| @@ -20,26 +20,66 @@ static INLINE int16x8_t vec_sign(int16x8_t a, int16x8_t b) { · 19 unmodified lines |
| 20 | 20 | return vec_xor(vec_add(a, mask), mask); |
| 21 | 21 | } |
| 22 | 22 | |
| 23 | +// Sets the value of a 32-bit integers to 1 when the corresponding value in a is |
| 24 | +// negative. |
| 25 | +static INLINE int32x4_t vec_is_neg(int32x4_t a) { |
| 26 | + return vec_sr(a, vec_shift_sign_s32); |
| 27 | +} |
| 28 | + |
| 23 | 29 | // Multiply the packed 16-bit integers in a and b, producing intermediate 32-bit |
| 24 | 30 | // integers, and return the high 16 bits of the intermediate integers. |
| 31 | +// (a * b) >> 16 |
| 25 | 32 | static INLINE int16x8_t vec_mulhi(int16x8_t a, int16x8_t b) { |
| 26 | 33 | // madds does ((A * B) >>15) + C, we need >> 16, so we perform an extra right |
| 27 | 34 | // shift. |
| 28 | | − return vec_sra(vec_madds(a, b, vec_zeros_s16), vec_ones_s16); |
| 35 | + return vec_sra(vec_madds(a, b, vec_zeros_s16), vec_ones_u16); |
| 29 | 36 | } |
| 30 | 37 | |
| 38 | +// Quantization function used for 4x4, 8x8 and 16x16 blocks. |
| 31 | 39 | static INLINE int16x8_t quantize_coeff(int16x8_t coeff, int16x8_t coeff_abs, |
| 32 | 40 | int16x8_t round, int16x8_t quant, |
| 33 | 41 | int16x8_t quant_shift, bool16x8_t mask) { |
| 34 | | − int16x8_t rounded, qcoeff; |
| 35 | | − rounded = vec_vaddshs(coeff_abs, round); |
| 36 | | − qcoeff = vec_mulhi(rounded, quant); |
| 42 | + const int16x8_t rounded = vec_vaddshs(coeff_abs, round); |
| 43 | + int16x8_t qcoeff = vec_mulhi(rounded, quant); |
| 37 | 44 | qcoeff = vec_add(qcoeff, rounded); |
| 38 | 45 | qcoeff = vec_mulhi(qcoeff, quant_shift); |
| 39 | 46 | qcoeff = vec_sign(qcoeff, coeff); |
| 40 | 47 | return vec_and(qcoeff, mask); |
| 41 | 48 | } |
| 42 | 49 | |
| 50 | +// Quantization function used for 32x32 blocks. |
| 51 | +static INLINE int16x8_t quantize_coeff_32(int16x8_t coeff, int16x8_t coeff_abs, |
| 52 | + int16x8_t round, int16x8_t quant, |
| 53 | + int16x8_t quant_shift, |
| 54 | + bool16x8_t mask) { |
| 55 | + const int16x8_t rounded = vec_vaddshs(coeff_abs, round); |
| 56 | + int16x8_t qcoeff = vec_mulhi(rounded, quant); |
| 57 | + qcoeff = vec_add(qcoeff, rounded); |
| 58 | + // 32x32 blocks require an extra multiplication by 2, this compensates for the |
| 59 | + // extra right shift added in vec_mulhi, as such vec_madds can be used |
| 60 | + // directly instead of vec_mulhi (((a * b) >> 15) >> 1) << 1 == (a * b >> 15) |
| 61 | + qcoeff = vec_madds(qcoeff, quant_shift, vec_zeros_s16); |
| 62 | + qcoeff = vec_sign(qcoeff, coeff); |
| 63 | + return vec_and(qcoeff, mask); |
| 64 | +} |
| 65 | + |
| 66 | +// DeQuantization function used for 32x32 blocks. Quantized coeff of 32x32 |
| 67 | +// blocks are twice as big as for other block sizes. As such, using |
| 68 | +// vec_mladd results in overflow. |
| 69 | +static INLINE int16x8_t dequantize_coeff_32(int16x8_t qcoeff, |
| 70 | + int16x8_t dequant) { |
| 71 | + int16x8_t dqcoeff; |
| 72 | + int32x4_t dqcoeffe = vec_mule(qcoeff, dequant); |
| 73 | + int32x4_t dqcoeffo = vec_mulo(qcoeff, dequant); |
| 74 | + // Add 1 if negative to round towards zero because the C uses division. |
| 75 | + dqcoeffe = vec_add(dqcoeffe, vec_is_neg(dqcoeffe)); |
| 76 | + dqcoeffo = vec_add(dqcoeffo, vec_is_neg(dqcoeffo)); |
| 77 | + dqcoeffe = vec_sra(dqcoeffe, vec_ones_u32); |
| 78 | + dqcoeffo = vec_sra(dqcoeffo, vec_ones_u32); |
| 79 | + dqcoeff = vec_pack(dqcoeffe, dqcoeffo); |
| 80 | + return vec_perm(dqcoeff, dqcoeff, vec_perm_merge); |
| 81 | +} |
| 82 | + |
| 43 | 83 | static INLINE int16x8_t nonzero_scanindex(int16x8_t qcoeff, bool16x8_t mask, |
| 44 | 84 | const int16_t *iscan_ptr, int index) { |
| 45 | 85 | int16x8_t scan = vec_vsx_ld(index, iscan_ptr); |
| @@ -157,3 +197,111 @@ void vpx_quantize_b_vsx(const tran_low_t *coeff_ptr, intptr_t n_coeffs, · 111 unmodified lines |
| 157 | 197 | eob = vec_max_across(eob); |
| 158 | 198 | *eob_ptr = eob[0]; |
| 159 | 199 | } |
| 200 | + |
| 201 | +void vpx_quantize_b_32x32_vsx( |
| 202 | + const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, |
| 203 | + const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, |
| 204 | + const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, |
| 205 | + tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, |
| 206 | + const int16_t *scan_ptr, const int16_t *iscan_ptr) { |
| 207 | + // In stage 1, we quantize 16 coeffs (DC + 15 AC) |
| 208 | + // In stage 2, we loop 42 times and quantize 24 coeffs per iteration |
| 209 | + // (32 * 32 - 16) / 24 = 42 |
| 210 | + int num_itr = 42; |
| 211 | + // Offsets are in bytes, 16 coeffs = 32 bytes |
| 212 | + int off0 = 32; |
| 213 | + int off1 = 48; |
| 214 | + int off2 = 64; |
| 215 | + |
| 216 | + int16x8_t qcoeff0, qcoeff1, eob; |
| 217 | + bool16x8_t zero_mask0, zero_mask1; |
| 218 | + |
| 219 | + int16x8_t zbin = vec_vsx_ld(0, zbin_ptr); |
| 220 | + int16x8_t round = vec_vsx_ld(0, round_ptr); |
| 221 | + int16x8_t quant = vec_vsx_ld(0, quant_ptr); |
| 222 | + int16x8_t dequant = vec_vsx_ld(0, dequant_ptr); |
| 223 | + int16x8_t quant_shift = vec_vsx_ld(0, quant_shift_ptr); |
| 224 | + |
| 225 | + int16x8_t coeff0 = vec_vsx_ld(0, coeff_ptr); |
| 226 | + int16x8_t coeff1 = vec_vsx_ld(16, coeff_ptr); |
| 227 | + |
| 228 | + int16x8_t coeff0_abs = vec_abs(coeff0); |
| 229 | + int16x8_t coeff1_abs = vec_abs(coeff1); |
| 230 | + |
| 231 | + (void)scan_ptr; |
| 232 | + (void)skip_block; |
| 233 | + (void)n_coeffs; |
| 234 | + assert(!skip_block); |
| 235 | + |
| 236 | + // 32x32 quantization requires that zbin and round be divided by 2 |
| 237 | + zbin = vec_sra(vec_add(zbin, vec_ones_s16), vec_ones_u16); |
| 238 | + round = vec_sra(vec_add(round, vec_ones_s16), vec_ones_u16); |
| 239 | + |
| 240 | + zero_mask0 = vec_cmpge(coeff0_abs, zbin); |
| 241 | + zbin = vec_splat(zbin, 1); // remove DC from zbin |
| 242 | + zero_mask1 = vec_cmpge(coeff1_abs, zbin); |
| 243 | + |
| 244 | + qcoeff0 = quantize_coeff_32(coeff0, coeff0_abs, round, quant, quant_shift, |
| 245 | + zero_mask0); |
| 246 | + round = vec_splat(round, 1); // remove DC from round |
| 247 | + quant = vec_splat(quant, 1); // remove DC from quant |
| 248 | + quant_shift = vec_splat(quant_shift, 1); // remove DC from quant_shift |
| 249 | + qcoeff1 = quantize_coeff_32(coeff1, coeff1_abs, round, quant, quant_shift, |
| 250 | + zero_mask1); |
| 251 | + |
| 252 | + vec_vsx_st(qcoeff0, 0, qcoeff_ptr); |
| 253 | + vec_vsx_st(qcoeff1, 16, qcoeff_ptr); |
| 254 | + |
| 255 | + vec_vsx_st(dequantize_coeff_32(qcoeff0, dequant), 0, dqcoeff_ptr); |
| 256 | + dequant = vec_splat(dequant, 1); // remove DC from dequant |
| 257 | + vec_vsx_st(dequantize_coeff_32(qcoeff1, dequant), 16, dqcoeff_ptr); |
| 258 | + |
| 259 | + eob = vec_max(nonzero_scanindex(qcoeff0, zero_mask0, iscan_ptr, 0), |
| 260 | + nonzero_scanindex(qcoeff1, zero_mask1, iscan_ptr, 16)); |
| 261 | + |
| 262 | + do { |
| 263 | + int16x8_t coeff2, coeff2_abs, qcoeff2, eob2; |
| 264 | + bool16x8_t zero_mask2; |
| 265 | + |
| 266 | + coeff0 = vec_vsx_ld(off0, coeff_ptr); |
| 267 | + coeff1 = vec_vsx_ld(off1, coeff_ptr); |
| 268 | + coeff2 = vec_vsx_ld(off2, coeff_ptr); |
| 269 | + |
| 270 | + coeff0_abs = vec_abs(coeff0); |
| 271 | + coeff1_abs = vec_abs(coeff1); |
| 272 | + coeff2_abs = vec_abs(coeff2); |
| 273 | + |
| 274 | + zero_mask0 = vec_cmpge(coeff0_abs, zbin); |
| 275 | + zero_mask1 = vec_cmpge(coeff1_abs, zbin); |
| 276 | + zero_mask2 = vec_cmpge(coeff2_abs, zbin); |
| 277 | + |
| 278 | + qcoeff0 = quantize_coeff_32(coeff0, coeff0_abs, round, quant, quant_shift, |
| 279 | + zero_mask0); |
| 280 | + qcoeff1 = quantize_coeff_32(coeff1, coeff1_abs, round, quant, quant_shift, |
| 281 | + zero_mask1); |
| 282 | + qcoeff2 = quantize_coeff_32(coeff2, coeff2_abs, round, quant, quant_shift, |
| 283 | + zero_mask2); |
| 284 | + |
| 285 | + vec_vsx_st(qcoeff0, off0, qcoeff_ptr); |
| 286 | + vec_vsx_st(qcoeff1, off1, qcoeff_ptr); |
| 287 | + vec_vsx_st(qcoeff2, off2, qcoeff_ptr); |
| 288 | + |
| 289 | + vec_vsx_st(dequantize_coeff_32(qcoeff0, dequant), off0, dqcoeff_ptr); |
| 290 | + vec_vsx_st(dequantize_coeff_32(qcoeff1, dequant), off1, dqcoeff_ptr); |
| 291 | + vec_vsx_st(dequantize_coeff_32(qcoeff2, dequant), off2, dqcoeff_ptr); |
| 292 | + |
| 293 | + eob = vec_max(eob, nonzero_scanindex(qcoeff0, zero_mask0, iscan_ptr, off0)); |
| 294 | + eob2 = vec_max(nonzero_scanindex(qcoeff1, zero_mask1, iscan_ptr, off1), |
| 295 | + nonzero_scanindex(qcoeff2, zero_mask2, iscan_ptr, off2)); |
| 296 | + eob = vec_max(eob, eob2); |
| 297 | + |
| 298 | + // 24 int16_t is 48 bytes |
| 299 | + off0 += 48; |
| 300 | + off1 += 48; |
| 301 | + off2 += 48; |
| 302 | + num_itr--; |
| 303 | + } while (num_itr != 0); |
| 304 | + |
| 305 | + eob = vec_max_across(eob); |
| 306 | + *eob_ptr = eob[0]; |
| 307 | +} |