| @@ -20,26 +20,66 @@ static INLINE int16x8_t vec_sign(int16x8_t a, int16x8_t b) { · 19 unmodified lines |
| 20 | return vec_xor(vec_add(a, mask), mask); | 20 | return vec_xor(vec_add(a, mask), mask); |
| 21 | } | 21 | } |
| 22 | | 22 | |
| | 23 | // Sets the value of a 32-bit integers to 1 when the corresponding value in a is |
| | 24 | // negative. |
| | 25 | static INLINE int32x4_t vec_is_neg(int32x4_t a) { |
| | 26 | return vec_sr(a, vec_shift_sign_s32); |
| | 27 | } |
| | 28 | |
| 23 | // Multiply the packed 16-bit integers in a and b, producing intermediate 32-bit | 29 | // Multiply the packed 16-bit integers in a and b, producing intermediate 32-bit |
| 24 | // integers, and return the high 16 bits of the intermediate integers. | 30 | // integers, and return the high 16 bits of the intermediate integers. |
| | 31 | // (a * b) >> 16 |
| 25 | static INLINE int16x8_t vec_mulhi(int16x8_t a, int16x8_t b) { | 32 | static INLINE int16x8_t vec_mulhi(int16x8_t a, int16x8_t b) { |
| 26 | // madds does ((A * B) >>15) + C, we need >> 16, so we perform an extra right | 33 | // madds does ((A * B) >>15) + C, we need >> 16, so we perform an extra right |
| 27 | // shift. | 34 | // shift. |
| 28 | return vec_sra(vec_madds(a, b, vec_zeros_s16), vec_ones_s16); | 35 | return vec_sra(vec_madds(a, b, vec_zeros_s16), vec_ones_u16); |
| 29 | } | 36 | } |
| 30 | | 37 | |
| | 38 | // Quantization function used for 4x4, 8x8 and 16x16 blocks. |
| 31 | static INLINE int16x8_t quantize_coeff(int16x8_t coeff, int16x8_t coeff_abs, | 39 | static INLINE int16x8_t quantize_coeff(int16x8_t coeff, int16x8_t coeff_abs, |
| 32 | int16x8_t round, int16x8_t quant, | 40 | int16x8_t round, int16x8_t quant, |
| 33 | int16x8_t quant_shift, bool16x8_t mask) { | 41 | int16x8_t quant_shift, bool16x8_t mask) { |
| 34 | int16x8_t rounded, qcoeff; | 42 | const int16x8_t rounded = vec_vaddshs(coeff_abs, round); |
| 35 | rounded = vec_vaddshs(coeff_abs, round); | 43 | int16x8_t qcoeff = vec_mulhi(rounded, quant); |
| 36 | qcoeff = vec_mulhi(rounded, quant); | | |
| 37 | qcoeff = vec_add(qcoeff, rounded); | 44 | qcoeff = vec_add(qcoeff, rounded); |
| 38 | qcoeff = vec_mulhi(qcoeff, quant_shift); | 45 | qcoeff = vec_mulhi(qcoeff, quant_shift); |
| 39 | qcoeff = vec_sign(qcoeff, coeff); | 46 | qcoeff = vec_sign(qcoeff, coeff); |
| 40 | return vec_and(qcoeff, mask); | 47 | return vec_and(qcoeff, mask); |
| 41 | } | 48 | } |
| 42 | | 49 | |
| | 50 | // Quantization function used for 32x32 blocks. |
| | 51 | static INLINE int16x8_t quantize_coeff_32(int16x8_t coeff, int16x8_t coeff_abs, |
| | 52 | int16x8_t round, int16x8_t quant, |
| | 53 | int16x8_t quant_shift, |
| | 54 | bool16x8_t mask) { |
| | 55 | const int16x8_t rounded = vec_vaddshs(coeff_abs, round); |
| | 56 | int16x8_t qcoeff = vec_mulhi(rounded, quant); |
| | 57 | qcoeff = vec_add(qcoeff, rounded); |
| | 58 | // 32x32 blocks require an extra multiplication by 2, this compensates for the |
| | 59 | // extra right shift added in vec_mulhi, as such vec_madds can be used |
| | 60 | // directly instead of vec_mulhi (((a * b) >> 15) >> 1) << 1 == (a * b >> 15) |
| | 61 | qcoeff = vec_madds(qcoeff, quant_shift, vec_zeros_s16); |
| | 62 | qcoeff = vec_sign(qcoeff, coeff); |
| | 63 | return vec_and(qcoeff, mask); |
| | 64 | } |
| | 65 | |
| | 66 | // DeQuantization function used for 32x32 blocks. Quantized coeff of 32x32 |
| | 67 | // blocks are twice as big as for other block sizes. As such, using |
| | 68 | // vec_mladd results in overflow. |
| | 69 | static INLINE int16x8_t dequantize_coeff_32(int16x8_t qcoeff, |
| | 70 | int16x8_t dequant) { |
| | 71 | int16x8_t dqcoeff; |
| | 72 | int32x4_t dqcoeffe = vec_mule(qcoeff, dequant); |
| | 73 | int32x4_t dqcoeffo = vec_mulo(qcoeff, dequant); |
| | 74 | // Add 1 if negative to round towards zero because the C uses division. |
| | 75 | dqcoeffe = vec_add(dqcoeffe, vec_is_neg(dqcoeffe)); |
| | 76 | dqcoeffo = vec_add(dqcoeffo, vec_is_neg(dqcoeffo)); |
| | 77 | dqcoeffe = vec_sra(dqcoeffe, vec_ones_u32); |
| | 78 | dqcoeffo = vec_sra(dqcoeffo, vec_ones_u32); |
| | 79 | dqcoeff = vec_pack(dqcoeffe, dqcoeffo); |
| | 80 | return vec_perm(dqcoeff, dqcoeff, vec_perm_merge); |
| | 81 | } |
| | 82 | |
| 43 | static INLINE int16x8_t nonzero_scanindex(int16x8_t qcoeff, bool16x8_t mask, | 83 | static INLINE int16x8_t nonzero_scanindex(int16x8_t qcoeff, bool16x8_t mask, |
| 44 | const int16_t *iscan_ptr, int index) { | 84 | const int16_t *iscan_ptr, int index) { |
| 45 | int16x8_t scan = vec_vsx_ld(index, iscan_ptr); | 85 | int16x8_t scan = vec_vsx_ld(index, iscan_ptr); |
| @@ -157,3 +197,111 @@ void vpx_quantize_b_vsx(const tran_low_t *coeff_ptr, intptr_t n_coeffs, · 111 unmodified lines |
| 157 | eob = vec_max_across(eob); | 197 | eob = vec_max_across(eob); |
| 158 | *eob_ptr = eob[0]; | 198 | *eob_ptr = eob[0]; |
| 159 | } | 199 | } |
| | 200 | |
| | 201 | void vpx_quantize_b_32x32_vsx( |
| | 202 | const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, |
| | 203 | const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, |
| | 204 | const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, |
| | 205 | tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, |
| | 206 | const int16_t *scan_ptr, const int16_t *iscan_ptr) { |
| | 207 | // In stage 1, we quantize 16 coeffs (DC + 15 AC) |
| | 208 | // In stage 2, we loop 42 times and quantize 24 coeffs per iteration |
| | 209 | // (32 * 32 - 16) / 24 = 42 |
| | 210 | int num_itr = 42; |
| | 211 | // Offsets are in bytes, 16 coeffs = 32 bytes |
| | 212 | int off0 = 32; |
| | 213 | int off1 = 48; |
| | 214 | int off2 = 64; |
| | 215 | |
| | 216 | int16x8_t qcoeff0, qcoeff1, eob; |
| | 217 | bool16x8_t zero_mask0, zero_mask1; |
| | 218 | |
| | 219 | int16x8_t zbin = vec_vsx_ld(0, zbin_ptr); |
| | 220 | int16x8_t round = vec_vsx_ld(0, round_ptr); |
| | 221 | int16x8_t quant = vec_vsx_ld(0, quant_ptr); |
| | 222 | int16x8_t dequant = vec_vsx_ld(0, dequant_ptr); |
| | 223 | int16x8_t quant_shift = vec_vsx_ld(0, quant_shift_ptr); |
| | 224 | |
| | 225 | int16x8_t coeff0 = vec_vsx_ld(0, coeff_ptr); |
| | 226 | int16x8_t coeff1 = vec_vsx_ld(16, coeff_ptr); |
| | 227 | |
| | 228 | int16x8_t coeff0_abs = vec_abs(coeff0); |
| | 229 | int16x8_t coeff1_abs = vec_abs(coeff1); |
| | 230 | |
| | 231 | (void)scan_ptr; |
| | 232 | (void)skip_block; |
| | 233 | (void)n_coeffs; |
| | 234 | assert(!skip_block); |
| | 235 | |
| | 236 | // 32x32 quantization requires that zbin and round be divided by 2 |
| | 237 | zbin = vec_sra(vec_add(zbin, vec_ones_s16), vec_ones_u16); |
| | 238 | round = vec_sra(vec_add(round, vec_ones_s16), vec_ones_u16); |
| | 239 | |
| | 240 | zero_mask0 = vec_cmpge(coeff0_abs, zbin); |
| | 241 | zbin = vec_splat(zbin, 1); // remove DC from zbin |
| | 242 | zero_mask1 = vec_cmpge(coeff1_abs, zbin); |
| | 243 | |
| | 244 | qcoeff0 = quantize_coeff_32(coeff0, coeff0_abs, round, quant, quant_shift, |
| | 245 | zero_mask0); |
| | 246 | round = vec_splat(round, 1); // remove DC from round |
| | 247 | quant = vec_splat(quant, 1); // remove DC from quant |
| | 248 | quant_shift = vec_splat(quant_shift, 1); // remove DC from quant_shift |
| | 249 | qcoeff1 = quantize_coeff_32(coeff1, coeff1_abs, round, quant, quant_shift, |
| | 250 | zero_mask1); |
| | 251 | |
| | 252 | vec_vsx_st(qcoeff0, 0, qcoeff_ptr); |
| | 253 | vec_vsx_st(qcoeff1, 16, qcoeff_ptr); |
| | 254 | |
| | 255 | vec_vsx_st(dequantize_coeff_32(qcoeff0, dequant), 0, dqcoeff_ptr); |
| | 256 | dequant = vec_splat(dequant, 1); // remove DC from dequant |
| | 257 | vec_vsx_st(dequantize_coeff_32(qcoeff1, dequant), 16, dqcoeff_ptr); |
| | 258 | |
| | 259 | eob = vec_max(nonzero_scanindex(qcoeff0, zero_mask0, iscan_ptr, 0), |
| | 260 | nonzero_scanindex(qcoeff1, zero_mask1, iscan_ptr, 16)); |
| | 261 | |
| | 262 | do { |
| | 263 | int16x8_t coeff2, coeff2_abs, qcoeff2, eob2; |
| | 264 | bool16x8_t zero_mask2; |
| | 265 | |
| | 266 | coeff0 = vec_vsx_ld(off0, coeff_ptr); |
| | 267 | coeff1 = vec_vsx_ld(off1, coeff_ptr); |
| | 268 | coeff2 = vec_vsx_ld(off2, coeff_ptr); |
| | 269 | |
| | 270 | coeff0_abs = vec_abs(coeff0); |
| | 271 | coeff1_abs = vec_abs(coeff1); |
| | 272 | coeff2_abs = vec_abs(coeff2); |
| | 273 | |
| | 274 | zero_mask0 = vec_cmpge(coeff0_abs, zbin); |
| | 275 | zero_mask1 = vec_cmpge(coeff1_abs, zbin); |
| | 276 | zero_mask2 = vec_cmpge(coeff2_abs, zbin); |
| | 277 | |
| | 278 | qcoeff0 = quantize_coeff_32(coeff0, coeff0_abs, round, quant, quant_shift, |
| | 279 | zero_mask0); |
| | 280 | qcoeff1 = quantize_coeff_32(coeff1, coeff1_abs, round, quant, quant_shift, |
| | 281 | zero_mask1); |
| | 282 | qcoeff2 = quantize_coeff_32(coeff2, coeff2_abs, round, quant, quant_shift, |
| | 283 | zero_mask2); |
| | 284 | |
| | 285 | vec_vsx_st(qcoeff0, off0, qcoeff_ptr); |
| | 286 | vec_vsx_st(qcoeff1, off1, qcoeff_ptr); |
| | 287 | vec_vsx_st(qcoeff2, off2, qcoeff_ptr); |
| | 288 | |
| | 289 | vec_vsx_st(dequantize_coeff_32(qcoeff0, dequant), off0, dqcoeff_ptr); |
| | 290 | vec_vsx_st(dequantize_coeff_32(qcoeff1, dequant), off1, dqcoeff_ptr); |
| | 291 | vec_vsx_st(dequantize_coeff_32(qcoeff2, dequant), off2, dqcoeff_ptr); |
| | 292 | |
| | 293 | eob = vec_max(eob, nonzero_scanindex(qcoeff0, zero_mask0, iscan_ptr, off0)); |
| | 294 | eob2 = vec_max(nonzero_scanindex(qcoeff1, zero_mask1, iscan_ptr, off1), |
| | 295 | nonzero_scanindex(qcoeff2, zero_mask2, iscan_ptr, off2)); |
| | 296 | eob = vec_max(eob, eob2); |
| | 297 | |
| | 298 | // 24 int16_t is 48 bytes |
| | 299 | off0 += 48; |
| | 300 | off1 += 48; |
| | 301 | off2 += 48; |
| | 302 | num_itr--; |
| | 303 | } while (num_itr != 0); |
| | 304 | |
| | 305 | eob = vec_max_across(eob); |
| | 306 | *eob_ptr = eob[0]; |
| | 307 | } |