| @@ -76,6 +76,8 @@ static int16x8_t cospi29_v = { 2404, 2404, 2404, 2404, 2404, 2404, 2404, 2404 }; · 75 unmodified lines |
| 76 | 76 | static int16x8_t cospi30_v = { 1606, 1606, 1606, 1606, 1606, 1606, 1606, 1606 }; |
| 77 | 77 | static int16x8_t cospi31_v = { 804, 804, 804, 804, 804, 804, 804, 804 }; |
| 78 | 78 | |
| 79 | +static uint8x16_t mask1 = { 0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, |
| 80 | + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17 }; |
| 79 | 81 | #define ROUND_SHIFT_INIT \ |
| 80 | 82 | const int32x4_t shift = vec_sl(vec_splat_s32(1), vec_splat_u32(13)); \ |
| 81 | 83 | const uint32x4_t shift14 = vec_splat_u32(14); |
| @@ -107,6 +109,15 @@ static int16x8_t cospi31_v = { 804, 804, 804, 804, 804, 804, 804, 804 }; · 25 unmodified lines |
| 107 | 109 | out1 = vec_sub(step0, step1); \ |
| 108 | 110 | out1 = vec_perm(out1, out1, mask0); |
| 109 | 111 | |
| 112 | +#define PACK_STORE(v0, v1) \ |
| 113 | + tmp16_0 = vec_add(vec_perm(d_u0, d_u1, mask1), v0); \ |
| 114 | + tmp16_1 = vec_add(vec_perm(d_u2, d_u3, mask1), v1); \ |
| 115 | + output_v = vec_packsu(tmp16_0, tmp16_1); \ |
| 116 | + \ |
| 117 | + vec_vsx_st(output_v, 0, tmp_dest); \ |
| 118 | + for (i = 0; i < 4; i++) \ |
| 119 | + for (j = 0; j < 4; j++) dest[j * stride + i] = tmp_dest[j * 4 + i]; |
| 120 | + |
| 110 | 121 | void vpx_idct4x4_16_add_vsx(const tran_low_t *input, uint8_t *dest, |
| 111 | 122 | int stride) { |
| 112 | 123 | int i, j; |
| @@ -114,13 +125,10 @@ void vpx_idct4x4_16_add_vsx(const tran_low_t *input, uint8_t *dest, · 1 unmodified line |
| 114 | 125 | int16x8_t step0, step1, tmp16_0, tmp16_1, t_out0, t_out1; |
| 115 | 126 | uint8x16_t mask0 = { 0x8, 0x9, 0xA, 0xB, 0xC, 0xD, 0xE, 0xF, |
| 116 | 127 | 0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7 }; |
| 117 | | − uint8x16_t mask1 = { 0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, |
| 118 | | − 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17 }; |
| 119 | 128 | int16x8_t v0 = load_tran_low(0, input); |
| 120 | 129 | int16x8_t v1 = load_tran_low(8 * sizeof(*input), input); |
| 121 | 130 | int16x8_t t0 = vec_mergeh(v0, v1); |
| 122 | 131 | int16x8_t t1 = vec_mergel(v0, v1); |
| 123 | | − |
| 124 | 132 | uint8x16_t dest0 = vec_vsx_ld(0, dest); |
| 125 | 133 | uint8x16_t dest1 = vec_vsx_ld(stride, dest); |
| 126 | 134 | uint8x16_t dest2 = vec_vsx_ld(2 * stride, dest); |
| @@ -130,6 +138,7 @@ void vpx_idct4x4_16_add_vsx(const tran_low_t *input, uint8_t *dest, · 3 unmodified lines |
| 130 | 138 | int16x8_t d_u1 = (int16x8_t)vec_mergeh(dest1, zerov); |
| 131 | 139 | int16x8_t d_u2 = (int16x8_t)vec_mergeh(dest2, zerov); |
| 132 | 140 | int16x8_t d_u3 = (int16x8_t)vec_mergeh(dest3, zerov); |
| 141 | + |
| 133 | 142 | uint8x16_t output_v; |
| 134 | 143 | uint8_t tmp_dest[16]; |
| 135 | 144 | ROUND_SHIFT_INIT |
| @@ -148,13 +157,8 @@ void vpx_idct4x4_16_add_vsx(const tran_low_t *input, uint8_t *dest, · 12 unmodified lines |
| 148 | 157 | |
| 149 | 158 | PIXEL_ADD4(v0, t_out0); |
| 150 | 159 | PIXEL_ADD4(v1, t_out1); |
| 151 | | − tmp16_0 = vec_add(vec_perm(d_u0, d_u1, mask1), v0); |
| 152 | | − tmp16_1 = vec_add(vec_perm(d_u2, d_u3, mask1), v1); |
| 153 | | − output_v = vec_packsu(tmp16_0, tmp16_1); |
| 154 | 160 | |
| 155 | | − vec_vsx_st(output_v, 0, tmp_dest); |
| 156 | | − for (i = 0; i < 4; i++) |
| 157 | | − for (j = 0; j < 4; j++) dest[j * stride + i] = tmp_dest[j * 4 + i]; |
| 161 | + PACK_STORE(v0, v1); |
| 158 | 162 | } |
| 159 | 163 | |
| 160 | 164 | #define TRANSPOSE8x8(in0, in1, in2, in3, in4, in5, in6, in7, out0, out1, out2, \ |
| @@ -1062,3 +1066,67 @@ void vpx_idct32x32_1024_add_vsx(const tran_low_t *input, uint8_t *dest, · 901 unmodified lines |
| 1062 | 1066 | ADD_STORE_BLOCK(src2, 16); |
| 1063 | 1067 | ADD_STORE_BLOCK(src3, 24); |
| 1064 | 1068 | } |
| 1069 | + |
| 1070 | +#define TRANSFORM_COLS \ |
| 1071 | + v32_a = vec_add(v32_a, v32_c); \ |
| 1072 | + v32_d = vec_sub(v32_d, v32_b); \ |
| 1073 | + v32_e = vec_sub(v32_a, v32_d); \ |
| 1074 | + v32_e = vec_sra(v32_e, one); \ |
| 1075 | + v32_b = vec_sub(v32_e, v32_b); \ |
| 1076 | + v32_c = vec_sub(v32_e, v32_c); \ |
| 1077 | + v32_a = vec_sub(v32_a, v32_b); \ |
| 1078 | + v32_d = vec_add(v32_d, v32_c); \ |
| 1079 | + v_a = vec_packs(v32_a, v32_b); \ |
| 1080 | + v_c = vec_packs(v32_c, v32_d); |
| 1081 | + |
| 1082 | +#define TRANSPOSE_WHT \ |
| 1083 | + tmp_a = vec_mergeh(v_a, v_c); \ |
| 1084 | + tmp_c = vec_mergel(v_a, v_c); \ |
| 1085 | + v_a = vec_mergeh(tmp_a, tmp_c); \ |
| 1086 | + v_c = vec_mergel(tmp_a, tmp_c); |
| 1087 | + |
| 1088 | +void vpx_iwht4x4_16_add_vsx(const tran_low_t *input, uint8_t *dest, |
| 1089 | + int stride) { |
| 1090 | + int16x8_t v_a = load_tran_low(0, input); |
| 1091 | + int16x8_t v_c = load_tran_low(8 * sizeof(*input), input); |
| 1092 | + int16x8_t tmp_a, tmp_c; |
| 1093 | + uint16x8_t two = vec_splat_u16(2); |
| 1094 | + uint32x4_t one = vec_splat_u32(1); |
| 1095 | + int16x8_t tmp16_0, tmp16_1; |
| 1096 | + int32x4_t v32_a, v32_c, v32_d, v32_b, v32_e; |
| 1097 | + uint8x16_t dest0 = vec_vsx_ld(0, dest); |
| 1098 | + uint8x16_t dest1 = vec_vsx_ld(stride, dest); |
| 1099 | + uint8x16_t dest2 = vec_vsx_ld(2 * stride, dest); |
| 1100 | + uint8x16_t dest3 = vec_vsx_ld(3 * stride, dest); |
| 1101 | + int16x8_t d_u0 = (int16x8_t)unpack_to_u16_h(dest0); |
| 1102 | + int16x8_t d_u1 = (int16x8_t)unpack_to_u16_h(dest1); |
| 1103 | + int16x8_t d_u2 = (int16x8_t)unpack_to_u16_h(dest2); |
| 1104 | + int16x8_t d_u3 = (int16x8_t)unpack_to_u16_h(dest3); |
| 1105 | + uint8x16_t output_v; |
| 1106 | + uint8_t tmp_dest[16]; |
| 1107 | + int i, j; |
| 1108 | + |
| 1109 | + v_a = vec_sra(v_a, two); |
| 1110 | + v_c = vec_sra(v_c, two); |
| 1111 | + |
| 1112 | + TRANSPOSE_WHT; |
| 1113 | + |
| 1114 | + v32_a = vec_unpackh(v_a); |
| 1115 | + v32_c = vec_unpackl(v_a); |
| 1116 | + |
| 1117 | + v32_d = vec_unpackh(v_c); |
| 1118 | + v32_b = vec_unpackl(v_c); |
| 1119 | + |
| 1120 | + TRANSFORM_COLS; |
| 1121 | + |
| 1122 | + TRANSPOSE_WHT; |
| 1123 | + |
| 1124 | + v32_a = vec_unpackh(v_a); |
| 1125 | + v32_c = vec_unpackl(v_a); |
| 1126 | + v32_d = vec_unpackh(v_c); |
| 1127 | + v32_b = vec_unpackl(v_c); |
| 1128 | + |
| 1129 | + TRANSFORM_COLS; |
| 1130 | + |
| 1131 | + PACK_STORE(v_a, v_c); |
| 1132 | +} |