History

Update sad4d x86 functions

Speed change is marginal.

Linfeng Zhang committed 8 years ago

Commit 39de45d1 parent debd86e2 files · +93 −127Browse files

Unified · Split

2 files
  1. test/sad_test.cc +2 −2
  2. vpx_dsp/x86/sad4d_avx2.c +91 −125
test/sad_test.cc+2−2
@@ -85,7 +85,7 @@ class SADTestBase : public ::testing::TestWithParam<ParamType> { · 84 unmodified lines
85#endif // CONFIG_VP9_HIGHBITDEPTH85#endif // CONFIG_VP9_HIGHBITDEPTH
86 }86 }
87 mask_ = (1 << bit_depth_) - 1;87 mask_ = (1 << bit_depth_) - 1;
88 source_stride_ = (params_.width + 31) & ~31;88 source_stride_ = (params_.width + 63) & ~63;
89 reference_stride_ = params_.width * 2;89 reference_stride_ = params_.width * 2;
90 rnd_.Reset(ACMRandom::DeterministicSeed());90 rnd_.Reset(ACMRandom::DeterministicSeed());
91 }91 }
@@ -109,7 +109,7 @@ class SADTestBase : public ::testing::TestWithParam<ParamType> { · 17 unmodified lines
109 109
110 protected:110 protected:
111 // Handle blocks up to 4 blocks 64x64 with stride up to 128111 // Handle blocks up to 4 blocks 64x64 with stride up to 128
112 static const int kDataAlignment = 16;112 static const int kDataAlignment = 32;
113 static const int kDataBlockSize = 64 * 128;113 static const int kDataBlockSize = 64 * 128;
114 static const int kDataBufferSize = 4 * kDataBlockSize;114 static const int kDataBufferSize = 4 * kDataBlockSize;
115 115
vpx_dsp/x86/sad4d_avx2.c+91−125
@@ -11,154 +11,120 @@ · 10 unmodified lines
11#include "./vpx_dsp_rtcd.h"11#include "./vpx_dsp_rtcd.h"
12#include "vpx/vpx_integer.h"12#include "vpx/vpx_integer.h"
13 13
14static INLINE void calc_final(const __m256i *const sums /*[4]*/,
15 uint32_t res[4]) {
16 const __m256i t0 = _mm256_hadd_epi32(sums[0], sums[1]);
17 const __m256i t1 = _mm256_hadd_epi32(sums[2], sums[3]);
18 const __m256i t2 = _mm256_hadd_epi32(t0, t1);
19 const __m128i sum = _mm_add_epi32(_mm256_castsi256_si128(t2),
20 _mm256_extractf128_si256(t2, 1));
21 _mm_storeu_si128((__m128i *)res, sum);
22}
23
14void vpx_sad32x32x4d_avx2(const uint8_t *src, int src_stride,24void vpx_sad32x32x4d_avx2(const uint8_t *src, int src_stride,
15 const uint8_t *const ref[4], int ref_stride,25 const uint8_t *const ref[4], int ref_stride,
16 uint32_t res[4]) {26 uint32_t res[4]) {
17 __m256i src_reg, ref0_reg, ref1_reg, ref2_reg, ref3_reg;
18 __m256i sum_ref0, sum_ref1, sum_ref2, sum_ref3;
19 __m256i sum_mlow, sum_mhigh;
20 int i;27 int i;
21 const uint8_t *ref0, *ref1, *ref2, *ref3;28 const uint8_t *refs[4];
22 29 __m256i sums[4];
23 ref0 = ref[0];30
24 ref1 = ref[1];31 refs[0] = ref[0];
25 ref2 = ref[2];32 refs[1] = ref[1];
26 ref3 = ref[3];33 refs[2] = ref[2];
27 sum_ref0 = _mm256_set1_epi16(0);34 refs[3] = ref[3];
28 sum_ref1 = _mm256_set1_epi16(0);35 sums[0] = _mm256_setzero_si256();
29 sum_ref2 = _mm256_set1_epi16(0);36 sums[1] = _mm256_setzero_si256();
30 sum_ref3 = _mm256_set1_epi16(0);37 sums[2] = _mm256_setzero_si256();
38 sums[3] = _mm256_setzero_si256();
39
31 for (i = 0; i < 32; i++) {40 for (i = 0; i < 32; i++) {
41 __m256i r[4];
42
32 // load src and all refs43 // load src and all refs
33 src_reg = _mm256_loadu_si256((const __m256i *)src);44 const __m256i s = _mm256_load_si256((const __m256i *)src);
34 ref0_reg = _mm256_loadu_si256((const __m256i *)ref0);45 r[0] = _mm256_loadu_si256((const __m256i *)refs[0]);
35 ref1_reg = _mm256_loadu_si256((const __m256i *)ref1);46 r[1] = _mm256_loadu_si256((const __m256i *)refs[1]);
36 ref2_reg = _mm256_loadu_si256((const __m256i *)ref2);47 r[2] = _mm256_loadu_si256((const __m256i *)refs[2]);
37 ref3_reg = _mm256_loadu_si256((const __m256i *)ref3);48 r[3] = _mm256_loadu_si256((const __m256i *)refs[3]);
49
38 // sum of the absolute differences between every ref-i to src50 // sum of the absolute differences between every ref-i to src
39 ref0_reg = _mm256_sad_epu8(ref0_reg, src_reg);51 r[0] = _mm256_sad_epu8(r[0], s);
40 ref1_reg = _mm256_sad_epu8(ref1_reg, src_reg);52 r[1] = _mm256_sad_epu8(r[1], s);
41 ref2_reg = _mm256_sad_epu8(ref2_reg, src_reg);53 r[2] = _mm256_sad_epu8(r[2], s);
42 ref3_reg = _mm256_sad_epu8(ref3_reg, src_reg);54 r[3] = _mm256_sad_epu8(r[3], s);
55
43 // sum every ref-i56 // sum every ref-i
44 sum_ref0 = _mm256_add_epi32(sum_ref0, ref0_reg);57 sums[0] = _mm256_add_epi32(sums[0], r[0]);
45 sum_ref1 = _mm256_add_epi32(sum_ref1, ref1_reg);58 sums[1] = _mm256_add_epi32(sums[1], r[1]);
46 sum_ref2 = _mm256_add_epi32(sum_ref2, ref2_reg);59 sums[2] = _mm256_add_epi32(sums[2], r[2]);
47 sum_ref3 = _mm256_add_epi32(sum_ref3, ref3_reg);60 sums[3] = _mm256_add_epi32(sums[3], r[3]);
48 61
49 src += src_stride;62 src += src_stride;
50 ref0 += ref_stride;63 refs[0] += ref_stride;
51 ref1 += ref_stride;64 refs[1] += ref_stride;
52 ref2 += ref_stride;65 refs[2] += ref_stride;
53 ref3 += ref_stride;66 refs[3] += ref_stride;
54 }
55 {
56 __m128i sum;
57 // in sum_ref-i the result is saved in the first 4 bytes
58 // the other 4 bytes are zeroed.
59 // sum_ref1 and sum_ref3 are shifted left by 4 bytes
60 sum_ref1 = _mm256_slli_si256(sum_ref1, 4);
61 sum_ref3 = _mm256_slli_si256(sum_ref3, 4);
62
63 // merge sum_ref0 and sum_ref1 also sum_ref2 and sum_ref3
64 sum_ref0 = _mm256_or_si256(sum_ref0, sum_ref1);
65 sum_ref2 = _mm256_or_si256(sum_ref2, sum_ref3);
66
67 // merge every 64 bit from each sum_ref-i
68 sum_mlow = _mm256_unpacklo_epi64(sum_ref0, sum_ref2);
69 sum_mhigh = _mm256_unpackhi_epi64(sum_ref0, sum_ref2);
70
71 // add the low 64 bit to the high 64 bit
72 sum_mlow = _mm256_add_epi32(sum_mlow, sum_mhigh);
73
74 // add the low 128 bit to the high 128 bit
75 sum = _mm_add_epi32(_mm256_castsi256_si128(sum_mlow),
76 _mm256_extractf128_si256(sum_mlow, 1));
77
78 _mm_storeu_si128((__m128i *)(res), sum);
79 }67 }
68
69 calc_final(sums, res);
80}70}
81 71
82void vpx_sad64x64x4d_avx2(const uint8_t *src, int src_stride,72void vpx_sad64x64x4d_avx2(const uint8_t *src, int src_stride,
83 const uint8_t *const ref[4], int ref_stride,73 const uint8_t *const ref[4], int ref_stride,
84 uint32_t res[4]) {74 uint32_t res[4]) {
85 __m256i src_reg, srcnext_reg, ref0_reg, ref0next_reg;75 __m256i sums[4];
86 __m256i ref1_reg, ref1next_reg, ref2_reg, ref2next_reg;
87 __m256i ref3_reg, ref3next_reg;
88 __m256i sum_ref0, sum_ref1, sum_ref2, sum_ref3;
89 __m256i sum_mlow, sum_mhigh;
90 int i;76 int i;
91 const uint8_t *ref0, *ref1, *ref2, *ref3;77 const uint8_t *refs[4];
92 78
93 ref0 = ref[0];79 refs[0] = ref[0];
94 ref1 = ref[1];80 refs[1] = ref[1];
95 ref2 = ref[2];81 refs[2] = ref[2];
96 ref3 = ref[3];82 refs[3] = ref[3];
97 sum_ref0 = _mm256_set1_epi16(0);83 sums[0] = _mm256_setzero_si256();
98 sum_ref1 = _mm256_set1_epi16(0);84 sums[1] = _mm256_setzero_si256();
99 sum_ref2 = _mm256_set1_epi16(0);85 sums[2] = _mm256_setzero_si256();
100 sum_ref3 = _mm256_set1_epi16(0);86 sums[3] = _mm256_setzero_si256();
87
101 for (i = 0; i < 64; i++) {88 for (i = 0; i < 64; i++) {
89 __m256i r_lo[4], r_hi[4];
102 // load 64 bytes from src and all refs90 // load 64 bytes from src and all refs
103 src_reg = _mm256_loadu_si256((const __m256i *)src);91 const __m256i s_lo = _mm256_load_si256((const __m256i *)src);
104 srcnext_reg = _mm256_loadu_si256((const __m256i *)(src + 32));92 const __m256i s_hi = _mm256_load_si256((const __m256i *)(src + 32));
105 ref0_reg = _mm256_loadu_si256((const __m256i *)ref0);93 r_lo[0] = _mm256_loadu_si256((const __m256i *)refs[0]);
106 ref0next_reg = _mm256_loadu_si256((const __m256i *)(ref0 + 32));94 r_hi[0] = _mm256_loadu_si256((const __m256i *)(refs[0] + 32));
107 ref1_reg = _mm256_loadu_si256((const __m256i *)ref1);95 r_lo[1] = _mm256_loadu_si256((const __m256i *)refs[1]);
108 ref1next_reg = _mm256_loadu_si256((const __m256i *)(ref1 + 32));96 r_hi[1] = _mm256_loadu_si256((const __m256i *)(refs[1] + 32));
109 ref2_reg = _mm256_loadu_si256((const __m256i *)ref2);97 r_lo[2] = _mm256_loadu_si256((const __m256i *)refs[2]);
110 ref2next_reg = _mm256_loadu_si256((const __m256i *)(ref2 + 32));98 r_hi[2] = _mm256_loadu_si256((const __m256i *)(refs[2] + 32));
111 ref3_reg = _mm256_loadu_si256((const __m256i *)ref3);99 r_lo[3] = _mm256_loadu_si256((const __m256i *)refs[3]);
112 ref3next_reg = _mm256_loadu_si256((const __m256i *)(ref3 + 32));100 r_hi[3] = _mm256_loadu_si256((const __m256i *)(refs[3] + 32));
101
113 // sum of the absolute differences between every ref-i to src102 // sum of the absolute differences between every ref-i to src
114 ref0_reg = _mm256_sad_epu8(ref0_reg, src_reg);103 r_lo[0] = _mm256_sad_epu8(r_lo[0], s_lo);
115 ref1_reg = _mm256_sad_epu8(ref1_reg, src_reg);104 r_lo[1] = _mm256_sad_epu8(r_lo[1], s_lo);
116 ref2_reg = _mm256_sad_epu8(ref2_reg, src_reg);105 r_lo[2] = _mm256_sad_epu8(r_lo[2], s_lo);
117 ref3_reg = _mm256_sad_epu8(ref3_reg, src_reg);106 r_lo[3] = _mm256_sad_epu8(r_lo[3], s_lo);
118 ref0next_reg = _mm256_sad_epu8(ref0next_reg, srcnext_reg);107 r_hi[0] = _mm256_sad_epu8(r_hi[0], s_hi);
119 ref1next_reg = _mm256_sad_epu8(ref1next_reg, srcnext_reg);108 r_hi[1] = _mm256_sad_epu8(r_hi[1], s_hi);
120 ref2next_reg = _mm256_sad_epu8(ref2next_reg, srcnext_reg);109 r_hi[2] = _mm256_sad_epu8(r_hi[2], s_hi);
121 ref3next_reg = _mm256_sad_epu8(ref3next_reg, srcnext_reg);110 r_hi[3] = _mm256_sad_epu8(r_hi[3], s_hi);
122 111
123 // sum every ref-i112 // sum every ref-i
124 sum_ref0 = _mm256_add_epi32(sum_ref0, ref0_reg);113 sums[0] = _mm256_add_epi32(sums[0], r_lo[0]);
125 sum_ref1 = _mm256_add_epi32(sum_ref1, ref1_reg);114 sums[1] = _mm256_add_epi32(sums[1], r_lo[1]);
126 sum_ref2 = _mm256_add_epi32(sum_ref2, ref2_reg);115 sums[2] = _mm256_add_epi32(sums[2], r_lo[2]);
127 sum_ref3 = _mm256_add_epi32(sum_ref3, ref3_reg);116 sums[3] = _mm256_add_epi32(sums[3], r_lo[3]);
128 sum_ref0 = _mm256_add_epi32(sum_ref0, ref0next_reg);117 sums[0] = _mm256_add_epi32(sums[0], r_hi[0]);
129 sum_ref1 = _mm256_add_epi32(sum_ref1, ref1next_reg);118 sums[1] = _mm256_add_epi32(sums[1], r_hi[1]);
130 sum_ref2 = _mm256_add_epi32(sum_ref2, ref2next_reg);119 sums[2] = _mm256_add_epi32(sums[2], r_hi[2]);
131 sum_ref3 = _mm256_add_epi32(sum_ref3, ref3next_reg);120 sums[3] = _mm256_add_epi32(sums[3], r_hi[3]);
121
132 src += src_stride;122 src += src_stride;
133 ref0 += ref_stride;123 refs[0] += ref_stride;
134 ref1 += ref_stride;124 refs[1] += ref_stride;
135 ref2 += ref_stride;125 refs[2] += ref_stride;
136 ref3 += ref_stride;126 refs[3] += ref_stride;
137 }127 }
138 {
139 __m128i sum;
140
141 // in sum_ref-i the result is saved in the first 4 bytes
142 // the other 4 bytes are zeroed.
143 // sum_ref1 and sum_ref3 are shifted left by 4 bytes
144 sum_ref1 = _mm256_slli_si256(sum_ref1, 4);
145 sum_ref3 = _mm256_slli_si256(sum_ref3, 4);
146
147 // merge sum_ref0 and sum_ref1 also sum_ref2 and sum_ref3
148 sum_ref0 = _mm256_or_si256(sum_ref0, sum_ref1);
149 sum_ref2 = _mm256_or_si256(sum_ref2, sum_ref3);
150 128
151 // merge every 64 bit from each sum_ref-i129 calc_final(sums, res);
152 sum_mlow = _mm256_unpacklo_epi64(sum_ref0, sum_ref2);
153 sum_mhigh = _mm256_unpackhi_epi64(sum_ref0, sum_ref2);
154
155 // add the low 64 bit to the high 64 bit
156 sum_mlow = _mm256_add_epi32(sum_mlow, sum_mhigh);
157
158 // add the low 128 bit to the high 128 bit
159 sum = _mm_add_epi32(_mm256_castsi256_si128(sum_mlow),
160 _mm256_extractf128_si256(sum_mlow, 1));
161
162 _mm_storeu_si128((__m128i *)(res), sum);
163 }
164}130}

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t