History

Update sad4d x86 functions

Speed change is marginal.

Linfeng Zhang committed 8 years ago

Commit 39de45d1 parent debd86e2 files · +93 −127Browse files

Unified · Split

2 files
  1. test/sad_test.cc +2 −2
  2. vpx_dsp/x86/sad4d_avx2.c +91 −125
test/sad_test.cc+2−2
@@ -85,7 +85,7 @@ class SADTestBase : public ::testing::TestWithParam<ParamType> { · 84 unmodified lines
8585 #endif // CONFIG_VP9_HIGHBITDEPTH
8686 }
8787 mask_ = (1 << bit_depth_) - 1;
88− source_stride_ = (params_.width + 31) & ~31;
88+ source_stride_ = (params_.width + 63) & ~63;
8989 reference_stride_ = params_.width * 2;
9090 rnd_.Reset(ACMRandom::DeterministicSeed());
9191 }
@@ -109,7 +109,7 @@ class SADTestBase : public ::testing::TestWithParam<ParamType> { · 17 unmodified lines
109109
110110 protected:
111111 // Handle blocks up to 4 blocks 64x64 with stride up to 128
112− static const int kDataAlignment = 16;
112+ static const int kDataAlignment = 32;
113113 static const int kDataBlockSize = 64 * 128;
114114 static const int kDataBufferSize = 4 * kDataBlockSize;
115115
vpx_dsp/x86/sad4d_avx2.c+91−125
@@ -11,154 +11,120 @@ · 10 unmodified lines
1111 #include "./vpx_dsp_rtcd.h"
1212 #include "vpx/vpx_integer.h"
1313
14+static INLINE void calc_final(const __m256i *const sums /*[4]*/,
15+ uint32_t res[4]) {
16+ const __m256i t0 = _mm256_hadd_epi32(sums[0], sums[1]);
17+ const __m256i t1 = _mm256_hadd_epi32(sums[2], sums[3]);
18+ const __m256i t2 = _mm256_hadd_epi32(t0, t1);
19+ const __m128i sum = _mm_add_epi32(_mm256_castsi256_si128(t2),
20+ _mm256_extractf128_si256(t2, 1));
21+ _mm_storeu_si128((__m128i *)res, sum);
22+}
23+
1424 void vpx_sad32x32x4d_avx2(const uint8_t *src, int src_stride,
1525 const uint8_t *const ref[4], int ref_stride,
1626 uint32_t res[4]) {
17− __m256i src_reg, ref0_reg, ref1_reg, ref2_reg, ref3_reg;
18− __m256i sum_ref0, sum_ref1, sum_ref2, sum_ref3;
19− __m256i sum_mlow, sum_mhigh;
2027 int i;
21− const uint8_t *ref0, *ref1, *ref2, *ref3;
22−
23− ref0 = ref[0];
24− ref1 = ref[1];
25− ref2 = ref[2];
26− ref3 = ref[3];
27− sum_ref0 = _mm256_set1_epi16(0);
28− sum_ref1 = _mm256_set1_epi16(0);
29− sum_ref2 = _mm256_set1_epi16(0);
30− sum_ref3 = _mm256_set1_epi16(0);
28+ const uint8_t *refs[4];
29+ __m256i sums[4];
30+
31+ refs[0] = ref[0];
32+ refs[1] = ref[1];
33+ refs[2] = ref[2];
34+ refs[3] = ref[3];
35+ sums[0] = _mm256_setzero_si256();
36+ sums[1] = _mm256_setzero_si256();
37+ sums[2] = _mm256_setzero_si256();
38+ sums[3] = _mm256_setzero_si256();
39+
3140 for (i = 0; i < 32; i++) {
41+ __m256i r[4];
42+
3243 // load src and all refs
33− src_reg = _mm256_loadu_si256((const __m256i *)src);
34− ref0_reg = _mm256_loadu_si256((const __m256i *)ref0);
35− ref1_reg = _mm256_loadu_si256((const __m256i *)ref1);
36− ref2_reg = _mm256_loadu_si256((const __m256i *)ref2);
37− ref3_reg = _mm256_loadu_si256((const __m256i *)ref3);
44+ const __m256i s = _mm256_load_si256((const __m256i *)src);
45+ r[0] = _mm256_loadu_si256((const __m256i *)refs[0]);
46+ r[1] = _mm256_loadu_si256((const __m256i *)refs[1]);
47+ r[2] = _mm256_loadu_si256((const __m256i *)refs[2]);
48+ r[3] = _mm256_loadu_si256((const __m256i *)refs[3]);
49+
3850 // sum of the absolute differences between every ref-i to src
39− ref0_reg = _mm256_sad_epu8(ref0_reg, src_reg);
40− ref1_reg = _mm256_sad_epu8(ref1_reg, src_reg);
41− ref2_reg = _mm256_sad_epu8(ref2_reg, src_reg);
42− ref3_reg = _mm256_sad_epu8(ref3_reg, src_reg);
51+ r[0] = _mm256_sad_epu8(r[0], s);
52+ r[1] = _mm256_sad_epu8(r[1], s);
53+ r[2] = _mm256_sad_epu8(r[2], s);
54+ r[3] = _mm256_sad_epu8(r[3], s);
55+
4356 // sum every ref-i
44− sum_ref0 = _mm256_add_epi32(sum_ref0, ref0_reg);
45− sum_ref1 = _mm256_add_epi32(sum_ref1, ref1_reg);
46− sum_ref2 = _mm256_add_epi32(sum_ref2, ref2_reg);
47− sum_ref3 = _mm256_add_epi32(sum_ref3, ref3_reg);
57+ sums[0] = _mm256_add_epi32(sums[0], r[0]);
58+ sums[1] = _mm256_add_epi32(sums[1], r[1]);
59+ sums[2] = _mm256_add_epi32(sums[2], r[2]);
60+ sums[3] = _mm256_add_epi32(sums[3], r[3]);
4861
4962 src += src_stride;
50− ref0 += ref_stride;
51− ref1 += ref_stride;
52− ref2 += ref_stride;
53− ref3 += ref_stride;
54− }
55− {
56− __m128i sum;
57− // in sum_ref-i the result is saved in the first 4 bytes
58− // the other 4 bytes are zeroed.
59− // sum_ref1 and sum_ref3 are shifted left by 4 bytes
60− sum_ref1 = _mm256_slli_si256(sum_ref1, 4);
61− sum_ref3 = _mm256_slli_si256(sum_ref3, 4);
62−
63− // merge sum_ref0 and sum_ref1 also sum_ref2 and sum_ref3
64− sum_ref0 = _mm256_or_si256(sum_ref0, sum_ref1);
65− sum_ref2 = _mm256_or_si256(sum_ref2, sum_ref3);
66−
67− // merge every 64 bit from each sum_ref-i
68− sum_mlow = _mm256_unpacklo_epi64(sum_ref0, sum_ref2);
69− sum_mhigh = _mm256_unpackhi_epi64(sum_ref0, sum_ref2);
70−
71− // add the low 64 bit to the high 64 bit
72− sum_mlow = _mm256_add_epi32(sum_mlow, sum_mhigh);
73−
74− // add the low 128 bit to the high 128 bit
75− sum = _mm_add_epi32(_mm256_castsi256_si128(sum_mlow),
76− _mm256_extractf128_si256(sum_mlow, 1));
77−
78− _mm_storeu_si128((__m128i *)(res), sum);
63+ refs[0] += ref_stride;
64+ refs[1] += ref_stride;
65+ refs[2] += ref_stride;
66+ refs[3] += ref_stride;
7967 }
68+
69+ calc_final(sums, res);
8070 }
8171
8272 void vpx_sad64x64x4d_avx2(const uint8_t *src, int src_stride,
8373 const uint8_t *const ref[4], int ref_stride,
8474 uint32_t res[4]) {
85− __m256i src_reg, srcnext_reg, ref0_reg, ref0next_reg;
86− __m256i ref1_reg, ref1next_reg, ref2_reg, ref2next_reg;
87− __m256i ref3_reg, ref3next_reg;
88− __m256i sum_ref0, sum_ref1, sum_ref2, sum_ref3;
89− __m256i sum_mlow, sum_mhigh;
75+ __m256i sums[4];
9076 int i;
91− const uint8_t *ref0, *ref1, *ref2, *ref3;
92−
93− ref0 = ref[0];
94− ref1 = ref[1];
95− ref2 = ref[2];
96− ref3 = ref[3];
97− sum_ref0 = _mm256_set1_epi16(0);
98− sum_ref1 = _mm256_set1_epi16(0);
99− sum_ref2 = _mm256_set1_epi16(0);
100− sum_ref3 = _mm256_set1_epi16(0);
77+ const uint8_t *refs[4];
78+
79+ refs[0] = ref[0];
80+ refs[1] = ref[1];
81+ refs[2] = ref[2];
82+ refs[3] = ref[3];
83+ sums[0] = _mm256_setzero_si256();
84+ sums[1] = _mm256_setzero_si256();
85+ sums[2] = _mm256_setzero_si256();
86+ sums[3] = _mm256_setzero_si256();
87+
10188 for (i = 0; i < 64; i++) {
89+ __m256i r_lo[4], r_hi[4];
10290 // load 64 bytes from src and all refs
103− src_reg = _mm256_loadu_si256((const __m256i *)src);
104− srcnext_reg = _mm256_loadu_si256((const __m256i *)(src + 32));
105− ref0_reg = _mm256_loadu_si256((const __m256i *)ref0);
106− ref0next_reg = _mm256_loadu_si256((const __m256i *)(ref0 + 32));
107− ref1_reg = _mm256_loadu_si256((const __m256i *)ref1);
108− ref1next_reg = _mm256_loadu_si256((const __m256i *)(ref1 + 32));
109− ref2_reg = _mm256_loadu_si256((const __m256i *)ref2);
110− ref2next_reg = _mm256_loadu_si256((const __m256i *)(ref2 + 32));
111− ref3_reg = _mm256_loadu_si256((const __m256i *)ref3);
112− ref3next_reg = _mm256_loadu_si256((const __m256i *)(ref3 + 32));
91+ const __m256i s_lo = _mm256_load_si256((const __m256i *)src);
92+ const __m256i s_hi = _mm256_load_si256((const __m256i *)(src + 32));
93+ r_lo[0] = _mm256_loadu_si256((const __m256i *)refs[0]);
94+ r_hi[0] = _mm256_loadu_si256((const __m256i *)(refs[0] + 32));
95+ r_lo[1] = _mm256_loadu_si256((const __m256i *)refs[1]);
96+ r_hi[1] = _mm256_loadu_si256((const __m256i *)(refs[1] + 32));
97+ r_lo[2] = _mm256_loadu_si256((const __m256i *)refs[2]);
98+ r_hi[2] = _mm256_loadu_si256((const __m256i *)(refs[2] + 32));
99+ r_lo[3] = _mm256_loadu_si256((const __m256i *)refs[3]);
100+ r_hi[3] = _mm256_loadu_si256((const __m256i *)(refs[3] + 32));
101+
113102 // sum of the absolute differences between every ref-i to src
114− ref0_reg = _mm256_sad_epu8(ref0_reg, src_reg);
115− ref1_reg = _mm256_sad_epu8(ref1_reg, src_reg);
116− ref2_reg = _mm256_sad_epu8(ref2_reg, src_reg);
117− ref3_reg = _mm256_sad_epu8(ref3_reg, src_reg);
118− ref0next_reg = _mm256_sad_epu8(ref0next_reg, srcnext_reg);
119− ref1next_reg = _mm256_sad_epu8(ref1next_reg, srcnext_reg);
120− ref2next_reg = _mm256_sad_epu8(ref2next_reg, srcnext_reg);
121− ref3next_reg = _mm256_sad_epu8(ref3next_reg, srcnext_reg);
103+ r_lo[0] = _mm256_sad_epu8(r_lo[0], s_lo);
104+ r_lo[1] = _mm256_sad_epu8(r_lo[1], s_lo);
105+ r_lo[2] = _mm256_sad_epu8(r_lo[2], s_lo);
106+ r_lo[3] = _mm256_sad_epu8(r_lo[3], s_lo);
107+ r_hi[0] = _mm256_sad_epu8(r_hi[0], s_hi);
108+ r_hi[1] = _mm256_sad_epu8(r_hi[1], s_hi);
109+ r_hi[2] = _mm256_sad_epu8(r_hi[2], s_hi);
110+ r_hi[3] = _mm256_sad_epu8(r_hi[3], s_hi);
122111
123112 // sum every ref-i
124− sum_ref0 = _mm256_add_epi32(sum_ref0, ref0_reg);
125− sum_ref1 = _mm256_add_epi32(sum_ref1, ref1_reg);
126− sum_ref2 = _mm256_add_epi32(sum_ref2, ref2_reg);
127− sum_ref3 = _mm256_add_epi32(sum_ref3, ref3_reg);
128− sum_ref0 = _mm256_add_epi32(sum_ref0, ref0next_reg);
129− sum_ref1 = _mm256_add_epi32(sum_ref1, ref1next_reg);
130− sum_ref2 = _mm256_add_epi32(sum_ref2, ref2next_reg);
131− sum_ref3 = _mm256_add_epi32(sum_ref3, ref3next_reg);
113+ sums[0] = _mm256_add_epi32(sums[0], r_lo[0]);
114+ sums[1] = _mm256_add_epi32(sums[1], r_lo[1]);
115+ sums[2] = _mm256_add_epi32(sums[2], r_lo[2]);
116+ sums[3] = _mm256_add_epi32(sums[3], r_lo[3]);
117+ sums[0] = _mm256_add_epi32(sums[0], r_hi[0]);
118+ sums[1] = _mm256_add_epi32(sums[1], r_hi[1]);
119+ sums[2] = _mm256_add_epi32(sums[2], r_hi[2]);
120+ sums[3] = _mm256_add_epi32(sums[3], r_hi[3]);
121+
132122 src += src_stride;
133− ref0 += ref_stride;
134− ref1 += ref_stride;
135− ref2 += ref_stride;
136− ref3 += ref_stride;
123+ refs[0] += ref_stride;
124+ refs[1] += ref_stride;
125+ refs[2] += ref_stride;
126+ refs[3] += ref_stride;
137127 }
138− {
139− __m128i sum;
140−
141− // in sum_ref-i the result is saved in the first 4 bytes
142− // the other 4 bytes are zeroed.
143− // sum_ref1 and sum_ref3 are shifted left by 4 bytes
144− sum_ref1 = _mm256_slli_si256(sum_ref1, 4);
145− sum_ref3 = _mm256_slli_si256(sum_ref3, 4);
146−
147− // merge sum_ref0 and sum_ref1 also sum_ref2 and sum_ref3
148− sum_ref0 = _mm256_or_si256(sum_ref0, sum_ref1);
149− sum_ref2 = _mm256_or_si256(sum_ref2, sum_ref3);
150128
151− // merge every 64 bit from each sum_ref-i
152− sum_mlow = _mm256_unpacklo_epi64(sum_ref0, sum_ref2);
153− sum_mhigh = _mm256_unpackhi_epi64(sum_ref0, sum_ref2);
154−
155− // add the low 64 bit to the high 64 bit
156− sum_mlow = _mm256_add_epi32(sum_mlow, sum_mhigh);
157−
158− // add the low 128 bit to the high 128 bit
159− sum = _mm_add_epi32(_mm256_castsi256_si128(sum_mlow),
160− _mm256_extractf128_si256(sum_mlow, 1));
161−
162− _mm_storeu_si128((__m128i *)(res), sum);
163− }
129+ calc_final(sums, res);
164130 }

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t