Add SSE4 version of new apply_temporal_filter

This adds a preliminary version of vp9_apply_temporal_filter in SSE4.1. This patch merely adds the function and does not enable it yet. Speed Up: | ss_x=1 | ss_x=0 | ss_y=1 | 19.80X | 19.04X | ss_y=0 | 21.09X | 20.21X | BUG=webm:1591

chiyotsai committed 7 years ago

Commit b580f761 parent e3210937 files · +1131 −48Browse files

Unified · Split

7 files

test/yuv_temporal_filter_test.cc +14 −2
vp9/common/vp9_rtcd_defs.pl +1 −0
vp9/encoder/vp9_temporal_filter.c +9 −1
vp9/encoder/vp9_temporal_filter.h +1 −1
vp9/encoder/x86/temporal_filter_constants.h +232 −0
vp9/encoder/x86/temporal_filter_sse4.c +872 −44
vp9/vp9cx.mk +2 −0

test/yuv_temporal_filter_test.cc+14−2

@@ -42,7 +42,14 @@ int GetFilterWeight(unsigned int row, unsigned int col, · 41 unmodified lines
42	42
43	43	int GetModIndex(int sum_dist, int index, int rounding, int strength,
44	44	int filter_weight) {
45		− int mod = (sum_dist * 3) / index;
	45	+ unsigned int index_mult[14] = {
	46	+ 0, 0, 0, 0, 49152, 39322, 32768, 28087, 24576, 21846, 19661, 17874, 0, 15124
	47	+ };
	48	+
	49	+ assert(index >= 0 && index <= 13);
	50	+ assert(index_mult[index] != 0);
	51	+
	52	+ int mod = (clamp(sum_dist, 0, UINT16_MAX) * index_mult[index]) >> 16;
46	53	mod += rounding;
47	54	mod >>= strength;
48	55
@@ -564,5 +571,10 @@ TEST_P(YUVTemporalFilterTest, DISABLED_Speed) { · 515 unmodified lines
564	571	}
565	572
566	573	INSTANTIATE_TEST_CASE_P(C, YUVTemporalFilterTest,
567		− ::testing::Values(&vp9_apply_temporal_filter));
	574	+ ::testing::Values(&vp9_apply_temporal_filter_c));
	575	+
	576	+#if HAVE_SSE4_1
	577	+INSTANTIATE_TEST_CASE_P(SSE4, YUVTemporalFilterTest,
	578	+ ::testing::Values(&vp9_apply_temporal_filter_sse4_1));
	579	+#endif // HAVE_SSE4_1
568	580	} // namespace

vp9/common/vp9_rtcd_defs.pl+1−0

@@ -188,6 +188,7 @@ add_proto qw/void vp9_temporal_filter_apply/, "const uint8_t *frame1, unsigned i · 187 unmodified lines
188	188	specialize qw/vp9_temporal_filter_apply sse4_1/;
189	189
190	190	add_proto qw/void vp9_apply_temporal_filter/, "const uint8_t y_src, int y_src_stride, const uint8_t y_pre, int y_pre_stride, const uint8_t u_src, const uint8_t v_src, int uv_src_stride, const uint8_t u_pre, const uint8_t v_pre, int uv_pre_stride, unsigned int block_width, unsigned int block_height, int ss_x, int ss_y, int strength, const int const blk_fw, int use_32x32, uint32_t y_accumulator, uint16_t y_count, uint32_t u_accumulator, uint16_t u_count, uint32_t v_accumulator, uint16_t *v_count";
	191	+specialize qw/vp9_apply_temporal_filter sse4_1/;
191	192	}
192	193
193	194	if (vpx_config("CONFIG_VP9_HIGHBITDEPTH") eq "yes") {

vp9/encoder/vp9_temporal_filter.c+9−1

@@ -34,6 +34,9 @@ · 33 unmodified lines
34	34	#include "vpx_scale/vpx_scale.h"
35	35
36	36	static int fixed_divide[512];
	37	+static unsigned int index_mult[14] = {
	38	+ 0, 0, 0, 0, 49152, 39322, 32768, 28087, 24576, 21846, 19661, 17874, 0, 15124
	39	+};
37	40
38	41	static void temporal_filter_predictors_mb_c(
39	42	MACROBLOCKD xd, uint8_t y_mb_ptr, uint8_t u_mb_ptr, uint8_t v_mb_ptr,
@@ -184,7 +187,12 @@ void vp9_temporal_filter_init(void) { · 144 unmodified lines
184	187
185	188	static INLINE int mod_index(int sum_dist, int index, int rounding, int strength,
186	189	int filter_weight) {
187		− int mod = (sum_dist * 3) / index;
	190	+ int mod;
	191	+
	192	+ assert(index >= 0 && index <= 13);
	193	+ assert(index_mult[index] != 0);
	194	+
	195	+ mod = (clamp(sum_dist, 0, UINT16_MAX) * index_mult[index]) >> 16;
188	196	mod += rounding;
189	197	mod >>= strength;
190	198

vp9/encoder/vp9_temporal_filter.h+1−1

@@ -24,7 +24,7 @@ static const MV kZeroMv = { 0, 0 }; · 23 unmodified lines
24	24	#define BH_LOG2 5
25	25	#define BW 32
26	26	#define BW_LOG2 5
27		−#define BLK_PELS 1024 // Pixels in the block
	27	+#define BLK_PELS ((BH) * (BW)) // Pixels in the block
28	28	#define TF_SHIFT 2
29	29	#define TF_ROUND 3
30	30	#define THR_SHIFT 2

vp9/encoder/x86/temporal_filter_constants.h+232−0

@@ -0,0 +1,232 @@
	1	+/*
	2	+ * Copyright (c) 2019 The WebM project authors. All Rights Reserved.
	3	+ *
	4	+ * Use of this source code is governed by a BSD-style license
	5	+ * that can be found in the LICENSE file in the root of the source
	6	+ * tree. An additional intellectual property rights grant can be found
	7	+ * in the file PATENTS. All contributing project authors may
	8	+ * be found in the AUTHORS file in the root of the source tree.
	9	+ */
	10	+
	11	+#include "./vpx_config.h"
	12	+
	13	+// Division using multiplication and shifting. The C implementation does:
	14	+// modifier *= 3;
	15	+// modifier /= index;
	16	+// where 'modifier' is a set of summed values and 'index' is the number of
	17	+// summed values.
	18	+//
	19	+// This equation works out to (m * 3) / i which reduces to:
	20	+// m * 3/4
	21	+// m * 1/2
	22	+// m * 1/3
	23	+//
	24	+// By pairing the multiply with a down shift by 16 (_mm_mulhi_epu16):
	25	+// m * C / 65536
	26	+// we can create a C to replicate the division.
	27	+//
	28	+// m * 49152 / 65536 = m * 3/4
	29	+// m * 32758 / 65536 = m * 1/2
	30	+// m * 21846 / 65536 = m * 0.3333
	31	+//
	32	+// These are loaded using an instruction expecting int16_t values but are used
	33	+// with _mm_mulhi_epu16(), which treats them as unsigned.
	34	+#define NEIGHBOR_CONSTANT_4 (int16_t)49152
	35	+#define NEIGHBOR_CONSTANT_5 (int16_t)39322
	36	+#define NEIGHBOR_CONSTANT_6 (int16_t)32768
	37	+#define NEIGHBOR_CONSTANT_7 (int16_t)28087
	38	+#define NEIGHBOR_CONSTANT_8 (int16_t)24576
	39	+#define NEIGHBOR_CONSTANT_9 (int16_t)21846
	40	+#define NEIGHBOR_CONSTANT_10 (int16_t)19661
	41	+#define NEIGHBOR_CONSTANT_11 (int16_t)17874
	42	+#define NEIGHBOR_CONSTANT_13 (int16_t)15124
	43	+
	44	+DECLARE_ALIGNED(16, static const int16_t, LEFT_CORNER_NEIGHBORS_PLUS_1[8]) = {
	45	+ NEIGHBOR_CONSTANT_5, NEIGHBOR_CONSTANT_7, NEIGHBOR_CONSTANT_7,
	46	+ NEIGHBOR_CONSTANT_7, NEIGHBOR_CONSTANT_7, NEIGHBOR_CONSTANT_7,
	47	+ NEIGHBOR_CONSTANT_7, NEIGHBOR_CONSTANT_7
	48	+};
	49	+
	50	+DECLARE_ALIGNED(16, static const int16_t, RIGHT_CORNER_NEIGHBORS_PLUS_1[8]) = {
	51	+ NEIGHBOR_CONSTANT_7, NEIGHBOR_CONSTANT_7, NEIGHBOR_CONSTANT_7,
	52	+ NEIGHBOR_CONSTANT_7, NEIGHBOR_CONSTANT_7, NEIGHBOR_CONSTANT_7,
	53	+ NEIGHBOR_CONSTANT_7, NEIGHBOR_CONSTANT_5
	54	+};
	55	+
	56	+DECLARE_ALIGNED(16, static const int16_t, LEFT_EDGE_NEIGHBORS_PLUS_1[8]) = {
	57	+ NEIGHBOR_CONSTANT_7, NEIGHBOR_CONSTANT_10, NEIGHBOR_CONSTANT_10,
	58	+ NEIGHBOR_CONSTANT_10, NEIGHBOR_CONSTANT_10, NEIGHBOR_CONSTANT_10,
	59	+ NEIGHBOR_CONSTANT_10, NEIGHBOR_CONSTANT_10
	60	+};
	61	+
	62	+DECLARE_ALIGNED(16, static const int16_t, RIGHT_EDGE_NEIGHBORS_PLUS_1[8]) = {
	63	+ NEIGHBOR_CONSTANT_10, NEIGHBOR_CONSTANT_10, NEIGHBOR_CONSTANT_10,
	64	+ NEIGHBOR_CONSTANT_10, NEIGHBOR_CONSTANT_10, NEIGHBOR_CONSTANT_10,
	65	+ NEIGHBOR_CONSTANT_10, NEIGHBOR_CONSTANT_7
	66	+};
	67	+
	68	+DECLARE_ALIGNED(16, static const int16_t, MIDDLE_EDGE_NEIGHBORS_PLUS_1[8]) = {
	69	+ NEIGHBOR_CONSTANT_7, NEIGHBOR_CONSTANT_7, NEIGHBOR_CONSTANT_7,
	70	+ NEIGHBOR_CONSTANT_7, NEIGHBOR_CONSTANT_7, NEIGHBOR_CONSTANT_7,
	71	+ NEIGHBOR_CONSTANT_7, NEIGHBOR_CONSTANT_7
	72	+};
	73	+
	74	+DECLARE_ALIGNED(16, static const int16_t, MIDDLE_CENTER_NEIGHBORS_PLUS_1[8]) = {
	75	+ NEIGHBOR_CONSTANT_10, NEIGHBOR_CONSTANT_10, NEIGHBOR_CONSTANT_10,
	76	+ NEIGHBOR_CONSTANT_10, NEIGHBOR_CONSTANT_10, NEIGHBOR_CONSTANT_10,
	77	+ NEIGHBOR_CONSTANT_10, NEIGHBOR_CONSTANT_10
	78	+};
	79	+
	80	+DECLARE_ALIGNED(16, static const int16_t, LEFT_CORNER_NEIGHBORS_PLUS_2[8]) = {
	81	+ NEIGHBOR_CONSTANT_6, NEIGHBOR_CONSTANT_8, NEIGHBOR_CONSTANT_8,
	82	+ NEIGHBOR_CONSTANT_8, NEIGHBOR_CONSTANT_8, NEIGHBOR_CONSTANT_8,
	83	+ NEIGHBOR_CONSTANT_8, NEIGHBOR_CONSTANT_8
	84	+};
	85	+
	86	+DECLARE_ALIGNED(16, static const int16_t, RIGHT_CORNER_NEIGHBORS_PLUS_2[8]) = {
	87	+ NEIGHBOR_CONSTANT_8, NEIGHBOR_CONSTANT_8, NEIGHBOR_CONSTANT_8,
	88	+ NEIGHBOR_CONSTANT_8, NEIGHBOR_CONSTANT_8, NEIGHBOR_CONSTANT_8,
	89	+ NEIGHBOR_CONSTANT_8, NEIGHBOR_CONSTANT_6
	90	+};
	91	+
	92	+DECLARE_ALIGNED(16, static const int16_t, LEFT_EDGE_NEIGHBORS_PLUS_2[8]) = {
	93	+ NEIGHBOR_CONSTANT_8, NEIGHBOR_CONSTANT_11, NEIGHBOR_CONSTANT_11,
	94	+ NEIGHBOR_CONSTANT_11, NEIGHBOR_CONSTANT_11, NEIGHBOR_CONSTANT_11,
	95	+ NEIGHBOR_CONSTANT_11, NEIGHBOR_CONSTANT_11
	96	+};
	97	+
	98	+DECLARE_ALIGNED(16, static const int16_t, RIGHT_EDGE_NEIGHBORS_PLUS_2[8]) = {
	99	+ NEIGHBOR_CONSTANT_11, NEIGHBOR_CONSTANT_11, NEIGHBOR_CONSTANT_11,
	100	+ NEIGHBOR_CONSTANT_11, NEIGHBOR_CONSTANT_11, NEIGHBOR_CONSTANT_11,
	101	+ NEIGHBOR_CONSTANT_11, NEIGHBOR_CONSTANT_8
	102	+};
	103	+
	104	+DECLARE_ALIGNED(16, static const int16_t, MIDDLE_EDGE_NEIGHBORS_PLUS_2[8]) = {
	105	+ NEIGHBOR_CONSTANT_8, NEIGHBOR_CONSTANT_8, NEIGHBOR_CONSTANT_8,
	106	+ NEIGHBOR_CONSTANT_8, NEIGHBOR_CONSTANT_8, NEIGHBOR_CONSTANT_8,
	107	+ NEIGHBOR_CONSTANT_8, NEIGHBOR_CONSTANT_8
	108	+};
	109	+
	110	+DECLARE_ALIGNED(16, static const int16_t, MIDDLE_CENTER_NEIGHBORS_PLUS_2[8]) = {
	111	+ NEIGHBOR_CONSTANT_11, NEIGHBOR_CONSTANT_11, NEIGHBOR_CONSTANT_11,
	112	+ NEIGHBOR_CONSTANT_11, NEIGHBOR_CONSTANT_11, NEIGHBOR_CONSTANT_11,
	113	+ NEIGHBOR_CONSTANT_11, NEIGHBOR_CONSTANT_11
	114	+};
	115	+
	116	+DECLARE_ALIGNED(16, static const int16_t, TWO_CORNER_NEIGHBORS_PLUS_2[8]) = {
	117	+ NEIGHBOR_CONSTANT_6, NEIGHBOR_CONSTANT_8, NEIGHBOR_CONSTANT_8,
	118	+ NEIGHBOR_CONSTANT_8, NEIGHBOR_CONSTANT_8, NEIGHBOR_CONSTANT_8,
	119	+ NEIGHBOR_CONSTANT_8, NEIGHBOR_CONSTANT_6
	120	+};
	121	+
	122	+DECLARE_ALIGNED(16, static const int16_t, TWO_EDGE_NEIGHBORS_PLUS_2[8]) = {
	123	+ NEIGHBOR_CONSTANT_8, NEIGHBOR_CONSTANT_11, NEIGHBOR_CONSTANT_11,
	124	+ NEIGHBOR_CONSTANT_11, NEIGHBOR_CONSTANT_11, NEIGHBOR_CONSTANT_11,
	125	+ NEIGHBOR_CONSTANT_11, NEIGHBOR_CONSTANT_8
	126	+};
	127	+
	128	+DECLARE_ALIGNED(16, static const int16_t, LEFT_CORNER_NEIGHBORS_PLUS_4[8]) = {
	129	+ NEIGHBOR_CONSTANT_8, NEIGHBOR_CONSTANT_10, NEIGHBOR_CONSTANT_10,
	130	+ NEIGHBOR_CONSTANT_10, NEIGHBOR_CONSTANT_10, NEIGHBOR_CONSTANT_10,
	131	+ NEIGHBOR_CONSTANT_10, NEIGHBOR_CONSTANT_10
	132	+};
	133	+
	134	+DECLARE_ALIGNED(16, static const int16_t, RIGHT_CORNER_NEIGHBORS_PLUS_4[8]) = {
	135	+ NEIGHBOR_CONSTANT_10, NEIGHBOR_CONSTANT_10, NEIGHBOR_CONSTANT_10,
	136	+ NEIGHBOR_CONSTANT_10, NEIGHBOR_CONSTANT_10, NEIGHBOR_CONSTANT_10,
	137	+ NEIGHBOR_CONSTANT_10, NEIGHBOR_CONSTANT_8
	138	+};
	139	+
	140	+DECLARE_ALIGNED(16, static const int16_t, LEFT_EDGE_NEIGHBORS_PLUS_4[8]) = {
	141	+ NEIGHBOR_CONSTANT_10, NEIGHBOR_CONSTANT_13, NEIGHBOR_CONSTANT_13,
	142	+ NEIGHBOR_CONSTANT_13, NEIGHBOR_CONSTANT_13, NEIGHBOR_CONSTANT_13,
	143	+ NEIGHBOR_CONSTANT_13, NEIGHBOR_CONSTANT_13
	144	+};
	145	+
	146	+DECLARE_ALIGNED(16, static const int16_t, RIGHT_EDGE_NEIGHBORS_PLUS_4[8]) = {
	147	+ NEIGHBOR_CONSTANT_13, NEIGHBOR_CONSTANT_13, NEIGHBOR_CONSTANT_13,
	148	+ NEIGHBOR_CONSTANT_13, NEIGHBOR_CONSTANT_13, NEIGHBOR_CONSTANT_13,
	149	+ NEIGHBOR_CONSTANT_13, NEIGHBOR_CONSTANT_10
	150	+};
	151	+
	152	+DECLARE_ALIGNED(16, static const int16_t, MIDDLE_EDGE_NEIGHBORS_PLUS_4[8]) = {
	153	+ NEIGHBOR_CONSTANT_10, NEIGHBOR_CONSTANT_10, NEIGHBOR_CONSTANT_10,
	154	+ NEIGHBOR_CONSTANT_10, NEIGHBOR_CONSTANT_10, NEIGHBOR_CONSTANT_10,
	155	+ NEIGHBOR_CONSTANT_10, NEIGHBOR_CONSTANT_10
	156	+};
	157	+
	158	+DECLARE_ALIGNED(16, static const int16_t, MIDDLE_CENTER_NEIGHBORS_PLUS_4[8]) = {
	159	+ NEIGHBOR_CONSTANT_13, NEIGHBOR_CONSTANT_13, NEIGHBOR_CONSTANT_13,
	160	+ NEIGHBOR_CONSTANT_13, NEIGHBOR_CONSTANT_13, NEIGHBOR_CONSTANT_13,
	161	+ NEIGHBOR_CONSTANT_13, NEIGHBOR_CONSTANT_13
	162	+};
	163	+
	164	+DECLARE_ALIGNED(16, static const int16_t, TWO_CORNER_NEIGHBORS_PLUS_4[8]) = {
	165	+ NEIGHBOR_CONSTANT_8, NEIGHBOR_CONSTANT_10, NEIGHBOR_CONSTANT_10,
	166	+ NEIGHBOR_CONSTANT_10, NEIGHBOR_CONSTANT_10, NEIGHBOR_CONSTANT_10,
	167	+ NEIGHBOR_CONSTANT_10, NEIGHBOR_CONSTANT_8
	168	+};
	169	+
	170	+DECLARE_ALIGNED(16, static const int16_t, TWO_EDGE_NEIGHBORS_PLUS_4[8]) = {
	171	+ NEIGHBOR_CONSTANT_10, NEIGHBOR_CONSTANT_13, NEIGHBOR_CONSTANT_13,
	172	+ NEIGHBOR_CONSTANT_13, NEIGHBOR_CONSTANT_13, NEIGHBOR_CONSTANT_13,
	173	+ NEIGHBOR_CONSTANT_13, NEIGHBOR_CONSTANT_10
	174	+};
	175	+
	176	+static const int16_t *const LUMA_LEFT_COLUMN_NEIGHBORS[2] = {
	177	+ LEFT_CORNER_NEIGHBORS_PLUS_2, LEFT_EDGE_NEIGHBORS_PLUS_2
	178	+};
	179	+
	180	+static const int16_t *const LUMA_MIDDLE_COLUMN_NEIGHBORS[2] = {
	181	+ MIDDLE_EDGE_NEIGHBORS_PLUS_2, MIDDLE_CENTER_NEIGHBORS_PLUS_2
	182	+};
	183	+
	184	+static const int16_t *const LUMA_RIGHT_COLUMN_NEIGHBORS[2] = {
	185	+ RIGHT_CORNER_NEIGHBORS_PLUS_2, RIGHT_EDGE_NEIGHBORS_PLUS_2
	186	+};
	187	+
	188	+static const int16_t *const CHROMA_NO_SS_LEFT_COLUMN_NEIGHBORS[2] = {
	189	+ LEFT_CORNER_NEIGHBORS_PLUS_1, LEFT_EDGE_NEIGHBORS_PLUS_1
	190	+};
	191	+
	192	+static const int16_t *const CHROMA_NO_SS_MIDDLE_COLUMN_NEIGHBORS[2] = {
	193	+ MIDDLE_EDGE_NEIGHBORS_PLUS_1, MIDDLE_CENTER_NEIGHBORS_PLUS_1
	194	+};
	195	+
	196	+static const int16_t *const CHROMA_NO_SS_RIGHT_COLUMN_NEIGHBORS[2] = {
	197	+ RIGHT_CORNER_NEIGHBORS_PLUS_1, RIGHT_EDGE_NEIGHBORS_PLUS_1
	198	+};
	199	+
	200	+static const int16_t *const CHROMA_SINGLE_SS_LEFT_COLUMN_NEIGHBORS[2] = {
	201	+ LEFT_CORNER_NEIGHBORS_PLUS_2, LEFT_EDGE_NEIGHBORS_PLUS_2
	202	+};
	203	+
	204	+static const int16_t *const CHROMA_SINGLE_SS_MIDDLE_COLUMN_NEIGHBORS[2] = {
	205	+ MIDDLE_EDGE_NEIGHBORS_PLUS_2, MIDDLE_CENTER_NEIGHBORS_PLUS_2
	206	+};
	207	+
	208	+static const int16_t *const CHROMA_SINGLE_SS_RIGHT_COLUMN_NEIGHBORS[2] = {
	209	+ RIGHT_CORNER_NEIGHBORS_PLUS_2, RIGHT_EDGE_NEIGHBORS_PLUS_2
	210	+};
	211	+
	212	+static const int16_t *const CHROMA_SINGLE_SS_SINGLE_COLUMN_NEIGHBORS[2] = {
	213	+ TWO_CORNER_NEIGHBORS_PLUS_2, TWO_EDGE_NEIGHBORS_PLUS_2
	214	+};
	215	+
	216	+static const int16_t *const CHROMA_DOUBLE_SS_LEFT_COLUMN_NEIGHBORS[2] = {
	217	+ LEFT_CORNER_NEIGHBORS_PLUS_4, LEFT_EDGE_NEIGHBORS_PLUS_4
	218	+};
	219	+
	220	+static const int16_t *const CHROMA_DOUBLE_SS_MIDDLE_COLUMN_NEIGHBORS[2] = {
	221	+ MIDDLE_EDGE_NEIGHBORS_PLUS_4, MIDDLE_CENTER_NEIGHBORS_PLUS_4
	222	+};
	223	+
	224	+static const int16_t *const CHROMA_DOUBLE_SS_RIGHT_COLUMN_NEIGHBORS[2] = {
	225	+ RIGHT_CORNER_NEIGHBORS_PLUS_4, RIGHT_EDGE_NEIGHBORS_PLUS_4
	226	+};
	227	+
	228	+static const int16_t *const CHROMA_DOUBLE_SS_SINGLE_COLUMN_NEIGHBORS[2] = {
	229	+ TWO_CORNER_NEIGHBORS_PLUS_4, TWO_EDGE_NEIGHBORS_PLUS_4
	230	+};
	231	+
	232	+#define DIST_STRIDE ((BW) + 2)

vp9/encoder/x86/temporal_filter_sse4.c+872−44

@@ -14,32 +14,9 @@ · 13 unmodified lines
14	14	#include "./vp9_rtcd.h"
15	15	#include "./vpx_config.h"
16	16	#include "vpx/vpx_integer.h"
17		−
18		−// Division using multiplication and shifting. The C implementation does:
19		−// modifier *= 3;
20		−// modifier /= index;
21		−// where 'modifier' is a set of summed values and 'index' is the number of
22		−// summed values. 'index' may be 4, 6, or 9, representing a block of 9 values
23		−// which may be bound by the edges of the block being filtered.
24		−//
25		−// This equation works out to (m * 3) / i which reduces to:
26		−// m * 3/4
27		−// m * 1/2
28		−// m * 1/3
29		−//
30		−// By pairing the multiply with a down shift by 16 (_mm_mulhi_epu16):
31		−// m * C / 65536
32		−// we can create a C to replicate the division.
33		−//
34		−// m * 49152 / 65536 = m * 3/4
35		−// m * 32758 / 65536 = m * 1/2
36		−// m * 21846 / 65536 = m * 0.3333
37		−//
38		−// These are loaded using an instruction expecting int16_t values but are used
39		−// with _mm_mulhi_epu16(), which treats them as unsigned.
40		−#define NEIGHBOR_CONSTANT_4 (int16_t)49152
41		−#define NEIGHBOR_CONSTANT_6 (int16_t)32768
42		−#define NEIGHBOR_CONSTANT_9 (int16_t)21846
	17	+#include "vp9/encoder/vp9_encoder.h"
	18	+#include "vp9/encoder/vp9_temporal_filter.h"
	19	+#include "vp9/encoder/x86/temporal_filter_constants.h"
43	20
44	21	// Load values from 'a' and 'b'. Compute the difference squared and sum
45	22	// neighboring values such that:
@@ -106,12 +83,62 @@ static void sum_16(const uint8_t a, const uint8_t b, __m128i *sum_0, · 60 unmodified lines
106	83	*sum_1 = sum_u16;
107	84	}
108	85
	86	+// Read in 8 pixels from a and b as 8-bit unsigned integers, compute the
	87	+// difference squared, and store as unsigned 16-bit integer to dst.
	88	+static INLINE void store_dist_8(const uint8_t a, const uint8_t b,
	89	+ uint16_t *dst) {
	90	+ const __m128i a_reg = _mm_loadl_epi64((const __m128i *)a);
	91	+ const __m128i b_reg = _mm_loadl_epi64((const __m128i *)b);
	92	+
	93	+ const __m128i a_first = _mm_cvtepu8_epi16(a_reg);
	94	+ const __m128i b_first = _mm_cvtepu8_epi16(b_reg);
	95	+
	96	+ __m128i dist_first;
	97	+
	98	+ dist_first = _mm_sub_epi16(a_first, b_first);
	99	+ dist_first = _mm_mullo_epi16(dist_first, dist_first);
	100	+
	101	+ _mm_storeu_si128((__m128i *)dst, dist_first);
	102	+}
	103	+
	104	+static INLINE void store_dist_16(const uint8_t a, const uint8_t b,
	105	+ uint16_t *dst) {
	106	+ const __m128i zero = _mm_setzero_si128();
	107	+ const __m128i a_reg = _mm_loadu_si128((const __m128i *)a);
	108	+ const __m128i b_reg = _mm_loadu_si128((const __m128i *)b);
	109	+
	110	+ const __m128i a_first = _mm_cvtepu8_epi16(a_reg);
	111	+ const __m128i a_second = _mm_unpackhi_epi8(a_reg, zero);
	112	+ const __m128i b_first = _mm_cvtepu8_epi16(b_reg);
	113	+ const __m128i b_second = _mm_unpackhi_epi8(b_reg, zero);
	114	+
	115	+ __m128i dist_first, dist_second;
	116	+
	117	+ dist_first = _mm_sub_epi16(a_first, b_first);
	118	+ dist_second = _mm_sub_epi16(a_second, b_second);
	119	+ dist_first = _mm_mullo_epi16(dist_first, dist_first);
	120	+ dist_second = _mm_mullo_epi16(dist_second, dist_second);
	121	+
	122	+ _mm_storeu_si128((__m128i *)dst, dist_first);
	123	+ _mm_storeu_si128((__m128i *)(dst + 8), dist_second);
	124	+}
	125	+
	126	+static INLINE void read_dist_8(const uint16_t dist, __m128i dist_reg) {
	127	+ dist_reg = _mm_loadu_si128((const __m128i )dist);
	128	+}
	129	+
	130	+static INLINE void read_dist_16(const uint16_t dist, __m128i reg_first,
	131	+ __m128i *reg_second) {
	132	+ read_dist_8(dist, reg_first);
	133	+ read_dist_8(dist + 8, reg_second);
	134	+}
	135	+
109	136	// Average the value based on the number of values summed (9 for pixels away
110	137	// from the border, 4 for pixels in corners, and 6 for other edge values).
111	138	//
112	139	// Add in the rounding factor and shift, clamp to 16, invert and shift. Multiply
113	140	// by weight.
114		−static __m128i average_8(__m128i sum, const __m128i mul_constants,
	141	+static __m128i average_8(__m128i sum, const __m128i *mul_constants,
115	142	const int strength, const int rounding,
116	143	const int weight) {
117	144	// _mm_srl_epi16 uses the lower 64 bit value for the shift.
@@ -121,7 +148,34 @@ static __m128i average_8(__m128i sum, const __m128i mul_constants, · 3 unmodified lines
121	148	const __m128i sixteen = _mm_set1_epi16(16);
122	149
123	150	// modifier * 3 / index;
124		− sum = _mm_mulhi_epu16(sum, mul_constants);
	151	+ sum = _mm_mulhi_epu16(sum, *mul_constants);
	152	+
	153	+ sum = _mm_adds_epu16(sum, rounding_u16);
	154	+ sum = _mm_srl_epi16(sum, strength_u128);
	155	+
	156	+ // The maximum input to this comparison is UINT16_MAX * NEIGHBOR_CONSTANT_4
	157	+ // >> 16 (also NEIGHBOR_CONSTANT_4 -1) which is 49151 / 0xbfff / -16385
	158	+ // So this needs to use the epu16 version which did not come until SSE4.
	159	+ sum = _mm_min_epu16(sum, sixteen);
	160	+
	161	+ sum = _mm_sub_epi16(sixteen, sum);
	162	+
	163	+ return _mm_mullo_epi16(sum, weight_u16);
	164	+}
	165	+
	166	+static __m128i average_4_4(__m128i sum, const __m128i *mul_constants,
	167	+ const int strength, const int rounding,
	168	+ const int weight_0, const int weight_1) {
	169	+ // _mm_srl_epi16 uses the lower 64 bit value for the shift.
	170	+ const __m128i strength_u128 = _mm_set_epi32(0, 0, 0, strength);
	171	+ const __m128i rounding_u16 = _mm_set1_epi16(rounding);
	172	+ const __m128i weight_u16 =
	173	+ _mm_setr_epi16(weight_0, weight_0, weight_0, weight_0, weight_1, weight_1,
	174	+ weight_1, weight_1);
	175	+ const __m128i sixteen = _mm_set1_epi16(16);
	176	+
	177	+ // modifier * 3 / index;
	178	+ sum = _mm_mulhi_epu16(sum, *mul_constants);
125	179
126	180	sum = _mm_adds_epu16(sum, rounding_u16);
127	181	sum = _mm_srl_epi16(sum, strength_u128);
@@ -136,20 +190,21 @@ static __m128i average_8(__m128i sum, const __m128i mul_constants, · 8 unmodified lines
136	190	return _mm_mullo_epi16(sum, weight_u16);
137	191	}
138	192
139		−static void average_16(__m128i sum_0_u16, __m128i sum_1_u16,
140		− const __m128i mul_constants_0,
141		− const __m128i mul_constants_1, const int strength,
142		− const int rounding, const int weight) {
	193	+static INLINE void average_16(__m128i sum_0_u16, __m128i sum_1_u16,
	194	+ const __m128i *mul_constants_0,
	195	+ const __m128i *mul_constants_1,
	196	+ const int strength, const int rounding,
	197	+ const int weight) {
143	198	const __m128i strength_u128 = _mm_set_epi32(0, 0, 0, strength);
144	199	const __m128i rounding_u16 = _mm_set1_epi16(rounding);
145	200	const __m128i weight_u16 = _mm_set1_epi16(weight);
146	201	const __m128i sixteen = _mm_set1_epi16(16);
147	202	__m128i input_0, input_1;
148	203
149		− input_0 = _mm_mulhi_epu16(*sum_0_u16, mul_constants_0);
	204	+ input_0 = _mm_mulhi_epu16(sum_0_u16, mul_constants_0);
150	205	input_0 = _mm_adds_epu16(input_0, rounding_u16);
151	206
152		− input_1 = _mm_mulhi_epu16(*sum_1_u16, mul_constants_1);
	207	+ input_1 = _mm_mulhi_epu16(sum_1_u16, mul_constants_1);
153	208	input_1 = _mm_adds_epu16(input_1, rounding_u16);
154	209
155	210	input_0 = _mm_srl_epi16(input_0, strength_u128);
@@ -192,10 +247,10 @@ static void accumulate_and_store_8(const __m128i sum_u16, const uint8_t *pred, · 36 unmodified lines
192	247	_mm_storeu_si128((__m128i *)(accumulator + 4), accum_1_u32);
193	248	}
194	249
195		−static void accumulate_and_store_16(const __m128i sum_0_u16,
196		− const __m128i sum_1_u16,
197		− const uint8_t pred, uint16_t count,
198		− uint32_t *accumulator) {
	250	+static INLINE void accumulate_and_store_16(const __m128i sum_0_u16,
	251	+ const __m128i sum_1_u16,
	252	+ const uint8_t pred, uint16_t count,
	253	+ uint32_t *accumulator) {
199	254	const __m128i pred_u8 = _mm_loadu_si128((const __m128i *)pred);
200	255	const __m128i zero = _mm_setzero_si128();
201	256	__m128i count_0_u16 = _mm_loadu_si128((const __m128i *)count),
@@ -235,6 +290,28 @@ static void accumulate_and_store_16(const __m128i sum_0_u16, · 33 unmodified lines
235	290	_mm_storeu_si128((__m128i *)(accumulator + 12), accum_3_u32);
236	291	}
237	292
	293	+// Read in 8 pixels from y_dist. For each index i, compute y_dist[i-1] +
	294	+// y_dist[i] + y_dist[i+1] and store in sum as 16-bit unsigned int.
	295	+static INLINE void get_sum_8(const uint16_t y_dist, __m128i sum) {
	296	+ __m128i dist_reg, dist_left, dist_right;
	297	+
	298	+ dist_reg = _mm_loadu_si128((const __m128i *)y_dist);
	299	+ dist_left = _mm_loadu_si128((const __m128i *)(y_dist - 1));
	300	+ dist_right = _mm_loadu_si128((const __m128i *)(y_dist + 1));
	301	+
	302	+ *sum = _mm_adds_epu16(dist_reg, dist_left);
	303	+ sum = _mm_adds_epu16(sum, dist_right);
	304	+}
	305	+
	306	+// Read in 16 pixels from y_dist. For each index i, compute y_dist[i-1] +
	307	+// y_dist[i] + y_dist[i+1]. Store the result for first 8 pixels in sum_first and
	308	+// the rest in sum_second.
	309	+static INLINE void get_sum_16(const uint16_t y_dist, __m128i sum_first,
	310	+ __m128i *sum_second) {
	311	+ get_sum_8(y_dist, sum_first);
	312	+ get_sum_8(y_dist + 8, sum_second);
	313	+}
	314	+
238	315	void vp9_temporal_filter_apply_sse4_1(const uint8_t *a, unsigned int stride,
239	316	const uint8_t *b, unsigned int width,
240	317	unsigned int height, int strength,
@@ -261,7 +338,8 @@ void vp9_temporal_filter_apply_sse4_1(const uint8_t *a, unsigned int stride, · 20 unmodified lines
261	338	sum_8(a, b, &sum_row_a);
262	339	sum_8(a + stride, b + width, &sum_row_b);
263	340	sum_row_c = _mm_adds_epu16(sum_row_a, sum_row_b);
264		− sum_row_c = average_8(sum_row_c, mul_constants, strength, rounding, weight);
	341	+ sum_row_c =
	342	+ average_8(sum_row_c, &mul_constants, strength, rounding, weight);
265	343	accumulate_and_store_8(sum_row_c, b, count, accumulator);
266	344
267	345	a += stride + stride;
@@ -279,7 +357,7 @@ void vp9_temporal_filter_apply_sse4_1(const uint8_t *a, unsigned int stride, · 11 unmodified lines
279	357	sum_row_a = _mm_adds_epu16(sum_row_a, sum_row_b);
280	358	sum_row_a = _mm_adds_epu16(sum_row_a, sum_row_c);
281	359	sum_row_a =
282		− average_8(sum_row_a, mul_constants, strength, rounding, weight);
	360	+ average_8(sum_row_a, &mul_constants, strength, rounding, weight);
283	361	accumulate_and_store_8(sum_row_a, b, count, accumulator);
284	362
285	363	a += stride;
@@ -296,7 +374,8 @@ void vp9_temporal_filter_apply_sse4_1(const uint8_t *a, unsigned int stride, · 10 unmodified lines
296	374	NEIGHBOR_CONSTANT_6, NEIGHBOR_CONSTANT_6,
297	375	NEIGHBOR_CONSTANT_6, NEIGHBOR_CONSTANT_4);
298	376	sum_row_a = _mm_adds_epu16(sum_row_a, sum_row_b);
299		− sum_row_a = average_8(sum_row_a, mul_constants, strength, rounding, weight);
	377	+ sum_row_a =
	378	+ average_8(sum_row_a, &mul_constants, strength, rounding, weight);
300	379	accumulate_and_store_8(sum_row_a, b, count, accumulator);
301	380
302	381	} else { // width == 16
@@ -318,7 +397,7 @@ void vp9_temporal_filter_apply_sse4_1(const uint8_t *a, unsigned int stride, · 15 unmodified lines
318	397	sum_row_c_0 = _mm_adds_epu16(sum_row_a_0, sum_row_b_0);
319	398	sum_row_c_1 = _mm_adds_epu16(sum_row_a_1, sum_row_b_1);
320	399
321		− average_16(&sum_row_c_0, &sum_row_c_1, mul_constants_0, mul_constants_1,
	400	+ average_16(&sum_row_c_0, &sum_row_c_1, &mul_constants_0, &mul_constants_1,
322	401	strength, rounding, weight);
323	402	accumulate_and_store_16(sum_row_c_0, sum_row_c_1, b, count, accumulator);
324	403
@@ -343,7 +422,7 @@ void vp9_temporal_filter_apply_sse4_1(const uint8_t *a, unsigned int stride, · 18 unmodified lines
343	422	sum_row_a_1 = _mm_adds_epu16(sum_row_a_1, sum_row_b_1);
344	423	sum_row_a_1 = _mm_adds_epu16(sum_row_a_1, sum_row_c_1);
345	424
346		− average_16(&sum_row_a_0, &sum_row_a_1, mul_constants_0, mul_constants_1,
	425	+ average_16(&sum_row_a_0, &sum_row_a_1, &mul_constants_0, &mul_constants_1,
347	426	strength, rounding, weight);
348	427	accumulate_and_store_16(sum_row_a_0, sum_row_a_1, b, count, accumulator);
349	428
@@ -369,8 +448,757 @@ void vp9_temporal_filter_apply_sse4_1(const uint8_t *a, unsigned int stride, · 19 unmodified lines
369	448	sum_row_c_0 = _mm_adds_epu16(sum_row_a_0, sum_row_b_0);
370	449	sum_row_c_1 = _mm_adds_epu16(sum_row_a_1, sum_row_b_1);
371	450
372		− average_16(&sum_row_c_0, &sum_row_c_1, mul_constants_0, mul_constants_1,
	451	+ average_16(&sum_row_c_0, &sum_row_c_1, &mul_constants_0, &mul_constants_1,
373	452	strength, rounding, weight);
374	453	accumulate_and_store_16(sum_row_c_0, sum_row_c_1, b, count, accumulator);
375	454	}
376	455	}
	456	+
	457	+// Read in a row of chroma values corresponds to a row of 16 luma values.
	458	+static INLINE void read_chroma_dist_row_16(int ss_x, const uint16_t *u_dist,
	459	+ const uint16_t *v_dist,
	460	+ __m128i u_first, __m128i u_second,
	461	+ __m128i *v_first,
	462	+ __m128i *v_second) {
	463	+ if (!ss_x) {
	464	+ // If there is no chroma subsampling in the horizaontal direction, then we
	465	+ // need to load 16 entries from chroma.
	466	+ read_dist_16(u_dist, u_first, u_second);
	467	+ read_dist_16(v_dist, v_first, v_second);
	468	+ } else { // ss_x == 1
	469	+ // Otherwise, we only need to load 8 entries
	470	+ __m128i u_reg, v_reg;
	471	+
	472	+ read_dist_8(u_dist, &u_reg);
	473	+
	474	+ *u_first = _mm_unpacklo_epi16(u_reg, u_reg);
	475	+ *u_second = _mm_unpackhi_epi16(u_reg, u_reg);
	476	+
	477	+ read_dist_8(v_dist, &v_reg);
	478	+
	479	+ *v_first = _mm_unpacklo_epi16(v_reg, v_reg);
	480	+ *v_second = _mm_unpackhi_epi16(v_reg, v_reg);
	481	+ }
	482	+}
	483	+
	484	+// Horizonta add unsigned 16-bit ints in src and store them as signed 32-bit int
	485	+// in dst.
	486	+static INLINE void hadd_epu16(__m128i src, __m128i dst) {
	487	+ const __m128i zero = _mm_setzero_si128();
	488	+ const __m128i shift_right = _mm_srli_si128(*src, 2);
	489	+
	490	+ const __m128i odd = _mm_blend_epi16(shift_right, zero, 170);
	491	+ const __m128i even = _mm_blend_epi16(*src, zero, 170);
	492	+
	493	+ *dst = _mm_add_epi32(even, odd);
	494	+}
	495	+
	496	+// Add a row of luma distortion to 8 corresponding chroma mods.
	497	+static INLINE void add_luma_dist_to_8_chroma_mod(const uint16_t *y_dist,
	498	+ int ss_x, int ss_y,
	499	+ __m128i *u_mod,
	500	+ __m128i *v_mod) {
	501	+ __m128i y_reg;
	502	+ if (!ss_x) {
	503	+ read_dist_8(y_dist, &y_reg);
	504	+ if (ss_y == 1) {
	505	+ __m128i y_tmp;
	506	+ read_dist_8(y_dist + DIST_STRIDE, &y_tmp);
	507	+
	508	+ y_reg = _mm_adds_epu16(y_reg, y_tmp);
	509	+ }
	510	+ } else {
	511	+ __m128i y_first, y_second;
	512	+ read_dist_16(y_dist, &y_first, &y_second);
	513	+ if (ss_y == 1) {
	514	+ __m128i y_tmp_0, y_tmp_1;
	515	+ read_dist_16(y_dist + DIST_STRIDE, &y_tmp_0, &y_tmp_1);
	516	+
	517	+ y_first = _mm_adds_epu16(y_first, y_tmp_0);
	518	+ y_second = _mm_adds_epu16(y_second, y_tmp_1);
	519	+ }
	520	+
	521	+ hadd_epu16(&y_first, &y_first);
	522	+ hadd_epu16(&y_second, &y_second);
	523	+
	524	+ y_reg = _mm_packus_epi32(y_first, y_second);
	525	+ }
	526	+
	527	+ u_mod = _mm_adds_epu16(u_mod, y_reg);
	528	+ v_mod = _mm_adds_epu16(v_mod, y_reg);
	529	+}
	530	+
	531	+// Apply temporal filter to the luma components. This performs temporal
	532	+// filtering on a luma block of 16 X block_height. Use blk_fw as an array of
	533	+// size 4for the weights for each of the 4 subblocks if blk_fw is not NULL,
	534	+// else use top_weight for top half, and bottom weight for bottom half.
	535	+static void vp9_apply_temporal_filter_luma_16(
	536	+ const uint8_t y_src, int y_src_stride, const uint8_t y_pre,
	537	+ int y_pre_stride, const uint8_t u_src, const uint8_t v_src,
	538	+ int uv_src_stride, const uint8_t u_pre, const uint8_t v_pre,
	539	+ int uv_pre_stride, unsigned int block_width, unsigned int block_height,
	540	+ int ss_x, int ss_y, int strength, int use_whole_blk, uint32_t *y_accum,
	541	+ uint16_t y_count, const uint16_t y_dist, const uint16_t *u_dist,
	542	+ const uint16_t v_dist, const int16_t const *neighbors_first,
	543	+ const int16_t const neighbors_second, int top_weight, int bottom_weight,
	544	+ const int *blk_fw) {
	545	+ const int rounding = (1 << strength) >> 1;
	546	+ int weight = top_weight;
	547	+
	548	+ __m128i mul_first, mul_second;
	549	+
	550	+ __m128i sum_row_1_first, sum_row_1_second;
	551	+ __m128i sum_row_2_first, sum_row_2_second;
	552	+ __m128i sum_row_3_first, sum_row_3_second;
	553	+
	554	+ __m128i u_first, u_second;
	555	+ __m128i v_first, v_second;
	556	+
	557	+ __m128i sum_row_first;
	558	+ __m128i sum_row_second;
	559	+
	560	+ assert(strength >= 0);
	561	+ assert(strength <= 6);
	562	+
	563	+ assert(block_width == 16);
	564	+
	565	+ (void)block_width;
	566	+
	567	+ // First row
	568	+ mul_first = _mm_loadu_si128((const __m128i *)neighbors_first[0]);
	569	+ mul_second = _mm_loadu_si128((const __m128i *)neighbors_second[0]);
	570	+
	571	+ // Add luma values
	572	+ get_sum_16(y_dist, &sum_row_2_first, &sum_row_2_second);
	573	+ get_sum_16(y_dist + DIST_STRIDE, &sum_row_3_first, &sum_row_3_second);
	574	+
	575	+ sum_row_first = _mm_adds_epu16(sum_row_2_first, sum_row_3_first);
	576	+ sum_row_second = _mm_adds_epu16(sum_row_2_second, sum_row_3_second);
	577	+
	578	+ // Add chroma values
	579	+ read_chroma_dist_row_16(ss_x, u_dist, v_dist, &u_first, &u_second, &v_first,
	580	+ &v_second);
	581	+
	582	+ sum_row_first = _mm_adds_epu16(sum_row_first, u_first);
	583	+ sum_row_second = _mm_adds_epu16(sum_row_second, u_second);
	584	+
	585	+ sum_row_first = _mm_adds_epu16(sum_row_first, v_first);
	586	+ sum_row_second = _mm_adds_epu16(sum_row_second, v_second);
	587	+
	588	+ // Get modifier and store result
	589	+ if (blk_fw) {
	590	+ sum_row_first =
	591	+ average_8(sum_row_first, &mul_first, strength, rounding, blk_fw[0]);
	592	+ sum_row_second =
	593	+ average_8(sum_row_second, &mul_second, strength, rounding, blk_fw[1]);
	594	+ } else {
	595	+ average_16(&sum_row_first, &sum_row_second, &mul_first, &mul_second,
	596	+ strength, rounding, weight);
	597	+ }
	598	+ accumulate_and_store_16(sum_row_first, sum_row_second, y_pre, y_count,
	599	+ y_accum);
	600	+
	601	+ y_src += y_src_stride;
	602	+ y_pre += y_pre_stride;
	603	+ y_count += y_pre_stride;
	604	+ y_accum += y_pre_stride;
	605	+ y_dist += DIST_STRIDE;
	606	+
	607	+ u_src += uv_src_stride;
	608	+ u_pre += uv_pre_stride;
	609	+ u_dist += DIST_STRIDE;
	610	+ v_src += uv_src_stride;
	611	+ v_pre += uv_pre_stride;
	612	+ v_dist += DIST_STRIDE;
	613	+
	614	+ // Then all the rows except the last one
	615	+ mul_first = _mm_loadu_si128((const __m128i *)neighbors_first[1]);
	616	+ mul_second = _mm_loadu_si128((const __m128i *)neighbors_second[1]);
	617	+
	618	+ for (unsigned int h = 1; h < block_height - 1; ++h) {
	619	+ // Move the weight to bottom half
	620	+ if (!use_whole_blk && h == block_height / 2) {
	621	+ if (blk_fw) {
	622	+ blk_fw += 2;
	623	+ } else {
	624	+ weight = bottom_weight;
	625	+ }
	626	+ }
	627	+ // Shift the rows up
	628	+ sum_row_1_first = sum_row_2_first;
	629	+ sum_row_1_second = sum_row_2_second;
	630	+ sum_row_2_first = sum_row_3_first;
	631	+ sum_row_2_second = sum_row_3_second;
	632	+
	633	+ // Add luma values to the modifier
	634	+ sum_row_first = _mm_adds_epu16(sum_row_1_first, sum_row_2_first);
	635	+ sum_row_second = _mm_adds_epu16(sum_row_1_second, sum_row_2_second);
	636	+
	637	+ get_sum_16(y_dist + DIST_STRIDE, &sum_row_3_first, &sum_row_3_second);
	638	+
	639	+ sum_row_first = _mm_adds_epu16(sum_row_first, sum_row_3_first);
	640	+ sum_row_second = _mm_adds_epu16(sum_row_second, sum_row_3_second);
	641	+
	642	+ // Add chroma values to the modifier
	643	+ if (ss_y == 0 \|\| h % 2 == 0) {
	644	+ // Only calculate the new chroma distortion if we are at a pixel that
	645	+ // corresponds to a new chroma row
	646	+ read_chroma_dist_row_16(ss_x, u_dist, v_dist, &u_first, &u_second,
	647	+ &v_first, &v_second);
	648	+
	649	+ u_src += uv_src_stride;
	650	+ u_pre += uv_pre_stride;
	651	+ u_dist += DIST_STRIDE;
	652	+ v_src += uv_src_stride;
	653	+ v_pre += uv_pre_stride;
	654	+ v_dist += DIST_STRIDE;
	655	+ }
	656	+
	657	+ sum_row_first = _mm_adds_epu16(sum_row_first, u_first);
	658	+ sum_row_second = _mm_adds_epu16(sum_row_second, u_second);
	659	+ sum_row_first = _mm_adds_epu16(sum_row_first, v_first);
	660	+ sum_row_second = _mm_adds_epu16(sum_row_second, v_second);
	661	+
	662	+ // Get modifier and store result
	663	+ if (blk_fw) {
	664	+ sum_row_first =
	665	+ average_8(sum_row_first, &mul_first, strength, rounding, blk_fw[0]);
	666	+ sum_row_second =
	667	+ average_8(sum_row_second, &mul_second, strength, rounding, blk_fw[1]);
	668	+ } else {
	669	+ average_16(&sum_row_first, &sum_row_second, &mul_first, &mul_second,
	670	+ strength, rounding, weight);
	671	+ }
	672	+ accumulate_and_store_16(sum_row_first, sum_row_second, y_pre, y_count,
	673	+ y_accum);
	674	+
	675	+ y_src += y_src_stride;
	676	+ y_pre += y_pre_stride;
	677	+ y_count += y_pre_stride;
	678	+ y_accum += y_pre_stride;
	679	+ y_dist += DIST_STRIDE;
	680	+ }
	681	+
	682	+ // The last row
	683	+ mul_first = _mm_loadu_si128((const __m128i *)neighbors_first[0]);
	684	+ mul_second = _mm_loadu_si128((const __m128i *)neighbors_second[0]);
	685	+
	686	+ // Shift the rows up
	687	+ sum_row_1_first = sum_row_2_first;
	688	+ sum_row_1_second = sum_row_2_second;
	689	+ sum_row_2_first = sum_row_3_first;
	690	+ sum_row_2_second = sum_row_3_second;
	691	+
	692	+ // Add luma values to the modifier
	693	+ sum_row_first = _mm_adds_epu16(sum_row_1_first, sum_row_2_first);
	694	+ sum_row_second = _mm_adds_epu16(sum_row_1_second, sum_row_2_second);
	695	+
	696	+ // Add chroma values to the modifier
	697	+ if (ss_y == 0) {
	698	+ // Only calculate the new chroma distortion if we are at a pixel that
	699	+ // corresponds to a new chroma row
	700	+ read_chroma_dist_row_16(ss_x, u_dist, v_dist, &u_first, &u_second, &v_first,
	701	+ &v_second);
	702	+ }
	703	+
	704	+ sum_row_first = _mm_adds_epu16(sum_row_first, u_first);
	705	+ sum_row_second = _mm_adds_epu16(sum_row_second, u_second);
	706	+ sum_row_first = _mm_adds_epu16(sum_row_first, v_first);
	707	+ sum_row_second = _mm_adds_epu16(sum_row_second, v_second);
	708	+
	709	+ // Get modifier and store result
	710	+ if (blk_fw) {
	711	+ sum_row_first =
	712	+ average_8(sum_row_first, &mul_first, strength, rounding, blk_fw[0]);
	713	+ sum_row_second =
	714	+ average_8(sum_row_second, &mul_second, strength, rounding, blk_fw[1]);
	715	+ } else {
	716	+ average_16(&sum_row_first, &sum_row_second, &mul_first, &mul_second,
	717	+ strength, rounding, weight);
	718	+ }
	719	+ accumulate_and_store_16(sum_row_first, sum_row_second, y_pre, y_count,
	720	+ y_accum);
	721	+}
	722	+
	723	+// Perform temporal filter for the luma component.
	724	+static void vp9_apply_temporal_filter_luma(
	725	+ const uint8_t y_src, int y_src_stride, const uint8_t y_pre,
	726	+ int y_pre_stride, const uint8_t u_src, const uint8_t v_src,
	727	+ int uv_src_stride, const uint8_t u_pre, const uint8_t v_pre,
	728	+ int uv_pre_stride, unsigned int block_width, unsigned int block_height,
	729	+ int ss_x, int ss_y, int strength, const int *blk_fw, int use_whole_blk,
	730	+ uint32_t y_accum, uint16_t y_count, const uint16_t *y_dist,
	731	+ const uint16_t u_dist, const uint16_t v_dist) {
	732	+ unsigned int blk_col = 0, uv_blk_col = 0;
	733	+ const unsigned int blk_col_step = 16, uv_blk_col_step = 16 >> ss_x;
	734	+ const unsigned int mid_width = block_width >> 1,
	735	+ last_width = block_width - blk_col_step;
	736	+ int top_weight = blk_fw[0],
	737	+ bottom_weight = use_whole_blk ? blk_fw[0] : blk_fw[2];
	738	+ const int16_t const neighbors_first;
	739	+ const int16_t const neighbors_second;
	740	+
	741	+ if (block_width == 16) {
	742	+ // Special Case: The blockwidth is 16 and we are operating on a row of 16
	743	+ // chroma pixels. In this case, we can't use the usualy left-midle-right
	744	+ // pattern. We also don't support splitting now.
	745	+ neighbors_first = LUMA_LEFT_COLUMN_NEIGHBORS;
	746	+ neighbors_second = LUMA_RIGHT_COLUMN_NEIGHBORS;
	747	+ if (use_whole_blk) {
	748	+ vp9_apply_temporal_filter_luma_16(
	749	+ y_src + blk_col, y_src_stride, y_pre + blk_col, y_pre_stride,
	750	+ u_src + uv_blk_col, v_src + uv_blk_col, uv_src_stride,
	751	+ u_pre + uv_blk_col, v_pre + uv_blk_col, uv_pre_stride, 16,
	752	+ block_height, ss_x, ss_y, strength, use_whole_blk, y_accum + blk_col,
	753	+ y_count + blk_col, y_dist + blk_col, u_dist + uv_blk_col,
	754	+ v_dist + uv_blk_col, neighbors_first, neighbors_second, top_weight,
	755	+ bottom_weight, NULL);
	756	+ } else {
	757	+ vp9_apply_temporal_filter_luma_16(
	758	+ y_src + blk_col, y_src_stride, y_pre + blk_col, y_pre_stride,
	759	+ u_src + uv_blk_col, v_src + uv_blk_col, uv_src_stride,
	760	+ u_pre + uv_blk_col, v_pre + uv_blk_col, uv_pre_stride, 16,
	761	+ block_height, ss_x, ss_y, strength, use_whole_blk, y_accum + blk_col,
	762	+ y_count + blk_col, y_dist + blk_col, u_dist + uv_blk_col,
	763	+ v_dist + uv_blk_col, neighbors_first, neighbors_second, 0, 0, blk_fw);
	764	+ }
	765	+
	766	+ return;
	767	+ }
	768	+
	769	+ // Left
	770	+ neighbors_first = LUMA_LEFT_COLUMN_NEIGHBORS;
	771	+ neighbors_second = LUMA_MIDDLE_COLUMN_NEIGHBORS;
	772	+ vp9_apply_temporal_filter_luma_16(
	773	+ y_src + blk_col, y_src_stride, y_pre + blk_col, y_pre_stride,
	774	+ u_src + uv_blk_col, v_src + uv_blk_col, uv_src_stride, u_pre + uv_blk_col,
	775	+ v_pre + uv_blk_col, uv_pre_stride, 16, block_height, ss_x, ss_y, strength,
	776	+ use_whole_blk, y_accum + blk_col, y_count + blk_col, y_dist + blk_col,
	777	+ u_dist + uv_blk_col, v_dist + uv_blk_col, neighbors_first,
	778	+ neighbors_second, top_weight, bottom_weight, NULL);
	779	+
	780	+ blk_col += blk_col_step;
	781	+ uv_blk_col += uv_blk_col_step;
	782	+
	783	+ // Middle First
	784	+ neighbors_first = LUMA_MIDDLE_COLUMN_NEIGHBORS;
	785	+ for (; blk_col < mid_width;
	786	+ blk_col += blk_col_step, uv_blk_col += uv_blk_col_step) {
	787	+ vp9_apply_temporal_filter_luma_16(
	788	+ y_src + blk_col, y_src_stride, y_pre + blk_col, y_pre_stride,
	789	+ u_src + uv_blk_col, v_src + uv_blk_col, uv_src_stride,
	790	+ u_pre + uv_blk_col, v_pre + uv_blk_col, uv_pre_stride, 16, block_height,
	791	+ ss_x, ss_y, strength, use_whole_blk, y_accum + blk_col,
	792	+ y_count + blk_col, y_dist + blk_col, u_dist + uv_blk_col,
	793	+ v_dist + uv_blk_col, neighbors_first, neighbors_second, top_weight,
	794	+ bottom_weight, NULL);
	795	+ }
	796	+
	797	+ if (!use_whole_blk) {
	798	+ top_weight = blk_fw[1];
	799	+ bottom_weight = blk_fw[3];
	800	+ }
	801	+
	802	+ // Middle Second
	803	+ for (; blk_col < last_width;
	804	+ blk_col += blk_col_step, uv_blk_col += uv_blk_col_step) {
	805	+ vp9_apply_temporal_filter_luma_16(
	806	+ y_src + blk_col, y_src_stride, y_pre + blk_col, y_pre_stride,
	807	+ u_src + uv_blk_col, v_src + uv_blk_col, uv_src_stride,
	808	+ u_pre + uv_blk_col, v_pre + uv_blk_col, uv_pre_stride, 16, block_height,
	809	+ ss_x, ss_y, strength, use_whole_blk, y_accum + blk_col,
	810	+ y_count + blk_col, y_dist + blk_col, u_dist + uv_blk_col,
	811	+ v_dist + uv_blk_col, neighbors_first, neighbors_second, top_weight,
	812	+ bottom_weight, NULL);
	813	+ }
	814	+
	815	+ // Right
	816	+ neighbors_second = LUMA_RIGHT_COLUMN_NEIGHBORS;
	817	+ vp9_apply_temporal_filter_luma_16(
	818	+ y_src + blk_col, y_src_stride, y_pre + blk_col, y_pre_stride,
	819	+ u_src + uv_blk_col, v_src + uv_blk_col, uv_src_stride, u_pre + uv_blk_col,
	820	+ v_pre + uv_blk_col, uv_pre_stride, 16, block_height, ss_x, ss_y, strength,
	821	+ use_whole_blk, y_accum + blk_col, y_count + blk_col, y_dist + blk_col,
	822	+ u_dist + uv_blk_col, v_dist + uv_blk_col, neighbors_first,
	823	+ neighbors_second, top_weight, bottom_weight, NULL);
	824	+}
	825	+
	826	+// Apply temporal filter to the chroma components. This performs temporal
	827	+// filtering on a chroma block of 8 X uv_height. If blk_fw is not NULL, use
	828	+// blk_fw as an array of size 4 for the weights for each of the 4 subblocks,
	829	+// else use top_weight for top half, and bottom weight for bottom half.
	830	+static void vp9_apply_temporal_filter_chroma_8(
	831	+ const uint8_t y_src, int y_src_stride, const uint8_t y_pre,
	832	+ int y_pre_stride, const uint8_t u_src, const uint8_t v_src,
	833	+ int uv_src_stride, const uint8_t u_pre, const uint8_t v_pre,
	834	+ int uv_pre_stride, unsigned int uv_block_width,
	835	+ unsigned int uv_block_height, int ss_x, int ss_y, int strength,
	836	+ uint32_t u_accum, uint16_t u_count, uint32_t v_accum, uint16_t v_count,
	837	+ const uint16_t y_dist, const uint16_t u_dist, const uint16_t *v_dist,
	838	+ const int16_t const neighbors, int top_weight, int bottom_weight,
	839	+ const int *blk_fw) {
	840	+ const int rounding = (1 << strength) >> 1;
	841	+ int weight = top_weight;
	842	+
	843	+ __m128i mul;
	844	+
	845	+ __m128i u_sum_row_1, u_sum_row_2, u_sum_row_3;
	846	+ __m128i v_sum_row_1, v_sum_row_2, v_sum_row_3;
	847	+
	848	+ __m128i u_sum_row, v_sum_row;
	849	+
	850	+ (void)uv_block_width;
	851	+
	852	+ // First row
	853	+ mul = _mm_loadu_si128((const __m128i *)neighbors[0]);
	854	+
	855	+ // Add chroma values
	856	+ get_sum_8(u_dist, &u_sum_row_2);
	857	+ get_sum_8(u_dist + DIST_STRIDE, &u_sum_row_3);
	858	+
	859	+ u_sum_row = _mm_adds_epu16(u_sum_row_2, u_sum_row_3);
	860	+
	861	+ get_sum_8(v_dist, &v_sum_row_2);
	862	+ get_sum_8(v_dist + DIST_STRIDE, &v_sum_row_3);
	863	+
	864	+ v_sum_row = _mm_adds_epu16(v_sum_row_2, v_sum_row_3);
	865	+
	866	+ // Add luma values
	867	+ add_luma_dist_to_8_chroma_mod(y_dist, ss_x, ss_y, &u_sum_row, &v_sum_row);
	868	+
	869	+ // Get modifier and store result
	870	+ if (blk_fw) {
	871	+ u_sum_row =
	872	+ average_4_4(u_sum_row, &mul, strength, rounding, blk_fw[0], blk_fw[1]);
	873	+ v_sum_row =
	874	+ average_4_4(v_sum_row, &mul, strength, rounding, blk_fw[0], blk_fw[1]);
	875	+ } else {
	876	+ u_sum_row = average_8(u_sum_row, &mul, strength, rounding, weight);
	877	+ v_sum_row = average_8(v_sum_row, &mul, strength, rounding, weight);
	878	+ }
	879	+ accumulate_and_store_8(u_sum_row, u_pre, u_count, u_accum);
	880	+ accumulate_and_store_8(v_sum_row, v_pre, v_count, v_accum);
	881	+
	882	+ u_src += uv_src_stride;
	883	+ u_pre += uv_pre_stride;
	884	+ u_dist += DIST_STRIDE;
	885	+ v_src += uv_src_stride;
	886	+ v_pre += uv_pre_stride;
	887	+ v_dist += DIST_STRIDE;
	888	+ u_count += uv_pre_stride;
	889	+ u_accum += uv_pre_stride;
	890	+ v_count += uv_pre_stride;
	891	+ v_accum += uv_pre_stride;
	892	+
	893	+ y_src += y_src_stride * (1 + ss_y);
	894	+ y_pre += y_pre_stride * (1 + ss_y);
	895	+ y_dist += DIST_STRIDE * (1 + ss_y);
	896	+
	897	+ // Then all the rows except the last one
	898	+ mul = _mm_loadu_si128((const __m128i *)neighbors[1]);
	899	+
	900	+ for (unsigned int h = 1; h < uv_block_height - 1; ++h) {
	901	+ // Move the weight pointer to the bottom half of the blocks
	902	+ if (h == uv_block_height / 2) {
	903	+ if (blk_fw) {
	904	+ blk_fw += 2;
	905	+ } else {
	906	+ weight = bottom_weight;
	907	+ }
	908	+ }
	909	+
	910	+ // Shift the rows up
	911	+ u_sum_row_1 = u_sum_row_2;
	912	+ u_sum_row_2 = u_sum_row_3;
	913	+
	914	+ v_sum_row_1 = v_sum_row_2;
	915	+ v_sum_row_2 = v_sum_row_3;
	916	+
	917	+ // Add chroma values
	918	+ u_sum_row = _mm_adds_epu16(u_sum_row_1, u_sum_row_2);
	919	+ get_sum_8(u_dist + DIST_STRIDE, &u_sum_row_3);
	920	+ u_sum_row = _mm_adds_epu16(u_sum_row, u_sum_row_3);
	921	+
	922	+ v_sum_row = _mm_adds_epu16(v_sum_row_1, v_sum_row_2);
	923	+ get_sum_8(v_dist + DIST_STRIDE, &v_sum_row_3);
	924	+ v_sum_row = _mm_adds_epu16(v_sum_row, v_sum_row_3);
	925	+
	926	+ // Add luma values
	927	+ add_luma_dist_to_8_chroma_mod(y_dist, ss_x, ss_y, &u_sum_row, &v_sum_row);
	928	+
	929	+ // Get modifier and store result
	930	+ if (blk_fw) {
	931	+ u_sum_row = average_4_4(u_sum_row, &mul, strength, rounding, blk_fw[0],
	932	+ blk_fw[1]);
	933	+ v_sum_row = average_4_4(v_sum_row, &mul, strength, rounding, blk_fw[0],
	934	+ blk_fw[1]);
	935	+ } else {
	936	+ u_sum_row = average_8(u_sum_row, &mul, strength, rounding, weight);
	937	+ v_sum_row = average_8(v_sum_row, &mul, strength, rounding, weight);
	938	+ }
	939	+
	940	+ accumulate_and_store_8(u_sum_row, u_pre, u_count, u_accum);
	941	+ accumulate_and_store_8(v_sum_row, v_pre, v_count, v_accum);
	942	+
	943	+ u_src += uv_src_stride;
	944	+ u_pre += uv_pre_stride;
	945	+ u_dist += DIST_STRIDE;
	946	+ v_src += uv_src_stride;
	947	+ v_pre += uv_pre_stride;
	948	+ v_dist += DIST_STRIDE;
	949	+ u_count += uv_pre_stride;
	950	+ u_accum += uv_pre_stride;
	951	+ v_count += uv_pre_stride;
	952	+ v_accum += uv_pre_stride;
	953	+
	954	+ y_src += y_src_stride * (1 + ss_y);
	955	+ y_pre += y_pre_stride * (1 + ss_y);
	956	+ y_dist += DIST_STRIDE * (1 + ss_y);
	957	+ }
	958	+
	959	+ // The last row
	960	+ mul = _mm_loadu_si128((const __m128i *)neighbors[0]);
	961	+
	962	+ // Shift the rows up
	963	+ u_sum_row_1 = u_sum_row_2;
	964	+ u_sum_row_2 = u_sum_row_3;
	965	+
	966	+ v_sum_row_1 = v_sum_row_2;
	967	+ v_sum_row_2 = v_sum_row_3;
	968	+
	969	+ // Add chroma values
	970	+ u_sum_row = _mm_adds_epu8(u_sum_row_1, u_sum_row_2);
	971	+ v_sum_row = _mm_adds_epu8(v_sum_row_1, v_sum_row_2);
	972	+
	973	+ // Add luma values
	974	+ add_luma_dist_to_8_chroma_mod(y_dist, ss_x, ss_y, &u_sum_row, &v_sum_row);
	975	+
	976	+ // Get modifier and store result
	977	+ if (blk_fw) {
	978	+ u_sum_row =
	979	+ average_4_4(u_sum_row, &mul, strength, rounding, blk_fw[0], blk_fw[1]);
	980	+ v_sum_row =
	981	+ average_4_4(v_sum_row, &mul, strength, rounding, blk_fw[0], blk_fw[1]);
	982	+ } else {
	983	+ u_sum_row = average_8(u_sum_row, &mul, strength, rounding, weight);
	984	+ v_sum_row = average_8(v_sum_row, &mul, strength, rounding, weight);
	985	+ }
	986	+
	987	+ accumulate_and_store_8(u_sum_row, u_pre, u_count, u_accum);
	988	+ accumulate_and_store_8(v_sum_row, v_pre, v_count, v_accum);
	989	+}
	990	+
	991	+// Perform temporal filter for the chroma components.
	992	+static void vp9_apply_temporal_filter_chroma(
	993	+ const uint8_t y_src, int y_src_stride, const uint8_t y_pre,
	994	+ int y_pre_stride, const uint8_t u_src, const uint8_t v_src,
	995	+ int uv_src_stride, const uint8_t u_pre, const uint8_t v_pre,
	996	+ int uv_pre_stride, unsigned int block_width, unsigned int block_height,
	997	+ int ss_x, int ss_y, int strength, const int *blk_fw, int use_whole_blk,
	998	+ uint32_t u_accum, uint16_t u_count, uint32_t v_accum, uint16_t v_count,
	999	+ const uint16_t y_dist, const uint16_t u_dist, const uint16_t *v_dist) {
	1000	+ const unsigned int uv_width = block_width >> ss_x,
	1001	+ uv_height = block_height >> ss_y;
	1002	+
	1003	+ unsigned int blk_col = 0, uv_blk_col = 0;
	1004	+ const unsigned int uv_blk_col_step = 8, blk_col_step = 8 << ss_x;
	1005	+ const unsigned int uv_mid_width = uv_width >> 1,
	1006	+ uv_last_width = uv_width - uv_blk_col_step;
	1007	+ int top_weight = blk_fw[0],
	1008	+ bottom_weight = use_whole_blk ? blk_fw[0] : blk_fw[2];
	1009	+ const int16_t const neighbors;
	1010	+
	1011	+ if (uv_width == 8) {
	1012	+ // Special Case: We are subsampling in x direction on a 16x16 block. Since
	1013	+ // we are operating on a row of 8 chroma pixels, we can't use the usual
	1014	+ // left-middle-right pattern.
	1015	+ assert(ss_x);
	1016	+
	1017	+ if (ss_y) {
	1018	+ neighbors = CHROMA_DOUBLE_SS_SINGLE_COLUMN_NEIGHBORS;
	1019	+ } else {
	1020	+ neighbors = CHROMA_SINGLE_SS_SINGLE_COLUMN_NEIGHBORS;
	1021	+ }
	1022	+
	1023	+ if (use_whole_blk) {
	1024	+ vp9_apply_temporal_filter_chroma_8(
	1025	+ y_src + blk_col, y_src_stride, y_pre + blk_col, y_pre_stride,
	1026	+ u_src + uv_blk_col, v_src + uv_blk_col, uv_src_stride,
	1027	+ u_pre + uv_blk_col, v_pre + uv_blk_col, uv_pre_stride, uv_width,
	1028	+ uv_height, ss_x, ss_y, strength, u_accum + uv_blk_col,
	1029	+ u_count + uv_blk_col, v_accum + uv_blk_col, v_count + uv_blk_col,
	1030	+ y_dist + blk_col, u_dist + uv_blk_col, v_dist + uv_blk_col, neighbors,
	1031	+ top_weight, bottom_weight, NULL);
	1032	+ } else {
	1033	+ vp9_apply_temporal_filter_chroma_8(
	1034	+ y_src + blk_col, y_src_stride, y_pre + blk_col, y_pre_stride,
	1035	+ u_src + uv_blk_col, v_src + uv_blk_col, uv_src_stride,
	1036	+ u_pre + uv_blk_col, v_pre + uv_blk_col, uv_pre_stride, uv_width,
	1037	+ uv_height, ss_x, ss_y, strength, u_accum + uv_blk_col,
	1038	+ u_count + uv_blk_col, v_accum + uv_blk_col, v_count + uv_blk_col,
	1039	+ y_dist + blk_col, u_dist + uv_blk_col, v_dist + uv_blk_col, neighbors,
	1040	+ 0, 0, blk_fw);
	1041	+ }
	1042	+
	1043	+ return;
	1044	+ }
	1045	+
	1046	+ // Left
	1047	+ if (ss_x && ss_y) {
	1048	+ neighbors = CHROMA_DOUBLE_SS_LEFT_COLUMN_NEIGHBORS;
	1049	+ } else if (ss_x \|\| ss_y) {
	1050	+ neighbors = CHROMA_SINGLE_SS_LEFT_COLUMN_NEIGHBORS;
	1051	+ } else {
	1052	+ neighbors = CHROMA_NO_SS_LEFT_COLUMN_NEIGHBORS;
	1053	+ }
	1054	+
	1055	+ vp9_apply_temporal_filter_chroma_8(
	1056	+ y_src + blk_col, y_src_stride, y_pre + blk_col, y_pre_stride,
	1057	+ u_src + uv_blk_col, v_src + uv_blk_col, uv_src_stride, u_pre + uv_blk_col,
	1058	+ v_pre + uv_blk_col, uv_pre_stride, uv_width, uv_height, ss_x, ss_y,
	1059	+ strength, u_accum + uv_blk_col, u_count + uv_blk_col,
	1060	+ v_accum + uv_blk_col, v_count + uv_blk_col, y_dist + blk_col,
	1061	+ u_dist + uv_blk_col, v_dist + uv_blk_col, neighbors, top_weight,
	1062	+ bottom_weight, NULL);
	1063	+
	1064	+ blk_col += blk_col_step;
	1065	+ uv_blk_col += uv_blk_col_step;
	1066	+
	1067	+ // Middle First
	1068	+ if (ss_x && ss_y) {
	1069	+ neighbors = CHROMA_DOUBLE_SS_MIDDLE_COLUMN_NEIGHBORS;
	1070	+ } else if (ss_x \|\| ss_y) {
	1071	+ neighbors = CHROMA_SINGLE_SS_MIDDLE_COLUMN_NEIGHBORS;
	1072	+ } else {
	1073	+ neighbors = CHROMA_NO_SS_MIDDLE_COLUMN_NEIGHBORS;
	1074	+ }
	1075	+
	1076	+ for (; uv_blk_col < uv_mid_width;
	1077	+ blk_col += blk_col_step, uv_blk_col += uv_blk_col_step) {
	1078	+ vp9_apply_temporal_filter_chroma_8(
	1079	+ y_src + blk_col, y_src_stride, y_pre + blk_col, y_pre_stride,
	1080	+ u_src + uv_blk_col, v_src + uv_blk_col, uv_src_stride,
	1081	+ u_pre + uv_blk_col, v_pre + uv_blk_col, uv_pre_stride, uv_width,
	1082	+ uv_height, ss_x, ss_y, strength, u_accum + uv_blk_col,
	1083	+ u_count + uv_blk_col, v_accum + uv_blk_col, v_count + uv_blk_col,
	1084	+ y_dist + blk_col, u_dist + uv_blk_col, v_dist + uv_blk_col, neighbors,
	1085	+ top_weight, bottom_weight, NULL);
	1086	+ }
	1087	+
	1088	+ if (!use_whole_blk) {
	1089	+ top_weight = blk_fw[1];
	1090	+ bottom_weight = blk_fw[3];
	1091	+ }
	1092	+
	1093	+ // Middle Second
	1094	+ for (; uv_blk_col < uv_last_width;
	1095	+ blk_col += blk_col_step, uv_blk_col += uv_blk_col_step) {
	1096	+ vp9_apply_temporal_filter_chroma_8(
	1097	+ y_src + blk_col, y_src_stride, y_pre + blk_col, y_pre_stride,
	1098	+ u_src + uv_blk_col, v_src + uv_blk_col, uv_src_stride,
	1099	+ u_pre + uv_blk_col, v_pre + uv_blk_col, uv_pre_stride, uv_width,
	1100	+ uv_height, ss_x, ss_y, strength, u_accum + uv_blk_col,
	1101	+ u_count + uv_blk_col, v_accum + uv_blk_col, v_count + uv_blk_col,
	1102	+ y_dist + blk_col, u_dist + uv_blk_col, v_dist + uv_blk_col, neighbors,
	1103	+ top_weight, bottom_weight, NULL);
	1104	+ }
	1105	+
	1106	+ // Right
	1107	+ if (ss_x && ss_y) {
	1108	+ neighbors = CHROMA_DOUBLE_SS_RIGHT_COLUMN_NEIGHBORS;
	1109	+ } else if (ss_x \|\| ss_y) {
	1110	+ neighbors = CHROMA_SINGLE_SS_RIGHT_COLUMN_NEIGHBORS;
	1111	+ } else {
	1112	+ neighbors = CHROMA_NO_SS_RIGHT_COLUMN_NEIGHBORS;
	1113	+ }
	1114	+
	1115	+ vp9_apply_temporal_filter_chroma_8(
	1116	+ y_src + blk_col, y_src_stride, y_pre + blk_col, y_pre_stride,
	1117	+ u_src + uv_blk_col, v_src + uv_blk_col, uv_src_stride, u_pre + uv_blk_col,
	1118	+ v_pre + uv_blk_col, uv_pre_stride, uv_width, uv_height, ss_x, ss_y,
	1119	+ strength, u_accum + uv_blk_col, u_count + uv_blk_col,
	1120	+ v_accum + uv_blk_col, v_count + uv_blk_col, y_dist + blk_col,
	1121	+ u_dist + uv_blk_col, v_dist + uv_blk_col, neighbors, top_weight,
	1122	+ bottom_weight, NULL);
	1123	+}
	1124	+
	1125	+void vp9_apply_temporal_filter_sse4_1(
	1126	+ const uint8_t y_src, int y_src_stride, const uint8_t y_pre,
	1127	+ int y_pre_stride, const uint8_t u_src, const uint8_t v_src,
	1128	+ int uv_src_stride, const uint8_t u_pre, const uint8_t v_pre,
	1129	+ int uv_pre_stride, unsigned int block_width, unsigned int block_height,
	1130	+ int ss_x, int ss_y, int strength, const int *const blk_fw,
	1131	+ int use_whole_blk, uint32_t y_accum, uint16_t y_count, uint32_t *u_accum,
	1132	+ uint16_t u_count, uint32_t v_accum, uint16_t *v_count) {
	1133	+ const unsigned int chroma_height = block_height >> ss_y,
	1134	+ chroma_width = block_width >> ss_x;
	1135	+
	1136	+ DECLARE_ALIGNED(16, uint16_t, y_dist[BH * DIST_STRIDE]) = { 0 };
	1137	+ DECLARE_ALIGNED(16, uint16_t, u_dist[BH * DIST_STRIDE]) = { 0 };
	1138	+ DECLARE_ALIGNED(16, uint16_t, v_dist[BH * DIST_STRIDE]) = { 0 };
	1139	+ const int *blk_fw_ptr = blk_fw;
	1140	+
	1141	+ uint16_t y_dist_ptr = y_dist + 1, u_dist_ptr = u_dist + 1,
	1142	+ *v_dist_ptr = v_dist + 1;
	1143	+ const uint8_t y_src_ptr = y_src, u_src_ptr = u_src, *v_src_ptr = v_src;
	1144	+ const uint8_t y_pre_ptr = y_pre, u_pre_ptr = u_pre, *v_pre_ptr = v_pre;
	1145	+
	1146	+ assert(block_width <= BW && "block width too large");
	1147	+ assert(block_height <= BH && "block height too large");
	1148	+ assert(block_width % 16 == 0 && "block width must be multiple of 16");
	1149	+ assert(block_height % 2 == 0 && "block height must be even");
	1150	+ assert((ss_x == 0 \|\| ss_x == 1) && (ss_y == 0 \|\| ss_y == 1) &&
	1151	+ "invalid chroma subsampling");
	1152	+ assert(strength >= 0 && strength <= 6 && "invalid temporal filter strength");
	1153	+ assert(blk_fw[0] >= 0 && "filter weight must be positive");
	1154	+ assert(
	1155	+ (use_whole_blk \|\| (blk_fw[1] >= 0 && blk_fw[2] >= 0 && blk_fw[3] >= 0)) &&
	1156	+ "subblock filter weight must be positive");
	1157	+ assert(blk_fw[0] <= 2 && "sublock filter weight must be less than 2");
	1158	+ assert(
	1159	+ (use_whole_blk \|\| (blk_fw[1] <= 2 && blk_fw[2] <= 2 && blk_fw[3] <= 2)) &&
	1160	+ "subblock filter weight must be less than 2");
	1161	+
	1162	+ // Precompute the difference sqaured
	1163	+ for (unsigned int row = 0; row < block_height; row++) {
	1164	+ for (unsigned int blk_col = 0; blk_col < block_width; blk_col += 16) {
	1165	+ store_dist_16(y_src_ptr + blk_col, y_pre_ptr + blk_col,
	1166	+ y_dist_ptr + blk_col);
	1167	+ }
	1168	+ y_src_ptr += y_src_stride;
	1169	+ y_pre_ptr += y_pre_stride;
	1170	+ y_dist_ptr += DIST_STRIDE;
	1171	+ }
	1172	+
	1173	+ for (unsigned int row = 0; row < chroma_height; row++) {
	1174	+ for (unsigned int blk_col = 0; blk_col < chroma_width; blk_col += 8) {
	1175	+ store_dist_8(u_src_ptr + blk_col, u_pre_ptr + blk_col,
	1176	+ u_dist_ptr + blk_col);
	1177	+ store_dist_8(v_src_ptr + blk_col, v_pre_ptr + blk_col,
	1178	+ v_dist_ptr + blk_col);
	1179	+ }
	1180	+
	1181	+ u_src_ptr += uv_src_stride;
	1182	+ u_pre_ptr += uv_pre_stride;
	1183	+ u_dist_ptr += DIST_STRIDE;
	1184	+ v_src_ptr += uv_src_stride;
	1185	+ v_pre_ptr += uv_pre_stride;
	1186	+ v_dist_ptr += DIST_STRIDE;
	1187	+ }
	1188	+
	1189	+ y_dist_ptr = y_dist + 1;
	1190	+ u_dist_ptr = u_dist + 1;
	1191	+ v_dist_ptr = v_dist + 1;
	1192	+
	1193	+ vp9_apply_temporal_filter_luma(
	1194	+ y_src, y_src_stride, y_pre, y_pre_stride, u_src, v_src, uv_src_stride,
	1195	+ u_pre, v_pre, uv_pre_stride, block_width, block_height, ss_x, ss_y,
	1196	+ strength, blk_fw_ptr, use_whole_blk, y_accum, y_count, y_dist_ptr,
	1197	+ u_dist_ptr, v_dist_ptr);
	1198	+
	1199	+ vp9_apply_temporal_filter_chroma(
	1200	+ y_src, y_src_stride, y_pre, y_pre_stride, u_src, v_src, uv_src_stride,
	1201	+ u_pre, v_pre, uv_pre_stride, block_width, block_height, ss_x, ss_y,
	1202	+ strength, blk_fw_ptr, use_whole_blk, u_accum, u_count, v_accum, v_count,
	1203	+ y_dist_ptr, u_dist_ptr, v_dist_ptr);
	1204	+}

vp9/vp9cx.mk+2−0

@@ -103,6 +103,7 @@ VP9_CX_SRCS-yes += encoder/vp9_mbgraph.c · 102 unmodified lines
103	103	VP9_CX_SRCS-yes += encoder/vp9_mbgraph.h
104	104
105	105	VP9_CX_SRCS-$(HAVE_SSE4_1) += encoder/x86/temporal_filter_sse4.c
	106	+VP9_CX_SRCS-$(HAVE_SSE4_1) += encoder/x86/temporal_filter_constants.h
106	107
107	108	VP9_CX_SRCS-$(HAVE_SSE2) += encoder/x86/vp9_quantize_sse2.c
108	109	VP9_CX_SRCS-$(HAVE_AVX2) += encoder/x86/vp9_quantize_avx2.c
@@ -149,5 +150,6 @@ VP9_CX_SRCS_REMOVE-$(CONFIG_REALTIME_ONLY) += encoder/vp9_firstpass.c · 40 unmodified lines
149	150	VP9_CX_SRCS_REMOVE-$(CONFIG_REALTIME_ONLY) += encoder/vp9_mbgraph.c
150	151	VP9_CX_SRCS_REMOVE-$(CONFIG_REALTIME_ONLY) += encoder/vp9_temporal_filter.c
151	152	VP9_CX_SRCS_REMOVE-$(CONFIG_REALTIME_ONLY) += encoder/x86/temporal_filter_sse4.c
	153	+VP9_CX_SRCS_REMOVE-$(CONFIG_REALTIME_ONLY) += encoder/x86/temporal_filter_constants.h
152	154
153	155	VP9_CX_SRCS-yes := $(filter-out $(VP9_CX_SRCS_REMOVE-yes),$(VP9_CX_SRCS-yes))

Add SSE4 version of new apply_temporal_filter

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code