Add SSE4 version of new apply_temporal_filter

This adds a preliminary version of vp9_apply_temporal_filter in SSE4.1. This patch merely adds the function and does not enable it yet. Speed Up: | ss_x=1 | ss_x=0 | ss_y=1 | 19.80X | 19.04X | ss_y=0 | 21.09X | 20.21X | BUG=webm:1591

chiyotsai committed 7 years ago

Commit b580f761 parent e3210937 files · +1131 −48Browse files

Unified · Split

7 files

test/yuv_temporal_filter_test.cc +14 −2
vp9/common/vp9_rtcd_defs.pl +1 −0
vp9/encoder/vp9_temporal_filter.c +9 −1
vp9/encoder/vp9_temporal_filter.h +1 −1
vp9/encoder/x86/temporal_filter_constants.h +232 −0
vp9/encoder/x86/temporal_filter_sse4.c +872 −44
vp9/vp9cx.mk +2 −0

test/yuv_temporal_filter_test.cc+14−2

 int GetModIndex(int sum_dist, int index, int rounding, int strength,
                 int filter_weight) {
-  int mod = (sum_dist * 3) / index;
+  unsigned int index_mult[14] = {
+    0, 0, 0, 0, 49152, 39322, 32768, 28087, 24576, 21846, 19661, 17874, 0, 15124
+  };
+  assert(index >= 0 && index <= 13);
+  assert(index_mult[index] != 0);
+  int mod = (clamp(sum_dist, 0, UINT16_MAX) * index_mult[index]) >> 16;
   mod += rounding;
   mod >>= strength;
 }
 INSTANTIATE_TEST_CASE_P(C, YUVTemporalFilterTest,
-                        ::testing::Values(&vp9_apply_temporal_filter));
+                        ::testing::Values(&vp9_apply_temporal_filter_c));
+#if HAVE_SSE4_1
+INSTANTIATE_TEST_CASE_P(SSE4, YUVTemporalFilterTest,
+                        ::testing::Values(&vp9_apply_temporal_filter_sse4_1));
+#endif  // HAVE_SSE4_1
 }  // namespace

vp9/common/vp9_rtcd_defs.pl+1−0

@@ -188,6 +188,7 @@ add_proto qw/void vp9_temporal_filter_apply/, "const uint8_t *frame1, unsigned i · 187 unmodified lines
188	specialize qw/vp9_temporal_filter_apply sse4_1/;	188	specialize qw/vp9_temporal_filter_apply sse4_1/;
189		189
190	add_proto qw/void vp9_apply_temporal_filter/, "const uint8_t y_src, int y_src_stride, const uint8_t y_pre, int y_pre_stride, const uint8_t u_src, const uint8_t v_src, int uv_src_stride, const uint8_t u_pre, const uint8_t v_pre, int uv_pre_stride, unsigned int block_width, unsigned int block_height, int ss_x, int ss_y, int strength, const int const blk_fw, int use_32x32, uint32_t y_accumulator, uint16_t y_count, uint32_t u_accumulator, uint16_t u_count, uint32_t v_accumulator, uint16_t *v_count";	190	add_proto qw/void vp9_apply_temporal_filter/, "const uint8_t y_src, int y_src_stride, const uint8_t y_pre, int y_pre_stride, const uint8_t u_src, const uint8_t v_src, int uv_src_stride, const uint8_t u_pre, const uint8_t v_pre, int uv_pre_stride, unsigned int block_width, unsigned int block_height, int ss_x, int ss_y, int strength, const int const blk_fw, int use_32x32, uint32_t y_accumulator, uint16_t y_count, uint32_t u_accumulator, uint16_t u_count, uint32_t v_accumulator, uint16_t *v_count";
		191	specialize qw/vp9_apply_temporal_filter sse4_1/;
191	}	192	}
192		193
193	if (vpx_config("CONFIG_VP9_HIGHBITDEPTH") eq "yes") {	194	if (vpx_config("CONFIG_VP9_HIGHBITDEPTH") eq "yes") {

vp9/encoder/vp9_temporal_filter.c+9−1

 #include "vpx_scale/vpx_scale.h"
 static int fixed_divide[512];
+static unsigned int index_mult[14] = {
+  0, 0, 0, 0, 49152, 39322, 32768, 28087, 24576, 21846, 19661, 17874, 0, 15124
+};
 static void temporal_filter_predictors_mb_c(
     MACROBLOCKD *xd, uint8_t *y_mb_ptr, uint8_t *u_mb_ptr, uint8_t *v_mb_ptr,
 static INLINE int mod_index(int sum_dist, int index, int rounding, int strength,
                             int filter_weight) {
-  int mod = (sum_dist * 3) / index;
+  int mod;
+  assert(index >= 0 && index <= 13);
+  assert(index_mult[index] != 0);
+  mod = (clamp(sum_dist, 0, UINT16_MAX) * index_mult[index]) >> 16;
   mod += rounding;
   mod >>= strength;

vp9/encoder/vp9_temporal_filter.h+1−1

@@ -24,7 +24,7 @@ static const MV kZeroMv = { 0, 0 }; · 23 unmodified lines
24	#define BH_LOG2 5	24	#define BH_LOG2 5
25	#define BW 32	25	#define BW 32
26	#define BW_LOG2 5	26	#define BW_LOG2 5
27	#define BLK_PELS 1024 // Pixels in the block	27	#define BLK_PELS ((BH) * (BW)) // Pixels in the block
28	#define TF_SHIFT 2	28	#define TF_SHIFT 2
29	#define TF_ROUND 3	29	#define TF_ROUND 3
30	#define THR_SHIFT 2	30	#define THR_SHIFT 2

vp9/encoder/x86/temporal_filter_constants.h+232−0

@@ -0,0 +1,232 @@
	1	/*
	2	* Copyright (c) 2019 The WebM project authors. All Rights Reserved.
	3	*
	4	* Use of this source code is governed by a BSD-style license
	5	* that can be found in the LICENSE file in the root of the source
	6	* tree. An additional intellectual property rights grant can be found
	7	* in the file PATENTS. All contributing project authors may
	8	* be found in the AUTHORS file in the root of the source tree.
	9	*/
	10
	11	#include "./vpx_config.h"
	12
	13	// Division using multiplication and shifting. The C implementation does:
	14	// modifier *= 3;
	15	// modifier /= index;
	16	// where 'modifier' is a set of summed values and 'index' is the number of
	17	// summed values.
	18	//
	19	// This equation works out to (m * 3) / i which reduces to:
	20	// m * 3/4
	21	// m * 1/2
	22	// m * 1/3
	23	//
	24	// By pairing the multiply with a down shift by 16 (_mm_mulhi_epu16):
	25	// m * C / 65536
	26	// we can create a C to replicate the division.
	27	//
	28	// m * 49152 / 65536 = m * 3/4
	29	// m * 32758 / 65536 = m * 1/2
	30	// m * 21846 / 65536 = m * 0.3333
	31	//
	32	// These are loaded using an instruction expecting int16_t values but are used
	33	// with _mm_mulhi_epu16(), which treats them as unsigned.
	34	#define NEIGHBOR_CONSTANT_4 (int16_t)49152
	35	#define NEIGHBOR_CONSTANT_5 (int16_t)39322
	36	#define NEIGHBOR_CONSTANT_6 (int16_t)32768
	37	#define NEIGHBOR_CONSTANT_7 (int16_t)28087
	38	#define NEIGHBOR_CONSTANT_8 (int16_t)24576
	39	#define NEIGHBOR_CONSTANT_9 (int16_t)21846
	40	#define NEIGHBOR_CONSTANT_10 (int16_t)19661
	41	#define NEIGHBOR_CONSTANT_11 (int16_t)17874
	42	#define NEIGHBOR_CONSTANT_13 (int16_t)15124
	43
	44	DECLARE_ALIGNED(16, static const int16_t, LEFT_CORNER_NEIGHBORS_PLUS_1[8]) = {
	45	NEIGHBOR_CONSTANT_5, NEIGHBOR_CONSTANT_7, NEIGHBOR_CONSTANT_7,
	46	NEIGHBOR_CONSTANT_7, NEIGHBOR_CONSTANT_7, NEIGHBOR_CONSTANT_7,
	47	NEIGHBOR_CONSTANT_7, NEIGHBOR_CONSTANT_7
	48	};
	49
	50	DECLARE_ALIGNED(16, static const int16_t, RIGHT_CORNER_NEIGHBORS_PLUS_1[8]) = {
	51	NEIGHBOR_CONSTANT_7, NEIGHBOR_CONSTANT_7, NEIGHBOR_CONSTANT_7,
	52	NEIGHBOR_CONSTANT_7, NEIGHBOR_CONSTANT_7, NEIGHBOR_CONSTANT_7,
	53	NEIGHBOR_CONSTANT_7, NEIGHBOR_CONSTANT_5
	54	};
	55
	56	DECLARE_ALIGNED(16, static const int16_t, LEFT_EDGE_NEIGHBORS_PLUS_1[8]) = {
	57	NEIGHBOR_CONSTANT_7, NEIGHBOR_CONSTANT_10, NEIGHBOR_CONSTANT_10,
	58	NEIGHBOR_CONSTANT_10, NEIGHBOR_CONSTANT_10, NEIGHBOR_CONSTANT_10,
	59	NEIGHBOR_CONSTANT_10, NEIGHBOR_CONSTANT_10
	60	};
	61
	62	DECLARE_ALIGNED(16, static const int16_t, RIGHT_EDGE_NEIGHBORS_PLUS_1[8]) = {
	63	NEIGHBOR_CONSTANT_10, NEIGHBOR_CONSTANT_10, NEIGHBOR_CONSTANT_10,
	64	NEIGHBOR_CONSTANT_10, NEIGHBOR_CONSTANT_10, NEIGHBOR_CONSTANT_10,
	65	NEIGHBOR_CONSTANT_10, NEIGHBOR_CONSTANT_7
	66	};
	67
	68	DECLARE_ALIGNED(16, static const int16_t, MIDDLE_EDGE_NEIGHBORS_PLUS_1[8]) = {
	69	NEIGHBOR_CONSTANT_7, NEIGHBOR_CONSTANT_7, NEIGHBOR_CONSTANT_7,
	70	NEIGHBOR_CONSTANT_7, NEIGHBOR_CONSTANT_7, NEIGHBOR_CONSTANT_7,
	71	NEIGHBOR_CONSTANT_7, NEIGHBOR_CONSTANT_7
	72	};
	73
	74	DECLARE_ALIGNED(16, static const int16_t, MIDDLE_CENTER_NEIGHBORS_PLUS_1[8]) = {
	75	NEIGHBOR_CONSTANT_10, NEIGHBOR_CONSTANT_10, NEIGHBOR_CONSTANT_10,
	76	NEIGHBOR_CONSTANT_10, NEIGHBOR_CONSTANT_10, NEIGHBOR_CONSTANT_10,
	77	NEIGHBOR_CONSTANT_10, NEIGHBOR_CONSTANT_10
	78	};
	79
	80	DECLARE_ALIGNED(16, static const int16_t, LEFT_CORNER_NEIGHBORS_PLUS_2[8]) = {
	81	NEIGHBOR_CONSTANT_6, NEIGHBOR_CONSTANT_8, NEIGHBOR_CONSTANT_8,
	82	NEIGHBOR_CONSTANT_8, NEIGHBOR_CONSTANT_8, NEIGHBOR_CONSTANT_8,
	83	NEIGHBOR_CONSTANT_8, NEIGHBOR_CONSTANT_8
	84	};
	85
	86	DECLARE_ALIGNED(16, static const int16_t, RIGHT_CORNER_NEIGHBORS_PLUS_2[8]) = {
	87	NEIGHBOR_CONSTANT_8, NEIGHBOR_CONSTANT_8, NEIGHBOR_CONSTANT_8,
	88	NEIGHBOR_CONSTANT_8, NEIGHBOR_CONSTANT_8, NEIGHBOR_CONSTANT_8,
	89	NEIGHBOR_CONSTANT_8, NEIGHBOR_CONSTANT_6
	90	};
	91
	92	DECLARE_ALIGNED(16, static const int16_t, LEFT_EDGE_NEIGHBORS_PLUS_2[8]) = {
	93	NEIGHBOR_CONSTANT_8, NEIGHBOR_CONSTANT_11, NEIGHBOR_CONSTANT_11,
	94	NEIGHBOR_CONSTANT_11, NEIGHBOR_CONSTANT_11, NEIGHBOR_CONSTANT_11,
	95	NEIGHBOR_CONSTANT_11, NEIGHBOR_CONSTANT_11
	96	};
	97
	98	DECLARE_ALIGNED(16, static const int16_t, RIGHT_EDGE_NEIGHBORS_PLUS_2[8]) = {
	99	NEIGHBOR_CONSTANT_11, NEIGHBOR_CONSTANT_11, NEIGHBOR_CONSTANT_11,
	100	NEIGHBOR_CONSTANT_11, NEIGHBOR_CONSTANT_11, NEIGHBOR_CONSTANT_11,
	101	NEIGHBOR_CONSTANT_11, NEIGHBOR_CONSTANT_8
	102	};
	103
	104	DECLARE_ALIGNED(16, static const int16_t, MIDDLE_EDGE_NEIGHBORS_PLUS_2[8]) = {
	105	NEIGHBOR_CONSTANT_8, NEIGHBOR_CONSTANT_8, NEIGHBOR_CONSTANT_8,
	106	NEIGHBOR_CONSTANT_8, NEIGHBOR_CONSTANT_8, NEIGHBOR_CONSTANT_8,
	107	NEIGHBOR_CONSTANT_8, NEIGHBOR_CONSTANT_8
	108	};
	109
	110	DECLARE_ALIGNED(16, static const int16_t, MIDDLE_CENTER_NEIGHBORS_PLUS_2[8]) = {
	111	NEIGHBOR_CONSTANT_11, NEIGHBOR_CONSTANT_11, NEIGHBOR_CONSTANT_11,
	112	NEIGHBOR_CONSTANT_11, NEIGHBOR_CONSTANT_11, NEIGHBOR_CONSTANT_11,
	113	NEIGHBOR_CONSTANT_11, NEIGHBOR_CONSTANT_11
	114	};
	115
	116	DECLARE_ALIGNED(16, static const int16_t, TWO_CORNER_NEIGHBORS_PLUS_2[8]) = {
	117	NEIGHBOR_CONSTANT_6, NEIGHBOR_CONSTANT_8, NEIGHBOR_CONSTANT_8,
	118	NEIGHBOR_CONSTANT_8, NEIGHBOR_CONSTANT_8, NEIGHBOR_CONSTANT_8,
	119	NEIGHBOR_CONSTANT_8, NEIGHBOR_CONSTANT_6
	120	};
	121
	122	DECLARE_ALIGNED(16, static const int16_t, TWO_EDGE_NEIGHBORS_PLUS_2[8]) = {
	123	NEIGHBOR_CONSTANT_8, NEIGHBOR_CONSTANT_11, NEIGHBOR_CONSTANT_11,
	124	NEIGHBOR_CONSTANT_11, NEIGHBOR_CONSTANT_11, NEIGHBOR_CONSTANT_11,
	125	NEIGHBOR_CONSTANT_11, NEIGHBOR_CONSTANT_8
	126	};
	127
	128	DECLARE_ALIGNED(16, static const int16_t, LEFT_CORNER_NEIGHBORS_PLUS_4[8]) = {
	129	NEIGHBOR_CONSTANT_8, NEIGHBOR_CONSTANT_10, NEIGHBOR_CONSTANT_10,
	130	NEIGHBOR_CONSTANT_10, NEIGHBOR_CONSTANT_10, NEIGHBOR_CONSTANT_10,
	131	NEIGHBOR_CONSTANT_10, NEIGHBOR_CONSTANT_10
	132	};
	133
	134	DECLARE_ALIGNED(16, static const int16_t, RIGHT_CORNER_NEIGHBORS_PLUS_4[8]) = {
	135	NEIGHBOR_CONSTANT_10, NEIGHBOR_CONSTANT_10, NEIGHBOR_CONSTANT_10,
	136	NEIGHBOR_CONSTANT_10, NEIGHBOR_CONSTANT_10, NEIGHBOR_CONSTANT_10,
	137	NEIGHBOR_CONSTANT_10, NEIGHBOR_CONSTANT_8
	138	};
	139
	140	DECLARE_ALIGNED(16, static const int16_t, LEFT_EDGE_NEIGHBORS_PLUS_4[8]) = {
	141	NEIGHBOR_CONSTANT_10, NEIGHBOR_CONSTANT_13, NEIGHBOR_CONSTANT_13,
	142	NEIGHBOR_CONSTANT_13, NEIGHBOR_CONSTANT_13, NEIGHBOR_CONSTANT_13,
	143	NEIGHBOR_CONSTANT_13, NEIGHBOR_CONSTANT_13
	144	};
	145
	146	DECLARE_ALIGNED(16, static const int16_t, RIGHT_EDGE_NEIGHBORS_PLUS_4[8]) = {
	147	NEIGHBOR_CONSTANT_13, NEIGHBOR_CONSTANT_13, NEIGHBOR_CONSTANT_13,
	148	NEIGHBOR_CONSTANT_13, NEIGHBOR_CONSTANT_13, NEIGHBOR_CONSTANT_13,
	149	NEIGHBOR_CONSTANT_13, NEIGHBOR_CONSTANT_10
	150	};
	151
	152	DECLARE_ALIGNED(16, static const int16_t, MIDDLE_EDGE_NEIGHBORS_PLUS_4[8]) = {
	153	NEIGHBOR_CONSTANT_10, NEIGHBOR_CONSTANT_10, NEIGHBOR_CONSTANT_10,
	154	NEIGHBOR_CONSTANT_10, NEIGHBOR_CONSTANT_10, NEIGHBOR_CONSTANT_10,
	155	NEIGHBOR_CONSTANT_10, NEIGHBOR_CONSTANT_10
	156	};
	157
	158	DECLARE_ALIGNED(16, static const int16_t, MIDDLE_CENTER_NEIGHBORS_PLUS_4[8]) = {
	159	NEIGHBOR_CONSTANT_13, NEIGHBOR_CONSTANT_13, NEIGHBOR_CONSTANT_13,
	160	NEIGHBOR_CONSTANT_13, NEIGHBOR_CONSTANT_13, NEIGHBOR_CONSTANT_13,
	161	NEIGHBOR_CONSTANT_13, NEIGHBOR_CONSTANT_13
	162	};
	163
	164	DECLARE_ALIGNED(16, static const int16_t, TWO_CORNER_NEIGHBORS_PLUS_4[8]) = {
	165	NEIGHBOR_CONSTANT_8, NEIGHBOR_CONSTANT_10, NEIGHBOR_CONSTANT_10,
	166	NEIGHBOR_CONSTANT_10, NEIGHBOR_CONSTANT_10, NEIGHBOR_CONSTANT_10,
	167	NEIGHBOR_CONSTANT_10, NEIGHBOR_CONSTANT_8
	168	};
	169
	170	DECLARE_ALIGNED(16, static const int16_t, TWO_EDGE_NEIGHBORS_PLUS_4[8]) = {
	171	NEIGHBOR_CONSTANT_10, NEIGHBOR_CONSTANT_13, NEIGHBOR_CONSTANT_13,
	172	NEIGHBOR_CONSTANT_13, NEIGHBOR_CONSTANT_13, NEIGHBOR_CONSTANT_13,
	173	NEIGHBOR_CONSTANT_13, NEIGHBOR_CONSTANT_10
	174	};
	175
	176	static const int16_t *const LUMA_LEFT_COLUMN_NEIGHBORS[2] = {
	177	LEFT_CORNER_NEIGHBORS_PLUS_2, LEFT_EDGE_NEIGHBORS_PLUS_2
	178	};
	179
	180	static const int16_t *const LUMA_MIDDLE_COLUMN_NEIGHBORS[2] = {
	181	MIDDLE_EDGE_NEIGHBORS_PLUS_2, MIDDLE_CENTER_NEIGHBORS_PLUS_2
	182	};
	183
	184	static const int16_t *const LUMA_RIGHT_COLUMN_NEIGHBORS[2] = {
	185	RIGHT_CORNER_NEIGHBORS_PLUS_2, RIGHT_EDGE_NEIGHBORS_PLUS_2
	186	};
	187
	188	static const int16_t *const CHROMA_NO_SS_LEFT_COLUMN_NEIGHBORS[2] = {
	189	LEFT_CORNER_NEIGHBORS_PLUS_1, LEFT_EDGE_NEIGHBORS_PLUS_1
	190	};
	191
	192	static const int16_t *const CHROMA_NO_SS_MIDDLE_COLUMN_NEIGHBORS[2] = {
	193	MIDDLE_EDGE_NEIGHBORS_PLUS_1, MIDDLE_CENTER_NEIGHBORS_PLUS_1
	194	};
	195
	196	static const int16_t *const CHROMA_NO_SS_RIGHT_COLUMN_NEIGHBORS[2] = {
	197	RIGHT_CORNER_NEIGHBORS_PLUS_1, RIGHT_EDGE_NEIGHBORS_PLUS_1
	198	};
	199
	200	static const int16_t *const CHROMA_SINGLE_SS_LEFT_COLUMN_NEIGHBORS[2] = {
	201	LEFT_CORNER_NEIGHBORS_PLUS_2, LEFT_EDGE_NEIGHBORS_PLUS_2
	202	};
	203
	204	static const int16_t *const CHROMA_SINGLE_SS_MIDDLE_COLUMN_NEIGHBORS[2] = {
	205	MIDDLE_EDGE_NEIGHBORS_PLUS_2, MIDDLE_CENTER_NEIGHBORS_PLUS_2
	206	};
	207
	208	static const int16_t *const CHROMA_SINGLE_SS_RIGHT_COLUMN_NEIGHBORS[2] = {
	209	RIGHT_CORNER_NEIGHBORS_PLUS_2, RIGHT_EDGE_NEIGHBORS_PLUS_2
	210	};
	211
	212	static const int16_t *const CHROMA_SINGLE_SS_SINGLE_COLUMN_NEIGHBORS[2] = {
	213	TWO_CORNER_NEIGHBORS_PLUS_2, TWO_EDGE_NEIGHBORS_PLUS_2
	214	};
	215
	216	static const int16_t *const CHROMA_DOUBLE_SS_LEFT_COLUMN_NEIGHBORS[2] = {
	217	LEFT_CORNER_NEIGHBORS_PLUS_4, LEFT_EDGE_NEIGHBORS_PLUS_4
	218	};
	219
	220	static const int16_t *const CHROMA_DOUBLE_SS_MIDDLE_COLUMN_NEIGHBORS[2] = {
	221	MIDDLE_EDGE_NEIGHBORS_PLUS_4, MIDDLE_CENTER_NEIGHBORS_PLUS_4
	222	};
	223
	224	static const int16_t *const CHROMA_DOUBLE_SS_RIGHT_COLUMN_NEIGHBORS[2] = {
	225	RIGHT_CORNER_NEIGHBORS_PLUS_4, RIGHT_EDGE_NEIGHBORS_PLUS_4
	226	};
	227
	228	static const int16_t *const CHROMA_DOUBLE_SS_SINGLE_COLUMN_NEIGHBORS[2] = {
	229	TWO_CORNER_NEIGHBORS_PLUS_4, TWO_EDGE_NEIGHBORS_PLUS_4
	230	};
	231
	232	#define DIST_STRIDE ((BW) + 2)

vp9/encoder/x86/temporal_filter_sse4.c+872−44

@@ -14,32 +14,9 @@ · 13 unmodified lines
14	#include "./vp9_rtcd.h"	14	#include "./vp9_rtcd.h"
15	#include "./vpx_config.h"	15	#include "./vpx_config.h"
16	#include "vpx/vpx_integer.h"	16	#include "vpx/vpx_integer.h"
17		17	#include "vp9/encoder/vp9_encoder.h"
18	// Division using multiplication and shifting. The C implementation does:	18	#include "vp9/encoder/vp9_temporal_filter.h"
19	// modifier *= 3;	19	#include "vp9/encoder/x86/temporal_filter_constants.h"
20	// modifier /= index;
21	// where 'modifier' is a set of summed values and 'index' is the number of
22	// summed values. 'index' may be 4, 6, or 9, representing a block of 9 values
23	// which may be bound by the edges of the block being filtered.
24	//
25	// This equation works out to (m * 3) / i which reduces to:
26	// m * 3/4
27	// m * 1/2
28	// m * 1/3
29	//
30	// By pairing the multiply with a down shift by 16 (_mm_mulhi_epu16):
31	// m * C / 65536
32	// we can create a C to replicate the division.
33	//
34	// m * 49152 / 65536 = m * 3/4
35	// m * 32758 / 65536 = m * 1/2
36	// m * 21846 / 65536 = m * 0.3333
37	//
38	// These are loaded using an instruction expecting int16_t values but are used
39	// with _mm_mulhi_epu16(), which treats them as unsigned.
40	#define NEIGHBOR_CONSTANT_4 (int16_t)49152
41	#define NEIGHBOR_CONSTANT_6 (int16_t)32768
42	#define NEIGHBOR_CONSTANT_9 (int16_t)21846
43		20
44	// Load values from 'a' and 'b'. Compute the difference squared and sum	21	// Load values from 'a' and 'b'. Compute the difference squared and sum
45	// neighboring values such that:	22	// neighboring values such that:
@@ -106,12 +83,62 @@ static void sum_16(const uint8_t a, const uint8_t b, __m128i *sum_0, · 60 unmodified lines
106	*sum_1 = sum_u16;	83	*sum_1 = sum_u16;
107	}	84	}
108		85
		86	// Read in 8 pixels from a and b as 8-bit unsigned integers, compute the
		87	// difference squared, and store as unsigned 16-bit integer to dst.
		88	static INLINE void store_dist_8(const uint8_t a, const uint8_t b,
		89	uint16_t *dst) {
		90	const __m128i a_reg = _mm_loadl_epi64((const __m128i *)a);
		91	const __m128i b_reg = _mm_loadl_epi64((const __m128i *)b);
		92
		93	const __m128i a_first = _mm_cvtepu8_epi16(a_reg);
		94	const __m128i b_first = _mm_cvtepu8_epi16(b_reg);
		95
		96	__m128i dist_first;
		97
		98	dist_first = _mm_sub_epi16(a_first, b_first);
		99	dist_first = _mm_mullo_epi16(dist_first, dist_first);
		100
		101	_mm_storeu_si128((__m128i *)dst, dist_first);
		102	}
		103
		104	static INLINE void store_dist_16(const uint8_t a, const uint8_t b,
		105	uint16_t *dst) {
		106	const __m128i zero = _mm_setzero_si128();
		107	const __m128i a_reg = _mm_loadu_si128((const __m128i *)a);
		108	const __m128i b_reg = _mm_loadu_si128((const __m128i *)b);
		109
		110	const __m128i a_first = _mm_cvtepu8_epi16(a_reg);
		111	const __m128i a_second = _mm_unpackhi_epi8(a_reg, zero);
		112	const __m128i b_first = _mm_cvtepu8_epi16(b_reg);
		113	const __m128i b_second = _mm_unpackhi_epi8(b_reg, zero);
		114
		115	__m128i dist_first, dist_second;
		116
		117	dist_first = _mm_sub_epi16(a_first, b_first);
		118	dist_second = _mm_sub_epi16(a_second, b_second);
		119	dist_first = _mm_mullo_epi16(dist_first, dist_first);
		120	dist_second = _mm_mullo_epi16(dist_second, dist_second);
		121
		122	_mm_storeu_si128((__m128i *)dst, dist_first);
		123	_mm_storeu_si128((__m128i *)(dst + 8), dist_second);
		124	}
		125
		126	static INLINE void read_dist_8(const uint16_t dist, __m128i dist_reg) {
		127	dist_reg = _mm_loadu_si128((const __m128i )dist);
		128	}
		129
		130	static INLINE void read_dist_16(const uint16_t dist, __m128i reg_first,
		131	__m128i *reg_second) {
		132	read_dist_8(dist, reg_first);
		133	read_dist_8(dist + 8, reg_second);
		134	}
		135
109	// Average the value based on the number of values summed (9 for pixels away	136	// Average the value based on the number of values summed (9 for pixels away
110	// from the border, 4 for pixels in corners, and 6 for other edge values).	137	// from the border, 4 for pixels in corners, and 6 for other edge values).
111	//	138	//
112	// Add in the rounding factor and shift, clamp to 16, invert and shift. Multiply	139	// Add in the rounding factor and shift, clamp to 16, invert and shift. Multiply
113	// by weight.	140	// by weight.
114	static __m128i average_8(__m128i sum, const __m128i mul_constants,	141	static __m128i average_8(__m128i sum, const __m128i *mul_constants,
115	const int strength, const int rounding,	142	const int strength, const int rounding,
116	const int weight) {	143	const int weight) {
117	// _mm_srl_epi16 uses the lower 64 bit value for the shift.	144	// _mm_srl_epi16 uses the lower 64 bit value for the shift.
@@ -121,7 +148,34 @@ static __m128i average_8(__m128i sum, const __m128i mul_constants, · 3 unmodified lines
121	const __m128i sixteen = _mm_set1_epi16(16);	148	const __m128i sixteen = _mm_set1_epi16(16);
122		149
123	// modifier * 3 / index;	150	// modifier * 3 / index;
124	sum = _mm_mulhi_epu16(sum, mul_constants);	151	sum = _mm_mulhi_epu16(sum, *mul_constants);
		152
		153	sum = _mm_adds_epu16(sum, rounding_u16);
		154	sum = _mm_srl_epi16(sum, strength_u128);
		155
		156	// The maximum input to this comparison is UINT16_MAX * NEIGHBOR_CONSTANT_4
		157	// >> 16 (also NEIGHBOR_CONSTANT_4 -1) which is 49151 / 0xbfff / -16385
		158	// So this needs to use the epu16 version which did not come until SSE4.
		159	sum = _mm_min_epu16(sum, sixteen);
		160
		161	sum = _mm_sub_epi16(sixteen, sum);
		162
		163	return _mm_mullo_epi16(sum, weight_u16);
		164	}
		165
		166	static __m128i average_4_4(__m128i sum, const __m128i *mul_constants,
		167	const int strength, const int rounding,
		168	const int weight_0, const int weight_1) {
		169	// _mm_srl_epi16 uses the lower 64 bit value for the shift.
		170	const __m128i strength_u128 = _mm_set_epi32(0, 0, 0, strength);
		171	const __m128i rounding_u16 = _mm_set1_epi16(rounding);
		172	const __m128i weight_u16 =
		173	_mm_setr_epi16(weight_0, weight_0, weight_0, weight_0, weight_1, weight_1,
		174	weight_1, weight_1);
		175	const __m128i sixteen = _mm_set1_epi16(16);
		176
		177	// modifier * 3 / index;
		178	sum = _mm_mulhi_epu16(sum, *mul_constants);
125		179
126	sum = _mm_adds_epu16(sum, rounding_u16);	180	sum = _mm_adds_epu16(sum, rounding_u16);
127	sum = _mm_srl_epi16(sum, strength_u128);	181	sum = _mm_srl_epi16(sum, strength_u128);
@@ -136,20 +190,21 @@ static __m128i average_8(__m128i sum, const __m128i mul_constants, · 8 unmodified lines
136	return _mm_mullo_epi16(sum, weight_u16);	190	return _mm_mullo_epi16(sum, weight_u16);
137	}	191	}
138		192
139	static void average_16(__m128i sum_0_u16, __m128i sum_1_u16,	193	static INLINE void average_16(__m128i sum_0_u16, __m128i sum_1_u16,
140	const __m128i mul_constants_0,	194	const __m128i *mul_constants_0,
141	const __m128i mul_constants_1, const int strength,	195	const __m128i *mul_constants_1,
142	const int rounding, const int weight) {	196	const int strength, const int rounding,
		197	const int weight) {
143	const __m128i strength_u128 = _mm_set_epi32(0, 0, 0, strength);	198	const __m128i strength_u128 = _mm_set_epi32(0, 0, 0, strength);
144	const __m128i rounding_u16 = _mm_set1_epi16(rounding);	199	const __m128i rounding_u16 = _mm_set1_epi16(rounding);
145	const __m128i weight_u16 = _mm_set1_epi16(weight);	200	const __m128i weight_u16 = _mm_set1_epi16(weight);
146	const __m128i sixteen = _mm_set1_epi16(16);	201	const __m128i sixteen = _mm_set1_epi16(16);
147	__m128i input_0, input_1;	202	__m128i input_0, input_1;
148		203
149	input_0 = _mm_mulhi_epu16(*sum_0_u16, mul_constants_0);	204	input_0 = _mm_mulhi_epu16(sum_0_u16, mul_constants_0);
150	input_0 = _mm_adds_epu16(input_0, rounding_u16);	205	input_0 = _mm_adds_epu16(input_0, rounding_u16);
151		206
152	input_1 = _mm_mulhi_epu16(*sum_1_u16, mul_constants_1);	207	input_1 = _mm_mulhi_epu16(sum_1_u16, mul_constants_1);
153	input_1 = _mm_adds_epu16(input_1, rounding_u16);	208	input_1 = _mm_adds_epu16(input_1, rounding_u16);
154		209
155	input_0 = _mm_srl_epi16(input_0, strength_u128);	210	input_0 = _mm_srl_epi16(input_0, strength_u128);
@@ -192,10 +247,10 @@ static void accumulate_and_store_8(const __m128i sum_u16, const uint8_t *pred, · 36 unmodified lines
192	_mm_storeu_si128((__m128i *)(accumulator + 4), accum_1_u32);	247	_mm_storeu_si128((__m128i *)(accumulator + 4), accum_1_u32);
193	}	248	}
194		249
195	static void accumulate_and_store_16(const __m128i sum_0_u16,	250	static INLINE void accumulate_and_store_16(const __m128i sum_0_u16,
196	const __m128i sum_1_u16,	251	const __m128i sum_1_u16,
197	const uint8_t pred, uint16_t count,	252	const uint8_t pred, uint16_t count,
198	uint32_t *accumulator) {	253	uint32_t *accumulator) {
199	const __m128i pred_u8 = _mm_loadu_si128((const __m128i *)pred);	254	const __m128i pred_u8 = _mm_loadu_si128((const __m128i *)pred);
200	const __m128i zero = _mm_setzero_si128();	255	const __m128i zero = _mm_setzero_si128();
201	__m128i count_0_u16 = _mm_loadu_si128((const __m128i *)count),	256	__m128i count_0_u16 = _mm_loadu_si128((const __m128i *)count),
@@ -235,6 +290,28 @@ static void accumulate_and_store_16(const __m128i sum_0_u16, · 33 unmodified lines
235	_mm_storeu_si128((__m128i *)(accumulator + 12), accum_3_u32);	290	_mm_storeu_si128((__m128i *)(accumulator + 12), accum_3_u32);
236	}	291	}
237		292
		293	// Read in 8 pixels from y_dist. For each index i, compute y_dist[i-1] +
		294	// y_dist[i] + y_dist[i+1] and store in sum as 16-bit unsigned int.
		295	static INLINE void get_sum_8(const uint16_t y_dist, __m128i sum) {
		296	__m128i dist_reg, dist_left, dist_right;
		297
		298	dist_reg = _mm_loadu_si128((const __m128i *)y_dist);
		299	dist_left = _mm_loadu_si128((const __m128i *)(y_dist - 1));
		300	dist_right = _mm_loadu_si128((const __m128i *)(y_dist + 1));
		301
		302	*sum = _mm_adds_epu16(dist_reg, dist_left);
		303	sum = _mm_adds_epu16(sum, dist_right);
		304	}
		305
		306	// Read in 16 pixels from y_dist. For each index i, compute y_dist[i-1] +
		307	// y_dist[i] + y_dist[i+1]. Store the result for first 8 pixels in sum_first and
		308	// the rest in sum_second.
		309	static INLINE void get_sum_16(const uint16_t y_dist, __m128i sum_first,
		310	__m128i *sum_second) {
		311	get_sum_8(y_dist, sum_first);
		312	get_sum_8(y_dist + 8, sum_second);
		313	}
		314
238	void vp9_temporal_filter_apply_sse4_1(const uint8_t *a, unsigned int stride,	315	void vp9_temporal_filter_apply_sse4_1(const uint8_t *a, unsigned int stride,
239	const uint8_t *b, unsigned int width,	316	const uint8_t *b, unsigned int width,
240	unsigned int height, int strength,	317	unsigned int height, int strength,
@@ -261,7 +338,8 @@ void vp9_temporal_filter_apply_sse4_1(const uint8_t *a, unsigned int stride, · 20 unmodified lines
261	sum_8(a, b, &sum_row_a);	338	sum_8(a, b, &sum_row_a);
262	sum_8(a + stride, b + width, &sum_row_b);	339	sum_8(a + stride, b + width, &sum_row_b);
263	sum_row_c = _mm_adds_epu16(sum_row_a, sum_row_b);	340	sum_row_c = _mm_adds_epu16(sum_row_a, sum_row_b);
264	sum_row_c = average_8(sum_row_c, mul_constants, strength, rounding, weight);	341	sum_row_c =
		342	average_8(sum_row_c, &mul_constants, strength, rounding, weight);
265	accumulate_and_store_8(sum_row_c, b, count, accumulator);	343	accumulate_and_store_8(sum_row_c, b, count, accumulator);
266		344
267	a += stride + stride;	345	a += stride + stride;
@@ -279,7 +357,7 @@ void vp9_temporal_filter_apply_sse4_1(const uint8_t *a, unsigned int stride, · 11 unmodified lines
279	sum_row_a = _mm_adds_epu16(sum_row_a, sum_row_b);	357	sum_row_a = _mm_adds_epu16(sum_row_a, sum_row_b);
280	sum_row_a = _mm_adds_epu16(sum_row_a, sum_row_c);	358	sum_row_a = _mm_adds_epu16(sum_row_a, sum_row_c);
281	sum_row_a =	359	sum_row_a =
282	average_8(sum_row_a, mul_constants, strength, rounding, weight);	360	average_8(sum_row_a, &mul_constants, strength, rounding, weight);
283	accumulate_and_store_8(sum_row_a, b, count, accumulator);	361	accumulate_and_store_8(sum_row_a, b, count, accumulator);
284		362
285	a += stride;	363	a += stride;
@@ -296,7 +374,8 @@ void vp9_temporal_filter_apply_sse4_1(const uint8_t *a, unsigned int stride, · 10 unmodified lines
296	NEIGHBOR_CONSTANT_6, NEIGHBOR_CONSTANT_6,	374	NEIGHBOR_CONSTANT_6, NEIGHBOR_CONSTANT_6,
297	NEIGHBOR_CONSTANT_6, NEIGHBOR_CONSTANT_4);	375	NEIGHBOR_CONSTANT_6, NEIGHBOR_CONSTANT_4);
298	sum_row_a = _mm_adds_epu16(sum_row_a, sum_row_b);	376	sum_row_a = _mm_adds_epu16(sum_row_a, sum_row_b);
299	sum_row_a = average_8(sum_row_a, mul_constants, strength, rounding, weight);	377	sum_row_a =
		378	average_8(sum_row_a, &mul_constants, strength, rounding, weight);
300	accumulate_and_store_8(sum_row_a, b, count, accumulator);	379	accumulate_and_store_8(sum_row_a, b, count, accumulator);
301		380
302	} else { // width == 16	381	} else { // width == 16
@@ -318,7 +397,7 @@ void vp9_temporal_filter_apply_sse4_1(const uint8_t *a, unsigned int stride, · 15 unmodified lines
318	sum_row_c_0 = _mm_adds_epu16(sum_row_a_0, sum_row_b_0);	397	sum_row_c_0 = _mm_adds_epu16(sum_row_a_0, sum_row_b_0);
319	sum_row_c_1 = _mm_adds_epu16(sum_row_a_1, sum_row_b_1);	398	sum_row_c_1 = _mm_adds_epu16(sum_row_a_1, sum_row_b_1);
320		399
321	average_16(&sum_row_c_0, &sum_row_c_1, mul_constants_0, mul_constants_1,	400	average_16(&sum_row_c_0, &sum_row_c_1, &mul_constants_0, &mul_constants_1,
322	strength, rounding, weight);	401	strength, rounding, weight);
323	accumulate_and_store_16(sum_row_c_0, sum_row_c_1, b, count, accumulator);	402	accumulate_and_store_16(sum_row_c_0, sum_row_c_1, b, count, accumulator);
324		403
@@ -343,7 +422,7 @@ void vp9_temporal_filter_apply_sse4_1(const uint8_t *a, unsigned int stride, · 18 unmodified lines
343	sum_row_a_1 = _mm_adds_epu16(sum_row_a_1, sum_row_b_1);	422	sum_row_a_1 = _mm_adds_epu16(sum_row_a_1, sum_row_b_1);
344	sum_row_a_1 = _mm_adds_epu16(sum_row_a_1, sum_row_c_1);	423	sum_row_a_1 = _mm_adds_epu16(sum_row_a_1, sum_row_c_1);
345		424
346	average_16(&sum_row_a_0, &sum_row_a_1, mul_constants_0, mul_constants_1,	425	average_16(&sum_row_a_0, &sum_row_a_1, &mul_constants_0, &mul_constants_1,
347	strength, rounding, weight);	426	strength, rounding, weight);
348	accumulate_and_store_16(sum_row_a_0, sum_row_a_1, b, count, accumulator);	427	accumulate_and_store_16(sum_row_a_0, sum_row_a_1, b, count, accumulator);
349		428
@@ -369,8 +448,757 @@ void vp9_temporal_filter_apply_sse4_1(const uint8_t *a, unsigned int stride, · 19 unmodified lines
369	sum_row_c_0 = _mm_adds_epu16(sum_row_a_0, sum_row_b_0);	448	sum_row_c_0 = _mm_adds_epu16(sum_row_a_0, sum_row_b_0);
370	sum_row_c_1 = _mm_adds_epu16(sum_row_a_1, sum_row_b_1);	449	sum_row_c_1 = _mm_adds_epu16(sum_row_a_1, sum_row_b_1);
371		450
372	average_16(&sum_row_c_0, &sum_row_c_1, mul_constants_0, mul_constants_1,	451	average_16(&sum_row_c_0, &sum_row_c_1, &mul_constants_0, &mul_constants_1,
373	strength, rounding, weight);	452	strength, rounding, weight);
374	accumulate_and_store_16(sum_row_c_0, sum_row_c_1, b, count, accumulator);	453	accumulate_and_store_16(sum_row_c_0, sum_row_c_1, b, count, accumulator);
375	}	454	}
376	}	455	}
		456
		457	// Read in a row of chroma values corresponds to a row of 16 luma values.
		458	static INLINE void read_chroma_dist_row_16(int ss_x, const uint16_t *u_dist,
		459	const uint16_t *v_dist,
		460	__m128i u_first, __m128i u_second,
		461	__m128i *v_first,
		462	__m128i *v_second) {
		463	if (!ss_x) {
		464	// If there is no chroma subsampling in the horizaontal direction, then we
		465	// need to load 16 entries from chroma.
		466	read_dist_16(u_dist, u_first, u_second);
		467	read_dist_16(v_dist, v_first, v_second);
		468	} else { // ss_x == 1
		469	// Otherwise, we only need to load 8 entries
		470	__m128i u_reg, v_reg;
		471
		472	read_dist_8(u_dist, &u_reg);
		473
		474	*u_first = _mm_unpacklo_epi16(u_reg, u_reg);
		475	*u_second = _mm_unpackhi_epi16(u_reg, u_reg);
		476
		477	read_dist_8(v_dist, &v_reg);
		478
		479	*v_first = _mm_unpacklo_epi16(v_reg, v_reg);
		480	*v_second = _mm_unpackhi_epi16(v_reg, v_reg);
		481	}
		482	}
		483
		484	// Horizonta add unsigned 16-bit ints in src and store them as signed 32-bit int
		485	// in dst.
		486	static INLINE void hadd_epu16(__m128i src, __m128i dst) {
		487	const __m128i zero = _mm_setzero_si128();
		488	const __m128i shift_right = _mm_srli_si128(*src, 2);
		489
		490	const __m128i odd = _mm_blend_epi16(shift_right, zero, 170);
		491	const __m128i even = _mm_blend_epi16(*src, zero, 170);
		492
		493	*dst = _mm_add_epi32(even, odd);
		494	}
		495
		496	// Add a row of luma distortion to 8 corresponding chroma mods.
		497	static INLINE void add_luma_dist_to_8_chroma_mod(const uint16_t *y_dist,
		498	int ss_x, int ss_y,
		499	__m128i *u_mod,
		500	__m128i *v_mod) {
		501	__m128i y_reg;
		502	if (!ss_x) {
		503	read_dist_8(y_dist, &y_reg);
		504	if (ss_y == 1) {
		505	__m128i y_tmp;
		506	read_dist_8(y_dist + DIST_STRIDE, &y_tmp);
		507
		508	y_reg = _mm_adds_epu16(y_reg, y_tmp);
		509	}
		510	} else {
		511	__m128i y_first, y_second;
		512	read_dist_16(y_dist, &y_first, &y_second);
		513	if (ss_y == 1) {
		514	__m128i y_tmp_0, y_tmp_1;
		515	read_dist_16(y_dist + DIST_STRIDE, &y_tmp_0, &y_tmp_1);
		516
		517	y_first = _mm_adds_epu16(y_first, y_tmp_0);
		518	y_second = _mm_adds_epu16(y_second, y_tmp_1);
		519	}
		520
		521	hadd_epu16(&y_first, &y_first);
		522	hadd_epu16(&y_second, &y_second);
		523
		524	y_reg = _mm_packus_epi32(y_first, y_second);
		525	}
		526
		527	u_mod = _mm_adds_epu16(u_mod, y_reg);
		528	v_mod = _mm_adds_epu16(v_mod, y_reg);
		529	}
		530
		531	// Apply temporal filter to the luma components. This performs temporal
		532	// filtering on a luma block of 16 X block_height. Use blk_fw as an array of
		533	// size 4for the weights for each of the 4 subblocks if blk_fw is not NULL,
		534	// else use top_weight for top half, and bottom weight for bottom half.
		535	static void vp9_apply_temporal_filter_luma_16(
		536	const uint8_t y_src, int y_src_stride, const uint8_t y_pre,
		537	int y_pre_stride, const uint8_t u_src, const uint8_t v_src,
		538	int uv_src_stride, const uint8_t u_pre, const uint8_t v_pre,
		539	int uv_pre_stride, unsigned int block_width, unsigned int block_height,
		540	int ss_x, int ss_y, int strength, int use_whole_blk, uint32_t *y_accum,
		541	uint16_t y_count, const uint16_t y_dist, const uint16_t *u_dist,
		542	const uint16_t v_dist, const int16_t const *neighbors_first,
		543	const int16_t const neighbors_second, int top_weight, int bottom_weight,
		544	const int *blk_fw) {
		545	const int rounding = (1 << strength) >> 1;
		546	int weight = top_weight;
		547
		548	__m128i mul_first, mul_second;
		549
		550	__m128i sum_row_1_first, sum_row_1_second;
		551	__m128i sum_row_2_first, sum_row_2_second;
		552	__m128i sum_row_3_first, sum_row_3_second;
		553
		554	__m128i u_first, u_second;
		555	__m128i v_first, v_second;
		556
		557	__m128i sum_row_first;
		558	__m128i sum_row_second;
		559
		560	assert(strength >= 0);
		561	assert(strength <= 6);
		562
		563	assert(block_width == 16);
		564
		565	(void)block_width;
		566
		567	// First row
		568	mul_first = _mm_loadu_si128((const __m128i *)neighbors_first[0]);
		569	mul_second = _mm_loadu_si128((const __m128i *)neighbors_second[0]);
		570
		571	// Add luma values
		572	get_sum_16(y_dist, &sum_row_2_first, &sum_row_2_second);
		573	get_sum_16(y_dist + DIST_STRIDE, &sum_row_3_first, &sum_row_3_second);
		574
		575	sum_row_first = _mm_adds_epu16(sum_row_2_first, sum_row_3_first);
		576	sum_row_second = _mm_adds_epu16(sum_row_2_second, sum_row_3_second);
		577
		578	// Add chroma values
		579	read_chroma_dist_row_16(ss_x, u_dist, v_dist, &u_first, &u_second, &v_first,
		580	&v_second);
		581
		582	sum_row_first = _mm_adds_epu16(sum_row_first, u_first);
		583	sum_row_second = _mm_adds_epu16(sum_row_second, u_second);
		584
		585	sum_row_first = _mm_adds_epu16(sum_row_first, v_first);
		586	sum_row_second = _mm_adds_epu16(sum_row_second, v_second);
		587
		588	// Get modifier and store result
		589	if (blk_fw) {
		590	sum_row_first =
		591	average_8(sum_row_first, &mul_first, strength, rounding, blk_fw[0]);
		592	sum_row_second =
		593	average_8(sum_row_second, &mul_second, strength, rounding, blk_fw[1]);
		594	} else {
		595	average_16(&sum_row_first, &sum_row_second, &mul_first, &mul_second,
		596	strength, rounding, weight);
		597	}
		598	accumulate_and_store_16(sum_row_first, sum_row_second, y_pre, y_count,
		599	y_accum);
		600
		601	y_src += y_src_stride;
		602	y_pre += y_pre_stride;
		603	y_count += y_pre_stride;
		604	y_accum += y_pre_stride;
		605	y_dist += DIST_STRIDE;
		606
		607	u_src += uv_src_stride;
		608	u_pre += uv_pre_stride;
		609	u_dist += DIST_STRIDE;
		610	v_src += uv_src_stride;
		611	v_pre += uv_pre_stride;
		612	v_dist += DIST_STRIDE;
		613
		614	// Then all the rows except the last one
		615	mul_first = _mm_loadu_si128((const __m128i *)neighbors_first[1]);
		616	mul_second = _mm_loadu_si128((const __m128i *)neighbors_second[1]);
		617
		618	for (unsigned int h = 1; h < block_height - 1; ++h) {
		619	// Move the weight to bottom half
		620	if (!use_whole_blk && h == block_height / 2) {
		621	if (blk_fw) {
		622	blk_fw += 2;
		623	} else {
		624	weight = bottom_weight;
		625	}
		626	}
		627	// Shift the rows up
		628	sum_row_1_first = sum_row_2_first;
		629	sum_row_1_second = sum_row_2_second;
		630	sum_row_2_first = sum_row_3_first;
		631	sum_row_2_second = sum_row_3_second;
		632
		633	// Add luma values to the modifier
		634	sum_row_first = _mm_adds_epu16(sum_row_1_first, sum_row_2_first);
		635	sum_row_second = _mm_adds_epu16(sum_row_1_second, sum_row_2_second);
		636
		637	get_sum_16(y_dist + DIST_STRIDE, &sum_row_3_first, &sum_row_3_second);
		638
		639	sum_row_first = _mm_adds_epu16(sum_row_first, sum_row_3_first);
		640	sum_row_second = _mm_adds_epu16(sum_row_second, sum_row_3_second);
		641
		642	// Add chroma values to the modifier
		643	if (ss_y == 0 \|\| h % 2 == 0) {
		644	// Only calculate the new chroma distortion if we are at a pixel that
		645	// corresponds to a new chroma row
		646	read_chroma_dist_row_16(ss_x, u_dist, v_dist, &u_first, &u_second,
		647	&v_first, &v_second);
		648
		649	u_src += uv_src_stride;
		650	u_pre += uv_pre_stride;
		651	u_dist += DIST_STRIDE;
		652	v_src += uv_src_stride;
		653	v_pre += uv_pre_stride;
		654	v_dist += DIST_STRIDE;
		655	}
		656
		657	sum_row_first = _mm_adds_epu16(sum_row_first, u_first);
		658	sum_row_second = _mm_adds_epu16(sum_row_second, u_second);
		659	sum_row_first = _mm_adds_epu16(sum_row_first, v_first);
		660	sum_row_second = _mm_adds_epu16(sum_row_second, v_second);
		661
		662	// Get modifier and store result
		663	if (blk_fw) {
		664	sum_row_first =
		665	average_8(sum_row_first, &mul_first, strength, rounding, blk_fw[0]);
		666	sum_row_second =
		667	average_8(sum_row_second, &mul_second, strength, rounding, blk_fw[1]);
		668	} else {
		669	average_16(&sum_row_first, &sum_row_second, &mul_first, &mul_second,
		670	strength, rounding, weight);
		671	}
		672	accumulate_and_store_16(sum_row_first, sum_row_second, y_pre, y_count,
		673	y_accum);
		674
		675	y_src += y_src_stride;
		676	y_pre += y_pre_stride;
		677	y_count += y_pre_stride;
		678	y_accum += y_pre_stride;
		679	y_dist += DIST_STRIDE;
		680	}
		681
		682	// The last row
		683	mul_first = _mm_loadu_si128((const __m128i *)neighbors_first[0]);
		684	mul_second = _mm_loadu_si128((const __m128i *)neighbors_second[0]);
		685
		686	// Shift the rows up
		687	sum_row_1_first = sum_row_2_first;
		688	sum_row_1_second = sum_row_2_second;
		689	sum_row_2_first = sum_row_3_first;
		690	sum_row_2_second = sum_row_3_second;
		691
		692	// Add luma values to the modifier
		693	sum_row_first = _mm_adds_epu16(sum_row_1_first, sum_row_2_first);
		694	sum_row_second = _mm_adds_epu16(sum_row_1_second, sum_row_2_second);
		695
		696	// Add chroma values to the modifier
		697	if (ss_y == 0) {
		698	// Only calculate the new chroma distortion if we are at a pixel that
		699	// corresponds to a new chroma row
		700	read_chroma_dist_row_16(ss_x, u_dist, v_dist, &u_first, &u_second, &v_first,
		701	&v_second);
		702	}
		703
		704	sum_row_first = _mm_adds_epu16(sum_row_first, u_first);
		705	sum_row_second = _mm_adds_epu16(sum_row_second, u_second);
		706	sum_row_first = _mm_adds_epu16(sum_row_first, v_first);
		707	sum_row_second = _mm_adds_epu16(sum_row_second, v_second);
		708
		709	// Get modifier and store result
		710	if (blk_fw) {
		711	sum_row_first =
		712	average_8(sum_row_first, &mul_first, strength, rounding, blk_fw[0]);
		713	sum_row_second =
		714	average_8(sum_row_second, &mul_second, strength, rounding, blk_fw[1]);
		715	} else {
		716	average_16(&sum_row_first, &sum_row_second, &mul_first, &mul_second,
		717	strength, rounding, weight);
		718	}
		719	accumulate_and_store_16(sum_row_first, sum_row_second, y_pre, y_count,
		720	y_accum);
		721	}
		722
		723	// Perform temporal filter for the luma component.
		724	static void vp9_apply_temporal_filter_luma(
		725	const uint8_t y_src, int y_src_stride, const uint8_t y_pre,
		726	int y_pre_stride, const uint8_t u_src, const uint8_t v_src,
		727	int uv_src_stride, const uint8_t u_pre, const uint8_t v_pre,
		728	int uv_pre_stride, unsigned int block_width, unsigned int block_height,
		729	int ss_x, int ss_y, int strength, const int *blk_fw, int use_whole_blk,
		730	uint32_t y_accum, uint16_t y_count, const uint16_t *y_dist,
		731	const uint16_t u_dist, const uint16_t v_dist) {
		732	unsigned int blk_col = 0, uv_blk_col = 0;
		733	const unsigned int blk_col_step = 16, uv_blk_col_step = 16 >> ss_x;
		734	const unsigned int mid_width = block_width >> 1,
		735	last_width = block_width - blk_col_step;
		736	int top_weight = blk_fw[0],
		737	bottom_weight = use_whole_blk ? blk_fw[0] : blk_fw[2];
		738	const int16_t const neighbors_first;
		739	const int16_t const neighbors_second;
		740
		741	if (block_width == 16) {
		742	// Special Case: The blockwidth is 16 and we are operating on a row of 16
		743	// chroma pixels. In this case, we can't use the usualy left-midle-right
		744	// pattern. We also don't support splitting now.
		745	neighbors_first = LUMA_LEFT_COLUMN_NEIGHBORS;
		746	neighbors_second = LUMA_RIGHT_COLUMN_NEIGHBORS;
		747	if (use_whole_blk) {
		748	vp9_apply_temporal_filter_luma_16(
		749	y_src + blk_col, y_src_stride, y_pre + blk_col, y_pre_stride,
		750	u_src + uv_blk_col, v_src + uv_blk_col, uv_src_stride,
		751	u_pre + uv_blk_col, v_pre + uv_blk_col, uv_pre_stride, 16,
		752	block_height, ss_x, ss_y, strength, use_whole_blk, y_accum + blk_col,
		753	y_count + blk_col, y_dist + blk_col, u_dist + uv_blk_col,
		754	v_dist + uv_blk_col, neighbors_first, neighbors_second, top_weight,
		755	bottom_weight, NULL);
		756	} else {
		757	vp9_apply_temporal_filter_luma_16(
		758	y_src + blk_col, y_src_stride, y_pre + blk_col, y_pre_stride,
		759	u_src + uv_blk_col, v_src + uv_blk_col, uv_src_stride,
		760	u_pre + uv_blk_col, v_pre + uv_blk_col, uv_pre_stride, 16,
		761	block_height, ss_x, ss_y, strength, use_whole_blk, y_accum + blk_col,
		762	y_count + blk_col, y_dist + blk_col, u_dist + uv_blk_col,
		763	v_dist + uv_blk_col, neighbors_first, neighbors_second, 0, 0, blk_fw);
		764	}
		765
		766	return;
		767	}
		768
		769	// Left
		770	neighbors_first = LUMA_LEFT_COLUMN_NEIGHBORS;
		771	neighbors_second = LUMA_MIDDLE_COLUMN_NEIGHBORS;
		772	vp9_apply_temporal_filter_luma_16(
		773	y_src + blk_col, y_src_stride, y_pre + blk_col, y_pre_stride,
		774	u_src + uv_blk_col, v_src + uv_blk_col, uv_src_stride, u_pre + uv_blk_col,
		775	v_pre + uv_blk_col, uv_pre_stride, 16, block_height, ss_x, ss_y, strength,
		776	use_whole_blk, y_accum + blk_col, y_count + blk_col, y_dist + blk_col,
		777	u_dist + uv_blk_col, v_dist + uv_blk_col, neighbors_first,
		778	neighbors_second, top_weight, bottom_weight, NULL);
		779
		780	blk_col += blk_col_step;
		781	uv_blk_col += uv_blk_col_step;
		782
		783	// Middle First
		784	neighbors_first = LUMA_MIDDLE_COLUMN_NEIGHBORS;
		785	for (; blk_col < mid_width;
		786	blk_col += blk_col_step, uv_blk_col += uv_blk_col_step) {
		787	vp9_apply_temporal_filter_luma_16(
		788	y_src + blk_col, y_src_stride, y_pre + blk_col, y_pre_stride,
		789	u_src + uv_blk_col, v_src + uv_blk_col, uv_src_stride,
		790	u_pre + uv_blk_col, v_pre + uv_blk_col, uv_pre_stride, 16, block_height,
		791	ss_x, ss_y, strength, use_whole_blk, y_accum + blk_col,
		792	y_count + blk_col, y_dist + blk_col, u_dist + uv_blk_col,
		793	v_dist + uv_blk_col, neighbors_first, neighbors_second, top_weight,
		794	bottom_weight, NULL);
		795	}
		796
		797	if (!use_whole_blk) {
		798	top_weight = blk_fw[1];
		799	bottom_weight = blk_fw[3];
		800	}
		801
		802	// Middle Second
		803	for (; blk_col < last_width;
		804	blk_col += blk_col_step, uv_blk_col += uv_blk_col_step) {
		805	vp9_apply_temporal_filter_luma_16(
		806	y_src + blk_col, y_src_stride, y_pre + blk_col, y_pre_stride,
		807	u_src + uv_blk_col, v_src + uv_blk_col, uv_src_stride,
		808	u_pre + uv_blk_col, v_pre + uv_blk_col, uv_pre_stride, 16, block_height,
		809	ss_x, ss_y, strength, use_whole_blk, y_accum + blk_col,
		810	y_count + blk_col, y_dist + blk_col, u_dist + uv_blk_col,
		811	v_dist + uv_blk_col, neighbors_first, neighbors_second, top_weight,
		812	bottom_weight, NULL);
		813	}
		814
		815	// Right
		816	neighbors_second = LUMA_RIGHT_COLUMN_NEIGHBORS;
		817	vp9_apply_temporal_filter_luma_16(
		818	y_src + blk_col, y_src_stride, y_pre + blk_col, y_pre_stride,
		819	u_src + uv_blk_col, v_src + uv_blk_col, uv_src_stride, u_pre + uv_blk_col,
		820	v_pre + uv_blk_col, uv_pre_stride, 16, block_height, ss_x, ss_y, strength,
		821	use_whole_blk, y_accum + blk_col, y_count + blk_col, y_dist + blk_col,
		822	u_dist + uv_blk_col, v_dist + uv_blk_col, neighbors_first,
		823	neighbors_second, top_weight, bottom_weight, NULL);
		824	}
		825
		826	// Apply temporal filter to the chroma components. This performs temporal
		827	// filtering on a chroma block of 8 X uv_height. If blk_fw is not NULL, use
		828	// blk_fw as an array of size 4 for the weights for each of the 4 subblocks,
		829	// else use top_weight for top half, and bottom weight for bottom half.
		830	static void vp9_apply_temporal_filter_chroma_8(
		831	const uint8_t y_src, int y_src_stride, const uint8_t y_pre,
		832	int y_pre_stride, const uint8_t u_src, const uint8_t v_src,
		833	int uv_src_stride, const uint8_t u_pre, const uint8_t v_pre,
		834	int uv_pre_stride, unsigned int uv_block_width,
		835	unsigned int uv_block_height, int ss_x, int ss_y, int strength,
		836	uint32_t u_accum, uint16_t u_count, uint32_t v_accum, uint16_t v_count,
		837	const uint16_t y_dist, const uint16_t u_dist, const uint16_t *v_dist,
		838	const int16_t const neighbors, int top_weight, int bottom_weight,
		839	const int *blk_fw) {
		840	const int rounding = (1 << strength) >> 1;
		841	int weight = top_weight;
		842
		843	__m128i mul;
		844
		845	__m128i u_sum_row_1, u_sum_row_2, u_sum_row_3;
		846	__m128i v_sum_row_1, v_sum_row_2, v_sum_row_3;
		847
		848	__m128i u_sum_row, v_sum_row;
		849
		850	(void)uv_block_width;
		851
		852	// First row
		853	mul = _mm_loadu_si128((const __m128i *)neighbors[0]);
		854
		855	// Add chroma values
		856	get_sum_8(u_dist, &u_sum_row_2);
		857	get_sum_8(u_dist + DIST_STRIDE, &u_sum_row_3);
		858
		859	u_sum_row = _mm_adds_epu16(u_sum_row_2, u_sum_row_3);
		860
		861	get_sum_8(v_dist, &v_sum_row_2);
		862	get_sum_8(v_dist + DIST_STRIDE, &v_sum_row_3);
		863
		864	v_sum_row = _mm_adds_epu16(v_sum_row_2, v_sum_row_3);
		865
		866	// Add luma values
		867	add_luma_dist_to_8_chroma_mod(y_dist, ss_x, ss_y, &u_sum_row, &v_sum_row);
		868
		869	// Get modifier and store result
		870	if (blk_fw) {
		871	u_sum_row =
		872	average_4_4(u_sum_row, &mul, strength, rounding, blk_fw[0], blk_fw[1]);
		873	v_sum_row =
		874	average_4_4(v_sum_row, &mul, strength, rounding, blk_fw[0], blk_fw[1]);
		875	} else {
		876	u_sum_row = average_8(u_sum_row, &mul, strength, rounding, weight);
		877	v_sum_row = average_8(v_sum_row, &mul, strength, rounding, weight);
		878	}
		879	accumulate_and_store_8(u_sum_row, u_pre, u_count, u_accum);
		880	accumulate_and_store_8(v_sum_row, v_pre, v_count, v_accum);
		881
		882	u_src += uv_src_stride;
		883	u_pre += uv_pre_stride;
		884	u_dist += DIST_STRIDE;
		885	v_src += uv_src_stride;
		886	v_pre += uv_pre_stride;
		887	v_dist += DIST_STRIDE;
		888	u_count += uv_pre_stride;
		889	u_accum += uv_pre_stride;
		890	v_count += uv_pre_stride;
		891	v_accum += uv_pre_stride;
		892
		893	y_src += y_src_stride * (1 + ss_y);
		894	y_pre += y_pre_stride * (1 + ss_y);
		895	y_dist += DIST_STRIDE * (1 + ss_y);
		896
		897	// Then all the rows except the last one
		898	mul = _mm_loadu_si128((const __m128i *)neighbors[1]);
		899
		900	for (unsigned int h = 1; h < uv_block_height - 1; ++h) {
		901	// Move the weight pointer to the bottom half of the blocks
		902	if (h == uv_block_height / 2) {
		903	if (blk_fw) {
		904	blk_fw += 2;
		905	} else {
		906	weight = bottom_weight;
		907	}
		908	}
		909
		910	// Shift the rows up
		911	u_sum_row_1 = u_sum_row_2;
		912	u_sum_row_2 = u_sum_row_3;
		913
		914	v_sum_row_1 = v_sum_row_2;
		915	v_sum_row_2 = v_sum_row_3;
		916
		917	// Add chroma values
		918	u_sum_row = _mm_adds_epu16(u_sum_row_1, u_sum_row_2);
		919	get_sum_8(u_dist + DIST_STRIDE, &u_sum_row_3);
		920	u_sum_row = _mm_adds_epu16(u_sum_row, u_sum_row_3);
		921
		922	v_sum_row = _mm_adds_epu16(v_sum_row_1, v_sum_row_2);
		923	get_sum_8(v_dist + DIST_STRIDE, &v_sum_row_3);
		924	v_sum_row = _mm_adds_epu16(v_sum_row, v_sum_row_3);
		925
		926	// Add luma values
		927	add_luma_dist_to_8_chroma_mod(y_dist, ss_x, ss_y, &u_sum_row, &v_sum_row);
		928
		929	// Get modifier and store result
		930	if (blk_fw) {
		931	u_sum_row = average_4_4(u_sum_row, &mul, strength, rounding, blk_fw[0],
		932	blk_fw[1]);
		933	v_sum_row = average_4_4(v_sum_row, &mul, strength, rounding, blk_fw[0],
		934	blk_fw[1]);
		935	} else {
		936	u_sum_row = average_8(u_sum_row, &mul, strength, rounding, weight);
		937	v_sum_row = average_8(v_sum_row, &mul, strength, rounding, weight);
		938	}
		939
		940	accumulate_and_store_8(u_sum_row, u_pre, u_count, u_accum);
		941	accumulate_and_store_8(v_sum_row, v_pre, v_count, v_accum);
		942
		943	u_src += uv_src_stride;
		944	u_pre += uv_pre_stride;
		945	u_dist += DIST_STRIDE;
		946	v_src += uv_src_stride;
		947	v_pre += uv_pre_stride;
		948	v_dist += DIST_STRIDE;
		949	u_count += uv_pre_stride;
		950	u_accum += uv_pre_stride;
		951	v_count += uv_pre_stride;
		952	v_accum += uv_pre_stride;
		953
		954	y_src += y_src_stride * (1 + ss_y);
		955	y_pre += y_pre_stride * (1 + ss_y);
		956	y_dist += DIST_STRIDE * (1 + ss_y);
		957	}
		958
		959	// The last row
		960	mul = _mm_loadu_si128((const __m128i *)neighbors[0]);
		961
		962	// Shift the rows up
		963	u_sum_row_1 = u_sum_row_2;
		964	u_sum_row_2 = u_sum_row_3;
		965
		966	v_sum_row_1 = v_sum_row_2;
		967	v_sum_row_2 = v_sum_row_3;
		968
		969	// Add chroma values
		970	u_sum_row = _mm_adds_epu8(u_sum_row_1, u_sum_row_2);
		971	v_sum_row = _mm_adds_epu8(v_sum_row_1, v_sum_row_2);
		972
		973	// Add luma values
		974	add_luma_dist_to_8_chroma_mod(y_dist, ss_x, ss_y, &u_sum_row, &v_sum_row);
		975
		976	// Get modifier and store result
		977	if (blk_fw) {
		978	u_sum_row =
		979	average_4_4(u_sum_row, &mul, strength, rounding, blk_fw[0], blk_fw[1]);
		980	v_sum_row =
		981	average_4_4(v_sum_row, &mul, strength, rounding, blk_fw[0], blk_fw[1]);
		982	} else {
		983	u_sum_row = average_8(u_sum_row, &mul, strength, rounding, weight);
		984	v_sum_row = average_8(v_sum_row, &mul, strength, rounding, weight);
		985	}
		986
		987	accumulate_and_store_8(u_sum_row, u_pre, u_count, u_accum);
		988	accumulate_and_store_8(v_sum_row, v_pre, v_count, v_accum);
		989	}
		990
		991	// Perform temporal filter for the chroma components.
		992	static void vp9_apply_temporal_filter_chroma(
		993	const uint8_t y_src, int y_src_stride, const uint8_t y_pre,
		994	int y_pre_stride, const uint8_t u_src, const uint8_t v_src,
		995	int uv_src_stride, const uint8_t u_pre, const uint8_t v_pre,
		996	int uv_pre_stride, unsigned int block_width, unsigned int block_height,
		997	int ss_x, int ss_y, int strength, const int *blk_fw, int use_whole_blk,
		998	uint32_t u_accum, uint16_t u_count, uint32_t v_accum, uint16_t v_count,
		999	const uint16_t y_dist, const uint16_t u_dist, const uint16_t *v_dist) {
		1000	const unsigned int uv_width = block_width >> ss_x,
		1001	uv_height = block_height >> ss_y;
		1002
		1003	unsigned int blk_col = 0, uv_blk_col = 0;
		1004	const unsigned int uv_blk_col_step = 8, blk_col_step = 8 << ss_x;
		1005	const unsigned int uv_mid_width = uv_width >> 1,
		1006	uv_last_width = uv_width - uv_blk_col_step;
		1007	int top_weight = blk_fw[0],
		1008	bottom_weight = use_whole_blk ? blk_fw[0] : blk_fw[2];
		1009	const int16_t const neighbors;
		1010
		1011	if (uv_width == 8) {
		1012	// Special Case: We are subsampling in x direction on a 16x16 block. Since
		1013	// we are operating on a row of 8 chroma pixels, we can't use the usual
		1014	// left-middle-right pattern.
		1015	assert(ss_x);
		1016
		1017	if (ss_y) {
		1018	neighbors = CHROMA_DOUBLE_SS_SINGLE_COLUMN_NEIGHBORS;
		1019	} else {
		1020	neighbors = CHROMA_SINGLE_SS_SINGLE_COLUMN_NEIGHBORS;
		1021	}
		1022
		1023	if (use_whole_blk) {
		1024	vp9_apply_temporal_filter_chroma_8(
		1025	y_src + blk_col, y_src_stride, y_pre + blk_col, y_pre_stride,
		1026	u_src + uv_blk_col, v_src + uv_blk_col, uv_src_stride,
		1027	u_pre + uv_blk_col, v_pre + uv_blk_col, uv_pre_stride, uv_width,
		1028	uv_height, ss_x, ss_y, strength, u_accum + uv_blk_col,
		1029	u_count + uv_blk_col, v_accum + uv_blk_col, v_count + uv_blk_col,
		1030	y_dist + blk_col, u_dist + uv_blk_col, v_dist + uv_blk_col, neighbors,
		1031	top_weight, bottom_weight, NULL);
		1032	} else {
		1033	vp9_apply_temporal_filter_chroma_8(
		1034	y_src + blk_col, y_src_stride, y_pre + blk_col, y_pre_stride,
		1035	u_src + uv_blk_col, v_src + uv_blk_col, uv_src_stride,
		1036	u_pre + uv_blk_col, v_pre + uv_blk_col, uv_pre_stride, uv_width,
		1037	uv_height, ss_x, ss_y, strength, u_accum + uv_blk_col,
		1038	u_count + uv_blk_col, v_accum + uv_blk_col, v_count + uv_blk_col,
		1039	y_dist + blk_col, u_dist + uv_blk_col, v_dist + uv_blk_col, neighbors,
		1040	0, 0, blk_fw);
		1041	}
		1042
		1043	return;
		1044	}
		1045
		1046	// Left
		1047	if (ss_x && ss_y) {
		1048	neighbors = CHROMA_DOUBLE_SS_LEFT_COLUMN_NEIGHBORS;
		1049	} else if (ss_x \|\| ss_y) {
		1050	neighbors = CHROMA_SINGLE_SS_LEFT_COLUMN_NEIGHBORS;
		1051	} else {
		1052	neighbors = CHROMA_NO_SS_LEFT_COLUMN_NEIGHBORS;
		1053	}
		1054
		1055	vp9_apply_temporal_filter_chroma_8(
		1056	y_src + blk_col, y_src_stride, y_pre + blk_col, y_pre_stride,
		1057	u_src + uv_blk_col, v_src + uv_blk_col, uv_src_stride, u_pre + uv_blk_col,
		1058	v_pre + uv_blk_col, uv_pre_stride, uv_width, uv_height, ss_x, ss_y,
		1059	strength, u_accum + uv_blk_col, u_count + uv_blk_col,
		1060	v_accum + uv_blk_col, v_count + uv_blk_col, y_dist + blk_col,
		1061	u_dist + uv_blk_col, v_dist + uv_blk_col, neighbors, top_weight,
		1062	bottom_weight, NULL);
		1063
		1064	blk_col += blk_col_step;
		1065	uv_blk_col += uv_blk_col_step;
		1066
		1067	// Middle First
		1068	if (ss_x && ss_y) {
		1069	neighbors = CHROMA_DOUBLE_SS_MIDDLE_COLUMN_NEIGHBORS;
		1070	} else if (ss_x \|\| ss_y) {
		1071	neighbors = CHROMA_SINGLE_SS_MIDDLE_COLUMN_NEIGHBORS;
		1072	} else {
		1073	neighbors = CHROMA_NO_SS_MIDDLE_COLUMN_NEIGHBORS;
		1074	}
		1075
		1076	for (; uv_blk_col < uv_mid_width;
		1077	blk_col += blk_col_step, uv_blk_col += uv_blk_col_step) {
		1078	vp9_apply_temporal_filter_chroma_8(
		1079	y_src + blk_col, y_src_stride, y_pre + blk_col, y_pre_stride,
		1080	u_src + uv_blk_col, v_src + uv_blk_col, uv_src_stride,
		1081	u_pre + uv_blk_col, v_pre + uv_blk_col, uv_pre_stride, uv_width,
		1082	uv_height, ss_x, ss_y, strength, u_accum + uv_blk_col,
		1083	u_count + uv_blk_col, v_accum + uv_blk_col, v_count + uv_blk_col,
		1084	y_dist + blk_col, u_dist + uv_blk_col, v_dist + uv_blk_col, neighbors,
		1085	top_weight, bottom_weight, NULL);
		1086	}
		1087
		1088	if (!use_whole_blk) {
		1089	top_weight = blk_fw[1];
		1090	bottom_weight = blk_fw[3];
		1091	}
		1092
		1093	// Middle Second
		1094	for (; uv_blk_col < uv_last_width;
		1095	blk_col += blk_col_step, uv_blk_col += uv_blk_col_step) {
		1096	vp9_apply_temporal_filter_chroma_8(
		1097	y_src + blk_col, y_src_stride, y_pre + blk_col, y_pre_stride,
		1098	u_src + uv_blk_col, v_src + uv_blk_col, uv_src_stride,
		1099	u_pre + uv_blk_col, v_pre + uv_blk_col, uv_pre_stride, uv_width,
		1100	uv_height, ss_x, ss_y, strength, u_accum + uv_blk_col,
		1101	u_count + uv_blk_col, v_accum + uv_blk_col, v_count + uv_blk_col,
		1102	y_dist + blk_col, u_dist + uv_blk_col, v_dist + uv_blk_col, neighbors,
		1103	top_weight, bottom_weight, NULL);
		1104	}
		1105
		1106	// Right
		1107	if (ss_x && ss_y) {
		1108	neighbors = CHROMA_DOUBLE_SS_RIGHT_COLUMN_NEIGHBORS;
		1109	} else if (ss_x \|\| ss_y) {
		1110	neighbors = CHROMA_SINGLE_SS_RIGHT_COLUMN_NEIGHBORS;
		1111	} else {
		1112	neighbors = CHROMA_NO_SS_RIGHT_COLUMN_NEIGHBORS;
		1113	}
		1114
		1115	vp9_apply_temporal_filter_chroma_8(
		1116	y_src + blk_col, y_src_stride, y_pre + blk_col, y_pre_stride,
		1117	u_src + uv_blk_col, v_src + uv_blk_col, uv_src_stride, u_pre + uv_blk_col,
		1118	v_pre + uv_blk_col, uv_pre_stride, uv_width, uv_height, ss_x, ss_y,
		1119	strength, u_accum + uv_blk_col, u_count + uv_blk_col,
		1120	v_accum + uv_blk_col, v_count + uv_blk_col, y_dist + blk_col,
		1121	u_dist + uv_blk_col, v_dist + uv_blk_col, neighbors, top_weight,
		1122	bottom_weight, NULL);
		1123	}
		1124
		1125	void vp9_apply_temporal_filter_sse4_1(
		1126	const uint8_t y_src, int y_src_stride, const uint8_t y_pre,
		1127	int y_pre_stride, const uint8_t u_src, const uint8_t v_src,
		1128	int uv_src_stride, const uint8_t u_pre, const uint8_t v_pre,
		1129	int uv_pre_stride, unsigned int block_width, unsigned int block_height,
		1130	int ss_x, int ss_y, int strength, const int *const blk_fw,
		1131	int use_whole_blk, uint32_t y_accum, uint16_t y_count, uint32_t *u_accum,
		1132	uint16_t u_count, uint32_t v_accum, uint16_t *v_count) {
		1133	const unsigned int chroma_height = block_height >> ss_y,
		1134	chroma_width = block_width >> ss_x;
		1135
		1136	DECLARE_ALIGNED(16, uint16_t, y_dist[BH * DIST_STRIDE]) = { 0 };
		1137	DECLARE_ALIGNED(16, uint16_t, u_dist[BH * DIST_STRIDE]) = { 0 };
		1138	DECLARE_ALIGNED(16, uint16_t, v_dist[BH * DIST_STRIDE]) = { 0 };
		1139	const int *blk_fw_ptr = blk_fw;
		1140
		1141	uint16_t y_dist_ptr = y_dist + 1, u_dist_ptr = u_dist + 1,
		1142	*v_dist_ptr = v_dist + 1;
		1143	const uint8_t y_src_ptr = y_src, u_src_ptr = u_src, *v_src_ptr = v_src;
		1144	const uint8_t y_pre_ptr = y_pre, u_pre_ptr = u_pre, *v_pre_ptr = v_pre;
		1145
		1146	assert(block_width <= BW && "block width too large");
		1147	assert(block_height <= BH && "block height too large");
		1148	assert(block_width % 16 == 0 && "block width must be multiple of 16");
		1149	assert(block_height % 2 == 0 && "block height must be even");
		1150	assert((ss_x == 0 \|\| ss_x == 1) && (ss_y == 0 \|\| ss_y == 1) &&
		1151	"invalid chroma subsampling");
		1152	assert(strength >= 0 && strength <= 6 && "invalid temporal filter strength");
		1153	assert(blk_fw[0] >= 0 && "filter weight must be positive");
		1154	assert(
		1155	(use_whole_blk \|\| (blk_fw[1] >= 0 && blk_fw[2] >= 0 && blk_fw[3] >= 0)) &&
		1156	"subblock filter weight must be positive");
		1157	assert(blk_fw[0] <= 2 && "sublock filter weight must be less than 2");
		1158	assert(
		1159	(use_whole_blk \|\| (blk_fw[1] <= 2 && blk_fw[2] <= 2 && blk_fw[3] <= 2)) &&
		1160	"subblock filter weight must be less than 2");
		1161
		1162	// Precompute the difference sqaured
		1163	for (unsigned int row = 0; row < block_height; row++) {
		1164	for (unsigned int blk_col = 0; blk_col < block_width; blk_col += 16) {
		1165	store_dist_16(y_src_ptr + blk_col, y_pre_ptr + blk_col,
		1166	y_dist_ptr + blk_col);
		1167	}
		1168	y_src_ptr += y_src_stride;
		1169	y_pre_ptr += y_pre_stride;
		1170	y_dist_ptr += DIST_STRIDE;
		1171	}
		1172
		1173	for (unsigned int row = 0; row < chroma_height; row++) {
		1174	for (unsigned int blk_col = 0; blk_col < chroma_width; blk_col += 8) {
		1175	store_dist_8(u_src_ptr + blk_col, u_pre_ptr + blk_col,
		1176	u_dist_ptr + blk_col);
		1177	store_dist_8(v_src_ptr + blk_col, v_pre_ptr + blk_col,
		1178	v_dist_ptr + blk_col);
		1179	}
		1180
		1181	u_src_ptr += uv_src_stride;
		1182	u_pre_ptr += uv_pre_stride;
		1183	u_dist_ptr += DIST_STRIDE;
		1184	v_src_ptr += uv_src_stride;
		1185	v_pre_ptr += uv_pre_stride;
		1186	v_dist_ptr += DIST_STRIDE;
		1187	}
		1188
		1189	y_dist_ptr = y_dist + 1;
		1190	u_dist_ptr = u_dist + 1;
		1191	v_dist_ptr = v_dist + 1;
		1192
		1193	vp9_apply_temporal_filter_luma(
		1194	y_src, y_src_stride, y_pre, y_pre_stride, u_src, v_src, uv_src_stride,
		1195	u_pre, v_pre, uv_pre_stride, block_width, block_height, ss_x, ss_y,
		1196	strength, blk_fw_ptr, use_whole_blk, y_accum, y_count, y_dist_ptr,
		1197	u_dist_ptr, v_dist_ptr);
		1198
		1199	vp9_apply_temporal_filter_chroma(
		1200	y_src, y_src_stride, y_pre, y_pre_stride, u_src, v_src, uv_src_stride,
		1201	u_pre, v_pre, uv_pre_stride, block_width, block_height, ss_x, ss_y,
		1202	strength, blk_fw_ptr, use_whole_blk, u_accum, u_count, v_accum, v_count,
		1203	y_dist_ptr, u_dist_ptr, v_dist_ptr);
		1204	}

vp9/vp9cx.mk+2−0

 VP9_CX_SRCS-yes += encoder/vp9_mbgraph.h
 VP9_CX_SRCS-$(HAVE_SSE4_1) += encoder/x86/temporal_filter_sse4.c
+VP9_CX_SRCS-$(HAVE_SSE4_1) += encoder/x86/temporal_filter_constants.h
 VP9_CX_SRCS-$(HAVE_SSE2) += encoder/x86/vp9_quantize_sse2.c
 VP9_CX_SRCS-$(HAVE_AVX2) += encoder/x86/vp9_quantize_avx2.c
 VP9_CX_SRCS_REMOVE-$(CONFIG_REALTIME_ONLY) += encoder/vp9_mbgraph.c
 VP9_CX_SRCS_REMOVE-$(CONFIG_REALTIME_ONLY) += encoder/vp9_temporal_filter.c
 VP9_CX_SRCS_REMOVE-$(CONFIG_REALTIME_ONLY) += encoder/x86/temporal_filter_sse4.c
+VP9_CX_SRCS_REMOVE-$(CONFIG_REALTIME_ONLY) += encoder/x86/temporal_filter_constants.h
 VP9_CX_SRCS-yes := $(filter-out $(VP9_CX_SRCS_REMOVE-yes),$(VP9_CX_SRCS-yes))

Add SSE4 version of new apply_temporal_filter

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code