avcodec/mips: MSA (MIPS-SIMD-Arch) optimizations for HEVC biw mc functions

This patch adds MSA (MIPS-SIMD-Arch) optimizations for HEVC biw mc functions (qpel as well as epel) in new file hevc_mc_biw_msa.c Adds new generic macros (needed for this patch) in libavutil/mips/generic_macros_msa.h

Shivraj Patil committed 11 years ago

Commit 88188f51 parent 12002894 files · +5744 −1Browse files

Unified · Split

4 files

libavcodec/mips/Makefile +2 −1
libavcodec/mips/hevc_mc_biw_msa.c +5572 −0
libavcodec/mips/hevcdsp_init_mips.c +83 −0
libavcodec/mips/hevcdsp_mips.h +87 −0

libavcodec/mips/Makefile+2−1

 MSA-OBJS-$(CONFIG_HEVC_DECODER)           += mips/hevcdsp_msa.o            \
                                              mips/hevc_mc_uni_msa.o        \
                                              mips/hevc_mc_uniw_msa.o       \
-                                             mips/hevc_mc_bi_msa.o
+                                             mips/hevc_mc_bi_msa.o         \
+                                             mips/hevc_mc_biw_msa.o
 MSA-OBJS-$(CONFIG_H264DSP)                += mips/h264dsp_msa.o
 LOONGSON3-OBJS-$(CONFIG_H264DSP)          += mips/h264dsp_mmi.o
 LOONGSON3-OBJS-$(CONFIG_H264CHROMA)       += mips/h264chroma_mmi.o

libavcodec/mips/hevc_mc_biw_msa.c+5572−0

@@ -0,0 +1,5572 @@
	1	/*
	2	* Copyright (c) 2015 Manojkumar Bhosale (Manojkumar.Bhosale@imgtec.com)
	3	*
	4	* This file is part of FFmpeg.
	5	*
	6	* FFmpeg is free software; you can redistribute it and/or
	7	* modify it under the terms of the GNU Lesser General Public
	8	* License as published by the Free Software Foundation; either
	9	* version 2.1 of the License, or (at your option) any later version.
	10	*
	11	* FFmpeg is distributed in the hope that it will be useful,
	12	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	13	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	14	* Lesser General Public License for more details.
	15	*
	16	* You should have received a copy of the GNU Lesser General Public
	17	* License along with FFmpeg; if not, write to the Free Software
	18	* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
	19	*/
	20
	21	#include "libavutil/mips/generic_macros_msa.h"
	22	#include "libavcodec/mips/hevcdsp_mips.h"
	23	#include "libavcodec/mips/hevc_macros_msa.h"
	24
	25	#define HEVC_BIW_RND_CLIP2(in0, in1, vec0, vec1, wgt, rnd, offset, \
	26	out0_r, out1_r, out0_l, out1_l) \
	27	{ \
	28	ILVR_H2_SW(in0, vec0, in1, vec1, out0_r, out1_r); \
	29	ILVL_H2_SW(in0, vec0, in1, vec1, out0_l, out1_l); \
	30	\
	31	out0_r = __msa_dpadd_s_w(offset, (v8i16) out0_r, (v8i16) wgt); \
	32	out1_r = __msa_dpadd_s_w(offset, (v8i16) out1_r, (v8i16) wgt); \
	33	out0_l = __msa_dpadd_s_w(offset, (v8i16) out0_l, (v8i16) wgt); \
	34	out1_l = __msa_dpadd_s_w(offset, (v8i16) out1_l, (v8i16) wgt); \
	35	\
	36	SRAR_W4_SW(out0_r, out1_r, out0_l, out1_l, rnd); \
	37	\
	38	out0_r = CLIP_SW_0_255(out0_r); \
	39	out1_r = CLIP_SW_0_255(out1_r); \
	40	out0_l = CLIP_SW_0_255(out0_l); \
	41	out1_l = CLIP_SW_0_255(out1_l); \
	42	}
	43
	44	#define HEVC_BIW_RND_CLIP4(in0, in1, in2, in3, vec0, vec1, vec2, vec3, \
	45	wgt, rnd, offset, \
	46	out0_r, out1_r, out2_r, out3_r, \
	47	out0_l, out1_l, out2_l, out3_l) \
	48	{ \
	49	HEVC_BIW_RND_CLIP2(in0, in1, vec0, vec1, wgt, rnd, offset, \
	50	out0_r, out1_r, out0_l, out1_l) \
	51	HEVC_BIW_RND_CLIP2(in2, in3, vec2, vec3, wgt, rnd, offset, \
	52	out2_r, out3_r, out2_l, out3_l) \
	53	}
	54
	55	#define HEVC_BI_RND_CLIP2(in0, in1, vec0, vec1, rnd_val, out0, out1) \
	56	{ \
	57	ADDS_SH2_SH(vec0, in0, vec1, in1, out0, out1); \
	58	SRARI_H2_SH(out0, out1, rnd_val); \
	59	CLIP_SH2_0_255(out0, out1); \
	60	}
	61
	62	#define HEVC_BI_RND_CLIP4(in0, in1, in2, in3, \
	63	vec0, vec1, vec2, vec3, rnd_val, \
	64	out0, out1, out2, out3) \
	65	{ \
	66	HEVC_BI_RND_CLIP2(in0, in1, vec0, vec1, rnd_val, out0, out1); \
	67	HEVC_BI_RND_CLIP2(in2, in3, vec2, vec3, rnd_val, out2, out3); \
	68	}
	69
	70	static void hevc_biwgt_copy_4w_msa(uint8_t *src0_ptr,
	71	int32_t src_stride,
	72	int16_t *src1_ptr,
	73	int32_t src2_stride,
	74	uint8_t *dst,
	75	int32_t dst_stride,
	76	int32_t height,
	77	int32_t weight0,
	78	int32_t weight1,
	79	int32_t offset0,
	80	int32_t offset1,
	81	int32_t rnd_val)
	82	{
	83	int32_t offset, weight;
	84	v16i8 zero = { 0 };
	85	v4i32 weight_vec, offset_vec, rnd_vec;
	86
	87	offset = (offset0 + offset1) << rnd_val;
	88	weight0 = weight0 & 0x0000FFFF;
	89	weight = weight0 \| (weight1 << 16);
	90
	91	offset_vec = __msa_fill_w(offset);
	92	weight_vec = __msa_fill_w(weight);
	93	rnd_vec = __msa_fill_w(rnd_val + 1);
	94
	95	if (2 == height) {
	96	v16i8 src0, src1;
	97	v8i16 in0, in1, dst0;
	98	v4i32 dst0_r, dst0_l;
	99
	100	LD_SB2(src0_ptr, src_stride, src0, src1);
	101	LD_SH2(src1_ptr, src2_stride, in0, in1);
	102	in0 = (v8i16) __msa_ilvr_d((v2i64) in1, (v2i64) in0);
	103	src0 = (v16i8) __msa_ilvr_w((v4i32) src1, (v4i32) src0);
	104
	105	dst0 = (v8i16) __msa_ilvr_b(zero, src0);
	106	dst0 <<= 6;
	107
	108	ILVRL_H2_SW(dst0, in0, dst0_r, dst0_l);
	109	dst0_r = __msa_dpadd_s_w(offset_vec, (v8i16) dst0_r,
	110	(v8i16) weight_vec);
	111	dst0_l = __msa_dpadd_s_w(offset_vec, (v8i16) dst0_l,
	112	(v8i16) weight_vec);
	113	SRAR_W2_SW(dst0_r, dst0_l, rnd_vec);
	114	dst0_r = CLIP_SW_0_255(dst0_r);
	115	dst0_l = CLIP_SW_0_255(dst0_l);
	116
	117	HEVC_PCK_SW_SB2(dst0_l, dst0_r, dst0_r);
	118	ST4x2_UB(dst0_r, dst, dst_stride);
	119	} else if (4 == height) {
	120	v16i8 src0, src1, src2, src3;
	121	v8i16 in0, in1, in2, in3;
	122	v8i16 dst0, dst1;
	123	v4i32 dst0_r, dst1_r, dst0_l, dst1_l;
	124
	125	LD_SB4(src0_ptr, src_stride, src0, src1, src2, src3);
	126	LD_SH4(src1_ptr, src2_stride, in0, in1, in2, in3);
	127	ILVR_D2_SH(in1, in0, in3, in2, in0, in1);
	128	ILVR_W2_SB(src1, src0, src3, src2, src0, src1);
	129	ILVR_B2_SH(zero, src0, zero, src1, dst0, dst1);
	130	dst0 <<= 6;
	131	dst1 <<= 6;
	132	HEVC_BIW_RND_CLIP2(dst0, dst1, in0, in1,
	133	weight_vec, rnd_vec, offset_vec,
	134	dst0_r, dst1_r, dst0_l, dst1_l);
	135
	136	HEVC_PCK_SW_SB4(dst0_l, dst0_r, dst1_l, dst1_r, dst0_r);
	137	ST4x4_UB(dst0_r, dst0_r, 0, 1, 2, 3, dst, dst_stride);
	138	} else if (0 == height % 8) {
	139	uint32_t loop_cnt;
	140	v16i8 src0, src1, src2, src3, src4, src5, src6, src7;
	141	v8i16 in0, in1, in2, in3, in4, in5, in6, in7;
	142	v8i16 dst0, dst1, dst2, dst3;
	143	v4i32 dst0_r, dst1_r, dst2_r, dst3_r, dst0_l, dst1_l, dst2_l, dst3_l;
	144
	145	for (loop_cnt = (height >> 3); loop_cnt--;) {
	146	LD_SB8(src0_ptr, src_stride,
	147	src0, src1, src2, src3, src4, src5, src6, src7);
	148	src0_ptr += (8 * src_stride);
	149	LD_SH8(src1_ptr, src2_stride,
	150	in0, in1, in2, in3, in4, in5, in6, in7);
	151	src1_ptr += (8 * src2_stride);
	152
	153	ILVR_D2_SH(in1, in0, in3, in2, in0, in1);
	154	ILVR_D2_SH(in5, in4, in7, in6, in2, in3);
	155	ILVR_W4_SB(src1, src0, src3, src2, src5, src4, src7, src6,
	156	src0, src1, src2, src3);
	157	ILVR_B4_SH(zero, src0, zero, src1, zero, src2, zero, src3,
	158	dst0, dst1, dst2, dst3);
	159
	160	SLLI_4V(dst0, dst1, dst2, dst3, 6);
	161	HEVC_BIW_RND_CLIP4(dst0, dst1, dst2, dst3,
	162	in0, in1, in2, in3,
	163	weight_vec, rnd_vec, offset_vec,
	164	dst0_r, dst1_r, dst2_r, dst3_r,
	165	dst0_l, dst1_l, dst2_l, dst3_l);
	166
	167	HEVC_PCK_SW_SB8(dst0_l, dst0_r, dst1_l, dst1_r,
	168	dst2_l, dst2_r, dst3_l, dst3_r, dst0_r, dst1_r);
	169	ST4x8_UB(dst0_r, dst1_r, dst, dst_stride);
	170	dst += (8 * dst_stride);
	171	}
	172	}
	173	}
	174
	175	static void hevc_biwgt_copy_6w_msa(uint8_t *src0_ptr,
	176	int32_t src_stride,
	177	int16_t *src1_ptr,
	178	int32_t src2_stride,
	179	uint8_t *dst,
	180	int32_t dst_stride,
	181	int32_t height,
	182	int32_t weight0,
	183	int32_t weight1,
	184	int32_t offset0,
	185	int32_t offset1,
	186	int32_t rnd_val)
	187	{
	188	uint32_t loop_cnt;
	189	int32_t offset, weight;
	190	v16i8 zero = { 0 };
	191	v16i8 src0, src1, src2, src3;
	192	v8i16 in0, in1, in2, in3;
	193	v8i16 dst0, dst1, dst2, dst3;
	194	v4i32 dst0_r, dst1_r, dst2_r, dst3_r, dst0_l, dst1_l, dst2_l, dst3_l;
	195	v4i32 offset_vec, weight_vec, rnd_vec;
	196
	197	offset = (offset0 + offset1) << rnd_val;
	198	weight0 = weight0 & 0x0000FFFF;
	199	weight = weight0 \| (weight1 << 16);
	200
	201	weight_vec = __msa_fill_w(weight);
	202	offset_vec = __msa_fill_w(offset);
	203	rnd_vec = __msa_fill_w(rnd_val + 1);
	204
	205	for (loop_cnt = (height >> 2); loop_cnt--;) {
	206	LD_SB4(src0_ptr, src_stride, src0, src1, src2, src3);
	207	src0_ptr += (4 * src_stride);
	208	LD_SH4(src1_ptr, src2_stride, in0, in1, in2, in3);
	209	src1_ptr += (4 * src2_stride);
	210	ILVR_B4_SH(zero, src0, zero, src1, zero, src2, zero, src3,
	211	dst0, dst1, dst2, dst3);
	212
	213	SLLI_4V(dst0, dst1, dst2, dst3, 6);
	214	HEVC_BIW_RND_CLIP4(dst0, dst1, dst2, dst3,
	215	in0, in1, in2, in3,
	216	weight_vec, rnd_vec, offset_vec,
	217	dst0_r, dst1_r, dst2_r, dst3_r,
	218	dst0_l, dst1_l, dst2_l, dst3_l);
	219
	220	HEVC_PCK_SW_SB8(dst0_l, dst0_r, dst1_l, dst1_r,
	221	dst2_l, dst2_r, dst3_l, dst3_r, dst0_r, dst1_r);
	222	ST6x4_UB(dst0_r, dst1_r, dst, dst_stride);
	223	dst += (4 * dst_stride);
	224	}
	225	}
	226
	227	static void hevc_biwgt_copy_8w_msa(uint8_t *src0_ptr,
	228	int32_t src_stride,
	229	int16_t *src1_ptr,
	230	int32_t src2_stride,
	231	uint8_t *dst,
	232	int32_t dst_stride,
	233	int32_t height,
	234	int32_t weight0,
	235	int32_t weight1,
	236	int32_t offset0,
	237	int32_t offset1,
	238	int32_t rnd_val)
	239	{
	240	int32_t offset, weight;
	241	v16i8 zero = { 0 };
	242	v4i32 offset_vec, weight_vec, rnd_vec;
	243
	244	offset = (offset0 + offset1) << rnd_val;
	245	weight0 = weight0 & 0x0000FFFF;
	246	weight = weight0 \| (weight1 << 16);
	247
	248	offset_vec = __msa_fill_w(offset);
	249	weight_vec = __msa_fill_w(weight);
	250	rnd_vec = __msa_fill_w(rnd_val + 1);
	251
	252	if (2 == height) {
	253	v16i8 src0, src1;
	254	v8i16 in0, in1, dst0, dst1;
	255	v4i32 dst0_r, dst1_r, dst0_l, dst1_l;
	256
	257	LD_SB2(src0_ptr, src_stride, src0, src1);
	258	LD_SH2(src1_ptr, src2_stride, in0, in1);
	259
	260	ILVR_B2_SH(zero, src0, zero, src1, dst0, dst1);
	261
	262	dst0 <<= 6;
	263	dst1 <<= 6;
	264	HEVC_BIW_RND_CLIP2(dst0, dst1, in0, in1,
	265	weight_vec, rnd_vec, offset_vec,
	266	dst0_r, dst1_r, dst0_l, dst1_l);
	267
	268	HEVC_PCK_SW_SB4(dst0_l, dst0_r, dst1_l, dst1_r, dst0_r);
	269	ST8x2_UB(dst0_r, dst, dst_stride);
	270	} else if (6 == height) {
	271	v16i8 src0, src1, src2, src3, src4, src5;
	272	v8i16 in0, in1, in2, in3, in4, in5;
	273	v8i16 dst0, dst1, dst2, dst3, dst4, dst5;
	274	v4i32 dst0_r, dst1_r, dst2_r, dst3_r, dst4_r, dst5_r;
	275	v4i32 dst0_l, dst1_l, dst2_l, dst3_l, dst4_l, dst5_l;
	276
	277	LD_SB6(src0_ptr, src_stride, src0, src1, src2, src3, src4, src5);
	278	LD_SH6(src1_ptr, src2_stride, in0, in1, in2, in3, in4, in5);
	279	ILVR_B4_SH(zero, src0, zero, src1, zero, src2, zero, src3,
	280	dst0, dst1, dst2, dst3);
	281	ILVR_B2_SH(zero, src4, zero, src5, dst4, dst5);
	282
	283	SLLI_4V(dst0, dst1, dst2, dst3, 6);
	284	dst4 <<= 6;
	285	dst5 <<= 6;
	286	HEVC_BIW_RND_CLIP4(dst0, dst1, dst2, dst3,
	287	in0, in1, in2, in3,
	288	weight_vec, rnd_vec, offset_vec,
	289	dst0_r, dst1_r, dst2_r, dst3_r,
	290	dst0_l, dst1_l, dst2_l, dst3_l);
	291	HEVC_BIW_RND_CLIP2(dst4, dst5, in4, in5,
	292	weight_vec, rnd_vec, offset_vec,
	293	dst4_r, dst5_r, dst4_l, dst5_l);
	294
	295	HEVC_PCK_SW_SB12(dst0_l, dst0_r, dst1_l, dst1_r,
	296	dst2_l, dst2_r, dst3_l, dst3_r,
	297	dst4_l, dst4_r, dst5_l, dst5_r,
	298	dst0_r, dst1_r, dst2_r);
	299	ST8x4_UB(dst0_r, dst1_r, dst, dst_stride);
	300	dst += (4 * dst_stride);
	301	ST8x2_UB(dst2_r, dst, dst_stride);
	302	} else if (0 == height % 4) {
	303	uint32_t loop_cnt;
	304	v16i8 src0, src1, src2, src3;
	305	v8i16 in0, in1, in2, in3;
	306	v8i16 dst0, dst1, dst2, dst3;
	307	v4i32 dst0_r, dst1_r, dst2_r, dst3_r, dst0_l, dst1_l, dst2_l, dst3_l;
	308
	309	for (loop_cnt = (height >> 2); loop_cnt--;) {
	310	LD_SB4(src0_ptr, src_stride, src0, src1, src2, src3);
	311	src0_ptr += (4 * src_stride);
	312	LD_SH4(src1_ptr, src2_stride, in0, in1, in2, in3);
	313	src1_ptr += (4 * src2_stride);
	314	ILVR_B4_SH(zero, src0, zero, src1, zero, src2, zero, src3,
	315	dst0, dst1, dst2, dst3);
	316
	317	SLLI_4V(dst0, dst1, dst2, dst3, 6);
	318	HEVC_BIW_RND_CLIP4(dst0, dst1, dst2, dst3,
	319	in0, in1, in2, in3,
	320	weight_vec, rnd_vec, offset_vec,
	321	dst0_r, dst1_r, dst2_r, dst3_r,
	322	dst0_l, dst1_l, dst2_l, dst3_l);
	323
	324	HEVC_PCK_SW_SB8(dst0_l, dst0_r, dst1_l, dst1_r,
	325	dst2_l, dst2_r, dst3_l, dst3_r, dst0_r, dst1_r);
	326	ST8x4_UB(dst0_r, dst1_r, dst, dst_stride);
	327	dst += (4 * dst_stride);
	328	}
	329	}
	330	}
	331
	332	static void hevc_biwgt_copy_12w_msa(uint8_t *src0_ptr,
	333	int32_t src_stride,
	334	int16_t *src1_ptr,
	335	int32_t src2_stride,
	336	uint8_t *dst,
	337	int32_t dst_stride,
	338	int32_t height,
	339	int32_t weight0,
	340	int32_t weight1,
	341	int32_t offset0,
	342	int32_t offset1,
	343	int32_t rnd_val)
	344	{
	345	uint32_t loop_cnt;
	346	int32_t offset, weight;
	347	v16i8 zero = { 0 };
	348	v16i8 src0, src1, src2, src3;
	349	v8i16 in0, in1, in2, in3, in4, in5, in6, in7;
	350	v8i16 dst0, dst1, dst2, dst3, dst4, dst5;
	351	v4i32 dst0_r, dst1_r, dst2_r, dst3_r, dst4_r, dst5_r;
	352	v4i32 dst0_l, dst1_l, dst2_l, dst3_l, dst4_l, dst5_l;
	353	v4i32 offset_vec, weight_vec, rnd_vec;
	354
	355	offset = (offset0 + offset1) << rnd_val;
	356	weight0 = weight0 & 0x0000FFFF;
	357	weight = weight0 \| (weight1 << 16);
	358
	359	offset_vec = __msa_fill_w(offset);
	360	weight_vec = __msa_fill_w(weight);
	361	rnd_vec = __msa_fill_w(rnd_val + 1);
	362
	363	for (loop_cnt = (16 >> 2); loop_cnt--;) {
	364	LD_SB4(src0_ptr, src_stride, src0, src1, src2, src3);
	365	src0_ptr += (4 * src_stride);
	366	LD_SH4(src1_ptr, src2_stride, in0, in1, in2, in3);
	367	LD_SH4(src1_ptr + 8, src2_stride, in4, in5, in6, in7);
	368	src1_ptr += (4 * src2_stride);
	369
	370	ILVR_D2_SH(in5, in4, in7, in6, in4, in5);
	371	ILVR_B4_SH(zero, src0, zero, src1, zero, src2, zero, src3,
	372	dst0, dst1, dst2, dst3);
	373
	374	SLLI_4V(dst0, dst1, dst2, dst3, 6);
	375	ILVL_W2_SB(src1, src0, src3, src2, src0, src1);
	376	ILVR_B2_SH(zero, src0, zero, src1, dst4, dst5);
	377
	378	dst4 <<= 6;
	379	dst5 <<= 6;
	380	HEVC_BIW_RND_CLIP4(dst0, dst1, dst2, dst3,
	381	in0, in1, in2, in3,
	382	weight_vec, rnd_vec, offset_vec,
	383	dst0_r, dst1_r, dst2_r, dst3_r,
	384	dst0_l, dst1_l, dst2_l, dst3_l);
	385	HEVC_BIW_RND_CLIP2(dst4, dst5, in4, in5,
	386	weight_vec, rnd_vec, offset_vec,
	387	dst4_r, dst5_r, dst4_l, dst5_l);
	388
	389	HEVC_PCK_SW_SB12(dst0_l, dst0_r, dst1_l, dst1_r,
	390	dst2_l, dst2_r, dst3_l, dst3_r,
	391	dst4_l, dst4_r, dst5_l, dst5_r,
	392	dst0_r, dst1_r, dst2_r);
	393	ST12x4_UB(dst0_r, dst1_r, dst2_r, dst, dst_stride);
	394	dst += (4 * dst_stride);
	395	}
	396	}
	397
	398	static void hevc_biwgt_copy_16multx4mult_msa(uint8_t *src0_ptr,
	399	int32_t src_stride,
	400	int16_t *src1_ptr,
	401	int32_t src2_stride,
	402	uint8_t *dst,
	403	int32_t dst_stride,
	404	int32_t height,
	405	int32_t weight0,
	406	int32_t weight1,
	407	int32_t offset0,
	408	int32_t offset1,
	409	int32_t rnd_val,
	410	int32_t width)
	411	{
	412	uint32_t loop_cnt, cnt;
	413	uint8_t *src0_ptr_tmp;
	414	int16_t *src1_ptr_tmp;
	415	uint8_t *dst_tmp;
	416	int32_t offset, weight;
	417	v16i8 zero = { 0 };
	418	v4i32 offset_vec, weight_vec, rnd_vec;
	419
	420	offset = (offset0 + offset1) << rnd_val;
	421	weight0 = weight0 & 0x0000FFFF;
	422	weight = weight0 \| (weight1 << 16);
	423
	424	offset_vec = __msa_fill_w(offset);
	425	weight_vec = __msa_fill_w(weight);
	426	rnd_vec = __msa_fill_w(rnd_val + 1);
	427
	428	for (cnt = (width >> 4); cnt--;) {
	429	src0_ptr_tmp = src0_ptr;
	430	src1_ptr_tmp = src1_ptr;
	431	dst_tmp = dst;
	432
	433	for (loop_cnt = (height >> 2); loop_cnt--;) {
	434	v16i8 src0, src1, src2, src3;
	435	v8i16 in0, in1, in2, in3, in4, in5, in6, in7;
	436	v8i16 tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7;
	437	v4i32 dst0_r, dst1_r, dst2_r, dst3_r;
	438	v4i32 dst0_l, dst1_l, dst2_l, dst3_l;
	439
	440	LD_SB4(src0_ptr_tmp, src_stride, src0, src1, src2, src3);
	441	src0_ptr_tmp += (4 * src_stride);
	442	LD_SH4(src1_ptr_tmp, src2_stride, in0, in1, in2, in3);
	443	LD_SH4(src1_ptr_tmp + 8, src2_stride, in4, in5, in6, in7);
	444	src1_ptr_tmp += (4 * src2_stride);
	445
	446	ILVR_B4_SH(zero, src0, zero, src1, zero, src2, zero, src3,
	447	tmp0, tmp1, tmp2, tmp3);
	448	ILVL_B4_SH(zero, src0, zero, src1, zero, src2, zero, src3,
	449	tmp4, tmp5, tmp6, tmp7);
	450
	451	SLLI_4V(tmp0, tmp1, tmp2, tmp3, 6);
	452	SLLI_4V(tmp4, tmp5, tmp6, tmp7, 6);
	453	HEVC_BIW_RND_CLIP4(tmp0, tmp1, tmp4, tmp5,
	454	in0, in1, in4, in5,
	455	weight_vec, rnd_vec, offset_vec,
	456	dst0_r, dst1_r, dst2_r, dst3_r,
	457	dst0_l, dst1_l, dst2_l, dst3_l);
	458
	459	HEVC_PCK_SW_SB8(dst0_l, dst0_r, dst2_l, dst2_r,
	460	dst1_l, dst1_r, dst3_l, dst3_r, dst0_r, dst1_r);
	461	ST_SW2(dst0_r, dst1_r, dst_tmp, dst_stride);
	462	dst_tmp += (2 * dst_stride);
	463
	464	HEVC_BIW_RND_CLIP4(tmp2, tmp3, tmp6, tmp7,
	465	in2, in3, in6, in7,
	466	weight_vec, rnd_vec, offset_vec,
	467	dst0_r, dst1_r, dst2_r, dst3_r,
	468	dst0_l, dst1_l, dst2_l, dst3_l);
	469
	470	HEVC_PCK_SW_SB8(dst0_l, dst0_r, dst2_l, dst2_r,
	471	dst1_l, dst1_r, dst3_l, dst3_r, dst0_r, dst1_r);
	472	ST_SW2(dst0_r, dst1_r, dst_tmp, dst_stride);
	473	dst_tmp += (2 * dst_stride);
	474	}
	475
	476	src0_ptr += 16;
	477	src1_ptr += 16;
	478	dst += 16;
	479	}
	480	}
	481
	482	static void hevc_biwgt_copy_16w_msa(uint8_t *src0_ptr,
	483	int32_t src_stride,
	484	int16_t *src1_ptr,
	485	int32_t src2_stride,
	486	uint8_t *dst,
	487	int32_t dst_stride,
	488	int32_t height,
	489	int32_t weight0,
	490	int32_t weight1,
	491	int32_t offset0,
	492	int32_t offset1,
	493	int32_t rnd_val)
	494	{
	495	hevc_biwgt_copy_16multx4mult_msa(src0_ptr, src_stride,
	496	src1_ptr, src2_stride,
	497	dst, dst_stride, height, weight0,
	498	weight1, offset0, offset1, rnd_val, 16);
	499	}
	500
	501	static void hevc_biwgt_copy_24w_msa(uint8_t *src0_ptr,
	502	int32_t src_stride,
	503	int16_t *src1_ptr,
	504	int32_t src2_stride,
	505	uint8_t *dst,
	506	int32_t dst_stride,
	507	int32_t height,
	508	int32_t weight0,
	509	int32_t weight1,
	510	int32_t offset0,
	511	int32_t offset1,
	512	int32_t rnd_val)
	513	{
	514	hevc_biwgt_copy_16multx4mult_msa(src0_ptr, src_stride,
	515	src1_ptr, src2_stride,
	516	dst, dst_stride, height, weight0,
	517	weight1, offset0, offset1, rnd_val, 16);
	518	hevc_biwgt_copy_8w_msa(src0_ptr + 16, src_stride,
	519	src1_ptr + 16, src2_stride,
	520	dst + 16, dst_stride, height, weight0,
	521	weight1, offset0, offset1, rnd_val);
	522	}
	523
	524	static void hevc_biwgt_copy_32w_msa(uint8_t *src0_ptr,
	525	int32_t src_stride,
	526	int16_t *src1_ptr,
	527	int32_t src2_stride,
	528	uint8_t *dst,
	529	int32_t dst_stride,
	530	int32_t height,
	531	int32_t weight0,
	532	int32_t weight1,
	533	int32_t offset0,
	534	int32_t offset1,
	535	int32_t rnd_val)
	536	{
	537	hevc_biwgt_copy_16multx4mult_msa(src0_ptr, src_stride,
	538	src1_ptr, src2_stride,
	539	dst, dst_stride, height, weight0,
	540	weight1, offset0, offset1, rnd_val, 32);
	541	}
	542
	543	static void hevc_biwgt_copy_48w_msa(uint8_t *src0_ptr,
	544	int32_t src_stride,
	545	int16_t *src1_ptr,
	546	int32_t src2_stride,
	547	uint8_t *dst,
	548	int32_t dst_stride,
	549	int32_t height,
	550	int32_t weight0,
	551	int32_t weight1,
	552	int32_t offset0,
	553	int32_t offset1,
	554	int32_t rnd_val)
	555	{
	556	hevc_biwgt_copy_16multx4mult_msa(src0_ptr, src_stride,
	557	src1_ptr, src2_stride,
	558	dst, dst_stride, height, weight0,
	559	weight1, offset0, offset1, rnd_val, 48);
	560	}
	561
	562	static void hevc_biwgt_copy_64w_msa(uint8_t *src0_ptr,
	563	int32_t src_stride,
	564	int16_t *src1_ptr,
	565	int32_t src2_stride,
	566	uint8_t *dst,
	567	int32_t dst_stride,
	568	int32_t height,
	569	int32_t weight0,
	570	int32_t weight1,
	571	int32_t offset0,
	572	int32_t offset1,
	573	int32_t rnd_val)
	574	{
	575	hevc_biwgt_copy_16multx4mult_msa(src0_ptr, src_stride,
	576	src1_ptr, src2_stride,
	577	dst, dst_stride, height, weight0,
	578	weight1, offset0, offset1, rnd_val, 64);
	579	}
	580
	581	static void hevc_hz_biwgt_8t_4w_msa(uint8_t *src0_ptr,
	582	int32_t src_stride,
	583	int16_t *src1_ptr,
	584	int32_t src2_stride,
	585	uint8_t *dst,
	586	int32_t dst_stride,
	587	const int8_t *filter,
	588	int32_t height,
	589	int32_t weight0,
	590	int32_t weight1,
	591	int32_t offset0,
	592	int32_t offset1,
	593	int32_t rnd_val)
	594	{
	595	uint32_t loop_cnt;
	596	int32_t offset, weight;
	597	v8i16 filt0, filt1, filt2, filt3;
	598	v16i8 src0, src1, src2, src3;
	599	v16i8 mask1, mask2, mask3;
	600	v16i8 vec0, vec1, vec2, vec3;
	601	v8i16 dst0, dst1;
	602	v8i16 in0, in1, in2, in3;
	603	v4i32 dst0_r, dst1_r, dst0_l, dst1_l;
	604	v8i16 filter_vec, const_vec;
	605	v4i32 weight_vec, offset_vec, rnd_vec;
	606	v16i8 mask0 = { 0, 1, 1, 2, 2, 3, 3, 4, 16, 17, 17, 18, 18, 19, 19, 20 };
	607
	608	src0_ptr -= 3;
	609	filter_vec = LD_SH(filter);
	610	SPLATI_H4_SH(filter_vec, 0, 1, 2, 3, filt0, filt1, filt2, filt3);
	611
	612	mask1 = mask0 + 2;
	613	mask2 = mask0 + 4;
	614	mask3 = mask0 + 6;
	615
	616	offset = (offset0 + offset1) << rnd_val;
	617	weight0 = weight0 & 0x0000FFFF;
	618	weight = weight0 \| (weight1 << 16);
	619
	620	const_vec = __msa_ldi_h(128);
	621	const_vec <<= 6;
	622	offset_vec = __msa_fill_w(offset);
	623	weight_vec = __msa_fill_w(weight);
	624	rnd_vec = __msa_fill_w(rnd_val + 1);
	625
	626	for (loop_cnt = (height >> 2); loop_cnt--;) {
	627	LD_SB4(src0_ptr, src_stride, src0, src1, src2, src3);
	628	src0_ptr += (4 * src_stride);
	629	LD_SH4(src1_ptr, src2_stride, in0, in1, in2, in3);
	630	src1_ptr += (4 * src2_stride);
	631	ILVR_D2_SH(in1, in0, in3, in2, in0, in1);
	632	XORI_B4_128_SB(src0, src1, src2, src3);
	633
	634	VSHF_B4_SB(src0, src1, mask0, mask1, mask2, mask3,
	635	vec0, vec1, vec2, vec3);
	636	dst0 = const_vec;
	637	DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2, filt3,
	638	dst0, dst0, dst0, dst0);
	639	VSHF_B4_SB(src2, src3, mask0, mask1, mask2, mask3,
	640	vec0, vec1, vec2, vec3);
	641	dst1 = const_vec;
	642	DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2, filt3,
	643	dst1, dst1, dst1, dst1);
	644
	645	HEVC_BIW_RND_CLIP2(dst0, dst1, in0, in1,
	646	weight_vec, rnd_vec, offset_vec,
	647	dst0_r, dst1_r, dst0_l, dst1_l);
	648
	649	HEVC_PCK_SW_SB4(dst0_l, dst0_r, dst1_l, dst1_r, dst0_r);
	650	ST4x4_UB(dst0_r, dst0_r, 0, 1, 2, 3, dst, dst_stride);
	651	dst += (4 * dst_stride);
	652	}
	653	}
	654
	655	static void hevc_hz_biwgt_8t_8w_msa(uint8_t *src0_ptr,
	656	int32_t src_stride,
	657	int16_t *src1_ptr,
	658	int32_t src2_stride,
	659	uint8_t *dst,
	660	int32_t dst_stride,
	661	const int8_t *filter,
	662	int32_t height,
	663	int32_t weight0,
	664	int32_t weight1,
	665	int32_t offset0,
	666	int32_t offset1,
	667	int32_t rnd_val)
	668	{
	669	uint32_t loop_cnt;
	670	int32_t offset, weight;
	671	v8i16 filt0, filt1, filt2, filt3;
	672	v16i8 src0, src1, src2, src3;
	673	v16i8 mask1, mask2, mask3;
	674	v16i8 vec0, vec1, vec2, vec3;
	675	v8i16 dst0, dst1, dst2, dst3;
	676	v8i16 in0, in1, in2, in3;
	677	v4i32 dst0_r, dst1_r, dst0_l, dst1_l, dst2_r, dst3_r, dst2_l, dst3_l;
	678	v8i16 filter_vec, const_vec;
	679	v4i32 weight_vec, offset_vec, rnd_vec;
	680	v16i8 mask0 = { 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8 };
	681
	682	src0_ptr -= 3;
	683	offset = (offset0 + offset1) << rnd_val;
	684	weight0 = weight0 & 0x0000FFFF;
	685	weight = weight0 \| (weight1 << 16);
	686
	687	const_vec = __msa_ldi_h(128);
	688	const_vec <<= 6;
	689	offset_vec = __msa_fill_w(offset);
	690	weight_vec = __msa_fill_w(weight);
	691	rnd_vec = __msa_fill_w(rnd_val + 1);
	692
	693	filter_vec = LD_SH(filter);
	694	SPLATI_H4_SH(filter_vec, 0, 1, 2, 3, filt0, filt1, filt2, filt3);
	695
	696	mask1 = mask0 + 2;
	697	mask2 = mask0 + 4;
	698	mask3 = mask0 + 6;
	699
	700	for (loop_cnt = (height >> 2); loop_cnt--;) {
	701	LD_SB4(src0_ptr, src_stride, src0, src1, src2, src3);
	702	src0_ptr += (4 * src_stride);
	703	LD_SH4(src1_ptr, src2_stride, in0, in1, in2, in3);
	704	src1_ptr += (4 * src2_stride);
	705	XORI_B4_128_SB(src0, src1, src2, src3);
	706
	707	VSHF_B4_SB(src0, src0, mask0, mask1, mask2, mask3,
	708	vec0, vec1, vec2, vec3);
	709	dst0 = const_vec;
	710	DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2, filt3,
	711	dst0, dst0, dst0, dst0);
	712	VSHF_B4_SB(src1, src1, mask0, mask1, mask2, mask3,
	713	vec0, vec1, vec2, vec3);
	714	dst1 = const_vec;
	715	DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2, filt3,
	716	dst1, dst1, dst1, dst1);
	717	VSHF_B4_SB(src2, src2, mask0, mask1, mask2, mask3,
	718	vec0, vec1, vec2, vec3);
	719	dst2 = const_vec;
	720	DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2, filt3,
	721	dst2, dst2, dst2, dst2);
	722	VSHF_B4_SB(src3, src3, mask0, mask1, mask2, mask3,
	723	vec0, vec1, vec2, vec3);
	724	dst3 = const_vec;
	725	DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2, filt3,
	726	dst3, dst3, dst3, dst3);
	727
	728	HEVC_BIW_RND_CLIP4(dst0, dst1, dst2, dst3,
	729	in0, in1, in2, in3,
	730	weight_vec, rnd_vec, offset_vec,
	731	dst0_r, dst1_r, dst2_r, dst3_r,
	732	dst0_l, dst1_l, dst2_l, dst3_l);
	733
	734	HEVC_PCK_SW_SB8(dst0_l, dst0_r, dst1_l, dst1_r,
	735	dst2_l, dst2_r, dst3_l, dst3_r, dst0_r, dst1_r);
	736	ST8x4_UB(dst0_r, dst1_r, dst, dst_stride);
	737	dst += (4 * dst_stride);
	738	}
	739	}
	740
	741	static void hevc_hz_biwgt_8t_12w_msa(uint8_t *src0_ptr,
	742	int32_t src_stride,
	743	int16_t *src1_ptr,
	744	int32_t src2_stride,
	745	uint8_t *dst,
	746	int32_t dst_stride,
	747	const int8_t *filter,
	748	int32_t height,
	749	int32_t weight0,
	750	int32_t weight1,
	751	int32_t offset0,
	752	int32_t offset1,
	753	int32_t rnd_val)
	754	{
	755	hevc_hz_biwgt_8t_8w_msa(src0_ptr, src_stride, src1_ptr, src2_stride,
	756	dst, dst_stride, filter, height,
	757	weight0, weight1, offset0, offset1, rnd_val);
	758	hevc_hz_biwgt_8t_4w_msa(src0_ptr + 8, src_stride, src1_ptr + 8, src2_stride,
	759	dst + 8, dst_stride, filter, height,
	760	weight0, weight1, offset0, offset1, rnd_val);
	761	}
	762
	763	static void hevc_hz_biwgt_8t_16w_msa(uint8_t *src0_ptr,
	764	int32_t src_stride,
	765	int16_t *src1_ptr,
	766	int32_t src2_stride,
	767	uint8_t *dst,
	768	int32_t dst_stride,
	769	const int8_t *filter,
	770	int32_t height,
	771	int32_t weight0,
	772	int32_t weight1,
	773	int32_t offset0,
	774	int32_t offset1,
	775	int32_t rnd_val)
	776	{
	777	uint32_t loop_cnt;
	778	int32_t offset, weight;
	779	v16i8 src0, src1, src2, src3;
	780	v8i16 in0, in1, in2, in3;
	781	v8i16 filt0, filt1, filt2, filt3;
	782	v16i8 mask1, mask2, mask3;
	783	v8i16 filter_vec, const_vec;
	784	v16i8 vec0, vec1, vec2, vec3;
	785	v8i16 dst0, dst1, dst2, dst3;
	786	v4i32 dst0_r, dst1_r, dst2_r, dst3_r, dst0_l, dst1_l, dst2_l, dst3_l;
	787	v4i32 weight_vec, offset_vec, rnd_vec;
	788	v16i8 mask0 = { 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8 };
	789
	790	src0_ptr -= 3;
	791	offset = (offset0 + offset1) << rnd_val;
	792	weight0 = weight0 & 0x0000FFFF;
	793	weight = weight0 \| (weight1 << 16);
	794
	795	const_vec = __msa_ldi_h(128);
	796	const_vec <<= 6;
	797	offset_vec = __msa_fill_w(offset);
	798	weight_vec = __msa_fill_w(weight);
	799	rnd_vec = __msa_fill_w(rnd_val + 1);
	800
	801	filter_vec = LD_SH(filter);
	802	SPLATI_H4_SH(filter_vec, 0, 1, 2, 3, filt0, filt1, filt2, filt3);
	803
	804	mask1 = mask0 + 2;
	805	mask2 = mask0 + 4;
	806	mask3 = mask0 + 6;
	807
	808	for (loop_cnt = (height >> 1); loop_cnt--;) {
	809	LD_SB2(src0_ptr, 8, src0, src1);
	810	src0_ptr += src_stride;
	811	LD_SB2(src0_ptr, 8, src2, src3);
	812	src0_ptr += src_stride;
	813	LD_SH2(src1_ptr, 8, in0, in1);
	814	src1_ptr += src2_stride;
	815	LD_SH2(src1_ptr, 8, in2, in3);
	816	src1_ptr += src2_stride;
	817	XORI_B4_128_SB(src0, src1, src2, src3);
	818
	819	VSHF_B4_SB(src0, src0, mask0, mask1, mask2, mask3,
	820	vec0, vec1, vec2, vec3);
	821	dst0 = const_vec;
	822	DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2, filt3,
	823	dst0, dst0, dst0, dst0);
	824	VSHF_B4_SB(src1, src1, mask0, mask1, mask2, mask3,
	825	vec0, vec1, vec2, vec3);
	826	dst1 = const_vec;
	827	DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2, filt3,
	828	dst1, dst1, dst1, dst1);
	829	VSHF_B4_SB(src2, src2, mask0, mask1, mask2, mask3,
	830	vec0, vec1, vec2, vec3);
	831	dst2 = const_vec;
	832	DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2, filt3,
	833	dst2, dst2, dst2, dst2);
	834	VSHF_B4_SB(src3, src3, mask0, mask1, mask2, mask3,
	835	vec0, vec1, vec2, vec3);
	836	dst3 = const_vec;
	837	DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2, filt3,
	838	dst3, dst3, dst3, dst3);
	839
	840	HEVC_BIW_RND_CLIP4(dst0, dst1, dst2, dst3,
	841	in0, in1, in2, in3,
	842	weight_vec, rnd_vec, offset_vec,
	843	dst0_r, dst1_r, dst2_r, dst3_r,
	844	dst0_l, dst1_l, dst2_l, dst3_l);
	845
	846	HEVC_PCK_SW_SB8(dst0_l, dst0_r, dst1_l, dst1_r,
	847	dst2_l, dst2_r, dst3_l, dst3_r, dst0_r, dst1_r);
	848	ST_SW2(dst0_r, dst1_r, dst, dst_stride);
	849	dst += (2 * dst_stride);
	850	}
	851	}
	852
	853	static void hevc_hz_biwgt_8t_24w_msa(uint8_t *src0_ptr,
	854	int32_t src_stride,
	855	int16_t *src1_ptr,
	856	int32_t src2_stride,
	857	uint8_t *dst,
	858	int32_t dst_stride,
	859	const int8_t *filter,
	860	int32_t height,
	861	int32_t weight0,
	862	int32_t weight1,
	863	int32_t offset0,
	864	int32_t offset1,
	865	int32_t rnd_val)
	866	{
	867	uint32_t loop_cnt;
	868	uint64_t dst_val0;
	869	int32_t offset, weight;
	870	v16i8 src0, src1;
	871	v8i16 in0, in1, in2;
	872	v8i16 filt0, filt1, filt2, filt3;
	873	v16i8 mask1, mask2, mask3, mask4, mask5, mask6, mask7;
	874	v16i8 vec0, vec1, vec2, vec3;
	875	v8i16 dst0, dst1, dst2;
	876	v4i32 dst0_r, dst1_r, dst2_r, dst0_l, dst1_l, dst2_l;
	877	v8i16 filter_vec, const_vec;
	878	v4i32 weight_vec, offset_vec, rnd_vec;
	879	v16i8 mask0 = { 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8 };
	880
	881	src0_ptr = src0_ptr - 3;
	882	offset = (offset0 + offset1) << rnd_val;
	883	weight0 = weight0 & 0x0000FFFF;
	884	weight = weight0 \| (weight1 << 16);
	885
	886	const_vec = __msa_ldi_h(128);
	887	const_vec <<= 6;
	888	offset_vec = __msa_fill_w(offset);
	889	weight_vec = __msa_fill_w(weight);
	890	rnd_vec = __msa_fill_w(rnd_val + 1);
	891
	892	filter_vec = LD_SH(filter);
	893	SPLATI_H4_SH(filter_vec, 0, 1, 2, 3, filt0, filt1, filt2, filt3);
	894
	895	mask1 = mask0 + 2;
	896	mask2 = mask0 + 4;
	897	mask3 = mask0 + 6;
	898	mask4 = mask0 + 8;
	899	mask5 = mask0 + 10;
	900	mask6 = mask0 + 12;
	901	mask7 = mask0 + 14;
	902
	903	for (loop_cnt = height; loop_cnt--;) {
	904	LD_SB2(src0_ptr, 16, src0, src1);
	905	src0_ptr += src_stride;
	906	LD_SH2(src1_ptr, 8, in0, in1);
	907	in2 = LD_SH(src1_ptr + 16);
	908	src1_ptr += src2_stride;
	909	XORI_B2_128_SB(src0, src1);
	910
	911	VSHF_B4_SB(src0, src0, mask0, mask1, mask2, mask3,
	912	vec0, vec1, vec2, vec3);
	913	dst0 = const_vec;
	914	DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2, filt3,
	915	dst0, dst0, dst0, dst0);
	916	VSHF_B4_SB(src0, src1, mask4, mask5, mask6, mask7,
	917	vec0, vec1, vec2, vec3);
	918	dst1 = const_vec;
	919	DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2, filt3,
	920	dst1, dst1, dst1, dst1);
	921	VSHF_B4_SB(src1, src1, mask0, mask1, mask2, mask3,
	922	vec0, vec1, vec2, vec3);
	923	dst2 = const_vec;
	924	DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2, filt3,
	925	dst2, dst2, dst2, dst2);
	926
	927	HEVC_BIW_RND_CLIP2(dst0, dst1, in0, in1,
	928	weight_vec, rnd_vec, offset_vec,
	929	dst0_r, dst1_r, dst0_l, dst1_l);
	930
	931	ILVRL_H2_SW(dst2, in2, dst2_r, dst2_l);
	932	dst2_r = __msa_dpadd_s_w(offset_vec, (v8i16) dst2_r,
	933	(v8i16) weight_vec);
	934	dst2_l = __msa_dpadd_s_w(offset_vec, (v8i16) dst2_l,
	935	(v8i16) weight_vec);
	936	SRAR_W2_SW(dst2_r, dst2_l, rnd_vec);
	937	dst2_r = CLIP_SW_0_255(dst2_r);
	938	dst2_l = CLIP_SW_0_255(dst2_l);
	939
	940	HEVC_PCK_SW_SB4(dst0_l, dst0_r, dst1_l, dst1_r, dst0_r);
	941	HEVC_PCK_SW_SB2(dst2_l, dst2_r, dst2_r);
	942	dst_val0 = __msa_copy_u_d((v2i64) dst2_r, 0);
	943	ST_SW(dst0_r, dst);
	944	SD(dst_val0, dst + 16);
	945	dst += dst_stride;
	946	}
	947	}
	948
	949	static void hevc_hz_biwgt_8t_32w_msa(uint8_t *src0_ptr,
	950	int32_t src_stride,
	951	int16_t *src1_ptr,
	952	int32_t src2_stride,
	953	uint8_t *dst,
	954	int32_t dst_stride,
	955	const int8_t *filter,
	956	int32_t height,
	957	int32_t weight0,
	958	int32_t weight1,
	959	int32_t offset0,
	960	int32_t offset1,
	961	int32_t rnd_val)
	962	{
	963	uint32_t loop_cnt;
	964	int32_t offset, weight;
	965	v16i8 src0, src1, src2;
	966	v8i16 in0, in1, in2, in3;
	967	v8i16 filt0, filt1, filt2, filt3;
	968	v16i8 mask0 = { 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8 };
	969	v16i8 mask1, mask2, mask3, mask4, mask5, mask6, mask7;
	970	v16i8 vec0, vec1, vec2, vec3;
	971	v8i16 dst0, dst1, dst2, dst3;
	972	v8i16 filter_vec, const_vec;
	973	v4i32 dst0_r, dst1_r, dst2_r, dst3_r, dst0_l, dst1_l, dst2_l, dst3_l;
	974	v4i32 weight_vec, offset_vec, rnd_vec;
	975
	976	src0_ptr -= 3;
	977	offset = (offset0 + offset1) << rnd_val;
	978	weight0 = weight0 & 0x0000FFFF;
	979	weight = weight0 \| (weight1 << 16);
	980
	981	const_vec = __msa_ldi_h(128);
	982	const_vec <<= 6;
	983	offset_vec = __msa_fill_w(offset);
	984	weight_vec = __msa_fill_w(weight);
	985	rnd_vec = __msa_fill_w(rnd_val + 1);
	986
	987	filter_vec = LD_SH(filter);
	988	SPLATI_H4_SH(filter_vec, 0, 1, 2, 3, filt0, filt1, filt2, filt3);
	989
	990	mask1 = mask0 + 2;
	991	mask2 = mask0 + 4;
	992	mask3 = mask0 + 6;
	993	mask4 = mask0 + 8;
	994	mask5 = mask0 + 10;
	995	mask6 = mask0 + 12;
	996	mask7 = mask0 + 14;
	997
	998	for (loop_cnt = height; loop_cnt--;) {
	999	LD_SB2(src0_ptr, 16, src0, src1);
	1000	src2 = LD_SB(src0_ptr + 24);
	1001	src0_ptr += src_stride;
	1002	LD_SH4(src1_ptr, 8, in0, in1, in2, in3);
	1003	src1_ptr += src2_stride;
	1004
	1005	XORI_B3_128_SB(src0, src1, src2);
	1006
	1007	VSHF_B4_SB(src0, src0, mask0, mask1, mask2, mask3,
	1008	vec0, vec1, vec2, vec3);
	1009	dst0 = const_vec;
	1010	DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2, filt3,
	1011	dst0, dst0, dst0, dst0);
	1012	VSHF_B4_SB(src0, src1, mask4, mask5, mask6, mask7,
	1013	vec0, vec1, vec2, vec3);
	1014	dst1 = const_vec;
	1015	DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2, filt3,
	1016	dst1, dst1, dst1, dst1);
	1017	VSHF_B4_SB(src1, src1, mask0, mask1, mask2, mask3,
	1018	vec0, vec1, vec2, vec3);
	1019	dst2 = const_vec;
	1020	DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2, filt3,
	1021	dst2, dst2, dst2, dst2);
	1022	VSHF_B4_SB(src2, src2, mask0, mask1, mask2, mask3,
	1023	vec0, vec1, vec2, vec3);
	1024	dst3 = const_vec;
	1025	DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2, filt3,
	1026	dst3, dst3, dst3, dst3);
	1027
	1028	HEVC_BIW_RND_CLIP4(dst0, dst1, dst2, dst3,
	1029	in0, in1, in2, in3,
	1030	weight_vec, rnd_vec, offset_vec,
	1031	dst0_r, dst1_r, dst2_r, dst3_r,
	1032	dst0_l, dst1_l, dst2_l, dst3_l);
	1033
	1034	HEVC_PCK_SW_SB8(dst0_l, dst0_r, dst1_l, dst1_r,
	1035	dst2_l, dst2_r, dst3_l, dst3_r, dst0_r, dst1_r);
	1036	ST_SW2(dst0_r, dst1_r, dst, 16);
	1037	dst += dst_stride;
	1038	}
	1039	}
	1040
	1041	static void hevc_hz_biwgt_8t_48w_msa(uint8_t *src0_ptr,
	1042	int32_t src_stride,
	1043	int16_t *src1_ptr,
	1044	int32_t src2_stride,
	1045	uint8_t *dst,
	1046	int32_t dst_stride,
	1047	const int8_t *filter,
	1048	int32_t height,
	1049	int32_t weight0,
	1050	int32_t weight1,
	1051	int32_t offset0,
	1052	int32_t offset1,
	1053	int32_t rnd_val)
	1054	{
	1055	uint32_t loop_cnt;
	1056	int32_t offset, weight;
	1057	uint64_t dst_val0;
	1058	v16i8 src0, src1, src2, src3;
	1059	v8i16 in0, in1, in2, in3, in4, in5;
	1060	v8i16 filt0, filt1, filt2, filt3;
	1061	v16i8 mask0 = { 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8 };
	1062	v16i8 mask1, mask2, mask3, mask4, mask5, mask6, mask7;
	1063	v16i8 vec0, vec1, vec2, vec3;
	1064	v8i16 dst0, dst1, dst2, dst3, dst4, dst5;
	1065	v8i16 filter_vec, const_vec;
	1066	v4i32 dst0_r, dst1_r, dst2_r, dst3_r, dst4_r, dst5_r;
	1067	v4i32 dst0_l, dst1_l, dst2_l, dst3_l, dst4_l, dst5_l;
	1068	v4i32 weight_vec, offset_vec, rnd_vec;
	1069
	1070	src0_ptr -= 3;
	1071	offset = (offset0 + offset1) << rnd_val;
	1072	weight0 = weight0 & 0x0000FFFF;
	1073	weight = weight0 \| (weight1 << 16);
	1074
	1075	const_vec = __msa_ldi_h(128);
	1076	const_vec <<= 6;
	1077	offset_vec = __msa_fill_w(offset);
	1078	weight_vec = __msa_fill_w(weight);
	1079	rnd_vec = __msa_fill_w(rnd_val + 1);
	1080
	1081	filter_vec = LD_SH(filter);
	1082	SPLATI_H4_SH(filter_vec, 0, 1, 2, 3, filt0, filt1, filt2, filt3);
	1083
	1084	mask1 = mask0 + 2;
	1085	mask2 = mask0 + 4;
	1086	mask3 = mask0 + 6;
	1087	mask4 = mask0 + 8;
	1088	mask5 = mask0 + 10;
	1089	mask6 = mask0 + 12;
	1090	mask7 = mask0 + 14;
	1091
	1092	for (loop_cnt = height; loop_cnt--;) {
	1093	LD_SB3(src0_ptr, 16, src0, src1, src2);
	1094	src3 = LD_SB(src0_ptr + 40);
	1095	src0_ptr += src_stride;
	1096	LD_SH2(src1_ptr, 8, in0, in1);
	1097	in2 = LD_SH(src1_ptr + 16);
	1098	XORI_B4_128_SB(src0, src1, src2, src3);
	1099
	1100	VSHF_B4_SB(src0, src0, mask0, mask1, mask2, mask3,
	1101	vec0, vec1, vec2, vec3);
	1102	dst0 = const_vec;
	1103	DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2, filt3,
	1104	dst0, dst0, dst0, dst0);
	1105	VSHF_B4_SB(src0, src1, mask4, mask5, mask6, mask7,
	1106	vec0, vec1, vec2, vec3);
	1107	dst1 = const_vec;
	1108	DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2, filt3,
	1109	dst1, dst1, dst1, dst1);
	1110	VSHF_B4_SB(src1, src1, mask0, mask1, mask2, mask3,
	1111	vec0, vec1, vec2, vec3);
	1112	dst2 = const_vec;
	1113	DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2, filt3,
	1114	dst2, dst2, dst2, dst2);
	1115	VSHF_B4_SB(src1, src2, mask4, mask5, mask6, mask7,
	1116	vec0, vec1, vec2, vec3);
	1117	dst3 = const_vec;
	1118	DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2, filt3,
	1119	dst3, dst3, dst3, dst3);
	1120	VSHF_B4_SB(src2, src2, mask0, mask1, mask2, mask3,
	1121	vec0, vec1, vec2, vec3);
	1122	dst4 = const_vec;
	1123	DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2, filt3,
	1124	dst4, dst4, dst4, dst4);
	1125	VSHF_B4_SB(src3, src3, mask0, mask1, mask2, mask3,
	1126	vec0, vec1, vec2, vec3);
	1127	dst5 = const_vec;
	1128	DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2, filt3,
	1129	dst5, dst5, dst5, dst5);
	1130
	1131	HEVC_BIW_RND_CLIP2(dst0, dst1, in0, in1,
	1132	weight_vec, rnd_vec, offset_vec,
	1133	dst0_r, dst1_r, dst0_l, dst1_l);
	1134
	1135	ILVRL_H2_SW(dst2, in2, dst2_r, dst2_l);
	1136	dst2_r = __msa_dpadd_s_w(offset_vec, (v8i16) dst2_r,
	1137	(v8i16) weight_vec);
	1138	dst2_l = __msa_dpadd_s_w(offset_vec, (v8i16) dst2_l,
	1139	(v8i16) weight_vec);
	1140	SRAR_W2_SW(dst2_r, dst2_l, rnd_vec);
	1141	dst2_r = CLIP_SW_0_255(dst2_r);
	1142	dst2_l = CLIP_SW_0_255(dst2_l);
	1143
	1144	HEVC_PCK_SW_SB4(dst0_l, dst0_r, dst1_l, dst1_r, dst0_r);
	1145	HEVC_PCK_SW_SB2(dst2_l, dst2_r, dst2_r);
	1146	dst_val0 = __msa_copy_u_d((v2i64) dst2_r, 0);
	1147	ST_SW(dst0_r, dst);
	1148	SD(dst_val0, dst + 16);
	1149
	1150	LD_SH2(src1_ptr + 24, 8, in3, in4);
	1151	in5 = LD_SH(src1_ptr + 40);
	1152	src1_ptr += src2_stride;
	1153
	1154	HEVC_BIW_RND_CLIP2(dst3, dst4, in3, in4,
	1155	weight_vec, rnd_vec, offset_vec,
	1156	dst3_r, dst4_r, dst3_l, dst4_l);
	1157
	1158	ILVRL_H2_SW(dst5, in5, dst5_r, dst5_l);
	1159	dst5_r = __msa_dpadd_s_w(offset_vec, (v8i16) dst5_r,
	1160	(v8i16) weight_vec);
	1161	dst5_l = __msa_dpadd_s_w(offset_vec, (v8i16) dst5_l,
	1162	(v8i16) weight_vec);
	1163	SRAR_W2_SW(dst5_r, dst5_l, rnd_vec);
	1164	dst5_r = CLIP_SW_0_255(dst5_r);
	1165	dst5_l = CLIP_SW_0_255(dst5_l);
	1166
	1167	HEVC_PCK_SW_SB4(dst4_l, dst4_r, dst5_l, dst5_r, dst4_r);
	1168	HEVC_PCK_SW_SB2(dst3_l, dst3_r, dst3_r);
	1169	dst_val0 = __msa_copy_u_d((v2i64) dst3_r, 0);
	1170	SD(dst_val0, dst + 24);
	1171	ST_SW(dst4_r, dst + 32);
	1172	dst += dst_stride;
	1173	}
	1174	}
	1175
	1176	static void hevc_hz_biwgt_8t_64w_msa(uint8_t *src0_ptr,
	1177	int32_t src_stride,
	1178	int16_t *src1_ptr,
	1179	int32_t src2_stride,
	1180	uint8_t *dst,
	1181	int32_t dst_stride,
	1182	const int8_t *filter,
	1183	int32_t height,
	1184	int32_t weight0,
	1185	int32_t weight1,
	1186	int32_t offset0,
	1187	int32_t offset1,
	1188	int32_t rnd_val)
	1189	{
	1190	uint8_t *src0_ptr_tmp;
	1191	uint8_t *dst_tmp;
	1192	int16_t *src1_ptr_tmp;
	1193	uint32_t loop_cnt, cnt;
	1194	int32_t offset, weight;
	1195	v16i8 src0, src1, src2;
	1196	v8i16 in0, in1, in2, in3;
	1197	v8i16 filt0, filt1, filt2, filt3;
	1198	v16i8 mask0 = { 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8 };
	1199	v16i8 mask1, mask2, mask3, mask4, mask5, mask6, mask7;
	1200	v16i8 vec0, vec1, vec2, vec3;
	1201	v8i16 dst0, dst1, dst2, dst3;
	1202	v8i16 filter_vec, const_vec;
	1203	v4i32 dst0_r, dst1_r, dst2_r, dst3_r, dst0_l, dst1_l, dst2_l, dst3_l;
	1204	v4i32 weight_vec, offset_vec, rnd_vec;
	1205
	1206	src0_ptr -= 3;
	1207	offset = (offset0 + offset1) << rnd_val;
	1208	weight0 = weight0 & 0x0000FFFF;
	1209	weight = weight0 \| (weight1 << 16);
	1210
	1211	const_vec = __msa_ldi_h(128);
	1212	const_vec <<= 6;
	1213	offset_vec = __msa_fill_w(offset);
	1214	weight_vec = __msa_fill_w(weight);
	1215	rnd_vec = __msa_fill_w(rnd_val + 1);
	1216
	1217	filter_vec = LD_SH(filter);
	1218	SPLATI_H4_SH(filter_vec, 0, 1, 2, 3, filt0, filt1, filt2, filt3);
	1219
	1220	mask1 = mask0 + 2;
	1221	mask2 = mask0 + 4;
	1222	mask3 = mask0 + 6;
	1223	mask4 = mask0 + 8;
	1224	mask5 = mask0 + 10;
	1225	mask6 = mask0 + 12;
	1226	mask7 = mask0 + 14;
	1227
	1228	for (loop_cnt = height; loop_cnt--;) {
	1229	src0_ptr_tmp = src0_ptr;
	1230	dst_tmp = dst;
	1231	src1_ptr_tmp = src1_ptr;
	1232
	1233	for (cnt = 2; cnt--;) {
	1234	LD_SB2(src0_ptr_tmp, 16, src0, src1);
	1235	src2 = LD_SB(src0_ptr_tmp + 24);
	1236	src0_ptr_tmp += 32;
	1237	LD_SH4(src1_ptr_tmp, 8, in0, in1, in2, in3);
	1238	src1_ptr_tmp += 32;
	1239	XORI_B3_128_SB(src0, src1, src2);
	1240
	1241	VSHF_B4_SB(src0, src0, mask0, mask1, mask2, mask3,
	1242	vec0, vec1, vec2, vec3);
	1243	dst0 = const_vec;
	1244	DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2, filt3,
	1245	dst0, dst0, dst0, dst0);
	1246	VSHF_B4_SB(src0, src1, mask4, mask5, mask6, mask7,
	1247	vec0, vec1, vec2, vec3);
	1248	dst1 = const_vec;
	1249	DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2, filt3,
	1250	dst1, dst1, dst1, dst1);
	1251	VSHF_B4_SB(src1, src1, mask0, mask1, mask2, mask3,
	1252	vec0, vec1, vec2, vec3);
	1253	dst2 = const_vec;
	1254	DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2, filt3,
	1255	dst2, dst2, dst2, dst2);
	1256	VSHF_B4_SB(src2, src2, mask0, mask1, mask2, mask3,
	1257	vec0, vec1, vec2, vec3);
	1258	dst3 = const_vec;
	1259	DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2, filt3,
	1260	dst3, dst3, dst3, dst3);
	1261
	1262	HEVC_BIW_RND_CLIP4(dst0, dst1, dst2, dst3,
	1263	in0, in1, in2, in3,
	1264	weight_vec, rnd_vec, offset_vec,
	1265	dst0_r, dst1_r, dst2_r, dst3_r,
	1266	dst0_l, dst1_l, dst2_l, dst3_l);
	1267
	1268	HEVC_PCK_SW_SB8(dst0_l, dst0_r, dst1_l, dst1_r,
	1269	dst2_l, dst2_r, dst3_l, dst3_r, dst0_r, dst1_r);
	1270	ST_SW2(dst0_r, dst1_r, dst_tmp, 16);
	1271	dst_tmp += 32;
	1272	}
	1273
	1274	src0_ptr += src_stride;
	1275	src1_ptr += src2_stride;
	1276	dst += dst_stride;
	1277
	1278	}
	1279	}
	1280
	1281	static void hevc_vt_biwgt_8t_4w_msa(uint8_t *src0_ptr,
	1282	int32_t src_stride,
	1283	int16_t *src1_ptr,
	1284	int32_t src2_stride,
	1285	uint8_t *dst,
	1286	int32_t dst_stride,
	1287	const int8_t *filter,
	1288	int32_t height,
	1289	int32_t weight0,
	1290	int32_t weight1,
	1291	int32_t offset0,
	1292	int32_t offset1,
	1293	int32_t rnd_val)
	1294	{
	1295	uint32_t loop_cnt;
	1296	int32_t offset, weight;
	1297	v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10;
	1298	v16i8 src11, src12, src13, src14;
	1299	v8i16 in0, in1, in2, in3, in4, in5, in6, in7;
	1300	v16i8 src10_r, src32_r, src54_r, src76_r, src98_r;
	1301	v16i8 src21_r, src43_r, src65_r, src87_r, src109_r;
	1302	v16i8 src1110_r, src1211_r, src1312_r, src1413_r;
	1303	v16i8 src2110, src4332, src6554, src8776, src10998;
	1304	v16i8 src12111110, src14131312;
	1305	v8i16 dst10, dst32, dst54, dst76;
	1306	v4i32 dst10_r, dst32_r, dst54_r, dst76_r;
	1307	v4i32 dst10_l, dst32_l, dst54_l, dst76_l;
	1308	v8i16 filt0, filt1, filt2, filt3;
	1309	v8i16 filter_vec, const_vec;
	1310	v4i32 weight_vec, offset_vec, rnd_vec;
	1311
	1312	src0_ptr -= (3 * src_stride);
	1313	offset = (offset0 + offset1) << rnd_val;
	1314	weight0 = weight0 & 0x0000FFFF;
	1315	weight = weight0 \| (weight1 << 16);
	1316
	1317	const_vec = __msa_ldi_h(128);
	1318	const_vec <<= 6;
	1319	offset_vec = __msa_fill_w(offset);
	1320	weight_vec = __msa_fill_w(weight);
	1321	rnd_vec = __msa_fill_w(rnd_val + 1);
	1322
	1323	filter_vec = LD_SH(filter);
	1324	SPLATI_H4_SH(filter_vec, 0, 1, 2, 3, filt0, filt1, filt2, filt3);
	1325
	1326	LD_SB7(src0_ptr, src_stride, src0, src1, src2, src3, src4, src5, src6);
	1327	src0_ptr += (7 * src_stride);
	1328
	1329	ILVR_B4_SB(src1, src0, src3, src2, src5, src4, src2, src1,
	1330	src10_r, src32_r, src54_r, src21_r);
	1331	ILVR_B2_SB(src4, src3, src6, src5, src43_r, src65_r);
	1332	ILVR_D3_SB(src21_r, src10_r, src43_r, src32_r, src65_r, src54_r,
	1333	src2110, src4332, src6554);
	1334	XORI_B3_128_SB(src2110, src4332, src6554);
	1335
	1336	for (loop_cnt = (height >> 3); loop_cnt--;) {
	1337	LD_SB8(src0_ptr, src_stride,
	1338	src7, src8, src9, src10, src11, src12, src13, src14);
	1339	src0_ptr += (8 * src_stride);
	1340	LD_SH8(src1_ptr, src2_stride, in0, in1, in2, in3, in4, in5, in6, in7);
	1341	src1_ptr += (8 * src2_stride);
	1342
	1343	ILVR_D2_SH(in1, in0, in3, in2, in0, in1);
	1344	ILVR_D2_SH(in5, in4, in7, in6, in2, in3);
	1345	ILVR_B4_SB(src7, src6, src8, src7, src9, src8, src10, src9,
	1346	src76_r, src87_r, src98_r, src109_r);
	1347	ILVR_B4_SB(src11, src10, src12, src11, src13, src12, src14, src13,
	1348	src1110_r, src1211_r, src1312_r, src1413_r);
	1349	ILVR_D4_SB(src87_r, src76_r, src109_r, src98_r, src1211_r, src1110_r,
	1350	src1413_r, src1312_r,
	1351	src8776, src10998, src12111110, src14131312);
	1352	XORI_B4_128_SB(src8776, src10998, src12111110, src14131312);
	1353
	1354	dst10 = const_vec;
	1355	DPADD_SB4_SH(src2110, src4332, src6554, src8776, filt0, filt1,
	1356	filt2, filt3, dst10, dst10, dst10, dst10);
	1357	dst32 = const_vec;
	1358	DPADD_SB4_SH(src4332, src6554, src8776, src10998,
	1359	filt0, filt1, filt2, filt3, dst32, dst32, dst32, dst32);
	1360	dst54 = const_vec;
	1361	DPADD_SB4_SH(src6554, src8776, src10998, src12111110,
	1362	filt0, filt1, filt2, filt3, dst54, dst54, dst54, dst54);
	1363	dst76 = const_vec;
	1364	DPADD_SB4_SH(src8776, src10998, src12111110, src14131312,
	1365	filt0, filt1, filt2, filt3, dst76, dst76, dst76, dst76);
	1366
	1367	HEVC_BIW_RND_CLIP4(dst10, dst32, dst54, dst76,
	1368	in0, in1, in2, in3,
	1369	weight_vec, rnd_vec, offset_vec,
	1370	dst10_r, dst32_r, dst54_r, dst76_r,
	1371	dst10_l, dst32_l, dst54_l, dst76_l);
	1372
	1373	HEVC_PCK_SW_SB8(dst10_l, dst10_r, dst32_l, dst32_r,
	1374	dst54_l, dst54_r, dst76_l, dst76_r, dst10_r, dst54_r);
	1375	ST4x8_UB(dst10_r, dst54_r, dst, dst_stride);
	1376	dst += (8 * dst_stride);
	1377
	1378	src2110 = src10998;
	1379	src4332 = src12111110;
	1380	src6554 = src14131312;
	1381	src6 = src14;
	1382	}
	1383	}
	1384
	1385	static void hevc_vt_biwgt_8t_8w_msa(uint8_t *src0_ptr,
	1386	int32_t src_stride,
	1387	int16_t *src1_ptr,
	1388	int32_t src2_stride,
	1389	uint8_t *dst,
	1390	int32_t dst_stride,
	1391	const int8_t *filter,
	1392	int32_t height,
	1393	int32_t weight0,
	1394	int32_t weight1,
	1395	int32_t offset0,
	1396	int32_t offset1,
	1397	int32_t rnd_val)
	1398	{
	1399	uint32_t loop_cnt;
	1400	int32_t offset, weight;
	1401	v16i8 src0, src1, src2, src3, src4, src5;
	1402	v16i8 src6, src7, src8, src9, src10;
	1403	v8i16 in0, in1, in2, in3;
	1404	v16i8 src10_r, src32_r, src54_r, src76_r, src98_r;
	1405	v16i8 src21_r, src43_r, src65_r, src87_r, src109_r;
	1406	v8i16 tmp0, tmp1, tmp2, tmp3;
	1407	v8i16 filt0, filt1, filt2, filt3;
	1408	v8i16 filter_vec, const_vec;
	1409	v4i32 dst0_r, dst1_r, dst2_r, dst3_r, dst0_l, dst1_l, dst2_l, dst3_l;
	1410	v4i32 weight_vec, offset_vec, rnd_vec;
	1411
	1412	src0_ptr -= (3 * src_stride);
	1413	offset = (offset0 + offset1) << rnd_val;
	1414	weight0 = weight0 & 0x0000FFFF;
	1415	weight = weight0 \| (weight1 << 16);
	1416
	1417	const_vec = __msa_ldi_h(128);
	1418	const_vec <<= 6;
	1419	offset_vec = __msa_fill_w(offset);
	1420	weight_vec = __msa_fill_w(weight);
	1421	rnd_vec = __msa_fill_w(rnd_val + 1);
	1422
	1423	filter_vec = LD_SH(filter);
	1424	SPLATI_H4_SH(filter_vec, 0, 1, 2, 3, filt0, filt1, filt2, filt3);
	1425
	1426	LD_SB7(src0_ptr, src_stride, src0, src1, src2, src3, src4, src5, src6);
	1427	src0_ptr += (7 * src_stride);
	1428	XORI_B7_128_SB(src0, src1, src2, src3, src4, src5, src6);
	1429
	1430	ILVR_B4_SB(src1, src0, src3, src2, src5, src4, src2, src1,
	1431	src10_r, src32_r, src54_r, src21_r);
	1432	ILVR_B2_SB(src4, src3, src6, src5, src43_r, src65_r);
	1433
	1434	for (loop_cnt = (height >> 2); loop_cnt--;) {
	1435	LD_SB4(src0_ptr, src_stride, src7, src8, src9, src10);
	1436	src0_ptr += (4 * src_stride);
	1437	LD_SH4(src1_ptr, src2_stride, in0, in1, in2, in3);
	1438	src1_ptr += (4 * src2_stride);
	1439
	1440	XORI_B4_128_SB(src7, src8, src9, src10);
	1441	ILVR_B4_SB(src7, src6, src8, src7, src9, src8, src10, src9,
	1442	src76_r, src87_r, src98_r, src109_r);
	1443
	1444	tmp0 = const_vec;
	1445	DPADD_SB4_SH(src10_r, src32_r, src54_r, src76_r,
	1446	filt0, filt1, filt2, filt3, tmp0, tmp0, tmp0, tmp0);
	1447	tmp1 = const_vec;
	1448	DPADD_SB4_SH(src21_r, src43_r, src65_r, src87_r,
	1449	filt0, filt1, filt2, filt3, tmp1, tmp1, tmp1, tmp1);
	1450	tmp2 = const_vec;
	1451	DPADD_SB4_SH(src32_r, src54_r, src76_r, src98_r,
	1452	filt0, filt1, filt2, filt3, tmp2, tmp2, tmp2, tmp2);
	1453	tmp3 = const_vec;
	1454	DPADD_SB4_SH(src43_r, src65_r, src87_r, src109_r,
	1455	filt0, filt1, filt2, filt3, tmp3, tmp3, tmp3, tmp3);
	1456
	1457	HEVC_BIW_RND_CLIP4(tmp0, tmp1, tmp2, tmp3,
	1458	in0, in1, in2, in3,
	1459	weight_vec, rnd_vec, offset_vec,
	1460	dst0_r, dst1_r, dst2_r, dst3_r,
	1461	dst0_l, dst1_l, dst2_l, dst3_l);
	1462
	1463	HEVC_PCK_SW_SB8(dst0_l, dst0_r, dst1_l, dst1_r,
	1464	dst2_l, dst2_r, dst3_l, dst3_r, dst0_r, dst1_r);
	1465	ST8x4_UB(dst0_r, dst1_r, dst, dst_stride);
	1466	dst += (4 * dst_stride);
	1467
	1468	src10_r = src54_r;
	1469	src32_r = src76_r;
	1470	src54_r = src98_r;
	1471	src21_r = src65_r;
	1472	src43_r = src87_r;
	1473	src65_r = src109_r;
	1474	src6 = src10;
	1475	}
	1476	}
	1477
	1478	static void hevc_vt_biwgt_8t_12w_msa(uint8_t *src0_ptr,
	1479	int32_t src_stride,
	1480	int16_t *src1_ptr,
	1481	int32_t src2_stride,
	1482	uint8_t *dst,
	1483	int32_t dst_stride,
	1484	const int8_t *filter,
	1485	int32_t height,
	1486	int32_t weight0,
	1487	int32_t weight1,
	1488	int32_t offset0,
	1489	int32_t offset1,
	1490	int32_t rnd_val)
	1491	{
	1492	uint32_t loop_cnt;
	1493	int32_t offset, weight;
	1494	v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8;
	1495	v8i16 in0, in1, in2, in3;
	1496	v16i8 src10_r, src32_r, src54_r, src76_r;
	1497	v16i8 src21_r, src43_r, src65_r, src87_r;
	1498	v8i16 tmp0, tmp1, tmp2;
	1499	v16i8 src10_l, src32_l, src54_l, src76_l;
	1500	v16i8 src21_l, src43_l, src65_l, src87_l;
	1501	v16i8 src2110, src4332, src6554, src8776;
	1502	v8i16 filt0, filt1, filt2, filt3;
	1503	v8i16 filter_vec, const_vec;
	1504	v4i32 dst0_r, dst1_r, dst2_r, dst0_l, dst1_l, dst2_l;
	1505	v4i32 weight_vec, offset_vec, rnd_vec;
	1506
	1507	src0_ptr -= (3 * src_stride);
	1508	offset = (offset0 + offset1) << rnd_val;
	1509	weight0 = weight0 & 0x0000FFFF;
	1510	weight = weight0 \| (weight1 << 16);
	1511
	1512	const_vec = __msa_ldi_h(128);
	1513	const_vec <<= 6;
	1514	offset_vec = __msa_fill_w(offset);
	1515	weight_vec = __msa_fill_w(weight);
	1516	rnd_vec = __msa_fill_w(rnd_val + 1);
	1517
	1518	filter_vec = LD_SH(filter);
	1519	SPLATI_H4_SH(filter_vec, 0, 1, 2, 3, filt0, filt1, filt2, filt3);
	1520
	1521	LD_SB7(src0_ptr, src_stride, src0, src1, src2, src3, src4, src5, src6);
	1522	src0_ptr += (7 * src_stride);
	1523	XORI_B7_128_SB(src0, src1, src2, src3, src4, src5, src6);
	1524
	1525	ILVR_B4_SB(src1, src0, src3, src2, src5, src4, src2, src1,
	1526	src10_r, src32_r, src54_r, src21_r);
	1527	ILVR_B2_SB(src4, src3, src6, src5, src43_r, src65_r);
	1528	ILVL_B4_SB(src1, src0, src3, src2, src5, src4, src2, src1,
	1529	src10_l, src32_l, src54_l, src21_l);
	1530	ILVL_B2_SB(src4, src3, src6, src5, src43_l, src65_l);
	1531	ILVR_D3_SB(src21_l, src10_l, src43_l, src32_l, src65_l, src54_l,
	1532	src2110, src4332, src6554);
	1533
	1534	for (loop_cnt = (height >> 1); loop_cnt--;) {
	1535	LD_SB2(src0_ptr, src_stride, src7, src8);
	1536	src0_ptr += (2 * src_stride);
	1537	LD_SH2(src1_ptr, src2_stride, in0, in1);
	1538	LD_SH2((src1_ptr + 8), src2_stride, in2, in3);
	1539	src1_ptr += (2 * src2_stride);
	1540	in2 = (v8i16) __msa_ilvr_d((v2i64) in3, (v2i64) in2);
	1541	XORI_B2_128_SB(src7, src8);
	1542
	1543	ILVR_B2_SB(src7, src6, src8, src7, src76_r, src87_r);
	1544	ILVL_B2_SB(src7, src6, src8, src7, src76_l, src87_l);
	1545	src8776 = (v16i8) __msa_ilvr_d((v2i64) src87_l, (v2i64) src76_l);
	1546
	1547	tmp0 = const_vec;
	1548	DPADD_SB4_SH(src10_r, src32_r, src54_r, src76_r,
	1549	filt0, filt1, filt2, filt3, tmp0, tmp0, tmp0, tmp0);
	1550	tmp1 = const_vec;
	1551	DPADD_SB4_SH(src21_r, src43_r, src65_r, src87_r,
	1552	filt0, filt1, filt2, filt3, tmp1, tmp1, tmp1, tmp1);
	1553	tmp2 = const_vec;
	1554	DPADD_SB4_SH(src2110, src4332, src6554, src8776,
	1555	filt0, filt1, filt2, filt3, tmp2, tmp2, tmp2, tmp2);
	1556
	1557	HEVC_BIW_RND_CLIP2(tmp0, tmp1, in0, in1,
	1558	weight_vec, rnd_vec, offset_vec,
	1559	dst0_r, dst1_r, dst0_l, dst1_l);
	1560
	1561	ILVRL_H2_SW(tmp2, in2, dst2_r, dst2_l);
	1562	dst2_r = __msa_dpadd_s_w(offset_vec, (v8i16) dst2_r,
	1563	(v8i16) weight_vec);
	1564	dst2_l = __msa_dpadd_s_w(offset_vec, (v8i16) dst2_l,
	1565	(v8i16) weight_vec);
	1566	SRAR_W2_SW(dst2_r, dst2_l, rnd_vec);
	1567	dst2_r = CLIP_SW_0_255(dst2_r);
	1568	dst2_l = CLIP_SW_0_255(dst2_l);
	1569
	1570	HEVC_PCK_SW_SB4(dst0_l, dst0_r, dst1_l, dst1_r, dst0_r);
	1571	HEVC_PCK_SW_SB2(dst2_l, dst2_r, dst2_r);
	1572	ST8x2_UB(dst0_r, dst, dst_stride);
	1573	ST4x2_UB(dst2_r, dst + 8, dst_stride);
	1574	dst += (2 * dst_stride);
	1575
	1576	src10_r = src32_r;
	1577	src32_r = src54_r;
	1578	src54_r = src76_r;
	1579	src21_r = src43_r;
	1580	src43_r = src65_r;
	1581	src65_r = src87_r;
	1582	src2110 = src4332;
	1583	src4332 = src6554;
	1584	src6554 = src8776;
	1585	src6 = src8;
	1586	}
	1587	}
	1588
	1589	static void hevc_vt_biwgt_8t_16multx2mult_msa(uint8_t *src0_ptr,
	1590	int32_t src_stride,
	1591	int16_t *src1_ptr,
	1592	int32_t src2_stride,
	1593	uint8_t *dst,
	1594	int32_t dst_stride,
	1595	const int8_t *filter,
	1596	int32_t height,
	1597	int32_t weight0,
	1598	int32_t weight1,
	1599	int32_t offset0,
	1600	int32_t offset1,
	1601	int32_t rnd_val,
	1602	int32_t width)
	1603	{
	1604	uint8_t *src0_ptr_tmp;
	1605	int16_t *src1_ptr_tmp;
	1606	uint8_t *dst_tmp;
	1607	uint32_t loop_cnt, cnt;
	1608	int32_t offset, weight;
	1609	v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8;
	1610	v8i16 in0, in1, in2, in3;
	1611	v16i8 src10_r, src32_r, src54_r, src76_r;
	1612	v16i8 src21_r, src43_r, src65_r, src87_r;
	1613	v16i8 src10_l, src32_l, src54_l, src76_l;
	1614	v16i8 src21_l, src43_l, src65_l, src87_l;
	1615	v8i16 tmp0, tmp1, tmp2, tmp3;
	1616	v8i16 filt0, filt1, filt2, filt3;
	1617	v8i16 filter_vec, const_vec;
	1618	v4i32 dst0_r, dst1_r, dst2_r, dst3_r, dst0_l, dst1_l, dst2_l, dst3_l;
	1619	v4i32 weight_vec, offset_vec, rnd_vec;
	1620
	1621	src0_ptr -= (3 * src_stride);
	1622
	1623	offset = (offset0 + offset1) << rnd_val;
	1624	weight0 = weight0 & 0x0000FFFF;
	1625	weight = weight0 \| (weight1 << 16);
	1626
	1627	const_vec = __msa_ldi_h(128);
	1628	const_vec <<= 6;
	1629	offset_vec = __msa_fill_w(offset);
	1630	weight_vec = __msa_fill_w(weight);
	1631	rnd_vec = __msa_fill_w(rnd_val + 1);
	1632
	1633	filter_vec = LD_SH(filter);
	1634	SPLATI_H4_SH(filter_vec, 0, 1, 2, 3, filt0, filt1, filt2, filt3);
	1635
	1636	for (cnt = (width >> 4); cnt--;) {
	1637	src0_ptr_tmp = src0_ptr;
	1638	src1_ptr_tmp = src1_ptr;
	1639	dst_tmp = dst;
	1640
	1641	LD_SB7(src0_ptr_tmp, src_stride,
	1642	src0, src1, src2, src3, src4, src5, src6);
	1643	src0_ptr_tmp += (7 * src_stride);
	1644
	1645	XORI_B7_128_SB(src0, src1, src2, src3, src4, src5, src6);
	1646	ILVR_B4_SB(src1, src0, src3, src2, src5, src4, src2, src1,
	1647	src10_r, src32_r, src54_r, src21_r);
	1648	ILVR_B2_SB(src4, src3, src6, src5, src43_r, src65_r);
	1649	ILVL_B4_SB(src1, src0, src3, src2, src5, src4, src2, src1,
	1650	src10_l, src32_l, src54_l, src21_l);
	1651	ILVL_B2_SB(src4, src3, src6, src5, src43_l, src65_l);
	1652
	1653	for (loop_cnt = (height >> 1); loop_cnt--;) {
	1654	LD_SB2(src0_ptr_tmp, src_stride, src7, src8);
	1655	src0_ptr_tmp += (2 * src_stride);
	1656	LD_SH2(src1_ptr_tmp, src2_stride, in0, in1);
	1657	LD_SH2((src1_ptr_tmp + 8), src2_stride, in2, in3);
	1658	src1_ptr_tmp += (2 * src2_stride);
	1659
	1660	XORI_B2_128_SB(src7, src8);
	1661	ILVR_B2_SB(src7, src6, src8, src7, src76_r, src87_r);
	1662	ILVL_B2_SB(src7, src6, src8, src7, src76_l, src87_l);
	1663
	1664	tmp0 = const_vec;
	1665	DPADD_SB4_SH(src10_r, src32_r, src54_r, src76_r,
	1666	filt0, filt1, filt2, filt3, tmp0, tmp0, tmp0, tmp0);
	1667	tmp1 = const_vec;
	1668	DPADD_SB4_SH(src21_r, src43_r, src65_r, src87_r,
	1669	filt0, filt1, filt2, filt3, tmp1, tmp1, tmp1, tmp1);
	1670	tmp2 = const_vec;
	1671	DPADD_SB4_SH(src10_l, src32_l, src54_l, src76_l,
	1672	filt0, filt1, filt2, filt3, tmp2, tmp2, tmp2, tmp2);
	1673	tmp3 = const_vec;
	1674	DPADD_SB4_SH(src21_l, src43_l, src65_l, src87_l,
	1675	filt0, filt1, filt2, filt3, tmp3, tmp3, tmp3, tmp3);
	1676
	1677	HEVC_BIW_RND_CLIP4(tmp0, tmp1, tmp2, tmp3,
	1678	in0, in1, in2, in3,
	1679	weight_vec, rnd_vec, offset_vec,
	1680	dst0_r, dst1_r, dst2_r, dst3_r,
	1681	dst0_l, dst1_l, dst2_l, dst3_l);
	1682
	1683	HEVC_PCK_SW_SB8(dst0_l, dst0_r, dst2_l, dst2_r,
	1684	dst1_l, dst1_r, dst3_l, dst3_r, dst0_r, dst1_r);
	1685	ST_SW2(dst0_r, dst1_r, dst_tmp, dst_stride);
	1686	dst_tmp += (2 * dst_stride);
	1687
	1688	src10_r = src32_r;
	1689	src32_r = src54_r;
	1690	src54_r = src76_r;
	1691	src21_r = src43_r;
	1692	src43_r = src65_r;
	1693	src65_r = src87_r;
	1694	src10_l = src32_l;
	1695	src32_l = src54_l;
	1696	src54_l = src76_l;
	1697	src21_l = src43_l;
	1698	src43_l = src65_l;
	1699	src65_l = src87_l;
	1700	src6 = src8;
	1701	}
	1702
	1703	src0_ptr += 16;
	1704	src1_ptr += 16;
	1705	dst += 16;
	1706	}
	1707	}
	1708
	1709	static void hevc_vt_biwgt_8t_16w_msa(uint8_t *src0_ptr,
	1710	int32_t src_stride,
	1711	int16_t *src1_ptr,
	1712	int32_t src2_stride,
	1713	uint8_t *dst,
	1714	int32_t dst_stride,
	1715	const int8_t *filter,
	1716	int32_t height,
	1717	int32_t weight0,
	1718	int32_t weight1,
	1719	int32_t offset0,
	1720	int32_t offset1,
	1721	int32_t rnd_val)
	1722	{
	1723	hevc_vt_biwgt_8t_16multx2mult_msa(src0_ptr, src_stride,
	1724	src1_ptr, src2_stride,
	1725	dst, dst_stride, filter, height,
	1726	weight0, weight1, offset0, offset1,
	1727	rnd_val, 16);
	1728	}
	1729
	1730	static void hevc_vt_biwgt_8t_24w_msa(uint8_t *src0_ptr,
	1731	int32_t src_stride,
	1732	int16_t *src1_ptr,
	1733	int32_t src2_stride,
	1734	uint8_t *dst,
	1735	int32_t dst_stride,
	1736	const int8_t *filter,
	1737	int32_t height,
	1738	int32_t weight0,
	1739	int32_t weight1,
	1740	int32_t offset0,
	1741	int32_t offset1,
	1742	int32_t rnd_val)
	1743	{
	1744	hevc_vt_biwgt_8t_16multx2mult_msa(src0_ptr, src_stride,
	1745	src1_ptr, src2_stride,
	1746	dst, dst_stride, filter, height,
	1747	weight0, weight1, offset0, offset1,
	1748	rnd_val, 16);
	1749	hevc_vt_biwgt_8t_8w_msa(src0_ptr + 16, src_stride,
	1750	src1_ptr + 16, src2_stride,
	1751	dst + 16, dst_stride, filter, height,
	1752	weight0, weight1, offset0, offset1, rnd_val);
	1753	}
	1754
	1755	static void hevc_vt_biwgt_8t_32w_msa(uint8_t *src0_ptr,
	1756	int32_t src_stride,
	1757	int16_t *src1_ptr,
	1758	int32_t src2_stride,
	1759	uint8_t *dst,
	1760	int32_t dst_stride,
	1761	const int8_t *filter,
	1762	int32_t height,
	1763	int32_t weight0,
	1764	int32_t weight1,
	1765	int32_t offset0,
	1766	int32_t offset1,
	1767	int32_t rnd_val)
	1768	{
	1769	hevc_vt_biwgt_8t_16multx2mult_msa(src0_ptr, src_stride,
	1770	src1_ptr, src2_stride,
	1771	dst, dst_stride, filter, height,
	1772	weight0, weight1, offset0, offset1,
	1773	rnd_val, 32);
	1774	}
	1775
	1776	static void hevc_vt_biwgt_8t_48w_msa(uint8_t *src0_ptr,
	1777	int32_t src_stride,
	1778	int16_t *src1_ptr,
	1779	int32_t src2_stride,
	1780	uint8_t *dst,
	1781	int32_t dst_stride,
	1782	const int8_t *filter,
	1783	int32_t height,
	1784	int32_t weight0,
	1785	int32_t weight1,
	1786	int32_t offset0,
	1787	int32_t offset1,
	1788	int32_t rnd_val)
	1789	{
	1790	hevc_vt_biwgt_8t_16multx2mult_msa(src0_ptr, src_stride,
	1791	src1_ptr, src2_stride,
	1792	dst, dst_stride, filter, height,
	1793	weight0, weight1, offset0, offset1,
	1794	rnd_val, 48);
	1795	}
	1796
	1797	static void hevc_vt_biwgt_8t_64w_msa(uint8_t *src0_ptr,
	1798	int32_t src_stride,
	1799	int16_t *src1_ptr,
	1800	int32_t src2_stride,
	1801	uint8_t *dst,
	1802	int32_t dst_stride,
	1803	const int8_t *filter,
	1804	int32_t height,
	1805	int32_t weight0,
	1806	int32_t weight1,
	1807	int32_t offset0,
	1808	int32_t offset1,
	1809	int32_t rnd_val)
	1810	{
	1811	hevc_vt_biwgt_8t_16multx2mult_msa(src0_ptr, src_stride,
	1812	src1_ptr, src2_stride,
	1813	dst, dst_stride, filter, height,
	1814	weight0, weight1, offset0, offset1,
	1815	rnd_val, 64);
	1816	}
	1817
	1818	static void hevc_hv_biwgt_8t_4w_msa(uint8_t *src0_ptr,
	1819	int32_t src_stride,
	1820	int16_t *src1_ptr,
	1821	int32_t src2_stride,
	1822	uint8_t *dst,
	1823	int32_t dst_stride,
	1824	const int8_t *filter_x,
	1825	const int8_t *filter_y,
	1826	int32_t height,
	1827	int32_t weight0,
	1828	int32_t weight1,
	1829	int32_t offset0,
	1830	int32_t offset1,
	1831	int32_t rnd_val)
	1832	{
	1833	uint32_t loop_cnt;
	1834	int32_t offset;
	1835	v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8;
	1836	v8i16 in0, in1;
	1837	v8i16 filt0, filt1, filt2, filt3;
	1838	v4i32 filt_h0, filt_h1, filt_h2, filt_h3;
	1839	v16i8 mask1, mask2, mask3;
	1840	v8i16 filter_vec, const_vec;
	1841	v16i8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
	1842	v16i8 vec8, vec9, vec10, vec11, vec12, vec13, vec14, vec15;
	1843	v8i16 dst30, dst41, dst52, dst63, dst66, dst87;
	1844	v4i32 dst0_r, dst1_r;
	1845	v4i32 tmp1, tmp2;
	1846	v4i32 weight_vec0, weight_vec1, offset_vec, rnd_vec;
	1847	v8i16 dst10_r, dst32_r, dst54_r, dst76_r;
	1848	v8i16 dst21_r, dst43_r, dst65_r, dst87_r;
	1849	v16i8 mask0 = { 0, 1, 1, 2, 2, 3, 3, 4, 16, 17, 17, 18, 18, 19, 19, 20 };
	1850	v8u16 mask4 = { 0, 4, 1, 5, 2, 6, 3, 7 };
	1851
	1852	src0_ptr -= ((3 * src_stride) + 3);
	1853
	1854	filter_vec = LD_SH(filter_x);
	1855	SPLATI_H4_SH(filter_vec, 0, 1, 2, 3, filt0, filt1, filt2, filt3);
	1856
	1857	filter_vec = LD_SH(filter_y);
	1858	vec0 = __msa_clti_s_b((v16i8) filter_vec, 0);
	1859	filter_vec = (v8i16) __msa_ilvr_b(vec0, (v16i8) filter_vec);
	1860
	1861	SPLATI_W4_SW(filter_vec, filt_h0, filt_h1, filt_h2, filt_h3);
	1862
	1863	mask1 = mask0 + 2;
	1864	mask2 = mask0 + 4;
	1865	mask3 = mask0 + 6;
	1866
	1867	offset = (offset0 + offset1) << rnd_val;
	1868	weight0 = weight0 & 0x0000FFFF;
	1869
	1870	const_vec = __msa_ldi_h(128);
	1871	const_vec <<= 6;
	1872	offset_vec = __msa_fill_w(offset);
	1873	weight_vec0 = __msa_fill_w(weight0);
	1874	weight_vec1 = __msa_fill_w(weight1);
	1875	rnd_vec = __msa_fill_w(rnd_val + 1);
	1876
	1877	LD_SB7(src0_ptr, src_stride, src0, src1, src2, src3, src4, src5, src6);
	1878	src0_ptr += (7 * src_stride);
	1879
	1880	XORI_B7_128_SB(src0, src1, src2, src3, src4, src5, src6);
	1881
	1882	VSHF_B4_SB(src0, src3, mask0, mask1, mask2, mask3, vec0, vec1, vec2, vec3);
	1883	VSHF_B4_SB(src1, src4, mask0, mask1, mask2, mask3, vec4, vec5, vec6, vec7);
	1884	VSHF_B4_SB(src2, src5, mask0, mask1, mask2, mask3,
	1885	vec8, vec9, vec10, vec11);
	1886	VSHF_B4_SB(src3, src6, mask0, mask1, mask2, mask3,
	1887	vec12, vec13, vec14, vec15);
	1888
	1889	dst30 = const_vec;
	1890	DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2, filt3,
	1891	dst30, dst30, dst30, dst30);
	1892	dst41 = const_vec;
	1893	DPADD_SB4_SH(vec4, vec5, vec6, vec7, filt0, filt1, filt2, filt3,
	1894	dst41, dst41, dst41, dst41);
	1895	dst52 = const_vec;
	1896	DPADD_SB4_SH(vec8, vec9, vec10, vec11, filt0, filt1, filt2, filt3,
	1897	dst52, dst52, dst52, dst52);
	1898	dst63 = const_vec;
	1899	DPADD_SB4_SH(vec12, vec13, vec14, vec15, filt0, filt1, filt2, filt3,
	1900	dst63, dst63, dst63, dst63);
	1901
	1902	ILVR_H3_SH(dst41, dst30, dst52, dst41, dst63, dst52,
	1903	dst10_r, dst21_r, dst32_r);
	1904	dst43_r = __msa_ilvl_h(dst41, dst30);
	1905	dst54_r = __msa_ilvl_h(dst52, dst41);
	1906	dst65_r = __msa_ilvl_h(dst63, dst52);
	1907	dst66 = (v8i16) __msa_splati_d((v2i64) dst63, 1);
	1908
	1909	for (loop_cnt = height >> 1; loop_cnt--;) {
	1910	LD_SB2(src0_ptr, src_stride, src7, src8);
	1911	src0_ptr += (2 * src_stride);
	1912	LD_SH2(src1_ptr, src2_stride, in0, in1);
	1913	src1_ptr += (2 * src2_stride);
	1914
	1915	in0 = (v8i16) __msa_ilvr_d((v2i64) in1, (v2i64) in0);
	1916	XORI_B2_128_SB(src7, src8);
	1917
	1918	VSHF_B4_SB(src7, src8, mask0, mask1, mask2, mask3,
	1919	vec0, vec1, vec2, vec3);
	1920	dst87 = const_vec;
	1921	DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2, filt3,
	1922	dst87, dst87, dst87, dst87);
	1923	dst76_r = __msa_ilvr_h(dst87, dst66);
	1924	dst0_r = HEVC_FILT_8TAP(dst10_r, dst32_r, dst54_r, dst76_r,
	1925	filt_h0, filt_h1, filt_h2, filt_h3);
	1926	dst87_r = __msa_vshf_h((v8i16) mask4, dst87, dst87);
	1927	dst1_r = HEVC_FILT_8TAP(dst21_r, dst43_r, dst65_r, dst87_r,
	1928	filt_h0, filt_h1, filt_h2, filt_h3);
	1929
	1930	dst0_r >>= 6;
	1931	dst1_r >>= 6;
	1932
	1933	ILVRL_H2_SW(in0, in0, tmp1, tmp2);
	1934	tmp1 = __msa_dpadd_s_w(offset_vec, (v8i16) tmp1, (v8i16) weight_vec0);
	1935	tmp2 = __msa_dpadd_s_w(offset_vec, (v8i16) tmp2, (v8i16) weight_vec0);
	1936	tmp1 += dst0_r * weight_vec1;
	1937	tmp2 += dst1_r * weight_vec1;
	1938	SRAR_W2_SW(tmp1, tmp2, rnd_vec);
	1939	tmp1 = CLIP_SW_0_255(tmp1);
	1940	tmp2 = CLIP_SW_0_255(tmp2);
	1941
	1942	HEVC_PCK_SW_SB2(tmp2, tmp1, tmp1);
	1943	ST4x2_UB(tmp1, dst, dst_stride);
	1944	dst += (2 * dst_stride);
	1945
	1946	dst10_r = dst32_r;
	1947	dst32_r = dst54_r;
	1948	dst54_r = dst76_r;
	1949	dst21_r = dst43_r;
	1950	dst43_r = dst65_r;
	1951	dst65_r = dst87_r;
	1952	dst66 = (v8i16) __msa_splati_d((v2i64) dst87, 1);
	1953	}
	1954	}
	1955
	1956	static void hevc_hv_biwgt_8t_8multx2mult_msa(uint8_t *src0_ptr,
	1957	int32_t src_stride,
	1958	int16_t *src1_ptr,
	1959	int32_t src2_stride,
	1960	uint8_t *dst,
	1961	int32_t dst_stride,
	1962	const int8_t *filter_x,
	1963	const int8_t *filter_y,
	1964	int32_t height,
	1965	int32_t weight0,
	1966	int32_t weight1,
	1967	int32_t offset0,
	1968	int32_t offset1,
	1969	int32_t rnd_val,
	1970	int32_t width)
	1971	{
	1972	uint32_t loop_cnt, cnt;
	1973	int32_t offset;
	1974	uint8_t *src0_ptr_tmp;
	1975	int16_t *src1_ptr_tmp;
	1976	uint8_t *dst_tmp;
	1977	v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8;
	1978	v8i16 in0, in1;
	1979	v8i16 filt0, filt1, filt2, filt3;
	1980	v4i32 filt_h0, filt_h1, filt_h2, filt_h3;
	1981	v16i8 mask0 = { 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8 };
	1982	v16i8 mask1, mask2, mask3;
	1983	v8i16 filter_vec, const_vec;
	1984	v16i8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
	1985	v16i8 vec8, vec9, vec10, vec11, vec12, vec13, vec14, vec15;
	1986	v8i16 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7, dst8;
	1987	v4i32 dst0_r, dst0_l, dst1_r, dst1_l;
	1988	v4i32 tmp0, tmp1, tmp2, tmp3;
	1989	v8i16 dst10_r, dst32_r, dst54_r, dst76_r;
	1990	v8i16 dst10_l, dst32_l, dst54_l, dst76_l;
	1991	v8i16 dst21_r, dst43_r, dst65_r, dst87_r;
	1992	v8i16 dst21_l, dst43_l, dst65_l, dst87_l;
	1993	v4i32 weight_vec0, weight_vec1, offset_vec, rnd_vec;
	1994
	1995	src0_ptr -= ((3 * src_stride) + 3);
	1996
	1997	offset = (offset0 + offset1) << rnd_val;
	1998	weight0 = weight0 & 0x0000FFFF;
	1999
	2000	const_vec = __msa_ldi_h(128);
	2001	const_vec <<= 6;
	2002	offset_vec = __msa_fill_w(offset);
	2003	weight_vec0 = __msa_fill_w(weight0);
	2004	weight_vec1 = __msa_fill_w(weight1);
	2005	rnd_vec = __msa_fill_w(rnd_val + 1);
	2006
	2007	filter_vec = LD_SH(filter_x);
	2008	SPLATI_H4_SH(filter_vec, 0, 1, 2, 3, filt0, filt1, filt2, filt3);
	2009
	2010	filter_vec = LD_SH(filter_y);
	2011	vec0 = __msa_clti_s_b((v16i8) filter_vec, 0);
	2012	filter_vec = (v8i16) __msa_ilvr_b(vec0, (v16i8) filter_vec);
	2013
	2014	SPLATI_W4_SW(filter_vec, filt_h0, filt_h1, filt_h2, filt_h3);
	2015
	2016	mask1 = mask0 + 2;
	2017	mask2 = mask0 + 4;
	2018	mask3 = mask0 + 6;
	2019
	2020	for (cnt = width >> 3; cnt--;) {
	2021	src0_ptr_tmp = src0_ptr;
	2022	src1_ptr_tmp = src1_ptr;
	2023	dst_tmp = dst;
	2024
	2025	LD_SB7(src0_ptr_tmp, src_stride,
	2026	src0, src1, src2, src3, src4, src5, src6);
	2027	src0_ptr_tmp += (7 * src_stride);
	2028
	2029	XORI_B7_128_SB(src0, src1, src2, src3, src4, src5, src6);
	2030
	2031	/* row 0 row 1 row 2 row 3 */
	2032	VSHF_B4_SB(src0, src0, mask0, mask1, mask2, mask3,
	2033	vec0, vec1, vec2, vec3);
	2034	VSHF_B4_SB(src1, src1, mask0, mask1, mask2, mask3,
	2035	vec4, vec5, vec6, vec7);
	2036	VSHF_B4_SB(src2, src2, mask0, mask1, mask2, mask3,
	2037	vec8, vec9, vec10, vec11);
	2038	VSHF_B4_SB(src3, src3, mask0, mask1, mask2, mask3,
	2039	vec12, vec13, vec14, vec15);
	2040
	2041	dst0 = const_vec;
	2042	DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2, filt3,
	2043	dst0, dst0, dst0, dst0);
	2044	dst1 = const_vec;
	2045	DPADD_SB4_SH(vec4, vec5, vec6, vec7, filt0, filt1, filt2, filt3,
	2046	dst1, dst1, dst1, dst1);
	2047	dst2 = const_vec;
	2048	DPADD_SB4_SH(vec8, vec9, vec10, vec11, filt0, filt1, filt2, filt3,
	2049	dst2, dst2, dst2, dst2);
	2050	dst3 = const_vec;
	2051	DPADD_SB4_SH(vec12, vec13, vec14, vec15, filt0, filt1, filt2, filt3,
	2052	dst3, dst3, dst3, dst3);
	2053
	2054	/* row 4 row 5 row 6 */
	2055	VSHF_B4_SB(src4, src4, mask0, mask1, mask2, mask3,
	2056	vec0, vec1, vec2, vec3);
	2057	VSHF_B4_SB(src5, src5, mask0, mask1, mask2, mask3,
	2058	vec4, vec5, vec6, vec7);
	2059	VSHF_B4_SB(src6, src6, mask0, mask1, mask2, mask3,
	2060	vec8, vec9, vec10, vec11);
	2061
	2062	dst4 = const_vec;
	2063	DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2, filt3,
	2064	dst4, dst4, dst4, dst4);
	2065	dst5 = const_vec;
	2066	DPADD_SB4_SH(vec4, vec5, vec6, vec7, filt0, filt1, filt2, filt3,
	2067	dst5, dst5, dst5, dst5);
	2068	dst6 = const_vec;
	2069	DPADD_SB4_SH(vec8, vec9, vec10, vec11, filt0, filt1, filt2, filt3,
	2070	dst6, dst6, dst6, dst6);
	2071
	2072	ILVR_H4_SH(dst1, dst0, dst3, dst2, dst5, dst4, dst2, dst1,
	2073	dst10_r, dst32_r, dst54_r, dst21_r);
	2074	ILVR_H2_SH(dst4, dst3, dst6, dst5, dst43_r, dst65_r);
	2075	ILVL_H4_SH(dst1, dst0, dst3, dst2, dst5, dst4, dst2, dst1,
	2076	dst10_l, dst32_l, dst54_l, dst21_l);
	2077	ILVL_H2_SH(dst4, dst3, dst6, dst5, dst43_l, dst65_l);
	2078
	2079	for (loop_cnt = height >> 1; loop_cnt--;) {
	2080	LD_SB2(src0_ptr_tmp, src_stride, src7, src8);
	2081	XORI_B2_128_SB(src7, src8);
	2082	src0_ptr_tmp += 2 * src_stride;
	2083
	2084	LD_SH2(src1_ptr_tmp, src2_stride, in0, in1);
	2085	src1_ptr_tmp += (2 * src2_stride);
	2086
	2087	VSHF_B4_SB(src7, src7, mask0, mask1, mask2, mask3,
	2088	vec0, vec1, vec2, vec3);
	2089	dst7 = const_vec;
	2090	DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2, filt3,
	2091	dst7, dst7, dst7, dst7);
	2092
	2093	ILVRL_H2_SH(dst7, dst6, dst76_r, dst76_l);
	2094	dst0_r = HEVC_FILT_8TAP(dst10_r, dst32_r, dst54_r, dst76_r,
	2095	filt_h0, filt_h1, filt_h2, filt_h3);
	2096	dst0_l = HEVC_FILT_8TAP(dst10_l, dst32_l, dst54_l, dst76_l,
	2097	filt_h0, filt_h1, filt_h2, filt_h3);
	2098
	2099	dst0_r >>= 6;
	2100	dst0_l >>= 6;
	2101
	2102	/* row 8 */
	2103	VSHF_B4_SB(src8, src8, mask0, mask1, mask2, mask3,
	2104	vec0, vec1, vec2, vec3);
	2105
	2106	dst8 = const_vec;
	2107	DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2, filt3,
	2108	dst8, dst8, dst8, dst8);
	2109
	2110	ILVRL_H2_SH(dst8, dst7, dst87_r, dst87_l);
	2111	dst1_r = HEVC_FILT_8TAP(dst21_r, dst43_r, dst65_r, dst87_r,
	2112	filt_h0, filt_h1, filt_h2, filt_h3);
	2113	dst1_l = HEVC_FILT_8TAP(dst21_l, dst43_l, dst65_l, dst87_l,
	2114	filt_h0, filt_h1, filt_h2, filt_h3);
	2115
	2116	dst1_r >>= 6;
	2117	dst1_l >>= 6;
	2118
	2119	ILVRL_H2_SW(in0, in0, tmp0, tmp1);
	2120	ILVRL_H2_SW(in1, in1, tmp2, tmp3);
	2121	tmp0 = __msa_dpadd_s_w(offset_vec, (v8i16) tmp0,
	2122	(v8i16) weight_vec0);
	2123	tmp1 = __msa_dpadd_s_w(offset_vec, (v8i16) tmp1,
	2124	(v8i16) weight_vec0);
	2125	tmp2 = __msa_dpadd_s_w(offset_vec, (v8i16) tmp2,
	2126	(v8i16) weight_vec0);
	2127	tmp3 = __msa_dpadd_s_w(offset_vec, (v8i16) tmp3,
	2128	(v8i16) weight_vec0);
	2129
	2130	tmp0 += (dst0_r * weight_vec1);
	2131	tmp1 += (dst0_l * weight_vec1);
	2132	tmp2 += (dst1_r * weight_vec1);
	2133	tmp3 += (dst1_l * weight_vec1);
	2134
	2135	SRAR_W4_SW(tmp0, tmp1, tmp2, tmp3, rnd_vec);
	2136	tmp0 = CLIP_SW_0_255(tmp0);
	2137	tmp1 = CLIP_SW_0_255(tmp1);
	2138	tmp2 = CLIP_SW_0_255(tmp2);
	2139	tmp3 = CLIP_SW_0_255(tmp3);
	2140	HEVC_PCK_SW_SB4(tmp1, tmp0, tmp3, tmp2, dst0_r);
	2141	ST8x2_UB(dst0_r, dst_tmp, dst_stride);
	2142	dst_tmp += (2 * dst_stride);
	2143
	2144	dst10_r = dst32_r;
	2145	dst32_r = dst54_r;
	2146	dst54_r = dst76_r;
	2147	dst10_l = dst32_l;
	2148	dst32_l = dst54_l;
	2149	dst54_l = dst76_l;
	2150	dst21_r = dst43_r;
	2151	dst43_r = dst65_r;
	2152	dst65_r = dst87_r;
	2153	dst21_l = dst43_l;
	2154	dst43_l = dst65_l;
	2155	dst65_l = dst87_l;
	2156	dst6 = dst8;
	2157	}
	2158
	2159	src0_ptr += 8;
	2160	src1_ptr += 8;
	2161	dst += 8;
	2162	}
	2163	}
	2164
	2165	static void hevc_hv_biwgt_8t_8w_msa(uint8_t *src0_ptr,
	2166	int32_t src_stride,
	2167	int16_t *src1_ptr,
	2168	int32_t src2_stride,
	2169	uint8_t *dst,
	2170	int32_t dst_stride,
	2171	const int8_t *filter_x,
	2172	const int8_t *filter_y,
	2173	int32_t height,
	2174	int32_t weight0,
	2175	int32_t weight1,
	2176	int32_t offset0,
	2177	int32_t offset1,
	2178	int32_t rnd_val)
	2179	{
	2180	hevc_hv_biwgt_8t_8multx2mult_msa(src0_ptr, src_stride,
	2181	src1_ptr, src2_stride,
	2182	dst, dst_stride, filter_x, filter_y,
	2183	height, weight0, weight1, offset0,
	2184	offset1, rnd_val, 8);
	2185	}
	2186
	2187	static void hevc_hv_biwgt_8t_12w_msa(uint8_t *src0_ptr,
	2188	int32_t src_stride,
	2189	int16_t *src1_ptr,
	2190	int32_t src2_stride,
	2191	uint8_t *dst,
	2192	int32_t dst_stride,
	2193	const int8_t *filter_x,
	2194	const int8_t *filter_y,
	2195	int32_t height,
	2196	int32_t weight0,
	2197	int32_t weight1,
	2198	int32_t offset0,
	2199	int32_t offset1,
	2200	int32_t rnd_val)
	2201	{
	2202	hevc_hv_biwgt_8t_8multx2mult_msa(src0_ptr, src_stride,
	2203	src1_ptr, src2_stride,
	2204	dst, dst_stride, filter_x, filter_y,
	2205	height, weight0, weight1, offset0,
	2206	offset1, rnd_val, 8);
	2207	hevc_hv_biwgt_8t_4w_msa(src0_ptr + 8, src_stride,
	2208	src1_ptr + 8, src2_stride,
	2209	dst + 8, dst_stride, filter_x, filter_y,
	2210	height, weight0, weight1, offset0, offset1,
	2211	rnd_val);
	2212	}
	2213
	2214	static void hevc_hv_biwgt_8t_16w_msa(uint8_t *src0_ptr,
	2215	int32_t src_stride,
	2216	int16_t *src1_ptr,
	2217	int32_t src2_stride,
	2218	uint8_t *dst,
	2219	int32_t dst_stride,
	2220	const int8_t *filter_x,
	2221	const int8_t *filter_y,
	2222	int32_t height,
	2223	int32_t weight0,
	2224	int32_t weight1,
	2225	int32_t offset0,
	2226	int32_t offset1,
	2227	int32_t rnd_val)
	2228	{
	2229	hevc_hv_biwgt_8t_8multx2mult_msa(src0_ptr, src_stride,
	2230	src1_ptr, src2_stride,
	2231	dst, dst_stride, filter_x, filter_y,
	2232	height, weight0, weight1, offset0,
	2233	offset1, rnd_val, 16);
	2234	}
	2235
	2236	static void hevc_hv_biwgt_8t_24w_msa(uint8_t *src0_ptr,
	2237	int32_t src_stride,
	2238	int16_t *src1_ptr,
	2239	int32_t src2_stride,
	2240	uint8_t *dst,
	2241	int32_t dst_stride,
	2242	const int8_t *filter_x,
	2243	const int8_t *filter_y,
	2244	int32_t height,
	2245	int32_t weight0,
	2246	int32_t weight1,
	2247	int32_t offset0,
	2248	int32_t offset1,
	2249	int32_t rnd_val)
	2250	{
	2251	hevc_hv_biwgt_8t_8multx2mult_msa(src0_ptr, src_stride,
	2252	src1_ptr, src2_stride,
	2253	dst, dst_stride, filter_x, filter_y,
	2254	height, weight0, weight1, offset0,
	2255	offset1, rnd_val, 24);
	2256	}
	2257
	2258	static void hevc_hv_biwgt_8t_32w_msa(uint8_t *src0_ptr,
	2259	int32_t src_stride,
	2260	int16_t *src1_ptr,
	2261	int32_t src2_stride,
	2262	uint8_t *dst,
	2263	int32_t dst_stride,
	2264	const int8_t *filter_x,
	2265	const int8_t *filter_y,
	2266	int32_t height,
	2267	int32_t weight0,
	2268	int32_t weight1,
	2269	int32_t offset0,
	2270	int32_t offset1,
	2271	int32_t rnd_val)
	2272	{
	2273	hevc_hv_biwgt_8t_8multx2mult_msa(src0_ptr, src_stride,
	2274	src1_ptr, src2_stride,
	2275	dst, dst_stride, filter_x, filter_y,
	2276	height, weight0, weight1, offset0,
	2277	offset1, rnd_val, 32);
	2278	}
	2279
	2280	static void hevc_hv_biwgt_8t_48w_msa(uint8_t *src0_ptr,
	2281	int32_t src_stride,
	2282	int16_t *src1_ptr,
	2283	int32_t src2_stride,
	2284	uint8_t *dst,
	2285	int32_t dst_stride,
	2286	const int8_t *filter_x,
	2287	const int8_t *filter_y,
	2288	int32_t height,
	2289	int32_t weight0,
	2290	int32_t weight1,
	2291	int32_t offset0,
	2292	int32_t offset1,
	2293	int32_t rnd_val)
	2294	{
	2295	hevc_hv_biwgt_8t_8multx2mult_msa(src0_ptr, src_stride,
	2296	src1_ptr, src2_stride,
	2297	dst, dst_stride, filter_x, filter_y,
	2298	height, weight0, weight1, offset0,
	2299	offset1, rnd_val, 48);
	2300	}
	2301
	2302	static void hevc_hv_biwgt_8t_64w_msa(uint8_t *src0_ptr,
	2303	int32_t src_stride,
	2304	int16_t *src1_ptr,
	2305	int32_t src2_stride,
	2306	uint8_t *dst,
	2307	int32_t dst_stride,
	2308	const int8_t *filter_x,
	2309	const int8_t *filter_y,
	2310	int32_t height,
	2311	int32_t weight0,
	2312	int32_t weight1,
	2313	int32_t offset0,
	2314	int32_t offset1,
	2315	int32_t rnd_val)
	2316	{
	2317	hevc_hv_biwgt_8t_8multx2mult_msa(src0_ptr, src_stride,
	2318	src1_ptr, src2_stride,
	2319	dst, dst_stride, filter_x, filter_y,
	2320	height, weight0, weight1, offset0,
	2321	offset1, rnd_val, 64);
	2322	}
	2323
	2324	static void hevc_hz_biwgt_4t_4x2_msa(uint8_t *src0_ptr,
	2325	int32_t src_stride,
	2326	int16_t *src1_ptr,
	2327	int32_t src2_stride,
	2328	uint8_t *dst,
	2329	int32_t dst_stride,
	2330	const int8_t *filter,
	2331	int32_t height,
	2332	int32_t weight0,
	2333	int32_t weight1,
	2334	int32_t offset0,
	2335	int32_t offset1,
	2336	int32_t rnd_val)
	2337	{
	2338	int32_t offset, weight;
	2339	v8i16 filt0, filt1;
	2340	v16i8 src0, src1;
	2341	v8i16 in0, in1;
	2342	v16i8 mask0 = { 0, 1, 1, 2, 2, 3, 3, 4, 16, 17, 17, 18, 18, 19, 19, 20 };
	2343	v16i8 mask1, vec0, vec1;
	2344	v8i16 dst0;
	2345	v4i32 dst0_r, dst0_l;
	2346	v8i16 filter_vec, const_vec;
	2347	v4i32 weight_vec, offset_vec, rnd_vec;
	2348
	2349	src0_ptr -= 1;
	2350
	2351	filter_vec = LD_SH(filter);
	2352	SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
	2353
	2354	mask1 = mask0 + 2;
	2355
	2356	offset = (offset0 + offset1) << rnd_val;
	2357	weight0 = weight0 & 0x0000FFFF;
	2358	weight = weight0 \| (weight1 << 16);
	2359
	2360	const_vec = __msa_ldi_h(128);
	2361	const_vec <<= 6;
	2362	offset_vec = __msa_fill_w(offset);
	2363	weight_vec = __msa_fill_w(weight);
	2364	rnd_vec = __msa_fill_w(rnd_val + 1);
	2365
	2366	LD_SB2(src0_ptr, src_stride, src0, src1);
	2367	LD_SH2(src1_ptr, src2_stride, in0, in1);
	2368	in0 = (v8i16) __msa_ilvr_d((v2i64) in1, (v2i64) in0);
	2369	XORI_B2_128_SB(src0, src1);
	2370
	2371	VSHF_B2_SB(src0, src1, src0, src1, mask0, mask1, vec0, vec1);
	2372	dst0 = const_vec;
	2373	DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst0, dst0);
	2374
	2375	ILVRL_H2_SW(dst0, in0, dst0_r, dst0_l);
	2376	dst0_r = __msa_dpadd_s_w(offset_vec, (v8i16) dst0_r, (v8i16) weight_vec);
	2377	dst0_l = __msa_dpadd_s_w(offset_vec, (v8i16) dst0_l, (v8i16) weight_vec);
	2378	SRAR_W2_SW(dst0_r, dst0_l, rnd_vec);
	2379	dst0_r = CLIP_SW_0_255(dst0_r);
	2380	dst0_l = CLIP_SW_0_255(dst0_l);
	2381
	2382	HEVC_PCK_SW_SB2(dst0_l, dst0_r, dst0_r);
	2383	ST4x2_UB(dst0_r, dst, dst_stride);
	2384	}
	2385
	2386	static void hevc_hz_biwgt_4t_4x4_msa(uint8_t *src0_ptr,
	2387	int32_t src_stride,
	2388	int16_t *src1_ptr,
	2389	int32_t src2_stride,
	2390	uint8_t *dst,
	2391	int32_t dst_stride,
	2392	const int8_t *filter,
	2393	int32_t height,
	2394	int32_t weight0,
	2395	int32_t weight1,
	2396	int32_t offset0,
	2397	int32_t offset1,
	2398	int32_t rnd_val)
	2399	{
	2400	int32_t offset, weight;
	2401	v8i16 filt0, filt1;
	2402	v16i8 src0, src1, src2, src3;
	2403	v16i8 mask0 = { 0, 1, 1, 2, 2, 3, 3, 4, 16, 17, 17, 18, 18, 19, 19, 20 };
	2404	v16i8 mask1;
	2405	v8i16 dst0, dst1;
	2406	v16i8 vec0, vec1;
	2407	v8i16 in0, in1, in2, in3;
	2408	v4i32 dst0_r, dst1_r, dst0_l, dst1_l;
	2409	v8i16 filter_vec, const_vec;
	2410	v4i32 weight_vec, offset_vec, rnd_vec;
	2411
	2412	src0_ptr -= 1;
	2413
	2414	/* rearranging filter */
	2415	filter_vec = LD_SH(filter);
	2416	SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
	2417
	2418	mask1 = mask0 + 2;
	2419
	2420	offset = (offset0 + offset1) << rnd_val;
	2421	weight0 = weight0 & 0x0000FFFF;
	2422	weight = weight0 \| (weight1 << 16);
	2423
	2424	const_vec = __msa_ldi_h(128);
	2425	const_vec <<= 6;
	2426	offset_vec = __msa_fill_w(offset);
	2427	weight_vec = __msa_fill_w(weight);
	2428	rnd_vec = __msa_fill_w(rnd_val + 1);
	2429
	2430	LD_SB4(src0_ptr, src_stride, src0, src1, src2, src3);
	2431	XORI_B4_128_SB(src0, src1, src2, src3);
	2432	LD_SH4(src1_ptr, src2_stride, in0, in1, in2, in3);
	2433	ILVR_D2_SH(in1, in0, in3, in2, in0, in1);
	2434
	2435	VSHF_B2_SB(src0, src1, src0, src1, mask0, mask1, vec0, vec1);
	2436	dst0 = const_vec;
	2437	DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst0, dst0);
	2438	VSHF_B2_SB(src2, src3, src2, src3, mask0, mask1, vec0, vec1);
	2439	dst1 = const_vec;
	2440	DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst1, dst1);
	2441	HEVC_BIW_RND_CLIP2(dst0, dst1, in0, in1,
	2442	weight_vec, rnd_vec, offset_vec,
	2443	dst0_r, dst1_r, dst0_l, dst1_l);
	2444
	2445	HEVC_PCK_SW_SB4(dst0_l, dst0_r, dst1_l, dst1_r, dst0_r);
	2446	ST4x4_UB(dst0_r, dst0_r, 0, 1, 2, 3, dst, dst_stride);
	2447	}
	2448
	2449	static void hevc_hz_biwgt_4t_4x8multiple_msa(uint8_t *src0_ptr,
	2450	int32_t src_stride,
	2451	int16_t *src1_ptr,
	2452	int32_t src2_stride,
	2453	uint8_t *dst,
	2454	int32_t dst_stride,
	2455	const int8_t *filter,
	2456	int32_t height,
	2457	int32_t weight0,
	2458	int32_t weight1,
	2459	int32_t offset0,
	2460	int32_t offset1,
	2461	int32_t rnd_val)
	2462	{
	2463	uint32_t loop_cnt;
	2464	int32_t weight, offset;
	2465	v8i16 filt0, filt1;
	2466	v16i8 src0, src1, src2, src3, src4, src5, src6, src7;
	2467	v16i8 mask0 = { 0, 1, 1, 2, 2, 3, 3, 4, 16, 17, 17, 18, 18, 19, 19, 20 };
	2468	v16i8 mask1;
	2469	v16i8 vec0, vec1;
	2470	v8i16 dst0, dst1, dst2, dst3;
	2471	v4i32 dst0_r, dst1_r, dst2_r, dst3_r, dst0_l, dst1_l, dst2_l, dst3_l;
	2472	v8i16 in0, in1, in2, in3, in4, in5, in6, in7;
	2473	v8i16 filter_vec, const_vec;
	2474	v4i32 weight_vec, offset_vec, rnd_vec;
	2475
	2476	src0_ptr -= 1;
	2477
	2478	filter_vec = LD_SH(filter);
	2479	SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
	2480
	2481	offset = (offset0 + offset1) << rnd_val;
	2482	weight0 = weight0 & 0x0000FFFF;
	2483	weight = weight0 \| (weight1 << 16);
	2484
	2485	const_vec = __msa_ldi_h(128);
	2486	const_vec <<= 6;
	2487	offset_vec = __msa_fill_w(offset);
	2488	weight_vec = __msa_fill_w(weight);
	2489	rnd_vec = __msa_fill_w(rnd_val + 1);
	2490
	2491	mask1 = mask0 + 2;
	2492
	2493	for (loop_cnt = (height >> 3); loop_cnt--;) {
	2494	LD_SB8(src0_ptr, src_stride,
	2495	src0, src1, src2, src3, src4, src5, src6, src7);
	2496	src0_ptr += (8 * src_stride);
	2497	LD_SH4(src1_ptr, src2_stride, in0, in1, in2, in3);
	2498	src1_ptr += (4 * src2_stride);
	2499	LD_SH4(src1_ptr, src2_stride, in4, in5, in6, in7);
	2500	src1_ptr += (4 * src2_stride);
	2501	ILVR_D2_SH(in1, in0, in3, in2, in0, in1);
	2502	ILVR_D2_SH(in5, in4, in7, in6, in2, in3);
	2503	XORI_B8_128_SB(src0, src1, src2, src3, src4, src5, src6, src7);
	2504
	2505	VSHF_B2_SB(src0, src1, src0, src1, mask0, mask1, vec0, vec1);
	2506	dst0 = const_vec;
	2507	DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst0, dst0);
	2508	VSHF_B2_SB(src2, src3, src2, src3, mask0, mask1, vec0, vec1);
	2509	dst1 = const_vec;
	2510	DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst1, dst1);
	2511	VSHF_B2_SB(src4, src5, src4, src5, mask0, mask1, vec0, vec1);
	2512	dst2 = const_vec;
	2513	DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst2, dst2);
	2514	VSHF_B2_SB(src6, src7, src6, src7, mask0, mask1, vec0, vec1);
	2515	dst3 = const_vec;
	2516	DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst3, dst3);
	2517	HEVC_BIW_RND_CLIP4(dst0, dst1, dst2, dst3,
	2518	in0, in1, in2, in3,
	2519	weight_vec, rnd_vec, offset_vec,
	2520	dst0_r, dst1_r, dst2_r, dst3_r,
	2521	dst0_l, dst1_l, dst2_l, dst3_l);
	2522
	2523	HEVC_PCK_SW_SB8(dst0_l, dst0_r, dst1_l, dst1_r,
	2524	dst2_l, dst2_r, dst3_l, dst3_r, dst0_r, dst1_r);
	2525	ST4x8_UB(dst0_r, dst1_r, dst, dst_stride);
	2526	dst += (8 * dst_stride);
	2527	}
	2528	}
	2529
	2530	static void hevc_hz_biwgt_4t_4w_msa(uint8_t *src0_ptr,
	2531	int32_t src_stride,
	2532	int16_t *src1_ptr,
	2533	int32_t src2_stride,
	2534	uint8_t *dst,
	2535	int32_t dst_stride,
	2536	const int8_t *filter,
	2537	int32_t height,
	2538	int32_t weight0,
	2539	int32_t weight1,
	2540	int32_t offset0,
	2541	int32_t offset1,
	2542	int32_t rnd_val)
	2543	{
	2544	if (2 == height) {
	2545	hevc_hz_biwgt_4t_4x2_msa(src0_ptr, src_stride, src1_ptr, src2_stride,
	2546	dst, dst_stride, filter, height,
	2547	weight0, weight1, offset0, offset1, rnd_val);
	2548	} else if (4 == height) {
	2549	hevc_hz_biwgt_4t_4x4_msa(src0_ptr, src_stride, src1_ptr, src2_stride,
	2550	dst, dst_stride, filter, height,
	2551	weight0, weight1, offset0, offset1, rnd_val);
	2552	} else if (0 == (height % 8)) {
	2553	hevc_hz_biwgt_4t_4x8multiple_msa(src0_ptr, src_stride,
	2554	src1_ptr, src2_stride,
	2555	dst, dst_stride, filter, height,
	2556	weight0, weight1, offset0, offset1,
	2557	rnd_val);
	2558	}
	2559	}
	2560
	2561	static void hevc_hz_biwgt_4t_6w_msa(uint8_t *src0_ptr,
	2562	int32_t src_stride,
	2563	int16_t *src1_ptr,
	2564	int32_t src2_stride,
	2565	uint8_t *dst,
	2566	int32_t dst_stride,
	2567	const int8_t *filter,
	2568	int32_t height,
	2569	int32_t weight0,
	2570	int32_t weight1,
	2571	int32_t offset0,
	2572	int32_t offset1,
	2573	int32_t rnd_val)
	2574	{
	2575	uint32_t loop_cnt;
	2576	int32_t offset, weight;
	2577	v8i16 filt0, filt1;
	2578	v16i8 src0, src1, src2, src3;
	2579	v16i8 mask0 = { 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8 };
	2580	v16i8 mask1;
	2581	v16i8 vec0, vec1;
	2582	v8i16 in0, in1, in2, in3;
	2583	v8i16 dst0, dst1, dst2, dst3;
	2584	v4i32 dst0_r, dst1_r, dst2_r, dst3_r, dst0_l, dst1_l, dst2_l, dst3_l;
	2585	v8i16 filter_vec, const_vec;
	2586	v4i32 weight_vec, offset_vec, rnd_vec;
	2587
	2588	src0_ptr -= 1;
	2589
	2590	filter_vec = LD_SH(filter);
	2591	SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
	2592
	2593	offset = (offset0 + offset1) << rnd_val;
	2594	weight0 = weight0 & 0x0000FFFF;
	2595	weight = weight0 \| (weight1 << 16);
	2596
	2597	const_vec = __msa_ldi_h(128);
	2598	const_vec <<= 6;
	2599	offset_vec = __msa_fill_w(offset);
	2600	weight_vec = __msa_fill_w(weight);
	2601	rnd_vec = __msa_fill_w(rnd_val + 1);
	2602
	2603	mask1 = mask0 + 2;
	2604
	2605	for (loop_cnt = (height >> 2); loop_cnt--;) {
	2606	LD_SB4(src0_ptr, src_stride, src0, src1, src2, src3);
	2607	src0_ptr += (4 * src_stride);
	2608	LD_SH4(src1_ptr, src2_stride, in0, in1, in2, in3);
	2609	src1_ptr += (4 * src2_stride);
	2610	XORI_B4_128_SB(src0, src1, src2, src3);
	2611
	2612	VSHF_B2_SB(src0, src0, src0, src0, mask0, mask1, vec0, vec1);
	2613	dst0 = const_vec;
	2614	DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst0, dst0);
	2615	VSHF_B2_SB(src1, src1, src1, src1, mask0, mask1, vec0, vec1);
	2616	dst1 = const_vec;
	2617	DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst1, dst1);
	2618	VSHF_B2_SB(src2, src2, src2, src2, mask0, mask1, vec0, vec1);
	2619	dst2 = const_vec;
	2620	DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst2, dst2);
	2621	VSHF_B2_SB(src3, src3, src3, src3, mask0, mask1, vec0, vec1);
	2622	dst3 = const_vec;
	2623	DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst3, dst3);
	2624
	2625	HEVC_BIW_RND_CLIP4(dst0, dst1, dst2, dst3,
	2626	in0, in1, in2, in3,
	2627	weight_vec, rnd_vec, offset_vec,
	2628	dst0_r, dst1_r, dst2_r, dst3_r,
	2629	dst0_l, dst1_l, dst2_l, dst3_l);
	2630
	2631	HEVC_PCK_SW_SB8(dst0_l, dst0_r, dst1_l, dst1_r,
	2632	dst2_l, dst2_r, dst3_l, dst3_r, dst0_r, dst1_r);
	2633	ST6x4_UB(dst0_r, dst1_r, dst, dst_stride);
	2634	dst += (4 * dst_stride);
	2635	}
	2636	}
	2637
	2638	static void hevc_hz_biwgt_4t_8x2_msa(uint8_t *src0_ptr,
	2639	int32_t src_stride,
	2640	int16_t *src1_ptr,
	2641	int32_t src2_stride,
	2642	uint8_t *dst,
	2643	int32_t dst_stride,
	2644	const int8_t *filter,
	2645	int32_t height,
	2646	int32_t weight0,
	2647	int32_t weight1,
	2648	int32_t offset0,
	2649	int32_t offset1,
	2650	int32_t rnd_val)
	2651	{
	2652	int32_t offset, weight;
	2653	v8i16 filt0, filt1;
	2654	v16i8 src0, src1;
	2655	v8i16 in0, in1;
	2656	v16i8 mask0 = { 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8 };
	2657	v16i8 mask1, vec0, vec1;
	2658	v8i16 dst0, dst1;
	2659	v8i16 filter_vec, const_vec;
	2660	v4i32 dst0_r, dst1_r, dst0_l, dst1_l;
	2661	v4i32 weight_vec, offset_vec, rnd_vec;
	2662
	2663	src0_ptr -= 1;
	2664
	2665	filter_vec = LD_SH(filter);
	2666	SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
	2667
	2668	offset = (offset0 + offset1) << rnd_val;
	2669	weight0 = weight0 & 0x0000FFFF;
	2670	weight = weight0 \| (weight1 << 16);
	2671
	2672	const_vec = __msa_ldi_h(128);
	2673	const_vec <<= 6;
	2674	offset_vec = __msa_fill_w(offset);
	2675	weight_vec = __msa_fill_w(weight);
	2676	rnd_vec = __msa_fill_w(rnd_val + 1);
	2677
	2678	mask1 = mask0 + 2;
	2679
	2680	LD_SB2(src0_ptr, src_stride, src0, src1);
	2681	LD_SH2(src1_ptr, src2_stride, in0, in1);
	2682	XORI_B2_128_SB(src0, src1);
	2683	VSHF_B2_SB(src0, src0, src0, src0, mask0, mask1, vec0, vec1);
	2684	dst0 = const_vec;
	2685	DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst0, dst0);
	2686	VSHF_B2_SB(src1, src1, src1, src1, mask0, mask1, vec0, vec1);
	2687	dst1 = const_vec;
	2688	DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst1, dst1);
	2689	HEVC_BIW_RND_CLIP2(dst0, dst1, in0, in1,
	2690	weight_vec, rnd_vec, offset_vec,
	2691	dst0_r, dst1_r, dst0_l, dst1_l);
	2692
	2693	HEVC_PCK_SW_SB4(dst0_l, dst0_r, dst1_l, dst1_r, dst0_r);
	2694	ST8x2_UB(dst0_r, dst, dst_stride);
	2695	}
	2696
	2697	static void hevc_hz_biwgt_4t_8x6_msa(uint8_t *src0_ptr,
	2698	int32_t src_stride,
	2699	int16_t *src1_ptr,
	2700	int32_t src2_stride,
	2701	uint8_t *dst,
	2702	int32_t dst_stride,
	2703	const int8_t *filter,
	2704	int32_t height,
	2705	int32_t weight0,
	2706	int32_t weight1,
	2707	int32_t offset0,
	2708	int32_t offset1,
	2709	int32_t rnd_val)
	2710	{
	2711	int32_t weight, offset;
	2712	v8i16 filt0, filt1;
	2713	v16i8 src0, src1, src2, src3, src4, src5;
	2714	v8i16 in0, in1, in2, in3, in4, in5;
	2715	v16i8 mask0 = { 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8 };
	2716	v16i8 mask1;
	2717	v16i8 vec0, vec1;
	2718	v8i16 dst0, dst1, dst2, dst3, dst4, dst5;
	2719	v8i16 filter_vec, const_vec;
	2720	v4i32 dst0_r, dst1_r, dst2_r, dst3_r, dst4_r, dst5_r;
	2721	v4i32 dst0_l, dst1_l, dst2_l, dst3_l, dst4_l, dst5_l;
	2722	v4i32 weight_vec, offset_vec, rnd_vec;
	2723
	2724	src0_ptr -= 1;
	2725
	2726	filter_vec = LD_SH(filter);
	2727	SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
	2728
	2729	offset = (offset0 + offset1) << rnd_val;
	2730	weight0 = weight0 & 0x0000FFFF;
	2731	weight = weight0 \| (weight1 << 16);
	2732
	2733	const_vec = __msa_ldi_h(128);
	2734	const_vec <<= 6;
	2735	offset_vec = __msa_fill_w(offset);
	2736	weight_vec = __msa_fill_w(weight);
	2737	rnd_vec = __msa_fill_w(rnd_val + 1);
	2738
	2739	mask1 = mask0 + 2;
	2740
	2741	LD_SB6(src0_ptr, src_stride, src0, src1, src2, src3, src4, src5);
	2742
	2743	LD_SH4(src1_ptr, src2_stride, in0, in1, in2, in3);
	2744	src1_ptr += (4 * src2_stride);
	2745	LD_SH2(src1_ptr, src2_stride, in4, in5);
	2746	XORI_B6_128_SB(src0, src1, src2, src3, src4, src5);
	2747	VSHF_B2_SB(src0, src0, src0, src0, mask0, mask1, vec0, vec1);
	2748	dst0 = const_vec;
	2749	DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst0, dst0);
	2750	VSHF_B2_SB(src1, src1, src1, src1, mask0, mask1, vec0, vec1);
	2751	dst1 = const_vec;
	2752	DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst1, dst1);
	2753	VSHF_B2_SB(src2, src2, src2, src2, mask0, mask1, vec0, vec1);
	2754	dst2 = const_vec;
	2755	DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst2, dst2);
	2756	VSHF_B2_SB(src3, src3, src3, src3, mask0, mask1, vec0, vec1);
	2757	dst3 = const_vec;
	2758	DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst3, dst3);
	2759	VSHF_B2_SB(src4, src4, src4, src4, mask0, mask1, vec0, vec1);
	2760	dst4 = const_vec;
	2761	DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst4, dst4);
	2762	VSHF_B2_SB(src5, src5, src5, src5, mask0, mask1, vec0, vec1);
	2763	dst5 = const_vec;
	2764	DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst5, dst5);
	2765	HEVC_BIW_RND_CLIP4(dst0, dst1, dst2, dst3,
	2766	in0, in1, in2, in3,
	2767	weight_vec, rnd_vec, offset_vec,
	2768	dst0_r, dst1_r, dst2_r, dst3_r,
	2769	dst0_l, dst1_l, dst2_l, dst3_l);
	2770	HEVC_BIW_RND_CLIP2(dst4, dst5, in4, in5,
	2771	weight_vec, rnd_vec, offset_vec,
	2772	dst4_r, dst5_r, dst4_l, dst5_l);
	2773
	2774	HEVC_PCK_SW_SB12(dst0_l, dst0_r, dst1_l, dst1_r,
	2775	dst2_l, dst2_r, dst3_l, dst3_r,
	2776	dst4_l, dst4_r, dst5_l, dst5_r, dst0_r, dst1_r, dst2_r);
	2777	ST8x4_UB(dst0_r, dst1_r, dst, dst_stride);
	2778	dst += (4 * dst_stride);
	2779	ST8x2_UB(dst2_r, dst, dst_stride);
	2780	}
	2781
	2782	static void hevc_hz_biwgt_4t_8x4multiple_msa(uint8_t *src0_ptr,
	2783	int32_t src_stride,
	2784	int16_t *src1_ptr,
	2785	int32_t src2_stride,
	2786	uint8_t *dst,
	2787	int32_t dst_stride,
	2788	const int8_t *filter,
	2789	int32_t height,
	2790	int32_t weight0,
	2791	int32_t weight1,
	2792	int32_t offset0,
	2793	int32_t offset1,
	2794	int32_t rnd_val)
	2795	{
	2796	uint32_t loop_cnt;
	2797	int32_t offset, weight;
	2798	v8i16 filt0, filt1;
	2799	v16i8 src0, src1, src2, src3;
	2800	v16i8 mask0 = { 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8 };
	2801	v16i8 mask1;
	2802	v16i8 vec0, vec1;
	2803	v8i16 in0, in1, in2, in3;
	2804	v8i16 dst0, dst1, dst2, dst3;
	2805	v4i32 dst0_r, dst1_r, dst2_r, dst3_r, dst0_l, dst1_l, dst2_l, dst3_l;
	2806	v8i16 filter_vec, const_vec;
	2807	v4i32 weight_vec, offset_vec, rnd_vec;
	2808
	2809	src0_ptr -= 1;
	2810
	2811	filter_vec = LD_SH(filter);
	2812	SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
	2813
	2814	offset = (offset0 + offset1) << rnd_val;
	2815	weight0 = weight0 & 0x0000FFFF;
	2816	weight = weight0 \| (weight1 << 16);
	2817
	2818	const_vec = __msa_ldi_h(128);
	2819	const_vec <<= 6;
	2820	offset_vec = __msa_fill_w(offset);
	2821	weight_vec = __msa_fill_w(weight);
	2822	rnd_vec = __msa_fill_w(rnd_val + 1);
	2823
	2824	mask1 = mask0 + 2;
	2825
	2826	for (loop_cnt = (height >> 2); loop_cnt--;) {
	2827	LD_SB4(src0_ptr, src_stride, src0, src1, src2, src3);
	2828	src0_ptr += (4 * src_stride);
	2829	LD_SH4(src1_ptr, src2_stride, in0, in1, in2, in3);
	2830	src1_ptr += (4 * src2_stride);
	2831	XORI_B4_128_SB(src0, src1, src2, src3);
	2832
	2833	VSHF_B2_SB(src0, src0, src0, src0, mask0, mask1, vec0, vec1);
	2834	dst0 = const_vec;
	2835	DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst0, dst0);
	2836	VSHF_B2_SB(src1, src1, src1, src1, mask0, mask1, vec0, vec1);
	2837	dst1 = const_vec;
	2838	DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst1, dst1);
	2839	VSHF_B2_SB(src2, src2, src2, src2, mask0, mask1, vec0, vec1);
	2840	dst2 = const_vec;
	2841	DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst2, dst2);
	2842	VSHF_B2_SB(src3, src3, src3, src3, mask0, mask1, vec0, vec1);
	2843	dst3 = const_vec;
	2844	DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst3, dst3);
	2845	HEVC_BIW_RND_CLIP4(dst0, dst1, dst2, dst3,
	2846	in0, in1, in2, in3,
	2847	weight_vec, rnd_vec, offset_vec,
	2848	dst0_r, dst1_r, dst2_r, dst3_r,
	2849	dst0_l, dst1_l, dst2_l, dst3_l);
	2850
	2851	HEVC_PCK_SW_SB8(dst0_l, dst0_r, dst1_l, dst1_r,
	2852	dst2_l, dst2_r, dst3_l, dst3_r, dst0_r, dst1_r);
	2853	ST8x4_UB(dst0_r, dst1_r, dst, dst_stride);
	2854	dst += (4 * dst_stride);
	2855	}
	2856	}
	2857
	2858	static void hevc_hz_biwgt_4t_8w_msa(uint8_t *src0_ptr,
	2859	int32_t src_stride,
	2860	int16_t *src1_ptr,
	2861	int32_t src2_stride,
	2862	uint8_t *dst,
	2863	int32_t dst_stride,
	2864	const int8_t *filter,
	2865	int32_t height,
	2866	int32_t weight0,
	2867	int32_t weight1,
	2868	int32_t offset0,
	2869	int32_t offset1,
	2870	int32_t rnd_val)
	2871	{
	2872	if (2 == height) {
	2873	hevc_hz_biwgt_4t_8x2_msa(src0_ptr, src_stride, src1_ptr, src2_stride,
	2874	dst, dst_stride, filter, height,
	2875	weight0, weight1, offset0, offset1, rnd_val);
	2876	} else if (6 == height) {
	2877	hevc_hz_biwgt_4t_8x6_msa(src0_ptr, src_stride, src1_ptr, src2_stride,
	2878	dst, dst_stride, filter, height,
	2879	weight0, weight1, offset0, offset1, rnd_val);
	2880	} else if (0 == (height % 4)) {
	2881	hevc_hz_biwgt_4t_8x4multiple_msa(src0_ptr, src_stride,
	2882	src1_ptr, src2_stride,
	2883	dst, dst_stride, filter, height,
	2884	weight0, weight1, offset0, offset1,
	2885	rnd_val);
	2886	}
	2887	}
	2888
	2889	static void hevc_hz_biwgt_4t_12w_msa(uint8_t *src0_ptr,
	2890	int32_t src_stride,
	2891	int16_t *src1_ptr,
	2892	int32_t src2_stride,
	2893	uint8_t *dst,
	2894	int32_t dst_stride,
	2895	const int8_t *filter,
	2896	int32_t height,
	2897	int32_t weight0,
	2898	int32_t weight1,
	2899	int32_t offset0,
	2900	int32_t offset1,
	2901	int32_t rnd_val)
	2902	{
	2903	uint32_t loop_cnt;
	2904	int32_t offset, weight;
	2905	v8i16 filt0, filt1;
	2906	v16i8 src0, src1, src2, src3;
	2907	v8i16 in0, in1, in2, in3, in4, in5, in6, in7;
	2908	v16i8 mask0 = { 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8 };
	2909	v16i8 mask2 = {
	2910	8, 9, 9, 10, 10, 11, 11, 12, 24, 25, 25, 26, 26, 27, 27, 28
	2911	};
	2912	v16i8 mask1, mask3;
	2913	v16i8 vec0, vec1;
	2914	v8i16 dst0, dst1, dst2, dst3, dst4, dst5;
	2915	v8i16 filter_vec, const_vec;
	2916	v4i32 dst0_r, dst1_r, dst2_r, dst3_r, dst4_r, dst5_r;
	2917	v4i32 dst0_l, dst1_l, dst2_l, dst3_l, dst4_l, dst5_l;
	2918	v4i32 weight_vec, offset_vec, rnd_vec;
	2919
	2920	src0_ptr -= 1;
	2921
	2922	filter_vec = LD_SH(filter);
	2923	SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
	2924
	2925	offset = (offset0 + offset1) << rnd_val;
	2926	weight0 = weight0 & 0x0000FFFF;
	2927	weight = weight0 \| (weight1 << 16);
	2928
	2929	const_vec = __msa_ldi_h(128);
	2930	const_vec <<= 6;
	2931	offset_vec = __msa_fill_w(offset);
	2932	weight_vec = __msa_fill_w(weight);
	2933	rnd_vec = __msa_fill_w(rnd_val + 1);
	2934
	2935	mask1 = mask0 + 2;
	2936	mask3 = mask2 + 2;
	2937
	2938	for (loop_cnt = (height >> 2); loop_cnt--;) {
	2939	LD_SB4(src0_ptr, src_stride, src0, src1, src2, src3);
	2940	src0_ptr += (4 * src_stride);
	2941	LD_SH4(src1_ptr, src2_stride, in0, in1, in2, in3);
	2942	LD_SH4(src1_ptr + 8, src2_stride, in4, in5, in6, in7);
	2943	src1_ptr += (4 * src2_stride);
	2944	ILVR_D2_SH(in5, in4, in7, in6, in4, in5);
	2945	XORI_B4_128_SB(src0, src1, src2, src3);
	2946
	2947	VSHF_B2_SB(src0, src0, src0, src0, mask0, mask1, vec0, vec1);
	2948	dst0 = const_vec;
	2949	DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst0, dst0);
	2950	VSHF_B2_SB(src1, src1, src1, src1, mask0, mask1, vec0, vec1);
	2951	dst1 = const_vec;
	2952	DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst1, dst1);
	2953	VSHF_B2_SB(src2, src2, src2, src2, mask0, mask1, vec0, vec1);
	2954	dst2 = const_vec;
	2955	DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst2, dst2);
	2956	VSHF_B2_SB(src3, src3, src3, src3, mask0, mask1, vec0, vec1);
	2957	dst3 = const_vec;
	2958	DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst3, dst3);
	2959	VSHF_B2_SB(src0, src1, src0, src1, mask2, mask3, vec0, vec1);
	2960	dst4 = const_vec;
	2961	DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst4, dst4);
	2962	VSHF_B2_SB(src2, src3, src2, src3, mask2, mask3, vec0, vec1);
	2963	dst5 = const_vec;
	2964	DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst5, dst5);
	2965
	2966	HEVC_BIW_RND_CLIP4(dst0, dst1, dst2, dst3,
	2967	in0, in1, in2, in3,
	2968	weight_vec, rnd_vec, offset_vec,
	2969	dst0_r, dst1_r, dst2_r, dst3_r,
	2970	dst0_l, dst1_l, dst2_l, dst3_l);
	2971	HEVC_BIW_RND_CLIP2(dst4, dst5, in4, in5,
	2972	weight_vec, rnd_vec, offset_vec,
	2973	dst4_r, dst5_r, dst4_l, dst5_l);
	2974
	2975	HEVC_PCK_SW_SB12(dst0_l, dst0_r, dst1_l, dst1_r,
	2976	dst2_l, dst2_r, dst3_l, dst3_r,
	2977	dst4_l, dst4_r, dst5_l, dst5_r,
	2978	dst0_r, dst1_r, dst2_r);
	2979	ST12x4_UB(dst0_r, dst1_r, dst2_r, dst, dst_stride);
	2980	dst += (4 * dst_stride);
	2981	}
	2982	}
	2983
	2984	static void hevc_hz_biwgt_4t_16w_msa(uint8_t *src0_ptr,
	2985	int32_t src_stride,
	2986	int16_t *src1_ptr,
	2987	int32_t src2_stride,
	2988	uint8_t *dst,
	2989	int32_t dst_stride,
	2990	const int8_t *filter,
	2991	int32_t height,
	2992	int32_t weight0,
	2993	int32_t weight1,
	2994	int32_t offset0,
	2995	int32_t offset1,
	2996	int32_t rnd_val)
	2997	{
	2998	uint32_t loop_cnt;
	2999	int32_t offset, weight;
	3000	v16i8 src0, src1, src2, src3, src4, src5, src6, src7;
	3001	v8i16 in0, in1, in2, in3, in4, in5, in6, in7;
	3002	v8i16 filt0, filt1;
	3003	v16i8 mask0 = { 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8 };
	3004	v16i8 mask1;
	3005	v8i16 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7;
	3006	v16i8 vec0, vec1;
	3007	v8i16 filter_vec, const_vec;
	3008	v4i32 dst0_r, dst1_r, dst2_r, dst3_r, dst0_l, dst1_l, dst2_l, dst3_l;
	3009	v4i32 weight_vec, offset_vec, rnd_vec;
	3010
	3011	src0_ptr -= 1;
	3012
	3013	filter_vec = LD_SH(filter);
	3014	SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
	3015
	3016	offset = (offset0 + offset1) << rnd_val;
	3017	weight0 = weight0 & 0x0000FFFF;
	3018	weight = weight0 \| (weight1 << 16);
	3019
	3020	const_vec = __msa_ldi_h(128);
	3021	const_vec <<= 6;
	3022	offset_vec = __msa_fill_w(offset);
	3023	weight_vec = __msa_fill_w(weight);
	3024	rnd_vec = __msa_fill_w(rnd_val + 1);
	3025
	3026	mask1 = mask0 + 2;
	3027
	3028	for (loop_cnt = (height >> 2); loop_cnt--;) {
	3029	LD_SB4(src0_ptr, src_stride, src0, src2, src4, src6);
	3030	LD_SB4(src0_ptr + 8, src_stride, src1, src3, src5, src7);
	3031	src0_ptr += (4 * src_stride);
	3032	LD_SH4(src1_ptr, src2_stride, in0, in2, in4, in6);
	3033	LD_SH4(src1_ptr + 8, src2_stride, in1, in3, in5, in7);
	3034	src1_ptr += (4 * src2_stride);
	3035	XORI_B8_128_SB(src0, src1, src2, src3, src4, src5, src6, src7);
	3036
	3037	VSHF_B2_SB(src0, src0, src0, src0, mask0, mask1, vec0, vec1);
	3038	dst0 = const_vec;
	3039	DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst0, dst0);
	3040	VSHF_B2_SB(src1, src1, src1, src1, mask0, mask1, vec0, vec1);
	3041	dst1 = const_vec;
	3042	DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst1, dst1);
	3043	VSHF_B2_SB(src2, src2, src2, src2, mask0, mask1, vec0, vec1);
	3044	dst2 = const_vec;
	3045	DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst2, dst2);
	3046	VSHF_B2_SB(src3, src3, src3, src3, mask0, mask1, vec0, vec1);
	3047	dst3 = const_vec;
	3048	DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst3, dst3);
	3049	VSHF_B2_SB(src4, src4, src4, src4, mask0, mask1, vec0, vec1);
	3050	dst4 = const_vec;
	3051	DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst4, dst4);
	3052	VSHF_B2_SB(src5, src5, src5, src5, mask0, mask1, vec0, vec1);
	3053	dst5 = const_vec;
	3054	DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst5, dst5);
	3055	VSHF_B2_SB(src6, src6, src6, src6, mask0, mask1, vec0, vec1);
	3056	dst6 = const_vec;
	3057	DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst6, dst6);
	3058	VSHF_B2_SB(src7, src7, src7, src7, mask0, mask1, vec0, vec1);
	3059	dst7 = const_vec;
	3060	DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst7, dst7);
	3061	HEVC_BIW_RND_CLIP4(dst0, dst1, dst2, dst3,
	3062	in0, in1, in2, in3,
	3063	weight_vec, rnd_vec, offset_vec,
	3064	dst0_r, dst1_r, dst2_r, dst3_r,
	3065	dst0_l, dst1_l, dst2_l, dst3_l);
	3066
	3067	HEVC_PCK_SW_SB8(dst0_l, dst0_r, dst1_l, dst1_r,
	3068	dst2_l, dst2_r, dst3_l, dst3_r, dst0_r, dst1_r);
	3069	ST_SW2(dst0_r, dst1_r, dst, dst_stride);
	3070	dst += (2 * dst_stride);
	3071
	3072	HEVC_BIW_RND_CLIP4(dst4, dst5, dst6, dst7,
	3073	in4, in5, in6, in7,
	3074	weight_vec, rnd_vec, offset_vec,
	3075	dst0_r, dst1_r, dst2_r, dst3_r,
	3076	dst0_l, dst1_l, dst2_l, dst3_l);
	3077
	3078	HEVC_PCK_SW_SB8(dst0_l, dst0_r, dst1_l, dst1_r,
	3079	dst2_l, dst2_r, dst3_l, dst3_r, dst0_r, dst1_r);
	3080	ST_SW2(dst0_r, dst1_r, dst, dst_stride);
	3081	dst += (2 * dst_stride);
	3082	}
	3083	}
	3084
	3085	static void hevc_hz_biwgt_4t_24w_msa(uint8_t *src0_ptr,
	3086	int32_t src_stride,
	3087	int16_t *src1_ptr,
	3088	int32_t src2_stride,
	3089	uint8_t *dst,
	3090	int32_t dst_stride,
	3091	const int8_t *filter,
	3092	int32_t height,
	3093	int32_t weight0,
	3094	int32_t weight1,
	3095	int32_t offset0,
	3096	int32_t offset1,
	3097	int32_t rnd_val)
	3098	{
	3099	uint32_t loop_cnt;
	3100	int32_t offset, weight;
	3101	uint8_t *dst_tmp = dst + 16;
	3102	v16i8 src0, src1, src2, src3;
	3103	v8i16 filt0, filt1;
	3104	v16i8 mask0 = { 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8 };
	3105	v16i8 mask1, mask2, mask3;
	3106	v16i8 vec0, vec1;
	3107	v8i16 dst0, dst1, dst2, dst3;
	3108	v8i16 in0, in1, in2, in3, in4, in5;
	3109	v8i16 filter_vec, const_vec;
	3110	v4i32 dst0_r, dst1_r, dst2_r, dst3_r, dst0_l, dst1_l, dst2_l, dst3_l;
	3111	v4i32 weight_vec, offset_vec, rnd_vec;
	3112
	3113	src0_ptr -= 1;
	3114
	3115	filter_vec = LD_SH(filter);
	3116	SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
	3117
	3118	offset = (offset0 + offset1) << rnd_val;
	3119	weight0 = weight0 & 0x0000FFFF;
	3120	weight = weight0 \| (weight1 << 16);
	3121
	3122	const_vec = __msa_ldi_h(128);
	3123	const_vec <<= 6;
	3124	offset_vec = __msa_fill_w(offset);
	3125	weight_vec = __msa_fill_w(weight);
	3126	rnd_vec = __msa_fill_w(rnd_val + 1);
	3127
	3128	mask1 = mask0 + 2;
	3129	mask2 = mask0 + 8;
	3130	mask3 = mask0 + 10;
	3131
	3132	for (loop_cnt = (height >> 1); loop_cnt--;) {
	3133	LD_SB2(src0_ptr, src_stride, src0, src2);
	3134	LD_SB2(src0_ptr + 16, src_stride, src1, src3);
	3135	src0_ptr += (2 * src_stride);
	3136	LD_SH2(src1_ptr, src2_stride, in0, in2);
	3137	LD_SH2(src1_ptr + 8, src2_stride, in1, in3);
	3138	LD_SH2(src1_ptr + 16, src2_stride, in4, in5);
	3139	src1_ptr += (2 * src2_stride);
	3140	XORI_B4_128_SB(src0, src1, src2, src3);
	3141
	3142	VSHF_B2_SB(src0, src0, src0, src0, mask0, mask1, vec0, vec1);
	3143	dst0 = const_vec;
	3144	DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst0, dst0);
	3145	VSHF_B2_SB(src0, src1, src0, src1, mask2, mask3, vec0, vec1);
	3146	dst1 = const_vec;
	3147	DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst1, dst1);
	3148	VSHF_B2_SB(src2, src2, src2, src2, mask0, mask1, vec0, vec1);
	3149	dst2 = const_vec;
	3150	DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst2, dst2);
	3151	VSHF_B2_SB(src2, src3, src2, src3, mask2, mask3, vec0, vec1);
	3152	dst3 = const_vec;
	3153	DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst3, dst3);
	3154	HEVC_BIW_RND_CLIP4(dst0, dst1, dst2, dst3,
	3155	in0, in1, in2, in3,
	3156	weight_vec, rnd_vec, offset_vec,
	3157	dst0_r, dst1_r, dst2_r, dst3_r,
	3158	dst0_l, dst1_l, dst2_l, dst3_l);
	3159
	3160	HEVC_PCK_SW_SB8(dst0_l, dst0_r, dst1_l, dst1_r,
	3161	dst2_l, dst2_r, dst3_l, dst3_r, dst0_r, dst1_r);
	3162	ST_SW2(dst0_r, dst1_r, dst, dst_stride);
	3163	dst += (2 * dst_stride);
	3164	/* 8 width */
	3165	VSHF_B2_SB(src1, src1, src1, src1, mask0, mask1, vec0, vec1);
	3166	dst0 = const_vec;
	3167	DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst0, dst0);
	3168	VSHF_B2_SB(src3, src3, src3, src3, mask0, mask1, vec0, vec1);
	3169	dst1 = const_vec;
	3170	DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst1, dst1);
	3171	HEVC_BIW_RND_CLIP2(dst0, dst1, in4, in5,
	3172	weight_vec, rnd_vec, offset_vec,
	3173	dst0_r, dst1_r, dst0_l, dst1_l);
	3174
	3175	HEVC_PCK_SW_SB4(dst0_l, dst0_r, dst1_l, dst1_r, dst0_r);
	3176	ST8x2_UB(dst0_r, dst_tmp, dst_stride);
	3177	dst_tmp += (2 * dst_stride);
	3178	}
	3179	}
	3180
	3181	static void hevc_hz_biwgt_4t_32w_msa(uint8_t *src0_ptr,
	3182	int32_t src_stride,
	3183	int16_t *src1_ptr,
	3184	int32_t src2_stride,
	3185	uint8_t *dst,
	3186	int32_t dst_stride,
	3187	const int8_t *filter,
	3188	int32_t height,
	3189	int32_t weight0,
	3190	int32_t weight1,
	3191	int32_t offset0,
	3192	int32_t offset1,
	3193	int32_t rnd_val)
	3194	{
	3195	uint32_t loop_cnt;
	3196	int32_t offset, weight;
	3197	v16i8 src0, src1, src2;
	3198	v8i16 filt0, filt1;
	3199	v16i8 mask0 = { 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8 };
	3200	v16i8 mask1, mask2, mask3;
	3201	v8i16 dst0, dst1, dst2, dst3;
	3202	v16i8 vec0, vec1;
	3203	v8i16 in0, in1, in2, in3;
	3204	v8i16 filter_vec, const_vec;
	3205	v4i32 dst0_r, dst1_r, dst2_r, dst3_r, dst0_l, dst1_l, dst2_l, dst3_l;
	3206	v4i32 weight_vec, offset_vec, rnd_vec;
	3207
	3208	src0_ptr -= 1;
	3209
	3210	filter_vec = LD_SH(filter);
	3211	SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
	3212
	3213	offset = (offset0 + offset1) << rnd_val;
	3214	weight0 = weight0 & 0x0000FFFF;
	3215	weight = weight0 \| (weight1 << 16);
	3216
	3217	const_vec = __msa_ldi_h(128);
	3218	const_vec <<= 6;
	3219	offset_vec = __msa_fill_w(offset);
	3220	weight_vec = __msa_fill_w(weight);
	3221	rnd_vec = __msa_fill_w(rnd_val + 1);
	3222
	3223	mask1 = mask0 + 2;
	3224	mask2 = mask0 + 8;
	3225	mask3 = mask0 + 10;
	3226
	3227	for (loop_cnt = height; loop_cnt--;) {
	3228	LD_SB2(src0_ptr, 16, src0, src1);
	3229	src2 = LD_SB(src0_ptr + 24);
	3230	src0_ptr += src_stride;
	3231	LD_SH4(src1_ptr, 8, in0, in1, in2, in3);
	3232	src1_ptr += src2_stride;
	3233	XORI_B3_128_SB(src0, src1, src2);
	3234
	3235	VSHF_B2_SB(src0, src0, src0, src0, mask0, mask1, vec0, vec1);
	3236	dst0 = const_vec;
	3237	DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst0, dst0);
	3238	VSHF_B2_SB(src0, src1, src0, src1, mask2, mask3, vec0, vec1);
	3239	dst1 = const_vec;
	3240	DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst1, dst1);
	3241	VSHF_B2_SB(src1, src1, src1, src1, mask0, mask1, vec0, vec1);
	3242	dst2 = const_vec;
	3243	DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst2, dst2);
	3244	VSHF_B2_SB(src2, src2, src2, src2, mask0, mask1, vec0, vec1);
	3245	dst3 = const_vec;
	3246	DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst3, dst3);
	3247	HEVC_BIW_RND_CLIP4(dst0, dst1, dst2, dst3,
	3248	in0, in1, in2, in3,
	3249	weight_vec, rnd_vec, offset_vec,
	3250	dst0_r, dst1_r, dst2_r, dst3_r,
	3251	dst0_l, dst1_l, dst2_l, dst3_l);
	3252
	3253	HEVC_PCK_SW_SB8(dst0_l, dst0_r, dst1_l, dst1_r,
	3254	dst2_l, dst2_r, dst3_l, dst3_r, dst0_r, dst1_r);
	3255	ST_SW2(dst0_r, dst1_r, dst, 16);
	3256	dst += dst_stride;
	3257	}
	3258	}
	3259
	3260	static void hevc_vt_biwgt_4t_4x2_msa(uint8_t *src0_ptr,
	3261	int32_t src_stride,
	3262	int16_t *src1_ptr,
	3263	int32_t src2_stride,
	3264	uint8_t *dst,
	3265	int32_t dst_stride,
	3266	const int8_t *filter,
	3267	int32_t height,
	3268	int32_t weight0,
	3269	int32_t weight1,
	3270	int32_t offset0,
	3271	int32_t offset1,
	3272	int32_t rnd_val)
	3273	{
	3274	int32_t weight, offset;
	3275	v16i8 src0, src1, src2, src3, src4;
	3276	v8i16 in0, in1, dst10;
	3277	v16i8 src10_r, src32_r, src21_r, src43_r, src2110, src4332;
	3278	v4i32 dst10_r, dst10_l;
	3279	v8i16 filt0, filt1;
	3280	v8i16 filter_vec, const_vec;
	3281	v4i32 weight_vec, offset_vec, rnd_vec;
	3282
	3283	src0_ptr -= src_stride;
	3284
	3285	offset = (offset0 + offset1) << rnd_val;
	3286	weight0 = weight0 & 0x0000FFFF;
	3287	weight = weight0 \| (weight1 << 16);
	3288
	3289	const_vec = __msa_ldi_h(128);
	3290	const_vec <<= 6;
	3291	offset_vec = __msa_fill_w(offset);
	3292	weight_vec = __msa_fill_w(weight);
	3293	rnd_vec = __msa_fill_w(rnd_val + 1);
	3294
	3295	filter_vec = LD_SH(filter);
	3296	SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
	3297
	3298	LD_SB3(src0_ptr, src_stride, src0, src1, src2);
	3299	src0_ptr += (3 * src_stride);
	3300	ILVR_B2_SB(src1, src0, src2, src1, src10_r, src21_r);
	3301	src2110 = (v16i8) __msa_ilvr_d((v2i64) src21_r, (v2i64) src10_r);
	3302	src2110 = (v16i8) __msa_xori_b((v16u8) src2110, 128);
	3303	LD_SB2(src0_ptr, src_stride, src3, src4);
	3304	src0_ptr += (2 * src_stride);
	3305	LD_SH2(src1_ptr, src2_stride, in0, in1);
	3306	src1_ptr += (2 * src2_stride);
	3307
	3308	in0 = (v8i16) __msa_ilvr_d((v2i64) in1, (v2i64) in0);
	3309	ILVR_B2_SB(src3, src2, src4, src3, src32_r, src43_r);
	3310	src4332 = (v16i8) __msa_ilvr_d((v2i64) src43_r, (v2i64) src32_r);
	3311	src4332 = (v16i8) __msa_xori_b((v16u8) src4332, 128);
	3312
	3313	dst10 = const_vec;
	3314	DPADD_SB2_SH(src2110, src4332, filt0, filt1, dst10, dst10);
	3315
	3316	ILVRL_H2_SW(dst10, in0, dst10_r, dst10_l);
	3317	dst10_r = __msa_dpadd_s_w(offset_vec, (v8i16) dst10_r, (v8i16) weight_vec);
	3318	dst10_l = __msa_dpadd_s_w(offset_vec, (v8i16) dst10_l, (v8i16) weight_vec);
	3319	SRAR_W2_SW(dst10_r, dst10_l, rnd_vec);
	3320	dst10_r = CLIP_SW_0_255(dst10_r);
	3321	dst10_l = CLIP_SW_0_255(dst10_l);
	3322
	3323	HEVC_PCK_SW_SB2(dst10_l, dst10_r, dst10_r);
	3324	ST4x2_UB(dst10_r, dst, dst_stride);
	3325	}
	3326
	3327	static void hevc_vt_biwgt_4t_4x4_msa(uint8_t *src0_ptr,
	3328	int32_t src_stride,
	3329	int16_t *src1_ptr,
	3330	int32_t src2_stride,
	3331	uint8_t *dst,
	3332	int32_t dst_stride,
	3333	const int8_t *filter,
	3334	int32_t height,
	3335	int32_t weight0,
	3336	int32_t weight1,
	3337	int32_t offset0,
	3338	int32_t offset1,
	3339	int32_t rnd_val)
	3340	{
	3341	int32_t weight, offset;
	3342	v16i8 src0, src1, src2, src3, src4, src5, src6;
	3343	v8i16 in0, in1, in2, in3;
	3344	v16i8 src10_r, src32_r, src54_r, src21_r, src43_r, src65_r;
	3345	v16i8 src2110, src4332, src6554;
	3346	v8i16 dst10, dst32;
	3347	v4i32 dst10_r, dst32_r, dst10_l, dst32_l;
	3348	v8i16 filt0, filt1;
	3349	v8i16 filter_vec, const_vec;
	3350	v4i32 weight_vec, offset_vec, rnd_vec;
	3351
	3352	src0_ptr -= src_stride;
	3353
	3354	offset = (offset0 + offset1) << rnd_val;
	3355	weight0 = weight0 & 0x0000FFFF;
	3356	weight = weight0 \| (weight1 << 16);
	3357
	3358	const_vec = __msa_ldi_h(128);
	3359	const_vec <<= 6;
	3360	offset_vec = __msa_fill_w(offset);
	3361	weight_vec = __msa_fill_w(weight);
	3362	rnd_vec = __msa_fill_w(rnd_val + 1);
	3363
	3364	filter_vec = LD_SH(filter);
	3365	SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
	3366
	3367	LD_SB3(src0_ptr, src_stride, src0, src1, src2);
	3368	src0_ptr += (3 * src_stride);
	3369	ILVR_B2_SB(src1, src0, src2, src1, src10_r, src21_r);
	3370	src2110 = (v16i8) __msa_ilvr_d((v2i64) src21_r, (v2i64) src10_r);
	3371	src2110 = (v16i8) __msa_xori_b((v16u8) src2110, 128);
	3372
	3373	LD_SB4(src0_ptr, src_stride, src3, src4, src5, src6);
	3374	src0_ptr += (4 * src_stride);
	3375	LD_SH4(src1_ptr, src2_stride, in0, in1, in2, in3);
	3376	src1_ptr += (4 * src2_stride);
	3377	ILVR_D2_SH(in1, in0, in3, in2, in0, in1);
	3378	ILVR_B4_SB(src3, src2, src4, src3, src5, src4, src6, src5,
	3379	src32_r, src43_r, src54_r, src65_r);
	3380	ILVR_D2_SB(src43_r, src32_r, src65_r, src54_r, src4332, src6554);
	3381	XORI_B2_128_SB(src4332, src6554);
	3382
	3383	dst10 = const_vec;
	3384	DPADD_SB2_SH(src2110, src4332, filt0, filt1, dst10, dst10);
	3385	dst32 = const_vec;
	3386	DPADD_SB2_SH(src4332, src6554, filt0, filt1, dst32, dst32);
	3387
	3388	HEVC_BIW_RND_CLIP2(dst10, dst32, in0, in1,
	3389	weight_vec, rnd_vec, offset_vec,
	3390	dst10_r, dst32_r, dst10_l, dst32_l);
	3391
	3392	HEVC_PCK_SW_SB4(dst10_l, dst10_r, dst32_l, dst32_r, dst10_r);
	3393	ST4x4_UB(dst10_r, dst10_r, 0, 1, 2, 3, dst, dst_stride);
	3394	dst += (4 * dst_stride);
	3395	}
	3396
	3397	static void hevc_vt_biwgt_4t_4x8multiple_msa(uint8_t *src0_ptr,
	3398	int32_t src_stride,
	3399	int16_t *src1_ptr,
	3400	int32_t src2_stride,
	3401	uint8_t *dst,
	3402	int32_t dst_stride,
	3403	const int8_t *filter,
	3404	int32_t height,
	3405	int32_t weight0,
	3406	int32_t weight1,
	3407	int32_t offset0,
	3408	int32_t offset1,
	3409	int32_t rnd_val)
	3410	{
	3411	uint32_t loop_cnt;
	3412	int32_t weight, offset;
	3413	v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8, src9;
	3414	v8i16 in0, in1, in2, in3, in4, in5, in6, in7;
	3415	v16i8 src10_r, src32_r, src54_r, src76_r, src98_r;
	3416	v16i8 src21_r, src43_r, src65_r, src87_r, src109_r;
	3417	v16i8 src2110, src4332, src6554, src8776;
	3418	v8i16 dst10, dst32, dst54, dst76;
	3419	v4i32 dst10_r, dst32_r, dst54_r, dst76_r;
	3420	v4i32 dst10_l, dst32_l, dst54_l, dst76_l;
	3421	v8i16 filt0, filt1;
	3422	v8i16 filter_vec, const_vec;
	3423	v4i32 weight_vec, offset_vec, rnd_vec;
	3424
	3425	src0_ptr -= src_stride;
	3426
	3427	offset = (offset0 + offset1) << rnd_val;
	3428	weight0 = weight0 & 0x0000FFFF;
	3429	weight = weight0 \| (weight1 << 16);
	3430
	3431	const_vec = __msa_ldi_h(128);
	3432	const_vec <<= 6;
	3433	offset_vec = __msa_fill_w(offset);
	3434	weight_vec = __msa_fill_w(weight);
	3435	rnd_vec = __msa_fill_w(rnd_val + 1);
	3436
	3437	filter_vec = LD_SH(filter);
	3438	SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
	3439
	3440	LD_SB3(src0_ptr, src_stride, src0, src1, src2);
	3441	src0_ptr += (3 * src_stride);
	3442	ILVR_B2_SB(src1, src0, src2, src1, src10_r, src21_r);
	3443	src2110 = (v16i8) __msa_ilvr_d((v2i64) src21_r, (v2i64) src10_r);
	3444	src2110 = (v16i8) __msa_xori_b((v16u8) src2110, 128);
	3445
	3446	for (loop_cnt = (height >> 3); loop_cnt--;) {
	3447	LD_SB6(src0_ptr, src_stride, src3, src4, src5, src6, src7, src8);
	3448	src0_ptr += (6 * src_stride);
	3449	LD_SH8(src1_ptr, src2_stride, in0, in1, in2, in3, in4, in5, in6, in7);
	3450	src1_ptr += (8 * src2_stride);
	3451
	3452	ILVR_D2_SH(in1, in0, in3, in2, in0, in1);
	3453	ILVR_D2_SH(in5, in4, in7, in6, in2, in3);
	3454
	3455	ILVR_B4_SB(src3, src2, src4, src3, src5, src4, src6, src5,
	3456	src32_r, src43_r, src54_r, src65_r);
	3457	ILVR_B2_SB(src7, src6, src8, src7, src76_r, src87_r);
	3458	ILVR_D3_SB(src43_r, src32_r, src65_r, src54_r, src87_r, src76_r,
	3459	src4332, src6554, src8776);
	3460	XORI_B3_128_SB(src4332, src6554, src8776);
	3461
	3462	dst10 = const_vec;
	3463	DPADD_SB2_SH(src2110, src4332, filt0, filt1, dst10, dst10);
	3464	dst32 = const_vec;
	3465	DPADD_SB2_SH(src4332, src6554, filt0, filt1, dst32, dst32);
	3466	dst54 = const_vec;
	3467	DPADD_SB2_SH(src6554, src8776, filt0, filt1, dst54, dst54);
	3468
	3469	LD_SB2(src0_ptr, src_stride, src9, src2);
	3470	src0_ptr += (2 * src_stride);
	3471	ILVR_B2_SB(src9, src8, src2, src9, src98_r, src109_r);
	3472	src2110 = (v16i8) __msa_ilvr_d((v2i64) src109_r, (v2i64) src98_r);
	3473	src2110 = (v16i8) __msa_xori_b((v16u8) src2110, 128);
	3474
	3475	dst76 = const_vec;
	3476	DPADD_SB2_SH(src8776, src2110, filt0, filt1, dst76, dst76);
	3477	HEVC_BIW_RND_CLIP4(dst10, dst32, dst54, dst76,
	3478	in0, in1, in2, in3,
	3479	weight_vec, rnd_vec, offset_vec,
	3480	dst10_r, dst32_r, dst54_r, dst76_r,
	3481	dst10_l, dst32_l, dst54_l, dst76_l);
	3482
	3483	HEVC_PCK_SW_SB8(dst10_l, dst10_r, dst32_l, dst32_r,
	3484	dst54_l, dst54_r, dst76_l, dst76_r, dst10_r, dst54_r);
	3485	ST4x8_UB(dst10_r, dst54_r, dst, dst_stride);
	3486	dst += (8 * dst_stride);
	3487	}
	3488	}
	3489
	3490	static void hevc_vt_biwgt_4t_4w_msa(uint8_t *src0_ptr,
	3491	int32_t src_stride,
	3492	int16_t *src1_ptr,
	3493	int32_t src2_stride,
	3494	uint8_t *dst,
	3495	int32_t dst_stride,
	3496	const int8_t *filter,
	3497	int32_t height,
	3498	int32_t weight0,
	3499	int32_t weight1,
	3500	int32_t offset0,
	3501	int32_t offset1,
	3502	int32_t rnd_val)
	3503	{
	3504	if (2 == height) {
	3505	hevc_vt_biwgt_4t_4x2_msa(src0_ptr, src_stride, src1_ptr, src2_stride,
	3506	dst, dst_stride, filter, height,
	3507	weight0, weight1, offset0, offset1, rnd_val);
	3508	} else if (4 == height) {
	3509	hevc_vt_biwgt_4t_4x4_msa(src0_ptr, src_stride, src1_ptr, src2_stride,
	3510	dst, dst_stride, filter, height,
	3511	weight0, weight1, offset0, offset1, rnd_val);
	3512	} else if (0 == (height % 8)) {
	3513	hevc_vt_biwgt_4t_4x8multiple_msa(src0_ptr, src_stride,
	3514	src1_ptr, src2_stride,
	3515	dst, dst_stride, filter, height,
	3516	weight0, weight1, offset0, offset1,
	3517	rnd_val);
	3518	}
	3519	}
	3520
	3521	static void hevc_vt_biwgt_4t_6w_msa(uint8_t *src0_ptr,
	3522	int32_t src_stride,
	3523	int16_t *src1_ptr,
	3524	int32_t src2_stride,
	3525	uint8_t *dst,
	3526	int32_t dst_stride,
	3527	const int8_t *filter,
	3528	int32_t height,
	3529	int32_t weight0,
	3530	int32_t weight1,
	3531	int32_t offset0,
	3532	int32_t offset1,
	3533	int32_t rnd_val)
	3534	{
	3535	uint32_t loop_cnt;
	3536	int32_t offset, weight;
	3537	v16i8 src0, src1, src2, src3, src4;
	3538	v8i16 in0, in1, in2, in3;
	3539	v16i8 src10_r, src32_r, src21_r, src43_r;
	3540	v8i16 tmp0, tmp1, tmp2, tmp3;
	3541	v8i16 filt0, filt1;
	3542	v8i16 filter_vec, const_vec;
	3543	v4i32 weight_vec, offset_vec, rnd_vec;
	3544	v4i32 dst0_r, dst1_r, dst2_r, dst3_r, dst0_l, dst1_l, dst2_l, dst3_l;
	3545
	3546	src0_ptr -= src_stride;
	3547
	3548	offset = (offset0 + offset1) << rnd_val;
	3549	weight0 = weight0 & 0x0000FFFF;
	3550	weight = weight0 \| (weight1 << 16);
	3551
	3552	const_vec = __msa_ldi_h(128);
	3553	const_vec <<= 6;
	3554	offset_vec = __msa_fill_w(offset);
	3555	weight_vec = __msa_fill_w(weight);
	3556	rnd_vec = __msa_fill_w(rnd_val + 1);
	3557
	3558	filter_vec = LD_SH(filter);
	3559	SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
	3560
	3561	LD_SB3(src0_ptr, src_stride, src0, src1, src2);
	3562	src0_ptr += (3 * src_stride);
	3563	XORI_B3_128_SB(src0, src1, src2);
	3564	ILVR_B2_SB(src1, src0, src2, src1, src10_r, src21_r);
	3565
	3566	for (loop_cnt = (height >> 2); loop_cnt--;) {
	3567	LD_SB2(src0_ptr, src_stride, src3, src4);
	3568	src0_ptr += (2 * src_stride);
	3569	LD_SH4(src1_ptr, src2_stride, in0, in1, in2, in3);
	3570	src1_ptr += (4 * src2_stride);
	3571	XORI_B2_128_SB(src3, src4);
	3572	ILVR_B2_SB(src3, src2, src4, src3, src32_r, src43_r);
	3573
	3574	tmp0 = const_vec;
	3575	DPADD_SB2_SH(src10_r, src32_r, filt0, filt1, tmp0, tmp0);
	3576	tmp1 = const_vec;
	3577	DPADD_SB2_SH(src21_r, src43_r, filt0, filt1, tmp1, tmp1);
	3578
	3579	LD_SB2(src0_ptr, src_stride, src1, src2);
	3580	src0_ptr += (2 * src_stride);
	3581	XORI_B2_128_SB(src1, src2);
	3582	ILVR_B2_SB(src1, src4, src2, src1, src10_r, src21_r);
	3583
	3584	tmp2 = const_vec;
	3585	DPADD_SB2_SH(src32_r, src10_r, filt0, filt1, tmp2, tmp2);
	3586	tmp3 = const_vec;
	3587	DPADD_SB2_SH(src43_r, src21_r, filt0, filt1, tmp3, tmp3);
	3588	HEVC_BIW_RND_CLIP4(tmp0, tmp1, tmp2, tmp3,
	3589	in0, in1, in2, in3,
	3590	weight_vec, rnd_vec, offset_vec,
	3591	dst0_r, dst1_r, dst2_r, dst3_r,
	3592	dst0_l, dst1_l, dst2_l, dst3_l);
	3593
	3594	HEVC_PCK_SW_SB8(dst0_l, dst0_r, dst1_l, dst1_r,
	3595	dst2_l, dst2_r, dst3_l, dst3_r, dst0_r, dst1_r);
	3596	ST6x4_UB(dst0_r, dst1_r, dst, dst_stride);
	3597	dst += (4 * dst_stride);
	3598	}
	3599	}
	3600
	3601	static void hevc_vt_biwgt_4t_8x2_msa(uint8_t *src0_ptr,
	3602	int32_t src_stride,
	3603	int16_t *src1_ptr,
	3604	int32_t src2_stride,
	3605	uint8_t *dst,
	3606	int32_t dst_stride,
	3607	const int8_t *filter,
	3608	int32_t height,
	3609	int32_t weight0,
	3610	int32_t weight1,
	3611	int32_t offset0,
	3612	int32_t offset1,
	3613	int32_t rnd_val)
	3614	{
	3615	int32_t offset, weight;
	3616	v16i8 src0, src1, src2, src3, src4;
	3617	v8i16 in0, in1, tmp0, tmp1;
	3618	v16i8 src10_r, src32_r, src21_r, src43_r;
	3619	v8i16 filt0, filt1;
	3620	v8i16 filter_vec, const_vec;
	3621	v4i32 weight_vec, offset_vec, rnd_vec;
	3622	v4i32 dst0_r, dst1_r, dst0_l, dst1_l;
	3623
	3624	src0_ptr -= src_stride;
	3625
	3626	offset = (offset0 + offset1) << rnd_val;
	3627	weight0 = weight0 & 0x0000FFFF;
	3628	weight = weight0 \| (weight1 << 16);
	3629
	3630	const_vec = __msa_ldi_h(128);
	3631	const_vec <<= 6;
	3632	offset_vec = __msa_fill_w(offset);
	3633	weight_vec = __msa_fill_w(weight);
	3634	rnd_vec = __msa_fill_w(rnd_val + 1);
	3635
	3636	filter_vec = LD_SH(filter);
	3637	SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
	3638
	3639	LD_SB3(src0_ptr, src_stride, src0, src1, src2);
	3640	src0_ptr += (3 * src_stride);
	3641	XORI_B3_128_SB(src0, src1, src2);
	3642	ILVR_B2_SB(src1, src0, src2, src1, src10_r, src21_r);
	3643
	3644	LD_SB2(src0_ptr, src_stride, src3, src4);
	3645	LD_SH2(src1_ptr, src2_stride, in0, in1);
	3646	XORI_B2_128_SB(src3, src4);
	3647	ILVR_B2_SB(src3, src2, src4, src3, src32_r, src43_r);
	3648
	3649	tmp0 = const_vec;
	3650	DPADD_SB2_SH(src10_r, src32_r, filt0, filt1, tmp0, tmp0);
	3651	tmp1 = const_vec;
	3652	DPADD_SB2_SH(src21_r, src43_r, filt0, filt1, tmp1, tmp1);
	3653	HEVC_BIW_RND_CLIP2(tmp0, tmp1, in0, in1,
	3654	weight_vec, rnd_vec, offset_vec,
	3655	dst0_r, dst1_r, dst0_l, dst1_l);
	3656
	3657	HEVC_PCK_SW_SB4(dst0_l, dst0_r, dst1_l, dst1_r, dst0_r);
	3658	ST8x2_UB(dst0_r, dst, dst_stride);
	3659	}
	3660
	3661	static void hevc_vt_biwgt_4t_8x6_msa(uint8_t *src0_ptr,
	3662	int32_t src_stride,
	3663	int16_t *src1_ptr,
	3664	int32_t src2_stride,
	3665	uint8_t *dst,
	3666	int32_t dst_stride,
	3667	const int8_t *filter,
	3668	int32_t height,
	3669	int32_t weight0,
	3670	int32_t weight1,
	3671	int32_t offset0,
	3672	int32_t offset1,
	3673	int32_t rnd_val)
	3674	{
	3675	int32_t offset, weight;
	3676	v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8;
	3677	v8i16 in0, in1, in2, in3, in4, in5;
	3678	v16i8 src10_r, src32_r, src54_r, src76_r;
	3679	v16i8 src21_r, src43_r, src65_r, src87_r;
	3680	v8i16 tmp0, tmp1, tmp2, tmp3, tmp4, tmp5;
	3681	v8i16 filt0, filt1;
	3682	v8i16 filter_vec, const_vec;
	3683	v4i32 weight_vec, offset_vec, rnd_vec;
	3684	v4i32 dst0_r, dst1_r, dst2_r, dst3_r, dst4_r, dst5_r;
	3685	v4i32 dst0_l, dst1_l, dst2_l, dst3_l, dst4_l, dst5_l;
	3686
	3687	src0_ptr -= src_stride;
	3688
	3689	offset = (offset0 + offset1) << rnd_val;
	3690	weight0 = weight0 & 0x0000FFFF;
	3691	weight = weight0 \| (weight1 << 16);
	3692
	3693	const_vec = __msa_ldi_h(128);
	3694	const_vec <<= 6;
	3695	offset_vec = __msa_fill_w(offset);
	3696	weight_vec = __msa_fill_w(weight);
	3697	rnd_vec = __msa_fill_w(rnd_val + 1);
	3698
	3699	filter_vec = LD_SH(filter);
	3700	SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
	3701
	3702	LD_SB3(src0_ptr, src_stride, src0, src1, src2);
	3703	src0_ptr += (3 * src_stride);
	3704	XORI_B3_128_SB(src0, src1, src2);
	3705	ILVR_B2_SB(src1, src0, src2, src1, src10_r, src21_r);
	3706
	3707	LD_SB6(src0_ptr, src_stride, src3, src4, src5, src6, src7, src8);
	3708	LD_SH6(src1_ptr, src2_stride, in0, in1, in2, in3, in4, in5);
	3709	XORI_B6_128_SB(src3, src4, src5, src6, src7, src8);
	3710	ILVR_B4_SB(src3, src2, src4, src3, src5, src4, src6, src5,
	3711	src32_r, src43_r, src54_r, src65_r);
	3712	ILVR_B2_SB(src7, src6, src8, src7, src76_r, src87_r);
	3713
	3714	tmp0 = const_vec;
	3715	DPADD_SB2_SH(src10_r, src32_r, filt0, filt1, tmp0, tmp0);
	3716	tmp1 = const_vec;
	3717	DPADD_SB2_SH(src21_r, src43_r, filt0, filt1, tmp1, tmp1);
	3718	tmp2 = const_vec;
	3719	DPADD_SB2_SH(src32_r, src54_r, filt0, filt1, tmp2, tmp2);
	3720	tmp3 = const_vec;
	3721	DPADD_SB2_SH(src43_r, src65_r, filt0, filt1, tmp3, tmp3);
	3722	tmp4 = const_vec;
	3723	DPADD_SB2_SH(src54_r, src76_r, filt0, filt1, tmp4, tmp4);
	3724	tmp5 = const_vec;
	3725	DPADD_SB2_SH(src65_r, src87_r, filt0, filt1, tmp5, tmp5);
	3726	HEVC_BIW_RND_CLIP4(tmp0, tmp1, tmp2, tmp3,
	3727	in0, in1, in2, in3,
	3728	weight_vec, rnd_vec, offset_vec,
	3729	dst0_r, dst1_r, dst2_r, dst3_r,
	3730	dst0_l, dst1_l, dst2_l, dst3_l);
	3731	HEVC_BIW_RND_CLIP2(tmp4, tmp5, in4, in5,
	3732	weight_vec, rnd_vec, offset_vec,
	3733	dst4_r, dst5_r, dst4_l, dst5_l);
	3734
	3735	HEVC_PCK_SW_SB12(dst0_l, dst0_r, dst1_l, dst1_r,
	3736	dst2_l, dst2_r, dst3_l, dst3_r,
	3737	dst4_l, dst4_r, dst5_l, dst5_r, dst0_r, dst1_r, dst2_r);
	3738	ST8x4_UB(dst0_r, dst1_r, dst, dst_stride);
	3739	dst += (4 * dst_stride);
	3740	ST8x2_UB(dst2_r, dst, dst_stride);
	3741	}
	3742
	3743	static void hevc_vt_biwgt_4t_8x4multiple_msa(uint8_t *src0_ptr,
	3744	int32_t src_stride,
	3745	int16_t *src1_ptr,
	3746	int32_t src2_stride,
	3747	uint8_t *dst,
	3748	int32_t dst_stride,
	3749	const int8_t *filter,
	3750	int32_t height,
	3751	int32_t weight0,
	3752	int32_t weight1,
	3753	int32_t offset0,
	3754	int32_t offset1,
	3755	int32_t rnd_val)
	3756	{
	3757	uint32_t loop_cnt;
	3758	int32_t offset, weight;
	3759	v16i8 src0, src1, src2, src3, src4;
	3760	v8i16 in0, in1, in2, in3;
	3761	v16i8 src10_r, src32_r, src21_r, src43_r;
	3762	v8i16 tmp0, tmp1, tmp2, tmp3;
	3763	v8i16 filt0, filt1;
	3764	v8i16 filter_vec, const_vec;
	3765	v4i32 weight_vec, offset_vec, rnd_vec;
	3766	v4i32 dst0_r, dst1_r, dst2_r, dst3_r, dst0_l, dst1_l, dst2_l, dst3_l;
	3767
	3768	src0_ptr -= src_stride;
	3769
	3770	offset = (offset0 + offset1) << rnd_val;
	3771	weight0 = weight0 & 0x0000FFFF;
	3772	weight = weight0 \| (weight1 << 16);
	3773
	3774	const_vec = __msa_ldi_h(128);
	3775	const_vec <<= 6;
	3776	offset_vec = __msa_fill_w(offset);
	3777	weight_vec = __msa_fill_w(weight);
	3778	rnd_vec = __msa_fill_w(rnd_val + 1);
	3779
	3780	filter_vec = LD_SH(filter);
	3781	SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
	3782
	3783	LD_SB3(src0_ptr, src_stride, src0, src1, src2);
	3784	src0_ptr += (3 * src_stride);
	3785	XORI_B3_128_SB(src0, src1, src2);
	3786	ILVR_B2_SB(src1, src0, src2, src1, src10_r, src21_r);
	3787
	3788	for (loop_cnt = (height >> 2); loop_cnt--;) {
	3789	LD_SB2(src0_ptr, src_stride, src3, src4);
	3790	src0_ptr += (2 * src_stride);
	3791	LD_SH4(src1_ptr, src2_stride, in0, in1, in2, in3);
	3792	src1_ptr += (4 * src2_stride);
	3793	XORI_B2_128_SB(src3, src4);
	3794	ILVR_B2_SB(src3, src2, src4, src3, src32_r, src43_r);
	3795
	3796	tmp0 = const_vec;
	3797	DPADD_SB2_SH(src10_r, src32_r, filt0, filt1, tmp0, tmp0);
	3798	tmp1 = const_vec;
	3799	DPADD_SB2_SH(src21_r, src43_r, filt0, filt1, tmp1, tmp1);
	3800
	3801	LD_SB2(src0_ptr, src_stride, src1, src2);
	3802	src0_ptr += (2 * src_stride);
	3803	XORI_B2_128_SB(src1, src2);
	3804	ILVR_B2_SB(src1, src4, src2, src1, src10_r, src21_r);
	3805
	3806	tmp2 = const_vec;
	3807	DPADD_SB2_SH(src32_r, src10_r, filt0, filt1, tmp2, tmp2);
	3808	tmp3 = const_vec;
	3809	DPADD_SB2_SH(src43_r, src21_r, filt0, filt1, tmp3, tmp3);
	3810	HEVC_BIW_RND_CLIP4(tmp0, tmp1, tmp2, tmp3,
	3811	in0, in1, in2, in3,
	3812	weight_vec, rnd_vec, offset_vec,
	3813	dst0_r, dst1_r, dst2_r, dst3_r,
	3814	dst0_l, dst1_l, dst2_l, dst3_l);
	3815
	3816	HEVC_PCK_SW_SB8(dst0_l, dst0_r, dst1_l, dst1_r,
	3817	dst2_l, dst2_r, dst3_l, dst3_r, dst0_r, dst1_r);
	3818	ST8x4_UB(dst0_r, dst1_r, dst, dst_stride);
	3819	dst += (4 * dst_stride);
	3820	}
	3821	}
	3822
	3823	static void hevc_vt_biwgt_4t_8w_msa(uint8_t *src0_ptr,
	3824	int32_t src_stride,
	3825	int16_t *src1_ptr,
	3826	int32_t src2_stride,
	3827	uint8_t *dst,
	3828	int32_t dst_stride,
	3829	const int8_t *filter,
	3830	int32_t height,
	3831	int32_t weight0,
	3832	int32_t weight1,
	3833	int32_t offset0,
	3834	int32_t offset1,
	3835	int32_t rnd_val)
	3836	{
	3837	if (2 == height) {
	3838	hevc_vt_biwgt_4t_8x2_msa(src0_ptr, src_stride, src1_ptr, src2_stride,
	3839	dst, dst_stride, filter, height,
	3840	weight0, weight1, offset0, offset1, rnd_val);
	3841	} else if (6 == height) {
	3842	hevc_vt_biwgt_4t_8x6_msa(src0_ptr, src_stride, src1_ptr, src2_stride,
	3843	dst, dst_stride, filter, height,
	3844	weight0, weight1, offset0, offset1, rnd_val);
	3845	} else {
	3846	hevc_vt_biwgt_4t_8x4multiple_msa(src0_ptr, src_stride,
	3847	src1_ptr, src2_stride,
	3848	dst, dst_stride, filter, height,
	3849	weight0, weight1, offset0, offset1,
	3850	rnd_val);
	3851	}
	3852	}
	3853
	3854	static void hevc_vt_biwgt_4t_12w_msa(uint8_t *src0_ptr,
	3855	int32_t src_stride,
	3856	int16_t *src1_ptr,
	3857	int32_t src2_stride,
	3858	uint8_t *dst,
	3859	int32_t dst_stride,
	3860	const int8_t *filter,
	3861	int32_t height,
	3862	int32_t weight0,
	3863	int32_t weight1,
	3864	int32_t offset0,
	3865	int32_t offset1,
	3866	int32_t rnd_val)
	3867	{
	3868	uint32_t loop_cnt;
	3869	int32_t offset, weight;
	3870	v16i8 src0, src1, src2, src3, src4, src5;
	3871	v8i16 in0, in1, in2, in3, in4, in5, in6, in7;
	3872	v16i8 src10_r, src32_r, src21_r, src43_r;
	3873	v8i16 tmp0, tmp1, tmp2, tmp3, tmp4, tmp5;
	3874	v16i8 src10_l, src32_l, src54_l, src21_l, src43_l, src65_l;
	3875	v16i8 src2110, src4332;
	3876	v8i16 filt0, filt1;
	3877	v8i16 filter_vec, const_vec;
	3878	v4i32 weight_vec, offset_vec, rnd_vec;
	3879	v4i32 dst0_r, dst1_r, dst2_r, dst3_r, dst4_r, dst5_r;
	3880	v4i32 dst0_l, dst1_l, dst2_l, dst3_l, dst4_l, dst5_l;
	3881
	3882	src0_ptr -= (1 * src_stride);
	3883
	3884	offset = (offset0 + offset1) << rnd_val;
	3885	weight0 = weight0 & 0x0000FFFF;
	3886	weight = weight0 \| (weight1 << 16);
	3887
	3888	const_vec = __msa_ldi_h(128);
	3889	const_vec <<= 6;
	3890	offset_vec = __msa_fill_w(offset);
	3891	weight_vec = __msa_fill_w(weight);
	3892	rnd_vec = __msa_fill_w(rnd_val + 1);
	3893
	3894	filter_vec = LD_SH(filter);
	3895	SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
	3896
	3897	LD_SB3(src0_ptr, src_stride, src0, src1, src2);
	3898	src0_ptr += (3 * src_stride);
	3899	XORI_B3_128_SB(src0, src1, src2);
	3900	ILVR_B2_SB(src1, src0, src2, src1, src10_r, src21_r);
	3901	ILVL_B2_SB(src1, src0, src2, src1, src10_l, src21_l);
	3902	src2110 = (v16i8) __msa_ilvr_d((v2i64) src21_l, (v2i64) src10_l);
	3903
	3904	for (loop_cnt = (height >> 2); loop_cnt--;) {
	3905	LD_SB2(src0_ptr, src_stride, src3, src4);
	3906	src0_ptr += (2 * src_stride);
	3907	LD_SH4(src1_ptr, src2_stride, in0, in1, in2, in3);
	3908	LD_SH4(src1_ptr + 8, src2_stride, in4, in5, in6, in7);
	3909	src1_ptr += (4 * src2_stride);
	3910	ILVR_D2_SH(in5, in4, in7, in6, in4, in5);
	3911	XORI_B2_128_SB(src3, src4);
	3912
	3913	ILVR_B2_SB(src3, src2, src4, src3, src32_r, src43_r);
	3914	ILVL_B2_SB(src3, src2, src4, src3, src32_l, src43_l);
	3915	src4332 = (v16i8) __msa_ilvr_d((v2i64) src43_l, (v2i64) src32_l);
	3916
	3917	tmp0 = const_vec;
	3918	DPADD_SB2_SH(src10_r, src32_r, filt0, filt1, tmp0, tmp0);
	3919	tmp1 = const_vec;
	3920	DPADD_SB2_SH(src21_r, src43_r, filt0, filt1, tmp1, tmp1);
	3921	tmp4 = const_vec;
	3922	DPADD_SB2_SH(src2110, src4332, filt0, filt1, tmp4, tmp4);
	3923
	3924	LD_SB2(src0_ptr, src_stride, src5, src2);
	3925	src0_ptr += (2 * src_stride);
	3926	XORI_B2_128_SB(src5, src2);
	3927	ILVR_B2_SB(src5, src4, src2, src5, src10_r, src21_r);
	3928	ILVL_B2_SB(src5, src4, src2, src5, src54_l, src65_l);
	3929	src2110 = (v16i8) __msa_ilvr_d((v2i64) src65_l, (v2i64) src54_l);
	3930
	3931	tmp2 = const_vec;
	3932	DPADD_SB2_SH(src32_r, src10_r, filt0, filt1, tmp2, tmp2);
	3933	tmp3 = const_vec;
	3934	DPADD_SB2_SH(src43_r, src21_r, filt0, filt1, tmp3, tmp3);
	3935	tmp5 = const_vec;
	3936	DPADD_SB2_SH(src4332, src2110, filt0, filt1, tmp5, tmp5);
	3937	HEVC_BIW_RND_CLIP4(tmp0, tmp1, tmp2, tmp3,
	3938	in0, in1, in2, in3,
	3939	weight_vec, rnd_vec, offset_vec,
	3940	dst0_r, dst1_r, dst2_r, dst3_r,
	3941	dst0_l, dst1_l, dst2_l, dst3_l);
	3942	HEVC_BIW_RND_CLIP2(tmp4, tmp5, in4, in5,
	3943	weight_vec, rnd_vec, offset_vec,
	3944	dst4_r, dst5_r, dst4_l, dst5_l);
	3945
	3946	HEVC_PCK_SW_SB12(dst0_l, dst0_r, dst1_l, dst1_r,
	3947	dst2_l, dst2_r, dst3_l, dst3_r,
	3948	dst4_l, dst4_r, dst5_l, dst5_r,
	3949	dst0_r, dst1_r, dst2_r);
	3950	ST12x4_UB(dst0_r, dst1_r, dst2_r, dst, dst_stride);
	3951	dst += (4 * dst_stride);
	3952	}
	3953	}
	3954
	3955	static void hevc_vt_biwgt_4t_16w_msa(uint8_t *src0_ptr,
	3956	int32_t src_stride,
	3957	int16_t *src1_ptr,
	3958	int32_t src2_stride,
	3959	uint8_t *dst,
	3960	int32_t dst_stride,
	3961	const int8_t *filter,
	3962	int32_t height,
	3963	int32_t weight0,
	3964	int32_t weight1,
	3965	int32_t offset0,
	3966	int32_t offset1,
	3967	int32_t rnd_val)
	3968	{
	3969	uint32_t loop_cnt;
	3970	int32_t offset, weight;
	3971	v16i8 src0, src1, src2, src3, src4, src5;
	3972	v8i16 in0, in1, in2, in3;
	3973	v16i8 src10_r, src32_r, src21_r, src43_r;
	3974	v16i8 src10_l, src32_l, src21_l, src43_l;
	3975	v8i16 tmp0, tmp1, tmp2, tmp3;
	3976	v8i16 filt0, filt1;
	3977	v8i16 filter_vec, const_vec;
	3978	v4i32 weight_vec, offset_vec, rnd_vec;
	3979	v4i32 dst0_r, dst1_r, dst2_r, dst3_r, dst0_l, dst1_l, dst2_l, dst3_l;
	3980
	3981	src0_ptr -= src_stride;
	3982
	3983	offset = (offset0 + offset1) << rnd_val;
	3984	weight0 = weight0 & 0x0000FFFF;
	3985	weight = weight0 \| (weight1 << 16);
	3986
	3987	const_vec = __msa_ldi_h(128);
	3988	const_vec <<= 6;
	3989	offset_vec = __msa_fill_w(offset);
	3990	weight_vec = __msa_fill_w(weight);
	3991	rnd_vec = __msa_fill_w(rnd_val + 1);
	3992
	3993	filter_vec = LD_SH(filter);
	3994	SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
	3995
	3996	LD_SB3(src0_ptr, src_stride, src0, src1, src2);
	3997	src0_ptr += (3 * src_stride);
	3998	XORI_B3_128_SB(src0, src1, src2);
	3999	ILVR_B2_SB(src1, src0, src2, src1, src10_r, src21_r);
	4000	ILVL_B2_SB(src1, src0, src2, src1, src10_l, src21_l);
	4001
	4002	for (loop_cnt = (height >> 2); loop_cnt--;) {
	4003	LD_SB2(src0_ptr, src_stride, src3, src4);
	4004	src0_ptr += (2 * src_stride);
	4005	LD_SH2(src1_ptr, src2_stride, in0, in1);
	4006	LD_SH2(src1_ptr + 8, src2_stride, in2, in3);
	4007	src1_ptr += (2 * src2_stride);
	4008	XORI_B2_128_SB(src3, src4);
	4009	ILVR_B2_SB(src3, src2, src4, src3, src32_r, src43_r);
	4010	ILVL_B2_SB(src3, src2, src4, src3, src32_l, src43_l);
	4011
	4012	tmp0 = const_vec;
	4013	DPADD_SB2_SH(src10_r, src32_r, filt0, filt1, tmp0, tmp0);
	4014	tmp1 = const_vec;
	4015	DPADD_SB2_SH(src21_r, src43_r, filt0, filt1, tmp1, tmp1);
	4016	tmp2 = const_vec;
	4017	DPADD_SB2_SH(src10_l, src32_l, filt0, filt1, tmp2, tmp2);
	4018	tmp3 = const_vec;
	4019	DPADD_SB2_SH(src21_l, src43_l, filt0, filt1, tmp3, tmp3);
	4020
	4021	HEVC_BIW_RND_CLIP4(tmp0, tmp1, tmp2, tmp3,
	4022	in0, in1, in2, in3,
	4023	weight_vec, rnd_vec, offset_vec,
	4024	dst0_r, dst1_r, dst2_r, dst3_r,
	4025	dst0_l, dst1_l, dst2_l, dst3_l);
	4026	HEVC_PCK_SW_SB8(dst0_l, dst0_r, dst2_l, dst2_r,
	4027	dst1_l, dst1_r, dst3_l, dst3_r, dst0_r, dst1_r);
	4028	ST_SW2(dst0_r, dst1_r, dst, dst_stride);
	4029	dst += (2 * dst_stride);
	4030	LD_SB2(src0_ptr, src_stride, src5, src2);
	4031	src0_ptr += (2 * src_stride);
	4032
	4033	LD_SH2(src1_ptr, src2_stride, in0, in1);
	4034	LD_SH2(src1_ptr + 8, src2_stride, in2, in3);
	4035	src1_ptr += (2 * src2_stride);
	4036	XORI_B2_128_SB(src5, src2);
	4037	ILVR_B2_SB(src5, src4, src2, src5, src10_r, src21_r);
	4038	ILVL_B2_SB(src5, src4, src2, src5, src10_l, src21_l);
	4039
	4040	tmp0 = const_vec;
	4041	DPADD_SB2_SH(src32_r, src10_r, filt0, filt1, tmp0, tmp0);
	4042	tmp1 = const_vec;
	4043	DPADD_SB2_SH(src43_r, src21_r, filt0, filt1, tmp1, tmp1);
	4044	tmp2 = const_vec;
	4045	DPADD_SB2_SH(src32_l, src10_l, filt0, filt1, tmp2, tmp2);
	4046	tmp3 = const_vec;
	4047	DPADD_SB2_SH(src43_l, src21_l, filt0, filt1, tmp3, tmp3);
	4048	HEVC_BIW_RND_CLIP4(tmp0, tmp1, tmp2, tmp3,
	4049	in0, in1, in2, in3,
	4050	weight_vec, rnd_vec, offset_vec,
	4051	dst0_r, dst1_r, dst2_r, dst3_r,
	4052	dst0_l, dst1_l, dst2_l, dst3_l);
	4053
	4054	HEVC_PCK_SW_SB8(dst0_l, dst0_r, dst2_l, dst2_r,
	4055	dst1_l, dst1_r, dst3_l, dst3_r, dst0_r, dst1_r);
	4056	ST_SW2(dst0_r, dst1_r, dst, dst_stride);
	4057	dst += (2 * dst_stride);
	4058	}
	4059	}
	4060
	4061	static void hevc_vt_biwgt_4t_24w_msa(uint8_t *src0_ptr,
	4062	int32_t src_stride,
	4063	int16_t *src1_ptr,
	4064	int32_t src2_stride,
	4065	uint8_t *dst,
	4066	int32_t dst_stride,
	4067	const int8_t *filter,
	4068	int32_t height,
	4069	int32_t weight0,
	4070	int32_t weight1,
	4071	int32_t offset0,
	4072	int32_t offset1,
	4073	int32_t rnd_val)
	4074	{
	4075	uint32_t loop_cnt;
	4076	int32_t offset, weight;
	4077	v16i8 src0, src1, src2, src3, src4, src5;
	4078	v16i8 src6, src7, src8, src9, src10, src11;
	4079	v8i16 in0, in1, in2, in3, in4, in5;
	4080	v16i8 src10_r, src32_r, src76_r, src98_r;
	4081	v16i8 src10_l, src32_l, src21_l, src43_l;
	4082	v16i8 src21_r, src43_r, src87_r, src109_r;
	4083	v8i16 tmp0, tmp1, tmp2, tmp3, tmp4, tmp5;
	4084	v8i16 filt0, filt1;
	4085	v8i16 filter_vec, const_vec;
	4086	v4i32 weight_vec, offset_vec, rnd_vec;
	4087	v4i32 dst0_r, dst1_r, dst2_r, dst3_r, dst4_r, dst5_r;
	4088	v4i32 dst0_l, dst1_l, dst2_l, dst3_l, dst4_l, dst5_l;
	4089
	4090	src0_ptr -= src_stride;
	4091
	4092	offset = (offset0 + offset1) << rnd_val;
	4093	weight0 = weight0 & 0x0000FFFF;
	4094	weight = weight0 \| (weight1 << 16);
	4095
	4096	const_vec = __msa_ldi_h(128);
	4097	const_vec <<= 6;
	4098	offset_vec = __msa_fill_w(offset);
	4099	weight_vec = __msa_fill_w(weight);
	4100	rnd_vec = __msa_fill_w(rnd_val + 1);
	4101
	4102	filter_vec = LD_SH(filter);
	4103	SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
	4104
	4105	/* 16width */
	4106	LD_SB3(src0_ptr, src_stride, src0, src1, src2);
	4107	XORI_B3_128_SB(src0, src1, src2);
	4108	ILVR_B2_SB(src1, src0, src2, src1, src10_r, src21_r);
	4109	ILVL_B2_SB(src1, src0, src2, src1, src10_l, src21_l);
	4110	/* 8width */
	4111	LD_SB3(src0_ptr + 16, src_stride, src6, src7, src8);
	4112	src0_ptr += (3 * src_stride);
	4113	XORI_B3_128_SB(src6, src7, src8);
	4114	ILVR_B2_SB(src7, src6, src8, src7, src76_r, src87_r);
	4115
	4116	for (loop_cnt = (height >> 2); loop_cnt--;) {
	4117	/* 16width */
	4118	LD_SB2(src0_ptr, src_stride, src3, src4);
	4119	LD_SH2(src1_ptr, src2_stride, in0, in1);
	4120	LD_SH2(src1_ptr + 8, src2_stride, in2, in3);
	4121	XORI_B2_128_SB(src3, src4);
	4122	ILVR_B2_SB(src3, src2, src4, src3, src32_r, src43_r);
	4123	ILVL_B2_SB(src3, src2, src4, src3, src32_l, src43_l);
	4124
	4125	/* 8width */
	4126	LD_SB2(src0_ptr + 16, src_stride, src9, src10);
	4127	src0_ptr += (2 * src_stride);
	4128	LD_SH2(src1_ptr + 16, src2_stride, in4, in5);
	4129	src1_ptr += (2 * src2_stride);
	4130	XORI_B2_128_SB(src9, src10);
	4131	ILVR_B2_SB(src9, src8, src10, src9, src98_r, src109_r);
	4132	/* 16width */
	4133	tmp0 = const_vec;
	4134	DPADD_SB2_SH(src10_r, src32_r, filt0, filt1, tmp0, tmp0);
	4135	tmp4 = const_vec;
	4136	DPADD_SB2_SH(src10_l, src32_l, filt0, filt1, tmp4, tmp4);
	4137	tmp1 = const_vec;
	4138	DPADD_SB2_SH(src21_r, src43_r, filt0, filt1, tmp1, tmp1);
	4139	tmp5 = const_vec;
	4140	DPADD_SB2_SH(src21_l, src43_l, filt0, filt1, tmp5, tmp5);
	4141	/* 8width */
	4142	tmp2 = const_vec;
	4143	DPADD_SB2_SH(src76_r, src98_r, filt0, filt1, tmp2, tmp2);
	4144	tmp3 = const_vec;
	4145	DPADD_SB2_SH(src87_r, src109_r, filt0, filt1, tmp3, tmp3);
	4146	/* 16width */
	4147	HEVC_BIW_RND_CLIP4(tmp0, tmp1, tmp4, tmp5,
	4148	in0, in1, in2, in3,
	4149	weight_vec, rnd_vec, offset_vec,
	4150	dst0_r, dst1_r, dst2_r, dst3_r,
	4151	dst0_l, dst1_l, dst2_l, dst3_l);
	4152	/* 8width */
	4153	HEVC_BIW_RND_CLIP2(tmp2, tmp3, in4, in5,
	4154	weight_vec, rnd_vec, offset_vec,
	4155	dst4_r, dst5_r, dst4_l, dst5_l);
	4156	/* 16width */
	4157	HEVC_PCK_SW_SB8(dst0_l, dst0_r, dst2_l, dst2_r,
	4158	dst1_l, dst1_r, dst3_l, dst3_r, dst0_r, dst1_r);
	4159	/* 8width */
	4160	HEVC_PCK_SW_SB4(dst4_l, dst4_r, dst5_l, dst5_r, dst4_r);
	4161	ST_SW2(dst0_r, dst1_r, dst, dst_stride);
	4162	ST8x2_UB(dst4_r, dst + 16, dst_stride);
	4163	dst += (2 * dst_stride);
	4164
	4165	/* 16width */
	4166	LD_SB2(src0_ptr, src_stride, src5, src2);
	4167	LD_SH2(src1_ptr, src2_stride, in0, in1);
	4168	LD_SH2(src1_ptr + 8, src2_stride, in2, in3);
	4169	XORI_B2_128_SB(src5, src2);
	4170	ILVR_B2_SB(src5, src4, src2, src5, src10_r, src21_r);
	4171	ILVL_B2_SB(src5, src4, src2, src5, src10_l, src21_l);
	4172	/* 8width */
	4173	LD_SB2(src0_ptr + 16, src_stride, src11, src8);
	4174	src0_ptr += (2 * src_stride);
	4175	LD_SH2(src1_ptr + 16, src2_stride, in4, in5);
	4176	src1_ptr += (2 * src2_stride);
	4177	XORI_B2_128_SB(src11, src8);
	4178	ILVR_B2_SB(src11, src10, src8, src11, src76_r, src87_r);
	4179	/* 16width */
	4180	tmp0 = const_vec;
	4181	DPADD_SB2_SH(src32_r, src10_r, filt0, filt1, tmp0, tmp0);
	4182	tmp4 = const_vec;
	4183	DPADD_SB2_SH(src32_l, src10_l, filt0, filt1, tmp4, tmp4);
	4184	tmp1 = const_vec;
	4185	DPADD_SB2_SH(src43_r, src21_r, filt0, filt1, tmp1, tmp1);
	4186	tmp5 = const_vec;
	4187	DPADD_SB2_SH(src43_l, src21_l, filt0, filt1, tmp5, tmp5);
	4188	/* 8width */
	4189	tmp2 = const_vec;
	4190	DPADD_SB2_SH(src98_r, src76_r, filt0, filt1, tmp2, tmp2);
	4191	tmp3 = const_vec;
	4192	DPADD_SB2_SH(src109_r, src87_r, filt0, filt1, tmp3, tmp3);
	4193	/* 16width */
	4194	HEVC_BIW_RND_CLIP4(tmp0, tmp1, tmp4, tmp5,
	4195	in0, in1, in2, in3,
	4196	weight_vec, rnd_vec, offset_vec,
	4197	dst0_r, dst1_r, dst2_r, dst3_r,
	4198	dst0_l, dst1_l, dst2_l, dst3_l);
	4199	/* 8width */
	4200	HEVC_BIW_RND_CLIP2(tmp2, tmp3, in4, in5,
	4201	weight_vec, rnd_vec, offset_vec,
	4202	dst4_r, dst5_r, dst4_l, dst5_l);
	4203	/* 16width */
	4204	HEVC_PCK_SW_SB8(dst0_l, dst0_r, dst2_l, dst2_r,
	4205	dst1_l, dst1_r, dst3_l, dst3_r, dst0_r, dst1_r);
	4206
	4207	/* 8width */
	4208	HEVC_PCK_SW_SB4(dst4_l, dst4_r, dst5_l, dst5_r, dst4_r);
	4209	ST_SW2(dst0_r, dst1_r, dst, dst_stride);
	4210	ST8x2_UB(dst4_r, dst + 16, dst_stride);
	4211	dst += (2 * dst_stride);
	4212	}
	4213	}
	4214
	4215	static void hevc_vt_biwgt_4t_32w_msa(uint8_t *src0_ptr,
	4216	int32_t src_stride,
	4217	int16_t *src1_ptr,
	4218	int32_t src2_stride,
	4219	uint8_t *dst,
	4220	int32_t dst_stride,
	4221	const int8_t *filter,
	4222	int32_t height,
	4223	int32_t weight0,
	4224	int32_t weight1,
	4225	int32_t offset0,
	4226	int32_t offset1,
	4227	int32_t rnd_val)
	4228	{
	4229	uint32_t loop_cnt;
	4230	uint8_t *dst_tmp = dst + 16;
	4231	int32_t offset, weight;
	4232	v16i8 src0, src1, src2, src3, src4, src6, src7, src8, src9, src10;
	4233	v8i16 in0, in1, in2, in3, in4, in5, in6, in7;
	4234	v16i8 src10_r, src32_r, src76_r, src98_r;
	4235	v16i8 src21_r, src43_r, src87_r, src109_r;
	4236	v8i16 tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7;
	4237	v16i8 src10_l, src32_l, src76_l, src98_l;
	4238	v16i8 src21_l, src43_l, src87_l, src109_l;
	4239	v8i16 filt0, filt1;
	4240	v8i16 filter_vec, const_vec;
	4241	v4i32 weight_vec, offset_vec, rnd_vec;
	4242	v4i32 dst0_r, dst1_r, dst2_r, dst3_r, dst4_r, dst5_r, dst6_r, dst7_r;
	4243	v4i32 dst0_l, dst1_l, dst2_l, dst3_l, dst4_l, dst5_l, dst6_l, dst7_l;
	4244
	4245	src0_ptr -= src_stride;
	4246
	4247	offset = (offset0 + offset1) << rnd_val;
	4248	weight0 = weight0 & 0x0000FFFF;
	4249	weight = weight0 \| (weight1 << 16);
	4250
	4251	const_vec = __msa_ldi_h(128);
	4252	const_vec <<= 6;
	4253	offset_vec = __msa_fill_w(offset);
	4254	weight_vec = __msa_fill_w(weight);
	4255	rnd_vec = __msa_fill_w(rnd_val + 1);
	4256
	4257	filter_vec = LD_SH(filter);
	4258	SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
	4259
	4260	/* 16width */
	4261	LD_SB3(src0_ptr, src_stride, src0, src1, src2);
	4262	XORI_B3_128_SB(src0, src1, src2);
	4263	ILVR_B2_SB(src1, src0, src2, src1, src10_r, src21_r);
	4264	ILVL_B2_SB(src1, src0, src2, src1, src10_l, src21_l);
	4265	/* next 16width */
	4266	LD_SB3(src0_ptr + 16, src_stride, src6, src7, src8);
	4267	src0_ptr += (3 * src_stride);
	4268	XORI_B3_128_SB(src6, src7, src8);
	4269	ILVR_B2_SB(src7, src6, src8, src7, src76_r, src87_r);
	4270	ILVL_B2_SB(src7, src6, src8, src7, src76_l, src87_l);
	4271
	4272	for (loop_cnt = (height >> 1); loop_cnt--;) {
	4273	/* 16width */
	4274	LD_SB2(src0_ptr, src_stride, src3, src4);
	4275	LD_SH2(src1_ptr, src2_stride, in0, in1);
	4276	LD_SH2(src1_ptr + 8, src2_stride, in2, in3);
	4277	XORI_B2_128_SB(src3, src4);
	4278	ILVR_B2_SB(src3, src2, src4, src3, src32_r, src43_r);
	4279	ILVL_B2_SB(src3, src2, src4, src3, src32_l, src43_l);
	4280
	4281	/* 16width */
	4282	tmp0 = const_vec;
	4283	DPADD_SB2_SH(src10_r, src32_r, filt0, filt1, tmp0, tmp0);
	4284	tmp4 = const_vec;
	4285	DPADD_SB2_SH(src10_l, src32_l, filt0, filt1, tmp4, tmp4);
	4286	tmp1 = const_vec;
	4287	DPADD_SB2_SH(src21_r, src43_r, filt0, filt1, tmp1, tmp1);
	4288	tmp5 = const_vec;
	4289	DPADD_SB2_SH(src21_l, src43_l, filt0, filt1, tmp5, tmp5);
	4290	/* 16width */
	4291	HEVC_BIW_RND_CLIP4(tmp0, tmp1, tmp4, tmp5,
	4292	in0, in1, in2, in3,
	4293	weight_vec, rnd_vec, offset_vec,
	4294	dst0_r, dst1_r, dst2_r, dst3_r,
	4295	dst0_l, dst1_l, dst2_l, dst3_l);
	4296	/* 16width */
	4297	HEVC_PCK_SW_SB8(dst0_l, dst0_r, dst2_l, dst2_r,
	4298	dst1_l, dst1_r, dst3_l, dst3_r, dst0_r, dst1_r);
	4299	ST_SW2(dst0_r, dst1_r, dst, dst_stride);
	4300	dst += (2 * dst_stride);
	4301
	4302	src10_r = src32_r;
	4303	src21_r = src43_r;
	4304	src10_l = src32_l;
	4305	src21_l = src43_l;
	4306	src2 = src4;
	4307
	4308	/* next 16width */
	4309	LD_SB2(src0_ptr + 16, src_stride, src9, src10);
	4310	src0_ptr += (2 * src_stride);
	4311	LD_SH2(src1_ptr + 16, src2_stride, in4, in5);
	4312	LD_SH2(src1_ptr + 24, src2_stride, in6, in7);
	4313	src1_ptr += (2 * src2_stride);
	4314	XORI_B2_128_SB(src9, src10);
	4315	ILVR_B2_SB(src9, src8, src10, src9, src98_r, src109_r);
	4316	ILVL_B2_SB(src9, src8, src10, src9, src98_l, src109_l);
	4317	/* next 16width */
	4318	tmp2 = const_vec;
	4319	DPADD_SB2_SH(src76_r, src98_r, filt0, filt1, tmp2, tmp2);
	4320	tmp6 = const_vec;
	4321	DPADD_SB2_SH(src76_l, src98_l, filt0, filt1, tmp6, tmp6);
	4322	tmp3 = const_vec;
	4323	DPADD_SB2_SH(src87_r, src109_r, filt0, filt1, tmp3, tmp3);
	4324	tmp7 = const_vec;
	4325	DPADD_SB2_SH(src87_l, src109_l, filt0, filt1, tmp7, tmp7);
	4326	/* next 16width */
	4327	HEVC_BIW_RND_CLIP4(tmp2, tmp3, tmp6, tmp7,
	4328	in4, in5, in6, in7,
	4329	weight_vec, rnd_vec, offset_vec,
	4330	dst4_r, dst5_r, dst6_r, dst7_r,
	4331	dst4_l, dst5_l, dst6_l, dst7_l);
	4332
	4333	/* next 16width */
	4334	HEVC_PCK_SW_SB8(dst4_l, dst4_r, dst6_l, dst6_r,
	4335	dst5_l, dst5_r, dst7_l, dst7_r, dst4_r, dst5_r);
	4336	ST_SW2(dst4_r, dst5_r, dst_tmp, dst_stride);
	4337	dst_tmp += (2 * dst_stride);
	4338
	4339	src76_r = src98_r;
	4340	src87_r = src109_r;
	4341	src76_l = src98_l;
	4342	src87_l = src109_l;
	4343	src8 = src10;
	4344	}
	4345	}
	4346
	4347	static void hevc_hv_biwgt_4t_4x2_msa(uint8_t *src0_ptr,
	4348	int32_t src_stride,
	4349	int16_t *src1_ptr,
	4350	int32_t src2_stride,
	4351	uint8_t *dst,
	4352	int32_t dst_stride,
	4353	const int8_t *filter_x,
	4354	const int8_t *filter_y,
	4355	int32_t height,
	4356	int32_t weight0,
	4357	int32_t weight1,
	4358	int32_t offset0,
	4359	int32_t offset1,
	4360	int32_t rnd_val)
	4361	{
	4362	int32_t offset, weight;
	4363	v8i16 in0, in1;
	4364	v16i8 src0, src1, src2, src3, src4;
	4365	v8i16 filt0, filt1;
	4366	v4i32 filt_h0, filt_h1;
	4367	v16i8 mask0 = { 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8 };
	4368	v16i8 mask1;
	4369	v8i16 filter_vec, const_vec;
	4370	v16i8 vec0, vec1, vec2, vec3, vec4, vec5;
	4371	v8i16 dst0, dst1, dst2, dst3, dst4;
	4372	v4i32 dst0_r, dst1_r, dst0_l;
	4373	v8i16 dst10_r, dst32_r, dst21_r, dst43_r;
	4374	v4i32 weight_vec, offset_vec, rnd_vec;
	4375
	4376	src0_ptr -= (src_stride + 1);
	4377
	4378	filter_vec = LD_SH(filter_x);
	4379	SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
	4380
	4381	filter_vec = LD_SH(filter_y);
	4382	vec0 = __msa_clti_s_b((v16i8) filter_vec, 0);
	4383	filter_vec = (v8i16) __msa_ilvr_b(vec0, (v16i8) filter_vec);
	4384
	4385	SPLATI_W2_SW(filter_vec, 0, filt_h0, filt_h1);
	4386
	4387	mask1 = mask0 + 2;
	4388
	4389	offset = (offset0 + offset1) << rnd_val;
	4390	weight0 = weight0 & 0x0000FFFF;
	4391	weight = weight0 \| (weight1 << 16);
	4392
	4393	const_vec = __msa_ldi_h(128);
	4394	const_vec <<= 6;
	4395	offset_vec = __msa_fill_w(offset);
	4396	weight_vec = __msa_fill_w(weight);
	4397	rnd_vec = __msa_fill_w(rnd_val + 1);
	4398
	4399	LD_SB3(src0_ptr, src_stride, src0, src1, src2);
	4400	src0_ptr += (3 * src_stride);
	4401	XORI_B3_128_SB(src0, src1, src2);
	4402
	4403	VSHF_B2_SB(src0, src0, src0, src0, mask0, mask1, vec0, vec1);
	4404	VSHF_B2_SB(src1, src1, src1, src1, mask0, mask1, vec2, vec3);
	4405	VSHF_B2_SB(src2, src2, src2, src2, mask0, mask1, vec4, vec5);
	4406	dst0 = const_vec;
	4407	DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst0, dst0);
	4408	dst1 = const_vec;
	4409	DPADD_SB2_SH(vec2, vec3, filt0, filt1, dst1, dst1);
	4410	dst2 = const_vec;
	4411	DPADD_SB2_SH(vec4, vec5, filt0, filt1, dst2, dst2);
	4412	ILVR_H2_SH(dst1, dst0, dst2, dst1, dst10_r, dst21_r);
	4413
	4414	LD_SB2(src0_ptr, src_stride, src3, src4);
	4415	LD_SH2(src1_ptr, src2_stride, in0, in1);
	4416	in0 = (v8i16) __msa_ilvr_d((v2i64) in1, (v2i64) in0);
	4417	XORI_B2_128_SB(src3, src4);
	4418	/* row 3 */
	4419	VSHF_B2_SB(src3, src3, src3, src3, mask0, mask1, vec0, vec1);
	4420	dst3 = const_vec;
	4421	DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst3, dst3);
	4422	dst32_r = __msa_ilvr_h(dst3, dst2);
	4423	dst0_r = HEVC_FILT_4TAP(dst10_r, dst32_r, filt_h0, filt_h1);
	4424	dst0_r >>= 6;
	4425	/* row 4 */
	4426	VSHF_B2_SB(src4, src4, src4, src4, mask0, mask1, vec0, vec1);
	4427	dst4 = const_vec;
	4428	DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst4, dst4);
	4429	dst43_r = __msa_ilvr_h(dst4, dst3);
	4430	dst1_r = HEVC_FILT_4TAP(dst21_r, dst43_r, filt_h0, filt_h1);
	4431	dst1_r >>= 6;
	4432	dst1_r = (v4i32) __msa_pckev_h((v8i16) dst1_r, (v8i16) dst0_r);
	4433
	4434	ILVRL_H2_SW(dst1_r, in0, dst0_r, dst0_l);
	4435	dst0_r = __msa_dpadd_s_w(offset_vec, (v8i16) dst0_r, (v8i16) weight_vec);
	4436	dst0_l = __msa_dpadd_s_w(offset_vec, (v8i16) dst0_l, (v8i16) weight_vec);
	4437	SRAR_W2_SW(dst0_r, dst0_l, rnd_vec);
	4438	dst0_r = CLIP_SW_0_255(dst0_r);
	4439	dst0_l = CLIP_SW_0_255(dst0_l);
	4440
	4441	HEVC_PCK_SW_SB2(dst0_l, dst0_r, dst0_r);
	4442	ST4x2_UB(dst0_r, dst, dst_stride);
	4443	}
	4444
	4445	static void hevc_hv_biwgt_4t_4x4_msa(uint8_t *src0_ptr,
	4446	int32_t src_stride,
	4447	int16_t *src1_ptr,
	4448	int32_t src2_stride,
	4449	uint8_t *dst,
	4450	int32_t dst_stride,
	4451	const int8_t *filter_x,
	4452	const int8_t *filter_y,
	4453	int32_t height,
	4454	int32_t weight0,
	4455	int32_t weight1,
	4456	int32_t offset0,
	4457	int32_t offset1,
	4458	int32_t rnd_val)
	4459	{
	4460	int32_t offset, weight;
	4461	v8i16 in0, in1, in2, in3;
	4462	v16i8 src0, src1, src2, src3, src4, src5, src6;
	4463	v8i16 filt0, filt1;
	4464	v4i32 filt_h0, filt_h1;
	4465	v16i8 mask0 = { 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8 };
	4466	v16i8 mask1;
	4467	v8i16 filter_vec, const_vec;
	4468	v16i8 vec0, vec1, vec2, vec3, vec4, vec5;
	4469	v8i16 dst0, dst1, dst2, dst3, dst4, dst5;
	4470	v8i16 tmp0, tmp1;
	4471	v4i32 dst0_l, dst1_l;
	4472	v4i32 dst0_r, dst1_r, dst2_r, dst3_r;
	4473	v8i16 dst10_r, dst32_r, dst21_r, dst43_r;
	4474	v4i32 weight_vec, offset_vec, rnd_vec;
	4475
	4476	src0_ptr -= (src_stride + 1);
	4477
	4478	filter_vec = LD_SH(filter_x);
	4479	SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
	4480
	4481	filter_vec = LD_SH(filter_y);
	4482	vec0 = __msa_clti_s_b((v16i8) filter_vec, 0);
	4483	filter_vec = (v8i16) __msa_ilvr_b(vec0, (v16i8) filter_vec);
	4484
	4485	SPLATI_W2_SW(filter_vec, 0, filt_h0, filt_h1);
	4486
	4487	mask1 = mask0 + 2;
	4488
	4489	offset = (offset0 + offset1) << rnd_val;
	4490	weight0 = weight0 & 0x0000FFFF;
	4491	weight = weight0 \| (weight1 << 16);
	4492
	4493	const_vec = __msa_ldi_h(128);
	4494	const_vec <<= 6;
	4495	offset_vec = __msa_fill_w(offset);
	4496	weight_vec = __msa_fill_w(weight);
	4497	rnd_vec = __msa_fill_w(rnd_val + 1);
	4498
	4499	LD_SB3(src0_ptr, src_stride, src0, src1, src2);
	4500	src0_ptr += (3 * src_stride);
	4501	XORI_B3_128_SB(src0, src1, src2);
	4502
	4503	VSHF_B2_SB(src0, src0, src0, src0, mask0, mask1, vec0, vec1);
	4504	VSHF_B2_SB(src1, src1, src1, src1, mask0, mask1, vec2, vec3);
	4505	VSHF_B2_SB(src2, src2, src2, src2, mask0, mask1, vec4, vec5);
	4506	dst0 = const_vec;
	4507	DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst0, dst0);
	4508	dst1 = const_vec;
	4509	DPADD_SB2_SH(vec2, vec3, filt0, filt1, dst1, dst1);
	4510	dst2 = const_vec;
	4511	DPADD_SB2_SH(vec4, vec5, filt0, filt1, dst2, dst2);
	4512	ILVR_H2_SH(dst1, dst0, dst2, dst1, dst10_r, dst21_r);
	4513
	4514	LD_SB4(src0_ptr, src_stride, src3, src4, src5, src6);
	4515	LD_SH4(src1_ptr, src2_stride, in0, in1, in2, in3);
	4516	ILVR_D2_SH(in1, in0, in3, in2, in0, in1);
	4517	XORI_B4_128_SB(src3, src4, src5, src6);
	4518	/* row 3 */
	4519	VSHF_B2_SB(src3, src3, src3, src3, mask0, mask1, vec0, vec1);
	4520	dst3 = const_vec;
	4521	DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst3, dst3);
	4522	dst32_r = __msa_ilvr_h(dst3, dst2);
	4523	dst0_r = HEVC_FILT_4TAP(dst10_r, dst32_r, filt_h0, filt_h1);
	4524	dst0_r >>= 6;
	4525	/* row 4 */
	4526	VSHF_B2_SB(src4, src4, src4, src4, mask0, mask1, vec0, vec1);
	4527	dst4 = const_vec;
	4528	DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst4, dst4);
	4529	dst43_r = __msa_ilvr_h(dst4, dst3);
	4530	dst1_r = HEVC_FILT_4TAP(dst21_r, dst43_r, filt_h0, filt_h1);
	4531	dst1_r >>= 6;
	4532	/* row 5 */
	4533	VSHF_B2_SB(src5, src5, src5, src5, mask0, mask1, vec0, vec1);
	4534	dst5 = const_vec;
	4535	DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst5, dst5);
	4536	dst10_r = __msa_ilvr_h(dst5, dst4);
	4537	dst2_r = HEVC_FILT_4TAP(dst32_r, dst10_r, filt_h0, filt_h1);
	4538	dst2_r >>= 6;
	4539	/* row 6 */
	4540	VSHF_B2_SB(src6, src6, src6, src6, mask0, mask1, vec0, vec1);
	4541	dst2 = const_vec;
	4542	DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst2, dst2);
	4543	dst21_r = __msa_ilvr_h(dst2, dst5);
	4544	dst3_r = HEVC_FILT_4TAP(dst43_r, dst21_r, filt_h0, filt_h1);
	4545	dst3_r >>= 6;
	4546	PCKEV_H2_SH(dst1_r, dst0_r, dst3_r, dst2_r, tmp0, tmp1);
	4547	HEVC_BIW_RND_CLIP2(tmp0, tmp1, in0, in1,
	4548	weight_vec, rnd_vec, offset_vec,
	4549	dst0_r, dst1_r, dst0_l, dst1_l);
	4550
	4551	HEVC_PCK_SW_SB4(dst0_l, dst0_r, dst1_l, dst1_r, dst0_r);
	4552	ST4x4_UB(dst0_r, dst0_r, 0, 1, 2, 3, dst, dst_stride);
	4553	}
	4554
	4555	static void hevc_hv_biwgt_4t_4multx8mult_msa(uint8_t *src0_ptr,
	4556	int32_t src_stride,
	4557	int16_t *src1_ptr,
	4558	int32_t src2_stride,
	4559	uint8_t *dst,
	4560	int32_t dst_stride,
	4561	const int8_t *filter_x,
	4562	const int8_t *filter_y,
	4563	int32_t height,
	4564	int32_t weight0,
	4565	int32_t weight1,
	4566	int32_t offset0,
	4567	int32_t offset1,
	4568	int32_t rnd_val)
	4569	{
	4570	uint32_t loop_cnt;
	4571	int32_t offset, weight;
	4572	v8i16 in0, in1, in2, in3, in4, in5, in6, in7;
	4573	v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10;
	4574	v8i16 filt0, filt1;
	4575	v4i32 filt_h0, filt_h1;
	4576	v16i8 mask0 = { 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8 };
	4577	v16i8 mask1;
	4578	v8i16 filter_vec, const_vec;
	4579	v16i8 vec0, vec1, vec2, vec3, vec4, vec5;
	4580	v8i16 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7, dst8, dst9;
	4581	v4i32 dst0_r, dst1_r, dst2_r, dst3_r, dst4_r, dst5_r, dst6_r, dst7_r;
	4582	v4i32 dst0_l, dst1_l, dst2_l, dst3_l;
	4583	v8i16 tmp0, tmp1, tmp2, tmp3;
	4584	v8i16 dst10_r, dst32_r, dst54_r, dst76_r;
	4585	v8i16 dst21_r, dst43_r, dst65_r, dst87_r;
	4586	v4i32 weight_vec, offset_vec, rnd_vec;
	4587
	4588	src0_ptr -= (src_stride + 1);
	4589
	4590	filter_vec = LD_SH(filter_x);
	4591	SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
	4592
	4593	filter_vec = LD_SH(filter_y);
	4594	vec0 = __msa_clti_s_b((v16i8) filter_vec, 0);
	4595	filter_vec = (v8i16) __msa_ilvr_b(vec0, (v16i8) filter_vec);
	4596
	4597	SPLATI_W2_SW(filter_vec, 0, filt_h0, filt_h1);
	4598
	4599	mask1 = mask0 + 2;
	4600
	4601	offset = (offset0 + offset1) << rnd_val;
	4602	weight0 = weight0 & 0x0000FFFF;
	4603	weight = weight0 \| (weight1 << 16);
	4604
	4605	const_vec = __msa_ldi_h(128);
	4606	const_vec <<= 6;
	4607	offset_vec = __msa_fill_w(offset);
	4608	weight_vec = __msa_fill_w(weight);
	4609	rnd_vec = __msa_fill_w(rnd_val + 1);
	4610
	4611	LD_SB3(src0_ptr, src_stride, src0, src1, src2);
	4612	src0_ptr += (3 * src_stride);
	4613	XORI_B3_128_SB(src0, src1, src2);
	4614
	4615	VSHF_B2_SB(src0, src0, src0, src0, mask0, mask1, vec0, vec1);
	4616	VSHF_B2_SB(src1, src1, src1, src1, mask0, mask1, vec2, vec3);
	4617	VSHF_B2_SB(src2, src2, src2, src2, mask0, mask1, vec4, vec5);
	4618	dst0 = const_vec;
	4619	DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst0, dst0);
	4620	dst1 = const_vec;
	4621	DPADD_SB2_SH(vec2, vec3, filt0, filt1, dst1, dst1);
	4622	dst2 = const_vec;
	4623	DPADD_SB2_SH(vec4, vec5, filt0, filt1, dst2, dst2);
	4624	ILVR_H2_SH(dst1, dst0, dst2, dst1, dst10_r, dst21_r);
	4625
	4626	for (loop_cnt = height >> 3; loop_cnt--;) {
	4627	LD_SB8(src0_ptr, src_stride,
	4628	src3, src4, src5, src6, src7, src8, src9, src10);
	4629	src0_ptr += (8 * src_stride);
	4630	LD_SH8(src1_ptr, src2_stride, in0, in1, in2, in3, in4, in5, in6, in7);
	4631	src1_ptr += (8 * src2_stride);
	4632	ILVR_D2_SH(in1, in0, in3, in2, in0, in1);
	4633	ILVR_D2_SH(in5, in4, in7, in6, in2, in3);
	4634	XORI_B8_128_SB(src3, src4, src5, src6, src7, src8, src9, src10);
	4635	/* row 3 */
	4636	VSHF_B2_SB(src3, src3, src3, src3, mask0, mask1, vec0, vec1);
	4637	dst3 = const_vec;
	4638	DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst3, dst3);
	4639	dst32_r = __msa_ilvr_h(dst3, dst2);
	4640	dst0_r = HEVC_FILT_4TAP(dst10_r, dst32_r, filt_h0, filt_h1);
	4641	dst0_r >>= 6;
	4642	/* row 4 */
	4643	VSHF_B2_SB(src4, src4, src4, src4, mask0, mask1, vec0, vec1);
	4644	dst4 = const_vec;
	4645	DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst4, dst4);
	4646	dst43_r = __msa_ilvr_h(dst4, dst3);
	4647	dst1_r = HEVC_FILT_4TAP(dst21_r, dst43_r, filt_h0, filt_h1);
	4648	dst1_r >>= 6;
	4649	/* row 5 */
	4650	VSHF_B2_SB(src5, src5, src5, src5, mask0, mask1, vec0, vec1);
	4651	dst5 = const_vec;
	4652	DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst5, dst5);
	4653	dst54_r = __msa_ilvr_h(dst5, dst4);
	4654	dst2_r = HEVC_FILT_4TAP(dst32_r, dst54_r, filt_h0, filt_h1);
	4655	dst2_r >>= 6;
	4656	/* row 6 */
	4657	VSHF_B2_SB(src6, src6, src6, src6, mask0, mask1, vec0, vec1);
	4658	dst6 = const_vec;
	4659	DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst6, dst6);
	4660	dst65_r = __msa_ilvr_h(dst6, dst5);
	4661	dst3_r = HEVC_FILT_4TAP(dst43_r, dst65_r, filt_h0, filt_h1);
	4662	dst3_r >>= 6;
	4663	VSHF_B2_SB(src7, src7, src7, src7, mask0, mask1, vec0, vec1);
	4664	dst7 = const_vec;
	4665	DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst7, dst7);
	4666	dst76_r = __msa_ilvr_h(dst7, dst6);
	4667	dst4_r = HEVC_FILT_4TAP(dst54_r, dst76_r, filt_h0, filt_h1);
	4668	dst4_r >>= 6;
	4669	VSHF_B2_SB(src8, src8, src8, src8, mask0, mask1, vec0, vec1);
	4670	dst8 = const_vec;
	4671	DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst8, dst8);
	4672	dst87_r = __msa_ilvr_h(dst8, dst7);
	4673	dst5_r = HEVC_FILT_4TAP(dst65_r, dst87_r, filt_h0, filt_h1);
	4674	dst5_r >>= 6;
	4675	VSHF_B2_SB(src9, src9, src9, src9, mask0, mask1, vec0, vec1);
	4676	dst9 = const_vec;
	4677	DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst9, dst9);
	4678	dst10_r = __msa_ilvr_h(dst9, dst8);
	4679	dst6_r = HEVC_FILT_4TAP(dst76_r, dst10_r, filt_h0, filt_h1);
	4680	dst6_r >>= 6;
	4681	VSHF_B2_SB(src10, src10, src10, src10, mask0, mask1, vec0, vec1);
	4682	dst2 = const_vec;
	4683	DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst2, dst2);
	4684	dst21_r = __msa_ilvr_h(dst2, dst9);
	4685	dst7_r = HEVC_FILT_4TAP(dst87_r, dst21_r, filt_h0, filt_h1);
	4686	dst7_r >>= 6;
	4687	PCKEV_H4_SH(dst1_r, dst0_r, dst3_r, dst2_r,
	4688	dst5_r, dst4_r, dst7_r, dst6_r, tmp0, tmp1, tmp2, tmp3);
	4689	HEVC_BIW_RND_CLIP4(tmp0, tmp1, tmp2, tmp3,
	4690	in0, in1, in2, in3,
	4691	weight_vec, rnd_vec, offset_vec,
	4692	dst0_r, dst1_r, dst2_r, dst3_r,
	4693	dst0_l, dst1_l, dst2_l, dst3_l);
	4694
	4695	HEVC_PCK_SW_SB8(dst0_l, dst0_r, dst1_l, dst1_r,
	4696	dst2_l, dst2_r, dst3_l, dst3_r, dst0_r, dst1_r);
	4697	ST4x8_UB(dst0_r, dst1_r, dst, dst_stride);
	4698	dst += (8 * dst_stride);
	4699	}
	4700	}
	4701
	4702	static void hevc_hv_biwgt_4t_4w_msa(uint8_t *src0_ptr,
	4703	int32_t src_stride,
	4704	int16_t *src1_ptr,
	4705	int32_t src2_stride,
	4706	uint8_t *dst,
	4707	int32_t dst_stride,
	4708	const int8_t *filter_x,
	4709	const int8_t *filter_y,
	4710	int32_t height,
	4711	int32_t weight0,
	4712	int32_t weight1,
	4713	int32_t offset0,
	4714	int32_t offset1,
	4715	int32_t rnd_val)
	4716	{
	4717	if (2 == height) {
	4718	hevc_hv_biwgt_4t_4x2_msa(src0_ptr, src_stride, src1_ptr, src2_stride,
	4719	dst, dst_stride, filter_x, filter_y,
	4720	height, weight0, weight1, offset0, offset1,
	4721	rnd_val);
	4722	} else if (4 == height) {
	4723	hevc_hv_biwgt_4t_4x4_msa(src0_ptr, src_stride, src1_ptr, src2_stride,
	4724	dst, dst_stride, filter_x, filter_y,
	4725	height, weight0, weight1, offset0, offset1,
	4726	rnd_val);
	4727	} else if (0 == (height % 8)) {
	4728	hevc_hv_biwgt_4t_4multx8mult_msa(src0_ptr, src_stride,
	4729	src1_ptr, src2_stride,
	4730	dst, dst_stride, filter_x, filter_y,
	4731	height, weight0, weight1,
	4732	offset0, offset1, rnd_val);
	4733	}
	4734	}
	4735
	4736	static void hevc_hv_biwgt_4t_6w_msa(uint8_t *src0_ptr,
	4737	int32_t src_stride,
	4738	int16_t *src1_ptr,
	4739	int32_t src2_stride,
	4740	uint8_t *dst,
	4741	int32_t dst_stride,
	4742	const int8_t *filter_x,
	4743	const int8_t *filter_y,
	4744	int32_t height,
	4745	int32_t weight0,
	4746	int32_t weight1,
	4747	int32_t offset0,
	4748	int32_t offset1,
	4749	int32_t rnd_val)
	4750	{
	4751	uint32_t loop_cnt;
	4752	int32_t offset, weight;
	4753	v16i8 src0, src1, src2, src3, src4, src5, src6;
	4754	v8i16 in0, in1, in2, in3;
	4755	v8i16 filt0, filt1;
	4756	v4i32 filt_h0, filt_h1;
	4757	v16i8 mask0 = { 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8 };
	4758	v16i8 mask1;
	4759	v8i16 filter_vec, const_vec;
	4760	v16i8 vec0, vec1, vec2, vec3, vec4, vec5;
	4761	v8i16 dst0, dst1, dst2, dst3, dst4, dst5;
	4762	v4i32 dst0_r, dst0_l, dst1_r, dst1_l, dst2_r, dst2_l, dst3_r, dst3_l;
	4763	v8i16 tmp0, tmp1, tmp2, tmp3;
	4764	v8i16 dst10_r, dst32_r, dst21_r, dst43_r;
	4765	v8i16 dst10_l, dst32_l, dst21_l, dst43_l;
	4766	v4i32 weight_vec, offset_vec, rnd_vec;
	4767
	4768	src0_ptr -= (src_stride + 1);
	4769
	4770	filter_vec = LD_SH(filter_x);
	4771	SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
	4772
	4773	filter_vec = LD_SH(filter_y);
	4774	vec0 = __msa_clti_s_b((v16i8) filter_vec, 0);
	4775	filter_vec = (v8i16) __msa_ilvr_b(vec0, (v16i8) filter_vec);
	4776
	4777	SPLATI_W2_SW(filter_vec, 0, filt_h0, filt_h1);
	4778
	4779	mask1 = mask0 + 2;
	4780
	4781	offset = (offset0 + offset1) << rnd_val;
	4782	weight0 = weight0 & 0x0000FFFF;
	4783	weight = weight0 \| (weight1 << 16);
	4784
	4785	const_vec = __msa_ldi_h(128);
	4786	const_vec <<= 6;
	4787	offset_vec = __msa_fill_w(offset);
	4788	weight_vec = __msa_fill_w(weight);
	4789	rnd_vec = __msa_fill_w(rnd_val + 1);
	4790
	4791	LD_SB3(src0_ptr, src_stride, src0, src1, src2);
	4792	src0_ptr += (3 * src_stride);
	4793	XORI_B3_128_SB(src0, src1, src2);
	4794
	4795	VSHF_B2_SB(src0, src0, src0, src0, mask0, mask1, vec0, vec1);
	4796	VSHF_B2_SB(src1, src1, src1, src1, mask0, mask1, vec2, vec3);
	4797	VSHF_B2_SB(src2, src2, src2, src2, mask0, mask1, vec4, vec5);
	4798	dst0 = const_vec;
	4799	DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst0, dst0);
	4800	dst1 = const_vec;
	4801	DPADD_SB2_SH(vec2, vec3, filt0, filt1, dst1, dst1);
	4802	dst2 = const_vec;
	4803	DPADD_SB2_SH(vec4, vec5, filt0, filt1, dst2, dst2);
	4804
	4805	ILVRL_H2_SH(dst1, dst0, dst10_r, dst10_l);
	4806	ILVRL_H2_SH(dst2, dst1, dst21_r, dst21_l);
	4807
	4808	for (loop_cnt = height >> 2; loop_cnt--;) {
	4809	LD_SB4(src0_ptr, src_stride, src3, src4, src5, src6);
	4810	src0_ptr += (4 * src_stride);
	4811	LD_SH4(src1_ptr, src2_stride, in0, in1, in2, in3);
	4812	src1_ptr += (4 * src2_stride);
	4813	XORI_B4_128_SB(src3, src4, src5, src6);
	4814
	4815	VSHF_B2_SB(src3, src3, src3, src3, mask0, mask1, vec0, vec1);
	4816	dst3 = const_vec;
	4817	DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst3, dst3);
	4818
	4819	ILVRL_H2_SH(dst3, dst2, dst32_r, dst32_l);
	4820	dst0_r = HEVC_FILT_4TAP(dst10_r, dst32_r, filt_h0, filt_h1);
	4821	dst0_l = HEVC_FILT_4TAP(dst10_l, dst32_l, filt_h0, filt_h1);
	4822	dst0_r >>= 6;
	4823	dst0_l >>= 6;
	4824
	4825	VSHF_B2_SB(src4, src4, src4, src4, mask0, mask1, vec0, vec1);
	4826	dst4 = const_vec;
	4827	DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst4, dst4);
	4828
	4829	ILVRL_H2_SH(dst4, dst3, dst43_r, dst43_l);
	4830	dst1_r = HEVC_FILT_4TAP(dst21_r, dst43_r, filt_h0, filt_h1);
	4831	dst1_l = HEVC_FILT_4TAP(dst21_l, dst43_l, filt_h0, filt_h1);
	4832	dst1_r >>= 6;
	4833	dst1_l >>= 6;
	4834
	4835	VSHF_B2_SB(src5, src5, src5, src5, mask0, mask1, vec0, vec1);
	4836	dst5 = const_vec;
	4837	DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst5, dst5);
	4838
	4839	ILVRL_H2_SH(dst5, dst4, dst10_r, dst10_l);
	4840	dst2_r = HEVC_FILT_4TAP(dst32_r, dst10_r, filt_h0, filt_h1);
	4841	dst2_l = HEVC_FILT_4TAP(dst32_l, dst10_l, filt_h0, filt_h1);
	4842	dst2_r >>= 6;
	4843	dst2_l >>= 6;
	4844	VSHF_B2_SB(src6, src6, src6, src6, mask0, mask1, vec0, vec1);
	4845	dst2 = const_vec;
	4846	DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst2, dst2);
	4847
	4848	ILVRL_H2_SH(dst2, dst5, dst21_r, dst21_l);
	4849	dst3_r = HEVC_FILT_4TAP(dst43_r, dst21_r, filt_h0, filt_h1);
	4850	dst3_l = HEVC_FILT_4TAP(dst43_l, dst21_l, filt_h0, filt_h1);
	4851	dst3_r >>= 6;
	4852	dst3_l >>= 6;
	4853	PCKEV_H4_SH(dst0_l, dst0_r, dst1_l, dst1_r,
	4854	dst2_l, dst2_r, dst3_l, dst3_r, tmp0, tmp1, tmp2, tmp3);
	4855	HEVC_BIW_RND_CLIP4(tmp0, tmp1, tmp2, tmp3,
	4856	in0, in1, in2, in3,
	4857	weight_vec, rnd_vec, offset_vec,
	4858	dst0_r, dst1_r, dst2_r, dst3_r,
	4859	dst0_l, dst1_l, dst2_l, dst3_l);
	4860
	4861	HEVC_PCK_SW_SB8(dst0_l, dst0_r, dst1_l, dst1_r,
	4862	dst2_l, dst2_r, dst3_l, dst3_r, dst0_r, dst1_r);
	4863	ST6x4_UB(dst0_r, dst1_r, dst, dst_stride);
	4864	dst += (4 * dst_stride);
	4865	}
	4866	}
	4867
	4868	static void hevc_hv_biwgt_4t_8x2_msa(uint8_t *src0_ptr,
	4869	int32_t src_stride,
	4870	int16_t *src1_ptr,
	4871	int32_t src2_stride,
	4872	uint8_t *dst,
	4873	int32_t dst_stride,
	4874	const int8_t *filter_x,
	4875	const int8_t *filter_y,
	4876	int32_t height,
	4877	int32_t weight0,
	4878	int32_t weight1,
	4879	int32_t offset0,
	4880	int32_t offset1,
	4881	int32_t rnd_val)
	4882	{
	4883	int32_t weight, offset;
	4884	v16i8 src0, src1, src2, src3, src4;
	4885	v8i16 filt0, filt1;
	4886	v4i32 filt_h0, filt_h1;
	4887	v16i8 mask0 = { 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8 };
	4888	v16i8 mask1;
	4889	v8i16 filter_vec, const_vec;
	4890	v16i8 vec0, vec1, vec2, vec3, vec4, vec5;
	4891	v8i16 dst0, dst1, dst2, dst3, dst4;
	4892	v8i16 in0, in1;
	4893	v4i32 dst0_r, dst0_l, dst1_r, dst1_l;
	4894	v8i16 dst10_r, dst32_r, dst21_r, dst43_r;
	4895	v8i16 dst10_l, dst32_l, dst21_l, dst43_l;
	4896	v8i16 tmp0, tmp1;
	4897	v4i32 weight_vec, offset_vec, rnd_vec;
	4898
	4899	src0_ptr -= (src_stride + 1);
	4900
	4901	filter_vec = LD_SH(filter_x);
	4902	SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
	4903
	4904	filter_vec = LD_SH(filter_y);
	4905	vec0 = __msa_clti_s_b((v16i8) filter_vec, 0);
	4906	filter_vec = (v8i16) __msa_ilvr_b(vec0, (v16i8) filter_vec);
	4907
	4908	SPLATI_W2_SW(filter_vec, 0, filt_h0, filt_h1);
	4909
	4910	mask1 = mask0 + 2;
	4911
	4912	offset = (offset0 + offset1) << rnd_val;
	4913	weight0 = weight0 & 0x0000FFFF;
	4914	weight = weight0 \| (weight1 << 16);
	4915
	4916	const_vec = __msa_ldi_h(128);
	4917	const_vec <<= 6;
	4918	offset_vec = __msa_fill_w(offset);
	4919	weight_vec = __msa_fill_w(weight);
	4920	rnd_vec = __msa_fill_w(rnd_val + 1);
	4921
	4922	LD_SB3(src0_ptr, src_stride, src0, src1, src2);
	4923	src0_ptr += (3 * src_stride);
	4924	XORI_B3_128_SB(src0, src1, src2);
	4925
	4926	VSHF_B2_SB(src0, src0, src0, src0, mask0, mask1, vec0, vec1);
	4927	VSHF_B2_SB(src1, src1, src1, src1, mask0, mask1, vec2, vec3);
	4928	VSHF_B2_SB(src2, src2, src2, src2, mask0, mask1, vec4, vec5);
	4929	dst0 = const_vec;
	4930	DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst0, dst0);
	4931	dst1 = const_vec;
	4932	DPADD_SB2_SH(vec2, vec3, filt0, filt1, dst1, dst1);
	4933	dst2 = const_vec;
	4934	DPADD_SB2_SH(vec4, vec5, filt0, filt1, dst2, dst2);
	4935
	4936	ILVRL_H2_SH(dst1, dst0, dst10_r, dst10_l);
	4937	ILVRL_H2_SH(dst2, dst1, dst21_r, dst21_l);
	4938
	4939	LD_SB2(src0_ptr, src_stride, src3, src4);
	4940
	4941	LD_SH2(src1_ptr, src2_stride, in0, in1);
	4942	XORI_B2_128_SB(src3, src4);
	4943
	4944	VSHF_B2_SB(src3, src3, src3, src3, mask0, mask1, vec0, vec1);
	4945	dst3 = const_vec;
	4946	DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst3, dst3);
	4947
	4948	ILVRL_H2_SH(dst3, dst2, dst32_r, dst32_l);
	4949	dst0_r = HEVC_FILT_4TAP(dst10_r, dst32_r, filt_h0, filt_h1);
	4950	dst0_l = HEVC_FILT_4TAP(dst10_l, dst32_l, filt_h0, filt_h1);
	4951	dst0_r >>= 6;
	4952	dst0_l >>= 6;
	4953	tmp0 = __msa_pckev_h((v8i16) dst0_l, (v8i16) dst0_r);
	4954
	4955	VSHF_B2_SB(src4, src4, src4, src4, mask0, mask1, vec0, vec1);
	4956	dst4 = const_vec;
	4957	DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst4, dst4);
	4958
	4959	ILVRL_H2_SH(dst4, dst3, dst43_r, dst43_l);
	4960	dst1_r = HEVC_FILT_4TAP(dst21_r, dst43_r, filt_h0, filt_h1);
	4961	dst1_l = HEVC_FILT_4TAP(dst21_l, dst43_l, filt_h0, filt_h1);
	4962	dst1_r >>= 6;
	4963	dst1_l >>= 6;
	4964	tmp1 = __msa_pckev_h((v8i16) dst1_l, (v8i16) dst1_r);
	4965
	4966	HEVC_BIW_RND_CLIP2(tmp0, tmp1, in0, in1,
	4967	weight_vec, rnd_vec, offset_vec,
	4968	dst0_r, dst1_r, dst0_l, dst1_l);
	4969	HEVC_PCK_SW_SB4(dst0_l, dst0_r, dst1_l, dst1_r, dst0_r);
	4970	ST8x2_UB(dst0_r, dst, dst_stride);
	4971	}
	4972
	4973	static void hevc_hv_biwgt_4t_8x6_msa(uint8_t *src0_ptr,
	4974	int32_t src_stride,
	4975	int16_t *src1_ptr,
	4976	int32_t src2_stride,
	4977	uint8_t *dst,
	4978	int32_t dst_stride,
	4979	const int8_t *filter_x,
	4980	const int8_t *filter_y,
	4981	int32_t height,
	4982	int32_t weight0,
	4983	int32_t weight1,
	4984	int32_t offset0,
	4985	int32_t offset1,
	4986	int32_t rnd_val)
	4987	{
	4988	uint32_t offset, weight;
	4989	v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8;
	4990	v8i16 filt0, filt1;
	4991	v4i32 filt_h0, filt_h1;
	4992	v16i8 mask0 = { 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8 };
	4993	v16i8 mask1;
	4994	v8i16 filter_vec, const_vec;
	4995	v16i8 vec0, vec1, vec2, vec3, vec4, vec5;
	4996	v8i16 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7, dst8;
	4997	v4i32 dst0_r, dst0_l, dst1_r, dst1_l, dst2_r, dst2_l, dst3_r, dst3_l;
	4998	v4i32 dst4_r, dst4_l, dst5_r, dst5_l;
	4999	v8i16 dst10_r, dst32_r, dst10_l, dst32_l;

573 more lines aren’t shown here.

libavcodec/mips/hevcdsp_init_mips.c+83−0

@@ -257,6 +257,89 @@ static av_cold void hevc_dsp_init_msa(HEVCDSPContext *c, · 256 unmodified lines
257	c->put_hevc_epel_bi[5][1][1] = ff_hevc_put_hevc_bi_epel_hv16_8_msa;	257	c->put_hevc_epel_bi[5][1][1] = ff_hevc_put_hevc_bi_epel_hv16_8_msa;
258	c->put_hevc_epel_bi[6][1][1] = ff_hevc_put_hevc_bi_epel_hv24_8_msa;	258	c->put_hevc_epel_bi[6][1][1] = ff_hevc_put_hevc_bi_epel_hv24_8_msa;
259	c->put_hevc_epel_bi[7][1][1] = ff_hevc_put_hevc_bi_epel_hv32_8_msa;	259	c->put_hevc_epel_bi[7][1][1] = ff_hevc_put_hevc_bi_epel_hv32_8_msa;
		260
		261	c->put_hevc_qpel_bi_w[1][0][0] =
		262	ff_hevc_put_hevc_bi_w_pel_pixels4_8_msa;
		263	c->put_hevc_qpel_bi_w[3][0][0] =
		264	ff_hevc_put_hevc_bi_w_pel_pixels8_8_msa;
		265	c->put_hevc_qpel_bi_w[4][0][0] =
		266	ff_hevc_put_hevc_bi_w_pel_pixels12_8_msa;
		267	c->put_hevc_qpel_bi_w[5][0][0] =
		268	ff_hevc_put_hevc_bi_w_pel_pixels16_8_msa;
		269	c->put_hevc_qpel_bi_w[6][0][0] =
		270	ff_hevc_put_hevc_bi_w_pel_pixels24_8_msa;
		271	c->put_hevc_qpel_bi_w[7][0][0] =
		272	ff_hevc_put_hevc_bi_w_pel_pixels32_8_msa;
		273	c->put_hevc_qpel_bi_w[8][0][0] =
		274	ff_hevc_put_hevc_bi_w_pel_pixels48_8_msa;
		275	c->put_hevc_qpel_bi_w[9][0][0] =
		276	ff_hevc_put_hevc_bi_w_pel_pixels64_8_msa;
		277
		278	c->put_hevc_qpel_bi_w[1][0][1] = ff_hevc_put_hevc_bi_w_qpel_h4_8_msa;
		279	c->put_hevc_qpel_bi_w[3][0][1] = ff_hevc_put_hevc_bi_w_qpel_h8_8_msa;
		280	c->put_hevc_qpel_bi_w[4][0][1] = ff_hevc_put_hevc_bi_w_qpel_h12_8_msa;
		281	c->put_hevc_qpel_bi_w[5][0][1] = ff_hevc_put_hevc_bi_w_qpel_h16_8_msa;
		282	c->put_hevc_qpel_bi_w[6][0][1] = ff_hevc_put_hevc_bi_w_qpel_h24_8_msa;
		283	c->put_hevc_qpel_bi_w[7][0][1] = ff_hevc_put_hevc_bi_w_qpel_h32_8_msa;
		284	c->put_hevc_qpel_bi_w[8][0][1] = ff_hevc_put_hevc_bi_w_qpel_h48_8_msa;
		285	c->put_hevc_qpel_bi_w[9][0][1] = ff_hevc_put_hevc_bi_w_qpel_h64_8_msa;
		286
		287	c->put_hevc_qpel_bi_w[1][1][0] = ff_hevc_put_hevc_bi_w_qpel_v4_8_msa;
		288	c->put_hevc_qpel_bi_w[3][1][0] = ff_hevc_put_hevc_bi_w_qpel_v8_8_msa;
		289	c->put_hevc_qpel_bi_w[4][1][0] = ff_hevc_put_hevc_bi_w_qpel_v12_8_msa;
		290	c->put_hevc_qpel_bi_w[5][1][0] = ff_hevc_put_hevc_bi_w_qpel_v16_8_msa;
		291	c->put_hevc_qpel_bi_w[6][1][0] = ff_hevc_put_hevc_bi_w_qpel_v24_8_msa;
		292	c->put_hevc_qpel_bi_w[7][1][0] = ff_hevc_put_hevc_bi_w_qpel_v32_8_msa;
		293	c->put_hevc_qpel_bi_w[8][1][0] = ff_hevc_put_hevc_bi_w_qpel_v48_8_msa;
		294	c->put_hevc_qpel_bi_w[9][1][0] = ff_hevc_put_hevc_bi_w_qpel_v64_8_msa;
		295
		296	c->put_hevc_qpel_bi_w[1][1][1] = ff_hevc_put_hevc_bi_w_qpel_hv4_8_msa;
		297	c->put_hevc_qpel_bi_w[3][1][1] = ff_hevc_put_hevc_bi_w_qpel_hv8_8_msa;
		298	c->put_hevc_qpel_bi_w[4][1][1] = ff_hevc_put_hevc_bi_w_qpel_hv12_8_msa;
		299	c->put_hevc_qpel_bi_w[5][1][1] = ff_hevc_put_hevc_bi_w_qpel_hv16_8_msa;
		300	c->put_hevc_qpel_bi_w[6][1][1] = ff_hevc_put_hevc_bi_w_qpel_hv24_8_msa;
		301	c->put_hevc_qpel_bi_w[7][1][1] = ff_hevc_put_hevc_bi_w_qpel_hv32_8_msa;
		302	c->put_hevc_qpel_bi_w[8][1][1] = ff_hevc_put_hevc_bi_w_qpel_hv48_8_msa;
		303	c->put_hevc_qpel_bi_w[9][1][1] = ff_hevc_put_hevc_bi_w_qpel_hv64_8_msa;
		304
		305	c->put_hevc_epel_bi_w[1][0][0] =
		306	ff_hevc_put_hevc_bi_w_pel_pixels4_8_msa;
		307	c->put_hevc_epel_bi_w[2][0][0] =
		308	ff_hevc_put_hevc_bi_w_pel_pixels6_8_msa;
		309	c->put_hevc_epel_bi_w[3][0][0] =
		310	ff_hevc_put_hevc_bi_w_pel_pixels8_8_msa;
		311	c->put_hevc_epel_bi_w[4][0][0] =
		312	ff_hevc_put_hevc_bi_w_pel_pixels12_8_msa;
		313	c->put_hevc_epel_bi_w[5][0][0] =
		314	ff_hevc_put_hevc_bi_w_pel_pixels16_8_msa;
		315	c->put_hevc_epel_bi_w[6][0][0] =
		316	ff_hevc_put_hevc_bi_w_pel_pixels24_8_msa;
		317	c->put_hevc_epel_bi_w[7][0][0] =
		318	ff_hevc_put_hevc_bi_w_pel_pixels32_8_msa;
		319
		320	c->put_hevc_epel_bi_w[1][0][1] = ff_hevc_put_hevc_bi_w_epel_h4_8_msa;
		321	c->put_hevc_epel_bi_w[2][0][1] = ff_hevc_put_hevc_bi_w_epel_h6_8_msa;
		322	c->put_hevc_epel_bi_w[3][0][1] = ff_hevc_put_hevc_bi_w_epel_h8_8_msa;
		323	c->put_hevc_epel_bi_w[4][0][1] = ff_hevc_put_hevc_bi_w_epel_h12_8_msa;
		324	c->put_hevc_epel_bi_w[5][0][1] = ff_hevc_put_hevc_bi_w_epel_h16_8_msa;
		325	c->put_hevc_epel_bi_w[6][0][1] = ff_hevc_put_hevc_bi_w_epel_h24_8_msa;
		326	c->put_hevc_epel_bi_w[7][0][1] = ff_hevc_put_hevc_bi_w_epel_h32_8_msa;
		327
		328	c->put_hevc_epel_bi_w[1][1][0] = ff_hevc_put_hevc_bi_w_epel_v4_8_msa;
		329	c->put_hevc_epel_bi_w[2][1][0] = ff_hevc_put_hevc_bi_w_epel_v6_8_msa;
		330	c->put_hevc_epel_bi_w[3][1][0] = ff_hevc_put_hevc_bi_w_epel_v8_8_msa;
		331	c->put_hevc_epel_bi_w[4][1][0] = ff_hevc_put_hevc_bi_w_epel_v12_8_msa;
		332	c->put_hevc_epel_bi_w[5][1][0] = ff_hevc_put_hevc_bi_w_epel_v16_8_msa;
		333	c->put_hevc_epel_bi_w[6][1][0] = ff_hevc_put_hevc_bi_w_epel_v24_8_msa;
		334	c->put_hevc_epel_bi_w[7][1][0] = ff_hevc_put_hevc_bi_w_epel_v32_8_msa;
		335
		336	c->put_hevc_epel_bi_w[1][1][1] = ff_hevc_put_hevc_bi_w_epel_hv4_8_msa;
		337	c->put_hevc_epel_bi_w[2][1][1] = ff_hevc_put_hevc_bi_w_epel_hv6_8_msa;
		338	c->put_hevc_epel_bi_w[3][1][1] = ff_hevc_put_hevc_bi_w_epel_hv8_8_msa;
		339	c->put_hevc_epel_bi_w[4][1][1] = ff_hevc_put_hevc_bi_w_epel_hv12_8_msa;
		340	c->put_hevc_epel_bi_w[5][1][1] = ff_hevc_put_hevc_bi_w_epel_hv16_8_msa;
		341	c->put_hevc_epel_bi_w[6][1][1] = ff_hevc_put_hevc_bi_w_epel_hv24_8_msa;
		342	c->put_hevc_epel_bi_w[7][1][1] = ff_hevc_put_hevc_bi_w_epel_hv32_8_msa;
260	}	343	}
261	}	344	}
262	#endif // #if HAVE_MSA	345	#endif // #if HAVE_MSA

libavcodec/mips/hevcdsp_mips.h+87−0

@@ -280,3 +280,90 @@ BI_MC(epel, hv, 48); · 279 unmodified lines
280	BI_MC(epel, hv, 64);	280	BI_MC(epel, hv, 64);
281		281
282	#undef BI_MC	282	#undef BI_MC
		283
		284	#define BI_W_MC(PEL, DIR, WIDTH) \
		285	void ff_hevc_put_hevc_bi_w_##PEL##_##DIR####WIDTH##_8_msa(uint8_t *dst, \
		286	ptrdiff_t \
		287	dst_stride, \
		288	uint8_t *src, \
		289	ptrdiff_t \
		290	src_stride, \
		291	int16_t *src_16bit, \
		292	int height, \
		293	int denom, \
		294	int weight0, \
		295	int weight1, \
		296	int offset0, \
		297	int offset1, \
		298	intptr_t mx, \
		299	intptr_t my, \
		300	int width)
		301
		302	BI_W_MC(pel, pixels, 4);
		303	BI_W_MC(pel, pixels, 6);
		304	BI_W_MC(pel, pixels, 8);
		305	BI_W_MC(pel, pixels, 12);
		306	BI_W_MC(pel, pixels, 16);
		307	BI_W_MC(pel, pixels, 24);
		308	BI_W_MC(pel, pixels, 32);
		309	BI_W_MC(pel, pixels, 48);
		310	BI_W_MC(pel, pixels, 64);
		311
		312	BI_W_MC(qpel, h, 4);
		313	BI_W_MC(qpel, h, 8);
		314	BI_W_MC(qpel, h, 12);
		315	BI_W_MC(qpel, h, 16);
		316	BI_W_MC(qpel, h, 24);
		317	BI_W_MC(qpel, h, 32);
		318	BI_W_MC(qpel, h, 48);
		319	BI_W_MC(qpel, h, 64);
		320
		321	BI_W_MC(qpel, v, 4);
		322	BI_W_MC(qpel, v, 8);
		323	BI_W_MC(qpel, v, 12);
		324	BI_W_MC(qpel, v, 16);
		325	BI_W_MC(qpel, v, 24);
		326	BI_W_MC(qpel, v, 32);
		327	BI_W_MC(qpel, v, 48);
		328	BI_W_MC(qpel, v, 64);
		329
		330	BI_W_MC(qpel, hv, 4);
		331	BI_W_MC(qpel, hv, 8);
		332	BI_W_MC(qpel, hv, 12);
		333	BI_W_MC(qpel, hv, 16);
		334	BI_W_MC(qpel, hv, 24);
		335	BI_W_MC(qpel, hv, 32);
		336	BI_W_MC(qpel, hv, 48);
		337	BI_W_MC(qpel, hv, 64);
		338
		339	BI_W_MC(epel, h, 4);
		340	BI_W_MC(epel, h, 6);
		341	BI_W_MC(epel, h, 8);
		342	BI_W_MC(epel, h, 12);
		343	BI_W_MC(epel, h, 16);
		344	BI_W_MC(epel, h, 24);
		345	BI_W_MC(epel, h, 32);
		346	BI_W_MC(epel, h, 48);
		347	BI_W_MC(epel, h, 64);
		348
		349	BI_W_MC(epel, v, 4);
		350	BI_W_MC(epel, v, 6);
		351	BI_W_MC(epel, v, 8);
		352	BI_W_MC(epel, v, 12);
		353	BI_W_MC(epel, v, 16);
		354	BI_W_MC(epel, v, 24);
		355	BI_W_MC(epel, v, 32);
		356	BI_W_MC(epel, v, 48);
		357	BI_W_MC(epel, v, 64);
		358
		359	BI_W_MC(epel, hv, 4);
		360	BI_W_MC(epel, hv, 6);
		361	BI_W_MC(epel, hv, 8);
		362	BI_W_MC(epel, hv, 12);
		363	BI_W_MC(epel, hv, 16);
		364	BI_W_MC(epel, hv, 24);
		365	BI_W_MC(epel, hv, 32);
		366	BI_W_MC(epel, hv, 48);
		367	BI_W_MC(epel, hv, 64);
		368
		369	#undef BI_W_MC

avcodec/mips: MSA (MIPS-SIMD-Arch) optimizations for HEVC biw mc functions

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code