avcodec/mips: MSA (MIPS-SIMD-Arch) optimizations for hpel functions

This patch adds MSA (MIPS-SIMD-Arch) optimizations for hpel functions in new file hpeldsp_msa.c Adds new generic macros (needed for this patch) in libavutil/mips/generic_macros_msa.h

Shivraj Patil committed 11 years ago

Commit ee3ef5f1 parent d277b057 files · +1825 −0Browse files

Showing the changes to libavcodec/mips (4 of 7 files). Show all

Unified · Split

4 files

libavcodec/mips/Makefile +2 −0
libavcodec/mips/hpeldsp_init_mips.c +73 −0
libavcodec/mips/hpeldsp_mips.h +87 −0
libavcodec/mips/hpeldsp_msa.c +1498 −0

libavcodec/mips/Makefile+2−0

 OBJS-$(CONFIG_H264PRED)                   += mips/h264pred_init_mips.o
 OBJS-$(CONFIG_H263DSP)                    += mips/h263dsp_init_mips.o
 OBJS-$(CONFIG_QPELDSP)                    += mips/qpeldsp_init_mips.o
+OBJS-$(CONFIG_HPELDSP)                    += mips/hpeldsp_init_mips.o
 MSA-OBJS-$(CONFIG_HEVC_DECODER)           += mips/hevcdsp_msa.o            \
                                              mips/hevc_mc_uni_msa.o        \
                                              mips/hevc_mc_uniw_msa.o       \
 MSA-OBJS-$(CONFIG_H264PRED)               += mips/h264pred_msa.o
 MSA-OBJS-$(CONFIG_H263DSP)                += mips/h263dsp_msa.o
 MSA-OBJS-$(CONFIG_QPELDSP)                += mips/qpeldsp_msa.o
+MSA-OBJS-$(CONFIG_HPELDSP)                += mips/hpeldsp_msa.o
 LOONGSON3-OBJS-$(CONFIG_H264DSP)          += mips/h264dsp_mmi.o
 LOONGSON3-OBJS-$(CONFIG_H264CHROMA)       += mips/h264chroma_mmi.o

libavcodec/mips/hpeldsp_init_mips.c+73−0

@@ -0,0 +1,73 @@
	1	/*
	2	* Copyright (c) 2015 Parag Salasakar (Parag.Salasakar@imgtec.com)
	3	*
	4	* This file is part of FFmpeg.
	5	*
	6	* FFmpeg is free software; you can redistribute it and/or
	7	* modify it under the terms of the GNU Lesser General Public
	8	* License as published by the Free Software Foundation; either
	9	* version 2.1 of the License, or (at your option) any later version.
	10	*
	11	* FFmpeg is distributed in the hope that it will be useful,
	12	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	13	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	14	* Lesser General Public License for more details.
	15	*
	16	* You should have received a copy of the GNU Lesser General Public
	17	* License along with FFmpeg; if not, write to the Free Software
	18	* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
	19	*/
	20
	21	#include "../hpeldsp.h"
	22	#include "libavcodec/mips/hpeldsp_mips.h"
	23
	24	#if HAVE_MSA
	25	static void ff_hpeldsp_init_msa(HpelDSPContext *c, int flags)
	26	{
	27	c->put_pixels_tab[0][0] = ff_put_pixels16_msa;
	28	c->put_pixels_tab[0][1] = ff_put_pixels16_x2_msa;
	29	c->put_pixels_tab[0][2] = ff_put_pixels16_y2_msa;
	30	c->put_pixels_tab[0][3] = ff_put_pixels16_xy2_msa;
	31
	32	c->put_pixels_tab[1][0] = ff_put_pixels8_msa;
	33	c->put_pixels_tab[1][1] = ff_put_pixels8_x2_msa;
	34	c->put_pixels_tab[1][2] = ff_put_pixels8_y2_msa;
	35	c->put_pixels_tab[1][3] = ff_put_pixels8_xy2_msa;
	36
	37	c->put_pixels_tab[2][1] = ff_put_pixels4_x2_msa;
	38	c->put_pixels_tab[2][2] = ff_put_pixels4_y2_msa;
	39	c->put_pixels_tab[2][3] = ff_put_pixels4_xy2_msa;
	40
	41	c->put_no_rnd_pixels_tab[0][0] = ff_put_pixels16_msa;
	42	c->put_no_rnd_pixels_tab[0][1] = ff_put_no_rnd_pixels16_x2_msa;
	43	c->put_no_rnd_pixels_tab[0][2] = ff_put_no_rnd_pixels16_y2_msa;
	44	c->put_no_rnd_pixels_tab[0][3] = ff_put_no_rnd_pixels16_xy2_msa;
	45
	46	c->put_no_rnd_pixels_tab[1][0] = ff_put_pixels8_msa;
	47	c->put_no_rnd_pixels_tab[1][1] = ff_put_no_rnd_pixels8_x2_msa;
	48	c->put_no_rnd_pixels_tab[1][2] = ff_put_no_rnd_pixels8_y2_msa;
	49	c->put_no_rnd_pixels_tab[1][3] = ff_put_no_rnd_pixels8_xy2_msa;
	50
	51	c->avg_pixels_tab[0][0] = ff_avg_pixels16_msa;
	52	c->avg_pixels_tab[0][1] = ff_avg_pixels16_x2_msa;
	53	c->avg_pixels_tab[0][2] = ff_avg_pixels16_y2_msa;
	54	c->avg_pixels_tab[0][3] = ff_avg_pixels16_xy2_msa;
	55
	56	c->avg_pixels_tab[1][0] = ff_avg_pixels8_msa;
	57	c->avg_pixels_tab[1][1] = ff_avg_pixels8_x2_msa;
	58	c->avg_pixels_tab[1][2] = ff_avg_pixels8_y2_msa;
	59	c->avg_pixels_tab[1][3] = ff_avg_pixels8_xy2_msa;
	60
	61	c->avg_pixels_tab[2][0] = ff_avg_pixels4_msa;
	62	c->avg_pixels_tab[2][1] = ff_avg_pixels4_x2_msa;
	63	c->avg_pixels_tab[2][2] = ff_avg_pixels4_y2_msa;
	64	c->avg_pixels_tab[2][3] = ff_avg_pixels4_xy2_msa;
	65	}
	66	#endif // #if HAVE_MSA
	67
	68	void ff_hpeldsp_init_mips(HpelDSPContext *c, int flags)
	69	{
	70	#if HAVE_MSA
	71	ff_hpeldsp_init_msa(c, flags);
	72	#endif // #if HAVE_MSA
	73	}

libavcodec/mips/hpeldsp_mips.h+87−0

@@ -0,0 +1,87 @@
	1	/*
	2	* Copyright (c) 2015 Parag Salasakar (Parag.Salasakar@imgtec.com)
	3	*
	4	* This file is part of FFmpeg.
	5	*
	6	* FFmpeg is free software; you can redistribute it and/or
	7	* modify it under the terms of the GNU Lesser General Public
	8	* License as published by the Free Software Foundation; either
	9	* version 2.1 of the License, or (at your option) any later version.
	10	*
	11	* FFmpeg is distributed in the hope that it will be useful,
	12	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	13	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	14	* Lesser General Public License for more details.
	15	*
	16	* You should have received a copy of the GNU Lesser General Public
	17	* License along with FFmpeg; if not, write to the Free Software
	18	* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
	19	*/
	20
	21	#ifndef AVCODEC_MIPS_HPELDSP_MIPS_H
	22	#define AVCODEC_MIPS_HPELDSP_MIPS_H
	23
	24	#include "libavcodec/bit_depth_template.c"
	25
	26	void ff_put_pixels16_msa(uint8_t block, const uint8_t pixels,
	27	ptrdiff_t line_size, int32_t h);
	28	void ff_put_pixels16_x2_msa(uint8_t block, const uint8_t pixels,
	29	ptrdiff_t line_size, int32_t h);
	30	void ff_put_pixels16_y2_msa(uint8_t block, const uint8_t pixels,
	31	ptrdiff_t line_size, int32_t h);
	32	void ff_put_pixels16_xy2_msa(uint8_t block, const uint8_t pixels,
	33	ptrdiff_t line_size, int32_t h);
	34	void ff_put_pixels8_msa(uint8_t block, const uint8_t pixels,
	35	ptrdiff_t line_size, int32_t h);
	36	void ff_put_pixels8_x2_msa(uint8_t block, const uint8_t pixels,
	37	ptrdiff_t line_size, int32_t h);
	38	void ff_put_pixels8_y2_msa(uint8_t block, const uint8_t pixels,
	39	ptrdiff_t line_size, int32_t h);
	40	void ff_put_pixels8_xy2_msa(uint8_t block, const uint8_t pixels,
	41	ptrdiff_t line_size, int32_t h);
	42	void ff_put_pixels4_msa(uint8_t block, const uint8_t pixels,
	43	ptrdiff_t line_size, int32_t h);
	44	void ff_put_pixels4_x2_msa(uint8_t block, const uint8_t pixels,
	45	ptrdiff_t line_size, int32_t h);
	46	void ff_put_pixels4_y2_msa(uint8_t block, const uint8_t pixels,
	47	ptrdiff_t line_size, int32_t h);
	48	void ff_put_pixels4_xy2_msa(uint8_t block, const uint8_t pixels,
	49	ptrdiff_t line_size, int32_t h);
	50	void ff_put_no_rnd_pixels16_x2_msa(uint8_t block, const uint8_t pixels,
	51	ptrdiff_t line_size, int32_t h);
	52	void ff_put_no_rnd_pixels16_y2_msa(uint8_t block, const uint8_t pixels,
	53	ptrdiff_t line_size, int32_t h);
	54	void ff_put_no_rnd_pixels16_xy2_msa(uint8_t block, const uint8_t pixels,
	55	ptrdiff_t line_size, int32_t h);
	56	void ff_put_no_rnd_pixels8_x2_msa(uint8_t block, const uint8_t pixels,
	57	ptrdiff_t line_size, int32_t h);
	58	void ff_put_no_rnd_pixels8_y2_msa(uint8_t block, const uint8_t pixels,
	59	ptrdiff_t line_size, int32_t h);
	60	void ff_put_no_rnd_pixels8_xy2_msa(uint8_t block, const uint8_t pixels,
	61	ptrdiff_t line_size, int32_t h);
	62	void ff_avg_pixels16_msa(uint8_t block, const uint8_t pixels,
	63	ptrdiff_t line_size, int32_t h);
	64	void ff_avg_pixels16_x2_msa(uint8_t block, const uint8_t pixels,
	65	ptrdiff_t line_size, int32_t h);
	66	void ff_avg_pixels16_y2_msa(uint8_t block, const uint8_t pixels,
	67	ptrdiff_t line_size, int32_t h);
	68	void ff_avg_pixels16_xy2_msa(uint8_t block, const uint8_t pixels,
	69	ptrdiff_t line_size, int32_t h);
	70	void ff_avg_pixels8_msa(uint8_t block, const uint8_t pixels,
	71	ptrdiff_t line_size, int32_t h);
	72	void ff_avg_pixels8_x2_msa(uint8_t block, const uint8_t pixels,
	73	ptrdiff_t line_size, int32_t h);
	74	void ff_avg_pixels8_y2_msa(uint8_t block, const uint8_t pixels,
	75	ptrdiff_t line_size, int32_t h);
	76	void ff_avg_pixels8_xy2_msa(uint8_t block, const uint8_t pixels,
	77	ptrdiff_t line_size, int32_t h);
	78	void ff_avg_pixels4_msa(uint8_t block, const uint8_t pixels,
	79	ptrdiff_t line_size, int32_t h);
	80	void ff_avg_pixels4_x2_msa(uint8_t block, const uint8_t pixels,
	81	ptrdiff_t line_size, int32_t h);
	82	void ff_avg_pixels4_y2_msa(uint8_t block, const uint8_t pixels,
	83	ptrdiff_t line_size, int32_t h);
	84	void ff_avg_pixels4_xy2_msa(uint8_t block, const uint8_t pixels,
	85	ptrdiff_t line_size, int32_t h);
	86
	87	#endif // #ifndef AVCODEC_MIPS_HPELDSP_MIPS_H

libavcodec/mips/hpeldsp_msa.c+1498−0

@@ -0,0 +1,1498 @@
	1	/*
	2	* Copyright (c) 2015 Parag Salasakar (Parag.Salasakar@imgtec.com)
	3	*
	4	* This file is part of FFmpeg.
	5	*
	6	* FFmpeg is free software; you can redistribute it and/or
	7	* modify it under the terms of the GNU Lesser General Public
	8	* License as published by the Free Software Foundation; either
	9	* version 2.1 of the License, or (at your option) any later version.
	10	*
	11	* FFmpeg is distributed in the hope that it will be useful,
	12	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	13	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	14	* Lesser General Public License for more details.
	15	*
	16	* You should have received a copy of the GNU Lesser General Public
	17	* License along with FFmpeg; if not, write to the Free Software
	18	* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
	19	*/
	20
	21	#include "libavutil/mips/generic_macros_msa.h"
	22	#include "libavcodec/mips/hpeldsp_mips.h"
	23
	24	#define PCKEV_AVG_ST_UB(in0, in1, dst, pdst) \
	25	{ \
	26	v16u8 tmp_m; \
	27	\
	28	tmp_m = (v16u8) __msa_pckev_b((v16i8) in0, (v16i8) in1); \
	29	tmp_m = __msa_aver_u_b(tmp_m, (v16u8) dst); \
	30	ST_UB(tmp_m, (pdst)); \
	31	}
	32
	33	#define PCKEV_ST_SB4(in0, in1, in2, in3, in4, in5, in6, in7, pdst, stride) \
	34	{ \
	35	v16i8 tmp0_m, tmp1_m, tmp2_m, tmp3_m; \
	36	uint8_t pdst_m = (uint8_t ) (pdst); \
	37	\
	38	PCKEV_B4_SB(in0, in1, in2, in3, in4, in5, in6, in7, \
	39	tmp0_m, tmp1_m, tmp2_m, tmp3_m); \
	40	ST_SB4(tmp0_m, tmp1_m, tmp2_m, tmp3_m, pdst_m, stride); \
	41	}
	42
	43	#define PCKEV_AVG_ST8x4_UB(in1, dst0, in2, dst1, in3, dst2, in4, dst3, \
	44	pdst, stride) \
	45	{ \
	46	v16u8 tmp0_m, tmp1_m, tmp2_m, tmp3_m; \
	47	uint8_t pdst_m = (uint8_t ) (pdst); \
	48	\
	49	PCKEV_B2_UB(in2, in1, in4, in3, tmp0_m, tmp1_m); \
	50	PCKEV_D2_UB(dst1, dst0, dst3, dst2, tmp2_m, tmp3_m); \
	51	AVER_UB2_UB(tmp0_m, tmp2_m, tmp1_m, tmp3_m, tmp0_m, tmp1_m); \
	52	ST8x4_UB(tmp0_m, tmp1_m, pdst_m, stride); \
	53	}
	54
	55	static void common_hz_bil_4w_msa(const uint8_t *src, int32_t src_stride,
	56	uint8_t *dst, int32_t dst_stride,
	57	uint8_t height)
	58	{
	59	uint8_t loop_cnt;
	60	uint32_t out0, out1;
	61	v16u8 src0, src1, src0_sld1, src1_sld1, res0, res1;
	62
	63	for (loop_cnt = (height >> 1); loop_cnt--;) {
	64	LD_UB2(src, src_stride, src0, src1);
	65	src += (2 * src_stride);
	66
	67	SLDI_B2_0_UB(src0, src1, src0_sld1, src1_sld1, 1);
	68	AVER_UB2_UB(src0_sld1, src0, src1_sld1, src1, res0, res1);
	69
	70	out0 = __msa_copy_u_w((v4i32) res0, 0);
	71	out1 = __msa_copy_u_w((v4i32) res1, 0);
	72	SW(out0, dst);
	73	dst += dst_stride;
	74	SW(out1, dst);
	75	dst += dst_stride;
	76	}
	77	}
	78
	79	static void common_hz_bil_8w_msa(const uint8_t *src, int32_t src_stride,
	80	uint8_t *dst, int32_t dst_stride,
	81	uint8_t height)
	82	{
	83	uint8_t loop_cnt;
	84	v16i8 src0, src1, src2, src3, src0_sld1, src1_sld1, src2_sld1, src3_sld1;
	85
	86	for (loop_cnt = (height >> 2); loop_cnt--;) {
	87	LD_SB4(src, src_stride, src0, src1, src2, src3);
	88	src += (4 * src_stride);
	89
	90	SLDI_B4_0_SB(src0, src1, src2, src3,
	91	src0_sld1, src1_sld1, src2_sld1, src3_sld1, 1);
	92	AVER_ST8x4_UB(src0, src0_sld1, src1, src1_sld1,
	93	src2, src2_sld1, src3, src3_sld1, dst, dst_stride);
	94	dst += (4 * dst_stride);
	95	}
	96	}
	97
	98	static void common_hz_bil_16w_msa(const uint8_t *src, int32_t src_stride,
	99	uint8_t *dst, int32_t dst_stride,
	100	uint8_t height)
	101	{
	102	uint8_t loop_cnt;
	103	v16u8 src0, src1, src2, src3, src4, src5, src6, src7;
	104	v16u8 src8, src9, src10, src11, src12, src13, src14, src15;
	105
	106	for (loop_cnt = (height >> 3); loop_cnt--;) {
	107	LD_UB8(src, src_stride, src0, src1, src2, src3, src4, src5, src6, src7);
	108	LD_UB8((src + 1), src_stride,
	109	src8, src9, src10, src11, src12, src13, src14, src15);
	110	src += (8 * src_stride);
	111
	112	AVER_ST16x4_UB(src0, src8, src1, src9, src2, src10, src3, src11,
	113	dst, dst_stride);
	114	dst += (4 * dst_stride);
	115
	116	AVER_ST16x4_UB(src4, src12, src5, src13, src6, src14, src7, src15,
	117	dst, dst_stride);
	118	dst += (4 * dst_stride);
	119	}
	120	}
	121
	122	static void common_hz_bil_no_rnd_8x8_msa(const uint8_t *src, int32_t src_stride,
	123	uint8_t *dst, int32_t dst_stride)
	124	{
	125	v16i8 src0, src1, src2, src3, src4, src5, src6, src7;
	126	v16i8 src0_sld1, src1_sld1, src2_sld1, src3_sld1;
	127	v16i8 src4_sld1, src5_sld1, src6_sld1, src7_sld1;
	128
	129	LD_SB8(src, src_stride, src0, src1, src2, src3, src4, src5, src6, src7);
	130	src += (8 * src_stride);
	131
	132	SLDI_B4_0_SB(src0, src1, src2, src3,
	133	src0_sld1, src1_sld1, src2_sld1, src3_sld1, 1);
	134	SLDI_B4_0_SB(src4, src5, src6, src7,
	135	src4_sld1, src5_sld1, src6_sld1, src7_sld1, 1);
	136
	137	AVE_ST8x4_UB(src0, src0_sld1, src1, src1_sld1,
	138	src2, src2_sld1, src3, src3_sld1, dst, dst_stride);
	139	dst += (4 * dst_stride);
	140	AVE_ST8x4_UB(src4, src4_sld1, src5, src5_sld1,
	141	src6, src6_sld1, src7, src7_sld1, dst, dst_stride);
	142	}
	143
	144	static void common_hz_bil_no_rnd_4x8_msa(const uint8_t *src, int32_t src_stride,
	145	uint8_t *dst, int32_t dst_stride)
	146	{
	147	v16i8 src0, src1, src2, src3, src0_sld1, src1_sld1, src2_sld1, src3_sld1;
	148
	149	LD_SB4(src, src_stride, src0, src1, src2, src3);
	150	SLDI_B4_0_SB(src0, src1, src2, src3,
	151	src0_sld1, src1_sld1, src2_sld1, src3_sld1, 1);
	152	AVE_ST8x4_UB(src0, src0_sld1, src1, src1_sld1,
	153	src2, src2_sld1, src3, src3_sld1, dst, dst_stride);
	154	}
	155
	156	static void common_hz_bil_no_rnd_16x16_msa(const uint8_t *src,
	157	int32_t src_stride,
	158	uint8_t *dst, int32_t dst_stride)
	159	{
	160	v16u8 src0, src1, src2, src3, src4, src5, src6, src7, src8;
	161	v16u8 src9, src10, src11, src12, src13, src14, src15;
	162
	163	LD_UB8(src, src_stride, src0, src1, src2, src3, src4, src5, src6, src7);
	164	LD_UB8((src + 1), src_stride,
	165	src8, src9, src10, src11, src12, src13, src14, src15);
	166	src += (8 * src_stride);
	167
	168	AVE_ST16x4_UB(src0, src8, src1, src9, src2, src10, src3, src11,
	169	dst, dst_stride);
	170	dst += (4 * dst_stride);
	171
	172	LD_UB4(src, src_stride, src0, src1, src2, src3);
	173	LD_UB4((src + 1), src_stride, src8, src9, src10, src11);
	174	src += (4 * src_stride);
	175
	176	AVE_ST16x4_UB(src4, src12, src5, src13, src6, src14, src7, src15,
	177	dst, dst_stride);
	178	dst += (4 * dst_stride);
	179
	180	LD_UB4(src, src_stride, src4, src5, src6, src7);
	181	LD_UB4((src + 1), src_stride, src12, src13, src14, src15);
	182	src += (4 * src_stride);
	183
	184	AVE_ST16x4_UB(src0, src8, src1, src9, src2, src10, src3, src11,
	185	dst, dst_stride);
	186	dst += (4 * dst_stride);
	187	AVE_ST16x4_UB(src4, src12, src5, src13, src6, src14, src7, src15,
	188	dst, dst_stride);
	189	}
	190
	191	static void common_hz_bil_no_rnd_8x16_msa(const uint8_t *src,
	192	int32_t src_stride,
	193	uint8_t *dst, int32_t dst_stride)
	194	{
	195	v16u8 src0, src1, src2, src3, src4, src5, src6, src7, src8;
	196	v16u8 src9, src10, src11, src12, src13, src14, src15;
	197
	198	LD_UB8(src, src_stride, src0, src1, src2, src3, src4, src5, src6, src7);
	199	LD_UB8((src + 1), src_stride,
	200	src8, src9, src10, src11, src12, src13, src14, src15);
	201
	202	AVE_ST16x4_UB(src0, src8, src1, src9, src2, src10, src3, src11,
	203	dst, dst_stride);
	204	dst += (4 * dst_stride);
	205	AVE_ST16x4_UB(src4, src12, src5, src13, src6, src14, src7, src15,
	206	dst, dst_stride);
	207	}
	208
	209	static void common_hz_bil_and_aver_dst_4w_msa(const uint8_t *src,
	210	int32_t src_stride,
	211	uint8_t *dst, int32_t dst_stride,
	212	uint8_t height)
	213	{
	214	uint8_t loop_cnt;
	215	uint32_t dst0, dst1, out0, out1;
	216	v16u8 src0, src1, src0_sld1, src1_sld1, res0, res1;
	217	v16u8 tmp0 = { 0 };
	218	v16u8 tmp1 = { 0 };
	219
	220	for (loop_cnt = (height >> 1); loop_cnt--;) {
	221	LD_UB2(src, src_stride, src0, src1);
	222	src += (2 * src_stride);
	223
	224	SLDI_B2_0_UB(src0, src1, src0_sld1, src1_sld1, 1);
	225
	226	dst0 = LW(dst);
	227	dst1 = LW(dst + dst_stride);
	228	tmp0 = (v16u8) __msa_insert_w((v4i32) tmp0, 0, dst0);
	229	tmp1 = (v16u8) __msa_insert_w((v4i32) tmp1, 0, dst1);
	230
	231	AVER_UB2_UB(src0_sld1, src0, src1_sld1, src1, res0, res1);
	232	AVER_UB2_UB(res0, tmp0, res1, tmp1, res0, res1);
	233
	234	out0 = __msa_copy_u_w((v4i32) res0, 0);
	235	out1 = __msa_copy_u_w((v4i32) res1, 0);
	236	SW(out0, dst);
	237	dst += dst_stride;
	238	SW(out1, dst);
	239	dst += dst_stride;
	240	}
	241	}
	242
	243	static void common_hz_bil_and_aver_dst_8w_msa(const uint8_t *src,
	244	int32_t src_stride,
	245	uint8_t *dst, int32_t dst_stride,
	246	uint8_t height)
	247	{
	248	uint8_t loop_cnt;
	249	v16i8 src0, src1, src2, src3, src0_sld1, src1_sld1, src2_sld1, src3_sld1;
	250
	251	for (loop_cnt = (height >> 2); loop_cnt--;) {
	252	LD_SB4(src, src_stride, src0, src1, src2, src3);
	253	src += (4 * src_stride);
	254
	255	SLDI_B4_0_SB(src0, src1, src2, src3,
	256	src0_sld1, src1_sld1, src2_sld1, src3_sld1, 1);
	257
	258	AVER_DST_ST8x4_UB(src0, src0_sld1, src1, src1_sld1, src2, src2_sld1,
	259	src3, src3_sld1, dst, dst_stride);
	260	dst += (4 * dst_stride);
	261	}
	262	}
	263
	264	static void common_hz_bil_and_aver_dst_16w_msa(const uint8_t *src,
	265	int32_t src_stride,
	266	uint8_t *dst, int32_t dst_stride,
	267	uint8_t height)
	268	{
	269	uint8_t loop_cnt;
	270	v16u8 src0, src1, src2, src3, src4, src5, src6, src7, src8;
	271	v16u8 src9, src10, src11, src12, src13, src14, src15;
	272
	273	for (loop_cnt = (height >> 3); loop_cnt--;) {
	274	LD_UB8(src, src_stride, src0, src1, src2, src3, src4, src5, src6, src7);
	275	LD_UB8((src + 1), src_stride,
	276	src8, src9, src10, src11, src12, src13, src14, src15);
	277	src += (8 * src_stride);
	278
	279	AVER_DST_ST16x4_UB(src0, src8, src1, src9, src2, src10, src3, src11,
	280	dst, dst_stride);
	281	dst += (4 * dst_stride);
	282	AVER_DST_ST16x4_UB(src4, src12, src5, src13, src6, src14, src7, src15,
	283	dst, dst_stride);
	284	dst += (4 * dst_stride);
	285	}
	286	}
	287
	288	static void common_vt_bil_4w_msa(const uint8_t *src, int32_t src_stride,
	289	uint8_t *dst, int32_t dst_stride,
	290	uint8_t height)
	291	{
	292	uint8_t loop_cnt;
	293	uint32_t out0, out1;
	294	v16u8 src0, src1, src2, res0, res1;
	295
	296	src0 = LD_UB(src);
	297	src += src_stride;
	298
	299	for (loop_cnt = (height >> 1); loop_cnt--;) {
	300	LD_UB2(src, src_stride, src1, src2);
	301	src += (2 * src_stride);
	302
	303	AVER_UB2_UB(src0, src1, src1, src2, res0, res1);
	304
	305	out0 = __msa_copy_u_w((v4i32) res0, 0);
	306	out1 = __msa_copy_u_w((v4i32) res1, 0);
	307	SW(out0, dst);
	308	dst += dst_stride;
	309	SW(out1, dst);
	310	dst += dst_stride;
	311
	312	src0 = src2;
	313	}
	314	}
	315
	316	static void common_vt_bil_8w_msa(const uint8_t *src, int32_t src_stride,
	317	uint8_t *dst, int32_t dst_stride,
	318	uint8_t height)
	319	{
	320	uint8_t loop_cnt;
	321	v16u8 src0, src1, src2, src3, src4;
	322
	323	src0 = LD_UB(src);
	324	src += src_stride;
	325
	326	for (loop_cnt = (height >> 2); loop_cnt--;) {
	327	LD_UB4(src, src_stride, src1, src2, src3, src4);
	328	src += (4 * src_stride);
	329
	330	AVER_ST8x4_UB(src0, src1, src1, src2, src2, src3, src3, src4,
	331	dst, dst_stride);
	332	dst += (4 * dst_stride);
	333
	334	src0 = src4;
	335	}
	336	}
	337
	338	static void common_vt_bil_16w_msa(const uint8_t *src, int32_t src_stride,
	339	uint8_t *dst, int32_t dst_stride,
	340	uint8_t height)
	341	{
	342	uint8_t loop_cnt;
	343	v16u8 src0, src1, src2, src3, src4, src5, src6, src7, src8;
	344
	345	src0 = LD_UB(src);
	346	src += src_stride;
	347
	348	for (loop_cnt = (height >> 3); loop_cnt--;) {
	349	LD_UB8(src, src_stride, src1, src2, src3, src4, src5, src6, src7, src8);
	350	src += (8 * src_stride);
	351
	352	AVER_ST16x4_UB(src0, src1, src1, src2, src2, src3, src3, src4,
	353	dst, dst_stride);
	354	dst += (4 * dst_stride);
	355	AVER_ST16x4_UB(src4, src5, src5, src6, src6, src7, src7, src8,
	356	dst, dst_stride);
	357	dst += (4 * dst_stride);
	358
	359	src0 = src8;
	360	}
	361	}
	362
	363	static void common_vt_bil_no_rnd_8x8_msa(const uint8_t *src, int32_t src_stride,
	364	uint8_t *dst, int32_t dst_stride)
	365	{
	366	v16u8 src0, src1, src2, src3, src4, src5, src6, src7, src8;
	367
	368	LD_UB8(src, src_stride, src0, src1, src2, src3, src4, src5, src6, src7);
	369	src += (8 * src_stride);
	370	src8 = LD_UB(src);
	371
	372	AVE_ST8x4_UB(src0, src1, src1, src2, src2, src3, src3, src4,
	373	dst, dst_stride);
	374	dst += (4 * dst_stride);
	375
	376	AVE_ST8x4_UB(src4, src5, src5, src6, src6, src7, src7, src8,
	377	dst, dst_stride);
	378	}
	379
	380	static void common_vt_bil_no_rnd_4x8_msa(const uint8_t *src, int32_t src_stride,
	381	uint8_t *dst, int32_t dst_stride)
	382	{
	383	v16u8 src0, src1, src2, src3, src4;
	384
	385	LD_UB5(src, src_stride, src0, src1, src2, src3, src4);
	386	AVE_ST8x4_UB(src0, src1, src1, src2, src2, src3, src3, src4,
	387	dst, dst_stride);
	388	}
	389
	390	static void common_vt_bil_no_rnd_16x16_msa(const uint8_t *src,
	391	int32_t src_stride,
	392	uint8_t *dst, int32_t dst_stride)
	393	{
	394	v16u8 src0, src1, src2, src3, src4, src5, src6, src7, src8;
	395	v16u8 src9, src10, src11, src12, src13, src14, src15, src16;
	396
	397	LD_UB8(src, src_stride, src0, src1, src2, src3, src4, src5, src6, src7);
	398	src += (8 * src_stride);
	399	LD_UB8(src, src_stride,
	400	src8, src9, src10, src11, src12, src13, src14, src15);
	401	src += (8 * src_stride);
	402	src16 = LD_UB(src);
	403
	404	AVE_ST16x4_UB(src0, src1, src1, src2, src2, src3, src3, src4,
	405	dst, dst_stride);
	406	dst += (4 * dst_stride);
	407	AVE_ST16x4_UB(src4, src5, src5, src6, src6, src7, src7, src8,
	408	dst, dst_stride);
	409	dst += (4 * dst_stride);
	410	AVE_ST16x4_UB(src8, src9, src9, src10, src10, src11, src11, src12,
	411	dst, dst_stride);
	412	dst += (4 * dst_stride);
	413	AVE_ST16x4_UB(src12, src13, src13, src14,
	414	src14, src15, src15, src16, dst, dst_stride);
	415	}
	416
	417	static void common_vt_bil_no_rnd_8x16_msa(const uint8_t *src,
	418	int32_t src_stride,
	419	uint8_t *dst, int32_t dst_stride)
	420	{
	421	v16u8 src0, src1, src2, src3, src4, src5, src6, src7, src8;
	422
	423	LD_UB8(src, src_stride, src0, src1, src2, src3, src4, src5, src6, src7);
	424	src += (8 * src_stride);
	425	src8 = LD_UB(src);
	426
	427	AVE_ST16x4_UB(src0, src1, src1, src2, src2, src3, src3, src4,
	428	dst, dst_stride);
	429	dst += (4 * dst_stride);
	430	AVE_ST16x4_UB(src4, src5, src5, src6, src6, src7, src7, src8,
	431	dst, dst_stride);
	432	}
	433
	434	static void common_vt_bil_and_aver_dst_4w_msa(const uint8_t *src,
	435	int32_t src_stride,
	436	uint8_t *dst, int32_t dst_stride,
	437	uint8_t height)
	438	{
	439	uint8_t loop_cnt;
	440	uint32_t out0, out1, dst0, dst1;
	441	v16u8 src0, src1, src2;
	442	v16u8 tmp0 = { 0 };
	443	v16u8 tmp1 = { 0 };
	444	v16u8 res0, res1;
	445
	446	src0 = LD_UB(src);
	447	src += src_stride;
	448
	449	for (loop_cnt = (height >> 1); loop_cnt--;) {
	450	LD_UB2(src, src_stride, src1, src2);
	451	src += (2 * src_stride);
	452	dst0 = LW(dst);
	453	dst1 = LW(dst + dst_stride);
	454	tmp0 = (v16u8) __msa_insert_w((v4i32) tmp0, 0, dst0);
	455	tmp1 = (v16u8) __msa_insert_w((v4i32) tmp1, 0, dst1);
	456	AVER_UB2_UB(src0, src1, src1, src2, res0, res1);
	457	AVER_UB2_UB(res0, tmp0, res1, tmp1, res0, res1);
	458	out0 = __msa_copy_u_w((v4i32) res0, 0);
	459	out1 = __msa_copy_u_w((v4i32) res1, 0);
	460	SW(out0, dst);
	461	dst += dst_stride;
	462	SW(out1, dst);
	463	dst += dst_stride;
	464	src0 = src2;
	465	}
	466	}
	467
	468	static void common_vt_bil_and_aver_dst_8w_msa(const uint8_t *src,
	469	int32_t src_stride,
	470	uint8_t *dst, int32_t dst_stride,
	471	uint8_t height)
	472	{
	473	uint8_t loop_cnt;
	474	v16u8 src0, src1, src2, src3, src4;
	475
	476	src0 = LD_UB(src);
	477	src += src_stride;
	478
	479	for (loop_cnt = (height >> 2); loop_cnt--;) {
	480	LD_UB4(src, src_stride, src1, src2, src3, src4);
	481	src += (4 * src_stride);
	482
	483	AVER_DST_ST8x4_UB(src0, src1, src1, src2, src2, src3, src3, src4,
	484	dst, dst_stride);
	485	dst += (4 * dst_stride);
	486	src0 = src4;
	487	}
	488	}
	489
	490	static void common_vt_bil_and_aver_dst_16w_msa(const uint8_t *src,
	491	int32_t src_stride,
	492	uint8_t *dst, int32_t dst_stride,
	493	uint8_t height)
	494	{
	495	uint8_t loop_cnt;
	496	v16u8 src0, src1, src2, src3, src4, src5, src6, src7, src8;
	497	v16u8 res0, res1, res2, res3, res4, res5, res6, res7;
	498	v16u8 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7;
	499
	500	src0 = LD_UB(src);
	501	src += src_stride;
	502
	503	for (loop_cnt = (height >> 3); loop_cnt--;) {
	504	LD_UB8(src, src_stride, src1, src2, src3, src4, src5, src6, src7, src8);
	505	src += (8 * src_stride);
	506	AVER_UB4_UB(src0, src1, src1, src2, src2, src3, src3, src4,
	507	res0, res1, res2, res3);
	508	AVER_UB4_UB(src4, src5, src5, src6, src6, src7, src7, src8,
	509	res4, res5, res6, res7);
	510
	511	LD_UB8(dst, dst_stride, dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7);
	512	AVER_UB4_UB(dst0, res0, dst1, res1, dst2, res2, dst3, res3,
	513	res0, res1, res2, res3);
	514	AVER_UB4_UB(dst4, res4, dst5, res5, dst6, res6, dst7, res7,
	515	res4, res5, res6, res7);
	516	ST_UB8(res0, res1, res2, res3, res4, res5, res6, res7, dst, dst_stride);
	517	dst += (8 * dst_stride);
	518
	519	src0 = src8;
	520	}
	521	}
	522
	523	static void common_hv_bil_4w_msa(const uint8_t *src, int32_t src_stride,
	524	uint8_t *dst, int32_t dst_stride,
	525	uint8_t height)
	526	{
	527	uint8_t loop_cnt;
	528	uint32_t res0, res1;
	529	v16i8 src0, src1, src2, src0_sld1, src1_sld1, src2_sld1;
	530	v16u8 src0_r, src1_r, src2_r, res;
	531	v8u16 add0, add1, add2, sum0, sum1;
	532
	533	src0 = LD_SB(src);
	534	src += src_stride;
	535
	536	for (loop_cnt = (height >> 1); loop_cnt--;) {
	537	LD_SB2(src, src_stride, src1, src2);
	538	src += (2 * src_stride);
	539
	540	SLDI_B3_0_SB(src0, src1, src2, src0_sld1, src1_sld1, src2_sld1, 1);
	541	ILVR_B3_UB(src0_sld1, src0, src1_sld1, src1, src2_sld1, src2,
	542	src0_r, src1_r, src2_r);
	543	HADD_UB3_UH(src0_r, src1_r, src2_r, add0, add1, add2);
	544	ADD2(add0, add1, add1, add2, sum0, sum1);
	545	SRARI_H2_UH(sum0, sum1, 2);
	546	res = (v16u8) __msa_pckev_b((v16i8) sum1, (v16i8) sum0);
	547	res0 = __msa_copy_u_w((v4i32) res, 0);
	548	res1 = __msa_copy_u_w((v4i32) res, 2);
	549	SW(res0, dst);
	550	dst += dst_stride;
	551	SW(res1, dst);
	552	dst += dst_stride;
	553
	554	src0 = src2;
	555	}
	556	}
	557
	558	static void common_hv_bil_8w_msa(const uint8_t *src, int32_t src_stride,
	559	uint8_t *dst, int32_t dst_stride,
	560	uint8_t height)
	561	{
	562	uint8_t loop_cnt;
	563	v16i8 src0, src1, src2, src3, src4;
	564	v16i8 src0_sld1, src1_sld1, src2_sld1, src3_sld1, src4_sld1;
	565	v16u8 src0_r, src1_r, src2_r, src3_r, src4_r;
	566	v8u16 add0, add1, add2, add3, add4;
	567	v8u16 sum0, sum1, sum2, sum3;
	568
	569	src0 = LD_SB(src);
	570	src += src_stride;
	571
	572	for (loop_cnt = (height >> 2); loop_cnt--;) {
	573	LD_SB4(src, src_stride, src1, src2, src3, src4);
	574	src += (4 * src_stride);
	575
	576	SLDI_B3_0_SB(src0, src1, src2, src0_sld1, src1_sld1, src2_sld1, 1);
	577	SLDI_B2_0_SB(src3, src4, src3_sld1, src4_sld1, 1);
	578	ILVR_B3_UB(src0_sld1, src0, src1_sld1, src1, src2_sld1, src2, src0_r,
	579	src1_r, src2_r);
	580	ILVR_B2_UB(src3_sld1, src3, src4_sld1, src4, src3_r, src4_r);
	581	HADD_UB3_UH(src0_r, src1_r, src2_r, add0, add1, add2);
	582	HADD_UB2_UH(src3_r, src4_r, add3, add4);
	583	ADD4(add0, add1, add1, add2, add2, add3, add3, add4,
	584	sum0, sum1, sum2, sum3);
	585	SRARI_H4_UH(sum0, sum1, sum2, sum3, 2);
	586	PCKEV_B2_SB(sum1, sum0, sum3, sum2, src0, src1);
	587	ST8x4_UB(src0, src1, dst, dst_stride);
	588	dst += (4 * dst_stride);
	589	src0 = src4;
	590	}
	591	}
	592
	593	static void common_hv_bil_16w_msa(const uint8_t *src, int32_t src_stride,
	594	uint8_t *dst, int32_t dst_stride,
	595	uint8_t height)
	596	{
	597	uint8_t loop_cnt;
	598	v16u8 src0, src1, src2, src3, src4, src5, src6, src7, src8, src9;
	599	v16u8 src10, src11, src12, src13, src14, src15, src16, src17;
	600	v8u16 src0_r, src1_r, src2_r, src3_r, src4_r, src5_r, src6_r, src7_r;
	601	v8u16 src8_r, src0_l, src1_l, src2_l, src3_l, src4_l, src5_l, src6_l;
	602	v8u16 src7_l, src8_l;
	603	v8u16 sum0_r, sum1_r, sum2_r, sum3_r, sum4_r, sum5_r, sum6_r, sum7_r;
	604	v8u16 sum0_l, sum1_l, sum2_l, sum3_l, sum4_l, sum5_l, sum6_l, sum7_l;
	605
	606	for (loop_cnt = (height >> 3); loop_cnt--;) {
	607	LD_UB8(src, src_stride, src0, src1, src2, src3, src4, src5, src6, src7);
	608	LD_UB8((src + 1), src_stride,
	609	src9, src10, src11, src12, src13, src14, src15, src16);
	610	src += (8 * src_stride);
	611
	612	src8 = LD_UB(src);
	613	src17 = LD_UB(src + 1);
	614
	615	ILVRL_B2_UH(src9, src0, src0_r, src0_l);
	616	ILVRL_B2_UH(src10, src1, src1_r, src1_l);
	617	ILVRL_B2_UH(src11, src2, src2_r, src2_l);
	618	ILVRL_B2_UH(src12, src3, src3_r, src3_l);
	619	ILVRL_B2_UH(src13, src4, src4_r, src4_l);
	620	ILVRL_B2_UH(src14, src5, src5_r, src5_l);
	621	ILVRL_B2_UH(src15, src6, src6_r, src6_l);
	622	ILVRL_B2_UH(src16, src7, src7_r, src7_l);
	623	ILVRL_B2_UH(src17, src8, src8_r, src8_l);
	624	HADD_UB3_UH(src0_r, src1_r, src2_r, src0_r, src1_r, src2_r);
	625	HADD_UB3_UH(src3_r, src4_r, src5_r, src3_r, src4_r, src5_r);
	626	HADD_UB3_UH(src6_r, src7_r, src8_r, src6_r, src7_r, src8_r);
	627	HADD_UB3_UH(src0_l, src1_l, src2_l, src0_l, src1_l, src2_l);
	628	HADD_UB3_UH(src3_l, src4_l, src5_l, src3_l, src4_l, src5_l);
	629	HADD_UB3_UH(src6_l, src7_l, src8_l, src6_l, src7_l, src8_l);
	630	ADD4(src0_r, src1_r, src1_r, src2_r, src2_r, src3_r, src3_r, src4_r,
	631	sum0_r, sum1_r, sum2_r, sum3_r);
	632	ADD4(src4_r, src5_r, src5_r, src6_r, src6_r, src7_r, src7_r, src8_r,
	633	sum4_r, sum5_r, sum6_r, sum7_r);
	634	ADD4(src0_l, src1_l, src1_l, src2_l, src2_l, src3_l, src3_l, src4_l,
	635	sum0_l, sum1_l, sum2_l, sum3_l);
	636	ADD4(src4_l, src5_l, src5_l, src6_l, src6_l, src7_l, src7_l, src8_l,
	637	sum4_l, sum5_l, sum6_l, sum7_l);
	638	SRARI_H4_UH(sum0_r, sum1_r, sum2_r, sum3_r, 2);
	639	SRARI_H4_UH(sum4_r, sum5_r, sum6_r, sum7_r, 2);
	640	SRARI_H4_UH(sum0_l, sum1_l, sum2_l, sum3_l, 2);
	641	SRARI_H4_UH(sum4_l, sum5_l, sum6_l, sum7_l, 2);
	642	PCKEV_ST_SB4(sum0_l, sum0_r, sum1_l, sum1_r, sum2_l, sum2_r,
	643	sum3_l, sum3_r, dst, dst_stride);
	644	dst += (4 * dst_stride);
	645	PCKEV_ST_SB4(sum4_l, sum4_r, sum5_l, sum5_r, sum6_l, sum6_r,
	646	sum7_l, sum7_r, dst, dst_stride);
	647	dst += (4 * dst_stride);
	648	}
	649	}
	650
	651	static void common_hv_bil_no_rnd_8x8_msa(const uint8_t *src, int32_t src_stride,
	652	uint8_t *dst, int32_t dst_stride)
	653	{
	654	v16u8 src0, src1, src2, src3, src4, src5, src6, src7, src8;
	655	v16u8 src0_sld1, src1_sld1, src2_sld1, src3_sld1;
	656	v16u8 src4_sld1, src5_sld1, src6_sld1, src7_sld1, src8_sld1;
	657	v8u16 src0_r, src1_r, src2_r, src3_r;
	658	v8u16 src4_r, src5_r, src6_r, src7_r, src8_r;
	659	v8u16 add0, add1, add2, add3, add4, add5, add6, add7, add8;
	660	v8u16 sum0, sum1, sum2, sum3, sum4, sum5, sum6, sum7;
	661	v16i8 out0, out1;
	662
	663	LD_UB8(src, src_stride, src0, src1, src2, src3, src4, src5, src6, src7);
	664	src += (8 * src_stride);
	665	src8 = LD_UB(src);
	666
	667	SLDI_B4_0_UB(src0, src1, src2, src3, src0_sld1, src1_sld1, src2_sld1,
	668	src3_sld1, 1);
	669	SLDI_B3_0_UB(src4, src5, src6, src4_sld1, src5_sld1, src6_sld1, 1);
	670	SLDI_B2_0_UB(src7, src8, src7_sld1, src8_sld1, 1);
	671	ILVR_B4_UH(src0_sld1, src0, src1_sld1, src1, src2_sld1, src2, src3_sld1,
	672	src3, src0_r, src1_r, src2_r, src3_r);
	673	ILVR_B3_UH(src4_sld1, src4, src5_sld1, src5, src6_sld1, src6, src4_r,
	674	src5_r, src6_r);
	675	ILVR_B2_UH(src7_sld1, src7, src8_sld1, src8, src7_r, src8_r);
	676	HADD_UB3_UH(src0_r, src1_r, src2_r, add0, add1, add2);
	677	HADD_UB3_UH(src3_r, src4_r, src5_r, add3, add4, add5);
	678	HADD_UB3_UH(src6_r, src7_r, src8_r, add6, add7, add8);
	679
	680	sum0 = add0 + add1 + 1;
	681	sum1 = add1 + add2 + 1;
	682	sum2 = add2 + add3 + 1;
	683	sum3 = add3 + add4 + 1;
	684	sum4 = add4 + add5 + 1;
	685	sum5 = add5 + add6 + 1;
	686	sum6 = add6 + add7 + 1;
	687	sum7 = add7 + add8 + 1;
	688
	689	SRA_4V(sum0, sum1, sum2, sum3, 2);
	690	SRA_4V(sum4, sum5, sum6, sum7, 2);
	691	PCKEV_B2_SB(sum1, sum0, sum3, sum2, out0, out1);
	692	ST8x4_UB(out0, out1, dst, dst_stride);
	693	PCKEV_B2_SB(sum5, sum4, sum7, sum6, out0, out1);
	694	ST8x4_UB(out0, out1, dst + 4 * dst_stride, dst_stride);
	695	}
	696
	697	static void common_hv_bil_no_rnd_4x8_msa(const uint8_t *src, int32_t src_stride,
	698	uint8_t *dst, int32_t dst_stride)
	699	{
	700	v16i8 src0, src1, src2, src3, src4;
	701	v16i8 src0_sld1, src1_sld1, src2_sld1, src3_sld1, src4_sld1;
	702	v8u16 src0_r, src1_r, src2_r, src3_r, src4_r;
	703	v8u16 add0, add1, add2, add3, add4;
	704	v8u16 sum0, sum1, sum2, sum3;
	705	v16i8 out0, out1;
	706
	707	LD_SB4(src, src_stride, src0, src1, src2, src3);
	708	src += (4 * src_stride);
	709	src4 = LD_SB(src);
	710
	711	SLDI_B3_0_SB(src0, src1, src2, src0_sld1, src1_sld1, src2_sld1, 1);
	712	SLDI_B2_0_SB(src3, src4, src3_sld1, src4_sld1, 1);
	713	ILVR_B3_UH(src0_sld1, src0, src1_sld1, src1, src2_sld1, src2, src0_r,
	714	src1_r, src2_r);
	715	ILVR_B2_UH(src3_sld1, src3, src4_sld1, src4, src3_r, src4_r);
	716	HADD_UB3_UH(src0_r, src1_r, src2_r, add0, add1, add2);
	717	HADD_UB2_UH(src3_r, src4_r, add3, add4);
	718
	719	sum0 = add0 + add1 + 1;
	720	sum1 = add1 + add2 + 1;
	721	sum2 = add2 + add3 + 1;
	722	sum3 = add3 + add4 + 1;
	723
	724	SRA_4V(sum0, sum1, sum2, sum3, 2);
	725	PCKEV_B2_SB(sum1, sum0, sum3, sum2, out0, out1);
	726	ST8x4_UB(out0, out1, dst, dst_stride);
	727	}
	728
	729	static void common_hv_bil_no_rnd_16x16_msa(const uint8_t *src,
	730	int32_t src_stride,
	731	uint8_t *dst, int32_t dst_stride)
	732	{
	733	v16u8 src0, src1, src2, src3, src4, src5, src6, src7, src8, src9;
	734	v16u8 src10, src11, src12, src13, src14, src15, src16, src17;
	735	v8u16 src0_r, src1_r, src2_r, src3_r, src4_r, src5_r, src6_r, src7_r;
	736	v8u16 src8_r, src0_l, src1_l, src2_l, src3_l, src4_l, src5_l, src6_l;
	737	v8u16 src7_l, src8_l;
	738	v8u16 sum0_r, sum1_r, sum2_r, sum3_r, sum4_r, sum5_r, sum6_r, sum7_r;
	739	v8u16 sum0_l, sum1_l, sum2_l, sum3_l, sum4_l, sum5_l, sum6_l, sum7_l;
	740
	741	LD_UB8(src, src_stride, src0, src1, src2, src3, src4, src5, src6, src7);
	742	LD_UB8((src + 1), src_stride,
	743	src9, src10, src11, src12, src13, src14, src15, src16);
	744	src += (8 * src_stride);
	745	src8 = LD_UB(src);
	746	src17 = LD_UB(src + 1);
	747
	748	ILVRL_B2_UH(src9, src0, src0_r, src0_l);
	749	ILVRL_B2_UH(src10, src1, src1_r, src1_l);
	750	ILVRL_B2_UH(src11, src2, src2_r, src2_l);
	751	ILVRL_B2_UH(src12, src3, src3_r, src3_l);
	752	ILVRL_B2_UH(src13, src4, src4_r, src4_l);
	753	ILVRL_B2_UH(src14, src5, src5_r, src5_l);
	754	ILVRL_B2_UH(src15, src6, src6_r, src6_l);
	755	ILVRL_B2_UH(src16, src7, src7_r, src7_l);
	756	ILVRL_B2_UH(src17, src8, src8_r, src8_l);
	757
	758	HADD_UB3_UH(src0_r, src1_r, src2_r, src0_r, src1_r, src2_r);
	759	HADD_UB3_UH(src3_r, src4_r, src5_r, src3_r, src4_r, src5_r);
	760	HADD_UB3_UH(src6_r, src7_r, src8_r, src6_r, src7_r, src8_r);
	761	HADD_UB3_UH(src0_l, src1_l, src2_l, src0_l, src1_l, src2_l);
	762	HADD_UB3_UH(src3_l, src4_l, src5_l, src3_l, src4_l, src5_l);
	763	HADD_UB3_UH(src6_l, src7_l, src8_l, src6_l, src7_l, src8_l);
	764
	765	sum0_r = src0_r + src1_r + 1;
	766	sum1_r = src1_r + src2_r + 1;
	767	sum2_r = src2_r + src3_r + 1;
	768	sum3_r = src3_r + src4_r + 1;
	769	sum4_r = src4_r + src5_r + 1;
	770	sum5_r = src5_r + src6_r + 1;
	771	sum6_r = src6_r + src7_r + 1;
	772	sum7_r = src7_r + src8_r + 1;
	773	sum0_l = src0_l + src1_l + 1;
	774	sum1_l = src1_l + src2_l + 1;
	775	sum2_l = src2_l + src3_l + 1;
	776	sum3_l = src3_l + src4_l + 1;
	777	sum4_l = src4_l + src5_l + 1;
	778	sum5_l = src5_l + src6_l + 1;
	779	sum6_l = src6_l + src7_l + 1;
	780	sum7_l = src7_l + src8_l + 1;
	781
	782	SRA_4V(sum0_r, sum1_r, sum2_r, sum3_r, 2);
	783	SRA_4V(sum4_r, sum5_r, sum6_r, sum7_r, 2);
	784	SRA_4V(sum0_l, sum1_l, sum2_l, sum3_l, 2);
	785	SRA_4V(sum4_l, sum5_l, sum6_l, sum7_l, 2);
	786	PCKEV_ST_SB4(sum0_l, sum0_r, sum1_l, sum1_r,
	787	sum2_l, sum2_r, sum3_l, sum3_r, dst, dst_stride);
	788	dst += (4 * dst_stride);
	789
	790	LD_UB8(src, src_stride, src0, src1, src2, src3, src4, src5, src6, src7);
	791	LD_UB8((src + 1), src_stride,
	792	src9, src10, src11, src12, src13, src14, src15, src16);
	793	src += (8 * src_stride);
	794	src8 = LD_UB(src);
	795	src17 = LD_UB(src + 1);
	796
	797	PCKEV_ST_SB4(sum4_l, sum4_r, sum5_l, sum5_r,
	798	sum6_l, sum6_r, sum7_l, sum7_r, dst, dst_stride);
	799	dst += (4 * dst_stride);
	800
	801	ILVRL_B2_UH(src9, src0, src0_r, src0_l);
	802	ILVRL_B2_UH(src10, src1, src1_r, src1_l);
	803	ILVRL_B2_UH(src11, src2, src2_r, src2_l);
	804	ILVRL_B2_UH(src12, src3, src3_r, src3_l);
	805	ILVRL_B2_UH(src13, src4, src4_r, src4_l);
	806	ILVRL_B2_UH(src14, src5, src5_r, src5_l);
	807	ILVRL_B2_UH(src15, src6, src6_r, src6_l);
	808	ILVRL_B2_UH(src16, src7, src7_r, src7_l);
	809	ILVRL_B2_UH(src17, src8, src8_r, src8_l);
	810
	811	HADD_UB3_UH(src0_r, src1_r, src2_r, src0_r, src1_r, src2_r);
	812	HADD_UB3_UH(src3_r, src4_r, src5_r, src3_r, src4_r, src5_r);
	813	HADD_UB3_UH(src6_r, src7_r, src8_r, src6_r, src7_r, src8_r);
	814	HADD_UB3_UH(src0_l, src1_l, src2_l, src0_l, src1_l, src2_l);
	815	HADD_UB3_UH(src3_l, src4_l, src5_l, src3_l, src4_l, src5_l);
	816	HADD_UB3_UH(src6_l, src7_l, src8_l, src6_l, src7_l, src8_l);
	817
	818	sum0_r = src0_r + src1_r + 1;
	819	sum1_r = src1_r + src2_r + 1;
	820	sum2_r = src2_r + src3_r + 1;
	821	sum3_r = src3_r + src4_r + 1;
	822	sum4_r = src4_r + src5_r + 1;
	823	sum5_r = src5_r + src6_r + 1;
	824	sum6_r = src6_r + src7_r + 1;
	825	sum7_r = src7_r + src8_r + 1;
	826	sum0_l = src0_l + src1_l + 1;
	827	sum1_l = src1_l + src2_l + 1;
	828	sum2_l = src2_l + src3_l + 1;
	829	sum3_l = src3_l + src4_l + 1;
	830	sum4_l = src4_l + src5_l + 1;
	831	sum5_l = src5_l + src6_l + 1;
	832	sum6_l = src6_l + src7_l + 1;
	833	sum7_l = src7_l + src8_l + 1;
	834
	835	SRA_4V(sum0_r, sum1_r, sum2_r, sum3_r, 2);
	836	SRA_4V(sum4_r, sum5_r, sum6_r, sum7_r, 2);
	837	SRA_4V(sum0_l, sum1_l, sum2_l, sum3_l, 2);
	838	SRA_4V(sum4_l, sum5_l, sum6_l, sum7_l, 2);
	839	PCKEV_ST_SB4(sum0_l, sum0_r, sum1_l, sum1_r,
	840	sum2_l, sum2_r, sum3_l, sum3_r, dst, dst_stride);
	841	dst += (4 * dst_stride);
	842	PCKEV_ST_SB4(sum4_l, sum4_r, sum5_l, sum5_r,
	843	sum6_l, sum6_r, sum7_l, sum7_r, dst, dst_stride);
	844	}
	845
	846	static void common_hv_bil_no_rnd_8x16_msa(const uint8_t *src,
	847	int32_t src_stride,
	848	uint8_t *dst, int32_t dst_stride)
	849	{
	850	v16u8 src0, src1, src2, src3, src4, src5, src6, src7, src8, src9;
	851	v16u8 src10, src11, src12, src13, src14, src15, src16, src17;
	852	v8u16 src0_r, src1_r, src2_r, src3_r, src4_r, src5_r, src6_r, src7_r;
	853	v8u16 src8_r, src0_l, src1_l, src2_l, src3_l, src4_l, src5_l, src6_l;
	854	v8u16 src7_l, src8_l;
	855	v8u16 sum0_r, sum1_r, sum2_r, sum3_r, sum4_r, sum5_r, sum6_r, sum7_r;
	856	v8u16 sum0_l, sum1_l, sum2_l, sum3_l, sum4_l, sum5_l, sum6_l, sum7_l;
	857
	858	LD_UB8(src, src_stride, src0, src1, src2, src3, src4, src5, src6, src7);
	859	LD_UB8((src + 1), src_stride,
	860	src9, src10, src11, src12, src13, src14, src15, src16);
	861	src += (8 * src_stride);
	862	src8 = LD_UB(src);
	863	src17 = LD_UB(src + 1);
	864
	865	ILVRL_B2_UH(src9, src0, src0_r, src0_l);
	866	ILVRL_B2_UH(src10, src1, src1_r, src1_l);
	867	ILVRL_B2_UH(src11, src2, src2_r, src2_l);
	868	ILVRL_B2_UH(src12, src3, src3_r, src3_l);
	869	ILVRL_B2_UH(src13, src4, src4_r, src4_l);
	870	ILVRL_B2_UH(src14, src5, src5_r, src5_l);
	871	ILVRL_B2_UH(src15, src6, src6_r, src6_l);
	872	ILVRL_B2_UH(src16, src7, src7_r, src7_l);
	873	ILVRL_B2_UH(src17, src8, src8_r, src8_l);
	874
	875	HADD_UB3_UH(src0_r, src1_r, src2_r, src0_r, src1_r, src2_r);
	876	HADD_UB3_UH(src3_r, src4_r, src5_r, src3_r, src4_r, src5_r);
	877	HADD_UB3_UH(src6_r, src7_r, src8_r, src6_r, src7_r, src8_r);
	878	HADD_UB3_UH(src0_l, src1_l, src2_l, src0_l, src1_l, src2_l);
	879	HADD_UB3_UH(src3_l, src4_l, src5_l, src3_l, src4_l, src5_l);
	880	HADD_UB3_UH(src6_l, src7_l, src8_l, src6_l, src7_l, src8_l);
	881
	882	sum0_r = src0_r + src1_r + 1;
	883	sum1_r = src1_r + src2_r + 1;
	884	sum2_r = src2_r + src3_r + 1;
	885	sum3_r = src3_r + src4_r + 1;
	886	sum4_r = src4_r + src5_r + 1;
	887	sum5_r = src5_r + src6_r + 1;
	888	sum6_r = src6_r + src7_r + 1;
	889	sum7_r = src7_r + src8_r + 1;
	890	sum0_l = src0_l + src1_l + 1;
	891	sum1_l = src1_l + src2_l + 1;
	892	sum2_l = src2_l + src3_l + 1;
	893	sum3_l = src3_l + src4_l + 1;
	894	sum4_l = src4_l + src5_l + 1;
	895	sum5_l = src5_l + src6_l + 1;
	896	sum6_l = src6_l + src7_l + 1;
	897	sum7_l = src7_l + src8_l + 1;
	898
	899	SRA_4V(sum0_r, sum1_r, sum2_r, sum3_r, 2);
	900	SRA_4V(sum4_r, sum5_r, sum6_r, sum7_r, 2);
	901	SRA_4V(sum0_l, sum1_l, sum2_l, sum3_l, 2);
	902	SRA_4V(sum4_l, sum5_l, sum6_l, sum7_l, 2);
	903	PCKEV_ST_SB4(sum0_l, sum0_r, sum1_l, sum1_r,
	904	sum2_l, sum2_r, sum3_l, sum3_r, dst, dst_stride);
	905	dst += (4 * dst_stride);
	906	PCKEV_ST_SB4(sum4_l, sum4_r, sum5_l, sum5_r,
	907	sum6_l, sum6_r, sum7_l, sum7_r, dst, dst_stride);
	908	}
	909
	910	static void common_hv_bil_and_aver_dst_4w_msa(const uint8_t *src,
	911	int32_t src_stride,
	912	uint8_t *dst, int32_t dst_stride,
	913	uint8_t height)
	914	{
	915	uint8_t loop_cnt;
	916	uint32_t out0, out1;
	917	v16i8 src0, src1, src2, src0_sld1, src1_sld1, src2_sld1;
	918	v16u8 src0_r, src1_r, src2_r;
	919	v8u16 add0, add1, add2, sum0, sum1;
	920	v16u8 dst0, dst1, res0, res1;
	921
	922	src0 = LD_SB(src);
	923	src += src_stride;
	924
	925	for (loop_cnt = (height >> 1); loop_cnt--;) {
	926	LD_SB2(src, src_stride, src1, src2);
	927	src += (2 * src_stride);
	928
	929	LD_UB2(dst, dst_stride, dst0, dst1);
	930	SLDI_B3_0_SB(src0, src1, src2, src0_sld1, src1_sld1, src2_sld1, 1);
	931	ILVR_B3_UB(src0_sld1, src0, src1_sld1, src1, src2_sld1, src2, src0_r,
	932	src1_r, src2_r);
	933	HADD_UB3_UH(src0_r, src1_r, src2_r, add0, add1, add2);
	934	ADD2(add0, add1, add1, add2, sum0, sum1);
	935	SRARI_H2_UH(sum0, sum1, 2);
	936	PCKEV_B2_UB(sum0, sum0, sum1, sum1, res0, res1);
	937	AVER_UB2_UB(dst0, res0, dst1, res1, res0, res1);
	938
	939	out0 = __msa_copy_u_w((v4i32) res0, 0);
	940	out1 = __msa_copy_u_w((v4i32) res1, 0);
	941	SW(out0, dst);
	942	dst += dst_stride;
	943	SW(out1, dst);
	944	dst += dst_stride;
	945
	946	src0 = src2;
	947	}
	948	}
	949
	950	static void common_hv_bil_and_aver_dst_8w_msa(const uint8_t *src,
	951	int32_t src_stride,
	952	uint8_t *dst, int32_t dst_stride,
	953	uint8_t height)
	954	{
	955	uint8_t loop_cnt;
	956	v16i8 src0, src1, src2, src3, src4;
	957	v16i8 src0_sld1, src1_sld1, src2_sld1, src3_sld1, src4_sld1;
	958	v16u8 dst0, dst1, dst2, dst3;
	959	v16u8 src0_r, src1_r, src2_r, src3_r, src4_r;
	960	v8u16 add0, add1, add2, add3, add4;
	961	v8u16 sum0, sum1, sum2, sum3;
	962
	963	src0 = LD_SB(src);
	964	src += src_stride;
	965
	966	for (loop_cnt = (height >> 2); loop_cnt--;) {
	967	LD_SB4(src, src_stride, src1, src2, src3, src4);
	968	src += (4 * src_stride);
	969
	970	LD_UB4(dst, dst_stride, dst0, dst1, dst2, dst3);
	971	SLDI_B3_0_SB(src0, src1, src2, src0_sld1, src1_sld1, src2_sld1, 1);
	972	SLDI_B2_0_SB(src3, src4, src3_sld1, src4_sld1, 1);
	973	ILVR_B3_UB(src0_sld1, src0, src1_sld1, src1, src2_sld1, src2, src0_r,
	974	src1_r, src2_r);
	975	ILVR_B2_UB(src3_sld1, src3, src4_sld1, src4, src3_r, src4_r);
	976	HADD_UB3_UH(src0_r, src1_r, src2_r, add0, add1, add2);
	977	HADD_UB2_UH(src3_r, src4_r, add3, add4);
	978	ADD4(add0, add1, add1, add2, add2, add3, add3, add4,
	979	sum0, sum1, sum2, sum3);
	980	SRARI_H4_UH(sum0, sum1, sum2, sum3, 2);
	981	PCKEV_AVG_ST8x4_UB(sum0, dst0, sum1, dst1,
	982	sum2, dst2, sum3, dst3, dst, dst_stride);
	983	dst += (4 * dst_stride);
	984	src0 = src4;
	985	}
	986	}
	987
	988	static void common_hv_bil_and_aver_dst_16w_msa(const uint8_t *src,
	989	int32_t src_stride,
	990	uint8_t *dst, int32_t dst_stride,
	991	uint8_t height)
	992	{
	993	uint8_t loop_cnt;
	994	v16u8 src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10;
	995	v16u8 src11, src12, src13, src14, src15, src16, src17;
	996	v16u8 src0_r, src1_r, src2_r, src3_r, src4_r, src5_r, src6_r, src7_r;
	997	v16u8 src8_r, src0_l, src1_l, src2_l, src3_l, src4_l, src5_l, src6_l;
	998	v16u8 src7_l, src8_l;
	999	v16u8 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7;
	1000	v8u16 sum0_r, sum1_r, sum2_r, sum3_r, sum4_r, sum5_r, sum6_r, sum7_r;
	1001	v8u16 sum0_l, sum1_l, sum2_l, sum3_l, sum4_l, sum5_l, sum6_l, sum7_l;
	1002	v8u16 add0, add1, add2, add3, add4, add5, add6, add7, add8;
	1003
	1004	for (loop_cnt = (height >> 3); loop_cnt--;) {
	1005	LD_UB8(src, src_stride, src0, src1, src2, src3, src4, src5, src6, src7);
	1006	LD_UB8((src + 1), src_stride,
	1007	src9, src10, src11, src12, src13, src14, src15, src16);
	1008	src += (8 * src_stride);
	1009
	1010	src8 = LD_UB(src);
	1011	src17 = LD_UB(src + 1);
	1012
	1013	ILVRL_B2_UB(src9, src0, src0_r, src0_l);
	1014	ILVRL_B2_UB(src10, src1, src1_r, src1_l);
	1015	ILVRL_B2_UB(src11, src2, src2_r, src2_l);
	1016	ILVRL_B2_UB(src12, src3, src3_r, src3_l);
	1017	ILVRL_B2_UB(src13, src4, src4_r, src4_l);
	1018	ILVRL_B2_UB(src14, src5, src5_r, src5_l);
	1019	ILVRL_B2_UB(src15, src6, src6_r, src6_l);
	1020	ILVRL_B2_UB(src16, src7, src7_r, src7_l);
	1021	ILVRL_B2_UB(src17, src8, src8_r, src8_l);
	1022	HADD_UB3_UH(src0_r, src1_r, src2_r, add0, add1, add2);
	1023	HADD_UB3_UH(src3_r, src4_r, src5_r, add3, add4, add5);
	1024	HADD_UB3_UH(src6_r, src7_r, src8_r, add6, add7, add8);
	1025	ADD4(add0, add1, add1, add2, add2, add3, add3, add4, sum0_r, sum1_r,
	1026	sum2_r, sum3_r);
	1027	ADD4(add4, add5, add5, add6, add6, add7, add7, add8, sum4_r, sum5_r,
	1028	sum6_r, sum7_r);
	1029	HADD_UB3_UH(src0_l, src1_l, src2_l, add0, add1, add2);
	1030	HADD_UB3_UH(src3_l, src4_l, src5_l, add3, add4, add5);
	1031	HADD_UB3_UH(src6_l, src7_l, src8_l, add6, add7, add8);
	1032	ADD4(add0, add1, add1, add2, add2, add3, add3, add4, sum0_l, sum1_l,
	1033	sum2_l, sum3_l);
	1034	ADD4(add4, add5, add5, add6, add6, add7, add7, add8, sum4_l, sum5_l,
	1035	sum6_l, sum7_l);
	1036	SRARI_H4_UH(sum0_r, sum1_r, sum2_r, sum3_r, 2);
	1037	SRARI_H4_UH(sum4_r, sum5_r, sum6_r, sum7_r, 2);
	1038	SRARI_H4_UH(sum0_l, sum1_l, sum2_l, sum3_l, 2);
	1039	SRARI_H4_UH(sum4_l, sum5_l, sum6_l, sum7_l, 2);
	1040	LD_UB8(dst, dst_stride, dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7);
	1041	PCKEV_AVG_ST_UB(sum0_l, sum0_r, dst0, dst);
	1042	dst += dst_stride;
	1043	PCKEV_AVG_ST_UB(sum1_l, sum1_r, dst1, dst);
	1044	dst += dst_stride;
	1045	PCKEV_AVG_ST_UB(sum2_l, sum2_r, dst2, dst);
	1046	dst += dst_stride;
	1047	PCKEV_AVG_ST_UB(sum3_l, sum3_r, dst3, dst);
	1048	dst += dst_stride;
	1049	PCKEV_AVG_ST_UB(sum4_l, sum4_r, dst4, dst);
	1050	dst += dst_stride;
	1051	PCKEV_AVG_ST_UB(sum5_l, sum5_r, dst5, dst);
	1052	dst += dst_stride;
	1053	PCKEV_AVG_ST_UB(sum6_l, sum6_r, dst6, dst);
	1054	dst += dst_stride;
	1055	PCKEV_AVG_ST_UB(sum7_l, sum7_r, dst7, dst);
	1056	dst += dst_stride;
	1057	}
	1058	}
	1059
	1060	static void copy_width8_msa(const uint8_t *src, int32_t src_stride,
	1061	uint8_t *dst, int32_t dst_stride,
	1062	int32_t height)
	1063	{
	1064	int32_t cnt;
	1065	uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
	1066	v16u8 src0, src1, src2, src3, src4, src5, src6, src7;
	1067
	1068	if (0 == height % 12) {
	1069	for (cnt = (height / 12); cnt--;) {
	1070	LD_UB8(src, src_stride,
	1071	src0, src1, src2, src3, src4, src5, src6, src7);
	1072	src += (8 * src_stride);
	1073
	1074	out0 = __msa_copy_u_d((v2i64) src0, 0);
	1075	out1 = __msa_copy_u_d((v2i64) src1, 0);
	1076	out2 = __msa_copy_u_d((v2i64) src2, 0);
	1077	out3 = __msa_copy_u_d((v2i64) src3, 0);
	1078	out4 = __msa_copy_u_d((v2i64) src4, 0);
	1079	out5 = __msa_copy_u_d((v2i64) src5, 0);
	1080	out6 = __msa_copy_u_d((v2i64) src6, 0);
	1081	out7 = __msa_copy_u_d((v2i64) src7, 0);
	1082
	1083	SD4(out0, out1, out2, out3, dst, dst_stride);
	1084	dst += (4 * dst_stride);
	1085	SD4(out4, out5, out6, out7, dst, dst_stride);
	1086	dst += (4 * dst_stride);
	1087
	1088	LD_UB4(src, src_stride, src0, src1, src2, src3);
	1089	src += (4 * src_stride);
	1090
	1091	out0 = __msa_copy_u_d((v2i64) src0, 0);
	1092	out1 = __msa_copy_u_d((v2i64) src1, 0);
	1093	out2 = __msa_copy_u_d((v2i64) src2, 0);
	1094	out3 = __msa_copy_u_d((v2i64) src3, 0);
	1095
	1096	SD4(out0, out1, out2, out3, dst, dst_stride);
	1097	dst += (4 * dst_stride);
	1098	}
	1099	} else if (0 == height % 8) {
	1100	for (cnt = height >> 3; cnt--;) {
	1101	LD_UB8(src, src_stride,
	1102	src0, src1, src2, src3, src4, src5, src6, src7);
	1103	src += (8 * src_stride);
	1104
	1105	out0 = __msa_copy_u_d((v2i64) src0, 0);
	1106	out1 = __msa_copy_u_d((v2i64) src1, 0);
	1107	out2 = __msa_copy_u_d((v2i64) src2, 0);
	1108	out3 = __msa_copy_u_d((v2i64) src3, 0);
	1109	out4 = __msa_copy_u_d((v2i64) src4, 0);
	1110	out5 = __msa_copy_u_d((v2i64) src5, 0);
	1111	out6 = __msa_copy_u_d((v2i64) src6, 0);
	1112	out7 = __msa_copy_u_d((v2i64) src7, 0);
	1113
	1114	SD4(out0, out1, out2, out3, dst, dst_stride);
	1115	dst += (4 * dst_stride);
	1116	SD4(out4, out5, out6, out7, dst, dst_stride);
	1117	dst += (4 * dst_stride);
	1118	}
	1119	} else if (0 == height % 4) {
	1120	for (cnt = (height / 4); cnt--;) {
	1121	LD_UB4(src, src_stride, src0, src1, src2, src3);
	1122	src += (4 * src_stride);
	1123	out0 = __msa_copy_u_d((v2i64) src0, 0);
	1124	out1 = __msa_copy_u_d((v2i64) src1, 0);
	1125	out2 = __msa_copy_u_d((v2i64) src2, 0);
	1126	out3 = __msa_copy_u_d((v2i64) src3, 0);
	1127
	1128	SD4(out0, out1, out2, out3, dst, dst_stride);
	1129	dst += (4 * dst_stride);
	1130	}
	1131	} else if (0 == height % 2) {
	1132	for (cnt = (height / 2); cnt--;) {
	1133	LD_UB2(src, src_stride, src0, src1);
	1134	src += (2 * src_stride);
	1135	out0 = __msa_copy_u_d((v2i64) src0, 0);
	1136	out1 = __msa_copy_u_d((v2i64) src1, 0);
	1137
	1138	SD(out0, dst);
	1139	dst += dst_stride;
	1140	SD(out1, dst);
	1141	dst += dst_stride;
	1142	}
	1143	}
	1144	}
	1145
	1146	static void copy_16multx8mult_msa(const uint8_t *src, int32_t src_stride,
	1147	uint8_t *dst, int32_t dst_stride,
	1148	int32_t height, int32_t width)
	1149	{
	1150	int32_t cnt, loop_cnt;
	1151	const uint8_t *src_tmp;
	1152	uint8_t *dst_tmp;
	1153	v16u8 src0, src1, src2, src3, src4, src5, src6, src7;
	1154
	1155	for (cnt = (width >> 4); cnt--;) {
	1156	src_tmp = src;
	1157	dst_tmp = dst;
	1158
	1159	for (loop_cnt = (height >> 3); loop_cnt--;) {
	1160	LD_UB8(src_tmp, src_stride,
	1161	src0, src1, src2, src3, src4, src5, src6, src7);
	1162	src_tmp += (8 * src_stride);
	1163
	1164	ST_UB8(src0, src1, src2, src3, src4, src5, src6, src7,
	1165	dst_tmp, dst_stride);
	1166	dst_tmp += (8 * dst_stride);
	1167	}
	1168
	1169	src += 16;
	1170	dst += 16;
	1171	}
	1172	}
	1173
	1174	static void copy_width16_msa(const uint8_t *src, int32_t src_stride,
	1175	uint8_t *dst, int32_t dst_stride,
	1176	int32_t height)
	1177	{
	1178	int32_t cnt;
	1179	v16u8 src0, src1, src2, src3, src4, src5, src6, src7;
	1180
	1181	if (0 == height % 12) {
	1182	for (cnt = (height / 12); cnt--;) {
	1183	LD_UB8(src, src_stride,
	1184	src0, src1, src2, src3, src4, src5, src6, src7);
	1185	src += (8 * src_stride);
	1186	ST_UB8(src0, src1, src2, src3, src4, src5, src6, src7,
	1187	dst, dst_stride);
	1188	dst += (8 * dst_stride);
	1189
	1190	LD_UB4(src, src_stride, src0, src1, src2, src3);
	1191	src += (4 * src_stride);
	1192	ST_UB4(src0, src1, src2, src3, dst, dst_stride);
	1193	dst += (4 * dst_stride);
	1194	}
	1195	} else if (0 == height % 8) {
	1196	copy_16multx8mult_msa(src, src_stride, dst, dst_stride, height, 16);
	1197	} else if (0 == height % 4) {
	1198	for (cnt = (height >> 2); cnt--;) {
	1199	LD_UB4(src, src_stride, src0, src1, src2, src3);
	1200	src += (4 * src_stride);
	1201
	1202	ST_UB4(src0, src1, src2, src3, dst, dst_stride);
	1203	dst += (4 * dst_stride);
	1204	}
	1205	}
	1206	}
	1207
	1208	static void avg_width4_msa(const uint8_t *src, int32_t src_stride,
	1209	uint8_t *dst, int32_t dst_stride,
	1210	int32_t height)
	1211	{
	1212	int32_t cnt;
	1213	uint32_t out0, out1, out2, out3;
	1214	v16u8 src0, src1, src2, src3;
	1215	v16u8 dst0, dst1, dst2, dst3;
	1216
	1217	if (0 == (height % 4)) {
	1218	for (cnt = (height / 4); cnt--;) {
	1219	LD_UB4(src, src_stride, src0, src1, src2, src3);
	1220	src += (4 * src_stride);
	1221
	1222	LD_UB4(dst, dst_stride, dst0, dst1, dst2, dst3);
	1223
	1224	AVER_UB4_UB(src0, dst0, src1, dst1, src2, dst2, src3, dst3,
	1225	dst0, dst1, dst2, dst3);
	1226
	1227	out0 = __msa_copy_u_w((v4i32) dst0, 0);
	1228	out1 = __msa_copy_u_w((v4i32) dst1, 0);
	1229	out2 = __msa_copy_u_w((v4i32) dst2, 0);
	1230	out3 = __msa_copy_u_w((v4i32) dst3, 0);
	1231	SW4(out0, out1, out2, out3, dst, dst_stride);
	1232	dst += (4 * dst_stride);
	1233	}
	1234	} else if (0 == (height % 2)) {
	1235	for (cnt = (height / 2); cnt--;) {
	1236	LD_UB2(src, src_stride, src0, src1);
	1237	src += (2 * src_stride);
	1238
	1239	LD_UB2(dst, dst_stride, dst0, dst1);
	1240
	1241	AVER_UB2_UB(src0, dst0, src1, dst1, dst0, dst1);
	1242
	1243	out0 = __msa_copy_u_w((v4i32) dst0, 0);
	1244	out1 = __msa_copy_u_w((v4i32) dst1, 0);
	1245	SW(out0, dst);
	1246	dst += dst_stride;
	1247	SW(out1, dst);
	1248	dst += dst_stride;
	1249	}
	1250	}
	1251	}
	1252
	1253	static void avg_width8_msa(const uint8_t *src, int32_t src_stride,
	1254	uint8_t *dst, int32_t dst_stride,
	1255	int32_t height)
	1256	{
	1257	int32_t cnt;
	1258	uint64_t out0, out1, out2, out3;
	1259	v16u8 src0, src1, src2, src3;
	1260	v16u8 dst0, dst1, dst2, dst3;
	1261
	1262	for (cnt = (height / 4); cnt--;) {
	1263	LD_UB4(src, src_stride, src0, src1, src2, src3);
	1264	src += (4 * src_stride);
	1265	LD_UB4(dst, dst_stride, dst0, dst1, dst2, dst3);
	1266
	1267	AVER_UB4_UB(src0, dst0, src1, dst1, src2, dst2, src3, dst3,
	1268	dst0, dst1, dst2, dst3);
	1269
	1270	out0 = __msa_copy_u_d((v2i64) dst0, 0);
	1271	out1 = __msa_copy_u_d((v2i64) dst1, 0);
	1272	out2 = __msa_copy_u_d((v2i64) dst2, 0);
	1273	out3 = __msa_copy_u_d((v2i64) dst3, 0);
	1274	SD4(out0, out1, out2, out3, dst, dst_stride);
	1275	dst += (4 * dst_stride);
	1276	}
	1277	}
	1278
	1279	static void avg_width16_msa(const uint8_t *src, int32_t src_stride,
	1280	uint8_t *dst, int32_t dst_stride,
	1281	int32_t height)
	1282	{
	1283	int32_t cnt;
	1284	v16u8 src0, src1, src2, src3, src4, src5, src6, src7;
	1285	v16u8 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7;
	1286
	1287	for (cnt = (height / 8); cnt--;) {
	1288	LD_UB8(src, src_stride, src0, src1, src2, src3, src4, src5, src6, src7);
	1289	src += (8 * src_stride);
	1290	LD_UB8(dst, dst_stride, dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7);
	1291
	1292	AVER_UB4_UB(src0, dst0, src1, dst1, src2, dst2, src3, dst3,
	1293	dst0, dst1, dst2, dst3);
	1294	AVER_UB4_UB(src4, dst4, src5, dst5, src6, dst6, src7, dst7,
	1295	dst4, dst5, dst6, dst7);
	1296	ST_UB8(dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7, dst, dst_stride);
	1297	dst += (8 * dst_stride);
	1298	}
	1299	}
	1300
	1301	void ff_put_pixels16_msa(uint8_t block, const uint8_t pixels,
	1302	ptrdiff_t line_size, int h)
	1303	{
	1304	copy_width16_msa(pixels, line_size, block, line_size, h);
	1305	}
	1306
	1307	void ff_put_pixels16_x2_msa(uint8_t block, const uint8_t pixels,
	1308	ptrdiff_t line_size, int h)
	1309	{
	1310	common_hz_bil_16w_msa(pixels, line_size, block, line_size, h);
	1311	}
	1312
	1313	void ff_put_pixels16_y2_msa(uint8_t block, const uint8_t pixels,
	1314	ptrdiff_t line_size, int h)
	1315	{
	1316	common_vt_bil_16w_msa(pixels, line_size, block, line_size, h);
	1317	}
	1318
	1319	void ff_put_pixels16_xy2_msa(uint8_t block, const uint8_t pixels,
	1320	ptrdiff_t line_size, int h)
	1321	{
	1322	common_hv_bil_16w_msa(pixels, line_size, block, line_size, h);
	1323	}
	1324
	1325	void ff_put_pixels8_msa(uint8_t block, const uint8_t pixels,
	1326	ptrdiff_t line_size, int h)
	1327	{
	1328	copy_width8_msa(pixels, line_size, block, line_size, h);
	1329	}
	1330
	1331	void ff_put_pixels8_x2_msa(uint8_t block, const uint8_t pixels,
	1332	ptrdiff_t line_size, int h)
	1333	{
	1334	common_hz_bil_8w_msa(pixels, line_size, block, line_size, h);
	1335	}
	1336
	1337	void ff_put_pixels8_y2_msa(uint8_t block, const uint8_t pixels,
	1338	ptrdiff_t line_size, int h)
	1339	{
	1340	common_vt_bil_8w_msa(pixels, line_size, block, line_size, h);
	1341	}
	1342
	1343	void ff_put_pixels8_xy2_msa(uint8_t block, const uint8_t pixels,
	1344	ptrdiff_t line_size, int h)
	1345	{
	1346	common_hv_bil_8w_msa(pixels, line_size, block, line_size, h);
	1347	}
	1348
	1349	void ff_put_pixels4_x2_msa(uint8_t block, const uint8_t pixels,
	1350	ptrdiff_t line_size, int h)
	1351	{
	1352	common_hz_bil_4w_msa(pixels, line_size, block, line_size, h);
	1353	}
	1354
	1355	void ff_put_pixels4_y2_msa(uint8_t block, const uint8_t pixels,
	1356	ptrdiff_t line_size, int h)
	1357	{
	1358	common_vt_bil_4w_msa(pixels, line_size, block, line_size, h);
	1359	}
	1360
	1361	void ff_put_pixels4_xy2_msa(uint8_t block, const uint8_t pixels,
	1362	ptrdiff_t line_size, int h)
	1363	{
	1364	common_hv_bil_4w_msa(pixels, line_size, block, line_size, h);
	1365	}
	1366
	1367	void ff_put_no_rnd_pixels16_x2_msa(uint8_t block, const uint8_t pixels,
	1368	ptrdiff_t line_size, int h)
	1369	{
	1370	if (h == 16) {
	1371	common_hz_bil_no_rnd_16x16_msa(pixels, line_size, block, line_size);
	1372	} else if (h == 8) {
	1373	common_hz_bil_no_rnd_8x16_msa(pixels, line_size, block, line_size);
	1374	}
	1375	}
	1376
	1377	void ff_put_no_rnd_pixels16_y2_msa(uint8_t block, const uint8_t pixels,
	1378	ptrdiff_t line_size, int h)
	1379	{
	1380	if (h == 16) {
	1381	common_vt_bil_no_rnd_16x16_msa(pixels, line_size, block, line_size);
	1382	} else if (h == 8) {
	1383	common_vt_bil_no_rnd_8x16_msa(pixels, line_size, block, line_size);
	1384	}
	1385	}
	1386
	1387	void ff_put_no_rnd_pixels16_xy2_msa(uint8_t *block,
	1388	const uint8_t *pixels,
	1389	ptrdiff_t line_size, int h)
	1390	{
	1391	if (h == 16) {
	1392	common_hv_bil_no_rnd_16x16_msa(pixels, line_size, block, line_size);
	1393	} else if (h == 8) {
	1394	common_hv_bil_no_rnd_8x16_msa(pixels, line_size, block, line_size);
	1395	}
	1396	}
	1397
	1398	void ff_put_no_rnd_pixels8_x2_msa(uint8_t block, const uint8_t pixels,
	1399	ptrdiff_t line_size, int h)
	1400	{
	1401	if (h == 8) {
	1402	common_hz_bil_no_rnd_8x8_msa(pixels, line_size, block, line_size);
	1403	} else if (h == 4) {
	1404	common_hz_bil_no_rnd_4x8_msa(pixels, line_size, block, line_size);
	1405	}
	1406	}
	1407
	1408	void ff_put_no_rnd_pixels8_y2_msa(uint8_t block, const uint8_t pixels,
	1409	ptrdiff_t line_size, int h)
	1410	{
	1411	if (h == 8) {
	1412	common_vt_bil_no_rnd_8x8_msa(pixels, line_size, block, line_size);
	1413	} else if (h == 4) {
	1414	common_vt_bil_no_rnd_4x8_msa(pixels, line_size, block, line_size);
	1415	}
	1416	}
	1417
	1418	void ff_put_no_rnd_pixels8_xy2_msa(uint8_t block, const uint8_t pixels,
	1419	ptrdiff_t line_size, int h)
	1420	{
	1421	if (h == 8) {
	1422	common_hv_bil_no_rnd_8x8_msa(pixels, line_size, block, line_size);
	1423	} else if (h == 4) {
	1424	common_hv_bil_no_rnd_4x8_msa(pixels, line_size, block, line_size);
	1425	}
	1426	}
	1427
	1428	void ff_avg_pixels16_msa(uint8_t block, const uint8_t pixels,
	1429	ptrdiff_t line_size, int h)
	1430	{
	1431	avg_width16_msa(pixels, line_size, block, line_size, h);
	1432	}
	1433
	1434	void ff_avg_pixels16_x2_msa(uint8_t block, const uint8_t pixels,
	1435	ptrdiff_t line_size, int h)
	1436	{
	1437	common_hz_bil_and_aver_dst_16w_msa(pixels, line_size, block, line_size, h);
	1438	}
	1439
	1440	void ff_avg_pixels16_y2_msa(uint8_t block, const uint8_t pixels,
	1441	ptrdiff_t line_size, int h)
	1442	{
	1443	common_vt_bil_and_aver_dst_16w_msa(pixels, line_size, block, line_size, h);
	1444	}
	1445
	1446	void ff_avg_pixels16_xy2_msa(uint8_t block, const uint8_t pixels,
	1447	ptrdiff_t line_size, int h)
	1448	{
	1449	common_hv_bil_and_aver_dst_16w_msa(pixels, line_size, block, line_size, h);
	1450	}
	1451
	1452	void ff_avg_pixels8_msa(uint8_t block, const uint8_t pixels,
	1453	ptrdiff_t line_size, int h)
	1454	{
	1455	avg_width8_msa(pixels, line_size, block, line_size, h);
	1456	}
	1457
	1458	void ff_avg_pixels8_x2_msa(uint8_t block, const uint8_t pixels,
	1459	ptrdiff_t line_size, int h)
	1460	{
	1461	common_hz_bil_and_aver_dst_8w_msa(pixels, line_size, block, line_size, h);
	1462	}
	1463
	1464	void ff_avg_pixels8_y2_msa(uint8_t block, const uint8_t pixels,
	1465	ptrdiff_t line_size, int h)
	1466	{
	1467	common_vt_bil_and_aver_dst_8w_msa(pixels, line_size, block, line_size, h);
	1468	}
	1469
	1470	void ff_avg_pixels8_xy2_msa(uint8_t block, const uint8_t pixels,
	1471	ptrdiff_t line_size, int h)
	1472	{
	1473	common_hv_bil_and_aver_dst_8w_msa(pixels, line_size, block, line_size, h);
	1474	}
	1475
	1476	void ff_avg_pixels4_msa(uint8_t block, const uint8_t pixels,
	1477	ptrdiff_t line_size, int h)
	1478	{
	1479	avg_width4_msa(pixels, line_size, block, line_size, h);
	1480	}
	1481
	1482	void ff_avg_pixels4_x2_msa(uint8_t block, const uint8_t pixels,
	1483	ptrdiff_t line_size, int h)
	1484	{
	1485	common_hz_bil_and_aver_dst_4w_msa(pixels, line_size, block, line_size, h);
	1486	}
	1487
	1488	void ff_avg_pixels4_y2_msa(uint8_t block, const uint8_t pixels,
	1489	ptrdiff_t line_size, int h)
	1490	{
	1491	common_vt_bil_and_aver_dst_4w_msa(pixels, line_size, block, line_size, h);
	1492	}
	1493
	1494	void ff_avg_pixels4_xy2_msa(uint8_t block, const uint8_t pixels,
	1495	ptrdiff_t line_size, int h)
	1496	{
	1497	common_hv_bil_and_aver_dst_4w_msa(pixels, line_size, block, line_size, h);
	1498	}

avcodec/mips: MSA (MIPS-SIMD-Arch) optimizations for hpel functions

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code