avcodec/mips: MSA (MIPS-SIMD-Arch) optimizations for hpel functions

This patch adds MSA (MIPS-SIMD-Arch) optimizations for hpel functions in new file hpeldsp_msa.c Adds new generic macros (needed for this patch) in libavutil/mips/generic_macros_msa.h

Shivraj Patil committed 11 years ago

Commit ee3ef5f1 parent d277b057 files · +1825 −0Browse files

Unified · Split

7 files

libavcodec/hpeldsp.c +2 −0
libavcodec/hpeldsp.h +1 −0
libavcodec/mips/Makefile +2 −0
libavcodec/mips/hpeldsp_init_mips.c +73 −0
libavcodec/mips/hpeldsp_mips.h +87 −0
libavcodec/mips/hpeldsp_msa.c +1498 −0
libavutil/mips/generic_macros_msa.h +162 −0

libavcodec/hpeldsp.c+2−0

@@ -365,4 +365,6 @@ av_cold void ff_hpeldsp_init(HpelDSPContext *c, int flags) · 364 unmodified lines
365	365	ff_hpeldsp_init_ppc(c, flags);
366	366	if (ARCH_X86)
367	367	ff_hpeldsp_init_x86(c, flags);
	368	+ if (ARCH_MIPS)
	369	+ ff_hpeldsp_init_mips(c, flags);
368	370	}

libavcodec/hpeldsp.h+1−0

@@ -99,5 +99,6 @@ void ff_hpeldsp_init_alpha(HpelDSPContext *c, int flags); · 98 unmodified lines
99	99	void ff_hpeldsp_init_arm(HpelDSPContext *c, int flags);
100	100	void ff_hpeldsp_init_ppc(HpelDSPContext *c, int flags);
101	101	void ff_hpeldsp_init_x86(HpelDSPContext *c, int flags);
	102	+void ff_hpeldsp_init_mips(HpelDSPContext *c, int flags);
102	103
103	104	#endif /* AVCODEC_HPELDSP_H */

libavcodec/mips/Makefile+2−0

@@ -26,6 +26,7 @@ OBJS-$(CONFIG_H264CHROMA) += mips/h264chroma_init_mips.o · 25 unmodified lines
26	26	OBJS-$(CONFIG_H264PRED) += mips/h264pred_init_mips.o
27	27	OBJS-$(CONFIG_H263DSP) += mips/h263dsp_init_mips.o
28	28	OBJS-$(CONFIG_QPELDSP) += mips/qpeldsp_init_mips.o
	29	+OBJS-$(CONFIG_HPELDSP) += mips/hpeldsp_init_mips.o
29	30	MSA-OBJS-$(CONFIG_HEVC_DECODER) += mips/hevcdsp_msa.o \
30	31	mips/hevc_mc_uni_msa.o \
31	32	mips/hevc_mc_uniw_msa.o \
@@ -41,5 +42,6 @@ MSA-OBJS-$(CONFIG_H264CHROMA) += mips/h264chroma_msa.o · 9 unmodified lines
41	42	MSA-OBJS-$(CONFIG_H264PRED) += mips/h264pred_msa.o
42	43	MSA-OBJS-$(CONFIG_H263DSP) += mips/h263dsp_msa.o
43	44	MSA-OBJS-$(CONFIG_QPELDSP) += mips/qpeldsp_msa.o
	45	+MSA-OBJS-$(CONFIG_HPELDSP) += mips/hpeldsp_msa.o
44	46	LOONGSON3-OBJS-$(CONFIG_H264DSP) += mips/h264dsp_mmi.o
45	47	LOONGSON3-OBJS-$(CONFIG_H264CHROMA) += mips/h264chroma_mmi.o

libavcodec/mips/hpeldsp_init_mips.c+73−0

@@ -0,0 +1,73 @@
	1	+/*
	2	+ * Copyright (c) 2015 Parag Salasakar (Parag.Salasakar@imgtec.com)
	3	+ *
	4	+ * This file is part of FFmpeg.
	5	+ *
	6	+ * FFmpeg is free software; you can redistribute it and/or
	7	+ * modify it under the terms of the GNU Lesser General Public
	8	+ * License as published by the Free Software Foundation; either
	9	+ * version 2.1 of the License, or (at your option) any later version.
	10	+ *
	11	+ * FFmpeg is distributed in the hope that it will be useful,
	12	+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
	13	+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	14	+ * Lesser General Public License for more details.
	15	+ *
	16	+ * You should have received a copy of the GNU Lesser General Public
	17	+ * License along with FFmpeg; if not, write to the Free Software
	18	+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
	19	+ */
	20	+
	21	+#include "../hpeldsp.h"
	22	+#include "libavcodec/mips/hpeldsp_mips.h"
	23	+
	24	+#if HAVE_MSA
	25	+static void ff_hpeldsp_init_msa(HpelDSPContext *c, int flags)
	26	+{
	27	+ c->put_pixels_tab[0][0] = ff_put_pixels16_msa;
	28	+ c->put_pixels_tab[0][1] = ff_put_pixels16_x2_msa;
	29	+ c->put_pixels_tab[0][2] = ff_put_pixels16_y2_msa;
	30	+ c->put_pixels_tab[0][3] = ff_put_pixels16_xy2_msa;
	31	+
	32	+ c->put_pixels_tab[1][0] = ff_put_pixels8_msa;
	33	+ c->put_pixels_tab[1][1] = ff_put_pixels8_x2_msa;
	34	+ c->put_pixels_tab[1][2] = ff_put_pixels8_y2_msa;
	35	+ c->put_pixels_tab[1][3] = ff_put_pixels8_xy2_msa;
	36	+
	37	+ c->put_pixels_tab[2][1] = ff_put_pixels4_x2_msa;
	38	+ c->put_pixels_tab[2][2] = ff_put_pixels4_y2_msa;
	39	+ c->put_pixels_tab[2][3] = ff_put_pixels4_xy2_msa;
	40	+
	41	+ c->put_no_rnd_pixels_tab[0][0] = ff_put_pixels16_msa;
	42	+ c->put_no_rnd_pixels_tab[0][1] = ff_put_no_rnd_pixels16_x2_msa;
	43	+ c->put_no_rnd_pixels_tab[0][2] = ff_put_no_rnd_pixels16_y2_msa;
	44	+ c->put_no_rnd_pixels_tab[0][3] = ff_put_no_rnd_pixels16_xy2_msa;
	45	+
	46	+ c->put_no_rnd_pixels_tab[1][0] = ff_put_pixels8_msa;
	47	+ c->put_no_rnd_pixels_tab[1][1] = ff_put_no_rnd_pixels8_x2_msa;
	48	+ c->put_no_rnd_pixels_tab[1][2] = ff_put_no_rnd_pixels8_y2_msa;
	49	+ c->put_no_rnd_pixels_tab[1][3] = ff_put_no_rnd_pixels8_xy2_msa;
	50	+
	51	+ c->avg_pixels_tab[0][0] = ff_avg_pixels16_msa;
	52	+ c->avg_pixels_tab[0][1] = ff_avg_pixels16_x2_msa;
	53	+ c->avg_pixels_tab[0][2] = ff_avg_pixels16_y2_msa;
	54	+ c->avg_pixels_tab[0][3] = ff_avg_pixels16_xy2_msa;
	55	+
	56	+ c->avg_pixels_tab[1][0] = ff_avg_pixels8_msa;
	57	+ c->avg_pixels_tab[1][1] = ff_avg_pixels8_x2_msa;
	58	+ c->avg_pixels_tab[1][2] = ff_avg_pixels8_y2_msa;
	59	+ c->avg_pixels_tab[1][3] = ff_avg_pixels8_xy2_msa;
	60	+
	61	+ c->avg_pixels_tab[2][0] = ff_avg_pixels4_msa;
	62	+ c->avg_pixels_tab[2][1] = ff_avg_pixels4_x2_msa;
	63	+ c->avg_pixels_tab[2][2] = ff_avg_pixels4_y2_msa;
	64	+ c->avg_pixels_tab[2][3] = ff_avg_pixels4_xy2_msa;
	65	+}
	66	+#endif // #if HAVE_MSA
	67	+
	68	+void ff_hpeldsp_init_mips(HpelDSPContext *c, int flags)
	69	+{
	70	+#if HAVE_MSA
	71	+ ff_hpeldsp_init_msa(c, flags);
	72	+#endif // #if HAVE_MSA
	73	+}

libavcodec/mips/hpeldsp_mips.h+87−0

@@ -0,0 +1,87 @@
	1	+/*
	2	+ * Copyright (c) 2015 Parag Salasakar (Parag.Salasakar@imgtec.com)
	3	+ *
	4	+ * This file is part of FFmpeg.
	5	+ *
	6	+ * FFmpeg is free software; you can redistribute it and/or
	7	+ * modify it under the terms of the GNU Lesser General Public
	8	+ * License as published by the Free Software Foundation; either
	9	+ * version 2.1 of the License, or (at your option) any later version.
	10	+ *
	11	+ * FFmpeg is distributed in the hope that it will be useful,
	12	+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
	13	+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	14	+ * Lesser General Public License for more details.
	15	+ *
	16	+ * You should have received a copy of the GNU Lesser General Public
	17	+ * License along with FFmpeg; if not, write to the Free Software
	18	+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
	19	+ */
	20	+
	21	+#ifndef AVCODEC_MIPS_HPELDSP_MIPS_H
	22	+#define AVCODEC_MIPS_HPELDSP_MIPS_H
	23	+
	24	+#include "libavcodec/bit_depth_template.c"
	25	+
	26	+void ff_put_pixels16_msa(uint8_t block, const uint8_t pixels,
	27	+ ptrdiff_t line_size, int32_t h);
	28	+void ff_put_pixels16_x2_msa(uint8_t block, const uint8_t pixels,
	29	+ ptrdiff_t line_size, int32_t h);
	30	+void ff_put_pixels16_y2_msa(uint8_t block, const uint8_t pixels,
	31	+ ptrdiff_t line_size, int32_t h);
	32	+void ff_put_pixels16_xy2_msa(uint8_t block, const uint8_t pixels,
	33	+ ptrdiff_t line_size, int32_t h);
	34	+void ff_put_pixels8_msa(uint8_t block, const uint8_t pixels,
	35	+ ptrdiff_t line_size, int32_t h);
	36	+void ff_put_pixels8_x2_msa(uint8_t block, const uint8_t pixels,
	37	+ ptrdiff_t line_size, int32_t h);
	38	+void ff_put_pixels8_y2_msa(uint8_t block, const uint8_t pixels,
	39	+ ptrdiff_t line_size, int32_t h);
	40	+void ff_put_pixels8_xy2_msa(uint8_t block, const uint8_t pixels,
	41	+ ptrdiff_t line_size, int32_t h);
	42	+void ff_put_pixels4_msa(uint8_t block, const uint8_t pixels,
	43	+ ptrdiff_t line_size, int32_t h);
	44	+void ff_put_pixels4_x2_msa(uint8_t block, const uint8_t pixels,
	45	+ ptrdiff_t line_size, int32_t h);
	46	+void ff_put_pixels4_y2_msa(uint8_t block, const uint8_t pixels,
	47	+ ptrdiff_t line_size, int32_t h);
	48	+void ff_put_pixels4_xy2_msa(uint8_t block, const uint8_t pixels,
	49	+ ptrdiff_t line_size, int32_t h);
	50	+void ff_put_no_rnd_pixels16_x2_msa(uint8_t block, const uint8_t pixels,
	51	+ ptrdiff_t line_size, int32_t h);
	52	+void ff_put_no_rnd_pixels16_y2_msa(uint8_t block, const uint8_t pixels,
	53	+ ptrdiff_t line_size, int32_t h);
	54	+void ff_put_no_rnd_pixels16_xy2_msa(uint8_t block, const uint8_t pixels,
	55	+ ptrdiff_t line_size, int32_t h);
	56	+void ff_put_no_rnd_pixels8_x2_msa(uint8_t block, const uint8_t pixels,
	57	+ ptrdiff_t line_size, int32_t h);
	58	+void ff_put_no_rnd_pixels8_y2_msa(uint8_t block, const uint8_t pixels,
	59	+ ptrdiff_t line_size, int32_t h);
	60	+void ff_put_no_rnd_pixels8_xy2_msa(uint8_t block, const uint8_t pixels,
	61	+ ptrdiff_t line_size, int32_t h);
	62	+void ff_avg_pixels16_msa(uint8_t block, const uint8_t pixels,
	63	+ ptrdiff_t line_size, int32_t h);
	64	+void ff_avg_pixels16_x2_msa(uint8_t block, const uint8_t pixels,
	65	+ ptrdiff_t line_size, int32_t h);
	66	+void ff_avg_pixels16_y2_msa(uint8_t block, const uint8_t pixels,
	67	+ ptrdiff_t line_size, int32_t h);
	68	+void ff_avg_pixels16_xy2_msa(uint8_t block, const uint8_t pixels,
	69	+ ptrdiff_t line_size, int32_t h);
	70	+void ff_avg_pixels8_msa(uint8_t block, const uint8_t pixels,
	71	+ ptrdiff_t line_size, int32_t h);
	72	+void ff_avg_pixels8_x2_msa(uint8_t block, const uint8_t pixels,
	73	+ ptrdiff_t line_size, int32_t h);
	74	+void ff_avg_pixels8_y2_msa(uint8_t block, const uint8_t pixels,
	75	+ ptrdiff_t line_size, int32_t h);
	76	+void ff_avg_pixels8_xy2_msa(uint8_t block, const uint8_t pixels,
	77	+ ptrdiff_t line_size, int32_t h);
	78	+void ff_avg_pixels4_msa(uint8_t block, const uint8_t pixels,
	79	+ ptrdiff_t line_size, int32_t h);
	80	+void ff_avg_pixels4_x2_msa(uint8_t block, const uint8_t pixels,
	81	+ ptrdiff_t line_size, int32_t h);
	82	+void ff_avg_pixels4_y2_msa(uint8_t block, const uint8_t pixels,
	83	+ ptrdiff_t line_size, int32_t h);
	84	+void ff_avg_pixels4_xy2_msa(uint8_t block, const uint8_t pixels,
	85	+ ptrdiff_t line_size, int32_t h);
	86	+
	87	+#endif // #ifndef AVCODEC_MIPS_HPELDSP_MIPS_H

libavcodec/mips/hpeldsp_msa.c+1498−0

@@ -0,0 +1,1498 @@
	1	+/*
	2	+ * Copyright (c) 2015 Parag Salasakar (Parag.Salasakar@imgtec.com)
	3	+ *
	4	+ * This file is part of FFmpeg.
	5	+ *
	6	+ * FFmpeg is free software; you can redistribute it and/or
	7	+ * modify it under the terms of the GNU Lesser General Public
	8	+ * License as published by the Free Software Foundation; either
	9	+ * version 2.1 of the License, or (at your option) any later version.
	10	+ *
	11	+ * FFmpeg is distributed in the hope that it will be useful,
	12	+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
	13	+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	14	+ * Lesser General Public License for more details.
	15	+ *
	16	+ * You should have received a copy of the GNU Lesser General Public
	17	+ * License along with FFmpeg; if not, write to the Free Software
	18	+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
	19	+ */
	20	+
	21	+#include "libavutil/mips/generic_macros_msa.h"
	22	+#include "libavcodec/mips/hpeldsp_mips.h"
	23	+
	24	+#define PCKEV_AVG_ST_UB(in0, in1, dst, pdst) \
	25	+{ \
	26	+ v16u8 tmp_m; \
	27	+ \
	28	+ tmp_m = (v16u8) __msa_pckev_b((v16i8) in0, (v16i8) in1); \
	29	+ tmp_m = __msa_aver_u_b(tmp_m, (v16u8) dst); \
	30	+ ST_UB(tmp_m, (pdst)); \
	31	+}
	32	+
	33	+#define PCKEV_ST_SB4(in0, in1, in2, in3, in4, in5, in6, in7, pdst, stride) \
	34	+{ \
	35	+ v16i8 tmp0_m, tmp1_m, tmp2_m, tmp3_m; \
	36	+ uint8_t pdst_m = (uint8_t ) (pdst); \
	37	+ \
	38	+ PCKEV_B4_SB(in0, in1, in2, in3, in4, in5, in6, in7, \
	39	+ tmp0_m, tmp1_m, tmp2_m, tmp3_m); \
	40	+ ST_SB4(tmp0_m, tmp1_m, tmp2_m, tmp3_m, pdst_m, stride); \
	41	+}
	42	+
	43	+#define PCKEV_AVG_ST8x4_UB(in1, dst0, in2, dst1, in3, dst2, in4, dst3, \
	44	+ pdst, stride) \
	45	+{ \
	46	+ v16u8 tmp0_m, tmp1_m, tmp2_m, tmp3_m; \
	47	+ uint8_t pdst_m = (uint8_t ) (pdst); \
	48	+ \
	49	+ PCKEV_B2_UB(in2, in1, in4, in3, tmp0_m, tmp1_m); \
	50	+ PCKEV_D2_UB(dst1, dst0, dst3, dst2, tmp2_m, tmp3_m); \
	51	+ AVER_UB2_UB(tmp0_m, tmp2_m, tmp1_m, tmp3_m, tmp0_m, tmp1_m); \
	52	+ ST8x4_UB(tmp0_m, tmp1_m, pdst_m, stride); \
	53	+}
	54	+
	55	+static void common_hz_bil_4w_msa(const uint8_t *src, int32_t src_stride,
	56	+ uint8_t *dst, int32_t dst_stride,
	57	+ uint8_t height)
	58	+{
	59	+ uint8_t loop_cnt;
	60	+ uint32_t out0, out1;
	61	+ v16u8 src0, src1, src0_sld1, src1_sld1, res0, res1;
	62	+
	63	+ for (loop_cnt = (height >> 1); loop_cnt--;) {
	64	+ LD_UB2(src, src_stride, src0, src1);
	65	+ src += (2 * src_stride);
	66	+
	67	+ SLDI_B2_0_UB(src0, src1, src0_sld1, src1_sld1, 1);
	68	+ AVER_UB2_UB(src0_sld1, src0, src1_sld1, src1, res0, res1);
	69	+
	70	+ out0 = __msa_copy_u_w((v4i32) res0, 0);
	71	+ out1 = __msa_copy_u_w((v4i32) res1, 0);
	72	+ SW(out0, dst);
	73	+ dst += dst_stride;
	74	+ SW(out1, dst);
	75	+ dst += dst_stride;
	76	+ }
	77	+}
	78	+
	79	+static void common_hz_bil_8w_msa(const uint8_t *src, int32_t src_stride,
	80	+ uint8_t *dst, int32_t dst_stride,
	81	+ uint8_t height)
	82	+{
	83	+ uint8_t loop_cnt;
	84	+ v16i8 src0, src1, src2, src3, src0_sld1, src1_sld1, src2_sld1, src3_sld1;
	85	+
	86	+ for (loop_cnt = (height >> 2); loop_cnt--;) {
	87	+ LD_SB4(src, src_stride, src0, src1, src2, src3);
	88	+ src += (4 * src_stride);
	89	+
	90	+ SLDI_B4_0_SB(src0, src1, src2, src3,
	91	+ src0_sld1, src1_sld1, src2_sld1, src3_sld1, 1);
	92	+ AVER_ST8x4_UB(src0, src0_sld1, src1, src1_sld1,
	93	+ src2, src2_sld1, src3, src3_sld1, dst, dst_stride);
	94	+ dst += (4 * dst_stride);
	95	+ }
	96	+}
	97	+
	98	+static void common_hz_bil_16w_msa(const uint8_t *src, int32_t src_stride,
	99	+ uint8_t *dst, int32_t dst_stride,
	100	+ uint8_t height)
	101	+{
	102	+ uint8_t loop_cnt;
	103	+ v16u8 src0, src1, src2, src3, src4, src5, src6, src7;
	104	+ v16u8 src8, src9, src10, src11, src12, src13, src14, src15;
	105	+
	106	+ for (loop_cnt = (height >> 3); loop_cnt--;) {
	107	+ LD_UB8(src, src_stride, src0, src1, src2, src3, src4, src5, src6, src7);
	108	+ LD_UB8((src + 1), src_stride,
	109	+ src8, src9, src10, src11, src12, src13, src14, src15);
	110	+ src += (8 * src_stride);
	111	+
	112	+ AVER_ST16x4_UB(src0, src8, src1, src9, src2, src10, src3, src11,
	113	+ dst, dst_stride);
	114	+ dst += (4 * dst_stride);
	115	+
	116	+ AVER_ST16x4_UB(src4, src12, src5, src13, src6, src14, src7, src15,
	117	+ dst, dst_stride);
	118	+ dst += (4 * dst_stride);
	119	+ }
	120	+}
	121	+
	122	+static void common_hz_bil_no_rnd_8x8_msa(const uint8_t *src, int32_t src_stride,
	123	+ uint8_t *dst, int32_t dst_stride)
	124	+{
	125	+ v16i8 src0, src1, src2, src3, src4, src5, src6, src7;
	126	+ v16i8 src0_sld1, src1_sld1, src2_sld1, src3_sld1;
	127	+ v16i8 src4_sld1, src5_sld1, src6_sld1, src7_sld1;
	128	+
	129	+ LD_SB8(src, src_stride, src0, src1, src2, src3, src4, src5, src6, src7);
	130	+ src += (8 * src_stride);
	131	+
	132	+ SLDI_B4_0_SB(src0, src1, src2, src3,
	133	+ src0_sld1, src1_sld1, src2_sld1, src3_sld1, 1);
	134	+ SLDI_B4_0_SB(src4, src5, src6, src7,
	135	+ src4_sld1, src5_sld1, src6_sld1, src7_sld1, 1);
	136	+
	137	+ AVE_ST8x4_UB(src0, src0_sld1, src1, src1_sld1,
	138	+ src2, src2_sld1, src3, src3_sld1, dst, dst_stride);
	139	+ dst += (4 * dst_stride);
	140	+ AVE_ST8x4_UB(src4, src4_sld1, src5, src5_sld1,
	141	+ src6, src6_sld1, src7, src7_sld1, dst, dst_stride);
	142	+}
	143	+
	144	+static void common_hz_bil_no_rnd_4x8_msa(const uint8_t *src, int32_t src_stride,
	145	+ uint8_t *dst, int32_t dst_stride)
	146	+{
	147	+ v16i8 src0, src1, src2, src3, src0_sld1, src1_sld1, src2_sld1, src3_sld1;
	148	+
	149	+ LD_SB4(src, src_stride, src0, src1, src2, src3);
	150	+ SLDI_B4_0_SB(src0, src1, src2, src3,
	151	+ src0_sld1, src1_sld1, src2_sld1, src3_sld1, 1);
	152	+ AVE_ST8x4_UB(src0, src0_sld1, src1, src1_sld1,
	153	+ src2, src2_sld1, src3, src3_sld1, dst, dst_stride);
	154	+}
	155	+
	156	+static void common_hz_bil_no_rnd_16x16_msa(const uint8_t *src,
	157	+ int32_t src_stride,
	158	+ uint8_t *dst, int32_t dst_stride)
	159	+{
	160	+ v16u8 src0, src1, src2, src3, src4, src5, src6, src7, src8;
	161	+ v16u8 src9, src10, src11, src12, src13, src14, src15;
	162	+
	163	+ LD_UB8(src, src_stride, src0, src1, src2, src3, src4, src5, src6, src7);
	164	+ LD_UB8((src + 1), src_stride,
	165	+ src8, src9, src10, src11, src12, src13, src14, src15);
	166	+ src += (8 * src_stride);
	167	+
	168	+ AVE_ST16x4_UB(src0, src8, src1, src9, src2, src10, src3, src11,
	169	+ dst, dst_stride);
	170	+ dst += (4 * dst_stride);
	171	+
	172	+ LD_UB4(src, src_stride, src0, src1, src2, src3);
	173	+ LD_UB4((src + 1), src_stride, src8, src9, src10, src11);
	174	+ src += (4 * src_stride);
	175	+
	176	+ AVE_ST16x4_UB(src4, src12, src5, src13, src6, src14, src7, src15,
	177	+ dst, dst_stride);
	178	+ dst += (4 * dst_stride);
	179	+
	180	+ LD_UB4(src, src_stride, src4, src5, src6, src7);
	181	+ LD_UB4((src + 1), src_stride, src12, src13, src14, src15);
	182	+ src += (4 * src_stride);
	183	+
	184	+ AVE_ST16x4_UB(src0, src8, src1, src9, src2, src10, src3, src11,
	185	+ dst, dst_stride);
	186	+ dst += (4 * dst_stride);
	187	+ AVE_ST16x4_UB(src4, src12, src5, src13, src6, src14, src7, src15,
	188	+ dst, dst_stride);
	189	+}
	190	+
	191	+static void common_hz_bil_no_rnd_8x16_msa(const uint8_t *src,
	192	+ int32_t src_stride,
	193	+ uint8_t *dst, int32_t dst_stride)
	194	+{
	195	+ v16u8 src0, src1, src2, src3, src4, src5, src6, src7, src8;
	196	+ v16u8 src9, src10, src11, src12, src13, src14, src15;
	197	+
	198	+ LD_UB8(src, src_stride, src0, src1, src2, src3, src4, src5, src6, src7);
	199	+ LD_UB8((src + 1), src_stride,
	200	+ src8, src9, src10, src11, src12, src13, src14, src15);
	201	+
	202	+ AVE_ST16x4_UB(src0, src8, src1, src9, src2, src10, src3, src11,
	203	+ dst, dst_stride);
	204	+ dst += (4 * dst_stride);
	205	+ AVE_ST16x4_UB(src4, src12, src5, src13, src6, src14, src7, src15,
	206	+ dst, dst_stride);
	207	+}
	208	+
	209	+static void common_hz_bil_and_aver_dst_4w_msa(const uint8_t *src,
	210	+ int32_t src_stride,
	211	+ uint8_t *dst, int32_t dst_stride,
	212	+ uint8_t height)
	213	+{
	214	+ uint8_t loop_cnt;
	215	+ uint32_t dst0, dst1, out0, out1;
	216	+ v16u8 src0, src1, src0_sld1, src1_sld1, res0, res1;
	217	+ v16u8 tmp0 = { 0 };
	218	+ v16u8 tmp1 = { 0 };
	219	+
	220	+ for (loop_cnt = (height >> 1); loop_cnt--;) {
	221	+ LD_UB2(src, src_stride, src0, src1);
	222	+ src += (2 * src_stride);
	223	+
	224	+ SLDI_B2_0_UB(src0, src1, src0_sld1, src1_sld1, 1);
	225	+
	226	+ dst0 = LW(dst);
	227	+ dst1 = LW(dst + dst_stride);
	228	+ tmp0 = (v16u8) __msa_insert_w((v4i32) tmp0, 0, dst0);
	229	+ tmp1 = (v16u8) __msa_insert_w((v4i32) tmp1, 0, dst1);
	230	+
	231	+ AVER_UB2_UB(src0_sld1, src0, src1_sld1, src1, res0, res1);
	232	+ AVER_UB2_UB(res0, tmp0, res1, tmp1, res0, res1);
	233	+
	234	+ out0 = __msa_copy_u_w((v4i32) res0, 0);
	235	+ out1 = __msa_copy_u_w((v4i32) res1, 0);
	236	+ SW(out0, dst);
	237	+ dst += dst_stride;
	238	+ SW(out1, dst);
	239	+ dst += dst_stride;
	240	+ }
	241	+}
	242	+
	243	+static void common_hz_bil_and_aver_dst_8w_msa(const uint8_t *src,
	244	+ int32_t src_stride,
	245	+ uint8_t *dst, int32_t dst_stride,
	246	+ uint8_t height)
	247	+{
	248	+ uint8_t loop_cnt;
	249	+ v16i8 src0, src1, src2, src3, src0_sld1, src1_sld1, src2_sld1, src3_sld1;
	250	+
	251	+ for (loop_cnt = (height >> 2); loop_cnt--;) {
	252	+ LD_SB4(src, src_stride, src0, src1, src2, src3);
	253	+ src += (4 * src_stride);
	254	+
	255	+ SLDI_B4_0_SB(src0, src1, src2, src3,
	256	+ src0_sld1, src1_sld1, src2_sld1, src3_sld1, 1);
	257	+
	258	+ AVER_DST_ST8x4_UB(src0, src0_sld1, src1, src1_sld1, src2, src2_sld1,
	259	+ src3, src3_sld1, dst, dst_stride);
	260	+ dst += (4 * dst_stride);
	261	+ }
	262	+}
	263	+
	264	+static void common_hz_bil_and_aver_dst_16w_msa(const uint8_t *src,
	265	+ int32_t src_stride,
	266	+ uint8_t *dst, int32_t dst_stride,
	267	+ uint8_t height)
	268	+{
	269	+ uint8_t loop_cnt;
	270	+ v16u8 src0, src1, src2, src3, src4, src5, src6, src7, src8;
	271	+ v16u8 src9, src10, src11, src12, src13, src14, src15;
	272	+
	273	+ for (loop_cnt = (height >> 3); loop_cnt--;) {
	274	+ LD_UB8(src, src_stride, src0, src1, src2, src3, src4, src5, src6, src7);
	275	+ LD_UB8((src + 1), src_stride,
	276	+ src8, src9, src10, src11, src12, src13, src14, src15);
	277	+ src += (8 * src_stride);
	278	+
	279	+ AVER_DST_ST16x4_UB(src0, src8, src1, src9, src2, src10, src3, src11,
	280	+ dst, dst_stride);
	281	+ dst += (4 * dst_stride);
	282	+ AVER_DST_ST16x4_UB(src4, src12, src5, src13, src6, src14, src7, src15,
	283	+ dst, dst_stride);
	284	+ dst += (4 * dst_stride);
	285	+ }
	286	+}
	287	+
	288	+static void common_vt_bil_4w_msa(const uint8_t *src, int32_t src_stride,
	289	+ uint8_t *dst, int32_t dst_stride,
	290	+ uint8_t height)
	291	+{
	292	+ uint8_t loop_cnt;
	293	+ uint32_t out0, out1;
	294	+ v16u8 src0, src1, src2, res0, res1;
	295	+
	296	+ src0 = LD_UB(src);
	297	+ src += src_stride;
	298	+
	299	+ for (loop_cnt = (height >> 1); loop_cnt--;) {
	300	+ LD_UB2(src, src_stride, src1, src2);
	301	+ src += (2 * src_stride);
	302	+
	303	+ AVER_UB2_UB(src0, src1, src1, src2, res0, res1);
	304	+
	305	+ out0 = __msa_copy_u_w((v4i32) res0, 0);
	306	+ out1 = __msa_copy_u_w((v4i32) res1, 0);
	307	+ SW(out0, dst);
	308	+ dst += dst_stride;
	309	+ SW(out1, dst);
	310	+ dst += dst_stride;
	311	+
	312	+ src0 = src2;
	313	+ }
	314	+}
	315	+
	316	+static void common_vt_bil_8w_msa(const uint8_t *src, int32_t src_stride,
	317	+ uint8_t *dst, int32_t dst_stride,
	318	+ uint8_t height)
	319	+{
	320	+ uint8_t loop_cnt;
	321	+ v16u8 src0, src1, src2, src3, src4;
	322	+
	323	+ src0 = LD_UB(src);
	324	+ src += src_stride;
	325	+
	326	+ for (loop_cnt = (height >> 2); loop_cnt--;) {
	327	+ LD_UB4(src, src_stride, src1, src2, src3, src4);
	328	+ src += (4 * src_stride);
	329	+
	330	+ AVER_ST8x4_UB(src0, src1, src1, src2, src2, src3, src3, src4,
	331	+ dst, dst_stride);
	332	+ dst += (4 * dst_stride);
	333	+
	334	+ src0 = src4;
	335	+ }
	336	+}
	337	+
	338	+static void common_vt_bil_16w_msa(const uint8_t *src, int32_t src_stride,
	339	+ uint8_t *dst, int32_t dst_stride,
	340	+ uint8_t height)
	341	+{
	342	+ uint8_t loop_cnt;
	343	+ v16u8 src0, src1, src2, src3, src4, src5, src6, src7, src8;
	344	+
	345	+ src0 = LD_UB(src);
	346	+ src += src_stride;
	347	+
	348	+ for (loop_cnt = (height >> 3); loop_cnt--;) {
	349	+ LD_UB8(src, src_stride, src1, src2, src3, src4, src5, src6, src7, src8);
	350	+ src += (8 * src_stride);
	351	+
	352	+ AVER_ST16x4_UB(src0, src1, src1, src2, src2, src3, src3, src4,
	353	+ dst, dst_stride);
	354	+ dst += (4 * dst_stride);
	355	+ AVER_ST16x4_UB(src4, src5, src5, src6, src6, src7, src7, src8,
	356	+ dst, dst_stride);
	357	+ dst += (4 * dst_stride);
	358	+
	359	+ src0 = src8;
	360	+ }
	361	+}
	362	+
	363	+static void common_vt_bil_no_rnd_8x8_msa(const uint8_t *src, int32_t src_stride,
	364	+ uint8_t *dst, int32_t dst_stride)
	365	+{
	366	+ v16u8 src0, src1, src2, src3, src4, src5, src6, src7, src8;
	367	+
	368	+ LD_UB8(src, src_stride, src0, src1, src2, src3, src4, src5, src6, src7);
	369	+ src += (8 * src_stride);
	370	+ src8 = LD_UB(src);
	371	+
	372	+ AVE_ST8x4_UB(src0, src1, src1, src2, src2, src3, src3, src4,
	373	+ dst, dst_stride);
	374	+ dst += (4 * dst_stride);
	375	+
	376	+ AVE_ST8x4_UB(src4, src5, src5, src6, src6, src7, src7, src8,
	377	+ dst, dst_stride);
	378	+}
	379	+
	380	+static void common_vt_bil_no_rnd_4x8_msa(const uint8_t *src, int32_t src_stride,
	381	+ uint8_t *dst, int32_t dst_stride)
	382	+{
	383	+ v16u8 src0, src1, src2, src3, src4;
	384	+
	385	+ LD_UB5(src, src_stride, src0, src1, src2, src3, src4);
	386	+ AVE_ST8x4_UB(src0, src1, src1, src2, src2, src3, src3, src4,
	387	+ dst, dst_stride);
	388	+}
	389	+
	390	+static void common_vt_bil_no_rnd_16x16_msa(const uint8_t *src,
	391	+ int32_t src_stride,
	392	+ uint8_t *dst, int32_t dst_stride)
	393	+{
	394	+ v16u8 src0, src1, src2, src3, src4, src5, src6, src7, src8;
	395	+ v16u8 src9, src10, src11, src12, src13, src14, src15, src16;
	396	+
	397	+ LD_UB8(src, src_stride, src0, src1, src2, src3, src4, src5, src6, src7);
	398	+ src += (8 * src_stride);
	399	+ LD_UB8(src, src_stride,
	400	+ src8, src9, src10, src11, src12, src13, src14, src15);
	401	+ src += (8 * src_stride);
	402	+ src16 = LD_UB(src);
	403	+
	404	+ AVE_ST16x4_UB(src0, src1, src1, src2, src2, src3, src3, src4,
	405	+ dst, dst_stride);
	406	+ dst += (4 * dst_stride);
	407	+ AVE_ST16x4_UB(src4, src5, src5, src6, src6, src7, src7, src8,
	408	+ dst, dst_stride);
	409	+ dst += (4 * dst_stride);
	410	+ AVE_ST16x4_UB(src8, src9, src9, src10, src10, src11, src11, src12,
	411	+ dst, dst_stride);
	412	+ dst += (4 * dst_stride);
	413	+ AVE_ST16x4_UB(src12, src13, src13, src14,
	414	+ src14, src15, src15, src16, dst, dst_stride);
	415	+}
	416	+
	417	+static void common_vt_bil_no_rnd_8x16_msa(const uint8_t *src,
	418	+ int32_t src_stride,
	419	+ uint8_t *dst, int32_t dst_stride)
	420	+{
	421	+ v16u8 src0, src1, src2, src3, src4, src5, src6, src7, src8;
	422	+
	423	+ LD_UB8(src, src_stride, src0, src1, src2, src3, src4, src5, src6, src7);
	424	+ src += (8 * src_stride);
	425	+ src8 = LD_UB(src);
	426	+
	427	+ AVE_ST16x4_UB(src0, src1, src1, src2, src2, src3, src3, src4,
	428	+ dst, dst_stride);
	429	+ dst += (4 * dst_stride);
	430	+ AVE_ST16x4_UB(src4, src5, src5, src6, src6, src7, src7, src8,
	431	+ dst, dst_stride);
	432	+}
	433	+
	434	+static void common_vt_bil_and_aver_dst_4w_msa(const uint8_t *src,
	435	+ int32_t src_stride,
	436	+ uint8_t *dst, int32_t dst_stride,
	437	+ uint8_t height)
	438	+{
	439	+ uint8_t loop_cnt;
	440	+ uint32_t out0, out1, dst0, dst1;
	441	+ v16u8 src0, src1, src2;
	442	+ v16u8 tmp0 = { 0 };
	443	+ v16u8 tmp1 = { 0 };
	444	+ v16u8 res0, res1;
	445	+
	446	+ src0 = LD_UB(src);
	447	+ src += src_stride;
	448	+
	449	+ for (loop_cnt = (height >> 1); loop_cnt--;) {
	450	+ LD_UB2(src, src_stride, src1, src2);
	451	+ src += (2 * src_stride);
	452	+ dst0 = LW(dst);
	453	+ dst1 = LW(dst + dst_stride);
	454	+ tmp0 = (v16u8) __msa_insert_w((v4i32) tmp0, 0, dst0);
	455	+ tmp1 = (v16u8) __msa_insert_w((v4i32) tmp1, 0, dst1);
	456	+ AVER_UB2_UB(src0, src1, src1, src2, res0, res1);
	457	+ AVER_UB2_UB(res0, tmp0, res1, tmp1, res0, res1);
	458	+ out0 = __msa_copy_u_w((v4i32) res0, 0);
	459	+ out1 = __msa_copy_u_w((v4i32) res1, 0);
	460	+ SW(out0, dst);
	461	+ dst += dst_stride;
	462	+ SW(out1, dst);
	463	+ dst += dst_stride;
	464	+ src0 = src2;
	465	+ }
	466	+}
	467	+
	468	+static void common_vt_bil_and_aver_dst_8w_msa(const uint8_t *src,
	469	+ int32_t src_stride,
	470	+ uint8_t *dst, int32_t dst_stride,
	471	+ uint8_t height)
	472	+{
	473	+ uint8_t loop_cnt;
	474	+ v16u8 src0, src1, src2, src3, src4;
	475	+
	476	+ src0 = LD_UB(src);
	477	+ src += src_stride;
	478	+
	479	+ for (loop_cnt = (height >> 2); loop_cnt--;) {
	480	+ LD_UB4(src, src_stride, src1, src2, src3, src4);
	481	+ src += (4 * src_stride);
	482	+
	483	+ AVER_DST_ST8x4_UB(src0, src1, src1, src2, src2, src3, src3, src4,
	484	+ dst, dst_stride);
	485	+ dst += (4 * dst_stride);
	486	+ src0 = src4;
	487	+ }
	488	+}
	489	+
	490	+static void common_vt_bil_and_aver_dst_16w_msa(const uint8_t *src,
	491	+ int32_t src_stride,
	492	+ uint8_t *dst, int32_t dst_stride,
	493	+ uint8_t height)
	494	+{
	495	+ uint8_t loop_cnt;
	496	+ v16u8 src0, src1, src2, src3, src4, src5, src6, src7, src8;
	497	+ v16u8 res0, res1, res2, res3, res4, res5, res6, res7;
	498	+ v16u8 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7;
	499	+
	500	+ src0 = LD_UB(src);
	501	+ src += src_stride;
	502	+
	503	+ for (loop_cnt = (height >> 3); loop_cnt--;) {
	504	+ LD_UB8(src, src_stride, src1, src2, src3, src4, src5, src6, src7, src8);
	505	+ src += (8 * src_stride);
	506	+ AVER_UB4_UB(src0, src1, src1, src2, src2, src3, src3, src4,
	507	+ res0, res1, res2, res3);
	508	+ AVER_UB4_UB(src4, src5, src5, src6, src6, src7, src7, src8,
	509	+ res4, res5, res6, res7);
	510	+
	511	+ LD_UB8(dst, dst_stride, dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7);
	512	+ AVER_UB4_UB(dst0, res0, dst1, res1, dst2, res2, dst3, res3,
	513	+ res0, res1, res2, res3);
	514	+ AVER_UB4_UB(dst4, res4, dst5, res5, dst6, res6, dst7, res7,
	515	+ res4, res5, res6, res7);
	516	+ ST_UB8(res0, res1, res2, res3, res4, res5, res6, res7, dst, dst_stride);
	517	+ dst += (8 * dst_stride);
	518	+
	519	+ src0 = src8;
	520	+ }
	521	+}
	522	+
	523	+static void common_hv_bil_4w_msa(const uint8_t *src, int32_t src_stride,
	524	+ uint8_t *dst, int32_t dst_stride,
	525	+ uint8_t height)
	526	+{
	527	+ uint8_t loop_cnt;
	528	+ uint32_t res0, res1;
	529	+ v16i8 src0, src1, src2, src0_sld1, src1_sld1, src2_sld1;
	530	+ v16u8 src0_r, src1_r, src2_r, res;
	531	+ v8u16 add0, add1, add2, sum0, sum1;
	532	+
	533	+ src0 = LD_SB(src);
	534	+ src += src_stride;
	535	+
	536	+ for (loop_cnt = (height >> 1); loop_cnt--;) {
	537	+ LD_SB2(src, src_stride, src1, src2);
	538	+ src += (2 * src_stride);
	539	+
	540	+ SLDI_B3_0_SB(src0, src1, src2, src0_sld1, src1_sld1, src2_sld1, 1);
	541	+ ILVR_B3_UB(src0_sld1, src0, src1_sld1, src1, src2_sld1, src2,
	542	+ src0_r, src1_r, src2_r);
	543	+ HADD_UB3_UH(src0_r, src1_r, src2_r, add0, add1, add2);
	544	+ ADD2(add0, add1, add1, add2, sum0, sum1);
	545	+ SRARI_H2_UH(sum0, sum1, 2);
	546	+ res = (v16u8) __msa_pckev_b((v16i8) sum1, (v16i8) sum0);
	547	+ res0 = __msa_copy_u_w((v4i32) res, 0);
	548	+ res1 = __msa_copy_u_w((v4i32) res, 2);
	549	+ SW(res0, dst);
	550	+ dst += dst_stride;
	551	+ SW(res1, dst);
	552	+ dst += dst_stride;
	553	+
	554	+ src0 = src2;
	555	+ }
	556	+}
	557	+
	558	+static void common_hv_bil_8w_msa(const uint8_t *src, int32_t src_stride,
	559	+ uint8_t *dst, int32_t dst_stride,
	560	+ uint8_t height)
	561	+{
	562	+ uint8_t loop_cnt;
	563	+ v16i8 src0, src1, src2, src3, src4;
	564	+ v16i8 src0_sld1, src1_sld1, src2_sld1, src3_sld1, src4_sld1;
	565	+ v16u8 src0_r, src1_r, src2_r, src3_r, src4_r;
	566	+ v8u16 add0, add1, add2, add3, add4;
	567	+ v8u16 sum0, sum1, sum2, sum3;
	568	+
	569	+ src0 = LD_SB(src);
	570	+ src += src_stride;
	571	+
	572	+ for (loop_cnt = (height >> 2); loop_cnt--;) {
	573	+ LD_SB4(src, src_stride, src1, src2, src3, src4);
	574	+ src += (4 * src_stride);
	575	+
	576	+ SLDI_B3_0_SB(src0, src1, src2, src0_sld1, src1_sld1, src2_sld1, 1);
	577	+ SLDI_B2_0_SB(src3, src4, src3_sld1, src4_sld1, 1);
	578	+ ILVR_B3_UB(src0_sld1, src0, src1_sld1, src1, src2_sld1, src2, src0_r,
	579	+ src1_r, src2_r);
	580	+ ILVR_B2_UB(src3_sld1, src3, src4_sld1, src4, src3_r, src4_r);
	581	+ HADD_UB3_UH(src0_r, src1_r, src2_r, add0, add1, add2);
	582	+ HADD_UB2_UH(src3_r, src4_r, add3, add4);
	583	+ ADD4(add0, add1, add1, add2, add2, add3, add3, add4,
	584	+ sum0, sum1, sum2, sum3);
	585	+ SRARI_H4_UH(sum0, sum1, sum2, sum3, 2);
	586	+ PCKEV_B2_SB(sum1, sum0, sum3, sum2, src0, src1);
	587	+ ST8x4_UB(src0, src1, dst, dst_stride);
	588	+ dst += (4 * dst_stride);
	589	+ src0 = src4;
	590	+ }
	591	+}
	592	+
	593	+static void common_hv_bil_16w_msa(const uint8_t *src, int32_t src_stride,
	594	+ uint8_t *dst, int32_t dst_stride,
	595	+ uint8_t height)
	596	+{
	597	+ uint8_t loop_cnt;
	598	+ v16u8 src0, src1, src2, src3, src4, src5, src6, src7, src8, src9;
	599	+ v16u8 src10, src11, src12, src13, src14, src15, src16, src17;
	600	+ v8u16 src0_r, src1_r, src2_r, src3_r, src4_r, src5_r, src6_r, src7_r;
	601	+ v8u16 src8_r, src0_l, src1_l, src2_l, src3_l, src4_l, src5_l, src6_l;
	602	+ v8u16 src7_l, src8_l;
	603	+ v8u16 sum0_r, sum1_r, sum2_r, sum3_r, sum4_r, sum5_r, sum6_r, sum7_r;
	604	+ v8u16 sum0_l, sum1_l, sum2_l, sum3_l, sum4_l, sum5_l, sum6_l, sum7_l;
	605	+
	606	+ for (loop_cnt = (height >> 3); loop_cnt--;) {
	607	+ LD_UB8(src, src_stride, src0, src1, src2, src3, src4, src5, src6, src7);
	608	+ LD_UB8((src + 1), src_stride,
	609	+ src9, src10, src11, src12, src13, src14, src15, src16);
	610	+ src += (8 * src_stride);
	611	+
	612	+ src8 = LD_UB(src);
	613	+ src17 = LD_UB(src + 1);
	614	+
	615	+ ILVRL_B2_UH(src9, src0, src0_r, src0_l);
	616	+ ILVRL_B2_UH(src10, src1, src1_r, src1_l);
	617	+ ILVRL_B2_UH(src11, src2, src2_r, src2_l);
	618	+ ILVRL_B2_UH(src12, src3, src3_r, src3_l);
	619	+ ILVRL_B2_UH(src13, src4, src4_r, src4_l);
	620	+ ILVRL_B2_UH(src14, src5, src5_r, src5_l);
	621	+ ILVRL_B2_UH(src15, src6, src6_r, src6_l);
	622	+ ILVRL_B2_UH(src16, src7, src7_r, src7_l);
	623	+ ILVRL_B2_UH(src17, src8, src8_r, src8_l);
	624	+ HADD_UB3_UH(src0_r, src1_r, src2_r, src0_r, src1_r, src2_r);
	625	+ HADD_UB3_UH(src3_r, src4_r, src5_r, src3_r, src4_r, src5_r);
	626	+ HADD_UB3_UH(src6_r, src7_r, src8_r, src6_r, src7_r, src8_r);
	627	+ HADD_UB3_UH(src0_l, src1_l, src2_l, src0_l, src1_l, src2_l);
	628	+ HADD_UB3_UH(src3_l, src4_l, src5_l, src3_l, src4_l, src5_l);
	629	+ HADD_UB3_UH(src6_l, src7_l, src8_l, src6_l, src7_l, src8_l);
	630	+ ADD4(src0_r, src1_r, src1_r, src2_r, src2_r, src3_r, src3_r, src4_r,
	631	+ sum0_r, sum1_r, sum2_r, sum3_r);
	632	+ ADD4(src4_r, src5_r, src5_r, src6_r, src6_r, src7_r, src7_r, src8_r,
	633	+ sum4_r, sum5_r, sum6_r, sum7_r);
	634	+ ADD4(src0_l, src1_l, src1_l, src2_l, src2_l, src3_l, src3_l, src4_l,
	635	+ sum0_l, sum1_l, sum2_l, sum3_l);
	636	+ ADD4(src4_l, src5_l, src5_l, src6_l, src6_l, src7_l, src7_l, src8_l,
	637	+ sum4_l, sum5_l, sum6_l, sum7_l);
	638	+ SRARI_H4_UH(sum0_r, sum1_r, sum2_r, sum3_r, 2);
	639	+ SRARI_H4_UH(sum4_r, sum5_r, sum6_r, sum7_r, 2);
	640	+ SRARI_H4_UH(sum0_l, sum1_l, sum2_l, sum3_l, 2);
	641	+ SRARI_H4_UH(sum4_l, sum5_l, sum6_l, sum7_l, 2);
	642	+ PCKEV_ST_SB4(sum0_l, sum0_r, sum1_l, sum1_r, sum2_l, sum2_r,
	643	+ sum3_l, sum3_r, dst, dst_stride);
	644	+ dst += (4 * dst_stride);
	645	+ PCKEV_ST_SB4(sum4_l, sum4_r, sum5_l, sum5_r, sum6_l, sum6_r,
	646	+ sum7_l, sum7_r, dst, dst_stride);
	647	+ dst += (4 * dst_stride);
	648	+ }
	649	+}
	650	+
	651	+static void common_hv_bil_no_rnd_8x8_msa(const uint8_t *src, int32_t src_stride,
	652	+ uint8_t *dst, int32_t dst_stride)
	653	+{
	654	+ v16u8 src0, src1, src2, src3, src4, src5, src6, src7, src8;
	655	+ v16u8 src0_sld1, src1_sld1, src2_sld1, src3_sld1;
	656	+ v16u8 src4_sld1, src5_sld1, src6_sld1, src7_sld1, src8_sld1;
	657	+ v8u16 src0_r, src1_r, src2_r, src3_r;
	658	+ v8u16 src4_r, src5_r, src6_r, src7_r, src8_r;
	659	+ v8u16 add0, add1, add2, add3, add4, add5, add6, add7, add8;
	660	+ v8u16 sum0, sum1, sum2, sum3, sum4, sum5, sum6, sum7;
	661	+ v16i8 out0, out1;
	662	+
	663	+ LD_UB8(src, src_stride, src0, src1, src2, src3, src4, src5, src6, src7);
	664	+ src += (8 * src_stride);
	665	+ src8 = LD_UB(src);
	666	+
	667	+ SLDI_B4_0_UB(src0, src1, src2, src3, src0_sld1, src1_sld1, src2_sld1,
	668	+ src3_sld1, 1);
	669	+ SLDI_B3_0_UB(src4, src5, src6, src4_sld1, src5_sld1, src6_sld1, 1);
	670	+ SLDI_B2_0_UB(src7, src8, src7_sld1, src8_sld1, 1);
	671	+ ILVR_B4_UH(src0_sld1, src0, src1_sld1, src1, src2_sld1, src2, src3_sld1,
	672	+ src3, src0_r, src1_r, src2_r, src3_r);
	673	+ ILVR_B3_UH(src4_sld1, src4, src5_sld1, src5, src6_sld1, src6, src4_r,
	674	+ src5_r, src6_r);
	675	+ ILVR_B2_UH(src7_sld1, src7, src8_sld1, src8, src7_r, src8_r);
	676	+ HADD_UB3_UH(src0_r, src1_r, src2_r, add0, add1, add2);
	677	+ HADD_UB3_UH(src3_r, src4_r, src5_r, add3, add4, add5);
	678	+ HADD_UB3_UH(src6_r, src7_r, src8_r, add6, add7, add8);
	679	+
	680	+ sum0 = add0 + add1 + 1;
	681	+ sum1 = add1 + add2 + 1;
	682	+ sum2 = add2 + add3 + 1;
	683	+ sum3 = add3 + add4 + 1;
	684	+ sum4 = add4 + add5 + 1;
	685	+ sum5 = add5 + add6 + 1;
	686	+ sum6 = add6 + add7 + 1;
	687	+ sum7 = add7 + add8 + 1;
	688	+
	689	+ SRA_4V(sum0, sum1, sum2, sum3, 2);
	690	+ SRA_4V(sum4, sum5, sum6, sum7, 2);
	691	+ PCKEV_B2_SB(sum1, sum0, sum3, sum2, out0, out1);
	692	+ ST8x4_UB(out0, out1, dst, dst_stride);
	693	+ PCKEV_B2_SB(sum5, sum4, sum7, sum6, out0, out1);
	694	+ ST8x4_UB(out0, out1, dst + 4 * dst_stride, dst_stride);
	695	+}
	696	+
	697	+static void common_hv_bil_no_rnd_4x8_msa(const uint8_t *src, int32_t src_stride,
	698	+ uint8_t *dst, int32_t dst_stride)
	699	+{
	700	+ v16i8 src0, src1, src2, src3, src4;
	701	+ v16i8 src0_sld1, src1_sld1, src2_sld1, src3_sld1, src4_sld1;
	702	+ v8u16 src0_r, src1_r, src2_r, src3_r, src4_r;
	703	+ v8u16 add0, add1, add2, add3, add4;
	704	+ v8u16 sum0, sum1, sum2, sum3;
	705	+ v16i8 out0, out1;
	706	+
	707	+ LD_SB4(src, src_stride, src0, src1, src2, src3);
	708	+ src += (4 * src_stride);
	709	+ src4 = LD_SB(src);
	710	+
	711	+ SLDI_B3_0_SB(src0, src1, src2, src0_sld1, src1_sld1, src2_sld1, 1);
	712	+ SLDI_B2_0_SB(src3, src4, src3_sld1, src4_sld1, 1);
	713	+ ILVR_B3_UH(src0_sld1, src0, src1_sld1, src1, src2_sld1, src2, src0_r,
	714	+ src1_r, src2_r);
	715	+ ILVR_B2_UH(src3_sld1, src3, src4_sld1, src4, src3_r, src4_r);
	716	+ HADD_UB3_UH(src0_r, src1_r, src2_r, add0, add1, add2);
	717	+ HADD_UB2_UH(src3_r, src4_r, add3, add4);
	718	+
	719	+ sum0 = add0 + add1 + 1;
	720	+ sum1 = add1 + add2 + 1;
	721	+ sum2 = add2 + add3 + 1;
	722	+ sum3 = add3 + add4 + 1;
	723	+
	724	+ SRA_4V(sum0, sum1, sum2, sum3, 2);
	725	+ PCKEV_B2_SB(sum1, sum0, sum3, sum2, out0, out1);
	726	+ ST8x4_UB(out0, out1, dst, dst_stride);
	727	+}
	728	+
	729	+static void common_hv_bil_no_rnd_16x16_msa(const uint8_t *src,
	730	+ int32_t src_stride,
	731	+ uint8_t *dst, int32_t dst_stride)
	732	+{
	733	+ v16u8 src0, src1, src2, src3, src4, src5, src6, src7, src8, src9;
	734	+ v16u8 src10, src11, src12, src13, src14, src15, src16, src17;
	735	+ v8u16 src0_r, src1_r, src2_r, src3_r, src4_r, src5_r, src6_r, src7_r;
	736	+ v8u16 src8_r, src0_l, src1_l, src2_l, src3_l, src4_l, src5_l, src6_l;
	737	+ v8u16 src7_l, src8_l;
	738	+ v8u16 sum0_r, sum1_r, sum2_r, sum3_r, sum4_r, sum5_r, sum6_r, sum7_r;
	739	+ v8u16 sum0_l, sum1_l, sum2_l, sum3_l, sum4_l, sum5_l, sum6_l, sum7_l;
	740	+
	741	+ LD_UB8(src, src_stride, src0, src1, src2, src3, src4, src5, src6, src7);
	742	+ LD_UB8((src + 1), src_stride,
	743	+ src9, src10, src11, src12, src13, src14, src15, src16);
	744	+ src += (8 * src_stride);
	745	+ src8 = LD_UB(src);
	746	+ src17 = LD_UB(src + 1);
	747	+
	748	+ ILVRL_B2_UH(src9, src0, src0_r, src0_l);
	749	+ ILVRL_B2_UH(src10, src1, src1_r, src1_l);
	750	+ ILVRL_B2_UH(src11, src2, src2_r, src2_l);
	751	+ ILVRL_B2_UH(src12, src3, src3_r, src3_l);
	752	+ ILVRL_B2_UH(src13, src4, src4_r, src4_l);
	753	+ ILVRL_B2_UH(src14, src5, src5_r, src5_l);
	754	+ ILVRL_B2_UH(src15, src6, src6_r, src6_l);
	755	+ ILVRL_B2_UH(src16, src7, src7_r, src7_l);
	756	+ ILVRL_B2_UH(src17, src8, src8_r, src8_l);
	757	+
	758	+ HADD_UB3_UH(src0_r, src1_r, src2_r, src0_r, src1_r, src2_r);
	759	+ HADD_UB3_UH(src3_r, src4_r, src5_r, src3_r, src4_r, src5_r);
	760	+ HADD_UB3_UH(src6_r, src7_r, src8_r, src6_r, src7_r, src8_r);
	761	+ HADD_UB3_UH(src0_l, src1_l, src2_l, src0_l, src1_l, src2_l);
	762	+ HADD_UB3_UH(src3_l, src4_l, src5_l, src3_l, src4_l, src5_l);
	763	+ HADD_UB3_UH(src6_l, src7_l, src8_l, src6_l, src7_l, src8_l);
	764	+
	765	+ sum0_r = src0_r + src1_r + 1;
	766	+ sum1_r = src1_r + src2_r + 1;
	767	+ sum2_r = src2_r + src3_r + 1;
	768	+ sum3_r = src3_r + src4_r + 1;
	769	+ sum4_r = src4_r + src5_r + 1;
	770	+ sum5_r = src5_r + src6_r + 1;
	771	+ sum6_r = src6_r + src7_r + 1;
	772	+ sum7_r = src7_r + src8_r + 1;
	773	+ sum0_l = src0_l + src1_l + 1;
	774	+ sum1_l = src1_l + src2_l + 1;
	775	+ sum2_l = src2_l + src3_l + 1;
	776	+ sum3_l = src3_l + src4_l + 1;
	777	+ sum4_l = src4_l + src5_l + 1;
	778	+ sum5_l = src5_l + src6_l + 1;
	779	+ sum6_l = src6_l + src7_l + 1;
	780	+ sum7_l = src7_l + src8_l + 1;
	781	+
	782	+ SRA_4V(sum0_r, sum1_r, sum2_r, sum3_r, 2);
	783	+ SRA_4V(sum4_r, sum5_r, sum6_r, sum7_r, 2);
	784	+ SRA_4V(sum0_l, sum1_l, sum2_l, sum3_l, 2);
	785	+ SRA_4V(sum4_l, sum5_l, sum6_l, sum7_l, 2);
	786	+ PCKEV_ST_SB4(sum0_l, sum0_r, sum1_l, sum1_r,
	787	+ sum2_l, sum2_r, sum3_l, sum3_r, dst, dst_stride);
	788	+ dst += (4 * dst_stride);
	789	+
	790	+ LD_UB8(src, src_stride, src0, src1, src2, src3, src4, src5, src6, src7);
	791	+ LD_UB8((src + 1), src_stride,
	792	+ src9, src10, src11, src12, src13, src14, src15, src16);
	793	+ src += (8 * src_stride);
	794	+ src8 = LD_UB(src);
	795	+ src17 = LD_UB(src + 1);
	796	+
	797	+ PCKEV_ST_SB4(sum4_l, sum4_r, sum5_l, sum5_r,
	798	+ sum6_l, sum6_r, sum7_l, sum7_r, dst, dst_stride);
	799	+ dst += (4 * dst_stride);
	800	+
	801	+ ILVRL_B2_UH(src9, src0, src0_r, src0_l);
	802	+ ILVRL_B2_UH(src10, src1, src1_r, src1_l);
	803	+ ILVRL_B2_UH(src11, src2, src2_r, src2_l);
	804	+ ILVRL_B2_UH(src12, src3, src3_r, src3_l);
	805	+ ILVRL_B2_UH(src13, src4, src4_r, src4_l);
	806	+ ILVRL_B2_UH(src14, src5, src5_r, src5_l);
	807	+ ILVRL_B2_UH(src15, src6, src6_r, src6_l);
	808	+ ILVRL_B2_UH(src16, src7, src7_r, src7_l);
	809	+ ILVRL_B2_UH(src17, src8, src8_r, src8_l);
	810	+
	811	+ HADD_UB3_UH(src0_r, src1_r, src2_r, src0_r, src1_r, src2_r);
	812	+ HADD_UB3_UH(src3_r, src4_r, src5_r, src3_r, src4_r, src5_r);
	813	+ HADD_UB3_UH(src6_r, src7_r, src8_r, src6_r, src7_r, src8_r);
	814	+ HADD_UB3_UH(src0_l, src1_l, src2_l, src0_l, src1_l, src2_l);
	815	+ HADD_UB3_UH(src3_l, src4_l, src5_l, src3_l, src4_l, src5_l);
	816	+ HADD_UB3_UH(src6_l, src7_l, src8_l, src6_l, src7_l, src8_l);
	817	+
	818	+ sum0_r = src0_r + src1_r + 1;
	819	+ sum1_r = src1_r + src2_r + 1;
	820	+ sum2_r = src2_r + src3_r + 1;
	821	+ sum3_r = src3_r + src4_r + 1;
	822	+ sum4_r = src4_r + src5_r + 1;
	823	+ sum5_r = src5_r + src6_r + 1;
	824	+ sum6_r = src6_r + src7_r + 1;
	825	+ sum7_r = src7_r + src8_r + 1;
	826	+ sum0_l = src0_l + src1_l + 1;
	827	+ sum1_l = src1_l + src2_l + 1;
	828	+ sum2_l = src2_l + src3_l + 1;
	829	+ sum3_l = src3_l + src4_l + 1;
	830	+ sum4_l = src4_l + src5_l + 1;
	831	+ sum5_l = src5_l + src6_l + 1;
	832	+ sum6_l = src6_l + src7_l + 1;
	833	+ sum7_l = src7_l + src8_l + 1;
	834	+
	835	+ SRA_4V(sum0_r, sum1_r, sum2_r, sum3_r, 2);
	836	+ SRA_4V(sum4_r, sum5_r, sum6_r, sum7_r, 2);
	837	+ SRA_4V(sum0_l, sum1_l, sum2_l, sum3_l, 2);
	838	+ SRA_4V(sum4_l, sum5_l, sum6_l, sum7_l, 2);
	839	+ PCKEV_ST_SB4(sum0_l, sum0_r, sum1_l, sum1_r,
	840	+ sum2_l, sum2_r, sum3_l, sum3_r, dst, dst_stride);
	841	+ dst += (4 * dst_stride);
	842	+ PCKEV_ST_SB4(sum4_l, sum4_r, sum5_l, sum5_r,
	843	+ sum6_l, sum6_r, sum7_l, sum7_r, dst, dst_stride);
	844	+}
	845	+
	846	+static void common_hv_bil_no_rnd_8x16_msa(const uint8_t *src,
	847	+ int32_t src_stride,
	848	+ uint8_t *dst, int32_t dst_stride)
	849	+{
	850	+ v16u8 src0, src1, src2, src3, src4, src5, src6, src7, src8, src9;
	851	+ v16u8 src10, src11, src12, src13, src14, src15, src16, src17;
	852	+ v8u16 src0_r, src1_r, src2_r, src3_r, src4_r, src5_r, src6_r, src7_r;
	853	+ v8u16 src8_r, src0_l, src1_l, src2_l, src3_l, src4_l, src5_l, src6_l;
	854	+ v8u16 src7_l, src8_l;
	855	+ v8u16 sum0_r, sum1_r, sum2_r, sum3_r, sum4_r, sum5_r, sum6_r, sum7_r;
	856	+ v8u16 sum0_l, sum1_l, sum2_l, sum3_l, sum4_l, sum5_l, sum6_l, sum7_l;
	857	+
	858	+ LD_UB8(src, src_stride, src0, src1, src2, src3, src4, src5, src6, src7);
	859	+ LD_UB8((src + 1), src_stride,
	860	+ src9, src10, src11, src12, src13, src14, src15, src16);
	861	+ src += (8 * src_stride);
	862	+ src8 = LD_UB(src);
	863	+ src17 = LD_UB(src + 1);
	864	+
	865	+ ILVRL_B2_UH(src9, src0, src0_r, src0_l);
	866	+ ILVRL_B2_UH(src10, src1, src1_r, src1_l);
	867	+ ILVRL_B2_UH(src11, src2, src2_r, src2_l);
	868	+ ILVRL_B2_UH(src12, src3, src3_r, src3_l);
	869	+ ILVRL_B2_UH(src13, src4, src4_r, src4_l);
	870	+ ILVRL_B2_UH(src14, src5, src5_r, src5_l);
	871	+ ILVRL_B2_UH(src15, src6, src6_r, src6_l);
	872	+ ILVRL_B2_UH(src16, src7, src7_r, src7_l);
	873	+ ILVRL_B2_UH(src17, src8, src8_r, src8_l);
	874	+
	875	+ HADD_UB3_UH(src0_r, src1_r, src2_r, src0_r, src1_r, src2_r);
	876	+ HADD_UB3_UH(src3_r, src4_r, src5_r, src3_r, src4_r, src5_r);
	877	+ HADD_UB3_UH(src6_r, src7_r, src8_r, src6_r, src7_r, src8_r);
	878	+ HADD_UB3_UH(src0_l, src1_l, src2_l, src0_l, src1_l, src2_l);
	879	+ HADD_UB3_UH(src3_l, src4_l, src5_l, src3_l, src4_l, src5_l);
	880	+ HADD_UB3_UH(src6_l, src7_l, src8_l, src6_l, src7_l, src8_l);
	881	+
	882	+ sum0_r = src0_r + src1_r + 1;
	883	+ sum1_r = src1_r + src2_r + 1;
	884	+ sum2_r = src2_r + src3_r + 1;
	885	+ sum3_r = src3_r + src4_r + 1;
	886	+ sum4_r = src4_r + src5_r + 1;
	887	+ sum5_r = src5_r + src6_r + 1;
	888	+ sum6_r = src6_r + src7_r + 1;
	889	+ sum7_r = src7_r + src8_r + 1;
	890	+ sum0_l = src0_l + src1_l + 1;
	891	+ sum1_l = src1_l + src2_l + 1;
	892	+ sum2_l = src2_l + src3_l + 1;
	893	+ sum3_l = src3_l + src4_l + 1;
	894	+ sum4_l = src4_l + src5_l + 1;
	895	+ sum5_l = src5_l + src6_l + 1;
	896	+ sum6_l = src6_l + src7_l + 1;
	897	+ sum7_l = src7_l + src8_l + 1;
	898	+
	899	+ SRA_4V(sum0_r, sum1_r, sum2_r, sum3_r, 2);
	900	+ SRA_4V(sum4_r, sum5_r, sum6_r, sum7_r, 2);
	901	+ SRA_4V(sum0_l, sum1_l, sum2_l, sum3_l, 2);
	902	+ SRA_4V(sum4_l, sum5_l, sum6_l, sum7_l, 2);
	903	+ PCKEV_ST_SB4(sum0_l, sum0_r, sum1_l, sum1_r,
	904	+ sum2_l, sum2_r, sum3_l, sum3_r, dst, dst_stride);
	905	+ dst += (4 * dst_stride);
	906	+ PCKEV_ST_SB4(sum4_l, sum4_r, sum5_l, sum5_r,
	907	+ sum6_l, sum6_r, sum7_l, sum7_r, dst, dst_stride);
	908	+}
	909	+
	910	+static void common_hv_bil_and_aver_dst_4w_msa(const uint8_t *src,
	911	+ int32_t src_stride,
	912	+ uint8_t *dst, int32_t dst_stride,
	913	+ uint8_t height)
	914	+{
	915	+ uint8_t loop_cnt;
	916	+ uint32_t out0, out1;
	917	+ v16i8 src0, src1, src2, src0_sld1, src1_sld1, src2_sld1;
	918	+ v16u8 src0_r, src1_r, src2_r;
	919	+ v8u16 add0, add1, add2, sum0, sum1;
	920	+ v16u8 dst0, dst1, res0, res1;
	921	+
	922	+ src0 = LD_SB(src);
	923	+ src += src_stride;
	924	+
	925	+ for (loop_cnt = (height >> 1); loop_cnt--;) {
	926	+ LD_SB2(src, src_stride, src1, src2);
	927	+ src += (2 * src_stride);
	928	+
	929	+ LD_UB2(dst, dst_stride, dst0, dst1);
	930	+ SLDI_B3_0_SB(src0, src1, src2, src0_sld1, src1_sld1, src2_sld1, 1);
	931	+ ILVR_B3_UB(src0_sld1, src0, src1_sld1, src1, src2_sld1, src2, src0_r,
	932	+ src1_r, src2_r);
	933	+ HADD_UB3_UH(src0_r, src1_r, src2_r, add0, add1, add2);
	934	+ ADD2(add0, add1, add1, add2, sum0, sum1);
	935	+ SRARI_H2_UH(sum0, sum1, 2);
	936	+ PCKEV_B2_UB(sum0, sum0, sum1, sum1, res0, res1);
	937	+ AVER_UB2_UB(dst0, res0, dst1, res1, res0, res1);
	938	+
	939	+ out0 = __msa_copy_u_w((v4i32) res0, 0);
	940	+ out1 = __msa_copy_u_w((v4i32) res1, 0);
	941	+ SW(out0, dst);
	942	+ dst += dst_stride;
	943	+ SW(out1, dst);
	944	+ dst += dst_stride;
	945	+
	946	+ src0 = src2;
	947	+ }
	948	+}
	949	+
	950	+static void common_hv_bil_and_aver_dst_8w_msa(const uint8_t *src,
	951	+ int32_t src_stride,
	952	+ uint8_t *dst, int32_t dst_stride,
	953	+ uint8_t height)
	954	+{
	955	+ uint8_t loop_cnt;
	956	+ v16i8 src0, src1, src2, src3, src4;
	957	+ v16i8 src0_sld1, src1_sld1, src2_sld1, src3_sld1, src4_sld1;
	958	+ v16u8 dst0, dst1, dst2, dst3;
	959	+ v16u8 src0_r, src1_r, src2_r, src3_r, src4_r;
	960	+ v8u16 add0, add1, add2, add3, add4;
	961	+ v8u16 sum0, sum1, sum2, sum3;
	962	+
	963	+ src0 = LD_SB(src);
	964	+ src += src_stride;
	965	+
	966	+ for (loop_cnt = (height >> 2); loop_cnt--;) {
	967	+ LD_SB4(src, src_stride, src1, src2, src3, src4);
	968	+ src += (4 * src_stride);
	969	+
	970	+ LD_UB4(dst, dst_stride, dst0, dst1, dst2, dst3);
	971	+ SLDI_B3_0_SB(src0, src1, src2, src0_sld1, src1_sld1, src2_sld1, 1);
	972	+ SLDI_B2_0_SB(src3, src4, src3_sld1, src4_sld1, 1);
	973	+ ILVR_B3_UB(src0_sld1, src0, src1_sld1, src1, src2_sld1, src2, src0_r,
	974	+ src1_r, src2_r);
	975	+ ILVR_B2_UB(src3_sld1, src3, src4_sld1, src4, src3_r, src4_r);
	976	+ HADD_UB3_UH(src0_r, src1_r, src2_r, add0, add1, add2);
	977	+ HADD_UB2_UH(src3_r, src4_r, add3, add4);
	978	+ ADD4(add0, add1, add1, add2, add2, add3, add3, add4,
	979	+ sum0, sum1, sum2, sum3);
	980	+ SRARI_H4_UH(sum0, sum1, sum2, sum3, 2);
	981	+ PCKEV_AVG_ST8x4_UB(sum0, dst0, sum1, dst1,
	982	+ sum2, dst2, sum3, dst3, dst, dst_stride);
	983	+ dst += (4 * dst_stride);
	984	+ src0 = src4;
	985	+ }
	986	+}
	987	+
	988	+static void common_hv_bil_and_aver_dst_16w_msa(const uint8_t *src,
	989	+ int32_t src_stride,
	990	+ uint8_t *dst, int32_t dst_stride,
	991	+ uint8_t height)
	992	+{
	993	+ uint8_t loop_cnt;
	994	+ v16u8 src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10;
	995	+ v16u8 src11, src12, src13, src14, src15, src16, src17;
	996	+ v16u8 src0_r, src1_r, src2_r, src3_r, src4_r, src5_r, src6_r, src7_r;
	997	+ v16u8 src8_r, src0_l, src1_l, src2_l, src3_l, src4_l, src5_l, src6_l;
	998	+ v16u8 src7_l, src8_l;
	999	+ v16u8 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7;
	1000	+ v8u16 sum0_r, sum1_r, sum2_r, sum3_r, sum4_r, sum5_r, sum6_r, sum7_r;
	1001	+ v8u16 sum0_l, sum1_l, sum2_l, sum3_l, sum4_l, sum5_l, sum6_l, sum7_l;
	1002	+ v8u16 add0, add1, add2, add3, add4, add5, add6, add7, add8;
	1003	+
	1004	+ for (loop_cnt = (height >> 3); loop_cnt--;) {
	1005	+ LD_UB8(src, src_stride, src0, src1, src2, src3, src4, src5, src6, src7);
	1006	+ LD_UB8((src + 1), src_stride,
	1007	+ src9, src10, src11, src12, src13, src14, src15, src16);
	1008	+ src += (8 * src_stride);
	1009	+
	1010	+ src8 = LD_UB(src);
	1011	+ src17 = LD_UB(src + 1);
	1012	+
	1013	+ ILVRL_B2_UB(src9, src0, src0_r, src0_l);
	1014	+ ILVRL_B2_UB(src10, src1, src1_r, src1_l);
	1015	+ ILVRL_B2_UB(src11, src2, src2_r, src2_l);
	1016	+ ILVRL_B2_UB(src12, src3, src3_r, src3_l);
	1017	+ ILVRL_B2_UB(src13, src4, src4_r, src4_l);
	1018	+ ILVRL_B2_UB(src14, src5, src5_r, src5_l);
	1019	+ ILVRL_B2_UB(src15, src6, src6_r, src6_l);
	1020	+ ILVRL_B2_UB(src16, src7, src7_r, src7_l);
	1021	+ ILVRL_B2_UB(src17, src8, src8_r, src8_l);
	1022	+ HADD_UB3_UH(src0_r, src1_r, src2_r, add0, add1, add2);
	1023	+ HADD_UB3_UH(src3_r, src4_r, src5_r, add3, add4, add5);
	1024	+ HADD_UB3_UH(src6_r, src7_r, src8_r, add6, add7, add8);
	1025	+ ADD4(add0, add1, add1, add2, add2, add3, add3, add4, sum0_r, sum1_r,
	1026	+ sum2_r, sum3_r);
	1027	+ ADD4(add4, add5, add5, add6, add6, add7, add7, add8, sum4_r, sum5_r,
	1028	+ sum6_r, sum7_r);
	1029	+ HADD_UB3_UH(src0_l, src1_l, src2_l, add0, add1, add2);
	1030	+ HADD_UB3_UH(src3_l, src4_l, src5_l, add3, add4, add5);
	1031	+ HADD_UB3_UH(src6_l, src7_l, src8_l, add6, add7, add8);
	1032	+ ADD4(add0, add1, add1, add2, add2, add3, add3, add4, sum0_l, sum1_l,
	1033	+ sum2_l, sum3_l);
	1034	+ ADD4(add4, add5, add5, add6, add6, add7, add7, add8, sum4_l, sum5_l,
	1035	+ sum6_l, sum7_l);
	1036	+ SRARI_H4_UH(sum0_r, sum1_r, sum2_r, sum3_r, 2);
	1037	+ SRARI_H4_UH(sum4_r, sum5_r, sum6_r, sum7_r, 2);
	1038	+ SRARI_H4_UH(sum0_l, sum1_l, sum2_l, sum3_l, 2);
	1039	+ SRARI_H4_UH(sum4_l, sum5_l, sum6_l, sum7_l, 2);
	1040	+ LD_UB8(dst, dst_stride, dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7);
	1041	+ PCKEV_AVG_ST_UB(sum0_l, sum0_r, dst0, dst);
	1042	+ dst += dst_stride;
	1043	+ PCKEV_AVG_ST_UB(sum1_l, sum1_r, dst1, dst);
	1044	+ dst += dst_stride;
	1045	+ PCKEV_AVG_ST_UB(sum2_l, sum2_r, dst2, dst);
	1046	+ dst += dst_stride;
	1047	+ PCKEV_AVG_ST_UB(sum3_l, sum3_r, dst3, dst);
	1048	+ dst += dst_stride;
	1049	+ PCKEV_AVG_ST_UB(sum4_l, sum4_r, dst4, dst);
	1050	+ dst += dst_stride;
	1051	+ PCKEV_AVG_ST_UB(sum5_l, sum5_r, dst5, dst);
	1052	+ dst += dst_stride;
	1053	+ PCKEV_AVG_ST_UB(sum6_l, sum6_r, dst6, dst);
	1054	+ dst += dst_stride;
	1055	+ PCKEV_AVG_ST_UB(sum7_l, sum7_r, dst7, dst);
	1056	+ dst += dst_stride;
	1057	+ }
	1058	+}
	1059	+
	1060	+static void copy_width8_msa(const uint8_t *src, int32_t src_stride,
	1061	+ uint8_t *dst, int32_t dst_stride,
	1062	+ int32_t height)
	1063	+{
	1064	+ int32_t cnt;
	1065	+ uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
	1066	+ v16u8 src0, src1, src2, src3, src4, src5, src6, src7;
	1067	+
	1068	+ if (0 == height % 12) {
	1069	+ for (cnt = (height / 12); cnt--;) {
	1070	+ LD_UB8(src, src_stride,
	1071	+ src0, src1, src2, src3, src4, src5, src6, src7);
	1072	+ src += (8 * src_stride);
	1073	+
	1074	+ out0 = __msa_copy_u_d((v2i64) src0, 0);
	1075	+ out1 = __msa_copy_u_d((v2i64) src1, 0);
	1076	+ out2 = __msa_copy_u_d((v2i64) src2, 0);
	1077	+ out3 = __msa_copy_u_d((v2i64) src3, 0);
	1078	+ out4 = __msa_copy_u_d((v2i64) src4, 0);
	1079	+ out5 = __msa_copy_u_d((v2i64) src5, 0);
	1080	+ out6 = __msa_copy_u_d((v2i64) src6, 0);
	1081	+ out7 = __msa_copy_u_d((v2i64) src7, 0);
	1082	+
	1083	+ SD4(out0, out1, out2, out3, dst, dst_stride);
	1084	+ dst += (4 * dst_stride);
	1085	+ SD4(out4, out5, out6, out7, dst, dst_stride);
	1086	+ dst += (4 * dst_stride);
	1087	+
	1088	+ LD_UB4(src, src_stride, src0, src1, src2, src3);
	1089	+ src += (4 * src_stride);
	1090	+
	1091	+ out0 = __msa_copy_u_d((v2i64) src0, 0);
	1092	+ out1 = __msa_copy_u_d((v2i64) src1, 0);
	1093	+ out2 = __msa_copy_u_d((v2i64) src2, 0);
	1094	+ out3 = __msa_copy_u_d((v2i64) src3, 0);
	1095	+
	1096	+ SD4(out0, out1, out2, out3, dst, dst_stride);
	1097	+ dst += (4 * dst_stride);
	1098	+ }
	1099	+ } else if (0 == height % 8) {
	1100	+ for (cnt = height >> 3; cnt--;) {
	1101	+ LD_UB8(src, src_stride,
	1102	+ src0, src1, src2, src3, src4, src5, src6, src7);
	1103	+ src += (8 * src_stride);
	1104	+
	1105	+ out0 = __msa_copy_u_d((v2i64) src0, 0);
	1106	+ out1 = __msa_copy_u_d((v2i64) src1, 0);
	1107	+ out2 = __msa_copy_u_d((v2i64) src2, 0);
	1108	+ out3 = __msa_copy_u_d((v2i64) src3, 0);
	1109	+ out4 = __msa_copy_u_d((v2i64) src4, 0);
	1110	+ out5 = __msa_copy_u_d((v2i64) src5, 0);
	1111	+ out6 = __msa_copy_u_d((v2i64) src6, 0);
	1112	+ out7 = __msa_copy_u_d((v2i64) src7, 0);
	1113	+
	1114	+ SD4(out0, out1, out2, out3, dst, dst_stride);
	1115	+ dst += (4 * dst_stride);
	1116	+ SD4(out4, out5, out6, out7, dst, dst_stride);
	1117	+ dst += (4 * dst_stride);
	1118	+ }
	1119	+ } else if (0 == height % 4) {
	1120	+ for (cnt = (height / 4); cnt--;) {
	1121	+ LD_UB4(src, src_stride, src0, src1, src2, src3);
	1122	+ src += (4 * src_stride);
	1123	+ out0 = __msa_copy_u_d((v2i64) src0, 0);
	1124	+ out1 = __msa_copy_u_d((v2i64) src1, 0);
	1125	+ out2 = __msa_copy_u_d((v2i64) src2, 0);
	1126	+ out3 = __msa_copy_u_d((v2i64) src3, 0);
	1127	+
	1128	+ SD4(out0, out1, out2, out3, dst, dst_stride);
	1129	+ dst += (4 * dst_stride);
	1130	+ }
	1131	+ } else if (0 == height % 2) {
	1132	+ for (cnt = (height / 2); cnt--;) {
	1133	+ LD_UB2(src, src_stride, src0, src1);
	1134	+ src += (2 * src_stride);
	1135	+ out0 = __msa_copy_u_d((v2i64) src0, 0);
	1136	+ out1 = __msa_copy_u_d((v2i64) src1, 0);
	1137	+
	1138	+ SD(out0, dst);
	1139	+ dst += dst_stride;
	1140	+ SD(out1, dst);
	1141	+ dst += dst_stride;
	1142	+ }
	1143	+ }
	1144	+}
	1145	+
	1146	+static void copy_16multx8mult_msa(const uint8_t *src, int32_t src_stride,
	1147	+ uint8_t *dst, int32_t dst_stride,
	1148	+ int32_t height, int32_t width)
	1149	+{
	1150	+ int32_t cnt, loop_cnt;
	1151	+ const uint8_t *src_tmp;
	1152	+ uint8_t *dst_tmp;
	1153	+ v16u8 src0, src1, src2, src3, src4, src5, src6, src7;
	1154	+
	1155	+ for (cnt = (width >> 4); cnt--;) {
	1156	+ src_tmp = src;
	1157	+ dst_tmp = dst;
	1158	+
	1159	+ for (loop_cnt = (height >> 3); loop_cnt--;) {
	1160	+ LD_UB8(src_tmp, src_stride,
	1161	+ src0, src1, src2, src3, src4, src5, src6, src7);
	1162	+ src_tmp += (8 * src_stride);
	1163	+
	1164	+ ST_UB8(src0, src1, src2, src3, src4, src5, src6, src7,
	1165	+ dst_tmp, dst_stride);
	1166	+ dst_tmp += (8 * dst_stride);
	1167	+ }
	1168	+
	1169	+ src += 16;
	1170	+ dst += 16;
	1171	+ }
	1172	+}
	1173	+
	1174	+static void copy_width16_msa(const uint8_t *src, int32_t src_stride,
	1175	+ uint8_t *dst, int32_t dst_stride,
	1176	+ int32_t height)
	1177	+{
	1178	+ int32_t cnt;
	1179	+ v16u8 src0, src1, src2, src3, src4, src5, src6, src7;
	1180	+
	1181	+ if (0 == height % 12) {
	1182	+ for (cnt = (height / 12); cnt--;) {
	1183	+ LD_UB8(src, src_stride,
	1184	+ src0, src1, src2, src3, src4, src5, src6, src7);
	1185	+ src += (8 * src_stride);
	1186	+ ST_UB8(src0, src1, src2, src3, src4, src5, src6, src7,
	1187	+ dst, dst_stride);
	1188	+ dst += (8 * dst_stride);
	1189	+
	1190	+ LD_UB4(src, src_stride, src0, src1, src2, src3);
	1191	+ src += (4 * src_stride);
	1192	+ ST_UB4(src0, src1, src2, src3, dst, dst_stride);
	1193	+ dst += (4 * dst_stride);
	1194	+ }
	1195	+ } else if (0 == height % 8) {
	1196	+ copy_16multx8mult_msa(src, src_stride, dst, dst_stride, height, 16);
	1197	+ } else if (0 == height % 4) {
	1198	+ for (cnt = (height >> 2); cnt--;) {
	1199	+ LD_UB4(src, src_stride, src0, src1, src2, src3);
	1200	+ src += (4 * src_stride);
	1201	+
	1202	+ ST_UB4(src0, src1, src2, src3, dst, dst_stride);
	1203	+ dst += (4 * dst_stride);
	1204	+ }
	1205	+ }
	1206	+}
	1207	+
	1208	+static void avg_width4_msa(const uint8_t *src, int32_t src_stride,
	1209	+ uint8_t *dst, int32_t dst_stride,
	1210	+ int32_t height)
	1211	+{
	1212	+ int32_t cnt;
	1213	+ uint32_t out0, out1, out2, out3;
	1214	+ v16u8 src0, src1, src2, src3;
	1215	+ v16u8 dst0, dst1, dst2, dst3;
	1216	+
	1217	+ if (0 == (height % 4)) {
	1218	+ for (cnt = (height / 4); cnt--;) {
	1219	+ LD_UB4(src, src_stride, src0, src1, src2, src3);
	1220	+ src += (4 * src_stride);
	1221	+
	1222	+ LD_UB4(dst, dst_stride, dst0, dst1, dst2, dst3);
	1223	+
	1224	+ AVER_UB4_UB(src0, dst0, src1, dst1, src2, dst2, src3, dst3,
	1225	+ dst0, dst1, dst2, dst3);
	1226	+
	1227	+ out0 = __msa_copy_u_w((v4i32) dst0, 0);
	1228	+ out1 = __msa_copy_u_w((v4i32) dst1, 0);
	1229	+ out2 = __msa_copy_u_w((v4i32) dst2, 0);
	1230	+ out3 = __msa_copy_u_w((v4i32) dst3, 0);
	1231	+ SW4(out0, out1, out2, out3, dst, dst_stride);
	1232	+ dst += (4 * dst_stride);
	1233	+ }
	1234	+ } else if (0 == (height % 2)) {
	1235	+ for (cnt = (height / 2); cnt--;) {
	1236	+ LD_UB2(src, src_stride, src0, src1);
	1237	+ src += (2 * src_stride);
	1238	+
	1239	+ LD_UB2(dst, dst_stride, dst0, dst1);
	1240	+
	1241	+ AVER_UB2_UB(src0, dst0, src1, dst1, dst0, dst1);
	1242	+
	1243	+ out0 = __msa_copy_u_w((v4i32) dst0, 0);
	1244	+ out1 = __msa_copy_u_w((v4i32) dst1, 0);
	1245	+ SW(out0, dst);
	1246	+ dst += dst_stride;
	1247	+ SW(out1, dst);
	1248	+ dst += dst_stride;
	1249	+ }
	1250	+ }
	1251	+}
	1252	+
	1253	+static void avg_width8_msa(const uint8_t *src, int32_t src_stride,
	1254	+ uint8_t *dst, int32_t dst_stride,
	1255	+ int32_t height)
	1256	+{
	1257	+ int32_t cnt;
	1258	+ uint64_t out0, out1, out2, out3;
	1259	+ v16u8 src0, src1, src2, src3;
	1260	+ v16u8 dst0, dst1, dst2, dst3;
	1261	+
	1262	+ for (cnt = (height / 4); cnt--;) {
	1263	+ LD_UB4(src, src_stride, src0, src1, src2, src3);
	1264	+ src += (4 * src_stride);
	1265	+ LD_UB4(dst, dst_stride, dst0, dst1, dst2, dst3);
	1266	+
	1267	+ AVER_UB4_UB(src0, dst0, src1, dst1, src2, dst2, src3, dst3,
	1268	+ dst0, dst1, dst2, dst3);
	1269	+
	1270	+ out0 = __msa_copy_u_d((v2i64) dst0, 0);
	1271	+ out1 = __msa_copy_u_d((v2i64) dst1, 0);
	1272	+ out2 = __msa_copy_u_d((v2i64) dst2, 0);
	1273	+ out3 = __msa_copy_u_d((v2i64) dst3, 0);
	1274	+ SD4(out0, out1, out2, out3, dst, dst_stride);
	1275	+ dst += (4 * dst_stride);
	1276	+ }
	1277	+}
	1278	+
	1279	+static void avg_width16_msa(const uint8_t *src, int32_t src_stride,
	1280	+ uint8_t *dst, int32_t dst_stride,
	1281	+ int32_t height)
	1282	+{
	1283	+ int32_t cnt;
	1284	+ v16u8 src0, src1, src2, src3, src4, src5, src6, src7;
	1285	+ v16u8 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7;
	1286	+
	1287	+ for (cnt = (height / 8); cnt--;) {
	1288	+ LD_UB8(src, src_stride, src0, src1, src2, src3, src4, src5, src6, src7);
	1289	+ src += (8 * src_stride);
	1290	+ LD_UB8(dst, dst_stride, dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7);
	1291	+
	1292	+ AVER_UB4_UB(src0, dst0, src1, dst1, src2, dst2, src3, dst3,
	1293	+ dst0, dst1, dst2, dst3);
	1294	+ AVER_UB4_UB(src4, dst4, src5, dst5, src6, dst6, src7, dst7,
	1295	+ dst4, dst5, dst6, dst7);
	1296	+ ST_UB8(dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7, dst, dst_stride);
	1297	+ dst += (8 * dst_stride);
	1298	+ }
	1299	+}
	1300	+
	1301	+void ff_put_pixels16_msa(uint8_t block, const uint8_t pixels,
	1302	+ ptrdiff_t line_size, int h)
	1303	+{
	1304	+ copy_width16_msa(pixels, line_size, block, line_size, h);
	1305	+}
	1306	+
	1307	+void ff_put_pixels16_x2_msa(uint8_t block, const uint8_t pixels,
	1308	+ ptrdiff_t line_size, int h)
	1309	+{
	1310	+ common_hz_bil_16w_msa(pixels, line_size, block, line_size, h);
	1311	+}
	1312	+
	1313	+void ff_put_pixels16_y2_msa(uint8_t block, const uint8_t pixels,
	1314	+ ptrdiff_t line_size, int h)
	1315	+{
	1316	+ common_vt_bil_16w_msa(pixels, line_size, block, line_size, h);
	1317	+}
	1318	+
	1319	+void ff_put_pixels16_xy2_msa(uint8_t block, const uint8_t pixels,
	1320	+ ptrdiff_t line_size, int h)
	1321	+{
	1322	+ common_hv_bil_16w_msa(pixels, line_size, block, line_size, h);
	1323	+}
	1324	+
	1325	+void ff_put_pixels8_msa(uint8_t block, const uint8_t pixels,
	1326	+ ptrdiff_t line_size, int h)
	1327	+{
	1328	+ copy_width8_msa(pixels, line_size, block, line_size, h);
	1329	+}
	1330	+
	1331	+void ff_put_pixels8_x2_msa(uint8_t block, const uint8_t pixels,
	1332	+ ptrdiff_t line_size, int h)
	1333	+{
	1334	+ common_hz_bil_8w_msa(pixels, line_size, block, line_size, h);
	1335	+}
	1336	+
	1337	+void ff_put_pixels8_y2_msa(uint8_t block, const uint8_t pixels,
	1338	+ ptrdiff_t line_size, int h)
	1339	+{
	1340	+ common_vt_bil_8w_msa(pixels, line_size, block, line_size, h);
	1341	+}
	1342	+
	1343	+void ff_put_pixels8_xy2_msa(uint8_t block, const uint8_t pixels,
	1344	+ ptrdiff_t line_size, int h)
	1345	+{
	1346	+ common_hv_bil_8w_msa(pixels, line_size, block, line_size, h);
	1347	+}
	1348	+
	1349	+void ff_put_pixels4_x2_msa(uint8_t block, const uint8_t pixels,
	1350	+ ptrdiff_t line_size, int h)
	1351	+{
	1352	+ common_hz_bil_4w_msa(pixels, line_size, block, line_size, h);
	1353	+}
	1354	+
	1355	+void ff_put_pixels4_y2_msa(uint8_t block, const uint8_t pixels,
	1356	+ ptrdiff_t line_size, int h)
	1357	+{
	1358	+ common_vt_bil_4w_msa(pixels, line_size, block, line_size, h);
	1359	+}
	1360	+
	1361	+void ff_put_pixels4_xy2_msa(uint8_t block, const uint8_t pixels,
	1362	+ ptrdiff_t line_size, int h)
	1363	+{
	1364	+ common_hv_bil_4w_msa(pixels, line_size, block, line_size, h);
	1365	+}
	1366	+
	1367	+void ff_put_no_rnd_pixels16_x2_msa(uint8_t block, const uint8_t pixels,
	1368	+ ptrdiff_t line_size, int h)
	1369	+{
	1370	+ if (h == 16) {
	1371	+ common_hz_bil_no_rnd_16x16_msa(pixels, line_size, block, line_size);
	1372	+ } else if (h == 8) {
	1373	+ common_hz_bil_no_rnd_8x16_msa(pixels, line_size, block, line_size);
	1374	+ }
	1375	+}
	1376	+
	1377	+void ff_put_no_rnd_pixels16_y2_msa(uint8_t block, const uint8_t pixels,
	1378	+ ptrdiff_t line_size, int h)
	1379	+{
	1380	+ if (h == 16) {
	1381	+ common_vt_bil_no_rnd_16x16_msa(pixels, line_size, block, line_size);
	1382	+ } else if (h == 8) {
	1383	+ common_vt_bil_no_rnd_8x16_msa(pixels, line_size, block, line_size);
	1384	+ }
	1385	+}
	1386	+
	1387	+void ff_put_no_rnd_pixels16_xy2_msa(uint8_t *block,
	1388	+ const uint8_t *pixels,
	1389	+ ptrdiff_t line_size, int h)
	1390	+{
	1391	+ if (h == 16) {
	1392	+ common_hv_bil_no_rnd_16x16_msa(pixels, line_size, block, line_size);
	1393	+ } else if (h == 8) {
	1394	+ common_hv_bil_no_rnd_8x16_msa(pixels, line_size, block, line_size);
	1395	+ }
	1396	+}
	1397	+
	1398	+void ff_put_no_rnd_pixels8_x2_msa(uint8_t block, const uint8_t pixels,
	1399	+ ptrdiff_t line_size, int h)
	1400	+{
	1401	+ if (h == 8) {
	1402	+ common_hz_bil_no_rnd_8x8_msa(pixels, line_size, block, line_size);
	1403	+ } else if (h == 4) {
	1404	+ common_hz_bil_no_rnd_4x8_msa(pixels, line_size, block, line_size);
	1405	+ }
	1406	+}
	1407	+
	1408	+void ff_put_no_rnd_pixels8_y2_msa(uint8_t block, const uint8_t pixels,
	1409	+ ptrdiff_t line_size, int h)
	1410	+{
	1411	+ if (h == 8) {
	1412	+ common_vt_bil_no_rnd_8x8_msa(pixels, line_size, block, line_size);
	1413	+ } else if (h == 4) {
	1414	+ common_vt_bil_no_rnd_4x8_msa(pixels, line_size, block, line_size);
	1415	+ }
	1416	+}
	1417	+
	1418	+void ff_put_no_rnd_pixels8_xy2_msa(uint8_t block, const uint8_t pixels,
	1419	+ ptrdiff_t line_size, int h)
	1420	+{
	1421	+ if (h == 8) {
	1422	+ common_hv_bil_no_rnd_8x8_msa(pixels, line_size, block, line_size);
	1423	+ } else if (h == 4) {
	1424	+ common_hv_bil_no_rnd_4x8_msa(pixels, line_size, block, line_size);
	1425	+ }
	1426	+}
	1427	+
	1428	+void ff_avg_pixels16_msa(uint8_t block, const uint8_t pixels,
	1429	+ ptrdiff_t line_size, int h)
	1430	+{
	1431	+ avg_width16_msa(pixels, line_size, block, line_size, h);
	1432	+}
	1433	+
	1434	+void ff_avg_pixels16_x2_msa(uint8_t block, const uint8_t pixels,
	1435	+ ptrdiff_t line_size, int h)
	1436	+{
	1437	+ common_hz_bil_and_aver_dst_16w_msa(pixels, line_size, block, line_size, h);
	1438	+}
	1439	+
	1440	+void ff_avg_pixels16_y2_msa(uint8_t block, const uint8_t pixels,
	1441	+ ptrdiff_t line_size, int h)
	1442	+{
	1443	+ common_vt_bil_and_aver_dst_16w_msa(pixels, line_size, block, line_size, h);
	1444	+}
	1445	+
	1446	+void ff_avg_pixels16_xy2_msa(uint8_t block, const uint8_t pixels,
	1447	+ ptrdiff_t line_size, int h)
	1448	+{
	1449	+ common_hv_bil_and_aver_dst_16w_msa(pixels, line_size, block, line_size, h);
	1450	+}
	1451	+
	1452	+void ff_avg_pixels8_msa(uint8_t block, const uint8_t pixels,
	1453	+ ptrdiff_t line_size, int h)
	1454	+{
	1455	+ avg_width8_msa(pixels, line_size, block, line_size, h);
	1456	+}
	1457	+
	1458	+void ff_avg_pixels8_x2_msa(uint8_t block, const uint8_t pixels,
	1459	+ ptrdiff_t line_size, int h)
	1460	+{
	1461	+ common_hz_bil_and_aver_dst_8w_msa(pixels, line_size, block, line_size, h);
	1462	+}
	1463	+
	1464	+void ff_avg_pixels8_y2_msa(uint8_t block, const uint8_t pixels,
	1465	+ ptrdiff_t line_size, int h)
	1466	+{
	1467	+ common_vt_bil_and_aver_dst_8w_msa(pixels, line_size, block, line_size, h);
	1468	+}
	1469	+
	1470	+void ff_avg_pixels8_xy2_msa(uint8_t block, const uint8_t pixels,
	1471	+ ptrdiff_t line_size, int h)
	1472	+{
	1473	+ common_hv_bil_and_aver_dst_8w_msa(pixels, line_size, block, line_size, h);
	1474	+}
	1475	+
	1476	+void ff_avg_pixels4_msa(uint8_t block, const uint8_t pixels,
	1477	+ ptrdiff_t line_size, int h)
	1478	+{
	1479	+ avg_width4_msa(pixels, line_size, block, line_size, h);
	1480	+}
	1481	+
	1482	+void ff_avg_pixels4_x2_msa(uint8_t block, const uint8_t pixels,
	1483	+ ptrdiff_t line_size, int h)
	1484	+{
	1485	+ common_hz_bil_and_aver_dst_4w_msa(pixels, line_size, block, line_size, h);
	1486	+}
	1487	+
	1488	+void ff_avg_pixels4_y2_msa(uint8_t block, const uint8_t pixels,
	1489	+ ptrdiff_t line_size, int h)
	1490	+{
	1491	+ common_vt_bil_and_aver_dst_4w_msa(pixels, line_size, block, line_size, h);
	1492	+}
	1493	+
	1494	+void ff_avg_pixels4_xy2_msa(uint8_t block, const uint8_t pixels,
	1495	+ ptrdiff_t line_size, int h)
	1496	+{
	1497	+ common_hv_bil_and_aver_dst_4w_msa(pixels, line_size, block, line_size, h);
	1498	+}

libavutil/mips/generic_macros_msa.h+162−0

@@ -804,6 +804,15 @@ · 803 unmodified lines
804	804	#define SLDI_B2_0_SB(...) SLDI_B2_0(v16i8, __VA_ARGS__)
805	805	#define SLDI_B2_0_SW(...) SLDI_B2_0(v4i32, __VA_ARGS__)
806	806
	807	+#define SLDI_B3_0(RTYPE, in0, in1, in2, out0, out1, out2, slide_val) \
	808	+{ \
	809	+ v16i8 zero_m = { 0 }; \
	810	+ SLDI_B2_0(RTYPE, in0, in1, out0, out1, slide_val); \
	811	+ out2 = (RTYPE) __msa_sldi_b((v16i8) zero_m, (v16i8) in2, slide_val); \
	812	+}
	813	+#define SLDI_B3_0_UB(...) SLDI_B3_0(v16u8, __VA_ARGS__)
	814	+#define SLDI_B3_0_SB(...) SLDI_B3_0(v16i8, __VA_ARGS__)
	815	+
807	816	#define SLDI_B4_0(RTYPE, in0, in1, in2, in3, \
808	817	out0, out1, out2, out3, slide_val) \
809	818	{ \
@@ -1174,6 +1183,13 @@ · 364 unmodified lines
1174	1183	}
1175	1184	#define HADD_UB2_UH(...) HADD_UB2(v8u16, __VA_ARGS__)
1176	1185
	1186	+#define HADD_UB3(RTYPE, in0, in1, in2, out0, out1, out2) \
	1187	+{ \
	1188	+ HADD_UB2(RTYPE, in0, in1, out0, out1); \
	1189	+ out2 = (RTYPE) __msa_hadd_u_h((v16u8) in2, (v16u8) in2); \
	1190	+}
	1191	+#define HADD_UB3_UH(...) HADD_UB3(v8u16, __VA_ARGS__)
	1192	+
1177	1193	/* Description : Horizontal subtraction of unsigned byte vector elements
1178	1194	Arguments : Inputs - in0, in1
1179	1195	Outputs - out0, out1
@@ -2408,6 +2424,67 @@ · 1228 unmodified lines
2408	2424	out3 = (v4i32) __msa_ilvl_d((v2i64) s3_m, (v2i64) s1_m); \
2409	2425	}
2410	2426
	2427	+/* Description : Average byte elements from pair of vectors and store 8x4 byte
	2428	+ block in destination memory
	2429	+ Arguments : Inputs - in0, in1, in2, in3, in4, in5, in6, in7, pdst, stride
	2430	+ Outputs -
	2431	+ Return Type -
	2432	+ Details : Each byte element from input vector pair 'in0' and 'in1' are
	2433	+ averaged (a + b)/2 and stored in 'tmp0_m'
	2434	+ Each byte element from input vector pair 'in2' and 'in3' are
	2435	+ averaged (a + b)/2 and stored in 'tmp1_m'
	2436	+ Each byte element from input vector pair 'in4' and 'in5' are
	2437	+ averaged (a + b)/2 and stored in 'tmp2_m'
	2438	+ Each byte element from input vector pair 'in6' and 'in7' are
	2439	+ averaged (a + b)/2 and stored in 'tmp3_m'
	2440	+ The half vector results from all 4 vectors are stored in
	2441	+ destination memory as 8x4 byte block
	2442	+*/
	2443	+#define AVE_ST8x4_UB(in0, in1, in2, in3, in4, in5, in6, in7, pdst, stride) \
	2444	+{ \
	2445	+ uint64_t out0_m, out1_m, out2_m, out3_m; \
	2446	+ v16u8 tmp0_m, tmp1_m, tmp2_m, tmp3_m; \
	2447	+ \
	2448	+ tmp0_m = __msa_ave_u_b((v16u8) in0, (v16u8) in1); \
	2449	+ tmp1_m = __msa_ave_u_b((v16u8) in2, (v16u8) in3); \
	2450	+ tmp2_m = __msa_ave_u_b((v16u8) in4, (v16u8) in5); \
	2451	+ tmp3_m = __msa_ave_u_b((v16u8) in6, (v16u8) in7); \
	2452	+ \
	2453	+ out0_m = __msa_copy_u_d((v2i64) tmp0_m, 0); \
	2454	+ out1_m = __msa_copy_u_d((v2i64) tmp1_m, 0); \
	2455	+ out2_m = __msa_copy_u_d((v2i64) tmp2_m, 0); \
	2456	+ out3_m = __msa_copy_u_d((v2i64) tmp3_m, 0); \
	2457	+ SD4(out0_m, out1_m, out2_m, out3_m, pdst, stride); \
	2458	+}
	2459	+
	2460	+/* Description : Average byte elements from pair of vectors and store 16x4 byte
	2461	+ block in destination memory
	2462	+ Arguments : Inputs - in0, in1, in2, in3, in4, in5, in6, in7, pdst, stride
	2463	+ Outputs -
	2464	+ Return Type -
	2465	+ Details : Each byte element from input vector pair 'in0' and 'in1' are
	2466	+ averaged (a + b)/2 and stored in 'tmp0_m'
	2467	+ Each byte element from input vector pair 'in2' and 'in3' are
	2468	+ averaged (a + b)/2 and stored in 'tmp1_m'
	2469	+ Each byte element from input vector pair 'in4' and 'in5' are
	2470	+ averaged (a + b)/2 and stored in 'tmp2_m'
	2471	+ Each byte element from input vector pair 'in6' and 'in7' are
	2472	+ averaged (a + b)/2 and stored in 'tmp3_m'
	2473	+ The results from all 4 vectors are stored in destination
	2474	+ memory as 16x4 byte block
	2475	+*/
	2476	+#define AVE_ST16x4_UB(in0, in1, in2, in3, in4, in5, in6, in7, pdst, stride) \
	2477	+{ \
	2478	+ v16u8 tmp0_m, tmp1_m, tmp2_m, tmp3_m; \
	2479	+ \
	2480	+ tmp0_m = __msa_ave_u_b((v16u8) in0, (v16u8) in1); \
	2481	+ tmp1_m = __msa_ave_u_b((v16u8) in2, (v16u8) in3); \
	2482	+ tmp2_m = __msa_ave_u_b((v16u8) in4, (v16u8) in5); \
	2483	+ tmp3_m = __msa_ave_u_b((v16u8) in6, (v16u8) in7); \
	2484	+ \
	2485	+ ST_UB4(tmp0_m, tmp1_m, tmp2_m, tmp3_m, pdst, stride); \
	2486	+}
	2487	+
2411	2488	/* Description : Average rounded byte elements from pair of vectors and store
2412	2489	8x4 byte block in destination memory
2413	2490	Arguments : Inputs - in0, in1, in2, in3, in4, in5, in6, in7, pdst, stride
@@ -2439,6 +2516,91 @@ · 25 unmodified lines
2439	2516	SD4(out0_m, out1_m, out2_m, out3_m, pdst, stride); \
2440	2517	}
2441	2518
	2519	+/* Description : Average rounded byte elements from pair of vectors and store
	2520	+ 16x4 byte block in destination memory
	2521	+ Arguments : Inputs - in0, in1, in2, in3, in4, in5, in6, in7, pdst, stride
	2522	+ Outputs -
	2523	+ Return Type -
	2524	+ Details : Each byte element from input vector pair 'in0' and 'in1' are
	2525	+ average rounded (a + b + 1)/2 and stored in 'tmp0_m'
	2526	+ Each byte element from input vector pair 'in2' and 'in3' are
	2527	+ average rounded (a + b + 1)/2 and stored in 'tmp1_m'
	2528	+ Each byte element from input vector pair 'in4' and 'in5' are
	2529	+ average rounded (a + b + 1)/2 and stored in 'tmp2_m'
	2530	+ Each byte element from input vector pair 'in6' and 'in7' are
	2531	+ average rounded (a + b + 1)/2 and stored in 'tmp3_m'
	2532	+ The vector results from all 4 vectors are stored in
	2533	+ destination memory as 16x4 byte block
	2534	+*/
	2535	+#define AVER_ST16x4_UB(in0, in1, in2, in3, in4, in5, in6, in7, pdst, stride) \
	2536	+{ \
	2537	+ v16u8 t0_m, t1_m, t2_m, t3_m; \
	2538	+ \
	2539	+ AVER_UB4_UB(in0, in1, in2, in3, in4, in5, in6, in7, \
	2540	+ t0_m, t1_m, t2_m, t3_m); \
	2541	+ ST_UB4(t0_m, t1_m, t2_m, t3_m, pdst, stride); \
	2542	+}
	2543	+
	2544	+/* Description : Average rounded byte elements from pair of vectors,
	2545	+ average rounded with destination and store 8x4 byte block
	2546	+ in destination memory
	2547	+ Arguments : Inputs - in0, in1, in2, in3, in4, in5, in6, in7, pdst, stride
	2548	+ Outputs -
	2549	+ Return Type -
	2550	+ Details : Each byte element from input vector pair 'in0' and 'in1' are
	2551	+ average rounded (a + b + 1)/2 and stored in 'tmp0_m'
	2552	+ Each byte element from input vector pair 'in2' and 'in3' are
	2553	+ average rounded (a + b + 1)/2 and stored in 'tmp1_m'
	2554	+ Each byte element from input vector pair 'in4' and 'in5' are
	2555	+ average rounded (a + b + 1)/2 and stored in 'tmp2_m'
	2556	+ Each byte element from input vector pair 'in6' and 'in7' are
	2557	+ average rounded (a + b + 1)/2 and stored in 'tmp3_m'
	2558	+ The half vector results from all 4 vectors are stored in
	2559	+ destination memory as 8x4 byte block
	2560	+*/
	2561	+#define AVER_DST_ST8x4_UB(in0, in1, in2, in3, in4, in5, in6, in7, \
	2562	+ pdst, stride) \
	2563	+{ \
	2564	+ v16u8 tmp0_m, tmp1_m, tmp2_m, tmp3_m; \
	2565	+ v16u8 dst0_m, dst1_m, dst2_m, dst3_m; \
	2566	+ \
	2567	+ LD_UB4(pdst, stride, dst0_m, dst1_m, dst2_m, dst3_m); \
	2568	+ AVER_UB4_UB(in0, in1, in2, in3, in4, in5, in6, in7, \
	2569	+ tmp0_m, tmp1_m, tmp2_m, tmp3_m); \
	2570	+ AVER_ST8x4_UB(dst0_m, tmp0_m, dst1_m, tmp1_m, \
	2571	+ dst2_m, tmp2_m, dst3_m, tmp3_m, pdst, stride); \
	2572	+}
	2573	+
	2574	+/* Description : Average rounded byte elements from pair of vectors,
	2575	+ average rounded with destination and store 16x4 byte block
	2576	+ in destination memory
	2577	+ Arguments : Inputs - in0, in1, in2, in3, in4, in5, in6, in7, pdst, stride
	2578	+ Outputs -
	2579	+ Return Type -
	2580	+ Details : Each byte element from input vector pair 'in0' and 'in1' are
	2581	+ average rounded (a + b + 1)/2 and stored in 'tmp0_m'
	2582	+ Each byte element from input vector pair 'in2' and 'in3' are
	2583	+ average rounded (a + b + 1)/2 and stored in 'tmp1_m'
	2584	+ Each byte element from input vector pair 'in4' and 'in5' are
	2585	+ average rounded (a + b + 1)/2 and stored in 'tmp2_m'
	2586	+ Each byte element from input vector pair 'in6' and 'in7' are
	2587	+ average rounded (a + b + 1)/2 and stored in 'tmp3_m'
	2588	+ The vector results from all 4 vectors are stored in
	2589	+ destination memory as 16x4 byte block
	2590	+*/
	2591	+#define AVER_DST_ST16x4_UB(in0, in1, in2, in3, in4, in5, in6, in7, \
	2592	+ pdst, stride) \
	2593	+{ \
	2594	+ v16u8 tmp0_m, tmp1_m, tmp2_m, tmp3_m; \
	2595	+ v16u8 dst0_m, dst1_m, dst2_m, dst3_m; \
	2596	+ \
	2597	+ LD_UB4(pdst, stride, dst0_m, dst1_m, dst2_m, dst3_m); \
	2598	+ AVER_UB4_UB(in0, in1, in2, in3, in4, in5, in6, in7, \
	2599	+ tmp0_m, tmp1_m, tmp2_m, tmp3_m); \
	2600	+ AVER_ST16x4_UB(dst0_m, tmp0_m, dst1_m, tmp1_m, \
	2601	+ dst2_m, tmp2_m, dst3_m, tmp3_m, pdst, stride); \
	2602	+}
	2603	+
2442	2604	/* Description : Add block 4x4
2443	2605	Arguments : Inputs - in0, in1, in2, in3, pdst, stride
2444	2606	Outputs -

avcodec/mips: MSA (MIPS-SIMD-Arch) optimizations for hpel functions

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code