Multiframe quality enhancement postprocessing

Adds a multiframe postprocessing module to enhance the quality of certain frames that are coded at lower quality than preceding frames. The module can be invoked from the commandline by use of the --mfqe option, and will be most beneficial for enhancing the quality of frames decoded using scalable patterns. Uses the vp8_variance_var16x16 and vp8_variance_sad16x16 function pointers to compute SAD and Variance of blocks.

Deb Mukherjee committed 14 years ago

Commit 87aa8461 parent 056bcc87 files · +234 −11Browse files

Unified · Split

7 files

examples/postproc.txt +1 −1
vp8/common/postproc.c +220 −7
vp8/common/postproc.h +1 −0
vp8/common/ppflags.h +2 −1
vp8/vp8_dx_iface.c +1 −1
vpx/vp8.h +1 −0
vpxdec.c +8 −1

examples/postproc.txt+1−1

@@ -58,7 +58,7 @@ if(frame_cnt%30 == 1) { · 57 unmodified lines
58	58	if(vpx_codec_control(&codec, VP8_SET_POSTPROC, &pp))
59	59	die_codec(&codec, "Failed to turn off postproc");
60	60	} else if(frame_cnt%30 == 16) {
61		− vp8_postproc_cfg_t pp = {VP8_DEBLOCK \| VP8_DEMACROBLOCK, 4, 0};
	61	+ vp8_postproc_cfg_t pp = {VP8_DEBLOCK \| VP8_DEMACROBLOCK \| VP8_MFQE, 4, 0};
62	62
63	63	if(vpx_codec_control(&codec, VP8_SET_POSTPROC, &pp))
64	64	die_codec(&codec, "Failed to turn on postproc");

vp8/common/postproc.c+220−7

@@ -12,9 +12,12 @@ · 11 unmodified lines
12	12	#include "vpx_config.h"
13	13	#include "vpx_scale/yv12config.h"
14	14	#include "postproc.h"
	15	+#include "common.h"
	16	+#include "recon.h"
15	17	#include "vpx_scale/yv12extend.h"
16	18	#include "vpx_scale/vpxscale.h"
17	19	#include "systemdependent.h"
	20	+#include "../encoder/variance.h"
18	21
19	22	#include <math.h>
20	23	#include <stdlib.h>
@@ -121,7 +124,6 @@ const short vp8_rv[] = · 100 unmodified lines
121	124	0, 9, 5, 5, 11, 10, 13, 9, 10, 13,
122	125	};
123	126
124		−
125	127	extern void vp8_blit_text(const char msg, unsigned char address, const int pitch);
126	128	extern void vp8_blit_line(int x0, int x1, int y0, int y1, unsigned char *image, const int pitch);
127	129	/***********************************************************************************************************
@@ -323,11 +325,11 @@ static void vp8_deblock_and_de_macro_block(YV12_BUFFER_CONFIG *source, · 195 unmodified lines
323	325	}
324	326
325	327	void vp8_deblock(YV12_BUFFER_CONFIG *source,
326		− YV12_BUFFER_CONFIG *post,
327		− int q,
328		− int low_var_thresh,
329		− int flag,
330		− vp8_postproc_rtcd_vtable_t *rtcd)
	328	+ YV12_BUFFER_CONFIG *post,
	329	+ int q,
	330	+ int low_var_thresh,
	331	+ int flag,
	332	+ vp8_postproc_rtcd_vtable_t *rtcd)
331	333	{
332	334	double level = 6.0e-05 * q * q * q - .0067 * q * q + .306 * q + .0065;
333	335	int ppl = (int)(level + .5);
@@ -671,6 +673,210 @@ static void constrain_line (int x0, int x1, int y0, int y1, int width, int hei · 337 unmodified lines
671	673	}
672	674	}
673	675
	676	+static void multiframe_quality_enhance_block
	677	+(
	678	+ int blksize, /* Currently only values supported are 16, 8, 4 */
	679	+ int qcurr,
	680	+ int qprev,
	681	+ unsigned char *y,
	682	+ unsigned char *u,
	683	+ unsigned char *v,
	684	+ int y_stride,
	685	+ int uv_stride,
	686	+ unsigned char *yd,
	687	+ unsigned char *ud,
	688	+ unsigned char *vd,
	689	+ int yd_stride,
	690	+ int uvd_stride
	691	+)
	692	+{
	693	+ static const unsigned char VP8_ZEROS[16]=
	694	+ {
	695	+ 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
	696	+ };
	697	+ int blksizeby2 = blksize >> 1;
	698	+ int blksizesq = blksize * blksize;
	699	+
	700	+ int i, j;
	701	+ unsigned char *yp;
	702	+ unsigned char *ydp;
	703	+ unsigned char *up;
	704	+ unsigned char *udp;
	705	+ unsigned char *vp;
	706	+ unsigned char *vdp;
	707	+
	708	+ unsigned int act, sse, sad, thr;
	709	+ if (blksize == 16)
	710	+ {
	711	+ act = vp8_variance_var16x16(y, y_stride, VP8_ZEROS, 0, &sse);
	712	+ sad = vp8_variance_sad16x16(y, y_stride, yd, yd_stride, 0);
	713	+ }
	714	+ else if (blksize == 8)
	715	+ {
	716	+ act = vp8_variance_var8x8(y, y_stride, VP8_ZEROS, 0, &sse);
	717	+ sad = vp8_variance_sad8x8(y, y_stride, yd, yd_stride, 0);
	718	+ }
	719	+ else
	720	+ {
	721	+ act = vp8_variance_var4x4(y, y_stride, VP8_ZEROS, 0, &sse);
	722	+ sad = vp8_variance_sad4x4(y, y_stride, yd, yd_stride, 0);
	723	+ }
	724	+
	725	+ thr = 6 * blksizesq + (act >> 3);
	726	+ if (thr > 12 * blksizesq) thr = 12 * blksizesq;
	727	+ // These thresholds should be adapted later based on qcurr and qprev
	728	+ if (sad < thr)
	729	+ {
	730	+ static const int precision = 4;
	731	+ static const int roundoff = (1 << (precision - 1));
	732	+ int ifactor = (sad << precision) / thr;
	733	+ // TODO: SIMD optimize this section
	734	+ if (ifactor)
	735	+ {
	736	+ int icfactor = (1 << precision) - ifactor;
	737	+ for (yp = y, ydp = yd, i = 0; i < blksize; ++i, yp += y_stride, ydp += yd_stride)
	738	+ {
	739	+ for (j = 0; j < blksize; ++j)
	740	+ ydp[j] = (int)((yp[j] * ifactor + ydp[j] * icfactor + roundoff) >> precision);
	741	+ }
	742	+ for (up = u, udp = ud, i = 0; i < blksizeby2; ++i, up += uv_stride, udp += uvd_stride)
	743	+ {
	744	+ for (j = 0; j < blksizeby2; ++j)
	745	+ udp[j] = (int)((up[j] * ifactor + udp[j] * icfactor + roundoff) >> precision);
	746	+ }
	747	+ for (vp = v, vdp = vd, i = 0; i < blksizeby2; ++i, vp += uv_stride, vdp += uvd_stride)
	748	+ {
	749	+ for (j = 0; j < blksizeby2; ++j)
	750	+ vdp[j] = (int)((vp[j] * ifactor + vdp[j] * icfactor + roundoff) >> precision);
	751	+ }
	752	+ }
	753	+ }
	754	+ else
	755	+ {
	756	+ if (blksize == 16)
	757	+ {
	758	+ vp8_recon_copy16x16(y, y_stride, yd, yd_stride);
	759	+ vp8_recon_copy8x8(u, uv_stride, ud, uvd_stride);
	760	+ vp8_recon_copy8x8(v, uv_stride, vd, uvd_stride);
	761	+ }
	762	+ else if (blksize == 8)
	763	+ {
	764	+ vp8_recon_copy8x8(y, y_stride, yd, yd_stride);
	765	+ for (up = u, udp = ud, i = 0; i < blksizeby2; ++i, up += uv_stride, udp += uvd_stride)
	766	+ vpx_memcpy(udp, up, blksizeby2);
	767	+ for (vp = v, vdp = vd, i = 0; i < blksizeby2; ++i, vp += uv_stride, vdp += uvd_stride)
	768	+ vpx_memcpy(vdp, vp, blksizeby2);
	769	+ }
	770	+ else
	771	+ {
	772	+ for (yp = y, ydp = yd, i = 0; i < blksize; ++i, yp += y_stride, ydp += yd_stride)
	773	+ vpx_memcpy(ydp, yp, blksize);
	774	+ for (up = u, udp = ud, i = 0; i < blksizeby2; ++i, up += uv_stride, udp += uvd_stride)
	775	+ vpx_memcpy(udp, up, blksizeby2);
	776	+ for (vp = v, vdp = vd, i = 0; i < blksizeby2; ++i, vp += uv_stride, vdp += uvd_stride)
	777	+ vpx_memcpy(vdp, vp, blksizeby2);
	778	+ }
	779	+ }
	780	+}
	781	+
	782	+void vp8_multiframe_quality_enhance
	783	+(
	784	+ VP8_COMMON *cm
	785	+)
	786	+{
	787	+ YV12_BUFFER_CONFIG *show = cm->frame_to_show;
	788	+ YV12_BUFFER_CONFIG *dest = &cm->post_proc_buffer;
	789	+
	790	+ FRAME_TYPE frame_type = cm->frame_type;
	791	+ /* Point at base of Mb MODE_INFO list has motion vectors etc */
	792	+ const MODE_INFO *mode_info_context = cm->mi;
	793	+ int qcurr = cm->base_qindex;
	794	+ int qprev = cm->postproc_state.last_base_qindex;
	795	+
	796	+ int mb_row;
	797	+ int mb_col;
	798	+
	799	+ unsigned char y_ptr, u_ptr, *v_ptr;
	800	+ unsigned char yd_ptr, ud_ptr, *vd_ptr;
	801	+
	802	+ /* Set up the buffer pointers */
	803	+ y_ptr = show->y_buffer;
	804	+ u_ptr = show->u_buffer;
	805	+ v_ptr = show->v_buffer;
	806	+ yd_ptr = dest->y_buffer;
	807	+ ud_ptr = dest->u_buffer;
	808	+ vd_ptr = dest->v_buffer;
	809	+
	810	+ /* postprocess each macro block */
	811	+ for (mb_row = 0; mb_row < cm->mb_rows; mb_row++)
	812	+ {
	813	+ for (mb_col = 0; mb_col < cm->mb_cols; mb_col++)
	814	+ {
	815	+ /* if motion is high there will likely be no benefit */
	816	+ if (((frame_type == INTER_FRAME &&
	817	+ abs(mode_info_context->mbmi.mv.as_mv.row) <= 10 &&
	818	+ abs(mode_info_context->mbmi.mv.as_mv.col) <= 10) \|\|
	819	+ (frame_type == KEY_FRAME)) &&
	820	+ mode_info_context->mbmi.mode != B_PRED)
	821	+ {
	822	+ multiframe_quality_enhance_block(16,
	823	+ qcurr,
	824	+ qprev,
	825	+ y_ptr,
	826	+ u_ptr,
	827	+ v_ptr,
	828	+ show->y_stride,
	829	+ show->uv_stride,
	830	+ yd_ptr,
	831	+ ud_ptr,
	832	+ vd_ptr,
	833	+ dest->y_stride,
	834	+ dest->uv_stride);
	835	+ }
	836	+ else if (mode_info_context->mbmi.mode == B_PRED)
	837	+ {
	838	+ int i, j;
	839	+ for (i=0; i<2; ++i)
	840	+ for (j=0; j<2; ++j)
	841	+ multiframe_quality_enhance_block(8,
	842	+ qcurr,
	843	+ qprev,
	844	+ y_ptr + 8(ishow->y_stride+j),
	845	+ u_ptr + 4(ishow->uv_stride+j),
	846	+ v_ptr + 4(ishow->uv_stride+j),
	847	+ show->y_stride,
	848	+ show->uv_stride,
	849	+ yd_ptr + 8(idest->y_stride+j),
	850	+ ud_ptr + 4(idest->uv_stride+j),
	851	+ vd_ptr + 4(idest->uv_stride+j),
	852	+ dest->y_stride,
	853	+ dest->uv_stride);
	854	+ }
	855	+ else
	856	+ {
	857	+ vp8_recon_copy16x16(y_ptr, show->y_stride, yd_ptr, dest->y_stride);
	858	+ vp8_recon_copy8x8(u_ptr, show->uv_stride, ud_ptr, dest->uv_stride);
	859	+ vp8_recon_copy8x8(v_ptr, show->uv_stride, vd_ptr, dest->uv_stride);
	860	+ }
	861	+ y_ptr += 16;
	862	+ u_ptr += 8;
	863	+ v_ptr += 8;
	864	+ yd_ptr += 16;
	865	+ ud_ptr += 8;
	866	+ vd_ptr += 8;
	867	+ mode_info_context++; /* step to next MB */
	868	+ }
	869	+
	870	+ y_ptr += show->y_stride * 16 - 16 * cm->mb_cols;
	871	+ u_ptr += show->uv_stride * 8 - 8 * cm->mb_cols;
	872	+ v_ptr += show->uv_stride * 8 - 8 * cm->mb_cols;
	873	+ yd_ptr += dest->y_stride * 16 - 16 * cm->mb_cols;
	874	+ ud_ptr += dest->uv_stride * 8 - 8 * cm->mb_cols;
	875	+ vd_ptr += dest->uv_stride * 8 - 8 * cm->mb_cols;
	876	+
	877	+ mode_info_context++; /* Skip border mb */
	878	+ }
	879	+}
674	880
675	881	#if CONFIG_RUNTIME_CPU_DETECT
676	882	#define RTCD_VTABLE(oci) (&(oci)->rtcd.postproc)
@@ -699,8 +905,8 @@ int vp8_post_proc_frame(VP8_COMMON oci, YV12_BUFFER_CONFIG dest, vp8_ppflags_t · 22 unmodified lines
699	905	dest->y_width = oci->Width;
700	906	dest->y_height = oci->Height;
701	907	dest->uv_height = dest->y_height / 2;
	908	+ oci->postproc_state.last_base_qindex = oci->base_qindex;
702	909	return 0;
703		−
704	910	}
705	911
706	912	#if ARCH_X86\|\|ARCH_X86_64
@@ -717,6 +923,12 @@ int vp8_post_proc_frame(VP8_COMMON oci, YV12_BUFFER_CONFIG dest, vp8_ppflags_t · 10 unmodified lines
717	923	vp8_deblock(oci->frame_to_show, &oci->post_proc_buffer,
718	924	q, 1, 0, RTCD_VTABLE(oci));
719	925	}
	926	+ else if ((flags & VP8D_MFQE) &&
	927	+ oci->current_video_frame >= 2 &&
	928	+ oci->base_qindex - oci->postproc_state.last_base_qindex >= 10)
	929	+ {
	930	+ vp8_multiframe_quality_enhance(oci);
	931	+ }
720	932	else
721	933	{
722	934	vp8_yv12_copy_frame_ptr(oci->frame_to_show, &oci->post_proc_buffer);
@@ -1105,5 +1317,6 @@ int vp8_post_proc_frame(VP8_COMMON oci, YV12_BUFFER_CONFIG dest, vp8_ppflags_t · 382 unmodified lines
1105	1317	dest->y_width = oci->Width;
1106	1318	dest->y_height = oci->Height;
1107	1319	dest->uv_height = dest->y_height / 2;
	1320	+ oci->postproc_state.last_base_qindex = oci->base_qindex;
1108	1321	return 0;
1109	1322	}

vp8/common/postproc.h+1−0

@@ -104,6 +104,7 @@ struct postproc_state · 103 unmodified lines
104	104	int last_q;
105	105	int last_noise;
106	106	char noise[3072];
	107	+ int last_base_qindex;
107	108	DECLARE_ALIGNED(16, char, blackclamp[16]);
108	109	DECLARE_ALIGNED(16, char, whiteclamp[16]);
109	110	DECLARE_ALIGNED(16, char, bothclamp[16]);

vp8/common/ppflags.h+2−1

@@ -23,7 +23,8 @@ enum · 22 unmodified lines
23	23	VP8D_DEBUG_TXT_RATE_INFO = 1<<6,
24	24	VP8D_DEBUG_DRAW_MV = 1<<7,
25	25	VP8D_DEBUG_CLR_BLK_MODES = 1<<8,
26		− VP8D_DEBUG_CLR_FRM_REF_BLKS = 1<<9
	26	+ VP8D_DEBUG_CLR_FRM_REF_BLKS = 1<<9,
	27	+ VP8D_MFQE = 1<<10
27	28	};
28	29
29	30	typedef struct

vp8/vp8_dx_iface.c+1−1

@@ -412,7 +412,7 @@ static vpx_codec_err_t vp8_decode(vpx_codec_alg_priv_t *ctx, · 411 unmodified lines
412	412	&& (ctx->base.init_flags & VPX_CODEC_USE_POSTPROC))
413	413	{
414	414	ctx->postproc_cfg.post_proc_flag =
415		− VP8_DEBLOCK \| VP8_DEMACROBLOCK;
	415	+ VP8_DEBLOCK \| VP8_DEMACROBLOCK \| VP8_MFQE;
416	416	ctx->postproc_cfg.deblocking_level = 4;
417	417	ctx->postproc_cfg.noise_level = 0;
418	418	}

vpx/vp8.h+1−0

@@ -63,6 +63,7 @@ enum vp8_postproc_level · 62 unmodified lines
63	63	VP8_DEBUG_TXT_MBLK_MODES = 1<<4, /*< print macro block modes over each macro block /
64	64	VP8_DEBUG_TXT_DC_DIFF = 1<<5, /*< print dc diff for each macro block /
65	65	VP8_DEBUG_TXT_RATE_INFO = 1<<6, /*< print video rate info (encoder only) /
	66	+ VP8_MFQE = 1<<10,
66	67	};
67	68
68	69	/*!\brief post process flags

vpxdec.c+8−1

@@ -124,11 +124,13 @@ static const arg_def_t pp_disp_b_modes = ARG_DEF(NULL, "pp-dbg-b-modes", 1, · 123 unmodified lines
124	124	"Display only selected block modes");
125	125	static const arg_def_t pp_disp_mvs = ARG_DEF(NULL, "pp-dbg-mvs", 1,
126	126	"Draw only selected motion vectors");
	127	+static const arg_def_t mfqe = ARG_DEF(NULL, "mfqe", 0,
	128	+ "Enable multiframe quality enhancement");
127	129
128	130	static const arg_def_t *vp8_pp_args[] =
129	131	{
130	132	&addnoise_level, &deblock, &demacroblock_level, &pp_debug_info,
131		− &pp_disp_ref_frame, &pp_disp_mb_modes, &pp_disp_b_modes, &pp_disp_mvs,
	133	+ &pp_disp_ref_frame, &pp_disp_mb_modes, &pp_disp_b_modes, &pp_disp_mvs, &mfqe,
132	134	NULL
133	135	};
134	136	#endif
@@ -803,6 +805,11 @@ int main(int argc, const char **argv_) · 668 unmodified lines
803	805	postproc = 1;
804	806	vp8_pp_cfg.post_proc_flag \|= VP8_DEBLOCK;
805	807	}
	808	+ else if (arg_match(&arg, &mfqe, argi))
	809	+ {
	810	+ postproc = 1;
	811	+ vp8_pp_cfg.post_proc_flag \|= VP8_MFQE;
	812	+ }
806	813	else if (arg_match(&arg, &pp_debug_info, argi))
807	814	{
808	815	unsigned int level = arg_parse_uint(&arg);

Multiframe quality enhancement postprocessing

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code