vp9 svc: Allow denoising next to highest resolution.

Denoise 2 spatial layes at most. Add noise sensitivity level 2 for vp9 such that applications can control whether to denoise the second highest spatial layer. Add tests to cover this case.

Jerome Jiang committed 8 years ago

Commit c1e511f1 parent 14dbdd910 files · +210 −110Browse files

Unified · Split

10 files

examples/vpx_temporal_svc_encoder.c +17 −9
test/datarate_test.cc +21 −16
vp9/encoder/vp9_denoiser.c +118 −61
vp9/encoder/vp9_denoiser.h +10 −6
vp9/encoder/vp9_encoder.c +16 −6
vp9/encoder/vp9_encoder.h +1 −1
vp9/encoder/vp9_noise_estimate.c +15 −6
vp9/encoder/vp9_svc_layercontext.c +4 −3
vp9/encoder/vp9_svc_layercontext.h +7 −1
vpx/vp8cx.h +1 −1

examples/vpx_temporal_svc_encoder.c+17−9

@@ -32,13 +32,21 @@ static const char *exec_name; · 31 unmodified lines
32	32
33	33	void usage_exit(void) { exit(EXIT_FAILURE); }
34	34
35		−// Denoiser states, for temporal denoising.
36		−enum denoiserState {
37		− kDenoiserOff,
38		− kDenoiserOnYOnly,
39		− kDenoiserOnYUV,
40		− kDenoiserOnYUVAggressive,
41		− kDenoiserOnAdaptive
	35	+// Denoiser states for vp8, for temporal denoising.
	36	+enum denoiserStateVp8 {
	37	+ kVp8DenoiserOff,
	38	+ kVp8DenoiserOnYOnly,
	39	+ kVp8DenoiserOnYUV,
	40	+ kVp8DenoiserOnYUVAggressive,
	41	+ kVp8DenoiserOnAdaptive
	42	+};
	43	+
	44	+// Denoiser states for vp9, for temporal denoising.
	45	+enum denoiserStateVp9 {
	46	+ kVp9DenoiserOff,
	47	+ kVp9DenoiserOnYOnly,
	48	+ // For SVC: denoise the top two spatial layers.
	49	+ kVp9DenoiserOnYTwoSpatialLayers
42	50	};
43	51
44	52	static int mode_to_num_layers[13] = { 1, 2, 2, 3, 3, 3, 3, 5, 2, 3, 3, 3, 3 };
@@ -755,7 +763,7 @@ int main(int argc, char **argv) { · 710 unmodified lines
755	763
756	764	if (strncmp(encoder->name, "vp8", 3) == 0) {
757	765	vpx_codec_control(&codec, VP8E_SET_CPUUSED, -speed);
758		− vpx_codec_control(&codec, VP8E_SET_NOISE_SENSITIVITY, kDenoiserOff);
	766	+ vpx_codec_control(&codec, VP8E_SET_NOISE_SENSITIVITY, kVp8DenoiserOff);
759	767	vpx_codec_control(&codec, VP8E_SET_STATIC_THRESHOLD, 1);
760	768	vpx_codec_control(&codec, VP8E_SET_GF_CBR_BOOST_PCT, 0);
761	769	#if VP8_ROI_MAP
@@ -772,7 +780,7 @@ int main(int argc, char **argv) { · 10 unmodified lines
772	780	vpx_codec_control(&codec, VP9E_SET_GF_CBR_BOOST_PCT, 0);
773	781	vpx_codec_control(&codec, VP9E_SET_FRAME_PARALLEL_DECODING, 0);
774	782	vpx_codec_control(&codec, VP9E_SET_FRAME_PERIODIC_BOOST, 0);
775		− vpx_codec_control(&codec, VP9E_SET_NOISE_SENSITIVITY, kDenoiserOff);
	783	+ vpx_codec_control(&codec, VP9E_SET_NOISE_SENSITIVITY, kVp9DenoiserOff);
776	784	vpx_codec_control(&codec, VP8E_SET_STATIC_THRESHOLD, 1);
777	785	vpx_codec_control(&codec, VP9E_SET_TUNE_CONTENT, 0);
778	786	vpx_codec_control(&codec, VP9E_SET_TILE_COLUMNS, (cfg.g_threads >> 1));

test/datarate_test.cc+21−16

@@ -1449,24 +1449,29 @@ TEST_P(DatarateOnePassCbrSvc, OnePassCbrSvc2SL3TLDenoiserOn) { · 1448 unmodified lines
1449	1449	::libvpx_test::Y4mVideoSource video("niklas_1280_720_30.y4m", 0, 300);
1450	1450	// TODO(marpan): Check that effective_datarate for each layer hits the
1451	1451	// layer target_bitrate.
1452		− for (int i = 600; i <= 1000; i += 200) {
1453		− cfg_.rc_target_bitrate = i;
1454		− ResetModel();
1455		− denoiser_on_ = 1;
1456		− assign_layer_bitrates(&cfg_, &svc_params_, cfg_.ss_number_layers,
1457		− cfg_.ts_number_layers, cfg_.temporal_layering_mode);
1458		− ASSERT_NO_FATAL_FAILURE(RunLoop(&video));
1459		− ASSERT_GE(cfg_.rc_target_bitrate, file_datarate_ * 0.78)
1460		− << " The datarate for the file exceeds the target by too much!";
1461		− ASSERT_LE(cfg_.rc_target_bitrate, file_datarate_ * 1.15)
1462		− << " The datarate for the file is lower than the target by too much!";
	1452	+ // For SVC, noise_sen = 1 means denoising only the top spatial layer
	1453	+ // noise_sen = 2 means denoising the two top spatial layers.
	1454	+ for (int noise_sen = 1; noise_sen <= 2; noise_sen++) {
	1455	+ for (int i = 600; i <= 1000; i += 200) {
	1456	+ cfg_.rc_target_bitrate = i;
	1457	+ ResetModel();
	1458	+ denoiser_on_ = noise_sen;
	1459	+ assign_layer_bitrates(&cfg_, &svc_params_, cfg_.ss_number_layers,
	1460	+ cfg_.ts_number_layers, cfg_.temporal_layering_mode);
	1461	+ ASSERT_NO_FATAL_FAILURE(RunLoop(&video));
	1462	+ ASSERT_GE(cfg_.rc_target_bitrate, file_datarate_ * 0.78)
	1463	+ << " The datarate for the file exceeds the target by too much!";
	1464	+ ASSERT_LE(cfg_.rc_target_bitrate, file_datarate_ * 1.15)
	1465	+ << " The datarate for the file is lower than the target by too much!";
1463	1466	#if CONFIG_VP9_DECODER
1464		− // Number of temporal layers > 1, so half of the frames in this SVC pattern
1465		− // will be non-reference frame and hence encoder will avoid loopfilter.
1466		− // Since frame dropper is off, we can expcet 150 (half of the sequence)
1467		− // mismatched frames.
1468		− EXPECT_EQ(static_cast<unsigned int>(150), GetMismatchFrames());
	1467	+ // Number of temporal layers > 1, so half of the frames in this SVC
	1468	+ // pattern
	1469	+ // will be non-reference frame and hence encoder will avoid loopfilter.
	1470	+ // Since frame dropper is off, we can expcet 150 (half of the sequence)
	1471	+ // mismatched frames.
	1472	+ EXPECT_EQ(static_cast<unsigned int>(150), GetMismatchFrames());
1469	1473	#endif
	1474	+ }
1470	1475	}
1471	1476	}
1472	1477

vp9/encoder/vp9_denoiser.c+118−61

@@ -189,11 +189,12 @@ static VP9_DENOISER_DECISION perform_motion_compensation( · 188 unmodified lines
189	189	int increase_denoising, int mi_row, int mi_col, PICK_MODE_CONTEXT *ctx,
190	190	int motion_magnitude, int is_skin, int *zeromv_filter, int consec_zeromv,
191	191	int num_spatial_layers, int width, int lst_fb_idx, int gld_fb_idx,
192		− int use_svc) {
	192	+ int use_svc, int spatial_layer) {
193	193	const int sse_diff = (ctx->newmv_sse == UINT_MAX)
194	194	? 0
195	195	: ((int)ctx->zeromv_sse - (int)ctx->newmv_sse);
196	196	int frame;
	197	+ int denoise_layer_idx = 0;
197	198	MACROBLOCKD *filter_mbd = &mb->e_mbd;
198	199	MODE_INFO *mi = filter_mbd->mi[0];
199	200	MODE_INFO saved_mi;
@@ -254,6 +255,10 @@ static VP9_DENOISER_DECISION perform_motion_compensation( · 54 unmodified lines
254	255	frame = lst_fb_idx + 1;
255	256	else if (frame == GOLDEN_FRAME)
256	257	frame = gld_fb_idx + 1;
	258	+ // Shift for the second spatial layer.
	259	+ if (num_spatial_layers - spatial_layer == 2)
	260	+ frame = frame + denoiser->num_ref_frames;
	261	+ denoise_layer_idx = num_spatial_layers - spatial_layer - 1;
257	262	}
258	263
259	264	if (ctx->newmv_sse > sse_thresh(bs, increase_denoising)) {
@@ -289,18 +294,21 @@ static VP9_DENOISER_DECISION perform_motion_compensation( · 29 unmodified lines
289	294	denoiser->running_avg_y[frame].uv_stride, mi_row, mi_col);
290	295	filter_mbd->plane[2].pre[0].stride = denoiser->running_avg_y[frame].uv_stride;
291	296
292		− filter_mbd->plane[0].dst.buf =
293		− block_start(denoiser->mc_running_avg_y.y_buffer,
294		− denoiser->mc_running_avg_y.y_stride, mi_row, mi_col);
295		− filter_mbd->plane[0].dst.stride = denoiser->mc_running_avg_y.y_stride;
296		− filter_mbd->plane[1].dst.buf =
297		− block_start(denoiser->mc_running_avg_y.u_buffer,
298		− denoiser->mc_running_avg_y.uv_stride, mi_row, mi_col);
299		− filter_mbd->plane[1].dst.stride = denoiser->mc_running_avg_y.uv_stride;
300		− filter_mbd->plane[2].dst.buf =
301		− block_start(denoiser->mc_running_avg_y.v_buffer,
302		− denoiser->mc_running_avg_y.uv_stride, mi_row, mi_col);
303		− filter_mbd->plane[2].dst.stride = denoiser->mc_running_avg_y.uv_stride;
	297	+ filter_mbd->plane[0].dst.buf = block_start(
	298	+ denoiser->mc_running_avg_y[denoise_layer_idx].y_buffer,
	299	+ denoiser->mc_running_avg_y[denoise_layer_idx].y_stride, mi_row, mi_col);
	300	+ filter_mbd->plane[0].dst.stride =
	301	+ denoiser->mc_running_avg_y[denoise_layer_idx].y_stride;
	302	+ filter_mbd->plane[1].dst.buf = block_start(
	303	+ denoiser->mc_running_avg_y[denoise_layer_idx].u_buffer,
	304	+ denoiser->mc_running_avg_y[denoise_layer_idx].uv_stride, mi_row, mi_col);
	305	+ filter_mbd->plane[1].dst.stride =
	306	+ denoiser->mc_running_avg_y[denoise_layer_idx].uv_stride;
	307	+ filter_mbd->plane[2].dst.buf = block_start(
	308	+ denoiser->mc_running_avg_y[denoise_layer_idx].v_buffer,
	309	+ denoiser->mc_running_avg_y[denoise_layer_idx].uv_stride, mi_row, mi_col);
	310	+ filter_mbd->plane[2].dst.stride =
	311	+ denoiser->mc_running_avg_y[denoise_layer_idx].uv_stride;
304	312
305	313	set_ref_ptrs(cm, filter_mbd, saved_frame, NONE);
306	314	vp9_build_inter_predictors_sby(filter_mbd, mi_row, mi_col, bs);
@@ -324,9 +332,17 @@ void vp9_denoiser_denoise(VP9_COMP cpi, MACROBLOCK mb, int mi_row, int mi_col, · 17 unmodified lines
324	332	int zeromv_filter = 0;
325	333	VP9_DENOISER *denoiser = &cpi->denoiser;
326	334	VP9_DENOISER_DECISION decision = COPY_BLOCK;
327		− YV12_BUFFER_CONFIG avg = denoiser->running_avg_y[INTRA_FRAME];
328		− YV12_BUFFER_CONFIG mc_avg = denoiser->mc_running_avg_y;
	335	+
	336	+ const int shift =
	337	+ cpi->svc.number_spatial_layers - cpi->svc.spatial_layer_id == 2
	338	+ ? denoiser->num_ref_frames
	339	+ : 0;
	340	+ YV12_BUFFER_CONFIG avg = denoiser->running_avg_y[INTRA_FRAME + shift];
	341	+ const int denoise_layer_index =
	342	+ cpi->svc.number_spatial_layers - cpi->svc.spatial_layer_id - 1;
	343	+ YV12_BUFFER_CONFIG mc_avg = denoiser->mc_running_avg_y[denoise_layer_index];
329	344	uint8_t *avg_start = block_start(avg.y_buffer, avg.y_stride, mi_row, mi_col);
	345	+
330	346	uint8_t *mc_avg_start =
331	347	block_start(mc_avg.y_buffer, mc_avg.y_stride, mi_row, mi_col);
332	348	struct buf_2d src = mb->plane[0].src;
@@ -381,7 +397,7 @@ void vp9_denoiser_denoise(VP9_COMP cpi, MACROBLOCK mb, int mi_row, int mi_col, · 48 unmodified lines
381	397	&cpi->common, denoiser, mb, bs, increase_denoising, mi_row, mi_col, ctx,
382	398	motion_magnitude, is_skin, &zeromv_filter, consec_zeromv,
383	399	cpi->svc.number_spatial_layers, cpi->Source->y_width, cpi->lst_fb_idx,
384		− cpi->gld_fb_idx, cpi->use_svc);
	400	+ cpi->gld_fb_idx, cpi->use_svc, cpi->svc.spatial_layer_id);
385	401
386	402	if (decision == FILTER_BLOCK) {
387	403	decision = vp9_denoiser_filter(src.buf, src.stride, mc_avg_start,
@@ -432,7 +448,8 @@ void vp9_denoiser_update_frame_info( · 44 unmodified lines
432	448	VP9_DENOISER *denoiser, YV12_BUFFER_CONFIG src, FRAME_TYPE frame_type,
433	449	int refresh_alt_ref_frame, int refresh_golden_frame, int refresh_last_frame,
434	450	int alt_fb_idx, int gld_fb_idx, int lst_fb_idx, int resized,
435		− int svc_base_is_key) {
	451	+ int svc_base_is_key, int second_spatial_layer) {
	452	+ const int shift = second_spatial_layer ? denoiser->num_ref_frames : 0;
436	453	// Copy source into denoised reference buffers on KEY_FRAME or
437	454	// if the just encoded frame was resized. For SVC, copy source if the base
438	455	// spatial layer was key frame.
@@ -441,8 +458,8 @@ void vp9_denoiser_update_frame_info( · 2 unmodified lines
441	458	int i;
442	459	// Start at 1 so as not to overwrite the INTRA_FRAME
443	460	for (i = 1; i < denoiser->num_ref_frames; ++i) {
444		− if (denoiser->running_avg_y[i].buffer_alloc != NULL)
445		− copy_frame(&denoiser->running_avg_y[i], &src);
	461	+ if (denoiser->running_avg_y[i + shift].buffer_alloc != NULL)
	462	+ copy_frame(&denoiser->running_avg_y[i + shift], &src);
446	463	}
447	464	denoiser->reset = 0;
448	465	return;
@@ -451,29 +468,29 @@ void vp9_denoiser_update_frame_info( · 2 unmodified lines
451	468	// If more than one refresh occurs, must copy frame buffer.
452	469	if ((refresh_alt_ref_frame + refresh_golden_frame + refresh_last_frame) > 1) {
453	470	if (refresh_alt_ref_frame) {
454		− copy_frame(&denoiser->running_avg_y[alt_fb_idx + 1],
455		− &denoiser->running_avg_y[INTRA_FRAME]);
	471	+ copy_frame(&denoiser->running_avg_y[alt_fb_idx + 1 + shift],
	472	+ &denoiser->running_avg_y[INTRA_FRAME + shift]);
456	473	}
457	474	if (refresh_golden_frame) {
458		− copy_frame(&denoiser->running_avg_y[gld_fb_idx + 1],
459		− &denoiser->running_avg_y[INTRA_FRAME]);
	475	+ copy_frame(&denoiser->running_avg_y[gld_fb_idx + 1 + shift],
	476	+ &denoiser->running_avg_y[INTRA_FRAME + shift]);
460	477	}
461	478	if (refresh_last_frame) {
462		− copy_frame(&denoiser->running_avg_y[lst_fb_idx + 1],
463		− &denoiser->running_avg_y[INTRA_FRAME]);
	479	+ copy_frame(&denoiser->running_avg_y[lst_fb_idx + 1 + shift],
	480	+ &denoiser->running_avg_y[INTRA_FRAME + shift]);
464	481	}
465	482	} else {
466	483	if (refresh_alt_ref_frame) {
467		− swap_frame_buffer(&denoiser->running_avg_y[alt_fb_idx + 1],
468		− &denoiser->running_avg_y[INTRA_FRAME]);
	484	+ swap_frame_buffer(&denoiser->running_avg_y[alt_fb_idx + 1 + shift],
	485	+ &denoiser->running_avg_y[INTRA_FRAME + shift]);
469	486	}
470	487	if (refresh_golden_frame) {
471		− swap_frame_buffer(&denoiser->running_avg_y[gld_fb_idx + 1],
472		− &denoiser->running_avg_y[INTRA_FRAME]);
	488	+ swap_frame_buffer(&denoiser->running_avg_y[gld_fb_idx + 1 + shift],
	489	+ &denoiser->running_avg_y[INTRA_FRAME + shift]);
473	490	}
474	491	if (refresh_last_frame) {
475		− swap_frame_buffer(&denoiser->running_avg_y[lst_fb_idx + 1],
476		− &denoiser->running_avg_y[INTRA_FRAME]);
	492	+ swap_frame_buffer(&denoiser->running_avg_y[lst_fb_idx + 1 + shift],
	493	+ &denoiser->running_avg_y[INTRA_FRAME + shift]);
477	494	}
478	495	}
479	496	}
@@ -522,44 +539,90 @@ static int vp9_denoiser_realloc_svc_helper(VP9_COMMON *cm, · 42 unmodified lines
522	539	}
523	540
524	541	int vp9_denoiser_realloc_svc(VP9_COMMON cm, VP9_DENOISER denoiser,
525		− int refresh_alt, int refresh_gld, int refresh_lst,
526		− int alt_fb_idx, int gld_fb_idx, int lst_fb_idx) {
	542	+ int svc_buf_shift, int refresh_alt,
	543	+ int refresh_gld, int refresh_lst, int alt_fb_idx,
	544	+ int gld_fb_idx, int lst_fb_idx) {
527	545	int fail = 0;
528	546	if (refresh_alt) {
529	547	// Increase the frame buffer index by 1 to map it to the buffer index in the
530	548	// denoiser.
531		− fail = vp9_denoiser_realloc_svc_helper(cm, denoiser, alt_fb_idx + 1);
	549	+ fail = vp9_denoiser_realloc_svc_helper(cm, denoiser,
	550	+ alt_fb_idx + 1 + svc_buf_shift);
532	551	if (fail) return 1;
533	552	}
534	553	if (refresh_gld) {
535		− fail = vp9_denoiser_realloc_svc_helper(cm, denoiser, gld_fb_idx + 1);
	554	+ fail = vp9_denoiser_realloc_svc_helper(cm, denoiser,
	555	+ gld_fb_idx + 1 + svc_buf_shift);
536	556	if (fail) return 1;
537	557	}
538	558	if (refresh_lst) {
539		− fail = vp9_denoiser_realloc_svc_helper(cm, denoiser, lst_fb_idx + 1);
	559	+ fail = vp9_denoiser_realloc_svc_helper(cm, denoiser,
	560	+ lst_fb_idx + 1 + svc_buf_shift);
540	561	if (fail) return 1;
541	562	}
542	563	return 0;
543	564	}
544	565
545		−int vp9_denoiser_alloc(VP9_COMMON cm, int use_svc, VP9_DENOISER denoiser,
546		− int width, int height, int ssx, int ssy,
	566	+int vp9_denoiser_alloc(VP9_COMMON cm, struct SVC svc, VP9_DENOISER *denoiser,
	567	+ int use_svc, int noise_sen, int width, int height,
	568	+ int ssx, int ssy,
547	569	#if CONFIG_VP9_HIGHBITDEPTH
548	570	int use_highbitdepth,
549	571	#endif
550	572	int border) {
551		− int i, fail, init_num_ref_frames;
	573	+ int i, layer, fail, init_num_ref_frames;
552	574	const int legacy_byte_alignment = 0;
	575	+ int num_layers = 1;
	576	+ int scaled_width = width;
	577	+ int scaled_height = height;
	578	+ if (use_svc) {
	579	+ LAYER_CONTEXT lc = &svc->layer_context[svc->spatial_layer_id
	580	+ svc->number_temporal_layers +
	581	+ svc->temporal_layer_id];
	582	+ get_layer_resolution(width, height, lc->scaling_factor_num,
	583	+ lc->scaling_factor_den, &scaled_width, &scaled_height);
	584	+ // For SVC: only denoise at most 2 spatial (highest) layers.
	585	+ if (noise_sen >= 2)
	586	+ // Denoise from one spatial layer below the top.
	587	+ svc->first_layer_denoise = VPXMAX(svc->number_spatial_layers - 2, 0);
	588	+ else
	589	+ // Only denoise the top spatial layer.
	590	+ svc->first_layer_denoise = VPXMAX(svc->number_spatial_layers - 1, 0);
	591	+ num_layers = svc->number_spatial_layers - svc->first_layer_denoise;
	592	+ }
553	593	assert(denoiser != NULL);
554		−
555	594	denoiser->num_ref_frames = use_svc ? SVC_REF_FRAMES : NONSVC_REF_FRAMES;
556	595	init_num_ref_frames = use_svc ? MAX_REF_FRAMES : NONSVC_REF_FRAMES;
	596	+ denoiser->num_layers = num_layers;
	597	+ CHECK_MEM_ERROR(cm, denoiser->running_avg_y,
	598	+ vpx_calloc(denoiser->num_ref_frames * num_layers,
	599	+ sizeof(denoiser->running_avg_y[0])));
557	600	CHECK_MEM_ERROR(
558		− cm, denoiser->running_avg_y,
559		− vpx_calloc(denoiser->num_ref_frames, sizeof(denoiser->running_avg_y[0])));
560		− for (i = 0; i < init_num_ref_frames; ++i) {
561		− fail = vpx_alloc_frame_buffer(&denoiser->running_avg_y[i], width, height,
562		− ssx, ssy,
	601	+ cm, denoiser->mc_running_avg_y,
	602	+ vpx_calloc(num_layers, sizeof(denoiser->mc_running_avg_y[0])));
	603	+
	604	+ for (layer = 0; layer < num_layers; ++layer) {
	605	+ const int denoise_width = (layer == 0) ? width : scaled_width;
	606	+ const int denoise_height = (layer == 0) ? height : scaled_height;
	607	+ for (i = 0; i < init_num_ref_frames; ++i) {
	608	+ fail = vpx_alloc_frame_buffer(
	609	+ &denoiser->running_avg_y[i + denoiser->num_ref_frames * layer],
	610	+ denoise_width, denoise_height, ssx, ssy,
	611	+#if CONFIG_VP9_HIGHBITDEPTH
	612	+ use_highbitdepth,
	613	+#endif
	614	+ border, legacy_byte_alignment);
	615	+ if (fail) {
	616	+ vp9_denoiser_free(denoiser);
	617	+ return 1;
	618	+ }
	619	+#ifdef OUTPUT_YUV_DENOISED
	620	+ make_grayscale(&denoiser->running_avg_y[i]);
	621	+#endif
	622	+ }
	623	+
	624	+ fail = vpx_alloc_frame_buffer(&denoiser->mc_running_avg_y[layer],
	625	+ denoise_width, denoise_height, ssx, ssy,
563	626	#if CONFIG_VP9_HIGHBITDEPTH
564	627	use_highbitdepth,
565	628	#endif
@@ -568,22 +631,10 @@ int vp9_denoiser_alloc(VP9_COMMON cm, int use_svc, VP9_DENOISER denoiser, · 2 unmodified lines
568	631	vp9_denoiser_free(denoiser);
569	632	return 1;
570	633	}
571		−#ifdef OUTPUT_YUV_DENOISED
572		− make_grayscale(&denoiser->running_avg_y[i]);
573		−#endif
574		− }
575		−
576		− fail = vpx_alloc_frame_buffer(&denoiser->mc_running_avg_y, width, height, ssx,
577		− ssy,
578		−#if CONFIG_VP9_HIGHBITDEPTH
579		− use_highbitdepth,
580		−#endif
581		− border, legacy_byte_alignment);
582		− if (fail) {
583		− vp9_denoiser_free(denoiser);
584		− return 1;
585	634	}
586	635
	636	+ // denoiser->last_source only used for noise_estimation, so only for top
	637	+ // layer.
587	638	fail = vpx_alloc_frame_buffer(&denoiser->last_source, width, height, ssx, ssy,
588	639	#if CONFIG_VP9_HIGHBITDEPTH
589	640	use_highbitdepth,
@@ -609,12 +660,18 @@ void vp9_denoiser_free(VP9_DENOISER *denoiser) { · 19 unmodified lines
609	660	return;
610	661	}
611	662	denoiser->frame_buffer_initialized = 0;
612		− for (i = 0; i < denoiser->num_ref_frames; ++i) {
	663	+ for (i = 0; i < denoiser->num_ref_frames * denoiser->num_layers; ++i) {
613	664	vpx_free_frame_buffer(&denoiser->running_avg_y[i]);
614	665	}
615	666	vpx_free(denoiser->running_avg_y);
616	667	denoiser->running_avg_y = NULL;
617		− vpx_free_frame_buffer(&denoiser->mc_running_avg_y);
	668	+
	669	+ for (i = 0; i < denoiser->num_layers; ++i) {
	670	+ vpx_free_frame_buffer(&denoiser->mc_running_avg_y[i]);
	671	+ }
	672	+
	673	+ vpx_free(denoiser->mc_running_avg_y);
	674	+ denoiser->mc_running_avg_y = NULL;
618	675	vpx_free_frame_buffer(&denoiser->last_source);
619	676	}
620	677

vp9/encoder/vp9_denoiser.h+10−6

@@ -44,11 +44,12 @@ typedef enum vp9_denoiser_level { · 43 unmodified lines
44	44
45	45	typedef struct vp9_denoiser {
46	46	YV12_BUFFER_CONFIG *running_avg_y;
47		− YV12_BUFFER_CONFIG mc_running_avg_y;
	47	+ YV12_BUFFER_CONFIG *mc_running_avg_y;
48	48	YV12_BUFFER_CONFIG last_source;
49	49	int frame_buffer_initialized;
50	50	int reset;
51	51	int num_ref_frames;
	52	+ int num_layers;
52	53	VP9_DENOISER_LEVEL denoising_level;
53	54	VP9_DENOISER_LEVEL prev_denoising_level;
54	55	} VP9_DENOISER;
@@ -66,12 +67,13 @@ typedef struct { · 11 unmodified lines
66	67	} VP9_PICKMODE_CTX_DEN;
67	68
68	69	struct VP9_COMP;
	70	+struct SVC;
69	71
70	72	void vp9_denoiser_update_frame_info(
71	73	VP9_DENOISER *denoiser, YV12_BUFFER_CONFIG src, FRAME_TYPE frame_type,
72	74	int refresh_alt_ref_frame, int refresh_golden_frame, int refresh_last_frame,
73	75	int alt_fb_idx, int gld_fb_idx, int lst_fb_idx, int resized,
74		− int svc_base_is_key);
	76	+ int svc_base_is_key, int second_spatial_layer);
75	77
76	78	void vp9_denoiser_denoise(struct VP9_COMP cpi, MACROBLOCK mb, int mi_row,
77	79	int mi_col, BLOCK_SIZE bs, PICK_MODE_CONTEXT *ctx,
@@ -84,11 +86,13 @@ void vp9_denoiser_update_frame_stats(MODE_INFO *mi, unsigned int sse, · 6 unmodified lines
84	86	PICK_MODE_CONTEXT *ctx);
85	87
86	88	int vp9_denoiser_realloc_svc(VP9_COMMON cm, VP9_DENOISER denoiser,
87		− int refresh_alt, int refresh_gld, int refresh_lst,
88		− int alt_fb_idx, int gld_fb_idx, int lst_fb_idx);
	89	+ int svc_buf_shift, int refresh_alt,
	90	+ int refresh_gld, int refresh_lst, int alt_fb_idx,
	91	+ int gld_fb_idx, int lst_fb_idx);
89	92
90		−int vp9_denoiser_alloc(VP9_COMMON cm, int use_svc, VP9_DENOISER denoiser,
91		− int width, int height, int ssx, int ssy,
	93	+int vp9_denoiser_alloc(VP9_COMMON cm, struct SVC svc, VP9_DENOISER *denoiser,
	94	+ int use_svc, int noise_sen, int width, int height,
	95	+ int ssx, int ssy,
92	96	#if CONFIG_VP9_HIGHBITDEPTH
93	97	int use_highbitdepth,
94	98	#endif

vp9/encoder/vp9_encoder.c+16−6

@@ -2861,18 +2861,26 @@ void vp9_update_reference_frames(VP9_COMP *cpi) { · 2860 unmodified lines
2861	2861	if (cpi->oxcf.noise_sensitivity > 0 && denoise_svc(cpi) &&
2862	2862	cpi->denoiser.denoising_level > kDenLowLow) {
2863	2863	int svc_base_is_key = 0;
	2864	+ int denoise_svc_second_layer = 0;
2864	2865	if (cpi->use_svc) {
2865	2866	int realloc_fail = 0;
	2867	+ const int svc_buf_shift =
	2868	+ cpi->svc.number_spatial_layers - cpi->svc.spatial_layer_id == 2
	2869	+ ? cpi->denoiser.num_ref_frames
	2870	+ : 0;
2866	2871	int layer = LAYER_IDS_TO_IDX(cpi->svc.spatial_layer_id,
2867	2872	cpi->svc.temporal_layer_id,
2868	2873	cpi->svc.number_temporal_layers);
2869	2874	LAYER_CONTEXT *lc = &cpi->svc.layer_context[layer];
2870	2875	svc_base_is_key = lc->is_key_frame;
2871		−
2872		− // Check if we need to allocate extra buffers in the denoiser for
	2876	+ denoise_svc_second_layer =
	2877	+ cpi->svc.number_spatial_layers - cpi->svc.spatial_layer_id == 2 ? 1
	2878	+ : 0;
	2879	+ // Check if we need to allocate extra buffers in the denoiser
	2880	+ // for
2873	2881	// refreshed frames.
2874	2882	realloc_fail = vp9_denoiser_realloc_svc(
2875		− cm, &cpi->denoiser, cpi->refresh_alt_ref_frame,
	2883	+ cm, &cpi->denoiser, svc_buf_shift, cpi->refresh_alt_ref_frame,
2876	2884	cpi->refresh_golden_frame, cpi->refresh_last_frame, cpi->alt_fb_idx,
2877	2885	cpi->gld_fb_idx, cpi->lst_fb_idx);
2878	2886	if (realloc_fail)
@@ -2883,7 +2891,8 @@ void vp9_update_reference_frames(VP9_COMP *cpi) { · 4 unmodified lines
2883	2891	&cpi->denoiser, *cpi->Source, cpi->common.frame_type,
2884	2892	cpi->refresh_alt_ref_frame, cpi->refresh_golden_frame,
2885	2893	cpi->refresh_last_frame, cpi->alt_fb_idx, cpi->gld_fb_idx,
2886		− cpi->lst_fb_idx, cpi->resize_pending, svc_base_is_key);
	2894	+ cpi->lst_fb_idx, cpi->resize_pending, svc_base_is_key,
	2895	+ denoise_svc_second_layer);
2887	2896	}
2888	2897	#endif
2889	2898	if (is_one_pass_cbr_svc(cpi)) {
@@ -3318,8 +3327,9 @@ static void setup_denoiser_buffer(VP9_COMP *cpi) { · 428 unmodified lines
3318	3327	VP9_COMMON *const cm = &cpi->common;
3319	3328	if (cpi->oxcf.noise_sensitivity > 0 &&
3320	3329	!cpi->denoiser.frame_buffer_initialized) {
3321		− if (vp9_denoiser_alloc(cm, cpi->use_svc, &cpi->denoiser, cm->width,
3322		− cm->height, cm->subsampling_x, cm->subsampling_y,
	3330	+ if (vp9_denoiser_alloc(cm, &cpi->svc, &cpi->denoiser, cpi->use_svc,
	3331	+ cpi->oxcf.noise_sensitivity, cm->width, cm->height,
	3332	+ cm->subsampling_x, cm->subsampling_y,
3323	3333	#if CONFIG_VP9_HIGHBITDEPTH
3324	3334	cm->use_highbitdepth,
3325	3335	#endif

vp9/encoder/vp9_encoder.h+1−1

@@ -870,7 +870,7 @@ static INLINE int is_one_pass_cbr_svc(const struct VP9_COMP *const cpi) { · 869 unmodified lines
870	870	static INLINE int denoise_svc(const struct VP9_COMP *const cpi) {
871	871	return (!cpi->use_svc \|\|
872	872	(cpi->use_svc &&
873		− cpi->svc.spatial_layer_id == cpi->svc.number_spatial_layers - 1));
	873	+ cpi->svc.spatial_layer_id >= cpi->svc.first_layer_denoise));
874	874	}
875	875	#endif
876	876

vp9/encoder/vp9_noise_estimate.c+15−6

@@ -21,6 +21,15 @@ · 20 unmodified lines
21	21	#include "vp9/encoder/vp9_noise_estimate.h"
22	22	#include "vp9/encoder/vp9_encoder.h"
23	23
	24	+#if CONFIG_VP9_TEMPORAL_DENOISING
	25	+// For SVC: only do noise estimation on top spatial layer.
	26	+static INLINE int noise_est_svc(const struct VP9_COMP *const cpi) {
	27	+ return (!cpi->use_svc \|\|
	28	+ (cpi->use_svc &&
	29	+ cpi->svc.spatial_layer_id == cpi->svc.number_spatial_layers - 1));
	30	+}
	31	+#endif
	32	+
24	33	void vp9_noise_estimate_init(NOISE_ESTIMATE *const ne, int width, int height) {
25	34	ne->enabled = 0;
26	35	ne->level = kLowLow;
@@ -45,7 +54,7 @@ static int enable_noise_estimation(VP9_COMP *const cpi) { · 18 unmodified lines
45	54	#endif
46	55	// Enable noise estimation if denoising is on.
47	56	#if CONFIG_VP9_TEMPORAL_DENOISING
48		− if (cpi->oxcf.noise_sensitivity > 0 && denoise_svc(cpi) &&
	57	+ if (cpi->oxcf.noise_sensitivity > 0 && noise_est_svc(cpi) &&
49	58	cpi->common.width >= 320 && cpi->common.height >= 180)
50	59	return 1;
51	60	#endif
@@ -111,7 +120,7 @@ void vp9_update_noise_estimate(VP9_COMP *const cpi) { · 59 unmodified lines
111	120	// Estimate is between current source and last source.
112	121	YV12_BUFFER_CONFIG *last_source = cpi->Last_Source;
113	122	#if CONFIG_VP9_TEMPORAL_DENOISING
114		− if (cpi->oxcf.noise_sensitivity > 0 && denoise_svc(cpi)) {
	123	+ if (cpi->oxcf.noise_sensitivity > 0 && noise_est_svc(cpi)) {
115	124	last_source = &cpi->denoiser.last_source;
116	125	// Tune these thresholds for different resolutions when denoising is
117	126	// enabled.
@@ -131,7 +140,7 @@ void vp9_update_noise_estimate(VP9_COMP *const cpi) { · 13 unmodified lines
131	140	(cpi->svc.number_spatial_layers == 1 &&
132	141	(ne->last_w != cm->width \|\| ne->last_h != cm->height))) {
133	142	#if CONFIG_VP9_TEMPORAL_DENOISING
134		− if (cpi->oxcf.noise_sensitivity > 0 && denoise_svc(cpi))
	143	+ if (cpi->oxcf.noise_sensitivity > 0 && noise_est_svc(cpi))
135	144	copy_frame(&cpi->denoiser.last_source, cpi->Source);
136	145	#endif
137	146	if (last_source != NULL) {
@@ -146,7 +155,7 @@ void vp9_update_noise_estimate(VP9_COMP *const cpi) { · 8 unmodified lines
146	155	ne->count = 0;
147	156	ne->num_frames_estimate = 10;
148	157	#if CONFIG_VP9_TEMPORAL_DENOISING
149		− if (cpi->oxcf.noise_sensitivity > 0 && denoise_svc(cpi) &&
	158	+ if (cpi->oxcf.noise_sensitivity > 0 && noise_est_svc(cpi) &&
150	159	cpi->svc.current_superframe > 1) {
151	160	vp9_denoiser_set_noise_level(&cpi->denoiser, ne->level);
152	161	copy_frame(&cpi->denoiser.last_source, cpi->Source);
@@ -257,14 +266,14 @@ void vp9_update_noise_estimate(VP9_COMP *const cpi) { · 104 unmodified lines
257	266	ne->count = 0;
258	267	ne->level = vp9_noise_estimate_extract_level(ne);
259	268	#if CONFIG_VP9_TEMPORAL_DENOISING
260		− if (cpi->oxcf.noise_sensitivity > 0 && denoise_svc(cpi))
	269	+ if (cpi->oxcf.noise_sensitivity > 0 && noise_est_svc(cpi))
261	270	vp9_denoiser_set_noise_level(&cpi->denoiser, ne->level);
262	271	#endif
263	272	}
264	273	}
265	274	}
266	275	#if CONFIG_VP9_TEMPORAL_DENOISING
267		− if (cpi->oxcf.noise_sensitivity > 0 && denoise_svc(cpi))
	276	+ if (cpi->oxcf.noise_sensitivity > 0 && noise_est_svc(cpi))
268	277	copy_frame(&cpi->denoiser.last_source, cpi->Source);
269	278	#endif
270	279	}

vp9/encoder/vp9_svc_layercontext.c+4−3

@@ -37,6 +37,7 @@ void vp9_init_layer_context(VP9_COMP *const cpi) { · 36 unmodified lines
37	37	svc->scaled_one_half = 0;
38	38	svc->current_superframe = 0;
39	39	svc->non_reference_frame = 0;
	40	+
40	41	for (i = 0; i < REF_FRAMES; ++i) svc->ref_frame_index[i] = -1;
41	42	for (sl = 0; sl < oxcf->ss_number_layers; ++sl) {
42	43	svc->ext_frame_flags[sl] = 0;
@@ -389,9 +390,9 @@ int vp9_is_upper_layer_key_frame(const VP9_COMP *const cpi) { · 346 unmodified lines
389	390	.is_key_frame;
390	391	}
391	392
392		−static void get_layer_resolution(const int width_org, const int height_org,
393		− const int num, const int den, int *width_out,
394		− int *height_out) {
	393	+void get_layer_resolution(const int width_org, const int height_org,
	394	+ const int num, const int den, int *width_out,
	395	+ int *height_out) {
395	396	int w, h;
396	397
397	398	if (width_out == NULL \|\| height_out == NULL \|\| den == 0) return;

vp9/encoder/vp9_svc_layercontext.h+7−1

@@ -49,7 +49,7 @@ typedef struct { · 48 unmodified lines
49	49	uint8_t speed;
50	50	} LAYER_CONTEXT;
51	51
52		−typedef struct {
	52	+typedef struct SVC {
53	53	int spatial_layer_id;
54	54	int temporal_layer_id;
55	55	int number_spatial_layers;
@@ -99,6 +99,8 @@ typedef struct { · 43 unmodified lines
99	99
100	100	BLOCK_SIZE *prev_partition_svc;
101	101	int mi_stride[VPX_MAX_LAYERS];
	102	+
	103	+ int first_layer_denoise;
102	104	} SVC;
103	105
104	106	struct VP9_COMP;
@@ -128,6 +130,10 @@ void vp9_save_layer_context(struct VP9_COMP *const cpi); · 23 unmodified lines
128	130	// Initialize second pass rc for spatial svc.
129	131	void vp9_init_second_pass_spatial_svc(struct VP9_COMP *cpi);
130	132
	133	+void get_layer_resolution(const int width_org, const int height_org,
	134	+ const int num, const int den, int *width_out,
	135	+ int *height_out);
	136	+
131	137	// Increment number of video frames in layer
132	138	void vp9_inc_frame_in_layer(struct VP9_COMP *const cpi);
133	139

vpx/vp8cx.h+1−1

@@ -408,7 +408,7 @@ enum vp8e_enc_control_id { · 407 unmodified lines
408	408
409	409	/*!\brief Codec control function to set noise sensitivity.
410	410	*
411		− * 0: off, 1: On(YOnly)
	411	+ * 0: off, 1: On(YOnly), 2: For SVC only, on top two spatial layers(YOnly)
412	412	*
413	413	* Supported in codecs: VP9
414	414	*/

vp9 svc: Allow denoising next to highest resolution.

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code