avcodec/me_cmp, motion_est: Sanitize permissible cmp_funcs

Several of the potential choices of comparison functions need an initialized MpegEncContext (initialized for encoding, not only ff_mpv_common_init()) or they crash when called. Modify ff_set_cmp() to check for this.

Andreas Rheinhardt committed 2 years ago

Commit d163eef1 parent b1a31b38 files · +73 −77Browse files

Showing the changes to libavcodec (8 of 8 files). Show all

Unified · Split

8 files

libavcodec/dvenc.c +1 −1
libavcodec/me_cmp.c +52 −64
libavcodec/me_cmp.h +8 −1
libavcodec/motion_est.c +6 −5
libavcodec/motion_est.h +1 −1
libavcodec/mpegvideo_enc.c +3 −3
libavcodec/snowenc.c +1 −1
libavcodec/svq1enc.c +1 −1

libavcodec/dvenc.c+1−1

@@ -100,7 +100,7 @@ static av_cold int dvvideo_encode_init(AVCodecContext *avctx) · 99 unmodified lines
100	100
101	101	memset(&mecc,0, sizeof(mecc));
102	102	ff_me_cmp_init(&mecc, avctx);
103		− ret = ff_set_cmp(&mecc, ildct_cmp, avctx->ildct_cmp);
	103	+ ret = ff_set_cmp(&mecc, ildct_cmp, avctx->ildct_cmp, 0);
104	104	if (ret < 0)
105	105	return ret;
106	106	if (!ildct_cmp[5])

libavcodec/me_cmp.c+52−64

@@ -20,6 +20,8 @@ · 19 unmodified lines
20	20	* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
21	21	*/
22	22
	23	+#include <stddef.h>
	24	+
23	25	#include "libavutil/attributes.h"
24	26	#include "libavutil/internal.h"
25	27	#include "libavutil/mem_internal.h"
@@ -473,74 +475,60 @@ static int zero_cmp(MpegEncContext s, const uint8_t a, const uint8_t *b, · 447 unmodified lines
473	475	return 0;
474	476	}
475	477
476		−int ff_set_cmp(const MECmpContext c, me_cmp_func cmp, int type)
	478	+av_cold int ff_set_cmp(const MECmpContext c, me_cmp_func cmp, int type, int mpvenc)
477	479	{
478		− int ret = 0;
479		− int i;
480		−
481		− memset(cmp, 0, sizeof(void ) 6);
482		−
483		− for (i = 0; i < 6; i++) {
484		− switch (type & 0xFF) {
485		− case FF_CMP_SAD:
486		− cmp[i] = c->sad[i];
487		− break;
488		− case FF_CMP_MEDIAN_SAD:
489		− cmp[i] = c->median_sad[i];
490		− break;
491		− case FF_CMP_SATD:
492		− cmp[i] = c->hadamard8_diff[i];
493		− break;
494		− case FF_CMP_SSE:
495		− cmp[i] = c->sse[i];
496		− break;
497		− case FF_CMP_DCT:
498		− cmp[i] = c->dct_sad[i];
499		− break;
500		− case FF_CMP_DCT264:
501		− cmp[i] = c->dct264_sad[i];
502		− break;
503		− case FF_CMP_DCTMAX:
504		− cmp[i] = c->dct_max[i];
505		− break;
506		− case FF_CMP_PSNR:
507		− cmp[i] = c->quant_psnr[i];
508		− break;
509		− case FF_CMP_BIT:
510		− cmp[i] = c->bit[i];
511		− break;
512		− case FF_CMP_RD:
513		− cmp[i] = c->rd[i];
514		− break;
515		− case FF_CMP_VSAD:
516		− cmp[i] = c->vsad[i];
517		− break;
518		− case FF_CMP_VSSE:
519		− cmp[i] = c->vsse[i];
520		− break;
521		− case FF_CMP_ZERO:
522		− cmp[i] = zero_cmp;
523		− break;
524		− case FF_CMP_NSSE:
525		− cmp[i] = c->nsse[i];
526		− break;
527		−#if CONFIG_DWT
528		− case FF_CMP_W53:
529		− cmp[i]= c->w53[i];
530		− break;
531		− case FF_CMP_W97:
532		− cmp[i]= c->w97[i];
533		− break;
	480	+#define ENTRY(CMP_FLAG, ARRAY, MPVENC_ONLY) \
	481	+ [FF_CMP_ ## CMP_FLAG] = { \
	482	+ .offset = offsetof(MECmpContext, ARRAY), \
	483	+ .mpv_only = MPVENC_ONLY, \
	484	+ .available = 1, \
	485	+ }
	486	+ static const struct {
	487	+ char available;
	488	+ char mpv_only;
	489	+ uint16_t offset;
	490	+ } cmp_func_list[] = {
	491	+ ENTRY(SAD, sad, 0),
	492	+ ENTRY(SSE, sse, 0),
	493	+ ENTRY(SATD, hadamard8_diff, 0),
	494	+ ENTRY(DCT, dct_sad, 1),
	495	+ ENTRY(PSNR, quant_psnr, 1),
	496	+ ENTRY(BIT, bit, 1),
	497	+ ENTRY(RD, rd, 1),
	498	+ ENTRY(VSAD, vsad, 0),
	499	+ ENTRY(VSSE, vsse, 0),
	500	+ ENTRY(NSSE, nsse, 0),
	501	+#if CONFIG_SNOW_DECODER \|\| CONFIG_SNOW_ENCODER
	502	+ ENTRY(W53, w53, 0),
	503	+ ENTRY(W97, w97, 0),
534	504	#endif
535		− default:
536		− av_log(NULL, AV_LOG_ERROR,
537		− "invalid cmp function selection\n");
538		− ret = -1;
539		− break;
540		− }
	505	+ ENTRY(DCTMAX, dct_max, 1),
	506	+#if CONFIG_GPL
	507	+ ENTRY(DCT264, dct264_sad, 1),
	508	+#endif
	509	+ ENTRY(MEDIAN_SAD, median_sad, 0),
	510	+ };
	511	+ const me_cmp_func *me_cmp_func_array;
	512	+
	513	+ type &= 0xFF;
	514	+
	515	+ if (type == FF_CMP_ZERO) {
	516	+ for (int i = 0; i < 6; i++)
	517	+ cmp[i] = zero_cmp;
	518	+ return 0;
	519	+ }
	520	+ if (type > FF_ARRAY_ELEMS(cmp_func_list) \|\|
	521	+ !cmp_func_list[type].available \|\|
	522	+ !mpvenc && cmp_func_list[type].mpv_only) {
	523	+ av_log(NULL, AV_LOG_ERROR,
	524	+ "invalid cmp function selection\n");
	525	+ return AVERROR(EINVAL);
541	526	}
	527	+ me_cmp_func_array = (const me_cmp_func)(((const char)c) + cmp_func_list[type].offset);
	528	+ for (int i = 0; i < 6; i++)
	529	+ cmp[i] = me_cmp_func_array[i];
542	530
543		− return ret;
	531	+ return 0;
544	532	}
545	533
546	534	#define BUTTERFLY2(o1, o2, i1, i2) \

libavcodec/me_cmp.h+8−1

@@ -83,7 +83,14 @@ void ff_me_cmp_init_riscv(MECmpContext c, AVCodecContext avctx); · 82 unmodified lines
83	83	void ff_me_cmp_init_x86(MECmpContext c, AVCodecContext avctx);
84	84	void ff_me_cmp_init_mips(MECmpContext c, AVCodecContext avctx);
85	85
86		−int ff_set_cmp(const MECmpContext c, me_cmp_func cmp, int type);
	86	+/**
	87	+ * Fill the function pointer array cmp[6] with me_cmp_funcs from
	88	+ * c based upon type. If mpvenc is not set, an error is returned
	89	+ * if the type of comparison functions requires an initialized
	90	+ * MpegEncContext.
	91	+ */
	92	+int ff_set_cmp(const MECmpContext c, me_cmp_func cmp,
	93	+ int type, int mpvenc);
87	94
88	95	void ff_dsputil_init_dwt(MECmpContext *c);
89	96

libavcodec/motion_est.c+6−5

@@ -305,7 +305,8 @@ static int zero_cmp(MpegEncContext s, const uint8_t a, const uint8_t *b, · 304 unmodified lines
305	305	static void zero_hpel(uint8_t a, const uint8_t b, ptrdiff_t stride, int h){
306	306	}
307	307
308		−av_cold int ff_me_init(MotionEstContext c, AVCodecContext avctx, const MECmpContext *mecc)
	308	+av_cold int ff_me_init(MotionEstContext c, AVCodecContext avctx,
	309	+ const MECmpContext *mecc, int mpvenc)
309	310	{
310	311	int cache_size = FFMIN(ME_MAP_SIZE>>ME_MAP_SHIFT, 1<<ME_MAP_SHIFT);
311	312	int dia_size = FFMAX(FFABS(avctx->dia_size) & 255, FFABS(avctx->pre_dia_size) & 255);
@@ -324,10 +325,10 @@ av_cold int ff_me_init(MotionEstContext c, AVCodecContext avctx, const MECmpCo · 12 unmodified lines
324	325	if (cache_size < 2 * dia_size)
325	326	av_log(avctx, AV_LOG_INFO, "ME_MAP size may be a little small for the selected diamond size\n");
326	327
327		− ret = ff_set_cmp(mecc, c->me_pre_cmp, avctx->me_pre_cmp);
328		− ret \|= ff_set_cmp(mecc, c->me_cmp, avctx->me_cmp);
329		− ret \|= ff_set_cmp(mecc, c->me_sub_cmp, avctx->me_sub_cmp);
330		− ret \|= ff_set_cmp(mecc, c->mb_cmp, avctx->mb_cmp);
	328	+ ret = ff_set_cmp(mecc, c->me_pre_cmp, avctx->me_pre_cmp, mpvenc);
	329	+ ret \|= ff_set_cmp(mecc, c->me_cmp, avctx->me_cmp, mpvenc);
	330	+ ret \|= ff_set_cmp(mecc, c->me_sub_cmp, avctx->me_sub_cmp, mpvenc);
	331	+ ret \|= ff_set_cmp(mecc, c->mb_cmp, avctx->mb_cmp, mpvenc);
331	332	if (ret < 0)
332	333	return ret;
333	334

libavcodec/motion_est.h+1−1

@@ -115,7 +115,7 @@ static inline int ff_h263_round_chroma(int x) · 114 unmodified lines
115	115	* Performs one-time initialization of the MotionEstContext.
116	116	*/
117	117	int ff_me_init(MotionEstContext c, struct AVCodecContext avctx,
118		− const struct MECmpContext *mecc);
	118	+ const struct MECmpContext *mecc, int mpvenc);
119	119
120	120	void ff_me_init_pic(struct MpegEncContext *s);
121	121

libavcodec/mpegvideo_enc.c+3−3

@@ -312,15 +312,15 @@ static av_cold int me_cmp_init(MpegEncContext s, AVCodecContext avctx) · 311 unmodified lines
312	312	int ret;
313	313
314	314	ff_me_cmp_init(&s->mecc, avctx);
315		− ret = ff_me_init(&s->me, avctx, &s->mecc);
	315	+ ret = ff_me_init(&s->me, avctx, &s->mecc, 1);
316	316	if (ret < 0)
317	317	return ret;
318		− ret = ff_set_cmp(&s->mecc, me_cmp, s->frame_skip_cmp);
	318	+ ret = ff_set_cmp(&s->mecc, me_cmp, s->frame_skip_cmp, 1);
319	319	if (ret < 0)
320	320	return ret;
321	321	s->frame_skip_cmp_fn = me_cmp[1];
322	322	if (avctx->flags & AV_CODEC_FLAG_INTERLACED_DCT) {
323		− ret = ff_set_cmp(&s->mecc, me_cmp, avctx->ildct_cmp);
	323	+ ret = ff_set_cmp(&s->mecc, me_cmp, avctx->ildct_cmp, 1);
324	324	if (ret < 0)
325	325	return ret;
326	326	if (!me_cmp[0] \|\| !me_cmp[4])

libavcodec/snowenc.c+1−1

@@ -217,7 +217,7 @@ static av_cold int encode_init(AVCodecContext *avctx) · 216 unmodified lines
217	217	mcf(12,12)
218	218
219	219	ff_me_cmp_init(&enc->mecc, avctx);
220		− ret = ff_me_init(&enc->m.me, avctx, &enc->mecc);
	220	+ ret = ff_me_init(&enc->m.me, avctx, &enc->mecc, 0);
221	221	if (ret < 0)
222	222	return ret;
223	223	ff_mpegvideoencdsp_init(&enc->mpvencdsp, avctx);

libavcodec/svq1enc.c+1−1

@@ -589,7 +589,7 @@ static av_cold int svq1_encode_init(AVCodecContext *avctx) · 588 unmodified lines
589	589
590	590	ff_hpeldsp_init(&s->hdsp, avctx->flags);
591	591	ff_me_cmp_init(&s->mecc, avctx);
592		− ret = ff_me_init(&s->m.me, avctx, &s->mecc);
	592	+ ret = ff_me_init(&s->m.me, avctx, &s->mecc, 0);
593	593	if (ret < 0)
594	594	return ret;
595	595	ff_mpegvideoencdsp_init(&s->m.mpvencdsp, avctx);

avcodec/me_cmp, motion_est: Sanitize permissible cmp_funcs

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code