History

avcodec/me_cmp, motion_est: Sanitize permissible cmp_funcs

Several of the potential choices of comparison functions need an initialized MpegEncContext (initialized for encoding, not only ff_mpv_common_init()) or they crash when called. Modify ff_set_cmp() to check for this.

Andreas Rheinhardt committed 2 years ago

Commit d163eef1 parent b1a31b38 files · +73 −77Browse files

Showing the changes to libavcodec (8 of 8 files). Show all

Unified · Split

8 files
  1. libavcodec/dvenc.c +1 −1
  2. libavcodec/me_cmp.c +52 −64
  3. libavcodec/me_cmp.h +8 −1
  4. libavcodec/motion_est.c +6 −5
  5. libavcodec/motion_est.h +1 −1
  6. libavcodec/mpegvideo_enc.c +3 −3
  7. libavcodec/snowenc.c +1 −1
  8. libavcodec/svq1enc.c +1 −1
libavcodec/dvenc.c+1−1
@@ -100,7 +100,7 @@ static av_cold int dvvideo_encode_init(AVCodecContext *avctx) · 99 unmodified lines
100 100
101 memset(&mecc,0, sizeof(mecc));101 memset(&mecc,0, sizeof(mecc));
102 ff_me_cmp_init(&mecc, avctx);102 ff_me_cmp_init(&mecc, avctx);
103 ret = ff_set_cmp(&mecc, ildct_cmp, avctx->ildct_cmp);103 ret = ff_set_cmp(&mecc, ildct_cmp, avctx->ildct_cmp, 0);
104 if (ret < 0)104 if (ret < 0)
105 return ret;105 return ret;
106 if (!ildct_cmp[5])106 if (!ildct_cmp[5])
libavcodec/me_cmp.c+52−64
@@ -20,6 +20,8 @@ · 19 unmodified lines
20 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA20 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
21 */21 */
22 22
23#include <stddef.h>
24
23#include "libavutil/attributes.h"25#include "libavutil/attributes.h"
24#include "libavutil/internal.h"26#include "libavutil/internal.h"
25#include "libavutil/mem_internal.h"27#include "libavutil/mem_internal.h"
@@ -473,74 +475,60 @@ static int zero_cmp(MpegEncContext *s, const uint8_t *a, const uint8_t *b, · 447 unmodified lines
473 return 0;475 return 0;
474}476}
475 477
476int ff_set_cmp(const MECmpContext *c, me_cmp_func *cmp, int type)478av_cold int ff_set_cmp(const MECmpContext *c, me_cmp_func *cmp, int type, int mpvenc)
477{479{
478 int ret = 0;480#define ENTRY(CMP_FLAG, ARRAY, MPVENC_ONLY) \
479 int i;481 [FF_CMP_ ## CMP_FLAG] = { \
480 482 .offset = offsetof(MECmpContext, ARRAY), \
481 memset(cmp, 0, sizeof(void *) * 6);483 .mpv_only = MPVENC_ONLY, \
482 484 .available = 1, \
483 for (i = 0; i < 6; i++) {485 }
484 switch (type & 0xFF) {486 static const struct {
485 case FF_CMP_SAD:487 char available;
486 cmp[i] = c->sad[i];488 char mpv_only;
487 break;489 uint16_t offset;
488 case FF_CMP_MEDIAN_SAD:490 } cmp_func_list[] = {
489 cmp[i] = c->median_sad[i];491 ENTRY(SAD, sad, 0),
490 break;492 ENTRY(SSE, sse, 0),
491 case FF_CMP_SATD:493 ENTRY(SATD, hadamard8_diff, 0),
492 cmp[i] = c->hadamard8_diff[i];494 ENTRY(DCT, dct_sad, 1),
493 break;495 ENTRY(PSNR, quant_psnr, 1),
494 case FF_CMP_SSE:496 ENTRY(BIT, bit, 1),
495 cmp[i] = c->sse[i];497 ENTRY(RD, rd, 1),
496 break;498 ENTRY(VSAD, vsad, 0),
497 case FF_CMP_DCT:499 ENTRY(VSSE, vsse, 0),
498 cmp[i] = c->dct_sad[i];500 ENTRY(NSSE, nsse, 0),
499 break;501#if CONFIG_SNOW_DECODER || CONFIG_SNOW_ENCODER
500 case FF_CMP_DCT264:502 ENTRY(W53, w53, 0),
501 cmp[i] = c->dct264_sad[i];503 ENTRY(W97, w97, 0),
502 break;
503 case FF_CMP_DCTMAX:
504 cmp[i] = c->dct_max[i];
505 break;
506 case FF_CMP_PSNR:
507 cmp[i] = c->quant_psnr[i];
508 break;
509 case FF_CMP_BIT:
510 cmp[i] = c->bit[i];
511 break;
512 case FF_CMP_RD:
513 cmp[i] = c->rd[i];
514 break;
515 case FF_CMP_VSAD:
516 cmp[i] = c->vsad[i];
517 break;
518 case FF_CMP_VSSE:
519 cmp[i] = c->vsse[i];
520 break;
521 case FF_CMP_ZERO:
522 cmp[i] = zero_cmp;
523 break;
524 case FF_CMP_NSSE:
525 cmp[i] = c->nsse[i];
526 break;
527#if CONFIG_DWT
528 case FF_CMP_W53:
529 cmp[i]= c->w53[i];
530 break;
531 case FF_CMP_W97:
532 cmp[i]= c->w97[i];
533 break;
534#endif504#endif
535 default:505 ENTRY(DCTMAX, dct_max, 1),
536 av_log(NULL, AV_LOG_ERROR,506#if CONFIG_GPL
537 "invalid cmp function selection\n");507 ENTRY(DCT264, dct264_sad, 1),
538 ret = -1;508#endif
539 break;509 ENTRY(MEDIAN_SAD, median_sad, 0),
540 }510 };
511 const me_cmp_func *me_cmp_func_array;
512
513 type &= 0xFF;
514
515 if (type == FF_CMP_ZERO) {
516 for (int i = 0; i < 6; i++)
517 cmp[i] = zero_cmp;
518 return 0;
519 }
520 if (type > FF_ARRAY_ELEMS(cmp_func_list) ||
521 !cmp_func_list[type].available ||
522 !mpvenc && cmp_func_list[type].mpv_only) {
523 av_log(NULL, AV_LOG_ERROR,
524 "invalid cmp function selection\n");
525 return AVERROR(EINVAL);
541 }526 }
527 me_cmp_func_array = (const me_cmp_func*)(((const char*)c) + cmp_func_list[type].offset);
528 for (int i = 0; i < 6; i++)
529 cmp[i] = me_cmp_func_array[i];
542 530
543 return ret;531 return 0;
544}532}
545 533
546#define BUTTERFLY2(o1, o2, i1, i2) \534#define BUTTERFLY2(o1, o2, i1, i2) \
libavcodec/me_cmp.h+8−1
@@ -83,7 +83,14 @@ void ff_me_cmp_init_riscv(MECmpContext *c, AVCodecContext *avctx); · 82 unmodified lines
83void ff_me_cmp_init_x86(MECmpContext *c, AVCodecContext *avctx);83void ff_me_cmp_init_x86(MECmpContext *c, AVCodecContext *avctx);
84void ff_me_cmp_init_mips(MECmpContext *c, AVCodecContext *avctx);84void ff_me_cmp_init_mips(MECmpContext *c, AVCodecContext *avctx);
85 85
86int ff_set_cmp(const MECmpContext *c, me_cmp_func *cmp, int type);86/**
87 * Fill the function pointer array cmp[6] with me_cmp_funcs from
88 * c based upon type. If mpvenc is not set, an error is returned
89 * if the type of comparison functions requires an initialized
90 * MpegEncContext.
91 */
92int ff_set_cmp(const MECmpContext *c, me_cmp_func *cmp,
93 int type, int mpvenc);
87 94
88void ff_dsputil_init_dwt(MECmpContext *c);95void ff_dsputil_init_dwt(MECmpContext *c);
89 96
libavcodec/motion_est.c+6−5
@@ -305,7 +305,8 @@ static int zero_cmp(MpegEncContext *s, const uint8_t *a, const uint8_t *b, · 304 unmodified lines
305static void zero_hpel(uint8_t *a, const uint8_t *b, ptrdiff_t stride, int h){305static void zero_hpel(uint8_t *a, const uint8_t *b, ptrdiff_t stride, int h){
306}306}
307 307
308av_cold int ff_me_init(MotionEstContext *c, AVCodecContext *avctx, const MECmpContext *mecc)308av_cold int ff_me_init(MotionEstContext *c, AVCodecContext *avctx,
309 const MECmpContext *mecc, int mpvenc)
309{310{
310 int cache_size = FFMIN(ME_MAP_SIZE>>ME_MAP_SHIFT, 1<<ME_MAP_SHIFT);311 int cache_size = FFMIN(ME_MAP_SIZE>>ME_MAP_SHIFT, 1<<ME_MAP_SHIFT);
311 int dia_size = FFMAX(FFABS(avctx->dia_size) & 255, FFABS(avctx->pre_dia_size) & 255);312 int dia_size = FFMAX(FFABS(avctx->dia_size) & 255, FFABS(avctx->pre_dia_size) & 255);
@@ -324,10 +325,10 @@ av_cold int ff_me_init(MotionEstContext *c, AVCodecContext *avctx, const MECmpCo · 12 unmodified lines
324 if (cache_size < 2 * dia_size)325 if (cache_size < 2 * dia_size)
325 av_log(avctx, AV_LOG_INFO, "ME_MAP size may be a little small for the selected diamond size\n");326 av_log(avctx, AV_LOG_INFO, "ME_MAP size may be a little small for the selected diamond size\n");
326 327
327 ret = ff_set_cmp(mecc, c->me_pre_cmp, avctx->me_pre_cmp);328 ret = ff_set_cmp(mecc, c->me_pre_cmp, avctx->me_pre_cmp, mpvenc);
328 ret |= ff_set_cmp(mecc, c->me_cmp, avctx->me_cmp);329 ret |= ff_set_cmp(mecc, c->me_cmp, avctx->me_cmp, mpvenc);
329 ret |= ff_set_cmp(mecc, c->me_sub_cmp, avctx->me_sub_cmp);330 ret |= ff_set_cmp(mecc, c->me_sub_cmp, avctx->me_sub_cmp, mpvenc);
330 ret |= ff_set_cmp(mecc, c->mb_cmp, avctx->mb_cmp);331 ret |= ff_set_cmp(mecc, c->mb_cmp, avctx->mb_cmp, mpvenc);
331 if (ret < 0)332 if (ret < 0)
332 return ret;333 return ret;
333 334
libavcodec/motion_est.h+1−1
@@ -115,7 +115,7 @@ static inline int ff_h263_round_chroma(int x) · 114 unmodified lines
115 * Performs one-time initialization of the MotionEstContext.115 * Performs one-time initialization of the MotionEstContext.
116 */116 */
117int ff_me_init(MotionEstContext *c, struct AVCodecContext *avctx,117int ff_me_init(MotionEstContext *c, struct AVCodecContext *avctx,
118 const struct MECmpContext *mecc);118 const struct MECmpContext *mecc, int mpvenc);
119 119
120void ff_me_init_pic(struct MpegEncContext *s);120void ff_me_init_pic(struct MpegEncContext *s);
121 121
libavcodec/mpegvideo_enc.c+3−3
@@ -312,15 +312,15 @@ static av_cold int me_cmp_init(MpegEncContext *s, AVCodecContext *avctx) · 311 unmodified lines
312 int ret;312 int ret;
313 313
314 ff_me_cmp_init(&s->mecc, avctx);314 ff_me_cmp_init(&s->mecc, avctx);
315 ret = ff_me_init(&s->me, avctx, &s->mecc);315 ret = ff_me_init(&s->me, avctx, &s->mecc, 1);
316 if (ret < 0)316 if (ret < 0)
317 return ret;317 return ret;
318 ret = ff_set_cmp(&s->mecc, me_cmp, s->frame_skip_cmp);318 ret = ff_set_cmp(&s->mecc, me_cmp, s->frame_skip_cmp, 1);
319 if (ret < 0)319 if (ret < 0)
320 return ret;320 return ret;
321 s->frame_skip_cmp_fn = me_cmp[1];321 s->frame_skip_cmp_fn = me_cmp[1];
322 if (avctx->flags & AV_CODEC_FLAG_INTERLACED_DCT) {322 if (avctx->flags & AV_CODEC_FLAG_INTERLACED_DCT) {
323 ret = ff_set_cmp(&s->mecc, me_cmp, avctx->ildct_cmp);323 ret = ff_set_cmp(&s->mecc, me_cmp, avctx->ildct_cmp, 1);
324 if (ret < 0)324 if (ret < 0)
325 return ret;325 return ret;
326 if (!me_cmp[0] || !me_cmp[4])326 if (!me_cmp[0] || !me_cmp[4])
libavcodec/snowenc.c+1−1
@@ -217,7 +217,7 @@ static av_cold int encode_init(AVCodecContext *avctx) · 216 unmodified lines
217 mcf(12,12)217 mcf(12,12)
218 218
219 ff_me_cmp_init(&enc->mecc, avctx);219 ff_me_cmp_init(&enc->mecc, avctx);
220 ret = ff_me_init(&enc->m.me, avctx, &enc->mecc);220 ret = ff_me_init(&enc->m.me, avctx, &enc->mecc, 0);
221 if (ret < 0)221 if (ret < 0)
222 return ret;222 return ret;
223 ff_mpegvideoencdsp_init(&enc->mpvencdsp, avctx);223 ff_mpegvideoencdsp_init(&enc->mpvencdsp, avctx);
libavcodec/svq1enc.c+1−1
@@ -589,7 +589,7 @@ static av_cold int svq1_encode_init(AVCodecContext *avctx) · 588 unmodified lines
589 589
590 ff_hpeldsp_init(&s->hdsp, avctx->flags);590 ff_hpeldsp_init(&s->hdsp, avctx->flags);
591 ff_me_cmp_init(&s->mecc, avctx);591 ff_me_cmp_init(&s->mecc, avctx);
592 ret = ff_me_init(&s->m.me, avctx, &s->mecc);592 ret = ff_me_init(&s->m.me, avctx, &s->mecc, 0);
593 if (ret < 0)593 if (ret < 0)
594 return ret;594 return ret;
595 ff_mpegvideoencdsp_init(&s->m.mpvencdsp, avctx);595 ff_mpegvideoencdsp_init(&s->m.mpvencdsp, avctx);

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t