History

avcodec/dnxhdenc: retry increasing qscale to not overflow max_bits

Increase mb_bits type from uint16_t to uint32_t to fix possible overflows in bit size calculations. Update fate test that needs change.

Paul B Mahol committed 4 years ago

Commit 044c09c1 parent 37480b13 files · +8 −6Browse files

Unified · Split

3 files
  1. libavcodec/dnxhdenc.c +5 −3
  2. libavcodec/dnxhdenc.h +1 −1
  3. tests/ref/lavf/mxf_opatom +2 −2
libavcodec/dnxhdenc.c+5−3
@@ -924,7 +924,7 @@ static void dnxhd_setup_threads_slices(DNXHDEncContext *ctx) · 923 unmodified lines
924924 unsigned mb = mb_y * ctx->m.mb_width + mb_x;
925925 ctx->slice_size[mb_y] += ctx->mb_bits[mb];
926926 }
927− ctx->slice_size[mb_y] = (ctx->slice_size[mb_y] + 31) & ~31;
927+ ctx->slice_size[mb_y] = (ctx->slice_size[mb_y] + 31U) & ~31U;
928928 ctx->slice_size[mb_y] >>= 3;
929929 thread_size = ctx->slice_size[mb_y];
930930 offset += thread_size;
@@ -1220,17 +1220,19 @@ static int dnxhd_encode_fast(AVCodecContext *avctx, DNXHDEncContext *ctx) · 289 unmodified lines
12201220 avctx->execute2(avctx, dnxhd_mb_var_thread,
12211221 NULL, NULL, ctx->m.mb_height);
12221222 radix_sort(ctx->mb_cmp, ctx->mb_cmp_tmp, ctx->m.mb_num);
1223+retry:
12231224 for (x = 0; x < ctx->m.mb_num && max_bits > ctx->frame_bits; x++) {
12241225 int mb = ctx->mb_cmp[x].mb;
12251226 int rc = (ctx->qscale * ctx->m.mb_num ) + mb;
12261227 max_bits -= ctx->mb_rc[rc].bits -
12271228 ctx->mb_rc[rc + ctx->m.mb_num].bits;
1228− ctx->mb_qscale[mb] = ctx->qscale + 1;
1229+ if (ctx->mb_qscale[mb] < 255)
1230+ ctx->mb_qscale[mb]++;
12291231 ctx->mb_bits[mb] = ctx->mb_rc[rc + ctx->m.mb_num].bits;
12301232 }
12311233
12321234 if (max_bits > ctx->frame_bits)
1233− return AVERROR(EINVAL);
1235+ goto retry;
12341236 }
12351237 return 0;
12361238 }
libavcodec/dnxhdenc.h+1−1
@@ -100,7 +100,7 @@ typedef struct DNXHDEncContext { · 99 unmodified lines
100100 unsigned qscale;
101101 unsigned lambda;
102102
103− uint16_t *mb_bits;
103+ uint32_t *mb_bits;
104104 uint8_t *mb_qscale;
105105
106106 RCCMPEntry *mb_cmp;
tests/ref/lavf/mxf_opatom+2−2
@@ -1,3 +1,3 @@
1−aab6397829bd90f0c77a3f9fde53bb9c *tests/data/lavf/lavf.mxf_opatom
1+b1e32792b835ac51b0b1d5dcb9497e4e *tests/data/lavf/lavf.mxf_opatom
22 4717625 tests/data/lavf/lavf.mxf_opatom
3−tests/data/lavf/lavf.mxf_opatom CRC=0xf55aa22a
3+tests/data/lavf/lavf.mxf_opatom CRC=0xb13ba2f8

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t