History

avcodec/dnxhdenc: retry increasing qscale to not overflow max_bits

Increase mb_bits type from uint16_t to uint32_t to fix possible overflows in bit size calculations. Update fate test that needs change.

Paul B Mahol committed 4 years ago

Commit 044c09c1 parent 37480b13 files · +8 −6Browse files

Unified · Split

3 files
  1. libavcodec/dnxhdenc.c +5 −3
  2. libavcodec/dnxhdenc.h +1 −1
  3. tests/ref/lavf/mxf_opatom +2 −2
libavcodec/dnxhdenc.c+5−3
@@ -924,7 +924,7 @@ static void dnxhd_setup_threads_slices(DNXHDEncContext *ctx) · 923 unmodified lines
924 unsigned mb = mb_y * ctx->m.mb_width + mb_x;924 unsigned mb = mb_y * ctx->m.mb_width + mb_x;
925 ctx->slice_size[mb_y] += ctx->mb_bits[mb];925 ctx->slice_size[mb_y] += ctx->mb_bits[mb];
926 }926 }
927 ctx->slice_size[mb_y] = (ctx->slice_size[mb_y] + 31) & ~31;927 ctx->slice_size[mb_y] = (ctx->slice_size[mb_y] + 31U) & ~31U;
928 ctx->slice_size[mb_y] >>= 3;928 ctx->slice_size[mb_y] >>= 3;
929 thread_size = ctx->slice_size[mb_y];929 thread_size = ctx->slice_size[mb_y];
930 offset += thread_size;930 offset += thread_size;
@@ -1220,17 +1220,19 @@ static int dnxhd_encode_fast(AVCodecContext *avctx, DNXHDEncContext *ctx) · 289 unmodified lines
1220 avctx->execute2(avctx, dnxhd_mb_var_thread,1220 avctx->execute2(avctx, dnxhd_mb_var_thread,
1221 NULL, NULL, ctx->m.mb_height);1221 NULL, NULL, ctx->m.mb_height);
1222 radix_sort(ctx->mb_cmp, ctx->mb_cmp_tmp, ctx->m.mb_num);1222 radix_sort(ctx->mb_cmp, ctx->mb_cmp_tmp, ctx->m.mb_num);
1223retry:
1223 for (x = 0; x < ctx->m.mb_num && max_bits > ctx->frame_bits; x++) {1224 for (x = 0; x < ctx->m.mb_num && max_bits > ctx->frame_bits; x++) {
1224 int mb = ctx->mb_cmp[x].mb;1225 int mb = ctx->mb_cmp[x].mb;
1225 int rc = (ctx->qscale * ctx->m.mb_num ) + mb;1226 int rc = (ctx->qscale * ctx->m.mb_num ) + mb;
1226 max_bits -= ctx->mb_rc[rc].bits -1227 max_bits -= ctx->mb_rc[rc].bits -
1227 ctx->mb_rc[rc + ctx->m.mb_num].bits;1228 ctx->mb_rc[rc + ctx->m.mb_num].bits;
1228 ctx->mb_qscale[mb] = ctx->qscale + 1;1229 if (ctx->mb_qscale[mb] < 255)
1230 ctx->mb_qscale[mb]++;
1229 ctx->mb_bits[mb] = ctx->mb_rc[rc + ctx->m.mb_num].bits;1231 ctx->mb_bits[mb] = ctx->mb_rc[rc + ctx->m.mb_num].bits;
1230 }1232 }
1231 1233
1232 if (max_bits > ctx->frame_bits)1234 if (max_bits > ctx->frame_bits)
1233 return AVERROR(EINVAL);1235 goto retry;
1234 }1236 }
1235 return 0;1237 return 0;
1236}1238}
libavcodec/dnxhdenc.h+1−1
@@ -100,7 +100,7 @@ typedef struct DNXHDEncContext { · 99 unmodified lines
100 unsigned qscale;100 unsigned qscale;
101 unsigned lambda;101 unsigned lambda;
102 102
103 uint16_t *mb_bits;103 uint32_t *mb_bits;
104 uint8_t *mb_qscale;104 uint8_t *mb_qscale;
105 105
106 RCCMPEntry *mb_cmp;106 RCCMPEntry *mb_cmp;
tests/ref/lavf/mxf_opatom+2−2
@@ -1,3 +1,3 @@
1aab6397829bd90f0c77a3f9fde53bb9c *tests/data/lavf/lavf.mxf_opatom1b1e32792b835ac51b0b1d5dcb9497e4e *tests/data/lavf/lavf.mxf_opatom
24717625 tests/data/lavf/lavf.mxf_opatom24717625 tests/data/lavf/lavf.mxf_opatom
3tests/data/lavf/lavf.mxf_opatom CRC=0xf55aa22a3tests/data/lavf/lavf.mxf_opatom CRC=0xb13ba2f8

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t