This also affected other users of VLCs from msmpeg4dec, namely vc1_block and wmv2dec.
Unified · Split
| @@ -59,7 +59,7 @@ void ff_msmpeg4_encode_mb(MpegEncContext *s, int16_t block[6][64], · 58 unmodified lines | |||
| 59 | int ff_msmpeg4_decode_init(AVCodecContext *avctx); | 59 | int ff_msmpeg4_decode_init(AVCodecContext *avctx); |
| 60 | int ff_msmpeg4_decode_picture_header(MpegEncContext *s); | 60 | int ff_msmpeg4_decode_picture_header(MpegEncContext *s); |
| 61 | int ff_msmpeg4_decode_ext_header(MpegEncContext *s, int buf_size); | 61 | int ff_msmpeg4_decode_ext_header(MpegEncContext *s, int buf_size); |
| 62 | int ff_msmpeg4_decode_motion(MpegEncContext * s, int *mx_ptr, int *my_ptr); | 62 | void ff_msmpeg4_decode_motion(MpegEncContext * s, int *mx_ptr, int *my_ptr); |
| 63 | int ff_msmpeg4_decode_block(MpegEncContext * s, int16_t * block, | 63 | int ff_msmpeg4_decode_block(MpegEncContext * s, int16_t * block, |
| 64 | int n, int coded, const uint8_t *scan_table); | 64 | int n, int coded, const uint8_t *scan_table); |
| 65 | int ff_msmpeg4_pred_dc(MpegEncContext *s, int n, | 65 | int ff_msmpeg4_pred_dc(MpegEncContext *s, int n, |
| @@ -228,8 +228,6 @@ static int msmpeg4v34_decode_mb(MpegEncContext *s, int16_t block[6][64]) · 227 unmodified lines | |||
| 228 | } | 228 | } |
| 229 | 229 | ||
| 230 | code = get_vlc2(&s->gb, ff_mb_non_intra_vlc[DEFAULT_INTER_INDEX].table, MB_NON_INTRA_VLC_BITS, 3); | 230 | code = get_vlc2(&s->gb, ff_mb_non_intra_vlc[DEFAULT_INTER_INDEX].table, MB_NON_INTRA_VLC_BITS, 3); |
| 231 | if (code < 0) | ||
| 232 | return -1; | ||
| 233 | //s->mb_intra = (code & 0x40) ? 0 : 1; | 231 | //s->mb_intra = (code & 0x40) ? 0 : 1; |
| 234 | s->mb_intra = (~code & 0x40) >> 6; | 232 | s->mb_intra = (~code & 0x40) >> 6; |
| 235 | 233 | ||
| @@ -237,8 +235,6 @@ static int msmpeg4v34_decode_mb(MpegEncContext *s, int16_t block[6][64]) · 1 unmodified line | |||
| 237 | } else { | 235 | } else { |
| 238 | s->mb_intra = 1; | 236 | s->mb_intra = 1; |
| 239 | code = get_vlc2(&s->gb, ff_msmp4_mb_i_vlc.table, MB_INTRA_VLC_BITS, 2); | 237 | code = get_vlc2(&s->gb, ff_msmp4_mb_i_vlc.table, MB_INTRA_VLC_BITS, 2); |
| 240 | if (code < 0) | ||
| 241 | return -1; | ||
| 242 | /* predict coded block pattern */ | 238 | /* predict coded block pattern */ |
| 243 | cbp = 0; | 239 | cbp = 0; |
| 244 | for(i=0;i<6;i++) { | 240 | for(i=0;i<6;i++) { |
| @@ -259,8 +255,7 @@ static int msmpeg4v34_decode_mb(MpegEncContext *s, int16_t block[6][64]) · 14 unmodified lines | |||
| 259 | s->rl_chroma_table_index = s->rl_table_index; | 255 | s->rl_chroma_table_index = s->rl_table_index; |
| 260 | } | 256 | } |
| 261 | ff_h263_pred_motion(s, 0, 0, &mx, &my); | 257 | ff_h263_pred_motion(s, 0, 0, &mx, &my); |
| 262 | if (ff_msmpeg4_decode_motion(s, &mx, &my) < 0) | 258 | ff_msmpeg4_decode_motion(s, &mx, &my); |
| 263 | return -1; | ||
| 264 | s->mv_dir = MV_DIR_FORWARD; | 259 | s->mv_dir = MV_DIR_FORWARD; |
| 265 | s->mv_type = MV_TYPE_16X16; | 260 | s->mv_type = MV_TYPE_16X16; |
| 266 | s->mv[0][0][0] = mx; | 261 | s->mv[0][0][0] = mx; |
| @@ -612,11 +607,6 @@ static int msmpeg4_decode_dc(MpegEncContext * s, int n, int *dir_ptr) · 345 unmodified lines | |||
| 612 | } else { | 607 | } else { |
| 613 | level = get_vlc2(&s->gb, ff_msmp4_dc_chroma_vlc[s->dc_table_index].table, DC_VLC_BITS, 3); | 608 | level = get_vlc2(&s->gb, ff_msmp4_dc_chroma_vlc[s->dc_table_index].table, DC_VLC_BITS, 3); |
| 614 | } | 609 | } |
| 615 | if (level < 0){ | ||
| 616 | av_log(s->avctx, AV_LOG_ERROR, "illegal dc vlc\n"); | ||
| 617 | *dir_ptr = 0; | ||
| 618 | return -1; | ||
| 619 | } | ||
| 620 | 610 | ||
| 621 | if (level == DC_MAX) { | 611 | if (level == DC_MAX) { |
| 622 | level = get_bits(&s->gb, 8); | 612 | level = get_bits(&s->gb, 8); |
| @@ -838,8 +828,7 @@ int ff_msmpeg4_decode_block(MpegEncContext * s, int16_t * block, · 215 unmodified lines | |||
| 838 | return 0; | 828 | return 0; |
| 839 | } | 829 | } |
| 840 | 830 | ||
| 841 | int ff_msmpeg4_decode_motion(MpegEncContext * s, | 831 | void ff_msmpeg4_decode_motion(MpegEncContext *s, int *mx_ptr, int *my_ptr) |
| 842 | int *mx_ptr, int *my_ptr) | ||
| 843 | { | 832 | { |
| 844 | MVTable *mv; | 833 | MVTable *mv; |
| 845 | int code, mx, my; | 834 | int code, mx, my; |
| @@ -847,10 +836,6 @@ int ff_msmpeg4_decode_motion(MpegEncContext * s, · 1 unmodified line | |||
| 847 | mv = &ff_mv_tables[s->mv_table_index]; | 836 | mv = &ff_mv_tables[s->mv_table_index]; |
| 848 | 837 | ||
| 849 | code = get_vlc2(&s->gb, mv->vlc.table, MV_VLC_BITS, 2); | 838 | code = get_vlc2(&s->gb, mv->vlc.table, MV_VLC_BITS, 2); |
| 850 | if (code < 0){ | ||
| 851 | av_log(s->avctx, AV_LOG_ERROR, "illegal MV code at %d %d\n", s->mb_x, s->mb_y); | ||
| 852 | return -1; | ||
| 853 | } | ||
| 854 | if (code == mv->n) { | 839 | if (code == mv->n) { |
| 855 | mx = get_bits(&s->gb, 6); | 840 | mx = get_bits(&s->gb, 6); |
| 856 | my = get_bits(&s->gb, 6); | 841 | my = get_bits(&s->gb, 6); |
| @@ -873,7 +858,6 @@ int ff_msmpeg4_decode_motion(MpegEncContext * s, · 16 unmodified lines | |||
| 873 | my -= 64; | 858 | my -= 64; |
| 874 | *mx_ptr = mx; | 859 | *mx_ptr = mx; |
| 875 | *my_ptr = my; | 860 | *my_ptr = my; |
| 876 | return 0; | ||
| 877 | } | 861 | } |
| 878 | 862 | ||
| 879 | AVCodec ff_msmpeg4v1_decoder = { | 863 | AVCodec ff_msmpeg4v1_decoder = { |
| @@ -592,10 +592,6 @@ static int vc1_decode_i_block(VC1Context *v, int16_t block[64], int n, · 591 unmodified lines | |||
| 592 | } else { | 592 | } else { |
| 593 | dcdiff = get_vlc2(&s->gb, ff_msmp4_dc_chroma_vlc[s->dc_table_index].table, DC_VLC_BITS, 3); | 593 | dcdiff = get_vlc2(&s->gb, ff_msmp4_dc_chroma_vlc[s->dc_table_index].table, DC_VLC_BITS, 3); |
| 594 | } | 594 | } |
| 595 | if (dcdiff < 0) { | ||
| 596 | av_log(s->avctx, AV_LOG_ERROR, "Illegal DC VLC\n"); | ||
| 597 | return -1; | ||
| 598 | } | ||
| 599 | if (dcdiff) { | 595 | if (dcdiff) { |
| 600 | const int m = (v->pq == 1 || v->pq == 2) ? 3 - v->pq : 0; | 596 | const int m = (v->pq == 1 || v->pq == 2) ? 3 - v->pq : 0; |
| 601 | if (dcdiff == 119 /* ESC index value */) { | 597 | if (dcdiff == 119 /* ESC index value */) { |
| @@ -740,10 +736,6 @@ static int vc1_decode_i_block_adv(VC1Context *v, int16_t block[64], int n, · 138 unmodified lines | |||
| 740 | } else { | 736 | } else { |
| 741 | dcdiff = get_vlc2(&s->gb, ff_msmp4_dc_chroma_vlc[s->dc_table_index].table, DC_VLC_BITS, 3); | 737 | dcdiff = get_vlc2(&s->gb, ff_msmp4_dc_chroma_vlc[s->dc_table_index].table, DC_VLC_BITS, 3); |
| 742 | } | 738 | } |
| 743 | if (dcdiff < 0) { | ||
| 744 | av_log(s->avctx, AV_LOG_ERROR, "Illegal DC VLC\n"); | ||
| 745 | return -1; | ||
| 746 | } | ||
| 747 | if (dcdiff) { | 739 | if (dcdiff) { |
| 748 | const int m = (quant == 1 || quant == 2) ? 3 - quant : 0; | 740 | const int m = (quant == 1 || quant == 2) ? 3 - quant : 0; |
| 749 | if (dcdiff == 119 /* ESC index value */) { | 741 | if (dcdiff == 119 /* ESC index value */) { |
| @@ -944,10 +936,6 @@ static int vc1_decode_intra_block(VC1Context *v, int16_t block[64], int n, · 194 unmodified lines | |||
| 944 | } else { | 936 | } else { |
| 945 | dcdiff = get_vlc2(&s->gb, ff_msmp4_dc_chroma_vlc[s->dc_table_index].table, DC_VLC_BITS, 3); | 937 | dcdiff = get_vlc2(&s->gb, ff_msmp4_dc_chroma_vlc[s->dc_table_index].table, DC_VLC_BITS, 3); |
| 946 | } | 938 | } |
| 947 | if (dcdiff < 0) { | ||
| 948 | av_log(s->avctx, AV_LOG_ERROR, "Illegal DC VLC\n"); | ||
| 949 | return -1; | ||
| 950 | } | ||
| 951 | if (dcdiff) { | 939 | if (dcdiff) { |
| 952 | const int m = (quant == 1 || quant == 2) ? 3 - quant : 0; | 940 | const int m = (quant == 1 || quant == 2) ? 3 - quant : 0; |
| 953 | if (dcdiff == 119 /* ESC index value */) { | 941 | if (dcdiff == 119 /* ESC index value */) { |
| @@ -278,22 +278,16 @@ int ff_wmv2_decode_secondary_picture_header(MpegEncContext *s) · 277 unmodified lines | |||
| 278 | return 0; | 278 | return 0; |
| 279 | } | 279 | } |
| 280 | 280 | ||
| 281 | static inline int wmv2_decode_motion(Wmv2Context *w, int *mx_ptr, int *my_ptr) | 281 | static inline void wmv2_decode_motion(Wmv2Context *w, int *mx_ptr, int *my_ptr) |
| 282 | { | 282 | { |
| 283 | MpegEncContext *const s = &w->s; | 283 | MpegEncContext *const s = &w->s; |
| 284 | int ret; | ||
| 285 | |||
| 286 | ret = ff_msmpeg4_decode_motion(s, mx_ptr, my_ptr); | ||
| 287 | 284 | ||
| 288 | if (ret < 0) | 285 | ff_msmpeg4_decode_motion(s, mx_ptr, my_ptr); |
| 289 | return ret; | ||
| 290 | 286 | ||
| 291 | if ((((*mx_ptr) | (*my_ptr)) & 1) && s->mspel) | 287 | if ((((*mx_ptr) | (*my_ptr)) & 1) && s->mspel) |
| 292 | w->hshift = get_bits1(&s->gb); | 288 | w->hshift = get_bits1(&s->gb); |
| 293 | else | 289 | else |
| 294 | w->hshift = 0; | 290 | w->hshift = 0; |
| 295 | |||
| 296 | return 0; | ||
| 297 | } | 291 | } |
| 298 | 292 | ||
| 299 | static int16_t *wmv2_pred_motion(Wmv2Context *w, int *px, int *py) | 293 | static int16_t *wmv2_pred_motion(Wmv2Context *w, int *px, int *py) |
| @@ -409,8 +403,6 @@ int ff_wmv2_decode_mb(MpegEncContext *s, int16_t block[6][64]) · 109 unmodified lines | |||
| 409 | 403 | ||
| 410 | code = get_vlc2(&s->gb, ff_mb_non_intra_vlc[w->cbp_table_index].table, | 404 | code = get_vlc2(&s->gb, ff_mb_non_intra_vlc[w->cbp_table_index].table, |
| 411 | MB_NON_INTRA_VLC_BITS, 3); | 405 | MB_NON_INTRA_VLC_BITS, 3); |
| 412 | if (code < 0) | ||
| 413 | return AVERROR_INVALIDDATA; | ||
| 414 | s->mb_intra = (~code & 0x40) >> 6; | 406 | s->mb_intra = (~code & 0x40) >> 6; |
| 415 | 407 | ||
| 416 | cbp = code & 0x3f; | 408 | cbp = code & 0x3f; |
| @@ -419,11 +411,6 @@ int ff_wmv2_decode_mb(MpegEncContext *s, int16_t block[6][64]) · 2 unmodified lines | |||
| 419 | if (get_bits_left(&s->gb) <= 0) | 411 | if (get_bits_left(&s->gb) <= 0) |
| 420 | return AVERROR_INVALIDDATA; | 412 | return AVERROR_INVALIDDATA; |
| 421 | code = get_vlc2(&s->gb, ff_msmp4_mb_i_vlc.table, MB_INTRA_VLC_BITS, 2); | 413 | code = get_vlc2(&s->gb, ff_msmp4_mb_i_vlc.table, MB_INTRA_VLC_BITS, 2); |
| 422 | if (code < 0) { | ||
| 423 | av_log(s->avctx, AV_LOG_ERROR, | ||
| 424 | "II-cbp illegal at %d %d\n", s->mb_x, s->mb_y); | ||
| 425 | return AVERROR_INVALIDDATA; | ||
| 426 | } | ||
| 427 | /* predict coded block pattern */ | 414 | /* predict coded block pattern */ |
| 428 | cbp = 0; | 415 | cbp = 0; |
| 429 | for (i = 0; i < 6; i++) { | 416 | for (i = 0; i < 6; i++) { |
| @@ -456,8 +443,7 @@ int ff_wmv2_decode_mb(MpegEncContext *s, int16_t block[6][64]) · 26 unmodified lines | |||
| 456 | w->per_block_abt = 0; | 443 | w->per_block_abt = 0; |
| 457 | } | 444 | } |
| 458 | 445 | ||
| 459 | if ((ret = wmv2_decode_motion(w, &mx, &my)) < 0) | 446 | wmv2_decode_motion(w, &mx, &my); |
| 460 | return ret; | ||
| 461 | 447 | ||
| 462 | s->mv_dir = MV_DIR_FORWARD; | 448 | s->mv_dir = MV_DIR_FORWARD; |
| 463 | s->mv_type = MV_TYPE_16X16; | 449 | s->mv_type = MV_TYPE_16X16; |