Needed to avoid integer overflows for 24-bit encoding.
Unified · Split
| @@ -442,10 +442,10 @@ static void copy_samples(FlacEncodeContext *s, const int16_t *samples) · 441 unmodified lines | ||
| 442 | 442 | } |
| 443 | 443 | |
| 444 | 444 | |
| 445 | −static int rice_count_exact(int32_t *res, int n, int k) | |
| 445 | +static uint64_t rice_count_exact(int32_t *res, int n, int k) | |
| 446 | 446 | { |
| 447 | 447 | int i; |
| 448 | − int count = 0; | |
| 448 | + uint64_t count = 0; | |
| 449 | 449 | |
| 450 | 450 | for (i = 0; i < n; i++) { |
| 451 | 451 | int32_t v = -2 * res[i] - 1; |
| @@ -456,12 +456,12 @@ static int rice_count_exact(int32_t *res, int n, int k) · 4 unmodified lines | ||
| 456 | 456 | } |
| 457 | 457 | |
| 458 | 458 | |
| 459 | −static int subframe_count_exact(FlacEncodeContext *s, FlacSubframe *sub, | |
| 460 | − int pred_order) | |
| 459 | +static uint64_t subframe_count_exact(FlacEncodeContext *s, FlacSubframe *sub, | |
| 460 | + int pred_order) | |
| 461 | 461 | { |
| 462 | 462 | int p, porder, psize; |
| 463 | 463 | int i, part_end; |
| 464 | − int count = 0; | |
| 464 | + uint64_t count = 0; | |
| 465 | 465 | |
| 466 | 466 | /* subframe header */ |
| 467 | 467 | count += 8; |
| @@ -508,25 +508,25 @@ static int subframe_count_exact(FlacEncodeContext *s, FlacSubframe *sub, · 40 unmodified lines | ||
| 508 | 508 | /** |
| 509 | 509 | * Solve for d/dk(rice_encode_count) = n-((sum-(n>>1))>>(k+1)) = 0. |
| 510 | 510 | */ |
| 511 | −static int find_optimal_param(uint32_t sum, int n) | |
| 511 | +static int find_optimal_param(uint64_t sum, int n) | |
| 512 | 512 | { |
| 513 | 513 | int k; |
| 514 | − uint32_t sum2; | |
| 514 | + uint64_t sum2; | |
| 515 | 515 | |
| 516 | 516 | if (sum <= n >> 1) |
| 517 | 517 | return 0; |
| 518 | 518 | sum2 = sum - (n >> 1); |
| 519 | − k = av_log2(n < 256 ? FASTDIV(sum2, n) : sum2 / n); | |
| 519 | + k = av_log2(av_clipl_int32(sum2 / n)); | |
| 520 | 520 | return FFMIN(k, MAX_RICE_PARAM); |
| 521 | 521 | } |
| 522 | 522 | |
| 523 | 523 | |
| 524 | −static uint32_t calc_optimal_rice_params(RiceContext *rc, int porder, | |
| 525 | − uint32_t *sums, int n, int pred_order) | |
| 524 | +static uint64_t calc_optimal_rice_params(RiceContext *rc, int porder, | |
| 525 | + uint64_t *sums, int n, int pred_order) | |
| 526 | 526 | { |
| 527 | 527 | int i; |
| 528 | 528 | int k, cnt, part; |
| 529 | − uint32_t all_bits; | |
| 529 | + uint64_t all_bits; | |
| 530 | 530 | |
| 531 | 531 | part = (1 << porder); |
| 532 | 532 | all_bits = 4 * part; |
| @@ -546,7 +546,7 @@ static uint32_t calc_optimal_rice_params(RiceContext *rc, int porder, · 13 unmodified lines | ||
| 546 | 546 | |
| 547 | 547 | |
| 548 | 548 | static void calc_sums(int pmin, int pmax, uint32_t *data, int n, int pred_order, |
| 549 | − uint32_t sums[][MAX_PARTITIONS]) | |
| 549 | + uint64_t sums[][MAX_PARTITIONS]) | |
| 550 | 550 | { |
| 551 | 551 | int i, j; |
| 552 | 552 | int parts; |
| @@ -557,7 +557,7 @@ static void calc_sums(int pmin, int pmax, uint32_t *data, int n, int pred_order, · 4 unmodified lines | ||
| 557 | 557 | res = &data[pred_order]; |
| 558 | 558 | res_end = &data[n >> pmax]; |
| 559 | 559 | for (i = 0; i < parts; i++) { |
| 560 | − uint32_t sum = 0; | |
| 560 | + uint64_t sum = 0; | |
| 561 | 561 | while (res < res_end) |
| 562 | 562 | sum += *(res++); |
| 563 | 563 | sums[pmax][i] = sum; |
| @@ -572,15 +572,15 @@ static void calc_sums(int pmin, int pmax, uint32_t *data, int n, int pred_order, · 8 unmodified lines | ||
| 572 | 572 | } |
| 573 | 573 | |
| 574 | 574 | |
| 575 | −static uint32_t calc_rice_params(RiceContext *rc, int pmin, int pmax, | |
| 575 | +static uint64_t calc_rice_params(RiceContext *rc, int pmin, int pmax, | |
| 576 | 576 | int32_t *data, int n, int pred_order) |
| 577 | 577 | { |
| 578 | 578 | int i; |
| 579 | − uint32_t bits[MAX_PARTITION_ORDER+1]; | |
| 579 | + uint64_t bits[MAX_PARTITION_ORDER+1]; | |
| 580 | 580 | int opt_porder; |
| 581 | 581 | RiceContext tmp_rc; |
| 582 | 582 | uint32_t *udata; |
| 583 | − uint32_t sums[MAX_PARTITION_ORDER+1][MAX_PARTITIONS]; | |
| 583 | + uint64_t sums[MAX_PARTITION_ORDER+1][MAX_PARTITIONS]; | |
| 584 | 584 | |
| 585 | 585 | assert(pmin >= 0 && pmin <= MAX_PARTITION_ORDER); |
| 586 | 586 | assert(pmax >= 0 && pmax <= MAX_PARTITION_ORDER); |
| @@ -616,7 +616,7 @@ static int get_max_p_order(int max_porder, int n, int order) · 29 unmodified lines | ||
| 616 | 616 | } |
| 617 | 617 | |
| 618 | 618 | |
| 619 | −static uint32_t find_subframe_rice_params(FlacEncodeContext *s, | |
| 619 | +static uint64_t find_subframe_rice_params(FlacEncodeContext *s, | |
| 620 | 620 | FlacSubframe *sub, int pred_order) |
| 621 | 621 | { |
| 622 | 622 | int pmin = get_max_p_order(s->options.min_partition_order, |
| @@ -624,7 +624,7 @@ static uint32_t find_subframe_rice_params(FlacEncodeContext *s, · 1 unmodified line | ||
| 624 | 624 | int pmax = get_max_p_order(s->options.max_partition_order, |
| 625 | 625 | s->frame.blocksize, pred_order); |
| 626 | 626 | |
| 627 | − uint32_t bits = 8 + pred_order * sub->obits + 2 + 4; | |
| 627 | + uint64_t bits = 8 + pred_order * sub->obits + 2 + 4; | |
| 628 | 628 | if (sub->type == FLAC_SUBFRAME_LPC) |
| 629 | 629 | bits += 4 + 5 + pred_order * s->options.lpc_coeff_precision; |
| 630 | 630 | bits += calc_rice_params(&sub->rc, pmin, pmax, sub->residual, |
| @@ -829,7 +829,7 @@ static int encode_residual_ch(FlacEncodeContext *s, int ch) · 198 unmodified lines | ||
| 829 | 829 | sub->type = FLAC_SUBFRAME_FIXED; |
| 830 | 830 | if (s->options.lpc_type == FF_LPC_TYPE_NONE || |
| 831 | 831 | s->options.lpc_type == FF_LPC_TYPE_FIXED || n <= max_order) { |
| 832 | − uint32_t bits[MAX_FIXED_ORDER+1]; | |
| 832 | + uint64_t bits[MAX_FIXED_ORDER+1]; | |
| 833 | 833 | if (max_order > MAX_FIXED_ORDER) |
| 834 | 834 | max_order = MAX_FIXED_ORDER; |
| 835 | 835 | opt_order = 0; |
| @@ -860,7 +860,7 @@ static int encode_residual_ch(FlacEncodeContext *s, int ch) · 24 unmodified lines | ||
| 860 | 860 | omethod == ORDER_METHOD_4LEVEL || |
| 861 | 861 | omethod == ORDER_METHOD_8LEVEL) { |
| 862 | 862 | int levels = 1 << omethod; |
| 863 | − uint32_t bits[1 << ORDER_METHOD_8LEVEL]; | |
| 863 | + uint64_t bits[1 << ORDER_METHOD_8LEVEL]; | |
| 864 | 864 | int order; |
| 865 | 865 | int opt_index = levels-1; |
| 866 | 866 | opt_order = max_order-1; |
| @@ -879,7 +879,7 @@ static int encode_residual_ch(FlacEncodeContext *s, int ch) · 12 unmodified lines | ||
| 879 | 879 | opt_order++; |
| 880 | 880 | } else if (omethod == ORDER_METHOD_SEARCH) { |
| 881 | 881 | // brute-force optimal order search |
| 882 | − uint32_t bits[MAX_LPC_ORDER]; | |
| 882 | + uint64_t bits[MAX_LPC_ORDER]; | |
| 883 | 883 | opt_order = 0; |
| 884 | 884 | bits[0] = UINT32_MAX; |
| 885 | 885 | for (i = min_order-1; i < max_order; i++) { |
| @@ -890,7 +890,7 @@ static int encode_residual_ch(FlacEncodeContext *s, int ch) · 4 unmodified lines | ||
| 890 | 890 | } |
| 891 | 891 | opt_order++; |
| 892 | 892 | } else if (omethod == ORDER_METHOD_LOG) { |
| 893 | − uint32_t bits[MAX_LPC_ORDER]; | |
| 893 | + uint64_t bits[MAX_LPC_ORDER]; | |
| 894 | 894 | int step; |
| 895 | 895 | |
| 896 | 896 | opt_order = min_order - 1 + (max_order-min_order)/3; |
| @@ -962,7 +962,8 @@ static int count_frame_header(FlacEncodeContext *s) · 65 unmodified lines | ||
| 962 | 962 | |
| 963 | 963 | static int encode_frame(FlacEncodeContext *s) |
| 964 | 964 | { |
| 965 | − int ch, count; | |
| 965 | + int ch; | |
| 966 | + uint64_t count; | |
| 966 | 967 | |
| 967 | 968 | count = count_frame_header(s); |
| 968 | 969 | |
| @@ -972,7 +973,10 @@ static int encode_frame(FlacEncodeContext *s) · 3 unmodified lines | ||
| 972 | 973 | count += (8 - (count & 7)) & 7; // byte alignment |
| 973 | 974 | count += 16; // CRC-16 |
| 974 | 975 | |
| 975 | − return count >> 3; | |
| 976 | + count >>= 3; | |
| 977 | + if (count > INT_MAX) | |
| 978 | + return AVERROR_BUG; | |
| 979 | + return count; | |
| 976 | 980 | } |
| 977 | 981 | |
| 978 | 982 | |
| @@ -1274,9 +1278,13 @@ static int flac_encode_frame(AVCodecContext *avctx, AVPacket *avpkt, · 295 unmodified lines | ||
| 1274 | 1278 | |
| 1275 | 1279 | /* fallback to verbatim mode if the compressed frame is larger than it |
| 1276 | 1280 | would be if encoded uncompressed. */ |
| 1277 | − if (frame_bytes > s->max_framesize) { | |
| 1281 | + if (frame_bytes < 0 || frame_bytes > s->max_framesize) { | |
| 1278 | 1282 | s->frame.verbatim_only = 1; |
| 1279 | 1283 | frame_bytes = encode_frame(s); |
| 1284 | + if (frame_bytes < 0) { | |
| 1285 | + av_log(avctx, AV_LOG_ERROR, "Bad frame count\n"); | |
| 1286 | + return frame_bytes; | |
| 1287 | + } | |
| 1280 | 1288 | } |
| 1281 | 1289 | |
| 1282 | 1290 | if ((ret = ff_alloc_packet(avpkt, frame_bytes))) { |