see https://fate.ffmpeg.org/report.cgi?slot=x86_64-archlinux-gcc-valgrind&time=20240105201935 If tc is zero, the max_len_q, max_len_p are uninitialized.
Unified · Split
| @@ -551,81 +551,80 @@ static void FUNC(vvc_loop_filter_luma)(uint8_t* _pix, ptrdiff_t _xstride, ptrdif · 550 unmodified lines | ||
| 551 | 551 | const ptrdiff_t ystride = _ystride / sizeof(pixel); |
| 552 | 552 | |
| 553 | 553 | for (int i = 0; i < 2; i++) { |
| 554 | − pixel* pix = (pixel*)_pix + i * 4 * ystride; | |
| 555 | − const int dp0 = abs(P2 - 2 * P1 + P0); | |
| 556 | − const int dq0 = abs(Q2 - 2 * Q1 + Q0); | |
| 557 | − const int dp3 = abs(TP2 - 2 * TP1 + TP0); | |
| 558 | − const int dq3 = abs(TQ2 - 2 * TQ1 + TQ0); | |
| 559 | − const int d0 = dp0 + dq0; | |
| 560 | − const int d3 = dp3 + dq3; | |
| 561 | 554 | #if BIT_DEPTH < 10 |
| 562 | 555 | const int tc = (_tc[i] + (1 << (9 - BIT_DEPTH))) >> (10 - BIT_DEPTH); |
| 563 | 556 | #else |
| 564 | 557 | const int tc = _tc[i] << (BIT_DEPTH - 10); |
| 565 | 558 | #endif |
| 566 | − const int tc25 = ((tc * 5 + 1) >> 1); | |
| 567 | − | |
| 568 | − const int no_p = _no_p[i]; | |
| 569 | − const int no_q = _no_q[i]; | |
| 570 | − | |
| 571 | − int max_len_p = _max_len_p[i]; | |
| 572 | − int max_len_q = _max_len_q[i]; | |
| 573 | − | |
| 574 | − const int large_p = (max_len_p > 3 && !hor_ctu_edge); | |
| 575 | − const int large_q = max_len_q > 3; | |
| 576 | − const int beta = _beta[i] << BIT_DEPTH - 8; | |
| 577 | − | |
| 578 | − const int beta_3 = beta >> 3; | |
| 579 | − const int beta_2 = beta >> 2; | |
| 580 | − | |
| 581 | − if (!tc) | |
| 582 | − continue; | |
| 583 | − | |
| 584 | − if (large_p || large_q) { | |
| 585 | − const int dp0l = large_p ? ((dp0 + abs(P5 - 2 * P4 + P3) + 1) >> 1) : dp0; | |
| 586 | − const int dq0l = large_q ? ((dq0 + abs(Q5 - 2 * Q4 + Q3) + 1) >> 1) : dq0; | |
| 587 | − const int dp3l = large_p ? ((dp3 + abs(TP5 - 2 * TP4 + TP3) + 1) >> 1) : dp3; | |
| 588 | − const int dq3l = large_q ? ((dq3 + abs(TQ5 - 2 * TQ4 + TQ3) + 1) >> 1) : dq3; | |
| 589 | − const int d0l = dp0l + dq0l; | |
| 590 | − const int d3l = dp3l + dq3l; | |
| 591 | − const int beta53 = beta * 3 >> 5; | |
| 592 | − const int beta_4 = beta >> 4; | |
| 593 | − max_len_p = large_p ? max_len_p : 3; | |
| 594 | − max_len_q = large_q ? max_len_q : 3; | |
| 595 | − | |
| 596 | − if (d0l + d3l < beta) { | |
| 597 | − const int sp0l = abs(P3 - P0) + (max_len_p == 7 ? abs(P7 - P6 - P5 + P4) : 0); | |
| 598 | − const int sq0l = abs(Q0 - Q3) + (max_len_q == 7 ? abs(Q4 - Q5 - Q6 + Q7) : 0); | |
| 599 | − const int sp3l = abs(TP3 - TP0) + (max_len_p == 7 ? abs(TP7 - TP6 - TP5 + TP4) : 0); | |
| 600 | − const int sq3l = abs(TQ0 - TQ3) + (max_len_q == 7 ? abs(TQ4 - TQ5 - TQ6 + TQ7) : 0); | |
| 601 | − const int sp0 = large_p ? ((sp0l + abs(P3 - P(max_len_p)) + 1) >> 1) : sp0l; | |
| 602 | − const int sp3 = large_p ? ((sp3l + abs(TP3 - TP(max_len_p)) + 1) >> 1) : sp3l; | |
| 603 | − const int sq0 = large_q ? ((sq0l + abs(Q3 - Q(max_len_q)) + 1) >> 1) : sq0l; | |
| 604 | − const int sq3 = large_q ? ((sq3l + abs(TQ3 - TQ(max_len_q)) + 1) >> 1) : sq3l; | |
| 605 | − if (sp0 + sq0 < beta53 && abs(P0 - Q0) < tc25 && | |
| 606 | − sp3 + sq3 < beta53 && abs(TP0 - TQ0) < tc25 && | |
| 607 | − (d0l << 1) < beta_4 && (d3l << 1) < beta_4) { | |
| 608 | − FUNC(loop_filter_luma_large)(pix, xstride, ystride, tc, no_p, no_q, max_len_p, max_len_q); | |
| 609 | − continue; | |
| 559 | + if (tc) { | |
| 560 | + pixel* pix = (pixel*)_pix + i * 4 * ystride; | |
| 561 | + const int dp0 = abs(P2 - 2 * P1 + P0); | |
| 562 | + const int dq0 = abs(Q2 - 2 * Q1 + Q0); | |
| 563 | + const int dp3 = abs(TP2 - 2 * TP1 + TP0); | |
| 564 | + const int dq3 = abs(TQ2 - 2 * TQ1 + TQ0); | |
| 565 | + const int d0 = dp0 + dq0; | |
| 566 | + const int d3 = dp3 + dq3; | |
| 567 | + const int tc25 = ((tc * 5 + 1) >> 1); | |
| 568 | + | |
| 569 | + const int no_p = _no_p[i]; | |
| 570 | + const int no_q = _no_q[i]; | |
| 571 | + | |
| 572 | + int max_len_p = _max_len_p[i]; | |
| 573 | + int max_len_q = _max_len_q[i]; | |
| 574 | + | |
| 575 | + const int large_p = (max_len_p > 3 && !hor_ctu_edge); | |
| 576 | + const int large_q = max_len_q > 3; | |
| 577 | + | |
| 578 | + const int beta = _beta[i] << BIT_DEPTH - 8; | |
| 579 | + const int beta_3 = beta >> 3; | |
| 580 | + const int beta_2 = beta >> 2; | |
| 581 | + | |
| 582 | + if (large_p || large_q) { | |
| 583 | + const int dp0l = large_p ? ((dp0 + abs(P5 - 2 * P4 + P3) + 1) >> 1) : dp0; | |
| 584 | + const int dq0l = large_q ? ((dq0 + abs(Q5 - 2 * Q4 + Q3) + 1) >> 1) : dq0; | |
| 585 | + const int dp3l = large_p ? ((dp3 + abs(TP5 - 2 * TP4 + TP3) + 1) >> 1) : dp3; | |
| 586 | + const int dq3l = large_q ? ((dq3 + abs(TQ5 - 2 * TQ4 + TQ3) + 1) >> 1) : dq3; | |
| 587 | + const int d0l = dp0l + dq0l; | |
| 588 | + const int d3l = dp3l + dq3l; | |
| 589 | + const int beta53 = beta * 3 >> 5; | |
| 590 | + const int beta_4 = beta >> 4; | |
| 591 | + max_len_p = large_p ? max_len_p : 3; | |
| 592 | + max_len_q = large_q ? max_len_q : 3; | |
| 593 | + | |
| 594 | + if (d0l + d3l < beta) { | |
| 595 | + const int sp0l = abs(P3 - P0) + (max_len_p == 7 ? abs(P7 - P6 - P5 + P4) : 0); | |
| 596 | + const int sq0l = abs(Q0 - Q3) + (max_len_q == 7 ? abs(Q4 - Q5 - Q6 + Q7) : 0); | |
| 597 | + const int sp3l = abs(TP3 - TP0) + (max_len_p == 7 ? abs(TP7 - TP6 - TP5 + TP4) : 0); | |
| 598 | + const int sq3l = abs(TQ0 - TQ3) + (max_len_q == 7 ? abs(TQ4 - TQ5 - TQ6 + TQ7) : 0); | |
| 599 | + const int sp0 = large_p ? ((sp0l + abs(P3 - P(max_len_p)) + 1) >> 1) : sp0l; | |
| 600 | + const int sp3 = large_p ? ((sp3l + abs(TP3 - TP(max_len_p)) + 1) >> 1) : sp3l; | |
| 601 | + const int sq0 = large_q ? ((sq0l + abs(Q3 - Q(max_len_q)) + 1) >> 1) : sq0l; | |
| 602 | + const int sq3 = large_q ? ((sq3l + abs(TQ3 - TQ(max_len_q)) + 1) >> 1) : sq3l; | |
| 603 | + if (sp0 + sq0 < beta53 && abs(P0 - Q0) < tc25 && | |
| 604 | + sp3 + sq3 < beta53 && abs(TP0 - TQ0) < tc25 && | |
| 605 | + (d0l << 1) < beta_4 && (d3l << 1) < beta_4) { | |
| 606 | + FUNC(loop_filter_luma_large)(pix, xstride, ystride, tc, no_p, no_q, max_len_p, max_len_q); | |
| 607 | + continue; | |
| 608 | + } | |
| 610 | 609 | } |
| 611 | 610 | } |
| 612 | − } | |
| 613 | − if (d0 + d3 < beta) { | |
| 614 | − if (max_len_p > 2 && max_len_q > 2 && | |
| 615 | − abs(P3 - P0) + abs(Q3 - Q0) < beta_3 && abs(P0 - Q0) < tc25 && | |
| 616 | − abs(TP3 - TP0) + abs(TQ3 - TQ0) < beta_3 && abs(TP0 - TQ0) < tc25 && | |
| 617 | − (d0 << 1) < beta_2 && (d3 << 1) < beta_2) { | |
| 618 | − FUNC(loop_filter_luma_strong)(pix, xstride, ystride, tc, tc << 1, tc * 3, no_p, no_q); | |
| 619 | − } else { // weak filtering | |
| 620 | − int nd_p = 1; | |
| 621 | − int nd_q = 1; | |
| 622 | − if (max_len_p > 1 && max_len_q > 1) { | |
| 623 | − if (dp0 + dp3 < ((beta + (beta >> 1)) >> 3)) | |
| 624 | − nd_p = 2; | |
| 625 | − if (dq0 + dq3 < ((beta + (beta >> 1)) >> 3)) | |
| 626 | − nd_q = 2; | |
| 611 | + if (d0 + d3 < beta) { | |
| 612 | + if (max_len_p > 2 && max_len_q > 2 && | |
| 613 | + abs(P3 - P0) + abs(Q3 - Q0) < beta_3 && abs(P0 - Q0) < tc25 && | |
| 614 | + abs(TP3 - TP0) + abs(TQ3 - TQ0) < beta_3 && abs(TP0 - TQ0) < tc25 && | |
| 615 | + (d0 << 1) < beta_2 && (d3 << 1) < beta_2) { | |
| 616 | + FUNC(loop_filter_luma_strong)(pix, xstride, ystride, tc, tc << 1, tc * 3, no_p, no_q); | |
| 617 | + } else { | |
| 618 | + int nd_p = 1; | |
| 619 | + int nd_q = 1; | |
| 620 | + if (max_len_p > 1 && max_len_q > 1) { | |
| 621 | + if (dp0 + dp3 < ((beta + (beta >> 1)) >> 3)) | |
| 622 | + nd_p = 2; | |
| 623 | + if (dq0 + dq3 < ((beta + (beta >> 1)) >> 3)) | |
| 624 | + nd_q = 2; | |
| 625 | + } | |
| 626 | + FUNC(loop_filter_luma_weak)(pix, xstride, ystride, tc, beta, no_p, no_q, nd_p, nd_q); | |
| 627 | 627 | } |
| 628 | − FUNC(loop_filter_luma_weak)(pix, xstride, ystride, tc, beta, no_p, no_q, nd_p, nd_q); | |
| 629 | 628 | } |
| 630 | 629 | } |
| 631 | 630 | } |
| @@ -689,65 +688,68 @@ static void FUNC(vvc_loop_filter_chroma)(uint8_t *_pix, const ptrdiff_t _xstrid · 57 unmodified lines | ||
| 689 | 688 | const int end = 8 / size; // 8 samples a loop |
| 690 | 689 | |
| 691 | 690 | for (int i = 0; i < end; i++) { |
| 692 | − pixel *pix = (pixel *)_pix + i * size * ystride; | |
| 693 | − const uint8_t no_p = _no_p[i]; | |
| 694 | − const uint8_t no_q = _no_q[i]; | |
| 695 | − const int beta = _beta[i] << (BIT_DEPTH - 8); | |
| 696 | − const int beta_3 = beta >> 3; | |
| 697 | − const int beta_2 = beta >> 2; | |
| 698 | − | |
| 699 | 691 | #if BIT_DEPTH < 10 |
| 700 | 692 | const int tc = (_tc[i] + (1 << (9 - BIT_DEPTH))) >> (10 - BIT_DEPTH); |
| 701 | 693 | #else |
| 702 | 694 | const int tc = _tc[i] << (BIT_DEPTH - 10); |
| 703 | 695 | #endif |
| 704 | − const int tc25 = ((tc * 5 + 1) >> 1); | |
| 705 | − | |
| 706 | − uint8_t max_len_p = _max_len_p[i]; | |
| 707 | − uint8_t max_len_q = _max_len_q[i]; | |
| 708 | − | |
| 709 | − if (!max_len_p || !max_len_q || !tc) | |
| 710 | − continue; | |
| 711 | − | |
| 712 | − if (max_len_q == 3){ | |
| 713 | − const int p1n = shift ? FP1 : TP1; | |
| 714 | − const int p2n = max_len_p == 1 ? p1n : (shift ? FP2 : TP2); | |
| 715 | − const int p0n = shift ? FP0 : TP0; | |
| 716 | − const int q0n = shift ? FQ0 : TQ0; | |
| 717 | − const int q1n = shift ? FQ1 : TQ1; | |
| 718 | − const int q2n = shift ? FQ2 : TQ2; | |
| 719 | − const int p3 = max_len_p == 1 ? P1 : P3; | |
| 720 | − const int p2 = max_len_p == 1 ? P1 : P2; | |
| 721 | − const int p1 = P1; | |
| 722 | − const int p0 = P0; | |
| 723 | − const int dp0 = abs(p2 - 2 * p1 + p0); | |
| 724 | − const int dq0 = abs(Q2 - 2 * Q1 + Q0); | |
| 696 | + if (tc) { | |
| 697 | + pixel *pix = (pixel *)_pix + i * size * ystride; | |
| 698 | + const uint8_t no_p = _no_p[i]; | |
| 699 | + const uint8_t no_q = _no_q[i]; | |
| 700 | + | |
| 701 | + const int beta = _beta[i] << (BIT_DEPTH - 8); | |
| 702 | + const int beta_3 = beta >> 3; | |
| 703 | + const int beta_2 = beta >> 2; | |
| 704 | + | |
| 705 | + const int tc25 = ((tc * 5 + 1) >> 1); | |
| 706 | + | |
| 707 | + uint8_t max_len_p = _max_len_p[i]; | |
| 708 | + uint8_t max_len_q = _max_len_q[i]; | |
| 725 | 709 | |
| 726 | − const int dp1 = abs(p2n - 2 * p1n + p0n); | |
| 727 | − const int dq1 = abs(q2n - 2 * q1n + q0n); | |
| 728 | − const int d0 = dp0 + dq0; | |
| 729 | − const int d1 = dp1 + dq1; | |
| 730 | − | |
| 731 | − if (d0 + d1 < beta) { | |
| 732 | − const int p3n = max_len_p == 1 ? p1n : (shift ? FP3 : TP3); | |
| 733 | − const int q3n = shift ? FQ3 : TQ3; | |
| 734 | − const int dsam0 = (d0 << 1) < beta_2 && (abs(p3 - p0) + abs(Q0 - Q3) < beta_3) && | |
| 735 | − abs(p0 - Q0) < tc25; | |
| 736 | − const int dsam1 = (d1 << 1) < beta_2 && (abs(p3n - p0n) + abs(q0n - q3n) < beta_3) && | |
| 737 | − abs(p0n - q0n) < tc25; | |
| 738 | − if (!dsam0 || !dsam1) | |
| 710 | + if (!max_len_p || !max_len_q) | |
| 711 | + continue; | |
| 712 | + | |
| 713 | + if (max_len_q == 3){ | |
| 714 | + const int p1n = shift ? FP1 : TP1; | |
| 715 | + const int p2n = max_len_p == 1 ? p1n : (shift ? FP2 : TP2); | |
| 716 | + const int p0n = shift ? FP0 : TP0; | |
| 717 | + const int q0n = shift ? FQ0 : TQ0; | |
| 718 | + const int q1n = shift ? FQ1 : TQ1; | |
| 719 | + const int q2n = shift ? FQ2 : TQ2; | |
| 720 | + const int p3 = max_len_p == 1 ? P1 : P3; | |
| 721 | + const int p2 = max_len_p == 1 ? P1 : P2; | |
| 722 | + const int p1 = P1; | |
| 723 | + const int p0 = P0; | |
| 724 | + const int dp0 = abs(p2 - 2 * p1 + p0); | |
| 725 | + const int dq0 = abs(Q2 - 2 * Q1 + Q0); | |
| 726 | + | |
| 727 | + const int dp1 = abs(p2n - 2 * p1n + p0n); | |
| 728 | + const int dq1 = abs(q2n - 2 * q1n + q0n); | |
| 729 | + const int d0 = dp0 + dq0; | |
| 730 | + const int d1 = dp1 + dq1; | |
| 731 | + | |
| 732 | + if (d0 + d1 < beta) { | |
| 733 | + const int p3n = max_len_p == 1 ? p1n : (shift ? FP3 : TP3); | |
| 734 | + const int q3n = shift ? FQ3 : TQ3; | |
| 735 | + const int dsam0 = (d0 << 1) < beta_2 && (abs(p3 - p0) + abs(Q0 - Q3) < beta_3) && | |
| 736 | + abs(p0 - Q0) < tc25; | |
| 737 | + const int dsam1 = (d1 << 1) < beta_2 && (abs(p3n - p0n) + abs(q0n - q3n) < beta_3) && | |
| 738 | + abs(p0n - q0n) < tc25; | |
| 739 | + if (!dsam0 || !dsam1) | |
| 740 | + max_len_p = max_len_q = 1; | |
| 741 | + } else { | |
| 739 | 742 | max_len_p = max_len_q = 1; |
| 740 | − } else { | |
| 741 | − max_len_p = max_len_q = 1; | |
| 743 | + } | |
| 742 | 744 | } |
| 743 | − } | |
| 744 | 745 | |
| 745 | − if (max_len_p == 3 && max_len_q == 3) | |
| 746 | − FUNC(loop_filter_chroma_strong)(pix, xstride, ystride, size, tc, no_p, no_q); | |
| 747 | − else if (max_len_q == 3) | |
| 748 | − FUNC(loop_filter_chroma_strong_one_side)(pix, xstride, ystride, size, tc, no_p, no_q); | |
| 749 | − else | |
| 750 | − FUNC(loop_filter_chroma_weak)(pix, xstride, ystride, size, tc, no_p, no_q); | |
| 746 | + if (max_len_p == 3 && max_len_q == 3) | |
| 747 | + FUNC(loop_filter_chroma_strong)(pix, xstride, ystride, size, tc, no_p, no_q); | |
| 748 | + else if (max_len_q == 3) | |
| 749 | + FUNC(loop_filter_chroma_strong_one_side)(pix, xstride, ystride, size, tc, no_p, no_q); | |
| 750 | + else | |
| 751 | + FUNC(loop_filter_chroma_weak)(pix, xstride, ystride, size, tc, no_p, no_q); | |
| 752 | + } | |
| 751 | 753 | } |
| 752 | 754 | } |
| 753 | 755 | |