History

avcodec/vvcdec: deblock, fix uninitialized values

see https://fate.ffmpeg.org/report.cgi?slot=x86_64-archlinux-gcc-valgrind&time=20240105201935 If tc is zero, the max_len_q, max_len_p are uninitialized.

Nuo Mi committed 2 years ago

Commit 375dcf41 parent be4fcf01 file · +119 −117Browse files

Unified · Split

libavcodec/vvc/vvc_filter_template.c+119−117
@@ -551,81 +551,80 @@ static void FUNC(vvc_loop_filter_luma)(uint8_t* _pix, ptrdiff_t _xstride, ptrdif · 550 unmodified lines
551 const ptrdiff_t ystride = _ystride / sizeof(pixel);551 const ptrdiff_t ystride = _ystride / sizeof(pixel);
552 552
553 for (int i = 0; i < 2; i++) {553 for (int i = 0; i < 2; i++) {
554 pixel* pix = (pixel*)_pix + i * 4 * ystride;
555 const int dp0 = abs(P2 - 2 * P1 + P0);
556 const int dq0 = abs(Q2 - 2 * Q1 + Q0);
557 const int dp3 = abs(TP2 - 2 * TP1 + TP0);
558 const int dq3 = abs(TQ2 - 2 * TQ1 + TQ0);
559 const int d0 = dp0 + dq0;
560 const int d3 = dp3 + dq3;
561#if BIT_DEPTH < 10554#if BIT_DEPTH < 10
562 const int tc = (_tc[i] + (1 << (9 - BIT_DEPTH))) >> (10 - BIT_DEPTH);555 const int tc = (_tc[i] + (1 << (9 - BIT_DEPTH))) >> (10 - BIT_DEPTH);
563#else556#else
564 const int tc = _tc[i] << (BIT_DEPTH - 10);557 const int tc = _tc[i] << (BIT_DEPTH - 10);
565#endif558#endif
566 const int tc25 = ((tc * 5 + 1) >> 1);559 if (tc) {
567 560 pixel* pix = (pixel*)_pix + i * 4 * ystride;
568 const int no_p = _no_p[i];561 const int dp0 = abs(P2 - 2 * P1 + P0);
569 const int no_q = _no_q[i];562 const int dq0 = abs(Q2 - 2 * Q1 + Q0);
570 563 const int dp3 = abs(TP2 - 2 * TP1 + TP0);
571 int max_len_p = _max_len_p[i];564 const int dq3 = abs(TQ2 - 2 * TQ1 + TQ0);
572 int max_len_q = _max_len_q[i];565 const int d0 = dp0 + dq0;
573 566 const int d3 = dp3 + dq3;
574 const int large_p = (max_len_p > 3 && !hor_ctu_edge);567 const int tc25 = ((tc * 5 + 1) >> 1);
575 const int large_q = max_len_q > 3;568
576 const int beta = _beta[i] << BIT_DEPTH - 8;569 const int no_p = _no_p[i];
577 570 const int no_q = _no_q[i];
578 const int beta_3 = beta >> 3;571
579 const int beta_2 = beta >> 2;572 int max_len_p = _max_len_p[i];
580 573 int max_len_q = _max_len_q[i];
581 if (!tc)574
582 continue;575 const int large_p = (max_len_p > 3 && !hor_ctu_edge);
583 576 const int large_q = max_len_q > 3;
584 if (large_p || large_q) {577
585 const int dp0l = large_p ? ((dp0 + abs(P5 - 2 * P4 + P3) + 1) >> 1) : dp0;578 const int beta = _beta[i] << BIT_DEPTH - 8;
586 const int dq0l = large_q ? ((dq0 + abs(Q5 - 2 * Q4 + Q3) + 1) >> 1) : dq0;579 const int beta_3 = beta >> 3;
587 const int dp3l = large_p ? ((dp3 + abs(TP5 - 2 * TP4 + TP3) + 1) >> 1) : dp3;580 const int beta_2 = beta >> 2;
588 const int dq3l = large_q ? ((dq3 + abs(TQ5 - 2 * TQ4 + TQ3) + 1) >> 1) : dq3;581
589 const int d0l = dp0l + dq0l;582 if (large_p || large_q) {
590 const int d3l = dp3l + dq3l;583 const int dp0l = large_p ? ((dp0 + abs(P5 - 2 * P4 + P3) + 1) >> 1) : dp0;
591 const int beta53 = beta * 3 >> 5;584 const int dq0l = large_q ? ((dq0 + abs(Q5 - 2 * Q4 + Q3) + 1) >> 1) : dq0;
592 const int beta_4 = beta >> 4;585 const int dp3l = large_p ? ((dp3 + abs(TP5 - 2 * TP4 + TP3) + 1) >> 1) : dp3;
593 max_len_p = large_p ? max_len_p : 3;586 const int dq3l = large_q ? ((dq3 + abs(TQ5 - 2 * TQ4 + TQ3) + 1) >> 1) : dq3;
594 max_len_q = large_q ? max_len_q : 3;587 const int d0l = dp0l + dq0l;
595 588 const int d3l = dp3l + dq3l;
596 if (d0l + d3l < beta) {589 const int beta53 = beta * 3 >> 5;
597 const int sp0l = abs(P3 - P0) + (max_len_p == 7 ? abs(P7 - P6 - P5 + P4) : 0);590 const int beta_4 = beta >> 4;
598 const int sq0l = abs(Q0 - Q3) + (max_len_q == 7 ? abs(Q4 - Q5 - Q6 + Q7) : 0);591 max_len_p = large_p ? max_len_p : 3;
599 const int sp3l = abs(TP3 - TP0) + (max_len_p == 7 ? abs(TP7 - TP6 - TP5 + TP4) : 0);592 max_len_q = large_q ? max_len_q : 3;
600 const int sq3l = abs(TQ0 - TQ3) + (max_len_q == 7 ? abs(TQ4 - TQ5 - TQ6 + TQ7) : 0);593
601 const int sp0 = large_p ? ((sp0l + abs(P3 - P(max_len_p)) + 1) >> 1) : sp0l;594 if (d0l + d3l < beta) {
602 const int sp3 = large_p ? ((sp3l + abs(TP3 - TP(max_len_p)) + 1) >> 1) : sp3l;595 const int sp0l = abs(P3 - P0) + (max_len_p == 7 ? abs(P7 - P6 - P5 + P4) : 0);
603 const int sq0 = large_q ? ((sq0l + abs(Q3 - Q(max_len_q)) + 1) >> 1) : sq0l;596 const int sq0l = abs(Q0 - Q3) + (max_len_q == 7 ? abs(Q4 - Q5 - Q6 + Q7) : 0);
604 const int sq3 = large_q ? ((sq3l + abs(TQ3 - TQ(max_len_q)) + 1) >> 1) : sq3l;597 const int sp3l = abs(TP3 - TP0) + (max_len_p == 7 ? abs(TP7 - TP6 - TP5 + TP4) : 0);
605 if (sp0 + sq0 < beta53 && abs(P0 - Q0) < tc25 &&598 const int sq3l = abs(TQ0 - TQ3) + (max_len_q == 7 ? abs(TQ4 - TQ5 - TQ6 + TQ7) : 0);
606 sp3 + sq3 < beta53 && abs(TP0 - TQ0) < tc25 &&599 const int sp0 = large_p ? ((sp0l + abs(P3 - P(max_len_p)) + 1) >> 1) : sp0l;
607 (d0l << 1) < beta_4 && (d3l << 1) < beta_4) {600 const int sp3 = large_p ? ((sp3l + abs(TP3 - TP(max_len_p)) + 1) >> 1) : sp3l;
608 FUNC(loop_filter_luma_large)(pix, xstride, ystride, tc, no_p, no_q, max_len_p, max_len_q);601 const int sq0 = large_q ? ((sq0l + abs(Q3 - Q(max_len_q)) + 1) >> 1) : sq0l;
609 continue;602 const int sq3 = large_q ? ((sq3l + abs(TQ3 - TQ(max_len_q)) + 1) >> 1) : sq3l;
603 if (sp0 + sq0 < beta53 && abs(P0 - Q0) < tc25 &&
604 sp3 + sq3 < beta53 && abs(TP0 - TQ0) < tc25 &&
605 (d0l << 1) < beta_4 && (d3l << 1) < beta_4) {
606 FUNC(loop_filter_luma_large)(pix, xstride, ystride, tc, no_p, no_q, max_len_p, max_len_q);
607 continue;
608 }
610 }609 }
611 }610 }
612 }611 if (d0 + d3 < beta) {
613 if (d0 + d3 < beta) {612 if (max_len_p > 2 && max_len_q > 2 &&
614 if (max_len_p > 2 && max_len_q > 2 &&613 abs(P3 - P0) + abs(Q3 - Q0) < beta_3 && abs(P0 - Q0) < tc25 &&
615 abs(P3 - P0) + abs(Q3 - Q0) < beta_3 && abs(P0 - Q0) < tc25 &&614 abs(TP3 - TP0) + abs(TQ3 - TQ0) < beta_3 && abs(TP0 - TQ0) < tc25 &&
616 abs(TP3 - TP0) + abs(TQ3 - TQ0) < beta_3 && abs(TP0 - TQ0) < tc25 &&615 (d0 << 1) < beta_2 && (d3 << 1) < beta_2) {
617 (d0 << 1) < beta_2 && (d3 << 1) < beta_2) {616 FUNC(loop_filter_luma_strong)(pix, xstride, ystride, tc, tc << 1, tc * 3, no_p, no_q);
618 FUNC(loop_filter_luma_strong)(pix, xstride, ystride, tc, tc << 1, tc * 3, no_p, no_q);617 } else {
619 } else { // weak filtering618 int nd_p = 1;
620 int nd_p = 1;619 int nd_q = 1;
621 int nd_q = 1;620 if (max_len_p > 1 && max_len_q > 1) {
622 if (max_len_p > 1 && max_len_q > 1) {621 if (dp0 + dp3 < ((beta + (beta >> 1)) >> 3))
623 if (dp0 + dp3 < ((beta + (beta >> 1)) >> 3))622 nd_p = 2;
624 nd_p = 2;623 if (dq0 + dq3 < ((beta + (beta >> 1)) >> 3))
625 if (dq0 + dq3 < ((beta + (beta >> 1)) >> 3))624 nd_q = 2;
626 nd_q = 2;625 }
626 FUNC(loop_filter_luma_weak)(pix, xstride, ystride, tc, beta, no_p, no_q, nd_p, nd_q);
627 }627 }
628 FUNC(loop_filter_luma_weak)(pix, xstride, ystride, tc, beta, no_p, no_q, nd_p, nd_q);
629 }628 }
630 }629 }
631 }630 }
@@ -689,65 +688,68 @@ static void FUNC(vvc_loop_filter_chroma)(uint8_t *_pix, const ptrdiff_t _xstrid · 57 unmodified lines
689 const int end = 8 / size; // 8 samples a loop688 const int end = 8 / size; // 8 samples a loop
690 689
691 for (int i = 0; i < end; i++) {690 for (int i = 0; i < end; i++) {
692 pixel *pix = (pixel *)_pix + i * size * ystride;
693 const uint8_t no_p = _no_p[i];
694 const uint8_t no_q = _no_q[i];
695 const int beta = _beta[i] << (BIT_DEPTH - 8);
696 const int beta_3 = beta >> 3;
697 const int beta_2 = beta >> 2;
698
699#if BIT_DEPTH < 10691#if BIT_DEPTH < 10
700 const int tc = (_tc[i] + (1 << (9 - BIT_DEPTH))) >> (10 - BIT_DEPTH);692 const int tc = (_tc[i] + (1 << (9 - BIT_DEPTH))) >> (10 - BIT_DEPTH);
701#else693#else
702 const int tc = _tc[i] << (BIT_DEPTH - 10);694 const int tc = _tc[i] << (BIT_DEPTH - 10);
703#endif695#endif
704 const int tc25 = ((tc * 5 + 1) >> 1);696 if (tc) {
705 697 pixel *pix = (pixel *)_pix + i * size * ystride;
706 uint8_t max_len_p = _max_len_p[i];698 const uint8_t no_p = _no_p[i];
707 uint8_t max_len_q = _max_len_q[i];699 const uint8_t no_q = _no_q[i];
708 700
709 if (!max_len_p || !max_len_q || !tc)701 const int beta = _beta[i] << (BIT_DEPTH - 8);
710 continue;702 const int beta_3 = beta >> 3;
711 703 const int beta_2 = beta >> 2;
712 if (max_len_q == 3){704
713 const int p1n = shift ? FP1 : TP1;705 const int tc25 = ((tc * 5 + 1) >> 1);
714 const int p2n = max_len_p == 1 ? p1n : (shift ? FP2 : TP2);706
715 const int p0n = shift ? FP0 : TP0;707 uint8_t max_len_p = _max_len_p[i];
716 const int q0n = shift ? FQ0 : TQ0;708 uint8_t max_len_q = _max_len_q[i];
717 const int q1n = shift ? FQ1 : TQ1;
718 const int q2n = shift ? FQ2 : TQ2;
719 const int p3 = max_len_p == 1 ? P1 : P3;
720 const int p2 = max_len_p == 1 ? P1 : P2;
721 const int p1 = P1;
722 const int p0 = P0;
723 const int dp0 = abs(p2 - 2 * p1 + p0);
724 const int dq0 = abs(Q2 - 2 * Q1 + Q0);
725 709
726 const int dp1 = abs(p2n - 2 * p1n + p0n);710 if (!max_len_p || !max_len_q)
727 const int dq1 = abs(q2n - 2 * q1n + q0n);711 continue;
728 const int d0 = dp0 + dq0;712
729 const int d1 = dp1 + dq1;713 if (max_len_q == 3){
730 714 const int p1n = shift ? FP1 : TP1;
731 if (d0 + d1 < beta) {715 const int p2n = max_len_p == 1 ? p1n : (shift ? FP2 : TP2);
732 const int p3n = max_len_p == 1 ? p1n : (shift ? FP3 : TP3);716 const int p0n = shift ? FP0 : TP0;
733 const int q3n = shift ? FQ3 : TQ3;717 const int q0n = shift ? FQ0 : TQ0;
734 const int dsam0 = (d0 << 1) < beta_2 && (abs(p3 - p0) + abs(Q0 - Q3) < beta_3) &&718 const int q1n = shift ? FQ1 : TQ1;
735 abs(p0 - Q0) < tc25;719 const int q2n = shift ? FQ2 : TQ2;
736 const int dsam1 = (d1 << 1) < beta_2 && (abs(p3n - p0n) + abs(q0n - q3n) < beta_3) &&720 const int p3 = max_len_p == 1 ? P1 : P3;
737 abs(p0n - q0n) < tc25;721 const int p2 = max_len_p == 1 ? P1 : P2;
738 if (!dsam0 || !dsam1)722 const int p1 = P1;
723 const int p0 = P0;
724 const int dp0 = abs(p2 - 2 * p1 + p0);
725 const int dq0 = abs(Q2 - 2 * Q1 + Q0);
726
727 const int dp1 = abs(p2n - 2 * p1n + p0n);
728 const int dq1 = abs(q2n - 2 * q1n + q0n);
729 const int d0 = dp0 + dq0;
730 const int d1 = dp1 + dq1;
731
732 if (d0 + d1 < beta) {
733 const int p3n = max_len_p == 1 ? p1n : (shift ? FP3 : TP3);
734 const int q3n = shift ? FQ3 : TQ3;
735 const int dsam0 = (d0 << 1) < beta_2 && (abs(p3 - p0) + abs(Q0 - Q3) < beta_3) &&
736 abs(p0 - Q0) < tc25;
737 const int dsam1 = (d1 << 1) < beta_2 && (abs(p3n - p0n) + abs(q0n - q3n) < beta_3) &&
738 abs(p0n - q0n) < tc25;
739 if (!dsam0 || !dsam1)
740 max_len_p = max_len_q = 1;
741 } else {
739 max_len_p = max_len_q = 1;742 max_len_p = max_len_q = 1;
740 } else {743 }
741 max_len_p = max_len_q = 1;
742 }744 }
743 }
744 745
745 if (max_len_p == 3 && max_len_q == 3)746 if (max_len_p == 3 && max_len_q == 3)
746 FUNC(loop_filter_chroma_strong)(pix, xstride, ystride, size, tc, no_p, no_q);747 FUNC(loop_filter_chroma_strong)(pix, xstride, ystride, size, tc, no_p, no_q);
747 else if (max_len_q == 3)748 else if (max_len_q == 3)
748 FUNC(loop_filter_chroma_strong_one_side)(pix, xstride, ystride, size, tc, no_p, no_q);749 FUNC(loop_filter_chroma_strong_one_side)(pix, xstride, ystride, size, tc, no_p, no_q);
749 else750 else
750 FUNC(loop_filter_chroma_weak)(pix, xstride, ystride, size, tc, no_p, no_q);751 FUNC(loop_filter_chroma_weak)(pix, xstride, ystride, size, tc, no_p, no_q);
752 }
751 }753 }
752}754}
753 755

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t