History

avcodec/vvcdec: deblock, fix uninitialized values

see https://fate.ffmpeg.org/report.cgi?slot=x86_64-archlinux-gcc-valgrind&time=20240105201935 If tc is zero, the max_len_q, max_len_p are uninitialized.

Nuo Mi committed 2 years ago

Commit 375dcf41 parent be4fcf01 file · +119 −117Browse files

Unified · Split

libavcodec/vvc/vvc_filter_template.c+119−117
@@ -551,81 +551,80 @@ static void FUNC(vvc_loop_filter_luma)(uint8_t* _pix, ptrdiff_t _xstride, ptrdif · 550 unmodified lines
551551 const ptrdiff_t ystride = _ystride / sizeof(pixel);
552552
553553 for (int i = 0; i < 2; i++) {
554− pixel* pix = (pixel*)_pix + i * 4 * ystride;
555− const int dp0 = abs(P2 - 2 * P1 + P0);
556− const int dq0 = abs(Q2 - 2 * Q1 + Q0);
557− const int dp3 = abs(TP2 - 2 * TP1 + TP0);
558− const int dq3 = abs(TQ2 - 2 * TQ1 + TQ0);
559− const int d0 = dp0 + dq0;
560− const int d3 = dp3 + dq3;
561554 #if BIT_DEPTH < 10
562555 const int tc = (_tc[i] + (1 << (9 - BIT_DEPTH))) >> (10 - BIT_DEPTH);
563556 #else
564557 const int tc = _tc[i] << (BIT_DEPTH - 10);
565558 #endif
566− const int tc25 = ((tc * 5 + 1) >> 1);
567−
568− const int no_p = _no_p[i];
569− const int no_q = _no_q[i];
570−
571− int max_len_p = _max_len_p[i];
572− int max_len_q = _max_len_q[i];
573−
574− const int large_p = (max_len_p > 3 && !hor_ctu_edge);
575− const int large_q = max_len_q > 3;
576− const int beta = _beta[i] << BIT_DEPTH - 8;
577−
578− const int beta_3 = beta >> 3;
579− const int beta_2 = beta >> 2;
580−
581− if (!tc)
582− continue;
583−
584− if (large_p || large_q) {
585− const int dp0l = large_p ? ((dp0 + abs(P5 - 2 * P4 + P3) + 1) >> 1) : dp0;
586− const int dq0l = large_q ? ((dq0 + abs(Q5 - 2 * Q4 + Q3) + 1) >> 1) : dq0;
587− const int dp3l = large_p ? ((dp3 + abs(TP5 - 2 * TP4 + TP3) + 1) >> 1) : dp3;
588− const int dq3l = large_q ? ((dq3 + abs(TQ5 - 2 * TQ4 + TQ3) + 1) >> 1) : dq3;
589− const int d0l = dp0l + dq0l;
590− const int d3l = dp3l + dq3l;
591− const int beta53 = beta * 3 >> 5;
592− const int beta_4 = beta >> 4;
593− max_len_p = large_p ? max_len_p : 3;
594− max_len_q = large_q ? max_len_q : 3;
595−
596− if (d0l + d3l < beta) {
597− const int sp0l = abs(P3 - P0) + (max_len_p == 7 ? abs(P7 - P6 - P5 + P4) : 0);
598− const int sq0l = abs(Q0 - Q3) + (max_len_q == 7 ? abs(Q4 - Q5 - Q6 + Q7) : 0);
599− const int sp3l = abs(TP3 - TP0) + (max_len_p == 7 ? abs(TP7 - TP6 - TP5 + TP4) : 0);
600− const int sq3l = abs(TQ0 - TQ3) + (max_len_q == 7 ? abs(TQ4 - TQ5 - TQ6 + TQ7) : 0);
601− const int sp0 = large_p ? ((sp0l + abs(P3 - P(max_len_p)) + 1) >> 1) : sp0l;
602− const int sp3 = large_p ? ((sp3l + abs(TP3 - TP(max_len_p)) + 1) >> 1) : sp3l;
603− const int sq0 = large_q ? ((sq0l + abs(Q3 - Q(max_len_q)) + 1) >> 1) : sq0l;
604− const int sq3 = large_q ? ((sq3l + abs(TQ3 - TQ(max_len_q)) + 1) >> 1) : sq3l;
605− if (sp0 + sq0 < beta53 && abs(P0 - Q0) < tc25 &&
606− sp3 + sq3 < beta53 && abs(TP0 - TQ0) < tc25 &&
607− (d0l << 1) < beta_4 && (d3l << 1) < beta_4) {
608− FUNC(loop_filter_luma_large)(pix, xstride, ystride, tc, no_p, no_q, max_len_p, max_len_q);
609− continue;
559+ if (tc) {
560+ pixel* pix = (pixel*)_pix + i * 4 * ystride;
561+ const int dp0 = abs(P2 - 2 * P1 + P0);
562+ const int dq0 = abs(Q2 - 2 * Q1 + Q0);
563+ const int dp3 = abs(TP2 - 2 * TP1 + TP0);
564+ const int dq3 = abs(TQ2 - 2 * TQ1 + TQ0);
565+ const int d0 = dp0 + dq0;
566+ const int d3 = dp3 + dq3;
567+ const int tc25 = ((tc * 5 + 1) >> 1);
568+
569+ const int no_p = _no_p[i];
570+ const int no_q = _no_q[i];
571+
572+ int max_len_p = _max_len_p[i];
573+ int max_len_q = _max_len_q[i];
574+
575+ const int large_p = (max_len_p > 3 && !hor_ctu_edge);
576+ const int large_q = max_len_q > 3;
577+
578+ const int beta = _beta[i] << BIT_DEPTH - 8;
579+ const int beta_3 = beta >> 3;
580+ const int beta_2 = beta >> 2;
581+
582+ if (large_p || large_q) {
583+ const int dp0l = large_p ? ((dp0 + abs(P5 - 2 * P4 + P3) + 1) >> 1) : dp0;
584+ const int dq0l = large_q ? ((dq0 + abs(Q5 - 2 * Q4 + Q3) + 1) >> 1) : dq0;
585+ const int dp3l = large_p ? ((dp3 + abs(TP5 - 2 * TP4 + TP3) + 1) >> 1) : dp3;
586+ const int dq3l = large_q ? ((dq3 + abs(TQ5 - 2 * TQ4 + TQ3) + 1) >> 1) : dq3;
587+ const int d0l = dp0l + dq0l;
588+ const int d3l = dp3l + dq3l;
589+ const int beta53 = beta * 3 >> 5;
590+ const int beta_4 = beta >> 4;
591+ max_len_p = large_p ? max_len_p : 3;
592+ max_len_q = large_q ? max_len_q : 3;
593+
594+ if (d0l + d3l < beta) {
595+ const int sp0l = abs(P3 - P0) + (max_len_p == 7 ? abs(P7 - P6 - P5 + P4) : 0);
596+ const int sq0l = abs(Q0 - Q3) + (max_len_q == 7 ? abs(Q4 - Q5 - Q6 + Q7) : 0);
597+ const int sp3l = abs(TP3 - TP0) + (max_len_p == 7 ? abs(TP7 - TP6 - TP5 + TP4) : 0);
598+ const int sq3l = abs(TQ0 - TQ3) + (max_len_q == 7 ? abs(TQ4 - TQ5 - TQ6 + TQ7) : 0);
599+ const int sp0 = large_p ? ((sp0l + abs(P3 - P(max_len_p)) + 1) >> 1) : sp0l;
600+ const int sp3 = large_p ? ((sp3l + abs(TP3 - TP(max_len_p)) + 1) >> 1) : sp3l;
601+ const int sq0 = large_q ? ((sq0l + abs(Q3 - Q(max_len_q)) + 1) >> 1) : sq0l;
602+ const int sq3 = large_q ? ((sq3l + abs(TQ3 - TQ(max_len_q)) + 1) >> 1) : sq3l;
603+ if (sp0 + sq0 < beta53 && abs(P0 - Q0) < tc25 &&
604+ sp3 + sq3 < beta53 && abs(TP0 - TQ0) < tc25 &&
605+ (d0l << 1) < beta_4 && (d3l << 1) < beta_4) {
606+ FUNC(loop_filter_luma_large)(pix, xstride, ystride, tc, no_p, no_q, max_len_p, max_len_q);
607+ continue;
608+ }
610609 }
611610 }
612− }
613− if (d0 + d3 < beta) {
614− if (max_len_p > 2 && max_len_q > 2 &&
615− abs(P3 - P0) + abs(Q3 - Q0) < beta_3 && abs(P0 - Q0) < tc25 &&
616− abs(TP3 - TP0) + abs(TQ3 - TQ0) < beta_3 && abs(TP0 - TQ0) < tc25 &&
617− (d0 << 1) < beta_2 && (d3 << 1) < beta_2) {
618− FUNC(loop_filter_luma_strong)(pix, xstride, ystride, tc, tc << 1, tc * 3, no_p, no_q);
619− } else { // weak filtering
620− int nd_p = 1;
621− int nd_q = 1;
622− if (max_len_p > 1 && max_len_q > 1) {
623− if (dp0 + dp3 < ((beta + (beta >> 1)) >> 3))
624− nd_p = 2;
625− if (dq0 + dq3 < ((beta + (beta >> 1)) >> 3))
626− nd_q = 2;
611+ if (d0 + d3 < beta) {
612+ if (max_len_p > 2 && max_len_q > 2 &&
613+ abs(P3 - P0) + abs(Q3 - Q0) < beta_3 && abs(P0 - Q0) < tc25 &&
614+ abs(TP3 - TP0) + abs(TQ3 - TQ0) < beta_3 && abs(TP0 - TQ0) < tc25 &&
615+ (d0 << 1) < beta_2 && (d3 << 1) < beta_2) {
616+ FUNC(loop_filter_luma_strong)(pix, xstride, ystride, tc, tc << 1, tc * 3, no_p, no_q);
617+ } else {
618+ int nd_p = 1;
619+ int nd_q = 1;
620+ if (max_len_p > 1 && max_len_q > 1) {
621+ if (dp0 + dp3 < ((beta + (beta >> 1)) >> 3))
622+ nd_p = 2;
623+ if (dq0 + dq3 < ((beta + (beta >> 1)) >> 3))
624+ nd_q = 2;
625+ }
626+ FUNC(loop_filter_luma_weak)(pix, xstride, ystride, tc, beta, no_p, no_q, nd_p, nd_q);
627627 }
628− FUNC(loop_filter_luma_weak)(pix, xstride, ystride, tc, beta, no_p, no_q, nd_p, nd_q);
629628 }
630629 }
631630 }
@@ -689,65 +688,68 @@ static void FUNC(vvc_loop_filter_chroma)(uint8_t *_pix, const ptrdiff_t _xstrid · 57 unmodified lines
689688 const int end = 8 / size; // 8 samples a loop
690689
691690 for (int i = 0; i < end; i++) {
692− pixel *pix = (pixel *)_pix + i * size * ystride;
693− const uint8_t no_p = _no_p[i];
694− const uint8_t no_q = _no_q[i];
695− const int beta = _beta[i] << (BIT_DEPTH - 8);
696− const int beta_3 = beta >> 3;
697− const int beta_2 = beta >> 2;
698−
699691 #if BIT_DEPTH < 10
700692 const int tc = (_tc[i] + (1 << (9 - BIT_DEPTH))) >> (10 - BIT_DEPTH);
701693 #else
702694 const int tc = _tc[i] << (BIT_DEPTH - 10);
703695 #endif
704− const int tc25 = ((tc * 5 + 1) >> 1);
705−
706− uint8_t max_len_p = _max_len_p[i];
707− uint8_t max_len_q = _max_len_q[i];
708−
709− if (!max_len_p || !max_len_q || !tc)
710− continue;
711−
712− if (max_len_q == 3){
713− const int p1n = shift ? FP1 : TP1;
714− const int p2n = max_len_p == 1 ? p1n : (shift ? FP2 : TP2);
715− const int p0n = shift ? FP0 : TP0;
716− const int q0n = shift ? FQ0 : TQ0;
717− const int q1n = shift ? FQ1 : TQ1;
718− const int q2n = shift ? FQ2 : TQ2;
719− const int p3 = max_len_p == 1 ? P1 : P3;
720− const int p2 = max_len_p == 1 ? P1 : P2;
721− const int p1 = P1;
722− const int p0 = P0;
723− const int dp0 = abs(p2 - 2 * p1 + p0);
724− const int dq0 = abs(Q2 - 2 * Q1 + Q0);
696+ if (tc) {
697+ pixel *pix = (pixel *)_pix + i * size * ystride;
698+ const uint8_t no_p = _no_p[i];
699+ const uint8_t no_q = _no_q[i];
700+
701+ const int beta = _beta[i] << (BIT_DEPTH - 8);
702+ const int beta_3 = beta >> 3;
703+ const int beta_2 = beta >> 2;
704+
705+ const int tc25 = ((tc * 5 + 1) >> 1);
706+
707+ uint8_t max_len_p = _max_len_p[i];
708+ uint8_t max_len_q = _max_len_q[i];
725709
726− const int dp1 = abs(p2n - 2 * p1n + p0n);
727− const int dq1 = abs(q2n - 2 * q1n + q0n);
728− const int d0 = dp0 + dq0;
729− const int d1 = dp1 + dq1;
730−
731− if (d0 + d1 < beta) {
732− const int p3n = max_len_p == 1 ? p1n : (shift ? FP3 : TP3);
733− const int q3n = shift ? FQ3 : TQ3;
734− const int dsam0 = (d0 << 1) < beta_2 && (abs(p3 - p0) + abs(Q0 - Q3) < beta_3) &&
735− abs(p0 - Q0) < tc25;
736− const int dsam1 = (d1 << 1) < beta_2 && (abs(p3n - p0n) + abs(q0n - q3n) < beta_3) &&
737− abs(p0n - q0n) < tc25;
738− if (!dsam0 || !dsam1)
710+ if (!max_len_p || !max_len_q)
711+ continue;
712+
713+ if (max_len_q == 3){
714+ const int p1n = shift ? FP1 : TP1;
715+ const int p2n = max_len_p == 1 ? p1n : (shift ? FP2 : TP2);
716+ const int p0n = shift ? FP0 : TP0;
717+ const int q0n = shift ? FQ0 : TQ0;
718+ const int q1n = shift ? FQ1 : TQ1;
719+ const int q2n = shift ? FQ2 : TQ2;
720+ const int p3 = max_len_p == 1 ? P1 : P3;
721+ const int p2 = max_len_p == 1 ? P1 : P2;
722+ const int p1 = P1;
723+ const int p0 = P0;
724+ const int dp0 = abs(p2 - 2 * p1 + p0);
725+ const int dq0 = abs(Q2 - 2 * Q1 + Q0);
726+
727+ const int dp1 = abs(p2n - 2 * p1n + p0n);
728+ const int dq1 = abs(q2n - 2 * q1n + q0n);
729+ const int d0 = dp0 + dq0;
730+ const int d1 = dp1 + dq1;
731+
732+ if (d0 + d1 < beta) {
733+ const int p3n = max_len_p == 1 ? p1n : (shift ? FP3 : TP3);
734+ const int q3n = shift ? FQ3 : TQ3;
735+ const int dsam0 = (d0 << 1) < beta_2 && (abs(p3 - p0) + abs(Q0 - Q3) < beta_3) &&
736+ abs(p0 - Q0) < tc25;
737+ const int dsam1 = (d1 << 1) < beta_2 && (abs(p3n - p0n) + abs(q0n - q3n) < beta_3) &&
738+ abs(p0n - q0n) < tc25;
739+ if (!dsam0 || !dsam1)
740+ max_len_p = max_len_q = 1;
741+ } else {
739742 max_len_p = max_len_q = 1;
740− } else {
741− max_len_p = max_len_q = 1;
743+ }
742744 }
743− }
744745
745− if (max_len_p == 3 && max_len_q == 3)
746− FUNC(loop_filter_chroma_strong)(pix, xstride, ystride, size, tc, no_p, no_q);
747− else if (max_len_q == 3)
748− FUNC(loop_filter_chroma_strong_one_side)(pix, xstride, ystride, size, tc, no_p, no_q);
749− else
750− FUNC(loop_filter_chroma_weak)(pix, xstride, ystride, size, tc, no_p, no_q);
746+ if (max_len_p == 3 && max_len_q == 3)
747+ FUNC(loop_filter_chroma_strong)(pix, xstride, ystride, size, tc, no_p, no_q);
748+ else if (max_len_q == 3)
749+ FUNC(loop_filter_chroma_strong_one_side)(pix, xstride, ystride, size, tc, no_p, no_q);
750+ else
751+ FUNC(loop_filter_chroma_weak)(pix, xstride, ystride, size, tc, no_p, no_q);
752+ }
751753 }
752754 }
753755

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t