Fixes use of uninitialized memory Fixes: 93728afd9aa074ba14a09bfd93a632fd-asan_static-oob_124a17d_1445_cov_1021181966_DBLK_D_VIXS_1.bit Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind Signed-off-by: Michael Niedermayer <michaelni@gmx.at>
Unified · Split
| @@ -1634,6 +1634,7 @@ static int hls_coding_unit(HEVCContext *s, int x0, int y0, int log2_cb_size) · 1633 unmodified lines | ||
| 1634 | 1634 | int x_cb = x0 >> log2_min_cb_size; |
| 1635 | 1635 | int y_cb = y0 >> log2_min_cb_size; |
| 1636 | 1636 | int x, y, ret; |
| 1637 | + int qp_block_mask = (1<<(s->sps->log2_ctb_size - s->pps->diff_cu_qp_delta_depth)) - 1; | |
| 1637 | 1638 | |
| 1638 | 1639 | lc->cu.x = x0; |
| 1639 | 1640 | lc->cu.y = y0; |
| @@ -1771,6 +1772,11 @@ static int hls_coding_unit(HEVCContext *s, int x0, int y0, int log2_cb_size) · 131 unmodified lines | ||
| 1771 | 1772 | x += min_cb_width; |
| 1772 | 1773 | } |
| 1773 | 1774 | |
| 1775 | + if(((x0 + (1<<log2_cb_size)) & qp_block_mask) == 0 && | |
| 1776 | + ((y0 + (1<<log2_cb_size)) & qp_block_mask) == 0) { | |
| 1777 | + lc->qPy_pred = lc->qp_y; | |
| 1778 | + } | |
| 1779 | + | |
| 1774 | 1780 | set_ct_depth(s, x0, y0, log2_cb_size, lc->ct.depth); |
| 1775 | 1781 | |
| 1776 | 1782 | return 0; |
| @@ -1782,6 +1788,7 @@ static int hls_coding_quadtree(HEVCContext *s, int x0, int y0, · 5 unmodified lines | ||
| 1782 | 1788 | HEVCLocalContext *lc = s->HEVClc; |
| 1783 | 1789 | const int cb_size = 1 << log2_cb_size; |
| 1784 | 1790 | int ret; |
| 1791 | + int qp_block_mask = (1<<(s->sps->log2_ctb_size - s->pps->diff_cu_qp_delta_depth)) - 1; | |
| 1785 | 1792 | |
| 1786 | 1793 | lc->ct.depth = cb_depth; |
| 1787 | 1794 | if (x0 + cb_size <= s->sps->width && |
| @@ -1822,8 +1829,15 @@ static int hls_coding_quadtree(HEVCContext *s, int x0, int y0, · 34 unmodified lines | ||
| 1822 | 1829 | } |
| 1823 | 1830 | if (more_data && x1 < s->sps->width && |
| 1824 | 1831 | y1 < s->sps->height) { |
| 1825 | − return hls_coding_quadtree(s, x1, y1, log2_cb_size - 1, cb_depth + 1); | |
| 1832 | + more_data = hls_coding_quadtree(s, x1, y1, log2_cb_size - 1, cb_depth + 1); | |
| 1833 | + if (more_data < 0) | |
| 1834 | + return more_data; | |
| 1826 | 1835 | } |
| 1836 | + | |
| 1837 | + if(((x0 + (1<<log2_cb_size)) & qp_block_mask) == 0 && | |
| 1838 | + ((y0 + (1<<log2_cb_size)) & qp_block_mask) == 0) | |
| 1839 | + lc->qPy_pred = lc->qp_y; | |
| 1840 | + | |
| 1827 | 1841 | if (more_data) |
| 1828 | 1842 | return ((x1 + cb_size_split) < s->sps->width || |
| 1829 | 1843 | (y1 + cb_size_split) < s->sps->height); |
| @@ -723,6 +723,8 @@ typedef struct HEVCLocalContext { · 722 unmodified lines | ||
| 723 | 723 | int8_t qp_y; |
| 724 | 724 | int8_t curr_qp_y; |
| 725 | 725 | |
| 726 | + int qPy_pred; | |
| 727 | + | |
| 726 | 728 | TransformUnit tu; |
| 727 | 729 | |
| 728 | 730 | uint8_t ctb_left_flag; |
| @@ -96,46 +96,7 @@ static int get_qPy_pred(HEVCContext *s, int xC, int yC, · 95 unmodified lines | ||
| 96 | 96 | lc->first_qp_group = !lc->tu.is_cu_qp_delta_coded; |
| 97 | 97 | qPy_pred = s->sh.slice_qp; |
| 98 | 98 | } else { |
| 99 | − qPy_pred = lc->qp_y; | |
| 100 | − if (log2_cb_size < s->sps->log2_ctb_size - | |
| 101 | − s->pps->diff_cu_qp_delta_depth) { | |
| 102 | − static const int offsetX[8][8] = { | |
| 103 | − { -1, 1, 3, 1, 7, 1, 3, 1 }, | |
| 104 | − { 0, 0, 0, 0, 0, 0, 0, 0 }, | |
| 105 | − { 1, 3, 1, 3, 1, 3, 1, 3 }, | |
| 106 | − { 2, 2, 2, 2, 2, 2, 2, 2 }, | |
| 107 | − { 3, 5, 7, 5, 3, 5, 7, 5 }, | |
| 108 | − { 4, 4, 4, 4, 4, 4, 4, 4 }, | |
| 109 | − { 5, 7, 5, 7, 5, 7, 5, 7 }, | |
| 110 | − { 6, 6, 6, 6, 6, 6, 6, 6 } | |
| 111 | − }; | |
| 112 | − static const int offsetY[8][8] = { | |
| 113 | − { 7, 0, 1, 2, 3, 4, 5, 6 }, | |
| 114 | − { 0, 1, 2, 3, 4, 5, 6, 7 }, | |
| 115 | − { 1, 0, 3, 2, 5, 4, 7, 6 }, | |
| 116 | − { 0, 1, 2, 3, 4, 5, 6, 7 }, | |
| 117 | − { 3, 0, 1, 2, 7, 4, 5, 6 }, | |
| 118 | − { 0, 1, 2, 3, 4, 5, 6, 7 }, | |
| 119 | − { 1, 0, 3, 2, 5, 4, 7, 6 }, | |
| 120 | − { 0, 1, 2, 3, 4, 5, 6, 7 } | |
| 121 | − }; | |
| 122 | − int xC0b = (xC - (xC & ctb_size_mask)) >> s->sps->log2_min_cb_size; | |
| 123 | − int yC0b = (yC - (yC & ctb_size_mask)) >> s->sps->log2_min_cb_size; | |
| 124 | − int idxX = (xQgBase & ctb_size_mask) >> s->sps->log2_min_cb_size; | |
| 125 | − int idxY = (yQgBase & ctb_size_mask) >> s->sps->log2_min_cb_size; | |
| 126 | − int idx_mask = ctb_size_mask >> s->sps->log2_min_cb_size; | |
| 127 | − int x, y; | |
| 128 | − | |
| 129 | − x = FFMIN(xC0b + offsetX[idxX][idxY], min_cb_width - 1); | |
| 130 | − y = FFMIN(yC0b + (offsetY[idxX][idxY] & idx_mask), min_cb_height - 1); | |
| 131 | − | |
| 132 | − if (xC0b == (lc->start_of_tiles_x >> s->sps->log2_min_cb_size) && | |
| 133 | − offsetX[idxX][idxY] == -1) { | |
| 134 | − x = (lc->end_of_tiles_x >> s->sps->log2_min_cb_size) - 1; | |
| 135 | − y = yC0b - 1; | |
| 136 | − } | |
| 137 | − qPy_pred = s->qp_y_tab[y * min_cb_width + x]; | |
| 138 | − } | |
| 99 | + qPy_pred = lc->qPy_pred; | |
| 139 | 100 | } |
| 140 | 101 | |
| 141 | 102 | // qPy_a |