Fixes use of uninitialized memory Fixes: 93728afd9aa074ba14a09bfd93a632fd-asan_static-oob_124a17d_1445_cov_1021181966_DBLK_D_VIXS_1.bit Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind Signed-off-by: Michael Niedermayer <michaelni@gmx.at>
Unified · Split
| @@ -1634,6 +1634,7 @@ static int hls_coding_unit(HEVCContext *s, int x0, int y0, int log2_cb_size) · 1633 unmodified lines | |||
| 1634 | int x_cb = x0 >> log2_min_cb_size; | 1634 | int x_cb = x0 >> log2_min_cb_size; |
| 1635 | int y_cb = y0 >> log2_min_cb_size; | 1635 | int y_cb = y0 >> log2_min_cb_size; |
| 1636 | int x, y, ret; | 1636 | int x, y, ret; |
| 1637 | int qp_block_mask = (1<<(s->sps->log2_ctb_size - s->pps->diff_cu_qp_delta_depth)) - 1; | ||
| 1637 | 1638 | ||
| 1638 | lc->cu.x = x0; | 1639 | lc->cu.x = x0; |
| 1639 | lc->cu.y = y0; | 1640 | lc->cu.y = y0; |
| @@ -1771,6 +1772,11 @@ static int hls_coding_unit(HEVCContext *s, int x0, int y0, int log2_cb_size) · 131 unmodified lines | |||
| 1771 | x += min_cb_width; | 1772 | x += min_cb_width; |
| 1772 | } | 1773 | } |
| 1773 | 1774 | ||
| 1775 | if(((x0 + (1<<log2_cb_size)) & qp_block_mask) == 0 && | ||
| 1776 | ((y0 + (1<<log2_cb_size)) & qp_block_mask) == 0) { | ||
| 1777 | lc->qPy_pred = lc->qp_y; | ||
| 1778 | } | ||
| 1779 | |||
| 1774 | set_ct_depth(s, x0, y0, log2_cb_size, lc->ct.depth); | 1780 | set_ct_depth(s, x0, y0, log2_cb_size, lc->ct.depth); |
| 1775 | 1781 | ||
| 1776 | return 0; | 1782 | return 0; |
| @@ -1782,6 +1788,7 @@ static int hls_coding_quadtree(HEVCContext *s, int x0, int y0, · 5 unmodified lines | |||
| 1782 | HEVCLocalContext *lc = s->HEVClc; | 1788 | HEVCLocalContext *lc = s->HEVClc; |
| 1783 | const int cb_size = 1 << log2_cb_size; | 1789 | const int cb_size = 1 << log2_cb_size; |
| 1784 | int ret; | 1790 | int ret; |
| 1791 | int qp_block_mask = (1<<(s->sps->log2_ctb_size - s->pps->diff_cu_qp_delta_depth)) - 1; | ||
| 1785 | 1792 | ||
| 1786 | lc->ct.depth = cb_depth; | 1793 | lc->ct.depth = cb_depth; |
| 1787 | if (x0 + cb_size <= s->sps->width && | 1794 | if (x0 + cb_size <= s->sps->width && |
| @@ -1822,8 +1829,15 @@ static int hls_coding_quadtree(HEVCContext *s, int x0, int y0, · 34 unmodified lines | |||
| 1822 | } | 1829 | } |
| 1823 | if (more_data && x1 < s->sps->width && | 1830 | if (more_data && x1 < s->sps->width && |
| 1824 | y1 < s->sps->height) { | 1831 | y1 < s->sps->height) { |
| 1825 | return hls_coding_quadtree(s, x1, y1, log2_cb_size - 1, cb_depth + 1); | 1832 | more_data = hls_coding_quadtree(s, x1, y1, log2_cb_size - 1, cb_depth + 1); |
| 1833 | if (more_data < 0) | ||
| 1834 | return more_data; | ||
| 1826 | } | 1835 | } |
| 1836 | |||
| 1837 | if(((x0 + (1<<log2_cb_size)) & qp_block_mask) == 0 && | ||
| 1838 | ((y0 + (1<<log2_cb_size)) & qp_block_mask) == 0) | ||
| 1839 | lc->qPy_pred = lc->qp_y; | ||
| 1840 | |||
| 1827 | if (more_data) | 1841 | if (more_data) |
| 1828 | return ((x1 + cb_size_split) < s->sps->width || | 1842 | return ((x1 + cb_size_split) < s->sps->width || |
| 1829 | (y1 + cb_size_split) < s->sps->height); | 1843 | (y1 + cb_size_split) < s->sps->height); |
| @@ -723,6 +723,8 @@ typedef struct HEVCLocalContext { · 722 unmodified lines | |||
| 723 | int8_t qp_y; | 723 | int8_t qp_y; |
| 724 | int8_t curr_qp_y; | 724 | int8_t curr_qp_y; |
| 725 | 725 | ||
| 726 | int qPy_pred; | ||
| 727 | |||
| 726 | TransformUnit tu; | 728 | TransformUnit tu; |
| 727 | 729 | ||
| 728 | uint8_t ctb_left_flag; | 730 | uint8_t ctb_left_flag; |
| @@ -96,46 +96,7 @@ static int get_qPy_pred(HEVCContext *s, int xC, int yC, · 95 unmodified lines | |||
| 96 | lc->first_qp_group = !lc->tu.is_cu_qp_delta_coded; | 96 | lc->first_qp_group = !lc->tu.is_cu_qp_delta_coded; |
| 97 | qPy_pred = s->sh.slice_qp; | 97 | qPy_pred = s->sh.slice_qp; |
| 98 | } else { | 98 | } else { |
| 99 | qPy_pred = lc->qp_y; | 99 | qPy_pred = lc->qPy_pred; |
| 100 | if (log2_cb_size < s->sps->log2_ctb_size - | ||
| 101 | s->pps->diff_cu_qp_delta_depth) { | ||
| 102 | static const int offsetX[8][8] = { | ||
| 103 | { -1, 1, 3, 1, 7, 1, 3, 1 }, | ||
| 104 | { 0, 0, 0, 0, 0, 0, 0, 0 }, | ||
| 105 | { 1, 3, 1, 3, 1, 3, 1, 3 }, | ||
| 106 | { 2, 2, 2, 2, 2, 2, 2, 2 }, | ||
| 107 | { 3, 5, 7, 5, 3, 5, 7, 5 }, | ||
| 108 | { 4, 4, 4, 4, 4, 4, 4, 4 }, | ||
| 109 | { 5, 7, 5, 7, 5, 7, 5, 7 }, | ||
| 110 | { 6, 6, 6, 6, 6, 6, 6, 6 } | ||
| 111 | }; | ||
| 112 | static const int offsetY[8][8] = { | ||
| 113 | { 7, 0, 1, 2, 3, 4, 5, 6 }, | ||
| 114 | { 0, 1, 2, 3, 4, 5, 6, 7 }, | ||
| 115 | { 1, 0, 3, 2, 5, 4, 7, 6 }, | ||
| 116 | { 0, 1, 2, 3, 4, 5, 6, 7 }, | ||
| 117 | { 3, 0, 1, 2, 7, 4, 5, 6 }, | ||
| 118 | { 0, 1, 2, 3, 4, 5, 6, 7 }, | ||
| 119 | { 1, 0, 3, 2, 5, 4, 7, 6 }, | ||
| 120 | { 0, 1, 2, 3, 4, 5, 6, 7 } | ||
| 121 | }; | ||
| 122 | int xC0b = (xC - (xC & ctb_size_mask)) >> s->sps->log2_min_cb_size; | ||
| 123 | int yC0b = (yC - (yC & ctb_size_mask)) >> s->sps->log2_min_cb_size; | ||
| 124 | int idxX = (xQgBase & ctb_size_mask) >> s->sps->log2_min_cb_size; | ||
| 125 | int idxY = (yQgBase & ctb_size_mask) >> s->sps->log2_min_cb_size; | ||
| 126 | int idx_mask = ctb_size_mask >> s->sps->log2_min_cb_size; | ||
| 127 | int x, y; | ||
| 128 | |||
| 129 | x = FFMIN(xC0b + offsetX[idxX][idxY], min_cb_width - 1); | ||
| 130 | y = FFMIN(yC0b + (offsetY[idxX][idxY] & idx_mask), min_cb_height - 1); | ||
| 131 | |||
| 132 | if (xC0b == (lc->start_of_tiles_x >> s->sps->log2_min_cb_size) && | ||
| 133 | offsetX[idxX][idxY] == -1) { | ||
| 134 | x = (lc->end_of_tiles_x >> s->sps->log2_min_cb_size) - 1; | ||
| 135 | y = yC0b - 1; | ||
| 136 | } | ||
| 137 | qPy_pred = s->qp_y_tab[y * min_cb_width + x]; | ||
| 138 | } | ||
| 139 | } | 100 | } |
| 140 | 101 | ||
| 141 | // qPy_a | 102 | // qPy_a |