The clamping of idxYInv from H.266(V3) section 8.8.2.3 was missing. This could lead to OOB reads from lmcs->pivot or input_pivot. I also changed the derivation of the forward LMCS idx to use a shift rather than a division for speed and as this is actually how the variable is declared in the specification (8.7.5.2).
Unified · Split
| @@ -835,7 +835,7 @@ static int lmcs_derive_lut(VVCLMCS *lmcs, const H266RawAPS *rlmcs, const H266Raw · 834 unmodified lines | |||
| 835 | 835 | ||
| 836 | //derive lmcs_fwd_lut | 836 | //derive lmcs_fwd_lut |
| 837 | for (uint16_t sample = 0; sample < max; sample++) { | 837 | for (uint16_t sample = 0; sample < max; sample++) { |
| 838 | const int idx_y = sample / org_cw; | 838 | const int idx_y = sample >> shift; |
| 839 | const uint16_t fwd_sample = lmcs_derive_lut_sample(sample, lmcs->pivot, | 839 | const uint16_t fwd_sample = lmcs_derive_lut_sample(sample, lmcs->pivot, |
| 840 | input_pivot, scale_coeff, idx_y, max); | 840 | input_pivot, scale_coeff, idx_y, max); |
| 841 | if (bit_depth > 8) | 841 | if (bit_depth > 8) |
| @@ -851,6 +851,7 @@ static int lmcs_derive_lut(VVCLMCS *lmcs, const H266RawAPS *rlmcs, const H266Raw · 9 unmodified lines | |||
| 851 | uint16_t inv_sample; | 851 | uint16_t inv_sample; |
| 852 | while (i <= lmcs->max_bin_idx && sample >= lmcs->pivot[i + 1]) | 852 | while (i <= lmcs->max_bin_idx && sample >= lmcs->pivot[i + 1]) |
| 853 | i++; | 853 | i++; |
| 854 | i = FFMIN(i, LMCS_MAX_BIN_SIZE - 1); | ||
| 854 | 855 | ||
| 855 | inv_sample = lmcs_derive_lut_sample(sample, input_pivot, lmcs->pivot, | 856 | inv_sample = lmcs_derive_lut_sample(sample, input_pivot, lmcs->pivot, |
| 856 | inv_scale_coeff, i, max); | 857 | inv_scale_coeff, i, max); |