It can't be higher than vps_max_sub_layers. Do this while keeping the workaround for qsvenc_hevc calling ff_hevc_parse_sps() without a vps_list, as in some cases it needs to parse an sps to generate a fake vps derived from it.
Unified · Split
| @@ -1158,6 +1158,7 @@ int ff_hevc_parse_sps(HEVCSPS *sps, GetBitContext *gb, unsigned int *sps_id, · 1157 unmodified lines | |||
| 1158 | HEVCWindow *ow; | 1158 | HEVCWindow *ow; |
| 1159 | int ret = 0; | 1159 | int ret = 0; |
| 1160 | int bit_depth_chroma, num_comps, multi_layer_ext; | 1160 | int bit_depth_chroma, num_comps, multi_layer_ext; |
| 1161 | int vps_max_sub_layers; | ||
| 1161 | int i; | 1162 | int i; |
| 1162 | 1163 | ||
| 1163 | // Coded parameters | 1164 | // Coded parameters |
| @@ -1182,7 +1183,10 @@ int ff_hevc_parse_sps(HEVCSPS *sps, GetBitContext *gb, unsigned int *sps_id, · 18 unmodified lines | |||
| 1182 | 1183 | ||
| 1183 | sps->max_sub_layers = sps->vps->vps_max_sub_layers; | 1184 | sps->max_sub_layers = sps->vps->vps_max_sub_layers; |
| 1184 | } | 1185 | } |
| 1185 | if (sps->max_sub_layers > HEVC_MAX_SUB_LAYERS) { | 1186 | vps_max_sub_layers = sps->vps ? sps->vps->vps_max_sub_layers |
| 1187 | : FFMIN(sps->max_sub_layers, HEVC_MAX_SUB_LAYERS); | ||
| 1188 | |||
| 1189 | if (sps->max_sub_layers > vps_max_sub_layers) { | ||
| 1186 | av_log(avctx, AV_LOG_ERROR, "sps_max_sub_layers out of range: %d\n", | 1190 | av_log(avctx, AV_LOG_ERROR, "sps_max_sub_layers out of range: %d\n", |
| 1187 | sps->max_sub_layers); | 1191 | sps->max_sub_layers); |
| 1188 | return AVERROR_INVALIDDATA; | 1192 | return AVERROR_INVALIDDATA; |