History

lavc/hevc/hevcdec: implement MV-HEVC inter-layer prediction

The per-frame reference picture set contains two more lists - INTER_LAYER[01]. Assuming at most two layers, INTER_LAYER1 is always empty, but is added anyway for completeness. When inter-layer prediction is enabled, INTER_LAYER0 for the second-layer frame will contain the base-layer frame from the same access unit, if it exists. The new lists are then used in per-slice reference picture set construction as per F.8.3.4 "Decoding process for reference picture lists construction".

Anton Khirnov committed 2 years ago

Commit 75914b51 parent 02a94355 files · +59 −18Browse files

Unified · Split

5 files
  1. libavcodec/hevc/hevcdec.c +6 −1
  2. libavcodec/hevc/hevcdec.h +6 −2
  3. libavcodec/hevc/refs.c +45 −13
  4. libavcodec/nvdec_hevc.c +1 −1
  5. libavcodec/vdpau_hevc.c +1 −1
libavcodec/hevc/hevcdec.c+6−1
@@ -784,7 +784,7 @@ static int hls_slice_header(SliceHeader *sh, const HEVCContext *s, GetBitContext · 783 unmodified lines
784 784
785 sh->rpl_modification_flag[0] = 0;785 sh->rpl_modification_flag[0] = 0;
786 sh->rpl_modification_flag[1] = 0;786 sh->rpl_modification_flag[1] = 0;
787 nb_refs = ff_hevc_frame_nb_refs(sh, pps);787 nb_refs = ff_hevc_frame_nb_refs(sh, pps, layer_idx);
788 if (!nb_refs) {788 if (!nb_refs) {
789 av_log(s->avctx, AV_LOG_ERROR, "Zero refs for a frame with P or B slices.\n");789 av_log(s->avctx, AV_LOG_ERROR, "Zero refs for a frame with P or B slices.\n");
790 return AVERROR_INVALIDDATA;790 return AVERROR_INVALIDDATA;
@@ -3356,6 +3356,11 @@ static int decode_nal_units(HEVCContext *s, const uint8_t *buf, int length) · 2565 unmodified lines
3356 s->eos = 0;3356 s->eos = 0;
3357 s->slice_initialized = 0;3357 s->slice_initialized = 0;
3358 3358
3359 for (int i = 0; i < FF_ARRAY_ELEMS(s->layers); i++) {
3360 HEVCLayerContext *l = &s->layers[i];
3361 l->cur_frame = NULL;
3362 }
3363
3359 /* split the input packet into NAL units, so we know the upper bound on the3364 /* split the input packet into NAL units, so we know the upper bound on the
3360 * number of slices in the frame */3365 * number of slices in the frame */
3361 ret = ff_h2645_packet_split(&s->pkt, buf, length, s->avctx,3366 ret = ff_h2645_packet_split(&s->pkt, buf, length, s->avctx,
libavcodec/hevc/hevcdec.h+6−2
@@ -84,6 +84,8 @@ enum RPSType { · 83 unmodified lines
84 ST_FOLL,84 ST_FOLL,
85 LT_CURR,85 LT_CURR,
86 LT_FOLL,86 LT_FOLL,
87 INTER_LAYER0,
88 INTER_LAYER1,
87 NB_RPS_TYPE,89 NB_RPS_TYPE,
88};90};
89 91
@@ -442,6 +444,7 @@ typedef struct HEVCLocalContext { · 352 unmodified lines
442 444
443typedef struct HEVCLayerContext {445typedef struct HEVCLayerContext {
444 HEVCFrame DPB[32];446 HEVCFrame DPB[32];
447 HEVCFrame *cur_frame;
445 448
446 const HEVCSPS *sps; // RefStruct reference449 const HEVCSPS *sps; // RefStruct reference
447 450
@@ -498,7 +501,7 @@ typedef struct HEVCContext { · 50 unmodified lines
498 struct AVMD5 *md5_ctx;501 struct AVMD5 *md5_ctx;
499 502
500 ///< candidate references for the current frame503 ///< candidate references for the current frame
501 RefPicList rps[5];504 RefPicList rps[NB_RPS_TYPE];
502 505
503 const HEVCVPS *vps; ///< RefStruct reference506 const HEVCVPS *vps; ///< RefStruct reference
504 const HEVCPPS *pps; ///< RefStruct reference507 const HEVCPPS *pps; ///< RefStruct reference
@@ -615,7 +618,8 @@ int ff_hevc_res_scale_sign_flag(HEVCLocalContext *lc, int idx); · 110 unmodified lines
615/**618/**
616 * Get the number of candidate references for the current frame.619 * Get the number of candidate references for the current frame.
617 */620 */
618int ff_hevc_frame_nb_refs(const SliceHeader *sh, const HEVCPPS *pps);621int ff_hevc_frame_nb_refs(const SliceHeader *sh, const HEVCPPS *pps,
622 unsigned layer_idx);
619 623
620int ff_hevc_set_new_ref(HEVCContext *s, HEVCLayerContext *l, int poc);624int ff_hevc_set_new_ref(HEVCContext *s, HEVCLayerContext *l, int poc);
621 625
libavcodec/hevc/refs.c+45−13
@@ -162,6 +162,7 @@ int ff_hevc_set_new_ref(HEVCContext *s, HEVCLayerContext *l, int poc) · 161 unmodified lines
162 return AVERROR(ENOMEM);162 return AVERROR(ENOMEM);
163 163
164 s->cur_frame = ref;164 s->cur_frame = ref;
165 l->cur_frame = ref;
165 s->collocated_ref = NULL;166 s->collocated_ref = NULL;
166 167
167 if (s->sh.pic_output_flag)168 if (s->sh.pic_output_flag)
@@ -261,7 +262,9 @@ int ff_hevc_slice_rpl(HEVCContext *s) · 93 unmodified lines
261 return ret;262 return ret;
262 263
263 if (!(s->rps[ST_CURR_BEF].nb_refs + s->rps[ST_CURR_AFT].nb_refs +264 if (!(s->rps[ST_CURR_BEF].nb_refs + s->rps[ST_CURR_AFT].nb_refs +
264 s->rps[LT_CURR].nb_refs) && !s->pps->pps_curr_pic_ref_enabled_flag) {265 s->rps[LT_CURR].nb_refs +
266 s->rps[INTER_LAYER0].nb_refs + s->rps[INTER_LAYER1].nb_refs) &&
267 !s->pps->pps_curr_pic_ref_enabled_flag) {
265 av_log(s->avctx, AV_LOG_ERROR, "Zero refs in the frame RPS.\n");268 av_log(s->avctx, AV_LOG_ERROR, "Zero refs in the frame RPS.\n");
266 return AVERROR_INVALIDDATA;269 return AVERROR_INVALIDDATA;
267 }270 }
@@ -271,11 +274,14 @@ int ff_hevc_slice_rpl(HEVCContext *s) · 3 unmodified lines
271 RefPicList *rpl = &s->cur_frame->refPicList[list_idx];274 RefPicList *rpl = &s->cur_frame->refPicList[list_idx];
272 275
273 /* The order of the elements is276 /* The order of the elements is
274 * ST_CURR_BEF - ST_CURR_AFT - LT_CURR for the L0 and277 * ST_CURR_BEF - INTER_LAYER0 - ST_CURR_AFT - LT_CURR - INTER_LAYER1 for the L0 and
275 * ST_CURR_AFT - ST_CURR_BEF - LT_CURR for the L1 */278 * ST_CURR_AFT - INTER_LAYER1 - ST_CURR_BEF - LT_CURR - INTER_LAYER0 for the L1 */
276 int cand_lists[3] = { list_idx ? ST_CURR_AFT : ST_CURR_BEF,279 int cand_lists[] = { list_idx ? ST_CURR_AFT : ST_CURR_BEF,
277 list_idx ? ST_CURR_BEF : ST_CURR_AFT,280 list_idx ? INTER_LAYER1 : INTER_LAYER0,
278 LT_CURR };281 list_idx ? ST_CURR_BEF : ST_CURR_AFT,
282 LT_CURR,
283 list_idx ? INTER_LAYER0 : INTER_LAYER1
284 };
279 285
280 /* concatenate the candidate lists for the current frame */286 /* concatenate the candidate lists for the current frame */
281 while (rpl_tmp.nb_refs < sh->nb_refs[list_idx]) {287 while (rpl_tmp.nb_refs < sh->nb_refs[list_idx]) {
@@ -284,7 +290,11 @@ int ff_hevc_slice_rpl(HEVCContext *s) · 2 unmodified lines
284 for (j = 0; j < rps->nb_refs && rpl_tmp.nb_refs < HEVC_MAX_REFS; j++) {290 for (j = 0; j < rps->nb_refs && rpl_tmp.nb_refs < HEVC_MAX_REFS; j++) {
285 rpl_tmp.list[rpl_tmp.nb_refs] = rps->list[j];291 rpl_tmp.list[rpl_tmp.nb_refs] = rps->list[j];
286 rpl_tmp.ref[rpl_tmp.nb_refs] = rps->ref[j];292 rpl_tmp.ref[rpl_tmp.nb_refs] = rps->ref[j];
287 rpl_tmp.isLongTerm[rpl_tmp.nb_refs] = i == 2;293 // multiview inter-layer refs are treated as long-term here,
294 // cf. G.8.1.3
295 rpl_tmp.isLongTerm[rpl_tmp.nb_refs] = cand_lists[i] == LT_CURR ||
296 cand_lists[i] == INTER_LAYER0 ||
297 cand_lists[i] == INTER_LAYER1;
288 rpl_tmp.nb_refs++;298 rpl_tmp.nb_refs++;
289 }299 }
290 }300 }
@@ -423,11 +433,6 @@ int ff_hevc_frame_rps(HEVCContext *s, HEVCLayerContext *l) · 132 unmodified lines
423 RefPicList *rps = s->rps;433 RefPicList *rps = s->rps;
424 int i, ret = 0;434 int i, ret = 0;
425 435
426 if (!short_rps) {
427 rps[0].nb_refs = rps[1].nb_refs = 0;
428 return 0;
429 }
430
431 unref_missing_refs(l);436 unref_missing_refs(l);
432 437
433 /* clear the reference flags on all frames except the current one */438 /* clear the reference flags on all frames except the current one */
@@ -443,6 +448,9 @@ int ff_hevc_frame_rps(HEVCContext *s, HEVCLayerContext *l) · 9 unmodified lines
443 for (i = 0; i < NB_RPS_TYPE; i++)448 for (i = 0; i < NB_RPS_TYPE; i++)
444 rps[i].nb_refs = 0;449 rps[i].nb_refs = 0;
445 450
451 if (!short_rps)
452 goto inter_layer;
453
446 /* add the short refs */454 /* add the short refs */
447 for (i = 0; i < short_rps->num_delta_pocs; i++) {455 for (i = 0; i < short_rps->num_delta_pocs; i++) {
448 int poc = s->poc + short_rps->delta_poc[i];456 int poc = s->poc + short_rps->delta_poc[i];
@@ -472,6 +480,24 @@ int ff_hevc_frame_rps(HEVCContext *s, HEVCLayerContext *l) · 23 unmodified lines
472 goto fail;480 goto fail;
473 }481 }
474 482
483inter_layer:
484 /* add inter-layer refs */
485 if (s->sh.inter_layer_pred) {
486 HEVCLayerContext *l0 = &s->layers[0];
487
488 av_assert0(l != l0);
489
490 /* Given the assumption of at most two layers, refPicSet0Flag is
491 * always 1, so only RefPicSetInterLayer0 can ever contain a frame. */
492 if (l0->cur_frame) {
493 // inter-layer refs are treated as short-term here, cf. F.8.1.6
494 ret = add_candidate_ref(s, l0, &rps[INTER_LAYER0], l0->cur_frame->poc,
495 HEVC_FRAME_FLAG_SHORT_REF, 1);
496 if (ret < 0)
497 goto fail;
498 }
499 }
500
475fail:501fail:
476 /* release any frames that are now unused */502 /* release any frames that are now unused */
477 for (i = 0; i < FF_ARRAY_ELEMS(l->DPB); i++)503 for (i = 0; i < FF_ARRAY_ELEMS(l->DPB); i++)
@@ -480,7 +506,8 @@ fail: · 2 unmodified lines
480 return ret;506 return ret;
481}507}
482 508
483int ff_hevc_frame_nb_refs(const SliceHeader *sh, const HEVCPPS *pps)509int ff_hevc_frame_nb_refs(const SliceHeader *sh, const HEVCPPS *pps,
510 unsigned layer_idx)
484{511{
485 int ret = 0;512 int ret = 0;
486 int i;513 int i;
@@ -499,6 +526,11 @@ int ff_hevc_frame_nb_refs(const SliceHeader *sh, const HEVCPPS *pps) · 12 unmodified lines
499 ret += !!long_rps->used[i];526 ret += !!long_rps->used[i];
500 }527 }
501 528
529 if (sh->inter_layer_pred) {
530 av_assert0(pps->sps->vps->num_direct_ref_layers[layer_idx] < 2);
531 ret++;
532 }
533
502 if (pps->pps_curr_pic_ref_enabled_flag)534 if (pps->pps_curr_pic_ref_enabled_flag)
503 ret++;535 ret++;
504 536
libavcodec/nvdec_hevc.c+1−1
@@ -188,7 +188,7 @@ static int nvdec_hevc_start_frame(AVCodecContext *avctx, · 187 unmodified lines
188 188
189 .NumBitsForShortTermRPSInSlice = s->sh.short_term_rps ? s->sh.short_term_ref_pic_set_size : 0,189 .NumBitsForShortTermRPSInSlice = s->sh.short_term_rps ? s->sh.short_term_ref_pic_set_size : 0,
190 .NumDeltaPocsOfRefRpsIdx = s->sh.short_term_rps ? s->sh.short_term_rps->rps_idx_num_delta_pocs : 0,190 .NumDeltaPocsOfRefRpsIdx = s->sh.short_term_rps ? s->sh.short_term_rps->rps_idx_num_delta_pocs : 0,
191 .NumPocTotalCurr = ff_hevc_frame_nb_refs(&s->sh, pps),191 .NumPocTotalCurr = ff_hevc_frame_nb_refs(&s->sh, pps, s->cur_layer),
192 .NumPocStCurrBefore = s->rps[ST_CURR_BEF].nb_refs,192 .NumPocStCurrBefore = s->rps[ST_CURR_BEF].nb_refs,
193 .NumPocStCurrAfter = s->rps[ST_CURR_AFT].nb_refs,193 .NumPocStCurrAfter = s->rps[ST_CURR_AFT].nb_refs,
194 .NumPocLtCurr = s->rps[LT_CURR].nb_refs,194 .NumPocLtCurr = s->rps[LT_CURR].nb_refs,
libavcodec/vdpau_hevc.c+1−1
@@ -206,7 +206,7 @@ static int vdpau_hevc_start_frame(AVCodecContext *avctx, · 205 unmodified lines
206 }206 }
207 }207 }
208 /* See section 7.4.7.2 of the specification. */208 /* See section 7.4.7.2 of the specification. */
209 info->NumPocTotalCurr = ff_hevc_frame_nb_refs(&h->sh, pps);209 info->NumPocTotalCurr = ff_hevc_frame_nb_refs(&h->sh, pps, h->cur_layer);
210 if (sh->short_term_ref_pic_set_sps_flag == 0 && sh->short_term_rps) {210 if (sh->short_term_ref_pic_set_sps_flag == 0 && sh->short_term_rps) {
211 /* Corresponds to specification field, NumDeltaPocs[RefRpsIdx].211 /* Corresponds to specification field, NumDeltaPocs[RefRpsIdx].
212 Only applicable when short_term_ref_pic_set_sps_flag == 0.212 Only applicable when short_term_ref_pic_set_sps_flag == 0.

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t