History

lavc/hevc/hevcdec: implement MV-HEVC inter-layer prediction

The per-frame reference picture set contains two more lists - INTER_LAYER[01]. Assuming at most two layers, INTER_LAYER1 is always empty, but is added anyway for completeness. When inter-layer prediction is enabled, INTER_LAYER0 for the second-layer frame will contain the base-layer frame from the same access unit, if it exists. The new lists are then used in per-slice reference picture set construction as per F.8.3.4 "Decoding process for reference picture lists construction".

Anton Khirnov committed 2 years ago

Commit 75914b51 parent 02a94355 files · +59 −18Browse files

Unified · Split

5 files
  1. libavcodec/hevc/hevcdec.c +6 −1
  2. libavcodec/hevc/hevcdec.h +6 −2
  3. libavcodec/hevc/refs.c +45 −13
  4. libavcodec/nvdec_hevc.c +1 −1
  5. libavcodec/vdpau_hevc.c +1 −1
libavcodec/hevc/hevcdec.c+6−1
@@ -784,7 +784,7 @@ static int hls_slice_header(SliceHeader *sh, const HEVCContext *s, GetBitContext · 783 unmodified lines
784784
785785 sh->rpl_modification_flag[0] = 0;
786786 sh->rpl_modification_flag[1] = 0;
787− nb_refs = ff_hevc_frame_nb_refs(sh, pps);
787+ nb_refs = ff_hevc_frame_nb_refs(sh, pps, layer_idx);
788788 if (!nb_refs) {
789789 av_log(s->avctx, AV_LOG_ERROR, "Zero refs for a frame with P or B slices.\n");
790790 return AVERROR_INVALIDDATA;
@@ -3356,6 +3356,11 @@ static int decode_nal_units(HEVCContext *s, const uint8_t *buf, int length) · 2565 unmodified lines
33563356 s->eos = 0;
33573357 s->slice_initialized = 0;
33583358
3359+ for (int i = 0; i < FF_ARRAY_ELEMS(s->layers); i++) {
3360+ HEVCLayerContext *l = &s->layers[i];
3361+ l->cur_frame = NULL;
3362+ }
3363+
33593364 /* split the input packet into NAL units, so we know the upper bound on the
33603365 * number of slices in the frame */
33613366 ret = ff_h2645_packet_split(&s->pkt, buf, length, s->avctx,
libavcodec/hevc/hevcdec.h+6−2
@@ -84,6 +84,8 @@ enum RPSType { · 83 unmodified lines
8484 ST_FOLL,
8585 LT_CURR,
8686 LT_FOLL,
87+ INTER_LAYER0,
88+ INTER_LAYER1,
8789 NB_RPS_TYPE,
8890 };
8991
@@ -442,6 +444,7 @@ typedef struct HEVCLocalContext { · 352 unmodified lines
442444
443445 typedef struct HEVCLayerContext {
444446 HEVCFrame DPB[32];
447+ HEVCFrame *cur_frame;
445448
446449 const HEVCSPS *sps; // RefStruct reference
447450
@@ -498,7 +501,7 @@ typedef struct HEVCContext { · 50 unmodified lines
498501 struct AVMD5 *md5_ctx;
499502
500503 ///< candidate references for the current frame
501− RefPicList rps[5];
504+ RefPicList rps[NB_RPS_TYPE];
502505
503506 const HEVCVPS *vps; ///< RefStruct reference
504507 const HEVCPPS *pps; ///< RefStruct reference
@@ -615,7 +618,8 @@ int ff_hevc_res_scale_sign_flag(HEVCLocalContext *lc, int idx); · 110 unmodified lines
615618 /**
616619 * Get the number of candidate references for the current frame.
617620 */
618−int ff_hevc_frame_nb_refs(const SliceHeader *sh, const HEVCPPS *pps);
621+int ff_hevc_frame_nb_refs(const SliceHeader *sh, const HEVCPPS *pps,
622+ unsigned layer_idx);
619623
620624 int ff_hevc_set_new_ref(HEVCContext *s, HEVCLayerContext *l, int poc);
621625
libavcodec/hevc/refs.c+45−13
@@ -162,6 +162,7 @@ int ff_hevc_set_new_ref(HEVCContext *s, HEVCLayerContext *l, int poc) · 161 unmodified lines
162162 return AVERROR(ENOMEM);
163163
164164 s->cur_frame = ref;
165+ l->cur_frame = ref;
165166 s->collocated_ref = NULL;
166167
167168 if (s->sh.pic_output_flag)
@@ -261,7 +262,9 @@ int ff_hevc_slice_rpl(HEVCContext *s) · 93 unmodified lines
261262 return ret;
262263
263264 if (!(s->rps[ST_CURR_BEF].nb_refs + s->rps[ST_CURR_AFT].nb_refs +
264− s->rps[LT_CURR].nb_refs) && !s->pps->pps_curr_pic_ref_enabled_flag) {
265+ s->rps[LT_CURR].nb_refs +
266+ s->rps[INTER_LAYER0].nb_refs + s->rps[INTER_LAYER1].nb_refs) &&
267+ !s->pps->pps_curr_pic_ref_enabled_flag) {
265268 av_log(s->avctx, AV_LOG_ERROR, "Zero refs in the frame RPS.\n");
266269 return AVERROR_INVALIDDATA;
267270 }
@@ -271,11 +274,14 @@ int ff_hevc_slice_rpl(HEVCContext *s) · 3 unmodified lines
271274 RefPicList *rpl = &s->cur_frame->refPicList[list_idx];
272275
273276 /* The order of the elements is
274− * ST_CURR_BEF - ST_CURR_AFT - LT_CURR for the L0 and
275− * ST_CURR_AFT - ST_CURR_BEF - LT_CURR for the L1 */
276− int cand_lists[3] = { list_idx ? ST_CURR_AFT : ST_CURR_BEF,
277− list_idx ? ST_CURR_BEF : ST_CURR_AFT,
278− LT_CURR };
277+ * ST_CURR_BEF - INTER_LAYER0 - ST_CURR_AFT - LT_CURR - INTER_LAYER1 for the L0 and
278+ * ST_CURR_AFT - INTER_LAYER1 - ST_CURR_BEF - LT_CURR - INTER_LAYER0 for the L1 */
279+ int cand_lists[] = { list_idx ? ST_CURR_AFT : ST_CURR_BEF,
280+ list_idx ? INTER_LAYER1 : INTER_LAYER0,
281+ list_idx ? ST_CURR_BEF : ST_CURR_AFT,
282+ LT_CURR,
283+ list_idx ? INTER_LAYER0 : INTER_LAYER1
284+ };
279285
280286 /* concatenate the candidate lists for the current frame */
281287 while (rpl_tmp.nb_refs < sh->nb_refs[list_idx]) {
@@ -284,7 +290,11 @@ int ff_hevc_slice_rpl(HEVCContext *s) · 2 unmodified lines
284290 for (j = 0; j < rps->nb_refs && rpl_tmp.nb_refs < HEVC_MAX_REFS; j++) {
285291 rpl_tmp.list[rpl_tmp.nb_refs] = rps->list[j];
286292 rpl_tmp.ref[rpl_tmp.nb_refs] = rps->ref[j];
287− rpl_tmp.isLongTerm[rpl_tmp.nb_refs] = i == 2;
293+ // multiview inter-layer refs are treated as long-term here,
294+ // cf. G.8.1.3
295+ rpl_tmp.isLongTerm[rpl_tmp.nb_refs] = cand_lists[i] == LT_CURR ||
296+ cand_lists[i] == INTER_LAYER0 ||
297+ cand_lists[i] == INTER_LAYER1;
288298 rpl_tmp.nb_refs++;
289299 }
290300 }
@@ -423,11 +433,6 @@ int ff_hevc_frame_rps(HEVCContext *s, HEVCLayerContext *l) · 132 unmodified lines
423433 RefPicList *rps = s->rps;
424434 int i, ret = 0;
425435
426− if (!short_rps) {
427− rps[0].nb_refs = rps[1].nb_refs = 0;
428− return 0;
429− }
430−
431436 unref_missing_refs(l);
432437
433438 /* clear the reference flags on all frames except the current one */
@@ -443,6 +448,9 @@ int ff_hevc_frame_rps(HEVCContext *s, HEVCLayerContext *l) · 9 unmodified lines
443448 for (i = 0; i < NB_RPS_TYPE; i++)
444449 rps[i].nb_refs = 0;
445450
451+ if (!short_rps)
452+ goto inter_layer;
453+
446454 /* add the short refs */
447455 for (i = 0; i < short_rps->num_delta_pocs; i++) {
448456 int poc = s->poc + short_rps->delta_poc[i];
@@ -472,6 +480,24 @@ int ff_hevc_frame_rps(HEVCContext *s, HEVCLayerContext *l) · 23 unmodified lines
472480 goto fail;
473481 }
474482
483+inter_layer:
484+ /* add inter-layer refs */
485+ if (s->sh.inter_layer_pred) {
486+ HEVCLayerContext *l0 = &s->layers[0];
487+
488+ av_assert0(l != l0);
489+
490+ /* Given the assumption of at most two layers, refPicSet0Flag is
491+ * always 1, so only RefPicSetInterLayer0 can ever contain a frame. */
492+ if (l0->cur_frame) {
493+ // inter-layer refs are treated as short-term here, cf. F.8.1.6
494+ ret = add_candidate_ref(s, l0, &rps[INTER_LAYER0], l0->cur_frame->poc,
495+ HEVC_FRAME_FLAG_SHORT_REF, 1);
496+ if (ret < 0)
497+ goto fail;
498+ }
499+ }
500+
475501 fail:
476502 /* release any frames that are now unused */
477503 for (i = 0; i < FF_ARRAY_ELEMS(l->DPB); i++)
@@ -480,7 +506,8 @@ fail: · 2 unmodified lines
480506 return ret;
481507 }
482508
483−int ff_hevc_frame_nb_refs(const SliceHeader *sh, const HEVCPPS *pps)
509+int ff_hevc_frame_nb_refs(const SliceHeader *sh, const HEVCPPS *pps,
510+ unsigned layer_idx)
484511 {
485512 int ret = 0;
486513 int i;
@@ -499,6 +526,11 @@ int ff_hevc_frame_nb_refs(const SliceHeader *sh, const HEVCPPS *pps) · 12 unmodified lines
499526 ret += !!long_rps->used[i];
500527 }
501528
529+ if (sh->inter_layer_pred) {
530+ av_assert0(pps->sps->vps->num_direct_ref_layers[layer_idx] < 2);
531+ ret++;
532+ }
533+
502534 if (pps->pps_curr_pic_ref_enabled_flag)
503535 ret++;
504536
libavcodec/nvdec_hevc.c+1−1
@@ -188,7 +188,7 @@ static int nvdec_hevc_start_frame(AVCodecContext *avctx, · 187 unmodified lines
188188
189189 .NumBitsForShortTermRPSInSlice = s->sh.short_term_rps ? s->sh.short_term_ref_pic_set_size : 0,
190190 .NumDeltaPocsOfRefRpsIdx = s->sh.short_term_rps ? s->sh.short_term_rps->rps_idx_num_delta_pocs : 0,
191− .NumPocTotalCurr = ff_hevc_frame_nb_refs(&s->sh, pps),
191+ .NumPocTotalCurr = ff_hevc_frame_nb_refs(&s->sh, pps, s->cur_layer),
192192 .NumPocStCurrBefore = s->rps[ST_CURR_BEF].nb_refs,
193193 .NumPocStCurrAfter = s->rps[ST_CURR_AFT].nb_refs,
194194 .NumPocLtCurr = s->rps[LT_CURR].nb_refs,
libavcodec/vdpau_hevc.c+1−1
@@ -206,7 +206,7 @@ static int vdpau_hevc_start_frame(AVCodecContext *avctx, · 205 unmodified lines
206206 }
207207 }
208208 /* See section 7.4.7.2 of the specification. */
209− info->NumPocTotalCurr = ff_hevc_frame_nb_refs(&h->sh, pps);
209+ info->NumPocTotalCurr = ff_hevc_frame_nb_refs(&h->sh, pps, h->cur_layer);
210210 if (sh->short_term_ref_pic_set_sps_flag == 0 && sh->short_term_rps) {
211211 /* Corresponds to specification field, NumDeltaPocs[RefRpsIdx].
212212 Only applicable when short_term_ref_pic_set_sps_flag == 0.

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t