History

avformat/mov: add support for Immersive Audio Model and Formats in ISOBMFF

James Almer committed 2 years ago

Commit fe637161 parent 9ba327e3 files · +290 −17Browse files

Unified · Split

3 files
  1. configure +1 −1
  2. libavformat/isom.h +3 −0
  3. libavformat/mov.c +286 −16
configure+1−1
@@ -3552,7 +3552,7 @@ matroska_demuxer_suggest="bzlib zlib" · 3551 unmodified lines
35523552 matroska_muxer_select="mpeg4audio riffenc aac_adtstoasc_bsf pgs_frame_merge_bsf vp9_superframe_bsf"
35533553 mlp_demuxer_select="mlp_parser"
35543554 mmf_muxer_select="riffenc"
3555−mov_demuxer_select="iso_media riffdec"
3555+mov_demuxer_select="iso_media riffdec iamfdec"
35563556 mov_demuxer_suggest="zlib"
35573557 mov_muxer_select="iso_media riffenc rtpenc_chain vp9_superframe_bsf aac_adtstoasc_bsf ac3_parser"
35583558 mp3_demuxer_select="mpegaudio_parser"
libavformat/isom.h+3−0
@@ -168,6 +168,7 @@ typedef struct MOVStreamContext { · 167 unmodified lines
168168 AVIOContext *pb;
169169 int refcount;
170170 int pb_is_copied;
171+ int id; ///< AVStream id
171172 int ffindex; ///< AVStream index
172173 int next_chunk;
173174 unsigned int chunk_count;
@@ -264,6 +265,8 @@ typedef struct MOVStreamContext { · 90 unmodified lines
264265 AVEncryptionInfo *default_encrypted_sample;
265266 MOVEncryptionIndex *encryption_index;
266267 } cenc;
268+
269+ struct IAMFDemuxContext *iamf;
267270 } MOVStreamContext;
268271
269272 typedef struct HEIFItem {
libavformat/mov.c+286−16
@@ -58,6 +58,8 @@ · 57 unmodified lines
5858 #include "internal.h"
5959 #include "avio_internal.h"
6060 #include "demux.h"
61+#include "iamf_parse.h"
62+#include "iamf_reader.h"
6163 #include "dovi_isom.h"
6264 #include "riff.h"
6365 #include "isom.h"
@@ -212,6 +214,7 @@ static int mov_read_covr(MOVContext *c, AVIOContext *pb, int type, int len) · 148 unmodified lines
212214 }
213215 st = c->fc->streams[c->fc->nb_streams - 1];
214216 st->priv_data = sc;
217+ sc->id = st->id;
215218 sc->refcount = 1;
216219
217220 if (st->attached_pic.size >= 8 && id != AV_CODEC_ID_BMP) {
@@ -836,6 +839,183 @@ static int mov_read_dac3(MOVContext *c, AVIOContext *pb, MOVAtom atom) · 618 unmodified lines
836839 return 0;
837840 }
838841
842+static int mov_read_iacb(MOVContext *c, AVIOContext *pb, MOVAtom atom)
843+{
844+ AVStream *st;
845+ MOVStreamContext *sc;
846+ FFIOContext b;
847+ AVIOContext *descriptor_pb;
848+ AVDictionary *metadata;
849+ IAMFContext *iamf;
850+ int64_t start_time, duration;
851+ unsigned descriptors_size;
852+ int nb_frames, disposition;
853+ int version, ret;
854+
855+ if (atom.size < 5)
856+ return AVERROR_INVALIDDATA;
857+
858+ if (c->fc->nb_streams < 1)
859+ return 0;
860+
861+ version = avio_r8(pb);
862+ if (version != 1) {
863+ av_log(c->fc, AV_LOG_ERROR, "%s configurationVersion %d",
864+ version < 1 ? "invalid" : "unsupported", version);
865+ return AVERROR_INVALIDDATA;
866+ }
867+
868+ descriptors_size = ffio_read_leb(pb);
869+ if (!descriptors_size || descriptors_size > INT_MAX)
870+ return AVERROR_INVALIDDATA;
871+
872+ st = c->fc->streams[c->fc->nb_streams - 1];
873+ sc = st->priv_data;
874+
875+ sc->iamf = av_mallocz(sizeof(*sc->iamf));
876+ if (!sc->iamf)
877+ return AVERROR(ENOMEM);
878+ iamf = &sc->iamf->iamf;
879+
880+ st->codecpar->extradata = av_malloc(descriptors_size);
881+ if (!st->codecpar->extradata)
882+ return AVERROR(ENOMEM);
883+ st->codecpar->extradata_size = descriptors_size;
884+
885+ ret = avio_read(pb, st->codecpar->extradata, descriptors_size);
886+ if (ret != descriptors_size)
887+ return ret < 0 ? ret : AVERROR_INVALIDDATA;
888+
889+ ffio_init_read_context(&b, st->codecpar->extradata, descriptors_size);
890+ descriptor_pb = &b.pub;
891+
892+ ret = ff_iamfdec_read_descriptors(iamf, descriptor_pb, descriptors_size, c->fc);
893+ if (ret < 0)
894+ return ret;
895+
896+ metadata = st->metadata;
897+ st->metadata = NULL;
898+ start_time = st->start_time;
899+ nb_frames = st->nb_frames;
900+ duration = st->duration;
901+ disposition = st->disposition;
902+
903+ for (int i = 0; i < iamf->nb_audio_elements; i++) {
904+ IAMFAudioElement *audio_element = iamf->audio_elements[i];
905+ AVStreamGroup *stg =
906+ avformat_stream_group_create(c->fc, AV_STREAM_GROUP_PARAMS_IAMF_AUDIO_ELEMENT, NULL);
907+
908+ if (!stg) {
909+ ret = AVERROR(ENOMEM);
910+ goto fail;
911+ }
912+
913+ av_iamf_audio_element_free(&stg->params.iamf_audio_element);
914+ stg->id = audio_element->audio_element_id;
915+ /* Transfer ownership */
916+ stg->params.iamf_audio_element = audio_element->element;
917+ audio_element->element = NULL;
918+
919+ for (int j = 0; j < audio_element->nb_substreams; j++) {
920+ IAMFSubStream *substream = &audio_element->substreams[j];
921+ AVStream *stream;
922+
923+ if (!i && !j) {
924+ if (audio_element->layers[0].substream_count != 1)
925+ st->disposition &= ~AV_DISPOSITION_DEFAULT;
926+ stream = st;
927+ } else
928+ stream = avformat_new_stream(c->fc, NULL);
929+ if (!stream) {
930+ ret = AVERROR(ENOMEM);
931+ goto fail;
932+ }
933+
934+ stream->start_time = start_time;
935+ stream->nb_frames = nb_frames;
936+ stream->duration = duration;
937+ stream->disposition = disposition;
938+ if (stream != st) {
939+ stream->priv_data = sc;
940+ sc->refcount++;
941+ }
942+
943+ if (i || j) {
944+ stream->disposition |= AV_DISPOSITION_DEPENDENT;
945+ if (audio_element->layers[0].substream_count == 1)
946+ stream->disposition &= ~AV_DISPOSITION_DEFAULT;
947+ }
948+
949+ ret = avcodec_parameters_copy(stream->codecpar, substream->codecpar);
950+ if (ret < 0)
951+ goto fail;
952+
953+ stream->id = substream->audio_substream_id;
954+
955+ avpriv_set_pts_info(st, 64, 1, sc->time_scale);
956+
957+ ret = avformat_stream_group_add_stream(stg, stream);
958+ if (ret < 0)
959+ goto fail;
960+ }
961+
962+ ret = av_dict_copy(&stg->metadata, metadata, 0);
963+ if (ret < 0)
964+ goto fail;
965+ }
966+
967+ for (int i = 0; i < iamf->nb_mix_presentations; i++) {
968+ IAMFMixPresentation *mix_presentation = iamf->mix_presentations[i];
969+ const AVIAMFMixPresentation *mix = mix_presentation->cmix;
970+ AVStreamGroup *stg =
971+ avformat_stream_group_create(c->fc, AV_STREAM_GROUP_PARAMS_IAMF_MIX_PRESENTATION, NULL);
972+
973+ if (!stg) {
974+ ret = AVERROR(ENOMEM);
975+ goto fail;
976+ }
977+
978+ av_iamf_mix_presentation_free(&stg->params.iamf_mix_presentation);
979+ stg->id = mix_presentation->mix_presentation_id;
980+ /* Transfer ownership */
981+ stg->params.iamf_mix_presentation = mix_presentation->mix;
982+ mix_presentation->mix = NULL;
983+
984+ for (int j = 0; j < mix->nb_submixes; j++) {
985+ const AVIAMFSubmix *submix = mix->submixes[j];
986+
987+ for (int k = 0; k < submix->nb_elements; k++) {
988+ const AVIAMFSubmixElement *submix_element = submix->elements[k];
989+ const AVStreamGroup *audio_element = NULL;
990+
991+ for (int l = 0; l < c->fc->nb_stream_groups; l++)
992+ if (c->fc->stream_groups[l]->type == AV_STREAM_GROUP_PARAMS_IAMF_AUDIO_ELEMENT &&
993+ c->fc->stream_groups[l]->id == submix_element->audio_element_id) {
994+ audio_element = c->fc->stream_groups[l];
995+ break;
996+ }
997+ av_assert0(audio_element);
998+
999+ for (int l = 0; l < audio_element->nb_streams; l++) {
1000+ ret = avformat_stream_group_add_stream(stg, audio_element->streams[l]);
1001+ if (ret < 0 && ret != AVERROR(EEXIST))
1002+ goto fail;
1003+ }
1004+ }
1005+ }
1006+
1007+ ret = av_dict_copy(&stg->metadata, metadata, 0);
1008+ if (ret < 0)
1009+ goto fail;
1010+ }
1011+
1012+ ret = 0;
1013+fail:
1014+ av_dict_free(&metadata);
1015+
1016+ return ret;
1017+}
1018+
8391019 static int mov_read_dec3(MOVContext *c, AVIOContext *pb, MOVAtom atom)
8401020 {
8411021 AVStream *st;
@@ -1331,7 +1511,7 @@ static int64_t get_frag_time(AVFormatContext *s, AVStream *dst_st, · 489 unmodified lines
13311511 // If the stream is referenced by any sidx, limit the search
13321512 // to fragments that referenced this stream in the sidx
13331513 if (sc->has_sidx) {
1334− frag_stream_info = get_frag_stream_info(frag_index, index, dst_st->id);
1514+ frag_stream_info = get_frag_stream_info(frag_index, index, sc->id);
13351515 if (frag_stream_info->sidx_pts != AV_NOPTS_VALUE)
13361516 return frag_stream_info->sidx_pts;
13371517 if (frag_stream_info->first_tfra_pts != AV_NOPTS_VALUE)
@@ -1342,9 +1522,11 @@ static int64_t get_frag_time(AVFormatContext *s, AVStream *dst_st, · 4 unmodified lines
13421522 for (i = 0; i < frag_index->item[index].nb_stream_info; i++) {
13431523 AVStream *frag_stream = NULL;
13441524 frag_stream_info = &frag_index->item[index].stream_info[i];
1345− for (j = 0; j < s->nb_streams; j++)
1346− if (s->streams[j]->id == frag_stream_info->id)
1525+ for (j = 0; j < s->nb_streams; j++) {
1526+ MOVStreamContext *sc2 = s->streams[j]->priv_data;
1527+ if (sc2->id == frag_stream_info->id)
13471528 frag_stream = s->streams[j];
1529+ }
13481530 if (!frag_stream) {
13491531 av_log(s, AV_LOG_WARNING, "No stream matching sidx ID found.\n");
13501532 continue;
@@ -1410,12 +1592,13 @@ static int update_frag_index(MOVContext *c, int64_t offset) · 59 unmodified lines
14101592
14111593 for (i = 0; i < c->fc->nb_streams; i++) {
14121594 // Avoid building frag index if streams lack track id.
1413− if (c->fc->streams[i]->id < 0) {
1595+ MOVStreamContext *sc = c->fc->streams[i]->priv_data;
1596+ if (sc->id < 0) {
14141597 av_free(frag_stream_info);
14151598 return AVERROR_INVALIDDATA;
14161599 }
14171600
1418− frag_stream_info[i].id = c->fc->streams[i]->id;
1601+ frag_stream_info[i].id = sc->id;
14191602 frag_stream_info[i].sidx_pts = AV_NOPTS_VALUE;
14201603 frag_stream_info[i].tfdt_dts = AV_NOPTS_VALUE;
14211604 frag_stream_info[i].next_trun_dts = AV_NOPTS_VALUE;
@@ -3210,7 +3393,7 @@ static int mov_read_stts(MOVContext *c, AVIOContext *pb, MOVAtom atom) · 1788 unmodified lines
32103393 "All samples in data stream index:id [%d:%d] have zero "
32113394 "duration, stream set to be discarded by default. Override "
32123395 "using AVStream->discard or -discard for ffmpeg command.\n",
3213− st->index, st->id);
3396+ st->index, sc->id);
32143397 st->discard = AVDISCARD_ALL;
32153398 }
32163399 sc->track_end = duration;
@@ -4590,6 +4773,50 @@ static void fix_timescale(MOVContext *c, MOVStreamContext *sc) · 1373 unmodified lines
45904773 }
45914774 }
45924775
4776+static int mov_update_iamf_streams(MOVContext *c, const AVStream *st)
4777+{
4778+ const MOVStreamContext *sc = st->priv_data;
4779+ const IAMFContext *iamf = &sc->iamf->iamf;
4780+
4781+ for (int i = 0; i < iamf->nb_audio_elements; i++) {
4782+ const AVStreamGroup *stg = NULL;
4783+
4784+ for (int j = 0; j < c->fc->nb_stream_groups; j++)
4785+ if (c->fc->stream_groups[j]->id == iamf->audio_elements[i]->audio_element_id)
4786+ stg = c->fc->stream_groups[j];
4787+ av_assert0(stg);
4788+
4789+ for (int j = 0; j < stg->nb_streams; j++) {
4790+ const FFStream *sti = cffstream(st);
4791+ AVStream *out = stg->streams[j];
4792+ FFStream *out_sti = ffstream(stg->streams[j]);
4793+
4794+ out->codecpar->bit_rate = 0;
4795+
4796+ if (out == st)
4797+ continue;
4798+
4799+ out->time_base = st->time_base;
4800+ out->start_time = st->start_time;
4801+ out->duration = st->duration;
4802+ out->nb_frames = st->nb_frames;
4803+ out->discard = st->discard;
4804+
4805+ av_assert0(!out_sti->index_entries);
4806+ out_sti->index_entries = av_malloc(sti->index_entries_allocated_size);
4807+ if (!out_sti->index_entries)
4808+ return AVERROR(ENOMEM);
4809+
4810+ out_sti->index_entries_allocated_size = sti->index_entries_allocated_size;
4811+ out_sti->nb_index_entries = sti->nb_index_entries;
4812+ out_sti->skip_samples = sti->skip_samples;
4813+ memcpy(out_sti->index_entries, sti->index_entries, sti->index_entries_allocated_size);
4814+ }
4815+ }
4816+
4817+ return 0;
4818+}
4819+
45934820 static int mov_read_trak(MOVContext *c, AVIOContext *pb, MOVAtom atom)
45944821 {
45954822 AVStream *st;
@@ -4670,6 +4897,12 @@ static int mov_read_trak(MOVContext *c, AVIOContext *pb, MOVAtom atom) · 74 unmodified lines
46704897
46714898 mov_build_index(c, st);
46724899
4900+ if (sc->iamf) {
4901+ ret = mov_update_iamf_streams(c, st);
4902+ if (ret < 0)
4903+ return ret;
4904+ }
4905+
46734906 if (sc->dref_id-1 < sc->drefs_count && sc->drefs[sc->dref_id-1].path) {
46744907 MOVDref *dref = &sc->drefs[sc->dref_id - 1];
46754908 if (c->enable_drefs) {
@@ -4902,6 +5135,7 @@ static int heif_add_stream(MOVContext *c, HEIFItem *item) · 226 unmodified lines
49025135 st->priv_data = sc;
49035136 st->codecpar->codec_type = AVMEDIA_TYPE_VIDEO;
49045137 st->codecpar->codec_id = mov_codec_id(st, item->type);
5138+ sc->id = st->id;
49055139 sc->ffindex = st->index;
49065140 c->trak_index = st->index;
49075141 st->avg_frame_rate.num = st->avg_frame_rate.den = 1;
@@ -5000,6 +5234,7 @@ static int mov_read_tkhd(MOVContext *c, AVIOContext *pb, MOVAtom atom) · 92 unmodified lines
50005234 avio_rb32(pb); /* modification time */
50015235 }
50025236 st->id = (int)avio_rb32(pb); /* track id (NOT 0 !)*/
5237+ sc->id = st->id;
50035238 avio_rb32(pb); /* reserved */
50045239
50055240 /* highlevel (considering edits) duration in movie timebase */
@@ -5174,7 +5409,8 @@ static int mov_read_tfdt(MOVContext *c, AVIOContext *pb, MOVAtom atom) · 168 unmodified lines
51745409 int64_t base_media_decode_time;
51755410
51765411 for (i = 0; i < c->fc->nb_streams; i++) {
5177− if (c->fc->streams[i]->id == frag->track_id) {
5412+ sc = c->fc->streams[i]->priv_data;
5413+ if (sc->id == frag->track_id) {
51785414 st = c->fc->streams[i];
51795415 break;
51805416 }
@@ -5227,7 +5463,8 @@ static int mov_read_trun(MOVContext *c, AVIOContext *pb, MOVAtom atom) · 46 unmodified lines
52275463 }
52285464
52295465 for (i = 0; i < c->fc->nb_streams; i++) {
5230− if (c->fc->streams[i]->id == frag->track_id) {
5466+ sc = c->fc->streams[i]->priv_data;
5467+ if (sc->id == frag->track_id) {
52315468 st = c->fc->streams[i];
52325469 sti = ffstream(st);
52335470 break;
@@ -5530,7 +5767,8 @@ static int mov_read_sidx(MOVContext *c, AVIOContext *pb, MOVAtom atom) · 296 unmodified lines
55305767
55315768 track_id = avio_rb32(pb); // Reference ID
55325769 for (i = 0; i < c->fc->nb_streams; i++) {
5533− if (c->fc->streams[i]->id == track_id) {
5770+ sc = c->fc->streams[i]->priv_data;
5771+ if (sc->id == track_id) {
55345772 st = c->fc->streams[i];
55355773 break;
55365774 }
@@ -6447,7 +6685,8 @@ static int get_current_encryption_info(MOVContext *c, MOVEncryptionIndex **encry · 910 unmodified lines
64476685 frag_stream_info = get_current_frag_stream_info(&c->frag_index);
64486686 if (frag_stream_info) {
64496687 for (i = 0; i < c->fc->nb_streams; i++) {
6450− if (c->fc->streams[i]->id == frag_stream_info->id) {
6688+ *sc = c->fc->streams[i]->priv_data;
6689+ if ((*sc)->id == frag_stream_info->id) {
64516690 st = c->fc->streams[i];
64526691 break;
64536692 }
@@ -7391,7 +7630,7 @@ static int cenc_filter(MOVContext *mov, AVStream* st, MOVStreamContext *sc, AVPa · 937 unmodified lines
73917630 AVEncryptionInfo *encrypted_sample;
73927631 int encrypted_index, ret;
73937632
7394− frag_stream_info = get_frag_stream_info_from_pkt(&mov->frag_index, pkt, st->id);
7633+ frag_stream_info = get_frag_stream_info_from_pkt(&mov->frag_index, pkt, sc->id);
73957634 encrypted_index = current_index;
73967635 encryption_index = NULL;
73977636 if (frag_stream_info) {
@@ -8194,6 +8433,7 @@ static const MOVParseTableEntry mov_default_parse_table[] = { · 796 unmodified lines
81948433 { MKTAG('i','p','r','p'), mov_read_iprp },
81958434 { MKTAG('i','i','n','f'), mov_read_iinf },
81968435 { MKTAG('a','m','v','e'), mov_read_amve }, /* ambient viewing environment box */
8436+{ MKTAG('i','a','c','b'), mov_read_iacb },
81978437 { 0, NULL }
81988438 };
81998439
@@ -8425,11 +8665,13 @@ static void mov_read_chapters(AVFormatContext *s) · 225 unmodified lines
84258665 AVStream *st = NULL;
84268666 FFStream *sti = NULL;
84278667 chapter_track = mov->chapter_tracks[j];
8428− for (i = 0; i < s->nb_streams; i++)
8429− if (s->streams[i]->id == chapter_track) {
8668+ for (i = 0; i < s->nb_streams; i++) {
8669+ sc = mov->fc->streams[i]->priv_data;
8670+ if (sc->id == chapter_track) {
84308671 st = s->streams[i];
84318672 break;
84328673 }
8674+ }
84338675 if (!st) {
84348676 av_log(s, AV_LOG_ERROR, "Referenced QT chapter track not found\n");
84358677 continue;
@@ -8662,6 +8904,10 @@ static void mov_free_stream_context(AVFormatContext *s, AVStream *st) · 226 unmodified lines
86628904 av_freep(&sc->mastering);
86638905 av_freep(&sc->coll);
86648906 av_freep(&sc->ambient);
8907+
8908+ if (sc->iamf)
8909+ ff_iamf_read_deinit(sc->iamf);
8910+ av_freep(&sc->iamf);
86658911 }
86668912
86678913 static int mov_read_close(AVFormatContext *s)
@@ -8916,9 +9162,11 @@ static int mov_read_header(AVFormatContext *s) · 248 unmodified lines
89169162 AVDictionaryEntry *tcr;
89179163 int tmcd_st_id = -1;
89189164
8919− for (j = 0; j < s->nb_streams; j++)
8920− if (s->streams[j]->id == sc->timecode_track)
9165+ for (j = 0; j < s->nb_streams; j++) {
9166+ MOVStreamContext *sc2 = s->streams[j]->priv_data;
9167+ if (sc2->id == sc->timecode_track)
89219168 tmcd_st_id = j;
9169+ }
89229170
89239171 if (tmcd_st_id < 0 || tmcd_st_id == i)
89249172 continue;
@@ -9298,7 +9546,29 @@ static int mov_read_packet(AVFormatContext *s, AVPacket *pkt) · 373 unmodified lines
92989546
92999547 if (st->codecpar->codec_id == AV_CODEC_ID_EIA_608 && sample->size > 8)
93009548 ret = get_eia608_packet(sc->pb, pkt, sample->size);
9301− else
9549+ else if (sc->iamf) {
9550+ int64_t pts, dts, pos, duration;
9551+ int flags, size = sample->size;
9552+ ret = mov_finalize_packet(s, st, sample, current_index, pkt);
9553+ pts = pkt->pts; dts = pkt->dts;
9554+ pos = pkt->pos; flags = pkt->flags;
9555+ duration = pkt->duration;
9556+ while (!ret && size > 0) {
9557+ ret = ff_iamf_read_packet(s, sc->iamf, sc->pb, size, pkt);
9558+ if (ret < 0) {
9559+ if (should_retry(sc->pb, ret))
9560+ mov_current_sample_dec(sc);
9561+ return ret;
9562+ }
9563+ size -= ret;
9564+ pkt->pts = pts; pkt->dts = dts;
9565+ pkt->pos = pos; pkt->flags |= flags;
9566+ pkt->duration = duration;
9567+ ret = ff_buffer_packet(s, pkt);
9568+ }
9569+ if (!ret)
9570+ return FFERROR_REDO;
9571+ } else
93029572 ret = av_get_packet(sc->pb, pkt, sample->size);
93039573 if (ret < 0) {
93049574 if (should_retry(sc->pb, ret)) {

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t