avformat: add AV1 RTP depacketizer and packetizer

Add RTP packetizer and depacketizer according to (most) of the official AV1 RTP specification. This enables streaming via RTSP between ffmpeg and ffmpeg and has also been tested to work with AV1 RTSP streams via GStreamer. It also adds the required SDP attributes for AV1. AV1 RTP encoding is marked as experimental due to draft specification status, debug amount reduced and other changes suggested by Tristan. Added optional code for searching the sequence header to determine the first packet for broken AV1 encoders / parsers. Stops depacketizing on corruption until next keyframe, no longer prematurely issues packet on decoding if temporal unit was not complete yet.

Chris Hodges committed 2 years ago

Commit 330c8f81 parent b3d73df10 files · +996 −0Browse files

Unified · Split

10 files

libavformat/Makefile +2 −0
libavformat/demux.c +1 −0
libavformat/rtp_av1.h +128 −0
libavformat/rtpdec.c +1 −0
libavformat/rtpdec_av1.c +461 −0
libavformat/rtpdec_formats.h +1 −0
libavformat/rtpenc.c +14 −0
libavformat/rtpenc.h +1 −0
libavformat/rtpenc_av1.c +357 −0
libavformat/sdp.c +30 −0

libavformat/Makefile+2−0

@@ -47,6 +47,7 @@ OBJS-$(CONFIG_RTPDEC) += rdt.o \ · 46 unmodified lines
47	47	rtpdec_ac3.o \
48	48	rtpdec_amr.o \
49	49	rtpdec_asf.o \
	50	+ rtpdec_av1.o \
50	51	rtpdec_dv.o \
51	52	rtpdec_g726.o \
52	53	rtpdec_h261.o \
@@ -517,6 +518,7 @@ OBJS-$(CONFIG_RTP_MUXER) += rtp.o \ · 464 unmodified lines
517	518	rtpenc_aac.o \
518	519	rtpenc_latm.o \
519	520	rtpenc_amr.o \
	521	+ rtpenc_av1.o \
520	522	rtpenc_h261.o \
521	523	rtpenc_h263.o \
522	524	rtpenc_h263_rfc2190.o \

libavformat/demux.c+1−0

@@ -111,6 +111,7 @@ static int set_codec_from_probe_data(AVFormatContext s, AVStream st, · 110 unmodified lines
111	111	{ "aac", AV_CODEC_ID_AAC, AVMEDIA_TYPE_AUDIO },
112	112	{ "ac3", AV_CODEC_ID_AC3, AVMEDIA_TYPE_AUDIO },
113	113	{ "aptx", AV_CODEC_ID_APTX, AVMEDIA_TYPE_AUDIO },
	114	+ { "av1", AV_CODEC_ID_AV1, AVMEDIA_TYPE_VIDEO },
114	115	{ "dts", AV_CODEC_ID_DTS, AVMEDIA_TYPE_AUDIO },
115	116	{ "dvbsub", AV_CODEC_ID_DVB_SUBTITLE, AVMEDIA_TYPE_SUBTITLE },
116	117	{ "dvbtxt", AV_CODEC_ID_DVB_TELETEXT, AVMEDIA_TYPE_SUBTITLE },

libavformat/rtp_av1.h+128−0

@@ -0,0 +1,128 @@
	1	+/*
	2	+ * Shared definitions and helper functions for
	3	+ * AV1 (de)packetization.
	4	+ * Copyright (c) 2024 Axis Communications
	5	+ *
	6	+ * This file is part of FFmpeg.
	7	+ *
	8	+ * FFmpeg is free software; you can redistribute it and/or
	9	+ * modify it under the terms of the GNU Lesser General Public
	10	+ * License as published by the Free Software Foundation; either
	11	+ * version 2.1 of the License, or (at your option) any later version.
	12	+ *
	13	+ * FFmpeg is distributed in the hope that it will be useful,
	14	+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
	15	+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	16	+ * Lesser General Public License for more details.
	17	+ *
	18	+ * You should have received a copy of the GNU Lesser General Public
	19	+ * License along with FFmpeg; if not, write to the Free Software
	20	+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
	21	+ */
	22	+
	23	+/**
	24	+ * @file
	25	+ * @brief shared defines and functions for AV1 RTP dec/enc
	26	+ * @author Chris Hodges <chris.hodges@axis.com>
	27	+ */
	28	+
	29	+#ifndef AVFORMAT_RTP_AV1_H
	30	+#define AVFORMAT_RTP_AV1_H
	31	+
	32	+// define a couple of flags and bit fields
	33	+#define AV1B_OBU_FORBIDDEN 7
	34	+#define AV1F_OBU_FORBIDDEN (1u << AV1B_OBU_FORBIDDEN)
	35	+#define AV1S_OBU_TYPE 3
	36	+#define AV1M_OBU_TYPE 15
	37	+#define AV1B_OBU_EXTENSION_FLAG 2
	38	+#define AV1F_OBU_EXTENSION_FLAG (1u << AV1B_OBU_EXTENSION_FLAG)
	39	+#define AV1B_OBU_HAS_SIZE_FIELD 1
	40	+#define AV1F_OBU_HAS_SIZE_FIELD (1u << AV1B_OBU_HAS_SIZE_FIELD)
	41	+#define AV1B_OBU_RESERVED_1BIT 0
	42	+#define AV1F_OBU_RESERVED_1BIT (1u << AV1B_OBU_RESERVED_1BIT)
	43	+
	44	+#define AV1B_AGGR_HDR_FRAG_CONT 7
	45	+#define AV1F_AGGR_HDR_FRAG_CONT (1u << AV1B_AGGR_HDR_FRAG_CONT)
	46	+#define AV1B_AGGR_HDR_LAST_FRAG 6
	47	+#define AV1F_AGGR_HDR_LAST_FRAG (1u << AV1B_AGGR_HDR_LAST_FRAG)
	48	+#define AV1S_AGGR_HDR_NUM_OBUS 4
	49	+#define AV1M_AGGR_HDR_NUM_OBUS 3
	50	+#define AV1B_AGGR_HDR_FIRST_PKT 3
	51	+#define AV1F_AGGR_HDR_FIRST_PKT (1u << AV1B_AGGR_HDR_FIRST_PKT)
	52	+
	53	+/// calculate number of required LEB bytes for the given length
	54	+static inline unsigned int calc_leb_size(uint32_t length) {
	55	+ unsigned int num_lebs = 0;
	56	+ do {
	57	+ num_lebs++;
	58	+ length >>= 7;
	59	+ } while (length);
	60	+ return num_lebs;
	61	+}
	62	+
	63	+/// write out variable number of LEB bytes for the given length
	64	+static inline unsigned int write_leb(uint8_t *lebptr, uint32_t length) {
	65	+ unsigned int num_lebs = 0;
	66	+ do {
	67	+ num_lebs++;
	68	+ if (length < 0x80) {
	69	+ *lebptr = length;
	70	+ break;
	71	+ }
	72	+ *lebptr++ = length \| 0x80; // no need to mask out
	73	+ length >>= 7;
	74	+ } while (1);
	75	+ return num_lebs;
	76	+}
	77	+
	78	+/// write out fixed number of LEB bytes (may have "unused" bytes)
	79	+static inline void write_leb_n(uint8_t *lebptr, uint32_t length, unsigned int num_lebs) {
	80	+ for (int i = 0; i < num_lebs; i++) {
	81	+ if (i == num_lebs - 1) {
	82	+ *lebptr = length & 0x7f;
	83	+ } else {
	84	+ *lebptr++ = length \| 0x80; // no need to mask out
	85	+ }
	86	+ length >>= 7;
	87	+ }
	88	+}
	89	+
	90	+/// securely parse LEB bytes and return the resulting encoded length
	91	+static inline unsigned int parse_leb(AVFormatContext ctx, const uint8_t buf_ptr,
	92	+ uint32_t buffer_size, uint32_t *obu_size) {
	93	+ uint8_t leb128;
	94	+ unsigned int num_lebs = 0;
	95	+ *obu_size = 0;
	96	+ do {
	97	+ uint32_t leb7;
	98	+ if (!buffer_size) {
	99	+ av_log(ctx, AV_LOG_ERROR, "AV1: Out of data in OBU size field AV1 RTP packet\n");
	100	+ return 0;
	101	+ }
	102	+ leb128 = *buf_ptr++;
	103	+ leb7 = leb128 & 0x7f;
	104	+ buffer_size--;
	105	+ /* AV1 spec says that the maximum value returned from leb128 must fit in
	106	+ * 32 bits, so if the next byte will shift data out, we have some kind
	107	+ * of violation here. It is legal, though, to have the most significant
	108	+ * bytes with all zero bits (in the lower 7 bits). */
	109	+ if (((num_lebs == 4) && (leb7 >= 0x10)) \|\| ((num_lebs > 4) && leb7)) {
	110	+ av_log(ctx, AV_LOG_ERROR, "AV1: OBU size field exceeds 32 bit in AV1 RTP packet\n");
	111	+ return 0;
	112	+ }
	113	+ if ((num_lebs == 7) && (leb128 >= 0x80)) {
	114	+ /* leb128 is defined to be up to 8 bytes (why???), 8th byte MUST NOT
	115	+ * indicate continuation */
	116	+ av_log(ctx, AV_LOG_ERROR, "AV1: OBU size field consists of too many bytes in AV1 RTP packet\n");
	117	+ return 0;
	118	+ }
	119	+ // shifts >= 32 are undefined in C!
	120	+ if (num_lebs <= 4) {
	121	+ obu_size \|= leb7 << (7 num_lebs);
	122	+ }
	123	+ num_lebs++;
	124	+ } while (leb128 >= 0x80);
	125	+ return num_lebs;
	126	+}
	127	+
	128	+#endif /* AVFORMAT_RTP_AV1_H */

libavformat/rtpdec.c+1−0

@@ -83,6 +83,7 @@ static const RTPDynamicProtocolHandler *const rtp_dynamic_protocol_handler_list[ · 82 unmodified lines
83	83	&ff_ac3_dynamic_handler,
84	84	&ff_amr_nb_dynamic_handler,
85	85	&ff_amr_wb_dynamic_handler,
	86	+ &ff_av1_dynamic_handler,
86	87	&ff_dv_dynamic_handler,
87	88	&ff_g726_16_dynamic_handler,
88	89	&ff_g726_24_dynamic_handler,

libavformat/rtpdec_av1.c+461−0

@@ -0,0 +1,461 @@
	1	+/*
	2	+ * Depacketization for RTP Payload Format For AV1 (v1.0)
	3	+ * https://aomediacodec.github.io/av1-rtp-spec/
	4	+ * Copyright (c) 2024 Axis Communications
	5	+ *
	6	+ * This file is part of FFmpeg.
	7	+ *
	8	+ * FFmpeg is free software; you can redistribute it and/or
	9	+ * modify it under the terms of the GNU Lesser General Public
	10	+ * License as published by the Free Software Foundation; either
	11	+ * version 2.1 of the License, or (at your option) any later version.
	12	+ *
	13	+ * FFmpeg is distributed in the hope that it will be useful,
	14	+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
	15	+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	16	+ * Lesser General Public License for more details.
	17	+ *
	18	+ * You should have received a copy of the GNU Lesser General Public
	19	+ * License along with FFmpeg; if not, write to the Free Software
	20	+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
	21	+ */
	22	+
	23	+/**
	24	+ * @file
	25	+ * @brief AV1 / RTP depacketization code (RTP Payload Format For AV1 (v1.0))
	26	+ * @author Chris Hodges <chris.hodges@axis.com>
	27	+ * @note The process will restore TDs and put back size fields into headers.
	28	+ * It will also try to keep complete OBUs and remove partial OBUs
	29	+ * caused by packet drops and thus keep the stream syntactically intact.
	30	+ */
	31	+
	32	+#include "libavutil/avstring.h"
	33	+#include "libavutil/mem.h"
	34	+#include "avformat.h"
	35	+
	36	+#include "rtpdec.h"
	37	+#include "libavcodec/av1.h"
	38	+#include "rtp_av1.h"
	39	+
	40	+// enable tracing of packet data
	41	+//#define RTPDEC_AV1_VERBOSE_TRACE
	42	+
	43	+/**
	44	+ * RTP/AV1 specific private data.
	45	+ */
	46	+struct PayloadContext {
	47	+ uint32_t timestamp; ///< last received timestamp for frame
	48	+ uint8_t profile; ///< profile (main/high/professional)
	49	+ uint8_t level_idx; ///< level (0-31)
	50	+ uint8_t tier; ///< main tier or high tier
	51	+ uint16_t prev_seq; ///< sequence number of previous packet
	52	+ unsigned int frag_obu_size; ///< current total size of fragmented OBU
	53	+ unsigned int frag_pkt_leb_pos; ///< offset in buffer where OBU LEB starts
	54	+ unsigned int frag_lebs_res; ///< number of bytes reserved for LEB
	55	+ unsigned int frag_header_size; ///< size of OBU header (1 or 2)
	56	+ int needs_td; ///< indicates that a TD should be output
	57	+ int drop_fragment; ///< drop all fragments until next frame
	58	+ int keyframe_seen; ///< keyframe was seen
	59	+ int wait_for_keyframe; ///< message about waiting for keyframe has been issued
	60	+};
	61	+
	62	+static int sdp_parse_fmtp_config_av1(AVFormatContext *s,
	63	+ AVStream *stream,
	64	+ PayloadContext *av1_data,
	65	+ const char attr, const char value) {
	66	+ if (!strcmp(attr, "profile")) {
	67	+ av1_data->profile = atoi(value);
	68	+ av_log(s, AV_LOG_DEBUG, "RTP AV1 profile: %u\n", av1_data->profile);
	69	+ } else if (!strcmp(attr, "level-idx")) {
	70	+ av1_data->level_idx = atoi(value);
	71	+ av_log(s, AV_LOG_DEBUG, "RTP AV1 level: %u\n", av1_data->profile);
	72	+ } else if (!strcmp(attr, "tier")) {
	73	+ av1_data->tier = atoi(value);
	74	+ av_log(s, AV_LOG_DEBUG, "RTP AV1 tier: %u\n", av1_data->tier);
	75	+ }
	76	+ return 0;
	77	+}
	78	+
	79	+// return 0 on complete packet, -1 on partial packet
	80	+static int av1_handle_packet(AVFormatContext ctx, PayloadContext data,
	81	+ AVStream st, AVPacket pkt, uint32_t *timestamp,
	82	+ const uint8_t *buf, int len, uint16_t seq,
	83	+ int flags) {
	84	+ uint8_t aggr_hdr;
	85	+ int result = 0;
	86	+ int is_frag_cont;
	87	+ int is_last_fragmented;
	88	+ int is_first_pkt;
	89	+ unsigned int num_obus;
	90	+ unsigned int obu_cnt = 1;
	91	+ unsigned int rem_pkt_size = len;
	92	+ unsigned int pktpos;
	93	+ const uint8_t *buf_ptr = buf;
	94	+ uint16_t expected_seq = data->prev_seq + 1;
	95	+ int16_t seq_diff = seq - expected_seq;
	96	+
	97	+ data->prev_seq = seq;
	98	+
	99	+ if (!len) {
	100	+ av_log(ctx, AV_LOG_ERROR, "Empty AV1 RTP packet\n");
	101	+ return AVERROR_INVALIDDATA;
	102	+ }
	103	+ if (len < 2) {
	104	+ av_log(ctx, AV_LOG_ERROR, "AV1 RTP packet too short\n");
	105	+ return AVERROR_INVALIDDATA;
	106	+ }
	107	+
	108	+ /* The payload structure is supposed to be straight-forward, but there are a
	109	+ * couple of edge cases which need to be tackled and make things a bit more
	110	+ * complex.
	111	+ * These are mainly due to:
	112	+ * - To reconstruct the OBU size for fragmented packets and place it the OBU
	113	+ * header, the final size will not be known until the last fragment has
	114	+ * been parsed. However, the number LEBs in the header is variable
	115	+ * depending on the length of the payload.
	116	+ * - We are increasing the out-packet size while we are getting fragmented
	117	+ * OBUs. If an RTP packet gets dropped, we would create corrupted OBUs.
	118	+ * In this case we decide to drop the whole frame.
	119	+ */
	120	+
	121	+#ifdef RTPDEC_AV1_VERBOSE_TRACE
	122	+ av_log(ctx, AV_LOG_TRACE, "RTP Packet %d in (%x), len=%d:\n",
	123	+ seq, flags, len);
	124	+ av_hex_dump_log(ctx, AV_LOG_TRACE, buf, FFMIN(len, 64));
	125	+ av_log(ctx, AV_LOG_TRACE, "... end at offset %x:\n", FFMAX(len - 64, 0));
	126	+ av_hex_dump_log(ctx, AV_LOG_TRACE, buf + FFMAX(len - 64, 0), FFMIN(len - 64, 64));
	127	+#endif
	128	+
	129	+ /* 8 bit aggregate header: Z Y W W N - - - */
	130	+ aggr_hdr = *buf_ptr++;
	131	+ rem_pkt_size--;
	132	+
	133	+ /* Z: MUST be set to 1 if the first OBU element is an OBU fragment that is a
	134	+ * continuation of an OBU fragment from the previous packet, and MUST be set
	135	+ * to 0 otherwise */
	136	+ is_frag_cont = (aggr_hdr >> AV1B_AGGR_HDR_FRAG_CONT) & 1;
	137	+
	138	+ /* Y: MUST be set to 1 if the last OBU element is an OBU fragment that will
	139	+ * continue in the next packet, and MUST be set to 0 otherwise */
	140	+ is_last_fragmented = (aggr_hdr >> AV1B_AGGR_HDR_LAST_FRAG) & 1;
	141	+
	142	+ /* W: two bit field that describes the number of OBU elements in the packet.
	143	+ * This field MUST be set equal to 0 or equal to the number of OBU elements
	144	+ * contained in the packet.
	145	+ * If set to 0, each OBU element MUST be preceded by a length field.
	146	+ * If not set to 0 (i.e., W = 1, 2 or 3) the last OBU element MUST NOT be
	147	+ * preceded by a length field (it's derived from RTP packet size minus other
	148	+ * known lengths). */
	149	+ num_obus = (aggr_hdr >> AV1S_AGGR_HDR_NUM_OBUS) & AV1M_AGGR_HDR_NUM_OBUS;
	150	+
	151	+ /* N: MUST be set to 1 if the packet is the first packet of a coded video
	152	+ * sequence, and MUST be set to 0 otherwise.*/
	153	+ is_first_pkt = (aggr_hdr >> AV1B_AGGR_HDR_FIRST_PKT) & 1;
	154	+
	155	+ if (is_frag_cont) {
	156	+ if (data->drop_fragment) {
	157	+ return AVERROR_INVALIDDATA;
	158	+ }
	159	+ if (is_first_pkt) {
	160	+ av_log(ctx, AV_LOG_ERROR, "Illegal aggregation header in first AV1 RTP packet\n");
	161	+ return AVERROR_INVALIDDATA;
	162	+ }
	163	+ if (seq_diff) {
	164	+ av_log(ctx, AV_LOG_WARNING, "AV1 RTP frag packet sequence mismatch (%d != %d), dropping temporal unit\n",
	165	+ seq, expected_seq);
	166	+ goto drop_fragment;
	167	+ }
	168	+ if (!pkt->size \|\| !data->frag_obu_size) {
	169	+ av_log(ctx, AV_LOG_WARNING, "Unexpected fragment continuation in AV1 RTP packet\n");
	170	+ goto drop_fragment; // avoid repeated output for the same fragment
	171	+ }
	172	+ } else {
	173	+ if (!is_first_pkt && !data->keyframe_seen) {
	174	+ if (!data->wait_for_keyframe) {
	175	+ data->wait_for_keyframe = 1;
	176	+ av_log(ctx, AV_LOG_WARNING, "AV1 RTP packet before keyframe, dropping and waiting for next keyframe\n");
	177	+ }
	178	+ goto drop_fragment;
	179	+ }
	180	+ if (seq_diff && !is_first_pkt) {
	181	+ av_log(ctx, AV_LOG_WARNING, "AV1 RTP unfrag packet sequence mismatch (%d != %d), dropping temporal unit\n",
	182	+ seq, expected_seq);
	183	+ goto drop_fragment;
	184	+ }
	185	+ data->drop_fragment = 0;
	186	+ if (!data->needs_td && ((data->timestamp != *timestamp) \|\| is_first_pkt)) {
	187	+ av_log(ctx, AV_LOG_TRACE, "Timestamp changed to %u (or first pkt %d), forcing TD\n", *timestamp, is_first_pkt);
	188	+ data->needs_td = 1;
	189	+ data->frag_obu_size = 0; // new temporal unit might have been caused by dropped packets
	190	+ }
	191	+ if (data->frag_obu_size) {
	192	+ data->frag_obu_size = 0; // make sure we recover
	193	+ av_log(ctx, AV_LOG_ERROR, "Missing fragment continuation in AV1 RTP packet\n");
	194	+ return AVERROR_INVALIDDATA;
	195	+ }
	196	+ // update the timestamp in the frame packet with the one from the RTP packet
	197	+ data->timestamp = *timestamp;
	198	+ }
	199	+ pktpos = pkt->size;
	200	+
	201	+#ifdef RTPDEC_AV1_VERBOSE_TRACE
	202	+ av_log(ctx, AV_LOG_TRACE, "Input buffer size %d, aggr head 0x%02x fc %d, lf %d, no %d, fp %d\n",
	203	+ len, aggr_hdr, is_frag_cont, is_last_fragmented, num_obus, is_first_pkt);
	204	+#endif
	205	+
	206	+ if (is_first_pkt) {
	207	+ pkt->flags \|= AV_PKT_FLAG_KEY;
	208	+ data->keyframe_seen = 1;
	209	+ data->wait_for_keyframe = 0;
	210	+ }
	211	+
	212	+ // loop over OBU elements
	213	+ while (rem_pkt_size) {
	214	+ uint32_t obu_size;
	215	+ int num_lebs;
	216	+ int needs_size_field;
	217	+ int output_size;
	218	+ unsigned int obu_payload_size;
	219	+ uint8_t obu_hdr;
	220	+
	221	+ obu_size = rem_pkt_size;
	222	+ if (!num_obus \|\| obu_cnt < num_obus) {
	223	+ // read out explicit OBU element size (which almost corresponds to the original OBU size)
	224	+ num_lebs = parse_leb(ctx, buf_ptr, rem_pkt_size, &obu_size);
	225	+ if (!num_lebs) {
	226	+ return AVERROR_INVALIDDATA;
	227	+ }
	228	+ rem_pkt_size -= num_lebs;
	229	+ buf_ptr += num_lebs;
	230	+ }
	231	+ // read first byte (which is the header byte only for non-fragmented elements)
	232	+ obu_hdr = *buf_ptr;
	233	+ if (obu_size > rem_pkt_size) {
	234	+ av_log(ctx, AV_LOG_ERROR, "AV1 OBU size %u larger than remaining pkt size %d\n", obu_size, rem_pkt_size);
	235	+ return AVERROR_INVALIDDATA;
	236	+ }
	237	+
	238	+ if (!obu_size) {
	239	+ av_log(ctx, AV_LOG_ERROR, "Unreasonable AV1 OBU size %u\n", obu_size);
	240	+ return AVERROR_INVALIDDATA;
	241	+ }
	242	+
	243	+ if (!is_frag_cont) {
	244	+ uint8_t obu_type = (obu_hdr >> AV1S_OBU_TYPE) & AV1M_OBU_TYPE;
	245	+ if (obu_hdr & AV1F_OBU_FORBIDDEN) {
	246	+ av_log(ctx, AV_LOG_ERROR, "Forbidden bit set in AV1 OBU header (0x%02x)\n", obu_hdr);
	247	+ return AVERROR_INVALIDDATA;
	248	+ }
	249	+ // ignore and remove OBUs according to spec
	250	+ if ((obu_type == AV1_OBU_TEMPORAL_DELIMITER) \|\|
	251	+ (obu_type == AV1_OBU_TILE_LIST)) {
	252	+ pktpos += obu_size;
	253	+ rem_pkt_size -= obu_size;
	254	+ // TODO: This probably breaks if the OBU_TILE_LIST is fragmented
	255	+ // into the next RTP packet, so at least check and fail here
	256	+ if (rem_pkt_size == 0 && is_last_fragmented) {
	257	+ avpriv_report_missing_feature(ctx, "AV1 OBU_TILE_LIST (should not be there!) to be ignored but is fragmented\n");
	258	+ return AVERROR_PATCHWELCOME;
	259	+ }
	260	+ obu_cnt++;
	261	+ continue;
	262	+ }
	263	+ }
	264	+
	265	+ // If we need to add a size field, out size will be different
	266	+ output_size = obu_size;
	267	+ // Spec says the OBUs should have their size fields removed,
	268	+ // but this is not mandatory
	269	+ if (is_frag_cont \|\| (obu_hdr & AV1F_OBU_HAS_SIZE_FIELD)) {
	270	+ needs_size_field = 0;
	271	+ } else {
	272	+ needs_size_field = 1;
	273	+ // (re)calculate number of LEB bytes needed (if it was implicit, there were no LEBs)
	274	+ output_size += calc_leb_size(obu_size - (1 + ((obu_hdr & AV1F_OBU_EXTENSION_FLAG) ? 1 : 0)));
	275	+ }
	276	+
	277	+ if (!is_frag_cont && (obu_cnt == 1)) {
	278	+ if (data->needs_td) {
	279	+ output_size += 2; // for Temporal Delimiter (TD)
	280	+ }
	281	+ if (pkt->data) {
	282	+ if ((result = av_grow_packet(pkt, output_size)) < 0)
	283	+ return result;
	284	+ } else {
	285	+ if ((result = av_new_packet(pkt, output_size) < 0))
	286	+ return result;
	287	+ }
	288	+
	289	+ if (data->needs_td) {
	290	+ // restore TD
	291	+ pkt->data[pktpos++] = 0x12;
	292	+ pkt->data[pktpos++] = 0x00;
	293	+ }
	294	+ data->needs_td = 0;
	295	+ } else {
	296	+ if ((result = av_grow_packet(pkt, output_size)) < 0)
	297	+ return result;
	298	+ }
	299	+
	300	+ obu_payload_size = obu_size;
	301	+ // do we need to restore the OBU size field?
	302	+ if (needs_size_field) {
	303	+ // set obu_has_size_field in header byte
	304	+ pkt->data[pktpos++] = *buf_ptr++ \| AV1F_OBU_HAS_SIZE_FIELD;
	305	+ data->frag_header_size = 1;
	306	+ obu_payload_size--;
	307	+
	308	+ // copy extension byte, if available
	309	+ if (obu_hdr & AV1F_OBU_EXTENSION_FLAG) {
	310	+ /* TODO we cannot handle the edge case where last element is a
	311	+ * fragment of exactly one byte AND the header has the extension
	312	+ * flag set. Note that it would be more efficient to not send a
	313	+ * fragment of one byte and instead drop the size field of the
	314	+ * prior element */
	315	+ if (!obu_payload_size) {
	316	+ av_log(ctx, AV_LOG_ERROR, "AV1 OBU too short for extension byte (0x%02x)\n",
	317	+ obu_hdr);
	318	+ return AVERROR_INVALIDDATA;
	319	+ }
	320	+ pkt->data[pktpos++] = *buf_ptr++;
	321	+ data->frag_header_size = 2;
	322	+ obu_payload_size--;
	323	+ }
	324	+
	325	+ // remember start position of LEB for possibly fragmented packet to
	326	+ // fixup OBU size later
	327	+ data->frag_pkt_leb_pos = pktpos;
	328	+ // write intermediate OBU size field
	329	+ num_lebs = write_leb(pkt->data + pktpos, obu_payload_size);
	330	+ data->frag_lebs_res = num_lebs;
	331	+ pktpos += num_lebs;
	332	+ }
	333	+ // copy verbatim or without above header size patch
	334	+ memcpy(pkt->data + pktpos, buf_ptr, obu_payload_size);
	335	+ pktpos += obu_payload_size;
	336	+ buf_ptr += obu_payload_size;
	337	+ rem_pkt_size -= obu_size;
	338	+
	339	+ // if we were handling a fragmented packet and this was the last
	340	+ // fragment, correct OBU size field
	341	+ if (data->frag_obu_size && (rem_pkt_size \|\| !is_last_fragmented)) {
	342	+ uint32_t final_obu_size = data->frag_obu_size + obu_size - data->frag_header_size;
	343	+ uint8_t *lebptr = pkt->data + data->frag_pkt_leb_pos;
	344	+ num_lebs = calc_leb_size(final_obu_size);
	345	+
	346	+ // check if we had allocated enough LEB bytes in header,
	347	+ // otherwise make some extra space
	348	+ if (num_lebs > data->frag_lebs_res) {
	349	+ int extra_bytes = num_lebs - data->frag_lebs_res;
	350	+ if ((result = av_grow_packet(pkt, extra_bytes)) < 0)
	351	+ return result;
	352	+ // update pointer in case buffer address changed
	353	+ lebptr = pkt->data + data->frag_pkt_leb_pos;
	354	+ // move existing data for OBU back a bit
	355	+ memmove(lebptr + extra_bytes, lebptr,
	356	+ pkt->size - extra_bytes - data->frag_pkt_leb_pos);
	357	+ // move pktpos further down for following OBUs in same packet.
	358	+ pktpos += extra_bytes;
	359	+ }
	360	+
	361	+ // update OBU size field
	362	+ write_leb(lebptr, final_obu_size);
	363	+
	364	+ data->frag_obu_size = 0; // signal end of fragment
	365	+ } else if (is_last_fragmented && !rem_pkt_size) {
	366	+ // add to total OBU size, so we can fix that in OBU header
	367	+ // (but only if the OBU size was missing!)
	368	+ if (needs_size_field \|\| data->frag_obu_size) {
	369	+ data->frag_obu_size += obu_size;
	370	+ }
	371	+ // fragment not yet finished!
	372	+ result = -1;
	373	+ }
	374	+ is_frag_cont = 0;
	375	+
	376	+ if (!rem_pkt_size && !num_obus && (num_obus != obu_cnt)) {
	377	+ av_log(ctx, AV_LOG_WARNING, "AV1 aggregation header indicated %u OBU elements, was %u\n",
	378	+ num_obus, obu_cnt);
	379	+ }
	380	+ obu_cnt++;
	381	+ }
	382	+
	383	+ if (flags & RTP_FLAG_MARKER) {
	384	+ av_log(ctx, AV_LOG_TRACE, "TD on next packet due to marker\n");
	385	+ data->needs_td = 1;
	386	+ } else {
	387	+ // fragment may be complete, but temporal unit is not yet finished
	388	+ result = -1;
	389	+ }
	390	+
	391	+ if (!is_last_fragmented) {
	392	+ data->frag_obu_size = 0;
	393	+ data->frag_pkt_leb_pos = 0;
	394	+ }
	395	+
	396	+#ifdef RTPDEC_AV1_VERBOSE_TRACE
	397	+ if (!result) {
	398	+ av_log(ctx, AV_LOG_TRACE, "AV1 out pkt-size: %d\n", pkt->size);
	399	+ av_hex_dump_log(ctx, AV_LOG_TRACE, pkt->data, FFMIN(pkt->size, 64));
	400	+ av_log(ctx, AV_LOG_TRACE, "... end at offset %x:\n", FFMAX(pkt->size - 64, 0));
	401	+ av_hex_dump_log(ctx, AV_LOG_TRACE, pkt->data + FFMAX(pkt->size - 64, 0), FFMIN(pkt->size, 64));
	402	+ }
	403	+#endif
	404	+ pkt->stream_index = st->index;
	405	+
	406	+ return result;
	407	+
	408	+drop_fragment:
	409	+ data->keyframe_seen = 0;
	410	+ data->drop_fragment = 1;
	411	+ data->frag_obu_size = 0;
	412	+ data->needs_td = 1;
	413	+ if (pkt->size) {
	414	+ av_log(ctx, AV_LOG_TRACE, "Dumping current AV1 frame packet\n");
	415	+ // we can't seem to deallocate the fragmented packet, but we can shrink it to 0
	416	+ av_shrink_packet(pkt, 0);
	417	+ }
	418	+ return AVERROR_INVALIDDATA;
	419	+}
	420	+
	421	+static void av1_close_context(PayloadContext *data) {
	422	+}
	423	+
	424	+static int av1_need_keyframe(PayloadContext *data)
	425	+{
	426	+ return !data->keyframe_seen;
	427	+}
	428	+
	429	+static int parse_av1_sdp_line(AVFormatContext *s, int st_index,
	430	+ PayloadContext av1_data, const char line) {
	431	+ AVStream * stream;
	432	+ const char *p = line;
	433	+ int result = 0;
	434	+
	435	+ if (st_index < 0)
	436	+ return 0;
	437	+
	438	+ stream = s->streams[st_index];
	439	+
	440	+ /* Optional parameters are profile, level-idx, and tier.
	441	+ * See Section 7.2.1 of https://aomediacodec.github.io/av1-rtp-spec/ */
	442	+ if (av_strstart(p, "fmtp:", &p)) {
	443	+ result = ff_parse_fmtp(s, stream, av1_data, p, sdp_parse_fmtp_config_av1);
	444	+ av_log(s, AV_LOG_DEBUG, "RTP AV1 Profile: %u, Level: %u, Tier: %u\n",
	445	+ av1_data->profile, av1_data->level_idx, av1_data->tier);
	446	+ }
	447	+
	448	+ return result;
	449	+}
	450	+
	451	+const RTPDynamicProtocolHandler ff_av1_dynamic_handler = {
	452	+ .enc_name = "AV1",
	453	+ .codec_type = AVMEDIA_TYPE_VIDEO,
	454	+ .codec_id = AV_CODEC_ID_AV1,
	455	+ .need_parsing = AVSTREAM_PARSE_FULL,
	456	+ .priv_data_size = sizeof(PayloadContext),
	457	+ .parse_sdp_a_line = parse_av1_sdp_line,
	458	+ .close = av1_close_context,
	459	+ .parse_packet = av1_handle_packet,
	460	+ .need_keyframe = av1_need_keyframe,
	461	+};

libavformat/rtpdec_formats.h+1−0

@@ -50,6 +50,7 @@ void ff_h264_parse_framesize(AVCodecParameters par, const char p); · 49 unmodified lines
50	50	extern const RTPDynamicProtocolHandler ff_ac3_dynamic_handler;
51	51	extern const RTPDynamicProtocolHandler ff_amr_nb_dynamic_handler;
52	52	extern const RTPDynamicProtocolHandler ff_amr_wb_dynamic_handler;
	53	+extern const RTPDynamicProtocolHandler ff_av1_dynamic_handler;
53	54	extern const RTPDynamicProtocolHandler ff_dv_dynamic_handler;
54	55	extern const RTPDynamicProtocolHandler ff_g726_16_dynamic_handler;
55	56	extern const RTPDynamicProtocolHandler ff_g726_24_dynamic_handler;

libavformat/rtpenc.c+14−0

@@ -79,6 +79,7 @@ static int is_supported(enum AVCodecID id) · 78 unmodified lines
79	79	case AV_CODEC_ID_THEORA:
80	80	case AV_CODEC_ID_VP8:
81	81	case AV_CODEC_ID_VP9:
	82	+ case AV_CODEC_ID_AV1:
82	83	case AV_CODEC_ID_ADPCM_G722:
83	84	case AV_CODEC_ID_ADPCM_G726:
84	85	case AV_CODEC_ID_ADPCM_G726LE:
@@ -228,6 +229,16 @@ static int rtp_write_header(AVFormatContext *s1) · 143 unmodified lines
228	229	goto fail;
229	230	}
230	231	break;
	232	+ case AV_CODEC_ID_AV1:
	233	+ if (s1->strict_std_compliance > FF_COMPLIANCE_EXPERIMENTAL) {
	234	+ av_log(s, AV_LOG_ERROR,
	235	+ "Packetizing AV1 is experimental and its specification is "
	236	+ "still in draft state. "
	237	+ "Please set -strict experimental in order to enable it.\n");
	238	+ ret = AVERROR_EXPERIMENTAL;
	239	+ goto fail;
	240	+ }
	241	+ break;
231	242	case AV_CODEC_ID_VORBIS:
232	243	case AV_CODEC_ID_THEORA:
233	244	s->max_frames_per_packet = 15;
@@ -579,6 +590,9 @@ static int rtp_write_packet(AVFormatContext s1, AVPacket pkt) · 345 unmodified lines
579	590	case AV_CODEC_ID_AMR_WB:
580	591	ff_rtp_send_amr(s1, pkt->data, size);
581	592	break;
	593	+ case AV_CODEC_ID_AV1:
	594	+ ff_rtp_send_av1(s1, pkt->data, size, (pkt->flags & AV_PKT_FLAG_KEY) ? 1 : 0);
	595	+ break;
582	596	case AV_CODEC_ID_MPEG2TS:
583	597	rtp_send_mpegts_raw(s1, pkt->data, size);
584	598	break;

libavformat/rtpenc.h+1−0

@@ -94,6 +94,7 @@ void ff_rtp_send_xiph(AVFormatContext s1, const uint8_t buff, int size); · 93 unmodified lines
94	94	void ff_rtp_send_vc2hq(AVFormatContext s1, const uint8_t buf, int size, int interlaced);
95	95	void ff_rtp_send_vp8(AVFormatContext s1, const uint8_t buff, int size);
96	96	void ff_rtp_send_vp9(AVFormatContext s1, const uint8_t buff, int size);
	97	+void ff_rtp_send_av1(AVFormatContext s1, const uint8_t buf1, int size, int is_keyframe);
97	98	void ff_rtp_send_jpeg(AVFormatContext s1, const uint8_t buff, int size);
98	99	void ff_rtp_send_raw_rfc4175(AVFormatContext s1, const uint8_t buf, int size, int interlaced, int field);
99	100

libavformat/rtpenc_av1.c+357−0

@@ -0,0 +1,357 @@
	1	+/*
	2	+ * Packetization for RTP Payload Format For AV1 (v1.0)
	3	+ * https://aomediacodec.github.io/av1-rtp-spec/
	4	+ * Copyright (c) 2024 Axis Communications
	5	+ *
	6	+ * This file is part of FFmpeg.
	7	+ *
	8	+ * FFmpeg is free software; you can redistribute it and/or
	9	+ * modify it under the terms of the GNU Lesser General Public
	10	+ * License as published by the Free Software Foundation; either
	11	+ * version 2.1 of the License, or (at your option) any later version.
	12	+ *
	13	+ * FFmpeg is distributed in the hope that it will be useful,
	14	+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
	15	+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	16	+ * Lesser General Public License for more details.
	17	+ *
	18	+ * You should have received a copy of the GNU Lesser General Public
	19	+ * License along with FFmpeg; if not, write to the Free Software
	20	+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
	21	+ */
	22	+
	23	+/**
	24	+ * @file
	25	+ * @brief AV1 / RTP packetization code (RTP Payload Format For AV1 (v1.0))
	26	+ * @author Chris Hodges <chris.hodges@axis.com>
	27	+ * @note This will remove TDs and OBU size fields
	28	+ */
	29	+
	30	+#include "avformat.h"
	31	+#include "rtpenc.h"
	32	+#include "libavcodec/av1.h"
	33	+#include "rtp_av1.h"
	34	+
	35	+// enable tracing of packet data
	36	+//#define RTPENC_AV1_VERBOSE_TRACE
	37	+
	38	+// enable searching for sequence header as workaround for AV1 encoders
	39	+// that do not set AV_PKT_FLAG_KEY correctly
	40	+#define RTPENC_AV1_SEARCH_SEQ_HEADER 1
	41	+
	42	+void ff_rtp_send_av1(AVFormatContext ctx, const uint8_t frame_buf, int frame_size, int is_keyframe) {
	43	+ uint8_t aggr_hdr = 0;
	44	+ int last_packet_of_frame = 0;
	45	+ RTPMuxContext *rtp_ctx = ctx->priv_data;
	46	+ const uint8_t *obu_ptr = frame_buf;
	47	+ int start_new_packet = 0;
	48	+ unsigned int num_obus = 0;
	49	+ unsigned int rem_pkt_size = rtp_ctx->max_payload_size - 1;
	50	+ uint8_t *pkt_ptr = NULL;
	51	+
	52	+ const uint8_t *curr_obu_ptr = NULL;
	53	+ uint32_t curr_elem_size = 0;
	54	+ int curr_obu_hdr = -1;
	55	+ int curr_obu_ext = -1;
	56	+ const uint8_t *last_obu_ptr = NULL;
	57	+ uint32_t last_elem_size = 0;
	58	+ int last_obu_hdr = -1;
	59	+ int last_obu_ext = -1;
	60	+
	61	+ rtp_ctx->timestamp = rtp_ctx->cur_timestamp;
	62	+
	63	+ /* The payload structure is supposed to be straight-forward, but there are a
	64	+ * couple of edge cases to be tackled and make things very complex.
	65	+ * These are mainly due to:
	66	+ * - the OBU element size being optional for the last element, but MANDATORY
	67	+ * if there are more than 3 elements
	68	+ * - the size field of the element is made up of a variable number of
	69	+ * LEB bytes
	70	+ * - the latter in combination with the desire to fill the max packet size
	71	+ * could cause a catch22
	72	+ * - if there's less than 2 bytes remaining (depending on the required LEB),
	73	+ * one would not have space for the payload of an element and must instead
	74	+ * start the next packet
	75	+ * - if there's less than 3 bytes remaining, the header byte plus the
	76	+ * optional extension byte will not fit in the fragment making the
	77	+ * handling even more complicated
	78	+ * - as some OBU types are supposed to be filtered out, it is hard to decide
	79	+ * via the remaining length whether the outputted OBU element will
	80	+ * actually be the last one
	81	+ *
	82	+ * There are two major ways to tackle that: Pre-parsing of all OBUs within a
	83	+ * frame (adds memory complexity) or lazy copying of the prior element.
	84	+ * Here, the latter is implemented.
	85	+ */
	86	+
	87	+ if (is_keyframe) {
	88	+#if RTPENC_AV1_SEARCH_SEQ_HEADER
	89	+ /* search for OBU_SEQUENCE_HEADER to get a better indication that
	90	+ * the frame was marked as keyframe is really a KEY_FRAME and not
	91	+ * a INTRA_ONLY frame. This might be unnecessary if the AV1 parser/
	92	+ * encoder always correctly specifies AV_PKT_FLAG_KEY.
	93	+ *
	94	+ * Note: Spec does NOT prohibit resending bit-identical
	95	+ * OBU_SEQUENCE_HEADER for ANY kind of frame, though!
	96	+ */
	97	+ int rem_size = frame_size;
	98	+ const uint8_t *buf_ptr = frame_buf;
	99	+ while (rem_size > 0) {
	100	+ uint32_t obu_size;
	101	+ uint8_t obu_hdr = *buf_ptr++;
	102	+ uint8_t obu_type = (obu_hdr >> AV1S_OBU_TYPE) & AV1M_OBU_TYPE;
	103	+ int num_lebs;
	104	+
	105	+ if (obu_type == AV1_OBU_SEQUENCE_HEADER) {
	106	+ av_log(ctx, AV_LOG_DEBUG, "Marking FIRST packet\n");
	107	+ aggr_hdr \|= AV1F_AGGR_HDR_FIRST_PKT;
	108	+ break;
	109	+ }
	110	+ if (!(obu_hdr & AV1F_OBU_HAS_SIZE_FIELD)) {
	111	+ break;
	112	+ }
	113	+ rem_size--;
	114	+ // read out explicit OBU size
	115	+ num_lebs = parse_leb(ctx, buf_ptr, rem_size, &obu_size);
	116	+ if (!num_lebs) {
	117	+ break;
	118	+ }
	119	+ buf_ptr += num_lebs + obu_size;
	120	+ rem_size -= num_lebs + obu_size;
	121	+ }
	122	+#else // RTPENC_AV1_SEARCH_SEQ_HEADER
	123	+ av_log(ctx, AV_LOG_DEBUG, "Marking FIRST packet\n");
	124	+ aggr_hdr \|= AV1F_AGGR_HDR_FIRST_PKT;
	125	+#endif // RTPENC_AV1_SEARCH_SEQ_HEADER
	126	+ }
	127	+ rem_pkt_size = rtp_ctx->max_payload_size - 1;
	128	+ pkt_ptr = rtp_ctx->buf + 1;
	129	+
	130	+#ifdef RTPENC_AV1_VERBOSE_TRACE
	131	+ av_log(ctx, AV_LOG_TRACE, "AV1 Frame %d in (%x), size=%d:\n",
	132	+ rtp_ctx->seq, rtp_ctx->flags, frame_size);
	133	+ av_hex_dump_log(ctx, AV_LOG_TRACE, frame_buf, FFMIN(frame_size, 128));
	134	+#endif
	135	+
	136	+ while (frame_size) {
	137	+ uint32_t obu_size;
	138	+ int num_lebs = 0;
	139	+ int ext_byte = -1;
	140	+
	141	+ uint8_t obu_hdr = *obu_ptr++;
	142	+ uint8_t obu_type = (obu_hdr >> AV1S_OBU_TYPE) & AV1M_OBU_TYPE;
	143	+ frame_size--;
	144	+
	145	+ if (obu_hdr & AV1F_OBU_FORBIDDEN) {
	146	+ av_log(ctx, AV_LOG_ERROR, "Forbidden bit set in AV1 OBU header (0x%02x)\n", obu_hdr);
	147	+ return;
	148	+ }
	149	+
	150	+ if (obu_hdr & AV1F_OBU_EXTENSION_FLAG) {
	151	+ if (!frame_size) {
	152	+ av_log(ctx, AV_LOG_ERROR, "Out of data for AV1 OBU header extension byte\n");
	153	+ return;
	154	+ }
	155	+ ext_byte = *obu_ptr++;
	156	+ frame_size--;
	157	+ }
	158	+
	159	+ if (obu_hdr & AV1F_OBU_HAS_SIZE_FIELD) {
	160	+ obu_hdr &= ~AV1F_OBU_HAS_SIZE_FIELD; // remove size field
	161	+ // read out explicit OBU size
	162	+ num_lebs = parse_leb(ctx, obu_ptr, frame_size, &obu_size);
	163	+ if (!num_lebs) {
	164	+ return;
	165	+ }
	166	+ obu_ptr += num_lebs;
	167	+ frame_size -= num_lebs;
	168	+ } else {
	169	+ av_log(ctx, AV_LOG_ERROR, "Cannot handle AV1 OBUs without size fields\n");
	170	+ return;
	171	+ }
	172	+
	173	+ if ((long) obu_size > frame_size) {
	174	+ av_log(ctx, AV_LOG_ERROR, "AV1 OBU size %d larger than remaining frame size %d\n", obu_size, frame_size);
	175	+ return;
	176	+ }
	177	+
	178	+ if (obu_size > 0xfffffffd) {
	179	+ av_log(ctx, AV_LOG_ERROR, "AV1 OBU size 0x%x might overflow (attack?)\n", obu_size);
	180	+ return;
	181	+ }
	182	+
	183	+ frame_size -= obu_size;
	184	+
	185	+ if ((obu_type == AV1_OBU_TEMPORAL_DELIMITER) \|\|
	186	+ (obu_type == AV1_OBU_TILE_LIST) \|\|
	187	+ (obu_type == AV1_OBU_PADDING)) {
	188	+ // ignore and remove according to spec (note that OBU_PADDING is not
	189	+ // mentioned in spec, but it does not make sense to transmit it).
	190	+ obu_ptr += obu_size;
	191	+ // additional handling if the ignored OBU was the last one
	192	+ if (!frame_size) {
	193	+ // we're done, flush the last packet, set RTP marker bit
	194	+ last_packet_of_frame = 1;
	195	+ goto flush_last_packet;
	196	+ }
	197	+ continue;
	198	+ }
	199	+
	200	+ /* if the last OBU had a temporal or spatial ID, they need to match to
	201	+ * current; otherwise start new packet */
	202	+ if ((last_obu_ext >= 0) && (curr_obu_ext != last_obu_ext)) {
	203	+ start_new_packet = 1;
	204	+ }
	205	+
	206	+flush_last_packet:
	207	+ last_obu_ptr = curr_obu_ptr;
	208	+ last_elem_size = curr_elem_size;
	209	+ last_obu_hdr = curr_obu_hdr;
	210	+ last_obu_ext = curr_obu_ext;
	211	+
	212	+ curr_obu_ptr = obu_ptr; // behind header
	213	+ curr_elem_size = obu_size + 1 + ((ext_byte >= 0) ? 1 : 0);
	214	+ curr_obu_hdr = obu_hdr;
	215	+ curr_obu_ext = ext_byte;
	216	+
	217	+ obu_ptr += obu_size;
	218	+
	219	+ if (last_obu_ptr) {
	220	+ unsigned int first_elem_with_size = last_elem_size + calc_leb_size(last_elem_size);
	221	+ // check if last packet fits completely and has reasonable space for
	222	+ // at least a fragment of the next
	223	+ if (!last_packet_of_frame && (first_elem_with_size + 10 < rem_pkt_size)) {
	224	+ num_lebs = write_leb(pkt_ptr, last_elem_size);
	225	+ pkt_ptr += num_lebs;
	226	+ rem_pkt_size -= num_lebs;
	227	+ } else {
	228	+ if ((num_obus >= 3) && (last_packet_of_frame \|\| (first_elem_with_size <= rem_pkt_size))) {
	229	+ // last fits with forced size, but nothing else
	230	+ num_lebs = write_leb(pkt_ptr, last_elem_size);
	231	+ pkt_ptr += num_lebs;
	232	+ rem_pkt_size -= num_lebs;
	233	+ }
	234	+ // force new packet
	235	+ start_new_packet = 1;
	236	+ }
	237	+
	238	+ // write header and optional extension byte (if not a continued fragment)
	239	+ if (last_obu_hdr >= 0) {
	240	+ *pkt_ptr++ = last_obu_hdr;
	241	+ last_elem_size--;
	242	+ rem_pkt_size--;
	243	+ if (last_obu_ext >= 0) {
	244	+ *pkt_ptr++ = last_obu_ext;
	245	+ last_elem_size--;
	246	+ rem_pkt_size--;
	247	+ }
	248	+ }
	249	+ // copy payload
	250	+ memcpy(pkt_ptr, last_obu_ptr, last_elem_size);
	251	+ pkt_ptr += last_elem_size;
	252	+ rem_pkt_size -= last_elem_size;
	253	+ num_obus++;
	254	+ }
	255	+
	256	+ if (start_new_packet \|\| last_packet_of_frame) {
	257	+ if (num_obus < 4) {
	258	+ aggr_hdr \|= num_obus << AV1S_AGGR_HDR_NUM_OBUS;
	259	+ }
	260	+ rtp_ctx->buf[0] = aggr_hdr;
	261	+
	262	+#ifdef RTPENC_AV1_VERBOSE_TRACE
	263	+ av_log(ctx, AV_LOG_TRACE, "Sending NON-FRAG packet no %d, %ld/%d, %d OBUs (marker=%d)\n",
	264	+ ((RTPMuxContext *) ctx->priv_data)->seq,
	265	+ pkt_ptr - rtp_ctx->buf, rtp_ctx->max_payload_size, num_obus, last_packet_of_frame);
	266	+ av_hex_dump_log(ctx, AV_LOG_TRACE, rtp_ctx->buf, FFMIN(pkt_ptr - rtp_ctx->buf, 64));
	267	+ av_log(ctx, AV_LOG_TRACE, "... end at offset %lx:\n", FFMAX((pkt_ptr - rtp_ctx->buf) - 64, 0));
	268	+ av_hex_dump_log(ctx, AV_LOG_TRACE, rtp_ctx->buf + FFMAX((pkt_ptr - rtp_ctx->buf) - 64, 0), FFMIN(pkt_ptr - rtp_ctx->buf, 64));
	269	+#endif
	270	+
	271	+ ff_rtp_send_data(ctx, rtp_ctx->buf, pkt_ptr - rtp_ctx->buf, last_packet_of_frame);
	272	+
	273	+ rem_pkt_size = rtp_ctx->max_payload_size - 1;
	274	+ pkt_ptr = rtp_ctx->buf + 1;
	275	+ aggr_hdr = 0;
	276	+ num_obus = 0;
	277	+ }
	278	+
	279	+ if (last_packet_of_frame) {
	280	+ break;
	281	+ }
	282	+
	283	+ // check if element needs to be fragmented, otherwise we will deal with
	284	+ // it in the next iteration
	285	+ if ((curr_elem_size > rem_pkt_size) \|\|
	286	+ ((num_obus >= 3) && (curr_elem_size + calc_leb_size(curr_elem_size)) > rem_pkt_size)) {
	287	+ uint32_t frag_size = rem_pkt_size;
	288	+
	289	+ // if there are going more than 3 OBU elements, we are obliged to
	290	+ // have the length field for the last
	291	+ if (num_obus >= 3) {
	292	+ // that's an upper limit of LEBs
	293	+ num_lebs = calc_leb_size(rem_pkt_size - 1);
	294	+ frag_size -= num_lebs;
	295	+
	296	+ // write a fixed number of LEBs, in case the frag_size could
	297	+ // now be specified with one less byte
	298	+ write_leb_n(pkt_ptr, frag_size, num_lebs);
	299	+ pkt_ptr += num_lebs;
	300	+ rem_pkt_size -= num_lebs;
	301	+ }
	302	+
	303	+ // write header and optional extension byte
	304	+ *pkt_ptr++ = curr_obu_hdr;
	305	+ curr_elem_size--;
	306	+ rem_pkt_size--;
	307	+ if (curr_obu_ext >= 0) {
	308	+ *pkt_ptr++ = curr_obu_ext;
	309	+ curr_elem_size--;
	310	+ rem_pkt_size--;
	311	+ }
	312	+
	313	+ // disable header writing for final fragment
	314	+ curr_obu_hdr = -1;
	315	+ curr_obu_ext = -1;
	316	+
	317	+ // send more full packet sized fragments
	318	+ do {
	319	+ // copy payload
	320	+ memcpy(pkt_ptr, curr_obu_ptr, rem_pkt_size);
	321	+ pkt_ptr += rem_pkt_size;
	322	+ curr_obu_ptr += rem_pkt_size;
	323	+ curr_elem_size -= rem_pkt_size;
	324	+ num_obus++;
	325	+
	326	+ aggr_hdr \|= AV1F_AGGR_HDR_LAST_FRAG;
	327	+ if (num_obus < 4) {
	328	+ aggr_hdr \|= num_obus << AV1S_AGGR_HDR_NUM_OBUS;
	329	+ }
	330	+ rtp_ctx->buf[0] = aggr_hdr;
	331	+
	332	+#ifdef RTPENC_AV1_VERBOSE_TRACE
	333	+ av_log(ctx, AV_LOG_DEBUG, "Sending FRAG packet no %d, %ld/%d, %d OBUs\n",
	334	+ ((RTPMuxContext *) ctx->priv_data)->seq,
	335	+ pkt_ptr - rtp_ctx->buf, rtp_ctx->max_payload_size, num_obus);
	336	+ av_hex_dump_log(ctx, AV_LOG_TRACE, rtp_ctx->buf, FFMIN(pkt_ptr - rtp_ctx->buf, 64));
	337	+ av_log(ctx, AV_LOG_TRACE, "... end at offset %lx:\n", FFMAX((pkt_ptr - rtp_ctx->buf) - 64, 0));
	338	+ av_hex_dump_log(ctx, AV_LOG_TRACE, rtp_ctx->buf + FFMAX((pkt_ptr - rtp_ctx->buf) - 64, 0), FFMIN(pkt_ptr - rtp_ctx->buf, 64));
	339	+#endif
	340	+
	341	+ ff_rtp_send_data(ctx, rtp_ctx->buf, pkt_ptr - rtp_ctx->buf, 0);
	342	+ rem_pkt_size = rtp_ctx->max_payload_size - 1;
	343	+ pkt_ptr = rtp_ctx->buf + 1;
	344	+
	345	+ aggr_hdr = AV1F_AGGR_HDR_FRAG_CONT;
	346	+ num_obus = 0;
	347	+ } while (curr_elem_size > rem_pkt_size);
	348	+ start_new_packet = 0;
	349	+ }
	350	+
	351	+ if (!frame_size) {
	352	+ // we're done, flush the last packet, set RTP marker bit
	353	+ last_packet_of_frame = 1;
	354	+ goto flush_last_packet;
	355	+ }
	356	+ }
	357	+}

libavformat/sdp.c+30−0

@@ -31,6 +31,7 @@ · 30 unmodified lines
31	31	#include "libavcodec/mpeg4audio.h"
32	32	#include "avformat.h"
33	33	#include "internal.h"
	34	+#include "av1.h"
34	35	#include "avc.h"
35	36	#include "hevc.h"
36	37	#include "nal.h"
@@ -155,6 +156,26 @@ static int sdp_get_address(char dest_addr, int size, int ttl, const char *url) · 118 unmodified lines
155	156	return port;
156	157	}
157	158
	159	+static int extradata2psets_av1(AVFormatContext s, const AVCodecParameters par,
	160	+ char **out)
	161	+{
	162	+ char *psets;
	163	+ AV1SequenceParameters seq;
	164	+
	165	+ if (ff_av1_parse_seq_header(&seq, par->extradata, par->extradata_size) < 0)
	166	+ return AVERROR_INVALIDDATA;
	167	+
	168	+ psets = av_mallocz(64);
	169	+ if (!psets) {
	170	+ av_log(s, AV_LOG_ERROR, "Cannot allocate memory for the parameter sets.\n");
	171	+ return AVERROR(ENOMEM);
	172	+ }
	173	+ av_strlcatf(psets, 64, "profile=%u;level-idx=%u;tier=%u",
	174	+ seq.profile, seq.level, seq.tier);
	175	+ *out = psets;
	176	+ return 0;
	177	+}
	178	+
158	179	#define MAX_PSET_SIZE 1024
159	180	static int extradata2psets(AVFormatContext s, const AVCodecParameters par,
160	181	char **out)
@@ -523,6 +544,15 @@ static int sdp_write_media_attributes(char buff, int size, const AVStream st, · 362 unmodified lines
523	544	int ret = 0;
524	545
525	546	switch (p->codec_id) {
	547	+ case AV_CODEC_ID_AV1:
	548	+ av_strlcatf(buff, size, "a=rtpmap:%d AV1/90000\r\n", payload_type);
	549	+ if (p->extradata_size) {
	550	+ ret = extradata2psets_av1(fmt, p, &config);
	551	+ if (ret < 0)
	552	+ return ret;
	553	+ av_strlcatf(buff, size, "a=fmtp:%d %s\r\n", payload_type, config);
	554	+ }
	555	+ break;
526	556	case AV_CODEC_ID_DIRAC:
527	557	av_strlcatf(buff, size, "a=rtpmap:%d VC2/90000\r\n", payload_type);
528	558	break;

avformat: add AV1 RTP depacketizer and packetizer

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code