History

avformat/hls demuxer: Add WebVTT subtitle support

This add support for WebVTT subtitles in HLS streams. Just like for separate audio streams, it supports all available WebVTT streams in all renditions. No new options are added, it just works and provides subtitles streams like any other demuxer. The code prevents downloading subtitle segments which are farther in the future than the main segments, to avoid loading hundreds of subtitle segments in advance.

softworkz committed 1 year ago

Commit fcc50c41 parent ec3f3451 file · +176 −37Browse files

Unified · Split

libavformat/hls.c+176−37
@@ -173,6 +173,7 @@ struct playlist { · 172 unmodified lines
173 * playlist, if any. */173 * playlist, if any. */
174 int n_init_sections;174 int n_init_sections;
175 struct segment **init_sections;175 struct segment **init_sections;
176 int is_subtitle; /* Indicates if it's a subtitle playlist */
176};177};
177 178
178/*179/*
@@ -515,13 +516,6 @@ static struct rendition *new_rendition(HLSContext *c, struct rendition_info *inf · 336 unmodified lines
515 return NULL;516 return NULL;
516 }517 }
517 518
518 /* TODO: handle subtitles (each segment has to parsed separately) */
519 if (c->ctx->strict_std_compliance > FF_COMPLIANCE_EXPERIMENTAL)
520 if (type == AVMEDIA_TYPE_SUBTITLE) {
521 av_log(c->ctx, AV_LOG_WARNING, "Can't support the subtitle(uri: %s)\n", info->uri);
522 return NULL;
523 }
524
525 rend = av_mallocz(sizeof(struct rendition));519 rend = av_mallocz(sizeof(struct rendition));
526 if (!rend)520 if (!rend)
527 return NULL;521 return NULL;
@@ -536,9 +530,14 @@ static struct rendition *new_rendition(HLSContext *c, struct rendition_info *inf · 8 unmodified lines
536 /* add the playlist if this is an external rendition */530 /* add the playlist if this is an external rendition */
537 if (info->uri[0]) {531 if (info->uri[0]) {
538 rend->playlist = new_playlist(c, info->uri, url_base);532 rend->playlist = new_playlist(c, info->uri, url_base);
539 if (rend->playlist)533 if (rend->playlist) {
534 if (type == AVMEDIA_TYPE_SUBTITLE) {
535 rend->playlist->is_subtitle = 1;
536 rend->playlist->is_id3_timestamped = 0;
537 }
540 dynarray_add(&rend->playlist->renditions,538 dynarray_add(&rend->playlist->renditions,
541 &rend->playlist->n_renditions, rend);539 &rend->playlist->n_renditions, rend);
540 }
542 }541 }
543 542
544 if (info->assoc_language[0]) {543 if (info->assoc_language[0]) {
@@ -761,6 +760,9 @@ static int test_segment(AVFormatContext *s, const AVInputFormat *in_fmt, struct · 216 unmodified lines
761 } else if (!strcmp(in_fmt->name, "mpegts")) {760 } else if (!strcmp(in_fmt->name, "mpegts")) {
762 matchF = av_match_ext( seg->url, "ts,m2t,m2ts,mts,mpg,m4s,mpeg,mpegts")761 matchF = av_match_ext( seg->url, "ts,m2t,m2ts,mts,mpg,m4s,mpeg,mpegts")
763 + 2*(ff_match_url_ext(seg->url, "ts,m2t,m2ts,mts,mpg,m4s,mpeg,mpegts") > 0);762 + 2*(ff_match_url_ext(seg->url, "ts,m2t,m2ts,mts,mpg,m4s,mpeg,mpegts") > 0);
763 } else if (!strcmp(in_fmt->name, "webvtt")) {
764 matchF = av_match_ext( seg->url, "vtt,webvtt")
765 + 2*(ff_match_url_ext(seg->url, "vtt,webvtt") > 0);
764 }766 }
765 767
766 if (!(matchA & matchF)) {768 if (!(matchA & matchF)) {
@@ -969,7 +971,7 @@ static int parse_playlist(HLSContext *c, const char *url, · 202 unmodified lines
969 if (ptr)971 if (ptr)
970 seg_offset = strtoll(ptr+1, NULL, 10);972 seg_offset = strtoll(ptr+1, NULL, 10);
971 } else if (av_strstart(line, "#", NULL)) {973 } else if (av_strstart(line, "#", NULL)) {
972 av_log(c->ctx, AV_LOG_INFO, "Skip ('%s')\n", line);974 av_log(c->ctx, AV_LOG_VERBOSE, "Skip ('%s')\n", line);
973 continue;975 continue;
974 } else if (line[0]) {976 } else if (line[0]) {
975 if (is_variant) {977 if (is_variant) {
@@ -1484,7 +1486,7 @@ static int playlist_needed(struct playlist *pls) · 508 unmodified lines
1484 int first_st;1486 int first_st;
1485 1487
1486 /* If there is no context or streams yet, the playlist is needed */1488 /* If there is no context or streams yet, the playlist is needed */
1487 if (!pls->ctx || !pls->n_main_streams)1489 if ((!pls->ctx || !pls->n_main_streams) && !pls->is_subtitle)
1488 return 1;1490 return 1;
1489 1491
1490 /* check if any of the streams in the playlist are needed */1492 /* check if any of the streams in the playlist are needed */
@@ -1522,17 +1524,13 @@ static int playlist_needed(struct playlist *pls) · 31 unmodified lines
1522 return 0;1524 return 0;
1523}1525}
1524 1526
1525static int read_data(void *opaque, uint8_t *buf, int buf_size)1527static int reload_playlist(struct playlist *v, HLSContext *c)
1526{1528{
1527 struct playlist *v = opaque;1529 int ret = 0;
1528 HLSContext *c = v->parent->priv_data;
1529 int ret;
1530 int just_opened = 0;
1531 int reload_count = 0;1530 int reload_count = 0;
1532 int segment_retries = 0;
1533 struct segment *seg;
1534 1531
1535restart:1532 v->needed = playlist_needed(v);
1533
1536 if (!v->needed)1534 if (!v->needed)
1537 return AVERROR_EOF;1535 return AVERROR_EOF;
1538 1536
@@ -1588,7 +1586,7 @@ reload: · 49 unmodified lines
1588 av_log(v->parent, AV_LOG_WARNING, "The m3u8 list sequence may have been wrapped.\n");1586 av_log(v->parent, AV_LOG_WARNING, "The m3u8 list sequence may have been wrapped.\n");
1589 }1587 }
1590 if (v->cur_seq_no >= v->start_seq_no + v->n_segments) {1588 if (v->cur_seq_no >= v->start_seq_no + v->n_segments) {
1591 if (v->finished)1589 if (v->finished || v->is_subtitle)
1592 return AVERROR_EOF;1590 return AVERROR_EOF;
1593 while (av_gettime_relative() - v->last_load_time < reload_interval) {1591 while (av_gettime_relative() - v->last_load_time < reload_interval) {
1594 if (ff_check_interrupt(c->interrupt_callback))1592 if (ff_check_interrupt(c->interrupt_callback))
@@ -1599,9 +1597,35 @@ reload: · 4 unmodified lines
1599 goto reload;1597 goto reload;
1600 }1598 }
1601 1599
1602 v->input_read_done = 0;1600 }
1603 seg = current_segment(v);1601 return ret;
1602}
1604 1603
1604static int read_data_continuous(void *opaque, uint8_t *buf, int buf_size)
1605{
1606 struct playlist *v = opaque;
1607 HLSContext *c = v->parent->priv_data;
1608 int ret;
1609 int just_opened = 0;
1610 int segment_retries = 0;
1611 struct segment *seg;
1612
1613 if (c->http_persistent && v->input_read_done) {
1614 ret = reload_playlist(v, c);
1615 if (ret < 0)
1616 return ret;
1617 }
1618
1619 v->input_read_done = 0;
1620
1621restart:
1622 ret = reload_playlist(v, c);
1623 if (ret < 0)
1624 return ret;
1625
1626 seg = current_segment(v);
1627
1628 if (!v->input || (c->http_persistent && v->input_read_done)) {
1605 /* load/update Media Initialization Section, if any */1629 /* load/update Media Initialization Section, if any */
1606 ret = update_init_section(v, seg);1630 ret = update_init_section(v, seg);
1607 if (ret)1631 if (ret)
@@ -1630,7 +1654,7 @@ reload: · 22 unmodified lines
1630 } else {1654 } else {
1631 segment_retries++;1655 segment_retries++;
1632 }1656 }
1633 goto reload;1657 goto restart;
1634 }1658 }
1635 segment_retries = 0;1659 segment_retries = 0;
1636 just_opened = 1;1660 just_opened = 1;
@@ -1692,6 +1716,110 @@ reload: · 55 unmodified lines
1692 goto restart;1716 goto restart;
1693}1717}
1694 1718
1719static int read_data_subtitle_segment(void *opaque, uint8_t *buf, int buf_size)
1720{
1721 struct playlist *v = opaque;
1722 HLSContext *c = v->parent->priv_data;
1723 int ret;
1724 struct segment *seg;
1725
1726 if (!v->needed || v->cur_seq_no - v->start_seq_no >= v->n_segments) {
1727 return AVERROR_EOF;
1728 } else {
1729 seg = current_segment(v);
1730 }
1731
1732 if (!v->input) {
1733 ret = open_input(c, v, seg, &v->input);
1734 if (ret < 0) {
1735 if (ff_check_interrupt(c->interrupt_callback))
1736 return AVERROR_EXIT;
1737 av_log(v->parent, AV_LOG_WARNING, "Failed to open segment of playlist %d\n",
1738 v->index);
1739 return ret;
1740 }
1741 }
1742
1743 return read_from_url(v, seg, buf, buf_size);
1744}
1745
1746static int nested_io_open(AVFormatContext *s, AVIOContext **pb, const char *url,
1747 int flags, AVDictionary **opts)
1748{
1749 av_log(s, AV_LOG_ERROR,
1750 "A HLS playlist item '%s' referred to an external file '%s'. "
1751 "Opening this file was forbidden for security reasons\n",
1752 s->url, url);
1753 return AVERROR(EPERM);
1754}
1755
1756static int init_subtitle_context(struct playlist *pls)
1757{
1758 HLSContext *c = pls->parent->priv_data;
1759 const AVInputFormat *in_fmt;
1760 AVDictionary *opts = NULL;
1761 int ret;
1762
1763 if (!(pls->ctx = avformat_alloc_context()))
1764 return AVERROR(ENOMEM);
1765
1766 pls->read_buffer = av_malloc(INITIAL_BUFFER_SIZE);
1767 if (!pls->read_buffer) {
1768 avformat_free_context(pls->ctx);
1769 pls->ctx = NULL;
1770 return AVERROR(ENOMEM);
1771 }
1772
1773 ffio_init_context(&pls->pb, pls->read_buffer, INITIAL_BUFFER_SIZE, 0, pls,
1774 read_data_subtitle_segment, NULL, NULL);
1775 pls->pb.pub.seekable = 0;
1776 pls->ctx->pb = &pls->pb.pub;
1777 pls->ctx->io_open = nested_io_open;
1778
1779 ret = ff_copy_whiteblacklists(pls->ctx, pls->parent);
1780 if (ret < 0)
1781 return ret;
1782
1783 in_fmt = av_find_input_format("webvtt");
1784 av_dict_copy(&opts, c->seg_format_opts, 0);
1785 ret = avformat_open_input(&pls->ctx, current_segment(pls)->url, in_fmt, &opts);
1786 av_dict_free(&opts);
1787
1788 return ret;
1789}
1790
1791static int read_subtitle_packet(struct playlist *v, AVPacket *pkt)
1792{
1793 HLSContext *c = v->parent->priv_data;
1794 int ret;
1795
1796restart:
1797 ret = reload_playlist(v, c);
1798 if (ret < 0)
1799 return ret;
1800
1801 if (v->input && !v->ctx)
1802 ff_format_io_close(v->parent, &v->input);
1803
1804 if (!v->input && !v->ctx) {
1805 ret = init_subtitle_context(v);
1806 if (ret < 0)
1807 return ret;
1808 }
1809
1810 ret = av_read_frame(v->ctx, v->pkt);
1811 if (!ret) {
1812 return ret;
1813 }
1814 ff_format_io_close(v->parent, &v->input);
1815 v->cur_seq_no++;
1816 c->cur_seq_no = v->cur_seq_no;
1817
1818 avformat_close_input(&v->ctx);
1819
1820 goto restart;
1821}
1822
1695static void add_renditions_to_variant(HLSContext *c, struct variant *var,1823static void add_renditions_to_variant(HLSContext *c, struct variant *var,
1696 enum AVMediaType type, const char *group_id)1824 enum AVMediaType type, const char *group_id)
1697{1825{
@@ -1853,16 +1981,6 @@ static int64_t select_cur_seq_no(HLSContext *c, struct playlist *pls) · 155 unmodified lines
1853 return pls->start_seq_no;1981 return pls->start_seq_no;
1854}1982}
1855 1983
1856static int nested_io_open(AVFormatContext *s, AVIOContext **pb, const char *url,
1857 int flags, AVDictionary **opts)
1858{
1859 av_log(s, AV_LOG_ERROR,
1860 "A HLS playlist item '%s' referred to an external file '%s'. "
1861 "Opening this file was forbidden for security reasons\n",
1862 s->url, url);
1863 return AVERROR(EPERM);
1864}
1865
1866static void add_stream_to_programs(AVFormatContext *s, struct playlist *pls, AVStream *stream)1984static void add_stream_to_programs(AVFormatContext *s, struct playlist *pls, AVStream *stream)
1867{1985{
1868 HLSContext *c = s->priv_data;1986 HLSContext *c = s->priv_data;
@@ -2070,6 +2188,8 @@ static int hls_read_header(AVFormatContext *s) · 201 unmodified lines
2070 highest_cur_seq_no = FFMAX(highest_cur_seq_no, pls->cur_seq_no);2188 highest_cur_seq_no = FFMAX(highest_cur_seq_no, pls->cur_seq_no);
2071 }2189 }
2072 2190
2191 av_dict_set(&c->seg_format_opts, "prefer_hls_mpegts_pts", "1", 0);
2192
2073 /* Open the demuxer for each playlist */2193 /* Open the demuxer for each playlist */
2074 for (i = 0; i < c->n_playlists; i++) {2194 for (i = 0; i < c->n_playlists; i++) {
2075 struct playlist *pls = c->playlists[i];2195 struct playlist *pls = c->playlists[i];
@@ -2107,8 +2227,12 @@ static int hls_read_header(AVFormatContext *s) · 31 unmodified lines
2107 return AVERROR(ENOMEM);2227 return AVERROR(ENOMEM);
2108 }2228 }
2109 2229
2110 ffio_init_context(&pls->pb, pls->read_buffer, INITIAL_BUFFER_SIZE, 0, pls,2230 if (pls->is_subtitle)
2111 read_data, NULL, NULL);2231 ffio_init_context(&pls->pb, (unsigned char*)av_strdup("WEBVTT\n"), (int)strlen("WEBVTT\n"), 0, pls,
2232 NULL, NULL, NULL);
2233 else
2234 ffio_init_context(&pls->pb, pls->read_buffer, INITIAL_BUFFER_SIZE, 0, pls,
2235 read_data_continuous, NULL, NULL);
2112 2236
2113 /*2237 /*
2114 * If encryption scheme is SAMPLE-AES, try to read ID3 tags of2238 * If encryption scheme is SAMPLE-AES, try to read ID3 tags of
@@ -2254,6 +2378,13 @@ static int hls_read_header(AVFormatContext *s) · 139 unmodified lines
2254 if (pls->n_main_streams)2378 if (pls->n_main_streams)
2255 av_dict_copy(&pls->main_streams[0]->metadata, pls->ctx->metadata, 0);2379 av_dict_copy(&pls->main_streams[0]->metadata, pls->ctx->metadata, 0);
2256 2380
2381 if (pls->is_subtitle) {
2382 avformat_free_context(pls->ctx);
2383 pls->ctx = NULL;
2384 pls->needed = 0;
2385 pls->main_streams[0]->discard = AVDISCARD_ALL;
2386 }
2387
2257 add_metadata_from_renditions(s, pls, AVMEDIA_TYPE_AUDIO);2388 add_metadata_from_renditions(s, pls, AVMEDIA_TYPE_AUDIO);
2258 add_metadata_from_renditions(s, pls, AVMEDIA_TYPE_VIDEO);2389 add_metadata_from_renditions(s, pls, AVMEDIA_TYPE_VIDEO);
2259 add_metadata_from_renditions(s, pls, AVMEDIA_TYPE_SUBTITLE);2390 add_metadata_from_renditions(s, pls, AVMEDIA_TYPE_SUBTITLE);
@@ -2296,6 +2427,8 @@ static int recheck_discard_flags(AVFormatContext *s, int first) · 36 unmodified lines
2296 pls->input_read_done = 0;2427 pls->input_read_done = 0;
2297 ff_format_io_close(pls->parent, &pls->input_next);2428 ff_format_io_close(pls->parent, &pls->input_next);
2298 pls->input_next_requested = 0;2429 pls->input_next_requested = 0;
2430 if (pls->is_subtitle)
2431 avformat_close_input(&pls->ctx);
2299 pls->needed = 0;2432 pls->needed = 0;
2300 changed = 1;2433 changed = 1;
2301 av_log(s, AV_LOG_INFO, "No longer receiving playlist %d\n", i);2434 av_log(s, AV_LOG_INFO, "No longer receiving playlist %d\n", i);
@@ -2363,7 +2496,10 @@ static int hls_read_packet(AVFormatContext *s, AVPacket *pkt) · 61 unmodified lines
2363 int64_t ts_diff;2496 int64_t ts_diff;
2364 AVRational tb;2497 AVRational tb;
2365 struct segment *seg = NULL;2498 struct segment *seg = NULL;
2366 ret = av_read_frame(pls->ctx, pls->pkt);2499 if (pls->is_subtitle)
2500 ret = read_subtitle_packet(pls, pls->pkt);
2501 else
2502 ret = av_read_frame(pls->ctx, pls->pkt);
2367 if (ret < 0) {2503 if (ret < 0) {
2368 if (!avio_feof(&pls->pb.pub) && ret != AVERROR_EOF)2504 if (!avio_feof(&pls->pb.pub) && ret != AVERROR_EOF)
2369 return ret;2505 return ret;
@@ -2559,7 +2695,10 @@ static int hls_read_seek(AVFormatContext *s, int stream_index, · 189 unmodified lines
2559 /* Reset the pos, to let the mpegts/mov demuxer know we've seeked. */2695 /* Reset the pos, to let the mpegts/mov demuxer know we've seeked. */
2560 pb->pos = 0;2696 pb->pos = 0;
2561 /* Flush the packet queue of the subdemuxer. */2697 /* Flush the packet queue of the subdemuxer. */
2562 ff_read_frame_flush(pls->ctx);2698 if (pls->ctx)
2699 ff_read_frame_flush(pls->ctx);
2700 if (pls->is_subtitle)
2701 avformat_close_input(&pls->ctx);
2563 2702
2564 /* Reset the init segment so it's re-fetched and served appropiately */2703 /* Reset the init segment so it's re-fetched and served appropiately */
2565 pls->cur_init_section = NULL;2704 pls->cur_init_section = NULL;
@@ -2628,7 +2767,7 @@ static const AVOption hls_options[] = { · 62 unmodified lines
2628 OFFSET(prefer_x_start), AV_OPT_TYPE_BOOL, { .i64 = 0 }, 0, 1, FLAGS},2767 OFFSET(prefer_x_start), AV_OPT_TYPE_BOOL, { .i64 = 0 }, 0, 1, FLAGS},
2629 {"allowed_extensions", "List of file extensions that hls is allowed to access",2768 {"allowed_extensions", "List of file extensions that hls is allowed to access",
2630 OFFSET(allowed_extensions), AV_OPT_TYPE_STRING,2769 OFFSET(allowed_extensions), AV_OPT_TYPE_STRING,
2631 {.str = "3gp,aac,avi,ac3,eac3,flac,mkv,m3u8,m4a,m4s,m4v,mpg,mov,mp2,mp3,mp4,mpeg,mpegts,ogg,ogv,oga,ts,vob,wav"},2770 {.str = "3gp,aac,avi,ac3,eac3,flac,mkv,m3u8,m4a,m4s,m4v,mpg,mov,mp2,mp3,mp4,mpeg,mpegts,ogg,ogv,oga,ts,vob,vtt,wav,webvtt"},
2632 INT_MIN, INT_MAX, FLAGS},2771 INT_MIN, INT_MAX, FLAGS},
2633 {"extension_picky", "Be picky with all extensions matching",2772 {"extension_picky", "Be picky with all extensions matching",
2634 OFFSET(extension_picky), AV_OPT_TYPE_BOOL, {.i64 = 1}, 0, 1, FLAGS},2773 OFFSET(extension_picky), AV_OPT_TYPE_BOOL, {.i64 = 1}, 0, 1, FLAGS},

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t