This add support for WebVTT subtitles in HLS streams. Just like for separate audio streams, it supports all available WebVTT streams in all renditions. No new options are added, it just works and provides subtitles streams like any other demuxer. The code prevents downloading subtitle segments which are farther in the future than the main segments, to avoid loading hundreds of subtitle segments in advance.
Showing the changes to libavformat (1 of 1 file). Show all
Unified · Split
| @@ -173,6 +173,7 @@ struct playlist { · 172 unmodified lines | ||
| 173 | 173 | * playlist, if any. */ |
| 174 | 174 | int n_init_sections; |
| 175 | 175 | struct segment **init_sections; |
| 176 | + int is_subtitle; /* Indicates if it's a subtitle playlist */ | |
| 176 | 177 | }; |
| 177 | 178 | |
| 178 | 179 | /* |
| @@ -515,13 +516,6 @@ static struct rendition *new_rendition(HLSContext *c, struct rendition_info *inf · 336 unmodified lines | ||
| 515 | 516 | return NULL; |
| 516 | 517 | } |
| 517 | 518 | |
| 518 | − /* TODO: handle subtitles (each segment has to parsed separately) */ | |
| 519 | − if (c->ctx->strict_std_compliance > FF_COMPLIANCE_EXPERIMENTAL) | |
| 520 | − if (type == AVMEDIA_TYPE_SUBTITLE) { | |
| 521 | − av_log(c->ctx, AV_LOG_WARNING, "Can't support the subtitle(uri: %s)\n", info->uri); | |
| 522 | − return NULL; | |
| 523 | − } | |
| 524 | − | |
| 525 | 519 | rend = av_mallocz(sizeof(struct rendition)); |
| 526 | 520 | if (!rend) |
| 527 | 521 | return NULL; |
| @@ -536,9 +530,14 @@ static struct rendition *new_rendition(HLSContext *c, struct rendition_info *inf · 8 unmodified lines | ||
| 536 | 530 | /* add the playlist if this is an external rendition */ |
| 537 | 531 | if (info->uri[0]) { |
| 538 | 532 | rend->playlist = new_playlist(c, info->uri, url_base); |
| 539 | − if (rend->playlist) | |
| 533 | + if (rend->playlist) { | |
| 534 | + if (type == AVMEDIA_TYPE_SUBTITLE) { | |
| 535 | + rend->playlist->is_subtitle = 1; | |
| 536 | + rend->playlist->is_id3_timestamped = 0; | |
| 537 | + } | |
| 540 | 538 | dynarray_add(&rend->playlist->renditions, |
| 541 | 539 | &rend->playlist->n_renditions, rend); |
| 540 | + } | |
| 542 | 541 | } |
| 543 | 542 | |
| 544 | 543 | if (info->assoc_language[0]) { |
| @@ -761,6 +760,9 @@ static int test_segment(AVFormatContext *s, const AVInputFormat *in_fmt, struct · 216 unmodified lines | ||
| 761 | 760 | } else if (!strcmp(in_fmt->name, "mpegts")) { |
| 762 | 761 | matchF = av_match_ext( seg->url, "ts,m2t,m2ts,mts,mpg,m4s,mpeg,mpegts") |
| 763 | 762 | + 2*(ff_match_url_ext(seg->url, "ts,m2t,m2ts,mts,mpg,m4s,mpeg,mpegts") > 0); |
| 763 | + } else if (!strcmp(in_fmt->name, "webvtt")) { | |
| 764 | + matchF = av_match_ext( seg->url, "vtt,webvtt") | |
| 765 | + + 2*(ff_match_url_ext(seg->url, "vtt,webvtt") > 0); | |
| 764 | 766 | } |
| 765 | 767 | |
| 766 | 768 | if (!(matchA & matchF)) { |
| @@ -969,7 +971,7 @@ static int parse_playlist(HLSContext *c, const char *url, · 202 unmodified lines | ||
| 969 | 971 | if (ptr) |
| 970 | 972 | seg_offset = strtoll(ptr+1, NULL, 10); |
| 971 | 973 | } else if (av_strstart(line, "#", NULL)) { |
| 972 | − av_log(c->ctx, AV_LOG_INFO, "Skip ('%s')\n", line); | |
| 974 | + av_log(c->ctx, AV_LOG_VERBOSE, "Skip ('%s')\n", line); | |
| 973 | 975 | continue; |
| 974 | 976 | } else if (line[0]) { |
| 975 | 977 | if (is_variant) { |
| @@ -1484,7 +1486,7 @@ static int playlist_needed(struct playlist *pls) · 508 unmodified lines | ||
| 1484 | 1486 | int first_st; |
| 1485 | 1487 | |
| 1486 | 1488 | /* If there is no context or streams yet, the playlist is needed */ |
| 1487 | − if (!pls->ctx || !pls->n_main_streams) | |
| 1489 | + if ((!pls->ctx || !pls->n_main_streams) && !pls->is_subtitle) | |
| 1488 | 1490 | return 1; |
| 1489 | 1491 | |
| 1490 | 1492 | /* check if any of the streams in the playlist are needed */ |
| @@ -1522,17 +1524,13 @@ static int playlist_needed(struct playlist *pls) · 31 unmodified lines | ||
| 1522 | 1524 | return 0; |
| 1523 | 1525 | } |
| 1524 | 1526 | |
| 1525 | −static int read_data(void *opaque, uint8_t *buf, int buf_size) | |
| 1527 | +static int reload_playlist(struct playlist *v, HLSContext *c) | |
| 1526 | 1528 | { |
| 1527 | − struct playlist *v = opaque; | |
| 1528 | − HLSContext *c = v->parent->priv_data; | |
| 1529 | − int ret; | |
| 1530 | − int just_opened = 0; | |
| 1529 | + int ret = 0; | |
| 1531 | 1530 | int reload_count = 0; |
| 1532 | − int segment_retries = 0; | |
| 1533 | − struct segment *seg; | |
| 1534 | 1531 | |
| 1535 | −restart: | |
| 1532 | + v->needed = playlist_needed(v); | |
| 1533 | + | |
| 1536 | 1534 | if (!v->needed) |
| 1537 | 1535 | return AVERROR_EOF; |
| 1538 | 1536 | |
| @@ -1588,7 +1586,7 @@ reload: · 49 unmodified lines | ||
| 1588 | 1586 | av_log(v->parent, AV_LOG_WARNING, "The m3u8 list sequence may have been wrapped.\n"); |
| 1589 | 1587 | } |
| 1590 | 1588 | if (v->cur_seq_no >= v->start_seq_no + v->n_segments) { |
| 1591 | − if (v->finished) | |
| 1589 | + if (v->finished || v->is_subtitle) | |
| 1592 | 1590 | return AVERROR_EOF; |
| 1593 | 1591 | while (av_gettime_relative() - v->last_load_time < reload_interval) { |
| 1594 | 1592 | if (ff_check_interrupt(c->interrupt_callback)) |
| @@ -1599,9 +1597,35 @@ reload: · 4 unmodified lines | ||
| 1599 | 1597 | goto reload; |
| 1600 | 1598 | } |
| 1601 | 1599 | |
| 1602 | − v->input_read_done = 0; | |
| 1603 | − seg = current_segment(v); | |
| 1600 | + } | |
| 1601 | + return ret; | |
| 1602 | +} | |
| 1604 | 1603 | |
| 1604 | +static int read_data_continuous(void *opaque, uint8_t *buf, int buf_size) | |
| 1605 | +{ | |
| 1606 | + struct playlist *v = opaque; | |
| 1607 | + HLSContext *c = v->parent->priv_data; | |
| 1608 | + int ret; | |
| 1609 | + int just_opened = 0; | |
| 1610 | + int segment_retries = 0; | |
| 1611 | + struct segment *seg; | |
| 1612 | + | |
| 1613 | + if (c->http_persistent && v->input_read_done) { | |
| 1614 | + ret = reload_playlist(v, c); | |
| 1615 | + if (ret < 0) | |
| 1616 | + return ret; | |
| 1617 | + } | |
| 1618 | + | |
| 1619 | + v->input_read_done = 0; | |
| 1620 | + | |
| 1621 | +restart: | |
| 1622 | + ret = reload_playlist(v, c); | |
| 1623 | + if (ret < 0) | |
| 1624 | + return ret; | |
| 1625 | + | |
| 1626 | + seg = current_segment(v); | |
| 1627 | + | |
| 1628 | + if (!v->input || (c->http_persistent && v->input_read_done)) { | |
| 1605 | 1629 | /* load/update Media Initialization Section, if any */ |
| 1606 | 1630 | ret = update_init_section(v, seg); |
| 1607 | 1631 | if (ret) |
| @@ -1630,7 +1654,7 @@ reload: · 22 unmodified lines | ||
| 1630 | 1654 | } else { |
| 1631 | 1655 | segment_retries++; |
| 1632 | 1656 | } |
| 1633 | − goto reload; | |
| 1657 | + goto restart; | |
| 1634 | 1658 | } |
| 1635 | 1659 | segment_retries = 0; |
| 1636 | 1660 | just_opened = 1; |
| @@ -1692,6 +1716,110 @@ reload: · 55 unmodified lines | ||
| 1692 | 1716 | goto restart; |
| 1693 | 1717 | } |
| 1694 | 1718 | |
| 1719 | +static int read_data_subtitle_segment(void *opaque, uint8_t *buf, int buf_size) | |
| 1720 | +{ | |
| 1721 | + struct playlist *v = opaque; | |
| 1722 | + HLSContext *c = v->parent->priv_data; | |
| 1723 | + int ret; | |
| 1724 | + struct segment *seg; | |
| 1725 | + | |
| 1726 | + if (!v->needed || v->cur_seq_no - v->start_seq_no >= v->n_segments) { | |
| 1727 | + return AVERROR_EOF; | |
| 1728 | + } else { | |
| 1729 | + seg = current_segment(v); | |
| 1730 | + } | |
| 1731 | + | |
| 1732 | + if (!v->input) { | |
| 1733 | + ret = open_input(c, v, seg, &v->input); | |
| 1734 | + if (ret < 0) { | |
| 1735 | + if (ff_check_interrupt(c->interrupt_callback)) | |
| 1736 | + return AVERROR_EXIT; | |
| 1737 | + av_log(v->parent, AV_LOG_WARNING, "Failed to open segment of playlist %d\n", | |
| 1738 | + v->index); | |
| 1739 | + return ret; | |
| 1740 | + } | |
| 1741 | + } | |
| 1742 | + | |
| 1743 | + return read_from_url(v, seg, buf, buf_size); | |
| 1744 | +} | |
| 1745 | + | |
| 1746 | +static int nested_io_open(AVFormatContext *s, AVIOContext **pb, const char *url, | |
| 1747 | + int flags, AVDictionary **opts) | |
| 1748 | +{ | |
| 1749 | + av_log(s, AV_LOG_ERROR, | |
| 1750 | + "A HLS playlist item '%s' referred to an external file '%s'. " | |
| 1751 | + "Opening this file was forbidden for security reasons\n", | |
| 1752 | + s->url, url); | |
| 1753 | + return AVERROR(EPERM); | |
| 1754 | +} | |
| 1755 | + | |
| 1756 | +static int init_subtitle_context(struct playlist *pls) | |
| 1757 | +{ | |
| 1758 | + HLSContext *c = pls->parent->priv_data; | |
| 1759 | + const AVInputFormat *in_fmt; | |
| 1760 | + AVDictionary *opts = NULL; | |
| 1761 | + int ret; | |
| 1762 | + | |
| 1763 | + if (!(pls->ctx = avformat_alloc_context())) | |
| 1764 | + return AVERROR(ENOMEM); | |
| 1765 | + | |
| 1766 | + pls->read_buffer = av_malloc(INITIAL_BUFFER_SIZE); | |
| 1767 | + if (!pls->read_buffer) { | |
| 1768 | + avformat_free_context(pls->ctx); | |
| 1769 | + pls->ctx = NULL; | |
| 1770 | + return AVERROR(ENOMEM); | |
| 1771 | + } | |
| 1772 | + | |
| 1773 | + ffio_init_context(&pls->pb, pls->read_buffer, INITIAL_BUFFER_SIZE, 0, pls, | |
| 1774 | + read_data_subtitle_segment, NULL, NULL); | |
| 1775 | + pls->pb.pub.seekable = 0; | |
| 1776 | + pls->ctx->pb = &pls->pb.pub; | |
| 1777 | + pls->ctx->io_open = nested_io_open; | |
| 1778 | + | |
| 1779 | + ret = ff_copy_whiteblacklists(pls->ctx, pls->parent); | |
| 1780 | + if (ret < 0) | |
| 1781 | + return ret; | |
| 1782 | + | |
| 1783 | + in_fmt = av_find_input_format("webvtt"); | |
| 1784 | + av_dict_copy(&opts, c->seg_format_opts, 0); | |
| 1785 | + ret = avformat_open_input(&pls->ctx, current_segment(pls)->url, in_fmt, &opts); | |
| 1786 | + av_dict_free(&opts); | |
| 1787 | + | |
| 1788 | + return ret; | |
| 1789 | +} | |
| 1790 | + | |
| 1791 | +static int read_subtitle_packet(struct playlist *v, AVPacket *pkt) | |
| 1792 | +{ | |
| 1793 | + HLSContext *c = v->parent->priv_data; | |
| 1794 | + int ret; | |
| 1795 | + | |
| 1796 | +restart: | |
| 1797 | + ret = reload_playlist(v, c); | |
| 1798 | + if (ret < 0) | |
| 1799 | + return ret; | |
| 1800 | + | |
| 1801 | + if (v->input && !v->ctx) | |
| 1802 | + ff_format_io_close(v->parent, &v->input); | |
| 1803 | + | |
| 1804 | + if (!v->input && !v->ctx) { | |
| 1805 | + ret = init_subtitle_context(v); | |
| 1806 | + if (ret < 0) | |
| 1807 | + return ret; | |
| 1808 | + } | |
| 1809 | + | |
| 1810 | + ret = av_read_frame(v->ctx, v->pkt); | |
| 1811 | + if (!ret) { | |
| 1812 | + return ret; | |
| 1813 | + } | |
| 1814 | + ff_format_io_close(v->parent, &v->input); | |
| 1815 | + v->cur_seq_no++; | |
| 1816 | + c->cur_seq_no = v->cur_seq_no; | |
| 1817 | + | |
| 1818 | + avformat_close_input(&v->ctx); | |
| 1819 | + | |
| 1820 | + goto restart; | |
| 1821 | +} | |
| 1822 | + | |
| 1695 | 1823 | static void add_renditions_to_variant(HLSContext *c, struct variant *var, |
| 1696 | 1824 | enum AVMediaType type, const char *group_id) |
| 1697 | 1825 | { |
| @@ -1853,16 +1981,6 @@ static int64_t select_cur_seq_no(HLSContext *c, struct playlist *pls) · 155 unmodified lines | ||
| 1853 | 1981 | return pls->start_seq_no; |
| 1854 | 1982 | } |
| 1855 | 1983 | |
| 1856 | −static int nested_io_open(AVFormatContext *s, AVIOContext **pb, const char *url, | |
| 1857 | − int flags, AVDictionary **opts) | |
| 1858 | −{ | |
| 1859 | − av_log(s, AV_LOG_ERROR, | |
| 1860 | − "A HLS playlist item '%s' referred to an external file '%s'. " | |
| 1861 | − "Opening this file was forbidden for security reasons\n", | |
| 1862 | − s->url, url); | |
| 1863 | − return AVERROR(EPERM); | |
| 1864 | −} | |
| 1865 | − | |
| 1866 | 1984 | static void add_stream_to_programs(AVFormatContext *s, struct playlist *pls, AVStream *stream) |
| 1867 | 1985 | { |
| 1868 | 1986 | HLSContext *c = s->priv_data; |
| @@ -2070,6 +2188,8 @@ static int hls_read_header(AVFormatContext *s) · 201 unmodified lines | ||
| 2070 | 2188 | highest_cur_seq_no = FFMAX(highest_cur_seq_no, pls->cur_seq_no); |
| 2071 | 2189 | } |
| 2072 | 2190 | |
| 2191 | + av_dict_set(&c->seg_format_opts, "prefer_hls_mpegts_pts", "1", 0); | |
| 2192 | + | |
| 2073 | 2193 | /* Open the demuxer for each playlist */ |
| 2074 | 2194 | for (i = 0; i < c->n_playlists; i++) { |
| 2075 | 2195 | struct playlist *pls = c->playlists[i]; |
| @@ -2107,8 +2227,12 @@ static int hls_read_header(AVFormatContext *s) · 31 unmodified lines | ||
| 2107 | 2227 | return AVERROR(ENOMEM); |
| 2108 | 2228 | } |
| 2109 | 2229 | |
| 2110 | − ffio_init_context(&pls->pb, pls->read_buffer, INITIAL_BUFFER_SIZE, 0, pls, | |
| 2111 | − read_data, NULL, NULL); | |
| 2230 | + if (pls->is_subtitle) | |
| 2231 | + ffio_init_context(&pls->pb, (unsigned char*)av_strdup("WEBVTT\n"), (int)strlen("WEBVTT\n"), 0, pls, | |
| 2232 | + NULL, NULL, NULL); | |
| 2233 | + else | |
| 2234 | + ffio_init_context(&pls->pb, pls->read_buffer, INITIAL_BUFFER_SIZE, 0, pls, | |
| 2235 | + read_data_continuous, NULL, NULL); | |
| 2112 | 2236 | |
| 2113 | 2237 | /* |
| 2114 | 2238 | * If encryption scheme is SAMPLE-AES, try to read ID3 tags of |
| @@ -2254,6 +2378,13 @@ static int hls_read_header(AVFormatContext *s) · 139 unmodified lines | ||
| 2254 | 2378 | if (pls->n_main_streams) |
| 2255 | 2379 | av_dict_copy(&pls->main_streams[0]->metadata, pls->ctx->metadata, 0); |
| 2256 | 2380 | |
| 2381 | + if (pls->is_subtitle) { | |
| 2382 | + avformat_free_context(pls->ctx); | |
| 2383 | + pls->ctx = NULL; | |
| 2384 | + pls->needed = 0; | |
| 2385 | + pls->main_streams[0]->discard = AVDISCARD_ALL; | |
| 2386 | + } | |
| 2387 | + | |
| 2257 | 2388 | add_metadata_from_renditions(s, pls, AVMEDIA_TYPE_AUDIO); |
| 2258 | 2389 | add_metadata_from_renditions(s, pls, AVMEDIA_TYPE_VIDEO); |
| 2259 | 2390 | add_metadata_from_renditions(s, pls, AVMEDIA_TYPE_SUBTITLE); |
| @@ -2296,6 +2427,8 @@ static int recheck_discard_flags(AVFormatContext *s, int first) · 36 unmodified lines | ||
| 2296 | 2427 | pls->input_read_done = 0; |
| 2297 | 2428 | ff_format_io_close(pls->parent, &pls->input_next); |
| 2298 | 2429 | pls->input_next_requested = 0; |
| 2430 | + if (pls->is_subtitle) | |
| 2431 | + avformat_close_input(&pls->ctx); | |
| 2299 | 2432 | pls->needed = 0; |
| 2300 | 2433 | changed = 1; |
| 2301 | 2434 | av_log(s, AV_LOG_INFO, "No longer receiving playlist %d\n", i); |
| @@ -2363,7 +2496,10 @@ static int hls_read_packet(AVFormatContext *s, AVPacket *pkt) · 61 unmodified lines | ||
| 2363 | 2496 | int64_t ts_diff; |
| 2364 | 2497 | AVRational tb; |
| 2365 | 2498 | struct segment *seg = NULL; |
| 2366 | − ret = av_read_frame(pls->ctx, pls->pkt); | |
| 2499 | + if (pls->is_subtitle) | |
| 2500 | + ret = read_subtitle_packet(pls, pls->pkt); | |
| 2501 | + else | |
| 2502 | + ret = av_read_frame(pls->ctx, pls->pkt); | |
| 2367 | 2503 | if (ret < 0) { |
| 2368 | 2504 | if (!avio_feof(&pls->pb.pub) && ret != AVERROR_EOF) |
| 2369 | 2505 | return ret; |
| @@ -2559,7 +2695,10 @@ static int hls_read_seek(AVFormatContext *s, int stream_index, · 189 unmodified lines | ||
| 2559 | 2695 | /* Reset the pos, to let the mpegts/mov demuxer know we've seeked. */ |
| 2560 | 2696 | pb->pos = 0; |
| 2561 | 2697 | /* Flush the packet queue of the subdemuxer. */ |
| 2562 | − ff_read_frame_flush(pls->ctx); | |
| 2698 | + if (pls->ctx) | |
| 2699 | + ff_read_frame_flush(pls->ctx); | |
| 2700 | + if (pls->is_subtitle) | |
| 2701 | + avformat_close_input(&pls->ctx); | |
| 2563 | 2702 | |
| 2564 | 2703 | /* Reset the init segment so it's re-fetched and served appropiately */ |
| 2565 | 2704 | pls->cur_init_section = NULL; |
| @@ -2628,7 +2767,7 @@ static const AVOption hls_options[] = { · 62 unmodified lines | ||
| 2628 | 2767 | OFFSET(prefer_x_start), AV_OPT_TYPE_BOOL, { .i64 = 0 }, 0, 1, FLAGS}, |
| 2629 | 2768 | {"allowed_extensions", "List of file extensions that hls is allowed to access", |
| 2630 | 2769 | OFFSET(allowed_extensions), AV_OPT_TYPE_STRING, |
| 2631 | − {.str = "3gp,aac,avi,ac3,eac3,flac,mkv,m3u8,m4a,m4s,m4v,mpg,mov,mp2,mp3,mp4,mpeg,mpegts,ogg,ogv,oga,ts,vob,wav"}, | |
| 2770 | + {.str = "3gp,aac,avi,ac3,eac3,flac,mkv,m3u8,m4a,m4s,m4v,mpg,mov,mp2,mp3,mp4,mpeg,mpegts,ogg,ogv,oga,ts,vob,vtt,wav,webvtt"}, | |
| 2632 | 2771 | INT_MIN, INT_MAX, FLAGS}, |
| 2633 | 2772 | {"extension_picky", "Be picky with all extensions matching", |
| 2634 | 2773 | OFFSET(extension_picky), AV_OPT_TYPE_BOOL, {.i64 = 1}, 0, 1, FLAGS}, |