History

avformat/http: Add support for Retry-After header

429 and 503 codes can, and often do (e.g. all Google Cloud Storage URLs can), return a Retry-After header with the error, indicating how long to wait, asd either a date, or in seconds, before retrying again. If it is not respected by, for example, using our default backoff stratetgy instead, chances of success are very unlikely. Some references: * https://datatracker.ietf.org/doc/html/rfc6585 * https://datatracker.ietf.org/doc/html/rfc7231#section-7.1.3 This adds an AVOption to respect that header.

Derek Buitenhuis committed 2 years ago

Commit 2d5fa811 parent 6d89fd43 files · +30 −1Browse files

Unified · Split

3 files
  1. doc/protocols.texi +5 −0
  2. libavformat/http.c +24 −0
  3. libavformat/version.h +1 −1
doc/protocols.texi+5−0
@@ -569,6 +569,11 @@ Set the maximum number of times to retry a connection. Default unset. · 568 unmodified lines
569569 @item reconnect_delay_total_max
570570 Set the maximum total delay in seconds after which to give up reconnecting.
571571
572+@item respect_retry_after
573+If enabled, and a Retry-After header is encountered, its requested reconnection
574+delay will be honored, rather than using exponential backoff. Useful for 429 and
575+503 errors. Default enabled.
576+
572577 @item listen
573578 If set to 1 enables experimental HTTP server. This can be used to send data when
574579 used as an output option, or read data from a client with HTTP POST when used as
libavformat/http.c+24−0
@@ -31,6 +31,7 @@ · 30 unmodified lines
3131 #include "libavutil/avstring.h"
3232 #include "libavutil/bprint.h"
3333 #include "libavutil/getenv_utf8.h"
34+#include "libavutil/macros.h"
3435 #include "libavutil/mem.h"
3536 #include "libavutil/opt.h"
3637 #include "libavutil/time.h"
@@ -138,6 +139,8 @@ typedef struct HTTPContext { · 101 unmodified lines
138139 char *new_location;
139140 AVDictionary *redirect_cache;
140141 uint64_t filesize_from_content_range;
142+ int respect_retry_after;
143+ unsigned int retry_after;
141144 int reconnect_max_retries;
142145 int reconnect_delay_total_max;
143146 } HTTPContext;
@@ -180,6 +183,7 @@ static const AVOption options[] = { · 36 unmodified lines
180183 { "reconnect_delay_max", "max reconnect delay in seconds after which to give up", OFFSET(reconnect_delay_max), AV_OPT_TYPE_INT, { .i64 = 120 }, 0, UINT_MAX/1000/1000, D },
181184 { "reconnect_max_retries", "the max number of times to retry a connection", OFFSET(reconnect_max_retries), AV_OPT_TYPE_INT, { .i64 = -1 }, -1, INT_MAX, D },
182185 { "reconnect_delay_total_max", "max total reconnect delay in seconds after which to give up", OFFSET(reconnect_delay_total_max), AV_OPT_TYPE_INT, { .i64 = 256 }, 0, UINT_MAX/1000/1000, D },
186+ { "respect_retry_after", "respect the Retry-After header when retrying connections", OFFSET(respect_retry_after), AV_OPT_TYPE_BOOL, { .i64 = 1 }, 0, 1, D },
183187 { "listen", "listen on HTTP", OFFSET(listen), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, 2, D | E },
184188 { "resource", "The resource requested by a client", OFFSET(resource), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, E },
185189 { "reply_code", "The http status code to return to a client", OFFSET(reply_code), AV_OPT_TYPE_INT, { .i64 = 200}, INT_MIN, 599, E},
@@ -393,6 +397,14 @@ redo: · 207 unmodified lines
393397 reconnect_delay_total > s->reconnect_delay_total_max)
394398 goto fail;
395399
400+ /* Both fields here are in seconds. */
401+ if (s->respect_retry_after && s->retry_after > 0) {
402+ reconnect_delay = s->retry_after;
403+ if (reconnect_delay > s->reconnect_delay_max)
404+ goto fail;
405+ s->retry_after = 0;
406+ }
407+
396408 av_log(h, AV_LOG_WARNING, "Will reconnect at %"PRIu64" in %d second(s).\n", off, reconnect_delay);
397409 ret = ff_network_sleep_interruptible(1000U * 1000 * reconnect_delay, &h->interrupt_callback);
398410 if (ret != AVERROR(ETIMEDOUT))
@@ -1241,6 +1253,18 @@ static int process_line(URLContext *h, char *line, int line_count, int *parsed_h · 842 unmodified lines
12411253 parse_expires(s, p);
12421254 } else if (!av_strcasecmp(tag, "Cache-Control")) {
12431255 parse_cache_control(s, p);
1256+ } else if (!av_strcasecmp(tag, "Retry-After")) {
1257+ /* The header can be either an integer that represents seconds, or a date. */
1258+ struct tm tm;
1259+ int date_ret = parse_http_date(p, &tm);
1260+ if (!date_ret) {
1261+ time_t retry = av_timegm(&tm);
1262+ int64_t now = av_gettime() / 1000000;
1263+ int64_t diff = ((int64_t) retry) - now;
1264+ s->retry_after = (unsigned int) FFMAX(0, diff);
1265+ } else {
1266+ s->retry_after = strtoul(p, NULL, 10);
1267+ }
12441268 }
12451269 }
12461270 return 1;
libavformat/version.h+1−1
@@ -32,7 +32,7 @@ · 31 unmodified lines
3232 #include "version_major.h"
3333
3434 #define LIBAVFORMAT_VERSION_MINOR 3
35−#define LIBAVFORMAT_VERSION_MICRO 102
35+#define LIBAVFORMAT_VERSION_MICRO 103
3636
3737 #define LIBAVFORMAT_VERSION_INT AV_VERSION_INT(LIBAVFORMAT_VERSION_MAJOR, \
3838 LIBAVFORMAT_VERSION_MINOR, \

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t