avformat/http: Add support for Retry-After header

429 and 503 codes can, and often do (e.g. all Google Cloud Storage URLs can), return a Retry-After header with the error, indicating how long to wait, asd either a date, or in seconds, before retrying again. If it is not respected by, for example, using our default backoff stratetgy instead, chances of success are very unlikely. Some references: * https://datatracker.ietf.org/doc/html/rfc6585 * https://datatracker.ietf.org/doc/html/rfc7231#section-7.1.3 This adds an AVOption to respect that header.

Derek Buitenhuis committed 2 years ago

Commit 2d5fa811 parent 6d89fd43 files · +30 −1Browse files

Unified · Split

3 files

doc/protocols.texi +5 −0
libavformat/http.c +24 −0
libavformat/version.h +1 −1

doc/protocols.texi+5−0

@@ -569,6 +569,11 @@ Set the maximum number of times to retry a connection. Default unset. · 568 unmodified lines
569	@item reconnect_delay_total_max	569	@item reconnect_delay_total_max
570	Set the maximum total delay in seconds after which to give up reconnecting.	570	Set the maximum total delay in seconds after which to give up reconnecting.
571		571
		572	@item respect_retry_after
		573	If enabled, and a Retry-After header is encountered, its requested reconnection
		574	delay will be honored, rather than using exponential backoff. Useful for 429 and
		575	503 errors. Default enabled.
		576
572	@item listen	577	@item listen
573	If set to 1 enables experimental HTTP server. This can be used to send data when	578	If set to 1 enables experimental HTTP server. This can be used to send data when
574	used as an output option, or read data from a client with HTTP POST when used as	579	used as an output option, or read data from a client with HTTP POST when used as

libavformat/http.c+24−0

 #include "libavutil/avstring.h"
 #include "libavutil/bprint.h"
 #include "libavutil/getenv_utf8.h"
+#include "libavutil/macros.h"
 #include "libavutil/mem.h"
 #include "libavutil/opt.h"
 #include "libavutil/time.h"
     char *new_location;
     AVDictionary *redirect_cache;
     uint64_t filesize_from_content_range;
+    int respect_retry_after;
+    unsigned int retry_after;
     int reconnect_max_retries;
     int reconnect_delay_total_max;
 } HTTPContext;
     { "reconnect_delay_max", "max reconnect delay in seconds after which to give up", OFFSET(reconnect_delay_max), AV_OPT_TYPE_INT, { .i64 = 120 }, 0, UINT_MAX/1000/1000, D },
     { "reconnect_max_retries", "the max number of times to retry a connection", OFFSET(reconnect_max_retries), AV_OPT_TYPE_INT, { .i64 = -1 }, -1, INT_MAX, D },
     { "reconnect_delay_total_max", "max total reconnect delay in seconds after which to give up", OFFSET(reconnect_delay_total_max), AV_OPT_TYPE_INT, { .i64 = 256 }, 0, UINT_MAX/1000/1000, D },
+    { "respect_retry_after", "respect the Retry-After header when retrying connections", OFFSET(respect_retry_after), AV_OPT_TYPE_BOOL, { .i64 = 1 }, 0, 1, D },
     { "listen", "listen on HTTP", OFFSET(listen), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, 2, D | E },
     { "resource", "The resource requested by a client", OFFSET(resource), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, E },
     { "reply_code", "The http status code to return to a client", OFFSET(reply_code), AV_OPT_TYPE_INT, { .i64 = 200}, INT_MIN, 599, E},
             reconnect_delay_total > s->reconnect_delay_total_max)
             goto fail;
+        /* Both fields here are in seconds. */
+        if (s->respect_retry_after && s->retry_after > 0) {
+            reconnect_delay = s->retry_after;
+            if (reconnect_delay > s->reconnect_delay_max)
+                goto fail;
+            s->retry_after = 0;
+        }
         av_log(h, AV_LOG_WARNING, "Will reconnect at %"PRIu64" in %d second(s).\n", off, reconnect_delay);
         ret = ff_network_sleep_interruptible(1000U * 1000 * reconnect_delay, &h->interrupt_callback);
         if (ret != AVERROR(ETIMEDOUT))
             parse_expires(s, p);
         } else if (!av_strcasecmp(tag, "Cache-Control")) {
             parse_cache_control(s, p);
+        } else if (!av_strcasecmp(tag, "Retry-After")) {
+            /* The header can be either an integer that represents seconds, or a date. */
+            struct tm tm;
+            int date_ret = parse_http_date(p, &tm);
+            if (!date_ret) {
+                time_t retry   = av_timegm(&tm);
+                int64_t now    = av_gettime() / 1000000;
+                int64_t diff   = ((int64_t) retry) - now;
+                s->retry_after = (unsigned int) FFMAX(0, diff);
+            } else {
+                s->retry_after = strtoul(p, NULL, 10);
+            }
         }
     }
     return 1;

libavformat/version.h+1−1

@@ -32,7 +32,7 @@ · 31 unmodified lines
32	#include "version_major.h"	32	#include "version_major.h"
33		33
34	#define LIBAVFORMAT_VERSION_MINOR 3	34	#define LIBAVFORMAT_VERSION_MINOR 3
35	#define LIBAVFORMAT_VERSION_MICRO 102	35	#define LIBAVFORMAT_VERSION_MICRO 103
36		36
37	#define LIBAVFORMAT_VERSION_INT AV_VERSION_INT(LIBAVFORMAT_VERSION_MAJOR, \	37	#define LIBAVFORMAT_VERSION_INT AV_VERSION_INT(LIBAVFORMAT_VERSION_MAJOR, \
38	LIBAVFORMAT_VERSION_MINOR, \	38	LIBAVFORMAT_VERSION_MINOR, \

avformat/http: Add support for Retry-After header

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code