swscale: introduce new, dynamic scaling API

As part of a larger, ongoing effort to modernize and partially rewrite libswscale, it was decided and generally agreed upon to introduce a new public API for libswscale. This API is designed to be less stateful, more explicitly defined, and considerably easier to use than the existing one. Most of the API work has been already accomplished in the previous commits, this commit merely introduces the ability to use sws_scale_frame() dynamically, without prior sws_init_context() calls. Instead, the new API takes frame properties from the frames themselves, and the implementation is based on the new SwsGraph API, which we simply reinitialize as needed. This high-level wrapper also recreates the logic that used to live inside vf_scale for scaling interlaced frames, enabling it to be reused more easily by end users. Finally, this function is designed to simply copy refs directly when nothing needs to be done, substantially improving throughput of the noop fast path.

Niklas Haas committed 1 year ago

Commit 2a091d41 parent bf738417 files · +276 −30Browse files

Unified · Split

7 files

doc/APIchanges +4 −0
libswscale/swscale.c +190 −6
libswscale/swscale.h +68 −21
libswscale/swscale_internal.h +8 −1
libswscale/utils.c +4 −0
libswscale/version.h +1 −1
libswscale/x86/output.asm +1 −1

doc/APIchanges+4−0

@@ -2,6 +2,10 @@ The last version increases of all libraries were on 2024-03-07 · 1 unmodified line
2	2
3	3	API changes, most recent first:
4	4
	5	+2024-11-25 - xxxxxxxxxx - lsws 8.12.100 - swscale.h
	6	+ Allow using sws_frame_scale() dynamically, without first initializing the
	7	+ SwsContext. Deprecate sws_init_context(). Add sws_frame_setup() instead.
	8	+
5	9	2024-11-25 - xxxxxxxxxx - lsws 8.11.100 - swscale.h
6	10	Replace #define-based SWS_* flags by enum SwsFlags.
7	11

libswscale/swscale.c+190−6

@@ -1219,21 +1219,205 @@ int sws_receive_slice(SwsContext *sws, unsigned int slice_start, · 1218 unmodified lines
1219	1219	dst, c->frame_dst->linesize, slice_start, slice_height);
1220	1220	}
1221	1221
	1222	+static void get_frame_pointers(const AVFrame frame, uint8_t data[4],
	1223	+ int linesize[4], int field)
	1224	+{
	1225	+ for (int i = 0; i < 4; i++) {
	1226	+ data[i] = frame->data[i];
	1227	+ linesize[i] = frame->linesize[i];
	1228	+ }
	1229	+
	1230	+ if (!(frame->flags & AV_FRAME_FLAG_INTERLACED)) {
	1231	+ av_assert1(!field);
	1232	+ return;
	1233	+ }
	1234	+
	1235	+ if (field == FIELD_BOTTOM) {
	1236	+ /* Odd rows, offset by one line */
	1237	+ const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(frame->format);
	1238	+ for (int i = 0; i < 4; i++) {
	1239	+ data[i] += linesize[i];
	1240	+ if (desc->flags & AV_PIX_FMT_FLAG_PAL)
	1241	+ break;
	1242	+ }
	1243	+ }
	1244	+
	1245	+ /* Take only every second line */
	1246	+ for (int i = 0; i < 4; i++)
	1247	+ linesize[i] <<= 1;
	1248	+}
	1249	+
	1250	+/* Subset of av_frame_ref() that only references (video) data buffers */
	1251	+static int frame_ref(AVFrame dst, const AVFrame src)
	1252	+{
	1253	+ /* ref the buffers */
	1254	+ for (int i = 0; i < FF_ARRAY_ELEMS(src->buf); i++) {
	1255	+ if (!src->buf[i])
	1256	+ continue;
	1257	+ dst->buf[i] = av_buffer_ref(src->buf[i]);
	1258	+ if (!dst->buf[i])
	1259	+ return AVERROR(ENOMEM);
	1260	+ }
	1261	+
	1262	+ memcpy(dst->data, src->data, sizeof(src->data));
	1263	+ memcpy(dst->linesize, src->linesize, sizeof(src->linesize));
	1264	+ return 0;
	1265	+}
	1266	+
1222	1267	int sws_scale_frame(SwsContext sws, AVFrame dst, const AVFrame *src)
1223	1268	{
1224	1269	int ret;
	1270	+ SwsInternal *c = sws_internal(sws);
	1271	+ if (!src \|\| !dst)
	1272	+ return AVERROR(EINVAL);
	1273	+
	1274	+ if (c->frame_src) {
	1275	+ /* Context has been initialized with explicit values, fall back to
	1276	+ * legacy API */
	1277	+ ret = sws_frame_start(sws, dst, src);
	1278	+ if (ret < 0)
	1279	+ return ret;
	1280	+
	1281	+ ret = sws_send_slice(sws, 0, src->height);
	1282	+ if (ret >= 0)
	1283	+ ret = sws_receive_slice(sws, 0, dst->height);
1225	1284
1226		− ret = sws_frame_start(sws, dst, src);
	1285	+ sws_frame_end(sws);
	1286	+
	1287	+ return ret;
	1288	+ }
	1289	+
	1290	+ ret = sws_frame_setup(sws, dst, src);
1227	1291	if (ret < 0)
1228	1292	return ret;
1229	1293
1230		− ret = sws_send_slice(sws, 0, src->height);
1231		− if (ret >= 0)
1232		− ret = sws_receive_slice(sws, 0, dst->height);
	1294	+ if (!src->data[0])
	1295	+ return 0;
1233	1296
1234		− sws_frame_end(sws);
	1297	+ if (c->graph[FIELD_TOP]->noop &&
	1298	+ (!c->graph[FIELD_BOTTOM] \|\| c->graph[FIELD_BOTTOM]->noop) &&
	1299	+ src->buf[0] && !dst->buf[0] && !dst->data[0])
	1300	+ {
	1301	+ /* Lightweight refcopy */
	1302	+ ret = frame_ref(dst, src);
	1303	+ if (ret < 0)
	1304	+ return ret;
	1305	+ } else {
	1306	+ if (!dst->data[0]) {
	1307	+ ret = av_frame_get_buffer(dst, 0);
	1308	+ if (ret < 0)
	1309	+ return ret;
	1310	+ }
1235	1311
1236		− return ret;
	1312	+ for (int field = 0; field < 2; field++) {
	1313	+ SwsGraph *graph = c->graph[field];
	1314	+ uint8_t dst_data[4], src_data[4];
	1315	+ int dst_linesize[4], src_linesize[4];
	1316	+ get_frame_pointers(dst, dst_data, dst_linesize, field);
	1317	+ get_frame_pointers(src, src_data, src_linesize, field);
	1318	+ sws_graph_run(graph, dst_data, dst_linesize,
	1319	+ (const uint8_t **) src_data, src_linesize);
	1320	+ if (!graph->dst.interlaced)
	1321	+ break;
	1322	+ }
	1323	+ }
	1324	+
	1325	+ return 0;
	1326	+}
	1327	+
	1328	+static int validate_params(SwsContext *ctx)
	1329	+{
	1330	+#define VALIDATE(field, min, max) \
	1331	+ if (ctx->field < min \|\| ctx->field > max) { \
	1332	+ av_log(ctx, AV_LOG_ERROR, "'%s' (%d) out of range [%d, %d]\n", \
	1333	+ #field, (int) ctx->field, min, max); \
	1334	+ return AVERROR(EINVAL); \
	1335	+ }
	1336	+
	1337	+ VALIDATE(threads, 0, SWS_MAX_THREADS);
	1338	+ VALIDATE(dither, 0, SWS_DITHER_NB - 1)
	1339	+ VALIDATE(alpha_blend, 0, SWS_ALPHA_BLEND_NB - 1)
	1340	+ return 0;
	1341	+}
	1342	+
	1343	+int sws_frame_setup(SwsContext ctx, const AVFrame dst, const AVFrame *src)
	1344	+{
	1345	+ SwsInternal *s = sws_internal(ctx);
	1346	+ const char *err_msg;
	1347	+ int ret;
	1348	+
	1349	+ if (!src \|\| !dst)
	1350	+ return AVERROR(EINVAL);
	1351	+ if ((ret = validate_params(ctx)) < 0)
	1352	+ return ret;
	1353	+
	1354	+ for (int field = 0; field < 2; field++) {
	1355	+ SwsFormat src_fmt = ff_fmt_from_frame(src, field);
	1356	+ SwsFormat dst_fmt = ff_fmt_from_frame(dst, field);
	1357	+
	1358	+ if ((src->flags ^ dst->flags) & AV_FRAME_FLAG_INTERLACED) {
	1359	+ err_msg = "Cannot convert interlaced to progressive frames or vice versa.\n";
	1360	+ ret = AVERROR(EINVAL);
	1361	+ goto fail;
	1362	+ }
	1363	+
	1364	+ /* TODO: remove once implemented */
	1365	+ if ((dst_fmt.prim != src_fmt.prim \|\| dst_fmt.trc != src_fmt.trc) &&
	1366	+ !s->color_conversion_warned)
	1367	+ {
	1368	+ av_log(ctx, AV_LOG_WARNING, "Conversions between different primaries / "
	1369	+ "transfer functions are not currently implemented, expect "
	1370	+ "wrong results.\n");
	1371	+ s->color_conversion_warned = 1;
	1372	+ }
	1373	+
	1374	+ if (!ff_test_fmt(&src_fmt, 0)) {
	1375	+ err_msg = "Unsupported input";
	1376	+ ret = AVERROR(ENOTSUP);
	1377	+ goto fail;
	1378	+ }
	1379	+
	1380	+ if (!ff_test_fmt(&dst_fmt, 1)) {
	1381	+ err_msg = "Unsupported output";
	1382	+ ret = AVERROR(ENOTSUP);
	1383	+ goto fail;
	1384	+ }
	1385	+
	1386	+ ret = sws_graph_reinit(ctx, &dst_fmt, &src_fmt, field, &s->graph[field]);
	1387	+ if (ret < 0) {
	1388	+ err_msg = "Failed initializing scaling graph";
	1389	+ goto fail;
	1390	+ }
	1391	+
	1392	+ if (s->graph[field]->incomplete && ctx->flags & SWS_STRICT) {
	1393	+ err_msg = "Incomplete scaling graph";
	1394	+ ret = AVERROR(EINVAL);
	1395	+ goto fail;
	1396	+ }
	1397	+
	1398	+ if (!src_fmt.interlaced) {
	1399	+ sws_graph_free(&s->graph[FIELD_BOTTOM]);
	1400	+ break;
	1401	+ }
	1402	+
	1403	+ continue;
	1404	+
	1405	+ fail:
	1406	+ av_log(ctx, AV_LOG_ERROR, "%s (%s): fmt:%s csp:%s prim:%s trc:%s ->"
	1407	+ " fmt:%s csp:%s prim:%s trc:%s\n",
	1408	+ err_msg, av_err2str(ret),
	1409	+ av_get_pix_fmt_name(src_fmt.format), av_color_space_name(src_fmt.csp),
	1410	+ av_color_primaries_name(src_fmt.prim), av_color_transfer_name(src_fmt.trc),
	1411	+ av_get_pix_fmt_name(dst_fmt.format), av_color_space_name(dst_fmt.csp),
	1412	+ av_color_primaries_name(dst_fmt.prim), av_color_transfer_name(dst_fmt.trc));
	1413	+
	1414	+ for (int i = 0; i < FF_ARRAY_ELEMS(s->graph); i++)
	1415	+ sws_graph_free(&s->graph[i]);
	1416	+
	1417	+ return ret;
	1418	+ }
	1419	+
	1420	+ return 0;
1237	1421	}
1238	1422
1239	1423	/**

libswscale/swscale.h+68−21

@@ -107,6 +107,12 @@ typedef enum SwsFlags { · 106 unmodified lines
107	107	SWS_LANCZOS = 1 << 9, ///< 3-tap sinc/sinc
108	108	SWS_SPLINE = 1 << 10, ///< cubic Keys spline
109	109
	110	+ /**
	111	+ * Return an error on underspecified conversions. Without this flag,
	112	+ * unspecified fields are defaulted to sensible values.
	113	+ */
	114	+ SWS_STRICT = 1 << 11,
	115	+
110	116	/**
111	117	* Emit verbose log of scaling parameters.
112	118	*/
@@ -204,7 +210,10 @@ typedef struct SwsContext { · 91 unmodified lines
204	210	int gamma_flag;
205	211
206	212	/**
207		− * Frame property overrides.
	213	+ * Deprecated frame property overrides, for the legacy API only.
	214	+ *
	215	+ * Ignored by sws_scale_frame() when used in dynamic mode, in which
	216	+ * case all properties are instead taken from the frame directly.
208	217	*/
209	218	int src_w, src_h; ///< Width and height of the source frame
210	219	int dst_w, dst_h; ///< Width and height of the destination frame
@@ -216,6 +225,8 @@ typedef struct SwsContext { · 5 unmodified lines
216	225	int src_h_chr_pos; ///< Source horizontal chroma position
217	226	int dst_v_chr_pos; ///< Destination vertical chroma position
218	227	int dst_h_chr_pos; ///< Destination horizontal chroma position
	228	+
	229	+ /* Remember to add new fields to graph.c:opts_equal() */
219	230	} SwsContext;
220	231
221	232	/**
@@ -284,12 +295,57 @@ int sws_test_transfer(enum AVColorTransferCharacteristic trc, int output); · 62 unmodified lines
284	295	*/
285	296	int sws_test_frame(const AVFrame *frame, int output);
286	297
	298	+/**
	299	+ * Like `sws_scale_frame`, but without actually scaling. It will instead
	300	+ * merely initialize internal state that would be required to perform the
	301	+ * operation, as well as returning the correct error code for unsupported
	302	+ * frame combinations.
	303	+ *
	304	+ * @param ctx The scaling context.
	305	+ * @param dst The destination frame to consider.
	306	+ * @param src The source frame to consider.
	307	+ * @return 0 on success, a negative AVERROR code on failure.
	308	+ */
	309	+int sws_frame_setup(SwsContext ctx, const AVFrame dst, const AVFrame *src);
	310	+
	311	+/********************
	312	+ * Main scaling API *
	313	+ ********************/
	314	+
287	315	/**
288	316	* Check if a given conversion is a noop. Returns a positive integer if
289	317	* no operation needs to be performed, 0 otherwise.
290	318	*/
291	319	int sws_is_noop(const AVFrame dst, const AVFrame src);
292	320
	321	+/**
	322	+ * Scale source data from `src` and write the output to `dst`.
	323	+ *
	324	+ * This function can be used directly on an allocated context, without setting
	325	+ * up any frame properties or calling `sws_init_context()`. Such usage is fully
	326	+ * dynamic and does not require reallocation if the frame properties change.
	327	+ *
	328	+ * Alternatively, this function can be called on a context that has been
	329	+ * explicitly initialized. However, this is provided only for backwards
	330	+ * compatibility. In this usage mode, all frame properties must be correctly
	331	+ * set at init time, and may no longer change after initialization.
	332	+ *
	333	+ * @param ctx The scaling context.
	334	+ * @param dst The destination frame. The data buffers may either be already
	335	+ * allocated by the caller or left clear, in which case they will
	336	+ * be allocated by the scaler. The latter may have performance
	337	+ * advantages - e.g. in certain cases some (or all) output planes
	338	+ * may be references to input planes, rather than copies.
	339	+ * @param src The source frame. If the data buffers are set to NULL, then
	340	+ * this function behaves identically to `sws_frame_setup`.
	341	+ * @return 0 on success, a negative AVERROR code on failure.
	342	+ */
	343	+int sws_scale_frame(SwsContext c, AVFrame dst, const AVFrame *src);
	344	+
	345	+/*************************
	346	+ * Legacy (stateful) API *
	347	+ *************************/
	348	+
293	349	#define SWS_SRC_V_CHR_DROP_MASK 0x30000
294	350	#define SWS_SRC_V_CHR_DROP_SHIFT 16
295	351
@@ -352,6 +408,11 @@ int sws_isSupportedEndiannessConversion(enum AVPixelFormat pix_fmt); · 56 unmodified lines
352	408	/**
353	409	* Initialize the swscaler context sws_context.
354	410	*
	411	+ * This function is considered deprecated, and provided only for backwards
	412	+ * compatibility with sws_scale() and sws_start_frame(). The preferred way to
	413	+ * use libswscale is to set all frame properties correctly and call
	414	+ * sws_scale_frame() directly, without explicitly initializing the context.
	415	+ *
355	416	* @return zero or positive value on success, a negative value on
356	417	* error
357	418	*/
@@ -393,7 +454,8 @@ SwsContext *sws_getContext(int srcW, int srcH, enum AVPixelFormat srcFormat, · 35 unmodified lines
393	454	/**
394	455	* Scale the image slice in srcSlice and put the resulting scaled
395	456	* slice in the image in dst. A slice is a sequence of consecutive
396		− * rows in an image.
	457	+ * rows in an image. Requires a context that has been previously
	458	+ * been initialized with sws_init_context().
397	459	*
398	460	* Slices have to be provided in sequential order, either in
399	461	* top-bottom or bottom-top order. If slices are provided in
@@ -420,27 +482,11 @@ int sws_scale(SwsContext c, const uint8_t const srcSlice[], · 20 unmodified lines
420	482	const int srcStride[], int srcSliceY, int srcSliceH,
421	483	uint8_t *const dst[], const int dstStride[]);
422	484
423		−/**
424		− * Scale source data from src and write the output to dst.
425		− *
426		− * This is merely a convenience wrapper around
427		− * - sws_frame_start()
428		− * - sws_send_slice(0, src->height)
429		− * - sws_receive_slice(0, dst->height)
430		− * - sws_frame_end()
431		− *
432		− * @param c The scaling context
433		− * @param dst The destination frame. See documentation for sws_frame_start() for
434		− * more details.
435		− * @param src The source frame.
436		− *
437		− * @return 0 on success, a negative AVERROR code on failure
438		− */
439		−int sws_scale_frame(SwsContext c, AVFrame dst, const AVFrame *src);
440		−
441	485	/**
442	486	* Initialize the scaling process for a given pair of source/destination frames.
443	487	* Must be called before any calls to sws_send_slice() and sws_receive_slice().
	488	+ * Requires a context that has been previously been initialized with
	489	+ * sws_init_context().
444	490	*
445	491	* This function will retain references to src and dst, so they must both use
446	492	* refcounted buffers (if allocated by the caller, in case of dst).
@@ -511,7 +557,8 @@ int sws_receive_slice(SwsContext *c, unsigned int slice_start, · 64 unmodified lines
511	557	unsigned int slice_height);
512	558
513	559	/**
514		− * Get the alignment required for slices
	560	+ * Get the alignment required for slices. Requires a context that has been
	561	+ * previously been initialized with sws_init_context().
515	562	*
516	563	* @param c The scaling context
517	564	* @return alignment required for output slices requested with sws_receive_slice().

libswscale/swscale_internal.h+8−1

@@ -26,6 +26,7 @@ · 25 unmodified lines
26	26
27	27	#include "config.h"
28	28	#include "swscale.h"
	29	+#include "graph.h"
29	30
30	31	#include "libavutil/avassert.h"
31	32	#include "libavutil/common.h"
@@ -48,6 +49,8 @@ · 16 unmodified lines
48	49
49	50	#define MAX_FILTER_SIZE SWS_MAX_FILTER_SIZE
50	51
	52	+#define SWS_MAX_THREADS 8192 /* sanity clamp */
	53	+
51	54	#if HAVE_BIGENDIAN
52	55	#define ALT32_CORR (-1)
53	56	#else
@@ -323,6 +326,9 @@ struct SwsInternal { · 269 unmodified lines
323	326	int *slice_err;
324	327	int nb_slice_ctx;
325	328
	329	+ /* Scaling graph, reinitialized dynamically as needed. */
	330	+ SwsGraph graph[2]; / top, bottom fields */
	331	+
326	332	// values passed to current sws_receive_slice() call
327	333	int dst_slice_start;
328	334	int dst_slice_height;
@@ -663,6 +669,7 @@ struct SwsInternal { · 334 unmodified lines
663	669	unsigned int dst_slice_align;
664	670	atomic_int stride_unaligned_warned;
665	671	atomic_int data_unaligned_warned;
	672	+ int color_conversion_warned;
666	673
667	674	Half2FloatTables *h2f_tables;
668	675	};
@@ -674,7 +681,7 @@ static_assert(offsetof(SwsInternal, redDither) + DITHER32_INT == offsetof(SwsInt · 5 unmodified lines
674	681	#if ARCH_X86_64
675	682	/* x86 yuv2gbrp uses the SwsInternal for yuv coefficients
676	683	if struct offsets change the asm needs to be updated too */
677		−static_assert(offsetof(SwsInternal, yuv2rgb_y_offset) == 40316,
	684	+static_assert(offsetof(SwsInternal, yuv2rgb_y_offset) == 40332,
678	685	"yuv2rgb_y_offset must be updated in x86 asm");
679	686	#endif
680	687

libswscale/utils.c+4−0

@@ -61,6 +61,7 @@ · 60 unmodified lines
61	61	#include "swscale.h"
62	62	#include "swscale_internal.h"
63	63	#include "utils.h"
	64	+#include "graph.h"
64	65
65	66	typedef struct FormatEntry {
66	67	uint8_t is_supported_in :1;
@@ -2450,6 +2451,9 @@ void sws_freeContext(SwsContext *sws) · 2383 unmodified lines
2450	2451	if (!c)
2451	2452	return;
2452	2453
	2454	+ for (i = 0; i < FF_ARRAY_ELEMS(c->graph); i++)
	2455	+ sws_graph_free(&c->graph[i]);
	2456	+
2453	2457	for (i = 0; i < c->nb_slice_ctx; i++)
2454	2458	sws_freeContext(c->slice_ctx[i]);
2455	2459	av_freep(&c->slice_ctx);

libswscale/version.h+1−1

@@ -28,7 +28,7 @@ · 27 unmodified lines
28	28
29	29	#include "version_major.h"
30	30
31		−#define LIBSWSCALE_VERSION_MINOR 11
	31	+#define LIBSWSCALE_VERSION_MINOR 12
32	32	#define LIBSWSCALE_VERSION_MICRO 100
33	33
34	34	#define LIBSWSCALE_VERSION_INT AV_VERSION_INT(LIBSWSCALE_VERSION_MAJOR, \

libswscale/x86/output.asm+1−1

@@ -582,7 +582,7 @@ yuv2nv12cX_fn yuv2nv21 · 581 unmodified lines
582	582
583	583	%if ARCH_X86_64
584	584	struc SwsInternal
585		− .padding: resb 40316 ; offsetof(SwsInternal, yuv2rgb_y_offset)
	585	+ .padding: resb 40332 ; offsetof(SwsInternal, yuv2rgb_y_offset)
586	586	.yuv2rgb_y_offset: resd 1
587	587	.yuv2rgb_y_coeff: resd 1
588	588	.yuv2rgb_v2r_coeff: resd 1

swscale: introduce new, dynamic scaling API

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code