History

swscale: introduce new, dynamic scaling API

As part of a larger, ongoing effort to modernize and partially rewrite libswscale, it was decided and generally agreed upon to introduce a new public API for libswscale. This API is designed to be less stateful, more explicitly defined, and considerably easier to use than the existing one. Most of the API work has been already accomplished in the previous commits, this commit merely introduces the ability to use sws_scale_frame() dynamically, without prior sws_init_context() calls. Instead, the new API takes frame properties from the frames themselves, and the implementation is based on the new SwsGraph API, which we simply reinitialize as needed. This high-level wrapper also recreates the logic that used to live inside vf_scale for scaling interlaced frames, enabling it to be reused more easily by end users. Finally, this function is designed to simply copy refs directly when nothing needs to be done, substantially improving throughput of the noop fast path.

Niklas Haas committed 1 year ago

Commit 2a091d41 parent bf738417 files · +276 −30Browse files

Unified · Split

7 files
  1. doc/APIchanges +4 −0
  2. libswscale/swscale.c +190 −6
  3. libswscale/swscale.h +68 −21
  4. libswscale/swscale_internal.h +8 −1
  5. libswscale/utils.c +4 −0
  6. libswscale/version.h +1 −1
  7. libswscale/x86/output.asm +1 −1
doc/APIchanges+4−0
@@ -2,6 +2,10 @@ The last version increases of all libraries were on 2024-03-07 · 1 unmodified line
2 2
3API changes, most recent first:3API changes, most recent first:
4 4
52024-11-25 - xxxxxxxxxx - lsws 8.12.100 - swscale.h
6 Allow using sws_frame_scale() dynamically, without first initializing the
7 SwsContext. Deprecate sws_init_context(). Add sws_frame_setup() instead.
8
52024-11-25 - xxxxxxxxxx - lsws 8.11.100 - swscale.h92024-11-25 - xxxxxxxxxx - lsws 8.11.100 - swscale.h
6 Replace #define-based SWS_* flags by enum SwsFlags.10 Replace #define-based SWS_* flags by enum SwsFlags.
7 11
libswscale/swscale.c+190−6
@@ -1219,21 +1219,205 @@ int sws_receive_slice(SwsContext *sws, unsigned int slice_start, · 1218 unmodified lines
1219 dst, c->frame_dst->linesize, slice_start, slice_height);1219 dst, c->frame_dst->linesize, slice_start, slice_height);
1220}1220}
1221 1221
1222static void get_frame_pointers(const AVFrame *frame, uint8_t *data[4],
1223 int linesize[4], int field)
1224{
1225 for (int i = 0; i < 4; i++) {
1226 data[i] = frame->data[i];
1227 linesize[i] = frame->linesize[i];
1228 }
1229
1230 if (!(frame->flags & AV_FRAME_FLAG_INTERLACED)) {
1231 av_assert1(!field);
1232 return;
1233 }
1234
1235 if (field == FIELD_BOTTOM) {
1236 /* Odd rows, offset by one line */
1237 const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(frame->format);
1238 for (int i = 0; i < 4; i++) {
1239 data[i] += linesize[i];
1240 if (desc->flags & AV_PIX_FMT_FLAG_PAL)
1241 break;
1242 }
1243 }
1244
1245 /* Take only every second line */
1246 for (int i = 0; i < 4; i++)
1247 linesize[i] <<= 1;
1248}
1249
1250/* Subset of av_frame_ref() that only references (video) data buffers */
1251static int frame_ref(AVFrame *dst, const AVFrame *src)
1252{
1253 /* ref the buffers */
1254 for (int i = 0; i < FF_ARRAY_ELEMS(src->buf); i++) {
1255 if (!src->buf[i])
1256 continue;
1257 dst->buf[i] = av_buffer_ref(src->buf[i]);
1258 if (!dst->buf[i])
1259 return AVERROR(ENOMEM);
1260 }
1261
1262 memcpy(dst->data, src->data, sizeof(src->data));
1263 memcpy(dst->linesize, src->linesize, sizeof(src->linesize));
1264 return 0;
1265}
1266
1222int sws_scale_frame(SwsContext *sws, AVFrame *dst, const AVFrame *src)1267int sws_scale_frame(SwsContext *sws, AVFrame *dst, const AVFrame *src)
1223{1268{
1224 int ret;1269 int ret;
1270 SwsInternal *c = sws_internal(sws);
1271 if (!src || !dst)
1272 return AVERROR(EINVAL);
1273
1274 if (c->frame_src) {
1275 /* Context has been initialized with explicit values, fall back to
1276 * legacy API */
1277 ret = sws_frame_start(sws, dst, src);
1278 if (ret < 0)
1279 return ret;
1280
1281 ret = sws_send_slice(sws, 0, src->height);
1282 if (ret >= 0)
1283 ret = sws_receive_slice(sws, 0, dst->height);
1225 1284
1226 ret = sws_frame_start(sws, dst, src);1285 sws_frame_end(sws);
1286
1287 return ret;
1288 }
1289
1290 ret = sws_frame_setup(sws, dst, src);
1227 if (ret < 0)1291 if (ret < 0)
1228 return ret;1292 return ret;
1229 1293
1230 ret = sws_send_slice(sws, 0, src->height);1294 if (!src->data[0])
1231 if (ret >= 0)1295 return 0;
1232 ret = sws_receive_slice(sws, 0, dst->height);
1233 1296
1234 sws_frame_end(sws);1297 if (c->graph[FIELD_TOP]->noop &&
1298 (!c->graph[FIELD_BOTTOM] || c->graph[FIELD_BOTTOM]->noop) &&
1299 src->buf[0] && !dst->buf[0] && !dst->data[0])
1300 {
1301 /* Lightweight refcopy */
1302 ret = frame_ref(dst, src);
1303 if (ret < 0)
1304 return ret;
1305 } else {
1306 if (!dst->data[0]) {
1307 ret = av_frame_get_buffer(dst, 0);
1308 if (ret < 0)
1309 return ret;
1310 }
1235 1311
1236 return ret;1312 for (int field = 0; field < 2; field++) {
1313 SwsGraph *graph = c->graph[field];
1314 uint8_t *dst_data[4], *src_data[4];
1315 int dst_linesize[4], src_linesize[4];
1316 get_frame_pointers(dst, dst_data, dst_linesize, field);
1317 get_frame_pointers(src, src_data, src_linesize, field);
1318 sws_graph_run(graph, dst_data, dst_linesize,
1319 (const uint8_t **) src_data, src_linesize);
1320 if (!graph->dst.interlaced)
1321 break;
1322 }
1323 }
1324
1325 return 0;
1326}
1327
1328static int validate_params(SwsContext *ctx)
1329{
1330#define VALIDATE(field, min, max) \
1331 if (ctx->field < min || ctx->field > max) { \
1332 av_log(ctx, AV_LOG_ERROR, "'%s' (%d) out of range [%d, %d]\n", \
1333 #field, (int) ctx->field, min, max); \
1334 return AVERROR(EINVAL); \
1335 }
1336
1337 VALIDATE(threads, 0, SWS_MAX_THREADS);
1338 VALIDATE(dither, 0, SWS_DITHER_NB - 1)
1339 VALIDATE(alpha_blend, 0, SWS_ALPHA_BLEND_NB - 1)
1340 return 0;
1341}
1342
1343int sws_frame_setup(SwsContext *ctx, const AVFrame *dst, const AVFrame *src)
1344{
1345 SwsInternal *s = sws_internal(ctx);
1346 const char *err_msg;
1347 int ret;
1348
1349 if (!src || !dst)
1350 return AVERROR(EINVAL);
1351 if ((ret = validate_params(ctx)) < 0)
1352 return ret;
1353
1354 for (int field = 0; field < 2; field++) {
1355 SwsFormat src_fmt = ff_fmt_from_frame(src, field);
1356 SwsFormat dst_fmt = ff_fmt_from_frame(dst, field);
1357
1358 if ((src->flags ^ dst->flags) & AV_FRAME_FLAG_INTERLACED) {
1359 err_msg = "Cannot convert interlaced to progressive frames or vice versa.\n";
1360 ret = AVERROR(EINVAL);
1361 goto fail;
1362 }
1363
1364 /* TODO: remove once implemented */
1365 if ((dst_fmt.prim != src_fmt.prim || dst_fmt.trc != src_fmt.trc) &&
1366 !s->color_conversion_warned)
1367 {
1368 av_log(ctx, AV_LOG_WARNING, "Conversions between different primaries / "
1369 "transfer functions are not currently implemented, expect "
1370 "wrong results.\n");
1371 s->color_conversion_warned = 1;
1372 }
1373
1374 if (!ff_test_fmt(&src_fmt, 0)) {
1375 err_msg = "Unsupported input";
1376 ret = AVERROR(ENOTSUP);
1377 goto fail;
1378 }
1379
1380 if (!ff_test_fmt(&dst_fmt, 1)) {
1381 err_msg = "Unsupported output";
1382 ret = AVERROR(ENOTSUP);
1383 goto fail;
1384 }
1385
1386 ret = sws_graph_reinit(ctx, &dst_fmt, &src_fmt, field, &s->graph[field]);
1387 if (ret < 0) {
1388 err_msg = "Failed initializing scaling graph";
1389 goto fail;
1390 }
1391
1392 if (s->graph[field]->incomplete && ctx->flags & SWS_STRICT) {
1393 err_msg = "Incomplete scaling graph";
1394 ret = AVERROR(EINVAL);
1395 goto fail;
1396 }
1397
1398 if (!src_fmt.interlaced) {
1399 sws_graph_free(&s->graph[FIELD_BOTTOM]);
1400 break;
1401 }
1402
1403 continue;
1404
1405 fail:
1406 av_log(ctx, AV_LOG_ERROR, "%s (%s): fmt:%s csp:%s prim:%s trc:%s ->"
1407 " fmt:%s csp:%s prim:%s trc:%s\n",
1408 err_msg, av_err2str(ret),
1409 av_get_pix_fmt_name(src_fmt.format), av_color_space_name(src_fmt.csp),
1410 av_color_primaries_name(src_fmt.prim), av_color_transfer_name(src_fmt.trc),
1411 av_get_pix_fmt_name(dst_fmt.format), av_color_space_name(dst_fmt.csp),
1412 av_color_primaries_name(dst_fmt.prim), av_color_transfer_name(dst_fmt.trc));
1413
1414 for (int i = 0; i < FF_ARRAY_ELEMS(s->graph); i++)
1415 sws_graph_free(&s->graph[i]);
1416
1417 return ret;
1418 }
1419
1420 return 0;
1237}1421}
1238 1422
1239/**1423/**
libswscale/swscale.h+68−21
@@ -107,6 +107,12 @@ typedef enum SwsFlags { · 106 unmodified lines
107 SWS_LANCZOS = 1 << 9, ///< 3-tap sinc/sinc107 SWS_LANCZOS = 1 << 9, ///< 3-tap sinc/sinc
108 SWS_SPLINE = 1 << 10, ///< cubic Keys spline108 SWS_SPLINE = 1 << 10, ///< cubic Keys spline
109 109
110 /**
111 * Return an error on underspecified conversions. Without this flag,
112 * unspecified fields are defaulted to sensible values.
113 */
114 SWS_STRICT = 1 << 11,
115
110 /**116 /**
111 * Emit verbose log of scaling parameters.117 * Emit verbose log of scaling parameters.
112 */118 */
@@ -204,7 +210,10 @@ typedef struct SwsContext { · 91 unmodified lines
204 int gamma_flag;210 int gamma_flag;
205 211
206 /**212 /**
207 * Frame property overrides.213 * Deprecated frame property overrides, for the legacy API only.
214 *
215 * Ignored by sws_scale_frame() when used in dynamic mode, in which
216 * case all properties are instead taken from the frame directly.
208 */217 */
209 int src_w, src_h; ///< Width and height of the source frame218 int src_w, src_h; ///< Width and height of the source frame
210 int dst_w, dst_h; ///< Width and height of the destination frame219 int dst_w, dst_h; ///< Width and height of the destination frame
@@ -216,6 +225,8 @@ typedef struct SwsContext { · 5 unmodified lines
216 int src_h_chr_pos; ///< Source horizontal chroma position225 int src_h_chr_pos; ///< Source horizontal chroma position
217 int dst_v_chr_pos; ///< Destination vertical chroma position226 int dst_v_chr_pos; ///< Destination vertical chroma position
218 int dst_h_chr_pos; ///< Destination horizontal chroma position227 int dst_h_chr_pos; ///< Destination horizontal chroma position
228
229 /* Remember to add new fields to graph.c:opts_equal() */
219} SwsContext;230} SwsContext;
220 231
221/**232/**
@@ -284,12 +295,57 @@ int sws_test_transfer(enum AVColorTransferCharacteristic trc, int output); · 62 unmodified lines
284 */295 */
285int sws_test_frame(const AVFrame *frame, int output);296int sws_test_frame(const AVFrame *frame, int output);
286 297
298/**
299 * Like `sws_scale_frame`, but without actually scaling. It will instead
300 * merely initialize internal state that *would* be required to perform the
301 * operation, as well as returning the correct error code for unsupported
302 * frame combinations.
303 *
304 * @param ctx The scaling context.
305 * @param dst The destination frame to consider.
306 * @param src The source frame to consider.
307 * @return 0 on success, a negative AVERROR code on failure.
308 */
309int sws_frame_setup(SwsContext *ctx, const AVFrame *dst, const AVFrame *src);
310
311/********************
312 * Main scaling API *
313 ********************/
314
287/**315/**
288 * Check if a given conversion is a noop. Returns a positive integer if316 * Check if a given conversion is a noop. Returns a positive integer if
289 * no operation needs to be performed, 0 otherwise.317 * no operation needs to be performed, 0 otherwise.
290 */318 */
291int sws_is_noop(const AVFrame *dst, const AVFrame *src);319int sws_is_noop(const AVFrame *dst, const AVFrame *src);
292 320
321/**
322 * Scale source data from `src` and write the output to `dst`.
323 *
324 * This function can be used directly on an allocated context, without setting
325 * up any frame properties or calling `sws_init_context()`. Such usage is fully
326 * dynamic and does not require reallocation if the frame properties change.
327 *
328 * Alternatively, this function can be called on a context that has been
329 * explicitly initialized. However, this is provided only for backwards
330 * compatibility. In this usage mode, all frame properties must be correctly
331 * set at init time, and may no longer change after initialization.
332 *
333 * @param ctx The scaling context.
334 * @param dst The destination frame. The data buffers may either be already
335 * allocated by the caller or left clear, in which case they will
336 * be allocated by the scaler. The latter may have performance
337 * advantages - e.g. in certain cases some (or all) output planes
338 * may be references to input planes, rather than copies.
339 * @param src The source frame. If the data buffers are set to NULL, then
340 * this function behaves identically to `sws_frame_setup`.
341 * @return 0 on success, a negative AVERROR code on failure.
342 */
343int sws_scale_frame(SwsContext *c, AVFrame *dst, const AVFrame *src);
344
345/*************************
346 * Legacy (stateful) API *
347 *************************/
348
293#define SWS_SRC_V_CHR_DROP_MASK 0x30000349#define SWS_SRC_V_CHR_DROP_MASK 0x30000
294#define SWS_SRC_V_CHR_DROP_SHIFT 16350#define SWS_SRC_V_CHR_DROP_SHIFT 16
295 351
@@ -352,6 +408,11 @@ int sws_isSupportedEndiannessConversion(enum AVPixelFormat pix_fmt); · 56 unmodified lines
352/**408/**
353 * Initialize the swscaler context sws_context.409 * Initialize the swscaler context sws_context.
354 *410 *
411 * This function is considered deprecated, and provided only for backwards
412 * compatibility with sws_scale() and sws_start_frame(). The preferred way to
413 * use libswscale is to set all frame properties correctly and call
414 * sws_scale_frame() directly, without explicitly initializing the context.
415 *
355 * @return zero or positive value on success, a negative value on416 * @return zero or positive value on success, a negative value on
356 * error417 * error
357 */418 */
@@ -393,7 +454,8 @@ SwsContext *sws_getContext(int srcW, int srcH, enum AVPixelFormat srcFormat, · 35 unmodified lines
393/**454/**
394 * Scale the image slice in srcSlice and put the resulting scaled455 * Scale the image slice in srcSlice and put the resulting scaled
395 * slice in the image in dst. A slice is a sequence of consecutive456 * slice in the image in dst. A slice is a sequence of consecutive
396 * rows in an image.457 * rows in an image. Requires a context that has been previously
458 * been initialized with sws_init_context().
397 *459 *
398 * Slices have to be provided in sequential order, either in460 * Slices have to be provided in sequential order, either in
399 * top-bottom or bottom-top order. If slices are provided in461 * top-bottom or bottom-top order. If slices are provided in
@@ -420,27 +482,11 @@ int sws_scale(SwsContext *c, const uint8_t *const srcSlice[], · 20 unmodified lines
420 const int srcStride[], int srcSliceY, int srcSliceH,482 const int srcStride[], int srcSliceY, int srcSliceH,
421 uint8_t *const dst[], const int dstStride[]);483 uint8_t *const dst[], const int dstStride[]);
422 484
423/**
424 * Scale source data from src and write the output to dst.
425 *
426 * This is merely a convenience wrapper around
427 * - sws_frame_start()
428 * - sws_send_slice(0, src->height)
429 * - sws_receive_slice(0, dst->height)
430 * - sws_frame_end()
431 *
432 * @param c The scaling context
433 * @param dst The destination frame. See documentation for sws_frame_start() for
434 * more details.
435 * @param src The source frame.
436 *
437 * @return 0 on success, a negative AVERROR code on failure
438 */
439int sws_scale_frame(SwsContext *c, AVFrame *dst, const AVFrame *src);
440
441/**485/**
442 * Initialize the scaling process for a given pair of source/destination frames.486 * Initialize the scaling process for a given pair of source/destination frames.
443 * Must be called before any calls to sws_send_slice() and sws_receive_slice().487 * Must be called before any calls to sws_send_slice() and sws_receive_slice().
488 * Requires a context that has been previously been initialized with
489 * sws_init_context().
444 *490 *
445 * This function will retain references to src and dst, so they must both use491 * This function will retain references to src and dst, so they must both use
446 * refcounted buffers (if allocated by the caller, in case of dst).492 * refcounted buffers (if allocated by the caller, in case of dst).
@@ -511,7 +557,8 @@ int sws_receive_slice(SwsContext *c, unsigned int slice_start, · 64 unmodified lines
511 unsigned int slice_height);557 unsigned int slice_height);
512 558
513/**559/**
514 * Get the alignment required for slices560 * Get the alignment required for slices. Requires a context that has been
561 * previously been initialized with sws_init_context().
515 *562 *
516 * @param c The scaling context563 * @param c The scaling context
517 * @return alignment required for output slices requested with sws_receive_slice().564 * @return alignment required for output slices requested with sws_receive_slice().
libswscale/swscale_internal.h+8−1
@@ -26,6 +26,7 @@ · 25 unmodified lines
26 26
27#include "config.h"27#include "config.h"
28#include "swscale.h"28#include "swscale.h"
29#include "graph.h"
29 30
30#include "libavutil/avassert.h"31#include "libavutil/avassert.h"
31#include "libavutil/common.h"32#include "libavutil/common.h"
@@ -48,6 +49,8 @@ · 16 unmodified lines
48 49
49#define MAX_FILTER_SIZE SWS_MAX_FILTER_SIZE50#define MAX_FILTER_SIZE SWS_MAX_FILTER_SIZE
50 51
52#define SWS_MAX_THREADS 8192 /* sanity clamp */
53
51#if HAVE_BIGENDIAN54#if HAVE_BIGENDIAN
52#define ALT32_CORR (-1)55#define ALT32_CORR (-1)
53#else56#else
@@ -323,6 +326,9 @@ struct SwsInternal { · 269 unmodified lines
323 int *slice_err;326 int *slice_err;
324 int nb_slice_ctx;327 int nb_slice_ctx;
325 328
329 /* Scaling graph, reinitialized dynamically as needed. */
330 SwsGraph *graph[2]; /* top, bottom fields */
331
326 // values passed to current sws_receive_slice() call332 // values passed to current sws_receive_slice() call
327 int dst_slice_start;333 int dst_slice_start;
328 int dst_slice_height;334 int dst_slice_height;
@@ -663,6 +669,7 @@ struct SwsInternal { · 334 unmodified lines
663 unsigned int dst_slice_align;669 unsigned int dst_slice_align;
664 atomic_int stride_unaligned_warned;670 atomic_int stride_unaligned_warned;
665 atomic_int data_unaligned_warned;671 atomic_int data_unaligned_warned;
672 int color_conversion_warned;
666 673
667 Half2FloatTables *h2f_tables;674 Half2FloatTables *h2f_tables;
668};675};
@@ -674,7 +681,7 @@ static_assert(offsetof(SwsInternal, redDither) + DITHER32_INT == offsetof(SwsInt · 5 unmodified lines
674#if ARCH_X86_64681#if ARCH_X86_64
675/* x86 yuv2gbrp uses the SwsInternal for yuv coefficients682/* x86 yuv2gbrp uses the SwsInternal for yuv coefficients
676 if struct offsets change the asm needs to be updated too */683 if struct offsets change the asm needs to be updated too */
677static_assert(offsetof(SwsInternal, yuv2rgb_y_offset) == 40316,684static_assert(offsetof(SwsInternal, yuv2rgb_y_offset) == 40332,
678 "yuv2rgb_y_offset must be updated in x86 asm");685 "yuv2rgb_y_offset must be updated in x86 asm");
679#endif686#endif
680 687
libswscale/utils.c+4−0
@@ -61,6 +61,7 @@ · 60 unmodified lines
61#include "swscale.h"61#include "swscale.h"
62#include "swscale_internal.h"62#include "swscale_internal.h"
63#include "utils.h"63#include "utils.h"
64#include "graph.h"
64 65
65typedef struct FormatEntry {66typedef struct FormatEntry {
66 uint8_t is_supported_in :1;67 uint8_t is_supported_in :1;
@@ -2450,6 +2451,9 @@ void sws_freeContext(SwsContext *sws) · 2383 unmodified lines
2450 if (!c)2451 if (!c)
2451 return;2452 return;
2452 2453
2454 for (i = 0; i < FF_ARRAY_ELEMS(c->graph); i++)
2455 sws_graph_free(&c->graph[i]);
2456
2453 for (i = 0; i < c->nb_slice_ctx; i++)2457 for (i = 0; i < c->nb_slice_ctx; i++)
2454 sws_freeContext(c->slice_ctx[i]);2458 sws_freeContext(c->slice_ctx[i]);
2455 av_freep(&c->slice_ctx);2459 av_freep(&c->slice_ctx);
libswscale/version.h+1−1
@@ -28,7 +28,7 @@ · 27 unmodified lines
28 28
29#include "version_major.h"29#include "version_major.h"
30 30
31#define LIBSWSCALE_VERSION_MINOR 1131#define LIBSWSCALE_VERSION_MINOR 12
32#define LIBSWSCALE_VERSION_MICRO 10032#define LIBSWSCALE_VERSION_MICRO 100
33 33
34#define LIBSWSCALE_VERSION_INT AV_VERSION_INT(LIBSWSCALE_VERSION_MAJOR, \34#define LIBSWSCALE_VERSION_INT AV_VERSION_INT(LIBSWSCALE_VERSION_MAJOR, \
libswscale/x86/output.asm+1−1
@@ -582,7 +582,7 @@ yuv2nv12cX_fn yuv2nv21 · 581 unmodified lines
582 582
583%if ARCH_X86_64583%if ARCH_X86_64
584struc SwsInternal584struc SwsInternal
585 .padding: resb 40316 ; offsetof(SwsInternal, yuv2rgb_y_offset)585 .padding: resb 40332 ; offsetof(SwsInternal, yuv2rgb_y_offset)
586 .yuv2rgb_y_offset: resd 1586 .yuv2rgb_y_offset: resd 1
587 .yuv2rgb_y_coeff: resd 1587 .yuv2rgb_y_coeff: resd 1
588 .yuv2rgb_v2r_coeff: resd 1588 .yuv2rgb_v2r_coeff: resd 1

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t