History

avfilter/vf_colordetect: add new color range detection filter

This filter can detect various properties about the image, including whether or not there are out-of-range values, or whether the input appears to use straight or premultiplied alpha. Of course, these can only be heuristics, with "undetermined" as the base case. While we can definitely prove the existence of full range or straight alpha colors, we can never infer the opposite.

Niklas Haas committed 1 year ago

Commit 545f7211 parent 8cdb47e5 files · +443 −0Browse files

Unified · Split

5 files
  1. doc/filters.texi +27 −0
  2. libavfilter/Makefile +1 −0
  3. libavfilter/allfilters.c +1 −0
  4. libavfilter/vf_colordetect.c +265 −0
  5. libavfilter/vf_colordetect.h +149 −0
doc/filters.texi+27−0
@@ -9753,6 +9753,33 @@ colorchannelmixer=.393:.769:.189:0:.349:.686:.168:0:.272:.534:.131 · 9752 unmodified lines
97539753
97549754 This filter supports the all above options as @ref{commands}.
97559755
9756+@section colordetect
9757+Analyze the video frames to determine the effective value range and alpha
9758+mode.
9759+
9760+The filter accepts the following options:
9761+
9762+@table @option
9763+@item mode
9764+Set of properties to detect. Unavailable properties, such as alpha mode for
9765+an input image without an alpha channel, will be ignored automatically.
9766+
9767+Accepts a combination of the following flags:
9768+
9769+@table @samp
9770+@item color_range
9771+Detect if the source countains luma pixels outside the limited (MPEG) range,
9772+which indicates that this is a full range YUV source.
9773+@item alpha_mode
9774+Detect if the source contains color values above the alpha channel, which
9775+indicates that the alpha channel is independent (straight), rather than
9776+premultiplied.
9777+@item all
9778+Enable detection of all of the above properties. This is the default.
9779+@end table
9780+
9781+@end table
9782+
97569783 @section colorize
97579784 Overlay a solid color on the video stream.
97589785
libavfilter/Makefile+1−0
@@ -237,6 +237,7 @@ OBJS-$(CONFIG_COLORBALANCE_FILTER) += vf_colorbalance.o · 236 unmodified lines
237237 OBJS-$(CONFIG_COLORCHANNELMIXER_FILTER) += vf_colorchannelmixer.o
238238 OBJS-$(CONFIG_COLORCONTRAST_FILTER) += vf_colorcontrast.o
239239 OBJS-$(CONFIG_COLORCORRECT_FILTER) += vf_colorcorrect.o
240+OBJS-$(CONFIG_COLORDETECT_FILTER) += vf_colordetect.o
240241 OBJS-$(CONFIG_COLORIZE_FILTER) += vf_colorize.o
241242 OBJS-$(CONFIG_COLORKEY_FILTER) += vf_colorkey.o
242243 OBJS-$(CONFIG_COLORKEY_OPENCL_FILTER) += vf_colorkey_opencl.o opencl.o \
libavfilter/allfilters.c+1−0
@@ -218,6 +218,7 @@ extern const FFFilter ff_vf_colorbalance; · 217 unmodified lines
218218 extern const FFFilter ff_vf_colorchannelmixer;
219219 extern const FFFilter ff_vf_colorcontrast;
220220 extern const FFFilter ff_vf_colorcorrect;
221+extern const FFFilter ff_vf_colordetect;
221222 extern const FFFilter ff_vf_colorize;
222223 extern const FFFilter ff_vf_colorkey;
223224 extern const FFFilter ff_vf_colorkey_opencl;
libavfilter/vf_colordetect.c+265−0
@@ -0,0 +1,265 @@
1+/*
2+ * Copyright (c) 2025 Niklas Haas
3+ *
4+ * This file is part of FFmpeg.
5+ *
6+ * FFmpeg is free software; you can redistribute it and/or
7+ * modify it under the terms of the GNU Lesser General Public
8+ * License as published by the Free Software Foundation; either
9+ * version 2.1 of the License, or (at your option) any later version.
10+ *
11+ * FFmpeg is distributed in the hope that it will be useful,
12+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
13+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14+ * Lesser General Public License for more details.
15+ *
16+ * You should have received a copy of the GNU Lesser General Public
17+ * License along with FFmpeg; if not, write to the Free Software
18+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19+ */
20+
21+/**
22+ * @file
23+ * Video color space detector, tries to auto-detect YUV range and alpha mode.
24+ */
25+
26+#include <stdbool.h>
27+#include <stdatomic.h>
28+
29+#include "config.h"
30+
31+#include "libavutil/mem.h"
32+#include "libavutil/opt.h"
33+#include "libavutil/pixdesc.h"
34+
35+#include "avfilter.h"
36+#include "filters.h"
37+#include "formats.h"
38+#include "video.h"
39+
40+#include "vf_colordetect.h"
41+
42+enum AlphaMode {
43+ ALPHA_NONE = -1,
44+ ALPHA_UNDETERMINED = 0,
45+ ALPHA_STRAIGHT,
46+ /* No way to positively identify premultiplied alpha */
47+};
48+
49+enum ColorDetectMode {
50+ COLOR_DETECT_COLOR_RANGE = 1 << 0,
51+ COLOR_DETECT_ALPHA_MODE = 1 << 1,
52+};
53+
54+typedef struct ColorDetectContext {
55+ const AVClass *class;
56+ FFColorDetectDSPContext dsp;
57+ unsigned mode;
58+
59+ const AVPixFmtDescriptor *desc;
60+ int nb_threads;
61+ int depth;
62+ int idx_a;
63+ int mpeg_min;
64+ int mpeg_max;
65+
66+ atomic_int detected_range; // enum AVColorRange
67+ atomic_int detected_alpha; // enum AlphaMode
68+} ColorDetectContext;
69+
70+#define OFFSET(x) offsetof(ColorDetectContext, x)
71+#define FLAGS AV_OPT_FLAG_VIDEO_PARAM|AV_OPT_FLAG_FILTERING_PARAM
72+
73+static const AVOption colordetect_options[] = {
74+ { "mode", "Image properties to detect", OFFSET(mode), AV_OPT_TYPE_FLAGS, {.i64 = -1}, 0, UINT_MAX, FLAGS, .unit = "mode" },
75+ { "color_range", "Detect (YUV) color range", 0, AV_OPT_TYPE_CONST, {.i64 = COLOR_DETECT_COLOR_RANGE}, 0, 0, FLAGS, .unit = "mode" },
76+ { "alpha_mode", "Detect alpha mode", 0, AV_OPT_TYPE_CONST, {.i64 = COLOR_DETECT_ALPHA_MODE }, 0, 0, FLAGS, .unit = "mode" },
77+ { "all", "Detect all supported properties", 0, AV_OPT_TYPE_CONST, {.i64 = -1}, 0, 0, FLAGS, .unit = "mode" },
78+ { NULL }
79+};
80+
81+AVFILTER_DEFINE_CLASS(colordetect);
82+
83+static int query_format(const AVFilterContext *ctx,
84+ AVFilterFormatsConfig **cfg_in,
85+ AVFilterFormatsConfig **cfg_out)
86+{
87+ int want_flags = AV_PIX_FMT_FLAG_PLANAR;
88+ int reject_flags = AV_PIX_FMT_FLAG_PAL | AV_PIX_FMT_FLAG_HWACCEL |
89+ AV_PIX_FMT_FLAG_BITSTREAM | AV_PIX_FMT_FLAG_FLOAT |
90+ AV_PIX_FMT_FLAG_BAYER | AV_PIX_FMT_FLAG_XYZ;
91+
92+ if (HAVE_BIGENDIAN) {
93+ want_flags |= AV_PIX_FMT_FLAG_BE;
94+ } else {
95+ reject_flags |= AV_PIX_FMT_FLAG_BE;
96+ }
97+
98+ AVFilterFormats *formats = ff_formats_pixdesc_filter(want_flags, reject_flags);
99+ return ff_set_common_formats2(ctx, cfg_in, cfg_out, formats);
100+}
101+
102+static int config_input(AVFilterLink *inlink)
103+{
104+ AVFilterContext *ctx = inlink->dst;
105+ ColorDetectContext *s = ctx->priv;
106+ const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(inlink->format);
107+ const int depth = desc->comp[0].depth;
108+ const int mpeg_min = 16 << (depth - 8);
109+ const int mpeg_max = 235 << (depth - 8);
110+ if (depth > 16) /* not currently possible; prevent future bugs */
111+ return AVERROR(ENOTSUP);
112+
113+ s->desc = desc;
114+ s->depth = depth;
115+ s->mpeg_min = mpeg_min;
116+ s->mpeg_max = mpeg_max;
117+ s->nb_threads = ff_filter_get_nb_threads(ctx);
118+
119+ if (desc->flags & AV_PIX_FMT_FLAG_RGB) {
120+ atomic_init(&s->detected_range, AVCOL_RANGE_JPEG);
121+ } else {
122+ atomic_init(&s->detected_range, AVCOL_RANGE_UNSPECIFIED);
123+ }
124+
125+ if (desc->flags & AV_PIX_FMT_FLAG_ALPHA) {
126+ s->idx_a = desc->comp[desc->nb_components - 1].plane;
127+ atomic_init(&s->detected_alpha, ALPHA_UNDETERMINED);
128+ } else {
129+ atomic_init(&s->detected_alpha, ALPHA_NONE);
130+ }
131+
132+ ff_color_detect_dsp_init(&s->dsp, depth, inlink->color_range);
133+ return 0;
134+}
135+
136+static int detect_range(AVFilterContext *ctx, void *arg,
137+ int jobnr, int nb_jobs)
138+{
139+ ColorDetectContext *s = ctx->priv;
140+ const AVFrame *in = arg;
141+ const ptrdiff_t stride = in->linesize[0];
142+ const int y_start = (in->height * jobnr) / nb_jobs;
143+ const int y_end = (in->height * (jobnr + 1)) / nb_jobs;
144+ const int h_slice = y_end - y_start;
145+
146+ if (s->dsp.detect_range(in->data[0] + y_start * stride, stride,
147+ in->width, h_slice, s->mpeg_min, s->mpeg_max))
148+ atomic_store(&s->detected_range, AVCOL_RANGE_JPEG);
149+
150+ return 0;
151+}
152+
153+static int detect_alpha(AVFilterContext *ctx, void *arg,
154+ int jobnr, int nb_jobs)
155+{
156+ ColorDetectContext *s = ctx->priv;
157+ const AVFrame *in = arg;
158+ const int w = in->width;
159+ const int h = in->height;
160+ const int y_start = (h * jobnr) / nb_jobs;
161+ const int y_end = (h * (jobnr + 1)) / nb_jobs;
162+ const int h_slice = y_end - y_start;
163+
164+ const int nb_planes = (s->desc->flags & AV_PIX_FMT_FLAG_RGB) ? 3 : 1;
165+ const ptrdiff_t alpha_stride = in->linesize[s->idx_a];
166+ const uint8_t *alpha = in->data[s->idx_a] + y_start * alpha_stride;
167+
168+ /**
169+ * To check if a value is out of range, we need to compare the color value
170+ * against the maximum possible color for a given alpha value.
171+ * x > ((mpeg_max - mpeg_min) / pixel_max) * a + mpeg_min
172+ *
173+ * This simplifies to:
174+ * (x - mpeg_min) * pixel_max > (mpeg_max - mpeg_min) * a
175+ * = P * x - K > Q * a in the below formula.
176+ *
177+ * We subtract an additional offset of (1 << (depth - 1)) to account for
178+ * rounding errors in the value of `x`, and an extra safety margin of
179+ * Q because vf_premultiply.c et al. add an offset of (a >> 1) & 1.
180+ */
181+ const int p = (1 << s->depth) - 1;
182+ const int q = s->mpeg_max - s->mpeg_min;
183+ const int k = p * s->mpeg_min + q + (1 << (s->depth - 1));
184+
185+ for (int i = 0; i < nb_planes; i++) {
186+ const ptrdiff_t stride = in->linesize[i];
187+ if (s->dsp.detect_alpha(in->data[i] + y_start * stride, stride,
188+ alpha, alpha_stride, w, h_slice, p, q, k)) {
189+ atomic_store(&s->detected_alpha, ALPHA_STRAIGHT);
190+ return 0;
191+ }
192+ }
193+
194+ return 0;
195+}
196+
197+static int filter_frame(AVFilterLink *inlink, AVFrame *in)
198+{
199+ AVFilterContext *ctx = inlink->dst;
200+ ColorDetectContext *s = ctx->priv;
201+ const int nb_threads = FFMIN(inlink->h, s->nb_threads);
202+
203+ if (s->mode & COLOR_DETECT_COLOR_RANGE && s->detected_range == AVCOL_RANGE_UNSPECIFIED)
204+ ff_filter_execute(ctx, detect_range, in, NULL, nb_threads);
205+ if (s->mode & COLOR_DETECT_ALPHA_MODE && s->detected_alpha == ALPHA_UNDETERMINED)
206+ ff_filter_execute(ctx, detect_alpha, in, NULL, nb_threads);
207+
208+ return ff_filter_frame(inlink->dst->outputs[0], in);
209+}
210+
211+static av_cold void uninit(AVFilterContext *ctx)
212+{
213+ ColorDetectContext *s = ctx->priv;
214+ if (!s->mode)
215+ return;
216+
217+ av_log(ctx, AV_LOG_INFO, "Detected color properties:\n");
218+ if (s->mode & COLOR_DETECT_COLOR_RANGE) {
219+ av_log(ctx, AV_LOG_INFO, " Color range: %s\n",
220+ s->detected_range == AVCOL_RANGE_JPEG ? "JPEG / full range"
221+ : "undetermined");
222+ }
223+
224+ if (s->mode & COLOR_DETECT_ALPHA_MODE) {
225+ av_log(ctx, AV_LOG_INFO, " Alpha mode: %s\n",
226+ s->detected_alpha == ALPHA_NONE ? "none" :
227+ s->detected_alpha == ALPHA_STRAIGHT ? "straight / independent"
228+ : "undetermined");
229+ }
230+}
231+
232+av_cold void ff_color_detect_dsp_init(FFColorDetectDSPContext *dsp, int depth,
233+ enum AVColorRange color_range)
234+{
235+ if (!dsp->detect_range)
236+ dsp->detect_range = depth > 8 ? ff_detect_range16_c : ff_detect_range_c;
237+ if (!dsp->detect_alpha) {
238+ if (color_range == AVCOL_RANGE_JPEG) {
239+ dsp->detect_alpha = depth > 8 ? ff_detect_alpha16_full_c : ff_detect_alpha_full_c;
240+ } else {
241+ dsp->detect_alpha = depth > 8 ? ff_detect_alpha16_limited_c : ff_detect_alpha_limited_c;
242+ }
243+ }
244+}
245+
246+static const AVFilterPad colordetect_inputs[] = {
247+ {
248+ .name = "default",
249+ .type = AVMEDIA_TYPE_VIDEO,
250+ .config_props = config_input,
251+ .filter_frame = filter_frame,
252+ },
253+};
254+
255+const FFFilter ff_vf_colordetect = {
256+ .p.name = "colordetect",
257+ .p.description = NULL_IF_CONFIG_SMALL("Detect video color properties."),
258+ .p.priv_class = &colordetect_class,
259+ .p.flags = AVFILTER_FLAG_SLICE_THREADS | AVFILTER_FLAG_METADATA_ONLY,
260+ .priv_size = sizeof(ColorDetectContext),
261+ FILTER_INPUTS(colordetect_inputs),
262+ FILTER_OUTPUTS(ff_video_default_filterpad),
263+ FILTER_QUERY_FUNC2(query_format),
264+ .uninit = uninit,
265+};
libavfilter/vf_colordetect.h+149−0
@@ -0,0 +1,149 @@
1+/*
2+ * This file is part of FFmpeg.
3+ *
4+ * FFmpeg is free software; you can redistribute it and/or
5+ * modify it under the terms of the GNU Lesser General Public
6+ * License as published by the Free Software Foundation; either
7+ * version 2.1 of the License, or (at your option) any later version.
8+ *
9+ * FFmpeg is distributed in the hope that it will be useful,
10+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
11+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12+ * Lesser General Public License for more details.
13+ *
14+ * You should have received a copy of the GNU Lesser General Public
15+ * License along with FFmpeg; if not, write to the Free Software
16+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17+ */
18+
19+#ifndef AVFILTER_COLORDETECT_H
20+#define AVFILTER_COLORDETECT_H
21+
22+#include <stddef.h>
23+#include <stdint.h>
24+
25+#include <libavutil/macros.h>
26+#include <libavutil/pixfmt.h>
27+
28+typedef struct FFColorDetectDSPContext {
29+ /* Returns 1 if an out-of-range value was detected, 0 otherwise */
30+ int (*detect_range)(const uint8_t *data, ptrdiff_t stride,
31+ ptrdiff_t width, ptrdiff_t height,
32+ int mpeg_min, int mpeg_max);
33+
34+ /* Returns 1 if the color value exceeds the alpha value, 0 otherwise */
35+ int (*detect_alpha)(const uint8_t *color, ptrdiff_t color_stride,
36+ const uint8_t *alpha, ptrdiff_t alpha_stride,
37+ ptrdiff_t width, ptrdiff_t height,
38+ int p, int q, int k);
39+} FFColorDetectDSPContext;
40+
41+void ff_color_detect_dsp_init(FFColorDetectDSPContext *dsp, int depth,
42+ enum AVColorRange color_range);
43+
44+static inline int ff_detect_range_c(const uint8_t *data, ptrdiff_t stride,
45+ ptrdiff_t width, ptrdiff_t height,
46+ int mpeg_min, int mpeg_max)
47+{
48+ while (height--) {
49+ for (int x = 0; x < width; x++) {
50+ const uint8_t val = data[x];
51+ if (val < mpeg_min || val > mpeg_max)
52+ return 1;
53+ }
54+ data += stride;
55+ }
56+
57+ return 0;
58+}
59+
60+static inline int ff_detect_range16_c(const uint8_t *data, ptrdiff_t stride,
61+ ptrdiff_t width, ptrdiff_t height,
62+ int mpeg_min, int mpeg_max)
63+{
64+ while (height--) {
65+ const uint16_t *data16 = (const uint16_t *) data;
66+ for (int x = 0; x < width; x++) {
67+ const uint16_t val = data16[x];
68+ if (val < mpeg_min || val > mpeg_max)
69+ return 1;
70+ }
71+ data += stride;
72+ }
73+
74+ return 0;
75+}
76+
77+static inline int
78+ff_detect_alpha_full_c(const uint8_t *color, ptrdiff_t color_stride,
79+ const uint8_t *alpha, ptrdiff_t alpha_stride,
80+ ptrdiff_t width, ptrdiff_t height,
81+ int p, int q, int k)
82+{
83+ while (height--) {
84+ for (int x = 0; x < width; x++) {
85+ if (color[x] > alpha[x])
86+ return 1;
87+ }
88+ color += color_stride;
89+ alpha += alpha_stride;
90+ }
91+ return 0;
92+}
93+
94+static inline int
95+ff_detect_alpha_limited_c(const uint8_t *color, ptrdiff_t color_stride,
96+ const uint8_t *alpha, ptrdiff_t alpha_stride,
97+ ptrdiff_t width, ptrdiff_t height,
98+ int p, int q, int k)
99+{
100+ while (height--) {
101+ for (int x = 0; x < width; x++) {
102+ if (p * color[x] - k > q * alpha[x])
103+ return 1;
104+ }
105+ color += color_stride;
106+ alpha += alpha_stride;
107+ }
108+ return 0;
109+}
110+
111+static inline int
112+ff_detect_alpha16_full_c(const uint8_t *color, ptrdiff_t color_stride,
113+ const uint8_t *alpha, ptrdiff_t alpha_stride,
114+ ptrdiff_t width, ptrdiff_t height,
115+ int p, int q, int k)
116+{
117+ while (height--) {
118+ const uint16_t *color16 = (const uint16_t *) color;
119+ const uint16_t *alpha16 = (const uint16_t *) alpha;
120+ for (int x = 0; x < width; x++) {
121+ if (color16[x] > alpha16[x])
122+ return 1;
123+ }
124+ color += color_stride;
125+ alpha += alpha_stride;
126+ }
127+ return 0;
128+}
129+
130+static inline int
131+ff_detect_alpha16_limited_c(const uint8_t *color, ptrdiff_t color_stride,
132+ const uint8_t *alpha, ptrdiff_t alpha_stride,
133+ ptrdiff_t width, ptrdiff_t height,
134+ int p, int q, int k)
135+{
136+ while (height--) {
137+ const uint16_t *color16 = (const uint16_t *) color;
138+ const uint16_t *alpha16 = (const uint16_t *) alpha;
139+ for (int x = 0; x < width; x++) {
140+ if ((int64_t) p * color16[x] - k > (int64_t) q * alpha16[x])
141+ return 1;
142+ }
143+ color += color_stride;
144+ alpha += alpha_stride;
145+ }
146+ return 0;
147+}
148+
149+#endif /* AVFILTER_COLORDETECT_H */

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t