History

avfilter/vf_colordetect: add new color range detection filter

This filter can detect various properties about the image, including whether or not there are out-of-range values, or whether the input appears to use straight or premultiplied alpha. Of course, these can only be heuristics, with "undetermined" as the base case. While we can definitely prove the existence of full range or straight alpha colors, we can never infer the opposite.

Niklas Haas committed 1 year ago

Commit 545f7211 parent 8cdb47e5 files · +443 −0Browse files

Unified · Split

5 files
  1. doc/filters.texi +27 −0
  2. libavfilter/Makefile +1 −0
  3. libavfilter/allfilters.c +1 −0
  4. libavfilter/vf_colordetect.c +265 −0
  5. libavfilter/vf_colordetect.h +149 −0
doc/filters.texi+27−0
@@ -9753,6 +9753,33 @@ colorchannelmixer=.393:.769:.189:0:.349:.686:.168:0:.272:.534:.131 · 9752 unmodified lines
9753 9753
9754This filter supports the all above options as @ref{commands}.9754This filter supports the all above options as @ref{commands}.
9755 9755
9756@section colordetect
9757Analyze the video frames to determine the effective value range and alpha
9758mode.
9759
9760The filter accepts the following options:
9761
9762@table @option
9763@item mode
9764Set of properties to detect. Unavailable properties, such as alpha mode for
9765an input image without an alpha channel, will be ignored automatically.
9766
9767Accepts a combination of the following flags:
9768
9769@table @samp
9770@item color_range
9771Detect if the source countains luma pixels outside the limited (MPEG) range,
9772which indicates that this is a full range YUV source.
9773@item alpha_mode
9774Detect if the source contains color values above the alpha channel, which
9775indicates that the alpha channel is independent (straight), rather than
9776premultiplied.
9777@item all
9778Enable detection of all of the above properties. This is the default.
9779@end table
9780
9781@end table
9782
9756@section colorize9783@section colorize
9757Overlay a solid color on the video stream.9784Overlay a solid color on the video stream.
9758 9785
libavfilter/Makefile+1−0
@@ -237,6 +237,7 @@ OBJS-$(CONFIG_COLORBALANCE_FILTER) += vf_colorbalance.o · 236 unmodified lines
237OBJS-$(CONFIG_COLORCHANNELMIXER_FILTER) += vf_colorchannelmixer.o237OBJS-$(CONFIG_COLORCHANNELMIXER_FILTER) += vf_colorchannelmixer.o
238OBJS-$(CONFIG_COLORCONTRAST_FILTER) += vf_colorcontrast.o238OBJS-$(CONFIG_COLORCONTRAST_FILTER) += vf_colorcontrast.o
239OBJS-$(CONFIG_COLORCORRECT_FILTER) += vf_colorcorrect.o239OBJS-$(CONFIG_COLORCORRECT_FILTER) += vf_colorcorrect.o
240OBJS-$(CONFIG_COLORDETECT_FILTER) += vf_colordetect.o
240OBJS-$(CONFIG_COLORIZE_FILTER) += vf_colorize.o241OBJS-$(CONFIG_COLORIZE_FILTER) += vf_colorize.o
241OBJS-$(CONFIG_COLORKEY_FILTER) += vf_colorkey.o242OBJS-$(CONFIG_COLORKEY_FILTER) += vf_colorkey.o
242OBJS-$(CONFIG_COLORKEY_OPENCL_FILTER) += vf_colorkey_opencl.o opencl.o \243OBJS-$(CONFIG_COLORKEY_OPENCL_FILTER) += vf_colorkey_opencl.o opencl.o \
libavfilter/allfilters.c+1−0
@@ -218,6 +218,7 @@ extern const FFFilter ff_vf_colorbalance; · 217 unmodified lines
218extern const FFFilter ff_vf_colorchannelmixer;218extern const FFFilter ff_vf_colorchannelmixer;
219extern const FFFilter ff_vf_colorcontrast;219extern const FFFilter ff_vf_colorcontrast;
220extern const FFFilter ff_vf_colorcorrect;220extern const FFFilter ff_vf_colorcorrect;
221extern const FFFilter ff_vf_colordetect;
221extern const FFFilter ff_vf_colorize;222extern const FFFilter ff_vf_colorize;
222extern const FFFilter ff_vf_colorkey;223extern const FFFilter ff_vf_colorkey;
223extern const FFFilter ff_vf_colorkey_opencl;224extern const FFFilter ff_vf_colorkey_opencl;
libavfilter/vf_colordetect.c+265−0
@@ -0,0 +1,265 @@
1/*
2 * Copyright (c) 2025 Niklas Haas
3 *
4 * This file is part of FFmpeg.
5 *
6 * FFmpeg is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
10 *
11 * FFmpeg is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with FFmpeg; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19 */
20
21/**
22 * @file
23 * Video color space detector, tries to auto-detect YUV range and alpha mode.
24 */
25
26#include <stdbool.h>
27#include <stdatomic.h>
28
29#include "config.h"
30
31#include "libavutil/mem.h"
32#include "libavutil/opt.h"
33#include "libavutil/pixdesc.h"
34
35#include "avfilter.h"
36#include "filters.h"
37#include "formats.h"
38#include "video.h"
39
40#include "vf_colordetect.h"
41
42enum AlphaMode {
43 ALPHA_NONE = -1,
44 ALPHA_UNDETERMINED = 0,
45 ALPHA_STRAIGHT,
46 /* No way to positively identify premultiplied alpha */
47};
48
49enum ColorDetectMode {
50 COLOR_DETECT_COLOR_RANGE = 1 << 0,
51 COLOR_DETECT_ALPHA_MODE = 1 << 1,
52};
53
54typedef struct ColorDetectContext {
55 const AVClass *class;
56 FFColorDetectDSPContext dsp;
57 unsigned mode;
58
59 const AVPixFmtDescriptor *desc;
60 int nb_threads;
61 int depth;
62 int idx_a;
63 int mpeg_min;
64 int mpeg_max;
65
66 atomic_int detected_range; // enum AVColorRange
67 atomic_int detected_alpha; // enum AlphaMode
68} ColorDetectContext;
69
70#define OFFSET(x) offsetof(ColorDetectContext, x)
71#define FLAGS AV_OPT_FLAG_VIDEO_PARAM|AV_OPT_FLAG_FILTERING_PARAM
72
73static const AVOption colordetect_options[] = {
74 { "mode", "Image properties to detect", OFFSET(mode), AV_OPT_TYPE_FLAGS, {.i64 = -1}, 0, UINT_MAX, FLAGS, .unit = "mode" },
75 { "color_range", "Detect (YUV) color range", 0, AV_OPT_TYPE_CONST, {.i64 = COLOR_DETECT_COLOR_RANGE}, 0, 0, FLAGS, .unit = "mode" },
76 { "alpha_mode", "Detect alpha mode", 0, AV_OPT_TYPE_CONST, {.i64 = COLOR_DETECT_ALPHA_MODE }, 0, 0, FLAGS, .unit = "mode" },
77 { "all", "Detect all supported properties", 0, AV_OPT_TYPE_CONST, {.i64 = -1}, 0, 0, FLAGS, .unit = "mode" },
78 { NULL }
79};
80
81AVFILTER_DEFINE_CLASS(colordetect);
82
83static int query_format(const AVFilterContext *ctx,
84 AVFilterFormatsConfig **cfg_in,
85 AVFilterFormatsConfig **cfg_out)
86{
87 int want_flags = AV_PIX_FMT_FLAG_PLANAR;
88 int reject_flags = AV_PIX_FMT_FLAG_PAL | AV_PIX_FMT_FLAG_HWACCEL |
89 AV_PIX_FMT_FLAG_BITSTREAM | AV_PIX_FMT_FLAG_FLOAT |
90 AV_PIX_FMT_FLAG_BAYER | AV_PIX_FMT_FLAG_XYZ;
91
92 if (HAVE_BIGENDIAN) {
93 want_flags |= AV_PIX_FMT_FLAG_BE;
94 } else {
95 reject_flags |= AV_PIX_FMT_FLAG_BE;
96 }
97
98 AVFilterFormats *formats = ff_formats_pixdesc_filter(want_flags, reject_flags);
99 return ff_set_common_formats2(ctx, cfg_in, cfg_out, formats);
100}
101
102static int config_input(AVFilterLink *inlink)
103{
104 AVFilterContext *ctx = inlink->dst;
105 ColorDetectContext *s = ctx->priv;
106 const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(inlink->format);
107 const int depth = desc->comp[0].depth;
108 const int mpeg_min = 16 << (depth - 8);
109 const int mpeg_max = 235 << (depth - 8);
110 if (depth > 16) /* not currently possible; prevent future bugs */
111 return AVERROR(ENOTSUP);
112
113 s->desc = desc;
114 s->depth = depth;
115 s->mpeg_min = mpeg_min;
116 s->mpeg_max = mpeg_max;
117 s->nb_threads = ff_filter_get_nb_threads(ctx);
118
119 if (desc->flags & AV_PIX_FMT_FLAG_RGB) {
120 atomic_init(&s->detected_range, AVCOL_RANGE_JPEG);
121 } else {
122 atomic_init(&s->detected_range, AVCOL_RANGE_UNSPECIFIED);
123 }
124
125 if (desc->flags & AV_PIX_FMT_FLAG_ALPHA) {
126 s->idx_a = desc->comp[desc->nb_components - 1].plane;
127 atomic_init(&s->detected_alpha, ALPHA_UNDETERMINED);
128 } else {
129 atomic_init(&s->detected_alpha, ALPHA_NONE);
130 }
131
132 ff_color_detect_dsp_init(&s->dsp, depth, inlink->color_range);
133 return 0;
134}
135
136static int detect_range(AVFilterContext *ctx, void *arg,
137 int jobnr, int nb_jobs)
138{
139 ColorDetectContext *s = ctx->priv;
140 const AVFrame *in = arg;
141 const ptrdiff_t stride = in->linesize[0];
142 const int y_start = (in->height * jobnr) / nb_jobs;
143 const int y_end = (in->height * (jobnr + 1)) / nb_jobs;
144 const int h_slice = y_end - y_start;
145
146 if (s->dsp.detect_range(in->data[0] + y_start * stride, stride,
147 in->width, h_slice, s->mpeg_min, s->mpeg_max))
148 atomic_store(&s->detected_range, AVCOL_RANGE_JPEG);
149
150 return 0;
151}
152
153static int detect_alpha(AVFilterContext *ctx, void *arg,
154 int jobnr, int nb_jobs)
155{
156 ColorDetectContext *s = ctx->priv;
157 const AVFrame *in = arg;
158 const int w = in->width;
159 const int h = in->height;
160 const int y_start = (h * jobnr) / nb_jobs;
161 const int y_end = (h * (jobnr + 1)) / nb_jobs;
162 const int h_slice = y_end - y_start;
163
164 const int nb_planes = (s->desc->flags & AV_PIX_FMT_FLAG_RGB) ? 3 : 1;
165 const ptrdiff_t alpha_stride = in->linesize[s->idx_a];
166 const uint8_t *alpha = in->data[s->idx_a] + y_start * alpha_stride;
167
168 /**
169 * To check if a value is out of range, we need to compare the color value
170 * against the maximum possible color for a given alpha value.
171 * x > ((mpeg_max - mpeg_min) / pixel_max) * a + mpeg_min
172 *
173 * This simplifies to:
174 * (x - mpeg_min) * pixel_max > (mpeg_max - mpeg_min) * a
175 * = P * x - K > Q * a in the below formula.
176 *
177 * We subtract an additional offset of (1 << (depth - 1)) to account for
178 * rounding errors in the value of `x`, and an extra safety margin of
179 * Q because vf_premultiply.c et al. add an offset of (a >> 1) & 1.
180 */
181 const int p = (1 << s->depth) - 1;
182 const int q = s->mpeg_max - s->mpeg_min;
183 const int k = p * s->mpeg_min + q + (1 << (s->depth - 1));
184
185 for (int i = 0; i < nb_planes; i++) {
186 const ptrdiff_t stride = in->linesize[i];
187 if (s->dsp.detect_alpha(in->data[i] + y_start * stride, stride,
188 alpha, alpha_stride, w, h_slice, p, q, k)) {
189 atomic_store(&s->detected_alpha, ALPHA_STRAIGHT);
190 return 0;
191 }
192 }
193
194 return 0;
195}
196
197static int filter_frame(AVFilterLink *inlink, AVFrame *in)
198{
199 AVFilterContext *ctx = inlink->dst;
200 ColorDetectContext *s = ctx->priv;
201 const int nb_threads = FFMIN(inlink->h, s->nb_threads);
202
203 if (s->mode & COLOR_DETECT_COLOR_RANGE && s->detected_range == AVCOL_RANGE_UNSPECIFIED)
204 ff_filter_execute(ctx, detect_range, in, NULL, nb_threads);
205 if (s->mode & COLOR_DETECT_ALPHA_MODE && s->detected_alpha == ALPHA_UNDETERMINED)
206 ff_filter_execute(ctx, detect_alpha, in, NULL, nb_threads);
207
208 return ff_filter_frame(inlink->dst->outputs[0], in);
209}
210
211static av_cold void uninit(AVFilterContext *ctx)
212{
213 ColorDetectContext *s = ctx->priv;
214 if (!s->mode)
215 return;
216
217 av_log(ctx, AV_LOG_INFO, "Detected color properties:\n");
218 if (s->mode & COLOR_DETECT_COLOR_RANGE) {
219 av_log(ctx, AV_LOG_INFO, " Color range: %s\n",
220 s->detected_range == AVCOL_RANGE_JPEG ? "JPEG / full range"
221 : "undetermined");
222 }
223
224 if (s->mode & COLOR_DETECT_ALPHA_MODE) {
225 av_log(ctx, AV_LOG_INFO, " Alpha mode: %s\n",
226 s->detected_alpha == ALPHA_NONE ? "none" :
227 s->detected_alpha == ALPHA_STRAIGHT ? "straight / independent"
228 : "undetermined");
229 }
230}
231
232av_cold void ff_color_detect_dsp_init(FFColorDetectDSPContext *dsp, int depth,
233 enum AVColorRange color_range)
234{
235 if (!dsp->detect_range)
236 dsp->detect_range = depth > 8 ? ff_detect_range16_c : ff_detect_range_c;
237 if (!dsp->detect_alpha) {
238 if (color_range == AVCOL_RANGE_JPEG) {
239 dsp->detect_alpha = depth > 8 ? ff_detect_alpha16_full_c : ff_detect_alpha_full_c;
240 } else {
241 dsp->detect_alpha = depth > 8 ? ff_detect_alpha16_limited_c : ff_detect_alpha_limited_c;
242 }
243 }
244}
245
246static const AVFilterPad colordetect_inputs[] = {
247 {
248 .name = "default",
249 .type = AVMEDIA_TYPE_VIDEO,
250 .config_props = config_input,
251 .filter_frame = filter_frame,
252 },
253};
254
255const FFFilter ff_vf_colordetect = {
256 .p.name = "colordetect",
257 .p.description = NULL_IF_CONFIG_SMALL("Detect video color properties."),
258 .p.priv_class = &colordetect_class,
259 .p.flags = AVFILTER_FLAG_SLICE_THREADS | AVFILTER_FLAG_METADATA_ONLY,
260 .priv_size = sizeof(ColorDetectContext),
261 FILTER_INPUTS(colordetect_inputs),
262 FILTER_OUTPUTS(ff_video_default_filterpad),
263 FILTER_QUERY_FUNC2(query_format),
264 .uninit = uninit,
265};
libavfilter/vf_colordetect.h+149−0
@@ -0,0 +1,149 @@
1/*
2 * This file is part of FFmpeg.
3 *
4 * FFmpeg is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU Lesser General Public
6 * License as published by the Free Software Foundation; either
7 * version 2.1 of the License, or (at your option) any later version.
8 *
9 * FFmpeg is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 * Lesser General Public License for more details.
13 *
14 * You should have received a copy of the GNU Lesser General Public
15 * License along with FFmpeg; if not, write to the Free Software
16 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 */
18
19#ifndef AVFILTER_COLORDETECT_H
20#define AVFILTER_COLORDETECT_H
21
22#include <stddef.h>
23#include <stdint.h>
24
25#include <libavutil/macros.h>
26#include <libavutil/pixfmt.h>
27
28typedef struct FFColorDetectDSPContext {
29 /* Returns 1 if an out-of-range value was detected, 0 otherwise */
30 int (*detect_range)(const uint8_t *data, ptrdiff_t stride,
31 ptrdiff_t width, ptrdiff_t height,
32 int mpeg_min, int mpeg_max);
33
34 /* Returns 1 if the color value exceeds the alpha value, 0 otherwise */
35 int (*detect_alpha)(const uint8_t *color, ptrdiff_t color_stride,
36 const uint8_t *alpha, ptrdiff_t alpha_stride,
37 ptrdiff_t width, ptrdiff_t height,
38 int p, int q, int k);
39} FFColorDetectDSPContext;
40
41void ff_color_detect_dsp_init(FFColorDetectDSPContext *dsp, int depth,
42 enum AVColorRange color_range);
43
44static inline int ff_detect_range_c(const uint8_t *data, ptrdiff_t stride,
45 ptrdiff_t width, ptrdiff_t height,
46 int mpeg_min, int mpeg_max)
47{
48 while (height--) {
49 for (int x = 0; x < width; x++) {
50 const uint8_t val = data[x];
51 if (val < mpeg_min || val > mpeg_max)
52 return 1;
53 }
54 data += stride;
55 }
56
57 return 0;
58}
59
60static inline int ff_detect_range16_c(const uint8_t *data, ptrdiff_t stride,
61 ptrdiff_t width, ptrdiff_t height,
62 int mpeg_min, int mpeg_max)
63{
64 while (height--) {
65 const uint16_t *data16 = (const uint16_t *) data;
66 for (int x = 0; x < width; x++) {
67 const uint16_t val = data16[x];
68 if (val < mpeg_min || val > mpeg_max)
69 return 1;
70 }
71 data += stride;
72 }
73
74 return 0;
75}
76
77static inline int
78ff_detect_alpha_full_c(const uint8_t *color, ptrdiff_t color_stride,
79 const uint8_t *alpha, ptrdiff_t alpha_stride,
80 ptrdiff_t width, ptrdiff_t height,
81 int p, int q, int k)
82{
83 while (height--) {
84 for (int x = 0; x < width; x++) {
85 if (color[x] > alpha[x])
86 return 1;
87 }
88 color += color_stride;
89 alpha += alpha_stride;
90 }
91 return 0;
92}
93
94static inline int
95ff_detect_alpha_limited_c(const uint8_t *color, ptrdiff_t color_stride,
96 const uint8_t *alpha, ptrdiff_t alpha_stride,
97 ptrdiff_t width, ptrdiff_t height,
98 int p, int q, int k)
99{
100 while (height--) {
101 for (int x = 0; x < width; x++) {
102 if (p * color[x] - k > q * alpha[x])
103 return 1;
104 }
105 color += color_stride;
106 alpha += alpha_stride;
107 }
108 return 0;
109}
110
111static inline int
112ff_detect_alpha16_full_c(const uint8_t *color, ptrdiff_t color_stride,
113 const uint8_t *alpha, ptrdiff_t alpha_stride,
114 ptrdiff_t width, ptrdiff_t height,
115 int p, int q, int k)
116{
117 while (height--) {
118 const uint16_t *color16 = (const uint16_t *) color;
119 const uint16_t *alpha16 = (const uint16_t *) alpha;
120 for (int x = 0; x < width; x++) {
121 if (color16[x] > alpha16[x])
122 return 1;
123 }
124 color += color_stride;
125 alpha += alpha_stride;
126 }
127 return 0;
128}
129
130static inline int
131ff_detect_alpha16_limited_c(const uint8_t *color, ptrdiff_t color_stride,
132 const uint8_t *alpha, ptrdiff_t alpha_stride,
133 ptrdiff_t width, ptrdiff_t height,
134 int p, int q, int k)
135{
136 while (height--) {
137 const uint16_t *color16 = (const uint16_t *) color;
138 const uint16_t *alpha16 = (const uint16_t *) alpha;
139 for (int x = 0; x < width; x++) {
140 if ((int64_t) p * color16[x] - k > (int64_t) q * alpha16[x])
141 return 1;
142 }
143 color += color_stride;
144 alpha += alpha_stride;
145 }
146 return 0;
147}
148
149#endif /* AVFILTER_COLORDETECT_H */

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t