| @@ -0,0 +1,386 @@ |
| 1 | +/* |
| 2 | + * Copyright (c) 2017 Richard Ling |
| 3 | + * |
| 4 | + * This file is part of FFmpeg. |
| 5 | + * |
| 6 | + * FFmpeg is free software; you can redistribute it and/or |
| 7 | + * modify it under the terms of the GNU Lesser General Public |
| 8 | + * License as published by the Free Software Foundation; either |
| 9 | + * version 2.1 of the License, or (at your option) any later version. |
| 10 | + * |
| 11 | + * FFmpeg is distributed in the hope that it will be useful, |
| 12 | + * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 13 | + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
| 14 | + * Lesser General Public License for more details. |
| 15 | + * |
| 16 | + * You should have received a copy of the GNU Lesser General Public |
| 17 | + * License along with FFmpeg; if not, write to the Free Software |
| 18 | + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA |
| 19 | + */ |
| 20 | + |
| 21 | +/* |
| 22 | + * Normalize RGB video (aka histogram stretching, contrast stretching). |
| 23 | + * See: https://en.wikipedia.org/wiki/Normalization_(image_processing) |
| 24 | + * |
| 25 | + * For each channel of each frame, the filter computes the input range and maps |
| 26 | + * it linearly to the user-specified output range. The output range defaults |
| 27 | + * to the full dynamic range from pure black to pure white. |
| 28 | + * |
| 29 | + * Naively maximising the dynamic range of each frame of video in isolation |
| 30 | + * may cause flickering (rapid changes in brightness of static objects in the |
| 31 | + * scene) when small dark or bright objects enter or leave the scene. This |
| 32 | + * filter can apply temporal smoothing to the input range to reduce flickering. |
| 33 | + * Temporal smoothing is similar to the auto-exposure (automatic gain control) |
| 34 | + * on a video camera, which performs the same function; and, like a video |
| 35 | + * camera, it may cause a period of over- or under-exposure of the video. |
| 36 | + * |
| 37 | + * The filter can normalize the R,G,B channels independently, which may cause |
| 38 | + * color shifting, or link them together as a single channel, which prevents |
| 39 | + * color shifting. More precisely, linked normalization preserves hue (as it's |
| 40 | + * defined in HSV/HSL color spaces) while independent normalization does not. |
| 41 | + * Independent normalization can be used to remove color casts, such as the |
| 42 | + * blue cast from underwater video, restoring more natural colors. The filter |
| 43 | + * can also combine independent and linked normalization in any ratio. |
| 44 | + * |
| 45 | + * Finally the overall strength of the filter can be adjusted, from no effect |
| 46 | + * to full normalization. |
| 47 | + * |
| 48 | + * The 5 AVOptions are: |
| 49 | + * blackpt, Colors which define the output range. The minimum input value |
| 50 | + * whitept is mapped to the blackpt. The maximum input value is mapped to |
| 51 | + * the whitept. The defaults are black and white respectively. |
| 52 | + * Specifying white for blackpt and black for whitept will give |
| 53 | + * color-inverted, normalized video. Shades of grey can be used |
| 54 | + * to reduce the dynamic range (contrast). Specifying saturated |
| 55 | + * colors here can create some interesting effects. |
| 56 | + * |
| 57 | + * smoothing The amount of temporal smoothing, expressed in frames (>=0). |
| 58 | + * the minimum and maximum input values of each channel are |
| 59 | + * smoothed using a rolling average over the current frame and |
| 60 | + * that many previous frames of video. Defaults to 0 (no temporal |
| 61 | + * smoothing). |
| 62 | + * |
| 63 | + * independence |
| 64 | + * Controls the ratio of independent (color shifting) channel |
| 65 | + * normalization to linked (color preserving) normalization. 0.0 |
| 66 | + * is fully linked, 1.0 is fully independent. Defaults to fully |
| 67 | + * independent. |
| 68 | + * |
| 69 | + * strength Overall strength of the filter. 1.0 is full strength. 0.0 is |
| 70 | + * a rather expensive no-op. Values in between can give a gentle |
| 71 | + * boost to low-contrast video without creating an artificial |
| 72 | + * over-processed look. The default is full strength. |
| 73 | + */ |
| 74 | + |
| 75 | +#include "libavutil/imgutils.h" |
| 76 | +#include "libavutil/opt.h" |
| 77 | +#include "libavutil/pixdesc.h" |
| 78 | +#include "avfilter.h" |
| 79 | +#include "formats.h" |
| 80 | +#include "internal.h" |
| 81 | +#include "video.h" |
| 82 | + |
| 83 | +typedef struct NormalizeContext { |
| 84 | + const AVClass *class; |
| 85 | + |
| 86 | + // Storage for the corresponding AVOptions |
| 87 | + uint8_t blackpt[4]; |
| 88 | + uint8_t whitept[4]; |
| 89 | + int smoothing; |
| 90 | + float independence; |
| 91 | + float strength; |
| 92 | + |
| 93 | + int co[4]; // Offsets to R,G,B,A bytes respectively in each pixel |
| 94 | + int num_components; // Number of components in the pixel format |
| 95 | + int history_len; // Number of frames to average; based on smoothing factor |
| 96 | + int frame_num; // Increments on each frame, starting from 0. |
| 97 | + |
| 98 | + // Per-extremum, per-channel history, for temporal smoothing. |
| 99 | + struct { |
| 100 | + uint8_t *history; // History entries. |
| 101 | + uint32_t history_sum; // Sum of history entries. |
| 102 | + } min[3], max[3]; // Min and max for each channel in {R,G,B}. |
| 103 | + uint8_t *history_mem; // Single allocation for above history entries |
| 104 | + |
| 105 | +} NormalizeContext; |
| 106 | + |
| 107 | +#define OFFSET(x) offsetof(NormalizeContext, x) |
| 108 | +#define FLAGS AV_OPT_FLAG_VIDEO_PARAM|AV_OPT_FLAG_FILTERING_PARAM |
| 109 | + |
| 110 | +static const AVOption normalize_options[] = { |
| 111 | + { "blackpt", "output color to which darkest input color is mapped", OFFSET(blackpt), AV_OPT_TYPE_COLOR, { .str = "black" }, CHAR_MIN, CHAR_MAX, FLAGS }, |
| 112 | + { "whitept", "output color to which brightest input color is mapped", OFFSET(whitept), AV_OPT_TYPE_COLOR, { .str = "white" }, CHAR_MIN, CHAR_MAX, FLAGS }, |
| 113 | + { "smoothing", "amount of temporal smoothing of the input range, to reduce flicker", OFFSET(smoothing), AV_OPT_TYPE_INT, {.i64=0}, 0, INT_MAX/8, FLAGS }, |
| 114 | + { "independence", "proportion of independent to linked channel normalization", OFFSET(independence), AV_OPT_TYPE_FLOAT, {.dbl=1.0}, 0.0, 1.0, FLAGS }, |
| 115 | + { "strength", "strength of filter, from no effect to full normalization", OFFSET(strength), AV_OPT_TYPE_FLOAT, {.dbl=1.0}, 0.0, 1.0, FLAGS }, |
| 116 | + { NULL } |
| 117 | +}; |
| 118 | + |
| 119 | +AVFILTER_DEFINE_CLASS(normalize); |
| 120 | + |
| 121 | +// This function is the main guts of the filter. Normalizes the input frame |
| 122 | +// into the output frame. The frames are known to have the same dimensions |
| 123 | +// and pixel format. |
| 124 | +static void normalize(NormalizeContext *s, AVFrame *in, AVFrame *out) |
| 125 | +{ |
| 126 | + // Per-extremum, per-channel local variables. |
| 127 | + struct { |
| 128 | + uint8_t in; // Original input byte value for this frame. |
| 129 | + float smoothed; // Smoothed input value [0,255]. |
| 130 | + float out; // Output value [0,255]. |
| 131 | + } min[3], max[3]; // Min and max for each channel in {R,G,B}. |
| 132 | + |
| 133 | + float rgb_min_smoothed; // Min input range for linked normalization |
| 134 | + float rgb_max_smoothed; // Max input range for linked normalization |
| 135 | + uint8_t lut[3][256]; // Lookup table |
| 136 | + int x, y, c; |
| 137 | + |
| 138 | + // First, scan the input frame to find, for each channel, the minimum |
| 139 | + // (min.in) and maximum (max.in) values present in the channel. |
| 140 | + for (c = 0; c < 3; c++) |
| 141 | + min[c].in = max[c].in = in->data[0][s->co[c]]; |
| 142 | + for (y = 0; y < in->height; y++) { |
| 143 | + uint8_t *inp = in->data[0] + y * in->linesize[0]; |
| 144 | + uint8_t *outp = out->data[0] + y * out->linesize[0]; |
| 145 | + for (x = 0; x < in->width; x++) { |
| 146 | + for (c = 0; c < 3; c++) { |
| 147 | + min[c].in = FFMIN(min[c].in, inp[s->co[c]]); |
| 148 | + max[c].in = FFMAX(max[c].in, inp[s->co[c]]); |
| 149 | + } |
| 150 | + inp += s->num_components; |
| 151 | + outp += s->num_components; |
| 152 | + } |
| 153 | + } |
| 154 | + |
| 155 | + // Next, for each channel, push min.in and max.in into their respective |
| 156 | + // histories, to determine the min.smoothed and max.smoothed for this frame. |
| 157 | + { |
| 158 | + int history_idx = s->frame_num % s->history_len; |
| 159 | + // Assume the history is not yet full; num_history_vals is the number |
| 160 | + // of frames received so far including the current frame. |
| 161 | + int num_history_vals = s->frame_num + 1; |
| 162 | + if (s->frame_num >= s->history_len) { |
| 163 | + //The history is full; drop oldest value and cap num_history_vals. |
| 164 | + for (c = 0; c < 3; c++) { |
| 165 | + s->min[c].history_sum -= s->min[c].history[history_idx]; |
| 166 | + s->max[c].history_sum -= s->max[c].history[history_idx]; |
| 167 | + } |
| 168 | + num_history_vals = s->history_len; |
| 169 | + } |
| 170 | + // For each extremum, update history_sum and calculate smoothed value |
| 171 | + // as the rolling average of the history entries. |
| 172 | + for (c = 0; c < 3; c++) { |
| 173 | + s->min[c].history_sum += (s->min[c].history[history_idx] = min[c].in); |
| 174 | + min[c].smoothed = s->min[c].history_sum / (float)num_history_vals; |
| 175 | + s->max[c].history_sum += (s->max[c].history[history_idx] = max[c].in); |
| 176 | + max[c].smoothed = s->max[c].history_sum / (float)num_history_vals; |
| 177 | + } |
| 178 | + } |
| 179 | + |
| 180 | + // Determine the input range for linked normalization. This is simply the |
| 181 | + // minimum of the per-channel minimums, and the maximum of the per-channel |
| 182 | + // maximums. |
| 183 | + rgb_min_smoothed = FFMIN3(min[0].smoothed, min[1].smoothed, min[2].smoothed); |
| 184 | + rgb_max_smoothed = FFMAX3(max[0].smoothed, max[1].smoothed, max[2].smoothed); |
| 185 | + |
| 186 | + // Now, process each channel to determine the input and output range and |
| 187 | + // build the lookup tables. |
| 188 | + for (c = 0; c < 3; c++) { |
| 189 | + int in_val; |
| 190 | + // Adjust the input range for this channel [min.smoothed,max.smoothed] |
| 191 | + // by mixing in the correct proportion of the linked normalization |
| 192 | + // input range [rgb_min_smoothed,rgb_max_smoothed]. |
| 193 | + min[c].smoothed = (min[c].smoothed * s->independence) |
| 194 | + + (rgb_min_smoothed * (1.0f - s->independence)); |
| 195 | + max[c].smoothed = (max[c].smoothed * s->independence) |
| 196 | + + (rgb_max_smoothed * (1.0f - s->independence)); |
| 197 | + |
| 198 | + // Calculate the output range [min.out,max.out] as a ratio of the full- |
| 199 | + // strength output range [blackpt,whitept] and the original input range |
| 200 | + // [min.in,max.in], based on the user-specified filter strength. |
| 201 | + min[c].out = (s->blackpt[c] * s->strength) |
| 202 | + + (min[c].in * (1.0f - s->strength)); |
| 203 | + max[c].out = (s->whitept[c] * s->strength) |
| 204 | + + (max[c].in * (1.0f - s->strength)); |
| 205 | + |
| 206 | + // Now, build a lookup table which linearly maps the adjusted input range |
| 207 | + // [min.smoothed,max.smoothed] to the output range [min.out,max.out]. |
| 208 | + // Perform the linear interpolation for each x: |
| 209 | + // lut[x] = (int)(float(x - min.smoothed) * scale + max.out + 0.5) |
| 210 | + // where scale = (max.out - min.out) / (max.smoothed - min.smoothed) |
| 211 | + if (min[c].smoothed == max[c].smoothed) { |
| 212 | + // There is no dynamic range to expand. No mapping for this channel. |
| 213 | + for (in_val = min[c].in; in_val <= max[c].in; in_val++) |
| 214 | + lut[c][in_val] = min[c].out; |
| 215 | + } else { |
| 216 | + // We must set lookup values for all values in the original input |
| 217 | + // range [min.in,max.in]. Since the original input range may be |
| 218 | + // larger than [min.smoothed,max.smoothed], some output values may |
| 219 | + // fall outside the [0,255] dynamic range. We need to clamp them. |
| 220 | + float scale = (max[c].out - min[c].out) / (max[c].smoothed - min[c].smoothed); |
| 221 | + for (in_val = min[c].in; in_val <= max[c].in; in_val++) { |
| 222 | + int out_val = (in_val - min[c].smoothed) * scale + min[c].out + 0.5f; |
| 223 | + out_val = FFMAX(out_val, 0); |
| 224 | + out_val = FFMIN(out_val, 255); |
| 225 | + lut[c][in_val] = out_val; |
| 226 | + } |
| 227 | + } |
| 228 | + } |
| 229 | + |
| 230 | + // Finally, process the pixels of the input frame using the lookup tables. |
| 231 | + for (y = 0; y < in->height; y++) { |
| 232 | + uint8_t *inp = in->data[0] + y * in->linesize[0]; |
| 233 | + uint8_t *outp = out->data[0] + y * out->linesize[0]; |
| 234 | + for (x = 0; x < in->width; x++) { |
| 235 | + for (c = 0; c < 3; c++) |
| 236 | + outp[s->co[c]] = lut[c][inp[s->co[c]]]; |
| 237 | + if (s->num_components == 4) |
| 238 | + // Copy alpha as-is. |
| 239 | + outp[s->co[3]] = inp[s->co[3]]; |
| 240 | + inp += s->num_components; |
| 241 | + outp += s->num_components; |
| 242 | + } |
| 243 | + } |
| 244 | + |
| 245 | + s->frame_num++; |
| 246 | +} |
| 247 | + |
| 248 | +// Now we define all the functions accessible from the ff_vf_normalize class, |
| 249 | +// which is ffmpeg's interface to our filter. See doc/filter_design.txt and |
| 250 | +// doc/writing_filters.txt for descriptions of what these interface functions |
| 251 | +// are expected to do. |
| 252 | + |
| 253 | +// Set the pixel formats that our filter supports. We should be able to process |
| 254 | +// any 8-bit RGB formats. 16-bit support might be useful one day. |
| 255 | +static int query_formats(AVFilterContext *ctx) |
| 256 | +{ |
| 257 | + static const enum AVPixelFormat pixel_fmts[] = { |
| 258 | + AV_PIX_FMT_RGB24, |
| 259 | + AV_PIX_FMT_BGR24, |
| 260 | + AV_PIX_FMT_ARGB, |
| 261 | + AV_PIX_FMT_RGBA, |
| 262 | + AV_PIX_FMT_ABGR, |
| 263 | + AV_PIX_FMT_BGRA, |
| 264 | + AV_PIX_FMT_0RGB, |
| 265 | + AV_PIX_FMT_RGB0, |
| 266 | + AV_PIX_FMT_0BGR, |
| 267 | + AV_PIX_FMT_BGR0, |
| 268 | + AV_PIX_FMT_NONE |
| 269 | + }; |
| 270 | + // According to filter_design.txt, using ff_set_common_formats() this way |
| 271 | + // ensures the pixel formats of the input and output will be the same. That |
| 272 | + // saves a bit of effort possibly needing to handle format conversions. |
| 273 | + AVFilterFormats *formats = ff_make_format_list(pixel_fmts); |
| 274 | + if (!formats) |
| 275 | + return AVERROR(ENOMEM); |
| 276 | + return ff_set_common_formats(ctx, formats); |
| 277 | +} |
| 278 | + |
| 279 | +// At this point we know the pixel format used for both input and output. We |
| 280 | +// can also access the frame rate of the input video and allocate some memory |
| 281 | +// appropriately |
| 282 | +static int config_input(AVFilterLink *inlink) |
| 283 | +{ |
| 284 | + NormalizeContext *s = inlink->dst->priv; |
| 285 | + // Store offsets to R,G,B,A bytes respectively in each pixel |
| 286 | + const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(inlink->format); |
| 287 | + int c; |
| 288 | + |
| 289 | + for (c = 0; c < 4; ++c) |
| 290 | + s->co[c] = desc->comp[c].offset; |
| 291 | + s->num_components = desc->nb_components; |
| 292 | + // Convert smoothing value to history_len (a count of frames to average, |
| 293 | + // must be at least 1). Currently this is a direct assignment, but the |
| 294 | + // smoothing value was originally envisaged as a number of seconds. In |
| 295 | + // future it would be nice to set history_len using a number of seconds, |
| 296 | + // but VFR video is currently an obstacle to doing so. |
| 297 | + s->history_len = s->smoothing + 1; |
| 298 | + // Allocate the history buffers -- there are 6 -- one for each extrema. |
| 299 | + // s->smoothing is limited to INT_MAX/8, so that (s->history_len * 6) |
| 300 | + // can't overflow on 32bit causing a too-small allocation. |
| 301 | + s->history_mem = av_malloc(s->history_len * 6); |
| 302 | + if (s->history_mem == NULL) |
| 303 | + return AVERROR(ENOMEM); |
| 304 | + |
| 305 | + for (c = 0; c < 3; c++) { |
| 306 | + s->min[c].history = s->history_mem + (c*2) * s->history_len; |
| 307 | + s->max[c].history = s->history_mem + (c*2+1) * s->history_len; |
| 308 | + } |
| 309 | + return 0; |
| 310 | +} |
| 311 | + |
| 312 | +// Free any memory allocations here |
| 313 | +static av_cold void uninit(AVFilterContext *ctx) |
| 314 | +{ |
| 315 | + NormalizeContext *s = ctx->priv; |
| 316 | + |
| 317 | + av_freep(&s->history_mem); |
| 318 | +} |
| 319 | + |
| 320 | +// This function is pretty much standard from doc/writing_filters.txt. It |
| 321 | +// tries to do in-place filtering where possible, only allocating a new output |
| 322 | +// frame when absolutely necessary. |
| 323 | +static int filter_frame(AVFilterLink *inlink, AVFrame *in) |
| 324 | +{ |
| 325 | + AVFilterContext *ctx = inlink->dst; |
| 326 | + AVFilterLink *outlink = ctx->outputs[0]; |
| 327 | + NormalizeContext *s = ctx->priv; |
| 328 | + AVFrame *out; |
| 329 | + // Set 'direct' if we can modify the input frame in-place. Otherwise we |
| 330 | + // need to retrieve a new frame from the output link. |
| 331 | + int direct = av_frame_is_writable(in) && !ctx->is_disabled; |
| 332 | + |
| 333 | + if (direct) { |
| 334 | + out = in; |
| 335 | + } else { |
| 336 | + out = ff_get_video_buffer(outlink, outlink->w, outlink->h); |
| 337 | + if (!out) { |
| 338 | + av_frame_free(&in); |
| 339 | + return AVERROR(ENOMEM); |
| 340 | + } |
| 341 | + av_frame_copy_props(out, in); |
| 342 | + } |
| 343 | + |
| 344 | + // Now we've got the input and output frames (which may be the same frame) |
| 345 | + // perform the filtering with our custom function. |
| 346 | + normalize(s, in, out); |
| 347 | + |
| 348 | + if (ctx->is_disabled) { |
| 349 | + av_frame_free(&out); |
| 350 | + return ff_filter_frame(outlink, in); |
| 351 | + } |
| 352 | + |
| 353 | + if (!direct) |
| 354 | + av_frame_free(&in); |
| 355 | + |
| 356 | + return ff_filter_frame(outlink, out); |
| 357 | +} |
| 358 | + |
| 359 | +static const AVFilterPad inputs[] = { |
| 360 | + { |
| 361 | + .name = "default", |
| 362 | + .type = AVMEDIA_TYPE_VIDEO, |
| 363 | + .filter_frame = filter_frame, |
| 364 | + .config_props = config_input, |
| 365 | + }, |
| 366 | + { NULL } |
| 367 | +}; |
| 368 | + |
| 369 | +static const AVFilterPad outputs[] = { |
| 370 | + { |
| 371 | + .name = "default", |
| 372 | + .type = AVMEDIA_TYPE_VIDEO, |
| 373 | + }, |
| 374 | + { NULL } |
| 375 | +}; |
| 376 | + |
| 377 | +AVFilter ff_vf_normalize = { |
| 378 | + .name = "normalize", |
| 379 | + .description = NULL_IF_CONFIG_SMALL("Normalize RGB video."), |
| 380 | + .priv_size = sizeof(NormalizeContext), |
| 381 | + .priv_class = &normalize_class, |
| 382 | + .uninit = uninit, |
| 383 | + .query_formats = query_formats, |
| 384 | + .inputs = inputs, |
| 385 | + .outputs = outputs, |
| 386 | +}; |