| @@ -0,0 +1,375 @@ |
| 1 | +/* |
| 2 | + * Copyright (c) 2019 Paul B Mahol |
| 3 | + * |
| 4 | + * This file is part of FFmpeg. |
| 5 | + * |
| 6 | + * FFmpeg is free software; you can redistribute it and/or |
| 7 | + * modify it under the terms of the GNU Lesser General Public |
| 8 | + * License as published by the Free Software Foundation; either |
| 9 | + * version 2.1 of the License, or (at your option) any later version. |
| 10 | + * |
| 11 | + * FFmpeg is distributed in the hope that it will be useful, |
| 12 | + * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 13 | + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
| 14 | + * Lesser General Public License for more details. |
| 15 | + * |
| 16 | + * You should have received a copy of the GNU Lesser General Public |
| 17 | + * License along with FFmpeg; if not, write to the Free Software |
| 18 | + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA |
| 19 | + */ |
| 20 | + |
| 21 | +#include <float.h> |
| 22 | +#include <math.h> |
| 23 | + |
| 24 | +#include "libavcodec/avfft.h" |
| 25 | +#include "libavutil/audio_fifo.h" |
| 26 | +#include "libavutil/avassert.h" |
| 27 | +#include "libavutil/channel_layout.h" |
| 28 | +#include "libavutil/opt.h" |
| 29 | +#include "libavutil/parseutils.h" |
| 30 | +#include "audio.h" |
| 31 | +#include "video.h" |
| 32 | +#include "avfilter.h" |
| 33 | +#include "filters.h" |
| 34 | +#include "internal.h" |
| 35 | +#include "window_func.h" |
| 36 | + |
| 37 | +typedef struct ShowSpatialContext { |
| 38 | + const AVClass *class; |
| 39 | + int w, h; |
| 40 | + AVRational frame_rate; |
| 41 | + FFTContext *fft[2]; ///< Fast Fourier Transform context |
| 42 | + FFTContext *ifft[2]; ///< Inverse Fast Fourier Transform context |
| 43 | + int fft_bits; ///< number of bits (FFT window size = 1<<fft_bits) |
| 44 | + FFTComplex *fft_data[2]; ///< bins holder for each (displayed) channels |
| 45 | + float *window_func_lut; ///< Window function LUT |
| 46 | + int win_func; |
| 47 | + int win_size; |
| 48 | + int buf_size; |
| 49 | + float overlap; |
| 50 | + int consumed; |
| 51 | + int hop_size; |
| 52 | + AVAudioFifo *fifo; |
| 53 | + int64_t pts; |
| 54 | +} ShowSpatialContext; |
| 55 | + |
| 56 | +#define OFFSET(x) offsetof(ShowSpatialContext, x) |
| 57 | +#define FLAGS AV_OPT_FLAG_FILTERING_PARAM|AV_OPT_FLAG_VIDEO_PARAM |
| 58 | + |
| 59 | +static const AVOption showspatial_options[] = { |
| 60 | + { "size", "set video size", OFFSET(w), AV_OPT_TYPE_IMAGE_SIZE, {.str = "512x512"}, 0, 0, FLAGS }, |
| 61 | + { "s", "set video size", OFFSET(w), AV_OPT_TYPE_IMAGE_SIZE, {.str = "512x512"}, 0, 0, FLAGS }, |
| 62 | + { "win_size", "set window size", OFFSET(win_size), AV_OPT_TYPE_INT, {.i64 = 4096}, 1024, 65536, FLAGS }, |
| 63 | + { "win_func", "set window function", OFFSET(win_func), AV_OPT_TYPE_INT, {.i64 = WFUNC_HANNING}, 0, NB_WFUNC-1, FLAGS, "win_func" }, |
| 64 | + { "rect", "Rectangular", 0, AV_OPT_TYPE_CONST, {.i64=WFUNC_RECT}, 0, 0, FLAGS, "win_func" }, |
| 65 | + { "bartlett", "Bartlett", 0, AV_OPT_TYPE_CONST, {.i64=WFUNC_BARTLETT}, 0, 0, FLAGS, "win_func" }, |
| 66 | + { "hann", "Hann", 0, AV_OPT_TYPE_CONST, {.i64=WFUNC_HANNING}, 0, 0, FLAGS, "win_func" }, |
| 67 | + { "hanning", "Hanning", 0, AV_OPT_TYPE_CONST, {.i64=WFUNC_HANNING}, 0, 0, FLAGS, "win_func" }, |
| 68 | + { "hamming", "Hamming", 0, AV_OPT_TYPE_CONST, {.i64=WFUNC_HAMMING}, 0, 0, FLAGS, "win_func" }, |
| 69 | + { "blackman", "Blackman", 0, AV_OPT_TYPE_CONST, {.i64=WFUNC_BLACKMAN}, 0, 0, FLAGS, "win_func" }, |
| 70 | + { "welch", "Welch", 0, AV_OPT_TYPE_CONST, {.i64=WFUNC_WELCH}, 0, 0, FLAGS, "win_func" }, |
| 71 | + { "flattop", "Flat-top", 0, AV_OPT_TYPE_CONST, {.i64=WFUNC_FLATTOP}, 0, 0, FLAGS, "win_func" }, |
| 72 | + { "bharris", "Blackman-Harris", 0, AV_OPT_TYPE_CONST, {.i64=WFUNC_BHARRIS}, 0, 0, FLAGS, "win_func" }, |
| 73 | + { "bnuttall", "Blackman-Nuttall", 0, AV_OPT_TYPE_CONST, {.i64=WFUNC_BNUTTALL}, 0, 0, FLAGS, "win_func" }, |
| 74 | + { "bhann", "Bartlett-Hann", 0, AV_OPT_TYPE_CONST, {.i64=WFUNC_BHANN}, 0, 0, FLAGS, "win_func" }, |
| 75 | + { "sine", "Sine", 0, AV_OPT_TYPE_CONST, {.i64=WFUNC_SINE}, 0, 0, FLAGS, "win_func" }, |
| 76 | + { "nuttall", "Nuttall", 0, AV_OPT_TYPE_CONST, {.i64=WFUNC_NUTTALL}, 0, 0, FLAGS, "win_func" }, |
| 77 | + { "lanczos", "Lanczos", 0, AV_OPT_TYPE_CONST, {.i64=WFUNC_LANCZOS}, 0, 0, FLAGS, "win_func" }, |
| 78 | + { "gauss", "Gauss", 0, AV_OPT_TYPE_CONST, {.i64=WFUNC_GAUSS}, 0, 0, FLAGS, "win_func" }, |
| 79 | + { "tukey", "Tukey", 0, AV_OPT_TYPE_CONST, {.i64=WFUNC_TUKEY}, 0, 0, FLAGS, "win_func" }, |
| 80 | + { "dolph", "Dolph-Chebyshev", 0, AV_OPT_TYPE_CONST, {.i64=WFUNC_DOLPH}, 0, 0, FLAGS, "win_func" }, |
| 81 | + { "cauchy", "Cauchy", 0, AV_OPT_TYPE_CONST, {.i64=WFUNC_CAUCHY}, 0, 0, FLAGS, "win_func" }, |
| 82 | + { "parzen", "Parzen", 0, AV_OPT_TYPE_CONST, {.i64=WFUNC_PARZEN}, 0, 0, FLAGS, "win_func" }, |
| 83 | + { "poisson", "Poisson", 0, AV_OPT_TYPE_CONST, {.i64=WFUNC_POISSON}, 0, 0, FLAGS, "win_func" }, |
| 84 | + { "bohman", "Bohman", 0, AV_OPT_TYPE_CONST, {.i64=WFUNC_BOHMAN}, 0, 0, FLAGS, "win_func" }, |
| 85 | + { "overlap", "set window overlap", OFFSET(overlap), AV_OPT_TYPE_FLOAT, {.dbl=0.5}, 0, 1, FLAGS }, |
| 86 | + { NULL } |
| 87 | +}; |
| 88 | + |
| 89 | +AVFILTER_DEFINE_CLASS(showspatial); |
| 90 | + |
| 91 | +static av_cold void uninit(AVFilterContext *ctx) |
| 92 | +{ |
| 93 | + ShowSpatialContext *s = ctx->priv; |
| 94 | + int i; |
| 95 | + |
| 96 | + for (i = 0; i < 2; i++) |
| 97 | + av_fft_end(s->fft[i]); |
| 98 | + for (i = 0; i < 2; i++) |
| 99 | + av_fft_end(s->ifft[i]); |
| 100 | + for (i = 0; i < 2; i++) |
| 101 | + av_freep(&s->fft_data[i]); |
| 102 | + av_freep(&s->window_func_lut); |
| 103 | + av_audio_fifo_free(s->fifo); |
| 104 | +} |
| 105 | + |
| 106 | +static int query_formats(AVFilterContext *ctx) |
| 107 | +{ |
| 108 | + AVFilterFormats *formats = NULL; |
| 109 | + AVFilterChannelLayouts *layout = NULL; |
| 110 | + AVFilterLink *inlink = ctx->inputs[0]; |
| 111 | + AVFilterLink *outlink = ctx->outputs[0]; |
| 112 | + static const enum AVSampleFormat sample_fmts[] = { AV_SAMPLE_FMT_FLTP, AV_SAMPLE_FMT_NONE }; |
| 113 | + static const enum AVPixelFormat pix_fmts[] = { AV_PIX_FMT_GBRP, AV_PIX_FMT_NONE }; |
| 114 | + int ret; |
| 115 | + |
| 116 | + formats = ff_make_format_list(sample_fmts); |
| 117 | + if ((ret = ff_formats_ref (formats, &inlink->out_formats )) < 0 || |
| 118 | + (ret = ff_add_channel_layout (&layout, AV_CH_LAYOUT_STEREO )) < 0 || |
| 119 | + (ret = ff_channel_layouts_ref (layout , &inlink->out_channel_layouts)) < 0) |
| 120 | + return ret; |
| 121 | + |
| 122 | + formats = ff_all_samplerates(); |
| 123 | + if ((ret = ff_formats_ref(formats, &inlink->out_samplerates)) < 0) |
| 124 | + return ret; |
| 125 | + |
| 126 | + formats = ff_make_format_list(pix_fmts); |
| 127 | + if ((ret = ff_formats_ref(formats, &outlink->in_formats)) < 0) |
| 128 | + return ret; |
| 129 | + |
| 130 | + return 0; |
| 131 | +} |
| 132 | + |
| 133 | +static int run_channel_fft(AVFilterContext *ctx, void *arg, int jobnr, int nb_jobs) |
| 134 | +{ |
| 135 | + ShowSpatialContext *s = ctx->priv; |
| 136 | + const float *window_func_lut = s->window_func_lut; |
| 137 | + AVFrame *fin = arg; |
| 138 | + const int ch = jobnr; |
| 139 | + const float *p = (float *)fin->extended_data[ch]; |
| 140 | + |
| 141 | + for (int n = 0; n < fin->nb_samples; n++) { |
| 142 | + s->fft_data[ch][n].re = p[n] * window_func_lut[n]; |
| 143 | + s->fft_data[ch][n].im = 0; |
| 144 | + } |
| 145 | + |
| 146 | + av_fft_permute(s->fft[ch], s->fft_data[ch]); |
| 147 | + av_fft_calc(s->fft[ch], s->fft_data[ch]); |
| 148 | + |
| 149 | + return 0; |
| 150 | +} |
| 151 | + |
| 152 | +static int config_output(AVFilterLink *outlink) |
| 153 | +{ |
| 154 | + AVFilterContext *ctx = outlink->src; |
| 155 | + AVFilterLink *inlink = ctx->inputs[0]; |
| 156 | + ShowSpatialContext *s = ctx->priv; |
| 157 | + int i, fft_bits, h, w; |
| 158 | + float overlap; |
| 159 | + |
| 160 | + outlink->w = s->w; |
| 161 | + outlink->h = s->h; |
| 162 | + outlink->sample_aspect_ratio = (AVRational){1,1}; |
| 163 | + |
| 164 | + h = s->h; |
| 165 | + w = s->w; |
| 166 | + |
| 167 | + s->buf_size = 1 << av_log2(s->win_size); |
| 168 | + s->win_size = s->buf_size; |
| 169 | + fft_bits = av_log2(s->win_size); |
| 170 | + |
| 171 | + /* (re-)configuration if the video output changed (or first init) */ |
| 172 | + if (fft_bits != s->fft_bits) { |
| 173 | + s->fft_bits = fft_bits; |
| 174 | + |
| 175 | + /* FFT buffers: x2 for each channel buffer. |
| 176 | + * Note: we use free and malloc instead of a realloc-like function to |
| 177 | + * make sure the buffer is aligned in memory for the FFT functions. */ |
| 178 | + for (i = 0; i < 2; i++) { |
| 179 | + av_fft_end(s->fft[i]); |
| 180 | + av_freep(&s->fft_data[i]); |
| 181 | + } |
| 182 | + for (i = 0; i < 2; i++) { |
| 183 | + s->fft[i] = av_fft_init(fft_bits, 0); |
| 184 | + if (!s->fft[i]) { |
| 185 | + av_log(ctx, AV_LOG_ERROR, "Unable to create FFT context. " |
| 186 | + "The window size might be too high.\n"); |
| 187 | + return AVERROR(EINVAL); |
| 188 | + } |
| 189 | + } |
| 190 | + |
| 191 | + for (i = 0; i < 2; i++) { |
| 192 | + s->fft_data[i] = av_calloc(s->buf_size, sizeof(**s->fft_data)); |
| 193 | + if (!s->fft_data[i]) |
| 194 | + return AVERROR(ENOMEM); |
| 195 | + } |
| 196 | + |
| 197 | + /* pre-calc windowing function */ |
| 198 | + s->window_func_lut = |
| 199 | + av_realloc_f(s->window_func_lut, s->win_size, |
| 200 | + sizeof(*s->window_func_lut)); |
| 201 | + if (!s->window_func_lut) |
| 202 | + return AVERROR(ENOMEM); |
| 203 | + generate_window_func(s->window_func_lut, s->win_size, s->win_func, &overlap); |
| 204 | + if (s->overlap == 1) |
| 205 | + s->overlap = overlap; |
| 206 | + |
| 207 | + s->hop_size = (1.f - s->overlap) * s->win_size; |
| 208 | + if (s->hop_size < 1) { |
| 209 | + av_log(ctx, AV_LOG_ERROR, "overlap %f too big\n", s->overlap); |
| 210 | + return AVERROR(EINVAL); |
| 211 | + } |
| 212 | + } |
| 213 | + |
| 214 | + outlink->time_base = av_inv_q(outlink->frame_rate); |
| 215 | + |
| 216 | + av_audio_fifo_free(s->fifo); |
| 217 | + s->fifo = av_audio_fifo_alloc(inlink->format, inlink->channels, s->win_size); |
| 218 | + if (!s->fifo) |
| 219 | + return AVERROR(ENOMEM); |
| 220 | + return 0; |
| 221 | +} |
| 222 | + |
| 223 | +#define RE(y, ch) s->fft_data[ch][y].re |
| 224 | +#define IM(y, ch) s->fft_data[ch][y].im |
| 225 | + |
| 226 | +static void draw_dot(uint8_t *dst, int linesize, int value) |
| 227 | +{ |
| 228 | + dst[0] = value; |
| 229 | + dst[1] = value; |
| 230 | + dst[-1] = value; |
| 231 | + dst[linesize] = value; |
| 232 | + dst[-linesize] = value; |
| 233 | +} |
| 234 | + |
| 235 | +static int draw_spatial(AVFilterLink *inlink, AVFrame *insamples) |
| 236 | +{ |
| 237 | + AVFilterContext *ctx = inlink->dst; |
| 238 | + AVFilterLink *outlink = ctx->outputs[0]; |
| 239 | + ShowSpatialContext *s = ctx->priv; |
| 240 | + AVFrame *outpicref; |
| 241 | + int h = s->h - 2; |
| 242 | + int w = s->w - 2; |
| 243 | + int z = s->win_size / 2; |
| 244 | + |
| 245 | + outpicref = ff_get_video_buffer(outlink, outlink->w, outlink->h); |
| 246 | + if (!outpicref) |
| 247 | + return AVERROR(ENOMEM); |
| 248 | + |
| 249 | + outpicref->sample_aspect_ratio = (AVRational){1,1}; |
| 250 | + for (int i = 0; i < outlink->h; i++) { |
| 251 | + memset(outpicref->data[0] + i * outpicref->linesize[0], 0, outlink->w); |
| 252 | + memset(outpicref->data[1] + i * outpicref->linesize[1], 0, outlink->w); |
| 253 | + memset(outpicref->data[2] + i * outpicref->linesize[2], 0, outlink->w); |
| 254 | + } |
| 255 | + |
| 256 | + for (int j = 0; j < z; j++) { |
| 257 | + const int idx = z - 1 - j; |
| 258 | + float l = hypotf(RE(idx, 0), IM(idx, 0)); |
| 259 | + float r = hypotf(RE(idx, 1), IM(idx, 1)); |
| 260 | + float sum = l + r; |
| 261 | + float lp = atan2f(IM(idx, 0), RE(idx, 0)); |
| 262 | + float rp = atan2f(IM(idx, 1), RE(idx, 1)); |
| 263 | + float diffp = ((rp - lp) / (2.f * M_PI) + 1.f) * 0.5f; |
| 264 | + float diff = (sum < 0.000001f ? 0.f : (r - l) / sum) * 0.5f + 0.5f; |
| 265 | + float cr = av_clipf(cbrtf(l / sum), 0, 1) * 255.f; |
| 266 | + float cb = av_clipf(cbrtf(r / sum), 0, 1) * 255.f; |
| 267 | + float cg; |
| 268 | + int x, y; |
| 269 | + |
| 270 | + cg = diffp * 255.f; |
| 271 | + x = av_clip(w * diff, 0, w - 2) + 1; |
| 272 | + y = av_clip(h * diffp, 0, h - 2) + 1; |
| 273 | + |
| 274 | + draw_dot(outpicref->data[0] + outpicref->linesize[0] * y + x, outpicref->linesize[0], cg); |
| 275 | + draw_dot(outpicref->data[1] + outpicref->linesize[1] * y + x, outpicref->linesize[1], cb); |
| 276 | + draw_dot(outpicref->data[2] + outpicref->linesize[2] * y + x, outpicref->linesize[2], cr); |
| 277 | + } |
| 278 | + |
| 279 | + outpicref->pts = av_rescale_q(insamples->pts, inlink->time_base, outlink->time_base); |
| 280 | + |
| 281 | + return ff_filter_frame(outlink, outpicref); |
| 282 | +} |
| 283 | + |
| 284 | +static int spatial_activate(AVFilterContext *ctx) |
| 285 | +{ |
| 286 | + AVFilterLink *inlink = ctx->inputs[0]; |
| 287 | + AVFilterLink *outlink = ctx->outputs[0]; |
| 288 | + ShowSpatialContext *s = ctx->priv; |
| 289 | + int ret; |
| 290 | + |
| 291 | + FF_FILTER_FORWARD_STATUS_BACK(outlink, inlink); |
| 292 | + |
| 293 | + if (av_audio_fifo_size(s->fifo) < s->win_size) { |
| 294 | + AVFrame *frame = NULL; |
| 295 | + |
| 296 | + ret = ff_inlink_consume_frame(inlink, &frame); |
| 297 | + if (ret < 0) |
| 298 | + return ret; |
| 299 | + if (ret > 0) { |
| 300 | + s->pts = frame->pts; |
| 301 | + s->consumed = 0; |
| 302 | + |
| 303 | + av_audio_fifo_write(s->fifo, (void **)frame->extended_data, frame->nb_samples); |
| 304 | + av_frame_free(&frame); |
| 305 | + } |
| 306 | + } |
| 307 | + |
| 308 | + if (av_audio_fifo_size(s->fifo) >= s->win_size) { |
| 309 | + AVFrame *fin = ff_get_audio_buffer(inlink, s->win_size); |
| 310 | + if (!fin) |
| 311 | + return AVERROR(ENOMEM); |
| 312 | + |
| 313 | + fin->pts = s->pts + s->consumed; |
| 314 | + s->consumed += s->hop_size; |
| 315 | + ret = av_audio_fifo_peek(s->fifo, (void **)fin->extended_data, |
| 316 | + FFMIN(s->win_size, av_audio_fifo_size(s->fifo))); |
| 317 | + if (ret < 0) { |
| 318 | + av_frame_free(&fin); |
| 319 | + return ret; |
| 320 | + } |
| 321 | + |
| 322 | + av_assert0(fin->nb_samples == s->win_size); |
| 323 | + |
| 324 | + ctx->internal->execute(ctx, run_channel_fft, fin, NULL, 2); |
| 325 | + |
| 326 | + ret = draw_spatial(inlink, fin); |
| 327 | + |
| 328 | + av_frame_free(&fin); |
| 329 | + av_audio_fifo_drain(s->fifo, s->hop_size); |
| 330 | + if (ret <= 0) |
| 331 | + return ret; |
| 332 | + } |
| 333 | + |
| 334 | + FF_FILTER_FORWARD_STATUS(inlink, outlink); |
| 335 | + if (ff_outlink_frame_wanted(outlink) && av_audio_fifo_size(s->fifo) < s->win_size) { |
| 336 | + ff_inlink_request_frame(inlink); |
| 337 | + return 0; |
| 338 | + } |
| 339 | + |
| 340 | + if (av_audio_fifo_size(s->fifo) >= s->win_size) { |
| 341 | + ff_filter_set_ready(ctx, 10); |
| 342 | + return 0; |
| 343 | + } |
| 344 | + return FFERROR_NOT_READY; |
| 345 | +} |
| 346 | + |
| 347 | +static const AVFilterPad showspatial_inputs[] = { |
| 348 | + { |
| 349 | + .name = "default", |
| 350 | + .type = AVMEDIA_TYPE_AUDIO, |
| 351 | + }, |
| 352 | + { NULL } |
| 353 | +}; |
| 354 | + |
| 355 | +static const AVFilterPad showspatial_outputs[] = { |
| 356 | + { |
| 357 | + .name = "default", |
| 358 | + .type = AVMEDIA_TYPE_VIDEO, |
| 359 | + .config_props = config_output, |
| 360 | + }, |
| 361 | + { NULL } |
| 362 | +}; |
| 363 | + |
| 364 | +AVFilter ff_avf_showspatial = { |
| 365 | + .name = "showspatial", |
| 366 | + .description = NULL_IF_CONFIG_SMALL("Convert input audio to a spatial video output."), |
| 367 | + .uninit = uninit, |
| 368 | + .query_formats = query_formats, |
| 369 | + .priv_size = sizeof(ShowSpatialContext), |
| 370 | + .inputs = showspatial_inputs, |
| 371 | + .outputs = showspatial_outputs, |
| 372 | + .activate = spatial_activate, |
| 373 | + .priv_class = &showspatial_class, |
| 374 | + .flags = AVFILTER_FLAG_SLICE_THREADS, |
| 375 | +}; |