| @@ -0,0 +1,375 @@ |
| | 1 | /* |
| | 2 | * Copyright (c) 2019 Paul B Mahol |
| | 3 | * |
| | 4 | * This file is part of FFmpeg. |
| | 5 | * |
| | 6 | * FFmpeg is free software; you can redistribute it and/or |
| | 7 | * modify it under the terms of the GNU Lesser General Public |
| | 8 | * License as published by the Free Software Foundation; either |
| | 9 | * version 2.1 of the License, or (at your option) any later version. |
| | 10 | * |
| | 11 | * FFmpeg is distributed in the hope that it will be useful, |
| | 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| | 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
| | 14 | * Lesser General Public License for more details. |
| | 15 | * |
| | 16 | * You should have received a copy of the GNU Lesser General Public |
| | 17 | * License along with FFmpeg; if not, write to the Free Software |
| | 18 | * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA |
| | 19 | */ |
| | 20 | |
| | 21 | #include <float.h> |
| | 22 | #include <math.h> |
| | 23 | |
| | 24 | #include "libavcodec/avfft.h" |
| | 25 | #include "libavutil/audio_fifo.h" |
| | 26 | #include "libavutil/avassert.h" |
| | 27 | #include "libavutil/channel_layout.h" |
| | 28 | #include "libavutil/opt.h" |
| | 29 | #include "libavutil/parseutils.h" |
| | 30 | #include "audio.h" |
| | 31 | #include "video.h" |
| | 32 | #include "avfilter.h" |
| | 33 | #include "filters.h" |
| | 34 | #include "internal.h" |
| | 35 | #include "window_func.h" |
| | 36 | |
| | 37 | typedef struct ShowSpatialContext { |
| | 38 | const AVClass *class; |
| | 39 | int w, h; |
| | 40 | AVRational frame_rate; |
| | 41 | FFTContext *fft[2]; ///< Fast Fourier Transform context |
| | 42 | FFTContext *ifft[2]; ///< Inverse Fast Fourier Transform context |
| | 43 | int fft_bits; ///< number of bits (FFT window size = 1<<fft_bits) |
| | 44 | FFTComplex *fft_data[2]; ///< bins holder for each (displayed) channels |
| | 45 | float *window_func_lut; ///< Window function LUT |
| | 46 | int win_func; |
| | 47 | int win_size; |
| | 48 | int buf_size; |
| | 49 | float overlap; |
| | 50 | int consumed; |
| | 51 | int hop_size; |
| | 52 | AVAudioFifo *fifo; |
| | 53 | int64_t pts; |
| | 54 | } ShowSpatialContext; |
| | 55 | |
| | 56 | #define OFFSET(x) offsetof(ShowSpatialContext, x) |
| | 57 | #define FLAGS AV_OPT_FLAG_FILTERING_PARAM|AV_OPT_FLAG_VIDEO_PARAM |
| | 58 | |
| | 59 | static const AVOption showspatial_options[] = { |
| | 60 | { "size", "set video size", OFFSET(w), AV_OPT_TYPE_IMAGE_SIZE, {.str = "512x512"}, 0, 0, FLAGS }, |
| | 61 | { "s", "set video size", OFFSET(w), AV_OPT_TYPE_IMAGE_SIZE, {.str = "512x512"}, 0, 0, FLAGS }, |
| | 62 | { "win_size", "set window size", OFFSET(win_size), AV_OPT_TYPE_INT, {.i64 = 4096}, 1024, 65536, FLAGS }, |
| | 63 | { "win_func", "set window function", OFFSET(win_func), AV_OPT_TYPE_INT, {.i64 = WFUNC_HANNING}, 0, NB_WFUNC-1, FLAGS, "win_func" }, |
| | 64 | { "rect", "Rectangular", 0, AV_OPT_TYPE_CONST, {.i64=WFUNC_RECT}, 0, 0, FLAGS, "win_func" }, |
| | 65 | { "bartlett", "Bartlett", 0, AV_OPT_TYPE_CONST, {.i64=WFUNC_BARTLETT}, 0, 0, FLAGS, "win_func" }, |
| | 66 | { "hann", "Hann", 0, AV_OPT_TYPE_CONST, {.i64=WFUNC_HANNING}, 0, 0, FLAGS, "win_func" }, |
| | 67 | { "hanning", "Hanning", 0, AV_OPT_TYPE_CONST, {.i64=WFUNC_HANNING}, 0, 0, FLAGS, "win_func" }, |
| | 68 | { "hamming", "Hamming", 0, AV_OPT_TYPE_CONST, {.i64=WFUNC_HAMMING}, 0, 0, FLAGS, "win_func" }, |
| | 69 | { "blackman", "Blackman", 0, AV_OPT_TYPE_CONST, {.i64=WFUNC_BLACKMAN}, 0, 0, FLAGS, "win_func" }, |
| | 70 | { "welch", "Welch", 0, AV_OPT_TYPE_CONST, {.i64=WFUNC_WELCH}, 0, 0, FLAGS, "win_func" }, |
| | 71 | { "flattop", "Flat-top", 0, AV_OPT_TYPE_CONST, {.i64=WFUNC_FLATTOP}, 0, 0, FLAGS, "win_func" }, |
| | 72 | { "bharris", "Blackman-Harris", 0, AV_OPT_TYPE_CONST, {.i64=WFUNC_BHARRIS}, 0, 0, FLAGS, "win_func" }, |
| | 73 | { "bnuttall", "Blackman-Nuttall", 0, AV_OPT_TYPE_CONST, {.i64=WFUNC_BNUTTALL}, 0, 0, FLAGS, "win_func" }, |
| | 74 | { "bhann", "Bartlett-Hann", 0, AV_OPT_TYPE_CONST, {.i64=WFUNC_BHANN}, 0, 0, FLAGS, "win_func" }, |
| | 75 | { "sine", "Sine", 0, AV_OPT_TYPE_CONST, {.i64=WFUNC_SINE}, 0, 0, FLAGS, "win_func" }, |
| | 76 | { "nuttall", "Nuttall", 0, AV_OPT_TYPE_CONST, {.i64=WFUNC_NUTTALL}, 0, 0, FLAGS, "win_func" }, |
| | 77 | { "lanczos", "Lanczos", 0, AV_OPT_TYPE_CONST, {.i64=WFUNC_LANCZOS}, 0, 0, FLAGS, "win_func" }, |
| | 78 | { "gauss", "Gauss", 0, AV_OPT_TYPE_CONST, {.i64=WFUNC_GAUSS}, 0, 0, FLAGS, "win_func" }, |
| | 79 | { "tukey", "Tukey", 0, AV_OPT_TYPE_CONST, {.i64=WFUNC_TUKEY}, 0, 0, FLAGS, "win_func" }, |
| | 80 | { "dolph", "Dolph-Chebyshev", 0, AV_OPT_TYPE_CONST, {.i64=WFUNC_DOLPH}, 0, 0, FLAGS, "win_func" }, |
| | 81 | { "cauchy", "Cauchy", 0, AV_OPT_TYPE_CONST, {.i64=WFUNC_CAUCHY}, 0, 0, FLAGS, "win_func" }, |
| | 82 | { "parzen", "Parzen", 0, AV_OPT_TYPE_CONST, {.i64=WFUNC_PARZEN}, 0, 0, FLAGS, "win_func" }, |
| | 83 | { "poisson", "Poisson", 0, AV_OPT_TYPE_CONST, {.i64=WFUNC_POISSON}, 0, 0, FLAGS, "win_func" }, |
| | 84 | { "bohman", "Bohman", 0, AV_OPT_TYPE_CONST, {.i64=WFUNC_BOHMAN}, 0, 0, FLAGS, "win_func" }, |
| | 85 | { "overlap", "set window overlap", OFFSET(overlap), AV_OPT_TYPE_FLOAT, {.dbl=0.5}, 0, 1, FLAGS }, |
| | 86 | { NULL } |
| | 87 | }; |
| | 88 | |
| | 89 | AVFILTER_DEFINE_CLASS(showspatial); |
| | 90 | |
| | 91 | static av_cold void uninit(AVFilterContext *ctx) |
| | 92 | { |
| | 93 | ShowSpatialContext *s = ctx->priv; |
| | 94 | int i; |
| | 95 | |
| | 96 | for (i = 0; i < 2; i++) |
| | 97 | av_fft_end(s->fft[i]); |
| | 98 | for (i = 0; i < 2; i++) |
| | 99 | av_fft_end(s->ifft[i]); |
| | 100 | for (i = 0; i < 2; i++) |
| | 101 | av_freep(&s->fft_data[i]); |
| | 102 | av_freep(&s->window_func_lut); |
| | 103 | av_audio_fifo_free(s->fifo); |
| | 104 | } |
| | 105 | |
| | 106 | static int query_formats(AVFilterContext *ctx) |
| | 107 | { |
| | 108 | AVFilterFormats *formats = NULL; |
| | 109 | AVFilterChannelLayouts *layout = NULL; |
| | 110 | AVFilterLink *inlink = ctx->inputs[0]; |
| | 111 | AVFilterLink *outlink = ctx->outputs[0]; |
| | 112 | static const enum AVSampleFormat sample_fmts[] = { AV_SAMPLE_FMT_FLTP, AV_SAMPLE_FMT_NONE }; |
| | 113 | static const enum AVPixelFormat pix_fmts[] = { AV_PIX_FMT_GBRP, AV_PIX_FMT_NONE }; |
| | 114 | int ret; |
| | 115 | |
| | 116 | formats = ff_make_format_list(sample_fmts); |
| | 117 | if ((ret = ff_formats_ref (formats, &inlink->out_formats )) < 0 || |
| | 118 | (ret = ff_add_channel_layout (&layout, AV_CH_LAYOUT_STEREO )) < 0 || |
| | 119 | (ret = ff_channel_layouts_ref (layout , &inlink->out_channel_layouts)) < 0) |
| | 120 | return ret; |
| | 121 | |
| | 122 | formats = ff_all_samplerates(); |
| | 123 | if ((ret = ff_formats_ref(formats, &inlink->out_samplerates)) < 0) |
| | 124 | return ret; |
| | 125 | |
| | 126 | formats = ff_make_format_list(pix_fmts); |
| | 127 | if ((ret = ff_formats_ref(formats, &outlink->in_formats)) < 0) |
| | 128 | return ret; |
| | 129 | |
| | 130 | return 0; |
| | 131 | } |
| | 132 | |
| | 133 | static int run_channel_fft(AVFilterContext *ctx, void *arg, int jobnr, int nb_jobs) |
| | 134 | { |
| | 135 | ShowSpatialContext *s = ctx->priv; |
| | 136 | const float *window_func_lut = s->window_func_lut; |
| | 137 | AVFrame *fin = arg; |
| | 138 | const int ch = jobnr; |
| | 139 | const float *p = (float *)fin->extended_data[ch]; |
| | 140 | |
| | 141 | for (int n = 0; n < fin->nb_samples; n++) { |
| | 142 | s->fft_data[ch][n].re = p[n] * window_func_lut[n]; |
| | 143 | s->fft_data[ch][n].im = 0; |
| | 144 | } |
| | 145 | |
| | 146 | av_fft_permute(s->fft[ch], s->fft_data[ch]); |
| | 147 | av_fft_calc(s->fft[ch], s->fft_data[ch]); |
| | 148 | |
| | 149 | return 0; |
| | 150 | } |
| | 151 | |
| | 152 | static int config_output(AVFilterLink *outlink) |
| | 153 | { |
| | 154 | AVFilterContext *ctx = outlink->src; |
| | 155 | AVFilterLink *inlink = ctx->inputs[0]; |
| | 156 | ShowSpatialContext *s = ctx->priv; |
| | 157 | int i, fft_bits, h, w; |
| | 158 | float overlap; |
| | 159 | |
| | 160 | outlink->w = s->w; |
| | 161 | outlink->h = s->h; |
| | 162 | outlink->sample_aspect_ratio = (AVRational){1,1}; |
| | 163 | |
| | 164 | h = s->h; |
| | 165 | w = s->w; |
| | 166 | |
| | 167 | s->buf_size = 1 << av_log2(s->win_size); |
| | 168 | s->win_size = s->buf_size; |
| | 169 | fft_bits = av_log2(s->win_size); |
| | 170 | |
| | 171 | /* (re-)configuration if the video output changed (or first init) */ |
| | 172 | if (fft_bits != s->fft_bits) { |
| | 173 | s->fft_bits = fft_bits; |
| | 174 | |
| | 175 | /* FFT buffers: x2 for each channel buffer. |
| | 176 | * Note: we use free and malloc instead of a realloc-like function to |
| | 177 | * make sure the buffer is aligned in memory for the FFT functions. */ |
| | 178 | for (i = 0; i < 2; i++) { |
| | 179 | av_fft_end(s->fft[i]); |
| | 180 | av_freep(&s->fft_data[i]); |
| | 181 | } |
| | 182 | for (i = 0; i < 2; i++) { |
| | 183 | s->fft[i] = av_fft_init(fft_bits, 0); |
| | 184 | if (!s->fft[i]) { |
| | 185 | av_log(ctx, AV_LOG_ERROR, "Unable to create FFT context. " |
| | 186 | "The window size might be too high.\n"); |
| | 187 | return AVERROR(EINVAL); |
| | 188 | } |
| | 189 | } |
| | 190 | |
| | 191 | for (i = 0; i < 2; i++) { |
| | 192 | s->fft_data[i] = av_calloc(s->buf_size, sizeof(**s->fft_data)); |
| | 193 | if (!s->fft_data[i]) |
| | 194 | return AVERROR(ENOMEM); |
| | 195 | } |
| | 196 | |
| | 197 | /* pre-calc windowing function */ |
| | 198 | s->window_func_lut = |
| | 199 | av_realloc_f(s->window_func_lut, s->win_size, |
| | 200 | sizeof(*s->window_func_lut)); |
| | 201 | if (!s->window_func_lut) |
| | 202 | return AVERROR(ENOMEM); |
| | 203 | generate_window_func(s->window_func_lut, s->win_size, s->win_func, &overlap); |
| | 204 | if (s->overlap == 1) |
| | 205 | s->overlap = overlap; |
| | 206 | |
| | 207 | s->hop_size = (1.f - s->overlap) * s->win_size; |
| | 208 | if (s->hop_size < 1) { |
| | 209 | av_log(ctx, AV_LOG_ERROR, "overlap %f too big\n", s->overlap); |
| | 210 | return AVERROR(EINVAL); |
| | 211 | } |
| | 212 | } |
| | 213 | |
| | 214 | outlink->time_base = av_inv_q(outlink->frame_rate); |
| | 215 | |
| | 216 | av_audio_fifo_free(s->fifo); |
| | 217 | s->fifo = av_audio_fifo_alloc(inlink->format, inlink->channels, s->win_size); |
| | 218 | if (!s->fifo) |
| | 219 | return AVERROR(ENOMEM); |
| | 220 | return 0; |
| | 221 | } |
| | 222 | |
| | 223 | #define RE(y, ch) s->fft_data[ch][y].re |
| | 224 | #define IM(y, ch) s->fft_data[ch][y].im |
| | 225 | |
| | 226 | static void draw_dot(uint8_t *dst, int linesize, int value) |
| | 227 | { |
| | 228 | dst[0] = value; |
| | 229 | dst[1] = value; |
| | 230 | dst[-1] = value; |
| | 231 | dst[linesize] = value; |
| | 232 | dst[-linesize] = value; |
| | 233 | } |
| | 234 | |
| | 235 | static int draw_spatial(AVFilterLink *inlink, AVFrame *insamples) |
| | 236 | { |
| | 237 | AVFilterContext *ctx = inlink->dst; |
| | 238 | AVFilterLink *outlink = ctx->outputs[0]; |
| | 239 | ShowSpatialContext *s = ctx->priv; |
| | 240 | AVFrame *outpicref; |
| | 241 | int h = s->h - 2; |
| | 242 | int w = s->w - 2; |
| | 243 | int z = s->win_size / 2; |
| | 244 | |
| | 245 | outpicref = ff_get_video_buffer(outlink, outlink->w, outlink->h); |
| | 246 | if (!outpicref) |
| | 247 | return AVERROR(ENOMEM); |
| | 248 | |
| | 249 | outpicref->sample_aspect_ratio = (AVRational){1,1}; |
| | 250 | for (int i = 0; i < outlink->h; i++) { |
| | 251 | memset(outpicref->data[0] + i * outpicref->linesize[0], 0, outlink->w); |
| | 252 | memset(outpicref->data[1] + i * outpicref->linesize[1], 0, outlink->w); |
| | 253 | memset(outpicref->data[2] + i * outpicref->linesize[2], 0, outlink->w); |
| | 254 | } |
| | 255 | |
| | 256 | for (int j = 0; j < z; j++) { |
| | 257 | const int idx = z - 1 - j; |
| | 258 | float l = hypotf(RE(idx, 0), IM(idx, 0)); |
| | 259 | float r = hypotf(RE(idx, 1), IM(idx, 1)); |
| | 260 | float sum = l + r; |
| | 261 | float lp = atan2f(IM(idx, 0), RE(idx, 0)); |
| | 262 | float rp = atan2f(IM(idx, 1), RE(idx, 1)); |
| | 263 | float diffp = ((rp - lp) / (2.f * M_PI) + 1.f) * 0.5f; |
| | 264 | float diff = (sum < 0.000001f ? 0.f : (r - l) / sum) * 0.5f + 0.5f; |
| | 265 | float cr = av_clipf(cbrtf(l / sum), 0, 1) * 255.f; |
| | 266 | float cb = av_clipf(cbrtf(r / sum), 0, 1) * 255.f; |
| | 267 | float cg; |
| | 268 | int x, y; |
| | 269 | |
| | 270 | cg = diffp * 255.f; |
| | 271 | x = av_clip(w * diff, 0, w - 2) + 1; |
| | 272 | y = av_clip(h * diffp, 0, h - 2) + 1; |
| | 273 | |
| | 274 | draw_dot(outpicref->data[0] + outpicref->linesize[0] * y + x, outpicref->linesize[0], cg); |
| | 275 | draw_dot(outpicref->data[1] + outpicref->linesize[1] * y + x, outpicref->linesize[1], cb); |
| | 276 | draw_dot(outpicref->data[2] + outpicref->linesize[2] * y + x, outpicref->linesize[2], cr); |
| | 277 | } |
| | 278 | |
| | 279 | outpicref->pts = av_rescale_q(insamples->pts, inlink->time_base, outlink->time_base); |
| | 280 | |
| | 281 | return ff_filter_frame(outlink, outpicref); |
| | 282 | } |
| | 283 | |
| | 284 | static int spatial_activate(AVFilterContext *ctx) |
| | 285 | { |
| | 286 | AVFilterLink *inlink = ctx->inputs[0]; |
| | 287 | AVFilterLink *outlink = ctx->outputs[0]; |
| | 288 | ShowSpatialContext *s = ctx->priv; |
| | 289 | int ret; |
| | 290 | |
| | 291 | FF_FILTER_FORWARD_STATUS_BACK(outlink, inlink); |
| | 292 | |
| | 293 | if (av_audio_fifo_size(s->fifo) < s->win_size) { |
| | 294 | AVFrame *frame = NULL; |
| | 295 | |
| | 296 | ret = ff_inlink_consume_frame(inlink, &frame); |
| | 297 | if (ret < 0) |
| | 298 | return ret; |
| | 299 | if (ret > 0) { |
| | 300 | s->pts = frame->pts; |
| | 301 | s->consumed = 0; |
| | 302 | |
| | 303 | av_audio_fifo_write(s->fifo, (void **)frame->extended_data, frame->nb_samples); |
| | 304 | av_frame_free(&frame); |
| | 305 | } |
| | 306 | } |
| | 307 | |
| | 308 | if (av_audio_fifo_size(s->fifo) >= s->win_size) { |
| | 309 | AVFrame *fin = ff_get_audio_buffer(inlink, s->win_size); |
| | 310 | if (!fin) |
| | 311 | return AVERROR(ENOMEM); |
| | 312 | |
| | 313 | fin->pts = s->pts + s->consumed; |
| | 314 | s->consumed += s->hop_size; |
| | 315 | ret = av_audio_fifo_peek(s->fifo, (void **)fin->extended_data, |
| | 316 | FFMIN(s->win_size, av_audio_fifo_size(s->fifo))); |
| | 317 | if (ret < 0) { |
| | 318 | av_frame_free(&fin); |
| | 319 | return ret; |
| | 320 | } |
| | 321 | |
| | 322 | av_assert0(fin->nb_samples == s->win_size); |
| | 323 | |
| | 324 | ctx->internal->execute(ctx, run_channel_fft, fin, NULL, 2); |
| | 325 | |
| | 326 | ret = draw_spatial(inlink, fin); |
| | 327 | |
| | 328 | av_frame_free(&fin); |
| | 329 | av_audio_fifo_drain(s->fifo, s->hop_size); |
| | 330 | if (ret <= 0) |
| | 331 | return ret; |
| | 332 | } |
| | 333 | |
| | 334 | FF_FILTER_FORWARD_STATUS(inlink, outlink); |
| | 335 | if (ff_outlink_frame_wanted(outlink) && av_audio_fifo_size(s->fifo) < s->win_size) { |
| | 336 | ff_inlink_request_frame(inlink); |
| | 337 | return 0; |
| | 338 | } |
| | 339 | |
| | 340 | if (av_audio_fifo_size(s->fifo) >= s->win_size) { |
| | 341 | ff_filter_set_ready(ctx, 10); |
| | 342 | return 0; |
| | 343 | } |
| | 344 | return FFERROR_NOT_READY; |
| | 345 | } |
| | 346 | |
| | 347 | static const AVFilterPad showspatial_inputs[] = { |
| | 348 | { |
| | 349 | .name = "default", |
| | 350 | .type = AVMEDIA_TYPE_AUDIO, |
| | 351 | }, |
| | 352 | { NULL } |
| | 353 | }; |
| | 354 | |
| | 355 | static const AVFilterPad showspatial_outputs[] = { |
| | 356 | { |
| | 357 | .name = "default", |
| | 358 | .type = AVMEDIA_TYPE_VIDEO, |
| | 359 | .config_props = config_output, |
| | 360 | }, |
| | 361 | { NULL } |
| | 362 | }; |
| | 363 | |
| | 364 | AVFilter ff_avf_showspatial = { |
| | 365 | .name = "showspatial", |
| | 366 | .description = NULL_IF_CONFIG_SMALL("Convert input audio to a spatial video output."), |
| | 367 | .uninit = uninit, |
| | 368 | .query_formats = query_formats, |
| | 369 | .priv_size = sizeof(ShowSpatialContext), |
| | 370 | .inputs = showspatial_inputs, |
| | 371 | .outputs = showspatial_outputs, |
| | 372 | .activate = spatial_activate, |
| | 373 | .priv_class = &showspatial_class, |
| | 374 | .flags = AVFILTER_FLAG_SLICE_THREADS, |
| | 375 | }; |