| @@ -0,0 +1,811 @@ |
| 1 | +/* |
| 2 | + * Copyright (C) 2017 Paul B Mahol |
| 3 | + * Copyright (C) 2013-2015 Andreas Fuchs, Wolfgang Hrauda |
| 4 | + * This file is part of FFmpeg. |
| 5 | + * |
| 6 | + * FFmpeg is free software; you can redistribute it and/or |
| 7 | + * modify it under the terms of the GNU Lesser General Public |
| 8 | + * License as published by the Free Software Foundation; either |
| 9 | + * version 2.1 of the License, or (at your option) any later version. |
| 10 | + * |
| 11 | + * FFmpeg is distributed in the hope that it will be useful, |
| 12 | + * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 13 | + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
| 14 | + * Lesser General Public License for more details. |
| 15 | + * |
| 16 | + * You should have received a copy of the GNU Lesser General Public |
| 17 | + * License along with FFmpeg; if not, write to the Free Software |
| 18 | + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA |
| 19 | + */ |
| 20 | + |
| 21 | +#include <math.h> |
| 22 | + |
| 23 | +#include "libavutil/audio_fifo.h" |
| 24 | +#include "libavutil/avstring.h" |
| 25 | +#include "libavutil/channel_layout.h" |
| 26 | +#include "libavutil/float_dsp.h" |
| 27 | +#include "libavutil/intmath.h" |
| 28 | +#include "libavutil/opt.h" |
| 29 | +#include "libavcodec/avfft.h" |
| 30 | + |
| 31 | +#include "avfilter.h" |
| 32 | +#include "internal.h" |
| 33 | +#include "audio.h" |
| 34 | + |
| 35 | +#define TIME_DOMAIN 0 |
| 36 | +#define FREQUENCY_DOMAIN 1 |
| 37 | + |
| 38 | +typedef struct HeadphoneContext { |
| 39 | + const AVClass *class; |
| 40 | + |
| 41 | + char *map; |
| 42 | + int type; |
| 43 | + |
| 44 | + int lfe_channel; |
| 45 | + |
| 46 | + int have_hrirs; |
| 47 | + int eof_hrirs; |
| 48 | + int64_t pts; |
| 49 | + |
| 50 | + int ir_len; |
| 51 | + |
| 52 | + int mapping[64]; |
| 53 | + |
| 54 | + int nb_inputs; |
| 55 | + |
| 56 | + int nb_irs; |
| 57 | + |
| 58 | + float gain; |
| 59 | + float lfe_gain, gain_lfe; |
| 60 | + |
| 61 | + float *ringbuffer[2]; |
| 62 | + int write[2]; |
| 63 | + |
| 64 | + int buffer_length; |
| 65 | + int n_fft; |
| 66 | + int size; |
| 67 | + |
| 68 | + int *delay[2]; |
| 69 | + float *data_ir[2]; |
| 70 | + float *temp_src[2]; |
| 71 | + FFTComplex *temp_fft[2]; |
| 72 | + |
| 73 | + FFTContext *fft[2], *ifft[2]; |
| 74 | + FFTComplex *data_hrtf[2]; |
| 75 | + |
| 76 | + AVFloatDSPContext *fdsp; |
| 77 | + struct headphone_inputs { |
| 78 | + AVAudioFifo *fifo; |
| 79 | + AVFrame *frame; |
| 80 | + int ir_len; |
| 81 | + int delay_l; |
| 82 | + int delay_r; |
| 83 | + int eof; |
| 84 | + } *in; |
| 85 | +} HeadphoneContext; |
| 86 | + |
| 87 | +static int parse_channel_name(HeadphoneContext *s, int x, char **arg, int *rchannel, char *buf) |
| 88 | +{ |
| 89 | + int len, i, channel_id = 0; |
| 90 | + int64_t layout, layout0; |
| 91 | + |
| 92 | + if (sscanf(*arg, "%7[A-Z]%n", buf, &len)) { |
| 93 | + layout0 = layout = av_get_channel_layout(buf); |
| 94 | + if (layout == AV_CH_LOW_FREQUENCY) |
| 95 | + s->lfe_channel = x; |
| 96 | + for (i = 32; i > 0; i >>= 1) { |
| 97 | + if (layout >= 1LL << i) { |
| 98 | + channel_id += i; |
| 99 | + layout >>= i; |
| 100 | + } |
| 101 | + } |
| 102 | + if (channel_id >= 64 || layout0 != 1LL << channel_id) |
| 103 | + return AVERROR(EINVAL); |
| 104 | + *rchannel = channel_id; |
| 105 | + *arg += len; |
| 106 | + return 0; |
| 107 | + } |
| 108 | + return AVERROR(EINVAL); |
| 109 | +} |
| 110 | + |
| 111 | +static void parse_map(AVFilterContext *ctx) |
| 112 | +{ |
| 113 | + HeadphoneContext *s = ctx->priv; |
| 114 | + char *arg, *tokenizer, *p, *args = av_strdup(s->map); |
| 115 | + int i; |
| 116 | + |
| 117 | + if (!args) |
| 118 | + return; |
| 119 | + p = args; |
| 120 | + |
| 121 | + s->lfe_channel = -1; |
| 122 | + s->nb_inputs = 1; |
| 123 | + |
| 124 | + for (i = 0; i < 64; i++) { |
| 125 | + s->mapping[i] = -1; |
| 126 | + } |
| 127 | + |
| 128 | + while ((arg = av_strtok(p, "|", &tokenizer))) { |
| 129 | + int out_ch_id; |
| 130 | + char buf[8]; |
| 131 | + |
| 132 | + p = NULL; |
| 133 | + if (parse_channel_name(s, s->nb_inputs - 1, &arg, &out_ch_id, buf)) { |
| 134 | + av_log(ctx, AV_LOG_WARNING, "Failed to parse \'%s\' as channel name.\n", buf); |
| 135 | + continue; |
| 136 | + } |
| 137 | + s->mapping[s->nb_inputs - 1] = out_ch_id; |
| 138 | + s->nb_inputs++; |
| 139 | + } |
| 140 | + s->nb_irs = s->nb_inputs - 1; |
| 141 | + |
| 142 | + av_free(args); |
| 143 | +} |
| 144 | + |
| 145 | +typedef struct ThreadData { |
| 146 | + AVFrame *in, *out; |
| 147 | + int *write; |
| 148 | + int **delay; |
| 149 | + float **ir; |
| 150 | + int *n_clippings; |
| 151 | + float **ringbuffer; |
| 152 | + float **temp_src; |
| 153 | + FFTComplex **temp_fft; |
| 154 | +} ThreadData; |
| 155 | + |
| 156 | +static int headphone_convolute(AVFilterContext *ctx, void *arg, int jobnr, int nb_jobs) |
| 157 | +{ |
| 158 | + HeadphoneContext *s = ctx->priv; |
| 159 | + ThreadData *td = arg; |
| 160 | + AVFrame *in = td->in, *out = td->out; |
| 161 | + int offset = jobnr; |
| 162 | + int *write = &td->write[jobnr]; |
| 163 | + const int *const delay = td->delay[jobnr]; |
| 164 | + const float *const ir = td->ir[jobnr]; |
| 165 | + int *n_clippings = &td->n_clippings[jobnr]; |
| 166 | + float *ringbuffer = td->ringbuffer[jobnr]; |
| 167 | + float *temp_src = td->temp_src[jobnr]; |
| 168 | + const int ir_len = s->ir_len; |
| 169 | + const float *src = (const float *)in->data[0]; |
| 170 | + float *dst = (float *)out->data[0]; |
| 171 | + const int in_channels = in->channels; |
| 172 | + const int buffer_length = s->buffer_length; |
| 173 | + const uint32_t modulo = (uint32_t)buffer_length - 1; |
| 174 | + float *buffer[16]; |
| 175 | + int wr = *write; |
| 176 | + int read; |
| 177 | + int i, l; |
| 178 | + |
| 179 | + dst += offset; |
| 180 | + for (l = 0; l < in_channels; l++) { |
| 181 | + buffer[l] = ringbuffer + l * buffer_length; |
| 182 | + } |
| 183 | + |
| 184 | + for (i = 0; i < in->nb_samples; i++) { |
| 185 | + const float *temp_ir = ir; |
| 186 | + |
| 187 | + *dst = 0; |
| 188 | + for (l = 0; l < in_channels; l++) { |
| 189 | + *(buffer[l] + wr) = src[l]; |
| 190 | + } |
| 191 | + |
| 192 | + for (l = 0; l < in_channels; l++) { |
| 193 | + const float *const bptr = buffer[l]; |
| 194 | + |
| 195 | + if (l == s->lfe_channel) { |
| 196 | + *dst += *(buffer[s->lfe_channel] + wr) * s->gain_lfe; |
| 197 | + temp_ir += FFALIGN(ir_len, 16); |
| 198 | + continue; |
| 199 | + } |
| 200 | + |
| 201 | + read = (wr - *(delay + l) - (ir_len - 1) + buffer_length) & modulo; |
| 202 | + |
| 203 | + if (read + ir_len < buffer_length) { |
| 204 | + memcpy(temp_src, bptr + read, ir_len * sizeof(*temp_src)); |
| 205 | + } else { |
| 206 | + int len = FFMIN(ir_len - (read % ir_len), buffer_length - read); |
| 207 | + |
| 208 | + memcpy(temp_src, bptr + read, len * sizeof(*temp_src)); |
| 209 | + memcpy(temp_src + len, bptr, (ir_len - len) * sizeof(*temp_src)); |
| 210 | + } |
| 211 | + |
| 212 | + dst[0] += s->fdsp->scalarproduct_float(temp_ir, temp_src, ir_len); |
| 213 | + temp_ir += FFALIGN(ir_len, 16); |
| 214 | + } |
| 215 | + |
| 216 | + if (fabs(*dst) > 1) |
| 217 | + *n_clippings += 1; |
| 218 | + |
| 219 | + dst += 2; |
| 220 | + src += in_channels; |
| 221 | + wr = (wr + 1) & modulo; |
| 222 | + } |
| 223 | + |
| 224 | + *write = wr; |
| 225 | + |
| 226 | + return 0; |
| 227 | +} |
| 228 | + |
| 229 | +static int headphone_fast_convolute(AVFilterContext *ctx, void *arg, int jobnr, int nb_jobs) |
| 230 | +{ |
| 231 | + HeadphoneContext *s = ctx->priv; |
| 232 | + ThreadData *td = arg; |
| 233 | + AVFrame *in = td->in, *out = td->out; |
| 234 | + int offset = jobnr; |
| 235 | + int *write = &td->write[jobnr]; |
| 236 | + FFTComplex *hrtf = s->data_hrtf[jobnr]; |
| 237 | + int *n_clippings = &td->n_clippings[jobnr]; |
| 238 | + float *ringbuffer = td->ringbuffer[jobnr]; |
| 239 | + const int ir_len = s->ir_len; |
| 240 | + const float *src = (const float *)in->data[0]; |
| 241 | + float *dst = (float *)out->data[0]; |
| 242 | + const int in_channels = in->channels; |
| 243 | + const int buffer_length = s->buffer_length; |
| 244 | + const uint32_t modulo = (uint32_t)buffer_length - 1; |
| 245 | + FFTComplex *fft_in = s->temp_fft[jobnr]; |
| 246 | + FFTContext *ifft = s->ifft[jobnr]; |
| 247 | + FFTContext *fft = s->fft[jobnr]; |
| 248 | + const int n_fft = s->n_fft; |
| 249 | + const float fft_scale = 1.0f / s->n_fft; |
| 250 | + FFTComplex *hrtf_offset; |
| 251 | + int wr = *write; |
| 252 | + int n_read; |
| 253 | + int i, j; |
| 254 | + |
| 255 | + dst += offset; |
| 256 | + |
| 257 | + n_read = FFMIN(s->ir_len, in->nb_samples); |
| 258 | + for (j = 0; j < n_read; j++) { |
| 259 | + dst[2 * j] = ringbuffer[wr]; |
| 260 | + ringbuffer[wr] = 0.0; |
| 261 | + wr = (wr + 1) & modulo; |
| 262 | + } |
| 263 | + |
| 264 | + for (j = n_read; j < in->nb_samples; j++) { |
| 265 | + dst[2 * j] = 0; |
| 266 | + } |
| 267 | + |
| 268 | + for (i = 0; i < in_channels; i++) { |
| 269 | + if (i == s->lfe_channel) { |
| 270 | + for (j = 0; j < in->nb_samples; j++) { |
| 271 | + dst[2 * j] += src[i + j * in_channels] * s->gain_lfe; |
| 272 | + } |
| 273 | + continue; |
| 274 | + } |
| 275 | + |
| 276 | + offset = i * n_fft; |
| 277 | + hrtf_offset = hrtf + offset; |
| 278 | + |
| 279 | + memset(fft_in, 0, sizeof(FFTComplex) * n_fft); |
| 280 | + |
| 281 | + for (j = 0; j < in->nb_samples; j++) { |
| 282 | + fft_in[j].re = src[j * in_channels + i]; |
| 283 | + } |
| 284 | + |
| 285 | + av_fft_permute(fft, fft_in); |
| 286 | + av_fft_calc(fft, fft_in); |
| 287 | + for (j = 0; j < n_fft; j++) { |
| 288 | + const FFTComplex *hcomplex = hrtf_offset + j; |
| 289 | + const float re = fft_in[j].re; |
| 290 | + const float im = fft_in[j].im; |
| 291 | + |
| 292 | + fft_in[j].re = re * hcomplex->re - im * hcomplex->im; |
| 293 | + fft_in[j].im = re * hcomplex->im + im * hcomplex->re; |
| 294 | + } |
| 295 | + |
| 296 | + av_fft_permute(ifft, fft_in); |
| 297 | + av_fft_calc(ifft, fft_in); |
| 298 | + |
| 299 | + for (j = 0; j < in->nb_samples; j++) { |
| 300 | + dst[2 * j] += fft_in[j].re * fft_scale; |
| 301 | + } |
| 302 | + |
| 303 | + for (j = 0; j < ir_len - 1; j++) { |
| 304 | + int write_pos = (wr + j) & modulo; |
| 305 | + |
| 306 | + *(ringbuffer + write_pos) += fft_in[in->nb_samples + j].re * fft_scale; |
| 307 | + } |
| 308 | + } |
| 309 | + |
| 310 | + for (i = 0; i < out->nb_samples; i++) { |
| 311 | + if (fabs(*dst) > 1) { |
| 312 | + n_clippings[0]++; |
| 313 | + } |
| 314 | + |
| 315 | + dst += 2; |
| 316 | + } |
| 317 | + |
| 318 | + *write = wr; |
| 319 | + |
| 320 | + return 0; |
| 321 | +} |
| 322 | + |
| 323 | +static int read_ir(AVFilterLink *inlink, AVFrame *frame) |
| 324 | +{ |
| 325 | + AVFilterContext *ctx = inlink->dst; |
| 326 | + HeadphoneContext *s = ctx->priv; |
| 327 | + int ir_len, max_ir_len, input_number; |
| 328 | + |
| 329 | + for (input_number = 0; input_number < s->nb_inputs; input_number++) |
| 330 | + if (inlink == ctx->inputs[input_number]) |
| 331 | + break; |
| 332 | + |
| 333 | + av_audio_fifo_write(s->in[input_number].fifo, (void **)frame->extended_data, |
| 334 | + frame->nb_samples); |
| 335 | + av_frame_free(&frame); |
| 336 | + |
| 337 | + ir_len = av_audio_fifo_size(s->in[input_number].fifo); |
| 338 | + max_ir_len = 4096; |
| 339 | + if (ir_len > max_ir_len) { |
| 340 | + av_log(ctx, AV_LOG_ERROR, "Too big length of IRs: %d > %d.\n", ir_len, max_ir_len); |
| 341 | + return AVERROR(EINVAL); |
| 342 | + } |
| 343 | + s->in[input_number].ir_len = ir_len; |
| 344 | + s->ir_len = FFMAX(ir_len, s->ir_len); |
| 345 | + |
| 346 | + return 0; |
| 347 | +} |
| 348 | + |
| 349 | +static int headphone_frame(HeadphoneContext *s, AVFilterLink *outlink) |
| 350 | +{ |
| 351 | + AVFilterContext *ctx = outlink->src; |
| 352 | + AVFrame *in = s->in[0].frame; |
| 353 | + int n_clippings[2] = { 0 }; |
| 354 | + ThreadData td; |
| 355 | + AVFrame *out; |
| 356 | + |
| 357 | + av_audio_fifo_read(s->in[0].fifo, (void **)in->extended_data, s->size); |
| 358 | + |
| 359 | + out = ff_get_audio_buffer(outlink, in->nb_samples); |
| 360 | + if (!out) { |
| 361 | + av_frame_free(&in); |
| 362 | + return AVERROR(ENOMEM); |
| 363 | + } |
| 364 | + out->pts = s->pts; |
| 365 | + if (s->pts != AV_NOPTS_VALUE) |
| 366 | + s->pts += av_rescale_q(out->nb_samples, (AVRational){1, outlink->sample_rate}, outlink->time_base); |
| 367 | + |
| 368 | + td.in = in; td.out = out; td.write = s->write; |
| 369 | + td.delay = s->delay; td.ir = s->data_ir; td.n_clippings = n_clippings; |
| 370 | + td.ringbuffer = s->ringbuffer; td.temp_src = s->temp_src; |
| 371 | + td.temp_fft = s->temp_fft; |
| 372 | + |
| 373 | + if (s->type == TIME_DOMAIN) { |
| 374 | + ctx->internal->execute(ctx, headphone_convolute, &td, NULL, 2); |
| 375 | + } else { |
| 376 | + ctx->internal->execute(ctx, headphone_fast_convolute, &td, NULL, 2); |
| 377 | + } |
| 378 | + emms_c(); |
| 379 | + |
| 380 | + if (n_clippings[0] + n_clippings[1] > 0) { |
| 381 | + av_log(ctx, AV_LOG_WARNING, "%d of %d samples clipped. Please reduce gain.\n", |
| 382 | + n_clippings[0] + n_clippings[1], out->nb_samples * 2); |
| 383 | + } |
| 384 | + |
| 385 | + return ff_filter_frame(outlink, out); |
| 386 | +} |
| 387 | + |
| 388 | +static int convert_coeffs(AVFilterContext *ctx, AVFilterLink *inlink) |
| 389 | +{ |
| 390 | + struct HeadphoneContext *s = ctx->priv; |
| 391 | + const int ir_len = s->ir_len; |
| 392 | + int nb_irs = s->nb_irs; |
| 393 | + int nb_input_channels = ctx->inputs[0]->channels; |
| 394 | + float gain_lin = expf((s->gain - 3 * nb_input_channels) / 20 * M_LN10); |
| 395 | + FFTComplex *data_hrtf_l = NULL; |
| 396 | + FFTComplex *data_hrtf_r = NULL; |
| 397 | + FFTComplex *fft_in_l = NULL; |
| 398 | + FFTComplex *fft_in_r = NULL; |
| 399 | + float *data_ir_l = NULL; |
| 400 | + float *data_ir_r = NULL; |
| 401 | + int offset = 0; |
| 402 | + int n_fft; |
| 403 | + int i, j; |
| 404 | + |
| 405 | + s->buffer_length = 1 << (32 - ff_clz(s->ir_len)); |
| 406 | + s->n_fft = n_fft = 1 << (32 - ff_clz(s->ir_len + inlink->sample_rate)); |
| 407 | + |
| 408 | + if (s->type == FREQUENCY_DOMAIN) { |
| 409 | + fft_in_l = av_calloc(n_fft, sizeof(*fft_in_l)); |
| 410 | + fft_in_r = av_calloc(n_fft, sizeof(*fft_in_r)); |
| 411 | + if (!fft_in_l || !fft_in_r) { |
| 412 | + return AVERROR(ENOMEM); |
| 413 | + } |
| 414 | + |
| 415 | + av_fft_end(s->fft[0]); |
| 416 | + av_fft_end(s->fft[1]); |
| 417 | + s->fft[0] = av_fft_init(log2(s->n_fft), 0); |
| 418 | + s->fft[1] = av_fft_init(log2(s->n_fft), 0); |
| 419 | + av_fft_end(s->ifft[0]); |
| 420 | + av_fft_end(s->ifft[1]); |
| 421 | + s->ifft[0] = av_fft_init(log2(s->n_fft), 1); |
| 422 | + s->ifft[1] = av_fft_init(log2(s->n_fft), 1); |
| 423 | + |
| 424 | + if (!s->fft[0] || !s->fft[1] || !s->ifft[0] || !s->ifft[1]) { |
| 425 | + av_log(ctx, AV_LOG_ERROR, "Unable to create FFT contexts of size %d.\n", s->n_fft); |
| 426 | + return AVERROR(ENOMEM); |
| 427 | + } |
| 428 | + } |
| 429 | + |
| 430 | + s->data_ir[0] = av_calloc(FFALIGN(s->ir_len, 16), sizeof(float) * s->nb_irs); |
| 431 | + s->data_ir[1] = av_calloc(FFALIGN(s->ir_len, 16), sizeof(float) * s->nb_irs); |
| 432 | + s->delay[0] = av_malloc_array(s->nb_irs, sizeof(float)); |
| 433 | + s->delay[1] = av_malloc_array(s->nb_irs, sizeof(float)); |
| 434 | + |
| 435 | + if (s->type == TIME_DOMAIN) { |
| 436 | + s->ringbuffer[0] = av_calloc(s->buffer_length, sizeof(float) * nb_input_channels); |
| 437 | + s->ringbuffer[1] = av_calloc(s->buffer_length, sizeof(float) * nb_input_channels); |
| 438 | + } else { |
| 439 | + s->ringbuffer[0] = av_calloc(s->buffer_length, sizeof(float)); |
| 440 | + s->ringbuffer[1] = av_calloc(s->buffer_length, sizeof(float)); |
| 441 | + s->temp_fft[0] = av_malloc_array(s->n_fft, sizeof(FFTComplex)); |
| 442 | + s->temp_fft[1] = av_malloc_array(s->n_fft, sizeof(FFTComplex)); |
| 443 | + if (!s->temp_fft[0] || !s->temp_fft[1]) |
| 444 | + return AVERROR(ENOMEM); |
| 445 | + } |
| 446 | + |
| 447 | + if (!s->data_ir[0] || !s->data_ir[1] || |
| 448 | + !s->ringbuffer[0] || !s->ringbuffer[1]) |
| 449 | + return AVERROR(ENOMEM); |
| 450 | + |
| 451 | + s->in[0].frame = ff_get_audio_buffer(ctx->inputs[0], s->size); |
| 452 | + if (!s->in[0].frame) |
| 453 | + return AVERROR(ENOMEM); |
| 454 | + for (i = 0; i < s->nb_irs; i++) { |
| 455 | + s->in[i + 1].frame = ff_get_audio_buffer(ctx->inputs[i + 1], s->ir_len); |
| 456 | + if (!s->in[i + 1].frame) |
| 457 | + return AVERROR(ENOMEM); |
| 458 | + } |
| 459 | + |
| 460 | + if (s->type == TIME_DOMAIN) { |
| 461 | + s->temp_src[0] = av_calloc(FFALIGN(ir_len, 16), sizeof(float)); |
| 462 | + s->temp_src[1] = av_calloc(FFALIGN(ir_len, 16), sizeof(float)); |
| 463 | + |
| 464 | + data_ir_l = av_calloc(nb_irs * FFALIGN(ir_len, 16), sizeof(*data_ir_l)); |
| 465 | + data_ir_r = av_calloc(nb_irs * FFALIGN(ir_len, 16), sizeof(*data_ir_r)); |
| 466 | + if (!data_ir_r || !data_ir_l || !s->temp_src[0] || !s->temp_src[1]) { |
| 467 | + av_free(data_ir_l); |
| 468 | + av_free(data_ir_r); |
| 469 | + return AVERROR(ENOMEM); |
| 470 | + } |
| 471 | + } else { |
| 472 | + data_hrtf_l = av_malloc_array(n_fft, sizeof(*data_hrtf_l) * nb_irs); |
| 473 | + data_hrtf_r = av_malloc_array(n_fft, sizeof(*data_hrtf_r) * nb_irs); |
| 474 | + if (!data_hrtf_r || !data_hrtf_l) { |
| 475 | + av_free(data_hrtf_l); |
| 476 | + av_free(data_hrtf_r); |
| 477 | + return AVERROR(ENOMEM); |
| 478 | + } |
| 479 | + } |
| 480 | + |
| 481 | + for (i = 0; i < s->nb_irs; i++) { |
| 482 | + int len = s->in[i + 1].ir_len; |
| 483 | + int delay_l = s->in[i + 1].delay_l; |
| 484 | + int delay_r = s->in[i + 1].delay_r; |
| 485 | + int idx = -1; |
| 486 | + float *ptr; |
| 487 | + |
| 488 | + for (j = 0; j < inlink->channels; j++) { |
| 489 | + if (s->mapping[i] < 0) { |
| 490 | + continue; |
| 491 | + } |
| 492 | + |
| 493 | + if ((av_channel_layout_extract_channel(inlink->channel_layout, j)) == (1LL << s->mapping[i])) { |
| 494 | + idx = j; |
| 495 | + break; |
| 496 | + } |
| 497 | + } |
| 498 | + if (idx == -1) |
| 499 | + continue; |
| 500 | + |
| 501 | + av_audio_fifo_read(s->in[i + 1].fifo, (void **)s->in[i + 1].frame->extended_data, len); |
| 502 | + ptr = (float *)s->in[i + 1].frame->extended_data[0]; |
| 503 | + |
| 504 | + if (s->type == TIME_DOMAIN) { |
| 505 | + offset = idx * FFALIGN(len, 16); |
| 506 | + for (j = 0; j < len; j++) { |
| 507 | + data_ir_l[offset + j] = ptr[len * 2 - j * 2 - 2] * gain_lin; |
| 508 | + data_ir_r[offset + j] = ptr[len * 2 - j * 2 - 1] * gain_lin; |
| 509 | + } |
| 510 | + } else { |
| 511 | + memset(fft_in_l, 0, n_fft * sizeof(*fft_in_l)); |
| 512 | + memset(fft_in_r, 0, n_fft * sizeof(*fft_in_r)); |
| 513 | + |
| 514 | + offset = idx * n_fft; |
| 515 | + for (j = 0; j < len; j++) { |
| 516 | + fft_in_l[delay_l + j].re = ptr[j * 2 ] * gain_lin; |
| 517 | + fft_in_r[delay_r + j].re = ptr[j * 2 + 1] * gain_lin; |
| 518 | + } |
| 519 | + |
| 520 | + av_fft_permute(s->fft[0], fft_in_l); |
| 521 | + av_fft_calc(s->fft[0], fft_in_l); |
| 522 | + memcpy(data_hrtf_l + offset, fft_in_l, n_fft * sizeof(*fft_in_l)); |
| 523 | + av_fft_permute(s->fft[0], fft_in_r); |
| 524 | + av_fft_calc(s->fft[0], fft_in_r); |
| 525 | + memcpy(data_hrtf_r + offset, fft_in_r, n_fft * sizeof(*fft_in_r)); |
| 526 | + } |
| 527 | + } |
| 528 | + |
| 529 | + if (s->type == TIME_DOMAIN) { |
| 530 | + memcpy(s->data_ir[0], data_ir_l, sizeof(float) * nb_irs * FFALIGN(ir_len, 16)); |
| 531 | + memcpy(s->data_ir[1], data_ir_r, sizeof(float) * nb_irs * FFALIGN(ir_len, 16)); |
| 532 | + |
| 533 | + av_freep(&data_ir_l); |
| 534 | + av_freep(&data_ir_r); |
| 535 | + } else { |
| 536 | + s->data_hrtf[0] = av_malloc_array(n_fft * s->nb_irs, sizeof(FFTComplex)); |
| 537 | + s->data_hrtf[1] = av_malloc_array(n_fft * s->nb_irs, sizeof(FFTComplex)); |
| 538 | + if (!s->data_hrtf[0] || !s->data_hrtf[1]) { |
| 539 | + av_freep(&data_hrtf_l); |
| 540 | + av_freep(&data_hrtf_r); |
| 541 | + av_freep(&fft_in_l); |
| 542 | + av_freep(&fft_in_r); |
| 543 | + return AVERROR(ENOMEM); |
| 544 | + } |
| 545 | + |
| 546 | + memcpy(s->data_hrtf[0], data_hrtf_l, |
| 547 | + sizeof(FFTComplex) * nb_irs * n_fft); |
| 548 | + memcpy(s->data_hrtf[1], data_hrtf_r, |
| 549 | + sizeof(FFTComplex) * nb_irs * n_fft); |
| 550 | + |
| 551 | + av_freep(&data_hrtf_l); |
| 552 | + av_freep(&data_hrtf_r); |
| 553 | + |
| 554 | + av_freep(&fft_in_l); |
| 555 | + av_freep(&fft_in_r); |
| 556 | + } |
| 557 | + |
| 558 | + s->have_hrirs = 1; |
| 559 | + |
| 560 | + return 0; |
| 561 | +} |
| 562 | + |
| 563 | +static int filter_frame(AVFilterLink *inlink, AVFrame *in) |
| 564 | +{ |
| 565 | + AVFilterContext *ctx = inlink->dst; |
| 566 | + HeadphoneContext *s = ctx->priv; |
| 567 | + AVFilterLink *outlink = ctx->outputs[0]; |
| 568 | + int ret = 0; |
| 569 | + |
| 570 | + av_audio_fifo_write(s->in[0].fifo, (void **)in->extended_data, |
| 571 | + in->nb_samples); |
| 572 | + if (s->pts == AV_NOPTS_VALUE) |
| 573 | + s->pts = in->pts; |
| 574 | + |
| 575 | + av_frame_free(&in); |
| 576 | + |
| 577 | + if (!s->have_hrirs && s->eof_hrirs) { |
| 578 | + ret = convert_coeffs(ctx, inlink); |
| 579 | + if (ret < 0) |
| 580 | + return ret; |
| 581 | + } |
| 582 | + |
| 583 | + if (s->have_hrirs) { |
| 584 | + while (av_audio_fifo_size(s->in[0].fifo) >= s->size) { |
| 585 | + ret = headphone_frame(s, outlink); |
| 586 | + if (ret < 0) |
| 587 | + break; |
| 588 | + } |
| 589 | + } |
| 590 | + return ret; |
| 591 | +} |
| 592 | + |
| 593 | +static int query_formats(AVFilterContext *ctx) |
| 594 | +{ |
| 595 | + struct HeadphoneContext *s = ctx->priv; |
| 596 | + AVFilterFormats *formats = NULL; |
| 597 | + AVFilterChannelLayouts *layouts = NULL; |
| 598 | + int ret, i; |
| 599 | + |
| 600 | + ret = ff_add_format(&formats, AV_SAMPLE_FMT_FLT); |
| 601 | + if (ret) |
| 602 | + return ret; |
| 603 | + ret = ff_set_common_formats(ctx, formats); |
| 604 | + if (ret) |
| 605 | + return ret; |
| 606 | + |
| 607 | + layouts = ff_all_channel_layouts(); |
| 608 | + if (!layouts) |
| 609 | + return AVERROR(ENOMEM); |
| 610 | + |
| 611 | + ret = ff_channel_layouts_ref(layouts, &ctx->inputs[0]->out_channel_layouts); |
| 612 | + if (ret) |
| 613 | + return ret; |
| 614 | + |
| 615 | + layouts = NULL; |
| 616 | + ret = ff_add_channel_layout(&layouts, AV_CH_LAYOUT_STEREO); |
| 617 | + if (ret) |
| 618 | + return ret; |
| 619 | + |
| 620 | + for (i = 1; i < s->nb_inputs; i++) { |
| 621 | + ret = ff_channel_layouts_ref(layouts, &ctx->inputs[i]->out_channel_layouts); |
| 622 | + if (ret) |
| 623 | + return ret; |
| 624 | + } |
| 625 | + |
| 626 | + ret = ff_channel_layouts_ref(layouts, &ctx->outputs[0]->in_channel_layouts); |
| 627 | + if (ret) |
| 628 | + return ret; |
| 629 | + |
| 630 | + formats = ff_all_samplerates(); |
| 631 | + if (!formats) |
| 632 | + return AVERROR(ENOMEM); |
| 633 | + return ff_set_common_samplerates(ctx, formats); |
| 634 | +} |
| 635 | + |
| 636 | +static int config_input(AVFilterLink *inlink) |
| 637 | +{ |
| 638 | + AVFilterContext *ctx = inlink->dst; |
| 639 | + HeadphoneContext *s = ctx->priv; |
| 640 | + |
| 641 | + if (s->type == FREQUENCY_DOMAIN) { |
| 642 | + inlink->partial_buf_size = |
| 643 | + inlink->min_samples = |
| 644 | + inlink->max_samples = inlink->sample_rate; |
| 645 | + } |
| 646 | + |
| 647 | + if (s->nb_irs < inlink->channels) { |
| 648 | + av_log(ctx, AV_LOG_ERROR, "Number of inputs must be >= %d.\n", inlink->channels + 1); |
| 649 | + return AVERROR(EINVAL); |
| 650 | + } |
| 651 | + |
| 652 | + return 0; |
| 653 | +} |
| 654 | + |
| 655 | +static av_cold int init(AVFilterContext *ctx) |
| 656 | +{ |
| 657 | + HeadphoneContext *s = ctx->priv; |
| 658 | + int i; |
| 659 | + |
| 660 | + AVFilterPad pad = { |
| 661 | + .name = "in0", |
| 662 | + .type = AVMEDIA_TYPE_AUDIO, |
| 663 | + .config_props = config_input, |
| 664 | + .filter_frame = filter_frame, |
| 665 | + }; |
| 666 | + ff_insert_inpad(ctx, 0, &pad); |
| 667 | + |
| 668 | + if (!s->map) { |
| 669 | + av_log(ctx, AV_LOG_ERROR, "Valid mapping must be set.\n"); |
| 670 | + return AVERROR(EINVAL); |
| 671 | + } |
| 672 | + |
| 673 | + parse_map(ctx); |
| 674 | + |
| 675 | + s->in = av_calloc(s->nb_inputs, sizeof(*s->in)); |
| 676 | + if (!s->in) |
| 677 | + return AVERROR(ENOMEM); |
| 678 | + |
| 679 | + for (i = 1; i < s->nb_inputs; i++) { |
| 680 | + char *name = av_asprintf("hrir%d", i - 1); |
| 681 | + AVFilterPad pad = { |
| 682 | + .name = name, |
| 683 | + .type = AVMEDIA_TYPE_AUDIO, |
| 684 | + .filter_frame = read_ir, |
| 685 | + }; |
| 686 | + if (!name) |
| 687 | + return AVERROR(ENOMEM); |
| 688 | + ff_insert_inpad(ctx, i, &pad); |
| 689 | + } |
| 690 | + |
| 691 | + s->fdsp = avpriv_float_dsp_alloc(0); |
| 692 | + if (!s->fdsp) |
| 693 | + return AVERROR(ENOMEM); |
| 694 | + s->pts = AV_NOPTS_VALUE; |
| 695 | + |
| 696 | + return 0; |
| 697 | +} |
| 698 | + |
| 699 | +static int config_output(AVFilterLink *outlink) |
| 700 | +{ |
| 701 | + AVFilterContext *ctx = outlink->src; |
| 702 | + HeadphoneContext *s = ctx->priv; |
| 703 | + AVFilterLink *inlink = ctx->inputs[0]; |
| 704 | + int i; |
| 705 | + |
| 706 | + if (s->type == TIME_DOMAIN) |
| 707 | + s->size = 1024; |
| 708 | + else |
| 709 | + s->size = inlink->sample_rate; |
| 710 | + |
| 711 | + for (i = 0; i < s->nb_inputs; i++) { |
| 712 | + s->in[i].fifo = av_audio_fifo_alloc(ctx->inputs[i]->format, ctx->inputs[i]->channels, 1024); |
| 713 | + if (!s->in[i].fifo) |
| 714 | + return AVERROR(ENOMEM); |
| 715 | + } |
| 716 | + s->gain_lfe = expf((s->gain - 3 * inlink->channels - 6 + s->lfe_gain) / 20 * M_LN10); |
| 717 | + |
| 718 | + return 0; |
| 719 | +} |
| 720 | + |
| 721 | +static int request_frame(AVFilterLink *outlink) |
| 722 | +{ |
| 723 | + AVFilterContext *ctx = outlink->src; |
| 724 | + HeadphoneContext *s = ctx->priv; |
| 725 | + int i, ret; |
| 726 | + |
| 727 | + for (i = 1; !s->eof_hrirs && i < s->nb_inputs; i++) { |
| 728 | + if (!s->in[i].eof) { |
| 729 | + ret = ff_request_frame(ctx->inputs[i]); |
| 730 | + if (ret == AVERROR_EOF) { |
| 731 | + s->in[i].eof = 1; |
| 732 | + ret = 0; |
| 733 | + } |
| 734 | + return ret; |
| 735 | + } else { |
| 736 | + if (i == s->nb_inputs - 1) |
| 737 | + s->eof_hrirs = 1; |
| 738 | + } |
| 739 | + } |
| 740 | + return ff_request_frame(ctx->inputs[0]); |
| 741 | +} |
| 742 | + |
| 743 | +static av_cold void uninit(AVFilterContext *ctx) |
| 744 | +{ |
| 745 | + HeadphoneContext *s = ctx->priv; |
| 746 | + int i; |
| 747 | + |
| 748 | + av_fft_end(s->ifft[0]); |
| 749 | + av_fft_end(s->ifft[1]); |
| 750 | + av_fft_end(s->fft[0]); |
| 751 | + av_fft_end(s->fft[1]); |
| 752 | + av_freep(&s->delay[0]); |
| 753 | + av_freep(&s->delay[1]); |
| 754 | + av_freep(&s->data_ir[0]); |
| 755 | + av_freep(&s->data_ir[1]); |
| 756 | + av_freep(&s->ringbuffer[0]); |
| 757 | + av_freep(&s->ringbuffer[1]); |
| 758 | + av_freep(&s->temp_src[0]); |
| 759 | + av_freep(&s->temp_src[1]); |
| 760 | + av_freep(&s->temp_fft[0]); |
| 761 | + av_freep(&s->temp_fft[1]); |
| 762 | + av_freep(&s->data_hrtf[0]); |
| 763 | + av_freep(&s->data_hrtf[1]); |
| 764 | + av_freep(&s->fdsp); |
| 765 | + |
| 766 | + for (i = 0; i < s->nb_inputs; i++) { |
| 767 | + av_frame_free(&s->in[i].frame); |
| 768 | + av_audio_fifo_free(s->in[i].fifo); |
| 769 | + if (ctx->input_pads && i) |
| 770 | + av_freep(&ctx->input_pads[i].name); |
| 771 | + } |
| 772 | + av_freep(&s->in); |
| 773 | +} |
| 774 | + |
| 775 | +#define OFFSET(x) offsetof(HeadphoneContext, x) |
| 776 | +#define FLAGS AV_OPT_FLAG_AUDIO_PARAM|AV_OPT_FLAG_FILTERING_PARAM |
| 777 | + |
| 778 | +static const AVOption headphone_options[] = { |
| 779 | + { "map", "set channels convolution mappings", OFFSET(map), AV_OPT_TYPE_STRING, {.str=NULL}, .flags = FLAGS }, |
| 780 | + { "gain", "set gain in dB", OFFSET(gain), AV_OPT_TYPE_FLOAT, {.dbl=0}, -20, 40, .flags = FLAGS }, |
| 781 | + { "lfe", "set lfe gain in dB", OFFSET(lfe_gain), AV_OPT_TYPE_FLOAT, {.dbl=0}, -20, 40, .flags = FLAGS }, |
| 782 | + { "type", "set processing", OFFSET(type), AV_OPT_TYPE_INT, {.i64=1}, 0, 1, .flags = FLAGS, "type" }, |
| 783 | + { "time", "time domain", 0, AV_OPT_TYPE_CONST, {.i64=0}, 0, 0, .flags = FLAGS, "type" }, |
| 784 | + { "freq", "frequency domain", 0, AV_OPT_TYPE_CONST, {.i64=1}, 0, 0, .flags = FLAGS, "type" }, |
| 785 | + { NULL } |
| 786 | +}; |
| 787 | + |
| 788 | +AVFILTER_DEFINE_CLASS(headphone); |
| 789 | + |
| 790 | +static const AVFilterPad outputs[] = { |
| 791 | + { |
| 792 | + .name = "default", |
| 793 | + .type = AVMEDIA_TYPE_AUDIO, |
| 794 | + .config_props = config_output, |
| 795 | + .request_frame = request_frame, |
| 796 | + }, |
| 797 | + { NULL } |
| 798 | +}; |
| 799 | + |
| 800 | +AVFilter ff_af_headphone = { |
| 801 | + .name = "headphone", |
| 802 | + .description = NULL_IF_CONFIG_SMALL("Apply headphone binaural spatialization with HRTFs in additional streams."), |
| 803 | + .priv_size = sizeof(HeadphoneContext), |
| 804 | + .priv_class = &headphone_class, |
| 805 | + .init = init, |
| 806 | + .uninit = uninit, |
| 807 | + .query_formats = query_formats, |
| 808 | + .inputs = NULL, |
| 809 | + .outputs = outputs, |
| 810 | + .flags = AVFILTER_FLAG_SLICE_THREADS | AVFILTER_FLAG_DYNAMIC_INPUTS, |
| 811 | +}; |