| @@ -129,7 +129,7 @@ typedef struct DenoiseState { · 128 unmodified lines |
| 129 | float mem_hp_x[2]; | 129 | float mem_hp_x[2]; |
| 130 | float lastg[NB_BANDS]; | 130 | float lastg[NB_BANDS]; |
| 131 | float history[FRAME_SIZE]; | 131 | float history[FRAME_SIZE]; |
| 132 | RNNState rnn; | 132 | RNNState rnn[2]; |
| 133 | AVTXContext *tx, *txi; | 133 | AVTXContext *tx, *txi; |
| 134 | av_tx_fn tx_fn, txi_fn; | 134 | av_tx_fn tx_fn, txi_fn; |
| 135 | } DenoiseState; | 135 | } DenoiseState; |
| @@ -146,7 +146,7 @@ typedef struct AudioRNNContext { · 10 unmodified lines |
| 146 | DECLARE_ALIGNED(32, float, window)[WINDOW_SIZE]; | 146 | DECLARE_ALIGNED(32, float, window)[WINDOW_SIZE]; |
| 147 | DECLARE_ALIGNED(32, float, dct_table)[FFALIGN(NB_BANDS, 4)][FFALIGN(NB_BANDS, 4)]; | 147 | DECLARE_ALIGNED(32, float, dct_table)[FFALIGN(NB_BANDS, 4)][FFALIGN(NB_BANDS, 4)]; |
| 148 | | 148 | |
| 149 | RNNModel *model; | 149 | RNNModel *model[2]; |
| 150 | | 150 | |
| 151 | AVFloatDSPContext *fdsp; | 151 | AVFloatDSPContext *fdsp; |
| 152 | } AudioRNNContext; | 152 | } AudioRNNContext; |
| @@ -350,27 +350,34 @@ static int config_input(AVFilterLink *inlink) · 197 unmodified lines |
| 350 | | 350 | |
| 351 | s->channels = inlink->channels; | 351 | s->channels = inlink->channels; |
| 352 | | 352 | |
| 353 | s->st = av_calloc(s->channels, sizeof(DenoiseState)); | 353 | if (!s->st) |
| | 354 | s->st = av_calloc(s->channels, sizeof(DenoiseState)); |
| 354 | if (!s->st) | 355 | if (!s->st) |
| 355 | return AVERROR(ENOMEM); | 356 | return AVERROR(ENOMEM); |
| 356 | | 357 | |
| 357 | for (int i = 0; i < s->channels; i++) { | 358 | for (int i = 0; i < s->channels; i++) { |
| 358 | DenoiseState *st = &s->st[i]; | 359 | DenoiseState *st = &s->st[i]; |
| 359 | | 360 | |
| 360 | st->rnn.model = s->model; | 361 | st->rnn[0].model = s->model[0]; |
| 361 | st->rnn.vad_gru_state = av_calloc(sizeof(float), FFALIGN(s->model->vad_gru_size, 16)); | 362 | st->rnn[0].vad_gru_state = av_calloc(sizeof(float), FFALIGN(s->model[0]->vad_gru_size, 16)); |
| 362 | st->rnn.noise_gru_state = av_calloc(sizeof(float), FFALIGN(s->model->noise_gru_size, 16)); | 363 | st->rnn[0].noise_gru_state = av_calloc(sizeof(float), FFALIGN(s->model[0]->noise_gru_size, 16)); |
| 363 | st->rnn.denoise_gru_state = av_calloc(sizeof(float), FFALIGN(s->model->denoise_gru_size, 16)); | 364 | st->rnn[0].denoise_gru_state = av_calloc(sizeof(float), FFALIGN(s->model[0]->denoise_gru_size, 16)); |
| 364 | if (!st->rnn.vad_gru_state || | 365 | if (!st->rnn[0].vad_gru_state || |
| 365 | !st->rnn.noise_gru_state || | 366 | !st->rnn[0].noise_gru_state || |
| 366 | !st->rnn.denoise_gru_state) | 367 | !st->rnn[0].denoise_gru_state) |
| 367 | return AVERROR(ENOMEM); | 368 | return AVERROR(ENOMEM); |
| | 369 | } |
| 368 | | 370 | |
| 369 | ret = av_tx_init(&st->tx, &st->tx_fn, AV_TX_FLOAT_FFT, 0, WINDOW_SIZE, NULL, 0); | 371 | for (int i = 0; i < s->channels; i++) { |
| | 372 | DenoiseState *st = &s->st[i]; |
| | 373 | |
| | 374 | if (!st->tx) |
| | 375 | ret = av_tx_init(&st->tx, &st->tx_fn, AV_TX_FLOAT_FFT, 0, WINDOW_SIZE, NULL, 0); |
| 370 | if (ret < 0) | 376 | if (ret < 0) |
| 371 | return ret; | 377 | return ret; |
| 372 | | 378 | |
| 373 | ret = av_tx_init(&st->txi, &st->txi_fn, AV_TX_FLOAT_FFT, 1, WINDOW_SIZE, NULL, 0); | 379 | if (!st->txi) |
| | 380 | ret = av_tx_init(&st->txi, &st->txi_fn, AV_TX_FLOAT_FFT, 1, WINDOW_SIZE, NULL, 0); |
| 374 | if (ret < 0) | 381 | if (ret < 0) |
| 375 | return ret; | 382 | return ret; |
| 376 | } | 383 | } |
| @@ -1368,7 +1375,7 @@ static float rnnoise_channel(AudioRNNContext *s, DenoiseState *st, float *out, c · 991 unmodified lines |
| 1368 | silence = compute_frame_features(s, st, X, P, Ex, Ep, Exp, features, x); | 1375 | silence = compute_frame_features(s, st, X, P, Ex, Ep, Exp, features, x); |
| 1369 | | 1376 | |
| 1370 | if (!silence && !disabled) { | 1377 | if (!silence && !disabled) { |
| 1371 | compute_rnn(s, &st->rnn, g, &vad_prob, features); | 1378 | compute_rnn(s, &st->rnn[0], g, &vad_prob, features); |
| 1372 | pitch_filter(X, P, Ex, Ep, Exp, g); | 1379 | pitch_filter(X, P, Ex, Ep, Exp, g); |
| 1373 | for (int i = 0; i < NB_BANDS; i++) { | 1380 | for (int i = 0; i < NB_BANDS; i++) { |
| 1374 | float alpha = .6f; | 1381 | float alpha = .6f; |
| @@ -1458,26 +1465,38 @@ static int activate(AVFilterContext *ctx) · 83 unmodified lines |
| 1458 | return FFERROR_NOT_READY; | 1465 | return FFERROR_NOT_READY; |
| 1459 | } | 1466 | } |
| 1460 | | 1467 | |
| 1461 | static av_cold int init(AVFilterContext *ctx) | 1468 | static int open_model(AVFilterContext *ctx, RNNModel **model) |
| 1462 | { | 1469 | { |
| 1463 | AudioRNNContext *s = ctx->priv; | 1470 | AudioRNNContext *s = ctx->priv; |
| 1464 | FILE *f; | 1471 | FILE *f; |
| 1465 | | 1472 | |
| 1466 | s->fdsp = avpriv_float_dsp_alloc(0); | | |
| 1467 | if (!s->fdsp) | | |
| 1468 | return AVERROR(ENOMEM); | | |
| 1469 | | | |
| 1470 | if (!s->model_name) | 1473 | if (!s->model_name) |
| 1471 | return AVERROR(EINVAL); | 1474 | return AVERROR(EINVAL); |
| 1472 | f = av_fopen_utf8(s->model_name, "r"); | 1475 | f = av_fopen_utf8(s->model_name, "r"); |
| 1473 | if (!f) | 1476 | if (!f) |
| 1474 | return AVERROR(EINVAL); | 1477 | return AVERROR(EINVAL); |
| 1475 | | 1478 | |
| 1476 | s->model = rnnoise_model_from_file(f); | 1479 | *model = rnnoise_model_from_file(f); |
| 1477 | fclose(f); | 1480 | fclose(f); |
| 1478 | if (!s->model) | 1481 | if (!*model) |
| 1479 | return AVERROR(EINVAL); | 1482 | return AVERROR(EINVAL); |
| 1480 | | 1483 | |
| | 1484 | return 0; |
| | 1485 | } |
| | 1486 | |
| | 1487 | static av_cold int init(AVFilterContext *ctx) |
| | 1488 | { |
| | 1489 | AudioRNNContext *s = ctx->priv; |
| | 1490 | int ret; |
| | 1491 | |
| | 1492 | s->fdsp = avpriv_float_dsp_alloc(0); |
| | 1493 | if (!s->fdsp) |
| | 1494 | return AVERROR(ENOMEM); |
| | 1495 | |
| | 1496 | ret = open_model(ctx, &s->model[0]); |
| | 1497 | if (ret < 0) |
| | 1498 | return ret; |
| | 1499 | |
| 1481 | for (int i = 0; i < FRAME_SIZE; i++) { | 1500 | for (int i = 0; i < FRAME_SIZE; i++) { |
| 1482 | s->window[i] = sin(.5*M_PI*sin(.5*M_PI*(i+.5)/FRAME_SIZE) * sin(.5*M_PI*(i+.5)/FRAME_SIZE)); | 1501 | s->window[i] = sin(.5*M_PI*sin(.5*M_PI*(i+.5)/FRAME_SIZE) * sin(.5*M_PI*(i+.5)/FRAME_SIZE)); |
| 1483 | s->window[WINDOW_SIZE - 1 - i] = s->window[i]; | 1502 | s->window[WINDOW_SIZE - 1 - i] = s->window[i]; |
| @@ -1494,22 +1513,59 @@ static av_cold int init(AVFilterContext *ctx) · 10 unmodified lines |
| 1494 | return 0; | 1513 | return 0; |
| 1495 | } | 1514 | } |
| 1496 | | 1515 | |
| | 1516 | static void free_model(AVFilterContext *ctx, int n) |
| | 1517 | { |
| | 1518 | AudioRNNContext *s = ctx->priv; |
| | 1519 | |
| | 1520 | rnnoise_model_free(s->model[n]); |
| | 1521 | s->model[n] = NULL; |
| | 1522 | |
| | 1523 | for (int ch = 0; ch < s->channels && s->st; ch++) { |
| | 1524 | av_freep(&s->st[ch].rnn[n].vad_gru_state); |
| | 1525 | av_freep(&s->st[ch].rnn[n].noise_gru_state); |
| | 1526 | av_freep(&s->st[ch].rnn[n].denoise_gru_state); |
| | 1527 | } |
| | 1528 | } |
| | 1529 | |
| | 1530 | static int process_command(AVFilterContext *ctx, const char *cmd, const char *args, |
| | 1531 | char *res, int res_len, int flags) |
| | 1532 | { |
| | 1533 | AudioRNNContext *s = ctx->priv; |
| | 1534 | int ret; |
| | 1535 | |
| | 1536 | ret = ff_filter_process_command(ctx, cmd, args, res, res_len, flags); |
| | 1537 | if (ret < 0) |
| | 1538 | return ret; |
| | 1539 | |
| | 1540 | ret = open_model(ctx, &s->model[1]); |
| | 1541 | if (ret < 0) |
| | 1542 | return ret; |
| | 1543 | |
| | 1544 | FFSWAP(RNNModel *, s->model[0], s->model[1]); |
| | 1545 | for (int ch = 0; ch < s->channels; ch++) |
| | 1546 | FFSWAP(RNNState, s->st[ch].rnn[0], s->st[ch].rnn[1]); |
| | 1547 | |
| | 1548 | ret = config_input(ctx->inputs[0]); |
| | 1549 | if (ret < 0) { |
| | 1550 | for (int ch = 0; ch < s->channels; ch++) |
| | 1551 | FFSWAP(RNNState, s->st[ch].rnn[0], s->st[ch].rnn[1]); |
| | 1552 | FFSWAP(RNNModel *, s->model[0], s->model[1]); |
| | 1553 | return ret; |
| | 1554 | } |
| | 1555 | |
| | 1556 | free_model(ctx, 1); |
| | 1557 | return 0; |
| | 1558 | } |
| | 1559 | |
| 1497 | static av_cold void uninit(AVFilterContext *ctx) | 1560 | static av_cold void uninit(AVFilterContext *ctx) |
| 1498 | { | 1561 | { |
| 1499 | AudioRNNContext *s = ctx->priv; | 1562 | AudioRNNContext *s = ctx->priv; |
| 1500 | | 1563 | |
| 1501 | av_freep(&s->fdsp); | 1564 | av_freep(&s->fdsp); |
| 1502 | rnnoise_model_free(s->model); | 1565 | free_model(ctx, 0); |
| 1503 | s->model = NULL; | 1566 | for (int ch = 0; ch < s->channels && s->st; ch++) { |
| 1504 | | 1567 | av_tx_uninit(&s->st[ch].tx); |
| 1505 | if (s->st) { | 1568 | av_tx_uninit(&s->st[ch].txi); |
| 1506 | for (int ch = 0; ch < s->channels; ch++) { | | |
| 1507 | av_freep(&s->st[ch].rnn.vad_gru_state); | | |
| 1508 | av_freep(&s->st[ch].rnn.noise_gru_state); | | |
| 1509 | av_freep(&s->st[ch].rnn.denoise_gru_state); | | |
| 1510 | av_tx_uninit(&s->st[ch].tx); | | |
| 1511 | av_tx_uninit(&s->st[ch].txi); | | |
| 1512 | } | | |
| 1513 | } | 1569 | } |
| 1514 | av_freep(&s->st); | 1570 | av_freep(&s->st); |
| 1515 | } | 1571 | } |
| @@ -1532,7 +1588,7 @@ static const AVFilterPad outputs[] = { · 16 unmodified lines |
| 1532 | }; | 1588 | }; |
| 1533 | | 1589 | |
| 1534 | #define OFFSET(x) offsetof(AudioRNNContext, x) | 1590 | #define OFFSET(x) offsetof(AudioRNNContext, x) |
| 1535 | #define AF AV_OPT_FLAG_AUDIO_PARAM|AV_OPT_FLAG_FILTERING_PARAM | 1591 | #define AF AV_OPT_FLAG_AUDIO_PARAM|AV_OPT_FLAG_FILTERING_PARAM|AV_OPT_FLAG_RUNTIME_PARAM |
| 1536 | | 1592 | |
| 1537 | static const AVOption arnndn_options[] = { | 1593 | static const AVOption arnndn_options[] = { |
| 1538 | { "model", "set model name", OFFSET(model_name), AV_OPT_TYPE_STRING, {.str=NULL}, 0, 0, AF }, | 1594 | { "model", "set model name", OFFSET(model_name), AV_OPT_TYPE_STRING, {.str=NULL}, 0, 0, AF }, |
| @@ -1556,4 +1612,5 @@ AVFilter ff_af_arnndn = { · 17 unmodified lines |
| 1556 | .outputs = outputs, | 1612 | .outputs = outputs, |
| 1557 | .flags = AVFILTER_FLAG_SUPPORT_TIMELINE_INTERNAL | | 1613 | .flags = AVFILTER_FLAG_SUPPORT_TIMELINE_INTERNAL | |
| 1558 | AVFILTER_FLAG_SLICE_THREADS, | 1614 | AVFILTER_FLAG_SLICE_THREADS, |
| | 1615 | .process_command = process_command, |
| 1559 | }; | 1616 | }; |