History

libavfilter: Whisper audio filter

It adds a new audio filter for running audio transcriptions with the whisper model. Documentation and examples are included into the patch.

Vittorio Palmisano committed 1 year ago

Commit 13ce36f1 parent 2eaf0446 files · +579 −0Browse files

Showing the changes to libavfilter (3 of 6 files). Show all

Unified · Split

3 files
  1. libavfilter/Makefile +2 −0
  2. libavfilter/af_whisper.c +463 −0
  3. libavfilter/allfilters.c +1 −0
libavfilter/Makefile+2−0
@@ -188,6 +188,8 @@ OBJS-$(CONFIG_HILBERT_FILTER) += asrc_hilbert.o · 187 unmodified lines
188OBJS-$(CONFIG_SINC_FILTER) += asrc_sinc.o188OBJS-$(CONFIG_SINC_FILTER) += asrc_sinc.o
189OBJS-$(CONFIG_SINE_FILTER) += asrc_sine.o189OBJS-$(CONFIG_SINE_FILTER) += asrc_sine.o
190 190
191OBJS-$(CONFIG_WHISPER_FILTER) += af_whisper.o
192
191OBJS-$(CONFIG_ANULLSINK_FILTER) += asink_anullsink.o193OBJS-$(CONFIG_ANULLSINK_FILTER) += asink_anullsink.o
192 194
193# video filters195# video filters
libavfilter/af_whisper.c+463−0
@@ -0,0 +1,463 @@
1/*
2 * Copyright (c) 2025 Vittorio Palmisano
3 *
4 * This file is part of FFmpeg.
5 *
6 * FFmpeg is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public License
8 * as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
10 *
11 * FFmpeg is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public License
17 * along with FFmpeg; if not, write to the Free Software Foundation, Inc.,
18 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19 */
20
21#include <stdio.h>
22#include <stdint.h>
23#include <stdlib.h>
24
25#include <whisper.h>
26
27#include "libavutil/avutil.h"
28#include "libavutil/opt.h"
29#include "libavutil/channel_layout.h"
30#include "libavutil/samplefmt.h"
31#include "libavfilter/avfilter.h"
32#include "libavfilter/audio.h"
33#include "libavutil/mem.h"
34#include "libavutil/avstring.h"
35#include "libavutil/internal.h"
36#include "libavformat/avio.h"
37#include "libavutil/thread.h"
38
39#include "formats.h"
40
41typedef struct WhisperContext {
42 const AVClass *class;
43 char *model_path;
44 char *language;
45 bool use_gpu;
46 int gpu_device;
47 char *vad_model_path;
48 float vad_threshold;
49 int64_t vad_min_speech_duration;
50 int64_t vad_min_silence_duration;
51
52 int64_t queue;
53 char *destination;
54 char *format;
55
56 struct whisper_context *ctx_wsp;
57 struct whisper_vad_context *ctx_vad;
58 struct whisper_vad_params vad_params;
59
60 float *audio_buffer;
61 int audio_buffer_queue_size;
62 int audio_buffer_fill_size;
63 int audio_buffer_vad_size;
64 int64_t audio_buffer_start_ms;
65
66 int eof;
67 int64_t next_pts;
68
69 AVIOContext *avio_context;
70 int index;
71} WhisperContext;
72
73static void cb_log(enum ggml_log_level level, const char *text, void *user_data)
74{
75 AVFilterContext *ctx = user_data;
76 int av_log_level = AV_LOG_DEBUG;
77 switch (level) {
78 case GGML_LOG_LEVEL_ERROR:
79 av_log_level = AV_LOG_ERROR;
80 break;
81 case GGML_LOG_LEVEL_WARN:
82 av_log_level = AV_LOG_WARNING;
83 break;
84 }
85 av_log(ctx, av_log_level, "%s", text);
86}
87
88static int init(AVFilterContext *ctx)
89{
90 WhisperContext *wctx = ctx->priv;
91
92 static AVOnce init_static_once = AV_ONCE_INIT;
93 ff_thread_once(&init_static_once, ggml_backend_load_all);
94
95 whisper_log_set(cb_log, ctx);
96
97 // Init whisper context
98 if (!wctx->model_path) {
99 av_log(ctx, AV_LOG_ERROR, "No whisper model path specified. Use the 'model' option.\n");
100 return AVERROR(EINVAL);
101 }
102
103 struct whisper_context_params params = whisper_context_default_params();
104 params.use_gpu = wctx->use_gpu;
105 params.gpu_device = wctx->gpu_device;
106
107 wctx->ctx_wsp = whisper_init_from_file_with_params(wctx->model_path, params);
108 if (wctx->ctx_wsp == NULL) {
109 av_log(ctx, AV_LOG_ERROR, "Failed to initialize whisper context from model: %s\n", wctx->model_path);
110 return AVERROR(EIO);
111 }
112
113 // Init buffer
114 wctx->audio_buffer_queue_size = av_rescale(wctx->queue, WHISPER_SAMPLE_RATE, AV_TIME_BASE);
115 wctx->audio_buffer = av_malloc_array(wctx->audio_buffer_queue_size, sizeof(*wctx->audio_buffer));
116 if (!wctx->audio_buffer)
117 return AVERROR(ENOMEM);
118
119 // Init VAD model context
120 if (wctx->vad_model_path) {
121 struct whisper_vad_context_params ctx_params = whisper_vad_default_context_params();
122 ctx_params.n_threads = ff_filter_get_nb_threads(ctx);
123 // ctx_params.use_gpu = wctx->use_gpu; TODO (see: whisper_vad_init_context)
124 ctx_params.gpu_device = wctx->gpu_device;
125 wctx->ctx_vad = whisper_vad_init_from_file_with_params(wctx->vad_model_path, ctx_params);
126
127 wctx->vad_params = whisper_vad_default_params();
128 wctx->vad_params.threshold = wctx->vad_threshold;
129 wctx->vad_params.min_speech_duration_ms = av_rescale(wctx->vad_min_speech_duration, 1000, AV_TIME_BASE);
130 wctx->vad_params.min_silence_duration_ms = av_rescale(wctx->vad_min_silence_duration, 1000, AV_TIME_BASE);
131 wctx->vad_params.max_speech_duration_s = av_rescale(wctx->queue, 1, AV_TIME_BASE);
132 wctx->vad_params.speech_pad_ms = 0;
133 wctx->vad_params.samples_overlap = 0;
134 }
135
136 wctx->next_pts = AV_NOPTS_VALUE;
137
138 if (wctx->destination && strcmp("", wctx->destination)) {
139 const char *dst = wctx->destination;
140 if (!strcmp("-", dst))
141 dst = "pipe:1";
142 int ret = avio_open(&wctx->avio_context, dst, AVIO_FLAG_WRITE);
143
144 if (ret < 0) {
145 av_log(ctx, AV_LOG_ERROR, "Could not open %s: %s\n", wctx->destination, av_err2str(ret));
146 return ret;
147 }
148
149 wctx->avio_context->direct = AVIO_FLAG_DIRECT;
150 }
151
152 av_log(ctx, AV_LOG_INFO,
153 "Whisper filter initialized: model: %s lang: %s queue: %ld ms\n",
154 wctx->model_path, wctx->language, wctx->queue / 1000);
155
156 return 0;
157}
158
159static void uninit(AVFilterContext *ctx)
160{
161 WhisperContext *wctx = ctx->priv;
162
163 if (wctx->audio_buffer_fill_size > 0) {
164 av_log(ctx, AV_LOG_WARNING,
165 "Remaining audio buffer %d samples (%d seconds) after stopping\n",
166 wctx->audio_buffer_fill_size, wctx->audio_buffer_fill_size / WHISPER_SAMPLE_RATE);
167 }
168
169 if (wctx->ctx_vad) {
170 whisper_vad_free(wctx->ctx_vad);
171 wctx->ctx_vad = NULL;
172 }
173
174 if (wctx->ctx_wsp) {
175 whisper_free(wctx->ctx_wsp);
176 wctx->ctx_wsp = NULL;
177 }
178
179 av_freep(&wctx->audio_buffer);
180
181 if (wctx->avio_context)
182 avio_closep(&wctx->avio_context);
183}
184
185static void run_transcription(AVFilterContext *ctx, AVFrame *frame, int samples)
186{
187 WhisperContext *wctx = ctx->priv;
188 samples = FFMAX(0, FFMIN(samples, wctx->audio_buffer_fill_size));
189
190 if (!wctx->ctx_wsp || samples == 0)
191 return;
192
193 const int64_t timestamp_ms = wctx->audio_buffer_start_ms;
194 const float duration = (float) samples / WHISPER_SAMPLE_RATE;
195
196 av_log(ctx, AV_LOG_INFO,
197 "run transcription at %ld ms, %d/%d samples (%.2f seconds)...\n",
198 timestamp_ms, samples, wctx->audio_buffer_fill_size, duration);
199
200 struct whisper_full_params params = whisper_full_default_params(WHISPER_SAMPLING_GREEDY);
201 params.language = wctx->language;
202 params.n_threads = ff_filter_get_nb_threads(ctx);
203 params.print_special = 0;
204 params.print_progress = 0;
205 params.print_realtime = 0;
206 params.print_timestamps = 0;
207
208 if (whisper_full(wctx->ctx_wsp, params, wctx->audio_buffer, samples) != 0) {
209 av_log(ctx, AV_LOG_ERROR, "Failed to process audio with whisper.cpp\n");
210 return;
211 }
212
213 const int n_segments = whisper_full_n_segments(wctx->ctx_wsp);
214 char *segments_text = NULL;
215
216 for (int i = 0; i < n_segments; ++i) {
217 const char *text = whisper_full_get_segment_text(wctx->ctx_wsp, i);
218 char *text_cleaned = av_strireplace(text + 1, "[BLANK_AUDIO]", "");
219
220 if (av_strnlen(text_cleaned, 1) == 0) {
221 av_freep(&text_cleaned);
222 continue;
223 }
224
225 const bool turn = whisper_full_get_segment_speaker_turn_next(wctx->ctx_wsp, i);
226 const int64_t t0_ms = whisper_full_get_segment_t0(wctx->ctx_wsp, i) * 10;
227 const int64_t t1_ms = whisper_full_get_segment_t1(wctx->ctx_wsp, i) * 10;
228
229 av_log(ctx, AV_LOG_DEBUG, " [%ld-%ld%s]: \"%s\"\n",
230 timestamp_ms + t0_ms, timestamp_ms + t1_ms, turn ? " (turn)" : "", text_cleaned);
231
232 if (segments_text) {
233 char *new_text = av_asprintf("%s%s", segments_text, text_cleaned);
234 av_freep(&segments_text);
235 segments_text = new_text;
236 } else
237 segments_text = av_strdup(text_cleaned);
238
239 if (wctx->avio_context) {
240 const int64_t start_t = timestamp_ms + t0_ms;
241 const int64_t end_t = timestamp_ms + t1_ms;
242 char *buf = NULL;
243
244 if (!av_strcasecmp(wctx->format, "srt")) {
245 buf =
246 av_asprintf
247 ("%d\n%02ld:%02ld:%02ld.%03ld --> %02ld:%02ld:%02ld.%03ld\n%s\n\n",
248 wctx->index, start_t / 3600000,
249 (start_t / 60000) % 60, (start_t / 1000) % 60,
250 start_t % 1000, end_t / 3600000, (end_t / 60000) % 60,
251 (end_t / 1000) % 60, end_t % 1000, text_cleaned);
252 } else if (!av_strcasecmp(wctx->format, "json")) {
253 buf = av_asprintf("{\"start\":%ld,\"end\":%ld,\"text\":\"%s\"}\n", start_t, end_t, text_cleaned);
254 } else
255 buf = av_strdup(text_cleaned);
256
257 if (buf) {
258 avio_write(wctx->avio_context, buf, strlen(buf));
259 av_freep(&buf);
260 }
261 }
262
263 av_freep(&text_cleaned);
264 }
265
266 wctx->index++;
267
268 AVDictionary **metadata = &frame->metadata;
269 if (metadata && segments_text) {
270 av_dict_set(metadata, "lavfi.whisper.text", segments_text, 0);
271 char *duration_text = av_asprintf("%f", duration);
272 av_dict_set(metadata, "lavfi.whisper.duration", duration_text, AV_DICT_DONT_STRDUP_VAL);
273 }
274 av_freep(&segments_text);
275
276 if (wctx->audio_buffer_fill_size > samples) {
277 memcpy(wctx->audio_buffer, wctx->audio_buffer + samples,
278 (wctx->audio_buffer_fill_size - samples) * sizeof(*wctx->audio_buffer));
279 wctx->audio_buffer_start_ms += duration * 1000;
280 }
281 wctx->audio_buffer_fill_size -= samples;
282 wctx->audio_buffer_vad_size = wctx->audio_buffer_fill_size;
283}
284
285static int filter_frame(AVFilterLink *inlink, AVFrame *frame)
286{
287 AVFilterContext *ctx = inlink->dst;
288 WhisperContext *wctx = ctx->priv;
289 AVFilterLink *outlink = ctx->outputs[0];
290
291 const int samples = frame->nb_samples;
292 const float *input_data = (const float *) frame->data[0];
293
294 if (wctx->audio_buffer_fill_size + samples > wctx->audio_buffer_queue_size) {
295 run_transcription(ctx, frame, wctx->audio_buffer_fill_size);
296 }
297
298 if (!wctx->audio_buffer_fill_size)
299 wctx->audio_buffer_start_ms = av_rescale_q(frame->pts,
300 (AVRational) {1000, 1},
301 (AVRational) {inlink->time_base.den, inlink->time_base.num});
302 memcpy(wctx->audio_buffer + wctx->audio_buffer_fill_size, input_data, samples * sizeof(*wctx->audio_buffer));
303 wctx->audio_buffer_fill_size += samples;
304
305 if (wctx->ctx_vad
306 && (wctx->audio_buffer_fill_size - wctx->audio_buffer_vad_size) >=
307 av_rescale(wctx->vad_min_speech_duration + wctx->vad_min_silence_duration, WHISPER_SAMPLE_RATE, AV_TIME_BASE)) {
308 struct whisper_vad_segments *segments = whisper_vad_segments_from_samples(wctx->ctx_vad,
309 wctx->vad_params,
310 wctx->audio_buffer,
311 wctx->audio_buffer_fill_size);
312 wctx->audio_buffer_vad_size = wctx->audio_buffer_fill_size;
313
314 if (!segments) {
315 av_log(ctx, AV_LOG_ERROR, "failed to detect VAD\n");
316 } else {
317 int n_segments = whisper_vad_segments_n_segments(segments);
318
319 if (n_segments > 0) {
320 const float start_ms = whisper_vad_segments_get_segment_t0(segments, 0) * 10.0;
321 const float end_ms = whisper_vad_segments_get_segment_t1(segments, n_segments - 1) * 10.0;
322 int end_pos = (int) (end_ms * WHISPER_SAMPLE_RATE / 1000);
323
324 if (end_pos <= wctx->audio_buffer_fill_size -
325 av_rescale(wctx->vad_min_silence_duration, WHISPER_SAMPLE_RATE, AV_TIME_BASE)) {
326 av_log(ctx, AV_LOG_INFO,
327 "VAD detected %d segments, start: %.0f ms, end: %.0f ms (buffer: %d ms)\n",
328 n_segments, start_ms, end_ms, 1000 * wctx->audio_buffer_fill_size / WHISPER_SAMPLE_RATE);
329 run_transcription(ctx, frame, end_pos);
330 }
331 }
332
333 whisper_vad_free_segments(segments);
334 }
335 } else if (wctx->audio_buffer_fill_size >= wctx->audio_buffer_queue_size)
336 run_transcription(ctx, frame, wctx->audio_buffer_fill_size);
337
338 wctx->next_pts = frame->pts + av_rescale_q(samples, (AVRational) {
339 1, inlink->sample_rate}
340 , inlink->time_base);
341 return ff_filter_frame(outlink, frame);
342}
343
344static int push_last_frame(AVFilterLink *outlink)
345{
346 AVFilterContext *ctx = outlink->src;
347 WhisperContext *wctx = ctx->priv;
348 AVFrame *frame;
349 int n_out = 1;
350
351 if (ctx->is_disabled || wctx->audio_buffer_fill_size == 0)
352 return 0;
353 frame = ff_get_audio_buffer(outlink, n_out);
354 if (!frame)
355 return AVERROR(ENOMEM);
356
357 av_samples_set_silence(frame->extended_data, 0, n_out, frame->ch_layout.nb_channels, frame->format);
358
359 frame->pts = wctx->next_pts;
360 if (wctx->next_pts != AV_NOPTS_VALUE)
361 wctx->next_pts += av_rescale_q(n_out, (AVRational) {
362 1, outlink->sample_rate}
363 , outlink->time_base);
364
365 run_transcription(ctx, frame, wctx->audio_buffer_fill_size);
366
367 return ff_filter_frame(outlink, frame);
368}
369
370static int activate(AVFilterContext *ctx)
371{
372 AVFilterLink *inlink = ctx->inputs[0];
373 AVFilterLink *outlink = ctx->outputs[0];
374 WhisperContext *wctx = ctx->priv;
375 int64_t pts;
376 int status;
377
378 FF_FILTER_FORWARD_STATUS_BACK(outlink, inlink);
379
380 if (!wctx->eof && ff_inlink_queued_frames(inlink)) {
381 AVFrame *frame = NULL;
382 int ret;
383
384 ret = ff_inlink_consume_frame(inlink, &frame);
385 if (ret < 0)
386 return ret;
387 if (ret > 0)
388 return filter_frame(inlink, frame);
389 }
390
391 if (!wctx->eof && ff_inlink_acknowledge_status(inlink, &status, &pts))
392 wctx->eof = status == AVERROR_EOF;
393
394 if (wctx->eof) {
395 push_last_frame(outlink);
396
397 ff_outlink_set_status(outlink, AVERROR_EOF, wctx->next_pts);
398 return 0;
399 }
400
401 FF_FILTER_FORWARD_WANTED(outlink, inlink);
402
403 return FFERROR_NOT_READY;
404}
405
406static int query_formats(const AVFilterContext *ctx,
407 AVFilterFormatsConfig **cfg_in,
408 AVFilterFormatsConfig **cfg_out)
409{
410 static const enum AVSampleFormat sample_fmts[] = { AV_SAMPLE_FMT_FLT, AV_SAMPLE_FMT_NONE };
411 AVChannelLayout chlayouts[] = { FF_COUNT2LAYOUT(1), { 0 } };
412 int sample_rates[] = { WHISPER_SAMPLE_RATE, -1 };
413 int ret;
414
415 ret = ff_set_common_formats_from_list2(ctx, cfg_in, cfg_out, sample_fmts);
416 if (ret < 0)
417 return ret;
418
419 ret = ff_set_common_channel_layouts_from_list2(ctx, cfg_in, cfg_out, chlayouts);
420 if (ret < 0)
421 return ret;
422
423 return ff_set_common_samplerates_from_list2(ctx, cfg_in, cfg_out, sample_rates);
424}
425
426#define OFFSET(x) offsetof(WhisperContext, x)
427#define FLAGS AV_OPT_FLAG_AUDIO_PARAM | AV_OPT_FLAG_FILTERING_PARAM
428#define HOURS 3600000000
429
430static const AVOption whisper_options[] = {
431 { "model", "Path to the whisper.cpp model file", OFFSET(model_path), AV_OPT_TYPE_STRING,.flags = FLAGS }, { "language", "Language for transcription ('auto' for auto-detect)", OFFSET(language), AV_OPT_TYPE_STRING, {.str = "auto"}, .flags = FLAGS },
432 { "queue", "Audio queue size", OFFSET(queue), AV_OPT_TYPE_DURATION, {.i64 = 3000000}, 20000, HOURS, .flags = FLAGS },
433 { "use_gpu", "Use GPU for processing", OFFSET(use_gpu), AV_OPT_TYPE_BOOL, {.i64 = 1}, 0, 1, .flags = FLAGS },
434 { "gpu_device", "GPU device to use", OFFSET(gpu_device), AV_OPT_TYPE_INT, {.i64 = 0}, 0, INT_MAX, .flags = FLAGS },
435 { "destination", "Output destination", OFFSET(destination), AV_OPT_TYPE_STRING, {.str = ""}, .flags = FLAGS },
436 { "format", "Output format (text|srt|json)", OFFSET(format), AV_OPT_TYPE_STRING, {.str = "text"},.flags = FLAGS },
437 { "vad_model", "Path to the VAD model file", OFFSET(vad_model_path), AV_OPT_TYPE_STRING,.flags = FLAGS },
438 { "vad_threshold", "VAD threshold", OFFSET(vad_threshold), AV_OPT_TYPE_FLOAT, {.dbl = 0.5}, 0.0, 1.0, .flags = FLAGS },
439 { "vad_min_speech_duration", "Minimum speech duration for VAD", OFFSET(vad_min_speech_duration), AV_OPT_TYPE_DURATION, {.i64 = 100000}, 20000, HOURS, .flags = FLAGS },
440 { "vad_min_silence_duration", "Minimum silence duration for VAD", OFFSET(vad_min_silence_duration), AV_OPT_TYPE_DURATION, {.i64 = 500000}, 0, HOURS, .flags = FLAGS },
441 { NULL }
442};
443
444static const AVClass whisper_class = {
445 .class_name = "whisper",
446 .item_name = av_default_item_name,
447 .option = whisper_options,
448 .version = LIBAVUTIL_VERSION_INT,
449};
450
451const FFFilter ff_af_whisper = {
452 .p.name = "whisper",
453 .p.description = NULL_IF_CONFIG_SMALL("Transcribe audio using whisper.cpp."),
454 .p.priv_class = &whisper_class,
455 .p.flags = AVFILTER_FLAG_METADATA_ONLY,
456 .init = init,
457 .uninit = uninit,
458 .activate = activate,
459 .priv_size = sizeof(WhisperContext),
460 FILTER_INPUTS(ff_audio_default_filterpad),
461 FILTER_OUTPUTS(ff_audio_default_filterpad),
462 FILTER_QUERY_FUNC2(query_formats),
463};
libavfilter/allfilters.c+1−0
@@ -162,6 +162,7 @@ extern const FFFilter ff_af_vibrato; · 161 unmodified lines
162extern const FFFilter ff_af_virtualbass;162extern const FFFilter ff_af_virtualbass;
163extern const FFFilter ff_af_volume;163extern const FFFilter ff_af_volume;
164extern const FFFilter ff_af_volumedetect;164extern const FFFilter ff_af_volumedetect;
165extern const FFFilter ff_af_whisper;
165 166
166extern const FFFilter ff_asrc_aevalsrc;167extern const FFFilter ff_asrc_aevalsrc;
167extern const FFFilter ff_asrc_afdelaysrc;168extern const FFFilter ff_asrc_afdelaysrc;

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t