History

libavfilter: Whisper audio filter

It adds a new audio filter for running audio transcriptions with the whisper model. Documentation and examples are included into the patch.

Vittorio Palmisano committed 1 year ago

Commit 13ce36f1 parent 2eaf0446 files · +579 −0Browse files

Showing the changes to libavfilter (3 of 6 files). Show all

Unified · Split

3 files
  1. libavfilter/Makefile +2 −0
  2. libavfilter/af_whisper.c +463 −0
  3. libavfilter/allfilters.c +1 −0
libavfilter/Makefile+2−0
@@ -188,6 +188,8 @@ OBJS-$(CONFIG_HILBERT_FILTER) += asrc_hilbert.o · 187 unmodified lines
188188 OBJS-$(CONFIG_SINC_FILTER) += asrc_sinc.o
189189 OBJS-$(CONFIG_SINE_FILTER) += asrc_sine.o
190190
191+OBJS-$(CONFIG_WHISPER_FILTER) += af_whisper.o
192+
191193 OBJS-$(CONFIG_ANULLSINK_FILTER) += asink_anullsink.o
192194
193195 # video filters
libavfilter/af_whisper.c+463−0
@@ -0,0 +1,463 @@
1+/*
2+ * Copyright (c) 2025 Vittorio Palmisano
3+ *
4+ * This file is part of FFmpeg.
5+ *
6+ * FFmpeg is free software; you can redistribute it and/or
7+ * modify it under the terms of the GNU Lesser General Public License
8+ * as published by the Free Software Foundation; either
9+ * version 2.1 of the License, or (at your option) any later version.
10+ *
11+ * FFmpeg is distributed in the hope that it will be useful,
12+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
13+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14+ * GNU Lesser General Public License for more details.
15+ *
16+ * You should have received a copy of the GNU Lesser General Public License
17+ * along with FFmpeg; if not, write to the Free Software Foundation, Inc.,
18+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19+ */
20+
21+#include <stdio.h>
22+#include <stdint.h>
23+#include <stdlib.h>
24+
25+#include <whisper.h>
26+
27+#include "libavutil/avutil.h"
28+#include "libavutil/opt.h"
29+#include "libavutil/channel_layout.h"
30+#include "libavutil/samplefmt.h"
31+#include "libavfilter/avfilter.h"
32+#include "libavfilter/audio.h"
33+#include "libavutil/mem.h"
34+#include "libavutil/avstring.h"
35+#include "libavutil/internal.h"
36+#include "libavformat/avio.h"
37+#include "libavutil/thread.h"
38+
39+#include "formats.h"
40+
41+typedef struct WhisperContext {
42+ const AVClass *class;
43+ char *model_path;
44+ char *language;
45+ bool use_gpu;
46+ int gpu_device;
47+ char *vad_model_path;
48+ float vad_threshold;
49+ int64_t vad_min_speech_duration;
50+ int64_t vad_min_silence_duration;
51+
52+ int64_t queue;
53+ char *destination;
54+ char *format;
55+
56+ struct whisper_context *ctx_wsp;
57+ struct whisper_vad_context *ctx_vad;
58+ struct whisper_vad_params vad_params;
59+
60+ float *audio_buffer;
61+ int audio_buffer_queue_size;
62+ int audio_buffer_fill_size;
63+ int audio_buffer_vad_size;
64+ int64_t audio_buffer_start_ms;
65+
66+ int eof;
67+ int64_t next_pts;
68+
69+ AVIOContext *avio_context;
70+ int index;
71+} WhisperContext;
72+
73+static void cb_log(enum ggml_log_level level, const char *text, void *user_data)
74+{
75+ AVFilterContext *ctx = user_data;
76+ int av_log_level = AV_LOG_DEBUG;
77+ switch (level) {
78+ case GGML_LOG_LEVEL_ERROR:
79+ av_log_level = AV_LOG_ERROR;
80+ break;
81+ case GGML_LOG_LEVEL_WARN:
82+ av_log_level = AV_LOG_WARNING;
83+ break;
84+ }
85+ av_log(ctx, av_log_level, "%s", text);
86+}
87+
88+static int init(AVFilterContext *ctx)
89+{
90+ WhisperContext *wctx = ctx->priv;
91+
92+ static AVOnce init_static_once = AV_ONCE_INIT;
93+ ff_thread_once(&init_static_once, ggml_backend_load_all);
94+
95+ whisper_log_set(cb_log, ctx);
96+
97+ // Init whisper context
98+ if (!wctx->model_path) {
99+ av_log(ctx, AV_LOG_ERROR, "No whisper model path specified. Use the 'model' option.\n");
100+ return AVERROR(EINVAL);
101+ }
102+
103+ struct whisper_context_params params = whisper_context_default_params();
104+ params.use_gpu = wctx->use_gpu;
105+ params.gpu_device = wctx->gpu_device;
106+
107+ wctx->ctx_wsp = whisper_init_from_file_with_params(wctx->model_path, params);
108+ if (wctx->ctx_wsp == NULL) {
109+ av_log(ctx, AV_LOG_ERROR, "Failed to initialize whisper context from model: %s\n", wctx->model_path);
110+ return AVERROR(EIO);
111+ }
112+
113+ // Init buffer
114+ wctx->audio_buffer_queue_size = av_rescale(wctx->queue, WHISPER_SAMPLE_RATE, AV_TIME_BASE);
115+ wctx->audio_buffer = av_malloc_array(wctx->audio_buffer_queue_size, sizeof(*wctx->audio_buffer));
116+ if (!wctx->audio_buffer)
117+ return AVERROR(ENOMEM);
118+
119+ // Init VAD model context
120+ if (wctx->vad_model_path) {
121+ struct whisper_vad_context_params ctx_params = whisper_vad_default_context_params();
122+ ctx_params.n_threads = ff_filter_get_nb_threads(ctx);
123+ // ctx_params.use_gpu = wctx->use_gpu; TODO (see: whisper_vad_init_context)
124+ ctx_params.gpu_device = wctx->gpu_device;
125+ wctx->ctx_vad = whisper_vad_init_from_file_with_params(wctx->vad_model_path, ctx_params);
126+
127+ wctx->vad_params = whisper_vad_default_params();
128+ wctx->vad_params.threshold = wctx->vad_threshold;
129+ wctx->vad_params.min_speech_duration_ms = av_rescale(wctx->vad_min_speech_duration, 1000, AV_TIME_BASE);
130+ wctx->vad_params.min_silence_duration_ms = av_rescale(wctx->vad_min_silence_duration, 1000, AV_TIME_BASE);
131+ wctx->vad_params.max_speech_duration_s = av_rescale(wctx->queue, 1, AV_TIME_BASE);
132+ wctx->vad_params.speech_pad_ms = 0;
133+ wctx->vad_params.samples_overlap = 0;
134+ }
135+
136+ wctx->next_pts = AV_NOPTS_VALUE;
137+
138+ if (wctx->destination && strcmp("", wctx->destination)) {
139+ const char *dst = wctx->destination;
140+ if (!strcmp("-", dst))
141+ dst = "pipe:1";
142+ int ret = avio_open(&wctx->avio_context, dst, AVIO_FLAG_WRITE);
143+
144+ if (ret < 0) {
145+ av_log(ctx, AV_LOG_ERROR, "Could not open %s: %s\n", wctx->destination, av_err2str(ret));
146+ return ret;
147+ }
148+
149+ wctx->avio_context->direct = AVIO_FLAG_DIRECT;
150+ }
151+
152+ av_log(ctx, AV_LOG_INFO,
153+ "Whisper filter initialized: model: %s lang: %s queue: %ld ms\n",
154+ wctx->model_path, wctx->language, wctx->queue / 1000);
155+
156+ return 0;
157+}
158+
159+static void uninit(AVFilterContext *ctx)
160+{
161+ WhisperContext *wctx = ctx->priv;
162+
163+ if (wctx->audio_buffer_fill_size > 0) {
164+ av_log(ctx, AV_LOG_WARNING,
165+ "Remaining audio buffer %d samples (%d seconds) after stopping\n",
166+ wctx->audio_buffer_fill_size, wctx->audio_buffer_fill_size / WHISPER_SAMPLE_RATE);
167+ }
168+
169+ if (wctx->ctx_vad) {
170+ whisper_vad_free(wctx->ctx_vad);
171+ wctx->ctx_vad = NULL;
172+ }
173+
174+ if (wctx->ctx_wsp) {
175+ whisper_free(wctx->ctx_wsp);
176+ wctx->ctx_wsp = NULL;
177+ }
178+
179+ av_freep(&wctx->audio_buffer);
180+
181+ if (wctx->avio_context)
182+ avio_closep(&wctx->avio_context);
183+}
184+
185+static void run_transcription(AVFilterContext *ctx, AVFrame *frame, int samples)
186+{
187+ WhisperContext *wctx = ctx->priv;
188+ samples = FFMAX(0, FFMIN(samples, wctx->audio_buffer_fill_size));
189+
190+ if (!wctx->ctx_wsp || samples == 0)
191+ return;
192+
193+ const int64_t timestamp_ms = wctx->audio_buffer_start_ms;
194+ const float duration = (float) samples / WHISPER_SAMPLE_RATE;
195+
196+ av_log(ctx, AV_LOG_INFO,
197+ "run transcription at %ld ms, %d/%d samples (%.2f seconds)...\n",
198+ timestamp_ms, samples, wctx->audio_buffer_fill_size, duration);
199+
200+ struct whisper_full_params params = whisper_full_default_params(WHISPER_SAMPLING_GREEDY);
201+ params.language = wctx->language;
202+ params.n_threads = ff_filter_get_nb_threads(ctx);
203+ params.print_special = 0;
204+ params.print_progress = 0;
205+ params.print_realtime = 0;
206+ params.print_timestamps = 0;
207+
208+ if (whisper_full(wctx->ctx_wsp, params, wctx->audio_buffer, samples) != 0) {
209+ av_log(ctx, AV_LOG_ERROR, "Failed to process audio with whisper.cpp\n");
210+ return;
211+ }
212+
213+ const int n_segments = whisper_full_n_segments(wctx->ctx_wsp);
214+ char *segments_text = NULL;
215+
216+ for (int i = 0; i < n_segments; ++i) {
217+ const char *text = whisper_full_get_segment_text(wctx->ctx_wsp, i);
218+ char *text_cleaned = av_strireplace(text + 1, "[BLANK_AUDIO]", "");
219+
220+ if (av_strnlen(text_cleaned, 1) == 0) {
221+ av_freep(&text_cleaned);
222+ continue;
223+ }
224+
225+ const bool turn = whisper_full_get_segment_speaker_turn_next(wctx->ctx_wsp, i);
226+ const int64_t t0_ms = whisper_full_get_segment_t0(wctx->ctx_wsp, i) * 10;
227+ const int64_t t1_ms = whisper_full_get_segment_t1(wctx->ctx_wsp, i) * 10;
228+
229+ av_log(ctx, AV_LOG_DEBUG, " [%ld-%ld%s]: \"%s\"\n",
230+ timestamp_ms + t0_ms, timestamp_ms + t1_ms, turn ? " (turn)" : "", text_cleaned);
231+
232+ if (segments_text) {
233+ char *new_text = av_asprintf("%s%s", segments_text, text_cleaned);
234+ av_freep(&segments_text);
235+ segments_text = new_text;
236+ } else
237+ segments_text = av_strdup(text_cleaned);
238+
239+ if (wctx->avio_context) {
240+ const int64_t start_t = timestamp_ms + t0_ms;
241+ const int64_t end_t = timestamp_ms + t1_ms;
242+ char *buf = NULL;
243+
244+ if (!av_strcasecmp(wctx->format, "srt")) {
245+ buf =
246+ av_asprintf
247+ ("%d\n%02ld:%02ld:%02ld.%03ld --> %02ld:%02ld:%02ld.%03ld\n%s\n\n",
248+ wctx->index, start_t / 3600000,
249+ (start_t / 60000) % 60, (start_t / 1000) % 60,
250+ start_t % 1000, end_t / 3600000, (end_t / 60000) % 60,
251+ (end_t / 1000) % 60, end_t % 1000, text_cleaned);
252+ } else if (!av_strcasecmp(wctx->format, "json")) {
253+ buf = av_asprintf("{\"start\":%ld,\"end\":%ld,\"text\":\"%s\"}\n", start_t, end_t, text_cleaned);
254+ } else
255+ buf = av_strdup(text_cleaned);
256+
257+ if (buf) {
258+ avio_write(wctx->avio_context, buf, strlen(buf));
259+ av_freep(&buf);
260+ }
261+ }
262+
263+ av_freep(&text_cleaned);
264+ }
265+
266+ wctx->index++;
267+
268+ AVDictionary **metadata = &frame->metadata;
269+ if (metadata && segments_text) {
270+ av_dict_set(metadata, "lavfi.whisper.text", segments_text, 0);
271+ char *duration_text = av_asprintf("%f", duration);
272+ av_dict_set(metadata, "lavfi.whisper.duration", duration_text, AV_DICT_DONT_STRDUP_VAL);
273+ }
274+ av_freep(&segments_text);
275+
276+ if (wctx->audio_buffer_fill_size > samples) {
277+ memcpy(wctx->audio_buffer, wctx->audio_buffer + samples,
278+ (wctx->audio_buffer_fill_size - samples) * sizeof(*wctx->audio_buffer));
279+ wctx->audio_buffer_start_ms += duration * 1000;
280+ }
281+ wctx->audio_buffer_fill_size -= samples;
282+ wctx->audio_buffer_vad_size = wctx->audio_buffer_fill_size;
283+}
284+
285+static int filter_frame(AVFilterLink *inlink, AVFrame *frame)
286+{
287+ AVFilterContext *ctx = inlink->dst;
288+ WhisperContext *wctx = ctx->priv;
289+ AVFilterLink *outlink = ctx->outputs[0];
290+
291+ const int samples = frame->nb_samples;
292+ const float *input_data = (const float *) frame->data[0];
293+
294+ if (wctx->audio_buffer_fill_size + samples > wctx->audio_buffer_queue_size) {
295+ run_transcription(ctx, frame, wctx->audio_buffer_fill_size);
296+ }
297+
298+ if (!wctx->audio_buffer_fill_size)
299+ wctx->audio_buffer_start_ms = av_rescale_q(frame->pts,
300+ (AVRational) {1000, 1},
301+ (AVRational) {inlink->time_base.den, inlink->time_base.num});
302+ memcpy(wctx->audio_buffer + wctx->audio_buffer_fill_size, input_data, samples * sizeof(*wctx->audio_buffer));
303+ wctx->audio_buffer_fill_size += samples;
304+
305+ if (wctx->ctx_vad
306+ && (wctx->audio_buffer_fill_size - wctx->audio_buffer_vad_size) >=
307+ av_rescale(wctx->vad_min_speech_duration + wctx->vad_min_silence_duration, WHISPER_SAMPLE_RATE, AV_TIME_BASE)) {
308+ struct whisper_vad_segments *segments = whisper_vad_segments_from_samples(wctx->ctx_vad,
309+ wctx->vad_params,
310+ wctx->audio_buffer,
311+ wctx->audio_buffer_fill_size);
312+ wctx->audio_buffer_vad_size = wctx->audio_buffer_fill_size;
313+
314+ if (!segments) {
315+ av_log(ctx, AV_LOG_ERROR, "failed to detect VAD\n");
316+ } else {
317+ int n_segments = whisper_vad_segments_n_segments(segments);
318+
319+ if (n_segments > 0) {
320+ const float start_ms = whisper_vad_segments_get_segment_t0(segments, 0) * 10.0;
321+ const float end_ms = whisper_vad_segments_get_segment_t1(segments, n_segments - 1) * 10.0;
322+ int end_pos = (int) (end_ms * WHISPER_SAMPLE_RATE / 1000);
323+
324+ if (end_pos <= wctx->audio_buffer_fill_size -
325+ av_rescale(wctx->vad_min_silence_duration, WHISPER_SAMPLE_RATE, AV_TIME_BASE)) {
326+ av_log(ctx, AV_LOG_INFO,
327+ "VAD detected %d segments, start: %.0f ms, end: %.0f ms (buffer: %d ms)\n",
328+ n_segments, start_ms, end_ms, 1000 * wctx->audio_buffer_fill_size / WHISPER_SAMPLE_RATE);
329+ run_transcription(ctx, frame, end_pos);
330+ }
331+ }
332+
333+ whisper_vad_free_segments(segments);
334+ }
335+ } else if (wctx->audio_buffer_fill_size >= wctx->audio_buffer_queue_size)
336+ run_transcription(ctx, frame, wctx->audio_buffer_fill_size);
337+
338+ wctx->next_pts = frame->pts + av_rescale_q(samples, (AVRational) {
339+ 1, inlink->sample_rate}
340+ , inlink->time_base);
341+ return ff_filter_frame(outlink, frame);
342+}
343+
344+static int push_last_frame(AVFilterLink *outlink)
345+{
346+ AVFilterContext *ctx = outlink->src;
347+ WhisperContext *wctx = ctx->priv;
348+ AVFrame *frame;
349+ int n_out = 1;
350+
351+ if (ctx->is_disabled || wctx->audio_buffer_fill_size == 0)
352+ return 0;
353+ frame = ff_get_audio_buffer(outlink, n_out);
354+ if (!frame)
355+ return AVERROR(ENOMEM);
356+
357+ av_samples_set_silence(frame->extended_data, 0, n_out, frame->ch_layout.nb_channels, frame->format);
358+
359+ frame->pts = wctx->next_pts;
360+ if (wctx->next_pts != AV_NOPTS_VALUE)
361+ wctx->next_pts += av_rescale_q(n_out, (AVRational) {
362+ 1, outlink->sample_rate}
363+ , outlink->time_base);
364+
365+ run_transcription(ctx, frame, wctx->audio_buffer_fill_size);
366+
367+ return ff_filter_frame(outlink, frame);
368+}
369+
370+static int activate(AVFilterContext *ctx)
371+{
372+ AVFilterLink *inlink = ctx->inputs[0];
373+ AVFilterLink *outlink = ctx->outputs[0];
374+ WhisperContext *wctx = ctx->priv;
375+ int64_t pts;
376+ int status;
377+
378+ FF_FILTER_FORWARD_STATUS_BACK(outlink, inlink);
379+
380+ if (!wctx->eof && ff_inlink_queued_frames(inlink)) {
381+ AVFrame *frame = NULL;
382+ int ret;
383+
384+ ret = ff_inlink_consume_frame(inlink, &frame);
385+ if (ret < 0)
386+ return ret;
387+ if (ret > 0)
388+ return filter_frame(inlink, frame);
389+ }
390+
391+ if (!wctx->eof && ff_inlink_acknowledge_status(inlink, &status, &pts))
392+ wctx->eof = status == AVERROR_EOF;
393+
394+ if (wctx->eof) {
395+ push_last_frame(outlink);
396+
397+ ff_outlink_set_status(outlink, AVERROR_EOF, wctx->next_pts);
398+ return 0;
399+ }
400+
401+ FF_FILTER_FORWARD_WANTED(outlink, inlink);
402+
403+ return FFERROR_NOT_READY;
404+}
405+
406+static int query_formats(const AVFilterContext *ctx,
407+ AVFilterFormatsConfig **cfg_in,
408+ AVFilterFormatsConfig **cfg_out)
409+{
410+ static const enum AVSampleFormat sample_fmts[] = { AV_SAMPLE_FMT_FLT, AV_SAMPLE_FMT_NONE };
411+ AVChannelLayout chlayouts[] = { FF_COUNT2LAYOUT(1), { 0 } };
412+ int sample_rates[] = { WHISPER_SAMPLE_RATE, -1 };
413+ int ret;
414+
415+ ret = ff_set_common_formats_from_list2(ctx, cfg_in, cfg_out, sample_fmts);
416+ if (ret < 0)
417+ return ret;
418+
419+ ret = ff_set_common_channel_layouts_from_list2(ctx, cfg_in, cfg_out, chlayouts);
420+ if (ret < 0)
421+ return ret;
422+
423+ return ff_set_common_samplerates_from_list2(ctx, cfg_in, cfg_out, sample_rates);
424+}
425+
426+#define OFFSET(x) offsetof(WhisperContext, x)
427+#define FLAGS AV_OPT_FLAG_AUDIO_PARAM | AV_OPT_FLAG_FILTERING_PARAM
428+#define HOURS 3600000000
429+
430+static const AVOption whisper_options[] = {
431+ { "model", "Path to the whisper.cpp model file", OFFSET(model_path), AV_OPT_TYPE_STRING,.flags = FLAGS }, { "language", "Language for transcription ('auto' for auto-detect)", OFFSET(language), AV_OPT_TYPE_STRING, {.str = "auto"}, .flags = FLAGS },
432+ { "queue", "Audio queue size", OFFSET(queue), AV_OPT_TYPE_DURATION, {.i64 = 3000000}, 20000, HOURS, .flags = FLAGS },
433+ { "use_gpu", "Use GPU for processing", OFFSET(use_gpu), AV_OPT_TYPE_BOOL, {.i64 = 1}, 0, 1, .flags = FLAGS },
434+ { "gpu_device", "GPU device to use", OFFSET(gpu_device), AV_OPT_TYPE_INT, {.i64 = 0}, 0, INT_MAX, .flags = FLAGS },
435+ { "destination", "Output destination", OFFSET(destination), AV_OPT_TYPE_STRING, {.str = ""}, .flags = FLAGS },
436+ { "format", "Output format (text|srt|json)", OFFSET(format), AV_OPT_TYPE_STRING, {.str = "text"},.flags = FLAGS },
437+ { "vad_model", "Path to the VAD model file", OFFSET(vad_model_path), AV_OPT_TYPE_STRING,.flags = FLAGS },
438+ { "vad_threshold", "VAD threshold", OFFSET(vad_threshold), AV_OPT_TYPE_FLOAT, {.dbl = 0.5}, 0.0, 1.0, .flags = FLAGS },
439+ { "vad_min_speech_duration", "Minimum speech duration for VAD", OFFSET(vad_min_speech_duration), AV_OPT_TYPE_DURATION, {.i64 = 100000}, 20000, HOURS, .flags = FLAGS },
440+ { "vad_min_silence_duration", "Minimum silence duration for VAD", OFFSET(vad_min_silence_duration), AV_OPT_TYPE_DURATION, {.i64 = 500000}, 0, HOURS, .flags = FLAGS },
441+ { NULL }
442+};
443+
444+static const AVClass whisper_class = {
445+ .class_name = "whisper",
446+ .item_name = av_default_item_name,
447+ .option = whisper_options,
448+ .version = LIBAVUTIL_VERSION_INT,
449+};
450+
451+const FFFilter ff_af_whisper = {
452+ .p.name = "whisper",
453+ .p.description = NULL_IF_CONFIG_SMALL("Transcribe audio using whisper.cpp."),
454+ .p.priv_class = &whisper_class,
455+ .p.flags = AVFILTER_FLAG_METADATA_ONLY,
456+ .init = init,
457+ .uninit = uninit,
458+ .activate = activate,
459+ .priv_size = sizeof(WhisperContext),
460+ FILTER_INPUTS(ff_audio_default_filterpad),
461+ FILTER_OUTPUTS(ff_audio_default_filterpad),
462+ FILTER_QUERY_FUNC2(query_formats),
463+};
libavfilter/allfilters.c+1−0
@@ -162,6 +162,7 @@ extern const FFFilter ff_af_vibrato; · 161 unmodified lines
162162 extern const FFFilter ff_af_virtualbass;
163163 extern const FFFilter ff_af_volume;
164164 extern const FFFilter ff_af_volumedetect;
165+extern const FFFilter ff_af_whisper;
165166
166167 extern const FFFilter ff_asrc_aevalsrc;
167168 extern const FFFilter ff_asrc_afdelaysrc;

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t