History

libavfilter: Whisper audio filter

It adds a new audio filter for running audio transcriptions with the whisper model. Documentation and examples are included into the patch.

Vittorio Palmisano committed 1 year ago

Commit 13ce36f1 parent 2eaf0446 files · +579 −0Browse files

Unified · Split

6 files
  1. Changelog +1 −0
  2. configure +5 −0
  3. doc/filters.texi +107 −0
  4. libavfilter/Makefile +2 −0
  5. libavfilter/af_whisper.c +463 −0
  6. libavfilter/allfilters.c +1 −0
Changelog+1−0
@@ -2,6 +2,7 @@ Entries are sorted chronologically from oldest to youngest within each release, · 1 unmodified line
2releases are sorted from youngest to oldest.2releases are sorted from youngest to oldest.
3 3
4version <next>:4version <next>:
5- Whisper filter
5- Drop support for OpenSSL < 1.1.06- Drop support for OpenSSL < 1.1.0
6- yasm support dropped, users need to use nasm7- yasm support dropped, users need to use nasm
7- VVC VAAPI decoder8- VVC VAAPI decoder
configure+5−0
@@ -336,6 +336,7 @@ External library support: · 335 unmodified lines
336 --disable-securetransport disable Secure Transport, needed for TLS support336 --disable-securetransport disable Secure Transport, needed for TLS support
337 on OSX if openssl and gnutls are not used [autodetect]337 on OSX if openssl and gnutls are not used [autodetect]
338 --enable-vapoursynth enable VapourSynth demuxer [no]338 --enable-vapoursynth enable VapourSynth demuxer [no]
339 --enable-whisper enable whisper filter [no]
339 --disable-xlib disable xlib [autodetect]340 --disable-xlib disable xlib [autodetect]
340 --disable-zlib disable zlib [autodetect]341 --disable-zlib disable zlib [autodetect]
341 342
@@ -2005,6 +2006,7 @@ EXTERNAL_LIBRARY_LIST=" · 1663 unmodified lines
2005 pocketsphinx2006 pocketsphinx
2006 vapoursynth2007 vapoursynth
2007 vulkan_static2008 vulkan_static
2009 whisper
2008"2010"
2009 2011
2010HWACCEL_AUTODETECT_LIBRARY_LIST="2012HWACCEL_AUTODETECT_LIBRARY_LIST="
@@ -4085,6 +4087,7 @@ xstack_qsv_filter_deps="libmfx" · 2074 unmodified lines
4085xstack_qsv_filter_select="qsvvpp"4087xstack_qsv_filter_select="qsvvpp"
4086pad_vaapi_filter_deps="vaapi_1"4088pad_vaapi_filter_deps="vaapi_1"
4087drawbox_vaapi_filter_deps="vaapi_1"4089drawbox_vaapi_filter_deps="vaapi_1"
4090whisper_filter_deps="whisper"
4088 4091
4089# examples4092# examples
4090avio_http_serve_files_deps="avformat avutil fork"4093avio_http_serve_files_deps="avformat avutil fork"
@@ -7164,6 +7167,8 @@ enabled libvo_amrwbenc && require libvo_amrwbenc vo-amrwbenc/enc_if.h E_IF_in · 3073 unmodified lines
7164enabled libvorbis && require_pkg_config libvorbis vorbis vorbis/codec.h vorbis_info_init &&7167enabled libvorbis && require_pkg_config libvorbis vorbis vorbis/codec.h vorbis_info_init &&
7165 require_pkg_config libvorbisenc vorbisenc vorbis/vorbisenc.h vorbis_encode_init7168 require_pkg_config libvorbisenc vorbisenc vorbis/vorbisenc.h vorbis_encode_init
7166 7169
7170enabled whisper && require_pkg_config whisper "whisper >= 1.7.5" whisper.h whisper_init_from_file_with_params
7171
7167enabled libvpx && {7172enabled libvpx && {
7168 enabled libvpx_vp8_decoder && {7173 enabled libvpx_vp8_decoder && {
7169 check_pkg_config libvpx_vp8_decoder "vpx >= 1.4.0" "vpx/vpx_decoder.h vpx/vp8dx.h" vpx_codec_vp8_dx ||7174 check_pkg_config libvpx_vp8_decoder "vpx >= 1.4.0" "vpx/vpx_decoder.h vpx/vp8dx.h" vpx_codec_vp8_dx ||
doc/filters.texi+107−0
@@ -7682,6 +7682,113 @@ There are 6 samples at -4 dB, 62 at -5 dB, 286 at -6 dB, etc. · 7681 unmodified lines
7682In other words, raising the volume by +4 dB does not cause any clipping,7682In other words, raising the volume by +4 dB does not cause any clipping,
7683raising it by +5 dB causes clipping for 6 samples, etc.7683raising it by +5 dB causes clipping for 6 samples, etc.
7684 7684
7685@anchor{whisper}
7686@section whisper
7687
7688It runs automatic speech recognition using the OpenAI's Whisper model.
7689
7690It requires the whisper.cpp library (https://github.com/ggml-org/whisper.cpp)
7691as a prerequisite. After installing the library it can be enabled using:
7692@code{./configure --enable-whisper}.
7693
7694The filter has following options:
7695
7696@table @option
7697@item model
7698The file path of the downloaded whisper.cpp model (mandatory).
7699
7700@item language
7701The language to use for transcription ('auto' for auto-detect).
7702Default value: @code{"auto"}
7703
7704@item queue
7705The maximum size that will be queued into the filter before processing the audio
7706with whisper. Using a small value the audio stream will be processed more often,
7707but the transcription quality will be lower and the required processing power
7708will be higher. Using a large value (e.g. 10-20s) will produce more accurate
7709results using less CPU (as using the whisper-cli tool), but the transcription
7710latency will be higher, thus not useful to process real-time streams.
7711Consider using the vad_model option associated with a large queue value.
7712Default value: @code{"3"}
7713
7714@item use_gpu
7715If the GPU support should be enabled.
7716Default value: @code{"true"}
7717
7718@item gpu_device
7719The GPU device index to use.
7720Default value: @code{"0"}
7721
7722@item destination
7723If set, the transcription output will be sent to the specified file or URL
7724(use one of the FFmpeg AVIO protocols); otherwise, the output will be logged as
7725info messages.
7726The output will also be set in the "lavfi.whisper.text" frame metadata.
7727If the destination is a file and it already exists, it will be overwritten.
7728
7729@item format
7730The destination format string; it could be "text" (only the transcribed text
7731will be sent to the destination), "srt" (subtitle format) or "json".
7732Default value: @code{"text"}
7733
7734@item vad_model
7735Path to the VAD model file. If set, the filter will load an additional voice
7736activity detection module (https://github.com/snakers4/silero-vad) that will be
7737used to fragment the audio queue; use this option setting a valid path obtained
7738from the whisper.cpp repository (e.g. "../whisper.cpp/models/ggml-silero-v5.1.2.bin")
7739and increase the queue parameter to a higher value (e.g. 20).
7740
7741@item vad_threshold
7742The VAD threshold to use.
7743Default value: @code{"0.5"}
7744
7745@item vad_min_speech_duration
7746The minimum VAD speaking duration.
7747Default value: @code{"0.1"}
7748
7749@item vad_min_silence_duration
7750The minimum VAD silence duration.
7751Default value: @code{"0.5"}
7752
7753@end table
7754
7755@subsection Examples
7756@itemize
7757
7758@item
7759Run a transcription with srt file generation:
7760@example
7761ffmpeg -i input.mp4 -vn -af "whisper=model=../whisper.cpp/models/ggml-base.en.bin\
7762:language=en\
7763:queue=3\
7764:destination=output.srt\
7765:format=srt" -f null -
7766@end example
7767
7768@item
7769Run a transcription and send the output in JSON format to an HTTP service:
7770@example
7771ffmpeg -i input.mp4 -vn -af "whisper=model=../whisper.cpp/models/ggml-base.en.bin\
7772:language=en\
7773:queue=3\
7774:destination=http\\://localhost\\:3000\
7775:format=json' -f null -
7776@end example
7777
7778@item
7779Transcribe the microphone input using the VAD option:
7780@example
7781ffmpeg -loglevel warning -f pulse -i default \
7782-af 'highpass=f=200,lowpass=f=3000,whisper=model=../whisper.cpp/models/ggml-medium.bin\
7783:language=en\
7784:queue=10\
7785:destination=-\
7786:format=json\
7787:vad_model=../whisper.cpp/models/ggml-silero-v5.1.2.bin' -f null -
7788@end example
7789
7790@end itemize
7791
7685@c man end AUDIO FILTERS7792@c man end AUDIO FILTERS
7686 7793
7687@chapter Audio Sources7794@chapter Audio Sources
libavfilter/Makefile+2−0
@@ -188,6 +188,8 @@ OBJS-$(CONFIG_HILBERT_FILTER) += asrc_hilbert.o · 187 unmodified lines
188OBJS-$(CONFIG_SINC_FILTER) += asrc_sinc.o188OBJS-$(CONFIG_SINC_FILTER) += asrc_sinc.o
189OBJS-$(CONFIG_SINE_FILTER) += asrc_sine.o189OBJS-$(CONFIG_SINE_FILTER) += asrc_sine.o
190 190
191OBJS-$(CONFIG_WHISPER_FILTER) += af_whisper.o
192
191OBJS-$(CONFIG_ANULLSINK_FILTER) += asink_anullsink.o193OBJS-$(CONFIG_ANULLSINK_FILTER) += asink_anullsink.o
192 194
193# video filters195# video filters
libavfilter/af_whisper.c+463−0
@@ -0,0 +1,463 @@
1/*
2 * Copyright (c) 2025 Vittorio Palmisano
3 *
4 * This file is part of FFmpeg.
5 *
6 * FFmpeg is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public License
8 * as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
10 *
11 * FFmpeg is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public License
17 * along with FFmpeg; if not, write to the Free Software Foundation, Inc.,
18 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19 */
20
21#include <stdio.h>
22#include <stdint.h>
23#include <stdlib.h>
24
25#include <whisper.h>
26
27#include "libavutil/avutil.h"
28#include "libavutil/opt.h"
29#include "libavutil/channel_layout.h"
30#include "libavutil/samplefmt.h"
31#include "libavfilter/avfilter.h"
32#include "libavfilter/audio.h"
33#include "libavutil/mem.h"
34#include "libavutil/avstring.h"
35#include "libavutil/internal.h"
36#include "libavformat/avio.h"
37#include "libavutil/thread.h"
38
39#include "formats.h"
40
41typedef struct WhisperContext {
42 const AVClass *class;
43 char *model_path;
44 char *language;
45 bool use_gpu;
46 int gpu_device;
47 char *vad_model_path;
48 float vad_threshold;
49 int64_t vad_min_speech_duration;
50 int64_t vad_min_silence_duration;
51
52 int64_t queue;
53 char *destination;
54 char *format;
55
56 struct whisper_context *ctx_wsp;
57 struct whisper_vad_context *ctx_vad;
58 struct whisper_vad_params vad_params;
59
60 float *audio_buffer;
61 int audio_buffer_queue_size;
62 int audio_buffer_fill_size;
63 int audio_buffer_vad_size;
64 int64_t audio_buffer_start_ms;
65
66 int eof;
67 int64_t next_pts;
68
69 AVIOContext *avio_context;
70 int index;
71} WhisperContext;
72
73static void cb_log(enum ggml_log_level level, const char *text, void *user_data)
74{
75 AVFilterContext *ctx = user_data;
76 int av_log_level = AV_LOG_DEBUG;
77 switch (level) {
78 case GGML_LOG_LEVEL_ERROR:
79 av_log_level = AV_LOG_ERROR;
80 break;
81 case GGML_LOG_LEVEL_WARN:
82 av_log_level = AV_LOG_WARNING;
83 break;
84 }
85 av_log(ctx, av_log_level, "%s", text);
86}
87
88static int init(AVFilterContext *ctx)
89{
90 WhisperContext *wctx = ctx->priv;
91
92 static AVOnce init_static_once = AV_ONCE_INIT;
93 ff_thread_once(&init_static_once, ggml_backend_load_all);
94
95 whisper_log_set(cb_log, ctx);
96
97 // Init whisper context
98 if (!wctx->model_path) {
99 av_log(ctx, AV_LOG_ERROR, "No whisper model path specified. Use the 'model' option.\n");
100 return AVERROR(EINVAL);
101 }
102
103 struct whisper_context_params params = whisper_context_default_params();
104 params.use_gpu = wctx->use_gpu;
105 params.gpu_device = wctx->gpu_device;
106
107 wctx->ctx_wsp = whisper_init_from_file_with_params(wctx->model_path, params);
108 if (wctx->ctx_wsp == NULL) {
109 av_log(ctx, AV_LOG_ERROR, "Failed to initialize whisper context from model: %s\n", wctx->model_path);
110 return AVERROR(EIO);
111 }
112
113 // Init buffer
114 wctx->audio_buffer_queue_size = av_rescale(wctx->queue, WHISPER_SAMPLE_RATE, AV_TIME_BASE);
115 wctx->audio_buffer = av_malloc_array(wctx->audio_buffer_queue_size, sizeof(*wctx->audio_buffer));
116 if (!wctx->audio_buffer)
117 return AVERROR(ENOMEM);
118
119 // Init VAD model context
120 if (wctx->vad_model_path) {
121 struct whisper_vad_context_params ctx_params = whisper_vad_default_context_params();
122 ctx_params.n_threads = ff_filter_get_nb_threads(ctx);
123 // ctx_params.use_gpu = wctx->use_gpu; TODO (see: whisper_vad_init_context)
124 ctx_params.gpu_device = wctx->gpu_device;
125 wctx->ctx_vad = whisper_vad_init_from_file_with_params(wctx->vad_model_path, ctx_params);
126
127 wctx->vad_params = whisper_vad_default_params();
128 wctx->vad_params.threshold = wctx->vad_threshold;
129 wctx->vad_params.min_speech_duration_ms = av_rescale(wctx->vad_min_speech_duration, 1000, AV_TIME_BASE);
130 wctx->vad_params.min_silence_duration_ms = av_rescale(wctx->vad_min_silence_duration, 1000, AV_TIME_BASE);
131 wctx->vad_params.max_speech_duration_s = av_rescale(wctx->queue, 1, AV_TIME_BASE);
132 wctx->vad_params.speech_pad_ms = 0;
133 wctx->vad_params.samples_overlap = 0;
134 }
135
136 wctx->next_pts = AV_NOPTS_VALUE;
137
138 if (wctx->destination && strcmp("", wctx->destination)) {
139 const char *dst = wctx->destination;
140 if (!strcmp("-", dst))
141 dst = "pipe:1";
142 int ret = avio_open(&wctx->avio_context, dst, AVIO_FLAG_WRITE);
143
144 if (ret < 0) {
145 av_log(ctx, AV_LOG_ERROR, "Could not open %s: %s\n", wctx->destination, av_err2str(ret));
146 return ret;
147 }
148
149 wctx->avio_context->direct = AVIO_FLAG_DIRECT;
150 }
151
152 av_log(ctx, AV_LOG_INFO,
153 "Whisper filter initialized: model: %s lang: %s queue: %ld ms\n",
154 wctx->model_path, wctx->language, wctx->queue / 1000);
155
156 return 0;
157}
158
159static void uninit(AVFilterContext *ctx)
160{
161 WhisperContext *wctx = ctx->priv;
162
163 if (wctx->audio_buffer_fill_size > 0) {
164 av_log(ctx, AV_LOG_WARNING,
165 "Remaining audio buffer %d samples (%d seconds) after stopping\n",
166 wctx->audio_buffer_fill_size, wctx->audio_buffer_fill_size / WHISPER_SAMPLE_RATE);
167 }
168
169 if (wctx->ctx_vad) {
170 whisper_vad_free(wctx->ctx_vad);
171 wctx->ctx_vad = NULL;
172 }
173
174 if (wctx->ctx_wsp) {
175 whisper_free(wctx->ctx_wsp);
176 wctx->ctx_wsp = NULL;
177 }
178
179 av_freep(&wctx->audio_buffer);
180
181 if (wctx->avio_context)
182 avio_closep(&wctx->avio_context);
183}
184
185static void run_transcription(AVFilterContext *ctx, AVFrame *frame, int samples)
186{
187 WhisperContext *wctx = ctx->priv;
188 samples = FFMAX(0, FFMIN(samples, wctx->audio_buffer_fill_size));
189
190 if (!wctx->ctx_wsp || samples == 0)
191 return;
192
193 const int64_t timestamp_ms = wctx->audio_buffer_start_ms;
194 const float duration = (float) samples / WHISPER_SAMPLE_RATE;
195
196 av_log(ctx, AV_LOG_INFO,
197 "run transcription at %ld ms, %d/%d samples (%.2f seconds)...\n",
198 timestamp_ms, samples, wctx->audio_buffer_fill_size, duration);
199
200 struct whisper_full_params params = whisper_full_default_params(WHISPER_SAMPLING_GREEDY);
201 params.language = wctx->language;
202 params.n_threads = ff_filter_get_nb_threads(ctx);
203 params.print_special = 0;
204 params.print_progress = 0;
205 params.print_realtime = 0;
206 params.print_timestamps = 0;
207
208 if (whisper_full(wctx->ctx_wsp, params, wctx->audio_buffer, samples) != 0) {
209 av_log(ctx, AV_LOG_ERROR, "Failed to process audio with whisper.cpp\n");
210 return;
211 }
212
213 const int n_segments = whisper_full_n_segments(wctx->ctx_wsp);
214 char *segments_text = NULL;
215
216 for (int i = 0; i < n_segments; ++i) {
217 const char *text = whisper_full_get_segment_text(wctx->ctx_wsp, i);
218 char *text_cleaned = av_strireplace(text + 1, "[BLANK_AUDIO]", "");
219
220 if (av_strnlen(text_cleaned, 1) == 0) {
221 av_freep(&text_cleaned);
222 continue;
223 }
224
225 const bool turn = whisper_full_get_segment_speaker_turn_next(wctx->ctx_wsp, i);
226 const int64_t t0_ms = whisper_full_get_segment_t0(wctx->ctx_wsp, i) * 10;
227 const int64_t t1_ms = whisper_full_get_segment_t1(wctx->ctx_wsp, i) * 10;
228
229 av_log(ctx, AV_LOG_DEBUG, " [%ld-%ld%s]: \"%s\"\n",
230 timestamp_ms + t0_ms, timestamp_ms + t1_ms, turn ? " (turn)" : "", text_cleaned);
231
232 if (segments_text) {
233 char *new_text = av_asprintf("%s%s", segments_text, text_cleaned);
234 av_freep(&segments_text);
235 segments_text = new_text;
236 } else
237 segments_text = av_strdup(text_cleaned);
238
239 if (wctx->avio_context) {
240 const int64_t start_t = timestamp_ms + t0_ms;
241 const int64_t end_t = timestamp_ms + t1_ms;
242 char *buf = NULL;
243
244 if (!av_strcasecmp(wctx->format, "srt")) {
245 buf =
246 av_asprintf
247 ("%d\n%02ld:%02ld:%02ld.%03ld --> %02ld:%02ld:%02ld.%03ld\n%s\n\n",
248 wctx->index, start_t / 3600000,
249 (start_t / 60000) % 60, (start_t / 1000) % 60,
250 start_t % 1000, end_t / 3600000, (end_t / 60000) % 60,
251 (end_t / 1000) % 60, end_t % 1000, text_cleaned);
252 } else if (!av_strcasecmp(wctx->format, "json")) {
253 buf = av_asprintf("{\"start\":%ld,\"end\":%ld,\"text\":\"%s\"}\n", start_t, end_t, text_cleaned);
254 } else
255 buf = av_strdup(text_cleaned);
256
257 if (buf) {
258 avio_write(wctx->avio_context, buf, strlen(buf));
259 av_freep(&buf);
260 }
261 }
262
263 av_freep(&text_cleaned);
264 }
265
266 wctx->index++;
267
268 AVDictionary **metadata = &frame->metadata;
269 if (metadata && segments_text) {
270 av_dict_set(metadata, "lavfi.whisper.text", segments_text, 0);
271 char *duration_text = av_asprintf("%f", duration);
272 av_dict_set(metadata, "lavfi.whisper.duration", duration_text, AV_DICT_DONT_STRDUP_VAL);
273 }
274 av_freep(&segments_text);
275
276 if (wctx->audio_buffer_fill_size > samples) {
277 memcpy(wctx->audio_buffer, wctx->audio_buffer + samples,
278 (wctx->audio_buffer_fill_size - samples) * sizeof(*wctx->audio_buffer));
279 wctx->audio_buffer_start_ms += duration * 1000;
280 }
281 wctx->audio_buffer_fill_size -= samples;
282 wctx->audio_buffer_vad_size = wctx->audio_buffer_fill_size;
283}
284
285static int filter_frame(AVFilterLink *inlink, AVFrame *frame)
286{
287 AVFilterContext *ctx = inlink->dst;
288 WhisperContext *wctx = ctx->priv;
289 AVFilterLink *outlink = ctx->outputs[0];
290
291 const int samples = frame->nb_samples;
292 const float *input_data = (const float *) frame->data[0];
293
294 if (wctx->audio_buffer_fill_size + samples > wctx->audio_buffer_queue_size) {
295 run_transcription(ctx, frame, wctx->audio_buffer_fill_size);
296 }
297
298 if (!wctx->audio_buffer_fill_size)
299 wctx->audio_buffer_start_ms = av_rescale_q(frame->pts,
300 (AVRational) {1000, 1},
301 (AVRational) {inlink->time_base.den, inlink->time_base.num});
302 memcpy(wctx->audio_buffer + wctx->audio_buffer_fill_size, input_data, samples * sizeof(*wctx->audio_buffer));
303 wctx->audio_buffer_fill_size += samples;
304
305 if (wctx->ctx_vad
306 && (wctx->audio_buffer_fill_size - wctx->audio_buffer_vad_size) >=
307 av_rescale(wctx->vad_min_speech_duration + wctx->vad_min_silence_duration, WHISPER_SAMPLE_RATE, AV_TIME_BASE)) {
308 struct whisper_vad_segments *segments = whisper_vad_segments_from_samples(wctx->ctx_vad,
309 wctx->vad_params,
310 wctx->audio_buffer,
311 wctx->audio_buffer_fill_size);
312 wctx->audio_buffer_vad_size = wctx->audio_buffer_fill_size;
313
314 if (!segments) {
315 av_log(ctx, AV_LOG_ERROR, "failed to detect VAD\n");
316 } else {
317 int n_segments = whisper_vad_segments_n_segments(segments);
318
319 if (n_segments > 0) {
320 const float start_ms = whisper_vad_segments_get_segment_t0(segments, 0) * 10.0;
321 const float end_ms = whisper_vad_segments_get_segment_t1(segments, n_segments - 1) * 10.0;
322 int end_pos = (int) (end_ms * WHISPER_SAMPLE_RATE / 1000);
323
324 if (end_pos <= wctx->audio_buffer_fill_size -
325 av_rescale(wctx->vad_min_silence_duration, WHISPER_SAMPLE_RATE, AV_TIME_BASE)) {
326 av_log(ctx, AV_LOG_INFO,
327 "VAD detected %d segments, start: %.0f ms, end: %.0f ms (buffer: %d ms)\n",
328 n_segments, start_ms, end_ms, 1000 * wctx->audio_buffer_fill_size / WHISPER_SAMPLE_RATE);
329 run_transcription(ctx, frame, end_pos);
330 }
331 }
332
333 whisper_vad_free_segments(segments);
334 }
335 } else if (wctx->audio_buffer_fill_size >= wctx->audio_buffer_queue_size)
336 run_transcription(ctx, frame, wctx->audio_buffer_fill_size);
337
338 wctx->next_pts = frame->pts + av_rescale_q(samples, (AVRational) {
339 1, inlink->sample_rate}
340 , inlink->time_base);
341 return ff_filter_frame(outlink, frame);
342}
343
344static int push_last_frame(AVFilterLink *outlink)
345{
346 AVFilterContext *ctx = outlink->src;
347 WhisperContext *wctx = ctx->priv;
348 AVFrame *frame;
349 int n_out = 1;
350
351 if (ctx->is_disabled || wctx->audio_buffer_fill_size == 0)
352 return 0;
353 frame = ff_get_audio_buffer(outlink, n_out);
354 if (!frame)
355 return AVERROR(ENOMEM);
356
357 av_samples_set_silence(frame->extended_data, 0, n_out, frame->ch_layout.nb_channels, frame->format);
358
359 frame->pts = wctx->next_pts;
360 if (wctx->next_pts != AV_NOPTS_VALUE)
361 wctx->next_pts += av_rescale_q(n_out, (AVRational) {
362 1, outlink->sample_rate}
363 , outlink->time_base);
364
365 run_transcription(ctx, frame, wctx->audio_buffer_fill_size);
366
367 return ff_filter_frame(outlink, frame);
368}
369
370static int activate(AVFilterContext *ctx)
371{
372 AVFilterLink *inlink = ctx->inputs[0];
373 AVFilterLink *outlink = ctx->outputs[0];
374 WhisperContext *wctx = ctx->priv;
375 int64_t pts;
376 int status;
377
378 FF_FILTER_FORWARD_STATUS_BACK(outlink, inlink);
379
380 if (!wctx->eof && ff_inlink_queued_frames(inlink)) {
381 AVFrame *frame = NULL;
382 int ret;
383
384 ret = ff_inlink_consume_frame(inlink, &frame);
385 if (ret < 0)
386 return ret;
387 if (ret > 0)
388 return filter_frame(inlink, frame);
389 }
390
391 if (!wctx->eof && ff_inlink_acknowledge_status(inlink, &status, &pts))
392 wctx->eof = status == AVERROR_EOF;
393
394 if (wctx->eof) {
395 push_last_frame(outlink);
396
397 ff_outlink_set_status(outlink, AVERROR_EOF, wctx->next_pts);
398 return 0;
399 }
400
401 FF_FILTER_FORWARD_WANTED(outlink, inlink);
402
403 return FFERROR_NOT_READY;
404}
405
406static int query_formats(const AVFilterContext *ctx,
407 AVFilterFormatsConfig **cfg_in,
408 AVFilterFormatsConfig **cfg_out)
409{
410 static const enum AVSampleFormat sample_fmts[] = { AV_SAMPLE_FMT_FLT, AV_SAMPLE_FMT_NONE };
411 AVChannelLayout chlayouts[] = { FF_COUNT2LAYOUT(1), { 0 } };
412 int sample_rates[] = { WHISPER_SAMPLE_RATE, -1 };
413 int ret;
414
415 ret = ff_set_common_formats_from_list2(ctx, cfg_in, cfg_out, sample_fmts);
416 if (ret < 0)
417 return ret;
418
419 ret = ff_set_common_channel_layouts_from_list2(ctx, cfg_in, cfg_out, chlayouts);
420 if (ret < 0)
421 return ret;
422
423 return ff_set_common_samplerates_from_list2(ctx, cfg_in, cfg_out, sample_rates);
424}
425
426#define OFFSET(x) offsetof(WhisperContext, x)
427#define FLAGS AV_OPT_FLAG_AUDIO_PARAM | AV_OPT_FLAG_FILTERING_PARAM
428#define HOURS 3600000000
429
430static const AVOption whisper_options[] = {
431 { "model", "Path to the whisper.cpp model file", OFFSET(model_path), AV_OPT_TYPE_STRING,.flags = FLAGS }, { "language", "Language for transcription ('auto' for auto-detect)", OFFSET(language), AV_OPT_TYPE_STRING, {.str = "auto"}, .flags = FLAGS },
432 { "queue", "Audio queue size", OFFSET(queue), AV_OPT_TYPE_DURATION, {.i64 = 3000000}, 20000, HOURS, .flags = FLAGS },
433 { "use_gpu", "Use GPU for processing", OFFSET(use_gpu), AV_OPT_TYPE_BOOL, {.i64 = 1}, 0, 1, .flags = FLAGS },
434 { "gpu_device", "GPU device to use", OFFSET(gpu_device), AV_OPT_TYPE_INT, {.i64 = 0}, 0, INT_MAX, .flags = FLAGS },
435 { "destination", "Output destination", OFFSET(destination), AV_OPT_TYPE_STRING, {.str = ""}, .flags = FLAGS },
436 { "format", "Output format (text|srt|json)", OFFSET(format), AV_OPT_TYPE_STRING, {.str = "text"},.flags = FLAGS },
437 { "vad_model", "Path to the VAD model file", OFFSET(vad_model_path), AV_OPT_TYPE_STRING,.flags = FLAGS },
438 { "vad_threshold", "VAD threshold", OFFSET(vad_threshold), AV_OPT_TYPE_FLOAT, {.dbl = 0.5}, 0.0, 1.0, .flags = FLAGS },
439 { "vad_min_speech_duration", "Minimum speech duration for VAD", OFFSET(vad_min_speech_duration), AV_OPT_TYPE_DURATION, {.i64 = 100000}, 20000, HOURS, .flags = FLAGS },
440 { "vad_min_silence_duration", "Minimum silence duration for VAD", OFFSET(vad_min_silence_duration), AV_OPT_TYPE_DURATION, {.i64 = 500000}, 0, HOURS, .flags = FLAGS },
441 { NULL }
442};
443
444static const AVClass whisper_class = {
445 .class_name = "whisper",
446 .item_name = av_default_item_name,
447 .option = whisper_options,
448 .version = LIBAVUTIL_VERSION_INT,
449};
450
451const FFFilter ff_af_whisper = {
452 .p.name = "whisper",
453 .p.description = NULL_IF_CONFIG_SMALL("Transcribe audio using whisper.cpp."),
454 .p.priv_class = &whisper_class,
455 .p.flags = AVFILTER_FLAG_METADATA_ONLY,
456 .init = init,
457 .uninit = uninit,
458 .activate = activate,
459 .priv_size = sizeof(WhisperContext),
460 FILTER_INPUTS(ff_audio_default_filterpad),
461 FILTER_OUTPUTS(ff_audio_default_filterpad),
462 FILTER_QUERY_FUNC2(query_formats),
463};
libavfilter/allfilters.c+1−0
@@ -162,6 +162,7 @@ extern const FFFilter ff_af_vibrato; · 161 unmodified lines
162extern const FFFilter ff_af_virtualbass;162extern const FFFilter ff_af_virtualbass;
163extern const FFFilter ff_af_volume;163extern const FFFilter ff_af_volume;
164extern const FFFilter ff_af_volumedetect;164extern const FFFilter ff_af_volumedetect;
165extern const FFFilter ff_af_whisper;
165 166
166extern const FFFilter ff_asrc_aevalsrc;167extern const FFFilter ff_asrc_aevalsrc;
167extern const FFFilter ff_asrc_afdelaysrc;168extern const FFFilter ff_asrc_afdelaysrc;

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t