History

lavfi: add filter dnn_detect for object detection

Below are the example steps to do object detection: 1. download and install l_openvino_toolkit_p_2021.1.110.tgz from https://software.intel.com/content/www/us/en/develop/tools/openvino-toolkit/download.html or, we can get source code (tag 2021.1), build and install. 2. export LD_LIBRARY_PATH with openvino settings, for example: .../deployment_tools/inference_engine/lib/intel64/:.../deployment_tools/inference_engine/external/tbb/lib/ 3. rebuild ffmpeg from source code with configure option: --enable-libopenvino --extra-cflags='-I.../deployment_tools/inference_engine/include/' --extra-ldflags='-L.../deployment_tools/inference_engine/lib/intel64' 4. download model files and test image wget https://github.com/guoyejun/ffmpeg_dnn/raw/main/models/openvino/2021.1/face-detection-adas-0001.bin wget https://github.com/guoyejun/ffmpeg_dnn/raw/main/models/openvino/2021.1/face-detection-adas-0001.xml wget https://github.com/guoyejun/ffmpeg_dnn/raw/main/models/openvino/2021.1/face-detection-adas-0001.label wget https://github.com/guoyejun/ffmpeg_dnn/raw/main/images/cici.jpg 5. run ffmpeg with: ./ffmpeg -i cici.jpg -vf dnn_detect=dnn_backend=openvino:model=face-detection-adas-0001.xml:input=data:output=detection_out:confidence=0.6:labels=face-detection-adas-0001.label,showinfo -f null - We'll see the detect result as below: [Parsed_showinfo_1 @ 0x560c21ecbe40] side data - detection bounding boxes: [Parsed_showinfo_1 @ 0x560c21ecbe40] source: face-detection-adas-0001.xml [Parsed_showinfo_1 @ 0x560c21ecbe40] index: 0, region: (1005, 813) -> (1086, 905), label: face, confidence: 10000/10000. [Parsed_showinfo_1 @ 0x560c21ecbe40] index: 1, region: (888, 839) -> (967, 926), label: face, confidence: 6917/10000. There are two faces detected with confidence 100% and 69.17%.

Guo, Yejun committed 5 years ago

Commit aa9ffda1 parent e942b4b5 files · +464 −0Browse files

Unified · Split

5 files
  1. configure +1 −0
  2. doc/filters.texi +40 −0
  3. libavfilter/Makefile +1 −0
  4. libavfilter/allfilters.c +1 −0
  5. libavfilter/vf_dnn_detect.c +421 −0
configure+1−0
@@ -3555,6 +3555,7 @@ derain_filter_select="dnn" · 3554 unmodified lines
3555deshake_filter_select="pixelutils"3555deshake_filter_select="pixelutils"
3556deshake_opencl_filter_deps="opencl"3556deshake_opencl_filter_deps="opencl"
3557dilation_opencl_filter_deps="opencl"3557dilation_opencl_filter_deps="opencl"
3558dnn_detect_filter_select="dnn"
3558dnn_processing_filter_select="dnn"3559dnn_processing_filter_select="dnn"
3559drawtext_filter_deps="libfreetype"3560drawtext_filter_deps="libfreetype"
3560drawtext_filter_suggest="libfontconfig libfribidi"3561drawtext_filter_suggest="libfontconfig libfribidi"
doc/filters.texi+40−0
@@ -10127,6 +10127,46 @@ ffmpeg -i INPUT -f lavfi -i nullsrc=hd720,geq='r=128+80*(sin(sqrt((X-W/2)*(X-W/2 · 10126 unmodified lines
10127@end example10127@end example
10128@end itemize10128@end itemize
10129 10129
10130@section dnn_detect
10131
10132Do object detection with deep neural networks.
10133
10134The filter accepts the following options:
10135
10136@table @option
10137@item dnn_backend
10138Specify which DNN backend to use for model loading and execution. This option accepts
10139only openvino now, tensorflow backends will be added.
10140
10141@item model
10142Set path to model file specifying network architecture and its parameters.
10143Note that different backends use different file formats.
10144
10145@item input
10146Set the input name of the dnn network.
10147
10148@item output
10149Set the output name of the dnn network.
10150
10151@item confidence
10152Set the confidence threshold (default: 0.5).
10153
10154@item labels
10155Set path to label file specifying the mapping between label id and name.
10156Each label name is written in one line, tailing spaces and empty lines are skipped.
10157The first line is the name of label id 0 (usually it is 'background'),
10158and the second line is the name of label id 1, etc.
10159The label id is considered as name if the label file is not provided.
10160
10161@item backend_configs
10162Set the configs to be passed into backend
10163
10164@item async
10165use DNN async execution if set (default: set),
10166roll back to sync execution if the backend does not support async.
10167
10168@end table
10169
10130@anchor{dnn_processing}10170@anchor{dnn_processing}
10131@section dnn_processing10171@section dnn_processing
10132 10172
libavfilter/Makefile+1−0
@@ -245,6 +245,7 @@ OBJS-$(CONFIG_DILATION_FILTER) += vf_neighbor.o · 244 unmodified lines
245OBJS-$(CONFIG_DILATION_OPENCL_FILTER) += vf_neighbor_opencl.o opencl.o \245OBJS-$(CONFIG_DILATION_OPENCL_FILTER) += vf_neighbor_opencl.o opencl.o \
246 opencl/neighbor.o246 opencl/neighbor.o
247OBJS-$(CONFIG_DISPLACE_FILTER) += vf_displace.o framesync.o247OBJS-$(CONFIG_DISPLACE_FILTER) += vf_displace.o framesync.o
248OBJS-$(CONFIG_DNN_DETECT_FILTER) += vf_dnn_detect.o
248OBJS-$(CONFIG_DNN_PROCESSING_FILTER) += vf_dnn_processing.o249OBJS-$(CONFIG_DNN_PROCESSING_FILTER) += vf_dnn_processing.o
249OBJS-$(CONFIG_DOUBLEWEAVE_FILTER) += vf_weave.o250OBJS-$(CONFIG_DOUBLEWEAVE_FILTER) += vf_weave.o
250OBJS-$(CONFIG_DRAWBOX_FILTER) += vf_drawbox.o251OBJS-$(CONFIG_DRAWBOX_FILTER) += vf_drawbox.o
libavfilter/allfilters.c+1−0
@@ -230,6 +230,7 @@ extern AVFilter ff_vf_detelecine; · 229 unmodified lines
230extern AVFilter ff_vf_dilation;230extern AVFilter ff_vf_dilation;
231extern AVFilter ff_vf_dilation_opencl;231extern AVFilter ff_vf_dilation_opencl;
232extern AVFilter ff_vf_displace;232extern AVFilter ff_vf_displace;
233extern AVFilter ff_vf_dnn_detect;
233extern AVFilter ff_vf_dnn_processing;234extern AVFilter ff_vf_dnn_processing;
234extern AVFilter ff_vf_doubleweave;235extern AVFilter ff_vf_doubleweave;
235extern AVFilter ff_vf_drawbox;236extern AVFilter ff_vf_drawbox;
libavfilter/vf_dnn_detect.c+421−0
@@ -0,0 +1,421 @@
1/*
2 * This file is part of FFmpeg.
3 *
4 * FFmpeg is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU Lesser General Public
6 * License as published by the Free Software Foundation; either
7 * version 2.1 of the License, or (at your option) any later version.
8 *
9 * FFmpeg is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 * Lesser General Public License for more details.
13 *
14 * You should have received a copy of the GNU Lesser General Public
15 * License along with FFmpeg; if not, write to the Free Software
16 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 */
18
19/**
20 * @file
21 * implementing an object detecting filter using deep learning networks.
22 */
23
24#include "libavformat/avio.h"
25#include "libavutil/opt.h"
26#include "libavutil/pixdesc.h"
27#include "libavutil/avassert.h"
28#include "libavutil/imgutils.h"
29#include "filters.h"
30#include "dnn_filter_common.h"
31#include "formats.h"
32#include "internal.h"
33#include "libavutil/time.h"
34#include "libavutil/avstring.h"
35#include "libavutil/detection_bbox.h"
36
37typedef struct DnnDetectContext {
38 const AVClass *class;
39 DnnContext dnnctx;
40 float confidence;
41 char *labels_filename;
42 char **labels;
43 int label_count;
44} DnnDetectContext;
45
46#define OFFSET(x) offsetof(DnnDetectContext, dnnctx.x)
47#define OFFSET2(x) offsetof(DnnDetectContext, x)
48#define FLAGS AV_OPT_FLAG_FILTERING_PARAM | AV_OPT_FLAG_VIDEO_PARAM
49static const AVOption dnn_detect_options[] = {
50 { "dnn_backend", "DNN backend", OFFSET(backend_type), AV_OPT_TYPE_INT, { .i64 = 2 }, INT_MIN, INT_MAX, FLAGS, "backend" },
51#if (CONFIG_LIBOPENVINO == 1)
52 { "openvino", "openvino backend flag", 0, AV_OPT_TYPE_CONST, { .i64 = 2 }, 0, 0, FLAGS, "backend" },
53#endif
54 DNN_COMMON_OPTIONS
55 { "confidence", "threshold of confidence", OFFSET2(confidence), AV_OPT_TYPE_FLOAT, { .dbl = 0.5 }, 0, 1, FLAGS},
56 { "labels", "path to labels file", OFFSET2(labels_filename), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, FLAGS },
57 { NULL }
58};
59
60AVFILTER_DEFINE_CLASS(dnn_detect);
61
62static int dnn_detect_post_proc(AVFrame *frame, DNNData *output, uint32_t nb, AVFilterContext *filter_ctx)
63{
64 DnnDetectContext *ctx = filter_ctx->priv;
65 float conf_threshold = ctx->confidence;
66 int proposal_count = output->height;
67 int detect_size = output->width;
68 float *detections = output->data;
69 int nb_bboxes = 0;
70 AVFrameSideData *sd;
71 AVDetectionBBox *bbox;
72 AVDetectionBBoxHeader *header;
73
74 sd = av_frame_get_side_data(frame, AV_FRAME_DATA_DETECTION_BBOXES);
75 if (sd) {
76 av_log(filter_ctx, AV_LOG_ERROR, "already have bounding boxes in side data.\n");
77 return -1;
78 }
79
80 for (int i = 0; i < proposal_count; ++i) {
81 float conf = detections[i * detect_size + 2];
82 if (conf < conf_threshold) {
83 continue;
84 }
85 nb_bboxes++;
86 }
87
88 if (nb_bboxes == 0) {
89 av_log(filter_ctx, AV_LOG_VERBOSE, "nothing detected in this frame.\n");
90 return 0;
91 }
92
93 header = av_detection_bbox_create_side_data(frame, nb_bboxes);
94 if (!header) {
95 av_log(filter_ctx, AV_LOG_ERROR, "failed to create side data with %d bounding boxes\n", nb_bboxes);
96 return -1;
97 }
98
99 av_strlcpy(header->source, ctx->dnnctx.model_filename, sizeof(header->source));
100
101 for (int i = 0; i < proposal_count; ++i) {
102 int av_unused image_id = (int)detections[i * detect_size + 0];
103 int label_id = (int)detections[i * detect_size + 1];
104 float conf = detections[i * detect_size + 2];
105 float x0 = detections[i * detect_size + 3];
106 float y0 = detections[i * detect_size + 4];
107 float x1 = detections[i * detect_size + 5];
108 float y1 = detections[i * detect_size + 6];
109
110 bbox = av_get_detection_bbox(header, i);
111
112 if (conf < conf_threshold) {
113 continue;
114 }
115
116 bbox->x = (int)(x0 * frame->width);
117 bbox->w = (int)(x1 * frame->width) - bbox->x;
118 bbox->y = (int)(y0 * frame->height);
119 bbox->h = (int)(y1 * frame->height) - bbox->y;
120
121 bbox->detect_confidence = av_make_q((int)(conf * 10000), 10000);
122 bbox->classify_count = 0;
123
124 if (ctx->labels && label_id < ctx->label_count) {
125 av_strlcpy(bbox->detect_label, ctx->labels[label_id], sizeof(bbox->detect_label));
126 } else {
127 snprintf(bbox->detect_label, sizeof(bbox->detect_label), "%d", label_id);
128 }
129
130 nb_bboxes--;
131 if (nb_bboxes == 0) {
132 break;
133 }
134 }
135
136 return 0;
137}
138
139static void free_detect_labels(DnnDetectContext *ctx)
140{
141 for (int i = 0; i < ctx->label_count; i++) {
142 av_freep(&ctx->labels[i]);
143 }
144 ctx->label_count = 0;
145 av_freep(&ctx->labels);
146}
147
148static int read_detect_label_file(AVFilterContext *context)
149{
150 int line_len;
151 FILE *file;
152 DnnDetectContext *ctx = context->priv;
153
154 file = av_fopen_utf8(ctx->labels_filename, "r");
155 if (!file){
156 av_log(context, AV_LOG_ERROR, "failed to open file %s\n", ctx->labels_filename);
157 return AVERROR(EINVAL);
158 }
159
160 while (!feof(file)) {
161 char *label;
162 char buf[256];
163 if (!fgets(buf, 256, file)) {
164 break;
165 }
166
167 line_len = strlen(buf);
168 while (line_len) {
169 int i = line_len - 1;
170 if (buf[i] == '\n' || buf[i] == '\r' || buf[i] == ' ') {
171 buf[i] = '\0';
172 line_len--;
173 } else {
174 break;
175 }
176 }
177
178 if (line_len == 0) // empty line
179 continue;
180
181 if (line_len >= AV_DETECTION_BBOX_LABEL_NAME_MAX_SIZE) {
182 av_log(context, AV_LOG_ERROR, "label %s too long\n", buf);
183 fclose(file);
184 return AVERROR(EINVAL);
185 }
186
187 label = av_strdup(buf);
188 if (!label) {
189 av_log(context, AV_LOG_ERROR, "failed to allocate memory for label %s\n", buf);
190 fclose(file);
191 return AVERROR(ENOMEM);
192 }
193
194 if (av_dynarray_add_nofree(&ctx->labels, &ctx->label_count, label) < 0) {
195 av_log(context, AV_LOG_ERROR, "failed to do av_dynarray_add\n");
196 fclose(file);
197 av_freep(&label);
198 return AVERROR(ENOMEM);
199 }
200 }
201
202 fclose(file);
203 return 0;
204}
205
206static av_cold int dnn_detect_init(AVFilterContext *context)
207{
208 DnnDetectContext *ctx = context->priv;
209 int ret = ff_dnn_init(&ctx->dnnctx, DFT_ANALYTICS_DETECT, context);
210 if (ret < 0)
211 return ret;
212 ff_dnn_set_detect_post_proc(&ctx->dnnctx, dnn_detect_post_proc);
213
214 if (ctx->labels_filename) {
215 return read_detect_label_file(context);
216 }
217 return 0;
218}
219
220static int dnn_detect_query_formats(AVFilterContext *context)
221{
222 static const enum AVPixelFormat pix_fmts[] = {
223 AV_PIX_FMT_RGB24, AV_PIX_FMT_BGR24,
224 AV_PIX_FMT_GRAY8, AV_PIX_FMT_GRAYF32,
225 AV_PIX_FMT_YUV420P, AV_PIX_FMT_YUV422P,
226 AV_PIX_FMT_YUV444P, AV_PIX_FMT_YUV410P, AV_PIX_FMT_YUV411P,
227 AV_PIX_FMT_NV12,
228 AV_PIX_FMT_NONE
229 };
230 AVFilterFormats *fmts_list = ff_make_format_list(pix_fmts);
231 return ff_set_common_formats(context, fmts_list);
232}
233
234static int dnn_detect_filter_frame(AVFilterLink *inlink, AVFrame *in)
235{
236 AVFilterContext *context = inlink->dst;
237 AVFilterLink *outlink = context->outputs[0];
238 DnnDetectContext *ctx = context->priv;
239 DNNReturnType dnn_result;
240
241 dnn_result = ff_dnn_execute_model(&ctx->dnnctx, in, in);
242 if (dnn_result != DNN_SUCCESS){
243 av_log(ctx, AV_LOG_ERROR, "failed to execute model\n");
244 av_frame_free(&in);
245 return AVERROR(EIO);
246 }
247
248 return ff_filter_frame(outlink, in);
249}
250
251static int dnn_detect_activate_sync(AVFilterContext *filter_ctx)
252{
253 AVFilterLink *inlink = filter_ctx->inputs[0];
254 AVFilterLink *outlink = filter_ctx->outputs[0];
255 AVFrame *in = NULL;
256 int64_t pts;
257 int ret, status;
258 int got_frame = 0;
259
260 FF_FILTER_FORWARD_STATUS_BACK(outlink, inlink);
261
262 do {
263 // drain all input frames
264 ret = ff_inlink_consume_frame(inlink, &in);
265 if (ret < 0)
266 return ret;
267 if (ret > 0) {
268 ret = dnn_detect_filter_frame(inlink, in);
269 if (ret < 0)
270 return ret;
271 got_frame = 1;
272 }
273 } while (ret > 0);
274
275 // if frame got, schedule to next filter
276 if (got_frame)
277 return 0;
278
279 if (ff_inlink_acknowledge_status(inlink, &status, &pts)) {
280 if (status == AVERROR_EOF) {
281 ff_outlink_set_status(outlink, status, pts);
282 return ret;
283 }
284 }
285
286 FF_FILTER_FORWARD_WANTED(outlink, inlink);
287
288 return FFERROR_NOT_READY;
289}
290
291static int dnn_detect_flush_frame(AVFilterLink *outlink, int64_t pts, int64_t *out_pts)
292{
293 DnnDetectContext *ctx = outlink->src->priv;
294 int ret;
295 DNNAsyncStatusType async_state;
296
297 ret = ff_dnn_flush(&ctx->dnnctx);
298 if (ret != DNN_SUCCESS) {
299 return -1;
300 }
301
302 do {
303 AVFrame *in_frame = NULL;
304 AVFrame *out_frame = NULL;
305 async_state = ff_dnn_get_async_result(&ctx->dnnctx, &in_frame, &out_frame);
306 if (out_frame) {
307 av_assert0(in_frame == out_frame);
308 ret = ff_filter_frame(outlink, out_frame);
309 if (ret < 0)
310 return ret;
311 if (out_pts)
312 *out_pts = out_frame->pts + pts;
313 }
314 av_usleep(5000);
315 } while (async_state >= DAST_NOT_READY);
316
317 return 0;
318}
319
320static int dnn_detect_activate_async(AVFilterContext *filter_ctx)
321{
322 AVFilterLink *inlink = filter_ctx->inputs[0];
323 AVFilterLink *outlink = filter_ctx->outputs[0];
324 DnnDetectContext *ctx = filter_ctx->priv;
325 AVFrame *in = NULL;
326 int64_t pts;
327 int ret, status;
328 int got_frame = 0;
329 int async_state;
330
331 FF_FILTER_FORWARD_STATUS_BACK(outlink, inlink);
332
333 do {
334 // drain all input frames
335 ret = ff_inlink_consume_frame(inlink, &in);
336 if (ret < 0)
337 return ret;
338 if (ret > 0) {
339 if (ff_dnn_execute_model_async(&ctx->dnnctx, in, in) != DNN_SUCCESS) {
340 return AVERROR(EIO);
341 }
342 }
343 } while (ret > 0);
344
345 // drain all processed frames
346 do {
347 AVFrame *in_frame = NULL;
348 AVFrame *out_frame = NULL;
349 async_state = ff_dnn_get_async_result(&ctx->dnnctx, &in_frame, &out_frame);
350 if (out_frame) {
351 av_assert0(in_frame == out_frame);
352 ret = ff_filter_frame(outlink, out_frame);
353 if (ret < 0)
354 return ret;
355 got_frame = 1;
356 }
357 } while (async_state == DAST_SUCCESS);
358
359 // if frame got, schedule to next filter
360 if (got_frame)
361 return 0;
362
363 if (ff_inlink_acknowledge_status(inlink, &status, &pts)) {
364 if (status == AVERROR_EOF) {
365 int64_t out_pts = pts;
366 ret = dnn_detect_flush_frame(outlink, pts, &out_pts);
367 ff_outlink_set_status(outlink, status, out_pts);
368 return ret;
369 }
370 }
371
372 FF_FILTER_FORWARD_WANTED(outlink, inlink);
373
374 return 0;
375}
376
377static int dnn_detect_activate(AVFilterContext *filter_ctx)
378{
379 DnnDetectContext *ctx = filter_ctx->priv;
380
381 if (ctx->dnnctx.async)
382 return dnn_detect_activate_async(filter_ctx);
383 else
384 return dnn_detect_activate_sync(filter_ctx);
385}
386
387static av_cold void dnn_detect_uninit(AVFilterContext *context)
388{
389 DnnDetectContext *ctx = context->priv;
390 ff_dnn_uninit(&ctx->dnnctx);
391 free_detect_labels(ctx);
392}
393
394static const AVFilterPad dnn_detect_inputs[] = {
395 {
396 .name = "default",
397 .type = AVMEDIA_TYPE_VIDEO,
398 },
399 { NULL }
400};
401
402static const AVFilterPad dnn_detect_outputs[] = {
403 {
404 .name = "default",
405 .type = AVMEDIA_TYPE_VIDEO,
406 },
407 { NULL }
408};
409
410AVFilter ff_vf_dnn_detect = {
411 .name = "dnn_detect",
412 .description = NULL_IF_CONFIG_SMALL("Apply DNN detect filter to the input."),
413 .priv_size = sizeof(DnnDetectContext),
414 .init = dnn_detect_init,
415 .uninit = dnn_detect_uninit,
416 .query_formats = dnn_detect_query_formats,
417 .inputs = dnn_detect_inputs,
418 .outputs = dnn_detect_outputs,
419 .priv_class = &dnn_detect_class,
420 .activate = dnn_detect_activate,
421};

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t