History

dnn/vf_dnn_detect.c: add tensorflow output parse support

Testing model is tensorflow offical model in github repo, please refer https://github.com/tensorflow/models/blob/master/research/object_detection/g3doc/tf2_detection_zoo.md to download the detect model as you need. For example, local testing was carried on with 'ssd_mobilenet_v2_coco_2018_03_29.tar.gz', and used one image of dog in https://github.com/tensorflow/models/blob/master/research/object_detection/test_images/image1.jpg Testing command is: ./ffmpeg -i image1.jpg -vf dnn_detect=dnn_backend=tensorflow:input=image_tensor:output=\ "num_detections&detection_scores&detection_classes&detection_boxes":model=ssd_mobilenet_v2_coco.pb,\ showinfo -f null - We will see the result similar as below: [Parsed_showinfo_1 @ 0x33e65f0] side data - detection bounding boxes: [Parsed_showinfo_1 @ 0x33e65f0] source: ssd_mobilenet_v2_coco.pb [Parsed_showinfo_1 @ 0x33e65f0] index: 0, region: (382, 60) -> (1005, 593), label: 18, confidence: 9834/10000. [Parsed_showinfo_1 @ 0x33e65f0] index: 1, region: (12, 8) -> (328, 549), label: 18, confidence: 8555/10000. [Parsed_showinfo_1 @ 0x33e65f0] index: 2, region: (293, 7) -> (682, 458), label: 1, confidence: 8033/10000. [Parsed_showinfo_1 @ 0x33e65f0] index: 3, region: (342, 0) -> (690, 325), label: 1, confidence: 5878/10000. There are two boxes of dog with cores 94.05% & 93.45% and two boxes of person with scores 80.33% & 58.78%.

Ting Fu committed 5 years ago

Commit c38bc561 parent e42125e1 file · +94 −1Browse files

Showing the changes to libavfilter (1 of 1 file). Show all

Unified · Split

libavfilter/vf_dnn_detect.c+94−1
@@ -48,6 +48,9 @@ typedef struct DnnDetectContext { · 47 unmodified lines
48#define FLAGS AV_OPT_FLAG_FILTERING_PARAM | AV_OPT_FLAG_VIDEO_PARAM48#define FLAGS AV_OPT_FLAG_FILTERING_PARAM | AV_OPT_FLAG_VIDEO_PARAM
49static const AVOption dnn_detect_options[] = {49static const AVOption dnn_detect_options[] = {
50 { "dnn_backend", "DNN backend", OFFSET(backend_type), AV_OPT_TYPE_INT, { .i64 = 2 }, INT_MIN, INT_MAX, FLAGS, "backend" },50 { "dnn_backend", "DNN backend", OFFSET(backend_type), AV_OPT_TYPE_INT, { .i64 = 2 }, INT_MIN, INT_MAX, FLAGS, "backend" },
51#if (CONFIG_LIBTENSORFLOW == 1)
52 { "tensorflow", "tensorflow backend flag", 0, AV_OPT_TYPE_CONST, { .i64 = 1 }, 0, 0, FLAGS, "backend" },
53#endif
51#if (CONFIG_LIBOPENVINO == 1)54#if (CONFIG_LIBOPENVINO == 1)
52 { "openvino", "openvino backend flag", 0, AV_OPT_TYPE_CONST, { .i64 = 2 }, 0, 0, FLAGS, "backend" },55 { "openvino", "openvino backend flag", 0, AV_OPT_TYPE_CONST, { .i64 = 2 }, 0, 0, FLAGS, "backend" },
53#endif56#endif
@@ -59,7 +62,7 @@ static const AVOption dnn_detect_options[] = { · 5 unmodified lines
59 62
60AVFILTER_DEFINE_CLASS(dnn_detect);63AVFILTER_DEFINE_CLASS(dnn_detect);
61 64
62static int dnn_detect_post_proc(AVFrame *frame, DNNData *output, uint32_t nb, AVFilterContext *filter_ctx)65static int dnn_detect_post_proc_ov(AVFrame *frame, DNNData *output, AVFilterContext *filter_ctx)
63{66{
64 DnnDetectContext *ctx = filter_ctx->priv;67 DnnDetectContext *ctx = filter_ctx->priv;
65 float conf_threshold = ctx->confidence;68 float conf_threshold = ctx->confidence;
@@ -136,6 +139,96 @@ static int dnn_detect_post_proc(AVFrame *frame, DNNData *output, uint32_t nb, AV · 70 unmodified lines
136 return 0;139 return 0;
137}140}
138 141
142static int dnn_detect_post_proc_tf(AVFrame *frame, DNNData *output, AVFilterContext *filter_ctx)
143{
144 DnnDetectContext *ctx = filter_ctx->priv;
145 int proposal_count;
146 float conf_threshold = ctx->confidence;
147 float *conf, *position, *label_id, x0, y0, x1, y1;
148 int nb_bboxes = 0;
149 AVFrameSideData *sd;
150 AVDetectionBBox *bbox;
151 AVDetectionBBoxHeader *header;
152
153 proposal_count = *(float *)(output[0].data);
154 conf = output[1].data;
155 position = output[3].data;
156 label_id = output[2].data;
157
158 sd = av_frame_get_side_data(frame, AV_FRAME_DATA_DETECTION_BBOXES);
159 if (sd) {
160 av_log(filter_ctx, AV_LOG_ERROR, "already have dnn bounding boxes in side data.\n");
161 return -1;
162 }
163
164 for (int i = 0; i < proposal_count; ++i) {
165 if (conf[i] < conf_threshold)
166 continue;
167 nb_bboxes++;
168 }
169
170 if (nb_bboxes == 0) {
171 av_log(filter_ctx, AV_LOG_VERBOSE, "nothing detected in this frame.\n");
172 return 0;
173 }
174
175 header = av_detection_bbox_create_side_data(frame, nb_bboxes);
176 if (!header) {
177 av_log(filter_ctx, AV_LOG_ERROR, "failed to create side data with %d bounding boxes\n", nb_bboxes);
178 return -1;
179 }
180
181 av_strlcpy(header->source, ctx->dnnctx.model_filename, sizeof(header->source));
182
183 for (int i = 0; i < proposal_count; ++i) {
184 y0 = position[i * 4];
185 x0 = position[i * 4 + 1];
186 y1 = position[i * 4 + 2];
187 x1 = position[i * 4 + 3];
188
189 bbox = av_get_detection_bbox(header, i);
190
191 if (conf[i] < conf_threshold) {
192 continue;
193 }
194
195 bbox->x = (int)(x0 * frame->width);
196 bbox->w = (int)(x1 * frame->width) - bbox->x;
197 bbox->y = (int)(y0 * frame->height);
198 bbox->h = (int)(y1 * frame->height) - bbox->y;
199
200 bbox->detect_confidence = av_make_q((int)(conf[i] * 10000), 10000);
201 bbox->classify_count = 0;
202
203 if (ctx->labels && label_id[i] < ctx->label_count) {
204 av_strlcpy(bbox->detect_label, ctx->labels[(int)label_id[i]], sizeof(bbox->detect_label));
205 } else {
206 snprintf(bbox->detect_label, sizeof(bbox->detect_label), "%d", (int)label_id[i]);
207 }
208
209 nb_bboxes--;
210 if (nb_bboxes == 0) {
211 break;
212 }
213 }
214 return 0;
215}
216
217static int dnn_detect_post_proc(AVFrame *frame, DNNData *output, uint32_t nb, AVFilterContext *filter_ctx)
218{
219 DnnDetectContext *ctx = filter_ctx->priv;
220 DnnContext *dnn_ctx = &ctx->dnnctx;
221 switch (dnn_ctx->backend_type) {
222 case DNN_OV:
223 return dnn_detect_post_proc_ov(frame, output, filter_ctx);
224 case DNN_TF:
225 return dnn_detect_post_proc_tf(frame, output, filter_ctx);
226 default:
227 avpriv_report_missing_feature(filter_ctx, "Current dnn backend does not support detect filter\n");
228 return AVERROR(EINVAL);
229 }
230}
231
139static void free_detect_labels(DnnDetectContext *ctx)232static void free_detect_labels(DnnDetectContext *ctx)
140{233{
141 for (int i = 0; i < ctx->label_count; i++) {234 for (int i = 0; i < ctx->label_count; i++) {

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t