Testing model is tensorflow offical model in github repo, please refer https://github.com/tensorflow/models/blob/master/research/object_detection/g3doc/tf2_detection_zoo.md to download the detect model as you need. For example, local testing was carried on with 'ssd_mobilenet_v2_coco_2018_03_29.tar.gz', and used one image of dog in https://github.com/tensorflow/models/blob/master/research/object_detection/test_images/image1.jpg Testing command is: ./ffmpeg -i image1.jpg -vf dnn_detect=dnn_backend=tensorflow:input=image_tensor:output=\ "num_detections&detection_scores&detection_classes&detection_boxes":model=ssd_mobilenet_v2_coco.pb,\ showinfo -f null - We will see the result similar as below: [Parsed_showinfo_1 @ 0x33e65f0] side data - detection bounding boxes: [Parsed_showinfo_1 @ 0x33e65f0] source: ssd_mobilenet_v2_coco.pb [Parsed_showinfo_1 @ 0x33e65f0] index: 0, region: (382, 60) -> (1005, 593), label: 18, confidence: 9834/10000. [Parsed_showinfo_1 @ 0x33e65f0] index: 1, region: (12, 8) -> (328, 549), label: 18, confidence: 8555/10000. [Parsed_showinfo_1 @ 0x33e65f0] index: 2, region: (293, 7) -> (682, 458), label: 1, confidence: 8033/10000. [Parsed_showinfo_1 @ 0x33e65f0] index: 3, region: (342, 0) -> (690, 325), label: 1, confidence: 5878/10000. There are two boxes of dog with cores 94.05% & 93.45% and two boxes of person with scores 80.33% & 58.78%.
Showing the changes to libavfilter (1 of 1 file). Show all
Unified · Split
| @@ -48,6 +48,9 @@ typedef struct DnnDetectContext { · 47 unmodified lines | |||
| 48 | #define FLAGS AV_OPT_FLAG_FILTERING_PARAM | AV_OPT_FLAG_VIDEO_PARAM | 48 | #define FLAGS AV_OPT_FLAG_FILTERING_PARAM | AV_OPT_FLAG_VIDEO_PARAM |
| 49 | static const AVOption dnn_detect_options[] = { | 49 | static const AVOption dnn_detect_options[] = { |
| 50 | { "dnn_backend", "DNN backend", OFFSET(backend_type), AV_OPT_TYPE_INT, { .i64 = 2 }, INT_MIN, INT_MAX, FLAGS, "backend" }, | 50 | { "dnn_backend", "DNN backend", OFFSET(backend_type), AV_OPT_TYPE_INT, { .i64 = 2 }, INT_MIN, INT_MAX, FLAGS, "backend" }, |
| 51 | #if (CONFIG_LIBTENSORFLOW == 1) | ||
| 52 | { "tensorflow", "tensorflow backend flag", 0, AV_OPT_TYPE_CONST, { .i64 = 1 }, 0, 0, FLAGS, "backend" }, | ||
| 53 | #endif | ||
| 51 | #if (CONFIG_LIBOPENVINO == 1) | 54 | #if (CONFIG_LIBOPENVINO == 1) |
| 52 | { "openvino", "openvino backend flag", 0, AV_OPT_TYPE_CONST, { .i64 = 2 }, 0, 0, FLAGS, "backend" }, | 55 | { "openvino", "openvino backend flag", 0, AV_OPT_TYPE_CONST, { .i64 = 2 }, 0, 0, FLAGS, "backend" }, |
| 53 | #endif | 56 | #endif |
| @@ -59,7 +62,7 @@ static const AVOption dnn_detect_options[] = { · 5 unmodified lines | |||
| 59 | 62 | ||
| 60 | AVFILTER_DEFINE_CLASS(dnn_detect); | 63 | AVFILTER_DEFINE_CLASS(dnn_detect); |
| 61 | 64 | ||
| 62 | static int dnn_detect_post_proc(AVFrame *frame, DNNData *output, uint32_t nb, AVFilterContext *filter_ctx) | 65 | static int dnn_detect_post_proc_ov(AVFrame *frame, DNNData *output, AVFilterContext *filter_ctx) |
| 63 | { | 66 | { |
| 64 | DnnDetectContext *ctx = filter_ctx->priv; | 67 | DnnDetectContext *ctx = filter_ctx->priv; |
| 65 | float conf_threshold = ctx->confidence; | 68 | float conf_threshold = ctx->confidence; |
| @@ -136,6 +139,96 @@ static int dnn_detect_post_proc(AVFrame *frame, DNNData *output, uint32_t nb, AV · 70 unmodified lines | |||
| 136 | return 0; | 139 | return 0; |
| 137 | } | 140 | } |
| 138 | 141 | ||
| 142 | static int dnn_detect_post_proc_tf(AVFrame *frame, DNNData *output, AVFilterContext *filter_ctx) | ||
| 143 | { | ||
| 144 | DnnDetectContext *ctx = filter_ctx->priv; | ||
| 145 | int proposal_count; | ||
| 146 | float conf_threshold = ctx->confidence; | ||
| 147 | float *conf, *position, *label_id, x0, y0, x1, y1; | ||
| 148 | int nb_bboxes = 0; | ||
| 149 | AVFrameSideData *sd; | ||
| 150 | AVDetectionBBox *bbox; | ||
| 151 | AVDetectionBBoxHeader *header; | ||
| 152 | |||
| 153 | proposal_count = *(float *)(output[0].data); | ||
| 154 | conf = output[1].data; | ||
| 155 | position = output[3].data; | ||
| 156 | label_id = output[2].data; | ||
| 157 | |||
| 158 | sd = av_frame_get_side_data(frame, AV_FRAME_DATA_DETECTION_BBOXES); | ||
| 159 | if (sd) { | ||
| 160 | av_log(filter_ctx, AV_LOG_ERROR, "already have dnn bounding boxes in side data.\n"); | ||
| 161 | return -1; | ||
| 162 | } | ||
| 163 | |||
| 164 | for (int i = 0; i < proposal_count; ++i) { | ||
| 165 | if (conf[i] < conf_threshold) | ||
| 166 | continue; | ||
| 167 | nb_bboxes++; | ||
| 168 | } | ||
| 169 | |||
| 170 | if (nb_bboxes == 0) { | ||
| 171 | av_log(filter_ctx, AV_LOG_VERBOSE, "nothing detected in this frame.\n"); | ||
| 172 | return 0; | ||
| 173 | } | ||
| 174 | |||
| 175 | header = av_detection_bbox_create_side_data(frame, nb_bboxes); | ||
| 176 | if (!header) { | ||
| 177 | av_log(filter_ctx, AV_LOG_ERROR, "failed to create side data with %d bounding boxes\n", nb_bboxes); | ||
| 178 | return -1; | ||
| 179 | } | ||
| 180 | |||
| 181 | av_strlcpy(header->source, ctx->dnnctx.model_filename, sizeof(header->source)); | ||
| 182 | |||
| 183 | for (int i = 0; i < proposal_count; ++i) { | ||
| 184 | y0 = position[i * 4]; | ||
| 185 | x0 = position[i * 4 + 1]; | ||
| 186 | y1 = position[i * 4 + 2]; | ||
| 187 | x1 = position[i * 4 + 3]; | ||
| 188 | |||
| 189 | bbox = av_get_detection_bbox(header, i); | ||
| 190 | |||
| 191 | if (conf[i] < conf_threshold) { | ||
| 192 | continue; | ||
| 193 | } | ||
| 194 | |||
| 195 | bbox->x = (int)(x0 * frame->width); | ||
| 196 | bbox->w = (int)(x1 * frame->width) - bbox->x; | ||
| 197 | bbox->y = (int)(y0 * frame->height); | ||
| 198 | bbox->h = (int)(y1 * frame->height) - bbox->y; | ||
| 199 | |||
| 200 | bbox->detect_confidence = av_make_q((int)(conf[i] * 10000), 10000); | ||
| 201 | bbox->classify_count = 0; | ||
| 202 | |||
| 203 | if (ctx->labels && label_id[i] < ctx->label_count) { | ||
| 204 | av_strlcpy(bbox->detect_label, ctx->labels[(int)label_id[i]], sizeof(bbox->detect_label)); | ||
| 205 | } else { | ||
| 206 | snprintf(bbox->detect_label, sizeof(bbox->detect_label), "%d", (int)label_id[i]); | ||
| 207 | } | ||
| 208 | |||
| 209 | nb_bboxes--; | ||
| 210 | if (nb_bboxes == 0) { | ||
| 211 | break; | ||
| 212 | } | ||
| 213 | } | ||
| 214 | return 0; | ||
| 215 | } | ||
| 216 | |||
| 217 | static int dnn_detect_post_proc(AVFrame *frame, DNNData *output, uint32_t nb, AVFilterContext *filter_ctx) | ||
| 218 | { | ||
| 219 | DnnDetectContext *ctx = filter_ctx->priv; | ||
| 220 | DnnContext *dnn_ctx = &ctx->dnnctx; | ||
| 221 | switch (dnn_ctx->backend_type) { | ||
| 222 | case DNN_OV: | ||
| 223 | return dnn_detect_post_proc_ov(frame, output, filter_ctx); | ||
| 224 | case DNN_TF: | ||
| 225 | return dnn_detect_post_proc_tf(frame, output, filter_ctx); | ||
| 226 | default: | ||
| 227 | avpriv_report_missing_feature(filter_ctx, "Current dnn backend does not support detect filter\n"); | ||
| 228 | return AVERROR(EINVAL); | ||
| 229 | } | ||
| 230 | } | ||
| 231 | |||
| 139 | static void free_detect_labels(DnnDetectContext *ctx) | 232 | static void free_detect_labels(DnnDetectContext *ctx) |
| 140 | { | 233 | { |
| 141 | for (int i = 0; i < ctx->label_count; i++) { | 234 | for (int i = 0; i < ctx->label_count; i++) { |