Add yolo support. Yolo model doesn't output final result. It outputs candidate boxes, so we need post-process to remove overlap boxes to get final results. Also, the box's coordinators relate to cell and anchors, so we need these information to calculate boxes as well. Model detail please refer to: https://github.com/openvinotoolkit/open_model_zoo/tree/master/models/public/yolo-v2-tf
Showing the changes to libavfilter (2 of 2 files). Show all
Unified · Split
| @@ -386,9 +386,9 @@ static void infer_completion_callback(void *args) · 385 unmodified lines | |||
| 386 | ov_shape_free(&output_shape); | 386 | ov_shape_free(&output_shape); |
| 387 | return; | 387 | return; |
| 388 | } | 388 | } |
| 389 | output.channels = dims[1]; | 389 | output.channels = output_shape.rank > 2 ? dims[output_shape.rank - 3] : 1; |
| 390 | output.height = dims[2]; | 390 | output.height = output_shape.rank > 1 ? dims[output_shape.rank - 2] : 1; |
| 391 | output.width = dims[3]; | 391 | output.width = output_shape.rank > 0 ? dims[output_shape.rank - 1] : 1; |
| 392 | av_assert0(request->lltask_count <= dims[0]); | 392 | av_assert0(request->lltask_count <= dims[0]); |
| 393 | ov_shape_free(&output_shape); | 393 | ov_shape_free(&output_shape); |
| 394 | #else | 394 | #else |
| @@ -30,9 +30,11 @@ · 29 unmodified lines | |||
| 30 | #include "libavutil/time.h" | 30 | #include "libavutil/time.h" |
| 31 | #include "libavutil/avstring.h" | 31 | #include "libavutil/avstring.h" |
| 32 | #include "libavutil/detection_bbox.h" | 32 | #include "libavutil/detection_bbox.h" |
| 33 | #include "libavutil/fifo.h" | ||
| 33 | 34 | ||
| 34 | typedef enum { | 35 | typedef enum { |
| 35 | DDMT_SSD | 36 | DDMT_SSD, |
| 37 | DDMT_YOLOV1V2, | ||
| 36 | } DNNDetectionModelType; | 38 | } DNNDetectionModelType; |
| 37 | 39 | ||
| 38 | typedef struct DnnDetectContext { | 40 | typedef struct DnnDetectContext { |
| @@ -43,6 +45,15 @@ typedef struct DnnDetectContext { · 4 unmodified lines | |||
| 43 | char **labels; | 45 | char **labels; |
| 44 | int label_count; | 46 | int label_count; |
| 45 | DNNDetectionModelType model_type; | 47 | DNNDetectionModelType model_type; |
| 48 | int cell_w; | ||
| 49 | int cell_h; | ||
| 50 | int nb_classes; | ||
| 51 | AVFifo *bboxes_fifo; | ||
| 52 | int scale_width; | ||
| 53 | int scale_height; | ||
| 54 | char *anchors_str; | ||
| 55 | float *anchors; | ||
| 56 | int nb_anchor; | ||
| 46 | } DnnDetectContext; | 57 | } DnnDetectContext; |
| 47 | 58 | ||
| 48 | #define OFFSET(x) offsetof(DnnDetectContext, dnnctx.x) | 59 | #define OFFSET(x) offsetof(DnnDetectContext, dnnctx.x) |
| @@ -61,11 +72,218 @@ static const AVOption dnn_detect_options[] = { · 12 unmodified lines | |||
| 61 | { "labels", "path to labels file", OFFSET2(labels_filename), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, FLAGS }, | 72 | { "labels", "path to labels file", OFFSET2(labels_filename), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, FLAGS }, |
| 62 | { "model_type", "DNN detection model type", OFFSET2(model_type), AV_OPT_TYPE_INT, { .i64 = DDMT_SSD }, INT_MIN, INT_MAX, FLAGS, "model_type" }, | 73 | { "model_type", "DNN detection model type", OFFSET2(model_type), AV_OPT_TYPE_INT, { .i64 = DDMT_SSD }, INT_MIN, INT_MAX, FLAGS, "model_type" }, |
| 63 | { "ssd", "output shape [1, 1, N, 7]", 0, AV_OPT_TYPE_CONST, { .i64 = DDMT_SSD }, 0, 0, FLAGS, "model_type" }, | 74 | { "ssd", "output shape [1, 1, N, 7]", 0, AV_OPT_TYPE_CONST, { .i64 = DDMT_SSD }, 0, 0, FLAGS, "model_type" }, |
| 75 | { "yolo", "output shape [1, N*Cx*Cy*DetectionBox]", 0, AV_OPT_TYPE_CONST, { .i64 = DDMT_YOLOV1V2 }, 0, 0, FLAGS, "model_type" }, | ||
| 76 | { "cell_w", "cell width", OFFSET2(cell_w), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, INTMAX_MAX, FLAGS }, | ||
| 77 | { "cell_h", "cell height", OFFSET2(cell_h), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, INTMAX_MAX, FLAGS }, | ||
| 78 | { "nb_classes", "The number of class", OFFSET2(nb_classes), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, INTMAX_MAX, FLAGS }, | ||
| 79 | { "anchors", "anchors, splited by '&'", OFFSET2(anchors_str), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, FLAGS }, | ||
| 64 | { NULL } | 80 | { NULL } |
| 65 | }; | 81 | }; |
| 66 | 82 | ||
| 67 | AVFILTER_DEFINE_CLASS(dnn_detect); | 83 | AVFILTER_DEFINE_CLASS(dnn_detect); |
| 68 | 84 | ||
| 85 | static int dnn_detect_get_label_id(int nb_classes, int cell_size, float *label_data) | ||
| 86 | { | ||
| 87 | float max_prob = 0; | ||
| 88 | int label_id = 0; | ||
| 89 | for (int i = 0; i < nb_classes; i++) { | ||
| 90 | if (label_data[i * cell_size] > max_prob) { | ||
| 91 | max_prob = label_data[i * cell_size]; | ||
| 92 | label_id = i; | ||
| 93 | } | ||
| 94 | } | ||
| 95 | return label_id; | ||
| 96 | } | ||
| 97 | |||
| 98 | static int dnn_detect_parse_anchors(char *anchors_str, float **anchors) | ||
| 99 | { | ||
| 100 | char *saveptr = NULL, *token; | ||
| 101 | float *anchors_buf; | ||
| 102 | int nb_anchor = 0, i = 0; | ||
| 103 | while(anchors_str[i] != '\0') { | ||
| 104 | if(anchors_str[i] == '&') | ||
| 105 | nb_anchor++; | ||
| 106 | i++; | ||
| 107 | } | ||
| 108 | nb_anchor++; | ||
| 109 | anchors_buf = av_mallocz(nb_anchor * sizeof(*anchors)); | ||
| 110 | if (!anchors_buf) { | ||
| 111 | return 0; | ||
| 112 | } | ||
| 113 | for (int i = 0; i < nb_anchor; i++) { | ||
| 114 | token = av_strtok(anchors_str, "&", &saveptr); | ||
| 115 | anchors_buf[i] = strtof(token, NULL); | ||
| 116 | anchors_str = NULL; | ||
| 117 | } | ||
| 118 | *anchors = anchors_buf; | ||
| 119 | return nb_anchor; | ||
| 120 | } | ||
| 121 | |||
| 122 | /* Calculate Intersection Over Union */ | ||
| 123 | static float dnn_detect_IOU(AVDetectionBBox *bbox1, AVDetectionBBox *bbox2) | ||
| 124 | { | ||
| 125 | float overlapping_width = FFMIN(bbox1->x + bbox1->w, bbox2->x + bbox2->w) - FFMAX(bbox1->x, bbox2->x); | ||
| 126 | float overlapping_height = FFMIN(bbox1->y + bbox1->h, bbox2->y + bbox2->h) - FFMAX(bbox1->y, bbox2->y); | ||
| 127 | float intersection_area = | ||
| 128 | (overlapping_width < 0 || overlapping_height < 0) ? 0 : overlapping_height * overlapping_width; | ||
| 129 | float union_area = bbox1->w * bbox1->h + bbox2->w * bbox2->h - intersection_area; | ||
| 130 | return intersection_area / union_area; | ||
| 131 | } | ||
| 132 | |||
| 133 | static int dnn_detect_parse_yolo_output(AVFrame *frame, DNNData *output, int output_index, | ||
| 134 | AVFilterContext *filter_ctx) | ||
| 135 | { | ||
| 136 | DnnDetectContext *ctx = filter_ctx->priv; | ||
| 137 | float conf_threshold = ctx->confidence; | ||
| 138 | int detection_boxes, box_size, cell_w, cell_h, scale_w, scale_h; | ||
| 139 | int nb_classes = ctx->nb_classes; | ||
| 140 | float *output_data = output[output_index].data; | ||
| 141 | float *anchors = ctx->anchors; | ||
| 142 | AVDetectionBBox *bbox; | ||
| 143 | |||
| 144 | if (ctx->model_type == DDMT_YOLOV1V2) { | ||
| 145 | cell_w = ctx->cell_w; | ||
| 146 | cell_h = ctx->cell_h; | ||
| 147 | scale_w = cell_w; | ||
| 148 | scale_h = cell_h; | ||
| 149 | } | ||
| 150 | box_size = nb_classes + 5; | ||
| 151 | |||
| 152 | if (!cell_h || !cell_w) { | ||
| 153 | av_log(filter_ctx, AV_LOG_ERROR, "cell_w and cell_h are detected\n"); | ||
| 154 | return AVERROR(EINVAL); | ||
| 155 | } | ||
| 156 | |||
| 157 | if (!nb_classes) { | ||
| 158 | av_log(filter_ctx, AV_LOG_ERROR, "nb_classes is not set\n"); | ||
| 159 | return AVERROR(EINVAL); | ||
| 160 | } | ||
| 161 | |||
| 162 | if (!anchors) { | ||
| 163 | av_log(filter_ctx, AV_LOG_ERROR, "anchors is not set\n"); | ||
| 164 | return AVERROR(EINVAL); | ||
| 165 | } | ||
| 166 | |||
| 167 | if (output[output_index].channels * output[output_index].width * | ||
| 168 | output[output_index].height % (box_size * cell_w * cell_h)) { | ||
| 169 | av_log(filter_ctx, AV_LOG_ERROR, "wrong cell_w, cell_h or nb_classes\n"); | ||
| 170 | return AVERROR(EINVAL); | ||
| 171 | } | ||
| 172 | detection_boxes = output[output_index].channels * | ||
| 173 | output[output_index].height * | ||
| 174 | output[output_index].width / box_size / cell_w / cell_h; | ||
| 175 | |||
| 176 | /** | ||
| 177 | * find all candidate bbox | ||
| 178 | * yolo output can be reshaped to [B, N*D, Cx, Cy] | ||
| 179 | * Detection box 'D' has format [`x`, `y`, `h`, `w`, `box_score`, `class_no_1`, ...,] | ||
| 180 | **/ | ||
| 181 | for (int box_id = 0; box_id < detection_boxes; box_id++) { | ||
| 182 | for (int cx = 0; cx < cell_w; cx++) | ||
| 183 | for (int cy = 0; cy < cell_h; cy++) { | ||
| 184 | float x, y, w, h, conf; | ||
| 185 | float *detection_boxes_data; | ||
| 186 | int label_id; | ||
| 187 | |||
| 188 | detection_boxes_data = output_data + box_id * box_size * cell_w * cell_h; | ||
| 189 | conf = detection_boxes_data[cy * cell_w + cx + 4 * cell_w * cell_h]; | ||
| 190 | if (conf < conf_threshold) { | ||
| 191 | continue; | ||
| 192 | } | ||
| 193 | |||
| 194 | x = detection_boxes_data[cy * cell_w + cx]; | ||
| 195 | y = detection_boxes_data[cy * cell_w + cx + cell_w * cell_h]; | ||
| 196 | w = detection_boxes_data[cy * cell_w + cx + 2 * cell_w * cell_h]; | ||
| 197 | h = detection_boxes_data[cy * cell_w + cx + 3 * cell_w * cell_h]; | ||
| 198 | label_id = dnn_detect_get_label_id(ctx->nb_classes, cell_w * cell_h, | ||
| 199 | detection_boxes_data + cy * cell_w + cx + 5 * cell_w * cell_h); | ||
| 200 | conf = conf * detection_boxes_data[cy * cell_w + cx + (label_id + 5) * cell_w * cell_h]; | ||
| 201 | |||
| 202 | bbox = av_mallocz(sizeof(*bbox)); | ||
| 203 | if (!bbox) | ||
| 204 | return AVERROR(ENOMEM); | ||
| 205 | |||
| 206 | bbox->w = exp(w) * anchors[box_id * 2] * frame->width / scale_w; | ||
| 207 | bbox->h = exp(h) * anchors[box_id * 2 + 1] * frame->height / scale_h; | ||
| 208 | bbox->x = (cx + x) / cell_w * frame->width - bbox->w / 2; | ||
| 209 | bbox->y = (cy + y) / cell_h * frame->height - bbox->h / 2; | ||
| 210 | bbox->detect_confidence = av_make_q((int)(conf * 10000), 10000); | ||
| 211 | if (ctx->labels && label_id < ctx->label_count) { | ||
| 212 | av_strlcpy(bbox->detect_label, ctx->labels[label_id], sizeof(bbox->detect_label)); | ||
| 213 | } else { | ||
| 214 | snprintf(bbox->detect_label, sizeof(bbox->detect_label), "%d", label_id); | ||
| 215 | } | ||
| 216 | |||
| 217 | if (av_fifo_write(ctx->bboxes_fifo, &bbox, 1) < 0) { | ||
| 218 | av_freep(&bbox); | ||
| 219 | return AVERROR(ENOMEM); | ||
| 220 | } | ||
| 221 | } | ||
| 222 | } | ||
| 223 | return 0; | ||
| 224 | } | ||
| 225 | |||
| 226 | static int dnn_detect_fill_side_data(AVFrame *frame, AVFilterContext *filter_ctx) | ||
| 227 | { | ||
| 228 | DnnDetectContext *ctx = filter_ctx->priv; | ||
| 229 | float conf_threshold = ctx->confidence; | ||
| 230 | AVDetectionBBox *bbox; | ||
| 231 | int nb_bboxes = 0; | ||
| 232 | AVDetectionBBoxHeader *header; | ||
| 233 | if (av_fifo_can_read(ctx->bboxes_fifo) == 0) { | ||
| 234 | av_log(filter_ctx, AV_LOG_VERBOSE, "nothing detected in this frame.\n"); | ||
| 235 | return 0; | ||
| 236 | } | ||
| 237 | |||
| 238 | /* remove overlap bboxes */ | ||
| 239 | for (int i = 0; i < av_fifo_can_read(ctx->bboxes_fifo); i++){ | ||
| 240 | av_fifo_peek(ctx->bboxes_fifo, &bbox, 1, i); | ||
| 241 | for (int j = 0; j < av_fifo_can_read(ctx->bboxes_fifo); j++) { | ||
| 242 | AVDetectionBBox *overlap_bbox; | ||
| 243 | av_fifo_peek(ctx->bboxes_fifo, &overlap_bbox, 1, j); | ||
| 244 | if (!strcmp(bbox->detect_label, overlap_bbox->detect_label) && | ||
| 245 | av_cmp_q(bbox->detect_confidence, overlap_bbox->detect_confidence) < 0 && | ||
| 246 | dnn_detect_IOU(bbox, overlap_bbox) >= conf_threshold) { | ||
| 247 | bbox->classify_count = -1; // bad result | ||
| 248 | nb_bboxes++; | ||
| 249 | break; | ||
| 250 | } | ||
| 251 | } | ||
| 252 | } | ||
| 253 | nb_bboxes = av_fifo_can_read(ctx->bboxes_fifo) - nb_bboxes; | ||
| 254 | header = av_detection_bbox_create_side_data(frame, nb_bboxes); | ||
| 255 | if (!header) { | ||
| 256 | av_log(filter_ctx, AV_LOG_ERROR, "failed to create side data with %d bounding boxes\n", nb_bboxes); | ||
| 257 | return -1; | ||
| 258 | } | ||
| 259 | av_strlcpy(header->source, ctx->dnnctx.model_filename, sizeof(header->source)); | ||
| 260 | |||
| 261 | while(av_fifo_can_read(ctx->bboxes_fifo)) { | ||
| 262 | AVDetectionBBox *candidate_bbox; | ||
| 263 | av_fifo_read(ctx->bboxes_fifo, &candidate_bbox, 1); | ||
| 264 | |||
| 265 | if (nb_bboxes > 0 && candidate_bbox->classify_count != -1) { | ||
| 266 | bbox = av_get_detection_bbox(header, header->nb_bboxes - nb_bboxes); | ||
| 267 | memcpy(bbox, candidate_bbox, sizeof(*bbox)); | ||
| 268 | nb_bboxes--; | ||
| 269 | } | ||
| 270 | av_freep(&candidate_bbox); | ||
| 271 | } | ||
| 272 | return 0; | ||
| 273 | } | ||
| 274 | |||
| 275 | static int dnn_detect_post_proc_yolo(AVFrame *frame, DNNData *output, AVFilterContext *filter_ctx) | ||
| 276 | { | ||
| 277 | int ret = 0; | ||
| 278 | ret = dnn_detect_parse_yolo_output(frame, output, 0, filter_ctx); | ||
| 279 | if (ret < 0) | ||
| 280 | return ret; | ||
| 281 | ret = dnn_detect_fill_side_data(frame, filter_ctx); | ||
| 282 | if (ret < 0) | ||
| 283 | return ret; | ||
| 284 | return 0; | ||
| 285 | } | ||
| 286 | |||
| 69 | static int dnn_detect_post_proc_ssd(AVFrame *frame, DNNData *output, AVFilterContext *filter_ctx) | 287 | static int dnn_detect_post_proc_ssd(AVFrame *frame, DNNData *output, AVFilterContext *filter_ctx) |
| 70 | { | 288 | { |
| 71 | DnnDetectContext *ctx = filter_ctx->priv; | 289 | DnnDetectContext *ctx = filter_ctx->priv; |
| @@ -158,6 +376,10 @@ static int dnn_detect_post_proc_ov(AVFrame *frame, DNNData *output, AVFilterCont · 86 unmodified lines | |||
| 158 | if (ret < 0) | 376 | if (ret < 0) |
| 159 | return ret; | 377 | return ret; |
| 160 | break; | 378 | break; |
| 379 | case DDMT_YOLOV1V2: | ||
| 380 | ret = dnn_detect_post_proc_yolo(frame, output, filter_ctx); | ||
| 381 | if (ret < 0) | ||
| 382 | return ret; | ||
| 161 | } | 383 | } |
| 162 | 384 | ||
| 163 | return 0; | 385 | return 0; |
| @@ -356,11 +578,22 @@ static av_cold int dnn_detect_init(AVFilterContext *context) · 192 unmodified lines | |||
| 356 | ret = check_output_nb(ctx, dnn_ctx->backend_type, dnn_ctx->nb_outputs); | 578 | ret = check_output_nb(ctx, dnn_ctx->backend_type, dnn_ctx->nb_outputs); |
| 357 | if (ret < 0) | 579 | if (ret < 0) |
| 358 | return ret; | 580 | return ret; |
| 581 | ctx->bboxes_fifo = av_fifo_alloc2(1, sizeof(AVDetectionBBox *), AV_FIFO_FLAG_AUTO_GROW); | ||
| 582 | if (!ctx->bboxes_fifo) | ||
| 583 | return AVERROR(ENOMEM); | ||
| 359 | ff_dnn_set_detect_post_proc(&ctx->dnnctx, dnn_detect_post_proc); | 584 | ff_dnn_set_detect_post_proc(&ctx->dnnctx, dnn_detect_post_proc); |
| 360 | 585 | ||
| 361 | if (ctx->labels_filename) { | 586 | if (ctx->labels_filename) { |
| 362 | return read_detect_label_file(context); | 587 | return read_detect_label_file(context); |
| 363 | } | 588 | } |
| 589 | if (ctx->anchors_str) { | ||
| 590 | ret = dnn_detect_parse_anchors(ctx->anchors_str, &ctx->anchors); | ||
| 591 | if (!ctx->anchors) { | ||
| 592 | av_log(context, AV_LOG_ERROR, "failed to parse anchors_str\n"); | ||
| 593 | return AVERROR(EINVAL); | ||
| 594 | } | ||
| 595 | ctx->nb_anchor = ret; | ||
| 596 | } | ||
| 364 | return 0; | 597 | return 0; |
| 365 | } | 598 | } |
| 366 | 599 | ||
| @@ -460,7 +693,14 @@ static int dnn_detect_activate(AVFilterContext *filter_ctx) · 93 unmodified lines | |||
| 460 | static av_cold void dnn_detect_uninit(AVFilterContext *context) | 693 | static av_cold void dnn_detect_uninit(AVFilterContext *context) |
| 461 | { | 694 | { |
| 462 | DnnDetectContext *ctx = context->priv; | 695 | DnnDetectContext *ctx = context->priv; |
| 696 | AVDetectionBBox *bbox; | ||
| 463 | ff_dnn_uninit(&ctx->dnnctx); | 697 | ff_dnn_uninit(&ctx->dnnctx); |
| 698 | while(av_fifo_can_read(ctx->bboxes_fifo)) { | ||
| 699 | av_fifo_read(ctx->bboxes_fifo, &bbox, 1); | ||
| 700 | av_freep(&bbox); | ||
| 701 | } | ||
| 702 | av_fifo_freep2(&ctx->bboxes_fifo); | ||
| 703 | av_freep(&ctx->anchors); | ||
| 464 | free_detect_labels(ctx); | 704 | free_detect_labels(ctx); |
| 465 | } | 705 | } |
| 466 | 706 | ||