| @@ -29,6 +29,7 @@ · 28 unmodified lines |
| 29 | 29 | #include "libavutil/avassert.h" |
| 30 | 30 | #include "libavutil/opt.h" |
| 31 | 31 | #include "libavutil/avstring.h" |
| 32 | +#include "libavutil/detection_bbox.h" |
| 32 | 33 | #include "../internal.h" |
| 33 | 34 | #include "queue.h" |
| 34 | 35 | #include "safe_queue.h" |
| @@ -74,6 +75,7 @@ typedef struct TaskItem { · 39 unmodified lines |
| 74 | 75 | // one task might have multiple inferences |
| 75 | 76 | typedef struct InferenceItem { |
| 76 | 77 | TaskItem *task; |
| 78 | + uint32_t bbox_index; |
| 77 | 79 | } InferenceItem; |
| 78 | 80 | |
| 79 | 81 | // one request for one call to openvino |
| @@ -182,12 +184,23 @@ static DNNReturnType fill_model_input_ov(OVModel *ov_model, RequestItem *request · 102 unmodified lines |
| 182 | 184 | request->inferences[i] = inference; |
| 183 | 185 | request->inference_count = i + 1; |
| 184 | 186 | task = inference->task; |
| 185 | | − if (task->do_ioproc) { |
| 186 | | − if (ov_model->model->frame_pre_proc != NULL) { |
| 187 | | − ov_model->model->frame_pre_proc(task->in_frame, &input, ov_model->model->filter_ctx); |
| 188 | | − } else { |
| 189 | | − ff_proc_from_frame_to_dnn(task->in_frame, &input, ov_model->model->func_type, ctx); |
| 187 | + switch (task->ov_model->model->func_type) { |
| 188 | + case DFT_PROCESS_FRAME: |
| 189 | + case DFT_ANALYTICS_DETECT: |
| 190 | + if (task->do_ioproc) { |
| 191 | + if (ov_model->model->frame_pre_proc != NULL) { |
| 192 | + ov_model->model->frame_pre_proc(task->in_frame, &input, ov_model->model->filter_ctx); |
| 193 | + } else { |
| 194 | + ff_proc_from_frame_to_dnn(task->in_frame, &input, ov_model->model->func_type, ctx); |
| 195 | + } |
| 190 | 196 | } |
| 197 | + break; |
| 198 | + case DFT_ANALYTICS_CLASSIFY: |
| 199 | + ff_frame_to_dnn_classify(task->in_frame, &input, inference->bbox_index, ctx); |
| 200 | + break; |
| 201 | + default: |
| 202 | + av_assert0(!"should not reach here"); |
| 203 | + break; |
| 191 | 204 | } |
| 192 | 205 | input.data = (uint8_t *)input.data |
| 193 | 206 | + input.width * input.height * input.channels * get_datatype_size(input.dt); |
| @@ -276,6 +289,13 @@ static void infer_completion_callback(void *args) · 82 unmodified lines |
| 276 | 289 | } |
| 277 | 290 | task->ov_model->model->detect_post_proc(task->out_frame, &output, 1, task->ov_model->model->filter_ctx); |
| 278 | 291 | break; |
| 292 | + case DFT_ANALYTICS_CLASSIFY: |
| 293 | + if (!task->ov_model->model->classify_post_proc) { |
| 294 | + av_log(ctx, AV_LOG_ERROR, "classify filter needs to provide post proc\n"); |
| 295 | + return; |
| 296 | + } |
| 297 | + task->ov_model->model->classify_post_proc(task->out_frame, &output, request->inferences[i]->bbox_index, task->ov_model->model->filter_ctx); |
| 298 | + break; |
| 279 | 299 | default: |
| 280 | 300 | av_assert0(!"should not reach here"); |
| 281 | 301 | break; |
| @@ -513,7 +533,44 @@ static DNNReturnType get_input_ov(void *model, DNNData *input, const char *input · 231 unmodified lines |
| 513 | 533 | return DNN_ERROR; |
| 514 | 534 | } |
| 515 | 535 | |
| 516 | | −static DNNReturnType extract_inference_from_task(DNNFunctionType func_type, TaskItem *task, Queue *inference_queue) |
| 536 | +static int contain_valid_detection_bbox(AVFrame *frame) |
| 537 | +{ |
| 538 | + AVFrameSideData *sd; |
| 539 | + const AVDetectionBBoxHeader *header; |
| 540 | + const AVDetectionBBox *bbox; |
| 541 | + |
| 542 | + sd = av_frame_get_side_data(frame, AV_FRAME_DATA_DETECTION_BBOXES); |
| 543 | + if (!sd) { // this frame has nothing detected |
| 544 | + return 0; |
| 545 | + } |
| 546 | + |
| 547 | + if (!sd->size) { |
| 548 | + return 0; |
| 549 | + } |
| 550 | + |
| 551 | + header = (const AVDetectionBBoxHeader *)sd->data; |
| 552 | + if (!header->nb_bboxes) { |
| 553 | + return 0; |
| 554 | + } |
| 555 | + |
| 556 | + for (uint32_t i = 0; i < header->nb_bboxes; i++) { |
| 557 | + bbox = av_get_detection_bbox(header, i); |
| 558 | + if (bbox->x < 0 || bbox->w < 0 || bbox->x + bbox->w >= frame->width) { |
| 559 | + return 0; |
| 560 | + } |
| 561 | + if (bbox->y < 0 || bbox->h < 0 || bbox->y + bbox->h >= frame->width) { |
| 562 | + return 0; |
| 563 | + } |
| 564 | + |
| 565 | + if (bbox->classify_count == AV_NUM_DETECTION_BBOX_CLASSIFY) { |
| 566 | + return 0; |
| 567 | + } |
| 568 | + } |
| 569 | + |
| 570 | + return 1; |
| 571 | +} |
| 572 | + |
| 573 | +static DNNReturnType extract_inference_from_task(DNNFunctionType func_type, TaskItem *task, Queue *inference_queue, DNNExecBaseParams *exec_params) |
| 517 | 574 | { |
| 518 | 575 | switch (func_type) { |
| 519 | 576 | case DFT_PROCESS_FRAME: |
| @@ -532,6 +589,45 @@ static DNNReturnType extract_inference_from_task(DNNFunctionType func_type, Task · 12 unmodified lines |
| 532 | 589 | } |
| 533 | 590 | return DNN_SUCCESS; |
| 534 | 591 | } |
| 592 | + case DFT_ANALYTICS_CLASSIFY: |
| 593 | + { |
| 594 | + const AVDetectionBBoxHeader *header; |
| 595 | + AVFrame *frame = task->in_frame; |
| 596 | + AVFrameSideData *sd; |
| 597 | + DNNExecClassificationParams *params = (DNNExecClassificationParams *)exec_params; |
| 598 | + |
| 599 | + task->inference_todo = 0; |
| 600 | + task->inference_done = 0; |
| 601 | + |
| 602 | + if (!contain_valid_detection_bbox(frame)) { |
| 603 | + return DNN_SUCCESS; |
| 604 | + } |
| 605 | + |
| 606 | + sd = av_frame_get_side_data(frame, AV_FRAME_DATA_DETECTION_BBOXES); |
| 607 | + header = (const AVDetectionBBoxHeader *)sd->data; |
| 608 | + |
| 609 | + for (uint32_t i = 0; i < header->nb_bboxes; i++) { |
| 610 | + InferenceItem *inference; |
| 611 | + const AVDetectionBBox *bbox = av_get_detection_bbox(header, i); |
| 612 | + |
| 613 | + if (av_strncasecmp(bbox->detect_label, params->target, sizeof(bbox->detect_label)) != 0) { |
| 614 | + continue; |
| 615 | + } |
| 616 | + |
| 617 | + inference = av_malloc(sizeof(*inference)); |
| 618 | + if (!inference) { |
| 619 | + return DNN_ERROR; |
| 620 | + } |
| 621 | + task->inference_todo++; |
| 622 | + inference->task = task; |
| 623 | + inference->bbox_index = i; |
| 624 | + if (ff_queue_push_back(inference_queue, inference) < 0) { |
| 625 | + av_freep(&inference); |
| 626 | + return DNN_ERROR; |
| 627 | + } |
| 628 | + } |
| 629 | + return DNN_SUCCESS; |
| 630 | + } |
| 535 | 631 | default: |
| 536 | 632 | av_assert0(!"should not reach here"); |
| 537 | 633 | return DNN_ERROR; |
| @@ -598,7 +694,7 @@ static DNNReturnType get_output_ov(void *model, const char *input_name, int inpu · 60 unmodified lines |
| 598 | 694 | task.out_frame = out_frame; |
| 599 | 695 | task.ov_model = ov_model; |
| 600 | 696 | |
| 601 | | − if (extract_inference_from_task(ov_model->model->func_type, &task, ov_model->inference_queue) != DNN_SUCCESS) { |
| 697 | + if (extract_inference_from_task(ov_model->model->func_type, &task, ov_model->inference_queue, NULL) != DNN_SUCCESS) { |
| 602 | 698 | av_frame_free(&out_frame); |
| 603 | 699 | av_frame_free(&in_frame); |
| 604 | 700 | av_log(ctx, AV_LOG_ERROR, "unable to extract inference from task.\n"); |
| @@ -690,6 +786,14 @@ DNNReturnType ff_dnn_execute_model_ov(const DNNModel *model, DNNExecBaseParams * · 85 unmodified lines |
| 690 | 786 | return DNN_ERROR; |
| 691 | 787 | } |
| 692 | 788 | |
| 789 | + if (model->func_type == DFT_ANALYTICS_CLASSIFY) { |
| 790 | + // Once we add async support for tensorflow backend and native backend, |
| 791 | + // we'll combine the two sync/async functions in dnn_interface.h to |
| 792 | + // simplify the code in filter, and async will be an option within backends. |
| 793 | + // so, do not support now, and classify filter will not call this function. |
| 794 | + return DNN_ERROR; |
| 795 | + } |
| 796 | + |
| 693 | 797 | if (ctx->options.batch_size > 1) { |
| 694 | 798 | avpriv_report_missing_feature(ctx, "batch mode for sync execution"); |
| 695 | 799 | return DNN_ERROR; |
| @@ -710,7 +814,7 @@ DNNReturnType ff_dnn_execute_model_ov(const DNNModel *model, DNNExecBaseParams * · 14 unmodified lines |
| 710 | 814 | task.out_frame = exec_params->out_frame ? exec_params->out_frame : exec_params->in_frame; |
| 711 | 815 | task.ov_model = ov_model; |
| 712 | 816 | |
| 713 | | − if (extract_inference_from_task(ov_model->model->func_type, &task, ov_model->inference_queue) != DNN_SUCCESS) { |
| 817 | + if (extract_inference_from_task(ov_model->model->func_type, &task, ov_model->inference_queue, exec_params) != DNN_SUCCESS) { |
| 714 | 818 | av_log(ctx, AV_LOG_ERROR, "unable to extract inference from task.\n"); |
| 715 | 819 | return DNN_ERROR; |
| 716 | 820 | } |
| @@ -730,6 +834,7 @@ DNNReturnType ff_dnn_execute_model_async_ov(const DNNModel *model, DNNExecBasePa · 13 unmodified lines |
| 730 | 834 | OVContext *ctx = &ov_model->ctx; |
| 731 | 835 | RequestItem *request; |
| 732 | 836 | TaskItem *task; |
| 837 | + DNNReturnType ret; |
| 733 | 838 | |
| 734 | 839 | if (ff_check_exec_params(ctx, DNN_OV, model->func_type, exec_params) != 0) { |
| 735 | 840 | return DNN_ERROR; |
| @@ -761,23 +866,25 @@ DNNReturnType ff_dnn_execute_model_async_ov(const DNNModel *model, DNNExecBasePa · 25 unmodified lines |
| 761 | 866 | return DNN_ERROR; |
| 762 | 867 | } |
| 763 | 868 | |
| 764 | | − if (extract_inference_from_task(ov_model->model->func_type, task, ov_model->inference_queue) != DNN_SUCCESS) { |
| 869 | + if (extract_inference_from_task(model->func_type, task, ov_model->inference_queue, exec_params) != DNN_SUCCESS) { |
| 765 | 870 | av_log(ctx, AV_LOG_ERROR, "unable to extract inference from task.\n"); |
| 766 | 871 | return DNN_ERROR; |
| 767 | 872 | } |
| 768 | 873 | |
| 769 | | − if (ff_queue_size(ov_model->inference_queue) < ctx->options.batch_size) { |
| 770 | | − // not enough inference items queued for a batch |
| 771 | | − return DNN_SUCCESS; |
| 772 | | − } |
| 874 | + while (ff_queue_size(ov_model->inference_queue) >= ctx->options.batch_size) { |
| 875 | + request = ff_safe_queue_pop_front(ov_model->request_queue); |
| 876 | + if (!request) { |
| 877 | + av_log(ctx, AV_LOG_ERROR, "unable to get infer request.\n"); |
| 878 | + return DNN_ERROR; |
| 879 | + } |
| 773 | 880 | |
| 774 | | − request = ff_safe_queue_pop_front(ov_model->request_queue); |
| 775 | | − if (!request) { |
| 776 | | − av_log(ctx, AV_LOG_ERROR, "unable to get infer request.\n"); |
| 777 | | − return DNN_ERROR; |
| 881 | + ret = execute_model_ov(request, ov_model->inference_queue); |
| 882 | + if (ret != DNN_SUCCESS) { |
| 883 | + return ret; |
| 884 | + } |
| 778 | 885 | } |
| 779 | 886 | |
| 780 | | − return execute_model_ov(request, ov_model->inference_queue); |
| 887 | + return DNN_SUCCESS; |
| 781 | 888 | } |
| 782 | 889 | |
| 783 | 890 | DNNAsyncStatusType ff_dnn_get_async_result_ov(const DNNModel *model, AVFrame **in, AVFrame **out) |