| @@ -29,6 +29,7 @@ · 28 unmodified lines |
| 29 | #include "libavutil/avassert.h" | 29 | #include "libavutil/avassert.h" |
| 30 | #include "libavutil/opt.h" | 30 | #include "libavutil/opt.h" |
| 31 | #include "libavutil/avstring.h" | 31 | #include "libavutil/avstring.h" |
| | 32 | #include "libavutil/detection_bbox.h" |
| 32 | #include "../internal.h" | 33 | #include "../internal.h" |
| 33 | #include "queue.h" | 34 | #include "queue.h" |
| 34 | #include "safe_queue.h" | 35 | #include "safe_queue.h" |
| @@ -74,6 +75,7 @@ typedef struct TaskItem { · 39 unmodified lines |
| 74 | // one task might have multiple inferences | 75 | // one task might have multiple inferences |
| 75 | typedef struct InferenceItem { | 76 | typedef struct InferenceItem { |
| 76 | TaskItem *task; | 77 | TaskItem *task; |
| | 78 | uint32_t bbox_index; |
| 77 | } InferenceItem; | 79 | } InferenceItem; |
| 78 | | 80 | |
| 79 | // one request for one call to openvino | 81 | // one request for one call to openvino |
| @@ -182,12 +184,23 @@ static DNNReturnType fill_model_input_ov(OVModel *ov_model, RequestItem *request · 102 unmodified lines |
| 182 | request->inferences[i] = inference; | 184 | request->inferences[i] = inference; |
| 183 | request->inference_count = i + 1; | 185 | request->inference_count = i + 1; |
| 184 | task = inference->task; | 186 | task = inference->task; |
| 185 | if (task->do_ioproc) { | 187 | switch (task->ov_model->model->func_type) { |
| 186 | if (ov_model->model->frame_pre_proc != NULL) { | 188 | case DFT_PROCESS_FRAME: |
| 187 | ov_model->model->frame_pre_proc(task->in_frame, &input, ov_model->model->filter_ctx); | 189 | case DFT_ANALYTICS_DETECT: |
| 188 | } else { | 190 | if (task->do_ioproc) { |
| 189 | ff_proc_from_frame_to_dnn(task->in_frame, &input, ov_model->model->func_type, ctx); | 191 | if (ov_model->model->frame_pre_proc != NULL) { |
| | 192 | ov_model->model->frame_pre_proc(task->in_frame, &input, ov_model->model->filter_ctx); |
| | 193 | } else { |
| | 194 | ff_proc_from_frame_to_dnn(task->in_frame, &input, ov_model->model->func_type, ctx); |
| | 195 | } |
| 190 | } | 196 | } |
| | 197 | break; |
| | 198 | case DFT_ANALYTICS_CLASSIFY: |
| | 199 | ff_frame_to_dnn_classify(task->in_frame, &input, inference->bbox_index, ctx); |
| | 200 | break; |
| | 201 | default: |
| | 202 | av_assert0(!"should not reach here"); |
| | 203 | break; |
| 191 | } | 204 | } |
| 192 | input.data = (uint8_t *)input.data | 205 | input.data = (uint8_t *)input.data |
| 193 | + input.width * input.height * input.channels * get_datatype_size(input.dt); | 206 | + input.width * input.height * input.channels * get_datatype_size(input.dt); |
| @@ -276,6 +289,13 @@ static void infer_completion_callback(void *args) · 82 unmodified lines |
| 276 | } | 289 | } |
| 277 | task->ov_model->model->detect_post_proc(task->out_frame, &output, 1, task->ov_model->model->filter_ctx); | 290 | task->ov_model->model->detect_post_proc(task->out_frame, &output, 1, task->ov_model->model->filter_ctx); |
| 278 | break; | 291 | break; |
| | 292 | case DFT_ANALYTICS_CLASSIFY: |
| | 293 | if (!task->ov_model->model->classify_post_proc) { |
| | 294 | av_log(ctx, AV_LOG_ERROR, "classify filter needs to provide post proc\n"); |
| | 295 | return; |
| | 296 | } |
| | 297 | task->ov_model->model->classify_post_proc(task->out_frame, &output, request->inferences[i]->bbox_index, task->ov_model->model->filter_ctx); |
| | 298 | break; |
| 279 | default: | 299 | default: |
| 280 | av_assert0(!"should not reach here"); | 300 | av_assert0(!"should not reach here"); |
| 281 | break; | 301 | break; |
| @@ -513,7 +533,44 @@ static DNNReturnType get_input_ov(void *model, DNNData *input, const char *input · 231 unmodified lines |
| 513 | return DNN_ERROR; | 533 | return DNN_ERROR; |
| 514 | } | 534 | } |
| 515 | | 535 | |
| 516 | static DNNReturnType extract_inference_from_task(DNNFunctionType func_type, TaskItem *task, Queue *inference_queue) | 536 | static int contain_valid_detection_bbox(AVFrame *frame) |
| | 537 | { |
| | 538 | AVFrameSideData *sd; |
| | 539 | const AVDetectionBBoxHeader *header; |
| | 540 | const AVDetectionBBox *bbox; |
| | 541 | |
| | 542 | sd = av_frame_get_side_data(frame, AV_FRAME_DATA_DETECTION_BBOXES); |
| | 543 | if (!sd) { // this frame has nothing detected |
| | 544 | return 0; |
| | 545 | } |
| | 546 | |
| | 547 | if (!sd->size) { |
| | 548 | return 0; |
| | 549 | } |
| | 550 | |
| | 551 | header = (const AVDetectionBBoxHeader *)sd->data; |
| | 552 | if (!header->nb_bboxes) { |
| | 553 | return 0; |
| | 554 | } |
| | 555 | |
| | 556 | for (uint32_t i = 0; i < header->nb_bboxes; i++) { |
| | 557 | bbox = av_get_detection_bbox(header, i); |
| | 558 | if (bbox->x < 0 || bbox->w < 0 || bbox->x + bbox->w >= frame->width) { |
| | 559 | return 0; |
| | 560 | } |
| | 561 | if (bbox->y < 0 || bbox->h < 0 || bbox->y + bbox->h >= frame->width) { |
| | 562 | return 0; |
| | 563 | } |
| | 564 | |
| | 565 | if (bbox->classify_count == AV_NUM_DETECTION_BBOX_CLASSIFY) { |
| | 566 | return 0; |
| | 567 | } |
| | 568 | } |
| | 569 | |
| | 570 | return 1; |
| | 571 | } |
| | 572 | |
| | 573 | static DNNReturnType extract_inference_from_task(DNNFunctionType func_type, TaskItem *task, Queue *inference_queue, DNNExecBaseParams *exec_params) |
| 517 | { | 574 | { |
| 518 | switch (func_type) { | 575 | switch (func_type) { |
| 519 | case DFT_PROCESS_FRAME: | 576 | case DFT_PROCESS_FRAME: |
| @@ -532,6 +589,45 @@ static DNNReturnType extract_inference_from_task(DNNFunctionType func_type, Task · 12 unmodified lines |
| 532 | } | 589 | } |
| 533 | return DNN_SUCCESS; | 590 | return DNN_SUCCESS; |
| 534 | } | 591 | } |
| | 592 | case DFT_ANALYTICS_CLASSIFY: |
| | 593 | { |
| | 594 | const AVDetectionBBoxHeader *header; |
| | 595 | AVFrame *frame = task->in_frame; |
| | 596 | AVFrameSideData *sd; |
| | 597 | DNNExecClassificationParams *params = (DNNExecClassificationParams *)exec_params; |
| | 598 | |
| | 599 | task->inference_todo = 0; |
| | 600 | task->inference_done = 0; |
| | 601 | |
| | 602 | if (!contain_valid_detection_bbox(frame)) { |
| | 603 | return DNN_SUCCESS; |
| | 604 | } |
| | 605 | |
| | 606 | sd = av_frame_get_side_data(frame, AV_FRAME_DATA_DETECTION_BBOXES); |
| | 607 | header = (const AVDetectionBBoxHeader *)sd->data; |
| | 608 | |
| | 609 | for (uint32_t i = 0; i < header->nb_bboxes; i++) { |
| | 610 | InferenceItem *inference; |
| | 611 | const AVDetectionBBox *bbox = av_get_detection_bbox(header, i); |
| | 612 | |
| | 613 | if (av_strncasecmp(bbox->detect_label, params->target, sizeof(bbox->detect_label)) != 0) { |
| | 614 | continue; |
| | 615 | } |
| | 616 | |
| | 617 | inference = av_malloc(sizeof(*inference)); |
| | 618 | if (!inference) { |
| | 619 | return DNN_ERROR; |
| | 620 | } |
| | 621 | task->inference_todo++; |
| | 622 | inference->task = task; |
| | 623 | inference->bbox_index = i; |
| | 624 | if (ff_queue_push_back(inference_queue, inference) < 0) { |
| | 625 | av_freep(&inference); |
| | 626 | return DNN_ERROR; |
| | 627 | } |
| | 628 | } |
| | 629 | return DNN_SUCCESS; |
| | 630 | } |
| 535 | default: | 631 | default: |
| 536 | av_assert0(!"should not reach here"); | 632 | av_assert0(!"should not reach here"); |
| 537 | return DNN_ERROR; | 633 | return DNN_ERROR; |
| @@ -598,7 +694,7 @@ static DNNReturnType get_output_ov(void *model, const char *input_name, int inpu · 60 unmodified lines |
| 598 | task.out_frame = out_frame; | 694 | task.out_frame = out_frame; |
| 599 | task.ov_model = ov_model; | 695 | task.ov_model = ov_model; |
| 600 | | 696 | |
| 601 | if (extract_inference_from_task(ov_model->model->func_type, &task, ov_model->inference_queue) != DNN_SUCCESS) { | 697 | if (extract_inference_from_task(ov_model->model->func_type, &task, ov_model->inference_queue, NULL) != DNN_SUCCESS) { |
| 602 | av_frame_free(&out_frame); | 698 | av_frame_free(&out_frame); |
| 603 | av_frame_free(&in_frame); | 699 | av_frame_free(&in_frame); |
| 604 | av_log(ctx, AV_LOG_ERROR, "unable to extract inference from task.\n"); | 700 | av_log(ctx, AV_LOG_ERROR, "unable to extract inference from task.\n"); |
| @@ -690,6 +786,14 @@ DNNReturnType ff_dnn_execute_model_ov(const DNNModel *model, DNNExecBaseParams * · 85 unmodified lines |
| 690 | return DNN_ERROR; | 786 | return DNN_ERROR; |
| 691 | } | 787 | } |
| 692 | | 788 | |
| | 789 | if (model->func_type == DFT_ANALYTICS_CLASSIFY) { |
| | 790 | // Once we add async support for tensorflow backend and native backend, |
| | 791 | // we'll combine the two sync/async functions in dnn_interface.h to |
| | 792 | // simplify the code in filter, and async will be an option within backends. |
| | 793 | // so, do not support now, and classify filter will not call this function. |
| | 794 | return DNN_ERROR; |
| | 795 | } |
| | 796 | |
| 693 | if (ctx->options.batch_size > 1) { | 797 | if (ctx->options.batch_size > 1) { |
| 694 | avpriv_report_missing_feature(ctx, "batch mode for sync execution"); | 798 | avpriv_report_missing_feature(ctx, "batch mode for sync execution"); |
| 695 | return DNN_ERROR; | 799 | return DNN_ERROR; |
| @@ -710,7 +814,7 @@ DNNReturnType ff_dnn_execute_model_ov(const DNNModel *model, DNNExecBaseParams * · 14 unmodified lines |
| 710 | task.out_frame = exec_params->out_frame ? exec_params->out_frame : exec_params->in_frame; | 814 | task.out_frame = exec_params->out_frame ? exec_params->out_frame : exec_params->in_frame; |
| 711 | task.ov_model = ov_model; | 815 | task.ov_model = ov_model; |
| 712 | | 816 | |
| 713 | if (extract_inference_from_task(ov_model->model->func_type, &task, ov_model->inference_queue) != DNN_SUCCESS) { | 817 | if (extract_inference_from_task(ov_model->model->func_type, &task, ov_model->inference_queue, exec_params) != DNN_SUCCESS) { |
| 714 | av_log(ctx, AV_LOG_ERROR, "unable to extract inference from task.\n"); | 818 | av_log(ctx, AV_LOG_ERROR, "unable to extract inference from task.\n"); |
| 715 | return DNN_ERROR; | 819 | return DNN_ERROR; |
| 716 | } | 820 | } |
| @@ -730,6 +834,7 @@ DNNReturnType ff_dnn_execute_model_async_ov(const DNNModel *model, DNNExecBasePa · 13 unmodified lines |
| 730 | OVContext *ctx = &ov_model->ctx; | 834 | OVContext *ctx = &ov_model->ctx; |
| 731 | RequestItem *request; | 835 | RequestItem *request; |
| 732 | TaskItem *task; | 836 | TaskItem *task; |
| | 837 | DNNReturnType ret; |
| 733 | | 838 | |
| 734 | if (ff_check_exec_params(ctx, DNN_OV, model->func_type, exec_params) != 0) { | 839 | if (ff_check_exec_params(ctx, DNN_OV, model->func_type, exec_params) != 0) { |
| 735 | return DNN_ERROR; | 840 | return DNN_ERROR; |
| @@ -761,23 +866,25 @@ DNNReturnType ff_dnn_execute_model_async_ov(const DNNModel *model, DNNExecBasePa · 25 unmodified lines |
| 761 | return DNN_ERROR; | 866 | return DNN_ERROR; |
| 762 | } | 867 | } |
| 763 | | 868 | |
| 764 | if (extract_inference_from_task(ov_model->model->func_type, task, ov_model->inference_queue) != DNN_SUCCESS) { | 869 | if (extract_inference_from_task(model->func_type, task, ov_model->inference_queue, exec_params) != DNN_SUCCESS) { |
| 765 | av_log(ctx, AV_LOG_ERROR, "unable to extract inference from task.\n"); | 870 | av_log(ctx, AV_LOG_ERROR, "unable to extract inference from task.\n"); |
| 766 | return DNN_ERROR; | 871 | return DNN_ERROR; |
| 767 | } | 872 | } |
| 768 | | 873 | |
| 769 | if (ff_queue_size(ov_model->inference_queue) < ctx->options.batch_size) { | 874 | while (ff_queue_size(ov_model->inference_queue) >= ctx->options.batch_size) { |
| 770 | // not enough inference items queued for a batch | 875 | request = ff_safe_queue_pop_front(ov_model->request_queue); |
| 771 | return DNN_SUCCESS; | 876 | if (!request) { |
| 772 | } | 877 | av_log(ctx, AV_LOG_ERROR, "unable to get infer request.\n"); |
| | 878 | return DNN_ERROR; |
| | 879 | } |
| 773 | | 880 | |
| 774 | request = ff_safe_queue_pop_front(ov_model->request_queue); | 881 | ret = execute_model_ov(request, ov_model->inference_queue); |
| 775 | if (!request) { | 882 | if (ret != DNN_SUCCESS) { |
| 776 | av_log(ctx, AV_LOG_ERROR, "unable to get infer request.\n"); | 883 | return ret; |
| 777 | return DNN_ERROR; | 884 | } |
| 778 | } | 885 | } |
| 779 | | 886 | |
| 780 | return execute_model_ov(request, ov_model->inference_queue); | 887 | return DNN_SUCCESS; |
| 781 | } | 888 | } |
| 782 | | 889 | |
| 783 | DNNAsyncStatusType ff_dnn_get_async_result_ov(const DNNModel *model, AVFrame **in, AVFrame **out) | 890 | DNNAsyncStatusType ff_dnn_get_async_result_ov(const DNNModel *model, AVFrame **in, AVFrame **out) |