| @@ -37,6 +37,7 @@ · 36 unmodified lines |
| 37 | 37 | typedef struct OVOptions{ |
| 38 | 38 | char *device_type; |
| 39 | 39 | int nireq; |
| 40 | + int batch_size; |
| 40 | 41 | } OVOptions; |
| 41 | 42 | |
| 42 | 43 | typedef struct OVContext { |
| @@ -70,7 +71,8 @@ typedef struct TaskItem { · 27 unmodified lines |
| 70 | 71 | |
| 71 | 72 | typedef struct RequestItem { |
| 72 | 73 | ie_infer_request_t *infer_request; |
| 73 | | − TaskItem *task; |
| 74 | + TaskItem **tasks; |
| 75 | + int task_count; |
| 74 | 76 | ie_complete_call_back_t callback; |
| 75 | 77 | } RequestItem; |
| 76 | 78 | |
| @@ -83,6 +85,7 @@ typedef struct RequestItem { · 6 unmodified lines |
| 83 | 85 | static const AVOption dnn_openvino_options[] = { |
| 84 | 86 | { "device", "device to run model", OFFSET(options.device_type), AV_OPT_TYPE_STRING, { .str = "CPU" }, 0, 0, FLAGS }, |
| 85 | 87 | { "nireq", "number of request", OFFSET(options.nireq), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, INT_MAX, FLAGS }, |
| 88 | + { "batch_size", "batch size per request", OFFSET(options.batch_size), AV_OPT_TYPE_INT, { .i64 = 1 }, 1, 1000, FLAGS}, |
| 86 | 89 | { NULL } |
| 87 | 90 | }; |
| 88 | 91 | |
| @@ -100,7 +103,19 @@ static DNNDataType precision_to_datatype(precision_e precision) · 11 unmodified lines |
| 100 | 103 | } |
| 101 | 104 | } |
| 102 | 105 | |
| 103 | | −static DNNReturnType fill_model_input_ov(OVModel *ov_model, TaskItem *task, RequestItem *request) |
| 106 | +static int get_datatype_size(DNNDataType dt) |
| 107 | +{ |
| 108 | + switch (dt) |
| 109 | + { |
| 110 | + case DNN_FLOAT: |
| 111 | + return sizeof(float); |
| 112 | + default: |
| 113 | + av_assert0(!"not supported yet."); |
| 114 | + return 1; |
| 115 | + } |
| 116 | +} |
| 117 | + |
| 118 | +static DNNReturnType fill_model_input_ov(OVModel *ov_model, RequestItem *request) |
| 104 | 119 | { |
| 105 | 120 | dimensions_t dims; |
| 106 | 121 | precision_e precision; |
| @@ -109,6 +124,7 @@ static DNNReturnType fill_model_input_ov(OVModel *ov_model, TaskItem *task, Requ · 2 unmodified lines |
| 109 | 124 | IEStatusCode status; |
| 110 | 125 | DNNData input; |
| 111 | 126 | ie_blob_t *input_blob = NULL; |
| 127 | + TaskItem *task = request->tasks[0]; |
| 112 | 128 | |
| 113 | 129 | status = ie_infer_request_get_blob(request->infer_request, task->input_name, &input_blob); |
| 114 | 130 | if (status != OK) { |
| @@ -134,12 +150,19 @@ static DNNReturnType fill_model_input_ov(OVModel *ov_model, TaskItem *task, Requ · 19 unmodified lines |
| 134 | 150 | input.channels = dims.dims[1]; |
| 135 | 151 | input.data = blob_buffer.buffer; |
| 136 | 152 | input.dt = precision_to_datatype(precision); |
| 137 | | − if (task->do_ioproc) { |
| 138 | | − if (ov_model->model->pre_proc != NULL) { |
| 139 | | − ov_model->model->pre_proc(task->in_frame, &input, ov_model->model->filter_ctx); |
| 140 | | − } else { |
| 141 | | − proc_from_frame_to_dnn(task->in_frame, &input, ctx); |
| 153 | + |
| 154 | + av_assert0(request->task_count <= dims.dims[0]); |
| 155 | + for (int i = 0; i < request->task_count; ++i) { |
| 156 | + task = request->tasks[i]; |
| 157 | + if (task->do_ioproc) { |
| 158 | + if (ov_model->model->pre_proc != NULL) { |
| 159 | + ov_model->model->pre_proc(task->in_frame, &input, ov_model->model->filter_ctx); |
| 160 | + } else { |
| 161 | + proc_from_frame_to_dnn(task->in_frame, &input, ctx); |
| 162 | + } |
| 142 | 163 | } |
| 164 | + input.data = (uint8_t *)input.data |
| 165 | + + input.width * input.height * input.channels * get_datatype_size(input.dt); |
| 143 | 166 | } |
| 144 | 167 | ie_blob_free(&input_blob); |
| 145 | 168 | |
| @@ -152,7 +175,7 @@ static void infer_completion_callback(void *args) · 6 unmodified lines |
| 152 | 175 | precision_e precision; |
| 153 | 176 | IEStatusCode status; |
| 154 | 177 | RequestItem *request = args; |
| 155 | | − TaskItem *task = request->task; |
| 178 | + TaskItem *task = request->tasks[0]; |
| 156 | 179 | ie_blob_t *output_blob = NULL; |
| 157 | 180 | ie_blob_buffer_t blob_buffer; |
| 158 | 181 | DNNData output; |
| @@ -194,41 +217,56 @@ static void infer_completion_callback(void *args) · 35 unmodified lines |
| 194 | 217 | output.width = dims.dims[3]; |
| 195 | 218 | output.dt = precision_to_datatype(precision); |
| 196 | 219 | output.data = blob_buffer.buffer; |
| 197 | | − if (task->do_ioproc) { |
| 198 | | − if (task->ov_model->model->post_proc != NULL) { |
| 199 | | − task->ov_model->model->post_proc(task->out_frame, &output, task->ov_model->model->filter_ctx); |
| 220 | + |
| 221 | + av_assert0(request->task_count <= dims.dims[0]); |
| 222 | + av_assert0(request->task_count >= 1); |
| 223 | + for (int i = 0; i < request->task_count; ++i) { |
| 224 | + task = request->tasks[i]; |
| 225 | + if (task->do_ioproc) { |
| 226 | + if (task->ov_model->model->post_proc != NULL) { |
| 227 | + task->ov_model->model->post_proc(task->out_frame, &output, task->ov_model->model->filter_ctx); |
| 228 | + } else { |
| 229 | + proc_from_dnn_to_frame(task->out_frame, &output, ctx); |
| 230 | + } |
| 200 | 231 | } else { |
| 201 | | − proc_from_dnn_to_frame(task->out_frame, &output, ctx); |
| 232 | + task->out_frame->width = output.width; |
| 233 | + task->out_frame->height = output.height; |
| 202 | 234 | } |
| 203 | | − } else { |
| 204 | | − task->out_frame->width = output.width; |
| 205 | | − task->out_frame->height = output.height; |
| 235 | + task->done = 1; |
| 236 | + output.data = (uint8_t *)output.data |
| 237 | + + output.width * output.height * output.channels * get_datatype_size(output.dt); |
| 206 | 238 | } |
| 207 | 239 | ie_blob_free(&output_blob); |
| 208 | 240 | |
| 241 | + request->task_count = 0; |
| 242 | + |
| 209 | 243 | if (task->async) { |
| 210 | | − request->task = NULL; |
| 211 | 244 | if (ff_safe_queue_push_back(task->ov_model->request_queue, request) < 0) { |
| 212 | 245 | av_log(ctx, AV_LOG_ERROR, "Failed to push back request_queue.\n"); |
| 213 | 246 | return; |
| 214 | 247 | } |
| 215 | 248 | } |
| 216 | | − |
| 217 | | − task->done = 1; |
| 218 | 249 | } |
| 219 | 250 | |
| 220 | | −static DNNReturnType execute_model_ov(TaskItem *task, RequestItem *request) |
| 251 | +static DNNReturnType execute_model_ov(RequestItem *request) |
| 221 | 252 | { |
| 222 | 253 | IEStatusCode status; |
| 254 | + DNNReturnType ret; |
| 255 | + TaskItem *task = request->tasks[0]; |
| 223 | 256 | OVContext *ctx = &task->ov_model->ctx; |
| 224 | 257 | |
| 225 | | − DNNReturnType ret = fill_model_input_ov(task->ov_model, task, request); |
| 226 | | − if (ret != DNN_SUCCESS) { |
| 227 | | − return ret; |
| 228 | | − } |
| 229 | | − |
| 230 | 258 | if (task->async) { |
| 231 | | − request->task = task; |
| 259 | + if (request->task_count < ctx->options.batch_size) { |
| 260 | + if (ff_safe_queue_push_front(task->ov_model->request_queue, request) < 0) { |
| 261 | + av_log(ctx, AV_LOG_ERROR, "Failed to push back request_queue.\n"); |
| 262 | + return DNN_ERROR; |
| 263 | + } |
| 264 | + return DNN_SUCCESS; |
| 265 | + } |
| 266 | + ret = fill_model_input_ov(task->ov_model, request); |
| 267 | + if (ret != DNN_SUCCESS) { |
| 268 | + return ret; |
| 269 | + } |
| 232 | 270 | status = ie_infer_set_completion_callback(request->infer_request, &request->callback); |
| 233 | 271 | if (status != OK) { |
| 234 | 272 | av_log(ctx, AV_LOG_ERROR, "Failed to set completion callback for inference\n"); |
| @@ -241,12 +279,15 @@ static DNNReturnType execute_model_ov(TaskItem *task, RequestItem *request) · 6 unmodified lines |
| 241 | 279 | } |
| 242 | 280 | return DNN_SUCCESS; |
| 243 | 281 | } else { |
| 282 | + ret = fill_model_input_ov(task->ov_model, request); |
| 283 | + if (ret != DNN_SUCCESS) { |
| 284 | + return ret; |
| 285 | + } |
| 244 | 286 | status = ie_infer_request_infer(request->infer_request); |
| 245 | 287 | if (status != OK) { |
| 246 | 288 | av_log(ctx, AV_LOG_ERROR, "Failed to start synchronous model inference\n"); |
| 247 | 289 | return DNN_ERROR; |
| 248 | 290 | } |
| 249 | | − request->task = task; |
| 250 | 291 | infer_completion_callback(request); |
| 251 | 292 | return task->done ? DNN_SUCCESS : DNN_ERROR; |
| 252 | 293 | } |
| @@ -319,6 +360,7 @@ static DNNReturnType get_output_ov(void *model, const char *input_name, int inpu · 66 unmodified lines |
| 319 | 360 | RequestItem request; |
| 320 | 361 | AVFrame *in_frame = av_frame_alloc(); |
| 321 | 362 | AVFrame *out_frame = NULL; |
| 363 | + TaskItem *ptask = &task; |
| 322 | 364 | |
| 323 | 365 | if (!in_frame) { |
| 324 | 366 | av_log(ctx, AV_LOG_ERROR, "Failed to allocate memory for input frame\n"); |
| @@ -343,8 +385,10 @@ static DNNReturnType get_output_ov(void *model, const char *input_name, int inpu · 18 unmodified lines |
| 343 | 385 | task.ov_model = ov_model; |
| 344 | 386 | |
| 345 | 387 | request.infer_request = ov_model->infer_request; |
| 388 | + request.task_count = 1; |
| 389 | + request.tasks = &ptask; |
| 346 | 390 | |
| 347 | | − ret = execute_model_ov(&task, &request); |
| 391 | + ret = execute_model_ov(&request); |
| 348 | 392 | *output_width = out_frame->width; |
| 349 | 393 | *output_height = out_frame->height; |
| 350 | 394 | |
| @@ -393,6 +437,24 @@ DNNModel *ff_dnn_load_model_ov(const char *model_filename, const char *options, · 42 unmodified lines |
| 393 | 437 | if (status != OK) |
| 394 | 438 | goto err; |
| 395 | 439 | |
| 440 | + // batch size |
| 441 | + if (ctx->options.batch_size <= 0) { |
| 442 | + ctx->options.batch_size = 1; |
| 443 | + } |
| 444 | + |
| 445 | + if (ctx->options.batch_size > 1) { |
| 446 | + input_shapes_t input_shapes; |
| 447 | + status = ie_network_get_input_shapes(ov_model->network, &input_shapes); |
| 448 | + if (status != OK) |
| 449 | + goto err; |
| 450 | + for (int i = 0; i < input_shapes.shape_num; i++) |
| 451 | + input_shapes.shapes[i].shape.dims[0] = ctx->options.batch_size; |
| 452 | + status = ie_network_reshape(ov_model->network, input_shapes); |
| 453 | + ie_network_input_shapes_free(&input_shapes); |
| 454 | + if (status != OK) |
| 455 | + goto err; |
| 456 | + } |
| 457 | + |
| 396 | 458 | status = ie_core_load_network(ov_model->core, ov_model->network, ctx->options.device_type, &config, &ov_model->exe_network); |
| 397 | 459 | if (status != OK) { |
| 398 | 460 | av_log(ctx, AV_LOG_ERROR, "Failed to init OpenVINO model\n"); |
| @@ -426,17 +488,24 @@ DNNModel *ff_dnn_load_model_ov(const char *model_filename, const char *options, · 27 unmodified lines |
| 426 | 488 | } |
| 427 | 489 | |
| 428 | 490 | for (int i = 0; i < ctx->options.nireq; i++) { |
| 429 | | − ie_infer_request_t *request; |
| 430 | 491 | RequestItem *item = av_mallocz(sizeof(*item)); |
| 431 | 492 | if (!item) { |
| 432 | 493 | goto err; |
| 433 | 494 | } |
| 434 | | − status = ie_exec_network_create_infer_request(ov_model->exe_network, &request); |
| 495 | + |
| 496 | + status = ie_exec_network_create_infer_request(ov_model->exe_network, &item->infer_request); |
| 435 | 497 | if (status != OK) { |
| 436 | 498 | av_freep(&item); |
| 437 | 499 | goto err; |
| 438 | 500 | } |
| 439 | | − item->infer_request = request; |
| 501 | + |
| 502 | + item->tasks = av_malloc_array(ctx->options.batch_size, sizeof(*item->tasks)); |
| 503 | + if (!item->tasks) { |
| 504 | + av_freep(&item); |
| 505 | + goto err; |
| 506 | + } |
| 507 | + item->task_count = 0; |
| 508 | + |
| 440 | 509 | item->callback.completeCallBackFunc = infer_completion_callback; |
| 441 | 510 | item->callback.args = item; |
| 442 | 511 | if (ff_safe_queue_push_back(ov_model->request_queue, item) < 0) { |
| @@ -469,6 +538,7 @@ DNNReturnType ff_dnn_execute_model_ov(const DNNModel *model, const char *input_n · 26 unmodified lines |
| 469 | 538 | OVContext *ctx = &ov_model->ctx; |
| 470 | 539 | TaskItem task; |
| 471 | 540 | RequestItem request; |
| 541 | + TaskItem *ptask = &task; |
| 472 | 542 | |
| 473 | 543 | if (!in_frame) { |
| 474 | 544 | av_log(ctx, AV_LOG_ERROR, "in frame is NULL when execute model.\n"); |
| @@ -487,6 +557,11 @@ DNNReturnType ff_dnn_execute_model_ov(const DNNModel *model, const char *input_n · 12 unmodified lines |
| 487 | 557 | return DNN_ERROR; |
| 488 | 558 | } |
| 489 | 559 | |
| 560 | + if (ctx->options.batch_size > 1) { |
| 561 | + av_log(ctx, AV_LOG_ERROR, "do not support batch mode for sync execution.\n"); |
| 562 | + return DNN_ERROR; |
| 563 | + } |
| 564 | + |
| 490 | 565 | task.done = 0; |
| 491 | 566 | task.do_ioproc = 1; |
| 492 | 567 | task.async = 0; |
| @@ -497,8 +572,10 @@ DNNReturnType ff_dnn_execute_model_ov(const DNNModel *model, const char *input_n · 4 unmodified lines |
| 497 | 572 | task.ov_model = ov_model; |
| 498 | 573 | |
| 499 | 574 | request.infer_request = ov_model->infer_request; |
| 575 | + request.task_count = 1; |
| 576 | + request.tasks = &ptask; |
| 500 | 577 | |
| 501 | | − return execute_model_ov(&task, &request); |
| 578 | + return execute_model_ov(&request); |
| 502 | 579 | } |
| 503 | 580 | |
| 504 | 581 | DNNReturnType ff_dnn_execute_model_async_ov(const DNNModel *model, const char *input_name, AVFrame *in_frame, |
| @@ -545,7 +622,8 @@ DNNReturnType ff_dnn_execute_model_async_ov(const DNNModel *model, const char *i · 40 unmodified lines |
| 545 | 622 | return DNN_ERROR; |
| 546 | 623 | } |
| 547 | 624 | |
| 548 | | − return execute_model_ov(task, request); |
| 625 | + request->tasks[request->task_count++] = task; |
| 626 | + return execute_model_ov(request); |
| 549 | 627 | } |
| 550 | 628 | |
| 551 | 629 | DNNAsyncStatusType ff_dnn_get_async_result_ov(const DNNModel *model, AVFrame **in, AVFrame **out) |
| @@ -569,6 +647,48 @@ DNNAsyncStatusType ff_dnn_get_async_result_ov(const DNNModel *model, AVFrame **i · 17 unmodified lines |
| 569 | 647 | return DAST_SUCCESS; |
| 570 | 648 | } |
| 571 | 649 | |
| 650 | +DNNReturnType ff_dnn_flush_ov(const DNNModel *model) |
| 651 | +{ |
| 652 | + OVModel *ov_model = (OVModel *)model->model; |
| 653 | + OVContext *ctx = &ov_model->ctx; |
| 654 | + RequestItem *request; |
| 655 | + IEStatusCode status; |
| 656 | + DNNReturnType ret; |
| 657 | + |
| 658 | + request = ff_safe_queue_pop_front(ov_model->request_queue); |
| 659 | + if (!request) { |
| 660 | + av_log(ctx, AV_LOG_ERROR, "unable to get infer request.\n"); |
| 661 | + return DNN_ERROR; |
| 662 | + } |
| 663 | + |
| 664 | + if (request->task_count == 0) { |
| 665 | + // no pending task need to flush |
| 666 | + if (ff_safe_queue_push_back(ov_model->request_queue, request) < 0) { |
| 667 | + av_log(ctx, AV_LOG_ERROR, "Failed to push back request_queue.\n"); |
| 668 | + return DNN_ERROR; |
| 669 | + } |
| 670 | + return DNN_SUCCESS; |
| 671 | + } |
| 672 | + |
| 673 | + ret = fill_model_input_ov(ov_model, request); |
| 674 | + if (ret != DNN_SUCCESS) { |
| 675 | + av_log(ctx, AV_LOG_ERROR, "Failed to fill model input.\n"); |
| 676 | + return ret; |
| 677 | + } |
| 678 | + status = ie_infer_set_completion_callback(request->infer_request, &request->callback); |
| 679 | + if (status != OK) { |
| 680 | + av_log(ctx, AV_LOG_ERROR, "Failed to set completion callback for inference\n"); |
| 681 | + return DNN_ERROR; |
| 682 | + } |
| 683 | + status = ie_infer_request_infer_async(request->infer_request); |
| 684 | + if (status != OK) { |
| 685 | + av_log(ctx, AV_LOG_ERROR, "Failed to start async inference\n"); |
| 686 | + return DNN_ERROR; |
| 687 | + } |
| 688 | + |
| 689 | + return DNN_SUCCESS; |
| 690 | +} |
| 691 | + |
| 572 | 692 | void ff_dnn_free_model_ov(DNNModel **model) |
| 573 | 693 | { |
| 574 | 694 | if (*model){ |
| @@ -578,12 +698,15 @@ void ff_dnn_free_model_ov(DNNModel **model) · 3 unmodified lines |
| 578 | 698 | if (item && item->infer_request) { |
| 579 | 699 | ie_infer_request_free(&item->infer_request); |
| 580 | 700 | } |
| 701 | + av_freep(&item->tasks); |
| 581 | 702 | av_freep(&item); |
| 582 | 703 | } |
| 583 | 704 | ff_safe_queue_destroy(ov_model->request_queue); |
| 584 | 705 | |
| 585 | 706 | while (ff_queue_size(ov_model->task_queue) != 0) { |
| 586 | 707 | TaskItem *item = ff_queue_pop_front(ov_model->task_queue); |
| 708 | + av_frame_free(&item->in_frame); |
| 709 | + av_frame_free(&item->out_frame); |
| 587 | 710 | av_freep(&item); |
| 588 | 711 | } |
| 589 | 712 | ff_queue_destroy(ov_model->task_queue); |