lavfi/dnn: Add OpenVINO API 2.0 support

OpenVINO API 2.0 was released in March 2022, which introduced new features. This commit implements current OpenVINO features with new 2.0 APIs. And will add other features in API 2.0. Please add installation path, which include openvino.pc, to PKG_CONFIG_PATH mannually for new OpenVINO libs config.

Wenbin Chen committed 3 years ago

Commit e79bd1f1 parent 80ad0e22 files · +487 −34Browse files

Showing the changes to libavfilter (1 of 2 files). Show all

Unified · Split

libavfilter/dnn/dnn_backend_openvino.c+483−32

@@ -32,7 +32,11 @@ · 31 unmodified lines
32	32	#include "libavutil/detection_bbox.h"
33	33	#include "../internal.h"
34	34	#include "safe_queue.h"
	35	+#if HAVE_OPENVINO2
	36	+#include <openvino/c/openvino.h>
	37	+#else
35	38	#include <c_api/ie_c_api.h>
	39	+#endif
36	40	#include "dnn_backend_common.h"
37	41
38	42	typedef struct OVOptions{
@@ -51,9 +55,20 @@ typedef struct OVContext { · 12 unmodified lines
51	55	typedef struct OVModel{
52	56	OVContext ctx;
53	57	DNNModel *model;
	58	+#if HAVE_OPENVINO2
	59	+ ov_core_t *core;
	60	+ ov_model_t *ov_model;
	61	+ ov_compiled_model_t *compiled_model;
	62	+ ov_output_const_port_t* input_port;
	63	+ ov_preprocess_input_info_t* input_info;
	64	+ ov_output_const_port_t* output_port;
	65	+ ov_preprocess_output_info_t* output_info;
	66	+ ov_preprocess_prepostprocessor_t* preprocess;
	67	+#else
54	68	ie_core_t *core;
55	69	ie_network_t *network;
56	70	ie_executable_network_t *exe_network;
	71	+#endif
57	72	SafeQueue *request_queue; // holds OVRequestItem
58	73	Queue *task_queue; // holds TaskItem
59	74	Queue *lltask_queue; // holds LastLevelTaskItem
@@ -63,10 +78,15 @@ typedef struct OVModel{ · 3 unmodified lines
63	78
64	79	// one request for one call to openvino
65	80	typedef struct OVRequestItem {
66		− ie_infer_request_t *infer_request;
67	81	LastLevelTaskItem **lltasks;
68	82	uint32_t lltask_count;
	83	+#if HAVE_OPENVINO2
	84	+ ov_infer_request_t *infer_request;
	85	+ ov_callback_t callback;
	86	+#else
69	87	ie_complete_call_back_t callback;
	88	+ ie_infer_request_t *infer_request;
	89	+#endif
70	90	} OVRequestItem;
71	91
72	92	#define APPEND_STRING(generated_string, iterate_string) \
@@ -85,11 +105,61 @@ static const AVOption dnn_openvino_options[] = { · 12 unmodified lines
85	105
86	106	AVFILTER_DEFINE_CLASS(dnn_openvino);
87	107
	108	+#if HAVE_OPENVINO2
	109	+static const struct {
	110	+ ov_status_e status;
	111	+ int av_err;
	112	+ const char *desc;
	113	+} ov2_errors[] = {
	114	+ { OK, 0, "success" },
	115	+ { GENERAL_ERROR, AVERROR_EXTERNAL, "general error" },
	116	+ { NOT_IMPLEMENTED, AVERROR(ENOSYS), "not implemented" },
	117	+ { NETWORK_NOT_LOADED, AVERROR_EXTERNAL, "network not loaded" },
	118	+ { PARAMETER_MISMATCH, AVERROR(EINVAL), "parameter mismatch" },
	119	+ { NOT_FOUND, AVERROR_EXTERNAL, "not found" },
	120	+ { OUT_OF_BOUNDS, AVERROR(EOVERFLOW), "out of bounds" },
	121	+ { UNEXPECTED, AVERROR_EXTERNAL, "unexpected" },
	122	+ { REQUEST_BUSY, AVERROR(EBUSY), "request busy" },
	123	+ { RESULT_NOT_READY, AVERROR(EBUSY), "result not ready" },
	124	+ { NOT_ALLOCATED, AVERROR(ENODATA), "not allocated" },
	125	+ { INFER_NOT_STARTED, AVERROR_EXTERNAL, "infer not started" },
	126	+ { NETWORK_NOT_READ, AVERROR_EXTERNAL, "network not read" },
	127	+ { INFER_CANCELLED, AVERROR(ECANCELED), "infer cancelled" },
	128	+ { INVALID_C_PARAM, AVERROR(EINVAL), "invalid C parameter" },
	129	+ { UNKNOWN_C_ERROR, AVERROR_UNKNOWN, "unknown C error" },
	130	+ { NOT_IMPLEMENT_C_METHOD, AVERROR(ENOSYS), "not implement C method" },
	131	+ { UNKNOW_EXCEPTION, AVERROR_UNKNOWN, "unknown exception" },
	132	+};
	133	+
	134	+static int ov2_map_error(ov_status_e status, const char **desc)
	135	+{
	136	+ int i;
	137	+ for (i = 0; i < FF_ARRAY_ELEMS(ov2_errors); i++) {
	138	+ if (ov2_errors[i].status == status) {
	139	+ if (desc)
	140	+ *desc = ov2_errors[i].desc;
	141	+ return ov2_errors[i].av_err;
	142	+ }
	143	+ }
	144	+ if (desc)
	145	+ *desc = "unknown error";
	146	+ return AVERROR_UNKNOWN;
	147	+}
	148	+#endif
	149	+
	150	+#if HAVE_OPENVINO2
	151	+static DNNDataType precision_to_datatype(ov_element_type_e precision)
	152	+#else
88	153	static DNNDataType precision_to_datatype(precision_e precision)
	154	+#endif
89	155	{
90	156	switch (precision)
91	157	{
	158	+#if HAVE_OPENVINO2
	159	+ case F32:
	160	+#else
92	161	case FP32:
	162	+#endif
93	163	return DNN_FLOAT;
94	164	case U8:
95	165	return DNN_UINT8;
@@ -115,20 +185,61 @@ static int get_datatype_size(DNNDataType dt) · 19 unmodified lines
115	185
116	186	static int fill_model_input_ov(OVModel ov_model, OVRequestItem request)
117	187	{
	188	+ DNNData input;
	189	+ LastLevelTaskItem *lltask;
	190	+ TaskItem *task;
	191	+ OVContext *ctx = &ov_model->ctx;
	192	+#if HAVE_OPENVINO2
	193	+ int64_t* dims;
	194	+ ov_status_e status;
	195	+ ov_tensor_t* tensor = NULL;
	196	+ ov_shape_t input_shape = {0};
	197	+ ov_element_type_e precision;
	198	+ void *input_data_ptr = NULL;
	199	+#else
118	200	dimensions_t dims;
119	201	precision_e precision;
120	202	ie_blob_buffer_t blob_buffer;
121		− OVContext *ctx = &ov_model->ctx;
122	203	IEStatusCode status;
123		− DNNData input;
124	204	ie_blob_t *input_blob = NULL;
125		− LastLevelTaskItem *lltask;
126		− TaskItem *task;
	205	+#endif
127	206
128	207	lltask = ff_queue_peek_front(ov_model->lltask_queue);
129	208	av_assert0(lltask);
130	209	task = lltask->task;
131	210
	211	+#if HAVE_OPENVINO2
	212	+ if (!ov_model_is_dynamic(ov_model->ov_model)) {
	213	+ ov_output_const_port_free(ov_model->input_port);
	214	+ status = ov_model_const_input_by_name(ov_model->ov_model, task->input_name, &ov_model->input_port);
	215	+ if (status != OK) {
	216	+ av_log(ctx, AV_LOG_ERROR, "Failed to get input port shape.\n");
	217	+ return ov2_map_error(status, NULL);
	218	+ }
	219	+ status = ov_const_port_get_shape(ov_model->input_port, &input_shape);
	220	+ if (status != OK) {
	221	+ av_log(ctx, AV_LOG_ERROR, "Failed to get input port shape.\n");
	222	+ return ov2_map_error(status, NULL);
	223	+ }
	224	+ dims = input_shape.dims;
	225	+ status = ov_port_get_element_type(ov_model->input_port, &precision);
	226	+ if (status != OK) {
	227	+ av_log(ctx, AV_LOG_ERROR, "Failed to get input port data type.\n");
	228	+ return ov2_map_error(status, NULL);
	229	+ }
	230	+ } else {
	231	+ avpriv_report_missing_feature(ctx, "Do not support dynamic model.");
	232	+ return AVERROR(ENOSYS);
	233	+ }
	234	+ input.height = dims[2];
	235	+ input.width = dims[3];
	236	+ input.channels = dims[1];
	237	+ input.dt = precision_to_datatype(precision);
	238	+ input.data = av_malloc(input.height * input.width * input.channels * get_datatype_size(input.dt));
	239	+ if (!input.data)
	240	+ return AVERROR(ENOMEM);
	241	+ input_data_ptr = input.data;
	242	+#else
132	243	status = ie_infer_request_get_blob(request->infer_request, task->input_name, &input_blob);
133	244	if (status != OK) {
134	245	av_log(ctx, AV_LOG_ERROR, "Failed to get input blob with name %s\n", task->input_name);
@@ -149,12 +260,12 @@ static int fill_model_input_ov(OVModel ov_model, OVRequestItem request) · 14 unmodified lines
149	260	av_log(ctx, AV_LOG_ERROR, "Failed to get input blob buffer\n");
150	261	return DNN_GENERIC_ERROR;
151	262	}
152		−
153	263	input.height = dims.dims[2];
154	264	input.width = dims.dims[3];
155	265	input.channels = dims.dims[1];
156	266	input.data = blob_buffer.buffer;
157	267	input.dt = precision_to_datatype(precision);
	268	+#endif
158	269	// all models in openvino open model zoo use BGR as input,
159	270	// change to be an option when necessary.
160	271	input.order = DCO_BGR;
@@ -187,29 +298,82 @@ static int fill_model_input_ov(OVModel ov_model, OVRequestItem request) · 26 unmodified lines
187	298	av_assert0(!"should not reach here");
188	299	break;
189	300	}
	301	+#if HAVE_OPENVINO2
	302	+ status = ov_tensor_create_from_host_ptr(precision, input_shape, input.data, &tensor);
	303	+ if (status != OK) {
	304	+ av_log(ctx, AV_LOG_ERROR, "Failed to create tensor from host prt.\n");
	305	+ return ov2_map_error(status, NULL);
	306	+ }
	307	+ status = ov_infer_request_set_input_tensor(request->infer_request, tensor);
	308	+ if (status != OK) {
	309	+ av_log(ctx, AV_LOG_ERROR, "Failed to Set an input tensor for the model.\n");
	310	+ return ov2_map_error(status, NULL);
	311	+ }
	312	+#endif
190	313	input.data = (uint8_t *)input.data
191	314	+ input.width * input.height * input.channels * get_datatype_size(input.dt);
192	315	}
	316	+#if HAVE_OPENVINO2
	317	+ av_freep(&input_data_ptr);
	318	+#else
193	319	ie_blob_free(&input_blob);
	320	+#endif
194	321
195	322	return 0;
196	323	}
197	324
198	325	static void infer_completion_callback(void *args)
199	326	{
200		− dimensions_t dims;
201		− precision_e precision;
202		− IEStatusCode status;
203	327	OVRequestItem *request = args;
204	328	LastLevelTaskItem *lltask = request->lltasks[0];
205	329	TaskItem *task = lltask->task;
206	330	OVModel *ov_model = task->model;
207	331	SafeQueue *requestq = ov_model->request_queue;
208		− ie_blob_t *output_blob = NULL;
209		− ie_blob_buffer_t blob_buffer;
210	332	DNNData output;
211	333	OVContext *ctx = &ov_model->ctx;
	334	+#if HAVE_OPENVINO2
	335	+ size_t* dims;
	336	+ ov_status_e status;
	337	+ ov_tensor_t *output_tensor;
	338	+ ov_shape_t output_shape = {0};
	339	+ ov_element_type_e precision;
	340	+
	341	+ status = ov_infer_request_get_output_tensor_by_index(request->infer_request, 0, &output_tensor);
	342	+ if (status != OK) {
	343	+ av_log(ctx, AV_LOG_ERROR,
	344	+ "Failed to get output tensor.");
	345	+ return;
	346	+ }
	347	+
	348	+ status = ov_tensor_data(output_tensor, &output.data);
	349	+ if (status != OK) {
	350	+ av_log(ctx, AV_LOG_ERROR,
	351	+ "Failed to get output data.");
	352	+ return;
	353	+ }
212	354
	355	+ status = ov_tensor_get_shape(output_tensor, &output_shape);
	356	+ if (status != OK) {
	357	+ av_log(ctx, AV_LOG_ERROR, "Failed to get output port shape.\n");
	358	+ return;
	359	+ }
	360	+ dims = output_shape.dims;
	361	+
	362	+ status = ov_port_get_element_type(ov_model->output_port, &precision);
	363	+ if (status != OK) {
	364	+ av_log(ctx, AV_LOG_ERROR, "Failed to get output port data type.\n");
	365	+ return;
	366	+ }
	367	+ output.channels = dims[1];
	368	+ output.height = dims[2];
	369	+ output.width = dims[3];
	370	+ av_assert0(request->lltask_count <= dims[0]);
	371	+#else
	372	+ IEStatusCode status;
	373	+ dimensions_t dims;
	374	+ ie_blob_t *output_blob = NULL;
	375	+ ie_blob_buffer_t blob_buffer;
	376	+ precision_e precision;
213	377	status = ie_infer_request_get_blob(request->infer_request, task->output_names[0], &output_blob);
214	378	if (status != OK) {
215	379	av_log(ctx, AV_LOG_ERROR,
@@ -232,14 +396,14 @@ static void infer_completion_callback(void *args) · 16 unmodified lines
232	396	av_log(ctx, AV_LOG_ERROR, "Failed to get dims or precision of output\n");
233	397	return;
234	398	}
235		−
	399	+ output.data = blob_buffer.buffer;
236	400	output.channels = dims.dims[1];
237	401	output.height = dims.dims[2];
238	402	output.width = dims.dims[3];
	403	+ av_assert0(request->lltask_count <= dims.dims[0]);
	404	+#endif
239	405	output.dt = precision_to_datatype(precision);
240		− output.data = blob_buffer.buffer;
241	406
242		− av_assert0(request->lltask_count <= dims.dims[0]);
243	407	av_assert0(request->lltask_count >= 1);
244	408	for (int i = 0; i < request->lltask_count; ++i) {
245	409	task = request->lltasks[i]->task;
@@ -281,11 +445,16 @@ static void infer_completion_callback(void *args) · 35 unmodified lines
281	445	output.data = (uint8_t *)output.data
282	446	+ output.width * output.height * output.channels * get_datatype_size(output.dt);
283	447	}
	448	+#if !HAVE_OPENVINO2
284	449	ie_blob_free(&output_blob);
285		−
	450	+#endif
286	451	request->lltask_count = 0;
287	452	if (ff_safe_queue_push_back(requestq, request) < 0) {
	453	+#if HAVE_OPENVINO2
	454	+ ov_infer_request_free(request->infer_request);
	455	+#else
288	456	ie_infer_request_free(&request->infer_request);
	457	+#endif
289	458	av_freep(&request);
290	459	av_log(ctx, AV_LOG_ERROR, "Failed to push back request_queue.\n");
291	460	return;
@@ -299,7 +468,11 @@ static void dnn_free_model_ov(DNNModel **model) · 7 unmodified lines
299	468	while (ff_safe_queue_size(ov_model->request_queue) != 0) {
300	469	OVRequestItem *item = ff_safe_queue_pop_front(ov_model->request_queue);
301	470	if (item && item->infer_request) {
	471	+#if HAVE_OPENVINO2
	472	+ ov_infer_request_free(item->infer_request);
	473	+#else
302	474	ie_infer_request_free(&item->infer_request);
	475	+#endif
303	476	}
304	477	av_freep(&item->lltasks);
305	478	av_freep(&item);
@@ -319,13 +492,23 @@ static void dnn_free_model_ov(DNNModel **model) · 13 unmodified lines
319	492	av_freep(&item);
320	493	}
321	494	ff_queue_destroy(ov_model->task_queue);
322		−
	495	+#if HAVE_OPENVINO2
	496	+ if (ov_model->preprocess)
	497	+ ov_preprocess_prepostprocessor_free(ov_model->preprocess);
	498	+ if (ov_model->compiled_model)
	499	+ ov_compiled_model_free(ov_model->compiled_model);
	500	+ if (ov_model->ov_model)
	501	+ ov_model_free(ov_model->ov_model);
	502	+ if (ov_model->core)
	503	+ ov_core_free(ov_model->core);
	504	+#else
323	505	if (ov_model->exe_network)
324	506	ie_exec_network_free(&ov_model->exe_network);
325	507	if (ov_model->network)
326	508	ie_network_free(&ov_model->network);
327	509	if (ov_model->core)
328	510	ie_core_free(&ov_model->core);
	511	+#endif
329	512	av_freep(&ov_model);
330	513	av_freep(model);
331	514	}
@@ -336,16 +519,106 @@ static int init_model_ov(OVModel ov_model, const char input_name, const char * · 4 unmodified lines
336	519	{
337	520	int ret = 0;
338	521	OVContext *ctx = &ov_model->ctx;
	522	+#if HAVE_OPENVINO2
	523	+ ov_status_e status;
	524	+ ov_preprocess_input_tensor_info_t* input_tensor_info;
	525	+ ov_preprocess_output_tensor_info_t* output_tensor_info;
	526	+ ov_model_t *tmp_ov_model;
	527	+ ov_layout_t* NHWC_layout = NULL;
	528	+ const char* NHWC_desc = "NHWC";
	529	+ const char* device = ctx->options.device_type;
	530	+#else
339	531	IEStatusCode status;
340	532	ie_available_devices_t a_dev;
341	533	ie_config_t config = {NULL, NULL, NULL};
342	534	char *all_dev_names = NULL;
	535	+#endif
343	536
344	537	// batch size
345	538	if (ctx->options.batch_size <= 0) {
346	539	ctx->options.batch_size = 1;
347	540	}
	541	+#if HAVE_OPENVINO2
	542	+ if (ctx->options.batch_size > 1) {
	543	+ avpriv_report_missing_feature(ctx, "Do not support batch_size > 1 for now,"
	544	+ "change batch_size to 1.\n");
	545	+ ctx->options.batch_size = 1;
	546	+ }
	547	+
	548	+ status = ov_preprocess_prepostprocessor_create(ov_model->ov_model, &ov_model->preprocess);
	549	+ if (status != OK) {
	550	+ av_log(ctx, AV_LOG_ERROR, "Failed to create preprocess for ov_model.\n");
	551	+ ret = ov2_map_error(status, NULL);
	552	+ goto err;
	553	+ }
	554	+
	555	+ status = ov_preprocess_prepostprocessor_get_input_info_by_name(ov_model->preprocess, input_name, &ov_model->input_info);
	556	+ status \|= ov_preprocess_prepostprocessor_get_output_info_by_name(ov_model->preprocess, output_name, &ov_model->output_info);
	557	+ if (status != OK) {
	558	+ av_log(ctx, AV_LOG_ERROR, "Failed to get input/output info from preprocess.\n");
	559	+ ret = ov2_map_error(status, NULL);
	560	+ goto err;
	561	+ }
	562	+
	563	+ status = ov_preprocess_input_info_get_tensor_info(ov_model->input_info, &input_tensor_info);
	564	+ status \|= ov_preprocess_output_info_get_tensor_info(ov_model->output_info, &output_tensor_info);
	565	+ if (status != OK) {
	566	+ av_log(ctx, AV_LOG_ERROR, "Failed to get tensor info from input/output.\n");
	567	+ ret = ov2_map_error(status, NULL);
	568	+ goto err;
	569	+ }
	570	+
	571	+ //set input layout
	572	+ status = ov_layout_create(NHWC_desc, &NHWC_layout);
	573	+ if (status != OK) {
	574	+ av_log(ctx, AV_LOG_ERROR, "Failed to create layout for input.\n");
	575	+ ret = ov2_map_error(status, NULL);
	576	+ goto err;
	577	+ }
	578	+
	579	+ status = ov_preprocess_input_tensor_info_set_layout(input_tensor_info, NHWC_layout);
	580	+ if (status != OK) {
	581	+ av_log(ctx, AV_LOG_ERROR, "Failed to set input tensor layout\n");
	582	+ ret = ov2_map_error(status, NULL);
	583	+ goto err;
	584	+ }
	585	+
	586	+ if (ov_model->model->func_type != DFT_PROCESS_FRAME)
	587	+ //set precision only for detect and classify
	588	+ status = ov_preprocess_input_tensor_info_set_element_type(input_tensor_info, U8);
	589	+ status \|= ov_preprocess_output_set_element_type(output_tensor_info, F32);
	590	+ if (status != OK) {
	591	+ av_log(ctx, AV_LOG_ERROR, "Failed to set input/output element type\n");
	592	+ ret = ov2_map_error(status, NULL);
	593	+ goto err;
	594	+ }
	595	+
	596	+ //update model
	597	+ if(ov_model->ov_model)
	598	+ tmp_ov_model = ov_model->ov_model;
	599	+ status = ov_preprocess_prepostprocessor_build(ov_model->preprocess, &ov_model->ov_model);
	600	+ if (status != OK) {
	601	+ av_log(ctx, AV_LOG_ERROR, "Failed to update OV model\n");
	602	+ ret = ov2_map_error(status, NULL);
	603	+ goto err;
	604	+ }
	605	+ ov_model_free(tmp_ov_model);
348	606
	607	+ //update output_port
	608	+ if (ov_model->output_port)
	609	+ ov_output_const_port_free(ov_model->output_port);
	610	+ status = ov_model_const_output_by_name(ov_model->ov_model, output_name, &ov_model->output_port);
	611	+ if (status != OK) {
	612	+ av_log(ctx, AV_LOG_ERROR, "Failed to get output port.\n");
	613	+ goto err;
	614	+ }
	615	+ //compile network
	616	+ status = ov_core_compile_model(ov_model->core, ov_model->ov_model, device, 0, &ov_model->compiled_model);
	617	+ if (status != OK) {
	618	+ ret = ov2_map_error(status, NULL);
	619	+ goto err;
	620	+ }
	621	+#else
349	622	if (ctx->options.batch_size > 1) {
350	623	input_shapes_t input_shapes;
351	624	status = ie_network_get_input_shapes(ov_model->network, &input_shapes);
@@ -420,7 +693,7 @@ static int init_model_ov(OVModel ov_model, const char input_name, const char * · 68 unmodified lines
420	693	ret = AVERROR(ENODEV);
421	694	goto err;
422	695	}
423		−
	696	+#endif
424	697	// create infer_requests for async execution
425	698	if (ctx->options.nireq <= 0) {
426	699	// the default value is a rough estimation
@@ -440,7 +713,11 @@ static int init_model_ov(OVModel ov_model, const char input_name, const char * · 13 unmodified lines
440	713	goto err;
441	714	}
442	715
	716	+#if HAVE_OPENVINO2
	717	+ item->callback.callback_func = infer_completion_callback;
	718	+#else
443	719	item->callback.completeCallBackFunc = infer_completion_callback;
	720	+#endif
444	721	item->callback.args = item;
445	722	if (ff_safe_queue_push_back(ov_model->request_queue, item) < 0) {
446	723	av_freep(&item);
@@ -448,11 +725,19 @@ static int init_model_ov(OVModel ov_model, const char input_name, const char * · 1 unmodified line
448	725	goto err;
449	726	}
450	727
	728	+#if HAVE_OPENVINO2
	729	+ status = ov_compiled_model_create_infer_request(ov_model->compiled_model, &item->infer_request);
	730	+ if (status != OK) {
	731	+ av_log(ctx, AV_LOG_ERROR, "Failed to Creates an inference request object.\n");
	732	+ goto err;
	733	+ }
	734	+#else
451	735	status = ie_exec_network_create_infer_request(ov_model->exe_network, &item->infer_request);
452	736	if (status != OK) {
453	737	ret = DNN_GENERIC_ERROR;
454	738	goto err;
455	739	}
	740	+#endif
456	741
457	742	item->lltasks = av_malloc_array(ctx->options.batch_size, sizeof(*item->lltasks));
458	743	if (!item->lltasks) {
@@ -483,7 +768,11 @@ err: · 24 unmodified lines
483	768
484	769	static int execute_model_ov(OVRequestItem request, Queue inferenceq)
485	770	{
	771	+#if HAVE_OPENVINO2
	772	+ ov_status_e status;
	773	+#else
486	774	IEStatusCode status;
	775	+#endif
487	776	LastLevelTaskItem *lltask;
488	777	int ret = 0;
489	778	TaskItem *task;
@@ -491,7 +780,11 @@ static int execute_model_ov(OVRequestItem request, Queue inferenceq) · 1 unmodified line
491	780	OVModel *ov_model;
492	781
493	782	if (ff_queue_size(inferenceq) == 0) {
	783	+#if HAVE_OPENVINO2
	784	+ ov_infer_request_free(request->infer_request);
	785	+#else
494	786	ie_infer_request_free(&request->infer_request);
	787	+#endif
495	788	av_freep(&request);
496	789	return 0;
497	790	}
@@ -501,11 +794,39 @@ static int execute_model_ov(OVRequestItem request, Queue inferenceq) · 3 unmodified lines
501	794	ov_model = task->model;
502	795	ctx = &ov_model->ctx;
503	796
	797	+ ret = fill_model_input_ov(ov_model, request);
	798	+ if (ret != 0) {
	799	+ goto err;
	800	+ }
	801	+
	802	+#if HAVE_OPENVINO2
504	803	if (task->async) {
505		− ret = fill_model_input_ov(ov_model, request);
506		− if (ret != 0) {
	804	+ status = ov_infer_request_set_callback(request->infer_request, &request->callback);
	805	+ if (status != OK) {
	806	+ av_log(ctx, AV_LOG_ERROR, "Failed to set completion callback for inference\n");
	807	+ ret = ov2_map_error(status, NULL);
	808	+ goto err;
	809	+ }
	810	+
	811	+ status = ov_infer_request_start_async(request->infer_request);
	812	+ if (status != OK) {
	813	+ av_log(ctx, AV_LOG_ERROR, "Failed to start async inference\n");
	814	+ ret = ov2_map_error(status, NULL);
	815	+ goto err;
	816	+ }
	817	+ return 0;
	818	+ } else {
	819	+ status = ov_infer_request_infer(request->infer_request);
	820	+ if (status != OK) {
	821	+ av_log(NULL, AV_LOG_ERROR, "Failed to start synchronous model inference for OV2\n");
	822	+ ret = ov2_map_error(status, NULL);
507	823	goto err;
508	824	}
	825	+ infer_completion_callback(request);
	826	+ return (task->inference_done == task->inference_todo) ? 0 : DNN_GENERIC_ERROR;
	827	+ }
	828	+#else
	829	+ if (task->async) {
509	830	status = ie_infer_set_completion_callback(request->infer_request, &request->callback);
510	831	if (status != OK) {
511	832	av_log(ctx, AV_LOG_ERROR, "Failed to set completion callback for inference\n");
@@ -520,10 +841,6 @@ static int execute_model_ov(OVRequestItem request, Queue inferenceq) · 8 unmodified lines
520	841	}
521	842	return 0;
522	843	} else {
523		− ret = fill_model_input_ov(ov_model, request);
524		− if (ret != 0) {
525		− goto err;
526		− }
527	844	status = ie_infer_request_infer(request->infer_request);
528	845	if (status != OK) {
529	846	av_log(ctx, AV_LOG_ERROR, "Failed to start synchronous model inference\n");
@@ -533,9 +850,14 @@ static int execute_model_ov(OVRequestItem request, Queue inferenceq) · 3 unmodified lines
533	850	infer_completion_callback(request);
534	851	return (task->inference_done == task->inference_todo) ? 0 : DNN_GENERIC_ERROR;
535	852	}
	853	+#endif
536	854	err:
537	855	if (ff_safe_queue_push_back(ov_model->request_queue, request) < 0) {
	856	+#if HAVE_OPENVINO2
	857	+ ov_infer_request_free(request->infer_request);
	858	+#else
538	859	ie_infer_request_free(&request->infer_request);
	860	+#endif
539	861	av_freep(&request);
540	862	}
541	863	return ret;
@@ -545,19 +867,54 @@ static int get_input_ov(void model, DNNData input, const char *input_name) · 3 unmodified lines
545	867	{
546	868	OVModel *ov_model = model;
547	869	OVContext *ctx = &ov_model->ctx;
	870	+ int input_resizable = ctx->options.input_resizable;
	871	+
	872	+#if HAVE_OPENVINO2
	873	+ ov_shape_t input_shape = {0};
	874	+ ov_element_type_e precision;
	875	+ int64_t* dims;
	876	+ ov_status_e status;
	877	+ if (!ov_model_is_dynamic(ov_model->ov_model)) {
	878	+ status = ov_model_const_input_by_name(ov_model->ov_model, input_name, &ov_model->input_port);
	879	+ if (status != OK) {
	880	+ av_log(ctx, AV_LOG_ERROR, "Failed to get input port shape.\n");
	881	+ return ov2_map_error(status, NULL);
	882	+ }
	883	+
	884	+ status = ov_const_port_get_shape(ov_model->input_port, &input_shape);
	885	+ if (status != OK) {
	886	+ av_log(ctx, AV_LOG_ERROR, "Failed to get input port shape.\n");
	887	+ return ov2_map_error(status, NULL);
	888	+ }
	889	+ dims = input_shape.dims;
	890	+
	891	+ status = ov_port_get_element_type(ov_model->input_port, &precision);
	892	+ if (status != OK) {
	893	+ av_log(ctx, AV_LOG_ERROR, "Failed to get input port data type.\n");
	894	+ return ov2_map_error(status, NULL);
	895	+ }
	896	+ } else {
	897	+ avpriv_report_missing_feature(ctx, "Do not support dynamic model now.");
	898	+ return AVERROR(ENOSYS);
	899	+ }
	900	+
	901	+ input->channels = dims[1];
	902	+ input->height = input_resizable ? -1 : dims[2];
	903	+ input->width = input_resizable ? -1 : dims[3];
	904	+ input->dt = precision_to_datatype(precision);
	905	+
	906	+ return 0;
	907	+#else
548	908	char *model_input_name = NULL;
549	909	IEStatusCode status;
550	910	size_t model_input_count = 0;
551	911	dimensions_t dims;
552	912	precision_e precision;
553		− int input_resizable = ctx->options.input_resizable;
554		−
555	913	status = ie_network_get_inputs_number(ov_model->network, &model_input_count);
556	914	if (status != OK) {
557	915	av_log(ctx, AV_LOG_ERROR, "Failed to get input count\n");
558	916	return DNN_GENERIC_ERROR;
559	917	}
560		−
561	918	for (size_t i = 0; i < model_input_count; i++) {
562	919	status = ie_network_get_input_name(ov_model->network, i, &model_input_name);
563	920	if (status != OK) {
@@ -585,6 +942,7 @@ static int get_input_ov(void model, DNNData input, const char *input_name) · 21 unmodified lines
585	942
586	943	av_log(ctx, AV_LOG_ERROR, "Could not find \"%s\" in model, all input(s) are: \"%s\"\n", input_name, ov_model->all_input_names);
587	944	return AVERROR(EINVAL);
	945	+#endif
588	946	}
589	947
590	948	static int contain_valid_detection_bbox(AVFrame *frame)
@@ -693,13 +1051,20 @@ static int extract_lltask_from_task(DNNFunctionType func_type, TaskItem *task, Q · 102 unmodified lines
693	1051	static int get_output_ov(void model, const char input_name, int input_width, int input_height,
694	1052	const char output_name, int output_width, int *output_height)
695	1053	{
	1054	+#if HAVE_OPENVINO2
	1055	+ ov_dimension_t dims[4] = {{1, 1}, {1, 1}, {input_height, input_height}, {input_width, input_width}};
	1056	+ ov_status_e status;
	1057	+ ov_shape_t input_shape = {0};
	1058	+ ov_partial_shape_t partial_shape;
	1059	+#else
	1060	+ IEStatusCode status;
	1061	+ input_shapes_t input_shapes;
	1062	+#endif
696	1063	int ret;
697	1064	OVModel *ov_model = model;
698	1065	OVContext *ctx = &ov_model->ctx;
699	1066	TaskItem task;
700	1067	OVRequestItem *request;
701		− IEStatusCode status;
702		− input_shapes_t input_shapes;
703	1068	DNNExecBaseParams exec_params = {
704	1069	.input_name = input_name,
705	1070	.output_names = &output_name,
@@ -713,6 +1078,46 @@ static int get_output_ov(void model, const char input_name, int input_width, i · 7 unmodified lines
713	1078	return AVERROR(EINVAL);
714	1079	}
715	1080
	1081	+#if HAVE_OPENVINO2
	1082	+ if (ctx->options.input_resizable) {
	1083	+ if (!ov_model_is_dynamic(ov_model->ov_model)) {
	1084	+ status = ov_partial_shape_create(4, dims, &partial_shape);
	1085	+ if (status != OK) {
	1086	+ av_log(ctx, AV_LOG_ERROR, "Failed create partial shape.\n");
	1087	+ goto err;
	1088	+ }
	1089	+ status = ov_const_port_get_shape(ov_model->input_port, &input_shape);
	1090	+ input_shape.dims[2] = input_height;
	1091	+ input_shape.dims[3] = input_width;
	1092	+ if (status != OK) {
	1093	+ av_log(ctx, AV_LOG_ERROR, "Failed create shape for model input resize.\n");
	1094	+ goto err;
	1095	+ }
	1096	+
	1097	+ status = ov_shape_to_partial_shape(input_shape, &partial_shape);
	1098	+ if (status != OK) {
	1099	+ av_log(ctx, AV_LOG_ERROR, "Failed create partial shape for model input resize.\n");
	1100	+ goto err;
	1101	+ }
	1102	+
	1103	+ status = ov_model_reshape_single_input(ov_model->ov_model, partial_shape);
	1104	+ if (status != OK) {
	1105	+ av_log(ctx, AV_LOG_ERROR, "Failed to reszie model input.\n");
	1106	+ goto err;
	1107	+ }
	1108	+ } else {
	1109	+ avpriv_report_missing_feature(ctx, "Do not support dynamic model.");
	1110	+ goto err;
	1111	+ }
	1112	+ }
	1113	+
	1114	+ status = ov_model_const_output_by_name(ov_model->ov_model, output_name, &ov_model->output_port);
	1115	+ if (status != OK) {
	1116	+ av_log(ctx, AV_LOG_ERROR, "Failed to get output port.\n");
	1117	+ goto err;
	1118	+ }
	1119	+ if (!ov_model->compiled_model) {
	1120	+#else
716	1121	if (ctx->options.input_resizable) {
717	1122	status = ie_network_get_input_shapes(ov_model->network, &input_shapes);
718	1123	input_shapes.shapes->shape.dims[2] = input_height;
@@ -724,8 +1129,8 @@ static int get_output_ov(void model, const char input_name, int input_width, i · 5 unmodified lines
724	1129	return DNN_GENERIC_ERROR;
725	1130	}
726	1131	}
727		−
728	1132	if (!ov_model->exe_network) {
	1133	+#endif
729	1134	ret = init_model_ov(ov_model, input_name, output_name);
730	1135	if (ret != 0) {
731	1136	av_log(ctx, AV_LOG_ERROR, "Failed init OpenVINO exectuable network or inference request\n");
@@ -765,9 +1170,15 @@ static DNNModel dnn_load_model_ov(const char model_filename, DNNFunctionType f · 33 unmodified lines
765	1170	DNNModel *model = NULL;
766	1171	OVModel *ov_model = NULL;
767	1172	OVContext *ctx = NULL;
768		− IEStatusCode status;
	1173	+#if HAVE_OPENVINO2
	1174	+ ov_core_t* core = NULL;
	1175	+ ov_model_t* ovmodel = NULL;
	1176	+ ov_status_e status;
	1177	+#else
769	1178	size_t node_count = 0;
770	1179	char *node_name = NULL;
	1180	+ IEStatusCode status;
	1181	+#endif
771	1182
772	1183	model = av_mallocz(sizeof(DNNModel));
773	1184	if (!model){
@@ -783,8 +1194,6 @@ static DNNModel dnn_load_model_ov(const char model_filename, DNNFunctionType f · 9 unmodified lines
783	1194	ov_model->model = model;
784	1195	ov_model->ctx.class = &dnn_openvino_class;
785	1196	ctx = &ov_model->ctx;
786		− ov_model->all_input_names = NULL;
787		− ov_model->all_output_names = NULL;
788	1197
789	1198	//parse options
790	1199	av_opt_set_defaults(ctx);
@@ -793,6 +1202,31 @@ static DNNModel dnn_load_model_ov(const char model_filename, DNNFunctionType f · 2 unmodified lines
793	1202	goto err;
794	1203	}
795	1204
	1205	+#if HAVE_OPENVINO2
	1206	+ status = ov_core_create(&core);
	1207	+ if (status != OK) {
	1208	+ goto err;
	1209	+ }
	1210	+
	1211	+ status = ov_core_read_model(core, model_filename, NULL, &ovmodel);
	1212	+ if (status != OK) {
	1213	+ ov_version_t ver;
	1214	+ status = ov_get_openvino_version(&ver);
	1215	+ av_log(NULL, AV_LOG_ERROR, "Failed to read the network from model file %s,\n"
	1216	+ "Please check if the model version matches the runtime OpenVINO Version:\n",
	1217	+ model_filename);
	1218	+ if (status == OK) {
	1219	+ av_log(NULL, AV_LOG_ERROR, "BuildNumber: %s\n", ver.buildNumber);
	1220	+ }
	1221	+ ov_version_free(&ver);
	1222	+ goto err;
	1223	+ }
	1224	+ ov_model->ov_model = ovmodel;
	1225	+ ov_model->core = core;
	1226	+#else
	1227	+ ov_model->all_input_names = NULL;
	1228	+ ov_model->all_output_names = NULL;
	1229	+
796	1230	status = ie_core_create("", &ov_model->core);
797	1231	if (status != OK)
798	1232	goto err;
@@ -835,6 +1269,7 @@ static DNNModel dnn_load_model_ov(const char model_filename, DNNFunctionType f · 36 unmodified lines
835	1269	}
836	1270	APPEND_STRING(ov_model->all_output_names, node_name)
837	1271	}
	1272	+#endif
838	1273
839	1274	model->get_input = &get_input_ov;
840	1275	model->get_output = &get_output_ov;
@@ -862,7 +1297,11 @@ static int dnn_execute_model_ov(const DNNModel model, DNNExecBaseParams exec_p · 21 unmodified lines
862	1297	return ret;
863	1298	}
864	1299
	1300	+#if HAVE_OPENVINO2
	1301	+ if (!ov_model->compiled_model) {
	1302	+#else
865	1303	if (!ov_model->exe_network) {
	1304	+#endif
866	1305	ret = init_model_ov(ov_model, exec_params->input_name, exec_params->output_names[0]);
867	1306	if (ret != 0) {
868	1307	av_log(ctx, AV_LOG_ERROR, "Failed init OpenVINO exectuable network or inference request\n");
@@ -943,7 +1382,11 @@ static int dnn_flush_ov(const DNNModel *model) · 74 unmodified lines
943	1382	OVModel *ov_model = model->model;
944	1383	OVContext *ctx = &ov_model->ctx;
945	1384	OVRequestItem *request;
	1385	+#if HAVE_OPENVINO2
	1386	+ ov_status_e status;
	1387	+#else
946	1388	IEStatusCode status;
	1389	+#endif
947	1390	int ret;
948	1391
949	1392	if (ff_queue_size(ov_model->lltask_queue) == 0) {
@@ -962,6 +1405,13 @@ static int dnn_flush_ov(const DNNModel *model) · 12 unmodified lines
962	1405	av_log(ctx, AV_LOG_ERROR, "Failed to fill model input.\n");
963	1406	return ret;
964	1407	}
	1408	+#if HAVE_OPENVINO2
	1409	+ status = ov_infer_request_infer(request->infer_request);
	1410	+ if (status != OK) {
	1411	+ av_log(ctx, AV_LOG_ERROR, "Failed to start sync inference for OV2\n");
	1412	+ return ov2_map_error(status, NULL);
	1413	+ }
	1414	+#else
965	1415	status = ie_infer_set_completion_callback(request->infer_request, &request->callback);
966	1416	if (status != OK) {
967	1417	av_log(ctx, AV_LOG_ERROR, "Failed to set completion callback for inference\n");
@@ -972,6 +1422,7 @@ static int dnn_flush_ov(const DNNModel *model) · 4 unmodified lines
972	1422	av_log(ctx, AV_LOG_ERROR, "Failed to start async inference\n");
973	1423	return DNN_GENERIC_ERROR;
974	1424	}
	1425	+#endif
975	1426
976	1427	return 0;
977	1428	}

lavfi/dnn: Add OpenVINO API 2.0 support

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code