History

dnn: add openvino as one of dnn backend

OpenVINO is a Deep Learning Deployment Toolkit at https://github.com/openvinotoolkit/openvino, it supports CPU, GPU and heterogeneous plugins to accelerate deep learning inferencing. Please refer to https://github.com/openvinotoolkit/openvino/blob/master/build-instruction.md to build openvino (c library is built at the same time). Please add option -DENABLE_MKL_DNN=ON for cmake to enable CPU path. The header files and libraries are installed to /usr/local/deployment_tools/inference_engine/ with default options on my system. To build FFmpeg with openvion, take my system as an example, run with: $ export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/deployment_tools/inference_engine/lib/intel64/:/usr/local/deployment_tools/inference_engine/external/tbb/lib/ $ ../ffmpeg/configure --enable-libopenvino --extra-cflags=-I/usr/local/deployment_tools/inference_engine/include/ --extra-ldflags=-L/usr/local/deployment_tools/inference_engine/lib/intel64 $ make Here are the features provided by OpenVINO inference engine: - support more DNN model formats It supports TensorFlow, Caffe, ONNX, MXNet and Kaldi by converting them into OpenVINO format with a python script. And torth model can be first converted into ONNX and then to OpenVINO format. see the script at https://github.com/openvinotoolkit/openvino/tree/master/model-optimizer/mo.py which also does some optimization at model level. - optimize at inference stage It optimizes for X86 CPUs with SSE, AVX etc. It also optimizes based on OpenCL for Intel GPUs. (only Intel GPU supported becuase Intel OpenCL extension is used for optimization)

Guo, Yejun committed 6 years ago

Commit ff37eba1 parent 1884d886 files · +317 −2Browse files

Showing the changes to libavfilter (5 of 6 files). Show all

Unified · Split

5 files
  1. libavfilter/dnn/Makefile +1 −0
  2. libavfilter/dnn/dnn_backend_openvino.c +261 −0
  3. libavfilter/dnn/dnn_backend_openvino.h +38 −0
  4. libavfilter/dnn/dnn_interface.c +11 −0
  5. libavfilter/dnn_interface.h +1 −1
libavfilter/dnn/Makefile+1−0
@@ -9,5 +9,6 @@ OBJS-$(CONFIG_DNN) += dnn/dnn_backend_native_layer_mat · 8 unmodified lines
9OBJS-$(CONFIG_DNN) += dnn/dnn_backend_native_layer_mathunary.o9OBJS-$(CONFIG_DNN) += dnn/dnn_backend_native_layer_mathunary.o
10 10
11DNN-OBJS-$(CONFIG_LIBTENSORFLOW) += dnn/dnn_backend_tf.o11DNN-OBJS-$(CONFIG_LIBTENSORFLOW) += dnn/dnn_backend_tf.o
12DNN-OBJS-$(CONFIG_LIBOPENVINO) += dnn/dnn_backend_openvino.o
12 13
13OBJS-$(CONFIG_DNN) += $(DNN-OBJS-yes)14OBJS-$(CONFIG_DNN) += $(DNN-OBJS-yes)
libavfilter/dnn/dnn_backend_openvino.c+261−0
@@ -0,0 +1,261 @@
1/*
2 * Copyright (c) 2020
3 *
4 * This file is part of FFmpeg.
5 *
6 * FFmpeg is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
10 *
11 * FFmpeg is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with FFmpeg; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19 */
20
21/**
22 * @file
23 * DNN OpenVINO backend implementation.
24 */
25
26#include "dnn_backend_openvino.h"
27#include "libavformat/avio.h"
28#include "libavutil/avassert.h"
29#include <c_api/ie_c_api.h>
30
31typedef struct OVModel{
32 ie_core_t *core;
33 ie_network_t *network;
34 ie_executable_network_t *exe_network;
35 ie_infer_request_t *infer_request;
36 ie_blob_t *input_blob;
37 ie_blob_t **output_blobs;
38 uint32_t nb_output;
39} OVModel;
40
41static DNNDataType precision_to_datatype(precision_e precision)
42{
43 switch (precision)
44 {
45 case FP32:
46 return DNN_FLOAT;
47 default:
48 av_assert0(!"not supported yet.");
49 return DNN_FLOAT;
50 }
51}
52
53static DNNReturnType get_input_ov(void *model, DNNData *input, const char *input_name)
54{
55 OVModel *ov_model = (OVModel *)model;
56 char *model_input_name = NULL;
57 IEStatusCode status;
58 size_t model_input_count = 0;
59 dimensions_t dims;
60 precision_e precision;
61
62 status = ie_network_get_inputs_number(ov_model->network, &model_input_count);
63 if (status != OK)
64 return DNN_ERROR;
65
66 for (size_t i = 0; i < model_input_count; i++) {
67 status = ie_network_get_input_name(ov_model->network, i, &model_input_name);
68 if (status != OK)
69 return DNN_ERROR;
70 if (strcmp(model_input_name, input_name) == 0) {
71 ie_network_name_free(&model_input_name);
72 status |= ie_network_get_input_dims(ov_model->network, input_name, &dims);
73 status |= ie_network_get_input_precision(ov_model->network, input_name, &precision);
74 if (status != OK)
75 return DNN_ERROR;
76
77 // The order of dims in the openvino is fixed and it is always NCHW for 4-D data.
78 // while we pass NHWC data from FFmpeg to openvino
79 status = ie_network_set_input_layout(ov_model->network, input_name, NHWC);
80 if (status != OK)
81 return DNN_ERROR;
82
83 input->channels = dims.dims[1];
84 input->height = dims.dims[2];
85 input->width = dims.dims[3];
86 input->dt = precision_to_datatype(precision);
87 return DNN_SUCCESS;
88 }
89
90 ie_network_name_free(&model_input_name);
91 }
92
93 return DNN_ERROR;
94}
95
96static DNNReturnType set_input_output_ov(void *model, DNNData *input, const char *input_name, const char **output_names, uint32_t nb_output)
97{
98 OVModel *ov_model = (OVModel *)model;
99 IEStatusCode status;
100 dimensions_t dims;
101 precision_e precision;
102 ie_blob_buffer_t blob_buffer;
103
104 status = ie_exec_network_create_infer_request(ov_model->exe_network, &ov_model->infer_request);
105 if (status != OK)
106 goto err;
107
108 status = ie_infer_request_get_blob(ov_model->infer_request, input_name, &ov_model->input_blob);
109 if (status != OK)
110 goto err;
111
112 status |= ie_blob_get_dims(ov_model->input_blob, &dims);
113 status |= ie_blob_get_precision(ov_model->input_blob, &precision);
114 if (status != OK)
115 goto err;
116
117 av_assert0(input->channels == dims.dims[1]);
118 av_assert0(input->height == dims.dims[2]);
119 av_assert0(input->width == dims.dims[3]);
120 av_assert0(input->dt == precision_to_datatype(precision));
121
122 status = ie_blob_get_buffer(ov_model->input_blob, &blob_buffer);
123 if (status != OK)
124 goto err;
125 input->data = blob_buffer.buffer;
126
127 // outputs
128 ov_model->nb_output = 0;
129 av_freep(&ov_model->output_blobs);
130 ov_model->output_blobs = av_mallocz_array(nb_output, sizeof(*ov_model->output_blobs));
131 if (!ov_model->output_blobs)
132 goto err;
133
134 for (int i = 0; i < nb_output; i++) {
135 const char *output_name = output_names[i];
136 status = ie_infer_request_get_blob(ov_model->infer_request, output_name, &(ov_model->output_blobs[i]));
137 if (status != OK)
138 goto err;
139 ov_model->nb_output++;
140 }
141
142 return DNN_SUCCESS;
143
144err:
145 if (ov_model->output_blobs) {
146 for (uint32_t i = 0; i < ov_model->nb_output; i++) {
147 ie_blob_free(&(ov_model->output_blobs[i]));
148 }
149 av_freep(&ov_model->output_blobs);
150 }
151 if (ov_model->input_blob)
152 ie_blob_free(&ov_model->input_blob);
153 if (ov_model->infer_request)
154 ie_infer_request_free(&ov_model->infer_request);
155 return DNN_ERROR;
156}
157
158DNNModel *ff_dnn_load_model_ov(const char *model_filename)
159{
160 DNNModel *model = NULL;
161 OVModel *ov_model = NULL;
162 IEStatusCode status;
163 ie_config_t config = {NULL, NULL, NULL};
164
165 model = av_malloc(sizeof(DNNModel));
166 if (!model){
167 return NULL;
168 }
169
170 ov_model = av_mallocz(sizeof(OVModel));
171 if (!ov_model)
172 goto err;
173
174 status = ie_core_create("", &ov_model->core);
175 if (status != OK)
176 goto err;
177
178 status = ie_core_read_network(ov_model->core, model_filename, NULL, &ov_model->network);
179 if (status != OK)
180 goto err;
181
182 status = ie_core_load_network(ov_model->core, ov_model->network, "CPU", &config, &ov_model->exe_network);
183 if (status != OK)
184 goto err;
185
186 model->model = (void *)ov_model;
187 model->set_input_output = &set_input_output_ov;
188 model->get_input = &get_input_ov;
189
190 return model;
191
192err:
193 if (model)
194 av_freep(&model);
195 if (ov_model) {
196 if (ov_model->exe_network)
197 ie_exec_network_free(&ov_model->exe_network);
198 if (ov_model->network)
199 ie_network_free(&ov_model->network);
200 if (ov_model->core)
201 ie_core_free(&ov_model->core);
202 av_freep(&ov_model);
203 }
204 return NULL;
205}
206
207DNNReturnType ff_dnn_execute_model_ov(const DNNModel *model, DNNData *outputs, uint32_t nb_output)
208{
209 dimensions_t dims;
210 precision_e precision;
211 ie_blob_buffer_t blob_buffer;
212 OVModel *ov_model = (OVModel *)model->model;
213 uint32_t nb = FFMIN(nb_output, ov_model->nb_output);
214 IEStatusCode status = ie_infer_request_infer(ov_model->infer_request);
215 if (status != OK)
216 return DNN_ERROR;
217
218 for (uint32_t i = 0; i < nb; ++i) {
219 status = ie_blob_get_buffer(ov_model->output_blobs[i], &blob_buffer);
220 if (status != OK)
221 return DNN_ERROR;
222
223 status |= ie_blob_get_dims(ov_model->output_blobs[i], &dims);
224 status |= ie_blob_get_precision(ov_model->output_blobs[i], &precision);
225 if (status != OK)
226 return DNN_ERROR;
227
228 outputs[i].channels = dims.dims[1];
229 outputs[i].height = dims.dims[2];
230 outputs[i].width = dims.dims[3];
231 outputs[i].dt = precision_to_datatype(precision);
232 outputs[i].data = blob_buffer.buffer;
233 }
234
235 return DNN_SUCCESS;
236}
237
238void ff_dnn_free_model_ov(DNNModel **model)
239{
240 if (*model){
241 OVModel *ov_model = (OVModel *)(*model)->model;
242 if (ov_model->output_blobs) {
243 for (uint32_t i = 0; i < ov_model->nb_output; i++) {
244 ie_blob_free(&(ov_model->output_blobs[i]));
245 }
246 av_freep(&ov_model->output_blobs);
247 }
248 if (ov_model->input_blob)
249 ie_blob_free(&ov_model->input_blob);
250 if (ov_model->infer_request)
251 ie_infer_request_free(&ov_model->infer_request);
252 if (ov_model->exe_network)
253 ie_exec_network_free(&ov_model->exe_network);
254 if (ov_model->network)
255 ie_network_free(&ov_model->network);
256 if (ov_model->core)
257 ie_core_free(&ov_model->core);
258 av_freep(&ov_model);
259 av_freep(model);
260 }
261}
libavfilter/dnn/dnn_backend_openvino.h+38−0
@@ -0,0 +1,38 @@
1/*
2 * Copyright (c) 2020
3 *
4 * This file is part of FFmpeg.
5 *
6 * FFmpeg is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
10 *
11 * FFmpeg is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with FFmpeg; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19 */
20
21/**
22 * @file
23 * DNN inference functions interface for OpenVINO backend.
24 */
25
26
27#ifndef AVFILTER_DNN_DNN_BACKEND_OPENVINO_H
28#define AVFILTER_DNN_DNN_BACKEND_OPENVINO_H
29
30#include "../dnn_interface.h"
31
32DNNModel *ff_dnn_load_model_ov(const char *model_filename);
33
34DNNReturnType ff_dnn_execute_model_ov(const DNNModel *model, DNNData *outputs, uint32_t nb_output);
35
36void ff_dnn_free_model_ov(DNNModel **model);
37
38#endif
libavfilter/dnn/dnn_interface.c+11−0
@@ -26,6 +26,7 @@ · 25 unmodified lines
26#include "../dnn_interface.h"26#include "../dnn_interface.h"
27#include "dnn_backend_native.h"27#include "dnn_backend_native.h"
28#include "dnn_backend_tf.h"28#include "dnn_backend_tf.h"
29#include "dnn_backend_openvino.h"
29#include "libavutil/mem.h"30#include "libavutil/mem.h"
30 31
31DNNModule *ff_get_dnn_module(DNNBackendType backend_type)32DNNModule *ff_get_dnn_module(DNNBackendType backend_type)
@@ -53,6 +54,16 @@ DNNModule *ff_get_dnn_module(DNNBackendType backend_type) · 21 unmodified lines
53 return NULL;54 return NULL;
54 #endif55 #endif
55 break;56 break;
57 case DNN_OV:
58 #if (CONFIG_LIBOPENVINO == 1)
59 dnn_module->load_model = &ff_dnn_load_model_ov;
60 dnn_module->execute_model = &ff_dnn_execute_model_ov;
61 dnn_module->free_model = &ff_dnn_free_model_ov;
62 #else
63 av_freep(&dnn_module);
64 return NULL;
65 #endif
66 break;
56 default:67 default:
57 av_log(NULL, AV_LOG_ERROR, "Module backend_type is not native or tensorflow\n");68 av_log(NULL, AV_LOG_ERROR, "Module backend_type is not native or tensorflow\n");
58 av_freep(&dnn_module);69 av_freep(&dnn_module);
libavfilter/dnn_interface.h+1−1
@@ -30,7 +30,7 @@ · 29 unmodified lines
30 30
31typedef enum {DNN_SUCCESS, DNN_ERROR} DNNReturnType;31typedef enum {DNN_SUCCESS, DNN_ERROR} DNNReturnType;
32 32
33typedef enum {DNN_NATIVE, DNN_TF} DNNBackendType;33typedef enum {DNN_NATIVE, DNN_TF, DNN_OV} DNNBackendType;
34 34
35typedef enum {DNN_FLOAT = 1, DNN_UINT8 = 4} DNNDataType;35typedef enum {DNN_FLOAT = 1, DNN_UINT8 = 4} DNNDataType;
36 36

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t