| @@ -0,0 +1,261 @@ |
| | 1 | /* |
| | 2 | * Copyright (c) 2020 |
| | 3 | * |
| | 4 | * This file is part of FFmpeg. |
| | 5 | * |
| | 6 | * FFmpeg is free software; you can redistribute it and/or |
| | 7 | * modify it under the terms of the GNU Lesser General Public |
| | 8 | * License as published by the Free Software Foundation; either |
| | 9 | * version 2.1 of the License, or (at your option) any later version. |
| | 10 | * |
| | 11 | * FFmpeg is distributed in the hope that it will be useful, |
| | 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| | 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
| | 14 | * Lesser General Public License for more details. |
| | 15 | * |
| | 16 | * You should have received a copy of the GNU Lesser General Public |
| | 17 | * License along with FFmpeg; if not, write to the Free Software |
| | 18 | * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA |
| | 19 | */ |
| | 20 | |
| | 21 | /** |
| | 22 | * @file |
| | 23 | * DNN OpenVINO backend implementation. |
| | 24 | */ |
| | 25 | |
| | 26 | #include "dnn_backend_openvino.h" |
| | 27 | #include "libavformat/avio.h" |
| | 28 | #include "libavutil/avassert.h" |
| | 29 | #include <c_api/ie_c_api.h> |
| | 30 | |
| | 31 | typedef struct OVModel{ |
| | 32 | ie_core_t *core; |
| | 33 | ie_network_t *network; |
| | 34 | ie_executable_network_t *exe_network; |
| | 35 | ie_infer_request_t *infer_request; |
| | 36 | ie_blob_t *input_blob; |
| | 37 | ie_blob_t **output_blobs; |
| | 38 | uint32_t nb_output; |
| | 39 | } OVModel; |
| | 40 | |
| | 41 | static DNNDataType precision_to_datatype(precision_e precision) |
| | 42 | { |
| | 43 | switch (precision) |
| | 44 | { |
| | 45 | case FP32: |
| | 46 | return DNN_FLOAT; |
| | 47 | default: |
| | 48 | av_assert0(!"not supported yet."); |
| | 49 | return DNN_FLOAT; |
| | 50 | } |
| | 51 | } |
| | 52 | |
| | 53 | static DNNReturnType get_input_ov(void *model, DNNData *input, const char *input_name) |
| | 54 | { |
| | 55 | OVModel *ov_model = (OVModel *)model; |
| | 56 | char *model_input_name = NULL; |
| | 57 | IEStatusCode status; |
| | 58 | size_t model_input_count = 0; |
| | 59 | dimensions_t dims; |
| | 60 | precision_e precision; |
| | 61 | |
| | 62 | status = ie_network_get_inputs_number(ov_model->network, &model_input_count); |
| | 63 | if (status != OK) |
| | 64 | return DNN_ERROR; |
| | 65 | |
| | 66 | for (size_t i = 0; i < model_input_count; i++) { |
| | 67 | status = ie_network_get_input_name(ov_model->network, i, &model_input_name); |
| | 68 | if (status != OK) |
| | 69 | return DNN_ERROR; |
| | 70 | if (strcmp(model_input_name, input_name) == 0) { |
| | 71 | ie_network_name_free(&model_input_name); |
| | 72 | status |= ie_network_get_input_dims(ov_model->network, input_name, &dims); |
| | 73 | status |= ie_network_get_input_precision(ov_model->network, input_name, &precision); |
| | 74 | if (status != OK) |
| | 75 | return DNN_ERROR; |
| | 76 | |
| | 77 | // The order of dims in the openvino is fixed and it is always NCHW for 4-D data. |
| | 78 | // while we pass NHWC data from FFmpeg to openvino |
| | 79 | status = ie_network_set_input_layout(ov_model->network, input_name, NHWC); |
| | 80 | if (status != OK) |
| | 81 | return DNN_ERROR; |
| | 82 | |
| | 83 | input->channels = dims.dims[1]; |
| | 84 | input->height = dims.dims[2]; |
| | 85 | input->width = dims.dims[3]; |
| | 86 | input->dt = precision_to_datatype(precision); |
| | 87 | return DNN_SUCCESS; |
| | 88 | } |
| | 89 | |
| | 90 | ie_network_name_free(&model_input_name); |
| | 91 | } |
| | 92 | |
| | 93 | return DNN_ERROR; |
| | 94 | } |
| | 95 | |
| | 96 | static DNNReturnType set_input_output_ov(void *model, DNNData *input, const char *input_name, const char **output_names, uint32_t nb_output) |
| | 97 | { |
| | 98 | OVModel *ov_model = (OVModel *)model; |
| | 99 | IEStatusCode status; |
| | 100 | dimensions_t dims; |
| | 101 | precision_e precision; |
| | 102 | ie_blob_buffer_t blob_buffer; |
| | 103 | |
| | 104 | status = ie_exec_network_create_infer_request(ov_model->exe_network, &ov_model->infer_request); |
| | 105 | if (status != OK) |
| | 106 | goto err; |
| | 107 | |
| | 108 | status = ie_infer_request_get_blob(ov_model->infer_request, input_name, &ov_model->input_blob); |
| | 109 | if (status != OK) |
| | 110 | goto err; |
| | 111 | |
| | 112 | status |= ie_blob_get_dims(ov_model->input_blob, &dims); |
| | 113 | status |= ie_blob_get_precision(ov_model->input_blob, &precision); |
| | 114 | if (status != OK) |
| | 115 | goto err; |
| | 116 | |
| | 117 | av_assert0(input->channels == dims.dims[1]); |
| | 118 | av_assert0(input->height == dims.dims[2]); |
| | 119 | av_assert0(input->width == dims.dims[3]); |
| | 120 | av_assert0(input->dt == precision_to_datatype(precision)); |
| | 121 | |
| | 122 | status = ie_blob_get_buffer(ov_model->input_blob, &blob_buffer); |
| | 123 | if (status != OK) |
| | 124 | goto err; |
| | 125 | input->data = blob_buffer.buffer; |
| | 126 | |
| | 127 | // outputs |
| | 128 | ov_model->nb_output = 0; |
| | 129 | av_freep(&ov_model->output_blobs); |
| | 130 | ov_model->output_blobs = av_mallocz_array(nb_output, sizeof(*ov_model->output_blobs)); |
| | 131 | if (!ov_model->output_blobs) |
| | 132 | goto err; |
| | 133 | |
| | 134 | for (int i = 0; i < nb_output; i++) { |
| | 135 | const char *output_name = output_names[i]; |
| | 136 | status = ie_infer_request_get_blob(ov_model->infer_request, output_name, &(ov_model->output_blobs[i])); |
| | 137 | if (status != OK) |
| | 138 | goto err; |
| | 139 | ov_model->nb_output++; |
| | 140 | } |
| | 141 | |
| | 142 | return DNN_SUCCESS; |
| | 143 | |
| | 144 | err: |
| | 145 | if (ov_model->output_blobs) { |
| | 146 | for (uint32_t i = 0; i < ov_model->nb_output; i++) { |
| | 147 | ie_blob_free(&(ov_model->output_blobs[i])); |
| | 148 | } |
| | 149 | av_freep(&ov_model->output_blobs); |
| | 150 | } |
| | 151 | if (ov_model->input_blob) |
| | 152 | ie_blob_free(&ov_model->input_blob); |
| | 153 | if (ov_model->infer_request) |
| | 154 | ie_infer_request_free(&ov_model->infer_request); |
| | 155 | return DNN_ERROR; |
| | 156 | } |
| | 157 | |
| | 158 | DNNModel *ff_dnn_load_model_ov(const char *model_filename) |
| | 159 | { |
| | 160 | DNNModel *model = NULL; |
| | 161 | OVModel *ov_model = NULL; |
| | 162 | IEStatusCode status; |
| | 163 | ie_config_t config = {NULL, NULL, NULL}; |
| | 164 | |
| | 165 | model = av_malloc(sizeof(DNNModel)); |
| | 166 | if (!model){ |
| | 167 | return NULL; |
| | 168 | } |
| | 169 | |
| | 170 | ov_model = av_mallocz(sizeof(OVModel)); |
| | 171 | if (!ov_model) |
| | 172 | goto err; |
| | 173 | |
| | 174 | status = ie_core_create("", &ov_model->core); |
| | 175 | if (status != OK) |
| | 176 | goto err; |
| | 177 | |
| | 178 | status = ie_core_read_network(ov_model->core, model_filename, NULL, &ov_model->network); |
| | 179 | if (status != OK) |
| | 180 | goto err; |
| | 181 | |
| | 182 | status = ie_core_load_network(ov_model->core, ov_model->network, "CPU", &config, &ov_model->exe_network); |
| | 183 | if (status != OK) |
| | 184 | goto err; |
| | 185 | |
| | 186 | model->model = (void *)ov_model; |
| | 187 | model->set_input_output = &set_input_output_ov; |
| | 188 | model->get_input = &get_input_ov; |
| | 189 | |
| | 190 | return model; |
| | 191 | |
| | 192 | err: |
| | 193 | if (model) |
| | 194 | av_freep(&model); |
| | 195 | if (ov_model) { |
| | 196 | if (ov_model->exe_network) |
| | 197 | ie_exec_network_free(&ov_model->exe_network); |
| | 198 | if (ov_model->network) |
| | 199 | ie_network_free(&ov_model->network); |
| | 200 | if (ov_model->core) |
| | 201 | ie_core_free(&ov_model->core); |
| | 202 | av_freep(&ov_model); |
| | 203 | } |
| | 204 | return NULL; |
| | 205 | } |
| | 206 | |
| | 207 | DNNReturnType ff_dnn_execute_model_ov(const DNNModel *model, DNNData *outputs, uint32_t nb_output) |
| | 208 | { |
| | 209 | dimensions_t dims; |
| | 210 | precision_e precision; |
| | 211 | ie_blob_buffer_t blob_buffer; |
| | 212 | OVModel *ov_model = (OVModel *)model->model; |
| | 213 | uint32_t nb = FFMIN(nb_output, ov_model->nb_output); |
| | 214 | IEStatusCode status = ie_infer_request_infer(ov_model->infer_request); |
| | 215 | if (status != OK) |
| | 216 | return DNN_ERROR; |
| | 217 | |
| | 218 | for (uint32_t i = 0; i < nb; ++i) { |
| | 219 | status = ie_blob_get_buffer(ov_model->output_blobs[i], &blob_buffer); |
| | 220 | if (status != OK) |
| | 221 | return DNN_ERROR; |
| | 222 | |
| | 223 | status |= ie_blob_get_dims(ov_model->output_blobs[i], &dims); |
| | 224 | status |= ie_blob_get_precision(ov_model->output_blobs[i], &precision); |
| | 225 | if (status != OK) |
| | 226 | return DNN_ERROR; |
| | 227 | |
| | 228 | outputs[i].channels = dims.dims[1]; |
| | 229 | outputs[i].height = dims.dims[2]; |
| | 230 | outputs[i].width = dims.dims[3]; |
| | 231 | outputs[i].dt = precision_to_datatype(precision); |
| | 232 | outputs[i].data = blob_buffer.buffer; |
| | 233 | } |
| | 234 | |
| | 235 | return DNN_SUCCESS; |
| | 236 | } |
| | 237 | |
| | 238 | void ff_dnn_free_model_ov(DNNModel **model) |
| | 239 | { |
| | 240 | if (*model){ |
| | 241 | OVModel *ov_model = (OVModel *)(*model)->model; |
| | 242 | if (ov_model->output_blobs) { |
| | 243 | for (uint32_t i = 0; i < ov_model->nb_output; i++) { |
| | 244 | ie_blob_free(&(ov_model->output_blobs[i])); |
| | 245 | } |
| | 246 | av_freep(&ov_model->output_blobs); |
| | 247 | } |
| | 248 | if (ov_model->input_blob) |
| | 249 | ie_blob_free(&ov_model->input_blob); |
| | 250 | if (ov_model->infer_request) |
| | 251 | ie_infer_request_free(&ov_model->infer_request); |
| | 252 | if (ov_model->exe_network) |
| | 253 | ie_exec_network_free(&ov_model->exe_network); |
| | 254 | if (ov_model->network) |
| | 255 | ie_network_free(&ov_model->network); |
| | 256 | if (ov_model->core) |
| | 257 | ie_core_free(&ov_model->core); |
| | 258 | av_freep(&ov_model); |
| | 259 | av_freep(model); |
| | 260 | } |
| | 261 | } |