History

libavfilter/dnn: add layout option to openvino backend

Dnn models have different input layout (NCHW or NHWC), so a "layout" option is added Use openvino's API to do layout conversion for input data. Use swscale to do layout conversion for output data as openvino doesn't have similiar C API for output.

Wenbin Chen committed 3 years ago

Commit 74ce1d21 parent 58d9b5c3 files · +185 −20Browse files

Unified · Split

3 files
  1. libavfilter/dnn/dnn_backend_openvino.c +42 −5
  2. libavfilter/dnn/dnn_io_proc.c +136 −15
  3. libavfilter/dnn_interface.h +7 −0
libavfilter/dnn/dnn_backend_openvino.c+42−5
@@ -45,6 +45,7 @@ typedef struct OVOptions{ · 44 unmodified lines
4545 uint8_t async;
4646 int batch_size;
4747 int input_resizable;
48+ DNNLayout layout;
4849 } OVOptions;
4950
5051 typedef struct OVContext {
@@ -100,6 +101,10 @@ static const AVOption dnn_openvino_options[] = { · 49 unmodified lines
100101 DNN_BACKEND_COMMON_OPTIONS
101102 { "batch_size", "batch size per request", OFFSET(options.batch_size), AV_OPT_TYPE_INT, { .i64 = 1 }, 1, 1000, FLAGS},
102103 { "input_resizable", "can input be resizable or not", OFFSET(options.input_resizable), AV_OPT_TYPE_BOOL, { .i64 = 0 }, 0, 1, FLAGS },
104+ { "layout", "input layout of model", OFFSET(options.layout), AV_OPT_TYPE_INT, { .i64 = DL_NONE}, DL_NONE, DL_NHWC, FLAGS, "layout" },
105+ { "none", "none", 0, AV_OPT_TYPE_CONST, { .i64 = DL_NONE }, 0, 0, FLAGS, "layout"},
106+ { "nchw", "nchw", 0, AV_OPT_TYPE_CONST, { .i64 = DL_NCHW }, 0, 0, FLAGS, "layout"},
107+ { "nhwc", "nhwc", 0, AV_OPT_TYPE_CONST, { .i64 = DL_NHWC }, 0, 0, FLAGS, "layout"},
103108 { NULL }
104109 };
105110
@@ -235,9 +240,9 @@ static int fill_model_input_ov(OVModel *ov_model, OVRequestItem *request) · 129 unmodified lines
235240 avpriv_report_missing_feature(ctx, "Do not support dynamic model.");
236241 return AVERROR(ENOSYS);
237242 }
238− input.height = dims[2];
239− input.width = dims[3];
240− input.channels = dims[1];
243+ input.height = dims[1];
244+ input.width = dims[2];
245+ input.channels = dims[3];
241246 input.dt = precision_to_datatype(precision);
242247 input.data = av_malloc(input.height * input.width * input.channels * get_datatype_size(input.dt));
243248 if (!input.data) {
@@ -412,6 +417,7 @@ static void infer_completion_callback(void *args) · 168 unmodified lines
412417 av_assert0(request->lltask_count <= dims.dims[0]);
413418 #endif
414419 output.dt = precision_to_datatype(precision);
420+ output.layout = ctx->options.layout;
415421
416422 av_assert0(request->lltask_count >= 1);
417423 for (int i = 0; i < request->lltask_count; ++i) {
@@ -540,11 +546,14 @@ static int init_model_ov(OVModel *ov_model, const char *input_name, const char * · 122 unmodified lines
540546 OVContext *ctx = &ov_model->ctx;
541547 #if HAVE_OPENVINO2
542548 ov_status_e status;
543− ov_preprocess_input_tensor_info_t* input_tensor_info;
544− ov_preprocess_output_tensor_info_t* output_tensor_info;
549+ ov_preprocess_input_tensor_info_t* input_tensor_info = NULL;
550+ ov_preprocess_output_tensor_info_t* output_tensor_info = NULL;
551+ ov_preprocess_input_model_info_t* input_model_info = NULL;
545552 ov_model_t *tmp_ov_model;
546553 ov_layout_t* NHWC_layout = NULL;
554+ ov_layout_t* NCHW_layout = NULL;
547555 const char* NHWC_desc = "NHWC";
556+ const char* NCHW_desc = "NCHW";
548557 const char* device = ctx->options.device_type;
549558 #else
550559 IEStatusCode status;
@@ -589,6 +598,7 @@ static int init_model_ov(OVModel *ov_model, const char *input_name, const char * · 38 unmodified lines
589598
590599 //set input layout
591600 status = ov_layout_create(NHWC_desc, &NHWC_layout);
601+ status |= ov_layout_create(NCHW_desc, &NCHW_layout);
592602 if (status != OK) {
593603 av_log(ctx, AV_LOG_ERROR, "Failed to create layout for input.\n");
594604 ret = ov2_map_error(status, NULL);
@@ -602,6 +612,22 @@ static int init_model_ov(OVModel *ov_model, const char *input_name, const char * · 7 unmodified lines
602612 goto err;
603613 }
604614
615+ status = ov_preprocess_input_info_get_model_info(ov_model->input_info, &input_model_info);
616+ if (status != OK) {
617+ av_log(ctx, AV_LOG_ERROR, "Failed to get input model info\n");
618+ ret = ov2_map_error(status, NULL);
619+ goto err;
620+ }
621+ if (ctx->options.layout == DL_NCHW)
622+ status = ov_preprocess_input_model_info_set_layout(input_model_info, NCHW_layout);
623+ else if (ctx->options.layout == DL_NHWC)
624+ status = ov_preprocess_input_model_info_set_layout(input_model_info, NHWC_layout);
625+ if (status != OK) {
626+ av_log(ctx, AV_LOG_ERROR, "Failed to get set input model layout\n");
627+ ret = ov2_map_error(status, NULL);
628+ goto err;
629+ }
630+
605631 if (ov_model->model->func_type != DFT_PROCESS_FRAME)
606632 //set precision only for detect and classify
607633 status = ov_preprocess_input_tensor_info_set_element_type(input_tensor_info, U8);
@@ -639,6 +665,9 @@ static int init_model_ov(OVModel *ov_model, const char *input_name, const char * · 31 unmodified lines
639665 ret = ov2_map_error(status, NULL);
640666 goto err;
641667 }
668+ ov_preprocess_input_model_info_free(input_model_info);
669+ ov_layout_free(NCHW_layout);
670+ ov_layout_free(NHWC_layout);
642671 #else
643672 if (ctx->options.batch_size > 1) {
644673 input_shapes_t input_shapes;
@@ -783,6 +812,14 @@ static int init_model_ov(OVModel *ov_model, const char *input_name, const char * · 138 unmodified lines
783812 return 0;
784813
785814 err:
815+#if HAVE_OPENVINO2
816+ if (NCHW_layout)
817+ ov_layout_free(NCHW_layout);
818+ if (NHWC_layout)
819+ ov_layout_free(NHWC_layout);
820+ if (input_model_info)
821+ ov_preprocess_input_model_info_free(input_model_info);
822+#endif
786823 dnn_free_model_ov(&ov_model->model);
787824 return ret;
788825 }
libavfilter/dnn/dnn_io_proc.c+136−15
@@ -27,6 +27,12 @@ · 26 unmodified lines
2727 int ff_proc_from_dnn_to_frame(AVFrame *frame, DNNData *output, void *log_ctx)
2828 {
2929 struct SwsContext *sws_ctx;
30+ int ret = 0;
31+ int linesize[4] = { 0 };
32+ void **dst_data = NULL;
33+ void *middle_data = NULL;
34+ uint8_t *planar_data[4] = { 0 };
35+ int plane_size = frame->width * frame->height * sizeof(uint8_t);
3036 int bytewidth = av_image_get_linesize(frame->format, frame->width, 0);
3137 if (bytewidth < 0) {
3238 return AVERROR(EINVAL);
@@ -35,6 +41,17 @@ int ff_proc_from_dnn_to_frame(AVFrame *frame, DNNData *output, void *log_ctx) · 2 unmodified lines
3541 avpriv_report_missing_feature(log_ctx, "data type rather than DNN_FLOAT");
3642 return AVERROR(ENOSYS);
3743 }
44+ dst_data = (void **)frame->data;
45+ linesize[0] = frame->linesize[0];
46+ if (output->layout == DL_NCHW) {
47+ middle_data = av_malloc(plane_size * output->channels);
48+ if (!middle_data) {
49+ ret = AVERROR(ENOMEM);
50+ goto err;
51+ }
52+ dst_data = &middle_data;
53+ linesize[0] = frame->width * 3;
54+ }
3855
3956 switch (frame->format) {
4057 case AV_PIX_FMT_RGB24:
@@ -51,18 +68,52 @@ int ff_proc_from_dnn_to_frame(AVFrame *frame, DNNData *output, void *log_ctx) · 10 unmodified lines
5168 "fmt:%s s:%dx%d -> fmt:%s s:%dx%d\n",
5269 av_get_pix_fmt_name(AV_PIX_FMT_GRAYF32), frame->width * 3, frame->height,
5370 av_get_pix_fmt_name(AV_PIX_FMT_GRAY8), frame->width * 3, frame->height);
54− return AVERROR(EINVAL);
71+ ret = AVERROR(EINVAL);
72+ goto err;
5573 }
5674 sws_scale(sws_ctx, (const uint8_t *[4]){(const uint8_t *)output->data, 0, 0, 0},
5775 (const int[4]){frame->width * 3 * sizeof(float), 0, 0, 0}, 0, frame->height,
58− (uint8_t * const*)frame->data, frame->linesize);
76+ (uint8_t * const*)dst_data, linesize);
5977 sws_freeContext(sws_ctx);
60− return 0;
78+ // convert data from planar to packed
79+ if (output->layout == DL_NCHW) {
80+ sws_ctx = sws_getContext(frame->width,
81+ frame->height,
82+ AV_PIX_FMT_GBRP,
83+ frame->width,
84+ frame->height,
85+ frame->format,
86+ 0, NULL, NULL, NULL);
87+ if (!sws_ctx) {
88+ av_log(log_ctx, AV_LOG_ERROR, "Impossible to create scale context for the conversion "
89+ "fmt:%s s:%dx%d -> fmt:%s s:%dx%d\n",
90+ av_get_pix_fmt_name(AV_PIX_FMT_GBRP), frame->width, frame->height,
91+ av_get_pix_fmt_name(frame->format),frame->width, frame->height);
92+ ret = AVERROR(EINVAL);
93+ goto err;
94+ }
95+ if (frame->format == AV_PIX_FMT_RGB24) {
96+ planar_data[0] = (uint8_t *)middle_data + plane_size;
97+ planar_data[1] = (uint8_t *)middle_data + plane_size * 2;
98+ planar_data[2] = (uint8_t *)middle_data;
99+ } else if (frame->format == AV_PIX_FMT_BGR24) {
100+ planar_data[0] = (uint8_t *)middle_data + plane_size;
101+ planar_data[1] = (uint8_t *)middle_data;
102+ planar_data[2] = (uint8_t *)middle_data + plane_size * 2;
103+ }
104+ sws_scale(sws_ctx, (const uint8_t * const *)planar_data,
105+ (const int [4]){frame->width * sizeof(uint8_t),
106+ frame->width * sizeof(uint8_t),
107+ frame->width * sizeof(uint8_t), 0},
108+ 0, frame->height, frame->data, frame->linesize);
109+ sws_freeContext(sws_ctx);
110+ }
111+ break;
61112 case AV_PIX_FMT_GRAYF32:
62113 av_image_copy_plane(frame->data[0], frame->linesize[0],
63114 output->data, bytewidth,
64115 bytewidth, frame->height);
65− return 0;
116+ break;
66117 case AV_PIX_FMT_YUV420P:
67118 case AV_PIX_FMT_YUV422P:
68119 case AV_PIX_FMT_YUV444P:
@@ -82,24 +133,34 @@ int ff_proc_from_dnn_to_frame(AVFrame *frame, DNNData *output, void *log_ctx) · 13 unmodified lines
82133 "fmt:%s s:%dx%d -> fmt:%s s:%dx%d\n",
83134 av_get_pix_fmt_name(AV_PIX_FMT_GRAYF32), frame->width, frame->height,
84135 av_get_pix_fmt_name(AV_PIX_FMT_GRAY8), frame->width, frame->height);
85− return AVERROR(EINVAL);
136+ ret = AVERROR(EINVAL);
137+ goto err;
86138 }
87139 sws_scale(sws_ctx, (const uint8_t *[4]){(const uint8_t *)output->data, 0, 0, 0},
88140 (const int[4]){frame->width * sizeof(float), 0, 0, 0}, 0, frame->height,
89141 (uint8_t * const*)frame->data, frame->linesize);
90142 sws_freeContext(sws_ctx);
91− return 0;
143+ break;
92144 default:
93145 avpriv_report_missing_feature(log_ctx, "%s", av_get_pix_fmt_name(frame->format));
94− return AVERROR(ENOSYS);
146+ ret = AVERROR(ENOSYS);
147+ goto err;
95148 }
96149
97− return 0;
150+err:
151+ av_free(middle_data);
152+ return ret;
98153 }
99154
100155 int ff_proc_from_frame_to_dnn(AVFrame *frame, DNNData *input, void *log_ctx)
101156 {
102157 struct SwsContext *sws_ctx;
158+ int ret = 0;
159+ int linesize[4] = { 0 };
160+ void **src_data = NULL;
161+ void *middle_data = NULL;
162+ uint8_t *planar_data[4] = { 0 };
163+ int plane_size = frame->width * frame->height * sizeof(uint8_t);
103164 int bytewidth = av_image_get_linesize(frame->format, frame->width, 0);
104165 if (bytewidth < 0) {
105166 return AVERROR(EINVAL);
@@ -109,9 +170,54 @@ int ff_proc_from_frame_to_dnn(AVFrame *frame, DNNData *input, void *log_ctx) · 3 unmodified lines
109170 return AVERROR(ENOSYS);
110171 }
111172
173+ src_data = (void **)frame->data;
174+ linesize[0] = frame->linesize[0];
175+ if (input->layout == DL_NCHW) {
176+ middle_data = av_malloc(plane_size * input->channels);
177+ if (!middle_data) {
178+ ret = AVERROR(ENOMEM);
179+ goto err;
180+ }
181+ src_data = &middle_data;
182+ linesize[0] = frame->width * 3;
183+ }
184+
112185 switch (frame->format) {
113186 case AV_PIX_FMT_RGB24:
114187 case AV_PIX_FMT_BGR24:
188+ // convert data from planar to packed
189+ if (input->layout == DL_NCHW) {
190+ sws_ctx = sws_getContext(frame->width,
191+ frame->height,
192+ frame->format,
193+ frame->width,
194+ frame->height,
195+ AV_PIX_FMT_GBRP,
196+ 0, NULL, NULL, NULL);
197+ if (!sws_ctx) {
198+ av_log(log_ctx, AV_LOG_ERROR, "Impossible to create scale context for the conversion "
199+ "fmt:%s s:%dx%d -> fmt:%s s:%dx%d\n",
200+ av_get_pix_fmt_name(frame->format), frame->width, frame->height,
201+ av_get_pix_fmt_name(AV_PIX_FMT_GBRP),frame->width, frame->height);
202+ ret = AVERROR(EINVAL);
203+ goto err;
204+ }
205+ if (frame->format == AV_PIX_FMT_RGB24) {
206+ planar_data[0] = (uint8_t *)middle_data + plane_size;
207+ planar_data[1] = (uint8_t *)middle_data + plane_size * 2;
208+ planar_data[2] = (uint8_t *)middle_data;
209+ } else if (frame->format == AV_PIX_FMT_BGR24) {
210+ planar_data[0] = (uint8_t *)middle_data + plane_size;
211+ planar_data[1] = (uint8_t *)middle_data;
212+ planar_data[2] = (uint8_t *)middle_data + plane_size * 2;
213+ }
214+ sws_scale(sws_ctx, (const uint8_t * const *)frame->data,
215+ frame->linesize, 0, frame->height, planar_data,
216+ (const int [4]){frame->width * sizeof(uint8_t),
217+ frame->width * sizeof(uint8_t),
218+ frame->width * sizeof(uint8_t), 0});
219+ sws_freeContext(sws_ctx);
220+ }
115221 sws_ctx = sws_getContext(frame->width * 3,
116222 frame->height,
117223 AV_PIX_FMT_GRAY8,
@@ -124,10 +230,11 @@ int ff_proc_from_frame_to_dnn(AVFrame *frame, DNNData *input, void *log_ctx) · 6 unmodified lines
124230 "fmt:%s s:%dx%d -> fmt:%s s:%dx%d\n",
125231 av_get_pix_fmt_name(AV_PIX_FMT_GRAY8), frame->width * 3, frame->height,
126232 av_get_pix_fmt_name(AV_PIX_FMT_GRAYF32),frame->width * 3, frame->height);
127− return AVERROR(EINVAL);
233+ ret = AVERROR(EINVAL);
234+ goto err;
128235 }
129− sws_scale(sws_ctx, (const uint8_t **)frame->data,
130− frame->linesize, 0, frame->height,
236+ sws_scale(sws_ctx, (const uint8_t **)src_data,
237+ linesize, 0, frame->height,
131238 (uint8_t * const [4]){input->data, 0, 0, 0},
132239 (const int [4]){frame->width * 3 * sizeof(float), 0, 0, 0});
133240 sws_freeContext(sws_ctx);
@@ -156,7 +263,8 @@ int ff_proc_from_frame_to_dnn(AVFrame *frame, DNNData *input, void *log_ctx) · 22 unmodified lines
156263 "fmt:%s s:%dx%d -> fmt:%s s:%dx%d\n",
157264 av_get_pix_fmt_name(AV_PIX_FMT_GRAY8), frame->width, frame->height,
158265 av_get_pix_fmt_name(AV_PIX_FMT_GRAYF32),frame->width, frame->height);
159− return AVERROR(EINVAL);
266+ ret = AVERROR(EINVAL);
267+ goto err;
160268 }
161269 sws_scale(sws_ctx, (const uint8_t **)frame->data,
162270 frame->linesize, 0, frame->height,
@@ -166,10 +274,12 @@ int ff_proc_from_frame_to_dnn(AVFrame *frame, DNNData *input, void *log_ctx) · 3 unmodified lines
166274 break;
167275 default:
168276 avpriv_report_missing_feature(log_ctx, "%s", av_get_pix_fmt_name(frame->format));
169− return AVERROR(ENOSYS);
277+ ret = AVERROR(ENOSYS);
278+ goto err;
170279 }
171−
172− return 0;
280+err:
281+ av_free(middle_data);
282+ return ret;
173283 }
174284
175285 static enum AVPixelFormat get_pixel_format(DNNData *data)
@@ -205,6 +315,11 @@ int ff_frame_to_dnn_classify(AVFrame *frame, DNNData *input, uint32_t bbox_index · 29 unmodified lines
205315 AVFrameSideData *sd = av_frame_get_side_data(frame, AV_FRAME_DATA_DETECTION_BBOXES);
206316 av_assert0(sd);
207317
318+ if (input->layout == DL_NCHW) {
319+ av_log(log_ctx, AV_LOG_ERROR, "dnn_classify input data doesn't support layout: NCHW\n");
320+ return AVERROR(ENOSYS);
321+ }
322+
208323 header = (const AVDetectionBBoxHeader *)sd->data;
209324 bbox = av_get_detection_bbox(header, bbox_index);
210325
@@ -257,6 +372,12 @@ int ff_frame_to_dnn_detect(AVFrame *frame, DNNData *input, void *log_ctx) · 46 unmodified lines
257372 int linesizes[4];
258373 int ret = 0;
259374 enum AVPixelFormat fmt = get_pixel_format(input);
375+
376+ if (input->layout == DL_NCHW) {
377+ av_log(log_ctx, AV_LOG_ERROR, "dnn_detect input data doesn't support layout: NCHW\n");
378+ return AVERROR(ENOSYS);
379+ }
380+
260381 sws_ctx = sws_getContext(frame->width, frame->height, frame->format,
261382 input->width, input->height, fmt,
262383 SWS_FAST_BILINEAR, NULL, NULL, NULL);
libavfilter/dnn_interface.h+7−0
@@ -56,12 +56,19 @@ typedef enum { · 55 unmodified lines
5656 DFT_ANALYTICS_CLASSIFY, // classify for each bounding box
5757 }DNNFunctionType;
5858
59+typedef enum {
60+ DL_NONE,
61+ DL_NCHW,
62+ DL_NHWC,
63+} DNNLayout;
64+
5965 typedef struct DNNData{
6066 void *data;
6167 int width, height, channels;
6268 // dt and order together decide the color format
6369 DNNDataType dt;
6470 DNNColorOrder order;
71+ DNNLayout layout;
6572 } DNNData;
6673
6774 typedef struct DNNExecBaseParams {

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t