libavfilter/dnn: Add scale and mean preprocess to openvino backend

Dnn models has different data preprocess requirements. Scale and mean parameters are added to preprocess input data.

Wenbin Chen committed 3 years ago

Commit c8c925d1 parent 74ce1d23 files · +108 −19Browse files

Unified · Split

3 files

libavfilter/dnn/dnn_backend_openvino.c +39 −4
libavfilter/dnn/dnn_io_proc.c +67 −15
libavfilter/dnn_interface.h +2 −0

libavfilter/dnn/dnn_backend_openvino.c+39−4

@@ -46,6 +46,8 @@ typedef struct OVOptions{ · 45 unmodified lines
46	46	int batch_size;
47	47	int input_resizable;
48	48	DNNLayout layout;
	49	+ float scale;
	50	+ float mean;
49	51	} OVOptions;
50	52
51	53	typedef struct OVContext {
@@ -105,6 +107,8 @@ static const AVOption dnn_openvino_options[] = { · 53 unmodified lines
105	107	{ "none", "none", 0, AV_OPT_TYPE_CONST, { .i64 = DL_NONE }, 0, 0, FLAGS, "layout"},
106	108	{ "nchw", "nchw", 0, AV_OPT_TYPE_CONST, { .i64 = DL_NCHW }, 0, 0, FLAGS, "layout"},
107	109	{ "nhwc", "nhwc", 0, AV_OPT_TYPE_CONST, { .i64 = DL_NHWC }, 0, 0, FLAGS, "layout"},
	110	+ { "scale", "Add scale preprocess operation. Divide each element of input by specified value.", OFFSET(options.scale), AV_OPT_TYPE_FLOAT, { .dbl = 0 }, INT_MIN, INT_MAX, FLAGS},
	111	+ { "mean", "Add mean preprocess operation. Subtract specified value from each element of input.", OFFSET(options.mean), AV_OPT_TYPE_FLOAT, { .dbl = 0 }, INT_MIN, INT_MAX, FLAGS},
108	112	{ NULL }
109	113	};
110	114
@@ -209,6 +213,7 @@ static int fill_model_input_ov(OVModel ov_model, OVRequestItem request) · 98 unmodified lines
209	213	ie_blob_t *input_blob = NULL;
210	214	#endif
211	215
	216	+ memset(&input, 0, sizeof(input));
212	217	lltask = ff_queue_peek_front(ov_model->lltask_queue);
213	218	av_assert0(lltask);
214	219	task = lltask->task;
@@ -280,6 +285,9 @@ static int fill_model_input_ov(OVModel ov_model, OVRequestItem request) · 65 unmodified lines
280	285	// all models in openvino open model zoo use BGR as input,
281	286	// change to be an option when necessary.
282	287	input.order = DCO_BGR;
	288	+ // We use preprocess_steps to scale input data, so disable scale and mean here.
	289	+ input.scale = 1;
	290	+ input.mean = 0;
283	291
284	292	for (int i = 0; i < ctx->options.batch_size; ++i) {
285	293	lltask = ff_queue_pop_front(ov_model->lltask_queue);
@@ -350,6 +358,7 @@ static void infer_completion_callback(void *args) · 64 unmodified lines
350	358	ov_shape_t output_shape = {0};
351	359	ov_element_type_e precision;
352	360
	361	+ memset(&output, 0, sizeof(output));
353	362	status = ov_infer_request_get_output_tensor_by_index(request->infer_request, 0, &output_tensor);
354	363	if (status != OK) {
355	364	av_log(ctx, AV_LOG_ERROR,
@@ -418,6 +427,8 @@ static void infer_completion_callback(void *args) · 62 unmodified lines
418	427	#endif
419	428	output.dt = precision_to_datatype(precision);
420	429	output.layout = ctx->options.layout;
	430	+ output.scale = ctx->options.scale;
	431	+ output.mean = ctx->options.mean;
421	432
422	433	av_assert0(request->lltask_count >= 1);
423	434	for (int i = 0; i < request->lltask_count; ++i) {
@@ -561,7 +572,9 @@ static int init_model_ov(OVModel ov_model, const char input_name, const char * · 137 unmodified lines
561	572	ie_config_t config = {NULL, NULL, NULL};
562	573	char *all_dev_names = NULL;
563	574	#endif
564		−
	575	+ // We scale pixel by default when do frame processing.
	576	+ if (fabsf(ctx->options.scale) < 1e-6f)
	577	+ ctx->options.scale = ov_model->model->func_type == DFT_PROCESS_FRAME ? 255 : 1;
565	578	// batch size
566	579	if (ctx->options.batch_size <= 0) {
567	580	ctx->options.batch_size = 1;
@@ -628,15 +641,37 @@ static int init_model_ov(OVModel ov_model, const char input_name, const char * · 60 unmodified lines
628	641	goto err;
629	642	}
630	643
	644	+ status = ov_preprocess_input_tensor_info_set_element_type(input_tensor_info, U8);
631	645	if (ov_model->model->func_type != DFT_PROCESS_FRAME)
632		− //set precision only for detect and classify
633		− status = ov_preprocess_input_tensor_info_set_element_type(input_tensor_info, U8);
634		− status \|= ov_preprocess_output_set_element_type(output_tensor_info, F32);
	646	+ status \|= ov_preprocess_output_set_element_type(output_tensor_info, F32);
	647	+ else if (fabsf(ctx->options.scale - 1) > 1e-6f \|\| fabsf(ctx->options.mean) > 1e-6f)
	648	+ status \|= ov_preprocess_output_set_element_type(output_tensor_info, F32);
	649	+ else
	650	+ status \|= ov_preprocess_output_set_element_type(output_tensor_info, U8);
635	651	if (status != OK) {
636	652	av_log(ctx, AV_LOG_ERROR, "Failed to set input/output element type\n");
637	653	ret = ov2_map_error(status, NULL);
638	654	goto err;
639	655	}
	656	+ // set preprocess steps.
	657	+ if (fabsf(ctx->options.scale - 1) > 1e-6f \|\| fabsf(ctx->options.mean) > 1e-6f) {
	658	+ ov_preprocess_preprocess_steps_t* input_process_steps = NULL;
	659	+ status = ov_preprocess_input_info_get_preprocess_steps(ov_model->input_info, &input_process_steps);
	660	+ if (status != OK) {
	661	+ av_log(ctx, AV_LOG_ERROR, "Failed to get preprocess steps\n");
	662	+ ret = ov2_map_error(status, NULL);
	663	+ goto err;
	664	+ }
	665	+ status = ov_preprocess_preprocess_steps_convert_element_type(input_process_steps, F32);
	666	+ status \|= ov_preprocess_preprocess_steps_mean(input_process_steps, ctx->options.mean);
	667	+ status \|= ov_preprocess_preprocess_steps_scale(input_process_steps, ctx->options.scale);
	668	+ if (status != OK) {
	669	+ av_log(ctx, AV_LOG_ERROR, "Failed to set preprocess steps\n");
	670	+ ret = ov2_map_error(status, NULL);
	671	+ goto err;
	672	+ }
	673	+ ov_preprocess_preprocess_steps_free(input_process_steps);
	674	+ }
640	675
641	676	//update model
642	677	if(ov_model->ov_model)

libavfilter/dnn/dnn_io_proc.c+67−15

@@ -24,6 +24,20 @@ · 23 unmodified lines
24	24	#include "libavutil/avassert.h"
25	25	#include "libavutil/detection_bbox.h"
26	26
	27	+static int get_datatype_size(DNNDataType dt)
	28	+{
	29	+ switch (dt)
	30	+ {
	31	+ case DNN_FLOAT:
	32	+ return sizeof(float);
	33	+ case DNN_UINT8:
	34	+ return sizeof(uint8_t);
	35	+ default:
	36	+ av_assert0(!"not supported yet.");
	37	+ return 1;
	38	+ }
	39	+}
	40	+
27	41	int ff_proc_from_dnn_to_frame(AVFrame frame, DNNData output, void *log_ctx)
28	42	{
29	43	struct SwsContext *sws_ctx;
@@ -33,14 +47,26 @@ int ff_proc_from_dnn_to_frame(AVFrame frame, DNNData output, void *log_ctx) · 3 unmodified lines
33	47	void *middle_data = NULL;
34	48	uint8_t *planar_data[4] = { 0 };
35	49	int plane_size = frame->width * frame->height * sizeof(uint8_t);
	50	+ enum AVPixelFormat src_fmt = AV_PIX_FMT_NONE;
	51	+ int src_datatype_size = get_datatype_size(output->dt);
	52	+
36	53	int bytewidth = av_image_get_linesize(frame->format, frame->width, 0);
37	54	if (bytewidth < 0) {
38	55	return AVERROR(EINVAL);
39	56	}
40		− if (output->dt != DNN_FLOAT) {
41		− avpriv_report_missing_feature(log_ctx, "data type rather than DNN_FLOAT");
	57	+ /* scale == 1 and mean == 0 and dt == UINT8: passthrough */
	58	+ if (fabsf(output->scale - 1) < 1e-6f && fabsf(output->mean) < 1e-6 && output->dt == DNN_UINT8)
	59	+ src_fmt = AV_PIX_FMT_GRAY8;
	60	+ /* (scale == 255 or scale == 0) and mean == 0 and dt == FLOAT: normalization */
	61	+ else if ((fabsf(output->scale - 255) < 1e-6f \|\| fabsf(output->scale) < 1e-6f) &&
	62	+ fabsf(output->mean) < 1e-6 && output->dt == DNN_FLOAT)
	63	+ src_fmt = AV_PIX_FMT_GRAYF32;
	64	+ else {
	65	+ av_log(log_ctx, AV_LOG_ERROR, "dnn_process output data doesn't type: UINT8 "
	66	+ "scale: %f, mean: %f\n", output->scale, output->mean);
42	67	return AVERROR(ENOSYS);
43	68	}
	69	+
44	70	dst_data = (void **)frame->data;
45	71	linesize[0] = frame->linesize[0];
46	72	if (output->layout == DL_NCHW) {
@@ -58,7 +84,7 @@ int ff_proc_from_dnn_to_frame(AVFrame frame, DNNData output, void *log_ctx) · 11 unmodified lines
58	84	case AV_PIX_FMT_BGR24:
59	85	sws_ctx = sws_getContext(frame->width * 3,
60	86	frame->height,
61		− AV_PIX_FMT_GRAYF32,
	87	+ src_fmt,
62	88	frame->width * 3,
63	89	frame->height,
64	90	AV_PIX_FMT_GRAY8,
@@ -66,13 +92,13 @@ int ff_proc_from_dnn_to_frame(AVFrame frame, DNNData output, void *log_ctx) · 1 unmodified line
66	92	if (!sws_ctx) {
67	93	av_log(log_ctx, AV_LOG_ERROR, "Impossible to create scale context for the conversion "
68	94	"fmt:%s s:%dx%d -> fmt:%s s:%dx%d\n",
69		− av_get_pix_fmt_name(AV_PIX_FMT_GRAYF32), frame->width * 3, frame->height,
	95	+ av_get_pix_fmt_name(src_fmt), frame->width * 3, frame->height,
70	96	av_get_pix_fmt_name(AV_PIX_FMT_GRAY8), frame->width * 3, frame->height);
71	97	ret = AVERROR(EINVAL);
72	98	goto err;
73	99	}
74	100	sws_scale(sws_ctx, (const uint8_t [4]){(const uint8_t )output->data, 0, 0, 0},
75		− (const int[4]){frame->width * 3 * sizeof(float), 0, 0, 0}, 0, frame->height,
	101	+ (const int[4]){frame->width * 3 * src_datatype_size, 0, 0, 0}, 0, frame->height,
76	102	(uint8_t * const*)dst_data, linesize);
77	103	sws_freeContext(sws_ctx);
78	104	// convert data from planar to packed
@@ -131,13 +157,13 @@ int ff_proc_from_dnn_to_frame(AVFrame frame, DNNData output, void *log_ctx) · 52 unmodified lines
131	157	if (!sws_ctx) {
132	158	av_log(log_ctx, AV_LOG_ERROR, "Impossible to create scale context for the conversion "
133	159	"fmt:%s s:%dx%d -> fmt:%s s:%dx%d\n",
134		− av_get_pix_fmt_name(AV_PIX_FMT_GRAYF32), frame->width, frame->height,
	160	+ av_get_pix_fmt_name(src_fmt), frame->width, frame->height,
135	161	av_get_pix_fmt_name(AV_PIX_FMT_GRAY8), frame->width, frame->height);
136	162	ret = AVERROR(EINVAL);
137	163	goto err;
138	164	}
139	165	sws_scale(sws_ctx, (const uint8_t [4]){(const uint8_t )output->data, 0, 0, 0},
140		− (const int[4]){frame->width * sizeof(float), 0, 0, 0}, 0, frame->height,
	166	+ (const int[4]){frame->width * src_datatype_size, 0, 0, 0}, 0, frame->height,
141	167	(uint8_t * const*)frame->data, frame->linesize);
142	168	sws_freeContext(sws_ctx);
143	169	break;
@@ -161,12 +187,22 @@ int ff_proc_from_frame_to_dnn(AVFrame frame, DNNData input, void *log_ctx) · 17 unmodified lines
161	187	void *middle_data = NULL;
162	188	uint8_t *planar_data[4] = { 0 };
163	189	int plane_size = frame->width * frame->height * sizeof(uint8_t);
	190	+ enum AVPixelFormat dst_fmt = AV_PIX_FMT_NONE;
	191	+ int dst_datatype_size = get_datatype_size(input->dt);
164	192	int bytewidth = av_image_get_linesize(frame->format, frame->width, 0);
165	193	if (bytewidth < 0) {
166	194	return AVERROR(EINVAL);
167	195	}
168		− if (input->dt != DNN_FLOAT) {
169		− avpriv_report_missing_feature(log_ctx, "data type rather than DNN_FLOAT");
	196	+ /* scale == 1 and mean == 0 and dt == UINT8: passthrough */
	197	+ if (fabsf(input->scale - 1) < 1e-6f && fabsf(input->mean) < 1e-6 && input->dt == DNN_UINT8)
	198	+ dst_fmt = AV_PIX_FMT_GRAY8;
	199	+ /* (scale == 255 or scale == 0) and mean == 0 and dt == FLOAT: normalization */
	200	+ else if ((fabsf(input->scale - 255) < 1e-6f \|\| fabsf(input->scale) < 1e-6f) &&
	201	+ fabsf(input->mean) < 1e-6 && input->dt == DNN_FLOAT)
	202	+ dst_fmt = AV_PIX_FMT_GRAYF32;
	203	+ else {
	204	+ av_log(log_ctx, AV_LOG_ERROR, "dnn_process input data doesn't support type: UINT8 "
	205	+ "scale: %f, mean: %f\n", input->scale, input->mean);
170	206	return AVERROR(ENOSYS);
171	207	}
172	208
@@ -223,20 +259,20 @@ int ff_proc_from_frame_to_dnn(AVFrame frame, DNNData input, void *log_ctx) · 50 unmodified lines
223	259	AV_PIX_FMT_GRAY8,
224	260	frame->width * 3,
225	261	frame->height,
226		− AV_PIX_FMT_GRAYF32,
	262	+ dst_fmt,
227	263	0, NULL, NULL, NULL);
228	264	if (!sws_ctx) {
229	265	av_log(log_ctx, AV_LOG_ERROR, "Impossible to create scale context for the conversion "
230	266	"fmt:%s s:%dx%d -> fmt:%s s:%dx%d\n",
231	267	av_get_pix_fmt_name(AV_PIX_FMT_GRAY8), frame->width * 3, frame->height,
232		− av_get_pix_fmt_name(AV_PIX_FMT_GRAYF32),frame->width * 3, frame->height);
	268	+ av_get_pix_fmt_name(dst_fmt),frame->width * 3, frame->height);
233	269	ret = AVERROR(EINVAL);
234	270	goto err;
235	271	}
236	272	sws_scale(sws_ctx, (const uint8_t **)src_data,
237	273	linesize, 0, frame->height,
238	274	(uint8_t * const [4]){input->data, 0, 0, 0},
239		− (const int [4]){frame->width * 3 * sizeof(float), 0, 0, 0});
	275	+ (const int [4]){frame->width * 3 * dst_datatype_size, 0, 0, 0});
240	276	sws_freeContext(sws_ctx);
241	277	break;
242	278	case AV_PIX_FMT_GRAYF32:
@@ -256,20 +292,20 @@ int ff_proc_from_frame_to_dnn(AVFrame frame, DNNData input, void *log_ctx) · 13 unmodified lines
256	292	AV_PIX_FMT_GRAY8,
257	293	frame->width,
258	294	frame->height,
259		− AV_PIX_FMT_GRAYF32,
	295	+ dst_fmt,
260	296	0, NULL, NULL, NULL);
261	297	if (!sws_ctx) {
262	298	av_log(log_ctx, AV_LOG_ERROR, "Impossible to create scale context for the conversion "
263	299	"fmt:%s s:%dx%d -> fmt:%s s:%dx%d\n",
264	300	av_get_pix_fmt_name(AV_PIX_FMT_GRAY8), frame->width, frame->height,
265		− av_get_pix_fmt_name(AV_PIX_FMT_GRAYF32),frame->width, frame->height);
	301	+ av_get_pix_fmt_name(dst_fmt),frame->width, frame->height);
266	302	ret = AVERROR(EINVAL);
267	303	goto err;
268	304	}
269	305	sws_scale(sws_ctx, (const uint8_t **)frame->data,
270	306	frame->linesize, 0, frame->height,
271	307	(uint8_t * const [4]){input->data, 0, 0, 0},
272		− (const int [4]){frame->width * sizeof(float), 0, 0, 0});
	308	+ (const int [4]){frame->width * dst_datatype_size, 0, 0, 0});
273	309	sws_freeContext(sws_ctx);
274	310	break;
275	311	default:
@@ -315,6 +351,14 @@ int ff_frame_to_dnn_classify(AVFrame frame, DNNData input, uint32_t bbox_index · 39 unmodified lines
315	351	AVFrameSideData *sd = av_frame_get_side_data(frame, AV_FRAME_DATA_DETECTION_BBOXES);
316	352	av_assert0(sd);
317	353
	354	+ /* (scale != 1 and scale != 0) or mean != 0 */
	355	+ if ((fabsf(input->scale - 1) > 1e-6f && fabsf(input->scale) > 1e-6f) \|\|
	356	+ fabsf(input->mean) > 1e-6f) {
	357	+ av_log(log_ctx, AV_LOG_ERROR, "dnn_classify input data doesn't support "
	358	+ "scale: %f, mean: %f\n", input->scale, input->mean);
	359	+ return AVERROR(ENOSYS);
	360	+ }
	361	+
318	362	if (input->layout == DL_NCHW) {
319	363	av_log(log_ctx, AV_LOG_ERROR, "dnn_classify input data doesn't support layout: NCHW\n");
320	364	return AVERROR(ENOSYS);
@@ -373,6 +417,14 @@ int ff_frame_to_dnn_detect(AVFrame frame, DNNData input, void *log_ctx) · 52 unmodified lines
373	417	int ret = 0;
374	418	enum AVPixelFormat fmt = get_pixel_format(input);
375	419
	420	+ /* (scale != 1 and scale != 0) or mean != 0 */
	421	+ if ((fabsf(input->scale - 1) > 1e-6f && fabsf(input->scale) > 1e-6f) \|\|
	422	+ fabsf(input->mean) > 1e-6f) {
	423	+ av_log(log_ctx, AV_LOG_ERROR, "dnn_detect input data doesn't support "
	424	+ "scale: %f, mean: %f\n", input->scale, input->mean);
	425	+ return AVERROR(ENOSYS);
	426	+ }
	427	+
376	428	if (input->layout == DL_NCHW) {
377	429	av_log(log_ctx, AV_LOG_ERROR, "dnn_detect input data doesn't support layout: NCHW\n");
378	430	return AVERROR(ENOSYS);

libavfilter/dnn_interface.h+2−0

@@ -69,6 +69,8 @@ typedef struct DNNData{ · 68 unmodified lines
69	69	DNNDataType dt;
70	70	DNNColorOrder order;
71	71	DNNLayout layout;
	72	+ float scale;
	73	+ float mean;
72	74	} DNNData;
73	75
74	76	typedef struct DNNExecBaseParams {

libavfilter/dnn: Add scale and mean preprocess to openvino backend

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code