Dnn models has different data preprocess requirements. Scale and mean parameters are added to preprocess input data.
Showing the changes to libavfilter (3 of 3 files). Show all
Unified · Split
| @@ -46,6 +46,8 @@ typedef struct OVOptions{ · 45 unmodified lines | ||
| 46 | 46 | int batch_size; |
| 47 | 47 | int input_resizable; |
| 48 | 48 | DNNLayout layout; |
| 49 | + float scale; | |
| 50 | + float mean; | |
| 49 | 51 | } OVOptions; |
| 50 | 52 | |
| 51 | 53 | typedef struct OVContext { |
| @@ -105,6 +107,8 @@ static const AVOption dnn_openvino_options[] = { · 53 unmodified lines | ||
| 105 | 107 | { "none", "none", 0, AV_OPT_TYPE_CONST, { .i64 = DL_NONE }, 0, 0, FLAGS, "layout"}, |
| 106 | 108 | { "nchw", "nchw", 0, AV_OPT_TYPE_CONST, { .i64 = DL_NCHW }, 0, 0, FLAGS, "layout"}, |
| 107 | 109 | { "nhwc", "nhwc", 0, AV_OPT_TYPE_CONST, { .i64 = DL_NHWC }, 0, 0, FLAGS, "layout"}, |
| 110 | + { "scale", "Add scale preprocess operation. Divide each element of input by specified value.", OFFSET(options.scale), AV_OPT_TYPE_FLOAT, { .dbl = 0 }, INT_MIN, INT_MAX, FLAGS}, | |
| 111 | + { "mean", "Add mean preprocess operation. Subtract specified value from each element of input.", OFFSET(options.mean), AV_OPT_TYPE_FLOAT, { .dbl = 0 }, INT_MIN, INT_MAX, FLAGS}, | |
| 108 | 112 | { NULL } |
| 109 | 113 | }; |
| 110 | 114 | |
| @@ -209,6 +213,7 @@ static int fill_model_input_ov(OVModel *ov_model, OVRequestItem *request) · 98 unmodified lines | ||
| 209 | 213 | ie_blob_t *input_blob = NULL; |
| 210 | 214 | #endif |
| 211 | 215 | |
| 216 | + memset(&input, 0, sizeof(input)); | |
| 212 | 217 | lltask = ff_queue_peek_front(ov_model->lltask_queue); |
| 213 | 218 | av_assert0(lltask); |
| 214 | 219 | task = lltask->task; |
| @@ -280,6 +285,9 @@ static int fill_model_input_ov(OVModel *ov_model, OVRequestItem *request) · 65 unmodified lines | ||
| 280 | 285 | // all models in openvino open model zoo use BGR as input, |
| 281 | 286 | // change to be an option when necessary. |
| 282 | 287 | input.order = DCO_BGR; |
| 288 | + // We use preprocess_steps to scale input data, so disable scale and mean here. | |
| 289 | + input.scale = 1; | |
| 290 | + input.mean = 0; | |
| 283 | 291 | |
| 284 | 292 | for (int i = 0; i < ctx->options.batch_size; ++i) { |
| 285 | 293 | lltask = ff_queue_pop_front(ov_model->lltask_queue); |
| @@ -350,6 +358,7 @@ static void infer_completion_callback(void *args) · 64 unmodified lines | ||
| 350 | 358 | ov_shape_t output_shape = {0}; |
| 351 | 359 | ov_element_type_e precision; |
| 352 | 360 | |
| 361 | + memset(&output, 0, sizeof(output)); | |
| 353 | 362 | status = ov_infer_request_get_output_tensor_by_index(request->infer_request, 0, &output_tensor); |
| 354 | 363 | if (status != OK) { |
| 355 | 364 | av_log(ctx, AV_LOG_ERROR, |
| @@ -418,6 +427,8 @@ static void infer_completion_callback(void *args) · 62 unmodified lines | ||
| 418 | 427 | #endif |
| 419 | 428 | output.dt = precision_to_datatype(precision); |
| 420 | 429 | output.layout = ctx->options.layout; |
| 430 | + output.scale = ctx->options.scale; | |
| 431 | + output.mean = ctx->options.mean; | |
| 421 | 432 | |
| 422 | 433 | av_assert0(request->lltask_count >= 1); |
| 423 | 434 | for (int i = 0; i < request->lltask_count; ++i) { |
| @@ -561,7 +572,9 @@ static int init_model_ov(OVModel *ov_model, const char *input_name, const char * · 137 unmodified lines | ||
| 561 | 572 | ie_config_t config = {NULL, NULL, NULL}; |
| 562 | 573 | char *all_dev_names = NULL; |
| 563 | 574 | #endif |
| 564 | − | |
| 575 | + // We scale pixel by default when do frame processing. | |
| 576 | + if (fabsf(ctx->options.scale) < 1e-6f) | |
| 577 | + ctx->options.scale = ov_model->model->func_type == DFT_PROCESS_FRAME ? 255 : 1; | |
| 565 | 578 | // batch size |
| 566 | 579 | if (ctx->options.batch_size <= 0) { |
| 567 | 580 | ctx->options.batch_size = 1; |
| @@ -628,15 +641,37 @@ static int init_model_ov(OVModel *ov_model, const char *input_name, const char * · 60 unmodified lines | ||
| 628 | 641 | goto err; |
| 629 | 642 | } |
| 630 | 643 | |
| 644 | + status = ov_preprocess_input_tensor_info_set_element_type(input_tensor_info, U8); | |
| 631 | 645 | if (ov_model->model->func_type != DFT_PROCESS_FRAME) |
| 632 | − //set precision only for detect and classify | |
| 633 | − status = ov_preprocess_input_tensor_info_set_element_type(input_tensor_info, U8); | |
| 634 | − status |= ov_preprocess_output_set_element_type(output_tensor_info, F32); | |
| 646 | + status |= ov_preprocess_output_set_element_type(output_tensor_info, F32); | |
| 647 | + else if (fabsf(ctx->options.scale - 1) > 1e-6f || fabsf(ctx->options.mean) > 1e-6f) | |
| 648 | + status |= ov_preprocess_output_set_element_type(output_tensor_info, F32); | |
| 649 | + else | |
| 650 | + status |= ov_preprocess_output_set_element_type(output_tensor_info, U8); | |
| 635 | 651 | if (status != OK) { |
| 636 | 652 | av_log(ctx, AV_LOG_ERROR, "Failed to set input/output element type\n"); |
| 637 | 653 | ret = ov2_map_error(status, NULL); |
| 638 | 654 | goto err; |
| 639 | 655 | } |
| 656 | + // set preprocess steps. | |
| 657 | + if (fabsf(ctx->options.scale - 1) > 1e-6f || fabsf(ctx->options.mean) > 1e-6f) { | |
| 658 | + ov_preprocess_preprocess_steps_t* input_process_steps = NULL; | |
| 659 | + status = ov_preprocess_input_info_get_preprocess_steps(ov_model->input_info, &input_process_steps); | |
| 660 | + if (status != OK) { | |
| 661 | + av_log(ctx, AV_LOG_ERROR, "Failed to get preprocess steps\n"); | |
| 662 | + ret = ov2_map_error(status, NULL); | |
| 663 | + goto err; | |
| 664 | + } | |
| 665 | + status = ov_preprocess_preprocess_steps_convert_element_type(input_process_steps, F32); | |
| 666 | + status |= ov_preprocess_preprocess_steps_mean(input_process_steps, ctx->options.mean); | |
| 667 | + status |= ov_preprocess_preprocess_steps_scale(input_process_steps, ctx->options.scale); | |
| 668 | + if (status != OK) { | |
| 669 | + av_log(ctx, AV_LOG_ERROR, "Failed to set preprocess steps\n"); | |
| 670 | + ret = ov2_map_error(status, NULL); | |
| 671 | + goto err; | |
| 672 | + } | |
| 673 | + ov_preprocess_preprocess_steps_free(input_process_steps); | |
| 674 | + } | |
| 640 | 675 | |
| 641 | 676 | //update model |
| 642 | 677 | if(ov_model->ov_model) |
| @@ -24,6 +24,20 @@ · 23 unmodified lines | ||
| 24 | 24 | #include "libavutil/avassert.h" |
| 25 | 25 | #include "libavutil/detection_bbox.h" |
| 26 | 26 | |
| 27 | +static int get_datatype_size(DNNDataType dt) | |
| 28 | +{ | |
| 29 | + switch (dt) | |
| 30 | + { | |
| 31 | + case DNN_FLOAT: | |
| 32 | + return sizeof(float); | |
| 33 | + case DNN_UINT8: | |
| 34 | + return sizeof(uint8_t); | |
| 35 | + default: | |
| 36 | + av_assert0(!"not supported yet."); | |
| 37 | + return 1; | |
| 38 | + } | |
| 39 | +} | |
| 40 | + | |
| 27 | 41 | int ff_proc_from_dnn_to_frame(AVFrame *frame, DNNData *output, void *log_ctx) |
| 28 | 42 | { |
| 29 | 43 | struct SwsContext *sws_ctx; |
| @@ -33,14 +47,26 @@ int ff_proc_from_dnn_to_frame(AVFrame *frame, DNNData *output, void *log_ctx) · 3 unmodified lines | ||
| 33 | 47 | void *middle_data = NULL; |
| 34 | 48 | uint8_t *planar_data[4] = { 0 }; |
| 35 | 49 | int plane_size = frame->width * frame->height * sizeof(uint8_t); |
| 50 | + enum AVPixelFormat src_fmt = AV_PIX_FMT_NONE; | |
| 51 | + int src_datatype_size = get_datatype_size(output->dt); | |
| 52 | + | |
| 36 | 53 | int bytewidth = av_image_get_linesize(frame->format, frame->width, 0); |
| 37 | 54 | if (bytewidth < 0) { |
| 38 | 55 | return AVERROR(EINVAL); |
| 39 | 56 | } |
| 40 | − if (output->dt != DNN_FLOAT) { | |
| 41 | − avpriv_report_missing_feature(log_ctx, "data type rather than DNN_FLOAT"); | |
| 57 | + /* scale == 1 and mean == 0 and dt == UINT8: passthrough */ | |
| 58 | + if (fabsf(output->scale - 1) < 1e-6f && fabsf(output->mean) < 1e-6 && output->dt == DNN_UINT8) | |
| 59 | + src_fmt = AV_PIX_FMT_GRAY8; | |
| 60 | + /* (scale == 255 or scale == 0) and mean == 0 and dt == FLOAT: normalization */ | |
| 61 | + else if ((fabsf(output->scale - 255) < 1e-6f || fabsf(output->scale) < 1e-6f) && | |
| 62 | + fabsf(output->mean) < 1e-6 && output->dt == DNN_FLOAT) | |
| 63 | + src_fmt = AV_PIX_FMT_GRAYF32; | |
| 64 | + else { | |
| 65 | + av_log(log_ctx, AV_LOG_ERROR, "dnn_process output data doesn't type: UINT8 " | |
| 66 | + "scale: %f, mean: %f\n", output->scale, output->mean); | |
| 42 | 67 | return AVERROR(ENOSYS); |
| 43 | 68 | } |
| 69 | + | |
| 44 | 70 | dst_data = (void **)frame->data; |
| 45 | 71 | linesize[0] = frame->linesize[0]; |
| 46 | 72 | if (output->layout == DL_NCHW) { |
| @@ -58,7 +84,7 @@ int ff_proc_from_dnn_to_frame(AVFrame *frame, DNNData *output, void *log_ctx) · 11 unmodified lines | ||
| 58 | 84 | case AV_PIX_FMT_BGR24: |
| 59 | 85 | sws_ctx = sws_getContext(frame->width * 3, |
| 60 | 86 | frame->height, |
| 61 | − AV_PIX_FMT_GRAYF32, | |
| 87 | + src_fmt, | |
| 62 | 88 | frame->width * 3, |
| 63 | 89 | frame->height, |
| 64 | 90 | AV_PIX_FMT_GRAY8, |
| @@ -66,13 +92,13 @@ int ff_proc_from_dnn_to_frame(AVFrame *frame, DNNData *output, void *log_ctx) · 1 unmodified line | ||
| 66 | 92 | if (!sws_ctx) { |
| 67 | 93 | av_log(log_ctx, AV_LOG_ERROR, "Impossible to create scale context for the conversion " |
| 68 | 94 | "fmt:%s s:%dx%d -> fmt:%s s:%dx%d\n", |
| 69 | − av_get_pix_fmt_name(AV_PIX_FMT_GRAYF32), frame->width * 3, frame->height, | |
| 95 | + av_get_pix_fmt_name(src_fmt), frame->width * 3, frame->height, | |
| 70 | 96 | av_get_pix_fmt_name(AV_PIX_FMT_GRAY8), frame->width * 3, frame->height); |
| 71 | 97 | ret = AVERROR(EINVAL); |
| 72 | 98 | goto err; |
| 73 | 99 | } |
| 74 | 100 | sws_scale(sws_ctx, (const uint8_t *[4]){(const uint8_t *)output->data, 0, 0, 0}, |
| 75 | − (const int[4]){frame->width * 3 * sizeof(float), 0, 0, 0}, 0, frame->height, | |
| 101 | + (const int[4]){frame->width * 3 * src_datatype_size, 0, 0, 0}, 0, frame->height, | |
| 76 | 102 | (uint8_t * const*)dst_data, linesize); |
| 77 | 103 | sws_freeContext(sws_ctx); |
| 78 | 104 | // convert data from planar to packed |
| @@ -131,13 +157,13 @@ int ff_proc_from_dnn_to_frame(AVFrame *frame, DNNData *output, void *log_ctx) · 52 unmodified lines | ||
| 131 | 157 | if (!sws_ctx) { |
| 132 | 158 | av_log(log_ctx, AV_LOG_ERROR, "Impossible to create scale context for the conversion " |
| 133 | 159 | "fmt:%s s:%dx%d -> fmt:%s s:%dx%d\n", |
| 134 | − av_get_pix_fmt_name(AV_PIX_FMT_GRAYF32), frame->width, frame->height, | |
| 160 | + av_get_pix_fmt_name(src_fmt), frame->width, frame->height, | |
| 135 | 161 | av_get_pix_fmt_name(AV_PIX_FMT_GRAY8), frame->width, frame->height); |
| 136 | 162 | ret = AVERROR(EINVAL); |
| 137 | 163 | goto err; |
| 138 | 164 | } |
| 139 | 165 | sws_scale(sws_ctx, (const uint8_t *[4]){(const uint8_t *)output->data, 0, 0, 0}, |
| 140 | − (const int[4]){frame->width * sizeof(float), 0, 0, 0}, 0, frame->height, | |
| 166 | + (const int[4]){frame->width * src_datatype_size, 0, 0, 0}, 0, frame->height, | |
| 141 | 167 | (uint8_t * const*)frame->data, frame->linesize); |
| 142 | 168 | sws_freeContext(sws_ctx); |
| 143 | 169 | break; |
| @@ -161,12 +187,22 @@ int ff_proc_from_frame_to_dnn(AVFrame *frame, DNNData *input, void *log_ctx) · 17 unmodified lines | ||
| 161 | 187 | void *middle_data = NULL; |
| 162 | 188 | uint8_t *planar_data[4] = { 0 }; |
| 163 | 189 | int plane_size = frame->width * frame->height * sizeof(uint8_t); |
| 190 | + enum AVPixelFormat dst_fmt = AV_PIX_FMT_NONE; | |
| 191 | + int dst_datatype_size = get_datatype_size(input->dt); | |
| 164 | 192 | int bytewidth = av_image_get_linesize(frame->format, frame->width, 0); |
| 165 | 193 | if (bytewidth < 0) { |
| 166 | 194 | return AVERROR(EINVAL); |
| 167 | 195 | } |
| 168 | − if (input->dt != DNN_FLOAT) { | |
| 169 | − avpriv_report_missing_feature(log_ctx, "data type rather than DNN_FLOAT"); | |
| 196 | + /* scale == 1 and mean == 0 and dt == UINT8: passthrough */ | |
| 197 | + if (fabsf(input->scale - 1) < 1e-6f && fabsf(input->mean) < 1e-6 && input->dt == DNN_UINT8) | |
| 198 | + dst_fmt = AV_PIX_FMT_GRAY8; | |
| 199 | + /* (scale == 255 or scale == 0) and mean == 0 and dt == FLOAT: normalization */ | |
| 200 | + else if ((fabsf(input->scale - 255) < 1e-6f || fabsf(input->scale) < 1e-6f) && | |
| 201 | + fabsf(input->mean) < 1e-6 && input->dt == DNN_FLOAT) | |
| 202 | + dst_fmt = AV_PIX_FMT_GRAYF32; | |
| 203 | + else { | |
| 204 | + av_log(log_ctx, AV_LOG_ERROR, "dnn_process input data doesn't support type: UINT8 " | |
| 205 | + "scale: %f, mean: %f\n", input->scale, input->mean); | |
| 170 | 206 | return AVERROR(ENOSYS); |
| 171 | 207 | } |
| 172 | 208 | |
| @@ -223,20 +259,20 @@ int ff_proc_from_frame_to_dnn(AVFrame *frame, DNNData *input, void *log_ctx) · 50 unmodified lines | ||
| 223 | 259 | AV_PIX_FMT_GRAY8, |
| 224 | 260 | frame->width * 3, |
| 225 | 261 | frame->height, |
| 226 | − AV_PIX_FMT_GRAYF32, | |
| 262 | + dst_fmt, | |
| 227 | 263 | 0, NULL, NULL, NULL); |
| 228 | 264 | if (!sws_ctx) { |
| 229 | 265 | av_log(log_ctx, AV_LOG_ERROR, "Impossible to create scale context for the conversion " |
| 230 | 266 | "fmt:%s s:%dx%d -> fmt:%s s:%dx%d\n", |
| 231 | 267 | av_get_pix_fmt_name(AV_PIX_FMT_GRAY8), frame->width * 3, frame->height, |
| 232 | − av_get_pix_fmt_name(AV_PIX_FMT_GRAYF32),frame->width * 3, frame->height); | |
| 268 | + av_get_pix_fmt_name(dst_fmt),frame->width * 3, frame->height); | |
| 233 | 269 | ret = AVERROR(EINVAL); |
| 234 | 270 | goto err; |
| 235 | 271 | } |
| 236 | 272 | sws_scale(sws_ctx, (const uint8_t **)src_data, |
| 237 | 273 | linesize, 0, frame->height, |
| 238 | 274 | (uint8_t * const [4]){input->data, 0, 0, 0}, |
| 239 | − (const int [4]){frame->width * 3 * sizeof(float), 0, 0, 0}); | |
| 275 | + (const int [4]){frame->width * 3 * dst_datatype_size, 0, 0, 0}); | |
| 240 | 276 | sws_freeContext(sws_ctx); |
| 241 | 277 | break; |
| 242 | 278 | case AV_PIX_FMT_GRAYF32: |
| @@ -256,20 +292,20 @@ int ff_proc_from_frame_to_dnn(AVFrame *frame, DNNData *input, void *log_ctx) · 13 unmodified lines | ||
| 256 | 292 | AV_PIX_FMT_GRAY8, |
| 257 | 293 | frame->width, |
| 258 | 294 | frame->height, |
| 259 | − AV_PIX_FMT_GRAYF32, | |
| 295 | + dst_fmt, | |
| 260 | 296 | 0, NULL, NULL, NULL); |
| 261 | 297 | if (!sws_ctx) { |
| 262 | 298 | av_log(log_ctx, AV_LOG_ERROR, "Impossible to create scale context for the conversion " |
| 263 | 299 | "fmt:%s s:%dx%d -> fmt:%s s:%dx%d\n", |
| 264 | 300 | av_get_pix_fmt_name(AV_PIX_FMT_GRAY8), frame->width, frame->height, |
| 265 | − av_get_pix_fmt_name(AV_PIX_FMT_GRAYF32),frame->width, frame->height); | |
| 301 | + av_get_pix_fmt_name(dst_fmt),frame->width, frame->height); | |
| 266 | 302 | ret = AVERROR(EINVAL); |
| 267 | 303 | goto err; |
| 268 | 304 | } |
| 269 | 305 | sws_scale(sws_ctx, (const uint8_t **)frame->data, |
| 270 | 306 | frame->linesize, 0, frame->height, |
| 271 | 307 | (uint8_t * const [4]){input->data, 0, 0, 0}, |
| 272 | − (const int [4]){frame->width * sizeof(float), 0, 0, 0}); | |
| 308 | + (const int [4]){frame->width * dst_datatype_size, 0, 0, 0}); | |
| 273 | 309 | sws_freeContext(sws_ctx); |
| 274 | 310 | break; |
| 275 | 311 | default: |
| @@ -315,6 +351,14 @@ int ff_frame_to_dnn_classify(AVFrame *frame, DNNData *input, uint32_t bbox_index · 39 unmodified lines | ||
| 315 | 351 | AVFrameSideData *sd = av_frame_get_side_data(frame, AV_FRAME_DATA_DETECTION_BBOXES); |
| 316 | 352 | av_assert0(sd); |
| 317 | 353 | |
| 354 | + /* (scale != 1 and scale != 0) or mean != 0 */ | |
| 355 | + if ((fabsf(input->scale - 1) > 1e-6f && fabsf(input->scale) > 1e-6f) || | |
| 356 | + fabsf(input->mean) > 1e-6f) { | |
| 357 | + av_log(log_ctx, AV_LOG_ERROR, "dnn_classify input data doesn't support " | |
| 358 | + "scale: %f, mean: %f\n", input->scale, input->mean); | |
| 359 | + return AVERROR(ENOSYS); | |
| 360 | + } | |
| 361 | + | |
| 318 | 362 | if (input->layout == DL_NCHW) { |
| 319 | 363 | av_log(log_ctx, AV_LOG_ERROR, "dnn_classify input data doesn't support layout: NCHW\n"); |
| 320 | 364 | return AVERROR(ENOSYS); |
| @@ -373,6 +417,14 @@ int ff_frame_to_dnn_detect(AVFrame *frame, DNNData *input, void *log_ctx) · 52 unmodified lines | ||
| 373 | 417 | int ret = 0; |
| 374 | 418 | enum AVPixelFormat fmt = get_pixel_format(input); |
| 375 | 419 | |
| 420 | + /* (scale != 1 and scale != 0) or mean != 0 */ | |
| 421 | + if ((fabsf(input->scale - 1) > 1e-6f && fabsf(input->scale) > 1e-6f) || | |
| 422 | + fabsf(input->mean) > 1e-6f) { | |
| 423 | + av_log(log_ctx, AV_LOG_ERROR, "dnn_detect input data doesn't support " | |
| 424 | + "scale: %f, mean: %f\n", input->scale, input->mean); | |
| 425 | + return AVERROR(ENOSYS); | |
| 426 | + } | |
| 427 | + | |
| 376 | 428 | if (input->layout == DL_NCHW) { |
| 377 | 429 | av_log(log_ctx, AV_LOG_ERROR, "dnn_detect input data doesn't support layout: NCHW\n"); |
| 378 | 430 | return AVERROR(ENOSYS); |
| @@ -69,6 +69,8 @@ typedef struct DNNData{ · 68 unmodified lines | ||
| 69 | 69 | DNNDataType dt; |
| 70 | 70 | DNNColorOrder order; |
| 71 | 71 | DNNLayout layout; |
| 72 | + float scale; | |
| 73 | + float mean; | |
| 72 | 74 | } DNNData; |
| 73 | 75 | |
| 74 | 76 | typedef struct DNNExecBaseParams { |