History

libavfilter/dnn_interface: use dims to represent shapes

For detect and classify output, width and height make no sence, so change width, height to dims to represent the shape of tensor. Use layout and dims to get width, height and channel.

Wenbin Chen committed 2 years ago

Commit 3de38b91 parent c695de57 files · +146 −90Browse files

Showing the changes to libavfilter (7 of 7 files). Show all

Unified · Split

7 files
  1. libavfilter/dnn/dnn_backend_openvino.c +44 −36
  2. libavfilter/dnn/dnn_backend_tf.c +21 −11
  3. libavfilter/dnn/dnn_io_proc.c +21 −9
  4. libavfilter/dnn_interface.h +16 −1
  5. libavfilter/vf_dnn_classify.c +3 −3
  6. libavfilter/vf_dnn_detect.c +27 −23
  7. libavfilter/vf_dnn_processing.c +14 −7
libavfilter/dnn/dnn_backend_openvino.c+44−36
@@ -253,9 +253,9 @@ static int fill_model_input_ov(OVModel *ov_model, OVRequestItem *request) · 252 unmodified lines
253 ov_shape_free(&input_shape);253 ov_shape_free(&input_shape);
254 return ov2_map_error(status, NULL);254 return ov2_map_error(status, NULL);
255 }255 }
256 input.height = dims[1];256 for (int i = 0; i < input_shape.rank; i++)
257 input.width = dims[2];257 input.dims[i] = dims[i];
258 input.channels = dims[3];258 input.layout = DL_NHWC;
259 input.dt = precision_to_datatype(precision);259 input.dt = precision_to_datatype(precision);
260#else260#else
261 status = ie_infer_request_get_blob(request->infer_request, task->input_name, &input_blob);261 status = ie_infer_request_get_blob(request->infer_request, task->input_name, &input_blob);
@@ -278,9 +278,9 @@ static int fill_model_input_ov(OVModel *ov_model, OVRequestItem *request) · 16 unmodified lines
278 av_log(ctx, AV_LOG_ERROR, "Failed to get input blob buffer\n");278 av_log(ctx, AV_LOG_ERROR, "Failed to get input blob buffer\n");
279 return DNN_GENERIC_ERROR;279 return DNN_GENERIC_ERROR;
280 }280 }
281 input.height = dims.dims[2];281 for (int i = 0; i < input_shape.rank; i++)
282 input.width = dims.dims[3];282 input.dims[i] = dims[i];
283 input.channels = dims.dims[1];283 input.layout = DL_NCHW;
284 input.data = blob_buffer.buffer;284 input.data = blob_buffer.buffer;
285 input.dt = precision_to_datatype(precision);285 input.dt = precision_to_datatype(precision);
286#endif286#endif
@@ -339,8 +339,8 @@ static int fill_model_input_ov(OVModel *ov_model, OVRequestItem *request) · 52 unmodified lines
339 av_assert0(!"should not reach here");339 av_assert0(!"should not reach here");
340 break;340 break;
341 }341 }
342 input.data = (uint8_t *)input.data342 input.data = (uint8_t *)input.data +
343 + input.width * input.height * input.channels * get_datatype_size(input.dt);343 input.dims[1] * input.dims[2] * input.dims[3] * get_datatype_size(input.dt);
344 }344 }
345#if HAVE_OPENVINO2345#if HAVE_OPENVINO2
346 ov_tensor_free(tensor);346 ov_tensor_free(tensor);
@@ -403,10 +403,11 @@ static void infer_completion_callback(void *args) · 56 unmodified lines
403 goto end;403 goto end;
404 }404 }
405 outputs[i].dt = precision_to_datatype(precision);405 outputs[i].dt = precision_to_datatype(precision);
406 406 outputs[i].layout = DL_NCHW;
407 outputs[i].channels = output_shape.rank > 2 ? dims[output_shape.rank - 3] : 1;407 outputs[i].dims[0] = 1;
408 outputs[i].height = output_shape.rank > 1 ? dims[output_shape.rank - 2] : 1;408 outputs[i].dims[1] = output_shape.rank > 2 ? dims[output_shape.rank - 3] : 1;
409 outputs[i].width = output_shape.rank > 0 ? dims[output_shape.rank - 1] : 1;409 outputs[i].dims[2] = output_shape.rank > 1 ? dims[output_shape.rank - 2] : 1;
410 outputs[i].dims[3] = output_shape.rank > 0 ? dims[output_shape.rank - 1] : 1;
410 av_assert0(request->lltask_count <= dims[0]);411 av_assert0(request->lltask_count <= dims[0]);
411 outputs[i].layout = ctx->options.layout;412 outputs[i].layout = ctx->options.layout;
412 outputs[i].scale = ctx->options.scale;413 outputs[i].scale = ctx->options.scale;
@@ -445,9 +446,9 @@ static void infer_completion_callback(void *args) · 32 unmodified lines
445 return;446 return;
446 }447 }
447 output.data = blob_buffer.buffer;448 output.data = blob_buffer.buffer;
448 output.channels = dims.dims[1];449 output.layout = DL_NCHW;
449 output.height = dims.dims[2];450 for (int i = 0; i < 4; i++)
450 output.width = dims.dims[3];451 output.dims[i] = dims.dims[i];
451 av_assert0(request->lltask_count <= dims.dims[0]);452 av_assert0(request->lltask_count <= dims.dims[0]);
452 output.dt = precision_to_datatype(precision);453 output.dt = precision_to_datatype(precision);
453 output.layout = ctx->options.layout;454 output.layout = ctx->options.layout;
@@ -469,8 +470,10 @@ static void infer_completion_callback(void *args) · 15 unmodified lines
469 ff_proc_from_dnn_to_frame(task->out_frame, outputs, ctx);470 ff_proc_from_dnn_to_frame(task->out_frame, outputs, ctx);
470 }471 }
471 } else {472 } else {
472 task->out_frame->width = outputs[0].width;473 task->out_frame->width =
473 task->out_frame->height = outputs[0].height;474 outputs[0].dims[dnn_get_width_idx_by_layout(outputs[0].layout)];
475 task->out_frame->height =
476 outputs[0].dims[dnn_get_height_idx_by_layout(outputs[0].layout)];
474 }477 }
475 break;478 break;
476 case DFT_ANALYTICS_DETECT:479 case DFT_ANALYTICS_DETECT:
@@ -501,7 +504,8 @@ static void infer_completion_callback(void *args) · 24 unmodified lines
501 av_freep(&request->lltasks[i]);504 av_freep(&request->lltasks[i]);
502 for (int i = 0; i < ov_model->nb_outputs; i++)505 for (int i = 0; i < ov_model->nb_outputs; i++)
503 outputs[i].data = (uint8_t *)outputs[i].data +506 outputs[i].data = (uint8_t *)outputs[i].data +
504 outputs[i].width * outputs[i].height * outputs[i].channels * get_datatype_size(outputs[i].dt);507 outputs[i].dims[1] * outputs[i].dims[2] * outputs[i].dims[3] *
508 get_datatype_size(outputs[i].dt);
505 }509 }
506end:510end:
507#if HAVE_OPENVINO2511#if HAVE_OPENVINO2
@@ -1085,7 +1089,6 @@ static int get_input_ov(void *model, DNNData *input, const char *input_name) · 577 unmodified lines
1085#if HAVE_OPENVINO21089#if HAVE_OPENVINO2
1086 ov_shape_t input_shape = {0};1090 ov_shape_t input_shape = {0};
1087 ov_element_type_e precision;1091 ov_element_type_e precision;
1088 int64_t* dims;
1089 ov_status_e status;1092 ov_status_e status;
1090 if (input_name)1093 if (input_name)
1091 status = ov_model_const_input_by_name(ov_model->ov_model, input_name, &ov_model->input_port);1094 status = ov_model_const_input_by_name(ov_model->ov_model, input_name, &ov_model->input_port);
@@ -1105,16 +1108,18 @@ static int get_input_ov(void *model, DNNData *input, const char *input_name) · 13 unmodified lines
1105 av_log(ctx, AV_LOG_ERROR, "Failed to get input port shape.\n");1108 av_log(ctx, AV_LOG_ERROR, "Failed to get input port shape.\n");
1106 return ov2_map_error(status, NULL);1109 return ov2_map_error(status, NULL);
1107 }1110 }
1108 dims = input_shape.dims;1111 for (int i = 0; i < 4; i++)
1109 if (dims[1] <= 3) { // NCHW1112 input->dims[i] = input_shape.dims[i];
1110 input->channels = dims[1];1113 if (input_resizable) {
1111 input->height = input_resizable ? -1 : dims[2];1114 input->dims[dnn_get_width_idx_by_layout(input->layout)] = -1;
1112 input->width = input_resizable ? -1 : dims[3];1115 input->dims[dnn_get_height_idx_by_layout(input->layout)] = -1;
1113 } else { // NHWC
1114 input->height = input_resizable ? -1 : dims[1];
1115 input->width = input_resizable ? -1 : dims[2];
1116 input->channels = dims[3];
1117 }1116 }
1117
1118 if (input_shape.dims[1] <= 3) // NCHW
1119 input->layout = DL_NCHW;
1120 else // NHWC
1121 input->layout = DL_NHWC;
1122
1118 input->dt = precision_to_datatype(precision);1123 input->dt = precision_to_datatype(precision);
1119 ov_shape_free(&input_shape);1124 ov_shape_free(&input_shape);
1120 return 0;1125 return 0;
@@ -1144,15 +1149,18 @@ static int get_input_ov(void *model, DNNData *input, const char *input_name) · 23 unmodified lines
1144 return DNN_GENERIC_ERROR;1149 return DNN_GENERIC_ERROR;
1145 }1150 }
1146 1151
1147 if (dims[1] <= 3) { // NCHW1152 for (int i = 0; i < 4; i++)
1148 input->channels = dims[1];1153 input->dims[i] = input_shape.dims[i];
1149 input->height = input_resizable ? -1 : dims[2];1154 if (input_resizable) {
1150 input->width = input_resizable ? -1 : dims[3];1155 input->dims[dnn_get_width_idx_by_layout(input->layout)] = -1;
1151 } else { // NHWC1156 input->dims[dnn_get_height_idx_by_layout(input->layout)] = -1;
1152 input->height = input_resizable ? -1 : dims[1];
1153 input->width = input_resizable ? -1 : dims[2];
1154 input->channels = dims[3];
1155 }1157 }
1158
1159 if (input_shape.dims[1] <= 3) // NCHW
1160 input->layout = DL_NCHW;
1161 else // NHWC
1162 input->layout = DL_NHWC;
1163
1156 input->dt = precision_to_datatype(precision);1164 input->dt = precision_to_datatype(precision);
1157 return 0;1165 return 0;
1158 }1166 }
libavfilter/dnn/dnn_backend_tf.c+21−11
@@ -251,7 +251,12 @@ static TF_Tensor *allocate_input_tensor(const DNNData *input) · 250 unmodified lines
251{251{
252 TF_DataType dt;252 TF_DataType dt;
253 size_t size;253 size_t size;
254 int64_t input_dims[] = {1, input->height, input->width, input->channels};254 int64_t input_dims[4] = { 0 };
255
256 input_dims[0] = 1;
257 input_dims[1] = input->dims[dnn_get_height_idx_by_layout(input->layout)];
258 input_dims[2] = input->dims[dnn_get_width_idx_by_layout(input->layout)];
259 input_dims[3] = input->dims[dnn_get_channel_idx_by_layout(input->layout)];
255 switch (input->dt) {260 switch (input->dt) {
256 case DNN_FLOAT:261 case DNN_FLOAT:
257 dt = TF_FLOAT;262 dt = TF_FLOAT;
@@ -310,9 +315,9 @@ static int get_input_tf(void *model, DNNData *input, const char *input_name) · 52 unmodified lines
310 315
311 // currently only NHWC is supported316 // currently only NHWC is supported
312 av_assert0(dims[0] == 1 || dims[0] == -1);317 av_assert0(dims[0] == 1 || dims[0] == -1);
313 input->height = dims[1];318 for (int i = 0; i < 4; i++)
314 input->width = dims[2];319 input->dims[i] = dims[i];
315 input->channels = dims[3];320 input->layout = DL_NHWC;
316 321
317 return 0;322 return 0;
318}323}
@@ -640,8 +645,8 @@ static int fill_model_input_tf(TFModel *tf_model, TFRequestItem *request) { · 321 unmodified lines
640 }645 }
641 646
642 infer_request = request->infer_request;647 infer_request = request->infer_request;
643 input.height = task->in_frame->height;648 input.dims[1] = task->in_frame->height;
644 input.width = task->in_frame->width;649 input.dims[2] = task->in_frame->width;
645 650
646 infer_request->tf_input = av_malloc(sizeof(TF_Output));651 infer_request->tf_input = av_malloc(sizeof(TF_Output));
647 if (!infer_request->tf_input) {652 if (!infer_request->tf_input) {
@@ -731,9 +736,12 @@ static void infer_completion_callback(void *args) { · 83 unmodified lines
731 }736 }
732 737
733 for (uint32_t i = 0; i < task->nb_output; ++i) {738 for (uint32_t i = 0; i < task->nb_output; ++i) {
734 outputs[i].height = TF_Dim(infer_request->output_tensors[i], 1);739 outputs[i].dims[dnn_get_height_idx_by_layout(outputs[i].layout)] =
735 outputs[i].width = TF_Dim(infer_request->output_tensors[i], 2);740 TF_Dim(infer_request->output_tensors[i], 1);
736 outputs[i].channels = TF_Dim(infer_request->output_tensors[i], 3);741 outputs[i].dims[dnn_get_width_idx_by_layout(outputs[i].layout)] =
742 TF_Dim(infer_request->output_tensors[i], 2);
743 outputs[i].dims[dnn_get_channel_idx_by_layout(outputs[i].layout)] =
744 TF_Dim(infer_request->output_tensors[i], 3);
737 outputs[i].data = TF_TensorData(infer_request->output_tensors[i]);745 outputs[i].data = TF_TensorData(infer_request->output_tensors[i]);
738 outputs[i].dt = (DNNDataType)TF_TensorType(infer_request->output_tensors[i]);746 outputs[i].dt = (DNNDataType)TF_TensorType(infer_request->output_tensors[i]);
739 }747 }
@@ -747,8 +755,10 @@ static void infer_completion_callback(void *args) { · 7 unmodified lines
747 ff_proc_from_dnn_to_frame(task->out_frame, outputs, ctx);755 ff_proc_from_dnn_to_frame(task->out_frame, outputs, ctx);
748 }756 }
749 } else {757 } else {
750 task->out_frame->width = outputs[0].width;758 task->out_frame->width =
751 task->out_frame->height = outputs[0].height;759 outputs[0].dims[dnn_get_width_idx_by_layout(outputs[0].layout)];
760 task->out_frame->height =
761 outputs[0].dims[dnn_get_height_idx_by_layout(outputs[0].layout)];
752 }762 }
753 break;763 break;
754 case DFT_ANALYTICS_DETECT:764 case DFT_ANALYTICS_DETECT:
libavfilter/dnn/dnn_io_proc.c+21−9
@@ -70,7 +70,7 @@ int ff_proc_from_dnn_to_frame(AVFrame *frame, DNNData *output, void *log_ctx) · 69 unmodified lines
70 dst_data = (void **)frame->data;70 dst_data = (void **)frame->data;
71 linesize[0] = frame->linesize[0];71 linesize[0] = frame->linesize[0];
72 if (output->layout == DL_NCHW) {72 if (output->layout == DL_NCHW) {
73 middle_data = av_malloc(plane_size * output->channels);73 middle_data = av_malloc(plane_size * output->dims[1]);
74 if (!middle_data) {74 if (!middle_data) {
75 ret = AVERROR(ENOMEM);75 ret = AVERROR(ENOMEM);
76 goto err;76 goto err;
@@ -209,7 +209,7 @@ int ff_proc_from_frame_to_dnn(AVFrame *frame, DNNData *input, void *log_ctx) · 132 unmodified lines
209 src_data = (void **)frame->data;209 src_data = (void **)frame->data;
210 linesize[0] = frame->linesize[0];210 linesize[0] = frame->linesize[0];
211 if (input->layout == DL_NCHW) {211 if (input->layout == DL_NCHW) {
212 middle_data = av_malloc(plane_size * input->channels);212 middle_data = av_malloc(plane_size * input->dims[1]);
213 if (!middle_data) {213 if (!middle_data) {
214 ret = AVERROR(ENOMEM);214 ret = AVERROR(ENOMEM);
215 goto err;215 goto err;
@@ -346,6 +346,7 @@ int ff_frame_to_dnn_classify(AVFrame *frame, DNNData *input, uint32_t bbox_index · 130 unmodified lines
346 int ret = 0;346 int ret = 0;
347 enum AVPixelFormat fmt;347 enum AVPixelFormat fmt;
348 int left, top, width, height;348 int left, top, width, height;
349 int width_idx, height_idx;
349 const AVDetectionBBoxHeader *header;350 const AVDetectionBBoxHeader *header;
350 const AVDetectionBBox *bbox;351 const AVDetectionBBox *bbox;
351 AVFrameSideData *sd = av_frame_get_side_data(frame, AV_FRAME_DATA_DETECTION_BBOXES);352 AVFrameSideData *sd = av_frame_get_side_data(frame, AV_FRAME_DATA_DETECTION_BBOXES);
@@ -364,6 +365,9 @@ int ff_frame_to_dnn_classify(AVFrame *frame, DNNData *input, uint32_t bbox_index · 12 unmodified lines
364 return AVERROR(ENOSYS);365 return AVERROR(ENOSYS);
365 }366 }
366 367
368 width_idx = dnn_get_width_idx_by_layout(input->layout);
369 height_idx = dnn_get_height_idx_by_layout(input->layout);
370
367 header = (const AVDetectionBBoxHeader *)sd->data;371 header = (const AVDetectionBBoxHeader *)sd->data;
368 bbox = av_get_detection_bbox(header, bbox_index);372 bbox = av_get_detection_bbox(header, bbox_index);
369 373
@@ -374,17 +378,20 @@ int ff_frame_to_dnn_classify(AVFrame *frame, DNNData *input, uint32_t bbox_index · 4 unmodified lines
374 378
375 fmt = get_pixel_format(input);379 fmt = get_pixel_format(input);
376 sws_ctx = sws_getContext(width, height, frame->format,380 sws_ctx = sws_getContext(width, height, frame->format,
377 input->width, input->height, fmt,381 input->dims[width_idx],
382 input->dims[height_idx], fmt,
378 SWS_FAST_BILINEAR, NULL, NULL, NULL);383 SWS_FAST_BILINEAR, NULL, NULL, NULL);
379 if (!sws_ctx) {384 if (!sws_ctx) {
380 av_log(log_ctx, AV_LOG_ERROR, "Failed to create scale context for the conversion "385 av_log(log_ctx, AV_LOG_ERROR, "Failed to create scale context for the conversion "
381 "fmt:%s s:%dx%d -> fmt:%s s:%dx%d\n",386 "fmt:%s s:%dx%d -> fmt:%s s:%dx%d\n",
382 av_get_pix_fmt_name(frame->format), width, height,387 av_get_pix_fmt_name(frame->format), width, height,
383 av_get_pix_fmt_name(fmt), input->width, input->height);388 av_get_pix_fmt_name(fmt),
389 input->dims[width_idx],
390 input->dims[height_idx]);
384 return AVERROR(EINVAL);391 return AVERROR(EINVAL);
385 }392 }
386 393
387 ret = av_image_fill_linesizes(linesizes, fmt, input->width);394 ret = av_image_fill_linesizes(linesizes, fmt, input->dims[width_idx]);
388 if (ret < 0) {395 if (ret < 0) {
389 av_log(log_ctx, AV_LOG_ERROR, "unable to get linesizes with av_image_fill_linesizes");396 av_log(log_ctx, AV_LOG_ERROR, "unable to get linesizes with av_image_fill_linesizes");
390 sws_freeContext(sws_ctx);397 sws_freeContext(sws_ctx);
@@ -414,7 +421,7 @@ int ff_frame_to_dnn_detect(AVFrame *frame, DNNData *input, void *log_ctx) · 23 unmodified lines
414{421{
415 struct SwsContext *sws_ctx;422 struct SwsContext *sws_ctx;
416 int linesizes[4];423 int linesizes[4];
417 int ret = 0;424 int ret = 0, width_idx, height_idx;
418 enum AVPixelFormat fmt = get_pixel_format(input);425 enum AVPixelFormat fmt = get_pixel_format(input);
419 426
420 /* (scale != 1 and scale != 0) or mean != 0 */427 /* (scale != 1 and scale != 0) or mean != 0 */
@@ -430,18 +437,23 @@ int ff_frame_to_dnn_detect(AVFrame *frame, DNNData *input, void *log_ctx) · 9 unmodified lines
430 return AVERROR(ENOSYS);437 return AVERROR(ENOSYS);
431 }438 }
432 439
440 width_idx = dnn_get_width_idx_by_layout(input->layout);
441 height_idx = dnn_get_height_idx_by_layout(input->layout);
442
433 sws_ctx = sws_getContext(frame->width, frame->height, frame->format,443 sws_ctx = sws_getContext(frame->width, frame->height, frame->format,
434 input->width, input->height, fmt,444 input->dims[width_idx],
445 input->dims[height_idx], fmt,
435 SWS_FAST_BILINEAR, NULL, NULL, NULL);446 SWS_FAST_BILINEAR, NULL, NULL, NULL);
436 if (!sws_ctx) {447 if (!sws_ctx) {
437 av_log(log_ctx, AV_LOG_ERROR, "Impossible to create scale context for the conversion "448 av_log(log_ctx, AV_LOG_ERROR, "Impossible to create scale context for the conversion "
438 "fmt:%s s:%dx%d -> fmt:%s s:%dx%d\n",449 "fmt:%s s:%dx%d -> fmt:%s s:%dx%d\n",
439 av_get_pix_fmt_name(frame->format), frame->width, frame->height,450 av_get_pix_fmt_name(frame->format), frame->width, frame->height,
440 av_get_pix_fmt_name(fmt), input->width, input->height);451 av_get_pix_fmt_name(fmt), input->dims[width_idx],
452 input->dims[height_idx]);
441 return AVERROR(EINVAL);453 return AVERROR(EINVAL);
442 }454 }
443 455
444 ret = av_image_fill_linesizes(linesizes, fmt, input->width);456 ret = av_image_fill_linesizes(linesizes, fmt, input->dims[width_idx]);
445 if (ret < 0) {457 if (ret < 0) {
446 av_log(log_ctx, AV_LOG_ERROR, "unable to get linesizes with av_image_fill_linesizes");458 av_log(log_ctx, AV_LOG_ERROR, "unable to get linesizes with av_image_fill_linesizes");
447 sws_freeContext(sws_ctx);459 sws_freeContext(sws_ctx);
libavfilter/dnn_interface.h+16−1
@@ -64,7 +64,7 @@ typedef enum { · 63 unmodified lines
64 64
65typedef struct DNNData{65typedef struct DNNData{
66 void *data;66 void *data;
67 int width, height, channels;67 int dims[4];
68 // dt and order together decide the color format68 // dt and order together decide the color format
69 DNNDataType dt;69 DNNDataType dt;
70 DNNColorOrder order;70 DNNColorOrder order;
@@ -134,4 +134,19 @@ typedef struct DNNModule{ · 63 unmodified lines
134// Initializes DNNModule depending on chosen backend.134// Initializes DNNModule depending on chosen backend.
135const DNNModule *ff_get_dnn_module(DNNBackendType backend_type, void *log_ctx);135const DNNModule *ff_get_dnn_module(DNNBackendType backend_type, void *log_ctx);
136 136
137static inline int dnn_get_width_idx_by_layout(DNNLayout layout)
138{
139 return layout == DL_NHWC ? 2 : 3;
140}
141
142static inline int dnn_get_height_idx_by_layout(DNNLayout layout)
143{
144 return layout == DL_NHWC ? 1 : 2;
145}
146
147static inline int dnn_get_channel_idx_by_layout(DNNLayout layout)
148{
149 return layout == DL_NHWC ? 3 : 1;
150}
151
137#endif152#endif
libavfilter/vf_dnn_classify.c+3−3
@@ -68,8 +68,8 @@ static int dnn_classify_post_proc(AVFrame *frame, DNNData *output, uint32_t bbox · 67 unmodified lines
68 uint32_t label_id;68 uint32_t label_id;
69 float confidence;69 float confidence;
70 AVFrameSideData *sd;70 AVFrameSideData *sd;
71 71 int output_size = output->dims[3] * output->dims[2] * output->dims[1];
72 if (output->channels <= 0) {72 if (output_size <= 0) {
73 return -1;73 return -1;
74 }74 }
75 75
@@ -88,7 +88,7 @@ static int dnn_classify_post_proc(AVFrame *frame, DNNData *output, uint32_t bbox · 12 unmodified lines
88 classifications = output->data;88 classifications = output->data;
89 label_id = 0;89 label_id = 0;
90 confidence= classifications[0];90 confidence= classifications[0];
91 for (int i = 1; i < output->channels; i++) {91 for (int i = 1; i < output_size; i++) {
92 if (classifications[i] > confidence) {92 if (classifications[i] > confidence) {
93 label_id = i;93 label_id = i;
94 confidence= classifications[i];94 confidence= classifications[i];
libavfilter/vf_dnn_detect.c+27−23
@@ -166,14 +166,14 @@ static int dnn_detect_parse_yolo_output(AVFrame *frame, DNNData *output, int out · 165 unmodified lines
166 scale_w = cell_w;166 scale_w = cell_w;
167 scale_h = cell_h;167 scale_h = cell_h;
168 } else {168 } else {
169 if (output[output_index].height != output[output_index].width &&169 if (output[output_index].dims[2] != output[output_index].dims[3] &&
170 output[output_index].height == output[output_index].channels) {170 output[output_index].dims[2] == output[output_index].dims[1]) {
171 is_NHWC = 1;171 is_NHWC = 1;
172 cell_w = output[output_index].height;172 cell_w = output[output_index].dims[2];
173 cell_h = output[output_index].channels;173 cell_h = output[output_index].dims[1];
174 } else {174 } else {
175 cell_w = output[output_index].width;175 cell_w = output[output_index].dims[3];
176 cell_h = output[output_index].height;176 cell_h = output[output_index].dims[2];
177 }177 }
178 scale_w = ctx->scale_width;178 scale_w = ctx->scale_width;
179 scale_h = ctx->scale_height;179 scale_h = ctx->scale_height;
@@ -205,14 +205,14 @@ static int dnn_detect_parse_yolo_output(AVFrame *frame, DNNData *output, int out · 25 unmodified lines
205 return AVERROR(EINVAL);205 return AVERROR(EINVAL);
206 }206 }
207 207
208 if (output[output_index].channels * output[output_index].width *208 if (output[output_index].dims[1] * output[output_index].dims[2] *
209 output[output_index].height % (box_size * cell_w * cell_h)) {209 output[output_index].dims[3] % (box_size * cell_w * cell_h)) {
210 av_log(filter_ctx, AV_LOG_ERROR, "wrong cell_w, cell_h or nb_classes\n");210 av_log(filter_ctx, AV_LOG_ERROR, "wrong cell_w, cell_h or nb_classes\n");
211 return AVERROR(EINVAL);211 return AVERROR(EINVAL);
212 }212 }
213 detection_boxes = output[output_index].channels *213 detection_boxes = output[output_index].dims[1] *
214 output[output_index].height *214 output[output_index].dims[2] *
215 output[output_index].width / box_size / cell_w / cell_h;215 output[output_index].dims[3] / box_size / cell_w / cell_h;
216 216
217 anchors = anchors + (detection_boxes * output_index * 2);217 anchors = anchors + (detection_boxes * output_index * 2);
218 /**218 /**
@@ -373,18 +373,18 @@ static int dnn_detect_post_proc_ssd(AVFrame *frame, DNNData *output, int nb_outp · 154 unmodified lines
373 int scale_w = ctx->scale_width;373 int scale_w = ctx->scale_width;
374 int scale_h = ctx->scale_height;374 int scale_h = ctx->scale_height;
375 375
376 if (nb_outputs == 1 && output->width == 7) {376 if (nb_outputs == 1 && output->dims[3] == 7) {
377 proposal_count = output->height;377 proposal_count = output->dims[2];
378 detect_size = output->width;378 detect_size = output->dims[3];
379 detections = output->data;379 detections = output->data;
380 } else if (nb_outputs == 2 && output[0].width == 5) {380 } else if (nb_outputs == 2 && output[0].dims[3] == 5) {
381 proposal_count = output[0].height;381 proposal_count = output[0].dims[2];
382 detect_size = output[0].width;382 detect_size = output[0].dims[3];
383 detections = output[0].data;383 detections = output[0].data;
384 labels = output[1].data;384 labels = output[1].data;
385 } else if (nb_outputs == 2 && output[1].width == 5) {385 } else if (nb_outputs == 2 && output[1].dims[3] == 5) {
386 proposal_count = output[1].height;386 proposal_count = output[1].dims[2];
387 detect_size = output[1].width;387 detect_size = output[1].dims[3];
388 detections = output[1].data;388 detections = output[1].data;
389 labels = output[0].data;389 labels = output[0].data;
390 } else {390 } else {
@@ -821,15 +821,19 @@ static int config_input(AVFilterLink *inlink) · 430 unmodified lines
821 AVFilterContext *context = inlink->dst;821 AVFilterContext *context = inlink->dst;
822 DnnDetectContext *ctx = context->priv;822 DnnDetectContext *ctx = context->priv;
823 DNNData model_input;823 DNNData model_input;
824 int ret;824 int ret, width_idx, height_idx;
825 825
826 ret = ff_dnn_get_input(&ctx->dnnctx, &model_input);826 ret = ff_dnn_get_input(&ctx->dnnctx, &model_input);
827 if (ret != 0) {827 if (ret != 0) {
828 av_log(ctx, AV_LOG_ERROR, "could not get input from the model\n");828 av_log(ctx, AV_LOG_ERROR, "could not get input from the model\n");
829 return ret;829 return ret;
830 }830 }
831 ctx->scale_width = model_input.width == -1 ? inlink->w : model_input.width;831 width_idx = dnn_get_width_idx_by_layout(model_input.layout);
832 ctx->scale_height = model_input.height == -1 ? inlink->h : model_input.height;832 height_idx = dnn_get_height_idx_by_layout(model_input.layout);
833 ctx->scale_width = model_input.dims[width_idx] == -1 ? inlink->w :
834 model_input.dims[width_idx];
835 ctx->scale_height = model_input.dims[height_idx] == -1 ? inlink->h :
836 model_input.dims[height_idx];
833 837
834 return 0;838 return 0;
835}839}
libavfilter/vf_dnn_processing.c+14−7
@@ -77,22 +77,29 @@ static const enum AVPixelFormat pix_fmts[] = { · 76 unmodified lines
77 "the frame's format %s does not match " \77 "the frame's format %s does not match " \
78 "the model input channel %d\n", \78 "the model input channel %d\n", \
79 av_get_pix_fmt_name(fmt), \79 av_get_pix_fmt_name(fmt), \
80 model_input->channels);80 model_input->dims[dnn_get_channel_idx_by_layout(model_input->layout)]);
81 81
82static int check_modelinput_inlink(const DNNData *model_input, const AVFilterLink *inlink)82static int check_modelinput_inlink(const DNNData *model_input, const AVFilterLink *inlink)
83{83{
84 AVFilterContext *ctx = inlink->dst;84 AVFilterContext *ctx = inlink->dst;
85 enum AVPixelFormat fmt = inlink->format;85 enum AVPixelFormat fmt = inlink->format;
86 int width_idx, height_idx;
86 87
88 width_idx = dnn_get_width_idx_by_layout(model_input->layout);
89 height_idx = dnn_get_height_idx_by_layout(model_input->layout);
87 // the design is to add explicit scale filter before this filter90 // the design is to add explicit scale filter before this filter
88 if (model_input->height != -1 && model_input->height != inlink->h) {91 if (model_input->dims[height_idx] != -1 &&
92 model_input->dims[height_idx] != inlink->h) {
89 av_log(ctx, AV_LOG_ERROR, "the model requires frame height %d but got %d\n",93 av_log(ctx, AV_LOG_ERROR, "the model requires frame height %d but got %d\n",
90 model_input->height, inlink->h);94 model_input->dims[height_idx],
95 inlink->h);
91 return AVERROR(EIO);96 return AVERROR(EIO);
92 }97 }
93 if (model_input->width != -1 && model_input->width != inlink->w) {98 if (model_input->dims[width_idx] != -1 &&
99 model_input->dims[width_idx] != inlink->w) {
94 av_log(ctx, AV_LOG_ERROR, "the model requires frame width %d but got %d\n",100 av_log(ctx, AV_LOG_ERROR, "the model requires frame width %d but got %d\n",
95 model_input->width, inlink->w);101 model_input->dims[width_idx],
102 inlink->w);
96 return AVERROR(EIO);103 return AVERROR(EIO);
97 }104 }
98 if (model_input->dt != DNN_FLOAT) {105 if (model_input->dt != DNN_FLOAT) {
@@ -103,7 +110,7 @@ static int check_modelinput_inlink(const DNNData *model_input, const AVFilterLin · 4 unmodified lines
103 switch (fmt) {110 switch (fmt) {
104 case AV_PIX_FMT_RGB24:111 case AV_PIX_FMT_RGB24:
105 case AV_PIX_FMT_BGR24:112 case AV_PIX_FMT_BGR24:
106 if (model_input->channels != 3) {113 if (model_input->dims[dnn_get_channel_idx_by_layout(model_input->layout)] != 3) {
107 LOG_FORMAT_CHANNEL_MISMATCH();114 LOG_FORMAT_CHANNEL_MISMATCH();
108 return AVERROR(EIO);115 return AVERROR(EIO);
109 }116 }
@@ -116,7 +123,7 @@ static int check_modelinput_inlink(const DNNData *model_input, const AVFilterLin · 6 unmodified lines
116 case AV_PIX_FMT_YUV410P:123 case AV_PIX_FMT_YUV410P:
117 case AV_PIX_FMT_YUV411P:124 case AV_PIX_FMT_YUV411P:
118 case AV_PIX_FMT_NV12:125 case AV_PIX_FMT_NV12:
119 if (model_input->channels != 1) {126 if (model_input->dims[dnn_get_channel_idx_by_layout(model_input->layout)] != 1) {
120 LOG_FORMAT_CHANNEL_MISMATCH();127 LOG_FORMAT_CHANNEL_MISMATCH();
121 return AVERROR(EIO);128 return AVERROR(EIO);
122 }129 }

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t