| @@ -24,6 +24,20 @@ · 23 unmodified lines |
| 24 | #include "libavutil/avassert.h" | 24 | #include "libavutil/avassert.h" |
| 25 | #include "libavutil/detection_bbox.h" | 25 | #include "libavutil/detection_bbox.h" |
| 26 | | 26 | |
| | 27 | static int get_datatype_size(DNNDataType dt) |
| | 28 | { |
| | 29 | switch (dt) |
| | 30 | { |
| | 31 | case DNN_FLOAT: |
| | 32 | return sizeof(float); |
| | 33 | case DNN_UINT8: |
| | 34 | return sizeof(uint8_t); |
| | 35 | default: |
| | 36 | av_assert0(!"not supported yet."); |
| | 37 | return 1; |
| | 38 | } |
| | 39 | } |
| | 40 | |
| 27 | int ff_proc_from_dnn_to_frame(AVFrame *frame, DNNData *output, void *log_ctx) | 41 | int ff_proc_from_dnn_to_frame(AVFrame *frame, DNNData *output, void *log_ctx) |
| 28 | { | 42 | { |
| 29 | struct SwsContext *sws_ctx; | 43 | struct SwsContext *sws_ctx; |
| @@ -33,14 +47,26 @@ int ff_proc_from_dnn_to_frame(AVFrame *frame, DNNData *output, void *log_ctx) · 3 unmodified lines |
| 33 | void *middle_data = NULL; | 47 | void *middle_data = NULL; |
| 34 | uint8_t *planar_data[4] = { 0 }; | 48 | uint8_t *planar_data[4] = { 0 }; |
| 35 | int plane_size = frame->width * frame->height * sizeof(uint8_t); | 49 | int plane_size = frame->width * frame->height * sizeof(uint8_t); |
| | 50 | enum AVPixelFormat src_fmt = AV_PIX_FMT_NONE; |
| | 51 | int src_datatype_size = get_datatype_size(output->dt); |
| | 52 | |
| 36 | int bytewidth = av_image_get_linesize(frame->format, frame->width, 0); | 53 | int bytewidth = av_image_get_linesize(frame->format, frame->width, 0); |
| 37 | if (bytewidth < 0) { | 54 | if (bytewidth < 0) { |
| 38 | return AVERROR(EINVAL); | 55 | return AVERROR(EINVAL); |
| 39 | } | 56 | } |
| 40 | if (output->dt != DNN_FLOAT) { | 57 | /* scale == 1 and mean == 0 and dt == UINT8: passthrough */ |
| 41 | avpriv_report_missing_feature(log_ctx, "data type rather than DNN_FLOAT"); | 58 | if (fabsf(output->scale - 1) < 1e-6f && fabsf(output->mean) < 1e-6 && output->dt == DNN_UINT8) |
| | 59 | src_fmt = AV_PIX_FMT_GRAY8; |
| | 60 | /* (scale == 255 or scale == 0) and mean == 0 and dt == FLOAT: normalization */ |
| | 61 | else if ((fabsf(output->scale - 255) < 1e-6f || fabsf(output->scale) < 1e-6f) && |
| | 62 | fabsf(output->mean) < 1e-6 && output->dt == DNN_FLOAT) |
| | 63 | src_fmt = AV_PIX_FMT_GRAYF32; |
| | 64 | else { |
| | 65 | av_log(log_ctx, AV_LOG_ERROR, "dnn_process output data doesn't type: UINT8 " |
| | 66 | "scale: %f, mean: %f\n", output->scale, output->mean); |
| 42 | return AVERROR(ENOSYS); | 67 | return AVERROR(ENOSYS); |
| 43 | } | 68 | } |
| | 69 | |
| 44 | dst_data = (void **)frame->data; | 70 | dst_data = (void **)frame->data; |
| 45 | linesize[0] = frame->linesize[0]; | 71 | linesize[0] = frame->linesize[0]; |
| 46 | if (output->layout == DL_NCHW) { | 72 | if (output->layout == DL_NCHW) { |
| @@ -58,7 +84,7 @@ int ff_proc_from_dnn_to_frame(AVFrame *frame, DNNData *output, void *log_ctx) · 11 unmodified lines |
| 58 | case AV_PIX_FMT_BGR24: | 84 | case AV_PIX_FMT_BGR24: |
| 59 | sws_ctx = sws_getContext(frame->width * 3, | 85 | sws_ctx = sws_getContext(frame->width * 3, |
| 60 | frame->height, | 86 | frame->height, |
| 61 | AV_PIX_FMT_GRAYF32, | 87 | src_fmt, |
| 62 | frame->width * 3, | 88 | frame->width * 3, |
| 63 | frame->height, | 89 | frame->height, |
| 64 | AV_PIX_FMT_GRAY8, | 90 | AV_PIX_FMT_GRAY8, |
| @@ -66,13 +92,13 @@ int ff_proc_from_dnn_to_frame(AVFrame *frame, DNNData *output, void *log_ctx) · 1 unmodified line |
| 66 | if (!sws_ctx) { | 92 | if (!sws_ctx) { |
| 67 | av_log(log_ctx, AV_LOG_ERROR, "Impossible to create scale context for the conversion " | 93 | av_log(log_ctx, AV_LOG_ERROR, "Impossible to create scale context for the conversion " |
| 68 | "fmt:%s s:%dx%d -> fmt:%s s:%dx%d\n", | 94 | "fmt:%s s:%dx%d -> fmt:%s s:%dx%d\n", |
| 69 | av_get_pix_fmt_name(AV_PIX_FMT_GRAYF32), frame->width * 3, frame->height, | 95 | av_get_pix_fmt_name(src_fmt), frame->width * 3, frame->height, |
| 70 | av_get_pix_fmt_name(AV_PIX_FMT_GRAY8), frame->width * 3, frame->height); | 96 | av_get_pix_fmt_name(AV_PIX_FMT_GRAY8), frame->width * 3, frame->height); |
| 71 | ret = AVERROR(EINVAL); | 97 | ret = AVERROR(EINVAL); |
| 72 | goto err; | 98 | goto err; |
| 73 | } | 99 | } |
| 74 | sws_scale(sws_ctx, (const uint8_t *[4]){(const uint8_t *)output->data, 0, 0, 0}, | 100 | sws_scale(sws_ctx, (const uint8_t *[4]){(const uint8_t *)output->data, 0, 0, 0}, |
| 75 | (const int[4]){frame->width * 3 * sizeof(float), 0, 0, 0}, 0, frame->height, | 101 | (const int[4]){frame->width * 3 * src_datatype_size, 0, 0, 0}, 0, frame->height, |
| 76 | (uint8_t * const*)dst_data, linesize); | 102 | (uint8_t * const*)dst_data, linesize); |
| 77 | sws_freeContext(sws_ctx); | 103 | sws_freeContext(sws_ctx); |
| 78 | // convert data from planar to packed | 104 | // convert data from planar to packed |
| @@ -131,13 +157,13 @@ int ff_proc_from_dnn_to_frame(AVFrame *frame, DNNData *output, void *log_ctx) · 52 unmodified lines |
| 131 | if (!sws_ctx) { | 157 | if (!sws_ctx) { |
| 132 | av_log(log_ctx, AV_LOG_ERROR, "Impossible to create scale context for the conversion " | 158 | av_log(log_ctx, AV_LOG_ERROR, "Impossible to create scale context for the conversion " |
| 133 | "fmt:%s s:%dx%d -> fmt:%s s:%dx%d\n", | 159 | "fmt:%s s:%dx%d -> fmt:%s s:%dx%d\n", |
| 134 | av_get_pix_fmt_name(AV_PIX_FMT_GRAYF32), frame->width, frame->height, | 160 | av_get_pix_fmt_name(src_fmt), frame->width, frame->height, |
| 135 | av_get_pix_fmt_name(AV_PIX_FMT_GRAY8), frame->width, frame->height); | 161 | av_get_pix_fmt_name(AV_PIX_FMT_GRAY8), frame->width, frame->height); |
| 136 | ret = AVERROR(EINVAL); | 162 | ret = AVERROR(EINVAL); |
| 137 | goto err; | 163 | goto err; |
| 138 | } | 164 | } |
| 139 | sws_scale(sws_ctx, (const uint8_t *[4]){(const uint8_t *)output->data, 0, 0, 0}, | 165 | sws_scale(sws_ctx, (const uint8_t *[4]){(const uint8_t *)output->data, 0, 0, 0}, |
| 140 | (const int[4]){frame->width * sizeof(float), 0, 0, 0}, 0, frame->height, | 166 | (const int[4]){frame->width * src_datatype_size, 0, 0, 0}, 0, frame->height, |
| 141 | (uint8_t * const*)frame->data, frame->linesize); | 167 | (uint8_t * const*)frame->data, frame->linesize); |
| 142 | sws_freeContext(sws_ctx); | 168 | sws_freeContext(sws_ctx); |
| 143 | break; | 169 | break; |
| @@ -161,12 +187,22 @@ int ff_proc_from_frame_to_dnn(AVFrame *frame, DNNData *input, void *log_ctx) · 17 unmodified lines |
| 161 | void *middle_data = NULL; | 187 | void *middle_data = NULL; |
| 162 | uint8_t *planar_data[4] = { 0 }; | 188 | uint8_t *planar_data[4] = { 0 }; |
| 163 | int plane_size = frame->width * frame->height * sizeof(uint8_t); | 189 | int plane_size = frame->width * frame->height * sizeof(uint8_t); |
| | 190 | enum AVPixelFormat dst_fmt = AV_PIX_FMT_NONE; |
| | 191 | int dst_datatype_size = get_datatype_size(input->dt); |
| 164 | int bytewidth = av_image_get_linesize(frame->format, frame->width, 0); | 192 | int bytewidth = av_image_get_linesize(frame->format, frame->width, 0); |
| 165 | if (bytewidth < 0) { | 193 | if (bytewidth < 0) { |
| 166 | return AVERROR(EINVAL); | 194 | return AVERROR(EINVAL); |
| 167 | } | 195 | } |
| 168 | if (input->dt != DNN_FLOAT) { | 196 | /* scale == 1 and mean == 0 and dt == UINT8: passthrough */ |
| 169 | avpriv_report_missing_feature(log_ctx, "data type rather than DNN_FLOAT"); | 197 | if (fabsf(input->scale - 1) < 1e-6f && fabsf(input->mean) < 1e-6 && input->dt == DNN_UINT8) |
| | 198 | dst_fmt = AV_PIX_FMT_GRAY8; |
| | 199 | /* (scale == 255 or scale == 0) and mean == 0 and dt == FLOAT: normalization */ |
| | 200 | else if ((fabsf(input->scale - 255) < 1e-6f || fabsf(input->scale) < 1e-6f) && |
| | 201 | fabsf(input->mean) < 1e-6 && input->dt == DNN_FLOAT) |
| | 202 | dst_fmt = AV_PIX_FMT_GRAYF32; |
| | 203 | else { |
| | 204 | av_log(log_ctx, AV_LOG_ERROR, "dnn_process input data doesn't support type: UINT8 " |
| | 205 | "scale: %f, mean: %f\n", input->scale, input->mean); |
| 170 | return AVERROR(ENOSYS); | 206 | return AVERROR(ENOSYS); |
| 171 | } | 207 | } |
| 172 | | 208 | |
| @@ -223,20 +259,20 @@ int ff_proc_from_frame_to_dnn(AVFrame *frame, DNNData *input, void *log_ctx) · 50 unmodified lines |
| 223 | AV_PIX_FMT_GRAY8, | 259 | AV_PIX_FMT_GRAY8, |
| 224 | frame->width * 3, | 260 | frame->width * 3, |
| 225 | frame->height, | 261 | frame->height, |
| 226 | AV_PIX_FMT_GRAYF32, | 262 | dst_fmt, |
| 227 | 0, NULL, NULL, NULL); | 263 | 0, NULL, NULL, NULL); |
| 228 | if (!sws_ctx) { | 264 | if (!sws_ctx) { |
| 229 | av_log(log_ctx, AV_LOG_ERROR, "Impossible to create scale context for the conversion " | 265 | av_log(log_ctx, AV_LOG_ERROR, "Impossible to create scale context for the conversion " |
| 230 | "fmt:%s s:%dx%d -> fmt:%s s:%dx%d\n", | 266 | "fmt:%s s:%dx%d -> fmt:%s s:%dx%d\n", |
| 231 | av_get_pix_fmt_name(AV_PIX_FMT_GRAY8), frame->width * 3, frame->height, | 267 | av_get_pix_fmt_name(AV_PIX_FMT_GRAY8), frame->width * 3, frame->height, |
| 232 | av_get_pix_fmt_name(AV_PIX_FMT_GRAYF32),frame->width * 3, frame->height); | 268 | av_get_pix_fmt_name(dst_fmt),frame->width * 3, frame->height); |
| 233 | ret = AVERROR(EINVAL); | 269 | ret = AVERROR(EINVAL); |
| 234 | goto err; | 270 | goto err; |
| 235 | } | 271 | } |
| 236 | sws_scale(sws_ctx, (const uint8_t **)src_data, | 272 | sws_scale(sws_ctx, (const uint8_t **)src_data, |
| 237 | linesize, 0, frame->height, | 273 | linesize, 0, frame->height, |
| 238 | (uint8_t * const [4]){input->data, 0, 0, 0}, | 274 | (uint8_t * const [4]){input->data, 0, 0, 0}, |
| 239 | (const int [4]){frame->width * 3 * sizeof(float), 0, 0, 0}); | 275 | (const int [4]){frame->width * 3 * dst_datatype_size, 0, 0, 0}); |
| 240 | sws_freeContext(sws_ctx); | 276 | sws_freeContext(sws_ctx); |
| 241 | break; | 277 | break; |
| 242 | case AV_PIX_FMT_GRAYF32: | 278 | case AV_PIX_FMT_GRAYF32: |
| @@ -256,20 +292,20 @@ int ff_proc_from_frame_to_dnn(AVFrame *frame, DNNData *input, void *log_ctx) · 13 unmodified lines |
| 256 | AV_PIX_FMT_GRAY8, | 292 | AV_PIX_FMT_GRAY8, |
| 257 | frame->width, | 293 | frame->width, |
| 258 | frame->height, | 294 | frame->height, |
| 259 | AV_PIX_FMT_GRAYF32, | 295 | dst_fmt, |
| 260 | 0, NULL, NULL, NULL); | 296 | 0, NULL, NULL, NULL); |
| 261 | if (!sws_ctx) { | 297 | if (!sws_ctx) { |
| 262 | av_log(log_ctx, AV_LOG_ERROR, "Impossible to create scale context for the conversion " | 298 | av_log(log_ctx, AV_LOG_ERROR, "Impossible to create scale context for the conversion " |
| 263 | "fmt:%s s:%dx%d -> fmt:%s s:%dx%d\n", | 299 | "fmt:%s s:%dx%d -> fmt:%s s:%dx%d\n", |
| 264 | av_get_pix_fmt_name(AV_PIX_FMT_GRAY8), frame->width, frame->height, | 300 | av_get_pix_fmt_name(AV_PIX_FMT_GRAY8), frame->width, frame->height, |
| 265 | av_get_pix_fmt_name(AV_PIX_FMT_GRAYF32),frame->width, frame->height); | 301 | av_get_pix_fmt_name(dst_fmt),frame->width, frame->height); |
| 266 | ret = AVERROR(EINVAL); | 302 | ret = AVERROR(EINVAL); |
| 267 | goto err; | 303 | goto err; |
| 268 | } | 304 | } |
| 269 | sws_scale(sws_ctx, (const uint8_t **)frame->data, | 305 | sws_scale(sws_ctx, (const uint8_t **)frame->data, |
| 270 | frame->linesize, 0, frame->height, | 306 | frame->linesize, 0, frame->height, |
| 271 | (uint8_t * const [4]){input->data, 0, 0, 0}, | 307 | (uint8_t * const [4]){input->data, 0, 0, 0}, |
| 272 | (const int [4]){frame->width * sizeof(float), 0, 0, 0}); | 308 | (const int [4]){frame->width * dst_datatype_size, 0, 0, 0}); |
| 273 | sws_freeContext(sws_ctx); | 309 | sws_freeContext(sws_ctx); |
| 274 | break; | 310 | break; |
| 275 | default: | 311 | default: |
| @@ -315,6 +351,14 @@ int ff_frame_to_dnn_classify(AVFrame *frame, DNNData *input, uint32_t bbox_index · 39 unmodified lines |
| 315 | AVFrameSideData *sd = av_frame_get_side_data(frame, AV_FRAME_DATA_DETECTION_BBOXES); | 351 | AVFrameSideData *sd = av_frame_get_side_data(frame, AV_FRAME_DATA_DETECTION_BBOXES); |
| 316 | av_assert0(sd); | 352 | av_assert0(sd); |
| 317 | | 353 | |
| | 354 | /* (scale != 1 and scale != 0) or mean != 0 */ |
| | 355 | if ((fabsf(input->scale - 1) > 1e-6f && fabsf(input->scale) > 1e-6f) || |
| | 356 | fabsf(input->mean) > 1e-6f) { |
| | 357 | av_log(log_ctx, AV_LOG_ERROR, "dnn_classify input data doesn't support " |
| | 358 | "scale: %f, mean: %f\n", input->scale, input->mean); |
| | 359 | return AVERROR(ENOSYS); |
| | 360 | } |
| | 361 | |
| 318 | if (input->layout == DL_NCHW) { | 362 | if (input->layout == DL_NCHW) { |
| 319 | av_log(log_ctx, AV_LOG_ERROR, "dnn_classify input data doesn't support layout: NCHW\n"); | 363 | av_log(log_ctx, AV_LOG_ERROR, "dnn_classify input data doesn't support layout: NCHW\n"); |
| 320 | return AVERROR(ENOSYS); | 364 | return AVERROR(ENOSYS); |
| @@ -373,6 +417,14 @@ int ff_frame_to_dnn_detect(AVFrame *frame, DNNData *input, void *log_ctx) · 52 unmodified lines |
| 373 | int ret = 0; | 417 | int ret = 0; |
| 374 | enum AVPixelFormat fmt = get_pixel_format(input); | 418 | enum AVPixelFormat fmt = get_pixel_format(input); |
| 375 | | 419 | |
| | 420 | /* (scale != 1 and scale != 0) or mean != 0 */ |
| | 421 | if ((fabsf(input->scale - 1) > 1e-6f && fabsf(input->scale) > 1e-6f) || |
| | 422 | fabsf(input->mean) > 1e-6f) { |
| | 423 | av_log(log_ctx, AV_LOG_ERROR, "dnn_detect input data doesn't support " |
| | 424 | "scale: %f, mean: %f\n", input->scale, input->mean); |
| | 425 | return AVERROR(ENOSYS); |
| | 426 | } |
| | 427 | |
| 376 | if (input->layout == DL_NCHW) { | 428 | if (input->layout == DL_NCHW) { |
| 377 | av_log(log_ctx, AV_LOG_ERROR, "dnn_detect input data doesn't support layout: NCHW\n"); | 429 | av_log(log_ctx, AV_LOG_ERROR, "dnn_detect input data doesn't support layout: NCHW\n"); |
| 378 | return AVERROR(ENOSYS); | 430 | return AVERROR(ENOSYS); |