| @@ -27,6 +27,12 @@ · 26 unmodified lines |
| 27 | int ff_proc_from_dnn_to_frame(AVFrame *frame, DNNData *output, void *log_ctx) | 27 | int ff_proc_from_dnn_to_frame(AVFrame *frame, DNNData *output, void *log_ctx) |
| 28 | { | 28 | { |
| 29 | struct SwsContext *sws_ctx; | 29 | struct SwsContext *sws_ctx; |
| | 30 | int ret = 0; |
| | 31 | int linesize[4] = { 0 }; |
| | 32 | void **dst_data = NULL; |
| | 33 | void *middle_data = NULL; |
| | 34 | uint8_t *planar_data[4] = { 0 }; |
| | 35 | int plane_size = frame->width * frame->height * sizeof(uint8_t); |
| 30 | int bytewidth = av_image_get_linesize(frame->format, frame->width, 0); | 36 | int bytewidth = av_image_get_linesize(frame->format, frame->width, 0); |
| 31 | if (bytewidth < 0) { | 37 | if (bytewidth < 0) { |
| 32 | return AVERROR(EINVAL); | 38 | return AVERROR(EINVAL); |
| @@ -35,6 +41,17 @@ int ff_proc_from_dnn_to_frame(AVFrame *frame, DNNData *output, void *log_ctx) · 2 unmodified lines |
| 35 | avpriv_report_missing_feature(log_ctx, "data type rather than DNN_FLOAT"); | 41 | avpriv_report_missing_feature(log_ctx, "data type rather than DNN_FLOAT"); |
| 36 | return AVERROR(ENOSYS); | 42 | return AVERROR(ENOSYS); |
| 37 | } | 43 | } |
| | 44 | dst_data = (void **)frame->data; |
| | 45 | linesize[0] = frame->linesize[0]; |
| | 46 | if (output->layout == DL_NCHW) { |
| | 47 | middle_data = av_malloc(plane_size * output->channels); |
| | 48 | if (!middle_data) { |
| | 49 | ret = AVERROR(ENOMEM); |
| | 50 | goto err; |
| | 51 | } |
| | 52 | dst_data = &middle_data; |
| | 53 | linesize[0] = frame->width * 3; |
| | 54 | } |
| 38 | | 55 | |
| 39 | switch (frame->format) { | 56 | switch (frame->format) { |
| 40 | case AV_PIX_FMT_RGB24: | 57 | case AV_PIX_FMT_RGB24: |
| @@ -51,18 +68,52 @@ int ff_proc_from_dnn_to_frame(AVFrame *frame, DNNData *output, void *log_ctx) · 10 unmodified lines |
| 51 | "fmt:%s s:%dx%d -> fmt:%s s:%dx%d\n", | 68 | "fmt:%s s:%dx%d -> fmt:%s s:%dx%d\n", |
| 52 | av_get_pix_fmt_name(AV_PIX_FMT_GRAYF32), frame->width * 3, frame->height, | 69 | av_get_pix_fmt_name(AV_PIX_FMT_GRAYF32), frame->width * 3, frame->height, |
| 53 | av_get_pix_fmt_name(AV_PIX_FMT_GRAY8), frame->width * 3, frame->height); | 70 | av_get_pix_fmt_name(AV_PIX_FMT_GRAY8), frame->width * 3, frame->height); |
| 54 | return AVERROR(EINVAL); | 71 | ret = AVERROR(EINVAL); |
| | 72 | goto err; |
| 55 | } | 73 | } |
| 56 | sws_scale(sws_ctx, (const uint8_t *[4]){(const uint8_t *)output->data, 0, 0, 0}, | 74 | sws_scale(sws_ctx, (const uint8_t *[4]){(const uint8_t *)output->data, 0, 0, 0}, |
| 57 | (const int[4]){frame->width * 3 * sizeof(float), 0, 0, 0}, 0, frame->height, | 75 | (const int[4]){frame->width * 3 * sizeof(float), 0, 0, 0}, 0, frame->height, |
| 58 | (uint8_t * const*)frame->data, frame->linesize); | 76 | (uint8_t * const*)dst_data, linesize); |
| 59 | sws_freeContext(sws_ctx); | 77 | sws_freeContext(sws_ctx); |
| 60 | return 0; | 78 | // convert data from planar to packed |
| | 79 | if (output->layout == DL_NCHW) { |
| | 80 | sws_ctx = sws_getContext(frame->width, |
| | 81 | frame->height, |
| | 82 | AV_PIX_FMT_GBRP, |
| | 83 | frame->width, |
| | 84 | frame->height, |
| | 85 | frame->format, |
| | 86 | 0, NULL, NULL, NULL); |
| | 87 | if (!sws_ctx) { |
| | 88 | av_log(log_ctx, AV_LOG_ERROR, "Impossible to create scale context for the conversion " |
| | 89 | "fmt:%s s:%dx%d -> fmt:%s s:%dx%d\n", |
| | 90 | av_get_pix_fmt_name(AV_PIX_FMT_GBRP), frame->width, frame->height, |
| | 91 | av_get_pix_fmt_name(frame->format),frame->width, frame->height); |
| | 92 | ret = AVERROR(EINVAL); |
| | 93 | goto err; |
| | 94 | } |
| | 95 | if (frame->format == AV_PIX_FMT_RGB24) { |
| | 96 | planar_data[0] = (uint8_t *)middle_data + plane_size; |
| | 97 | planar_data[1] = (uint8_t *)middle_data + plane_size * 2; |
| | 98 | planar_data[2] = (uint8_t *)middle_data; |
| | 99 | } else if (frame->format == AV_PIX_FMT_BGR24) { |
| | 100 | planar_data[0] = (uint8_t *)middle_data + plane_size; |
| | 101 | planar_data[1] = (uint8_t *)middle_data; |
| | 102 | planar_data[2] = (uint8_t *)middle_data + plane_size * 2; |
| | 103 | } |
| | 104 | sws_scale(sws_ctx, (const uint8_t * const *)planar_data, |
| | 105 | (const int [4]){frame->width * sizeof(uint8_t), |
| | 106 | frame->width * sizeof(uint8_t), |
| | 107 | frame->width * sizeof(uint8_t), 0}, |
| | 108 | 0, frame->height, frame->data, frame->linesize); |
| | 109 | sws_freeContext(sws_ctx); |
| | 110 | } |
| | 111 | break; |
| 61 | case AV_PIX_FMT_GRAYF32: | 112 | case AV_PIX_FMT_GRAYF32: |
| 62 | av_image_copy_plane(frame->data[0], frame->linesize[0], | 113 | av_image_copy_plane(frame->data[0], frame->linesize[0], |
| 63 | output->data, bytewidth, | 114 | output->data, bytewidth, |
| 64 | bytewidth, frame->height); | 115 | bytewidth, frame->height); |
| 65 | return 0; | 116 | break; |
| 66 | case AV_PIX_FMT_YUV420P: | 117 | case AV_PIX_FMT_YUV420P: |
| 67 | case AV_PIX_FMT_YUV422P: | 118 | case AV_PIX_FMT_YUV422P: |
| 68 | case AV_PIX_FMT_YUV444P: | 119 | case AV_PIX_FMT_YUV444P: |
| @@ -82,24 +133,34 @@ int ff_proc_from_dnn_to_frame(AVFrame *frame, DNNData *output, void *log_ctx) · 13 unmodified lines |
| 82 | "fmt:%s s:%dx%d -> fmt:%s s:%dx%d\n", | 133 | "fmt:%s s:%dx%d -> fmt:%s s:%dx%d\n", |
| 83 | av_get_pix_fmt_name(AV_PIX_FMT_GRAYF32), frame->width, frame->height, | 134 | av_get_pix_fmt_name(AV_PIX_FMT_GRAYF32), frame->width, frame->height, |
| 84 | av_get_pix_fmt_name(AV_PIX_FMT_GRAY8), frame->width, frame->height); | 135 | av_get_pix_fmt_name(AV_PIX_FMT_GRAY8), frame->width, frame->height); |
| 85 | return AVERROR(EINVAL); | 136 | ret = AVERROR(EINVAL); |
| | 137 | goto err; |
| 86 | } | 138 | } |
| 87 | sws_scale(sws_ctx, (const uint8_t *[4]){(const uint8_t *)output->data, 0, 0, 0}, | 139 | sws_scale(sws_ctx, (const uint8_t *[4]){(const uint8_t *)output->data, 0, 0, 0}, |
| 88 | (const int[4]){frame->width * sizeof(float), 0, 0, 0}, 0, frame->height, | 140 | (const int[4]){frame->width * sizeof(float), 0, 0, 0}, 0, frame->height, |
| 89 | (uint8_t * const*)frame->data, frame->linesize); | 141 | (uint8_t * const*)frame->data, frame->linesize); |
| 90 | sws_freeContext(sws_ctx); | 142 | sws_freeContext(sws_ctx); |
| 91 | return 0; | 143 | break; |
| 92 | default: | 144 | default: |
| 93 | avpriv_report_missing_feature(log_ctx, "%s", av_get_pix_fmt_name(frame->format)); | 145 | avpriv_report_missing_feature(log_ctx, "%s", av_get_pix_fmt_name(frame->format)); |
| 94 | return AVERROR(ENOSYS); | 146 | ret = AVERROR(ENOSYS); |
| | 147 | goto err; |
| 95 | } | 148 | } |
| 96 | | 149 | |
| 97 | return 0; | 150 | err: |
| | 151 | av_free(middle_data); |
| | 152 | return ret; |
| 98 | } | 153 | } |
| 99 | | 154 | |
| 100 | int ff_proc_from_frame_to_dnn(AVFrame *frame, DNNData *input, void *log_ctx) | 155 | int ff_proc_from_frame_to_dnn(AVFrame *frame, DNNData *input, void *log_ctx) |
| 101 | { | 156 | { |
| 102 | struct SwsContext *sws_ctx; | 157 | struct SwsContext *sws_ctx; |
| | 158 | int ret = 0; |
| | 159 | int linesize[4] = { 0 }; |
| | 160 | void **src_data = NULL; |
| | 161 | void *middle_data = NULL; |
| | 162 | uint8_t *planar_data[4] = { 0 }; |
| | 163 | int plane_size = frame->width * frame->height * sizeof(uint8_t); |
| 103 | int bytewidth = av_image_get_linesize(frame->format, frame->width, 0); | 164 | int bytewidth = av_image_get_linesize(frame->format, frame->width, 0); |
| 104 | if (bytewidth < 0) { | 165 | if (bytewidth < 0) { |
| 105 | return AVERROR(EINVAL); | 166 | return AVERROR(EINVAL); |
| @@ -109,9 +170,54 @@ int ff_proc_from_frame_to_dnn(AVFrame *frame, DNNData *input, void *log_ctx) · 3 unmodified lines |
| 109 | return AVERROR(ENOSYS); | 170 | return AVERROR(ENOSYS); |
| 110 | } | 171 | } |
| 111 | | 172 | |
| | 173 | src_data = (void **)frame->data; |
| | 174 | linesize[0] = frame->linesize[0]; |
| | 175 | if (input->layout == DL_NCHW) { |
| | 176 | middle_data = av_malloc(plane_size * input->channels); |
| | 177 | if (!middle_data) { |
| | 178 | ret = AVERROR(ENOMEM); |
| | 179 | goto err; |
| | 180 | } |
| | 181 | src_data = &middle_data; |
| | 182 | linesize[0] = frame->width * 3; |
| | 183 | } |
| | 184 | |
| 112 | switch (frame->format) { | 185 | switch (frame->format) { |
| 113 | case AV_PIX_FMT_RGB24: | 186 | case AV_PIX_FMT_RGB24: |
| 114 | case AV_PIX_FMT_BGR24: | 187 | case AV_PIX_FMT_BGR24: |
| | 188 | // convert data from planar to packed |
| | 189 | if (input->layout == DL_NCHW) { |
| | 190 | sws_ctx = sws_getContext(frame->width, |
| | 191 | frame->height, |
| | 192 | frame->format, |
| | 193 | frame->width, |
| | 194 | frame->height, |
| | 195 | AV_PIX_FMT_GBRP, |
| | 196 | 0, NULL, NULL, NULL); |
| | 197 | if (!sws_ctx) { |
| | 198 | av_log(log_ctx, AV_LOG_ERROR, "Impossible to create scale context for the conversion " |
| | 199 | "fmt:%s s:%dx%d -> fmt:%s s:%dx%d\n", |
| | 200 | av_get_pix_fmt_name(frame->format), frame->width, frame->height, |
| | 201 | av_get_pix_fmt_name(AV_PIX_FMT_GBRP),frame->width, frame->height); |
| | 202 | ret = AVERROR(EINVAL); |
| | 203 | goto err; |
| | 204 | } |
| | 205 | if (frame->format == AV_PIX_FMT_RGB24) { |
| | 206 | planar_data[0] = (uint8_t *)middle_data + plane_size; |
| | 207 | planar_data[1] = (uint8_t *)middle_data + plane_size * 2; |
| | 208 | planar_data[2] = (uint8_t *)middle_data; |
| | 209 | } else if (frame->format == AV_PIX_FMT_BGR24) { |
| | 210 | planar_data[0] = (uint8_t *)middle_data + plane_size; |
| | 211 | planar_data[1] = (uint8_t *)middle_data; |
| | 212 | planar_data[2] = (uint8_t *)middle_data + plane_size * 2; |
| | 213 | } |
| | 214 | sws_scale(sws_ctx, (const uint8_t * const *)frame->data, |
| | 215 | frame->linesize, 0, frame->height, planar_data, |
| | 216 | (const int [4]){frame->width * sizeof(uint8_t), |
| | 217 | frame->width * sizeof(uint8_t), |
| | 218 | frame->width * sizeof(uint8_t), 0}); |
| | 219 | sws_freeContext(sws_ctx); |
| | 220 | } |
| 115 | sws_ctx = sws_getContext(frame->width * 3, | 221 | sws_ctx = sws_getContext(frame->width * 3, |
| 116 | frame->height, | 222 | frame->height, |
| 117 | AV_PIX_FMT_GRAY8, | 223 | AV_PIX_FMT_GRAY8, |
| @@ -124,10 +230,11 @@ int ff_proc_from_frame_to_dnn(AVFrame *frame, DNNData *input, void *log_ctx) · 6 unmodified lines |
| 124 | "fmt:%s s:%dx%d -> fmt:%s s:%dx%d\n", | 230 | "fmt:%s s:%dx%d -> fmt:%s s:%dx%d\n", |
| 125 | av_get_pix_fmt_name(AV_PIX_FMT_GRAY8), frame->width * 3, frame->height, | 231 | av_get_pix_fmt_name(AV_PIX_FMT_GRAY8), frame->width * 3, frame->height, |
| 126 | av_get_pix_fmt_name(AV_PIX_FMT_GRAYF32),frame->width * 3, frame->height); | 232 | av_get_pix_fmt_name(AV_PIX_FMT_GRAYF32),frame->width * 3, frame->height); |
| 127 | return AVERROR(EINVAL); | 233 | ret = AVERROR(EINVAL); |
| | 234 | goto err; |
| 128 | } | 235 | } |
| 129 | sws_scale(sws_ctx, (const uint8_t **)frame->data, | 236 | sws_scale(sws_ctx, (const uint8_t **)src_data, |
| 130 | frame->linesize, 0, frame->height, | 237 | linesize, 0, frame->height, |
| 131 | (uint8_t * const [4]){input->data, 0, 0, 0}, | 238 | (uint8_t * const [4]){input->data, 0, 0, 0}, |
| 132 | (const int [4]){frame->width * 3 * sizeof(float), 0, 0, 0}); | 239 | (const int [4]){frame->width * 3 * sizeof(float), 0, 0, 0}); |
| 133 | sws_freeContext(sws_ctx); | 240 | sws_freeContext(sws_ctx); |
| @@ -156,7 +263,8 @@ int ff_proc_from_frame_to_dnn(AVFrame *frame, DNNData *input, void *log_ctx) · 22 unmodified lines |
| 156 | "fmt:%s s:%dx%d -> fmt:%s s:%dx%d\n", | 263 | "fmt:%s s:%dx%d -> fmt:%s s:%dx%d\n", |
| 157 | av_get_pix_fmt_name(AV_PIX_FMT_GRAY8), frame->width, frame->height, | 264 | av_get_pix_fmt_name(AV_PIX_FMT_GRAY8), frame->width, frame->height, |
| 158 | av_get_pix_fmt_name(AV_PIX_FMT_GRAYF32),frame->width, frame->height); | 265 | av_get_pix_fmt_name(AV_PIX_FMT_GRAYF32),frame->width, frame->height); |
| 159 | return AVERROR(EINVAL); | 266 | ret = AVERROR(EINVAL); |
| | 267 | goto err; |
| 160 | } | 268 | } |
| 161 | sws_scale(sws_ctx, (const uint8_t **)frame->data, | 269 | sws_scale(sws_ctx, (const uint8_t **)frame->data, |
| 162 | frame->linesize, 0, frame->height, | 270 | frame->linesize, 0, frame->height, |
| @@ -166,10 +274,12 @@ int ff_proc_from_frame_to_dnn(AVFrame *frame, DNNData *input, void *log_ctx) · 3 unmodified lines |
| 166 | break; | 274 | break; |
| 167 | default: | 275 | default: |
| 168 | avpriv_report_missing_feature(log_ctx, "%s", av_get_pix_fmt_name(frame->format)); | 276 | avpriv_report_missing_feature(log_ctx, "%s", av_get_pix_fmt_name(frame->format)); |
| 169 | return AVERROR(ENOSYS); | 277 | ret = AVERROR(ENOSYS); |
| | 278 | goto err; |
| 170 | } | 279 | } |
| 171 | | 280 | err: |
| 172 | return 0; | 281 | av_free(middle_data); |
| | 282 | return ret; |
| 173 | } | 283 | } |
| 174 | | 284 | |
| 175 | static enum AVPixelFormat get_pixel_format(DNNData *data) | 285 | static enum AVPixelFormat get_pixel_format(DNNData *data) |
| @@ -205,6 +315,11 @@ int ff_frame_to_dnn_classify(AVFrame *frame, DNNData *input, uint32_t bbox_index · 29 unmodified lines |
| 205 | AVFrameSideData *sd = av_frame_get_side_data(frame, AV_FRAME_DATA_DETECTION_BBOXES); | 315 | AVFrameSideData *sd = av_frame_get_side_data(frame, AV_FRAME_DATA_DETECTION_BBOXES); |
| 206 | av_assert0(sd); | 316 | av_assert0(sd); |
| 207 | | 317 | |
| | 318 | if (input->layout == DL_NCHW) { |
| | 319 | av_log(log_ctx, AV_LOG_ERROR, "dnn_classify input data doesn't support layout: NCHW\n"); |
| | 320 | return AVERROR(ENOSYS); |
| | 321 | } |
| | 322 | |
| 208 | header = (const AVDetectionBBoxHeader *)sd->data; | 323 | header = (const AVDetectionBBoxHeader *)sd->data; |
| 209 | bbox = av_get_detection_bbox(header, bbox_index); | 324 | bbox = av_get_detection_bbox(header, bbox_index); |
| 210 | | 325 | |
| @@ -257,6 +372,12 @@ int ff_frame_to_dnn_detect(AVFrame *frame, DNNData *input, void *log_ctx) · 46 unmodified lines |
| 257 | int linesizes[4]; | 372 | int linesizes[4]; |
| 258 | int ret = 0; | 373 | int ret = 0; |
| 259 | enum AVPixelFormat fmt = get_pixel_format(input); | 374 | enum AVPixelFormat fmt = get_pixel_format(input); |
| | 375 | |
| | 376 | if (input->layout == DL_NCHW) { |
| | 377 | av_log(log_ctx, AV_LOG_ERROR, "dnn_detect input data doesn't support layout: NCHW\n"); |
| | 378 | return AVERROR(ENOSYS); |
| | 379 | } |
| | 380 | |
| 260 | sws_ctx = sws_getContext(frame->width, frame->height, frame->format, | 381 | sws_ctx = sws_getContext(frame->width, frame->height, frame->format, |
| 261 | input->width, input->height, fmt, | 382 | input->width, input->height, fmt, |
| 262 | SWS_FAST_BILINEAR, NULL, NULL, NULL); | 383 | SWS_FAST_BILINEAR, NULL, NULL, NULL); |