The following is a python script to halve the value of the gray image. It demos how to setup and execute dnn model with python+tensorflow. It also generates .pb file which will be used by ffmpeg. import tensorflow as tf import numpy as np from skimage import color from skimage import io in_img = io.imread('input.jpg') in_img = color.rgb2gray(in_img) io.imsave('ori_gray.jpg', np.squeeze(in_img)) in_data = np.expand_dims(in_img, axis=0) in_data = np.expand_dims(in_data, axis=3) filter_data = np.array([0.5]).reshape(1,1,1,1).astype(np.float32) filter = tf.Variable(filter_data) x = tf.placeholder(tf.float32, shape=[1, None, None, 1], name='dnn_in') y = tf.nn.conv2d(x, filter, strides=[1, 1, 1, 1], padding='VALID', name='dnn_out') sess=tf.Session() sess.run(tf.global_variables_initializer()) graph_def = tf.graph_util.convert_variables_to_constants(sess, sess.graph_def, ['dnn_out']) tf.train.write_graph(graph_def, '.', 'halve_gray_float.pb', as_text=False) print("halve_gray_float.pb generated, please use \ path_to_ffmpeg/tools/python/convert.py to generate halve_gray_float.model\n") output = sess.run(y, feed_dict={x: in_data}) output = output * 255.0 output = output.astype(np.uint8) io.imsave("out.jpg", np.squeeze(output)) To do the same thing with ffmpeg: - generate halve_gray_float.pb with the above script - generate halve_gray_float.model with tools/python/convert.py - try with following commands ./ffmpeg -i input.jpg -vf format=grayf32,dnn_processing=model=halve_gray_float.model:input=dnn_in:output=dnn_out:dnn_backend=native out.native.png ./ffmpeg -i input.jpg -vf format=grayf32,dnn_processing=model=halve_gray_float.pb:input=dnn_in:output=dnn_out:dnn_backend=tensorflow out.tf.png
Showing the changes to libavfilter (1 of 2 files). Show all
Unified · Split
| @@ -104,12 +104,20 @@ static int query_formats(AVFilterContext *context) · 103 unmodified lines | |||
| 104 | { | 104 | { |
| 105 | static const enum AVPixelFormat pix_fmts[] = { | 105 | static const enum AVPixelFormat pix_fmts[] = { |
| 106 | AV_PIX_FMT_RGB24, AV_PIX_FMT_BGR24, | 106 | AV_PIX_FMT_RGB24, AV_PIX_FMT_BGR24, |
| 107 | AV_PIX_FMT_GRAY8, AV_PIX_FMT_GRAYF32, | ||
| 107 | AV_PIX_FMT_NONE | 108 | AV_PIX_FMT_NONE |
| 108 | }; | 109 | }; |
| 109 | AVFilterFormats *fmts_list = ff_make_format_list(pix_fmts); | 110 | AVFilterFormats *fmts_list = ff_make_format_list(pix_fmts); |
| 110 | return ff_set_common_formats(context, fmts_list); | 111 | return ff_set_common_formats(context, fmts_list); |
| 111 | } | 112 | } |
| 112 | 113 | ||
| 114 | #define LOG_FORMAT_CHANNEL_MISMATCH() \ | ||
| 115 | av_log(ctx, AV_LOG_ERROR, \ | ||
| 116 | "the frame's format %s does not match " \ | ||
| 117 | "the model input channel %d\n", \ | ||
| 118 | av_get_pix_fmt_name(fmt), \ | ||
| 119 | model_input->channels); | ||
| 120 | |||
| 113 | static int check_modelinput_inlink(const DNNData *model_input, const AVFilterLink *inlink) | 121 | static int check_modelinput_inlink(const DNNData *model_input, const AVFilterLink *inlink) |
| 114 | { | 122 | { |
| 115 | AVFilterContext *ctx = inlink->dst; | 123 | AVFilterContext *ctx = inlink->dst; |
| @@ -131,17 +139,34 @@ static int check_modelinput_inlink(const DNNData *model_input, const AVFilterLin · 15 unmodified lines | |||
| 131 | case AV_PIX_FMT_RGB24: | 139 | case AV_PIX_FMT_RGB24: |
| 132 | case AV_PIX_FMT_BGR24: | 140 | case AV_PIX_FMT_BGR24: |
| 133 | if (model_input->channels != 3) { | 141 | if (model_input->channels != 3) { |
| 134 | av_log(ctx, AV_LOG_ERROR, "the frame's input format %s does not match " | 142 | LOG_FORMAT_CHANNEL_MISMATCH(); |
| 135 | "the model input channels %d\n", | ||
| 136 | av_get_pix_fmt_name(fmt), | ||
| 137 | model_input->channels); | ||
| 138 | return AVERROR(EIO); | 143 | return AVERROR(EIO); |
| 139 | } | 144 | } |
| 140 | if (model_input->dt != DNN_FLOAT && model_input->dt != DNN_UINT8) { | 145 | if (model_input->dt != DNN_FLOAT && model_input->dt != DNN_UINT8) { |
| 141 | av_log(ctx, AV_LOG_ERROR, "only support dnn models with input data type as float32 and uint8.\n"); | 146 | av_log(ctx, AV_LOG_ERROR, "only support dnn models with input data type as float32 and uint8.\n"); |
| 142 | return AVERROR(EIO); | 147 | return AVERROR(EIO); |
| 143 | } | 148 | } |
| 144 | break; | 149 | return 0; |
| 150 | case AV_PIX_FMT_GRAY8: | ||
| 151 | if (model_input->channels != 1) { | ||
| 152 | LOG_FORMAT_CHANNEL_MISMATCH(); | ||
| 153 | return AVERROR(EIO); | ||
| 154 | } | ||
| 155 | if (model_input->dt != DNN_UINT8) { | ||
| 156 | av_log(ctx, AV_LOG_ERROR, "only support dnn models with input data type uint8.\n"); | ||
| 157 | return AVERROR(EIO); | ||
| 158 | } | ||
| 159 | return 0; | ||
| 160 | case AV_PIX_FMT_GRAYF32: | ||
| 161 | if (model_input->channels != 1) { | ||
| 162 | LOG_FORMAT_CHANNEL_MISMATCH(); | ||
| 163 | return AVERROR(EIO); | ||
| 164 | } | ||
| 165 | if (model_input->dt != DNN_FLOAT) { | ||
| 166 | av_log(ctx, AV_LOG_ERROR, "only support dnn models with input data type float32.\n"); | ||
| 167 | return AVERROR(EIO); | ||
| 168 | } | ||
| 169 | return 0; | ||
| 145 | default: | 170 | default: |
| 146 | av_log(ctx, AV_LOG_ERROR, "%s not supported.\n", av_get_pix_fmt_name(fmt)); | 171 | av_log(ctx, AV_LOG_ERROR, "%s not supported.\n", av_get_pix_fmt_name(fmt)); |
| 147 | return AVERROR(EIO); | 172 | return AVERROR(EIO); |
| @@ -206,28 +231,58 @@ static int config_output(AVFilterLink *outlink) · 58 unmodified lines | |||
| 206 | 231 | ||
| 207 | static int copy_from_frame_to_dnn(DNNData *dnn_input, const AVFrame *frame) | 232 | static int copy_from_frame_to_dnn(DNNData *dnn_input, const AVFrame *frame) |
| 208 | { | 233 | { |
| 209 | // extend this function to support more formats | 234 | switch (frame->format) { |
| 210 | av_assert0(frame->format == AV_PIX_FMT_RGB24 || frame->format == AV_PIX_FMT_BGR24); | 235 | case AV_PIX_FMT_RGB24: |
| 211 | 236 | case AV_PIX_FMT_BGR24: | |
| 212 | if (dnn_input->dt == DNN_FLOAT) { | 237 | if (dnn_input->dt == DNN_FLOAT) { |
| 213 | float *dnn_input_data = dnn_input->data; | 238 | float *dnn_input_data = dnn_input->data; |
| 214 | for (int i = 0; i < frame->height; i++) { | 239 | for (int i = 0; i < frame->height; i++) { |
| 215 | for(int j = 0; j < frame->width * 3; j++) { | 240 | for(int j = 0; j < frame->width * 3; j++) { |
| 216 | int k = i * frame->linesize[0] + j; | 241 | int k = i * frame->linesize[0] + j; |
| 217 | int t = i * frame->width * 3 + j; | 242 | int t = i * frame->width * 3 + j; |
| 218 | dnn_input_data[t] = frame->data[0][k] / 255.0f; | 243 | dnn_input_data[t] = frame->data[0][k] / 255.0f; |
| 244 | } | ||
| 245 | } | ||
| 246 | } else { | ||
| 247 | uint8_t *dnn_input_data = dnn_input->data; | ||
| 248 | av_assert0(dnn_input->dt == DNN_UINT8); | ||
| 249 | for (int i = 0; i < frame->height; i++) { | ||
| 250 | for(int j = 0; j < frame->width * 3; j++) { | ||
| 251 | int k = i * frame->linesize[0] + j; | ||
| 252 | int t = i * frame->width * 3 + j; | ||
| 253 | dnn_input_data[t] = frame->data[0][k]; | ||
| 254 | } | ||
| 219 | } | 255 | } |
| 220 | } | 256 | } |
| 221 | } else { | 257 | return 0; |
| 222 | uint8_t *dnn_input_data = dnn_input->data; | 258 | case AV_PIX_FMT_GRAY8: |
| 223 | av_assert0(dnn_input->dt == DNN_UINT8); | 259 | { |
| 224 | for (int i = 0; i < frame->height; i++) { | 260 | uint8_t *dnn_input_data = dnn_input->data; |
| 225 | for(int j = 0; j < frame->width * 3; j++) { | 261 | av_assert0(dnn_input->dt == DNN_UINT8); |
| 226 | int k = i * frame->linesize[0] + j; | 262 | for (int i = 0; i < frame->height; i++) { |
| 227 | int t = i * frame->width * 3 + j; | 263 | for(int j = 0; j < frame->width; j++) { |
| 228 | dnn_input_data[t] = frame->data[0][k]; | 264 | int k = i * frame->linesize[0] + j; |
| 265 | int t = i * frame->width + j; | ||
| 266 | dnn_input_data[t] = frame->data[0][k]; | ||
| 267 | } | ||
| 229 | } | 268 | } |
| 230 | } | 269 | } |
| 270 | return 0; | ||
| 271 | case AV_PIX_FMT_GRAYF32: | ||
| 272 | { | ||
| 273 | float *dnn_input_data = dnn_input->data; | ||
| 274 | av_assert0(dnn_input->dt == DNN_FLOAT); | ||
| 275 | for (int i = 0; i < frame->height; i++) { | ||
| 276 | for(int j = 0; j < frame->width; j++) { | ||
| 277 | int k = i * frame->linesize[0] + j * sizeof(float); | ||
| 278 | int t = i * frame->width + j; | ||
| 279 | dnn_input_data[t] = *(float*)(frame->data[0] + k); | ||
| 280 | } | ||
| 281 | } | ||
| 282 | } | ||
| 283 | return 0; | ||
| 284 | default: | ||
| 285 | return AVERROR(EIO); | ||
| 231 | } | 286 | } |
| 232 | 287 | ||
| 233 | return 0; | 288 | return 0; |
| @@ -235,28 +290,58 @@ static int copy_from_frame_to_dnn(DNNData *dnn_input, const AVFrame *frame) · 1 unmodified line | |||
| 235 | 290 | ||
| 236 | static int copy_from_dnn_to_frame(AVFrame *frame, const DNNData *dnn_output) | 291 | static int copy_from_dnn_to_frame(AVFrame *frame, const DNNData *dnn_output) |
| 237 | { | 292 | { |
| 238 | // extend this function to support more formats | 293 | switch (frame->format) { |
| 239 | av_assert0(frame->format == AV_PIX_FMT_RGB24 || frame->format == AV_PIX_FMT_BGR24); | 294 | case AV_PIX_FMT_RGB24: |
| 240 | 295 | case AV_PIX_FMT_BGR24: | |
| 241 | if (dnn_output->dt == DNN_FLOAT) { | 296 | if (dnn_output->dt == DNN_FLOAT) { |
| 242 | float *dnn_output_data = dnn_output->data; | 297 | float *dnn_output_data = dnn_output->data; |
| 243 | for (int i = 0; i < frame->height; i++) { | 298 | for (int i = 0; i < frame->height; i++) { |
| 244 | for(int j = 0; j < frame->width * 3; j++) { | 299 | for(int j = 0; j < frame->width * 3; j++) { |
| 245 | int k = i * frame->linesize[0] + j; | 300 | int k = i * frame->linesize[0] + j; |
| 246 | int t = i * frame->width * 3 + j; | 301 | int t = i * frame->width * 3 + j; |
| 247 | frame->data[0][k] = av_clip_uintp2((int)(dnn_output_data[t] * 255.0f), 8); | 302 | frame->data[0][k] = av_clip_uintp2((int)(dnn_output_data[t] * 255.0f), 8); |
| 303 | } | ||
| 304 | } | ||
| 305 | } else { | ||
| 306 | uint8_t *dnn_output_data = dnn_output->data; | ||
| 307 | av_assert0(dnn_output->dt == DNN_UINT8); | ||
| 308 | for (int i = 0; i < frame->height; i++) { | ||
| 309 | for(int j = 0; j < frame->width * 3; j++) { | ||
| 310 | int k = i * frame->linesize[0] + j; | ||
| 311 | int t = i * frame->width * 3 + j; | ||
| 312 | frame->data[0][k] = dnn_output_data[t]; | ||
| 313 | } | ||
| 314 | } | ||
| 315 | } | ||
| 316 | return 0; | ||
| 317 | case AV_PIX_FMT_GRAY8: | ||
| 318 | { | ||
| 319 | uint8_t *dnn_output_data = dnn_output->data; | ||
| 320 | av_assert0(dnn_output->dt == DNN_UINT8); | ||
| 321 | for (int i = 0; i < frame->height; i++) { | ||
| 322 | for(int j = 0; j < frame->width; j++) { | ||
| 323 | int k = i * frame->linesize[0] + j; | ||
| 324 | int t = i * frame->width + j; | ||
| 325 | frame->data[0][k] = dnn_output_data[t]; | ||
| 326 | } | ||
| 248 | } | 327 | } |
| 249 | } | 328 | } |
| 250 | } else { | 329 | return 0; |
| 251 | uint8_t *dnn_output_data = dnn_output->data; | 330 | case AV_PIX_FMT_GRAYF32: |
| 252 | av_assert0(dnn_output->dt == DNN_UINT8); | 331 | { |
| 253 | for (int i = 0; i < frame->height; i++) { | 332 | float *dnn_output_data = dnn_output->data; |
| 254 | for(int j = 0; j < frame->width * 3; j++) { | 333 | av_assert0(dnn_output->dt == DNN_FLOAT); |
| 255 | int k = i * frame->linesize[0] + j; | 334 | for (int i = 0; i < frame->height; i++) { |
| 256 | int t = i * frame->width * 3 + j; | 335 | for(int j = 0; j < frame->width; j++) { |
| 257 | frame->data[0][k] = dnn_output_data[t]; | 336 | int k = i * frame->linesize[0] + j * sizeof(float); |
| 337 | int t = i * frame->width + j; | ||
| 338 | *(float*)(frame->data[0] + k) = dnn_output_data[t]; | ||
| 339 | } | ||
| 258 | } | 340 | } |
| 259 | } | 341 | } |
| 342 | return 0; | ||
| 343 | default: | ||
| 344 | return AVERROR(EIO); | ||
| 260 | } | 345 | } |
| 261 | 346 | ||
| 262 | return 0; | 347 | return 0; |
| @@ -278,7 +363,6 @@ static int filter_frame(AVFilterLink *inlink, AVFrame *in) · 15 unmodified lines | |||
| 278 | av_frame_free(&in); | 363 | av_frame_free(&in); |
| 279 | return AVERROR(EIO); | 364 | return AVERROR(EIO); |
| 280 | } | 365 | } |
| 281 | av_assert0(ctx->output.channels == 3); | ||
| 282 | 366 | ||
| 283 | out = ff_get_video_buffer(outlink, outlink->w, outlink->h); | 367 | out = ff_get_video_buffer(outlink, outlink->w, outlink->h); |
| 284 | if (!out) { | 368 | if (!out) { |