The following is a python script to halve the value of the gray image. It demos how to setup and execute dnn model with python+tensorflow. It also generates .pb file which will be used by ffmpeg. import tensorflow as tf import numpy as np from skimage import color from skimage import io in_img = io.imread('input.jpg') in_img = color.rgb2gray(in_img) io.imsave('ori_gray.jpg', np.squeeze(in_img)) in_data = np.expand_dims(in_img, axis=0) in_data = np.expand_dims(in_data, axis=3) filter_data = np.array([0.5]).reshape(1,1,1,1).astype(np.float32) filter = tf.Variable(filter_data) x = tf.placeholder(tf.float32, shape=[1, None, None, 1], name='dnn_in') y = tf.nn.conv2d(x, filter, strides=[1, 1, 1, 1], padding='VALID', name='dnn_out') sess=tf.Session() sess.run(tf.global_variables_initializer()) graph_def = tf.graph_util.convert_variables_to_constants(sess, sess.graph_def, ['dnn_out']) tf.train.write_graph(graph_def, '.', 'halve_gray_float.pb', as_text=False) print("halve_gray_float.pb generated, please use \ path_to_ffmpeg/tools/python/convert.py to generate halve_gray_float.model\n") output = sess.run(y, feed_dict={x: in_data}) output = output * 255.0 output = output.astype(np.uint8) io.imsave("out.jpg", np.squeeze(output)) To do the same thing with ffmpeg: - generate halve_gray_float.pb with the above script - generate halve_gray_float.model with tools/python/convert.py - try with following commands ./ffmpeg -i input.jpg -vf format=grayf32,dnn_processing=model=halve_gray_float.model:input=dnn_in:output=dnn_out:dnn_backend=native out.native.png ./ffmpeg -i input.jpg -vf format=grayf32,dnn_processing=model=halve_gray_float.pb:input=dnn_in:output=dnn_out:dnn_backend=tensorflow out.tf.png
Showing the changes to libavfilter (1 of 2 files). Show all
Unified · Split
| @@ -104,12 +104,20 @@ static int query_formats(AVFilterContext *context) · 103 unmodified lines | ||
| 104 | 104 | { |
| 105 | 105 | static const enum AVPixelFormat pix_fmts[] = { |
| 106 | 106 | AV_PIX_FMT_RGB24, AV_PIX_FMT_BGR24, |
| 107 | + AV_PIX_FMT_GRAY8, AV_PIX_FMT_GRAYF32, | |
| 107 | 108 | AV_PIX_FMT_NONE |
| 108 | 109 | }; |
| 109 | 110 | AVFilterFormats *fmts_list = ff_make_format_list(pix_fmts); |
| 110 | 111 | return ff_set_common_formats(context, fmts_list); |
| 111 | 112 | } |
| 112 | 113 | |
| 114 | +#define LOG_FORMAT_CHANNEL_MISMATCH() \ | |
| 115 | + av_log(ctx, AV_LOG_ERROR, \ | |
| 116 | + "the frame's format %s does not match " \ | |
| 117 | + "the model input channel %d\n", \ | |
| 118 | + av_get_pix_fmt_name(fmt), \ | |
| 119 | + model_input->channels); | |
| 120 | + | |
| 113 | 121 | static int check_modelinput_inlink(const DNNData *model_input, const AVFilterLink *inlink) |
| 114 | 122 | { |
| 115 | 123 | AVFilterContext *ctx = inlink->dst; |
| @@ -131,17 +139,34 @@ static int check_modelinput_inlink(const DNNData *model_input, const AVFilterLin · 15 unmodified lines | ||
| 131 | 139 | case AV_PIX_FMT_RGB24: |
| 132 | 140 | case AV_PIX_FMT_BGR24: |
| 133 | 141 | if (model_input->channels != 3) { |
| 134 | − av_log(ctx, AV_LOG_ERROR, "the frame's input format %s does not match " | |
| 135 | − "the model input channels %d\n", | |
| 136 | − av_get_pix_fmt_name(fmt), | |
| 137 | − model_input->channels); | |
| 142 | + LOG_FORMAT_CHANNEL_MISMATCH(); | |
| 138 | 143 | return AVERROR(EIO); |
| 139 | 144 | } |
| 140 | 145 | if (model_input->dt != DNN_FLOAT && model_input->dt != DNN_UINT8) { |
| 141 | 146 | av_log(ctx, AV_LOG_ERROR, "only support dnn models with input data type as float32 and uint8.\n"); |
| 142 | 147 | return AVERROR(EIO); |
| 143 | 148 | } |
| 144 | − break; | |
| 149 | + return 0; | |
| 150 | + case AV_PIX_FMT_GRAY8: | |
| 151 | + if (model_input->channels != 1) { | |
| 152 | + LOG_FORMAT_CHANNEL_MISMATCH(); | |
| 153 | + return AVERROR(EIO); | |
| 154 | + } | |
| 155 | + if (model_input->dt != DNN_UINT8) { | |
| 156 | + av_log(ctx, AV_LOG_ERROR, "only support dnn models with input data type uint8.\n"); | |
| 157 | + return AVERROR(EIO); | |
| 158 | + } | |
| 159 | + return 0; | |
| 160 | + case AV_PIX_FMT_GRAYF32: | |
| 161 | + if (model_input->channels != 1) { | |
| 162 | + LOG_FORMAT_CHANNEL_MISMATCH(); | |
| 163 | + return AVERROR(EIO); | |
| 164 | + } | |
| 165 | + if (model_input->dt != DNN_FLOAT) { | |
| 166 | + av_log(ctx, AV_LOG_ERROR, "only support dnn models with input data type float32.\n"); | |
| 167 | + return AVERROR(EIO); | |
| 168 | + } | |
| 169 | + return 0; | |
| 145 | 170 | default: |
| 146 | 171 | av_log(ctx, AV_LOG_ERROR, "%s not supported.\n", av_get_pix_fmt_name(fmt)); |
| 147 | 172 | return AVERROR(EIO); |
| @@ -206,28 +231,58 @@ static int config_output(AVFilterLink *outlink) · 58 unmodified lines | ||
| 206 | 231 | |
| 207 | 232 | static int copy_from_frame_to_dnn(DNNData *dnn_input, const AVFrame *frame) |
| 208 | 233 | { |
| 209 | − // extend this function to support more formats | |
| 210 | − av_assert0(frame->format == AV_PIX_FMT_RGB24 || frame->format == AV_PIX_FMT_BGR24); | |
| 211 | − | |
| 212 | − if (dnn_input->dt == DNN_FLOAT) { | |
| 213 | − float *dnn_input_data = dnn_input->data; | |
| 214 | − for (int i = 0; i < frame->height; i++) { | |
| 215 | − for(int j = 0; j < frame->width * 3; j++) { | |
| 216 | − int k = i * frame->linesize[0] + j; | |
| 217 | − int t = i * frame->width * 3 + j; | |
| 218 | − dnn_input_data[t] = frame->data[0][k] / 255.0f; | |
| 234 | + switch (frame->format) { | |
| 235 | + case AV_PIX_FMT_RGB24: | |
| 236 | + case AV_PIX_FMT_BGR24: | |
| 237 | + if (dnn_input->dt == DNN_FLOAT) { | |
| 238 | + float *dnn_input_data = dnn_input->data; | |
| 239 | + for (int i = 0; i < frame->height; i++) { | |
| 240 | + for(int j = 0; j < frame->width * 3; j++) { | |
| 241 | + int k = i * frame->linesize[0] + j; | |
| 242 | + int t = i * frame->width * 3 + j; | |
| 243 | + dnn_input_data[t] = frame->data[0][k] / 255.0f; | |
| 244 | + } | |
| 245 | + } | |
| 246 | + } else { | |
| 247 | + uint8_t *dnn_input_data = dnn_input->data; | |
| 248 | + av_assert0(dnn_input->dt == DNN_UINT8); | |
| 249 | + for (int i = 0; i < frame->height; i++) { | |
| 250 | + for(int j = 0; j < frame->width * 3; j++) { | |
| 251 | + int k = i * frame->linesize[0] + j; | |
| 252 | + int t = i * frame->width * 3 + j; | |
| 253 | + dnn_input_data[t] = frame->data[0][k]; | |
| 254 | + } | |
| 219 | 255 | } |
| 220 | 256 | } |
| 221 | − } else { | |
| 222 | − uint8_t *dnn_input_data = dnn_input->data; | |
| 223 | − av_assert0(dnn_input->dt == DNN_UINT8); | |
| 224 | − for (int i = 0; i < frame->height; i++) { | |
| 225 | − for(int j = 0; j < frame->width * 3; j++) { | |
| 226 | − int k = i * frame->linesize[0] + j; | |
| 227 | − int t = i * frame->width * 3 + j; | |
| 228 | − dnn_input_data[t] = frame->data[0][k]; | |
| 257 | + return 0; | |
| 258 | + case AV_PIX_FMT_GRAY8: | |
| 259 | + { | |
| 260 | + uint8_t *dnn_input_data = dnn_input->data; | |
| 261 | + av_assert0(dnn_input->dt == DNN_UINT8); | |
| 262 | + for (int i = 0; i < frame->height; i++) { | |
| 263 | + for(int j = 0; j < frame->width; j++) { | |
| 264 | + int k = i * frame->linesize[0] + j; | |
| 265 | + int t = i * frame->width + j; | |
| 266 | + dnn_input_data[t] = frame->data[0][k]; | |
| 267 | + } | |
| 229 | 268 | } |
| 230 | 269 | } |
| 270 | + return 0; | |
| 271 | + case AV_PIX_FMT_GRAYF32: | |
| 272 | + { | |
| 273 | + float *dnn_input_data = dnn_input->data; | |
| 274 | + av_assert0(dnn_input->dt == DNN_FLOAT); | |
| 275 | + for (int i = 0; i < frame->height; i++) { | |
| 276 | + for(int j = 0; j < frame->width; j++) { | |
| 277 | + int k = i * frame->linesize[0] + j * sizeof(float); | |
| 278 | + int t = i * frame->width + j; | |
| 279 | + dnn_input_data[t] = *(float*)(frame->data[0] + k); | |
| 280 | + } | |
| 281 | + } | |
| 282 | + } | |
| 283 | + return 0; | |
| 284 | + default: | |
| 285 | + return AVERROR(EIO); | |
| 231 | 286 | } |
| 232 | 287 | |
| 233 | 288 | return 0; |
| @@ -235,28 +290,58 @@ static int copy_from_frame_to_dnn(DNNData *dnn_input, const AVFrame *frame) · 1 unmodified line | ||
| 235 | 290 | |
| 236 | 291 | static int copy_from_dnn_to_frame(AVFrame *frame, const DNNData *dnn_output) |
| 237 | 292 | { |
| 238 | − // extend this function to support more formats | |
| 239 | − av_assert0(frame->format == AV_PIX_FMT_RGB24 || frame->format == AV_PIX_FMT_BGR24); | |
| 240 | − | |
| 241 | − if (dnn_output->dt == DNN_FLOAT) { | |
| 242 | − float *dnn_output_data = dnn_output->data; | |
| 243 | − for (int i = 0; i < frame->height; i++) { | |
| 244 | − for(int j = 0; j < frame->width * 3; j++) { | |
| 245 | − int k = i * frame->linesize[0] + j; | |
| 246 | − int t = i * frame->width * 3 + j; | |
| 247 | − frame->data[0][k] = av_clip_uintp2((int)(dnn_output_data[t] * 255.0f), 8); | |
| 293 | + switch (frame->format) { | |
| 294 | + case AV_PIX_FMT_RGB24: | |
| 295 | + case AV_PIX_FMT_BGR24: | |
| 296 | + if (dnn_output->dt == DNN_FLOAT) { | |
| 297 | + float *dnn_output_data = dnn_output->data; | |
| 298 | + for (int i = 0; i < frame->height; i++) { | |
| 299 | + for(int j = 0; j < frame->width * 3; j++) { | |
| 300 | + int k = i * frame->linesize[0] + j; | |
| 301 | + int t = i * frame->width * 3 + j; | |
| 302 | + frame->data[0][k] = av_clip_uintp2((int)(dnn_output_data[t] * 255.0f), 8); | |
| 303 | + } | |
| 304 | + } | |
| 305 | + } else { | |
| 306 | + uint8_t *dnn_output_data = dnn_output->data; | |
| 307 | + av_assert0(dnn_output->dt == DNN_UINT8); | |
| 308 | + for (int i = 0; i < frame->height; i++) { | |
| 309 | + for(int j = 0; j < frame->width * 3; j++) { | |
| 310 | + int k = i * frame->linesize[0] + j; | |
| 311 | + int t = i * frame->width * 3 + j; | |
| 312 | + frame->data[0][k] = dnn_output_data[t]; | |
| 313 | + } | |
| 314 | + } | |
| 315 | + } | |
| 316 | + return 0; | |
| 317 | + case AV_PIX_FMT_GRAY8: | |
| 318 | + { | |
| 319 | + uint8_t *dnn_output_data = dnn_output->data; | |
| 320 | + av_assert0(dnn_output->dt == DNN_UINT8); | |
| 321 | + for (int i = 0; i < frame->height; i++) { | |
| 322 | + for(int j = 0; j < frame->width; j++) { | |
| 323 | + int k = i * frame->linesize[0] + j; | |
| 324 | + int t = i * frame->width + j; | |
| 325 | + frame->data[0][k] = dnn_output_data[t]; | |
| 326 | + } | |
| 248 | 327 | } |
| 249 | 328 | } |
| 250 | − } else { | |
| 251 | − uint8_t *dnn_output_data = dnn_output->data; | |
| 252 | − av_assert0(dnn_output->dt == DNN_UINT8); | |
| 253 | − for (int i = 0; i < frame->height; i++) { | |
| 254 | − for(int j = 0; j < frame->width * 3; j++) { | |
| 255 | − int k = i * frame->linesize[0] + j; | |
| 256 | − int t = i * frame->width * 3 + j; | |
| 257 | − frame->data[0][k] = dnn_output_data[t]; | |
| 329 | + return 0; | |
| 330 | + case AV_PIX_FMT_GRAYF32: | |
| 331 | + { | |
| 332 | + float *dnn_output_data = dnn_output->data; | |
| 333 | + av_assert0(dnn_output->dt == DNN_FLOAT); | |
| 334 | + for (int i = 0; i < frame->height; i++) { | |
| 335 | + for(int j = 0; j < frame->width; j++) { | |
| 336 | + int k = i * frame->linesize[0] + j * sizeof(float); | |
| 337 | + int t = i * frame->width + j; | |
| 338 | + *(float*)(frame->data[0] + k) = dnn_output_data[t]; | |
| 339 | + } | |
| 258 | 340 | } |
| 259 | 341 | } |
| 342 | + return 0; | |
| 343 | + default: | |
| 344 | + return AVERROR(EIO); | |
| 260 | 345 | } |
| 261 | 346 | |
| 262 | 347 | return 0; |
| @@ -278,7 +363,6 @@ static int filter_frame(AVFilterLink *inlink, AVFrame *in) · 15 unmodified lines | ||
| 278 | 363 | av_frame_free(&in); |
| 279 | 364 | return AVERROR(EIO); |
| 280 | 365 | } |
| 281 | − av_assert0(ctx->output.channels == 3); | |
| 282 | 366 | |
| 283 | 367 | out = ff_get_video_buffer(outlink, outlink->w, outlink->h); |
| 284 | 368 | if (!out) { |