History

vf_dnn_processing: add support for more formats gray8 and grayf32

The following is a python script to halve the value of the gray image. It demos how to setup and execute dnn model with python+tensorflow. It also generates .pb file which will be used by ffmpeg. import tensorflow as tf import numpy as np from skimage import color from skimage import io in_img = io.imread('input.jpg') in_img = color.rgb2gray(in_img) io.imsave('ori_gray.jpg', np.squeeze(in_img)) in_data = np.expand_dims(in_img, axis=0) in_data = np.expand_dims(in_data, axis=3) filter_data = np.array([0.5]).reshape(1,1,1,1).astype(np.float32) filter = tf.Variable(filter_data) x = tf.placeholder(tf.float32, shape=[1, None, None, 1], name='dnn_in') y = tf.nn.conv2d(x, filter, strides=[1, 1, 1, 1], padding='VALID', name='dnn_out') sess=tf.Session() sess.run(tf.global_variables_initializer()) graph_def = tf.graph_util.convert_variables_to_constants(sess, sess.graph_def, ['dnn_out']) tf.train.write_graph(graph_def, '.', 'halve_gray_float.pb', as_text=False) print("halve_gray_float.pb generated, please use \ path_to_ffmpeg/tools/python/convert.py to generate halve_gray_float.model\n") output = sess.run(y, feed_dict={x: in_data}) output = output * 255.0 output = output.astype(np.uint8) io.imsave("out.jpg", np.squeeze(output)) To do the same thing with ffmpeg: - generate halve_gray_float.pb with the above script - generate halve_gray_float.model with tools/python/convert.py - try with following commands ./ffmpeg -i input.jpg -vf format=grayf32,dnn_processing=model=halve_gray_float.model:input=dnn_in:output=dnn_out:dnn_backend=native out.native.png ./ffmpeg -i input.jpg -vf format=grayf32,dnn_processing=model=halve_gray_float.pb:input=dnn_in:output=dnn_out:dnn_backend=tensorflow out.tf.png

Guo, Yejun committed 6 years ago

Commit 37d24a61 parent 04e6f8a2 files · +132 −42Browse files

Unified · Split

2 files
  1. doc/filters.texi +6 −0
  2. libavfilter/vf_dnn_processing.c +126 −42
doc/filters.texi+6−0
@@ -9115,6 +9115,12 @@ Halve the red channle of the frame with format rgb24: · 9114 unmodified lines
91159115 ffmpeg -i input.jpg -vf format=rgb24,dnn_processing=model=halve_first_channel.model:input=dnn_in:output=dnn_out:dnn_backend=native out.native.png
91169116 @end example
91179117
9118+@item
9119+Halve the pixel value of the frame with format gray32f:
9120+@example
9121+ffmpeg -i input.jpg -vf format=grayf32,dnn_processing=model=halve_gray_float.model:input=dnn_in:output=dnn_out:dnn_backend=native -y out.native.png
9122+@end example
9123+
91189124 @end itemize
91199125
91209126 @section drawbox
libavfilter/vf_dnn_processing.c+126−42
@@ -104,12 +104,20 @@ static int query_formats(AVFilterContext *context) · 103 unmodified lines
104104 {
105105 static const enum AVPixelFormat pix_fmts[] = {
106106 AV_PIX_FMT_RGB24, AV_PIX_FMT_BGR24,
107+ AV_PIX_FMT_GRAY8, AV_PIX_FMT_GRAYF32,
107108 AV_PIX_FMT_NONE
108109 };
109110 AVFilterFormats *fmts_list = ff_make_format_list(pix_fmts);
110111 return ff_set_common_formats(context, fmts_list);
111112 }
112113
114+#define LOG_FORMAT_CHANNEL_MISMATCH() \
115+ av_log(ctx, AV_LOG_ERROR, \
116+ "the frame's format %s does not match " \
117+ "the model input channel %d\n", \
118+ av_get_pix_fmt_name(fmt), \
119+ model_input->channels);
120+
113121 static int check_modelinput_inlink(const DNNData *model_input, const AVFilterLink *inlink)
114122 {
115123 AVFilterContext *ctx = inlink->dst;
@@ -131,17 +139,34 @@ static int check_modelinput_inlink(const DNNData *model_input, const AVFilterLin · 15 unmodified lines
131139 case AV_PIX_FMT_RGB24:
132140 case AV_PIX_FMT_BGR24:
133141 if (model_input->channels != 3) {
134− av_log(ctx, AV_LOG_ERROR, "the frame's input format %s does not match "
135− "the model input channels %d\n",
136− av_get_pix_fmt_name(fmt),
137− model_input->channels);
142+ LOG_FORMAT_CHANNEL_MISMATCH();
138143 return AVERROR(EIO);
139144 }
140145 if (model_input->dt != DNN_FLOAT && model_input->dt != DNN_UINT8) {
141146 av_log(ctx, AV_LOG_ERROR, "only support dnn models with input data type as float32 and uint8.\n");
142147 return AVERROR(EIO);
143148 }
144− break;
149+ return 0;
150+ case AV_PIX_FMT_GRAY8:
151+ if (model_input->channels != 1) {
152+ LOG_FORMAT_CHANNEL_MISMATCH();
153+ return AVERROR(EIO);
154+ }
155+ if (model_input->dt != DNN_UINT8) {
156+ av_log(ctx, AV_LOG_ERROR, "only support dnn models with input data type uint8.\n");
157+ return AVERROR(EIO);
158+ }
159+ return 0;
160+ case AV_PIX_FMT_GRAYF32:
161+ if (model_input->channels != 1) {
162+ LOG_FORMAT_CHANNEL_MISMATCH();
163+ return AVERROR(EIO);
164+ }
165+ if (model_input->dt != DNN_FLOAT) {
166+ av_log(ctx, AV_LOG_ERROR, "only support dnn models with input data type float32.\n");
167+ return AVERROR(EIO);
168+ }
169+ return 0;
145170 default:
146171 av_log(ctx, AV_LOG_ERROR, "%s not supported.\n", av_get_pix_fmt_name(fmt));
147172 return AVERROR(EIO);
@@ -206,28 +231,58 @@ static int config_output(AVFilterLink *outlink) · 58 unmodified lines
206231
207232 static int copy_from_frame_to_dnn(DNNData *dnn_input, const AVFrame *frame)
208233 {
209− // extend this function to support more formats
210− av_assert0(frame->format == AV_PIX_FMT_RGB24 || frame->format == AV_PIX_FMT_BGR24);
211−
212− if (dnn_input->dt == DNN_FLOAT) {
213− float *dnn_input_data = dnn_input->data;
214− for (int i = 0; i < frame->height; i++) {
215− for(int j = 0; j < frame->width * 3; j++) {
216− int k = i * frame->linesize[0] + j;
217− int t = i * frame->width * 3 + j;
218− dnn_input_data[t] = frame->data[0][k] / 255.0f;
234+ switch (frame->format) {
235+ case AV_PIX_FMT_RGB24:
236+ case AV_PIX_FMT_BGR24:
237+ if (dnn_input->dt == DNN_FLOAT) {
238+ float *dnn_input_data = dnn_input->data;
239+ for (int i = 0; i < frame->height; i++) {
240+ for(int j = 0; j < frame->width * 3; j++) {
241+ int k = i * frame->linesize[0] + j;
242+ int t = i * frame->width * 3 + j;
243+ dnn_input_data[t] = frame->data[0][k] / 255.0f;
244+ }
245+ }
246+ } else {
247+ uint8_t *dnn_input_data = dnn_input->data;
248+ av_assert0(dnn_input->dt == DNN_UINT8);
249+ for (int i = 0; i < frame->height; i++) {
250+ for(int j = 0; j < frame->width * 3; j++) {
251+ int k = i * frame->linesize[0] + j;
252+ int t = i * frame->width * 3 + j;
253+ dnn_input_data[t] = frame->data[0][k];
254+ }
219255 }
220256 }
221− } else {
222− uint8_t *dnn_input_data = dnn_input->data;
223− av_assert0(dnn_input->dt == DNN_UINT8);
224− for (int i = 0; i < frame->height; i++) {
225− for(int j = 0; j < frame->width * 3; j++) {
226− int k = i * frame->linesize[0] + j;
227− int t = i * frame->width * 3 + j;
228− dnn_input_data[t] = frame->data[0][k];
257+ return 0;
258+ case AV_PIX_FMT_GRAY8:
259+ {
260+ uint8_t *dnn_input_data = dnn_input->data;
261+ av_assert0(dnn_input->dt == DNN_UINT8);
262+ for (int i = 0; i < frame->height; i++) {
263+ for(int j = 0; j < frame->width; j++) {
264+ int k = i * frame->linesize[0] + j;
265+ int t = i * frame->width + j;
266+ dnn_input_data[t] = frame->data[0][k];
267+ }
229268 }
230269 }
270+ return 0;
271+ case AV_PIX_FMT_GRAYF32:
272+ {
273+ float *dnn_input_data = dnn_input->data;
274+ av_assert0(dnn_input->dt == DNN_FLOAT);
275+ for (int i = 0; i < frame->height; i++) {
276+ for(int j = 0; j < frame->width; j++) {
277+ int k = i * frame->linesize[0] + j * sizeof(float);
278+ int t = i * frame->width + j;
279+ dnn_input_data[t] = *(float*)(frame->data[0] + k);
280+ }
281+ }
282+ }
283+ return 0;
284+ default:
285+ return AVERROR(EIO);
231286 }
232287
233288 return 0;
@@ -235,28 +290,58 @@ static int copy_from_frame_to_dnn(DNNData *dnn_input, const AVFrame *frame) · 1 unmodified line
235290
236291 static int copy_from_dnn_to_frame(AVFrame *frame, const DNNData *dnn_output)
237292 {
238− // extend this function to support more formats
239− av_assert0(frame->format == AV_PIX_FMT_RGB24 || frame->format == AV_PIX_FMT_BGR24);
240−
241− if (dnn_output->dt == DNN_FLOAT) {
242− float *dnn_output_data = dnn_output->data;
243− for (int i = 0; i < frame->height; i++) {
244− for(int j = 0; j < frame->width * 3; j++) {
245− int k = i * frame->linesize[0] + j;
246− int t = i * frame->width * 3 + j;
247− frame->data[0][k] = av_clip_uintp2((int)(dnn_output_data[t] * 255.0f), 8);
293+ switch (frame->format) {
294+ case AV_PIX_FMT_RGB24:
295+ case AV_PIX_FMT_BGR24:
296+ if (dnn_output->dt == DNN_FLOAT) {
297+ float *dnn_output_data = dnn_output->data;
298+ for (int i = 0; i < frame->height; i++) {
299+ for(int j = 0; j < frame->width * 3; j++) {
300+ int k = i * frame->linesize[0] + j;
301+ int t = i * frame->width * 3 + j;
302+ frame->data[0][k] = av_clip_uintp2((int)(dnn_output_data[t] * 255.0f), 8);
303+ }
304+ }
305+ } else {
306+ uint8_t *dnn_output_data = dnn_output->data;
307+ av_assert0(dnn_output->dt == DNN_UINT8);
308+ for (int i = 0; i < frame->height; i++) {
309+ for(int j = 0; j < frame->width * 3; j++) {
310+ int k = i * frame->linesize[0] + j;
311+ int t = i * frame->width * 3 + j;
312+ frame->data[0][k] = dnn_output_data[t];
313+ }
314+ }
315+ }
316+ return 0;
317+ case AV_PIX_FMT_GRAY8:
318+ {
319+ uint8_t *dnn_output_data = dnn_output->data;
320+ av_assert0(dnn_output->dt == DNN_UINT8);
321+ for (int i = 0; i < frame->height; i++) {
322+ for(int j = 0; j < frame->width; j++) {
323+ int k = i * frame->linesize[0] + j;
324+ int t = i * frame->width + j;
325+ frame->data[0][k] = dnn_output_data[t];
326+ }
248327 }
249328 }
250− } else {
251− uint8_t *dnn_output_data = dnn_output->data;
252− av_assert0(dnn_output->dt == DNN_UINT8);
253− for (int i = 0; i < frame->height; i++) {
254− for(int j = 0; j < frame->width * 3; j++) {
255− int k = i * frame->linesize[0] + j;
256− int t = i * frame->width * 3 + j;
257− frame->data[0][k] = dnn_output_data[t];
329+ return 0;
330+ case AV_PIX_FMT_GRAYF32:
331+ {
332+ float *dnn_output_data = dnn_output->data;
333+ av_assert0(dnn_output->dt == DNN_FLOAT);
334+ for (int i = 0; i < frame->height; i++) {
335+ for(int j = 0; j < frame->width; j++) {
336+ int k = i * frame->linesize[0] + j * sizeof(float);
337+ int t = i * frame->width + j;
338+ *(float*)(frame->data[0] + k) = dnn_output_data[t];
339+ }
258340 }
259341 }
342+ return 0;
343+ default:
344+ return AVERROR(EIO);
260345 }
261346
262347 return 0;
@@ -278,7 +363,6 @@ static int filter_frame(AVFilterLink *inlink, AVFrame *in) · 15 unmodified lines
278363 av_frame_free(&in);
279364 return AVERROR(EIO);
280365 }
281− av_assert0(ctx->output.channels == 3);
282366
283367 out = ff_get_video_buffer(outlink, outlink->w, outlink->h);
284368 if (!out) {

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t