History

vf_dnn_processing: add support for more formats gray8 and grayf32

The following is a python script to halve the value of the gray image. It demos how to setup and execute dnn model with python+tensorflow. It also generates .pb file which will be used by ffmpeg. import tensorflow as tf import numpy as np from skimage import color from skimage import io in_img = io.imread('input.jpg') in_img = color.rgb2gray(in_img) io.imsave('ori_gray.jpg', np.squeeze(in_img)) in_data = np.expand_dims(in_img, axis=0) in_data = np.expand_dims(in_data, axis=3) filter_data = np.array([0.5]).reshape(1,1,1,1).astype(np.float32) filter = tf.Variable(filter_data) x = tf.placeholder(tf.float32, shape=[1, None, None, 1], name='dnn_in') y = tf.nn.conv2d(x, filter, strides=[1, 1, 1, 1], padding='VALID', name='dnn_out') sess=tf.Session() sess.run(tf.global_variables_initializer()) graph_def = tf.graph_util.convert_variables_to_constants(sess, sess.graph_def, ['dnn_out']) tf.train.write_graph(graph_def, '.', 'halve_gray_float.pb', as_text=False) print("halve_gray_float.pb generated, please use \ path_to_ffmpeg/tools/python/convert.py to generate halve_gray_float.model\n") output = sess.run(y, feed_dict={x: in_data}) output = output * 255.0 output = output.astype(np.uint8) io.imsave("out.jpg", np.squeeze(output)) To do the same thing with ffmpeg: - generate halve_gray_float.pb with the above script - generate halve_gray_float.model with tools/python/convert.py - try with following commands ./ffmpeg -i input.jpg -vf format=grayf32,dnn_processing=model=halve_gray_float.model:input=dnn_in:output=dnn_out:dnn_backend=native out.native.png ./ffmpeg -i input.jpg -vf format=grayf32,dnn_processing=model=halve_gray_float.pb:input=dnn_in:output=dnn_out:dnn_backend=tensorflow out.tf.png

Guo, Yejun committed 6 years ago

Commit 37d24a61 parent 04e6f8a2 files · +132 −42Browse files

Showing the changes to libavfilter (1 of 2 files). Show all

Unified · Split

libavfilter/vf_dnn_processing.c+126−42
@@ -104,12 +104,20 @@ static int query_formats(AVFilterContext *context) · 103 unmodified lines
104{104{
105 static const enum AVPixelFormat pix_fmts[] = {105 static const enum AVPixelFormat pix_fmts[] = {
106 AV_PIX_FMT_RGB24, AV_PIX_FMT_BGR24,106 AV_PIX_FMT_RGB24, AV_PIX_FMT_BGR24,
107 AV_PIX_FMT_GRAY8, AV_PIX_FMT_GRAYF32,
107 AV_PIX_FMT_NONE108 AV_PIX_FMT_NONE
108 };109 };
109 AVFilterFormats *fmts_list = ff_make_format_list(pix_fmts);110 AVFilterFormats *fmts_list = ff_make_format_list(pix_fmts);
110 return ff_set_common_formats(context, fmts_list);111 return ff_set_common_formats(context, fmts_list);
111}112}
112 113
114#define LOG_FORMAT_CHANNEL_MISMATCH() \
115 av_log(ctx, AV_LOG_ERROR, \
116 "the frame's format %s does not match " \
117 "the model input channel %d\n", \
118 av_get_pix_fmt_name(fmt), \
119 model_input->channels);
120
113static int check_modelinput_inlink(const DNNData *model_input, const AVFilterLink *inlink)121static int check_modelinput_inlink(const DNNData *model_input, const AVFilterLink *inlink)
114{122{
115 AVFilterContext *ctx = inlink->dst;123 AVFilterContext *ctx = inlink->dst;
@@ -131,17 +139,34 @@ static int check_modelinput_inlink(const DNNData *model_input, const AVFilterLin · 15 unmodified lines
131 case AV_PIX_FMT_RGB24:139 case AV_PIX_FMT_RGB24:
132 case AV_PIX_FMT_BGR24:140 case AV_PIX_FMT_BGR24:
133 if (model_input->channels != 3) {141 if (model_input->channels != 3) {
134 av_log(ctx, AV_LOG_ERROR, "the frame's input format %s does not match "142 LOG_FORMAT_CHANNEL_MISMATCH();
135 "the model input channels %d\n",
136 av_get_pix_fmt_name(fmt),
137 model_input->channels);
138 return AVERROR(EIO);143 return AVERROR(EIO);
139 }144 }
140 if (model_input->dt != DNN_FLOAT && model_input->dt != DNN_UINT8) {145 if (model_input->dt != DNN_FLOAT && model_input->dt != DNN_UINT8) {
141 av_log(ctx, AV_LOG_ERROR, "only support dnn models with input data type as float32 and uint8.\n");146 av_log(ctx, AV_LOG_ERROR, "only support dnn models with input data type as float32 and uint8.\n");
142 return AVERROR(EIO);147 return AVERROR(EIO);
143 }148 }
144 break;149 return 0;
150 case AV_PIX_FMT_GRAY8:
151 if (model_input->channels != 1) {
152 LOG_FORMAT_CHANNEL_MISMATCH();
153 return AVERROR(EIO);
154 }
155 if (model_input->dt != DNN_UINT8) {
156 av_log(ctx, AV_LOG_ERROR, "only support dnn models with input data type uint8.\n");
157 return AVERROR(EIO);
158 }
159 return 0;
160 case AV_PIX_FMT_GRAYF32:
161 if (model_input->channels != 1) {
162 LOG_FORMAT_CHANNEL_MISMATCH();
163 return AVERROR(EIO);
164 }
165 if (model_input->dt != DNN_FLOAT) {
166 av_log(ctx, AV_LOG_ERROR, "only support dnn models with input data type float32.\n");
167 return AVERROR(EIO);
168 }
169 return 0;
145 default:170 default:
146 av_log(ctx, AV_LOG_ERROR, "%s not supported.\n", av_get_pix_fmt_name(fmt));171 av_log(ctx, AV_LOG_ERROR, "%s not supported.\n", av_get_pix_fmt_name(fmt));
147 return AVERROR(EIO);172 return AVERROR(EIO);
@@ -206,28 +231,58 @@ static int config_output(AVFilterLink *outlink) · 58 unmodified lines
206 231
207static int copy_from_frame_to_dnn(DNNData *dnn_input, const AVFrame *frame)232static int copy_from_frame_to_dnn(DNNData *dnn_input, const AVFrame *frame)
208{233{
209 // extend this function to support more formats234 switch (frame->format) {
210 av_assert0(frame->format == AV_PIX_FMT_RGB24 || frame->format == AV_PIX_FMT_BGR24);235 case AV_PIX_FMT_RGB24:
211 236 case AV_PIX_FMT_BGR24:
212 if (dnn_input->dt == DNN_FLOAT) {237 if (dnn_input->dt == DNN_FLOAT) {
213 float *dnn_input_data = dnn_input->data;238 float *dnn_input_data = dnn_input->data;
214 for (int i = 0; i < frame->height; i++) {239 for (int i = 0; i < frame->height; i++) {
215 for(int j = 0; j < frame->width * 3; j++) {240 for(int j = 0; j < frame->width * 3; j++) {
216 int k = i * frame->linesize[0] + j;241 int k = i * frame->linesize[0] + j;
217 int t = i * frame->width * 3 + j;242 int t = i * frame->width * 3 + j;
218 dnn_input_data[t] = frame->data[0][k] / 255.0f;243 dnn_input_data[t] = frame->data[0][k] / 255.0f;
244 }
245 }
246 } else {
247 uint8_t *dnn_input_data = dnn_input->data;
248 av_assert0(dnn_input->dt == DNN_UINT8);
249 for (int i = 0; i < frame->height; i++) {
250 for(int j = 0; j < frame->width * 3; j++) {
251 int k = i * frame->linesize[0] + j;
252 int t = i * frame->width * 3 + j;
253 dnn_input_data[t] = frame->data[0][k];
254 }
219 }255 }
220 }256 }
221 } else {257 return 0;
222 uint8_t *dnn_input_data = dnn_input->data;258 case AV_PIX_FMT_GRAY8:
223 av_assert0(dnn_input->dt == DNN_UINT8);259 {
224 for (int i = 0; i < frame->height; i++) {260 uint8_t *dnn_input_data = dnn_input->data;
225 for(int j = 0; j < frame->width * 3; j++) {261 av_assert0(dnn_input->dt == DNN_UINT8);
226 int k = i * frame->linesize[0] + j;262 for (int i = 0; i < frame->height; i++) {
227 int t = i * frame->width * 3 + j;263 for(int j = 0; j < frame->width; j++) {
228 dnn_input_data[t] = frame->data[0][k];264 int k = i * frame->linesize[0] + j;
265 int t = i * frame->width + j;
266 dnn_input_data[t] = frame->data[0][k];
267 }
229 }268 }
230 }269 }
270 return 0;
271 case AV_PIX_FMT_GRAYF32:
272 {
273 float *dnn_input_data = dnn_input->data;
274 av_assert0(dnn_input->dt == DNN_FLOAT);
275 for (int i = 0; i < frame->height; i++) {
276 for(int j = 0; j < frame->width; j++) {
277 int k = i * frame->linesize[0] + j * sizeof(float);
278 int t = i * frame->width + j;
279 dnn_input_data[t] = *(float*)(frame->data[0] + k);
280 }
281 }
282 }
283 return 0;
284 default:
285 return AVERROR(EIO);
231 }286 }
232 287
233 return 0;288 return 0;
@@ -235,28 +290,58 @@ static int copy_from_frame_to_dnn(DNNData *dnn_input, const AVFrame *frame) · 1 unmodified line
235 290
236static int copy_from_dnn_to_frame(AVFrame *frame, const DNNData *dnn_output)291static int copy_from_dnn_to_frame(AVFrame *frame, const DNNData *dnn_output)
237{292{
238 // extend this function to support more formats293 switch (frame->format) {
239 av_assert0(frame->format == AV_PIX_FMT_RGB24 || frame->format == AV_PIX_FMT_BGR24);294 case AV_PIX_FMT_RGB24:
240 295 case AV_PIX_FMT_BGR24:
241 if (dnn_output->dt == DNN_FLOAT) {296 if (dnn_output->dt == DNN_FLOAT) {
242 float *dnn_output_data = dnn_output->data;297 float *dnn_output_data = dnn_output->data;
243 for (int i = 0; i < frame->height; i++) {298 for (int i = 0; i < frame->height; i++) {
244 for(int j = 0; j < frame->width * 3; j++) {299 for(int j = 0; j < frame->width * 3; j++) {
245 int k = i * frame->linesize[0] + j;300 int k = i * frame->linesize[0] + j;
246 int t = i * frame->width * 3 + j;301 int t = i * frame->width * 3 + j;
247 frame->data[0][k] = av_clip_uintp2((int)(dnn_output_data[t] * 255.0f), 8);302 frame->data[0][k] = av_clip_uintp2((int)(dnn_output_data[t] * 255.0f), 8);
303 }
304 }
305 } else {
306 uint8_t *dnn_output_data = dnn_output->data;
307 av_assert0(dnn_output->dt == DNN_UINT8);
308 for (int i = 0; i < frame->height; i++) {
309 for(int j = 0; j < frame->width * 3; j++) {
310 int k = i * frame->linesize[0] + j;
311 int t = i * frame->width * 3 + j;
312 frame->data[0][k] = dnn_output_data[t];
313 }
314 }
315 }
316 return 0;
317 case AV_PIX_FMT_GRAY8:
318 {
319 uint8_t *dnn_output_data = dnn_output->data;
320 av_assert0(dnn_output->dt == DNN_UINT8);
321 for (int i = 0; i < frame->height; i++) {
322 for(int j = 0; j < frame->width; j++) {
323 int k = i * frame->linesize[0] + j;
324 int t = i * frame->width + j;
325 frame->data[0][k] = dnn_output_data[t];
326 }
248 }327 }
249 }328 }
250 } else {329 return 0;
251 uint8_t *dnn_output_data = dnn_output->data;330 case AV_PIX_FMT_GRAYF32:
252 av_assert0(dnn_output->dt == DNN_UINT8);331 {
253 for (int i = 0; i < frame->height; i++) {332 float *dnn_output_data = dnn_output->data;
254 for(int j = 0; j < frame->width * 3; j++) {333 av_assert0(dnn_output->dt == DNN_FLOAT);
255 int k = i * frame->linesize[0] + j;334 for (int i = 0; i < frame->height; i++) {
256 int t = i * frame->width * 3 + j;335 for(int j = 0; j < frame->width; j++) {
257 frame->data[0][k] = dnn_output_data[t];336 int k = i * frame->linesize[0] + j * sizeof(float);
337 int t = i * frame->width + j;
338 *(float*)(frame->data[0] + k) = dnn_output_data[t];
339 }
258 }340 }
259 }341 }
342 return 0;
343 default:
344 return AVERROR(EIO);
260 }345 }
261 346
262 return 0;347 return 0;
@@ -278,7 +363,6 @@ static int filter_frame(AVFilterLink *inlink, AVFrame *in) · 15 unmodified lines
278 av_frame_free(&in);363 av_frame_free(&in);
279 return AVERROR(EIO);364 return AVERROR(EIO);
280 }365 }
281 av_assert0(ctx->output.channels == 3);
282 366
283 out = ff_get_video_buffer(outlink, outlink->w, outlink->h);367 out = ff_get_video_buffer(outlink, outlink->w, outlink->h);
284 if (!out) {368 if (!out) {

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t