History

dnn: add a new interface DNNModel.get_output

for some cases (for example, super resolution), the DNN model changes the frame size which impacts the filter behavior, so the filter needs to know the out frame size at very beginning. Currently, the filter reuses DNNModule.execute_model to query the out frame size, it is not clear from interface perspective, so add a new explict interface DNNModel.get_output for such query.

Guo, Yejun committed 6 years ago

Commit e71d73b1 parent fce3e3e6 files · +185 −58Browse files

Unified · Split

6 files
  1. libavfilter/dnn/dnn_backend_native.c +56 −10
  2. libavfilter/dnn/dnn_backend_openvino.c +56 −10
  3. libavfilter/dnn/dnn_backend_tf.c +56 −10
  4. libavfilter/dnn_interface.h +3 −0
  5. libavfilter/vf_dnn_processing.c +4 −13
  6. libavfilter/vf_sr.c +10 −15
libavfilter/dnn/dnn_backend_native.c+56−10
@@ -44,6 +44,10 @@ const AVClass dnn_native_class = { · 43 unmodified lines
4444 .category = AV_CLASS_CATEGORY_FILTER,
4545 };
4646
47+static DNNReturnType execute_model_native(const DNNModel *model, const char *input_name, AVFrame *in_frame,
48+ const char **output_names, uint32_t nb_output, AVFrame *out_frame,
49+ int do_ioproc);
50+
4751 static DNNReturnType get_input_native(void *model, DNNData *input, const char *input_name)
4852 {
4953 NativeModel *native_model = (NativeModel *)model;
@@ -70,6 +74,25 @@ static DNNReturnType get_input_native(void *model, DNNData *input, const char *i · 20 unmodified lines
7074 return DNN_ERROR;
7175 }
7276
77+static DNNReturnType get_output_native(void *model, const char *input_name, int input_width, int input_height,
78+ const char *output_name, int *output_width, int *output_height)
79+{
80+ DNNReturnType ret;
81+ NativeModel *native_model = (NativeModel *)model;
82+ AVFrame *in_frame = av_frame_alloc();
83+ AVFrame *out_frame = av_frame_alloc();
84+ in_frame->width = input_width;
85+ in_frame->height = input_height;
86+
87+ ret = execute_model_native(native_model->model, input_name, in_frame, &output_name, 1, out_frame, 0);
88+ *output_width = out_frame->width;
89+ *output_height = out_frame->height;
90+
91+ av_frame_free(&out_frame);
92+ av_frame_free(&in_frame);
93+ return ret;
94+}
95+
7396 // Loads model and its parameters that are stored in a binary file with following structure:
7497 // layers_num,layer_type,layer_parameterss,layer_type,layer_parameters...
7598 // For CONV layer: activation_function, input_num, output_num, kernel_size, kernel, biases
@@ -216,6 +239,7 @@ DNNModel *ff_dnn_load_model_native(const char *model_filename, const char *optio · 140 unmodified lines
216239 }
217240
218241 model->get_input = &get_input_native;
242+ model->get_output = &get_output_native;
219243 model->userdata = userdata;
220244
221245 return model;
@@ -226,8 +250,9 @@ fail: · 4 unmodified lines
226250 return NULL;
227251 }
228252
229−DNNReturnType ff_dnn_execute_model_native(const DNNModel *model, const char *input_name, AVFrame *in_frame,
230− const char **output_names, uint32_t nb_output, AVFrame *out_frame)
253+static DNNReturnType execute_model_native(const DNNModel *model, const char *input_name, AVFrame *in_frame,
254+ const char **output_names, uint32_t nb_output, AVFrame *out_frame,
255+ int do_ioproc)
231256 {
232257 NativeModel *native_model = (NativeModel *)model->model;
233258 NativeContext *ctx = &native_model->ctx;
@@ -276,10 +301,12 @@ DNNReturnType ff_dnn_execute_model_native(const DNNModel *model, const char *inp · 42 unmodified lines
276301 input.channels = oprd->dims[3];
277302 input.data = oprd->data;
278303 input.dt = oprd->data_type;
279− if (native_model->model->pre_proc != NULL) {
280− native_model->model->pre_proc(in_frame, &input, native_model->model->userdata);
281− } else {
282− proc_from_frame_to_dnn(in_frame, &input, ctx);
304+ if (do_ioproc) {
305+ if (native_model->model->pre_proc != NULL) {
306+ native_model->model->pre_proc(in_frame, &input, native_model->model->userdata);
307+ } else {
308+ proc_from_frame_to_dnn(in_frame, &input, ctx);
309+ }
283310 }
284311
285312 if (nb_output != 1) {
@@ -322,21 +349,40 @@ DNNReturnType ff_dnn_execute_model_native(const DNNModel *model, const char *inp · 36 unmodified lines
322349 output.channels = oprd->dims[3];
323350 output.dt = oprd->data_type;
324351
325− if (out_frame->width != output.width || out_frame->height != output.height) {
326− out_frame->width = output.width;
327− out_frame->height = output.height;
328− } else {
352+ if (do_ioproc) {
329353 if (native_model->model->post_proc != NULL) {
330354 native_model->model->post_proc(out_frame, &output, native_model->model->userdata);
331355 } else {
332356 proc_from_dnn_to_frame(out_frame, &output, ctx);
333357 }
358+ } else {
359+ out_frame->width = output.width;
360+ out_frame->height = output.height;
334361 }
335362 }
336363
337364 return DNN_SUCCESS;
338365 }
339366
367+DNNReturnType ff_dnn_execute_model_native(const DNNModel *model, const char *input_name, AVFrame *in_frame,
368+ const char **output_names, uint32_t nb_output, AVFrame *out_frame)
369+{
370+ NativeModel *native_model = (NativeModel *)model->model;
371+ NativeContext *ctx = &native_model->ctx;
372+
373+ if (!in_frame) {
374+ av_log(ctx, AV_LOG_ERROR, "in frame is NULL when execute model.\n");
375+ return DNN_ERROR;
376+ }
377+
378+ if (!out_frame) {
379+ av_log(ctx, AV_LOG_ERROR, "out frame is NULL when execute model.\n");
380+ return DNN_ERROR;
381+ }
382+
383+ return execute_model_native(model, input_name, in_frame, output_names, nb_output, out_frame, 1);
384+}
385+
340386 int32_t calculate_operand_dims_count(const DnnOperand *oprd)
341387 {
342388 int32_t result = 1;
libavfilter/dnn/dnn_backend_openvino.c+56−10
@@ -63,6 +63,10 @@ static const AVOption dnn_openvino_options[] = { · 62 unmodified lines
6363
6464 AVFILTER_DEFINE_CLASS(dnn_openvino);
6565
66+static DNNReturnType execute_model_ov(const DNNModel *model, const char *input_name, AVFrame *in_frame,
67+ const char **output_names, uint32_t nb_output, AVFrame *out_frame,
68+ int do_ioproc);
69+
6670 static DNNDataType precision_to_datatype(precision_e precision)
6771 {
6872 switch (precision)
@@ -132,6 +136,25 @@ static DNNReturnType get_input_ov(void *model, DNNData *input, const char *input · 63 unmodified lines
132136 return DNN_ERROR;
133137 }
134138
139+static DNNReturnType get_output_ov(void *model, const char *input_name, int input_width, int input_height,
140+ const char *output_name, int *output_width, int *output_height)
141+{
142+ DNNReturnType ret;
143+ OVModel *ov_model = (OVModel *)model;
144+ AVFrame *in_frame = av_frame_alloc();
145+ AVFrame *out_frame = av_frame_alloc();
146+ in_frame->width = input_width;
147+ in_frame->height = input_height;
148+
149+ ret = execute_model_ov(ov_model->model, input_name, in_frame, &output_name, 1, out_frame, 0);
150+ *output_width = out_frame->width;
151+ *output_height = out_frame->height;
152+
153+ av_frame_free(&out_frame);
154+ av_frame_free(&in_frame);
155+ return ret;
156+}
157+
135158 DNNModel *ff_dnn_load_model_ov(const char *model_filename, const char *options, void *userdata)
136159 {
137160 char *all_dev_names = NULL;
@@ -191,6 +214,7 @@ DNNModel *ff_dnn_load_model_ov(const char *model_filename, const char *options, · 53 unmodified lines
191214
192215 model->model = (void *)ov_model;
193216 model->get_input = &get_input_ov;
217+ model->get_output = &get_output_ov;
194218 model->options = options;
195219 model->userdata = userdata;
196220
@@ -213,8 +237,9 @@ err: · 16 unmodified lines
213237 return NULL;
214238 }
215239
216−DNNReturnType ff_dnn_execute_model_ov(const DNNModel *model, const char *input_name, AVFrame *in_frame,
217− const char **output_names, uint32_t nb_output, AVFrame *out_frame)
240+static DNNReturnType execute_model_ov(const DNNModel *model, const char *input_name, AVFrame *in_frame,
241+ const char **output_names, uint32_t nb_output, AVFrame *out_frame,
242+ int do_ioproc)
218243 {
219244 char *model_output_name = NULL;
220245 char *all_output_names = NULL;
@@ -252,10 +277,12 @@ DNNReturnType ff_dnn_execute_model_ov(const DNNModel *model, const char *input_n · 31 unmodified lines
252277 input.channels = dims.dims[1];
253278 input.data = blob_buffer.buffer;
254279 input.dt = precision_to_datatype(precision);
255− if (ov_model->model->pre_proc != NULL) {
256− ov_model->model->pre_proc(in_frame, &input, ov_model->model->userdata);
257− } else {
258− proc_from_frame_to_dnn(in_frame, &input, ctx);
280+ if (do_ioproc) {
281+ if (ov_model->model->pre_proc != NULL) {
282+ ov_model->model->pre_proc(in_frame, &input, ov_model->model->userdata);
283+ } else {
284+ proc_from_frame_to_dnn(in_frame, &input, ctx);
285+ }
259286 }
260287 ie_blob_free(&input_blob);
261288
@@ -308,15 +335,15 @@ DNNReturnType ff_dnn_execute_model_ov(const DNNModel *model, const char *input_n · 46 unmodified lines
308335 output.width = dims.dims[3];
309336 output.dt = precision_to_datatype(precision);
310337 output.data = blob_buffer.buffer;
311− if (out_frame->width != output.width || out_frame->height != output.height) {
312− out_frame->width = output.width;
313− out_frame->height = output.height;
314− } else {
338+ if (do_ioproc) {
315339 if (ov_model->model->post_proc != NULL) {
316340 ov_model->model->post_proc(out_frame, &output, ov_model->model->userdata);
317341 } else {
318342 proc_from_dnn_to_frame(out_frame, &output, ctx);
319343 }
344+ } else {
345+ out_frame->width = output.width;
346+ out_frame->height = output.height;
320347 }
321348 ie_blob_free(&output_blob);
322349 }
@@ -324,6 +351,25 @@ DNNReturnType ff_dnn_execute_model_ov(const DNNModel *model, const char *input_n · 1 unmodified line
324351 return DNN_SUCCESS;
325352 }
326353
354+DNNReturnType ff_dnn_execute_model_ov(const DNNModel *model, const char *input_name, AVFrame *in_frame,
355+ const char **output_names, uint32_t nb_output, AVFrame *out_frame)
356+{
357+ OVModel *ov_model = (OVModel *)model->model;
358+ OVContext *ctx = &ov_model->ctx;
359+
360+ if (!in_frame) {
361+ av_log(ctx, AV_LOG_ERROR, "in frame is NULL when execute model.\n");
362+ return DNN_ERROR;
363+ }
364+
365+ if (!out_frame) {
366+ av_log(ctx, AV_LOG_ERROR, "out frame is NULL when execute model.\n");
367+ return DNN_ERROR;
368+ }
369+
370+ return execute_model_ov(model, input_name, in_frame, output_names, nb_output, out_frame, 1);
371+}
372+
327373 void ff_dnn_free_model_ov(DNNModel **model)
328374 {
329375 if (*model){
libavfilter/dnn/dnn_backend_tf.c+56−10
@@ -55,6 +55,10 @@ static const AVClass dnn_tensorflow_class = { · 54 unmodified lines
5555 .category = AV_CLASS_CATEGORY_FILTER,
5656 };
5757
58+static DNNReturnType execute_model_tf(const DNNModel *model, const char *input_name, AVFrame *in_frame,
59+ const char **output_names, uint32_t nb_output, AVFrame *out_frame,
60+ int do_ioproc);
61+
5862 static void free_buffer(void *data, size_t length)
5963 {
6064 av_freep(&data);
@@ -150,6 +154,25 @@ static DNNReturnType get_input_tf(void *model, DNNData *input, const char *input · 89 unmodified lines
150154 return DNN_SUCCESS;
151155 }
152156
157+static DNNReturnType get_output_tf(void *model, const char *input_name, int input_width, int input_height,
158+ const char *output_name, int *output_width, int *output_height)
159+{
160+ DNNReturnType ret;
161+ TFModel *tf_model = (TFModel *)model;
162+ AVFrame *in_frame = av_frame_alloc();
163+ AVFrame *out_frame = av_frame_alloc();
164+ in_frame->width = input_width;
165+ in_frame->height = input_height;
166+
167+ ret = execute_model_tf(tf_model->model, input_name, in_frame, &output_name, 1, out_frame, 0);
168+ *output_width = out_frame->width;
169+ *output_height = out_frame->height;
170+
171+ av_frame_free(&out_frame);
172+ av_frame_free(&in_frame);
173+ return ret;
174+}
175+
153176 static DNNReturnType load_tf_model(TFModel *tf_model, const char *model_filename)
154177 {
155178 TFContext *ctx = &tf_model->ctx;
@@ -583,14 +606,16 @@ DNNModel *ff_dnn_load_model_tf(const char *model_filename, const char *options, · 427 unmodified lines
583606
584607 model->model = (void *)tf_model;
585608 model->get_input = &get_input_tf;
609+ model->get_output = &get_output_tf;
586610 model->options = options;
587611 model->userdata = userdata;
588612
589613 return model;
590614 }
591615
592−DNNReturnType ff_dnn_execute_model_tf(const DNNModel *model, const char *input_name, AVFrame *in_frame,
593− const char **output_names, uint32_t nb_output, AVFrame *out_frame)
616+static DNNReturnType execute_model_tf(const DNNModel *model, const char *input_name, AVFrame *in_frame,
617+ const char **output_names, uint32_t nb_output, AVFrame *out_frame,
618+ int do_ioproc)
594619 {
595620 TF_Output *tf_outputs;
596621 TFModel *tf_model = (TFModel *)model->model;
@@ -618,10 +643,12 @@ DNNReturnType ff_dnn_execute_model_tf(const DNNModel *model, const char *input_n · 21 unmodified lines
618643 }
619644 input.data = (float *)TF_TensorData(input_tensor);
620645
621− if (tf_model->model->pre_proc != NULL) {
622− tf_model->model->pre_proc(in_frame, &input, tf_model->model->userdata);
623− } else {
624− proc_from_frame_to_dnn(in_frame, &input, ctx);
646+ if (do_ioproc) {
647+ if (tf_model->model->pre_proc != NULL) {
648+ tf_model->model->pre_proc(in_frame, &input, tf_model->model->userdata);
649+ } else {
650+ proc_from_frame_to_dnn(in_frame, &input, ctx);
651+ }
625652 }
626653
627654 if (nb_output != 1) {
@@ -673,15 +700,15 @@ DNNReturnType ff_dnn_execute_model_tf(const DNNModel *model, const char *input_n · 45 unmodified lines
673700 output.data = TF_TensorData(output_tensors[i]);
674701 output.dt = TF_TensorType(output_tensors[i]);
675702
676− if (out_frame->width != output.width || out_frame->height != output.height) {
677− out_frame->width = output.width;
678− out_frame->height = output.height;
679− } else {
703+ if (do_ioproc) {
680704 if (tf_model->model->post_proc != NULL) {
681705 tf_model->model->post_proc(out_frame, &output, tf_model->model->userdata);
682706 } else {
683707 proc_from_dnn_to_frame(out_frame, &output, ctx);
684708 }
709+ } else {
710+ out_frame->width = output.width;
711+ out_frame->height = output.height;
685712 }
686713 }
687714
@@ -696,6 +723,25 @@ DNNReturnType ff_dnn_execute_model_tf(const DNNModel *model, const char *input_n · 8 unmodified lines
696723 return DNN_SUCCESS;
697724 }
698725
726+DNNReturnType ff_dnn_execute_model_tf(const DNNModel *model, const char *input_name, AVFrame *in_frame,
727+ const char **output_names, uint32_t nb_output, AVFrame *out_frame)
728+{
729+ TFModel *tf_model = (TFModel *)model->model;
730+ TFContext *ctx = &tf_model->ctx;
731+
732+ if (!in_frame) {
733+ av_log(ctx, AV_LOG_ERROR, "in frame is NULL when execute model.\n");
734+ return DNN_ERROR;
735+ }
736+
737+ if (!out_frame) {
738+ av_log(ctx, AV_LOG_ERROR, "out frame is NULL when execute model.\n");
739+ return DNN_ERROR;
740+ }
741+
742+ return execute_model_tf(model, input_name, in_frame, output_names, nb_output, out_frame, 1);
743+}
744+
699745 void ff_dnn_free_model_tf(DNNModel **model)
700746 {
701747 TFModel *tf_model;
libavfilter/dnn_interface.h+3−0
@@ -51,6 +51,9 @@ typedef struct DNNModel{ · 50 unmodified lines
5151 // Gets model input information
5252 // Just reuse struct DNNData here, actually the DNNData.data field is not needed.
5353 DNNReturnType (*get_input)(void *model, DNNData *input, const char *input_name);
54+ // Gets model output width/height with given input w/h
55+ DNNReturnType (*get_output)(void *model, const char *input_name, int input_width, int input_height,
56+ const char *output_name, int *output_width, int *output_height);
5457 // set the pre process to transfer data from AVFrame to DNNData
5558 // the default implementation within DNN is used if it is not provided by the filter
5659 int (*pre_proc)(AVFrame *frame_in, DNNData *model_input, void *user_data);
libavfilter/vf_dnn_processing.c+4−13
@@ -233,24 +233,15 @@ static int config_output(AVFilterLink *outlink) · 232 unmodified lines
233233 DnnProcessingContext *ctx = context->priv;
234234 DNNReturnType result;
235235 AVFilterLink *inlink = context->inputs[0];
236− AVFrame *out = NULL;
237−
238− AVFrame *fake_in = ff_get_video_buffer(inlink, inlink->w, inlink->h);
239236
240237 // have a try run in case that the dnn model resize the frame
241− out = ff_get_video_buffer(inlink, inlink->w, inlink->h);
242− result = (ctx->dnn_module->execute_model)(ctx->model, ctx->model_inputname, fake_in,
243− (const char **)&ctx->model_outputname, 1, out);
244− if (result != DNN_SUCCESS){
245− av_log(ctx, AV_LOG_ERROR, "failed to execute model\n");
238+ result = ctx->model->get_output(ctx->model->model, ctx->model_inputname, inlink->w, inlink->h,
239+ ctx->model_outputname, &outlink->w, &outlink->h);
240+ if (result != DNN_SUCCESS) {
241+ av_log(ctx, AV_LOG_ERROR, "could not get output from the model\n");
246242 return AVERROR(EIO);
247243 }
248244
249− outlink->w = out->width;
250− outlink->h = out->height;
251−
252− av_frame_free(&fake_in);
253− av_frame_free(&out);
254245 prepare_uv_scale(outlink);
255246
256247 return 0;
libavfilter/vf_sr.c+10−15
@@ -111,23 +111,20 @@ static int config_output(AVFilterLink *outlink) · 110 unmodified lines
111111 SRContext *ctx = context->priv;
112112 DNNReturnType result;
113113 AVFilterLink *inlink = context->inputs[0];
114− AVFrame *out = NULL;
115− const char *model_output_name = "y";
114+ int out_width, out_height;
116115
117116 // have a try run in case that the dnn model resize the frame
118− AVFrame *fake_in = ff_get_video_buffer(inlink, inlink->w, inlink->h);
119− out = ff_get_video_buffer(inlink, inlink->w, inlink->h);
120− result = (ctx->dnn_module->execute_model)(ctx->model, "x", fake_in,
121− (const char **)&model_output_name, 1, out);
122− if (result != DNN_SUCCESS){
123− av_log(context, AV_LOG_ERROR, "failed to execute loaded model\n");
117+ result = ctx->model->get_output(ctx->model->model, "x", inlink->w, inlink->h,
118+ "y", &out_width, &out_height);
119+ if (result != DNN_SUCCESS) {
120+ av_log(ctx, AV_LOG_ERROR, "could not get output from the model\n");
124121 return AVERROR(EIO);
125122 }
126123
127− if (fake_in->width != out->width || fake_in->height != out->height) {
124+ if (inlink->w != out_width || inlink->h != out_height) {
128125 //espcn
129− outlink->w = out->width;
130− outlink->h = out->height;
126+ outlink->w = out_width;
127+ outlink->h = out_height;
131128 if (inlink->format != AV_PIX_FMT_GRAY8){
132129 const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(inlink->format);
133130 int sws_src_h = AV_CEIL_RSHIFT(inlink->h, desc->log2_chroma_h);
@@ -141,15 +138,13 @@ static int config_output(AVFilterLink *outlink) · 7 unmodified lines
141138 }
142139 } else {
143140 //srcnn
144− outlink->w = out->width * ctx->scale_factor;
145− outlink->h = out->height * ctx->scale_factor;
141+ outlink->w = out_width * ctx->scale_factor;
142+ outlink->h = out_height * ctx->scale_factor;
146143 ctx->sws_pre_scale = sws_getContext(inlink->w, inlink->h, inlink->format,
147144 outlink->w, outlink->h, outlink->format,
148145 SWS_BICUBIC, NULL, NULL, NULL);
149146 }
150147
151− av_frame_free(&fake_in);
152− av_frame_free(&out);
153148 return 0;
154149 }
155150

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t