History

dnn: convert tf.pad to native model in python script, and load/execute it in the c code.

since tf.pad is enabled, the conv2d(valid) changes back to its original behavior.

Guo, Yejun committed 7 years ago

Commit ccbab411 parent 3805aae3 files · +54 −6Browse files

Unified · Split

3 files
  1. libavfilter/dnn/dnn_backend_native.c +35 −0
  2. libavfilter/dnn/dnn_backend_native.h +1 −1
  3. tools/python/convert_from_tensorflow.py +18 −5
libavfilter/dnn/dnn_backend_native.c+35−0
@@ -25,6 +25,7 @@ · 24 unmodified lines
25 25
26#include "dnn_backend_native.h"26#include "dnn_backend_native.h"
27#include "libavutil/avassert.h"27#include "libavutil/avassert.h"
28#include "dnn_backend_native_layer_pad.h"
28 29
29static DNNReturnType set_input_output_native(void *model, DNNInputData *input, const char *input_name, const char **output_names, uint32_t nb_output)30static DNNReturnType set_input_output_native(void *model, DNNInputData *input, const char *input_name, const char **output_names, uint32_t nb_output)
30{31{
@@ -32,6 +33,7 @@ static DNNReturnType set_input_output_native(void *model, DNNInputData *input, c · 1 unmodified line
32 InputParams *input_params;33 InputParams *input_params;
33 ConvolutionalParams *conv_params;34 ConvolutionalParams *conv_params;
34 DepthToSpaceParams *depth_to_space_params;35 DepthToSpaceParams *depth_to_space_params;
36 LayerPadParams *pad_params;
35 int cur_width, cur_height, cur_channels;37 int cur_width, cur_height, cur_channels;
36 int32_t layer;38 int32_t layer;
37 39
@@ -77,6 +79,12 @@ static DNNReturnType set_input_output_native(void *model, DNNInputData *input, c · 39 unmodified lines
77 cur_height *= depth_to_space_params->block_size;79 cur_height *= depth_to_space_params->block_size;
78 cur_width *= depth_to_space_params->block_size;80 cur_width *= depth_to_space_params->block_size;
79 break;81 break;
82 case MIRROR_PAD:
83 pad_params = (LayerPadParams *)network->layers[layer].params;
84 cur_height = cur_height + pad_params->paddings[1][0] + pad_params->paddings[1][1];
85 cur_width = cur_width + pad_params->paddings[2][0] + pad_params->paddings[2][1];
86 cur_channels = cur_channels + pad_params->paddings[3][0] + pad_params->paddings[3][1];
87 break;
80 default:88 default:
81 return DNN_ERROR;89 return DNN_ERROR;
82 }90 }
@@ -110,6 +118,7 @@ DNNModel *ff_dnn_load_model_native(const char *model_filename) · 27 unmodified lines
110 DNNLayerType layer_type;118 DNNLayerType layer_type;
111 ConvolutionalParams *conv_params;119 ConvolutionalParams *conv_params;
112 DepthToSpaceParams *depth_to_space_params;120 DepthToSpaceParams *depth_to_space_params;
121 LayerPadParams *pad_params;
113 122
114 model = av_malloc(sizeof(DNNModel));123 model = av_malloc(sizeof(DNNModel));
115 if (!model){124 if (!model){
@@ -207,6 +216,23 @@ DNNModel *ff_dnn_load_model_native(const char *model_filename) · 91 unmodified lines
207 network->layers[layer].type = DEPTH_TO_SPACE;216 network->layers[layer].type = DEPTH_TO_SPACE;
208 network->layers[layer].params = depth_to_space_params;217 network->layers[layer].params = depth_to_space_params;
209 break;218 break;
219 case MIRROR_PAD:
220 pad_params = av_malloc(sizeof(LayerPadParams));
221 if (!pad_params){
222 avio_closep(&model_file_context);
223 ff_dnn_free_model_native(&model);
224 return NULL;
225 }
226 pad_params->mode = (int32_t)avio_rl32(model_file_context);
227 dnn_size += 4;
228 for (i = 0; i < 4; ++i) {
229 pad_params->paddings[i][0] = avio_rl32(model_file_context);
230 pad_params->paddings[i][1] = avio_rl32(model_file_context);
231 dnn_size += 8;
232 }
233 network->layers[layer].type = MIRROR_PAD;
234 network->layers[layer].params = pad_params;
235 break;
210 default:236 default:
211 avio_closep(&model_file_context);237 avio_closep(&model_file_context);
212 ff_dnn_free_model_native(&model);238 ff_dnn_free_model_native(&model);
@@ -314,6 +340,7 @@ DNNReturnType ff_dnn_execute_model_native(const DNNModel *model, DNNData *output · 101 unmodified lines
314 InputParams *input_params;340 InputParams *input_params;
315 ConvolutionalParams *conv_params;341 ConvolutionalParams *conv_params;
316 DepthToSpaceParams *depth_to_space_params;342 DepthToSpaceParams *depth_to_space_params;
343 LayerPadParams *pad_params;
317 344
318 if (network->layers_num <= 0 || network->layers[0].type != INPUT || !network->layers[0].output){345 if (network->layers_num <= 0 || network->layers[0].type != INPUT || !network->layers[0].output){
319 return DNN_ERROR;346 return DNN_ERROR;
@@ -348,6 +375,14 @@ DNNReturnType ff_dnn_execute_model_native(const DNNModel *model, DNNData *output · 28 unmodified lines
348 cur_width *= depth_to_space_params->block_size;375 cur_width *= depth_to_space_params->block_size;
349 cur_channels /= depth_to_space_params->block_size * depth_to_space_params->block_size;376 cur_channels /= depth_to_space_params->block_size * depth_to_space_params->block_size;
350 break;377 break;
378 case MIRROR_PAD:
379 pad_params = (LayerPadParams *)network->layers[layer].params;
380 dnn_execute_layer_pad(network->layers[layer - 1].output, network->layers[layer].output,
381 pad_params, 1, cur_height, cur_width, cur_channels);
382 cur_height = cur_height + pad_params->paddings[1][0] + pad_params->paddings[1][1];
383 cur_width = cur_width + pad_params->paddings[2][0] + pad_params->paddings[2][1];
384 cur_channels = cur_channels + pad_params->paddings[3][0] + pad_params->paddings[3][1];
385 break;
351 case INPUT:386 case INPUT:
352 return DNN_ERROR;387 return DNN_ERROR;
353 }388 }
libavfilter/dnn/dnn_backend_native.h+1−1
@@ -30,7 +30,7 @@ · 29 unmodified lines
30#include "../dnn_interface.h"30#include "../dnn_interface.h"
31#include "libavformat/avio.h"31#include "libavformat/avio.h"
32 32
33typedef enum {INPUT, CONV, DEPTH_TO_SPACE} DNNLayerType;33typedef enum {INPUT, CONV, DEPTH_TO_SPACE, MIRROR_PAD} DNNLayerType;
34 34
35typedef enum {RELU, TANH, SIGMOID, NONE, LEAKY_RELU} DNNActivationFunc;35typedef enum {RELU, TANH, SIGMOID, NONE, LEAKY_RELU} DNNActivationFunc;
36 36
tools/python/convert_from_tensorflow.py+18−5
@@ -23,9 +23,6 @@ import sys, struct · 22 unmodified lines
23 23
24__all__ = ['convert_from_tensorflow']24__all__ = ['convert_from_tensorflow']
25 25
26# as the first step to be compatible with vf_sr, it is not general.
27# it will be refined step by step.
28
29class TFConverter:26class TFConverter:
30 def __init__(self, graph_def, nodes, outfile):27 def __init__(self, graph_def, nodes, outfile):
31 self.graph_def = graph_def28 self.graph_def = graph_def
@@ -36,9 +33,10 @@ class TFConverter: · 4 unmodified lines
36 self.name_node_dict = {}33 self.name_node_dict = {}
37 self.edges = {}34 self.edges = {}
38 self.conv_activations = {'Relu':0, 'Tanh':1, 'Sigmoid':2, 'LeakyRelu':4}35 self.conv_activations = {'Relu':0, 'Tanh':1, 'Sigmoid':2, 'LeakyRelu':4}
39 self.conv_paddings = {'VALID':2, 'SAME':1}36 self.conv_paddings = {'VALID':0, 'SAME':1}
40 self.converted_nodes = set()37 self.converted_nodes = set()
41 self.op2code = {'Conv2D':1, 'DepthToSpace':2}38 self.op2code = {'Conv2D':1, 'DepthToSpace':2, 'MirrorPad':3}
39 self.mirrorpad_mode = {'CONSTANT':0, 'REFLECT':1, 'SYMMETRIC':2}
42 40
43 41
44 def dump_for_tensorboard(self):42 def dump_for_tensorboard(self):
@@ -101,6 +99,19 @@ class TFConverter: · 56 unmodified lines
101 self.converted_nodes.add(node.name)99 self.converted_nodes.add(node.name)
102 100
103 101
102 def dump_mirrorpad_to_file(self, node, f):
103 assert(node.op == 'MirrorPad')
104 self.layer_number = self.layer_number + 1
105 mode = node.attr['mode'].s
106 mode = self.mirrorpad_mode[mode.decode("utf-8")]
107 np.array([self.op2code[node.op], mode], dtype=np.uint32).tofile(f)
108 pnode = self.name_node_dict[node.input[1]]
109 self.converted_nodes.add(pnode.name)
110 paddings = pnode.attr['value'].tensor.tensor_content
111 f.write(paddings)
112 self.converted_nodes.add(node.name)
113
114
104 def generate_layer_number(self):115 def generate_layer_number(self):
105 # in current hard code implementation, the layer number is the first data written to the native model file116 # in current hard code implementation, the layer number is the first data written to the native model file
106 # it is not easy to know it at the beginning time in the general converter, so first do a dry run for compatibility117 # it is not easy to know it at the beginning time in the general converter, so first do a dry run for compatibility
@@ -118,6 +129,8 @@ class TFConverter: · 11 unmodified lines
118 self.dump_conv2d_to_file(node, f)129 self.dump_conv2d_to_file(node, f)
119 elif node.op == 'DepthToSpace':130 elif node.op == 'DepthToSpace':
120 self.dump_depth2space_to_file(node, f)131 self.dump_depth2space_to_file(node, f)
132 elif node.op == 'MirrorPad':
133 self.dump_mirrorpad_to_file(node, f)
121 134
122 135
123 def dump_to_file(self):136 def dump_to_file(self):

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t