dnn: convert tf.pad to native model in python script, and load/execute it in the c code.

since tf.pad is enabled, the conv2d(valid) changes back to its original behavior.

Guo, Yejun committed 7 years ago

Commit ccbab411 parent 3805aae3 files · +54 −6Browse files

Unified · Split

3 files

libavfilter/dnn/dnn_backend_native.c +35 −0
libavfilter/dnn/dnn_backend_native.h +1 −1
tools/python/convert_from_tensorflow.py +18 −5

libavfilter/dnn/dnn_backend_native.c+35−0

@@ -25,6 +25,7 @@ · 24 unmodified lines
25	25
26	26	#include "dnn_backend_native.h"
27	27	#include "libavutil/avassert.h"
	28	+#include "dnn_backend_native_layer_pad.h"
28	29
29	30	static DNNReturnType set_input_output_native(void model, DNNInputData input, const char input_name, const char *output_names, uint32_t nb_output)
30	31	{
@@ -32,6 +33,7 @@ static DNNReturnType set_input_output_native(void model, DNNInputData input, c · 1 unmodified line
32	33	InputParams *input_params;
33	34	ConvolutionalParams *conv_params;
34	35	DepthToSpaceParams *depth_to_space_params;
	36	+ LayerPadParams *pad_params;
35	37	int cur_width, cur_height, cur_channels;
36	38	int32_t layer;
37	39
@@ -77,6 +79,12 @@ static DNNReturnType set_input_output_native(void model, DNNInputData input, c · 39 unmodified lines
77	79	cur_height *= depth_to_space_params->block_size;
78	80	cur_width *= depth_to_space_params->block_size;
79	81	break;
	82	+ case MIRROR_PAD:
	83	+ pad_params = (LayerPadParams *)network->layers[layer].params;
	84	+ cur_height = cur_height + pad_params->paddings[1][0] + pad_params->paddings[1][1];
	85	+ cur_width = cur_width + pad_params->paddings[2][0] + pad_params->paddings[2][1];
	86	+ cur_channels = cur_channels + pad_params->paddings[3][0] + pad_params->paddings[3][1];
	87	+ break;
80	88	default:
81	89	return DNN_ERROR;
82	90	}
@@ -110,6 +118,7 @@ DNNModel ff_dnn_load_model_native(const char model_filename) · 27 unmodified lines
110	118	DNNLayerType layer_type;
111	119	ConvolutionalParams *conv_params;
112	120	DepthToSpaceParams *depth_to_space_params;
	121	+ LayerPadParams *pad_params;
113	122
114	123	model = av_malloc(sizeof(DNNModel));
115	124	if (!model){
@@ -207,6 +216,23 @@ DNNModel ff_dnn_load_model_native(const char model_filename) · 91 unmodified lines
207	216	network->layers[layer].type = DEPTH_TO_SPACE;
208	217	network->layers[layer].params = depth_to_space_params;
209	218	break;
	219	+ case MIRROR_PAD:
	220	+ pad_params = av_malloc(sizeof(LayerPadParams));
	221	+ if (!pad_params){
	222	+ avio_closep(&model_file_context);
	223	+ ff_dnn_free_model_native(&model);
	224	+ return NULL;
	225	+ }
	226	+ pad_params->mode = (int32_t)avio_rl32(model_file_context);
	227	+ dnn_size += 4;
	228	+ for (i = 0; i < 4; ++i) {
	229	+ pad_params->paddings[i][0] = avio_rl32(model_file_context);
	230	+ pad_params->paddings[i][1] = avio_rl32(model_file_context);
	231	+ dnn_size += 8;
	232	+ }
	233	+ network->layers[layer].type = MIRROR_PAD;
	234	+ network->layers[layer].params = pad_params;
	235	+ break;
210	236	default:
211	237	avio_closep(&model_file_context);
212	238	ff_dnn_free_model_native(&model);
@@ -314,6 +340,7 @@ DNNReturnType ff_dnn_execute_model_native(const DNNModel model, DNNData output · 101 unmodified lines
314	340	InputParams *input_params;
315	341	ConvolutionalParams *conv_params;
316	342	DepthToSpaceParams *depth_to_space_params;
	343	+ LayerPadParams *pad_params;
317	344
318	345	if (network->layers_num <= 0 \|\| network->layers[0].type != INPUT \|\| !network->layers[0].output){
319	346	return DNN_ERROR;
@@ -348,6 +375,14 @@ DNNReturnType ff_dnn_execute_model_native(const DNNModel model, DNNData output · 28 unmodified lines
348	375	cur_width *= depth_to_space_params->block_size;
349	376	cur_channels /= depth_to_space_params->block_size * depth_to_space_params->block_size;
350	377	break;
	378	+ case MIRROR_PAD:
	379	+ pad_params = (LayerPadParams *)network->layers[layer].params;
	380	+ dnn_execute_layer_pad(network->layers[layer - 1].output, network->layers[layer].output,
	381	+ pad_params, 1, cur_height, cur_width, cur_channels);
	382	+ cur_height = cur_height + pad_params->paddings[1][0] + pad_params->paddings[1][1];
	383	+ cur_width = cur_width + pad_params->paddings[2][0] + pad_params->paddings[2][1];
	384	+ cur_channels = cur_channels + pad_params->paddings[3][0] + pad_params->paddings[3][1];
	385	+ break;
351	386	case INPUT:
352	387	return DNN_ERROR;
353	388	}

libavfilter/dnn/dnn_backend_native.h+1−1

@@ -30,7 +30,7 @@ · 29 unmodified lines
30	30	#include "../dnn_interface.h"
31	31	#include "libavformat/avio.h"
32	32
33		−typedef enum {INPUT, CONV, DEPTH_TO_SPACE} DNNLayerType;
	33	+typedef enum {INPUT, CONV, DEPTH_TO_SPACE, MIRROR_PAD} DNNLayerType;
34	34
35	35	typedef enum {RELU, TANH, SIGMOID, NONE, LEAKY_RELU} DNNActivationFunc;
36	36

tools/python/convert_from_tensorflow.py+18−5

@@ -23,9 +23,6 @@ import sys, struct · 22 unmodified lines
23	23
24	24	__all__ = ['convert_from_tensorflow']
25	25
26		−# as the first step to be compatible with vf_sr, it is not general.
27		−# it will be refined step by step.
28		−
29	26	class TFConverter:
30	27	def __init__(self, graph_def, nodes, outfile):
31	28	self.graph_def = graph_def
@@ -36,9 +33,10 @@ class TFConverter: · 4 unmodified lines
36	33	self.name_node_dict = {}
37	34	self.edges = {}
38	35	self.conv_activations = {'Relu':0, 'Tanh':1, 'Sigmoid':2, 'LeakyRelu':4}
39		− self.conv_paddings = {'VALID':2, 'SAME':1}
	36	+ self.conv_paddings = {'VALID':0, 'SAME':1}
40	37	self.converted_nodes = set()
41		− self.op2code = {'Conv2D':1, 'DepthToSpace':2}
	38	+ self.op2code = {'Conv2D':1, 'DepthToSpace':2, 'MirrorPad':3}
	39	+ self.mirrorpad_mode = {'CONSTANT':0, 'REFLECT':1, 'SYMMETRIC':2}
42	40
43	41
44	42	def dump_for_tensorboard(self):
@@ -101,6 +99,19 @@ class TFConverter: · 56 unmodified lines
101	99	self.converted_nodes.add(node.name)
102	100
103	101
	102	+ def dump_mirrorpad_to_file(self, node, f):
	103	+ assert(node.op == 'MirrorPad')
	104	+ self.layer_number = self.layer_number + 1
	105	+ mode = node.attr['mode'].s
	106	+ mode = self.mirrorpad_mode[mode.decode("utf-8")]
	107	+ np.array([self.op2code[node.op], mode], dtype=np.uint32).tofile(f)
	108	+ pnode = self.name_node_dict[node.input[1]]
	109	+ self.converted_nodes.add(pnode.name)
	110	+ paddings = pnode.attr['value'].tensor.tensor_content
	111	+ f.write(paddings)
	112	+ self.converted_nodes.add(node.name)
	113	+
	114	+
104	115	def generate_layer_number(self):
105	116	# in current hard code implementation, the layer number is the first data written to the native model file
106	117	# it is not easy to know it at the beginning time in the general converter, so first do a dry run for compatibility
@@ -118,6 +129,8 @@ class TFConverter: · 11 unmodified lines
118	129	self.dump_conv2d_to_file(node, f)
119	130	elif node.op == 'DepthToSpace':
120	131	self.dump_depth2space_to_file(node, f)
	132	+ elif node.op == 'MirrorPad':
	133	+ self.dump_mirrorpad_to_file(node, f)
121	134
122	135
123	136	def dump_to_file(self):

dnn: convert tf.pad to native model in python script, and load/execute it in the c code.

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code