netfilter: nf_tables: add rule blob layout

This patch adds a blob layout per chain to represent the ruleset in the packet datapath. size (unsigned long) struct nft_rule_dp struct nft_expr ... struct nft_rule_dp struct nft_expr ... struct nft_rule_dp (is_last=1) The new structure nft_rule_dp represents the rule in a more compact way (smaller memory footprint) compared to the control-plane nft_rule structure. The ruleset blob is a read-only data structure. The first field contains the blob size, then the rules containing expressions. There is a trailing rule which is used by the tracing infrastructure which is equivalent to the NULL rule marker in the previous representation. The blob size field does not include the size of this trailing rule marker. The ruleset blob is generated from the commit path. This patch reuses the infrastructure available since 0cbc06b3faba ("netfilter: nf_tables: remove synchronize_rcu in commit phase") to build the array of rules per chain.

Pablo Neira Ayuso committed 4 years ago

Commit 2c865a81 parent 3b9e2ea4 files · +147 −67Browse files

Unified · Split

4 files

include/net/netfilter/nf_tables.h +18 −4
net/netfilter/nf_tables_api.c +101 −48
net/netfilter/nf_tables_core.c +27 −14
net/netfilter/nf_tables_trace.c +1 −1

include/net/netfilter/nf_tables.h+18−4

@@ -974,6 +974,20 @@ static inline void nft_set_elem_update_expr(const struct nft_set_ext *ext, · 973 unmodified lines
974	974
975	975	#define NFT_CHAIN_POLICY_UNSET U8_MAX
976	976
	977	+struct nft_rule_dp {
	978	+ u64 is_last:1,
	979	+ dlen:12,
	980	+ handle:42; /* for tracing */
	981	+ unsigned char data[]
	982	+ __attribute__((aligned(__alignof__(struct nft_expr))));
	983	+};
	984	+
	985	+struct nft_rule_blob {
	986	+ unsigned long size;
	987	+ unsigned char data[]
	988	+ __attribute__((aligned(__alignof__(struct nft_rule_dp))));
	989	+};
	990	+
977	991	/**
978	992	* struct nft_chain - nf_tables chain
979	993	*
@@ -987,8 +1001,8 @@ static inline void nft_set_elem_update_expr(const struct nft_set_ext *ext, · 7 unmodified lines
987	1001	* @name: name of the chain
988	1002	*/
989	1003	struct nft_chain {
990		− struct nft_rule __rcu rules_gen_0;
991		− struct nft_rule __rcu rules_gen_1;
	1004	+ struct nft_rule_blob __rcu *blob_gen_0;
	1005	+ struct nft_rule_blob __rcu *blob_gen_1;
992	1006	struct list_head rules;
993	1007	struct list_head list;
994	1008	struct rhlist_head rhlhead;
@@ -1003,7 +1017,7 @@ struct nft_chain { · 8 unmodified lines
1003	1017	u8 *udata;
1004	1018
1005	1019	/* Only used during control plane commit phase: */
1006		− struct nft_rule **rules_next;
	1020	+ struct nft_rule_blob *blob_next;
1007	1021	};
1008	1022
1009	1023	int nft_chain_validate(const struct nft_ctx ctx, const struct nft_chain chain);
@@ -1321,7 +1335,7 @@ struct nft_traceinfo { · 311 unmodified lines
1321	1335	const struct nft_pktinfo *pkt;
1322	1336	const struct nft_base_chain *basechain;
1323	1337	const struct nft_chain *chain;
1324		− const struct nft_rule *rule;
	1338	+ const struct nft_rule_dp *rule;
1325	1339	const struct nft_verdict *verdict;
1326	1340	enum nft_trace_types type;
1327	1341	bool packet_dumped;

net/netfilter/nf_tables_api.c+101−48

@@ -1747,16 +1747,16 @@ static void nft_chain_stats_replace(struct nft_trans *trans) · 1746 unmodified lines
1747	1747
1748	1748	static void nf_tables_chain_free_chain_rules(struct nft_chain *chain)
1749	1749	{
1750		− struct nft_rule **g0 = rcu_dereference_raw(chain->rules_gen_0);
1751		− struct nft_rule **g1 = rcu_dereference_raw(chain->rules_gen_1);
	1750	+ struct nft_rule_blob *g0 = rcu_dereference_raw(chain->blob_gen_0);
	1751	+ struct nft_rule_blob *g1 = rcu_dereference_raw(chain->blob_gen_1);
1752	1752
1753	1753	if (g0 != g1)
1754	1754	kvfree(g1);
1755	1755	kvfree(g0);
1756	1756
1757	1757	/* should be NULL either via abort or via successful commit */
1758		− WARN_ON_ONCE(chain->rules_next);
1759		− kvfree(chain->rules_next);
	1758	+ WARN_ON_ONCE(chain->blob_next);
	1759	+ kvfree(chain->blob_next);
1760	1760	}
1761	1761
1762	1762	void nf_tables_chain_destroy(struct nft_ctx *ctx)
@@ -2002,23 +2002,39 @@ static void nft_chain_release_hook(struct nft_chain_hook *hook) · 239 unmodified lines
2002	2002
2003	2003	struct nft_rules_old {
2004	2004	struct rcu_head h;
2005		− struct nft_rule **start;
	2005	+ struct nft_rule_blob *blob;
2006	2006	};
2007	2007
2008		−static struct nft_rule *nf_tables_chain_alloc_rules(const struct nft_chain chain,
2009		− unsigned int alloc)
	2008	+static void nft_last_rule(struct nft_rule_blob blob, const void ptr)
2010	2009	{
2011		− if (alloc > INT_MAX)
	2010	+ struct nft_rule_dp *prule;
	2011	+
	2012	+ prule = (struct nft_rule_dp *)ptr;
	2013	+ prule->is_last = 1;
	2014	+ ptr += offsetof(struct nft_rule_dp, data);
	2015	+ /* blob size does not include the trailer rule */
	2016	+}
	2017	+
	2018	+static struct nft_rule_blob *nf_tables_chain_alloc_rules(unsigned int size)
	2019	+{
	2020	+ struct nft_rule_blob *blob;
	2021	+
	2022	+ /* size must include room for the last rule */
	2023	+ if (size < offsetof(struct nft_rule_dp, data))
	2024	+ return NULL;
	2025	+
	2026	+ size += sizeof(struct nft_rule_blob) + sizeof(struct nft_rules_old);
	2027	+ if (size > INT_MAX)
2012	2028	return NULL;
2013	2029
2014		− alloc += 1; /* NULL, ends rules */
2015		− if (sizeof(struct nft_rule *) > INT_MAX / alloc)
	2030	+ blob = kvmalloc(size, GFP_KERNEL);
	2031	+ if (!blob)
2016	2032	return NULL;
2017	2033
2018		− alloc = sizeof(struct nft_rule );
2019		− alloc += sizeof(struct nft_rules_old);
	2034	+ blob->size = 0;
	2035	+ nft_last_rule(blob, blob->data);
2020	2036
2021		− return kvmalloc(alloc, GFP_KERNEL);
	2037	+ return blob;
2022	2038	}
2023	2039
2024	2040	static void nft_basechain_hook_init(struct nf_hook_ops *ops, u8 family,
@@ -2091,9 +2107,10 @@ static int nf_tables_addchain(struct nft_ctx *ctx, u8 family, u8 genmask, · 66 unmodified lines
2091	2107	struct nft_stats __percpu *stats;
2092	2108	struct net *net = ctx->net;
2093	2109	char name[NFT_NAME_MAXLEN];
	2110	+ struct nft_rule_blob *blob;
2094	2111	struct nft_trans *trans;
2095	2112	struct nft_chain *chain;
2096		− struct nft_rule **rules;
	2113	+ unsigned int data_size;
2097	2114	int err;
2098	2115
2099	2116	if (table->use == UINT_MAX)
@@ -2178,15 +2195,15 @@ static int nf_tables_addchain(struct nft_ctx *ctx, u8 family, u8 genmask, · 78 unmodified lines
2178	2195	chain->udlen = nla_len(nla[NFTA_CHAIN_USERDATA]);
2179	2196	}
2180	2197
2181		− rules = nf_tables_chain_alloc_rules(chain, 0);
2182		− if (!rules) {
	2198	+ data_size = offsetof(struct nft_rule_dp, data); /* last rule */
	2199	+ blob = nf_tables_chain_alloc_rules(data_size);
	2200	+ if (!blob) {
2183	2201	err = -ENOMEM;
2184	2202	goto err_destroy_chain;
2185	2203	}
2186	2204
2187		− *rules = NULL;
2188		− rcu_assign_pointer(chain->rules_gen_0, rules);
2189		− rcu_assign_pointer(chain->rules_gen_1, rules);
	2205	+ RCU_INIT_POINTER(chain->blob_gen_0, blob);
	2206	+ RCU_INIT_POINTER(chain->blob_gen_1, blob);
2190	2207
2191	2208	err = nf_tables_register_hook(net, table, chain);
2192	2209	if (err < 0)
@@ -8241,32 +8258,72 @@ EXPORT_SYMBOL_GPL(nf_tables_trans_destroy_flush_work); · 6048 unmodified lines
8241	8258
8242	8259	static int nf_tables_commit_chain_prepare(struct net net, struct nft_chain chain)
8243	8260	{
	8261	+ const struct nft_expr expr, last;
	8262	+ unsigned int size, data_size;
	8263	+ void data, data_boundary;
	8264	+ struct nft_rule_dp *prule;
8244	8265	struct nft_rule *rule;
8245		− unsigned int alloc = 0;
8246	8266	int i;
8247	8267
8248	8268	/* already handled or inactive chain? */
8249		− if (chain->rules_next \|\| !nft_is_active_next(net, chain))
	8269	+ if (chain->blob_next \|\| !nft_is_active_next(net, chain))
8250	8270	return 0;
8251	8271
8252	8272	rule = list_entry(&chain->rules, struct nft_rule, list);
8253	8273	i = 0;
8254	8274
8255	8275	list_for_each_entry_continue(rule, &chain->rules, list) {
8256		− if (nft_is_active_next(net, rule))
8257		− alloc++;
	8276	+ if (nft_is_active_next(net, rule)) {
	8277	+ data_size += sizeof(*prule) + rule->dlen;
	8278	+ if (data_size > INT_MAX)
	8279	+ return -ENOMEM;
	8280	+ }
8258	8281	}
	8282	+ data_size += offsetof(struct nft_rule_dp, data); /* last rule */
8259	8283
8260		− chain->rules_next = nf_tables_chain_alloc_rules(chain, alloc);
8261		− if (!chain->rules_next)
	8284	+ chain->blob_next = nf_tables_chain_alloc_rules(data_size);
	8285	+ if (!chain->blob_next)
8262	8286	return -ENOMEM;
8263	8287
	8288	+ data = (void *)chain->blob_next->data;
	8289	+ data_boundary = data + data_size;
	8290	+ size = 0;
	8291	+
8264	8292	list_for_each_entry_continue(rule, &chain->rules, list) {
8265		− if (nft_is_active_next(net, rule))
8266		− chain->rules_next[i++] = rule;
	8293	+ if (!nft_is_active_next(net, rule))
	8294	+ continue;
	8295	+
	8296	+ prule = (struct nft_rule_dp *)data;
	8297	+ data += offsetof(struct nft_rule_dp, data);
	8298	+ if (WARN_ON_ONCE(data > data_boundary))
	8299	+ return -ENOMEM;
	8300	+
	8301	+ nft_rule_for_each_expr(expr, last, rule) {
	8302	+ if (WARN_ON_ONCE(data + expr->ops->size > data_boundary))
	8303	+ return -ENOMEM;
	8304	+
	8305	+ memcpy(data + size, expr, expr->ops->size);
	8306	+ size += expr->ops->size;
	8307	+ }
	8308	+ if (WARN_ON_ONCE(size >= 1 << 12))
	8309	+ return -ENOMEM;
	8310	+
	8311	+ prule->handle = rule->handle;
	8312	+ prule->dlen = size;
	8313	+ prule->is_last = 0;
	8314	+
	8315	+ data += size;
	8316	+ size = 0;
	8317	+ chain->blob_next->size += (unsigned long)(data - (void *)prule);
8267	8318	}
8268	8319
8269		− chain->rules_next[i] = NULL;
	8320	+ prule = (struct nft_rule_dp *)data;
	8321	+ data += offsetof(struct nft_rule_dp, data);
	8322	+ if (WARN_ON_ONCE(data > data_boundary))
	8323	+ return -ENOMEM;
	8324	+
	8325	+ nft_last_rule(chain->blob_next, prule);
	8326	+
8270	8327	return 0;
8271	8328	}
8272	8329
@@ -8280,8 +8337,8 @@ static void nf_tables_commit_chain_prepare_cancel(struct net *net) · 7 unmodified lines
8280	8337
8281	8338	if (trans->msg_type == NFT_MSG_NEWRULE \|\|
8282	8339	trans->msg_type == NFT_MSG_DELRULE) {
8283		− kvfree(chain->rules_next);
8284		− chain->rules_next = NULL;
	8340	+ kvfree(chain->blob_next);
	8341	+ chain->blob_next = NULL;
8285	8342	}
8286	8343	}
8287	8344	}
@@ -8290,38 +8347,34 @@ static void __nf_tables_commit_chain_free_rules_old(struct rcu_head *h) · 2 unmodified lines
8290	8347	{
8291	8348	struct nft_rules_old *o = container_of(h, struct nft_rules_old, h);
8292	8349
8293		− kvfree(o->start);
	8350	+ kvfree(o->blob);
8294	8351	}
8295	8352
8296		−static void nf_tables_commit_chain_free_rules_old(struct nft_rule **rules)
	8353	+static void nf_tables_commit_chain_free_rules_old(struct nft_rule_blob *blob)
8297	8354	{
8298		− struct nft_rule **r = rules;
8299	8355	struct nft_rules_old *old;
8300	8356
8301		− while (*r)
8302		− r++;
8303		−
8304		− r++; /* rcu_head is after end marker */
8305		− old = (void *) r;
8306		− old->start = rules;
	8357	+ /* rcu_head is after end marker */
	8358	+ old = (void )blob + sizeof(blob) + blob->size;
	8359	+ old->blob = blob;
8307	8360
8308	8361	call_rcu(&old->h, __nf_tables_commit_chain_free_rules_old);
8309	8362	}
8310	8363
8311	8364	static void nf_tables_commit_chain(struct net net, struct nft_chain chain)
8312	8365	{
8313		− struct nft_rule g0, g1;
	8366	+ struct nft_rule_blob g0, g1;
8314	8367	bool next_genbit;
8315	8368
8316	8369	next_genbit = nft_gencursor_next(net);
8317	8370
8318		− g0 = rcu_dereference_protected(chain->rules_gen_0,
	8371	+ g0 = rcu_dereference_protected(chain->blob_gen_0,
8319	8372	lockdep_commit_lock_is_held(net));
8320		− g1 = rcu_dereference_protected(chain->rules_gen_1,
	8373	+ g1 = rcu_dereference_protected(chain->blob_gen_1,
8321	8374	lockdep_commit_lock_is_held(net));
8322	8375
8323	8376	/* No changes to this chain? */
8324		− if (chain->rules_next == NULL) {
	8377	+ if (chain->blob_next == NULL) {
8325	8378	/* chain had no change in last or next generation */
8326	8379	if (g0 == g1)
8327	8380	return;
@@ -8330,10 +8383,10 @@ static void nf_tables_commit_chain(struct net net, struct nft_chain chain) · 2 unmodified lines
8330	8383	* one uses same rules as current generation.
8331	8384	*/
8332	8385	if (next_genbit) {
8333		− rcu_assign_pointer(chain->rules_gen_1, g0);
	8386	+ rcu_assign_pointer(chain->blob_gen_1, g0);
8334	8387	nf_tables_commit_chain_free_rules_old(g1);
8335	8388	} else {
8336		− rcu_assign_pointer(chain->rules_gen_0, g1);
	8389	+ rcu_assign_pointer(chain->blob_gen_0, g1);
8337	8390	nf_tables_commit_chain_free_rules_old(g0);
8338	8391	}
8339	8392
@@ -8341,11 +8394,11 @@ static void nf_tables_commit_chain(struct net net, struct nft_chain chain) · 1 unmodified line
8341	8394	}
8342	8395
8343	8396	if (next_genbit)
8344		− rcu_assign_pointer(chain->rules_gen_1, chain->rules_next);
	8397	+ rcu_assign_pointer(chain->blob_gen_1, chain->blob_next);
8345	8398	else
8346		− rcu_assign_pointer(chain->rules_gen_0, chain->rules_next);
	8399	+ rcu_assign_pointer(chain->blob_gen_0, chain->blob_next);
8347	8400
8348		− chain->rules_next = NULL;
	8401	+ chain->blob_next = NULL;
8349	8402
8350	8403	if (g0 == g1)
8351	8404	return;

net/netfilter/nf_tables_core.c+27−14

@@ -38,7 +38,7 @@ static noinline void __nft_trace_packet(struct nft_traceinfo *info, · 37 unmodified lines
38	38
39	39	static inline void nft_trace_packet(struct nft_traceinfo *info,
40	40	const struct nft_chain *chain,
41		− const struct nft_rule *rule,
	41	+ const struct nft_rule_dp *rule,
42	42	enum nft_trace_types type)
43	43	{
44	44	if (static_branch_unlikely(&nft_trace_enabled)) {
@@ -88,7 +88,7 @@ static noinline void __nft_trace_verdict(struct nft_traceinfo *info, · 43 unmodified lines
88	88
89	89	static inline void nft_trace_verdict(struct nft_traceinfo *info,
90	90	const struct nft_chain *chain,
91		− const struct nft_rule *rule,
	91	+ const struct nft_rule_dp *rule,
92	92	const struct nft_regs *regs)
93	93	{
94	94	if (static_branch_unlikely(&nft_trace_enabled)) {
@@ -153,8 +153,9 @@ static noinline void nft_update_chain_stats(const struct nft_chain *chain, · 58 unmodified lines
153	153	}
154	154
155	155	struct nft_jumpstack {
156		− const struct nft_chain *chain;
157		− struct nft_rule const rules;
	156	+ const struct nft_chain *chain;
	157	+ const struct nft_rule_dp *rule;
	158	+ const struct nft_rule_dp *last_rule;
158	159	};
159	160
160	161	static void expr_call_ops_eval(const struct nft_expr *expr,
@@ -183,18 +184,28 @@ static void expr_call_ops_eval(const struct nft_expr *expr, · 22 unmodified lines
183	184	expr->ops->eval(expr, regs, pkt);
184	185	}
185	186
	187	+#define nft_rule_expr_first(rule) (struct nft_expr *)&rule->data[0]
	188	+#define nft_rule_expr_next(expr) ((void *)expr) + expr->ops->size
	189	+#define nft_rule_expr_last(rule) (struct nft_expr *)&rule->data[rule->dlen]
	190	+#define nft_rule_next(rule) (void )rule + sizeof(rule) + rule->dlen
	191	+
	192	+#define nft_rule_dp_for_each_expr(expr, last, rule) \
	193	+ for ((expr) = nft_rule_expr_first(rule), (last) = nft_rule_expr_last(rule); \
	194	+ (expr) != (last); \
	195	+ (expr) = nft_rule_expr_next(expr))
	196	+
186	197	unsigned int
187	198	nft_do_chain(struct nft_pktinfo pkt, void priv)
188	199	{
189	200	const struct nft_chain chain = priv, basechain = chain;
	201	+ const struct nft_rule_dp rule, last_rule;
190	202	const struct net *net = nft_net(pkt);
191		− struct nft_rule const rules;
192		− const struct nft_rule *rule;
193	203	const struct nft_expr expr, last;
194	204	struct nft_regs regs;
195	205	unsigned int stackptr = 0;
196	206	struct nft_jumpstack jumpstack[NFT_JUMP_STACK_SIZE];
197	207	bool genbit = READ_ONCE(net->nft.gencursor);
	208	+ struct nft_rule_blob *blob;
198	209	struct nft_traceinfo info;
199	210
200	211	info.trace = false;
@@ -202,16 +213,16 @@ nft_do_chain(struct nft_pktinfo pkt, void priv) · 1 unmodified line
202	213	nft_trace_init(&info, pkt, &regs.verdict, basechain);
203	214	do_chain:
204	215	if (genbit)
205		− rules = rcu_dereference(chain->rules_gen_1);
	216	+ blob = rcu_dereference(chain->blob_gen_1);
206	217	else
207		− rules = rcu_dereference(chain->rules_gen_0);
	218	+ blob = rcu_dereference(chain->blob_gen_0);
208	219
	220	+ rule = (struct nft_rule_dp *)blob->data;
	221	+ last_rule = (void *)blob->data + blob->size;
209	222	next_rule:
210		− rule = *rules;
211	223	regs.verdict.code = NFT_CONTINUE;
212		− for (; *rules ; rules++) {
213		− rule = *rules;
214		− nft_rule_for_each_expr(expr, last, rule) {
	224	+ for (; rule < last_rule; rule = nft_rule_next(rule)) {
	225	+ nft_rule_dp_for_each_expr(expr, last, rule) {
215	226	if (expr->ops == &nft_cmp_fast_ops)
216	227	nft_cmp_fast_eval(expr, &regs);
217	228	else if (expr->ops == &nft_bitwise_fast_ops)
@@ -251,7 +262,8 @@ next_rule: · 33 unmodified lines
251	262	if (WARN_ON_ONCE(stackptr >= NFT_JUMP_STACK_SIZE))
252	263	return NF_DROP;
253	264	jumpstack[stackptr].chain = chain;
254		− jumpstack[stackptr].rules = rules + 1;
	265	+ jumpstack[stackptr].rule = nft_rule_next(rule);
	266	+ jumpstack[stackptr].last_rule = last_rule;
255	267	stackptr++;
256	268	fallthrough;
257	269	case NFT_GOTO:
@@ -267,7 +279,8 @@ next_rule: · 9 unmodified lines
267	279	if (stackptr > 0) {
268	280	stackptr--;
269	281	chain = jumpstack[stackptr].chain;
270		− rules = jumpstack[stackptr].rules;
	282	+ rule = jumpstack[stackptr].rule;
	283	+ last_rule = jumpstack[stackptr].last_rule;
271	284	goto next_rule;
272	285	}
273	286

net/netfilter/nf_tables_trace.c+1−1

@@ -142,7 +142,7 @@ static int nf_trace_fill_pkt_info(struct sk_buff *nlskb, · 141 unmodified lines
142	142	static int nf_trace_fill_rule_info(struct sk_buff *nlskb,
143	143	const struct nft_traceinfo *info)
144	144	{
145		− if (!info->rule)
	145	+ if (!info->rule \|\| info->rule->is_last)
146	146	return 0;
147	147
148	148	/* a continue verdict with ->type == RETURN means that this is

netfilter: nf_tables: add rule blob layout

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code