| @@ -1747,16 +1747,16 @@ static void nft_chain_stats_replace(struct nft_trans *trans) · 1746 unmodified lines |
| 1747 | | 1747 | |
| 1748 | static void nf_tables_chain_free_chain_rules(struct nft_chain *chain) | 1748 | static void nf_tables_chain_free_chain_rules(struct nft_chain *chain) |
| 1749 | { | 1749 | { |
| 1750 | struct nft_rule **g0 = rcu_dereference_raw(chain->rules_gen_0); | 1750 | struct nft_rule_blob *g0 = rcu_dereference_raw(chain->blob_gen_0); |
| 1751 | struct nft_rule **g1 = rcu_dereference_raw(chain->rules_gen_1); | 1751 | struct nft_rule_blob *g1 = rcu_dereference_raw(chain->blob_gen_1); |
| 1752 | | 1752 | |
| 1753 | if (g0 != g1) | 1753 | if (g0 != g1) |
| 1754 | kvfree(g1); | 1754 | kvfree(g1); |
| 1755 | kvfree(g0); | 1755 | kvfree(g0); |
| 1756 | | 1756 | |
| 1757 | /* should be NULL either via abort or via successful commit */ | 1757 | /* should be NULL either via abort or via successful commit */ |
| 1758 | WARN_ON_ONCE(chain->rules_next); | 1758 | WARN_ON_ONCE(chain->blob_next); |
| 1759 | kvfree(chain->rules_next); | 1759 | kvfree(chain->blob_next); |
| 1760 | } | 1760 | } |
| 1761 | | 1761 | |
| 1762 | void nf_tables_chain_destroy(struct nft_ctx *ctx) | 1762 | void nf_tables_chain_destroy(struct nft_ctx *ctx) |
| @@ -2002,23 +2002,39 @@ static void nft_chain_release_hook(struct nft_chain_hook *hook) · 239 unmodified lines |
| 2002 | | 2002 | |
| 2003 | struct nft_rules_old { | 2003 | struct nft_rules_old { |
| 2004 | struct rcu_head h; | 2004 | struct rcu_head h; |
| 2005 | struct nft_rule **start; | 2005 | struct nft_rule_blob *blob; |
| 2006 | }; | 2006 | }; |
| 2007 | | 2007 | |
| 2008 | static struct nft_rule **nf_tables_chain_alloc_rules(const struct nft_chain *chain, | 2008 | static void nft_last_rule(struct nft_rule_blob *blob, const void *ptr) |
| 2009 | unsigned int alloc) | | |
| 2010 | { | 2009 | { |
| 2011 | if (alloc > INT_MAX) | 2010 | struct nft_rule_dp *prule; |
| | 2011 | |
| | 2012 | prule = (struct nft_rule_dp *)ptr; |
| | 2013 | prule->is_last = 1; |
| | 2014 | ptr += offsetof(struct nft_rule_dp, data); |
| | 2015 | /* blob size does not include the trailer rule */ |
| | 2016 | } |
| | 2017 | |
| | 2018 | static struct nft_rule_blob *nf_tables_chain_alloc_rules(unsigned int size) |
| | 2019 | { |
| | 2020 | struct nft_rule_blob *blob; |
| | 2021 | |
| | 2022 | /* size must include room for the last rule */ |
| | 2023 | if (size < offsetof(struct nft_rule_dp, data)) |
| | 2024 | return NULL; |
| | 2025 | |
| | 2026 | size += sizeof(struct nft_rule_blob) + sizeof(struct nft_rules_old); |
| | 2027 | if (size > INT_MAX) |
| 2012 | return NULL; | 2028 | return NULL; |
| 2013 | | 2029 | |
| 2014 | alloc += 1; /* NULL, ends rules */ | 2030 | blob = kvmalloc(size, GFP_KERNEL); |
| 2015 | if (sizeof(struct nft_rule *) > INT_MAX / alloc) | 2031 | if (!blob) |
| 2016 | return NULL; | 2032 | return NULL; |
| 2017 | | 2033 | |
| 2018 | alloc *= sizeof(struct nft_rule *); | 2034 | blob->size = 0; |
| 2019 | alloc += sizeof(struct nft_rules_old); | 2035 | nft_last_rule(blob, blob->data); |
| 2020 | | 2036 | |
| 2021 | return kvmalloc(alloc, GFP_KERNEL); | 2037 | return blob; |
| 2022 | } | 2038 | } |
| 2023 | | 2039 | |
| 2024 | static void nft_basechain_hook_init(struct nf_hook_ops *ops, u8 family, | 2040 | static void nft_basechain_hook_init(struct nf_hook_ops *ops, u8 family, |
| @@ -2091,9 +2107,10 @@ static int nf_tables_addchain(struct nft_ctx *ctx, u8 family, u8 genmask, · 66 unmodified lines |
| 2091 | struct nft_stats __percpu *stats; | 2107 | struct nft_stats __percpu *stats; |
| 2092 | struct net *net = ctx->net; | 2108 | struct net *net = ctx->net; |
| 2093 | char name[NFT_NAME_MAXLEN]; | 2109 | char name[NFT_NAME_MAXLEN]; |
| | 2110 | struct nft_rule_blob *blob; |
| 2094 | struct nft_trans *trans; | 2111 | struct nft_trans *trans; |
| 2095 | struct nft_chain *chain; | 2112 | struct nft_chain *chain; |
| 2096 | struct nft_rule **rules; | 2113 | unsigned int data_size; |
| 2097 | int err; | 2114 | int err; |
| 2098 | | 2115 | |
| 2099 | if (table->use == UINT_MAX) | 2116 | if (table->use == UINT_MAX) |
| @@ -2178,15 +2195,15 @@ static int nf_tables_addchain(struct nft_ctx *ctx, u8 family, u8 genmask, · 78 unmodified lines |
| 2178 | chain->udlen = nla_len(nla[NFTA_CHAIN_USERDATA]); | 2195 | chain->udlen = nla_len(nla[NFTA_CHAIN_USERDATA]); |
| 2179 | } | 2196 | } |
| 2180 | | 2197 | |
| 2181 | rules = nf_tables_chain_alloc_rules(chain, 0); | 2198 | data_size = offsetof(struct nft_rule_dp, data); /* last rule */ |
| 2182 | if (!rules) { | 2199 | blob = nf_tables_chain_alloc_rules(data_size); |
| | 2200 | if (!blob) { |
| 2183 | err = -ENOMEM; | 2201 | err = -ENOMEM; |
| 2184 | goto err_destroy_chain; | 2202 | goto err_destroy_chain; |
| 2185 | } | 2203 | } |
| 2186 | | 2204 | |
| 2187 | *rules = NULL; | 2205 | RCU_INIT_POINTER(chain->blob_gen_0, blob); |
| 2188 | rcu_assign_pointer(chain->rules_gen_0, rules); | 2206 | RCU_INIT_POINTER(chain->blob_gen_1, blob); |
| 2189 | rcu_assign_pointer(chain->rules_gen_1, rules); | | |
| 2190 | | 2207 | |
| 2191 | err = nf_tables_register_hook(net, table, chain); | 2208 | err = nf_tables_register_hook(net, table, chain); |
| 2192 | if (err < 0) | 2209 | if (err < 0) |
| @@ -8241,32 +8258,72 @@ EXPORT_SYMBOL_GPL(nf_tables_trans_destroy_flush_work); · 6048 unmodified lines |
| 8241 | | 8258 | |
| 8242 | static int nf_tables_commit_chain_prepare(struct net *net, struct nft_chain *chain) | 8259 | static int nf_tables_commit_chain_prepare(struct net *net, struct nft_chain *chain) |
| 8243 | { | 8260 | { |
| | 8261 | const struct nft_expr *expr, *last; |
| | 8262 | unsigned int size, data_size; |
| | 8263 | void *data, *data_boundary; |
| | 8264 | struct nft_rule_dp *prule; |
| 8244 | struct nft_rule *rule; | 8265 | struct nft_rule *rule; |
| 8245 | unsigned int alloc = 0; | | |
| 8246 | int i; | 8266 | int i; |
| 8247 | | 8267 | |
| 8248 | /* already handled or inactive chain? */ | 8268 | /* already handled or inactive chain? */ |
| 8249 | if (chain->rules_next || !nft_is_active_next(net, chain)) | 8269 | if (chain->blob_next || !nft_is_active_next(net, chain)) |
| 8250 | return 0; | 8270 | return 0; |
| 8251 | | 8271 | |
| 8252 | rule = list_entry(&chain->rules, struct nft_rule, list); | 8272 | rule = list_entry(&chain->rules, struct nft_rule, list); |
| 8253 | i = 0; | 8273 | i = 0; |
| 8254 | | 8274 | |
| 8255 | list_for_each_entry_continue(rule, &chain->rules, list) { | 8275 | list_for_each_entry_continue(rule, &chain->rules, list) { |
| 8256 | if (nft_is_active_next(net, rule)) | 8276 | if (nft_is_active_next(net, rule)) { |
| 8257 | alloc++; | 8277 | data_size += sizeof(*prule) + rule->dlen; |
| | 8278 | if (data_size > INT_MAX) |
| | 8279 | return -ENOMEM; |
| | 8280 | } |
| 8258 | } | 8281 | } |
| | 8282 | data_size += offsetof(struct nft_rule_dp, data); /* last rule */ |
| 8259 | | 8283 | |
| 8260 | chain->rules_next = nf_tables_chain_alloc_rules(chain, alloc); | 8284 | chain->blob_next = nf_tables_chain_alloc_rules(data_size); |
| 8261 | if (!chain->rules_next) | 8285 | if (!chain->blob_next) |
| 8262 | return -ENOMEM; | 8286 | return -ENOMEM; |
| 8263 | | 8287 | |
| | 8288 | data = (void *)chain->blob_next->data; |
| | 8289 | data_boundary = data + data_size; |
| | 8290 | size = 0; |
| | 8291 | |
| 8264 | list_for_each_entry_continue(rule, &chain->rules, list) { | 8292 | list_for_each_entry_continue(rule, &chain->rules, list) { |
| 8265 | if (nft_is_active_next(net, rule)) | 8293 | if (!nft_is_active_next(net, rule)) |
| 8266 | chain->rules_next[i++] = rule; | 8294 | continue; |
| | 8295 | |
| | 8296 | prule = (struct nft_rule_dp *)data; |
| | 8297 | data += offsetof(struct nft_rule_dp, data); |
| | 8298 | if (WARN_ON_ONCE(data > data_boundary)) |
| | 8299 | return -ENOMEM; |
| | 8300 | |
| | 8301 | nft_rule_for_each_expr(expr, last, rule) { |
| | 8302 | if (WARN_ON_ONCE(data + expr->ops->size > data_boundary)) |
| | 8303 | return -ENOMEM; |
| | 8304 | |
| | 8305 | memcpy(data + size, expr, expr->ops->size); |
| | 8306 | size += expr->ops->size; |
| | 8307 | } |
| | 8308 | if (WARN_ON_ONCE(size >= 1 << 12)) |
| | 8309 | return -ENOMEM; |
| | 8310 | |
| | 8311 | prule->handle = rule->handle; |
| | 8312 | prule->dlen = size; |
| | 8313 | prule->is_last = 0; |
| | 8314 | |
| | 8315 | data += size; |
| | 8316 | size = 0; |
| | 8317 | chain->blob_next->size += (unsigned long)(data - (void *)prule); |
| 8267 | } | 8318 | } |
| 8268 | | 8319 | |
| 8269 | chain->rules_next[i] = NULL; | 8320 | prule = (struct nft_rule_dp *)data; |
| | 8321 | data += offsetof(struct nft_rule_dp, data); |
| | 8322 | if (WARN_ON_ONCE(data > data_boundary)) |
| | 8323 | return -ENOMEM; |
| | 8324 | |
| | 8325 | nft_last_rule(chain->blob_next, prule); |
| | 8326 | |
| 8270 | return 0; | 8327 | return 0; |
| 8271 | } | 8328 | } |
| 8272 | | 8329 | |
| @@ -8280,8 +8337,8 @@ static void nf_tables_commit_chain_prepare_cancel(struct net *net) · 7 unmodified lines |
| 8280 | | 8337 | |
| 8281 | if (trans->msg_type == NFT_MSG_NEWRULE || | 8338 | if (trans->msg_type == NFT_MSG_NEWRULE || |
| 8282 | trans->msg_type == NFT_MSG_DELRULE) { | 8339 | trans->msg_type == NFT_MSG_DELRULE) { |
| 8283 | kvfree(chain->rules_next); | 8340 | kvfree(chain->blob_next); |
| 8284 | chain->rules_next = NULL; | 8341 | chain->blob_next = NULL; |
| 8285 | } | 8342 | } |
| 8286 | } | 8343 | } |
| 8287 | } | 8344 | } |
| @@ -8290,38 +8347,34 @@ static void __nf_tables_commit_chain_free_rules_old(struct rcu_head *h) · 2 unmodified lines |
| 8290 | { | 8347 | { |
| 8291 | struct nft_rules_old *o = container_of(h, struct nft_rules_old, h); | 8348 | struct nft_rules_old *o = container_of(h, struct nft_rules_old, h); |
| 8292 | | 8349 | |
| 8293 | kvfree(o->start); | 8350 | kvfree(o->blob); |
| 8294 | } | 8351 | } |
| 8295 | | 8352 | |
| 8296 | static void nf_tables_commit_chain_free_rules_old(struct nft_rule **rules) | 8353 | static void nf_tables_commit_chain_free_rules_old(struct nft_rule_blob *blob) |
| 8297 | { | 8354 | { |
| 8298 | struct nft_rule **r = rules; | | |
| 8299 | struct nft_rules_old *old; | 8355 | struct nft_rules_old *old; |
| 8300 | | 8356 | |
| 8301 | while (*r) | 8357 | /* rcu_head is after end marker */ |
| 8302 | r++; | 8358 | old = (void *)blob + sizeof(*blob) + blob->size; |
| 8303 | | 8359 | old->blob = blob; |
| 8304 | r++; /* rcu_head is after end marker */ | | |
| 8305 | old = (void *) r; | | |
| 8306 | old->start = rules; | | |
| 8307 | | 8360 | |
| 8308 | call_rcu(&old->h, __nf_tables_commit_chain_free_rules_old); | 8361 | call_rcu(&old->h, __nf_tables_commit_chain_free_rules_old); |
| 8309 | } | 8362 | } |
| 8310 | | 8363 | |
| 8311 | static void nf_tables_commit_chain(struct net *net, struct nft_chain *chain) | 8364 | static void nf_tables_commit_chain(struct net *net, struct nft_chain *chain) |
| 8312 | { | 8365 | { |
| 8313 | struct nft_rule **g0, **g1; | 8366 | struct nft_rule_blob *g0, *g1; |
| 8314 | bool next_genbit; | 8367 | bool next_genbit; |
| 8315 | | 8368 | |
| 8316 | next_genbit = nft_gencursor_next(net); | 8369 | next_genbit = nft_gencursor_next(net); |
| 8317 | | 8370 | |
| 8318 | g0 = rcu_dereference_protected(chain->rules_gen_0, | 8371 | g0 = rcu_dereference_protected(chain->blob_gen_0, |
| 8319 | lockdep_commit_lock_is_held(net)); | 8372 | lockdep_commit_lock_is_held(net)); |
| 8320 | g1 = rcu_dereference_protected(chain->rules_gen_1, | 8373 | g1 = rcu_dereference_protected(chain->blob_gen_1, |
| 8321 | lockdep_commit_lock_is_held(net)); | 8374 | lockdep_commit_lock_is_held(net)); |
| 8322 | | 8375 | |
| 8323 | /* No changes to this chain? */ | 8376 | /* No changes to this chain? */ |
| 8324 | if (chain->rules_next == NULL) { | 8377 | if (chain->blob_next == NULL) { |
| 8325 | /* chain had no change in last or next generation */ | 8378 | /* chain had no change in last or next generation */ |
| 8326 | if (g0 == g1) | 8379 | if (g0 == g1) |
| 8327 | return; | 8380 | return; |
| @@ -8330,10 +8383,10 @@ static void nf_tables_commit_chain(struct net *net, struct nft_chain *chain) · 2 unmodified lines |
| 8330 | * one uses same rules as current generation. | 8383 | * one uses same rules as current generation. |
| 8331 | */ | 8384 | */ |
| 8332 | if (next_genbit) { | 8385 | if (next_genbit) { |
| 8333 | rcu_assign_pointer(chain->rules_gen_1, g0); | 8386 | rcu_assign_pointer(chain->blob_gen_1, g0); |
| 8334 | nf_tables_commit_chain_free_rules_old(g1); | 8387 | nf_tables_commit_chain_free_rules_old(g1); |
| 8335 | } else { | 8388 | } else { |
| 8336 | rcu_assign_pointer(chain->rules_gen_0, g1); | 8389 | rcu_assign_pointer(chain->blob_gen_0, g1); |
| 8337 | nf_tables_commit_chain_free_rules_old(g0); | 8390 | nf_tables_commit_chain_free_rules_old(g0); |
| 8338 | } | 8391 | } |
| 8339 | | 8392 | |
| @@ -8341,11 +8394,11 @@ static void nf_tables_commit_chain(struct net *net, struct nft_chain *chain) · 1 unmodified line |
| 8341 | } | 8394 | } |
| 8342 | | 8395 | |
| 8343 | if (next_genbit) | 8396 | if (next_genbit) |
| 8344 | rcu_assign_pointer(chain->rules_gen_1, chain->rules_next); | 8397 | rcu_assign_pointer(chain->blob_gen_1, chain->blob_next); |
| 8345 | else | 8398 | else |
| 8346 | rcu_assign_pointer(chain->rules_gen_0, chain->rules_next); | 8399 | rcu_assign_pointer(chain->blob_gen_0, chain->blob_next); |
| 8347 | | 8400 | |
| 8348 | chain->rules_next = NULL; | 8401 | chain->blob_next = NULL; |
| 8349 | | 8402 | |
| 8350 | if (g0 == g1) | 8403 | if (g0 == g1) |
| 8351 | return; | 8404 | return; |