History

Merge tag 'nf-23-08-10' of git://git.kernel.org/pub/scm/linux/kernel/git/netfilter/nf

Pablo Neira Ayuso says: ==================== Netfilter fixes for net The existing attempt to resolve races between control plane and GC work is error prone, as reported by Bien Pham <phamnnb@sea.com>, some places forgot to call nft_set_elem_mark_busy(), leading to double-deactivation of elements. This series contains the following patches: 1) Do not skip expired elements during walk otherwise elements might never decrement the reference counter on data, leading to memleak. 2) Add a GC transaction API to replace the former attempt to deal with races between control plane and GC. GC worker sets on NFT_SET_ELEM_DEAD_BIT on elements and it creates a GC transaction to remove the expired elements, GC transaction could abort in case of interference with control plane and retried later (GC async). Set backends such as rbtree and pipapo also perform GC from control plane (GC sync), in such case, element deactivation and removal is safe because mutex is held then collected elements are released via call_rcu(). 3) Adapt existing set backends to use the GC transaction API. 4) Update rhash set backend to set on _DEAD bit to report deleted elements from datapath for GC. 5) Remove old GC batch API and the NFT_SET_ELEM_BUSY_BIT. * tag 'nf-23-08-10' of git://git.kernel.org/pub/scm/linux/kernel/git/netfilter/nf: netfilter: nf_tables: remove busy mark and gc batch API netfilter: nft_set_hash: mark set element as dead when deleting from packet path netfilter: nf_tables: adapt set backend to use GC transaction API netfilter: nf_tables: GC transaction API to avoid race with control plane netfilter: nf_tables: don't skip expired elements during walk ====================

Jakub Kicinski committed 3 years ago

Commit 3e91b0e2 parents 62d02fc a2dd0235 files · +476 −248Browse files

Unified · Split

5 files
  1. include/net/netfilter/nf_tables.h +45 −75
  2. net/netfilter/nf_tables_api.c +244 −63
  3. net/netfilter/nft_set_hash.c +52 −33
  4. net/netfilter/nft_set_pipapo.c +48 −18
  5. net/netfilter/nft_set_rbtree.c +87 −59
include/net/netfilter/nf_tables.h+45−75
@@ -512,6 +512,7 @@ struct nft_set_elem_expr { · 511 unmodified lines
512 *512 *
513 * @list: table set list node513 * @list: table set list node
514 * @bindings: list of set bindings514 * @bindings: list of set bindings
515 * @refs: internal refcounting for async set destruction
515 * @table: table this set belongs to516 * @table: table this set belongs to
516 * @net: netnamespace this set belongs to517 * @net: netnamespace this set belongs to
517 * @name: name of the set518 * @name: name of the set
@@ -541,6 +542,7 @@ struct nft_set_elem_expr { · 23 unmodified lines
541struct nft_set {542struct nft_set {
542 struct list_head list;543 struct list_head list;
543 struct list_head bindings;544 struct list_head bindings;
545 refcount_t refs;
544 struct nft_table *table;546 struct nft_table *table;
545 possible_net_t net;547 possible_net_t net;
546 char *name;548 char *name;
@@ -562,7 +564,8 @@ struct nft_set { · 15 unmodified lines
562 struct list_head pending_update;564 struct list_head pending_update;
563 /* runtime data below here */565 /* runtime data below here */
564 const struct nft_set_ops *ops ____cacheline_aligned;566 const struct nft_set_ops *ops ____cacheline_aligned;
565 u16 flags:14,567 u16 flags:13,
568 dead:1,
566 genmask:2;569 genmask:2;
567 u8 klen;570 u8 klen;
568 u8 dlen;571 u8 dlen;
@@ -596,7 +599,6 @@ struct nft_set *nft_set_lookup_global(const struct net *net, · 27 unmodified lines
596 599
597struct nft_set_ext *nft_set_catchall_lookup(const struct net *net,600struct nft_set_ext *nft_set_catchall_lookup(const struct net *net,
598 const struct nft_set *set);601 const struct nft_set *set);
599void *nft_set_catchall_gc(const struct nft_set *set);
600 602
601static inline unsigned long nft_set_gc_interval(const struct nft_set *set)603static inline unsigned long nft_set_gc_interval(const struct nft_set *set)
602{604{
@@ -813,62 +815,6 @@ void nft_set_elem_destroy(const struct nft_set *set, void *elem, · 210 unmodified lines
813void nf_tables_set_elem_destroy(const struct nft_ctx *ctx,815void nf_tables_set_elem_destroy(const struct nft_ctx *ctx,
814 const struct nft_set *set, void *elem);816 const struct nft_set *set, void *elem);
815 817
816/**
817 * struct nft_set_gc_batch_head - nf_tables set garbage collection batch
818 *
819 * @rcu: rcu head
820 * @set: set the elements belong to
821 * @cnt: count of elements
822 */
823struct nft_set_gc_batch_head {
824 struct rcu_head rcu;
825 const struct nft_set *set;
826 unsigned int cnt;
827};
828
829#define NFT_SET_GC_BATCH_SIZE ((PAGE_SIZE - \
830 sizeof(struct nft_set_gc_batch_head)) / \
831 sizeof(void *))
832
833/**
834 * struct nft_set_gc_batch - nf_tables set garbage collection batch
835 *
836 * @head: GC batch head
837 * @elems: garbage collection elements
838 */
839struct nft_set_gc_batch {
840 struct nft_set_gc_batch_head head;
841 void *elems[NFT_SET_GC_BATCH_SIZE];
842};
843
844struct nft_set_gc_batch *nft_set_gc_batch_alloc(const struct nft_set *set,
845 gfp_t gfp);
846void nft_set_gc_batch_release(struct rcu_head *rcu);
847
848static inline void nft_set_gc_batch_complete(struct nft_set_gc_batch *gcb)
849{
850 if (gcb != NULL)
851 call_rcu(&gcb->head.rcu, nft_set_gc_batch_release);
852}
853
854static inline struct nft_set_gc_batch *
855nft_set_gc_batch_check(const struct nft_set *set, struct nft_set_gc_batch *gcb,
856 gfp_t gfp)
857{
858 if (gcb != NULL) {
859 if (gcb->head.cnt + 1 < ARRAY_SIZE(gcb->elems))
860 return gcb;
861 nft_set_gc_batch_complete(gcb);
862 }
863 return nft_set_gc_batch_alloc(set, gfp);
864}
865
866static inline void nft_set_gc_batch_add(struct nft_set_gc_batch *gcb,
867 void *elem)
868{
869 gcb->elems[gcb->head.cnt++] = elem;
870}
871
872struct nft_expr_ops;818struct nft_expr_ops;
873/**819/**
874 * struct nft_expr_type - nf_tables expression type820 * struct nft_expr_type - nf_tables expression type
@@ -1557,39 +1503,30 @@ static inline void nft_set_elem_change_active(const struct net *net, · 682 unmodified lines
1557 1503
1558#endif /* IS_ENABLED(CONFIG_NF_TABLES) */1504#endif /* IS_ENABLED(CONFIG_NF_TABLES) */
1559 1505
1560/*1506#define NFT_SET_ELEM_DEAD_MASK (1 << 2)
1561 * We use a free bit in the genmask field to indicate the element
1562 * is busy, meaning it is currently being processed either by
1563 * the netlink API or GC.
1564 *
1565 * Even though the genmask is only a single byte wide, this works
1566 * because the extension structure if fully constant once initialized,
1567 * so there are no non-atomic write accesses unless it is already
1568 * marked busy.
1569 */
1570#define NFT_SET_ELEM_BUSY_MASK (1 << 2)
1571 1507
1572#if defined(__LITTLE_ENDIAN_BITFIELD)1508#if defined(__LITTLE_ENDIAN_BITFIELD)
1573#define NFT_SET_ELEM_BUSY_BIT 21509#define NFT_SET_ELEM_DEAD_BIT 2
1574#elif defined(__BIG_ENDIAN_BITFIELD)1510#elif defined(__BIG_ENDIAN_BITFIELD)
1575#define NFT_SET_ELEM_BUSY_BIT (BITS_PER_LONG - BITS_PER_BYTE + 2)1511#define NFT_SET_ELEM_DEAD_BIT (BITS_PER_LONG - BITS_PER_BYTE + 2)
1576#else1512#else
1577#error1513#error
1578#endif1514#endif
1579 1515
1580static inline int nft_set_elem_mark_busy(struct nft_set_ext *ext)1516static inline void nft_set_elem_dead(struct nft_set_ext *ext)
1581{1517{
1582 unsigned long *word = (unsigned long *)ext;1518 unsigned long *word = (unsigned long *)ext;
1583 1519
1584 BUILD_BUG_ON(offsetof(struct nft_set_ext, genmask) != 0);1520 BUILD_BUG_ON(offsetof(struct nft_set_ext, genmask) != 0);
1585 return test_and_set_bit(NFT_SET_ELEM_BUSY_BIT, word);1521 set_bit(NFT_SET_ELEM_DEAD_BIT, word);
1586}1522}
1587 1523
1588static inline void nft_set_elem_clear_busy(struct nft_set_ext *ext)1524static inline int nft_set_elem_is_dead(const struct nft_set_ext *ext)
1589{1525{
1590 unsigned long *word = (unsigned long *)ext;1526 unsigned long *word = (unsigned long *)ext;
1591 1527
1592 clear_bit(NFT_SET_ELEM_BUSY_BIT, word);1528 BUILD_BUG_ON(offsetof(struct nft_set_ext, genmask) != 0);
1529 return test_bit(NFT_SET_ELEM_DEAD_BIT, word);
1593}1530}
1594 1531
1595/**1532/**
@@ -1732,6 +1669,38 @@ struct nft_trans_flowtable { · 136 unmodified lines
1732#define nft_trans_flowtable_flags(trans) \1669#define nft_trans_flowtable_flags(trans) \
1733 (((struct nft_trans_flowtable *)trans->data)->flags)1670 (((struct nft_trans_flowtable *)trans->data)->flags)
1734 1671
1672#define NFT_TRANS_GC_BATCHCOUNT 256
1673
1674struct nft_trans_gc {
1675 struct list_head list;
1676 struct net *net;
1677 struct nft_set *set;
1678 u32 seq;
1679 u8 count;
1680 void *priv[NFT_TRANS_GC_BATCHCOUNT];
1681 struct rcu_head rcu;
1682};
1683
1684struct nft_trans_gc *nft_trans_gc_alloc(struct nft_set *set,
1685 unsigned int gc_seq, gfp_t gfp);
1686void nft_trans_gc_destroy(struct nft_trans_gc *trans);
1687
1688struct nft_trans_gc *nft_trans_gc_queue_async(struct nft_trans_gc *gc,
1689 unsigned int gc_seq, gfp_t gfp);
1690void nft_trans_gc_queue_async_done(struct nft_trans_gc *gc);
1691
1692struct nft_trans_gc *nft_trans_gc_queue_sync(struct nft_trans_gc *gc, gfp_t gfp);
1693void nft_trans_gc_queue_sync_done(struct nft_trans_gc *trans);
1694
1695void nft_trans_gc_elem_add(struct nft_trans_gc *gc, void *priv);
1696
1697struct nft_trans_gc *nft_trans_gc_catchall(struct nft_trans_gc *gc,
1698 unsigned int gc_seq);
1699
1700void nft_setelem_data_deactivate(const struct net *net,
1701 const struct nft_set *set,
1702 struct nft_set_elem *elem);
1703
1735int __init nft_chain_filter_init(void);1704int __init nft_chain_filter_init(void);
1736void nft_chain_filter_fini(void);1705void nft_chain_filter_fini(void);
1737 1706
@@ -1758,6 +1727,7 @@ struct nftables_pernet { · 20 unmodified lines
1758 struct mutex commit_mutex;1727 struct mutex commit_mutex;
1759 u64 table_handle;1728 u64 table_handle;
1760 unsigned int base_seq;1729 unsigned int base_seq;
1730 unsigned int gc_seq;
1761};1731};
1762 1732
1763extern unsigned int nf_tables_net_id;1733extern unsigned int nf_tables_net_id;
net/netfilter/nf_tables_api.c+244−63
@@ -31,7 +31,9 @@ static LIST_HEAD(nf_tables_expressions); · 30 unmodified lines
31static LIST_HEAD(nf_tables_objects);31static LIST_HEAD(nf_tables_objects);
32static LIST_HEAD(nf_tables_flowtables);32static LIST_HEAD(nf_tables_flowtables);
33static LIST_HEAD(nf_tables_destroy_list);33static LIST_HEAD(nf_tables_destroy_list);
34static LIST_HEAD(nf_tables_gc_list);
34static DEFINE_SPINLOCK(nf_tables_destroy_list_lock);35static DEFINE_SPINLOCK(nf_tables_destroy_list_lock);
36static DEFINE_SPINLOCK(nf_tables_gc_list_lock);
35 37
36enum {38enum {
37 NFT_VALIDATE_SKIP = 0,39 NFT_VALIDATE_SKIP = 0,
@@ -120,6 +122,9 @@ static void nft_validate_state_update(struct nft_table *table, u8 new_validate_s · 82 unmodified lines
120static void nf_tables_trans_destroy_work(struct work_struct *w);122static void nf_tables_trans_destroy_work(struct work_struct *w);
121static DECLARE_WORK(trans_destroy_work, nf_tables_trans_destroy_work);123static DECLARE_WORK(trans_destroy_work, nf_tables_trans_destroy_work);
122 124
125static void nft_trans_gc_work(struct work_struct *work);
126static DECLARE_WORK(trans_gc_work, nft_trans_gc_work);
127
123static void nft_ctx_init(struct nft_ctx *ctx,128static void nft_ctx_init(struct nft_ctx *ctx,
124 struct net *net,129 struct net *net,
125 const struct sk_buff *skb,130 const struct sk_buff *skb,
@@ -582,10 +587,6 @@ static int nft_trans_set_add(const struct nft_ctx *ctx, int msg_type, · 456 unmodified lines
582 return __nft_trans_set_add(ctx, msg_type, set, NULL);587 return __nft_trans_set_add(ctx, msg_type, set, NULL);
583}588}
584 589
585static void nft_setelem_data_deactivate(const struct net *net,
586 const struct nft_set *set,
587 struct nft_set_elem *elem);
588
589static int nft_mapelem_deactivate(const struct nft_ctx *ctx,590static int nft_mapelem_deactivate(const struct nft_ctx *ctx,
590 struct nft_set *set,591 struct nft_set *set,
591 const struct nft_set_iter *iter,592 const struct nft_set_iter *iter,
@@ -5055,6 +5056,7 @@ static int nf_tables_newset(struct sk_buff *skb, const struct nfnl_info *info, · 4463 unmodified lines
5055 5056
5056 INIT_LIST_HEAD(&set->bindings);5057 INIT_LIST_HEAD(&set->bindings);
5057 INIT_LIST_HEAD(&set->catchall_list);5058 INIT_LIST_HEAD(&set->catchall_list);
5059 refcount_set(&set->refs, 1);
5058 set->table = table;5060 set->table = table;
5059 write_pnet(&set->net, net);5061 write_pnet(&set->net, net);
5060 set->ops = ops;5062 set->ops = ops;
@@ -5122,6 +5124,14 @@ static void nft_set_catchall_destroy(const struct nft_ctx *ctx, · 61 unmodified lines
5122 }5124 }
5123}5125}
5124 5126
5127static void nft_set_put(struct nft_set *set)
5128{
5129 if (refcount_dec_and_test(&set->refs)) {
5130 kfree(set->name);
5131 kvfree(set);
5132 }
5133}
5134
5125static void nft_set_destroy(const struct nft_ctx *ctx, struct nft_set *set)5135static void nft_set_destroy(const struct nft_ctx *ctx, struct nft_set *set)
5126{5136{
5127 int i;5137 int i;
@@ -5134,8 +5144,7 @@ static void nft_set_destroy(const struct nft_ctx *ctx, struct nft_set *set) · 6 unmodified lines
5134 5144
5135 set->ops->destroy(ctx, set);5145 set->ops->destroy(ctx, set);
5136 nft_set_catchall_destroy(ctx, set);5146 nft_set_catchall_destroy(ctx, set);
5137 kfree(set->name);5147 nft_set_put(set);
5138 kvfree(set);
5139}5148}
5140 5149
5141static int nf_tables_delset(struct sk_buff *skb, const struct nfnl_info *info,5150static int nf_tables_delset(struct sk_buff *skb, const struct nfnl_info *info,
@@ -5602,8 +5611,12 @@ static int nf_tables_dump_setelem(const struct nft_ctx *ctx, · 460 unmodified lines
5602 const struct nft_set_iter *iter,5611 const struct nft_set_iter *iter,
5603 struct nft_set_elem *elem)5612 struct nft_set_elem *elem)
5604{5613{
5614 const struct nft_set_ext *ext = nft_set_elem_ext(set, elem->priv);
5605 struct nft_set_dump_args *args;5615 struct nft_set_dump_args *args;
5606 5616
5617 if (nft_set_elem_expired(ext))
5618 return 0;
5619
5607 args = container_of(iter, struct nft_set_dump_args, iter);5620 args = container_of(iter, struct nft_set_dump_args, iter);
5608 return nf_tables_fill_setelem(args->skb, set, elem, args->reset);5621 return nf_tables_fill_setelem(args->skb, set, elem, args->reset);
5609}5622}
@@ -6274,7 +6287,8 @@ struct nft_set_ext *nft_set_catchall_lookup(const struct net *net, · 664 unmodified lines
6274 list_for_each_entry_rcu(catchall, &set->catchall_list, list) {6287 list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
6275 ext = nft_set_elem_ext(set, catchall->elem);6288 ext = nft_set_elem_ext(set, catchall->elem);
6276 if (nft_set_elem_active(ext, genmask) &&6289 if (nft_set_elem_active(ext, genmask) &&
6277 !nft_set_elem_expired(ext))6290 !nft_set_elem_expired(ext) &&
6291 !nft_set_elem_is_dead(ext))
6278 return ext;6292 return ext;
6279 }6293 }
6280 6294
@@ -6282,29 +6296,6 @@ struct nft_set_ext *nft_set_catchall_lookup(const struct net *net, · 1 unmodified line
6282}6296}
6283EXPORT_SYMBOL_GPL(nft_set_catchall_lookup);6297EXPORT_SYMBOL_GPL(nft_set_catchall_lookup);
6284 6298
6285void *nft_set_catchall_gc(const struct nft_set *set)
6286{
6287 struct nft_set_elem_catchall *catchall, *next;
6288 struct nft_set_ext *ext;
6289 void *elem = NULL;
6290
6291 list_for_each_entry_safe(catchall, next, &set->catchall_list, list) {
6292 ext = nft_set_elem_ext(set, catchall->elem);
6293
6294 if (!nft_set_elem_expired(ext) ||
6295 nft_set_elem_mark_busy(ext))
6296 continue;
6297
6298 elem = catchall->elem;
6299 list_del_rcu(&catchall->list);
6300 kfree_rcu(catchall, rcu);
6301 break;
6302 }
6303
6304 return elem;
6305}
6306EXPORT_SYMBOL_GPL(nft_set_catchall_gc);
6307
6308static int nft_setelem_catchall_insert(const struct net *net,6299static int nft_setelem_catchall_insert(const struct net *net,
6309 struct nft_set *set,6300 struct nft_set *set,
6310 const struct nft_set_elem *elem,6301 const struct nft_set_elem *elem,
@@ -6366,7 +6357,6 @@ static void nft_setelem_activate(struct net *net, struct nft_set *set, · 55 unmodified lines
6366 6357
6367 if (nft_setelem_is_catchall(set, elem)) {6358 if (nft_setelem_is_catchall(set, elem)) {
6368 nft_set_elem_change_active(net, set, ext);6359 nft_set_elem_change_active(net, set, ext);
6369 nft_set_elem_clear_busy(ext);
6370 } else {6360 } else {
6371 set->ops->activate(net, set, elem);6361 set->ops->activate(net, set, elem);
6372 }6362 }
@@ -6381,8 +6371,7 @@ static int nft_setelem_catchall_deactivate(const struct net *net, · 8 unmodified lines
6381 6371
6382 list_for_each_entry(catchall, &set->catchall_list, list) {6372 list_for_each_entry(catchall, &set->catchall_list, list) {
6383 ext = nft_set_elem_ext(set, catchall->elem);6373 ext = nft_set_elem_ext(set, catchall->elem);
6384 if (!nft_is_active(net, ext) ||6374 if (!nft_is_active(net, ext))
6385 nft_set_elem_mark_busy(ext))
6386 continue;6375 continue;
6387 6376
6388 kfree(elem->priv);6377 kfree(elem->priv);
@@ -6777,7 +6766,7 @@ static int nft_add_set_elem(struct nft_ctx *ctx, struct nft_set *set, · 388 unmodified lines
6777 goto err_elem_free;6766 goto err_elem_free;
6778 }6767 }
6779 6768
6780 ext->genmask = nft_genmask_cur(ctx->net) | NFT_SET_ELEM_BUSY_MASK;6769 ext->genmask = nft_genmask_cur(ctx->net);
6781 6770
6782 err = nft_setelem_insert(ctx->net, set, &elem, &ext2, flags);6771 err = nft_setelem_insert(ctx->net, set, &elem, &ext2, flags);
6783 if (err) {6772 if (err) {
@@ -6929,9 +6918,9 @@ static void nft_setelem_data_activate(const struct net *net, · 145 unmodified lines
6929 nft_use_inc_restore(&(*nft_set_ext_obj(ext))->use);6918 nft_use_inc_restore(&(*nft_set_ext_obj(ext))->use);
6930}6919}
6931 6920
6932static void nft_setelem_data_deactivate(const struct net *net,6921void nft_setelem_data_deactivate(const struct net *net,
6933 const struct nft_set *set,6922 const struct nft_set *set,
6934 struct nft_set_elem *elem)6923 struct nft_set_elem *elem)
6935{6924{
6936 const struct nft_set_ext *ext = nft_set_elem_ext(set, elem->priv);6925 const struct nft_set_ext *ext = nft_set_elem_ext(set, elem->priv);
6937 6926
@@ -7095,8 +7084,7 @@ static int nft_set_catchall_flush(const struct nft_ctx *ctx, · 157 unmodified lines
7095 7084
7096 list_for_each_entry_rcu(catchall, &set->catchall_list, list) {7085 list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
7097 ext = nft_set_elem_ext(set, catchall->elem);7086 ext = nft_set_elem_ext(set, catchall->elem);
7098 if (!nft_set_elem_active(ext, genmask) ||7087 if (!nft_set_elem_active(ext, genmask))
7099 nft_set_elem_mark_busy(ext))
7100 continue;7088 continue;
7101 7089
7102 elem.priv = catchall->elem;7090 elem.priv = catchall->elem;
@@ -7170,29 +7158,6 @@ static int nf_tables_delsetelem(struct sk_buff *skb, · 67 unmodified lines
7170 return err;7158 return err;
7171}7159}
7172 7160
7173void nft_set_gc_batch_release(struct rcu_head *rcu)
7174{
7175 struct nft_set_gc_batch *gcb;
7176 unsigned int i;
7177
7178 gcb = container_of(rcu, struct nft_set_gc_batch, head.rcu);
7179 for (i = 0; i < gcb->head.cnt; i++)
7180 nft_set_elem_destroy(gcb->head.set, gcb->elems[i], true);
7181 kfree(gcb);
7182}
7183
7184struct nft_set_gc_batch *nft_set_gc_batch_alloc(const struct nft_set *set,
7185 gfp_t gfp)
7186{
7187 struct nft_set_gc_batch *gcb;
7188
7189 gcb = kzalloc(sizeof(*gcb), gfp);
7190 if (gcb == NULL)
7191 return gcb;
7192 gcb->head.set = set;
7193 return gcb;
7194}
7195
7196/*7161/*
7197 * Stateful objects7162 * Stateful objects
7198 */7163 */
@@ -9414,6 +9379,207 @@ void nft_chain_del(struct nft_chain *chain) · 2215 unmodified lines
9414 list_del_rcu(&chain->list);9379 list_del_rcu(&chain->list);
9415}9380}
9416 9381
9382static void nft_trans_gc_setelem_remove(struct nft_ctx *ctx,
9383 struct nft_trans_gc *trans)
9384{
9385 void **priv = trans->priv;
9386 unsigned int i;
9387
9388 for (i = 0; i < trans->count; i++) {
9389 struct nft_set_elem elem = {
9390 .priv = priv[i],
9391 };
9392
9393 nft_setelem_data_deactivate(ctx->net, trans->set, &elem);
9394 nft_setelem_remove(ctx->net, trans->set, &elem);
9395 }
9396}
9397
9398void nft_trans_gc_destroy(struct nft_trans_gc *trans)
9399{
9400 nft_set_put(trans->set);
9401 put_net(trans->net);
9402 kfree(trans);
9403}
9404
9405static void nft_trans_gc_trans_free(struct rcu_head *rcu)
9406{
9407 struct nft_set_elem elem = {};
9408 struct nft_trans_gc *trans;
9409 struct nft_ctx ctx = {};
9410 unsigned int i;
9411
9412 trans = container_of(rcu, struct nft_trans_gc, rcu);
9413 ctx.net = read_pnet(&trans->set->net);
9414
9415 for (i = 0; i < trans->count; i++) {
9416 elem.priv = trans->priv[i];
9417 if (!nft_setelem_is_catchall(trans->set, &elem))
9418 atomic_dec(&trans->set->nelems);
9419
9420 nf_tables_set_elem_destroy(&ctx, trans->set, elem.priv);
9421 }
9422
9423 nft_trans_gc_destroy(trans);
9424}
9425
9426static bool nft_trans_gc_work_done(struct nft_trans_gc *trans)
9427{
9428 struct nftables_pernet *nft_net;
9429 struct nft_ctx ctx = {};
9430
9431 nft_net = nft_pernet(trans->net);
9432
9433 mutex_lock(&nft_net->commit_mutex);
9434
9435 /* Check for race with transaction, otherwise this batch refers to
9436 * stale objects that might not be there anymore. Skip transaction if
9437 * set has been destroyed from control plane transaction in case gc
9438 * worker loses race.
9439 */
9440 if (READ_ONCE(nft_net->gc_seq) != trans->seq || trans->set->dead) {
9441 mutex_unlock(&nft_net->commit_mutex);
9442 return false;
9443 }
9444
9445 ctx.net = trans->net;
9446 ctx.table = trans->set->table;
9447
9448 nft_trans_gc_setelem_remove(&ctx, trans);
9449 mutex_unlock(&nft_net->commit_mutex);
9450
9451 return true;
9452}
9453
9454static void nft_trans_gc_work(struct work_struct *work)
9455{
9456 struct nft_trans_gc *trans, *next;
9457 LIST_HEAD(trans_gc_list);
9458
9459 spin_lock(&nf_tables_destroy_list_lock);
9460 list_splice_init(&nf_tables_gc_list, &trans_gc_list);
9461 spin_unlock(&nf_tables_destroy_list_lock);
9462
9463 list_for_each_entry_safe(trans, next, &trans_gc_list, list) {
9464 list_del(&trans->list);
9465 if (!nft_trans_gc_work_done(trans)) {
9466 nft_trans_gc_destroy(trans);
9467 continue;
9468 }
9469 call_rcu(&trans->rcu, nft_trans_gc_trans_free);
9470 }
9471}
9472
9473struct nft_trans_gc *nft_trans_gc_alloc(struct nft_set *set,
9474 unsigned int gc_seq, gfp_t gfp)
9475{
9476 struct net *net = read_pnet(&set->net);
9477 struct nft_trans_gc *trans;
9478
9479 trans = kzalloc(sizeof(*trans), gfp);
9480 if (!trans)
9481 return NULL;
9482
9483 refcount_inc(&set->refs);
9484 trans->set = set;
9485 trans->net = get_net(net);
9486 trans->seq = gc_seq;
9487
9488 return trans;
9489}
9490
9491void nft_trans_gc_elem_add(struct nft_trans_gc *trans, void *priv)
9492{
9493 trans->priv[trans->count++] = priv;
9494}
9495
9496static void nft_trans_gc_queue_work(struct nft_trans_gc *trans)
9497{
9498 spin_lock(&nf_tables_gc_list_lock);
9499 list_add_tail(&trans->list, &nf_tables_gc_list);
9500 spin_unlock(&nf_tables_gc_list_lock);
9501
9502 schedule_work(&trans_gc_work);
9503}
9504
9505static int nft_trans_gc_space(struct nft_trans_gc *trans)
9506{
9507 return NFT_TRANS_GC_BATCHCOUNT - trans->count;
9508}
9509
9510struct nft_trans_gc *nft_trans_gc_queue_async(struct nft_trans_gc *gc,
9511 unsigned int gc_seq, gfp_t gfp)
9512{
9513 if (nft_trans_gc_space(gc))
9514 return gc;
9515
9516 nft_trans_gc_queue_work(gc);
9517
9518 return nft_trans_gc_alloc(gc->set, gc_seq, gfp);
9519}
9520
9521void nft_trans_gc_queue_async_done(struct nft_trans_gc *trans)
9522{
9523 if (trans->count == 0) {
9524 nft_trans_gc_destroy(trans);
9525 return;
9526 }
9527
9528 nft_trans_gc_queue_work(trans);
9529}
9530
9531struct nft_trans_gc *nft_trans_gc_queue_sync(struct nft_trans_gc *gc, gfp_t gfp)
9532{
9533 if (WARN_ON_ONCE(!lockdep_commit_lock_is_held(gc->net)))
9534 return NULL;
9535
9536 if (nft_trans_gc_space(gc))
9537 return gc;
9538
9539 call_rcu(&gc->rcu, nft_trans_gc_trans_free);
9540
9541 return nft_trans_gc_alloc(gc->set, 0, gfp);
9542}
9543
9544void nft_trans_gc_queue_sync_done(struct nft_trans_gc *trans)
9545{
9546 WARN_ON_ONCE(!lockdep_commit_lock_is_held(trans->net));
9547
9548 if (trans->count == 0) {
9549 nft_trans_gc_destroy(trans);
9550 return;
9551 }
9552
9553 call_rcu(&trans->rcu, nft_trans_gc_trans_free);
9554}
9555
9556struct nft_trans_gc *nft_trans_gc_catchall(struct nft_trans_gc *gc,
9557 unsigned int gc_seq)
9558{
9559 struct nft_set_elem_catchall *catchall;
9560 const struct nft_set *set = gc->set;
9561 struct nft_set_ext *ext;
9562
9563 list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
9564 ext = nft_set_elem_ext(set, catchall->elem);
9565
9566 if (!nft_set_elem_expired(ext))
9567 continue;
9568 if (nft_set_elem_is_dead(ext))
9569 goto dead_elem;
9570
9571 nft_set_elem_dead(ext);
9572dead_elem:
9573 gc = nft_trans_gc_queue_async(gc, gc_seq, GFP_ATOMIC);
9574 if (!gc)
9575 return NULL;
9576
9577 nft_trans_gc_elem_add(gc, catchall->elem);
9578 }
9579
9580 return gc;
9581}
9582
9417static void nf_tables_module_autoload_cleanup(struct net *net)9583static void nf_tables_module_autoload_cleanup(struct net *net)
9418{9584{
9419 struct nftables_pernet *nft_net = nft_pernet(net);9585 struct nftables_pernet *nft_net = nft_pernet(net);
@@ -9576,11 +9742,11 @@ static int nf_tables_commit(struct net *net, struct sk_buff *skb) · 156 unmodified lines
9576{9742{
9577 struct nftables_pernet *nft_net = nft_pernet(net);9743 struct nftables_pernet *nft_net = nft_pernet(net);
9578 struct nft_trans *trans, *next;9744 struct nft_trans *trans, *next;
9745 unsigned int base_seq, gc_seq;
9579 LIST_HEAD(set_update_list);9746 LIST_HEAD(set_update_list);
9580 struct nft_trans_elem *te;9747 struct nft_trans_elem *te;
9581 struct nft_chain *chain;9748 struct nft_chain *chain;
9582 struct nft_table *table;9749 struct nft_table *table;
9583 unsigned int base_seq;
9584 LIST_HEAD(adl);9750 LIST_HEAD(adl);
9585 int err;9751 int err;
9586 9752
@@ -9657,6 +9823,10 @@ static int nf_tables_commit(struct net *net, struct sk_buff *skb) · 70 unmodified lines
9657 9823
9658 WRITE_ONCE(nft_net->base_seq, base_seq);9824 WRITE_ONCE(nft_net->base_seq, base_seq);
9659 9825
9826 /* Bump gc counter, it becomes odd, this is the busy mark. */
9827 gc_seq = READ_ONCE(nft_net->gc_seq);
9828 WRITE_ONCE(nft_net->gc_seq, ++gc_seq);
9829
9660 /* step 3. Start new generation, rules_gen_X now in use. */9830 /* step 3. Start new generation, rules_gen_X now in use. */
9661 net->nft.gencursor = nft_gencursor_next(net);9831 net->nft.gencursor = nft_gencursor_next(net);
9662 9832
@@ -9764,6 +9934,7 @@ static int nf_tables_commit(struct net *net, struct sk_buff *skb) · 101 unmodified lines
9764 break;9934 break;
9765 case NFT_MSG_DELSET:9935 case NFT_MSG_DELSET:
9766 case NFT_MSG_DESTROYSET:9936 case NFT_MSG_DESTROYSET:
9937 nft_trans_set(trans)->dead = 1;
9767 list_del_rcu(&nft_trans_set(trans)->list);9938 list_del_rcu(&nft_trans_set(trans)->list);
9768 nf_tables_set_notify(&trans->ctx, nft_trans_set(trans),9939 nf_tables_set_notify(&trans->ctx, nft_trans_set(trans),
9769 trans->msg_type, GFP_KERNEL);9940 trans->msg_type, GFP_KERNEL);
@@ -9866,6 +10037,8 @@ static int nf_tables_commit(struct net *net, struct sk_buff *skb) · 96 unmodified lines
9866 nft_commit_notify(net, NETLINK_CB(skb).portid);10037 nft_commit_notify(net, NETLINK_CB(skb).portid);
9867 nf_tables_gen_notify(net, skb, NFT_MSG_NEWGEN);10038 nf_tables_gen_notify(net, skb, NFT_MSG_NEWGEN);
9868 nf_tables_commit_audit_log(&adl, nft_net->base_seq);10039 nf_tables_commit_audit_log(&adl, nft_net->base_seq);
10040
10041 WRITE_ONCE(nft_net->gc_seq, ++gc_seq);
9869 nf_tables_commit_release(net);10042 nf_tables_commit_release(net);
9870 10043
9871 return 0;10044 return 0;
@@ -10915,6 +11088,7 @@ static int __net_init nf_tables_init_net(struct net *net) · 1043 unmodified lines
10915 INIT_LIST_HEAD(&nft_net->notify_list);11088 INIT_LIST_HEAD(&nft_net->notify_list);
10916 mutex_init(&nft_net->commit_mutex);11089 mutex_init(&nft_net->commit_mutex);
10917 nft_net->base_seq = 1;11090 nft_net->base_seq = 1;
11091 nft_net->gc_seq = 0;
10918 11092
10919 return 0;11093 return 0;
10920}11094}
@@ -10943,10 +11117,16 @@ static void __net_exit nf_tables_exit_net(struct net *net) · 22 unmodified lines
10943 WARN_ON_ONCE(!list_empty(&nft_net->notify_list));11117 WARN_ON_ONCE(!list_empty(&nft_net->notify_list));
10944}11118}
10945 11119
11120static void nf_tables_exit_batch(struct list_head *net_exit_list)
11121{
11122 flush_work(&trans_gc_work);
11123}
11124
10946static struct pernet_operations nf_tables_net_ops = {11125static struct pernet_operations nf_tables_net_ops = {
10947 .init = nf_tables_init_net,11126 .init = nf_tables_init_net,
10948 .pre_exit = nf_tables_pre_exit_net,11127 .pre_exit = nf_tables_pre_exit_net,
10949 .exit = nf_tables_exit_net,11128 .exit = nf_tables_exit_net,
11129 .exit_batch = nf_tables_exit_batch,
10950 .id = &nf_tables_net_id,11130 .id = &nf_tables_net_id,
10951 .size = sizeof(struct nftables_pernet),11131 .size = sizeof(struct nftables_pernet),
10952};11132};
@@ -11018,6 +11198,7 @@ static void __exit nf_tables_module_exit(void) · 65 unmodified lines
11018 nft_chain_filter_fini();11198 nft_chain_filter_fini();
11019 nft_chain_route_fini();11199 nft_chain_route_fini();
11020 unregister_pernet_subsys(&nf_tables_net_ops);11200 unregister_pernet_subsys(&nf_tables_net_ops);
11201 cancel_work_sync(&trans_gc_work);
11021 cancel_work_sync(&trans_destroy_work);11202 cancel_work_sync(&trans_destroy_work);
11022 rcu_barrier();11203 rcu_barrier();
11023 rhltable_destroy(&nft_objname_ht);11204 rhltable_destroy(&nft_objname_ht);
net/netfilter/nft_set_hash.c+52−33
@@ -59,6 +59,8 @@ static inline int nft_rhash_cmp(struct rhashtable_compare_arg *arg, · 58 unmodified lines
59 59
60 if (memcmp(nft_set_ext_key(&he->ext), x->key, x->set->klen))60 if (memcmp(nft_set_ext_key(&he->ext), x->key, x->set->klen))
61 return 1;61 return 1;
62 if (nft_set_elem_is_dead(&he->ext))
63 return 1;
62 if (nft_set_elem_expired(&he->ext))64 if (nft_set_elem_expired(&he->ext))
63 return 1;65 return 1;
64 if (!nft_set_elem_active(&he->ext, x->genmask))66 if (!nft_set_elem_active(&he->ext, x->genmask))
@@ -188,7 +190,6 @@ static void nft_rhash_activate(const struct net *net, const struct nft_set *set, · 123 unmodified lines
188 struct nft_rhash_elem *he = elem->priv;190 struct nft_rhash_elem *he = elem->priv;
189 191
190 nft_set_elem_change_active(net, set, &he->ext);192 nft_set_elem_change_active(net, set, &he->ext);
191 nft_set_elem_clear_busy(&he->ext);
192}193}
193 194
194static bool nft_rhash_flush(const struct net *net,195static bool nft_rhash_flush(const struct net *net,
@@ -196,12 +197,9 @@ static bool nft_rhash_flush(const struct net *net, · 1 unmodified line
196{197{
197 struct nft_rhash_elem *he = priv;198 struct nft_rhash_elem *he = priv;
198 199
199 if (!nft_set_elem_mark_busy(&he->ext) ||200 nft_set_elem_change_active(net, set, &he->ext);
200 !nft_is_active(net, &he->ext)) {201
201 nft_set_elem_change_active(net, set, &he->ext);202 return true;
202 return true;
203 }
204 return false;
205}203}
206 204
207static void *nft_rhash_deactivate(const struct net *net,205static void *nft_rhash_deactivate(const struct net *net,
@@ -218,9 +216,8 @@ static void *nft_rhash_deactivate(const struct net *net, · 10 unmodified lines
218 216
219 rcu_read_lock();217 rcu_read_lock();
220 he = rhashtable_lookup(&priv->ht, &arg, nft_rhash_params);218 he = rhashtable_lookup(&priv->ht, &arg, nft_rhash_params);
221 if (he != NULL &&219 if (he)
222 !nft_rhash_flush(net, set, he))220 nft_set_elem_change_active(net, set, &he->ext);
223 he = NULL;
224 221
225 rcu_read_unlock();222 rcu_read_unlock();
226 223
@@ -252,7 +249,9 @@ static bool nft_rhash_delete(const struct nft_set *set, · 25 unmodified lines
252 if (he == NULL)249 if (he == NULL)
253 return false;250 return false;
254 251
255 return rhashtable_remove_fast(&priv->ht, &he->node, nft_rhash_params) == 0;252 nft_set_elem_dead(&he->ext);
253
254 return true;
256}255}
257 256
258static void nft_rhash_walk(const struct nft_ctx *ctx, struct nft_set *set,257static void nft_rhash_walk(const struct nft_ctx *ctx, struct nft_set *set,
@@ -278,8 +277,6 @@ static void nft_rhash_walk(const struct nft_ctx *ctx, struct nft_set *set, · 19 unmodified lines
278 277
279 if (iter->count < iter->skip)278 if (iter->count < iter->skip)
280 goto cont;279 goto cont;
281 if (nft_set_elem_expired(&he->ext))
282 goto cont;
283 if (!nft_set_elem_active(&he->ext, iter->genmask))280 if (!nft_set_elem_active(&he->ext, iter->genmask))
284 goto cont;281 goto cont;
285 282
@@ -314,25 +311,48 @@ static bool nft_rhash_expr_needs_gc_run(const struct nft_set *set, · 28 unmodified lines
314 311
315static void nft_rhash_gc(struct work_struct *work)312static void nft_rhash_gc(struct work_struct *work)
316{313{
314 struct nftables_pernet *nft_net;
317 struct nft_set *set;315 struct nft_set *set;
318 struct nft_rhash_elem *he;316 struct nft_rhash_elem *he;
319 struct nft_rhash *priv;317 struct nft_rhash *priv;
320 struct nft_set_gc_batch *gcb = NULL;
321 struct rhashtable_iter hti;318 struct rhashtable_iter hti;
319 struct nft_trans_gc *gc;
320 struct net *net;
321 u32 gc_seq;
322 322
323 priv = container_of(work, struct nft_rhash, gc_work.work);323 priv = container_of(work, struct nft_rhash, gc_work.work);
324 set = nft_set_container_of(priv);324 set = nft_set_container_of(priv);
325 net = read_pnet(&set->net);
326 nft_net = nft_pernet(net);
327 gc_seq = READ_ONCE(nft_net->gc_seq);
328
329 gc = nft_trans_gc_alloc(set, gc_seq, GFP_KERNEL);
330 if (!gc)
331 goto done;
325 332
326 rhashtable_walk_enter(&priv->ht, &hti);333 rhashtable_walk_enter(&priv->ht, &hti);
327 rhashtable_walk_start(&hti);334 rhashtable_walk_start(&hti);
328 335
329 while ((he = rhashtable_walk_next(&hti))) {336 while ((he = rhashtable_walk_next(&hti))) {
330 if (IS_ERR(he)) {337 if (IS_ERR(he)) {
331 if (PTR_ERR(he) != -EAGAIN)338 if (PTR_ERR(he) != -EAGAIN) {
332 break;339 nft_trans_gc_destroy(gc);
340 gc = NULL;
341 goto try_later;
342 }
333 continue;343 continue;
334 }344 }
335 345
346 /* Ruleset has been updated, try later. */
347 if (READ_ONCE(nft_net->gc_seq) != gc_seq) {
348 nft_trans_gc_destroy(gc);
349 gc = NULL;
350 goto try_later;
351 }
352
353 if (nft_set_elem_is_dead(&he->ext))
354 goto dead_elem;
355
336 if (nft_set_ext_exists(&he->ext, NFT_SET_EXT_EXPRESSIONS) &&356 if (nft_set_ext_exists(&he->ext, NFT_SET_EXT_EXPRESSIONS) &&
337 nft_rhash_expr_needs_gc_run(set, &he->ext))357 nft_rhash_expr_needs_gc_run(set, &he->ext))
338 goto needs_gc_run;358 goto needs_gc_run;
@@ -340,26 +360,26 @@ static void nft_rhash_gc(struct work_struct *work) · 1 unmodified line
340 if (!nft_set_elem_expired(&he->ext))360 if (!nft_set_elem_expired(&he->ext))
341 continue;361 continue;
342needs_gc_run:362needs_gc_run:
343 if (nft_set_elem_mark_busy(&he->ext))363 nft_set_elem_dead(&he->ext);
344 continue;364dead_elem:
365 gc = nft_trans_gc_queue_async(gc, gc_seq, GFP_ATOMIC);
366 if (!gc)
367 goto try_later;
345 368
346 gcb = nft_set_gc_batch_check(set, gcb, GFP_ATOMIC);369 nft_trans_gc_elem_add(gc, he);
347 if (gcb == NULL)
348 break;
349 rhashtable_remove_fast(&priv->ht, &he->node, nft_rhash_params);
350 atomic_dec(&set->nelems);
351 nft_set_gc_batch_add(gcb, he);
352 }370 }
371
372 gc = nft_trans_gc_catchall(gc, gc_seq);
373
374try_later:
375 /* catchall list iteration requires rcu read side lock. */
353 rhashtable_walk_stop(&hti);376 rhashtable_walk_stop(&hti);
354 rhashtable_walk_exit(&hti);377 rhashtable_walk_exit(&hti);
355 378
356 he = nft_set_catchall_gc(set);379 if (gc)
357 if (he) {380 nft_trans_gc_queue_async_done(gc);
358 gcb = nft_set_gc_batch_check(set, gcb, GFP_ATOMIC);381
359 if (gcb)382done:
360 nft_set_gc_batch_add(gcb, he);
361 }
362 nft_set_gc_batch_complete(gcb);
363 queue_delayed_work(system_power_efficient_wq, &priv->gc_work,383 queue_delayed_work(system_power_efficient_wq, &priv->gc_work,
364 nft_set_gc_interval(set));384 nft_set_gc_interval(set));
365}385}
@@ -394,7 +414,7 @@ static int nft_rhash_init(const struct nft_set *set, · 28 unmodified lines
394 return err;414 return err;
395 415
396 INIT_DEFERRABLE_WORK(&priv->gc_work, nft_rhash_gc);416 INIT_DEFERRABLE_WORK(&priv->gc_work, nft_rhash_gc);
397 if (set->flags & NFT_SET_TIMEOUT)417 if (set->flags & (NFT_SET_TIMEOUT | NFT_SET_EVAL))
398 nft_rhash_gc_init(set);418 nft_rhash_gc_init(set);
399 419
400 return 0;420 return 0;
@@ -422,7 +442,6 @@ static void nft_rhash_destroy(const struct nft_ctx *ctx, · 21 unmodified lines
422 };442 };
423 443
424 cancel_delayed_work_sync(&priv->gc_work);444 cancel_delayed_work_sync(&priv->gc_work);
425 rcu_barrier();
426 rhashtable_free_and_destroy(&priv->ht, nft_rhash_elem_destroy,445 rhashtable_free_and_destroy(&priv->ht, nft_rhash_elem_destroy,
427 (void *)&rhash_ctx);446 (void *)&rhash_ctx);
428}447}
net/netfilter/nft_set_pipapo.c+48−18
@@ -566,8 +566,7 @@ next_match: · 565 unmodified lines
566 goto out;566 goto out;
567 567
568 if (last) {568 if (last) {
569 if (nft_set_elem_expired(&f->mt[b].e->ext) ||569 if ((genmask &&
570 (genmask &&
571 !nft_set_elem_active(&f->mt[b].e->ext, genmask)))570 !nft_set_elem_active(&f->mt[b].e->ext, genmask)))
572 goto next_match;571 goto next_match;
573 572
@@ -601,8 +600,17 @@ out: · 27 unmodified lines
601static void *nft_pipapo_get(const struct net *net, const struct nft_set *set,600static void *nft_pipapo_get(const struct net *net, const struct nft_set *set,
602 const struct nft_set_elem *elem, unsigned int flags)601 const struct nft_set_elem *elem, unsigned int flags)
603{602{
604 return pipapo_get(net, set, (const u8 *)elem->key.val.data,603 struct nft_pipapo_elem *ret;
605 nft_genmask_cur(net));604
605 ret = pipapo_get(net, set, (const u8 *)elem->key.val.data,
606 nft_genmask_cur(net));
607 if (IS_ERR(ret))
608 return ret;
609
610 if (nft_set_elem_expired(&ret->ext))
611 return ERR_PTR(-ENOENT);
612
613 return ret;
606}614}
607 615
608/**616/**
@@ -1528,16 +1536,34 @@ static void pipapo_drop(struct nft_pipapo_match *m, · 919 unmodified lines
1528 }1536 }
1529}1537}
1530 1538
1539static void nft_pipapo_gc_deactivate(struct net *net, struct nft_set *set,
1540 struct nft_pipapo_elem *e)
1541
1542{
1543 struct nft_set_elem elem = {
1544 .priv = e,
1545 };
1546
1547 nft_setelem_data_deactivate(net, set, &elem);
1548}
1549
1531/**1550/**
1532 * pipapo_gc() - Drop expired entries from set, destroy start and end elements1551 * pipapo_gc() - Drop expired entries from set, destroy start and end elements
1533 * @set: nftables API set representation1552 * @set: nftables API set representation
1534 * @m: Matching data1553 * @m: Matching data
1535 */1554 */
1536static void pipapo_gc(const struct nft_set *set, struct nft_pipapo_match *m)1555static void pipapo_gc(const struct nft_set *_set, struct nft_pipapo_match *m)
1537{1556{
1557 struct nft_set *set = (struct nft_set *) _set;
1538 struct nft_pipapo *priv = nft_set_priv(set);1558 struct nft_pipapo *priv = nft_set_priv(set);
1559 struct net *net = read_pnet(&set->net);
1539 int rules_f0, first_rule = 0;1560 int rules_f0, first_rule = 0;
1540 struct nft_pipapo_elem *e;1561 struct nft_pipapo_elem *e;
1562 struct nft_trans_gc *gc;
1563
1564 gc = nft_trans_gc_alloc(set, 0, GFP_KERNEL);
1565 if (!gc)
1566 return;
1541 1567
1542 while ((rules_f0 = pipapo_rules_same_key(m->f, first_rule))) {1568 while ((rules_f0 = pipapo_rules_same_key(m->f, first_rule))) {
1543 union nft_pipapo_map_bucket rulemap[NFT_PIPAPO_MAX_FIELDS];1569 union nft_pipapo_map_bucket rulemap[NFT_PIPAPO_MAX_FIELDS];
@@ -1561,13 +1587,20 @@ static void pipapo_gc(const struct nft_set *set, struct nft_pipapo_match *m) · 17 unmodified lines
1561 f--;1587 f--;
1562 i--;1588 i--;
1563 e = f->mt[rulemap[i].to].e;1589 e = f->mt[rulemap[i].to].e;
1564 if (nft_set_elem_expired(&e->ext) &&1590
1565 !nft_set_elem_mark_busy(&e->ext)) {1591 /* synchronous gc never fails, there is no need to set on
1592 * NFT_SET_ELEM_DEAD_BIT.
1593 */
1594 if (nft_set_elem_expired(&e->ext)) {
1566 priv->dirty = true;1595 priv->dirty = true;
1567 pipapo_drop(m, rulemap);
1568 1596
1569 rcu_barrier();1597 gc = nft_trans_gc_queue_sync(gc, GFP_ATOMIC);
1570 nft_set_elem_destroy(set, e, true);1598 if (!gc)
1599 break;
1600
1601 nft_pipapo_gc_deactivate(net, set, e);
1602 pipapo_drop(m, rulemap);
1603 nft_trans_gc_elem_add(gc, e);
1571 1604
1572 /* And check again current first rule, which is now the1605 /* And check again current first rule, which is now the
1573 * first we haven't checked.1606 * first we haven't checked.
@@ -1577,11 +1610,11 @@ static void pipapo_gc(const struct nft_set *set, struct nft_pipapo_match *m) · 3 unmodified lines
1577 }1610 }
1578 }1611 }
1579 1612
1580 e = nft_set_catchall_gc(set);1613 gc = nft_trans_gc_catchall(gc, 0);
1581 if (e)1614 if (gc) {
1582 nft_set_elem_destroy(set, e, true);1615 nft_trans_gc_queue_sync_done(gc);
1583 1616 priv->last_gc = jiffies;
1584 priv->last_gc = jiffies;1617 }
1585}1618}
1586 1619
1587/**1620/**
@@ -1706,7 +1739,6 @@ static void nft_pipapo_activate(const struct net *net, · 118 unmodified lines
1706 return;1739 return;
1707 1740
1708 nft_set_elem_change_active(net, set, &e->ext);1741 nft_set_elem_change_active(net, set, &e->ext);
1709 nft_set_elem_clear_busy(&e->ext);
1710}1742}
1711 1743
1712/**1744/**
@@ -2005,8 +2037,6 @@ static void nft_pipapo_walk(const struct nft_ctx *ctx, struct nft_set *set, · 292 unmodified lines
2005 goto cont;2037 goto cont;
2006 2038
2007 e = f->mt[r].e;2039 e = f->mt[r].e;
2008 if (nft_set_elem_expired(&e->ext))
2009 goto cont;
2010 2040
2011 elem.priv = e;2041 elem.priv = e;
2012 2042
net/netfilter/nft_set_rbtree.c+87−59
@@ -46,6 +46,12 @@ static int nft_rbtree_cmp(const struct nft_set *set, · 45 unmodified lines
46 set->klen);46 set->klen);
47}47}
48 48
49static bool nft_rbtree_elem_expired(const struct nft_rbtree_elem *rbe)
50{
51 return nft_set_elem_expired(&rbe->ext) ||
52 nft_set_elem_is_dead(&rbe->ext);
53}
54
49static bool __nft_rbtree_lookup(const struct net *net, const struct nft_set *set,55static bool __nft_rbtree_lookup(const struct net *net, const struct nft_set *set,
50 const u32 *key, const struct nft_set_ext **ext,56 const u32 *key, const struct nft_set_ext **ext,
51 unsigned int seq)57 unsigned int seq)
@@ -80,7 +86,7 @@ static bool __nft_rbtree_lookup(const struct net *net, const struct nft_set *set · 28 unmodified lines
80 continue;86 continue;
81 }87 }
82 88
83 if (nft_set_elem_expired(&rbe->ext))89 if (nft_rbtree_elem_expired(rbe))
84 return false;90 return false;
85 91
86 if (nft_rbtree_interval_end(rbe)) {92 if (nft_rbtree_interval_end(rbe)) {
@@ -98,7 +104,7 @@ static bool __nft_rbtree_lookup(const struct net *net, const struct nft_set *set · 11 unmodified lines
98 104
99 if (set->flags & NFT_SET_INTERVAL && interval != NULL &&105 if (set->flags & NFT_SET_INTERVAL && interval != NULL &&
100 nft_set_elem_active(&interval->ext, genmask) &&106 nft_set_elem_active(&interval->ext, genmask) &&
101 !nft_set_elem_expired(&interval->ext) &&107 !nft_rbtree_elem_expired(interval) &&
102 nft_rbtree_interval_start(interval)) {108 nft_rbtree_interval_start(interval)) {
103 *ext = &interval->ext;109 *ext = &interval->ext;
104 return true;110 return true;
@@ -215,6 +221,18 @@ static void *nft_rbtree_get(const struct net *net, const struct nft_set *set, · 110 unmodified lines
215 return rbe;221 return rbe;
216}222}
217 223
224static void nft_rbtree_gc_remove(struct net *net, struct nft_set *set,
225 struct nft_rbtree *priv,
226 struct nft_rbtree_elem *rbe)
227{
228 struct nft_set_elem elem = {
229 .priv = rbe,
230 };
231
232 nft_setelem_data_deactivate(net, set, &elem);
233 rb_erase(&rbe->node, &priv->root);
234}
235
218static int nft_rbtree_gc_elem(const struct nft_set *__set,236static int nft_rbtree_gc_elem(const struct nft_set *__set,
219 struct nft_rbtree *priv,237 struct nft_rbtree *priv,
220 struct nft_rbtree_elem *rbe,238 struct nft_rbtree_elem *rbe,
@@ -222,11 +240,12 @@ static int nft_rbtree_gc_elem(const struct nft_set *__set, · 1 unmodified line
222{240{
223 struct nft_set *set = (struct nft_set *)__set;241 struct nft_set *set = (struct nft_set *)__set;
224 struct rb_node *prev = rb_prev(&rbe->node);242 struct rb_node *prev = rb_prev(&rbe->node);
243 struct net *net = read_pnet(&set->net);
225 struct nft_rbtree_elem *rbe_prev;244 struct nft_rbtree_elem *rbe_prev;
226 struct nft_set_gc_batch *gcb;245 struct nft_trans_gc *gc;
227 246
228 gcb = nft_set_gc_batch_check(set, NULL, GFP_ATOMIC);247 gc = nft_trans_gc_alloc(set, 0, GFP_ATOMIC);
229 if (!gcb)248 if (!gc)
230 return -ENOMEM;249 return -ENOMEM;
231 250
232 /* search for end interval coming before this element.251 /* search for end interval coming before this element.
@@ -244,17 +263,28 @@ static int nft_rbtree_gc_elem(const struct nft_set *__set, · 11 unmodified lines
244 263
245 if (prev) {264 if (prev) {
246 rbe_prev = rb_entry(prev, struct nft_rbtree_elem, node);265 rbe_prev = rb_entry(prev, struct nft_rbtree_elem, node);
266 nft_rbtree_gc_remove(net, set, priv, rbe_prev);
247 267
248 rb_erase(&rbe_prev->node, &priv->root);268 /* There is always room in this trans gc for this element,
249 atomic_dec(&set->nelems);269 * memory allocation never actually happens, hence, the warning
250 nft_set_gc_batch_add(gcb, rbe_prev);270 * splat in such case. No need to set NFT_SET_ELEM_DEAD_BIT,
271 * this is synchronous gc which never fails.
272 */
273 gc = nft_trans_gc_queue_sync(gc, GFP_ATOMIC);
274 if (WARN_ON_ONCE(!gc))
275 return -ENOMEM;
276
277 nft_trans_gc_elem_add(gc, rbe_prev);
251 }278 }
252 279
253 rb_erase(&rbe->node, &priv->root);280 nft_rbtree_gc_remove(net, set, priv, rbe);
254 atomic_dec(&set->nelems);281 gc = nft_trans_gc_queue_sync(gc, GFP_ATOMIC);
282 if (WARN_ON_ONCE(!gc))
283 return -ENOMEM;
284
285 nft_trans_gc_elem_add(gc, rbe);
255 286
256 nft_set_gc_batch_add(gcb, rbe);287 nft_trans_gc_queue_sync_done(gc);
257 nft_set_gc_batch_complete(gcb);
258 288
259 return 0;289 return 0;
260}290}
@@ -482,7 +512,6 @@ static void nft_rbtree_activate(const struct net *net, · 221 unmodified lines
482 struct nft_rbtree_elem *rbe = elem->priv;512 struct nft_rbtree_elem *rbe = elem->priv;
483 513
484 nft_set_elem_change_active(net, set, &rbe->ext);514 nft_set_elem_change_active(net, set, &rbe->ext);
485 nft_set_elem_clear_busy(&rbe->ext);
486}515}
487 516
488static bool nft_rbtree_flush(const struct net *net,517static bool nft_rbtree_flush(const struct net *net,
@@ -490,12 +519,9 @@ static bool nft_rbtree_flush(const struct net *net, · 1 unmodified line
490{519{
491 struct nft_rbtree_elem *rbe = priv;520 struct nft_rbtree_elem *rbe = priv;
492 521
493 if (!nft_set_elem_mark_busy(&rbe->ext) ||522 nft_set_elem_change_active(net, set, &rbe->ext);
494 !nft_is_active(net, &rbe->ext)) {523
495 nft_set_elem_change_active(net, set, &rbe->ext);524 return true;
496 return true;
497 }
498 return false;
499}525}
500 526
501static void *nft_rbtree_deactivate(const struct net *net,527static void *nft_rbtree_deactivate(const struct net *net,
@@ -552,8 +578,6 @@ static void nft_rbtree_walk(const struct nft_ctx *ctx, · 50 unmodified lines
552 578
553 if (iter->count < iter->skip)579 if (iter->count < iter->skip)
554 goto cont;580 goto cont;
555 if (nft_set_elem_expired(&rbe->ext))
556 goto cont;
557 if (!nft_set_elem_active(&rbe->ext, iter->genmask))581 if (!nft_set_elem_active(&rbe->ext, iter->genmask))
558 goto cont;582 goto cont;
559 583
@@ -572,26 +596,40 @@ cont: · 12 unmodified lines
572 596
573static void nft_rbtree_gc(struct work_struct *work)597static void nft_rbtree_gc(struct work_struct *work)
574{598{
575 struct nft_rbtree_elem *rbe, *rbe_end = NULL, *rbe_prev = NULL;599 struct nft_rbtree_elem *rbe, *rbe_end = NULL;
576 struct nft_set_gc_batch *gcb = NULL;600 struct nftables_pernet *nft_net;
577 struct nft_rbtree *priv;601 struct nft_rbtree *priv;
602 struct nft_trans_gc *gc;
578 struct rb_node *node;603 struct rb_node *node;
579 struct nft_set *set;604 struct nft_set *set;
605 unsigned int gc_seq;
580 struct net *net;606 struct net *net;
581 u8 genmask;
582 607
583 priv = container_of(work, struct nft_rbtree, gc_work.work);608 priv = container_of(work, struct nft_rbtree, gc_work.work);
584 set = nft_set_container_of(priv);609 set = nft_set_container_of(priv);
585 net = read_pnet(&set->net);610 net = read_pnet(&set->net);
586 genmask = nft_genmask_cur(net);611 nft_net = nft_pernet(net);
612 gc_seq = READ_ONCE(nft_net->gc_seq);
613
614 gc = nft_trans_gc_alloc(set, gc_seq, GFP_KERNEL);
615 if (!gc)
616 goto done;
587 617
588 write_lock_bh(&priv->lock);618 write_lock_bh(&priv->lock);
589 write_seqcount_begin(&priv->count);619 write_seqcount_begin(&priv->count);
590 for (node = rb_first(&priv->root); node != NULL; node = rb_next(node)) {620 for (node = rb_first(&priv->root); node != NULL; node = rb_next(node)) {
621
622 /* Ruleset has been updated, try later. */
623 if (READ_ONCE(nft_net->gc_seq) != gc_seq) {
624 nft_trans_gc_destroy(gc);
625 gc = NULL;
626 goto try_later;
627 }
628
591 rbe = rb_entry(node, struct nft_rbtree_elem, node);629 rbe = rb_entry(node, struct nft_rbtree_elem, node);
592 630
593 if (!nft_set_elem_active(&rbe->ext, genmask))631 if (nft_set_elem_is_dead(&rbe->ext))
594 continue;632 goto dead_elem;
595 633
596 /* elements are reversed in the rbtree for historical reasons,634 /* elements are reversed in the rbtree for historical reasons,
597 * from highest to lowest value, that is why end element is635 * from highest to lowest value, that is why end element is
@@ -604,46 +642,36 @@ static void nft_rbtree_gc(struct work_struct *work) · 6 unmodified lines
604 if (!nft_set_elem_expired(&rbe->ext))642 if (!nft_set_elem_expired(&rbe->ext))
605 continue;643 continue;
606 644
607 if (nft_set_elem_mark_busy(&rbe->ext)) {645 nft_set_elem_dead(&rbe->ext);
608 rbe_end = NULL;646
647 if (!rbe_end)
609 continue;648 continue;
610 }
611 649
612 if (rbe_prev) {650 nft_set_elem_dead(&rbe_end->ext);
613 rb_erase(&rbe_prev->node, &priv->root);
614 rbe_prev = NULL;
615 }
616 gcb = nft_set_gc_batch_check(set, gcb, GFP_ATOMIC);
617 if (!gcb)
618 break;
619 651
620 atomic_dec(&set->nelems);652 gc = nft_trans_gc_queue_async(gc, gc_seq, GFP_ATOMIC);
621 nft_set_gc_batch_add(gcb, rbe);653 if (!gc)
622 rbe_prev = rbe;654 goto try_later;
623 655
624 if (rbe_end) {656 nft_trans_gc_elem_add(gc, rbe_end);
625 atomic_dec(&set->nelems);657 rbe_end = NULL;
626 nft_set_gc_batch_add(gcb, rbe_end);658dead_elem:
627 rb_erase(&rbe_end->node, &priv->root);659 gc = nft_trans_gc_queue_async(gc, gc_seq, GFP_ATOMIC);
628 rbe_end = NULL;660 if (!gc)
629 }661 goto try_later;
630 node = rb_next(node);662
631 if (!node)663 nft_trans_gc_elem_add(gc, rbe);
632 break;
633 }664 }
634 if (rbe_prev)665
635 rb_erase(&rbe_prev->node, &priv->root);666 gc = nft_trans_gc_catchall(gc, gc_seq);
667
668try_later:
636 write_seqcount_end(&priv->count);669 write_seqcount_end(&priv->count);
637 write_unlock_bh(&priv->lock);670 write_unlock_bh(&priv->lock);
638 671
639 rbe = nft_set_catchall_gc(set);672 if (gc)
640 if (rbe) {673 nft_trans_gc_queue_async_done(gc);
641 gcb = nft_set_gc_batch_check(set, gcb, GFP_ATOMIC);674done:
642 if (gcb)
643 nft_set_gc_batch_add(gcb, rbe);
644 }
645 nft_set_gc_batch_complete(gcb);
646
647 queue_delayed_work(system_power_efficient_wq, &priv->gc_work,675 queue_delayed_work(system_power_efficient_wq, &priv->gc_work,
648 nft_set_gc_interval(set));676 nft_set_gc_interval(set));
649}677}

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t