History

Merge tag 'nf-23-08-10' of git://git.kernel.org/pub/scm/linux/kernel/git/netfilter/nf

Pablo Neira Ayuso says: ==================== Netfilter fixes for net The existing attempt to resolve races between control plane and GC work is error prone, as reported by Bien Pham <phamnnb@sea.com>, some places forgot to call nft_set_elem_mark_busy(), leading to double-deactivation of elements. This series contains the following patches: 1) Do not skip expired elements during walk otherwise elements might never decrement the reference counter on data, leading to memleak. 2) Add a GC transaction API to replace the former attempt to deal with races between control plane and GC. GC worker sets on NFT_SET_ELEM_DEAD_BIT on elements and it creates a GC transaction to remove the expired elements, GC transaction could abort in case of interference with control plane and retried later (GC async). Set backends such as rbtree and pipapo also perform GC from control plane (GC sync), in such case, element deactivation and removal is safe because mutex is held then collected elements are released via call_rcu(). 3) Adapt existing set backends to use the GC transaction API. 4) Update rhash set backend to set on _DEAD bit to report deleted elements from datapath for GC. 5) Remove old GC batch API and the NFT_SET_ELEM_BUSY_BIT. * tag 'nf-23-08-10' of git://git.kernel.org/pub/scm/linux/kernel/git/netfilter/nf: netfilter: nf_tables: remove busy mark and gc batch API netfilter: nft_set_hash: mark set element as dead when deleting from packet path netfilter: nf_tables: adapt set backend to use GC transaction API netfilter: nf_tables: GC transaction API to avoid race with control plane netfilter: nf_tables: don't skip expired elements during walk ====================

Jakub Kicinski committed 3 years ago

Commit 3e91b0e2 parents 62d02fc a2dd0235 files · +476 −248Browse files

Unified · Split

5 files
  1. include/net/netfilter/nf_tables.h +45 −75
  2. net/netfilter/nf_tables_api.c +244 −63
  3. net/netfilter/nft_set_hash.c +52 −33
  4. net/netfilter/nft_set_pipapo.c +48 −18
  5. net/netfilter/nft_set_rbtree.c +87 −59
include/net/netfilter/nf_tables.h+45−75
@@ -512,6 +512,7 @@ struct nft_set_elem_expr { · 511 unmodified lines
512512 *
513513 * @list: table set list node
514514 * @bindings: list of set bindings
515+ * @refs: internal refcounting for async set destruction
515516 * @table: table this set belongs to
516517 * @net: netnamespace this set belongs to
517518 * @name: name of the set
@@ -541,6 +542,7 @@ struct nft_set_elem_expr { · 23 unmodified lines
541542 struct nft_set {
542543 struct list_head list;
543544 struct list_head bindings;
545+ refcount_t refs;
544546 struct nft_table *table;
545547 possible_net_t net;
546548 char *name;
@@ -562,7 +564,8 @@ struct nft_set { · 15 unmodified lines
562564 struct list_head pending_update;
563565 /* runtime data below here */
564566 const struct nft_set_ops *ops ____cacheline_aligned;
565− u16 flags:14,
567+ u16 flags:13,
568+ dead:1,
566569 genmask:2;
567570 u8 klen;
568571 u8 dlen;
@@ -596,7 +599,6 @@ struct nft_set *nft_set_lookup_global(const struct net *net, · 27 unmodified lines
596599
597600 struct nft_set_ext *nft_set_catchall_lookup(const struct net *net,
598601 const struct nft_set *set);
599−void *nft_set_catchall_gc(const struct nft_set *set);
600602
601603 static inline unsigned long nft_set_gc_interval(const struct nft_set *set)
602604 {
@@ -813,62 +815,6 @@ void nft_set_elem_destroy(const struct nft_set *set, void *elem, · 210 unmodified lines
813815 void nf_tables_set_elem_destroy(const struct nft_ctx *ctx,
814816 const struct nft_set *set, void *elem);
815817
816−/**
817− * struct nft_set_gc_batch_head - nf_tables set garbage collection batch
818− *
819− * @rcu: rcu head
820− * @set: set the elements belong to
821− * @cnt: count of elements
822− */
823−struct nft_set_gc_batch_head {
824− struct rcu_head rcu;
825− const struct nft_set *set;
826− unsigned int cnt;
827−};
828−
829−#define NFT_SET_GC_BATCH_SIZE ((PAGE_SIZE - \
830− sizeof(struct nft_set_gc_batch_head)) / \
831− sizeof(void *))
832−
833−/**
834− * struct nft_set_gc_batch - nf_tables set garbage collection batch
835− *
836− * @head: GC batch head
837− * @elems: garbage collection elements
838− */
839−struct nft_set_gc_batch {
840− struct nft_set_gc_batch_head head;
841− void *elems[NFT_SET_GC_BATCH_SIZE];
842−};
843−
844−struct nft_set_gc_batch *nft_set_gc_batch_alloc(const struct nft_set *set,
845− gfp_t gfp);
846−void nft_set_gc_batch_release(struct rcu_head *rcu);
847−
848−static inline void nft_set_gc_batch_complete(struct nft_set_gc_batch *gcb)
849−{
850− if (gcb != NULL)
851− call_rcu(&gcb->head.rcu, nft_set_gc_batch_release);
852−}
853−
854−static inline struct nft_set_gc_batch *
855−nft_set_gc_batch_check(const struct nft_set *set, struct nft_set_gc_batch *gcb,
856− gfp_t gfp)
857−{
858− if (gcb != NULL) {
859− if (gcb->head.cnt + 1 < ARRAY_SIZE(gcb->elems))
860− return gcb;
861− nft_set_gc_batch_complete(gcb);
862− }
863− return nft_set_gc_batch_alloc(set, gfp);
864−}
865−
866−static inline void nft_set_gc_batch_add(struct nft_set_gc_batch *gcb,
867− void *elem)
868−{
869− gcb->elems[gcb->head.cnt++] = elem;
870−}
871−
872818 struct nft_expr_ops;
873819 /**
874820 * struct nft_expr_type - nf_tables expression type
@@ -1557,39 +1503,30 @@ static inline void nft_set_elem_change_active(const struct net *net, · 682 unmodified lines
15571503
15581504 #endif /* IS_ENABLED(CONFIG_NF_TABLES) */
15591505
1560−/*
1561− * We use a free bit in the genmask field to indicate the element
1562− * is busy, meaning it is currently being processed either by
1563− * the netlink API or GC.
1564− *
1565− * Even though the genmask is only a single byte wide, this works
1566− * because the extension structure if fully constant once initialized,
1567− * so there are no non-atomic write accesses unless it is already
1568− * marked busy.
1569− */
1570−#define NFT_SET_ELEM_BUSY_MASK (1 << 2)
1506+#define NFT_SET_ELEM_DEAD_MASK (1 << 2)
15711507
15721508 #if defined(__LITTLE_ENDIAN_BITFIELD)
1573−#define NFT_SET_ELEM_BUSY_BIT 2
1509+#define NFT_SET_ELEM_DEAD_BIT 2
15741510 #elif defined(__BIG_ENDIAN_BITFIELD)
1575−#define NFT_SET_ELEM_BUSY_BIT (BITS_PER_LONG - BITS_PER_BYTE + 2)
1511+#define NFT_SET_ELEM_DEAD_BIT (BITS_PER_LONG - BITS_PER_BYTE + 2)
15761512 #else
15771513 #error
15781514 #endif
15791515
1580−static inline int nft_set_elem_mark_busy(struct nft_set_ext *ext)
1516+static inline void nft_set_elem_dead(struct nft_set_ext *ext)
15811517 {
15821518 unsigned long *word = (unsigned long *)ext;
15831519
15841520 BUILD_BUG_ON(offsetof(struct nft_set_ext, genmask) != 0);
1585− return test_and_set_bit(NFT_SET_ELEM_BUSY_BIT, word);
1521+ set_bit(NFT_SET_ELEM_DEAD_BIT, word);
15861522 }
15871523
1588−static inline void nft_set_elem_clear_busy(struct nft_set_ext *ext)
1524+static inline int nft_set_elem_is_dead(const struct nft_set_ext *ext)
15891525 {
15901526 unsigned long *word = (unsigned long *)ext;
15911527
1592− clear_bit(NFT_SET_ELEM_BUSY_BIT, word);
1528+ BUILD_BUG_ON(offsetof(struct nft_set_ext, genmask) != 0);
1529+ return test_bit(NFT_SET_ELEM_DEAD_BIT, word);
15931530 }
15941531
15951532 /**
@@ -1732,6 +1669,38 @@ struct nft_trans_flowtable { · 136 unmodified lines
17321669 #define nft_trans_flowtable_flags(trans) \
17331670 (((struct nft_trans_flowtable *)trans->data)->flags)
17341671
1672+#define NFT_TRANS_GC_BATCHCOUNT 256
1673+
1674+struct nft_trans_gc {
1675+ struct list_head list;
1676+ struct net *net;
1677+ struct nft_set *set;
1678+ u32 seq;
1679+ u8 count;
1680+ void *priv[NFT_TRANS_GC_BATCHCOUNT];
1681+ struct rcu_head rcu;
1682+};
1683+
1684+struct nft_trans_gc *nft_trans_gc_alloc(struct nft_set *set,
1685+ unsigned int gc_seq, gfp_t gfp);
1686+void nft_trans_gc_destroy(struct nft_trans_gc *trans);
1687+
1688+struct nft_trans_gc *nft_trans_gc_queue_async(struct nft_trans_gc *gc,
1689+ unsigned int gc_seq, gfp_t gfp);
1690+void nft_trans_gc_queue_async_done(struct nft_trans_gc *gc);
1691+
1692+struct nft_trans_gc *nft_trans_gc_queue_sync(struct nft_trans_gc *gc, gfp_t gfp);
1693+void nft_trans_gc_queue_sync_done(struct nft_trans_gc *trans);
1694+
1695+void nft_trans_gc_elem_add(struct nft_trans_gc *gc, void *priv);
1696+
1697+struct nft_trans_gc *nft_trans_gc_catchall(struct nft_trans_gc *gc,
1698+ unsigned int gc_seq);
1699+
1700+void nft_setelem_data_deactivate(const struct net *net,
1701+ const struct nft_set *set,
1702+ struct nft_set_elem *elem);
1703+
17351704 int __init nft_chain_filter_init(void);
17361705 void nft_chain_filter_fini(void);
17371706
@@ -1758,6 +1727,7 @@ struct nftables_pernet { · 20 unmodified lines
17581727 struct mutex commit_mutex;
17591728 u64 table_handle;
17601729 unsigned int base_seq;
1730+ unsigned int gc_seq;
17611731 };
17621732
17631733 extern unsigned int nf_tables_net_id;
net/netfilter/nf_tables_api.c+244−63
@@ -31,7 +31,9 @@ static LIST_HEAD(nf_tables_expressions); · 30 unmodified lines
3131 static LIST_HEAD(nf_tables_objects);
3232 static LIST_HEAD(nf_tables_flowtables);
3333 static LIST_HEAD(nf_tables_destroy_list);
34+static LIST_HEAD(nf_tables_gc_list);
3435 static DEFINE_SPINLOCK(nf_tables_destroy_list_lock);
36+static DEFINE_SPINLOCK(nf_tables_gc_list_lock);
3537
3638 enum {
3739 NFT_VALIDATE_SKIP = 0,
@@ -120,6 +122,9 @@ static void nft_validate_state_update(struct nft_table *table, u8 new_validate_s · 82 unmodified lines
120122 static void nf_tables_trans_destroy_work(struct work_struct *w);
121123 static DECLARE_WORK(trans_destroy_work, nf_tables_trans_destroy_work);
122124
125+static void nft_trans_gc_work(struct work_struct *work);
126+static DECLARE_WORK(trans_gc_work, nft_trans_gc_work);
127+
123128 static void nft_ctx_init(struct nft_ctx *ctx,
124129 struct net *net,
125130 const struct sk_buff *skb,
@@ -582,10 +587,6 @@ static int nft_trans_set_add(const struct nft_ctx *ctx, int msg_type, · 456 unmodified lines
582587 return __nft_trans_set_add(ctx, msg_type, set, NULL);
583588 }
584589
585−static void nft_setelem_data_deactivate(const struct net *net,
586− const struct nft_set *set,
587− struct nft_set_elem *elem);
588−
589590 static int nft_mapelem_deactivate(const struct nft_ctx *ctx,
590591 struct nft_set *set,
591592 const struct nft_set_iter *iter,
@@ -5055,6 +5056,7 @@ static int nf_tables_newset(struct sk_buff *skb, const struct nfnl_info *info, · 4463 unmodified lines
50555056
50565057 INIT_LIST_HEAD(&set->bindings);
50575058 INIT_LIST_HEAD(&set->catchall_list);
5059+ refcount_set(&set->refs, 1);
50585060 set->table = table;
50595061 write_pnet(&set->net, net);
50605062 set->ops = ops;
@@ -5122,6 +5124,14 @@ static void nft_set_catchall_destroy(const struct nft_ctx *ctx, · 61 unmodified lines
51225124 }
51235125 }
51245126
5127+static void nft_set_put(struct nft_set *set)
5128+{
5129+ if (refcount_dec_and_test(&set->refs)) {
5130+ kfree(set->name);
5131+ kvfree(set);
5132+ }
5133+}
5134+
51255135 static void nft_set_destroy(const struct nft_ctx *ctx, struct nft_set *set)
51265136 {
51275137 int i;
@@ -5134,8 +5144,7 @@ static void nft_set_destroy(const struct nft_ctx *ctx, struct nft_set *set) · 6 unmodified lines
51345144
51355145 set->ops->destroy(ctx, set);
51365146 nft_set_catchall_destroy(ctx, set);
5137− kfree(set->name);
5138− kvfree(set);
5147+ nft_set_put(set);
51395148 }
51405149
51415150 static int nf_tables_delset(struct sk_buff *skb, const struct nfnl_info *info,
@@ -5602,8 +5611,12 @@ static int nf_tables_dump_setelem(const struct nft_ctx *ctx, · 460 unmodified lines
56025611 const struct nft_set_iter *iter,
56035612 struct nft_set_elem *elem)
56045613 {
5614+ const struct nft_set_ext *ext = nft_set_elem_ext(set, elem->priv);
56055615 struct nft_set_dump_args *args;
56065616
5617+ if (nft_set_elem_expired(ext))
5618+ return 0;
5619+
56075620 args = container_of(iter, struct nft_set_dump_args, iter);
56085621 return nf_tables_fill_setelem(args->skb, set, elem, args->reset);
56095622 }
@@ -6274,7 +6287,8 @@ struct nft_set_ext *nft_set_catchall_lookup(const struct net *net, · 664 unmodified lines
62746287 list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
62756288 ext = nft_set_elem_ext(set, catchall->elem);
62766289 if (nft_set_elem_active(ext, genmask) &&
6277− !nft_set_elem_expired(ext))
6290+ !nft_set_elem_expired(ext) &&
6291+ !nft_set_elem_is_dead(ext))
62786292 return ext;
62796293 }
62806294
@@ -6282,29 +6296,6 @@ struct nft_set_ext *nft_set_catchall_lookup(const struct net *net, · 1 unmodified line
62826296 }
62836297 EXPORT_SYMBOL_GPL(nft_set_catchall_lookup);
62846298
6285−void *nft_set_catchall_gc(const struct nft_set *set)
6286−{
6287− struct nft_set_elem_catchall *catchall, *next;
6288− struct nft_set_ext *ext;
6289− void *elem = NULL;
6290−
6291− list_for_each_entry_safe(catchall, next, &set->catchall_list, list) {
6292− ext = nft_set_elem_ext(set, catchall->elem);
6293−
6294− if (!nft_set_elem_expired(ext) ||
6295− nft_set_elem_mark_busy(ext))
6296− continue;
6297−
6298− elem = catchall->elem;
6299− list_del_rcu(&catchall->list);
6300− kfree_rcu(catchall, rcu);
6301− break;
6302− }
6303−
6304− return elem;
6305−}
6306−EXPORT_SYMBOL_GPL(nft_set_catchall_gc);
6307−
63086299 static int nft_setelem_catchall_insert(const struct net *net,
63096300 struct nft_set *set,
63106301 const struct nft_set_elem *elem,
@@ -6366,7 +6357,6 @@ static void nft_setelem_activate(struct net *net, struct nft_set *set, · 55 unmodified lines
63666357
63676358 if (nft_setelem_is_catchall(set, elem)) {
63686359 nft_set_elem_change_active(net, set, ext);
6369− nft_set_elem_clear_busy(ext);
63706360 } else {
63716361 set->ops->activate(net, set, elem);
63726362 }
@@ -6381,8 +6371,7 @@ static int nft_setelem_catchall_deactivate(const struct net *net, · 8 unmodified lines
63816371
63826372 list_for_each_entry(catchall, &set->catchall_list, list) {
63836373 ext = nft_set_elem_ext(set, catchall->elem);
6384− if (!nft_is_active(net, ext) ||
6385− nft_set_elem_mark_busy(ext))
6374+ if (!nft_is_active(net, ext))
63866375 continue;
63876376
63886377 kfree(elem->priv);
@@ -6777,7 +6766,7 @@ static int nft_add_set_elem(struct nft_ctx *ctx, struct nft_set *set, · 388 unmodified lines
67776766 goto err_elem_free;
67786767 }
67796768
6780− ext->genmask = nft_genmask_cur(ctx->net) | NFT_SET_ELEM_BUSY_MASK;
6769+ ext->genmask = nft_genmask_cur(ctx->net);
67816770
67826771 err = nft_setelem_insert(ctx->net, set, &elem, &ext2, flags);
67836772 if (err) {
@@ -6929,9 +6918,9 @@ static void nft_setelem_data_activate(const struct net *net, · 145 unmodified lines
69296918 nft_use_inc_restore(&(*nft_set_ext_obj(ext))->use);
69306919 }
69316920
6932−static void nft_setelem_data_deactivate(const struct net *net,
6933− const struct nft_set *set,
6934− struct nft_set_elem *elem)
6921+void nft_setelem_data_deactivate(const struct net *net,
6922+ const struct nft_set *set,
6923+ struct nft_set_elem *elem)
69356924 {
69366925 const struct nft_set_ext *ext = nft_set_elem_ext(set, elem->priv);
69376926
@@ -7095,8 +7084,7 @@ static int nft_set_catchall_flush(const struct nft_ctx *ctx, · 157 unmodified lines
70957084
70967085 list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
70977086 ext = nft_set_elem_ext(set, catchall->elem);
7098− if (!nft_set_elem_active(ext, genmask) ||
7099− nft_set_elem_mark_busy(ext))
7087+ if (!nft_set_elem_active(ext, genmask))
71007088 continue;
71017089
71027090 elem.priv = catchall->elem;
@@ -7170,29 +7158,6 @@ static int nf_tables_delsetelem(struct sk_buff *skb, · 67 unmodified lines
71707158 return err;
71717159 }
71727160
7173−void nft_set_gc_batch_release(struct rcu_head *rcu)
7174−{
7175− struct nft_set_gc_batch *gcb;
7176− unsigned int i;
7177−
7178− gcb = container_of(rcu, struct nft_set_gc_batch, head.rcu);
7179− for (i = 0; i < gcb->head.cnt; i++)
7180− nft_set_elem_destroy(gcb->head.set, gcb->elems[i], true);
7181− kfree(gcb);
7182−}
7183−
7184−struct nft_set_gc_batch *nft_set_gc_batch_alloc(const struct nft_set *set,
7185− gfp_t gfp)
7186−{
7187− struct nft_set_gc_batch *gcb;
7188−
7189− gcb = kzalloc(sizeof(*gcb), gfp);
7190− if (gcb == NULL)
7191− return gcb;
7192− gcb->head.set = set;
7193− return gcb;
7194−}
7195−
71967161 /*
71977162 * Stateful objects
71987163 */
@@ -9414,6 +9379,207 @@ void nft_chain_del(struct nft_chain *chain) · 2215 unmodified lines
94149379 list_del_rcu(&chain->list);
94159380 }
94169381
9382+static void nft_trans_gc_setelem_remove(struct nft_ctx *ctx,
9383+ struct nft_trans_gc *trans)
9384+{
9385+ void **priv = trans->priv;
9386+ unsigned int i;
9387+
9388+ for (i = 0; i < trans->count; i++) {
9389+ struct nft_set_elem elem = {
9390+ .priv = priv[i],
9391+ };
9392+
9393+ nft_setelem_data_deactivate(ctx->net, trans->set, &elem);
9394+ nft_setelem_remove(ctx->net, trans->set, &elem);
9395+ }
9396+}
9397+
9398+void nft_trans_gc_destroy(struct nft_trans_gc *trans)
9399+{
9400+ nft_set_put(trans->set);
9401+ put_net(trans->net);
9402+ kfree(trans);
9403+}
9404+
9405+static void nft_trans_gc_trans_free(struct rcu_head *rcu)
9406+{
9407+ struct nft_set_elem elem = {};
9408+ struct nft_trans_gc *trans;
9409+ struct nft_ctx ctx = {};
9410+ unsigned int i;
9411+
9412+ trans = container_of(rcu, struct nft_trans_gc, rcu);
9413+ ctx.net = read_pnet(&trans->set->net);
9414+
9415+ for (i = 0; i < trans->count; i++) {
9416+ elem.priv = trans->priv[i];
9417+ if (!nft_setelem_is_catchall(trans->set, &elem))
9418+ atomic_dec(&trans->set->nelems);
9419+
9420+ nf_tables_set_elem_destroy(&ctx, trans->set, elem.priv);
9421+ }
9422+
9423+ nft_trans_gc_destroy(trans);
9424+}
9425+
9426+static bool nft_trans_gc_work_done(struct nft_trans_gc *trans)
9427+{
9428+ struct nftables_pernet *nft_net;
9429+ struct nft_ctx ctx = {};
9430+
9431+ nft_net = nft_pernet(trans->net);
9432+
9433+ mutex_lock(&nft_net->commit_mutex);
9434+
9435+ /* Check for race with transaction, otherwise this batch refers to
9436+ * stale objects that might not be there anymore. Skip transaction if
9437+ * set has been destroyed from control plane transaction in case gc
9438+ * worker loses race.
9439+ */
9440+ if (READ_ONCE(nft_net->gc_seq) != trans->seq || trans->set->dead) {
9441+ mutex_unlock(&nft_net->commit_mutex);
9442+ return false;
9443+ }
9444+
9445+ ctx.net = trans->net;
9446+ ctx.table = trans->set->table;
9447+
9448+ nft_trans_gc_setelem_remove(&ctx, trans);
9449+ mutex_unlock(&nft_net->commit_mutex);
9450+
9451+ return true;
9452+}
9453+
9454+static void nft_trans_gc_work(struct work_struct *work)
9455+{
9456+ struct nft_trans_gc *trans, *next;
9457+ LIST_HEAD(trans_gc_list);
9458+
9459+ spin_lock(&nf_tables_destroy_list_lock);
9460+ list_splice_init(&nf_tables_gc_list, &trans_gc_list);
9461+ spin_unlock(&nf_tables_destroy_list_lock);
9462+
9463+ list_for_each_entry_safe(trans, next, &trans_gc_list, list) {
9464+ list_del(&trans->list);
9465+ if (!nft_trans_gc_work_done(trans)) {
9466+ nft_trans_gc_destroy(trans);
9467+ continue;
9468+ }
9469+ call_rcu(&trans->rcu, nft_trans_gc_trans_free);
9470+ }
9471+}
9472+
9473+struct nft_trans_gc *nft_trans_gc_alloc(struct nft_set *set,
9474+ unsigned int gc_seq, gfp_t gfp)
9475+{
9476+ struct net *net = read_pnet(&set->net);
9477+ struct nft_trans_gc *trans;
9478+
9479+ trans = kzalloc(sizeof(*trans), gfp);
9480+ if (!trans)
9481+ return NULL;
9482+
9483+ refcount_inc(&set->refs);
9484+ trans->set = set;
9485+ trans->net = get_net(net);
9486+ trans->seq = gc_seq;
9487+
9488+ return trans;
9489+}
9490+
9491+void nft_trans_gc_elem_add(struct nft_trans_gc *trans, void *priv)
9492+{
9493+ trans->priv[trans->count++] = priv;
9494+}
9495+
9496+static void nft_trans_gc_queue_work(struct nft_trans_gc *trans)
9497+{
9498+ spin_lock(&nf_tables_gc_list_lock);
9499+ list_add_tail(&trans->list, &nf_tables_gc_list);
9500+ spin_unlock(&nf_tables_gc_list_lock);
9501+
9502+ schedule_work(&trans_gc_work);
9503+}
9504+
9505+static int nft_trans_gc_space(struct nft_trans_gc *trans)
9506+{
9507+ return NFT_TRANS_GC_BATCHCOUNT - trans->count;
9508+}
9509+
9510+struct nft_trans_gc *nft_trans_gc_queue_async(struct nft_trans_gc *gc,
9511+ unsigned int gc_seq, gfp_t gfp)
9512+{
9513+ if (nft_trans_gc_space(gc))
9514+ return gc;
9515+
9516+ nft_trans_gc_queue_work(gc);
9517+
9518+ return nft_trans_gc_alloc(gc->set, gc_seq, gfp);
9519+}
9520+
9521+void nft_trans_gc_queue_async_done(struct nft_trans_gc *trans)
9522+{
9523+ if (trans->count == 0) {
9524+ nft_trans_gc_destroy(trans);
9525+ return;
9526+ }
9527+
9528+ nft_trans_gc_queue_work(trans);
9529+}
9530+
9531+struct nft_trans_gc *nft_trans_gc_queue_sync(struct nft_trans_gc *gc, gfp_t gfp)
9532+{
9533+ if (WARN_ON_ONCE(!lockdep_commit_lock_is_held(gc->net)))
9534+ return NULL;
9535+
9536+ if (nft_trans_gc_space(gc))
9537+ return gc;
9538+
9539+ call_rcu(&gc->rcu, nft_trans_gc_trans_free);
9540+
9541+ return nft_trans_gc_alloc(gc->set, 0, gfp);
9542+}
9543+
9544+void nft_trans_gc_queue_sync_done(struct nft_trans_gc *trans)
9545+{
9546+ WARN_ON_ONCE(!lockdep_commit_lock_is_held(trans->net));
9547+
9548+ if (trans->count == 0) {
9549+ nft_trans_gc_destroy(trans);
9550+ return;
9551+ }
9552+
9553+ call_rcu(&trans->rcu, nft_trans_gc_trans_free);
9554+}
9555+
9556+struct nft_trans_gc *nft_trans_gc_catchall(struct nft_trans_gc *gc,
9557+ unsigned int gc_seq)
9558+{
9559+ struct nft_set_elem_catchall *catchall;
9560+ const struct nft_set *set = gc->set;
9561+ struct nft_set_ext *ext;
9562+
9563+ list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
9564+ ext = nft_set_elem_ext(set, catchall->elem);
9565+
9566+ if (!nft_set_elem_expired(ext))
9567+ continue;
9568+ if (nft_set_elem_is_dead(ext))
9569+ goto dead_elem;
9570+
9571+ nft_set_elem_dead(ext);
9572+dead_elem:
9573+ gc = nft_trans_gc_queue_async(gc, gc_seq, GFP_ATOMIC);
9574+ if (!gc)
9575+ return NULL;
9576+
9577+ nft_trans_gc_elem_add(gc, catchall->elem);
9578+ }
9579+
9580+ return gc;
9581+}
9582+
94179583 static void nf_tables_module_autoload_cleanup(struct net *net)
94189584 {
94199585 struct nftables_pernet *nft_net = nft_pernet(net);
@@ -9576,11 +9742,11 @@ static int nf_tables_commit(struct net *net, struct sk_buff *skb) · 156 unmodified lines
95769742 {
95779743 struct nftables_pernet *nft_net = nft_pernet(net);
95789744 struct nft_trans *trans, *next;
9745+ unsigned int base_seq, gc_seq;
95799746 LIST_HEAD(set_update_list);
95809747 struct nft_trans_elem *te;
95819748 struct nft_chain *chain;
95829749 struct nft_table *table;
9583− unsigned int base_seq;
95849750 LIST_HEAD(adl);
95859751 int err;
95869752
@@ -9657,6 +9823,10 @@ static int nf_tables_commit(struct net *net, struct sk_buff *skb) · 70 unmodified lines
96579823
96589824 WRITE_ONCE(nft_net->base_seq, base_seq);
96599825
9826+ /* Bump gc counter, it becomes odd, this is the busy mark. */
9827+ gc_seq = READ_ONCE(nft_net->gc_seq);
9828+ WRITE_ONCE(nft_net->gc_seq, ++gc_seq);
9829+
96609830 /* step 3. Start new generation, rules_gen_X now in use. */
96619831 net->nft.gencursor = nft_gencursor_next(net);
96629832
@@ -9764,6 +9934,7 @@ static int nf_tables_commit(struct net *net, struct sk_buff *skb) · 101 unmodified lines
97649934 break;
97659935 case NFT_MSG_DELSET:
97669936 case NFT_MSG_DESTROYSET:
9937+ nft_trans_set(trans)->dead = 1;
97679938 list_del_rcu(&nft_trans_set(trans)->list);
97689939 nf_tables_set_notify(&trans->ctx, nft_trans_set(trans),
97699940 trans->msg_type, GFP_KERNEL);
@@ -9866,6 +10037,8 @@ static int nf_tables_commit(struct net *net, struct sk_buff *skb) · 96 unmodified lines
986610037 nft_commit_notify(net, NETLINK_CB(skb).portid);
986710038 nf_tables_gen_notify(net, skb, NFT_MSG_NEWGEN);
986810039 nf_tables_commit_audit_log(&adl, nft_net->base_seq);
10040+
10041+ WRITE_ONCE(nft_net->gc_seq, ++gc_seq);
986910042 nf_tables_commit_release(net);
987010043
987110044 return 0;
@@ -10915,6 +11088,7 @@ static int __net_init nf_tables_init_net(struct net *net) · 1043 unmodified lines
1091511088 INIT_LIST_HEAD(&nft_net->notify_list);
1091611089 mutex_init(&nft_net->commit_mutex);
1091711090 nft_net->base_seq = 1;
11091+ nft_net->gc_seq = 0;
1091811092
1091911093 return 0;
1092011094 }
@@ -10943,10 +11117,16 @@ static void __net_exit nf_tables_exit_net(struct net *net) · 22 unmodified lines
1094311117 WARN_ON_ONCE(!list_empty(&nft_net->notify_list));
1094411118 }
1094511119
11120+static void nf_tables_exit_batch(struct list_head *net_exit_list)
11121+{
11122+ flush_work(&trans_gc_work);
11123+}
11124+
1094611125 static struct pernet_operations nf_tables_net_ops = {
1094711126 .init = nf_tables_init_net,
1094811127 .pre_exit = nf_tables_pre_exit_net,
1094911128 .exit = nf_tables_exit_net,
11129+ .exit_batch = nf_tables_exit_batch,
1095011130 .id = &nf_tables_net_id,
1095111131 .size = sizeof(struct nftables_pernet),
1095211132 };
@@ -11018,6 +11198,7 @@ static void __exit nf_tables_module_exit(void) · 65 unmodified lines
1101811198 nft_chain_filter_fini();
1101911199 nft_chain_route_fini();
1102011200 unregister_pernet_subsys(&nf_tables_net_ops);
11201+ cancel_work_sync(&trans_gc_work);
1102111202 cancel_work_sync(&trans_destroy_work);
1102211203 rcu_barrier();
1102311204 rhltable_destroy(&nft_objname_ht);
net/netfilter/nft_set_hash.c+52−33
@@ -59,6 +59,8 @@ static inline int nft_rhash_cmp(struct rhashtable_compare_arg *arg, · 58 unmodified lines
5959
6060 if (memcmp(nft_set_ext_key(&he->ext), x->key, x->set->klen))
6161 return 1;
62+ if (nft_set_elem_is_dead(&he->ext))
63+ return 1;
6264 if (nft_set_elem_expired(&he->ext))
6365 return 1;
6466 if (!nft_set_elem_active(&he->ext, x->genmask))
@@ -188,7 +190,6 @@ static void nft_rhash_activate(const struct net *net, const struct nft_set *set, · 123 unmodified lines
188190 struct nft_rhash_elem *he = elem->priv;
189191
190192 nft_set_elem_change_active(net, set, &he->ext);
191− nft_set_elem_clear_busy(&he->ext);
192193 }
193194
194195 static bool nft_rhash_flush(const struct net *net,
@@ -196,12 +197,9 @@ static bool nft_rhash_flush(const struct net *net, · 1 unmodified line
196197 {
197198 struct nft_rhash_elem *he = priv;
198199
199− if (!nft_set_elem_mark_busy(&he->ext) ||
200− !nft_is_active(net, &he->ext)) {
201− nft_set_elem_change_active(net, set, &he->ext);
202− return true;
203− }
204− return false;
200+ nft_set_elem_change_active(net, set, &he->ext);
201+
202+ return true;
205203 }
206204
207205 static void *nft_rhash_deactivate(const struct net *net,
@@ -218,9 +216,8 @@ static void *nft_rhash_deactivate(const struct net *net, · 10 unmodified lines
218216
219217 rcu_read_lock();
220218 he = rhashtable_lookup(&priv->ht, &arg, nft_rhash_params);
221− if (he != NULL &&
222− !nft_rhash_flush(net, set, he))
223− he = NULL;
219+ if (he)
220+ nft_set_elem_change_active(net, set, &he->ext);
224221
225222 rcu_read_unlock();
226223
@@ -252,7 +249,9 @@ static bool nft_rhash_delete(const struct nft_set *set, · 25 unmodified lines
252249 if (he == NULL)
253250 return false;
254251
255− return rhashtable_remove_fast(&priv->ht, &he->node, nft_rhash_params) == 0;
252+ nft_set_elem_dead(&he->ext);
253+
254+ return true;
256255 }
257256
258257 static void nft_rhash_walk(const struct nft_ctx *ctx, struct nft_set *set,
@@ -278,8 +277,6 @@ static void nft_rhash_walk(const struct nft_ctx *ctx, struct nft_set *set, · 19 unmodified lines
278277
279278 if (iter->count < iter->skip)
280279 goto cont;
281− if (nft_set_elem_expired(&he->ext))
282− goto cont;
283280 if (!nft_set_elem_active(&he->ext, iter->genmask))
284281 goto cont;
285282
@@ -314,25 +311,48 @@ static bool nft_rhash_expr_needs_gc_run(const struct nft_set *set, · 28 unmodified lines
314311
315312 static void nft_rhash_gc(struct work_struct *work)
316313 {
314+ struct nftables_pernet *nft_net;
317315 struct nft_set *set;
318316 struct nft_rhash_elem *he;
319317 struct nft_rhash *priv;
320− struct nft_set_gc_batch *gcb = NULL;
321318 struct rhashtable_iter hti;
319+ struct nft_trans_gc *gc;
320+ struct net *net;
321+ u32 gc_seq;
322322
323323 priv = container_of(work, struct nft_rhash, gc_work.work);
324324 set = nft_set_container_of(priv);
325+ net = read_pnet(&set->net);
326+ nft_net = nft_pernet(net);
327+ gc_seq = READ_ONCE(nft_net->gc_seq);
328+
329+ gc = nft_trans_gc_alloc(set, gc_seq, GFP_KERNEL);
330+ if (!gc)
331+ goto done;
325332
326333 rhashtable_walk_enter(&priv->ht, &hti);
327334 rhashtable_walk_start(&hti);
328335
329336 while ((he = rhashtable_walk_next(&hti))) {
330337 if (IS_ERR(he)) {
331− if (PTR_ERR(he) != -EAGAIN)
332− break;
338+ if (PTR_ERR(he) != -EAGAIN) {
339+ nft_trans_gc_destroy(gc);
340+ gc = NULL;
341+ goto try_later;
342+ }
333343 continue;
334344 }
335345
346+ /* Ruleset has been updated, try later. */
347+ if (READ_ONCE(nft_net->gc_seq) != gc_seq) {
348+ nft_trans_gc_destroy(gc);
349+ gc = NULL;
350+ goto try_later;
351+ }
352+
353+ if (nft_set_elem_is_dead(&he->ext))
354+ goto dead_elem;
355+
336356 if (nft_set_ext_exists(&he->ext, NFT_SET_EXT_EXPRESSIONS) &&
337357 nft_rhash_expr_needs_gc_run(set, &he->ext))
338358 goto needs_gc_run;
@@ -340,26 +360,26 @@ static void nft_rhash_gc(struct work_struct *work) · 1 unmodified line
340360 if (!nft_set_elem_expired(&he->ext))
341361 continue;
342362 needs_gc_run:
343− if (nft_set_elem_mark_busy(&he->ext))
344− continue;
363+ nft_set_elem_dead(&he->ext);
364+dead_elem:
365+ gc = nft_trans_gc_queue_async(gc, gc_seq, GFP_ATOMIC);
366+ if (!gc)
367+ goto try_later;
345368
346− gcb = nft_set_gc_batch_check(set, gcb, GFP_ATOMIC);
347− if (gcb == NULL)
348− break;
349− rhashtable_remove_fast(&priv->ht, &he->node, nft_rhash_params);
350− atomic_dec(&set->nelems);
351− nft_set_gc_batch_add(gcb, he);
369+ nft_trans_gc_elem_add(gc, he);
352370 }
371+
372+ gc = nft_trans_gc_catchall(gc, gc_seq);
373+
374+try_later:
375+ /* catchall list iteration requires rcu read side lock. */
353376 rhashtable_walk_stop(&hti);
354377 rhashtable_walk_exit(&hti);
355378
356− he = nft_set_catchall_gc(set);
357− if (he) {
358− gcb = nft_set_gc_batch_check(set, gcb, GFP_ATOMIC);
359− if (gcb)
360− nft_set_gc_batch_add(gcb, he);
361− }
362− nft_set_gc_batch_complete(gcb);
379+ if (gc)
380+ nft_trans_gc_queue_async_done(gc);
381+
382+done:
363383 queue_delayed_work(system_power_efficient_wq, &priv->gc_work,
364384 nft_set_gc_interval(set));
365385 }
@@ -394,7 +414,7 @@ static int nft_rhash_init(const struct nft_set *set, · 28 unmodified lines
394414 return err;
395415
396416 INIT_DEFERRABLE_WORK(&priv->gc_work, nft_rhash_gc);
397− if (set->flags & NFT_SET_TIMEOUT)
417+ if (set->flags & (NFT_SET_TIMEOUT | NFT_SET_EVAL))
398418 nft_rhash_gc_init(set);
399419
400420 return 0;
@@ -422,7 +442,6 @@ static void nft_rhash_destroy(const struct nft_ctx *ctx, · 21 unmodified lines
422442 };
423443
424444 cancel_delayed_work_sync(&priv->gc_work);
425− rcu_barrier();
426445 rhashtable_free_and_destroy(&priv->ht, nft_rhash_elem_destroy,
427446 (void *)&rhash_ctx);
428447 }
net/netfilter/nft_set_pipapo.c+48−18
@@ -566,8 +566,7 @@ next_match: · 565 unmodified lines
566566 goto out;
567567
568568 if (last) {
569− if (nft_set_elem_expired(&f->mt[b].e->ext) ||
570− (genmask &&
569+ if ((genmask &&
571570 !nft_set_elem_active(&f->mt[b].e->ext, genmask)))
572571 goto next_match;
573572
@@ -601,8 +600,17 @@ out: · 27 unmodified lines
601600 static void *nft_pipapo_get(const struct net *net, const struct nft_set *set,
602601 const struct nft_set_elem *elem, unsigned int flags)
603602 {
604− return pipapo_get(net, set, (const u8 *)elem->key.val.data,
605− nft_genmask_cur(net));
603+ struct nft_pipapo_elem *ret;
604+
605+ ret = pipapo_get(net, set, (const u8 *)elem->key.val.data,
606+ nft_genmask_cur(net));
607+ if (IS_ERR(ret))
608+ return ret;
609+
610+ if (nft_set_elem_expired(&ret->ext))
611+ return ERR_PTR(-ENOENT);
612+
613+ return ret;
606614 }
607615
608616 /**
@@ -1528,16 +1536,34 @@ static void pipapo_drop(struct nft_pipapo_match *m, · 919 unmodified lines
15281536 }
15291537 }
15301538
1539+static void nft_pipapo_gc_deactivate(struct net *net, struct nft_set *set,
1540+ struct nft_pipapo_elem *e)
1541+
1542+{
1543+ struct nft_set_elem elem = {
1544+ .priv = e,
1545+ };
1546+
1547+ nft_setelem_data_deactivate(net, set, &elem);
1548+}
1549+
15311550 /**
15321551 * pipapo_gc() - Drop expired entries from set, destroy start and end elements
15331552 * @set: nftables API set representation
15341553 * @m: Matching data
15351554 */
1536−static void pipapo_gc(const struct nft_set *set, struct nft_pipapo_match *m)
1555+static void pipapo_gc(const struct nft_set *_set, struct nft_pipapo_match *m)
15371556 {
1557+ struct nft_set *set = (struct nft_set *) _set;
15381558 struct nft_pipapo *priv = nft_set_priv(set);
1559+ struct net *net = read_pnet(&set->net);
15391560 int rules_f0, first_rule = 0;
15401561 struct nft_pipapo_elem *e;
1562+ struct nft_trans_gc *gc;
1563+
1564+ gc = nft_trans_gc_alloc(set, 0, GFP_KERNEL);
1565+ if (!gc)
1566+ return;
15411567
15421568 while ((rules_f0 = pipapo_rules_same_key(m->f, first_rule))) {
15431569 union nft_pipapo_map_bucket rulemap[NFT_PIPAPO_MAX_FIELDS];
@@ -1561,13 +1587,20 @@ static void pipapo_gc(const struct nft_set *set, struct nft_pipapo_match *m) · 17 unmodified lines
15611587 f--;
15621588 i--;
15631589 e = f->mt[rulemap[i].to].e;
1564− if (nft_set_elem_expired(&e->ext) &&
1565− !nft_set_elem_mark_busy(&e->ext)) {
1590+
1591+ /* synchronous gc never fails, there is no need to set on
1592+ * NFT_SET_ELEM_DEAD_BIT.
1593+ */
1594+ if (nft_set_elem_expired(&e->ext)) {
15661595 priv->dirty = true;
1567− pipapo_drop(m, rulemap);
15681596
1569− rcu_barrier();
1570− nft_set_elem_destroy(set, e, true);
1597+ gc = nft_trans_gc_queue_sync(gc, GFP_ATOMIC);
1598+ if (!gc)
1599+ break;
1600+
1601+ nft_pipapo_gc_deactivate(net, set, e);
1602+ pipapo_drop(m, rulemap);
1603+ nft_trans_gc_elem_add(gc, e);
15711604
15721605 /* And check again current first rule, which is now the
15731606 * first we haven't checked.
@@ -1577,11 +1610,11 @@ static void pipapo_gc(const struct nft_set *set, struct nft_pipapo_match *m) · 3 unmodified lines
15771610 }
15781611 }
15791612
1580− e = nft_set_catchall_gc(set);
1581− if (e)
1582− nft_set_elem_destroy(set, e, true);
1583−
1584− priv->last_gc = jiffies;
1613+ gc = nft_trans_gc_catchall(gc, 0);
1614+ if (gc) {
1615+ nft_trans_gc_queue_sync_done(gc);
1616+ priv->last_gc = jiffies;
1617+ }
15851618 }
15861619
15871620 /**
@@ -1706,7 +1739,6 @@ static void nft_pipapo_activate(const struct net *net, · 118 unmodified lines
17061739 return;
17071740
17081741 nft_set_elem_change_active(net, set, &e->ext);
1709− nft_set_elem_clear_busy(&e->ext);
17101742 }
17111743
17121744 /**
@@ -2005,8 +2037,6 @@ static void nft_pipapo_walk(const struct nft_ctx *ctx, struct nft_set *set, · 292 unmodified lines
20052037 goto cont;
20062038
20072039 e = f->mt[r].e;
2008− if (nft_set_elem_expired(&e->ext))
2009− goto cont;
20102040
20112041 elem.priv = e;
20122042
net/netfilter/nft_set_rbtree.c+87−59
@@ -46,6 +46,12 @@ static int nft_rbtree_cmp(const struct nft_set *set, · 45 unmodified lines
4646 set->klen);
4747 }
4848
49+static bool nft_rbtree_elem_expired(const struct nft_rbtree_elem *rbe)
50+{
51+ return nft_set_elem_expired(&rbe->ext) ||
52+ nft_set_elem_is_dead(&rbe->ext);
53+}
54+
4955 static bool __nft_rbtree_lookup(const struct net *net, const struct nft_set *set,
5056 const u32 *key, const struct nft_set_ext **ext,
5157 unsigned int seq)
@@ -80,7 +86,7 @@ static bool __nft_rbtree_lookup(const struct net *net, const struct nft_set *set · 28 unmodified lines
8086 continue;
8187 }
8288
83− if (nft_set_elem_expired(&rbe->ext))
89+ if (nft_rbtree_elem_expired(rbe))
8490 return false;
8591
8692 if (nft_rbtree_interval_end(rbe)) {
@@ -98,7 +104,7 @@ static bool __nft_rbtree_lookup(const struct net *net, const struct nft_set *set · 11 unmodified lines
98104
99105 if (set->flags & NFT_SET_INTERVAL && interval != NULL &&
100106 nft_set_elem_active(&interval->ext, genmask) &&
101− !nft_set_elem_expired(&interval->ext) &&
107+ !nft_rbtree_elem_expired(interval) &&
102108 nft_rbtree_interval_start(interval)) {
103109 *ext = &interval->ext;
104110 return true;
@@ -215,6 +221,18 @@ static void *nft_rbtree_get(const struct net *net, const struct nft_set *set, · 110 unmodified lines
215221 return rbe;
216222 }
217223
224+static void nft_rbtree_gc_remove(struct net *net, struct nft_set *set,
225+ struct nft_rbtree *priv,
226+ struct nft_rbtree_elem *rbe)
227+{
228+ struct nft_set_elem elem = {
229+ .priv = rbe,
230+ };
231+
232+ nft_setelem_data_deactivate(net, set, &elem);
233+ rb_erase(&rbe->node, &priv->root);
234+}
235+
218236 static int nft_rbtree_gc_elem(const struct nft_set *__set,
219237 struct nft_rbtree *priv,
220238 struct nft_rbtree_elem *rbe,
@@ -222,11 +240,12 @@ static int nft_rbtree_gc_elem(const struct nft_set *__set, · 1 unmodified line
222240 {
223241 struct nft_set *set = (struct nft_set *)__set;
224242 struct rb_node *prev = rb_prev(&rbe->node);
243+ struct net *net = read_pnet(&set->net);
225244 struct nft_rbtree_elem *rbe_prev;
226− struct nft_set_gc_batch *gcb;
245+ struct nft_trans_gc *gc;
227246
228− gcb = nft_set_gc_batch_check(set, NULL, GFP_ATOMIC);
229− if (!gcb)
247+ gc = nft_trans_gc_alloc(set, 0, GFP_ATOMIC);
248+ if (!gc)
230249 return -ENOMEM;
231250
232251 /* search for end interval coming before this element.
@@ -244,17 +263,28 @@ static int nft_rbtree_gc_elem(const struct nft_set *__set, · 11 unmodified lines
244263
245264 if (prev) {
246265 rbe_prev = rb_entry(prev, struct nft_rbtree_elem, node);
266+ nft_rbtree_gc_remove(net, set, priv, rbe_prev);
247267
248− rb_erase(&rbe_prev->node, &priv->root);
249− atomic_dec(&set->nelems);
250− nft_set_gc_batch_add(gcb, rbe_prev);
268+ /* There is always room in this trans gc for this element,
269+ * memory allocation never actually happens, hence, the warning
270+ * splat in such case. No need to set NFT_SET_ELEM_DEAD_BIT,
271+ * this is synchronous gc which never fails.
272+ */
273+ gc = nft_trans_gc_queue_sync(gc, GFP_ATOMIC);
274+ if (WARN_ON_ONCE(!gc))
275+ return -ENOMEM;
276+
277+ nft_trans_gc_elem_add(gc, rbe_prev);
251278 }
252279
253− rb_erase(&rbe->node, &priv->root);
254− atomic_dec(&set->nelems);
280+ nft_rbtree_gc_remove(net, set, priv, rbe);
281+ gc = nft_trans_gc_queue_sync(gc, GFP_ATOMIC);
282+ if (WARN_ON_ONCE(!gc))
283+ return -ENOMEM;
284+
285+ nft_trans_gc_elem_add(gc, rbe);
255286
256− nft_set_gc_batch_add(gcb, rbe);
257− nft_set_gc_batch_complete(gcb);
287+ nft_trans_gc_queue_sync_done(gc);
258288
259289 return 0;
260290 }
@@ -482,7 +512,6 @@ static void nft_rbtree_activate(const struct net *net, · 221 unmodified lines
482512 struct nft_rbtree_elem *rbe = elem->priv;
483513
484514 nft_set_elem_change_active(net, set, &rbe->ext);
485− nft_set_elem_clear_busy(&rbe->ext);
486515 }
487516
488517 static bool nft_rbtree_flush(const struct net *net,
@@ -490,12 +519,9 @@ static bool nft_rbtree_flush(const struct net *net, · 1 unmodified line
490519 {
491520 struct nft_rbtree_elem *rbe = priv;
492521
493− if (!nft_set_elem_mark_busy(&rbe->ext) ||
494− !nft_is_active(net, &rbe->ext)) {
495− nft_set_elem_change_active(net, set, &rbe->ext);
496− return true;
497− }
498− return false;
522+ nft_set_elem_change_active(net, set, &rbe->ext);
523+
524+ return true;
499525 }
500526
501527 static void *nft_rbtree_deactivate(const struct net *net,
@@ -552,8 +578,6 @@ static void nft_rbtree_walk(const struct nft_ctx *ctx, · 50 unmodified lines
552578
553579 if (iter->count < iter->skip)
554580 goto cont;
555− if (nft_set_elem_expired(&rbe->ext))
556− goto cont;
557581 if (!nft_set_elem_active(&rbe->ext, iter->genmask))
558582 goto cont;
559583
@@ -572,26 +596,40 @@ cont: · 12 unmodified lines
572596
573597 static void nft_rbtree_gc(struct work_struct *work)
574598 {
575− struct nft_rbtree_elem *rbe, *rbe_end = NULL, *rbe_prev = NULL;
576− struct nft_set_gc_batch *gcb = NULL;
599+ struct nft_rbtree_elem *rbe, *rbe_end = NULL;
600+ struct nftables_pernet *nft_net;
577601 struct nft_rbtree *priv;
602+ struct nft_trans_gc *gc;
578603 struct rb_node *node;
579604 struct nft_set *set;
605+ unsigned int gc_seq;
580606 struct net *net;
581− u8 genmask;
582607
583608 priv = container_of(work, struct nft_rbtree, gc_work.work);
584609 set = nft_set_container_of(priv);
585610 net = read_pnet(&set->net);
586− genmask = nft_genmask_cur(net);
611+ nft_net = nft_pernet(net);
612+ gc_seq = READ_ONCE(nft_net->gc_seq);
613+
614+ gc = nft_trans_gc_alloc(set, gc_seq, GFP_KERNEL);
615+ if (!gc)
616+ goto done;
587617
588618 write_lock_bh(&priv->lock);
589619 write_seqcount_begin(&priv->count);
590620 for (node = rb_first(&priv->root); node != NULL; node = rb_next(node)) {
621+
622+ /* Ruleset has been updated, try later. */
623+ if (READ_ONCE(nft_net->gc_seq) != gc_seq) {
624+ nft_trans_gc_destroy(gc);
625+ gc = NULL;
626+ goto try_later;
627+ }
628+
591629 rbe = rb_entry(node, struct nft_rbtree_elem, node);
592630
593− if (!nft_set_elem_active(&rbe->ext, genmask))
594− continue;
631+ if (nft_set_elem_is_dead(&rbe->ext))
632+ goto dead_elem;
595633
596634 /* elements are reversed in the rbtree for historical reasons,
597635 * from highest to lowest value, that is why end element is
@@ -604,46 +642,36 @@ static void nft_rbtree_gc(struct work_struct *work) · 6 unmodified lines
604642 if (!nft_set_elem_expired(&rbe->ext))
605643 continue;
606644
607− if (nft_set_elem_mark_busy(&rbe->ext)) {
608− rbe_end = NULL;
645+ nft_set_elem_dead(&rbe->ext);
646+
647+ if (!rbe_end)
609648 continue;
610− }
611649
612− if (rbe_prev) {
613− rb_erase(&rbe_prev->node, &priv->root);
614− rbe_prev = NULL;
615− }
616− gcb = nft_set_gc_batch_check(set, gcb, GFP_ATOMIC);
617− if (!gcb)
618− break;
650+ nft_set_elem_dead(&rbe_end->ext);
619651
620− atomic_dec(&set->nelems);
621− nft_set_gc_batch_add(gcb, rbe);
622− rbe_prev = rbe;
652+ gc = nft_trans_gc_queue_async(gc, gc_seq, GFP_ATOMIC);
653+ if (!gc)
654+ goto try_later;
623655
624− if (rbe_end) {
625− atomic_dec(&set->nelems);
626− nft_set_gc_batch_add(gcb, rbe_end);
627− rb_erase(&rbe_end->node, &priv->root);
628− rbe_end = NULL;
629− }
630− node = rb_next(node);
631− if (!node)
632− break;
656+ nft_trans_gc_elem_add(gc, rbe_end);
657+ rbe_end = NULL;
658+dead_elem:
659+ gc = nft_trans_gc_queue_async(gc, gc_seq, GFP_ATOMIC);
660+ if (!gc)
661+ goto try_later;
662+
663+ nft_trans_gc_elem_add(gc, rbe);
633664 }
634− if (rbe_prev)
635− rb_erase(&rbe_prev->node, &priv->root);
665+
666+ gc = nft_trans_gc_catchall(gc, gc_seq);
667+
668+try_later:
636669 write_seqcount_end(&priv->count);
637670 write_unlock_bh(&priv->lock);
638671
639− rbe = nft_set_catchall_gc(set);
640− if (rbe) {
641− gcb = nft_set_gc_batch_check(set, gcb, GFP_ATOMIC);
642− if (gcb)
643− nft_set_gc_batch_add(gcb, rbe);
644− }
645− nft_set_gc_batch_complete(gcb);
646−
672+ if (gc)
673+ nft_trans_gc_queue_async_done(gc);
674+done:
647675 queue_delayed_work(system_power_efficient_wq, &priv->gc_work,
648676 nft_set_gc_interval(set));
649677 }

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t