/* Copyright (c) 2017 Covalent IO, Inc. http://covalent.io * * This program is free software; you can redistribute it and/or * modify it under the terms of version 2 of the GNU General Public * License as published by the Free Software Foundation. * * This program is distributed in the hope that it will be useful, but * WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. */ /* A BPF sock_map is used to store sock objects. This is primarly used * for doing socket redirect with BPF helper routines. * * A sock map may have BPF programs attached to it, currently a program * used to parse packets and a program to provide a verdict and redirect * decision on the packet are supported. Any programs attached to a sock * map are inherited by sock objects when they are added to the map. If * no BPF programs are attached the sock object may only be used for sock * redirect. * * A sock object may be in multiple maps, but can only inherit a single * parse or verdict program. If adding a sock object to a map would result * in having multiple parsing programs the update will return an EBUSY error. * * For reference this program is similar to devmap used in XDP context * reviewing these together may be useful. For an example please review * ./samples/bpf/sockmap/. */#include <linux/bpf.h>#include <net/sock.h>#include <linux/filter.h>#include <linux/errno.h>#include <linux/file.h>#include <linux/kernel.h>#include <linux/net.h>#include <linux/skbuff.h>#include <linux/workqueue.h>#include <linux/list.h>#include <net/strparser.h>#include <net/tcp.h> struct bpf_stab { struct bpf_map map; struct sock **sock_map; struct bpf_prog *bpf_parse; struct bpf_prog *bpf_verdict;}; enum smap_psock_state { SMAP_TX_RUNNING,}; struct smap_psock_map_entry { struct list_head list; struct sock **entry;}; struct smap_psock { struct rcu_head rcu; /* refcnt is used inside sk_callback_lock */ u32 refcnt; /* datapath variables */ struct sk_buff_head rxqueue; bool strp_enabled; /* datapath error path cache across tx work invocations */ int save_rem; int save_off; struct sk_buff *save_skb; struct strparser strp; struct bpf_prog *bpf_parse; struct bpf_prog *bpf_verdict; struct list_head maps; /* Back reference used when sock callback trigger sockmap operations */ struct sock *sock; unsigned long state; struct work_struct tx_work; struct work_struct gc_work; void (*save_data_ready)(struct sock *sk); void (*save_write_space)(struct sock *sk); void (*save_state_change)(struct sock *sk);}; static inline struct smap_psock *smap_psock_sk(const struct sock *sk){ return rcu_dereference_sk_user_data(sk);} /* compute the linear packet data range [data, data_end) for skb when * sk_skb type programs are in use. */static inline void bpf_compute_data_end_sk_skb(struct sk_buff *skb){ TCP_SKB_CB(skb)->bpf.data_end = skb->data + skb_headlen(skb);} enum __sk_action { __SK_DROP = 0, __SK_PASS, __SK_REDIRECT,}; static int smap_verdict_func(struct smap_psock *psock, struct sk_buff *skb){ struct bpf_prog *prog = READ_ONCE(psock->bpf_verdict); int rc; if (unlikely(!prog)) return __SK_DROP; skb_orphan(skb); /* We need to ensure that BPF metadata for maps is also cleared * when we orphan the skb so that we don't have the possibility * to reference a stale map. */ TCP_SKB_CB(skb)->bpf.map = NULL; skb->sk = psock->sock; bpf_compute_data_end_sk_skb(skb); preempt_disable(); rc = (*prog->bpf_func)(skb, prog->insnsi); preempt_enable(); skb->sk = NULL; /* Moving return codes from UAPI namespace into internal namespace */ return rc == SK_PASS ? (TCP_SKB_CB(skb)->bpf.map ? __SK_REDIRECT : __SK_PASS) : __SK_DROP;} static void smap_do_verdict(struct smap_psock *psock, struct sk_buff *skb){ struct sock *sk; int rc; rc = smap_verdict_func(psock, skb); switch (rc) { case __SK_REDIRECT: sk = do_sk_redirect_map(skb); if (likely(sk)) { struct smap_psock *peer = smap_psock_sk(sk); if (likely(peer && test_bit(SMAP_TX_RUNNING, &peer->state) && !sock_flag(sk, SOCK_DEAD) && sock_writeable(sk))) { skb_set_owner_w(skb, sk); skb_queue_tail(&peer->rxqueue, skb); schedule_work(&peer->tx_work); break; } } /* Fall through and free skb otherwise */ case __SK_DROP: default: kfree_skb(skb); }} static void smap_report_sk_error(struct smap_psock *psock, int err){ struct sock *sk = psock->sock; sk->sk_err = err; sk->sk_error_report(sk);} static void smap_release_sock(struct smap_psock *psock, struct sock *sock); /* Called with lock_sock(sk) held */static void smap_state_change(struct sock *sk){ struct smap_psock_map_entry *e, *tmp; struct smap_psock *psock; struct socket_wq *wq; struct sock *osk; rcu_read_lock(); /* Allowing transitions into an established syn_recv states allows * for early binding sockets to a smap object before the connection * is established. */ switch (sk->sk_state) { case TCP_SYN_SENT: case TCP_SYN_RECV: case TCP_ESTABLISHED: break; case TCP_CLOSE_WAIT: case TCP_CLOSING: case TCP_LAST_ACK: case TCP_FIN_WAIT1: case TCP_FIN_WAIT2: case TCP_LISTEN: break; case TCP_CLOSE: /* Only release if the map entry is in fact the sock in * question. There is a case where the operator deletes * the sock from the map, but the TCP sock is closed before * the psock is detached. Use cmpxchg to verify correct * sock is removed. */ psock = smap_psock_sk(sk); if (unlikely(!psock)) break; write_lock_bh(&sk->sk_callback_lock); list_for_each_entry_safe(e, tmp, &psock->maps, list) { osk = cmpxchg(e->entry, sk, NULL); if (osk == sk) { list_del(&e->list); smap_release_sock(psock, sk); } } write_unlock_bh(&sk->sk_callback_lock); break; default: psock = smap_psock_sk(sk); if (unlikely(!psock)) break; smap_report_sk_error(psock, EPIPE); break; } wq = rcu_dereference(sk->sk_wq); if (skwq_has_sleeper(wq)) wake_up_interruptible_all(&wq->wait); rcu_read_unlock();} static void smap_read_sock_strparser(struct strparser *strp, struct sk_buff *skb){ struct smap_psock *psock; rcu_read_lock(); psock = container_of(strp, struct smap_psock, strp); smap_do_verdict(psock, skb); rcu_read_unlock();} /* Called with lock held on socket */static void smap_data_ready(struct sock *sk){ struct smap_psock *psock; rcu_read_lock(); psock = smap_psock_sk(sk); if (likely(psock)) { write_lock_bh(&sk->sk_callback_lock); strp_data_ready(&psock->strp); write_unlock_bh(&sk->sk_callback_lock); } rcu_read_unlock();} static void smap_tx_work(struct work_struct *w){ struct smap_psock *psock; struct sk_buff *skb; int rem, off, n; psock = container_of(w, struct smap_psock, tx_work); /* lock sock to avoid losing sk_socket at some point during loop */ lock_sock(psock->sock); if (psock->save_skb) { skb = psock->save_skb; rem = psock->save_rem; off = psock->save_off; psock->save_skb = NULL; goto start; } while ((skb = skb_dequeue(&psock->rxqueue))) { rem = skb->len; off = 0;start: do { if (likely(psock->sock->sk_socket)) n = skb_send_sock_locked(psock->sock, skb, off, rem); else n = -EINVAL; if (n <= 0) { if (n == -EAGAIN) { /* Retry when space is available */ psock->save_skb = skb; psock->save_rem = rem; psock->save_off = off; goto out; } /* Hard errors break pipe and stop xmit */ smap_report_sk_error(psock, n ? -n : EPIPE); clear_bit(SMAP_TX_RUNNING, &psock->state); kfree_skb(skb); goto out; } rem -= n; off += n; } while (rem); kfree_skb(skb); }out: release_sock(psock->sock);} static void smap_write_space(struct sock *sk){ struct smap_psock *psock; rcu_read_lock(); psock = smap_psock_sk(sk); if (likely(psock && test_bit(SMAP_TX_RUNNING, &psock->state))) schedule_work(&psock->tx_work); rcu_read_unlock();} static void smap_stop_sock(struct smap_psock *psock, struct sock *sk){ if (!psock->strp_enabled) return; sk->sk_data_ready = psock->save_data_ready; sk->sk_write_space = psock->save_write_space; sk->sk_state_change = psock->save_state_change; psock->save_data_ready = NULL; psock->save_write_space = NULL; psock->save_state_change = NULL; strp_stop(&psock->strp); psock->strp_enabled = false;} static void smap_destroy_psock(struct rcu_head *rcu){ struct smap_psock *psock = container_of(rcu, struct smap_psock, rcu); /* Now that a grace period has passed there is no longer * any reference to this sock in the sockmap so we can * destroy the psock, strparser, and bpf programs. But, * because we use workqueue sync operations we can not * do it in rcu context */ schedule_work(&psock->gc_work);} static void smap_release_sock(struct smap_psock *psock, struct sock *sock){ psock->refcnt--; if (psock->refcnt) return; smap_stop_sock(psock, sock); clear_bit(SMAP_TX_RUNNING, &psock->state); rcu_assign_sk_user_data(sock, NULL); call_rcu_sched(&psock->rcu, smap_destroy_psock);} static int smap_parse_func_strparser(struct strparser *strp, struct sk_buff *skb){ struct smap_psock *psock; struct bpf_prog *prog; int rc; rcu_read_lock(); psock = container_of(strp, struct smap_psock, strp); prog = READ_ONCE(psock->bpf_parse); if (unlikely(!prog)) { rcu_read_unlock(); return skb->len; } /* Attach socket for bpf program to use if needed we can do this * because strparser clones the skb before handing it to a upper * layer, meaning skb_orphan has been called. We NULL sk on the * way out to ensure we don't trigger a BUG_ON in skb/sk operations * later and because we are not charging the memory of this skb to * any socket yet. */ skb->sk = psock->sock; bpf_compute_data_end_sk_skb(skb); rc = (*prog->bpf_func)(skb, prog->insnsi); skb->sk = NULL; rcu_read_unlock(); return rc;} static int smap_read_sock_done(struct strparser *strp, int err){ return err;} static int smap_init_sock(struct smap_psock *psock, struct sock *sk){ static const struct strp_callbacks cb = { .rcv_msg = smap_read_sock_strparser, .parse_msg = smap_parse_func_strparser, .read_sock_done = smap_read_sock_done, }; return strp_init(&psock->strp, sk, &cb);} static void smap_init_progs(struct smap_psock *psock, struct bpf_stab *stab, struct bpf_prog *verdict, struct bpf_prog *parse){ struct bpf_prog *orig_parse, *orig_verdict; orig_parse = xchg(&psock->bpf_parse, parse); orig_verdict = xchg(&psock->bpf_verdict, verdict); if (orig_verdict) bpf_prog_put(orig_verdict); if (orig_parse) bpf_prog_put(orig_parse);} static void smap_start_sock(struct smap_psock *psock, struct sock *sk){ if (sk->sk_data_ready == smap_data_ready) return; psock->save_data_ready = sk->sk_data_ready; psock->save_write_space = sk->sk_write_space; psock->save_state_change = sk->sk_state_change; sk->sk_data_ready = smap_data_ready; sk->sk_write_space = smap_write_space; sk->sk_state_change = smap_state_change; psock->strp_enabled = true;} static void sock_map_remove_complete(struct bpf_stab *stab){ bpf_map_area_free(stab->sock_map); kfree(stab);} static void smap_gc_work(struct work_struct *w){ struct smap_psock_map_entry *e, *tmp; struct smap_psock *psock; psock = container_of(w, struct smap_psock, gc_work); /* no callback lock needed because we already detached sockmap ops */ if (psock->strp_enabled) strp_done(&psock->strp); cancel_work_sync(&psock->tx_work); __skb_queue_purge(&psock->rxqueue); /* At this point all strparser and xmit work must be complete */ if (psock->bpf_parse) bpf_prog_put(psock->bpf_parse); if (psock->bpf_verdict) bpf_prog_put(psock->bpf_verdict); list_for_each_entry_safe(e, tmp, &psock->maps, list) { list_del(&e->list); kfree(e); } sock_put(psock->sock); kfree(psock);} static struct smap_psock *smap_init_psock(struct sock *sock, struct bpf_stab *stab){ struct smap_psock *psock; psock = kzalloc_node(sizeof(struct smap_psock), GFP_ATOMIC | __GFP_NOWARN, stab->map.numa_node); if (!psock) return ERR_PTR(-ENOMEM); psock->sock = sock; skb_queue_head_init(&psock->rxqueue); INIT_WORK(&psock->tx_work, smap_tx_work); INIT_WORK(&psock->gc_work, smap_gc_work); INIT_LIST_HEAD(&psock->maps); psock->refcnt = 1; rcu_assign_sk_user_data(sock, psock); sock_hold(sock); return psock;} static struct bpf_map *sock_map_alloc(union bpf_attr *attr)Lines 1–500 of 898. Load more
/* Copyright (c) 2017 Covalent IO, Inc. http://covalent.io * * This program is free software; you can redistribute it and/or * modify it under the terms of version 2 of the GNU General Public * License as published by the Free Software Foundation. * * This program is distributed in the hope that it will be useful, but * WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. */ /* A BPF sock_map is used to store sock objects. This is primarly used * for doing socket redirect with BPF helper routines. * * A sock map may have BPF programs attached to it, currently a program * used to parse packets and a program to provide a verdict and redirect * decision on the packet are supported. Any programs attached to a sock * map are inherited by sock objects when they are added to the map. If * no BPF programs are attached the sock object may only be used for sock * redirect. * * A sock object may be in multiple maps, but can only inherit a single * parse or verdict program. If adding a sock object to a map would result * in having multiple parsing programs the update will return an EBUSY error. * * For reference this program is similar to devmap used in XDP context * reviewing these together may be useful. For an example please review * ./samples/bpf/sockmap/. */ #include <linux/bpf.h> #include <net/sock.h> #include <linux/filter.h> #include <linux/errno.h> #include <linux/file.h> #include <linux/kernel.h> #include <linux/net.h> #include <linux/skbuff.h> #include <linux/workqueue.h> #include <linux/list.h> #include <net/strparser.h> #include <net/tcp.h> struct bpf_stab { struct bpf_map map; struct sock **sock_map; struct bpf_prog *bpf_parse; struct bpf_prog *bpf_verdict; }; enum smap_psock_state { SMAP_TX_RUNNING, }; struct smap_psock_map_entry { struct list_head list; struct sock **entry; }; struct smap_psock { struct rcu_head rcu; /* refcnt is used inside sk_callback_lock */ u32 refcnt; /* datapath variables */ struct sk_buff_head rxqueue; bool strp_enabled; /* datapath error path cache across tx work invocations */ int save_rem; int save_off; struct sk_buff *save_skb; struct strparser strp; struct bpf_prog *bpf_parse; struct bpf_prog *bpf_verdict; struct list_head maps; /* Back reference used when sock callback trigger sockmap operations */ struct sock *sock; unsigned long state; struct work_struct tx_work; struct work_struct gc_work; void (*save_data_ready)(struct sock *sk); void (*save_write_space)(struct sock *sk); void (*save_state_change)(struct sock *sk); }; static inline struct smap_psock *smap_psock_sk(const struct sock *sk) { return rcu_dereference_sk_user_data(sk); } /* compute the linear packet data range [data, data_end) for skb when * sk_skb type programs are in use. */ static inline void bpf_compute_data_end_sk_skb(struct sk_buff *skb) { TCP_SKB_CB(skb)->bpf.data_end = skb->data + skb_headlen(skb); } enum __sk_action { __SK_DROP = 0, __SK_PASS, __SK_REDIRECT, }; static int smap_verdict_func(struct smap_psock *psock, struct sk_buff *skb) { struct bpf_prog *prog = READ_ONCE(psock->bpf_verdict); int rc; if (unlikely(!prog)) return __SK_DROP; skb_orphan(skb); /* We need to ensure that BPF metadata for maps is also cleared * when we orphan the skb so that we don't have the possibility * to reference a stale map. */ TCP_SKB_CB(skb)->bpf.map = NULL; skb->sk = psock->sock; bpf_compute_data_end_sk_skb(skb); preempt_disable(); rc = (*prog->bpf_func)(skb, prog->insnsi); preempt_enable(); skb->sk = NULL; /* Moving return codes from UAPI namespace into internal namespace */ return rc == SK_PASS ? (TCP_SKB_CB(skb)->bpf.map ? __SK_REDIRECT : __SK_PASS) : __SK_DROP; } static void smap_do_verdict(struct smap_psock *psock, struct sk_buff *skb) { struct sock *sk; int rc; rc = smap_verdict_func(psock, skb); switch (rc) { case __SK_REDIRECT: sk = do_sk_redirect_map(skb); if (likely(sk)) { struct smap_psock *peer = smap_psock_sk(sk); if (likely(peer && test_bit(SMAP_TX_RUNNING, &peer->state) && !sock_flag(sk, SOCK_DEAD) && sock_writeable(sk))) { skb_set_owner_w(skb, sk); skb_queue_tail(&peer->rxqueue, skb); schedule_work(&peer->tx_work); break; } } /* Fall through and free skb otherwise */ case __SK_DROP: default: kfree_skb(skb); } } static void smap_report_sk_error(struct smap_psock *psock, int err) { struct sock *sk = psock->sock; sk->sk_err = err; sk->sk_error_report(sk); } static void smap_release_sock(struct smap_psock *psock, struct sock *sock); /* Called with lock_sock(sk) held */ static void smap_state_change(struct sock *sk) { struct smap_psock_map_entry *e, *tmp; struct smap_psock *psock; struct socket_wq *wq; struct sock *osk; rcu_read_lock(); /* Allowing transitions into an established syn_recv states allows * for early binding sockets to a smap object before the connection * is established. */ switch (sk->sk_state) { case TCP_SYN_SENT: case TCP_SYN_RECV: case TCP_ESTABLISHED: break; case TCP_CLOSE_WAIT: case TCP_CLOSING: case TCP_LAST_ACK: case TCP_FIN_WAIT1: case TCP_FIN_WAIT2: case TCP_LISTEN: break; case TCP_CLOSE: /* Only release if the map entry is in fact the sock in * question. There is a case where the operator deletes * the sock from the map, but the TCP sock is closed before * the psock is detached. Use cmpxchg to verify correct * sock is removed. */ psock = smap_psock_sk(sk); if (unlikely(!psock)) break; write_lock_bh(&sk->sk_callback_lock); list_for_each_entry_safe(e, tmp, &psock->maps, list) { osk = cmpxchg(e->entry, sk, NULL); if (osk == sk) { list_del(&e->list); smap_release_sock(psock, sk); } } write_unlock_bh(&sk->sk_callback_lock); break; default: psock = smap_psock_sk(sk); if (unlikely(!psock)) break; smap_report_sk_error(psock, EPIPE); break; } wq = rcu_dereference(sk->sk_wq); if (skwq_has_sleeper(wq)) wake_up_interruptible_all(&wq->wait); rcu_read_unlock(); } static void smap_read_sock_strparser(struct strparser *strp, struct sk_buff *skb) { struct smap_psock *psock; rcu_read_lock(); psock = container_of(strp, struct smap_psock, strp); smap_do_verdict(psock, skb); rcu_read_unlock(); } /* Called with lock held on socket */ static void smap_data_ready(struct sock *sk) { struct smap_psock *psock; rcu_read_lock(); psock = smap_psock_sk(sk); if (likely(psock)) { write_lock_bh(&sk->sk_callback_lock); strp_data_ready(&psock->strp); write_unlock_bh(&sk->sk_callback_lock); } rcu_read_unlock(); } static void smap_tx_work(struct work_struct *w) { struct smap_psock *psock; struct sk_buff *skb; int rem, off, n; psock = container_of(w, struct smap_psock, tx_work); /* lock sock to avoid losing sk_socket at some point during loop */ lock_sock(psock->sock); if (psock->save_skb) { skb = psock->save_skb; rem = psock->save_rem; off = psock->save_off; psock->save_skb = NULL; goto start; } while ((skb = skb_dequeue(&psock->rxqueue))) { rem = skb->len; off = 0; start: do { if (likely(psock->sock->sk_socket)) n = skb_send_sock_locked(psock->sock, skb, off, rem); else n = -EINVAL; if (n <= 0) { if (n == -EAGAIN) { /* Retry when space is available */ psock->save_skb = skb; psock->save_rem = rem; psock->save_off = off; goto out; } /* Hard errors break pipe and stop xmit */ smap_report_sk_error(psock, n ? -n : EPIPE); clear_bit(SMAP_TX_RUNNING, &psock->state); kfree_skb(skb); goto out; } rem -= n; off += n; } while (rem); kfree_skb(skb); } out: release_sock(psock->sock); } static void smap_write_space(struct sock *sk) { struct smap_psock *psock; rcu_read_lock(); psock = smap_psock_sk(sk); if (likely(psock && test_bit(SMAP_TX_RUNNING, &psock->state))) schedule_work(&psock->tx_work); rcu_read_unlock(); } static void smap_stop_sock(struct smap_psock *psock, struct sock *sk) { if (!psock->strp_enabled) return; sk->sk_data_ready = psock->save_data_ready; sk->sk_write_space = psock->save_write_space; sk->sk_state_change = psock->save_state_change; psock->save_data_ready = NULL; psock->save_write_space = NULL; psock->save_state_change = NULL; strp_stop(&psock->strp); psock->strp_enabled = false; } static void smap_destroy_psock(struct rcu_head *rcu) { struct smap_psock *psock = container_of(rcu, struct smap_psock, rcu); /* Now that a grace period has passed there is no longer * any reference to this sock in the sockmap so we can * destroy the psock, strparser, and bpf programs. But, * because we use workqueue sync operations we can not * do it in rcu context */ schedule_work(&psock->gc_work); } static void smap_release_sock(struct smap_psock *psock, struct sock *sock) { psock->refcnt--; if (psock->refcnt) return; smap_stop_sock(psock, sock); clear_bit(SMAP_TX_RUNNING, &psock->state); rcu_assign_sk_user_data(sock, NULL); call_rcu_sched(&psock->rcu, smap_destroy_psock); } static int smap_parse_func_strparser(struct strparser *strp, struct sk_buff *skb) { struct smap_psock *psock; struct bpf_prog *prog; int rc; rcu_read_lock(); psock = container_of(strp, struct smap_psock, strp); prog = READ_ONCE(psock->bpf_parse); if (unlikely(!prog)) { rcu_read_unlock(); return skb->len; } /* Attach socket for bpf program to use if needed we can do this * because strparser clones the skb before handing it to a upper * layer, meaning skb_orphan has been called. We NULL sk on the * way out to ensure we don't trigger a BUG_ON in skb/sk operations * later and because we are not charging the memory of this skb to * any socket yet. */ skb->sk = psock->sock; bpf_compute_data_end_sk_skb(skb); rc = (*prog->bpf_func)(skb, prog->insnsi); skb->sk = NULL; rcu_read_unlock(); return rc; } static int smap_read_sock_done(struct strparser *strp, int err) { return err; } static int smap_init_sock(struct smap_psock *psock, struct sock *sk) { static const struct strp_callbacks cb = { .rcv_msg = smap_read_sock_strparser, .parse_msg = smap_parse_func_strparser, .read_sock_done = smap_read_sock_done, }; return strp_init(&psock->strp, sk, &cb); } static void smap_init_progs(struct smap_psock *psock, struct bpf_stab *stab, struct bpf_prog *verdict, struct bpf_prog *parse) { struct bpf_prog *orig_parse, *orig_verdict; orig_parse = xchg(&psock->bpf_parse, parse); orig_verdict = xchg(&psock->bpf_verdict, verdict); if (orig_verdict) bpf_prog_put(orig_verdict); if (orig_parse) bpf_prog_put(orig_parse); } static void smap_start_sock(struct smap_psock *psock, struct sock *sk) { if (sk->sk_data_ready == smap_data_ready) return; psock->save_data_ready = sk->sk_data_ready; psock->save_write_space = sk->sk_write_space; psock->save_state_change = sk->sk_state_change; sk->sk_data_ready = smap_data_ready; sk->sk_write_space = smap_write_space; sk->sk_state_change = smap_state_change; psock->strp_enabled = true; } static void sock_map_remove_complete(struct bpf_stab *stab) { bpf_map_area_free(stab->sock_map); kfree(stab); } static void smap_gc_work(struct work_struct *w) { struct smap_psock_map_entry *e, *tmp; struct smap_psock *psock; psock = container_of(w, struct smap_psock, gc_work); /* no callback lock needed because we already detached sockmap ops */ if (psock->strp_enabled) strp_done(&psock->strp); cancel_work_sync(&psock->tx_work); __skb_queue_purge(&psock->rxqueue); /* At this point all strparser and xmit work must be complete */ if (psock->bpf_parse) bpf_prog_put(psock->bpf_parse); if (psock->bpf_verdict) bpf_prog_put(psock->bpf_verdict); list_for_each_entry_safe(e, tmp, &psock->maps, list) { list_del(&e->list); kfree(e); } sock_put(psock->sock); kfree(psock); } static struct smap_psock *smap_init_psock(struct sock *sock, struct bpf_stab *stab) { struct smap_psock *psock; psock = kzalloc_node(sizeof(struct smap_psock), GFP_ATOMIC | __GFP_NOWARN, stab->map.numa_node); if (!psock) return ERR_PTR(-ENOMEM); psock->sock = sock; skb_queue_head_init(&psock->rxqueue); INIT_WORK(&psock->tx_work, smap_tx_work); INIT_WORK(&psock->gc_work, smap_gc_work); INIT_LIST_HEAD(&psock->maps); psock->refcnt = 1; rcu_assign_sk_user_data(sock, psock); sock_hold(sock); return psock; } static struct bpf_map *sock_map_alloc(union bpf_attr *attr)