/* Copyright (c) 2017 Covalent IO, Inc. http://covalent.io * * This program is free software; you can redistribute it and/or * modify it under the terms of version 2 of the GNU General Public * License as published by the Free Software Foundation. * * This program is distributed in the hope that it will be useful, but * WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. */ /* A BPF sock_map is used to store sock objects. This is primarly used * for doing socket redirect with BPF helper routines. * * A sock map may have BPF programs attached to it, currently a program * used to parse packets and a program to provide a verdict and redirect * decision on the packet are supported. Any programs attached to a sock * map are inherited by sock objects when they are added to the map. If * no BPF programs are attached the sock object may only be used for sock * redirect. * * A sock object may be in multiple maps, but can only inherit a single * parse or verdict program. If adding a sock object to a map would result * in having multiple parsing programs the update will return an EBUSY error. * * For reference this program is similar to devmap used in XDP context * reviewing these together may be useful. For an example please review * ./samples/bpf/sockmap/. */#include <linux/bpf.h>#include <net/sock.h>#include <linux/filter.h>#include <linux/errno.h>#include <linux/file.h>#include <linux/kernel.h>#include <linux/net.h>#include <linux/skbuff.h>#include <linux/workqueue.h>#include <linux/list.h>#include <net/strparser.h>#include <net/tcp.h> #define SOCK_CREATE_FLAG_MASK \ (BPF_F_NUMA_NODE | BPF_F_RDONLY | BPF_F_WRONLY) struct bpf_stab { struct bpf_map map; struct sock **sock_map; struct bpf_prog *bpf_parse; struct bpf_prog *bpf_verdict;}; enum smap_psock_state { SMAP_TX_RUNNING,}; struct smap_psock_map_entry { struct list_head list; struct sock **entry;}; struct smap_psock { struct rcu_head rcu; /* refcnt is used inside sk_callback_lock */ u32 refcnt; /* datapath variables */ struct sk_buff_head rxqueue; bool strp_enabled; /* datapath error path cache across tx work invocations */ int save_rem; int save_off; struct sk_buff *save_skb; struct strparser strp; struct bpf_prog *bpf_parse; struct bpf_prog *bpf_verdict; struct list_head maps; /* Back reference used when sock callback trigger sockmap operations */ struct sock *sock; unsigned long state; struct work_struct tx_work; struct work_struct gc_work; struct proto *sk_proto; void (*save_close)(struct sock *sk, long timeout); void (*save_data_ready)(struct sock *sk); void (*save_write_space)(struct sock *sk);}; static inline struct smap_psock *smap_psock_sk(const struct sock *sk){ return rcu_dereference_sk_user_data(sk);} static struct proto tcp_bpf_proto;static int bpf_tcp_init(struct sock *sk){ struct smap_psock *psock; rcu_read_lock(); psock = smap_psock_sk(sk); if (unlikely(!psock)) { rcu_read_unlock(); return -EINVAL; } if (unlikely(psock->sk_proto)) { rcu_read_unlock(); return -EBUSY; } psock->save_close = sk->sk_prot->close; psock->sk_proto = sk->sk_prot; sk->sk_prot = &tcp_bpf_proto; rcu_read_unlock(); return 0;} static void bpf_tcp_release(struct sock *sk){ struct smap_psock *psock; rcu_read_lock(); psock = smap_psock_sk(sk); if (likely(psock)) { sk->sk_prot = psock->sk_proto; psock->sk_proto = NULL; } rcu_read_unlock();} static void smap_release_sock(struct smap_psock *psock, struct sock *sock); static void bpf_tcp_close(struct sock *sk, long timeout){ void (*close_fun)(struct sock *sk, long timeout); struct smap_psock_map_entry *e, *tmp; struct smap_psock *psock; struct sock *osk; rcu_read_lock(); psock = smap_psock_sk(sk); if (unlikely(!psock)) { rcu_read_unlock(); return sk->sk_prot->close(sk, timeout); } /* The psock may be destroyed anytime after exiting the RCU critial * section so by the time we use close_fun the psock may no longer * be valid. However, bpf_tcp_close is called with the sock lock * held so the close hook and sk are still valid. */ close_fun = psock->save_close; write_lock_bh(&sk->sk_callback_lock); list_for_each_entry_safe(e, tmp, &psock->maps, list) { osk = cmpxchg(e->entry, sk, NULL); if (osk == sk) { list_del(&e->list); smap_release_sock(psock, sk); } } write_unlock_bh(&sk->sk_callback_lock); rcu_read_unlock(); close_fun(sk, timeout);} enum __sk_action { __SK_DROP = 0, __SK_PASS, __SK_REDIRECT,}; static struct tcp_ulp_ops bpf_tcp_ulp_ops __read_mostly = { .name = "bpf_tcp", .uid = TCP_ULP_BPF, .user_visible = false, .owner = NULL, .init = bpf_tcp_init, .release = bpf_tcp_release,}; static int bpf_tcp_ulp_register(void){ tcp_bpf_proto = tcp_prot; tcp_bpf_proto.close = bpf_tcp_close; return tcp_register_ulp(&bpf_tcp_ulp_ops);} static int smap_verdict_func(struct smap_psock *psock, struct sk_buff *skb){ struct bpf_prog *prog = READ_ONCE(psock->bpf_verdict); int rc; if (unlikely(!prog)) return __SK_DROP; skb_orphan(skb); /* We need to ensure that BPF metadata for maps is also cleared * when we orphan the skb so that we don't have the possibility * to reference a stale map. */ TCP_SKB_CB(skb)->bpf.map = NULL; skb->sk = psock->sock; bpf_compute_data_pointers(skb); preempt_disable(); rc = (*prog->bpf_func)(skb, prog->insnsi); preempt_enable(); skb->sk = NULL; /* Moving return codes from UAPI namespace into internal namespace */ return rc == SK_PASS ? (TCP_SKB_CB(skb)->bpf.map ? __SK_REDIRECT : __SK_PASS) : __SK_DROP;} static void smap_do_verdict(struct smap_psock *psock, struct sk_buff *skb){ struct sock *sk; int rc; rc = smap_verdict_func(psock, skb); switch (rc) { case __SK_REDIRECT: sk = do_sk_redirect_map(skb); if (likely(sk)) { struct smap_psock *peer = smap_psock_sk(sk); if (likely(peer && test_bit(SMAP_TX_RUNNING, &peer->state) && !sock_flag(sk, SOCK_DEAD) && sock_writeable(sk))) { skb_set_owner_w(skb, sk); skb_queue_tail(&peer->rxqueue, skb); schedule_work(&peer->tx_work); break; } } /* Fall through and free skb otherwise */ case __SK_DROP: default: kfree_skb(skb); }} static void smap_report_sk_error(struct smap_psock *psock, int err){ struct sock *sk = psock->sock; sk->sk_err = err; sk->sk_error_report(sk);} static void smap_read_sock_strparser(struct strparser *strp, struct sk_buff *skb){ struct smap_psock *psock; rcu_read_lock(); psock = container_of(strp, struct smap_psock, strp); smap_do_verdict(psock, skb); rcu_read_unlock();} /* Called with lock held on socket */static void smap_data_ready(struct sock *sk){ struct smap_psock *psock; rcu_read_lock(); psock = smap_psock_sk(sk); if (likely(psock)) { write_lock_bh(&sk->sk_callback_lock); strp_data_ready(&psock->strp); write_unlock_bh(&sk->sk_callback_lock); } rcu_read_unlock();} static void smap_tx_work(struct work_struct *w){ struct smap_psock *psock; struct sk_buff *skb; int rem, off, n; psock = container_of(w, struct smap_psock, tx_work); /* lock sock to avoid losing sk_socket at some point during loop */ lock_sock(psock->sock); if (psock->save_skb) { skb = psock->save_skb; rem = psock->save_rem; off = psock->save_off; psock->save_skb = NULL; goto start; } while ((skb = skb_dequeue(&psock->rxqueue))) { rem = skb->len; off = 0;start: do { if (likely(psock->sock->sk_socket)) n = skb_send_sock_locked(psock->sock, skb, off, rem); else n = -EINVAL; if (n <= 0) { if (n == -EAGAIN) { /* Retry when space is available */ psock->save_skb = skb; psock->save_rem = rem; psock->save_off = off; goto out; } /* Hard errors break pipe and stop xmit */ smap_report_sk_error(psock, n ? -n : EPIPE); clear_bit(SMAP_TX_RUNNING, &psock->state); kfree_skb(skb); goto out; } rem -= n; off += n; } while (rem); kfree_skb(skb); }out: release_sock(psock->sock);} static void smap_write_space(struct sock *sk){ struct smap_psock *psock; rcu_read_lock(); psock = smap_psock_sk(sk); if (likely(psock && test_bit(SMAP_TX_RUNNING, &psock->state))) schedule_work(&psock->tx_work); rcu_read_unlock();} static void smap_stop_sock(struct smap_psock *psock, struct sock *sk){ if (!psock->strp_enabled) return; sk->sk_data_ready = psock->save_data_ready; sk->sk_write_space = psock->save_write_space; psock->save_data_ready = NULL; psock->save_write_space = NULL; strp_stop(&psock->strp); psock->strp_enabled = false;} static void smap_destroy_psock(struct rcu_head *rcu){ struct smap_psock *psock = container_of(rcu, struct smap_psock, rcu); /* Now that a grace period has passed there is no longer * any reference to this sock in the sockmap so we can * destroy the psock, strparser, and bpf programs. But, * because we use workqueue sync operations we can not * do it in rcu context */ schedule_work(&psock->gc_work);} static void smap_release_sock(struct smap_psock *psock, struct sock *sock){ psock->refcnt--; if (psock->refcnt) return; tcp_cleanup_ulp(sock); smap_stop_sock(psock, sock); clear_bit(SMAP_TX_RUNNING, &psock->state); rcu_assign_sk_user_data(sock, NULL); call_rcu_sched(&psock->rcu, smap_destroy_psock);} static int smap_parse_func_strparser(struct strparser *strp, struct sk_buff *skb){ struct smap_psock *psock; struct bpf_prog *prog; int rc; rcu_read_lock(); psock = container_of(strp, struct smap_psock, strp); prog = READ_ONCE(psock->bpf_parse); if (unlikely(!prog)) { rcu_read_unlock(); return skb->len; } /* Attach socket for bpf program to use if needed we can do this * because strparser clones the skb before handing it to a upper * layer, meaning skb_orphan has been called. We NULL sk on the * way out to ensure we don't trigger a BUG_ON in skb/sk operations * later and because we are not charging the memory of this skb to * any socket yet. */ skb->sk = psock->sock; bpf_compute_data_pointers(skb); rc = (*prog->bpf_func)(skb, prog->insnsi); skb->sk = NULL; rcu_read_unlock(); return rc;} static int smap_read_sock_done(struct strparser *strp, int err){ return err;} static int smap_init_sock(struct smap_psock *psock, struct sock *sk){ static const struct strp_callbacks cb = { .rcv_msg = smap_read_sock_strparser, .parse_msg = smap_parse_func_strparser, .read_sock_done = smap_read_sock_done, }; return strp_init(&psock->strp, sk, &cb);} static void smap_init_progs(struct smap_psock *psock, struct bpf_stab *stab, struct bpf_prog *verdict, struct bpf_prog *parse){ struct bpf_prog *orig_parse, *orig_verdict; orig_parse = xchg(&psock->bpf_parse, parse); orig_verdict = xchg(&psock->bpf_verdict, verdict); if (orig_verdict) bpf_prog_put(orig_verdict); if (orig_parse) bpf_prog_put(orig_parse);} static void smap_start_sock(struct smap_psock *psock, struct sock *sk){ if (sk->sk_data_ready == smap_data_ready) return; psock->save_data_ready = sk->sk_data_ready; psock->save_write_space = sk->sk_write_space; sk->sk_data_ready = smap_data_ready; sk->sk_write_space = smap_write_space; psock->strp_enabled = true;} static void sock_map_remove_complete(struct bpf_stab *stab){ bpf_map_area_free(stab->sock_map); kfree(stab);} static void smap_gc_work(struct work_struct *w){ struct smap_psock_map_entry *e, *tmp; struct smap_psock *psock; psock = container_of(w, struct smap_psock, gc_work); /* no callback lock needed because we already detached sockmap ops */ if (psock->strp_enabled) strp_done(&psock->strp); cancel_work_sync(&psock->tx_work); __skb_queue_purge(&psock->rxqueue); /* At this point all strparser and xmit work must be complete */ if (psock->bpf_parse) bpf_prog_put(psock->bpf_parse); if (psock->bpf_verdict) bpf_prog_put(psock->bpf_verdict); list_for_each_entry_safe(e, tmp, &psock->maps, list) { list_del(&e->list); kfree(e); } sock_put(psock->sock); kfree(psock);} static struct smap_psock *smap_init_psock(struct sock *sock, struct bpf_stab *stab){Lines 1–500 of 938. Load more
/* Copyright (c) 2017 Covalent IO, Inc. http://covalent.io * * This program is free software; you can redistribute it and/or * modify it under the terms of version 2 of the GNU General Public * License as published by the Free Software Foundation. * * This program is distributed in the hope that it will be useful, but * WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. */ /* A BPF sock_map is used to store sock objects. This is primarly used * for doing socket redirect with BPF helper routines. * * A sock map may have BPF programs attached to it, currently a program * used to parse packets and a program to provide a verdict and redirect * decision on the packet are supported. Any programs attached to a sock * map are inherited by sock objects when they are added to the map. If * no BPF programs are attached the sock object may only be used for sock * redirect. * * A sock object may be in multiple maps, but can only inherit a single * parse or verdict program. If adding a sock object to a map would result * in having multiple parsing programs the update will return an EBUSY error. * * For reference this program is similar to devmap used in XDP context * reviewing these together may be useful. For an example please review * ./samples/bpf/sockmap/. */ #include <linux/bpf.h> #include <net/sock.h> #include <linux/filter.h> #include <linux/errno.h> #include <linux/file.h> #include <linux/kernel.h> #include <linux/net.h> #include <linux/skbuff.h> #include <linux/workqueue.h> #include <linux/list.h> #include <net/strparser.h> #include <net/tcp.h> #define SOCK_CREATE_FLAG_MASK \ (BPF_F_NUMA_NODE | BPF_F_RDONLY | BPF_F_WRONLY) struct bpf_stab { struct bpf_map map; struct sock **sock_map; struct bpf_prog *bpf_parse; struct bpf_prog *bpf_verdict; }; enum smap_psock_state { SMAP_TX_RUNNING, }; struct smap_psock_map_entry { struct list_head list; struct sock **entry; }; struct smap_psock { struct rcu_head rcu; /* refcnt is used inside sk_callback_lock */ u32 refcnt; /* datapath variables */ struct sk_buff_head rxqueue; bool strp_enabled; /* datapath error path cache across tx work invocations */ int save_rem; int save_off; struct sk_buff *save_skb; struct strparser strp; struct bpf_prog *bpf_parse; struct bpf_prog *bpf_verdict; struct list_head maps; /* Back reference used when sock callback trigger sockmap operations */ struct sock *sock; unsigned long state; struct work_struct tx_work; struct work_struct gc_work; struct proto *sk_proto; void (*save_close)(struct sock *sk, long timeout); void (*save_data_ready)(struct sock *sk); void (*save_write_space)(struct sock *sk); }; static inline struct smap_psock *smap_psock_sk(const struct sock *sk) { return rcu_dereference_sk_user_data(sk); } static struct proto tcp_bpf_proto; static int bpf_tcp_init(struct sock *sk) { struct smap_psock *psock; rcu_read_lock(); psock = smap_psock_sk(sk); if (unlikely(!psock)) { rcu_read_unlock(); return -EINVAL; } if (unlikely(psock->sk_proto)) { rcu_read_unlock(); return -EBUSY; } psock->save_close = sk->sk_prot->close; psock->sk_proto = sk->sk_prot; sk->sk_prot = &tcp_bpf_proto; rcu_read_unlock(); return 0; } static void bpf_tcp_release(struct sock *sk) { struct smap_psock *psock; rcu_read_lock(); psock = smap_psock_sk(sk); if (likely(psock)) { sk->sk_prot = psock->sk_proto; psock->sk_proto = NULL; } rcu_read_unlock(); } static void smap_release_sock(struct smap_psock *psock, struct sock *sock); static void bpf_tcp_close(struct sock *sk, long timeout) { void (*close_fun)(struct sock *sk, long timeout); struct smap_psock_map_entry *e, *tmp; struct smap_psock *psock; struct sock *osk; rcu_read_lock(); psock = smap_psock_sk(sk); if (unlikely(!psock)) { rcu_read_unlock(); return sk->sk_prot->close(sk, timeout); } /* The psock may be destroyed anytime after exiting the RCU critial * section so by the time we use close_fun the psock may no longer * be valid. However, bpf_tcp_close is called with the sock lock * held so the close hook and sk are still valid. */ close_fun = psock->save_close; write_lock_bh(&sk->sk_callback_lock); list_for_each_entry_safe(e, tmp, &psock->maps, list) { osk = cmpxchg(e->entry, sk, NULL); if (osk == sk) { list_del(&e->list); smap_release_sock(psock, sk); } } write_unlock_bh(&sk->sk_callback_lock); rcu_read_unlock(); close_fun(sk, timeout); } enum __sk_action { __SK_DROP = 0, __SK_PASS, __SK_REDIRECT, }; static struct tcp_ulp_ops bpf_tcp_ulp_ops __read_mostly = { .name = "bpf_tcp", .uid = TCP_ULP_BPF, .user_visible = false, .owner = NULL, .init = bpf_tcp_init, .release = bpf_tcp_release, }; static int bpf_tcp_ulp_register(void) { tcp_bpf_proto = tcp_prot; tcp_bpf_proto.close = bpf_tcp_close; return tcp_register_ulp(&bpf_tcp_ulp_ops); } static int smap_verdict_func(struct smap_psock *psock, struct sk_buff *skb) { struct bpf_prog *prog = READ_ONCE(psock->bpf_verdict); int rc; if (unlikely(!prog)) return __SK_DROP; skb_orphan(skb); /* We need to ensure that BPF metadata for maps is also cleared * when we orphan the skb so that we don't have the possibility * to reference a stale map. */ TCP_SKB_CB(skb)->bpf.map = NULL; skb->sk = psock->sock; bpf_compute_data_pointers(skb); preempt_disable(); rc = (*prog->bpf_func)(skb, prog->insnsi); preempt_enable(); skb->sk = NULL; /* Moving return codes from UAPI namespace into internal namespace */ return rc == SK_PASS ? (TCP_SKB_CB(skb)->bpf.map ? __SK_REDIRECT : __SK_PASS) : __SK_DROP; } static void smap_do_verdict(struct smap_psock *psock, struct sk_buff *skb) { struct sock *sk; int rc; rc = smap_verdict_func(psock, skb); switch (rc) { case __SK_REDIRECT: sk = do_sk_redirect_map(skb); if (likely(sk)) { struct smap_psock *peer = smap_psock_sk(sk); if (likely(peer && test_bit(SMAP_TX_RUNNING, &peer->state) && !sock_flag(sk, SOCK_DEAD) && sock_writeable(sk))) { skb_set_owner_w(skb, sk); skb_queue_tail(&peer->rxqueue, skb); schedule_work(&peer->tx_work); break; } } /* Fall through and free skb otherwise */ case __SK_DROP: default: kfree_skb(skb); } } static void smap_report_sk_error(struct smap_psock *psock, int err) { struct sock *sk = psock->sock; sk->sk_err = err; sk->sk_error_report(sk); } static void smap_read_sock_strparser(struct strparser *strp, struct sk_buff *skb) { struct smap_psock *psock; rcu_read_lock(); psock = container_of(strp, struct smap_psock, strp); smap_do_verdict(psock, skb); rcu_read_unlock(); } /* Called with lock held on socket */ static void smap_data_ready(struct sock *sk) { struct smap_psock *psock; rcu_read_lock(); psock = smap_psock_sk(sk); if (likely(psock)) { write_lock_bh(&sk->sk_callback_lock); strp_data_ready(&psock->strp); write_unlock_bh(&sk->sk_callback_lock); } rcu_read_unlock(); } static void smap_tx_work(struct work_struct *w) { struct smap_psock *psock; struct sk_buff *skb; int rem, off, n; psock = container_of(w, struct smap_psock, tx_work); /* lock sock to avoid losing sk_socket at some point during loop */ lock_sock(psock->sock); if (psock->save_skb) { skb = psock->save_skb; rem = psock->save_rem; off = psock->save_off; psock->save_skb = NULL; goto start; } while ((skb = skb_dequeue(&psock->rxqueue))) { rem = skb->len; off = 0; start: do { if (likely(psock->sock->sk_socket)) n = skb_send_sock_locked(psock->sock, skb, off, rem); else n = -EINVAL; if (n <= 0) { if (n == -EAGAIN) { /* Retry when space is available */ psock->save_skb = skb; psock->save_rem = rem; psock->save_off = off; goto out; } /* Hard errors break pipe and stop xmit */ smap_report_sk_error(psock, n ? -n : EPIPE); clear_bit(SMAP_TX_RUNNING, &psock->state); kfree_skb(skb); goto out; } rem -= n; off += n; } while (rem); kfree_skb(skb); } out: release_sock(psock->sock); } static void smap_write_space(struct sock *sk) { struct smap_psock *psock; rcu_read_lock(); psock = smap_psock_sk(sk); if (likely(psock && test_bit(SMAP_TX_RUNNING, &psock->state))) schedule_work(&psock->tx_work); rcu_read_unlock(); } static void smap_stop_sock(struct smap_psock *psock, struct sock *sk) { if (!psock->strp_enabled) return; sk->sk_data_ready = psock->save_data_ready; sk->sk_write_space = psock->save_write_space; psock->save_data_ready = NULL; psock->save_write_space = NULL; strp_stop(&psock->strp); psock->strp_enabled = false; } static void smap_destroy_psock(struct rcu_head *rcu) { struct smap_psock *psock = container_of(rcu, struct smap_psock, rcu); /* Now that a grace period has passed there is no longer * any reference to this sock in the sockmap so we can * destroy the psock, strparser, and bpf programs. But, * because we use workqueue sync operations we can not * do it in rcu context */ schedule_work(&psock->gc_work); } static void smap_release_sock(struct smap_psock *psock, struct sock *sock) { psock->refcnt--; if (psock->refcnt) return; tcp_cleanup_ulp(sock); smap_stop_sock(psock, sock); clear_bit(SMAP_TX_RUNNING, &psock->state); rcu_assign_sk_user_data(sock, NULL); call_rcu_sched(&psock->rcu, smap_destroy_psock); } static int smap_parse_func_strparser(struct strparser *strp, struct sk_buff *skb) { struct smap_psock *psock; struct bpf_prog *prog; int rc; rcu_read_lock(); psock = container_of(strp, struct smap_psock, strp); prog = READ_ONCE(psock->bpf_parse); if (unlikely(!prog)) { rcu_read_unlock(); return skb->len; } /* Attach socket for bpf program to use if needed we can do this * because strparser clones the skb before handing it to a upper * layer, meaning skb_orphan has been called. We NULL sk on the * way out to ensure we don't trigger a BUG_ON in skb/sk operations * later and because we are not charging the memory of this skb to * any socket yet. */ skb->sk = psock->sock; bpf_compute_data_pointers(skb); rc = (*prog->bpf_func)(skb, prog->insnsi); skb->sk = NULL; rcu_read_unlock(); return rc; } static int smap_read_sock_done(struct strparser *strp, int err) { return err; } static int smap_init_sock(struct smap_psock *psock, struct sock *sk) { static const struct strp_callbacks cb = { .rcv_msg = smap_read_sock_strparser, .parse_msg = smap_parse_func_strparser, .read_sock_done = smap_read_sock_done, }; return strp_init(&psock->strp, sk, &cb); } static void smap_init_progs(struct smap_psock *psock, struct bpf_stab *stab, struct bpf_prog *verdict, struct bpf_prog *parse) { struct bpf_prog *orig_parse, *orig_verdict; orig_parse = xchg(&psock->bpf_parse, parse); orig_verdict = xchg(&psock->bpf_verdict, verdict); if (orig_verdict) bpf_prog_put(orig_verdict); if (orig_parse) bpf_prog_put(orig_parse); } static void smap_start_sock(struct smap_psock *psock, struct sock *sk) { if (sk->sk_data_ready == smap_data_ready) return; psock->save_data_ready = sk->sk_data_ready; psock->save_write_space = sk->sk_write_space; sk->sk_data_ready = smap_data_ready; sk->sk_write_space = smap_write_space; psock->strp_enabled = true; } static void sock_map_remove_complete(struct bpf_stab *stab) { bpf_map_area_free(stab->sock_map); kfree(stab); } static void smap_gc_work(struct work_struct *w) { struct smap_psock_map_entry *e, *tmp; struct smap_psock *psock; psock = container_of(w, struct smap_psock, gc_work); /* no callback lock needed because we already detached sockmap ops */ if (psock->strp_enabled) strp_done(&psock->strp); cancel_work_sync(&psock->tx_work); __skb_queue_purge(&psock->rxqueue); /* At this point all strparser and xmit work must be complete */ if (psock->bpf_parse) bpf_prog_put(psock->bpf_parse); if (psock->bpf_verdict) bpf_prog_put(psock->bpf_verdict); list_for_each_entry_safe(e, tmp, &psock->maps, list) { list_del(&e->list); kfree(e); } sock_put(psock->sock); kfree(psock); } static struct smap_psock *smap_init_psock(struct sock *sock, struct bpf_stab *stab) {