/* Copyright (c) 2017 Covalent IO, Inc. http://covalent.io * * This program is free software; you can redistribute it and/or * modify it under the terms of version 2 of the GNU General Public * License as published by the Free Software Foundation. * * This program is distributed in the hope that it will be useful, but * WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. */ /* A BPF sock_map is used to store sock objects. This is primarly used * for doing socket redirect with BPF helper routines. * * A sock map may have BPF programs attached to it, currently a program * used to parse packets and a program to provide a verdict and redirect * decision on the packet are supported. Any programs attached to a sock * map are inherited by sock objects when they are added to the map. If * no BPF programs are attached the sock object may only be used for sock * redirect. * * A sock object may be in multiple maps, but can only inherit a single * parse or verdict program. If adding a sock object to a map would result * in having multiple parsing programs the update will return an EBUSY error. * * For reference this program is similar to devmap used in XDP context * reviewing these together may be useful. For an example please review * ./samples/bpf/sockmap/. */#include <linux/bpf.h>#include <net/sock.h>#include <linux/filter.h>#include <linux/errno.h>#include <linux/file.h>#include <linux/kernel.h>#include <linux/net.h>#include <linux/skbuff.h>#include <linux/workqueue.h>#include <linux/list.h>#include <linux/mm.h>#include <net/strparser.h>#include <net/tcp.h>#include <linux/ptr_ring.h>#include <net/inet_common.h>#include <linux/sched/signal.h> #define SOCK_CREATE_FLAG_MASK \ (BPF_F_NUMA_NODE | BPF_F_RDONLY | BPF_F_WRONLY) struct bpf_sock_progs { struct bpf_prog *bpf_tx_msg; struct bpf_prog *bpf_parse; struct bpf_prog *bpf_verdict;}; struct bpf_stab { struct bpf_map map; struct sock **sock_map; struct bpf_sock_progs progs;}; struct bucket { struct hlist_head head; raw_spinlock_t lock;}; struct bpf_htab { struct bpf_map map; struct bucket *buckets; atomic_t count; u32 n_buckets; u32 elem_size; struct bpf_sock_progs progs; struct rcu_head rcu;}; struct htab_elem { struct rcu_head rcu; struct hlist_node hash_node; u32 hash; struct sock *sk; char key[0];}; enum smap_psock_state { SMAP_TX_RUNNING,}; struct smap_psock_map_entry { struct list_head list; struct sock **entry; struct htab_elem __rcu *hash_link; struct bpf_htab __rcu *htab;}; struct smap_psock { struct rcu_head rcu; refcount_t refcnt; /* datapath variables */ struct sk_buff_head rxqueue; bool strp_enabled; /* datapath error path cache across tx work invocations */ int save_rem; int save_off; struct sk_buff *save_skb; /* datapath variables for tx_msg ULP */ struct sock *sk_redir; int apply_bytes; int cork_bytes; int sg_size; int eval; struct sk_msg_buff *cork; struct list_head ingress; struct strparser strp; struct bpf_prog *bpf_tx_msg; struct bpf_prog *bpf_parse; struct bpf_prog *bpf_verdict; struct list_head maps; spinlock_t maps_lock; /* Back reference used when sock callback trigger sockmap operations */ struct sock *sock; unsigned long state; struct work_struct tx_work; struct work_struct gc_work; struct proto *sk_proto; void (*save_close)(struct sock *sk, long timeout); void (*save_data_ready)(struct sock *sk); void (*save_write_space)(struct sock *sk);}; static void smap_release_sock(struct smap_psock *psock, struct sock *sock);static int bpf_tcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int nonblock, int flags, int *addr_len);static int bpf_tcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t size);static int bpf_tcp_sendpage(struct sock *sk, struct page *page, int offset, size_t size, int flags);static void bpf_tcp_close(struct sock *sk, long timeout); static inline struct smap_psock *smap_psock_sk(const struct sock *sk){ return rcu_dereference_sk_user_data(sk);} static bool bpf_tcp_stream_read(const struct sock *sk){ struct smap_psock *psock; bool empty = true; rcu_read_lock(); psock = smap_psock_sk(sk); if (unlikely(!psock)) goto out; empty = list_empty(&psock->ingress);out: rcu_read_unlock(); return !empty;} enum { SOCKMAP_IPV4, SOCKMAP_IPV6, SOCKMAP_NUM_PROTS,}; enum { SOCKMAP_BASE, SOCKMAP_TX, SOCKMAP_NUM_CONFIGS,}; static struct proto *saved_tcpv6_prot __read_mostly;static DEFINE_SPINLOCK(tcpv6_prot_lock);static struct proto bpf_tcp_prots[SOCKMAP_NUM_PROTS][SOCKMAP_NUM_CONFIGS];static void build_protos(struct proto prot[SOCKMAP_NUM_CONFIGS], struct proto *base){ prot[SOCKMAP_BASE] = *base; prot[SOCKMAP_BASE].close = bpf_tcp_close; prot[SOCKMAP_BASE].recvmsg = bpf_tcp_recvmsg; prot[SOCKMAP_BASE].stream_memory_read = bpf_tcp_stream_read; prot[SOCKMAP_TX] = prot[SOCKMAP_BASE]; prot[SOCKMAP_TX].sendmsg = bpf_tcp_sendmsg; prot[SOCKMAP_TX].sendpage = bpf_tcp_sendpage;} static void update_sk_prot(struct sock *sk, struct smap_psock *psock){ int family = sk->sk_family == AF_INET6 ? SOCKMAP_IPV6 : SOCKMAP_IPV4; int conf = psock->bpf_tx_msg ? SOCKMAP_TX : SOCKMAP_BASE; sk->sk_prot = &bpf_tcp_prots[family][conf];} static int bpf_tcp_init(struct sock *sk){ struct smap_psock *psock; rcu_read_lock(); psock = smap_psock_sk(sk); if (unlikely(!psock)) { rcu_read_unlock(); return -EINVAL; } if (unlikely(psock->sk_proto)) { rcu_read_unlock(); return -EBUSY; } psock->save_close = sk->sk_prot->close; psock->sk_proto = sk->sk_prot; /* Build IPv6 sockmap whenever the address of tcpv6_prot changes */ if (sk->sk_family == AF_INET6 && unlikely(sk->sk_prot != smp_load_acquire(&saved_tcpv6_prot))) { spin_lock_bh(&tcpv6_prot_lock); if (likely(sk->sk_prot != saved_tcpv6_prot)) { build_protos(bpf_tcp_prots[SOCKMAP_IPV6], sk->sk_prot); smp_store_release(&saved_tcpv6_prot, sk->sk_prot); } spin_unlock_bh(&tcpv6_prot_lock); } update_sk_prot(sk, psock); rcu_read_unlock(); return 0;} static void smap_release_sock(struct smap_psock *psock, struct sock *sock);static int free_start_sg(struct sock *sk, struct sk_msg_buff *md); static void bpf_tcp_release(struct sock *sk){ struct smap_psock *psock; rcu_read_lock(); psock = smap_psock_sk(sk); if (unlikely(!psock)) goto out; if (psock->cork) { free_start_sg(psock->sock, psock->cork); kfree(psock->cork); psock->cork = NULL; } if (psock->sk_proto) { sk->sk_prot = psock->sk_proto; psock->sk_proto = NULL; }out: rcu_read_unlock();} static struct htab_elem *lookup_elem_raw(struct hlist_head *head, u32 hash, void *key, u32 key_size){ struct htab_elem *l; hlist_for_each_entry_rcu(l, head, hash_node) { if (l->hash == hash && !memcmp(&l->key, key, key_size)) return l; } return NULL;} static inline struct bucket *__select_bucket(struct bpf_htab *htab, u32 hash){ return &htab->buckets[hash & (htab->n_buckets - 1)];} static inline struct hlist_head *select_bucket(struct bpf_htab *htab, u32 hash){ return &__select_bucket(htab, hash)->head;} static void free_htab_elem(struct bpf_htab *htab, struct htab_elem *l){ atomic_dec(&htab->count); kfree_rcu(l, rcu);} static struct smap_psock_map_entry *psock_map_pop(struct sock *sk, struct smap_psock *psock){ struct smap_psock_map_entry *e; spin_lock_bh(&psock->maps_lock); e = list_first_entry_or_null(&psock->maps, struct smap_psock_map_entry, list); if (e) list_del(&e->list); spin_unlock_bh(&psock->maps_lock); return e;} static void bpf_tcp_close(struct sock *sk, long timeout){ void (*close_fun)(struct sock *sk, long timeout); struct smap_psock_map_entry *e; struct sk_msg_buff *md, *mtmp; struct smap_psock *psock; struct sock *osk; lock_sock(sk); rcu_read_lock(); psock = smap_psock_sk(sk); if (unlikely(!psock)) { rcu_read_unlock(); release_sock(sk); return sk->sk_prot->close(sk, timeout); } /* The psock may be destroyed anytime after exiting the RCU critial * section so by the time we use close_fun the psock may no longer * be valid. However, bpf_tcp_close is called with the sock lock * held so the close hook and sk are still valid. */ close_fun = psock->save_close; if (psock->cork) { free_start_sg(psock->sock, psock->cork); kfree(psock->cork); psock->cork = NULL; } list_for_each_entry_safe(md, mtmp, &psock->ingress, list) { list_del(&md->list); free_start_sg(psock->sock, md); kfree(md); } e = psock_map_pop(sk, psock); while (e) { if (e->entry) { osk = cmpxchg(e->entry, sk, NULL); if (osk == sk) { smap_release_sock(psock, sk); } } else { struct htab_elem *link = rcu_dereference(e->hash_link); struct bpf_htab *htab = rcu_dereference(e->htab); struct hlist_head *head; struct htab_elem *l; struct bucket *b; b = __select_bucket(htab, link->hash); head = &b->head; raw_spin_lock_bh(&b->lock); l = lookup_elem_raw(head, link->hash, link->key, htab->map.key_size); /* If another thread deleted this object skip deletion. * The refcnt on psock may or may not be zero. */ if (l) { hlist_del_rcu(&link->hash_node); smap_release_sock(psock, link->sk); free_htab_elem(htab, link); } raw_spin_unlock_bh(&b->lock); } e = psock_map_pop(sk, psock); } rcu_read_unlock(); release_sock(sk); close_fun(sk, timeout);} enum __sk_action { __SK_DROP = 0, __SK_PASS, __SK_REDIRECT, __SK_NONE,}; static struct tcp_ulp_ops bpf_tcp_ulp_ops __read_mostly = { .name = "bpf_tcp", .uid = TCP_ULP_BPF, .user_visible = false, .owner = NULL, .init = bpf_tcp_init, .release = bpf_tcp_release,}; static int memcopy_from_iter(struct sock *sk, struct sk_msg_buff *md, struct iov_iter *from, int bytes){ struct scatterlist *sg = md->sg_data; int i = md->sg_curr, rc = -ENOSPC; do { int copy; char *to; if (md->sg_copybreak >= sg[i].length) { md->sg_copybreak = 0; if (++i == MAX_SKB_FRAGS) i = 0; if (i == md->sg_end) break; } copy = sg[i].length - md->sg_copybreak; to = sg_virt(&sg[i]) + md->sg_copybreak; md->sg_copybreak += copy; if (sk->sk_route_caps & NETIF_F_NOCACHE_COPY) rc = copy_from_iter_nocache(to, copy, from); else rc = copy_from_iter(to, copy, from); if (rc != copy) { rc = -EFAULT; goto out; } bytes -= copy; if (!bytes) break; md->sg_copybreak = 0; if (++i == MAX_SKB_FRAGS) i = 0; } while (i != md->sg_end);out: md->sg_curr = i; return rc;} static int bpf_tcp_push(struct sock *sk, int apply_bytes, struct sk_msg_buff *md, int flags, bool uncharge){ bool apply = apply_bytes; struct scatterlist *sg; int offset, ret = 0; struct page *p; size_t size; while (1) { sg = md->sg_data + md->sg_start; size = (apply && apply_bytes < sg->length) ? apply_bytes : sg->length; offset = sg->offset; tcp_rate_check_app_limited(sk); p = sg_page(sg);retry: ret = do_tcp_sendpages(sk, p, offset, size, flags); if (ret != size) { if (ret > 0) { if (apply) apply_bytes -= ret; sg->offset += ret; sg->length -= ret; size -= ret; offset += ret; if (uncharge) sk_mem_uncharge(sk, ret); goto retry; } return ret; } if (apply) apply_bytes -= ret; sg->offset += ret; sg->length -= ret; if (uncharge) sk_mem_uncharge(sk, ret); if (!sg->length) { put_page(p); md->sg_start++; if (md->sg_start == MAX_SKB_FRAGS) md->sg_start = 0; sg_init_table(sg, 1); if (md->sg_start == md->sg_end) break; } if (apply && !apply_bytes) break;Lines 1–500 of 2550. Load more
/* Copyright (c) 2017 Covalent IO, Inc. http://covalent.io * * This program is free software; you can redistribute it and/or * modify it under the terms of version 2 of the GNU General Public * License as published by the Free Software Foundation. * * This program is distributed in the hope that it will be useful, but * WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. */ /* A BPF sock_map is used to store sock objects. This is primarly used * for doing socket redirect with BPF helper routines. * * A sock map may have BPF programs attached to it, currently a program * used to parse packets and a program to provide a verdict and redirect * decision on the packet are supported. Any programs attached to a sock * map are inherited by sock objects when they are added to the map. If * no BPF programs are attached the sock object may only be used for sock * redirect. * * A sock object may be in multiple maps, but can only inherit a single * parse or verdict program. If adding a sock object to a map would result * in having multiple parsing programs the update will return an EBUSY error. * * For reference this program is similar to devmap used in XDP context * reviewing these together may be useful. For an example please review * ./samples/bpf/sockmap/. */ #include <linux/bpf.h> #include <net/sock.h> #include <linux/filter.h> #include <linux/errno.h> #include <linux/file.h> #include <linux/kernel.h> #include <linux/net.h> #include <linux/skbuff.h> #include <linux/workqueue.h> #include <linux/list.h> #include <linux/mm.h> #include <net/strparser.h> #include <net/tcp.h> #include <linux/ptr_ring.h> #include <net/inet_common.h> #include <linux/sched/signal.h> #define SOCK_CREATE_FLAG_MASK \ (BPF_F_NUMA_NODE | BPF_F_RDONLY | BPF_F_WRONLY) struct bpf_sock_progs { struct bpf_prog *bpf_tx_msg; struct bpf_prog *bpf_parse; struct bpf_prog *bpf_verdict; }; struct bpf_stab { struct bpf_map map; struct sock **sock_map; struct bpf_sock_progs progs; }; struct bucket { struct hlist_head head; raw_spinlock_t lock; }; struct bpf_htab { struct bpf_map map; struct bucket *buckets; atomic_t count; u32 n_buckets; u32 elem_size; struct bpf_sock_progs progs; struct rcu_head rcu; }; struct htab_elem { struct rcu_head rcu; struct hlist_node hash_node; u32 hash; struct sock *sk; char key[0]; }; enum smap_psock_state { SMAP_TX_RUNNING, }; struct smap_psock_map_entry { struct list_head list; struct sock **entry; struct htab_elem __rcu *hash_link; struct bpf_htab __rcu *htab; }; struct smap_psock { struct rcu_head rcu; refcount_t refcnt; /* datapath variables */ struct sk_buff_head rxqueue; bool strp_enabled; /* datapath error path cache across tx work invocations */ int save_rem; int save_off; struct sk_buff *save_skb; /* datapath variables for tx_msg ULP */ struct sock *sk_redir; int apply_bytes; int cork_bytes; int sg_size; int eval; struct sk_msg_buff *cork; struct list_head ingress; struct strparser strp; struct bpf_prog *bpf_tx_msg; struct bpf_prog *bpf_parse; struct bpf_prog *bpf_verdict; struct list_head maps; spinlock_t maps_lock; /* Back reference used when sock callback trigger sockmap operations */ struct sock *sock; unsigned long state; struct work_struct tx_work; struct work_struct gc_work; struct proto *sk_proto; void (*save_close)(struct sock *sk, long timeout); void (*save_data_ready)(struct sock *sk); void (*save_write_space)(struct sock *sk); }; static void smap_release_sock(struct smap_psock *psock, struct sock *sock); static int bpf_tcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int nonblock, int flags, int *addr_len); static int bpf_tcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t size); static int bpf_tcp_sendpage(struct sock *sk, struct page *page, int offset, size_t size, int flags); static void bpf_tcp_close(struct sock *sk, long timeout); static inline struct smap_psock *smap_psock_sk(const struct sock *sk) { return rcu_dereference_sk_user_data(sk); } static bool bpf_tcp_stream_read(const struct sock *sk) { struct smap_psock *psock; bool empty = true; rcu_read_lock(); psock = smap_psock_sk(sk); if (unlikely(!psock)) goto out; empty = list_empty(&psock->ingress); out: rcu_read_unlock(); return !empty; } enum { SOCKMAP_IPV4, SOCKMAP_IPV6, SOCKMAP_NUM_PROTS, }; enum { SOCKMAP_BASE, SOCKMAP_TX, SOCKMAP_NUM_CONFIGS, }; static struct proto *saved_tcpv6_prot __read_mostly; static DEFINE_SPINLOCK(tcpv6_prot_lock); static struct proto bpf_tcp_prots[SOCKMAP_NUM_PROTS][SOCKMAP_NUM_CONFIGS]; static void build_protos(struct proto prot[SOCKMAP_NUM_CONFIGS], struct proto *base) { prot[SOCKMAP_BASE] = *base; prot[SOCKMAP_BASE].close = bpf_tcp_close; prot[SOCKMAP_BASE].recvmsg = bpf_tcp_recvmsg; prot[SOCKMAP_BASE].stream_memory_read = bpf_tcp_stream_read; prot[SOCKMAP_TX] = prot[SOCKMAP_BASE]; prot[SOCKMAP_TX].sendmsg = bpf_tcp_sendmsg; prot[SOCKMAP_TX].sendpage = bpf_tcp_sendpage; } static void update_sk_prot(struct sock *sk, struct smap_psock *psock) { int family = sk->sk_family == AF_INET6 ? SOCKMAP_IPV6 : SOCKMAP_IPV4; int conf = psock->bpf_tx_msg ? SOCKMAP_TX : SOCKMAP_BASE; sk->sk_prot = &bpf_tcp_prots[family][conf]; } static int bpf_tcp_init(struct sock *sk) { struct smap_psock *psock; rcu_read_lock(); psock = smap_psock_sk(sk); if (unlikely(!psock)) { rcu_read_unlock(); return -EINVAL; } if (unlikely(psock->sk_proto)) { rcu_read_unlock(); return -EBUSY; } psock->save_close = sk->sk_prot->close; psock->sk_proto = sk->sk_prot; /* Build IPv6 sockmap whenever the address of tcpv6_prot changes */ if (sk->sk_family == AF_INET6 && unlikely(sk->sk_prot != smp_load_acquire(&saved_tcpv6_prot))) { spin_lock_bh(&tcpv6_prot_lock); if (likely(sk->sk_prot != saved_tcpv6_prot)) { build_protos(bpf_tcp_prots[SOCKMAP_IPV6], sk->sk_prot); smp_store_release(&saved_tcpv6_prot, sk->sk_prot); } spin_unlock_bh(&tcpv6_prot_lock); } update_sk_prot(sk, psock); rcu_read_unlock(); return 0; } static void smap_release_sock(struct smap_psock *psock, struct sock *sock); static int free_start_sg(struct sock *sk, struct sk_msg_buff *md); static void bpf_tcp_release(struct sock *sk) { struct smap_psock *psock; rcu_read_lock(); psock = smap_psock_sk(sk); if (unlikely(!psock)) goto out; if (psock->cork) { free_start_sg(psock->sock, psock->cork); kfree(psock->cork); psock->cork = NULL; } if (psock->sk_proto) { sk->sk_prot = psock->sk_proto; psock->sk_proto = NULL; } out: rcu_read_unlock(); } static struct htab_elem *lookup_elem_raw(struct hlist_head *head, u32 hash, void *key, u32 key_size) { struct htab_elem *l; hlist_for_each_entry_rcu(l, head, hash_node) { if (l->hash == hash && !memcmp(&l->key, key, key_size)) return l; } return NULL; } static inline struct bucket *__select_bucket(struct bpf_htab *htab, u32 hash) { return &htab->buckets[hash & (htab->n_buckets - 1)]; } static inline struct hlist_head *select_bucket(struct bpf_htab *htab, u32 hash) { return &__select_bucket(htab, hash)->head; } static void free_htab_elem(struct bpf_htab *htab, struct htab_elem *l) { atomic_dec(&htab->count); kfree_rcu(l, rcu); } static struct smap_psock_map_entry *psock_map_pop(struct sock *sk, struct smap_psock *psock) { struct smap_psock_map_entry *e; spin_lock_bh(&psock->maps_lock); e = list_first_entry_or_null(&psock->maps, struct smap_psock_map_entry, list); if (e) list_del(&e->list); spin_unlock_bh(&psock->maps_lock); return e; } static void bpf_tcp_close(struct sock *sk, long timeout) { void (*close_fun)(struct sock *sk, long timeout); struct smap_psock_map_entry *e; struct sk_msg_buff *md, *mtmp; struct smap_psock *psock; struct sock *osk; lock_sock(sk); rcu_read_lock(); psock = smap_psock_sk(sk); if (unlikely(!psock)) { rcu_read_unlock(); release_sock(sk); return sk->sk_prot->close(sk, timeout); } /* The psock may be destroyed anytime after exiting the RCU critial * section so by the time we use close_fun the psock may no longer * be valid. However, bpf_tcp_close is called with the sock lock * held so the close hook and sk are still valid. */ close_fun = psock->save_close; if (psock->cork) { free_start_sg(psock->sock, psock->cork); kfree(psock->cork); psock->cork = NULL; } list_for_each_entry_safe(md, mtmp, &psock->ingress, list) { list_del(&md->list); free_start_sg(psock->sock, md); kfree(md); } e = psock_map_pop(sk, psock); while (e) { if (e->entry) { osk = cmpxchg(e->entry, sk, NULL); if (osk == sk) { smap_release_sock(psock, sk); } } else { struct htab_elem *link = rcu_dereference(e->hash_link); struct bpf_htab *htab = rcu_dereference(e->htab); struct hlist_head *head; struct htab_elem *l; struct bucket *b; b = __select_bucket(htab, link->hash); head = &b->head; raw_spin_lock_bh(&b->lock); l = lookup_elem_raw(head, link->hash, link->key, htab->map.key_size); /* If another thread deleted this object skip deletion. * The refcnt on psock may or may not be zero. */ if (l) { hlist_del_rcu(&link->hash_node); smap_release_sock(psock, link->sk); free_htab_elem(htab, link); } raw_spin_unlock_bh(&b->lock); } e = psock_map_pop(sk, psock); } rcu_read_unlock(); release_sock(sk); close_fun(sk, timeout); } enum __sk_action { __SK_DROP = 0, __SK_PASS, __SK_REDIRECT, __SK_NONE, }; static struct tcp_ulp_ops bpf_tcp_ulp_ops __read_mostly = { .name = "bpf_tcp", .uid = TCP_ULP_BPF, .user_visible = false, .owner = NULL, .init = bpf_tcp_init, .release = bpf_tcp_release, }; static int memcopy_from_iter(struct sock *sk, struct sk_msg_buff *md, struct iov_iter *from, int bytes) { struct scatterlist *sg = md->sg_data; int i = md->sg_curr, rc = -ENOSPC; do { int copy; char *to; if (md->sg_copybreak >= sg[i].length) { md->sg_copybreak = 0; if (++i == MAX_SKB_FRAGS) i = 0; if (i == md->sg_end) break; } copy = sg[i].length - md->sg_copybreak; to = sg_virt(&sg[i]) + md->sg_copybreak; md->sg_copybreak += copy; if (sk->sk_route_caps & NETIF_F_NOCACHE_COPY) rc = copy_from_iter_nocache(to, copy, from); else rc = copy_from_iter(to, copy, from); if (rc != copy) { rc = -EFAULT; goto out; } bytes -= copy; if (!bytes) break; md->sg_copybreak = 0; if (++i == MAX_SKB_FRAGS) i = 0; } while (i != md->sg_end); out: md->sg_curr = i; return rc; } static int bpf_tcp_push(struct sock *sk, int apply_bytes, struct sk_msg_buff *md, int flags, bool uncharge) { bool apply = apply_bytes; struct scatterlist *sg; int offset, ret = 0; struct page *p; size_t size; while (1) { sg = md->sg_data + md->sg_start; size = (apply && apply_bytes < sg->length) ? apply_bytes : sg->length; offset = sg->offset; tcp_rate_check_app_limited(sk); p = sg_page(sg); retry: ret = do_tcp_sendpages(sk, p, offset, size, flags); if (ret != size) { if (ret > 0) { if (apply) apply_bytes -= ret; sg->offset += ret; sg->length -= ret; size -= ret; offset += ret; if (uncharge) sk_mem_uncharge(sk, ret); goto retry; } return ret; } if (apply) apply_bytes -= ret; sg->offset += ret; sg->length -= ret; if (uncharge) sk_mem_uncharge(sk, ret); if (!sg->length) { put_page(p); md->sg_start++; if (md->sg_start == MAX_SKB_FRAGS) md->sg_start = 0; sg_init_table(sg, 1); if (md->sg_start == md->sg_end) break; } if (apply && !apply_bytes) break;