1924 lines · 44.4 KB · C
RawDownload
/* Copyright (c) 2017 Covalent IO, Inc. http://covalent.io * * This program is free software; you can redistribute it and/or * modify it under the terms of version 2 of the GNU General Public * License as published by the Free Software Foundation. * * This program is distributed in the hope that it will be useful, but * WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. */ /* A BPF sock_map is used to store sock objects. This is primarly used * for doing socket redirect with BPF helper routines. * * A sock map may have BPF programs attached to it, currently a program * used to parse packets and a program to provide a verdict and redirect * decision on the packet are supported. Any programs attached to a sock * map are inherited by sock objects when they are added to the map. If * no BPF programs are attached the sock object may only be used for sock * redirect. * * A sock object may be in multiple maps, but can only inherit a single * parse or verdict program. If adding a sock object to a map would result * in having multiple parsing programs the update will return an EBUSY error. * * For reference this program is similar to devmap used in XDP context * reviewing these together may be useful. For an example please review * ./samples/bpf/sockmap/. */#include <linux/bpf.h>#include <net/sock.h>#include <linux/filter.h>#include <linux/errno.h>#include <linux/file.h>#include <linux/kernel.h>#include <linux/net.h>#include <linux/skbuff.h>#include <linux/workqueue.h>#include <linux/list.h>#include <linux/mm.h>#include <net/strparser.h>#include <net/tcp.h>#include <linux/ptr_ring.h>#include <net/inet_common.h>#include <linux/sched/signal.h> #define SOCK_CREATE_FLAG_MASK \	(BPF_F_NUMA_NODE | BPF_F_RDONLY | BPF_F_WRONLY) struct bpf_stab {	struct bpf_map map;	struct sock **sock_map;	struct bpf_prog *bpf_tx_msg;	struct bpf_prog *bpf_parse;	struct bpf_prog *bpf_verdict;}; enum smap_psock_state {	SMAP_TX_RUNNING,}; struct smap_psock_map_entry {	struct list_head list;	struct sock **entry;}; struct smap_psock {	struct rcu_head	rcu;	refcount_t refcnt; 	/* datapath variables */	struct sk_buff_head rxqueue;	bool strp_enabled; 	/* datapath error path cache across tx work invocations */	int save_rem;	int save_off;	struct sk_buff *save_skb; 	/* datapath variables for tx_msg ULP */	struct sock *sk_redir;	int apply_bytes;	int cork_bytes;	int sg_size;	int eval;	struct sk_msg_buff *cork;	struct list_head ingress; 	struct strparser strp;	struct bpf_prog *bpf_tx_msg;	struct bpf_prog *bpf_parse;	struct bpf_prog *bpf_verdict;	struct list_head maps; 	/* Back reference used when sock callback trigger sockmap operations */	struct sock *sock;	unsigned long state; 	struct work_struct tx_work;	struct work_struct gc_work; 	struct proto *sk_proto;	void (*save_close)(struct sock *sk, long timeout);	void (*save_data_ready)(struct sock *sk);	void (*save_write_space)(struct sock *sk);}; static void smap_release_sock(struct smap_psock *psock, struct sock *sock);static int bpf_tcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len,			   int nonblock, int flags, int *addr_len);static int bpf_tcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t size);static int bpf_tcp_sendpage(struct sock *sk, struct page *page,			    int offset, size_t size, int flags); static inline struct smap_psock *smap_psock_sk(const struct sock *sk){	return rcu_dereference_sk_user_data(sk);} static bool bpf_tcp_stream_read(const struct sock *sk){	struct smap_psock *psock;	bool empty = true; 	rcu_read_lock();	psock = smap_psock_sk(sk);	if (unlikely(!psock))		goto out;	empty = list_empty(&psock->ingress);out:	rcu_read_unlock();	return !empty;} static struct proto tcp_bpf_proto;static int bpf_tcp_init(struct sock *sk){	struct smap_psock *psock; 	rcu_read_lock();	psock = smap_psock_sk(sk);	if (unlikely(!psock)) {		rcu_read_unlock();		return -EINVAL;	} 	if (unlikely(psock->sk_proto)) {		rcu_read_unlock();		return -EBUSY;	} 	psock->save_close = sk->sk_prot->close;	psock->sk_proto = sk->sk_prot; 	if (psock->bpf_tx_msg) {		tcp_bpf_proto.sendmsg = bpf_tcp_sendmsg;		tcp_bpf_proto.sendpage = bpf_tcp_sendpage;		tcp_bpf_proto.recvmsg = bpf_tcp_recvmsg;		tcp_bpf_proto.stream_memory_read = bpf_tcp_stream_read;	} 	sk->sk_prot = &tcp_bpf_proto;	rcu_read_unlock();	return 0;} static void smap_release_sock(struct smap_psock *psock, struct sock *sock);static int free_start_sg(struct sock *sk, struct sk_msg_buff *md); static void bpf_tcp_release(struct sock *sk){	struct smap_psock *psock; 	rcu_read_lock();	psock = smap_psock_sk(sk);	if (unlikely(!psock))		goto out; 	if (psock->cork) {		free_start_sg(psock->sock, psock->cork);		kfree(psock->cork);		psock->cork = NULL;	} 	if (psock->sk_proto) {		sk->sk_prot = psock->sk_proto;		psock->sk_proto = NULL;	}out:	rcu_read_unlock();} static void bpf_tcp_close(struct sock *sk, long timeout){	void (*close_fun)(struct sock *sk, long timeout);	struct smap_psock_map_entry *e, *tmp;	struct sk_msg_buff *md, *mtmp;	struct smap_psock *psock;	struct sock *osk; 	rcu_read_lock();	psock = smap_psock_sk(sk);	if (unlikely(!psock)) {		rcu_read_unlock();		return sk->sk_prot->close(sk, timeout);	} 	/* The psock may be destroyed anytime after exiting the RCU critial	 * section so by the time we use close_fun the psock may no longer	 * be valid. However, bpf_tcp_close is called with the sock lock	 * held so the close hook and sk are still valid.	 */	close_fun = psock->save_close; 	write_lock_bh(&sk->sk_callback_lock);	if (psock->cork) {		free_start_sg(psock->sock, psock->cork);		kfree(psock->cork);		psock->cork = NULL;	} 	list_for_each_entry_safe(md, mtmp, &psock->ingress, list) {		list_del(&md->list);		free_start_sg(psock->sock, md);		kfree(md);	} 	list_for_each_entry_safe(e, tmp, &psock->maps, list) {		osk = cmpxchg(e->entry, sk, NULL);		if (osk == sk) {			list_del(&e->list);			smap_release_sock(psock, sk);		}	}	write_unlock_bh(&sk->sk_callback_lock);	rcu_read_unlock();	close_fun(sk, timeout);} enum __sk_action {	__SK_DROP = 0,	__SK_PASS,	__SK_REDIRECT,	__SK_NONE,}; static struct tcp_ulp_ops bpf_tcp_ulp_ops __read_mostly = {	.name		= "bpf_tcp",	.uid		= TCP_ULP_BPF,	.user_visible	= false,	.owner		= NULL,	.init		= bpf_tcp_init,	.release	= bpf_tcp_release,}; static int memcopy_from_iter(struct sock *sk,			     struct sk_msg_buff *md,			     struct iov_iter *from, int bytes){	struct scatterlist *sg = md->sg_data;	int i = md->sg_curr, rc = -ENOSPC; 	do {		int copy;		char *to; 		if (md->sg_copybreak >= sg[i].length) {			md->sg_copybreak = 0; 			if (++i == MAX_SKB_FRAGS)				i = 0; 			if (i == md->sg_end)				break;		} 		copy = sg[i].length - md->sg_copybreak;		to = sg_virt(&sg[i]) + md->sg_copybreak;		md->sg_copybreak += copy; 		if (sk->sk_route_caps & NETIF_F_NOCACHE_COPY)			rc = copy_from_iter_nocache(to, copy, from);		else			rc = copy_from_iter(to, copy, from); 		if (rc != copy) {			rc = -EFAULT;			goto out;		} 		bytes -= copy;		if (!bytes)			break; 		md->sg_copybreak = 0;		if (++i == MAX_SKB_FRAGS)			i = 0;	} while (i != md->sg_end);out:	md->sg_curr = i;	return rc;} static int bpf_tcp_push(struct sock *sk, int apply_bytes,			struct sk_msg_buff *md,			int flags, bool uncharge){	bool apply = apply_bytes;	struct scatterlist *sg;	int offset, ret = 0;	struct page *p;	size_t size; 	while (1) {		sg = md->sg_data + md->sg_start;		size = (apply && apply_bytes < sg->length) ?			apply_bytes : sg->length;		offset = sg->offset; 		tcp_rate_check_app_limited(sk);		p = sg_page(sg);retry:		ret = do_tcp_sendpages(sk, p, offset, size, flags);		if (ret != size) {			if (ret > 0) {				if (apply)					apply_bytes -= ret; 				sg->offset += ret;				sg->length -= ret;				size -= ret;				offset += ret;				if (uncharge)					sk_mem_uncharge(sk, ret);				goto retry;			} 			return ret;		} 		if (apply)			apply_bytes -= ret;		sg->offset += ret;		sg->length -= ret;		if (uncharge)			sk_mem_uncharge(sk, ret); 		if (!sg->length) {			put_page(p);			md->sg_start++;			if (md->sg_start == MAX_SKB_FRAGS)				md->sg_start = 0;			sg_init_table(sg, 1); 			if (md->sg_start == md->sg_end)				break;		} 		if (apply && !apply_bytes)			break;	}	return 0;} static inline void bpf_compute_data_pointers_sg(struct sk_msg_buff *md){	struct scatterlist *sg = md->sg_data + md->sg_start; 	if (md->sg_copy[md->sg_start]) {		md->data = md->data_end = 0;	} else {		md->data = sg_virt(sg);		md->data_end = md->data + sg->length;	}} static void return_mem_sg(struct sock *sk, int bytes, struct sk_msg_buff *md){	struct scatterlist *sg = md->sg_data;	int i = md->sg_start; 	do {		int uncharge = (bytes < sg[i].length) ? bytes : sg[i].length; 		sk_mem_uncharge(sk, uncharge);		bytes -= uncharge;		if (!bytes)			break;		i++;		if (i == MAX_SKB_FRAGS)			i = 0;	} while (i != md->sg_end);} static void free_bytes_sg(struct sock *sk, int bytes,			  struct sk_msg_buff *md, bool charge){	struct scatterlist *sg = md->sg_data;	int i = md->sg_start, free; 	while (bytes && sg[i].length) {		free = sg[i].length;		if (bytes < free) {			sg[i].length -= bytes;			sg[i].offset += bytes;			if (charge)				sk_mem_uncharge(sk, bytes);			break;		} 		if (charge)			sk_mem_uncharge(sk, sg[i].length);		put_page(sg_page(&sg[i]));		bytes -= sg[i].length;		sg[i].length = 0;		sg[i].page_link = 0;		sg[i].offset = 0;		i++; 		if (i == MAX_SKB_FRAGS)			i = 0;	}	md->sg_start = i;} static int free_sg(struct sock *sk, int start, struct sk_msg_buff *md){	struct scatterlist *sg = md->sg_data;	int i = start, free = 0; 	while (sg[i].length) {		free += sg[i].length;		sk_mem_uncharge(sk, sg[i].length);		put_page(sg_page(&sg[i]));		sg[i].length = 0;		sg[i].page_link = 0;		sg[i].offset = 0;		i++; 		if (i == MAX_SKB_FRAGS)			i = 0;	} 	return free;} static int free_start_sg(struct sock *sk, struct sk_msg_buff *md){	int free = free_sg(sk, md->sg_start, md); 	md->sg_start = md->sg_end;	return free;} static int free_curr_sg(struct sock *sk, struct sk_msg_buff *md){	return free_sg(sk, md->sg_curr, md);} static int bpf_map_msg_verdict(int _rc, struct sk_msg_buff *md){	return ((_rc == SK_PASS) ?	       (md->map ? __SK_REDIRECT : __SK_PASS) :	       __SK_DROP);} static unsigned int smap_do_tx_msg(struct sock *sk,				   struct smap_psock *psock,				   struct sk_msg_buff *md){	struct bpf_prog *prog;	unsigned int rc, _rc; 	preempt_disable();	rcu_read_lock(); 	/* If the policy was removed mid-send then default to 'accept' */	prog = READ_ONCE(psock->bpf_tx_msg);	if (unlikely(!prog)) {		_rc = SK_PASS;		goto verdict;	} 	bpf_compute_data_pointers_sg(md);	rc = (*prog->bpf_func)(md, prog->insnsi);	psock->apply_bytes = md->apply_bytes; 	/* Moving return codes from UAPI namespace into internal namespace */	_rc = bpf_map_msg_verdict(rc, md); 	/* The psock has a refcount on the sock but not on the map and because	 * we need to drop rcu read lock here its possible the map could be	 * removed between here and when we need it to execute the sock	 * redirect. So do the map lookup now for future use.	 */	if (_rc == __SK_REDIRECT) {		if (psock->sk_redir)			sock_put(psock->sk_redir);		psock->sk_redir = do_msg_redirect_map(md);

Lines 1–500 of 1924.