// SPDX-License-Identifier: GPL-2.0-only/* Copyright (c) 2017 Facebook */#include <linux/bpf.h>#include <linux/btf.h>#include <linux/btf_ids.h>#include <linux/slab.h>#include <linux/init.h>#include <linux/vmalloc.h>#include <linux/etherdevice.h>#include <linux/filter.h>#include <linux/rcupdate_trace.h>#include <linux/sched/signal.h>#include <net/bpf_sk_storage.h>#include <net/hotdata.h>#include <net/sock.h>#include <net/tcp.h>#include <net/net_namespace.h>#include <net/page_pool/helpers.h>#include <linux/error-injection.h>#include <linux/smp.h>#include <linux/sock_diag.h>#include <linux/netfilter.h>#include <net/netdev_rx_queue.h>#include <net/xdp.h>#include <net/netfilter/nf_bpf_link.h> #define CREATE_TRACE_POINTS#include <trace/events/bpf_test_run.h> struct bpf_test_timer {	u32 i;	u64 time_start, time_spent;}; static void bpf_test_timer_enter(struct bpf_test_timer *t)	__acquires(rcu){	rcu_read_lock_dont_migrate();	t->time_start = ktime_get_ns();} static void bpf_test_timer_leave(struct bpf_test_timer *t)	__releases(rcu){	t->time_start = 0;	rcu_read_unlock_migrate();} static bool bpf_test_timer_continue(struct bpf_test_timer *t, int iterations,				    u32 repeat, int *err, u32 *duration)	__must_hold(rcu){	t->i += iterations;	if (t->i >= repeat) {		/* We're done. */		t->time_spent += ktime_get_ns() - t->time_start;		do_div(t->time_spent, t->i);		*duration = t->time_spent > U32_MAX ? U32_MAX : (u32)t->time_spent;		*err = 0;		goto reset;	} 	if (signal_pending(current)) {		/* During iteration: we've been cancelled, abort. */		*err = -EINTR;		goto reset;	} 	if (need_resched()) {		/* During iteration: we need to reschedule between runs. */		t->time_spent += ktime_get_ns() - t->time_start;		bpf_test_timer_leave(t);		cond_resched();		bpf_test_timer_enter(t);	} 	/* Do another round. */	return true; reset:	t->i = 0;	return false;} /* We put this struct at the head of each page with a context and frame * initialised when the page is allocated, so we don't have to do this on each * repetition of the test run. */struct xdp_page_head {	struct xdp_buff orig_ctx;	struct xdp_buff ctx;	union {		/* ::data_hard_start starts here */		DECLARE_FLEX_ARRAY(struct xdp_frame, frame);		DECLARE_FLEX_ARRAY(u8, data);	};}; struct xdp_test_data {	struct xdp_buff *orig_ctx;	struct xdp_rxq_info rxq;	struct net_device *dev;	struct page_pool *pp;	struct xdp_frame **frames;	struct sk_buff **skbs;	struct xdp_mem_info mem;	u32 batch_size;	u32 frame_cnt;}; /* tools/testing/selftests/bpf/prog_tests/xdp_do_redirect.c:%MAX_PKT_SIZE * must be updated accordingly this gets changed, otherwise BPF selftests * will fail. */#define TEST_XDP_FRAME_SIZE (PAGE_SIZE - sizeof(struct xdp_page_head))#define TEST_XDP_MAX_BATCH 256 static void xdp_test_run_init_page(netmem_ref netmem, void *arg){	struct xdp_page_head *head =		phys_to_virt(page_to_phys(netmem_to_page(netmem)));	struct xdp_buff *new_ctx, *orig_ctx;	u32 headroom = XDP_PACKET_HEADROOM;	struct xdp_test_data *xdp = arg;	size_t frm_len, meta_len;	struct xdp_frame *frm;	void *data; 	orig_ctx = xdp->orig_ctx;	frm_len = orig_ctx->data_end - orig_ctx->data_meta;	meta_len = orig_ctx->data - orig_ctx->data_meta;	headroom -= meta_len; 	new_ctx = &head->ctx;	frm = head->frame;	data = head->data;	memcpy(data + headroom, orig_ctx->data_meta, frm_len); 	xdp_init_buff(new_ctx, TEST_XDP_FRAME_SIZE, &xdp->rxq);	xdp_prepare_buff(new_ctx, data, headroom, frm_len, true);	new_ctx->data = new_ctx->data_meta + meta_len; 	xdp_update_frame_from_buff(new_ctx, frm);	frm->mem_type = new_ctx->rxq->mem.type; 	memcpy(&head->orig_ctx, new_ctx, sizeof(head->orig_ctx));} static int xdp_test_run_setup(struct xdp_test_data *xdp, struct xdp_buff *orig_ctx){	struct page_pool *pp;	int err = -ENOMEM;	struct page_pool_params pp_params = {		.order = 0,		.flags = 0,		.pool_size = xdp->batch_size,		.nid = NUMA_NO_NODE,		.init_callback = xdp_test_run_init_page,		.init_arg = xdp,	}; 	xdp->frames = kvmalloc_array(xdp->batch_size, sizeof(void *), GFP_KERNEL);	if (!xdp->frames)		return -ENOMEM; 	xdp->skbs = kvmalloc_array(xdp->batch_size, sizeof(void *), GFP_KERNEL);	if (!xdp->skbs)		goto err_skbs; 	pp = page_pool_create(&pp_params);	if (IS_ERR(pp)) {		err = PTR_ERR(pp);		goto err_pp;	} 	/* will copy 'mem.id' into pp->xdp_mem_id */	err = xdp_reg_mem_model(&xdp->mem, MEM_TYPE_PAGE_POOL, pp);	if (err)		goto err_mmodel; 	xdp->pp = pp; 	/* We create a 'fake' RXQ referencing the original dev, but with an	 * xdp_mem_info pointing to our page_pool	 */	xdp_rxq_info_reg(&xdp->rxq, orig_ctx->rxq->dev, 0, 0);	xdp->rxq.mem.type = MEM_TYPE_PAGE_POOL;	xdp->rxq.mem.id = pp->xdp_mem_id;	xdp->dev = orig_ctx->rxq->dev;	xdp->orig_ctx = orig_ctx; 	return 0; err_mmodel:	page_pool_destroy(pp);err_pp:	kvfree(xdp->skbs);err_skbs:	kvfree(xdp->frames);	return err;} static void xdp_test_run_teardown(struct xdp_test_data *xdp){	xdp_unreg_mem_model(&xdp->mem);	page_pool_destroy(xdp->pp);	kfree(xdp->frames);	kfree(xdp->skbs);} static bool frame_was_changed(const struct xdp_page_head *head){	/* xdp_scrub_frame() zeroes the data pointer, flags is the last field,	 * i.e. has the highest chances to be overwritten. If those two are	 * untouched, it's most likely safe to skip the context reset.	 */	return head->frame->data != head->orig_ctx.data ||	       head->frame->flags != head->orig_ctx.flags;} static bool ctx_was_changed(struct xdp_page_head *head){	return head->orig_ctx.data != head->ctx.data ||		head->orig_ctx.data_meta != head->ctx.data_meta ||		head->orig_ctx.data_end != head->ctx.data_end;} static void reset_ctx(struct xdp_page_head *head){	if (likely(!frame_was_changed(head) && !ctx_was_changed(head)))		return; 	head->ctx.data = head->orig_ctx.data;	head->ctx.data_meta = head->orig_ctx.data_meta;	head->ctx.data_end = head->orig_ctx.data_end;	xdp_update_frame_from_buff(&head->ctx, head->frame);	head->frame->mem_type = head->orig_ctx.rxq->mem.type;} static int xdp_recv_frames(struct xdp_frame **frames, int nframes,			   struct sk_buff **skbs,			   struct net_device *dev){	gfp_t gfp = __GFP_ZERO | GFP_ATOMIC;	int i;	LIST_HEAD(list); 	if (unlikely(!kmem_cache_alloc_bulk(net_hotdata.skbuff_cache, gfp,					    nframes, (void **)skbs))) {		for (i = 0; i < nframes; i++)			xdp_return_frame(frames[i]);		return -ENOMEM;	} 	for (i = 0; i < nframes; i++) {		struct xdp_frame *xdpf = frames[i];		struct sk_buff *skb = skbs[i]; 		skb = __xdp_build_skb_from_frame(xdpf, skb, dev);		if (!skb) {			xdp_return_frame(xdpf);			continue;		} 		list_add_tail(&skb->list, &list);	}	netif_receive_skb_list(&list); 	return 0;} static int xdp_test_run_batch(struct xdp_test_data *xdp, struct bpf_prog *prog,			      u32 repeat){	struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx;	int err = 0, act, ret, i, nframes = 0, batch_sz;	struct xdp_frame **frames = xdp->frames;	struct bpf_redirect_info *ri;	struct xdp_page_head *head;	struct xdp_frame *frm;	bool redirect = false;	struct xdp_buff *ctx;	struct page *page; 	batch_sz = min_t(u32, repeat, xdp->batch_size); 	local_bh_disable();	bpf_net_ctx = bpf_net_ctx_set(&__bpf_net_ctx);	ri = bpf_net_ctx_get_ri();	xdp_set_return_frame_no_direct(); 	for (i = 0; i < batch_sz; i++) {		page = page_pool_dev_alloc_pages(xdp->pp);		if (!page) {			err = -ENOMEM;			goto out;		} 		head = phys_to_virt(page_to_phys(page));		reset_ctx(head);		ctx = &head->ctx;		frm = head->frame;		xdp->frame_cnt++; 		act = bpf_prog_run_xdp(prog, ctx); 		/* if program changed pkt bounds we need to update the xdp_frame */		if (unlikely(ctx_was_changed(head))) {			ret = xdp_update_frame_from_buff(ctx, frm);			if (ret) {				xdp_return_buff(ctx);				continue;			}		} 		switch (act) {		case XDP_TX:			/* we can't do a real XDP_TX since we're not in the			 * driver, so turn it into a REDIRECT back to the same			 * index			 */			ri->tgt_index = xdp->dev->ifindex;			ri->map_id = INT_MAX;			ri->map_type = BPF_MAP_TYPE_UNSPEC;			fallthrough;		case XDP_REDIRECT:			redirect = true;			ret = xdp_do_redirect_frame(xdp->dev, ctx, frm, prog);			if (ret)				xdp_return_buff(ctx);			break;		case XDP_PASS:			frames[nframes++] = frm;			break;		default:			bpf_warn_invalid_xdp_action(NULL, prog, act);			fallthrough;		case XDP_DROP:			xdp_return_buff(ctx);			break;		}	} out:	if (redirect)		xdp_do_flush();	if (nframes) {		ret = xdp_recv_frames(frames, nframes, xdp->skbs, xdp->dev);		if (ret)			err = ret;	} 	xdp_clear_return_frame_no_direct();	bpf_net_ctx_clear(bpf_net_ctx);	local_bh_enable();	return err;} static int bpf_test_run_xdp_live(struct bpf_prog *prog, struct xdp_buff *ctx,				 u32 repeat, u32 batch_size, u32 *time) {	struct xdp_test_data xdp = { .batch_size = batch_size };	struct bpf_test_timer t = {};	int ret; 	if (!repeat)		repeat = 1; 	ret = xdp_test_run_setup(&xdp, ctx);	if (ret)		return ret; 	bpf_test_timer_enter(&t);	do {		xdp.frame_cnt = 0;		ret = xdp_test_run_batch(&xdp, prog, repeat - t.i);		if (unlikely(ret < 0))			break;	} while (bpf_test_timer_continue(&t, xdp.frame_cnt, repeat, &ret, time));	bpf_test_timer_leave(&t); 	xdp_test_run_teardown(&xdp);	return ret;} static int bpf_test_run(struct bpf_prog *prog, void *ctx, u32 repeat,			u32 *retval, u32 *time, bool xdp){	struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx;	struct bpf_prog_array_item item = {.prog = prog};	struct bpf_run_ctx *old_ctx;	struct bpf_cg_run_ctx run_ctx;	struct bpf_test_timer t = {};	enum bpf_cgroup_storage_type stype;	int ret; 	for_each_cgroup_storage_type(stype) {		item.cgroup_storage[stype] = bpf_cgroup_storage_alloc(prog, stype);		if (IS_ERR(item.cgroup_storage[stype])) {			item.cgroup_storage[stype] = NULL;			for_each_cgroup_storage_type(stype)				bpf_cgroup_storage_free(item.cgroup_storage[stype]);			return -ENOMEM;		}	} 	if (!repeat)		repeat = 1; 	bpf_test_timer_enter(&t);	old_ctx = bpf_set_run_ctx(&run_ctx.run_ctx);	do {		run_ctx.prog_item = &item;		local_bh_disable();		bpf_net_ctx = bpf_net_ctx_set(&__bpf_net_ctx); 		if (xdp)			*retval = bpf_prog_run_xdp(prog, ctx);		else			*retval = bpf_prog_run(prog, ctx); 		bpf_net_ctx_clear(bpf_net_ctx);		local_bh_enable();	} while (bpf_test_timer_continue(&t, 1, repeat, &ret, time));	bpf_reset_run_ctx(old_ctx);	bpf_test_timer_leave(&t); 	for_each_cgroup_storage_type(stype)		bpf_cgroup_storage_free(item.cgroup_storage[stype]); 	return ret;} static int bpf_test_finish(const union bpf_attr *kattr,			   union bpf_attr __user *uattr, const void *data,			   struct skb_shared_info *sinfo, u32 size, u32 frag_size,			   u32 retval, u32 duration){	void __user *data_out = u64_to_user_ptr(kattr->test.data_out);	int err = -EFAULT;	u32 copy_size = size; 	/* Clamp copy if the user has provided a size hint, but copy the full	 * buffer if not to retain old behaviour.	 */	if (kattr->test.data_size_out &&	    copy_size > kattr->test.data_size_out) {		copy_size = kattr->test.data_size_out;		err = -ENOSPC;	} 	if (data_out) {		u32 head_len = size - frag_size;		u32 len = min(copy_size, head_len); 		if (copy_to_user(data_out, data, len))			goto out; 		if (sinfo) {			int i, offset = len;			u32 data_len; 			for (i = 0; i < sinfo->nr_frags; i++) {				skb_frag_t *frag = &sinfo->frags[i]; 				if (offset >= copy_size) {					err = -ENOSPC;					break;				} 				data_len = min_t(u32, copy_size - offset,						 skb_frag_size(frag)); 				if (copy_to_user(data_out + offset,						 skb_frag_address(frag),						 data_len))					goto out; 				offset += data_len;			}		}	} 	if (copy_to_user(&uattr->test.data_size_out, &size, sizeof(size)))		goto out;	if (copy_to_user(&uattr->test.retval, &retval, sizeof(retval)))		goto out;	if (copy_to_user(&uattr->test.duration, &duration, sizeof(duration)))		goto out;	if (err != -ENOSPC)		err = 0;out:	trace_bpf_test_finish(&err);	return err;} /* Integer types of various sizes and pointer combinations cover variety of * architecture dependent calling conventions. 7+ can be supported in the

Lines 1–500 of 1910. Load more