Merge git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf

Daniel Borkmann says: ==================== pull-request: bpf 2020-12-28 The following pull-request contains BPF updates for your *net* tree. There is a small merge conflict between bpf tree commit 69ca310f3416 ("bpf: Save correct stopping point in file seq iteration") and net tree commit 66ed594409a1 ("bpf/task_iter: In task_file_seq_get_next use task_lookup_next_fd_rcu"). The get_files_struct() does not exist anymore in net, so take the hunk in HEAD and add the `info->tid = curr_tid` to the error path: [...] curr_task = task_seq_get_next(ns, &curr_tid, true); if (!curr_task) { info->task = NULL; info->tid = curr_tid; return NULL; } /* set info->task and info->tid */ [...] We've added 10 non-merge commits during the last 9 day(s) which contain a total of 11 files changed, 75 insertions(+), 20 deletions(-). The main changes are: 1) Various AF_XDP fixes such as fill/completion ring leak on failed bind and fixing a race in skb mode's backpressure mechanism, from Magnus Karlsson. 2) Fix latency spikes on lockdep enabled kernels by adding a rescheduling point to BPF hashtab initialization, from Eric Dumazet. 3) Fix a splat in task iterator by saving the correct stopping point in the seq file iteration, from Jonathan Lemon. 4) Fix BPF maps selftest by adding retries in case hashtab returns EBUSY errors on update/deletes, from Andrii Nakryiko. 5) Fix BPF selftest error reporting to something more user friendly if the vmlinux BTF cannot be found, from Kamal Mostafa. ====================

David S. Miller committed 5 years ago

Commit 4bfc4712 parents 1fef735 a61daaf11 files · +81 −27Browse files

Unified · Split

11 files

include/net/xdp_sock.h +0 −4
include/net/xsk_buff_pool.h +5 −0
kernel/bpf/hashtab.c +1 −0
kernel/bpf/syscall.c +0 −1
kernel/bpf/task_iter.c +9 −9
net/xdp/xsk.c +13 −3
net/xdp/xsk_buff_pool.c +1 −2
net/xdp/xsk_queue.h +5 −0
tools/testing/selftests/bpf/Makefile +3 −0
tools/testing/selftests/bpf/test_maps.c +42 −6
tools/testing/selftests/bpf/xdpxceiver.c +2 −2

include/net/xdp_sock.h+0−4

 	struct xsk_queue *tx ____cacheline_aligned_in_smp;
 	struct list_head tx_list;
-	/* Mutual exclusion of NAPI TX thread and sendmsg error paths
-	 * in the SKB destructor callback.
-	 */
-	spinlock_t tx_completion_lock;
 	/* Protects generic receive. */
 	spinlock_t rx_lock;

include/net/xsk_buff_pool.h+5−0

 	bool dma_need_sync;
 	bool unaligned;
 	void *addrs;
+	/* Mutual exclusion of the completion ring in the SKB mode. Two cases to protect:
+	 * NAPI TX thread and sendmsg error paths in the SKB destructor callback and when
+	 * sockets share a single cq when the same netdev and queue id is shared.
+	 */
+	spinlock_t cq_lock;
 	struct xdp_buff_xsk *free_heads[];
 };

kernel/bpf/hashtab.c+1−0

@@ -152,6 +152,7 @@ static void htab_init_buckets(struct bpf_htab *htab) · 151 unmodified lines
152	lockdep_set_class(&htab->buckets[i].lock,	152	lockdep_set_class(&htab->buckets[i].lock,
153	&htab->lockdep_key);	153	&htab->lockdep_key);
154	}	154	}
		155	cond_resched();
155	}	156	}
156	}	157	}
157		158

kernel/bpf/syscall.c+0−1

@@ -17,7 +17,6 @@ · 16 unmodified lines
17	#include <linux/fs.h>	17	#include <linux/fs.h>
18	#include <linux/license.h>	18	#include <linux/license.h>
19	#include <linux/filter.h>	19	#include <linux/filter.h>
20	#include <linux/version.h>
21	#include <linux/kernel.h>	20	#include <linux/kernel.h>
22	#include <linux/idr.h>	21	#include <linux/idr.h>
23	#include <linux/cred.h>	22	#include <linux/cred.h>

kernel/bpf/task_iter.c+9−9

@@ -37,7 +37,7 @@ retry: · 36 unmodified lines
37	if (!task) {	37	if (!task) {
38	++*tid;	38	++*tid;
39	goto retry;	39	goto retry;
40	} else if (skip_if_dup_files && task->tgid != task->pid &&	40	} else if (skip_if_dup_files && !thread_group_leader(task) &&
41	task->files == task->group_leader->files) {	41	task->files == task->group_leader->files) {
42	put_task_struct(task);	42	put_task_struct(task);
43	task = NULL;	43	task = NULL;
@@ -151,14 +151,14 @@ again: · 107 unmodified lines
151	curr_task = info->task;	151	curr_task = info->task;
152	curr_fd = info->fd;	152	curr_fd = info->fd;
153	} else {	153	} else {
154	curr_task = task_seq_get_next(ns, &curr_tid, true);	154	curr_task = task_seq_get_next(ns, &curr_tid, true);
155	if (!curr_task) {	155	if (!curr_task) {
156	info->task = NULL;	156	info->task = NULL;
157	return NULL;	157	info->tid = curr_tid;
158	}	158	return NULL;
159		159	}
160	/* set info->task and info->tid */	160
161	info->task = curr_task;	161	/* set info->task and info->tid */
162	if (curr_tid == info->tid) {	162	if (curr_tid == info->tid) {
163	curr_fd = info->fd;	163	curr_fd = info->fd;
164	} else {	164	} else {

net/xdp/xsk.c+13−3

 	struct xdp_sock *xs = xdp_sk(skb->sk);
 	unsigned long flags;
-	spin_lock_irqsave(&xs->tx_completion_lock, flags);
+	spin_lock_irqsave(&xs->pool->cq_lock, flags);
 	xskq_prod_submit_addr(xs->pool->cq, addr);
-	spin_unlock_irqrestore(&xs->tx_completion_lock, flags);
+	spin_unlock_irqrestore(&xs->pool->cq_lock, flags);
 	sock_wfree(skb);
 }
 	bool sent_frame = false;
 	struct xdp_desc desc;
 	struct sk_buff *skb;
+	unsigned long flags;
 	int err = 0;
 	mutex_lock(&xs->mutex);
 		 * if there is space in it. This avoids having to implement
 		 * any buffering in the Tx path.
 		 */
+		spin_lock_irqsave(&xs->pool->cq_lock, flags);
 		if (unlikely(err) || xskq_prod_reserve(xs->pool->cq)) {
+			spin_unlock_irqrestore(&xs->pool->cq_lock, flags);
 			kfree_skb(skb);
 			goto out;
 		}
+		spin_unlock_irqrestore(&xs->pool->cq_lock, flags);
 		skb->dev = xs->dev;
 		skb->priority = sk->sk_priority;
 		if  (err == NETDEV_TX_BUSY) {
 			/* Tell user-space to retry the send */
 			skb->destructor = sock_wfree;
+			spin_lock_irqsave(&xs->pool->cq_lock, flags);
+			xskq_prod_cancel(xs->pool->cq);
+			spin_unlock_irqrestore(&xs->pool->cq_lock, flags);
 			/* Free skb without triggering the perf drop trace */
 			consume_skb(skb);
 			err = -EAGAIN;
 		}
 	}
+	/* FQ and CQ are now owned by the buffer pool and cleaned up with it. */
+	xs->fq_tmp = NULL;
+	xs->cq_tmp = NULL;
 	xs->dev = dev;
 	xs->zc = xs->umem->zc;
 	xs->queue_id = qid;
 	xs->state = XSK_READY;
 	mutex_init(&xs->mutex);
 	spin_lock_init(&xs->rx_lock);
-	spin_lock_init(&xs->tx_completion_lock);
 	INIT_LIST_HEAD(&xs->map_list);
 	spin_lock_init(&xs->map_list_lock);

net/xdp/xsk_buff_pool.c+1−2

 	INIT_LIST_HEAD(&pool->free_list);
 	INIT_LIST_HEAD(&pool->xsk_tx_list);
 	spin_lock_init(&pool->xsk_tx_list_lock);
+	spin_lock_init(&pool->cq_lock);
 	refcount_set(&pool->users, 1);
 	pool->fq = xs->fq_tmp;
 	pool->cq = xs->cq_tmp;
-	xs->fq_tmp = NULL;
-	xs->cq_tmp = NULL;
 	for (i = 0; i < pool->free_heads_cnt; i++) {
 		xskb = &pool->heads[i];

net/xdp/xsk_queue.h+5−0

 	return xskq_prod_nb_free(q, 1) ? false : true;
 }
+static inline void xskq_prod_cancel(struct xsk_queue *q)
+{
+	q->cached_prod--;
+}
 static inline int xskq_prod_reserve(struct xsk_queue *q)
 {
 	if (xskq_prod_is_full(q))

tools/testing/selftests/bpf/Makefile+3−0

 		     /sys/kernel/btf/vmlinux				\
 		     /boot/vmlinux-$(shell uname -r)
 VMLINUX_BTF ?= $(abspath $(firstword $(wildcard $(VMLINUX_BTF_PATHS))))
+ifeq ($(VMLINUX_BTF),)
+$(error Cannot find a vmlinux for VMLINUX_BTF at any of "$(VMLINUX_BTF_PATHS)")
+endif
 # Define simple and short `make test_progs`, `make test_sysctl`, etc targets
 # to build individual tests.

tools/testing/selftests/bpf/test_maps.c+42−6

@@ -1312,22 +1312,58 @@ static void test_map_stress(void) · 1311 unmodified lines
1312	#define DO_UPDATE 1	1312	#define DO_UPDATE 1
1313	#define DO_DELETE 0	1313	#define DO_DELETE 0
1314		1314
		1315	#define MAP_RETRIES 20
		1316
		1317	static int map_update_retriable(int map_fd, const void key, const void value,
		1318	int flags, int attempts)
		1319	{
		1320	while (bpf_map_update_elem(map_fd, key, value, flags)) {
		1321	if (!attempts \|\| (errno != EAGAIN && errno != EBUSY))
		1322	return -errno;
		1323
		1324	usleep(1);
		1325	attempts--;
		1326	}
		1327
		1328	return 0;
		1329	}
		1330
		1331	static int map_delete_retriable(int map_fd, const void *key, int attempts)
		1332	{
		1333	while (bpf_map_delete_elem(map_fd, key)) {
		1334	if (!attempts \|\| (errno != EAGAIN && errno != EBUSY))
		1335	return -errno;
		1336
		1337	usleep(1);
		1338	attempts--;
		1339	}
		1340
		1341	return 0;
		1342	}
		1343
1315	static void test_update_delete(unsigned int fn, void *data)	1344	static void test_update_delete(unsigned int fn, void *data)
1316	{	1345	{
1317	int do_update = ((int *)data)[1];	1346	int do_update = ((int *)data)[1];
1318	int fd = ((int *)data)[0];	1347	int fd = ((int *)data)[0];
1319	int i, key, value;	1348	int i, key, value, err;
1320		1349
1321	for (i = fn; i < MAP_SIZE; i += TASKS) {	1350	for (i = fn; i < MAP_SIZE; i += TASKS) {
1322	key = value = i;	1351	key = value = i;
1323		1352
1324	if (do_update) {	1353	if (do_update) {
1325	assert(bpf_map_update_elem(fd, &key, &value,	1354	err = map_update_retriable(fd, &key, &value, BPF_NOEXIST, MAP_RETRIES);
1326	BPF_NOEXIST) == 0);	1355	if (err)
1327	assert(bpf_map_update_elem(fd, &key, &value,	1356	printf("error %d %d\n", err, errno);
1328	BPF_EXIST) == 0);	1357	assert(err == 0);
		1358	err = map_update_retriable(fd, &key, &value, BPF_EXIST, MAP_RETRIES);
		1359	if (err)
		1360	printf("error %d %d\n", err, errno);
		1361	assert(err == 0);
1329	} else {	1362	} else {
1330	assert(bpf_map_delete_elem(fd, &key) == 0);	1363	err = map_delete_retriable(fd, &key, MAP_RETRIES);
		1364	if (err)
		1365	printf("error %d %d\n", err, errno);
		1366	assert(err == 0);
1331	}	1367	}
1332	}	1368	}
1333	}	1369	}

tools/testing/selftests/bpf/xdpxceiver.c+2−2

@@ -715,7 +715,7 @@ static void worker_pkt_dump(void) · 714 unmodified lines
715	int payload = ((uint32_t )(pkt_buf[iter]->payload + PKT_HDR_SIZE));	715	int payload = ((uint32_t )(pkt_buf[iter]->payload + PKT_HDR_SIZE));
716		716
717	if (payload == EOT) {	717	if (payload == EOT) {
718	ksft_print_msg("End-of-tranmission frame received\n");	718	ksft_print_msg("End-of-transmission frame received\n");
719	fprintf(stdout, "---------------------------------------\n");	719	fprintf(stdout, "---------------------------------------\n");
720	break;	720	break;
721	}	721	}
@@ -747,7 +747,7 @@ static void worker_pkt_validate(void) · 25 unmodified lines
747	}	747	}
748		748
749	if (payloadseqnum == EOT) {	749	if (payloadseqnum == EOT) {
750	ksft_print_msg("End-of-tranmission frame received: PASS\n");	750	ksft_print_msg("End-of-transmission frame received: PASS\n");
751	sigvar = 1;	751	sigvar = 1;
752	break;	752	break;
753	}	753	}

Merge git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code