Merge git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next

Daniel Borkmann says: ==================== pull-request: bpf-next 2021-06-28 The following pull-request contains BPF updates for your *net-next* tree. We've added 37 non-merge commits during the last 12 day(s) which contain a total of 56 files changed, 394 insertions(+), 380 deletions(-). The main changes are: 1) XDP driver RCU cleanups, from Toke Høiland-Jørgensen and Paul E. McKenney. 2) Fix bpf_skb_change_proto() IPv4/v6 GSO handling, from Maciej Żenczykowski. 3) Fix false positive kmemleak report for BPF ringbuf alloc, from Rustam Kovhaev. 4) Fix x86 JIT's extable offset calculation for PROBE_LDX NULL, from Ravi Bangoria. 5) Enable libbpf fallback probing with tracing under RHEL7, from Jonathan Edwards. 6) Clean up x86 JIT to remove unused cnt tracking from EMIT macro, from Jiri Olsa. 7) Netlink cleanups for libbpf to please Coverity, from Kumar Kartikeya Dwivedi. 8) Allow to retrieve ancestor cgroup id in tracing programs, from Namhyung Kim. 9) Fix lirc BPF program query to use user-provided prog_cnt, from Sean Young. 10) Add initial libbpf doc including generated kdoc for its API, from Grant Seltzer. 11) Make xdp_rxq_info_unreg_mem_model() more robust, from Jakub Kicinski. 12) Fix up bpfilter startup log-level to info level, from Gary Lin. ====================

David S. Miller committed 5 years ago

Commit e1289cf2 parents 1fd07f3 a78cae256 files · +394 −380Browse files

Unified · Split

56 files

Documentation/RCU/checklist.rst +34 −21
Documentation/bpf/index.rst +13 −0
Documentation/bpf/libbpf/libbpf.rst +14 −0
Documentation/bpf/libbpf/libbpf_api.rst +27 −0
Documentation/bpf/libbpf/libbpf_build.rst +37 −0
Documentation/bpf/libbpf/libbpf_naming_convention.rst +12 −18
Documentation/networking/af_xdp.rst +16 −16
arch/x86/net/bpf_jit_comp.c +13 −33
drivers/media/rc/bpf-lirc.c +2 −1
drivers/net/ethernet/amazon/ena/ena_netdev.c +0 −3
drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c +0 −2
drivers/net/ethernet/cavium/thunder/nicvf_main.c +0 −2
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +1 −7
drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c +0 −3
drivers/net/ethernet/intel/i40e/i40e_txrx.c +0 −2
drivers/net/ethernet/intel/i40e/i40e_xsk.c +0 −3
drivers/net/ethernet/intel/ice/ice_txrx.c +1 −5
drivers/net/ethernet/intel/ice/ice_xsk.c +0 −3
drivers/net/ethernet/intel/igb/igb_main.c +0 −2
drivers/net/ethernet/intel/igc/igc_main.c +2 −5
drivers/net/ethernet/intel/ixgbe/ixgbe_main.c +0 −2
drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c +0 −3
drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c +0 −2
drivers/net/ethernet/marvell/mvneta.c +0 −2
drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c +0 −4
drivers/net/ethernet/mellanox/mlx4/en_rx.c +2 −6
drivers/net/ethernet/netronome/nfp/nfp_net_common.c +0 −2
drivers/net/ethernet/qlogic/qede/qede_fp.c +0 −6
drivers/net/ethernet/sfc/rx.c +2 −7
drivers/net/ethernet/socionext/netsec.c +0 −3
drivers/net/ethernet/stmicro/stmmac/stmmac_main.c +2 −8
drivers/net/ethernet/ti/cpsw_priv.c +2 −8
include/linux/filter.h +3 −5
include/linux/rcupdate.h +14 −0
include/net/xdp_sock.h +1 −1
kernel/bpf/cpumap.c +9 −4
kernel/bpf/devmap.c +21 −28
kernel/bpf/hashtab.c +14 −7
kernel/bpf/helpers.c +3 −3
kernel/bpf/lpm_trie.c +4 −2
kernel/bpf/ringbuf.c +2 −0
kernel/trace/bpf_trace.c +2 −0
net/bpfilter/main.c +1 −1
net/core/filter.c +37 −35
net/core/xdp.c +6 −5
net/sched/act_bpf.c +0 −2
net/sched/cls_bpf.c +0 −3
net/xdp/xsk.c +2 −2
net/xdp/xsk.h +2 −2
net/xdp/xskmap.c +17 −12
samples/bpf/xdp_redirect_user.c +2 −2
tools/lib/bpf/libbpf.c +4 −0
tools/lib/bpf/netlink.c +44 −71
tools/lib/bpf/nlattr.c +1 −1
tools/lib/bpf/nlattr.h +24 −14
tools/testing/selftests/bpf/prog_tests/ringbuf.c +1 −1

Documentation/RCU/checklist.rst+34−21

@@ -211,27 +211,40 @@ over a rather long period of time, but improvements are always welcome! · 210 unmodified lines
211	211	of the system, especially to real-time workloads running on
212	212	the rest of the system.
213	213
214		−7. As of v4.20, a given kernel implements only one RCU flavor,
215		− which is RCU-sched for PREEMPTION=n and RCU-preempt for PREEMPTION=y.
216		− If the updater uses call_rcu() or synchronize_rcu(),
217		− then the corresponding readers may use rcu_read_lock() and
218		− rcu_read_unlock(), rcu_read_lock_bh() and rcu_read_unlock_bh(),
219		− or any pair of primitives that disables and re-enables preemption,
220		− for example, rcu_read_lock_sched() and rcu_read_unlock_sched().
221		− If the updater uses synchronize_srcu() or call_srcu(),
222		− then the corresponding readers must use srcu_read_lock() and
223		− srcu_read_unlock(), and with the same srcu_struct. The rules for
224		− the expedited primitives are the same as for their non-expedited
225		− counterparts. Mixing things up will result in confusion and
226		− broken kernels, and has even resulted in an exploitable security
227		− issue.
228		−
229		− One exception to this rule: rcu_read_lock() and rcu_read_unlock()
230		− may be substituted for rcu_read_lock_bh() and rcu_read_unlock_bh()
231		− in cases where local bottom halves are already known to be
232		− disabled, for example, in irq or softirq context. Commenting
233		− such cases is a must, of course! And the jury is still out on
234		− whether the increased speed is worth it.
	214	+7. As of v4.20, a given kernel implements only one RCU flavor, which
	215	+ is RCU-sched for PREEMPTION=n and RCU-preempt for PREEMPTION=y.
	216	+ If the updater uses call_rcu() or synchronize_rcu(), then
	217	+ the corresponding readers may use: (1) rcu_read_lock() and
	218	+ rcu_read_unlock(), (2) any pair of primitives that disables
	219	+ and re-enables softirq, for example, rcu_read_lock_bh() and
	220	+ rcu_read_unlock_bh(), or (3) any pair of primitives that disables
	221	+ and re-enables preemption, for example, rcu_read_lock_sched() and
	222	+ rcu_read_unlock_sched(). If the updater uses synchronize_srcu()
	223	+ or call_srcu(), then the corresponding readers must use
	224	+ srcu_read_lock() and srcu_read_unlock(), and with the same
	225	+ srcu_struct. The rules for the expedited RCU grace-period-wait
	226	+ primitives are the same as for their non-expedited counterparts.
	227	+
	228	+ If the updater uses call_rcu_tasks() or synchronize_rcu_tasks(),
	229	+ then the readers must refrain from executing voluntary
	230	+ context switches, that is, from blocking. If the updater uses
	231	+ call_rcu_tasks_trace() or synchronize_rcu_tasks_trace(), then
	232	+ the corresponding readers must use rcu_read_lock_trace() and
	233	+ rcu_read_unlock_trace(). If an updater uses call_rcu_tasks_rude()
	234	+ or synchronize_rcu_tasks_rude(), then the corresponding readers
	235	+ must use anything that disables interrupts.
	236	+
	237	+ Mixing things up will result in confusion and broken kernels, and
	238	+ has even resulted in an exploitable security issue. Therefore,
	239	+ when using non-obvious pairs of primitives, commenting is
	240	+ of course a must. One example of non-obvious pairing is
	241	+ the XDP feature in networking, which calls BPF programs from
	242	+ network-driver NAPI (softirq) context. BPF relies heavily on RCU
	243	+ protection for its data structures, but because the BPF program
	244	+ invocation happens entirely within a single local_bh_disable()
	245	+ section in a NAPI poll cycle, this usage is safe. The reason
	246	+ that this usage is safe is that readers can use anything that
	247	+ disables BH when updaters use call_rcu() or synchronize_rcu().
235	248
236	249	8. Although synchronize_rcu() is slower than is call_rcu(), it
237	250	usually results in simpler code. So, unless update performance is

Documentation/bpf/index.rst+13−0

@@ -12,6 +12,19 @@ BPF instruction-set. · 11 unmodified lines
12	12	The Cilium project also maintains a `BPF and XDP Reference Guide`_
13	13	that goes into great technical depth about the BPF Architecture.
14	14
	15	+libbpf
	16	+======
	17	+
	18	+Libbpf is a userspace library for loading and interacting with bpf programs.
	19	+
	20	+.. toctree::
	21	+ :maxdepth: 1
	22	+
	23	+ libbpf/libbpf
	24	+ libbpf/libbpf_api
	25	+ libbpf/libbpf_build
	26	+ libbpf/libbpf_naming_convention
	27	+
15	28	BPF Type Format (BTF)
16	29	=====================
17	30

Documentation/bpf/libbpf/libbpf.rst+14−0

@@ -0,0 +1,14 @@
	1	+.. SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
	2	+
	3	+libbpf
	4	+======
	5	+
	6	+This is documentation for libbpf, a userspace library for loading and
	7	+interacting with bpf programs.
	8	+
	9	+All general BPF questions, including kernel functionality, libbpf APIs and
	10	+their application, should be sent to bpf@vger.kernel.org mailing list.
	11	+You can `subscribe <http://vger.kernel.org/vger-lists.html#bpf>`_ to the
	12	+mailing list search its `archive <https://lore.kernel.org/bpf/>`_.
	13	+Please search the archive before asking new questions. It very well might
	14	+be that this was already addressed or answered before.

Documentation/bpf/libbpf/libbpf_api.rst+27−0

@@ -0,0 +1,27 @@
	1	+.. SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
	2	+
	3	+API
	4	+===
	5	+
	6	+This documentation is autogenerated from header files in libbpf, tools/lib/bpf
	7	+
	8	+.. kernel-doc:: tools/lib/bpf/libbpf.h
	9	+ :internal:
	10	+
	11	+.. kernel-doc:: tools/lib/bpf/bpf.h
	12	+ :internal:
	13	+
	14	+.. kernel-doc:: tools/lib/bpf/btf.h
	15	+ :internal:
	16	+
	17	+.. kernel-doc:: tools/lib/bpf/xsk.h
	18	+ :internal:
	19	+
	20	+.. kernel-doc:: tools/lib/bpf/bpf_tracing.h
	21	+ :internal:
	22	+
	23	+.. kernel-doc:: tools/lib/bpf/bpf_core_read.h
	24	+ :internal:
	25	+
	26	+.. kernel-doc:: tools/lib/bpf/bpf_endian.h
	27	+ :internal:

Documentation/bpf/libbpf/libbpf_build.rst+37−0

@@ -0,0 +1,37 @@
	1	+.. SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
	2	+
	3	+Building libbpf
	4	+===============
	5	+
	6	+libelf and zlib are internal dependencies of libbpf and thus are required to link
	7	+against and must be installed on the system for applications to work.
	8	+pkg-config is used by default to find libelf, and the program called
	9	+can be overridden with PKG_CONFIG.
	10	+
	11	+If using pkg-config at build time is not desired, it can be disabled by
	12	+setting NO_PKG_CONFIG=1 when calling make.
	13	+
	14	+To build both static libbpf.a and shared libbpf.so:
	15	+
	16	+.. code-block:: bash
	17	+
	18	+ $ cd src
	19	+ $ make
	20	+
	21	+To build only static libbpf.a library in directory build/ and install them
	22	+together with libbpf headers in a staging directory root/:
	23	+
	24	+.. code-block:: bash
	25	+
	26	+ $ cd src
	27	+ $ mkdir build root
	28	+ $ BUILD_STATIC_ONLY=y OBJDIR=build DESTDIR=root make install
	29	+
	30	+To build both static libbpf.a and shared libbpf.so against a custom libelf
	31	+dependency installed in /build/root/ and install them together with libbpf
	32	+headers in a build directory /build/root/:
	33	+
	34	+.. code-block:: bash
	35	+
	36	+ $ cd src
	37	+ $ PKG_CONFIG_PATH=/build/root/lib64/pkgconfig DESTDIR=/build/root make

Documentation/bpf/libbpf/libbpf_naming_convention.rstrenamed from tools/lib/bpf/README.rst+12−18

@@ -1,7 +1,7 @@
1	1	.. SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
2	2
3		−libbpf API naming convention
4		−============================
	3	+API naming convention
	4	+=====================
5	5
6	6	libbpf API provides access to a few logically separated groups of
7	7	functions and types. Every group has its own naming convention
@@ -10,14 +10,14 @@ new function or type is added to keep libbpf API clean and consistent. · 2 unmodified lines
10	10
11	11	All types and functions provided by libbpf API should have one of the
12	12	following prefixes: ``bpf_``, ``btf_``, ``libbpf_``, ``xsk_``,
13		−``perf_buffer_``.
	13	+``btf_dump_``, ``ring_buffer_``, ``perf_buffer_``.
14	14
15	15	System call wrappers
16	16	--------------------
17	17
18	18	System call wrappers are simple wrappers for commands supported by
19	19	sys_bpf system call. These wrappers should go to ``bpf.h`` header file
20		−and map one-on-one to corresponding commands.
	20	+and map one to one to corresponding commands.
21	21
22	22	For example ``bpf_map_lookup_elem`` wraps ``BPF_MAP_LOOKUP_ELEM``
23	23	command of sys_bpf, ``bpf_prog_attach`` wraps ``BPF_PROG_ATTACH``, etc.
@@ -49,10 +49,6 @@ object, ``bpf_object``, double underscore and ``open`` that defines the · 25 unmodified lines
49	49	purpose of the function to open ELF file and create ``bpf_object`` from
50	50	it.
51	51
52		−Another example: ``bpf_program__load`` is named for corresponding
53		−object, ``bpf_program``, that is separated from other part of the name
54		−by double underscore.
55		−
56	52	All objects and corresponding functions other than BTF related should go
57	53	to ``libbpf.h``. BTF types and functions should go to ``btf.h``.
58	54
@@ -72,11 +68,7 @@ of both low-level ring access functions and high-level configuration · 13 unmodified lines
72	68	functions. These can be mixed and matched. Note that these functions
73	69	are not reentrant for performance reasons.
74	70
75		−Please take a look at Documentation/networking/af_xdp.rst in the Linux
76		−kernel source tree on how to use XDP sockets and for some common
77		−mistakes in case you do not get any traffic up to user space.
78		−
79		−libbpf ABI
	71	+ABI
80	72	==========
81	73
82	74	libbpf can be both linked statically or used as DSO. To avoid possible
@@ -116,7 +108,8 @@ This bump in ABI version is at most once per kernel development cycle. · 33 unmodified lines
116	108
117	109	For example, if current state of ``libbpf.map`` is:
118	110
119		−.. code-block::
	111	+.. code-block:: c
	112	+
120	113	LIBBPF_0.0.1 {
121	114	global:
122	115	bpf_func_a;
@@ -128,7 +121,8 @@ For example, if current state of ``libbpf.map`` is: · 5 unmodified lines
128	121	, and a new symbol ``bpf_func_c`` is being introduced, then
129	122	``libbpf.map`` should be changed like this:
130	123
131		−.. code-block::
	124	+.. code-block:: c
	125	+
132	126	LIBBPF_0.0.1 {
133	127	global:
134	128	bpf_func_a;
@@ -148,7 +142,7 @@ Format of version script and ways to handle ABI changes, including · 13 unmodified lines
148	142	incompatible ones, described in details in [1].
149	143
150	144	Stand-alone build
151		−=================
	145	+-------------------
152	146
153	147	Under https://github.com/libbpf/libbpf there is a (semi-)automated
154	148	mirror of the mainline's version of libbpf for a stand-alone build.
@@ -157,12 +151,12 @@ However, all changes to libbpf's code base must be upstreamed through · 2 unmodified lines
157	151	the mainline kernel tree.
158	152
159	153	License
160		−=======
	154	+-------------------
161	155
162	156	libbpf is dual-licensed under LGPL 2.1 and BSD 2-Clause.
163	157
164	158	Links
165		−=====
	159	+-------------------
166	160
167	161	[1] https://www.akkadia.org/drepper/dsohowto.pdf
168	162	(Chapter 3. Maintaining APIs and ABIs).

Documentation/networking/af_xdp.rst+16−16

@@ -290,19 +290,19 @@ round-robin example of distributing packets is shown below: · 289 unmodified lines
290	290	#define MAX_SOCKS 16
291	291
292	292	struct {
293		− __uint(type, BPF_MAP_TYPE_XSKMAP);
294		− __uint(max_entries, MAX_SOCKS);
295		− __uint(key_size, sizeof(int));
296		− __uint(value_size, sizeof(int));
	293	+ __uint(type, BPF_MAP_TYPE_XSKMAP);
	294	+ __uint(max_entries, MAX_SOCKS);
	295	+ __uint(key_size, sizeof(int));
	296	+ __uint(value_size, sizeof(int));
297	297	} xsks_map SEC(".maps");
298	298
299	299	static unsigned int rr;
300	300
301	301	SEC("xdp_sock") int xdp_sock_prog(struct xdp_md *ctx)
302	302	{
303		− rr = (rr + 1) & (MAX_SOCKS - 1);
	303	+ rr = (rr + 1) & (MAX_SOCKS - 1);
304	304
305		− return bpf_redirect_map(&xsks_map, rr, XDP_DROP);
	305	+ return bpf_redirect_map(&xsks_map, rr, XDP_DROP);
306	306	}
307	307
308	308	Note, that since there is only a single set of FILL and COMPLETION
@@ -379,7 +379,7 @@ would look like this for the TX path: · 70 unmodified lines
379	379	.. code-block:: c
380	380
381	381	if (xsk_ring_prod__needs_wakeup(&my_tx_ring))
382		− sendto(xsk_socket__fd(xsk_handle), NULL, 0, MSG_DONTWAIT, NULL, 0);
	382	+ sendto(xsk_socket__fd(xsk_handle), NULL, 0, MSG_DONTWAIT, NULL, 0);
383	383
384	384	I.e., only use the syscall if the flag is set.
385	385
@@ -442,9 +442,9 @@ purposes. The supported statistics are shown below: · 56 unmodified lines
442	442	.. code-block:: c
443	443
444	444	struct xdp_statistics {
445		− __u64 rx_dropped; /* Dropped for reasons other than invalid desc */
446		− __u64 rx_invalid_descs; /* Dropped due to invalid descriptor */
447		− __u64 tx_invalid_descs; /* Dropped due to invalid descriptor */
	445	+ __u64 rx_dropped; /* Dropped for reasons other than invalid desc */
	446	+ __u64 rx_invalid_descs; /* Dropped due to invalid descriptor */
	447	+ __u64 tx_invalid_descs; /* Dropped due to invalid descriptor */
448	448	};
449	449
450	450	XDP_OPTIONS getsockopt
@@ -483,15 +483,15 @@ like this: · 32 unmodified lines
483	483	.. code-block:: c
484	484
485	485	// struct xdp_rxtx_ring {
486		− // __u32 *producer;
487		− // __u32 *consumer;
488		− // struct xdp_desc *desc;
	486	+ // __u32 *producer;
	487	+ // __u32 *consumer;
	488	+ // struct xdp_desc *desc;
489	489	// };
490	490
491	491	// struct xdp_umem_ring {
492		− // __u32 *producer;
493		− // __u32 *consumer;
494		− // __u64 *desc;
	492	+ // __u32 *producer;
	493	+ // __u32 *consumer;
	494	+ // __u64 *desc;
495	495	// };
496	496
497	497	// typedef struct xdp_rxtx_ring RING;

arch/x86/net/bpf_jit_comp.c+13−33

@@ -31,7 +31,7 @@ static u8 emit_code(u8 ptr, u32 bytes, unsigned int len) · 30 unmodified lines
31	31	}
32	32
33	33	#define EMIT(bytes, len) \
34		− do { prog = emit_code(prog, bytes, len); cnt += len; } while (0)
	34	+ do { prog = emit_code(prog, bytes, len); } while (0)
35	35
36	36	#define EMIT1(b1) EMIT(b1, 1)
37	37	#define EMIT2(b1, b2) EMIT((b1) + ((b2) << 8), 2)
@@ -239,7 +239,6 @@ struct jit_context { · 201 unmodified lines
239	239	static void push_callee_regs(u8 *pprog, bool callee_regs_used)
240	240	{
241	241	u8 prog = pprog;
242		− int cnt = 0;
243	242
244	243	if (callee_regs_used[0])
245	244	EMIT1(0x53); /* push rbx */
@@ -255,7 +254,6 @@ static void push_callee_regs(u8 *pprog, bool callee_regs_used) · 9 unmodified lines
255	254	static void pop_callee_regs(u8 *pprog, bool callee_regs_used)
256	255	{
257	256	u8 prog = pprog;
258		− int cnt = 0;
259	257
260	258	if (callee_regs_used[3])
261	259	EMIT2(0x41, 0x5F); /* pop r15 */
@@ -277,13 +275,12 @@ static void emit_prologue(u8 **pprog, u32 stack_depth, bool ebpf_from_cbpf, · 15 unmodified lines
277	275	bool tail_call_reachable, bool is_subprog)
278	276	{
279	277	u8 prog = pprog;
280		− int cnt = X86_PATCH_SIZE;
281	278
282	279	/* BPF trampoline can be made to work without these nops,
283	280	* but let's waste 5 bytes for now and optimize later
284	281	*/
285		− memcpy(prog, x86_nops[5], cnt);
286		− prog += cnt;
	282	+ memcpy(prog, x86_nops[5], X86_PATCH_SIZE);
	283	+ prog += X86_PATCH_SIZE;
287	284	if (!ebpf_from_cbpf) {
288	285	if (tail_call_reachable && !is_subprog)
289	286	EMIT2(0x31, 0xC0); /* xor eax, eax */
@@ -303,7 +300,6 @@ static void emit_prologue(u8 **pprog, u32 stack_depth, bool ebpf_from_cbpf, · 13 unmodified lines
303	300	static int emit_patch(u8 *pprog, void func, void *ip, u8 opcode)
304	301	{
305	302	u8 prog = pprog;
306		− int cnt = 0;
307	303	s64 offset;
308	304
309	305	offset = func - (ip + X86_PATCH_SIZE);
@@ -423,7 +419,6 @@ static void emit_bpf_tail_call_indirect(u8 *pprog, bool callee_regs_used, · 113 unmodified lines
423	419	int off1 = 42;
424	420	int off2 = 31;
425	421	int off3 = 9;
426		− int cnt = 0;
427	422
428	423	/* count the additional bytes used for popping callee regs from stack
429	424	* that need to be taken into account for each of the offsets that
@@ -513,7 +508,6 @@ static void emit_bpf_tail_call_direct(struct bpf_jit_poke_descriptor *poke, · 83 unmodified lines
513	508	int pop_bytes = 0;
514	509	int off1 = 20;
515	510	int poke_off;
516		− int cnt = 0;
517	511
518	512	/* count the additional bytes used for popping callee regs to stack
519	513	* that need to be taken into account for jump offset that is used for
@@ -615,7 +609,6 @@ static void emit_mov_imm32(u8 **pprog, bool sign_propagate, · 95 unmodified lines
615	609	{
616	610	u8 prog = pprog;
617	611	u8 b1, b2, b3;
618		− int cnt = 0;
619	612
620	613	/*
621	614	* Optimization: if imm32 is positive, use 'mov %eax, imm32'
@@ -655,7 +648,6 @@ static void emit_mov_imm64(u8 **pprog, u32 dst_reg, · 33 unmodified lines
655	648	const u32 imm32_hi, const u32 imm32_lo)
656	649	{
657	650	u8 prog = pprog;
658		− int cnt = 0;
659	651
660	652	if (is_uimm32(((u64)imm32_hi << 32) \| (u32)imm32_lo)) {
661	653	/*
@@ -678,7 +670,6 @@ static void emit_mov_imm64(u8 **pprog, u32 dst_reg, · 16 unmodified lines
678	670	static void emit_mov_reg(u8 **pprog, bool is64, u32 dst_reg, u32 src_reg)
679	671	{
680	672	u8 prog = pprog;
681		− int cnt = 0;
682	673
683	674	if (is64) {
684	675	/* mov dst, src */
@@ -697,7 +688,6 @@ static void emit_mov_reg(u8 **pprog, bool is64, u32 dst_reg, u32 src_reg) · 12 unmodified lines
697	688	static void emit_insn_suffix(u8 **pprog, u32 ptr_reg, u32 val_reg, int off)
698	689	{
699	690	u8 prog = pprog;
700		− int cnt = 0;
701	691
702	692	if (is_imm8(off)) {
703	693	/* 1-byte signed displacement.
@@ -720,7 +710,6 @@ static void emit_insn_suffix(u8 **pprog, u32 ptr_reg, u32 val_reg, int off) · 16 unmodified lines
720	710	static void maybe_emit_mod(u8 **pprog, u32 dst_reg, u32 src_reg, bool is64)
721	711	{
722	712	u8 prog = pprog;
723		− int cnt = 0;
724	713
725	714	if (is64)
726	715	EMIT1(add_2mod(0x48, dst_reg, src_reg));
@@ -733,7 +722,6 @@ static void maybe_emit_mod(u8 **pprog, u32 dst_reg, u32 src_reg, bool is64) · 6 unmodified lines
733	722	static void emit_ldx(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, int off)
734	723	{
735	724	u8 prog = pprog;
736		− int cnt = 0;
737	725
738	726	switch (size) {
739	727	case BPF_B:
@@ -764,7 +752,6 @@ static void emit_ldx(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, int off) · 24 unmodified lines
764	752	static void emit_stx(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, int off)
765	753	{
766	754	u8 prog = pprog;
767		− int cnt = 0;
768	755
769	756	switch (size) {
770	757	case BPF_B:
@@ -799,7 +786,6 @@ static int emit_atomic(u8 **pprog, u8 atomic_op, · 28 unmodified lines
799	786	u32 dst_reg, u32 src_reg, s16 off, u8 bpf_size)
800	787	{
801	788	u8 prog = pprog;
802		− int cnt = 0;
803	789
804	790	EMIT1(0xF0); /* lock prefix */
805	791
@@ -869,10 +855,10 @@ static void detect_reg_usage(struct bpf_insn *insn, int insn_cnt, · 63 unmodified lines
869	855	}
870	856	}
871	857
872		−static int emit_nops(u8 **pprog, int len)
	858	+static void emit_nops(u8 **pprog, int len)
873	859	{
874	860	u8 prog = pprog;
875		− int i, noplen, cnt = 0;
	861	+ int i, noplen;
876	862
877	863	while (len > 0) {
878	864	noplen = len;
@@ -886,8 +872,6 @@ static int emit_nops(u8 **pprog, int len) · 7 unmodified lines
886	872	}
887	873
888	874	*pprog = prog;
889		−
890		− return cnt;
891	875	}
892	876
893	877	#define INSN_SZ_DIFF (((addrs[i] - addrs[i - 1]) - (prog - temp)))
@@ -902,7 +886,7 @@ static int do_jit(struct bpf_prog bpf_prog, int addrs, u8 *image, · 8 unmodified lines
902	886	bool tail_call_seen = false;
903	887	bool seen_exit = false;
904	888	u8 temp[BPF_MAX_INSN_SIZE + BPF_INSN_SAFETY];
905		− int i, cnt = 0, excnt = 0;
	889	+ int i, excnt = 0;
906	890	int ilen, proglen = 0;
907	891	u8 *prog = temp;
908	892	int err;
@@ -1297,7 +1281,7 @@ st: if (is_imm8(insn->off)) · 388 unmodified lines
1297	1281	emit_ldx(&prog, BPF_SIZE(insn->code), dst_reg, src_reg, insn->off);
1298	1282	if (BPF_MODE(insn->code) == BPF_PROBE_MEM) {
1299	1283	struct exception_table_entry *ex;
1300		− u8 *_insn = image + proglen;
	1284	+ u8 *_insn = image + proglen + (start_of_ldx - temp);
1301	1285	s64 delta;
1302	1286
1303	1287	/* populate jmp_offset for JMP above */
@@ -1576,7 +1560,7 @@ emit_cond_jmp: /* Convert BPF opcode to x86 */ · 272 unmodified lines
1576	1560	nops);
1577	1561	return -EFAULT;
1578	1562	}
1579		− cnt += emit_nops(&prog, nops);
	1563	+ emit_nops(&prog, nops);
1580	1564	}
1581	1565	EMIT2(jmp_cond, jmp_offset);
1582	1566	} else if (is_simm32(jmp_offset)) {
@@ -1622,7 +1606,7 @@ emit_cond_jmp: /* Convert BPF opcode to x86 */ · 39 unmodified lines
1622	1606	nops);
1623	1607	return -EFAULT;
1624	1608	}
1625		− cnt += emit_nops(&prog, nops);
	1609	+ emit_nops(&prog, nops);
1626	1610	}
1627	1611	break;
1628	1612	}
@@ -1647,7 +1631,7 @@ emit_jmp: · 18 unmodified lines
1647	1631	nops);
1648	1632	return -EFAULT;
1649	1633	}
1650		− cnt += emit_nops(&prog, INSN_SZ_DIFF - 2);
	1634	+ emit_nops(&prog, INSN_SZ_DIFF - 2);
1651	1635	}
1652	1636	EMIT2(0xEB, jmp_offset);
1653	1637	} else if (is_simm32(jmp_offset)) {
@@ -1754,7 +1738,6 @@ static int invoke_bpf_prog(const struct btf_func_model m, u8 *pprog, · 100 unmodified lines
1754	1738	{
1755	1739	u8 prog = pprog;
1756	1740	u8 *jmp_insn;
1757		− int cnt = 0;
1758	1741
1759	1742	/* arg1: mov rdi, progs[i] */
1760	1743	emit_mov_imm64(&prog, BPF_REG_1, (long) p >> 32, (u32) (long) p);
@@ -1822,7 +1805,6 @@ static void emit_align(u8 **pprog, u32 align) · 61 unmodified lines
1822	1805	static int emit_cond_near_jump(u8 *pprog, void func, void *ip, u8 jmp_cond)
1823	1806	{
1824	1807	u8 prog = pprog;
1825		− int cnt = 0;
1826	1808	s64 offset;
1827	1809
1828	1810	offset = func - (ip + 2 + 4);
@@ -1854,7 +1836,7 @@ static int invoke_bpf_mod_ret(const struct btf_func_model m, u8 *pprog, · 25 unmodified lines
1854	1836	u8 **branches)
1855	1837	{
1856	1838	u8 prog = pprog;
1857		− int i, cnt = 0;
	1839	+ int i;
1858	1840
1859	1841	/* The first fmod_ret program will receive a garbage return value.
1860	1842	* Set this to 0 to avoid confusing the program.
@@ -1950,7 +1932,7 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image im, void image, void *i · 89 unmodified lines
1950	1932	struct bpf_tramp_progs *tprogs,
1951	1933	void *orig_call)
1952	1934	{
1953		− int ret, i, cnt = 0, nr_args = m->nr_args;
	1935	+ int ret, i, nr_args = m->nr_args;
1954	1936	int stack_size = nr_args * 8;
1955	1937	struct bpf_tramp_progs *fentry = &tprogs[BPF_TRAMP_FENTRY];
1956	1938	struct bpf_tramp_progs *fexit = &tprogs[BPF_TRAMP_FEXIT];
@@ -2095,8 +2077,6 @@ static int emit_fallback_jump(u8 **pprog) · 138 unmodified lines
2095	2077	*/
2096	2078	err = emit_jump(&prog, __x86_indirect_thunk_rdx, prog);
2097	2079	#else
2098		− int cnt = 0;
2099		−
2100	2080	EMIT2(0xFF, 0xE2); /* jmp rdx */
2101	2081	#endif
2102	2082	*pprog = prog;
@@ -2106,7 +2086,7 @@ static int emit_fallback_jump(u8 **pprog) · 3 unmodified lines
2106	2086	static int emit_bpf_dispatcher(u8 *pprog, int a, int b, s64 progs)
2107	2087	{
2108	2088	u8 jg_reloc, prog = *pprog;
2109		− int pivot, err, jg_bytes = 1, cnt = 0;
	2089	+ int pivot, err, jg_bytes = 1;
2110	2090	s64 jg_offset;
2111	2091
2112	2092	if (a == b) {

drivers/media/rc/bpf-lirc.c+2−1

@@ -326,7 +326,8 @@ int lirc_prog_query(const union bpf_attr attr, union bpf_attr __user uattr) · 325 unmodified lines
326	326	}
327	327
328	328	if (attr->query.prog_cnt != 0 && prog_ids && cnt)
329		− ret = bpf_prog_array_copy_to_user(progs, prog_ids, cnt);
	329	+ ret = bpf_prog_array_copy_to_user(progs, prog_ids,
	330	+ attr->query.prog_cnt);
330	331
331	332	unlock:
332	333	mutex_unlock(&ir_raw_handler_lock);

drivers/net/ethernet/amazon/ena/ena_netdev.c+0−3

@@ -384,7 +384,6 @@ static int ena_xdp_execute(struct ena_ring rx_ring, struct xdp_buff xdp) · 383 unmodified lines
384	384	struct xdp_frame *xdpf;
385	385	u64 *xdp_stat;
386	386
387		− rcu_read_lock();
388	387	xdp_prog = READ_ONCE(rx_ring->xdp_bpf_prog);
389	388
390	389	if (!xdp_prog)
@@ -441,8 +440,6 @@ static int ena_xdp_execute(struct ena_ring rx_ring, struct xdp_buff xdp) · 50 unmodified lines
441	440
442	441	ena_increase_stat(xdp_stat, 1, &rx_ring->syncp);
443	442	out:
444		− rcu_read_unlock();
445		−
446	443	return verdict;
447	444	}
448	445

drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c+0−2

@@ -138,9 +138,7 @@ bool bnxt_rx_xdp(struct bnxt bp, struct bnxt_rx_ring_info rxr, u16 cons, · 137 unmodified lines
138	138	xdp_prepare_buff(&xdp, data_ptr - offset, offset, len, false);
139	139	orig_data = xdp.data;
140	140
141		− rcu_read_lock();
142	141	act = bpf_prog_run_xdp(xdp_prog, &xdp);
143		− rcu_read_unlock();
144	142
145	143	tx_avail = bnxt_tx_avail(bp, txr);
146	144	/* If the tx ring is not full, we must not update the rx producer yet

drivers/net/ethernet/cavium/thunder/nicvf_main.c+0−2

@@ -555,9 +555,7 @@ static inline bool nicvf_xdp_rx(struct nicvf nic, struct bpf_prog prog, · 554 unmodified lines
555	555	xdp_prepare_buff(&xdp, hard_start, data - hard_start, len, false);
556	556	orig_data = xdp.data;
557	557
558		− rcu_read_lock();
559	558	action = bpf_prog_run_xdp(prog, &xdp);
560		− rcu_read_unlock();
561	559
562	560	len = xdp.data_end - xdp.data;
563	561	/* Check if XDP program has changed headers */

drivers/net/ethernet/freescale/dpaa/dpaa_eth.c+1−7

@@ -2558,13 +2558,9 @@ static u32 dpaa_run_xdp(struct dpaa_priv priv, struct qm_fd fd, void *vaddr, · 2557 unmodified lines
2558	2558	u32 xdp_act;
2559	2559	int err;
2560	2560
2561		− rcu_read_lock();
2562		−
2563	2561	xdp_prog = READ_ONCE(priv->xdp_prog);
2564		− if (!xdp_prog) {
2565		− rcu_read_unlock();
	2562	+ if (!xdp_prog)
2566	2563	return XDP_PASS;
2567		− }
2568	2564
2569	2565	xdp_init_buff(&xdp, DPAA_BP_RAW_SIZE - DPAA_TX_PRIV_DATA_SIZE,
2570	2566	&dpaa_fq->xdp_rxq);
@@ -2638,8 +2634,6 @@ static u32 dpaa_run_xdp(struct dpaa_priv priv, struct qm_fd fd, void *vaddr, · 67 unmodified lines
2638	2634	break;
2639	2635	}
2640	2636
2641		− rcu_read_unlock();
2642		−
2643	2637	return xdp_act;
2644	2638	}
2645	2639

drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c+0−3

@@ -352,8 +352,6 @@ static u32 dpaa2_eth_run_xdp(struct dpaa2_eth_priv *priv, · 351 unmodified lines
352	352	u32 xdp_act = XDP_PASS;
353	353	int err, offset;
354	354
355		− rcu_read_lock();
356		−
357	355	xdp_prog = READ_ONCE(ch->xdp.prog);
358	356	if (!xdp_prog)
359	357	goto out;
@@ -414,7 +412,6 @@ static u32 dpaa2_eth_run_xdp(struct dpaa2_eth_priv *priv, · 54 unmodified lines
414	412
415	413	ch->xdp.res \|= xdp_act;
416	414	out:
417		− rcu_read_unlock();
418	415	return xdp_act;
419	416	}
420	417

drivers/net/ethernet/intel/i40e/i40e_txrx.c+0−2

@@ -2298,7 +2298,6 @@ static int i40e_run_xdp(struct i40e_ring rx_ring, struct xdp_buff xdp) · 2297 unmodified lines
2298	2298	struct bpf_prog *xdp_prog;
2299	2299	u32 act;
2300	2300
2301		− rcu_read_lock();
2302	2301	xdp_prog = READ_ONCE(rx_ring->xdp_prog);
2303	2302
2304	2303	if (!xdp_prog)
@@ -2334,7 +2333,6 @@ out_failure: · 29 unmodified lines
2334	2333	break;
2335	2334	}
2336	2335	xdp_out:
2337		− rcu_read_unlock();
2338	2336	return result;
2339	2337	}
2340	2338

drivers/net/ethernet/intel/i40e/i40e_xsk.c+0−3

@@ -153,7 +153,6 @@ static int i40e_run_xdp_zc(struct i40e_ring rx_ring, struct xdp_buff xdp) · 152 unmodified lines
153	153	struct bpf_prog *xdp_prog;
154	154	u32 act;
155	155
156		− rcu_read_lock();
157	156	/* NB! xdp_prog will always be !NULL, due to the fact that
158	157	* this path is enabled by setting an XDP program.
159	158	*/
@@ -164,7 +163,6 @@ static int i40e_run_xdp_zc(struct i40e_ring rx_ring, struct xdp_buff xdp) · 4 unmodified lines
164	163	err = xdp_do_redirect(rx_ring->netdev, xdp, xdp_prog);
165	164	if (err)
166	165	goto out_failure;
167		− rcu_read_unlock();
168	166	return I40E_XDP_REDIR;
169	167	}
170	168
@@ -188,7 +186,6 @@ out_failure: · 17 unmodified lines
188	186	result = I40E_XDP_CONSUMED;
189	187	break;
190	188	}
191		− rcu_read_unlock();
192	189	return result;
193	190	}
194	191

drivers/net/ethernet/intel/ice/ice_txrx.c+1−5

@@ -1140,15 +1140,11 @@ int ice_clean_rx_irq(struct ice_ring *rx_ring, int budget) · 1139 unmodified lines
1140	1140	xdp.frame_sz = ice_rx_frame_truesize(rx_ring, size);
1141	1141	#endif
1142	1142
1143		− rcu_read_lock();
1144	1143	xdp_prog = READ_ONCE(rx_ring->xdp_prog);
1145		− if (!xdp_prog) {
1146		− rcu_read_unlock();
	1144	+ if (!xdp_prog)
1147	1145	goto construct_skb;
1148		− }
1149	1146
1150	1147	xdp_res = ice_run_xdp(rx_ring, &xdp, xdp_prog);
1151		− rcu_read_unlock();
1152	1148	if (!xdp_res)
1153	1149	goto construct_skb;
1154	1150	if (xdp_res & (ICE_XDP_TX \| ICE_XDP_REDIR)) {

drivers/net/ethernet/intel/ice/ice_xsk.c+0−3

@@ -466,7 +466,6 @@ ice_run_xdp_zc(struct ice_ring rx_ring, struct xdp_buff xdp) · 465 unmodified lines
466	466	struct ice_ring *xdp_ring;
467	467	u32 act;
468	468
469		− rcu_read_lock();
470	469	/* ZC patch is enabled only when XDP program is set,
471	470	* so here it can not be NULL
472	471	*/
@@ -478,7 +477,6 @@ ice_run_xdp_zc(struct ice_ring rx_ring, struct xdp_buff xdp) · 5 unmodified lines
478	477	err = xdp_do_redirect(rx_ring->netdev, xdp, xdp_prog);
479	478	if (err)
480	479	goto out_failure;
481		− rcu_read_unlock();
482	480	return ICE_XDP_REDIR;
483	481	}
484	482
@@ -503,7 +501,6 @@ out_failure: · 18 unmodified lines
503	501	break;
504	502	}
505	503
506		− rcu_read_unlock();
507	504	return result;
508	505	}
509	506

drivers/net/ethernet/intel/igb/igb_main.c+0−2

@@ -8381,7 +8381,6 @@ static struct sk_buff igb_run_xdp(struct igb_adapter adapter, · 8380 unmodified lines
8381	8381	struct bpf_prog *xdp_prog;
8382	8382	u32 act;
8383	8383
8384		− rcu_read_lock();
8385	8384	xdp_prog = READ_ONCE(rx_ring->xdp_prog);
8386	8385
8387	8386	if (!xdp_prog)
@@ -8416,7 +8415,6 @@ out_failure: · 28 unmodified lines
8416	8415	break;
8417	8416	}
8418	8417	xdp_out:
8419		− rcu_read_unlock();
8420	8418	return ERR_PTR(-result);
8421	8419	}
8422	8420

drivers/net/ethernet/intel/igc/igc_main.c+2−5

@@ -2240,18 +2240,15 @@ static struct sk_buff igc_xdp_run_prog(struct igc_adapter adapter, · 2239 unmodified lines
2240	2240	struct bpf_prog *prog;
2241	2241	int res;
2242	2242
2243		− rcu_read_lock();
2244		−
2245	2243	prog = READ_ONCE(adapter->xdp_prog);
2246	2244	if (!prog) {
2247	2245	res = IGC_XDP_PASS;
2248		− goto unlock;
	2246	+ goto out;
2249	2247	}
2250	2248
2251	2249	res = __igc_xdp_run_prog(adapter, prog, xdp);
2252	2250
2253		−unlock:
2254		− rcu_read_unlock();
	2251	+out:
2255	2252	return ERR_PTR(-res);
2256	2253	}
2257	2254

drivers/net/ethernet/intel/ixgbe/ixgbe_main.c+0−2

@@ -2199,7 +2199,6 @@ static struct sk_buff ixgbe_run_xdp(struct ixgbe_adapter adapter, · 2198 unmodified lines
2199	2199	struct xdp_frame *xdpf;
2200	2200	u32 act;
2201	2201
2202		− rcu_read_lock();
2203	2202	xdp_prog = READ_ONCE(rx_ring->xdp_prog);
2204	2203
2205	2204	if (!xdp_prog)
@@ -2237,7 +2236,6 @@ out_failure: · 31 unmodified lines
2237	2236	break;
2238	2237	}
2239	2238	xdp_out:
2240		− rcu_read_unlock();
2241	2239	return ERR_PTR(-result);
2242	2240	}
2243	2241

drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c+0−3

@@ -100,7 +100,6 @@ static int ixgbe_run_xdp_zc(struct ixgbe_adapter *adapter, · 99 unmodified lines
100	100	struct xdp_frame *xdpf;
101	101	u32 act;
102	102
103		− rcu_read_lock();
104	103	xdp_prog = READ_ONCE(rx_ring->xdp_prog);
105	104	act = bpf_prog_run_xdp(xdp_prog, xdp);
106	105
@@ -108,7 +107,6 @@ static int ixgbe_run_xdp_zc(struct ixgbe_adapter *adapter, · 1 unmodified line
108	107	err = xdp_do_redirect(rx_ring->netdev, xdp, xdp_prog);
109	108	if (err)
110	109	goto out_failure;
111		− rcu_read_unlock();
112	110	return IXGBE_XDP_REDIR;
113	111	}
114	112
@@ -134,7 +132,6 @@ out_failure: · 19 unmodified lines
134	132	result = IXGBE_XDP_CONSUMED;
135	133	break;
136	134	}
137		− rcu_read_unlock();
138	135	return result;
139	136	}
140	137

drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c+0−2

@@ -1054,7 +1054,6 @@ static struct sk_buff ixgbevf_run_xdp(struct ixgbevf_adapter adapter, · 1053 unmodified lines
1054	1054	struct bpf_prog *xdp_prog;
1055	1055	u32 act;
1056	1056
1057		− rcu_read_lock();
1058	1057	xdp_prog = READ_ONCE(rx_ring->xdp_prog);
1059	1058
1060	1059	if (!xdp_prog)
@@ -1082,7 +1081,6 @@ out_failure: · 21 unmodified lines
1082	1081	break;
1083	1082	}
1084	1083	xdp_out:
1085		− rcu_read_unlock();
1086	1084	return ERR_PTR(-result);
1087	1085	}
1088	1086

drivers/net/ethernet/marvell/mvneta.c+0−2

@@ -2369,7 +2369,6 @@ static int mvneta_rx_swbm(struct napi_struct *napi, · 2368 unmodified lines
2369	2369	/* Get number of received packets */
2370	2370	rx_todo = mvneta_rxq_busy_desc_num_get(pp, rxq);
2371	2371
2372		− rcu_read_lock();
2373	2372	xdp_prog = READ_ONCE(pp->xdp_prog);
2374	2373
2375	2374	/* Fairness NAPI loop */
@@ -2447,7 +2446,6 @@ next: · 71 unmodified lines
2447	2446	xdp_buf.data_hard_start = NULL;
2448	2447	sinfo.nr_frags = 0;
2449	2448	}
2450		− rcu_read_unlock();
2451	2449
2452	2450	if (xdp_buf.data_hard_start)
2453	2451	mvneta_xdp_put_buff(pp, rxq, &xdp_buf, &sinfo, -1);

drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c+0−4

@@ -3877,8 +3877,6 @@ static int mvpp2_rx(struct mvpp2_port port, struct napi_struct napi, · 3876 unmodified lines
3877	3877	int rx_done = 0;
3878	3878	u32 xdp_ret = 0;
3879	3879
3880		− rcu_read_lock();
3881		−
3882	3880	xdp_prog = READ_ONCE(port->xdp_prog);
3883	3881
3884	3882	/* Get number of received packets and clamp the to-do */
@@ -4024,8 +4022,6 @@ err_drop_frame: · 139 unmodified lines
4024	4022	mvpp2_bm_pool_put(port, pool, dma_addr, phys_addr);
4025	4023	}
4026	4024
4027		− rcu_read_unlock();
4028		−
4029	4025	if (xdp_ret & MVPP2_XDP_REDIR)
4030	4026	xdp_do_flush_map();
4031	4027

drivers/net/ethernet/mellanox/mlx4/en_rx.c+2−6

@@ -679,9 +679,7 @@ int mlx4_en_process_rx_cq(struct net_device dev, struct mlx4_en_cq cq, int bud · 678 unmodified lines
679	679
680	680	ring = priv->rx_ring[cq_ring];
681	681
682		− /* Protect accesses to: ring->xdp_prog, priv->mac_hash list */
683		− rcu_read_lock();
684		− xdp_prog = rcu_dereference(ring->xdp_prog);
	682	+ xdp_prog = rcu_dereference_bh(ring->xdp_prog);
685	683	xdp_init_buff(&xdp, priv->frag_info[0].frag_stride, &ring->xdp_rxq);
686	684	doorbell_pending = false;
687	685
@@ -744,7 +742,7 @@ int mlx4_en_process_rx_cq(struct net_device dev, struct mlx4_en_cq cq, int bud · 56 unmodified lines
744	742	/* Drop the packet, since HW loopback-ed it */
745	743	mac_hash = ethh->h_source[MLX4_EN_MAC_HASH_IDX];
746	744	bucket = &priv->mac_hash[mac_hash];
747		− hlist_for_each_entry_rcu(entry, bucket, hlist) {
	745	+ hlist_for_each_entry_rcu_bh(entry, bucket, hlist) {
748	746	if (ether_addr_equal_64bits(entry->mac,
749	747	ethh->h_source))
750	748	goto next;
@@ -899,8 +897,6 @@ next: · 148 unmodified lines
899	897	break;
900	898	}
901	899
902		− rcu_read_unlock();
903		−
904	900	if (likely(polled)) {
905	901	if (doorbell_pending) {
906	902	priv->tx_cq[TX_XDP][cq_ring]->xdp_busy = true;

drivers/net/ethernet/netronome/nfp/nfp_net_common.c+0−2

@@ -1819,7 +1819,6 @@ static int nfp_net_rx(struct nfp_net_rx_ring *rx_ring, int budget) · 1818 unmodified lines
1819	1819	struct xdp_buff xdp;
1820	1820	int idx;
1821	1821
1822		− rcu_read_lock();
1823	1822	xdp_prog = READ_ONCE(dp->xdp_prog);
1824	1823	true_bufsz = xdp_prog ? PAGE_SIZE : dp->fl_bufsz;
1825	1824	xdp_init_buff(&xdp, PAGE_SIZE - NFP_NET_RX_BUF_HEADROOM,
@@ -2036,7 +2035,6 @@ static int nfp_net_rx(struct nfp_net_rx_ring *rx_ring, int budget) · 210 unmodified lines
2036	2035	if (!nfp_net_xdp_complete(tx_ring))
2037	2036	pkts_polled = budget;
2038	2037	}
2039		− rcu_read_unlock();
2040	2038
2041	2039	return pkts_polled;
2042	2040	}

drivers/net/ethernet/qlogic/qede/qede_fp.c+0−6

@@ -1089,13 +1089,7 @@ static bool qede_rx_xdp(struct qede_dev *edev, · 1088 unmodified lines
1089	1089	xdp_prepare_buff(&xdp, page_address(bd->data), *data_offset,
1090	1090	*len, false);
1091	1091
1092		− /* Queues always have a full reset currently, so for the time
1093		− * being until there's atomic program replace just mark read
1094		− * side for map helpers.
1095		− */
1096		− rcu_read_lock();
1097	1092	act = bpf_prog_run_xdp(prog, &xdp);
1098		− rcu_read_unlock();
1099	1093
1100	1094	/* Recalculate, as XDP might have changed the headers */
1101	1095	*data_offset = xdp.data - xdp.data_hard_start;

drivers/net/ethernet/sfc/rx.c+2−7

@@ -260,18 +260,14 @@ static bool efx_do_xdp(struct efx_nic efx, struct efx_channel channel, · 259 unmodified lines
260	260	s16 offset;
261	261	int err;
262	262
263		− rcu_read_lock();
264		− xdp_prog = rcu_dereference(efx->xdp_prog);
265		− if (!xdp_prog) {
266		− rcu_read_unlock();
	263	+ xdp_prog = rcu_dereference_bh(efx->xdp_prog);
	264	+ if (!xdp_prog)
267	265	return true;
268		− }
269	266
270	267	rx_queue = efx_channel_get_rx_queue(channel);
271	268
272	269	if (unlikely(channel->rx_pkt_n_frags > 1)) {
273	270	/* We can't do XDP on fragmented packets - drop. */
274		− rcu_read_unlock();
275	271	efx_free_rx_buffers(rx_queue, rx_buf,
276	272	channel->rx_pkt_n_frags);
277	273	if (net_ratelimit())
@@ -296,7 +292,6 @@ static bool efx_do_xdp(struct efx_nic efx, struct efx_channel channel, · 18 unmodified lines
296	292	rx_buf->len, false);
297	293
298	294	xdp_act = bpf_prog_run_xdp(xdp_prog, &xdp);
299		− rcu_read_unlock();
300	295
301	296	offset = (u8 )xdp.data - ehp;
302	297

drivers/net/ethernet/socionext/netsec.c+0−3

@@ -958,7 +958,6 @@ static int netsec_process_rx(struct netsec_priv *priv, int budget) · 957 unmodified lines
958	958
959	959	xdp_init_buff(&xdp, PAGE_SIZE, &dring->xdp_rxq);
960	960
961		− rcu_read_lock();
962	961	xdp_prog = READ_ONCE(priv->xdp_prog);
963	962	dma_dir = page_pool_get_dma_dir(dring->page_pool);
964	963
@@ -1069,8 +1068,6 @@ next: · 104 unmodified lines
1069	1068	}
1070	1069	netsec_finalize_xdp_rx(priv, xdp_act, xdp_xmit);
1071	1070
1072		− rcu_read_unlock();
1073		−
1074	1071	return done;
1075	1072	}
1076	1073

drivers/net/ethernet/stmicro/stmmac/stmmac_main.c+2−8

@@ -4651,7 +4651,6 @@ static int stmmac_xdp_xmit_back(struct stmmac_priv *priv, · 4650 unmodified lines
4651	4651	return res;
4652	4652	}
4653	4653
4654		−/* This function assumes rcu_read_lock() is held by the caller. */
4655	4654	static int __stmmac_xdp_run_prog(struct stmmac_priv *priv,
4656	4655	struct bpf_prog *prog,
4657	4656	struct xdp_buff *xdp)
@@ -4693,17 +4692,14 @@ static struct sk_buff stmmac_xdp_run_prog(struct stmmac_priv priv, · 35 unmodified lines
4693	4692	struct bpf_prog *prog;
4694	4693	int res;
4695	4694
4696		− rcu_read_lock();
4697		−
4698	4695	prog = READ_ONCE(priv->xdp_prog);
4699	4696	if (!prog) {
4700	4697	res = STMMAC_XDP_PASS;
4701		− goto unlock;
	4698	+ goto out;
4702	4699	}
4703	4700
4704	4701	res = __stmmac_xdp_run_prog(priv, prog, xdp);
4705		−unlock:
4706		− rcu_read_unlock();
	4702	+out:
4707	4703	return ERR_PTR(-res);
4708	4704	}
4709	4705
@@ -4973,10 +4969,8 @@ read_again: · 263 unmodified lines
4973	4969	buf->xdp->data_end = buf->xdp->data + buf1_len;
4974	4970	xsk_buff_dma_sync_for_cpu(buf->xdp, rx_q->xsk_pool);
4975	4971
4976		− rcu_read_lock();
4977	4972	prog = READ_ONCE(priv->xdp_prog);
4978	4973	res = __stmmac_xdp_run_prog(priv, prog, buf->xdp);
4979		− rcu_read_unlock();
4980	4974
4981	4975	switch (res) {
4982	4976	case STMMAC_XDP_PASS:

drivers/net/ethernet/ti/cpsw_priv.c+2−8

@@ -1328,13 +1328,9 @@ int cpsw_run_xdp(struct cpsw_priv priv, int ch, struct xdp_buff xdp, · 1327 unmodified lines
1328	1328	struct bpf_prog *prog;
1329	1329	u32 act;
1330	1330
1331		− rcu_read_lock();
1332		−
1333	1331	prog = READ_ONCE(priv->xdp_prog);
1334		− if (!prog) {
1335		− ret = CPSW_XDP_PASS;
1336		− goto out;
1337		− }
	1332	+ if (!prog)
	1333	+ return CPSW_XDP_PASS;
1338	1334
1339	1335	act = bpf_prog_run_xdp(prog, xdp);
1340	1336	/* XDP prog might have changed packet data and boundaries */
@@ -1378,10 +1374,8 @@ int cpsw_run_xdp(struct cpsw_priv priv, int ch, struct xdp_buff xdp, · 37 unmodified lines
1378	1374	ndev->stats.rx_bytes += *len;
1379	1375	ndev->stats.rx_packets++;
1380	1376	out:
1381		− rcu_read_unlock();
1382	1377	return ret;
1383	1378	drop:
1384		− rcu_read_unlock();
1385	1379	page_pool_recycle_direct(cpsw->page_pool[ch], page);
1386	1380	return ret;
1387	1381	}

include/linux/filter.h+3−5

@@ -763,11 +763,9 @@ DECLARE_BPF_DISPATCHER(xdp) · 762 unmodified lines
763	763	static __always_inline u32 bpf_prog_run_xdp(const struct bpf_prog *prog,
764	764	struct xdp_buff *xdp)
765	765	{
766		− /* Caller needs to hold rcu_read_lock() (!), otherwise program
767		− * can be released while still running, or map elements could be
768		− * freed early while still having concurrent users. XDP fastpath
769		− * already takes rcu_read_lock() when fetching the program, so
770		− * it's not necessary here anymore.
	766	+ /* Driver XDP hooks are invoked within a single NAPI poll cycle and thus
	767	+ * under local_bh_disable(), which provides the needed RCU protection
	768	+ * for accessing map entries.
771	769	*/
772	770	return __BPF_PROG_RUN(prog, xdp, BPF_DISPATCHER_FUNC(xdp));
773	771	}

include/linux/rcupdate.h+14−0

@@ -363,6 +363,20 @@ static inline void rcu_preempt_sleep_check(void) { } · 362 unmodified lines
363	363	#define rcu_check_sparse(p, space)
364	364	#endif /* #else #ifdef __CHECKER__ */
365	365
	366	+/**
	367	+ * unrcu_pointer - mark a pointer as not being RCU protected
	368	+ * @p: pointer needing to lose its __rcu property
	369	+ *
	370	+ * Converts @p from an __rcu pointer to a __kernel pointer.
	371	+ * This allows an __rcu pointer to be used with xchg() and friends.
	372	+ */
	373	+#define unrcu_pointer(p) \
	374	+({ \
	375	+ typeof(p) _________p1 = (typeof(p) __force)(p); \
	376	+ rcu_check_sparse(p, __rcu); \
	377	+ ((typeof(p) __force __kernel )(_________p1)); \
	378	+})
	379	+
366	380	#define __rcu_access_pointer(p, space) \
367	381	({ \
368	382	typeof(p) _________p1 = (typeof(p) __force)READ_ONCE(p); \

include/net/xdp_sock.h+1−1

@@ -37,7 +37,7 @@ struct xdp_umem { · 36 unmodified lines
37	37	struct xsk_map {
38	38	struct bpf_map map;
39	39	spinlock_t lock; /* Synchronize map updates */
40		− struct xdp_sock *xsk_map[];
	40	+ struct xdp_sock __rcu *xsk_map[];
41	41	};
42	42
43	43	struct xdp_sock {

kernel/bpf/cpumap.c+9−4

@@ -74,7 +74,7 @@ struct bpf_cpu_map_entry { · 73 unmodified lines
74	74	struct bpf_cpu_map {
75	75	struct bpf_map map;
76	76	/* Below members specific for map type */
77		− struct bpf_cpu_map_entry **cpu_map;
	77	+ struct bpf_cpu_map_entry __rcu **cpu_map;
78	78	};
79	79
80	80	static DEFINE_PER_CPU(struct list_head, cpu_map_flush_list);
@@ -469,7 +469,7 @@ static void __cpu_map_entry_replace(struct bpf_cpu_map *cmap, · 388 unmodified lines
469	469	{
470	470	struct bpf_cpu_map_entry *old_rcpu;
471	471
472		− old_rcpu = xchg(&cmap->cpu_map[key_cpu], rcpu);
	472	+ old_rcpu = unrcu_pointer(xchg(&cmap->cpu_map[key_cpu], RCU_INITIALIZER(rcpu)));
473	473	if (old_rcpu) {
474	474	call_rcu(&old_rcpu->rcu, __cpu_map_entry_free);
475	475	INIT_WORK(&old_rcpu->kthread_stop_wq, cpu_map_kthread_stop);
@@ -551,7 +551,7 @@ static void cpu_map_free(struct bpf_map *map) · 75 unmodified lines
551	551	for (i = 0; i < cmap->map.max_entries; i++) {
552	552	struct bpf_cpu_map_entry *rcpu;
553	553
554		− rcpu = READ_ONCE(cmap->cpu_map[i]);
	554	+ rcpu = rcu_dereference_raw(cmap->cpu_map[i]);
555	555	if (!rcpu)
556	556	continue;
557	557
@@ -562,6 +562,10 @@ static void cpu_map_free(struct bpf_map *map) · 4 unmodified lines
562	562	kfree(cmap);
563	563	}
564	564
	565	+/* Elements are kept alive by RCU; either by rcu_read_lock() (from syscall) or
	566	+ * by local_bh_disable() (from XDP calls inside NAPI). The
	567	+ * rcu_read_lock_bh_held() below makes lockdep accept both.
	568	+ */
565	569	static void __cpu_map_lookup_elem(struct bpf_map map, u32 key)
566	570	{
567	571	struct bpf_cpu_map *cmap = container_of(map, struct bpf_cpu_map, map);
@@ -570,7 +574,8 @@ static void __cpu_map_lookup_elem(struct bpf_map map, u32 key) · 2 unmodified lines
570	574	if (key >= map->max_entries)
571	575	return NULL;
572	576
573		− rcpu = READ_ONCE(cmap->cpu_map[key]);
	577	+ rcpu = rcu_dereference_check(cmap->cpu_map[key],
	578	+ rcu_read_lock_bh_held());
574	579	return rcpu;
575	580	}
576	581

kernel/bpf/devmap.c+21−28

@@ -73,7 +73,7 @@ struct bpf_dtab_netdev { · 72 unmodified lines
73	73
74	74	struct bpf_dtab {
75	75	struct bpf_map map;
76		− struct bpf_dtab_netdev *netdev_map; / DEVMAP type only */
	76	+ struct bpf_dtab_netdev __rcu *netdev_map; / DEVMAP type only */
77	77	struct list_head list;
78	78
79	79	/* these are only used for DEVMAP_HASH type maps */
@@ -226,7 +226,7 @@ static void dev_map_free(struct bpf_map *map) · 146 unmodified lines
226	226	for (i = 0; i < dtab->map.max_entries; i++) {
227	227	struct bpf_dtab_netdev *dev;
228	228
229		− dev = dtab->netdev_map[i];
	229	+ dev = rcu_dereference_raw(dtab->netdev_map[i]);
230	230	if (!dev)
231	231	continue;
232	232
@@ -259,6 +259,10 @@ static int dev_map_get_next_key(struct bpf_map map, void key, void *next_key) · 26 unmodified lines
259	259	return 0;
260	260	}
261	261
	262	+/* Elements are kept alive by RCU; either by rcu_read_lock() (from syscall) or
	263	+ * by local_bh_disable() (from XDP calls inside NAPI). The
	264	+ * rcu_read_lock_bh_held() below makes lockdep accept both.
	265	+ */
262	266	static void __dev_map_hash_lookup_elem(struct bpf_map map, u32 key)
263	267	{
264	268	struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map);
@@ -410,15 +414,9 @@ out: · 145 unmodified lines
410	414	trace_xdp_devmap_xmit(bq->dev_rx, dev, sent, cnt - sent, err);
411	415	}
412	416
413		−/* __dev_flush is called from xdp_do_flush() which _must_ be signaled
414		− * from the driver before returning from its napi->poll() routine. The poll()
415		− * routine is called either from busy_poll context or net_rx_action signaled
416		− * from NET_RX_SOFTIRQ. Either way the poll routine must complete before the
417		− * net device can be torn down. On devmap tear down we ensure the flush list
418		− * is empty before completing to ensure all flush operations have completed.
419		− * When drivers update the bpf program they may need to ensure any flush ops
420		− * are also complete. Using synchronize_rcu or call_rcu will suffice for this
421		− * because both wait for napi context to exit.
	417	+/* __dev_flush is called from xdp_do_flush() which _must_ be signalled from the
	418	+ * driver before returning from its napi->poll() routine. See the comment above
	419	+ * xdp_do_flush() in filter.c.
422	420	*/
423	421	void __dev_flush(void)
424	422	{
@@ -433,9 +431,9 @@ void __dev_flush(void) · 8 unmodified lines
433	431	}
434	432	}
435	433
436		−/* rcu_read_lock (from syscall and BPF contexts) ensures that if a delete and/or
437		− * update happens in parallel here a dev_put won't happen until after reading
438		− * the ifindex.
	434	+/* Elements are kept alive by RCU; either by rcu_read_lock() (from syscall) or
	435	+ * by local_bh_disable() (from XDP calls inside NAPI). The
	436	+ * rcu_read_lock_bh_held() below makes lockdep accept both.
439	437	*/
440	438	static void __dev_map_lookup_elem(struct bpf_map map, u32 key)
441	439	{
@@ -445,12 +443,14 @@ static void __dev_map_lookup_elem(struct bpf_map map, u32 key) · 3 unmodified lines
445	443	if (key >= map->max_entries)
446	444	return NULL;
447	445
448		− obj = READ_ONCE(dtab->netdev_map[key]);
	446	+ obj = rcu_dereference_check(dtab->netdev_map[key],
	447	+ rcu_read_lock_bh_held());
449	448	return obj;
450	449	}
451	450
452		−/* Runs under RCU-read-side, plus in softirq under NAPI protection.
453		− * Thus, safe percpu variable access.
	451	+/* Runs in NAPI, i.e., softirq under local_bh_disable(). Thus, safe percpu
	452	+ * variable access, and map elements stick around. See comment above
	453	+ * xdp_do_flush() in filter.c.
454	454	*/
455	455	static void bq_enqueue(struct net_device dev, struct xdp_frame xdpf,
456	456	struct net_device dev_rx, struct bpf_prog xdp_prog)
@@ -735,14 +735,7 @@ static int dev_map_delete_elem(struct bpf_map map, void key) · 278 unmodified lines
735	735	if (k >= map->max_entries)
736	736	return -EINVAL;
737	737
738		− /* Use call_rcu() here to ensure any rcu critical sections have
739		− * completed as well as any flush operations because call_rcu
740		− * will wait for preempt-disable region to complete, NAPI in this
741		− * context. And additionally, the driver tear down ensures all
742		− * soft irqs are complete before removing the net device in the
743		− * case of dev_put equals zero.
744		− */
745		− old_dev = xchg(&dtab->netdev_map[k], NULL);
	738	+ old_dev = unrcu_pointer(xchg(&dtab->netdev_map[k], NULL));
746	739	if (old_dev)
747	740	call_rcu(&old_dev->rcu, __dev_map_entry_free);
748	741	return 0;
@@ -851,7 +844,7 @@ static int __dev_map_update_elem(struct net net, struct bpf_map map, · 102 unmodified lines
851	844	* Remembering the driver side flush operation will happen before the
852	845	* net device is removed.
853	846	*/
854		− old_dev = xchg(&dtab->netdev_map[i], dev);
	847	+ old_dev = unrcu_pointer(xchg(&dtab->netdev_map[i], RCU_INITIALIZER(dev)));
855	848	if (old_dev)
856	849	call_rcu(&old_dev->rcu, __dev_map_entry_free);
857	850
@@ -1031,10 +1024,10 @@ static int dev_map_notification(struct notifier_block *notifier, · 173 unmodified lines
1031	1024	for (i = 0; i < dtab->map.max_entries; i++) {
1032	1025	struct bpf_dtab_netdev dev, odev;
1033	1026
1034		− dev = READ_ONCE(dtab->netdev_map[i]);
	1027	+ dev = rcu_dereference(dtab->netdev_map[i]);
1035	1028	if (!dev \|\| netdev != dev->dev)
1036	1029	continue;
1037		− odev = cmpxchg(&dtab->netdev_map[i], dev, NULL);
	1030	+ odev = unrcu_pointer(cmpxchg(&dtab->netdev_map[i], RCU_INITIALIZER(dev), NULL));
1038	1031	if (dev == odev)
1039	1032	call_rcu(&dev->rcu,
1040	1033	__dev_map_entry_free);

kernel/bpf/hashtab.c+14−7

@@ -596,7 +596,8 @@ static void __htab_map_lookup_elem(struct bpf_map map, void *key) · 595 unmodified lines
596	596	struct htab_elem *l;
597	597	u32 hash, key_size;
598	598
599		− WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held());
	599	+ WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held() &&
	600	+ !rcu_read_lock_bh_held());
600	601
601	602	key_size = map->key_size;
602	603
@@ -989,7 +990,8 @@ static int htab_map_update_elem(struct bpf_map map, void key, void *value, · 386 unmodified lines
989	990	/* unknown flags */
990	991	return -EINVAL;
991	992
992		− WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held());
	993	+ WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held() &&
	994	+ !rcu_read_lock_bh_held());
993	995
994	996	key_size = map->key_size;
995	997
@@ -1082,7 +1084,8 @@ static int htab_lru_map_update_elem(struct bpf_map map, void key, void *value, · 86 unmodified lines
1082	1084	/* unknown flags */
1083	1085	return -EINVAL;
1084	1086
1085		− WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held());
	1087	+ WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held() &&
	1088	+ !rcu_read_lock_bh_held());
1086	1089
1087	1090	key_size = map->key_size;
1088	1091
@@ -1148,7 +1151,8 @@ static int __htab_percpu_map_update_elem(struct bpf_map map, void key, · 59 unmodified lines
1148	1151	/* unknown flags */
1149	1152	return -EINVAL;
1150	1153
1151		− WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held());
	1154	+ WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held() &&
	1155	+ !rcu_read_lock_bh_held());
1152	1156
1153	1157	key_size = map->key_size;
1154	1158
@@ -1202,7 +1206,8 @@ static int __htab_lru_percpu_map_update_elem(struct bpf_map map, void key, · 47 unmodified lines
1202	1206	/* unknown flags */
1203	1207	return -EINVAL;
1204	1208
1205		− WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held());
	1209	+ WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held() &&
	1210	+ !rcu_read_lock_bh_held());
1206	1211
1207	1212	key_size = map->key_size;
1208	1213
@@ -1276,7 +1281,8 @@ static int htab_map_delete_elem(struct bpf_map map, void key) · 67 unmodified lines
1276	1281	u32 hash, key_size;
1277	1282	int ret;
1278	1283
1279		− WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held());
	1284	+ WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held() &&
	1285	+ !rcu_read_lock_bh_held());
1280	1286
1281	1287	key_size = map->key_size;
1282	1288
@@ -1311,7 +1317,8 @@ static int htab_lru_map_delete_elem(struct bpf_map map, void key) · 28 unmodified lines
1311	1317	u32 hash, key_size;
1312	1318	int ret;
1313	1319
1314		− WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held());
	1320	+ WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held() &&
	1321	+ !rcu_read_lock_bh_held());
1315	1322
1316	1323	key_size = map->key_size;
1317	1324

kernel/bpf/helpers.c+3−3

@@ -29,7 +29,7 @@ · 28 unmodified lines
29	29	*/
30	30	BPF_CALL_2(bpf_map_lookup_elem, struct bpf_map , map, void , key)
31	31	{
32		− WARN_ON_ONCE(!rcu_read_lock_held());
	32	+ WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_bh_held());
33	33	return (unsigned long) map->ops->map_lookup_elem(map, key);
34	34	}
35	35
@@ -45,7 +45,7 @@ const struct bpf_func_proto bpf_map_lookup_elem_proto = { · 9 unmodified lines
45	45	BPF_CALL_4(bpf_map_update_elem, struct bpf_map , map, void , key,
46	46	void *, value, u64, flags)
47	47	{
48		− WARN_ON_ONCE(!rcu_read_lock_held());
	48	+ WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_bh_held());
49	49	return map->ops->map_update_elem(map, key, value, flags);
50	50	}
51	51
@@ -62,7 +62,7 @@ const struct bpf_func_proto bpf_map_update_elem_proto = { · 10 unmodified lines
62	62
63	63	BPF_CALL_2(bpf_map_delete_elem, struct bpf_map , map, void , key)
64	64	{
65		− WARN_ON_ONCE(!rcu_read_lock_held());
	65	+ WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_bh_held());
66	66	return map->ops->map_delete_elem(map, key);
67	67	}
68	68

kernel/bpf/lpm_trie.c+4−2

@@ -232,7 +232,8 @@ static void trie_lookup_elem(struct bpf_map map, void *_key) · 231 unmodified lines
232	232
233	233	/* Start walking the trie from the root node ... */
234	234
235		− for (node = rcu_dereference(trie->root); node;) {
	235	+ for (node = rcu_dereference_check(trie->root, rcu_read_lock_bh_held());
	236	+ node;) {
236	237	unsigned int next_bit;
237	238	size_t matchlen;
238	239
@@ -264,7 +265,8 @@ static void trie_lookup_elem(struct bpf_map map, void *_key) · 25 unmodified lines
264	265	* traverse down.
265	266	*/
266	267	next_bit = extract_bit(key->data, node->prefixlen);
267		− node = rcu_dereference(node->child[next_bit]);
	268	+ node = rcu_dereference_check(node->child[next_bit],
	269	+ rcu_read_lock_bh_held());
268	270	}
269	271
270	272	if (!found)

kernel/bpf/ringbuf.c+2−0

@@ -8,6 +8,7 @@ · 7 unmodified lines
8	8	#include <linux/vmalloc.h>
9	9	#include <linux/wait.h>
10	10	#include <linux/poll.h>
	11	+#include <linux/kmemleak.h>
11	12	#include <uapi/linux/btf.h>
12	13
13	14	#define RINGBUF_CREATE_FLAG_MASK (BPF_F_NUMA_NODE)
@@ -105,6 +106,7 @@ static struct bpf_ringbuf *bpf_ringbuf_area_alloc(size_t data_sz, int numa_node) · 91 unmodified lines
105	106	rb = vmap(pages, nr_meta_pages + 2 * nr_data_pages,
106	107	VM_ALLOC \| VM_USERMAP, PAGE_KERNEL);
107	108	if (rb) {
	109	+ kmemleak_not_leak(pages);
108	110	rb->pages = pages;
109	111	rb->nr_pages = nr_pages;
110	112	return rb;

kernel/trace/bpf_trace.c+2−0

@@ -1017,6 +1017,8 @@ bpf_tracing_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) · 1016 unmodified lines
1017	1017	#ifdef CONFIG_CGROUPS
1018	1018	case BPF_FUNC_get_current_cgroup_id:
1019	1019	return &bpf_get_current_cgroup_id_proto;
	1020	+ case BPF_FUNC_get_current_ancestor_cgroup_id:
	1021	+ return &bpf_get_current_ancestor_cgroup_id_proto;
1020	1022	#endif
1021	1023	case BPF_FUNC_send_signal:
1022	1024	return &bpf_send_signal_proto;

net/bpfilter/main.c+1−1

@@ -57,7 +57,7 @@ int main(void) · 56 unmodified lines
57	57	{
58	58	debug_f = fopen("/dev/kmsg", "w");
59	59	setvbuf(debug_f, 0, _IOLBF, 0);
60		− fprintf(debug_f, "Started bpfilter\n");
	60	+ fprintf(debug_f, "<5>Started bpfilter\n");
61	61	loop();
62	62	fclose(debug_f);
63	63	return 0;

net/core/filter.c+37−35

@@ -3235,15 +3235,12 @@ static int bpf_skb_net_hdr_pop(struct sk_buff *skb, u32 off, u32 len) · 3234 unmodified lines
3235	3235	return ret;
3236	3236	}
3237	3237
3238		−static int bpf_skb_proto_4_to_6(struct sk_buff *skb, u64 flags)
	3238	+static int bpf_skb_proto_4_to_6(struct sk_buff *skb)
3239	3239	{
3240	3240	const u32 len_diff = sizeof(struct ipv6hdr) - sizeof(struct iphdr);
3241	3241	u32 off = skb_mac_header_len(skb);
3242	3242	int ret;
3243	3243
3244		− if (skb_is_gso(skb) && !skb_is_gso_tcp(skb))
3245		− return -ENOTSUPP;
3246		−
3247	3244	ret = skb_cow(skb, len_diff);
3248	3245	if (unlikely(ret < 0))
3249	3246	return ret;
@@ -3255,21 +3252,11 @@ static int bpf_skb_proto_4_to_6(struct sk_buff *skb, u64 flags) · 5 unmodified lines
3255	3252	if (skb_is_gso(skb)) {
3256	3253	struct skb_shared_info *shinfo = skb_shinfo(skb);
3257	3254
3258		− /* SKB_GSO_TCPV4 needs to be changed into
3259		− * SKB_GSO_TCPV6.
3260		− */
	3255	+ /* SKB_GSO_TCPV4 needs to be changed into SKB_GSO_TCPV6. */
3261	3256	if (shinfo->gso_type & SKB_GSO_TCPV4) {
3262	3257	shinfo->gso_type &= ~SKB_GSO_TCPV4;
3263	3258	shinfo->gso_type \|= SKB_GSO_TCPV6;
3264	3259	}
3265		−
3266		− /* Due to IPv6 header, MSS needs to be downgraded. */
3267		− if (!(flags & BPF_F_ADJ_ROOM_FIXED_GSO))
3268		− skb_decrease_gso_size(shinfo, len_diff);
3269		−
3270		− /* Header must be checked, and gso_segs recomputed. */
3271		− shinfo->gso_type \|= SKB_GSO_DODGY;
3272		− shinfo->gso_segs = 0;
3273	3260	}
3274	3261
3275	3262	skb->protocol = htons(ETH_P_IPV6);
@@ -3278,15 +3265,12 @@ static int bpf_skb_proto_4_to_6(struct sk_buff *skb, u64 flags) · 2 unmodified lines
3278	3265	return 0;
3279	3266	}
3280	3267
3281		−static int bpf_skb_proto_6_to_4(struct sk_buff *skb, u64 flags)
	3268	+static int bpf_skb_proto_6_to_4(struct sk_buff *skb)
3282	3269	{
3283	3270	const u32 len_diff = sizeof(struct ipv6hdr) - sizeof(struct iphdr);
3284	3271	u32 off = skb_mac_header_len(skb);
3285	3272	int ret;
3286	3273
3287		− if (skb_is_gso(skb) && !skb_is_gso_tcp(skb))
3288		− return -ENOTSUPP;
3289		−
3290	3274	ret = skb_unclone(skb, GFP_ATOMIC);
3291	3275	if (unlikely(ret < 0))
3292	3276	return ret;
@@ -3298,21 +3282,11 @@ static int bpf_skb_proto_6_to_4(struct sk_buff *skb, u64 flags) · 5 unmodified lines
3298	3282	if (skb_is_gso(skb)) {
3299	3283	struct skb_shared_info *shinfo = skb_shinfo(skb);
3300	3284
3301		− /* SKB_GSO_TCPV6 needs to be changed into
3302		− * SKB_GSO_TCPV4.
3303		− */
	3285	+ /* SKB_GSO_TCPV6 needs to be changed into SKB_GSO_TCPV4. */
3304	3286	if (shinfo->gso_type & SKB_GSO_TCPV6) {
3305	3287	shinfo->gso_type &= ~SKB_GSO_TCPV6;
3306	3288	shinfo->gso_type \|= SKB_GSO_TCPV4;
3307	3289	}
3308		−
3309		− /* Due to IPv4 header, MSS can be upgraded. */
3310		− if (!(flags & BPF_F_ADJ_ROOM_FIXED_GSO))
3311		− skb_increase_gso_size(shinfo, len_diff);
3312		−
3313		− /* Header must be checked, and gso_segs recomputed. */
3314		− shinfo->gso_type \|= SKB_GSO_DODGY;
3315		− shinfo->gso_segs = 0;
3316	3290	}
3317	3291
3318	3292	skb->protocol = htons(ETH_P_IP);
@@ -3321,17 +3295,17 @@ static int bpf_skb_proto_6_to_4(struct sk_buff *skb, u64 flags) · 2 unmodified lines
3321	3295	return 0;
3322	3296	}
3323	3297
3324		−static int bpf_skb_proto_xlat(struct sk_buff *skb, __be16 to_proto, u64 flags)
	3298	+static int bpf_skb_proto_xlat(struct sk_buff *skb, __be16 to_proto)
3325	3299	{
3326	3300	__be16 from_proto = skb->protocol;
3327	3301
3328	3302	if (from_proto == htons(ETH_P_IP) &&
3329	3303	to_proto == htons(ETH_P_IPV6))
3330		− return bpf_skb_proto_4_to_6(skb, flags);
	3304	+ return bpf_skb_proto_4_to_6(skb);
3331	3305
3332	3306	if (from_proto == htons(ETH_P_IPV6) &&
3333	3307	to_proto == htons(ETH_P_IP))
3334		− return bpf_skb_proto_6_to_4(skb, flags);
	3308	+ return bpf_skb_proto_6_to_4(skb);
3335	3309
3336	3310	return -ENOTSUPP;
3337	3311	}
@@ -3341,7 +3315,7 @@ BPF_CALL_3(bpf_skb_change_proto, struct sk_buff *, skb, __be16, proto, · 3 unmodified lines
3341	3315	{
3342	3316	int ret;
3343	3317
3344		− if (unlikely(flags & ~(BPF_F_ADJ_ROOM_FIXED_GSO)))
	3318	+ if (unlikely(flags))
3345	3319	return -EINVAL;
3346	3320
3347	3321	/* General idea is that this helper does the basic groundwork
@@ -3361,7 +3335,7 @@ BPF_CALL_3(bpf_skb_change_proto, struct sk_buff *, skb, __be16, proto, · 13 unmodified lines
3361	3335	* that. For offloads, we mark packet as dodgy, so that headers
3362	3336	* need to be verified first.
3363	3337	*/
3364		− ret = bpf_skb_proto_xlat(skb, proto, flags);
	3338	+ ret = bpf_skb_proto_xlat(skb, proto);
3365	3339	bpf_compute_data_pointers(skb);
3366	3340	return ret;
3367	3341	}
@@ -3923,6 +3897,34 @@ static const struct bpf_func_proto bpf_xdp_adjust_meta_proto = { · 555 unmodified lines
3923	3897	.arg2_type = ARG_ANYTHING,
3924	3898	};
3925	3899
	3900	+/* XDP_REDIRECT works by a three-step process, implemented in the functions
	3901	+ * below:
	3902	+ *
	3903	+ * 1. The bpf_redirect() and bpf_redirect_map() helpers will lookup the target
	3904	+ * of the redirect and store it (along with some other metadata) in a per-CPU
	3905	+ * struct bpf_redirect_info.
	3906	+ *
	3907	+ * 2. When the program returns the XDP_REDIRECT return code, the driver will
	3908	+ * call xdp_do_redirect() which will use the information in struct
	3909	+ * bpf_redirect_info to actually enqueue the frame into a map type-specific
	3910	+ * bulk queue structure.
	3911	+ *
	3912	+ * 3. Before exiting its NAPI poll loop, the driver will call xdp_do_flush(),
	3913	+ * which will flush all the different bulk queues, thus completing the
	3914	+ * redirect.
	3915	+ *
	3916	+ * Pointers to the map entries will be kept around for this whole sequence of
	3917	+ * steps, protected by RCU. However, there is no top-level rcu_read_lock() in
	3918	+ * the core code; instead, the RCU protection relies on everything happening
	3919	+ * inside a single NAPI poll sequence, which means it's between a pair of calls
	3920	+ * to local_bh_disable()/local_bh_enable().
	3921	+ *
	3922	+ * The map entries are marked as __rcu and the map code makes sure to
	3923	+ * dereference those pointers with rcu_dereference_check() in a way that works
	3924	+ * for both sections that to hold an rcu_read_lock() and sections that are
	3925	+ * called from NAPI without a separate rcu_read_lock(). The code below does not
	3926	+ * use RCU annotations, but relies on those in the map code.
	3927	+ */
3926	3928	void xdp_do_flush(void)
3927	3929	{
3928	3930	__dev_flush();

net/core/xdp.c+6−5

@@ -113,8 +113,13 @@ static void mem_allocator_disconnect(void *allocator) · 112 unmodified lines
113	113	void xdp_rxq_info_unreg_mem_model(struct xdp_rxq_info *xdp_rxq)
114	114	{
115	115	struct xdp_mem_allocator *xa;
	116	+ int type = xdp_rxq->mem.type;
116	117	int id = xdp_rxq->mem.id;
117	118
	119	+ /* Reset mem info to defaults */
	120	+ xdp_rxq->mem.id = 0;
	121	+ xdp_rxq->mem.type = 0;
	122	+
118	123	if (xdp_rxq->reg_state != REG_STATE_REGISTERED) {
119	124	WARN(1, "Missing register, driver bug");
120	125	return;
@@ -123,7 +128,7 @@ void xdp_rxq_info_unreg_mem_model(struct xdp_rxq_info *xdp_rxq) · 2 unmodified lines
123	128	if (id == 0)
124	129	return;
125	130
126		− if (xdp_rxq->mem.type == MEM_TYPE_PAGE_POOL) {
	131	+ if (type == MEM_TYPE_PAGE_POOL) {
127	132	rcu_read_lock();
128	133	xa = rhashtable_lookup(mem_id_ht, &id, mem_id_rht_params);
129	134	page_pool_destroy(xa->page_pool);
@@ -144,10 +149,6 @@ void xdp_rxq_info_unreg(struct xdp_rxq_info *xdp_rxq) · 14 unmodified lines
144	149
145	150	xdp_rxq->reg_state = REG_STATE_UNREGISTERED;
146	151	xdp_rxq->dev = NULL;
147		−
148		− /* Reset mem info to defaults */
149		− xdp_rxq->mem.id = 0;
150		− xdp_rxq->mem.type = 0;
151	152	}
152	153	EXPORT_SYMBOL_GPL(xdp_rxq_info_unreg);
153	154

net/sched/act_bpf.c+0−2

@@ -43,7 +43,6 @@ static int tcf_bpf_act(struct sk_buff skb, const struct tc_action act, · 42 unmodified lines
43	43	tcf_lastuse_update(&prog->tcf_tm);
44	44	bstats_cpu_update(this_cpu_ptr(prog->common.cpu_bstats), skb);
45	45
46		− rcu_read_lock();
47	46	filter = rcu_dereference(prog->filter);
48	47	if (at_ingress) {
49	48	__skb_push(skb, skb->mac_len);
@@ -56,7 +55,6 @@ static int tcf_bpf_act(struct sk_buff skb, const struct tc_action act, · 6 unmodified lines
56	55	}
57	56	if (skb_sk_is_prefetched(skb) && filter_res != TC_ACT_OK)
58	57	skb_orphan(skb);
59		− rcu_read_unlock();
60	58
61	59	/* A BPF program may overwrite the default action opcode.
62	60	* Similarly as in cls_bpf, if filter_res == -1 we use the

net/sched/cls_bpf.c+0−3

@@ -85,8 +85,6 @@ static int cls_bpf_classify(struct sk_buff skb, const struct tcf_proto tp, · 84 unmodified lines
85	85	struct cls_bpf_prog *prog;
86	86	int ret = -1;
87	87
88		− /* Needed here for accessing maps. */
89		− rcu_read_lock();
90	88	list_for_each_entry_rcu(prog, &head->plist, link) {
91	89	int filter_res;
92	90
@@ -131,7 +129,6 @@ static int cls_bpf_classify(struct sk_buff skb, const struct tcf_proto tp, · 38 unmodified lines
131	129
132	130	break;
133	131	}
134		− rcu_read_unlock();
135	132
136	133	return ret;
137	134	}

net/xdp/xsk.c+2−2

@@ -749,7 +749,7 @@ static void xsk_unbind_dev(struct xdp_sock *xs) · 748 unmodified lines
749	749	}
750	750
751	751	static struct xsk_map xsk_get_map_list_entry(struct xdp_sock xs,
752		− struct xdp_sock ***map_entry)
	752	+ struct xdp_sock __rcu ***map_entry)
753	753	{
754	754	struct xsk_map *map = NULL;
755	755	struct xsk_map_node *node;
@@ -785,7 +785,7 @@ static void xsk_delete_from_maps(struct xdp_sock *xs) · 29 unmodified lines
785	785	* might be updates to the map between
786	786	* xsk_get_map_list_entry() and xsk_map_try_sock_delete().
787	787	*/
788		− struct xdp_sock **map_entry = NULL;
	788	+ struct xdp_sock __rcu **map_entry = NULL;
789	789	struct xsk_map *map;
790	790
791	791	while ((map = xsk_get_map_list_entry(xs, &map_entry))) {

net/xdp/xsk.h+2−2

@@ -31,7 +31,7 @@ struct xdp_mmap_offsets_v1 { · 30 unmodified lines
31	31	struct xsk_map_node {
32	32	struct list_head node;
33	33	struct xsk_map *map;
34		− struct xdp_sock **map_entry;
	34	+ struct xdp_sock __rcu **map_entry;
35	35	};
36	36
37	37	static inline struct xdp_sock xdp_sk(struct sock sk)
@@ -40,7 +40,7 @@ static inline struct xdp_sock xdp_sk(struct sock sk) · 2 unmodified lines
40	40	}
41	41
42	42	void xsk_map_try_sock_delete(struct xsk_map map, struct xdp_sock xs,
43		− struct xdp_sock **map_entry);
	43	+ struct xdp_sock __rcu **map_entry);
44	44	void xsk_clear_pool_at_qid(struct net_device *dev, u16 queue_id);
45	45	int xsk_reg_pool_at_qid(struct net_device dev, struct xsk_buff_pool pool,
46	46	u16 queue_id);

net/xdp/xskmap.c+17−12

@@ -12,7 +12,7 @@ · 11 unmodified lines
12	12	#include "xsk.h"
13	13
14	14	static struct xsk_map_node xsk_map_node_alloc(struct xsk_map map,
15		− struct xdp_sock **map_entry)
	15	+ struct xdp_sock __rcu **map_entry)
16	16	{
17	17	struct xsk_map_node *node;
18	18
@@ -42,7 +42,7 @@ static void xsk_map_sock_add(struct xdp_sock xs, struct xsk_map_node node) · 23 unmodified lines
42	42	}
43	43
44	44	static void xsk_map_sock_delete(struct xdp_sock *xs,
45		− struct xdp_sock **map_entry)
	45	+ struct xdp_sock __rcu **map_entry)
46	46	{
47	47	struct xsk_map_node n, tmp;
48	48
@@ -124,6 +124,10 @@ static int xsk_map_gen_lookup(struct bpf_map map, struct bpf_insn insn_buf) · 75 unmodified lines
124	124	return insn - insn_buf;
125	125	}
126	126
	127	+/* Elements are kept alive by RCU; either by rcu_read_lock() (from syscall) or
	128	+ * by local_bh_disable() (from XDP calls inside NAPI). The
	129	+ * rcu_read_lock_bh_held() below makes lockdep accept both.
	130	+ */
127	131	static void __xsk_map_lookup_elem(struct bpf_map map, u32 key)
128	132	{
129	133	struct xsk_map *m = container_of(map, struct xsk_map, map);
@@ -131,12 +135,11 @@ static void __xsk_map_lookup_elem(struct bpf_map map, u32 key) · 1 unmodified line
131	135	if (key >= map->max_entries)
132	136	return NULL;
133	137
134		− return READ_ONCE(m->xsk_map[key]);
	138	+ return rcu_dereference_check(m->xsk_map[key], rcu_read_lock_bh_held());
135	139	}
136	140
137	141	static void xsk_map_lookup_elem(struct bpf_map map, void *key)
138	142	{
139		− WARN_ON_ONCE(!rcu_read_lock_held());
140	143	return __xsk_map_lookup_elem(map, (u32 )key);
141	144	}
142	145
@@ -149,7 +152,8 @@ static int xsk_map_update_elem(struct bpf_map map, void key, void *value, · 6 unmodified lines
149	152	u64 map_flags)
150	153	{
151	154	struct xsk_map *m = container_of(map, struct xsk_map, map);
152		− struct xdp_sock xs, old_xs, **map_entry;
	155	+ struct xdp_sock __rcu **map_entry;
	156	+ struct xdp_sock xs, old_xs;
153	157	u32 i = (u32 )key, fd = (u32 )value;
154	158	struct xsk_map_node *node;
155	159	struct socket *sock;
@@ -179,7 +183,7 @@ static int xsk_map_update_elem(struct bpf_map map, void key, void *value, · 23 unmodified lines
179	183	}
180	184
181	185	spin_lock_bh(&m->lock);
182		− old_xs = READ_ONCE(*map_entry);
	186	+ old_xs = rcu_dereference_protected(*map_entry, lockdep_is_held(&m->lock));
183	187	if (old_xs == xs) {
184	188	err = 0;
185	189	goto out;
@@ -191,7 +195,7 @@ static int xsk_map_update_elem(struct bpf_map map, void key, void *value, · 5 unmodified lines
191	195	goto out;
192	196	}
193	197	xsk_map_sock_add(xs, node);
194		− WRITE_ONCE(*map_entry, xs);
	198	+ rcu_assign_pointer(*map_entry, xs);
195	199	if (old_xs)
196	200	xsk_map_sock_delete(old_xs, map_entry);
197	201	spin_unlock_bh(&m->lock);
@@ -208,7 +212,8 @@ out: · 10 unmodified lines
208	212	static int xsk_map_delete_elem(struct bpf_map map, void key)
209	213	{
210	214	struct xsk_map *m = container_of(map, struct xsk_map, map);
211		− struct xdp_sock old_xs, *map_entry;
	215	+ struct xdp_sock __rcu **map_entry;
	216	+ struct xdp_sock *old_xs;
212	217	int k = (u32 )key;
213	218
214	219	if (k >= map->max_entries)
@@ -216,7 +221,7 @@ static int xsk_map_delete_elem(struct bpf_map map, void key) · 1 unmodified line
216	221
217	222	spin_lock_bh(&m->lock);
218	223	map_entry = &m->xsk_map[k];
219		− old_xs = xchg(map_entry, NULL);
	224	+ old_xs = unrcu_pointer(xchg(map_entry, NULL));
220	225	if (old_xs)
221	226	xsk_map_sock_delete(old_xs, map_entry);
222	227	spin_unlock_bh(&m->lock);
@@ -231,11 +236,11 @@ static int xsk_map_redirect(struct bpf_map *map, u32 ifindex, u64 flags) · 8 unmodified lines
231	236	}
232	237
233	238	void xsk_map_try_sock_delete(struct xsk_map map, struct xdp_sock xs,
234		− struct xdp_sock **map_entry)
	239	+ struct xdp_sock __rcu **map_entry)
235	240	{
236	241	spin_lock_bh(&map->lock);
237		− if (READ_ONCE(*map_entry) == xs) {
238		− WRITE_ONCE(*map_entry, NULL);
	242	+ if (rcu_access_pointer(*map_entry) == xs) {
	243	+ rcu_assign_pointer(*map_entry, NULL);
239	244	xsk_map_sock_delete(xs, map_entry);
240	245	}
241	246	spin_unlock_bh(&map->lock);

samples/bpf/xdp_redirect_user.c+2−2

@@ -130,7 +130,7 @@ int main(int argc, char **argv) · 129 unmodified lines
130	130	if (!(xdp_flags & XDP_FLAGS_SKB_MODE))
131	131	xdp_flags \|= XDP_FLAGS_DRV_MODE;
132	132
133		− if (optind == argc) {
	133	+ if (optind + 2 != argc) {
134	134	printf("usage: %s <IFNAME\|IFINDEX>_IN <IFNAME\|IFINDEX>_OUT\n", argv[0]);
135	135	return 1;
136	136	}
@@ -213,5 +213,5 @@ int main(int argc, char **argv) · 76 unmodified lines
213	213	poll_stats(2, ifindex_out);
214	214
215	215	out:
216		− return 0;
	216	+ return ret;
217	217	}

tools/lib/bpf/libbpf.c+4−0

@@ -4000,6 +4000,10 @@ bpf_object__probe_loading(struct bpf_object *obj) · 3999 unmodified lines
4000	4000	attr.license = "GPL";
4001	4001
4002	4002	ret = bpf_load_program_xattr(&attr, NULL, 0);
	4003	+ if (ret < 0) {
	4004	+ attr.prog_type = BPF_PROG_TYPE_TRACEPOINT;
	4005	+ ret = bpf_load_program_xattr(&attr, NULL, 0);
	4006	+ }
4003	4007	if (ret < 0) {
4004	4008	ret = errno;
4005	4009	cp = libbpf_strerror_r(ret, errmsg, sizeof(errmsg));

tools/lib/bpf/netlink.c+44−71

@@ -154,7 +154,7 @@ done: · 153 unmodified lines
154	154	return ret;
155	155	}
156	156
157		−static int libbpf_netlink_send_recv(struct nlmsghdr *nh,
	157	+static int libbpf_netlink_send_recv(struct libbpf_nla_req *req,
158	158	__dump_nlmsg_t parse_msg,
159	159	libbpf_dump_nlmsg_t parse_attr,
160	160	void *cookie)
@@ -166,15 +166,15 @@ static int libbpf_netlink_send_recv(struct nlmsghdr *nh, · 5 unmodified lines
166	166	if (sock < 0)
167	167	return sock;
168	168
169		− nh->nlmsg_pid = 0;
170		− nh->nlmsg_seq = time(NULL);
	169	+ req->nh.nlmsg_pid = 0;
	170	+ req->nh.nlmsg_seq = time(NULL);
171	171
172		− if (send(sock, nh, nh->nlmsg_len, 0) < 0) {
	172	+ if (send(sock, req, req->nh.nlmsg_len, 0) < 0) {
173	173	ret = -errno;
174	174	goto out;
175	175	}
176	176
177		− ret = libbpf_netlink_recv(sock, nl_pid, nh->nlmsg_seq,
	177	+ ret = libbpf_netlink_recv(sock, nl_pid, req->nh.nlmsg_seq,
178	178	parse_msg, parse_attr, cookie);
179	179	out:
180	180	libbpf_netlink_close(sock);
@@ -186,11 +186,7 @@ static int __bpf_set_link_xdp_fd_replace(int ifindex, int fd, int old_fd, · 5 unmodified lines
186	186	{
187	187	struct nlattr *nla;
188	188	int ret;
189		− struct {
190		− struct nlmsghdr nh;
191		− struct ifinfomsg ifinfo;
192		− char attrbuf[64];
193		− } req;
	189	+ struct libbpf_nla_req req;
194	190
195	191	memset(&req, 0, sizeof(req));
196	192	req.nh.nlmsg_len = NLMSG_LENGTH(sizeof(struct ifinfomsg));
@@ -199,27 +195,26 @@ static int __bpf_set_link_xdp_fd_replace(int ifindex, int fd, int old_fd, · 2 unmodified lines
199	195	req.ifinfo.ifi_family = AF_UNSPEC;
200	196	req.ifinfo.ifi_index = ifindex;
201	197
202		− nla = nlattr_begin_nested(&req.nh, sizeof(req), IFLA_XDP);
	198	+ nla = nlattr_begin_nested(&req, IFLA_XDP);
203	199	if (!nla)
204	200	return -EMSGSIZE;
205		− ret = nlattr_add(&req.nh, sizeof(req), IFLA_XDP_FD, &fd, sizeof(fd));
	201	+ ret = nlattr_add(&req, IFLA_XDP_FD, &fd, sizeof(fd));
206	202	if (ret < 0)
207	203	return ret;
208	204	if (flags) {
209		− ret = nlattr_add(&req.nh, sizeof(req), IFLA_XDP_FLAGS, &flags,
210		− sizeof(flags));
	205	+ ret = nlattr_add(&req, IFLA_XDP_FLAGS, &flags, sizeof(flags));
211	206	if (ret < 0)
212	207	return ret;
213	208	}
214	209	if (flags & XDP_FLAGS_REPLACE) {
215		− ret = nlattr_add(&req.nh, sizeof(req), IFLA_XDP_EXPECTED_FD,
216		− &old_fd, sizeof(old_fd));
	210	+ ret = nlattr_add(&req, IFLA_XDP_EXPECTED_FD, &old_fd,
	211	+ sizeof(old_fd));
217	212	if (ret < 0)
218	213	return ret;
219	214	}
220		− nlattr_end_nested(&req.nh, nla);
	215	+ nlattr_end_nested(&req, nla);
221	216
222		− return libbpf_netlink_send_recv(&req.nh, NULL, NULL, NULL);
	217	+ return libbpf_netlink_send_recv(&req, NULL, NULL, NULL);
223	218	}
224	219
225	220	int bpf_set_link_xdp_fd_opts(int ifindex, int fd, __u32 flags,
@@ -314,14 +309,11 @@ int bpf_get_link_xdp_info(int ifindex, struct xdp_link_info *info, · 88 unmodified lines
314	309	struct xdp_id_md xdp_id = {};
315	310	__u32 mask;
316	311	int ret;
317		− struct {
318		− struct nlmsghdr nh;
319		− struct ifinfomsg ifm;
320		− } req = {
321		− .nh.nlmsg_len = NLMSG_LENGTH(sizeof(struct ifinfomsg)),
322		− .nh.nlmsg_type = RTM_GETLINK,
323		− .nh.nlmsg_flags = NLM_F_DUMP \| NLM_F_REQUEST,
324		− .ifm.ifi_family = AF_PACKET,
	312	+ struct libbpf_nla_req req = {
	313	+ .nh.nlmsg_len = NLMSG_LENGTH(sizeof(struct ifinfomsg)),
	314	+ .nh.nlmsg_type = RTM_GETLINK,
	315	+ .nh.nlmsg_flags = NLM_F_DUMP \| NLM_F_REQUEST,
	316	+ .ifinfo.ifi_family = AF_PACKET,
325	317	};
326	318
327	319	if (flags & ~XDP_FLAGS_MASK \|\| !info_size)
@@ -336,7 +328,7 @@ int bpf_get_link_xdp_info(int ifindex, struct xdp_link_info *info, · 8 unmodified lines
336	328	xdp_id.ifindex = ifindex;
337	329	xdp_id.flags = flags;
338	330
339		− ret = libbpf_netlink_send_recv(&req.nh, __dump_link_nlmsg,
	331	+ ret = libbpf_netlink_send_recv(&req, __dump_link_nlmsg,
340	332	get_xdp_info, &xdp_id);
341	333	if (!ret) {
342	334	size_t sz = min(info_size, sizeof(xdp_id.info));
@@ -376,15 +368,14 @@ int bpf_get_link_xdp_id(int ifindex, __u32 *prog_id, __u32 flags) · 33 unmodified lines
376	368	return libbpf_err(ret);
377	369	}
378	370
379		−typedef int (qdisc_config_t)(struct nlmsghdr nh, struct tcmsg *t,
380		− size_t maxsz);
	371	+typedef int (qdisc_config_t)(struct libbpf_nla_req req);
381	372
382		−static int clsact_config(struct nlmsghdr nh, struct tcmsg t, size_t maxsz)
	373	+static int clsact_config(struct libbpf_nla_req *req)
383	374	{
384		− t->tcm_parent = TC_H_CLSACT;
385		− t->tcm_handle = TC_H_MAKE(TC_H_CLSACT, 0);
	375	+ req->tc.tcm_parent = TC_H_CLSACT;
	376	+ req->tc.tcm_handle = TC_H_MAKE(TC_H_CLSACT, 0);
386	377
387		− return nlattr_add(nh, maxsz, TCA_KIND, "clsact", sizeof("clsact"));
	378	+ return nlattr_add(req, TCA_KIND, "clsact", sizeof("clsact"));
388	379	}
389	380
390	381	static int attach_point_to_config(struct bpf_tc_hook *hook,
@@ -431,11 +422,7 @@ static int tc_qdisc_modify(struct bpf_tc_hook *hook, int cmd, int flags) · 40 unmodified lines
431	422	{
432	423	qdisc_config_t config;
433	424	int ret;
434		− struct {
435		− struct nlmsghdr nh;
436		− struct tcmsg tc;
437		− char buf[256];
438		− } req;
	425	+ struct libbpf_nla_req req;
439	426
440	427	ret = attach_point_to_config(hook, &config);
441	428	if (ret < 0)
@@ -448,11 +435,11 @@ static int tc_qdisc_modify(struct bpf_tc_hook *hook, int cmd, int flags) · 6 unmodified lines
448	435	req.tc.tcm_family = AF_UNSPEC;
449	436	req.tc.tcm_ifindex = OPTS_GET(hook, ifindex, 0);
450	437
451		− ret = config(&req.nh, &req.tc, sizeof(req));
	438	+ ret = config(&req);
452	439	if (ret < 0)
453	440	return ret;
454	441
455		− return libbpf_netlink_send_recv(&req.nh, NULL, NULL, NULL);
	442	+ return libbpf_netlink_send_recv(&req, NULL, NULL, NULL);
456	443	}
457	444
458	445	static int tc_qdisc_create_excl(struct bpf_tc_hook *hook)
@@ -537,14 +524,14 @@ static int get_tc_info(struct nlmsghdr *nh, libbpf_dump_nlmsg_t fn, · 78 unmodified lines
537	524	struct nlattr *tb[TCA_MAX + 1];
538	525
539	526	libbpf_nla_parse(tb, TCA_MAX,
540		− (struct nlattr )((char )tc + NLMSG_ALIGN(sizeof(*tc))),
	527	+ (struct nlattr )((void )tc + NLMSG_ALIGN(sizeof(*tc))),
541	528	NLMSG_PAYLOAD(nh, sizeof(*tc)), NULL);
542	529	if (!tb[TCA_KIND])
543	530	return NL_CONT;
544	531	return __get_tc_info(cookie, tc, tb, nh->nlmsg_flags & NLM_F_ECHO);
545	532	}
546	533
547		−static int tc_add_fd_and_name(struct nlmsghdr *nh, size_t maxsz, int fd)
	534	+static int tc_add_fd_and_name(struct libbpf_nla_req *req, int fd)
548	535	{
549	536	struct bpf_prog_info info = {};
550	537	__u32 info_len = sizeof(info);
@@ -555,7 +542,7 @@ static int tc_add_fd_and_name(struct nlmsghdr *nh, size_t maxsz, int fd) · 4 unmodified lines
555	542	if (ret < 0)
556	543	return ret;
557	544
558		− ret = nlattr_add(nh, maxsz, TCA_BPF_FD, &fd, sizeof(fd));
	545	+ ret = nlattr_add(req, TCA_BPF_FD, &fd, sizeof(fd));
559	546	if (ret < 0)
560	547	return ret;
561	548	len = snprintf(name, sizeof(name), "%s:[%u]", info.name, info.id);
@@ -563,7 +550,7 @@ static int tc_add_fd_and_name(struct nlmsghdr *nh, size_t maxsz, int fd) · 1 unmodified line
563	550	return -errno;
564	551	if (len >= sizeof(name))
565	552	return -ENAMETOOLONG;
566		− return nlattr_add(nh, maxsz, TCA_BPF_NAME, name, len + 1);
	553	+ return nlattr_add(req, TCA_BPF_NAME, name, len + 1);
567	554	}
568	555
569	556	int bpf_tc_attach(const struct bpf_tc_hook hook, struct bpf_tc_opts opts)
@@ -571,12 +558,8 @@ int bpf_tc_attach(const struct bpf_tc_hook hook, struct bpf_tc_opts opts) · 1 unmodified line
571	558	__u32 protocol, bpf_flags, handle, priority, parent, prog_id, flags;
572	559	int ret, ifindex, attach_point, prog_fd;
573	560	struct bpf_cb_ctx info = {};
	561	+ struct libbpf_nla_req req;
574	562	struct nlattr *nla;
575		− struct {
576		− struct nlmsghdr nh;
577		− struct tcmsg tc;
578		− char buf[256];
579		− } req;
580	563
581	564	if (!hook \|\| !opts \|\|
582	565	!OPTS_VALID(hook, bpf_tc_hook) \|\|
@@ -618,25 +601,24 @@ int bpf_tc_attach(const struct bpf_tc_hook hook, struct bpf_tc_opts opts) · 35 unmodified lines
618	601	return libbpf_err(ret);
619	602	req.tc.tcm_parent = parent;
620	603
621		− ret = nlattr_add(&req.nh, sizeof(req), TCA_KIND, "bpf", sizeof("bpf"));
	604	+ ret = nlattr_add(&req, TCA_KIND, "bpf", sizeof("bpf"));
622	605	if (ret < 0)
623	606	return libbpf_err(ret);
624		− nla = nlattr_begin_nested(&req.nh, sizeof(req), TCA_OPTIONS);
	607	+ nla = nlattr_begin_nested(&req, TCA_OPTIONS);
625	608	if (!nla)
626	609	return libbpf_err(-EMSGSIZE);
627		− ret = tc_add_fd_and_name(&req.nh, sizeof(req), prog_fd);
	610	+ ret = tc_add_fd_and_name(&req, prog_fd);
628	611	if (ret < 0)
629	612	return libbpf_err(ret);
630	613	bpf_flags = TCA_BPF_FLAG_ACT_DIRECT;
631		− ret = nlattr_add(&req.nh, sizeof(req), TCA_BPF_FLAGS, &bpf_flags,
632		− sizeof(bpf_flags));
	614	+ ret = nlattr_add(&req, TCA_BPF_FLAGS, &bpf_flags, sizeof(bpf_flags));
633	615	if (ret < 0)
634	616	return libbpf_err(ret);
635		− nlattr_end_nested(&req.nh, nla);
	617	+ nlattr_end_nested(&req, nla);
636	618
637	619	info.opts = opts;
638	620
639		− ret = libbpf_netlink_send_recv(&req.nh, get_tc_info, NULL, &info);
	621	+ ret = libbpf_netlink_send_recv(&req, get_tc_info, NULL, &info);
640	622	if (ret < 0)
641	623	return libbpf_err(ret);
642	624	if (!info.processed)
@@ -650,11 +632,7 @@ static int __bpf_tc_detach(const struct bpf_tc_hook *hook, · 7 unmodified lines
650	632	{
651	633	__u32 protocol = 0, handle, priority, parent, prog_id, flags;
652	634	int ret, ifindex, attach_point, prog_fd;
653		− struct {
654		− struct nlmsghdr nh;
655		− struct tcmsg tc;
656		− char buf[256];
657		− } req;
	635	+ struct libbpf_nla_req req;
658	636
659	637	if (!hook \|\|
660	638	!OPTS_VALID(hook, bpf_tc_hook) \|\|
@@ -701,13 +679,12 @@ static int __bpf_tc_detach(const struct bpf_tc_hook *hook, · 40 unmodified lines
701	679	req.tc.tcm_parent = parent;
702	680
703	681	if (!flush) {
704		− ret = nlattr_add(&req.nh, sizeof(req), TCA_KIND,
705		− "bpf", sizeof("bpf"));
	682	+ ret = nlattr_add(&req, TCA_KIND, "bpf", sizeof("bpf"));
706	683	if (ret < 0)
707	684	return ret;
708	685	}
709	686
710		− return libbpf_netlink_send_recv(&req.nh, NULL, NULL, NULL);
	687	+ return libbpf_netlink_send_recv(&req, NULL, NULL, NULL);
711	688	}
712	689
713	690	int bpf_tc_detach(const struct bpf_tc_hook *hook,
@@ -727,11 +704,7 @@ int bpf_tc_query(const struct bpf_tc_hook hook, struct bpf_tc_opts opts) · 13 unmodified lines
727	704	__u32 protocol, handle, priority, parent, prog_id, flags;
728	705	int ret, ifindex, attach_point, prog_fd;
729	706	struct bpf_cb_ctx info = {};
730		− struct {
731		− struct nlmsghdr nh;
732		− struct tcmsg tc;
733		− char buf[256];
734		− } req;
	707	+ struct libbpf_nla_req req;
735	708
736	709	if (!hook \|\| !opts \|\|
737	710	!OPTS_VALID(hook, bpf_tc_hook) \|\|
@@ -770,13 +743,13 @@ int bpf_tc_query(const struct bpf_tc_hook hook, struct bpf_tc_opts opts) · 32 unmodified lines
770	743	return libbpf_err(ret);
771	744	req.tc.tcm_parent = parent;
772	745
773		− ret = nlattr_add(&req.nh, sizeof(req), TCA_KIND, "bpf", sizeof("bpf"));
	746	+ ret = nlattr_add(&req, TCA_KIND, "bpf", sizeof("bpf"));
774	747	if (ret < 0)
775	748	return libbpf_err(ret);
776	749
777	750	info.opts = opts;
778	751
779		− ret = libbpf_netlink_send_recv(&req.nh, get_tc_info, NULL, &info);
	752	+ ret = libbpf_netlink_send_recv(&req, get_tc_info, NULL, &info);
780	753	if (ret < 0)
781	754	return libbpf_err(ret);
782	755	if (!info.processed)

tools/lib/bpf/nlattr.c+1−1

@@ -27,7 +27,7 @@ static struct nlattr nla_next(const struct nlattr nla, int *remaining) · 26 unmodified lines
27	27	int totlen = NLA_ALIGN(nla->nla_len);
28	28
29	29	*remaining -= totlen;
30		− return (struct nlattr ) ((char ) nla + totlen);
	30	+ return (struct nlattr )((void )nla + totlen);
31	31	}
32	32
33	33	static int nla_ok(const struct nlattr *nla, int remaining)

tools/lib/bpf/nlattr.h+24−14

@@ -13,6 +13,7 @@ · 12 unmodified lines
13	13	#include <string.h>
14	14	#include <errno.h>
15	15	#include <linux/netlink.h>
	16	+#include <linux/rtnetlink.h>
16	17
17	18	/* avoid multiple definition of netlink features */
18	19	#define __LINUX_NETLINK_H
@@ -52,6 +53,15 @@ struct libbpf_nla_policy { · 33 unmodified lines
52	53	uint16_t maxlen;
53	54	};
54	55
	56	+struct libbpf_nla_req {
	57	+ struct nlmsghdr nh;
	58	+ union {
	59	+ struct ifinfomsg ifinfo;
	60	+ struct tcmsg tc;
	61	+ };
	62	+ char buf[128];
	63	+};
	64	+
55	65	/**
56	66	* @ingroup attr
57	67	* Iterate over a stream of attributes
@@ -71,7 +81,7 @@ struct libbpf_nla_policy { · 13 unmodified lines
71	81	*/
72	82	static inline void libbpf_nla_data(const struct nlattr nla)
73	83	{
74		− return (char *) nla + NLA_HDRLEN;
	84	+ return (void *)nla + NLA_HDRLEN;
75	85	}
76	86
77	87	static inline uint8_t libbpf_nla_getattr_u8(const struct nlattr *nla)
@@ -108,47 +118,47 @@ int libbpf_nla_dump_errormsg(struct nlmsghdr *nlh); · 30 unmodified lines
108	118
109	119	static inline struct nlattr nla_data(struct nlattr nla)
110	120	{
111		− return (struct nlattr )((char )nla + NLA_HDRLEN);
	121	+ return (struct nlattr )((void )nla + NLA_HDRLEN);
112	122	}
113	123
114		−static inline struct nlattr nh_tail(struct nlmsghdr nh)
	124	+static inline struct nlattr req_tail(struct libbpf_nla_req req)
115	125	{
116		− return (struct nlattr )((char )nh + NLMSG_ALIGN(nh->nlmsg_len));
	126	+ return (struct nlattr )((void )req + NLMSG_ALIGN(req->nh.nlmsg_len));
117	127	}
118	128
119		−static inline int nlattr_add(struct nlmsghdr *nh, size_t maxsz, int type,
	129	+static inline int nlattr_add(struct libbpf_nla_req *req, int type,
120	130	const void *data, int len)
121	131	{
122	132	struct nlattr *nla;
123	133
124		− if (NLMSG_ALIGN(nh->nlmsg_len) + NLA_ALIGN(NLA_HDRLEN + len) > maxsz)
	134	+ if (NLMSG_ALIGN(req->nh.nlmsg_len) + NLA_ALIGN(NLA_HDRLEN + len) > sizeof(*req))
125	135	return -EMSGSIZE;
126	136	if (!!data != !!len)
127	137	return -EINVAL;
128	138
129		− nla = nh_tail(nh);
	139	+ nla = req_tail(req);
130	140	nla->nla_type = type;
131	141	nla->nla_len = NLA_HDRLEN + len;
132	142	if (data)
133	143	memcpy(nla_data(nla), data, len);
134		− nh->nlmsg_len = NLMSG_ALIGN(nh->nlmsg_len) + NLA_ALIGN(nla->nla_len);
	144	+ req->nh.nlmsg_len = NLMSG_ALIGN(req->nh.nlmsg_len) + NLA_ALIGN(nla->nla_len);
135	145	return 0;
136	146	}
137	147
138		−static inline struct nlattr nlattr_begin_nested(struct nlmsghdr nh,
139		− size_t maxsz, int type)
	148	+static inline struct nlattr nlattr_begin_nested(struct libbpf_nla_req req, int type)
140	149	{
141	150	struct nlattr *tail;
142	151
143		− tail = nh_tail(nh);
144		− if (nlattr_add(nh, maxsz, type \| NLA_F_NESTED, NULL, 0))
	152	+ tail = req_tail(req);
	153	+ if (nlattr_add(req, type \| NLA_F_NESTED, NULL, 0))
145	154	return NULL;
146	155	return tail;
147	156	}
148	157
149		−static inline void nlattr_end_nested(struct nlmsghdr nh, struct nlattr tail)
	158	+static inline void nlattr_end_nested(struct libbpf_nla_req *req,
	159	+ struct nlattr *tail)
150	160	{
151		− tail->nla_len = (char )nh_tail(nh) - (char )tail;
	161	+ tail->nla_len = (void )req_tail(req) - (void )tail;
152	162	}
153	163
154	164	#endif /* __LIBBPF_NLATTR_H */

tools/testing/selftests/bpf/prog_tests/ringbuf.c+1−1

@@ -100,7 +100,7 @@ void test_ringbuf(void) · 99 unmodified lines
100	100	if (CHECK(err != 0, "skel_load", "skeleton load failed\n"))
101	101	goto cleanup;
102	102
103		− rb_fd = bpf_map__fd(skel->maps.ringbuf);
	103	+ rb_fd = skel->maps.ringbuf.map_fd;
104	104	/* good read/write cons_pos */
105	105	mmap_ptr = mmap(NULL, page_size, PROT_READ \| PROT_WRITE, MAP_SHARED, rb_fd, 0);
106	106	ASSERT_OK_PTR(mmap_ptr, "rw_cons_pos");

Merge git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code