cgroup/rstat: validate cpu before css_rstat_cpu() access

css_rstat_updated() is exposed as a BPF kfunc and accepts a caller-provided cpu argument. The function uses cpu for per-cpu rstat lookups without checking whether it refers to a valid possible CPU. A BPF iter/cgroup program with CAP_BPF and CAP_PERFMON can pass an invalid cpu value. On an unfixed UBSCAN_BOUNDS test kernel, cpu == 0x7fffffff triggers: UBSAN: array-index-out-of-bounds in kernel/cgroup/rstat.c:31:9 index 2147483647 is out of range for type 'long unsigned int [64]' Call Trace: css_rstat_updated bpf_iter_run_prog cgroup_iter_seq_show bpf_seq_read Add cpu validation to the BPF-facing css_rstat_updated() kfunc and move the common implementation to __css_rstat_updated() for in-kernel callers.

Qing Ming committed 5 months ago

Commit 88170051 parent 345f4014 files · +25 −14Browse files

Unified · Split

4 files

block/blk-cgroup.c +1 −1
include/linux/cgroup.h +1 −0
kernel/cgroup/rstat.c +20 −10
mm/memcontrol.c +3 −3

block/blk-cgroup.c+1−1

@@ -2241,7 +2241,7 @@ void blk_cgroup_bio_start(struct bio *bio) · 2240 unmodified lines
2241	2241	}
2242	2242
2243	2243	u64_stats_update_end_irqrestore(&bis->sync, flags);
2244		− css_rstat_updated(&blkcg->css, cpu);
	2244	+ __css_rstat_updated(&blkcg->css, cpu);
2245	2245	put_cpu();
2246	2246	}
2247	2247

include/linux/cgroup.h+1−0

@@ -776,6 +776,7 @@ static inline void cgroup_path_from_kernfs_id(u64 id, char *buf, size_t buflen) · 775 unmodified lines
776	776	/*
777	777	* cgroup scalable recursive statistics.
778	778	*/
	779	+void __css_rstat_updated(struct cgroup_subsys_state *css, int cpu);
779	780	void css_rstat_updated(struct cgroup_subsys_state *css, int cpu);
780	781	void css_rstat_flush(struct cgroup_subsys_state *css);
781	782

kernel/cgroup/rstat.c+20−10

@@ -1,6 +1,7 @@
1	1	// SPDX-License-Identifier: GPL-2.0-only
2	2	#include "cgroup-internal.h"
3	3
	4	+#include <linux/cpumask.h>
4	5	#include <linux/sched/cputime.h>
5	6
6	7	#include <linux/bpf.h>
@@ -53,7 +54,7 @@ static inline struct llist_head ss_lhead_cpu(struct cgroup_subsys ss, int cpu) · 46 unmodified lines
53	54	}
54	55
55	56	/**
56		− * css_rstat_updated - keep track of updated rstat_cpu
	57	+ * __css_rstat_updated - keep track of updated rstat_cpu
57	58	* @css: target cgroup subsystem state
58	59	* @cpu: cpu on which rstat_cpu was updated
59	60	*
@@ -63,20 +64,17 @@ static inline struct llist_head ss_lhead_cpu(struct cgroup_subsys ss, int cpu) · 3 unmodified lines
63	64	*
64	65	* NOTE: if the user needs the guarantee that the updater either add itself in
65	66	* the lockless list or the concurrent flusher flushes its updated stats, a
66		− * memory barrier is needed before the call to css_rstat_updated() i.e. a
	67	+ * memory barrier is needed before the call to __css_rstat_updated() i.e. a
67	68	* barrier after updating the per-cpu stats and before calling
68		− * css_rstat_updated().
	69	+ * __css_rstat_updated().
69	70	*/
70		−__bpf_kfunc void css_rstat_updated(struct cgroup_subsys_state *css, int cpu)
	71	+void __css_rstat_updated(struct cgroup_subsys_state *css, int cpu)
71	72	{
72	73	struct llist_head *lhead;
73	74	struct css_rstat_cpu *rstatc;
74	75	struct llist_node *self;
75	76
76		− /*
77		− * Since bpf programs can call this function, prevent access to
78		− * uninitialized rstat pointers.
79		− */
	77	+ /* Prevent access to uninitialized rstat pointers. */
80	78	if (!css_uses_rstat(css))
81	79	return;
82	80
@@ -125,6 +123,18 @@ __bpf_kfunc void css_rstat_updated(struct cgroup_subsys_state *css, int cpu) · 42 unmodified lines
125	123	llist_add(&rstatc->lnode, lhead);
126	124	}
127	125
	126	+/*
	127	+ * BPF-facing wrapper for __css_rstat_updated(). Validate the caller-provided
	128	+ * CPU before passing it to the internal rstat updater.
	129	+ */
	130	+__bpf_kfunc void css_rstat_updated(struct cgroup_subsys_state *css, int cpu)
	131	+{
	132	+ if (unlikely(cpu < 0 \|\| cpu >= nr_cpu_ids \|\| !cpu_possible(cpu)))
	133	+ return;
	134	+
	135	+ __css_rstat_updated(css, cpu);
	136	+}
	137	+
128	138	static void __css_process_update_tree(struct cgroup_subsys_state *css, int cpu)
129	139	{
130	140	/* put @css and all ancestors on the corresponding updated lists */
@@ -170,7 +180,7 @@ static void css_process_update_tree(struct cgroup_subsys *ss, int cpu) · 39 unmodified lines
170	180	* flusher flush the stats updated by the updater who have
171	181	* observed that they are already on the list. The
172	182	* corresponding barrier pair for this one should be before
173		− * css_rstat_updated() by the user.
	183	+ * __css_rstat_updated() by the user.
174	184	*
175	185	* For now, there aren't any such user, so not adding the
176	186	* barrier here but if such a use-case arise, please add
@@ -614,7 +624,7 @@ static void cgroup_base_stat_cputime_account_end(struct cgroup *cgrp, · 437 unmodified lines
614	624	unsigned long flags)
615	625	{
616	626	u64_stats_update_end_irqrestore(&rstatbc->bsync, flags);
617		− css_rstat_updated(&cgrp->self, smp_processor_id());
	627	+ __css_rstat_updated(&cgrp->self, smp_processor_id());
618	628	put_cpu_ptr(rstatbc);
619	629	}
620	630

mm/memcontrol.c+3−3

@@ -579,7 +579,7 @@ static inline void memcg_rstat_updated(struct mem_cgroup *memcg, int val, · 578 unmodified lines
579	579	if (!val)
580	580	return;
581	581
582		− css_rstat_updated(&memcg->css, cpu);
	582	+ __css_rstat_updated(&memcg->css, cpu);
583	583	statc_pcpu = memcg->vmstats_percpu;
584	584	for (; statc_pcpu; statc_pcpu = statc->parent_pcpu) {
585	585	statc = this_cpu_ptr(statc_pcpu);
@@ -2608,7 +2608,7 @@ static inline void account_slab_nmi_safe(struct mem_cgroup *memcg, · 2022 unmodified lines
2608	2608	struct mem_cgroup_per_node *pn = memcg->nodeinfo[pgdat->node_id];
2609	2609
2610	2610	/* preemption is disabled in_nmi(). */
2611		− css_rstat_updated(&memcg->css, smp_processor_id());
	2611	+ __css_rstat_updated(&memcg->css, smp_processor_id());
2612	2612	if (idx == NR_SLAB_RECLAIMABLE_B)
2613	2613	atomic_add(nr, &pn->slab_reclaimable);
2614	2614	else
@@ -2832,7 +2832,7 @@ static inline void account_kmem_nmi_safe(struct mem_cgroup *memcg, int val) · 217 unmodified lines
2832	2832	mod_memcg_state(memcg, MEMCG_KMEM, val);
2833	2833	} else {
2834	2834	/* preemption is disabled in_nmi(). */
2835		− css_rstat_updated(&memcg->css, smp_processor_id());
	2835	+ __css_rstat_updated(&memcg->css, smp_processor_id());
2836	2836	atomic_add(val, &memcg->kmem_stat);
2837	2837	}
2838	2838	}