History

cgroup/rstat: validate cpu before css_rstat_cpu() access

css_rstat_updated() is exposed as a BPF kfunc and accepts a caller-provided cpu argument. The function uses cpu for per-cpu rstat lookups without checking whether it refers to a valid possible CPU. A BPF iter/cgroup program with CAP_BPF and CAP_PERFMON can pass an invalid cpu value. On an unfixed UBSCAN_BOUNDS test kernel, cpu == 0x7fffffff triggers: UBSAN: array-index-out-of-bounds in kernel/cgroup/rstat.c:31:9 index 2147483647 is out of range for type 'long unsigned int [64]' Call Trace: css_rstat_updated bpf_iter_run_prog cgroup_iter_seq_show bpf_seq_read Add cpu validation to the BPF-facing css_rstat_updated() kfunc and move the common implementation to __css_rstat_updated() for in-kernel callers.

Qing Ming committed 5 months ago

Commit 88170051 parent 345f4014 files · +25 −14Browse files

Unified · Split

4 files
  1. block/blk-cgroup.c +1 −1
  2. include/linux/cgroup.h +1 −0
  3. kernel/cgroup/rstat.c +20 −10
  4. mm/memcontrol.c +3 −3
block/blk-cgroup.c+1−1
@@ -2241,7 +2241,7 @@ void blk_cgroup_bio_start(struct bio *bio) · 2240 unmodified lines
2241 }2241 }
2242 2242
2243 u64_stats_update_end_irqrestore(&bis->sync, flags);2243 u64_stats_update_end_irqrestore(&bis->sync, flags);
2244 css_rstat_updated(&blkcg->css, cpu);2244 __css_rstat_updated(&blkcg->css, cpu);
2245 put_cpu();2245 put_cpu();
2246}2246}
2247 2247
include/linux/cgroup.h+1−0
@@ -776,6 +776,7 @@ static inline void cgroup_path_from_kernfs_id(u64 id, char *buf, size_t buflen) · 775 unmodified lines
776/*776/*
777 * cgroup scalable recursive statistics.777 * cgroup scalable recursive statistics.
778 */778 */
779void __css_rstat_updated(struct cgroup_subsys_state *css, int cpu);
779void css_rstat_updated(struct cgroup_subsys_state *css, int cpu);780void css_rstat_updated(struct cgroup_subsys_state *css, int cpu);
780void css_rstat_flush(struct cgroup_subsys_state *css);781void css_rstat_flush(struct cgroup_subsys_state *css);
781 782
kernel/cgroup/rstat.c+20−10
@@ -1,6 +1,7 @@
1// SPDX-License-Identifier: GPL-2.0-only1// SPDX-License-Identifier: GPL-2.0-only
2#include "cgroup-internal.h"2#include "cgroup-internal.h"
3 3
4#include <linux/cpumask.h>
4#include <linux/sched/cputime.h>5#include <linux/sched/cputime.h>
5 6
6#include <linux/bpf.h>7#include <linux/bpf.h>
@@ -53,7 +54,7 @@ static inline struct llist_head *ss_lhead_cpu(struct cgroup_subsys *ss, int cpu) · 46 unmodified lines
53}54}
54 55
55/**56/**
56 * css_rstat_updated - keep track of updated rstat_cpu57 * __css_rstat_updated - keep track of updated rstat_cpu
57 * @css: target cgroup subsystem state58 * @css: target cgroup subsystem state
58 * @cpu: cpu on which rstat_cpu was updated59 * @cpu: cpu on which rstat_cpu was updated
59 *60 *
@@ -63,20 +64,17 @@ static inline struct llist_head *ss_lhead_cpu(struct cgroup_subsys *ss, int cpu) · 3 unmodified lines
63 *64 *
64 * NOTE: if the user needs the guarantee that the updater either add itself in65 * NOTE: if the user needs the guarantee that the updater either add itself in
65 * the lockless list or the concurrent flusher flushes its updated stats, a66 * the lockless list or the concurrent flusher flushes its updated stats, a
66 * memory barrier is needed before the call to css_rstat_updated() i.e. a67 * memory barrier is needed before the call to __css_rstat_updated() i.e. a
67 * barrier after updating the per-cpu stats and before calling68 * barrier after updating the per-cpu stats and before calling
68 * css_rstat_updated().69 * __css_rstat_updated().
69 */70 */
70__bpf_kfunc void css_rstat_updated(struct cgroup_subsys_state *css, int cpu)71void __css_rstat_updated(struct cgroup_subsys_state *css, int cpu)
71{72{
72 struct llist_head *lhead;73 struct llist_head *lhead;
73 struct css_rstat_cpu *rstatc;74 struct css_rstat_cpu *rstatc;
74 struct llist_node *self;75 struct llist_node *self;
75 76
76 /*77 /* Prevent access to uninitialized rstat pointers. */
77 * Since bpf programs can call this function, prevent access to
78 * uninitialized rstat pointers.
79 */
80 if (!css_uses_rstat(css))78 if (!css_uses_rstat(css))
81 return;79 return;
82 80
@@ -125,6 +123,18 @@ __bpf_kfunc void css_rstat_updated(struct cgroup_subsys_state *css, int cpu) · 42 unmodified lines
125 llist_add(&rstatc->lnode, lhead);123 llist_add(&rstatc->lnode, lhead);
126}124}
127 125
126/*
127 * BPF-facing wrapper for __css_rstat_updated(). Validate the caller-provided
128 * CPU before passing it to the internal rstat updater.
129 */
130__bpf_kfunc void css_rstat_updated(struct cgroup_subsys_state *css, int cpu)
131{
132 if (unlikely(cpu < 0 || cpu >= nr_cpu_ids || !cpu_possible(cpu)))
133 return;
134
135 __css_rstat_updated(css, cpu);
136}
137
128static void __css_process_update_tree(struct cgroup_subsys_state *css, int cpu)138static void __css_process_update_tree(struct cgroup_subsys_state *css, int cpu)
129{139{
130 /* put @css and all ancestors on the corresponding updated lists */140 /* put @css and all ancestors on the corresponding updated lists */
@@ -170,7 +180,7 @@ static void css_process_update_tree(struct cgroup_subsys *ss, int cpu) · 39 unmodified lines
170 * flusher flush the stats updated by the updater who have180 * flusher flush the stats updated by the updater who have
171 * observed that they are already on the list. The181 * observed that they are already on the list. The
172 * corresponding barrier pair for this one should be before182 * corresponding barrier pair for this one should be before
173 * css_rstat_updated() by the user.183 * __css_rstat_updated() by the user.
174 *184 *
175 * For now, there aren't any such user, so not adding the185 * For now, there aren't any such user, so not adding the
176 * barrier here but if such a use-case arise, please add186 * barrier here but if such a use-case arise, please add
@@ -614,7 +624,7 @@ static void cgroup_base_stat_cputime_account_end(struct cgroup *cgrp, · 437 unmodified lines
614 unsigned long flags)624 unsigned long flags)
615{625{
616 u64_stats_update_end_irqrestore(&rstatbc->bsync, flags);626 u64_stats_update_end_irqrestore(&rstatbc->bsync, flags);
617 css_rstat_updated(&cgrp->self, smp_processor_id());627 __css_rstat_updated(&cgrp->self, smp_processor_id());
618 put_cpu_ptr(rstatbc);628 put_cpu_ptr(rstatbc);
619}629}
620 630
mm/memcontrol.c+3−3
@@ -579,7 +579,7 @@ static inline void memcg_rstat_updated(struct mem_cgroup *memcg, int val, · 578 unmodified lines
579 if (!val)579 if (!val)
580 return;580 return;
581 581
582 css_rstat_updated(&memcg->css, cpu);582 __css_rstat_updated(&memcg->css, cpu);
583 statc_pcpu = memcg->vmstats_percpu;583 statc_pcpu = memcg->vmstats_percpu;
584 for (; statc_pcpu; statc_pcpu = statc->parent_pcpu) {584 for (; statc_pcpu; statc_pcpu = statc->parent_pcpu) {
585 statc = this_cpu_ptr(statc_pcpu);585 statc = this_cpu_ptr(statc_pcpu);
@@ -2608,7 +2608,7 @@ static inline void account_slab_nmi_safe(struct mem_cgroup *memcg, · 2022 unmodified lines
2608 struct mem_cgroup_per_node *pn = memcg->nodeinfo[pgdat->node_id];2608 struct mem_cgroup_per_node *pn = memcg->nodeinfo[pgdat->node_id];
2609 2609
2610 /* preemption is disabled in_nmi(). */2610 /* preemption is disabled in_nmi(). */
2611 css_rstat_updated(&memcg->css, smp_processor_id());2611 __css_rstat_updated(&memcg->css, smp_processor_id());
2612 if (idx == NR_SLAB_RECLAIMABLE_B)2612 if (idx == NR_SLAB_RECLAIMABLE_B)
2613 atomic_add(nr, &pn->slab_reclaimable);2613 atomic_add(nr, &pn->slab_reclaimable);
2614 else2614 else
@@ -2832,7 +2832,7 @@ static inline void account_kmem_nmi_safe(struct mem_cgroup *memcg, int val) · 217 unmodified lines
2832 mod_memcg_state(memcg, MEMCG_KMEM, val);2832 mod_memcg_state(memcg, MEMCG_KMEM, val);
2833 } else {2833 } else {
2834 /* preemption is disabled in_nmi(). */2834 /* preemption is disabled in_nmi(). */
2835 css_rstat_updated(&memcg->css, smp_processor_id());2835 __css_rstat_updated(&memcg->css, smp_processor_id());
2836 atomic_add(val, &memcg->kmem_stat);2836 atomic_add(val, &memcg->kmem_stat);
2837 }2837 }
2838}2838}