History

bpf: Move backtracking logic to backtrack.c

Move precision propagation and backtracking logic to backtrack.c to reduce verifier.c size. No functional changes.

Alexei Starovoitov committed 7 months ago

Commit ed0b9711 parent c82834a4 files · +957 −945Browse files

Showing the changes to kernel (3 of 4 files). Show all

Unified · Split

3 files
  1. kernel/bpf/Makefile +1 −1
  2. kernel/bpf/backtrack.c +934 −0
  3. kernel/bpf/verifier.c +4 −944
kernel/bpf/Makefile+1−1
@@ -11,7 +11,7 @@ obj-$(CONFIG_BPF_SYSCALL) += bpf_iter.o map_iter.o task_iter.o prog_iter.o link_ · 10 unmodified lines
11obj-$(CONFIG_BPF_SYSCALL) += hashtab.o arraymap.o percpu_freelist.o bpf_lru_list.o lpm_trie.o map_in_map.o bloom_filter.o11obj-$(CONFIG_BPF_SYSCALL) += hashtab.o arraymap.o percpu_freelist.o bpf_lru_list.o lpm_trie.o map_in_map.o bloom_filter.o
12obj-$(CONFIG_BPF_SYSCALL) += local_storage.o queue_stack_maps.o ringbuf.o bpf_insn_array.o12obj-$(CONFIG_BPF_SYSCALL) += local_storage.o queue_stack_maps.o ringbuf.o bpf_insn_array.o
13obj-$(CONFIG_BPF_SYSCALL) += bpf_local_storage.o bpf_task_storage.o13obj-$(CONFIG_BPF_SYSCALL) += bpf_local_storage.o bpf_task_storage.o
14obj-$(CONFIG_BPF_SYSCALL) += fixups.o cfg.o states.o14obj-$(CONFIG_BPF_SYSCALL) += fixups.o cfg.o states.o backtrack.o
15obj-${CONFIG_BPF_LSM} += bpf_inode_storage.o15obj-${CONFIG_BPF_LSM} += bpf_inode_storage.o
16obj-$(CONFIG_BPF_SYSCALL) += disasm.o mprog.o16obj-$(CONFIG_BPF_SYSCALL) += disasm.o mprog.o
17obj-$(CONFIG_BPF_JIT) += trampoline.o17obj-$(CONFIG_BPF_JIT) += trampoline.o
kernel/bpf/backtrack.c+934−0
@@ -0,0 +1,934 @@
1// SPDX-License-Identifier: GPL-2.0-only
2/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */
3#include <linux/bpf.h>
4#include <linux/bpf_verifier.h>
5#include <linux/filter.h>
6#include <linux/bitmap.h>
7
8#define verbose(env, fmt, args...) bpf_verifier_log_write(env, fmt, ##args)
9
10/* for any branch, call, exit record the history of jmps in the given state */
11int bpf_push_jmp_history(struct bpf_verifier_env *env, struct bpf_verifier_state *cur,
12 int insn_flags, u64 linked_regs)
13{
14 u32 cnt = cur->jmp_history_cnt;
15 struct bpf_jmp_history_entry *p;
16 size_t alloc_size;
17
18 /* combine instruction flags if we already recorded this instruction */
19 if (env->cur_hist_ent) {
20 /* atomic instructions push insn_flags twice, for READ and
21 * WRITE sides, but they should agree on stack slot
22 */
23 verifier_bug_if((env->cur_hist_ent->flags & insn_flags) &&
24 (env->cur_hist_ent->flags & insn_flags) != insn_flags,
25 env, "insn history: insn_idx %d cur flags %x new flags %x",
26 env->insn_idx, env->cur_hist_ent->flags, insn_flags);
27 env->cur_hist_ent->flags |= insn_flags;
28 verifier_bug_if(env->cur_hist_ent->linked_regs != 0, env,
29 "insn history: insn_idx %d linked_regs: %#llx",
30 env->insn_idx, env->cur_hist_ent->linked_regs);
31 env->cur_hist_ent->linked_regs = linked_regs;
32 return 0;
33 }
34
35 cnt++;
36 alloc_size = kmalloc_size_roundup(size_mul(cnt, sizeof(*p)));
37 p = krealloc(cur->jmp_history, alloc_size, GFP_KERNEL_ACCOUNT);
38 if (!p)
39 return -ENOMEM;
40 cur->jmp_history = p;
41
42 p = &cur->jmp_history[cnt - 1];
43 p->idx = env->insn_idx;
44 p->prev_idx = env->prev_insn_idx;
45 p->flags = insn_flags;
46 p->linked_regs = linked_regs;
47 cur->jmp_history_cnt = cnt;
48 env->cur_hist_ent = p;
49
50 return 0;
51}
52
53static bool is_atomic_load_insn(const struct bpf_insn *insn)
54{
55 return BPF_CLASS(insn->code) == BPF_STX &&
56 BPF_MODE(insn->code) == BPF_ATOMIC &&
57 insn->imm == BPF_LOAD_ACQ;
58}
59
60static bool is_atomic_fetch_insn(const struct bpf_insn *insn)
61{
62 return BPF_CLASS(insn->code) == BPF_STX &&
63 BPF_MODE(insn->code) == BPF_ATOMIC &&
64 (insn->imm & BPF_FETCH);
65}
66
67static int insn_stack_access_spi(int insn_flags)
68{
69 return (insn_flags >> INSN_F_SPI_SHIFT) & INSN_F_SPI_MASK;
70}
71
72static int insn_stack_access_frameno(int insn_flags)
73{
74 return insn_flags & INSN_F_FRAMENO_MASK;
75}
76
77/* Backtrack one insn at a time. If idx is not at the top of recorded
78 * history then previous instruction came from straight line execution.
79 * Return -ENOENT if we exhausted all instructions within given state.
80 *
81 * It's legal to have a bit of a looping with the same starting and ending
82 * insn index within the same state, e.g.: 3->4->5->3, so just because current
83 * instruction index is the same as state's first_idx doesn't mean we are
84 * done. If there is still some jump history left, we should keep going. We
85 * need to take into account that we might have a jump history between given
86 * state's parent and itself, due to checkpointing. In this case, we'll have
87 * history entry recording a jump from last instruction of parent state and
88 * first instruction of given state.
89 */
90static int get_prev_insn_idx(struct bpf_verifier_state *st, int i,
91 u32 *history)
92{
93 u32 cnt = *history;
94
95 if (i == st->first_insn_idx) {
96 if (cnt == 0)
97 return -ENOENT;
98 if (cnt == 1 && st->jmp_history[0].idx == i)
99 return -ENOENT;
100 }
101
102 if (cnt && st->jmp_history[cnt - 1].idx == i) {
103 i = st->jmp_history[cnt - 1].prev_idx;
104 (*history)--;
105 } else {
106 i--;
107 }
108 return i;
109}
110
111static struct bpf_jmp_history_entry *get_jmp_hist_entry(struct bpf_verifier_state *st,
112 u32 hist_end, int insn_idx)
113{
114 if (hist_end > 0 && st->jmp_history[hist_end - 1].idx == insn_idx)
115 return &st->jmp_history[hist_end - 1];
116 return NULL;
117}
118
119static inline void bt_init(struct backtrack_state *bt, u32 frame)
120{
121 bt->frame = frame;
122}
123
124static inline void bt_reset(struct backtrack_state *bt)
125{
126 struct bpf_verifier_env *env = bt->env;
127
128 memset(bt, 0, sizeof(*bt));
129 bt->env = env;
130}
131
132static inline u32 bt_empty(struct backtrack_state *bt)
133{
134 u64 mask = 0;
135 int i;
136
137 for (i = 0; i <= bt->frame; i++)
138 mask |= bt->reg_masks[i] | bt->stack_masks[i];
139
140 return mask == 0;
141}
142
143static inline int bt_subprog_enter(struct backtrack_state *bt)
144{
145 if (bt->frame == MAX_CALL_FRAMES - 1) {
146 verifier_bug(bt->env, "subprog enter from frame %d", bt->frame);
147 return -EFAULT;
148 }
149 bt->frame++;
150 return 0;
151}
152
153static inline int bt_subprog_exit(struct backtrack_state *bt)
154{
155 if (bt->frame == 0) {
156 verifier_bug(bt->env, "subprog exit from frame 0");
157 return -EFAULT;
158 }
159 bt->frame--;
160 return 0;
161}
162
163static inline void bt_clear_frame_reg(struct backtrack_state *bt, u32 frame, u32 reg)
164{
165 bt->reg_masks[frame] &= ~(1 << reg);
166}
167
168static inline void bt_set_reg(struct backtrack_state *bt, u32 reg)
169{
170 bpf_bt_set_frame_reg(bt, bt->frame, reg);
171}
172
173static inline void bt_clear_reg(struct backtrack_state *bt, u32 reg)
174{
175 bt_clear_frame_reg(bt, bt->frame, reg);
176}
177
178static inline void bt_clear_frame_slot(struct backtrack_state *bt, u32 frame, u32 slot)
179{
180 bt->stack_masks[frame] &= ~(1ull << slot);
181}
182
183static inline u32 bt_frame_reg_mask(struct backtrack_state *bt, u32 frame)
184{
185 return bt->reg_masks[frame];
186}
187
188static inline u32 bt_reg_mask(struct backtrack_state *bt)
189{
190 return bt->reg_masks[bt->frame];
191}
192
193static inline u64 bt_frame_stack_mask(struct backtrack_state *bt, u32 frame)
194{
195 return bt->stack_masks[frame];
196}
197
198static inline u64 bt_stack_mask(struct backtrack_state *bt)
199{
200 return bt->stack_masks[bt->frame];
201}
202
203static inline bool bt_is_reg_set(struct backtrack_state *bt, u32 reg)
204{
205 return bt->reg_masks[bt->frame] & (1 << reg);
206}
207
208
209/* format registers bitmask, e.g., "r0,r2,r4" for 0x15 mask */
210static void fmt_reg_mask(char *buf, ssize_t buf_sz, u32 reg_mask)
211{
212 DECLARE_BITMAP(mask, 64);
213 bool first = true;
214 int i, n;
215
216 buf[0] = '\0';
217
218 bitmap_from_u64(mask, reg_mask);
219 for_each_set_bit(i, mask, 32) {
220 n = snprintf(buf, buf_sz, "%sr%d", first ? "" : ",", i);
221 first = false;
222 buf += n;
223 buf_sz -= n;
224 if (buf_sz < 0)
225 break;
226 }
227}
228/* format stack slots bitmask, e.g., "-8,-24,-40" for 0x15 mask */
229void bpf_fmt_stack_mask(char *buf, ssize_t buf_sz, u64 stack_mask)
230{
231 DECLARE_BITMAP(mask, 64);
232 bool first = true;
233 int i, n;
234
235 buf[0] = '\0';
236
237 bitmap_from_u64(mask, stack_mask);
238 for_each_set_bit(i, mask, 64) {
239 n = snprintf(buf, buf_sz, "%s%d", first ? "" : ",", -(i + 1) * 8);
240 first = false;
241 buf += n;
242 buf_sz -= n;
243 if (buf_sz < 0)
244 break;
245 }
246}
247
248
249/* For given verifier state backtrack_insn() is called from the last insn to
250 * the first insn. Its purpose is to compute a bitmask of registers and
251 * stack slots that needs precision in the parent verifier state.
252 *
253 * @idx is an index of the instruction we are currently processing;
254 * @subseq_idx is an index of the subsequent instruction that:
255 * - *would be* executed next, if jump history is viewed in forward order;
256 * - *was* processed previously during backtracking.
257 */
258static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx,
259 struct bpf_jmp_history_entry *hist, struct backtrack_state *bt)
260{
261 struct bpf_insn *insn = env->prog->insnsi + idx;
262 u8 class = BPF_CLASS(insn->code);
263 u8 opcode = BPF_OP(insn->code);
264 u8 mode = BPF_MODE(insn->code);
265 u32 dreg = insn->dst_reg;
266 u32 sreg = insn->src_reg;
267 u32 spi, i, fr;
268
269 if (insn->code == 0)
270 return 0;
271 if (env->log.level & BPF_LOG_LEVEL2) {
272 fmt_reg_mask(env->tmp_str_buf, TMP_STR_BUF_LEN, bt_reg_mask(bt));
273 verbose(env, "mark_precise: frame%d: regs=%s ",
274 bt->frame, env->tmp_str_buf);
275 bpf_fmt_stack_mask(env->tmp_str_buf, TMP_STR_BUF_LEN, bt_stack_mask(bt));
276 verbose(env, "stack=%s before ", env->tmp_str_buf);
277 verbose(env, "%d: ", idx);
278 bpf_verbose_insn(env, insn);
279 }
280
281 /* If there is a history record that some registers gained range at this insn,
282 * propagate precision marks to those registers, so that bt_is_reg_set()
283 * accounts for these registers.
284 */
285 bpf_bt_sync_linked_regs(bt, hist);
286
287 if (class == BPF_ALU || class == BPF_ALU64) {
288 if (!bt_is_reg_set(bt, dreg))
289 return 0;
290 if (opcode == BPF_END || opcode == BPF_NEG) {
291 /* sreg is reserved and unused
292 * dreg still need precision before this insn
293 */
294 return 0;
295 } else if (opcode == BPF_MOV) {
296 if (BPF_SRC(insn->code) == BPF_X) {
297 /* dreg = sreg or dreg = (s8, s16, s32)sreg
298 * dreg needs precision after this insn
299 * sreg needs precision before this insn
300 */
301 bt_clear_reg(bt, dreg);
302 if (sreg != BPF_REG_FP)
303 bt_set_reg(bt, sreg);
304 } else {
305 /* dreg = K
306 * dreg needs precision after this insn.
307 * Corresponding register is already marked
308 * as precise=true in this verifier state.
309 * No further markings in parent are necessary
310 */
311 bt_clear_reg(bt, dreg);
312 }
313 } else {
314 if (BPF_SRC(insn->code) == BPF_X) {
315 /* dreg += sreg
316 * both dreg and sreg need precision
317 * before this insn
318 */
319 if (sreg != BPF_REG_FP)
320 bt_set_reg(bt, sreg);
321 } /* else dreg += K
322 * dreg still needs precision before this insn
323 */
324 }
325 } else if (class == BPF_LDX ||
326 is_atomic_load_insn(insn) ||
327 is_atomic_fetch_insn(insn)) {
328 u32 load_reg = dreg;
329
330 /*
331 * Atomic fetch operation writes the old value into
332 * a register (sreg or r0) and if it was tracked for
333 * precision, propagate to the stack slot like we do
334 * in regular ldx.
335 */
336 if (is_atomic_fetch_insn(insn))
337 load_reg = insn->imm == BPF_CMPXCHG ?
338 BPF_REG_0 : sreg;
339
340 if (!bt_is_reg_set(bt, load_reg))
341 return 0;
342 bt_clear_reg(bt, load_reg);
343
344 /* scalars can only be spilled into stack w/o losing precision.
345 * Load from any other memory can be zero extended.
346 * The desire to keep that precision is already indicated
347 * by 'precise' mark in corresponding register of this state.
348 * No further tracking necessary.
349 */
350 if (!hist || !(hist->flags & INSN_F_STACK_ACCESS))
351 return 0;
352 /* dreg = *(u64 *)[fp - off] was a fill from the stack.
353 * that [fp - off] slot contains scalar that needs to be
354 * tracked with precision
355 */
356 spi = insn_stack_access_spi(hist->flags);
357 fr = insn_stack_access_frameno(hist->flags);
358 bpf_bt_set_frame_slot(bt, fr, spi);
359 } else if (class == BPF_STX || class == BPF_ST) {
360 if (bt_is_reg_set(bt, dreg))
361 /* stx & st shouldn't be using _scalar_ dst_reg
362 * to access memory. It means backtracking
363 * encountered a case of pointer subtraction.
364 */
365 return -ENOTSUPP;
366 /* scalars can only be spilled into stack */
367 if (!hist || !(hist->flags & INSN_F_STACK_ACCESS))
368 return 0;
369 spi = insn_stack_access_spi(hist->flags);
370 fr = insn_stack_access_frameno(hist->flags);
371 if (!bt_is_frame_slot_set(bt, fr, spi))
372 return 0;
373 bt_clear_frame_slot(bt, fr, spi);
374 if (class == BPF_STX)
375 bt_set_reg(bt, sreg);
376 } else if (class == BPF_JMP || class == BPF_JMP32) {
377 if (bpf_pseudo_call(insn)) {
378 int subprog_insn_idx, subprog;
379
380 subprog_insn_idx = idx + insn->imm + 1;
381 subprog = bpf_find_subprog(env, subprog_insn_idx);
382 if (subprog < 0)
383 return -EFAULT;
384
385 if (bpf_subprog_is_global(env, subprog)) {
386 /* check that jump history doesn't have any
387 * extra instructions from subprog; the next
388 * instruction after call to global subprog
389 * should be literally next instruction in
390 * caller program
391 */
392 verifier_bug_if(idx + 1 != subseq_idx, env,
393 "extra insn from subprog");
394 /* r1-r5 are invalidated after subprog call,
395 * so for global func call it shouldn't be set
396 * anymore
397 */
398 if (bt_reg_mask(bt) & BPF_REGMASK_ARGS) {
399 verifier_bug(env, "global subprog unexpected regs %x",
400 bt_reg_mask(bt));
401 return -EFAULT;
402 }
403 /* global subprog always sets R0 */
404 bt_clear_reg(bt, BPF_REG_0);
405 return 0;
406 } else {
407 /* static subprog call instruction, which
408 * means that we are exiting current subprog,
409 * so only r1-r5 could be still requested as
410 * precise, r0 and r6-r10 or any stack slot in
411 * the current frame should be zero by now
412 */
413 if (bt_reg_mask(bt) & ~BPF_REGMASK_ARGS) {
414 verifier_bug(env, "static subprog unexpected regs %x",
415 bt_reg_mask(bt));
416 return -EFAULT;
417 }
418 /* we are now tracking register spills correctly,
419 * so any instance of leftover slots is a bug
420 */
421 if (bt_stack_mask(bt) != 0) {
422 verifier_bug(env,
423 "static subprog leftover stack slots %llx",
424 bt_stack_mask(bt));
425 return -EFAULT;
426 }
427 /* propagate r1-r5 to the caller */
428 for (i = BPF_REG_1; i <= BPF_REG_5; i++) {
429 if (bt_is_reg_set(bt, i)) {
430 bt_clear_reg(bt, i);
431 bpf_bt_set_frame_reg(bt, bt->frame - 1, i);
432 }
433 }
434 if (bt_subprog_exit(bt))
435 return -EFAULT;
436 return 0;
437 }
438 } else if (bpf_is_sync_callback_calling_insn(insn) && idx != subseq_idx - 1) {
439 /* exit from callback subprog to callback-calling helper or
440 * kfunc call. Use idx/subseq_idx check to discern it from
441 * straight line code backtracking.
442 * Unlike the subprog call handling above, we shouldn't
443 * propagate precision of r1-r5 (if any requested), as they are
444 * not actually arguments passed directly to callback subprogs
445 */
446 if (bt_reg_mask(bt) & ~BPF_REGMASK_ARGS) {
447 verifier_bug(env, "callback unexpected regs %x",
448 bt_reg_mask(bt));
449 return -EFAULT;
450 }
451 if (bt_stack_mask(bt) != 0) {
452 verifier_bug(env, "callback leftover stack slots %llx",
453 bt_stack_mask(bt));
454 return -EFAULT;
455 }
456 /* clear r1-r5 in callback subprog's mask */
457 for (i = BPF_REG_1; i <= BPF_REG_5; i++)
458 bt_clear_reg(bt, i);
459 if (bt_subprog_exit(bt))
460 return -EFAULT;
461 return 0;
462 } else if (opcode == BPF_CALL) {
463 /* kfunc with imm==0 is invalid and fixup_kfunc_call will
464 * catch this error later. Make backtracking conservative
465 * with ENOTSUPP.
466 */
467 if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL && insn->imm == 0)
468 return -ENOTSUPP;
469 /* regular helper call sets R0 */
470 bt_clear_reg(bt, BPF_REG_0);
471 if (bt_reg_mask(bt) & BPF_REGMASK_ARGS) {
472 /* if backtracking was looking for registers R1-R5
473 * they should have been found already.
474 */
475 verifier_bug(env, "backtracking call unexpected regs %x",
476 bt_reg_mask(bt));
477 return -EFAULT;
478 }
479 if (insn->src_reg == BPF_REG_0 && insn->imm == BPF_FUNC_tail_call
480 && subseq_idx - idx != 1) {
481 if (bt_subprog_enter(bt))
482 return -EFAULT;
483 }
484 } else if (opcode == BPF_EXIT) {
485 bool r0_precise;
486
487 /* Backtracking to a nested function call, 'idx' is a part of
488 * the inner frame 'subseq_idx' is a part of the outer frame.
489 * In case of a regular function call, instructions giving
490 * precision to registers R1-R5 should have been found already.
491 * In case of a callback, it is ok to have R1-R5 marked for
492 * backtracking, as these registers are set by the function
493 * invoking callback.
494 */
495 if (subseq_idx >= 0 && bpf_calls_callback(env, subseq_idx))
496 for (i = BPF_REG_1; i <= BPF_REG_5; i++)
497 bt_clear_reg(bt, i);
498 if (bt_reg_mask(bt) & BPF_REGMASK_ARGS) {
499 verifier_bug(env, "backtracking exit unexpected regs %x",
500 bt_reg_mask(bt));
501 return -EFAULT;
502 }
503
504 /* BPF_EXIT in subprog or callback always returns
505 * right after the call instruction, so by checking
506 * whether the instruction at subseq_idx-1 is subprog
507 * call or not we can distinguish actual exit from
508 * *subprog* from exit from *callback*. In the former
509 * case, we need to propagate r0 precision, if
510 * necessary. In the former we never do that.
511 */
512 r0_precise = subseq_idx - 1 >= 0 &&
513 bpf_pseudo_call(&env->prog->insnsi[subseq_idx - 1]) &&
514 bt_is_reg_set(bt, BPF_REG_0);
515
516 bt_clear_reg(bt, BPF_REG_0);
517 if (bt_subprog_enter(bt))
518 return -EFAULT;
519
520 if (r0_precise)
521 bt_set_reg(bt, BPF_REG_0);
522 /* r6-r9 and stack slots will stay set in caller frame
523 * bitmasks until we return back from callee(s)
524 */
525 return 0;
526 } else if (BPF_SRC(insn->code) == BPF_X) {
527 if (!bt_is_reg_set(bt, dreg) && !bt_is_reg_set(bt, sreg))
528 return 0;
529 /* dreg <cond> sreg
530 * Both dreg and sreg need precision before
531 * this insn. If only sreg was marked precise
532 * before it would be equally necessary to
533 * propagate it to dreg.
534 */
535 if (!hist || !(hist->flags & INSN_F_SRC_REG_STACK))
536 bt_set_reg(bt, sreg);
537 if (!hist || !(hist->flags & INSN_F_DST_REG_STACK))
538 bt_set_reg(bt, dreg);
539 } else if (BPF_SRC(insn->code) == BPF_K) {
540 /* dreg <cond> K
541 * Only dreg still needs precision before
542 * this insn, so for the K-based conditional
543 * there is nothing new to be marked.
544 */
545 }
546 } else if (class == BPF_LD) {
547 if (!bt_is_reg_set(bt, dreg))
548 return 0;
549 bt_clear_reg(bt, dreg);
550 /* It's ld_imm64 or ld_abs or ld_ind.
551 * For ld_imm64 no further tracking of precision
552 * into parent is necessary
553 */
554 if (mode == BPF_IND || mode == BPF_ABS)
555 /* to be analyzed */
556 return -ENOTSUPP;
557 }
558 /* Propagate precision marks to linked registers, to account for
559 * registers marked as precise in this function.
560 */
561 bpf_bt_sync_linked_regs(bt, hist);
562 return 0;
563}
564
565/* the scalar precision tracking algorithm:
566 * . at the start all registers have precise=false.
567 * . scalar ranges are tracked as normal through alu and jmp insns.
568 * . once precise value of the scalar register is used in:
569 * . ptr + scalar alu
570 * . if (scalar cond K|scalar)
571 * . helper_call(.., scalar, ...) where ARG_CONST is expected
572 * backtrack through the verifier states and mark all registers and
573 * stack slots with spilled constants that these scalar registers
574 * should be precise.
575 * . during state pruning two registers (or spilled stack slots)
576 * are equivalent if both are not precise.
577 *
578 * Note the verifier cannot simply walk register parentage chain,
579 * since many different registers and stack slots could have been
580 * used to compute single precise scalar.
581 *
582 * The approach of starting with precise=true for all registers and then
583 * backtrack to mark a register as not precise when the verifier detects
584 * that program doesn't care about specific value (e.g., when helper
585 * takes register as ARG_ANYTHING parameter) is not safe.
586 *
587 * It's ok to walk single parentage chain of the verifier states.
588 * It's possible that this backtracking will go all the way till 1st insn.
589 * All other branches will be explored for needing precision later.
590 *
591 * The backtracking needs to deal with cases like:
592 * R8=map_value(id=0,off=0,ks=4,vs=1952,imm=0) R9_w=map_value(id=0,off=40,ks=4,vs=1952,imm=0)
593 * r9 -= r8
594 * r5 = r9
595 * if r5 > 0x79f goto pc+7
596 * R5_w=inv(id=0,umax_value=1951,var_off=(0x0; 0x7ff))
597 * r5 += 1
598 * ...
599 * call bpf_perf_event_output#25
600 * where .arg5_type = ARG_CONST_SIZE_OR_ZERO
601 *
602 * and this case:
603 * r6 = 1
604 * call foo // uses callee's r6 inside to compute r0
605 * r0 += r6
606 * if r0 == 0 goto
607 *
608 * to track above reg_mask/stack_mask needs to be independent for each frame.
609 *
610 * Also if parent's curframe > frame where backtracking started,
611 * the verifier need to mark registers in both frames, otherwise callees
612 * may incorrectly prune callers. This is similar to
613 * commit 7640ead93924 ("bpf: verifier: make sure callees don't prune with caller differences")
614 *
615 * For now backtracking falls back into conservative marking.
616 */
617void bpf_mark_all_scalars_precise(struct bpf_verifier_env *env,
618 struct bpf_verifier_state *st)
619{
620 struct bpf_func_state *func;
621 struct bpf_reg_state *reg;
622 int i, j;
623
624 if (env->log.level & BPF_LOG_LEVEL2) {
625 verbose(env, "mark_precise: frame%d: falling back to forcing all scalars precise\n",
626 st->curframe);
627 }
628
629 /* big hammer: mark all scalars precise in this path.
630 * pop_stack may still get !precise scalars.
631 * We also skip current state and go straight to first parent state,
632 * because precision markings in current non-checkpointed state are
633 * not needed. See why in the comment in __mark_chain_precision below.
634 */
635 for (st = st->parent; st; st = st->parent) {
636 for (i = 0; i <= st->curframe; i++) {
637 func = st->frame[i];
638 for (j = 0; j < BPF_REG_FP; j++) {
639 reg = &func->regs[j];
640 if (reg->type != SCALAR_VALUE || reg->precise)
641 continue;
642 reg->precise = true;
643 if (env->log.level & BPF_LOG_LEVEL2) {
644 verbose(env, "force_precise: frame%d: forcing r%d to be precise\n",
645 i, j);
646 }
647 }
648 for (j = 0; j < func->allocated_stack / BPF_REG_SIZE; j++) {
649 if (!bpf_is_spilled_reg(&func->stack[j]))
650 continue;
651 reg = &func->stack[j].spilled_ptr;
652 if (reg->type != SCALAR_VALUE || reg->precise)
653 continue;
654 reg->precise = true;
655 if (env->log.level & BPF_LOG_LEVEL2) {
656 verbose(env, "force_precise: frame%d: forcing fp%d to be precise\n",
657 i, -(j + 1) * 8);
658 }
659 }
660 }
661 }
662}
663
664/*
665 * bpf_mark_chain_precision() backtracks BPF program instruction sequence and
666 * chain of verifier states making sure that register *regno* (if regno >= 0)
667 * and/or stack slot *spi* (if spi >= 0) are marked as precisely tracked
668 * SCALARS, as well as any other registers and slots that contribute to
669 * a tracked state of given registers/stack slots, depending on specific BPF
670 * assembly instructions (see backtrack_insns() for exact instruction handling
671 * logic). This backtracking relies on recorded jmp_history and is able to
672 * traverse entire chain of parent states. This process ends only when all the
673 * necessary registers/slots and their transitive dependencies are marked as
674 * precise.
675 *
676 * One important and subtle aspect is that precise marks *do not matter* in
677 * the currently verified state (current state). It is important to understand
678 * why this is the case.
679 *
680 * First, note that current state is the state that is not yet "checkpointed",
681 * i.e., it is not yet put into env->explored_states, and it has no children
682 * states as well. It's ephemeral, and can end up either a) being discarded if
683 * compatible explored state is found at some point or BPF_EXIT instruction is
684 * reached or b) checkpointed and put into env->explored_states, branching out
685 * into one or more children states.
686 *
687 * In the former case, precise markings in current state are completely
688 * ignored by state comparison code (see regsafe() for details). Only
689 * checkpointed ("old") state precise markings are important, and if old
690 * state's register/slot is precise, regsafe() assumes current state's
691 * register/slot as precise and checks value ranges exactly and precisely. If
692 * states turn out to be compatible, current state's necessary precise
693 * markings and any required parent states' precise markings are enforced
694 * after the fact with propagate_precision() logic, after the fact. But it's
695 * important to realize that in this case, even after marking current state
696 * registers/slots as precise, we immediately discard current state. So what
697 * actually matters is any of the precise markings propagated into current
698 * state's parent states, which are always checkpointed (due to b) case above).
699 * As such, for scenario a) it doesn't matter if current state has precise
700 * markings set or not.
701 *
702 * Now, for the scenario b), checkpointing and forking into child(ren)
703 * state(s). Note that before current state gets to checkpointing step, any
704 * processed instruction always assumes precise SCALAR register/slot
705 * knowledge: if precise value or range is useful to prune jump branch, BPF
706 * verifier takes this opportunity enthusiastically. Similarly, when
707 * register's value is used to calculate offset or memory address, exact
708 * knowledge of SCALAR range is assumed, checked, and enforced. So, similar to
709 * what we mentioned above about state comparison ignoring precise markings
710 * during state comparison, BPF verifier ignores and also assumes precise
711 * markings *at will* during instruction verification process. But as verifier
712 * assumes precision, it also propagates any precision dependencies across
713 * parent states, which are not yet finalized, so can be further restricted
714 * based on new knowledge gained from restrictions enforced by their children
715 * states. This is so that once those parent states are finalized, i.e., when
716 * they have no more active children state, state comparison logic in
717 * is_state_visited() would enforce strict and precise SCALAR ranges, if
718 * required for correctness.
719 *
720 * To build a bit more intuition, note also that once a state is checkpointed,
721 * the path we took to get to that state is not important. This is crucial
722 * property for state pruning. When state is checkpointed and finalized at
723 * some instruction index, it can be correctly and safely used to "short
724 * circuit" any *compatible* state that reaches exactly the same instruction
725 * index. I.e., if we jumped to that instruction from a completely different
726 * code path than original finalized state was derived from, it doesn't
727 * matter, current state can be discarded because from that instruction
728 * forward having a compatible state will ensure we will safely reach the
729 * exit. States describe preconditions for further exploration, but completely
730 * forget the history of how we got here.
731 *
732 * This also means that even if we needed precise SCALAR range to get to
733 * finalized state, but from that point forward *that same* SCALAR register is
734 * never used in a precise context (i.e., it's precise value is not needed for
735 * correctness), it's correct and safe to mark such register as "imprecise"
736 * (i.e., precise marking set to false). This is what we rely on when we do
737 * not set precise marking in current state. If no child state requires
738 * precision for any given SCALAR register, it's safe to dictate that it can
739 * be imprecise. If any child state does require this register to be precise,
740 * we'll mark it precise later retroactively during precise markings
741 * propagation from child state to parent states.
742 *
743 * Skipping precise marking setting in current state is a mild version of
744 * relying on the above observation. But we can utilize this property even
745 * more aggressively by proactively forgetting any precise marking in the
746 * current state (which we inherited from the parent state), right before we
747 * checkpoint it and branch off into new child state. This is done by
748 * mark_all_scalars_imprecise() to hopefully get more permissive and generic
749 * finalized states which help in short circuiting more future states.
750 */
751int bpf_mark_chain_precision(struct bpf_verifier_env *env,
752 struct bpf_verifier_state *starting_state,
753 int regno,
754 bool *changed)
755{
756 struct bpf_verifier_state *st = starting_state;
757 struct backtrack_state *bt = &env->bt;
758 int first_idx = st->first_insn_idx;
759 int last_idx = starting_state->insn_idx;
760 int subseq_idx = -1;
761 struct bpf_func_state *func;
762 bool tmp, skip_first = true;
763 struct bpf_reg_state *reg;
764 int i, fr, err;
765
766 if (!env->bpf_capable)
767 return 0;
768
769 changed = changed ?: &tmp;
770 /* set frame number from which we are starting to backtrack */
771 bt_init(bt, starting_state->curframe);
772
773 /* Do sanity checks against current state of register and/or stack
774 * slot, but don't set precise flag in current state, as precision
775 * tracking in the current state is unnecessary.
776 */
777 func = st->frame[bt->frame];
778 if (regno >= 0) {
779 reg = &func->regs[regno];
780 if (reg->type != SCALAR_VALUE) {
781 verifier_bug(env, "backtracking misuse");
782 return -EFAULT;
783 }
784 bt_set_reg(bt, regno);
785 }
786
787 if (bt_empty(bt))
788 return 0;
789
790 for (;;) {
791 DECLARE_BITMAP(mask, 64);
792 u32 history = st->jmp_history_cnt;
793 struct bpf_jmp_history_entry *hist;
794
795 if (env->log.level & BPF_LOG_LEVEL2) {
796 verbose(env, "mark_precise: frame%d: last_idx %d first_idx %d subseq_idx %d \n",
797 bt->frame, last_idx, first_idx, subseq_idx);
798 }
799
800 if (last_idx < 0) {
801 /* we are at the entry into subprog, which
802 * is expected for global funcs, but only if
803 * requested precise registers are R1-R5
804 * (which are global func's input arguments)
805 */
806 if (st->curframe == 0 &&
807 st->frame[0]->subprogno > 0 &&
808 st->frame[0]->callsite == BPF_MAIN_FUNC &&
809 bt_stack_mask(bt) == 0 &&
810 (bt_reg_mask(bt) & ~BPF_REGMASK_ARGS) == 0) {
811 bitmap_from_u64(mask, bt_reg_mask(bt));
812 for_each_set_bit(i, mask, 32) {
813 reg = &st->frame[0]->regs[i];
814 bt_clear_reg(bt, i);
815 if (reg->type == SCALAR_VALUE) {
816 reg->precise = true;
817 *changed = true;
818 }
819 }
820 return 0;
821 }
822
823 verifier_bug(env, "backtracking func entry subprog %d reg_mask %x stack_mask %llx",
824 st->frame[0]->subprogno, bt_reg_mask(bt), bt_stack_mask(bt));
825 return -EFAULT;
826 }
827
828 for (i = last_idx;;) {
829 if (skip_first) {
830 err = 0;
831 skip_first = false;
832 } else {
833 hist = get_jmp_hist_entry(st, history, i);
834 err = backtrack_insn(env, i, subseq_idx, hist, bt);
835 }
836 if (err == -ENOTSUPP) {
837 bpf_mark_all_scalars_precise(env, starting_state);
838 bt_reset(bt);
839 return 0;
840 } else if (err) {
841 return err;
842 }
843 if (bt_empty(bt))
844 /* Found assignment(s) into tracked register in this state.
845 * Since this state is already marked, just return.
846 * Nothing to be tracked further in the parent state.
847 */
848 return 0;
849 subseq_idx = i;
850 i = get_prev_insn_idx(st, i, &history);
851 if (i == -ENOENT)
852 break;
853 if (i >= env->prog->len) {
854 /* This can happen if backtracking reached insn 0
855 * and there are still reg_mask or stack_mask
856 * to backtrack.
857 * It means the backtracking missed the spot where
858 * particular register was initialized with a constant.
859 */
860 verifier_bug(env, "backtracking idx %d", i);
861 return -EFAULT;
862 }
863 }
864 st = st->parent;
865 if (!st)
866 break;
867
868 for (fr = bt->frame; fr >= 0; fr--) {
869 func = st->frame[fr];
870 bitmap_from_u64(mask, bt_frame_reg_mask(bt, fr));
871 for_each_set_bit(i, mask, 32) {
872 reg = &func->regs[i];
873 if (reg->type != SCALAR_VALUE) {
874 bt_clear_frame_reg(bt, fr, i);
875 continue;
876 }
877 if (reg->precise) {
878 bt_clear_frame_reg(bt, fr, i);
879 } else {
880 reg->precise = true;
881 *changed = true;
882 }
883 }
884
885 bitmap_from_u64(mask, bt_frame_stack_mask(bt, fr));
886 for_each_set_bit(i, mask, 64) {
887 if (verifier_bug_if(i >= func->allocated_stack / BPF_REG_SIZE,
888 env, "stack slot %d, total slots %d",
889 i, func->allocated_stack / BPF_REG_SIZE))
890 return -EFAULT;
891
892 if (!bpf_is_spilled_scalar_reg(&func->stack[i])) {
893 bt_clear_frame_slot(bt, fr, i);
894 continue;
895 }
896 reg = &func->stack[i].spilled_ptr;
897 if (reg->precise) {
898 bt_clear_frame_slot(bt, fr, i);
899 } else {
900 reg->precise = true;
901 *changed = true;
902 }
903 }
904 if (env->log.level & BPF_LOG_LEVEL2) {
905 fmt_reg_mask(env->tmp_str_buf, TMP_STR_BUF_LEN,
906 bt_frame_reg_mask(bt, fr));
907 verbose(env, "mark_precise: frame%d: parent state regs=%s ",
908 fr, env->tmp_str_buf);
909 bpf_fmt_stack_mask(env->tmp_str_buf, TMP_STR_BUF_LEN,
910 bt_frame_stack_mask(bt, fr));
911 verbose(env, "stack=%s: ", env->tmp_str_buf);
912 print_verifier_state(env, st, fr, true);
913 }
914 }
915
916 if (bt_empty(bt))
917 return 0;
918
919 subseq_idx = first_idx;
920 last_idx = st->last_insn_idx;
921 first_idx = st->first_insn_idx;
922 }
923
924 /* if we still have requested precise regs or slots, we missed
925 * something (e.g., stack access through non-r10 register), so
926 * fallback to marking all precise
927 */
928 if (!bt_empty(bt)) {
929 bpf_mark_all_scalars_precise(env, starting_state);
930 bt_reset(bt);
931 }
932
933 return 0;
934}
kernel/bpf/verifier.c+4−944
@@ -513,20 +513,6 @@ static bool helper_multiple_ref_obj_use(enum bpf_func_id func_id, · 512 unmodified lines
513 return ref_obj_uses > 1;513 return ref_obj_uses > 1;
514}514}
515 515
516static bool is_atomic_load_insn(const struct bpf_insn *insn)
517{
518 return BPF_CLASS(insn->code) == BPF_STX &&
519 BPF_MODE(insn->code) == BPF_ATOMIC &&
520 insn->imm == BPF_LOAD_ACQ;
521}
522
523static bool is_atomic_fetch_insn(const struct bpf_insn *insn)
524{
525 return BPF_CLASS(insn->code) == BPF_STX &&
526 BPF_MODE(insn->code) == BPF_ATOMIC &&
527 (insn->imm & BPF_FETCH);
528}
529
530 516
531static bool is_spi_bounds_valid(struct bpf_func_state *state, int spi, int nr_slots)517static bool is_spi_bounds_valid(struct bpf_func_state *state, int spi, int nr_slots)
532{518{
@@ -1241,11 +1227,6 @@ static bool is_stack_slot_special(const struct bpf_stack_state *stack) · 708 unmodified lines
1241/* The reg state of a pointer or a bounded scalar was saved when1227/* The reg state of a pointer or a bounded scalar was saved when
1242 * it was spilled to the stack.1228 * it was spilled to the stack.
1243 */1229 */
1244static bool is_spilled_scalar_reg(const struct bpf_stack_state *stack)
1245{
1246 return stack->slot_type[BPF_REG_SIZE - 1] == STACK_SPILL &&
1247 stack->spilled_ptr.type == SCALAR_VALUE;
1248}
1249 1230
1250/*1231/*
1251 * Mark stack slot as STACK_MISC, unless it is already:1232 * Mark stack slot as STACK_MISC, unless it is already:
@@ -2590,7 +2571,6 @@ static struct bpf_retval_range retval_range(s32 minval, s32 maxval) · 1338 unmodified lines
2590 return (struct bpf_retval_range){ minval, maxval, false };2571 return (struct bpf_retval_range){ minval, maxval, false };
2591}2572}
2592 2573
2593#define BPF_MAIN_FUNC (-1)
2594static void init_func_state(struct bpf_verifier_env *env,2574static void init_func_state(struct bpf_verifier_env *env,
2595 struct bpf_func_state *state,2575 struct bpf_func_state *state,
2596 int callsite, int frameno, int subprogno)2576 int callsite, int frameno, int subprogno)
@@ -3517,16 +3497,6 @@ static int insn_stack_access_flags(int frameno, int spi) · 920 unmodified lines
3517 return INSN_F_STACK_ACCESS | (spi << INSN_F_SPI_SHIFT) | frameno;3497 return INSN_F_STACK_ACCESS | (spi << INSN_F_SPI_SHIFT) | frameno;
3518}3498}
3519 3499
3520static int insn_stack_access_spi(int insn_flags)
3521{
3522 return (insn_flags >> INSN_F_SPI_SHIFT) & INSN_F_SPI_MASK;
3523}
3524
3525static int insn_stack_access_frameno(int insn_flags)
3526{
3527 return insn_flags & INSN_F_FRAMENO_MASK;
3528}
3529
3530#define LR_FRAMENO_BITS 33500#define LR_FRAMENO_BITS 3
3531#define LR_SPI_BITS 63501#define LR_SPI_BITS 6
3532#define LR_ENTRY_BITS (LR_SPI_BITS + LR_FRAMENO_BITS + 1)3502#define LR_ENTRY_BITS (LR_SPI_BITS + LR_FRAMENO_BITS + 1)
@@ -3605,91 +3575,6 @@ static void linked_regs_unpack(u64 val, struct linked_regs *s) · 72 unmodified lines
3605 }3575 }
3606}3576}
3607 3577
3608/* for any branch, call, exit record the history of jmps in the given state */
3609int bpf_push_jmp_history(struct bpf_verifier_env *env, struct bpf_verifier_state *cur,
3610 int insn_flags, u64 linked_regs)
3611{
3612 u32 cnt = cur->jmp_history_cnt;
3613 struct bpf_jmp_history_entry *p;
3614 size_t alloc_size;
3615
3616 /* combine instruction flags if we already recorded this instruction */
3617 if (env->cur_hist_ent) {
3618 /* atomic instructions push insn_flags twice, for READ and
3619 * WRITE sides, but they should agree on stack slot
3620 */
3621 verifier_bug_if((env->cur_hist_ent->flags & insn_flags) &&
3622 (env->cur_hist_ent->flags & insn_flags) != insn_flags,
3623 env, "insn history: insn_idx %d cur flags %x new flags %x",
3624 env->insn_idx, env->cur_hist_ent->flags, insn_flags);
3625 env->cur_hist_ent->flags |= insn_flags;
3626 verifier_bug_if(env->cur_hist_ent->linked_regs != 0, env,
3627 "insn history: insn_idx %d linked_regs: %#llx",
3628 env->insn_idx, env->cur_hist_ent->linked_regs);
3629 env->cur_hist_ent->linked_regs = linked_regs;
3630 return 0;
3631 }
3632
3633 cnt++;
3634 alloc_size = kmalloc_size_roundup(size_mul(cnt, sizeof(*p)));
3635 p = krealloc(cur->jmp_history, alloc_size, GFP_KERNEL_ACCOUNT);
3636 if (!p)
3637 return -ENOMEM;
3638 cur->jmp_history = p;
3639
3640 p = &cur->jmp_history[cnt - 1];
3641 p->idx = env->insn_idx;
3642 p->prev_idx = env->prev_insn_idx;
3643 p->flags = insn_flags;
3644 p->linked_regs = linked_regs;
3645 cur->jmp_history_cnt = cnt;
3646 env->cur_hist_ent = p;
3647
3648 return 0;
3649}
3650
3651static struct bpf_jmp_history_entry *get_jmp_hist_entry(struct bpf_verifier_state *st,
3652 u32 hist_end, int insn_idx)
3653{
3654 if (hist_end > 0 && st->jmp_history[hist_end - 1].idx == insn_idx)
3655 return &st->jmp_history[hist_end - 1];
3656 return NULL;
3657}
3658
3659/* Backtrack one insn at a time. If idx is not at the top of recorded
3660 * history then previous instruction came from straight line execution.
3661 * Return -ENOENT if we exhausted all instructions within given state.
3662 *
3663 * It's legal to have a bit of a looping with the same starting and ending
3664 * insn index within the same state, e.g.: 3->4->5->3, so just because current
3665 * instruction index is the same as state's first_idx doesn't mean we are
3666 * done. If there is still some jump history left, we should keep going. We
3667 * need to take into account that we might have a jump history between given
3668 * state's parent and itself, due to checkpointing. In this case, we'll have
3669 * history entry recording a jump from last instruction of parent state and
3670 * first instruction of given state.
3671 */
3672static int get_prev_insn_idx(struct bpf_verifier_state *st, int i,
3673 u32 *history)
3674{
3675 u32 cnt = *history;
3676
3677 if (i == st->first_insn_idx) {
3678 if (cnt == 0)
3679 return -ENOENT;
3680 if (cnt == 1 && st->jmp_history[0].idx == i)
3681 return -ENOENT;
3682 }
3683
3684 if (cnt && st->jmp_history[cnt - 1].idx == i) {
3685 i = st->jmp_history[cnt - 1].prev_idx;
3686 (*history)--;
3687 } else {
3688 i--;
3689 }
3690 return i;
3691}
3692
3693static const char *disasm_kfunc_name(void *data, const struct bpf_insn *insn)3578static const char *disasm_kfunc_name(void *data, const struct bpf_insn *insn)
3694{3579{
3695 const struct btf_type *func;3580 const struct btf_type *func;
@@ -3717,148 +3602,10 @@ void bpf_verbose_insn(struct bpf_verifier_env *env, struct bpf_insn *insn) · 21 unmodified lines
3717 print_bpf_insn(&cbs, insn, env->allow_ptr_leaks);3602 print_bpf_insn(&cbs, insn, env->allow_ptr_leaks);
3718}3603}
3719 3604
3720static inline void bt_init(struct backtrack_state *bt, u32 frame)
3721{
3722 bt->frame = frame;
3723}
3724
3725static inline void bt_reset(struct backtrack_state *bt)
3726{
3727 struct bpf_verifier_env *env = bt->env;
3728
3729 memset(bt, 0, sizeof(*bt));
3730 bt->env = env;
3731}
3732
3733static inline u32 bt_empty(struct backtrack_state *bt)
3734{
3735 u64 mask = 0;
3736 int i;
3737
3738 for (i = 0; i <= bt->frame; i++)
3739 mask |= bt->reg_masks[i] | bt->stack_masks[i];
3740
3741 return mask == 0;
3742}
3743
3744static inline int bt_subprog_enter(struct backtrack_state *bt)
3745{
3746 if (bt->frame == MAX_CALL_FRAMES - 1) {
3747 verifier_bug(bt->env, "subprog enter from frame %d", bt->frame);
3748 return -EFAULT;
3749 }
3750 bt->frame++;
3751 return 0;
3752}
3753
3754static inline int bt_subprog_exit(struct backtrack_state *bt)
3755{
3756 if (bt->frame == 0) {
3757 verifier_bug(bt->env, "subprog exit from frame 0");
3758 return -EFAULT;
3759 }
3760 bt->frame--;
3761 return 0;
3762}
3763
3764static inline void bt_clear_frame_reg(struct backtrack_state *bt, u32 frame, u32 reg)
3765{
3766 bt->reg_masks[frame] &= ~(1 << reg);
3767}
3768
3769static inline void bt_set_reg(struct backtrack_state *bt, u32 reg)
3770{
3771 bpf_bt_set_frame_reg(bt, bt->frame, reg);
3772}
3773
3774static inline void bt_clear_reg(struct backtrack_state *bt, u32 reg)
3775{
3776 bt_clear_frame_reg(bt, bt->frame, reg);
3777}
3778
3779static inline void bt_clear_frame_slot(struct backtrack_state *bt, u32 frame, u32 slot)
3780{
3781 bt->stack_masks[frame] &= ~(1ull << slot);
3782}
3783
3784static inline u32 bt_frame_reg_mask(struct backtrack_state *bt, u32 frame)
3785{
3786 return bt->reg_masks[frame];
3787}
3788
3789static inline u32 bt_reg_mask(struct backtrack_state *bt)
3790{
3791 return bt->reg_masks[bt->frame];
3792}
3793
3794static inline u64 bt_frame_stack_mask(struct backtrack_state *bt, u32 frame)
3795{
3796 return bt->stack_masks[frame];
3797}
3798
3799static inline u64 bt_stack_mask(struct backtrack_state *bt)
3800{
3801 return bt->stack_masks[bt->frame];
3802}
3803
3804static inline bool bt_is_reg_set(struct backtrack_state *bt, u32 reg)
3805{
3806 return bt->reg_masks[bt->frame] & (1 << reg);
3807}
3808
3809static inline bool bt_is_frame_reg_set(struct backtrack_state *bt, u32 frame, u32 reg)
3810{
3811 return bt->reg_masks[frame] & (1 << reg);
3812}
3813
3814static inline bool bt_is_frame_slot_set(struct backtrack_state *bt, u32 frame, u32 slot)
3815{
3816 return bt->stack_masks[frame] & (1ull << slot);
3817}
3818
3819/* format registers bitmask, e.g., "r0,r2,r4" for 0x15 mask */
3820static void fmt_reg_mask(char *buf, ssize_t buf_sz, u32 reg_mask)
3821{
3822 DECLARE_BITMAP(mask, 64);
3823 bool first = true;
3824 int i, n;
3825
3826 buf[0] = '\0';
3827
3828 bitmap_from_u64(mask, reg_mask);
3829 for_each_set_bit(i, mask, 32) {
3830 n = snprintf(buf, buf_sz, "%sr%d", first ? "" : ",", i);
3831 first = false;
3832 buf += n;
3833 buf_sz -= n;
3834 if (buf_sz < 0)
3835 break;
3836 }
3837}
3838/* format stack slots bitmask, e.g., "-8,-24,-40" for 0x15 mask */
3839void bpf_fmt_stack_mask(char *buf, ssize_t buf_sz, u64 stack_mask)
3840{
3841 DECLARE_BITMAP(mask, 64);
3842 bool first = true;
3843 int i, n;
3844
3845 buf[0] = '\0';
3846
3847 bitmap_from_u64(mask, stack_mask);
3848 for_each_set_bit(i, mask, 64) {
3849 n = snprintf(buf, buf_sz, "%s%d", first ? "" : ",", -(i + 1) * 8);
3850 first = false;
3851 buf += n;
3852 buf_sz -= n;
3853 if (buf_sz < 0)
3854 break;
3855 }
3856}
3857
3858/* If any register R in hist->linked_regs is marked as precise in bt,3605/* If any register R in hist->linked_regs is marked as precise in bt,
3859 * do bt_set_frame_{reg,slot}(bt, R) for all registers in hist->linked_regs.3606 * do bt_set_frame_{reg,slot}(bt, R) for all registers in hist->linked_regs.
3860 */3607 */
3861static void bt_sync_linked_regs(struct backtrack_state *bt, struct bpf_jmp_history_entry *hist)3608void bpf_bt_sync_linked_regs(struct backtrack_state *bt, struct bpf_jmp_history_entry *hist)
3862{3609{
3863 struct linked_regs linked_regs;3610 struct linked_regs linked_regs;
3864 bool some_precise = false;3611 bool some_precise = false;
@@ -3891,693 +3638,6 @@ static void bt_sync_linked_regs(struct backtrack_state *bt, struct bpf_jmp_histo · 26 unmodified lines
3891 }3638 }
3892}3639}
3893 3640
3894/* For given verifier state backtrack_insn() is called from the last insn to
3895 * the first insn. Its purpose is to compute a bitmask of registers and
3896 * stack slots that needs precision in the parent verifier state.
3897 *
3898 * @idx is an index of the instruction we are currently processing;
3899 * @subseq_idx is an index of the subsequent instruction that:
3900 * - *would be* executed next, if jump history is viewed in forward order;
3901 * - *was* processed previously during backtracking.
3902 */
3903static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx,
3904 struct bpf_jmp_history_entry *hist, struct backtrack_state *bt)
3905{
3906 struct bpf_insn *insn = env->prog->insnsi + idx;
3907 u8 class = BPF_CLASS(insn->code);
3908 u8 opcode = BPF_OP(insn->code);
3909 u8 mode = BPF_MODE(insn->code);
3910 u32 dreg = insn->dst_reg;
3911 u32 sreg = insn->src_reg;
3912 u32 spi, i, fr;
3913
3914 if (insn->code == 0)
3915 return 0;
3916 if (env->log.level & BPF_LOG_LEVEL2) {
3917 fmt_reg_mask(env->tmp_str_buf, TMP_STR_BUF_LEN, bt_reg_mask(bt));
3918 verbose(env, "mark_precise: frame%d: regs=%s ",
3919 bt->frame, env->tmp_str_buf);
3920 bpf_fmt_stack_mask(env->tmp_str_buf, TMP_STR_BUF_LEN, bt_stack_mask(bt));
3921 verbose(env, "stack=%s before ", env->tmp_str_buf);
3922 verbose(env, "%d: ", idx);
3923 bpf_verbose_insn(env, insn);
3924 }
3925
3926 /* If there is a history record that some registers gained range at this insn,
3927 * propagate precision marks to those registers, so that bt_is_reg_set()
3928 * accounts for these registers.
3929 */
3930 bt_sync_linked_regs(bt, hist);
3931
3932 if (class == BPF_ALU || class == BPF_ALU64) {
3933 if (!bt_is_reg_set(bt, dreg))
3934 return 0;
3935 if (opcode == BPF_END || opcode == BPF_NEG) {
3936 /* sreg is reserved and unused
3937 * dreg still need precision before this insn
3938 */
3939 return 0;
3940 } else if (opcode == BPF_MOV) {
3941 if (BPF_SRC(insn->code) == BPF_X) {
3942 /* dreg = sreg or dreg = (s8, s16, s32)sreg
3943 * dreg needs precision after this insn
3944 * sreg needs precision before this insn
3945 */
3946 bt_clear_reg(bt, dreg);
3947 if (sreg != BPF_REG_FP)
3948 bt_set_reg(bt, sreg);
3949 } else {
3950 /* dreg = K
3951 * dreg needs precision after this insn.
3952 * Corresponding register is already marked
3953 * as precise=true in this verifier state.
3954 * No further markings in parent are necessary
3955 */
3956 bt_clear_reg(bt, dreg);
3957 }
3958 } else {
3959 if (BPF_SRC(insn->code) == BPF_X) {
3960 /* dreg += sreg
3961 * both dreg and sreg need precision
3962 * before this insn
3963 */
3964 if (sreg != BPF_REG_FP)
3965 bt_set_reg(bt, sreg);
3966 } /* else dreg += K
3967 * dreg still needs precision before this insn
3968 */
3969 }
3970 } else if (class == BPF_LDX ||
3971 is_atomic_load_insn(insn) ||
3972 is_atomic_fetch_insn(insn)) {
3973 u32 load_reg = dreg;
3974
3975 /*
3976 * Atomic fetch operation writes the old value into
3977 * a register (sreg or r0) and if it was tracked for
3978 * precision, propagate to the stack slot like we do
3979 * in regular ldx.
3980 */
3981 if (is_atomic_fetch_insn(insn))
3982 load_reg = insn->imm == BPF_CMPXCHG ?
3983 BPF_REG_0 : sreg;
3984
3985 if (!bt_is_reg_set(bt, load_reg))
3986 return 0;
3987 bt_clear_reg(bt, load_reg);
3988
3989 /* scalars can only be spilled into stack w/o losing precision.
3990 * Load from any other memory can be zero extended.
3991 * The desire to keep that precision is already indicated
3992 * by 'precise' mark in corresponding register of this state.
3993 * No further tracking necessary.
3994 */
3995 if (!hist || !(hist->flags & INSN_F_STACK_ACCESS))
3996 return 0;
3997 /* dreg = *(u64 *)[fp - off] was a fill from the stack.
3998 * that [fp - off] slot contains scalar that needs to be
3999 * tracked with precision
4000 */
4001 spi = insn_stack_access_spi(hist->flags);
4002 fr = insn_stack_access_frameno(hist->flags);
4003 bpf_bt_set_frame_slot(bt, fr, spi);
4004 } else if (class == BPF_STX || class == BPF_ST) {
4005 if (bt_is_reg_set(bt, dreg))
4006 /* stx & st shouldn't be using _scalar_ dst_reg
4007 * to access memory. It means backtracking
4008 * encountered a case of pointer subtraction.
4009 */
4010 return -ENOTSUPP;
4011 /* scalars can only be spilled into stack */
4012 if (!hist || !(hist->flags & INSN_F_STACK_ACCESS))
4013 return 0;
4014 spi = insn_stack_access_spi(hist->flags);
4015 fr = insn_stack_access_frameno(hist->flags);
4016 if (!bt_is_frame_slot_set(bt, fr, spi))
4017 return 0;
4018 bt_clear_frame_slot(bt, fr, spi);
4019 if (class == BPF_STX)
4020 bt_set_reg(bt, sreg);
4021 } else if (class == BPF_JMP || class == BPF_JMP32) {
4022 if (bpf_pseudo_call(insn)) {
4023 int subprog_insn_idx, subprog;
4024
4025 subprog_insn_idx = idx + insn->imm + 1;
4026 subprog = bpf_find_subprog(env, subprog_insn_idx);
4027 if (subprog < 0)
4028 return -EFAULT;
4029
4030 if (bpf_subprog_is_global(env, subprog)) {
4031 /* check that jump history doesn't have any
4032 * extra instructions from subprog; the next
4033 * instruction after call to global subprog
4034 * should be literally next instruction in
4035 * caller program
4036 */
4037 verifier_bug_if(idx + 1 != subseq_idx, env,
4038 "extra insn from subprog");
4039 /* r1-r5 are invalidated after subprog call,
4040 * so for global func call it shouldn't be set
4041 * anymore
4042 */
4043 if (bt_reg_mask(bt) & BPF_REGMASK_ARGS) {
4044 verifier_bug(env, "global subprog unexpected regs %x",
4045 bt_reg_mask(bt));
4046 return -EFAULT;
4047 }
4048 /* global subprog always sets R0 */
4049 bt_clear_reg(bt, BPF_REG_0);
4050 return 0;
4051 } else {
4052 /* static subprog call instruction, which
4053 * means that we are exiting current subprog,
4054 * so only r1-r5 could be still requested as
4055 * precise, r0 and r6-r10 or any stack slot in
4056 * the current frame should be zero by now
4057 */
4058 if (bt_reg_mask(bt) & ~BPF_REGMASK_ARGS) {
4059 verifier_bug(env, "static subprog unexpected regs %x",
4060 bt_reg_mask(bt));
4061 return -EFAULT;
4062 }
4063 /* we are now tracking register spills correctly,
4064 * so any instance of leftover slots is a bug
4065 */
4066 if (bt_stack_mask(bt) != 0) {
4067 verifier_bug(env,
4068 "static subprog leftover stack slots %llx",
4069 bt_stack_mask(bt));
4070 return -EFAULT;
4071 }
4072 /* propagate r1-r5 to the caller */
4073 for (i = BPF_REG_1; i <= BPF_REG_5; i++) {
4074 if (bt_is_reg_set(bt, i)) {
4075 bt_clear_reg(bt, i);
4076 bpf_bt_set_frame_reg(bt, bt->frame - 1, i);
4077 }
4078 }
4079 if (bt_subprog_exit(bt))
4080 return -EFAULT;
4081 return 0;
4082 }
4083 } else if (bpf_is_sync_callback_calling_insn(insn) && idx != subseq_idx - 1) {
4084 /* exit from callback subprog to callback-calling helper or
4085 * kfunc call. Use idx/subseq_idx check to discern it from
4086 * straight line code backtracking.
4087 * Unlike the subprog call handling above, we shouldn't
4088 * propagate precision of r1-r5 (if any requested), as they are
4089 * not actually arguments passed directly to callback subprogs
4090 */
4091 if (bt_reg_mask(bt) & ~BPF_REGMASK_ARGS) {
4092 verifier_bug(env, "callback unexpected regs %x",
4093 bt_reg_mask(bt));
4094 return -EFAULT;
4095 }
4096 if (bt_stack_mask(bt) != 0) {
4097 verifier_bug(env, "callback leftover stack slots %llx",
4098 bt_stack_mask(bt));
4099 return -EFAULT;
4100 }
4101 /* clear r1-r5 in callback subprog's mask */
4102 for (i = BPF_REG_1; i <= BPF_REG_5; i++)
4103 bt_clear_reg(bt, i);
4104 if (bt_subprog_exit(bt))
4105 return -EFAULT;
4106 return 0;
4107 } else if (opcode == BPF_CALL) {
4108 /* kfunc with imm==0 is invalid and fixup_kfunc_call will
4109 * catch this error later. Make backtracking conservative
4110 * with ENOTSUPP.
4111 */
4112 if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL && insn->imm == 0)
4113 return -ENOTSUPP;
4114 /* regular helper call sets R0 */
4115 bt_clear_reg(bt, BPF_REG_0);
4116 if (bt_reg_mask(bt) & BPF_REGMASK_ARGS) {
4117 /* if backtracking was looking for registers R1-R5
4118 * they should have been found already.
4119 */
4120 verifier_bug(env, "backtracking call unexpected regs %x",
4121 bt_reg_mask(bt));
4122 return -EFAULT;
4123 }
4124 if (insn->src_reg == BPF_REG_0 && insn->imm == BPF_FUNC_tail_call
4125 && subseq_idx - idx != 1) {
4126 if (bt_subprog_enter(bt))
4127 return -EFAULT;
4128 }
4129 } else if (opcode == BPF_EXIT) {
4130 bool r0_precise;
4131
4132 /* Backtracking to a nested function call, 'idx' is a part of
4133 * the inner frame 'subseq_idx' is a part of the outer frame.
4134 * In case of a regular function call, instructions giving
4135 * precision to registers R1-R5 should have been found already.
4136 * In case of a callback, it is ok to have R1-R5 marked for
4137 * backtracking, as these registers are set by the function
4138 * invoking callback.
4139 */
4140 if (subseq_idx >= 0 && bpf_calls_callback(env, subseq_idx))
4141 for (i = BPF_REG_1; i <= BPF_REG_5; i++)
4142 bt_clear_reg(bt, i);
4143 if (bt_reg_mask(bt) & BPF_REGMASK_ARGS) {
4144 verifier_bug(env, "backtracking exit unexpected regs %x",
4145 bt_reg_mask(bt));
4146 return -EFAULT;
4147 }
4148
4149 /* BPF_EXIT in subprog or callback always returns
4150 * right after the call instruction, so by checking
4151 * whether the instruction at subseq_idx-1 is subprog
4152 * call or not we can distinguish actual exit from
4153 * *subprog* from exit from *callback*. In the former
4154 * case, we need to propagate r0 precision, if
4155 * necessary. In the former we never do that.
4156 */
4157 r0_precise = subseq_idx - 1 >= 0 &&
4158 bpf_pseudo_call(&env->prog->insnsi[subseq_idx - 1]) &&
4159 bt_is_reg_set(bt, BPF_REG_0);
4160
4161 bt_clear_reg(bt, BPF_REG_0);
4162 if (bt_subprog_enter(bt))
4163 return -EFAULT;
4164
4165 if (r0_precise)
4166 bt_set_reg(bt, BPF_REG_0);
4167 /* r6-r9 and stack slots will stay set in caller frame
4168 * bitmasks until we return back from callee(s)
4169 */
4170 return 0;
4171 } else if (BPF_SRC(insn->code) == BPF_X) {
4172 if (!bt_is_reg_set(bt, dreg) && !bt_is_reg_set(bt, sreg))
4173 return 0;
4174 /* dreg <cond> sreg
4175 * Both dreg and sreg need precision before
4176 * this insn. If only sreg was marked precise
4177 * before it would be equally necessary to
4178 * propagate it to dreg.
4179 */
4180 if (!hist || !(hist->flags & INSN_F_SRC_REG_STACK))
4181 bt_set_reg(bt, sreg);
4182 if (!hist || !(hist->flags & INSN_F_DST_REG_STACK))
4183 bt_set_reg(bt, dreg);
4184 } else if (BPF_SRC(insn->code) == BPF_K) {
4185 /* dreg <cond> K
4186 * Only dreg still needs precision before
4187 * this insn, so for the K-based conditional
4188 * there is nothing new to be marked.
4189 */
4190 }
4191 } else if (class == BPF_LD) {
4192 if (!bt_is_reg_set(bt, dreg))
4193 return 0;
4194 bt_clear_reg(bt, dreg);
4195 /* It's ld_imm64 or ld_abs or ld_ind.
4196 * For ld_imm64 no further tracking of precision
4197 * into parent is necessary
4198 */
4199 if (mode == BPF_IND || mode == BPF_ABS)
4200 /* to be analyzed */
4201 return -ENOTSUPP;
4202 }
4203 /* Propagate precision marks to linked registers, to account for
4204 * registers marked as precise in this function.
4205 */
4206 bt_sync_linked_regs(bt, hist);
4207 return 0;
4208}
4209
4210/* the scalar precision tracking algorithm:
4211 * . at the start all registers have precise=false.
4212 * . scalar ranges are tracked as normal through alu and jmp insns.
4213 * . once precise value of the scalar register is used in:
4214 * . ptr + scalar alu
4215 * . if (scalar cond K|scalar)
4216 * . helper_call(.., scalar, ...) where ARG_CONST is expected
4217 * backtrack through the verifier states and mark all registers and
4218 * stack slots with spilled constants that these scalar registers
4219 * should be precise.
4220 * . during state pruning two registers (or spilled stack slots)
4221 * are equivalent if both are not precise.
4222 *
4223 * Note the verifier cannot simply walk register parentage chain,
4224 * since many different registers and stack slots could have been
4225 * used to compute single precise scalar.
4226 *
4227 * The approach of starting with precise=true for all registers and then
4228 * backtrack to mark a register as not precise when the verifier detects
4229 * that program doesn't care about specific value (e.g., when helper
4230 * takes register as ARG_ANYTHING parameter) is not safe.
4231 *
4232 * It's ok to walk single parentage chain of the verifier states.
4233 * It's possible that this backtracking will go all the way till 1st insn.
4234 * All other branches will be explored for needing precision later.
4235 *
4236 * The backtracking needs to deal with cases like:
4237 * R8=map_value(id=0,off=0,ks=4,vs=1952,imm=0) R9_w=map_value(id=0,off=40,ks=4,vs=1952,imm=0)
4238 * r9 -= r8
4239 * r5 = r9
4240 * if r5 > 0x79f goto pc+7
4241 * R5_w=inv(id=0,umax_value=1951,var_off=(0x0; 0x7ff))
4242 * r5 += 1
4243 * ...
4244 * call bpf_perf_event_output#25
4245 * where .arg5_type = ARG_CONST_SIZE_OR_ZERO
4246 *
4247 * and this case:
4248 * r6 = 1
4249 * call foo // uses callee's r6 inside to compute r0
4250 * r0 += r6
4251 * if r0 == 0 goto
4252 *
4253 * to track above reg_mask/stack_mask needs to be independent for each frame.
4254 *
4255 * Also if parent's curframe > frame where backtracking started,
4256 * the verifier need to mark registers in both frames, otherwise callees
4257 * may incorrectly prune callers. This is similar to
4258 * commit 7640ead93924 ("bpf: verifier: make sure callees don't prune with caller differences")
4259 *
4260 * For now backtracking falls back into conservative marking.
4261 */
4262void bpf_mark_all_scalars_precise(struct bpf_verifier_env *env,
4263 struct bpf_verifier_state *st)
4264{
4265 struct bpf_func_state *func;
4266 struct bpf_reg_state *reg;
4267 int i, j;
4268
4269 if (env->log.level & BPF_LOG_LEVEL2) {
4270 verbose(env, "mark_precise: frame%d: falling back to forcing all scalars precise\n",
4271 st->curframe);
4272 }
4273
4274 /* big hammer: mark all scalars precise in this path.
4275 * pop_stack may still get !precise scalars.
4276 * We also skip current state and go straight to first parent state,
4277 * because precision markings in current non-checkpointed state are
4278 * not needed. See why in the comment in __mark_chain_precision below.
4279 */
4280 for (st = st->parent; st; st = st->parent) {
4281 for (i = 0; i <= st->curframe; i++) {
4282 func = st->frame[i];
4283 for (j = 0; j < BPF_REG_FP; j++) {
4284 reg = &func->regs[j];
4285 if (reg->type != SCALAR_VALUE || reg->precise)
4286 continue;
4287 reg->precise = true;
4288 if (env->log.level & BPF_LOG_LEVEL2) {
4289 verbose(env, "force_precise: frame%d: forcing r%d to be precise\n",
4290 i, j);
4291 }
4292 }
4293 for (j = 0; j < func->allocated_stack / BPF_REG_SIZE; j++) {
4294 if (!bpf_is_spilled_reg(&func->stack[j]))
4295 continue;
4296 reg = &func->stack[j].spilled_ptr;
4297 if (reg->type != SCALAR_VALUE || reg->precise)
4298 continue;
4299 reg->precise = true;
4300 if (env->log.level & BPF_LOG_LEVEL2) {
4301 verbose(env, "force_precise: frame%d: forcing fp%d to be precise\n",
4302 i, -(j + 1) * 8);
4303 }
4304 }
4305 }
4306 }
4307}
4308
4309/*
4310 * bpf_mark_chain_precision() backtracks BPF program instruction sequence and
4311 * chain of verifier states making sure that register *regno* (if regno >= 0)
4312 * and/or stack slot *spi* (if spi >= 0) are marked as precisely tracked
4313 * SCALARS, as well as any other registers and slots that contribute to
4314 * a tracked state of given registers/stack slots, depending on specific BPF
4315 * assembly instructions (see backtrack_insns() for exact instruction handling
4316 * logic). This backtracking relies on recorded jmp_history and is able to
4317 * traverse entire chain of parent states. This process ends only when all the
4318 * necessary registers/slots and their transitive dependencies are marked as
4319 * precise.
4320 *
4321 * One important and subtle aspect is that precise marks *do not matter* in
4322 * the currently verified state (current state). It is important to understand
4323 * why this is the case.
4324 *
4325 * First, note that current state is the state that is not yet "checkpointed",
4326 * i.e., it is not yet put into env->explored_states, and it has no children
4327 * states as well. It's ephemeral, and can end up either a) being discarded if
4328 * compatible explored state is found at some point or BPF_EXIT instruction is
4329 * reached or b) checkpointed and put into env->explored_states, branching out
4330 * into one or more children states.
4331 *
4332 * In the former case, precise markings in current state are completely
4333 * ignored by state comparison code (see regsafe() for details). Only
4334 * checkpointed ("old") state precise markings are important, and if old
4335 * state's register/slot is precise, regsafe() assumes current state's
4336 * register/slot as precise and checks value ranges exactly and precisely. If
4337 * states turn out to be compatible, current state's necessary precise
4338 * markings and any required parent states' precise markings are enforced
4339 * after the fact with propagate_precision() logic, after the fact. But it's
4340 * important to realize that in this case, even after marking current state
4341 * registers/slots as precise, we immediately discard current state. So what
4342 * actually matters is any of the precise markings propagated into current
4343 * state's parent states, which are always checkpointed (due to b) case above).
4344 * As such, for scenario a) it doesn't matter if current state has precise
4345 * markings set or not.
4346 *
4347 * Now, for the scenario b), checkpointing and forking into child(ren)
4348 * state(s). Note that before current state gets to checkpointing step, any
4349 * processed instruction always assumes precise SCALAR register/slot
4350 * knowledge: if precise value or range is useful to prune jump branch, BPF
4351 * verifier takes this opportunity enthusiastically. Similarly, when
4352 * register's value is used to calculate offset or memory address, exact
4353 * knowledge of SCALAR range is assumed, checked, and enforced. So, similar to
4354 * what we mentioned above about state comparison ignoring precise markings
4355 * during state comparison, BPF verifier ignores and also assumes precise
4356 * markings *at will* during instruction verification process. But as verifier
4357 * assumes precision, it also propagates any precision dependencies across
4358 * parent states, which are not yet finalized, so can be further restricted
4359 * based on new knowledge gained from restrictions enforced by their children
4360 * states. This is so that once those parent states are finalized, i.e., when
4361 * they have no more active children state, state comparison logic in
4362 * is_state_visited() would enforce strict and precise SCALAR ranges, if
4363 * required for correctness.
4364 *
4365 * To build a bit more intuition, note also that once a state is checkpointed,
4366 * the path we took to get to that state is not important. This is crucial
4367 * property for state pruning. When state is checkpointed and finalized at
4368 * some instruction index, it can be correctly and safely used to "short
4369 * circuit" any *compatible* state that reaches exactly the same instruction
4370 * index. I.e., if we jumped to that instruction from a completely different
4371 * code path than original finalized state was derived from, it doesn't
4372 * matter, current state can be discarded because from that instruction
4373 * forward having a compatible state will ensure we will safely reach the
4374 * exit. States describe preconditions for further exploration, but completely
4375 * forget the history of how we got here.
4376 *
4377 * This also means that even if we needed precise SCALAR range to get to
4378 * finalized state, but from that point forward *that same* SCALAR register is
4379 * never used in a precise context (i.e., it's precise value is not needed for
4380 * correctness), it's correct and safe to mark such register as "imprecise"
4381 * (i.e., precise marking set to false). This is what we rely on when we do
4382 * not set precise marking in current state. If no child state requires
4383 * precision for any given SCALAR register, it's safe to dictate that it can
4384 * be imprecise. If any child state does require this register to be precise,
4385 * we'll mark it precise later retroactively during precise markings
4386 * propagation from child state to parent states.
4387 *
4388 * Skipping precise marking setting in current state is a mild version of
4389 * relying on the above observation. But we can utilize this property even
4390 * more aggressively by proactively forgetting any precise marking in the
4391 * current state (which we inherited from the parent state), right before we
4392 * checkpoint it and branch off into new child state. This is done by
4393 * mark_all_scalars_imprecise() to hopefully get more permissive and generic
4394 * finalized states which help in short circuiting more future states.
4395 */
4396int bpf_mark_chain_precision(struct bpf_verifier_env *env,
4397 struct bpf_verifier_state *starting_state,
4398 int regno,
4399 bool *changed)
4400{
4401 struct bpf_verifier_state *st = starting_state;
4402 struct backtrack_state *bt = &env->bt;
4403 int first_idx = st->first_insn_idx;
4404 int last_idx = starting_state->insn_idx;
4405 int subseq_idx = -1;
4406 struct bpf_func_state *func;
4407 bool tmp, skip_first = true;
4408 struct bpf_reg_state *reg;
4409 int i, fr, err;
4410
4411 if (!env->bpf_capable)
4412 return 0;
4413
4414 changed = changed ?: &tmp;
4415 /* set frame number from which we are starting to backtrack */
4416 bt_init(bt, starting_state->curframe);
4417
4418 /* Do sanity checks against current state of register and/or stack
4419 * slot, but don't set precise flag in current state, as precision
4420 * tracking in the current state is unnecessary.
4421 */
4422 func = st->frame[bt->frame];
4423 if (regno >= 0) {
4424 reg = &func->regs[regno];
4425 if (reg->type != SCALAR_VALUE) {
4426 verifier_bug(env, "backtracking misuse");
4427 return -EFAULT;
4428 }
4429 bt_set_reg(bt, regno);
4430 }
4431
4432 if (bt_empty(bt))
4433 return 0;
4434
4435 for (;;) {
4436 DECLARE_BITMAP(mask, 64);
4437 u32 history = st->jmp_history_cnt;
4438 struct bpf_jmp_history_entry *hist;
4439
4440 if (env->log.level & BPF_LOG_LEVEL2) {
4441 verbose(env, "mark_precise: frame%d: last_idx %d first_idx %d subseq_idx %d \n",
4442 bt->frame, last_idx, first_idx, subseq_idx);
4443 }
4444
4445 if (last_idx < 0) {
4446 /* we are at the entry into subprog, which
4447 * is expected for global funcs, but only if
4448 * requested precise registers are R1-R5
4449 * (which are global func's input arguments)
4450 */
4451 if (st->curframe == 0 &&
4452 st->frame[0]->subprogno > 0 &&
4453 st->frame[0]->callsite == BPF_MAIN_FUNC &&
4454 bt_stack_mask(bt) == 0 &&
4455 (bt_reg_mask(bt) & ~BPF_REGMASK_ARGS) == 0) {
4456 bitmap_from_u64(mask, bt_reg_mask(bt));
4457 for_each_set_bit(i, mask, 32) {
4458 reg = &st->frame[0]->regs[i];
4459 bt_clear_reg(bt, i);
4460 if (reg->type == SCALAR_VALUE) {
4461 reg->precise = true;
4462 *changed = true;
4463 }
4464 }
4465 return 0;
4466 }
4467
4468 verifier_bug(env, "backtracking func entry subprog %d reg_mask %x stack_mask %llx",
4469 st->frame[0]->subprogno, bt_reg_mask(bt), bt_stack_mask(bt));
4470 return -EFAULT;
4471 }
4472
4473 for (i = last_idx;;) {
4474 if (skip_first) {
4475 err = 0;
4476 skip_first = false;
4477 } else {
4478 hist = get_jmp_hist_entry(st, history, i);
4479 err = backtrack_insn(env, i, subseq_idx, hist, bt);
4480 }
4481 if (err == -ENOTSUPP) {
4482 bpf_mark_all_scalars_precise(env, starting_state);
4483 bt_reset(bt);
4484 return 0;
4485 } else if (err) {
4486 return err;
4487 }
4488 if (bt_empty(bt))
4489 /* Found assignment(s) into tracked register in this state.
4490 * Since this state is already marked, just return.
4491 * Nothing to be tracked further in the parent state.
4492 */
4493 return 0;
4494 subseq_idx = i;
4495 i = get_prev_insn_idx(st, i, &history);
4496 if (i == -ENOENT)
4497 break;
4498 if (i >= env->prog->len) {
4499 /* This can happen if backtracking reached insn 0
4500 * and there are still reg_mask or stack_mask
4501 * to backtrack.
4502 * It means the backtracking missed the spot where
4503 * particular register was initialized with a constant.
4504 */
4505 verifier_bug(env, "backtracking idx %d", i);
4506 return -EFAULT;
4507 }
4508 }
4509 st = st->parent;
4510 if (!st)
4511 break;
4512
4513 for (fr = bt->frame; fr >= 0; fr--) {
4514 func = st->frame[fr];
4515 bitmap_from_u64(mask, bt_frame_reg_mask(bt, fr));
4516 for_each_set_bit(i, mask, 32) {
4517 reg = &func->regs[i];
4518 if (reg->type != SCALAR_VALUE) {
4519 bt_clear_frame_reg(bt, fr, i);
4520 continue;
4521 }
4522 if (reg->precise) {
4523 bt_clear_frame_reg(bt, fr, i);
4524 } else {
4525 reg->precise = true;
4526 *changed = true;
4527 }
4528 }
4529
4530 bitmap_from_u64(mask, bt_frame_stack_mask(bt, fr));
4531 for_each_set_bit(i, mask, 64) {
4532 if (verifier_bug_if(i >= func->allocated_stack / BPF_REG_SIZE,
4533 env, "stack slot %d, total slots %d",
4534 i, func->allocated_stack / BPF_REG_SIZE))
4535 return -EFAULT;
4536
4537 if (!is_spilled_scalar_reg(&func->stack[i])) {
4538 bt_clear_frame_slot(bt, fr, i);
4539 continue;
4540 }
4541 reg = &func->stack[i].spilled_ptr;
4542 if (reg->precise) {
4543 bt_clear_frame_slot(bt, fr, i);
4544 } else {
4545 reg->precise = true;
4546 *changed = true;
4547 }
4548 }
4549 if (env->log.level & BPF_LOG_LEVEL2) {
4550 fmt_reg_mask(env->tmp_str_buf, TMP_STR_BUF_LEN,
4551 bt_frame_reg_mask(bt, fr));
4552 verbose(env, "mark_precise: frame%d: parent state regs=%s ",
4553 fr, env->tmp_str_buf);
4554 bpf_fmt_stack_mask(env->tmp_str_buf, TMP_STR_BUF_LEN,
4555 bt_frame_stack_mask(bt, fr));
4556 verbose(env, "stack=%s: ", env->tmp_str_buf);
4557 print_verifier_state(env, st, fr, true);
4558 }
4559 }
4560
4561 if (bt_empty(bt))
4562 return 0;
4563
4564 subseq_idx = first_idx;
4565 last_idx = st->last_insn_idx;
4566 first_idx = st->first_insn_idx;
4567 }
4568
4569 /* if we still have requested precise regs or slots, we missed
4570 * something (e.g., stack access through non-r10 register), so
4571 * fallback to marking all precise
4572 */
4573 if (!bt_empty(bt)) {
4574 bpf_mark_all_scalars_precise(env, starting_state);
4575 bt_reset(bt);
4576 }
4577
4578 return 0;
4579}
4580
4581int mark_chain_precision(struct bpf_verifier_env *env, int regno)3641int mark_chain_precision(struct bpf_verifier_env *env, int regno)
4582{3642{
4583 return bpf_mark_chain_precision(env, env->cur_state, regno, NULL);3643 return bpf_mark_chain_precision(env, env->cur_state, regno, NULL);
@@ -4759,7 +3819,7 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env, · 175 unmodified lines
4759 */3819 */
4760 if (!env->allow_ptr_leaks &&3820 if (!env->allow_ptr_leaks &&
4761 bpf_is_spilled_reg(&state->stack[spi]) &&3821 bpf_is_spilled_reg(&state->stack[spi]) &&
4762 !is_spilled_scalar_reg(&state->stack[spi]) &&3822 !bpf_is_spilled_scalar_reg(&state->stack[spi]) &&
4763 size != BPF_REG_SIZE) {3823 size != BPF_REG_SIZE) {
4764 verbose(env, "attempt to corrupt spilled pointer on stack\n");3824 verbose(env, "attempt to corrupt spilled pointer on stack\n");
4765 return -EACCES;3825 return -EACCES;
@@ -4938,7 +3998,7 @@ static int check_stack_write_var_off(struct bpf_verifier_env *env, · 172 unmodified lines
4938 * maintain the spill type.3998 * maintain the spill type.
4939 */3999 */
4940 if (writing_zero && *stype == STACK_SPILL &&4000 if (writing_zero && *stype == STACK_SPILL &&
4941 is_spilled_scalar_reg(&state->stack[spi])) {4001 bpf_is_spilled_scalar_reg(&state->stack[spi])) {
4942 struct bpf_reg_state *spill_reg = &state->stack[spi].spilled_ptr;4002 struct bpf_reg_state *spill_reg = &state->stack[spi].spilled_ptr;
4943 4003
4944 if (tnum_is_const(spill_reg->var_off) && spill_reg->var_off.value == 0) {4004 if (tnum_is_const(spill_reg->var_off) && spill_reg->var_off.value == 0) {
@@ -9380,7 +8440,7 @@ static int get_constant_map_key(struct bpf_verifier_env *env, · 4435 unmodified lines
9380 }8440 }
9381 8441
9382 /* Check that stack contains a scalar spill of expected size */8442 /* Check that stack contains a scalar spill of expected size */
9383 if (!is_spilled_scalar_reg(&state->stack[spi]))8443 if (!bpf_is_spilled_scalar_reg(&state->stack[spi]))
9384 return -EOPNOTSUPP;8444 return -EOPNOTSUPP;
9385 for (i = off; i >= 0 && stype[i] == STACK_SPILL; i--)8445 for (i = off; i >= 0 && stype[i] == STACK_SPILL; i--)
9386 spill_size++;8446 spill_size++;

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t