Move precision propagation and backtracking logic to backtrack.c to reduce verifier.c size. No functional changes.
Showing the changes to kernel (3 of 4 files). Show all
Unified · Split
| @@ -11,7 +11,7 @@ obj-$(CONFIG_BPF_SYSCALL) += bpf_iter.o map_iter.o task_iter.o prog_iter.o link_ · 10 unmodified lines | ||
| 11 | 11 | obj-$(CONFIG_BPF_SYSCALL) += hashtab.o arraymap.o percpu_freelist.o bpf_lru_list.o lpm_trie.o map_in_map.o bloom_filter.o |
| 12 | 12 | obj-$(CONFIG_BPF_SYSCALL) += local_storage.o queue_stack_maps.o ringbuf.o bpf_insn_array.o |
| 13 | 13 | obj-$(CONFIG_BPF_SYSCALL) += bpf_local_storage.o bpf_task_storage.o |
| 14 | −obj-$(CONFIG_BPF_SYSCALL) += fixups.o cfg.o states.o | |
| 14 | +obj-$(CONFIG_BPF_SYSCALL) += fixups.o cfg.o states.o backtrack.o | |
| 15 | 15 | obj-${CONFIG_BPF_LSM} += bpf_inode_storage.o |
| 16 | 16 | obj-$(CONFIG_BPF_SYSCALL) += disasm.o mprog.o |
| 17 | 17 | obj-$(CONFIG_BPF_JIT) += trampoline.o |
| @@ -0,0 +1,934 @@ | ||
| 1 | +// SPDX-License-Identifier: GPL-2.0-only | |
| 2 | +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ | |
| 3 | +#include <linux/bpf.h> | |
| 4 | +#include <linux/bpf_verifier.h> | |
| 5 | +#include <linux/filter.h> | |
| 6 | +#include <linux/bitmap.h> | |
| 7 | + | |
| 8 | +#define verbose(env, fmt, args...) bpf_verifier_log_write(env, fmt, ##args) | |
| 9 | + | |
| 10 | +/* for any branch, call, exit record the history of jmps in the given state */ | |
| 11 | +int bpf_push_jmp_history(struct bpf_verifier_env *env, struct bpf_verifier_state *cur, | |
| 12 | + int insn_flags, u64 linked_regs) | |
| 13 | +{ | |
| 14 | + u32 cnt = cur->jmp_history_cnt; | |
| 15 | + struct bpf_jmp_history_entry *p; | |
| 16 | + size_t alloc_size; | |
| 17 | + | |
| 18 | + /* combine instruction flags if we already recorded this instruction */ | |
| 19 | + if (env->cur_hist_ent) { | |
| 20 | + /* atomic instructions push insn_flags twice, for READ and | |
| 21 | + * WRITE sides, but they should agree on stack slot | |
| 22 | + */ | |
| 23 | + verifier_bug_if((env->cur_hist_ent->flags & insn_flags) && | |
| 24 | + (env->cur_hist_ent->flags & insn_flags) != insn_flags, | |
| 25 | + env, "insn history: insn_idx %d cur flags %x new flags %x", | |
| 26 | + env->insn_idx, env->cur_hist_ent->flags, insn_flags); | |
| 27 | + env->cur_hist_ent->flags |= insn_flags; | |
| 28 | + verifier_bug_if(env->cur_hist_ent->linked_regs != 0, env, | |
| 29 | + "insn history: insn_idx %d linked_regs: %#llx", | |
| 30 | + env->insn_idx, env->cur_hist_ent->linked_regs); | |
| 31 | + env->cur_hist_ent->linked_regs = linked_regs; | |
| 32 | + return 0; | |
| 33 | + } | |
| 34 | + | |
| 35 | + cnt++; | |
| 36 | + alloc_size = kmalloc_size_roundup(size_mul(cnt, sizeof(*p))); | |
| 37 | + p = krealloc(cur->jmp_history, alloc_size, GFP_KERNEL_ACCOUNT); | |
| 38 | + if (!p) | |
| 39 | + return -ENOMEM; | |
| 40 | + cur->jmp_history = p; | |
| 41 | + | |
| 42 | + p = &cur->jmp_history[cnt - 1]; | |
| 43 | + p->idx = env->insn_idx; | |
| 44 | + p->prev_idx = env->prev_insn_idx; | |
| 45 | + p->flags = insn_flags; | |
| 46 | + p->linked_regs = linked_regs; | |
| 47 | + cur->jmp_history_cnt = cnt; | |
| 48 | + env->cur_hist_ent = p; | |
| 49 | + | |
| 50 | + return 0; | |
| 51 | +} | |
| 52 | + | |
| 53 | +static bool is_atomic_load_insn(const struct bpf_insn *insn) | |
| 54 | +{ | |
| 55 | + return BPF_CLASS(insn->code) == BPF_STX && | |
| 56 | + BPF_MODE(insn->code) == BPF_ATOMIC && | |
| 57 | + insn->imm == BPF_LOAD_ACQ; | |
| 58 | +} | |
| 59 | + | |
| 60 | +static bool is_atomic_fetch_insn(const struct bpf_insn *insn) | |
| 61 | +{ | |
| 62 | + return BPF_CLASS(insn->code) == BPF_STX && | |
| 63 | + BPF_MODE(insn->code) == BPF_ATOMIC && | |
| 64 | + (insn->imm & BPF_FETCH); | |
| 65 | +} | |
| 66 | + | |
| 67 | +static int insn_stack_access_spi(int insn_flags) | |
| 68 | +{ | |
| 69 | + return (insn_flags >> INSN_F_SPI_SHIFT) & INSN_F_SPI_MASK; | |
| 70 | +} | |
| 71 | + | |
| 72 | +static int insn_stack_access_frameno(int insn_flags) | |
| 73 | +{ | |
| 74 | + return insn_flags & INSN_F_FRAMENO_MASK; | |
| 75 | +} | |
| 76 | + | |
| 77 | +/* Backtrack one insn at a time. If idx is not at the top of recorded | |
| 78 | + * history then previous instruction came from straight line execution. | |
| 79 | + * Return -ENOENT if we exhausted all instructions within given state. | |
| 80 | + * | |
| 81 | + * It's legal to have a bit of a looping with the same starting and ending | |
| 82 | + * insn index within the same state, e.g.: 3->4->5->3, so just because current | |
| 83 | + * instruction index is the same as state's first_idx doesn't mean we are | |
| 84 | + * done. If there is still some jump history left, we should keep going. We | |
| 85 | + * need to take into account that we might have a jump history between given | |
| 86 | + * state's parent and itself, due to checkpointing. In this case, we'll have | |
| 87 | + * history entry recording a jump from last instruction of parent state and | |
| 88 | + * first instruction of given state. | |
| 89 | + */ | |
| 90 | +static int get_prev_insn_idx(struct bpf_verifier_state *st, int i, | |
| 91 | + u32 *history) | |
| 92 | +{ | |
| 93 | + u32 cnt = *history; | |
| 94 | + | |
| 95 | + if (i == st->first_insn_idx) { | |
| 96 | + if (cnt == 0) | |
| 97 | + return -ENOENT; | |
| 98 | + if (cnt == 1 && st->jmp_history[0].idx == i) | |
| 99 | + return -ENOENT; | |
| 100 | + } | |
| 101 | + | |
| 102 | + if (cnt && st->jmp_history[cnt - 1].idx == i) { | |
| 103 | + i = st->jmp_history[cnt - 1].prev_idx; | |
| 104 | + (*history)--; | |
| 105 | + } else { | |
| 106 | + i--; | |
| 107 | + } | |
| 108 | + return i; | |
| 109 | +} | |
| 110 | + | |
| 111 | +static struct bpf_jmp_history_entry *get_jmp_hist_entry(struct bpf_verifier_state *st, | |
| 112 | + u32 hist_end, int insn_idx) | |
| 113 | +{ | |
| 114 | + if (hist_end > 0 && st->jmp_history[hist_end - 1].idx == insn_idx) | |
| 115 | + return &st->jmp_history[hist_end - 1]; | |
| 116 | + return NULL; | |
| 117 | +} | |
| 118 | + | |
| 119 | +static inline void bt_init(struct backtrack_state *bt, u32 frame) | |
| 120 | +{ | |
| 121 | + bt->frame = frame; | |
| 122 | +} | |
| 123 | + | |
| 124 | +static inline void bt_reset(struct backtrack_state *bt) | |
| 125 | +{ | |
| 126 | + struct bpf_verifier_env *env = bt->env; | |
| 127 | + | |
| 128 | + memset(bt, 0, sizeof(*bt)); | |
| 129 | + bt->env = env; | |
| 130 | +} | |
| 131 | + | |
| 132 | +static inline u32 bt_empty(struct backtrack_state *bt) | |
| 133 | +{ | |
| 134 | + u64 mask = 0; | |
| 135 | + int i; | |
| 136 | + | |
| 137 | + for (i = 0; i <= bt->frame; i++) | |
| 138 | + mask |= bt->reg_masks[i] | bt->stack_masks[i]; | |
| 139 | + | |
| 140 | + return mask == 0; | |
| 141 | +} | |
| 142 | + | |
| 143 | +static inline int bt_subprog_enter(struct backtrack_state *bt) | |
| 144 | +{ | |
| 145 | + if (bt->frame == MAX_CALL_FRAMES - 1) { | |
| 146 | + verifier_bug(bt->env, "subprog enter from frame %d", bt->frame); | |
| 147 | + return -EFAULT; | |
| 148 | + } | |
| 149 | + bt->frame++; | |
| 150 | + return 0; | |
| 151 | +} | |
| 152 | + | |
| 153 | +static inline int bt_subprog_exit(struct backtrack_state *bt) | |
| 154 | +{ | |
| 155 | + if (bt->frame == 0) { | |
| 156 | + verifier_bug(bt->env, "subprog exit from frame 0"); | |
| 157 | + return -EFAULT; | |
| 158 | + } | |
| 159 | + bt->frame--; | |
| 160 | + return 0; | |
| 161 | +} | |
| 162 | + | |
| 163 | +static inline void bt_clear_frame_reg(struct backtrack_state *bt, u32 frame, u32 reg) | |
| 164 | +{ | |
| 165 | + bt->reg_masks[frame] &= ~(1 << reg); | |
| 166 | +} | |
| 167 | + | |
| 168 | +static inline void bt_set_reg(struct backtrack_state *bt, u32 reg) | |
| 169 | +{ | |
| 170 | + bpf_bt_set_frame_reg(bt, bt->frame, reg); | |
| 171 | +} | |
| 172 | + | |
| 173 | +static inline void bt_clear_reg(struct backtrack_state *bt, u32 reg) | |
| 174 | +{ | |
| 175 | + bt_clear_frame_reg(bt, bt->frame, reg); | |
| 176 | +} | |
| 177 | + | |
| 178 | +static inline void bt_clear_frame_slot(struct backtrack_state *bt, u32 frame, u32 slot) | |
| 179 | +{ | |
| 180 | + bt->stack_masks[frame] &= ~(1ull << slot); | |
| 181 | +} | |
| 182 | + | |
| 183 | +static inline u32 bt_frame_reg_mask(struct backtrack_state *bt, u32 frame) | |
| 184 | +{ | |
| 185 | + return bt->reg_masks[frame]; | |
| 186 | +} | |
| 187 | + | |
| 188 | +static inline u32 bt_reg_mask(struct backtrack_state *bt) | |
| 189 | +{ | |
| 190 | + return bt->reg_masks[bt->frame]; | |
| 191 | +} | |
| 192 | + | |
| 193 | +static inline u64 bt_frame_stack_mask(struct backtrack_state *bt, u32 frame) | |
| 194 | +{ | |
| 195 | + return bt->stack_masks[frame]; | |
| 196 | +} | |
| 197 | + | |
| 198 | +static inline u64 bt_stack_mask(struct backtrack_state *bt) | |
| 199 | +{ | |
| 200 | + return bt->stack_masks[bt->frame]; | |
| 201 | +} | |
| 202 | + | |
| 203 | +static inline bool bt_is_reg_set(struct backtrack_state *bt, u32 reg) | |
| 204 | +{ | |
| 205 | + return bt->reg_masks[bt->frame] & (1 << reg); | |
| 206 | +} | |
| 207 | + | |
| 208 | + | |
| 209 | +/* format registers bitmask, e.g., "r0,r2,r4" for 0x15 mask */ | |
| 210 | +static void fmt_reg_mask(char *buf, ssize_t buf_sz, u32 reg_mask) | |
| 211 | +{ | |
| 212 | + DECLARE_BITMAP(mask, 64); | |
| 213 | + bool first = true; | |
| 214 | + int i, n; | |
| 215 | + | |
| 216 | + buf[0] = '\0'; | |
| 217 | + | |
| 218 | + bitmap_from_u64(mask, reg_mask); | |
| 219 | + for_each_set_bit(i, mask, 32) { | |
| 220 | + n = snprintf(buf, buf_sz, "%sr%d", first ? "" : ",", i); | |
| 221 | + first = false; | |
| 222 | + buf += n; | |
| 223 | + buf_sz -= n; | |
| 224 | + if (buf_sz < 0) | |
| 225 | + break; | |
| 226 | + } | |
| 227 | +} | |
| 228 | +/* format stack slots bitmask, e.g., "-8,-24,-40" for 0x15 mask */ | |
| 229 | +void bpf_fmt_stack_mask(char *buf, ssize_t buf_sz, u64 stack_mask) | |
| 230 | +{ | |
| 231 | + DECLARE_BITMAP(mask, 64); | |
| 232 | + bool first = true; | |
| 233 | + int i, n; | |
| 234 | + | |
| 235 | + buf[0] = '\0'; | |
| 236 | + | |
| 237 | + bitmap_from_u64(mask, stack_mask); | |
| 238 | + for_each_set_bit(i, mask, 64) { | |
| 239 | + n = snprintf(buf, buf_sz, "%s%d", first ? "" : ",", -(i + 1) * 8); | |
| 240 | + first = false; | |
| 241 | + buf += n; | |
| 242 | + buf_sz -= n; | |
| 243 | + if (buf_sz < 0) | |
| 244 | + break; | |
| 245 | + } | |
| 246 | +} | |
| 247 | + | |
| 248 | + | |
| 249 | +/* For given verifier state backtrack_insn() is called from the last insn to | |
| 250 | + * the first insn. Its purpose is to compute a bitmask of registers and | |
| 251 | + * stack slots that needs precision in the parent verifier state. | |
| 252 | + * | |
| 253 | + * @idx is an index of the instruction we are currently processing; | |
| 254 | + * @subseq_idx is an index of the subsequent instruction that: | |
| 255 | + * - *would be* executed next, if jump history is viewed in forward order; | |
| 256 | + * - *was* processed previously during backtracking. | |
| 257 | + */ | |
| 258 | +static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, | |
| 259 | + struct bpf_jmp_history_entry *hist, struct backtrack_state *bt) | |
| 260 | +{ | |
| 261 | + struct bpf_insn *insn = env->prog->insnsi + idx; | |
| 262 | + u8 class = BPF_CLASS(insn->code); | |
| 263 | + u8 opcode = BPF_OP(insn->code); | |
| 264 | + u8 mode = BPF_MODE(insn->code); | |
| 265 | + u32 dreg = insn->dst_reg; | |
| 266 | + u32 sreg = insn->src_reg; | |
| 267 | + u32 spi, i, fr; | |
| 268 | + | |
| 269 | + if (insn->code == 0) | |
| 270 | + return 0; | |
| 271 | + if (env->log.level & BPF_LOG_LEVEL2) { | |
| 272 | + fmt_reg_mask(env->tmp_str_buf, TMP_STR_BUF_LEN, bt_reg_mask(bt)); | |
| 273 | + verbose(env, "mark_precise: frame%d: regs=%s ", | |
| 274 | + bt->frame, env->tmp_str_buf); | |
| 275 | + bpf_fmt_stack_mask(env->tmp_str_buf, TMP_STR_BUF_LEN, bt_stack_mask(bt)); | |
| 276 | + verbose(env, "stack=%s before ", env->tmp_str_buf); | |
| 277 | + verbose(env, "%d: ", idx); | |
| 278 | + bpf_verbose_insn(env, insn); | |
| 279 | + } | |
| 280 | + | |
| 281 | + /* If there is a history record that some registers gained range at this insn, | |
| 282 | + * propagate precision marks to those registers, so that bt_is_reg_set() | |
| 283 | + * accounts for these registers. | |
| 284 | + */ | |
| 285 | + bpf_bt_sync_linked_regs(bt, hist); | |
| 286 | + | |
| 287 | + if (class == BPF_ALU || class == BPF_ALU64) { | |
| 288 | + if (!bt_is_reg_set(bt, dreg)) | |
| 289 | + return 0; | |
| 290 | + if (opcode == BPF_END || opcode == BPF_NEG) { | |
| 291 | + /* sreg is reserved and unused | |
| 292 | + * dreg still need precision before this insn | |
| 293 | + */ | |
| 294 | + return 0; | |
| 295 | + } else if (opcode == BPF_MOV) { | |
| 296 | + if (BPF_SRC(insn->code) == BPF_X) { | |
| 297 | + /* dreg = sreg or dreg = (s8, s16, s32)sreg | |
| 298 | + * dreg needs precision after this insn | |
| 299 | + * sreg needs precision before this insn | |
| 300 | + */ | |
| 301 | + bt_clear_reg(bt, dreg); | |
| 302 | + if (sreg != BPF_REG_FP) | |
| 303 | + bt_set_reg(bt, sreg); | |
| 304 | + } else { | |
| 305 | + /* dreg = K | |
| 306 | + * dreg needs precision after this insn. | |
| 307 | + * Corresponding register is already marked | |
| 308 | + * as precise=true in this verifier state. | |
| 309 | + * No further markings in parent are necessary | |
| 310 | + */ | |
| 311 | + bt_clear_reg(bt, dreg); | |
| 312 | + } | |
| 313 | + } else { | |
| 314 | + if (BPF_SRC(insn->code) == BPF_X) { | |
| 315 | + /* dreg += sreg | |
| 316 | + * both dreg and sreg need precision | |
| 317 | + * before this insn | |
| 318 | + */ | |
| 319 | + if (sreg != BPF_REG_FP) | |
| 320 | + bt_set_reg(bt, sreg); | |
| 321 | + } /* else dreg += K | |
| 322 | + * dreg still needs precision before this insn | |
| 323 | + */ | |
| 324 | + } | |
| 325 | + } else if (class == BPF_LDX || | |
| 326 | + is_atomic_load_insn(insn) || | |
| 327 | + is_atomic_fetch_insn(insn)) { | |
| 328 | + u32 load_reg = dreg; | |
| 329 | + | |
| 330 | + /* | |
| 331 | + * Atomic fetch operation writes the old value into | |
| 332 | + * a register (sreg or r0) and if it was tracked for | |
| 333 | + * precision, propagate to the stack slot like we do | |
| 334 | + * in regular ldx. | |
| 335 | + */ | |
| 336 | + if (is_atomic_fetch_insn(insn)) | |
| 337 | + load_reg = insn->imm == BPF_CMPXCHG ? | |
| 338 | + BPF_REG_0 : sreg; | |
| 339 | + | |
| 340 | + if (!bt_is_reg_set(bt, load_reg)) | |
| 341 | + return 0; | |
| 342 | + bt_clear_reg(bt, load_reg); | |
| 343 | + | |
| 344 | + /* scalars can only be spilled into stack w/o losing precision. | |
| 345 | + * Load from any other memory can be zero extended. | |
| 346 | + * The desire to keep that precision is already indicated | |
| 347 | + * by 'precise' mark in corresponding register of this state. | |
| 348 | + * No further tracking necessary. | |
| 349 | + */ | |
| 350 | + if (!hist || !(hist->flags & INSN_F_STACK_ACCESS)) | |
| 351 | + return 0; | |
| 352 | + /* dreg = *(u64 *)[fp - off] was a fill from the stack. | |
| 353 | + * that [fp - off] slot contains scalar that needs to be | |
| 354 | + * tracked with precision | |
| 355 | + */ | |
| 356 | + spi = insn_stack_access_spi(hist->flags); | |
| 357 | + fr = insn_stack_access_frameno(hist->flags); | |
| 358 | + bpf_bt_set_frame_slot(bt, fr, spi); | |
| 359 | + } else if (class == BPF_STX || class == BPF_ST) { | |
| 360 | + if (bt_is_reg_set(bt, dreg)) | |
| 361 | + /* stx & st shouldn't be using _scalar_ dst_reg | |
| 362 | + * to access memory. It means backtracking | |
| 363 | + * encountered a case of pointer subtraction. | |
| 364 | + */ | |
| 365 | + return -ENOTSUPP; | |
| 366 | + /* scalars can only be spilled into stack */ | |
| 367 | + if (!hist || !(hist->flags & INSN_F_STACK_ACCESS)) | |
| 368 | + return 0; | |
| 369 | + spi = insn_stack_access_spi(hist->flags); | |
| 370 | + fr = insn_stack_access_frameno(hist->flags); | |
| 371 | + if (!bt_is_frame_slot_set(bt, fr, spi)) | |
| 372 | + return 0; | |
| 373 | + bt_clear_frame_slot(bt, fr, spi); | |
| 374 | + if (class == BPF_STX) | |
| 375 | + bt_set_reg(bt, sreg); | |
| 376 | + } else if (class == BPF_JMP || class == BPF_JMP32) { | |
| 377 | + if (bpf_pseudo_call(insn)) { | |
| 378 | + int subprog_insn_idx, subprog; | |
| 379 | + | |
| 380 | + subprog_insn_idx = idx + insn->imm + 1; | |
| 381 | + subprog = bpf_find_subprog(env, subprog_insn_idx); | |
| 382 | + if (subprog < 0) | |
| 383 | + return -EFAULT; | |
| 384 | + | |
| 385 | + if (bpf_subprog_is_global(env, subprog)) { | |
| 386 | + /* check that jump history doesn't have any | |
| 387 | + * extra instructions from subprog; the next | |
| 388 | + * instruction after call to global subprog | |
| 389 | + * should be literally next instruction in | |
| 390 | + * caller program | |
| 391 | + */ | |
| 392 | + verifier_bug_if(idx + 1 != subseq_idx, env, | |
| 393 | + "extra insn from subprog"); | |
| 394 | + /* r1-r5 are invalidated after subprog call, | |
| 395 | + * so for global func call it shouldn't be set | |
| 396 | + * anymore | |
| 397 | + */ | |
| 398 | + if (bt_reg_mask(bt) & BPF_REGMASK_ARGS) { | |
| 399 | + verifier_bug(env, "global subprog unexpected regs %x", | |
| 400 | + bt_reg_mask(bt)); | |
| 401 | + return -EFAULT; | |
| 402 | + } | |
| 403 | + /* global subprog always sets R0 */ | |
| 404 | + bt_clear_reg(bt, BPF_REG_0); | |
| 405 | + return 0; | |
| 406 | + } else { | |
| 407 | + /* static subprog call instruction, which | |
| 408 | + * means that we are exiting current subprog, | |
| 409 | + * so only r1-r5 could be still requested as | |
| 410 | + * precise, r0 and r6-r10 or any stack slot in | |
| 411 | + * the current frame should be zero by now | |
| 412 | + */ | |
| 413 | + if (bt_reg_mask(bt) & ~BPF_REGMASK_ARGS) { | |
| 414 | + verifier_bug(env, "static subprog unexpected regs %x", | |
| 415 | + bt_reg_mask(bt)); | |
| 416 | + return -EFAULT; | |
| 417 | + } | |
| 418 | + /* we are now tracking register spills correctly, | |
| 419 | + * so any instance of leftover slots is a bug | |
| 420 | + */ | |
| 421 | + if (bt_stack_mask(bt) != 0) { | |
| 422 | + verifier_bug(env, | |
| 423 | + "static subprog leftover stack slots %llx", | |
| 424 | + bt_stack_mask(bt)); | |
| 425 | + return -EFAULT; | |
| 426 | + } | |
| 427 | + /* propagate r1-r5 to the caller */ | |
| 428 | + for (i = BPF_REG_1; i <= BPF_REG_5; i++) { | |
| 429 | + if (bt_is_reg_set(bt, i)) { | |
| 430 | + bt_clear_reg(bt, i); | |
| 431 | + bpf_bt_set_frame_reg(bt, bt->frame - 1, i); | |
| 432 | + } | |
| 433 | + } | |
| 434 | + if (bt_subprog_exit(bt)) | |
| 435 | + return -EFAULT; | |
| 436 | + return 0; | |
| 437 | + } | |
| 438 | + } else if (bpf_is_sync_callback_calling_insn(insn) && idx != subseq_idx - 1) { | |
| 439 | + /* exit from callback subprog to callback-calling helper or | |
| 440 | + * kfunc call. Use idx/subseq_idx check to discern it from | |
| 441 | + * straight line code backtracking. | |
| 442 | + * Unlike the subprog call handling above, we shouldn't | |
| 443 | + * propagate precision of r1-r5 (if any requested), as they are | |
| 444 | + * not actually arguments passed directly to callback subprogs | |
| 445 | + */ | |
| 446 | + if (bt_reg_mask(bt) & ~BPF_REGMASK_ARGS) { | |
| 447 | + verifier_bug(env, "callback unexpected regs %x", | |
| 448 | + bt_reg_mask(bt)); | |
| 449 | + return -EFAULT; | |
| 450 | + } | |
| 451 | + if (bt_stack_mask(bt) != 0) { | |
| 452 | + verifier_bug(env, "callback leftover stack slots %llx", | |
| 453 | + bt_stack_mask(bt)); | |
| 454 | + return -EFAULT; | |
| 455 | + } | |
| 456 | + /* clear r1-r5 in callback subprog's mask */ | |
| 457 | + for (i = BPF_REG_1; i <= BPF_REG_5; i++) | |
| 458 | + bt_clear_reg(bt, i); | |
| 459 | + if (bt_subprog_exit(bt)) | |
| 460 | + return -EFAULT; | |
| 461 | + return 0; | |
| 462 | + } else if (opcode == BPF_CALL) { | |
| 463 | + /* kfunc with imm==0 is invalid and fixup_kfunc_call will | |
| 464 | + * catch this error later. Make backtracking conservative | |
| 465 | + * with ENOTSUPP. | |
| 466 | + */ | |
| 467 | + if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL && insn->imm == 0) | |
| 468 | + return -ENOTSUPP; | |
| 469 | + /* regular helper call sets R0 */ | |
| 470 | + bt_clear_reg(bt, BPF_REG_0); | |
| 471 | + if (bt_reg_mask(bt) & BPF_REGMASK_ARGS) { | |
| 472 | + /* if backtracking was looking for registers R1-R5 | |
| 473 | + * they should have been found already. | |
| 474 | + */ | |
| 475 | + verifier_bug(env, "backtracking call unexpected regs %x", | |
| 476 | + bt_reg_mask(bt)); | |
| 477 | + return -EFAULT; | |
| 478 | + } | |
| 479 | + if (insn->src_reg == BPF_REG_0 && insn->imm == BPF_FUNC_tail_call | |
| 480 | + && subseq_idx - idx != 1) { | |
| 481 | + if (bt_subprog_enter(bt)) | |
| 482 | + return -EFAULT; | |
| 483 | + } | |
| 484 | + } else if (opcode == BPF_EXIT) { | |
| 485 | + bool r0_precise; | |
| 486 | + | |
| 487 | + /* Backtracking to a nested function call, 'idx' is a part of | |
| 488 | + * the inner frame 'subseq_idx' is a part of the outer frame. | |
| 489 | + * In case of a regular function call, instructions giving | |
| 490 | + * precision to registers R1-R5 should have been found already. | |
| 491 | + * In case of a callback, it is ok to have R1-R5 marked for | |
| 492 | + * backtracking, as these registers are set by the function | |
| 493 | + * invoking callback. | |
| 494 | + */ | |
| 495 | + if (subseq_idx >= 0 && bpf_calls_callback(env, subseq_idx)) | |
| 496 | + for (i = BPF_REG_1; i <= BPF_REG_5; i++) | |
| 497 | + bt_clear_reg(bt, i); | |
| 498 | + if (bt_reg_mask(bt) & BPF_REGMASK_ARGS) { | |
| 499 | + verifier_bug(env, "backtracking exit unexpected regs %x", | |
| 500 | + bt_reg_mask(bt)); | |
| 501 | + return -EFAULT; | |
| 502 | + } | |
| 503 | + | |
| 504 | + /* BPF_EXIT in subprog or callback always returns | |
| 505 | + * right after the call instruction, so by checking | |
| 506 | + * whether the instruction at subseq_idx-1 is subprog | |
| 507 | + * call or not we can distinguish actual exit from | |
| 508 | + * *subprog* from exit from *callback*. In the former | |
| 509 | + * case, we need to propagate r0 precision, if | |
| 510 | + * necessary. In the former we never do that. | |
| 511 | + */ | |
| 512 | + r0_precise = subseq_idx - 1 >= 0 && | |
| 513 | + bpf_pseudo_call(&env->prog->insnsi[subseq_idx - 1]) && | |
| 514 | + bt_is_reg_set(bt, BPF_REG_0); | |
| 515 | + | |
| 516 | + bt_clear_reg(bt, BPF_REG_0); | |
| 517 | + if (bt_subprog_enter(bt)) | |
| 518 | + return -EFAULT; | |
| 519 | + | |
| 520 | + if (r0_precise) | |
| 521 | + bt_set_reg(bt, BPF_REG_0); | |
| 522 | + /* r6-r9 and stack slots will stay set in caller frame | |
| 523 | + * bitmasks until we return back from callee(s) | |
| 524 | + */ | |
| 525 | + return 0; | |
| 526 | + } else if (BPF_SRC(insn->code) == BPF_X) { | |
| 527 | + if (!bt_is_reg_set(bt, dreg) && !bt_is_reg_set(bt, sreg)) | |
| 528 | + return 0; | |
| 529 | + /* dreg <cond> sreg | |
| 530 | + * Both dreg and sreg need precision before | |
| 531 | + * this insn. If only sreg was marked precise | |
| 532 | + * before it would be equally necessary to | |
| 533 | + * propagate it to dreg. | |
| 534 | + */ | |
| 535 | + if (!hist || !(hist->flags & INSN_F_SRC_REG_STACK)) | |
| 536 | + bt_set_reg(bt, sreg); | |
| 537 | + if (!hist || !(hist->flags & INSN_F_DST_REG_STACK)) | |
| 538 | + bt_set_reg(bt, dreg); | |
| 539 | + } else if (BPF_SRC(insn->code) == BPF_K) { | |
| 540 | + /* dreg <cond> K | |
| 541 | + * Only dreg still needs precision before | |
| 542 | + * this insn, so for the K-based conditional | |
| 543 | + * there is nothing new to be marked. | |
| 544 | + */ | |
| 545 | + } | |
| 546 | + } else if (class == BPF_LD) { | |
| 547 | + if (!bt_is_reg_set(bt, dreg)) | |
| 548 | + return 0; | |
| 549 | + bt_clear_reg(bt, dreg); | |
| 550 | + /* It's ld_imm64 or ld_abs or ld_ind. | |
| 551 | + * For ld_imm64 no further tracking of precision | |
| 552 | + * into parent is necessary | |
| 553 | + */ | |
| 554 | + if (mode == BPF_IND || mode == BPF_ABS) | |
| 555 | + /* to be analyzed */ | |
| 556 | + return -ENOTSUPP; | |
| 557 | + } | |
| 558 | + /* Propagate precision marks to linked registers, to account for | |
| 559 | + * registers marked as precise in this function. | |
| 560 | + */ | |
| 561 | + bpf_bt_sync_linked_regs(bt, hist); | |
| 562 | + return 0; | |
| 563 | +} | |
| 564 | + | |
| 565 | +/* the scalar precision tracking algorithm: | |
| 566 | + * . at the start all registers have precise=false. | |
| 567 | + * . scalar ranges are tracked as normal through alu and jmp insns. | |
| 568 | + * . once precise value of the scalar register is used in: | |
| 569 | + * . ptr + scalar alu | |
| 570 | + * . if (scalar cond K|scalar) | |
| 571 | + * . helper_call(.., scalar, ...) where ARG_CONST is expected | |
| 572 | + * backtrack through the verifier states and mark all registers and | |
| 573 | + * stack slots with spilled constants that these scalar registers | |
| 574 | + * should be precise. | |
| 575 | + * . during state pruning two registers (or spilled stack slots) | |
| 576 | + * are equivalent if both are not precise. | |
| 577 | + * | |
| 578 | + * Note the verifier cannot simply walk register parentage chain, | |
| 579 | + * since many different registers and stack slots could have been | |
| 580 | + * used to compute single precise scalar. | |
| 581 | + * | |
| 582 | + * The approach of starting with precise=true for all registers and then | |
| 583 | + * backtrack to mark a register as not precise when the verifier detects | |
| 584 | + * that program doesn't care about specific value (e.g., when helper | |
| 585 | + * takes register as ARG_ANYTHING parameter) is not safe. | |
| 586 | + * | |
| 587 | + * It's ok to walk single parentage chain of the verifier states. | |
| 588 | + * It's possible that this backtracking will go all the way till 1st insn. | |
| 589 | + * All other branches will be explored for needing precision later. | |
| 590 | + * | |
| 591 | + * The backtracking needs to deal with cases like: | |
| 592 | + * R8=map_value(id=0,off=0,ks=4,vs=1952,imm=0) R9_w=map_value(id=0,off=40,ks=4,vs=1952,imm=0) | |
| 593 | + * r9 -= r8 | |
| 594 | + * r5 = r9 | |
| 595 | + * if r5 > 0x79f goto pc+7 | |
| 596 | + * R5_w=inv(id=0,umax_value=1951,var_off=(0x0; 0x7ff)) | |
| 597 | + * r5 += 1 | |
| 598 | + * ... | |
| 599 | + * call bpf_perf_event_output#25 | |
| 600 | + * where .arg5_type = ARG_CONST_SIZE_OR_ZERO | |
| 601 | + * | |
| 602 | + * and this case: | |
| 603 | + * r6 = 1 | |
| 604 | + * call foo // uses callee's r6 inside to compute r0 | |
| 605 | + * r0 += r6 | |
| 606 | + * if r0 == 0 goto | |
| 607 | + * | |
| 608 | + * to track above reg_mask/stack_mask needs to be independent for each frame. | |
| 609 | + * | |
| 610 | + * Also if parent's curframe > frame where backtracking started, | |
| 611 | + * the verifier need to mark registers in both frames, otherwise callees | |
| 612 | + * may incorrectly prune callers. This is similar to | |
| 613 | + * commit 7640ead93924 ("bpf: verifier: make sure callees don't prune with caller differences") | |
| 614 | + * | |
| 615 | + * For now backtracking falls back into conservative marking. | |
| 616 | + */ | |
| 617 | +void bpf_mark_all_scalars_precise(struct bpf_verifier_env *env, | |
| 618 | + struct bpf_verifier_state *st) | |
| 619 | +{ | |
| 620 | + struct bpf_func_state *func; | |
| 621 | + struct bpf_reg_state *reg; | |
| 622 | + int i, j; | |
| 623 | + | |
| 624 | + if (env->log.level & BPF_LOG_LEVEL2) { | |
| 625 | + verbose(env, "mark_precise: frame%d: falling back to forcing all scalars precise\n", | |
| 626 | + st->curframe); | |
| 627 | + } | |
| 628 | + | |
| 629 | + /* big hammer: mark all scalars precise in this path. | |
| 630 | + * pop_stack may still get !precise scalars. | |
| 631 | + * We also skip current state and go straight to first parent state, | |
| 632 | + * because precision markings in current non-checkpointed state are | |
| 633 | + * not needed. See why in the comment in __mark_chain_precision below. | |
| 634 | + */ | |
| 635 | + for (st = st->parent; st; st = st->parent) { | |
| 636 | + for (i = 0; i <= st->curframe; i++) { | |
| 637 | + func = st->frame[i]; | |
| 638 | + for (j = 0; j < BPF_REG_FP; j++) { | |
| 639 | + reg = &func->regs[j]; | |
| 640 | + if (reg->type != SCALAR_VALUE || reg->precise) | |
| 641 | + continue; | |
| 642 | + reg->precise = true; | |
| 643 | + if (env->log.level & BPF_LOG_LEVEL2) { | |
| 644 | + verbose(env, "force_precise: frame%d: forcing r%d to be precise\n", | |
| 645 | + i, j); | |
| 646 | + } | |
| 647 | + } | |
| 648 | + for (j = 0; j < func->allocated_stack / BPF_REG_SIZE; j++) { | |
| 649 | + if (!bpf_is_spilled_reg(&func->stack[j])) | |
| 650 | + continue; | |
| 651 | + reg = &func->stack[j].spilled_ptr; | |
| 652 | + if (reg->type != SCALAR_VALUE || reg->precise) | |
| 653 | + continue; | |
| 654 | + reg->precise = true; | |
| 655 | + if (env->log.level & BPF_LOG_LEVEL2) { | |
| 656 | + verbose(env, "force_precise: frame%d: forcing fp%d to be precise\n", | |
| 657 | + i, -(j + 1) * 8); | |
| 658 | + } | |
| 659 | + } | |
| 660 | + } | |
| 661 | + } | |
| 662 | +} | |
| 663 | + | |
| 664 | +/* | |
| 665 | + * bpf_mark_chain_precision() backtracks BPF program instruction sequence and | |
| 666 | + * chain of verifier states making sure that register *regno* (if regno >= 0) | |
| 667 | + * and/or stack slot *spi* (if spi >= 0) are marked as precisely tracked | |
| 668 | + * SCALARS, as well as any other registers and slots that contribute to | |
| 669 | + * a tracked state of given registers/stack slots, depending on specific BPF | |
| 670 | + * assembly instructions (see backtrack_insns() for exact instruction handling | |
| 671 | + * logic). This backtracking relies on recorded jmp_history and is able to | |
| 672 | + * traverse entire chain of parent states. This process ends only when all the | |
| 673 | + * necessary registers/slots and their transitive dependencies are marked as | |
| 674 | + * precise. | |
| 675 | + * | |
| 676 | + * One important and subtle aspect is that precise marks *do not matter* in | |
| 677 | + * the currently verified state (current state). It is important to understand | |
| 678 | + * why this is the case. | |
| 679 | + * | |
| 680 | + * First, note that current state is the state that is not yet "checkpointed", | |
| 681 | + * i.e., it is not yet put into env->explored_states, and it has no children | |
| 682 | + * states as well. It's ephemeral, and can end up either a) being discarded if | |
| 683 | + * compatible explored state is found at some point or BPF_EXIT instruction is | |
| 684 | + * reached or b) checkpointed and put into env->explored_states, branching out | |
| 685 | + * into one or more children states. | |
| 686 | + * | |
| 687 | + * In the former case, precise markings in current state are completely | |
| 688 | + * ignored by state comparison code (see regsafe() for details). Only | |
| 689 | + * checkpointed ("old") state precise markings are important, and if old | |
| 690 | + * state's register/slot is precise, regsafe() assumes current state's | |
| 691 | + * register/slot as precise and checks value ranges exactly and precisely. If | |
| 692 | + * states turn out to be compatible, current state's necessary precise | |
| 693 | + * markings and any required parent states' precise markings are enforced | |
| 694 | + * after the fact with propagate_precision() logic, after the fact. But it's | |
| 695 | + * important to realize that in this case, even after marking current state | |
| 696 | + * registers/slots as precise, we immediately discard current state. So what | |
| 697 | + * actually matters is any of the precise markings propagated into current | |
| 698 | + * state's parent states, which are always checkpointed (due to b) case above). | |
| 699 | + * As such, for scenario a) it doesn't matter if current state has precise | |
| 700 | + * markings set or not. | |
| 701 | + * | |
| 702 | + * Now, for the scenario b), checkpointing and forking into child(ren) | |
| 703 | + * state(s). Note that before current state gets to checkpointing step, any | |
| 704 | + * processed instruction always assumes precise SCALAR register/slot | |
| 705 | + * knowledge: if precise value or range is useful to prune jump branch, BPF | |
| 706 | + * verifier takes this opportunity enthusiastically. Similarly, when | |
| 707 | + * register's value is used to calculate offset or memory address, exact | |
| 708 | + * knowledge of SCALAR range is assumed, checked, and enforced. So, similar to | |
| 709 | + * what we mentioned above about state comparison ignoring precise markings | |
| 710 | + * during state comparison, BPF verifier ignores and also assumes precise | |
| 711 | + * markings *at will* during instruction verification process. But as verifier | |
| 712 | + * assumes precision, it also propagates any precision dependencies across | |
| 713 | + * parent states, which are not yet finalized, so can be further restricted | |
| 714 | + * based on new knowledge gained from restrictions enforced by their children | |
| 715 | + * states. This is so that once those parent states are finalized, i.e., when | |
| 716 | + * they have no more active children state, state comparison logic in | |
| 717 | + * is_state_visited() would enforce strict and precise SCALAR ranges, if | |
| 718 | + * required for correctness. | |
| 719 | + * | |
| 720 | + * To build a bit more intuition, note also that once a state is checkpointed, | |
| 721 | + * the path we took to get to that state is not important. This is crucial | |
| 722 | + * property for state pruning. When state is checkpointed and finalized at | |
| 723 | + * some instruction index, it can be correctly and safely used to "short | |
| 724 | + * circuit" any *compatible* state that reaches exactly the same instruction | |
| 725 | + * index. I.e., if we jumped to that instruction from a completely different | |
| 726 | + * code path than original finalized state was derived from, it doesn't | |
| 727 | + * matter, current state can be discarded because from that instruction | |
| 728 | + * forward having a compatible state will ensure we will safely reach the | |
| 729 | + * exit. States describe preconditions for further exploration, but completely | |
| 730 | + * forget the history of how we got here. | |
| 731 | + * | |
| 732 | + * This also means that even if we needed precise SCALAR range to get to | |
| 733 | + * finalized state, but from that point forward *that same* SCALAR register is | |
| 734 | + * never used in a precise context (i.e., it's precise value is not needed for | |
| 735 | + * correctness), it's correct and safe to mark such register as "imprecise" | |
| 736 | + * (i.e., precise marking set to false). This is what we rely on when we do | |
| 737 | + * not set precise marking in current state. If no child state requires | |
| 738 | + * precision for any given SCALAR register, it's safe to dictate that it can | |
| 739 | + * be imprecise. If any child state does require this register to be precise, | |
| 740 | + * we'll mark it precise later retroactively during precise markings | |
| 741 | + * propagation from child state to parent states. | |
| 742 | + * | |
| 743 | + * Skipping precise marking setting in current state is a mild version of | |
| 744 | + * relying on the above observation. But we can utilize this property even | |
| 745 | + * more aggressively by proactively forgetting any precise marking in the | |
| 746 | + * current state (which we inherited from the parent state), right before we | |
| 747 | + * checkpoint it and branch off into new child state. This is done by | |
| 748 | + * mark_all_scalars_imprecise() to hopefully get more permissive and generic | |
| 749 | + * finalized states which help in short circuiting more future states. | |
| 750 | + */ | |
| 751 | +int bpf_mark_chain_precision(struct bpf_verifier_env *env, | |
| 752 | + struct bpf_verifier_state *starting_state, | |
| 753 | + int regno, | |
| 754 | + bool *changed) | |
| 755 | +{ | |
| 756 | + struct bpf_verifier_state *st = starting_state; | |
| 757 | + struct backtrack_state *bt = &env->bt; | |
| 758 | + int first_idx = st->first_insn_idx; | |
| 759 | + int last_idx = starting_state->insn_idx; | |
| 760 | + int subseq_idx = -1; | |
| 761 | + struct bpf_func_state *func; | |
| 762 | + bool tmp, skip_first = true; | |
| 763 | + struct bpf_reg_state *reg; | |
| 764 | + int i, fr, err; | |
| 765 | + | |
| 766 | + if (!env->bpf_capable) | |
| 767 | + return 0; | |
| 768 | + | |
| 769 | + changed = changed ?: &tmp; | |
| 770 | + /* set frame number from which we are starting to backtrack */ | |
| 771 | + bt_init(bt, starting_state->curframe); | |
| 772 | + | |
| 773 | + /* Do sanity checks against current state of register and/or stack | |
| 774 | + * slot, but don't set precise flag in current state, as precision | |
| 775 | + * tracking in the current state is unnecessary. | |
| 776 | + */ | |
| 777 | + func = st->frame[bt->frame]; | |
| 778 | + if (regno >= 0) { | |
| 779 | + reg = &func->regs[regno]; | |
| 780 | + if (reg->type != SCALAR_VALUE) { | |
| 781 | + verifier_bug(env, "backtracking misuse"); | |
| 782 | + return -EFAULT; | |
| 783 | + } | |
| 784 | + bt_set_reg(bt, regno); | |
| 785 | + } | |
| 786 | + | |
| 787 | + if (bt_empty(bt)) | |
| 788 | + return 0; | |
| 789 | + | |
| 790 | + for (;;) { | |
| 791 | + DECLARE_BITMAP(mask, 64); | |
| 792 | + u32 history = st->jmp_history_cnt; | |
| 793 | + struct bpf_jmp_history_entry *hist; | |
| 794 | + | |
| 795 | + if (env->log.level & BPF_LOG_LEVEL2) { | |
| 796 | + verbose(env, "mark_precise: frame%d: last_idx %d first_idx %d subseq_idx %d \n", | |
| 797 | + bt->frame, last_idx, first_idx, subseq_idx); | |
| 798 | + } | |
| 799 | + | |
| 800 | + if (last_idx < 0) { | |
| 801 | + /* we are at the entry into subprog, which | |
| 802 | + * is expected for global funcs, but only if | |
| 803 | + * requested precise registers are R1-R5 | |
| 804 | + * (which are global func's input arguments) | |
| 805 | + */ | |
| 806 | + if (st->curframe == 0 && | |
| 807 | + st->frame[0]->subprogno > 0 && | |
| 808 | + st->frame[0]->callsite == BPF_MAIN_FUNC && | |
| 809 | + bt_stack_mask(bt) == 0 && | |
| 810 | + (bt_reg_mask(bt) & ~BPF_REGMASK_ARGS) == 0) { | |
| 811 | + bitmap_from_u64(mask, bt_reg_mask(bt)); | |
| 812 | + for_each_set_bit(i, mask, 32) { | |
| 813 | + reg = &st->frame[0]->regs[i]; | |
| 814 | + bt_clear_reg(bt, i); | |
| 815 | + if (reg->type == SCALAR_VALUE) { | |
| 816 | + reg->precise = true; | |
| 817 | + *changed = true; | |
| 818 | + } | |
| 819 | + } | |
| 820 | + return 0; | |
| 821 | + } | |
| 822 | + | |
| 823 | + verifier_bug(env, "backtracking func entry subprog %d reg_mask %x stack_mask %llx", | |
| 824 | + st->frame[0]->subprogno, bt_reg_mask(bt), bt_stack_mask(bt)); | |
| 825 | + return -EFAULT; | |
| 826 | + } | |
| 827 | + | |
| 828 | + for (i = last_idx;;) { | |
| 829 | + if (skip_first) { | |
| 830 | + err = 0; | |
| 831 | + skip_first = false; | |
| 832 | + } else { | |
| 833 | + hist = get_jmp_hist_entry(st, history, i); | |
| 834 | + err = backtrack_insn(env, i, subseq_idx, hist, bt); | |
| 835 | + } | |
| 836 | + if (err == -ENOTSUPP) { | |
| 837 | + bpf_mark_all_scalars_precise(env, starting_state); | |
| 838 | + bt_reset(bt); | |
| 839 | + return 0; | |
| 840 | + } else if (err) { | |
| 841 | + return err; | |
| 842 | + } | |
| 843 | + if (bt_empty(bt)) | |
| 844 | + /* Found assignment(s) into tracked register in this state. | |
| 845 | + * Since this state is already marked, just return. | |
| 846 | + * Nothing to be tracked further in the parent state. | |
| 847 | + */ | |
| 848 | + return 0; | |
| 849 | + subseq_idx = i; | |
| 850 | + i = get_prev_insn_idx(st, i, &history); | |
| 851 | + if (i == -ENOENT) | |
| 852 | + break; | |
| 853 | + if (i >= env->prog->len) { | |
| 854 | + /* This can happen if backtracking reached insn 0 | |
| 855 | + * and there are still reg_mask or stack_mask | |
| 856 | + * to backtrack. | |
| 857 | + * It means the backtracking missed the spot where | |
| 858 | + * particular register was initialized with a constant. | |
| 859 | + */ | |
| 860 | + verifier_bug(env, "backtracking idx %d", i); | |
| 861 | + return -EFAULT; | |
| 862 | + } | |
| 863 | + } | |
| 864 | + st = st->parent; | |
| 865 | + if (!st) | |
| 866 | + break; | |
| 867 | + | |
| 868 | + for (fr = bt->frame; fr >= 0; fr--) { | |
| 869 | + func = st->frame[fr]; | |
| 870 | + bitmap_from_u64(mask, bt_frame_reg_mask(bt, fr)); | |
| 871 | + for_each_set_bit(i, mask, 32) { | |
| 872 | + reg = &func->regs[i]; | |
| 873 | + if (reg->type != SCALAR_VALUE) { | |
| 874 | + bt_clear_frame_reg(bt, fr, i); | |
| 875 | + continue; | |
| 876 | + } | |
| 877 | + if (reg->precise) { | |
| 878 | + bt_clear_frame_reg(bt, fr, i); | |
| 879 | + } else { | |
| 880 | + reg->precise = true; | |
| 881 | + *changed = true; | |
| 882 | + } | |
| 883 | + } | |
| 884 | + | |
| 885 | + bitmap_from_u64(mask, bt_frame_stack_mask(bt, fr)); | |
| 886 | + for_each_set_bit(i, mask, 64) { | |
| 887 | + if (verifier_bug_if(i >= func->allocated_stack / BPF_REG_SIZE, | |
| 888 | + env, "stack slot %d, total slots %d", | |
| 889 | + i, func->allocated_stack / BPF_REG_SIZE)) | |
| 890 | + return -EFAULT; | |
| 891 | + | |
| 892 | + if (!bpf_is_spilled_scalar_reg(&func->stack[i])) { | |
| 893 | + bt_clear_frame_slot(bt, fr, i); | |
| 894 | + continue; | |
| 895 | + } | |
| 896 | + reg = &func->stack[i].spilled_ptr; | |
| 897 | + if (reg->precise) { | |
| 898 | + bt_clear_frame_slot(bt, fr, i); | |
| 899 | + } else { | |
| 900 | + reg->precise = true; | |
| 901 | + *changed = true; | |
| 902 | + } | |
| 903 | + } | |
| 904 | + if (env->log.level & BPF_LOG_LEVEL2) { | |
| 905 | + fmt_reg_mask(env->tmp_str_buf, TMP_STR_BUF_LEN, | |
| 906 | + bt_frame_reg_mask(bt, fr)); | |
| 907 | + verbose(env, "mark_precise: frame%d: parent state regs=%s ", | |
| 908 | + fr, env->tmp_str_buf); | |
| 909 | + bpf_fmt_stack_mask(env->tmp_str_buf, TMP_STR_BUF_LEN, | |
| 910 | + bt_frame_stack_mask(bt, fr)); | |
| 911 | + verbose(env, "stack=%s: ", env->tmp_str_buf); | |
| 912 | + print_verifier_state(env, st, fr, true); | |
| 913 | + } | |
| 914 | + } | |
| 915 | + | |
| 916 | + if (bt_empty(bt)) | |
| 917 | + return 0; | |
| 918 | + | |
| 919 | + subseq_idx = first_idx; | |
| 920 | + last_idx = st->last_insn_idx; | |
| 921 | + first_idx = st->first_insn_idx; | |
| 922 | + } | |
| 923 | + | |
| 924 | + /* if we still have requested precise regs or slots, we missed | |
| 925 | + * something (e.g., stack access through non-r10 register), so | |
| 926 | + * fallback to marking all precise | |
| 927 | + */ | |
| 928 | + if (!bt_empty(bt)) { | |
| 929 | + bpf_mark_all_scalars_precise(env, starting_state); | |
| 930 | + bt_reset(bt); | |
| 931 | + } | |
| 932 | + | |
| 933 | + return 0; | |
| 934 | +} | |
| @@ -513,20 +513,6 @@ static bool helper_multiple_ref_obj_use(enum bpf_func_id func_id, · 512 unmodified lines | ||
| 513 | 513 | return ref_obj_uses > 1; |
| 514 | 514 | } |
| 515 | 515 | |
| 516 | −static bool is_atomic_load_insn(const struct bpf_insn *insn) | |
| 517 | −{ | |
| 518 | − return BPF_CLASS(insn->code) == BPF_STX && | |
| 519 | − BPF_MODE(insn->code) == BPF_ATOMIC && | |
| 520 | − insn->imm == BPF_LOAD_ACQ; | |
| 521 | −} | |
| 522 | − | |
| 523 | −static bool is_atomic_fetch_insn(const struct bpf_insn *insn) | |
| 524 | −{ | |
| 525 | − return BPF_CLASS(insn->code) == BPF_STX && | |
| 526 | − BPF_MODE(insn->code) == BPF_ATOMIC && | |
| 527 | − (insn->imm & BPF_FETCH); | |
| 528 | −} | |
| 529 | − | |
| 530 | 516 | |
| 531 | 517 | static bool is_spi_bounds_valid(struct bpf_func_state *state, int spi, int nr_slots) |
| 532 | 518 | { |
| @@ -1241,11 +1227,6 @@ static bool is_stack_slot_special(const struct bpf_stack_state *stack) · 708 unmodified lines | ||
| 1241 | 1227 | /* The reg state of a pointer or a bounded scalar was saved when |
| 1242 | 1228 | * it was spilled to the stack. |
| 1243 | 1229 | */ |
| 1244 | −static bool is_spilled_scalar_reg(const struct bpf_stack_state *stack) | |
| 1245 | −{ | |
| 1246 | − return stack->slot_type[BPF_REG_SIZE - 1] == STACK_SPILL && | |
| 1247 | − stack->spilled_ptr.type == SCALAR_VALUE; | |
| 1248 | −} | |
| 1249 | 1230 | |
| 1250 | 1231 | /* |
| 1251 | 1232 | * Mark stack slot as STACK_MISC, unless it is already: |
| @@ -2590,7 +2571,6 @@ static struct bpf_retval_range retval_range(s32 minval, s32 maxval) · 1338 unmodified lines | ||
| 2590 | 2571 | return (struct bpf_retval_range){ minval, maxval, false }; |
| 2591 | 2572 | } |
| 2592 | 2573 | |
| 2593 | −#define BPF_MAIN_FUNC (-1) | |
| 2594 | 2574 | static void init_func_state(struct bpf_verifier_env *env, |
| 2595 | 2575 | struct bpf_func_state *state, |
| 2596 | 2576 | int callsite, int frameno, int subprogno) |
| @@ -3517,16 +3497,6 @@ static int insn_stack_access_flags(int frameno, int spi) · 920 unmodified lines | ||
| 3517 | 3497 | return INSN_F_STACK_ACCESS | (spi << INSN_F_SPI_SHIFT) | frameno; |
| 3518 | 3498 | } |
| 3519 | 3499 | |
| 3520 | −static int insn_stack_access_spi(int insn_flags) | |
| 3521 | −{ | |
| 3522 | − return (insn_flags >> INSN_F_SPI_SHIFT) & INSN_F_SPI_MASK; | |
| 3523 | −} | |
| 3524 | − | |
| 3525 | −static int insn_stack_access_frameno(int insn_flags) | |
| 3526 | −{ | |
| 3527 | − return insn_flags & INSN_F_FRAMENO_MASK; | |
| 3528 | −} | |
| 3529 | − | |
| 3530 | 3500 | #define LR_FRAMENO_BITS 3 |
| 3531 | 3501 | #define LR_SPI_BITS 6 |
| 3532 | 3502 | #define LR_ENTRY_BITS (LR_SPI_BITS + LR_FRAMENO_BITS + 1) |
| @@ -3605,91 +3575,6 @@ static void linked_regs_unpack(u64 val, struct linked_regs *s) · 72 unmodified lines | ||
| 3605 | 3575 | } |
| 3606 | 3576 | } |
| 3607 | 3577 | |
| 3608 | −/* for any branch, call, exit record the history of jmps in the given state */ | |
| 3609 | −int bpf_push_jmp_history(struct bpf_verifier_env *env, struct bpf_verifier_state *cur, | |
| 3610 | − int insn_flags, u64 linked_regs) | |
| 3611 | −{ | |
| 3612 | − u32 cnt = cur->jmp_history_cnt; | |
| 3613 | − struct bpf_jmp_history_entry *p; | |
| 3614 | − size_t alloc_size; | |
| 3615 | − | |
| 3616 | − /* combine instruction flags if we already recorded this instruction */ | |
| 3617 | − if (env->cur_hist_ent) { | |
| 3618 | − /* atomic instructions push insn_flags twice, for READ and | |
| 3619 | − * WRITE sides, but they should agree on stack slot | |
| 3620 | − */ | |
| 3621 | − verifier_bug_if((env->cur_hist_ent->flags & insn_flags) && | |
| 3622 | − (env->cur_hist_ent->flags & insn_flags) != insn_flags, | |
| 3623 | − env, "insn history: insn_idx %d cur flags %x new flags %x", | |
| 3624 | − env->insn_idx, env->cur_hist_ent->flags, insn_flags); | |
| 3625 | − env->cur_hist_ent->flags |= insn_flags; | |
| 3626 | − verifier_bug_if(env->cur_hist_ent->linked_regs != 0, env, | |
| 3627 | − "insn history: insn_idx %d linked_regs: %#llx", | |
| 3628 | − env->insn_idx, env->cur_hist_ent->linked_regs); | |
| 3629 | − env->cur_hist_ent->linked_regs = linked_regs; | |
| 3630 | − return 0; | |
| 3631 | − } | |
| 3632 | − | |
| 3633 | − cnt++; | |
| 3634 | − alloc_size = kmalloc_size_roundup(size_mul(cnt, sizeof(*p))); | |
| 3635 | − p = krealloc(cur->jmp_history, alloc_size, GFP_KERNEL_ACCOUNT); | |
| 3636 | − if (!p) | |
| 3637 | − return -ENOMEM; | |
| 3638 | − cur->jmp_history = p; | |
| 3639 | − | |
| 3640 | − p = &cur->jmp_history[cnt - 1]; | |
| 3641 | − p->idx = env->insn_idx; | |
| 3642 | − p->prev_idx = env->prev_insn_idx; | |
| 3643 | − p->flags = insn_flags; | |
| 3644 | − p->linked_regs = linked_regs; | |
| 3645 | − cur->jmp_history_cnt = cnt; | |
| 3646 | − env->cur_hist_ent = p; | |
| 3647 | − | |
| 3648 | − return 0; | |
| 3649 | −} | |
| 3650 | − | |
| 3651 | −static struct bpf_jmp_history_entry *get_jmp_hist_entry(struct bpf_verifier_state *st, | |
| 3652 | − u32 hist_end, int insn_idx) | |
| 3653 | −{ | |
| 3654 | − if (hist_end > 0 && st->jmp_history[hist_end - 1].idx == insn_idx) | |
| 3655 | − return &st->jmp_history[hist_end - 1]; | |
| 3656 | − return NULL; | |
| 3657 | −} | |
| 3658 | − | |
| 3659 | −/* Backtrack one insn at a time. If idx is not at the top of recorded | |
| 3660 | − * history then previous instruction came from straight line execution. | |
| 3661 | − * Return -ENOENT if we exhausted all instructions within given state. | |
| 3662 | − * | |
| 3663 | − * It's legal to have a bit of a looping with the same starting and ending | |
| 3664 | − * insn index within the same state, e.g.: 3->4->5->3, so just because current | |
| 3665 | − * instruction index is the same as state's first_idx doesn't mean we are | |
| 3666 | − * done. If there is still some jump history left, we should keep going. We | |
| 3667 | − * need to take into account that we might have a jump history between given | |
| 3668 | − * state's parent and itself, due to checkpointing. In this case, we'll have | |
| 3669 | − * history entry recording a jump from last instruction of parent state and | |
| 3670 | − * first instruction of given state. | |
| 3671 | − */ | |
| 3672 | −static int get_prev_insn_idx(struct bpf_verifier_state *st, int i, | |
| 3673 | − u32 *history) | |
| 3674 | −{ | |
| 3675 | − u32 cnt = *history; | |
| 3676 | − | |
| 3677 | − if (i == st->first_insn_idx) { | |
| 3678 | − if (cnt == 0) | |
| 3679 | − return -ENOENT; | |
| 3680 | − if (cnt == 1 && st->jmp_history[0].idx == i) | |
| 3681 | − return -ENOENT; | |
| 3682 | − } | |
| 3683 | − | |
| 3684 | − if (cnt && st->jmp_history[cnt - 1].idx == i) { | |
| 3685 | − i = st->jmp_history[cnt - 1].prev_idx; | |
| 3686 | − (*history)--; | |
| 3687 | − } else { | |
| 3688 | − i--; | |
| 3689 | − } | |
| 3690 | − return i; | |
| 3691 | −} | |
| 3692 | − | |
| 3693 | 3578 | static const char *disasm_kfunc_name(void *data, const struct bpf_insn *insn) |
| 3694 | 3579 | { |
| 3695 | 3580 | const struct btf_type *func; |
| @@ -3717,148 +3602,10 @@ void bpf_verbose_insn(struct bpf_verifier_env *env, struct bpf_insn *insn) · 21 unmodified lines | ||
| 3717 | 3602 | print_bpf_insn(&cbs, insn, env->allow_ptr_leaks); |
| 3718 | 3603 | } |
| 3719 | 3604 | |
| 3720 | −static inline void bt_init(struct backtrack_state *bt, u32 frame) | |
| 3721 | −{ | |
| 3722 | − bt->frame = frame; | |
| 3723 | −} | |
| 3724 | − | |
| 3725 | −static inline void bt_reset(struct backtrack_state *bt) | |
| 3726 | −{ | |
| 3727 | − struct bpf_verifier_env *env = bt->env; | |
| 3728 | − | |
| 3729 | − memset(bt, 0, sizeof(*bt)); | |
| 3730 | − bt->env = env; | |
| 3731 | −} | |
| 3732 | − | |
| 3733 | −static inline u32 bt_empty(struct backtrack_state *bt) | |
| 3734 | −{ | |
| 3735 | − u64 mask = 0; | |
| 3736 | − int i; | |
| 3737 | − | |
| 3738 | − for (i = 0; i <= bt->frame; i++) | |
| 3739 | − mask |= bt->reg_masks[i] | bt->stack_masks[i]; | |
| 3740 | − | |
| 3741 | − return mask == 0; | |
| 3742 | −} | |
| 3743 | − | |
| 3744 | −static inline int bt_subprog_enter(struct backtrack_state *bt) | |
| 3745 | −{ | |
| 3746 | − if (bt->frame == MAX_CALL_FRAMES - 1) { | |
| 3747 | − verifier_bug(bt->env, "subprog enter from frame %d", bt->frame); | |
| 3748 | − return -EFAULT; | |
| 3749 | − } | |
| 3750 | − bt->frame++; | |
| 3751 | − return 0; | |
| 3752 | −} | |
| 3753 | − | |
| 3754 | −static inline int bt_subprog_exit(struct backtrack_state *bt) | |
| 3755 | −{ | |
| 3756 | − if (bt->frame == 0) { | |
| 3757 | − verifier_bug(bt->env, "subprog exit from frame 0"); | |
| 3758 | − return -EFAULT; | |
| 3759 | − } | |
| 3760 | − bt->frame--; | |
| 3761 | − return 0; | |
| 3762 | −} | |
| 3763 | − | |
| 3764 | −static inline void bt_clear_frame_reg(struct backtrack_state *bt, u32 frame, u32 reg) | |
| 3765 | −{ | |
| 3766 | − bt->reg_masks[frame] &= ~(1 << reg); | |
| 3767 | −} | |
| 3768 | − | |
| 3769 | −static inline void bt_set_reg(struct backtrack_state *bt, u32 reg) | |
| 3770 | −{ | |
| 3771 | − bpf_bt_set_frame_reg(bt, bt->frame, reg); | |
| 3772 | −} | |
| 3773 | − | |
| 3774 | −static inline void bt_clear_reg(struct backtrack_state *bt, u32 reg) | |
| 3775 | −{ | |
| 3776 | − bt_clear_frame_reg(bt, bt->frame, reg); | |
| 3777 | −} | |
| 3778 | − | |
| 3779 | −static inline void bt_clear_frame_slot(struct backtrack_state *bt, u32 frame, u32 slot) | |
| 3780 | −{ | |
| 3781 | − bt->stack_masks[frame] &= ~(1ull << slot); | |
| 3782 | −} | |
| 3783 | − | |
| 3784 | −static inline u32 bt_frame_reg_mask(struct backtrack_state *bt, u32 frame) | |
| 3785 | −{ | |
| 3786 | − return bt->reg_masks[frame]; | |
| 3787 | −} | |
| 3788 | − | |
| 3789 | −static inline u32 bt_reg_mask(struct backtrack_state *bt) | |
| 3790 | −{ | |
| 3791 | − return bt->reg_masks[bt->frame]; | |
| 3792 | −} | |
| 3793 | − | |
| 3794 | −static inline u64 bt_frame_stack_mask(struct backtrack_state *bt, u32 frame) | |
| 3795 | −{ | |
| 3796 | − return bt->stack_masks[frame]; | |
| 3797 | −} | |
| 3798 | − | |
| 3799 | −static inline u64 bt_stack_mask(struct backtrack_state *bt) | |
| 3800 | −{ | |
| 3801 | − return bt->stack_masks[bt->frame]; | |
| 3802 | −} | |
| 3803 | − | |
| 3804 | −static inline bool bt_is_reg_set(struct backtrack_state *bt, u32 reg) | |
| 3805 | −{ | |
| 3806 | − return bt->reg_masks[bt->frame] & (1 << reg); | |
| 3807 | −} | |
| 3808 | − | |
| 3809 | −static inline bool bt_is_frame_reg_set(struct backtrack_state *bt, u32 frame, u32 reg) | |
| 3810 | −{ | |
| 3811 | − return bt->reg_masks[frame] & (1 << reg); | |
| 3812 | −} | |
| 3813 | − | |
| 3814 | −static inline bool bt_is_frame_slot_set(struct backtrack_state *bt, u32 frame, u32 slot) | |
| 3815 | −{ | |
| 3816 | − return bt->stack_masks[frame] & (1ull << slot); | |
| 3817 | −} | |
| 3818 | − | |
| 3819 | −/* format registers bitmask, e.g., "r0,r2,r4" for 0x15 mask */ | |
| 3820 | −static void fmt_reg_mask(char *buf, ssize_t buf_sz, u32 reg_mask) | |
| 3821 | −{ | |
| 3822 | − DECLARE_BITMAP(mask, 64); | |
| 3823 | − bool first = true; | |
| 3824 | − int i, n; | |
| 3825 | − | |
| 3826 | − buf[0] = '\0'; | |
| 3827 | − | |
| 3828 | − bitmap_from_u64(mask, reg_mask); | |
| 3829 | − for_each_set_bit(i, mask, 32) { | |
| 3830 | − n = snprintf(buf, buf_sz, "%sr%d", first ? "" : ",", i); | |
| 3831 | − first = false; | |
| 3832 | − buf += n; | |
| 3833 | − buf_sz -= n; | |
| 3834 | − if (buf_sz < 0) | |
| 3835 | − break; | |
| 3836 | − } | |
| 3837 | −} | |
| 3838 | −/* format stack slots bitmask, e.g., "-8,-24,-40" for 0x15 mask */ | |
| 3839 | −void bpf_fmt_stack_mask(char *buf, ssize_t buf_sz, u64 stack_mask) | |
| 3840 | −{ | |
| 3841 | − DECLARE_BITMAP(mask, 64); | |
| 3842 | − bool first = true; | |
| 3843 | − int i, n; | |
| 3844 | − | |
| 3845 | − buf[0] = '\0'; | |
| 3846 | − | |
| 3847 | − bitmap_from_u64(mask, stack_mask); | |
| 3848 | − for_each_set_bit(i, mask, 64) { | |
| 3849 | − n = snprintf(buf, buf_sz, "%s%d", first ? "" : ",", -(i + 1) * 8); | |
| 3850 | − first = false; | |
| 3851 | − buf += n; | |
| 3852 | − buf_sz -= n; | |
| 3853 | − if (buf_sz < 0) | |
| 3854 | − break; | |
| 3855 | − } | |
| 3856 | −} | |
| 3857 | − | |
| 3858 | 3605 | /* If any register R in hist->linked_regs is marked as precise in bt, |
| 3859 | 3606 | * do bt_set_frame_{reg,slot}(bt, R) for all registers in hist->linked_regs. |
| 3860 | 3607 | */ |
| 3861 | −static void bt_sync_linked_regs(struct backtrack_state *bt, struct bpf_jmp_history_entry *hist) | |
| 3608 | +void bpf_bt_sync_linked_regs(struct backtrack_state *bt, struct bpf_jmp_history_entry *hist) | |
| 3862 | 3609 | { |
| 3863 | 3610 | struct linked_regs linked_regs; |
| 3864 | 3611 | bool some_precise = false; |
| @@ -3891,693 +3638,6 @@ static void bt_sync_linked_regs(struct backtrack_state *bt, struct bpf_jmp_histo · 26 unmodified lines | ||
| 3891 | 3638 | } |
| 3892 | 3639 | } |
| 3893 | 3640 | |
| 3894 | −/* For given verifier state backtrack_insn() is called from the last insn to | |
| 3895 | − * the first insn. Its purpose is to compute a bitmask of registers and | |
| 3896 | − * stack slots that needs precision in the parent verifier state. | |
| 3897 | − * | |
| 3898 | − * @idx is an index of the instruction we are currently processing; | |
| 3899 | − * @subseq_idx is an index of the subsequent instruction that: | |
| 3900 | − * - *would be* executed next, if jump history is viewed in forward order; | |
| 3901 | − * - *was* processed previously during backtracking. | |
| 3902 | − */ | |
| 3903 | −static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, | |
| 3904 | − struct bpf_jmp_history_entry *hist, struct backtrack_state *bt) | |
| 3905 | −{ | |
| 3906 | − struct bpf_insn *insn = env->prog->insnsi + idx; | |
| 3907 | − u8 class = BPF_CLASS(insn->code); | |
| 3908 | − u8 opcode = BPF_OP(insn->code); | |
| 3909 | − u8 mode = BPF_MODE(insn->code); | |
| 3910 | − u32 dreg = insn->dst_reg; | |
| 3911 | − u32 sreg = insn->src_reg; | |
| 3912 | − u32 spi, i, fr; | |
| 3913 | − | |
| 3914 | − if (insn->code == 0) | |
| 3915 | − return 0; | |
| 3916 | − if (env->log.level & BPF_LOG_LEVEL2) { | |
| 3917 | − fmt_reg_mask(env->tmp_str_buf, TMP_STR_BUF_LEN, bt_reg_mask(bt)); | |
| 3918 | − verbose(env, "mark_precise: frame%d: regs=%s ", | |
| 3919 | − bt->frame, env->tmp_str_buf); | |
| 3920 | − bpf_fmt_stack_mask(env->tmp_str_buf, TMP_STR_BUF_LEN, bt_stack_mask(bt)); | |
| 3921 | − verbose(env, "stack=%s before ", env->tmp_str_buf); | |
| 3922 | − verbose(env, "%d: ", idx); | |
| 3923 | − bpf_verbose_insn(env, insn); | |
| 3924 | − } | |
| 3925 | − | |
| 3926 | − /* If there is a history record that some registers gained range at this insn, | |
| 3927 | − * propagate precision marks to those registers, so that bt_is_reg_set() | |
| 3928 | − * accounts for these registers. | |
| 3929 | − */ | |
| 3930 | − bt_sync_linked_regs(bt, hist); | |
| 3931 | − | |
| 3932 | − if (class == BPF_ALU || class == BPF_ALU64) { | |
| 3933 | − if (!bt_is_reg_set(bt, dreg)) | |
| 3934 | − return 0; | |
| 3935 | − if (opcode == BPF_END || opcode == BPF_NEG) { | |
| 3936 | − /* sreg is reserved and unused | |
| 3937 | − * dreg still need precision before this insn | |
| 3938 | − */ | |
| 3939 | − return 0; | |
| 3940 | − } else if (opcode == BPF_MOV) { | |
| 3941 | − if (BPF_SRC(insn->code) == BPF_X) { | |
| 3942 | − /* dreg = sreg or dreg = (s8, s16, s32)sreg | |
| 3943 | − * dreg needs precision after this insn | |
| 3944 | − * sreg needs precision before this insn | |
| 3945 | − */ | |
| 3946 | − bt_clear_reg(bt, dreg); | |
| 3947 | − if (sreg != BPF_REG_FP) | |
| 3948 | − bt_set_reg(bt, sreg); | |
| 3949 | − } else { | |
| 3950 | − /* dreg = K | |
| 3951 | − * dreg needs precision after this insn. | |
| 3952 | − * Corresponding register is already marked | |
| 3953 | − * as precise=true in this verifier state. | |
| 3954 | − * No further markings in parent are necessary | |
| 3955 | − */ | |
| 3956 | − bt_clear_reg(bt, dreg); | |
| 3957 | − } | |
| 3958 | − } else { | |
| 3959 | − if (BPF_SRC(insn->code) == BPF_X) { | |
| 3960 | − /* dreg += sreg | |
| 3961 | − * both dreg and sreg need precision | |
| 3962 | − * before this insn | |
| 3963 | − */ | |
| 3964 | − if (sreg != BPF_REG_FP) | |
| 3965 | − bt_set_reg(bt, sreg); | |
| 3966 | − } /* else dreg += K | |
| 3967 | − * dreg still needs precision before this insn | |
| 3968 | − */ | |
| 3969 | − } | |
| 3970 | − } else if (class == BPF_LDX || | |
| 3971 | − is_atomic_load_insn(insn) || | |
| 3972 | − is_atomic_fetch_insn(insn)) { | |
| 3973 | − u32 load_reg = dreg; | |
| 3974 | − | |
| 3975 | − /* | |
| 3976 | − * Atomic fetch operation writes the old value into | |
| 3977 | − * a register (sreg or r0) and if it was tracked for | |
| 3978 | − * precision, propagate to the stack slot like we do | |
| 3979 | − * in regular ldx. | |
| 3980 | − */ | |
| 3981 | − if (is_atomic_fetch_insn(insn)) | |
| 3982 | − load_reg = insn->imm == BPF_CMPXCHG ? | |
| 3983 | − BPF_REG_0 : sreg; | |
| 3984 | − | |
| 3985 | − if (!bt_is_reg_set(bt, load_reg)) | |
| 3986 | − return 0; | |
| 3987 | − bt_clear_reg(bt, load_reg); | |
| 3988 | − | |
| 3989 | − /* scalars can only be spilled into stack w/o losing precision. | |
| 3990 | − * Load from any other memory can be zero extended. | |
| 3991 | − * The desire to keep that precision is already indicated | |
| 3992 | − * by 'precise' mark in corresponding register of this state. | |
| 3993 | − * No further tracking necessary. | |
| 3994 | − */ | |
| 3995 | − if (!hist || !(hist->flags & INSN_F_STACK_ACCESS)) | |
| 3996 | − return 0; | |
| 3997 | − /* dreg = *(u64 *)[fp - off] was a fill from the stack. | |
| 3998 | − * that [fp - off] slot contains scalar that needs to be | |
| 3999 | − * tracked with precision | |
| 4000 | − */ | |
| 4001 | − spi = insn_stack_access_spi(hist->flags); | |
| 4002 | − fr = insn_stack_access_frameno(hist->flags); | |
| 4003 | − bpf_bt_set_frame_slot(bt, fr, spi); | |
| 4004 | − } else if (class == BPF_STX || class == BPF_ST) { | |
| 4005 | − if (bt_is_reg_set(bt, dreg)) | |
| 4006 | − /* stx & st shouldn't be using _scalar_ dst_reg | |
| 4007 | − * to access memory. It means backtracking | |
| 4008 | − * encountered a case of pointer subtraction. | |
| 4009 | − */ | |
| 4010 | − return -ENOTSUPP; | |
| 4011 | − /* scalars can only be spilled into stack */ | |
| 4012 | − if (!hist || !(hist->flags & INSN_F_STACK_ACCESS)) | |
| 4013 | − return 0; | |
| 4014 | − spi = insn_stack_access_spi(hist->flags); | |
| 4015 | − fr = insn_stack_access_frameno(hist->flags); | |
| 4016 | − if (!bt_is_frame_slot_set(bt, fr, spi)) | |
| 4017 | − return 0; | |
| 4018 | − bt_clear_frame_slot(bt, fr, spi); | |
| 4019 | − if (class == BPF_STX) | |
| 4020 | − bt_set_reg(bt, sreg); | |
| 4021 | − } else if (class == BPF_JMP || class == BPF_JMP32) { | |
| 4022 | − if (bpf_pseudo_call(insn)) { | |
| 4023 | − int subprog_insn_idx, subprog; | |
| 4024 | − | |
| 4025 | − subprog_insn_idx = idx + insn->imm + 1; | |
| 4026 | − subprog = bpf_find_subprog(env, subprog_insn_idx); | |
| 4027 | − if (subprog < 0) | |
| 4028 | − return -EFAULT; | |
| 4029 | − | |
| 4030 | − if (bpf_subprog_is_global(env, subprog)) { | |
| 4031 | − /* check that jump history doesn't have any | |
| 4032 | − * extra instructions from subprog; the next | |
| 4033 | − * instruction after call to global subprog | |
| 4034 | − * should be literally next instruction in | |
| 4035 | − * caller program | |
| 4036 | − */ | |
| 4037 | − verifier_bug_if(idx + 1 != subseq_idx, env, | |
| 4038 | − "extra insn from subprog"); | |
| 4039 | − /* r1-r5 are invalidated after subprog call, | |
| 4040 | − * so for global func call it shouldn't be set | |
| 4041 | − * anymore | |
| 4042 | − */ | |
| 4043 | − if (bt_reg_mask(bt) & BPF_REGMASK_ARGS) { | |
| 4044 | − verifier_bug(env, "global subprog unexpected regs %x", | |
| 4045 | − bt_reg_mask(bt)); | |
| 4046 | − return -EFAULT; | |
| 4047 | − } | |
| 4048 | − /* global subprog always sets R0 */ | |
| 4049 | − bt_clear_reg(bt, BPF_REG_0); | |
| 4050 | − return 0; | |
| 4051 | − } else { | |
| 4052 | − /* static subprog call instruction, which | |
| 4053 | − * means that we are exiting current subprog, | |
| 4054 | − * so only r1-r5 could be still requested as | |
| 4055 | − * precise, r0 and r6-r10 or any stack slot in | |
| 4056 | − * the current frame should be zero by now | |
| 4057 | − */ | |
| 4058 | − if (bt_reg_mask(bt) & ~BPF_REGMASK_ARGS) { | |
| 4059 | − verifier_bug(env, "static subprog unexpected regs %x", | |
| 4060 | − bt_reg_mask(bt)); | |
| 4061 | − return -EFAULT; | |
| 4062 | − } | |
| 4063 | − /* we are now tracking register spills correctly, | |
| 4064 | − * so any instance of leftover slots is a bug | |
| 4065 | − */ | |
| 4066 | − if (bt_stack_mask(bt) != 0) { | |
| 4067 | − verifier_bug(env, | |
| 4068 | − "static subprog leftover stack slots %llx", | |
| 4069 | − bt_stack_mask(bt)); | |
| 4070 | − return -EFAULT; | |
| 4071 | − } | |
| 4072 | − /* propagate r1-r5 to the caller */ | |
| 4073 | − for (i = BPF_REG_1; i <= BPF_REG_5; i++) { | |
| 4074 | − if (bt_is_reg_set(bt, i)) { | |
| 4075 | − bt_clear_reg(bt, i); | |
| 4076 | − bpf_bt_set_frame_reg(bt, bt->frame - 1, i); | |
| 4077 | − } | |
| 4078 | − } | |
| 4079 | − if (bt_subprog_exit(bt)) | |
| 4080 | − return -EFAULT; | |
| 4081 | − return 0; | |
| 4082 | − } | |
| 4083 | − } else if (bpf_is_sync_callback_calling_insn(insn) && idx != subseq_idx - 1) { | |
| 4084 | − /* exit from callback subprog to callback-calling helper or | |
| 4085 | − * kfunc call. Use idx/subseq_idx check to discern it from | |
| 4086 | − * straight line code backtracking. | |
| 4087 | − * Unlike the subprog call handling above, we shouldn't | |
| 4088 | − * propagate precision of r1-r5 (if any requested), as they are | |
| 4089 | − * not actually arguments passed directly to callback subprogs | |
| 4090 | − */ | |
| 4091 | − if (bt_reg_mask(bt) & ~BPF_REGMASK_ARGS) { | |
| 4092 | − verifier_bug(env, "callback unexpected regs %x", | |
| 4093 | − bt_reg_mask(bt)); | |
| 4094 | − return -EFAULT; | |
| 4095 | − } | |
| 4096 | − if (bt_stack_mask(bt) != 0) { | |
| 4097 | − verifier_bug(env, "callback leftover stack slots %llx", | |
| 4098 | − bt_stack_mask(bt)); | |
| 4099 | − return -EFAULT; | |
| 4100 | − } | |
| 4101 | − /* clear r1-r5 in callback subprog's mask */ | |
| 4102 | − for (i = BPF_REG_1; i <= BPF_REG_5; i++) | |
| 4103 | − bt_clear_reg(bt, i); | |
| 4104 | − if (bt_subprog_exit(bt)) | |
| 4105 | − return -EFAULT; | |
| 4106 | − return 0; | |
| 4107 | − } else if (opcode == BPF_CALL) { | |
| 4108 | − /* kfunc with imm==0 is invalid and fixup_kfunc_call will | |
| 4109 | − * catch this error later. Make backtracking conservative | |
| 4110 | − * with ENOTSUPP. | |
| 4111 | − */ | |
| 4112 | − if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL && insn->imm == 0) | |
| 4113 | − return -ENOTSUPP; | |
| 4114 | − /* regular helper call sets R0 */ | |
| 4115 | − bt_clear_reg(bt, BPF_REG_0); | |
| 4116 | − if (bt_reg_mask(bt) & BPF_REGMASK_ARGS) { | |
| 4117 | − /* if backtracking was looking for registers R1-R5 | |
| 4118 | − * they should have been found already. | |
| 4119 | − */ | |
| 4120 | − verifier_bug(env, "backtracking call unexpected regs %x", | |
| 4121 | − bt_reg_mask(bt)); | |
| 4122 | − return -EFAULT; | |
| 4123 | − } | |
| 4124 | − if (insn->src_reg == BPF_REG_0 && insn->imm == BPF_FUNC_tail_call | |
| 4125 | − && subseq_idx - idx != 1) { | |
| 4126 | − if (bt_subprog_enter(bt)) | |
| 4127 | − return -EFAULT; | |
| 4128 | − } | |
| 4129 | − } else if (opcode == BPF_EXIT) { | |
| 4130 | − bool r0_precise; | |
| 4131 | − | |
| 4132 | − /* Backtracking to a nested function call, 'idx' is a part of | |
| 4133 | − * the inner frame 'subseq_idx' is a part of the outer frame. | |
| 4134 | − * In case of a regular function call, instructions giving | |
| 4135 | − * precision to registers R1-R5 should have been found already. | |
| 4136 | − * In case of a callback, it is ok to have R1-R5 marked for | |
| 4137 | − * backtracking, as these registers are set by the function | |
| 4138 | − * invoking callback. | |
| 4139 | − */ | |
| 4140 | − if (subseq_idx >= 0 && bpf_calls_callback(env, subseq_idx)) | |
| 4141 | − for (i = BPF_REG_1; i <= BPF_REG_5; i++) | |
| 4142 | − bt_clear_reg(bt, i); | |
| 4143 | − if (bt_reg_mask(bt) & BPF_REGMASK_ARGS) { | |
| 4144 | − verifier_bug(env, "backtracking exit unexpected regs %x", | |
| 4145 | − bt_reg_mask(bt)); | |
| 4146 | − return -EFAULT; | |
| 4147 | − } | |
| 4148 | − | |
| 4149 | − /* BPF_EXIT in subprog or callback always returns | |
| 4150 | − * right after the call instruction, so by checking | |
| 4151 | − * whether the instruction at subseq_idx-1 is subprog | |
| 4152 | − * call or not we can distinguish actual exit from | |
| 4153 | − * *subprog* from exit from *callback*. In the former | |
| 4154 | − * case, we need to propagate r0 precision, if | |
| 4155 | − * necessary. In the former we never do that. | |
| 4156 | − */ | |
| 4157 | − r0_precise = subseq_idx - 1 >= 0 && | |
| 4158 | − bpf_pseudo_call(&env->prog->insnsi[subseq_idx - 1]) && | |
| 4159 | − bt_is_reg_set(bt, BPF_REG_0); | |
| 4160 | − | |
| 4161 | − bt_clear_reg(bt, BPF_REG_0); | |
| 4162 | − if (bt_subprog_enter(bt)) | |
| 4163 | − return -EFAULT; | |
| 4164 | − | |
| 4165 | − if (r0_precise) | |
| 4166 | − bt_set_reg(bt, BPF_REG_0); | |
| 4167 | − /* r6-r9 and stack slots will stay set in caller frame | |
| 4168 | − * bitmasks until we return back from callee(s) | |
| 4169 | − */ | |
| 4170 | − return 0; | |
| 4171 | − } else if (BPF_SRC(insn->code) == BPF_X) { | |
| 4172 | − if (!bt_is_reg_set(bt, dreg) && !bt_is_reg_set(bt, sreg)) | |
| 4173 | − return 0; | |
| 4174 | − /* dreg <cond> sreg | |
| 4175 | − * Both dreg and sreg need precision before | |
| 4176 | − * this insn. If only sreg was marked precise | |
| 4177 | − * before it would be equally necessary to | |
| 4178 | − * propagate it to dreg. | |
| 4179 | − */ | |
| 4180 | − if (!hist || !(hist->flags & INSN_F_SRC_REG_STACK)) | |
| 4181 | − bt_set_reg(bt, sreg); | |
| 4182 | − if (!hist || !(hist->flags & INSN_F_DST_REG_STACK)) | |
| 4183 | − bt_set_reg(bt, dreg); | |
| 4184 | − } else if (BPF_SRC(insn->code) == BPF_K) { | |
| 4185 | − /* dreg <cond> K | |
| 4186 | − * Only dreg still needs precision before | |
| 4187 | − * this insn, so for the K-based conditional | |
| 4188 | − * there is nothing new to be marked. | |
| 4189 | − */ | |
| 4190 | − } | |
| 4191 | − } else if (class == BPF_LD) { | |
| 4192 | − if (!bt_is_reg_set(bt, dreg)) | |
| 4193 | − return 0; | |
| 4194 | − bt_clear_reg(bt, dreg); | |
| 4195 | − /* It's ld_imm64 or ld_abs or ld_ind. | |
| 4196 | − * For ld_imm64 no further tracking of precision | |
| 4197 | − * into parent is necessary | |
| 4198 | − */ | |
| 4199 | − if (mode == BPF_IND || mode == BPF_ABS) | |
| 4200 | − /* to be analyzed */ | |
| 4201 | − return -ENOTSUPP; | |
| 4202 | − } | |
| 4203 | − /* Propagate precision marks to linked registers, to account for | |
| 4204 | − * registers marked as precise in this function. | |
| 4205 | − */ | |
| 4206 | − bt_sync_linked_regs(bt, hist); | |
| 4207 | − return 0; | |
| 4208 | −} | |
| 4209 | − | |
| 4210 | −/* the scalar precision tracking algorithm: | |
| 4211 | − * . at the start all registers have precise=false. | |
| 4212 | − * . scalar ranges are tracked as normal through alu and jmp insns. | |
| 4213 | − * . once precise value of the scalar register is used in: | |
| 4214 | − * . ptr + scalar alu | |
| 4215 | − * . if (scalar cond K|scalar) | |
| 4216 | − * . helper_call(.., scalar, ...) where ARG_CONST is expected | |
| 4217 | − * backtrack through the verifier states and mark all registers and | |
| 4218 | − * stack slots with spilled constants that these scalar registers | |
| 4219 | − * should be precise. | |
| 4220 | − * . during state pruning two registers (or spilled stack slots) | |
| 4221 | − * are equivalent if both are not precise. | |
| 4222 | − * | |
| 4223 | − * Note the verifier cannot simply walk register parentage chain, | |
| 4224 | − * since many different registers and stack slots could have been | |
| 4225 | − * used to compute single precise scalar. | |
| 4226 | − * | |
| 4227 | − * The approach of starting with precise=true for all registers and then | |
| 4228 | − * backtrack to mark a register as not precise when the verifier detects | |
| 4229 | − * that program doesn't care about specific value (e.g., when helper | |
| 4230 | − * takes register as ARG_ANYTHING parameter) is not safe. | |
| 4231 | − * | |
| 4232 | − * It's ok to walk single parentage chain of the verifier states. | |
| 4233 | − * It's possible that this backtracking will go all the way till 1st insn. | |
| 4234 | − * All other branches will be explored for needing precision later. | |
| 4235 | − * | |
| 4236 | − * The backtracking needs to deal with cases like: | |
| 4237 | − * R8=map_value(id=0,off=0,ks=4,vs=1952,imm=0) R9_w=map_value(id=0,off=40,ks=4,vs=1952,imm=0) | |
| 4238 | − * r9 -= r8 | |
| 4239 | − * r5 = r9 | |
| 4240 | − * if r5 > 0x79f goto pc+7 | |
| 4241 | − * R5_w=inv(id=0,umax_value=1951,var_off=(0x0; 0x7ff)) | |
| 4242 | − * r5 += 1 | |
| 4243 | − * ... | |
| 4244 | − * call bpf_perf_event_output#25 | |
| 4245 | − * where .arg5_type = ARG_CONST_SIZE_OR_ZERO | |
| 4246 | − * | |
| 4247 | − * and this case: | |
| 4248 | − * r6 = 1 | |
| 4249 | − * call foo // uses callee's r6 inside to compute r0 | |
| 4250 | − * r0 += r6 | |
| 4251 | − * if r0 == 0 goto | |
| 4252 | − * | |
| 4253 | − * to track above reg_mask/stack_mask needs to be independent for each frame. | |
| 4254 | − * | |
| 4255 | − * Also if parent's curframe > frame where backtracking started, | |
| 4256 | − * the verifier need to mark registers in both frames, otherwise callees | |
| 4257 | − * may incorrectly prune callers. This is similar to | |
| 4258 | − * commit 7640ead93924 ("bpf: verifier: make sure callees don't prune with caller differences") | |
| 4259 | − * | |
| 4260 | − * For now backtracking falls back into conservative marking. | |
| 4261 | − */ | |
| 4262 | −void bpf_mark_all_scalars_precise(struct bpf_verifier_env *env, | |
| 4263 | − struct bpf_verifier_state *st) | |
| 4264 | −{ | |
| 4265 | − struct bpf_func_state *func; | |
| 4266 | − struct bpf_reg_state *reg; | |
| 4267 | − int i, j; | |
| 4268 | − | |
| 4269 | − if (env->log.level & BPF_LOG_LEVEL2) { | |
| 4270 | − verbose(env, "mark_precise: frame%d: falling back to forcing all scalars precise\n", | |
| 4271 | − st->curframe); | |
| 4272 | − } | |
| 4273 | − | |
| 4274 | − /* big hammer: mark all scalars precise in this path. | |
| 4275 | − * pop_stack may still get !precise scalars. | |
| 4276 | − * We also skip current state and go straight to first parent state, | |
| 4277 | − * because precision markings in current non-checkpointed state are | |
| 4278 | − * not needed. See why in the comment in __mark_chain_precision below. | |
| 4279 | − */ | |
| 4280 | − for (st = st->parent; st; st = st->parent) { | |
| 4281 | − for (i = 0; i <= st->curframe; i++) { | |
| 4282 | − func = st->frame[i]; | |
| 4283 | − for (j = 0; j < BPF_REG_FP; j++) { | |
| 4284 | − reg = &func->regs[j]; | |
| 4285 | − if (reg->type != SCALAR_VALUE || reg->precise) | |
| 4286 | − continue; | |
| 4287 | − reg->precise = true; | |
| 4288 | − if (env->log.level & BPF_LOG_LEVEL2) { | |
| 4289 | − verbose(env, "force_precise: frame%d: forcing r%d to be precise\n", | |
| 4290 | − i, j); | |
| 4291 | − } | |
| 4292 | − } | |
| 4293 | − for (j = 0; j < func->allocated_stack / BPF_REG_SIZE; j++) { | |
| 4294 | − if (!bpf_is_spilled_reg(&func->stack[j])) | |
| 4295 | − continue; | |
| 4296 | − reg = &func->stack[j].spilled_ptr; | |
| 4297 | − if (reg->type != SCALAR_VALUE || reg->precise) | |
| 4298 | − continue; | |
| 4299 | − reg->precise = true; | |
| 4300 | − if (env->log.level & BPF_LOG_LEVEL2) { | |
| 4301 | − verbose(env, "force_precise: frame%d: forcing fp%d to be precise\n", | |
| 4302 | − i, -(j + 1) * 8); | |
| 4303 | − } | |
| 4304 | − } | |
| 4305 | − } | |
| 4306 | − } | |
| 4307 | −} | |
| 4308 | − | |
| 4309 | −/* | |
| 4310 | − * bpf_mark_chain_precision() backtracks BPF program instruction sequence and | |
| 4311 | − * chain of verifier states making sure that register *regno* (if regno >= 0) | |
| 4312 | − * and/or stack slot *spi* (if spi >= 0) are marked as precisely tracked | |
| 4313 | − * SCALARS, as well as any other registers and slots that contribute to | |
| 4314 | − * a tracked state of given registers/stack slots, depending on specific BPF | |
| 4315 | − * assembly instructions (see backtrack_insns() for exact instruction handling | |
| 4316 | − * logic). This backtracking relies on recorded jmp_history and is able to | |
| 4317 | − * traverse entire chain of parent states. This process ends only when all the | |
| 4318 | − * necessary registers/slots and their transitive dependencies are marked as | |
| 4319 | − * precise. | |
| 4320 | − * | |
| 4321 | − * One important and subtle aspect is that precise marks *do not matter* in | |
| 4322 | − * the currently verified state (current state). It is important to understand | |
| 4323 | − * why this is the case. | |
| 4324 | − * | |
| 4325 | − * First, note that current state is the state that is not yet "checkpointed", | |
| 4326 | − * i.e., it is not yet put into env->explored_states, and it has no children | |
| 4327 | − * states as well. It's ephemeral, and can end up either a) being discarded if | |
| 4328 | − * compatible explored state is found at some point or BPF_EXIT instruction is | |
| 4329 | − * reached or b) checkpointed and put into env->explored_states, branching out | |
| 4330 | − * into one or more children states. | |
| 4331 | − * | |
| 4332 | − * In the former case, precise markings in current state are completely | |
| 4333 | − * ignored by state comparison code (see regsafe() for details). Only | |
| 4334 | − * checkpointed ("old") state precise markings are important, and if old | |
| 4335 | − * state's register/slot is precise, regsafe() assumes current state's | |
| 4336 | − * register/slot as precise and checks value ranges exactly and precisely. If | |
| 4337 | − * states turn out to be compatible, current state's necessary precise | |
| 4338 | − * markings and any required parent states' precise markings are enforced | |
| 4339 | − * after the fact with propagate_precision() logic, after the fact. But it's | |
| 4340 | − * important to realize that in this case, even after marking current state | |
| 4341 | − * registers/slots as precise, we immediately discard current state. So what | |
| 4342 | − * actually matters is any of the precise markings propagated into current | |
| 4343 | − * state's parent states, which are always checkpointed (due to b) case above). | |
| 4344 | − * As such, for scenario a) it doesn't matter if current state has precise | |
| 4345 | − * markings set or not. | |
| 4346 | − * | |
| 4347 | − * Now, for the scenario b), checkpointing and forking into child(ren) | |
| 4348 | − * state(s). Note that before current state gets to checkpointing step, any | |
| 4349 | − * processed instruction always assumes precise SCALAR register/slot | |
| 4350 | − * knowledge: if precise value or range is useful to prune jump branch, BPF | |
| 4351 | − * verifier takes this opportunity enthusiastically. Similarly, when | |
| 4352 | − * register's value is used to calculate offset or memory address, exact | |
| 4353 | − * knowledge of SCALAR range is assumed, checked, and enforced. So, similar to | |
| 4354 | − * what we mentioned above about state comparison ignoring precise markings | |
| 4355 | − * during state comparison, BPF verifier ignores and also assumes precise | |
| 4356 | − * markings *at will* during instruction verification process. But as verifier | |
| 4357 | − * assumes precision, it also propagates any precision dependencies across | |
| 4358 | − * parent states, which are not yet finalized, so can be further restricted | |
| 4359 | − * based on new knowledge gained from restrictions enforced by their children | |
| 4360 | − * states. This is so that once those parent states are finalized, i.e., when | |
| 4361 | − * they have no more active children state, state comparison logic in | |
| 4362 | − * is_state_visited() would enforce strict and precise SCALAR ranges, if | |
| 4363 | − * required for correctness. | |
| 4364 | − * | |
| 4365 | − * To build a bit more intuition, note also that once a state is checkpointed, | |
| 4366 | − * the path we took to get to that state is not important. This is crucial | |
| 4367 | − * property for state pruning. When state is checkpointed and finalized at | |
| 4368 | − * some instruction index, it can be correctly and safely used to "short | |
| 4369 | − * circuit" any *compatible* state that reaches exactly the same instruction | |
| 4370 | − * index. I.e., if we jumped to that instruction from a completely different | |
| 4371 | − * code path than original finalized state was derived from, it doesn't | |
| 4372 | − * matter, current state can be discarded because from that instruction | |
| 4373 | − * forward having a compatible state will ensure we will safely reach the | |
| 4374 | − * exit. States describe preconditions for further exploration, but completely | |
| 4375 | − * forget the history of how we got here. | |
| 4376 | − * | |
| 4377 | − * This also means that even if we needed precise SCALAR range to get to | |
| 4378 | − * finalized state, but from that point forward *that same* SCALAR register is | |
| 4379 | − * never used in a precise context (i.e., it's precise value is not needed for | |
| 4380 | − * correctness), it's correct and safe to mark such register as "imprecise" | |
| 4381 | − * (i.e., precise marking set to false). This is what we rely on when we do | |
| 4382 | − * not set precise marking in current state. If no child state requires | |
| 4383 | − * precision for any given SCALAR register, it's safe to dictate that it can | |
| 4384 | − * be imprecise. If any child state does require this register to be precise, | |
| 4385 | − * we'll mark it precise later retroactively during precise markings | |
| 4386 | − * propagation from child state to parent states. | |
| 4387 | − * | |
| 4388 | − * Skipping precise marking setting in current state is a mild version of | |
| 4389 | − * relying on the above observation. But we can utilize this property even | |
| 4390 | − * more aggressively by proactively forgetting any precise marking in the | |
| 4391 | − * current state (which we inherited from the parent state), right before we | |
| 4392 | − * checkpoint it and branch off into new child state. This is done by | |
| 4393 | − * mark_all_scalars_imprecise() to hopefully get more permissive and generic | |
| 4394 | − * finalized states which help in short circuiting more future states. | |
| 4395 | − */ | |
| 4396 | −int bpf_mark_chain_precision(struct bpf_verifier_env *env, | |
| 4397 | − struct bpf_verifier_state *starting_state, | |
| 4398 | − int regno, | |
| 4399 | − bool *changed) | |
| 4400 | −{ | |
| 4401 | − struct bpf_verifier_state *st = starting_state; | |
| 4402 | − struct backtrack_state *bt = &env->bt; | |
| 4403 | − int first_idx = st->first_insn_idx; | |
| 4404 | − int last_idx = starting_state->insn_idx; | |
| 4405 | − int subseq_idx = -1; | |
| 4406 | − struct bpf_func_state *func; | |
| 4407 | − bool tmp, skip_first = true; | |
| 4408 | − struct bpf_reg_state *reg; | |
| 4409 | − int i, fr, err; | |
| 4410 | − | |
| 4411 | − if (!env->bpf_capable) | |
| 4412 | − return 0; | |
| 4413 | − | |
| 4414 | − changed = changed ?: &tmp; | |
| 4415 | − /* set frame number from which we are starting to backtrack */ | |
| 4416 | − bt_init(bt, starting_state->curframe); | |
| 4417 | − | |
| 4418 | − /* Do sanity checks against current state of register and/or stack | |
| 4419 | − * slot, but don't set precise flag in current state, as precision | |
| 4420 | − * tracking in the current state is unnecessary. | |
| 4421 | − */ | |
| 4422 | − func = st->frame[bt->frame]; | |
| 4423 | − if (regno >= 0) { | |
| 4424 | − reg = &func->regs[regno]; | |
| 4425 | − if (reg->type != SCALAR_VALUE) { | |
| 4426 | − verifier_bug(env, "backtracking misuse"); | |
| 4427 | − return -EFAULT; | |
| 4428 | − } | |
| 4429 | − bt_set_reg(bt, regno); | |
| 4430 | − } | |
| 4431 | − | |
| 4432 | − if (bt_empty(bt)) | |
| 4433 | − return 0; | |
| 4434 | − | |
| 4435 | − for (;;) { | |
| 4436 | − DECLARE_BITMAP(mask, 64); | |
| 4437 | − u32 history = st->jmp_history_cnt; | |
| 4438 | − struct bpf_jmp_history_entry *hist; | |
| 4439 | − | |
| 4440 | − if (env->log.level & BPF_LOG_LEVEL2) { | |
| 4441 | − verbose(env, "mark_precise: frame%d: last_idx %d first_idx %d subseq_idx %d \n", | |
| 4442 | − bt->frame, last_idx, first_idx, subseq_idx); | |
| 4443 | − } | |
| 4444 | − | |
| 4445 | − if (last_idx < 0) { | |
| 4446 | − /* we are at the entry into subprog, which | |
| 4447 | − * is expected for global funcs, but only if | |
| 4448 | − * requested precise registers are R1-R5 | |
| 4449 | − * (which are global func's input arguments) | |
| 4450 | − */ | |
| 4451 | − if (st->curframe == 0 && | |
| 4452 | − st->frame[0]->subprogno > 0 && | |
| 4453 | − st->frame[0]->callsite == BPF_MAIN_FUNC && | |
| 4454 | − bt_stack_mask(bt) == 0 && | |
| 4455 | − (bt_reg_mask(bt) & ~BPF_REGMASK_ARGS) == 0) { | |
| 4456 | − bitmap_from_u64(mask, bt_reg_mask(bt)); | |
| 4457 | − for_each_set_bit(i, mask, 32) { | |
| 4458 | − reg = &st->frame[0]->regs[i]; | |
| 4459 | − bt_clear_reg(bt, i); | |
| 4460 | − if (reg->type == SCALAR_VALUE) { | |
| 4461 | − reg->precise = true; | |
| 4462 | − *changed = true; | |
| 4463 | − } | |
| 4464 | − } | |
| 4465 | − return 0; | |
| 4466 | − } | |
| 4467 | − | |
| 4468 | − verifier_bug(env, "backtracking func entry subprog %d reg_mask %x stack_mask %llx", | |
| 4469 | − st->frame[0]->subprogno, bt_reg_mask(bt), bt_stack_mask(bt)); | |
| 4470 | − return -EFAULT; | |
| 4471 | − } | |
| 4472 | − | |
| 4473 | − for (i = last_idx;;) { | |
| 4474 | − if (skip_first) { | |
| 4475 | − err = 0; | |
| 4476 | − skip_first = false; | |
| 4477 | − } else { | |
| 4478 | − hist = get_jmp_hist_entry(st, history, i); | |
| 4479 | − err = backtrack_insn(env, i, subseq_idx, hist, bt); | |
| 4480 | − } | |
| 4481 | − if (err == -ENOTSUPP) { | |
| 4482 | − bpf_mark_all_scalars_precise(env, starting_state); | |
| 4483 | − bt_reset(bt); | |
| 4484 | − return 0; | |
| 4485 | − } else if (err) { | |
| 4486 | − return err; | |
| 4487 | − } | |
| 4488 | − if (bt_empty(bt)) | |
| 4489 | − /* Found assignment(s) into tracked register in this state. | |
| 4490 | − * Since this state is already marked, just return. | |
| 4491 | − * Nothing to be tracked further in the parent state. | |
| 4492 | − */ | |
| 4493 | − return 0; | |
| 4494 | − subseq_idx = i; | |
| 4495 | − i = get_prev_insn_idx(st, i, &history); | |
| 4496 | − if (i == -ENOENT) | |
| 4497 | − break; | |
| 4498 | − if (i >= env->prog->len) { | |
| 4499 | − /* This can happen if backtracking reached insn 0 | |
| 4500 | − * and there are still reg_mask or stack_mask | |
| 4501 | − * to backtrack. | |
| 4502 | − * It means the backtracking missed the spot where | |
| 4503 | − * particular register was initialized with a constant. | |
| 4504 | − */ | |
| 4505 | − verifier_bug(env, "backtracking idx %d", i); | |
| 4506 | − return -EFAULT; | |
| 4507 | − } | |
| 4508 | − } | |
| 4509 | − st = st->parent; | |
| 4510 | − if (!st) | |
| 4511 | − break; | |
| 4512 | − | |
| 4513 | − for (fr = bt->frame; fr >= 0; fr--) { | |
| 4514 | − func = st->frame[fr]; | |
| 4515 | − bitmap_from_u64(mask, bt_frame_reg_mask(bt, fr)); | |
| 4516 | − for_each_set_bit(i, mask, 32) { | |
| 4517 | − reg = &func->regs[i]; | |
| 4518 | − if (reg->type != SCALAR_VALUE) { | |
| 4519 | − bt_clear_frame_reg(bt, fr, i); | |
| 4520 | − continue; | |
| 4521 | − } | |
| 4522 | − if (reg->precise) { | |
| 4523 | − bt_clear_frame_reg(bt, fr, i); | |
| 4524 | − } else { | |
| 4525 | − reg->precise = true; | |
| 4526 | − *changed = true; | |
| 4527 | − } | |
| 4528 | − } | |
| 4529 | − | |
| 4530 | − bitmap_from_u64(mask, bt_frame_stack_mask(bt, fr)); | |
| 4531 | − for_each_set_bit(i, mask, 64) { | |
| 4532 | − if (verifier_bug_if(i >= func->allocated_stack / BPF_REG_SIZE, | |
| 4533 | − env, "stack slot %d, total slots %d", | |
| 4534 | − i, func->allocated_stack / BPF_REG_SIZE)) | |
| 4535 | − return -EFAULT; | |
| 4536 | − | |
| 4537 | − if (!is_spilled_scalar_reg(&func->stack[i])) { | |
| 4538 | − bt_clear_frame_slot(bt, fr, i); | |
| 4539 | − continue; | |
| 4540 | − } | |
| 4541 | − reg = &func->stack[i].spilled_ptr; | |
| 4542 | − if (reg->precise) { | |
| 4543 | − bt_clear_frame_slot(bt, fr, i); | |
| 4544 | − } else { | |
| 4545 | − reg->precise = true; | |
| 4546 | − *changed = true; | |
| 4547 | − } | |
| 4548 | − } | |
| 4549 | − if (env->log.level & BPF_LOG_LEVEL2) { | |
| 4550 | − fmt_reg_mask(env->tmp_str_buf, TMP_STR_BUF_LEN, | |
| 4551 | − bt_frame_reg_mask(bt, fr)); | |
| 4552 | − verbose(env, "mark_precise: frame%d: parent state regs=%s ", | |
| 4553 | − fr, env->tmp_str_buf); | |
| 4554 | − bpf_fmt_stack_mask(env->tmp_str_buf, TMP_STR_BUF_LEN, | |
| 4555 | − bt_frame_stack_mask(bt, fr)); | |
| 4556 | − verbose(env, "stack=%s: ", env->tmp_str_buf); | |
| 4557 | − print_verifier_state(env, st, fr, true); | |
| 4558 | − } | |
| 4559 | − } | |
| 4560 | − | |
| 4561 | − if (bt_empty(bt)) | |
| 4562 | − return 0; | |
| 4563 | − | |
| 4564 | − subseq_idx = first_idx; | |
| 4565 | − last_idx = st->last_insn_idx; | |
| 4566 | − first_idx = st->first_insn_idx; | |
| 4567 | − } | |
| 4568 | − | |
| 4569 | − /* if we still have requested precise regs or slots, we missed | |
| 4570 | − * something (e.g., stack access through non-r10 register), so | |
| 4571 | − * fallback to marking all precise | |
| 4572 | − */ | |
| 4573 | − if (!bt_empty(bt)) { | |
| 4574 | − bpf_mark_all_scalars_precise(env, starting_state); | |
| 4575 | − bt_reset(bt); | |
| 4576 | − } | |
| 4577 | − | |
| 4578 | − return 0; | |
| 4579 | −} | |
| 4580 | − | |
| 4581 | 3641 | int mark_chain_precision(struct bpf_verifier_env *env, int regno) |
| 4582 | 3642 | { |
| 4583 | 3643 | return bpf_mark_chain_precision(env, env->cur_state, regno, NULL); |
| @@ -4759,7 +3819,7 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env, · 175 unmodified lines | ||
| 4759 | 3819 | */ |
| 4760 | 3820 | if (!env->allow_ptr_leaks && |
| 4761 | 3821 | bpf_is_spilled_reg(&state->stack[spi]) && |
| 4762 | − !is_spilled_scalar_reg(&state->stack[spi]) && | |
| 3822 | + !bpf_is_spilled_scalar_reg(&state->stack[spi]) && | |
| 4763 | 3823 | size != BPF_REG_SIZE) { |
| 4764 | 3824 | verbose(env, "attempt to corrupt spilled pointer on stack\n"); |
| 4765 | 3825 | return -EACCES; |
| @@ -4938,7 +3998,7 @@ static int check_stack_write_var_off(struct bpf_verifier_env *env, · 172 unmodified lines | ||
| 4938 | 3998 | * maintain the spill type. |
| 4939 | 3999 | */ |
| 4940 | 4000 | if (writing_zero && *stype == STACK_SPILL && |
| 4941 | − is_spilled_scalar_reg(&state->stack[spi])) { | |
| 4001 | + bpf_is_spilled_scalar_reg(&state->stack[spi])) { | |
| 4942 | 4002 | struct bpf_reg_state *spill_reg = &state->stack[spi].spilled_ptr; |
| 4943 | 4003 | |
| 4944 | 4004 | if (tnum_is_const(spill_reg->var_off) && spill_reg->var_off.value == 0) { |
| @@ -9380,7 +8440,7 @@ static int get_constant_map_key(struct bpf_verifier_env *env, · 4435 unmodified lines | ||
| 9380 | 8440 | } |
| 9381 | 8441 | |
| 9382 | 8442 | /* Check that stack contains a scalar spill of expected size */ |
| 9383 | − if (!is_spilled_scalar_reg(&state->stack[spi])) | |
| 8443 | + if (!bpf_is_spilled_scalar_reg(&state->stack[spi])) | |
| 9384 | 8444 | return -EOPNOTSUPP; |
| 9385 | 8445 | for (i = off; i >= 0 && stype[i] == STACK_SPILL; i--) |
| 9386 | 8446 | spill_size++; |