Move precision propagation and backtracking logic to backtrack.c to reduce verifier.c size. No functional changes.
Showing the changes to kernel (3 of 4 files). Show all
Unified · Split
| @@ -11,7 +11,7 @@ obj-$(CONFIG_BPF_SYSCALL) += bpf_iter.o map_iter.o task_iter.o prog_iter.o link_ · 10 unmodified lines | |||
| 11 | obj-$(CONFIG_BPF_SYSCALL) += hashtab.o arraymap.o percpu_freelist.o bpf_lru_list.o lpm_trie.o map_in_map.o bloom_filter.o | 11 | obj-$(CONFIG_BPF_SYSCALL) += hashtab.o arraymap.o percpu_freelist.o bpf_lru_list.o lpm_trie.o map_in_map.o bloom_filter.o |
| 12 | obj-$(CONFIG_BPF_SYSCALL) += local_storage.o queue_stack_maps.o ringbuf.o bpf_insn_array.o | 12 | obj-$(CONFIG_BPF_SYSCALL) += local_storage.o queue_stack_maps.o ringbuf.o bpf_insn_array.o |
| 13 | obj-$(CONFIG_BPF_SYSCALL) += bpf_local_storage.o bpf_task_storage.o | 13 | obj-$(CONFIG_BPF_SYSCALL) += bpf_local_storage.o bpf_task_storage.o |
| 14 | obj-$(CONFIG_BPF_SYSCALL) += fixups.o cfg.o states.o | 14 | obj-$(CONFIG_BPF_SYSCALL) += fixups.o cfg.o states.o backtrack.o |
| 15 | obj-${CONFIG_BPF_LSM} += bpf_inode_storage.o | 15 | obj-${CONFIG_BPF_LSM} += bpf_inode_storage.o |
| 16 | obj-$(CONFIG_BPF_SYSCALL) += disasm.o mprog.o | 16 | obj-$(CONFIG_BPF_SYSCALL) += disasm.o mprog.o |
| 17 | obj-$(CONFIG_BPF_JIT) += trampoline.o | 17 | obj-$(CONFIG_BPF_JIT) += trampoline.o |
| @@ -0,0 +1,934 @@ | |||
| 1 | // SPDX-License-Identifier: GPL-2.0-only | ||
| 2 | /* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ | ||
| 3 | #include <linux/bpf.h> | ||
| 4 | #include <linux/bpf_verifier.h> | ||
| 5 | #include <linux/filter.h> | ||
| 6 | #include <linux/bitmap.h> | ||
| 7 | |||
| 8 | #define verbose(env, fmt, args...) bpf_verifier_log_write(env, fmt, ##args) | ||
| 9 | |||
| 10 | /* for any branch, call, exit record the history of jmps in the given state */ | ||
| 11 | int bpf_push_jmp_history(struct bpf_verifier_env *env, struct bpf_verifier_state *cur, | ||
| 12 | int insn_flags, u64 linked_regs) | ||
| 13 | { | ||
| 14 | u32 cnt = cur->jmp_history_cnt; | ||
| 15 | struct bpf_jmp_history_entry *p; | ||
| 16 | size_t alloc_size; | ||
| 17 | |||
| 18 | /* combine instruction flags if we already recorded this instruction */ | ||
| 19 | if (env->cur_hist_ent) { | ||
| 20 | /* atomic instructions push insn_flags twice, for READ and | ||
| 21 | * WRITE sides, but they should agree on stack slot | ||
| 22 | */ | ||
| 23 | verifier_bug_if((env->cur_hist_ent->flags & insn_flags) && | ||
| 24 | (env->cur_hist_ent->flags & insn_flags) != insn_flags, | ||
| 25 | env, "insn history: insn_idx %d cur flags %x new flags %x", | ||
| 26 | env->insn_idx, env->cur_hist_ent->flags, insn_flags); | ||
| 27 | env->cur_hist_ent->flags |= insn_flags; | ||
| 28 | verifier_bug_if(env->cur_hist_ent->linked_regs != 0, env, | ||
| 29 | "insn history: insn_idx %d linked_regs: %#llx", | ||
| 30 | env->insn_idx, env->cur_hist_ent->linked_regs); | ||
| 31 | env->cur_hist_ent->linked_regs = linked_regs; | ||
| 32 | return 0; | ||
| 33 | } | ||
| 34 | |||
| 35 | cnt++; | ||
| 36 | alloc_size = kmalloc_size_roundup(size_mul(cnt, sizeof(*p))); | ||
| 37 | p = krealloc(cur->jmp_history, alloc_size, GFP_KERNEL_ACCOUNT); | ||
| 38 | if (!p) | ||
| 39 | return -ENOMEM; | ||
| 40 | cur->jmp_history = p; | ||
| 41 | |||
| 42 | p = &cur->jmp_history[cnt - 1]; | ||
| 43 | p->idx = env->insn_idx; | ||
| 44 | p->prev_idx = env->prev_insn_idx; | ||
| 45 | p->flags = insn_flags; | ||
| 46 | p->linked_regs = linked_regs; | ||
| 47 | cur->jmp_history_cnt = cnt; | ||
| 48 | env->cur_hist_ent = p; | ||
| 49 | |||
| 50 | return 0; | ||
| 51 | } | ||
| 52 | |||
| 53 | static bool is_atomic_load_insn(const struct bpf_insn *insn) | ||
| 54 | { | ||
| 55 | return BPF_CLASS(insn->code) == BPF_STX && | ||
| 56 | BPF_MODE(insn->code) == BPF_ATOMIC && | ||
| 57 | insn->imm == BPF_LOAD_ACQ; | ||
| 58 | } | ||
| 59 | |||
| 60 | static bool is_atomic_fetch_insn(const struct bpf_insn *insn) | ||
| 61 | { | ||
| 62 | return BPF_CLASS(insn->code) == BPF_STX && | ||
| 63 | BPF_MODE(insn->code) == BPF_ATOMIC && | ||
| 64 | (insn->imm & BPF_FETCH); | ||
| 65 | } | ||
| 66 | |||
| 67 | static int insn_stack_access_spi(int insn_flags) | ||
| 68 | { | ||
| 69 | return (insn_flags >> INSN_F_SPI_SHIFT) & INSN_F_SPI_MASK; | ||
| 70 | } | ||
| 71 | |||
| 72 | static int insn_stack_access_frameno(int insn_flags) | ||
| 73 | { | ||
| 74 | return insn_flags & INSN_F_FRAMENO_MASK; | ||
| 75 | } | ||
| 76 | |||
| 77 | /* Backtrack one insn at a time. If idx is not at the top of recorded | ||
| 78 | * history then previous instruction came from straight line execution. | ||
| 79 | * Return -ENOENT if we exhausted all instructions within given state. | ||
| 80 | * | ||
| 81 | * It's legal to have a bit of a looping with the same starting and ending | ||
| 82 | * insn index within the same state, e.g.: 3->4->5->3, so just because current | ||
| 83 | * instruction index is the same as state's first_idx doesn't mean we are | ||
| 84 | * done. If there is still some jump history left, we should keep going. We | ||
| 85 | * need to take into account that we might have a jump history between given | ||
| 86 | * state's parent and itself, due to checkpointing. In this case, we'll have | ||
| 87 | * history entry recording a jump from last instruction of parent state and | ||
| 88 | * first instruction of given state. | ||
| 89 | */ | ||
| 90 | static int get_prev_insn_idx(struct bpf_verifier_state *st, int i, | ||
| 91 | u32 *history) | ||
| 92 | { | ||
| 93 | u32 cnt = *history; | ||
| 94 | |||
| 95 | if (i == st->first_insn_idx) { | ||
| 96 | if (cnt == 0) | ||
| 97 | return -ENOENT; | ||
| 98 | if (cnt == 1 && st->jmp_history[0].idx == i) | ||
| 99 | return -ENOENT; | ||
| 100 | } | ||
| 101 | |||
| 102 | if (cnt && st->jmp_history[cnt - 1].idx == i) { | ||
| 103 | i = st->jmp_history[cnt - 1].prev_idx; | ||
| 104 | (*history)--; | ||
| 105 | } else { | ||
| 106 | i--; | ||
| 107 | } | ||
| 108 | return i; | ||
| 109 | } | ||
| 110 | |||
| 111 | static struct bpf_jmp_history_entry *get_jmp_hist_entry(struct bpf_verifier_state *st, | ||
| 112 | u32 hist_end, int insn_idx) | ||
| 113 | { | ||
| 114 | if (hist_end > 0 && st->jmp_history[hist_end - 1].idx == insn_idx) | ||
| 115 | return &st->jmp_history[hist_end - 1]; | ||
| 116 | return NULL; | ||
| 117 | } | ||
| 118 | |||
| 119 | static inline void bt_init(struct backtrack_state *bt, u32 frame) | ||
| 120 | { | ||
| 121 | bt->frame = frame; | ||
| 122 | } | ||
| 123 | |||
| 124 | static inline void bt_reset(struct backtrack_state *bt) | ||
| 125 | { | ||
| 126 | struct bpf_verifier_env *env = bt->env; | ||
| 127 | |||
| 128 | memset(bt, 0, sizeof(*bt)); | ||
| 129 | bt->env = env; | ||
| 130 | } | ||
| 131 | |||
| 132 | static inline u32 bt_empty(struct backtrack_state *bt) | ||
| 133 | { | ||
| 134 | u64 mask = 0; | ||
| 135 | int i; | ||
| 136 | |||
| 137 | for (i = 0; i <= bt->frame; i++) | ||
| 138 | mask |= bt->reg_masks[i] | bt->stack_masks[i]; | ||
| 139 | |||
| 140 | return mask == 0; | ||
| 141 | } | ||
| 142 | |||
| 143 | static inline int bt_subprog_enter(struct backtrack_state *bt) | ||
| 144 | { | ||
| 145 | if (bt->frame == MAX_CALL_FRAMES - 1) { | ||
| 146 | verifier_bug(bt->env, "subprog enter from frame %d", bt->frame); | ||
| 147 | return -EFAULT; | ||
| 148 | } | ||
| 149 | bt->frame++; | ||
| 150 | return 0; | ||
| 151 | } | ||
| 152 | |||
| 153 | static inline int bt_subprog_exit(struct backtrack_state *bt) | ||
| 154 | { | ||
| 155 | if (bt->frame == 0) { | ||
| 156 | verifier_bug(bt->env, "subprog exit from frame 0"); | ||
| 157 | return -EFAULT; | ||
| 158 | } | ||
| 159 | bt->frame--; | ||
| 160 | return 0; | ||
| 161 | } | ||
| 162 | |||
| 163 | static inline void bt_clear_frame_reg(struct backtrack_state *bt, u32 frame, u32 reg) | ||
| 164 | { | ||
| 165 | bt->reg_masks[frame] &= ~(1 << reg); | ||
| 166 | } | ||
| 167 | |||
| 168 | static inline void bt_set_reg(struct backtrack_state *bt, u32 reg) | ||
| 169 | { | ||
| 170 | bpf_bt_set_frame_reg(bt, bt->frame, reg); | ||
| 171 | } | ||
| 172 | |||
| 173 | static inline void bt_clear_reg(struct backtrack_state *bt, u32 reg) | ||
| 174 | { | ||
| 175 | bt_clear_frame_reg(bt, bt->frame, reg); | ||
| 176 | } | ||
| 177 | |||
| 178 | static inline void bt_clear_frame_slot(struct backtrack_state *bt, u32 frame, u32 slot) | ||
| 179 | { | ||
| 180 | bt->stack_masks[frame] &= ~(1ull << slot); | ||
| 181 | } | ||
| 182 | |||
| 183 | static inline u32 bt_frame_reg_mask(struct backtrack_state *bt, u32 frame) | ||
| 184 | { | ||
| 185 | return bt->reg_masks[frame]; | ||
| 186 | } | ||
| 187 | |||
| 188 | static inline u32 bt_reg_mask(struct backtrack_state *bt) | ||
| 189 | { | ||
| 190 | return bt->reg_masks[bt->frame]; | ||
| 191 | } | ||
| 192 | |||
| 193 | static inline u64 bt_frame_stack_mask(struct backtrack_state *bt, u32 frame) | ||
| 194 | { | ||
| 195 | return bt->stack_masks[frame]; | ||
| 196 | } | ||
| 197 | |||
| 198 | static inline u64 bt_stack_mask(struct backtrack_state *bt) | ||
| 199 | { | ||
| 200 | return bt->stack_masks[bt->frame]; | ||
| 201 | } | ||
| 202 | |||
| 203 | static inline bool bt_is_reg_set(struct backtrack_state *bt, u32 reg) | ||
| 204 | { | ||
| 205 | return bt->reg_masks[bt->frame] & (1 << reg); | ||
| 206 | } | ||
| 207 | |||
| 208 | |||
| 209 | /* format registers bitmask, e.g., "r0,r2,r4" for 0x15 mask */ | ||
| 210 | static void fmt_reg_mask(char *buf, ssize_t buf_sz, u32 reg_mask) | ||
| 211 | { | ||
| 212 | DECLARE_BITMAP(mask, 64); | ||
| 213 | bool first = true; | ||
| 214 | int i, n; | ||
| 215 | |||
| 216 | buf[0] = '\0'; | ||
| 217 | |||
| 218 | bitmap_from_u64(mask, reg_mask); | ||
| 219 | for_each_set_bit(i, mask, 32) { | ||
| 220 | n = snprintf(buf, buf_sz, "%sr%d", first ? "" : ",", i); | ||
| 221 | first = false; | ||
| 222 | buf += n; | ||
| 223 | buf_sz -= n; | ||
| 224 | if (buf_sz < 0) | ||
| 225 | break; | ||
| 226 | } | ||
| 227 | } | ||
| 228 | /* format stack slots bitmask, e.g., "-8,-24,-40" for 0x15 mask */ | ||
| 229 | void bpf_fmt_stack_mask(char *buf, ssize_t buf_sz, u64 stack_mask) | ||
| 230 | { | ||
| 231 | DECLARE_BITMAP(mask, 64); | ||
| 232 | bool first = true; | ||
| 233 | int i, n; | ||
| 234 | |||
| 235 | buf[0] = '\0'; | ||
| 236 | |||
| 237 | bitmap_from_u64(mask, stack_mask); | ||
| 238 | for_each_set_bit(i, mask, 64) { | ||
| 239 | n = snprintf(buf, buf_sz, "%s%d", first ? "" : ",", -(i + 1) * 8); | ||
| 240 | first = false; | ||
| 241 | buf += n; | ||
| 242 | buf_sz -= n; | ||
| 243 | if (buf_sz < 0) | ||
| 244 | break; | ||
| 245 | } | ||
| 246 | } | ||
| 247 | |||
| 248 | |||
| 249 | /* For given verifier state backtrack_insn() is called from the last insn to | ||
| 250 | * the first insn. Its purpose is to compute a bitmask of registers and | ||
| 251 | * stack slots that needs precision in the parent verifier state. | ||
| 252 | * | ||
| 253 | * @idx is an index of the instruction we are currently processing; | ||
| 254 | * @subseq_idx is an index of the subsequent instruction that: | ||
| 255 | * - *would be* executed next, if jump history is viewed in forward order; | ||
| 256 | * - *was* processed previously during backtracking. | ||
| 257 | */ | ||
| 258 | static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, | ||
| 259 | struct bpf_jmp_history_entry *hist, struct backtrack_state *bt) | ||
| 260 | { | ||
| 261 | struct bpf_insn *insn = env->prog->insnsi + idx; | ||
| 262 | u8 class = BPF_CLASS(insn->code); | ||
| 263 | u8 opcode = BPF_OP(insn->code); | ||
| 264 | u8 mode = BPF_MODE(insn->code); | ||
| 265 | u32 dreg = insn->dst_reg; | ||
| 266 | u32 sreg = insn->src_reg; | ||
| 267 | u32 spi, i, fr; | ||
| 268 | |||
| 269 | if (insn->code == 0) | ||
| 270 | return 0; | ||
| 271 | if (env->log.level & BPF_LOG_LEVEL2) { | ||
| 272 | fmt_reg_mask(env->tmp_str_buf, TMP_STR_BUF_LEN, bt_reg_mask(bt)); | ||
| 273 | verbose(env, "mark_precise: frame%d: regs=%s ", | ||
| 274 | bt->frame, env->tmp_str_buf); | ||
| 275 | bpf_fmt_stack_mask(env->tmp_str_buf, TMP_STR_BUF_LEN, bt_stack_mask(bt)); | ||
| 276 | verbose(env, "stack=%s before ", env->tmp_str_buf); | ||
| 277 | verbose(env, "%d: ", idx); | ||
| 278 | bpf_verbose_insn(env, insn); | ||
| 279 | } | ||
| 280 | |||
| 281 | /* If there is a history record that some registers gained range at this insn, | ||
| 282 | * propagate precision marks to those registers, so that bt_is_reg_set() | ||
| 283 | * accounts for these registers. | ||
| 284 | */ | ||
| 285 | bpf_bt_sync_linked_regs(bt, hist); | ||
| 286 | |||
| 287 | if (class == BPF_ALU || class == BPF_ALU64) { | ||
| 288 | if (!bt_is_reg_set(bt, dreg)) | ||
| 289 | return 0; | ||
| 290 | if (opcode == BPF_END || opcode == BPF_NEG) { | ||
| 291 | /* sreg is reserved and unused | ||
| 292 | * dreg still need precision before this insn | ||
| 293 | */ | ||
| 294 | return 0; | ||
| 295 | } else if (opcode == BPF_MOV) { | ||
| 296 | if (BPF_SRC(insn->code) == BPF_X) { | ||
| 297 | /* dreg = sreg or dreg = (s8, s16, s32)sreg | ||
| 298 | * dreg needs precision after this insn | ||
| 299 | * sreg needs precision before this insn | ||
| 300 | */ | ||
| 301 | bt_clear_reg(bt, dreg); | ||
| 302 | if (sreg != BPF_REG_FP) | ||
| 303 | bt_set_reg(bt, sreg); | ||
| 304 | } else { | ||
| 305 | /* dreg = K | ||
| 306 | * dreg needs precision after this insn. | ||
| 307 | * Corresponding register is already marked | ||
| 308 | * as precise=true in this verifier state. | ||
| 309 | * No further markings in parent are necessary | ||
| 310 | */ | ||
| 311 | bt_clear_reg(bt, dreg); | ||
| 312 | } | ||
| 313 | } else { | ||
| 314 | if (BPF_SRC(insn->code) == BPF_X) { | ||
| 315 | /* dreg += sreg | ||
| 316 | * both dreg and sreg need precision | ||
| 317 | * before this insn | ||
| 318 | */ | ||
| 319 | if (sreg != BPF_REG_FP) | ||
| 320 | bt_set_reg(bt, sreg); | ||
| 321 | } /* else dreg += K | ||
| 322 | * dreg still needs precision before this insn | ||
| 323 | */ | ||
| 324 | } | ||
| 325 | } else if (class == BPF_LDX || | ||
| 326 | is_atomic_load_insn(insn) || | ||
| 327 | is_atomic_fetch_insn(insn)) { | ||
| 328 | u32 load_reg = dreg; | ||
| 329 | |||
| 330 | /* | ||
| 331 | * Atomic fetch operation writes the old value into | ||
| 332 | * a register (sreg or r0) and if it was tracked for | ||
| 333 | * precision, propagate to the stack slot like we do | ||
| 334 | * in regular ldx. | ||
| 335 | */ | ||
| 336 | if (is_atomic_fetch_insn(insn)) | ||
| 337 | load_reg = insn->imm == BPF_CMPXCHG ? | ||
| 338 | BPF_REG_0 : sreg; | ||
| 339 | |||
| 340 | if (!bt_is_reg_set(bt, load_reg)) | ||
| 341 | return 0; | ||
| 342 | bt_clear_reg(bt, load_reg); | ||
| 343 | |||
| 344 | /* scalars can only be spilled into stack w/o losing precision. | ||
| 345 | * Load from any other memory can be zero extended. | ||
| 346 | * The desire to keep that precision is already indicated | ||
| 347 | * by 'precise' mark in corresponding register of this state. | ||
| 348 | * No further tracking necessary. | ||
| 349 | */ | ||
| 350 | if (!hist || !(hist->flags & INSN_F_STACK_ACCESS)) | ||
| 351 | return 0; | ||
| 352 | /* dreg = *(u64 *)[fp - off] was a fill from the stack. | ||
| 353 | * that [fp - off] slot contains scalar that needs to be | ||
| 354 | * tracked with precision | ||
| 355 | */ | ||
| 356 | spi = insn_stack_access_spi(hist->flags); | ||
| 357 | fr = insn_stack_access_frameno(hist->flags); | ||
| 358 | bpf_bt_set_frame_slot(bt, fr, spi); | ||
| 359 | } else if (class == BPF_STX || class == BPF_ST) { | ||
| 360 | if (bt_is_reg_set(bt, dreg)) | ||
| 361 | /* stx & st shouldn't be using _scalar_ dst_reg | ||
| 362 | * to access memory. It means backtracking | ||
| 363 | * encountered a case of pointer subtraction. | ||
| 364 | */ | ||
| 365 | return -ENOTSUPP; | ||
| 366 | /* scalars can only be spilled into stack */ | ||
| 367 | if (!hist || !(hist->flags & INSN_F_STACK_ACCESS)) | ||
| 368 | return 0; | ||
| 369 | spi = insn_stack_access_spi(hist->flags); | ||
| 370 | fr = insn_stack_access_frameno(hist->flags); | ||
| 371 | if (!bt_is_frame_slot_set(bt, fr, spi)) | ||
| 372 | return 0; | ||
| 373 | bt_clear_frame_slot(bt, fr, spi); | ||
| 374 | if (class == BPF_STX) | ||
| 375 | bt_set_reg(bt, sreg); | ||
| 376 | } else if (class == BPF_JMP || class == BPF_JMP32) { | ||
| 377 | if (bpf_pseudo_call(insn)) { | ||
| 378 | int subprog_insn_idx, subprog; | ||
| 379 | |||
| 380 | subprog_insn_idx = idx + insn->imm + 1; | ||
| 381 | subprog = bpf_find_subprog(env, subprog_insn_idx); | ||
| 382 | if (subprog < 0) | ||
| 383 | return -EFAULT; | ||
| 384 | |||
| 385 | if (bpf_subprog_is_global(env, subprog)) { | ||
| 386 | /* check that jump history doesn't have any | ||
| 387 | * extra instructions from subprog; the next | ||
| 388 | * instruction after call to global subprog | ||
| 389 | * should be literally next instruction in | ||
| 390 | * caller program | ||
| 391 | */ | ||
| 392 | verifier_bug_if(idx + 1 != subseq_idx, env, | ||
| 393 | "extra insn from subprog"); | ||
| 394 | /* r1-r5 are invalidated after subprog call, | ||
| 395 | * so for global func call it shouldn't be set | ||
| 396 | * anymore | ||
| 397 | */ | ||
| 398 | if (bt_reg_mask(bt) & BPF_REGMASK_ARGS) { | ||
| 399 | verifier_bug(env, "global subprog unexpected regs %x", | ||
| 400 | bt_reg_mask(bt)); | ||
| 401 | return -EFAULT; | ||
| 402 | } | ||
| 403 | /* global subprog always sets R0 */ | ||
| 404 | bt_clear_reg(bt, BPF_REG_0); | ||
| 405 | return 0; | ||
| 406 | } else { | ||
| 407 | /* static subprog call instruction, which | ||
| 408 | * means that we are exiting current subprog, | ||
| 409 | * so only r1-r5 could be still requested as | ||
| 410 | * precise, r0 and r6-r10 or any stack slot in | ||
| 411 | * the current frame should be zero by now | ||
| 412 | */ | ||
| 413 | if (bt_reg_mask(bt) & ~BPF_REGMASK_ARGS) { | ||
| 414 | verifier_bug(env, "static subprog unexpected regs %x", | ||
| 415 | bt_reg_mask(bt)); | ||
| 416 | return -EFAULT; | ||
| 417 | } | ||
| 418 | /* we are now tracking register spills correctly, | ||
| 419 | * so any instance of leftover slots is a bug | ||
| 420 | */ | ||
| 421 | if (bt_stack_mask(bt) != 0) { | ||
| 422 | verifier_bug(env, | ||
| 423 | "static subprog leftover stack slots %llx", | ||
| 424 | bt_stack_mask(bt)); | ||
| 425 | return -EFAULT; | ||
| 426 | } | ||
| 427 | /* propagate r1-r5 to the caller */ | ||
| 428 | for (i = BPF_REG_1; i <= BPF_REG_5; i++) { | ||
| 429 | if (bt_is_reg_set(bt, i)) { | ||
| 430 | bt_clear_reg(bt, i); | ||
| 431 | bpf_bt_set_frame_reg(bt, bt->frame - 1, i); | ||
| 432 | } | ||
| 433 | } | ||
| 434 | if (bt_subprog_exit(bt)) | ||
| 435 | return -EFAULT; | ||
| 436 | return 0; | ||
| 437 | } | ||
| 438 | } else if (bpf_is_sync_callback_calling_insn(insn) && idx != subseq_idx - 1) { | ||
| 439 | /* exit from callback subprog to callback-calling helper or | ||
| 440 | * kfunc call. Use idx/subseq_idx check to discern it from | ||
| 441 | * straight line code backtracking. | ||
| 442 | * Unlike the subprog call handling above, we shouldn't | ||
| 443 | * propagate precision of r1-r5 (if any requested), as they are | ||
| 444 | * not actually arguments passed directly to callback subprogs | ||
| 445 | */ | ||
| 446 | if (bt_reg_mask(bt) & ~BPF_REGMASK_ARGS) { | ||
| 447 | verifier_bug(env, "callback unexpected regs %x", | ||
| 448 | bt_reg_mask(bt)); | ||
| 449 | return -EFAULT; | ||
| 450 | } | ||
| 451 | if (bt_stack_mask(bt) != 0) { | ||
| 452 | verifier_bug(env, "callback leftover stack slots %llx", | ||
| 453 | bt_stack_mask(bt)); | ||
| 454 | return -EFAULT; | ||
| 455 | } | ||
| 456 | /* clear r1-r5 in callback subprog's mask */ | ||
| 457 | for (i = BPF_REG_1; i <= BPF_REG_5; i++) | ||
| 458 | bt_clear_reg(bt, i); | ||
| 459 | if (bt_subprog_exit(bt)) | ||
| 460 | return -EFAULT; | ||
| 461 | return 0; | ||
| 462 | } else if (opcode == BPF_CALL) { | ||
| 463 | /* kfunc with imm==0 is invalid and fixup_kfunc_call will | ||
| 464 | * catch this error later. Make backtracking conservative | ||
| 465 | * with ENOTSUPP. | ||
| 466 | */ | ||
| 467 | if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL && insn->imm == 0) | ||
| 468 | return -ENOTSUPP; | ||
| 469 | /* regular helper call sets R0 */ | ||
| 470 | bt_clear_reg(bt, BPF_REG_0); | ||
| 471 | if (bt_reg_mask(bt) & BPF_REGMASK_ARGS) { | ||
| 472 | /* if backtracking was looking for registers R1-R5 | ||
| 473 | * they should have been found already. | ||
| 474 | */ | ||
| 475 | verifier_bug(env, "backtracking call unexpected regs %x", | ||
| 476 | bt_reg_mask(bt)); | ||
| 477 | return -EFAULT; | ||
| 478 | } | ||
| 479 | if (insn->src_reg == BPF_REG_0 && insn->imm == BPF_FUNC_tail_call | ||
| 480 | && subseq_idx - idx != 1) { | ||
| 481 | if (bt_subprog_enter(bt)) | ||
| 482 | return -EFAULT; | ||
| 483 | } | ||
| 484 | } else if (opcode == BPF_EXIT) { | ||
| 485 | bool r0_precise; | ||
| 486 | |||
| 487 | /* Backtracking to a nested function call, 'idx' is a part of | ||
| 488 | * the inner frame 'subseq_idx' is a part of the outer frame. | ||
| 489 | * In case of a regular function call, instructions giving | ||
| 490 | * precision to registers R1-R5 should have been found already. | ||
| 491 | * In case of a callback, it is ok to have R1-R5 marked for | ||
| 492 | * backtracking, as these registers are set by the function | ||
| 493 | * invoking callback. | ||
| 494 | */ | ||
| 495 | if (subseq_idx >= 0 && bpf_calls_callback(env, subseq_idx)) | ||
| 496 | for (i = BPF_REG_1; i <= BPF_REG_5; i++) | ||
| 497 | bt_clear_reg(bt, i); | ||
| 498 | if (bt_reg_mask(bt) & BPF_REGMASK_ARGS) { | ||
| 499 | verifier_bug(env, "backtracking exit unexpected regs %x", | ||
| 500 | bt_reg_mask(bt)); | ||
| 501 | return -EFAULT; | ||
| 502 | } | ||
| 503 | |||
| 504 | /* BPF_EXIT in subprog or callback always returns | ||
| 505 | * right after the call instruction, so by checking | ||
| 506 | * whether the instruction at subseq_idx-1 is subprog | ||
| 507 | * call or not we can distinguish actual exit from | ||
| 508 | * *subprog* from exit from *callback*. In the former | ||
| 509 | * case, we need to propagate r0 precision, if | ||
| 510 | * necessary. In the former we never do that. | ||
| 511 | */ | ||
| 512 | r0_precise = subseq_idx - 1 >= 0 && | ||
| 513 | bpf_pseudo_call(&env->prog->insnsi[subseq_idx - 1]) && | ||
| 514 | bt_is_reg_set(bt, BPF_REG_0); | ||
| 515 | |||
| 516 | bt_clear_reg(bt, BPF_REG_0); | ||
| 517 | if (bt_subprog_enter(bt)) | ||
| 518 | return -EFAULT; | ||
| 519 | |||
| 520 | if (r0_precise) | ||
| 521 | bt_set_reg(bt, BPF_REG_0); | ||
| 522 | /* r6-r9 and stack slots will stay set in caller frame | ||
| 523 | * bitmasks until we return back from callee(s) | ||
| 524 | */ | ||
| 525 | return 0; | ||
| 526 | } else if (BPF_SRC(insn->code) == BPF_X) { | ||
| 527 | if (!bt_is_reg_set(bt, dreg) && !bt_is_reg_set(bt, sreg)) | ||
| 528 | return 0; | ||
| 529 | /* dreg <cond> sreg | ||
| 530 | * Both dreg and sreg need precision before | ||
| 531 | * this insn. If only sreg was marked precise | ||
| 532 | * before it would be equally necessary to | ||
| 533 | * propagate it to dreg. | ||
| 534 | */ | ||
| 535 | if (!hist || !(hist->flags & INSN_F_SRC_REG_STACK)) | ||
| 536 | bt_set_reg(bt, sreg); | ||
| 537 | if (!hist || !(hist->flags & INSN_F_DST_REG_STACK)) | ||
| 538 | bt_set_reg(bt, dreg); | ||
| 539 | } else if (BPF_SRC(insn->code) == BPF_K) { | ||
| 540 | /* dreg <cond> K | ||
| 541 | * Only dreg still needs precision before | ||
| 542 | * this insn, so for the K-based conditional | ||
| 543 | * there is nothing new to be marked. | ||
| 544 | */ | ||
| 545 | } | ||
| 546 | } else if (class == BPF_LD) { | ||
| 547 | if (!bt_is_reg_set(bt, dreg)) | ||
| 548 | return 0; | ||
| 549 | bt_clear_reg(bt, dreg); | ||
| 550 | /* It's ld_imm64 or ld_abs or ld_ind. | ||
| 551 | * For ld_imm64 no further tracking of precision | ||
| 552 | * into parent is necessary | ||
| 553 | */ | ||
| 554 | if (mode == BPF_IND || mode == BPF_ABS) | ||
| 555 | /* to be analyzed */ | ||
| 556 | return -ENOTSUPP; | ||
| 557 | } | ||
| 558 | /* Propagate precision marks to linked registers, to account for | ||
| 559 | * registers marked as precise in this function. | ||
| 560 | */ | ||
| 561 | bpf_bt_sync_linked_regs(bt, hist); | ||
| 562 | return 0; | ||
| 563 | } | ||
| 564 | |||
| 565 | /* the scalar precision tracking algorithm: | ||
| 566 | * . at the start all registers have precise=false. | ||
| 567 | * . scalar ranges are tracked as normal through alu and jmp insns. | ||
| 568 | * . once precise value of the scalar register is used in: | ||
| 569 | * . ptr + scalar alu | ||
| 570 | * . if (scalar cond K|scalar) | ||
| 571 | * . helper_call(.., scalar, ...) where ARG_CONST is expected | ||
| 572 | * backtrack through the verifier states and mark all registers and | ||
| 573 | * stack slots with spilled constants that these scalar registers | ||
| 574 | * should be precise. | ||
| 575 | * . during state pruning two registers (or spilled stack slots) | ||
| 576 | * are equivalent if both are not precise. | ||
| 577 | * | ||
| 578 | * Note the verifier cannot simply walk register parentage chain, | ||
| 579 | * since many different registers and stack slots could have been | ||
| 580 | * used to compute single precise scalar. | ||
| 581 | * | ||
| 582 | * The approach of starting with precise=true for all registers and then | ||
| 583 | * backtrack to mark a register as not precise when the verifier detects | ||
| 584 | * that program doesn't care about specific value (e.g., when helper | ||
| 585 | * takes register as ARG_ANYTHING parameter) is not safe. | ||
| 586 | * | ||
| 587 | * It's ok to walk single parentage chain of the verifier states. | ||
| 588 | * It's possible that this backtracking will go all the way till 1st insn. | ||
| 589 | * All other branches will be explored for needing precision later. | ||
| 590 | * | ||
| 591 | * The backtracking needs to deal with cases like: | ||
| 592 | * R8=map_value(id=0,off=0,ks=4,vs=1952,imm=0) R9_w=map_value(id=0,off=40,ks=4,vs=1952,imm=0) | ||
| 593 | * r9 -= r8 | ||
| 594 | * r5 = r9 | ||
| 595 | * if r5 > 0x79f goto pc+7 | ||
| 596 | * R5_w=inv(id=0,umax_value=1951,var_off=(0x0; 0x7ff)) | ||
| 597 | * r5 += 1 | ||
| 598 | * ... | ||
| 599 | * call bpf_perf_event_output#25 | ||
| 600 | * where .arg5_type = ARG_CONST_SIZE_OR_ZERO | ||
| 601 | * | ||
| 602 | * and this case: | ||
| 603 | * r6 = 1 | ||
| 604 | * call foo // uses callee's r6 inside to compute r0 | ||
| 605 | * r0 += r6 | ||
| 606 | * if r0 == 0 goto | ||
| 607 | * | ||
| 608 | * to track above reg_mask/stack_mask needs to be independent for each frame. | ||
| 609 | * | ||
| 610 | * Also if parent's curframe > frame where backtracking started, | ||
| 611 | * the verifier need to mark registers in both frames, otherwise callees | ||
| 612 | * may incorrectly prune callers. This is similar to | ||
| 613 | * commit 7640ead93924 ("bpf: verifier: make sure callees don't prune with caller differences") | ||
| 614 | * | ||
| 615 | * For now backtracking falls back into conservative marking. | ||
| 616 | */ | ||
| 617 | void bpf_mark_all_scalars_precise(struct bpf_verifier_env *env, | ||
| 618 | struct bpf_verifier_state *st) | ||
| 619 | { | ||
| 620 | struct bpf_func_state *func; | ||
| 621 | struct bpf_reg_state *reg; | ||
| 622 | int i, j; | ||
| 623 | |||
| 624 | if (env->log.level & BPF_LOG_LEVEL2) { | ||
| 625 | verbose(env, "mark_precise: frame%d: falling back to forcing all scalars precise\n", | ||
| 626 | st->curframe); | ||
| 627 | } | ||
| 628 | |||
| 629 | /* big hammer: mark all scalars precise in this path. | ||
| 630 | * pop_stack may still get !precise scalars. | ||
| 631 | * We also skip current state and go straight to first parent state, | ||
| 632 | * because precision markings in current non-checkpointed state are | ||
| 633 | * not needed. See why in the comment in __mark_chain_precision below. | ||
| 634 | */ | ||
| 635 | for (st = st->parent; st; st = st->parent) { | ||
| 636 | for (i = 0; i <= st->curframe; i++) { | ||
| 637 | func = st->frame[i]; | ||
| 638 | for (j = 0; j < BPF_REG_FP; j++) { | ||
| 639 | reg = &func->regs[j]; | ||
| 640 | if (reg->type != SCALAR_VALUE || reg->precise) | ||
| 641 | continue; | ||
| 642 | reg->precise = true; | ||
| 643 | if (env->log.level & BPF_LOG_LEVEL2) { | ||
| 644 | verbose(env, "force_precise: frame%d: forcing r%d to be precise\n", | ||
| 645 | i, j); | ||
| 646 | } | ||
| 647 | } | ||
| 648 | for (j = 0; j < func->allocated_stack / BPF_REG_SIZE; j++) { | ||
| 649 | if (!bpf_is_spilled_reg(&func->stack[j])) | ||
| 650 | continue; | ||
| 651 | reg = &func->stack[j].spilled_ptr; | ||
| 652 | if (reg->type != SCALAR_VALUE || reg->precise) | ||
| 653 | continue; | ||
| 654 | reg->precise = true; | ||
| 655 | if (env->log.level & BPF_LOG_LEVEL2) { | ||
| 656 | verbose(env, "force_precise: frame%d: forcing fp%d to be precise\n", | ||
| 657 | i, -(j + 1) * 8); | ||
| 658 | } | ||
| 659 | } | ||
| 660 | } | ||
| 661 | } | ||
| 662 | } | ||
| 663 | |||
| 664 | /* | ||
| 665 | * bpf_mark_chain_precision() backtracks BPF program instruction sequence and | ||
| 666 | * chain of verifier states making sure that register *regno* (if regno >= 0) | ||
| 667 | * and/or stack slot *spi* (if spi >= 0) are marked as precisely tracked | ||
| 668 | * SCALARS, as well as any other registers and slots that contribute to | ||
| 669 | * a tracked state of given registers/stack slots, depending on specific BPF | ||
| 670 | * assembly instructions (see backtrack_insns() for exact instruction handling | ||
| 671 | * logic). This backtracking relies on recorded jmp_history and is able to | ||
| 672 | * traverse entire chain of parent states. This process ends only when all the | ||
| 673 | * necessary registers/slots and their transitive dependencies are marked as | ||
| 674 | * precise. | ||
| 675 | * | ||
| 676 | * One important and subtle aspect is that precise marks *do not matter* in | ||
| 677 | * the currently verified state (current state). It is important to understand | ||
| 678 | * why this is the case. | ||
| 679 | * | ||
| 680 | * First, note that current state is the state that is not yet "checkpointed", | ||
| 681 | * i.e., it is not yet put into env->explored_states, and it has no children | ||
| 682 | * states as well. It's ephemeral, and can end up either a) being discarded if | ||
| 683 | * compatible explored state is found at some point or BPF_EXIT instruction is | ||
| 684 | * reached or b) checkpointed and put into env->explored_states, branching out | ||
| 685 | * into one or more children states. | ||
| 686 | * | ||
| 687 | * In the former case, precise markings in current state are completely | ||
| 688 | * ignored by state comparison code (see regsafe() for details). Only | ||
| 689 | * checkpointed ("old") state precise markings are important, and if old | ||
| 690 | * state's register/slot is precise, regsafe() assumes current state's | ||
| 691 | * register/slot as precise and checks value ranges exactly and precisely. If | ||
| 692 | * states turn out to be compatible, current state's necessary precise | ||
| 693 | * markings and any required parent states' precise markings are enforced | ||
| 694 | * after the fact with propagate_precision() logic, after the fact. But it's | ||
| 695 | * important to realize that in this case, even after marking current state | ||
| 696 | * registers/slots as precise, we immediately discard current state. So what | ||
| 697 | * actually matters is any of the precise markings propagated into current | ||
| 698 | * state's parent states, which are always checkpointed (due to b) case above). | ||
| 699 | * As such, for scenario a) it doesn't matter if current state has precise | ||
| 700 | * markings set or not. | ||
| 701 | * | ||
| 702 | * Now, for the scenario b), checkpointing and forking into child(ren) | ||
| 703 | * state(s). Note that before current state gets to checkpointing step, any | ||
| 704 | * processed instruction always assumes precise SCALAR register/slot | ||
| 705 | * knowledge: if precise value or range is useful to prune jump branch, BPF | ||
| 706 | * verifier takes this opportunity enthusiastically. Similarly, when | ||
| 707 | * register's value is used to calculate offset or memory address, exact | ||
| 708 | * knowledge of SCALAR range is assumed, checked, and enforced. So, similar to | ||
| 709 | * what we mentioned above about state comparison ignoring precise markings | ||
| 710 | * during state comparison, BPF verifier ignores and also assumes precise | ||
| 711 | * markings *at will* during instruction verification process. But as verifier | ||
| 712 | * assumes precision, it also propagates any precision dependencies across | ||
| 713 | * parent states, which are not yet finalized, so can be further restricted | ||
| 714 | * based on new knowledge gained from restrictions enforced by their children | ||
| 715 | * states. This is so that once those parent states are finalized, i.e., when | ||
| 716 | * they have no more active children state, state comparison logic in | ||
| 717 | * is_state_visited() would enforce strict and precise SCALAR ranges, if | ||
| 718 | * required for correctness. | ||
| 719 | * | ||
| 720 | * To build a bit more intuition, note also that once a state is checkpointed, | ||
| 721 | * the path we took to get to that state is not important. This is crucial | ||
| 722 | * property for state pruning. When state is checkpointed and finalized at | ||
| 723 | * some instruction index, it can be correctly and safely used to "short | ||
| 724 | * circuit" any *compatible* state that reaches exactly the same instruction | ||
| 725 | * index. I.e., if we jumped to that instruction from a completely different | ||
| 726 | * code path than original finalized state was derived from, it doesn't | ||
| 727 | * matter, current state can be discarded because from that instruction | ||
| 728 | * forward having a compatible state will ensure we will safely reach the | ||
| 729 | * exit. States describe preconditions for further exploration, but completely | ||
| 730 | * forget the history of how we got here. | ||
| 731 | * | ||
| 732 | * This also means that even if we needed precise SCALAR range to get to | ||
| 733 | * finalized state, but from that point forward *that same* SCALAR register is | ||
| 734 | * never used in a precise context (i.e., it's precise value is not needed for | ||
| 735 | * correctness), it's correct and safe to mark such register as "imprecise" | ||
| 736 | * (i.e., precise marking set to false). This is what we rely on when we do | ||
| 737 | * not set precise marking in current state. If no child state requires | ||
| 738 | * precision for any given SCALAR register, it's safe to dictate that it can | ||
| 739 | * be imprecise. If any child state does require this register to be precise, | ||
| 740 | * we'll mark it precise later retroactively during precise markings | ||
| 741 | * propagation from child state to parent states. | ||
| 742 | * | ||
| 743 | * Skipping precise marking setting in current state is a mild version of | ||
| 744 | * relying on the above observation. But we can utilize this property even | ||
| 745 | * more aggressively by proactively forgetting any precise marking in the | ||
| 746 | * current state (which we inherited from the parent state), right before we | ||
| 747 | * checkpoint it and branch off into new child state. This is done by | ||
| 748 | * mark_all_scalars_imprecise() to hopefully get more permissive and generic | ||
| 749 | * finalized states which help in short circuiting more future states. | ||
| 750 | */ | ||
| 751 | int bpf_mark_chain_precision(struct bpf_verifier_env *env, | ||
| 752 | struct bpf_verifier_state *starting_state, | ||
| 753 | int regno, | ||
| 754 | bool *changed) | ||
| 755 | { | ||
| 756 | struct bpf_verifier_state *st = starting_state; | ||
| 757 | struct backtrack_state *bt = &env->bt; | ||
| 758 | int first_idx = st->first_insn_idx; | ||
| 759 | int last_idx = starting_state->insn_idx; | ||
| 760 | int subseq_idx = -1; | ||
| 761 | struct bpf_func_state *func; | ||
| 762 | bool tmp, skip_first = true; | ||
| 763 | struct bpf_reg_state *reg; | ||
| 764 | int i, fr, err; | ||
| 765 | |||
| 766 | if (!env->bpf_capable) | ||
| 767 | return 0; | ||
| 768 | |||
| 769 | changed = changed ?: &tmp; | ||
| 770 | /* set frame number from which we are starting to backtrack */ | ||
| 771 | bt_init(bt, starting_state->curframe); | ||
| 772 | |||
| 773 | /* Do sanity checks against current state of register and/or stack | ||
| 774 | * slot, but don't set precise flag in current state, as precision | ||
| 775 | * tracking in the current state is unnecessary. | ||
| 776 | */ | ||
| 777 | func = st->frame[bt->frame]; | ||
| 778 | if (regno >= 0) { | ||
| 779 | reg = &func->regs[regno]; | ||
| 780 | if (reg->type != SCALAR_VALUE) { | ||
| 781 | verifier_bug(env, "backtracking misuse"); | ||
| 782 | return -EFAULT; | ||
| 783 | } | ||
| 784 | bt_set_reg(bt, regno); | ||
| 785 | } | ||
| 786 | |||
| 787 | if (bt_empty(bt)) | ||
| 788 | return 0; | ||
| 789 | |||
| 790 | for (;;) { | ||
| 791 | DECLARE_BITMAP(mask, 64); | ||
| 792 | u32 history = st->jmp_history_cnt; | ||
| 793 | struct bpf_jmp_history_entry *hist; | ||
| 794 | |||
| 795 | if (env->log.level & BPF_LOG_LEVEL2) { | ||
| 796 | verbose(env, "mark_precise: frame%d: last_idx %d first_idx %d subseq_idx %d \n", | ||
| 797 | bt->frame, last_idx, first_idx, subseq_idx); | ||
| 798 | } | ||
| 799 | |||
| 800 | if (last_idx < 0) { | ||
| 801 | /* we are at the entry into subprog, which | ||
| 802 | * is expected for global funcs, but only if | ||
| 803 | * requested precise registers are R1-R5 | ||
| 804 | * (which are global func's input arguments) | ||
| 805 | */ | ||
| 806 | if (st->curframe == 0 && | ||
| 807 | st->frame[0]->subprogno > 0 && | ||
| 808 | st->frame[0]->callsite == BPF_MAIN_FUNC && | ||
| 809 | bt_stack_mask(bt) == 0 && | ||
| 810 | (bt_reg_mask(bt) & ~BPF_REGMASK_ARGS) == 0) { | ||
| 811 | bitmap_from_u64(mask, bt_reg_mask(bt)); | ||
| 812 | for_each_set_bit(i, mask, 32) { | ||
| 813 | reg = &st->frame[0]->regs[i]; | ||
| 814 | bt_clear_reg(bt, i); | ||
| 815 | if (reg->type == SCALAR_VALUE) { | ||
| 816 | reg->precise = true; | ||
| 817 | *changed = true; | ||
| 818 | } | ||
| 819 | } | ||
| 820 | return 0; | ||
| 821 | } | ||
| 822 | |||
| 823 | verifier_bug(env, "backtracking func entry subprog %d reg_mask %x stack_mask %llx", | ||
| 824 | st->frame[0]->subprogno, bt_reg_mask(bt), bt_stack_mask(bt)); | ||
| 825 | return -EFAULT; | ||
| 826 | } | ||
| 827 | |||
| 828 | for (i = last_idx;;) { | ||
| 829 | if (skip_first) { | ||
| 830 | err = 0; | ||
| 831 | skip_first = false; | ||
| 832 | } else { | ||
| 833 | hist = get_jmp_hist_entry(st, history, i); | ||
| 834 | err = backtrack_insn(env, i, subseq_idx, hist, bt); | ||
| 835 | } | ||
| 836 | if (err == -ENOTSUPP) { | ||
| 837 | bpf_mark_all_scalars_precise(env, starting_state); | ||
| 838 | bt_reset(bt); | ||
| 839 | return 0; | ||
| 840 | } else if (err) { | ||
| 841 | return err; | ||
| 842 | } | ||
| 843 | if (bt_empty(bt)) | ||
| 844 | /* Found assignment(s) into tracked register in this state. | ||
| 845 | * Since this state is already marked, just return. | ||
| 846 | * Nothing to be tracked further in the parent state. | ||
| 847 | */ | ||
| 848 | return 0; | ||
| 849 | subseq_idx = i; | ||
| 850 | i = get_prev_insn_idx(st, i, &history); | ||
| 851 | if (i == -ENOENT) | ||
| 852 | break; | ||
| 853 | if (i >= env->prog->len) { | ||
| 854 | /* This can happen if backtracking reached insn 0 | ||
| 855 | * and there are still reg_mask or stack_mask | ||
| 856 | * to backtrack. | ||
| 857 | * It means the backtracking missed the spot where | ||
| 858 | * particular register was initialized with a constant. | ||
| 859 | */ | ||
| 860 | verifier_bug(env, "backtracking idx %d", i); | ||
| 861 | return -EFAULT; | ||
| 862 | } | ||
| 863 | } | ||
| 864 | st = st->parent; | ||
| 865 | if (!st) | ||
| 866 | break; | ||
| 867 | |||
| 868 | for (fr = bt->frame; fr >= 0; fr--) { | ||
| 869 | func = st->frame[fr]; | ||
| 870 | bitmap_from_u64(mask, bt_frame_reg_mask(bt, fr)); | ||
| 871 | for_each_set_bit(i, mask, 32) { | ||
| 872 | reg = &func->regs[i]; | ||
| 873 | if (reg->type != SCALAR_VALUE) { | ||
| 874 | bt_clear_frame_reg(bt, fr, i); | ||
| 875 | continue; | ||
| 876 | } | ||
| 877 | if (reg->precise) { | ||
| 878 | bt_clear_frame_reg(bt, fr, i); | ||
| 879 | } else { | ||
| 880 | reg->precise = true; | ||
| 881 | *changed = true; | ||
| 882 | } | ||
| 883 | } | ||
| 884 | |||
| 885 | bitmap_from_u64(mask, bt_frame_stack_mask(bt, fr)); | ||
| 886 | for_each_set_bit(i, mask, 64) { | ||
| 887 | if (verifier_bug_if(i >= func->allocated_stack / BPF_REG_SIZE, | ||
| 888 | env, "stack slot %d, total slots %d", | ||
| 889 | i, func->allocated_stack / BPF_REG_SIZE)) | ||
| 890 | return -EFAULT; | ||
| 891 | |||
| 892 | if (!bpf_is_spilled_scalar_reg(&func->stack[i])) { | ||
| 893 | bt_clear_frame_slot(bt, fr, i); | ||
| 894 | continue; | ||
| 895 | } | ||
| 896 | reg = &func->stack[i].spilled_ptr; | ||
| 897 | if (reg->precise) { | ||
| 898 | bt_clear_frame_slot(bt, fr, i); | ||
| 899 | } else { | ||
| 900 | reg->precise = true; | ||
| 901 | *changed = true; | ||
| 902 | } | ||
| 903 | } | ||
| 904 | if (env->log.level & BPF_LOG_LEVEL2) { | ||
| 905 | fmt_reg_mask(env->tmp_str_buf, TMP_STR_BUF_LEN, | ||
| 906 | bt_frame_reg_mask(bt, fr)); | ||
| 907 | verbose(env, "mark_precise: frame%d: parent state regs=%s ", | ||
| 908 | fr, env->tmp_str_buf); | ||
| 909 | bpf_fmt_stack_mask(env->tmp_str_buf, TMP_STR_BUF_LEN, | ||
| 910 | bt_frame_stack_mask(bt, fr)); | ||
| 911 | verbose(env, "stack=%s: ", env->tmp_str_buf); | ||
| 912 | print_verifier_state(env, st, fr, true); | ||
| 913 | } | ||
| 914 | } | ||
| 915 | |||
| 916 | if (bt_empty(bt)) | ||
| 917 | return 0; | ||
| 918 | |||
| 919 | subseq_idx = first_idx; | ||
| 920 | last_idx = st->last_insn_idx; | ||
| 921 | first_idx = st->first_insn_idx; | ||
| 922 | } | ||
| 923 | |||
| 924 | /* if we still have requested precise regs or slots, we missed | ||
| 925 | * something (e.g., stack access through non-r10 register), so | ||
| 926 | * fallback to marking all precise | ||
| 927 | */ | ||
| 928 | if (!bt_empty(bt)) { | ||
| 929 | bpf_mark_all_scalars_precise(env, starting_state); | ||
| 930 | bt_reset(bt); | ||
| 931 | } | ||
| 932 | |||
| 933 | return 0; | ||
| 934 | } | ||
| @@ -513,20 +513,6 @@ static bool helper_multiple_ref_obj_use(enum bpf_func_id func_id, · 512 unmodified lines | |||
| 513 | return ref_obj_uses > 1; | 513 | return ref_obj_uses > 1; |
| 514 | } | 514 | } |
| 515 | 515 | ||
| 516 | static bool is_atomic_load_insn(const struct bpf_insn *insn) | ||
| 517 | { | ||
| 518 | return BPF_CLASS(insn->code) == BPF_STX && | ||
| 519 | BPF_MODE(insn->code) == BPF_ATOMIC && | ||
| 520 | insn->imm == BPF_LOAD_ACQ; | ||
| 521 | } | ||
| 522 | |||
| 523 | static bool is_atomic_fetch_insn(const struct bpf_insn *insn) | ||
| 524 | { | ||
| 525 | return BPF_CLASS(insn->code) == BPF_STX && | ||
| 526 | BPF_MODE(insn->code) == BPF_ATOMIC && | ||
| 527 | (insn->imm & BPF_FETCH); | ||
| 528 | } | ||
| 529 | |||
| 530 | 516 | ||
| 531 | static bool is_spi_bounds_valid(struct bpf_func_state *state, int spi, int nr_slots) | 517 | static bool is_spi_bounds_valid(struct bpf_func_state *state, int spi, int nr_slots) |
| 532 | { | 518 | { |
| @@ -1241,11 +1227,6 @@ static bool is_stack_slot_special(const struct bpf_stack_state *stack) · 708 unmodified lines | |||
| 1241 | /* The reg state of a pointer or a bounded scalar was saved when | 1227 | /* The reg state of a pointer or a bounded scalar was saved when |
| 1242 | * it was spilled to the stack. | 1228 | * it was spilled to the stack. |
| 1243 | */ | 1229 | */ |
| 1244 | static bool is_spilled_scalar_reg(const struct bpf_stack_state *stack) | ||
| 1245 | { | ||
| 1246 | return stack->slot_type[BPF_REG_SIZE - 1] == STACK_SPILL && | ||
| 1247 | stack->spilled_ptr.type == SCALAR_VALUE; | ||
| 1248 | } | ||
| 1249 | 1230 | ||
| 1250 | /* | 1231 | /* |
| 1251 | * Mark stack slot as STACK_MISC, unless it is already: | 1232 | * Mark stack slot as STACK_MISC, unless it is already: |
| @@ -2590,7 +2571,6 @@ static struct bpf_retval_range retval_range(s32 minval, s32 maxval) · 1338 unmodified lines | |||
| 2590 | return (struct bpf_retval_range){ minval, maxval, false }; | 2571 | return (struct bpf_retval_range){ minval, maxval, false }; |
| 2591 | } | 2572 | } |
| 2592 | 2573 | ||
| 2593 | #define BPF_MAIN_FUNC (-1) | ||
| 2594 | static void init_func_state(struct bpf_verifier_env *env, | 2574 | static void init_func_state(struct bpf_verifier_env *env, |
| 2595 | struct bpf_func_state *state, | 2575 | struct bpf_func_state *state, |
| 2596 | int callsite, int frameno, int subprogno) | 2576 | int callsite, int frameno, int subprogno) |
| @@ -3517,16 +3497,6 @@ static int insn_stack_access_flags(int frameno, int spi) · 920 unmodified lines | |||
| 3517 | return INSN_F_STACK_ACCESS | (spi << INSN_F_SPI_SHIFT) | frameno; | 3497 | return INSN_F_STACK_ACCESS | (spi << INSN_F_SPI_SHIFT) | frameno; |
| 3518 | } | 3498 | } |
| 3519 | 3499 | ||
| 3520 | static int insn_stack_access_spi(int insn_flags) | ||
| 3521 | { | ||
| 3522 | return (insn_flags >> INSN_F_SPI_SHIFT) & INSN_F_SPI_MASK; | ||
| 3523 | } | ||
| 3524 | |||
| 3525 | static int insn_stack_access_frameno(int insn_flags) | ||
| 3526 | { | ||
| 3527 | return insn_flags & INSN_F_FRAMENO_MASK; | ||
| 3528 | } | ||
| 3529 | |||
| 3530 | #define LR_FRAMENO_BITS 3 | 3500 | #define LR_FRAMENO_BITS 3 |
| 3531 | #define LR_SPI_BITS 6 | 3501 | #define LR_SPI_BITS 6 |
| 3532 | #define LR_ENTRY_BITS (LR_SPI_BITS + LR_FRAMENO_BITS + 1) | 3502 | #define LR_ENTRY_BITS (LR_SPI_BITS + LR_FRAMENO_BITS + 1) |
| @@ -3605,91 +3575,6 @@ static void linked_regs_unpack(u64 val, struct linked_regs *s) · 72 unmodified lines | |||
| 3605 | } | 3575 | } |
| 3606 | } | 3576 | } |
| 3607 | 3577 | ||
| 3608 | /* for any branch, call, exit record the history of jmps in the given state */ | ||
| 3609 | int bpf_push_jmp_history(struct bpf_verifier_env *env, struct bpf_verifier_state *cur, | ||
| 3610 | int insn_flags, u64 linked_regs) | ||
| 3611 | { | ||
| 3612 | u32 cnt = cur->jmp_history_cnt; | ||
| 3613 | struct bpf_jmp_history_entry *p; | ||
| 3614 | size_t alloc_size; | ||
| 3615 | |||
| 3616 | /* combine instruction flags if we already recorded this instruction */ | ||
| 3617 | if (env->cur_hist_ent) { | ||
| 3618 | /* atomic instructions push insn_flags twice, for READ and | ||
| 3619 | * WRITE sides, but they should agree on stack slot | ||
| 3620 | */ | ||
| 3621 | verifier_bug_if((env->cur_hist_ent->flags & insn_flags) && | ||
| 3622 | (env->cur_hist_ent->flags & insn_flags) != insn_flags, | ||
| 3623 | env, "insn history: insn_idx %d cur flags %x new flags %x", | ||
| 3624 | env->insn_idx, env->cur_hist_ent->flags, insn_flags); | ||
| 3625 | env->cur_hist_ent->flags |= insn_flags; | ||
| 3626 | verifier_bug_if(env->cur_hist_ent->linked_regs != 0, env, | ||
| 3627 | "insn history: insn_idx %d linked_regs: %#llx", | ||
| 3628 | env->insn_idx, env->cur_hist_ent->linked_regs); | ||
| 3629 | env->cur_hist_ent->linked_regs = linked_regs; | ||
| 3630 | return 0; | ||
| 3631 | } | ||
| 3632 | |||
| 3633 | cnt++; | ||
| 3634 | alloc_size = kmalloc_size_roundup(size_mul(cnt, sizeof(*p))); | ||
| 3635 | p = krealloc(cur->jmp_history, alloc_size, GFP_KERNEL_ACCOUNT); | ||
| 3636 | if (!p) | ||
| 3637 | return -ENOMEM; | ||
| 3638 | cur->jmp_history = p; | ||
| 3639 | |||
| 3640 | p = &cur->jmp_history[cnt - 1]; | ||
| 3641 | p->idx = env->insn_idx; | ||
| 3642 | p->prev_idx = env->prev_insn_idx; | ||
| 3643 | p->flags = insn_flags; | ||
| 3644 | p->linked_regs = linked_regs; | ||
| 3645 | cur->jmp_history_cnt = cnt; | ||
| 3646 | env->cur_hist_ent = p; | ||
| 3647 | |||
| 3648 | return 0; | ||
| 3649 | } | ||
| 3650 | |||
| 3651 | static struct bpf_jmp_history_entry *get_jmp_hist_entry(struct bpf_verifier_state *st, | ||
| 3652 | u32 hist_end, int insn_idx) | ||
| 3653 | { | ||
| 3654 | if (hist_end > 0 && st->jmp_history[hist_end - 1].idx == insn_idx) | ||
| 3655 | return &st->jmp_history[hist_end - 1]; | ||
| 3656 | return NULL; | ||
| 3657 | } | ||
| 3658 | |||
| 3659 | /* Backtrack one insn at a time. If idx is not at the top of recorded | ||
| 3660 | * history then previous instruction came from straight line execution. | ||
| 3661 | * Return -ENOENT if we exhausted all instructions within given state. | ||
| 3662 | * | ||
| 3663 | * It's legal to have a bit of a looping with the same starting and ending | ||
| 3664 | * insn index within the same state, e.g.: 3->4->5->3, so just because current | ||
| 3665 | * instruction index is the same as state's first_idx doesn't mean we are | ||
| 3666 | * done. If there is still some jump history left, we should keep going. We | ||
| 3667 | * need to take into account that we might have a jump history between given | ||
| 3668 | * state's parent and itself, due to checkpointing. In this case, we'll have | ||
| 3669 | * history entry recording a jump from last instruction of parent state and | ||
| 3670 | * first instruction of given state. | ||
| 3671 | */ | ||
| 3672 | static int get_prev_insn_idx(struct bpf_verifier_state *st, int i, | ||
| 3673 | u32 *history) | ||
| 3674 | { | ||
| 3675 | u32 cnt = *history; | ||
| 3676 | |||
| 3677 | if (i == st->first_insn_idx) { | ||
| 3678 | if (cnt == 0) | ||
| 3679 | return -ENOENT; | ||
| 3680 | if (cnt == 1 && st->jmp_history[0].idx == i) | ||
| 3681 | return -ENOENT; | ||
| 3682 | } | ||
| 3683 | |||
| 3684 | if (cnt && st->jmp_history[cnt - 1].idx == i) { | ||
| 3685 | i = st->jmp_history[cnt - 1].prev_idx; | ||
| 3686 | (*history)--; | ||
| 3687 | } else { | ||
| 3688 | i--; | ||
| 3689 | } | ||
| 3690 | return i; | ||
| 3691 | } | ||
| 3692 | |||
| 3693 | static const char *disasm_kfunc_name(void *data, const struct bpf_insn *insn) | 3578 | static const char *disasm_kfunc_name(void *data, const struct bpf_insn *insn) |
| 3694 | { | 3579 | { |
| 3695 | const struct btf_type *func; | 3580 | const struct btf_type *func; |
| @@ -3717,148 +3602,10 @@ void bpf_verbose_insn(struct bpf_verifier_env *env, struct bpf_insn *insn) · 21 unmodified lines | |||
| 3717 | print_bpf_insn(&cbs, insn, env->allow_ptr_leaks); | 3602 | print_bpf_insn(&cbs, insn, env->allow_ptr_leaks); |
| 3718 | } | 3603 | } |
| 3719 | 3604 | ||
| 3720 | static inline void bt_init(struct backtrack_state *bt, u32 frame) | ||
| 3721 | { | ||
| 3722 | bt->frame = frame; | ||
| 3723 | } | ||
| 3724 | |||
| 3725 | static inline void bt_reset(struct backtrack_state *bt) | ||
| 3726 | { | ||
| 3727 | struct bpf_verifier_env *env = bt->env; | ||
| 3728 | |||
| 3729 | memset(bt, 0, sizeof(*bt)); | ||
| 3730 | bt->env = env; | ||
| 3731 | } | ||
| 3732 | |||
| 3733 | static inline u32 bt_empty(struct backtrack_state *bt) | ||
| 3734 | { | ||
| 3735 | u64 mask = 0; | ||
| 3736 | int i; | ||
| 3737 | |||
| 3738 | for (i = 0; i <= bt->frame; i++) | ||
| 3739 | mask |= bt->reg_masks[i] | bt->stack_masks[i]; | ||
| 3740 | |||
| 3741 | return mask == 0; | ||
| 3742 | } | ||
| 3743 | |||
| 3744 | static inline int bt_subprog_enter(struct backtrack_state *bt) | ||
| 3745 | { | ||
| 3746 | if (bt->frame == MAX_CALL_FRAMES - 1) { | ||
| 3747 | verifier_bug(bt->env, "subprog enter from frame %d", bt->frame); | ||
| 3748 | return -EFAULT; | ||
| 3749 | } | ||
| 3750 | bt->frame++; | ||
| 3751 | return 0; | ||
| 3752 | } | ||
| 3753 | |||
| 3754 | static inline int bt_subprog_exit(struct backtrack_state *bt) | ||
| 3755 | { | ||
| 3756 | if (bt->frame == 0) { | ||
| 3757 | verifier_bug(bt->env, "subprog exit from frame 0"); | ||
| 3758 | return -EFAULT; | ||
| 3759 | } | ||
| 3760 | bt->frame--; | ||
| 3761 | return 0; | ||
| 3762 | } | ||
| 3763 | |||
| 3764 | static inline void bt_clear_frame_reg(struct backtrack_state *bt, u32 frame, u32 reg) | ||
| 3765 | { | ||
| 3766 | bt->reg_masks[frame] &= ~(1 << reg); | ||
| 3767 | } | ||
| 3768 | |||
| 3769 | static inline void bt_set_reg(struct backtrack_state *bt, u32 reg) | ||
| 3770 | { | ||
| 3771 | bpf_bt_set_frame_reg(bt, bt->frame, reg); | ||
| 3772 | } | ||
| 3773 | |||
| 3774 | static inline void bt_clear_reg(struct backtrack_state *bt, u32 reg) | ||
| 3775 | { | ||
| 3776 | bt_clear_frame_reg(bt, bt->frame, reg); | ||
| 3777 | } | ||
| 3778 | |||
| 3779 | static inline void bt_clear_frame_slot(struct backtrack_state *bt, u32 frame, u32 slot) | ||
| 3780 | { | ||
| 3781 | bt->stack_masks[frame] &= ~(1ull << slot); | ||
| 3782 | } | ||
| 3783 | |||
| 3784 | static inline u32 bt_frame_reg_mask(struct backtrack_state *bt, u32 frame) | ||
| 3785 | { | ||
| 3786 | return bt->reg_masks[frame]; | ||
| 3787 | } | ||
| 3788 | |||
| 3789 | static inline u32 bt_reg_mask(struct backtrack_state *bt) | ||
| 3790 | { | ||
| 3791 | return bt->reg_masks[bt->frame]; | ||
| 3792 | } | ||
| 3793 | |||
| 3794 | static inline u64 bt_frame_stack_mask(struct backtrack_state *bt, u32 frame) | ||
| 3795 | { | ||
| 3796 | return bt->stack_masks[frame]; | ||
| 3797 | } | ||
| 3798 | |||
| 3799 | static inline u64 bt_stack_mask(struct backtrack_state *bt) | ||
| 3800 | { | ||
| 3801 | return bt->stack_masks[bt->frame]; | ||
| 3802 | } | ||
| 3803 | |||
| 3804 | static inline bool bt_is_reg_set(struct backtrack_state *bt, u32 reg) | ||
| 3805 | { | ||
| 3806 | return bt->reg_masks[bt->frame] & (1 << reg); | ||
| 3807 | } | ||
| 3808 | |||
| 3809 | static inline bool bt_is_frame_reg_set(struct backtrack_state *bt, u32 frame, u32 reg) | ||
| 3810 | { | ||
| 3811 | return bt->reg_masks[frame] & (1 << reg); | ||
| 3812 | } | ||
| 3813 | |||
| 3814 | static inline bool bt_is_frame_slot_set(struct backtrack_state *bt, u32 frame, u32 slot) | ||
| 3815 | { | ||
| 3816 | return bt->stack_masks[frame] & (1ull << slot); | ||
| 3817 | } | ||
| 3818 | |||
| 3819 | /* format registers bitmask, e.g., "r0,r2,r4" for 0x15 mask */ | ||
| 3820 | static void fmt_reg_mask(char *buf, ssize_t buf_sz, u32 reg_mask) | ||
| 3821 | { | ||
| 3822 | DECLARE_BITMAP(mask, 64); | ||
| 3823 | bool first = true; | ||
| 3824 | int i, n; | ||
| 3825 | |||
| 3826 | buf[0] = '\0'; | ||
| 3827 | |||
| 3828 | bitmap_from_u64(mask, reg_mask); | ||
| 3829 | for_each_set_bit(i, mask, 32) { | ||
| 3830 | n = snprintf(buf, buf_sz, "%sr%d", first ? "" : ",", i); | ||
| 3831 | first = false; | ||
| 3832 | buf += n; | ||
| 3833 | buf_sz -= n; | ||
| 3834 | if (buf_sz < 0) | ||
| 3835 | break; | ||
| 3836 | } | ||
| 3837 | } | ||
| 3838 | /* format stack slots bitmask, e.g., "-8,-24,-40" for 0x15 mask */ | ||
| 3839 | void bpf_fmt_stack_mask(char *buf, ssize_t buf_sz, u64 stack_mask) | ||
| 3840 | { | ||
| 3841 | DECLARE_BITMAP(mask, 64); | ||
| 3842 | bool first = true; | ||
| 3843 | int i, n; | ||
| 3844 | |||
| 3845 | buf[0] = '\0'; | ||
| 3846 | |||
| 3847 | bitmap_from_u64(mask, stack_mask); | ||
| 3848 | for_each_set_bit(i, mask, 64) { | ||
| 3849 | n = snprintf(buf, buf_sz, "%s%d", first ? "" : ",", -(i + 1) * 8); | ||
| 3850 | first = false; | ||
| 3851 | buf += n; | ||
| 3852 | buf_sz -= n; | ||
| 3853 | if (buf_sz < 0) | ||
| 3854 | break; | ||
| 3855 | } | ||
| 3856 | } | ||
| 3857 | |||
| 3858 | /* If any register R in hist->linked_regs is marked as precise in bt, | 3605 | /* If any register R in hist->linked_regs is marked as precise in bt, |
| 3859 | * do bt_set_frame_{reg,slot}(bt, R) for all registers in hist->linked_regs. | 3606 | * do bt_set_frame_{reg,slot}(bt, R) for all registers in hist->linked_regs. |
| 3860 | */ | 3607 | */ |
| 3861 | static void bt_sync_linked_regs(struct backtrack_state *bt, struct bpf_jmp_history_entry *hist) | 3608 | void bpf_bt_sync_linked_regs(struct backtrack_state *bt, struct bpf_jmp_history_entry *hist) |
| 3862 | { | 3609 | { |
| 3863 | struct linked_regs linked_regs; | 3610 | struct linked_regs linked_regs; |
| 3864 | bool some_precise = false; | 3611 | bool some_precise = false; |
| @@ -3891,693 +3638,6 @@ static void bt_sync_linked_regs(struct backtrack_state *bt, struct bpf_jmp_histo · 26 unmodified lines | |||
| 3891 | } | 3638 | } |
| 3892 | } | 3639 | } |
| 3893 | 3640 | ||
| 3894 | /* For given verifier state backtrack_insn() is called from the last insn to | ||
| 3895 | * the first insn. Its purpose is to compute a bitmask of registers and | ||
| 3896 | * stack slots that needs precision in the parent verifier state. | ||
| 3897 | * | ||
| 3898 | * @idx is an index of the instruction we are currently processing; | ||
| 3899 | * @subseq_idx is an index of the subsequent instruction that: | ||
| 3900 | * - *would be* executed next, if jump history is viewed in forward order; | ||
| 3901 | * - *was* processed previously during backtracking. | ||
| 3902 | */ | ||
| 3903 | static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, | ||
| 3904 | struct bpf_jmp_history_entry *hist, struct backtrack_state *bt) | ||
| 3905 | { | ||
| 3906 | struct bpf_insn *insn = env->prog->insnsi + idx; | ||
| 3907 | u8 class = BPF_CLASS(insn->code); | ||
| 3908 | u8 opcode = BPF_OP(insn->code); | ||
| 3909 | u8 mode = BPF_MODE(insn->code); | ||
| 3910 | u32 dreg = insn->dst_reg; | ||
| 3911 | u32 sreg = insn->src_reg; | ||
| 3912 | u32 spi, i, fr; | ||
| 3913 | |||
| 3914 | if (insn->code == 0) | ||
| 3915 | return 0; | ||
| 3916 | if (env->log.level & BPF_LOG_LEVEL2) { | ||
| 3917 | fmt_reg_mask(env->tmp_str_buf, TMP_STR_BUF_LEN, bt_reg_mask(bt)); | ||
| 3918 | verbose(env, "mark_precise: frame%d: regs=%s ", | ||
| 3919 | bt->frame, env->tmp_str_buf); | ||
| 3920 | bpf_fmt_stack_mask(env->tmp_str_buf, TMP_STR_BUF_LEN, bt_stack_mask(bt)); | ||
| 3921 | verbose(env, "stack=%s before ", env->tmp_str_buf); | ||
| 3922 | verbose(env, "%d: ", idx); | ||
| 3923 | bpf_verbose_insn(env, insn); | ||
| 3924 | } | ||
| 3925 | |||
| 3926 | /* If there is a history record that some registers gained range at this insn, | ||
| 3927 | * propagate precision marks to those registers, so that bt_is_reg_set() | ||
| 3928 | * accounts for these registers. | ||
| 3929 | */ | ||
| 3930 | bt_sync_linked_regs(bt, hist); | ||
| 3931 | |||
| 3932 | if (class == BPF_ALU || class == BPF_ALU64) { | ||
| 3933 | if (!bt_is_reg_set(bt, dreg)) | ||
| 3934 | return 0; | ||
| 3935 | if (opcode == BPF_END || opcode == BPF_NEG) { | ||
| 3936 | /* sreg is reserved and unused | ||
| 3937 | * dreg still need precision before this insn | ||
| 3938 | */ | ||
| 3939 | return 0; | ||
| 3940 | } else if (opcode == BPF_MOV) { | ||
| 3941 | if (BPF_SRC(insn->code) == BPF_X) { | ||
| 3942 | /* dreg = sreg or dreg = (s8, s16, s32)sreg | ||
| 3943 | * dreg needs precision after this insn | ||
| 3944 | * sreg needs precision before this insn | ||
| 3945 | */ | ||
| 3946 | bt_clear_reg(bt, dreg); | ||
| 3947 | if (sreg != BPF_REG_FP) | ||
| 3948 | bt_set_reg(bt, sreg); | ||
| 3949 | } else { | ||
| 3950 | /* dreg = K | ||
| 3951 | * dreg needs precision after this insn. | ||
| 3952 | * Corresponding register is already marked | ||
| 3953 | * as precise=true in this verifier state. | ||
| 3954 | * No further markings in parent are necessary | ||
| 3955 | */ | ||
| 3956 | bt_clear_reg(bt, dreg); | ||
| 3957 | } | ||
| 3958 | } else { | ||
| 3959 | if (BPF_SRC(insn->code) == BPF_X) { | ||
| 3960 | /* dreg += sreg | ||
| 3961 | * both dreg and sreg need precision | ||
| 3962 | * before this insn | ||
| 3963 | */ | ||
| 3964 | if (sreg != BPF_REG_FP) | ||
| 3965 | bt_set_reg(bt, sreg); | ||
| 3966 | } /* else dreg += K | ||
| 3967 | * dreg still needs precision before this insn | ||
| 3968 | */ | ||
| 3969 | } | ||
| 3970 | } else if (class == BPF_LDX || | ||
| 3971 | is_atomic_load_insn(insn) || | ||
| 3972 | is_atomic_fetch_insn(insn)) { | ||
| 3973 | u32 load_reg = dreg; | ||
| 3974 | |||
| 3975 | /* | ||
| 3976 | * Atomic fetch operation writes the old value into | ||
| 3977 | * a register (sreg or r0) and if it was tracked for | ||
| 3978 | * precision, propagate to the stack slot like we do | ||
| 3979 | * in regular ldx. | ||
| 3980 | */ | ||
| 3981 | if (is_atomic_fetch_insn(insn)) | ||
| 3982 | load_reg = insn->imm == BPF_CMPXCHG ? | ||
| 3983 | BPF_REG_0 : sreg; | ||
| 3984 | |||
| 3985 | if (!bt_is_reg_set(bt, load_reg)) | ||
| 3986 | return 0; | ||
| 3987 | bt_clear_reg(bt, load_reg); | ||
| 3988 | |||
| 3989 | /* scalars can only be spilled into stack w/o losing precision. | ||
| 3990 | * Load from any other memory can be zero extended. | ||
| 3991 | * The desire to keep that precision is already indicated | ||
| 3992 | * by 'precise' mark in corresponding register of this state. | ||
| 3993 | * No further tracking necessary. | ||
| 3994 | */ | ||
| 3995 | if (!hist || !(hist->flags & INSN_F_STACK_ACCESS)) | ||
| 3996 | return 0; | ||
| 3997 | /* dreg = *(u64 *)[fp - off] was a fill from the stack. | ||
| 3998 | * that [fp - off] slot contains scalar that needs to be | ||
| 3999 | * tracked with precision | ||
| 4000 | */ | ||
| 4001 | spi = insn_stack_access_spi(hist->flags); | ||
| 4002 | fr = insn_stack_access_frameno(hist->flags); | ||
| 4003 | bpf_bt_set_frame_slot(bt, fr, spi); | ||
| 4004 | } else if (class == BPF_STX || class == BPF_ST) { | ||
| 4005 | if (bt_is_reg_set(bt, dreg)) | ||
| 4006 | /* stx & st shouldn't be using _scalar_ dst_reg | ||
| 4007 | * to access memory. It means backtracking | ||
| 4008 | * encountered a case of pointer subtraction. | ||
| 4009 | */ | ||
| 4010 | return -ENOTSUPP; | ||
| 4011 | /* scalars can only be spilled into stack */ | ||
| 4012 | if (!hist || !(hist->flags & INSN_F_STACK_ACCESS)) | ||
| 4013 | return 0; | ||
| 4014 | spi = insn_stack_access_spi(hist->flags); | ||
| 4015 | fr = insn_stack_access_frameno(hist->flags); | ||
| 4016 | if (!bt_is_frame_slot_set(bt, fr, spi)) | ||
| 4017 | return 0; | ||
| 4018 | bt_clear_frame_slot(bt, fr, spi); | ||
| 4019 | if (class == BPF_STX) | ||
| 4020 | bt_set_reg(bt, sreg); | ||
| 4021 | } else if (class == BPF_JMP || class == BPF_JMP32) { | ||
| 4022 | if (bpf_pseudo_call(insn)) { | ||
| 4023 | int subprog_insn_idx, subprog; | ||
| 4024 | |||
| 4025 | subprog_insn_idx = idx + insn->imm + 1; | ||
| 4026 | subprog = bpf_find_subprog(env, subprog_insn_idx); | ||
| 4027 | if (subprog < 0) | ||
| 4028 | return -EFAULT; | ||
| 4029 | |||
| 4030 | if (bpf_subprog_is_global(env, subprog)) { | ||
| 4031 | /* check that jump history doesn't have any | ||
| 4032 | * extra instructions from subprog; the next | ||
| 4033 | * instruction after call to global subprog | ||
| 4034 | * should be literally next instruction in | ||
| 4035 | * caller program | ||
| 4036 | */ | ||
| 4037 | verifier_bug_if(idx + 1 != subseq_idx, env, | ||
| 4038 | "extra insn from subprog"); | ||
| 4039 | /* r1-r5 are invalidated after subprog call, | ||
| 4040 | * so for global func call it shouldn't be set | ||
| 4041 | * anymore | ||
| 4042 | */ | ||
| 4043 | if (bt_reg_mask(bt) & BPF_REGMASK_ARGS) { | ||
| 4044 | verifier_bug(env, "global subprog unexpected regs %x", | ||
| 4045 | bt_reg_mask(bt)); | ||
| 4046 | return -EFAULT; | ||
| 4047 | } | ||
| 4048 | /* global subprog always sets R0 */ | ||
| 4049 | bt_clear_reg(bt, BPF_REG_0); | ||
| 4050 | return 0; | ||
| 4051 | } else { | ||
| 4052 | /* static subprog call instruction, which | ||
| 4053 | * means that we are exiting current subprog, | ||
| 4054 | * so only r1-r5 could be still requested as | ||
| 4055 | * precise, r0 and r6-r10 or any stack slot in | ||
| 4056 | * the current frame should be zero by now | ||
| 4057 | */ | ||
| 4058 | if (bt_reg_mask(bt) & ~BPF_REGMASK_ARGS) { | ||
| 4059 | verifier_bug(env, "static subprog unexpected regs %x", | ||
| 4060 | bt_reg_mask(bt)); | ||
| 4061 | return -EFAULT; | ||
| 4062 | } | ||
| 4063 | /* we are now tracking register spills correctly, | ||
| 4064 | * so any instance of leftover slots is a bug | ||
| 4065 | */ | ||
| 4066 | if (bt_stack_mask(bt) != 0) { | ||
| 4067 | verifier_bug(env, | ||
| 4068 | "static subprog leftover stack slots %llx", | ||
| 4069 | bt_stack_mask(bt)); | ||
| 4070 | return -EFAULT; | ||
| 4071 | } | ||
| 4072 | /* propagate r1-r5 to the caller */ | ||
| 4073 | for (i = BPF_REG_1; i <= BPF_REG_5; i++) { | ||
| 4074 | if (bt_is_reg_set(bt, i)) { | ||
| 4075 | bt_clear_reg(bt, i); | ||
| 4076 | bpf_bt_set_frame_reg(bt, bt->frame - 1, i); | ||
| 4077 | } | ||
| 4078 | } | ||
| 4079 | if (bt_subprog_exit(bt)) | ||
| 4080 | return -EFAULT; | ||
| 4081 | return 0; | ||
| 4082 | } | ||
| 4083 | } else if (bpf_is_sync_callback_calling_insn(insn) && idx != subseq_idx - 1) { | ||
| 4084 | /* exit from callback subprog to callback-calling helper or | ||
| 4085 | * kfunc call. Use idx/subseq_idx check to discern it from | ||
| 4086 | * straight line code backtracking. | ||
| 4087 | * Unlike the subprog call handling above, we shouldn't | ||
| 4088 | * propagate precision of r1-r5 (if any requested), as they are | ||
| 4089 | * not actually arguments passed directly to callback subprogs | ||
| 4090 | */ | ||
| 4091 | if (bt_reg_mask(bt) & ~BPF_REGMASK_ARGS) { | ||
| 4092 | verifier_bug(env, "callback unexpected regs %x", | ||
| 4093 | bt_reg_mask(bt)); | ||
| 4094 | return -EFAULT; | ||
| 4095 | } | ||
| 4096 | if (bt_stack_mask(bt) != 0) { | ||
| 4097 | verifier_bug(env, "callback leftover stack slots %llx", | ||
| 4098 | bt_stack_mask(bt)); | ||
| 4099 | return -EFAULT; | ||
| 4100 | } | ||
| 4101 | /* clear r1-r5 in callback subprog's mask */ | ||
| 4102 | for (i = BPF_REG_1; i <= BPF_REG_5; i++) | ||
| 4103 | bt_clear_reg(bt, i); | ||
| 4104 | if (bt_subprog_exit(bt)) | ||
| 4105 | return -EFAULT; | ||
| 4106 | return 0; | ||
| 4107 | } else if (opcode == BPF_CALL) { | ||
| 4108 | /* kfunc with imm==0 is invalid and fixup_kfunc_call will | ||
| 4109 | * catch this error later. Make backtracking conservative | ||
| 4110 | * with ENOTSUPP. | ||
| 4111 | */ | ||
| 4112 | if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL && insn->imm == 0) | ||
| 4113 | return -ENOTSUPP; | ||
| 4114 | /* regular helper call sets R0 */ | ||
| 4115 | bt_clear_reg(bt, BPF_REG_0); | ||
| 4116 | if (bt_reg_mask(bt) & BPF_REGMASK_ARGS) { | ||
| 4117 | /* if backtracking was looking for registers R1-R5 | ||
| 4118 | * they should have been found already. | ||
| 4119 | */ | ||
| 4120 | verifier_bug(env, "backtracking call unexpected regs %x", | ||
| 4121 | bt_reg_mask(bt)); | ||
| 4122 | return -EFAULT; | ||
| 4123 | } | ||
| 4124 | if (insn->src_reg == BPF_REG_0 && insn->imm == BPF_FUNC_tail_call | ||
| 4125 | && subseq_idx - idx != 1) { | ||
| 4126 | if (bt_subprog_enter(bt)) | ||
| 4127 | return -EFAULT; | ||
| 4128 | } | ||
| 4129 | } else if (opcode == BPF_EXIT) { | ||
| 4130 | bool r0_precise; | ||
| 4131 | |||
| 4132 | /* Backtracking to a nested function call, 'idx' is a part of | ||
| 4133 | * the inner frame 'subseq_idx' is a part of the outer frame. | ||
| 4134 | * In case of a regular function call, instructions giving | ||
| 4135 | * precision to registers R1-R5 should have been found already. | ||
| 4136 | * In case of a callback, it is ok to have R1-R5 marked for | ||
| 4137 | * backtracking, as these registers are set by the function | ||
| 4138 | * invoking callback. | ||
| 4139 | */ | ||
| 4140 | if (subseq_idx >= 0 && bpf_calls_callback(env, subseq_idx)) | ||
| 4141 | for (i = BPF_REG_1; i <= BPF_REG_5; i++) | ||
| 4142 | bt_clear_reg(bt, i); | ||
| 4143 | if (bt_reg_mask(bt) & BPF_REGMASK_ARGS) { | ||
| 4144 | verifier_bug(env, "backtracking exit unexpected regs %x", | ||
| 4145 | bt_reg_mask(bt)); | ||
| 4146 | return -EFAULT; | ||
| 4147 | } | ||
| 4148 | |||
| 4149 | /* BPF_EXIT in subprog or callback always returns | ||
| 4150 | * right after the call instruction, so by checking | ||
| 4151 | * whether the instruction at subseq_idx-1 is subprog | ||
| 4152 | * call or not we can distinguish actual exit from | ||
| 4153 | * *subprog* from exit from *callback*. In the former | ||
| 4154 | * case, we need to propagate r0 precision, if | ||
| 4155 | * necessary. In the former we never do that. | ||
| 4156 | */ | ||
| 4157 | r0_precise = subseq_idx - 1 >= 0 && | ||
| 4158 | bpf_pseudo_call(&env->prog->insnsi[subseq_idx - 1]) && | ||
| 4159 | bt_is_reg_set(bt, BPF_REG_0); | ||
| 4160 | |||
| 4161 | bt_clear_reg(bt, BPF_REG_0); | ||
| 4162 | if (bt_subprog_enter(bt)) | ||
| 4163 | return -EFAULT; | ||
| 4164 | |||
| 4165 | if (r0_precise) | ||
| 4166 | bt_set_reg(bt, BPF_REG_0); | ||
| 4167 | /* r6-r9 and stack slots will stay set in caller frame | ||
| 4168 | * bitmasks until we return back from callee(s) | ||
| 4169 | */ | ||
| 4170 | return 0; | ||
| 4171 | } else if (BPF_SRC(insn->code) == BPF_X) { | ||
| 4172 | if (!bt_is_reg_set(bt, dreg) && !bt_is_reg_set(bt, sreg)) | ||
| 4173 | return 0; | ||
| 4174 | /* dreg <cond> sreg | ||
| 4175 | * Both dreg and sreg need precision before | ||
| 4176 | * this insn. If only sreg was marked precise | ||
| 4177 | * before it would be equally necessary to | ||
| 4178 | * propagate it to dreg. | ||
| 4179 | */ | ||
| 4180 | if (!hist || !(hist->flags & INSN_F_SRC_REG_STACK)) | ||
| 4181 | bt_set_reg(bt, sreg); | ||
| 4182 | if (!hist || !(hist->flags & INSN_F_DST_REG_STACK)) | ||
| 4183 | bt_set_reg(bt, dreg); | ||
| 4184 | } else if (BPF_SRC(insn->code) == BPF_K) { | ||
| 4185 | /* dreg <cond> K | ||
| 4186 | * Only dreg still needs precision before | ||
| 4187 | * this insn, so for the K-based conditional | ||
| 4188 | * there is nothing new to be marked. | ||
| 4189 | */ | ||
| 4190 | } | ||
| 4191 | } else if (class == BPF_LD) { | ||
| 4192 | if (!bt_is_reg_set(bt, dreg)) | ||
| 4193 | return 0; | ||
| 4194 | bt_clear_reg(bt, dreg); | ||
| 4195 | /* It's ld_imm64 or ld_abs or ld_ind. | ||
| 4196 | * For ld_imm64 no further tracking of precision | ||
| 4197 | * into parent is necessary | ||
| 4198 | */ | ||
| 4199 | if (mode == BPF_IND || mode == BPF_ABS) | ||
| 4200 | /* to be analyzed */ | ||
| 4201 | return -ENOTSUPP; | ||
| 4202 | } | ||
| 4203 | /* Propagate precision marks to linked registers, to account for | ||
| 4204 | * registers marked as precise in this function. | ||
| 4205 | */ | ||
| 4206 | bt_sync_linked_regs(bt, hist); | ||
| 4207 | return 0; | ||
| 4208 | } | ||
| 4209 | |||
| 4210 | /* the scalar precision tracking algorithm: | ||
| 4211 | * . at the start all registers have precise=false. | ||
| 4212 | * . scalar ranges are tracked as normal through alu and jmp insns. | ||
| 4213 | * . once precise value of the scalar register is used in: | ||
| 4214 | * . ptr + scalar alu | ||
| 4215 | * . if (scalar cond K|scalar) | ||
| 4216 | * . helper_call(.., scalar, ...) where ARG_CONST is expected | ||
| 4217 | * backtrack through the verifier states and mark all registers and | ||
| 4218 | * stack slots with spilled constants that these scalar registers | ||
| 4219 | * should be precise. | ||
| 4220 | * . during state pruning two registers (or spilled stack slots) | ||
| 4221 | * are equivalent if both are not precise. | ||
| 4222 | * | ||
| 4223 | * Note the verifier cannot simply walk register parentage chain, | ||
| 4224 | * since many different registers and stack slots could have been | ||
| 4225 | * used to compute single precise scalar. | ||
| 4226 | * | ||
| 4227 | * The approach of starting with precise=true for all registers and then | ||
| 4228 | * backtrack to mark a register as not precise when the verifier detects | ||
| 4229 | * that program doesn't care about specific value (e.g., when helper | ||
| 4230 | * takes register as ARG_ANYTHING parameter) is not safe. | ||
| 4231 | * | ||
| 4232 | * It's ok to walk single parentage chain of the verifier states. | ||
| 4233 | * It's possible that this backtracking will go all the way till 1st insn. | ||
| 4234 | * All other branches will be explored for needing precision later. | ||
| 4235 | * | ||
| 4236 | * The backtracking needs to deal with cases like: | ||
| 4237 | * R8=map_value(id=0,off=0,ks=4,vs=1952,imm=0) R9_w=map_value(id=0,off=40,ks=4,vs=1952,imm=0) | ||
| 4238 | * r9 -= r8 | ||
| 4239 | * r5 = r9 | ||
| 4240 | * if r5 > 0x79f goto pc+7 | ||
| 4241 | * R5_w=inv(id=0,umax_value=1951,var_off=(0x0; 0x7ff)) | ||
| 4242 | * r5 += 1 | ||
| 4243 | * ... | ||
| 4244 | * call bpf_perf_event_output#25 | ||
| 4245 | * where .arg5_type = ARG_CONST_SIZE_OR_ZERO | ||
| 4246 | * | ||
| 4247 | * and this case: | ||
| 4248 | * r6 = 1 | ||
| 4249 | * call foo // uses callee's r6 inside to compute r0 | ||
| 4250 | * r0 += r6 | ||
| 4251 | * if r0 == 0 goto | ||
| 4252 | * | ||
| 4253 | * to track above reg_mask/stack_mask needs to be independent for each frame. | ||
| 4254 | * | ||
| 4255 | * Also if parent's curframe > frame where backtracking started, | ||
| 4256 | * the verifier need to mark registers in both frames, otherwise callees | ||
| 4257 | * may incorrectly prune callers. This is similar to | ||
| 4258 | * commit 7640ead93924 ("bpf: verifier: make sure callees don't prune with caller differences") | ||
| 4259 | * | ||
| 4260 | * For now backtracking falls back into conservative marking. | ||
| 4261 | */ | ||
| 4262 | void bpf_mark_all_scalars_precise(struct bpf_verifier_env *env, | ||
| 4263 | struct bpf_verifier_state *st) | ||
| 4264 | { | ||
| 4265 | struct bpf_func_state *func; | ||
| 4266 | struct bpf_reg_state *reg; | ||
| 4267 | int i, j; | ||
| 4268 | |||
| 4269 | if (env->log.level & BPF_LOG_LEVEL2) { | ||
| 4270 | verbose(env, "mark_precise: frame%d: falling back to forcing all scalars precise\n", | ||
| 4271 | st->curframe); | ||
| 4272 | } | ||
| 4273 | |||
| 4274 | /* big hammer: mark all scalars precise in this path. | ||
| 4275 | * pop_stack may still get !precise scalars. | ||
| 4276 | * We also skip current state and go straight to first parent state, | ||
| 4277 | * because precision markings in current non-checkpointed state are | ||
| 4278 | * not needed. See why in the comment in __mark_chain_precision below. | ||
| 4279 | */ | ||
| 4280 | for (st = st->parent; st; st = st->parent) { | ||
| 4281 | for (i = 0; i <= st->curframe; i++) { | ||
| 4282 | func = st->frame[i]; | ||
| 4283 | for (j = 0; j < BPF_REG_FP; j++) { | ||
| 4284 | reg = &func->regs[j]; | ||
| 4285 | if (reg->type != SCALAR_VALUE || reg->precise) | ||
| 4286 | continue; | ||
| 4287 | reg->precise = true; | ||
| 4288 | if (env->log.level & BPF_LOG_LEVEL2) { | ||
| 4289 | verbose(env, "force_precise: frame%d: forcing r%d to be precise\n", | ||
| 4290 | i, j); | ||
| 4291 | } | ||
| 4292 | } | ||
| 4293 | for (j = 0; j < func->allocated_stack / BPF_REG_SIZE; j++) { | ||
| 4294 | if (!bpf_is_spilled_reg(&func->stack[j])) | ||
| 4295 | continue; | ||
| 4296 | reg = &func->stack[j].spilled_ptr; | ||
| 4297 | if (reg->type != SCALAR_VALUE || reg->precise) | ||
| 4298 | continue; | ||
| 4299 | reg->precise = true; | ||
| 4300 | if (env->log.level & BPF_LOG_LEVEL2) { | ||
| 4301 | verbose(env, "force_precise: frame%d: forcing fp%d to be precise\n", | ||
| 4302 | i, -(j + 1) * 8); | ||
| 4303 | } | ||
| 4304 | } | ||
| 4305 | } | ||
| 4306 | } | ||
| 4307 | } | ||
| 4308 | |||
| 4309 | /* | ||
| 4310 | * bpf_mark_chain_precision() backtracks BPF program instruction sequence and | ||
| 4311 | * chain of verifier states making sure that register *regno* (if regno >= 0) | ||
| 4312 | * and/or stack slot *spi* (if spi >= 0) are marked as precisely tracked | ||
| 4313 | * SCALARS, as well as any other registers and slots that contribute to | ||
| 4314 | * a tracked state of given registers/stack slots, depending on specific BPF | ||
| 4315 | * assembly instructions (see backtrack_insns() for exact instruction handling | ||
| 4316 | * logic). This backtracking relies on recorded jmp_history and is able to | ||
| 4317 | * traverse entire chain of parent states. This process ends only when all the | ||
| 4318 | * necessary registers/slots and their transitive dependencies are marked as | ||
| 4319 | * precise. | ||
| 4320 | * | ||
| 4321 | * One important and subtle aspect is that precise marks *do not matter* in | ||
| 4322 | * the currently verified state (current state). It is important to understand | ||
| 4323 | * why this is the case. | ||
| 4324 | * | ||
| 4325 | * First, note that current state is the state that is not yet "checkpointed", | ||
| 4326 | * i.e., it is not yet put into env->explored_states, and it has no children | ||
| 4327 | * states as well. It's ephemeral, and can end up either a) being discarded if | ||
| 4328 | * compatible explored state is found at some point or BPF_EXIT instruction is | ||
| 4329 | * reached or b) checkpointed and put into env->explored_states, branching out | ||
| 4330 | * into one or more children states. | ||
| 4331 | * | ||
| 4332 | * In the former case, precise markings in current state are completely | ||
| 4333 | * ignored by state comparison code (see regsafe() for details). Only | ||
| 4334 | * checkpointed ("old") state precise markings are important, and if old | ||
| 4335 | * state's register/slot is precise, regsafe() assumes current state's | ||
| 4336 | * register/slot as precise and checks value ranges exactly and precisely. If | ||
| 4337 | * states turn out to be compatible, current state's necessary precise | ||
| 4338 | * markings and any required parent states' precise markings are enforced | ||
| 4339 | * after the fact with propagate_precision() logic, after the fact. But it's | ||
| 4340 | * important to realize that in this case, even after marking current state | ||
| 4341 | * registers/slots as precise, we immediately discard current state. So what | ||
| 4342 | * actually matters is any of the precise markings propagated into current | ||
| 4343 | * state's parent states, which are always checkpointed (due to b) case above). | ||
| 4344 | * As such, for scenario a) it doesn't matter if current state has precise | ||
| 4345 | * markings set or not. | ||
| 4346 | * | ||
| 4347 | * Now, for the scenario b), checkpointing and forking into child(ren) | ||
| 4348 | * state(s). Note that before current state gets to checkpointing step, any | ||
| 4349 | * processed instruction always assumes precise SCALAR register/slot | ||
| 4350 | * knowledge: if precise value or range is useful to prune jump branch, BPF | ||
| 4351 | * verifier takes this opportunity enthusiastically. Similarly, when | ||
| 4352 | * register's value is used to calculate offset or memory address, exact | ||
| 4353 | * knowledge of SCALAR range is assumed, checked, and enforced. So, similar to | ||
| 4354 | * what we mentioned above about state comparison ignoring precise markings | ||
| 4355 | * during state comparison, BPF verifier ignores and also assumes precise | ||
| 4356 | * markings *at will* during instruction verification process. But as verifier | ||
| 4357 | * assumes precision, it also propagates any precision dependencies across | ||
| 4358 | * parent states, which are not yet finalized, so can be further restricted | ||
| 4359 | * based on new knowledge gained from restrictions enforced by their children | ||
| 4360 | * states. This is so that once those parent states are finalized, i.e., when | ||
| 4361 | * they have no more active children state, state comparison logic in | ||
| 4362 | * is_state_visited() would enforce strict and precise SCALAR ranges, if | ||
| 4363 | * required for correctness. | ||
| 4364 | * | ||
| 4365 | * To build a bit more intuition, note also that once a state is checkpointed, | ||
| 4366 | * the path we took to get to that state is not important. This is crucial | ||
| 4367 | * property for state pruning. When state is checkpointed and finalized at | ||
| 4368 | * some instruction index, it can be correctly and safely used to "short | ||
| 4369 | * circuit" any *compatible* state that reaches exactly the same instruction | ||
| 4370 | * index. I.e., if we jumped to that instruction from a completely different | ||
| 4371 | * code path than original finalized state was derived from, it doesn't | ||
| 4372 | * matter, current state can be discarded because from that instruction | ||
| 4373 | * forward having a compatible state will ensure we will safely reach the | ||
| 4374 | * exit. States describe preconditions for further exploration, but completely | ||
| 4375 | * forget the history of how we got here. | ||
| 4376 | * | ||
| 4377 | * This also means that even if we needed precise SCALAR range to get to | ||
| 4378 | * finalized state, but from that point forward *that same* SCALAR register is | ||
| 4379 | * never used in a precise context (i.e., it's precise value is not needed for | ||
| 4380 | * correctness), it's correct and safe to mark such register as "imprecise" | ||
| 4381 | * (i.e., precise marking set to false). This is what we rely on when we do | ||
| 4382 | * not set precise marking in current state. If no child state requires | ||
| 4383 | * precision for any given SCALAR register, it's safe to dictate that it can | ||
| 4384 | * be imprecise. If any child state does require this register to be precise, | ||
| 4385 | * we'll mark it precise later retroactively during precise markings | ||
| 4386 | * propagation from child state to parent states. | ||
| 4387 | * | ||
| 4388 | * Skipping precise marking setting in current state is a mild version of | ||
| 4389 | * relying on the above observation. But we can utilize this property even | ||
| 4390 | * more aggressively by proactively forgetting any precise marking in the | ||
| 4391 | * current state (which we inherited from the parent state), right before we | ||
| 4392 | * checkpoint it and branch off into new child state. This is done by | ||
| 4393 | * mark_all_scalars_imprecise() to hopefully get more permissive and generic | ||
| 4394 | * finalized states which help in short circuiting more future states. | ||
| 4395 | */ | ||
| 4396 | int bpf_mark_chain_precision(struct bpf_verifier_env *env, | ||
| 4397 | struct bpf_verifier_state *starting_state, | ||
| 4398 | int regno, | ||
| 4399 | bool *changed) | ||
| 4400 | { | ||
| 4401 | struct bpf_verifier_state *st = starting_state; | ||
| 4402 | struct backtrack_state *bt = &env->bt; | ||
| 4403 | int first_idx = st->first_insn_idx; | ||
| 4404 | int last_idx = starting_state->insn_idx; | ||
| 4405 | int subseq_idx = -1; | ||
| 4406 | struct bpf_func_state *func; | ||
| 4407 | bool tmp, skip_first = true; | ||
| 4408 | struct bpf_reg_state *reg; | ||
| 4409 | int i, fr, err; | ||
| 4410 | |||
| 4411 | if (!env->bpf_capable) | ||
| 4412 | return 0; | ||
| 4413 | |||
| 4414 | changed = changed ?: &tmp; | ||
| 4415 | /* set frame number from which we are starting to backtrack */ | ||
| 4416 | bt_init(bt, starting_state->curframe); | ||
| 4417 | |||
| 4418 | /* Do sanity checks against current state of register and/or stack | ||
| 4419 | * slot, but don't set precise flag in current state, as precision | ||
| 4420 | * tracking in the current state is unnecessary. | ||
| 4421 | */ | ||
| 4422 | func = st->frame[bt->frame]; | ||
| 4423 | if (regno >= 0) { | ||
| 4424 | reg = &func->regs[regno]; | ||
| 4425 | if (reg->type != SCALAR_VALUE) { | ||
| 4426 | verifier_bug(env, "backtracking misuse"); | ||
| 4427 | return -EFAULT; | ||
| 4428 | } | ||
| 4429 | bt_set_reg(bt, regno); | ||
| 4430 | } | ||
| 4431 | |||
| 4432 | if (bt_empty(bt)) | ||
| 4433 | return 0; | ||
| 4434 | |||
| 4435 | for (;;) { | ||
| 4436 | DECLARE_BITMAP(mask, 64); | ||
| 4437 | u32 history = st->jmp_history_cnt; | ||
| 4438 | struct bpf_jmp_history_entry *hist; | ||
| 4439 | |||
| 4440 | if (env->log.level & BPF_LOG_LEVEL2) { | ||
| 4441 | verbose(env, "mark_precise: frame%d: last_idx %d first_idx %d subseq_idx %d \n", | ||
| 4442 | bt->frame, last_idx, first_idx, subseq_idx); | ||
| 4443 | } | ||
| 4444 | |||
| 4445 | if (last_idx < 0) { | ||
| 4446 | /* we are at the entry into subprog, which | ||
| 4447 | * is expected for global funcs, but only if | ||
| 4448 | * requested precise registers are R1-R5 | ||
| 4449 | * (which are global func's input arguments) | ||
| 4450 | */ | ||
| 4451 | if (st->curframe == 0 && | ||
| 4452 | st->frame[0]->subprogno > 0 && | ||
| 4453 | st->frame[0]->callsite == BPF_MAIN_FUNC && | ||
| 4454 | bt_stack_mask(bt) == 0 && | ||
| 4455 | (bt_reg_mask(bt) & ~BPF_REGMASK_ARGS) == 0) { | ||
| 4456 | bitmap_from_u64(mask, bt_reg_mask(bt)); | ||
| 4457 | for_each_set_bit(i, mask, 32) { | ||
| 4458 | reg = &st->frame[0]->regs[i]; | ||
| 4459 | bt_clear_reg(bt, i); | ||
| 4460 | if (reg->type == SCALAR_VALUE) { | ||
| 4461 | reg->precise = true; | ||
| 4462 | *changed = true; | ||
| 4463 | } | ||
| 4464 | } | ||
| 4465 | return 0; | ||
| 4466 | } | ||
| 4467 | |||
| 4468 | verifier_bug(env, "backtracking func entry subprog %d reg_mask %x stack_mask %llx", | ||
| 4469 | st->frame[0]->subprogno, bt_reg_mask(bt), bt_stack_mask(bt)); | ||
| 4470 | return -EFAULT; | ||
| 4471 | } | ||
| 4472 | |||
| 4473 | for (i = last_idx;;) { | ||
| 4474 | if (skip_first) { | ||
| 4475 | err = 0; | ||
| 4476 | skip_first = false; | ||
| 4477 | } else { | ||
| 4478 | hist = get_jmp_hist_entry(st, history, i); | ||
| 4479 | err = backtrack_insn(env, i, subseq_idx, hist, bt); | ||
| 4480 | } | ||
| 4481 | if (err == -ENOTSUPP) { | ||
| 4482 | bpf_mark_all_scalars_precise(env, starting_state); | ||
| 4483 | bt_reset(bt); | ||
| 4484 | return 0; | ||
| 4485 | } else if (err) { | ||
| 4486 | return err; | ||
| 4487 | } | ||
| 4488 | if (bt_empty(bt)) | ||
| 4489 | /* Found assignment(s) into tracked register in this state. | ||
| 4490 | * Since this state is already marked, just return. | ||
| 4491 | * Nothing to be tracked further in the parent state. | ||
| 4492 | */ | ||
| 4493 | return 0; | ||
| 4494 | subseq_idx = i; | ||
| 4495 | i = get_prev_insn_idx(st, i, &history); | ||
| 4496 | if (i == -ENOENT) | ||
| 4497 | break; | ||
| 4498 | if (i >= env->prog->len) { | ||
| 4499 | /* This can happen if backtracking reached insn 0 | ||
| 4500 | * and there are still reg_mask or stack_mask | ||
| 4501 | * to backtrack. | ||
| 4502 | * It means the backtracking missed the spot where | ||
| 4503 | * particular register was initialized with a constant. | ||
| 4504 | */ | ||
| 4505 | verifier_bug(env, "backtracking idx %d", i); | ||
| 4506 | return -EFAULT; | ||
| 4507 | } | ||
| 4508 | } | ||
| 4509 | st = st->parent; | ||
| 4510 | if (!st) | ||
| 4511 | break; | ||
| 4512 | |||
| 4513 | for (fr = bt->frame; fr >= 0; fr--) { | ||
| 4514 | func = st->frame[fr]; | ||
| 4515 | bitmap_from_u64(mask, bt_frame_reg_mask(bt, fr)); | ||
| 4516 | for_each_set_bit(i, mask, 32) { | ||
| 4517 | reg = &func->regs[i]; | ||
| 4518 | if (reg->type != SCALAR_VALUE) { | ||
| 4519 | bt_clear_frame_reg(bt, fr, i); | ||
| 4520 | continue; | ||
| 4521 | } | ||
| 4522 | if (reg->precise) { | ||
| 4523 | bt_clear_frame_reg(bt, fr, i); | ||
| 4524 | } else { | ||
| 4525 | reg->precise = true; | ||
| 4526 | *changed = true; | ||
| 4527 | } | ||
| 4528 | } | ||
| 4529 | |||
| 4530 | bitmap_from_u64(mask, bt_frame_stack_mask(bt, fr)); | ||
| 4531 | for_each_set_bit(i, mask, 64) { | ||
| 4532 | if (verifier_bug_if(i >= func->allocated_stack / BPF_REG_SIZE, | ||
| 4533 | env, "stack slot %d, total slots %d", | ||
| 4534 | i, func->allocated_stack / BPF_REG_SIZE)) | ||
| 4535 | return -EFAULT; | ||
| 4536 | |||
| 4537 | if (!is_spilled_scalar_reg(&func->stack[i])) { | ||
| 4538 | bt_clear_frame_slot(bt, fr, i); | ||
| 4539 | continue; | ||
| 4540 | } | ||
| 4541 | reg = &func->stack[i].spilled_ptr; | ||
| 4542 | if (reg->precise) { | ||
| 4543 | bt_clear_frame_slot(bt, fr, i); | ||
| 4544 | } else { | ||
| 4545 | reg->precise = true; | ||
| 4546 | *changed = true; | ||
| 4547 | } | ||
| 4548 | } | ||
| 4549 | if (env->log.level & BPF_LOG_LEVEL2) { | ||
| 4550 | fmt_reg_mask(env->tmp_str_buf, TMP_STR_BUF_LEN, | ||
| 4551 | bt_frame_reg_mask(bt, fr)); | ||
| 4552 | verbose(env, "mark_precise: frame%d: parent state regs=%s ", | ||
| 4553 | fr, env->tmp_str_buf); | ||
| 4554 | bpf_fmt_stack_mask(env->tmp_str_buf, TMP_STR_BUF_LEN, | ||
| 4555 | bt_frame_stack_mask(bt, fr)); | ||
| 4556 | verbose(env, "stack=%s: ", env->tmp_str_buf); | ||
| 4557 | print_verifier_state(env, st, fr, true); | ||
| 4558 | } | ||
| 4559 | } | ||
| 4560 | |||
| 4561 | if (bt_empty(bt)) | ||
| 4562 | return 0; | ||
| 4563 | |||
| 4564 | subseq_idx = first_idx; | ||
| 4565 | last_idx = st->last_insn_idx; | ||
| 4566 | first_idx = st->first_insn_idx; | ||
| 4567 | } | ||
| 4568 | |||
| 4569 | /* if we still have requested precise regs or slots, we missed | ||
| 4570 | * something (e.g., stack access through non-r10 register), so | ||
| 4571 | * fallback to marking all precise | ||
| 4572 | */ | ||
| 4573 | if (!bt_empty(bt)) { | ||
| 4574 | bpf_mark_all_scalars_precise(env, starting_state); | ||
| 4575 | bt_reset(bt); | ||
| 4576 | } | ||
| 4577 | |||
| 4578 | return 0; | ||
| 4579 | } | ||
| 4580 | |||
| 4581 | int mark_chain_precision(struct bpf_verifier_env *env, int regno) | 3641 | int mark_chain_precision(struct bpf_verifier_env *env, int regno) |
| 4582 | { | 3642 | { |
| 4583 | return bpf_mark_chain_precision(env, env->cur_state, regno, NULL); | 3643 | return bpf_mark_chain_precision(env, env->cur_state, regno, NULL); |
| @@ -4759,7 +3819,7 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env, · 175 unmodified lines | |||
| 4759 | */ | 3819 | */ |
| 4760 | if (!env->allow_ptr_leaks && | 3820 | if (!env->allow_ptr_leaks && |
| 4761 | bpf_is_spilled_reg(&state->stack[spi]) && | 3821 | bpf_is_spilled_reg(&state->stack[spi]) && |
| 4762 | !is_spilled_scalar_reg(&state->stack[spi]) && | 3822 | !bpf_is_spilled_scalar_reg(&state->stack[spi]) && |
| 4763 | size != BPF_REG_SIZE) { | 3823 | size != BPF_REG_SIZE) { |
| 4764 | verbose(env, "attempt to corrupt spilled pointer on stack\n"); | 3824 | verbose(env, "attempt to corrupt spilled pointer on stack\n"); |
| 4765 | return -EACCES; | 3825 | return -EACCES; |
| @@ -4938,7 +3998,7 @@ static int check_stack_write_var_off(struct bpf_verifier_env *env, · 172 unmodified lines | |||
| 4938 | * maintain the spill type. | 3998 | * maintain the spill type. |
| 4939 | */ | 3999 | */ |
| 4940 | if (writing_zero && *stype == STACK_SPILL && | 4000 | if (writing_zero && *stype == STACK_SPILL && |
| 4941 | is_spilled_scalar_reg(&state->stack[spi])) { | 4001 | bpf_is_spilled_scalar_reg(&state->stack[spi])) { |
| 4942 | struct bpf_reg_state *spill_reg = &state->stack[spi].spilled_ptr; | 4002 | struct bpf_reg_state *spill_reg = &state->stack[spi].spilled_ptr; |
| 4943 | 4003 | ||
| 4944 | if (tnum_is_const(spill_reg->var_off) && spill_reg->var_off.value == 0) { | 4004 | if (tnum_is_const(spill_reg->var_off) && spill_reg->var_off.value == 0) { |
| @@ -9380,7 +8440,7 @@ static int get_constant_map_key(struct bpf_verifier_env *env, · 4435 unmodified lines | |||
| 9380 | } | 8440 | } |
| 9381 | 8441 | ||
| 9382 | /* Check that stack contains a scalar spill of expected size */ | 8442 | /* Check that stack contains a scalar spill of expected size */ |
| 9383 | if (!is_spilled_scalar_reg(&state->stack[spi])) | 8443 | if (!bpf_is_spilled_scalar_reg(&state->stack[spi])) |
| 9384 | return -EOPNOTSUPP; | 8444 | return -EOPNOTSUPP; |
| 9385 | for (i = off; i >= 0 && stype[i] == STACK_SPILL; i--) | 8445 | for (i = off; i >= 0 && stype[i] == STACK_SPILL; i--) |
| 9386 | spill_size++; | 8446 | spill_size++; |