// SPDX-License-Identifier: GPL-2.0-only/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */#include <linux/bpf.h>#include <linux/bpf_verifier.h>#include <linux/filter.h>#include <linux/bitmap.h> #define verbose(env, fmt, args...) bpf_verifier_log_write(env, fmt, ##args) /* for any branch, call, exit record the history of jmps in the given state */int bpf_push_jmp_history(struct bpf_verifier_env *env, struct bpf_verifier_state *cur, int insn_flags, int spi, int frame, u64 linked_regs){ u32 cnt = cur->jmp_history_cnt; struct bpf_jmp_history_entry *p; size_t alloc_size; /* combine instruction flags if we already recorded this instruction */ if (env->cur_hist_ent) { /* atomic instructions push insn_flags twice, for READ and * WRITE sides, but they should agree on stack slot */ verifier_bug_if((env->cur_hist_ent->flags & insn_flags) && (env->cur_hist_ent->flags & insn_flags) != insn_flags, env, "insn history: insn_idx %d cur flags %x new flags %x", env->insn_idx, env->cur_hist_ent->flags, insn_flags); env->cur_hist_ent->flags |= insn_flags; env->cur_hist_ent->spi = spi; env->cur_hist_ent->frame = frame; verifier_bug_if(env->cur_hist_ent->linked_regs != 0, env, "insn history: insn_idx %d linked_regs: %#llx", env->insn_idx, env->cur_hist_ent->linked_regs); env->cur_hist_ent->linked_regs = linked_regs; return 0; } cnt++; alloc_size = kmalloc_size_roundup(size_mul(cnt, sizeof(*p))); p = krealloc(cur->jmp_history, alloc_size, GFP_KERNEL_ACCOUNT); if (!p) return -ENOMEM; cur->jmp_history = p; p = &cur->jmp_history[cnt - 1]; p->idx = env->insn_idx; p->prev_idx = env->prev_insn_idx; p->flags = insn_flags; p->spi = spi; p->frame = frame; p->linked_regs = linked_regs; cur->jmp_history_cnt = cnt; env->cur_hist_ent = p; return 0;} static bool is_atomic_load_insn(const struct bpf_insn *insn){ return BPF_CLASS(insn->code) == BPF_STX && BPF_MODE(insn->code) == BPF_ATOMIC && insn->imm == BPF_LOAD_ACQ;} static bool is_atomic_fetch_insn(const struct bpf_insn *insn){ return BPF_CLASS(insn->code) == BPF_STX && BPF_MODE(insn->code) == BPF_ATOMIC && (insn->imm & BPF_FETCH);} /* Backtrack one insn at a time. If idx is not at the top of recorded * history then previous instruction came from straight line execution. * Return -ENOENT if we exhausted all instructions within given state. * * It's legal to have a bit of a looping with the same starting and ending * insn index within the same state, e.g.: 3->4->5->3, so just because current * instruction index is the same as state's first_idx doesn't mean we are * done. If there is still some jump history left, we should keep going. We * need to take into account that we might have a jump history between given * state's parent and itself, due to checkpointing. In this case, we'll have * history entry recording a jump from last instruction of parent state and * first instruction of given state. */static int get_prev_insn_idx(struct bpf_verifier_state *st, int i, u32 *history){ u32 cnt = *history; if (i == st->first_insn_idx) { if (cnt == 0) return -ENOENT; if (cnt == 1 && st->jmp_history[0].idx == i) return -ENOENT; } if (cnt && st->jmp_history[cnt - 1].idx == i) { i = st->jmp_history[cnt - 1].prev_idx; (*history)--; } else { i--; } return i;} static struct bpf_jmp_history_entry *get_jmp_hist_entry(struct bpf_verifier_state *st, u32 hist_end, int insn_idx){ if (hist_end > 0 && st->jmp_history[hist_end - 1].idx == insn_idx) return &st->jmp_history[hist_end - 1]; return NULL;} static inline void bt_init(struct backtrack_state *bt, u32 frame){ bt->frame = frame;} static inline void bt_reset(struct backtrack_state *bt){ struct bpf_verifier_env *env = bt->env; memset(bt, 0, sizeof(*bt)); bt->env = env;} static inline u32 bt_empty(struct backtrack_state *bt){ u64 mask = 0; int i; for (i = 0; i <= bt->frame; i++) mask |= bt->reg_masks[i] | bt->stack_masks[i] | bt->stack_arg_masks[i]; return mask == 0;} static inline void bt_clear_frame_stack_arg_slot(struct backtrack_state *bt, u32 frame, u32 slot){ bt->stack_arg_masks[frame] &= ~(1 << slot);} static inline bool bt_is_frame_stack_arg_slot_set(struct backtrack_state *bt, u32 frame, u32 slot){ return bt->stack_arg_masks[frame] & (1 << slot);} static inline int bt_subprog_enter(struct backtrack_state *bt){ if (bt->frame == MAX_CALL_FRAMES - 1) { verifier_bug(bt->env, "subprog enter from frame %d", bt->frame); return -EFAULT; } bt->frame++; return 0;} static inline int bt_subprog_exit(struct backtrack_state *bt){ if (bt->frame == 0) { verifier_bug(bt->env, "subprog exit from frame 0"); return -EFAULT; } bt->frame--; return 0;} static inline void bt_clear_frame_reg(struct backtrack_state *bt, u32 frame, u32 reg){ bt->reg_masks[frame] &= ~(1 << reg);} static inline void bt_set_reg(struct backtrack_state *bt, u32 reg){ bpf_bt_set_frame_reg(bt, bt->frame, reg);} static inline void bt_clear_reg(struct backtrack_state *bt, u32 reg){ bt_clear_frame_reg(bt, bt->frame, reg);} static inline void bt_clear_frame_slot(struct backtrack_state *bt, u32 frame, u32 slot){ bt->stack_masks[frame] &= ~(1ull << slot);} static inline u32 bt_frame_reg_mask(struct backtrack_state *bt, u32 frame){ return bt->reg_masks[frame];} static inline u32 bt_reg_mask(struct backtrack_state *bt){ return bt->reg_masks[bt->frame];} static inline u64 bt_frame_stack_mask(struct backtrack_state *bt, u32 frame){ return bt->stack_masks[frame];} static inline u64 bt_stack_mask(struct backtrack_state *bt){ return bt->stack_masks[bt->frame];} static inline u8 bt_stack_arg_mask(struct backtrack_state *bt){ return bt->stack_arg_masks[bt->frame];} static inline bool bt_is_reg_set(struct backtrack_state *bt, u32 reg){ return bt->reg_masks[bt->frame] & (1 << reg);} /* format registers bitmask, e.g., "r0,r2,r4" for 0x15 mask */static void fmt_reg_mask(char *buf, ssize_t buf_sz, u32 reg_mask){ DECLARE_BITMAP(mask, 64); bool first = true; int i, n; buf[0] = '\0'; bitmap_from_u64(mask, reg_mask); for_each_set_bit(i, mask, 32) { n = snprintf(buf, buf_sz, "%sr%d", first ? "" : ",", i); first = false; buf += n; buf_sz -= n; if (buf_sz < 0) break; }}/* format stack slots bitmask, e.g., "-8,-24,-40" for 0x15 mask */void bpf_fmt_stack_mask(char *buf, ssize_t buf_sz, u64 stack_mask){ DECLARE_BITMAP(mask, 64); bool first = true; int i, n; buf[0] = '\0'; bitmap_from_u64(mask, stack_mask); for_each_set_bit(i, mask, 64) { n = snprintf(buf, buf_sz, "%s%d", first ? "" : ",", -(i + 1) * 8); first = false; buf += n; buf_sz -= n; if (buf_sz < 0) break; }} /* For given verifier state backtrack_insn() is called from the last insn to * the first insn. Its purpose is to compute a bitmask of registers and * stack slots that needs precision in the parent verifier state. * * @idx is an index of the instruction we are currently processing; * @subseq_idx is an index of the subsequent instruction that: * - *would be* executed next, if jump history is viewed in forward order; * - *was* processed previously during backtracking. */static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, struct bpf_jmp_history_entry *hist, struct backtrack_state *bt){ struct bpf_insn *insn = env->prog->insnsi + idx; u8 class = BPF_CLASS(insn->code); u8 opcode = BPF_OP(insn->code); u8 mode = BPF_MODE(insn->code); u32 dreg = insn->dst_reg; u32 sreg = insn->src_reg; u32 spi, i, fr; if (insn->code == 0) return 0; if (env->log.level & BPF_LOG_LEVEL2) { fmt_reg_mask(env->tmp_str_buf, TMP_STR_BUF_LEN, bt_reg_mask(bt)); verbose(env, "mark_precise: frame%d: regs=%s ", bt->frame, env->tmp_str_buf); bpf_fmt_stack_mask(env->tmp_str_buf, TMP_STR_BUF_LEN, bt_stack_mask(bt)); verbose(env, "stack=%s before ", env->tmp_str_buf); verbose(env, "%d: ", idx); bpf_verbose_insn(env, insn); verbose(env, "\n"); } /* If there is a history record that some registers gained range at this insn, * propagate precision marks to those registers, so that bt_is_reg_set() * accounts for these registers. */ bpf_bt_sync_linked_regs(bt, hist); if (class == BPF_ALU || class == BPF_ALU64) { if (!bt_is_reg_set(bt, dreg)) return 0; if (opcode == BPF_END || opcode == BPF_NEG) { /* sreg is reserved and unused * dreg still need precision before this insn */ return 0; } else if (opcode == BPF_MOV) { if (BPF_SRC(insn->code) == BPF_X) { /* dreg = sreg or dreg = (s8, s16, s32)sreg * dreg needs precision after this insn * sreg needs precision before this insn */ bt_clear_reg(bt, dreg); if (sreg != BPF_REG_FP) bt_set_reg(bt, sreg); } else { /* dreg = K * dreg needs precision after this insn. * Corresponding register is already marked * as precise=true in this verifier state. * No further markings in parent are necessary */ bt_clear_reg(bt, dreg); } } else { if (BPF_SRC(insn->code) == BPF_X) { /* dreg += sreg * both dreg and sreg need precision * before this insn */ if (sreg != BPF_REG_FP) bt_set_reg(bt, sreg); } /* else dreg += K * dreg still needs precision before this insn */ } } else if (class == BPF_LDX || is_atomic_load_insn(insn) || is_atomic_fetch_insn(insn)) { u32 load_reg = dreg; /* * Atomic fetch operation writes the old value into * a register (sreg or r0) and if it was tracked for * precision, propagate to the stack slot like we do * in regular ldx. */ if (is_atomic_fetch_insn(insn)) load_reg = insn->imm == BPF_CMPXCHG ? BPF_REG_0 : sreg; if (!bt_is_reg_set(bt, load_reg)) return 0; bt_clear_reg(bt, load_reg); if (hist && hist->flags & INSN_F_STACK_ARG_ACCESS) { spi = hist->spi; /* * Stack arg read: callee reads from r11+off, but * the data lives in the caller's stack_arg_regs. * Set the mask in the caller frame so precision * is marked in the caller's slot at the callee * entry checkpoint. */ bt_set_frame_stack_arg_slot(bt, bt->frame - 1, spi); return 0; } /* scalars can only be spilled into stack w/o losing precision. * Load from any other memory can be zero extended. * The desire to keep that precision is already indicated * by 'precise' mark in corresponding register of this state. * No further tracking necessary. */ if (!hist || !(hist->flags & INSN_F_STACK_ACCESS)) return 0; /* dreg = *(u64 *)[fp - off] was a fill from the stack. * that [fp - off] slot contains scalar that needs to be * tracked with precision */ spi = hist->spi; fr = hist->frame; bpf_bt_set_frame_slot(bt, fr, spi); } else if (class == BPF_STX || class == BPF_ST) { if (bt_is_reg_set(bt, dreg)) /* stx & st shouldn't be using _scalar_ dst_reg * to access memory. It means backtracking * encountered a case of pointer subtraction. */ return -ENOTSUPP; if (hist && hist->flags & INSN_F_STACK_ARG_ACCESS) { spi = hist->spi; if (!bt_is_frame_stack_arg_slot_set(bt, bt->frame, spi)) return 0; bt_clear_frame_stack_arg_slot(bt, bt->frame, spi); if (class == BPF_STX) bt_set_reg(bt, sreg); return 0; } /* scalars can only be spilled into stack */ if (!hist || !(hist->flags & INSN_F_STACK_ACCESS)) return 0; spi = hist->spi; fr = hist->frame; if (!bt_is_frame_slot_set(bt, fr, spi)) return 0; bt_clear_frame_slot(bt, fr, spi); if (class == BPF_STX) bt_set_reg(bt, sreg); } else if (class == BPF_JMP || class == BPF_JMP32) { if (bpf_pseudo_call(insn)) { int subprog_insn_idx, subprog; subprog_insn_idx = idx + insn->imm + 1; subprog = bpf_find_subprog(env, subprog_insn_idx); if (subprog < 0) return -EFAULT; if (bpf_subprog_is_global(env, subprog)) { /* check that jump history doesn't have any * extra instructions from subprog; the next * instruction after call to global subprog * should be literally next instruction in * caller program */ verifier_bug_if(idx + 1 != subseq_idx, env, "extra insn from subprog"); /* r1-r5 are invalidated after subprog call, * so for global func call it shouldn't be set * anymore */ if (bt_reg_mask(bt) & BPF_REGMASK_ARGS) { verifier_bug(env, "global subprog unexpected regs %x", bt_reg_mask(bt)); return -EFAULT; } /* global subprog always sets R0 */ bt_clear_reg(bt, BPF_REG_0); return 0; } else { /* static subprog call instruction, which * means that we are exiting current subprog, * so only r1-r5 could be still requested as * precise, r0 and r6-r10 or any stack slot in * the current frame should be zero by now */ if (bt_reg_mask(bt) & ~BPF_REGMASK_ARGS) { verifier_bug(env, "static subprog unexpected regs %x", bt_reg_mask(bt)); return -EFAULT; } /* we are now tracking register spills correctly, * so any instance of leftover slots is a bug */ if (bt_stack_mask(bt) != 0) { verifier_bug(env, "static subprog leftover stack slots %llx", bt_stack_mask(bt)); return -EFAULT; } /* propagate r1-r5 to the caller */ for (i = BPF_REG_1; i <= BPF_REG_5; i++) { if (bt_is_reg_set(bt, i)) { bt_clear_reg(bt, i); bpf_bt_set_frame_reg(bt, bt->frame - 1, i); } } if (bt_stack_arg_mask(bt)) { verifier_bug(env, "static subprog leftover stack arg slots %x", bt_stack_arg_mask(bt)); return -EFAULT; } if (bt_subprog_exit(bt)) return -EFAULT; return 0; } } else if (bpf_is_sync_callback_calling_insn(insn) && idx != subseq_idx - 1) { /* exit from callback subprog to callback-calling helper or * kfunc call. Use idx/subseq_idx check to discern it from * straight line code backtracking. * Unlike the subprog call handling above, we shouldn't * propagate precision of r1-r5 (if any requested), as they are * not actually arguments passed directly to callback subprogs */ if (bt_reg_mask(bt) & ~BPF_REGMASK_ARGS) { verifier_bug(env, "callback unexpected regs %x", bt_reg_mask(bt)); return -EFAULT; } if (bt_stack_mask(bt) != 0) { verifier_bug(env, "callback leftover stack slots %llx", bt_stack_mask(bt)); return -EFAULT; } /* clear r1-r5 in callback subprog's mask */ for (i = BPF_REG_1; i <= BPF_REG_5; i++) bt_clear_reg(bt, i); if (bt_subprog_exit(bt)) return -EFAULT; return 0; } else if (opcode == BPF_CALL) {Lines 1–500 of 993. Load more
// SPDX-License-Identifier: GPL-2.0-only /* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ #include <linux/bpf.h> #include <linux/bpf_verifier.h> #include <linux/filter.h> #include <linux/bitmap.h> #define verbose(env, fmt, args...) bpf_verifier_log_write(env, fmt, ##args) /* for any branch, call, exit record the history of jmps in the given state */ int bpf_push_jmp_history(struct bpf_verifier_env *env, struct bpf_verifier_state *cur, int insn_flags, int spi, int frame, u64 linked_regs) { u32 cnt = cur->jmp_history_cnt; struct bpf_jmp_history_entry *p; size_t alloc_size; /* combine instruction flags if we already recorded this instruction */ if (env->cur_hist_ent) { /* atomic instructions push insn_flags twice, for READ and * WRITE sides, but they should agree on stack slot */ verifier_bug_if((env->cur_hist_ent->flags & insn_flags) && (env->cur_hist_ent->flags & insn_flags) != insn_flags, env, "insn history: insn_idx %d cur flags %x new flags %x", env->insn_idx, env->cur_hist_ent->flags, insn_flags); env->cur_hist_ent->flags |= insn_flags; env->cur_hist_ent->spi = spi; env->cur_hist_ent->frame = frame; verifier_bug_if(env->cur_hist_ent->linked_regs != 0, env, "insn history: insn_idx %d linked_regs: %#llx", env->insn_idx, env->cur_hist_ent->linked_regs); env->cur_hist_ent->linked_regs = linked_regs; return 0; } cnt++; alloc_size = kmalloc_size_roundup(size_mul(cnt, sizeof(*p))); p = krealloc(cur->jmp_history, alloc_size, GFP_KERNEL_ACCOUNT); if (!p) return -ENOMEM; cur->jmp_history = p; p = &cur->jmp_history[cnt - 1]; p->idx = env->insn_idx; p->prev_idx = env->prev_insn_idx; p->flags = insn_flags; p->spi = spi; p->frame = frame; p->linked_regs = linked_regs; cur->jmp_history_cnt = cnt; env->cur_hist_ent = p; return 0; } static bool is_atomic_load_insn(const struct bpf_insn *insn) { return BPF_CLASS(insn->code) == BPF_STX && BPF_MODE(insn->code) == BPF_ATOMIC && insn->imm == BPF_LOAD_ACQ; } static bool is_atomic_fetch_insn(const struct bpf_insn *insn) { return BPF_CLASS(insn->code) == BPF_STX && BPF_MODE(insn->code) == BPF_ATOMIC && (insn->imm & BPF_FETCH); } /* Backtrack one insn at a time. If idx is not at the top of recorded * history then previous instruction came from straight line execution. * Return -ENOENT if we exhausted all instructions within given state. * * It's legal to have a bit of a looping with the same starting and ending * insn index within the same state, e.g.: 3->4->5->3, so just because current * instruction index is the same as state's first_idx doesn't mean we are * done. If there is still some jump history left, we should keep going. We * need to take into account that we might have a jump history between given * state's parent and itself, due to checkpointing. In this case, we'll have * history entry recording a jump from last instruction of parent state and * first instruction of given state. */ static int get_prev_insn_idx(struct bpf_verifier_state *st, int i, u32 *history) { u32 cnt = *history; if (i == st->first_insn_idx) { if (cnt == 0) return -ENOENT; if (cnt == 1 && st->jmp_history[0].idx == i) return -ENOENT; } if (cnt && st->jmp_history[cnt - 1].idx == i) { i = st->jmp_history[cnt - 1].prev_idx; (*history)--; } else { i--; } return i; } static struct bpf_jmp_history_entry *get_jmp_hist_entry(struct bpf_verifier_state *st, u32 hist_end, int insn_idx) { if (hist_end > 0 && st->jmp_history[hist_end - 1].idx == insn_idx) return &st->jmp_history[hist_end - 1]; return NULL; } static inline void bt_init(struct backtrack_state *bt, u32 frame) { bt->frame = frame; } static inline void bt_reset(struct backtrack_state *bt) { struct bpf_verifier_env *env = bt->env; memset(bt, 0, sizeof(*bt)); bt->env = env; } static inline u32 bt_empty(struct backtrack_state *bt) { u64 mask = 0; int i; for (i = 0; i <= bt->frame; i++) mask |= bt->reg_masks[i] | bt->stack_masks[i] | bt->stack_arg_masks[i]; return mask == 0; } static inline void bt_clear_frame_stack_arg_slot(struct backtrack_state *bt, u32 frame, u32 slot) { bt->stack_arg_masks[frame] &= ~(1 << slot); } static inline bool bt_is_frame_stack_arg_slot_set(struct backtrack_state *bt, u32 frame, u32 slot) { return bt->stack_arg_masks[frame] & (1 << slot); } static inline int bt_subprog_enter(struct backtrack_state *bt) { if (bt->frame == MAX_CALL_FRAMES - 1) { verifier_bug(bt->env, "subprog enter from frame %d", bt->frame); return -EFAULT; } bt->frame++; return 0; } static inline int bt_subprog_exit(struct backtrack_state *bt) { if (bt->frame == 0) { verifier_bug(bt->env, "subprog exit from frame 0"); return -EFAULT; } bt->frame--; return 0; } static inline void bt_clear_frame_reg(struct backtrack_state *bt, u32 frame, u32 reg) { bt->reg_masks[frame] &= ~(1 << reg); } static inline void bt_set_reg(struct backtrack_state *bt, u32 reg) { bpf_bt_set_frame_reg(bt, bt->frame, reg); } static inline void bt_clear_reg(struct backtrack_state *bt, u32 reg) { bt_clear_frame_reg(bt, bt->frame, reg); } static inline void bt_clear_frame_slot(struct backtrack_state *bt, u32 frame, u32 slot) { bt->stack_masks[frame] &= ~(1ull << slot); } static inline u32 bt_frame_reg_mask(struct backtrack_state *bt, u32 frame) { return bt->reg_masks[frame]; } static inline u32 bt_reg_mask(struct backtrack_state *bt) { return bt->reg_masks[bt->frame]; } static inline u64 bt_frame_stack_mask(struct backtrack_state *bt, u32 frame) { return bt->stack_masks[frame]; } static inline u64 bt_stack_mask(struct backtrack_state *bt) { return bt->stack_masks[bt->frame]; } static inline u8 bt_stack_arg_mask(struct backtrack_state *bt) { return bt->stack_arg_masks[bt->frame]; } static inline bool bt_is_reg_set(struct backtrack_state *bt, u32 reg) { return bt->reg_masks[bt->frame] & (1 << reg); } /* format registers bitmask, e.g., "r0,r2,r4" for 0x15 mask */ static void fmt_reg_mask(char *buf, ssize_t buf_sz, u32 reg_mask) { DECLARE_BITMAP(mask, 64); bool first = true; int i, n; buf[0] = '\0'; bitmap_from_u64(mask, reg_mask); for_each_set_bit(i, mask, 32) { n = snprintf(buf, buf_sz, "%sr%d", first ? "" : ",", i); first = false; buf += n; buf_sz -= n; if (buf_sz < 0) break; } } /* format stack slots bitmask, e.g., "-8,-24,-40" for 0x15 mask */ void bpf_fmt_stack_mask(char *buf, ssize_t buf_sz, u64 stack_mask) { DECLARE_BITMAP(mask, 64); bool first = true; int i, n; buf[0] = '\0'; bitmap_from_u64(mask, stack_mask); for_each_set_bit(i, mask, 64) { n = snprintf(buf, buf_sz, "%s%d", first ? "" : ",", -(i + 1) * 8); first = false; buf += n; buf_sz -= n; if (buf_sz < 0) break; } } /* For given verifier state backtrack_insn() is called from the last insn to * the first insn. Its purpose is to compute a bitmask of registers and * stack slots that needs precision in the parent verifier state. * * @idx is an index of the instruction we are currently processing; * @subseq_idx is an index of the subsequent instruction that: * - *would be* executed next, if jump history is viewed in forward order; * - *was* processed previously during backtracking. */ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, struct bpf_jmp_history_entry *hist, struct backtrack_state *bt) { struct bpf_insn *insn = env->prog->insnsi + idx; u8 class = BPF_CLASS(insn->code); u8 opcode = BPF_OP(insn->code); u8 mode = BPF_MODE(insn->code); u32 dreg = insn->dst_reg; u32 sreg = insn->src_reg; u32 spi, i, fr; if (insn->code == 0) return 0; if (env->log.level & BPF_LOG_LEVEL2) { fmt_reg_mask(env->tmp_str_buf, TMP_STR_BUF_LEN, bt_reg_mask(bt)); verbose(env, "mark_precise: frame%d: regs=%s ", bt->frame, env->tmp_str_buf); bpf_fmt_stack_mask(env->tmp_str_buf, TMP_STR_BUF_LEN, bt_stack_mask(bt)); verbose(env, "stack=%s before ", env->tmp_str_buf); verbose(env, "%d: ", idx); bpf_verbose_insn(env, insn); verbose(env, "\n"); } /* If there is a history record that some registers gained range at this insn, * propagate precision marks to those registers, so that bt_is_reg_set() * accounts for these registers. */ bpf_bt_sync_linked_regs(bt, hist); if (class == BPF_ALU || class == BPF_ALU64) { if (!bt_is_reg_set(bt, dreg)) return 0; if (opcode == BPF_END || opcode == BPF_NEG) { /* sreg is reserved and unused * dreg still need precision before this insn */ return 0; } else if (opcode == BPF_MOV) { if (BPF_SRC(insn->code) == BPF_X) { /* dreg = sreg or dreg = (s8, s16, s32)sreg * dreg needs precision after this insn * sreg needs precision before this insn */ bt_clear_reg(bt, dreg); if (sreg != BPF_REG_FP) bt_set_reg(bt, sreg); } else { /* dreg = K * dreg needs precision after this insn. * Corresponding register is already marked * as precise=true in this verifier state. * No further markings in parent are necessary */ bt_clear_reg(bt, dreg); } } else { if (BPF_SRC(insn->code) == BPF_X) { /* dreg += sreg * both dreg and sreg need precision * before this insn */ if (sreg != BPF_REG_FP) bt_set_reg(bt, sreg); } /* else dreg += K * dreg still needs precision before this insn */ } } else if (class == BPF_LDX || is_atomic_load_insn(insn) || is_atomic_fetch_insn(insn)) { u32 load_reg = dreg; /* * Atomic fetch operation writes the old value into * a register (sreg or r0) and if it was tracked for * precision, propagate to the stack slot like we do * in regular ldx. */ if (is_atomic_fetch_insn(insn)) load_reg = insn->imm == BPF_CMPXCHG ? BPF_REG_0 : sreg; if (!bt_is_reg_set(bt, load_reg)) return 0; bt_clear_reg(bt, load_reg); if (hist && hist->flags & INSN_F_STACK_ARG_ACCESS) { spi = hist->spi; /* * Stack arg read: callee reads from r11+off, but * the data lives in the caller's stack_arg_regs. * Set the mask in the caller frame so precision * is marked in the caller's slot at the callee * entry checkpoint. */ bt_set_frame_stack_arg_slot(bt, bt->frame - 1, spi); return 0; } /* scalars can only be spilled into stack w/o losing precision. * Load from any other memory can be zero extended. * The desire to keep that precision is already indicated * by 'precise' mark in corresponding register of this state. * No further tracking necessary. */ if (!hist || !(hist->flags & INSN_F_STACK_ACCESS)) return 0; /* dreg = *(u64 *)[fp - off] was a fill from the stack. * that [fp - off] slot contains scalar that needs to be * tracked with precision */ spi = hist->spi; fr = hist->frame; bpf_bt_set_frame_slot(bt, fr, spi); } else if (class == BPF_STX || class == BPF_ST) { if (bt_is_reg_set(bt, dreg)) /* stx & st shouldn't be using _scalar_ dst_reg * to access memory. It means backtracking * encountered a case of pointer subtraction. */ return -ENOTSUPP; if (hist && hist->flags & INSN_F_STACK_ARG_ACCESS) { spi = hist->spi; if (!bt_is_frame_stack_arg_slot_set(bt, bt->frame, spi)) return 0; bt_clear_frame_stack_arg_slot(bt, bt->frame, spi); if (class == BPF_STX) bt_set_reg(bt, sreg); return 0; } /* scalars can only be spilled into stack */ if (!hist || !(hist->flags & INSN_F_STACK_ACCESS)) return 0; spi = hist->spi; fr = hist->frame; if (!bt_is_frame_slot_set(bt, fr, spi)) return 0; bt_clear_frame_slot(bt, fr, spi); if (class == BPF_STX) bt_set_reg(bt, sreg); } else if (class == BPF_JMP || class == BPF_JMP32) { if (bpf_pseudo_call(insn)) { int subprog_insn_idx, subprog; subprog_insn_idx = idx + insn->imm + 1; subprog = bpf_find_subprog(env, subprog_insn_idx); if (subprog < 0) return -EFAULT; if (bpf_subprog_is_global(env, subprog)) { /* check that jump history doesn't have any * extra instructions from subprog; the next * instruction after call to global subprog * should be literally next instruction in * caller program */ verifier_bug_if(idx + 1 != subseq_idx, env, "extra insn from subprog"); /* r1-r5 are invalidated after subprog call, * so for global func call it shouldn't be set * anymore */ if (bt_reg_mask(bt) & BPF_REGMASK_ARGS) { verifier_bug(env, "global subprog unexpected regs %x", bt_reg_mask(bt)); return -EFAULT; } /* global subprog always sets R0 */ bt_clear_reg(bt, BPF_REG_0); return 0; } else { /* static subprog call instruction, which * means that we are exiting current subprog, * so only r1-r5 could be still requested as * precise, r0 and r6-r10 or any stack slot in * the current frame should be zero by now */ if (bt_reg_mask(bt) & ~BPF_REGMASK_ARGS) { verifier_bug(env, "static subprog unexpected regs %x", bt_reg_mask(bt)); return -EFAULT; } /* we are now tracking register spills correctly, * so any instance of leftover slots is a bug */ if (bt_stack_mask(bt) != 0) { verifier_bug(env, "static subprog leftover stack slots %llx", bt_stack_mask(bt)); return -EFAULT; } /* propagate r1-r5 to the caller */ for (i = BPF_REG_1; i <= BPF_REG_5; i++) { if (bt_is_reg_set(bt, i)) { bt_clear_reg(bt, i); bpf_bt_set_frame_reg(bt, bt->frame - 1, i); } } if (bt_stack_arg_mask(bt)) { verifier_bug(env, "static subprog leftover stack arg slots %x", bt_stack_arg_mask(bt)); return -EFAULT; } if (bt_subprog_exit(bt)) return -EFAULT; return 0; } } else if (bpf_is_sync_callback_calling_insn(insn) && idx != subseq_idx - 1) { /* exit from callback subprog to callback-calling helper or * kfunc call. Use idx/subseq_idx check to discern it from * straight line code backtracking. * Unlike the subprog call handling above, we shouldn't * propagate precision of r1-r5 (if any requested), as they are * not actually arguments passed directly to callback subprogs */ if (bt_reg_mask(bt) & ~BPF_REGMASK_ARGS) { verifier_bug(env, "callback unexpected regs %x", bt_reg_mask(bt)); return -EFAULT; } if (bt_stack_mask(bt) != 0) { verifier_bug(env, "callback leftover stack slots %llx", bt_stack_mask(bt)); return -EFAULT; } /* clear r1-r5 in callback subprog's mask */ for (i = BPF_REG_1; i <= BPF_REG_5; i++) bt_clear_reg(bt, i); if (bt_subprog_exit(bt)) return -EFAULT; return 0; } else if (opcode == BPF_CALL) {