History

bpf: Track verifier register diagnostic events

Record material register and outgoing stack argument changes so diagnostics can explain how a value reached its current type, bounds, or unreadable state. Store old and new register types, scalar ranges, tnum value and mask, map and BTF type identity, and basic operand metadata in the environment-owned diagnostic event stream. Record invalidations when packet data moves, references are released, or borrowed references leave their protected region. Register-scoped history starts at the latest matching modification and then shows later branch outcomes. Also record fixed stack spills and overwrites, and tag register fills from stack so register-scoped history can follow value flow through spilled stack slots. The type_is_map_ptr() helper previously lived as a static function in kernel/bpf/log.c since commit 0c95c9fdb696 ("bpf: emit map name in register state if applicable and available"). Move it verbatim to include/linux/bpf_verifier.h as a static inline, next to the other type classifiers, so diagnostics.c can reuse it without duplicating the case list.

Kumar Kartikeya Dwivedi committed 2 months ago

Commit af4ea6e1 parent a6debd55 files · +515 −23Browse files

Unified · Split

5 files
  1. include/linux/bpf_verifier.h +17 −0
  2. kernel/bpf/diagnostics.c +356 −0
  3. kernel/bpf/diagnostics.h +23 −0
  4. kernel/bpf/log.c +0 −11
  5. kernel/bpf/verifier.c +119 −12
include/linux/bpf_verifier.h+17−0
@@ -354,6 +354,11 @@ struct bpf_func_state { · 353 unmodified lines
354 * 0 = main function, 1 = first callee.354 * 0 = main function, 1 = first callee.
355 */355 */
356 u32 frameno;356 u32 frameno;
357 /*
358 * Unique diagnostic identity for this function invocation. Frame depth is
359 * reused after returns, while this ID is preserved across state clones.
360 */
361 u32 diag_frame_id;
357 /* subprog number == index within subprog_info362 /* subprog number == index within subprog_info
358 * zero == main subprog363 * zero == main subprog
359 */364 */
@@ -1351,6 +1356,18 @@ static inline bool type_is_non_owning_ref(u32 type) · 991 unmodified lines
1351 return type_is_ptr_alloc_obj(type) && type_flag(type) & NON_OWN_REF;1356 return type_is_ptr_alloc_obj(type) && type_flag(type) & NON_OWN_REF;
1352}1357}
1353 1358
1359static inline bool type_is_map_ptr(enum bpf_reg_type type)
1360{
1361 switch (base_type(type)) {
1362 case CONST_PTR_TO_MAP:
1363 case PTR_TO_MAP_KEY:
1364 case PTR_TO_MAP_VALUE:
1365 return true;
1366 default:
1367 return false;
1368 }
1369}
1370
1354static inline bool type_is_pkt_pointer(enum bpf_reg_type type)1371static inline bool type_is_pkt_pointer(enum bpf_reg_type type)
1355{1372{
1356 type = base_type(type);1373 type = base_type(type);
kernel/bpf/diagnostics.c+356−0
@@ -25,8 +25,83 @@ · 24 unmodified lines
25#define BPF_DIAG_EVENT_LOG_MAX_SIZE (64U << 20)25#define BPF_DIAG_EVENT_LOG_MAX_SIZE (64U << 20)
26#define DISASM_LINE_LEN 16026#define DISASM_LINE_LEN 160
27 27
28enum bpf_diag_mod_target_kind {
29 BPF_DIAG_MOD_TARGET_NONE,
30 BPF_DIAG_MOD_TARGET_REG,
31 BPF_DIAG_MOD_TARGET_STACK_ARG,
32 BPF_DIAG_MOD_TARGET_STACK_SLOT,
33 BPF_DIAG_MOD_TARGET_STACK_RANGE,
34};
35
36struct bpf_diag_mod_target {
37 u32 frame_id;
38 union {
39 struct {
40 s16 min_off;
41 s16 max_off;
42 } range;
43 u16 spi;
44 u8 regno;
45 u8 stack_arg;
46 };
47 u8 frameno;
48 u8 kind;
49};
50
51static struct bpf_diag_mod_target diag_reg_target(u32 frame_id, u8 frameno, u8 regno)
52{
53 return (struct bpf_diag_mod_target){
54 .frame_id = frame_id,
55 .frameno = frameno,
56 .kind = BPF_DIAG_MOD_TARGET_REG,
57 .regno = regno,
58 };
59}
60
61static struct bpf_diag_mod_target diag_stack_arg_target(u32 frame_id, u8 frameno, u8 slot)
62{
63 return (struct bpf_diag_mod_target){
64 .frame_id = frame_id,
65 .frameno = frameno,
66 .kind = BPF_DIAG_MOD_TARGET_STACK_ARG,
67 .stack_arg = slot,
68 };
69}
70
71static struct bpf_diag_mod_target diag_stack_slot_target(u32 frame_id, u8 frameno, u16 spi)
72{
73 return (struct bpf_diag_mod_target){
74 .frame_id = frame_id,
75 .frameno = frameno,
76 .kind = BPF_DIAG_MOD_TARGET_STACK_SLOT,
77 .spi = spi,
78 };
79}
80
81static struct bpf_diag_mod_target diag_stack_range_target(u32 frame_id, u8 frameno,
82 s16 min_off, s16 max_off)
83{
84 return (struct bpf_diag_mod_target){
85 .frame_id = frame_id,
86 .frameno = frameno,
87 .kind = BPF_DIAG_MOD_TARGET_STACK_RANGE,
88 .range.min_off = min_off,
89 .range.max_off = max_off,
90 };
91}
92
93struct bpf_diag_reg_snapshot {
94 u32 type;
95 u32 btf_id;
96 const struct bpf_map *map_ptr;
97 const struct btf *btf;
98 struct tnum var_off;
99 struct cnum64 r64;
100};
101
28enum bpf_diag_history_kind {102enum bpf_diag_history_kind {
29 BPF_DIAG_HISTORY_BRANCH,103 BPF_DIAG_HISTORY_BRANCH,
104 BPF_DIAG_HISTORY_MOD,
30};105};
31 106
32struct bpf_diag_history_event {107struct bpf_diag_history_event {
@@ -37,6 +112,13 @@ struct bpf_diag_history_event { · 4 unmodified lines
37 struct {112 struct {
38 bool cond_true;113 bool cond_true;
39 } branch;114 } branch;
115 struct {
116 struct bpf_diag_mod_target target;
117 struct bpf_diag_mod_target origin;
118 struct bpf_diag_reg_snapshot old, new;
119 u8 reason;
120 bool origin_valid;
121 } mod;
40 };122 };
41};123};
42 124
@@ -77,10 +159,22 @@ struct bpf_diag_scratch { · 34 unmodified lines
77 struct disasm_line disasm_lines[BPF_DIAG_CONTEXT_CNT];159 struct disasm_line disasm_lines[BPF_DIAG_CONTEXT_CNT];
78};160};
79 161
162struct bpf_diag_mod_scope {
163 struct bpf_reg_state target_reg_snapshot;
164 struct bpf_diag_mod_target target;
165 struct bpf_diag_mod_target origin;
166 enum bpf_diag_mod_reason reason;
167 u32 insn_idx;
168 bool active;
169 bool origin_valid;
170};
171
80struct bpf_diag {172struct bpf_diag {
81 struct bpf_diag_log log;173 struct bpf_diag_log log;
82 struct bpf_diag_scratch scratch;174 struct bpf_diag_scratch scratch;
83 struct list_head fmt_chunks;175 struct list_head fmt_chunks;
176 struct bpf_diag_mod_scope mod;
177 u32 frame_id_gen;
84};178};
85 179
86bool bpf_diag_enabled(const struct bpf_verifier_env *env)180bool bpf_diag_enabled(const struct bpf_verifier_env *env)
@@ -103,6 +197,12 @@ int bpf_diag_init(struct bpf_verifier_env *env) · 16 unmodified lines
103 return 0;197 return 0;
104}198}
105 199
200void bpf_diag_init_frame(struct bpf_verifier_env *env, struct bpf_func_state *state)
201{
202 if (env->diag)
203 state->diag_frame_id = ++env->diag->frame_id_gen;
204}
205
106static char *diag_fmt_alloc(struct bpf_verifier_env *env, size_t size)206static char *diag_fmt_alloc(struct bpf_verifier_env *env, size_t size)
107{207{
108 struct bpf_diag *diag = env->diag;208 struct bpf_diag *diag = env->diag;
@@ -359,6 +459,28 @@ static void diag_print_wrapped_prefixed(struct bpf_verifier_env *env, const char · 250 unmodified lines
359 }459 }
360}460}
361 461
462const char *bpf_diag_fmt_btf_type(struct bpf_verifier_env *env, const struct btf *btf, u32 type_id)
463{
464 char *buf = bpf_diag_fmt_buf(env, BPF_DIAG_FMT_BUF_SIZE);
465 size_t len;
466 int ret;
467
468 if (!buf)
469 return "";
470
471 buf[0] = '\0';
472 ret = btf_type_name_to_buf(btf, type_id, buf, BPF_DIAG_FMT_BUF_SIZE);
473 if (ret < 0 || !buf[0]) {
474 scnprintf(buf, BPF_DIAG_FMT_BUF_SIZE, "BTF type ID %u", type_id);
475 return buf;
476 }
477
478 len = strlen(buf);
479 if (len && buf[len - 1] == '{')
480 buf[len - 1] = '\0';
481 return buf;
482}
483
362static int diag_line_width(unsigned int line)484static int diag_line_width(unsigned int line)
363{485{
364 int width = 1;486 int width = 1;
@@ -665,3 +787,237 @@ void bpf_diag_record_branch(struct bpf_verifier_env *env, u32 insn_idx, bool con · 300 unmodified lines
665 787
666 diag_append_history(env, &event);788 diag_append_history(env, &event);
667}789}
790
791static void diag_snapshot_reg(struct bpf_diag_reg_snapshot *snapshot,
792 const struct bpf_reg_state *reg)
793{
794 snapshot->type = reg->type;
795 if (type_is_map_ptr(reg->type))
796 snapshot->map_ptr = reg->map_ptr;
797 if (base_type(reg->type) == PTR_TO_BTF_ID && reg->btf && reg->btf_id) {
798 snapshot->btf_id = reg->btf_id;
799 snapshot->btf = reg->btf;
800 }
801 snapshot->var_off = reg->var_off;
802 snapshot->r64 = reg->r64;
803}
804
805static bool diag_mod_insn_origin(struct bpf_verifier_env *env, u32 insn_idx,
806 const struct bpf_diag_mod_target *target,
807 struct bpf_diag_mod_target *origin)
808{
809 const struct bpf_insn *insn = &env->prog->insnsi[insn_idx];
810 u8 class = BPF_CLASS(insn->code);
811 const struct bpf_func_state *state;
812
813 if (target->kind == BPF_DIAG_MOD_TARGET_REG && (class == BPF_ALU || class == BPF_ALU64) &&
814 BPF_OP(insn->code) == BPF_MOV && BPF_SRC(insn->code) == BPF_X) {
815 *origin = diag_reg_target(target->frame_id, target->frameno, insn->src_reg);
816 return true;
817 }
818
819 if ((target->kind != BPF_DIAG_MOD_TARGET_STACK_ARG &&
820 target->kind != BPF_DIAG_MOD_TARGET_STACK_SLOT) ||
821 class != BPF_STX)
822 return false;
823
824 state = env->cur_state->frame[env->cur_state->curframe];
825 *origin = diag_reg_target(state->diag_frame_id, state->frameno, insn->src_reg);
826 return true;
827}
828
829static bool diag_mod_keeps_lineage(struct bpf_verifier_env *env,
830 const struct bpf_diag_history_event *event)
831{
832 const struct bpf_insn *insn;
833 u8 class;
834
835 if (event->mod.reason != BPF_DIAG_MOD_WRITE ||
836 event->mod.target.kind != BPF_DIAG_MOD_TARGET_REG)
837 return false;
838
839 insn = &env->prog->insnsi[event->insn_idx];
840 class = BPF_CLASS(insn->code);
841 if (class != BPF_ALU && class != BPF_ALU64)
842 return false;
843
844 switch (BPF_OP(insn->code)) {
845 case BPF_ADD:
846 case BPF_SUB:
847 case BPF_MUL:
848 case BPF_OR:
849 case BPF_AND:
850 case BPF_LSH:
851 case BPF_RSH:
852 case BPF_ARSH:
853 case BPF_XOR:
854 case BPF_NEG:
855 case BPF_END:
856 return true;
857 default:
858 return false;
859 }
860}
861
862static void diag_record_mod(struct bpf_verifier_env *env, u32 insn_idx,
863 struct bpf_diag_mod_target target,
864 enum bpf_diag_mod_reason reason,
865 const struct bpf_reg_state *old_reg,
866 const struct bpf_reg_state *new_reg,
867 const struct bpf_diag_mod_target *origin)
868{
869 struct bpf_diag_history_event event = {
870 .insn_idx = insn_idx,
871 .kind = BPF_DIAG_HISTORY_MOD,
872 .mod = {
873 .target = target,
874 .reason = reason,
875 },
876 };
877
878 if (old_reg)
879 diag_snapshot_reg(&event.mod.old, old_reg);
880 if (new_reg)
881 diag_snapshot_reg(&event.mod.new, new_reg);
882 if (origin) {
883 event.mod.origin = *origin;
884 event.mod.origin_valid = true;
885 } else if (diag_mod_insn_origin(env, insn_idx, &target, &event.mod.origin)) {
886 event.mod.origin_valid = true;
887 }
888 if (old_reg && new_reg &&
889 (reason == BPF_DIAG_MOD_WRITE || reason == BPF_DIAG_MOD_SPILL) &&
890 !memcmp(&event.mod.old, &event.mod.new, sizeof(event.mod.old)) &&
891 !event.mod.origin_valid &&
892 diag_mod_keeps_lineage(env, &event))
893 return;
894
895 diag_append_history(env, &event);
896}
897
898static struct bpf_reg_state *target_to_reg(struct bpf_verifier_env *env,
899 const struct bpf_diag_mod_target *target)
900{
901 struct bpf_verifier_state *vstate = env->cur_state;
902 struct bpf_func_state *state;
903
904 state = target->frameno <= vstate->curframe ? vstate->frame[target->frameno] : NULL;
905
906 if (!state)
907 return NULL;
908 if (state->diag_frame_id != target->frame_id)
909 return NULL;
910
911 switch (target->kind) {
912 case BPF_DIAG_MOD_TARGET_REG:
913 if (target->regno >= MAX_BPF_REG)
914 return NULL;
915 return &state->regs[target->regno];
916 case BPF_DIAG_MOD_TARGET_STACK_ARG:
917 if (target->stack_arg >= state->out_stack_arg_cnt)
918 return NULL;
919 return &state->stack_arg_regs[target->stack_arg];
920 case BPF_DIAG_MOD_TARGET_STACK_SLOT:
921 if (target->spi >= state->allocated_stack / BPF_REG_SIZE)
922 return NULL;
923 return &state->stack[target->spi].spilled_ptr;
924 default:
925 return NULL;
926 }
927}
928
929static bool reg_to_target(struct bpf_verifier_env *env, const struct bpf_reg_state *reg,
930 struct bpf_diag_mod_target *target)
931{
932 struct bpf_verifier_state *vstate = env->cur_state;
933 unsigned long addr = (unsigned long)reg;
934 int frame;
935
936 for (frame = 0; frame <= vstate->curframe; frame++) {
937 struct bpf_func_state *state = vstate->frame[frame];
938 unsigned long start, end;
939 u32 nslots = state->allocated_stack / BPF_REG_SIZE;
940 int spi;
941
942 start = (unsigned long)state->regs;
943 end = (unsigned long)(state->regs + MAX_BPF_REG);
944 if (addr >= start && addr < end) {
945 *target = diag_reg_target(state->diag_frame_id, state->frameno,
946 reg - state->regs);
947 return true;
948 }
949
950 start = (unsigned long)state->stack_arg_regs;
951 end = (unsigned long)(state->stack_arg_regs + state->out_stack_arg_cnt);
952 if (state->out_stack_arg_cnt && addr >= start && addr < end) {
953 *target = diag_stack_arg_target(state->diag_frame_id, state->frameno,
954 reg - state->stack_arg_regs);
955 return true;
956 }
957
958 start = (unsigned long)state->stack;
959 end = (unsigned long)(state->stack + nslots);
960 if (nslots && addr >= start && addr < end) {
961 spi = ((const char *)reg - (const char *)state->stack) /
962 sizeof(*state->stack);
963 *target = diag_stack_slot_target(state->diag_frame_id, state->frameno, spi);
964 return true;
965 }
966 }
967 return false;
968}
969
970void bpf_diag_mod_begin(struct bpf_verifier_env *env, const struct bpf_reg_state *reg,
971 const struct bpf_reg_state *origin, enum bpf_diag_mod_reason reason)
972{
973 struct bpf_diag *diag = env->diag;
974
975 if (!diag)
976 return;
977 diag->mod.active = reg_to_target(env, reg, &diag->mod.target);
978 if (!diag->mod.active)
979 return;
980 diag->mod.target_reg_snapshot = *reg;
981 diag->mod.insn_idx = env->insn_idx;
982 diag->mod.reason = reason;
983 diag->mod.origin_valid = origin && reg_to_target(env, origin, &diag->mod.origin);
984}
985
986void bpf_diag_mod_end(struct bpf_verifier_env *env)
987{
988 struct bpf_diag *diag = env->diag;
989 const struct bpf_reg_state *new_reg;
990
991 if (!diag || !diag->mod.active)
992 return;
993 diag->mod.active = false;
994 /*
995 * Resolve the target again because the enclosing function state's stack
996 * may have been reallocated while the modification was in progress.
997 */
998 new_reg = target_to_reg(env, &diag->mod.target);
999 if (!new_reg)
1000 return;
1001 diag_record_mod(env, diag->mod.insn_idx, diag->mod.target, diag->mod.reason,
1002 &diag->mod.target_reg_snapshot, new_reg,
1003 diag->mod.origin_valid ? &diag->mod.origin : NULL);
1004}
1005
1006void bpf_diag_record_scrub(struct bpf_verifier_env *env, const struct bpf_reg_state *reg,
1007 enum bpf_diag_mod_reason reason)
1008{
1009 struct bpf_diag_mod_target target;
1010
1011 if (!env->diag || reg->type == NOT_INIT || !reg_to_target(env, reg, &target))
1012 return;
1013 diag_record_mod(env, env->insn_idx, target, reason, reg, NULL, NULL);
1014}
1015
1016void bpf_diag_record_scrub_stack(struct bpf_verifier_env *env,
1017 const struct bpf_func_state *state, s16 min_off, s16 max_off,
1018 enum bpf_diag_mod_reason reason)
1019{
1020 diag_record_mod(env, env->insn_idx,
1021 diag_stack_range_target(state->diag_frame_id, state->frameno, min_off, max_off),
1022 reason, NULL, NULL, NULL);
1023}
kernel/bpf/diagnostics.h+23−0
@@ -8,17 +8,40 @@ · 7 unmodified lines
8#include <linux/stdarg.h>8#include <linux/stdarg.h>
9#include <linux/types.h>9#include <linux/types.h>
10 10
11struct bpf_func_state;
12struct bpf_reg_state;
11struct bpf_verifier_env;13struct bpf_verifier_env;
14struct btf;
15
16enum bpf_diag_mod_reason {
17 BPF_DIAG_MOD_WRITE,
18 BPF_DIAG_MOD_SPILL,
19 BPF_DIAG_MOD_VAR_WRITE,
20 BPF_DIAG_MOD_REF_RELEASE,
21 BPF_DIAG_MOD_PKT_DATA_CHANGE,
22 BPF_DIAG_MOD_NON_OWN_REF,
23 BPF_DIAG_MOD_CALLER_SAVED,
24};
12 25
13bool bpf_diag_enabled(const struct bpf_verifier_env *env);26bool bpf_diag_enabled(const struct bpf_verifier_env *env);
14int bpf_diag_init(struct bpf_verifier_env *env);27int bpf_diag_init(struct bpf_verifier_env *env);
28void bpf_diag_init_frame(struct bpf_verifier_env *env, struct bpf_func_state *state);
15char *bpf_diag_fmt_buf(struct bpf_verifier_env *env, size_t size);29char *bpf_diag_fmt_buf(struct bpf_verifier_env *env, size_t size);
16const char *bpf_diag_vfmt(struct bpf_verifier_env *env, const char *fmt, va_list args)30const char *bpf_diag_vfmt(struct bpf_verifier_env *env, const char *fmt, va_list args)
17 __printf(2, 0);31 __printf(2, 0);
18const char *bpf_diag_fmt(struct bpf_verifier_env *env, const char *fmt, ...) __printf(2, 3);32const char *bpf_diag_fmt(struct bpf_verifier_env *env, const char *fmt, ...) __printf(2, 3);
33const char *bpf_diag_fmt_btf_type(struct bpf_verifier_env *env, const struct btf *btf, u32 type_id);
19u64 bpf_diag_event_log_save(struct bpf_verifier_env *env);34u64 bpf_diag_event_log_save(struct bpf_verifier_env *env);
20void bpf_diag_event_log_restore(struct bpf_verifier_env *env, u64 log_pos);35void bpf_diag_event_log_restore(struct bpf_verifier_env *env, u64 log_pos);
21void bpf_diag_free(struct bpf_verifier_env *env);36void bpf_diag_free(struct bpf_verifier_env *env);
22void bpf_diag_record_branch(struct bpf_verifier_env *env, u32 insn_idx, bool cond_true);37void bpf_diag_record_branch(struct bpf_verifier_env *env, u32 insn_idx, bool cond_true);
38void bpf_diag_mod_begin(struct bpf_verifier_env *env, const struct bpf_reg_state *reg,
39 const struct bpf_reg_state *origin, enum bpf_diag_mod_reason reason);
40void bpf_diag_mod_end(struct bpf_verifier_env *env);
41void bpf_diag_record_scrub(struct bpf_verifier_env *env, const struct bpf_reg_state *reg,
42 enum bpf_diag_mod_reason reason);
43void bpf_diag_record_scrub_stack(struct bpf_verifier_env *env,
44 const struct bpf_func_state *state, s16 min_off, s16 max_off,
45 enum bpf_diag_mod_reason reason);
23 46
24#endif /* __BPF_DIAGNOSTICS_H */47#endif /* __BPF_DIAGNOSTICS_H */
kernel/bpf/log.c+0−11
@@ -615,17 +615,6 @@ static void print_scalar_ranges(struct bpf_verifier_env *env, · 614 unmodified lines
615 }615 }
616}616}
617 617
618static bool type_is_map_ptr(enum bpf_reg_type t) {
619 switch (base_type(t)) {
620 case CONST_PTR_TO_MAP:
621 case PTR_TO_MAP_KEY:
622 case PTR_TO_MAP_VALUE:
623 return true;
624 default:
625 return false;
626 }
627}
628
629/*618/*
630 * _a stands for append, was shortened to avoid multiline statements below.619 * _a stands for append, was shortened to avoid multiline statements below.
631 * This macro is used to output a comma separated list of attributes.620 * This macro is used to output a comma separated list of attributes.
kernel/bpf/verifier.c+119−12
@@ -1792,6 +1792,17 @@ static const int caller_saved[CALLER_SAVED_REGS] = { · 1791 unmodified lines
1792 BPF_REG_0, BPF_REG_1, BPF_REG_2, BPF_REG_3, BPF_REG_4, BPF_REG_51792 BPF_REG_0, BPF_REG_1, BPF_REG_2, BPF_REG_3, BPF_REG_4, BPF_REG_5
1793};1793};
1794 1794
1795static void bpf_diag_record_caller_saved(struct bpf_verifier_env *env,
1796 struct bpf_reg_state *regs)
1797{
1798 int i;
1799
1800 for (i = 1; i < CALLER_SAVED_REGS; i++) {
1801 bpf_diag_record_scrub(env, &regs[caller_saved[i]],
1802 BPF_DIAG_MOD_CALLER_SAVED);
1803 }
1804}
1805
1795/* This helper doesn't clear reg->id */1806/* This helper doesn't clear reg->id */
1796static void ___mark_reg_known(struct bpf_reg_state *reg, u64 imm)1807static void ___mark_reg_known(struct bpf_reg_state *reg, u64 imm)
1797{1808{
@@ -2245,6 +2256,7 @@ static void init_func_state(struct bpf_verifier_env *env, · 447 unmodified lines
2245{2256{
2246 state->callsite = callsite;2257 state->callsite = callsite;
2247 state->frameno = frameno;2258 state->frameno = frameno;
2259 bpf_diag_init_frame(env, state);
2248 state->subprogno = subprogno;2260 state->subprogno = subprogno;
2249 state->callback_ret_range = retval_range(0, 0);2261 state->callback_ret_range = retval_range(0, 0);
2250 init_reg_state(env, state);2262 init_reg_state(env, state);
@@ -3362,6 +3374,7 @@ static void save_register_state(struct bpf_verifier_env *env, · 1111 unmodified lines
3362{3374{
3363 int i;3375 int i;
3364 3376
3377 bpf_diag_mod_begin(env, &state->stack[spi].spilled_ptr, reg, BPF_DIAG_MOD_SPILL);
3365 state->stack[spi].spilled_ptr = *reg;3378 state->stack[spi].spilled_ptr = *reg;
3366 3379
3367 for (i = BPF_REG_SIZE; i > BPF_REG_SIZE - size; i--)3380 for (i = BPF_REG_SIZE; i > BPF_REG_SIZE - size; i--)
@@ -3370,6 +3383,8 @@ static void save_register_state(struct bpf_verifier_env *env, · 2 unmodified lines
3370 /* size < 8 bytes spill */3383 /* size < 8 bytes spill */
3371 for (; i; i--)3384 for (; i; i--)
3372 mark_stack_slot_misc(env, &state->stack[spi].slot_type[i - 1]);3385 mark_stack_slot_misc(env, &state->stack[spi].slot_type[i - 1]);
3386
3387 bpf_diag_mod_end(env);
3373}3388}
3374 3389
3375static bool is_bpf_st_mem(struct bpf_insn *insn)3390static bool is_bpf_st_mem(struct bpf_insn *insn)
@@ -3506,6 +3521,9 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env, · 130 unmodified lines
3506 } else {3521 } else {
3507 u8 type = STACK_MISC;3522 u8 type = STACK_MISC;
3508 3523
3524 if (bpf_is_spilled_reg(&state->stack[spi]))
3525 bpf_diag_record_scrub(env, &state->stack[spi].spilled_ptr,
3526 BPF_DIAG_MOD_WRITE);
3509 scrub_special_slot(state, spi);3527 scrub_special_slot(state, spi);
3510 3528
3511 /* when we zero initialize stack slots mark them as such */3529 /* when we zero initialize stack slots mark them as such */
@@ -3666,6 +3684,8 @@ static int check_stack_write_var_off(struct bpf_verifier_env *env, · 154 unmodified lines
3666 if (err)3684 if (err)
3667 return err;3685 return err;
3668 }3686 }
3687 bpf_diag_record_scrub_stack(env, state, min_off, max_off,
3688 BPF_DIAG_MOD_VAR_WRITE);
3669 return 0;3689 return 0;
3670}3690}
3671 3691
@@ -3758,6 +3778,12 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env, · 86 unmodified lines
3758 mark_stack_slot_scratched(env, spi);3778 mark_stack_slot_scratched(env, spi);
3759 check_fastcall_stack_contract(env, state, env->insn_idx, off);3779 check_fastcall_stack_contract(env, state, env->insn_idx, off);
3760 3780
3781 /*
3782 * Refine the in-progress load record's origin to the source stack slot.
3783 */
3784 if (dst_regno >= 0)
3785 bpf_diag_mod_begin(env, &state->regs[dst_regno], reg, BPF_DIAG_MOD_WRITE);
3786
3761 if (bpf_is_spilled_reg(&reg_state->stack[spi])) {3787 if (bpf_is_spilled_reg(&reg_state->stack[spi])) {
3762 u8 spill_size = 1;3788 u8 spill_size = 1;
3763 3789
@@ -4051,14 +4077,17 @@ static int check_stack_arg_write(struct bpf_verifier_env *env, struct bpf_func_s · 287 unmodified lines
4051 if (spi + 1 > subprog->max_out_stack_arg_cnt)4077 if (spi + 1 > subprog->max_out_stack_arg_cnt)
4052 subprog->max_out_stack_arg_cnt = spi + 1;4078 subprog->max_out_stack_arg_cnt = spi + 1;
4053 4079
4080 arg = &state->stack_arg_regs[spi];
4081 bpf_diag_mod_begin(env, arg, value_reg, BPF_DIAG_MOD_WRITE);
4082
4054 if (value_reg) {4083 if (value_reg) {
4055 state->stack_arg_regs[spi] = *value_reg;4084 state->stack_arg_regs[spi] = *value_reg;
4056 } else {4085 } else {
4057 /* BPF_ST: store immediate, treat as scalar */4086 /* BPF_ST: store immediate, treat as scalar */
4058 arg = &state->stack_arg_regs[spi];
4059 arg->type = SCALAR_VALUE;4087 arg->type = SCALAR_VALUE;
4060 __mark_reg_known(arg, env->prog->insnsi[env->insn_idx].imm);4088 __mark_reg_known(arg, env->prog->insnsi[env->insn_idx].imm);
4061 }4089 }
4090 bpf_diag_mod_end(env);
4062 state->no_stack_arg_load = true;4091 state->no_stack_arg_load = true;
4063 return bpf_push_jmp_history(env, env->cur_state,4092 return bpf_push_jmp_history(env, env->cur_state,
4064 INSN_F_STACK_ARG_ACCESS, spi, 0, 0);4093 INSN_F_STACK_ARG_ACCESS, spi, 0, 0);
@@ -4091,7 +4120,9 @@ static int check_stack_arg_read(struct bpf_verifier_env *env, struct bpf_func_st · 26 unmodified lines
4091 caller = vstate->frame[vstate->curframe - 1];4120 caller = vstate->frame[vstate->curframe - 1];
4092 arg = &caller->stack_arg_regs[spi];4121 arg = &caller->stack_arg_regs[spi];
4093 cur = vstate->frame[vstate->curframe];4122 cur = vstate->frame[vstate->curframe];
4123 bpf_diag_mod_begin(env, &cur->regs[dst_regno], arg, BPF_DIAG_MOD_WRITE);
4094 cur->regs[dst_regno] = *arg;4124 cur->regs[dst_regno] = *arg;
4125 bpf_diag_mod_end(env);
4095 return bpf_push_jmp_history(env, env->cur_state,4126 return bpf_push_jmp_history(env, env->cur_state,
4096 INSN_F_STACK_ARG_ACCESS, spi, 0, 0);4127 INSN_F_STACK_ARG_ACCESS, spi, 0, 0);
4097}4128}
@@ -6426,15 +6457,19 @@ static int check_load_mem(struct bpf_verifier_env *env, struct bpf_insn *insn, · 2328 unmodified lines
6426 6457
6427 src_reg_type = regs[insn->src_reg].type;6458 src_reg_type = regs[insn->src_reg].type;
6428 6459
6429 /* Check if (src_reg + off) is readable. The state of dst_reg will be6460 /*
6430 * updated by this call.6461 * check_stack_read_fixed_off() may refine the modification's origin to
6462 * the source stack slot.
6431 */6463 */
6464 bpf_diag_mod_begin(env, &regs[insn->dst_reg], NULL, BPF_DIAG_MOD_WRITE);
6432 err = check_mem_access(env, env->insn_idx, regs + insn->src_reg, argno_from_reg(insn->src_reg), insn->off,6465 err = check_mem_access(env, env->insn_idx, regs + insn->src_reg, argno_from_reg(insn->src_reg), insn->off,
6433 BPF_SIZE(insn->code), BPF_READ, insn->dst_reg,6466 BPF_SIZE(insn->code), BPF_READ, insn->dst_reg,
6434 strict_alignment_once, is_ldsx);6467 strict_alignment_once, is_ldsx);
6435 err = err ?: save_aux_ptr_type(env, src_reg_type,6468 err = err ?: save_aux_ptr_type(env, src_reg_type,
6436 allow_trust_mismatch);6469 allow_trust_mismatch);
6437 err = err ?: reg_bounds_sanity_check(env, &regs[insn->dst_reg], ctx);6470 err = err ?: reg_bounds_sanity_check(env, &regs[insn->dst_reg], ctx);
6471 if (!err)
6472 bpf_diag_mod_end(env);
6438 6473
6439 return err;6474 return err;
6440}6475}
@@ -6540,10 +6575,14 @@ static int check_atomic_rmw(struct bpf_verifier_env *env, · 99 unmodified lines
6540 */6575 */
6541 err = check_mem_access(env, env->insn_idx, dst_reg, argno_from_reg(insn->dst_reg), insn->off,6576 err = check_mem_access(env, env->insn_idx, dst_reg, argno_from_reg(insn->dst_reg), insn->off,
6542 BPF_SIZE(insn->code), BPF_READ, -1, true, false);6577 BPF_SIZE(insn->code), BPF_READ, -1, true, false);
6543 if (!err && load_reg >= 0)6578 if (!err && load_reg >= 0) {
6579 bpf_diag_mod_begin(env, cur_regs(env) + load_reg, NULL, BPF_DIAG_MOD_WRITE);
6544 err = check_mem_access(env, env->insn_idx, dst_reg, argno_from_reg(insn->dst_reg),6580 err = check_mem_access(env, env->insn_idx, dst_reg, argno_from_reg(insn->dst_reg),
6545 insn->off, BPF_SIZE(insn->code),6581 insn->off, BPF_SIZE(insn->code),
6546 BPF_READ, load_reg, true, false);6582 BPF_READ, load_reg, true, false);
6583 if (!err)
6584 bpf_diag_mod_end(env);
6585 }
6547 if (err)6586 if (err)
6548 return err;6587 return err;
6549 6588
@@ -8945,8 +8984,10 @@ static void clear_all_pkt_pointers(struct bpf_verifier_env *env) · 2395 unmodified lines
8945 struct bpf_reg_state *reg;8984 struct bpf_reg_state *reg;
8946 8985
8947 bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({8986 bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({
8948 if (reg_is_pkt_pointer_any(reg) || reg_is_dynptr_slice_pkt(reg))8987 if (reg_is_pkt_pointer_any(reg) || reg_is_dynptr_slice_pkt(reg)) {
8988 bpf_diag_record_scrub(env, reg, BPF_DIAG_MOD_PKT_DATA_CHANGE);
8949 mark_reg_invalid(env, reg);8989 mark_reg_invalid(env, reg);
8990 }
8950 }));8991 }));
8951}8992}
8952 8993
@@ -9062,10 +9103,25 @@ static int release_reference(struct bpf_verifier_env *env, int id) · 109 unmodified lines
9062 return err;9103 return err;
9063 }9104 }
9064 9105
9106 /*
9107 * A dynptr occupies two stack slots that invalidate_dynptr()
9108 * clears together. Record both scrubs before invalidating it.
9109 */
9110 if (stack && stack->slot_type[BPF_REG_SIZE - 1] == STACK_DYNPTR) {
9111 struct bpf_stack_state *dyn_stack = stack;
9112
9113 if (reg->dynptr.first_slot)
9114 dyn_stack--;
9115 bpf_diag_record_scrub(env, &dyn_stack[0].spilled_ptr,
9116 BPF_DIAG_MOD_REF_RELEASE);
9117 bpf_diag_record_scrub(env, &dyn_stack[1].spilled_ptr,
9118 BPF_DIAG_MOD_REF_RELEASE);
9119 invalidate_dynptr(env, dyn_stack);
9120 continue;
9121 }
9122 bpf_diag_record_scrub(env, reg, BPF_DIAG_MOD_REF_RELEASE);
9065 if (!stack || stack->slot_type[BPF_REG_SIZE - 1] == STACK_SPILL)9123 if (!stack || stack->slot_type[BPF_REG_SIZE - 1] == STACK_SPILL)
9066 mark_reg_invalid(env, reg);9124 mark_reg_invalid(env, reg);
9067 else if (stack->slot_type[BPF_REG_SIZE - 1] == STACK_DYNPTR)
9068 invalidate_dynptr(env, stack);
9069 }));9125 }));
9070 }9126 }
9071 9127
@@ -9078,8 +9134,10 @@ static void invalidate_non_owning_refs(struct bpf_verifier_env *env) · 6 unmodified lines
9078 struct bpf_reg_state *reg;9134 struct bpf_reg_state *reg;
9079 9135
9080 bpf_for_each_reg_in_vstate(env->cur_state, unused, reg, ({9136 bpf_for_each_reg_in_vstate(env->cur_state, unused, reg, ({
9081 if (type_is_non_owning_ref(reg->type))9137 if (type_is_non_owning_ref(reg->type)) {
9138 bpf_diag_record_scrub(env, reg, BPF_DIAG_MOD_NON_OWN_REF);
9082 mark_reg_invalid(env, reg);9139 mark_reg_invalid(env, reg);
9140 }
9083 }));9141 }));
9084}9142}
9085 9143
@@ -9092,8 +9150,10 @@ static void invalidate_rcu_protected_refs(struct bpf_verifier_env *env) · 6 unmodified lines
9092 9150
9093 bpf_for_each_reg_in_vstate_mask(env->cur_state, state, reg, stack, clear_mask, ({9151 bpf_for_each_reg_in_vstate_mask(env->cur_state, state, reg, stack, clear_mask, ({
9094 if (reg->type & MEM_RCU) {9152 if (reg->type & MEM_RCU) {
9153 bpf_diag_mod_begin(env, reg, NULL, BPF_DIAG_MOD_WRITE);
9095 reg->type &= ~(MEM_RCU | PTR_MAYBE_NULL);9154 reg->type &= ~(MEM_RCU | PTR_MAYBE_NULL);
9096 reg->type |= PTR_UNTRUSTED;9155 reg->type |= PTR_UNTRUSTED;
9156 bpf_diag_mod_end(env);
9097 }9157 }
9098 }));9158 }));
9099}9159}
@@ -9110,9 +9170,11 @@ static int ref_convert_alloc_rcu_protected(struct bpf_verifier_env *env, u32 id) · 10 unmodified lines
9110 if (reg->id != id)9170 if (reg->id != id)
9111 continue;9171 continue;
9112 if ((reg->type & MEM_ALLOC) && (reg->type & MEM_PERCPU)) {9172 if ((reg->type & MEM_ALLOC) && (reg->type & MEM_PERCPU)) {
9173 bpf_diag_mod_begin(env, reg, NULL, BPF_DIAG_MOD_WRITE);
9113 reg->id = 0;9174 reg->id = 0;
9114 reg->type &= ~MEM_ALLOC;9175 reg->type &= ~MEM_ALLOC;
9115 reg->type |= MEM_RCU;9176 reg->type |= MEM_RCU;
9177 bpf_diag_mod_end(env);
9116 }9178 }
9117 }));9179 }));
9118 9180
@@ -9124,6 +9186,8 @@ static void clear_caller_saved_regs(struct bpf_verifier_env *env, · 5 unmodified lines
9124{9186{
9125 int i;9187 int i;
9126 9188
9189 bpf_diag_record_caller_saved(env, regs);
9190
9127 /* after the call registers r0 - r5 were scratched */9191 /* after the call registers r0 - r5 were scratched */
9128 for (i = 0; i < CALLER_SAVED_REGS; i++) {9192 for (i = 0; i < CALLER_SAVED_REGS; i++) {
9129 bpf_mark_reg_not_init(env, &regs[caller_saved[i]]);9193 bpf_mark_reg_not_init(env, &regs[caller_saved[i]]);
@@ -9131,13 +9195,15 @@ static void clear_caller_saved_regs(struct bpf_verifier_env *env, · 1 unmodified line
9131 }9195 }
9132}9196}
9133 9197
9134static void invalidate_outgoing_stack_args(const struct bpf_verifier_env *env,9198static void invalidate_outgoing_stack_args(struct bpf_verifier_env *env,
9135 struct bpf_func_state *state)9199 struct bpf_func_state *state)
9136{9200{
9137 int i, nslots = state->out_stack_arg_cnt;9201 int i, nslots = state->out_stack_arg_cnt;
9138 9202
9139 for (i = 0; i < nslots; i++)9203 for (i = 0; i < nslots; i++) {
9204 bpf_diag_record_scrub(env, &state->stack_arg_regs[i], BPF_DIAG_MOD_CALLER_SAVED);
9140 bpf_mark_reg_not_init(env, &state->stack_arg_regs[i]);9205 bpf_mark_reg_not_init(env, &state->stack_arg_regs[i]);
9206 }
9141}9207}
9142 9208
9143typedef int (*set_callee_state_fn)(struct bpf_verifier_env *env,9209typedef int (*set_callee_state_fn)(struct bpf_verifier_env *env,
@@ -9436,6 +9502,7 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, · 292 unmodified lines
9436 return err;9502 return err;
9437 if (bpf_subprog_is_global(env, subprog)) {9503 if (bpf_subprog_is_global(env, subprog)) {
9438 const char *sub_name = bpf_subprog_name(env, subprog);9504 const char *sub_name = bpf_subprog_name(env, subprog);
9505 bool returns_void;
9439 9506
9440 if (env->cur_state->active_locks) {9507 if (env->cur_state->active_locks) {
9441 verbose(env, "global function calls are not allowed while holding a lock,\n"9508 verbose(env, "global function calls are not allowed while holding a lock,\n"
@@ -9458,16 +9525,22 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, · 16 unmodified lines
9458 if (env->log.level & BPF_LOG_LEVEL)9525 if (env->log.level & BPF_LOG_LEVEL)
9459 verbose(env, "Func#%d ('%s') is global and assumed valid.\n",9526 verbose(env, "Func#%d ('%s') is global and assumed valid.\n",
9460 subprog, sub_name);9527 subprog, sub_name);
9528 returns_void = subprog_returns_void(env, subprog);
9461 if (env->subprog_info[subprog].changes_pkt_data)9529 if (env->subprog_info[subprog].changes_pkt_data)
9462 clear_all_pkt_pointers(env);9530 clear_all_pkt_pointers(env);
9463 /* mark global subprog for verifying after main prog */9531 /* mark global subprog for verifying after main prog */
9464 subprog_aux(env, subprog)->called = true;9532 subprog_aux(env, subprog)->called = true;
9533 if (returns_void)
9534 bpf_diag_record_scrub(env, &caller->regs[BPF_REG_0], BPF_DIAG_MOD_CALLER_SAVED);
9535 else
9536 bpf_diag_mod_begin(env, &caller->regs[BPF_REG_0], NULL, BPF_DIAG_MOD_WRITE);
9465 clear_caller_saved_regs(env, caller->regs);9537 clear_caller_saved_regs(env, caller->regs);
9466 invalidate_outgoing_stack_args(env, cur_func(env));9538 invalidate_outgoing_stack_args(env, cur_func(env));
9467 9539
9468 /* All non-void global functions return a 64-bit SCALAR_VALUE. */9540 /* All non-void global functions return a 64-bit SCALAR_VALUE. */
9469 if (!subprog_returns_void(env, subprog)) {9541 if (!returns_void) {
9470 mark_reg_unknown(env, caller->regs, BPF_REG_0);9542 mark_reg_unknown(env, caller->regs, BPF_REG_0);
9543 bpf_diag_mod_end(env);
9471 }9544 }
9472 9545
9473 if (env->subprog_info[subprog].might_throw) {9546 if (env->subprog_info[subprog].might_throw) {
@@ -9502,6 +9575,7 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, · 28 unmodified lines
9502 if (err)9575 if (err)
9503 return err;9576 return err;
9504 9577
9578 bpf_diag_record_scrub(env, &caller->regs[BPF_REG_0], BPF_DIAG_MOD_CALLER_SAVED);
9505 clear_caller_saved_regs(env, caller->regs);9579 clear_caller_saved_regs(env, caller->regs);
9506 9580
9507 /* and go analyze first insn of the callee */9581 /* and go analyze first insn of the callee */
@@ -9865,7 +9939,9 @@ static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx) · 357 unmodified lines
9865 }9939 }
9866 } else {9940 } else {
9867 /* return to the caller whatever r0 had in the callee */9941 /* return to the caller whatever r0 had in the callee */
9942 bpf_diag_mod_begin(env, &caller->regs[BPF_REG_0], r0, BPF_DIAG_MOD_WRITE);
9868 caller->regs[BPF_REG_0] = *r0;9943 caller->regs[BPF_REG_0] = *r0;
9944 bpf_diag_mod_end(env);
9869 }9945 }
9870 9946
9871 /* for callbacks like bpf_loop or bpf_for_each_map_elem go back to callsite,9947 /* for callbacks like bpf_loop or bpf_for_each_map_elem go back to callsite,
@@ -10518,12 +10594,14 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn · 646 unmodified lines
10518 return err;10594 return err;
10519 10595
10520 /* reset caller saved regs */10596 /* reset caller saved regs */
10597 bpf_diag_record_caller_saved(env, regs);
10521 for (i = 0; i < CALLER_SAVED_REGS; i++) {10598 for (i = 0; i < CALLER_SAVED_REGS; i++) {
10522 bpf_mark_reg_not_init(env, &regs[caller_saved[i]]);10599 bpf_mark_reg_not_init(env, &regs[caller_saved[i]]);
10523 check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK);10600 check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK);
10524 }10601 }
10525 invalidate_outgoing_stack_args(env, cur_func(env));10602 invalidate_outgoing_stack_args(env, cur_func(env));
10526 10603
10604 bpf_diag_mod_begin(env, &regs[BPF_REG_0], NULL, BPF_DIAG_MOD_WRITE);
10527 /* update return register (already marked as written above) */10605 /* update return register (already marked as written above) */
10528 ret_type = fn->ret_type;10606 ret_type = fn->ret_type;
10529 ret_flag = type_flag(ret_type);10607 ret_flag = type_flag(ret_type);
@@ -10672,6 +10750,8 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn · 142 unmodified lines
10672 if (err)10750 if (err)
10673 return err;10751 return err;
10674 10752
10753 bpf_diag_mod_end(env);
10754
10675 /*10755 /*
10676 * In order for a release of any of the original or cast pointers10756 * In order for a release of any of the original or cast pointers
10677 * to invalidate all other pointers, reuse the same reference id for10757 * to invalidate all other pointers, reuse the same reference id for
@@ -10688,6 +10768,7 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn · 10 unmodified lines
10688 __mark_reg_known_zero(r0);10768 __mark_reg_known_zero(r0);
10689 r0->type = SCALAR_VALUE;10769 r0->type = SCALAR_VALUE;
10690 10770
10771 bpf_diag_mod_begin(env, &regs[BPF_REG_0], NULL, BPF_DIAG_MOD_WRITE);
10691 regs[BPF_REG_0].type &= ~PTR_MAYBE_NULL;10772 regs[BPF_REG_0].type &= ~PTR_MAYBE_NULL;
10692 regs[BPF_REG_0].id = meta.ref_obj.id;10773 regs[BPF_REG_0].id = meta.ref_obj.id;
10693 } else if (is_acquire_function(func_id, meta.map.ptr)) {10774 } else if (is_acquire_function(func_id, meta.map.ptr)) {
@@ -10706,6 +10787,8 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn · 12 unmodified lines
10706 if (err)10787 if (err)
10707 return err;10788 return err;
10708 10789
10790 bpf_diag_mod_end(env);
10791
10709 err = check_map_func_compatibility(env, meta.map.ptr, func_id);10792 err = check_map_func_compatibility(env, meta.map.ptr, func_id);
10710 if (err)10793 if (err)
10711 return err;10794 return err;
@@ -13211,6 +13294,8 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, · 2499 unmodified lines
13211 }13294 }
13212 }13295 }
13213 13296
13297 bpf_diag_record_caller_saved(env, regs);
13298 bpf_diag_mod_begin(env, &regs[BPF_REG_0], NULL, BPF_DIAG_MOD_WRITE);
13214 for (i = 0; i < CALLER_SAVED_REGS; i++) {13299 for (i = 0; i < CALLER_SAVED_REGS; i++) {
13215 u32 regno = caller_saved[i];13300 u32 regno = caller_saved[i];
13216 13301
@@ -13362,6 +13447,12 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, · 145 unmodified lines
13362 caller_info->stack_arg_cnt = stack_arg_cnt;13447 caller_info->stack_arg_cnt = stack_arg_cnt;
13363 }13448 }
13364 13449
13450 /*
13451 * Record R0 before process_iter_next_call() snapshots the alternate
13452 * iterator path's diagnostic position.
13453 */
13454 bpf_diag_mod_end(env);
13455
13365 if (bpf_is_iter_next_kfunc(&meta)) {13456 if (bpf_is_iter_next_kfunc(&meta)) {
13366 err = process_iter_next_call(env, insn_idx, &meta);13457 err = process_iter_next_call(env, insn_idx, &meta);
13367 if (err)13458 if (err)
@@ -15004,6 +15095,8 @@ static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn) · 1636 unmodified lines
15004 u8 opcode = BPF_OP(insn->code);15095 u8 opcode = BPF_OP(insn->code);
15005 int err;15096 int err;
15006 15097
15098 bpf_diag_mod_begin(env, &regs[insn->dst_reg], NULL, BPF_DIAG_MOD_WRITE);
15099
15007 if (opcode == BPF_END || opcode == BPF_NEG) {15100 if (opcode == BPF_END || opcode == BPF_NEG) {
15008 /* check src operand */15101 /* check src operand */
15009 err = check_reg_arg(env, insn->dst_reg, SRC_OP);15102 err = check_reg_arg(env, insn->dst_reg, SRC_OP);
@@ -15177,7 +15270,12 @@ static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn) · 167 unmodified lines
15177 return err;15270 return err;
15178 }15271 }
15179 15272
15180 return reg_bounds_sanity_check(env, &regs[insn->dst_reg], "alu");15273 err = reg_bounds_sanity_check(env, &regs[insn->dst_reg], "alu");
15274 if (err)
15275 return err;
15276
15277 bpf_diag_mod_end(env);
15278 return 0;
15181}15279}
15182 15280
15183static void find_good_pkt_pointers(struct bpf_verifier_state *vstate,15281static void find_good_pkt_pointers(struct bpf_verifier_state *vstate,
@@ -16271,11 +16369,13 @@ static int check_ld_imm(struct bpf_verifier_env *env, struct bpf_insn *insn) · 1087 unmodified lines
16271 return err;16369 return err;
16272 16370
16273 dst_reg = &regs[insn->dst_reg];16371 dst_reg = &regs[insn->dst_reg];
16372 bpf_diag_mod_begin(env, dst_reg, NULL, BPF_DIAG_MOD_WRITE);
16274 if (insn->src_reg == 0) {16373 if (insn->src_reg == 0) {
16275 u64 imm = ((u64)(insn + 1)->imm << 32) | (u32)insn->imm;16374 u64 imm = ((u64)(insn + 1)->imm << 32) | (u32)insn->imm;
16276 16375
16277 dst_reg->type = SCALAR_VALUE;16376 dst_reg->type = SCALAR_VALUE;
16278 __mark_reg_known(&regs[insn->dst_reg], imm);16377 __mark_reg_known(&regs[insn->dst_reg], imm);
16378 bpf_diag_mod_end(env);
16279 return 0;16379 return 0;
16280 }16380 }
16281 16381
@@ -16299,6 +16399,7 @@ static int check_ld_imm(struct bpf_verifier_env *env, struct bpf_insn *insn) · 17 unmodified lines
16299 verifier_bug(env, "pseudo btf id: unexpected dst reg type");16399 verifier_bug(env, "pseudo btf id: unexpected dst reg type");
16300 return -EFAULT;16400 return -EFAULT;
16301 }16401 }
16402 bpf_diag_mod_end(env);
16302 return 0;16403 return 0;
16303 }16404 }
16304 16405
@@ -16318,6 +16419,7 @@ static int check_ld_imm(struct bpf_verifier_env *env, struct bpf_insn *insn) · 13 unmodified lines
16318 16419
16319 dst_reg->type = PTR_TO_FUNC;16420 dst_reg->type = PTR_TO_FUNC;
16320 dst_reg->subprogno = subprogno;16421 dst_reg->subprogno = subprogno;
16422 bpf_diag_mod_end(env);
16321 return 0;16423 return 0;
16322 }16424 }
16323 16425
@@ -16328,6 +16430,7 @@ static int check_ld_imm(struct bpf_verifier_env *env, struct bpf_insn *insn) · 4 unmodified lines
16328 if (map->map_type == BPF_MAP_TYPE_ARENA) {16430 if (map->map_type == BPF_MAP_TYPE_ARENA) {
16329 __mark_reg_unknown(env, dst_reg);16431 __mark_reg_unknown(env, dst_reg);
16330 dst_reg->map_ptr = map;16432 dst_reg->map_ptr = map;
16433 bpf_diag_mod_end(env);
16331 return 0;16434 return 0;
16332 }16435 }
16333 __mark_reg_known(dst_reg, aux->map_off);16436 __mark_reg_known(dst_reg, aux->map_off);
@@ -16345,6 +16448,7 @@ static int check_ld_imm(struct bpf_verifier_env *env, struct bpf_insn *insn) · 11 unmodified lines
16345 return -EFAULT;16448 return -EFAULT;
16346 }16449 }
16347 16450
16451 bpf_diag_mod_end(env);
16348 return 0;16452 return 0;
16349}16453}
16350 16454
@@ -16423,6 +16527,8 @@ static int check_ld_abs(struct bpf_verifier_env *env, struct bpf_insn *insn) · 72 unmodified lines
16423 return err;16527 return err;
16424 16528
16425 /* reset caller saved regs to unreadable */16529 /* reset caller saved regs to unreadable */
16530 bpf_diag_record_caller_saved(env, regs);
16531 bpf_diag_mod_begin(env, &regs[BPF_REG_0], NULL, BPF_DIAG_MOD_WRITE);
16426 for (i = 0; i < CALLER_SAVED_REGS; i++) {16532 for (i = 0; i < CALLER_SAVED_REGS; i++) {
16427 bpf_mark_reg_not_init(env, &regs[caller_saved[i]]);16533 bpf_mark_reg_not_init(env, &regs[caller_saved[i]]);
16428 check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK);16534 check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK);
@@ -16433,6 +16539,7 @@ static int check_ld_abs(struct bpf_verifier_env *env, struct bpf_insn *insn) · 4 unmodified lines
16433 * Already marked as written above.16539 * Already marked as written above.
16434 */16540 */
16435 mark_reg_unknown(env, regs, BPF_REG_0);16541 mark_reg_unknown(env, regs, BPF_REG_0);
16542 bpf_diag_mod_end(env);
16436 /*16543 /*
16437 * See bpf_gen_ld_abs() which emits a hidden BPF_EXIT with r0=016544 * See bpf_gen_ld_abs() which emits a hidden BPF_EXIT with r0=0
16438 * which must be explored by the verifier when in a subprog.16545 * which must be explored by the verifier when in a subprog.