History

bpf: record arg tracking results in bpf_liveness masks

After arg tracking reaches a fixed point, perform a single linear scan over the converged at_in[] state and translate each memory access into liveness read/write masks on the func_instance: - Load/store instructions: FP-derived pointer's frame and offset(s) are converted to half-slot masks targeting per_frame_masks->{may_read,must_write} - Helper/kfunc calls: record_call_access() queries bpf_helper_stack_access_bytes() / bpf_kfunc_stack_access_bytes() for each FP-derived argument to determine access size and direction. Unknown access size (S64_MIN) conservatively marks all slots from fp_off to fp+0 as read. - Imprecise pointers (frame == ARG_IMPRECISE): conservatively mark all slots in every frame covered by the pointer's frame bitmask as fully read. - Static subprog calls with unresolved arguments: conservatively mark all frames as fully read. Instead of a call to clean_live_states(), start cleaning the current state continuously as registers and stack become dead since the static analysis provides complete liveness information. This makes clean_live_states() and bpf_verifier_state->cleaned unnecessary.

Eduard Zingerman committed 7 months ago

Commit fed53db1 parent bf0c5713 files · +245 −63Browse files

Unified · Split

3 files
  1. include/linux/bpf_verifier.h +0 −1
  2. kernel/bpf/liveness.c +243 −0
  3. kernel/bpf/verifier.c +2 −62
include/linux/bpf_verifier.h+0−1
@@ -492,7 +492,6 @@ struct bpf_verifier_state { · 491 unmodified lines
492492
493493 bool speculative;
494494 bool in_sleepable;
495− bool cleaned;
496495
497496 /* first and last insn idx of this verifier state */
498497 u32 first_insn_idx;
kernel/bpf/liveness.c+243−0
@@ -1421,6 +1421,215 @@ static void arg_track_xfer(struct bpf_verifier_env *env, struct bpf_insn *insn, · 1420 unmodified lines
14211421 }
14221422 }
14231423
1424+/*
1425+ * Record access_bytes from helper/kfunc or load/store insn.
1426+ * access_bytes > 0: stack read
1427+ * access_bytes < 0: stack write
1428+ * access_bytes == S64_MIN: unknown — conservative, mark [0..slot] as read
1429+ * access_bytes == 0: no access
1430+ *
1431+ */
1432+static int record_stack_access_off(struct bpf_verifier_env *env,
1433+ struct func_instance *instance, s64 fp_off,
1434+ s64 access_bytes, u32 frame, u32 insn_idx)
1435+{
1436+ s32 slot_hi, slot_lo;
1437+ spis_t mask;
1438+
1439+ if (fp_off >= 0)
1440+ /*
1441+ * out of bounds stack access doesn't contribute
1442+ * into actual stack liveness. It will be rejected
1443+ * by the main verifier pass later.
1444+ */
1445+ return 0;
1446+ if (access_bytes == S64_MIN) {
1447+ /* helper/kfunc read unknown amount of bytes from fp_off until fp+0 */
1448+ slot_hi = (-fp_off - 1) / STACK_SLOT_SZ;
1449+ mask = SPIS_ZERO;
1450+ spis_or_range(&mask, 0, slot_hi);
1451+ return mark_stack_read(instance, frame, insn_idx, mask);
1452+ }
1453+ if (access_bytes > 0) {
1454+ /* Mark any touched slot as use */
1455+ slot_hi = (-fp_off - 1) / STACK_SLOT_SZ;
1456+ slot_lo = max_t(s32, (-fp_off - access_bytes) / STACK_SLOT_SZ, 0);
1457+ mask = SPIS_ZERO;
1458+ spis_or_range(&mask, slot_lo, slot_hi);
1459+ return mark_stack_read(instance, frame, insn_idx, mask);
1460+ } else if (access_bytes < 0) {
1461+ /* Mark only fully covered slots as def */
1462+ access_bytes = -access_bytes;
1463+ slot_hi = (-fp_off) / STACK_SLOT_SZ - 1;
1464+ slot_lo = max_t(s32, (-fp_off - access_bytes + STACK_SLOT_SZ - 1) / STACK_SLOT_SZ, 0);
1465+ if (slot_lo <= slot_hi) {
1466+ mask = SPIS_ZERO;
1467+ spis_or_range(&mask, slot_lo, slot_hi);
1468+ bpf_mark_stack_write(env, frame, mask);
1469+ }
1470+ }
1471+ return 0;
1472+}
1473+
1474+/*
1475+ * 'arg' is FP-derived argument to helper/kfunc or load/store that
1476+ * reads (positive) or writes (negative) 'access_bytes' into 'use' or 'def'.
1477+ */
1478+static int record_stack_access(struct bpf_verifier_env *env,
1479+ struct func_instance *instance,
1480+ const struct arg_track *arg,
1481+ s64 access_bytes, u32 frame, u32 insn_idx)
1482+{
1483+ int i, err;
1484+
1485+ if (access_bytes == 0)
1486+ return 0;
1487+ if (arg->off_cnt == 0) {
1488+ if (access_bytes > 0 || access_bytes == S64_MIN)
1489+ return mark_stack_read(instance, frame, insn_idx, SPIS_ALL);
1490+ return 0;
1491+ }
1492+ if (access_bytes != S64_MIN && access_bytes < 0 && arg->off_cnt != 1)
1493+ /* multi-offset write cannot set stack_def */
1494+ return 0;
1495+
1496+ for (i = 0; i < arg->off_cnt; i++) {
1497+ err = record_stack_access_off(env, instance, arg->off[i], access_bytes, frame, insn_idx);
1498+ if (err)
1499+ return err;
1500+ }
1501+ return 0;
1502+}
1503+
1504+/*
1505+ * When a pointer is ARG_IMPRECISE, conservatively mark every frame in
1506+ * the bitmask as fully used.
1507+ */
1508+static int record_imprecise(struct func_instance *instance, u32 mask, u32 insn_idx)
1509+{
1510+ int depth = instance->callchain.curframe;
1511+ int f, err;
1512+
1513+ for (f = 0; mask; f++, mask >>= 1) {
1514+ if (!(mask & 1))
1515+ continue;
1516+ if (f <= depth) {
1517+ err = mark_stack_read(instance, f, insn_idx, SPIS_ALL);
1518+ if (err)
1519+ return err;
1520+ }
1521+ }
1522+ return 0;
1523+}
1524+
1525+/* Record load/store access for a given 'at' state of 'insn'. */
1526+static int record_load_store_access(struct bpf_verifier_env *env,
1527+ struct func_instance *instance,
1528+ struct arg_track *at, int insn_idx)
1529+{
1530+ struct bpf_insn *insn = &env->prog->insnsi[insn_idx];
1531+ int depth = instance->callchain.curframe;
1532+ s32 sz = bpf_size_to_bytes(BPF_SIZE(insn->code));
1533+ u8 class = BPF_CLASS(insn->code);
1534+ struct arg_track resolved, *ptr;
1535+ int oi;
1536+
1537+ switch (class) {
1538+ case BPF_LDX:
1539+ ptr = &at[insn->src_reg];
1540+ break;
1541+ case BPF_STX:
1542+ if (BPF_MODE(insn->code) == BPF_ATOMIC) {
1543+ if (insn->imm == BPF_STORE_REL)
1544+ sz = -sz;
1545+ if (insn->imm == BPF_LOAD_ACQ)
1546+ ptr = &at[insn->src_reg];
1547+ else
1548+ ptr = &at[insn->dst_reg];
1549+ } else {
1550+ ptr = &at[insn->dst_reg];
1551+ sz = -sz;
1552+ }
1553+ break;
1554+ case BPF_ST:
1555+ ptr = &at[insn->dst_reg];
1556+ sz = -sz;
1557+ break;
1558+ default:
1559+ return 0;
1560+ }
1561+
1562+ /* Resolve offsets: fold insn->off into arg_track */
1563+ if (ptr->off_cnt > 0) {
1564+ resolved.off_cnt = ptr->off_cnt;
1565+ resolved.frame = ptr->frame;
1566+ for (oi = 0; oi < ptr->off_cnt; oi++) {
1567+ resolved.off[oi] = arg_add(ptr->off[oi], insn->off);
1568+ if (resolved.off[oi] == OFF_IMPRECISE) {
1569+ resolved.off_cnt = 0;
1570+ break;
1571+ }
1572+ }
1573+ ptr = &resolved;
1574+ }
1575+
1576+ if (ptr->frame >= 0 && ptr->frame <= depth)
1577+ return record_stack_access(env, instance, ptr, sz, ptr->frame, insn_idx);
1578+ if (ptr->frame == ARG_IMPRECISE)
1579+ return record_imprecise(instance, ptr->mask, insn_idx);
1580+ /* ARG_NONE: not derived from any frame pointer, skip */
1581+ return 0;
1582+}
1583+
1584+/* Record stack access for a given 'at' state of helper/kfunc 'insn' */
1585+static int record_call_access(struct bpf_verifier_env *env,
1586+ struct func_instance *instance,
1587+ struct arg_track *at,
1588+ int insn_idx)
1589+{
1590+ struct bpf_insn *insn = &env->prog->insnsi[insn_idx];
1591+ int depth = instance->callchain.curframe;
1592+ struct bpf_call_summary cs;
1593+ int r, err = 0, num_params = 5;
1594+
1595+ if (bpf_pseudo_call(insn))
1596+ return 0;
1597+
1598+ if (bpf_get_call_summary(env, insn, &cs))
1599+ num_params = cs.num_params;
1600+
1601+ for (r = BPF_REG_1; r < BPF_REG_1 + num_params; r++) {
1602+ int frame = at[r].frame;
1603+ s64 bytes;
1604+
1605+ if (!arg_is_fp(&at[r]))
1606+ continue;
1607+
1608+ if (bpf_helper_call(insn)) {
1609+ bytes = bpf_helper_stack_access_bytes(env, insn, r - 1, insn_idx);
1610+ } else if (bpf_pseudo_kfunc_call(insn)) {
1611+ bytes = bpf_kfunc_stack_access_bytes(env, insn, r - 1, insn_idx);
1612+ } else {
1613+ for (int f = 0; f <= depth; f++) {
1614+ err = mark_stack_read(instance, f, insn_idx, SPIS_ALL);
1615+ if (err)
1616+ return err;
1617+ }
1618+ return 0;
1619+ }
1620+ if (bytes == 0)
1621+ continue;
1622+
1623+ if (frame >= 0 && frame <= depth)
1624+ err = record_stack_access(env, instance, &at[r], bytes, frame, insn_idx);
1625+ else if (frame == ARG_IMPRECISE)
1626+ err = record_imprecise(instance, at[r].mask, insn_idx);
1627+ if (err)
1628+ return err;
1629+ }
1630+ return 0;
1631+}
1632+
14241633 /*
14251634 * For a calls_callback helper, find the callback subprog and determine
14261635 * which caller register maps to which callback register for FP passthrough.
@@ -1665,6 +1874,40 @@ redo: · 238 unmodified lines
16651874 if (changed)
16661875 goto redo;
16671876
1877+ /* Record memory accesses using converged at_in (RPO skips dead code) */
1878+ for (p = po_end - 1; p >= po_start; p--) {
1879+ int idx = env->cfg.insn_postorder[p];
1880+ int i = idx - start;
1881+ struct bpf_insn *insn = &insns[idx];
1882+
1883+ reset_stack_write_marks(env, instance);
1884+ err = record_load_store_access(env, instance, at_in[i], idx);
1885+ if (err)
1886+ goto err_free;
1887+
1888+ if (insn->code == (BPF_JMP | BPF_CALL)) {
1889+ err = record_call_access(env, instance, at_in[i], idx);
1890+ if (err)
1891+ goto err_free;
1892+ }
1893+
1894+ if (bpf_pseudo_call(insn) || bpf_calls_callback(env, idx)) {
1895+ kvfree(env->callsite_at_stack[idx]);
1896+ env->callsite_at_stack[idx] =
1897+ kvmalloc_objs(*env->callsite_at_stack[idx],
1898+ MAX_ARG_SPILL_SLOTS, GFP_KERNEL_ACCOUNT);
1899+ if (!env->callsite_at_stack[idx]) {
1900+ err = -ENOMEM;
1901+ goto err_free;
1902+ }
1903+ memcpy(env->callsite_at_stack[idx],
1904+ at_stack_in[i], sizeof(struct arg_track) * MAX_ARG_SPILL_SLOTS);
1905+ }
1906+ err = commit_stack_write_marks(env, instance, idx);
1907+ if (err)
1908+ goto err_free;
1909+ }
1910+
16681911 info->at_in = at_in;
16691912 at_in = NULL;
16701913 info->len = len;
kernel/bpf/verifier.c+2−62
@@ -1804,7 +1804,6 @@ static int copy_verifier_state(struct bpf_verifier_state *dst_state, · 1803 unmodified lines
18041804 return err;
18051805 dst_state->speculative = src->speculative;
18061806 dst_state->in_sleepable = src->in_sleepable;
1807− dst_state->cleaned = src->cleaned;
18081807 dst_state->curframe = src->curframe;
18091808 dst_state->branches = src->branches;
18101809 dst_state->parent = src->parent;
@@ -20254,8 +20253,6 @@ static int clean_verifier_state(struct bpf_verifier_env *env, · 18443 unmodified lines
2025420253 {
2025520254 int i, err;
2025620255
20257− if (env->cur_state != st)
20258− st->cleaned = true;
2025920256 err = bpf_live_stack_query_init(env, st);
2026020257 if (err)
2026120258 return err;
@@ -20268,37 +20265,6 @@ static int clean_verifier_state(struct bpf_verifier_env *env, · 6 unmodified lines
2026820265 return 0;
2026920266 }
2027020267
20271−/* the parentage chains form a tree.
20272− * the verifier states are added to state lists at given insn and
20273− * pushed into state stack for future exploration.
20274− * when the verifier reaches bpf_exit insn some of the verifier states
20275− * stored in the state lists have their final liveness state already,
20276− * but a lot of states will get revised from liveness point of view when
20277− * the verifier explores other branches.
20278− * Example:
20279− * 1: *(u64)(r10 - 8) = 1
20280− * 2: if r1 == 100 goto pc+1
20281− * 3: *(u64)(r10 - 8) = 2
20282− * 4: r0 = *(u64)(r10 - 8)
20283− * 5: exit
20284− * when the verifier reaches exit insn the stack slot -8 in the state list of
20285− * insn 2 is not yet marked alive. Then the verifier pops the other_branch
20286− * of insn 2 and goes exploring further. After the insn 4 read, liveness
20287− * analysis would propagate read mark for -8 at insn 2.
20288− *
20289− * Since the verifier pushes the branch states as it sees them while exploring
20290− * the program the condition of walking the branch instruction for the second
20291− * time means that all states below this branch were already explored and
20292− * their final liveness marks are already propagated.
20293− * Hence when the verifier completes the search of state list in is_state_visited()
20294− * we can call this clean_live_states() function to clear dead the registers and stack
20295− * slots to simplify state merging.
20296− *
20297− * Important note here that walking the same branch instruction in the callee
20298− * doesn't meant that the states are DONE. The verifier has to compare
20299− * the callsites
20300− */
20301−
2030220268 /* Find id in idset and increment its count, or add new entry */
2030320269 static void idset_cnt_inc(struct bpf_idset *idset, u32 id)
2030420270 {
@@ -20362,33 +20328,6 @@ static void clear_singular_ids(struct bpf_verifier_env *env, · 57 unmodified lines
2036220328 }));
2036320329 }
2036420330
20365−static int clean_live_states(struct bpf_verifier_env *env, int insn,
20366− struct bpf_verifier_state *cur)
20367−{
20368− struct bpf_verifier_state_list *sl;
20369− struct list_head *pos, *head;
20370− int err;
20371−
20372− head = explored_state(env, insn);
20373− list_for_each(pos, head) {
20374− sl = container_of(pos, struct bpf_verifier_state_list, node);
20375− if (sl->state.branches)
20376− continue;
20377− if (sl->state.insn_idx != insn ||
20378− !same_callsites(&sl->state, cur))
20379− continue;
20380− if (sl->state.cleaned)
20381− /* all regs in this state in all frames were already marked */
20382− continue;
20383− if (incomplete_read_marks(env, &sl->state))
20384− continue;
20385− err = clean_verifier_state(env, &sl->state);
20386− if (err)
20387− return err;
20388− }
20389− return 0;
20390−}
20391−
2039220331 static bool regs_exact(const struct bpf_reg_state *rold,
2039320332 const struct bpf_reg_state *rcur,
2039420333 struct bpf_idmap *idmap)
@@ -21089,7 +21028,8 @@ static int is_state_visited(struct bpf_verifier_env *env, int insn_idx) · 694 unmodified lines
2108921028 env->insn_processed - env->prev_insn_processed >= 8)
2109021029 add_new_state = true;
2109121030
21092− err = clean_live_states(env, insn_idx, cur);
21031+ /* keep cleaning the current state as registers/stack become dead */
21032+ err = clean_verifier_state(env, cur);
2109321033 if (err)
2109421034 return err;
2109521035