History

bpf: record arg tracking results in bpf_liveness masks

After arg tracking reaches a fixed point, perform a single linear scan over the converged at_in[] state and translate each memory access into liveness read/write masks on the func_instance: - Load/store instructions: FP-derived pointer's frame and offset(s) are converted to half-slot masks targeting per_frame_masks->{may_read,must_write} - Helper/kfunc calls: record_call_access() queries bpf_helper_stack_access_bytes() / bpf_kfunc_stack_access_bytes() for each FP-derived argument to determine access size and direction. Unknown access size (S64_MIN) conservatively marks all slots from fp_off to fp+0 as read. - Imprecise pointers (frame == ARG_IMPRECISE): conservatively mark all slots in every frame covered by the pointer's frame bitmask as fully read. - Static subprog calls with unresolved arguments: conservatively mark all frames as fully read. Instead of a call to clean_live_states(), start cleaning the current state continuously as registers and stack become dead since the static analysis provides complete liveness information. This makes clean_live_states() and bpf_verifier_state->cleaned unnecessary.

Eduard Zingerman committed 7 months ago

Commit fed53db1 parent bf0c5713 files · +245 −63Browse files

Unified · Split

3 files
  1. include/linux/bpf_verifier.h +0 −1
  2. kernel/bpf/liveness.c +243 −0
  3. kernel/bpf/verifier.c +2 −62
include/linux/bpf_verifier.h+0−1
@@ -492,7 +492,6 @@ struct bpf_verifier_state { · 491 unmodified lines
492 492
493 bool speculative;493 bool speculative;
494 bool in_sleepable;494 bool in_sleepable;
495 bool cleaned;
496 495
497 /* first and last insn idx of this verifier state */496 /* first and last insn idx of this verifier state */
498 u32 first_insn_idx;497 u32 first_insn_idx;
kernel/bpf/liveness.c+243−0
@@ -1421,6 +1421,215 @@ static void arg_track_xfer(struct bpf_verifier_env *env, struct bpf_insn *insn, · 1420 unmodified lines
1421 }1421 }
1422}1422}
1423 1423
1424/*
1425 * Record access_bytes from helper/kfunc or load/store insn.
1426 * access_bytes > 0: stack read
1427 * access_bytes < 0: stack write
1428 * access_bytes == S64_MIN: unknown — conservative, mark [0..slot] as read
1429 * access_bytes == 0: no access
1430 *
1431 */
1432static int record_stack_access_off(struct bpf_verifier_env *env,
1433 struct func_instance *instance, s64 fp_off,
1434 s64 access_bytes, u32 frame, u32 insn_idx)
1435{
1436 s32 slot_hi, slot_lo;
1437 spis_t mask;
1438
1439 if (fp_off >= 0)
1440 /*
1441 * out of bounds stack access doesn't contribute
1442 * into actual stack liveness. It will be rejected
1443 * by the main verifier pass later.
1444 */
1445 return 0;
1446 if (access_bytes == S64_MIN) {
1447 /* helper/kfunc read unknown amount of bytes from fp_off until fp+0 */
1448 slot_hi = (-fp_off - 1) / STACK_SLOT_SZ;
1449 mask = SPIS_ZERO;
1450 spis_or_range(&mask, 0, slot_hi);
1451 return mark_stack_read(instance, frame, insn_idx, mask);
1452 }
1453 if (access_bytes > 0) {
1454 /* Mark any touched slot as use */
1455 slot_hi = (-fp_off - 1) / STACK_SLOT_SZ;
1456 slot_lo = max_t(s32, (-fp_off - access_bytes) / STACK_SLOT_SZ, 0);
1457 mask = SPIS_ZERO;
1458 spis_or_range(&mask, slot_lo, slot_hi);
1459 return mark_stack_read(instance, frame, insn_idx, mask);
1460 } else if (access_bytes < 0) {
1461 /* Mark only fully covered slots as def */
1462 access_bytes = -access_bytes;
1463 slot_hi = (-fp_off) / STACK_SLOT_SZ - 1;
1464 slot_lo = max_t(s32, (-fp_off - access_bytes + STACK_SLOT_SZ - 1) / STACK_SLOT_SZ, 0);
1465 if (slot_lo <= slot_hi) {
1466 mask = SPIS_ZERO;
1467 spis_or_range(&mask, slot_lo, slot_hi);
1468 bpf_mark_stack_write(env, frame, mask);
1469 }
1470 }
1471 return 0;
1472}
1473
1474/*
1475 * 'arg' is FP-derived argument to helper/kfunc or load/store that
1476 * reads (positive) or writes (negative) 'access_bytes' into 'use' or 'def'.
1477 */
1478static int record_stack_access(struct bpf_verifier_env *env,
1479 struct func_instance *instance,
1480 const struct arg_track *arg,
1481 s64 access_bytes, u32 frame, u32 insn_idx)
1482{
1483 int i, err;
1484
1485 if (access_bytes == 0)
1486 return 0;
1487 if (arg->off_cnt == 0) {
1488 if (access_bytes > 0 || access_bytes == S64_MIN)
1489 return mark_stack_read(instance, frame, insn_idx, SPIS_ALL);
1490 return 0;
1491 }
1492 if (access_bytes != S64_MIN && access_bytes < 0 && arg->off_cnt != 1)
1493 /* multi-offset write cannot set stack_def */
1494 return 0;
1495
1496 for (i = 0; i < arg->off_cnt; i++) {
1497 err = record_stack_access_off(env, instance, arg->off[i], access_bytes, frame, insn_idx);
1498 if (err)
1499 return err;
1500 }
1501 return 0;
1502}
1503
1504/*
1505 * When a pointer is ARG_IMPRECISE, conservatively mark every frame in
1506 * the bitmask as fully used.
1507 */
1508static int record_imprecise(struct func_instance *instance, u32 mask, u32 insn_idx)
1509{
1510 int depth = instance->callchain.curframe;
1511 int f, err;
1512
1513 for (f = 0; mask; f++, mask >>= 1) {
1514 if (!(mask & 1))
1515 continue;
1516 if (f <= depth) {
1517 err = mark_stack_read(instance, f, insn_idx, SPIS_ALL);
1518 if (err)
1519 return err;
1520 }
1521 }
1522 return 0;
1523}
1524
1525/* Record load/store access for a given 'at' state of 'insn'. */
1526static int record_load_store_access(struct bpf_verifier_env *env,
1527 struct func_instance *instance,
1528 struct arg_track *at, int insn_idx)
1529{
1530 struct bpf_insn *insn = &env->prog->insnsi[insn_idx];
1531 int depth = instance->callchain.curframe;
1532 s32 sz = bpf_size_to_bytes(BPF_SIZE(insn->code));
1533 u8 class = BPF_CLASS(insn->code);
1534 struct arg_track resolved, *ptr;
1535 int oi;
1536
1537 switch (class) {
1538 case BPF_LDX:
1539 ptr = &at[insn->src_reg];
1540 break;
1541 case BPF_STX:
1542 if (BPF_MODE(insn->code) == BPF_ATOMIC) {
1543 if (insn->imm == BPF_STORE_REL)
1544 sz = -sz;
1545 if (insn->imm == BPF_LOAD_ACQ)
1546 ptr = &at[insn->src_reg];
1547 else
1548 ptr = &at[insn->dst_reg];
1549 } else {
1550 ptr = &at[insn->dst_reg];
1551 sz = -sz;
1552 }
1553 break;
1554 case BPF_ST:
1555 ptr = &at[insn->dst_reg];
1556 sz = -sz;
1557 break;
1558 default:
1559 return 0;
1560 }
1561
1562 /* Resolve offsets: fold insn->off into arg_track */
1563 if (ptr->off_cnt > 0) {
1564 resolved.off_cnt = ptr->off_cnt;
1565 resolved.frame = ptr->frame;
1566 for (oi = 0; oi < ptr->off_cnt; oi++) {
1567 resolved.off[oi] = arg_add(ptr->off[oi], insn->off);
1568 if (resolved.off[oi] == OFF_IMPRECISE) {
1569 resolved.off_cnt = 0;
1570 break;
1571 }
1572 }
1573 ptr = &resolved;
1574 }
1575
1576 if (ptr->frame >= 0 && ptr->frame <= depth)
1577 return record_stack_access(env, instance, ptr, sz, ptr->frame, insn_idx);
1578 if (ptr->frame == ARG_IMPRECISE)
1579 return record_imprecise(instance, ptr->mask, insn_idx);
1580 /* ARG_NONE: not derived from any frame pointer, skip */
1581 return 0;
1582}
1583
1584/* Record stack access for a given 'at' state of helper/kfunc 'insn' */
1585static int record_call_access(struct bpf_verifier_env *env,
1586 struct func_instance *instance,
1587 struct arg_track *at,
1588 int insn_idx)
1589{
1590 struct bpf_insn *insn = &env->prog->insnsi[insn_idx];
1591 int depth = instance->callchain.curframe;
1592 struct bpf_call_summary cs;
1593 int r, err = 0, num_params = 5;
1594
1595 if (bpf_pseudo_call(insn))
1596 return 0;
1597
1598 if (bpf_get_call_summary(env, insn, &cs))
1599 num_params = cs.num_params;
1600
1601 for (r = BPF_REG_1; r < BPF_REG_1 + num_params; r++) {
1602 int frame = at[r].frame;
1603 s64 bytes;
1604
1605 if (!arg_is_fp(&at[r]))
1606 continue;
1607
1608 if (bpf_helper_call(insn)) {
1609 bytes = bpf_helper_stack_access_bytes(env, insn, r - 1, insn_idx);
1610 } else if (bpf_pseudo_kfunc_call(insn)) {
1611 bytes = bpf_kfunc_stack_access_bytes(env, insn, r - 1, insn_idx);
1612 } else {
1613 for (int f = 0; f <= depth; f++) {
1614 err = mark_stack_read(instance, f, insn_idx, SPIS_ALL);
1615 if (err)
1616 return err;
1617 }
1618 return 0;
1619 }
1620 if (bytes == 0)
1621 continue;
1622
1623 if (frame >= 0 && frame <= depth)
1624 err = record_stack_access(env, instance, &at[r], bytes, frame, insn_idx);
1625 else if (frame == ARG_IMPRECISE)
1626 err = record_imprecise(instance, at[r].mask, insn_idx);
1627 if (err)
1628 return err;
1629 }
1630 return 0;
1631}
1632
1424/*1633/*
1425 * For a calls_callback helper, find the callback subprog and determine1634 * For a calls_callback helper, find the callback subprog and determine
1426 * which caller register maps to which callback register for FP passthrough.1635 * which caller register maps to which callback register for FP passthrough.
@@ -1665,6 +1874,40 @@ redo: · 238 unmodified lines
1665 if (changed)1874 if (changed)
1666 goto redo;1875 goto redo;
1667 1876
1877 /* Record memory accesses using converged at_in (RPO skips dead code) */
1878 for (p = po_end - 1; p >= po_start; p--) {
1879 int idx = env->cfg.insn_postorder[p];
1880 int i = idx - start;
1881 struct bpf_insn *insn = &insns[idx];
1882
1883 reset_stack_write_marks(env, instance);
1884 err = record_load_store_access(env, instance, at_in[i], idx);
1885 if (err)
1886 goto err_free;
1887
1888 if (insn->code == (BPF_JMP | BPF_CALL)) {
1889 err = record_call_access(env, instance, at_in[i], idx);
1890 if (err)
1891 goto err_free;
1892 }
1893
1894 if (bpf_pseudo_call(insn) || bpf_calls_callback(env, idx)) {
1895 kvfree(env->callsite_at_stack[idx]);
1896 env->callsite_at_stack[idx] =
1897 kvmalloc_objs(*env->callsite_at_stack[idx],
1898 MAX_ARG_SPILL_SLOTS, GFP_KERNEL_ACCOUNT);
1899 if (!env->callsite_at_stack[idx]) {
1900 err = -ENOMEM;
1901 goto err_free;
1902 }
1903 memcpy(env->callsite_at_stack[idx],
1904 at_stack_in[i], sizeof(struct arg_track) * MAX_ARG_SPILL_SLOTS);
1905 }
1906 err = commit_stack_write_marks(env, instance, idx);
1907 if (err)
1908 goto err_free;
1909 }
1910
1668 info->at_in = at_in;1911 info->at_in = at_in;
1669 at_in = NULL;1912 at_in = NULL;
1670 info->len = len;1913 info->len = len;
kernel/bpf/verifier.c+2−62
@@ -1804,7 +1804,6 @@ static int copy_verifier_state(struct bpf_verifier_state *dst_state, · 1803 unmodified lines
1804 return err;1804 return err;
1805 dst_state->speculative = src->speculative;1805 dst_state->speculative = src->speculative;
1806 dst_state->in_sleepable = src->in_sleepable;1806 dst_state->in_sleepable = src->in_sleepable;
1807 dst_state->cleaned = src->cleaned;
1808 dst_state->curframe = src->curframe;1807 dst_state->curframe = src->curframe;
1809 dst_state->branches = src->branches;1808 dst_state->branches = src->branches;
1810 dst_state->parent = src->parent;1809 dst_state->parent = src->parent;
@@ -20254,8 +20253,6 @@ static int clean_verifier_state(struct bpf_verifier_env *env, · 18443 unmodified lines
20254{20253{
20255 int i, err;20254 int i, err;
20256 20255
20257 if (env->cur_state != st)
20258 st->cleaned = true;
20259 err = bpf_live_stack_query_init(env, st);20256 err = bpf_live_stack_query_init(env, st);
20260 if (err)20257 if (err)
20261 return err;20258 return err;
@@ -20268,37 +20265,6 @@ static int clean_verifier_state(struct bpf_verifier_env *env, · 6 unmodified lines
20268 return 0;20265 return 0;
20269}20266}
20270 20267
20271/* the parentage chains form a tree.
20272 * the verifier states are added to state lists at given insn and
20273 * pushed into state stack for future exploration.
20274 * when the verifier reaches bpf_exit insn some of the verifier states
20275 * stored in the state lists have their final liveness state already,
20276 * but a lot of states will get revised from liveness point of view when
20277 * the verifier explores other branches.
20278 * Example:
20279 * 1: *(u64)(r10 - 8) = 1
20280 * 2: if r1 == 100 goto pc+1
20281 * 3: *(u64)(r10 - 8) = 2
20282 * 4: r0 = *(u64)(r10 - 8)
20283 * 5: exit
20284 * when the verifier reaches exit insn the stack slot -8 in the state list of
20285 * insn 2 is not yet marked alive. Then the verifier pops the other_branch
20286 * of insn 2 and goes exploring further. After the insn 4 read, liveness
20287 * analysis would propagate read mark for -8 at insn 2.
20288 *
20289 * Since the verifier pushes the branch states as it sees them while exploring
20290 * the program the condition of walking the branch instruction for the second
20291 * time means that all states below this branch were already explored and
20292 * their final liveness marks are already propagated.
20293 * Hence when the verifier completes the search of state list in is_state_visited()
20294 * we can call this clean_live_states() function to clear dead the registers and stack
20295 * slots to simplify state merging.
20296 *
20297 * Important note here that walking the same branch instruction in the callee
20298 * doesn't meant that the states are DONE. The verifier has to compare
20299 * the callsites
20300 */
20301
20302/* Find id in idset and increment its count, or add new entry */20268/* Find id in idset and increment its count, or add new entry */
20303static void idset_cnt_inc(struct bpf_idset *idset, u32 id)20269static void idset_cnt_inc(struct bpf_idset *idset, u32 id)
20304{20270{
@@ -20362,33 +20328,6 @@ static void clear_singular_ids(struct bpf_verifier_env *env, · 57 unmodified lines
20362 }));20328 }));
20363}20329}
20364 20330
20365static int clean_live_states(struct bpf_verifier_env *env, int insn,
20366 struct bpf_verifier_state *cur)
20367{
20368 struct bpf_verifier_state_list *sl;
20369 struct list_head *pos, *head;
20370 int err;
20371
20372 head = explored_state(env, insn);
20373 list_for_each(pos, head) {
20374 sl = container_of(pos, struct bpf_verifier_state_list, node);
20375 if (sl->state.branches)
20376 continue;
20377 if (sl->state.insn_idx != insn ||
20378 !same_callsites(&sl->state, cur))
20379 continue;
20380 if (sl->state.cleaned)
20381 /* all regs in this state in all frames were already marked */
20382 continue;
20383 if (incomplete_read_marks(env, &sl->state))
20384 continue;
20385 err = clean_verifier_state(env, &sl->state);
20386 if (err)
20387 return err;
20388 }
20389 return 0;
20390}
20391
20392static bool regs_exact(const struct bpf_reg_state *rold,20331static bool regs_exact(const struct bpf_reg_state *rold,
20393 const struct bpf_reg_state *rcur,20332 const struct bpf_reg_state *rcur,
20394 struct bpf_idmap *idmap)20333 struct bpf_idmap *idmap)
@@ -21089,7 +21028,8 @@ static int is_state_visited(struct bpf_verifier_env *env, int insn_idx) · 694 unmodified lines
21089 env->insn_processed - env->prev_insn_processed >= 8)21028 env->insn_processed - env->prev_insn_processed >= 8)
21090 add_new_state = true;21029 add_new_state = true;
21091 21030
21092 err = clean_live_states(env, insn_idx, cur);21031 /* keep cleaning the current state as registers/stack become dead */
21032 err = clean_verifier_state(env, cur);
21093 if (err)21033 if (err)
21094 return err;21034 return err;
21095 21035