After arg tracking reaches a fixed point, perform a single linear scan over the converged at_in[] state and translate each memory access into liveness read/write masks on the func_instance: - Load/store instructions: FP-derived pointer's frame and offset(s) are converted to half-slot masks targeting per_frame_masks->{may_read,must_write} - Helper/kfunc calls: record_call_access() queries bpf_helper_stack_access_bytes() / bpf_kfunc_stack_access_bytes() for each FP-derived argument to determine access size and direction. Unknown access size (S64_MIN) conservatively marks all slots from fp_off to fp+0 as read. - Imprecise pointers (frame == ARG_IMPRECISE): conservatively mark all slots in every frame covered by the pointer's frame bitmask as fully read. - Static subprog calls with unresolved arguments: conservatively mark all frames as fully read. Instead of a call to clean_live_states(), start cleaning the current state continuously as registers and stack become dead since the static analysis provides complete liveness information. This makes clean_live_states() and bpf_verifier_state->cleaned unnecessary.
Showing the changes to kernel (2 of 3 files). Show all
Unified · Split
| @@ -1421,6 +1421,215 @@ static void arg_track_xfer(struct bpf_verifier_env *env, struct bpf_insn *insn, · 1420 unmodified lines | |||
| 1421 | } | 1421 | } |
| 1422 | } | 1422 | } |
| 1423 | 1423 | ||
| 1424 | /* | ||
| 1425 | * Record access_bytes from helper/kfunc or load/store insn. | ||
| 1426 | * access_bytes > 0: stack read | ||
| 1427 | * access_bytes < 0: stack write | ||
| 1428 | * access_bytes == S64_MIN: unknown — conservative, mark [0..slot] as read | ||
| 1429 | * access_bytes == 0: no access | ||
| 1430 | * | ||
| 1431 | */ | ||
| 1432 | static int record_stack_access_off(struct bpf_verifier_env *env, | ||
| 1433 | struct func_instance *instance, s64 fp_off, | ||
| 1434 | s64 access_bytes, u32 frame, u32 insn_idx) | ||
| 1435 | { | ||
| 1436 | s32 slot_hi, slot_lo; | ||
| 1437 | spis_t mask; | ||
| 1438 | |||
| 1439 | if (fp_off >= 0) | ||
| 1440 | /* | ||
| 1441 | * out of bounds stack access doesn't contribute | ||
| 1442 | * into actual stack liveness. It will be rejected | ||
| 1443 | * by the main verifier pass later. | ||
| 1444 | */ | ||
| 1445 | return 0; | ||
| 1446 | if (access_bytes == S64_MIN) { | ||
| 1447 | /* helper/kfunc read unknown amount of bytes from fp_off until fp+0 */ | ||
| 1448 | slot_hi = (-fp_off - 1) / STACK_SLOT_SZ; | ||
| 1449 | mask = SPIS_ZERO; | ||
| 1450 | spis_or_range(&mask, 0, slot_hi); | ||
| 1451 | return mark_stack_read(instance, frame, insn_idx, mask); | ||
| 1452 | } | ||
| 1453 | if (access_bytes > 0) { | ||
| 1454 | /* Mark any touched slot as use */ | ||
| 1455 | slot_hi = (-fp_off - 1) / STACK_SLOT_SZ; | ||
| 1456 | slot_lo = max_t(s32, (-fp_off - access_bytes) / STACK_SLOT_SZ, 0); | ||
| 1457 | mask = SPIS_ZERO; | ||
| 1458 | spis_or_range(&mask, slot_lo, slot_hi); | ||
| 1459 | return mark_stack_read(instance, frame, insn_idx, mask); | ||
| 1460 | } else if (access_bytes < 0) { | ||
| 1461 | /* Mark only fully covered slots as def */ | ||
| 1462 | access_bytes = -access_bytes; | ||
| 1463 | slot_hi = (-fp_off) / STACK_SLOT_SZ - 1; | ||
| 1464 | slot_lo = max_t(s32, (-fp_off - access_bytes + STACK_SLOT_SZ - 1) / STACK_SLOT_SZ, 0); | ||
| 1465 | if (slot_lo <= slot_hi) { | ||
| 1466 | mask = SPIS_ZERO; | ||
| 1467 | spis_or_range(&mask, slot_lo, slot_hi); | ||
| 1468 | bpf_mark_stack_write(env, frame, mask); | ||
| 1469 | } | ||
| 1470 | } | ||
| 1471 | return 0; | ||
| 1472 | } | ||
| 1473 | |||
| 1474 | /* | ||
| 1475 | * 'arg' is FP-derived argument to helper/kfunc or load/store that | ||
| 1476 | * reads (positive) or writes (negative) 'access_bytes' into 'use' or 'def'. | ||
| 1477 | */ | ||
| 1478 | static int record_stack_access(struct bpf_verifier_env *env, | ||
| 1479 | struct func_instance *instance, | ||
| 1480 | const struct arg_track *arg, | ||
| 1481 | s64 access_bytes, u32 frame, u32 insn_idx) | ||
| 1482 | { | ||
| 1483 | int i, err; | ||
| 1484 | |||
| 1485 | if (access_bytes == 0) | ||
| 1486 | return 0; | ||
| 1487 | if (arg->off_cnt == 0) { | ||
| 1488 | if (access_bytes > 0 || access_bytes == S64_MIN) | ||
| 1489 | return mark_stack_read(instance, frame, insn_idx, SPIS_ALL); | ||
| 1490 | return 0; | ||
| 1491 | } | ||
| 1492 | if (access_bytes != S64_MIN && access_bytes < 0 && arg->off_cnt != 1) | ||
| 1493 | /* multi-offset write cannot set stack_def */ | ||
| 1494 | return 0; | ||
| 1495 | |||
| 1496 | for (i = 0; i < arg->off_cnt; i++) { | ||
| 1497 | err = record_stack_access_off(env, instance, arg->off[i], access_bytes, frame, insn_idx); | ||
| 1498 | if (err) | ||
| 1499 | return err; | ||
| 1500 | } | ||
| 1501 | return 0; | ||
| 1502 | } | ||
| 1503 | |||
| 1504 | /* | ||
| 1505 | * When a pointer is ARG_IMPRECISE, conservatively mark every frame in | ||
| 1506 | * the bitmask as fully used. | ||
| 1507 | */ | ||
| 1508 | static int record_imprecise(struct func_instance *instance, u32 mask, u32 insn_idx) | ||
| 1509 | { | ||
| 1510 | int depth = instance->callchain.curframe; | ||
| 1511 | int f, err; | ||
| 1512 | |||
| 1513 | for (f = 0; mask; f++, mask >>= 1) { | ||
| 1514 | if (!(mask & 1)) | ||
| 1515 | continue; | ||
| 1516 | if (f <= depth) { | ||
| 1517 | err = mark_stack_read(instance, f, insn_idx, SPIS_ALL); | ||
| 1518 | if (err) | ||
| 1519 | return err; | ||
| 1520 | } | ||
| 1521 | } | ||
| 1522 | return 0; | ||
| 1523 | } | ||
| 1524 | |||
| 1525 | /* Record load/store access for a given 'at' state of 'insn'. */ | ||
| 1526 | static int record_load_store_access(struct bpf_verifier_env *env, | ||
| 1527 | struct func_instance *instance, | ||
| 1528 | struct arg_track *at, int insn_idx) | ||
| 1529 | { | ||
| 1530 | struct bpf_insn *insn = &env->prog->insnsi[insn_idx]; | ||
| 1531 | int depth = instance->callchain.curframe; | ||
| 1532 | s32 sz = bpf_size_to_bytes(BPF_SIZE(insn->code)); | ||
| 1533 | u8 class = BPF_CLASS(insn->code); | ||
| 1534 | struct arg_track resolved, *ptr; | ||
| 1535 | int oi; | ||
| 1536 | |||
| 1537 | switch (class) { | ||
| 1538 | case BPF_LDX: | ||
| 1539 | ptr = &at[insn->src_reg]; | ||
| 1540 | break; | ||
| 1541 | case BPF_STX: | ||
| 1542 | if (BPF_MODE(insn->code) == BPF_ATOMIC) { | ||
| 1543 | if (insn->imm == BPF_STORE_REL) | ||
| 1544 | sz = -sz; | ||
| 1545 | if (insn->imm == BPF_LOAD_ACQ) | ||
| 1546 | ptr = &at[insn->src_reg]; | ||
| 1547 | else | ||
| 1548 | ptr = &at[insn->dst_reg]; | ||
| 1549 | } else { | ||
| 1550 | ptr = &at[insn->dst_reg]; | ||
| 1551 | sz = -sz; | ||
| 1552 | } | ||
| 1553 | break; | ||
| 1554 | case BPF_ST: | ||
| 1555 | ptr = &at[insn->dst_reg]; | ||
| 1556 | sz = -sz; | ||
| 1557 | break; | ||
| 1558 | default: | ||
| 1559 | return 0; | ||
| 1560 | } | ||
| 1561 | |||
| 1562 | /* Resolve offsets: fold insn->off into arg_track */ | ||
| 1563 | if (ptr->off_cnt > 0) { | ||
| 1564 | resolved.off_cnt = ptr->off_cnt; | ||
| 1565 | resolved.frame = ptr->frame; | ||
| 1566 | for (oi = 0; oi < ptr->off_cnt; oi++) { | ||
| 1567 | resolved.off[oi] = arg_add(ptr->off[oi], insn->off); | ||
| 1568 | if (resolved.off[oi] == OFF_IMPRECISE) { | ||
| 1569 | resolved.off_cnt = 0; | ||
| 1570 | break; | ||
| 1571 | } | ||
| 1572 | } | ||
| 1573 | ptr = &resolved; | ||
| 1574 | } | ||
| 1575 | |||
| 1576 | if (ptr->frame >= 0 && ptr->frame <= depth) | ||
| 1577 | return record_stack_access(env, instance, ptr, sz, ptr->frame, insn_idx); | ||
| 1578 | if (ptr->frame == ARG_IMPRECISE) | ||
| 1579 | return record_imprecise(instance, ptr->mask, insn_idx); | ||
| 1580 | /* ARG_NONE: not derived from any frame pointer, skip */ | ||
| 1581 | return 0; | ||
| 1582 | } | ||
| 1583 | |||
| 1584 | /* Record stack access for a given 'at' state of helper/kfunc 'insn' */ | ||
| 1585 | static int record_call_access(struct bpf_verifier_env *env, | ||
| 1586 | struct func_instance *instance, | ||
| 1587 | struct arg_track *at, | ||
| 1588 | int insn_idx) | ||
| 1589 | { | ||
| 1590 | struct bpf_insn *insn = &env->prog->insnsi[insn_idx]; | ||
| 1591 | int depth = instance->callchain.curframe; | ||
| 1592 | struct bpf_call_summary cs; | ||
| 1593 | int r, err = 0, num_params = 5; | ||
| 1594 | |||
| 1595 | if (bpf_pseudo_call(insn)) | ||
| 1596 | return 0; | ||
| 1597 | |||
| 1598 | if (bpf_get_call_summary(env, insn, &cs)) | ||
| 1599 | num_params = cs.num_params; | ||
| 1600 | |||
| 1601 | for (r = BPF_REG_1; r < BPF_REG_1 + num_params; r++) { | ||
| 1602 | int frame = at[r].frame; | ||
| 1603 | s64 bytes; | ||
| 1604 | |||
| 1605 | if (!arg_is_fp(&at[r])) | ||
| 1606 | continue; | ||
| 1607 | |||
| 1608 | if (bpf_helper_call(insn)) { | ||
| 1609 | bytes = bpf_helper_stack_access_bytes(env, insn, r - 1, insn_idx); | ||
| 1610 | } else if (bpf_pseudo_kfunc_call(insn)) { | ||
| 1611 | bytes = bpf_kfunc_stack_access_bytes(env, insn, r - 1, insn_idx); | ||
| 1612 | } else { | ||
| 1613 | for (int f = 0; f <= depth; f++) { | ||
| 1614 | err = mark_stack_read(instance, f, insn_idx, SPIS_ALL); | ||
| 1615 | if (err) | ||
| 1616 | return err; | ||
| 1617 | } | ||
| 1618 | return 0; | ||
| 1619 | } | ||
| 1620 | if (bytes == 0) | ||
| 1621 | continue; | ||
| 1622 | |||
| 1623 | if (frame >= 0 && frame <= depth) | ||
| 1624 | err = record_stack_access(env, instance, &at[r], bytes, frame, insn_idx); | ||
| 1625 | else if (frame == ARG_IMPRECISE) | ||
| 1626 | err = record_imprecise(instance, at[r].mask, insn_idx); | ||
| 1627 | if (err) | ||
| 1628 | return err; | ||
| 1629 | } | ||
| 1630 | return 0; | ||
| 1631 | } | ||
| 1632 | |||
| 1424 | /* | 1633 | /* |
| 1425 | * For a calls_callback helper, find the callback subprog and determine | 1634 | * For a calls_callback helper, find the callback subprog and determine |
| 1426 | * which caller register maps to which callback register for FP passthrough. | 1635 | * which caller register maps to which callback register for FP passthrough. |
| @@ -1665,6 +1874,40 @@ redo: · 238 unmodified lines | |||
| 1665 | if (changed) | 1874 | if (changed) |
| 1666 | goto redo; | 1875 | goto redo; |
| 1667 | 1876 | ||
| 1877 | /* Record memory accesses using converged at_in (RPO skips dead code) */ | ||
| 1878 | for (p = po_end - 1; p >= po_start; p--) { | ||
| 1879 | int idx = env->cfg.insn_postorder[p]; | ||
| 1880 | int i = idx - start; | ||
| 1881 | struct bpf_insn *insn = &insns[idx]; | ||
| 1882 | |||
| 1883 | reset_stack_write_marks(env, instance); | ||
| 1884 | err = record_load_store_access(env, instance, at_in[i], idx); | ||
| 1885 | if (err) | ||
| 1886 | goto err_free; | ||
| 1887 | |||
| 1888 | if (insn->code == (BPF_JMP | BPF_CALL)) { | ||
| 1889 | err = record_call_access(env, instance, at_in[i], idx); | ||
| 1890 | if (err) | ||
| 1891 | goto err_free; | ||
| 1892 | } | ||
| 1893 | |||
| 1894 | if (bpf_pseudo_call(insn) || bpf_calls_callback(env, idx)) { | ||
| 1895 | kvfree(env->callsite_at_stack[idx]); | ||
| 1896 | env->callsite_at_stack[idx] = | ||
| 1897 | kvmalloc_objs(*env->callsite_at_stack[idx], | ||
| 1898 | MAX_ARG_SPILL_SLOTS, GFP_KERNEL_ACCOUNT); | ||
| 1899 | if (!env->callsite_at_stack[idx]) { | ||
| 1900 | err = -ENOMEM; | ||
| 1901 | goto err_free; | ||
| 1902 | } | ||
| 1903 | memcpy(env->callsite_at_stack[idx], | ||
| 1904 | at_stack_in[i], sizeof(struct arg_track) * MAX_ARG_SPILL_SLOTS); | ||
| 1905 | } | ||
| 1906 | err = commit_stack_write_marks(env, instance, idx); | ||
| 1907 | if (err) | ||
| 1908 | goto err_free; | ||
| 1909 | } | ||
| 1910 | |||
| 1668 | info->at_in = at_in; | 1911 | info->at_in = at_in; |
| 1669 | at_in = NULL; | 1912 | at_in = NULL; |
| 1670 | info->len = len; | 1913 | info->len = len; |
| @@ -1804,7 +1804,6 @@ static int copy_verifier_state(struct bpf_verifier_state *dst_state, · 1803 unmodified lines | |||
| 1804 | return err; | 1804 | return err; |
| 1805 | dst_state->speculative = src->speculative; | 1805 | dst_state->speculative = src->speculative; |
| 1806 | dst_state->in_sleepable = src->in_sleepable; | 1806 | dst_state->in_sleepable = src->in_sleepable; |
| 1807 | dst_state->cleaned = src->cleaned; | ||
| 1808 | dst_state->curframe = src->curframe; | 1807 | dst_state->curframe = src->curframe; |
| 1809 | dst_state->branches = src->branches; | 1808 | dst_state->branches = src->branches; |
| 1810 | dst_state->parent = src->parent; | 1809 | dst_state->parent = src->parent; |
| @@ -20254,8 +20253,6 @@ static int clean_verifier_state(struct bpf_verifier_env *env, · 18443 unmodified lines | |||
| 20254 | { | 20253 | { |
| 20255 | int i, err; | 20254 | int i, err; |
| 20256 | 20255 | ||
| 20257 | if (env->cur_state != st) | ||
| 20258 | st->cleaned = true; | ||
| 20259 | err = bpf_live_stack_query_init(env, st); | 20256 | err = bpf_live_stack_query_init(env, st); |
| 20260 | if (err) | 20257 | if (err) |
| 20261 | return err; | 20258 | return err; |
| @@ -20268,37 +20265,6 @@ static int clean_verifier_state(struct bpf_verifier_env *env, · 6 unmodified lines | |||
| 20268 | return 0; | 20265 | return 0; |
| 20269 | } | 20266 | } |
| 20270 | 20267 | ||
| 20271 | /* the parentage chains form a tree. | ||
| 20272 | * the verifier states are added to state lists at given insn and | ||
| 20273 | * pushed into state stack for future exploration. | ||
| 20274 | * when the verifier reaches bpf_exit insn some of the verifier states | ||
| 20275 | * stored in the state lists have their final liveness state already, | ||
| 20276 | * but a lot of states will get revised from liveness point of view when | ||
| 20277 | * the verifier explores other branches. | ||
| 20278 | * Example: | ||
| 20279 | * 1: *(u64)(r10 - 8) = 1 | ||
| 20280 | * 2: if r1 == 100 goto pc+1 | ||
| 20281 | * 3: *(u64)(r10 - 8) = 2 | ||
| 20282 | * 4: r0 = *(u64)(r10 - 8) | ||
| 20283 | * 5: exit | ||
| 20284 | * when the verifier reaches exit insn the stack slot -8 in the state list of | ||
| 20285 | * insn 2 is not yet marked alive. Then the verifier pops the other_branch | ||
| 20286 | * of insn 2 and goes exploring further. After the insn 4 read, liveness | ||
| 20287 | * analysis would propagate read mark for -8 at insn 2. | ||
| 20288 | * | ||
| 20289 | * Since the verifier pushes the branch states as it sees them while exploring | ||
| 20290 | * the program the condition of walking the branch instruction for the second | ||
| 20291 | * time means that all states below this branch were already explored and | ||
| 20292 | * their final liveness marks are already propagated. | ||
| 20293 | * Hence when the verifier completes the search of state list in is_state_visited() | ||
| 20294 | * we can call this clean_live_states() function to clear dead the registers and stack | ||
| 20295 | * slots to simplify state merging. | ||
| 20296 | * | ||
| 20297 | * Important note here that walking the same branch instruction in the callee | ||
| 20298 | * doesn't meant that the states are DONE. The verifier has to compare | ||
| 20299 | * the callsites | ||
| 20300 | */ | ||
| 20301 | |||
| 20302 | /* Find id in idset and increment its count, or add new entry */ | 20268 | /* Find id in idset and increment its count, or add new entry */ |
| 20303 | static void idset_cnt_inc(struct bpf_idset *idset, u32 id) | 20269 | static void idset_cnt_inc(struct bpf_idset *idset, u32 id) |
| 20304 | { | 20270 | { |
| @@ -20362,33 +20328,6 @@ static void clear_singular_ids(struct bpf_verifier_env *env, · 57 unmodified lines | |||
| 20362 | })); | 20328 | })); |
| 20363 | } | 20329 | } |
| 20364 | 20330 | ||
| 20365 | static int clean_live_states(struct bpf_verifier_env *env, int insn, | ||
| 20366 | struct bpf_verifier_state *cur) | ||
| 20367 | { | ||
| 20368 | struct bpf_verifier_state_list *sl; | ||
| 20369 | struct list_head *pos, *head; | ||
| 20370 | int err; | ||
| 20371 | |||
| 20372 | head = explored_state(env, insn); | ||
| 20373 | list_for_each(pos, head) { | ||
| 20374 | sl = container_of(pos, struct bpf_verifier_state_list, node); | ||
| 20375 | if (sl->state.branches) | ||
| 20376 | continue; | ||
| 20377 | if (sl->state.insn_idx != insn || | ||
| 20378 | !same_callsites(&sl->state, cur)) | ||
| 20379 | continue; | ||
| 20380 | if (sl->state.cleaned) | ||
| 20381 | /* all regs in this state in all frames were already marked */ | ||
| 20382 | continue; | ||
| 20383 | if (incomplete_read_marks(env, &sl->state)) | ||
| 20384 | continue; | ||
| 20385 | err = clean_verifier_state(env, &sl->state); | ||
| 20386 | if (err) | ||
| 20387 | return err; | ||
| 20388 | } | ||
| 20389 | return 0; | ||
| 20390 | } | ||
| 20391 | |||
| 20392 | static bool regs_exact(const struct bpf_reg_state *rold, | 20331 | static bool regs_exact(const struct bpf_reg_state *rold, |
| 20393 | const struct bpf_reg_state *rcur, | 20332 | const struct bpf_reg_state *rcur, |
| 20394 | struct bpf_idmap *idmap) | 20333 | struct bpf_idmap *idmap) |
| @@ -21089,7 +21028,8 @@ static int is_state_visited(struct bpf_verifier_env *env, int insn_idx) · 694 unmodified lines | |||
| 21089 | env->insn_processed - env->prev_insn_processed >= 8) | 21028 | env->insn_processed - env->prev_insn_processed >= 8) |
| 21090 | add_new_state = true; | 21029 | add_new_state = true; |
| 21091 | 21030 | ||
| 21092 | err = clean_live_states(env, insn_idx, cur); | 21031 | /* keep cleaning the current state as registers/stack become dead */ |
| 21032 | err = clean_verifier_state(env, cur); | ||
| 21093 | if (err) | 21033 | if (err) |
| 21094 | return err; | 21034 | return err; |
| 21095 | 21035 | ||