Pull bpf fixes from Alexei Starovoitov: - Fix how linked registers track zero extension of subregisters (Daniel Borkmann) - Fix unsound scalar fork for OR instructions (Daniel Wade) - Fix exception exit lock check for subprogs (Ihor Solodrai) - Fix undefined behavior in interpreter for SDIV/SMOD instructions (Jenny Guanni Qu) - Release module's BTF when module is unloaded (Kumar Kartikeya Dwivedi) - Fix constant blinding for PROBE_MEM32 instructions (Sachin Kumar) - Reset register ID for END instructions to prevent incorrect value tracking (Yazhou Tang) * tag 'bpf-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf: selftests/bpf: Add a test cases for sync_linked_regs regarding zext propagation bpf: Fix sync_linked_regs regarding BPF_ADD_CONST32 zext propagation selftests/bpf: Add tests for maybe_fork_scalars() OR vs AND handling bpf: Fix unsound scalar forking in maybe_fork_scalars() for BPF_OR selftests/bpf: Add tests for sdiv32/smod32 with INT_MIN dividend bpf: Fix undefined behavior in interpreter sdiv/smod for INT_MIN selftests/bpf: Add tests for bpf_throw lock leak from subprogs bpf: Fix exception exit lock checking for subprogs bpf: Release module BTF IDR before module unload selftests/bpf: Fix pkg-config call on static builds bpf: Fix constant blinding for PROBE_MEM32 stores selftests/bpf: Add test for BPF_END register ID reset bpf: Reset register ID for BPF_END value tracking
Showing the changes to kernel (3 of 9 files). Show all
Unified · Split
| @@ -1787,7 +1787,16 @@ static void btf_free_id(struct btf *btf) · 1786 unmodified lines | ||
| 1787 | 1787 | * of the _bh() version. |
| 1788 | 1788 | */ |
| 1789 | 1789 | spin_lock_irqsave(&btf_idr_lock, flags); |
| 1790 | − idr_remove(&btf_idr, btf->id); | |
| 1790 | + if (btf->id) { | |
| 1791 | + idr_remove(&btf_idr, btf->id); | |
| 1792 | + /* | |
| 1793 | + * Clear the id here to make this function idempotent, since it will get | |
| 1794 | + * called a couple of times for module BTFs: on module unload, and then | |
| 1795 | + * the final btf_put(). btf_alloc_id() starts IDs with 1, so we can use | |
| 1796 | + * 0 as sentinel value. | |
| 1797 | + */ | |
| 1798 | + WRITE_ONCE(btf->id, 0); | |
| 1799 | + } | |
| 1791 | 1800 | spin_unlock_irqrestore(&btf_idr_lock, flags); |
| 1792 | 1801 | } |
| 1793 | 1802 | |
| @@ -8115,7 +8124,7 @@ static void bpf_btf_show_fdinfo(struct seq_file *m, struct file *filp) · 6321 unmodified lines | ||
| 8115 | 8124 | { |
| 8116 | 8125 | const struct btf *btf = filp->private_data; |
| 8117 | 8126 | |
| 8118 | − seq_printf(m, "btf_id:\t%u\n", btf->id); | |
| 8127 | + seq_printf(m, "btf_id:\t%u\n", READ_ONCE(btf->id)); | |
| 8119 | 8128 | } |
| 8120 | 8129 | #endif |
| 8121 | 8130 | |
| @@ -8197,7 +8206,7 @@ int btf_get_info_by_fd(const struct btf *btf, · 75 unmodified lines | ||
| 8197 | 8206 | if (copy_from_user(&info, uinfo, info_copy)) |
| 8198 | 8207 | return -EFAULT; |
| 8199 | 8208 | |
| 8200 | − info.id = btf->id; | |
| 8209 | + info.id = READ_ONCE(btf->id); | |
| 8201 | 8210 | ubtf = u64_to_user_ptr(info.btf); |
| 8202 | 8211 | btf_copy = min_t(u32, btf->data_size, info.btf_size); |
| 8203 | 8212 | if (copy_to_user(ubtf, btf->data, btf_copy)) |
| @@ -8260,7 +8269,7 @@ int btf_get_fd_by_id(u32 id) · 56 unmodified lines | ||
| 8260 | 8269 | |
| 8261 | 8270 | u32 btf_obj_id(const struct btf *btf) |
| 8262 | 8271 | { |
| 8263 | − return btf->id; | |
| 8272 | + return READ_ONCE(btf->id); | |
| 8264 | 8273 | } |
| 8265 | 8274 | |
| 8266 | 8275 | bool btf_is_kernel(const struct btf *btf) |
| @@ -8382,6 +8391,13 @@ static int btf_module_notify(struct notifier_block *nb, unsigned long op, · 115 unmodified lines | ||
| 8382 | 8391 | if (btf_mod->module != module) |
| 8383 | 8392 | continue; |
| 8384 | 8393 | |
| 8394 | + /* | |
| 8395 | + * For modules, we do the freeing of BTF IDR as soon as | |
| 8396 | + * module goes away to disable BTF discovery, since the | |
| 8397 | + * btf_try_get_module() on such BTFs will fail. This may | |
| 8398 | + * be called again on btf_put(), but it's ok to do so. | |
| 8399 | + */ | |
| 8400 | + btf_free_id(btf_mod->btf); | |
| 8385 | 8401 | list_del(&btf_mod->list); |
| 8386 | 8402 | if (btf_mod->sysfs_attr) |
| 8387 | 8403 | sysfs_remove_bin_file(btf_kobj, btf_mod->sysfs_attr); |
| @@ -1422,6 +1422,27 @@ static int bpf_jit_blind_insn(const struct bpf_insn *from, · 1421 unmodified lines | ||
| 1422 | 1422 | *to++ = BPF_ALU64_IMM(BPF_XOR, BPF_REG_AX, imm_rnd); |
| 1423 | 1423 | *to++ = BPF_STX_MEM(from->code, from->dst_reg, BPF_REG_AX, from->off); |
| 1424 | 1424 | break; |
| 1425 | + | |
| 1426 | + case BPF_ST | BPF_PROBE_MEM32 | BPF_DW: | |
| 1427 | + case BPF_ST | BPF_PROBE_MEM32 | BPF_W: | |
| 1428 | + case BPF_ST | BPF_PROBE_MEM32 | BPF_H: | |
| 1429 | + case BPF_ST | BPF_PROBE_MEM32 | BPF_B: | |
| 1430 | + *to++ = BPF_ALU64_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ | |
| 1431 | + from->imm); | |
| 1432 | + *to++ = BPF_ALU64_IMM(BPF_XOR, BPF_REG_AX, imm_rnd); | |
| 1433 | + /* | |
| 1434 | + * Cannot use BPF_STX_MEM() macro here as it | |
| 1435 | + * hardcodes BPF_MEM mode, losing PROBE_MEM32 | |
| 1436 | + * and breaking arena addressing in the JIT. | |
| 1437 | + */ | |
| 1438 | + *to++ = (struct bpf_insn) { | |
| 1439 | + .code = BPF_STX | BPF_PROBE_MEM32 | | |
| 1440 | + BPF_SIZE(from->code), | |
| 1441 | + .dst_reg = from->dst_reg, | |
| 1442 | + .src_reg = BPF_REG_AX, | |
| 1443 | + .off = from->off, | |
| 1444 | + }; | |
| 1445 | + break; | |
| 1425 | 1446 | } |
| 1426 | 1447 | out: |
| 1427 | 1448 | return to - to_buff; |
| @@ -1736,6 +1757,12 @@ bool bpf_opcode_in_insntable(u8 code) · 308 unmodified lines | ||
| 1736 | 1757 | } |
| 1737 | 1758 | |
| 1738 | 1759 | #ifndef CONFIG_BPF_JIT_ALWAYS_ON |
| 1760 | +/* Absolute value of s32 without undefined behavior for S32_MIN */ | |
| 1761 | +static u32 abs_s32(s32 x) | |
| 1762 | +{ | |
| 1763 | + return x >= 0 ? (u32)x : -(u32)x; | |
| 1764 | +} | |
| 1765 | + | |
| 1739 | 1766 | /** |
| 1740 | 1767 | * ___bpf_prog_run - run eBPF program on a given context |
| 1741 | 1768 | * @regs: is the array of MAX_BPF_EXT_REG eBPF pseudo-registers |
| @@ -1900,8 +1927,8 @@ select_insn: · 158 unmodified lines | ||
| 1900 | 1927 | DST = do_div(AX, (u32) SRC); |
| 1901 | 1928 | break; |
| 1902 | 1929 | case 1: |
| 1903 | − AX = abs((s32)DST); | |
| 1904 | − AX = do_div(AX, abs((s32)SRC)); | |
| 1930 | + AX = abs_s32((s32)DST); | |
| 1931 | + AX = do_div(AX, abs_s32((s32)SRC)); | |
| 1905 | 1932 | if ((s32)DST < 0) |
| 1906 | 1933 | DST = (u32)-AX; |
| 1907 | 1934 | else |
| @@ -1928,8 +1955,8 @@ select_insn: · 20 unmodified lines | ||
| 1928 | 1955 | DST = do_div(AX, (u32) IMM); |
| 1929 | 1956 | break; |
| 1930 | 1957 | case 1: |
| 1931 | − AX = abs((s32)DST); | |
| 1932 | − AX = do_div(AX, abs((s32)IMM)); | |
| 1958 | + AX = abs_s32((s32)DST); | |
| 1959 | + AX = do_div(AX, abs_s32((s32)IMM)); | |
| 1933 | 1960 | if ((s32)DST < 0) |
| 1934 | 1961 | DST = (u32)-AX; |
| 1935 | 1962 | else |
| @@ -1955,8 +1982,8 @@ select_insn: · 19 unmodified lines | ||
| 1955 | 1982 | DST = (u32) AX; |
| 1956 | 1983 | break; |
| 1957 | 1984 | case 1: |
| 1958 | − AX = abs((s32)DST); | |
| 1959 | − do_div(AX, abs((s32)SRC)); | |
| 1985 | + AX = abs_s32((s32)DST); | |
| 1986 | + do_div(AX, abs_s32((s32)SRC)); | |
| 1960 | 1987 | if (((s32)DST < 0) == ((s32)SRC < 0)) |
| 1961 | 1988 | DST = (u32)AX; |
| 1962 | 1989 | else |
| @@ -1982,8 +2009,8 @@ select_insn: · 19 unmodified lines | ||
| 1982 | 2009 | DST = (u32) AX; |
| 1983 | 2010 | break; |
| 1984 | 2011 | case 1: |
| 1985 | − AX = abs((s32)DST); | |
| 1986 | − do_div(AX, abs((s32)IMM)); | |
| 2012 | + AX = abs_s32((s32)DST); | |
| 2013 | + do_div(AX, abs_s32((s32)IMM)); | |
| 1987 | 2014 | if (((s32)DST < 0) == ((s32)IMM < 0)) |
| 1988 | 2015 | DST = (u32)AX; |
| 1989 | 2016 | else |
| @@ -15910,6 +15910,13 @@ static void scalar_byte_swap(struct bpf_reg_state *dst_reg, struct bpf_insn *ins · 15909 unmodified lines | ||
| 15910 | 15910 | /* Apply bswap if alu64 or switch between big-endian and little-endian machines */ |
| 15911 | 15911 | bool need_bswap = alu64 || (to_le == is_big_endian); |
| 15912 | 15912 | |
| 15913 | + /* | |
| 15914 | + * If the register is mutated, manually reset its scalar ID to break | |
| 15915 | + * any existing ties and avoid incorrect bounds propagation. | |
| 15916 | + */ | |
| 15917 | + if (need_bswap || insn->imm == 16 || insn->imm == 32) | |
| 15918 | + dst_reg->id = 0; | |
| 15919 | + | |
| 15913 | 15920 | if (need_bswap) { |
| 15914 | 15921 | if (insn->imm == 16) |
| 15915 | 15922 | dst_reg->var_off = tnum_bswap16(dst_reg->var_off); |
| @@ -15992,7 +15999,7 @@ static int maybe_fork_scalars(struct bpf_verifier_env *env, struct bpf_insn *ins · 76 unmodified lines | ||
| 15992 | 15999 | else |
| 15993 | 16000 | return 0; |
| 15994 | 16001 | |
| 15995 | − branch = push_stack(env, env->insn_idx + 1, env->insn_idx, false); | |
| 16002 | + branch = push_stack(env, env->insn_idx, env->insn_idx, false); | |
| 15996 | 16003 | if (IS_ERR(branch)) |
| 15997 | 16004 | return PTR_ERR(branch); |
| 15998 | 16005 | |
| @@ -17408,6 +17415,12 @@ static void sync_linked_regs(struct bpf_verifier_env *env, struct bpf_verifier_s · 1409 unmodified lines | ||
| 17408 | 17415 | continue; |
| 17409 | 17416 | if ((reg->id & ~BPF_ADD_CONST) != (known_reg->id & ~BPF_ADD_CONST)) |
| 17410 | 17417 | continue; |
| 17418 | + /* | |
| 17419 | + * Skip mixed 32/64-bit links: the delta relationship doesn't | |
| 17420 | + * hold across different ALU widths. | |
| 17421 | + */ | |
| 17422 | + if (((reg->id ^ known_reg->id) & BPF_ADD_CONST) == BPF_ADD_CONST) | |
| 17423 | + continue; | |
| 17411 | 17424 | if ((!(reg->id & BPF_ADD_CONST) && !(known_reg->id & BPF_ADD_CONST)) || |
| 17412 | 17425 | reg->off == known_reg->off) { |
| 17413 | 17426 | s32 saved_subreg_def = reg->subreg_def; |
| @@ -17435,7 +17448,7 @@ static void sync_linked_regs(struct bpf_verifier_env *env, struct bpf_verifier_s · 21 unmodified lines | ||
| 17435 | 17448 | scalar32_min_max_add(reg, &fake_reg); |
| 17436 | 17449 | scalar_min_max_add(reg, &fake_reg); |
| 17437 | 17450 | reg->var_off = tnum_add(reg->var_off, fake_reg.var_off); |
| 17438 | − if (known_reg->id & BPF_ADD_CONST32) | |
| 17451 | + if ((reg->id | known_reg->id) & BPF_ADD_CONST32) | |
| 17439 | 17452 | zext_32_to_64(reg); |
| 17440 | 17453 | reg_bounds_sync(reg); |
| 17441 | 17454 | } |
| @@ -19863,11 +19876,14 @@ static bool regsafe(struct bpf_verifier_env *env, struct bpf_reg_state *rold, · 2421 unmodified lines | ||
| 19863 | 19876 | * Also verify that new value satisfies old value range knowledge. |
| 19864 | 19877 | */ |
| 19865 | 19878 | |
| 19866 | − /* ADD_CONST mismatch: different linking semantics */ | |
| 19867 | − if ((rold->id & BPF_ADD_CONST) && !(rcur->id & BPF_ADD_CONST)) | |
| 19868 | − return false; | |
| 19869 | − | |
| 19870 | − if (rold->id && !(rold->id & BPF_ADD_CONST) && (rcur->id & BPF_ADD_CONST)) | |
| 19879 | + /* | |
| 19880 | + * ADD_CONST flags must match exactly: BPF_ADD_CONST32 and | |
| 19881 | + * BPF_ADD_CONST64 have different linking semantics in | |
| 19882 | + * sync_linked_regs() (alu32 zero-extends, alu64 does not), | |
| 19883 | + * so pruning across different flag types is unsafe. | |
| 19884 | + */ | |
| 19885 | + if (rold->id && | |
| 19886 | + (rold->id & BPF_ADD_CONST) != (rcur->id & BPF_ADD_CONST)) | |
| 19871 | 19887 | return false; |
| 19872 | 19888 | |
| 19873 | 19889 | /* Both have offset linkage: offsets must match */ |
| @@ -20904,7 +20920,8 @@ static int process_bpf_exit_full(struct bpf_verifier_env *env, · 1030 unmodified lines | ||
| 20904 | 20920 | * state when it exits. |
| 20905 | 20921 | */ |
| 20906 | 20922 | int err = check_resource_leak(env, exception_exit, |
| 20907 | − !env->cur_state->curframe, | |
| 20923 | + exception_exit || !env->cur_state->curframe, | |
| 20924 | + exception_exit ? "bpf_throw" : | |
| 20908 | 20925 | "BPF_EXIT instruction in main prog"); |
| 20909 | 20926 | if (err) |
| 20910 | 20927 | return err; |