History

Merge git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf

Daniel Borkmann says: ==================== pull-request: bpf 2021-07-29 The following pull-request contains BPF updates for your *net* tree. We've added 9 non-merge commits during the last 14 day(s) which contain a total of 20 files changed, 446 insertions(+), 138 deletions(-). The main changes are: 1) Fix UBSAN out-of-bounds splat for showing XDP link fdinfo, from Lorenz Bauer. 2) Fix insufficient Spectre v4 mitigation in BPF runtime, from Daniel Borkmann, Piotr Krysiuk and Benedict Schlueter. 3) Batch of fixes for BPF sockmap found under stress testing, from John Fastabend. ====================

David S. Miller committed 5 years ago

Commit fc16a532 parents 89fb62f 2039f2620 files · +446 −138Browse files

Showing the changes to kernel (3 of 20 files). Show all

Unified · Split

3 files
  1. kernel/bpf/core.c +18 −1
  2. kernel/bpf/disasm.c +9 −7
  3. kernel/bpf/verifier.c +49 −99
kernel/bpf/core.c+18−1
@@ -32,6 +32,8 @@ · 31 unmodified lines
32#include <linux/perf_event.h>32#include <linux/perf_event.h>
33#include <linux/extable.h>33#include <linux/extable.h>
34#include <linux/log2.h>34#include <linux/log2.h>
35
36#include <asm/barrier.h>
35#include <asm/unaligned.h>37#include <asm/unaligned.h>
36 38
37/* Registers */39/* Registers */
@@ -1377,6 +1379,7 @@ static u64 ___bpf_prog_run(u64 *regs, const struct bpf_insn *insn) · 1339 unmodified lines
1377 /* Non-UAPI available opcodes. */1379 /* Non-UAPI available opcodes. */
1378 [BPF_JMP | BPF_CALL_ARGS] = &&JMP_CALL_ARGS,1380 [BPF_JMP | BPF_CALL_ARGS] = &&JMP_CALL_ARGS,
1379 [BPF_JMP | BPF_TAIL_CALL] = &&JMP_TAIL_CALL,1381 [BPF_JMP | BPF_TAIL_CALL] = &&JMP_TAIL_CALL,
1382 [BPF_ST | BPF_NOSPEC] = &&ST_NOSPEC,
1380 [BPF_LDX | BPF_PROBE_MEM | BPF_B] = &&LDX_PROBE_MEM_B,1383 [BPF_LDX | BPF_PROBE_MEM | BPF_B] = &&LDX_PROBE_MEM_B,
1381 [BPF_LDX | BPF_PROBE_MEM | BPF_H] = &&LDX_PROBE_MEM_H,1384 [BPF_LDX | BPF_PROBE_MEM | BPF_H] = &&LDX_PROBE_MEM_H,
1382 [BPF_LDX | BPF_PROBE_MEM | BPF_W] = &&LDX_PROBE_MEM_W,1385 [BPF_LDX | BPF_PROBE_MEM | BPF_W] = &&LDX_PROBE_MEM_W,
@@ -1621,7 +1624,21 @@ out: · 238 unmodified lines
1621 COND_JMP(s, JSGE, >=)1624 COND_JMP(s, JSGE, >=)
1622 COND_JMP(s, JSLE, <=)1625 COND_JMP(s, JSLE, <=)
1623#undef COND_JMP1626#undef COND_JMP
1624 /* STX and ST and LDX*/1627 /* ST, STX and LDX*/
1628 ST_NOSPEC:
1629 /* Speculation barrier for mitigating Speculative Store Bypass.
1630 * In case of arm64, we rely on the firmware mitigation as
1631 * controlled via the ssbd kernel parameter. Whenever the
1632 * mitigation is enabled, it works for all of the kernel code
1633 * with no need to provide any additional instructions here.
1634 * In case of x86, we use 'lfence' insn for mitigation. We
1635 * reuse preexisting logic from Spectre v1 mitigation that
1636 * happens to produce the required code on x86 for v4 as well.
1637 */
1638#ifdef CONFIG_X86
1639 barrier_nospec();
1640#endif
1641 CONT;
1625#define LDST(SIZEOP, SIZE) \1642#define LDST(SIZEOP, SIZE) \
1626 STX_MEM_##SIZEOP: \1643 STX_MEM_##SIZEOP: \
1627 *(SIZE *)(unsigned long) (DST + insn->off) = SRC; \1644 *(SIZE *)(unsigned long) (DST + insn->off) = SRC; \
kernel/bpf/disasm.c+9−7
@@ -206,15 +206,17 @@ void print_bpf_insn(const struct bpf_insn_cbs *cbs, · 205 unmodified lines
206 verbose(cbs->private_data, "BUG_%02x\n", insn->code);206 verbose(cbs->private_data, "BUG_%02x\n", insn->code);
207 }207 }
208 } else if (class == BPF_ST) {208 } else if (class == BPF_ST) {
209 if (BPF_MODE(insn->code) != BPF_MEM) {209 if (BPF_MODE(insn->code) == BPF_MEM) {
210 verbose(cbs->private_data, "(%02x) *(%s *)(r%d %+d) = %d\n",
211 insn->code,
212 bpf_ldst_string[BPF_SIZE(insn->code) >> 3],
213 insn->dst_reg,
214 insn->off, insn->imm);
215 } else if (BPF_MODE(insn->code) == 0xc0 /* BPF_NOSPEC, no UAPI */) {
216 verbose(cbs->private_data, "(%02x) nospec\n", insn->code);
217 } else {
210 verbose(cbs->private_data, "BUG_st_%02x\n", insn->code);218 verbose(cbs->private_data, "BUG_st_%02x\n", insn->code);
211 return;
212 }219 }
213 verbose(cbs->private_data, "(%02x) *(%s *)(r%d %+d) = %d\n",
214 insn->code,
215 bpf_ldst_string[BPF_SIZE(insn->code) >> 3],
216 insn->dst_reg,
217 insn->off, insn->imm);
218 } else if (class == BPF_LDX) {220 } else if (class == BPF_LDX) {
219 if (BPF_MODE(insn->code) != BPF_MEM) {221 if (BPF_MODE(insn->code) != BPF_MEM) {
220 verbose(cbs->private_data, "BUG_ldx_%02x\n", insn->code);222 verbose(cbs->private_data, "BUG_ldx_%02x\n", insn->code);
kernel/bpf/verifier.c+49−99
@@ -2610,6 +2610,19 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env, · 2609 unmodified lines
2610 cur = env->cur_state->frame[env->cur_state->curframe];2610 cur = env->cur_state->frame[env->cur_state->curframe];
2611 if (value_regno >= 0)2611 if (value_regno >= 0)
2612 reg = &cur->regs[value_regno];2612 reg = &cur->regs[value_regno];
2613 if (!env->bypass_spec_v4) {
2614 bool sanitize = reg && is_spillable_regtype(reg->type);
2615
2616 for (i = 0; i < size; i++) {
2617 if (state->stack[spi].slot_type[i] == STACK_INVALID) {
2618 sanitize = true;
2619 break;
2620 }
2621 }
2622
2623 if (sanitize)
2624 env->insn_aux_data[insn_idx].sanitize_stack_spill = true;
2625 }
2613 2626
2614 if (reg && size == BPF_REG_SIZE && register_is_bounded(reg) &&2627 if (reg && size == BPF_REG_SIZE && register_is_bounded(reg) &&
2615 !register_is_null(reg) && env->bpf_capable) {2628 !register_is_null(reg) && env->bpf_capable) {
@@ -2632,47 +2645,10 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env, · 16 unmodified lines
2632 verbose(env, "invalid size of register spill\n");2645 verbose(env, "invalid size of register spill\n");
2633 return -EACCES;2646 return -EACCES;
2634 }2647 }
2635
2636 if (state != cur && reg->type == PTR_TO_STACK) {2648 if (state != cur && reg->type == PTR_TO_STACK) {
2637 verbose(env, "cannot spill pointers to stack into stack frame of the caller\n");2649 verbose(env, "cannot spill pointers to stack into stack frame of the caller\n");
2638 return -EINVAL;2650 return -EINVAL;
2639 }2651 }
2640
2641 if (!env->bypass_spec_v4) {
2642 bool sanitize = false;
2643
2644 if (state->stack[spi].slot_type[0] == STACK_SPILL &&
2645 register_is_const(&state->stack[spi].spilled_ptr))
2646 sanitize = true;
2647 for (i = 0; i < BPF_REG_SIZE; i++)
2648 if (state->stack[spi].slot_type[i] == STACK_MISC) {
2649 sanitize = true;
2650 break;
2651 }
2652 if (sanitize) {
2653 int *poff = &env->insn_aux_data[insn_idx].sanitize_stack_off;
2654 int soff = (-spi - 1) * BPF_REG_SIZE;
2655
2656 /* detected reuse of integer stack slot with a pointer
2657 * which means either llvm is reusing stack slot or
2658 * an attacker is trying to exploit CVE-2018-3639
2659 * (speculative store bypass)
2660 * Have to sanitize that slot with preemptive
2661 * store of zero.
2662 */
2663 if (*poff && *poff != soff) {
2664 /* disallow programs where single insn stores
2665 * into two different stack slots, since verifier
2666 * cannot sanitize them
2667 */
2668 verbose(env,
2669 "insn %d cannot access two stack slots fp%d and fp%d",
2670 insn_idx, *poff, soff);
2671 return -EINVAL;
2672 }
2673 *poff = soff;
2674 }
2675 }
2676 save_register_state(state, spi, reg);2652 save_register_state(state, spi, reg);
2677 } else {2653 } else {
2678 u8 type = STACK_MISC;2654 u8 type = STACK_MISC;
@@ -6561,6 +6537,12 @@ static int sanitize_ptr_alu(struct bpf_verifier_env *env, · 3882 unmodified lines
6561 alu_state |= off_is_imm ? BPF_ALU_IMMEDIATE : 0;6537 alu_state |= off_is_imm ? BPF_ALU_IMMEDIATE : 0;
6562 alu_state |= ptr_is_dst_reg ?6538 alu_state |= ptr_is_dst_reg ?
6563 BPF_ALU_SANITIZE_SRC : BPF_ALU_SANITIZE_DST;6539 BPF_ALU_SANITIZE_SRC : BPF_ALU_SANITIZE_DST;
6540
6541 /* Limit pruning on unknown scalars to enable deep search for
6542 * potential masking differences from other program paths.
6543 */
6544 if (!off_is_imm)
6545 env->explore_alu_limits = true;
6564 }6546 }
6565 6547
6566 err = update_alu_sanitation_state(aux, alu_state, alu_limit);6548 err = update_alu_sanitation_state(aux, alu_state, alu_limit);
@@ -9936,8 +9918,8 @@ next: · 3369 unmodified lines
9936}9918}
9937 9919
9938/* Returns true if (rold safe implies rcur safe) */9920/* Returns true if (rold safe implies rcur safe) */
9939static bool regsafe(struct bpf_reg_state *rold, struct bpf_reg_state *rcur,9921static bool regsafe(struct bpf_verifier_env *env, struct bpf_reg_state *rold,
9940 struct bpf_id_pair *idmap)9922 struct bpf_reg_state *rcur, struct bpf_id_pair *idmap)
9941{9923{
9942 bool equal;9924 bool equal;
9943 9925
@@ -9963,6 +9945,8 @@ static bool regsafe(struct bpf_reg_state *rold, struct bpf_reg_state *rcur, · 19 unmodified lines
9963 return false;9945 return false;
9964 switch (rold->type) {9946 switch (rold->type) {
9965 case SCALAR_VALUE:9947 case SCALAR_VALUE:
9948 if (env->explore_alu_limits)
9949 return false;
9966 if (rcur->type == SCALAR_VALUE) {9950 if (rcur->type == SCALAR_VALUE) {
9967 if (!rold->precise && !rcur->precise)9951 if (!rold->precise && !rcur->precise)
9968 return true;9952 return true;
@@ -10053,9 +10037,8 @@ static bool regsafe(struct bpf_reg_state *rold, struct bpf_reg_state *rcur, · 84 unmodified lines
10053 return false;10037 return false;
10054}10038}
10055 10039
10056static bool stacksafe(struct bpf_func_state *old,10040static bool stacksafe(struct bpf_verifier_env *env, struct bpf_func_state *old,
10057 struct bpf_func_state *cur,10041 struct bpf_func_state *cur, struct bpf_id_pair *idmap)
10058 struct bpf_id_pair *idmap)
10059{10042{
10060 int i, spi;10043 int i, spi;
10061 10044
@@ -10100,9 +10083,8 @@ static bool stacksafe(struct bpf_func_state *old, · 38 unmodified lines
10100 continue;10083 continue;
10101 if (old->stack[spi].slot_type[0] != STACK_SPILL)10084 if (old->stack[spi].slot_type[0] != STACK_SPILL)
10102 continue;10085 continue;
10103 if (!regsafe(&old->stack[spi].spilled_ptr,10086 if (!regsafe(env, &old->stack[spi].spilled_ptr,
10104 &cur->stack[spi].spilled_ptr,10087 &cur->stack[spi].spilled_ptr, idmap))
10105 idmap))
10106 /* when explored and current stack slot are both storing10088 /* when explored and current stack slot are both storing
10107 * spilled registers, check that stored pointers types10089 * spilled registers, check that stored pointers types
10108 * are the same as well.10090 * are the same as well.
@@ -10159,10 +10141,11 @@ static bool func_states_equal(struct bpf_verifier_env *env, struct bpf_func_stat · 50 unmodified lines
10159 10141
10160 memset(env->idmap_scratch, 0, sizeof(env->idmap_scratch));10142 memset(env->idmap_scratch, 0, sizeof(env->idmap_scratch));
10161 for (i = 0; i < MAX_BPF_REG; i++)10143 for (i = 0; i < MAX_BPF_REG; i++)
10162 if (!regsafe(&old->regs[i], &cur->regs[i], env->idmap_scratch))10144 if (!regsafe(env, &old->regs[i], &cur->regs[i],
10145 env->idmap_scratch))
10163 return false;10146 return false;
10164 10147
10165 if (!stacksafe(old, cur, env->idmap_scratch))10148 if (!stacksafe(env, old, cur, env->idmap_scratch))
10166 return false;10149 return false;
10167 10150
10168 if (!refsafe(old, cur))10151 if (!refsafe(old, cur))
@@ -11906,35 +11889,33 @@ static int convert_ctx_accesses(struct bpf_verifier_env *env) · 1737 unmodified lines
11906 11889
11907 for (i = 0; i < insn_cnt; i++, insn++) {11890 for (i = 0; i < insn_cnt; i++, insn++) {
11908 bpf_convert_ctx_access_t convert_ctx_access;11891 bpf_convert_ctx_access_t convert_ctx_access;
11892 bool ctx_access;
11909 11893
11910 if (insn->code == (BPF_LDX | BPF_MEM | BPF_B) ||11894 if (insn->code == (BPF_LDX | BPF_MEM | BPF_B) ||
11911 insn->code == (BPF_LDX | BPF_MEM | BPF_H) ||11895 insn->code == (BPF_LDX | BPF_MEM | BPF_H) ||
11912 insn->code == (BPF_LDX | BPF_MEM | BPF_W) ||11896 insn->code == (BPF_LDX | BPF_MEM | BPF_W) ||
11913 insn->code == (BPF_LDX | BPF_MEM | BPF_DW))11897 insn->code == (BPF_LDX | BPF_MEM | BPF_DW)) {
11914 type = BPF_READ;11898 type = BPF_READ;
11915 else if (insn->code == (BPF_STX | BPF_MEM | BPF_B) ||11899 ctx_access = true;
11916 insn->code == (BPF_STX | BPF_MEM | BPF_H) ||11900 } else if (insn->code == (BPF_STX | BPF_MEM | BPF_B) ||
11917 insn->code == (BPF_STX | BPF_MEM | BPF_W) ||11901 insn->code == (BPF_STX | BPF_MEM | BPF_H) ||
11918 insn->code == (BPF_STX | BPF_MEM | BPF_DW))11902 insn->code == (BPF_STX | BPF_MEM | BPF_W) ||
11903 insn->code == (BPF_STX | BPF_MEM | BPF_DW) ||
11904 insn->code == (BPF_ST | BPF_MEM | BPF_B) ||
11905 insn->code == (BPF_ST | BPF_MEM | BPF_H) ||
11906 insn->code == (BPF_ST | BPF_MEM | BPF_W) ||
11907 insn->code == (BPF_ST | BPF_MEM | BPF_DW)) {
11919 type = BPF_WRITE;11908 type = BPF_WRITE;
11920 else11909 ctx_access = BPF_CLASS(insn->code) == BPF_STX;
11910 } else {
11921 continue;11911 continue;
11912 }
11922 11913
11923 if (type == BPF_WRITE &&11914 if (type == BPF_WRITE &&
11924 env->insn_aux_data[i + delta].sanitize_stack_off) {11915 env->insn_aux_data[i + delta].sanitize_stack_spill) {
11925 struct bpf_insn patch[] = {11916 struct bpf_insn patch[] = {
11926 /* Sanitize suspicious stack slot with zero.
11927 * There are no memory dependencies for this store,
11928 * since it's only using frame pointer and immediate
11929 * constant of zero
11930 */
11931 BPF_ST_MEM(BPF_DW, BPF_REG_FP,
11932 env->insn_aux_data[i + delta].sanitize_stack_off,
11933 0),
11934 /* the original STX instruction will immediately
11935 * overwrite the same stack slot with appropriate value
11936 */
11937 *insn,11917 *insn,
11918 BPF_ST_NOSPEC(),
11938 };11919 };
11939 11920
11940 cnt = ARRAY_SIZE(patch);11921 cnt = ARRAY_SIZE(patch);
@@ -11948,6 +11929,9 @@ static int convert_ctx_accesses(struct bpf_verifier_env *env) · 7 unmodified lines
11948 continue;11929 continue;
11949 }11930 }
11950 11931
11932 if (!ctx_access)
11933 continue;
11934
11951 switch (env->insn_aux_data[i + delta].ptr_type) {11935 switch (env->insn_aux_data[i + delta].ptr_type) {
11952 case PTR_TO_CTX:11936 case PTR_TO_CTX:
11953 if (!ops->convert_ctx_access)11937 if (!ops->convert_ctx_access)
@@ -12752,37 +12736,6 @@ static void free_states(struct bpf_verifier_env *env) · 798 unmodified lines
12752 }12736 }
12753}12737}
12754 12738
12755/* The verifier is using insn_aux_data[] to store temporary data during
12756 * verification and to store information for passes that run after the
12757 * verification like dead code sanitization. do_check_common() for subprogram N
12758 * may analyze many other subprograms. sanitize_insn_aux_data() clears all
12759 * temporary data after do_check_common() finds that subprogram N cannot be
12760 * verified independently. pass_cnt counts the number of times
12761 * do_check_common() was run and insn->aux->seen tells the pass number
12762 * insn_aux_data was touched. These variables are compared to clear temporary
12763 * data from failed pass. For testing and experiments do_check_common() can be
12764 * run multiple times even when prior attempt to verify is unsuccessful.
12765 *
12766 * Note that special handling is needed on !env->bypass_spec_v1 if this is
12767 * ever called outside of error path with subsequent program rejection.
12768 */
12769static void sanitize_insn_aux_data(struct bpf_verifier_env *env)
12770{
12771 struct bpf_insn *insn = env->prog->insnsi;
12772 struct bpf_insn_aux_data *aux;
12773 int i, class;
12774
12775 for (i = 0; i < env->prog->len; i++) {
12776 class = BPF_CLASS(insn[i].code);
12777 if (class != BPF_LDX && class != BPF_STX)
12778 continue;
12779 aux = &env->insn_aux_data[i];
12780 if (aux->seen != env->pass_cnt)
12781 continue;
12782 memset(aux, 0, offsetof(typeof(*aux), orig_idx));
12783 }
12784}
12785
12786static int do_check_common(struct bpf_verifier_env *env, int subprog)12739static int do_check_common(struct bpf_verifier_env *env, int subprog)
12787{12740{
12788 bool pop_log = !(env->log.level & BPF_LOG_LEVEL2);12741 bool pop_log = !(env->log.level & BPF_LOG_LEVEL2);
@@ -12859,9 +12812,6 @@ out: · 70 unmodified lines
12859 if (!ret && pop_log)12812 if (!ret && pop_log)
12860 bpf_vlog_reset(&env->log, 0);12813 bpf_vlog_reset(&env->log, 0);
12861 free_states(env);12814 free_states(env);
12862 if (ret)
12863 /* clean aux data in case subprog was rejected */
12864 sanitize_insn_aux_data(env);
12865 return ret;12815 return ret;
12866}12816}
12867 12817

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t