History

Merge git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf

Daniel Borkmann says: ==================== pull-request: bpf 2021-07-29 The following pull-request contains BPF updates for your *net* tree. We've added 9 non-merge commits during the last 14 day(s) which contain a total of 20 files changed, 446 insertions(+), 138 deletions(-). The main changes are: 1) Fix UBSAN out-of-bounds splat for showing XDP link fdinfo, from Lorenz Bauer. 2) Fix insufficient Spectre v4 mitigation in BPF runtime, from Daniel Borkmann, Piotr Krysiuk and Benedict Schlueter. 3) Batch of fixes for BPF sockmap found under stress testing, from John Fastabend. ====================

David S. Miller committed 5 years ago

Commit fc16a532 parents 89fb62f 2039f2620 files · +446 −138Browse files

Showing the changes to kernel (3 of 20 files). Show all

Unified · Split

3 files
  1. kernel/bpf/core.c +18 −1
  2. kernel/bpf/disasm.c +9 −7
  3. kernel/bpf/verifier.c +49 −99
kernel/bpf/core.c+18−1
@@ -32,6 +32,8 @@ · 31 unmodified lines
3232 #include <linux/perf_event.h>
3333 #include <linux/extable.h>
3434 #include <linux/log2.h>
35+
36+#include <asm/barrier.h>
3537 #include <asm/unaligned.h>
3638
3739 /* Registers */
@@ -1377,6 +1379,7 @@ static u64 ___bpf_prog_run(u64 *regs, const struct bpf_insn *insn) · 1339 unmodified lines
13771379 /* Non-UAPI available opcodes. */
13781380 [BPF_JMP | BPF_CALL_ARGS] = &&JMP_CALL_ARGS,
13791381 [BPF_JMP | BPF_TAIL_CALL] = &&JMP_TAIL_CALL,
1382+ [BPF_ST | BPF_NOSPEC] = &&ST_NOSPEC,
13801383 [BPF_LDX | BPF_PROBE_MEM | BPF_B] = &&LDX_PROBE_MEM_B,
13811384 [BPF_LDX | BPF_PROBE_MEM | BPF_H] = &&LDX_PROBE_MEM_H,
13821385 [BPF_LDX | BPF_PROBE_MEM | BPF_W] = &&LDX_PROBE_MEM_W,
@@ -1621,7 +1624,21 @@ out: · 238 unmodified lines
16211624 COND_JMP(s, JSGE, >=)
16221625 COND_JMP(s, JSLE, <=)
16231626 #undef COND_JMP
1624− /* STX and ST and LDX*/
1627+ /* ST, STX and LDX*/
1628+ ST_NOSPEC:
1629+ /* Speculation barrier for mitigating Speculative Store Bypass.
1630+ * In case of arm64, we rely on the firmware mitigation as
1631+ * controlled via the ssbd kernel parameter. Whenever the
1632+ * mitigation is enabled, it works for all of the kernel code
1633+ * with no need to provide any additional instructions here.
1634+ * In case of x86, we use 'lfence' insn for mitigation. We
1635+ * reuse preexisting logic from Spectre v1 mitigation that
1636+ * happens to produce the required code on x86 for v4 as well.
1637+ */
1638+#ifdef CONFIG_X86
1639+ barrier_nospec();
1640+#endif
1641+ CONT;
16251642 #define LDST(SIZEOP, SIZE) \
16261643 STX_MEM_##SIZEOP: \
16271644 *(SIZE *)(unsigned long) (DST + insn->off) = SRC; \
kernel/bpf/disasm.c+9−7
@@ -206,15 +206,17 @@ void print_bpf_insn(const struct bpf_insn_cbs *cbs, · 205 unmodified lines
206206 verbose(cbs->private_data, "BUG_%02x\n", insn->code);
207207 }
208208 } else if (class == BPF_ST) {
209− if (BPF_MODE(insn->code) != BPF_MEM) {
209+ if (BPF_MODE(insn->code) == BPF_MEM) {
210+ verbose(cbs->private_data, "(%02x) *(%s *)(r%d %+d) = %d\n",
211+ insn->code,
212+ bpf_ldst_string[BPF_SIZE(insn->code) >> 3],
213+ insn->dst_reg,
214+ insn->off, insn->imm);
215+ } else if (BPF_MODE(insn->code) == 0xc0 /* BPF_NOSPEC, no UAPI */) {
216+ verbose(cbs->private_data, "(%02x) nospec\n", insn->code);
217+ } else {
210218 verbose(cbs->private_data, "BUG_st_%02x\n", insn->code);
211− return;
212219 }
213− verbose(cbs->private_data, "(%02x) *(%s *)(r%d %+d) = %d\n",
214− insn->code,
215− bpf_ldst_string[BPF_SIZE(insn->code) >> 3],
216− insn->dst_reg,
217− insn->off, insn->imm);
218220 } else if (class == BPF_LDX) {
219221 if (BPF_MODE(insn->code) != BPF_MEM) {
220222 verbose(cbs->private_data, "BUG_ldx_%02x\n", insn->code);
kernel/bpf/verifier.c+49−99
@@ -2610,6 +2610,19 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env, · 2609 unmodified lines
26102610 cur = env->cur_state->frame[env->cur_state->curframe];
26112611 if (value_regno >= 0)
26122612 reg = &cur->regs[value_regno];
2613+ if (!env->bypass_spec_v4) {
2614+ bool sanitize = reg && is_spillable_regtype(reg->type);
2615+
2616+ for (i = 0; i < size; i++) {
2617+ if (state->stack[spi].slot_type[i] == STACK_INVALID) {
2618+ sanitize = true;
2619+ break;
2620+ }
2621+ }
2622+
2623+ if (sanitize)
2624+ env->insn_aux_data[insn_idx].sanitize_stack_spill = true;
2625+ }
26132626
26142627 if (reg && size == BPF_REG_SIZE && register_is_bounded(reg) &&
26152628 !register_is_null(reg) && env->bpf_capable) {
@@ -2632,47 +2645,10 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env, · 16 unmodified lines
26322645 verbose(env, "invalid size of register spill\n");
26332646 return -EACCES;
26342647 }
2635−
26362648 if (state != cur && reg->type == PTR_TO_STACK) {
26372649 verbose(env, "cannot spill pointers to stack into stack frame of the caller\n");
26382650 return -EINVAL;
26392651 }
2640−
2641− if (!env->bypass_spec_v4) {
2642− bool sanitize = false;
2643−
2644− if (state->stack[spi].slot_type[0] == STACK_SPILL &&
2645− register_is_const(&state->stack[spi].spilled_ptr))
2646− sanitize = true;
2647− for (i = 0; i < BPF_REG_SIZE; i++)
2648− if (state->stack[spi].slot_type[i] == STACK_MISC) {
2649− sanitize = true;
2650− break;
2651− }
2652− if (sanitize) {
2653− int *poff = &env->insn_aux_data[insn_idx].sanitize_stack_off;
2654− int soff = (-spi - 1) * BPF_REG_SIZE;
2655−
2656− /* detected reuse of integer stack slot with a pointer
2657− * which means either llvm is reusing stack slot or
2658− * an attacker is trying to exploit CVE-2018-3639
2659− * (speculative store bypass)
2660− * Have to sanitize that slot with preemptive
2661− * store of zero.
2662− */
2663− if (*poff && *poff != soff) {
2664− /* disallow programs where single insn stores
2665− * into two different stack slots, since verifier
2666− * cannot sanitize them
2667− */
2668− verbose(env,
2669− "insn %d cannot access two stack slots fp%d and fp%d",
2670− insn_idx, *poff, soff);
2671− return -EINVAL;
2672− }
2673− *poff = soff;
2674− }
2675− }
26762652 save_register_state(state, spi, reg);
26772653 } else {
26782654 u8 type = STACK_MISC;
@@ -6561,6 +6537,12 @@ static int sanitize_ptr_alu(struct bpf_verifier_env *env, · 3882 unmodified lines
65616537 alu_state |= off_is_imm ? BPF_ALU_IMMEDIATE : 0;
65626538 alu_state |= ptr_is_dst_reg ?
65636539 BPF_ALU_SANITIZE_SRC : BPF_ALU_SANITIZE_DST;
6540+
6541+ /* Limit pruning on unknown scalars to enable deep search for
6542+ * potential masking differences from other program paths.
6543+ */
6544+ if (!off_is_imm)
6545+ env->explore_alu_limits = true;
65646546 }
65656547
65666548 err = update_alu_sanitation_state(aux, alu_state, alu_limit);
@@ -9936,8 +9918,8 @@ next: · 3369 unmodified lines
99369918 }
99379919
99389920 /* Returns true if (rold safe implies rcur safe) */
9939−static bool regsafe(struct bpf_reg_state *rold, struct bpf_reg_state *rcur,
9940− struct bpf_id_pair *idmap)
9921+static bool regsafe(struct bpf_verifier_env *env, struct bpf_reg_state *rold,
9922+ struct bpf_reg_state *rcur, struct bpf_id_pair *idmap)
99419923 {
99429924 bool equal;
99439925
@@ -9963,6 +9945,8 @@ static bool regsafe(struct bpf_reg_state *rold, struct bpf_reg_state *rcur, · 19 unmodified lines
99639945 return false;
99649946 switch (rold->type) {
99659947 case SCALAR_VALUE:
9948+ if (env->explore_alu_limits)
9949+ return false;
99669950 if (rcur->type == SCALAR_VALUE) {
99679951 if (!rold->precise && !rcur->precise)
99689952 return true;
@@ -10053,9 +10037,8 @@ static bool regsafe(struct bpf_reg_state *rold, struct bpf_reg_state *rcur, · 84 unmodified lines
1005310037 return false;
1005410038 }
1005510039
10056−static bool stacksafe(struct bpf_func_state *old,
10057− struct bpf_func_state *cur,
10058− struct bpf_id_pair *idmap)
10040+static bool stacksafe(struct bpf_verifier_env *env, struct bpf_func_state *old,
10041+ struct bpf_func_state *cur, struct bpf_id_pair *idmap)
1005910042 {
1006010043 int i, spi;
1006110044
@@ -10100,9 +10083,8 @@ static bool stacksafe(struct bpf_func_state *old, · 38 unmodified lines
1010010083 continue;
1010110084 if (old->stack[spi].slot_type[0] != STACK_SPILL)
1010210085 continue;
10103− if (!regsafe(&old->stack[spi].spilled_ptr,
10104− &cur->stack[spi].spilled_ptr,
10105− idmap))
10086+ if (!regsafe(env, &old->stack[spi].spilled_ptr,
10087+ &cur->stack[spi].spilled_ptr, idmap))
1010610088 /* when explored and current stack slot are both storing
1010710089 * spilled registers, check that stored pointers types
1010810090 * are the same as well.
@@ -10159,10 +10141,11 @@ static bool func_states_equal(struct bpf_verifier_env *env, struct bpf_func_stat · 50 unmodified lines
1015910141
1016010142 memset(env->idmap_scratch, 0, sizeof(env->idmap_scratch));
1016110143 for (i = 0; i < MAX_BPF_REG; i++)
10162− if (!regsafe(&old->regs[i], &cur->regs[i], env->idmap_scratch))
10144+ if (!regsafe(env, &old->regs[i], &cur->regs[i],
10145+ env->idmap_scratch))
1016310146 return false;
1016410147
10165− if (!stacksafe(old, cur, env->idmap_scratch))
10148+ if (!stacksafe(env, old, cur, env->idmap_scratch))
1016610149 return false;
1016710150
1016810151 if (!refsafe(old, cur))
@@ -11906,35 +11889,33 @@ static int convert_ctx_accesses(struct bpf_verifier_env *env) · 1737 unmodified lines
1190611889
1190711890 for (i = 0; i < insn_cnt; i++, insn++) {
1190811891 bpf_convert_ctx_access_t convert_ctx_access;
11892+ bool ctx_access;
1190911893
1191011894 if (insn->code == (BPF_LDX | BPF_MEM | BPF_B) ||
1191111895 insn->code == (BPF_LDX | BPF_MEM | BPF_H) ||
1191211896 insn->code == (BPF_LDX | BPF_MEM | BPF_W) ||
11913− insn->code == (BPF_LDX | BPF_MEM | BPF_DW))
11897+ insn->code == (BPF_LDX | BPF_MEM | BPF_DW)) {
1191411898 type = BPF_READ;
11915− else if (insn->code == (BPF_STX | BPF_MEM | BPF_B) ||
11916− insn->code == (BPF_STX | BPF_MEM | BPF_H) ||
11917− insn->code == (BPF_STX | BPF_MEM | BPF_W) ||
11918− insn->code == (BPF_STX | BPF_MEM | BPF_DW))
11899+ ctx_access = true;
11900+ } else if (insn->code == (BPF_STX | BPF_MEM | BPF_B) ||
11901+ insn->code == (BPF_STX | BPF_MEM | BPF_H) ||
11902+ insn->code == (BPF_STX | BPF_MEM | BPF_W) ||
11903+ insn->code == (BPF_STX | BPF_MEM | BPF_DW) ||
11904+ insn->code == (BPF_ST | BPF_MEM | BPF_B) ||
11905+ insn->code == (BPF_ST | BPF_MEM | BPF_H) ||
11906+ insn->code == (BPF_ST | BPF_MEM | BPF_W) ||
11907+ insn->code == (BPF_ST | BPF_MEM | BPF_DW)) {
1191911908 type = BPF_WRITE;
11920− else
11909+ ctx_access = BPF_CLASS(insn->code) == BPF_STX;
11910+ } else {
1192111911 continue;
11912+ }
1192211913
1192311914 if (type == BPF_WRITE &&
11924− env->insn_aux_data[i + delta].sanitize_stack_off) {
11915+ env->insn_aux_data[i + delta].sanitize_stack_spill) {
1192511916 struct bpf_insn patch[] = {
11926− /* Sanitize suspicious stack slot with zero.
11927− * There are no memory dependencies for this store,
11928− * since it's only using frame pointer and immediate
11929− * constant of zero
11930− */
11931− BPF_ST_MEM(BPF_DW, BPF_REG_FP,
11932− env->insn_aux_data[i + delta].sanitize_stack_off,
11933− 0),
11934− /* the original STX instruction will immediately
11935− * overwrite the same stack slot with appropriate value
11936− */
1193711917 *insn,
11918+ BPF_ST_NOSPEC(),
1193811919 };
1193911920
1194011921 cnt = ARRAY_SIZE(patch);
@@ -11948,6 +11929,9 @@ static int convert_ctx_accesses(struct bpf_verifier_env *env) · 7 unmodified lines
1194811929 continue;
1194911930 }
1195011931
11932+ if (!ctx_access)
11933+ continue;
11934+
1195111935 switch (env->insn_aux_data[i + delta].ptr_type) {
1195211936 case PTR_TO_CTX:
1195311937 if (!ops->convert_ctx_access)
@@ -12752,37 +12736,6 @@ static void free_states(struct bpf_verifier_env *env) · 798 unmodified lines
1275212736 }
1275312737 }
1275412738
12755−/* The verifier is using insn_aux_data[] to store temporary data during
12756− * verification and to store information for passes that run after the
12757− * verification like dead code sanitization. do_check_common() for subprogram N
12758− * may analyze many other subprograms. sanitize_insn_aux_data() clears all
12759− * temporary data after do_check_common() finds that subprogram N cannot be
12760− * verified independently. pass_cnt counts the number of times
12761− * do_check_common() was run and insn->aux->seen tells the pass number
12762− * insn_aux_data was touched. These variables are compared to clear temporary
12763− * data from failed pass. For testing and experiments do_check_common() can be
12764− * run multiple times even when prior attempt to verify is unsuccessful.
12765− *
12766− * Note that special handling is needed on !env->bypass_spec_v1 if this is
12767− * ever called outside of error path with subsequent program rejection.
12768− */
12769−static void sanitize_insn_aux_data(struct bpf_verifier_env *env)
12770−{
12771− struct bpf_insn *insn = env->prog->insnsi;
12772− struct bpf_insn_aux_data *aux;
12773− int i, class;
12774−
12775− for (i = 0; i < env->prog->len; i++) {
12776− class = BPF_CLASS(insn[i].code);
12777− if (class != BPF_LDX && class != BPF_STX)
12778− continue;
12779− aux = &env->insn_aux_data[i];
12780− if (aux->seen != env->pass_cnt)
12781− continue;
12782− memset(aux, 0, offsetof(typeof(*aux), orig_idx));
12783− }
12784−}
12785−
1278612739 static int do_check_common(struct bpf_verifier_env *env, int subprog)
1278712740 {
1278812741 bool pop_log = !(env->log.level & BPF_LOG_LEVEL2);
@@ -12859,9 +12812,6 @@ out: · 70 unmodified lines
1285912812 if (!ret && pop_log)
1286012813 bpf_vlog_reset(&env->log, 0);
1286112814 free_states(env);
12862− if (ret)
12863− /* clean aux data in case subprog was rejected */
12864− sanitize_insn_aux_data(env);
1286512815 return ret;
1286612816 }
1286712817

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t