Merge git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next

Alexei Starovoitov says: ==================== pull-request: bpf-next 2020-10-12 The main changes are: 1) The BPF verifier improvements to track register allocation pattern, from Alexei and Yonghong. 2) libbpf relocation support for different size load/store, from Andrii. 3) bpf_redirect_peer() helper and support for inner map array with different max_entries, from Daniel. 4) BPF support for per-cpu variables, form Hao. 5) sockmap improvements, from John. ====================

Jakub Kicinski committed 6 years ago

Commit ccdf7fa2 parents a308283 376dcfe73 files · +4339 −772Browse files

Showing the changes to kernel (9 of 73 files). Show all

Unified · Split

9 files

kernel/bpf/arraymap.c +11 −6
kernel/bpf/btf.c +0 −25
kernel/bpf/hashtab.c +3 −3
kernel/bpf/helpers.c +32 −0
kernel/bpf/percpu_freelist.c +96 −5
kernel/bpf/percpu_freelist.h +1 −0
kernel/bpf/syscall.c +3 −1
kernel/bpf/verifier.c +251 −19
kernel/trace/bpf_trace.c +6 −0

kernel/bpf/arraymap.c+11−6

@@ -16,7 +16,7 @@ · 15 unmodified lines
16	16
17	17	#define ARRAY_CREATE_FLAG_MASK \
18	18	(BPF_F_NUMA_NODE \| BPF_F_MMAPABLE \| BPF_F_ACCESS_MASK \| \
19		− BPF_F_PRESERVE_ELEMS)
	19	+ BPF_F_PRESERVE_ELEMS \| BPF_F_INNER_MAP)
20	20
21	21	static void bpf_array_free_percpu(struct bpf_array *array)
22	22	{
@@ -62,7 +62,7 @@ int array_map_alloc_check(union bpf_attr *attr) · 39 unmodified lines
62	62	return -EINVAL;
63	63
64	64	if (attr->map_type != BPF_MAP_TYPE_ARRAY &&
65		− attr->map_flags & BPF_F_MMAPABLE)
	65	+ attr->map_flags & (BPF_F_MMAPABLE \| BPF_F_INNER_MAP))
66	66	return -EINVAL;
67	67
68	68	if (attr->map_type != BPF_MAP_TYPE_PERF_EVENT_ARRAY &&
@@ -214,7 +214,7 @@ static int array_map_direct_value_meta(const struct bpf_map *map, u64 imm, · 145 unmodified lines
214	214	}
215	215
216	216	/* emit BPF instructions equivalent to C code of array_map_lookup_elem() */
217		−static u32 array_map_gen_lookup(struct bpf_map map, struct bpf_insn insn_buf)
	217	+static int array_map_gen_lookup(struct bpf_map map, struct bpf_insn insn_buf)
218	218	{
219	219	struct bpf_array *array = container_of(map, struct bpf_array, map);
220	220	struct bpf_insn *insn = insn_buf;
@@ -223,6 +223,9 @@ static u32 array_map_gen_lookup(struct bpf_map map, struct bpf_insn insn_buf) · 2 unmodified lines
223	223	const int map_ptr = BPF_REG_1;
224	224	const int index = BPF_REG_2;
225	225
	226	+ if (map->map_flags & BPF_F_INNER_MAP)
	227	+ return -EOPNOTSUPP;
	228	+
226	229	*insn++ = BPF_ALU64_IMM(BPF_ADD, map_ptr, offsetof(struct bpf_array, value));
227	230	*insn++ = BPF_LDX_MEM(BPF_W, ret, index, 0);
228	231	if (!map->bypass_spec_v1) {
@@ -496,8 +499,10 @@ static int array_map_mmap(struct bpf_map map, struct vm_area_struct vma) · 267 unmodified lines
496	499	static bool array_map_meta_equal(const struct bpf_map *meta0,
497	500	const struct bpf_map *meta1)
498	501	{
499		− return meta0->max_entries == meta1->max_entries &&
500		− bpf_map_meta_equal(meta0, meta1);
	502	+ if (!bpf_map_meta_equal(meta0, meta1))
	503	+ return false;
	504	+ return meta0->map_flags & BPF_F_INNER_MAP ? true :
	505	+ meta0->max_entries == meta1->max_entries;
501	506	}
502	507
503	508	struct bpf_iter_seq_array_map_info {
@@ -1251,7 +1256,7 @@ static void array_of_map_lookup_elem(struct bpf_map map, void *key) · 747 unmodified lines
1251	1256	return READ_ONCE(*inner_map);
1252	1257	}
1253	1258
1254		−static u32 array_of_map_gen_lookup(struct bpf_map *map,
	1259	+static int array_of_map_gen_lookup(struct bpf_map *map,
1255	1260	struct bpf_insn *insn_buf)
1256	1261	{
1257	1262	struct bpf_array *array = container_of(map, struct bpf_array, map);

kernel/bpf/btf.c+0−25

@@ -188,11 +188,6 @@ · 187 unmodified lines
188	188	i < btf_type_vlen(struct_type); \
189	189	i++, member++)
190	190
191		−#define for_each_vsi(i, struct_type, member) \
192		− for (i = 0, member = btf_type_var_secinfo(struct_type); \
193		− i < btf_type_vlen(struct_type); \
194		− i++, member++)
195		−
196	191	#define for_each_vsi_from(i, from, struct_type, member) \
197	192	for (i = from, member = btf_type_var_secinfo(struct_type) + from; \
198	193	i < btf_type_vlen(struct_type); \
@@ -440,16 +435,6 @@ static bool btf_type_nosize_or_null(const struct btf_type *t) · 241 unmodified lines
440	435	return !t \|\| btf_type_nosize(t);
441	436	}
442	437
443		−/* union is only a special case of struct:
444		− * all its offsetof(member) == 0
445		− */
446		−static bool btf_type_is_struct(const struct btf_type *t)
447		−{
448		− u8 kind = BTF_INFO_KIND(t->info);
449		−
450		− return kind == BTF_KIND_STRUCT \|\| kind == BTF_KIND_UNION;
451		−}
452		−
453	438	static bool __btf_type_is_struct(const struct btf_type *t)
454	439	{
455	440	return BTF_INFO_KIND(t->info) == BTF_KIND_STRUCT;
@@ -460,11 +445,6 @@ static bool btf_type_is_array(const struct btf_type *t) · 4 unmodified lines
460	445	return BTF_INFO_KIND(t->info) == BTF_KIND_ARRAY;
461	446	}
462	447
463		−static bool btf_type_is_var(const struct btf_type *t)
464		−{
465		− return BTF_INFO_KIND(t->info) == BTF_KIND_VAR;
466		−}
467		−
468	448	static bool btf_type_is_datasec(const struct btf_type *t)
469	449	{
470	450	return BTF_INFO_KIND(t->info) == BTF_KIND_DATASEC;
@@ -613,11 +593,6 @@ static const struct btf_var btf_type_var(const struct btf_type t) · 142 unmodified lines
613	593	return (const struct btf_var *)(t + 1);
614	594	}
615	595
616		−static const struct btf_var_secinfo btf_type_var_secinfo(const struct btf_type t)
617		−{
618		− return (const struct btf_var_secinfo *)(t + 1);
619		−}
620		−
621	596	static const struct btf_kind_operations btf_type_ops(const struct btf_type t)
622	597	{
623	598	return kind_ops[BTF_INFO_KIND(t->info)];

kernel/bpf/hashtab.c+3−3

@@ -612,7 +612,7 @@ static void htab_map_lookup_elem(struct bpf_map map, void *key) · 611 unmodified lines
612	612	* bpf_prog
613	613	* __htab_map_lookup_elem
614	614	*/
615		−static u32 htab_map_gen_lookup(struct bpf_map map, struct bpf_insn insn_buf)
	615	+static int htab_map_gen_lookup(struct bpf_map map, struct bpf_insn insn_buf)
616	616	{
617	617	struct bpf_insn *insn = insn_buf;
618	618	const int ret = BPF_REG_0;
@@ -651,7 +651,7 @@ static void htab_lru_map_lookup_elem_sys(struct bpf_map map, void *key) · 32 unmodified lines
651	651	return __htab_lru_map_lookup_elem(map, key, false);
652	652	}
653	653
654		−static u32 htab_lru_map_gen_lookup(struct bpf_map *map,
	654	+static int htab_lru_map_gen_lookup(struct bpf_map *map,
655	655	struct bpf_insn *insn_buf)
656	656	{
657	657	struct bpf_insn *insn = insn_buf;
@@ -2070,7 +2070,7 @@ static void htab_of_map_lookup_elem(struct bpf_map map, void *key) · 1412 unmodified lines
2070	2070	return READ_ONCE(*inner_map);
2071	2071	}
2072	2072
2073		−static u32 htab_of_map_gen_lookup(struct bpf_map *map,
	2073	+static int htab_of_map_gen_lookup(struct bpf_map *map,
2074	2074	struct bpf_insn *insn_buf)
2075	2075	{
2076	2076	struct bpf_insn *insn = insn_buf;

kernel/bpf/helpers.c+32−0

@@ -623,6 +623,34 @@ const struct bpf_func_proto bpf_copy_from_user_proto = { · 622 unmodified lines
623	623	.arg3_type = ARG_ANYTHING,
624	624	};
625	625
	626	+BPF_CALL_2(bpf_per_cpu_ptr, const void *, ptr, u32, cpu)
	627	+{
	628	+ if (cpu >= nr_cpu_ids)
	629	+ return (unsigned long)NULL;
	630	+
	631	+ return (unsigned long)per_cpu_ptr((const void __percpu *)ptr, cpu);
	632	+}
	633	+
	634	+const struct bpf_func_proto bpf_per_cpu_ptr_proto = {
	635	+ .func = bpf_per_cpu_ptr,
	636	+ .gpl_only = false,
	637	+ .ret_type = RET_PTR_TO_MEM_OR_BTF_ID_OR_NULL,
	638	+ .arg1_type = ARG_PTR_TO_PERCPU_BTF_ID,
	639	+ .arg2_type = ARG_ANYTHING,
	640	+};
	641	+
	642	+BPF_CALL_1(bpf_this_cpu_ptr, const void *, percpu_ptr)
	643	+{
	644	+ return (unsigned long)this_cpu_ptr((const void __percpu *)percpu_ptr);
	645	+}
	646	+
	647	+const struct bpf_func_proto bpf_this_cpu_ptr_proto = {
	648	+ .func = bpf_this_cpu_ptr,
	649	+ .gpl_only = false,
	650	+ .ret_type = RET_PTR_TO_MEM_OR_BTF_ID,
	651	+ .arg1_type = ARG_PTR_TO_PERCPU_BTF_ID,
	652	+};
	653	+
626	654	const struct bpf_func_proto bpf_get_current_task_proto __weak;
627	655	const struct bpf_func_proto bpf_probe_read_user_proto __weak;
628	656	const struct bpf_func_proto bpf_probe_read_user_str_proto __weak;
@@ -689,6 +717,10 @@ bpf_base_func_proto(enum bpf_func_id func_id) · 60 unmodified lines
689	717	return &bpf_snprintf_btf_proto;
690	718	case BPF_FUNC_jiffies64:
691	719	return &bpf_jiffies64_proto;
	720	+ case BPF_FUNC_bpf_per_cpu_ptr:
	721	+ return &bpf_per_cpu_ptr_proto;
	722	+ case BPF_FUNC_bpf_this_cpu_ptr:
	723	+ return &bpf_this_cpu_ptr_proto;
692	724	default:
693	725	break;
694	726	}

kernel/bpf/percpu_freelist.c+96−5

@@ -17,6 +17,8 @@ int pcpu_freelist_init(struct pcpu_freelist *s) · 16 unmodified lines
17	17	raw_spin_lock_init(&head->lock);
18	18	head->first = NULL;
19	19	}
	20	+ raw_spin_lock_init(&s->extralist.lock);
	21	+ s->extralist.first = NULL;
20	22	return 0;
21	23	}
22	24
@@ -40,12 +42,50 @@ static inline void ___pcpu_freelist_push(struct pcpu_freelist_head *head, · 17 unmodified lines
40	42	raw_spin_unlock(&head->lock);
41	43	}
42	44
	45	+static inline bool pcpu_freelist_try_push_extra(struct pcpu_freelist *s,
	46	+ struct pcpu_freelist_node *node)
	47	+{
	48	+ if (!raw_spin_trylock(&s->extralist.lock))
	49	+ return false;
	50	+
	51	+ pcpu_freelist_push_node(&s->extralist, node);
	52	+ raw_spin_unlock(&s->extralist.lock);
	53	+ return true;
	54	+}
	55	+
	56	+static inline void ___pcpu_freelist_push_nmi(struct pcpu_freelist *s,
	57	+ struct pcpu_freelist_node *node)
	58	+{
	59	+ int cpu, orig_cpu;
	60	+
	61	+ orig_cpu = cpu = raw_smp_processor_id();
	62	+ while (1) {
	63	+ struct pcpu_freelist_head *head;
	64	+
	65	+ head = per_cpu_ptr(s->freelist, cpu);
	66	+ if (raw_spin_trylock(&head->lock)) {
	67	+ pcpu_freelist_push_node(head, node);
	68	+ raw_spin_unlock(&head->lock);
	69	+ return;
	70	+ }
	71	+ cpu = cpumask_next(cpu, cpu_possible_mask);
	72	+ if (cpu >= nr_cpu_ids)
	73	+ cpu = 0;
	74	+
	75	+ /* cannot lock any per cpu lock, try extralist */
	76	+ if (cpu == orig_cpu &&
	77	+ pcpu_freelist_try_push_extra(s, node))
	78	+ return;
	79	+ }
	80	+}
	81	+
43	82	void __pcpu_freelist_push(struct pcpu_freelist *s,
44	83	struct pcpu_freelist_node *node)
45	84	{
46		− struct pcpu_freelist_head *head = this_cpu_ptr(s->freelist);
47		−
48		− ___pcpu_freelist_push(head, node);
	85	+ if (in_nmi())
	86	+ ___pcpu_freelist_push_nmi(s, node);
	87	+ else
	88	+ ___pcpu_freelist_push(this_cpu_ptr(s->freelist), node);
49	89	}
50	90
51	91	void pcpu_freelist_push(struct pcpu_freelist *s,
@@ -81,7 +121,7 @@ again: · 29 unmodified lines
81	121	}
82	122	}
83	123
84		−struct pcpu_freelist_node __pcpu_freelist_pop(struct pcpu_freelist s)
	124	+static struct pcpu_freelist_node ___pcpu_freelist_pop(struct pcpu_freelist s)
85	125	{
86	126	struct pcpu_freelist_head *head;
87	127	struct pcpu_freelist_node *node;
@@ -102,8 +142,59 @@ struct pcpu_freelist_node __pcpu_freelist_pop(struct pcpu_freelist s) · 14 unmodified lines
102	142	if (cpu >= nr_cpu_ids)
103	143	cpu = 0;
104	144	if (cpu == orig_cpu)
105		− return NULL;
	145	+ break;
	146	+ }
	147	+
	148	+ /* per cpu lists are all empty, try extralist */
	149	+ raw_spin_lock(&s->extralist.lock);
	150	+ node = s->extralist.first;
	151	+ if (node)
	152	+ s->extralist.first = node->next;
	153	+ raw_spin_unlock(&s->extralist.lock);
	154	+ return node;
	155	+}
	156	+
	157	+static struct pcpu_freelist_node *
	158	+___pcpu_freelist_pop_nmi(struct pcpu_freelist *s)
	159	+{
	160	+ struct pcpu_freelist_head *head;
	161	+ struct pcpu_freelist_node *node;
	162	+ int orig_cpu, cpu;
	163	+
	164	+ orig_cpu = cpu = raw_smp_processor_id();
	165	+ while (1) {
	166	+ head = per_cpu_ptr(s->freelist, cpu);
	167	+ if (raw_spin_trylock(&head->lock)) {
	168	+ node = head->first;
	169	+ if (node) {
	170	+ head->first = node->next;
	171	+ raw_spin_unlock(&head->lock);
	172	+ return node;
	173	+ }
	174	+ raw_spin_unlock(&head->lock);
	175	+ }
	176	+ cpu = cpumask_next(cpu, cpu_possible_mask);
	177	+ if (cpu >= nr_cpu_ids)
	178	+ cpu = 0;
	179	+ if (cpu == orig_cpu)
	180	+ break;
106	181	}
	182	+
	183	+ /* cannot pop from per cpu lists, try extralist */
	184	+ if (!raw_spin_trylock(&s->extralist.lock))
	185	+ return NULL;
	186	+ node = s->extralist.first;
	187	+ if (node)
	188	+ s->extralist.first = node->next;
	189	+ raw_spin_unlock(&s->extralist.lock);
	190	+ return node;
	191	+}
	192	+
	193	+struct pcpu_freelist_node __pcpu_freelist_pop(struct pcpu_freelist s)
	194	+{
	195	+ if (in_nmi())
	196	+ return ___pcpu_freelist_pop_nmi(s);
	197	+ return ___pcpu_freelist_pop(s);
107	198	}
108	199
109	200	struct pcpu_freelist_node pcpu_freelist_pop(struct pcpu_freelist s)

kernel/bpf/percpu_freelist.h+1−0

@@ -13,6 +13,7 @@ struct pcpu_freelist_head { · 12 unmodified lines
13	13
14	14	struct pcpu_freelist {
15	15	struct pcpu_freelist_head __percpu *freelist;
	16	+ struct pcpu_freelist_head extralist;
16	17	};
17	18
18	19	struct pcpu_freelist_node {

kernel/bpf/syscall.c+3−1

@@ -4323,8 +4323,10 @@ static int bpf_prog_bind_map(union bpf_attr *attr) · 4322 unmodified lines
4323	4323	used_maps_old = prog->aux->used_maps;
4324	4324
4325	4325	for (i = 0; i < prog->aux->used_map_cnt; i++)
4326		− if (used_maps_old[i] == map)
	4326	+ if (used_maps_old[i] == map) {
	4327	+ bpf_map_put(map);
4327	4328	goto out_unlock;
	4329	+ }
4328	4330
4329	4331	used_maps_new = kmalloc_array(prog->aux->used_map_cnt + 1,
4330	4332	sizeof(used_maps_new[0]),

kernel/bpf/verifier.c+251−19

@@ -238,6 +238,8 @@ struct bpf_call_arg_meta { · 237 unmodified lines
238	238	u64 msize_max_value;
239	239	int ref_obj_id;
240	240	int func_id;
	241	+ u32 btf_id;
	242	+ u32 ret_btf_id;
241	243	};
242	244
243	245	struct btf *btf_vmlinux;
@@ -517,6 +519,7 @@ static const char * const reg_type_str[] = { · 273 unmodified lines
517	519	[PTR_TO_XDP_SOCK] = "xdp_sock",
518	520	[PTR_TO_BTF_ID] = "ptr_",
519	521	[PTR_TO_BTF_ID_OR_NULL] = "ptr_or_null_",
	522	+ [PTR_TO_PERCPU_BTF_ID] = "percpu_ptr_",
520	523	[PTR_TO_MEM] = "mem",
521	524	[PTR_TO_MEM_OR_NULL] = "mem_or_null",
522	525	[PTR_TO_RDONLY_BUF] = "rdonly_buf",
@@ -583,7 +586,9 @@ static void print_verifier_state(struct bpf_verifier_env *env, · 60 unmodified lines
583	586	/* reg->off should be 0 for SCALAR_VALUE */
584	587	verbose(env, "%lld", reg->var_off.value + reg->off);
585	588	} else {
586		− if (t == PTR_TO_BTF_ID \|\| t == PTR_TO_BTF_ID_OR_NULL)
	589	+ if (t == PTR_TO_BTF_ID \|\|
	590	+ t == PTR_TO_BTF_ID_OR_NULL \|\|
	591	+ t == PTR_TO_PERCPU_BTF_ID)
587	592	verbose(env, "%s", kernel_type_name(reg->btf_id));
588	593	verbose(env, "(id=%d", reg->id);
589	594	if (reg_type_may_be_refcounted_or_null(t))
@@ -2204,6 +2209,7 @@ static bool is_spillable_regtype(enum bpf_reg_type type) · 1614 unmodified lines
2204	2209	case PTR_TO_RDONLY_BUF_OR_NULL:
2205	2210	case PTR_TO_RDWR_BUF:
2206	2211	case PTR_TO_RDWR_BUF_OR_NULL:
	2212	+ case PTR_TO_PERCPU_BTF_ID:
2207	2213	return true;
2208	2214	default:
2209	2215	return false;
@@ -2221,6 +2227,20 @@ static bool register_is_const(struct bpf_reg_state *reg) · 11 unmodified lines
2221	2227	return reg->type == SCALAR_VALUE && tnum_is_const(reg->var_off);
2222	2228	}
2223	2229
	2230	+static bool __is_scalar_unbounded(struct bpf_reg_state *reg)
	2231	+{
	2232	+ return tnum_is_unknown(reg->var_off) &&
	2233	+ reg->smin_value == S64_MIN && reg->smax_value == S64_MAX &&
	2234	+ reg->umin_value == 0 && reg->umax_value == U64_MAX &&
	2235	+ reg->s32_min_value == S32_MIN && reg->s32_max_value == S32_MAX &&
	2236	+ reg->u32_min_value == 0 && reg->u32_max_value == U32_MAX;
	2237	+}
	2238	+
	2239	+static bool register_is_bounded(struct bpf_reg_state *reg)
	2240	+{
	2241	+ return reg->type == SCALAR_VALUE && !__is_scalar_unbounded(reg);
	2242	+}
	2243	+
2224	2244	static bool __is_pointer_value(bool allow_ptr_leaks,
2225	2245	const struct bpf_reg_state *reg)
2226	2246	{
@@ -2272,7 +2292,7 @@ static int check_stack_write(struct bpf_verifier_env *env, · 45 unmodified lines
2272	2292	if (value_regno >= 0)
2273	2293	reg = &cur->regs[value_regno];
2274	2294
2275		− if (reg && size == BPF_REG_SIZE && register_is_const(reg) &&
	2295	+ if (reg && size == BPF_REG_SIZE && register_is_bounded(reg) &&
2276	2296	!register_is_null(reg) && env->bpf_capable) {
2277	2297	if (dst_reg != BPF_REG_FP) {
2278	2298	/* The backtracking logic can only recognize explicit
@@ -2667,7 +2687,7 @@ static bool may_access_direct_pkt_data(struct bpf_verifier_env *env, · 388 unmodified lines
2667	2687	case BPF_PROG_TYPE_CGROUP_SKB:
2668	2688	if (t == BPF_WRITE)
2669	2689	return false;
2670		− /* fallthrough */
	2690	+ fallthrough;
2671	2691
2672	2692	/* Program types with direct read + write access go here! */
2673	2693	case BPF_PROG_TYPE_SCHED_CLS:
@@ -3978,6 +3998,7 @@ static const struct bpf_reg_types sock_types = { · 1304 unmodified lines
3978	3998	},
3979	3999	};
3980	4000
	4001	+#ifdef CONFIG_NET
3981	4002	static const struct bpf_reg_types btf_id_sock_common_types = {
3982	4003	.types = {
3983	4004	PTR_TO_SOCK_COMMON,
@@ -3988,6 +4009,7 @@ static const struct bpf_reg_types btf_id_sock_common_types = { · 4 unmodified lines
3988	4009	},
3989	4010	.btf_id = &btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON],
3990	4011	};
	4012	+#endif
3991	4013
3992	4014	static const struct bpf_reg_types mem_types = {
3993	4015	.types = {
@@ -4017,6 +4039,7 @@ static const struct bpf_reg_types alloc_mem_types = { .types = { PTR_TO_MEM } }; · 23 unmodified lines
4017	4039	static const struct bpf_reg_types const_map_ptr_types = { .types = { CONST_PTR_TO_MAP } };
4018	4040	static const struct bpf_reg_types btf_ptr_types = { .types = { PTR_TO_BTF_ID } };
4019	4041	static const struct bpf_reg_types spin_lock_types = { .types = { PTR_TO_MAP_VALUE } };
	4042	+static const struct bpf_reg_types percpu_btf_ptr_types = { .types = { PTR_TO_PERCPU_BTF_ID } };
4020	4043
4021	4044	static const struct bpf_reg_types *compatible_reg_types[__BPF_ARG_TYPE_MAX] = {
4022	4045	[ARG_PTR_TO_MAP_KEY] = &map_key_value_types,
@@ -4030,7 +4053,9 @@ static const struct bpf_reg_types *compatible_reg_types[__BPF_ARG_TYPE_MAX] = { · 7 unmodified lines
4030	4053	[ARG_PTR_TO_CTX] = &context_types,
4031	4054	[ARG_PTR_TO_CTX_OR_NULL] = &context_types,
4032	4055	[ARG_PTR_TO_SOCK_COMMON] = &sock_types,
	4056	+#ifdef CONFIG_NET
4033	4057	[ARG_PTR_TO_BTF_ID_SOCK_COMMON] = &btf_id_sock_common_types,
	4058	+#endif
4034	4059	[ARG_PTR_TO_SOCKET] = &fullsock_types,
4035	4060	[ARG_PTR_TO_SOCKET_OR_NULL] = &fullsock_types,
4036	4061	[ARG_PTR_TO_BTF_ID] = &btf_ptr_types,
@@ -4042,6 +4067,7 @@ static const struct bpf_reg_types *compatible_reg_types[__BPF_ARG_TYPE_MAX] = { · 5 unmodified lines
4042	4067	[ARG_PTR_TO_ALLOC_MEM_OR_NULL] = &alloc_mem_types,
4043	4068	[ARG_PTR_TO_INT] = &int_ptr_types,
4044	4069	[ARG_PTR_TO_LONG] = &int_ptr_types,
	4070	+ [ARG_PTR_TO_PERCPU_BTF_ID] = &percpu_btf_ptr_types,
4045	4071	};
4046	4072
4047	4073	static int check_reg_type(struct bpf_verifier_env *env, u32 regno,
@@ -4205,6 +4231,12 @@ skip_type_check: · 157 unmodified lines
4205	4231	err = check_helper_mem_access(env, regno,
4206	4232	meta->map_ptr->value_size, false,
4207	4233	meta);
	4234	+ } else if (arg_type == ARG_PTR_TO_PERCPU_BTF_ID) {
	4235	+ if (!reg->btf_id) {
	4236	+ verbose(env, "Helper has invalid btf_id in R%d\n", regno);
	4237	+ return -EACCES;
	4238	+ }
	4239	+ meta->ret_btf_id = reg->btf_id;
4208	4240	} else if (arg_type == ARG_PTR_TO_SPIN_LOCK) {
4209	4241	if (meta->func_id == BPF_FUNC_spin_lock) {
4210	4242	if (process_spin_lock(env, regno, true))
@@ -5114,6 +5146,35 @@ static int check_helper_call(struct bpf_verifier_env *env, int func_id, int insn · 903 unmodified lines
5114	5146	regs[BPF_REG_0].type = PTR_TO_MEM_OR_NULL;
5115	5147	regs[BPF_REG_0].id = ++env->id_gen;
5116	5148	regs[BPF_REG_0].mem_size = meta.mem_size;
	5149	+ } else if (fn->ret_type == RET_PTR_TO_MEM_OR_BTF_ID_OR_NULL \|\|
	5150	+ fn->ret_type == RET_PTR_TO_MEM_OR_BTF_ID) {
	5151	+ const struct btf_type *t;
	5152	+
	5153	+ mark_reg_known_zero(env, regs, BPF_REG_0);
	5154	+ t = btf_type_skip_modifiers(btf_vmlinux, meta.ret_btf_id, NULL);
	5155	+ if (!btf_type_is_struct(t)) {
	5156	+ u32 tsize;
	5157	+ const struct btf_type *ret;
	5158	+ const char *tname;
	5159	+
	5160	+ /* resolve the type size of ksym. */
	5161	+ ret = btf_resolve_size(btf_vmlinux, t, &tsize);
	5162	+ if (IS_ERR(ret)) {
	5163	+ tname = btf_name_by_offset(btf_vmlinux, t->name_off);
	5164	+ verbose(env, "unable to resolve the size of type '%s': %ld\n",
	5165	+ tname, PTR_ERR(ret));
	5166	+ return -EINVAL;
	5167	+ }
	5168	+ regs[BPF_REG_0].type =
	5169	+ fn->ret_type == RET_PTR_TO_MEM_OR_BTF_ID ?
	5170	+ PTR_TO_MEM : PTR_TO_MEM_OR_NULL;
	5171	+ regs[BPF_REG_0].mem_size = tsize;
	5172	+ } else {
	5173	+ regs[BPF_REG_0].type =
	5174	+ fn->ret_type == RET_PTR_TO_MEM_OR_BTF_ID ?
	5175	+ PTR_TO_BTF_ID : PTR_TO_BTF_ID_OR_NULL;
	5176	+ regs[BPF_REG_0].btf_id = meta.ret_btf_id;
	5177	+ }
5117	5178	} else if (fn->ret_type == RET_PTR_TO_BTF_ID_OR_NULL) {
5118	5179	int ret_btf_id;
5119	5180
@@ -5432,7 +5493,7 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, · 312 unmodified lines
5432	5493	/* smin_val represents the known value */
5433	5494	if (known && smin_val == 0 && opcode == BPF_ADD)
5434	5495	break;
5435		− /* fall-through */
	5496	+ fallthrough;
5436	5497	case PTR_TO_PACKET_END:
5437	5498	case PTR_TO_SOCKET:
5438	5499	case PTR_TO_SOCKET_OR_NULL:
@@ -6389,6 +6450,11 @@ static int adjust_reg_min_max_vals(struct bpf_verifier_env *env, · 950 unmodified lines
6389	6450	src_reg = NULL;
6390	6451	if (dst_reg->type != SCALAR_VALUE)
6391	6452	ptr_reg = dst_reg;
	6453	+ else
	6454	+ /* Make sure ID is cleared otherwise dst_reg min/max could be
	6455	+ * incorrectly propagated into other registers by find_equal_scalars()
	6456	+ */
	6457	+ dst_reg->id = 0;
6392	6458	if (BPF_SRC(insn->code) == BPF_X) {
6393	6459	src_reg = &regs[insn->src_reg];
6394	6460	if (src_reg->type != SCALAR_VALUE) {
@@ -6522,6 +6588,12 @@ static int check_alu_op(struct bpf_verifier_env env, struct bpf_insn insn) · 127 unmodified lines
6522	6588	/* case: R1 = R2
6523	6589	* copy register state to dest reg
6524	6590	*/
	6591	+ if (src_reg->type == SCALAR_VALUE && !src_reg->id)
	6592	+ /* Assign src and dst registers the same ID
	6593	+ * that will be used by find_equal_scalars()
	6594	+ * to propagate min/max range.
	6595	+ */
	6596	+ src_reg->id = ++env->id_gen;
6525	6597	dst_reg = src_reg;
6526	6598	dst_reg->live \|= REG_LIVE_WRITTEN;
6527	6599	dst_reg->subreg_def = DEF_NOT_SUBREG;
@@ -6534,6 +6606,11 @@ static int check_alu_op(struct bpf_verifier_env env, struct bpf_insn insn) · 6 unmodified lines
6534	6606	return -EACCES;
6535	6607	} else if (src_reg->type == SCALAR_VALUE) {
6536	6608	dst_reg = src_reg;
	6609	+ /* Make sure ID is cleared otherwise
	6610	+ * dst_reg min/max could be incorrectly
	6611	+ * propagated into src_reg by find_equal_scalars()
	6612	+ */
	6613	+ dst_reg->id = 0;
6537	6614	dst_reg->live \|= REG_LIVE_WRITTEN;
6538	6615	dst_reg->subreg_def = env->insn_idx + 1;
6539	6616	} else {
@@ -7322,6 +7399,30 @@ static bool try_match_pkt_pointers(const struct bpf_insn *insn, · 782 unmodified lines
7322	7399	return true;
7323	7400	}
7324	7401
	7402	+static void find_equal_scalars(struct bpf_verifier_state *vstate,
	7403	+ struct bpf_reg_state *known_reg)
	7404	+{
	7405	+ struct bpf_func_state *state;
	7406	+ struct bpf_reg_state *reg;
	7407	+ int i, j;
	7408	+
	7409	+ for (i = 0; i <= vstate->curframe; i++) {
	7410	+ state = vstate->frame[i];
	7411	+ for (j = 0; j < MAX_BPF_REG; j++) {
	7412	+ reg = &state->regs[j];
	7413	+ if (reg->type == SCALAR_VALUE && reg->id == known_reg->id)
	7414	+ reg = known_reg;
	7415	+ }
	7416	+
	7417	+ bpf_for_each_spilled_reg(j, state, reg) {
	7418	+ if (!reg)
	7419	+ continue;
	7420	+ if (reg->type == SCALAR_VALUE && reg->id == known_reg->id)
	7421	+ reg = known_reg;
	7422	+ }
	7423	+ }
	7424	+}
	7425	+
7325	7426	static int check_cond_jmp_op(struct bpf_verifier_env *env,
7326	7427	struct bpf_insn insn, int insn_idx)
7327	7428	{
@@ -7450,6 +7551,11 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env, · 122 unmodified lines
7450	7551	reg_combine_min_max(&other_branch_regs[insn->src_reg],
7451	7552	&other_branch_regs[insn->dst_reg],
7452	7553	src_reg, dst_reg, opcode);
	7554	+ if (src_reg->id) {
	7555	+ find_equal_scalars(this_branch, src_reg);
	7556	+ find_equal_scalars(other_branch, &other_branch_regs[insn->src_reg]);
	7557	+ }
	7558	+
7453	7559	}
7454	7560	} else if (dst_reg->type == SCALAR_VALUE) {
7455	7561	reg_set_min_max(&other_branch_regs[insn->dst_reg],
@@ -7457,6 +7563,11 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env, · 1 unmodified line
7457	7563	opcode, is_jmp32);
7458	7564	}
7459	7565
	7566	+ if (dst_reg->type == SCALAR_VALUE && dst_reg->id) {
	7567	+ find_equal_scalars(this_branch, dst_reg);
	7568	+ find_equal_scalars(other_branch, &other_branch_regs[insn->dst_reg]);
	7569	+ }
	7570	+
7460	7571	/* detect if R == 0 where R is returned from bpf_map_lookup_elem().
7461	7572	* NOTE: these optimizations below are related with pointer comparison
7462	7573	* which will never be JMP32.
@@ -7488,6 +7599,7 @@ static int check_ld_imm(struct bpf_verifier_env env, struct bpf_insn insn) · 25 unmodified lines
7488	7599	{
7489	7600	struct bpf_insn_aux_data *aux = cur_aux(env);
7490	7601	struct bpf_reg_state *regs = cur_regs(env);
	7602	+ struct bpf_reg_state *dst_reg;
7491	7603	struct bpf_map *map;
7492	7604	int err;
7493	7605
@@ -7504,25 +7616,45 @@ static int check_ld_imm(struct bpf_verifier_env env, struct bpf_insn insn) · 10 unmodified lines
7504	7616	if (err)
7505	7617	return err;
7506	7618
	7619	+ dst_reg = &regs[insn->dst_reg];
7507	7620	if (insn->src_reg == 0) {
7508	7621	u64 imm = ((u64)(insn + 1)->imm << 32) \| (u32)insn->imm;
7509	7622
7510		− regs[insn->dst_reg].type = SCALAR_VALUE;
	7623	+ dst_reg->type = SCALAR_VALUE;
7511	7624	__mark_reg_known(&regs[insn->dst_reg], imm);
7512	7625	return 0;
7513	7626	}
7514	7627
	7628	+ if (insn->src_reg == BPF_PSEUDO_BTF_ID) {
	7629	+ mark_reg_known_zero(env, regs, insn->dst_reg);
	7630	+
	7631	+ dst_reg->type = aux->btf_var.reg_type;
	7632	+ switch (dst_reg->type) {
	7633	+ case PTR_TO_MEM:
	7634	+ dst_reg->mem_size = aux->btf_var.mem_size;
	7635	+ break;
	7636	+ case PTR_TO_BTF_ID:
	7637	+ case PTR_TO_PERCPU_BTF_ID:
	7638	+ dst_reg->btf_id = aux->btf_var.btf_id;
	7639	+ break;
	7640	+ default:
	7641	+ verbose(env, "bpf verifier is misconfigured\n");
	7642	+ return -EFAULT;
	7643	+ }
	7644	+ return 0;
	7645	+ }
	7646	+
7515	7647	map = env->used_maps[aux->map_index];
7516	7648	mark_reg_known_zero(env, regs, insn->dst_reg);
7517		− regs[insn->dst_reg].map_ptr = map;
	7649	+ dst_reg->map_ptr = map;
7518	7650
7519	7651	if (insn->src_reg == BPF_PSEUDO_MAP_VALUE) {
7520		− regs[insn->dst_reg].type = PTR_TO_MAP_VALUE;
7521		− regs[insn->dst_reg].off = aux->map_off;
	7652	+ dst_reg->type = PTR_TO_MAP_VALUE;
	7653	+ dst_reg->off = aux->map_off;
7522	7654	if (map_value_has_spin_lock(map))
7523		− regs[insn->dst_reg].id = ++env->id_gen;
	7655	+ dst_reg->id = ++env->id_gen;
7524	7656	} else if (insn->src_reg == BPF_PSEUDO_MAP_FD) {
7525		− regs[insn->dst_reg].type = CONST_PTR_TO_MAP;
	7657	+ dst_reg->type = CONST_PTR_TO_MAP;
7526	7658	} else {
7527	7659	verbose(env, "bpf verifier is misconfigured\n");
7528	7660	return -EINVAL;
@@ -9424,6 +9556,92 @@ process_bpf_exit: · 1895 unmodified lines
9424	9556	return 0;
9425	9557	}
9426	9558
	9559	+/* replace pseudo btf_id with kernel symbol address */
	9560	+static int check_pseudo_btf_id(struct bpf_verifier_env *env,
	9561	+ struct bpf_insn *insn,
	9562	+ struct bpf_insn_aux_data *aux)
	9563	+{
	9564	+ u32 datasec_id, type, id = insn->imm;
	9565	+ const struct btf_var_secinfo *vsi;
	9566	+ const struct btf_type *datasec;
	9567	+ const struct btf_type *t;
	9568	+ const char *sym_name;
	9569	+ bool percpu = false;
	9570	+ u64 addr;
	9571	+ int i;
	9572	+
	9573	+ if (!btf_vmlinux) {
	9574	+ verbose(env, "kernel is missing BTF, make sure CONFIG_DEBUG_INFO_BTF=y is specified in Kconfig.\n");
	9575	+ return -EINVAL;
	9576	+ }
	9577	+
	9578	+ if (insn[1].imm != 0) {
	9579	+ verbose(env, "reserved field (insn[1].imm) is used in pseudo_btf_id ldimm64 insn.\n");
	9580	+ return -EINVAL;
	9581	+ }
	9582	+
	9583	+ t = btf_type_by_id(btf_vmlinux, id);
	9584	+ if (!t) {
	9585	+ verbose(env, "ldimm64 insn specifies invalid btf_id %d.\n", id);
	9586	+ return -ENOENT;
	9587	+ }
	9588	+
	9589	+ if (!btf_type_is_var(t)) {
	9590	+ verbose(env, "pseudo btf_id %d in ldimm64 isn't KIND_VAR.\n",
	9591	+ id);
	9592	+ return -EINVAL;
	9593	+ }
	9594	+
	9595	+ sym_name = btf_name_by_offset(btf_vmlinux, t->name_off);
	9596	+ addr = kallsyms_lookup_name(sym_name);
	9597	+ if (!addr) {
	9598	+ verbose(env, "ldimm64 failed to find the address for kernel symbol '%s'.\n",
	9599	+ sym_name);
	9600	+ return -ENOENT;
	9601	+ }
	9602	+
	9603	+ datasec_id = btf_find_by_name_kind(btf_vmlinux, ".data..percpu",
	9604	+ BTF_KIND_DATASEC);
	9605	+ if (datasec_id > 0) {
	9606	+ datasec = btf_type_by_id(btf_vmlinux, datasec_id);
	9607	+ for_each_vsi(i, datasec, vsi) {
	9608	+ if (vsi->type == id) {
	9609	+ percpu = true;
	9610	+ break;
	9611	+ }
	9612	+ }
	9613	+ }
	9614	+
	9615	+ insn[0].imm = (u32)addr;
	9616	+ insn[1].imm = addr >> 32;
	9617	+
	9618	+ type = t->type;
	9619	+ t = btf_type_skip_modifiers(btf_vmlinux, type, NULL);
	9620	+ if (percpu) {
	9621	+ aux->btf_var.reg_type = PTR_TO_PERCPU_BTF_ID;
	9622	+ aux->btf_var.btf_id = type;
	9623	+ } else if (!btf_type_is_struct(t)) {
	9624	+ const struct btf_type *ret;
	9625	+ const char *tname;
	9626	+ u32 tsize;
	9627	+
	9628	+ /* resolve the type size of ksym. */
	9629	+ ret = btf_resolve_size(btf_vmlinux, t, &tsize);
	9630	+ if (IS_ERR(ret)) {
	9631	+ tname = btf_name_by_offset(btf_vmlinux, t->name_off);
	9632	+ verbose(env, "ldimm64 unable to resolve the size of type '%s': %ld\n",
	9633	+ tname, PTR_ERR(ret));
	9634	+ return -EINVAL;
	9635	+ }
	9636	+ aux->btf_var.reg_type = PTR_TO_MEM;
	9637	+ aux->btf_var.mem_size = tsize;
	9638	+ } else {
	9639	+ aux->btf_var.reg_type = PTR_TO_BTF_ID;
	9640	+ aux->btf_var.btf_id = type;
	9641	+ }
	9642	+ return 0;
	9643	+}
	9644	+
9427	9645	static int check_map_prealloc(struct bpf_map *map)
9428	9646	{
9429	9647	return (map->map_type != BPF_MAP_TYPE_HASH &&
@@ -9534,10 +9752,14 @@ static bool bpf_map_is_cgroup_storage(struct bpf_map *map) · 104 unmodified lines
9534	9752	map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE);
9535	9753	}
9536	9754
9537		−/* look for pseudo eBPF instructions that access map FDs and
9538		− * replace them with actual map pointers
	9755	+/* find and rewrite pseudo imm in ld_imm64 instructions:
	9756	+ *
	9757	+ * 1. if it accesses map FD, replace it with actual map pointer.
	9758	+ * 2. if it accesses btf_id of a VAR, replace it with pointer to the var.
	9759	+ *
	9760	+ * NOTE: btf_vmlinux is required for converting pseudo btf_id.
9539	9761	*/
9540		−static int replace_map_fd_with_map_ptr(struct bpf_verifier_env *env)
	9762	+static int resolve_pseudo_ldimm64(struct bpf_verifier_env *env)
9541	9763	{
9542	9764	struct bpf_insn *insn = env->prog->insnsi;
9543	9765	int insn_cnt = env->prog->len;
@@ -9578,6 +9800,14 @@ static int replace_map_fd_with_map_ptr(struct bpf_verifier_env *env) · 34 unmodified lines
9578	9800	/* valid generic load 64-bit imm */
9579	9801	goto next_insn;
9580	9802
	9803	+ if (insn[0].src_reg == BPF_PSEUDO_BTF_ID) {
	9804	+ aux = &env->insn_aux_data[i];
	9805	+ err = check_pseudo_btf_id(env, insn, aux);
	9806	+ if (err)
	9807	+ return err;
	9808	+ goto next_insn;
	9809	+ }
	9810	+
9581	9811	/* In final convert_pseudo_ld_imm64() step, this is
9582	9812	* converted into regular 64-bit imm load insn.
9583	9813	*/
@@ -10819,7 +11049,9 @@ static int fixup_bpf_calls(struct bpf_verifier_env *env) · 1235 unmodified lines
10819	11049	if (insn->imm == BPF_FUNC_map_lookup_elem &&
10820	11050	ops->map_gen_lookup) {
10821	11051	cnt = ops->map_gen_lookup(map_ptr, insn_buf);
10822		− if (cnt == 0 \|\| cnt >= ARRAY_SIZE(insn_buf)) {
	11052	+ if (cnt == -EOPNOTSUPP)
	11053	+ goto patch_map_ops_generic;
	11054	+ if (cnt <= 0 \|\| cnt >= ARRAY_SIZE(insn_buf)) {
10823	11055	verbose(env, "bpf verifier is misconfigured\n");
10824	11056	return -EINVAL;
10825	11057	}
@@ -10849,7 +11081,7 @@ static int fixup_bpf_calls(struct bpf_verifier_env *env) · 23 unmodified lines
10849	11081	(int ()(struct bpf_map map, void *value))NULL));
10850	11082	BUILD_BUG_ON(!__same_type(ops->map_peek_elem,
10851	11083	(int ()(struct bpf_map map, void *value))NULL));
10852		−
	11084	+patch_map_ops_generic:
10853	11085	switch (insn->imm) {
10854	11086	case BPF_FUNC_map_lookup_elem:
10855	11087	insn->imm = BPF_CAST_CALL(ops->map_lookup_elem) -
@@ -11633,10 +11865,6 @@ int bpf_check(struct bpf_prog *prog, union bpf_attr attr, · 777 unmodified lines
11633	11865	if (is_priv)
11634	11866	env->test_state_freq = attr->prog_flags & BPF_F_TEST_STATE_FREQ;
11635	11867
11636		− ret = replace_map_fd_with_map_ptr(env);
11637		− if (ret < 0)
11638		− goto skip_full_check;
11639		−
11640	11868	if (bpf_prog_is_dev_bound(env->prog->aux)) {
11641	11869	ret = bpf_prog_offload_verifier_prep(env->prog);
11642	11870	if (ret)
@@ -11662,6 +11890,10 @@ int bpf_check(struct bpf_prog *prog, union bpf_attr attr, · 19 unmodified lines
11662	11890	if (ret)
11663	11891	goto skip_full_check;
11664	11892
	11893	+ ret = resolve_pseudo_ldimm64(env);
	11894	+ if (ret < 0)
	11895	+ goto skip_full_check;
	11896	+
11665	11897	ret = check_cfg(env);
11666	11898	if (ret < 0)
11667	11899	goto skip_full_check;

kernel/trace/bpf_trace.c+6−0

@@ -1327,6 +1327,10 @@ bpf_tracing_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) · 1326 unmodified lines
1327	1327	return prog->aux->sleepable ? &bpf_copy_from_user_proto : NULL;
1328	1328	case BPF_FUNC_snprintf_btf:
1329	1329	return &bpf_snprintf_btf_proto;
	1330	+ case BPF_FUNC_bpf_per_cpu_ptr:
	1331	+ return &bpf_per_cpu_ptr_proto;
	1332	+ case BPF_FUNC_bpf_this_cpu_ptr:
	1333	+ return &bpf_this_cpu_ptr_proto;
1330	1334	default:
1331	1335	return NULL;
1332	1336	}
@@ -1776,7 +1780,9 @@ const struct bpf_verifier_ops raw_tracepoint_verifier_ops = { · 443 unmodified lines
1776	1780	};
1777	1781
1778	1782	const struct bpf_prog_ops raw_tracepoint_prog_ops = {
	1783	+#ifdef CONFIG_NET
1779	1784	.test_run = bpf_prog_test_run_raw_tp,
	1785	+#endif
1780	1786	};
1781	1787
1782	1788	const struct bpf_verifier_ops tracing_verifier_ops = {

Merge git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code