Merge git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next

Alexei Starovoitov says: ==================== pull-request: bpf-next 2020-10-12 The main changes are: 1) The BPF verifier improvements to track register allocation pattern, from Alexei and Yonghong. 2) libbpf relocation support for different size load/store, from Andrii. 3) bpf_redirect_peer() helper and support for inner map array with different max_entries, from Daniel. 4) BPF support for per-cpu variables, form Hao. 5) sockmap improvements, from John. ====================

Jakub Kicinski committed 6 years ago

Commit ccdf7fa2 parents a308283 376dcfe73 files · +4339 −772Browse files

Showing the changes to kernel (9 of 73 files). Show all

Unified · Split

9 files

kernel/bpf/arraymap.c +11 −6
kernel/bpf/btf.c +0 −25
kernel/bpf/hashtab.c +3 −3
kernel/bpf/helpers.c +32 −0
kernel/bpf/percpu_freelist.c +96 −5
kernel/bpf/percpu_freelist.h +1 −0
kernel/bpf/syscall.c +3 −1
kernel/bpf/verifier.c +251 −19
kernel/trace/bpf_trace.c +6 −0

kernel/bpf/arraymap.c+11−6

 #define ARRAY_CREATE_FLAG_MASK \
 	(BPF_F_NUMA_NODE | BPF_F_MMAPABLE | BPF_F_ACCESS_MASK | \
-	 BPF_F_PRESERVE_ELEMS)
+	 BPF_F_PRESERVE_ELEMS | BPF_F_INNER_MAP)
 static void bpf_array_free_percpu(struct bpf_array *array)
 {
 		return -EINVAL;
 	if (attr->map_type != BPF_MAP_TYPE_ARRAY &&
-	    attr->map_flags & BPF_F_MMAPABLE)
+	    attr->map_flags & (BPF_F_MMAPABLE | BPF_F_INNER_MAP))
 		return -EINVAL;
 	if (attr->map_type != BPF_MAP_TYPE_PERF_EVENT_ARRAY &&
 }
 /* emit BPF instructions equivalent to C code of array_map_lookup_elem() */
-static u32 array_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn_buf)
+static int array_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn_buf)
 {
 	struct bpf_array *array = container_of(map, struct bpf_array, map);
 	struct bpf_insn *insn = insn_buf;
 	const int map_ptr = BPF_REG_1;
 	const int index = BPF_REG_2;
+	if (map->map_flags & BPF_F_INNER_MAP)
+		return -EOPNOTSUPP;
 	*insn++ = BPF_ALU64_IMM(BPF_ADD, map_ptr, offsetof(struct bpf_array, value));
 	*insn++ = BPF_LDX_MEM(BPF_W, ret, index, 0);
 	if (!map->bypass_spec_v1) {
 static bool array_map_meta_equal(const struct bpf_map *meta0,
 				 const struct bpf_map *meta1)
 {
-	return meta0->max_entries == meta1->max_entries &&
+	if (!bpf_map_meta_equal(meta0, meta1))
-		bpf_map_meta_equal(meta0, meta1);
+		return false;
+	return meta0->map_flags & BPF_F_INNER_MAP ? true :
+	       meta0->max_entries == meta1->max_entries;
 }
 struct bpf_iter_seq_array_map_info {
 	return READ_ONCE(*inner_map);
 }
-static u32 array_of_map_gen_lookup(struct bpf_map *map,
+static int array_of_map_gen_lookup(struct bpf_map *map,
 				   struct bpf_insn *insn_buf)
 {
 	struct bpf_array *array = container_of(map, struct bpf_array, map);

kernel/bpf/btf.c+0−25

@@ -188,11 +188,6 @@ · 187 unmodified lines
188	i < btf_type_vlen(struct_type); \	188	i < btf_type_vlen(struct_type); \
189	i++, member++)	189	i++, member++)
190		190
191	#define for_each_vsi(i, struct_type, member) \
192	for (i = 0, member = btf_type_var_secinfo(struct_type); \
193	i < btf_type_vlen(struct_type); \
194	i++, member++)
195
196	#define for_each_vsi_from(i, from, struct_type, member) \	191	#define for_each_vsi_from(i, from, struct_type, member) \
197	for (i = from, member = btf_type_var_secinfo(struct_type) + from; \	192	for (i = from, member = btf_type_var_secinfo(struct_type) + from; \
198	i < btf_type_vlen(struct_type); \	193	i < btf_type_vlen(struct_type); \
@@ -440,16 +435,6 @@ static bool btf_type_nosize_or_null(const struct btf_type *t) · 241 unmodified lines
440	return !t \|\| btf_type_nosize(t);	435	return !t \|\| btf_type_nosize(t);
441	}	436	}
442		437
443	/* union is only a special case of struct:
444	* all its offsetof(member) == 0
445	*/
446	static bool btf_type_is_struct(const struct btf_type *t)
447	{
448	u8 kind = BTF_INFO_KIND(t->info);
449
450	return kind == BTF_KIND_STRUCT \|\| kind == BTF_KIND_UNION;
451	}
452
453	static bool __btf_type_is_struct(const struct btf_type *t)	438	static bool __btf_type_is_struct(const struct btf_type *t)
454	{	439	{
455	return BTF_INFO_KIND(t->info) == BTF_KIND_STRUCT;	440	return BTF_INFO_KIND(t->info) == BTF_KIND_STRUCT;
@@ -460,11 +445,6 @@ static bool btf_type_is_array(const struct btf_type *t) · 4 unmodified lines
460	return BTF_INFO_KIND(t->info) == BTF_KIND_ARRAY;	445	return BTF_INFO_KIND(t->info) == BTF_KIND_ARRAY;
461	}	446	}
462		447
463	static bool btf_type_is_var(const struct btf_type *t)
464	{
465	return BTF_INFO_KIND(t->info) == BTF_KIND_VAR;
466	}
467
468	static bool btf_type_is_datasec(const struct btf_type *t)	448	static bool btf_type_is_datasec(const struct btf_type *t)
469	{	449	{
470	return BTF_INFO_KIND(t->info) == BTF_KIND_DATASEC;	450	return BTF_INFO_KIND(t->info) == BTF_KIND_DATASEC;
@@ -613,11 +593,6 @@ static const struct btf_var btf_type_var(const struct btf_type t) · 142 unmodified lines
613	return (const struct btf_var *)(t + 1);	593	return (const struct btf_var *)(t + 1);
614	}	594	}
615		595
616	static const struct btf_var_secinfo btf_type_var_secinfo(const struct btf_type t)
617	{
618	return (const struct btf_var_secinfo *)(t + 1);
619	}
620
621	static const struct btf_kind_operations btf_type_ops(const struct btf_type t)	596	static const struct btf_kind_operations btf_type_ops(const struct btf_type t)
622	{	597	{
623	return kind_ops[BTF_INFO_KIND(t->info)];	598	return kind_ops[BTF_INFO_KIND(t->info)];

kernel/bpf/hashtab.c+3−3

@@ -612,7 +612,7 @@ static void htab_map_lookup_elem(struct bpf_map map, void *key) · 611 unmodified lines
612	* bpf_prog	612	* bpf_prog
613	* __htab_map_lookup_elem	613	* __htab_map_lookup_elem
614	*/	614	*/
615	static u32 htab_map_gen_lookup(struct bpf_map map, struct bpf_insn insn_buf)	615	static int htab_map_gen_lookup(struct bpf_map map, struct bpf_insn insn_buf)
616	{	616	{
617	struct bpf_insn *insn = insn_buf;	617	struct bpf_insn *insn = insn_buf;
618	const int ret = BPF_REG_0;	618	const int ret = BPF_REG_0;
@@ -651,7 +651,7 @@ static void htab_lru_map_lookup_elem_sys(struct bpf_map map, void *key) · 32 unmodified lines
651	return __htab_lru_map_lookup_elem(map, key, false);	651	return __htab_lru_map_lookup_elem(map, key, false);
652	}	652	}
653		653
654	static u32 htab_lru_map_gen_lookup(struct bpf_map *map,	654	static int htab_lru_map_gen_lookup(struct bpf_map *map,
655	struct bpf_insn *insn_buf)	655	struct bpf_insn *insn_buf)
656	{	656	{
657	struct bpf_insn *insn = insn_buf;	657	struct bpf_insn *insn = insn_buf;
@@ -2070,7 +2070,7 @@ static void htab_of_map_lookup_elem(struct bpf_map map, void *key) · 1412 unmodified lines
2070	return READ_ONCE(*inner_map);	2070	return READ_ONCE(*inner_map);
2071	}	2071	}
2072		2072
2073	static u32 htab_of_map_gen_lookup(struct bpf_map *map,	2073	static int htab_of_map_gen_lookup(struct bpf_map *map,
2074	struct bpf_insn *insn_buf)	2074	struct bpf_insn *insn_buf)
2075	{	2075	{
2076	struct bpf_insn *insn = insn_buf;	2076	struct bpf_insn *insn = insn_buf;

kernel/bpf/helpers.c+32−0

@@ -623,6 +623,34 @@ const struct bpf_func_proto bpf_copy_from_user_proto = { · 622 unmodified lines
623	.arg3_type = ARG_ANYTHING,	623	.arg3_type = ARG_ANYTHING,
624	};	624	};
625		625
		626	BPF_CALL_2(bpf_per_cpu_ptr, const void *, ptr, u32, cpu)
		627	{
		628	if (cpu >= nr_cpu_ids)
		629	return (unsigned long)NULL;
		630
		631	return (unsigned long)per_cpu_ptr((const void __percpu *)ptr, cpu);
		632	}
		633
		634	const struct bpf_func_proto bpf_per_cpu_ptr_proto = {
		635	.func = bpf_per_cpu_ptr,
		636	.gpl_only = false,
		637	.ret_type = RET_PTR_TO_MEM_OR_BTF_ID_OR_NULL,
		638	.arg1_type = ARG_PTR_TO_PERCPU_BTF_ID,
		639	.arg2_type = ARG_ANYTHING,
		640	};
		641
		642	BPF_CALL_1(bpf_this_cpu_ptr, const void *, percpu_ptr)
		643	{
		644	return (unsigned long)this_cpu_ptr((const void __percpu *)percpu_ptr);
		645	}
		646
		647	const struct bpf_func_proto bpf_this_cpu_ptr_proto = {
		648	.func = bpf_this_cpu_ptr,
		649	.gpl_only = false,
		650	.ret_type = RET_PTR_TO_MEM_OR_BTF_ID,
		651	.arg1_type = ARG_PTR_TO_PERCPU_BTF_ID,
		652	};
		653
626	const struct bpf_func_proto bpf_get_current_task_proto __weak;	654	const struct bpf_func_proto bpf_get_current_task_proto __weak;
627	const struct bpf_func_proto bpf_probe_read_user_proto __weak;	655	const struct bpf_func_proto bpf_probe_read_user_proto __weak;
628	const struct bpf_func_proto bpf_probe_read_user_str_proto __weak;	656	const struct bpf_func_proto bpf_probe_read_user_str_proto __weak;
@@ -689,6 +717,10 @@ bpf_base_func_proto(enum bpf_func_id func_id) · 60 unmodified lines
689	return &bpf_snprintf_btf_proto;	717	return &bpf_snprintf_btf_proto;
690	case BPF_FUNC_jiffies64:	718	case BPF_FUNC_jiffies64:
691	return &bpf_jiffies64_proto;	719	return &bpf_jiffies64_proto;
		720	case BPF_FUNC_bpf_per_cpu_ptr:
		721	return &bpf_per_cpu_ptr_proto;
		722	case BPF_FUNC_bpf_this_cpu_ptr:
		723	return &bpf_this_cpu_ptr_proto;
692	default:	724	default:
693	break;	725	break;
694	}	726	}

kernel/bpf/percpu_freelist.c+96−5

@@ -17,6 +17,8 @@ int pcpu_freelist_init(struct pcpu_freelist *s) · 16 unmodified lines
17	raw_spin_lock_init(&head->lock);	17	raw_spin_lock_init(&head->lock);
18	head->first = NULL;	18	head->first = NULL;
19	}	19	}
		20	raw_spin_lock_init(&s->extralist.lock);
		21	s->extralist.first = NULL;
20	return 0;	22	return 0;
21	}	23	}
22		24
@@ -40,12 +42,50 @@ static inline void ___pcpu_freelist_push(struct pcpu_freelist_head *head, · 17 unmodified lines
40	raw_spin_unlock(&head->lock);	42	raw_spin_unlock(&head->lock);
41	}	43	}
42		44
		45	static inline bool pcpu_freelist_try_push_extra(struct pcpu_freelist *s,
		46	struct pcpu_freelist_node *node)
		47	{
		48	if (!raw_spin_trylock(&s->extralist.lock))
		49	return false;
		50
		51	pcpu_freelist_push_node(&s->extralist, node);
		52	raw_spin_unlock(&s->extralist.lock);
		53	return true;
		54	}
		55
		56	static inline void ___pcpu_freelist_push_nmi(struct pcpu_freelist *s,
		57	struct pcpu_freelist_node *node)
		58	{
		59	int cpu, orig_cpu;
		60
		61	orig_cpu = cpu = raw_smp_processor_id();
		62	while (1) {
		63	struct pcpu_freelist_head *head;
		64
		65	head = per_cpu_ptr(s->freelist, cpu);
		66	if (raw_spin_trylock(&head->lock)) {
		67	pcpu_freelist_push_node(head, node);
		68	raw_spin_unlock(&head->lock);
		69	return;
		70	}
		71	cpu = cpumask_next(cpu, cpu_possible_mask);
		72	if (cpu >= nr_cpu_ids)
		73	cpu = 0;
		74
		75	/* cannot lock any per cpu lock, try extralist */
		76	if (cpu == orig_cpu &&
		77	pcpu_freelist_try_push_extra(s, node))
		78	return;
		79	}
		80	}
		81
43	void __pcpu_freelist_push(struct pcpu_freelist *s,	82	void __pcpu_freelist_push(struct pcpu_freelist *s,
44	struct pcpu_freelist_node *node)	83	struct pcpu_freelist_node *node)
45	{	84	{
46	struct pcpu_freelist_head *head = this_cpu_ptr(s->freelist);	85	if (in_nmi())
47		86	___pcpu_freelist_push_nmi(s, node);
48	___pcpu_freelist_push(head, node);	87	else
		88	___pcpu_freelist_push(this_cpu_ptr(s->freelist), node);
49	}	89	}
50		90
51	void pcpu_freelist_push(struct pcpu_freelist *s,	91	void pcpu_freelist_push(struct pcpu_freelist *s,
@@ -81,7 +121,7 @@ again: · 29 unmodified lines
81	}	121	}
82	}	122	}
83		123
84	struct pcpu_freelist_node __pcpu_freelist_pop(struct pcpu_freelist s)	124	static struct pcpu_freelist_node ___pcpu_freelist_pop(struct pcpu_freelist s)
85	{	125	{
86	struct pcpu_freelist_head *head;	126	struct pcpu_freelist_head *head;
87	struct pcpu_freelist_node *node;	127	struct pcpu_freelist_node *node;
@@ -102,8 +142,59 @@ struct pcpu_freelist_node __pcpu_freelist_pop(struct pcpu_freelist s) · 14 unmodified lines
102	if (cpu >= nr_cpu_ids)	142	if (cpu >= nr_cpu_ids)
103	cpu = 0;	143	cpu = 0;
104	if (cpu == orig_cpu)	144	if (cpu == orig_cpu)
105	return NULL;	145	break;
		146	}
		147
		148	/* per cpu lists are all empty, try extralist */
		149	raw_spin_lock(&s->extralist.lock);
		150	node = s->extralist.first;
		151	if (node)
		152	s->extralist.first = node->next;
		153	raw_spin_unlock(&s->extralist.lock);
		154	return node;
		155	}
		156
		157	static struct pcpu_freelist_node *
		158	___pcpu_freelist_pop_nmi(struct pcpu_freelist *s)
		159	{
		160	struct pcpu_freelist_head *head;
		161	struct pcpu_freelist_node *node;
		162	int orig_cpu, cpu;
		163
		164	orig_cpu = cpu = raw_smp_processor_id();
		165	while (1) {
		166	head = per_cpu_ptr(s->freelist, cpu);
		167	if (raw_spin_trylock(&head->lock)) {
		168	node = head->first;
		169	if (node) {
		170	head->first = node->next;
		171	raw_spin_unlock(&head->lock);
		172	return node;
		173	}
		174	raw_spin_unlock(&head->lock);
		175	}
		176	cpu = cpumask_next(cpu, cpu_possible_mask);
		177	if (cpu >= nr_cpu_ids)
		178	cpu = 0;
		179	if (cpu == orig_cpu)
		180	break;
106	}	181	}
		182
		183	/* cannot pop from per cpu lists, try extralist */
		184	if (!raw_spin_trylock(&s->extralist.lock))
		185	return NULL;
		186	node = s->extralist.first;
		187	if (node)
		188	s->extralist.first = node->next;
		189	raw_spin_unlock(&s->extralist.lock);
		190	return node;
		191	}
		192
		193	struct pcpu_freelist_node __pcpu_freelist_pop(struct pcpu_freelist s)
		194	{
		195	if (in_nmi())
		196	return ___pcpu_freelist_pop_nmi(s);
		197	return ___pcpu_freelist_pop(s);
107	}	198	}
108		199
109	struct pcpu_freelist_node pcpu_freelist_pop(struct pcpu_freelist s)	200	struct pcpu_freelist_node pcpu_freelist_pop(struct pcpu_freelist s)

kernel/bpf/percpu_freelist.h+1−0

@@ -13,6 +13,7 @@ struct pcpu_freelist_head { · 12 unmodified lines
13		13
14	struct pcpu_freelist {	14	struct pcpu_freelist {
15	struct pcpu_freelist_head __percpu *freelist;	15	struct pcpu_freelist_head __percpu *freelist;
		16	struct pcpu_freelist_head extralist;
16	};	17	};
17		18
18	struct pcpu_freelist_node {	19	struct pcpu_freelist_node {

kernel/bpf/syscall.c+3−1

 	used_maps_old = prog->aux->used_maps;
 	for (i = 0; i < prog->aux->used_map_cnt; i++)
-		if (used_maps_old[i] == map)
+		if (used_maps_old[i] == map) {
+			bpf_map_put(map);
 			goto out_unlock;
+		}
 	used_maps_new = kmalloc_array(prog->aux->used_map_cnt + 1,
 				      sizeof(used_maps_new[0]),

kernel/bpf/verifier.c+251−19

@@ -238,6 +238,8 @@ struct bpf_call_arg_meta { · 237 unmodified lines
238	u64 msize_max_value;	238	u64 msize_max_value;
239	int ref_obj_id;	239	int ref_obj_id;
240	int func_id;	240	int func_id;
		241	u32 btf_id;
		242	u32 ret_btf_id;
241	};	243	};
242		244
243	struct btf *btf_vmlinux;	245	struct btf *btf_vmlinux;
@@ -517,6 +519,7 @@ static const char * const reg_type_str[] = { · 273 unmodified lines
517	[PTR_TO_XDP_SOCK] = "xdp_sock",	519	[PTR_TO_XDP_SOCK] = "xdp_sock",
518	[PTR_TO_BTF_ID] = "ptr_",	520	[PTR_TO_BTF_ID] = "ptr_",
519	[PTR_TO_BTF_ID_OR_NULL] = "ptr_or_null_",	521	[PTR_TO_BTF_ID_OR_NULL] = "ptr_or_null_",
		522	[PTR_TO_PERCPU_BTF_ID] = "percpu_ptr_",
520	[PTR_TO_MEM] = "mem",	523	[PTR_TO_MEM] = "mem",
521	[PTR_TO_MEM_OR_NULL] = "mem_or_null",	524	[PTR_TO_MEM_OR_NULL] = "mem_or_null",
522	[PTR_TO_RDONLY_BUF] = "rdonly_buf",	525	[PTR_TO_RDONLY_BUF] = "rdonly_buf",
@@ -583,7 +586,9 @@ static void print_verifier_state(struct bpf_verifier_env *env, · 60 unmodified lines
583	/* reg->off should be 0 for SCALAR_VALUE */	586	/* reg->off should be 0 for SCALAR_VALUE */
584	verbose(env, "%lld", reg->var_off.value + reg->off);	587	verbose(env, "%lld", reg->var_off.value + reg->off);
585	} else {	588	} else {
586	if (t == PTR_TO_BTF_ID \|\| t == PTR_TO_BTF_ID_OR_NULL)	589	if (t == PTR_TO_BTF_ID \|\|
		590	t == PTR_TO_BTF_ID_OR_NULL \|\|
		591	t == PTR_TO_PERCPU_BTF_ID)
587	verbose(env, "%s", kernel_type_name(reg->btf_id));	592	verbose(env, "%s", kernel_type_name(reg->btf_id));
588	verbose(env, "(id=%d", reg->id);	593	verbose(env, "(id=%d", reg->id);
589	if (reg_type_may_be_refcounted_or_null(t))	594	if (reg_type_may_be_refcounted_or_null(t))
@@ -2204,6 +2209,7 @@ static bool is_spillable_regtype(enum bpf_reg_type type) · 1614 unmodified lines
2204	case PTR_TO_RDONLY_BUF_OR_NULL:	2209	case PTR_TO_RDONLY_BUF_OR_NULL:
2205	case PTR_TO_RDWR_BUF:	2210	case PTR_TO_RDWR_BUF:
2206	case PTR_TO_RDWR_BUF_OR_NULL:	2211	case PTR_TO_RDWR_BUF_OR_NULL:
		2212	case PTR_TO_PERCPU_BTF_ID:
2207	return true;	2213	return true;
2208	default:	2214	default:
2209	return false;	2215	return false;
@@ -2221,6 +2227,20 @@ static bool register_is_const(struct bpf_reg_state *reg) · 11 unmodified lines
2221	return reg->type == SCALAR_VALUE && tnum_is_const(reg->var_off);	2227	return reg->type == SCALAR_VALUE && tnum_is_const(reg->var_off);
2222	}	2228	}
2223		2229
		2230	static bool __is_scalar_unbounded(struct bpf_reg_state *reg)
		2231	{
		2232	return tnum_is_unknown(reg->var_off) &&
		2233	reg->smin_value == S64_MIN && reg->smax_value == S64_MAX &&
		2234	reg->umin_value == 0 && reg->umax_value == U64_MAX &&
		2235	reg->s32_min_value == S32_MIN && reg->s32_max_value == S32_MAX &&
		2236	reg->u32_min_value == 0 && reg->u32_max_value == U32_MAX;
		2237	}
		2238
		2239	static bool register_is_bounded(struct bpf_reg_state *reg)
		2240	{
		2241	return reg->type == SCALAR_VALUE && !__is_scalar_unbounded(reg);
		2242	}
		2243
2224	static bool __is_pointer_value(bool allow_ptr_leaks,	2244	static bool __is_pointer_value(bool allow_ptr_leaks,
2225	const struct bpf_reg_state *reg)	2245	const struct bpf_reg_state *reg)
2226	{	2246	{
@@ -2272,7 +2292,7 @@ static int check_stack_write(struct bpf_verifier_env *env, · 45 unmodified lines
2272	if (value_regno >= 0)	2292	if (value_regno >= 0)
2273	reg = &cur->regs[value_regno];	2293	reg = &cur->regs[value_regno];
2274		2294
2275	if (reg && size == BPF_REG_SIZE && register_is_const(reg) &&	2295	if (reg && size == BPF_REG_SIZE && register_is_bounded(reg) &&
2276	!register_is_null(reg) && env->bpf_capable) {	2296	!register_is_null(reg) && env->bpf_capable) {
2277	if (dst_reg != BPF_REG_FP) {	2297	if (dst_reg != BPF_REG_FP) {
2278	/* The backtracking logic can only recognize explicit	2298	/* The backtracking logic can only recognize explicit
@@ -2667,7 +2687,7 @@ static bool may_access_direct_pkt_data(struct bpf_verifier_env *env, · 388 unmodified lines
2667	case BPF_PROG_TYPE_CGROUP_SKB:	2687	case BPF_PROG_TYPE_CGROUP_SKB:
2668	if (t == BPF_WRITE)	2688	if (t == BPF_WRITE)
2669	return false;	2689	return false;
2670	/* fallthrough */	2690	fallthrough;
2671		2691
2672	/* Program types with direct read + write access go here! */	2692	/* Program types with direct read + write access go here! */
2673	case BPF_PROG_TYPE_SCHED_CLS:	2693	case BPF_PROG_TYPE_SCHED_CLS:
@@ -3978,6 +3998,7 @@ static const struct bpf_reg_types sock_types = { · 1304 unmodified lines
3978	},	3998	},
3979	};	3999	};
3980		4000
		4001	#ifdef CONFIG_NET
3981	static const struct bpf_reg_types btf_id_sock_common_types = {	4002	static const struct bpf_reg_types btf_id_sock_common_types = {
3982	.types = {	4003	.types = {
3983	PTR_TO_SOCK_COMMON,	4004	PTR_TO_SOCK_COMMON,
@@ -3988,6 +4009,7 @@ static const struct bpf_reg_types btf_id_sock_common_types = { · 4 unmodified lines
3988	},	4009	},
3989	.btf_id = &btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON],	4010	.btf_id = &btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON],
3990	};	4011	};
		4012	#endif
3991		4013
3992	static const struct bpf_reg_types mem_types = {	4014	static const struct bpf_reg_types mem_types = {
3993	.types = {	4015	.types = {
@@ -4017,6 +4039,7 @@ static const struct bpf_reg_types alloc_mem_types = { .types = { PTR_TO_MEM } }; · 23 unmodified lines
4017	static const struct bpf_reg_types const_map_ptr_types = { .types = { CONST_PTR_TO_MAP } };	4039	static const struct bpf_reg_types const_map_ptr_types = { .types = { CONST_PTR_TO_MAP } };
4018	static const struct bpf_reg_types btf_ptr_types = { .types = { PTR_TO_BTF_ID } };	4040	static const struct bpf_reg_types btf_ptr_types = { .types = { PTR_TO_BTF_ID } };
4019	static const struct bpf_reg_types spin_lock_types = { .types = { PTR_TO_MAP_VALUE } };	4041	static const struct bpf_reg_types spin_lock_types = { .types = { PTR_TO_MAP_VALUE } };
		4042	static const struct bpf_reg_types percpu_btf_ptr_types = { .types = { PTR_TO_PERCPU_BTF_ID } };
4020		4043
4021	static const struct bpf_reg_types *compatible_reg_types[__BPF_ARG_TYPE_MAX] = {	4044	static const struct bpf_reg_types *compatible_reg_types[__BPF_ARG_TYPE_MAX] = {
4022	[ARG_PTR_TO_MAP_KEY] = &map_key_value_types,	4045	[ARG_PTR_TO_MAP_KEY] = &map_key_value_types,
@@ -4030,7 +4053,9 @@ static const struct bpf_reg_types *compatible_reg_types[__BPF_ARG_TYPE_MAX] = { · 7 unmodified lines
4030	[ARG_PTR_TO_CTX] = &context_types,	4053	[ARG_PTR_TO_CTX] = &context_types,
4031	[ARG_PTR_TO_CTX_OR_NULL] = &context_types,	4054	[ARG_PTR_TO_CTX_OR_NULL] = &context_types,
4032	[ARG_PTR_TO_SOCK_COMMON] = &sock_types,	4055	[ARG_PTR_TO_SOCK_COMMON] = &sock_types,
		4056	#ifdef CONFIG_NET
4033	[ARG_PTR_TO_BTF_ID_SOCK_COMMON] = &btf_id_sock_common_types,	4057	[ARG_PTR_TO_BTF_ID_SOCK_COMMON] = &btf_id_sock_common_types,
		4058	#endif
4034	[ARG_PTR_TO_SOCKET] = &fullsock_types,	4059	[ARG_PTR_TO_SOCKET] = &fullsock_types,
4035	[ARG_PTR_TO_SOCKET_OR_NULL] = &fullsock_types,	4060	[ARG_PTR_TO_SOCKET_OR_NULL] = &fullsock_types,
4036	[ARG_PTR_TO_BTF_ID] = &btf_ptr_types,	4061	[ARG_PTR_TO_BTF_ID] = &btf_ptr_types,
@@ -4042,6 +4067,7 @@ static const struct bpf_reg_types *compatible_reg_types[__BPF_ARG_TYPE_MAX] = { · 5 unmodified lines
4042	[ARG_PTR_TO_ALLOC_MEM_OR_NULL] = &alloc_mem_types,	4067	[ARG_PTR_TO_ALLOC_MEM_OR_NULL] = &alloc_mem_types,
4043	[ARG_PTR_TO_INT] = &int_ptr_types,	4068	[ARG_PTR_TO_INT] = &int_ptr_types,
4044	[ARG_PTR_TO_LONG] = &int_ptr_types,	4069	[ARG_PTR_TO_LONG] = &int_ptr_types,
		4070	[ARG_PTR_TO_PERCPU_BTF_ID] = &percpu_btf_ptr_types,
4045	};	4071	};
4046		4072
4047	static int check_reg_type(struct bpf_verifier_env *env, u32 regno,	4073	static int check_reg_type(struct bpf_verifier_env *env, u32 regno,
@@ -4205,6 +4231,12 @@ skip_type_check: · 157 unmodified lines
4205	err = check_helper_mem_access(env, regno,	4231	err = check_helper_mem_access(env, regno,
4206	meta->map_ptr->value_size, false,	4232	meta->map_ptr->value_size, false,
4207	meta);	4233	meta);
		4234	} else if (arg_type == ARG_PTR_TO_PERCPU_BTF_ID) {
		4235	if (!reg->btf_id) {
		4236	verbose(env, "Helper has invalid btf_id in R%d\n", regno);
		4237	return -EACCES;
		4238	}
		4239	meta->ret_btf_id = reg->btf_id;
4208	} else if (arg_type == ARG_PTR_TO_SPIN_LOCK) {	4240	} else if (arg_type == ARG_PTR_TO_SPIN_LOCK) {
4209	if (meta->func_id == BPF_FUNC_spin_lock) {	4241	if (meta->func_id == BPF_FUNC_spin_lock) {
4210	if (process_spin_lock(env, regno, true))	4242	if (process_spin_lock(env, regno, true))
@@ -5114,6 +5146,35 @@ static int check_helper_call(struct bpf_verifier_env *env, int func_id, int insn · 903 unmodified lines
5114	regs[BPF_REG_0].type = PTR_TO_MEM_OR_NULL;	5146	regs[BPF_REG_0].type = PTR_TO_MEM_OR_NULL;
5115	regs[BPF_REG_0].id = ++env->id_gen;	5147	regs[BPF_REG_0].id = ++env->id_gen;
5116	regs[BPF_REG_0].mem_size = meta.mem_size;	5148	regs[BPF_REG_0].mem_size = meta.mem_size;
		5149	} else if (fn->ret_type == RET_PTR_TO_MEM_OR_BTF_ID_OR_NULL \|\|
		5150	fn->ret_type == RET_PTR_TO_MEM_OR_BTF_ID) {
		5151	const struct btf_type *t;
		5152
		5153	mark_reg_known_zero(env, regs, BPF_REG_0);
		5154	t = btf_type_skip_modifiers(btf_vmlinux, meta.ret_btf_id, NULL);
		5155	if (!btf_type_is_struct(t)) {
		5156	u32 tsize;
		5157	const struct btf_type *ret;
		5158	const char *tname;
		5159
		5160	/* resolve the type size of ksym. */
		5161	ret = btf_resolve_size(btf_vmlinux, t, &tsize);
		5162	if (IS_ERR(ret)) {
		5163	tname = btf_name_by_offset(btf_vmlinux, t->name_off);
		5164	verbose(env, "unable to resolve the size of type '%s': %ld\n",
		5165	tname, PTR_ERR(ret));
		5166	return -EINVAL;
		5167	}
		5168	regs[BPF_REG_0].type =
		5169	fn->ret_type == RET_PTR_TO_MEM_OR_BTF_ID ?
		5170	PTR_TO_MEM : PTR_TO_MEM_OR_NULL;
		5171	regs[BPF_REG_0].mem_size = tsize;
		5172	} else {
		5173	regs[BPF_REG_0].type =
		5174	fn->ret_type == RET_PTR_TO_MEM_OR_BTF_ID ?
		5175	PTR_TO_BTF_ID : PTR_TO_BTF_ID_OR_NULL;
		5176	regs[BPF_REG_0].btf_id = meta.ret_btf_id;
		5177	}
5117	} else if (fn->ret_type == RET_PTR_TO_BTF_ID_OR_NULL) {	5178	} else if (fn->ret_type == RET_PTR_TO_BTF_ID_OR_NULL) {
5118	int ret_btf_id;	5179	int ret_btf_id;
5119		5180
@@ -5432,7 +5493,7 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, · 312 unmodified lines
5432	/* smin_val represents the known value */	5493	/* smin_val represents the known value */
5433	if (known && smin_val == 0 && opcode == BPF_ADD)	5494	if (known && smin_val == 0 && opcode == BPF_ADD)
5434	break;	5495	break;
5435	/* fall-through */	5496	fallthrough;
5436	case PTR_TO_PACKET_END:	5497	case PTR_TO_PACKET_END:
5437	case PTR_TO_SOCKET:	5498	case PTR_TO_SOCKET:
5438	case PTR_TO_SOCKET_OR_NULL:	5499	case PTR_TO_SOCKET_OR_NULL:
@@ -6389,6 +6450,11 @@ static int adjust_reg_min_max_vals(struct bpf_verifier_env *env, · 950 unmodified lines
6389	src_reg = NULL;	6450	src_reg = NULL;
6390	if (dst_reg->type != SCALAR_VALUE)	6451	if (dst_reg->type != SCALAR_VALUE)
6391	ptr_reg = dst_reg;	6452	ptr_reg = dst_reg;
		6453	else
		6454	/* Make sure ID is cleared otherwise dst_reg min/max could be
		6455	* incorrectly propagated into other registers by find_equal_scalars()
		6456	*/
		6457	dst_reg->id = 0;
6392	if (BPF_SRC(insn->code) == BPF_X) {	6458	if (BPF_SRC(insn->code) == BPF_X) {
6393	src_reg = &regs[insn->src_reg];	6459	src_reg = &regs[insn->src_reg];
6394	if (src_reg->type != SCALAR_VALUE) {	6460	if (src_reg->type != SCALAR_VALUE) {
@@ -6522,6 +6588,12 @@ static int check_alu_op(struct bpf_verifier_env env, struct bpf_insn insn) · 127 unmodified lines
6522	/* case: R1 = R2	6588	/* case: R1 = R2
6523	* copy register state to dest reg	6589	* copy register state to dest reg
6524	*/	6590	*/
		6591	if (src_reg->type == SCALAR_VALUE && !src_reg->id)
		6592	/* Assign src and dst registers the same ID
		6593	* that will be used by find_equal_scalars()
		6594	* to propagate min/max range.
		6595	*/
		6596	src_reg->id = ++env->id_gen;
6525	dst_reg = src_reg;	6597	dst_reg = src_reg;
6526	dst_reg->live \|= REG_LIVE_WRITTEN;	6598	dst_reg->live \|= REG_LIVE_WRITTEN;
6527	dst_reg->subreg_def = DEF_NOT_SUBREG;	6599	dst_reg->subreg_def = DEF_NOT_SUBREG;
@@ -6534,6 +6606,11 @@ static int check_alu_op(struct bpf_verifier_env env, struct bpf_insn insn) · 6 unmodified lines
6534	return -EACCES;	6606	return -EACCES;
6535	} else if (src_reg->type == SCALAR_VALUE) {	6607	} else if (src_reg->type == SCALAR_VALUE) {
6536	dst_reg = src_reg;	6608	dst_reg = src_reg;
		6609	/* Make sure ID is cleared otherwise
		6610	* dst_reg min/max could be incorrectly
		6611	* propagated into src_reg by find_equal_scalars()
		6612	*/
		6613	dst_reg->id = 0;
6537	dst_reg->live \|= REG_LIVE_WRITTEN;	6614	dst_reg->live \|= REG_LIVE_WRITTEN;
6538	dst_reg->subreg_def = env->insn_idx + 1;	6615	dst_reg->subreg_def = env->insn_idx + 1;
6539	} else {	6616	} else {
@@ -7322,6 +7399,30 @@ static bool try_match_pkt_pointers(const struct bpf_insn *insn, · 782 unmodified lines
7322	return true;	7399	return true;
7323	}	7400	}
7324		7401
		7402	static void find_equal_scalars(struct bpf_verifier_state *vstate,
		7403	struct bpf_reg_state *known_reg)
		7404	{
		7405	struct bpf_func_state *state;
		7406	struct bpf_reg_state *reg;
		7407	int i, j;
		7408
		7409	for (i = 0; i <= vstate->curframe; i++) {
		7410	state = vstate->frame[i];
		7411	for (j = 0; j < MAX_BPF_REG; j++) {
		7412	reg = &state->regs[j];
		7413	if (reg->type == SCALAR_VALUE && reg->id == known_reg->id)
		7414	reg = known_reg;
		7415	}
		7416
		7417	bpf_for_each_spilled_reg(j, state, reg) {
		7418	if (!reg)
		7419	continue;
		7420	if (reg->type == SCALAR_VALUE && reg->id == known_reg->id)
		7421	reg = known_reg;
		7422	}
		7423	}
		7424	}
		7425
7325	static int check_cond_jmp_op(struct bpf_verifier_env *env,	7426	static int check_cond_jmp_op(struct bpf_verifier_env *env,
7326	struct bpf_insn insn, int insn_idx)	7427	struct bpf_insn insn, int insn_idx)
7327	{	7428	{
@@ -7450,6 +7551,11 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env, · 122 unmodified lines
7450	reg_combine_min_max(&other_branch_regs[insn->src_reg],	7551	reg_combine_min_max(&other_branch_regs[insn->src_reg],
7451	&other_branch_regs[insn->dst_reg],	7552	&other_branch_regs[insn->dst_reg],
7452	src_reg, dst_reg, opcode);	7553	src_reg, dst_reg, opcode);
		7554	if (src_reg->id) {
		7555	find_equal_scalars(this_branch, src_reg);
		7556	find_equal_scalars(other_branch, &other_branch_regs[insn->src_reg]);
		7557	}
		7558
7453	}	7559	}
7454	} else if (dst_reg->type == SCALAR_VALUE) {	7560	} else if (dst_reg->type == SCALAR_VALUE) {
7455	reg_set_min_max(&other_branch_regs[insn->dst_reg],	7561	reg_set_min_max(&other_branch_regs[insn->dst_reg],
@@ -7457,6 +7563,11 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env, · 1 unmodified line
7457	opcode, is_jmp32);	7563	opcode, is_jmp32);
7458	}	7564	}
7459		7565
		7566	if (dst_reg->type == SCALAR_VALUE && dst_reg->id) {
		7567	find_equal_scalars(this_branch, dst_reg);
		7568	find_equal_scalars(other_branch, &other_branch_regs[insn->dst_reg]);
		7569	}
		7570
7460	/* detect if R == 0 where R is returned from bpf_map_lookup_elem().	7571	/* detect if R == 0 where R is returned from bpf_map_lookup_elem().
7461	* NOTE: these optimizations below are related with pointer comparison	7572	* NOTE: these optimizations below are related with pointer comparison
7462	* which will never be JMP32.	7573	* which will never be JMP32.
@@ -7488,6 +7599,7 @@ static int check_ld_imm(struct bpf_verifier_env env, struct bpf_insn insn) · 25 unmodified lines
7488	{	7599	{
7489	struct bpf_insn_aux_data *aux = cur_aux(env);	7600	struct bpf_insn_aux_data *aux = cur_aux(env);
7490	struct bpf_reg_state *regs = cur_regs(env);	7601	struct bpf_reg_state *regs = cur_regs(env);
		7602	struct bpf_reg_state *dst_reg;
7491	struct bpf_map *map;	7603	struct bpf_map *map;
7492	int err;	7604	int err;
7493		7605
@@ -7504,25 +7616,45 @@ static int check_ld_imm(struct bpf_verifier_env env, struct bpf_insn insn) · 10 unmodified lines
7504	if (err)	7616	if (err)
7505	return err;	7617	return err;
7506		7618
		7619	dst_reg = &regs[insn->dst_reg];
7507	if (insn->src_reg == 0) {	7620	if (insn->src_reg == 0) {
7508	u64 imm = ((u64)(insn + 1)->imm << 32) \| (u32)insn->imm;	7621	u64 imm = ((u64)(insn + 1)->imm << 32) \| (u32)insn->imm;
7509		7622
7510	regs[insn->dst_reg].type = SCALAR_VALUE;	7623	dst_reg->type = SCALAR_VALUE;
7511	__mark_reg_known(&regs[insn->dst_reg], imm);	7624	__mark_reg_known(&regs[insn->dst_reg], imm);
7512	return 0;	7625	return 0;
7513	}	7626	}
7514		7627
		7628	if (insn->src_reg == BPF_PSEUDO_BTF_ID) {
		7629	mark_reg_known_zero(env, regs, insn->dst_reg);
		7630
		7631	dst_reg->type = aux->btf_var.reg_type;
		7632	switch (dst_reg->type) {
		7633	case PTR_TO_MEM:
		7634	dst_reg->mem_size = aux->btf_var.mem_size;
		7635	break;
		7636	case PTR_TO_BTF_ID:
		7637	case PTR_TO_PERCPU_BTF_ID:
		7638	dst_reg->btf_id = aux->btf_var.btf_id;
		7639	break;
		7640	default:
		7641	verbose(env, "bpf verifier is misconfigured\n");
		7642	return -EFAULT;
		7643	}
		7644	return 0;
		7645	}
		7646
7515	map = env->used_maps[aux->map_index];	7647	map = env->used_maps[aux->map_index];
7516	mark_reg_known_zero(env, regs, insn->dst_reg);	7648	mark_reg_known_zero(env, regs, insn->dst_reg);
7517	regs[insn->dst_reg].map_ptr = map;	7649	dst_reg->map_ptr = map;
7518		7650
7519	if (insn->src_reg == BPF_PSEUDO_MAP_VALUE) {	7651	if (insn->src_reg == BPF_PSEUDO_MAP_VALUE) {
7520	regs[insn->dst_reg].type = PTR_TO_MAP_VALUE;	7652	dst_reg->type = PTR_TO_MAP_VALUE;
7521	regs[insn->dst_reg].off = aux->map_off;	7653	dst_reg->off = aux->map_off;
7522	if (map_value_has_spin_lock(map))	7654	if (map_value_has_spin_lock(map))
7523	regs[insn->dst_reg].id = ++env->id_gen;	7655	dst_reg->id = ++env->id_gen;
7524	} else if (insn->src_reg == BPF_PSEUDO_MAP_FD) {	7656	} else if (insn->src_reg == BPF_PSEUDO_MAP_FD) {
7525	regs[insn->dst_reg].type = CONST_PTR_TO_MAP;	7657	dst_reg->type = CONST_PTR_TO_MAP;
7526	} else {	7658	} else {
7527	verbose(env, "bpf verifier is misconfigured\n");	7659	verbose(env, "bpf verifier is misconfigured\n");
7528	return -EINVAL;	7660	return -EINVAL;
@@ -9424,6 +9556,92 @@ process_bpf_exit: · 1895 unmodified lines
9424	return 0;	9556	return 0;
9425	}	9557	}
9426		9558
		9559	/* replace pseudo btf_id with kernel symbol address */
		9560	static int check_pseudo_btf_id(struct bpf_verifier_env *env,
		9561	struct bpf_insn *insn,
		9562	struct bpf_insn_aux_data *aux)
		9563	{
		9564	u32 datasec_id, type, id = insn->imm;
		9565	const struct btf_var_secinfo *vsi;
		9566	const struct btf_type *datasec;
		9567	const struct btf_type *t;
		9568	const char *sym_name;
		9569	bool percpu = false;
		9570	u64 addr;
		9571	int i;
		9572
		9573	if (!btf_vmlinux) {
		9574	verbose(env, "kernel is missing BTF, make sure CONFIG_DEBUG_INFO_BTF=y is specified in Kconfig.\n");
		9575	return -EINVAL;
		9576	}
		9577
		9578	if (insn[1].imm != 0) {
		9579	verbose(env, "reserved field (insn[1].imm) is used in pseudo_btf_id ldimm64 insn.\n");
		9580	return -EINVAL;
		9581	}
		9582
		9583	t = btf_type_by_id(btf_vmlinux, id);
		9584	if (!t) {
		9585	verbose(env, "ldimm64 insn specifies invalid btf_id %d.\n", id);
		9586	return -ENOENT;
		9587	}
		9588
		9589	if (!btf_type_is_var(t)) {
		9590	verbose(env, "pseudo btf_id %d in ldimm64 isn't KIND_VAR.\n",
		9591	id);
		9592	return -EINVAL;
		9593	}
		9594
		9595	sym_name = btf_name_by_offset(btf_vmlinux, t->name_off);
		9596	addr = kallsyms_lookup_name(sym_name);
		9597	if (!addr) {
		9598	verbose(env, "ldimm64 failed to find the address for kernel symbol '%s'.\n",
		9599	sym_name);
		9600	return -ENOENT;
		9601	}
		9602
		9603	datasec_id = btf_find_by_name_kind(btf_vmlinux, ".data..percpu",
		9604	BTF_KIND_DATASEC);
		9605	if (datasec_id > 0) {
		9606	datasec = btf_type_by_id(btf_vmlinux, datasec_id);
		9607	for_each_vsi(i, datasec, vsi) {
		9608	if (vsi->type == id) {
		9609	percpu = true;
		9610	break;
		9611	}
		9612	}
		9613	}
		9614
		9615	insn[0].imm = (u32)addr;
		9616	insn[1].imm = addr >> 32;
		9617
		9618	type = t->type;
		9619	t = btf_type_skip_modifiers(btf_vmlinux, type, NULL);
		9620	if (percpu) {
		9621	aux->btf_var.reg_type = PTR_TO_PERCPU_BTF_ID;
		9622	aux->btf_var.btf_id = type;
		9623	} else if (!btf_type_is_struct(t)) {
		9624	const struct btf_type *ret;
		9625	const char *tname;
		9626	u32 tsize;
		9627
		9628	/* resolve the type size of ksym. */
		9629	ret = btf_resolve_size(btf_vmlinux, t, &tsize);
		9630	if (IS_ERR(ret)) {
		9631	tname = btf_name_by_offset(btf_vmlinux, t->name_off);
		9632	verbose(env, "ldimm64 unable to resolve the size of type '%s': %ld\n",
		9633	tname, PTR_ERR(ret));
		9634	return -EINVAL;
		9635	}
		9636	aux->btf_var.reg_type = PTR_TO_MEM;
		9637	aux->btf_var.mem_size = tsize;
		9638	} else {
		9639	aux->btf_var.reg_type = PTR_TO_BTF_ID;
		9640	aux->btf_var.btf_id = type;
		9641	}
		9642	return 0;
		9643	}
		9644
9427	static int check_map_prealloc(struct bpf_map *map)	9645	static int check_map_prealloc(struct bpf_map *map)
9428	{	9646	{
9429	return (map->map_type != BPF_MAP_TYPE_HASH &&	9647	return (map->map_type != BPF_MAP_TYPE_HASH &&
@@ -9534,10 +9752,14 @@ static bool bpf_map_is_cgroup_storage(struct bpf_map *map) · 104 unmodified lines
9534	map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE);	9752	map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE);
9535	}	9753	}
9536		9754
9537	/* look for pseudo eBPF instructions that access map FDs and	9755	/* find and rewrite pseudo imm in ld_imm64 instructions:
9538	* replace them with actual map pointers	9756	*
		9757	* 1. if it accesses map FD, replace it with actual map pointer.
		9758	* 2. if it accesses btf_id of a VAR, replace it with pointer to the var.
		9759	*
		9760	* NOTE: btf_vmlinux is required for converting pseudo btf_id.
9539	*/	9761	*/
9540	static int replace_map_fd_with_map_ptr(struct bpf_verifier_env *env)	9762	static int resolve_pseudo_ldimm64(struct bpf_verifier_env *env)
9541	{	9763	{
9542	struct bpf_insn *insn = env->prog->insnsi;	9764	struct bpf_insn *insn = env->prog->insnsi;
9543	int insn_cnt = env->prog->len;	9765	int insn_cnt = env->prog->len;
@@ -9578,6 +9800,14 @@ static int replace_map_fd_with_map_ptr(struct bpf_verifier_env *env) · 34 unmodified lines
9578	/* valid generic load 64-bit imm */	9800	/* valid generic load 64-bit imm */
9579	goto next_insn;	9801	goto next_insn;
9580		9802
		9803	if (insn[0].src_reg == BPF_PSEUDO_BTF_ID) {
		9804	aux = &env->insn_aux_data[i];
		9805	err = check_pseudo_btf_id(env, insn, aux);
		9806	if (err)
		9807	return err;
		9808	goto next_insn;
		9809	}
		9810
9581	/* In final convert_pseudo_ld_imm64() step, this is	9811	/* In final convert_pseudo_ld_imm64() step, this is
9582	* converted into regular 64-bit imm load insn.	9812	* converted into regular 64-bit imm load insn.
9583	*/	9813	*/
@@ -10819,7 +11049,9 @@ static int fixup_bpf_calls(struct bpf_verifier_env *env) · 1235 unmodified lines
10819	if (insn->imm == BPF_FUNC_map_lookup_elem &&	11049	if (insn->imm == BPF_FUNC_map_lookup_elem &&
10820	ops->map_gen_lookup) {	11050	ops->map_gen_lookup) {
10821	cnt = ops->map_gen_lookup(map_ptr, insn_buf);	11051	cnt = ops->map_gen_lookup(map_ptr, insn_buf);
10822	if (cnt == 0 \|\| cnt >= ARRAY_SIZE(insn_buf)) {	11052	if (cnt == -EOPNOTSUPP)
		11053	goto patch_map_ops_generic;
		11054	if (cnt <= 0 \|\| cnt >= ARRAY_SIZE(insn_buf)) {
10823	verbose(env, "bpf verifier is misconfigured\n");	11055	verbose(env, "bpf verifier is misconfigured\n");
10824	return -EINVAL;	11056	return -EINVAL;
10825	}	11057	}
@@ -10849,7 +11081,7 @@ static int fixup_bpf_calls(struct bpf_verifier_env *env) · 23 unmodified lines
10849	(int ()(struct bpf_map map, void *value))NULL));	11081	(int ()(struct bpf_map map, void *value))NULL));
10850	BUILD_BUG_ON(!__same_type(ops->map_peek_elem,	11082	BUILD_BUG_ON(!__same_type(ops->map_peek_elem,
10851	(int ()(struct bpf_map map, void *value))NULL));	11083	(int ()(struct bpf_map map, void *value))NULL));
10852		11084	patch_map_ops_generic:
10853	switch (insn->imm) {	11085	switch (insn->imm) {
10854	case BPF_FUNC_map_lookup_elem:	11086	case BPF_FUNC_map_lookup_elem:
10855	insn->imm = BPF_CAST_CALL(ops->map_lookup_elem) -	11087	insn->imm = BPF_CAST_CALL(ops->map_lookup_elem) -
@@ -11633,10 +11865,6 @@ int bpf_check(struct bpf_prog *prog, union bpf_attr attr, · 777 unmodified lines
11633	if (is_priv)	11865	if (is_priv)
11634	env->test_state_freq = attr->prog_flags & BPF_F_TEST_STATE_FREQ;	11866	env->test_state_freq = attr->prog_flags & BPF_F_TEST_STATE_FREQ;
11635		11867
11636	ret = replace_map_fd_with_map_ptr(env);
11637	if (ret < 0)
11638	goto skip_full_check;
11639
11640	if (bpf_prog_is_dev_bound(env->prog->aux)) {	11868	if (bpf_prog_is_dev_bound(env->prog->aux)) {
11641	ret = bpf_prog_offload_verifier_prep(env->prog);	11869	ret = bpf_prog_offload_verifier_prep(env->prog);
11642	if (ret)	11870	if (ret)
@@ -11662,6 +11890,10 @@ int bpf_check(struct bpf_prog *prog, union bpf_attr attr, · 19 unmodified lines
11662	if (ret)	11890	if (ret)
11663	goto skip_full_check;	11891	goto skip_full_check;
11664		11892
		11893	ret = resolve_pseudo_ldimm64(env);
		11894	if (ret < 0)
		11895	goto skip_full_check;
		11896
11665	ret = check_cfg(env);	11897	ret = check_cfg(env);
11666	if (ret < 0)	11898	if (ret < 0)
11667	goto skip_full_check;	11899	goto skip_full_check;

kernel/trace/bpf_trace.c+6−0

 		return prog->aux->sleepable ? &bpf_copy_from_user_proto : NULL;
 	case BPF_FUNC_snprintf_btf:
 		return &bpf_snprintf_btf_proto;
+	case BPF_FUNC_bpf_per_cpu_ptr:
+		return &bpf_per_cpu_ptr_proto;
+	case BPF_FUNC_bpf_this_cpu_ptr:
+		return &bpf_this_cpu_ptr_proto;
 	default:
 		return NULL;
 	}
 };
 const struct bpf_prog_ops raw_tracepoint_prog_ops = {
+#ifdef CONFIG_NET
 	.test_run = bpf_prog_test_run_raw_tp,
+#endif
 };
 const struct bpf_verifier_ops tracing_verifier_ops = {

Merge git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code