bpf: Introduce bpf_per_cpu_ptr()

Add bpf_per_cpu_ptr() to help bpf programs access percpu vars. bpf_per_cpu_ptr() has the same semantic as per_cpu_ptr() in the kernel except that it may return NULL. This happens when the cpu parameter is out of range. So the caller must check the returned value.

Hao Luo committed 6 years ago

Commit eaa6bcb1 parent 2c2f6ab8 files · +132 −13Browse files

Unified · Split

8 files

include/linux/bpf.h +4 −0
include/linux/btf.h +11 −0
include/uapi/linux/bpf.h +18 −0
kernel/bpf/btf.c +0 −10
kernel/bpf/helpers.c +18 −0
kernel/bpf/verifier.c +61 −3
kernel/trace/bpf_trace.c +2 −0
tools/include/uapi/linux/bpf.h +18 −0

include/linux/bpf.h+4−0

 	ARG_PTR_TO_ALLOC_MEM_OR_NULL,	/* pointer to dynamically allocated memory or NULL */
 	ARG_CONST_ALLOC_SIZE_OR_ZERO,	/* number of allocated bytes requested */
 	ARG_PTR_TO_BTF_ID_SOCK_COMMON,	/* pointer to in-kernel sock_common or bpf-mirrored bpf_sock */
+	ARG_PTR_TO_PERCPU_BTF_ID,	/* pointer to in-kernel percpu type */
 	__BPF_ARG_TYPE_MAX,
 };
 	RET_PTR_TO_SOCK_COMMON_OR_NULL,	/* returns a pointer to a sock_common or NULL */
 	RET_PTR_TO_ALLOC_MEM_OR_NULL,	/* returns a pointer to dynamically allocated memory or NULL */
 	RET_PTR_TO_BTF_ID_OR_NULL,	/* returns a pointer to a btf_id or NULL */
+	RET_PTR_TO_MEM_OR_BTF_ID_OR_NULL, /* returns a pointer to a valid memory or a btf_id or NULL */
 };
 /* eBPF function prototype used by verifier to allow BPF_CALLs from eBPF programs
 	PTR_TO_RDONLY_BUF_OR_NULL, /* reg points to a readonly buffer or NULL */
 	PTR_TO_RDWR_BUF,	 /* reg points to a read/write buffer */
 	PTR_TO_RDWR_BUF_OR_NULL, /* reg points to a read/write buffer or NULL */
+	PTR_TO_PERCPU_BTF_ID,	 /* reg points to a percpu kernel variable */
 };
 /* The information passed from prog-specific *_is_valid_access
 extern const struct bpf_func_proto bpf_skc_to_udp6_sock_proto;
 extern const struct bpf_func_proto bpf_copy_from_user_proto;
 extern const struct bpf_func_proto bpf_snprintf_btf_proto;
+extern const struct bpf_func_proto bpf_per_cpu_ptr_proto;
 const struct bpf_func_proto *bpf_tracing_func_proto(
 	enum bpf_func_id func_id, const struct bpf_prog *prog);

include/linux/btf.h+11−0

 	     i < btf_type_vlen(struct_type);			\
 	     i++, member++)
+#define for_each_vsi(i, datasec_type, member)			\
+	for (i = 0, member = btf_type_var_secinfo(datasec_type);	\
+	     i < btf_type_vlen(datasec_type);			\
+	     i++, member++)
 static inline bool btf_type_is_ptr(const struct btf_type *t)
 {
 	return BTF_INFO_KIND(t->info) == BTF_KIND_PTR;
 	return (const struct btf_member *)(t + 1);
 }
+static inline const struct btf_var_secinfo *btf_type_var_secinfo(
+		const struct btf_type *t)
+{
+	return (const struct btf_var_secinfo *)(t + 1);
+}
 #ifdef CONFIG_BPF_SYSCALL
 const struct btf_type *btf_type_by_id(const struct btf *btf, u32 type_id);
 const char *btf_name_by_offset(const struct btf *btf, u32 offset);

include/uapi/linux/bpf.h+18−0

@@ -3686,6 +3686,23 @@ union bpf_attr { · 3685 unmodified lines
3686	* Return	3686	* Return
3687	* The helper returns TC_ACT_REDIRECT on success or	3687	* The helper returns TC_ACT_REDIRECT on success or
3688	* TC_ACT_SHOT on error.	3688	* TC_ACT_SHOT on error.
		3689	*
		3690	* void bpf_per_cpu_ptr(const void percpu_ptr, u32 cpu)
		3691	* Description
		3692	* Take a pointer to a percpu ksym, percpu_ptr, and return a
		3693	* pointer to the percpu kernel variable on cpu. A ksym is an
		3694	* extern variable decorated with '__ksym'. For ksym, there is a
		3695	* global var (either static or global) defined of the same name
		3696	* in the kernel. The ksym is percpu if the global var is percpu.
		3697	* The returned pointer points to the global percpu var on cpu.
		3698	*
		3699	* bpf_per_cpu_ptr() has the same semantic as per_cpu_ptr() in the
		3700	* kernel, except that bpf_per_cpu_ptr() may return NULL. This
		3701	* happens if cpu is larger than nr_cpu_ids. The caller of
		3702	* bpf_per_cpu_ptr() must check the returned value.
		3703	* Return
		3704	* A pointer pointing to the kernel percpu variable on cpu, or
		3705	* NULL, if cpu is invalid.
3689	*/	3706	*/
3690	#define __BPF_FUNC_MAPPER(FN) \	3707	#define __BPF_FUNC_MAPPER(FN) \
3691	FN(unspec), \	3708	FN(unspec), \
@@ -3841,6 +3858,7 @@ union bpf_attr { · 149 unmodified lines
3841	FN(seq_printf_btf), \	3858	FN(seq_printf_btf), \
3842	FN(skb_cgroup_classid), \	3859	FN(skb_cgroup_classid), \
3843	FN(redirect_neigh), \	3860	FN(redirect_neigh), \
		3861	FN(bpf_per_cpu_ptr), \
3844	/* */	3862	/* */
3845		3863
3846	/* integer value in 'imm' field of BPF_CALL instruction selects which helper	3864	/* integer value in 'imm' field of BPF_CALL instruction selects which helper

kernel/bpf/btf.c+0−10

 	     i < btf_type_vlen(struct_type);				\
 	     i++, member++)
-#define for_each_vsi(i, struct_type, member)			\
-	for (i = 0, member = btf_type_var_secinfo(struct_type);	\
-	     i < btf_type_vlen(struct_type);			\
-	     i++, member++)
 #define for_each_vsi_from(i, from, struct_type, member)				\
 	for (i = from, member = btf_type_var_secinfo(struct_type) + from;	\
 	     i < btf_type_vlen(struct_type);					\
 	return (const struct btf_var *)(t + 1);
 }
-static const struct btf_var_secinfo *btf_type_var_secinfo(const struct btf_type *t)
-{
-	return (const struct btf_var_secinfo *)(t + 1);
-}
 static const struct btf_kind_operations *btf_type_ops(const struct btf_type *t)
 {
 	return kind_ops[BTF_INFO_KIND(t->info)];

kernel/bpf/helpers.c+18−0

@@ -623,6 +623,22 @@ const struct bpf_func_proto bpf_copy_from_user_proto = { · 622 unmodified lines
623	.arg3_type = ARG_ANYTHING,	623	.arg3_type = ARG_ANYTHING,
624	};	624	};
625		625
		626	BPF_CALL_2(bpf_per_cpu_ptr, const void *, ptr, u32, cpu)
		627	{
		628	if (cpu >= nr_cpu_ids)
		629	return (unsigned long)NULL;
		630
		631	return (unsigned long)per_cpu_ptr((const void __percpu *)ptr, cpu);
		632	}
		633
		634	const struct bpf_func_proto bpf_per_cpu_ptr_proto = {
		635	.func = bpf_per_cpu_ptr,
		636	.gpl_only = false,
		637	.ret_type = RET_PTR_TO_MEM_OR_BTF_ID_OR_NULL,
		638	.arg1_type = ARG_PTR_TO_PERCPU_BTF_ID,
		639	.arg2_type = ARG_ANYTHING,
		640	};
		641
626	const struct bpf_func_proto bpf_get_current_task_proto __weak;	642	const struct bpf_func_proto bpf_get_current_task_proto __weak;
627	const struct bpf_func_proto bpf_probe_read_user_proto __weak;	643	const struct bpf_func_proto bpf_probe_read_user_proto __weak;
628	const struct bpf_func_proto bpf_probe_read_user_str_proto __weak;	644	const struct bpf_func_proto bpf_probe_read_user_str_proto __weak;
@@ -689,6 +705,8 @@ bpf_base_func_proto(enum bpf_func_id func_id) · 60 unmodified lines
689	return &bpf_snprintf_btf_proto;	705	return &bpf_snprintf_btf_proto;
690	case BPF_FUNC_jiffies64:	706	case BPF_FUNC_jiffies64:
691	return &bpf_jiffies64_proto;	707	return &bpf_jiffies64_proto;
		708	case BPF_FUNC_bpf_per_cpu_ptr:
		709	return &bpf_per_cpu_ptr_proto;
692	default:	710	default:
693	break;	711	break;
694	}	712	}

kernel/bpf/verifier.c+61−3

 	u64 msize_max_value;
 	int ref_obj_id;
 	int func_id;
+	u32 btf_id;
+	u32 ret_btf_id;
 };
 struct btf *btf_vmlinux;
 	[PTR_TO_XDP_SOCK]	= "xdp_sock",
 	[PTR_TO_BTF_ID]		= "ptr_",
 	[PTR_TO_BTF_ID_OR_NULL]	= "ptr_or_null_",
+	[PTR_TO_PERCPU_BTF_ID]	= "percpu_ptr_",
 	[PTR_TO_MEM]		= "mem",
 	[PTR_TO_MEM_OR_NULL]	= "mem_or_null",
 	[PTR_TO_RDONLY_BUF]	= "rdonly_buf",
 			/* reg->off should be 0 for SCALAR_VALUE */
 			verbose(env, "%lld", reg->var_off.value + reg->off);
 		} else {
-			if (t == PTR_TO_BTF_ID || t == PTR_TO_BTF_ID_OR_NULL)
+			if (t == PTR_TO_BTF_ID ||
+			    t == PTR_TO_BTF_ID_OR_NULL ||
+			    t == PTR_TO_PERCPU_BTF_ID)
 				verbose(env, "%s", kernel_type_name(reg->btf_id));
 			verbose(env, "(id=%d", reg->id);
 			if (reg_type_may_be_refcounted_or_null(t))
 	case PTR_TO_RDONLY_BUF_OR_NULL:
 	case PTR_TO_RDWR_BUF:
 	case PTR_TO_RDWR_BUF_OR_NULL:
+	case PTR_TO_PERCPU_BTF_ID:
 		return true;
 	default:
 		return false;
 static const struct bpf_reg_types const_map_ptr_types = { .types = { CONST_PTR_TO_MAP } };
 static const struct bpf_reg_types btf_ptr_types = { .types = { PTR_TO_BTF_ID } };
 static const struct bpf_reg_types spin_lock_types = { .types = { PTR_TO_MAP_VALUE } };
+static const struct bpf_reg_types percpu_btf_ptr_types = { .types = { PTR_TO_PERCPU_BTF_ID } };
 static const struct bpf_reg_types *compatible_reg_types[__BPF_ARG_TYPE_MAX] = {
 	[ARG_PTR_TO_MAP_KEY]		= &map_key_value_types,
 	[ARG_PTR_TO_ALLOC_MEM_OR_NULL]	= &alloc_mem_types,
 	[ARG_PTR_TO_INT]		= &int_ptr_types,
 	[ARG_PTR_TO_LONG]		= &int_ptr_types,
+	[ARG_PTR_TO_PERCPU_BTF_ID]	= &percpu_btf_ptr_types,
 };
 static int check_reg_type(struct bpf_verifier_env *env, u32 regno,
 		err = check_helper_mem_access(env, regno,
 					      meta->map_ptr->value_size, false,
 					      meta);
+	} else if (arg_type == ARG_PTR_TO_PERCPU_BTF_ID) {
+		if (!reg->btf_id) {
+			verbose(env, "Helper has invalid btf_id in R%d\n", regno);
+			return -EACCES;
+		}
+		meta->ret_btf_id = reg->btf_id;
 	} else if (arg_type == ARG_PTR_TO_SPIN_LOCK) {
 		if (meta->func_id == BPF_FUNC_spin_lock) {
 			if (process_spin_lock(env, regno, true))
 		regs[BPF_REG_0].type = PTR_TO_MEM_OR_NULL;
 		regs[BPF_REG_0].id = ++env->id_gen;
 		regs[BPF_REG_0].mem_size = meta.mem_size;
+	} else if (fn->ret_type == RET_PTR_TO_MEM_OR_BTF_ID_OR_NULL) {
+		const struct btf_type *t;
+		mark_reg_known_zero(env, regs, BPF_REG_0);
+		t = btf_type_skip_modifiers(btf_vmlinux, meta.ret_btf_id, NULL);
+		if (!btf_type_is_struct(t)) {
+			u32 tsize;
+			const struct btf_type *ret;
+			const char *tname;
+			/* resolve the type size of ksym. */
+			ret = btf_resolve_size(btf_vmlinux, t, &tsize);
+			if (IS_ERR(ret)) {
+				tname = btf_name_by_offset(btf_vmlinux, t->name_off);
+				verbose(env, "unable to resolve the size of type '%s': %ld\n",
+					tname, PTR_ERR(ret));
+				return -EINVAL;
+			}
+			regs[BPF_REG_0].type = PTR_TO_MEM_OR_NULL;
+			regs[BPF_REG_0].mem_size = tsize;
+		} else {
+			regs[BPF_REG_0].type = PTR_TO_BTF_ID_OR_NULL;
+			regs[BPF_REG_0].btf_id = meta.ret_btf_id;
+		}
 	} else if (fn->ret_type == RET_PTR_TO_BTF_ID_OR_NULL) {
 		int ret_btf_id;
 			dst_reg->mem_size = aux->btf_var.mem_size;
 			break;
 		case PTR_TO_BTF_ID:
+		case PTR_TO_PERCPU_BTF_ID:
 			dst_reg->btf_id = aux->btf_var.btf_id;
 			break;
 		default:
 			       struct bpf_insn *insn,
 			       struct bpf_insn_aux_data *aux)
 {
-	u32 type, id = insn->imm;
+	u32 datasec_id, type, id = insn->imm;
+	const struct btf_var_secinfo *vsi;
+	const struct btf_type *datasec;
 	const struct btf_type *t;
 	const char *sym_name;
+	bool percpu = false;
 	u64 addr;
+	int i;
 	if (!btf_vmlinux) {
 		verbose(env, "kernel is missing BTF, make sure CONFIG_DEBUG_INFO_BTF=y is specified in Kconfig.\n");
 		return -ENOENT;
 	}
+	datasec_id = btf_find_by_name_kind(btf_vmlinux, ".data..percpu",
+					   BTF_KIND_DATASEC);
+	if (datasec_id > 0) {
+		datasec = btf_type_by_id(btf_vmlinux, datasec_id);
+		for_each_vsi(i, datasec, vsi) {
+			if (vsi->type == id) {
+				percpu = true;
+				break;
+			}
+		}
+	}
 	insn[0].imm = (u32)addr;
 	insn[1].imm = addr >> 32;
 	type = t->type;
 	t = btf_type_skip_modifiers(btf_vmlinux, type, NULL);
-	if (!btf_type_is_struct(t)) {
+	if (percpu) {
+		aux->btf_var.reg_type = PTR_TO_PERCPU_BTF_ID;
+		aux->btf_var.btf_id = type;
+	} else if (!btf_type_is_struct(t)) {
 		const struct btf_type *ret;
 		const char *tname;
 		u32 tsize;

kernel/trace/bpf_trace.c+2−0

 		return prog->aux->sleepable ? &bpf_copy_from_user_proto : NULL;
 	case BPF_FUNC_snprintf_btf:
 		return &bpf_snprintf_btf_proto;
+	case BPF_FUNC_bpf_per_cpu_ptr:
+		return &bpf_per_cpu_ptr_proto;
 	default:
 		return NULL;
 	}

tools/include/uapi/linux/bpf.h+18−0

@@ -3686,6 +3686,23 @@ union bpf_attr { · 3685 unmodified lines
3686	* Return	3686	* Return
3687	* The helper returns TC_ACT_REDIRECT on success or	3687	* The helper returns TC_ACT_REDIRECT on success or
3688	* TC_ACT_SHOT on error.	3688	* TC_ACT_SHOT on error.
		3689	*
		3690	* void bpf_per_cpu_ptr(const void percpu_ptr, u32 cpu)
		3691	* Description
		3692	* Take a pointer to a percpu ksym, percpu_ptr, and return a
		3693	* pointer to the percpu kernel variable on cpu. A ksym is an
		3694	* extern variable decorated with '__ksym'. For ksym, there is a
		3695	* global var (either static or global) defined of the same name
		3696	* in the kernel. The ksym is percpu if the global var is percpu.
		3697	* The returned pointer points to the global percpu var on cpu.
		3698	*
		3699	* bpf_per_cpu_ptr() has the same semantic as per_cpu_ptr() in the
		3700	* kernel, except that bpf_per_cpu_ptr() may return NULL. This
		3701	* happens if cpu is larger than nr_cpu_ids. The caller of
		3702	* bpf_per_cpu_ptr() must check the returned value.
		3703	* Return
		3704	* A pointer pointing to the kernel percpu variable on cpu, or
		3705	* NULL, if cpu is invalid.
3689	*/	3706	*/
3690	#define __BPF_FUNC_MAPPER(FN) \	3707	#define __BPF_FUNC_MAPPER(FN) \
3691	FN(unspec), \	3708	FN(unspec), \
@@ -3841,6 +3858,7 @@ union bpf_attr { · 149 unmodified lines
3841	FN(seq_printf_btf), \	3858	FN(seq_printf_btf), \
3842	FN(skb_cgroup_classid), \	3859	FN(skb_cgroup_classid), \
3843	FN(redirect_neigh), \	3860	FN(redirect_neigh), \
		3861	FN(bpf_per_cpu_ptr), \
3844	/* */	3862	/* */
3845		3863
3846	/* integer value in 'imm' field of BPF_CALL instruction selects which helper	3864	/* integer value in 'imm' field of BPF_CALL instruction selects which helper

bpf: Introduce bpf_per_cpu_ptr()

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code