bpf: Remove gfp_flags plumbing from bpf_local_storage_update()

Remove the check that rejects sleepable BPF programs from doing BPF_ANY/BPF_EXIST updates on local storage. This restriction was added in commit b00fa38a9c1c ("bpf: Enable non-atomic allocations in local storage") because kzalloc(GFP_KERNEL) could sleep inside local_storage->lock. This is no longer a concern: all local storage allocations now use kmalloc_nolock() which never sleeps. In addition, since kmalloc_nolock() only accepts __GFP_ACCOUNT, __GFP_ZERO and __GFP_NO_OBJ_EXT, the gfp_flags parameter plumbing from bpf_*_storage_get() to bpf_local_storage_update() becomes dead code. Remove gfp_flags from bpf_selem_alloc(), bpf_local_storage_alloc() and bpf_local_storage_update(). Drop the hidden 5th argument from bpf_*_storage_get helpers, and remove the verifier patching that injected GFP_KERNEL/GFP_ATOMIC into the fifth argument.

Amery Hung committed 7 months ago

Commit 136deea1 parent 5063e777 files · +30 −67Browse files

Unified · Split

7 files

include/linux/bpf_local_storage.h +3 −4
kernel/bpf/bpf_cgrp_storage.c +4 −5
kernel/bpf/bpf_inode_storage.c +4 −5
kernel/bpf/bpf_local_storage.c +6 −10
kernel/bpf/bpf_task_storage.c +4 −5
kernel/bpf/verifier.c +0 −26
net/core/bpf_sk_storage.c +9 −12

include/linux/bpf_local_storage.h+3−4

 struct bpf_local_storage_elem *
 bpf_selem_alloc(struct bpf_local_storage_map *smap, void *owner, void *value,
-		bool swap_uptrs, gfp_t gfp_flags);
+		bool swap_uptrs);
 void bpf_selem_free(struct bpf_local_storage_elem *selem,
 		    bool reuse_now);
 int
 bpf_local_storage_alloc(void *owner,
 			struct bpf_local_storage_map *smap,
-			struct bpf_local_storage_elem *first_selem,
+			struct bpf_local_storage_elem *first_selem);
-			gfp_t gfp_flags);
 struct bpf_local_storage_data *
 bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap,
-			 void *value, u64 map_flags, bool swap_uptrs, gfp_t gfp_flags);
+			 void *value, u64 map_flags, bool swap_uptrs);
 u64 bpf_local_storage_map_mem_usage(const struct bpf_map *map);

kernel/bpf/bpf_cgrp_storage.c+4−5

 		return PTR_ERR(cgroup);
 	sdata = bpf_local_storage_update(cgroup, (struct bpf_local_storage_map *)map,
-					 value, map_flags, false, GFP_ATOMIC);
+					 value, map_flags, false);
 	cgroup_put(cgroup);
 	return PTR_ERR_OR_ZERO(sdata);
 }
 	bpf_local_storage_map_free(map, &cgroup_cache);
 }
-/* *gfp_flags* is a hidden argument provided by the verifier */
+BPF_CALL_4(bpf_cgrp_storage_get, struct bpf_map *, map, struct cgroup *, cgroup,
-BPF_CALL_5(bpf_cgrp_storage_get, struct bpf_map *, map, struct cgroup *, cgroup,
+	   void *, value, u64, flags)
-	   void *, value, u64, flags, gfp_t, gfp_flags)
 {
 	struct bpf_local_storage_data *sdata;
 	if (!percpu_ref_is_dying(&cgroup->self.refcnt) &&
 	    (flags & BPF_LOCAL_STORAGE_GET_F_CREATE))
 		sdata = bpf_local_storage_update(cgroup, (struct bpf_local_storage_map *)map,
-						 value, BPF_NOEXIST, false, gfp_flags);
+						 value, BPF_NOEXIST, false);
 out:
 	return IS_ERR_OR_NULL(sdata) ? (unsigned long)NULL : (unsigned long)sdata->data;

kernel/bpf/bpf_inode_storage.c+4−5

 	sdata = bpf_local_storage_update(file_inode(fd_file(f)),
 					 (struct bpf_local_storage_map *)map,
-					 value, map_flags, false, GFP_ATOMIC);
+					 value, map_flags, false);
 	return PTR_ERR_OR_ZERO(sdata);
 }
 	return inode_storage_delete(file_inode(fd_file(f)), map);
 }
-/* *gfp_flags* is a hidden argument provided by the verifier */
+BPF_CALL_4(bpf_inode_storage_get, struct bpf_map *, map, struct inode *, inode,
-BPF_CALL_5(bpf_inode_storage_get, struct bpf_map *, map, struct inode *, inode,
+	   void *, value, u64, flags)
-	   void *, value, u64, flags, gfp_t, gfp_flags)
 {
 	struct bpf_local_storage_data *sdata;
 	if (flags & BPF_LOCAL_STORAGE_GET_F_CREATE) {
 		sdata = bpf_local_storage_update(
 			inode, (struct bpf_local_storage_map *)map, value,
-			BPF_NOEXIST, false, gfp_flags);
+			BPF_NOEXIST, false);
 		return IS_ERR(sdata) ? (unsigned long)NULL :
 					     (unsigned long)sdata->data;
 	}

kernel/bpf/bpf_local_storage.c+6−10

 struct bpf_local_storage_elem *
 bpf_selem_alloc(struct bpf_local_storage_map *smap, void *owner,
-		void *value, bool swap_uptrs, gfp_t gfp_flags)
+		void *value, bool swap_uptrs)
 {
 	struct bpf_local_storage_elem *selem;
 int bpf_local_storage_alloc(void *owner,
 			    struct bpf_local_storage_map *smap,
-			    struct bpf_local_storage_elem *first_selem,
+			    struct bpf_local_storage_elem *first_selem)
-			    gfp_t gfp_flags)
 {
 	struct bpf_local_storage *prev_storage, *storage;
 	struct bpf_local_storage **owner_storage_ptr;
  */
 struct bpf_local_storage_data *
 bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap,
-			 void *value, u64 map_flags, bool swap_uptrs, gfp_t gfp_flags)
+			 void *value, u64 map_flags, bool swap_uptrs)
 {
 	struct bpf_local_storage_data *old_sdata = NULL;
 	struct bpf_local_storage_elem *alloc_selem, *selem = NULL;
 		     !btf_record_has_field(smap->map.record, BPF_SPIN_LOCK)))
 		return ERR_PTR(-EINVAL);
-	if (gfp_flags == GFP_KERNEL && (map_flags & ~BPF_F_LOCK) != BPF_NOEXIST)
-		return ERR_PTR(-EINVAL);
 	local_storage = rcu_dereference_check(*owner_storage(smap, owner),
 					      bpf_rcu_lock_held());
 	if (!local_storage || hlist_empty(&local_storage->list)) {
 		if (err)
 			return ERR_PTR(err);
-		selem = bpf_selem_alloc(smap, owner, value, swap_uptrs, gfp_flags);
+		selem = bpf_selem_alloc(smap, owner, value, swap_uptrs);
 		if (!selem)
 			return ERR_PTR(-ENOMEM);
-		err = bpf_local_storage_alloc(owner, smap, selem, gfp_flags);
+		err = bpf_local_storage_alloc(owner, smap, selem);
 		if (err) {
 			bpf_selem_free(selem, true);
 			mem_uncharge(smap, owner, smap->elem_size);
 	/* A lookup has just been done before and concluded a new selem is
 	 * needed. The chance of an unnecessary alloc is unlikely.
 	 */
-	alloc_selem = selem = bpf_selem_alloc(smap, owner, value, swap_uptrs, gfp_flags);
+	alloc_selem = selem = bpf_selem_alloc(smap, owner, value, swap_uptrs);
 	if (!alloc_selem)
 		return ERR_PTR(-ENOMEM);

kernel/bpf/bpf_task_storage.c+4−5

 	sdata = bpf_local_storage_update(
 		task, (struct bpf_local_storage_map *)map, value, map_flags,
-		true, GFP_ATOMIC);
+		true);
 	err = PTR_ERR_OR_ZERO(sdata);
 out:
 	return err;
 }
-/* *gfp_flags* is a hidden argument provided by the verifier */
+BPF_CALL_4(bpf_task_storage_get, struct bpf_map *, map, struct task_struct *,
-BPF_CALL_5(bpf_task_storage_get, struct bpf_map *, map, struct task_struct *,
+	   task, void *, value, u64, flags)
-	   task, void *, value, u64, flags, gfp_t, gfp_flags)
 {
 	struct bpf_local_storage_data *sdata;
 	    (flags & BPF_LOCAL_STORAGE_GET_F_CREATE)) {
 		sdata = bpf_local_storage_update(
 			task, (struct bpf_local_storage_map *)map, value,
-			BPF_NOEXIST, false, gfp_flags);
+			BPF_NOEXIST, false);
 		return IS_ERR(sdata) ? (unsigned long)NULL : (unsigned long)sdata->data;
 	}

kernel/bpf/verifier.c+0−26

@@ -589,14 +589,6 @@ static bool is_may_goto_insn_at(struct bpf_verifier_env *env, int insn_idx) · 588 unmodified lines
589	return bpf_is_may_goto_insn(&env->prog->insnsi[insn_idx]);	589	return bpf_is_may_goto_insn(&env->prog->insnsi[insn_idx]);
590	}	590	}
591		591
592	static bool is_storage_get_function(enum bpf_func_id func_id)
593	{
594	return func_id == BPF_FUNC_sk_storage_get \|\|
595	func_id == BPF_FUNC_inode_storage_get \|\|
596	func_id == BPF_FUNC_task_storage_get \|\|
597	func_id == BPF_FUNC_cgrp_storage_get;
598	}
599
600	static bool helper_multiple_ref_obj_use(enum bpf_func_id func_id,	592	static bool helper_multiple_ref_obj_use(enum bpf_func_id func_id,
601	const struct bpf_map *map)	593	const struct bpf_map *map)
602	{	594	{
@@ -24429,24 +24421,6 @@ static int do_misc_fixups(struct bpf_verifier_env *env) · 23826 unmodified lines
24429	goto patch_call_imm;	24421	goto patch_call_imm;
24430	}	24422	}
24431		24423
24432	if (is_storage_get_function(insn->imm)) {
24433	if (env->insn_aux_data[i + delta].non_sleepable)
24434	insn_buf[0] = BPF_MOV64_IMM(BPF_REG_5, (__force __s32)GFP_ATOMIC);
24435	else
24436	insn_buf[0] = BPF_MOV64_IMM(BPF_REG_5, (__force __s32)GFP_KERNEL);
24437	insn_buf[1] = *insn;
24438	cnt = 2;
24439
24440	new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
24441	if (!new_prog)
24442	return -ENOMEM;
24443
24444	delta += cnt - 1;
24445	env->prog = prog = new_prog;
24446	insn = new_prog->insnsi + i + delta;
24447	goto patch_call_imm;
24448	}
24449
24450	/* bpf_per_cpu_ptr() and bpf_this_cpu_ptr() */	24424	/* bpf_per_cpu_ptr() and bpf_this_cpu_ptr() */
24451	if (env->insn_aux_data[i + delta].call_with_percpu_alloc_ptr) {	24425	if (env->insn_aux_data[i + delta].call_with_percpu_alloc_ptr) {
24452	/* patch with 'r1 = (u64 )(r1 + 0)' since for percpu data,	24426	/* patch with 'r1 = (u64 )(r1 + 0)' since for percpu data,

net/core/bpf_sk_storage.c+9−12

 	if (sock) {
 		sdata = bpf_local_storage_update(
 			sock->sk, (struct bpf_local_storage_map *)map, value,
-			map_flags, false, GFP_ATOMIC);
+			map_flags, false);
 		sockfd_put(sock);
 		return PTR_ERR_OR_ZERO(sdata);
 	}
 {
 	struct bpf_local_storage_elem *copy_selem;
-	copy_selem = bpf_selem_alloc(smap, newsk, NULL, false, GFP_ATOMIC);
+	copy_selem = bpf_selem_alloc(smap, newsk, NULL, false);
 	if (!copy_selem)
 		return NULL;
 			}
 			bpf_selem_link_storage_nolock(new_sk_storage, copy_selem);
 		} else {
-			ret = bpf_local_storage_alloc(newsk, smap, copy_selem, GFP_ATOMIC);
+			ret = bpf_local_storage_alloc(newsk, smap, copy_selem);
 			if (ret) {
 				bpf_selem_free(copy_selem, true);
 				atomic_sub(smap->elem_size,
 	return ret;
 }
-/* *gfp_flags* is a hidden argument provided by the verifier */
+BPF_CALL_4(bpf_sk_storage_get, struct bpf_map *, map, struct sock *, sk,
-BPF_CALL_5(bpf_sk_storage_get, struct bpf_map *, map, struct sock *, sk,
+	   void *, value, u64, flags)
-	   void *, value, u64, flags, gfp_t, gfp_flags)
 {
 	struct bpf_local_storage_data *sdata;
 	    refcount_inc_not_zero(&sk->sk_refcnt)) {
 		sdata = bpf_local_storage_update(
 			sk, (struct bpf_local_storage_map *)map, value,
-			BPF_NOEXIST, false, gfp_flags);
+			BPF_NOEXIST, false);
 		/* sk must be a fullsock (guaranteed by verifier),
 		 * so sock_gen_put() is unnecessary.
 		 */
 	return false;
 }
-/* *gfp_flags* is a hidden argument provided by the verifier */
+BPF_CALL_4(bpf_sk_storage_get_tracing, struct bpf_map *, map, struct sock *, sk,
-BPF_CALL_5(bpf_sk_storage_get_tracing, struct bpf_map *, map, struct sock *, sk,
+	   void *, value, u64, flags)
-	   void *, value, u64, flags, gfp_t, gfp_flags)
 {
 	WARN_ON_ONCE(!bpf_rcu_lock_held());
 	if (in_hardirq() || in_nmi())
 		return (unsigned long)NULL;
-	return (unsigned long)____bpf_sk_storage_get(map, sk, value, flags,
+	return (unsigned long)____bpf_sk_storage_get(map, sk, value, flags);
-						     gfp_flags);
 }
 BPF_CALL_2(bpf_sk_storage_delete_tracing, struct bpf_map *, map,