Merge branch 'access-creds'

The access() (and faccessat()) credentials change can cause an unnecessary load on the RCU machinery because every access() call ends up freeing the temporary access credential using RCU. This isn't really noticeable on small machines, but if you have hundreds of cores you can cause huge slowdowns due to RCU storms. It's easy to avoid: the temporary access crededntials aren't actually normally accessed using RCU at all, so we can avoid the whole issue by just marking them as such. * access-creds: access: avoid the RCU grace period for the temporary subjective credentials

Linus Torvalds committed 7 years ago

Commit a29a0a42 parents bed38c3 d7852fb3 files · +45 −3Browse files

Unified · Split

3 files

fs/open.c +19 −0
include/linux/cred.h +7 −1
kernel/cred.c +19 −2

fs/open.c+19−0

@@ -374,6 +374,25 @@ long do_faccessat(int dfd, const char __user *filename, int mode) · 373 unmodified lines
374	374	override_cred->cap_permitted;
375	375	}
376	376
	377	+ /*
	378	+ * The new set of credentials can only be used in
	379	+ * task-synchronous circumstances, and does not need
	380	+ * RCU freeing, unless somebody then takes a separate
	381	+ * reference to it.
	382	+ *
	383	+ * NOTE! This is _only_ true because this credential
	384	+ * is used purely for override_creds() that installs
	385	+ * it as the subjective cred. Other threads will be
	386	+ * accessing ->real_cred, not the subjective cred.
	387	+ *
	388	+ * If somebody _does_ make a copy of this (using the
	389	+ * 'get_current_cred()' function), that will clear the
	390	+ * non_rcu field, because now that other user may be
	391	+ * expecting RCU freeing. But normal thread-synchronous
	392	+ * cred accesses will keep things non-RCY.
	393	+ */
	394	+ override_cred->non_rcu = 1;
	395	+
377	396	old_cred = override_creds(override_cred);
378	397	retry:
379	398	res = user_path_at(dfd, filename, lookup_flags, &path);

include/linux/cred.h+7−1

@@ -145,7 +145,11 @@ struct cred { · 144 unmodified lines
145	145	struct user_struct user; / real user ID subscription */
146	146	struct user_namespace user_ns; / user_ns the caps and keyrings are relative to. */
147	147	struct group_info group_info; / supplementary groups for euid/fsgid */
148		− struct rcu_head rcu; /* RCU deletion hook */
	148	+ /* RCU deletion */
	149	+ union {
	150	+ int non_rcu; /* Can we skip RCU deletion? */
	151	+ struct rcu_head rcu; /* RCU deletion hook */
	152	+ };
149	153	} __randomize_layout;
150	154
151	155	extern void __put_cred(struct cred *);
@@ -246,6 +250,7 @@ static inline const struct cred get_cred(const struct cred cred) · 94 unmodified lines
246	250	if (!cred)
247	251	return cred;
248	252	validate_creds(cred);
	253	+ nonconst_cred->non_rcu = 0;
249	254	return get_new_cred(nonconst_cred);
250	255	}
251	256
@@ -257,6 +262,7 @@ static inline const struct cred get_cred_rcu(const struct cred cred) · 5 unmodified lines
257	262	if (!atomic_inc_not_zero(&nonconst_cred->usage))
258	263	return NULL;
259	264	validate_creds(cred);
	265	+ nonconst_cred->non_rcu = 0;
260	266	return cred;
261	267	}
262	268

kernel/cred.c+19−2

@@ -144,7 +144,10 @@ void __put_cred(struct cred *cred) · 143 unmodified lines
144	144	BUG_ON(cred == current->cred);
145	145	BUG_ON(cred == current->real_cred);
146	146
147		− call_rcu(&cred->rcu, put_cred_rcu);
	147	+ if (cred->non_rcu)
	148	+ put_cred_rcu(&cred->rcu);
	149	+ else
	150	+ call_rcu(&cred->rcu, put_cred_rcu);
148	151	}
149	152	EXPORT_SYMBOL(__put_cred);
150	153
@@ -261,6 +264,7 @@ struct cred *prepare_creds(void) · 110 unmodified lines
261	264	old = task->cred;
262	265	memcpy(new, old, sizeof(struct cred));
263	266
	267	+ new->non_rcu = 0;
264	268	atomic_set(&new->usage, 1);
265	269	set_cred_subscribers(new, 0);
266	270	get_group_info(new->group_info);
@@ -544,7 +548,19 @@ const struct cred override_creds(const struct cred new) · 277 unmodified lines
544	548
545	549	validate_creds(old);
546	550	validate_creds(new);
547		− get_cred(new);
	551	+
	552	+ /*
	553	+ * NOTE! This uses 'get_new_cred()' rather than 'get_cred()'.
	554	+ *
	555	+ * That means that we do not clear the 'non_rcu' flag, since
	556	+ * we are only installing the cred into the thread-synchronous
	557	+ * '->cred' pointer, not the '->real_cred' pointer that is
	558	+ * visible to other threads under RCU.
	559	+ *
	560	+ * Also note that we did validate_creds() manually, not depending
	561	+ * on the validation in 'get_cred()'.
	562	+ */
	563	+ get_new_cred((struct cred *)new);
548	564	alter_cred_subscribers(new, 1);
549	565	rcu_assign_pointer(current->cred, new);
550	566	alter_cred_subscribers(old, -1);
@@ -681,6 +697,7 @@ struct cred prepare_kernel_cred(struct task_struct daemon) · 130 unmodified lines
681	697	validate_creds(old);
682	698
683	699	new = old;
	700	+ new->non_rcu = 0;
684	701	atomic_set(&new->usage, 1);
685	702	set_cred_subscribers(new, 0);
686	703	get_uid(new->user);