Merge branch 'access-creds'

The access() (and faccessat()) credentials change can cause an unnecessary load on the RCU machinery because every access() call ends up freeing the temporary access credential using RCU. This isn't really noticeable on small machines, but if you have hundreds of cores you can cause huge slowdowns due to RCU storms. It's easy to avoid: the temporary access crededntials aren't actually normally accessed using RCU at all, so we can avoid the whole issue by just marking them as such. * access-creds: access: avoid the RCU grace period for the temporary subjective credentials

Linus Torvalds committed 7 years ago

Commit a29a0a42 parents bed38c3 d7852fb3 files · +45 −3Browse files

Showing the changes to kernel (1 of 3 files). Show all

Unified · Split

kernel/cred.c+19−2

@@ -144,7 +144,10 @@ void __put_cred(struct cred *cred) · 143 unmodified lines
144	144	BUG_ON(cred == current->cred);
145	145	BUG_ON(cred == current->real_cred);
146	146
147		− call_rcu(&cred->rcu, put_cred_rcu);
	147	+ if (cred->non_rcu)
	148	+ put_cred_rcu(&cred->rcu);
	149	+ else
	150	+ call_rcu(&cred->rcu, put_cred_rcu);
148	151	}
149	152	EXPORT_SYMBOL(__put_cred);
150	153
@@ -261,6 +264,7 @@ struct cred *prepare_creds(void) · 110 unmodified lines
261	264	old = task->cred;
262	265	memcpy(new, old, sizeof(struct cred));
263	266
	267	+ new->non_rcu = 0;
264	268	atomic_set(&new->usage, 1);
265	269	set_cred_subscribers(new, 0);
266	270	get_group_info(new->group_info);
@@ -544,7 +548,19 @@ const struct cred override_creds(const struct cred new) · 277 unmodified lines
544	548
545	549	validate_creds(old);
546	550	validate_creds(new);
547		− get_cred(new);
	551	+
	552	+ /*
	553	+ * NOTE! This uses 'get_new_cred()' rather than 'get_cred()'.
	554	+ *
	555	+ * That means that we do not clear the 'non_rcu' flag, since
	556	+ * we are only installing the cred into the thread-synchronous
	557	+ * '->cred' pointer, not the '->real_cred' pointer that is
	558	+ * visible to other threads under RCU.
	559	+ *
	560	+ * Also note that we did validate_creds() manually, not depending
	561	+ * on the validation in 'get_cred()'.
	562	+ */
	563	+ get_new_cred((struct cred *)new);
548	564	alter_cred_subscribers(new, 1);
549	565	rcu_assign_pointer(current->cred, new);
550	566	alter_cred_subscribers(old, -1);
@@ -681,6 +697,7 @@ struct cred prepare_kernel_cred(struct task_struct daemon) · 130 unmodified lines
681	697	validate_creds(old);
682	698
683	699	new = old;
	700	+ new->non_rcu = 0;
684	701	atomic_set(&new->usage, 1);
685	702	set_cred_subscribers(new, 0);
686	703	get_uid(new->user);