The access() (and faccessat()) credentials change can cause an unnecessary load on the RCU machinery because every access() call ends up freeing the temporary access credential using RCU. This isn't really noticeable on small machines, but if you have hundreds of cores you can cause huge slowdowns due to RCU storms. It's easy to avoid: the temporary access crededntials aren't actually normally accessed using RCU at all, so we can avoid the whole issue by just marking them as such. * access-creds: access: avoid the RCU grace period for the temporary subjective credentials
Showing the changes to kernel (1 of 3 files). Show all
Unified · Split
| @@ -144,7 +144,10 @@ void __put_cred(struct cred *cred) · 143 unmodified lines | |||
| 144 | BUG_ON(cred == current->cred); | 144 | BUG_ON(cred == current->cred); |
| 145 | BUG_ON(cred == current->real_cred); | 145 | BUG_ON(cred == current->real_cred); |
| 146 | 146 | ||
| 147 | call_rcu(&cred->rcu, put_cred_rcu); | 147 | if (cred->non_rcu) |
| 148 | put_cred_rcu(&cred->rcu); | ||
| 149 | else | ||
| 150 | call_rcu(&cred->rcu, put_cred_rcu); | ||
| 148 | } | 151 | } |
| 149 | EXPORT_SYMBOL(__put_cred); | 152 | EXPORT_SYMBOL(__put_cred); |
| 150 | 153 | ||
| @@ -261,6 +264,7 @@ struct cred *prepare_creds(void) · 110 unmodified lines | |||
| 261 | old = task->cred; | 264 | old = task->cred; |
| 262 | memcpy(new, old, sizeof(struct cred)); | 265 | memcpy(new, old, sizeof(struct cred)); |
| 263 | 266 | ||
| 267 | new->non_rcu = 0; | ||
| 264 | atomic_set(&new->usage, 1); | 268 | atomic_set(&new->usage, 1); |
| 265 | set_cred_subscribers(new, 0); | 269 | set_cred_subscribers(new, 0); |
| 266 | get_group_info(new->group_info); | 270 | get_group_info(new->group_info); |
| @@ -544,7 +548,19 @@ const struct cred *override_creds(const struct cred *new) · 277 unmodified lines | |||
| 544 | 548 | ||
| 545 | validate_creds(old); | 549 | validate_creds(old); |
| 546 | validate_creds(new); | 550 | validate_creds(new); |
| 547 | get_cred(new); | 551 | |
| 552 | /* | ||
| 553 | * NOTE! This uses 'get_new_cred()' rather than 'get_cred()'. | ||
| 554 | * | ||
| 555 | * That means that we do not clear the 'non_rcu' flag, since | ||
| 556 | * we are only installing the cred into the thread-synchronous | ||
| 557 | * '->cred' pointer, not the '->real_cred' pointer that is | ||
| 558 | * visible to other threads under RCU. | ||
| 559 | * | ||
| 560 | * Also note that we did validate_creds() manually, not depending | ||
| 561 | * on the validation in 'get_cred()'. | ||
| 562 | */ | ||
| 563 | get_new_cred((struct cred *)new); | ||
| 548 | alter_cred_subscribers(new, 1); | 564 | alter_cred_subscribers(new, 1); |
| 549 | rcu_assign_pointer(current->cred, new); | 565 | rcu_assign_pointer(current->cred, new); |
| 550 | alter_cred_subscribers(old, -1); | 566 | alter_cred_subscribers(old, -1); |
| @@ -681,6 +697,7 @@ struct cred *prepare_kernel_cred(struct task_struct *daemon) · 130 unmodified lines | |||
| 681 | validate_creds(old); | 697 | validate_creds(old); |
| 682 | 698 | ||
| 683 | *new = *old; | 699 | *new = *old; |
| 700 | new->non_rcu = 0; | ||
| 684 | atomic_set(&new->usage, 1); | 701 | atomic_set(&new->usage, 1); |
| 685 | set_cred_subscribers(new, 0); | 702 | set_cred_subscribers(new, 0); |
| 686 | get_uid(new->user); | 703 | get_uid(new->user); |