padata_do_parallel currently returns -EINVAL if the callback CPU isn't in the callback cpumask. pcrypt tries to prevent this situation by keeping its own callback cpumask in sync with padata's and checks that the callback CPU it passes to padata is valid. Make padata handle this instead. padata_do_parallel now takes a pointer to the callback CPU and updates it for the caller if an alternate CPU is used. Overall behavior in terms of which callback CPUs are chosen stays the same. Prepares for removal of the padata cpumask notifier in pcrypt, which will fix a lockdep complaint about nested acquisition of the CPU hotplug lock later in the series.
Unified · Split
| @@ -57,35 +57,6 @@ struct pcrypt_aead_ctx { · 56 unmodified lines | ||
| 57 | 57 | unsigned int cb_cpu; |
| 58 | 58 | }; |
| 59 | 59 | |
| 60 | −static int pcrypt_do_parallel(struct padata_priv *padata, unsigned int *cb_cpu, | |
| 61 | − struct padata_pcrypt *pcrypt) | |
| 62 | −{ | |
| 63 | − unsigned int cpu_index, cpu, i; | |
| 64 | − struct pcrypt_cpumask *cpumask; | |
| 65 | − | |
| 66 | − cpu = *cb_cpu; | |
| 67 | − | |
| 68 | − rcu_read_lock_bh(); | |
| 69 | − cpumask = rcu_dereference_bh(pcrypt->cb_cpumask); | |
| 70 | − if (cpumask_test_cpu(cpu, cpumask->mask)) | |
| 71 | − goto out; | |
| 72 | − | |
| 73 | − if (!cpumask_weight(cpumask->mask)) | |
| 74 | − goto out; | |
| 75 | − | |
| 76 | − cpu_index = cpu % cpumask_weight(cpumask->mask); | |
| 77 | − | |
| 78 | − cpu = cpumask_first(cpumask->mask); | |
| 79 | − for (i = 0; i < cpu_index; i++) | |
| 80 | − cpu = cpumask_next(cpu, cpumask->mask); | |
| 81 | − | |
| 82 | − *cb_cpu = cpu; | |
| 83 | − | |
| 84 | −out: | |
| 85 | − rcu_read_unlock_bh(); | |
| 86 | − return padata_do_parallel(pcrypt->pinst, padata, cpu); | |
| 87 | −} | |
| 88 | − | |
| 89 | 60 | static int pcrypt_aead_setkey(struct crypto_aead *parent, |
| 90 | 61 | const u8 *key, unsigned int keylen) |
| 91 | 62 | { |
| @@ -157,7 +128,7 @@ static int pcrypt_aead_encrypt(struct aead_request *req) · 65 unmodified lines | ||
| 157 | 128 | req->cryptlen, req->iv); |
| 158 | 129 | aead_request_set_ad(creq, req->assoclen); |
| 159 | 130 | |
| 160 | − err = pcrypt_do_parallel(padata, &ctx->cb_cpu, &pencrypt); | |
| 131 | + err = padata_do_parallel(pencrypt.pinst, padata, &ctx->cb_cpu); | |
| 161 | 132 | if (!err) |
| 162 | 133 | return -EINPROGRESS; |
| 163 | 134 | |
| @@ -199,7 +170,7 @@ static int pcrypt_aead_decrypt(struct aead_request *req) · 35 unmodified lines | ||
| 199 | 170 | req->cryptlen, req->iv); |
| 200 | 171 | aead_request_set_ad(creq, req->assoclen); |
| 201 | 172 | |
| 202 | − err = pcrypt_do_parallel(padata, &ctx->cb_cpu, &pdecrypt); | |
| 173 | + err = padata_do_parallel(pdecrypt.pinst, padata, &ctx->cb_cpu); | |
| 203 | 174 | if (!err) |
| 204 | 175 | return -EINPROGRESS; |
| 205 | 176 | |
| @@ -154,7 +154,7 @@ struct padata_instance { · 153 unmodified lines | ||
| 154 | 154 | extern struct padata_instance *padata_alloc_possible(const char *name); |
| 155 | 155 | extern void padata_free(struct padata_instance *pinst); |
| 156 | 156 | extern int padata_do_parallel(struct padata_instance *pinst, |
| 157 | − struct padata_priv *padata, int cb_cpu); | |
| 157 | + struct padata_priv *padata, int *cb_cpu); | |
| 158 | 158 | extern void padata_do_serial(struct padata_priv *padata); |
| 159 | 159 | extern int padata_set_cpumask(struct padata_instance *pinst, int cpumask_type, |
| 160 | 160 | cpumask_var_t cpumask); |
| @@ -94,17 +94,19 @@ static void padata_parallel_worker(struct work_struct *parallel_work) · 93 unmodified lines | ||
| 94 | 94 | * |
| 95 | 95 | * @pinst: padata instance |
| 96 | 96 | * @padata: object to be parallelized |
| 97 | − * @cb_cpu: cpu the serialization callback function will run on, | |
| 98 | − * must be in the serial cpumask of padata(i.e. cpumask.cbcpu). | |
| 97 | + * @cb_cpu: pointer to the CPU that the serialization callback function should | |
| 98 | + * run on. If it's not in the serial cpumask of @pinst | |
| 99 | + * (i.e. cpumask.cbcpu), this function selects a fallback CPU and if | |
| 100 | + * none found, returns -EINVAL. | |
| 99 | 101 | * |
| 100 | 102 | * The parallelization callback function will run with BHs off. |
| 101 | 103 | * Note: Every object which is parallelized by padata_do_parallel |
| 102 | 104 | * must be seen by padata_do_serial. |
| 103 | 105 | */ |
| 104 | 106 | int padata_do_parallel(struct padata_instance *pinst, |
| 105 | − struct padata_priv *padata, int cb_cpu) | |
| 107 | + struct padata_priv *padata, int *cb_cpu) | |
| 106 | 108 | { |
| 107 | − int target_cpu, err; | |
| 109 | + int i, cpu, cpu_index, target_cpu, err; | |
| 108 | 110 | struct padata_parallel_queue *queue; |
| 109 | 111 | struct parallel_data *pd; |
| 110 | 112 | |
| @@ -116,8 +118,19 @@ int padata_do_parallel(struct padata_instance *pinst, · 5 unmodified lines | ||
| 116 | 118 | if (!(pinst->flags & PADATA_INIT) || pinst->flags & PADATA_INVALID) |
| 117 | 119 | goto out; |
| 118 | 120 | |
| 119 | − if (!cpumask_test_cpu(cb_cpu, pd->cpumask.cbcpu)) | |
| 120 | − goto out; | |
| 121 | + if (!cpumask_test_cpu(*cb_cpu, pd->cpumask.cbcpu)) { | |
| 122 | + if (!cpumask_weight(pd->cpumask.cbcpu)) | |
| 123 | + goto out; | |
| 124 | + | |
| 125 | + /* Select an alternate fallback CPU and notify the caller. */ | |
| 126 | + cpu_index = *cb_cpu % cpumask_weight(pd->cpumask.cbcpu); | |
| 127 | + | |
| 128 | + cpu = cpumask_first(pd->cpumask.cbcpu); | |
| 129 | + for (i = 0; i < cpu_index; i++) | |
| 130 | + cpu = cpumask_next(cpu, pd->cpumask.cbcpu); | |
| 131 | + | |
| 132 | + *cb_cpu = cpu; | |
| 133 | + } | |
| 121 | 134 | |
| 122 | 135 | err = -EBUSY; |
| 123 | 136 | if ((pinst->flags & PADATA_RESET)) |
| @@ -129,7 +142,7 @@ int padata_do_parallel(struct padata_instance *pinst, · 5 unmodified lines | ||
| 129 | 142 | err = 0; |
| 130 | 143 | atomic_inc(&pd->refcnt); |
| 131 | 144 | padata->pd = pd; |
| 132 | − padata->cb_cpu = cb_cpu; | |
| 145 | + padata->cb_cpu = *cb_cpu; | |
| 133 | 146 | |
| 134 | 147 | target_cpu = padata_cpu_hash(pd); |
| 135 | 148 | padata->cpu = target_cpu; |