Originally, all of the control requests share a single buffer( ctrl & input & ctrl_status fields in struct virtio_crypto), this allows queue depth 1 only, the performance of control queue gets limited by this design. In this patch, each request allocates request buffer dynamically, and free buffer after request, so the scope protected by ctrl_lock also get optimized here. It's possible to optimize control queue depth in the next step. A necessary comment is already in code, still describe it again: /* * Note: there are padding fields in request, clear them to zero before * sending to host to avoid to divulge any information. * Ex, virtio_crypto_ctrl_request::ctrl::u::destroy_session::padding[48] */ So use kzalloc to allocate buffer of struct virtio_crypto_ctrl_request. Potentially dereferencing uninitialized variables: Reported-by: kernel test robot <lkp@intel.com> Reported-by: Dan Carpenter <dan.carpenter@oracle.com>
Unified · Split
| @@ -108,16 +108,22 @@ static int virtio_crypto_alg_akcipher_init_session(struct virtio_crypto_akcipher · 107 unmodified lines | ||
| 108 | 108 | unsigned int num_out = 0, num_in = 0; |
| 109 | 109 | struct virtio_crypto_op_ctrl_req *ctrl; |
| 110 | 110 | struct virtio_crypto_session_input *input; |
| 111 | + struct virtio_crypto_ctrl_request *vc_ctrl_req; | |
| 111 | 112 | |
| 112 | 113 | pkey = kmemdup(key, keylen, GFP_ATOMIC); |
| 113 | 114 | if (!pkey) |
| 114 | 115 | return -ENOMEM; |
| 115 | 116 | |
| 116 | − spin_lock(&vcrypto->ctrl_lock); | |
| 117 | − ctrl = &vcrypto->ctrl; | |
| 117 | + vc_ctrl_req = kzalloc(sizeof(*vc_ctrl_req), GFP_KERNEL); | |
| 118 | + if (!vc_ctrl_req) { | |
| 119 | + err = -ENOMEM; | |
| 120 | + goto out; | |
| 121 | + } | |
| 122 | + | |
| 123 | + ctrl = &vc_ctrl_req->ctrl; | |
| 118 | 124 | memcpy(&ctrl->header, header, sizeof(ctrl->header)); |
| 119 | 125 | memcpy(&ctrl->u, para, sizeof(ctrl->u)); |
| 120 | − input = &vcrypto->input; | |
| 126 | + input = &vc_ctrl_req->input; | |
| 121 | 127 | input->status = cpu_to_le32(VIRTIO_CRYPTO_ERR); |
| 122 | 128 | |
| 123 | 129 | sg_init_one(&outhdr_sg, ctrl, sizeof(*ctrl)); |
| @@ -129,16 +135,22 @@ static int virtio_crypto_alg_akcipher_init_session(struct virtio_crypto_akcipher · 5 unmodified lines | ||
| 129 | 135 | sg_init_one(&inhdr_sg, input, sizeof(*input)); |
| 130 | 136 | sgs[num_out + num_in++] = &inhdr_sg; |
| 131 | 137 | |
| 138 | + spin_lock(&vcrypto->ctrl_lock); | |
| 132 | 139 | err = virtqueue_add_sgs(vcrypto->ctrl_vq, sgs, num_out, num_in, vcrypto, GFP_ATOMIC); |
| 133 | − if (err < 0) | |
| 140 | + if (err < 0) { | |
| 141 | + spin_unlock(&vcrypto->ctrl_lock); | |
| 134 | 142 | goto out; |
| 143 | + } | |
| 135 | 144 | |
| 136 | 145 | virtqueue_kick(vcrypto->ctrl_vq); |
| 137 | 146 | while (!virtqueue_get_buf(vcrypto->ctrl_vq, &inlen) && |
| 138 | 147 | !virtqueue_is_broken(vcrypto->ctrl_vq)) |
| 139 | 148 | cpu_relax(); |
| 149 | + spin_unlock(&vcrypto->ctrl_lock); | |
| 140 | 150 | |
| 141 | 151 | if (le32_to_cpu(input->status) != VIRTIO_CRYPTO_OK) { |
| 152 | + pr_err("virtio_crypto: Create session failed status: %u\n", | |
| 153 | + le32_to_cpu(input->status)); | |
| 142 | 154 | err = -EINVAL; |
| 143 | 155 | goto out; |
| 144 | 156 | } |
| @@ -148,13 +160,9 @@ static int virtio_crypto_alg_akcipher_init_session(struct virtio_crypto_akcipher · 3 unmodified lines | ||
| 148 | 160 | err = 0; |
| 149 | 161 | |
| 150 | 162 | out: |
| 151 | − spin_unlock(&vcrypto->ctrl_lock); | |
| 163 | + kfree(vc_ctrl_req); | |
| 152 | 164 | kfree_sensitive(pkey); |
| 153 | 165 | |
| 154 | − if (err < 0) | |
| 155 | − pr_err("virtio_crypto: Create session failed status: %u\n", | |
| 156 | − le32_to_cpu(input->status)); | |
| 157 | − | |
| 158 | 166 | return err; |
| 159 | 167 | } |
| 160 | 168 | |
| @@ -167,15 +175,18 @@ static int virtio_crypto_alg_akcipher_close_session(struct virtio_crypto_akciphe · 6 unmodified lines | ||
| 167 | 175 | int err; |
| 168 | 176 | struct virtio_crypto_op_ctrl_req *ctrl; |
| 169 | 177 | struct virtio_crypto_inhdr *ctrl_status; |
| 178 | + struct virtio_crypto_ctrl_request *vc_ctrl_req; | |
| 170 | 179 | |
| 171 | − spin_lock(&vcrypto->ctrl_lock); | |
| 172 | − if (!ctx->session_valid) { | |
| 173 | − err = 0; | |
| 174 | − goto out; | |
| 175 | − } | |
| 176 | − ctrl_status = &vcrypto->ctrl_status; | |
| 180 | + if (!ctx->session_valid) | |
| 181 | + return 0; | |
| 182 | + | |
| 183 | + vc_ctrl_req = kzalloc(sizeof(*vc_ctrl_req), GFP_KERNEL); | |
| 184 | + if (!vc_ctrl_req) | |
| 185 | + return -ENOMEM; | |
| 186 | + | |
| 187 | + ctrl_status = &vc_ctrl_req->ctrl_status; | |
| 177 | 188 | ctrl_status->status = VIRTIO_CRYPTO_ERR; |
| 178 | − ctrl = &vcrypto->ctrl; | |
| 189 | + ctrl = &vc_ctrl_req->ctrl; | |
| 179 | 190 | ctrl->header.opcode = cpu_to_le32(VIRTIO_CRYPTO_AKCIPHER_DESTROY_SESSION); |
| 180 | 191 | ctrl->header.queue_id = 0; |
| 181 | 192 | |
| @@ -188,16 +199,22 @@ static int virtio_crypto_alg_akcipher_close_session(struct virtio_crypto_akciphe · 6 unmodified lines | ||
| 188 | 199 | sg_init_one(&inhdr_sg, &ctrl_status->status, sizeof(ctrl_status->status)); |
| 189 | 200 | sgs[num_out + num_in++] = &inhdr_sg; |
| 190 | 201 | |
| 202 | + spin_lock(&vcrypto->ctrl_lock); | |
| 191 | 203 | err = virtqueue_add_sgs(vcrypto->ctrl_vq, sgs, num_out, num_in, vcrypto, GFP_ATOMIC); |
| 192 | − if (err < 0) | |
| 204 | + if (err < 0) { | |
| 205 | + spin_unlock(&vcrypto->ctrl_lock); | |
| 193 | 206 | goto out; |
| 207 | + } | |
| 194 | 208 | |
| 195 | 209 | virtqueue_kick(vcrypto->ctrl_vq); |
| 196 | 210 | while (!virtqueue_get_buf(vcrypto->ctrl_vq, &inlen) && |
| 197 | 211 | !virtqueue_is_broken(vcrypto->ctrl_vq)) |
| 198 | 212 | cpu_relax(); |
| 213 | + spin_unlock(&vcrypto->ctrl_lock); | |
| 199 | 214 | |
| 200 | 215 | if (ctrl_status->status != VIRTIO_CRYPTO_OK) { |
| 216 | + pr_err("virtio_crypto: Close session failed status: %u, session_id: 0x%llx\n", | |
| 217 | + ctrl_status->status, destroy_session->session_id); | |
| 201 | 218 | err = -EINVAL; |
| 202 | 219 | goto out; |
| 203 | 220 | } |
| @@ -206,11 +223,7 @@ static int virtio_crypto_alg_akcipher_close_session(struct virtio_crypto_akciphe · 2 unmodified lines | ||
| 206 | 223 | ctx->session_valid = false; |
| 207 | 224 | |
| 208 | 225 | out: |
| 209 | − spin_unlock(&vcrypto->ctrl_lock); | |
| 210 | − if (err < 0) { | |
| 211 | − pr_err("virtio_crypto: Close session failed status: %u, session_id: 0x%llx\n", | |
| 212 | − ctrl_status->status, destroy_session->session_id); | |
| 213 | − } | |
| 226 | + kfree(vc_ctrl_req); | |
| 214 | 227 | |
| 215 | 228 | return err; |
| 216 | 229 | } |
| @@ -13,6 +13,7 @@ · 12 unmodified lines | ||
| 13 | 13 | #include <crypto/aead.h> |
| 14 | 14 | #include <crypto/aes.h> |
| 15 | 15 | #include <crypto/engine.h> |
| 16 | +#include <uapi/linux/virtio_crypto.h> | |
| 16 | 17 | |
| 17 | 18 | |
| 18 | 19 | /* Internal representation of a data virtqueue */ |
| @@ -65,11 +66,6 @@ struct virtio_crypto { · 46 unmodified lines | ||
| 65 | 66 | /* Maximum size of per request */ |
| 66 | 67 | u64 max_size; |
| 67 | 68 | |
| 68 | − /* Control VQ buffers: protected by the ctrl_lock */ | |
| 69 | − struct virtio_crypto_op_ctrl_req ctrl; | |
| 70 | − struct virtio_crypto_session_input input; | |
| 71 | − struct virtio_crypto_inhdr ctrl_status; | |
| 72 | − | |
| 73 | 69 | unsigned long status; |
| 74 | 70 | atomic_t ref_count; |
| 75 | 71 | struct list_head list; |
| @@ -85,6 +81,17 @@ struct virtio_crypto_sym_session_info { · 9 unmodified lines | ||
| 85 | 81 | __u64 session_id; |
| 86 | 82 | }; |
| 87 | 83 | |
| 84 | +/* | |
| 85 | + * Note: there are padding fields in request, clear them to zero before | |
| 86 | + * sending to host to avoid to divulge any information. | |
| 87 | + * Ex, virtio_crypto_ctrl_request::ctrl::u::destroy_session::padding[48] | |
| 88 | + */ | |
| 89 | +struct virtio_crypto_ctrl_request { | |
| 90 | + struct virtio_crypto_op_ctrl_req ctrl; | |
| 91 | + struct virtio_crypto_session_input input; | |
| 92 | + struct virtio_crypto_inhdr ctrl_status; | |
| 93 | +}; | |
| 94 | + | |
| 88 | 95 | struct virtio_crypto_request; |
| 89 | 96 | typedef void (*virtio_crypto_data_callback) |
| 90 | 97 | (struct virtio_crypto_request *vc_req, int len); |
| @@ -126,6 +126,7 @@ static int virtio_crypto_alg_skcipher_init_session( · 125 unmodified lines | ||
| 126 | 126 | struct virtio_crypto_op_ctrl_req *ctrl; |
| 127 | 127 | struct virtio_crypto_session_input *input; |
| 128 | 128 | struct virtio_crypto_sym_create_session_req *sym_create_session; |
| 129 | + struct virtio_crypto_ctrl_request *vc_ctrl_req; | |
| 129 | 130 | |
| 130 | 131 | /* |
| 131 | 132 | * Avoid to do DMA from the stack, switch to using |
| @@ -136,15 +137,20 @@ static int virtio_crypto_alg_skcipher_init_session( · 4 unmodified lines | ||
| 136 | 137 | if (!cipher_key) |
| 137 | 138 | return -ENOMEM; |
| 138 | 139 | |
| 139 | − spin_lock(&vcrypto->ctrl_lock); | |
| 140 | + vc_ctrl_req = kzalloc(sizeof(*vc_ctrl_req), GFP_KERNEL); | |
| 141 | + if (!vc_ctrl_req) { | |
| 142 | + err = -ENOMEM; | |
| 143 | + goto out; | |
| 144 | + } | |
| 145 | + | |
| 140 | 146 | /* Pad ctrl header */ |
| 141 | − ctrl = &vcrypto->ctrl; | |
| 147 | + ctrl = &vc_ctrl_req->ctrl; | |
| 142 | 148 | ctrl->header.opcode = cpu_to_le32(VIRTIO_CRYPTO_CIPHER_CREATE_SESSION); |
| 143 | 149 | ctrl->header.algo = cpu_to_le32(alg); |
| 144 | 150 | /* Set the default dataqueue id to 0 */ |
| 145 | 151 | ctrl->header.queue_id = 0; |
| 146 | 152 | |
| 147 | − input = &vcrypto->input; | |
| 153 | + input = &vc_ctrl_req->input; | |
| 148 | 154 | input->status = cpu_to_le32(VIRTIO_CRYPTO_ERR); |
| 149 | 155 | /* Pad cipher's parameters */ |
| 150 | 156 | sym_create_session = &ctrl->u.sym_create_session; |
| @@ -164,12 +170,12 @@ static int virtio_crypto_alg_skcipher_init_session( · 13 unmodified lines | ||
| 164 | 170 | sg_init_one(&inhdr, input, sizeof(*input)); |
| 165 | 171 | sgs[num_out + num_in++] = &inhdr; |
| 166 | 172 | |
| 173 | + spin_lock(&vcrypto->ctrl_lock); | |
| 167 | 174 | err = virtqueue_add_sgs(vcrypto->ctrl_vq, sgs, num_out, |
| 168 | 175 | num_in, vcrypto, GFP_ATOMIC); |
| 169 | 176 | if (err < 0) { |
| 170 | 177 | spin_unlock(&vcrypto->ctrl_lock); |
| 171 | − kfree_sensitive(cipher_key); | |
| 172 | − return err; | |
| 178 | + goto out; | |
| 173 | 179 | } |
| 174 | 180 | virtqueue_kick(vcrypto->ctrl_vq); |
| 175 | 181 | |
| @@ -180,13 +186,13 @@ static int virtio_crypto_alg_skcipher_init_session( · 4 unmodified lines | ||
| 180 | 186 | while (!virtqueue_get_buf(vcrypto->ctrl_vq, &tmp) && |
| 181 | 187 | !virtqueue_is_broken(vcrypto->ctrl_vq)) |
| 182 | 188 | cpu_relax(); |
| 189 | + spin_unlock(&vcrypto->ctrl_lock); | |
| 183 | 190 | |
| 184 | 191 | if (le32_to_cpu(input->status) != VIRTIO_CRYPTO_OK) { |
| 185 | − spin_unlock(&vcrypto->ctrl_lock); | |
| 186 | 192 | pr_err("virtio_crypto: Create session failed status: %u\n", |
| 187 | 193 | le32_to_cpu(input->status)); |
| 188 | − kfree_sensitive(cipher_key); | |
| 189 | − return -EINVAL; | |
| 194 | + err = -EINVAL; | |
| 195 | + goto out; | |
| 190 | 196 | } |
| 191 | 197 | |
| 192 | 198 | if (encrypt) |
| @@ -194,10 +200,11 @@ static int virtio_crypto_alg_skcipher_init_session( · 1 unmodified line | ||
| 194 | 200 | else |
| 195 | 201 | ctx->dec_sess_info.session_id = le64_to_cpu(input->session_id); |
| 196 | 202 | |
| 197 | − spin_unlock(&vcrypto->ctrl_lock); | |
| 198 | − | |
| 203 | + err = 0; | |
| 204 | +out: | |
| 205 | + kfree(vc_ctrl_req); | |
| 199 | 206 | kfree_sensitive(cipher_key); |
| 200 | − return 0; | |
| 207 | + return err; | |
| 201 | 208 | } |
| 202 | 209 | |
| 203 | 210 | static int virtio_crypto_alg_skcipher_close_session( |
| @@ -212,12 +219,16 @@ static int virtio_crypto_alg_skcipher_close_session( · 8 unmodified lines | ||
| 212 | 219 | unsigned int num_out = 0, num_in = 0; |
| 213 | 220 | struct virtio_crypto_op_ctrl_req *ctrl; |
| 214 | 221 | struct virtio_crypto_inhdr *ctrl_status; |
| 222 | + struct virtio_crypto_ctrl_request *vc_ctrl_req; | |
| 215 | 223 | |
| 216 | − spin_lock(&vcrypto->ctrl_lock); | |
| 217 | − ctrl_status = &vcrypto->ctrl_status; | |
| 224 | + vc_ctrl_req = kzalloc(sizeof(*vc_ctrl_req), GFP_KERNEL); | |
| 225 | + if (!vc_ctrl_req) | |
| 226 | + return -ENOMEM; | |
| 227 | + | |
| 228 | + ctrl_status = &vc_ctrl_req->ctrl_status; | |
| 218 | 229 | ctrl_status->status = VIRTIO_CRYPTO_ERR; |
| 219 | 230 | /* Pad ctrl header */ |
| 220 | − ctrl = &vcrypto->ctrl; | |
| 231 | + ctrl = &vc_ctrl_req->ctrl; | |
| 221 | 232 | ctrl->header.opcode = cpu_to_le32(VIRTIO_CRYPTO_CIPHER_DESTROY_SESSION); |
| 222 | 233 | /* Set the default virtqueue id to 0 */ |
| 223 | 234 | ctrl->header.queue_id = 0; |
| @@ -236,28 +247,31 @@ static int virtio_crypto_alg_skcipher_close_session( · 12 unmodified lines | ||
| 236 | 247 | sg_init_one(&status_sg, &ctrl_status->status, sizeof(ctrl_status->status)); |
| 237 | 248 | sgs[num_out + num_in++] = &status_sg; |
| 238 | 249 | |
| 250 | + spin_lock(&vcrypto->ctrl_lock); | |
| 239 | 251 | err = virtqueue_add_sgs(vcrypto->ctrl_vq, sgs, num_out, |
| 240 | 252 | num_in, vcrypto, GFP_ATOMIC); |
| 241 | 253 | if (err < 0) { |
| 242 | 254 | spin_unlock(&vcrypto->ctrl_lock); |
| 243 | − return err; | |
| 255 | + goto out; | |
| 244 | 256 | } |
| 245 | 257 | virtqueue_kick(vcrypto->ctrl_vq); |
| 246 | 258 | |
| 247 | 259 | while (!virtqueue_get_buf(vcrypto->ctrl_vq, &tmp) && |
| 248 | 260 | !virtqueue_is_broken(vcrypto->ctrl_vq)) |
| 249 | 261 | cpu_relax(); |
| 262 | + spin_unlock(&vcrypto->ctrl_lock); | |
| 250 | 263 | |
| 251 | 264 | if (ctrl_status->status != VIRTIO_CRYPTO_OK) { |
| 252 | − spin_unlock(&vcrypto->ctrl_lock); | |
| 253 | 265 | pr_err("virtio_crypto: Close session failed status: %u, session_id: 0x%llx\n", |
| 254 | 266 | ctrl_status->status, destroy_session->session_id); |
| 255 | 267 | |
| 256 | 268 | return -EINVAL; |
| 257 | 269 | } |
| 258 | − spin_unlock(&vcrypto->ctrl_lock); | |
| 259 | 270 | |
| 260 | − return 0; | |
| 271 | + err = 0; | |
| 272 | +out: | |
| 273 | + kfree(vc_ctrl_req); | |
| 274 | + return err; | |
| 261 | 275 | } |
| 262 | 276 | |
| 263 | 277 | static int virtio_crypto_alg_skcipher_init_sessions( |