History

Merge tag 'for_linus' of git://git.kernel.org/pub/scm/linux/kernel/git/mst/vhost

Pull virtio updates from Michael Tsirkin: - vdpa generic device type support - more virtio hardening for broken devices (but on the same theme, revert some virtio hotplug hardening patches - they were misusing some interrupt flags and had to be reverted) - RSS support in virtio-net - max device MTU support in mlx5 vdpa - akcipher support in virtio-crypto - shared IRQ support in ifcvf vdpa - a minor performance improvement in vhost - enable virtio mem for ARM64 - beginnings of advance dma support - cleanups, fixes all over the place * tag 'for_linus' of git://git.kernel.org/pub/scm/linux/kernel/git/mst/vhost: (33 commits) vdpa/mlx5: Avoid processing works if workqueue was destroyed vhost: handle error while adding split ranges to iotlb vdpa: support exposing the count of vqs to userspace vdpa: change the type of nvqs to u32 vdpa: support exposing the config size to userspace vdpa/mlx5: re-create forwarding rules after mac modified virtio: pci: check bar values read from virtio config space Revert "virtio_pci: harden MSI-X interrupts" Revert "virtio-pci: harden INTX interrupts" drivers/net/virtio_net: Added RSS hash report control. drivers/net/virtio_net: Added RSS hash report. drivers/net/virtio_net: Added basic RSS support. drivers/net/virtio_net: Fixed padded vheader to use v1 with hash. virtio: use virtio_device_ready() in virtio_device_restore() tools/virtio: compile with -pthread tools/virtio: fix after premapped buf support virtio_ring: remove flags check for unmap packed indirect desc virtio_ring: remove flags check for unmap split indirect desc virtio_ring: rename vring_unmap_state_packed() to vring_unmap_extra_packed() net/mlx5: Add support for configuring max device MTU ...

Linus Torvalds committed 4 years ago

Commit f4f5d7c2 parents e729dbe ad6dc1d32 files · +1628 −286Browse files

Unified · Split

32 files
  1. drivers/crypto/virtio/Kconfig +3 −0
  2. drivers/crypto/virtio/Makefile +2 −1
  3. drivers/crypto/virtio/virtio_crypto_akcipher_algs.c +585 −0
  4. drivers/crypto/virtio/virtio_crypto_common.h +5 −2
  5. drivers/crypto/virtio/virtio_crypto_core.c +5 −1
  6. drivers/crypto/virtio/virtio_crypto_mgr.c +14 −3
  7. drivers/crypto/virtio/virtio_crypto_skcipher_algs.c +2 −2
  8. drivers/net/virtio_net.c +376 −13
  9. drivers/vdpa/ifcvf/ifcvf_base.c +44 −80
  10. drivers/vdpa/ifcvf/ifcvf_base.h +17 −7
  11. drivers/vdpa/ifcvf/ifcvf_main.c +284 −33
  12. drivers/vdpa/mlx5/net/mlx5_vnet.c +80 −4
  13. drivers/vdpa/vdpa.c +3 −3
  14. drivers/vhost/iotlb.c +5 −1
  15. drivers/vhost/vdpa.c +40 −5
  16. drivers/vhost/vhost.c +2 −1
  17. drivers/virtio/Kconfig +4 −3
  18. drivers/virtio/virtio.c +3 −2
  19. drivers/virtio/virtio_pci_common.c +7 −41
  20. drivers/virtio/virtio_pci_common.h +2 −5
  21. drivers/virtio/virtio_pci_legacy.c +2 −3
  22. drivers/virtio/virtio_pci_modern.c +11 −7
  23. drivers/virtio/virtio_pci_modern_dev.c +8 −1
  24. drivers/virtio/virtio_ring.c +18 −35
  25. include/linux/balloon_compaction.h +0 −22
  26. include/linux/vdpa.h +5 −4
  27. include/uapi/linux/vhost.h +7 −0
  28. include/uapi/linux/virtio_config.h +6 −0
  29. include/uapi/linux/virtio_crypto.h +81 −1
  30. mm/balloon_compaction.c +3 −3
  31. tools/virtio/Makefile +2 −1
  32. tools/virtio/linux/dma-mapping.h +2 −2
drivers/crypto/virtio/Kconfig+3−0
@@ -3,8 +3,11 @@ config CRYPTO_DEV_VIRTIO · 2 unmodified lines
33 tristate "VirtIO crypto driver"
44 depends on VIRTIO
55 select CRYPTO_AEAD
6+ select CRYPTO_AKCIPHER2
67 select CRYPTO_SKCIPHER
78 select CRYPTO_ENGINE
9+ select CRYPTO_RSA
10+ select MPILIB
811 help
912 This driver provides support for virtio crypto device. If you
1013 choose 'M' here, this module will be called virtio_crypto.
drivers/crypto/virtio/Makefile+2−1
@@ -1,6 +1,7 @@
11 # SPDX-License-Identifier: GPL-2.0
22 obj-$(CONFIG_CRYPTO_DEV_VIRTIO) += virtio_crypto.o
33 virtio_crypto-objs := \
4− virtio_crypto_algs.o \
4+ virtio_crypto_skcipher_algs.o \
5+ virtio_crypto_akcipher_algs.o \
56 virtio_crypto_mgr.o \
67 virtio_crypto_core.o
drivers/crypto/virtio/virtio_crypto_akcipher_algs.c+585−0
@@ -0,0 +1,585 @@
1+// SPDX-License-Identifier: GPL-2.0-or-later
2+ /* Asymmetric algorithms supported by virtio crypto device
3+ *
4+ * Authors: zhenwei pi <pizhenwei@bytedance.com>
5+ * lei he <helei.sig11@bytedance.com>
6+ *
7+ * Copyright 2022 Bytedance CO., LTD.
8+ */
9+
10+#include <linux/mpi.h>
11+#include <linux/scatterlist.h>
12+#include <crypto/algapi.h>
13+#include <crypto/internal/akcipher.h>
14+#include <crypto/internal/rsa.h>
15+#include <linux/err.h>
16+#include <crypto/scatterwalk.h>
17+#include <linux/atomic.h>
18+
19+#include <uapi/linux/virtio_crypto.h>
20+#include "virtio_crypto_common.h"
21+
22+struct virtio_crypto_rsa_ctx {
23+ MPI n;
24+};
25+
26+struct virtio_crypto_akcipher_ctx {
27+ struct crypto_engine_ctx enginectx;
28+ struct virtio_crypto *vcrypto;
29+ struct crypto_akcipher *tfm;
30+ bool session_valid;
31+ __u64 session_id;
32+ union {
33+ struct virtio_crypto_rsa_ctx rsa_ctx;
34+ };
35+};
36+
37+struct virtio_crypto_akcipher_request {
38+ struct virtio_crypto_request base;
39+ struct virtio_crypto_akcipher_ctx *akcipher_ctx;
40+ struct akcipher_request *akcipher_req;
41+ void *src_buf;
42+ void *dst_buf;
43+ uint32_t opcode;
44+};
45+
46+struct virtio_crypto_akcipher_algo {
47+ uint32_t algonum;
48+ uint32_t service;
49+ unsigned int active_devs;
50+ struct akcipher_alg algo;
51+};
52+
53+static DEFINE_MUTEX(algs_lock);
54+
55+static void virtio_crypto_akcipher_finalize_req(
56+ struct virtio_crypto_akcipher_request *vc_akcipher_req,
57+ struct akcipher_request *req, int err)
58+{
59+ virtcrypto_clear_request(&vc_akcipher_req->base);
60+
61+ crypto_finalize_akcipher_request(vc_akcipher_req->base.dataq->engine, req, err);
62+}
63+
64+static void virtio_crypto_dataq_akcipher_callback(struct virtio_crypto_request *vc_req, int len)
65+{
66+ struct virtio_crypto_akcipher_request *vc_akcipher_req =
67+ container_of(vc_req, struct virtio_crypto_akcipher_request, base);
68+ struct akcipher_request *akcipher_req;
69+ int error;
70+
71+ switch (vc_req->status) {
72+ case VIRTIO_CRYPTO_OK:
73+ error = 0;
74+ break;
75+ case VIRTIO_CRYPTO_INVSESS:
76+ case VIRTIO_CRYPTO_ERR:
77+ error = -EINVAL;
78+ break;
79+ case VIRTIO_CRYPTO_BADMSG:
80+ error = -EBADMSG;
81+ break;
82+
83+ case VIRTIO_CRYPTO_KEY_REJECTED:
84+ error = -EKEYREJECTED;
85+ break;
86+
87+ default:
88+ error = -EIO;
89+ break;
90+ }
91+
92+ akcipher_req = vc_akcipher_req->akcipher_req;
93+ if (vc_akcipher_req->opcode != VIRTIO_CRYPTO_AKCIPHER_VERIFY)
94+ sg_copy_from_buffer(akcipher_req->dst, sg_nents(akcipher_req->dst),
95+ vc_akcipher_req->dst_buf, akcipher_req->dst_len);
96+ virtio_crypto_akcipher_finalize_req(vc_akcipher_req, akcipher_req, error);
97+}
98+
99+static int virtio_crypto_alg_akcipher_init_session(struct virtio_crypto_akcipher_ctx *ctx,
100+ struct virtio_crypto_ctrl_header *header, void *para,
101+ const uint8_t *key, unsigned int keylen)
102+{
103+ struct scatterlist outhdr_sg, key_sg, inhdr_sg, *sgs[3];
104+ struct virtio_crypto *vcrypto = ctx->vcrypto;
105+ uint8_t *pkey;
106+ unsigned int inlen;
107+ int err;
108+ unsigned int num_out = 0, num_in = 0;
109+
110+ pkey = kmemdup(key, keylen, GFP_ATOMIC);
111+ if (!pkey)
112+ return -ENOMEM;
113+
114+ spin_lock(&vcrypto->ctrl_lock);
115+ memcpy(&vcrypto->ctrl.header, header, sizeof(vcrypto->ctrl.header));
116+ memcpy(&vcrypto->ctrl.u, para, sizeof(vcrypto->ctrl.u));
117+ vcrypto->input.status = cpu_to_le32(VIRTIO_CRYPTO_ERR);
118+
119+ sg_init_one(&outhdr_sg, &vcrypto->ctrl, sizeof(vcrypto->ctrl));
120+ sgs[num_out++] = &outhdr_sg;
121+
122+ sg_init_one(&key_sg, pkey, keylen);
123+ sgs[num_out++] = &key_sg;
124+
125+ sg_init_one(&inhdr_sg, &vcrypto->input, sizeof(vcrypto->input));
126+ sgs[num_out + num_in++] = &inhdr_sg;
127+
128+ err = virtqueue_add_sgs(vcrypto->ctrl_vq, sgs, num_out, num_in, vcrypto, GFP_ATOMIC);
129+ if (err < 0)
130+ goto out;
131+
132+ virtqueue_kick(vcrypto->ctrl_vq);
133+ while (!virtqueue_get_buf(vcrypto->ctrl_vq, &inlen) &&
134+ !virtqueue_is_broken(vcrypto->ctrl_vq))
135+ cpu_relax();
136+
137+ if (le32_to_cpu(vcrypto->input.status) != VIRTIO_CRYPTO_OK) {
138+ err = -EINVAL;
139+ goto out;
140+ }
141+
142+ ctx->session_id = le64_to_cpu(vcrypto->input.session_id);
143+ ctx->session_valid = true;
144+ err = 0;
145+
146+out:
147+ spin_unlock(&vcrypto->ctrl_lock);
148+ kfree_sensitive(pkey);
149+
150+ if (err < 0)
151+ pr_err("virtio_crypto: Create session failed status: %u\n",
152+ le32_to_cpu(vcrypto->input.status));
153+
154+ return err;
155+}
156+
157+static int virtio_crypto_alg_akcipher_close_session(struct virtio_crypto_akcipher_ctx *ctx)
158+{
159+ struct scatterlist outhdr_sg, inhdr_sg, *sgs[2];
160+ struct virtio_crypto_destroy_session_req *destroy_session;
161+ struct virtio_crypto *vcrypto = ctx->vcrypto;
162+ unsigned int num_out = 0, num_in = 0, inlen;
163+ int err;
164+
165+ spin_lock(&vcrypto->ctrl_lock);
166+ if (!ctx->session_valid) {
167+ err = 0;
168+ goto out;
169+ }
170+ vcrypto->ctrl_status.status = VIRTIO_CRYPTO_ERR;
171+ vcrypto->ctrl.header.opcode = cpu_to_le32(VIRTIO_CRYPTO_AKCIPHER_DESTROY_SESSION);
172+ vcrypto->ctrl.header.queue_id = 0;
173+
174+ destroy_session = &vcrypto->ctrl.u.destroy_session;
175+ destroy_session->session_id = cpu_to_le64(ctx->session_id);
176+
177+ sg_init_one(&outhdr_sg, &vcrypto->ctrl, sizeof(vcrypto->ctrl));
178+ sgs[num_out++] = &outhdr_sg;
179+
180+ sg_init_one(&inhdr_sg, &vcrypto->ctrl_status.status, sizeof(vcrypto->ctrl_status.status));
181+ sgs[num_out + num_in++] = &inhdr_sg;
182+
183+ err = virtqueue_add_sgs(vcrypto->ctrl_vq, sgs, num_out, num_in, vcrypto, GFP_ATOMIC);
184+ if (err < 0)
185+ goto out;
186+
187+ virtqueue_kick(vcrypto->ctrl_vq);
188+ while (!virtqueue_get_buf(vcrypto->ctrl_vq, &inlen) &&
189+ !virtqueue_is_broken(vcrypto->ctrl_vq))
190+ cpu_relax();
191+
192+ if (vcrypto->ctrl_status.status != VIRTIO_CRYPTO_OK) {
193+ err = -EINVAL;
194+ goto out;
195+ }
196+
197+ err = 0;
198+ ctx->session_valid = false;
199+
200+out:
201+ spin_unlock(&vcrypto->ctrl_lock);
202+ if (err < 0) {
203+ pr_err("virtio_crypto: Close session failed status: %u, session_id: 0x%llx\n",
204+ vcrypto->ctrl_status.status, destroy_session->session_id);
205+ }
206+
207+ return err;
208+}
209+
210+static int __virtio_crypto_akcipher_do_req(struct virtio_crypto_akcipher_request *vc_akcipher_req,
211+ struct akcipher_request *req, struct data_queue *data_vq)
212+{
213+ struct virtio_crypto_akcipher_ctx *ctx = vc_akcipher_req->akcipher_ctx;
214+ struct virtio_crypto_request *vc_req = &vc_akcipher_req->base;
215+ struct virtio_crypto *vcrypto = ctx->vcrypto;
216+ struct virtio_crypto_op_data_req *req_data = vc_req->req_data;
217+ struct scatterlist *sgs[4], outhdr_sg, inhdr_sg, srcdata_sg, dstdata_sg;
218+ void *src_buf = NULL, *dst_buf = NULL;
219+ unsigned int num_out = 0, num_in = 0;
220+ int node = dev_to_node(&vcrypto->vdev->dev);
221+ unsigned long flags;
222+ int ret = -ENOMEM;
223+ bool verify = vc_akcipher_req->opcode == VIRTIO_CRYPTO_AKCIPHER_VERIFY;
224+ unsigned int src_len = verify ? req->src_len + req->dst_len : req->src_len;
225+
226+ /* out header */
227+ sg_init_one(&outhdr_sg, req_data, sizeof(*req_data));
228+ sgs[num_out++] = &outhdr_sg;
229+
230+ /* src data */
231+ src_buf = kcalloc_node(src_len, 1, GFP_KERNEL, node);
232+ if (!src_buf)
233+ goto err;
234+
235+ if (verify) {
236+ /* for verify operation, both src and dst data work as OUT direction */
237+ sg_copy_to_buffer(req->src, sg_nents(req->src), src_buf, src_len);
238+ sg_init_one(&srcdata_sg, src_buf, src_len);
239+ sgs[num_out++] = &srcdata_sg;
240+ } else {
241+ sg_copy_to_buffer(req->src, sg_nents(req->src), src_buf, src_len);
242+ sg_init_one(&srcdata_sg, src_buf, src_len);
243+ sgs[num_out++] = &srcdata_sg;
244+
245+ /* dst data */
246+ dst_buf = kcalloc_node(req->dst_len, 1, GFP_KERNEL, node);
247+ if (!dst_buf)
248+ goto err;
249+
250+ sg_init_one(&dstdata_sg, dst_buf, req->dst_len);
251+ sgs[num_out + num_in++] = &dstdata_sg;
252+ }
253+
254+ vc_akcipher_req->src_buf = src_buf;
255+ vc_akcipher_req->dst_buf = dst_buf;
256+
257+ /* in header */
258+ sg_init_one(&inhdr_sg, &vc_req->status, sizeof(vc_req->status));
259+ sgs[num_out + num_in++] = &inhdr_sg;
260+
261+ spin_lock_irqsave(&data_vq->lock, flags);
262+ ret = virtqueue_add_sgs(data_vq->vq, sgs, num_out, num_in, vc_req, GFP_ATOMIC);
263+ virtqueue_kick(data_vq->vq);
264+ spin_unlock_irqrestore(&data_vq->lock, flags);
265+ if (ret)
266+ goto err;
267+
268+ return 0;
269+
270+err:
271+ kfree(src_buf);
272+ kfree(dst_buf);
273+
274+ return -ENOMEM;
275+}
276+
277+static int virtio_crypto_rsa_do_req(struct crypto_engine *engine, void *vreq)
278+{
279+ struct akcipher_request *req = container_of(vreq, struct akcipher_request, base);
280+ struct virtio_crypto_akcipher_request *vc_akcipher_req = akcipher_request_ctx(req);
281+ struct virtio_crypto_request *vc_req = &vc_akcipher_req->base;
282+ struct virtio_crypto_akcipher_ctx *ctx = vc_akcipher_req->akcipher_ctx;
283+ struct virtio_crypto *vcrypto = ctx->vcrypto;
284+ struct data_queue *data_vq = vc_req->dataq;
285+ struct virtio_crypto_op_header *header;
286+ struct virtio_crypto_akcipher_data_req *akcipher_req;
287+ int ret;
288+
289+ vc_req->sgs = NULL;
290+ vc_req->req_data = kzalloc_node(sizeof(*vc_req->req_data),
291+ GFP_KERNEL, dev_to_node(&vcrypto->vdev->dev));
292+ if (!vc_req->req_data)
293+ return -ENOMEM;
294+
295+ /* build request header */
296+ header = &vc_req->req_data->header;
297+ header->opcode = cpu_to_le32(vc_akcipher_req->opcode);
298+ header->algo = cpu_to_le32(VIRTIO_CRYPTO_AKCIPHER_RSA);
299+ header->session_id = cpu_to_le64(ctx->session_id);
300+
301+ /* build request akcipher data */
302+ akcipher_req = &vc_req->req_data->u.akcipher_req;
303+ akcipher_req->para.src_data_len = cpu_to_le32(req->src_len);
304+ akcipher_req->para.dst_data_len = cpu_to_le32(req->dst_len);
305+
306+ ret = __virtio_crypto_akcipher_do_req(vc_akcipher_req, req, data_vq);
307+ if (ret < 0) {
308+ kfree_sensitive(vc_req->req_data);
309+ vc_req->req_data = NULL;
310+ return ret;
311+ }
312+
313+ return 0;
314+}
315+
316+static int virtio_crypto_rsa_req(struct akcipher_request *req, uint32_t opcode)
317+{
318+ struct crypto_akcipher *atfm = crypto_akcipher_reqtfm(req);
319+ struct virtio_crypto_akcipher_ctx *ctx = akcipher_tfm_ctx(atfm);
320+ struct virtio_crypto_akcipher_request *vc_akcipher_req = akcipher_request_ctx(req);
321+ struct virtio_crypto_request *vc_req = &vc_akcipher_req->base;
322+ struct virtio_crypto *vcrypto = ctx->vcrypto;
323+ /* Use the first data virtqueue as default */
324+ struct data_queue *data_vq = &vcrypto->data_vq[0];
325+
326+ vc_req->dataq = data_vq;
327+ vc_req->alg_cb = virtio_crypto_dataq_akcipher_callback;
328+ vc_akcipher_req->akcipher_ctx = ctx;
329+ vc_akcipher_req->akcipher_req = req;
330+ vc_akcipher_req->opcode = opcode;
331+
332+ return crypto_transfer_akcipher_request_to_engine(data_vq->engine, req);
333+}
334+
335+static int virtio_crypto_rsa_encrypt(struct akcipher_request *req)
336+{
337+ return virtio_crypto_rsa_req(req, VIRTIO_CRYPTO_AKCIPHER_ENCRYPT);
338+}
339+
340+static int virtio_crypto_rsa_decrypt(struct akcipher_request *req)
341+{
342+ return virtio_crypto_rsa_req(req, VIRTIO_CRYPTO_AKCIPHER_DECRYPT);
343+}
344+
345+static int virtio_crypto_rsa_sign(struct akcipher_request *req)
346+{
347+ return virtio_crypto_rsa_req(req, VIRTIO_CRYPTO_AKCIPHER_SIGN);
348+}
349+
350+static int virtio_crypto_rsa_verify(struct akcipher_request *req)
351+{
352+ return virtio_crypto_rsa_req(req, VIRTIO_CRYPTO_AKCIPHER_VERIFY);
353+}
354+
355+static int virtio_crypto_rsa_set_key(struct crypto_akcipher *tfm,
356+ const void *key,
357+ unsigned int keylen,
358+ bool private,
359+ int padding_algo,
360+ int hash_algo)
361+{
362+ struct virtio_crypto_akcipher_ctx *ctx = akcipher_tfm_ctx(tfm);
363+ struct virtio_crypto_rsa_ctx *rsa_ctx = &ctx->rsa_ctx;
364+ struct virtio_crypto *vcrypto;
365+ struct virtio_crypto_ctrl_header header;
366+ struct virtio_crypto_akcipher_session_para para;
367+ struct rsa_key rsa_key = {0};
368+ int node = virtio_crypto_get_current_node();
369+ uint32_t keytype;
370+ int ret;
371+
372+ /* mpi_free will test n, just free it. */
373+ mpi_free(rsa_ctx->n);
374+ rsa_ctx->n = NULL;
375+
376+ if (private) {
377+ keytype = VIRTIO_CRYPTO_AKCIPHER_KEY_TYPE_PRIVATE;
378+ ret = rsa_parse_priv_key(&rsa_key, key, keylen);
379+ } else {
380+ keytype = VIRTIO_CRYPTO_AKCIPHER_KEY_TYPE_PUBLIC;
381+ ret = rsa_parse_pub_key(&rsa_key, key, keylen);
382+ }
383+
384+ if (ret)
385+ return ret;
386+
387+ rsa_ctx->n = mpi_read_raw_data(rsa_key.n, rsa_key.n_sz);
388+ if (!rsa_ctx->n)
389+ return -ENOMEM;
390+
391+ if (!ctx->vcrypto) {
392+ vcrypto = virtcrypto_get_dev_node(node, VIRTIO_CRYPTO_SERVICE_AKCIPHER,
393+ VIRTIO_CRYPTO_AKCIPHER_RSA);
394+ if (!vcrypto) {
395+ pr_err("virtio_crypto: Could not find a virtio device in the system or unsupported algo\n");
396+ return -ENODEV;
397+ }
398+
399+ ctx->vcrypto = vcrypto;
400+ } else {
401+ virtio_crypto_alg_akcipher_close_session(ctx);
402+ }
403+
404+ /* set ctrl header */
405+ header.opcode = cpu_to_le32(VIRTIO_CRYPTO_AKCIPHER_CREATE_SESSION);
406+ header.algo = cpu_to_le32(VIRTIO_CRYPTO_AKCIPHER_RSA);
407+ header.queue_id = 0;
408+
409+ /* set RSA para */
410+ para.algo = cpu_to_le32(VIRTIO_CRYPTO_AKCIPHER_RSA);
411+ para.keytype = cpu_to_le32(keytype);
412+ para.keylen = cpu_to_le32(keylen);
413+ para.u.rsa.padding_algo = cpu_to_le32(padding_algo);
414+ para.u.rsa.hash_algo = cpu_to_le32(hash_algo);
415+
416+ return virtio_crypto_alg_akcipher_init_session(ctx, &header, &para, key, keylen);
417+}
418+
419+static int virtio_crypto_rsa_raw_set_priv_key(struct crypto_akcipher *tfm,
420+ const void *key,
421+ unsigned int keylen)
422+{
423+ return virtio_crypto_rsa_set_key(tfm, key, keylen, 1,
424+ VIRTIO_CRYPTO_RSA_RAW_PADDING,
425+ VIRTIO_CRYPTO_RSA_NO_HASH);
426+}
427+
428+
429+static int virtio_crypto_p1pad_rsa_sha1_set_priv_key(struct crypto_akcipher *tfm,
430+ const void *key,
431+ unsigned int keylen)
432+{
433+ return virtio_crypto_rsa_set_key(tfm, key, keylen, 1,
434+ VIRTIO_CRYPTO_RSA_PKCS1_PADDING,
435+ VIRTIO_CRYPTO_RSA_SHA1);
436+}
437+
438+static int virtio_crypto_rsa_raw_set_pub_key(struct crypto_akcipher *tfm,
439+ const void *key,
440+ unsigned int keylen)
441+{
442+ return virtio_crypto_rsa_set_key(tfm, key, keylen, 0,
443+ VIRTIO_CRYPTO_RSA_RAW_PADDING,
444+ VIRTIO_CRYPTO_RSA_NO_HASH);
445+}
446+
447+static int virtio_crypto_p1pad_rsa_sha1_set_pub_key(struct crypto_akcipher *tfm,
448+ const void *key,
449+ unsigned int keylen)
450+{
451+ return virtio_crypto_rsa_set_key(tfm, key, keylen, 0,
452+ VIRTIO_CRYPTO_RSA_PKCS1_PADDING,
453+ VIRTIO_CRYPTO_RSA_SHA1);
454+}
455+
456+static unsigned int virtio_crypto_rsa_max_size(struct crypto_akcipher *tfm)
457+{
458+ struct virtio_crypto_akcipher_ctx *ctx = akcipher_tfm_ctx(tfm);
459+ struct virtio_crypto_rsa_ctx *rsa_ctx = &ctx->rsa_ctx;
460+
461+ return mpi_get_size(rsa_ctx->n);
462+}
463+
464+static int virtio_crypto_rsa_init_tfm(struct crypto_akcipher *tfm)
465+{
466+ struct virtio_crypto_akcipher_ctx *ctx = akcipher_tfm_ctx(tfm);
467+
468+ ctx->tfm = tfm;
469+ ctx->enginectx.op.do_one_request = virtio_crypto_rsa_do_req;
470+ ctx->enginectx.op.prepare_request = NULL;
471+ ctx->enginectx.op.unprepare_request = NULL;
472+
473+ return 0;
474+}
475+
476+static void virtio_crypto_rsa_exit_tfm(struct crypto_akcipher *tfm)
477+{
478+ struct virtio_crypto_akcipher_ctx *ctx = akcipher_tfm_ctx(tfm);
479+ struct virtio_crypto_rsa_ctx *rsa_ctx = &ctx->rsa_ctx;
480+
481+ virtio_crypto_alg_akcipher_close_session(ctx);
482+ virtcrypto_dev_put(ctx->vcrypto);
483+ mpi_free(rsa_ctx->n);
484+ rsa_ctx->n = NULL;
485+}
486+
487+static struct virtio_crypto_akcipher_algo virtio_crypto_akcipher_algs[] = {
488+ {
489+ .algonum = VIRTIO_CRYPTO_AKCIPHER_RSA,
490+ .service = VIRTIO_CRYPTO_SERVICE_AKCIPHER,
491+ .algo = {
492+ .encrypt = virtio_crypto_rsa_encrypt,
493+ .decrypt = virtio_crypto_rsa_decrypt,
494+ .set_pub_key = virtio_crypto_rsa_raw_set_pub_key,
495+ .set_priv_key = virtio_crypto_rsa_raw_set_priv_key,
496+ .max_size = virtio_crypto_rsa_max_size,
497+ .init = virtio_crypto_rsa_init_tfm,
498+ .exit = virtio_crypto_rsa_exit_tfm,
499+ .reqsize = sizeof(struct virtio_crypto_akcipher_request),
500+ .base = {
501+ .cra_name = "rsa",
502+ .cra_driver_name = "virtio-crypto-rsa",
503+ .cra_priority = 150,
504+ .cra_module = THIS_MODULE,
505+ .cra_ctxsize = sizeof(struct virtio_crypto_akcipher_ctx),
506+ },
507+ },
508+ },
509+ {
510+ .algonum = VIRTIO_CRYPTO_AKCIPHER_RSA,
511+ .service = VIRTIO_CRYPTO_SERVICE_AKCIPHER,
512+ .algo = {
513+ .encrypt = virtio_crypto_rsa_encrypt,
514+ .decrypt = virtio_crypto_rsa_decrypt,
515+ .sign = virtio_crypto_rsa_sign,
516+ .verify = virtio_crypto_rsa_verify,
517+ .set_pub_key = virtio_crypto_p1pad_rsa_sha1_set_pub_key,
518+ .set_priv_key = virtio_crypto_p1pad_rsa_sha1_set_priv_key,
519+ .max_size = virtio_crypto_rsa_max_size,
520+ .init = virtio_crypto_rsa_init_tfm,
521+ .exit = virtio_crypto_rsa_exit_tfm,
522+ .reqsize = sizeof(struct virtio_crypto_akcipher_request),
523+ .base = {
524+ .cra_name = "pkcs1pad(rsa,sha1)",
525+ .cra_driver_name = "virtio-pkcs1-rsa-with-sha1",
526+ .cra_priority = 150,
527+ .cra_module = THIS_MODULE,
528+ .cra_ctxsize = sizeof(struct virtio_crypto_akcipher_ctx),
529+ },
530+ },
531+ },
532+};
533+
534+int virtio_crypto_akcipher_algs_register(struct virtio_crypto *vcrypto)
535+{
536+ int ret = 0;
537+ int i = 0;
538+
539+ mutex_lock(&algs_lock);
540+
541+ for (i = 0; i < ARRAY_SIZE(virtio_crypto_akcipher_algs); i++) {
542+ uint32_t service = virtio_crypto_akcipher_algs[i].service;
543+ uint32_t algonum = virtio_crypto_akcipher_algs[i].algonum;
544+
545+ if (!virtcrypto_algo_is_supported(vcrypto, service, algonum))
546+ continue;
547+
548+ if (virtio_crypto_akcipher_algs[i].active_devs == 0) {
549+ ret = crypto_register_akcipher(&virtio_crypto_akcipher_algs[i].algo);
550+ if (ret)
551+ goto unlock;
552+ }
553+
554+ virtio_crypto_akcipher_algs[i].active_devs++;
555+ dev_info(&vcrypto->vdev->dev, "Registered akcipher algo %s\n",
556+ virtio_crypto_akcipher_algs[i].algo.base.cra_name);
557+ }
558+
559+unlock:
560+ mutex_unlock(&algs_lock);
561+ return ret;
562+}
563+
564+void virtio_crypto_akcipher_algs_unregister(struct virtio_crypto *vcrypto)
565+{
566+ int i = 0;
567+
568+ mutex_lock(&algs_lock);
569+
570+ for (i = 0; i < ARRAY_SIZE(virtio_crypto_akcipher_algs); i++) {
571+ uint32_t service = virtio_crypto_akcipher_algs[i].service;
572+ uint32_t algonum = virtio_crypto_akcipher_algs[i].algonum;
573+
574+ if (virtio_crypto_akcipher_algs[i].active_devs == 0 ||
575+ !virtcrypto_algo_is_supported(vcrypto, service, algonum))
576+ continue;
577+
578+ if (virtio_crypto_akcipher_algs[i].active_devs == 1)
579+ crypto_unregister_akcipher(&virtio_crypto_akcipher_algs[i].algo);
580+
581+ virtio_crypto_akcipher_algs[i].active_devs--;
582+ }
583+
584+ mutex_unlock(&algs_lock);
585+}
drivers/crypto/virtio/virtio_crypto_common.h+5−2
@@ -56,6 +56,7 @@ struct virtio_crypto { · 55 unmodified lines
5656 u32 mac_algo_l;
5757 u32 mac_algo_h;
5858 u32 aead_algo;
59+ u32 akcipher_algo;
5960
6061 /* Maximum length of cipher key */
6162 u32 max_cipher_key_len;
@@ -129,7 +130,9 @@ static inline int virtio_crypto_get_current_node(void) · 67 unmodified lines
129130 return node;
130131 }
131132
132−int virtio_crypto_algs_register(struct virtio_crypto *vcrypto);
133−void virtio_crypto_algs_unregister(struct virtio_crypto *vcrypto);
133+int virtio_crypto_skcipher_algs_register(struct virtio_crypto *vcrypto);
134+void virtio_crypto_skcipher_algs_unregister(struct virtio_crypto *vcrypto);
135+int virtio_crypto_akcipher_algs_register(struct virtio_crypto *vcrypto);
136+void virtio_crypto_akcipher_algs_unregister(struct virtio_crypto *vcrypto);
134137
135138 #endif /* _VIRTIO_CRYPTO_COMMON_H */
drivers/crypto/virtio/virtio_crypto_core.c+5−1
@@ -297,6 +297,7 @@ static int virtcrypto_probe(struct virtio_device *vdev) · 296 unmodified lines
297297 u32 mac_algo_l = 0;
298298 u32 mac_algo_h = 0;
299299 u32 aead_algo = 0;
300+ u32 akcipher_algo = 0;
300301 u32 crypto_services = 0;
301302
302303 if (!virtio_has_feature(vdev, VIRTIO_F_VERSION_1))
@@ -348,6 +349,9 @@ static int virtcrypto_probe(struct virtio_device *vdev) · 45 unmodified lines
348349 mac_algo_h, &mac_algo_h);
349350 virtio_cread_le(vdev, struct virtio_crypto_config,
350351 aead_algo, &aead_algo);
352+ if (crypto_services & (1 << VIRTIO_CRYPTO_SERVICE_AKCIPHER))
353+ virtio_cread_le(vdev, struct virtio_crypto_config,
354+ akcipher_algo, &akcipher_algo);
351355
352356 /* Add virtio crypto device to global table */
353357 err = virtcrypto_devmgr_add_dev(vcrypto);
@@ -374,7 +378,7 @@ static int virtcrypto_probe(struct virtio_device *vdev) · 20 unmodified lines
374378 vcrypto->mac_algo_h = mac_algo_h;
375379 vcrypto->hash_algo = hash_algo;
376380 vcrypto->aead_algo = aead_algo;
377−
381+ vcrypto->akcipher_algo = akcipher_algo;
378382
379383 dev_info(&vdev->dev,
380384 "max_queues: %u, max_cipher_key_len: %u, max_auth_key_len: %u, max_size 0x%llx\n",
drivers/crypto/virtio/virtio_crypto_mgr.c+14−3
@@ -237,8 +237,14 @@ struct virtio_crypto *virtcrypto_get_dev_node(int node, uint32_t service, · 236 unmodified lines
237237 */
238238 int virtcrypto_dev_start(struct virtio_crypto *vcrypto)
239239 {
240− if (virtio_crypto_algs_register(vcrypto)) {
241− pr_err("virtio_crypto: Failed to register crypto algs\n");
240+ if (virtio_crypto_skcipher_algs_register(vcrypto)) {
241+ pr_err("virtio_crypto: Failed to register crypto skcipher algs\n");
242+ return -EFAULT;
243+ }
244+
245+ if (virtio_crypto_akcipher_algs_register(vcrypto)) {
246+ pr_err("virtio_crypto: Failed to register crypto akcipher algs\n");
247+ virtio_crypto_skcipher_algs_unregister(vcrypto);
242248 return -EFAULT;
243249 }
244250
@@ -257,7 +263,8 @@ int virtcrypto_dev_start(struct virtio_crypto *vcrypto) · 12 unmodified lines
257263 */
258264 void virtcrypto_dev_stop(struct virtio_crypto *vcrypto)
259265 {
260− virtio_crypto_algs_unregister(vcrypto);
266+ virtio_crypto_skcipher_algs_unregister(vcrypto);
267+ virtio_crypto_akcipher_algs_unregister(vcrypto);
261268 }
262269
263270 /*
@@ -312,6 +319,10 @@ bool virtcrypto_algo_is_supported(struct virtio_crypto *vcrypto, · 48 unmodified lines
312319 case VIRTIO_CRYPTO_SERVICE_AEAD:
313320 algo_mask = vcrypto->aead_algo;
314321 break;
322+
323+ case VIRTIO_CRYPTO_SERVICE_AKCIPHER:
324+ algo_mask = vcrypto->akcipher_algo;
325+ break;
315326 }
316327
317328 if (!(algo_mask & (1u << algo)))
drivers/crypto/virtio/virtio_crypto_skcipher_algs.crenamed from drivers/crypto/virtio/virtio_crypto_algs.c+2−2
@@ -613,7 +613,7 @@ static struct virtio_crypto_algo virtio_crypto_algs[] = { { · 612 unmodified lines
613613 },
614614 } };
615615
616−int virtio_crypto_algs_register(struct virtio_crypto *vcrypto)
616+int virtio_crypto_skcipher_algs_register(struct virtio_crypto *vcrypto)
617617 {
618618 int ret = 0;
619619 int i = 0;
@@ -644,7 +644,7 @@ unlock: · 24 unmodified lines
644644 return ret;
645645 }
646646
647−void virtio_crypto_algs_unregister(struct virtio_crypto *vcrypto)
647+void virtio_crypto_skcipher_algs_unregister(struct virtio_crypto *vcrypto)
648648 {
649649 int i = 0;
650650
drivers/net/virtio_net.c+376−13
@@ -169,6 +169,24 @@ struct receive_queue { · 168 unmodified lines
169169 struct xdp_rxq_info xdp_rxq;
170170 };
171171
172+/* This structure can contain rss message with maximum settings for indirection table and keysize
173+ * Note, that default structure that describes RSS configuration virtio_net_rss_config
174+ * contains same info but can't handle table values.
175+ * In any case, structure would be passed to virtio hw through sg_buf split by parts
176+ * because table sizes may be differ according to the device configuration.
177+ */
178+#define VIRTIO_NET_RSS_MAX_KEY_SIZE 40
179+#define VIRTIO_NET_RSS_MAX_TABLE_LEN 128
180+struct virtio_net_ctrl_rss {
181+ u32 hash_types;
182+ u16 indirection_table_mask;
183+ u16 unclassified_queue;
184+ u16 indirection_table[VIRTIO_NET_RSS_MAX_TABLE_LEN];
185+ u16 max_tx_vq;
186+ u8 hash_key_length;
187+ u8 key[VIRTIO_NET_RSS_MAX_KEY_SIZE];
188+};
189+
172190 /* Control VQ buffers: protected by the rtnl lock */
173191 struct control_buf {
174192 struct virtio_net_ctrl_hdr hdr;
@@ -178,6 +196,7 @@ struct control_buf { · 3 unmodified lines
178196 u8 allmulti;
179197 __virtio16 vid;
180198 __virtio64 offloads;
199+ struct virtio_net_ctrl_rss rss;
181200 };
182201
183202 struct virtnet_info {
@@ -206,6 +225,14 @@ struct virtnet_info { · 22 unmodified lines
206225 /* Host will merge rx buffers for big packets (shake it! shake it!) */
207226 bool mergeable_rx_bufs;
208227
228+ /* Host supports rss and/or hash report */
229+ bool has_rss;
230+ bool has_rss_hash_report;
231+ u8 rss_key_size;
232+ u16 rss_indir_table_size;
233+ u32 rss_hash_types_supported;
234+ u32 rss_hash_types_saved;
235+
209236 /* Has control virtqueue */
210237 bool has_cvq;
211238
@@ -242,13 +269,13 @@ struct virtnet_info { · 30 unmodified lines
242269 };
243270
244271 struct padded_vnet_hdr {
245− struct virtio_net_hdr_mrg_rxbuf hdr;
272+ struct virtio_net_hdr_v1_hash hdr;
246273 /*
247274 * hdr is in a separate sg buffer, and data sg buffer shares same page
248275 * with this header sg. This padding makes next sg 16 byte aligned
249276 * after the header.
250277 */
251− char padding[4];
278+ char padding[12];
252279 };
253280
254281 static bool is_xdp_frame(void *ptr)
@@ -396,7 +423,7 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi, · 141 unmodified lines
396423
397424 hdr_len = vi->hdr_len;
398425 if (vi->mergeable_rx_bufs)
399− hdr_padded_len = sizeof(*hdr);
426+ hdr_padded_len = hdr_len;
400427 else
401428 hdr_padded_len = sizeof(struct padded_vnet_hdr);
402429
@@ -1123,6 +1150,35 @@ xdp_xmit: · 720 unmodified lines
11231150 return NULL;
11241151 }
11251152
1153+static void virtio_skb_set_hash(const struct virtio_net_hdr_v1_hash *hdr_hash,
1154+ struct sk_buff *skb)
1155+{
1156+ enum pkt_hash_types rss_hash_type;
1157+
1158+ if (!hdr_hash || !skb)
1159+ return;
1160+
1161+ switch ((int)hdr_hash->hash_report) {
1162+ case VIRTIO_NET_HASH_REPORT_TCPv4:
1163+ case VIRTIO_NET_HASH_REPORT_UDPv4:
1164+ case VIRTIO_NET_HASH_REPORT_TCPv6:
1165+ case VIRTIO_NET_HASH_REPORT_UDPv6:
1166+ case VIRTIO_NET_HASH_REPORT_TCPv6_EX:
1167+ case VIRTIO_NET_HASH_REPORT_UDPv6_EX:
1168+ rss_hash_type = PKT_HASH_TYPE_L4;
1169+ break;
1170+ case VIRTIO_NET_HASH_REPORT_IPv4:
1171+ case VIRTIO_NET_HASH_REPORT_IPv6:
1172+ case VIRTIO_NET_HASH_REPORT_IPv6_EX:
1173+ rss_hash_type = PKT_HASH_TYPE_L3;
1174+ break;
1175+ case VIRTIO_NET_HASH_REPORT_NONE:
1176+ default:
1177+ rss_hash_type = PKT_HASH_TYPE_NONE;
1178+ }
1179+ skb_set_hash(skb, (unsigned int)hdr_hash->hash_value, rss_hash_type);
1180+}
1181+
11261182 static void receive_buf(struct virtnet_info *vi, struct receive_queue *rq,
11271183 void *buf, unsigned int len, void **ctx,
11281184 unsigned int *xdp_xmit,
@@ -1157,6 +1213,8 @@ static void receive_buf(struct virtnet_info *vi, struct receive_queue *rq, · 28 unmodified lines
11571213 return;
11581214
11591215 hdr = skb_vnet_hdr(skb);
1216+ if (dev->features & NETIF_F_RXHASH && vi->has_rss_hash_report)
1217+ virtio_skb_set_hash((const struct virtio_net_hdr_v1_hash *)hdr, skb);
11601218
11611219 if (hdr->hdr.flags & VIRTIO_NET_HDR_F_DATA_VALID)
11621220 skb->ip_summed = CHECKSUM_UNNECESSARY;
@@ -1266,7 +1324,8 @@ static unsigned int get_mergeable_buf_len(struct receive_queue *rq, · 103 unmodified lines
12661324 struct ewma_pkt_len *avg_pkt_len,
12671325 unsigned int room)
12681326 {
1269− const size_t hdr_len = sizeof(struct virtio_net_hdr_mrg_rxbuf);
1327+ struct virtnet_info *vi = rq->vq->vdev->priv;
1328+ const size_t hdr_len = vi->hdr_len;
12701329 unsigned int len;
12711330
12721331 if (room)
@@ -2183,6 +2242,174 @@ static void virtnet_get_ringparam(struct net_device *dev, · 910 unmodified lines
21832242 ring->tx_pending = ring->tx_max_pending;
21842243 }
21852244
2245+static bool virtnet_commit_rss_command(struct virtnet_info *vi)
2246+{
2247+ struct net_device *dev = vi->dev;
2248+ struct scatterlist sgs[4];
2249+ unsigned int sg_buf_size;
2250+
2251+ /* prepare sgs */
2252+ sg_init_table(sgs, 4);
2253+
2254+ sg_buf_size = offsetof(struct virtio_net_ctrl_rss, indirection_table);
2255+ sg_set_buf(&sgs[0], &vi->ctrl->rss, sg_buf_size);
2256+
2257+ sg_buf_size = sizeof(uint16_t) * (vi->ctrl->rss.indirection_table_mask + 1);
2258+ sg_set_buf(&sgs[1], vi->ctrl->rss.indirection_table, sg_buf_size);
2259+
2260+ sg_buf_size = offsetof(struct virtio_net_ctrl_rss, key)
2261+ - offsetof(struct virtio_net_ctrl_rss, max_tx_vq);
2262+ sg_set_buf(&sgs[2], &vi->ctrl->rss.max_tx_vq, sg_buf_size);
2263+
2264+ sg_buf_size = vi->rss_key_size;
2265+ sg_set_buf(&sgs[3], vi->ctrl->rss.key, sg_buf_size);
2266+
2267+ if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MQ,
2268+ vi->has_rss ? VIRTIO_NET_CTRL_MQ_RSS_CONFIG
2269+ : VIRTIO_NET_CTRL_MQ_HASH_CONFIG, sgs)) {
2270+ dev_warn(&dev->dev, "VIRTIONET issue with committing RSS sgs\n");
2271+ return false;
2272+ }
2273+ return true;
2274+}
2275+
2276+static void virtnet_init_default_rss(struct virtnet_info *vi)
2277+{
2278+ u32 indir_val = 0;
2279+ int i = 0;
2280+
2281+ vi->ctrl->rss.hash_types = vi->rss_hash_types_supported;
2282+ vi->rss_hash_types_saved = vi->rss_hash_types_supported;
2283+ vi->ctrl->rss.indirection_table_mask = vi->rss_indir_table_size
2284+ ? vi->rss_indir_table_size - 1 : 0;
2285+ vi->ctrl->rss.unclassified_queue = 0;
2286+
2287+ for (; i < vi->rss_indir_table_size; ++i) {
2288+ indir_val = ethtool_rxfh_indir_default(i, vi->curr_queue_pairs);
2289+ vi->ctrl->rss.indirection_table[i] = indir_val;
2290+ }
2291+
2292+ vi->ctrl->rss.max_tx_vq = vi->curr_queue_pairs;
2293+ vi->ctrl->rss.hash_key_length = vi->rss_key_size;
2294+
2295+ netdev_rss_key_fill(vi->ctrl->rss.key, vi->rss_key_size);
2296+}
2297+
2298+static void virtnet_get_hashflow(const struct virtnet_info *vi, struct ethtool_rxnfc *info)
2299+{
2300+ info->data = 0;
2301+ switch (info->flow_type) {
2302+ case TCP_V4_FLOW:
2303+ if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_TCPv4) {
2304+ info->data = RXH_IP_SRC | RXH_IP_DST |
2305+ RXH_L4_B_0_1 | RXH_L4_B_2_3;
2306+ } else if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_IPv4) {
2307+ info->data = RXH_IP_SRC | RXH_IP_DST;
2308+ }
2309+ break;
2310+ case TCP_V6_FLOW:
2311+ if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_TCPv6) {
2312+ info->data = RXH_IP_SRC | RXH_IP_DST |
2313+ RXH_L4_B_0_1 | RXH_L4_B_2_3;
2314+ } else if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_IPv6) {
2315+ info->data = RXH_IP_SRC | RXH_IP_DST;
2316+ }
2317+ break;
2318+ case UDP_V4_FLOW:
2319+ if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_UDPv4) {
2320+ info->data = RXH_IP_SRC | RXH_IP_DST |
2321+ RXH_L4_B_0_1 | RXH_L4_B_2_3;
2322+ } else if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_IPv4) {
2323+ info->data = RXH_IP_SRC | RXH_IP_DST;
2324+ }
2325+ break;
2326+ case UDP_V6_FLOW:
2327+ if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_UDPv6) {
2328+ info->data = RXH_IP_SRC | RXH_IP_DST |
2329+ RXH_L4_B_0_1 | RXH_L4_B_2_3;
2330+ } else if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_IPv6) {
2331+ info->data = RXH_IP_SRC | RXH_IP_DST;
2332+ }
2333+ break;
2334+ case IPV4_FLOW:
2335+ if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_IPv4)
2336+ info->data = RXH_IP_SRC | RXH_IP_DST;
2337+
2338+ break;
2339+ case IPV6_FLOW:
2340+ if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_IPv6)
2341+ info->data = RXH_IP_SRC | RXH_IP_DST;
2342+
2343+ break;
2344+ default:
2345+ info->data = 0;
2346+ break;
2347+ }
2348+}
2349+
2350+static bool virtnet_set_hashflow(struct virtnet_info *vi, struct ethtool_rxnfc *info)
2351+{
2352+ u32 new_hashtypes = vi->rss_hash_types_saved;
2353+ bool is_disable = info->data & RXH_DISCARD;
2354+ bool is_l4 = info->data == (RXH_IP_SRC | RXH_IP_DST | RXH_L4_B_0_1 | RXH_L4_B_2_3);
2355+
2356+ /* supports only 'sd', 'sdfn' and 'r' */
2357+ if (!((info->data == (RXH_IP_SRC | RXH_IP_DST)) | is_l4 | is_disable))
2358+ return false;
2359+
2360+ switch (info->flow_type) {
2361+ case TCP_V4_FLOW:
2362+ new_hashtypes &= ~(VIRTIO_NET_RSS_HASH_TYPE_IPv4 | VIRTIO_NET_RSS_HASH_TYPE_TCPv4);
2363+ if (!is_disable)
2364+ new_hashtypes |= VIRTIO_NET_RSS_HASH_TYPE_IPv4
2365+ | (is_l4 ? VIRTIO_NET_RSS_HASH_TYPE_TCPv4 : 0);
2366+ break;
2367+ case UDP_V4_FLOW:
2368+ new_hashtypes &= ~(VIRTIO_NET_RSS_HASH_TYPE_IPv4 | VIRTIO_NET_RSS_HASH_TYPE_UDPv4);
2369+ if (!is_disable)
2370+ new_hashtypes |= VIRTIO_NET_RSS_HASH_TYPE_IPv4
2371+ | (is_l4 ? VIRTIO_NET_RSS_HASH_TYPE_UDPv4 : 0);
2372+ break;
2373+ case IPV4_FLOW:
2374+ new_hashtypes &= ~VIRTIO_NET_RSS_HASH_TYPE_IPv4;
2375+ if (!is_disable)
2376+ new_hashtypes = VIRTIO_NET_RSS_HASH_TYPE_IPv4;
2377+ break;
2378+ case TCP_V6_FLOW:
2379+ new_hashtypes &= ~(VIRTIO_NET_RSS_HASH_TYPE_IPv6 | VIRTIO_NET_RSS_HASH_TYPE_TCPv6);
2380+ if (!is_disable)
2381+ new_hashtypes |= VIRTIO_NET_RSS_HASH_TYPE_IPv6
2382+ | (is_l4 ? VIRTIO_NET_RSS_HASH_TYPE_TCPv6 : 0);
2383+ break;
2384+ case UDP_V6_FLOW:
2385+ new_hashtypes &= ~(VIRTIO_NET_RSS_HASH_TYPE_IPv6 | VIRTIO_NET_RSS_HASH_TYPE_UDPv6);
2386+ if (!is_disable)
2387+ new_hashtypes |= VIRTIO_NET_RSS_HASH_TYPE_IPv6
2388+ | (is_l4 ? VIRTIO_NET_RSS_HASH_TYPE_UDPv6 : 0);
2389+ break;
2390+ case IPV6_FLOW:
2391+ new_hashtypes &= ~VIRTIO_NET_RSS_HASH_TYPE_IPv6;
2392+ if (!is_disable)
2393+ new_hashtypes = VIRTIO_NET_RSS_HASH_TYPE_IPv6;
2394+ break;
2395+ default:
2396+ /* unsupported flow */
2397+ return false;
2398+ }
2399+
2400+ /* if unsupported hashtype was set */
2401+ if (new_hashtypes != (new_hashtypes & vi->rss_hash_types_supported))
2402+ return false;
2403+
2404+ if (new_hashtypes != vi->rss_hash_types_saved) {
2405+ vi->rss_hash_types_saved = new_hashtypes;
2406+ vi->ctrl->rss.hash_types = vi->rss_hash_types_saved;
2407+ if (vi->dev->features & NETIF_F_RXHASH)
2408+ return virtnet_commit_rss_command(vi);
2409+ }
2410+
2411+ return true;
2412+}
21862413
21872414 static void virtnet_get_drvinfo(struct net_device *dev,
21882415 struct ethtool_drvinfo *info)
@@ -2411,6 +2638,92 @@ static void virtnet_update_settings(struct virtnet_info *vi) · 222 unmodified lines
24112638 vi->duplex = duplex;
24122639 }
24132640
2641+static u32 virtnet_get_rxfh_key_size(struct net_device *dev)
2642+{
2643+ return ((struct virtnet_info *)netdev_priv(dev))->rss_key_size;
2644+}
2645+
2646+static u32 virtnet_get_rxfh_indir_size(struct net_device *dev)
2647+{
2648+ return ((struct virtnet_info *)netdev_priv(dev))->rss_indir_table_size;
2649+}
2650+
2651+static int virtnet_get_rxfh(struct net_device *dev, u32 *indir, u8 *key, u8 *hfunc)
2652+{
2653+ struct virtnet_info *vi = netdev_priv(dev);
2654+ int i;
2655+
2656+ if (indir) {
2657+ for (i = 0; i < vi->rss_indir_table_size; ++i)
2658+ indir[i] = vi->ctrl->rss.indirection_table[i];
2659+ }
2660+
2661+ if (key)
2662+ memcpy(key, vi->ctrl->rss.key, vi->rss_key_size);
2663+
2664+ if (hfunc)
2665+ *hfunc = ETH_RSS_HASH_TOP;
2666+
2667+ return 0;
2668+}
2669+
2670+static int virtnet_set_rxfh(struct net_device *dev, const u32 *indir, const u8 *key, const u8 hfunc)
2671+{
2672+ struct virtnet_info *vi = netdev_priv(dev);
2673+ int i;
2674+
2675+ if (hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != ETH_RSS_HASH_TOP)
2676+ return -EOPNOTSUPP;
2677+
2678+ if (indir) {
2679+ for (i = 0; i < vi->rss_indir_table_size; ++i)
2680+ vi->ctrl->rss.indirection_table[i] = indir[i];
2681+ }
2682+ if (key)
2683+ memcpy(vi->ctrl->rss.key, key, vi->rss_key_size);
2684+
2685+ virtnet_commit_rss_command(vi);
2686+
2687+ return 0;
2688+}
2689+
2690+static int virtnet_get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *info, u32 *rule_locs)
2691+{
2692+ struct virtnet_info *vi = netdev_priv(dev);
2693+ int rc = 0;
2694+
2695+ switch (info->cmd) {
2696+ case ETHTOOL_GRXRINGS:
2697+ info->data = vi->curr_queue_pairs;
2698+ break;
2699+ case ETHTOOL_GRXFH:
2700+ virtnet_get_hashflow(vi, info);
2701+ break;
2702+ default:
2703+ rc = -EOPNOTSUPP;
2704+ }
2705+
2706+ return rc;
2707+}
2708+
2709+static int virtnet_set_rxnfc(struct net_device *dev, struct ethtool_rxnfc *info)
2710+{
2711+ struct virtnet_info *vi = netdev_priv(dev);
2712+ int rc = 0;
2713+
2714+ switch (info->cmd) {
2715+ case ETHTOOL_SRXFH:
2716+ if (!virtnet_set_hashflow(vi, info))
2717+ rc = -EINVAL;
2718+
2719+ break;
2720+ default:
2721+ rc = -EOPNOTSUPP;
2722+ }
2723+
2724+ return rc;
2725+}
2726+
24142727 static const struct ethtool_ops virtnet_ethtool_ops = {
24152728 .supported_coalesce_params = ETHTOOL_COALESCE_MAX_FRAMES,
24162729 .get_drvinfo = virtnet_get_drvinfo,
@@ -2426,6 +2739,12 @@ static const struct ethtool_ops virtnet_ethtool_ops = { · 9 unmodified lines
24262739 .set_link_ksettings = virtnet_set_link_ksettings,
24272740 .set_coalesce = virtnet_set_coalesce,
24282741 .get_coalesce = virtnet_get_coalesce,
2742+ .get_rxfh_key_size = virtnet_get_rxfh_key_size,
2743+ .get_rxfh_indir_size = virtnet_get_rxfh_indir_size,
2744+ .get_rxfh = virtnet_get_rxfh,
2745+ .set_rxfh = virtnet_set_rxfh,
2746+ .get_rxnfc = virtnet_get_rxnfc,
2747+ .set_rxnfc = virtnet_set_rxnfc,
24292748 };
24302749
24312750 static void virtnet_freeze_down(struct virtio_device *vdev)
@@ -2678,6 +2997,16 @@ static int virtnet_set_features(struct net_device *dev, · 246 unmodified lines
26782997 vi->guest_offloads = offloads;
26792998 }
26802999
3000+ if ((dev->features ^ features) & NETIF_F_RXHASH) {
3001+ if (features & NETIF_F_RXHASH)
3002+ vi->ctrl->rss.hash_types = vi->rss_hash_types_saved;
3003+ else
3004+ vi->ctrl->rss.hash_types = VIRTIO_NET_HASH_REPORT_NONE;
3005+
3006+ if (!virtnet_commit_rss_command(vi))
3007+ return -EINVAL;
3008+ }
3009+
26813010 return 0;
26823011 }
26833012
@@ -2851,7 +3180,7 @@ static void virtnet_del_vqs(struct virtnet_info *vi) · 167 unmodified lines
28513180 */
28523181 static unsigned int mergeable_min_buf_len(struct virtnet_info *vi, struct virtqueue *vq)
28533182 {
2854− const unsigned int hdr_len = sizeof(struct virtio_net_hdr_mrg_rxbuf);
3183+ const unsigned int hdr_len = vi->hdr_len;
28553184 unsigned int rq_size = virtqueue_get_vring_size(vq);
28563185 unsigned int packet_len = vi->big_packets ? IP_MAX_MTU : vi->dev->max_mtu;
28573186 unsigned int buf_len = hdr_len + ETH_HLEN + VLAN_HLEN + packet_len;
@@ -3072,6 +3401,10 @@ static bool virtnet_validate_features(struct virtio_device *vdev) · 214 unmodified lines
30723401 "VIRTIO_NET_F_CTRL_VQ") ||
30733402 VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_MQ, "VIRTIO_NET_F_CTRL_VQ") ||
30743403 VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_CTRL_MAC_ADDR,
3404+ "VIRTIO_NET_F_CTRL_VQ") ||
3405+ VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_RSS,
3406+ "VIRTIO_NET_F_CTRL_VQ") ||
3407+ VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_HASH_REPORT,
30753408 "VIRTIO_NET_F_CTRL_VQ"))) {
30763409 return false;
30773410 }
@@ -3112,13 +3445,14 @@ static int virtnet_probe(struct virtio_device *vdev) · 34 unmodified lines
31123445 u16 max_queue_pairs;
31133446 int mtu;
31143447
3115− /* Find if host supports multiqueue virtio_net device */
3116− err = virtio_cread_feature(vdev, VIRTIO_NET_F_MQ,
3117− struct virtio_net_config,
3118− max_virtqueue_pairs, &max_queue_pairs);
3448+ /* Find if host supports multiqueue/rss virtio_net device */
3449+ max_queue_pairs = 1;
3450+ if (virtio_has_feature(vdev, VIRTIO_NET_F_MQ) || virtio_has_feature(vdev, VIRTIO_NET_F_RSS))
3451+ max_queue_pairs =
3452+ virtio_cread16(vdev, offsetof(struct virtio_net_config, max_virtqueue_pairs));
31193453
31203454 /* We need at least 2 queue's */
3121− if (err || max_queue_pairs < VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN ||
3455+ if (max_queue_pairs < VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN ||
31223456 max_queue_pairs > VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX ||
31233457 !virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ))
31243458 max_queue_pairs = 1;
@@ -3206,8 +3540,33 @@ static int virtnet_probe(struct virtio_device *vdev) · 81 unmodified lines
32063540 if (virtio_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF))
32073541 vi->mergeable_rx_bufs = true;
32083542
3209− if (virtio_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF) ||
3210− virtio_has_feature(vdev, VIRTIO_F_VERSION_1))
3543+ if (virtio_has_feature(vdev, VIRTIO_NET_F_HASH_REPORT))
3544+ vi->has_rss_hash_report = true;
3545+
3546+ if (virtio_has_feature(vdev, VIRTIO_NET_F_RSS))
3547+ vi->has_rss = true;
3548+
3549+ if (vi->has_rss || vi->has_rss_hash_report) {
3550+ vi->rss_indir_table_size =
3551+ virtio_cread16(vdev, offsetof(struct virtio_net_config,
3552+ rss_max_indirection_table_length));
3553+ vi->rss_key_size =
3554+ virtio_cread8(vdev, offsetof(struct virtio_net_config, rss_max_key_size));
3555+
3556+ vi->rss_hash_types_supported =
3557+ virtio_cread32(vdev, offsetof(struct virtio_net_config, supported_hash_types));
3558+ vi->rss_hash_types_supported &=
3559+ ~(VIRTIO_NET_RSS_HASH_TYPE_IP_EX |
3560+ VIRTIO_NET_RSS_HASH_TYPE_TCP_EX |
3561+ VIRTIO_NET_RSS_HASH_TYPE_UDP_EX);
3562+
3563+ dev->hw_features |= NETIF_F_RXHASH;
3564+ }
3565+
3566+ if (vi->has_rss_hash_report)
3567+ vi->hdr_len = sizeof(struct virtio_net_hdr_v1_hash);
3568+ else if (virtio_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF) ||
3569+ virtio_has_feature(vdev, VIRTIO_F_VERSION_1))
32113570 vi->hdr_len = sizeof(struct virtio_net_hdr_mrg_rxbuf);
32123571 else
32133572 vi->hdr_len = sizeof(struct virtio_net_hdr);
@@ -3274,6 +3633,9 @@ static int virtnet_probe(struct virtio_device *vdev) · 60 unmodified lines
32743633 }
32753634 }
32763635
3636+ if (vi->has_rss || vi->has_rss_hash_report)
3637+ virtnet_init_default_rss(vi);
3638+
32773639 err = register_netdev(dev);
32783640 if (err) {
32793641 pr_debug("virtio_net: registering device failed\n");
@@ -3405,7 +3767,8 @@ static struct virtio_device_id id_table[] = { · 125 unmodified lines
34053767 VIRTIO_NET_F_GUEST_ANNOUNCE, VIRTIO_NET_F_MQ, \
34063768 VIRTIO_NET_F_CTRL_MAC_ADDR, \
34073769 VIRTIO_NET_F_MTU, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS, \
3408− VIRTIO_NET_F_SPEED_DUPLEX, VIRTIO_NET_F_STANDBY
3770+ VIRTIO_NET_F_SPEED_DUPLEX, VIRTIO_NET_F_STANDBY, \
3771+ VIRTIO_NET_F_RSS, VIRTIO_NET_F_HASH_REPORT
34093772
34103773 static unsigned int features[] = {
34113774 VIRTNET_FEATURES,
drivers/vdpa/ifcvf/ifcvf_base.c+44−80
@@ -10,45 +10,29 @@ · 9 unmodified lines
1010
1111 #include "ifcvf_base.h"
1212
13−static inline u8 ifc_ioread8(u8 __iomem *addr)
14−{
15− return ioread8(addr);
16−}
17−static inline u16 ifc_ioread16 (__le16 __iomem *addr)
13+struct ifcvf_adapter *vf_to_adapter(struct ifcvf_hw *hw)
1814 {
19− return ioread16(addr);
15+ return container_of(hw, struct ifcvf_adapter, vf);
2016 }
2117
22−static inline u32 ifc_ioread32(__le32 __iomem *addr)
18+u16 ifcvf_set_vq_vector(struct ifcvf_hw *hw, u16 qid, int vector)
2319 {
24− return ioread32(addr);
25−}
20+ struct virtio_pci_common_cfg __iomem *cfg = hw->common_cfg;
2621
27−static inline void ifc_iowrite8(u8 value, u8 __iomem *addr)
28−{
29− iowrite8(value, addr);
30−}
22+ vp_iowrite16(qid, &cfg->queue_select);
23+ vp_iowrite16(vector, &cfg->queue_msix_vector);
3124
32−static inline void ifc_iowrite16(u16 value, __le16 __iomem *addr)
33−{
34− iowrite16(value, addr);
25+ return vp_ioread16(&cfg->queue_msix_vector);
3526 }
3627
37−static inline void ifc_iowrite32(u32 value, __le32 __iomem *addr)
28+u16 ifcvf_set_config_vector(struct ifcvf_hw *hw, int vector)
3829 {
39− iowrite32(value, addr);
40−}
30+ struct virtio_pci_common_cfg __iomem *cfg = hw->common_cfg;
4131
42−static void ifc_iowrite64_twopart(u64 val,
43− __le32 __iomem *lo, __le32 __iomem *hi)
44−{
45− ifc_iowrite32((u32)val, lo);
46− ifc_iowrite32(val >> 32, hi);
47−}
32+ cfg = hw->common_cfg;
33+ vp_iowrite16(vector, &cfg->msix_config);
4834
49−struct ifcvf_adapter *vf_to_adapter(struct ifcvf_hw *hw)
50−{
51− return container_of(hw, struct ifcvf_adapter, vf);
35+ return vp_ioread16(&cfg->msix_config);
5236 }
5337
5438 static void __iomem *get_cap_addr(struct ifcvf_hw *hw,
@@ -158,15 +142,16 @@ next: · 103 unmodified lines
158142 return -EIO;
159143 }
160144
161− hw->nr_vring = ifc_ioread16(&hw->common_cfg->num_queues);
145+ hw->nr_vring = vp_ioread16(&hw->common_cfg->num_queues);
162146
163147 for (i = 0; i < hw->nr_vring; i++) {
164− ifc_iowrite16(i, &hw->common_cfg->queue_select);
165− notify_off = ifc_ioread16(&hw->common_cfg->queue_notify_off);
148+ vp_iowrite16(i, &hw->common_cfg->queue_select);
149+ notify_off = vp_ioread16(&hw->common_cfg->queue_notify_off);
166150 hw->vring[i].notify_addr = hw->notify_base +
167151 notify_off * hw->notify_off_multiplier;
168152 hw->vring[i].notify_pa = hw->notify_base_pa +
169153 notify_off * hw->notify_off_multiplier;
154+ hw->vring[i].irq = -EINVAL;
170155 }
171156
172157 hw->lm_cfg = hw->base[IFCVF_LM_BAR];
@@ -176,17 +161,20 @@ next: · 3 unmodified lines
176161 hw->common_cfg, hw->notify_base, hw->isr,
177162 hw->dev_cfg, hw->notify_off_multiplier);
178163
164+ hw->vqs_reused_irq = -EINVAL;
165+ hw->config_irq = -EINVAL;
166+
179167 return 0;
180168 }
181169
182170 u8 ifcvf_get_status(struct ifcvf_hw *hw)
183171 {
184− return ifc_ioread8(&hw->common_cfg->device_status);
172+ return vp_ioread8(&hw->common_cfg->device_status);
185173 }
186174
187175 void ifcvf_set_status(struct ifcvf_hw *hw, u8 status)
188176 {
189− ifc_iowrite8(status, &hw->common_cfg->device_status);
177+ vp_iowrite8(status, &hw->common_cfg->device_status);
190178 }
191179
192180 void ifcvf_reset(struct ifcvf_hw *hw)
@@ -214,11 +202,11 @@ u64 ifcvf_get_hw_features(struct ifcvf_hw *hw) · 21 unmodified lines
214202 u32 features_lo, features_hi;
215203 u64 features;
216204
217− ifc_iowrite32(0, &cfg->device_feature_select);
218− features_lo = ifc_ioread32(&cfg->device_feature);
205+ vp_iowrite32(0, &cfg->device_feature_select);
206+ features_lo = vp_ioread32(&cfg->device_feature);
219207
220− ifc_iowrite32(1, &cfg->device_feature_select);
221− features_hi = ifc_ioread32(&cfg->device_feature);
208+ vp_iowrite32(1, &cfg->device_feature_select);
209+ features_hi = vp_ioread32(&cfg->device_feature);
222210
223211 features = ((u64)features_hi << 32) | features_lo;
224212
@@ -271,12 +259,12 @@ void ifcvf_read_dev_config(struct ifcvf_hw *hw, u64 offset, · 46 unmodified lines
271259
272260 WARN_ON(offset + length > hw->config_size);
273261 do {
274− old_gen = ifc_ioread8(&hw->common_cfg->config_generation);
262+ old_gen = vp_ioread8(&hw->common_cfg->config_generation);
275263 p = dst;
276264 for (i = 0; i < length; i++)
277− *p++ = ifc_ioread8(hw->dev_cfg + offset + i);
265+ *p++ = vp_ioread8(hw->dev_cfg + offset + i);
278266
279− new_gen = ifc_ioread8(&hw->common_cfg->config_generation);
267+ new_gen = vp_ioread8(&hw->common_cfg->config_generation);
280268 } while (old_gen != new_gen);
281269 }
282270
@@ -289,18 +277,18 @@ void ifcvf_write_dev_config(struct ifcvf_hw *hw, u64 offset, · 6 unmodified lines
289277 p = src;
290278 WARN_ON(offset + length > hw->config_size);
291279 for (i = 0; i < length; i++)
292− ifc_iowrite8(*p++, hw->dev_cfg + offset + i);
280+ vp_iowrite8(*p++, hw->dev_cfg + offset + i);
293281 }
294282
295283 static void ifcvf_set_features(struct ifcvf_hw *hw, u64 features)
296284 {
297285 struct virtio_pci_common_cfg __iomem *cfg = hw->common_cfg;
298286
299− ifc_iowrite32(0, &cfg->guest_feature_select);
300− ifc_iowrite32((u32)features, &cfg->guest_feature);
287+ vp_iowrite32(0, &cfg->guest_feature_select);
288+ vp_iowrite32((u32)features, &cfg->guest_feature);
301289
302− ifc_iowrite32(1, &cfg->guest_feature_select);
303− ifc_iowrite32(features >> 32, &cfg->guest_feature);
290+ vp_iowrite32(1, &cfg->guest_feature_select);
291+ vp_iowrite32(features >> 32, &cfg->guest_feature);
304292 }
305293
306294 static int ifcvf_config_features(struct ifcvf_hw *hw)
@@ -329,7 +317,7 @@ u16 ifcvf_get_vq_state(struct ifcvf_hw *hw, u16 qid) · 22 unmodified lines
329317 ifcvf_lm = (struct ifcvf_lm_cfg __iomem *)hw->lm_cfg;
330318 q_pair_id = qid / hw->nr_vring;
331319 avail_idx_addr = &ifcvf_lm->vring_lm_cfg[q_pair_id].idx_addr[qid % 2];
332− last_avail_idx = ifc_ioread16(avail_idx_addr);
320+ last_avail_idx = vp_ioread16(avail_idx_addr);
333321
334322 return last_avail_idx;
335323 }
@@ -344,7 +332,7 @@ int ifcvf_set_vq_state(struct ifcvf_hw *hw, u16 qid, u16 num) · 8 unmodified lines
344332 q_pair_id = qid / hw->nr_vring;
345333 avail_idx_addr = &ifcvf_lm->vring_lm_cfg[q_pair_id].idx_addr[qid % 2];
346334 hw->vring[qid].last_avail_idx = num;
347− ifc_iowrite16(num, avail_idx_addr);
335+ vp_iowrite16(num, avail_idx_addr);
348336
349337 return 0;
350338 }
@@ -352,41 +340,23 @@ int ifcvf_set_vq_state(struct ifcvf_hw *hw, u16 qid, u16 num) · 1 unmodified line
352340 static int ifcvf_hw_enable(struct ifcvf_hw *hw)
353341 {
354342 struct virtio_pci_common_cfg __iomem *cfg;
355− struct ifcvf_adapter *ifcvf;
356343 u32 i;
357344
358− ifcvf = vf_to_adapter(hw);
359345 cfg = hw->common_cfg;
360− ifc_iowrite16(IFCVF_MSI_CONFIG_OFF, &cfg->msix_config);
361−
362− if (ifc_ioread16(&cfg->msix_config) == VIRTIO_MSI_NO_VECTOR) {
363− IFCVF_ERR(ifcvf->pdev, "No msix vector for device config\n");
364− return -EINVAL;
365− }
366−
367346 for (i = 0; i < hw->nr_vring; i++) {
368347 if (!hw->vring[i].ready)
369348 break;
370349
371− ifc_iowrite16(i, &cfg->queue_select);
372− ifc_iowrite64_twopart(hw->vring[i].desc, &cfg->queue_desc_lo,
350+ vp_iowrite16(i, &cfg->queue_select);
351+ vp_iowrite64_twopart(hw->vring[i].desc, &cfg->queue_desc_lo,
373352 &cfg->queue_desc_hi);
374− ifc_iowrite64_twopart(hw->vring[i].avail, &cfg->queue_avail_lo,
353+ vp_iowrite64_twopart(hw->vring[i].avail, &cfg->queue_avail_lo,
375354 &cfg->queue_avail_hi);
376− ifc_iowrite64_twopart(hw->vring[i].used, &cfg->queue_used_lo,
355+ vp_iowrite64_twopart(hw->vring[i].used, &cfg->queue_used_lo,
377356 &cfg->queue_used_hi);
378− ifc_iowrite16(hw->vring[i].size, &cfg->queue_size);
379− ifc_iowrite16(i + IFCVF_MSI_QUEUE_OFF, &cfg->queue_msix_vector);
380−
381− if (ifc_ioread16(&cfg->queue_msix_vector) ==
382− VIRTIO_MSI_NO_VECTOR) {
383− IFCVF_ERR(ifcvf->pdev,
384− "No msix vector for queue %u\n", i);
385− return -EINVAL;
386− }
387−
357+ vp_iowrite16(hw->vring[i].size, &cfg->queue_size);
388358 ifcvf_set_vq_state(hw, i, hw->vring[i].last_avail_idx);
389− ifc_iowrite16(1, &cfg->queue_enable);
359+ vp_iowrite16(1, &cfg->queue_enable);
390360 }
391361
392362 return 0;
@@ -394,18 +364,12 @@ static int ifcvf_hw_enable(struct ifcvf_hw *hw) · 1 unmodified line
394364
395365 static void ifcvf_hw_disable(struct ifcvf_hw *hw)
396366 {
397− struct virtio_pci_common_cfg __iomem *cfg;
398367 u32 i;
399368
400− cfg = hw->common_cfg;
401− ifc_iowrite16(VIRTIO_MSI_NO_VECTOR, &cfg->msix_config);
402−
369+ ifcvf_set_config_vector(hw, VIRTIO_MSI_NO_VECTOR);
403370 for (i = 0; i < hw->nr_vring; i++) {
404− ifc_iowrite16(i, &cfg->queue_select);
405− ifc_iowrite16(VIRTIO_MSI_NO_VECTOR, &cfg->queue_msix_vector);
371+ ifcvf_set_vq_vector(hw, i, VIRTIO_MSI_NO_VECTOR);
406372 }
407−
408− ifc_ioread16(&cfg->queue_msix_vector);
409373 }
410374
411375 int ifcvf_start_hw(struct ifcvf_hw *hw)
@@ -433,5 +397,5 @@ void ifcvf_stop_hw(struct ifcvf_hw *hw) · 21 unmodified lines
433397
434398 void ifcvf_notify_queue(struct ifcvf_hw *hw, u16 qid)
435399 {
436− ifc_iowrite16(qid, hw->vring[qid].notify_addr);
400+ vp_iowrite16(qid, hw->vring[qid].notify_addr);
437401 }
drivers/vdpa/ifcvf/ifcvf_base.h+17−7
@@ -14,6 +14,7 @@ · 13 unmodified lines
1414 #include <linux/pci.h>
1515 #include <linux/pci_regs.h>
1616 #include <linux/vdpa.h>
17+#include <linux/virtio_pci_modern.h>
1718 #include <uapi/linux/virtio_net.h>
1819 #include <uapi/linux/virtio_blk.h>
1920 #include <uapi/linux/virtio_config.h>
@@ -27,8 +28,6 @@ · 7 unmodified lines
2728
2829 #define IFCVF_QUEUE_ALIGNMENT PAGE_SIZE
2930 #define IFCVF_QUEUE_MAX 32768
30−#define IFCVF_MSI_CONFIG_OFF 0
31−#define IFCVF_MSI_QUEUE_OFF 1
3231 #define IFCVF_PCI_MAX_RESOURCE 6
3332
3433 #define IFCVF_LM_CFG_SIZE 0x40
@@ -42,6 +41,13 @@ · 7 unmodified lines
4241 #define ifcvf_private_to_vf(adapter) \
4342 (&((struct ifcvf_adapter *)adapter)->vf)
4443
44+/* all vqs and config interrupt has its own vector */
45+#define MSIX_VECTOR_PER_VQ_AND_CONFIG 1
46+/* all vqs share a vector, and config interrupt has a separate vector */
47+#define MSIX_VECTOR_SHARED_VQ_AND_CONFIG 2
48+/* all vqs and config interrupt share a vector */
49+#define MSIX_VECTOR_DEV_SHARED 3
50+
4551 struct vring_info {
4652 u64 desc;
4753 u64 avail;
@@ -60,25 +66,27 @@ struct ifcvf_hw { · 12 unmodified lines
6066 u8 __iomem *isr;
6167 /* Live migration */
6268 u8 __iomem *lm_cfg;
63− u16 nr_vring;
6469 /* Notification bar number */
6570 u8 notify_bar;
71+ u8 msix_vector_status;
72+ /* virtio-net or virtio-blk device config size */
73+ u32 config_size;
6674 /* Notificaiton bar address */
6775 void __iomem *notify_base;
6876 phys_addr_t notify_base_pa;
6977 u32 notify_off_multiplier;
78+ u32 dev_type;
7079 u64 req_features;
7180 u64 hw_features;
72− u32 dev_type;
7381 struct virtio_pci_common_cfg __iomem *common_cfg;
7482 void __iomem *dev_cfg;
7583 struct vring_info vring[IFCVF_MAX_QUEUES];
7684 void __iomem * const *base;
7785 char config_msix_name[256];
7886 struct vdpa_callback config_cb;
79− unsigned int config_irq;
80− /* virtio-net or virtio-blk device config size */
81− u32 config_size;
87+ int config_irq;
88+ int vqs_reused_irq;
89+ u16 nr_vring;
8290 };
8391
8492 struct ifcvf_adapter {
@@ -123,4 +131,6 @@ int ifcvf_set_vq_state(struct ifcvf_hw *hw, u16 qid, u16 num); · 38 unmodified lines
123131 struct ifcvf_adapter *vf_to_adapter(struct ifcvf_hw *hw);
124132 int ifcvf_probed_virtio_net(struct ifcvf_hw *hw);
125133 u32 ifcvf_get_config_size(struct ifcvf_hw *hw);
134+u16 ifcvf_set_vq_vector(struct ifcvf_hw *hw, u16 qid, int vector);
135+u16 ifcvf_set_config_vector(struct ifcvf_hw *hw, int vector);
126136 #endif /* _IFCVF_H_ */
drivers/vdpa/ifcvf/ifcvf_main.c+284−33
@@ -27,7 +27,7 @@ static irqreturn_t ifcvf_config_changed(int irq, void *arg) · 26 unmodified lines
2727 return IRQ_HANDLED;
2828 }
2929
30−static irqreturn_t ifcvf_intr_handler(int irq, void *arg)
30+static irqreturn_t ifcvf_vq_intr_handler(int irq, void *arg)
3131 {
3232 struct vring_info *vring = arg;
3333
@@ -37,76 +37,324 @@ static irqreturn_t ifcvf_intr_handler(int irq, void *arg) · 3 unmodified lines
3737 return IRQ_HANDLED;
3838 }
3939
40+static irqreturn_t ifcvf_vqs_reused_intr_handler(int irq, void *arg)
41+{
42+ struct ifcvf_hw *vf = arg;
43+ struct vring_info *vring;
44+ int i;
45+
46+ for (i = 0; i < vf->nr_vring; i++) {
47+ vring = &vf->vring[i];
48+ if (vring->cb.callback)
49+ vring->cb.callback(vring->cb.private);
50+ }
51+
52+ return IRQ_HANDLED;
53+}
54+
55+static irqreturn_t ifcvf_dev_intr_handler(int irq, void *arg)
56+{
57+ struct ifcvf_hw *vf = arg;
58+ u8 isr;
59+
60+ isr = vp_ioread8(vf->isr);
61+ if (isr & VIRTIO_PCI_ISR_CONFIG)
62+ ifcvf_config_changed(irq, arg);
63+
64+ return ifcvf_vqs_reused_intr_handler(irq, arg);
65+}
66+
4067 static void ifcvf_free_irq_vectors(void *data)
4168 {
4269 pci_free_irq_vectors(data);
4370 }
4471
45−static void ifcvf_free_irq(struct ifcvf_adapter *adapter, int queues)
72+static void ifcvf_free_per_vq_irq(struct ifcvf_adapter *adapter)
4673 {
4774 struct pci_dev *pdev = adapter->pdev;
4875 struct ifcvf_hw *vf = &adapter->vf;
4976 int i;
5077
78+ for (i = 0; i < vf->nr_vring; i++) {
79+ if (vf->vring[i].irq != -EINVAL) {
80+ devm_free_irq(&pdev->dev, vf->vring[i].irq, &vf->vring[i]);
81+ vf->vring[i].irq = -EINVAL;
82+ }
83+ }
84+}
5185
52− for (i = 0; i < queues; i++) {
53− devm_free_irq(&pdev->dev, vf->vring[i].irq, &vf->vring[i]);
54− vf->vring[i].irq = -EINVAL;
86+static void ifcvf_free_vqs_reused_irq(struct ifcvf_adapter *adapter)
87+{
88+ struct pci_dev *pdev = adapter->pdev;
89+ struct ifcvf_hw *vf = &adapter->vf;
90+
91+ if (vf->vqs_reused_irq != -EINVAL) {
92+ devm_free_irq(&pdev->dev, vf->vqs_reused_irq, vf);
93+ vf->vqs_reused_irq = -EINVAL;
5594 }
5695
57− devm_free_irq(&pdev->dev, vf->config_irq, vf);
96+}
97+
98+static void ifcvf_free_vq_irq(struct ifcvf_adapter *adapter)
99+{
100+ struct ifcvf_hw *vf = &adapter->vf;
101+
102+ if (vf->msix_vector_status == MSIX_VECTOR_PER_VQ_AND_CONFIG)
103+ ifcvf_free_per_vq_irq(adapter);
104+ else
105+ ifcvf_free_vqs_reused_irq(adapter);
106+}
107+
108+static void ifcvf_free_config_irq(struct ifcvf_adapter *adapter)
109+{
110+ struct pci_dev *pdev = adapter->pdev;
111+ struct ifcvf_hw *vf = &adapter->vf;
112+
113+ if (vf->config_irq == -EINVAL)
114+ return;
115+
116+ /* If the irq is shared by all vqs and the config interrupt,
117+ * it is already freed in ifcvf_free_vq_irq, so here only
118+ * need to free config irq when msix_vector_status != MSIX_VECTOR_DEV_SHARED
119+ */
120+ if (vf->msix_vector_status != MSIX_VECTOR_DEV_SHARED) {
121+ devm_free_irq(&pdev->dev, vf->config_irq, vf);
122+ vf->config_irq = -EINVAL;
123+ }
124+}
125+
126+static void ifcvf_free_irq(struct ifcvf_adapter *adapter)
127+{
128+ struct pci_dev *pdev = adapter->pdev;
129+
130+ ifcvf_free_vq_irq(adapter);
131+ ifcvf_free_config_irq(adapter);
58132 ifcvf_free_irq_vectors(pdev);
59133 }
60134
61−static int ifcvf_request_irq(struct ifcvf_adapter *adapter)
135+/* ifcvf MSIX vectors allocator, this helper tries to allocate
136+ * vectors for all virtqueues and the config interrupt.
137+ * It returns the number of allocated vectors, negative
138+ * return value when fails.
139+ */
140+static int ifcvf_alloc_vectors(struct ifcvf_adapter *adapter)
62141 {
63142 struct pci_dev *pdev = adapter->pdev;
64143 struct ifcvf_hw *vf = &adapter->vf;
65− int vector, i, ret, irq;
66− u16 max_intr;
144+ int max_intr, ret;
67145
68146 /* all queues and config interrupt */
69147 max_intr = vf->nr_vring + 1;
148+ ret = pci_alloc_irq_vectors(pdev, 1, max_intr, PCI_IRQ_MSIX | PCI_IRQ_AFFINITY);
70149
71− ret = pci_alloc_irq_vectors(pdev, max_intr,
72− max_intr, PCI_IRQ_MSIX);
73150 if (ret < 0) {
74151 IFCVF_ERR(pdev, "Failed to alloc IRQ vectors\n");
75152 return ret;
76153 }
77154
155+ if (ret < max_intr)
156+ IFCVF_INFO(pdev,
157+ "Requested %u vectors, however only %u allocated, lower performance\n",
158+ max_intr, ret);
159+
160+ return ret;
161+}
162+
163+static int ifcvf_request_per_vq_irq(struct ifcvf_adapter *adapter)
164+{
165+ struct pci_dev *pdev = adapter->pdev;
166+ struct ifcvf_hw *vf = &adapter->vf;
167+ int i, vector, ret, irq;
168+
169+ vf->vqs_reused_irq = -EINVAL;
170+ for (i = 0; i < vf->nr_vring; i++) {
171+ snprintf(vf->vring[i].msix_name, 256, "ifcvf[%s]-%d\n", pci_name(pdev), i);
172+ vector = i;
173+ irq = pci_irq_vector(pdev, vector);
174+ ret = devm_request_irq(&pdev->dev, irq,
175+ ifcvf_vq_intr_handler, 0,
176+ vf->vring[i].msix_name,
177+ &vf->vring[i]);
178+ if (ret) {
179+ IFCVF_ERR(pdev, "Failed to request irq for vq %d\n", i);
180+ goto err;
181+ }
182+
183+ vf->vring[i].irq = irq;
184+ ret = ifcvf_set_vq_vector(vf, i, vector);
185+ if (ret == VIRTIO_MSI_NO_VECTOR) {
186+ IFCVF_ERR(pdev, "No msix vector for vq %u\n", i);
187+ goto err;
188+ }
189+ }
190+
191+ return 0;
192+err:
193+ ifcvf_free_irq(adapter);
194+
195+ return -EFAULT;
196+}
197+
198+static int ifcvf_request_vqs_reused_irq(struct ifcvf_adapter *adapter)
199+{
200+ struct pci_dev *pdev = adapter->pdev;
201+ struct ifcvf_hw *vf = &adapter->vf;
202+ int i, vector, ret, irq;
203+
204+ vector = 0;
205+ snprintf(vf->vring[0].msix_name, 256, "ifcvf[%s]-vqs-reused-irq\n", pci_name(pdev));
206+ irq = pci_irq_vector(pdev, vector);
207+ ret = devm_request_irq(&pdev->dev, irq,
208+ ifcvf_vqs_reused_intr_handler, 0,
209+ vf->vring[0].msix_name, vf);
210+ if (ret) {
211+ IFCVF_ERR(pdev, "Failed to request reused irq for the device\n");
212+ goto err;
213+ }
214+
215+ vf->vqs_reused_irq = irq;
216+ for (i = 0; i < vf->nr_vring; i++) {
217+ vf->vring[i].irq = -EINVAL;
218+ ret = ifcvf_set_vq_vector(vf, i, vector);
219+ if (ret == VIRTIO_MSI_NO_VECTOR) {
220+ IFCVF_ERR(pdev, "No msix vector for vq %u\n", i);
221+ goto err;
222+ }
223+ }
224+
225+ return 0;
226+err:
227+ ifcvf_free_irq(adapter);
228+
229+ return -EFAULT;
230+}
231+
232+static int ifcvf_request_dev_irq(struct ifcvf_adapter *adapter)
233+{
234+ struct pci_dev *pdev = adapter->pdev;
235+ struct ifcvf_hw *vf = &adapter->vf;
236+ int i, vector, ret, irq;
237+
238+ vector = 0;
239+ snprintf(vf->vring[0].msix_name, 256, "ifcvf[%s]-dev-irq\n", pci_name(pdev));
240+ irq = pci_irq_vector(pdev, vector);
241+ ret = devm_request_irq(&pdev->dev, irq,
242+ ifcvf_dev_intr_handler, 0,
243+ vf->vring[0].msix_name, vf);
244+ if (ret) {
245+ IFCVF_ERR(pdev, "Failed to request irq for the device\n");
246+ goto err;
247+ }
248+
249+ vf->vqs_reused_irq = irq;
250+ for (i = 0; i < vf->nr_vring; i++) {
251+ vf->vring[i].irq = -EINVAL;
252+ ret = ifcvf_set_vq_vector(vf, i, vector);
253+ if (ret == VIRTIO_MSI_NO_VECTOR) {
254+ IFCVF_ERR(pdev, "No msix vector for vq %u\n", i);
255+ goto err;
256+ }
257+ }
258+
259+ vf->config_irq = irq;
260+ ret = ifcvf_set_config_vector(vf, vector);
261+ if (ret == VIRTIO_MSI_NO_VECTOR) {
262+ IFCVF_ERR(pdev, "No msix vector for device config\n");
263+ goto err;
264+ }
265+
266+ return 0;
267+err:
268+ ifcvf_free_irq(adapter);
269+
270+ return -EFAULT;
271+
272+}
273+
274+static int ifcvf_request_vq_irq(struct ifcvf_adapter *adapter)
275+{
276+ struct ifcvf_hw *vf = &adapter->vf;
277+ int ret;
278+
279+ if (vf->msix_vector_status == MSIX_VECTOR_PER_VQ_AND_CONFIG)
280+ ret = ifcvf_request_per_vq_irq(adapter);
281+ else
282+ ret = ifcvf_request_vqs_reused_irq(adapter);
283+
284+ return ret;
285+}
286+
287+static int ifcvf_request_config_irq(struct ifcvf_adapter *adapter)
288+{
289+ struct pci_dev *pdev = adapter->pdev;
290+ struct ifcvf_hw *vf = &adapter->vf;
291+ int config_vector, ret;
292+
293+ if (vf->msix_vector_status == MSIX_VECTOR_DEV_SHARED)
294+ return 0;
295+
296+ if (vf->msix_vector_status == MSIX_VECTOR_PER_VQ_AND_CONFIG)
297+ /* vector 0 ~ vf->nr_vring for vqs, num vf->nr_vring vector for config interrupt */
298+ config_vector = vf->nr_vring;
299+
300+ if (vf->msix_vector_status == MSIX_VECTOR_SHARED_VQ_AND_CONFIG)
301+ /* vector 0 for vqs and 1 for config interrupt */
302+ config_vector = 1;
303+
78304 snprintf(vf->config_msix_name, 256, "ifcvf[%s]-config\n",
79305 pci_name(pdev));
80− vector = 0;
81− vf->config_irq = pci_irq_vector(pdev, vector);
306+ vf->config_irq = pci_irq_vector(pdev, config_vector);
82307 ret = devm_request_irq(&pdev->dev, vf->config_irq,
83308 ifcvf_config_changed, 0,
84309 vf->config_msix_name, vf);
85310 if (ret) {
86311 IFCVF_ERR(pdev, "Failed to request config irq\n");
87− return ret;
312+ goto err;
88313 }
89314
90− for (i = 0; i < vf->nr_vring; i++) {
91− snprintf(vf->vring[i].msix_name, 256, "ifcvf[%s]-%d\n",
92− pci_name(pdev), i);
93− vector = i + IFCVF_MSI_QUEUE_OFF;
94− irq = pci_irq_vector(pdev, vector);
95− ret = devm_request_irq(&pdev->dev, irq,
96− ifcvf_intr_handler, 0,
97− vf->vring[i].msix_name,
98− &vf->vring[i]);
99− if (ret) {
100− IFCVF_ERR(pdev,
101− "Failed to request irq for vq %d\n", i);
102− ifcvf_free_irq(adapter, i);
315+ ret = ifcvf_set_config_vector(vf, config_vector);
316+ if (ret == VIRTIO_MSI_NO_VECTOR) {
317+ IFCVF_ERR(pdev, "No msix vector for device config\n");
318+ goto err;
319+ }
103320
104− return ret;
105− }
321+ return 0;
322+err:
323+ ifcvf_free_irq(adapter);
106324
107− vf->vring[i].irq = irq;
325+ return -EFAULT;
326+}
327+
328+static int ifcvf_request_irq(struct ifcvf_adapter *adapter)
329+{
330+ struct ifcvf_hw *vf = &adapter->vf;
331+ int nvectors, ret, max_intr;
332+
333+ nvectors = ifcvf_alloc_vectors(adapter);
334+ if (nvectors <= 0)
335+ return -EFAULT;
336+
337+ vf->msix_vector_status = MSIX_VECTOR_PER_VQ_AND_CONFIG;
338+ max_intr = vf->nr_vring + 1;
339+ if (nvectors < max_intr)
340+ vf->msix_vector_status = MSIX_VECTOR_SHARED_VQ_AND_CONFIG;
341+
342+ if (nvectors == 1) {
343+ vf->msix_vector_status = MSIX_VECTOR_DEV_SHARED;
344+ ret = ifcvf_request_dev_irq(adapter);
345+
346+ return ret;
108347 }
109348
349+ ret = ifcvf_request_vq_irq(adapter);
350+ if (ret)
351+ return ret;
352+
353+ ret = ifcvf_request_config_irq(adapter);
354+
355+ if (ret)
356+ return ret;
357+
110358 return 0;
111359 }
112360
@@ -263,7 +511,7 @@ static int ifcvf_vdpa_reset(struct vdpa_device *vdpa_dev) · 150 unmodified lines
263511
264512 if (status_old & VIRTIO_CONFIG_S_DRIVER_OK) {
265513 ifcvf_stop_datapath(adapter);
266− ifcvf_free_irq(adapter, vf->nr_vring);
514+ ifcvf_free_irq(adapter);
267515 }
268516
269517 ifcvf_reset_vring(adapter);
@@ -348,7 +596,7 @@ static u32 ifcvf_vdpa_get_generation(struct vdpa_device *vdpa_dev) · 78 unmodified lines
348596 {
349597 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
350598
351− return ioread8(&vf->common_cfg->config_generation);
599+ return vp_ioread8(&vf->common_cfg->config_generation);
352600 }
353601
354602 static u32 ifcvf_vdpa_get_device_id(struct vdpa_device *vdpa_dev)
@@ -410,7 +658,10 @@ static int ifcvf_vdpa_get_vq_irq(struct vdpa_device *vdpa_dev, · 55 unmodified lines
410658 {
411659 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
412660
413− return vf->vring[qid].irq;
661+ if (vf->vqs_reused_irq < 0)
662+ return vf->vring[qid].irq;
663+ else
664+ return -EINVAL;
414665 }
415666
416667 static struct vdpa_notification_area ifcvf_get_vq_notification(struct vdpa_device *vdpa_dev,
drivers/vdpa/mlx5/net/mlx5_vnet.c+80−4
@@ -1475,7 +1475,7 @@ static virtio_net_ctrl_ack handle_ctrl_mac(struct mlx5_vdpa_dev *mvdev, u8 cmd) · 1474 unmodified lines
14751475 virtio_net_ctrl_ack status = VIRTIO_NET_ERR;
14761476 struct mlx5_core_dev *pfmdev;
14771477 size_t read;
1478− u8 mac[ETH_ALEN];
1478+ u8 mac[ETH_ALEN], mac_back[ETH_ALEN];
14791479
14801480 pfmdev = pci_get_drvdata(pci_physfn(mvdev->mdev->pdev));
14811481 switch (cmd) {
@@ -1489,6 +1489,9 @@ static virtio_net_ctrl_ack handle_ctrl_mac(struct mlx5_vdpa_dev *mvdev, u8 cmd) · 7 unmodified lines
14891489 break;
14901490 }
14911491
1492+ if (is_zero_ether_addr(mac))
1493+ break;
1494+
14921495 if (!is_zero_ether_addr(ndev->config.mac)) {
14931496 if (mlx5_mpfs_del_mac(pfmdev, ndev->config.mac)) {
14941497 mlx5_vdpa_warn(mvdev, "failed to delete old MAC %pM from MPFS table\n",
@@ -1503,7 +1506,47 @@ static virtio_net_ctrl_ack handle_ctrl_mac(struct mlx5_vdpa_dev *mvdev, u8 cmd) · 8 unmodified lines
15031506 break;
15041507 }
15051508
1509+ /* backup the original mac address so that if failed to add the forward rules
1510+ * we could restore it
1511+ */
1512+ memcpy(mac_back, ndev->config.mac, ETH_ALEN);
1513+
15061514 memcpy(ndev->config.mac, mac, ETH_ALEN);
1515+
1516+ /* Need recreate the flow table entry, so that the packet could forward back
1517+ */
1518+ remove_fwd_to_tir(ndev);
1519+
1520+ if (add_fwd_to_tir(ndev)) {
1521+ mlx5_vdpa_warn(mvdev, "failed to insert forward rules, try to restore\n");
1522+
1523+ /* Although it hardly run here, we still need double check */
1524+ if (is_zero_ether_addr(mac_back)) {
1525+ mlx5_vdpa_warn(mvdev, "restore mac failed: Original MAC is zero\n");
1526+ break;
1527+ }
1528+
1529+ /* Try to restore original mac address to MFPS table, and try to restore
1530+ * the forward rule entry.
1531+ */
1532+ if (mlx5_mpfs_del_mac(pfmdev, ndev->config.mac)) {
1533+ mlx5_vdpa_warn(mvdev, "restore mac failed: delete MAC %pM from MPFS table failed\n",
1534+ ndev->config.mac);
1535+ }
1536+
1537+ if (mlx5_mpfs_add_mac(pfmdev, mac_back)) {
1538+ mlx5_vdpa_warn(mvdev, "restore mac failed: insert old MAC %pM into MPFS table failed\n",
1539+ mac_back);
1540+ }
1541+
1542+ memcpy(ndev->config.mac, mac_back, ETH_ALEN);
1543+
1544+ if (add_fwd_to_tir(ndev))
1545+ mlx5_vdpa_warn(mvdev, "restore forward rules failed: insert forward rules failed\n");
1546+
1547+ break;
1548+ }
1549+
15071550 status = VIRTIO_NET_OK;
15081551 break;
15091552
@@ -1669,7 +1712,7 @@ static void mlx5_vdpa_kick_vq(struct vdpa_device *vdev, u16 idx) · 159 unmodified lines
16691712 return;
16701713
16711714 if (unlikely(is_ctrl_vq_idx(mvdev, idx))) {
1672− if (!mvdev->cvq.ready)
1715+ if (!mvdev->wq || !mvdev->cvq.ready)
16731716 return;
16741717
16751718 wqent = kzalloc(sizeof(*wqent), GFP_ATOMIC);
@@ -2565,6 +2608,28 @@ static int event_handler(struct notifier_block *nb, unsigned long event, void *p · 889 unmodified lines
25652608 return ret;
25662609 }
25672610
2611+static int config_func_mtu(struct mlx5_core_dev *mdev, u16 mtu)
2612+{
2613+ int inlen = MLX5_ST_SZ_BYTES(modify_nic_vport_context_in);
2614+ void *in;
2615+ int err;
2616+
2617+ in = kvzalloc(inlen, GFP_KERNEL);
2618+ if (!in)
2619+ return -ENOMEM;
2620+
2621+ MLX5_SET(modify_nic_vport_context_in, in, field_select.mtu, 1);
2622+ MLX5_SET(modify_nic_vport_context_in, in, nic_vport_context.mtu,
2623+ mtu + MLX5V_ETH_HARD_MTU);
2624+ MLX5_SET(modify_nic_vport_context_in, in, opcode,
2625+ MLX5_CMD_OP_MODIFY_NIC_VPORT_CONTEXT);
2626+
2627+ err = mlx5_cmd_exec_in(mdev, modify_nic_vport_context, in);
2628+
2629+ kvfree(in);
2630+ return err;
2631+}
2632+
25682633 static int mlx5_vdpa_dev_add(struct vdpa_mgmt_dev *v_mdev, const char *name,
25692634 const struct vdpa_dev_set_config *add_config)
25702635 {
@@ -2624,6 +2689,13 @@ static int mlx5_vdpa_dev_add(struct vdpa_mgmt_dev *v_mdev, const char *name, · 53 unmodified lines
26242689 init_mvqs(ndev);
26252690 mutex_init(&ndev->reslock);
26262691 config = &ndev->config;
2692+
2693+ if (add_config->mask & BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MTU)) {
2694+ err = config_func_mtu(mdev, add_config->net.mtu);
2695+ if (err)
2696+ goto err_mtu;
2697+ }
2698+
26272699 err = query_mtu(mdev, &mtu);
26282700 if (err)
26292701 goto err_mtu;
@@ -2707,9 +2779,12 @@ static void mlx5_vdpa_dev_del(struct vdpa_mgmt_dev *v_mdev, struct vdpa_device * · 77 unmodified lines
27072779 struct mlx5_vdpa_mgmtdev *mgtdev = container_of(v_mdev, struct mlx5_vdpa_mgmtdev, mgtdev);
27082780 struct mlx5_vdpa_dev *mvdev = to_mvdev(dev);
27092781 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev);
2782+ struct workqueue_struct *wq;
27102783
27112784 mlx5_notifier_unregister(mvdev->mdev, &ndev->nb);
2712− destroy_workqueue(mvdev->wq);
2785+ wq = mvdev->wq;
2786+ mvdev->wq = NULL;
2787+ destroy_workqueue(wq);
27132788 _vdpa_unregister_device(dev);
27142789 mgtdev->ndev = NULL;
27152790 }
@@ -2741,7 +2816,8 @@ static int mlx5v_probe(struct auxiliary_device *adev, · 25 unmodified lines
27412816 mgtdev->mgtdev.device = mdev->device;
27422817 mgtdev->mgtdev.id_table = id_table;
27432818 mgtdev->mgtdev.config_attr_mask = BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MACADDR) |
2744− BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MAX_VQP);
2819+ BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MAX_VQP) |
2820+ BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MTU);
27452821 mgtdev->mgtdev.max_supported_vqs =
27462822 MLX5_CAP_DEV_VDPA_EMULATION(mdev, max_num_virtio_queues) + 1;
27472823 mgtdev->mgtdev.supported_features = get_supported_features(mdev);
drivers/vdpa/vdpa.c+3−3
@@ -232,7 +232,7 @@ static int vdpa_name_match(struct device *dev, const void *data) · 231 unmodified lines
232232 return (strcmp(dev_name(&vdev->dev), data) == 0);
233233 }
234234
235−static int __vdpa_register_device(struct vdpa_device *vdev, int nvqs)
235+static int __vdpa_register_device(struct vdpa_device *vdev, u32 nvqs)
236236 {
237237 struct device *dev;
238238
@@ -257,7 +257,7 @@ static int __vdpa_register_device(struct vdpa_device *vdev, int nvqs) · 18 unmodified lines
257257 *
258258 * Return: Returns an error when fail to add device to vDPA bus
259259 */
260−int _vdpa_register_device(struct vdpa_device *vdev, int nvqs)
260+int _vdpa_register_device(struct vdpa_device *vdev, u32 nvqs)
261261 {
262262 if (!vdev->mdev)
263263 return -EINVAL;
@@ -274,7 +274,7 @@ EXPORT_SYMBOL_GPL(_vdpa_register_device); · 10 unmodified lines
274274 *
275275 * Return: Returns an error when fail to add to vDPA bus
276276 */
277−int vdpa_register_device(struct vdpa_device *vdev, int nvqs)
277+int vdpa_register_device(struct vdpa_device *vdev, u32 nvqs)
278278 {
279279 int err;
280280
drivers/vhost/iotlb.c+5−1
@@ -62,8 +62,12 @@ int vhost_iotlb_add_range_ctx(struct vhost_iotlb *iotlb, · 61 unmodified lines
6262 */
6363 if (start == 0 && last == ULONG_MAX) {
6464 u64 mid = last / 2;
65+ int err = vhost_iotlb_add_range_ctx(iotlb, start, mid, addr,
66+ perm, opaque);
67+
68+ if (err)
69+ return err;
6570
66− vhost_iotlb_add_range_ctx(iotlb, start, mid, addr, perm, opaque);
6771 addr += mid + 1;
6872 start = mid + 1;
6973 }
drivers/vhost/vdpa.c+40−5
@@ -42,7 +42,7 @@ struct vhost_vdpa { · 41 unmodified lines
4242 struct device dev;
4343 struct cdev cdev;
4444 atomic_t opened;
45− int nvqs;
45+ u32 nvqs;
4646 int virtio_id;
4747 int minor;
4848 struct eventfd_ctx *config_ctx;
@@ -97,8 +97,11 @@ static void vhost_vdpa_setup_vq_irq(struct vhost_vdpa *v, u16 qid) · 48 unmodified lines
9797 return;
9898
9999 irq = ops->get_vq_irq(vdpa, qid);
100+ if (irq < 0)
101+ return;
102+
100103 irq_bypass_unregister_producer(&vq->call_ctx.producer);
101− if (!vq->call_ctx.ctx || irq < 0)
104+ if (!vq->call_ctx.ctx)
102105 return;
103106
104107 vq->call_ctx.producer.token = vq->call_ctx.ctx;
@@ -158,7 +161,8 @@ static long vhost_vdpa_set_status(struct vhost_vdpa *v, u8 __user *statusp) · 53 unmodified lines
158161 struct vdpa_device *vdpa = v->vdpa;
159162 const struct vdpa_config_ops *ops = vdpa->config;
160163 u8 status, status_old;
161− int ret, nvqs = v->nvqs;
164+ u32 nvqs = v->nvqs;
165+ int ret;
162166 u16 i;
163167
164168 if (copy_from_user(&status, statusp, sizeof(status)))
@@ -355,6 +359,30 @@ static long vhost_vdpa_get_iova_range(struct vhost_vdpa *v, u32 __user *argp) · 190 unmodified lines
355359 return 0;
356360 }
357361
362+static long vhost_vdpa_get_config_size(struct vhost_vdpa *v, u32 __user *argp)
363+{
364+ struct vdpa_device *vdpa = v->vdpa;
365+ const struct vdpa_config_ops *ops = vdpa->config;
366+ u32 size;
367+
368+ size = ops->get_config_size(vdpa);
369+
370+ if (copy_to_user(argp, &size, sizeof(size)))
371+ return -EFAULT;
372+
373+ return 0;
374+}
375+
376+static long vhost_vdpa_get_vqs_count(struct vhost_vdpa *v, u32 __user *argp)
377+{
378+ struct vdpa_device *vdpa = v->vdpa;
379+
380+ if (copy_to_user(argp, &vdpa->nvqs, sizeof(vdpa->nvqs)))
381+ return -EFAULT;
382+
383+ return 0;
384+}
385+
358386 static long vhost_vdpa_vring_ioctl(struct vhost_vdpa *v, unsigned int cmd,
359387 void __user *argp)
360388 {
@@ -492,6 +520,12 @@ static long vhost_vdpa_unlocked_ioctl(struct file *filep, · 131 unmodified lines
492520 case VHOST_VDPA_GET_IOVA_RANGE:
493521 r = vhost_vdpa_get_iova_range(v, argp);
494522 break;
523+ case VHOST_VDPA_GET_CONFIG_SIZE:
524+ r = vhost_vdpa_get_config_size(v, argp);
525+ break;
526+ case VHOST_VDPA_GET_VQS_COUNT:
527+ r = vhost_vdpa_get_vqs_count(v, argp);
528+ break;
495529 default:
496530 r = vhost_dev_ioctl(&v->vdev, cmd, argp);
497531 if (r == -ENOIOCTLCMD)
@@ -948,7 +982,8 @@ static int vhost_vdpa_open(struct inode *inode, struct file *filep) · 450 unmodified lines
948982 struct vhost_vdpa *v;
949983 struct vhost_dev *dev;
950984 struct vhost_virtqueue **vqs;
951− int nvqs, i, r, opened;
985+ int r, opened;
986+ u32 i, nvqs;
952987
953988 v = container_of(inode->i_cdev, struct vhost_vdpa, cdev);
954989
@@ -1001,7 +1036,7 @@ err: · 46 unmodified lines
10011036
10021037 static void vhost_vdpa_clean_irq(struct vhost_vdpa *v)
10031038 {
1004− int i;
1039+ u32 i;
10051040
10061041 for (i = 0; i < v->nvqs; i++)
10071042 vhost_vdpa_unsetup_vq_irq(v, i);
drivers/vhost/vhost.c+2−1
@@ -2550,8 +2550,9 @@ bool vhost_enable_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq) · 2549 unmodified lines
25502550 &vq->avail->idx, r);
25512551 return false;
25522552 }
2553+ vq->avail_idx = vhost16_to_cpu(vq, avail_idx);
25532554
2554− return vhost16_to_cpu(vq, avail_idx) != vq->avail_idx;
2555+ return vq->avail_idx != vq->last_avail_idx;
25552556 }
25562557 EXPORT_SYMBOL_GPL(vhost_enable_notify);
25572558
drivers/virtio/Kconfig+4−3
@@ -105,7 +105,7 @@ config VIRTIO_BALLOON · 104 unmodified lines
105105
106106 config VIRTIO_MEM
107107 tristate "Virtio mem driver"
108− depends on X86_64
108+ depends on X86_64 || ARM64
109109 depends on VIRTIO
110110 depends on MEMORY_HOTPLUG
111111 depends on MEMORY_HOTREMOVE
@@ -115,8 +115,9 @@ config VIRTIO_MEM · 3 unmodified lines
115115 This driver provides access to virtio-mem paravirtualized memory
116116 devices, allowing to hotplug and hotunplug memory.
117117
118− This driver was only tested under x86-64, but should theoretically
119− work on all architectures that support memory hotplug and hotremove.
118+ This driver was only tested under x86-64 and arm64, but should
119+ theoretically work on all architectures that support memory hotplug
120+ and hotremove.
120121
121122 If unsure, say M.
122123
drivers/virtio/virtio.c+3−2
@@ -526,8 +526,9 @@ int virtio_device_restore(struct virtio_device *dev) · 525 unmodified lines
526526 goto err;
527527 }
528528
529− /* Finally, tell the device we're all set */
530− virtio_add_status(dev, VIRTIO_CONFIG_S_DRIVER_OK);
529+ /* If restore didn't do it, mark device DRIVER_OK ourselves. */
530+ if (!(dev->config->get_status(dev) & VIRTIO_CONFIG_S_DRIVER_OK))
531+ virtio_device_ready(dev);
531532
532533 virtio_config_enable(dev);
533534
drivers/virtio/virtio_pci_common.c+7−41
@@ -24,46 +24,17 @@ MODULE_PARM_DESC(force_legacy, · 23 unmodified lines
2424 "Force legacy mode for transitional virtio 1 devices");
2525 #endif
2626
27−/* disable irq handlers */
28−void vp_disable_cbs(struct virtio_device *vdev)
27+/* wait for pending irq handlers */
28+void vp_synchronize_vectors(struct virtio_device *vdev)
2929 {
3030 struct virtio_pci_device *vp_dev = to_vp_device(vdev);
3131 int i;
3232
33− if (vp_dev->intx_enabled) {
34− /*
35− * The below synchronize() guarantees that any
36− * interrupt for this line arriving after
37− * synchronize_irq() has completed is guaranteed to see
38− * intx_soft_enabled == false.
39− */
40− WRITE_ONCE(vp_dev->intx_soft_enabled, false);
33+ if (vp_dev->intx_enabled)
4134 synchronize_irq(vp_dev->pci_dev->irq);
42− }
43−
44− for (i = 0; i < vp_dev->msix_vectors; ++i)
45− disable_irq(pci_irq_vector(vp_dev->pci_dev, i));
46−}
47−
48−/* enable irq handlers */
49−void vp_enable_cbs(struct virtio_device *vdev)
50−{
51− struct virtio_pci_device *vp_dev = to_vp_device(vdev);
52− int i;
53−
54− if (vp_dev->intx_enabled) {
55− disable_irq(vp_dev->pci_dev->irq);
56− /*
57− * The above disable_irq() provides TSO ordering and
58− * as such promotes the below store to store-release.
59− */
60− WRITE_ONCE(vp_dev->intx_soft_enabled, true);
61− enable_irq(vp_dev->pci_dev->irq);
62− return;
63− }
6435
6536 for (i = 0; i < vp_dev->msix_vectors; ++i)
66− enable_irq(pci_irq_vector(vp_dev->pci_dev, i));
37+ synchronize_irq(pci_irq_vector(vp_dev->pci_dev, i));
6738 }
6839
6940 /* the notify function used when creating a virt queue */
@@ -113,9 +84,6 @@ static irqreturn_t vp_interrupt(int irq, void *opaque) · 43 unmodified lines
11384 struct virtio_pci_device *vp_dev = opaque;
11485 u8 isr;
11586
116− if (!READ_ONCE(vp_dev->intx_soft_enabled))
117− return IRQ_NONE;
118−
11987 /* reading the ISR has the effect of also clearing it so it's very
12088 * important to save off the value. */
12189 isr = ioread8(vp_dev->isr);
@@ -173,8 +141,7 @@ static int vp_request_msix_vectors(struct virtio_device *vdev, int nvectors, · 51 unmodified lines
173141 snprintf(vp_dev->msix_names[v], sizeof *vp_dev->msix_names,
174142 "%s-config", name);
175143 err = request_irq(pci_irq_vector(vp_dev->pci_dev, v),
176− vp_config_changed, IRQF_NO_AUTOEN,
177− vp_dev->msix_names[v],
144+ vp_config_changed, 0, vp_dev->msix_names[v],
178145 vp_dev);
179146 if (err)
180147 goto error;
@@ -193,8 +160,7 @@ static int vp_request_msix_vectors(struct virtio_device *vdev, int nvectors, · 12 unmodified lines
193160 snprintf(vp_dev->msix_names[v], sizeof *vp_dev->msix_names,
194161 "%s-virtqueues", name);
195162 err = request_irq(pci_irq_vector(vp_dev->pci_dev, v),
196− vp_vring_interrupt, IRQF_NO_AUTOEN,
197− vp_dev->msix_names[v],
163+ vp_vring_interrupt, 0, vp_dev->msix_names[v],
198164 vp_dev);
199165 if (err)
200166 goto error;
@@ -371,7 +337,7 @@ static int vp_find_vqs_msix(struct virtio_device *vdev, unsigned nvqs, · 170 unmodified lines
371337 "%s-%s",
372338 dev_name(&vp_dev->vdev.dev), names[i]);
373339 err = request_irq(pci_irq_vector(vp_dev->pci_dev, msix_vec),
374− vring_interrupt, IRQF_NO_AUTOEN,
340+ vring_interrupt, 0,
375341 vp_dev->msix_names[msix_vec],
376342 vqs[i]);
377343 if (err)
drivers/virtio/virtio_pci_common.h+2−5
@@ -63,7 +63,6 @@ struct virtio_pci_device { · 62 unmodified lines
6363 /* MSI-X support */
6464 int msix_enabled;
6565 int intx_enabled;
66− bool intx_soft_enabled;
6766 cpumask_var_t *msix_affinity_masks;
6867 /* Name strings for interrupts. This size should be enough,
6968 * and I'm too lazy to allocate each name separately. */
@@ -102,10 +101,8 @@ static struct virtio_pci_device *to_vp_device(struct virtio_device *vdev) · 32 unmodified lines
102101 return container_of(vdev, struct virtio_pci_device, vdev);
103102 }
104103
105−/* disable irq handlers */
106−void vp_disable_cbs(struct virtio_device *vdev);
107−/* enable irq handlers */
108−void vp_enable_cbs(struct virtio_device *vdev);
104+/* wait for pending irq handlers */
105+void vp_synchronize_vectors(struct virtio_device *vdev);
109106 /* the notify function used when creating a virt queue */
110107 bool vp_notify(struct virtqueue *vq);
111108 /* the config->del_vqs() implementation */
drivers/virtio/virtio_pci_legacy.c+2−3
@@ -98,8 +98,8 @@ static void vp_reset(struct virtio_device *vdev) · 97 unmodified lines
9898 /* Flush out the status write, and flush in device writes,
9999 * including MSi-X interrupts, if any. */
100100 vp_legacy_get_status(&vp_dev->ldev);
101− /* Disable VQ/configuration callbacks. */
102− vp_disable_cbs(vdev);
101+ /* Flush pending VQ/configuration callbacks. */
102+ vp_synchronize_vectors(vdev);
103103 }
104104
105105 static u16 vp_config_vector(struct virtio_pci_device *vp_dev, u16 vector)
@@ -185,7 +185,6 @@ static void del_vq(struct virtio_pci_vq_info *info) · 79 unmodified lines
185185 }
186186
187187 static const struct virtio_config_ops virtio_pci_config_ops = {
188− .enable_cbs = vp_enable_cbs,
189188 .get = vp_get,
190189 .set = vp_set,
191190 .get_status = vp_get_status,
drivers/virtio/virtio_pci_modern.c+11−7
@@ -172,8 +172,8 @@ static void vp_reset(struct virtio_device *vdev) · 171 unmodified lines
172172 */
173173 while (vp_modern_get_status(mdev))
174174 msleep(1);
175− /* Disable VQ/configuration callbacks. */
176− vp_disable_cbs(vdev);
175+ /* Flush pending VQ/configuration callbacks. */
176+ vp_synchronize_vectors(vdev);
177177 }
178178
179179 static u16 vp_config_vector(struct virtio_pci_device *vp_dev, u16 vector)
@@ -293,7 +293,7 @@ static int virtio_pci_find_shm_cap(struct pci_dev *dev, u8 required_id, · 113 unmodified lines
293293
294294 for (pos = pci_find_capability(dev, PCI_CAP_ID_VNDR); pos > 0;
295295 pos = pci_find_next_capability(dev, pos, PCI_CAP_ID_VNDR)) {
296− u8 type, cap_len, id;
296+ u8 type, cap_len, id, res_bar;
297297 u32 tmp32;
298298 u64 res_offset, res_length;
299299
@@ -315,9 +315,14 @@ static int virtio_pci_find_shm_cap(struct pci_dev *dev, u8 required_id, · 15 unmodified lines
315315 if (id != required_id)
316316 continue;
317317
318− /* Type, and ID match, looks good */
319318 pci_read_config_byte(dev, pos + offsetof(struct virtio_pci_cap,
320− bar), bar);
319+ bar), &res_bar);
320+ if (res_bar >= PCI_STD_NUM_BARS)
321+ continue;
322+
323+ /* Type and ID match, and the BAR value isn't reserved.
324+ * Looks good.
325+ */
321326
322327 /* Read the lower 32bit of length and offset */
323328 pci_read_config_dword(dev, pos + offsetof(struct virtio_pci_cap,
@@ -337,6 +342,7 @@ static int virtio_pci_find_shm_cap(struct pci_dev *dev, u8 required_id, · 13 unmodified lines
337342 length_hi), &tmp32);
338343 res_length |= ((u64)tmp32) << 32;
339344
345+ *bar = res_bar;
340346 *offset = res_offset;
341347 *len = res_length;
342348
@@ -380,7 +386,6 @@ static bool vp_get_shm_region(struct virtio_device *vdev, · 37 unmodified lines
380386 }
381387
382388 static const struct virtio_config_ops virtio_pci_config_nodev_ops = {
383− .enable_cbs = vp_enable_cbs,
384389 .get = NULL,
385390 .set = NULL,
386391 .generation = vp_generation,
@@ -398,7 +403,6 @@ static const struct virtio_config_ops virtio_pci_config_nodev_ops = { · 11 unmodified lines
398403 };
399404
400405 static const struct virtio_config_ops virtio_pci_config_ops = {
401− .enable_cbs = vp_enable_cbs,
402406 .get = vp_get,
403407 .set = vp_set,
404408 .generation = vp_generation,
drivers/virtio/virtio_pci_modern_dev.c+8−1
@@ -35,6 +35,13 @@ vp_modern_map_capability(struct virtio_pci_modern_device *mdev, int off, · 34 unmodified lines
3535 pci_read_config_dword(dev, off + offsetof(struct virtio_pci_cap, length),
3636 &length);
3737
38+ /* Check if the BAR may have changed since we requested the region. */
39+ if (bar >= PCI_STD_NUM_BARS || !(mdev->modern_bars & (1 << bar))) {
40+ dev_err(&dev->dev,
41+ "virtio_pci: bar unexpectedly changed to %u\n", bar);
42+ return NULL;
43+ }
44+
3845 if (length <= start) {
3946 dev_err(&dev->dev,
4047 "virtio_pci: bad capability len %u (>%u expected)\n",
@@ -120,7 +127,7 @@ static inline int virtio_pci_find_capability(struct pci_dev *dev, u8 cfg_type, · 79 unmodified lines
120127 &bar);
121128
122129 /* Ignore structures with reserved BAR values */
123− if (bar > 0x5)
130+ if (bar >= PCI_STD_NUM_BARS)
124131 continue;
125132
126133 if (type == cfg_type) {
drivers/virtio/virtio_ring.c+18−35
@@ -379,19 +379,11 @@ static void vring_unmap_one_split_indirect(const struct vring_virtqueue *vq, · 378 unmodified lines
379379
380380 flags = virtio16_to_cpu(vq->vq.vdev, desc->flags);
381381
382− if (flags & VRING_DESC_F_INDIRECT) {
383− dma_unmap_single(vring_dma_dev(vq),
384− virtio64_to_cpu(vq->vq.vdev, desc->addr),
385− virtio32_to_cpu(vq->vq.vdev, desc->len),
386− (flags & VRING_DESC_F_WRITE) ?
387− DMA_FROM_DEVICE : DMA_TO_DEVICE);
388− } else {
389− dma_unmap_page(vring_dma_dev(vq),
390− virtio64_to_cpu(vq->vq.vdev, desc->addr),
391− virtio32_to_cpu(vq->vq.vdev, desc->len),
392− (flags & VRING_DESC_F_WRITE) ?
393− DMA_FROM_DEVICE : DMA_TO_DEVICE);
394− }
382+ dma_unmap_page(vring_dma_dev(vq),
383+ virtio64_to_cpu(vq->vq.vdev, desc->addr),
384+ virtio32_to_cpu(vq->vq.vdev, desc->len),
385+ (flags & VRING_DESC_F_WRITE) ?
386+ DMA_FROM_DEVICE : DMA_TO_DEVICE);
395387 }
396388
397389 static unsigned int vring_unmap_one_split(const struct vring_virtqueue *vq,
@@ -984,24 +976,24 @@ static struct virtqueue *vring_create_virtqueue_split( · 586 unmodified lines
984976 * Packed ring specific functions - *_packed().
985977 */
986978
987−static void vring_unmap_state_packed(const struct vring_virtqueue *vq,
988− struct vring_desc_extra *state)
979+static void vring_unmap_extra_packed(const struct vring_virtqueue *vq,
980+ struct vring_desc_extra *extra)
989981 {
990982 u16 flags;
991983
992984 if (!vq->use_dma_api)
993985 return;
994986
995− flags = state->flags;
987+ flags = extra->flags;
996988
997989 if (flags & VRING_DESC_F_INDIRECT) {
998990 dma_unmap_single(vring_dma_dev(vq),
999− state->addr, state->len,
991+ extra->addr, extra->len,
1000992 (flags & VRING_DESC_F_WRITE) ?
1001993 DMA_FROM_DEVICE : DMA_TO_DEVICE);
1002994 } else {
1003995 dma_unmap_page(vring_dma_dev(vq),
1004− state->addr, state->len,
996+ extra->addr, extra->len,
1005997 (flags & VRING_DESC_F_WRITE) ?
1006998 DMA_FROM_DEVICE : DMA_TO_DEVICE);
1007999 }
@@ -1017,19 +1009,11 @@ static void vring_unmap_desc_packed(const struct vring_virtqueue *vq, · 9 unmodified lines
10171009
10181010 flags = le16_to_cpu(desc->flags);
10191011
1020− if (flags & VRING_DESC_F_INDIRECT) {
1021− dma_unmap_single(vring_dma_dev(vq),
1022− le64_to_cpu(desc->addr),
1023− le32_to_cpu(desc->len),
1024− (flags & VRING_DESC_F_WRITE) ?
1025− DMA_FROM_DEVICE : DMA_TO_DEVICE);
1026− } else {
1027− dma_unmap_page(vring_dma_dev(vq),
1028− le64_to_cpu(desc->addr),
1029− le32_to_cpu(desc->len),
1030− (flags & VRING_DESC_F_WRITE) ?
1031− DMA_FROM_DEVICE : DMA_TO_DEVICE);
1032− }
1012+ dma_unmap_page(vring_dma_dev(vq),
1013+ le64_to_cpu(desc->addr),
1014+ le32_to_cpu(desc->len),
1015+ (flags & VRING_DESC_F_WRITE) ?
1016+ DMA_FROM_DEVICE : DMA_TO_DEVICE);
10331017 }
10341018
10351019 static struct vring_packed_desc *alloc_indirect_packed(unsigned int total_sg,
@@ -1303,8 +1287,7 @@ unmap_release: · 267 unmodified lines
13031287 for (n = 0; n < total_sg; n++) {
13041288 if (i == err_idx)
13051289 break;
1306− vring_unmap_state_packed(vq,
1307− &vq->packed.desc_extra[curr]);
1290+ vring_unmap_extra_packed(vq, &vq->packed.desc_extra[curr]);
13081291 curr = vq->packed.desc_extra[curr].next;
13091292 i++;
13101293 if (i >= vq->packed.vring.num)
@@ -1383,8 +1366,8 @@ static void detach_buf_packed(struct vring_virtqueue *vq, · 72 unmodified lines
13831366 if (unlikely(vq->use_dma_api)) {
13841367 curr = id;
13851368 for (i = 0; i < state->num; i++) {
1386− vring_unmap_state_packed(vq,
1387− &vq->packed.desc_extra[curr]);
1369+ vring_unmap_extra_packed(vq,
1370+ &vq->packed.desc_extra[curr]);
13881371 curr = vq->packed.desc_extra[curr].next;
13891372 }
13901373 }
include/linux/balloon_compaction.h+0−22
@@ -80,12 +80,6 @@ static inline void balloon_devinfo_init(struct balloon_dev_info *balloon) · 79 unmodified lines
8080
8181 #ifdef CONFIG_BALLOON_COMPACTION
8282 extern const struct address_space_operations balloon_aops;
83−extern bool balloon_page_isolate(struct page *page,
84− isolate_mode_t mode);
85−extern void balloon_page_putback(struct page *page);
86−extern int balloon_page_migrate(struct address_space *mapping,
87− struct page *newpage,
88− struct page *page, enum migrate_mode mode);
8983
9084 /*
9185 * balloon_page_insert - insert a page into the balloon's page list and make
@@ -155,22 +149,6 @@ static inline void balloon_page_delete(struct page *page) · 63 unmodified lines
155149 list_del(&page->lru);
156150 }
157151
158−static inline bool balloon_page_isolate(struct page *page)
159−{
160− return false;
161−}
162−
163−static inline void balloon_page_putback(struct page *page)
164−{
165− return;
166−}
167−
168−static inline int balloon_page_migrate(struct page *newpage,
169− struct page *page, enum migrate_mode mode)
170−{
171− return 0;
172−}
173−
174152 static inline gfp_t balloon_mapping_gfp_mask(void)
175153 {
176154 return GFP_HIGHUSER;
include/linux/vdpa.h+5−4
@@ -83,7 +83,7 @@ struct vdpa_device { · 82 unmodified lines
8383 unsigned int index;
8484 bool features_valid;
8585 bool use_va;
86− int nvqs;
86+ u32 nvqs;
8787 struct vdpa_mgmt_dev *mdev;
8888 };
8989
@@ -207,7 +207,8 @@ struct vdpa_map_file { · 117 unmodified lines
207207 * @reset: Reset device
208208 * @vdev: vdpa device
209209 * Returns integer: success (0) or error (< 0)
210− * @get_config_size: Get the size of the configuration space
210+ * @get_config_size: Get the size of the configuration space includes
211+ * fields that are conditional on feature bits.
211212 * @vdev: vdpa device
212213 * Returns size_t: configuration size
213214 * @get_config: Read from device specific configuration space
@@ -337,10 +338,10 @@ struct vdpa_device *__vdpa_alloc_device(struct device *parent, · 123 unmodified lines
337338 dev_struct, member)), name, use_va), \
338339 dev_struct, member)
339340
340−int vdpa_register_device(struct vdpa_device *vdev, int nvqs);
341+int vdpa_register_device(struct vdpa_device *vdev, u32 nvqs);
341342 void vdpa_unregister_device(struct vdpa_device *vdev);
342343
343−int _vdpa_register_device(struct vdpa_device *vdev, int nvqs);
344+int _vdpa_register_device(struct vdpa_device *vdev, u32 nvqs);
344345 void _vdpa_unregister_device(struct vdpa_device *vdev);
345346
346347 /**
include/uapi/linux/vhost.h+7−0
@@ -150,4 +150,11 @@ · 149 unmodified lines
150150 /* Get the valid iova range */
151151 #define VHOST_VDPA_GET_IOVA_RANGE _IOR(VHOST_VIRTIO, 0x78, \
152152 struct vhost_vdpa_iova_range)
153+
154+/* Get the config size */
155+#define VHOST_VDPA_GET_CONFIG_SIZE _IOR(VHOST_VIRTIO, 0x79, __u32)
156+
157+/* Get the count of all virtqueues */
158+#define VHOST_VDPA_GET_VQS_COUNT _IOR(VHOST_VIRTIO, 0x80, __u32)
159+
153160 #endif
include/uapi/linux/virtio_config.h+6−0
@@ -82,6 +82,12 @@ · 81 unmodified lines
8282 /* This feature indicates support for the packed virtqueue layout. */
8383 #define VIRTIO_F_RING_PACKED 34
8484
85+/*
86+ * Inorder feature indicates that all buffers are used by the device
87+ * in the same order in which they have been made available.
88+ */
89+#define VIRTIO_F_IN_ORDER 35
90+
8591 /*
8692 * This feature indicates that memory accesses by the driver and the
8793 * device are ordered in a way described by the platform.
include/uapi/linux/virtio_crypto.h+81−1
@@ -37,6 +37,7 @@ · 36 unmodified lines
3737 #define VIRTIO_CRYPTO_SERVICE_HASH 1
3838 #define VIRTIO_CRYPTO_SERVICE_MAC 2
3939 #define VIRTIO_CRYPTO_SERVICE_AEAD 3
40+#define VIRTIO_CRYPTO_SERVICE_AKCIPHER 4
4041
4142 #define VIRTIO_CRYPTO_OPCODE(service, op) (((service) << 8) | (op))
4243
@@ -57,6 +58,10 @@ struct virtio_crypto_ctrl_header { · 14 unmodified lines
5758 VIRTIO_CRYPTO_OPCODE(VIRTIO_CRYPTO_SERVICE_AEAD, 0x02)
5859 #define VIRTIO_CRYPTO_AEAD_DESTROY_SESSION \
5960 VIRTIO_CRYPTO_OPCODE(VIRTIO_CRYPTO_SERVICE_AEAD, 0x03)
61+#define VIRTIO_CRYPTO_AKCIPHER_CREATE_SESSION \
62+ VIRTIO_CRYPTO_OPCODE(VIRTIO_CRYPTO_SERVICE_AKCIPHER, 0x04)
63+#define VIRTIO_CRYPTO_AKCIPHER_DESTROY_SESSION \
64+ VIRTIO_CRYPTO_OPCODE(VIRTIO_CRYPTO_SERVICE_AKCIPHER, 0x05)
6065 __le32 opcode;
6166 __le32 algo;
6267 __le32 flag;
@@ -180,6 +185,58 @@ struct virtio_crypto_aead_create_session_req { · 117 unmodified lines
180185 __u8 padding[32];
181186 };
182187
188+struct virtio_crypto_rsa_session_para {
189+#define VIRTIO_CRYPTO_RSA_RAW_PADDING 0
190+#define VIRTIO_CRYPTO_RSA_PKCS1_PADDING 1
191+ __le32 padding_algo;
192+
193+#define VIRTIO_CRYPTO_RSA_NO_HASH 0
194+#define VIRTIO_CRYPTO_RSA_MD2 1
195+#define VIRTIO_CRYPTO_RSA_MD3 2
196+#define VIRTIO_CRYPTO_RSA_MD4 3
197+#define VIRTIO_CRYPTO_RSA_MD5 4
198+#define VIRTIO_CRYPTO_RSA_SHA1 5
199+#define VIRTIO_CRYPTO_RSA_SHA256 6
200+#define VIRTIO_CRYPTO_RSA_SHA384 7
201+#define VIRTIO_CRYPTO_RSA_SHA512 8
202+#define VIRTIO_CRYPTO_RSA_SHA224 9
203+ __le32 hash_algo;
204+};
205+
206+struct virtio_crypto_ecdsa_session_para {
207+#define VIRTIO_CRYPTO_CURVE_UNKNOWN 0
208+#define VIRTIO_CRYPTO_CURVE_NIST_P192 1
209+#define VIRTIO_CRYPTO_CURVE_NIST_P224 2
210+#define VIRTIO_CRYPTO_CURVE_NIST_P256 3
211+#define VIRTIO_CRYPTO_CURVE_NIST_P384 4
212+#define VIRTIO_CRYPTO_CURVE_NIST_P521 5
213+ __le32 curve_id;
214+ __le32 padding;
215+};
216+
217+struct virtio_crypto_akcipher_session_para {
218+#define VIRTIO_CRYPTO_NO_AKCIPHER 0
219+#define VIRTIO_CRYPTO_AKCIPHER_RSA 1
220+#define VIRTIO_CRYPTO_AKCIPHER_DSA 2
221+#define VIRTIO_CRYPTO_AKCIPHER_ECDSA 3
222+ __le32 algo;
223+
224+#define VIRTIO_CRYPTO_AKCIPHER_KEY_TYPE_PUBLIC 1
225+#define VIRTIO_CRYPTO_AKCIPHER_KEY_TYPE_PRIVATE 2
226+ __le32 keytype;
227+ __le32 keylen;
228+
229+ union {
230+ struct virtio_crypto_rsa_session_para rsa;
231+ struct virtio_crypto_ecdsa_session_para ecdsa;
232+ } u;
233+};
234+
235+struct virtio_crypto_akcipher_create_session_req {
236+ struct virtio_crypto_akcipher_session_para para;
237+ __u8 padding[36];
238+};
239+
183240 struct virtio_crypto_alg_chain_session_para {
184241 #define VIRTIO_CRYPTO_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER 1
185242 #define VIRTIO_CRYPTO_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH 2
@@ -247,6 +304,8 @@ struct virtio_crypto_op_ctrl_req { · 61 unmodified lines
247304 mac_create_session;
248305 struct virtio_crypto_aead_create_session_req
249306 aead_create_session;
307+ struct virtio_crypto_akcipher_create_session_req
308+ akcipher_create_session;
250309 struct virtio_crypto_destroy_session_req
251310 destroy_session;
252311 __u8 padding[56];
@@ -266,6 +325,14 @@ struct virtio_crypto_op_header { · 13 unmodified lines
266325 VIRTIO_CRYPTO_OPCODE(VIRTIO_CRYPTO_SERVICE_AEAD, 0x00)
267326 #define VIRTIO_CRYPTO_AEAD_DECRYPT \
268327 VIRTIO_CRYPTO_OPCODE(VIRTIO_CRYPTO_SERVICE_AEAD, 0x01)
328+#define VIRTIO_CRYPTO_AKCIPHER_ENCRYPT \
329+ VIRTIO_CRYPTO_OPCODE(VIRTIO_CRYPTO_SERVICE_AKCIPHER, 0x00)
330+#define VIRTIO_CRYPTO_AKCIPHER_DECRYPT \
331+ VIRTIO_CRYPTO_OPCODE(VIRTIO_CRYPTO_SERVICE_AKCIPHER, 0x01)
332+#define VIRTIO_CRYPTO_AKCIPHER_SIGN \
333+ VIRTIO_CRYPTO_OPCODE(VIRTIO_CRYPTO_SERVICE_AKCIPHER, 0x02)
334+#define VIRTIO_CRYPTO_AKCIPHER_VERIFY \
335+ VIRTIO_CRYPTO_OPCODE(VIRTIO_CRYPTO_SERVICE_AKCIPHER, 0x03)
269336 __le32 opcode;
270337 /* algo should be service-specific algorithms */
271338 __le32 algo;
@@ -390,6 +457,16 @@ struct virtio_crypto_aead_data_req { · 118 unmodified lines
390457 __u8 padding[32];
391458 };
392459
460+struct virtio_crypto_akcipher_para {
461+ __le32 src_data_len;
462+ __le32 dst_data_len;
463+};
464+
465+struct virtio_crypto_akcipher_data_req {
466+ struct virtio_crypto_akcipher_para para;
467+ __u8 padding[40];
468+};
469+
393470 /* The request of the data virtqueue's packet */
394471 struct virtio_crypto_op_data_req {
395472 struct virtio_crypto_op_header header;
@@ -399,6 +476,7 @@ struct virtio_crypto_op_data_req { · 3 unmodified lines
399476 struct virtio_crypto_hash_data_req hash_req;
400477 struct virtio_crypto_mac_data_req mac_req;
401478 struct virtio_crypto_aead_data_req aead_req;
479+ struct virtio_crypto_akcipher_data_req akcipher_req;
402480 __u8 padding[48];
403481 } u;
404482 };
@@ -408,6 +486,8 @@ struct virtio_crypto_op_data_req { · 3 unmodified lines
408486 #define VIRTIO_CRYPTO_BADMSG 2
409487 #define VIRTIO_CRYPTO_NOTSUPP 3
410488 #define VIRTIO_CRYPTO_INVSESS 4 /* Invalid session id */
489+#define VIRTIO_CRYPTO_NOSPC 5 /* no free session ID */
490+#define VIRTIO_CRYPTO_KEY_REJECTED 6 /* Signature verification failed */
411491
412492 /* The accelerator hardware is ready */
413493 #define VIRTIO_CRYPTO_S_HW_READY (1 << 0)
@@ -438,7 +518,7 @@ struct virtio_crypto_config { · 24 unmodified lines
438518 __le32 max_cipher_key_len;
439519 /* Maximum length of authenticated key */
440520 __le32 max_auth_key_len;
441− __le32 reserve;
521+ __le32 akcipher_algo;
442522 /* Maximum size of each crypto request's content */
443523 __le64 max_size;
444524 };
mm/balloon_compaction.c+3−3
@@ -203,7 +203,7 @@ EXPORT_SYMBOL_GPL(balloon_page_dequeue); · 202 unmodified lines
203203
204204 #ifdef CONFIG_BALLOON_COMPACTION
205205
206−bool balloon_page_isolate(struct page *page, isolate_mode_t mode)
206+static bool balloon_page_isolate(struct page *page, isolate_mode_t mode)
207207
208208 {
209209 struct balloon_dev_info *b_dev_info = balloon_page_device(page);
@@ -217,7 +217,7 @@ bool balloon_page_isolate(struct page *page, isolate_mode_t mode) · 7 unmodified lines
217217 return true;
218218 }
219219
220−void balloon_page_putback(struct page *page)
220+static void balloon_page_putback(struct page *page)
221221 {
222222 struct balloon_dev_info *b_dev_info = balloon_page_device(page);
223223 unsigned long flags;
@@ -230,7 +230,7 @@ void balloon_page_putback(struct page *page) · 6 unmodified lines
230230
231231
232232 /* move_to_new_page() counterpart for a ballooned page */
233−int balloon_page_migrate(struct address_space *mapping,
233+static int balloon_page_migrate(struct address_space *mapping,
234234 struct page *newpage, struct page *page,
235235 enum migrate_mode mode)
236236 {
tools/virtio/Makefile+2−1
@@ -5,7 +5,8 @@ virtio_test: virtio_ring.o virtio_test.o · 4 unmodified lines
55 vringh_test: vringh_test.o vringh.o virtio_ring.o
66
77 CFLAGS += -g -O2 -Werror -Wno-maybe-uninitialized -Wall -I. -I../include/ -I ../../usr/include/ -Wno-pointer-sign -fno-strict-overflow -fno-strict-aliasing -fno-common -MMD -U_FORTIFY_SOURCE -include ../../include/linux/kconfig.h
8−LDFLAGS += -lpthread
8+CFLAGS += -pthread
9+LDFLAGS += -pthread
910 vpath %.c ../../drivers/virtio ../../drivers/vhost
1011 mod:
1112 ${MAKE} -C `pwd`/../.. M=`pwd`/vhost_test V=${V}
tools/virtio/linux/dma-mapping.h+2−2
@@ -26,8 +26,8 @@ enum dma_data_direction { · 25 unmodified lines
2626 #define dma_map_single(d, p, s, dir) (virt_to_phys(p))
2727 #define dma_mapping_error(...) (0)
2828
29−#define dma_unmap_single(...) do { } while (0)
30−#define dma_unmap_page(...) do { } while (0)
29+#define dma_unmap_single(d, a, s, r) do { (void)(d); (void)(a); (void)(s); (void)(r); } while (0)
30+#define dma_unmap_page(d, a, s, r) do { (void)(d); (void)(a); (void)(s); (void)(r); } while (0)
3131
3232 #define dma_max_mapping_size(...) SIZE_MAX
3333