lael-labs/linux

drivers/crypto/virtio/Kconfig+3−0

 	tristate "VirtIO crypto driver"
 	depends on VIRTIO
 	select CRYPTO_AEAD
+	select CRYPTO_AKCIPHER2
 	select CRYPTO_SKCIPHER
 	select CRYPTO_ENGINE
+	select CRYPTO_RSA
+	select MPILIB
 	help
 	  This driver provides support for virtio crypto device. If you
 	  choose 'M' here, this module will be called virtio_crypto.

drivers/crypto/virtio/Makefile+2−1

@@ -1,6 +1,7 @@
1	# SPDX-License-Identifier: GPL-2.0	1	# SPDX-License-Identifier: GPL-2.0
2	obj-$(CONFIG_CRYPTO_DEV_VIRTIO) += virtio_crypto.o	2	obj-$(CONFIG_CRYPTO_DEV_VIRTIO) += virtio_crypto.o
3	virtio_crypto-objs := \	3	virtio_crypto-objs := \
4	virtio_crypto_algs.o \	4	virtio_crypto_skcipher_algs.o \
		5	virtio_crypto_akcipher_algs.o \
5	virtio_crypto_mgr.o \	6	virtio_crypto_mgr.o \
6	virtio_crypto_core.o	7	virtio_crypto_core.o

drivers/crypto/virtio/virtio_crypto_akcipher_algs.c+585−0

@@ -0,0 +1,585 @@
	1	// SPDX-License-Identifier: GPL-2.0-or-later
	2	/* Asymmetric algorithms supported by virtio crypto device
	3	*
	4	* Authors: zhenwei pi <pizhenwei@bytedance.com>
	5	* lei he <helei.sig11@bytedance.com>
	6	*
	7	* Copyright 2022 Bytedance CO., LTD.
	8	*/
	9
	10	#include <linux/mpi.h>
	11	#include <linux/scatterlist.h>
	12	#include <crypto/algapi.h>
	13	#include <crypto/internal/akcipher.h>
	14	#include <crypto/internal/rsa.h>
	15	#include <linux/err.h>
	16	#include <crypto/scatterwalk.h>
	17	#include <linux/atomic.h>
	18
	19	#include <uapi/linux/virtio_crypto.h>
	20	#include "virtio_crypto_common.h"
	21
	22	struct virtio_crypto_rsa_ctx {
	23	MPI n;
	24	};
	25
	26	struct virtio_crypto_akcipher_ctx {
	27	struct crypto_engine_ctx enginectx;
	28	struct virtio_crypto *vcrypto;
	29	struct crypto_akcipher *tfm;
	30	bool session_valid;
	31	__u64 session_id;
	32	union {
	33	struct virtio_crypto_rsa_ctx rsa_ctx;
	34	};
	35	};
	36
	37	struct virtio_crypto_akcipher_request {
	38	struct virtio_crypto_request base;
	39	struct virtio_crypto_akcipher_ctx *akcipher_ctx;
	40	struct akcipher_request *akcipher_req;
	41	void *src_buf;
	42	void *dst_buf;
	43	uint32_t opcode;
	44	};
	45
	46	struct virtio_crypto_akcipher_algo {
	47	uint32_t algonum;
	48	uint32_t service;
	49	unsigned int active_devs;
	50	struct akcipher_alg algo;
	51	};
	52
	53	static DEFINE_MUTEX(algs_lock);
	54
	55	static void virtio_crypto_akcipher_finalize_req(
	56	struct virtio_crypto_akcipher_request *vc_akcipher_req,
	57	struct akcipher_request *req, int err)
	58	{
	59	virtcrypto_clear_request(&vc_akcipher_req->base);
	60
	61	crypto_finalize_akcipher_request(vc_akcipher_req->base.dataq->engine, req, err);
	62	}
	63
	64	static void virtio_crypto_dataq_akcipher_callback(struct virtio_crypto_request *vc_req, int len)
	65	{
	66	struct virtio_crypto_akcipher_request *vc_akcipher_req =
	67	container_of(vc_req, struct virtio_crypto_akcipher_request, base);
	68	struct akcipher_request *akcipher_req;
	69	int error;
	70
	71	switch (vc_req->status) {
	72	case VIRTIO_CRYPTO_OK:
	73	error = 0;
	74	break;
	75	case VIRTIO_CRYPTO_INVSESS:
	76	case VIRTIO_CRYPTO_ERR:
	77	error = -EINVAL;
	78	break;
	79	case VIRTIO_CRYPTO_BADMSG:
	80	error = -EBADMSG;
	81	break;
	82
	83	case VIRTIO_CRYPTO_KEY_REJECTED:
	84	error = -EKEYREJECTED;
	85	break;
	86
	87	default:
	88	error = -EIO;
	89	break;
	90	}
	91
	92	akcipher_req = vc_akcipher_req->akcipher_req;
	93	if (vc_akcipher_req->opcode != VIRTIO_CRYPTO_AKCIPHER_VERIFY)
	94	sg_copy_from_buffer(akcipher_req->dst, sg_nents(akcipher_req->dst),
	95	vc_akcipher_req->dst_buf, akcipher_req->dst_len);
	96	virtio_crypto_akcipher_finalize_req(vc_akcipher_req, akcipher_req, error);
	97	}
	98
	99	static int virtio_crypto_alg_akcipher_init_session(struct virtio_crypto_akcipher_ctx *ctx,
	100	struct virtio_crypto_ctrl_header header, void para,
	101	const uint8_t *key, unsigned int keylen)
	102	{
	103	struct scatterlist outhdr_sg, key_sg, inhdr_sg, *sgs[3];
	104	struct virtio_crypto *vcrypto = ctx->vcrypto;
	105	uint8_t *pkey;
	106	unsigned int inlen;
	107	int err;
	108	unsigned int num_out = 0, num_in = 0;
	109
	110	pkey = kmemdup(key, keylen, GFP_ATOMIC);
	111	if (!pkey)
	112	return -ENOMEM;
	113
	114	spin_lock(&vcrypto->ctrl_lock);
	115	memcpy(&vcrypto->ctrl.header, header, sizeof(vcrypto->ctrl.header));
	116	memcpy(&vcrypto->ctrl.u, para, sizeof(vcrypto->ctrl.u));
	117	vcrypto->input.status = cpu_to_le32(VIRTIO_CRYPTO_ERR);
	118
	119	sg_init_one(&outhdr_sg, &vcrypto->ctrl, sizeof(vcrypto->ctrl));
	120	sgs[num_out++] = &outhdr_sg;
	121
	122	sg_init_one(&key_sg, pkey, keylen);
	123	sgs[num_out++] = &key_sg;
	124
	125	sg_init_one(&inhdr_sg, &vcrypto->input, sizeof(vcrypto->input));
	126	sgs[num_out + num_in++] = &inhdr_sg;
	127
	128	err = virtqueue_add_sgs(vcrypto->ctrl_vq, sgs, num_out, num_in, vcrypto, GFP_ATOMIC);
	129	if (err < 0)
	130	goto out;
	131
	132	virtqueue_kick(vcrypto->ctrl_vq);
	133	while (!virtqueue_get_buf(vcrypto->ctrl_vq, &inlen) &&
	134	!virtqueue_is_broken(vcrypto->ctrl_vq))
	135	cpu_relax();
	136
	137	if (le32_to_cpu(vcrypto->input.status) != VIRTIO_CRYPTO_OK) {
	138	err = -EINVAL;
	139	goto out;
	140	}
	141
	142	ctx->session_id = le64_to_cpu(vcrypto->input.session_id);
	143	ctx->session_valid = true;
	144	err = 0;
	145
	146	out:
	147	spin_unlock(&vcrypto->ctrl_lock);
	148	kfree_sensitive(pkey);
	149
	150	if (err < 0)
	151	pr_err("virtio_crypto: Create session failed status: %u\n",
	152	le32_to_cpu(vcrypto->input.status));
	153
	154	return err;
	155	}
	156
	157	static int virtio_crypto_alg_akcipher_close_session(struct virtio_crypto_akcipher_ctx *ctx)
	158	{
	159	struct scatterlist outhdr_sg, inhdr_sg, *sgs[2];
	160	struct virtio_crypto_destroy_session_req *destroy_session;
	161	struct virtio_crypto *vcrypto = ctx->vcrypto;
	162	unsigned int num_out = 0, num_in = 0, inlen;
	163	int err;
	164
	165	spin_lock(&vcrypto->ctrl_lock);
	166	if (!ctx->session_valid) {
	167	err = 0;
	168	goto out;
	169	}
	170	vcrypto->ctrl_status.status = VIRTIO_CRYPTO_ERR;
	171	vcrypto->ctrl.header.opcode = cpu_to_le32(VIRTIO_CRYPTO_AKCIPHER_DESTROY_SESSION);
	172	vcrypto->ctrl.header.queue_id = 0;
	173
	174	destroy_session = &vcrypto->ctrl.u.destroy_session;
	175	destroy_session->session_id = cpu_to_le64(ctx->session_id);
	176
	177	sg_init_one(&outhdr_sg, &vcrypto->ctrl, sizeof(vcrypto->ctrl));
	178	sgs[num_out++] = &outhdr_sg;
	179
	180	sg_init_one(&inhdr_sg, &vcrypto->ctrl_status.status, sizeof(vcrypto->ctrl_status.status));
	181	sgs[num_out + num_in++] = &inhdr_sg;
	182
	183	err = virtqueue_add_sgs(vcrypto->ctrl_vq, sgs, num_out, num_in, vcrypto, GFP_ATOMIC);
	184	if (err < 0)
	185	goto out;
	186
	187	virtqueue_kick(vcrypto->ctrl_vq);
	188	while (!virtqueue_get_buf(vcrypto->ctrl_vq, &inlen) &&
	189	!virtqueue_is_broken(vcrypto->ctrl_vq))
	190	cpu_relax();
	191
	192	if (vcrypto->ctrl_status.status != VIRTIO_CRYPTO_OK) {
	193	err = -EINVAL;
	194	goto out;
	195	}
	196
	197	err = 0;
	198	ctx->session_valid = false;
	199
	200	out:
	201	spin_unlock(&vcrypto->ctrl_lock);
	202	if (err < 0) {
	203	pr_err("virtio_crypto: Close session failed status: %u, session_id: 0x%llx\n",
	204	vcrypto->ctrl_status.status, destroy_session->session_id);
	205	}
	206
	207	return err;
	208	}
	209
	210	static int __virtio_crypto_akcipher_do_req(struct virtio_crypto_akcipher_request *vc_akcipher_req,
	211	struct akcipher_request req, struct data_queue data_vq)
	212	{
	213	struct virtio_crypto_akcipher_ctx *ctx = vc_akcipher_req->akcipher_ctx;
	214	struct virtio_crypto_request *vc_req = &vc_akcipher_req->base;
	215	struct virtio_crypto *vcrypto = ctx->vcrypto;
	216	struct virtio_crypto_op_data_req *req_data = vc_req->req_data;
	217	struct scatterlist *sgs[4], outhdr_sg, inhdr_sg, srcdata_sg, dstdata_sg;
	218	void src_buf = NULL, dst_buf = NULL;
	219	unsigned int num_out = 0, num_in = 0;
	220	int node = dev_to_node(&vcrypto->vdev->dev);
	221	unsigned long flags;
	222	int ret = -ENOMEM;
	223	bool verify = vc_akcipher_req->opcode == VIRTIO_CRYPTO_AKCIPHER_VERIFY;
	224	unsigned int src_len = verify ? req->src_len + req->dst_len : req->src_len;
	225
	226	/* out header */
	227	sg_init_one(&outhdr_sg, req_data, sizeof(*req_data));
	228	sgs[num_out++] = &outhdr_sg;
	229
	230	/* src data */
	231	src_buf = kcalloc_node(src_len, 1, GFP_KERNEL, node);
	232	if (!src_buf)
	233	goto err;
	234
	235	if (verify) {
	236	/* for verify operation, both src and dst data work as OUT direction */
	237	sg_copy_to_buffer(req->src, sg_nents(req->src), src_buf, src_len);
	238	sg_init_one(&srcdata_sg, src_buf, src_len);
	239	sgs[num_out++] = &srcdata_sg;
	240	} else {
	241	sg_copy_to_buffer(req->src, sg_nents(req->src), src_buf, src_len);
	242	sg_init_one(&srcdata_sg, src_buf, src_len);
	243	sgs[num_out++] = &srcdata_sg;
	244
	245	/* dst data */
	246	dst_buf = kcalloc_node(req->dst_len, 1, GFP_KERNEL, node);
	247	if (!dst_buf)
	248	goto err;
	249
	250	sg_init_one(&dstdata_sg, dst_buf, req->dst_len);
	251	sgs[num_out + num_in++] = &dstdata_sg;
	252	}
	253
	254	vc_akcipher_req->src_buf = src_buf;
	255	vc_akcipher_req->dst_buf = dst_buf;
	256
	257	/* in header */
	258	sg_init_one(&inhdr_sg, &vc_req->status, sizeof(vc_req->status));
	259	sgs[num_out + num_in++] = &inhdr_sg;
	260
	261	spin_lock_irqsave(&data_vq->lock, flags);
	262	ret = virtqueue_add_sgs(data_vq->vq, sgs, num_out, num_in, vc_req, GFP_ATOMIC);
	263	virtqueue_kick(data_vq->vq);
	264	spin_unlock_irqrestore(&data_vq->lock, flags);
	265	if (ret)
	266	goto err;
	267
	268	return 0;
	269
	270	err:
	271	kfree(src_buf);
	272	kfree(dst_buf);
	273
	274	return -ENOMEM;
	275	}
	276
	277	static int virtio_crypto_rsa_do_req(struct crypto_engine engine, void vreq)
	278	{
	279	struct akcipher_request *req = container_of(vreq, struct akcipher_request, base);
	280	struct virtio_crypto_akcipher_request *vc_akcipher_req = akcipher_request_ctx(req);
	281	struct virtio_crypto_request *vc_req = &vc_akcipher_req->base;
	282	struct virtio_crypto_akcipher_ctx *ctx = vc_akcipher_req->akcipher_ctx;
	283	struct virtio_crypto *vcrypto = ctx->vcrypto;
	284	struct data_queue *data_vq = vc_req->dataq;
	285	struct virtio_crypto_op_header *header;
	286	struct virtio_crypto_akcipher_data_req *akcipher_req;
	287	int ret;
	288
	289	vc_req->sgs = NULL;
	290	vc_req->req_data = kzalloc_node(sizeof(*vc_req->req_data),
	291	GFP_KERNEL, dev_to_node(&vcrypto->vdev->dev));
	292	if (!vc_req->req_data)
	293	return -ENOMEM;
	294
	295	/* build request header */
	296	header = &vc_req->req_data->header;
	297	header->opcode = cpu_to_le32(vc_akcipher_req->opcode);
	298	header->algo = cpu_to_le32(VIRTIO_CRYPTO_AKCIPHER_RSA);
	299	header->session_id = cpu_to_le64(ctx->session_id);
	300
	301	/* build request akcipher data */
	302	akcipher_req = &vc_req->req_data->u.akcipher_req;
	303	akcipher_req->para.src_data_len = cpu_to_le32(req->src_len);
	304	akcipher_req->para.dst_data_len = cpu_to_le32(req->dst_len);
	305
	306	ret = __virtio_crypto_akcipher_do_req(vc_akcipher_req, req, data_vq);
	307	if (ret < 0) {
	308	kfree_sensitive(vc_req->req_data);
	309	vc_req->req_data = NULL;
	310	return ret;
	311	}
	312
	313	return 0;
	314	}
	315
	316	static int virtio_crypto_rsa_req(struct akcipher_request *req, uint32_t opcode)
	317	{
	318	struct crypto_akcipher *atfm = crypto_akcipher_reqtfm(req);
	319	struct virtio_crypto_akcipher_ctx *ctx = akcipher_tfm_ctx(atfm);
	320	struct virtio_crypto_akcipher_request *vc_akcipher_req = akcipher_request_ctx(req);
	321	struct virtio_crypto_request *vc_req = &vc_akcipher_req->base;
	322	struct virtio_crypto *vcrypto = ctx->vcrypto;
	323	/* Use the first data virtqueue as default */
	324	struct data_queue *data_vq = &vcrypto->data_vq[0];
	325
	326	vc_req->dataq = data_vq;
	327	vc_req->alg_cb = virtio_crypto_dataq_akcipher_callback;
	328	vc_akcipher_req->akcipher_ctx = ctx;
	329	vc_akcipher_req->akcipher_req = req;
	330	vc_akcipher_req->opcode = opcode;
	331
	332	return crypto_transfer_akcipher_request_to_engine(data_vq->engine, req);
	333	}
	334
	335	static int virtio_crypto_rsa_encrypt(struct akcipher_request *req)
	336	{
	337	return virtio_crypto_rsa_req(req, VIRTIO_CRYPTO_AKCIPHER_ENCRYPT);
	338	}
	339
	340	static int virtio_crypto_rsa_decrypt(struct akcipher_request *req)
	341	{
	342	return virtio_crypto_rsa_req(req, VIRTIO_CRYPTO_AKCIPHER_DECRYPT);
	343	}
	344
	345	static int virtio_crypto_rsa_sign(struct akcipher_request *req)
	346	{
	347	return virtio_crypto_rsa_req(req, VIRTIO_CRYPTO_AKCIPHER_SIGN);
	348	}
	349
	350	static int virtio_crypto_rsa_verify(struct akcipher_request *req)
	351	{
	352	return virtio_crypto_rsa_req(req, VIRTIO_CRYPTO_AKCIPHER_VERIFY);
	353	}
	354
	355	static int virtio_crypto_rsa_set_key(struct crypto_akcipher *tfm,
	356	const void *key,
	357	unsigned int keylen,
	358	bool private,
	359	int padding_algo,
	360	int hash_algo)
	361	{
	362	struct virtio_crypto_akcipher_ctx *ctx = akcipher_tfm_ctx(tfm);
	363	struct virtio_crypto_rsa_ctx *rsa_ctx = &ctx->rsa_ctx;
	364	struct virtio_crypto *vcrypto;
	365	struct virtio_crypto_ctrl_header header;
	366	struct virtio_crypto_akcipher_session_para para;
	367	struct rsa_key rsa_key = {0};
	368	int node = virtio_crypto_get_current_node();
	369	uint32_t keytype;
	370	int ret;
	371
	372	/* mpi_free will test n, just free it. */
	373	mpi_free(rsa_ctx->n);
	374	rsa_ctx->n = NULL;
	375
	376	if (private) {
	377	keytype = VIRTIO_CRYPTO_AKCIPHER_KEY_TYPE_PRIVATE;
	378	ret = rsa_parse_priv_key(&rsa_key, key, keylen);
	379	} else {
	380	keytype = VIRTIO_CRYPTO_AKCIPHER_KEY_TYPE_PUBLIC;
	381	ret = rsa_parse_pub_key(&rsa_key, key, keylen);
	382	}
	383
	384	if (ret)
	385	return ret;
	386
	387	rsa_ctx->n = mpi_read_raw_data(rsa_key.n, rsa_key.n_sz);
	388	if (!rsa_ctx->n)
	389	return -ENOMEM;
	390
	391	if (!ctx->vcrypto) {
	392	vcrypto = virtcrypto_get_dev_node(node, VIRTIO_CRYPTO_SERVICE_AKCIPHER,
	393	VIRTIO_CRYPTO_AKCIPHER_RSA);
	394	if (!vcrypto) {
	395	pr_err("virtio_crypto: Could not find a virtio device in the system or unsupported algo\n");
	396	return -ENODEV;
	397	}
	398
	399	ctx->vcrypto = vcrypto;
	400	} else {
	401	virtio_crypto_alg_akcipher_close_session(ctx);
	402	}
	403
	404	/* set ctrl header */
	405	header.opcode = cpu_to_le32(VIRTIO_CRYPTO_AKCIPHER_CREATE_SESSION);
	406	header.algo = cpu_to_le32(VIRTIO_CRYPTO_AKCIPHER_RSA);
	407	header.queue_id = 0;
	408
	409	/* set RSA para */
	410	para.algo = cpu_to_le32(VIRTIO_CRYPTO_AKCIPHER_RSA);
	411	para.keytype = cpu_to_le32(keytype);
	412	para.keylen = cpu_to_le32(keylen);
	413	para.u.rsa.padding_algo = cpu_to_le32(padding_algo);
	414	para.u.rsa.hash_algo = cpu_to_le32(hash_algo);
	415
	416	return virtio_crypto_alg_akcipher_init_session(ctx, &header, &para, key, keylen);
	417	}
	418
	419	static int virtio_crypto_rsa_raw_set_priv_key(struct crypto_akcipher *tfm,
	420	const void *key,
	421	unsigned int keylen)
	422	{
	423	return virtio_crypto_rsa_set_key(tfm, key, keylen, 1,
	424	VIRTIO_CRYPTO_RSA_RAW_PADDING,
	425	VIRTIO_CRYPTO_RSA_NO_HASH);
	426	}
	427
	428
	429	static int virtio_crypto_p1pad_rsa_sha1_set_priv_key(struct crypto_akcipher *tfm,
	430	const void *key,
	431	unsigned int keylen)
	432	{
	433	return virtio_crypto_rsa_set_key(tfm, key, keylen, 1,
	434	VIRTIO_CRYPTO_RSA_PKCS1_PADDING,
	435	VIRTIO_CRYPTO_RSA_SHA1);
	436	}
	437
	438	static int virtio_crypto_rsa_raw_set_pub_key(struct crypto_akcipher *tfm,
	439	const void *key,
	440	unsigned int keylen)
	441	{
	442	return virtio_crypto_rsa_set_key(tfm, key, keylen, 0,
	443	VIRTIO_CRYPTO_RSA_RAW_PADDING,
	444	VIRTIO_CRYPTO_RSA_NO_HASH);
	445	}
	446
	447	static int virtio_crypto_p1pad_rsa_sha1_set_pub_key(struct crypto_akcipher *tfm,
	448	const void *key,
	449	unsigned int keylen)
	450	{
	451	return virtio_crypto_rsa_set_key(tfm, key, keylen, 0,
	452	VIRTIO_CRYPTO_RSA_PKCS1_PADDING,
	453	VIRTIO_CRYPTO_RSA_SHA1);
	454	}
	455
	456	static unsigned int virtio_crypto_rsa_max_size(struct crypto_akcipher *tfm)
	457	{
	458	struct virtio_crypto_akcipher_ctx *ctx = akcipher_tfm_ctx(tfm);
	459	struct virtio_crypto_rsa_ctx *rsa_ctx = &ctx->rsa_ctx;
	460
	461	return mpi_get_size(rsa_ctx->n);
	462	}
	463
	464	static int virtio_crypto_rsa_init_tfm(struct crypto_akcipher *tfm)
	465	{
	466	struct virtio_crypto_akcipher_ctx *ctx = akcipher_tfm_ctx(tfm);
	467
	468	ctx->tfm = tfm;
	469	ctx->enginectx.op.do_one_request = virtio_crypto_rsa_do_req;
	470	ctx->enginectx.op.prepare_request = NULL;
	471	ctx->enginectx.op.unprepare_request = NULL;
	472
	473	return 0;
	474	}
	475
	476	static void virtio_crypto_rsa_exit_tfm(struct crypto_akcipher *tfm)
	477	{
	478	struct virtio_crypto_akcipher_ctx *ctx = akcipher_tfm_ctx(tfm);
	479	struct virtio_crypto_rsa_ctx *rsa_ctx = &ctx->rsa_ctx;
	480
	481	virtio_crypto_alg_akcipher_close_session(ctx);
	482	virtcrypto_dev_put(ctx->vcrypto);
	483	mpi_free(rsa_ctx->n);
	484	rsa_ctx->n = NULL;
	485	}
	486
	487	static struct virtio_crypto_akcipher_algo virtio_crypto_akcipher_algs[] = {
	488	{
	489	.algonum = VIRTIO_CRYPTO_AKCIPHER_RSA,
	490	.service = VIRTIO_CRYPTO_SERVICE_AKCIPHER,
	491	.algo = {
	492	.encrypt = virtio_crypto_rsa_encrypt,
	493	.decrypt = virtio_crypto_rsa_decrypt,
	494	.set_pub_key = virtio_crypto_rsa_raw_set_pub_key,
	495	.set_priv_key = virtio_crypto_rsa_raw_set_priv_key,
	496	.max_size = virtio_crypto_rsa_max_size,
	497	.init = virtio_crypto_rsa_init_tfm,
	498	.exit = virtio_crypto_rsa_exit_tfm,
	499	.reqsize = sizeof(struct virtio_crypto_akcipher_request),
	500	.base = {
	501	.cra_name = "rsa",
	502	.cra_driver_name = "virtio-crypto-rsa",
	503	.cra_priority = 150,
	504	.cra_module = THIS_MODULE,
	505	.cra_ctxsize = sizeof(struct virtio_crypto_akcipher_ctx),
	506	},
	507	},
	508	},
	509	{
	510	.algonum = VIRTIO_CRYPTO_AKCIPHER_RSA,
	511	.service = VIRTIO_CRYPTO_SERVICE_AKCIPHER,
	512	.algo = {
	513	.encrypt = virtio_crypto_rsa_encrypt,
	514	.decrypt = virtio_crypto_rsa_decrypt,
	515	.sign = virtio_crypto_rsa_sign,
	516	.verify = virtio_crypto_rsa_verify,
	517	.set_pub_key = virtio_crypto_p1pad_rsa_sha1_set_pub_key,
	518	.set_priv_key = virtio_crypto_p1pad_rsa_sha1_set_priv_key,
	519	.max_size = virtio_crypto_rsa_max_size,
	520	.init = virtio_crypto_rsa_init_tfm,
	521	.exit = virtio_crypto_rsa_exit_tfm,
	522	.reqsize = sizeof(struct virtio_crypto_akcipher_request),
	523	.base = {
	524	.cra_name = "pkcs1pad(rsa,sha1)",
	525	.cra_driver_name = "virtio-pkcs1-rsa-with-sha1",
	526	.cra_priority = 150,
	527	.cra_module = THIS_MODULE,
	528	.cra_ctxsize = sizeof(struct virtio_crypto_akcipher_ctx),
	529	},
	530	},
	531	},
	532	};
	533
	534	int virtio_crypto_akcipher_algs_register(struct virtio_crypto *vcrypto)
	535	{
	536	int ret = 0;
	537	int i = 0;
	538
	539	mutex_lock(&algs_lock);
	540
	541	for (i = 0; i < ARRAY_SIZE(virtio_crypto_akcipher_algs); i++) {
	542	uint32_t service = virtio_crypto_akcipher_algs[i].service;
	543	uint32_t algonum = virtio_crypto_akcipher_algs[i].algonum;
	544
	545	if (!virtcrypto_algo_is_supported(vcrypto, service, algonum))
	546	continue;
	547
	548	if (virtio_crypto_akcipher_algs[i].active_devs == 0) {
	549	ret = crypto_register_akcipher(&virtio_crypto_akcipher_algs[i].algo);
	550	if (ret)
	551	goto unlock;
	552	}
	553
	554	virtio_crypto_akcipher_algs[i].active_devs++;
	555	dev_info(&vcrypto->vdev->dev, "Registered akcipher algo %s\n",
	556	virtio_crypto_akcipher_algs[i].algo.base.cra_name);
	557	}
	558
	559	unlock:
	560	mutex_unlock(&algs_lock);
	561	return ret;
	562	}
	563
	564	void virtio_crypto_akcipher_algs_unregister(struct virtio_crypto *vcrypto)
	565	{
	566	int i = 0;
	567
	568	mutex_lock(&algs_lock);
	569
	570	for (i = 0; i < ARRAY_SIZE(virtio_crypto_akcipher_algs); i++) {
	571	uint32_t service = virtio_crypto_akcipher_algs[i].service;
	572	uint32_t algonum = virtio_crypto_akcipher_algs[i].algonum;
	573
	574	if (virtio_crypto_akcipher_algs[i].active_devs == 0 \|\|
	575	!virtcrypto_algo_is_supported(vcrypto, service, algonum))
	576	continue;
	577
	578	if (virtio_crypto_akcipher_algs[i].active_devs == 1)
	579	crypto_unregister_akcipher(&virtio_crypto_akcipher_algs[i].algo);
	580
	581	virtio_crypto_akcipher_algs[i].active_devs--;
	582	}
	583
	584	mutex_unlock(&algs_lock);
	585	}

drivers/crypto/virtio/virtio_crypto_common.h+5−2

 	u32 mac_algo_l;
 	u32 mac_algo_h;
 	u32 aead_algo;
+	u32 akcipher_algo;
 	/* Maximum length of cipher key */
 	u32 max_cipher_key_len;
 	return node;
 }
-int virtio_crypto_algs_register(struct virtio_crypto *vcrypto);
+int virtio_crypto_skcipher_algs_register(struct virtio_crypto *vcrypto);
-void virtio_crypto_algs_unregister(struct virtio_crypto *vcrypto);
+void virtio_crypto_skcipher_algs_unregister(struct virtio_crypto *vcrypto);
+int virtio_crypto_akcipher_algs_register(struct virtio_crypto *vcrypto);
+void virtio_crypto_akcipher_algs_unregister(struct virtio_crypto *vcrypto);
 #endif /* _VIRTIO_CRYPTO_COMMON_H */

drivers/crypto/virtio/virtio_crypto_core.c+5−1

@@ -297,6 +297,7 @@ static int virtcrypto_probe(struct virtio_device *vdev) · 296 unmodified lines
297	u32 mac_algo_l = 0;	297	u32 mac_algo_l = 0;
298	u32 mac_algo_h = 0;	298	u32 mac_algo_h = 0;
299	u32 aead_algo = 0;	299	u32 aead_algo = 0;
		300	u32 akcipher_algo = 0;
300	u32 crypto_services = 0;	301	u32 crypto_services = 0;
301		302
302	if (!virtio_has_feature(vdev, VIRTIO_F_VERSION_1))	303	if (!virtio_has_feature(vdev, VIRTIO_F_VERSION_1))
@@ -348,6 +349,9 @@ static int virtcrypto_probe(struct virtio_device *vdev) · 45 unmodified lines
348	mac_algo_h, &mac_algo_h);	349	mac_algo_h, &mac_algo_h);
349	virtio_cread_le(vdev, struct virtio_crypto_config,	350	virtio_cread_le(vdev, struct virtio_crypto_config,
350	aead_algo, &aead_algo);	351	aead_algo, &aead_algo);
		352	if (crypto_services & (1 << VIRTIO_CRYPTO_SERVICE_AKCIPHER))
		353	virtio_cread_le(vdev, struct virtio_crypto_config,
		354	akcipher_algo, &akcipher_algo);
351		355
352	/* Add virtio crypto device to global table */	356	/* Add virtio crypto device to global table */
353	err = virtcrypto_devmgr_add_dev(vcrypto);	357	err = virtcrypto_devmgr_add_dev(vcrypto);
@@ -374,7 +378,7 @@ static int virtcrypto_probe(struct virtio_device *vdev) · 20 unmodified lines
374	vcrypto->mac_algo_h = mac_algo_h;	378	vcrypto->mac_algo_h = mac_algo_h;
375	vcrypto->hash_algo = hash_algo;	379	vcrypto->hash_algo = hash_algo;
376	vcrypto->aead_algo = aead_algo;	380	vcrypto->aead_algo = aead_algo;
377		381	vcrypto->akcipher_algo = akcipher_algo;
378		382
379	dev_info(&vdev->dev,	383	dev_info(&vdev->dev,
380	"max_queues: %u, max_cipher_key_len: %u, max_auth_key_len: %u, max_size 0x%llx\n",	384	"max_queues: %u, max_cipher_key_len: %u, max_auth_key_len: %u, max_size 0x%llx\n",

drivers/crypto/virtio/virtio_crypto_mgr.c+14−3

  */
 int virtcrypto_dev_start(struct virtio_crypto *vcrypto)
 {
-	if (virtio_crypto_algs_register(vcrypto)) {
+	if (virtio_crypto_skcipher_algs_register(vcrypto)) {
-		pr_err("virtio_crypto: Failed to register crypto algs\n");
+		pr_err("virtio_crypto: Failed to register crypto skcipher algs\n");
+		return -EFAULT;
+	}
+	if (virtio_crypto_akcipher_algs_register(vcrypto)) {
+		pr_err("virtio_crypto: Failed to register crypto akcipher algs\n");
+		virtio_crypto_skcipher_algs_unregister(vcrypto);
 		return -EFAULT;
 	}
  */
 void virtcrypto_dev_stop(struct virtio_crypto *vcrypto)
 {
-	virtio_crypto_algs_unregister(vcrypto);
+	virtio_crypto_skcipher_algs_unregister(vcrypto);
+	virtio_crypto_akcipher_algs_unregister(vcrypto);
 }
 /*
 	case VIRTIO_CRYPTO_SERVICE_AEAD:
 		algo_mask = vcrypto->aead_algo;
 		break;
+	case VIRTIO_CRYPTO_SERVICE_AKCIPHER:
+		algo_mask = vcrypto->akcipher_algo;
+		break;
 	}
 	if (!(algo_mask & (1u << algo)))

drivers/crypto/virtio/virtio_crypto_skcipher_algs.crenamed from drivers/crypto/virtio/virtio_crypto_algs.c+2−2

@@ -613,7 +613,7 @@ static struct virtio_crypto_algo virtio_crypto_algs[] = { { · 612 unmodified lines
613	},	613	},
614	} };	614	} };
615		615
616	int virtio_crypto_algs_register(struct virtio_crypto *vcrypto)	616	int virtio_crypto_skcipher_algs_register(struct virtio_crypto *vcrypto)
617	{	617	{
618	int ret = 0;	618	int ret = 0;
619	int i = 0;	619	int i = 0;
@@ -644,7 +644,7 @@ unlock: · 24 unmodified lines
644	return ret;	644	return ret;
645	}	645	}
646		646
647	void virtio_crypto_algs_unregister(struct virtio_crypto *vcrypto)	647	void virtio_crypto_skcipher_algs_unregister(struct virtio_crypto *vcrypto)
648	{	648	{
649	int i = 0;	649	int i = 0;
650		650

drivers/net/virtio_net.c+376−13

@@ -169,6 +169,24 @@ struct receive_queue { · 168 unmodified lines
169	struct xdp_rxq_info xdp_rxq;	169	struct xdp_rxq_info xdp_rxq;
170	};	170	};
171		171
		172	/* This structure can contain rss message with maximum settings for indirection table and keysize
		173	* Note, that default structure that describes RSS configuration virtio_net_rss_config
		174	* contains same info but can't handle table values.
		175	* In any case, structure would be passed to virtio hw through sg_buf split by parts
		176	* because table sizes may be differ according to the device configuration.
		177	*/
		178	#define VIRTIO_NET_RSS_MAX_KEY_SIZE 40
		179	#define VIRTIO_NET_RSS_MAX_TABLE_LEN 128
		180	struct virtio_net_ctrl_rss {
		181	u32 hash_types;
		182	u16 indirection_table_mask;
		183	u16 unclassified_queue;
		184	u16 indirection_table[VIRTIO_NET_RSS_MAX_TABLE_LEN];
		185	u16 max_tx_vq;
		186	u8 hash_key_length;
		187	u8 key[VIRTIO_NET_RSS_MAX_KEY_SIZE];
		188	};
		189
172	/* Control VQ buffers: protected by the rtnl lock */	190	/* Control VQ buffers: protected by the rtnl lock */
173	struct control_buf {	191	struct control_buf {
174	struct virtio_net_ctrl_hdr hdr;	192	struct virtio_net_ctrl_hdr hdr;
@@ -178,6 +196,7 @@ struct control_buf { · 3 unmodified lines
178	u8 allmulti;	196	u8 allmulti;
179	__virtio16 vid;	197	__virtio16 vid;
180	__virtio64 offloads;	198	__virtio64 offloads;
		199	struct virtio_net_ctrl_rss rss;
181	};	200	};
182		201
183	struct virtnet_info {	202	struct virtnet_info {
@@ -206,6 +225,14 @@ struct virtnet_info { · 22 unmodified lines
206	/* Host will merge rx buffers for big packets (shake it! shake it!) */	225	/* Host will merge rx buffers for big packets (shake it! shake it!) */
207	bool mergeable_rx_bufs;	226	bool mergeable_rx_bufs;
208		227
		228	/* Host supports rss and/or hash report */
		229	bool has_rss;
		230	bool has_rss_hash_report;
		231	u8 rss_key_size;
		232	u16 rss_indir_table_size;
		233	u32 rss_hash_types_supported;
		234	u32 rss_hash_types_saved;
		235
209	/* Has control virtqueue */	236	/* Has control virtqueue */
210	bool has_cvq;	237	bool has_cvq;
211		238
@@ -242,13 +269,13 @@ struct virtnet_info { · 30 unmodified lines
242	};	269	};
243		270
244	struct padded_vnet_hdr {	271	struct padded_vnet_hdr {
245	struct virtio_net_hdr_mrg_rxbuf hdr;	272	struct virtio_net_hdr_v1_hash hdr;
246	/*	273	/*
247	* hdr is in a separate sg buffer, and data sg buffer shares same page	274	* hdr is in a separate sg buffer, and data sg buffer shares same page
248	* with this header sg. This padding makes next sg 16 byte aligned	275	* with this header sg. This padding makes next sg 16 byte aligned
249	* after the header.	276	* after the header.
250	*/	277	*/
251	char padding[4];	278	char padding[12];
252	};	279	};
253		280
254	static bool is_xdp_frame(void *ptr)	281	static bool is_xdp_frame(void *ptr)
@@ -396,7 +423,7 @@ static struct sk_buff page_to_skb(struct virtnet_info vi, · 141 unmodified lines
396		423
397	hdr_len = vi->hdr_len;	424	hdr_len = vi->hdr_len;
398	if (vi->mergeable_rx_bufs)	425	if (vi->mergeable_rx_bufs)
399	hdr_padded_len = sizeof(*hdr);	426	hdr_padded_len = hdr_len;
400	else	427	else
401	hdr_padded_len = sizeof(struct padded_vnet_hdr);	428	hdr_padded_len = sizeof(struct padded_vnet_hdr);
402		429
@@ -1123,6 +1150,35 @@ xdp_xmit: · 720 unmodified lines
1123	return NULL;	1150	return NULL;
1124	}	1151	}
1125		1152
		1153	static void virtio_skb_set_hash(const struct virtio_net_hdr_v1_hash *hdr_hash,
		1154	struct sk_buff *skb)
		1155	{
		1156	enum pkt_hash_types rss_hash_type;
		1157
		1158	if (!hdr_hash \|\| !skb)
		1159	return;
		1160
		1161	switch ((int)hdr_hash->hash_report) {
		1162	case VIRTIO_NET_HASH_REPORT_TCPv4:
		1163	case VIRTIO_NET_HASH_REPORT_UDPv4:
		1164	case VIRTIO_NET_HASH_REPORT_TCPv6:
		1165	case VIRTIO_NET_HASH_REPORT_UDPv6:
		1166	case VIRTIO_NET_HASH_REPORT_TCPv6_EX:
		1167	case VIRTIO_NET_HASH_REPORT_UDPv6_EX:
		1168	rss_hash_type = PKT_HASH_TYPE_L4;
		1169	break;
		1170	case VIRTIO_NET_HASH_REPORT_IPv4:
		1171	case VIRTIO_NET_HASH_REPORT_IPv6:
		1172	case VIRTIO_NET_HASH_REPORT_IPv6_EX:
		1173	rss_hash_type = PKT_HASH_TYPE_L3;
		1174	break;
		1175	case VIRTIO_NET_HASH_REPORT_NONE:
		1176	default:
		1177	rss_hash_type = PKT_HASH_TYPE_NONE;
		1178	}
		1179	skb_set_hash(skb, (unsigned int)hdr_hash->hash_value, rss_hash_type);
		1180	}
		1181
1126	static void receive_buf(struct virtnet_info vi, struct receive_queue rq,	1182	static void receive_buf(struct virtnet_info vi, struct receive_queue rq,
1127	void buf, unsigned int len, void *ctx,	1183	void buf, unsigned int len, void *ctx,
1128	unsigned int *xdp_xmit,	1184	unsigned int *xdp_xmit,
@@ -1157,6 +1213,8 @@ static void receive_buf(struct virtnet_info vi, struct receive_queue rq, · 28 unmodified lines
1157	return;	1213	return;
1158		1214
1159	hdr = skb_vnet_hdr(skb);	1215	hdr = skb_vnet_hdr(skb);
		1216	if (dev->features & NETIF_F_RXHASH && vi->has_rss_hash_report)
		1217	virtio_skb_set_hash((const struct virtio_net_hdr_v1_hash *)hdr, skb);
1160		1218
1161	if (hdr->hdr.flags & VIRTIO_NET_HDR_F_DATA_VALID)	1219	if (hdr->hdr.flags & VIRTIO_NET_HDR_F_DATA_VALID)
1162	skb->ip_summed = CHECKSUM_UNNECESSARY;	1220	skb->ip_summed = CHECKSUM_UNNECESSARY;
@@ -1266,7 +1324,8 @@ static unsigned int get_mergeable_buf_len(struct receive_queue *rq, · 103 unmodified lines
1266	struct ewma_pkt_len *avg_pkt_len,	1324	struct ewma_pkt_len *avg_pkt_len,
1267	unsigned int room)	1325	unsigned int room)
1268	{	1326	{
1269	const size_t hdr_len = sizeof(struct virtio_net_hdr_mrg_rxbuf);	1327	struct virtnet_info *vi = rq->vq->vdev->priv;
		1328	const size_t hdr_len = vi->hdr_len;
1270	unsigned int len;	1329	unsigned int len;
1271		1330
1272	if (room)	1331	if (room)
@@ -2183,6 +2242,174 @@ static void virtnet_get_ringparam(struct net_device *dev, · 910 unmodified lines
2183	ring->tx_pending = ring->tx_max_pending;	2242	ring->tx_pending = ring->tx_max_pending;
2184	}	2243	}
2185		2244
		2245	static bool virtnet_commit_rss_command(struct virtnet_info *vi)
		2246	{
		2247	struct net_device *dev = vi->dev;
		2248	struct scatterlist sgs[4];
		2249	unsigned int sg_buf_size;
		2250
		2251	/* prepare sgs */
		2252	sg_init_table(sgs, 4);
		2253
		2254	sg_buf_size = offsetof(struct virtio_net_ctrl_rss, indirection_table);
		2255	sg_set_buf(&sgs[0], &vi->ctrl->rss, sg_buf_size);
		2256
		2257	sg_buf_size = sizeof(uint16_t) * (vi->ctrl->rss.indirection_table_mask + 1);
		2258	sg_set_buf(&sgs[1], vi->ctrl->rss.indirection_table, sg_buf_size);
		2259
		2260	sg_buf_size = offsetof(struct virtio_net_ctrl_rss, key)
		2261	- offsetof(struct virtio_net_ctrl_rss, max_tx_vq);
		2262	sg_set_buf(&sgs[2], &vi->ctrl->rss.max_tx_vq, sg_buf_size);
		2263
		2264	sg_buf_size = vi->rss_key_size;
		2265	sg_set_buf(&sgs[3], vi->ctrl->rss.key, sg_buf_size);
		2266
		2267	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MQ,
		2268	vi->has_rss ? VIRTIO_NET_CTRL_MQ_RSS_CONFIG
		2269	: VIRTIO_NET_CTRL_MQ_HASH_CONFIG, sgs)) {
		2270	dev_warn(&dev->dev, "VIRTIONET issue with committing RSS sgs\n");
		2271	return false;
		2272	}
		2273	return true;
		2274	}
		2275
		2276	static void virtnet_init_default_rss(struct virtnet_info *vi)
		2277	{
		2278	u32 indir_val = 0;
		2279	int i = 0;
		2280
		2281	vi->ctrl->rss.hash_types = vi->rss_hash_types_supported;
		2282	vi->rss_hash_types_saved = vi->rss_hash_types_supported;
		2283	vi->ctrl->rss.indirection_table_mask = vi->rss_indir_table_size
		2284	? vi->rss_indir_table_size - 1 : 0;
		2285	vi->ctrl->rss.unclassified_queue = 0;
		2286
		2287	for (; i < vi->rss_indir_table_size; ++i) {
		2288	indir_val = ethtool_rxfh_indir_default(i, vi->curr_queue_pairs);
		2289	vi->ctrl->rss.indirection_table[i] = indir_val;
		2290	}
		2291
		2292	vi->ctrl->rss.max_tx_vq = vi->curr_queue_pairs;
		2293	vi->ctrl->rss.hash_key_length = vi->rss_key_size;
		2294
		2295	netdev_rss_key_fill(vi->ctrl->rss.key, vi->rss_key_size);
		2296	}
		2297
		2298	static void virtnet_get_hashflow(const struct virtnet_info vi, struct ethtool_rxnfc info)
		2299	{
		2300	info->data = 0;
		2301	switch (info->flow_type) {
		2302	case TCP_V4_FLOW:
		2303	if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_TCPv4) {
		2304	info->data = RXH_IP_SRC \| RXH_IP_DST \|
		2305	RXH_L4_B_0_1 \| RXH_L4_B_2_3;
		2306	} else if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_IPv4) {
		2307	info->data = RXH_IP_SRC \| RXH_IP_DST;
		2308	}
		2309	break;
		2310	case TCP_V6_FLOW:
		2311	if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_TCPv6) {
		2312	info->data = RXH_IP_SRC \| RXH_IP_DST \|
		2313	RXH_L4_B_0_1 \| RXH_L4_B_2_3;
		2314	} else if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_IPv6) {
		2315	info->data = RXH_IP_SRC \| RXH_IP_DST;
		2316	}
		2317	break;
		2318	case UDP_V4_FLOW:
		2319	if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_UDPv4) {
		2320	info->data = RXH_IP_SRC \| RXH_IP_DST \|
		2321	RXH_L4_B_0_1 \| RXH_L4_B_2_3;
		2322	} else if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_IPv4) {
		2323	info->data = RXH_IP_SRC \| RXH_IP_DST;
		2324	}
		2325	break;
		2326	case UDP_V6_FLOW:
		2327	if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_UDPv6) {
		2328	info->data = RXH_IP_SRC \| RXH_IP_DST \|
		2329	RXH_L4_B_0_1 \| RXH_L4_B_2_3;
		2330	} else if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_IPv6) {
		2331	info->data = RXH_IP_SRC \| RXH_IP_DST;
		2332	}
		2333	break;
		2334	case IPV4_FLOW:
		2335	if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_IPv4)
		2336	info->data = RXH_IP_SRC \| RXH_IP_DST;
		2337
		2338	break;
		2339	case IPV6_FLOW:
		2340	if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_IPv6)
		2341	info->data = RXH_IP_SRC \| RXH_IP_DST;
		2342
		2343	break;
		2344	default:
		2345	info->data = 0;
		2346	break;
		2347	}
		2348	}
		2349
		2350	static bool virtnet_set_hashflow(struct virtnet_info vi, struct ethtool_rxnfc info)
		2351	{
		2352	u32 new_hashtypes = vi->rss_hash_types_saved;
		2353	bool is_disable = info->data & RXH_DISCARD;
		2354	bool is_l4 = info->data == (RXH_IP_SRC \| RXH_IP_DST \| RXH_L4_B_0_1 \| RXH_L4_B_2_3);
		2355
		2356	/* supports only 'sd', 'sdfn' and 'r' */
		2357	if (!((info->data == (RXH_IP_SRC \| RXH_IP_DST)) \| is_l4 \| is_disable))
		2358	return false;
		2359
		2360	switch (info->flow_type) {
		2361	case TCP_V4_FLOW:
		2362	new_hashtypes &= ~(VIRTIO_NET_RSS_HASH_TYPE_IPv4 \| VIRTIO_NET_RSS_HASH_TYPE_TCPv4);
		2363	if (!is_disable)
		2364	new_hashtypes \|= VIRTIO_NET_RSS_HASH_TYPE_IPv4
		2365	\| (is_l4 ? VIRTIO_NET_RSS_HASH_TYPE_TCPv4 : 0);
		2366	break;
		2367	case UDP_V4_FLOW:
		2368	new_hashtypes &= ~(VIRTIO_NET_RSS_HASH_TYPE_IPv4 \| VIRTIO_NET_RSS_HASH_TYPE_UDPv4);
		2369	if (!is_disable)
		2370	new_hashtypes \|= VIRTIO_NET_RSS_HASH_TYPE_IPv4
		2371	\| (is_l4 ? VIRTIO_NET_RSS_HASH_TYPE_UDPv4 : 0);
		2372	break;
		2373	case IPV4_FLOW:
		2374	new_hashtypes &= ~VIRTIO_NET_RSS_HASH_TYPE_IPv4;
		2375	if (!is_disable)
		2376	new_hashtypes = VIRTIO_NET_RSS_HASH_TYPE_IPv4;
		2377	break;
		2378	case TCP_V6_FLOW:
		2379	new_hashtypes &= ~(VIRTIO_NET_RSS_HASH_TYPE_IPv6 \| VIRTIO_NET_RSS_HASH_TYPE_TCPv6);
		2380	if (!is_disable)
		2381	new_hashtypes \|= VIRTIO_NET_RSS_HASH_TYPE_IPv6
		2382	\| (is_l4 ? VIRTIO_NET_RSS_HASH_TYPE_TCPv6 : 0);
		2383	break;
		2384	case UDP_V6_FLOW:
		2385	new_hashtypes &= ~(VIRTIO_NET_RSS_HASH_TYPE_IPv6 \| VIRTIO_NET_RSS_HASH_TYPE_UDPv6);
		2386	if (!is_disable)
		2387	new_hashtypes \|= VIRTIO_NET_RSS_HASH_TYPE_IPv6
		2388	\| (is_l4 ? VIRTIO_NET_RSS_HASH_TYPE_UDPv6 : 0);
		2389	break;
		2390	case IPV6_FLOW:
		2391	new_hashtypes &= ~VIRTIO_NET_RSS_HASH_TYPE_IPv6;
		2392	if (!is_disable)
		2393	new_hashtypes = VIRTIO_NET_RSS_HASH_TYPE_IPv6;
		2394	break;
		2395	default:
		2396	/* unsupported flow */
		2397	return false;
		2398	}
		2399
		2400	/* if unsupported hashtype was set */
		2401	if (new_hashtypes != (new_hashtypes & vi->rss_hash_types_supported))
		2402	return false;
		2403
		2404	if (new_hashtypes != vi->rss_hash_types_saved) {
		2405	vi->rss_hash_types_saved = new_hashtypes;
		2406	vi->ctrl->rss.hash_types = vi->rss_hash_types_saved;
		2407	if (vi->dev->features & NETIF_F_RXHASH)
		2408	return virtnet_commit_rss_command(vi);
		2409	}
		2410
		2411	return true;
		2412	}
2186		2413
2187	static void virtnet_get_drvinfo(struct net_device *dev,	2414	static void virtnet_get_drvinfo(struct net_device *dev,
2188	struct ethtool_drvinfo *info)	2415	struct ethtool_drvinfo *info)
@@ -2411,6 +2638,92 @@ static void virtnet_update_settings(struct virtnet_info *vi) · 222 unmodified lines
2411	vi->duplex = duplex;	2638	vi->duplex = duplex;
2412	}	2639	}
2413		2640
		2641	static u32 virtnet_get_rxfh_key_size(struct net_device *dev)
		2642	{
		2643	return ((struct virtnet_info *)netdev_priv(dev))->rss_key_size;
		2644	}
		2645
		2646	static u32 virtnet_get_rxfh_indir_size(struct net_device *dev)
		2647	{
		2648	return ((struct virtnet_info *)netdev_priv(dev))->rss_indir_table_size;
		2649	}
		2650
		2651	static int virtnet_get_rxfh(struct net_device dev, u32 indir, u8 key, u8 hfunc)
		2652	{
		2653	struct virtnet_info *vi = netdev_priv(dev);
		2654	int i;
		2655
		2656	if (indir) {
		2657	for (i = 0; i < vi->rss_indir_table_size; ++i)
		2658	indir[i] = vi->ctrl->rss.indirection_table[i];
		2659	}
		2660
		2661	if (key)
		2662	memcpy(key, vi->ctrl->rss.key, vi->rss_key_size);
		2663
		2664	if (hfunc)
		2665	*hfunc = ETH_RSS_HASH_TOP;
		2666
		2667	return 0;
		2668	}
		2669
		2670	static int virtnet_set_rxfh(struct net_device dev, const u32 indir, const u8 *key, const u8 hfunc)
		2671	{
		2672	struct virtnet_info *vi = netdev_priv(dev);
		2673	int i;
		2674
		2675	if (hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != ETH_RSS_HASH_TOP)
		2676	return -EOPNOTSUPP;
		2677
		2678	if (indir) {
		2679	for (i = 0; i < vi->rss_indir_table_size; ++i)
		2680	vi->ctrl->rss.indirection_table[i] = indir[i];
		2681	}
		2682	if (key)
		2683	memcpy(vi->ctrl->rss.key, key, vi->rss_key_size);
		2684
		2685	virtnet_commit_rss_command(vi);
		2686
		2687	return 0;
		2688	}
		2689
		2690	static int virtnet_get_rxnfc(struct net_device dev, struct ethtool_rxnfc info, u32 *rule_locs)
		2691	{
		2692	struct virtnet_info *vi = netdev_priv(dev);
		2693	int rc = 0;
		2694
		2695	switch (info->cmd) {
		2696	case ETHTOOL_GRXRINGS:
		2697	info->data = vi->curr_queue_pairs;
		2698	break;
		2699	case ETHTOOL_GRXFH:
		2700	virtnet_get_hashflow(vi, info);
		2701	break;
		2702	default:
		2703	rc = -EOPNOTSUPP;
		2704	}
		2705
		2706	return rc;
		2707	}
		2708
		2709	static int virtnet_set_rxnfc(struct net_device dev, struct ethtool_rxnfc info)
		2710	{
		2711	struct virtnet_info *vi = netdev_priv(dev);
		2712	int rc = 0;
		2713
		2714	switch (info->cmd) {
		2715	case ETHTOOL_SRXFH:
		2716	if (!virtnet_set_hashflow(vi, info))
		2717	rc = -EINVAL;
		2718
		2719	break;
		2720	default:
		2721	rc = -EOPNOTSUPP;
		2722	}
		2723
		2724	return rc;
		2725	}
		2726
2414	static const struct ethtool_ops virtnet_ethtool_ops = {	2727	static const struct ethtool_ops virtnet_ethtool_ops = {
2415	.supported_coalesce_params = ETHTOOL_COALESCE_MAX_FRAMES,	2728	.supported_coalesce_params = ETHTOOL_COALESCE_MAX_FRAMES,
2416	.get_drvinfo = virtnet_get_drvinfo,	2729	.get_drvinfo = virtnet_get_drvinfo,
@@ -2426,6 +2739,12 @@ static const struct ethtool_ops virtnet_ethtool_ops = { · 9 unmodified lines
2426	.set_link_ksettings = virtnet_set_link_ksettings,	2739	.set_link_ksettings = virtnet_set_link_ksettings,
2427	.set_coalesce = virtnet_set_coalesce,	2740	.set_coalesce = virtnet_set_coalesce,
2428	.get_coalesce = virtnet_get_coalesce,	2741	.get_coalesce = virtnet_get_coalesce,
		2742	.get_rxfh_key_size = virtnet_get_rxfh_key_size,
		2743	.get_rxfh_indir_size = virtnet_get_rxfh_indir_size,
		2744	.get_rxfh = virtnet_get_rxfh,
		2745	.set_rxfh = virtnet_set_rxfh,
		2746	.get_rxnfc = virtnet_get_rxnfc,
		2747	.set_rxnfc = virtnet_set_rxnfc,
2429	};	2748	};
2430		2749
2431	static void virtnet_freeze_down(struct virtio_device *vdev)	2750	static void virtnet_freeze_down(struct virtio_device *vdev)
@@ -2678,6 +2997,16 @@ static int virtnet_set_features(struct net_device *dev, · 246 unmodified lines
2678	vi->guest_offloads = offloads;	2997	vi->guest_offloads = offloads;
2679	}	2998	}
2680		2999
		3000	if ((dev->features ^ features) & NETIF_F_RXHASH) {
		3001	if (features & NETIF_F_RXHASH)
		3002	vi->ctrl->rss.hash_types = vi->rss_hash_types_saved;
		3003	else
		3004	vi->ctrl->rss.hash_types = VIRTIO_NET_HASH_REPORT_NONE;
		3005
		3006	if (!virtnet_commit_rss_command(vi))
		3007	return -EINVAL;
		3008	}
		3009
2681	return 0;	3010	return 0;
2682	}	3011	}
2683		3012
@@ -2851,7 +3180,7 @@ static void virtnet_del_vqs(struct virtnet_info *vi) · 167 unmodified lines
2851	*/	3180	*/
2852	static unsigned int mergeable_min_buf_len(struct virtnet_info vi, struct virtqueue vq)	3181	static unsigned int mergeable_min_buf_len(struct virtnet_info vi, struct virtqueue vq)
2853	{	3182	{
2854	const unsigned int hdr_len = sizeof(struct virtio_net_hdr_mrg_rxbuf);	3183	const unsigned int hdr_len = vi->hdr_len;
2855	unsigned int rq_size = virtqueue_get_vring_size(vq);	3184	unsigned int rq_size = virtqueue_get_vring_size(vq);
2856	unsigned int packet_len = vi->big_packets ? IP_MAX_MTU : vi->dev->max_mtu;	3185	unsigned int packet_len = vi->big_packets ? IP_MAX_MTU : vi->dev->max_mtu;
2857	unsigned int buf_len = hdr_len + ETH_HLEN + VLAN_HLEN + packet_len;	3186	unsigned int buf_len = hdr_len + ETH_HLEN + VLAN_HLEN + packet_len;
@@ -3072,6 +3401,10 @@ static bool virtnet_validate_features(struct virtio_device *vdev) · 214 unmodified lines
3072	"VIRTIO_NET_F_CTRL_VQ") \|\|	3401	"VIRTIO_NET_F_CTRL_VQ") \|\|
3073	VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_MQ, "VIRTIO_NET_F_CTRL_VQ") \|\|	3402	VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_MQ, "VIRTIO_NET_F_CTRL_VQ") \|\|
3074	VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_CTRL_MAC_ADDR,	3403	VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_CTRL_MAC_ADDR,
		3404	"VIRTIO_NET_F_CTRL_VQ") \|\|
		3405	VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_RSS,
		3406	"VIRTIO_NET_F_CTRL_VQ") \|\|
		3407	VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_HASH_REPORT,
3075	"VIRTIO_NET_F_CTRL_VQ"))) {	3408	"VIRTIO_NET_F_CTRL_VQ"))) {
3076	return false;	3409	return false;
3077	}	3410	}
@@ -3112,13 +3445,14 @@ static int virtnet_probe(struct virtio_device *vdev) · 34 unmodified lines
3112	u16 max_queue_pairs;	3445	u16 max_queue_pairs;
3113	int mtu;	3446	int mtu;
3114		3447
3115	/* Find if host supports multiqueue virtio_net device */	3448	/* Find if host supports multiqueue/rss virtio_net device */
3116	err = virtio_cread_feature(vdev, VIRTIO_NET_F_MQ,	3449	max_queue_pairs = 1;
3117	struct virtio_net_config,	3450	if (virtio_has_feature(vdev, VIRTIO_NET_F_MQ) \|\| virtio_has_feature(vdev, VIRTIO_NET_F_RSS))
3118	max_virtqueue_pairs, &max_queue_pairs);	3451	max_queue_pairs =
		3452	virtio_cread16(vdev, offsetof(struct virtio_net_config, max_virtqueue_pairs));
3119		3453
3120	/* We need at least 2 queue's */	3454	/* We need at least 2 queue's */
3121	if (err \|\| max_queue_pairs < VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN \|\|	3455	if (max_queue_pairs < VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN \|\|
3122	max_queue_pairs > VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX \|\|	3456	max_queue_pairs > VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX \|\|
3123	!virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ))	3457	!virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ))
3124	max_queue_pairs = 1;	3458	max_queue_pairs = 1;
@@ -3206,8 +3540,33 @@ static int virtnet_probe(struct virtio_device *vdev) · 81 unmodified lines
3206	if (virtio_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF))	3540	if (virtio_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF))
3207	vi->mergeable_rx_bufs = true;	3541	vi->mergeable_rx_bufs = true;
3208		3542
3209	if (virtio_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF) \|\|	3543	if (virtio_has_feature(vdev, VIRTIO_NET_F_HASH_REPORT))
3210	virtio_has_feature(vdev, VIRTIO_F_VERSION_1))	3544	vi->has_rss_hash_report = true;
		3545
		3546	if (virtio_has_feature(vdev, VIRTIO_NET_F_RSS))
		3547	vi->has_rss = true;
		3548
		3549	if (vi->has_rss \|\| vi->has_rss_hash_report) {
		3550	vi->rss_indir_table_size =
		3551	virtio_cread16(vdev, offsetof(struct virtio_net_config,
		3552	rss_max_indirection_table_length));
		3553	vi->rss_key_size =
		3554	virtio_cread8(vdev, offsetof(struct virtio_net_config, rss_max_key_size));
		3555
		3556	vi->rss_hash_types_supported =
		3557	virtio_cread32(vdev, offsetof(struct virtio_net_config, supported_hash_types));
		3558	vi->rss_hash_types_supported &=
		3559	~(VIRTIO_NET_RSS_HASH_TYPE_IP_EX \|
		3560	VIRTIO_NET_RSS_HASH_TYPE_TCP_EX \|
		3561	VIRTIO_NET_RSS_HASH_TYPE_UDP_EX);
		3562
		3563	dev->hw_features \|= NETIF_F_RXHASH;
		3564	}
		3565
		3566	if (vi->has_rss_hash_report)
		3567	vi->hdr_len = sizeof(struct virtio_net_hdr_v1_hash);
		3568	else if (virtio_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF) \|\|
		3569	virtio_has_feature(vdev, VIRTIO_F_VERSION_1))
3211	vi->hdr_len = sizeof(struct virtio_net_hdr_mrg_rxbuf);	3570	vi->hdr_len = sizeof(struct virtio_net_hdr_mrg_rxbuf);
3212	else	3571	else
3213	vi->hdr_len = sizeof(struct virtio_net_hdr);	3572	vi->hdr_len = sizeof(struct virtio_net_hdr);
@@ -3274,6 +3633,9 @@ static int virtnet_probe(struct virtio_device *vdev) · 60 unmodified lines
3274	}	3633	}
3275	}	3634	}
3276		3635
		3636	if (vi->has_rss \|\| vi->has_rss_hash_report)
		3637	virtnet_init_default_rss(vi);
		3638
3277	err = register_netdev(dev);	3639	err = register_netdev(dev);
3278	if (err) {	3640	if (err) {
3279	pr_debug("virtio_net: registering device failed\n");	3641	pr_debug("virtio_net: registering device failed\n");
@@ -3405,7 +3767,8 @@ static struct virtio_device_id id_table[] = { · 125 unmodified lines
3405	VIRTIO_NET_F_GUEST_ANNOUNCE, VIRTIO_NET_F_MQ, \	3767	VIRTIO_NET_F_GUEST_ANNOUNCE, VIRTIO_NET_F_MQ, \
3406	VIRTIO_NET_F_CTRL_MAC_ADDR, \	3768	VIRTIO_NET_F_CTRL_MAC_ADDR, \
3407	VIRTIO_NET_F_MTU, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS, \	3769	VIRTIO_NET_F_MTU, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS, \
3408	VIRTIO_NET_F_SPEED_DUPLEX, VIRTIO_NET_F_STANDBY	3770	VIRTIO_NET_F_SPEED_DUPLEX, VIRTIO_NET_F_STANDBY, \
		3771	VIRTIO_NET_F_RSS, VIRTIO_NET_F_HASH_REPORT
3409		3772
3410	static unsigned int features[] = {	3773	static unsigned int features[] = {
3411	VIRTNET_FEATURES,	3774	VIRTNET_FEATURES,

drivers/vdpa/ifcvf/ifcvf_base.c+44−80

 #include "ifcvf_base.h"
-static inline u8 ifc_ioread8(u8 __iomem *addr)
+struct ifcvf_adapter *vf_to_adapter(struct ifcvf_hw *hw)
-{
-	return ioread8(addr);
-}
-static inline u16 ifc_ioread16 (__le16 __iomem *addr)
 {
-	return ioread16(addr);
+	return container_of(hw, struct ifcvf_adapter, vf);
 }
-static inline u32 ifc_ioread32(__le32 __iomem *addr)
+u16 ifcvf_set_vq_vector(struct ifcvf_hw *hw, u16 qid, int vector)
 {
-	return ioread32(addr);
+	struct virtio_pci_common_cfg __iomem *cfg = hw->common_cfg;
-}
-static inline void ifc_iowrite8(u8 value, u8 __iomem *addr)
+	vp_iowrite16(qid, &cfg->queue_select);
-{
+	vp_iowrite16(vector, &cfg->queue_msix_vector);
-	iowrite8(value, addr);
-}
-static inline void ifc_iowrite16(u16 value, __le16 __iomem *addr)
+	return vp_ioread16(&cfg->queue_msix_vector);
-{
-	iowrite16(value, addr);
 }
-static inline void ifc_iowrite32(u32 value, __le32 __iomem *addr)
+u16 ifcvf_set_config_vector(struct ifcvf_hw *hw, int vector)
 {
-	iowrite32(value, addr);
+	struct virtio_pci_common_cfg __iomem *cfg = hw->common_cfg;
-}
-static void ifc_iowrite64_twopart(u64 val,
+	cfg = hw->common_cfg;
-				  __le32 __iomem *lo, __le32 __iomem *hi)
+	vp_iowrite16(vector,  &cfg->msix_config);
-{
-	ifc_iowrite32((u32)val, lo);
-	ifc_iowrite32(val >> 32, hi);
-}
-struct ifcvf_adapter *vf_to_adapter(struct ifcvf_hw *hw)
+	return vp_ioread16(&cfg->msix_config);
-{
-	return container_of(hw, struct ifcvf_adapter, vf);
 }
 static void __iomem *get_cap_addr(struct ifcvf_hw *hw,
 		return -EIO;
 	}
-	hw->nr_vring = ifc_ioread16(&hw->common_cfg->num_queues);
+	hw->nr_vring = vp_ioread16(&hw->common_cfg->num_queues);
 	for (i = 0; i < hw->nr_vring; i++) {
-		ifc_iowrite16(i, &hw->common_cfg->queue_select);
+		vp_iowrite16(i, &hw->common_cfg->queue_select);
-		notify_off = ifc_ioread16(&hw->common_cfg->queue_notify_off);
+		notify_off = vp_ioread16(&hw->common_cfg->queue_notify_off);
 		hw->vring[i].notify_addr = hw->notify_base +
 			notify_off * hw->notify_off_multiplier;
 		hw->vring[i].notify_pa = hw->notify_base_pa +
 			notify_off * hw->notify_off_multiplier;
+		hw->vring[i].irq = -EINVAL;
 	}
 	hw->lm_cfg = hw->base[IFCVF_LM_BAR];
 		  hw->common_cfg, hw->notify_base, hw->isr,
 		  hw->dev_cfg, hw->notify_off_multiplier);
+	hw->vqs_reused_irq = -EINVAL;
+	hw->config_irq = -EINVAL;
 	return 0;
 }
 u8 ifcvf_get_status(struct ifcvf_hw *hw)
 {
-	return ifc_ioread8(&hw->common_cfg->device_status);
+	return vp_ioread8(&hw->common_cfg->device_status);
 }
 void ifcvf_set_status(struct ifcvf_hw *hw, u8 status)
 {
-	ifc_iowrite8(status, &hw->common_cfg->device_status);
+	vp_iowrite8(status, &hw->common_cfg->device_status);
 }
 void ifcvf_reset(struct ifcvf_hw *hw)
 	u32 features_lo, features_hi;
 	u64 features;
-	ifc_iowrite32(0, &cfg->device_feature_select);
+	vp_iowrite32(0, &cfg->device_feature_select);
-	features_lo = ifc_ioread32(&cfg->device_feature);
+	features_lo = vp_ioread32(&cfg->device_feature);
-	ifc_iowrite32(1, &cfg->device_feature_select);
+	vp_iowrite32(1, &cfg->device_feature_select);
-	features_hi = ifc_ioread32(&cfg->device_feature);
+	features_hi = vp_ioread32(&cfg->device_feature);
 	features = ((u64)features_hi << 32) | features_lo;
 	WARN_ON(offset + length > hw->config_size);
 	do {
-		old_gen = ifc_ioread8(&hw->common_cfg->config_generation);
+		old_gen = vp_ioread8(&hw->common_cfg->config_generation);
 		p = dst;
 		for (i = 0; i < length; i++)
-			*p++ = ifc_ioread8(hw->dev_cfg + offset + i);
+			*p++ = vp_ioread8(hw->dev_cfg + offset + i);
-		new_gen = ifc_ioread8(&hw->common_cfg->config_generation);
+		new_gen = vp_ioread8(&hw->common_cfg->config_generation);
 	} while (old_gen != new_gen);
 }
 	p = src;
 	WARN_ON(offset + length > hw->config_size);
 	for (i = 0; i < length; i++)
-		ifc_iowrite8(*p++, hw->dev_cfg + offset + i);
+		vp_iowrite8(*p++, hw->dev_cfg + offset + i);
 }
 static void ifcvf_set_features(struct ifcvf_hw *hw, u64 features)
 {
 	struct virtio_pci_common_cfg __iomem *cfg = hw->common_cfg;
-	ifc_iowrite32(0, &cfg->guest_feature_select);
+	vp_iowrite32(0, &cfg->guest_feature_select);
-	ifc_iowrite32((u32)features, &cfg->guest_feature);
+	vp_iowrite32((u32)features, &cfg->guest_feature);
-	ifc_iowrite32(1, &cfg->guest_feature_select);
+	vp_iowrite32(1, &cfg->guest_feature_select);
-	ifc_iowrite32(features >> 32, &cfg->guest_feature);
+	vp_iowrite32(features >> 32, &cfg->guest_feature);
 }
 static int ifcvf_config_features(struct ifcvf_hw *hw)
 	ifcvf_lm = (struct ifcvf_lm_cfg __iomem *)hw->lm_cfg;
 	q_pair_id = qid / hw->nr_vring;
 	avail_idx_addr = &ifcvf_lm->vring_lm_cfg[q_pair_id].idx_addr[qid % 2];
-	last_avail_idx = ifc_ioread16(avail_idx_addr);
+	last_avail_idx = vp_ioread16(avail_idx_addr);
 	return last_avail_idx;
 }
 	q_pair_id = qid / hw->nr_vring;
 	avail_idx_addr = &ifcvf_lm->vring_lm_cfg[q_pair_id].idx_addr[qid % 2];
 	hw->vring[qid].last_avail_idx = num;
-	ifc_iowrite16(num, avail_idx_addr);
+	vp_iowrite16(num, avail_idx_addr);
 	return 0;
 }
 static int ifcvf_hw_enable(struct ifcvf_hw *hw)
 {
 	struct virtio_pci_common_cfg __iomem *cfg;
-	struct ifcvf_adapter *ifcvf;
 	u32 i;
-	ifcvf = vf_to_adapter(hw);
 	cfg = hw->common_cfg;
-	ifc_iowrite16(IFCVF_MSI_CONFIG_OFF, &cfg->msix_config);
-	if (ifc_ioread16(&cfg->msix_config) == VIRTIO_MSI_NO_VECTOR) {
-		IFCVF_ERR(ifcvf->pdev, "No msix vector for device config\n");
-		return -EINVAL;
-	}
 	for (i = 0; i < hw->nr_vring; i++) {
 		if (!hw->vring[i].ready)
 			break;
-		ifc_iowrite16(i, &cfg->queue_select);
+		vp_iowrite16(i, &cfg->queue_select);
-		ifc_iowrite64_twopart(hw->vring[i].desc, &cfg->queue_desc_lo,
+		vp_iowrite64_twopart(hw->vring[i].desc, &cfg->queue_desc_lo,
 				     &cfg->queue_desc_hi);
-		ifc_iowrite64_twopart(hw->vring[i].avail, &cfg->queue_avail_lo,
+		vp_iowrite64_twopart(hw->vring[i].avail, &cfg->queue_avail_lo,
 				      &cfg->queue_avail_hi);
-		ifc_iowrite64_twopart(hw->vring[i].used, &cfg->queue_used_lo,
+		vp_iowrite64_twopart(hw->vring[i].used, &cfg->queue_used_lo,
 				     &cfg->queue_used_hi);
-		ifc_iowrite16(hw->vring[i].size, &cfg->queue_size);
+		vp_iowrite16(hw->vring[i].size, &cfg->queue_size);
-		ifc_iowrite16(i + IFCVF_MSI_QUEUE_OFF, &cfg->queue_msix_vector);
-		if (ifc_ioread16(&cfg->queue_msix_vector) ==
-		    VIRTIO_MSI_NO_VECTOR) {
-			IFCVF_ERR(ifcvf->pdev,
-				  "No msix vector for queue %u\n", i);
-			return -EINVAL;
-		}
 		ifcvf_set_vq_state(hw, i, hw->vring[i].last_avail_idx);
-		ifc_iowrite16(1, &cfg->queue_enable);
+		vp_iowrite16(1, &cfg->queue_enable);
 	}
 	return 0;
 static void ifcvf_hw_disable(struct ifcvf_hw *hw)
 {
-	struct virtio_pci_common_cfg __iomem *cfg;
 	u32 i;
-	cfg = hw->common_cfg;
+	ifcvf_set_config_vector(hw, VIRTIO_MSI_NO_VECTOR);
-	ifc_iowrite16(VIRTIO_MSI_NO_VECTOR, &cfg->msix_config);
 	for (i = 0; i < hw->nr_vring; i++) {
-		ifc_iowrite16(i, &cfg->queue_select);
+		ifcvf_set_vq_vector(hw, i, VIRTIO_MSI_NO_VECTOR);
-		ifc_iowrite16(VIRTIO_MSI_NO_VECTOR, &cfg->queue_msix_vector);
 	}
-	ifc_ioread16(&cfg->queue_msix_vector);
 }
 int ifcvf_start_hw(struct ifcvf_hw *hw)
 void ifcvf_notify_queue(struct ifcvf_hw *hw, u16 qid)
 {
-	ifc_iowrite16(qid, hw->vring[qid].notify_addr);
+	vp_iowrite16(qid, hw->vring[qid].notify_addr);
 }

drivers/vdpa/ifcvf/ifcvf_base.h+17−7

 #include <linux/pci.h>
 #include <linux/pci_regs.h>
 #include <linux/vdpa.h>
+#include <linux/virtio_pci_modern.h>
 #include <uapi/linux/virtio_net.h>
 #include <uapi/linux/virtio_blk.h>
 #include <uapi/linux/virtio_config.h>
 #define IFCVF_QUEUE_ALIGNMENT	PAGE_SIZE
 #define IFCVF_QUEUE_MAX		32768
-#define IFCVF_MSI_CONFIG_OFF	0
-#define IFCVF_MSI_QUEUE_OFF	1
 #define IFCVF_PCI_MAX_RESOURCE	6
 #define IFCVF_LM_CFG_SIZE		0x40
 #define ifcvf_private_to_vf(adapter) \
 	(&((struct ifcvf_adapter *)adapter)->vf)
+/* all vqs and config interrupt has its own vector */
+#define MSIX_VECTOR_PER_VQ_AND_CONFIG		1
+/* all vqs share a vector, and config interrupt has a separate vector */
+#define MSIX_VECTOR_SHARED_VQ_AND_CONFIG	2
+/* all vqs and config interrupt share a vector */
+#define MSIX_VECTOR_DEV_SHARED			3
 struct vring_info {
 	u64 desc;
 	u64 avail;
 	u8 __iomem *isr;
 	/* Live migration */
 	u8 __iomem *lm_cfg;
-	u16 nr_vring;
 	/* Notification bar number */
 	u8 notify_bar;
+	u8 msix_vector_status;
+	/* virtio-net or virtio-blk device config size */
+	u32 config_size;
 	/* Notificaiton bar address */
 	void __iomem *notify_base;
 	phys_addr_t notify_base_pa;
 	u32 notify_off_multiplier;
+	u32 dev_type;
 	u64 req_features;
 	u64 hw_features;
-	u32 dev_type;
 	struct virtio_pci_common_cfg __iomem *common_cfg;
 	void __iomem *dev_cfg;
 	struct vring_info vring[IFCVF_MAX_QUEUES];
 	void __iomem * const *base;
 	char config_msix_name[256];
 	struct vdpa_callback config_cb;
-	unsigned int config_irq;
+	int config_irq;
-	/* virtio-net or virtio-blk device config size */
+	int vqs_reused_irq;
-	u32 config_size;
+	u16 nr_vring;
 };
 struct ifcvf_adapter {
 struct ifcvf_adapter *vf_to_adapter(struct ifcvf_hw *hw);
 int ifcvf_probed_virtio_net(struct ifcvf_hw *hw);
 u32 ifcvf_get_config_size(struct ifcvf_hw *hw);
+u16 ifcvf_set_vq_vector(struct ifcvf_hw *hw, u16 qid, int vector);
+u16 ifcvf_set_config_vector(struct ifcvf_hw *hw, int vector);
 #endif /* _IFCVF_H_ */

drivers/vdpa/ifcvf/ifcvf_main.c+284−33

@@ -27,7 +27,7 @@ static irqreturn_t ifcvf_config_changed(int irq, void *arg) · 26 unmodified lines
27	return IRQ_HANDLED;	27	return IRQ_HANDLED;
28	}	28	}
29		29
30	static irqreturn_t ifcvf_intr_handler(int irq, void *arg)	30	static irqreturn_t ifcvf_vq_intr_handler(int irq, void *arg)
31	{	31	{
32	struct vring_info *vring = arg;	32	struct vring_info *vring = arg;
33		33
@@ -37,76 +37,324 @@ static irqreturn_t ifcvf_intr_handler(int irq, void *arg) · 3 unmodified lines
37	return IRQ_HANDLED;	37	return IRQ_HANDLED;
38	}	38	}
39		39
		40	static irqreturn_t ifcvf_vqs_reused_intr_handler(int irq, void *arg)
		41	{
		42	struct ifcvf_hw *vf = arg;
		43	struct vring_info *vring;
		44	int i;
		45
		46	for (i = 0; i < vf->nr_vring; i++) {
		47	vring = &vf->vring[i];
		48	if (vring->cb.callback)
		49	vring->cb.callback(vring->cb.private);
		50	}
		51
		52	return IRQ_HANDLED;
		53	}
		54
		55	static irqreturn_t ifcvf_dev_intr_handler(int irq, void *arg)
		56	{
		57	struct ifcvf_hw *vf = arg;
		58	u8 isr;
		59
		60	isr = vp_ioread8(vf->isr);
		61	if (isr & VIRTIO_PCI_ISR_CONFIG)
		62	ifcvf_config_changed(irq, arg);
		63
		64	return ifcvf_vqs_reused_intr_handler(irq, arg);
		65	}
		66
40	static void ifcvf_free_irq_vectors(void *data)	67	static void ifcvf_free_irq_vectors(void *data)
41	{	68	{
42	pci_free_irq_vectors(data);	69	pci_free_irq_vectors(data);
43	}	70	}
44		71
45	static void ifcvf_free_irq(struct ifcvf_adapter *adapter, int queues)	72	static void ifcvf_free_per_vq_irq(struct ifcvf_adapter *adapter)
46	{	73	{
47	struct pci_dev *pdev = adapter->pdev;	74	struct pci_dev *pdev = adapter->pdev;
48	struct ifcvf_hw *vf = &adapter->vf;	75	struct ifcvf_hw *vf = &adapter->vf;
49	int i;	76	int i;
50		77
		78	for (i = 0; i < vf->nr_vring; i++) {
		79	if (vf->vring[i].irq != -EINVAL) {
		80	devm_free_irq(&pdev->dev, vf->vring[i].irq, &vf->vring[i]);
		81	vf->vring[i].irq = -EINVAL;
		82	}
		83	}
		84	}
51		85
52	for (i = 0; i < queues; i++) {	86	static void ifcvf_free_vqs_reused_irq(struct ifcvf_adapter *adapter)
53	devm_free_irq(&pdev->dev, vf->vring[i].irq, &vf->vring[i]);	87	{
54	vf->vring[i].irq = -EINVAL;	88	struct pci_dev *pdev = adapter->pdev;
		89	struct ifcvf_hw *vf = &adapter->vf;
		90
		91	if (vf->vqs_reused_irq != -EINVAL) {
		92	devm_free_irq(&pdev->dev, vf->vqs_reused_irq, vf);
		93	vf->vqs_reused_irq = -EINVAL;
55	}	94	}
56		95
57	devm_free_irq(&pdev->dev, vf->config_irq, vf);	96	}
		97
		98	static void ifcvf_free_vq_irq(struct ifcvf_adapter *adapter)
		99	{
		100	struct ifcvf_hw *vf = &adapter->vf;
		101
		102	if (vf->msix_vector_status == MSIX_VECTOR_PER_VQ_AND_CONFIG)
		103	ifcvf_free_per_vq_irq(adapter);
		104	else
		105	ifcvf_free_vqs_reused_irq(adapter);
		106	}
		107
		108	static void ifcvf_free_config_irq(struct ifcvf_adapter *adapter)
		109	{
		110	struct pci_dev *pdev = adapter->pdev;
		111	struct ifcvf_hw *vf = &adapter->vf;
		112
		113	if (vf->config_irq == -EINVAL)
		114	return;
		115
		116	/* If the irq is shared by all vqs and the config interrupt,
		117	* it is already freed in ifcvf_free_vq_irq, so here only
		118	* need to free config irq when msix_vector_status != MSIX_VECTOR_DEV_SHARED
		119	*/
		120	if (vf->msix_vector_status != MSIX_VECTOR_DEV_SHARED) {
		121	devm_free_irq(&pdev->dev, vf->config_irq, vf);
		122	vf->config_irq = -EINVAL;
		123	}
		124	}
		125
		126	static void ifcvf_free_irq(struct ifcvf_adapter *adapter)
		127	{
		128	struct pci_dev *pdev = adapter->pdev;
		129
		130	ifcvf_free_vq_irq(adapter);
		131	ifcvf_free_config_irq(adapter);
58	ifcvf_free_irq_vectors(pdev);	132	ifcvf_free_irq_vectors(pdev);
59	}	133	}
60		134
61	static int ifcvf_request_irq(struct ifcvf_adapter *adapter)	135	/* ifcvf MSIX vectors allocator, this helper tries to allocate
		136	* vectors for all virtqueues and the config interrupt.
		137	* It returns the number of allocated vectors, negative
		138	* return value when fails.
		139	*/
		140	static int ifcvf_alloc_vectors(struct ifcvf_adapter *adapter)
62	{	141	{
63	struct pci_dev *pdev = adapter->pdev;	142	struct pci_dev *pdev = adapter->pdev;
64	struct ifcvf_hw *vf = &adapter->vf;	143	struct ifcvf_hw *vf = &adapter->vf;
65	int vector, i, ret, irq;	144	int max_intr, ret;
66	u16 max_intr;
67		145
68	/* all queues and config interrupt */	146	/* all queues and config interrupt */
69	max_intr = vf->nr_vring + 1;	147	max_intr = vf->nr_vring + 1;
		148	ret = pci_alloc_irq_vectors(pdev, 1, max_intr, PCI_IRQ_MSIX \| PCI_IRQ_AFFINITY);
70		149
71	ret = pci_alloc_irq_vectors(pdev, max_intr,
72	max_intr, PCI_IRQ_MSIX);
73	if (ret < 0) {	150	if (ret < 0) {
74	IFCVF_ERR(pdev, "Failed to alloc IRQ vectors\n");	151	IFCVF_ERR(pdev, "Failed to alloc IRQ vectors\n");
75	return ret;	152	return ret;
76	}	153	}
77		154
		155	if (ret < max_intr)
		156	IFCVF_INFO(pdev,
		157	"Requested %u vectors, however only %u allocated, lower performance\n",
		158	max_intr, ret);
		159
		160	return ret;
		161	}
		162
		163	static int ifcvf_request_per_vq_irq(struct ifcvf_adapter *adapter)
		164	{
		165	struct pci_dev *pdev = adapter->pdev;
		166	struct ifcvf_hw *vf = &adapter->vf;
		167	int i, vector, ret, irq;
		168
		169	vf->vqs_reused_irq = -EINVAL;
		170	for (i = 0; i < vf->nr_vring; i++) {
		171	snprintf(vf->vring[i].msix_name, 256, "ifcvf[%s]-%d\n", pci_name(pdev), i);
		172	vector = i;
		173	irq = pci_irq_vector(pdev, vector);
		174	ret = devm_request_irq(&pdev->dev, irq,
		175	ifcvf_vq_intr_handler, 0,
		176	vf->vring[i].msix_name,
		177	&vf->vring[i]);
		178	if (ret) {
		179	IFCVF_ERR(pdev, "Failed to request irq for vq %d\n", i);
		180	goto err;
		181	}
		182
		183	vf->vring[i].irq = irq;
		184	ret = ifcvf_set_vq_vector(vf, i, vector);
		185	if (ret == VIRTIO_MSI_NO_VECTOR) {
		186	IFCVF_ERR(pdev, "No msix vector for vq %u\n", i);
		187	goto err;
		188	}
		189	}
		190
		191	return 0;
		192	err:
		193	ifcvf_free_irq(adapter);
		194
		195	return -EFAULT;
		196	}
		197
		198	static int ifcvf_request_vqs_reused_irq(struct ifcvf_adapter *adapter)
		199	{
		200	struct pci_dev *pdev = adapter->pdev;
		201	struct ifcvf_hw *vf = &adapter->vf;
		202	int i, vector, ret, irq;
		203
		204	vector = 0;
		205	snprintf(vf->vring[0].msix_name, 256, "ifcvf[%s]-vqs-reused-irq\n", pci_name(pdev));
		206	irq = pci_irq_vector(pdev, vector);
		207	ret = devm_request_irq(&pdev->dev, irq,
		208	ifcvf_vqs_reused_intr_handler, 0,
		209	vf->vring[0].msix_name, vf);
		210	if (ret) {
		211	IFCVF_ERR(pdev, "Failed to request reused irq for the device\n");
		212	goto err;
		213	}
		214
		215	vf->vqs_reused_irq = irq;
		216	for (i = 0; i < vf->nr_vring; i++) {
		217	vf->vring[i].irq = -EINVAL;
		218	ret = ifcvf_set_vq_vector(vf, i, vector);
		219	if (ret == VIRTIO_MSI_NO_VECTOR) {
		220	IFCVF_ERR(pdev, "No msix vector for vq %u\n", i);
		221	goto err;
		222	}
		223	}
		224
		225	return 0;
		226	err:
		227	ifcvf_free_irq(adapter);
		228
		229	return -EFAULT;
		230	}
		231
		232	static int ifcvf_request_dev_irq(struct ifcvf_adapter *adapter)
		233	{
		234	struct pci_dev *pdev = adapter->pdev;
		235	struct ifcvf_hw *vf = &adapter->vf;
		236	int i, vector, ret, irq;
		237
		238	vector = 0;
		239	snprintf(vf->vring[0].msix_name, 256, "ifcvf[%s]-dev-irq\n", pci_name(pdev));
		240	irq = pci_irq_vector(pdev, vector);
		241	ret = devm_request_irq(&pdev->dev, irq,
		242	ifcvf_dev_intr_handler, 0,
		243	vf->vring[0].msix_name, vf);
		244	if (ret) {
		245	IFCVF_ERR(pdev, "Failed to request irq for the device\n");
		246	goto err;
		247	}
		248
		249	vf->vqs_reused_irq = irq;
		250	for (i = 0; i < vf->nr_vring; i++) {
		251	vf->vring[i].irq = -EINVAL;
		252	ret = ifcvf_set_vq_vector(vf, i, vector);
		253	if (ret == VIRTIO_MSI_NO_VECTOR) {
		254	IFCVF_ERR(pdev, "No msix vector for vq %u\n", i);
		255	goto err;
		256	}
		257	}
		258
		259	vf->config_irq = irq;
		260	ret = ifcvf_set_config_vector(vf, vector);
		261	if (ret == VIRTIO_MSI_NO_VECTOR) {
		262	IFCVF_ERR(pdev, "No msix vector for device config\n");
		263	goto err;
		264	}
		265
		266	return 0;
		267	err:
		268	ifcvf_free_irq(adapter);
		269
		270	return -EFAULT;
		271
		272	}
		273
		274	static int ifcvf_request_vq_irq(struct ifcvf_adapter *adapter)
		275	{
		276	struct ifcvf_hw *vf = &adapter->vf;
		277	int ret;
		278
		279	if (vf->msix_vector_status == MSIX_VECTOR_PER_VQ_AND_CONFIG)
		280	ret = ifcvf_request_per_vq_irq(adapter);
		281	else
		282	ret = ifcvf_request_vqs_reused_irq(adapter);
		283
		284	return ret;
		285	}
		286
		287	static int ifcvf_request_config_irq(struct ifcvf_adapter *adapter)
		288	{
		289	struct pci_dev *pdev = adapter->pdev;
		290	struct ifcvf_hw *vf = &adapter->vf;
		291	int config_vector, ret;
		292
		293	if (vf->msix_vector_status == MSIX_VECTOR_DEV_SHARED)
		294	return 0;
		295
		296	if (vf->msix_vector_status == MSIX_VECTOR_PER_VQ_AND_CONFIG)
		297	/* vector 0 ~ vf->nr_vring for vqs, num vf->nr_vring vector for config interrupt */
		298	config_vector = vf->nr_vring;
		299
		300	if (vf->msix_vector_status == MSIX_VECTOR_SHARED_VQ_AND_CONFIG)
		301	/* vector 0 for vqs and 1 for config interrupt */
		302	config_vector = 1;
		303
78	snprintf(vf->config_msix_name, 256, "ifcvf[%s]-config\n",	304	snprintf(vf->config_msix_name, 256, "ifcvf[%s]-config\n",
79	pci_name(pdev));	305	pci_name(pdev));
80	vector = 0;	306	vf->config_irq = pci_irq_vector(pdev, config_vector);
81	vf->config_irq = pci_irq_vector(pdev, vector);
82	ret = devm_request_irq(&pdev->dev, vf->config_irq,	307	ret = devm_request_irq(&pdev->dev, vf->config_irq,
83	ifcvf_config_changed, 0,	308	ifcvf_config_changed, 0,
84	vf->config_msix_name, vf);	309	vf->config_msix_name, vf);
85	if (ret) {	310	if (ret) {
86	IFCVF_ERR(pdev, "Failed to request config irq\n");	311	IFCVF_ERR(pdev, "Failed to request config irq\n");
87	return ret;	312	goto err;
88	}	313	}
89		314
90	for (i = 0; i < vf->nr_vring; i++) {	315	ret = ifcvf_set_config_vector(vf, config_vector);
91	snprintf(vf->vring[i].msix_name, 256, "ifcvf[%s]-%d\n",	316	if (ret == VIRTIO_MSI_NO_VECTOR) {
92	pci_name(pdev), i);	317	IFCVF_ERR(pdev, "No msix vector for device config\n");
93	vector = i + IFCVF_MSI_QUEUE_OFF;	318	goto err;
94	irq = pci_irq_vector(pdev, vector);	319	}
95	ret = devm_request_irq(&pdev->dev, irq,
96	ifcvf_intr_handler, 0,
97	vf->vring[i].msix_name,
98	&vf->vring[i]);
99	if (ret) {
100	IFCVF_ERR(pdev,
101	"Failed to request irq for vq %d\n", i);
102	ifcvf_free_irq(adapter, i);
103		320
104	return ret;	321	return 0;
105	}	322	err:
		323	ifcvf_free_irq(adapter);
106		324
107	vf->vring[i].irq = irq;	325	return -EFAULT;
		326	}
		327
		328	static int ifcvf_request_irq(struct ifcvf_adapter *adapter)
		329	{
		330	struct ifcvf_hw *vf = &adapter->vf;
		331	int nvectors, ret, max_intr;
		332
		333	nvectors = ifcvf_alloc_vectors(adapter);
		334	if (nvectors <= 0)
		335	return -EFAULT;
		336
		337	vf->msix_vector_status = MSIX_VECTOR_PER_VQ_AND_CONFIG;
		338	max_intr = vf->nr_vring + 1;
		339	if (nvectors < max_intr)
		340	vf->msix_vector_status = MSIX_VECTOR_SHARED_VQ_AND_CONFIG;
		341
		342	if (nvectors == 1) {
		343	vf->msix_vector_status = MSIX_VECTOR_DEV_SHARED;
		344	ret = ifcvf_request_dev_irq(adapter);
		345
		346	return ret;
108	}	347	}
109		348
		349	ret = ifcvf_request_vq_irq(adapter);
		350	if (ret)
		351	return ret;
		352
		353	ret = ifcvf_request_config_irq(adapter);
		354
		355	if (ret)
		356	return ret;
		357
110	return 0;	358	return 0;
111	}	359	}
112		360
@@ -263,7 +511,7 @@ static int ifcvf_vdpa_reset(struct vdpa_device *vdpa_dev) · 150 unmodified lines
263		511
264	if (status_old & VIRTIO_CONFIG_S_DRIVER_OK) {	512	if (status_old & VIRTIO_CONFIG_S_DRIVER_OK) {
265	ifcvf_stop_datapath(adapter);	513	ifcvf_stop_datapath(adapter);
266	ifcvf_free_irq(adapter, vf->nr_vring);	514	ifcvf_free_irq(adapter);
267	}	515	}
268		516
269	ifcvf_reset_vring(adapter);	517	ifcvf_reset_vring(adapter);
@@ -348,7 +596,7 @@ static u32 ifcvf_vdpa_get_generation(struct vdpa_device *vdpa_dev) · 78 unmodified lines
348	{	596	{
349	struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);	597	struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
350		598
351	return ioread8(&vf->common_cfg->config_generation);	599	return vp_ioread8(&vf->common_cfg->config_generation);
352	}	600	}
353		601
354	static u32 ifcvf_vdpa_get_device_id(struct vdpa_device *vdpa_dev)	602	static u32 ifcvf_vdpa_get_device_id(struct vdpa_device *vdpa_dev)
@@ -410,7 +658,10 @@ static int ifcvf_vdpa_get_vq_irq(struct vdpa_device *vdpa_dev, · 55 unmodified lines
410	{	658	{
411	struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);	659	struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
412		660
413	return vf->vring[qid].irq;	661	if (vf->vqs_reused_irq < 0)
		662	return vf->vring[qid].irq;
		663	else
		664	return -EINVAL;
414	}	665	}
415		666
416	static struct vdpa_notification_area ifcvf_get_vq_notification(struct vdpa_device *vdpa_dev,	667	static struct vdpa_notification_area ifcvf_get_vq_notification(struct vdpa_device *vdpa_dev,

drivers/vdpa/mlx5/net/mlx5_vnet.c+80−4

@@ -1475,7 +1475,7 @@ static virtio_net_ctrl_ack handle_ctrl_mac(struct mlx5_vdpa_dev *mvdev, u8 cmd) · 1474 unmodified lines
1475	virtio_net_ctrl_ack status = VIRTIO_NET_ERR;	1475	virtio_net_ctrl_ack status = VIRTIO_NET_ERR;
1476	struct mlx5_core_dev *pfmdev;	1476	struct mlx5_core_dev *pfmdev;
1477	size_t read;	1477	size_t read;
1478	u8 mac[ETH_ALEN];	1478	u8 mac[ETH_ALEN], mac_back[ETH_ALEN];
1479		1479
1480	pfmdev = pci_get_drvdata(pci_physfn(mvdev->mdev->pdev));	1480	pfmdev = pci_get_drvdata(pci_physfn(mvdev->mdev->pdev));
1481	switch (cmd) {	1481	switch (cmd) {
@@ -1489,6 +1489,9 @@ static virtio_net_ctrl_ack handle_ctrl_mac(struct mlx5_vdpa_dev *mvdev, u8 cmd) · 7 unmodified lines
1489	break;	1489	break;
1490	}	1490	}
1491		1491
		1492	if (is_zero_ether_addr(mac))
		1493	break;
		1494
1492	if (!is_zero_ether_addr(ndev->config.mac)) {	1495	if (!is_zero_ether_addr(ndev->config.mac)) {
1493	if (mlx5_mpfs_del_mac(pfmdev, ndev->config.mac)) {	1496	if (mlx5_mpfs_del_mac(pfmdev, ndev->config.mac)) {
1494	mlx5_vdpa_warn(mvdev, "failed to delete old MAC %pM from MPFS table\n",	1497	mlx5_vdpa_warn(mvdev, "failed to delete old MAC %pM from MPFS table\n",
@@ -1503,7 +1506,47 @@ static virtio_net_ctrl_ack handle_ctrl_mac(struct mlx5_vdpa_dev *mvdev, u8 cmd) · 8 unmodified lines
1503	break;	1506	break;
1504	}	1507	}
1505		1508
		1509	/* backup the original mac address so that if failed to add the forward rules
		1510	* we could restore it
		1511	*/
		1512	memcpy(mac_back, ndev->config.mac, ETH_ALEN);
		1513
1506	memcpy(ndev->config.mac, mac, ETH_ALEN);	1514	memcpy(ndev->config.mac, mac, ETH_ALEN);
		1515
		1516	/* Need recreate the flow table entry, so that the packet could forward back
		1517	*/
		1518	remove_fwd_to_tir(ndev);
		1519
		1520	if (add_fwd_to_tir(ndev)) {
		1521	mlx5_vdpa_warn(mvdev, "failed to insert forward rules, try to restore\n");
		1522
		1523	/* Although it hardly run here, we still need double check */
		1524	if (is_zero_ether_addr(mac_back)) {
		1525	mlx5_vdpa_warn(mvdev, "restore mac failed: Original MAC is zero\n");
		1526	break;
		1527	}
		1528
		1529	/* Try to restore original mac address to MFPS table, and try to restore
		1530	* the forward rule entry.
		1531	*/
		1532	if (mlx5_mpfs_del_mac(pfmdev, ndev->config.mac)) {
		1533	mlx5_vdpa_warn(mvdev, "restore mac failed: delete MAC %pM from MPFS table failed\n",
		1534	ndev->config.mac);
		1535	}
		1536
		1537	if (mlx5_mpfs_add_mac(pfmdev, mac_back)) {
		1538	mlx5_vdpa_warn(mvdev, "restore mac failed: insert old MAC %pM into MPFS table failed\n",
		1539	mac_back);
		1540	}
		1541
		1542	memcpy(ndev->config.mac, mac_back, ETH_ALEN);
		1543
		1544	if (add_fwd_to_tir(ndev))
		1545	mlx5_vdpa_warn(mvdev, "restore forward rules failed: insert forward rules failed\n");
		1546
		1547	break;
		1548	}
		1549
1507	status = VIRTIO_NET_OK;	1550	status = VIRTIO_NET_OK;
1508	break;	1551	break;
1509		1552
@@ -1669,7 +1712,7 @@ static void mlx5_vdpa_kick_vq(struct vdpa_device *vdev, u16 idx) · 159 unmodified lines
1669	return;	1712	return;
1670		1713
1671	if (unlikely(is_ctrl_vq_idx(mvdev, idx))) {	1714	if (unlikely(is_ctrl_vq_idx(mvdev, idx))) {
1672	if (!mvdev->cvq.ready)	1715	if (!mvdev->wq \|\| !mvdev->cvq.ready)
1673	return;	1716	return;
1674		1717
1675	wqent = kzalloc(sizeof(*wqent), GFP_ATOMIC);	1718	wqent = kzalloc(sizeof(*wqent), GFP_ATOMIC);
@@ -2565,6 +2608,28 @@ static int event_handler(struct notifier_block nb, unsigned long event, void p · 889 unmodified lines
2565	return ret;	2608	return ret;
2566	}	2609	}
2567		2610
		2611	static int config_func_mtu(struct mlx5_core_dev *mdev, u16 mtu)
		2612	{
		2613	int inlen = MLX5_ST_SZ_BYTES(modify_nic_vport_context_in);
		2614	void *in;
		2615	int err;
		2616
		2617	in = kvzalloc(inlen, GFP_KERNEL);
		2618	if (!in)
		2619	return -ENOMEM;
		2620
		2621	MLX5_SET(modify_nic_vport_context_in, in, field_select.mtu, 1);
		2622	MLX5_SET(modify_nic_vport_context_in, in, nic_vport_context.mtu,
		2623	mtu + MLX5V_ETH_HARD_MTU);
		2624	MLX5_SET(modify_nic_vport_context_in, in, opcode,
		2625	MLX5_CMD_OP_MODIFY_NIC_VPORT_CONTEXT);
		2626
		2627	err = mlx5_cmd_exec_in(mdev, modify_nic_vport_context, in);
		2628
		2629	kvfree(in);
		2630	return err;
		2631	}
		2632
2568	static int mlx5_vdpa_dev_add(struct vdpa_mgmt_dev v_mdev, const char name,	2633	static int mlx5_vdpa_dev_add(struct vdpa_mgmt_dev v_mdev, const char name,
2569	const struct vdpa_dev_set_config *add_config)	2634	const struct vdpa_dev_set_config *add_config)
2570	{	2635	{
@@ -2624,6 +2689,13 @@ static int mlx5_vdpa_dev_add(struct vdpa_mgmt_dev v_mdev, const char name, · 53 unmodified lines
2624	init_mvqs(ndev);	2689	init_mvqs(ndev);
2625	mutex_init(&ndev->reslock);	2690	mutex_init(&ndev->reslock);
2626	config = &ndev->config;	2691	config = &ndev->config;
		2692
		2693	if (add_config->mask & BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MTU)) {
		2694	err = config_func_mtu(mdev, add_config->net.mtu);
		2695	if (err)
		2696	goto err_mtu;
		2697	}
		2698
2627	err = query_mtu(mdev, &mtu);	2699	err = query_mtu(mdev, &mtu);
2628	if (err)	2700	if (err)
2629	goto err_mtu;	2701	goto err_mtu;
@@ -2707,9 +2779,12 @@ static void mlx5_vdpa_dev_del(struct vdpa_mgmt_dev v_mdev, struct vdpa_device · 77 unmodified lines
2707	struct mlx5_vdpa_mgmtdev *mgtdev = container_of(v_mdev, struct mlx5_vdpa_mgmtdev, mgtdev);	2779	struct mlx5_vdpa_mgmtdev *mgtdev = container_of(v_mdev, struct mlx5_vdpa_mgmtdev, mgtdev);
2708	struct mlx5_vdpa_dev *mvdev = to_mvdev(dev);	2780	struct mlx5_vdpa_dev *mvdev = to_mvdev(dev);
2709	struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev);	2781	struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev);
		2782	struct workqueue_struct *wq;
2710		2783
2711	mlx5_notifier_unregister(mvdev->mdev, &ndev->nb);	2784	mlx5_notifier_unregister(mvdev->mdev, &ndev->nb);
2712	destroy_workqueue(mvdev->wq);	2785	wq = mvdev->wq;
		2786	mvdev->wq = NULL;
		2787	destroy_workqueue(wq);
2713	_vdpa_unregister_device(dev);	2788	_vdpa_unregister_device(dev);
2714	mgtdev->ndev = NULL;	2789	mgtdev->ndev = NULL;
2715	}	2790	}
@@ -2741,7 +2816,8 @@ static int mlx5v_probe(struct auxiliary_device *adev, · 25 unmodified lines
2741	mgtdev->mgtdev.device = mdev->device;	2816	mgtdev->mgtdev.device = mdev->device;
2742	mgtdev->mgtdev.id_table = id_table;	2817	mgtdev->mgtdev.id_table = id_table;
2743	mgtdev->mgtdev.config_attr_mask = BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MACADDR) \|	2818	mgtdev->mgtdev.config_attr_mask = BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MACADDR) \|
2744	BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MAX_VQP);	2819	BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MAX_VQP) \|
		2820	BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MTU);
2745	mgtdev->mgtdev.max_supported_vqs =	2821	mgtdev->mgtdev.max_supported_vqs =
2746	MLX5_CAP_DEV_VDPA_EMULATION(mdev, max_num_virtio_queues) + 1;	2822	MLX5_CAP_DEV_VDPA_EMULATION(mdev, max_num_virtio_queues) + 1;
2747	mgtdev->mgtdev.supported_features = get_supported_features(mdev);	2823	mgtdev->mgtdev.supported_features = get_supported_features(mdev);

drivers/vdpa/vdpa.c+3−3

@@ -232,7 +232,7 @@ static int vdpa_name_match(struct device dev, const void data) · 231 unmodified lines
232	return (strcmp(dev_name(&vdev->dev), data) == 0);	232	return (strcmp(dev_name(&vdev->dev), data) == 0);
233	}	233	}
234		234
235	static int __vdpa_register_device(struct vdpa_device *vdev, int nvqs)	235	static int __vdpa_register_device(struct vdpa_device *vdev, u32 nvqs)
236	{	236	{
237	struct device *dev;	237	struct device *dev;
238		238
@@ -257,7 +257,7 @@ static int __vdpa_register_device(struct vdpa_device *vdev, int nvqs) · 18 unmodified lines
257	*	257	*
258	* Return: Returns an error when fail to add device to vDPA bus	258	* Return: Returns an error when fail to add device to vDPA bus
259	*/	259	*/
260	int _vdpa_register_device(struct vdpa_device *vdev, int nvqs)	260	int _vdpa_register_device(struct vdpa_device *vdev, u32 nvqs)
261	{	261	{
262	if (!vdev->mdev)	262	if (!vdev->mdev)
263	return -EINVAL;	263	return -EINVAL;
@@ -274,7 +274,7 @@ EXPORT_SYMBOL_GPL(_vdpa_register_device); · 10 unmodified lines
274	*	274	*
275	* Return: Returns an error when fail to add to vDPA bus	275	* Return: Returns an error when fail to add to vDPA bus
276	*/	276	*/
277	int vdpa_register_device(struct vdpa_device *vdev, int nvqs)	277	int vdpa_register_device(struct vdpa_device *vdev, u32 nvqs)
278	{	278	{
279	int err;	279	int err;
280		280

drivers/vhost/iotlb.c+5−1

 	 */
 	if (start == 0 && last == ULONG_MAX) {
 		u64 mid = last / 2;
+		int err = vhost_iotlb_add_range_ctx(iotlb, start, mid, addr,
+				perm, opaque);
+		if (err)
+			return err;
-		vhost_iotlb_add_range_ctx(iotlb, start, mid, addr, perm, opaque);
 		addr += mid + 1;
 		start = mid + 1;
 	}

drivers/vhost/vdpa.c+40−5

 	struct device dev;
 	struct cdev cdev;
 	atomic_t opened;
-	int nvqs;
+	u32 nvqs;
 	int virtio_id;
 	int minor;
 	struct eventfd_ctx *config_ctx;
 		return;
 	irq = ops->get_vq_irq(vdpa, qid);
+	if (irq < 0)
+		return;
 	irq_bypass_unregister_producer(&vq->call_ctx.producer);
-	if (!vq->call_ctx.ctx || irq < 0)
+	if (!vq->call_ctx.ctx)
 		return;
 	vq->call_ctx.producer.token = vq->call_ctx.ctx;
 	struct vdpa_device *vdpa = v->vdpa;
 	const struct vdpa_config_ops *ops = vdpa->config;
 	u8 status, status_old;
-	int ret, nvqs = v->nvqs;
+	u32 nvqs = v->nvqs;
+	int ret;
 	u16 i;
 	if (copy_from_user(&status, statusp, sizeof(status)))
 	return 0;
 }
+static long vhost_vdpa_get_config_size(struct vhost_vdpa *v, u32 __user *argp)
+{
+	struct vdpa_device *vdpa = v->vdpa;
+	const struct vdpa_config_ops *ops = vdpa->config;
+	u32 size;
+	size = ops->get_config_size(vdpa);
+	if (copy_to_user(argp, &size, sizeof(size)))
+		return -EFAULT;
+	return 0;
+}
+static long vhost_vdpa_get_vqs_count(struct vhost_vdpa *v, u32 __user *argp)
+{
+	struct vdpa_device *vdpa = v->vdpa;
+	if (copy_to_user(argp, &vdpa->nvqs, sizeof(vdpa->nvqs)))
+		return -EFAULT;
+	return 0;
+}
 static long vhost_vdpa_vring_ioctl(struct vhost_vdpa *v, unsigned int cmd,
 				   void __user *argp)
 {
 	case VHOST_VDPA_GET_IOVA_RANGE:
 		r = vhost_vdpa_get_iova_range(v, argp);
 		break;
+	case VHOST_VDPA_GET_CONFIG_SIZE:
+		r = vhost_vdpa_get_config_size(v, argp);
+		break;
+	case VHOST_VDPA_GET_VQS_COUNT:
+		r = vhost_vdpa_get_vqs_count(v, argp);
+		break;
 	default:
 		r = vhost_dev_ioctl(&v->vdev, cmd, argp);
 		if (r == -ENOIOCTLCMD)
 	struct vhost_vdpa *v;
 	struct vhost_dev *dev;
 	struct vhost_virtqueue **vqs;
-	int nvqs, i, r, opened;
+	int r, opened;
+	u32 i, nvqs;
 	v = container_of(inode->i_cdev, struct vhost_vdpa, cdev);
 static void vhost_vdpa_clean_irq(struct vhost_vdpa *v)
 {
-	int i;
+	u32 i;
 	for (i = 0; i < v->nvqs; i++)
 		vhost_vdpa_unsetup_vq_irq(v, i);

drivers/vhost/vhost.c+2−1

@@ -2550,8 +2550,9 @@ bool vhost_enable_notify(struct vhost_dev dev, struct vhost_virtqueue vq) · 2549 unmodified lines
2550	&vq->avail->idx, r);	2550	&vq->avail->idx, r);
2551	return false;	2551	return false;
2552	}	2552	}
		2553	vq->avail_idx = vhost16_to_cpu(vq, avail_idx);
2553		2554
2554	return vhost16_to_cpu(vq, avail_idx) != vq->avail_idx;	2555	return vq->avail_idx != vq->last_avail_idx;
2555	}	2556	}
2556	EXPORT_SYMBOL_GPL(vhost_enable_notify);	2557	EXPORT_SYMBOL_GPL(vhost_enable_notify);
2557		2558

drivers/virtio/Kconfig+4−3

 config VIRTIO_MEM
 	tristate "Virtio mem driver"
-	depends on X86_64
+	depends on X86_64 || ARM64
 	depends on VIRTIO
 	depends on MEMORY_HOTPLUG
 	depends on MEMORY_HOTREMOVE
 	 This driver provides access to virtio-mem paravirtualized memory
 	 devices, allowing to hotplug and hotunplug memory.
-	 This driver was only tested under x86-64, but should theoretically
+	 This driver was only tested under x86-64 and arm64, but should
-	 work on all architectures that support memory hotplug and hotremove.
+	 theoretically work on all architectures that support memory hotplug
+	 and hotremove.
 	 If unsure, say M.

drivers/virtio/virtio.c+3−2

@@ -526,8 +526,9 @@ int virtio_device_restore(struct virtio_device *dev) · 525 unmodified lines
526	goto err;	526	goto err;
527	}	527	}
528		528
529	/* Finally, tell the device we're all set */	529	/* If restore didn't do it, mark device DRIVER_OK ourselves. */
530	virtio_add_status(dev, VIRTIO_CONFIG_S_DRIVER_OK);	530	if (!(dev->config->get_status(dev) & VIRTIO_CONFIG_S_DRIVER_OK))
		531	virtio_device_ready(dev);
531		532
532	virtio_config_enable(dev);	533	virtio_config_enable(dev);
533		534

drivers/virtio/virtio_pci_common.c+7−41

 		 "Force legacy mode for transitional virtio 1 devices");
 #endif
-/* disable irq handlers */
+/* wait for pending irq handlers */
-void vp_disable_cbs(struct virtio_device *vdev)
+void vp_synchronize_vectors(struct virtio_device *vdev)
 {
 	struct virtio_pci_device *vp_dev = to_vp_device(vdev);
 	int i;
-	if (vp_dev->intx_enabled) {
+	if (vp_dev->intx_enabled)
-		/*
-		 * The below synchronize() guarantees that any
-		 * interrupt for this line arriving after
-		 * synchronize_irq() has completed is guaranteed to see
-		 * intx_soft_enabled == false.
-		 */
-		WRITE_ONCE(vp_dev->intx_soft_enabled, false);
 		synchronize_irq(vp_dev->pci_dev->irq);
-	}
-	for (i = 0; i < vp_dev->msix_vectors; ++i)
-		disable_irq(pci_irq_vector(vp_dev->pci_dev, i));
-}
-/* enable irq handlers */
-void vp_enable_cbs(struct virtio_device *vdev)
-{
-	struct virtio_pci_device *vp_dev = to_vp_device(vdev);
-	int i;
-	if (vp_dev->intx_enabled) {
-		disable_irq(vp_dev->pci_dev->irq);
-		/*
-		 * The above disable_irq() provides TSO ordering and
-		 * as such promotes the below store to store-release.
-		 */
-		WRITE_ONCE(vp_dev->intx_soft_enabled, true);
-		enable_irq(vp_dev->pci_dev->irq);
-		return;
-	}
 	for (i = 0; i < vp_dev->msix_vectors; ++i)
-		enable_irq(pci_irq_vector(vp_dev->pci_dev, i));
+		synchronize_irq(pci_irq_vector(vp_dev->pci_dev, i));
 }
 /* the notify function used when creating a virt queue */
 	struct virtio_pci_device *vp_dev = opaque;
 	u8 isr;
-	if (!READ_ONCE(vp_dev->intx_soft_enabled))
-		return IRQ_NONE;
 	/* reading the ISR has the effect of also clearing it so it's very
 	 * important to save off the value. */
 	isr = ioread8(vp_dev->isr);
 	snprintf(vp_dev->msix_names[v], sizeof *vp_dev->msix_names,
 		 "%s-config", name);
 	err = request_irq(pci_irq_vector(vp_dev->pci_dev, v),
-			  vp_config_changed, IRQF_NO_AUTOEN,
+			  vp_config_changed, 0, vp_dev->msix_names[v],
-			  vp_dev->msix_names[v],
 			  vp_dev);
 	if (err)
 		goto error;
 		snprintf(vp_dev->msix_names[v], sizeof *vp_dev->msix_names,
 			 "%s-virtqueues", name);
 		err = request_irq(pci_irq_vector(vp_dev->pci_dev, v),
-				  vp_vring_interrupt, IRQF_NO_AUTOEN,
+				  vp_vring_interrupt, 0, vp_dev->msix_names[v],
-				  vp_dev->msix_names[v],
 				  vp_dev);
 		if (err)
 			goto error;
 			 "%s-%s",
 			 dev_name(&vp_dev->vdev.dev), names[i]);
 		err = request_irq(pci_irq_vector(vp_dev->pci_dev, msix_vec),
-				  vring_interrupt, IRQF_NO_AUTOEN,
+				  vring_interrupt, 0,
 				  vp_dev->msix_names[msix_vec],
 				  vqs[i]);
 		if (err)

drivers/virtio/virtio_pci_common.h+2−5

 	/* MSI-X support */
 	int msix_enabled;
 	int intx_enabled;
-	bool intx_soft_enabled;
 	cpumask_var_t *msix_affinity_masks;
 	/* Name strings for interrupts. This size should be enough,
 	 * and I'm too lazy to allocate each name separately. */
 	return container_of(vdev, struct virtio_pci_device, vdev);
 }
-/* disable irq handlers */
+/* wait for pending irq handlers */
-void vp_disable_cbs(struct virtio_device *vdev);
+void vp_synchronize_vectors(struct virtio_device *vdev);
-/* enable irq handlers */
-void vp_enable_cbs(struct virtio_device *vdev);
 /* the notify function used when creating a virt queue */
 bool vp_notify(struct virtqueue *vq);
 /* the config->del_vqs() implementation */

drivers/virtio/virtio_pci_legacy.c+2−3

 	/* Flush out the status write, and flush in device writes,
 	 * including MSi-X interrupts, if any. */
 	vp_legacy_get_status(&vp_dev->ldev);
-	/* Disable VQ/configuration callbacks. */
+	/* Flush pending VQ/configuration callbacks. */
-	vp_disable_cbs(vdev);
+	vp_synchronize_vectors(vdev);
 }
 static u16 vp_config_vector(struct virtio_pci_device *vp_dev, u16 vector)
 }
 static const struct virtio_config_ops virtio_pci_config_ops = {
-	.enable_cbs	= vp_enable_cbs,
 	.get		= vp_get,
 	.set		= vp_set,
 	.get_status	= vp_get_status,

drivers/virtio/virtio_pci_modern.c+11−7

 	 */
 	while (vp_modern_get_status(mdev))
 		msleep(1);
-	/* Disable VQ/configuration callbacks. */
+	/* Flush pending VQ/configuration callbacks. */
-	vp_disable_cbs(vdev);
+	vp_synchronize_vectors(vdev);
 }
 static u16 vp_config_vector(struct virtio_pci_device *vp_dev, u16 vector)
 	for (pos = pci_find_capability(dev, PCI_CAP_ID_VNDR); pos > 0;
 	     pos = pci_find_next_capability(dev, pos, PCI_CAP_ID_VNDR)) {
-		u8 type, cap_len, id;
+		u8 type, cap_len, id, res_bar;
 		u32 tmp32;
 		u64 res_offset, res_length;
 		if (id != required_id)
 			continue;
-		/* Type, and ID match, looks good */
 		pci_read_config_byte(dev, pos + offsetof(struct virtio_pci_cap,
-							 bar), bar);
+							 bar), &res_bar);
+		if (res_bar >= PCI_STD_NUM_BARS)
+			continue;
+		/* Type and ID match, and the BAR value isn't reserved.
+		 * Looks good.
+		 */
 		/* Read the lower 32bit of length and offset */
 		pci_read_config_dword(dev, pos + offsetof(struct virtio_pci_cap,
 						     length_hi), &tmp32);
 		res_length |= ((u64)tmp32) << 32;
+		*bar = res_bar;
 		*offset = res_offset;
 		*len = res_length;
 }
 static const struct virtio_config_ops virtio_pci_config_nodev_ops = {
-	.enable_cbs	= vp_enable_cbs,
 	.get		= NULL,
 	.set		= NULL,
 	.generation	= vp_generation,
 };
 static const struct virtio_config_ops virtio_pci_config_ops = {
-	.enable_cbs	= vp_enable_cbs,
 	.get		= vp_get,
 	.set		= vp_set,
 	.generation	= vp_generation,

drivers/virtio/virtio_pci_modern_dev.c+8−1

 	pci_read_config_dword(dev, off + offsetof(struct virtio_pci_cap, length),
 			      &length);
+	/* Check if the BAR may have changed since we requested the region. */
+	if (bar >= PCI_STD_NUM_BARS || !(mdev->modern_bars & (1 << bar))) {
+		dev_err(&dev->dev,
+			"virtio_pci: bar unexpectedly changed to %u\n", bar);
+		return NULL;
+	}
 	if (length <= start) {
 		dev_err(&dev->dev,
 			"virtio_pci: bad capability len %u (>%u expected)\n",
 				     &bar);
 		/* Ignore structures with reserved BAR values */
-		if (bar > 0x5)
+		if (bar >= PCI_STD_NUM_BARS)
 			continue;
 		if (type == cfg_type) {

drivers/virtio/virtio_ring.c+18−35

 	flags = virtio16_to_cpu(vq->vq.vdev, desc->flags);
-	if (flags & VRING_DESC_F_INDIRECT) {
+	dma_unmap_page(vring_dma_dev(vq),
-		dma_unmap_single(vring_dma_dev(vq),
+		       virtio64_to_cpu(vq->vq.vdev, desc->addr),
-				 virtio64_to_cpu(vq->vq.vdev, desc->addr),
+		       virtio32_to_cpu(vq->vq.vdev, desc->len),
-				 virtio32_to_cpu(vq->vq.vdev, desc->len),
+		       (flags & VRING_DESC_F_WRITE) ?
-				 (flags & VRING_DESC_F_WRITE) ?
+		       DMA_FROM_DEVICE : DMA_TO_DEVICE);
-				 DMA_FROM_DEVICE : DMA_TO_DEVICE);
-	} else {
-		dma_unmap_page(vring_dma_dev(vq),
-			       virtio64_to_cpu(vq->vq.vdev, desc->addr),
-			       virtio32_to_cpu(vq->vq.vdev, desc->len),
-			       (flags & VRING_DESC_F_WRITE) ?
-			       DMA_FROM_DEVICE : DMA_TO_DEVICE);
-	}
 }
 static unsigned int vring_unmap_one_split(const struct vring_virtqueue *vq,
  * Packed ring specific functions - *_packed().
  */
-static void vring_unmap_state_packed(const struct vring_virtqueue *vq,
+static void vring_unmap_extra_packed(const struct vring_virtqueue *vq,
-				     struct vring_desc_extra *state)
+				     struct vring_desc_extra *extra)
 {
 	u16 flags;
 	if (!vq->use_dma_api)
 		return;
-	flags = state->flags;
+	flags = extra->flags;
 	if (flags & VRING_DESC_F_INDIRECT) {
 		dma_unmap_single(vring_dma_dev(vq),
-				 state->addr, state->len,
+				 extra->addr, extra->len,
 				 (flags & VRING_DESC_F_WRITE) ?
 				 DMA_FROM_DEVICE : DMA_TO_DEVICE);
 	} else {
 		dma_unmap_page(vring_dma_dev(vq),
-			       state->addr, state->len,
+			       extra->addr, extra->len,
 			       (flags & VRING_DESC_F_WRITE) ?
 			       DMA_FROM_DEVICE : DMA_TO_DEVICE);
 	}
 	flags = le16_to_cpu(desc->flags);
-	if (flags & VRING_DESC_F_INDIRECT) {
+	dma_unmap_page(vring_dma_dev(vq),
-		dma_unmap_single(vring_dma_dev(vq),
+		       le64_to_cpu(desc->addr),
-				 le64_to_cpu(desc->addr),
+		       le32_to_cpu(desc->len),
-				 le32_to_cpu(desc->len),
+		       (flags & VRING_DESC_F_WRITE) ?
-				 (flags & VRING_DESC_F_WRITE) ?
+		       DMA_FROM_DEVICE : DMA_TO_DEVICE);
-				 DMA_FROM_DEVICE : DMA_TO_DEVICE);
-	} else {
-		dma_unmap_page(vring_dma_dev(vq),
-			       le64_to_cpu(desc->addr),
-			       le32_to_cpu(desc->len),
-			       (flags & VRING_DESC_F_WRITE) ?
-			       DMA_FROM_DEVICE : DMA_TO_DEVICE);
-	}
 }
 static struct vring_packed_desc *alloc_indirect_packed(unsigned int total_sg,
 	for (n = 0; n < total_sg; n++) {
 		if (i == err_idx)
 			break;
-		vring_unmap_state_packed(vq,
+		vring_unmap_extra_packed(vq, &vq->packed.desc_extra[curr]);
-					 &vq->packed.desc_extra[curr]);
 		curr = vq->packed.desc_extra[curr].next;
 		i++;
 		if (i >= vq->packed.vring.num)
 	if (unlikely(vq->use_dma_api)) {
 		curr = id;
 		for (i = 0; i < state->num; i++) {
-			vring_unmap_state_packed(vq,
+			vring_unmap_extra_packed(vq,
-				&vq->packed.desc_extra[curr]);
+						 &vq->packed.desc_extra[curr]);
 			curr = vq->packed.desc_extra[curr].next;
 		}
 	}

include/linux/balloon_compaction.h+0−22

@@ -80,12 +80,6 @@ static inline void balloon_devinfo_init(struct balloon_dev_info *balloon) · 79 unmodified lines
80		80
81	#ifdef CONFIG_BALLOON_COMPACTION	81	#ifdef CONFIG_BALLOON_COMPACTION
82	extern const struct address_space_operations balloon_aops;	82	extern const struct address_space_operations balloon_aops;
83	extern bool balloon_page_isolate(struct page *page,
84	isolate_mode_t mode);
85	extern void balloon_page_putback(struct page *page);
86	extern int balloon_page_migrate(struct address_space *mapping,
87	struct page *newpage,
88	struct page *page, enum migrate_mode mode);
89		83
90	/*	84	/*
91	* balloon_page_insert - insert a page into the balloon's page list and make	85	* balloon_page_insert - insert a page into the balloon's page list and make
@@ -155,22 +149,6 @@ static inline void balloon_page_delete(struct page *page) · 63 unmodified lines
155	list_del(&page->lru);	149	list_del(&page->lru);
156	}	150	}
157		151
158	static inline bool balloon_page_isolate(struct page *page)
159	{
160	return false;
161	}
162
163	static inline void balloon_page_putback(struct page *page)
164	{
165	return;
166	}
167
168	static inline int balloon_page_migrate(struct page *newpage,
169	struct page *page, enum migrate_mode mode)
170	{
171	return 0;
172	}
173
174	static inline gfp_t balloon_mapping_gfp_mask(void)	152	static inline gfp_t balloon_mapping_gfp_mask(void)
175	{	153	{
176	return GFP_HIGHUSER;	154	return GFP_HIGHUSER;

include/linux/vdpa.h+5−4

 	unsigned int index;
 	bool features_valid;
 	bool use_va;
-	int nvqs;
+	u32 nvqs;
 	struct vdpa_mgmt_dev *mdev;
 };
  * @reset:			Reset device
  *				@vdev: vdpa device
  *				Returns integer: success (0) or error (< 0)
- * @get_config_size:		Get the size of the configuration space
+ * @get_config_size:		Get the size of the configuration space includes
+ *				fields that are conditional on feature bits.
  *				@vdev: vdpa device
  *				Returns size_t: configuration size
  * @get_config:			Read from device specific configuration space
 				       dev_struct, member)), name, use_va), \
 				       dev_struct, member)
-int vdpa_register_device(struct vdpa_device *vdev, int nvqs);
+int vdpa_register_device(struct vdpa_device *vdev, u32 nvqs);
 void vdpa_unregister_device(struct vdpa_device *vdev);
-int _vdpa_register_device(struct vdpa_device *vdev, int nvqs);
+int _vdpa_register_device(struct vdpa_device *vdev, u32 nvqs);
 void _vdpa_unregister_device(struct vdpa_device *vdev);
 /**

include/uapi/linux/vhost.h+7−0

@@ -150,4 +150,11 @@ · 149 unmodified lines
150	/* Get the valid iova range */	150	/* Get the valid iova range */
151	#define VHOST_VDPA_GET_IOVA_RANGE _IOR(VHOST_VIRTIO, 0x78, \	151	#define VHOST_VDPA_GET_IOVA_RANGE _IOR(VHOST_VIRTIO, 0x78, \
152	struct vhost_vdpa_iova_range)	152	struct vhost_vdpa_iova_range)
		153
		154	/* Get the config size */
		155	#define VHOST_VDPA_GET_CONFIG_SIZE _IOR(VHOST_VIRTIO, 0x79, __u32)
		156
		157	/* Get the count of all virtqueues */
		158	#define VHOST_VDPA_GET_VQS_COUNT _IOR(VHOST_VIRTIO, 0x80, __u32)
		159
153	#endif	160	#endif

include/uapi/linux/virtio_config.h+6−0

 /* This feature indicates support for the packed virtqueue layout. */
 #define VIRTIO_F_RING_PACKED		34
+/*
+ * Inorder feature indicates that all buffers are used by the device
+ * in the same order in which they have been made available.
+ */
+#define VIRTIO_F_IN_ORDER		35
 /*
  * This feature indicates that memory accesses by the driver and the
  * device are ordered in a way described by the platform.

include/uapi/linux/virtio_crypto.h+81−1

@@ -37,6 +37,7 @@ · 36 unmodified lines
37	#define VIRTIO_CRYPTO_SERVICE_HASH 1	37	#define VIRTIO_CRYPTO_SERVICE_HASH 1
38	#define VIRTIO_CRYPTO_SERVICE_MAC 2	38	#define VIRTIO_CRYPTO_SERVICE_MAC 2
39	#define VIRTIO_CRYPTO_SERVICE_AEAD 3	39	#define VIRTIO_CRYPTO_SERVICE_AEAD 3
		40	#define VIRTIO_CRYPTO_SERVICE_AKCIPHER 4
40		41
41	#define VIRTIO_CRYPTO_OPCODE(service, op) (((service) << 8) \| (op))	42	#define VIRTIO_CRYPTO_OPCODE(service, op) (((service) << 8) \| (op))
42		43
@@ -57,6 +58,10 @@ struct virtio_crypto_ctrl_header { · 14 unmodified lines
57	VIRTIO_CRYPTO_OPCODE(VIRTIO_CRYPTO_SERVICE_AEAD, 0x02)	58	VIRTIO_CRYPTO_OPCODE(VIRTIO_CRYPTO_SERVICE_AEAD, 0x02)
58	#define VIRTIO_CRYPTO_AEAD_DESTROY_SESSION \	59	#define VIRTIO_CRYPTO_AEAD_DESTROY_SESSION \
59	VIRTIO_CRYPTO_OPCODE(VIRTIO_CRYPTO_SERVICE_AEAD, 0x03)	60	VIRTIO_CRYPTO_OPCODE(VIRTIO_CRYPTO_SERVICE_AEAD, 0x03)
		61	#define VIRTIO_CRYPTO_AKCIPHER_CREATE_SESSION \
		62	VIRTIO_CRYPTO_OPCODE(VIRTIO_CRYPTO_SERVICE_AKCIPHER, 0x04)
		63	#define VIRTIO_CRYPTO_AKCIPHER_DESTROY_SESSION \
		64	VIRTIO_CRYPTO_OPCODE(VIRTIO_CRYPTO_SERVICE_AKCIPHER, 0x05)
60	__le32 opcode;	65	__le32 opcode;
61	__le32 algo;	66	__le32 algo;
62	__le32 flag;	67	__le32 flag;
@@ -180,6 +185,58 @@ struct virtio_crypto_aead_create_session_req { · 117 unmodified lines
180	__u8 padding[32];	185	__u8 padding[32];
181	};	186	};
182		187
		188	struct virtio_crypto_rsa_session_para {
		189	#define VIRTIO_CRYPTO_RSA_RAW_PADDING 0
		190	#define VIRTIO_CRYPTO_RSA_PKCS1_PADDING 1
		191	__le32 padding_algo;
		192
		193	#define VIRTIO_CRYPTO_RSA_NO_HASH 0
		194	#define VIRTIO_CRYPTO_RSA_MD2 1
		195	#define VIRTIO_CRYPTO_RSA_MD3 2
		196	#define VIRTIO_CRYPTO_RSA_MD4 3
		197	#define VIRTIO_CRYPTO_RSA_MD5 4
		198	#define VIRTIO_CRYPTO_RSA_SHA1 5
		199	#define VIRTIO_CRYPTO_RSA_SHA256 6
		200	#define VIRTIO_CRYPTO_RSA_SHA384 7
		201	#define VIRTIO_CRYPTO_RSA_SHA512 8
		202	#define VIRTIO_CRYPTO_RSA_SHA224 9
		203	__le32 hash_algo;
		204	};
		205
		206	struct virtio_crypto_ecdsa_session_para {
		207	#define VIRTIO_CRYPTO_CURVE_UNKNOWN 0
		208	#define VIRTIO_CRYPTO_CURVE_NIST_P192 1
		209	#define VIRTIO_CRYPTO_CURVE_NIST_P224 2
		210	#define VIRTIO_CRYPTO_CURVE_NIST_P256 3
		211	#define VIRTIO_CRYPTO_CURVE_NIST_P384 4
		212	#define VIRTIO_CRYPTO_CURVE_NIST_P521 5
		213	__le32 curve_id;
		214	__le32 padding;
		215	};
		216
		217	struct virtio_crypto_akcipher_session_para {
		218	#define VIRTIO_CRYPTO_NO_AKCIPHER 0
		219	#define VIRTIO_CRYPTO_AKCIPHER_RSA 1
		220	#define VIRTIO_CRYPTO_AKCIPHER_DSA 2
		221	#define VIRTIO_CRYPTO_AKCIPHER_ECDSA 3
		222	__le32 algo;
		223
		224	#define VIRTIO_CRYPTO_AKCIPHER_KEY_TYPE_PUBLIC 1
		225	#define VIRTIO_CRYPTO_AKCIPHER_KEY_TYPE_PRIVATE 2
		226	__le32 keytype;
		227	__le32 keylen;
		228
		229	union {
		230	struct virtio_crypto_rsa_session_para rsa;
		231	struct virtio_crypto_ecdsa_session_para ecdsa;
		232	} u;
		233	};
		234
		235	struct virtio_crypto_akcipher_create_session_req {
		236	struct virtio_crypto_akcipher_session_para para;
		237	__u8 padding[36];
		238	};
		239
183	struct virtio_crypto_alg_chain_session_para {	240	struct virtio_crypto_alg_chain_session_para {
184	#define VIRTIO_CRYPTO_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER 1	241	#define VIRTIO_CRYPTO_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER 1
185	#define VIRTIO_CRYPTO_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH 2	242	#define VIRTIO_CRYPTO_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH 2
@@ -247,6 +304,8 @@ struct virtio_crypto_op_ctrl_req { · 61 unmodified lines
247	mac_create_session;	304	mac_create_session;
248	struct virtio_crypto_aead_create_session_req	305	struct virtio_crypto_aead_create_session_req
249	aead_create_session;	306	aead_create_session;
		307	struct virtio_crypto_akcipher_create_session_req
		308	akcipher_create_session;
250	struct virtio_crypto_destroy_session_req	309	struct virtio_crypto_destroy_session_req
251	destroy_session;	310	destroy_session;
252	__u8 padding[56];	311	__u8 padding[56];
@@ -266,6 +325,14 @@ struct virtio_crypto_op_header { · 13 unmodified lines
266	VIRTIO_CRYPTO_OPCODE(VIRTIO_CRYPTO_SERVICE_AEAD, 0x00)	325	VIRTIO_CRYPTO_OPCODE(VIRTIO_CRYPTO_SERVICE_AEAD, 0x00)
267	#define VIRTIO_CRYPTO_AEAD_DECRYPT \	326	#define VIRTIO_CRYPTO_AEAD_DECRYPT \
268	VIRTIO_CRYPTO_OPCODE(VIRTIO_CRYPTO_SERVICE_AEAD, 0x01)	327	VIRTIO_CRYPTO_OPCODE(VIRTIO_CRYPTO_SERVICE_AEAD, 0x01)
		328	#define VIRTIO_CRYPTO_AKCIPHER_ENCRYPT \
		329	VIRTIO_CRYPTO_OPCODE(VIRTIO_CRYPTO_SERVICE_AKCIPHER, 0x00)
		330	#define VIRTIO_CRYPTO_AKCIPHER_DECRYPT \
		331	VIRTIO_CRYPTO_OPCODE(VIRTIO_CRYPTO_SERVICE_AKCIPHER, 0x01)
		332	#define VIRTIO_CRYPTO_AKCIPHER_SIGN \
		333	VIRTIO_CRYPTO_OPCODE(VIRTIO_CRYPTO_SERVICE_AKCIPHER, 0x02)
		334	#define VIRTIO_CRYPTO_AKCIPHER_VERIFY \
		335	VIRTIO_CRYPTO_OPCODE(VIRTIO_CRYPTO_SERVICE_AKCIPHER, 0x03)
269	__le32 opcode;	336	__le32 opcode;
270	/* algo should be service-specific algorithms */	337	/* algo should be service-specific algorithms */
271	__le32 algo;	338	__le32 algo;
@@ -390,6 +457,16 @@ struct virtio_crypto_aead_data_req { · 118 unmodified lines
390	__u8 padding[32];	457	__u8 padding[32];
391	};	458	};
392		459
		460	struct virtio_crypto_akcipher_para {
		461	__le32 src_data_len;
		462	__le32 dst_data_len;
		463	};
		464
		465	struct virtio_crypto_akcipher_data_req {
		466	struct virtio_crypto_akcipher_para para;
		467	__u8 padding[40];
		468	};
		469
393	/* The request of the data virtqueue's packet */	470	/* The request of the data virtqueue's packet */
394	struct virtio_crypto_op_data_req {	471	struct virtio_crypto_op_data_req {
395	struct virtio_crypto_op_header header;	472	struct virtio_crypto_op_header header;
@@ -399,6 +476,7 @@ struct virtio_crypto_op_data_req { · 3 unmodified lines
399	struct virtio_crypto_hash_data_req hash_req;	476	struct virtio_crypto_hash_data_req hash_req;
400	struct virtio_crypto_mac_data_req mac_req;	477	struct virtio_crypto_mac_data_req mac_req;
401	struct virtio_crypto_aead_data_req aead_req;	478	struct virtio_crypto_aead_data_req aead_req;
		479	struct virtio_crypto_akcipher_data_req akcipher_req;
402	__u8 padding[48];	480	__u8 padding[48];
403	} u;	481	} u;
404	};	482	};
@@ -408,6 +486,8 @@ struct virtio_crypto_op_data_req { · 3 unmodified lines
408	#define VIRTIO_CRYPTO_BADMSG 2	486	#define VIRTIO_CRYPTO_BADMSG 2
409	#define VIRTIO_CRYPTO_NOTSUPP 3	487	#define VIRTIO_CRYPTO_NOTSUPP 3
410	#define VIRTIO_CRYPTO_INVSESS 4 /* Invalid session id */	488	#define VIRTIO_CRYPTO_INVSESS 4 /* Invalid session id */
		489	#define VIRTIO_CRYPTO_NOSPC 5 /* no free session ID */
		490	#define VIRTIO_CRYPTO_KEY_REJECTED 6 /* Signature verification failed */
411		491
412	/* The accelerator hardware is ready */	492	/* The accelerator hardware is ready */
413	#define VIRTIO_CRYPTO_S_HW_READY (1 << 0)	493	#define VIRTIO_CRYPTO_S_HW_READY (1 << 0)
@@ -438,7 +518,7 @@ struct virtio_crypto_config { · 24 unmodified lines
438	__le32 max_cipher_key_len;	518	__le32 max_cipher_key_len;
439	/* Maximum length of authenticated key */	519	/* Maximum length of authenticated key */
440	__le32 max_auth_key_len;	520	__le32 max_auth_key_len;
441	__le32 reserve;	521	__le32 akcipher_algo;
442	/* Maximum size of each crypto request's content */	522	/* Maximum size of each crypto request's content */
443	__le64 max_size;	523	__le64 max_size;
444	};	524	};

mm/balloon_compaction.c+3−3

@@ -203,7 +203,7 @@ EXPORT_SYMBOL_GPL(balloon_page_dequeue); · 202 unmodified lines
203		203
204	#ifdef CONFIG_BALLOON_COMPACTION	204	#ifdef CONFIG_BALLOON_COMPACTION
205		205
206	bool balloon_page_isolate(struct page *page, isolate_mode_t mode)	206	static bool balloon_page_isolate(struct page *page, isolate_mode_t mode)
207		207
208	{	208	{
209	struct balloon_dev_info *b_dev_info = balloon_page_device(page);	209	struct balloon_dev_info *b_dev_info = balloon_page_device(page);
@@ -217,7 +217,7 @@ bool balloon_page_isolate(struct page *page, isolate_mode_t mode) · 7 unmodified lines
217	return true;	217	return true;
218	}	218	}
219		219
220	void balloon_page_putback(struct page *page)	220	static void balloon_page_putback(struct page *page)
221	{	221	{
222	struct balloon_dev_info *b_dev_info = balloon_page_device(page);	222	struct balloon_dev_info *b_dev_info = balloon_page_device(page);
223	unsigned long flags;	223	unsigned long flags;
@@ -230,7 +230,7 @@ void balloon_page_putback(struct page *page) · 6 unmodified lines
230		230
231		231
232	/* move_to_new_page() counterpart for a ballooned page */	232	/* move_to_new_page() counterpart for a ballooned page */
233	int balloon_page_migrate(struct address_space *mapping,	233	static int balloon_page_migrate(struct address_space *mapping,
234	struct page newpage, struct page page,	234	struct page newpage, struct page page,
235	enum migrate_mode mode)	235	enum migrate_mode mode)
236	{	236	{

tools/virtio/Makefile+2−1

@@ -5,7 +5,8 @@ virtio_test: virtio_ring.o virtio_test.o · 4 unmodified lines
5	vringh_test: vringh_test.o vringh.o virtio_ring.o	5	vringh_test: vringh_test.o vringh.o virtio_ring.o
6		6
7	CFLAGS += -g -O2 -Werror -Wno-maybe-uninitialized -Wall -I. -I../include/ -I ../../usr/include/ -Wno-pointer-sign -fno-strict-overflow -fno-strict-aliasing -fno-common -MMD -U_FORTIFY_SOURCE -include ../../include/linux/kconfig.h	7	CFLAGS += -g -O2 -Werror -Wno-maybe-uninitialized -Wall -I. -I../include/ -I ../../usr/include/ -Wno-pointer-sign -fno-strict-overflow -fno-strict-aliasing -fno-common -MMD -U_FORTIFY_SOURCE -include ../../include/linux/kconfig.h
8	LDFLAGS += -lpthread	8	CFLAGS += -pthread
		9	LDFLAGS += -pthread
9	vpath %.c ../../drivers/virtio ../../drivers/vhost	10	vpath %.c ../../drivers/virtio ../../drivers/vhost
10	mod:	11	mod:
11	${MAKE} -C `pwd`/../.. M=`pwd`/vhost_test V=${V}	12	${MAKE} -C `pwd`/../.. M=`pwd`/vhost_test V=${V}

tools/virtio/linux/dma-mapping.h+2−2

@@ -26,8 +26,8 @@ enum dma_data_direction { · 25 unmodified lines
26	#define dma_map_single(d, p, s, dir) (virt_to_phys(p))	26	#define dma_map_single(d, p, s, dir) (virt_to_phys(p))
27	#define dma_mapping_error(...) (0)	27	#define dma_mapping_error(...) (0)
28		28
29	#define dma_unmap_single(...) do { } while (0)	29	#define dma_unmap_single(d, a, s, r) do { (void)(d); (void)(a); (void)(s); (void)(r); } while (0)
30	#define dma_unmap_page(...) do { } while (0)	30	#define dma_unmap_page(d, a, s, r) do { (void)(d); (void)(a); (void)(s); (void)(r); } while (0)
31		31
32	#define dma_max_mapping_size(...) SIZE_MAX	32	#define dma_max_mapping_size(...) SIZE_MAX
33		33