lael-labs/linux

Documentation/devicetree/bindings/virtio/mmio.yaml+4−0

     description: Required for devices making accesses thru an IOMMU.
     maxItems: 1
+  wakeup-source:
+    type: boolean
+    description: Required for setting irq of a virtio_mmio device as wakeup source.
 required:
   - compatible
   - reg

arch/um/drivers/virtio_uml.c+2−1

 		goto error_create;
 	}
 	vq->priv = info;
+	vq->num_max = num;
 	num = virtqueue_get_vring_size(vq);
 	if (vu_dev->protocol_features &
 static int vu_find_vqs(struct virtio_device *vdev, unsigned nvqs,
 		       struct virtqueue *vqs[], vq_callback_t *callbacks[],
-		       const char * const names[], const bool *ctx,
+		       const char * const names[], u32 sizes[], const bool *ctx,
 		       struct irq_affinity *desc)
 {
 	struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev);

drivers/block/virtio_blk.c+10−14

 	}
 }
+static inline struct virtio_blk_vq *get_virtio_blk_vq(struct blk_mq_hw_ctx *hctx)
+{
+	struct virtio_blk *vblk = hctx->queue->queuedata;
+	struct virtio_blk_vq *vq = &vblk->vqs[hctx->queue_num];
+	return vq;
+}
 static int virtblk_add_req(struct virtqueue *vq, struct virtblk_req *vbr)
 {
 	struct scatterlist hdr, status, *sgs[3];
 	struct request *requeue_list = NULL;
 	rq_list_for_each_safe(rqlist, req, next) {
-		struct virtio_blk_vq *vq = req->mq_hctx->driver_data;
+		struct virtio_blk_vq *vq = get_virtio_blk_vq(req->mq_hctx);
 		bool kick;
 		if (!virtblk_prep_rq_batch(req)) {
 static int virtblk_poll(struct blk_mq_hw_ctx *hctx, struct io_comp_batch *iob)
 {
 	struct virtio_blk *vblk = hctx->queue->queuedata;
-	struct virtio_blk_vq *vq = hctx->driver_data;
+	struct virtio_blk_vq *vq = get_virtio_blk_vq(hctx);
 	struct virtblk_req *vbr;
 	unsigned long flags;
 	unsigned int len;
 	return found;
 }
-static int virtblk_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
-			  unsigned int hctx_idx)
-{
-	struct virtio_blk *vblk = data;
-	struct virtio_blk_vq *vq = &vblk->vqs[hctx_idx];
-	WARN_ON(vblk->tag_set.tags[hctx_idx] != hctx->tags);
-	hctx->driver_data = vq;
-	return 0;
-}
 static const struct blk_mq_ops virtio_mq_ops = {
 	.queue_rq	= virtio_queue_rq,
 	.queue_rqs	= virtio_queue_rqs,
 	.commit_rqs	= virtio_commit_rqs,
-	.init_hctx	= virtblk_init_hctx,
 	.complete	= virtblk_request_done,
 	.map_queues	= virtblk_map_queues,
 	.poll		= virtblk_poll,

drivers/net/virtio_net.c+286−39

@@ -135,6 +135,9 @@ struct send_queue { · 134 unmodified lines
135	struct virtnet_sq_stats stats;	135	struct virtnet_sq_stats stats;
136		136
137	struct napi_struct napi;	137	struct napi_struct napi;
		138
		139	/* Record whether sq is in reset state. */
		140	bool reset;
138	};	141	};
139		142
140	/* Internal representation of a receive virtqueue */	143	/* Internal representation of a receive virtqueue */
@@ -267,6 +270,12 @@ struct virtnet_info { · 126 unmodified lines
267	u8 duplex;	270	u8 duplex;
268	u32 speed;	271	u32 speed;
269		272
		273	/* Interrupt coalescing settings */
		274	u32 tx_usecs;
		275	u32 rx_usecs;
		276	u32 tx_max_packets;
		277	u32 rx_max_packets;
		278
270	unsigned long guest_offloads;	279	unsigned long guest_offloads;
271	unsigned long guest_offloads_capable;	280	unsigned long guest_offloads_capable;
272		281
@@ -284,6 +293,9 @@ struct padded_vnet_hdr { · 11 unmodified lines
284	char padding[12];	293	char padding[12];
285	};	294	};
286		295
		296	static void virtnet_rq_free_unused_buf(struct virtqueue vq, void buf);
		297	static void virtnet_sq_free_unused_buf(struct virtqueue vq, void buf);
		298
287	static bool is_xdp_frame(void *ptr)	299	static bool is_xdp_frame(void *ptr)
288	{	300	{
289	return (unsigned long)ptr & VIRTIO_XDP_FLAG;	301	return (unsigned long)ptr & VIRTIO_XDP_FLAG;
@@ -1628,6 +1640,11 @@ static void virtnet_poll_cleantx(struct receive_queue *rq) · 1338 unmodified lines
1628	return;	1640	return;
1629		1641
1630	if (__netif_tx_trylock(txq)) {	1642	if (__netif_tx_trylock(txq)) {
		1643	if (sq->reset) {
		1644	__netif_tx_unlock(txq);
		1645	return;
		1646	}
		1647
1631	do {	1648	do {
1632	virtqueue_disable_cb(sq->vq);	1649	virtqueue_disable_cb(sq->vq);
1633	free_old_xmit_skbs(sq, true);	1650	free_old_xmit_skbs(sq, true);
@@ -1875,6 +1892,70 @@ static netdev_tx_t start_xmit(struct sk_buff skb, struct net_device dev) · 241 unmodified lines
1875	return NETDEV_TX_OK;	1892	return NETDEV_TX_OK;
1876	}	1893	}
1877		1894
		1895	static int virtnet_rx_resize(struct virtnet_info *vi,
		1896	struct receive_queue *rq, u32 ring_num)
		1897	{
		1898	bool running = netif_running(vi->dev);
		1899	int err, qindex;
		1900
		1901	qindex = rq - vi->rq;
		1902
		1903	if (running)
		1904	napi_disable(&rq->napi);
		1905
		1906	err = virtqueue_resize(rq->vq, ring_num, virtnet_rq_free_unused_buf);
		1907	if (err)
		1908	netdev_err(vi->dev, "resize rx fail: rx queue index: %d err: %d\n", qindex, err);
		1909
		1910	if (!try_fill_recv(vi, rq, GFP_KERNEL))
		1911	schedule_delayed_work(&vi->refill, 0);
		1912
		1913	if (running)
		1914	virtnet_napi_enable(rq->vq, &rq->napi);
		1915	return err;
		1916	}
		1917
		1918	static int virtnet_tx_resize(struct virtnet_info *vi,
		1919	struct send_queue *sq, u32 ring_num)
		1920	{
		1921	bool running = netif_running(vi->dev);
		1922	struct netdev_queue *txq;
		1923	int err, qindex;
		1924
		1925	qindex = sq - vi->sq;
		1926
		1927	if (running)
		1928	virtnet_napi_tx_disable(&sq->napi);
		1929
		1930	txq = netdev_get_tx_queue(vi->dev, qindex);
		1931
		1932	/* 1. wait all ximt complete
		1933	* 2. fix the race of netif_stop_subqueue() vs netif_start_subqueue()
		1934	*/
		1935	__netif_tx_lock_bh(txq);
		1936
		1937	/* Prevent rx poll from accessing sq. */
		1938	sq->reset = true;
		1939
		1940	/* Prevent the upper layer from trying to send packets. */
		1941	netif_stop_subqueue(vi->dev, qindex);
		1942
		1943	__netif_tx_unlock_bh(txq);
		1944
		1945	err = virtqueue_resize(sq->vq, ring_num, virtnet_sq_free_unused_buf);
		1946	if (err)
		1947	netdev_err(vi->dev, "resize tx fail: tx queue index: %d err: %d\n", qindex, err);
		1948
		1949	__netif_tx_lock_bh(txq);
		1950	sq->reset = false;
		1951	netif_tx_wake_queue(txq);
		1952	__netif_tx_unlock_bh(txq);
		1953
		1954	if (running)
		1955	virtnet_napi_tx_enable(vi, sq->vq, &sq->napi);
		1956	return err;
		1957	}
		1958
1878	/*	1959	/*
1879	* Send command via the control virtqueue and check status. Commands	1960	* Send command via the control virtqueue and check status. Commands
1880	* supported by the hypervisor, as indicated by feature bits, should	1961	* supported by the hypervisor, as indicated by feature bits, should
@@ -2285,10 +2366,57 @@ static void virtnet_get_ringparam(struct net_device *dev, · 404 unmodified lines
2285	{	2366	{
2286	struct virtnet_info *vi = netdev_priv(dev);	2367	struct virtnet_info *vi = netdev_priv(dev);
2287		2368
2288	ring->rx_max_pending = virtqueue_get_vring_size(vi->rq[0].vq);	2369	ring->rx_max_pending = vi->rq[0].vq->num_max;
2289	ring->tx_max_pending = virtqueue_get_vring_size(vi->sq[0].vq);	2370	ring->tx_max_pending = vi->sq[0].vq->num_max;
2290	ring->rx_pending = ring->rx_max_pending;	2371	ring->rx_pending = virtqueue_get_vring_size(vi->rq[0].vq);
2291	ring->tx_pending = ring->tx_max_pending;	2372	ring->tx_pending = virtqueue_get_vring_size(vi->sq[0].vq);
		2373	}
		2374
		2375	static int virtnet_set_ringparam(struct net_device *dev,
		2376	struct ethtool_ringparam *ring,
		2377	struct kernel_ethtool_ringparam *kernel_ring,
		2378	struct netlink_ext_ack *extack)
		2379	{
		2380	struct virtnet_info *vi = netdev_priv(dev);
		2381	u32 rx_pending, tx_pending;
		2382	struct receive_queue *rq;
		2383	struct send_queue *sq;
		2384	int i, err;
		2385
		2386	if (ring->rx_mini_pending \|\| ring->rx_jumbo_pending)
		2387	return -EINVAL;
		2388
		2389	rx_pending = virtqueue_get_vring_size(vi->rq[0].vq);
		2390	tx_pending = virtqueue_get_vring_size(vi->sq[0].vq);
		2391
		2392	if (ring->rx_pending == rx_pending &&
		2393	ring->tx_pending == tx_pending)
		2394	return 0;
		2395
		2396	if (ring->rx_pending > vi->rq[0].vq->num_max)
		2397	return -EINVAL;
		2398
		2399	if (ring->tx_pending > vi->sq[0].vq->num_max)
		2400	return -EINVAL;
		2401
		2402	for (i = 0; i < vi->max_queue_pairs; i++) {
		2403	rq = vi->rq + i;
		2404	sq = vi->sq + i;
		2405
		2406	if (ring->tx_pending != tx_pending) {
		2407	err = virtnet_tx_resize(vi, sq, ring->tx_pending);
		2408	if (err)
		2409	return err;
		2410	}
		2411
		2412	if (ring->rx_pending != rx_pending) {
		2413	err = virtnet_rx_resize(vi, rq, ring->rx_pending);
		2414	if (err)
		2415	return err;
		2416	}
		2417	}
		2418
		2419	return 0;
2292	}	2420	}
2293		2421
2294	static bool virtnet_commit_rss_command(struct virtnet_info *vi)	2422	static bool virtnet_commit_rss_command(struct virtnet_info *vi)
@@ -2618,27 +2746,89 @@ static int virtnet_get_link_ksettings(struct net_device *dev, · 323 unmodified lines
2618	return 0;	2746	return 0;
2619	}	2747	}
2620		2748
		2749	static int virtnet_send_notf_coal_cmds(struct virtnet_info *vi,
		2750	struct ethtool_coalesce *ec)
		2751	{
		2752	struct scatterlist sgs_tx, sgs_rx;
		2753	struct virtio_net_ctrl_coal_tx coal_tx;
		2754	struct virtio_net_ctrl_coal_rx coal_rx;
		2755
		2756	coal_tx.tx_usecs = cpu_to_le32(ec->tx_coalesce_usecs);
		2757	coal_tx.tx_max_packets = cpu_to_le32(ec->tx_max_coalesced_frames);
		2758	sg_init_one(&sgs_tx, &coal_tx, sizeof(coal_tx));
		2759
		2760	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_NOTF_COAL,
		2761	VIRTIO_NET_CTRL_NOTF_COAL_TX_SET,
		2762	&sgs_tx))
		2763	return -EINVAL;
		2764
		2765	/* Save parameters */
		2766	vi->tx_usecs = ec->tx_coalesce_usecs;
		2767	vi->tx_max_packets = ec->tx_max_coalesced_frames;
		2768
		2769	coal_rx.rx_usecs = cpu_to_le32(ec->rx_coalesce_usecs);
		2770	coal_rx.rx_max_packets = cpu_to_le32(ec->rx_max_coalesced_frames);
		2771	sg_init_one(&sgs_rx, &coal_rx, sizeof(coal_rx));
		2772
		2773	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_NOTF_COAL,
		2774	VIRTIO_NET_CTRL_NOTF_COAL_RX_SET,
		2775	&sgs_rx))
		2776	return -EINVAL;
		2777
		2778	/* Save parameters */
		2779	vi->rx_usecs = ec->rx_coalesce_usecs;
		2780	vi->rx_max_packets = ec->rx_max_coalesced_frames;
		2781
		2782	return 0;
		2783	}
		2784
		2785	static int virtnet_coal_params_supported(struct ethtool_coalesce *ec)
		2786	{
		2787	/* usecs coalescing is supported only if VIRTIO_NET_F_NOTF_COAL
		2788	* feature is negotiated.
		2789	*/
		2790	if (ec->rx_coalesce_usecs \|\| ec->tx_coalesce_usecs)
		2791	return -EOPNOTSUPP;
		2792
		2793	if (ec->tx_max_coalesced_frames > 1 \|\|
		2794	ec->rx_max_coalesced_frames != 1)
		2795	return -EINVAL;
		2796
		2797	return 0;
		2798	}
		2799
2621	static int virtnet_set_coalesce(struct net_device *dev,	2800	static int virtnet_set_coalesce(struct net_device *dev,
2622	struct ethtool_coalesce *ec,	2801	struct ethtool_coalesce *ec,
2623	struct kernel_ethtool_coalesce *kernel_coal,	2802	struct kernel_ethtool_coalesce *kernel_coal,
2624	struct netlink_ext_ack *extack)	2803	struct netlink_ext_ack *extack)
2625	{	2804	{
2626	struct virtnet_info *vi = netdev_priv(dev);	2805	struct virtnet_info *vi = netdev_priv(dev);
2627	int i, napi_weight;	2806	int ret, i, napi_weight;
2628		2807	bool update_napi = false;
2629	if (ec->tx_max_coalesced_frames > 1 \|\|
2630	ec->rx_max_coalesced_frames != 1)
2631	return -EINVAL;
2632		2808
		2809	/* Can't change NAPI weight if the link is up */
2633	napi_weight = ec->tx_max_coalesced_frames ? NAPI_POLL_WEIGHT : 0;	2810	napi_weight = ec->tx_max_coalesced_frames ? NAPI_POLL_WEIGHT : 0;
2634	if (napi_weight ^ vi->sq[0].napi.weight) {	2811	if (napi_weight ^ vi->sq[0].napi.weight) {
2635	if (dev->flags & IFF_UP)	2812	if (dev->flags & IFF_UP)
2636	return -EBUSY;	2813	return -EBUSY;
		2814	else
		2815	update_napi = true;
		2816	}
		2817
		2818	if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_NOTF_COAL))
		2819	ret = virtnet_send_notf_coal_cmds(vi, ec);
		2820	else
		2821	ret = virtnet_coal_params_supported(ec);
		2822
		2823	if (ret)
		2824	return ret;
		2825
		2826	if (update_napi) {
2637	for (i = 0; i < vi->max_queue_pairs; i++)	2827	for (i = 0; i < vi->max_queue_pairs; i++)
2638	vi->sq[i].napi.weight = napi_weight;	2828	vi->sq[i].napi.weight = napi_weight;
2639	}	2829	}
2640		2830
2641	return 0;	2831	return ret;
2642	}	2832	}
2643		2833
2644	static int virtnet_get_coalesce(struct net_device *dev,	2834	static int virtnet_get_coalesce(struct net_device *dev,
@@ -2646,16 +2836,19 @@ static int virtnet_get_coalesce(struct net_device *dev, · 1 unmodified line
2646	struct kernel_ethtool_coalesce *kernel_coal,	2836	struct kernel_ethtool_coalesce *kernel_coal,
2647	struct netlink_ext_ack *extack)	2837	struct netlink_ext_ack *extack)
2648	{	2838	{
2649	struct ethtool_coalesce ec_default = {
2650	.cmd = ETHTOOL_GCOALESCE,
2651	.rx_max_coalesced_frames = 1,
2652	};
2653	struct virtnet_info *vi = netdev_priv(dev);	2839	struct virtnet_info *vi = netdev_priv(dev);
2654		2840
2655	memcpy(ec, &ec_default, sizeof(ec_default));	2841	if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_NOTF_COAL)) {
		2842	ec->rx_coalesce_usecs = vi->rx_usecs;
		2843	ec->tx_coalesce_usecs = vi->tx_usecs;
		2844	ec->tx_max_coalesced_frames = vi->tx_max_packets;
		2845	ec->rx_max_coalesced_frames = vi->rx_max_packets;
		2846	} else {
		2847	ec->rx_max_coalesced_frames = 1;
2656		2848
2657	if (vi->sq[0].napi.weight)	2849	if (vi->sq[0].napi.weight)
2658	ec->tx_max_coalesced_frames = 1;	2850	ec->tx_max_coalesced_frames = 1;
		2851	}
2659		2852
2660	return 0;	2853	return 0;
2661	}	2854	}
@@ -2774,10 +2967,12 @@ static int virtnet_set_rxnfc(struct net_device dev, struct ethtool_rxnfc info) · 112 unmodified lines
2774	}	2967	}
2775		2968
2776	static const struct ethtool_ops virtnet_ethtool_ops = {	2969	static const struct ethtool_ops virtnet_ethtool_ops = {
2777	.supported_coalesce_params = ETHTOOL_COALESCE_MAX_FRAMES,	2970	.supported_coalesce_params = ETHTOOL_COALESCE_MAX_FRAMES \|
		2971	ETHTOOL_COALESCE_USECS,
2778	.get_drvinfo = virtnet_get_drvinfo,	2972	.get_drvinfo = virtnet_get_drvinfo,
2779	.get_link = ethtool_op_get_link,	2973	.get_link = ethtool_op_get_link,
2780	.get_ringparam = virtnet_get_ringparam,	2974	.get_ringparam = virtnet_get_ringparam,
		2975	.set_ringparam = virtnet_set_ringparam,
2781	.get_strings = virtnet_get_strings,	2976	.get_strings = virtnet_get_strings,
2782	.get_sset_count = virtnet_get_sset_count,	2977	.get_sset_count = virtnet_get_sset_count,
2783	.get_ethtool_stats = virtnet_get_ethtool_stats,	2978	.get_ethtool_stats = virtnet_get_ethtool_stats,
@@ -3171,6 +3366,27 @@ static void free_receive_page_frags(struct virtnet_info *vi) · 387 unmodified lines
3171	put_page(vi->rq[i].alloc_frag.page);	3366	put_page(vi->rq[i].alloc_frag.page);
3172	}	3367	}
3173		3368
		3369	static void virtnet_sq_free_unused_buf(struct virtqueue vq, void buf)
		3370	{
		3371	if (!is_xdp_frame(buf))
		3372	dev_kfree_skb(buf);
		3373	else
		3374	xdp_return_frame(ptr_to_xdp(buf));
		3375	}
		3376
		3377	static void virtnet_rq_free_unused_buf(struct virtqueue vq, void buf)
		3378	{
		3379	struct virtnet_info *vi = vq->vdev->priv;
		3380	int i = vq2rxq(vq);
		3381
		3382	if (vi->mergeable_rx_bufs)
		3383	put_page(virt_to_head_page(buf));
		3384	else if (vi->big_packets)
		3385	give_pages(&vi->rq[i], buf);
		3386	else
		3387	put_page(virt_to_head_page(buf));
		3388	}
		3389
3174	static void free_unused_bufs(struct virtnet_info *vi)	3390	static void free_unused_bufs(struct virtnet_info *vi)
3175	{	3391	{
3176	void *buf;	3392	void *buf;
@@ -3178,26 +3394,14 @@ static void free_unused_bufs(struct virtnet_info *vi) · 1 unmodified line
3178		3394
3179	for (i = 0; i < vi->max_queue_pairs; i++) {	3395	for (i = 0; i < vi->max_queue_pairs; i++) {
3180	struct virtqueue *vq = vi->sq[i].vq;	3396	struct virtqueue *vq = vi->sq[i].vq;
3181	while ((buf = virtqueue_detach_unused_buf(vq)) != NULL) {	3397	while ((buf = virtqueue_detach_unused_buf(vq)) != NULL)
3182	if (!is_xdp_frame(buf))	3398	virtnet_sq_free_unused_buf(vq, buf);
3183	dev_kfree_skb(buf);
3184	else
3185	xdp_return_frame(ptr_to_xdp(buf));
3186	}
3187	}	3399	}
3188		3400
3189	for (i = 0; i < vi->max_queue_pairs; i++) {	3401	for (i = 0; i < vi->max_queue_pairs; i++) {
3190	struct virtqueue *vq = vi->rq[i].vq;	3402	struct virtqueue *vq = vi->rq[i].vq;
3191		3403	while ((buf = virtqueue_detach_unused_buf(vq)) != NULL)
3192	while ((buf = virtqueue_detach_unused_buf(vq)) != NULL) {	3404	virtnet_rq_free_unused_buf(vq, buf);
3193	if (vi->mergeable_rx_bufs) {
3194	put_page(virt_to_head_page(buf));
3195	} else if (vi->big_packets) {
3196	give_pages(&vi->rq[i], buf);
3197	} else {
3198	put_page(virt_to_head_page(buf));
3199	}
3200	}
3201	}	3405	}
3202	}	3406	}
3203		3407
@@ -3228,6 +3432,29 @@ static unsigned int mergeable_min_buf_len(struct virtnet_info *vi, struct virtqu · 24 unmodified lines
3228	(unsigned int)GOOD_PACKET_LEN);	3432	(unsigned int)GOOD_PACKET_LEN);
3229	}	3433	}
3230		3434
		3435	static void virtnet_config_sizes(struct virtnet_info vi, u32 sizes)
		3436	{
		3437	u32 i, rx_size, tx_size;
		3438
		3439	if (vi->speed == SPEED_UNKNOWN \|\| vi->speed < SPEED_10000) {
		3440	rx_size = 1024;
		3441	tx_size = 1024;
		3442
		3443	} else if (vi->speed < SPEED_40000) {
		3444	rx_size = 1024 * 4;
		3445	tx_size = 1024 * 4;
		3446
		3447	} else {
		3448	rx_size = 1024 * 8;
		3449	tx_size = 1024 * 8;
		3450	}
		3451
		3452	for (i = 0; i < vi->max_queue_pairs; i++) {
		3453	sizes[rxq2vq(i)] = rx_size;
		3454	sizes[txq2vq(i)] = tx_size;
		3455	}
		3456	}
		3457
3231	static int virtnet_find_vqs(struct virtnet_info *vi)	3458	static int virtnet_find_vqs(struct virtnet_info *vi)
3232	{	3459	{
3233	vq_callback_t **callbacks;	3460	vq_callback_t **callbacks;
@@ -3235,6 +3462,7 @@ static int virtnet_find_vqs(struct virtnet_info *vi) · 1 unmodified line
3235	int ret = -ENOMEM;	3462	int ret = -ENOMEM;
3236	int i, total_vqs;	3463	int i, total_vqs;
3237	const char **names;	3464	const char **names;
		3465	u32 *sizes;
3238	bool *ctx;	3466	bool *ctx;
3239		3467
3240	/* We expect 1 RX virtqueue followed by 1 TX virtqueue, followed by	3468	/* We expect 1 RX virtqueue followed by 1 TX virtqueue, followed by
@@ -3262,10 +3490,15 @@ static int virtnet_find_vqs(struct virtnet_info *vi) · 21 unmodified lines
3262	ctx = NULL;	3490	ctx = NULL;
3263	}	3491	}
3264		3492
		3493	sizes = kmalloc_array(total_vqs, sizeof(*sizes), GFP_KERNEL);
		3494	if (!sizes)
		3495	goto err_sizes;
		3496
3265	/* Parameters for control virtqueue, if any */	3497	/* Parameters for control virtqueue, if any */
3266	if (vi->has_cvq) {	3498	if (vi->has_cvq) {
3267	callbacks[total_vqs - 1] = NULL;	3499	callbacks[total_vqs - 1] = NULL;
3268	names[total_vqs - 1] = "control";	3500	names[total_vqs - 1] = "control";
		3501	sizes[total_vqs - 1] = 64;
3269	}	3502	}
3270		3503
3271	/* Allocate/initialize parameters for send/receive virtqueues */	3504	/* Allocate/initialize parameters for send/receive virtqueues */
@@ -3280,8 +3513,10 @@ static int virtnet_find_vqs(struct virtnet_info *vi) · 8 unmodified lines
3280	ctx[rxq2vq(i)] = true;	3513	ctx[rxq2vq(i)] = true;
3281	}	3514	}
3282		3515
3283	ret = virtio_find_vqs_ctx(vi->vdev, total_vqs, vqs, callbacks,	3516	virtnet_config_sizes(vi, sizes);
3284	names, ctx, NULL);	3517
		3518	ret = virtio_find_vqs_ctx_size(vi->vdev, total_vqs, vqs, callbacks,
		3519	names, sizes, ctx, NULL);
3285	if (ret)	3520	if (ret)
3286	goto err_find;	3521	goto err_find;
3287		3522
@@ -3301,6 +3536,8 @@ static int virtnet_find_vqs(struct virtnet_info *vi) · 13 unmodified lines
3301		3536
3302		3537
3303	err_find:	3538	err_find:
		3539	kfree(sizes);
		3540	err_sizes:
3304	kfree(ctx);	3541	kfree(ctx);
3305	err_ctx:	3542	err_ctx:
3306	kfree(names);	3543	kfree(names);
@@ -3444,6 +3681,8 @@ static bool virtnet_validate_features(struct virtio_device *vdev) · 137 unmodified lines
3444	VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_RSS,	3681	VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_RSS,
3445	"VIRTIO_NET_F_CTRL_VQ") \|\|	3682	"VIRTIO_NET_F_CTRL_VQ") \|\|
3446	VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_HASH_REPORT,	3683	VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_HASH_REPORT,
		3684	"VIRTIO_NET_F_CTRL_VQ") \|\|
		3685	VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_NOTF_COAL,
3447	"VIRTIO_NET_F_CTRL_VQ"))) {	3686	"VIRTIO_NET_F_CTRL_VQ"))) {
3448	return false;	3687	return false;
3449	}	3688	}
@@ -3580,6 +3819,13 @@ static int virtnet_probe(struct virtio_device *vdev) · 130 unmodified lines
3580	if (virtio_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF))	3819	if (virtio_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF))
3581	vi->mergeable_rx_bufs = true;	3820	vi->mergeable_rx_bufs = true;
3582		3821
		3822	if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_NOTF_COAL)) {
		3823	vi->rx_usecs = 0;
		3824	vi->tx_usecs = 0;
		3825	vi->tx_max_packets = 0;
		3826	vi->rx_max_packets = 0;
		3827	}
		3828
3583	if (virtio_has_feature(vdev, VIRTIO_NET_F_HASH_REPORT))	3829	if (virtio_has_feature(vdev, VIRTIO_NET_F_HASH_REPORT))
3584	vi->has_rss_hash_report = true;	3830	vi->has_rss_hash_report = true;
3585		3831
@@ -3651,6 +3897,9 @@ static int virtnet_probe(struct virtio_device *vdev) · 65 unmodified lines
3651	vi->curr_queue_pairs = num_online_cpus();	3897	vi->curr_queue_pairs = num_online_cpus();
3652	vi->max_queue_pairs = max_queue_pairs;	3898	vi->max_queue_pairs = max_queue_pairs;
3653		3899
		3900	virtnet_init_settings(dev);
		3901	virtnet_update_settings(vi);
		3902
3654	/* Allocate/initialize the rx/tx queues, and invoke find_vqs */	3903	/* Allocate/initialize the rx/tx queues, and invoke find_vqs */
3655	err = init_vqs(vi);	3904	err = init_vqs(vi);
3656	if (err)	3905	if (err)
@@ -3663,8 +3912,6 @@ static int virtnet_probe(struct virtio_device *vdev) · 6 unmodified lines
3663	netif_set_real_num_tx_queues(dev, vi->curr_queue_pairs);	3912	netif_set_real_num_tx_queues(dev, vi->curr_queue_pairs);
3664	netif_set_real_num_rx_queues(dev, vi->curr_queue_pairs);	3913	netif_set_real_num_rx_queues(dev, vi->curr_queue_pairs);
3665		3914
3666	virtnet_init_settings(dev);
3667
3668	if (virtio_has_feature(vdev, VIRTIO_NET_F_STANDBY)) {	3915	if (virtio_has_feature(vdev, VIRTIO_NET_F_STANDBY)) {
3669	vi->failover = net_failover_create(vi->dev);	3916	vi->failover = net_failover_create(vi->dev);
3670	if (IS_ERR(vi->failover)) {	3917	if (IS_ERR(vi->failover)) {
@@ -3814,7 +4061,7 @@ static struct virtio_device_id id_table[] = { · 143 unmodified lines
3814	VIRTIO_NET_F_CTRL_MAC_ADDR, \	4061	VIRTIO_NET_F_CTRL_MAC_ADDR, \
3815	VIRTIO_NET_F_MTU, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS, \	4062	VIRTIO_NET_F_MTU, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS, \
3816	VIRTIO_NET_F_SPEED_DUPLEX, VIRTIO_NET_F_STANDBY, \	4063	VIRTIO_NET_F_SPEED_DUPLEX, VIRTIO_NET_F_STANDBY, \
3817	VIRTIO_NET_F_RSS, VIRTIO_NET_F_HASH_REPORT	4064	VIRTIO_NET_F_RSS, VIRTIO_NET_F_HASH_REPORT, VIRTIO_NET_F_NOTF_COAL
3818		4065
3819	static unsigned int features[] = {	4066	static unsigned int features[] = {
3820	VIRTNET_FEATURES,	4067	VIRTNET_FEATURES,

drivers/nvdimm/virtio_pmem.c+8−1

 	ndr_desc.res = &res;
 	ndr_desc.numa_node = nid;
 	ndr_desc.flush = async_pmem_flush;
+	ndr_desc.provider_data = vdev;
 	set_bit(ND_REGION_PAGEMAP, &ndr_desc.flags);
 	set_bit(ND_REGION_ASYNC, &ndr_desc.flags);
+	/*
+	 * The NVDIMM region could be available before the
+	 * virtio_device_ready() that is called by
+	 * virtio_dev_probe(), so we set device ready here.
+	 */
+	virtio_device_ready(vdev);
 	nd_region = nvdimm_pmem_region_create(vpmem->nvdimm_bus, &ndr_desc);
 	if (!nd_region) {
 		dev_err(&vdev->dev, "failed to create nvdimm region\n");
 		err = -ENXIO;
 		goto out_nd;
 	}
-	nd_region->provider_data = dev_to_virtio(nd_region->dev.parent->parent);
 	return 0;
 out_nd:
+	virtio_reset_device(vdev);
 	nvdimm_bus_unregister(vpmem->nvdimm_bus);
 out_vq:
 	vdev->config->del_vqs(vdev);

drivers/platform/mellanox/mlxbf-tmfifo.c+3−0

 					struct virtqueue *vqs[],
 					vq_callback_t *callbacks[],
 					const char * const names[],
+					u32 sizes[],
 					const bool *ctx,
 					struct irq_affinity *desc)
 {
 			goto error;
 		}
+		vq->num_max = vring->num;
 		vqs[i] = vq;
 		vring->vq = vq;
 		vq->priv = vring;

drivers/remoteproc/remoteproc_core.c+2−2

@@ -335,7 +335,7 @@ int rproc_alloc_vring(struct rproc_vdev *rvdev, int i) · 334 unmodified lines
335	size_t size;	335	size_t size;
336		336
337	/* actual size of vring (in bytes) */	337	/* actual size of vring (in bytes) */
338	size = PAGE_ALIGN(vring_size(rvring->len, rvring->align));	338	size = PAGE_ALIGN(vring_size(rvring->num, rvring->align));
339		339
340	rsc = (void *)rproc->table_ptr + rvdev->rsc_offset;	340	rsc = (void *)rproc->table_ptr + rvdev->rsc_offset;
341		341
@@ -402,7 +402,7 @@ rproc_parse_vring(struct rproc_vdev rvdev, struct fw_rsc_vdev rsc, int i) · 60 unmodified lines
402	return -EINVAL;	402	return -EINVAL;
403	}	403	}
404		404
405	rvring->len = vring->num;	405	rvring->num = vring->num;
406	rvring->align = vring->align;	406	rvring->align = vring->align;
407	rvring->rvdev = rvdev;	407	rvring->rvdev = rvdev;
408		408

drivers/remoteproc/remoteproc_virtio.c+8−5

 	struct fw_rsc_vdev *rsc;
 	struct virtqueue *vq;
 	void *addr;
-	int len, size;
+	int num, size;
 	/* we're temporarily limited to two virtqueues per rvdev */
 	if (id >= ARRAY_SIZE(rvdev->vring))
 	rvring = &rvdev->vring[id];
 	addr = mem->va;
-	len = rvring->len;
+	num = rvring->num;
 	/* zero vring */
-	size = vring_size(len, rvring->align);
+	size = vring_size(num, rvring->align);
 	memset(addr, 0, size);
 	dev_dbg(dev, "vring%d: va %pK qsz %d notifyid %d\n",
-		id, addr, len, rvring->notifyid);
+		id, addr, num, rvring->notifyid);
 	/*
 	 * Create the new vq, and tell virtio we're not interested in
 	 * the 'weak' smp barriers, since we're talking with a real device.
 	 */
-	vq = vring_new_virtqueue(id, len, rvring->align, vdev, false, ctx,
+	vq = vring_new_virtqueue(id, num, rvring->align, vdev, false, ctx,
 				 addr, rproc_virtio_notify, callback, name);
 	if (!vq) {
 		dev_err(dev, "vring_new_virtqueue %s failed\n", name);
 		return ERR_PTR(-ENOMEM);
 	}
+	vq->num_max = num;
 	rvring->vq = vq;
 	vq->priv = rvring;
 				 struct virtqueue *vqs[],
 				 vq_callback_t *callbacks[],
 				 const char * const names[],
+				 u32 sizes[],
 				 const bool * ctx,
 				 struct irq_affinity *desc)
 {

drivers/s390/virtio/virtio_ccw.c+4−0

 		err = -ENOMEM;
 		goto out_err;
 	}
+	vq->num_max = info->num;
 	/* it may have been reduced */
 	info->num = virtqueue_get_vring_size(vq);
 			       struct virtqueue *vqs[],
 			       vq_callback_t *callbacks[],
 			       const char * const names[],
+			       u32 sizes[],
 			       const bool *ctx,
 			       struct irq_affinity *desc)
 {

drivers/vdpa/ifcvf/ifcvf_base.c+11−3

 {
 	struct virtio_pci_common_cfg __iomem *cfg = hw->common_cfg;
-	cfg = hw->common_cfg;
 	vp_iowrite16(vector,  &cfg->msix_config);
 	return vp_ioread16(&cfg->msix_config);
 			break;
 		case VIRTIO_PCI_CAP_DEVICE_CFG:
 			hw->dev_cfg = get_cap_addr(hw, &cap);
+			hw->cap_dev_config_size = le32_to_cpu(cap.length);
 			IFCVF_DBG(pdev, "hw->dev_cfg = %p\n", hw->dev_cfg);
 			break;
 		}
 u32 ifcvf_get_config_size(struct ifcvf_hw *hw)
 {
 	struct ifcvf_adapter *adapter;
+	u32 net_config_size = sizeof(struct virtio_net_config);
+	u32 blk_config_size = sizeof(struct virtio_blk_config);
+	u32 cap_size = hw->cap_dev_config_size;
 	u32 config_size;
 	adapter = vf_to_adapter(hw);
+	/* If the onboard device config space size is greater than
+	 * the size of struct virtio_net/blk_config, only the spec
+	 * implementing contents size is returned, this is very
+	 * unlikely, defensive programming.
+	 */
 	switch (hw->dev_type) {
 	case VIRTIO_ID_NET:
-		config_size = sizeof(struct virtio_net_config);
+		config_size = min(cap_size, net_config_size);
 		break;
 	case VIRTIO_ID_BLOCK:
-		config_size = sizeof(struct virtio_blk_config);
+		config_size = min(cap_size, blk_config_size);
 		break;
 	default:
 		config_size = 0;

drivers/vdpa/ifcvf/ifcvf_base.h+2−0

@@ -87,6 +87,8 @@ struct ifcvf_hw { · 86 unmodified lines
87	int config_irq;	87	int config_irq;
88	int vqs_reused_irq;	88	int vqs_reused_irq;
89	u16 nr_vring;	89	u16 nr_vring;
		90	/* VIRTIO_PCI_CAP_DEVICE_CFG size */
		91	u32 cap_dev_config_size;
90	};	92	};
91		93
92	struct ifcvf_adapter {	94	struct ifcvf_adapter {

drivers/vdpa/ifcvf/ifcvf_main.c+77−67

@@ -685,7 +685,7 @@ static struct vdpa_notification_area ifcvf_get_vq_notification(struct vdpa_devic · 684 unmodified lines
685	}	685	}
686		686
687	/*	687	/*
688	* IFCVF currently does't have on-chip IOMMU, so not	688	* IFCVF currently doesn't have on-chip IOMMU, so not
689	* implemented set_map()/dma_map()/dma_unmap()	689	* implemented set_map()/dma_map()/dma_unmap()
690	*/	690	*/
691	static const struct vdpa_config_ops ifc_vdpa_ops = {	691	static const struct vdpa_config_ops ifc_vdpa_ops = {
@@ -752,59 +752,36 @@ static int ifcvf_vdpa_dev_add(struct vdpa_mgmt_dev mdev, const char name, · 60 unmodified lines
752	{	752	{
753	struct ifcvf_vdpa_mgmt_dev *ifcvf_mgmt_dev;	753	struct ifcvf_vdpa_mgmt_dev *ifcvf_mgmt_dev;
754	struct ifcvf_adapter *adapter;	754	struct ifcvf_adapter *adapter;
		755	struct vdpa_device *vdpa_dev;
755	struct pci_dev *pdev;	756	struct pci_dev *pdev;
756	struct ifcvf_hw *vf;	757	struct ifcvf_hw *vf;
757	struct device *dev;	758	int ret;
758	int ret, i;
759		759
760	ifcvf_mgmt_dev = container_of(mdev, struct ifcvf_vdpa_mgmt_dev, mdev);	760	ifcvf_mgmt_dev = container_of(mdev, struct ifcvf_vdpa_mgmt_dev, mdev);
761	if (ifcvf_mgmt_dev->adapter)	761	if (!ifcvf_mgmt_dev->adapter)
762	return -EOPNOTSUPP;	762	return -EOPNOTSUPP;
763		763
764	pdev = ifcvf_mgmt_dev->pdev;	764	adapter = ifcvf_mgmt_dev->adapter;
765	dev = &pdev->dev;
766	adapter = vdpa_alloc_device(struct ifcvf_adapter, vdpa,
767	dev, &ifc_vdpa_ops, 1, 1, name, false);
768	if (IS_ERR(adapter)) {
769	IFCVF_ERR(pdev, "Failed to allocate vDPA structure");
770	return PTR_ERR(adapter);
771	}
772
773	ifcvf_mgmt_dev->adapter = adapter;
774
775	vf = &adapter->vf;	765	vf = &adapter->vf;
776	vf->dev_type = get_dev_type(pdev);	766	pdev = adapter->pdev;
777	vf->base = pcim_iomap_table(pdev);	767	vdpa_dev = &adapter->vdpa;
778		768
779	adapter->pdev = pdev;	769	if (name)
780	adapter->vdpa.dma_dev = &pdev->dev;	770	ret = dev_set_name(&vdpa_dev->dev, "%s", name);
781		771	else
782	ret = ifcvf_init_hw(vf, pdev);	772	ret = dev_set_name(&vdpa_dev->dev, "vdpa%u", vdpa_dev->index);
783	if (ret) {
784	IFCVF_ERR(pdev, "Failed to init IFCVF hw\n");
785	goto err;
786	}
787
788	for (i = 0; i < vf->nr_vring; i++)
789	vf->vring[i].irq = -EINVAL;
790
791	vf->hw_features = ifcvf_get_hw_features(vf);
792	vf->config_size = ifcvf_get_config_size(vf);
793		773
794	adapter->vdpa.mdev = &ifcvf_mgmt_dev->mdev;
795	ret = _vdpa_register_device(&adapter->vdpa, vf->nr_vring);	774	ret = _vdpa_register_device(&adapter->vdpa, vf->nr_vring);
796	if (ret) {	775	if (ret) {
		776	put_device(&adapter->vdpa.dev);
797	IFCVF_ERR(pdev, "Failed to register to vDPA bus");	777	IFCVF_ERR(pdev, "Failed to register to vDPA bus");
798	goto err;	778	return ret;
799	}	779	}
800		780
801	return 0;	781	return 0;
802
803	err:
804	put_device(&adapter->vdpa.dev);
805	return ret;
806	}	782	}
807		783
		784
808	static void ifcvf_vdpa_dev_del(struct vdpa_mgmt_dev mdev, struct vdpa_device dev)	785	static void ifcvf_vdpa_dev_del(struct vdpa_mgmt_dev mdev, struct vdpa_device dev)
809	{	786	{
810	struct ifcvf_vdpa_mgmt_dev *ifcvf_mgmt_dev;	787	struct ifcvf_vdpa_mgmt_dev *ifcvf_mgmt_dev;
@@ -823,61 +800,94 @@ static int ifcvf_probe(struct pci_dev pdev, const struct pci_device_id id) · 12 unmodified lines
823	{	800	{
824	struct ifcvf_vdpa_mgmt_dev *ifcvf_mgmt_dev;	801	struct ifcvf_vdpa_mgmt_dev *ifcvf_mgmt_dev;
825	struct device *dev = &pdev->dev;	802	struct device *dev = &pdev->dev;
		803	struct ifcvf_adapter *adapter;
		804	struct ifcvf_hw *vf;
826	u32 dev_type;	805	u32 dev_type;
827	int ret;	806	int ret, i;
828
829	ifcvf_mgmt_dev = kzalloc(sizeof(struct ifcvf_vdpa_mgmt_dev), GFP_KERNEL);
830	if (!ifcvf_mgmt_dev) {
831	IFCVF_ERR(pdev, "Failed to alloc memory for the vDPA management device\n");
832	return -ENOMEM;
833	}
834
835	dev_type = get_dev_type(pdev);
836	switch (dev_type) {
837	case VIRTIO_ID_NET:
838	ifcvf_mgmt_dev->mdev.id_table = id_table_net;
839	break;
840	case VIRTIO_ID_BLOCK:
841	ifcvf_mgmt_dev->mdev.id_table = id_table_blk;
842	break;
843	default:
844	IFCVF_ERR(pdev, "VIRTIO ID %u not supported\n", dev_type);
845	ret = -EOPNOTSUPP;
846	goto err;
847	}
848
849	ifcvf_mgmt_dev->mdev.ops = &ifcvf_vdpa_mgmt_dev_ops;
850	ifcvf_mgmt_dev->mdev.device = dev;
851	ifcvf_mgmt_dev->pdev = pdev;
852		807
853	ret = pcim_enable_device(pdev);	808	ret = pcim_enable_device(pdev);
854	if (ret) {	809	if (ret) {
855	IFCVF_ERR(pdev, "Failed to enable device\n");	810	IFCVF_ERR(pdev, "Failed to enable device\n");
856	goto err;	811	return ret;
857	}	812	}
858
859	ret = pcim_iomap_regions(pdev, BIT(0) \| BIT(2) \| BIT(4),	813	ret = pcim_iomap_regions(pdev, BIT(0) \| BIT(2) \| BIT(4),
860	IFCVF_DRIVER_NAME);	814	IFCVF_DRIVER_NAME);
861	if (ret) {	815	if (ret) {
862	IFCVF_ERR(pdev, "Failed to request MMIO region\n");	816	IFCVF_ERR(pdev, "Failed to request MMIO region\n");
863	goto err;	817	return ret;
864	}	818	}
865		819
866	ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));	820	ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));
867	if (ret) {	821	if (ret) {
868	IFCVF_ERR(pdev, "No usable DMA configuration\n");	822	IFCVF_ERR(pdev, "No usable DMA configuration\n");
869	goto err;	823	return ret;
870	}	824	}
871		825
872	ret = devm_add_action_or_reset(dev, ifcvf_free_irq_vectors, pdev);	826	ret = devm_add_action_or_reset(dev, ifcvf_free_irq_vectors, pdev);
873	if (ret) {	827	if (ret) {
874	IFCVF_ERR(pdev,	828	IFCVF_ERR(pdev,
875	"Failed for adding devres for freeing irq vectors\n");	829	"Failed for adding devres for freeing irq vectors\n");
876	goto err;	830	return ret;
877	}	831	}
878		832
879	pci_set_master(pdev);	833	pci_set_master(pdev);
880		834
		835	adapter = vdpa_alloc_device(struct ifcvf_adapter, vdpa,
		836	dev, &ifc_vdpa_ops, 1, 1, NULL, false);
		837	if (IS_ERR(adapter)) {
		838	IFCVF_ERR(pdev, "Failed to allocate vDPA structure");
		839	return PTR_ERR(adapter);
		840	}
		841
		842	vf = &adapter->vf;
		843	vf->dev_type = get_dev_type(pdev);
		844	vf->base = pcim_iomap_table(pdev);
		845
		846	adapter->pdev = pdev;
		847	adapter->vdpa.dma_dev = &pdev->dev;
		848
		849	ret = ifcvf_init_hw(vf, pdev);
		850	if (ret) {
		851	IFCVF_ERR(pdev, "Failed to init IFCVF hw\n");
		852	return ret;
		853	}
		854
		855	for (i = 0; i < vf->nr_vring; i++)
		856	vf->vring[i].irq = -EINVAL;
		857
		858	vf->hw_features = ifcvf_get_hw_features(vf);
		859	vf->config_size = ifcvf_get_config_size(vf);
		860
		861	ifcvf_mgmt_dev = kzalloc(sizeof(struct ifcvf_vdpa_mgmt_dev), GFP_KERNEL);
		862	if (!ifcvf_mgmt_dev) {
		863	IFCVF_ERR(pdev, "Failed to alloc memory for the vDPA management device\n");
		864	return -ENOMEM;
		865	}
		866
		867	ifcvf_mgmt_dev->mdev.ops = &ifcvf_vdpa_mgmt_dev_ops;
		868	ifcvf_mgmt_dev->mdev.device = dev;
		869	ifcvf_mgmt_dev->adapter = adapter;
		870
		871	dev_type = get_dev_type(pdev);
		872	switch (dev_type) {
		873	case VIRTIO_ID_NET:
		874	ifcvf_mgmt_dev->mdev.id_table = id_table_net;
		875	break;
		876	case VIRTIO_ID_BLOCK:
		877	ifcvf_mgmt_dev->mdev.id_table = id_table_blk;
		878	break;
		879	default:
		880	IFCVF_ERR(pdev, "VIRTIO ID %u not supported\n", dev_type);
		881	ret = -EOPNOTSUPP;
		882	goto err;
		883	}
		884
		885	ifcvf_mgmt_dev->mdev.max_supported_vqs = vf->nr_vring;
		886	ifcvf_mgmt_dev->mdev.supported_features = vf->hw_features;
		887
		888	adapter->vdpa.mdev = &ifcvf_mgmt_dev->mdev;
		889
		890
881	ret = vdpa_mgmtdev_register(&ifcvf_mgmt_dev->mdev);	891	ret = vdpa_mgmtdev_register(&ifcvf_mgmt_dev->mdev);
882	if (ret) {	892	if (ret) {
883	IFCVF_ERR(pdev,	893	IFCVF_ERR(pdev,

drivers/vdpa/mlx5/core/mlx5_vdpa.h+11−0

 	struct mlx5_vdpa_dev *mvdev;
 };
+enum {
+	MLX5_VDPA_DATAVQ_GROUP,
+	MLX5_VDPA_CVQ_GROUP,
+	MLX5_VDPA_NUMVQ_GROUPS
+};
+enum {
+	MLX5_VDPA_NUM_AS = MLX5_VDPA_NUMVQ_GROUPS
+};
 struct mlx5_vdpa_dev {
 	struct vdpa_device vdev;
 	struct mlx5_core_dev *mdev;
 	struct mlx5_vdpa_mr mr;
 	struct mlx5_control_vq cvq;
 	struct workqueue_struct *wq;
+	unsigned int group2asid[MLX5_VDPA_NUMVQ_GROUPS];
 };
 int mlx5_vdpa_alloc_pd(struct mlx5_vdpa_dev *dev, u32 *pdn, u16 uid);

drivers/vdpa/mlx5/net/mlx5_vnet.c+158−15

 	bool setup;
 	u32 cur_num_vqs;
 	u32 rqt_size;
+	bool nb_registered;
 	struct notifier_block nb;
 	struct vdpa_callback config_cb;
 	struct mlx5_vdpa_wq_ent cvq_ent;
 	if (err)
 		goto err_cmd;
+	mvq->fw_state = MLX5_VIRTIO_NET_Q_OBJECT_STATE_INIT;
 	kfree(in);
 	mvq->virtq_id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);
 		mlx5_vdpa_warn(&ndev->mvdev, "destroy virtqueue 0x%x\n", mvq->virtq_id);
 		return;
 	}
+	mvq->fw_state = MLX5_VIRTIO_NET_Q_OBJECT_NONE;
 	umems_destroy(ndev, mvq);
 }
 	return err;
 }
+static bool is_valid_state_change(int oldstate, int newstate)
+{
+	switch (oldstate) {
+	case MLX5_VIRTIO_NET_Q_OBJECT_STATE_INIT:
+		return newstate == MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY;
+	case MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY:
+		return newstate == MLX5_VIRTIO_NET_Q_OBJECT_STATE_SUSPEND;
+	case MLX5_VIRTIO_NET_Q_OBJECT_STATE_SUSPEND:
+	case MLX5_VIRTIO_NET_Q_OBJECT_STATE_ERR:
+	default:
+		return false;
+	}
+}
 static int modify_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, int state)
 {
 	int inlen = MLX5_ST_SZ_BYTES(modify_virtio_net_q_in);
 	void *in;
 	int err;
+	if (mvq->fw_state == MLX5_VIRTIO_NET_Q_OBJECT_NONE)
+		return 0;
+	if (!is_valid_state_change(mvq->fw_state, state))
+		return -EINVAL;
 	in = kzalloc(inlen, GFP_KERNEL);
 	if (!in)
 		return -ENOMEM;
 	headers_v = MLX5_ADDR_OF(fte_match_param, spec->match_value, outer_headers);
 	dmac_c = MLX5_ADDR_OF(fte_match_param, headers_c, outer_headers.dmac_47_16);
 	dmac_v = MLX5_ADDR_OF(fte_match_param, headers_v, outer_headers.dmac_47_16);
-	memset(dmac_c, 0xff, ETH_ALEN);
+	eth_broadcast_addr(dmac_c);
 	ether_addr_copy(dmac_v, mac);
 	MLX5_SET(fte_match_set_lyr_2_4, headers_c, cvlan_tag, 1);
 	if (tagged) {
 	struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev);
 	struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev);
 	struct mlx5_vdpa_virtqueue *mvq;
+	int err;
 	if (!mvdev->actual_features)
 		return;
 	}
 	mvq = &ndev->vqs[idx];
-	if (!ready)
+	if (!ready) {
 		suspend_vq(ndev, mvq);
+	} else {
+		err = modify_virtqueue(ndev, mvq, MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY);
+		if (err) {
+			mlx5_vdpa_warn(mvdev, "modify VQ %d to ready failed (%d)\n", idx, err);
+			ready = false;
+		}
+	}
 	mvq->ready = ready;
 }
 	return PAGE_SIZE;
 }
-static u32 mlx5_vdpa_get_vq_group(struct vdpa_device *vdpa, u16 idx)
+static u32 mlx5_vdpa_get_vq_group(struct vdpa_device *vdev, u16 idx)
 {
-	return 0;
+	struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev);
+	if (is_ctrl_vq_idx(mvdev, idx))
+		return MLX5_VDPA_CVQ_GROUP;
+	return MLX5_VDPA_DATAVQ_GROUP;
 }
 enum { MLX5_VIRTIO_NET_F_GUEST_CSUM = 1 << 9,
 	up_write(&ndev->reslock);
 }
+static void init_group_to_asid_map(struct mlx5_vdpa_dev *mvdev)
+{
+	int i;
+	/* default mapping all groups are mapped to asid 0 */
+	for (i = 0; i < MLX5_VDPA_NUMVQ_GROUPS; i++)
+		mvdev->group2asid[i] = 0;
+}
 static int mlx5_vdpa_reset(struct vdpa_device *vdev)
 {
 	struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev);
 	ndev->mvdev.cvq.completed_desc = 0;
 	memset(ndev->event_cbs, 0, sizeof(*ndev->event_cbs) * (mvdev->max_vqs + 1));
 	ndev->mvdev.actual_features = 0;
+	init_group_to_asid_map(mvdev);
 	++mvdev->generation;
 	if (MLX5_CAP_GEN(mvdev->mdev, umem_uid_0)) {
 		if (mlx5_vdpa_create_mr(mvdev, NULL))
 			mlx5_vdpa_warn(mvdev, "create MR failed\n");
 	return mvdev->generation;
 }
-static int mlx5_vdpa_set_map(struct vdpa_device *vdev, unsigned int asid,
+static int set_map_control(struct mlx5_vdpa_dev *mvdev, struct vhost_iotlb *iotlb)
-			     struct vhost_iotlb *iotlb)
+{
+	u64 start = 0ULL, last = 0ULL - 1;
+	struct vhost_iotlb_map *map;
+	int err = 0;
+	spin_lock(&mvdev->cvq.iommu_lock);
+	vhost_iotlb_reset(mvdev->cvq.iotlb);
+	for (map = vhost_iotlb_itree_first(iotlb, start, last); map;
+	     map = vhost_iotlb_itree_next(map, start, last)) {
+		err = vhost_iotlb_add_range(mvdev->cvq.iotlb, map->start,
+					    map->last, map->addr, map->perm);
+		if (err)
+			goto out;
+	}
+out:
+	spin_unlock(&mvdev->cvq.iommu_lock);
+	return err;
+}
+static int set_map_data(struct mlx5_vdpa_dev *mvdev, struct vhost_iotlb *iotlb)
 {
-	struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev);
-	struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev);
 	bool change_map;
 	int err;
-	down_write(&ndev->reslock);
 	err = mlx5_vdpa_handle_set_map(mvdev, iotlb, &change_map);
 	if (err) {
 		mlx5_vdpa_warn(mvdev, "set map failed(%d)\n", err);
-		goto err;
+		return err;
 	}
 	if (change_map)
 		err = mlx5_vdpa_change_map(mvdev, iotlb);
-err:
+	return err;
+}
+static int mlx5_vdpa_set_map(struct vdpa_device *vdev, unsigned int asid,
+			     struct vhost_iotlb *iotlb)
+{
+	struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev);
+	struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev);
+	int err = -EINVAL;
+	down_write(&ndev->reslock);
+	if (mvdev->group2asid[MLX5_VDPA_DATAVQ_GROUP] == asid) {
+		err = set_map_data(mvdev, iotlb);
+		if (err)
+			goto out;
+	}
+	if (mvdev->group2asid[MLX5_VDPA_CVQ_GROUP] == asid)
+		err = set_map_control(mvdev, iotlb);
+out:
 	up_write(&ndev->reslock);
 	return err;
 }
 	return err;
 }
+static void mlx5_vdpa_cvq_suspend(struct mlx5_vdpa_dev *mvdev)
+{
+	struct mlx5_control_vq *cvq;
+	if (!(mvdev->actual_features & BIT_ULL(VIRTIO_NET_F_CTRL_VQ)))
+		return;
+	cvq = &mvdev->cvq;
+	cvq->ready = false;
+}
+static int mlx5_vdpa_suspend(struct vdpa_device *vdev)
+{
+	struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev);
+	struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev);
+	struct mlx5_vdpa_virtqueue *mvq;
+	int i;
+	down_write(&ndev->reslock);
+	mlx5_notifier_unregister(mvdev->mdev, &ndev->nb);
+	ndev->nb_registered = false;
+	flush_workqueue(ndev->mvdev.wq);
+	for (i = 0; i < ndev->cur_num_vqs; i++) {
+		mvq = &ndev->vqs[i];
+		suspend_vq(ndev, mvq);
+	}
+	mlx5_vdpa_cvq_suspend(mvdev);
+	up_write(&ndev->reslock);
+	return 0;
+}
+static int mlx5_set_group_asid(struct vdpa_device *vdev, u32 group,
+			       unsigned int asid)
+{
+	struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev);
+	if (group >= MLX5_VDPA_NUMVQ_GROUPS)
+		return -EINVAL;
+	mvdev->group2asid[group] = asid;
+	return 0;
+}
 static const struct vdpa_config_ops mlx5_vdpa_ops = {
 	.set_vq_address = mlx5_vdpa_set_vq_address,
 	.set_vq_num = mlx5_vdpa_set_vq_num,
 	.set_config = mlx5_vdpa_set_config,
 	.get_generation = mlx5_vdpa_get_generation,
 	.set_map = mlx5_vdpa_set_map,
+	.set_group_asid = mlx5_set_group_asid,
 	.free = mlx5_vdpa_free,
+	.suspend = mlx5_vdpa_suspend,
 };
 static int query_mtu(struct mlx5_core_dev *mdev, u16 *mtu)
 		mvq->index = i;
 		mvq->ndev = ndev;
 		mvq->fwqp.fw = true;
+		mvq->fw_state = MLX5_VIRTIO_NET_Q_OBJECT_NONE;
 	}
 	for (; i < ndev->mvdev.max_vqs; i++) {
 		mvq = &ndev->vqs[i];
 		switch (eqe->sub_type) {
 		case MLX5_PORT_CHANGE_SUBTYPE_DOWN:
 		case MLX5_PORT_CHANGE_SUBTYPE_ACTIVE:
+			down_read(&ndev->reslock);
+			if (!ndev->nb_registered) {
+				up_read(&ndev->reslock);
+				return NOTIFY_DONE;
+			}
 			wqent = kzalloc(sizeof(*wqent), GFP_ATOMIC);
-			if (!wqent)
+			if (!wqent) {
+				up_read(&ndev->reslock);
 				return NOTIFY_DONE;
+			}
 			wqent->mvdev = &ndev->mvdev;
 			INIT_WORK(&wqent->work, update_carrier);
 			queue_work(ndev->mvdev.wq, &wqent->work);
+			up_read(&ndev->reslock);
 			ret = NOTIFY_OK;
 			break;
 		default:
 	}
 	ndev = vdpa_alloc_device(struct mlx5_vdpa_net, mvdev.vdev, mdev->device, &mlx5_vdpa_ops,
-				 1, 1, name, false);
+				 MLX5_VDPA_NUMVQ_GROUPS, MLX5_VDPA_NUM_AS, name, false);
 	if (IS_ERR(ndev))
 		return PTR_ERR(ndev);
 	ndev->nb.notifier_call = event_handler;
 	mlx5_notifier_register(mdev, &ndev->nb);
+	ndev->nb_registered = true;
 	mvdev->vdev.mdev = &mgtdev->mgtdev;
 	err = _vdpa_register_device(&mvdev->vdev, max_vqs + 1);
 	if (err)
 	struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev);
 	struct workqueue_struct *wq;
-	mlx5_notifier_unregister(mvdev->mdev, &ndev->nb);
+	if (ndev->nb_registered) {
+		mlx5_notifier_unregister(mvdev->mdev, &ndev->nb);
+		ndev->nb_registered = false;
+	}
 	wq = mvdev->wq;
 	mvdev->wq = NULL;
 	destroy_workqueue(wq);

drivers/vdpa/vdpa.c+3−11

 		    config.mac))
 		return -EMSGSIZE;
-	val_u16 = le16_to_cpu(config.status);
+	val_u16 = __virtio16_to_cpu(true, config.status);
 	if (nla_put_u16(msg, VDPA_ATTR_DEV_NET_STATUS, val_u16))
 		return -EMSGSIZE;
-	val_u16 = le16_to_cpu(config.mtu);
+	val_u16 = __virtio16_to_cpu(true, config.mtu);
 	if (nla_put_u16(msg, VDPA_ATTR_DEV_NET_CFG_MTU, val_u16))
 		return -EMSGSIZE;
 {
 	u32 device_id;
 	void *hdr;
-	u8 status;
 	int err;
 	down_read(&vdev->cf_lock);
-	status = vdev->config->get_status(vdev);
-	if (!(status & VIRTIO_CONFIG_S_FEATURES_OK)) {
-		NL_SET_ERR_MSG_MOD(extack, "Features negotiation not completed");
-		err = -EAGAIN;
-		goto out;
-	}
 	hdr = genlmsg_put(msg, portid, seq, &vdpa_nl_family, flags,
 			  VDPA_CMD_DEV_CONFIG_GET);
 	if (!hdr) {
 	}
 	vdpa_get_config_unlocked(vdev, 0, &config, sizeof(config));
-	max_vqp = le16_to_cpu(config.max_virtqueue_pairs);
+	max_vqp = __virtio16_to_cpu(true, config.max_virtqueue_pairs);
 	if (nla_put_u16(msg, VDPA_ATTR_DEV_NET_CFG_MAX_VQP, max_vqp))
 		return -EMSGSIZE;

drivers/vdpa/vdpa_sim/vdpa_sim.c+16−2

 static int max_iotlb_entries = 2048;
 module_param(max_iotlb_entries, int, 0444);
 MODULE_PARM_DESC(max_iotlb_entries,
-		 "Maximum number of iotlb entries. 0 means unlimited. (default: 2048)");
+		 "Maximum number of iotlb entries for each address space. 0 means unlimited. (default: 2048)");
 #define VDPASIM_QUEUE_ALIGN PAGE_SIZE
 #define VDPASIM_QUEUE_MAX 256
 	for (i = 0; i < vdpasim->dev_attr.nas; i++)
 		vhost_iotlb_reset(&vdpasim->iommu[i]);
+	vdpasim->running = true;
 	spin_unlock(&vdpasim->iommu_lock);
 	vdpasim->features = 0;
 		goto err_iommu;
 	for (i = 0; i < vdpasim->dev_attr.nas; i++)
-		vhost_iotlb_init(&vdpasim->iommu[i], 0, 0);
+		vhost_iotlb_init(&vdpasim->iommu[i], max_iotlb_entries, 0);
 	vdpasim->buffer = kvmalloc(dev_attr->buffer_size, GFP_KERNEL);
 	if (!vdpasim->buffer)
 	return 0;
 }
+static int vdpasim_suspend(struct vdpa_device *vdpa)
+{
+	struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
+	spin_lock(&vdpasim->lock);
+	vdpasim->running = false;
+	spin_unlock(&vdpasim->lock);
+	return 0;
+}
 static size_t vdpasim_get_config_size(struct vdpa_device *vdpa)
 {
 	struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
 	.get_status             = vdpasim_get_status,
 	.set_status             = vdpasim_set_status,
 	.reset			= vdpasim_reset,
+	.suspend		= vdpasim_suspend,
 	.get_config_size        = vdpasim_get_config_size,
 	.get_config             = vdpasim_get_config,
 	.set_config             = vdpasim_set_config,
 	.get_status             = vdpasim_get_status,
 	.set_status             = vdpasim_set_status,
 	.reset			= vdpasim_reset,
+	.suspend		= vdpasim_suspend,
 	.get_config_size        = vdpasim_get_config_size,
 	.get_config             = vdpasim_get_config,
 	.set_config             = vdpasim_set_config,

drivers/vdpa/vdpa_sim/vdpa_sim.h+1−0

@@ -66,6 +66,7 @@ struct vdpasim { · 65 unmodified lines
66	u32 generation;	66	u32 generation;
67	u64 features;	67	u64 features;
68	u32 groups;	68	u32 groups;
		69	bool running;
69	/* spinlock to synchronize iommu table */	70	/* spinlock to synchronize iommu table */
70	spinlock_t iommu_lock;	71	spinlock_t iommu_lock;
71	};	72	};

drivers/vdpa/vdpa_sim/vdpa_sim_blk.c+147−29

@@ -25,31 +25,49 @@ · 24 unmodified lines
25	#define DRV_LICENSE "GPL v2"	25	#define DRV_LICENSE "GPL v2"
26		26
27	#define VDPASIM_BLK_FEATURES (VDPASIM_FEATURES \| \	27	#define VDPASIM_BLK_FEATURES (VDPASIM_FEATURES \| \
		28	(1ULL << VIRTIO_BLK_F_FLUSH) \| \
28	(1ULL << VIRTIO_BLK_F_SIZE_MAX) \| \	29	(1ULL << VIRTIO_BLK_F_SIZE_MAX) \| \
29	(1ULL << VIRTIO_BLK_F_SEG_MAX) \| \	30	(1ULL << VIRTIO_BLK_F_SEG_MAX) \| \
30	(1ULL << VIRTIO_BLK_F_BLK_SIZE) \| \	31	(1ULL << VIRTIO_BLK_F_BLK_SIZE) \| \
31	(1ULL << VIRTIO_BLK_F_TOPOLOGY) \| \	32	(1ULL << VIRTIO_BLK_F_TOPOLOGY) \| \
32	(1ULL << VIRTIO_BLK_F_MQ))	33	(1ULL << VIRTIO_BLK_F_MQ) \| \
		34	(1ULL << VIRTIO_BLK_F_DISCARD) \| \
		35	(1ULL << VIRTIO_BLK_F_WRITE_ZEROES))
33		36
34	#define VDPASIM_BLK_CAPACITY 0x40000	37	#define VDPASIM_BLK_CAPACITY 0x40000
35	#define VDPASIM_BLK_SIZE_MAX 0x1000	38	#define VDPASIM_BLK_SIZE_MAX 0x1000
36	#define VDPASIM_BLK_SEG_MAX 32	39	#define VDPASIM_BLK_SEG_MAX 32
		40	#define VDPASIM_BLK_DWZ_MAX_SECTORS UINT_MAX
		41
		42	/* 1 virtqueue, 1 address space, 1 virtqueue group */
37	#define VDPASIM_BLK_VQ_NUM 1	43	#define VDPASIM_BLK_VQ_NUM 1
		44	#define VDPASIM_BLK_AS_NUM 1
		45	#define VDPASIM_BLK_GROUP_NUM 1
38		46
39	static char vdpasim_blk_id[VIRTIO_BLK_ID_BYTES] = "vdpa_blk_sim";	47	static char vdpasim_blk_id[VIRTIO_BLK_ID_BYTES] = "vdpa_blk_sim";
40		48
41	static bool vdpasim_blk_check_range(u64 start_sector, size_t range_size)	49	static bool vdpasim_blk_check_range(struct vdpasim *vdpasim, u64 start_sector,
		50	u64 num_sectors, u64 max_sectors)
42	{	51	{
43	u64 range_sectors = range_size >> SECTOR_SHIFT;	52	if (start_sector > VDPASIM_BLK_CAPACITY) {
44		53	dev_dbg(&vdpasim->vdpa.dev,
45	if (range_size > VDPASIM_BLK_SIZE_MAX * VDPASIM_BLK_SEG_MAX)	54	"starting sector exceeds the capacity - start: 0x%llx capacity: 0x%x\n",
46	return false;	55	start_sector, VDPASIM_BLK_CAPACITY);
		56	}
47		57
48	if (start_sector > VDPASIM_BLK_CAPACITY)	58	if (num_sectors > max_sectors) {
		59	dev_dbg(&vdpasim->vdpa.dev,
		60	"number of sectors exceeds the max allowed in a request - num: 0x%llx max: 0x%llx\n",
		61	num_sectors, max_sectors);
49	return false;	62	return false;
		63	}
50		64
51	if (range_sectors > VDPASIM_BLK_CAPACITY - start_sector)	65	if (num_sectors > VDPASIM_BLK_CAPACITY - start_sector) {
		66	dev_dbg(&vdpasim->vdpa.dev,
		67	"request exceeds the capacity - start: 0x%llx num: 0x%llx capacity: 0x%x\n",
		68	start_sector, num_sectors, VDPASIM_BLK_CAPACITY);
52	return false;	69	return false;
		70	}
53		71
54	return true;	72	return true;
55	}	73	}
@@ -63,6 +81,7 @@ static bool vdpasim_blk_handle_req(struct vdpasim *vdpasim, · 7 unmodified lines
63	{	81	{
64	size_t pushed = 0, to_pull, to_push;	82	size_t pushed = 0, to_pull, to_push;
65	struct virtio_blk_outhdr hdr;	83	struct virtio_blk_outhdr hdr;
		84	bool handled = false;
66	ssize_t bytes;	85	ssize_t bytes;
67	loff_t offset;	86	loff_t offset;
68	u64 sector;	87	u64 sector;
@@ -76,14 +95,14 @@ static bool vdpasim_blk_handle_req(struct vdpasim *vdpasim, · 7 unmodified lines
76	return false;	95	return false;
77		96
78	if (vq->out_iov.used < 1 \|\| vq->in_iov.used < 1) {	97	if (vq->out_iov.used < 1 \|\| vq->in_iov.used < 1) {
79	dev_err(&vdpasim->vdpa.dev, "missing headers - out_iov: %u in_iov %u\n",	98	dev_dbg(&vdpasim->vdpa.dev, "missing headers - out_iov: %u in_iov %u\n",
80	vq->out_iov.used, vq->in_iov.used);	99	vq->out_iov.used, vq->in_iov.used);
81	return false;	100	goto err;
82	}	101	}
83		102
84	if (vq->in_iov.iov[vq->in_iov.used - 1].iov_len < 1) {	103	if (vq->in_iov.iov[vq->in_iov.used - 1].iov_len < 1) {
85	dev_err(&vdpasim->vdpa.dev, "request in header too short\n");	104	dev_dbg(&vdpasim->vdpa.dev, "request in header too short\n");
86	return false;	105	goto err;
87	}	106	}
88		107
89	/* The last byte is the status and we checked if the last iov has	108	/* The last byte is the status and we checked if the last iov has
@@ -96,8 +115,8 @@ static bool vdpasim_blk_handle_req(struct vdpasim *vdpasim, · 6 unmodified lines
96	bytes = vringh_iov_pull_iotlb(&vq->vring, &vq->out_iov, &hdr,	115	bytes = vringh_iov_pull_iotlb(&vq->vring, &vq->out_iov, &hdr,
97	sizeof(hdr));	116	sizeof(hdr));
98	if (bytes != sizeof(hdr)) {	117	if (bytes != sizeof(hdr)) {
99	dev_err(&vdpasim->vdpa.dev, "request out header too short\n");	118	dev_dbg(&vdpasim->vdpa.dev, "request out header too short\n");
100	return false;	119	goto err;
101	}	120	}
102		121
103	to_pull -= bytes;	122	to_pull -= bytes;
@@ -107,12 +126,20 @@ static bool vdpasim_blk_handle_req(struct vdpasim *vdpasim, · 3 unmodified lines
107	offset = sector << SECTOR_SHIFT;	126	offset = sector << SECTOR_SHIFT;
108	status = VIRTIO_BLK_S_OK;	127	status = VIRTIO_BLK_S_OK;
109		128
		129	if (type != VIRTIO_BLK_T_IN && type != VIRTIO_BLK_T_OUT &&
		130	sector != 0) {
		131	dev_dbg(&vdpasim->vdpa.dev,
		132	"sector must be 0 for %u request - sector: 0x%llx\n",
		133	type, sector);
		134	status = VIRTIO_BLK_S_IOERR;
		135	goto err_status;
		136	}
		137
110	switch (type) {	138	switch (type) {
111	case VIRTIO_BLK_T_IN:	139	case VIRTIO_BLK_T_IN:
112	if (!vdpasim_blk_check_range(sector, to_push)) {	140	if (!vdpasim_blk_check_range(vdpasim, sector,
113	dev_err(&vdpasim->vdpa.dev,	141	to_push >> SECTOR_SHIFT,
114	"reading over the capacity - offset: 0x%llx len: 0x%zx\n",	142	VDPASIM_BLK_SIZE_MAX * VDPASIM_BLK_SEG_MAX)) {
115	offset, to_push);
116	status = VIRTIO_BLK_S_IOERR;	143	status = VIRTIO_BLK_S_IOERR;
117	break;	144	break;
118	}	145	}
@@ -121,7 +148,7 @@ static bool vdpasim_blk_handle_req(struct vdpasim *vdpasim, · 2 unmodified lines
121	vdpasim->buffer + offset,	148	vdpasim->buffer + offset,
122	to_push);	149	to_push);
123	if (bytes < 0) {	150	if (bytes < 0) {
124	dev_err(&vdpasim->vdpa.dev,	151	dev_dbg(&vdpasim->vdpa.dev,
125	"vringh_iov_push_iotlb() error: %zd offset: 0x%llx len: 0x%zx\n",	152	"vringh_iov_push_iotlb() error: %zd offset: 0x%llx len: 0x%zx\n",
126	bytes, offset, to_push);	153	bytes, offset, to_push);
127	status = VIRTIO_BLK_S_IOERR;	154	status = VIRTIO_BLK_S_IOERR;
@@ -132,10 +159,9 @@ static bool vdpasim_blk_handle_req(struct vdpasim *vdpasim, · 4 unmodified lines
132	break;	159	break;
133		160
134	case VIRTIO_BLK_T_OUT:	161	case VIRTIO_BLK_T_OUT:
135	if (!vdpasim_blk_check_range(sector, to_pull)) {	162	if (!vdpasim_blk_check_range(vdpasim, sector,
136	dev_err(&vdpasim->vdpa.dev,	163	to_pull >> SECTOR_SHIFT,
137	"writing over the capacity - offset: 0x%llx len: 0x%zx\n",	164	VDPASIM_BLK_SIZE_MAX * VDPASIM_BLK_SEG_MAX)) {
138	offset, to_pull);
139	status = VIRTIO_BLK_S_IOERR;	165	status = VIRTIO_BLK_S_IOERR;
140	break;	166	break;
141	}	167	}
@@ -144,7 +170,7 @@ static bool vdpasim_blk_handle_req(struct vdpasim *vdpasim, · 2 unmodified lines
144	vdpasim->buffer + offset,	170	vdpasim->buffer + offset,
145	to_pull);	171	to_pull);
146	if (bytes < 0) {	172	if (bytes < 0) {
147	dev_err(&vdpasim->vdpa.dev,	173	dev_dbg(&vdpasim->vdpa.dev,
148	"vringh_iov_pull_iotlb() error: %zd offset: 0x%llx len: 0x%zx\n",	174	"vringh_iov_pull_iotlb() error: %zd offset: 0x%llx len: 0x%zx\n",
149	bytes, offset, to_pull);	175	bytes, offset, to_pull);
150	status = VIRTIO_BLK_S_IOERR;	176	status = VIRTIO_BLK_S_IOERR;
@@ -157,7 +183,7 @@ static bool vdpasim_blk_handle_req(struct vdpasim *vdpasim, · 6 unmodified lines
157	vdpasim_blk_id,	183	vdpasim_blk_id,
158	VIRTIO_BLK_ID_BYTES);	184	VIRTIO_BLK_ID_BYTES);
159	if (bytes < 0) {	185	if (bytes < 0) {
160	dev_err(&vdpasim->vdpa.dev,	186	dev_dbg(&vdpasim->vdpa.dev,
161	"vringh_iov_push_iotlb() error: %zd\n", bytes);	187	"vringh_iov_push_iotlb() error: %zd\n", bytes);
162	status = VIRTIO_BLK_S_IOERR;	188	status = VIRTIO_BLK_S_IOERR;
163	break;	189	break;
@@ -166,13 +192,76 @@ static bool vdpasim_blk_handle_req(struct vdpasim *vdpasim, · 2 unmodified lines
166	pushed += bytes;	192	pushed += bytes;
167	break;	193	break;
168		194
		195	case VIRTIO_BLK_T_FLUSH:
		196	/* nothing to do */
		197	break;
		198
		199	case VIRTIO_BLK_T_DISCARD:
		200	case VIRTIO_BLK_T_WRITE_ZEROES: {
		201	struct virtio_blk_discard_write_zeroes range;
		202	u32 num_sectors, flags;
		203
		204	if (to_pull != sizeof(range)) {
		205	dev_dbg(&vdpasim->vdpa.dev,
		206	"discard/write_zeroes header len: 0x%zx [expected: 0x%zx]\n",
		207	to_pull, sizeof(range));
		208	status = VIRTIO_BLK_S_IOERR;
		209	break;
		210	}
		211
		212	bytes = vringh_iov_pull_iotlb(&vq->vring, &vq->out_iov, &range,
		213	to_pull);
		214	if (bytes < 0) {
		215	dev_dbg(&vdpasim->vdpa.dev,
		216	"vringh_iov_pull_iotlb() error: %zd offset: 0x%llx len: 0x%zx\n",
		217	bytes, offset, to_pull);
		218	status = VIRTIO_BLK_S_IOERR;
		219	break;
		220	}
		221
		222	sector = le64_to_cpu(range.sector);
		223	offset = sector << SECTOR_SHIFT;
		224	num_sectors = le32_to_cpu(range.num_sectors);
		225	flags = le32_to_cpu(range.flags);
		226
		227	if (type == VIRTIO_BLK_T_DISCARD && flags != 0) {
		228	dev_dbg(&vdpasim->vdpa.dev,
		229	"discard unexpected flags set - flags: 0x%x\n",
		230	flags);
		231	status = VIRTIO_BLK_S_UNSUPP;
		232	break;
		233	}
		234
		235	if (type == VIRTIO_BLK_T_WRITE_ZEROES &&
		236	flags & ~VIRTIO_BLK_WRITE_ZEROES_FLAG_UNMAP) {
		237	dev_dbg(&vdpasim->vdpa.dev,
		238	"write_zeroes unexpected flags set - flags: 0x%x\n",
		239	flags);
		240	status = VIRTIO_BLK_S_UNSUPP;
		241	break;
		242	}
		243
		244	if (!vdpasim_blk_check_range(vdpasim, sector, num_sectors,
		245	VDPASIM_BLK_DWZ_MAX_SECTORS)) {
		246	status = VIRTIO_BLK_S_IOERR;
		247	break;
		248	}
		249
		250	if (type == VIRTIO_BLK_T_WRITE_ZEROES) {
		251	memset(vdpasim->buffer + offset, 0,
		252	num_sectors << SECTOR_SHIFT);
		253	}
		254
		255	break;
		256	}
169	default:	257	default:
170	dev_warn(&vdpasim->vdpa.dev,	258	dev_dbg(&vdpasim->vdpa.dev,
171	"Unsupported request type %d\n", type);	259	"Unsupported request type %d\n", type);
172	status = VIRTIO_BLK_S_IOERR;	260	status = VIRTIO_BLK_S_IOERR;
173	break;	261	break;
174	}	262	}
175		263
		264	err_status:
176	/* If some operations fail, we need to skip the remaining bytes	265	/* If some operations fail, we need to skip the remaining bytes
177	* to put the status in the last byte	266	* to put the status in the last byte
178	*/	267	*/
@@ -182,21 +271,25 @@ static bool vdpasim_blk_handle_req(struct vdpasim *vdpasim, · 3 unmodified lines
182	/* Last byte is the status */	271	/* Last byte is the status */
183	bytes = vringh_iov_push_iotlb(&vq->vring, &vq->in_iov, &status, 1);	272	bytes = vringh_iov_push_iotlb(&vq->vring, &vq->in_iov, &status, 1);
184	if (bytes != 1)	273	if (bytes != 1)
185	return false;	274	goto err;
186		275
187	pushed += bytes;	276	pushed += bytes;
188		277
189	/* Make sure data is wrote before advancing index */	278	/* Make sure data is wrote before advancing index */
190	smp_wmb();	279	smp_wmb();
191		280
		281	handled = true;
		282
		283	err:
192	vringh_complete_iotlb(&vq->vring, vq->head, pushed);	284	vringh_complete_iotlb(&vq->vring, vq->head, pushed);
193		285
194	return true;	286	return handled;
195	}	287	}
196		288
197	static void vdpasim_blk_work(struct work_struct *work)	289	static void vdpasim_blk_work(struct work_struct *work)
198	{	290	{
199	struct vdpasim *vdpasim = container_of(work, struct vdpasim, work);	291	struct vdpasim *vdpasim = container_of(work, struct vdpasim, work);
		292	bool reschedule = false;
200	int i;	293	int i;
201		294
202	spin_lock(&vdpasim->lock);	295	spin_lock(&vdpasim->lock);
@@ -204,8 +297,12 @@ static void vdpasim_blk_work(struct work_struct *work) · 1 unmodified line
204	if (!(vdpasim->status & VIRTIO_CONFIG_S_DRIVER_OK))	297	if (!(vdpasim->status & VIRTIO_CONFIG_S_DRIVER_OK))
205	goto out;	298	goto out;
206		299
		300	if (!vdpasim->running)
		301	goto out;
		302
207	for (i = 0; i < VDPASIM_BLK_VQ_NUM; i++) {	303	for (i = 0; i < VDPASIM_BLK_VQ_NUM; i++) {
208	struct vdpasim_virtqueue *vq = &vdpasim->vqs[i];	304	struct vdpasim_virtqueue *vq = &vdpasim->vqs[i];
		305	int reqs = 0;
209		306
210	if (!vq->ready)	307	if (!vq->ready)
211	continue;	308	continue;
@@ -218,10 +315,18 @@ static void vdpasim_blk_work(struct work_struct *work) · 6 unmodified lines
218	if (vringh_need_notify_iotlb(&vq->vring) > 0)	315	if (vringh_need_notify_iotlb(&vq->vring) > 0)
219	vringh_notify(&vq->vring);	316	vringh_notify(&vq->vring);
220	local_bh_enable();	317	local_bh_enable();
		318
		319	if (++reqs > 4) {
		320	reschedule = true;
		321	break;
		322	}
221	}	323	}
222	}	324	}
223	out:	325	out:
224	spin_unlock(&vdpasim->lock);	326	spin_unlock(&vdpasim->lock);
		327
		328	if (reschedule)
		329	schedule_work(&vdpasim->work);
225	}	330	}
226		331
227	static void vdpasim_blk_get_config(struct vdpasim vdpasim, void config)	332	static void vdpasim_blk_get_config(struct vdpasim vdpasim, void config)
@@ -237,6 +342,17 @@ static void vdpasim_blk_get_config(struct vdpasim vdpasim, void config) · 9 unmodified lines
237	blk_config->min_io_size = cpu_to_vdpasim16(vdpasim, 1);	342	blk_config->min_io_size = cpu_to_vdpasim16(vdpasim, 1);
238	blk_config->opt_io_size = cpu_to_vdpasim32(vdpasim, 1);	343	blk_config->opt_io_size = cpu_to_vdpasim32(vdpasim, 1);
239	blk_config->blk_size = cpu_to_vdpasim32(vdpasim, SECTOR_SIZE);	344	blk_config->blk_size = cpu_to_vdpasim32(vdpasim, SECTOR_SIZE);
		345	/* VIRTIO_BLK_F_DISCARD */
		346	blk_config->discard_sector_alignment =
		347	cpu_to_vdpasim32(vdpasim, SECTOR_SIZE);
		348	blk_config->max_discard_sectors =
		349	cpu_to_vdpasim32(vdpasim, VDPASIM_BLK_DWZ_MAX_SECTORS);
		350	blk_config->max_discard_seg = cpu_to_vdpasim32(vdpasim, 1);
		351	/* VIRTIO_BLK_F_WRITE_ZEROES */
		352	blk_config->max_write_zeroes_sectors =
		353	cpu_to_vdpasim32(vdpasim, VDPASIM_BLK_DWZ_MAX_SECTORS);
		354	blk_config->max_write_zeroes_seg = cpu_to_vdpasim32(vdpasim, 1);
		355
240	}	356	}
241		357
242	static void vdpasim_blk_mgmtdev_release(struct device *dev)	358	static void vdpasim_blk_mgmtdev_release(struct device *dev)
@@ -260,6 +376,8 @@ static int vdpasim_blk_dev_add(struct vdpa_mgmt_dev mdev, const char name, · 17 unmodified lines
260	dev_attr.id = VIRTIO_ID_BLOCK;	376	dev_attr.id = VIRTIO_ID_BLOCK;
261	dev_attr.supported_features = VDPASIM_BLK_FEATURES;	377	dev_attr.supported_features = VDPASIM_BLK_FEATURES;
262	dev_attr.nvqs = VDPASIM_BLK_VQ_NUM;	378	dev_attr.nvqs = VDPASIM_BLK_VQ_NUM;
		379	dev_attr.ngroups = VDPASIM_BLK_GROUP_NUM;
		380	dev_attr.nas = VDPASIM_BLK_AS_NUM;
263	dev_attr.config_size = sizeof(struct virtio_blk_config);	381	dev_attr.config_size = sizeof(struct virtio_blk_config);
264	dev_attr.get_config = vdpasim_blk_get_config;	382	dev_attr.get_config = vdpasim_blk_get_config;
265	dev_attr.work_fn = vdpasim_blk_work;	383	dev_attr.work_fn = vdpasim_blk_work;

drivers/vdpa/vdpa_sim/vdpa_sim_net.c+3−0

@@ -154,6 +154,9 @@ static void vdpasim_net_work(struct work_struct *work) · 153 unmodified lines
154		154
155	spin_lock(&vdpasim->lock);	155	spin_lock(&vdpasim->lock);
156		156
		157	if (!vdpasim->running)
		158	goto out;
		159
157	if (!(vdpasim->status & VIRTIO_CONFIG_S_DRIVER_OK))	160	if (!(vdpasim->status & VIRTIO_CONFIG_S_DRIVER_OK))
158	goto out;	161	goto out;
159		162

drivers/vdpa/vdpa_user/iova_domain.c+89−13

 {
 	unsigned long pfn = PFN_DOWN(orig);
 	unsigned int offset = offset_in_page(orig);
-	char *buffer;
+	struct page *page;
 	unsigned int sz = 0;
 	while (size) {
 		sz = min_t(size_t, PAGE_SIZE - offset, size);
-		buffer = kmap_atomic(pfn_to_page(pfn));
+		page = pfn_to_page(pfn);
 		if (dir == DMA_TO_DEVICE)
-			memcpy(addr, buffer + offset, sz);
+			memcpy_from_page(addr, page, offset, sz);
 		else
-			memcpy(buffer + offset, addr, sz);
+			memcpy_to_page(page, offset, addr, sz);
-		kunmap_atomic(buffer);
 		size -= sz;
 		pfn++;
 			    map->orig_phys == INVALID_PHYS_ADDR))
 			return;
-		addr = page_address(map->bounce_page) + offset;
+		addr = kmap_local_page(map->bounce_page);
-		do_bounce(map->orig_phys + offset, addr, sz, dir);
+		do_bounce(map->orig_phys + offset, addr + offset, sz, dir);
+		kunmap_local(addr);
 		size -= sz;
 		iova += sz;
 	}
 	struct vduse_bounce_map *map;
 	struct page *page = NULL;
-	spin_lock(&domain->iotlb_lock);
+	read_lock(&domain->bounce_lock);
 	map = &domain->bounce_maps[iova >> PAGE_SHIFT];
-	if (!map->bounce_page)
+	if (domain->user_bounce_pages || !map->bounce_page)
 		goto out;
 	page = map->bounce_page;
 	get_page(page);
 out:
-	spin_unlock(&domain->iotlb_lock);
+	read_unlock(&domain->bounce_lock);
 	return page;
 }
 static void
-vduse_domain_free_bounce_pages(struct vduse_iova_domain *domain)
+vduse_domain_free_kernel_bounce_pages(struct vduse_iova_domain *domain)
 {
 	struct vduse_bounce_map *map;
 	unsigned long pfn, bounce_pfns;
 	}
 }
+int vduse_domain_add_user_bounce_pages(struct vduse_iova_domain *domain,
+				       struct page **pages, int count)
+{
+	struct vduse_bounce_map *map;
+	int i, ret;
+	/* Now we don't support partial mapping */
+	if (count != (domain->bounce_size >> PAGE_SHIFT))
+		return -EINVAL;
+	write_lock(&domain->bounce_lock);
+	ret = -EEXIST;
+	if (domain->user_bounce_pages)
+		goto out;
+	for (i = 0; i < count; i++) {
+		map = &domain->bounce_maps[i];
+		if (map->bounce_page) {
+			/* Copy kernel page to user page if it's in use */
+			if (map->orig_phys != INVALID_PHYS_ADDR)
+				memcpy_to_page(pages[i], 0,
+					       page_address(map->bounce_page),
+					       PAGE_SIZE);
+			__free_page(map->bounce_page);
+		}
+		map->bounce_page = pages[i];
+		get_page(pages[i]);
+	}
+	domain->user_bounce_pages = true;
+	ret = 0;
+out:
+	write_unlock(&domain->bounce_lock);
+	return ret;
+}
+void vduse_domain_remove_user_bounce_pages(struct vduse_iova_domain *domain)
+{
+	struct vduse_bounce_map *map;
+	unsigned long i, count;
+	write_lock(&domain->bounce_lock);
+	if (!domain->user_bounce_pages)
+		goto out;
+	count = domain->bounce_size >> PAGE_SHIFT;
+	for (i = 0; i < count; i++) {
+		struct page *page = NULL;
+		map = &domain->bounce_maps[i];
+		if (WARN_ON(!map->bounce_page))
+			continue;
+		/* Copy user page to kernel page if it's in use */
+		if (map->orig_phys != INVALID_PHYS_ADDR) {
+			page = alloc_page(GFP_ATOMIC | __GFP_NOFAIL);
+			memcpy_from_page(page_address(page),
+					 map->bounce_page, 0, PAGE_SIZE);
+		}
+		put_page(map->bounce_page);
+		map->bounce_page = page;
+	}
+	domain->user_bounce_pages = false;
+out:
+	write_unlock(&domain->bounce_lock);
+}
 void vduse_domain_reset_bounce_map(struct vduse_iova_domain *domain)
 {
 	if (!domain->bounce_map)
 	if (vduse_domain_init_bounce_map(domain))
 		goto err;
+	read_lock(&domain->bounce_lock);
 	if (vduse_domain_map_bounce_page(domain, (u64)iova, (u64)size, pa))
-		goto err;
+		goto err_unlock;
 	if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL)
 		vduse_domain_bounce(domain, iova, size, DMA_TO_DEVICE);
+	read_unlock(&domain->bounce_lock);
 	return iova;
+err_unlock:
+	read_unlock(&domain->bounce_lock);
 err:
 	vduse_domain_free_iova(iovad, iova, size);
 	return DMA_MAPPING_ERROR;
 {
 	struct iova_domain *iovad = &domain->stream_iovad;
+	read_lock(&domain->bounce_lock);
 	if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
 		vduse_domain_bounce(domain, dma_addr, size, DMA_FROM_DEVICE);
 	vduse_domain_unmap_bounce_page(domain, (u64)dma_addr, (u64)size);
+	read_unlock(&domain->bounce_lock);
 	vduse_domain_free_iova(iovad, dma_addr, size);
 }
 	spin_lock(&domain->iotlb_lock);
 	vduse_iotlb_del_range(domain, 0, ULLONG_MAX);
-	vduse_domain_free_bounce_pages(domain);
+	vduse_domain_remove_user_bounce_pages(domain);
+	vduse_domain_free_kernel_bounce_pages(domain);
 	spin_unlock(&domain->iotlb_lock);
 	put_iova_domain(&domain->stream_iovad);
 	put_iova_domain(&domain->consistent_iovad);
 		goto err_file;
 	domain->file = file;
+	rwlock_init(&domain->bounce_lock);
 	spin_lock_init(&domain->iotlb_lock);
 	init_iova_domain(&domain->stream_iovad,
 			PAGE_SIZE, IOVA_START_PFN);

drivers/vdpa/vdpa_user/iova_domain.h+8−0

 #include <linux/iova.h>
 #include <linux/dma-mapping.h>
 #include <linux/vhost_iotlb.h>
+#include <linux/rwlock.h>
 #define IOVA_START_PFN 1
 	struct vhost_iotlb *iotlb;
 	spinlock_t iotlb_lock;
 	struct file *file;
+	bool user_bounce_pages;
+	rwlock_t bounce_lock;
 };
 int vduse_domain_set_map(struct vduse_iova_domain *domain,
 void vduse_domain_reset_bounce_map(struct vduse_iova_domain *domain);
+int vduse_domain_add_user_bounce_pages(struct vduse_iova_domain *domain,
+				       struct page **pages, int count);
+void vduse_domain_remove_user_bounce_pages(struct vduse_iova_domain *domain);
 void vduse_domain_destroy(struct vduse_iova_domain *domain);
 struct vduse_iova_domain *vduse_domain_create(unsigned long iova_limit,

drivers/vdpa/vdpa_user/vduse_dev.c+180−0

@@ -21,6 +21,8 @@ · 20 unmodified lines
21	#include <linux/uio.h>	21	#include <linux/uio.h>
22	#include <linux/vdpa.h>	22	#include <linux/vdpa.h>
23	#include <linux/nospec.h>	23	#include <linux/nospec.h>
		24	#include <linux/vmalloc.h>
		25	#include <linux/sched/mm.h>
24	#include <uapi/linux/vduse.h>	26	#include <uapi/linux/vduse.h>
25	#include <uapi/linux/vdpa.h>	27	#include <uapi/linux/vdpa.h>
26	#include <uapi/linux/virtio_config.h>	28	#include <uapi/linux/virtio_config.h>
@@ -64,6 +66,13 @@ struct vduse_vdpa { · 37 unmodified lines
64	struct vduse_dev *dev;	66	struct vduse_dev *dev;
65	};	67	};
66		68
		69	struct vduse_umem {
		70	unsigned long iova;
		71	unsigned long npages;
		72	struct page **pages;
		73	struct mm_struct *mm;
		74	};
		75
67	struct vduse_dev {	76	struct vduse_dev {
68	struct vduse_vdpa *vdev;	77	struct vduse_vdpa *vdev;
69	struct device *dev;	78	struct device *dev;
@@ -95,6 +104,8 @@ struct vduse_dev { · 25 unmodified lines
95	u8 status;	104	u8 status;
96	u32 vq_num;	105	u32 vq_num;
97	u32 vq_align;	106	u32 vq_align;
		107	struct vduse_umem *umem;
		108	struct mutex mem_lock;
98	};	109	};
99		110
100	struct vduse_dev_msg {	111	struct vduse_dev_msg {
@@ -917,6 +928,102 @@ unlock: · 816 unmodified lines
917	return ret;	928	return ret;
918	}	929	}
919		930
		931	static int vduse_dev_dereg_umem(struct vduse_dev *dev,
		932	u64 iova, u64 size)
		933	{
		934	int ret;
		935
		936	mutex_lock(&dev->mem_lock);
		937	ret = -ENOENT;
		938	if (!dev->umem)
		939	goto unlock;
		940
		941	ret = -EINVAL;
		942	if (dev->umem->iova != iova \|\| size != dev->domain->bounce_size)
		943	goto unlock;
		944
		945	vduse_domain_remove_user_bounce_pages(dev->domain);
		946	unpin_user_pages_dirty_lock(dev->umem->pages,
		947	dev->umem->npages, true);
		948	atomic64_sub(dev->umem->npages, &dev->umem->mm->pinned_vm);
		949	mmdrop(dev->umem->mm);
		950	vfree(dev->umem->pages);
		951	kfree(dev->umem);
		952	dev->umem = NULL;
		953	ret = 0;
		954	unlock:
		955	mutex_unlock(&dev->mem_lock);
		956	return ret;
		957	}
		958
		959	static int vduse_dev_reg_umem(struct vduse_dev *dev,
		960	u64 iova, u64 uaddr, u64 size)
		961	{
		962	struct page **page_list = NULL;
		963	struct vduse_umem *umem = NULL;
		964	long pinned = 0;
		965	unsigned long npages, lock_limit;
		966	int ret;
		967
		968	if (!dev->domain->bounce_map \|\|
		969	size != dev->domain->bounce_size \|\|
		970	iova != 0 \|\| uaddr & ~PAGE_MASK)
		971	return -EINVAL;
		972
		973	mutex_lock(&dev->mem_lock);
		974	ret = -EEXIST;
		975	if (dev->umem)
		976	goto unlock;
		977
		978	ret = -ENOMEM;
		979	npages = size >> PAGE_SHIFT;
		980	page_list = __vmalloc(array_size(npages, sizeof(struct page *)),
		981	GFP_KERNEL_ACCOUNT);
		982	umem = kzalloc(sizeof(*umem), GFP_KERNEL);
		983	if (!page_list \|\| !umem)
		984	goto unlock;
		985
		986	mmap_read_lock(current->mm);
		987
		988	lock_limit = PFN_DOWN(rlimit(RLIMIT_MEMLOCK));
		989	if (npages + atomic64_read(&current->mm->pinned_vm) > lock_limit)
		990	goto out;
		991
		992	pinned = pin_user_pages(uaddr, npages, FOLL_LONGTERM \| FOLL_WRITE,
		993	page_list, NULL);
		994	if (pinned != npages) {
		995	ret = pinned < 0 ? pinned : -ENOMEM;
		996	goto out;
		997	}
		998
		999	ret = vduse_domain_add_user_bounce_pages(dev->domain,
		1000	page_list, pinned);
		1001	if (ret)
		1002	goto out;
		1003
		1004	atomic64_add(npages, &current->mm->pinned_vm);
		1005
		1006	umem->pages = page_list;
		1007	umem->npages = pinned;
		1008	umem->iova = iova;
		1009	umem->mm = current->mm;
		1010	mmgrab(current->mm);
		1011
		1012	dev->umem = umem;
		1013	out:
		1014	if (ret && pinned > 0)
		1015	unpin_user_pages(page_list, pinned);
		1016
		1017	mmap_read_unlock(current->mm);
		1018	unlock:
		1019	if (ret) {
		1020	vfree(page_list);
		1021	kfree(umem);
		1022	}
		1023	mutex_unlock(&dev->mem_lock);
		1024	return ret;
		1025	}
		1026
920	static long vduse_dev_ioctl(struct file *file, unsigned int cmd,	1027	static long vduse_dev_ioctl(struct file *file, unsigned int cmd,
921	unsigned long arg)	1028	unsigned long arg)
922	{	1029	{
@@ -1089,6 +1196,77 @@ static long vduse_dev_ioctl(struct file *file, unsigned int cmd, · 166 unmodified lines
1089	ret = vduse_dev_queue_irq_work(dev, &dev->vqs[index].inject);	1196	ret = vduse_dev_queue_irq_work(dev, &dev->vqs[index].inject);
1090	break;	1197	break;
1091	}	1198	}
		1199	case VDUSE_IOTLB_REG_UMEM: {
		1200	struct vduse_iova_umem umem;
		1201
		1202	ret = -EFAULT;
		1203	if (copy_from_user(&umem, argp, sizeof(umem)))
		1204	break;
		1205
		1206	ret = -EINVAL;
		1207	if (!is_mem_zero((const char *)umem.reserved,
		1208	sizeof(umem.reserved)))
		1209	break;
		1210
		1211	ret = vduse_dev_reg_umem(dev, umem.iova,
		1212	umem.uaddr, umem.size);
		1213	break;
		1214	}
		1215	case VDUSE_IOTLB_DEREG_UMEM: {
		1216	struct vduse_iova_umem umem;
		1217
		1218	ret = -EFAULT;
		1219	if (copy_from_user(&umem, argp, sizeof(umem)))
		1220	break;
		1221
		1222	ret = -EINVAL;
		1223	if (!is_mem_zero((const char *)umem.reserved,
		1224	sizeof(umem.reserved)))
		1225	break;
		1226
		1227	ret = vduse_dev_dereg_umem(dev, umem.iova,
		1228	umem.size);
		1229	break;
		1230	}
		1231	case VDUSE_IOTLB_GET_INFO: {
		1232	struct vduse_iova_info info;
		1233	struct vhost_iotlb_map *map;
		1234	struct vduse_iova_domain *domain = dev->domain;
		1235
		1236	ret = -EFAULT;
		1237	if (copy_from_user(&info, argp, sizeof(info)))
		1238	break;
		1239
		1240	ret = -EINVAL;
		1241	if (info.start > info.last)
		1242	break;
		1243
		1244	if (!is_mem_zero((const char *)info.reserved,
		1245	sizeof(info.reserved)))
		1246	break;
		1247
		1248	spin_lock(&domain->iotlb_lock);
		1249	map = vhost_iotlb_itree_first(domain->iotlb,
		1250	info.start, info.last);
		1251	if (map) {
		1252	info.start = map->start;
		1253	info.last = map->last;
		1254	info.capability = 0;
		1255	if (domain->bounce_map && map->start == 0 &&
		1256	map->last == domain->bounce_size - 1)
		1257	info.capability \|= VDUSE_IOVA_CAP_UMEM;
		1258	}
		1259	spin_unlock(&domain->iotlb_lock);
		1260	if (!map)
		1261	break;
		1262
		1263	ret = -EFAULT;
		1264	if (copy_to_user(argp, &info, sizeof(info)))
		1265	break;
		1266
		1267	ret = 0;
		1268	break;
		1269	}
1092	default:	1270	default:
1093	ret = -ENOIOCTLCMD;	1271	ret = -ENOIOCTLCMD;
1094	break;	1272	break;
@@ -1101,6 +1279,7 @@ static int vduse_dev_release(struct inode inode, struct file file) · 6 unmodified lines
1101	{	1279	{
1102	struct vduse_dev *dev = file->private_data;	1280	struct vduse_dev *dev = file->private_data;
1103		1281
		1282	vduse_dev_dereg_umem(dev, 0, dev->domain->bounce_size);
1104	spin_lock(&dev->msg_lock);	1283	spin_lock(&dev->msg_lock);
1105	/* Make sure the inflight messages can processed after reconncection */	1284	/* Make sure the inflight messages can processed after reconncection */
1106	list_splice_init(&dev->recv_list, &dev->send_list);	1285	list_splice_init(&dev->recv_list, &dev->send_list);
@@ -1163,6 +1342,7 @@ static struct vduse_dev *vduse_dev_create(void) · 56 unmodified lines
1163	return NULL;	1342	return NULL;
1164		1343
1165	mutex_init(&dev->lock);	1344	mutex_init(&dev->lock);
		1345	mutex_init(&dev->mem_lock);
1166	spin_lock_init(&dev->msg_lock);	1346	spin_lock_init(&dev->msg_lock);
1167	INIT_LIST_HEAD(&dev->send_list);	1347	INIT_LIST_HEAD(&dev->send_list);
1168	INIT_LIST_HEAD(&dev->recv_list);	1348	INIT_LIST_HEAD(&dev->recv_list);

drivers/vhost/scsi.c+61−24

 };
 #define VHOST_SCSI_MAX_TARGET	256
-#define VHOST_SCSI_MAX_VQ	128
+#define VHOST_SCSI_MAX_IO_VQ	1024
 #define VHOST_SCSI_MAX_EVENT	128
+static unsigned vhost_scsi_max_io_vqs = 128;
+module_param_named(max_io_vqs, vhost_scsi_max_io_vqs, uint, 0644);
+MODULE_PARM_DESC(max_io_vqs, "Set the max number of IO virtqueues a vhost scsi device can support. The default is 128. The max is 1024.");
 struct vhost_scsi_virtqueue {
 	struct vhost_virtqueue vq;
 	/*
 	char vs_vhost_wwpn[TRANSPORT_IQN_LEN];
 	struct vhost_dev dev;
-	struct vhost_scsi_virtqueue vqs[VHOST_SCSI_MAX_VQ];
+	struct vhost_scsi_virtqueue *vqs;
+	unsigned long *compl_bitmap;
+	struct vhost_scsi_inflight **old_inflight;
 	struct vhost_work vs_completion_work; /* cmd completion work item */
 	struct llist_head vs_completion_list; /* cmd completion queue */
 	struct vhost_virtqueue *vq;
 	int idx, i;
-	for (i = 0; i < VHOST_SCSI_MAX_VQ; i++) {
+	for (i = 0; i < vs->dev.nvqs;  i++) {
 		vq = &vs->vqs[i].vq;
 		mutex_lock(&vq->mutex);
 {
 	struct vhost_scsi *vs = container_of(work, struct vhost_scsi,
 					vs_completion_work);
-	DECLARE_BITMAP(signal, VHOST_SCSI_MAX_VQ);
 	struct virtio_scsi_cmd_resp v_rsp;
 	struct vhost_scsi_cmd *cmd, *t;
 	struct llist_node *llnode;
 	struct iov_iter iov_iter;
 	int ret, vq;
-	bitmap_zero(signal, VHOST_SCSI_MAX_VQ);
+	bitmap_zero(vs->compl_bitmap, vs->dev.nvqs);
 	llnode = llist_del_all(&vs->vs_completion_list);
 	llist_for_each_entry_safe(cmd, t, llnode, tvc_completion_list) {
 		se_cmd = &cmd->tvc_se_cmd;
 			vhost_add_used(cmd->tvc_vq, cmd->tvc_vq_desc, 0);
 			q = container_of(cmd->tvc_vq, struct vhost_scsi_virtqueue, vq);
 			vq = q - vs->vqs;
-			__set_bit(vq, signal);
+			__set_bit(vq, vs->compl_bitmap);
 		} else
 			pr_err("Faulted on virtio_scsi_cmd_resp\n");
 	}
 	vq = -1;
-	while ((vq = find_next_bit(signal, VHOST_SCSI_MAX_VQ, vq + 1))
+	while ((vq = find_next_bit(vs->compl_bitmap, vs->dev.nvqs, vq + 1))
-		< VHOST_SCSI_MAX_VQ)
+		< vs->dev.nvqs)
 		vhost_signal(&vs->dev, &vs->vqs[vq].vq);
 }
 /* Callers must hold dev mutex */
 static void vhost_scsi_flush(struct vhost_scsi *vs)
 {
-	struct vhost_scsi_inflight *old_inflight[VHOST_SCSI_MAX_VQ];
 	int i;
 	/* Init new inflight and remember the old inflight */
-	vhost_scsi_init_inflight(vs, old_inflight);
+	vhost_scsi_init_inflight(vs, vs->old_inflight);
 	/*
 	 * The inflight->kref was initialized to 1. We decrement it here to
 	 * indicate the start of the flush operation so that it will reach 0
 	 * when all the reqs are finished.
 	 */
-	for (i = 0; i < VHOST_SCSI_MAX_VQ; i++)
+	for (i = 0; i < vs->dev.nvqs; i++)
-		kref_put(&old_inflight[i]->kref, vhost_scsi_done_inflight);
+		kref_put(&vs->old_inflight[i]->kref, vhost_scsi_done_inflight);
 	/* Flush both the vhost poll and vhost work */
 	vhost_dev_flush(&vs->dev);
 	/* Wait for all reqs issued before the flush to be finished */
-	for (i = 0; i < VHOST_SCSI_MAX_VQ; i++)
+	for (i = 0; i < vs->dev.nvqs; i++)
-		wait_for_completion(&old_inflight[i]->comp);
+		wait_for_completion(&vs->old_inflight[i]->comp);
 }
 static void vhost_scsi_destroy_vq_cmds(struct vhost_virtqueue *vq)
 		memcpy(vs->vs_vhost_wwpn, t->vhost_wwpn,
 		       sizeof(vs->vs_vhost_wwpn));
-		for (i = VHOST_SCSI_VQ_IO; i < VHOST_SCSI_MAX_VQ; i++) {
+		for (i = VHOST_SCSI_VQ_IO; i < vs->dev.nvqs; i++) {
 			vq = &vs->vqs[i].vq;
 			if (!vhost_vq_is_setup(vq))
 				continue;
 				goto destroy_vq_cmds;
 		}
-		for (i = 0; i < VHOST_SCSI_MAX_VQ; i++) {
+		for (i = 0; i < vs->dev.nvqs; i++) {
 			vq = &vs->vqs[i].vq;
 			mutex_lock(&vq->mutex);
 			vhost_vq_set_backend(vq, vs_tpg);
 		target_undepend_item(&se_tpg->tpg_group.cg_item);
 	}
 	if (match) {
-		for (i = 0; i < VHOST_SCSI_MAX_VQ; i++) {
+		for (i = 0; i < vs->dev.nvqs; i++) {
 			vq = &vs->vqs[i].vq;
 			mutex_lock(&vq->mutex);
 			vhost_vq_set_backend(vq, NULL);
 		/* Make sure cmds are not running before tearing them down. */
 		vhost_scsi_flush(vs);
-		for (i = 0; i < VHOST_SCSI_MAX_VQ; i++) {
+		for (i = 0; i < vs->dev.nvqs; i++) {
 			vq = &vs->vqs[i].vq;
 			vhost_scsi_destroy_vq_cmds(vq);
 		}
 		return -EFAULT;
 	}
-	for (i = 0; i < VHOST_SCSI_MAX_VQ; i++) {
+	for (i = 0; i < vs->dev.nvqs; i++) {
 		vq = &vs->vqs[i].vq;
 		mutex_lock(&vq->mutex);
 		vq->acked_features = features;
 {
 	struct vhost_scsi *vs;
 	struct vhost_virtqueue **vqs;
-	int r = -ENOMEM, i;
+	int r = -ENOMEM, i, nvqs = vhost_scsi_max_io_vqs;
 	vs = kvzalloc(sizeof(*vs), GFP_KERNEL);
 	if (!vs)
 		goto err_vs;
-	vqs = kmalloc_array(VHOST_SCSI_MAX_VQ, sizeof(*vqs), GFP_KERNEL);
+	if (nvqs > VHOST_SCSI_MAX_IO_VQ) {
-	if (!vqs)
+		pr_err("Invalid max_io_vqs of %d. Using %d.\n", nvqs,
+		       VHOST_SCSI_MAX_IO_VQ);
+		nvqs = VHOST_SCSI_MAX_IO_VQ;
+	} else if (nvqs == 0) {
+		pr_err("Invalid max_io_vqs of %d. Using 1.\n", nvqs);
+		nvqs = 1;
+	}
+	nvqs += VHOST_SCSI_VQ_IO;
+	vs->compl_bitmap = bitmap_alloc(nvqs, GFP_KERNEL);
+	if (!vs->compl_bitmap)
+		goto err_compl_bitmap;
+	vs->old_inflight = kmalloc_array(nvqs, sizeof(*vs->old_inflight),
+					 GFP_KERNEL | __GFP_ZERO);
+	if (!vs->old_inflight)
+		goto err_inflight;
+	vs->vqs = kmalloc_array(nvqs, sizeof(*vs->vqs),
+				GFP_KERNEL | __GFP_ZERO);
+	if (!vs->vqs)
 		goto err_vqs;
+	vqs = kmalloc_array(nvqs, sizeof(*vqs), GFP_KERNEL);
+	if (!vqs)
+		goto err_local_vqs;
 	vhost_work_init(&vs->vs_completion_work, vhost_scsi_complete_cmd_work);
 	vhost_work_init(&vs->vs_event_work, vhost_scsi_evt_work);
 	vqs[VHOST_SCSI_VQ_EVT] = &vs->vqs[VHOST_SCSI_VQ_EVT].vq;
 	vs->vqs[VHOST_SCSI_VQ_CTL].vq.handle_kick = vhost_scsi_ctl_handle_kick;
 	vs->vqs[VHOST_SCSI_VQ_EVT].vq.handle_kick = vhost_scsi_evt_handle_kick;
-	for (i = VHOST_SCSI_VQ_IO; i < VHOST_SCSI_MAX_VQ; i++) {
+	for (i = VHOST_SCSI_VQ_IO; i < nvqs; i++) {
 		vqs[i] = &vs->vqs[i].vq;
 		vs->vqs[i].vq.handle_kick = vhost_scsi_handle_kick;
 	}
-	vhost_dev_init(&vs->dev, vqs, VHOST_SCSI_MAX_VQ, UIO_MAXIOV,
+	vhost_dev_init(&vs->dev, vqs, nvqs, UIO_MAXIOV,
 		       VHOST_SCSI_WEIGHT, 0, true, NULL);
 	vhost_scsi_init_inflight(vs, NULL);
 	f->private_data = vs;
 	return 0;
+err_local_vqs:
+	kfree(vs->vqs);
 err_vqs:
+	kfree(vs->old_inflight);
+err_inflight:
+	bitmap_free(vs->compl_bitmap);
+err_compl_bitmap:
 	kvfree(vs);
 err_vs:
 	return r;
 	vhost_dev_stop(&vs->dev);
 	vhost_dev_cleanup(&vs->dev);
 	kfree(vs->dev.vqs);
+	kfree(vs->vqs);
+	kfree(vs->old_inflight);
+	bitmap_free(vs->compl_bitmap);
 	kvfree(vs);
 	return 0;
 }

drivers/vhost/vdpa.c+36−2

 	return 0;
 }
+static bool vhost_vdpa_can_suspend(const struct vhost_vdpa *v)
+{
+	struct vdpa_device *vdpa = v->vdpa;
+	const struct vdpa_config_ops *ops = vdpa->config;
+	return ops->suspend;
+}
 static long vhost_vdpa_get_features(struct vhost_vdpa *v, u64 __user *featurep)
 {
 	struct vdpa_device *vdpa = v->vdpa;
 	return 0;
 }
+/* After a successful return of ioctl the device must not process more
+ * virtqueue descriptors. The device can answer to read or writes of config
+ * fields as if it were not suspended. In particular, writing to "queue_enable"
+ * with a value of 1 will not make the device start processing buffers.
+ */
+static long vhost_vdpa_suspend(struct vhost_vdpa *v)
+{
+	struct vdpa_device *vdpa = v->vdpa;
+	const struct vdpa_config_ops *ops = vdpa->config;
+	if (!ops->suspend)
+		return -EOPNOTSUPP;
+	return ops->suspend(vdpa);
+}
 static long vhost_vdpa_vring_ioctl(struct vhost_vdpa *v, unsigned int cmd,
 				   void __user *argp)
 {
 	if (cmd == VHOST_SET_BACKEND_FEATURES) {
 		if (copy_from_user(&features, featurep, sizeof(features)))
 			return -EFAULT;
-		if (features & ~VHOST_VDPA_BACKEND_FEATURES)
+		if (features & ~(VHOST_VDPA_BACKEND_FEATURES |
+				 BIT_ULL(VHOST_BACKEND_F_SUSPEND)))
+			return -EOPNOTSUPP;
+		if ((features & BIT_ULL(VHOST_BACKEND_F_SUSPEND)) &&
+		     !vhost_vdpa_can_suspend(v))
 			return -EOPNOTSUPP;
 		vhost_set_backend_features(&v->vdev, features);
 		return 0;
 		break;
 	case VHOST_GET_BACKEND_FEATURES:
 		features = VHOST_VDPA_BACKEND_FEATURES;
+		if (vhost_vdpa_can_suspend(v))
+			features |= BIT_ULL(VHOST_BACKEND_F_SUSPEND);
 		if (copy_to_user(featurep, &features, sizeof(features)))
 			r = -EFAULT;
 		break;
 	case VHOST_VDPA_GET_VQS_COUNT:
 		r = vhost_vdpa_get_vqs_count(v, argp);
 		break;
+	case VHOST_VDPA_SUSPEND:
+		r = vhost_vdpa_suspend(v);
+		break;
 	default:
 		r = vhost_dev_ioctl(&v->vdev, cmd, argp);
 		if (r == -ENOIOCTLCMD)
 	if (!bus)
 		return -EFAULT;
-	if (!iommu_capable(bus, IOMMU_CAP_CACHE_COHERENCY))
+	if (!device_iommu_capable(dma_dev, IOMMU_CAP_CACHE_COHERENCY))
 		return -ENOTSUPP;
 	v->domain = iommu_domain_alloc(bus);
 err:
 	put_device(&v->dev);
+	ida_simple_remove(&vhost_vdpa_ida, v->minor);
 	return r;
 }

drivers/vhost/vringh.c+56−22

 #if IS_REACHABLE(CONFIG_VHOST_IOTLB)
 static int iotlb_translate(const struct vringh *vrh,
-			   u64 addr, u64 len, struct bio_vec iov[],
+			   u64 addr, u64 len, u64 *translated,
+			   struct bio_vec iov[],
 			   int iov_size, u32 perm)
 {
 	struct vhost_iotlb_map *map;
 	spin_unlock(vrh->iotlb_lock);
+	if (translated)
+		*translated = min(len, s);
 	return ret;
 }
 static inline int copy_from_iotlb(const struct vringh *vrh, void *dst,
 				  void *src, size_t len)
 {
-	struct iov_iter iter;
+	u64 total_translated = 0;
-	struct bio_vec iov[16];
-	int ret;
-	ret = iotlb_translate(vrh, (u64)(uintptr_t)src,
+	while (total_translated < len) {
-			      len, iov, 16, VHOST_MAP_RO);
+		struct bio_vec iov[16];
-	if (ret < 0)
+		struct iov_iter iter;
-		return ret;
+		u64 translated;
+		int ret;
-	iov_iter_bvec(&iter, READ, iov, ret, len);
+		ret = iotlb_translate(vrh, (u64)(uintptr_t)src,
+				      len - total_translated, &translated,
+				      iov, ARRAY_SIZE(iov), VHOST_MAP_RO);
+		if (ret == -ENOBUFS)
+			ret = ARRAY_SIZE(iov);
+		else if (ret < 0)
+			return ret;
-	ret = copy_from_iter(dst, len, &iter);
+		iov_iter_bvec(&iter, READ, iov, ret, translated);
-	return ret;
+		ret = copy_from_iter(dst, translated, &iter);
+		if (ret < 0)
+			return ret;
+		src += translated;
+		dst += translated;
+		total_translated += translated;
+	}
+	return total_translated;
 }
 static inline int copy_to_iotlb(const struct vringh *vrh, void *dst,
 				void *src, size_t len)
 {
-	struct iov_iter iter;
+	u64 total_translated = 0;
-	struct bio_vec iov[16];
-	int ret;
-	ret = iotlb_translate(vrh, (u64)(uintptr_t)dst,
+	while (total_translated < len) {
-			      len, iov, 16, VHOST_MAP_WO);
+		struct bio_vec iov[16];
-	if (ret < 0)
+		struct iov_iter iter;
-		return ret;
+		u64 translated;
+		int ret;
+		ret = iotlb_translate(vrh, (u64)(uintptr_t)dst,
+				      len - total_translated, &translated,
+				      iov, ARRAY_SIZE(iov), VHOST_MAP_WO);
+		if (ret == -ENOBUFS)
+			ret = ARRAY_SIZE(iov);
+		else if (ret < 0)
+			return ret;
-	iov_iter_bvec(&iter, WRITE, iov, ret, len);
+		iov_iter_bvec(&iter, WRITE, iov, ret, translated);
+		ret = copy_to_iter(src, translated, &iter);
+		if (ret < 0)
+			return ret;
+		src += translated;
+		dst += translated;
+		total_translated += translated;
+	}
-	return copy_to_iter(src, len, &iter);
+	return total_translated;
 }
 static inline int getu16_iotlb(const struct vringh *vrh,
 	int ret;
 	/* Atomic read is needed for getu16 */
-	ret = iotlb_translate(vrh, (u64)(uintptr_t)p, sizeof(*p),
+	ret = iotlb_translate(vrh, (u64)(uintptr_t)p, sizeof(*p), NULL,
 			      &iov, 1, VHOST_MAP_RO);
 	if (ret < 0)
 		return ret;
 	int ret;
 	/* Atomic write is needed for putu16 */
-	ret = iotlb_translate(vrh, (u64)(uintptr_t)p, sizeof(*p),
+	ret = iotlb_translate(vrh, (u64)(uintptr_t)p, sizeof(*p), NULL,
 			      &iov, 1, VHOST_MAP_WO);
 	if (ret < 0)
 		return ret;

drivers/virtio/Kconfig+7−4

 config VIRTIO_HARDEN_NOTIFICATION
         bool "Harden virtio notification"
+        depends on BROKEN
         help
           Enable this to harden the device notifications and suppress
           those that happen at a time where notifications are illegal.
-          Experimental: Note that several drivers still have bugs that
+          Experimental: Note that several drivers still have issues that
           may cause crashes or hangs when correct handling of
           notifications is enforced; depending on the subset of
           drivers and devices you use, this may or may not work.
 	 This driver provides access to virtio-mem paravirtualized memory
 	 devices, allowing to hotplug and hotunplug memory.
-	 This driver was only tested under x86-64 and arm64, but should
+	 This driver currently only supports x86-64 and arm64. Although it
-	 theoretically work on all architectures that support memory hotplug
+	 should compile on other architectures that implement memory
-	 and hotremove.
+	 hot(un)plug, architecture-specific and/or common
+	 code changes may be required for virtio-mem, kdump and kexec to work as
+	 expected.
 	 If unsure, say M.

drivers/virtio/virtio.c+3−1

@@ -428,7 +428,9 @@ int register_virtio_device(struct virtio_device *dev) · 427 unmodified lines
428	goto out;	428	goto out;
429		429
430	dev->index = err;	430	dev->index = err;
431	dev_set_name(&dev->dev, "virtio%u", dev->index);	431	err = dev_set_name(&dev->dev, "virtio%u", dev->index);
		432	if (err)
		433	goto out_ida_remove;
432		434
433	err = virtio_device_of_init(dev);	435	err = virtio_device_of_init(dev);
434	if (err)	436	if (err)

drivers/virtio/virtio_mmio.c+12−2

 static struct virtqueue *vm_setup_vq(struct virtio_device *vdev, unsigned int index,
 				  void (*callback)(struct virtqueue *vq),
-				  const char *name, bool ctx)
+				  const char *name, u32 size, bool ctx)
 {
 	struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev);
 	struct virtio_mmio_vq_info *info;
 		goto error_new_virtqueue;
 	}
+	if (!size || size > num)
+		size = num;
 	/* Create the vring */
-	vq = vring_create_virtqueue(index, num, VIRTIO_MMIO_VRING_ALIGN, vdev,
+	vq = vring_create_virtqueue(index, size, VIRTIO_MMIO_VRING_ALIGN, vdev,
 				 true, true, ctx, vm_notify, callback, name);
 	if (!vq) {
 		err = -ENOMEM;
 		goto error_new_virtqueue;
 	}
+	vq->num_max = num;
 	/* Activate the queue */
 	writel(virtqueue_get_vring_size(vq), vm_dev->base + VIRTIO_MMIO_QUEUE_NUM);
 	if (vm_dev->version == 1) {
 		       struct virtqueue *vqs[],
 		       vq_callback_t *callbacks[],
 		       const char * const names[],
+		       u32 sizes[],
 		       const bool *ctx,
 		       struct irq_affinity *desc)
 {
 	if (err)
 		return err;
+	if (of_property_read_bool(vm_dev->pdev->dev.of_node, "wakeup-source"))
+		enable_irq_wake(irq);
 	for (i = 0; i < nvqs; ++i) {
 		if (!names[i]) {
 			vqs[i] = NULL;
 		}
 		vqs[i] = vm_setup_vq(vdev, queue_idx++, callbacks[i], names[i],
+				     sizes ? sizes[i] : 0,
 				     ctx ? ctx[i] : false);
 		if (IS_ERR(vqs[i])) {
 			vm_del_vqs(vdev);

drivers/virtio/virtio_pci_common.c+20−12

 static struct virtqueue *vp_setup_vq(struct virtio_device *vdev, unsigned int index,
 				     void (*callback)(struct virtqueue *vq),
 				     const char *name,
+				     u32 size,
 				     bool ctx,
 				     u16 msix_vec)
 {
 	if (!info)
 		return ERR_PTR(-ENOMEM);
-	vq = vp_dev->setup_vq(vp_dev, info, index, callback, name, ctx,
+	vq = vp_dev->setup_vq(vp_dev, info, index, callback, name, size, ctx,
 			      msix_vec);
 	if (IS_ERR(vq))
 		goto out_info;
 	struct virtio_pci_vq_info *info = vp_dev->vqs[vq->index];
 	unsigned long flags;
-	spin_lock_irqsave(&vp_dev->lock, flags);
+	/*
-	list_del(&info->node);
+	 * If it fails during re-enable reset vq. This way we won't rejoin
-	spin_unlock_irqrestore(&vp_dev->lock, flags);
+	 * info->node to the queue. Prevent unexpected irqs.
+	 */
+	if (!vq->reset) {
+		spin_lock_irqsave(&vp_dev->lock, flags);
+		list_del(&info->node);
+		spin_unlock_irqrestore(&vp_dev->lock, flags);
+	}
 	vp_dev->del_vq(info);
 	kfree(info);
 static int vp_find_vqs_msix(struct virtio_device *vdev, unsigned int nvqs,
 		struct virtqueue *vqs[], vq_callback_t *callbacks[],
-		const char * const names[], bool per_vq_vectors,
+		const char * const names[], u32 sizes[], bool per_vq_vectors,
 		const bool *ctx,
 		struct irq_affinity *desc)
 {
 		else
 			msix_vec = VP_MSIX_VQ_VECTOR;
 		vqs[i] = vp_setup_vq(vdev, queue_idx++, callbacks[i], names[i],
-				     ctx ? ctx[i] : false,
+				     sizes ? sizes[i] : 0,
-				     msix_vec);
+				     ctx ? ctx[i] : false, msix_vec);
 		if (IS_ERR(vqs[i])) {
 			err = PTR_ERR(vqs[i]);
 			goto error_find;
 static int vp_find_vqs_intx(struct virtio_device *vdev, unsigned int nvqs,
 		struct virtqueue *vqs[], vq_callback_t *callbacks[],
-		const char * const names[], const bool *ctx)
+		const char * const names[], u32 sizes[], const bool *ctx)
 {
 	struct virtio_pci_device *vp_dev = to_vp_device(vdev);
 	int i, err, queue_idx = 0;
 			continue;
 		}
 		vqs[i] = vp_setup_vq(vdev, queue_idx++, callbacks[i], names[i],
+				     sizes ? sizes[i] : 0,
 				     ctx ? ctx[i] : false,
 				     VIRTIO_MSI_NO_VECTOR);
 		if (IS_ERR(vqs[i])) {
 /* the config->find_vqs() implementation */
 int vp_find_vqs(struct virtio_device *vdev, unsigned int nvqs,
 		struct virtqueue *vqs[], vq_callback_t *callbacks[],
-		const char * const names[], const bool *ctx,
+		const char * const names[], u32 sizes[], const bool *ctx,
 		struct irq_affinity *desc)
 {
 	int err;
 	/* Try MSI-X with one vector per queue. */
-	err = vp_find_vqs_msix(vdev, nvqs, vqs, callbacks, names, true, ctx, desc);
+	err = vp_find_vqs_msix(vdev, nvqs, vqs, callbacks, names, sizes, true, ctx, desc);
 	if (!err)
 		return 0;
 	/* Fallback: MSI-X with one vector for config, one shared for queues. */
-	err = vp_find_vqs_msix(vdev, nvqs, vqs, callbacks, names, false, ctx, desc);
+	err = vp_find_vqs_msix(vdev, nvqs, vqs, callbacks, names, sizes, false, ctx, desc);
 	if (!err)
 		return 0;
 	/* Finally fall back to regular interrupts. */
-	return vp_find_vqs_intx(vdev, nvqs, vqs, callbacks, names, ctx);
+	return vp_find_vqs_intx(vdev, nvqs, vqs, callbacks, names, sizes, ctx);
 }
 const char *vp_bus_name(struct virtio_device *vdev)

drivers/virtio/virtio_pci_common.h+2−1

 				      unsigned int idx,
 				      void (*callback)(struct virtqueue *vq),
 				      const char *name,
+				      u32 size,
 				      bool ctx,
 				      u16 msix_vec);
 	void (*del_vq)(struct virtio_pci_vq_info *info);
 /* the config->find_vqs() implementation */
 int vp_find_vqs(struct virtio_device *vdev, unsigned int nvqs,
 		struct virtqueue *vqs[], vq_callback_t *callbacks[],
-		const char * const names[], const bool *ctx,
+		const char * const names[], u32 sizes[], const bool *ctx,
 		struct irq_affinity *desc);
 const char *vp_bus_name(struct virtio_device *vdev);

drivers/virtio/virtio_pci_legacy.c+7−1

 				  unsigned int index,
 				  void (*callback)(struct virtqueue *vq),
 				  const char *name,
+				  u32 size,
 				  bool ctx,
 				  u16 msix_vec)
 {
 	if (!num || vp_legacy_get_queue_enable(&vp_dev->ldev, index))
 		return ERR_PTR(-ENOENT);
+	if (!size || size > num)
+		size = num;
 	info->msix_vector = msix_vec;
 	/* create the vring */
-	vq = vring_create_virtqueue(index, num,
+	vq = vring_create_virtqueue(index, size,
 				    VIRTIO_PCI_VRING_ALIGN, &vp_dev->vdev,
 				    true, false, ctx,
 				    vp_notify, callback, name);
 	if (!vq)
 		return ERR_PTR(-ENOMEM);
+	vq->num_max = num;
 	q_pfn = virtqueue_get_desc_addr(vq) >> VIRTIO_PCI_QUEUE_ADDR_SHIFT;
 	if (q_pfn >> 32) {
 		dev_err(&vp_dev->pci_dev->dev,

drivers/virtio/virtio_pci_modern.c+130−23

@@ -34,6 +34,9 @@ static void vp_transport_features(struct virtio_device *vdev, u64 features) · 33 unmodified lines
34	if ((features & BIT_ULL(VIRTIO_F_SR_IOV)) &&	34	if ((features & BIT_ULL(VIRTIO_F_SR_IOV)) &&
35	pci_find_ext_capability(pci_dev, PCI_EXT_CAP_ID_SRIOV))	35	pci_find_ext_capability(pci_dev, PCI_EXT_CAP_ID_SRIOV))
36	__virtio_set_bit(vdev, VIRTIO_F_SR_IOV);	36	__virtio_set_bit(vdev, VIRTIO_F_SR_IOV);
		37
		38	if (features & BIT_ULL(VIRTIO_F_RING_RESET))
		39	__virtio_set_bit(vdev, VIRTIO_F_RING_RESET);
37	}	40	}
38		41
39	/* virtio config->finalize_features() implementation */	42	/* virtio config->finalize_features() implementation */
@@ -176,6 +179,110 @@ static void vp_reset(struct virtio_device *vdev) · 136 unmodified lines
176	vp_synchronize_vectors(vdev);	179	vp_synchronize_vectors(vdev);
177	}	180	}
178		181
		182	static int vp_active_vq(struct virtqueue *vq, u16 msix_vec)
		183	{
		184	struct virtio_pci_device *vp_dev = to_vp_device(vq->vdev);
		185	struct virtio_pci_modern_device *mdev = &vp_dev->mdev;
		186	unsigned long index;
		187
		188	index = vq->index;
		189
		190	/* activate the queue */
		191	vp_modern_set_queue_size(mdev, index, virtqueue_get_vring_size(vq));
		192	vp_modern_queue_address(mdev, index, virtqueue_get_desc_addr(vq),
		193	virtqueue_get_avail_addr(vq),
		194	virtqueue_get_used_addr(vq));
		195
		196	if (msix_vec != VIRTIO_MSI_NO_VECTOR) {
		197	msix_vec = vp_modern_queue_vector(mdev, index, msix_vec);
		198	if (msix_vec == VIRTIO_MSI_NO_VECTOR)
		199	return -EBUSY;
		200	}
		201
		202	return 0;
		203	}
		204
		205	static int vp_modern_disable_vq_and_reset(struct virtqueue *vq)
		206	{
		207	struct virtio_pci_device *vp_dev = to_vp_device(vq->vdev);
		208	struct virtio_pci_modern_device *mdev = &vp_dev->mdev;
		209	struct virtio_pci_vq_info *info;
		210	unsigned long flags;
		211
		212	if (!virtio_has_feature(vq->vdev, VIRTIO_F_RING_RESET))
		213	return -ENOENT;
		214
		215	vp_modern_set_queue_reset(mdev, vq->index);
		216
		217	info = vp_dev->vqs[vq->index];
		218
		219	/* delete vq from irq handler */
		220	spin_lock_irqsave(&vp_dev->lock, flags);
		221	list_del(&info->node);
		222	spin_unlock_irqrestore(&vp_dev->lock, flags);
		223
		224	INIT_LIST_HEAD(&info->node);
		225
		226	#ifdef CONFIG_VIRTIO_HARDEN_NOTIFICATION
		227	__virtqueue_break(vq);
		228	#endif
		229
		230	/* For the case where vq has an exclusive irq, call synchronize_irq() to
		231	* wait for completion.
		232	*
		233	* note: We can't use disable_irq() since it conflicts with the affinity
		234	* managed IRQ that is used by some drivers.
		235	*/
		236	if (vp_dev->per_vq_vectors && info->msix_vector != VIRTIO_MSI_NO_VECTOR)
		237	synchronize_irq(pci_irq_vector(vp_dev->pci_dev, info->msix_vector));
		238
		239	vq->reset = true;
		240
		241	return 0;
		242	}
		243
		244	static int vp_modern_enable_vq_after_reset(struct virtqueue *vq)
		245	{
		246	struct virtio_pci_device *vp_dev = to_vp_device(vq->vdev);
		247	struct virtio_pci_modern_device *mdev = &vp_dev->mdev;
		248	struct virtio_pci_vq_info *info;
		249	unsigned long flags, index;
		250	int err;
		251
		252	if (!vq->reset)
		253	return -EBUSY;
		254
		255	index = vq->index;
		256	info = vp_dev->vqs[index];
		257
		258	if (vp_modern_get_queue_reset(mdev, index))
		259	return -EBUSY;
		260
		261	if (vp_modern_get_queue_enable(mdev, index))
		262	return -EBUSY;
		263
		264	err = vp_active_vq(vq, info->msix_vector);
		265	if (err)
		266	return err;
		267
		268	if (vq->callback) {
		269	spin_lock_irqsave(&vp_dev->lock, flags);
		270	list_add(&info->node, &vp_dev->virtqueues);
		271	spin_unlock_irqrestore(&vp_dev->lock, flags);
		272	} else {
		273	INIT_LIST_HEAD(&info->node);
		274	}
		275
		276	#ifdef CONFIG_VIRTIO_HARDEN_NOTIFICATION
		277	__virtqueue_unbreak(vq);
		278	#endif
		279
		280	vp_modern_set_queue_enable(&vp_dev->mdev, index, true);
		281	vq->reset = false;
		282
		283	return 0;
		284	}
		285
179	static u16 vp_config_vector(struct virtio_pci_device *vp_dev, u16 vector)	286	static u16 vp_config_vector(struct virtio_pci_device *vp_dev, u16 vector)
180	{	287	{
181	return vp_modern_config_vector(&vp_dev->mdev, vector);	288	return vp_modern_config_vector(&vp_dev->mdev, vector);
@@ -186,6 +293,7 @@ static struct virtqueue setup_vq(struct virtio_pci_device vp_dev, · 4 unmodified lines
186	unsigned int index,	293	unsigned int index,
187	void (callback)(struct virtqueue vq),	294	void (callback)(struct virtqueue vq),
188	const char *name,	295	const char *name,
		296	u32 size,
189	bool ctx,	297	bool ctx,
190	u16 msix_vec)	298	u16 msix_vec)
191	{	299	{
@@ -203,47 +311,39 @@ static struct virtqueue setup_vq(struct virtio_pci_device vp_dev, · 11 unmodified lines
203	if (!num \|\| vp_modern_get_queue_enable(mdev, index))	311	if (!num \|\| vp_modern_get_queue_enable(mdev, index))
204	return ERR_PTR(-ENOENT);	312	return ERR_PTR(-ENOENT);
205		313
206	if (num & (num - 1)) {	314	if (!size \|\| size > num)
207	dev_warn(&vp_dev->pci_dev->dev, "bad queue size %u", num);	315	size = num;
		316
		317	if (size & (size - 1)) {
		318	dev_warn(&vp_dev->pci_dev->dev, "bad queue size %u", size);
208	return ERR_PTR(-EINVAL);	319	return ERR_PTR(-EINVAL);
209	}	320	}
210		321
211	info->msix_vector = msix_vec;	322	info->msix_vector = msix_vec;
212		323
213	/* create the vring */	324	/* create the vring */
214	vq = vring_create_virtqueue(index, num,	325	vq = vring_create_virtqueue(index, size,
215	SMP_CACHE_BYTES, &vp_dev->vdev,	326	SMP_CACHE_BYTES, &vp_dev->vdev,
216	true, true, ctx,	327	true, true, ctx,
217	vp_notify, callback, name);	328	vp_notify, callback, name);
218	if (!vq)	329	if (!vq)
219	return ERR_PTR(-ENOMEM);	330	return ERR_PTR(-ENOMEM);
220		331
221	/* activate the queue */	332	vq->num_max = num;
222	vp_modern_set_queue_size(mdev, index, virtqueue_get_vring_size(vq));	333
223	vp_modern_queue_address(mdev, index, virtqueue_get_desc_addr(vq),	334	err = vp_active_vq(vq, msix_vec);
224	virtqueue_get_avail_addr(vq),	335	if (err)
225	virtqueue_get_used_addr(vq));	336	goto err;
226		337
227	vq->priv = (void __force *)vp_modern_map_vq_notify(mdev, index, NULL);	338	vq->priv = (void __force *)vp_modern_map_vq_notify(mdev, index, NULL);
228	if (!vq->priv) {	339	if (!vq->priv) {
229	err = -ENOMEM;	340	err = -ENOMEM;
230	goto err_map_notify;	341	goto err;
231	}
232
233	if (msix_vec != VIRTIO_MSI_NO_VECTOR) {
234	msix_vec = vp_modern_queue_vector(mdev, index, msix_vec);
235	if (msix_vec == VIRTIO_MSI_NO_VECTOR) {
236	err = -EBUSY;
237	goto err_assign_vector;
238	}
239	}	342	}
240		343
241	return vq;	344	return vq;
242		345
243	err_assign_vector:	346	err:
244	if (!mdev->notify_base)
245	pci_iounmap(mdev->pci_dev, (void __iomem __force *)vq->priv);
246	err_map_notify:
247	vring_del_virtqueue(vq);	347	vring_del_virtqueue(vq);
248	return ERR_PTR(err);	348	return ERR_PTR(err);
249	}	349	}
@@ -251,12 +351,15 @@ err_map_notify: · 1 unmodified line
251	static int vp_modern_find_vqs(struct virtio_device *vdev, unsigned int nvqs,	351	static int vp_modern_find_vqs(struct virtio_device *vdev, unsigned int nvqs,
252	struct virtqueue *vqs[],	352	struct virtqueue *vqs[],
253	vq_callback_t *callbacks[],	353	vq_callback_t *callbacks[],
254	const char * const names[], const bool *ctx,	354	const char * const names[],
		355	u32 sizes[],
		356	const bool *ctx,
255	struct irq_affinity *desc)	357	struct irq_affinity *desc)
256	{	358	{
257	struct virtio_pci_device *vp_dev = to_vp_device(vdev);	359	struct virtio_pci_device *vp_dev = to_vp_device(vdev);
258	struct virtqueue *vq;	360	struct virtqueue *vq;
259	int rc = vp_find_vqs(vdev, nvqs, vqs, callbacks, names, ctx, desc);	361	int rc = vp_find_vqs(vdev, nvqs, vqs, callbacks, names, sizes, ctx,
		362	desc);
260		363
261	if (rc)	364	if (rc)
262	return rc;	365	return rc;
@@ -401,6 +504,8 @@ static const struct virtio_config_ops virtio_pci_config_nodev_ops = { · 138 unmodified lines
401	.set_vq_affinity = vp_set_vq_affinity,	504	.set_vq_affinity = vp_set_vq_affinity,
402	.get_vq_affinity = vp_get_vq_affinity,	505	.get_vq_affinity = vp_get_vq_affinity,
403	.get_shm_region = vp_get_shm_region,	506	.get_shm_region = vp_get_shm_region,
		507	.disable_vq_and_reset = vp_modern_disable_vq_and_reset,
		508	.enable_vq_after_reset = vp_modern_enable_vq_after_reset,
404	};	509	};
405		510
406	static const struct virtio_config_ops virtio_pci_config_ops = {	511	static const struct virtio_config_ops virtio_pci_config_ops = {
@@ -419,6 +524,8 @@ static const struct virtio_config_ops virtio_pci_config_ops = { · 12 unmodified lines
419	.set_vq_affinity = vp_set_vq_affinity,	524	.set_vq_affinity = vp_set_vq_affinity,
420	.get_vq_affinity = vp_get_vq_affinity,	525	.get_vq_affinity = vp_get_vq_affinity,
421	.get_shm_region = vp_get_shm_region,	526	.get_shm_region = vp_get_shm_region,
		527	.disable_vq_and_reset = vp_modern_disable_vq_and_reset,
		528	.enable_vq_after_reset = vp_modern_enable_vq_after_reset,
422	};	529	};
423		530
424	/* the PCI probing function */	531	/* the PCI probing function */

drivers/virtio/virtio_pci_modern_dev.c+39−0

@@ -3,6 +3,7 @@ · 2 unmodified lines
3	#include <linux/virtio_pci_modern.h>	3	#include <linux/virtio_pci_modern.h>
4	#include <linux/module.h>	4	#include <linux/module.h>
5	#include <linux/pci.h>	5	#include <linux/pci.h>
		6	#include <linux/delay.h>
6		7
7	/*	8	/*
8	* vp_modern_map_capability - map a part of virtio pci capability	9	* vp_modern_map_capability - map a part of virtio pci capability
@@ -474,6 +475,44 @@ void vp_modern_set_status(struct virtio_pci_modern_device *mdev, · 465 unmodified lines
474	}	475	}
475	EXPORT_SYMBOL_GPL(vp_modern_set_status);	476	EXPORT_SYMBOL_GPL(vp_modern_set_status);
476		477
		478	/*
		479	* vp_modern_get_queue_reset - get the queue reset status
		480	* @mdev: the modern virtio-pci device
		481	* @index: queue index
		482	*/
		483	int vp_modern_get_queue_reset(struct virtio_pci_modern_device *mdev, u16 index)
		484	{
		485	struct virtio_pci_modern_common_cfg __iomem *cfg;
		486
		487	cfg = (struct virtio_pci_modern_common_cfg __iomem *)mdev->common;
		488
		489	vp_iowrite16(index, &cfg->cfg.queue_select);
		490	return vp_ioread16(&cfg->queue_reset);
		491	}
		492	EXPORT_SYMBOL_GPL(vp_modern_get_queue_reset);
		493
		494	/*
		495	* vp_modern_set_queue_reset - reset the queue
		496	* @mdev: the modern virtio-pci device
		497	* @index: queue index
		498	*/
		499	void vp_modern_set_queue_reset(struct virtio_pci_modern_device *mdev, u16 index)
		500	{
		501	struct virtio_pci_modern_common_cfg __iomem *cfg;
		502
		503	cfg = (struct virtio_pci_modern_common_cfg __iomem *)mdev->common;
		504
		505	vp_iowrite16(index, &cfg->cfg.queue_select);
		506	vp_iowrite16(1, &cfg->queue_reset);
		507
		508	while (vp_ioread16(&cfg->queue_reset))
		509	msleep(1);
		510
		511	while (vp_ioread16(&cfg->cfg.queue_enable))
		512	msleep(1);
		513	}
		514	EXPORT_SYMBOL_GPL(vp_modern_set_queue_reset);
		515
477	/*	516	/*
478	* vp_modern_queue_vector - set the MSIX vector for a specific virtqueue	517	* vp_modern_queue_vector - set the MSIX vector for a specific virtqueue
479	* @mdev: the modern virtio-pci device	518	* @mdev: the modern virtio-pci device

drivers/virtio/virtio_ring.c+559−211

@@ -85,6 +85,71 @@ struct vring_desc_extra { · 84 unmodified lines
85	u16 next; /* The next desc state in a list. */	85	u16 next; /* The next desc state in a list. */
86	};	86	};
87		87
		88	struct vring_virtqueue_split {
		89	/* Actual memory layout for this queue. */
		90	struct vring vring;
		91
		92	/* Last written value to avail->flags */
		93	u16 avail_flags_shadow;
		94
		95	/*
		96	* Last written value to avail->idx in
		97	* guest byte order.
		98	*/
		99	u16 avail_idx_shadow;
		100
		101	/* Per-descriptor state. */
		102	struct vring_desc_state_split *desc_state;
		103	struct vring_desc_extra *desc_extra;
		104
		105	/* DMA address and size information */
		106	dma_addr_t queue_dma_addr;
		107	size_t queue_size_in_bytes;
		108
		109	/*
		110	* The parameters for creating vrings are reserved for creating new
		111	* vring.
		112	*/
		113	u32 vring_align;
		114	bool may_reduce_num;
		115	};
		116
		117	struct vring_virtqueue_packed {
		118	/* Actual memory layout for this queue. */
		119	struct {
		120	unsigned int num;
		121	struct vring_packed_desc *desc;
		122	struct vring_packed_desc_event *driver;
		123	struct vring_packed_desc_event *device;
		124	} vring;
		125
		126	/* Driver ring wrap counter. */
		127	bool avail_wrap_counter;
		128
		129	/* Avail used flags. */
		130	u16 avail_used_flags;
		131
		132	/* Index of the next avail descriptor. */
		133	u16 next_avail_idx;
		134
		135	/*
		136	* Last written value to driver->flags in
		137	* guest byte order.
		138	*/
		139	u16 event_flags_shadow;
		140
		141	/* Per-descriptor state. */
		142	struct vring_desc_state_packed *desc_state;
		143	struct vring_desc_extra *desc_extra;
		144
		145	/* DMA address and size information */
		146	dma_addr_t ring_dma_addr;
		147	dma_addr_t driver_event_dma_addr;
		148	dma_addr_t device_event_dma_addr;
		149	size_t ring_size_in_bytes;
		150	size_t event_size_in_bytes;
		151	};
		152
88	struct vring_virtqueue {	153	struct vring_virtqueue {
89	struct virtqueue vq;	154	struct virtqueue vq;
90		155
@@ -124,64 +189,10 @@ struct vring_virtqueue { · 33 unmodified lines
124		189
125	union {	190	union {
126	/* Available for split ring */	191	/* Available for split ring */
127	struct {	192	struct vring_virtqueue_split split;
128	/* Actual memory layout for this queue. */
129	struct vring vring;
130
131	/* Last written value to avail->flags */
132	u16 avail_flags_shadow;
133
134	/*
135	* Last written value to avail->idx in
136	* guest byte order.
137	*/
138	u16 avail_idx_shadow;
139
140	/* Per-descriptor state. */
141	struct vring_desc_state_split *desc_state;
142	struct vring_desc_extra *desc_extra;
143
144	/* DMA address and size information */
145	dma_addr_t queue_dma_addr;
146	size_t queue_size_in_bytes;
147	} split;
148		193
149	/* Available for packed ring */	194	/* Available for packed ring */
150	struct {	195	struct vring_virtqueue_packed packed;
151	/* Actual memory layout for this queue. */
152	struct {
153	unsigned int num;
154	struct vring_packed_desc *desc;
155	struct vring_packed_desc_event *driver;
156	struct vring_packed_desc_event *device;
157	} vring;
158
159	/* Driver ring wrap counter. */
160	bool avail_wrap_counter;
161
162	/* Avail used flags. */
163	u16 avail_used_flags;
164
165	/* Index of the next avail descriptor. */
166	u16 next_avail_idx;
167
168	/*
169	* Last written value to driver->flags in
170	* guest byte order.
171	*/
172	u16 event_flags_shadow;
173
174	/* Per-descriptor state. */
175	struct vring_desc_state_packed *desc_state;
176	struct vring_desc_extra *desc_extra;
177
178	/* DMA address and size information */
179	dma_addr_t ring_dma_addr;
180	dma_addr_t driver_event_dma_addr;
181	dma_addr_t device_event_dma_addr;
182	size_t ring_size_in_bytes;
183	size_t event_size_in_bytes;
184	} packed;
185	};	196	};
186		197
187	/* How to notify other side. FIXME: commonalize hcalls! */	198	/* How to notify other side. FIXME: commonalize hcalls! */
@@ -200,6 +211,16 @@ struct vring_virtqueue { · 12 unmodified lines
200	#endif	211	#endif
201	};	212	};
202		213
		214	static struct virtqueue *__vring_new_virtqueue(unsigned int index,
		215	struct vring_virtqueue_split *vring_split,
		216	struct virtio_device *vdev,
		217	bool weak_barriers,
		218	bool context,
		219	bool (notify)(struct virtqueue ),
		220	void (callback)(struct virtqueue ),
		221	const char *name);
		222	static struct vring_desc_extra *vring_alloc_desc_extra(unsigned int num);
		223	static void vring_free(struct virtqueue *_vq);
203		224
204	/*	225	/*
205	* Helpers.	226	* Helpers.
@@ -364,6 +385,24 @@ static int vring_mapping_error(const struct vring_virtqueue *vq, · 158 unmodified lines
364	return dma_mapping_error(vring_dma_dev(vq), addr);	385	return dma_mapping_error(vring_dma_dev(vq), addr);
365	}	386	}
366		387
		388	static void virtqueue_init(struct vring_virtqueue *vq, u32 num)
		389	{
		390	vq->vq.num_free = num;
		391
		392	if (vq->packed_ring)
		393	vq->last_used_idx = 0 \| (1 << VRING_PACKED_EVENT_F_WRAP_CTR);
		394	else
		395	vq->last_used_idx = 0;
		396
		397	vq->event_triggered = false;
		398	vq->num_added = 0;
		399
		400	#ifdef DEBUG
		401	vq->in_use = false;
		402	vq->last_add_time_valid = false;
		403	#endif
		404	}
		405
367		406
368	/*	407	/*
369	* Split ring specific functions - *_split().	408	* Split ring specific functions - *_split().
@@ -907,28 +946,107 @@ static void virtqueue_detach_unused_buf_split(struct virtqueue _vq) · 537 unmodified lines
907	return NULL;	946	return NULL;
908	}	947	}
909		948
910	static struct virtqueue *vring_create_virtqueue_split(	949	static void virtqueue_vring_init_split(struct vring_virtqueue_split *vring_split,
911	unsigned int index,	950	struct vring_virtqueue *vq)
912	unsigned int num,	951	{
913	unsigned int vring_align,	952	struct virtio_device *vdev;
914	struct virtio_device *vdev,	953
915	bool weak_barriers,	954	vdev = vq->vq.vdev;
916	bool may_reduce_num,	955
917	bool context,	956	vring_split->avail_flags_shadow = 0;
918	bool (notify)(struct virtqueue ),	957	vring_split->avail_idx_shadow = 0;
919	void (callback)(struct virtqueue ),	958
920	const char *name)	959	/* No callback? Tell other side not to bother us. */
		960	if (!vq->vq.callback) {
		961	vring_split->avail_flags_shadow \|= VRING_AVAIL_F_NO_INTERRUPT;
		962	if (!vq->event)
		963	vring_split->vring.avail->flags = cpu_to_virtio16(vdev,
		964	vring_split->avail_flags_shadow);
		965	}
		966	}
		967
		968	static void virtqueue_reinit_split(struct vring_virtqueue *vq)
		969	{
		970	int num;
		971
		972	num = vq->split.vring.num;
		973
		974	vq->split.vring.avail->flags = 0;
		975	vq->split.vring.avail->idx = 0;
		976
		977	/* reset avail event */
		978	vq->split.vring.avail->ring[num] = 0;
		979
		980	vq->split.vring.used->flags = 0;
		981	vq->split.vring.used->idx = 0;
		982
		983	/* reset used event */
		984	(__virtio16 )&(vq->split.vring.used->ring[num]) = 0;
		985
		986	virtqueue_init(vq, num);
		987
		988	virtqueue_vring_init_split(&vq->split, vq);
		989	}
		990
		991	static void virtqueue_vring_attach_split(struct vring_virtqueue *vq,
		992	struct vring_virtqueue_split *vring_split)
		993	{
		994	vq->split = *vring_split;
		995
		996	/* Put everything in free lists. */
		997	vq->free_head = 0;
		998	}
		999
		1000	static int vring_alloc_state_extra_split(struct vring_virtqueue_split *vring_split)
		1001	{
		1002	struct vring_desc_state_split *state;
		1003	struct vring_desc_extra *extra;
		1004	u32 num = vring_split->vring.num;
		1005
		1006	state = kmalloc_array(num, sizeof(struct vring_desc_state_split), GFP_KERNEL);
		1007	if (!state)
		1008	goto err_state;
		1009
		1010	extra = vring_alloc_desc_extra(num);
		1011	if (!extra)
		1012	goto err_extra;
		1013
		1014	memset(state, 0, num * sizeof(struct vring_desc_state_split));
		1015
		1016	vring_split->desc_state = state;
		1017	vring_split->desc_extra = extra;
		1018	return 0;
		1019
		1020	err_extra:
		1021	kfree(state);
		1022	err_state:
		1023	return -ENOMEM;
		1024	}
		1025
		1026	static void vring_free_split(struct vring_virtqueue_split *vring_split,
		1027	struct virtio_device *vdev)
		1028	{
		1029	vring_free_queue(vdev, vring_split->queue_size_in_bytes,
		1030	vring_split->vring.desc,
		1031	vring_split->queue_dma_addr);
		1032
		1033	kfree(vring_split->desc_state);
		1034	kfree(vring_split->desc_extra);
		1035	}
		1036
		1037	static int vring_alloc_queue_split(struct vring_virtqueue_split *vring_split,
		1038	struct virtio_device *vdev,
		1039	u32 num,
		1040	unsigned int vring_align,
		1041	bool may_reduce_num)
921	{	1042	{
922	struct virtqueue *vq;
923	void *queue = NULL;	1043	void *queue = NULL;
924	dma_addr_t dma_addr;	1044	dma_addr_t dma_addr;
925	size_t queue_size_in_bytes;
926	struct vring vring;
927		1045
928	/* We assume num is a power of 2. */	1046	/* We assume num is a power of 2. */
929	if (num & (num - 1)) {	1047	if (num & (num - 1)) {
930	dev_warn(&vdev->dev, "Bad virtqueue length %u\n", num);	1048	dev_warn(&vdev->dev, "Bad virtqueue length %u\n", num);
931	return NULL;	1049	return -EINVAL;
932	}	1050	}
933		1051
934	/* TODO: allocate each queue chunk individually */	1052	/* TODO: allocate each queue chunk individually */
@@ -939,11 +1057,11 @@ static struct virtqueue *vring_create_virtqueue_split( · 4 unmodified lines
939	if (queue)	1057	if (queue)
940	break;	1058	break;
941	if (!may_reduce_num)	1059	if (!may_reduce_num)
942	return NULL;	1060	return -ENOMEM;
943	}	1061	}
944		1062
945	if (!num)	1063	if (!num)
946	return NULL;	1064	return -ENOMEM;
947		1065
948	if (!queue) {	1066	if (!queue) {
949	/* Try to get a single page. You are my only hope! */	1067	/* Try to get a single page. You are my only hope! */
@@ -951,26 +1069,85 @@ static struct virtqueue *vring_create_virtqueue_split( · 1 unmodified line
951	&dma_addr, GFP_KERNEL\|__GFP_ZERO);	1069	&dma_addr, GFP_KERNEL\|__GFP_ZERO);
952	}	1070	}
953	if (!queue)	1071	if (!queue)
954	return NULL;	1072	return -ENOMEM;
		1073
		1074	vring_init(&vring_split->vring, num, queue, vring_align);
		1075
		1076	vring_split->queue_dma_addr = dma_addr;
		1077	vring_split->queue_size_in_bytes = vring_size(num, vring_align);
955		1078
956	queue_size_in_bytes = vring_size(num, vring_align);	1079	vring_split->vring_align = vring_align;
957	vring_init(&vring, num, queue, vring_align);	1080	vring_split->may_reduce_num = may_reduce_num;
958		1081
959	vq = __vring_new_virtqueue(index, vring, vdev, weak_barriers, context,	1082	return 0;
960	notify, callback, name);	1083	}
		1084
		1085	static struct virtqueue *vring_create_virtqueue_split(
		1086	unsigned int index,
		1087	unsigned int num,
		1088	unsigned int vring_align,
		1089	struct virtio_device *vdev,
		1090	bool weak_barriers,
		1091	bool may_reduce_num,
		1092	bool context,
		1093	bool (notify)(struct virtqueue ),
		1094	void (callback)(struct virtqueue ),
		1095	const char *name)
		1096	{
		1097	struct vring_virtqueue_split vring_split = {};
		1098	struct virtqueue *vq;
		1099	int err;
		1100
		1101	err = vring_alloc_queue_split(&vring_split, vdev, num, vring_align,
		1102	may_reduce_num);
		1103	if (err)
		1104	return NULL;
		1105
		1106	vq = __vring_new_virtqueue(index, &vring_split, vdev, weak_barriers,
		1107	context, notify, callback, name);
961	if (!vq) {	1108	if (!vq) {
962	vring_free_queue(vdev, queue_size_in_bytes, queue,	1109	vring_free_split(&vring_split, vdev);
963	dma_addr);
964	return NULL;	1110	return NULL;
965	}	1111	}
966		1112
967	to_vvq(vq)->split.queue_dma_addr = dma_addr;
968	to_vvq(vq)->split.queue_size_in_bytes = queue_size_in_bytes;
969	to_vvq(vq)->we_own_ring = true;	1113	to_vvq(vq)->we_own_ring = true;
970		1114
971	return vq;	1115	return vq;
972	}	1116	}
973		1117
		1118	static int virtqueue_resize_split(struct virtqueue *_vq, u32 num)
		1119	{
		1120	struct vring_virtqueue_split vring_split = {};
		1121	struct vring_virtqueue *vq = to_vvq(_vq);
		1122	struct virtio_device *vdev = _vq->vdev;
		1123	int err;
		1124
		1125	err = vring_alloc_queue_split(&vring_split, vdev, num,
		1126	vq->split.vring_align,
		1127	vq->split.may_reduce_num);
		1128	if (err)
		1129	goto err;
		1130
		1131	err = vring_alloc_state_extra_split(&vring_split);
		1132	if (err)
		1133	goto err_state_extra;
		1134
		1135	vring_free(&vq->vq);
		1136
		1137	virtqueue_vring_init_split(&vring_split, vq);
		1138
		1139	virtqueue_init(vq, vring_split.vring.num);
		1140	virtqueue_vring_attach_split(vq, &vring_split);
		1141
		1142	return 0;
		1143
		1144	err_state_extra:
		1145	vring_free_split(&vring_split, vdev);
		1146	err:
		1147	virtqueue_reinit_split(vq);
		1148	return -ENOMEM;
		1149	}
		1150
974		1151
975	/*	1152	/*
976	* Packed ring specific functions - *_packed().	1153	* Packed ring specific functions - *_packed().
@@ -1637,8 +1814,7 @@ static void virtqueue_detach_unused_buf_packed(struct virtqueue _vq) · 660 unmodified lines
1637	return NULL;	1814	return NULL;
1638	}	1815	}
1639		1816
1640	static struct vring_desc_extra vring_alloc_desc_extra(struct vring_virtqueue vq,	1817	static struct vring_desc_extra *vring_alloc_desc_extra(unsigned int num)
1641	unsigned int num)
1642	{	1818	{
1643	struct vring_desc_extra *desc_extra;	1819	struct vring_desc_extra *desc_extra;
1644	unsigned int i;	1820	unsigned int i;
@@ -1656,19 +1832,32 @@ static struct vring_desc_extra vring_alloc_desc_extra(struct vring_virtqueue v · 11 unmodified lines
1656	return desc_extra;	1832	return desc_extra;
1657	}	1833	}
1658		1834
1659	static struct virtqueue *vring_create_virtqueue_packed(	1835	static void vring_free_packed(struct vring_virtqueue_packed *vring_packed,
1660	unsigned int index,	1836	struct virtio_device *vdev)
1661	unsigned int num,	1837	{
1662	unsigned int vring_align,	1838	if (vring_packed->vring.desc)
1663	struct virtio_device *vdev,	1839	vring_free_queue(vdev, vring_packed->ring_size_in_bytes,
1664	bool weak_barriers,	1840	vring_packed->vring.desc,
1665	bool may_reduce_num,	1841	vring_packed->ring_dma_addr);
1666	bool context,	1842
1667	bool (notify)(struct virtqueue ),	1843	if (vring_packed->vring.driver)
1668	void (callback)(struct virtqueue ),	1844	vring_free_queue(vdev, vring_packed->event_size_in_bytes,
1669	const char *name)	1845	vring_packed->vring.driver,
		1846	vring_packed->driver_event_dma_addr);
		1847
		1848	if (vring_packed->vring.device)
		1849	vring_free_queue(vdev, vring_packed->event_size_in_bytes,
		1850	vring_packed->vring.device,
		1851	vring_packed->device_event_dma_addr);
		1852
		1853	kfree(vring_packed->desc_state);
		1854	kfree(vring_packed->desc_extra);
		1855	}
		1856
		1857	static int vring_alloc_queue_packed(struct vring_virtqueue_packed *vring_packed,
		1858	struct virtio_device *vdev,
		1859	u32 num)
1670	{	1860	{
1671	struct vring_virtqueue *vq;
1672	struct vring_packed_desc *ring;	1861	struct vring_packed_desc *ring;
1673	struct vring_packed_desc_event driver, device;	1862	struct vring_packed_desc_event driver, device;
1674	dma_addr_t ring_dma_addr, driver_event_dma_addr, device_event_dma_addr;	1863	dma_addr_t ring_dma_addr, driver_event_dma_addr, device_event_dma_addr;
@@ -1680,7 +1869,11 @@ static struct virtqueue *vring_create_virtqueue_packed( · 5 unmodified lines
1680	&ring_dma_addr,	1869	&ring_dma_addr,
1681	GFP_KERNEL\|__GFP_NOWARN\|__GFP_ZERO);	1870	GFP_KERNEL\|__GFP_NOWARN\|__GFP_ZERO);
1682	if (!ring)	1871	if (!ring)
1683	goto err_ring;	1872	goto err;
		1873
		1874	vring_packed->vring.desc = ring;
		1875	vring_packed->ring_dma_addr = ring_dma_addr;
		1876	vring_packed->ring_size_in_bytes = ring_size_in_bytes;
1684		1877
1685	event_size_in_bytes = sizeof(struct vring_packed_desc_event);	1878	event_size_in_bytes = sizeof(struct vring_packed_desc_event);
1686		1879
@@ -1688,13 +1881,112 @@ static struct virtqueue *vring_create_virtqueue_packed( · 1 unmodified line
1688	&driver_event_dma_addr,	1881	&driver_event_dma_addr,
1689	GFP_KERNEL\|__GFP_NOWARN\|__GFP_ZERO);	1882	GFP_KERNEL\|__GFP_NOWARN\|__GFP_ZERO);
1690	if (!driver)	1883	if (!driver)
1691	goto err_driver;	1884	goto err;
		1885
		1886	vring_packed->vring.driver = driver;
		1887	vring_packed->event_size_in_bytes = event_size_in_bytes;
		1888	vring_packed->driver_event_dma_addr = driver_event_dma_addr;
1692		1889
1693	device = vring_alloc_queue(vdev, event_size_in_bytes,	1890	device = vring_alloc_queue(vdev, event_size_in_bytes,
1694	&device_event_dma_addr,	1891	&device_event_dma_addr,
1695	GFP_KERNEL\|__GFP_NOWARN\|__GFP_ZERO);	1892	GFP_KERNEL\|__GFP_NOWARN\|__GFP_ZERO);
1696	if (!device)	1893	if (!device)
1697	goto err_device;	1894	goto err;
		1895
		1896	vring_packed->vring.device = device;
		1897	vring_packed->device_event_dma_addr = device_event_dma_addr;
		1898
		1899	vring_packed->vring.num = num;
		1900
		1901	return 0;
		1902
		1903	err:
		1904	vring_free_packed(vring_packed, vdev);
		1905	return -ENOMEM;
		1906	}
		1907
		1908	static int vring_alloc_state_extra_packed(struct vring_virtqueue_packed *vring_packed)
		1909	{
		1910	struct vring_desc_state_packed *state;
		1911	struct vring_desc_extra *extra;
		1912	u32 num = vring_packed->vring.num;
		1913
		1914	state = kmalloc_array(num, sizeof(struct vring_desc_state_packed), GFP_KERNEL);
		1915	if (!state)
		1916	goto err_desc_state;
		1917
		1918	memset(state, 0, num * sizeof(struct vring_desc_state_packed));
		1919
		1920	extra = vring_alloc_desc_extra(num);
		1921	if (!extra)
		1922	goto err_desc_extra;
		1923
		1924	vring_packed->desc_state = state;
		1925	vring_packed->desc_extra = extra;
		1926
		1927	return 0;
		1928
		1929	err_desc_extra:
		1930	kfree(state);
		1931	err_desc_state:
		1932	return -ENOMEM;
		1933	}
		1934
		1935	static void virtqueue_vring_init_packed(struct vring_virtqueue_packed *vring_packed,
		1936	bool callback)
		1937	{
		1938	vring_packed->next_avail_idx = 0;
		1939	vring_packed->avail_wrap_counter = 1;
		1940	vring_packed->event_flags_shadow = 0;
		1941	vring_packed->avail_used_flags = 1 << VRING_PACKED_DESC_F_AVAIL;
		1942
		1943	/* No callback? Tell other side not to bother us. */
		1944	if (!callback) {
		1945	vring_packed->event_flags_shadow = VRING_PACKED_EVENT_FLAG_DISABLE;
		1946	vring_packed->vring.driver->flags =
		1947	cpu_to_le16(vring_packed->event_flags_shadow);
		1948	}
		1949	}
		1950
		1951	static void virtqueue_vring_attach_packed(struct vring_virtqueue *vq,
		1952	struct vring_virtqueue_packed *vring_packed)
		1953	{
		1954	vq->packed = *vring_packed;
		1955
		1956	/* Put everything in free lists. */
		1957	vq->free_head = 0;
		1958	}
		1959
		1960	static void virtqueue_reinit_packed(struct vring_virtqueue *vq)
		1961	{
		1962	memset(vq->packed.vring.device, 0, vq->packed.event_size_in_bytes);
		1963	memset(vq->packed.vring.driver, 0, vq->packed.event_size_in_bytes);
		1964
		1965	/* we need to reset the desc.flags. For more, see is_used_desc_packed() */
		1966	memset(vq->packed.vring.desc, 0, vq->packed.ring_size_in_bytes);
		1967
		1968	virtqueue_init(vq, vq->packed.vring.num);
		1969	virtqueue_vring_init_packed(&vq->packed, !!vq->vq.callback);
		1970	}
		1971
		1972	static struct virtqueue *vring_create_virtqueue_packed(
		1973	unsigned int index,
		1974	unsigned int num,
		1975	unsigned int vring_align,
		1976	struct virtio_device *vdev,
		1977	bool weak_barriers,
		1978	bool may_reduce_num,
		1979	bool context,
		1980	bool (notify)(struct virtqueue ),
		1981	void (callback)(struct virtqueue ),
		1982	const char *name)
		1983	{
		1984	struct vring_virtqueue_packed vring_packed = {};
		1985	struct vring_virtqueue *vq;
		1986	int err;
		1987
		1988	if (vring_alloc_queue_packed(&vring_packed, vdev, num))
		1989	goto err_ring;
1698		1990
1699	vq = kmalloc(sizeof(*vq), GFP_KERNEL);	1991	vq = kmalloc(sizeof(*vq), GFP_KERNEL);
1700	if (!vq)	1992	if (!vq)
@@ -1703,8 +1995,8 @@ static struct virtqueue *vring_create_virtqueue_packed( · 2 unmodified lines
1703	vq->vq.callback = callback;	1995	vq->vq.callback = callback;
1704	vq->vq.vdev = vdev;	1996	vq->vq.vdev = vdev;
1705	vq->vq.name = name;	1997	vq->vq.name = name;
1706	vq->vq.num_free = num;
1707	vq->vq.index = index;	1998	vq->vq.index = index;
		1999	vq->vq.reset = false;
1708	vq->we_own_ring = true;	2000	vq->we_own_ring = true;
1709	vq->notify = notify;	2001	vq->notify = notify;
1710	vq->weak_barriers = weak_barriers;	2002	vq->weak_barriers = weak_barriers;
@@ -1713,15 +2005,8 @@ static struct virtqueue *vring_create_virtqueue_packed( · 2 unmodified lines
1713	#else	2005	#else
1714	vq->broken = false;	2006	vq->broken = false;
1715	#endif	2007	#endif
1716	vq->last_used_idx = 0 \| (1 << VRING_PACKED_EVENT_F_WRAP_CTR);
1717	vq->event_triggered = false;
1718	vq->num_added = 0;
1719	vq->packed_ring = true;	2008	vq->packed_ring = true;
1720	vq->use_dma_api = vring_use_dma_api(vdev);	2009	vq->use_dma_api = vring_use_dma_api(vdev);
1721	#ifdef DEBUG
1722	vq->in_use = false;
1723	vq->last_add_time_valid = false;
1724	#endif
1725		2010
1726	vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC) &&	2011	vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC) &&
1727	!context;	2012	!context;
@@ -1730,65 +2015,58 @@ static struct virtqueue *vring_create_virtqueue_packed( · 2 unmodified lines
1730	if (virtio_has_feature(vdev, VIRTIO_F_ORDER_PLATFORM))	2015	if (virtio_has_feature(vdev, VIRTIO_F_ORDER_PLATFORM))
1731	vq->weak_barriers = false;	2016	vq->weak_barriers = false;
1732		2017
1733	vq->packed.ring_dma_addr = ring_dma_addr;	2018	err = vring_alloc_state_extra_packed(&vring_packed);
1734	vq->packed.driver_event_dma_addr = driver_event_dma_addr;	2019	if (err)
1735	vq->packed.device_event_dma_addr = device_event_dma_addr;	2020	goto err_state_extra;
1736
1737	vq->packed.ring_size_in_bytes = ring_size_in_bytes;
1738	vq->packed.event_size_in_bytes = event_size_in_bytes;
1739
1740	vq->packed.vring.num = num;
1741	vq->packed.vring.desc = ring;
1742	vq->packed.vring.driver = driver;
1743	vq->packed.vring.device = device;
1744
1745	vq->packed.next_avail_idx = 0;
1746	vq->packed.avail_wrap_counter = 1;
1747	vq->packed.event_flags_shadow = 0;
1748	vq->packed.avail_used_flags = 1 << VRING_PACKED_DESC_F_AVAIL;
1749
1750	vq->packed.desc_state = kmalloc_array(num,
1751	sizeof(struct vring_desc_state_packed),
1752	GFP_KERNEL);
1753	if (!vq->packed.desc_state)
1754	goto err_desc_state;
1755
1756	memset(vq->packed.desc_state, 0,
1757	num * sizeof(struct vring_desc_state_packed));
1758
1759	/* Put everything in free lists. */
1760	vq->free_head = 0;
1761		2021
1762	vq->packed.desc_extra = vring_alloc_desc_extra(vq, num);	2022	virtqueue_vring_init_packed(&vring_packed, !!callback);
1763	if (!vq->packed.desc_extra)
1764	goto err_desc_extra;
1765		2023
1766	/* No callback? Tell other side not to bother us. */	2024	virtqueue_init(vq, num);
1767	if (!callback) {	2025	virtqueue_vring_attach_packed(vq, &vring_packed);
1768	vq->packed.event_flags_shadow = VRING_PACKED_EVENT_FLAG_DISABLE;
1769	vq->packed.vring.driver->flags =
1770	cpu_to_le16(vq->packed.event_flags_shadow);
1771	}
1772		2026
1773	spin_lock(&vdev->vqs_list_lock);	2027	spin_lock(&vdev->vqs_list_lock);
1774	list_add_tail(&vq->vq.list, &vdev->vqs);	2028	list_add_tail(&vq->vq.list, &vdev->vqs);
1775	spin_unlock(&vdev->vqs_list_lock);	2029	spin_unlock(&vdev->vqs_list_lock);
1776	return &vq->vq;	2030	return &vq->vq;
1777		2031
1778	err_desc_extra:	2032	err_state_extra:
1779	kfree(vq->packed.desc_state);
1780	err_desc_state:
1781	kfree(vq);	2033	kfree(vq);
1782	err_vq:	2034	err_vq:
1783	vring_free_queue(vdev, event_size_in_bytes, device, device_event_dma_addr);	2035	vring_free_packed(&vring_packed, vdev);
1784	err_device:
1785	vring_free_queue(vdev, event_size_in_bytes, driver, driver_event_dma_addr);
1786	err_driver:
1787	vring_free_queue(vdev, ring_size_in_bytes, ring, ring_dma_addr);
1788	err_ring:	2036	err_ring:
1789	return NULL;	2037	return NULL;
1790	}	2038	}
1791		2039
		2040	static int virtqueue_resize_packed(struct virtqueue *_vq, u32 num)
		2041	{
		2042	struct vring_virtqueue_packed vring_packed = {};
		2043	struct vring_virtqueue *vq = to_vvq(_vq);
		2044	struct virtio_device *vdev = _vq->vdev;
		2045	int err;
		2046
		2047	if (vring_alloc_queue_packed(&vring_packed, vdev, num))
		2048	goto err_ring;
		2049
		2050	err = vring_alloc_state_extra_packed(&vring_packed);
		2051	if (err)
		2052	goto err_state_extra;
		2053
		2054	vring_free(&vq->vq);
		2055
		2056	virtqueue_vring_init_packed(&vring_packed, !!vq->vq.callback);
		2057
		2058	virtqueue_init(vq, vring_packed.vring.num);
		2059	virtqueue_vring_attach_packed(vq, &vring_packed);
		2060
		2061	return 0;
		2062
		2063	err_state_extra:
		2064	vring_free_packed(&vring_packed, vdev);
		2065	err_ring:
		2066	virtqueue_reinit_packed(vq);
		2067	return -ENOMEM;
		2068	}
		2069
1792		2070
1793	/*	2071	/*
1794	* Generic functions and exported symbols.	2072	* Generic functions and exported symbols.
@@ -2131,8 +2409,8 @@ EXPORT_SYMBOL_GPL(virtqueue_enable_cb_delayed); · 336 unmodified lines
2131	* @_vq: the struct virtqueue we're talking about.	2409	* @_vq: the struct virtqueue we're talking about.
2132	*	2410	*
2133	* Returns NULL or the "data" token handed to virtqueue_add_*().	2411	* Returns NULL or the "data" token handed to virtqueue_add_*().
2134	* This is not valid on an active queue; it is useful only for device	2412	* This is not valid on an active queue; it is useful for device
2135	* shutdown.	2413	* shutdown or the reset queue.
2136	*/	2414	*/
2137	void virtqueue_detach_unused_buf(struct virtqueue _vq)	2415	void virtqueue_detach_unused_buf(struct virtqueue _vq)
2138	{	2416	{
@@ -2180,16 +2458,17 @@ irqreturn_t vring_interrupt(int irq, void *_vq) · 41 unmodified lines
2180	EXPORT_SYMBOL_GPL(vring_interrupt);	2458	EXPORT_SYMBOL_GPL(vring_interrupt);
2181		2459
2182	/* Only available for split ring */	2460	/* Only available for split ring */
2183	struct virtqueue *__vring_new_virtqueue(unsigned int index,	2461	static struct virtqueue *__vring_new_virtqueue(unsigned int index,
2184	struct vring vring,	2462	struct vring_virtqueue_split *vring_split,
2185	struct virtio_device *vdev,	2463	struct virtio_device *vdev,
2186	bool weak_barriers,	2464	bool weak_barriers,
2187	bool context,	2465	bool context,
2188	bool (notify)(struct virtqueue ),	2466	bool (notify)(struct virtqueue ),
2189	void (callback)(struct virtqueue ),	2467	void (callback)(struct virtqueue ),
2190	const char *name)	2468	const char *name)
2191	{	2469	{
2192	struct vring_virtqueue *vq;	2470	struct vring_virtqueue *vq;
		2471	int err;
2193		2472
2194	if (virtio_has_feature(vdev, VIRTIO_F_RING_PACKED))	2473	if (virtio_has_feature(vdev, VIRTIO_F_RING_PACKED))
2195	return NULL;	2474	return NULL;
@@ -2202,8 +2481,8 @@ struct virtqueue *__vring_new_virtqueue(unsigned int index, · 6 unmodified lines
2202	vq->vq.callback = callback;	2481	vq->vq.callback = callback;
2203	vq->vq.vdev = vdev;	2482	vq->vq.vdev = vdev;
2204	vq->vq.name = name;	2483	vq->vq.name = name;
2205	vq->vq.num_free = vring.num;
2206	vq->vq.index = index;	2484	vq->vq.index = index;
		2485	vq->vq.reset = false;
2207	vq->we_own_ring = false;	2486	vq->we_own_ring = false;
2208	vq->notify = notify;	2487	vq->notify = notify;
2209	vq->weak_barriers = weak_barriers;	2488	vq->weak_barriers = weak_barriers;
@@ -2212,14 +2491,7 @@ struct virtqueue *__vring_new_virtqueue(unsigned int index, · 2 unmodified lines
2212	#else	2491	#else
2213	vq->broken = false;	2492	vq->broken = false;
2214	#endif	2493	#endif
2215	vq->last_used_idx = 0;
2216	vq->event_triggered = false;
2217	vq->num_added = 0;
2218	vq->use_dma_api = vring_use_dma_api(vdev);	2494	vq->use_dma_api = vring_use_dma_api(vdev);
2219	#ifdef DEBUG
2220	vq->in_use = false;
2221	vq->last_add_time_valid = false;
2222	#endif
2223		2495
2224	vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC) &&	2496	vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC) &&
2225	!context;	2497	!context;
@@ -2228,47 +2500,22 @@ struct virtqueue *__vring_new_virtqueue(unsigned int index, · 2 unmodified lines
2228	if (virtio_has_feature(vdev, VIRTIO_F_ORDER_PLATFORM))	2500	if (virtio_has_feature(vdev, VIRTIO_F_ORDER_PLATFORM))
2229	vq->weak_barriers = false;	2501	vq->weak_barriers = false;
2230		2502
2231	vq->split.queue_dma_addr = 0;	2503	err = vring_alloc_state_extra_split(vring_split);
2232	vq->split.queue_size_in_bytes = 0;	2504	if (err) {
2233		2505	kfree(vq);
2234	vq->split.vring = vring;	2506	return NULL;
2235	vq->split.avail_flags_shadow = 0;
2236	vq->split.avail_idx_shadow = 0;
2237
2238	/* No callback? Tell other side not to bother us. */
2239	if (!callback) {
2240	vq->split.avail_flags_shadow \|= VRING_AVAIL_F_NO_INTERRUPT;
2241	if (!vq->event)
2242	vq->split.vring.avail->flags = cpu_to_virtio16(vdev,
2243	vq->split.avail_flags_shadow);
2244	}	2507	}
2245		2508
2246	vq->split.desc_state = kmalloc_array(vring.num,	2509	virtqueue_vring_init_split(vring_split, vq);
2247	sizeof(struct vring_desc_state_split), GFP_KERNEL);
2248	if (!vq->split.desc_state)
2249	goto err_state;
2250
2251	vq->split.desc_extra = vring_alloc_desc_extra(vq, vring.num);
2252	if (!vq->split.desc_extra)
2253	goto err_extra;
2254		2510
2255	/* Put everything in free lists. */	2511	virtqueue_init(vq, vring_split->vring.num);
2256	vq->free_head = 0;	2512	virtqueue_vring_attach_split(vq, vring_split);
2257	memset(vq->split.desc_state, 0, vring.num *
2258	sizeof(struct vring_desc_state_split));
2259		2513
2260	spin_lock(&vdev->vqs_list_lock);	2514	spin_lock(&vdev->vqs_list_lock);
2261	list_add_tail(&vq->vq.list, &vdev->vqs);	2515	list_add_tail(&vq->vq.list, &vdev->vqs);
2262	spin_unlock(&vdev->vqs_list_lock);	2516	spin_unlock(&vdev->vqs_list_lock);
2263	return &vq->vq;	2517	return &vq->vq;
2264
2265	err_extra:
2266	kfree(vq->split.desc_state);
2267	err_state:
2268	kfree(vq);
2269	return NULL;
2270	}	2518	}
2271	EXPORT_SYMBOL_GPL(__vring_new_virtqueue);
2272		2519
2273	struct virtqueue *vring_create_virtqueue(	2520	struct virtqueue *vring_create_virtqueue(
2274	unsigned int index,	2521	unsigned int index,
@@ -2294,6 +2541,75 @@ struct virtqueue *vring_create_virtqueue( · 19 unmodified lines
2294	}	2541	}
2295	EXPORT_SYMBOL_GPL(vring_create_virtqueue);	2542	EXPORT_SYMBOL_GPL(vring_create_virtqueue);
2296		2543
		2544	/**
		2545	* virtqueue_resize - resize the vring of vq
		2546	* @_vq: the struct virtqueue we're talking about.
		2547	* @num: new ring num
		2548	* @recycle: callback for recycle the useless buffer
		2549	*
		2550	* When it is really necessary to create a new vring, it will set the current vq
		2551	* into the reset state. Then call the passed callback to recycle the buffer
		2552	* that is no longer used. Only after the new vring is successfully created, the
		2553	* old vring will be released.
		2554	*
		2555	* Caller must ensure we don't call this with other virtqueue operations
		2556	* at the same time (except where noted).
		2557	*
		2558	* Returns zero or a negative error.
		2559	* 0: success.
		2560	* -ENOMEM: Failed to allocate a new ring, fall back to the original ring size.
		2561	* vq can still work normally
		2562	* -EBUSY: Failed to sync with device, vq may not work properly
		2563	* -ENOENT: Transport or device not supported
		2564	* -E2BIG/-EINVAL: num error
		2565	* -EPERM: Operation not permitted
		2566	*
		2567	*/
		2568	int virtqueue_resize(struct virtqueue *_vq, u32 num,
		2569	void (recycle)(struct virtqueue vq, void *buf))
		2570	{
		2571	struct vring_virtqueue *vq = to_vvq(_vq);
		2572	struct virtio_device *vdev = vq->vq.vdev;
		2573	void *buf;
		2574	int err;
		2575
		2576	if (!vq->we_own_ring)
		2577	return -EPERM;
		2578
		2579	if (num > vq->vq.num_max)
		2580	return -E2BIG;
		2581
		2582	if (!num)
		2583	return -EINVAL;
		2584
		2585	if ((vq->packed_ring ? vq->packed.vring.num : vq->split.vring.num) == num)
		2586	return 0;
		2587
		2588	if (!vdev->config->disable_vq_and_reset)
		2589	return -ENOENT;
		2590
		2591	if (!vdev->config->enable_vq_after_reset)
		2592	return -ENOENT;
		2593
		2594	err = vdev->config->disable_vq_and_reset(_vq);
		2595	if (err)
		2596	return err;
		2597
		2598	while ((buf = virtqueue_detach_unused_buf(_vq)) != NULL)
		2599	recycle(_vq, buf);
		2600
		2601	if (vq->packed_ring)
		2602	err = virtqueue_resize_packed(_vq, num);
		2603	else
		2604	err = virtqueue_resize_split(_vq, num);
		2605
		2606	if (vdev->config->enable_vq_after_reset(_vq))
		2607	return -EBUSY;
		2608
		2609	return err;
		2610	}
		2611	EXPORT_SYMBOL_GPL(virtqueue_resize);
		2612
2297	/* Only available for split ring */	2613	/* Only available for split ring */
2298	struct virtqueue *vring_new_virtqueue(unsigned int index,	2614	struct virtqueue *vring_new_virtqueue(unsigned int index,
2299	unsigned int num,	2615	unsigned int num,
@@ -2306,25 +2622,21 @@ struct virtqueue *vring_new_virtqueue(unsigned int index, · 6 unmodified lines
2306	void (callback)(struct virtqueue vq),	2622	void (callback)(struct virtqueue vq),
2307	const char *name)	2623	const char *name)
2308	{	2624	{
2309	struct vring vring;	2625	struct vring_virtqueue_split vring_split = {};
2310		2626
2311	if (virtio_has_feature(vdev, VIRTIO_F_RING_PACKED))	2627	if (virtio_has_feature(vdev, VIRTIO_F_RING_PACKED))
2312	return NULL;	2628	return NULL;
2313		2629
2314	vring_init(&vring, num, pages, vring_align);	2630	vring_init(&vring_split.vring, num, pages, vring_align);
2315	return __vring_new_virtqueue(index, vring, vdev, weak_barriers, context,	2631	return __vring_new_virtqueue(index, &vring_split, vdev, weak_barriers,
2316	notify, callback, name);	2632	context, notify, callback, name);
2317	}	2633	}
2318	EXPORT_SYMBOL_GPL(vring_new_virtqueue);	2634	EXPORT_SYMBOL_GPL(vring_new_virtqueue);
2319		2635
2320	void vring_del_virtqueue(struct virtqueue *_vq)	2636	static void vring_free(struct virtqueue *_vq)
2321	{	2637	{
2322	struct vring_virtqueue *vq = to_vvq(_vq);	2638	struct vring_virtqueue *vq = to_vvq(_vq);
2323		2639
2324	spin_lock(&vq->vq.vdev->vqs_list_lock);
2325	list_del(&_vq->list);
2326	spin_unlock(&vq->vq.vdev->vqs_list_lock);
2327
2328	if (vq->we_own_ring) {	2640	if (vq->we_own_ring) {
2329	if (vq->packed_ring) {	2641	if (vq->packed_ring) {
2330	vring_free_queue(vq->vq.vdev,	2642	vring_free_queue(vq->vq.vdev,
@@ -2355,6 +2667,18 @@ void vring_del_virtqueue(struct virtqueue *_vq) · 24 unmodified lines
2355	kfree(vq->split.desc_state);	2667	kfree(vq->split.desc_state);
2356	kfree(vq->split.desc_extra);	2668	kfree(vq->split.desc_extra);
2357	}	2669	}
		2670	}
		2671
		2672	void vring_del_virtqueue(struct virtqueue *_vq)
		2673	{
		2674	struct vring_virtqueue *vq = to_vvq(_vq);
		2675
		2676	spin_lock(&vq->vq.vdev->vqs_list_lock);
		2677	list_del(&_vq->list);
		2678	spin_unlock(&vq->vq.vdev->vqs_list_lock);
		2679
		2680	vring_free(_vq);
		2681
2358	kfree(vq);	2682	kfree(vq);
2359	}	2683	}
2360	EXPORT_SYMBOL_GPL(vring_del_virtqueue);	2684	EXPORT_SYMBOL_GPL(vring_del_virtqueue);
@@ -2402,6 +2726,30 @@ unsigned int virtqueue_get_vring_size(struct virtqueue *_vq) · 41 unmodified lines
2402	}	2726	}
2403	EXPORT_SYMBOL_GPL(virtqueue_get_vring_size);	2727	EXPORT_SYMBOL_GPL(virtqueue_get_vring_size);
2404		2728
		2729	/*
		2730	* This function should only be called by the core, not directly by the driver.
		2731	*/
		2732	void __virtqueue_break(struct virtqueue *_vq)
		2733	{
		2734	struct vring_virtqueue *vq = to_vvq(_vq);
		2735
		2736	/* Pairs with READ_ONCE() in virtqueue_is_broken(). */
		2737	WRITE_ONCE(vq->broken, true);
		2738	}
		2739	EXPORT_SYMBOL_GPL(__virtqueue_break);
		2740
		2741	/*
		2742	* This function should only be called by the core, not directly by the driver.
		2743	*/
		2744	void __virtqueue_unbreak(struct virtqueue *_vq)
		2745	{
		2746	struct vring_virtqueue *vq = to_vvq(_vq);
		2747
		2748	/* Pairs with READ_ONCE() in virtqueue_is_broken(). */
		2749	WRITE_ONCE(vq->broken, false);
		2750	}
		2751	EXPORT_SYMBOL_GPL(__virtqueue_unbreak);
		2752
2405	bool virtqueue_is_broken(struct virtqueue *_vq)	2753	bool virtqueue_is_broken(struct virtqueue *_vq)
2406	{	2754	{
2407	struct vring_virtqueue *vq = to_vvq(_vq);	2755	struct vring_virtqueue *vq = to_vvq(_vq);

drivers/virtio/virtio_vdpa.c+12−6

 static struct virtqueue *
 virtio_vdpa_setup_vq(struct virtio_device *vdev, unsigned int index,
 		     void (*callback)(struct virtqueue *vq),
-		     const char *name, bool ctx)
+		     const char *name, u32 size, bool ctx)
 {
 	struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vdev);
 	struct vdpa_device *vdpa = vd_get_vdpa(vdev);
 		goto error_new_virtqueue;
 	}
+	if (!size || size > max_num)
+		size = max_num;
 	if (ops->get_vq_num_min)
 		min_num = ops->get_vq_num_min(vdpa);
-	may_reduce_num = (max_num == min_num) ? false : true;
+	may_reduce_num = (size == min_num) ? false : true;
 	/* Create the vring */
 	align = ops->get_vq_align(vdpa);
-	vq = vring_create_virtqueue(index, max_num, align, vdev,
+	vq = vring_create_virtqueue(index, size, align, vdev,
 				    true, may_reduce_num, ctx,
 				    virtio_vdpa_notify, callback, name);
 	if (!vq) {
 		goto error_new_virtqueue;
 	}
+	vq->num_max = max_num;
 	/* Setup virtqueue callback */
 	cb.callback = callback ? virtio_vdpa_virtqueue_cb : NULL;
 	cb.private = info;
 				struct virtqueue *vqs[],
 				vq_callback_t *callbacks[],
 				const char * const names[],
+				u32 sizes[],
 				const bool *ctx,
 				struct irq_affinity *desc)
 {
 			continue;
 		}
-		vqs[i] = virtio_vdpa_setup_vq(vdev, queue_idx++,
+		vqs[i] = virtio_vdpa_setup_vq(vdev, queue_idx++, callbacks[i],
-					      callbacks[i], names[i], ctx ?
+						  names[i], sizes ? sizes[i] : 0,
-					      ctx[i] : false);
+						  ctx ? ctx[i] : false);
 		if (IS_ERR(vqs[i])) {
 			err = PTR_ERR(vqs[i]);
 			goto err_setup_vq;

include/linux/mlx5/mlx5_ifc_vdpa.h+8−0

@@ -150,6 +150,14 @@ enum { · 149 unmodified lines
150	MLX5_VIRTIO_NET_Q_OBJECT_STATE_ERR = 0x3,	150	MLX5_VIRTIO_NET_Q_OBJECT_STATE_ERR = 0x3,
151	};	151	};
152		152
		153	/* This indicates that the object was not created or has already
		154	* been desroyed. It is very safe to assume that this object will never
		155	* have so many states
		156	*/
		157	enum {
		158	MLX5_VIRTIO_NET_Q_OBJECT_NONE = 0xffffffff
		159	};
		160
153	enum {	161	enum {
154	MLX5_RQTC_LIST_Q_TYPE_RQ = 0x0,	162	MLX5_RQTC_LIST_Q_TYPE_RQ = 0x0,
155	MLX5_RQTC_LIST_Q_TYPE_VIRTIO_NET_Q = 0x1,	163	MLX5_RQTC_LIST_Q_TYPE_VIRTIO_NET_Q = 0x1,

include/linux/remoteproc.h+2−2

@@ -597,7 +597,7 @@ struct rproc_subdev { · 596 unmodified lines
597	/**	597	/**
598	* struct rproc_vring - remoteproc vring state	598	* struct rproc_vring - remoteproc vring state
599	* @va: virtual address	599	* @va: virtual address
600	* @len: length, in bytes	600	* @num: vring size
601	* @da: device address	601	* @da: device address
602	* @align: vring alignment	602	* @align: vring alignment
603	* @notifyid: rproc-specific unique vring index	603	* @notifyid: rproc-specific unique vring index
@@ -606,7 +606,7 @@ struct rproc_subdev { · 2 unmodified lines
606	*/	606	*/
607	struct rproc_vring {	607	struct rproc_vring {
608	void *va;	608	void *va;
609	int len;	609	int num;
610	u32 da;	610	u32 da;
611	u32 align;	611	u32 align;
612	int notifyid;	612	int notifyid;

include/linux/vdpa.h+4−0

  * @reset:			Reset device
  *				@vdev: vdpa device
  *				Returns integer: success (0) or error (< 0)
+ * @suspend:			Suspend or resume the device (optional)
+ *				@vdev: vdpa device
+ *				Returns integer: success (0) or error (< 0)
  * @get_config_size:		Get the size of the configuration space includes
  *				fields that are conditional on feature bits.
  *				@vdev: vdpa device
 	u8 (*get_status)(struct vdpa_device *vdev);
 	void (*set_status)(struct vdpa_device *vdev, u8 status);
 	int (*reset)(struct vdpa_device *vdev);
+	int (*suspend)(struct vdpa_device *vdev);
 	size_t (*get_config_size)(struct vdpa_device *vdev);
 	void (*get_config)(struct vdpa_device *vdev, unsigned int offset,
 			   void *buf, unsigned int len);

include/linux/virtio.h+10−0

  * @priv: a pointer for the virtqueue implementation to use.
  * @index: the zero-based ordinal number for this queue.
  * @num_free: number of elements we expect to be able to fit.
+ * @num_max: the maximum number of elements supported by the device.
+ * @reset: vq is in reset state or not.
  *
  * A note on @num_free: with indirect buffers, each buffer needs one
  * element in the queue, otherwise a buffer will need one element per
 	struct virtio_device *vdev;
 	unsigned int index;
 	unsigned int num_free;
+	unsigned int num_max;
 	void *priv;
+	bool reset;
 };
 int virtqueue_add_outbuf(struct virtqueue *vq,
 dma_addr_t virtqueue_get_avail_addr(struct virtqueue *vq);
 dma_addr_t virtqueue_get_used_addr(struct virtqueue *vq);
+int virtqueue_resize(struct virtqueue *vq, u32 num,
+		     void (*recycle)(struct virtqueue *vq, void *buf));
 /**
  * virtio_device - representation of a device using virtio
  * @index: unique position on the virtio bus
 void virtio_break_device(struct virtio_device *dev);
 void __virtio_unbreak_device(struct virtio_device *dev);
+void __virtqueue_break(struct virtqueue *_vq);
+void __virtqueue_unbreak(struct virtqueue *_vq);
 void virtio_config_changed(struct virtio_device *dev);
 #ifdef CONFIG_PM_SLEEP
 int virtio_device_freeze(struct virtio_device *dev);

include/linux/virtio_config.h+35−5

  *		include a NULL entry for vqs that do not need a callback
  *	names: array of virtqueue names (mainly for debugging)
  *		include a NULL entry for vqs unused by driver
+ *	sizes: array of virtqueue sizes
  *	Returns 0 on success or error status
  * @del_vqs: free virtqueues found by find_vqs().
  * @synchronize_cbs: synchronize with the virtqueue callbacks (optional)
  * @set_vq_affinity: set the affinity for a virtqueue (optional).
  * @get_vq_affinity: get the affinity for a virtqueue (optional).
  * @get_shm_region: get a shared memory region based on the index.
+ * @disable_vq_and_reset: reset a queue individually (optional).
+ *	vq: the virtqueue
+ *	Returns 0 on success or error status
+ *	disable_vq_and_reset will guarantee that the callbacks are disabled and
+ *	synchronized.
+ *	Except for the callback, the caller should guarantee that the vring is
+ *	not accessed by any functions of virtqueue.
+ * @enable_vq_after_reset: enable a reset queue
+ *	vq: the virtqueue
+ *	Returns 0 on success or error status
+ *	If disable_vq_and_reset is set, then enable_vq_after_reset must also be
+ *	set.
  */
 typedef void vq_callback_t(struct virtqueue *);
 struct virtio_config_ops {
 	void (*reset)(struct virtio_device *vdev);
 	int (*find_vqs)(struct virtio_device *, unsigned nvqs,
 			struct virtqueue *vqs[], vq_callback_t *callbacks[],
-			const char * const names[], const bool *ctx,
+			const char * const names[],
+			u32 sizes[],
+			const bool *ctx,
 			struct irq_affinity *desc);
 	void (*del_vqs)(struct virtio_device *);
 	void (*synchronize_cbs)(struct virtio_device *);
 			int index);
 	bool (*get_shm_region)(struct virtio_device *vdev,
 			       struct virtio_shm_region *region, u8 id);
+	int (*disable_vq_and_reset)(struct virtqueue *vq);
+	int (*enable_vq_after_reset)(struct virtqueue *vq);
 };
 /* If driver didn't advertise the feature, it will never appear. */
 	const char *names[] = { n };
 	struct virtqueue *vq;
 	int err = vdev->config->find_vqs(vdev, 1, &vq, callbacks, names, NULL,
-					 NULL);
+					 NULL, NULL);
 	if (err < 0)
 		return ERR_PTR(err);
 	return vq;
 			const char * const names[],
 			struct irq_affinity *desc)
 {
-	return vdev->config->find_vqs(vdev, nvqs, vqs, callbacks, names, NULL, desc);
+	return vdev->config->find_vqs(vdev, nvqs, vqs, callbacks, names, NULL,
+				      NULL, desc);
 }
 static inline
 			const char * const names[], const bool *ctx,
 			struct irq_affinity *desc)
 {
-	return vdev->config->find_vqs(vdev, nvqs, vqs, callbacks, names, ctx,
+	return vdev->config->find_vqs(vdev, nvqs, vqs, callbacks, names, NULL,
-				      desc);
+				      ctx, desc);
+}
+static inline
+int virtio_find_vqs_ctx_size(struct virtio_device *vdev, u32 nvqs,
+			     struct virtqueue *vqs[],
+			     vq_callback_t *callbacks[],
+			     const char * const names[],
+			     u32 sizes[],
+			     const bool *ctx, struct irq_affinity *desc)
+{
+	return vdev->config->find_vqs(vdev, nvqs, vqs, callbacks, names, sizes,
+				      ctx, desc);
 }
 /**

include/linux/virtio_pci_modern.h+9−0

 #include <linux/pci.h>
 #include <linux/virtio_pci.h>
+struct virtio_pci_modern_common_cfg {
+	struct virtio_pci_common_cfg cfg;
+	__le16 queue_notify_data;	/* read-write */
+	__le16 queue_reset;		/* read-write */
+};
 struct virtio_pci_modern_device {
 	struct pci_dev *pci_dev;
 				       u16 index, resource_size_t *pa);
 int vp_modern_probe(struct virtio_pci_modern_device *mdev);
 void vp_modern_remove(struct virtio_pci_modern_device *mdev);
+int vp_modern_get_queue_reset(struct virtio_pci_modern_device *mdev, u16 index);
+void vp_modern_set_queue_reset(struct virtio_pci_modern_device *mdev, u16 index);
 #endif

include/linux/virtio_ring.h+0−10

@@ -76,16 +76,6 @@ struct virtqueue *vring_create_virtqueue(unsigned int index, · 75 unmodified lines
76	void (callback)(struct virtqueue vq),	76	void (callback)(struct virtqueue vq),
77	const char *name);	77	const char *name);
78		78
79	/* Creates a virtqueue with a custom layout. */
80	struct virtqueue *__vring_new_virtqueue(unsigned int index,
81	struct vring vring,
82	struct virtio_device *vdev,
83	bool weak_barriers,
84	bool ctx,
85	bool (notify)(struct virtqueue ),
86	void (callback)(struct virtqueue ),
87	const char *name);
88
89	/*	79	/*
90	* Creates a virtqueue with a standard layout but a caller-allocated	80	* Creates a virtqueue with a standard layout but a caller-allocated
91	* ring.	81	* ring.

include/uapi/linux/vduse.h+47−0

@@ -210,6 +210,53 @@ struct vduse_vq_eventfd { · 209 unmodified lines
210	*/	210	*/
211	#define VDUSE_VQ_INJECT_IRQ _IOW(VDUSE_BASE, 0x17, __u32)	211	#define VDUSE_VQ_INJECT_IRQ _IOW(VDUSE_BASE, 0x17, __u32)
212		212
		213	/**
		214	* struct vduse_iova_umem - userspace memory configuration for one IOVA region
		215	* @uaddr: start address of userspace memory, it must be aligned to page size
		216	* @iova: start of the IOVA region
		217	* @size: size of the IOVA region
		218	* @reserved: for future use, needs to be initialized to zero
		219	*
		220	* Structure used by VDUSE_IOTLB_REG_UMEM and VDUSE_IOTLB_DEREG_UMEM
		221	* ioctls to register/de-register userspace memory for IOVA regions
		222	*/
		223	struct vduse_iova_umem {
		224	__u64 uaddr;
		225	__u64 iova;
		226	__u64 size;
		227	__u64 reserved[3];
		228	};
		229
		230	/* Register userspace memory for IOVA regions */
		231	#define VDUSE_IOTLB_REG_UMEM _IOW(VDUSE_BASE, 0x18, struct vduse_iova_umem)
		232
		233	/* De-register the userspace memory. Caller should set iova and size field. */
		234	#define VDUSE_IOTLB_DEREG_UMEM _IOW(VDUSE_BASE, 0x19, struct vduse_iova_umem)
		235
		236	/**
		237	* struct vduse_iova_info - information of one IOVA region
		238	* @start: start of the IOVA region
		239	* @last: last of the IOVA region
		240	* @capability: capability of the IOVA regsion
		241	* @reserved: for future use, needs to be initialized to zero
		242	*
		243	* Structure used by VDUSE_IOTLB_GET_INFO ioctl to get information of
		244	* one IOVA region.
		245	*/
		246	struct vduse_iova_info {
		247	__u64 start;
		248	__u64 last;
		249	#define VDUSE_IOVA_CAP_UMEM (1 << 0)
		250	__u64 capability;
		251	__u64 reserved[3];
		252	};
		253
		254	/*
		255	* Find the first IOVA region that overlaps with the range [start, last]
		256	* and return some information on it. Caller should set start and last fields.
		257	*/
		258	#define VDUSE_IOTLB_GET_INFO _IOWR(VDUSE_BASE, 0x1a, struct vduse_iova_info)
		259
213	/* The control messages definition for read(2)/write(2) on /dev/vduse/$NAME */	260	/* The control messages definition for read(2)/write(2) on /dev/vduse/$NAME */
214		261
215	/**	262	/**

include/uapi/linux/vhost.h+9−0

@@ -171,4 +171,13 @@ · 170 unmodified lines
171	#define VHOST_VDPA_SET_GROUP_ASID _IOW(VHOST_VIRTIO, 0x7C, \	171	#define VHOST_VDPA_SET_GROUP_ASID _IOW(VHOST_VIRTIO, 0x7C, \
172	struct vhost_vring_state)	172	struct vhost_vring_state)
173		173
		174	/* Suspend a device so it does not process virtqueue requests anymore
		175	*
		176	* After the return of ioctl the device must preserve all the necessary state
		177	* (the virtqueue vring base plus the possible device specific states) that is
		178	* required for restoring in the future. The device must not change its
		179	* configuration after that point.
		180	*/
		181	#define VHOST_VDPA_SUSPEND _IO(VHOST_VIRTIO, 0x7D)
		182
174	#endif	183	#endif

include/uapi/linux/vhost_types.h+2−0

@@ -161,5 +161,7 @@ struct vhost_vdpa_iova_range { · 160 unmodified lines
161	* message	161	* message
162	*/	162	*/
163	#define VHOST_BACKEND_F_IOTLB_ASID 0x3	163	#define VHOST_BACKEND_F_IOTLB_ASID 0x3
		164	/* Device can be suspended */
		165	#define VHOST_BACKEND_F_SUSPEND 0x4
164		166
165	#endif	167	#endif

include/uapi/linux/virtio_config.h+6−1

  * rest are per-device feature bits.
  */
 #define VIRTIO_TRANSPORT_F_START	28
-#define VIRTIO_TRANSPORT_F_END		38
+#define VIRTIO_TRANSPORT_F_END		41
 #ifndef VIRTIO_CONFIG_NO_LEGACY
 /* Do we get callbacks when the ring is completely used, even if we've
  * Does the device support Single Root I/O Virtualization?
  */
 #define VIRTIO_F_SR_IOV			37
+/*
+ * This feature indicates that the driver can reset a queue individually.
+ */
+#define VIRTIO_F_RING_RESET		40
 #endif /* _UAPI_LINUX_VIRTIO_CONFIG_H */

include/uapi/linux/virtio_net.h+33−1

@@ -56,7 +56,7 @@ · 55 unmodified lines
56	#define VIRTIO_NET_F_MQ 22 /* Device supports Receive Flow	56	#define VIRTIO_NET_F_MQ 22 /* Device supports Receive Flow
57	* Steering */	57	* Steering */
58	#define VIRTIO_NET_F_CTRL_MAC_ADDR 23 /* Set MAC address */	58	#define VIRTIO_NET_F_CTRL_MAC_ADDR 23 /* Set MAC address */
59		59	#define VIRTIO_NET_F_NOTF_COAL 53 /* Guest can handle notifications coalescing */
60	#define VIRTIO_NET_F_HASH_REPORT 57 /* Supports hash report */	60	#define VIRTIO_NET_F_HASH_REPORT 57 /* Supports hash report */
61	#define VIRTIO_NET_F_RSS 60 /* Supports RSS RX steering */	61	#define VIRTIO_NET_F_RSS 60 /* Supports RSS RX steering */
62	#define VIRTIO_NET_F_RSC_EXT 61 /* extended coalescing info */	62	#define VIRTIO_NET_F_RSC_EXT 61 /* extended coalescing info */
@@ -355,4 +355,36 @@ struct virtio_net_hash_config { · 292 unmodified lines
355	#define VIRTIO_NET_CTRL_GUEST_OFFLOADS 5	355	#define VIRTIO_NET_CTRL_GUEST_OFFLOADS 5
356	#define VIRTIO_NET_CTRL_GUEST_OFFLOADS_SET 0	356	#define VIRTIO_NET_CTRL_GUEST_OFFLOADS_SET 0
357		357
		358	/*
		359	* Control notifications coalescing.
		360	*
		361	* Request the device to change the notifications coalescing parameters.
		362	*
		363	* Available with the VIRTIO_NET_F_NOTF_COAL feature bit.
		364	*/
		365	#define VIRTIO_NET_CTRL_NOTF_COAL 6
		366	/*
		367	* Set the tx-usecs/tx-max-packets patameters.
		368	* tx-usecs - Maximum number of usecs to delay a TX notification.
		369	* tx-max-packets - Maximum number of packets to send before a TX notification.
		370	*/
		371	struct virtio_net_ctrl_coal_tx {
		372	__le32 tx_max_packets;
		373	__le32 tx_usecs;
		374	};
		375
		376	#define VIRTIO_NET_CTRL_NOTF_COAL_TX_SET 0
		377
		378	/*
		379	* Set the rx-usecs/rx-max-packets patameters.
		380	* rx-usecs - Maximum number of usecs to delay a RX notification.
		381	* rx-max-frames - Maximum number of packets to receive before a RX notification.
		382	*/
		383	struct virtio_net_ctrl_coal_rx {
		384	__le32 rx_max_packets;
		385	__le32 rx_usecs;
		386	};
		387
		388	#define VIRTIO_NET_CTRL_NOTF_COAL_RX_SET 1
		389
358	#endif /* _UAPI_LINUX_VIRTIO_NET_H */	390	#endif /* _UAPI_LINUX_VIRTIO_NET_H */

include/uapi/linux/virtio_pci.h+2−0

@@ -202,6 +202,8 @@ struct virtio_pci_cfg_cap { · 201 unmodified lines
202	#define VIRTIO_PCI_COMMON_Q_AVAILHI 44	202	#define VIRTIO_PCI_COMMON_Q_AVAILHI 44
203	#define VIRTIO_PCI_COMMON_Q_USEDLO 48	203	#define VIRTIO_PCI_COMMON_Q_USEDLO 48
204	#define VIRTIO_PCI_COMMON_Q_USEDHI 52	204	#define VIRTIO_PCI_COMMON_Q_USEDHI 52
		205	#define VIRTIO_PCI_COMMON_Q_NDATA 56
		206	#define VIRTIO_PCI_COMMON_Q_RESET 58
205		207
206	#endif /* VIRTIO_PCI_NO_MODERN */	208	#endif /* VIRTIO_PCI_NO_MODERN */
207		209

tools/virtio/linux/kernel.h+1−1

 #define READ                    0
 #define WRITE                   1
-typedef unsigned long long phys_addr_t;
 typedef unsigned long long dma_addr_t;
 typedef size_t __kernel_size_t;
 typedef unsigned int __wsum;
 #endif
 #define dev_err(dev, format, ...) fprintf (stderr, format, ## __VA_ARGS__)
 #define dev_warn(dev, format, ...) fprintf (stderr, format, ## __VA_ARGS__)
+#define dev_warn_once(dev, format, ...) fprintf (stderr, format, ## __VA_ARGS__)
 #define min(x, y) ({				\
 	typeof(x) _min1 = (x);			\

tools/virtio/linux/vringh.h+1−0

@@ -1 +1,2 @@
		1	#include <limits.h>
1	#include "../../../include/linux/vringh.h"	2	#include "../../../include/linux/vringh.h"

tools/virtio/virtio_test.c+2−2

@@ -102,8 +102,8 @@ static void vq_reset(struct vq_info info, int num, struct virtio_device vdev) · 101 unmodified lines
102		102
103	memset(info->ring, 0, vring_size(num, 4096));	103	memset(info->ring, 0, vring_size(num, 4096));
104	vring_init(&info->vring, num, info->ring, 4096);	104	vring_init(&info->vring, num, info->ring, 4096);
105	info->vq = __vring_new_virtqueue(info->idx, info->vring, vdev, true,	105	info->vq = vring_new_virtqueue(info->idx, num, 4096, vdev, true, false,
106	false, vq_notify, vq_callback, "test");	106	info->ring, vq_notify, vq_callback, "test");
107	assert(info->vq);	107	assert(info->vq);
108	info->vq->priv = info;	108	info->vq->priv = info;
109	}	109	}