Merge tag 'for_linus' of git://git.kernel.org/pub/scm/linux/kernel/git/mst/vhost

Pull virtio updates from Michael Tsirkin: - A huge patchset supporting vq resize using the new vq reset capability - Features, fixes, and cleanups all over the place * tag 'for_linus' of git://git.kernel.org/pub/scm/linux/kernel/git/mst/vhost: (88 commits) vdpa/mlx5: Fix possible uninitialized return value vdpa_sim_blk: add support for discard and write-zeroes vdpa_sim_blk: add support for VIRTIO_BLK_T_FLUSH vdpa_sim_blk: make vdpasim_blk_check_range usable by other requests vdpa_sim_blk: check if sector is 0 for commands other than read or write vdpa_sim: Implement suspend vdpa op vhost-vdpa: uAPI to suspend the device vhost-vdpa: introduce SUSPEND backend feature bit vdpa: Add suspend operation virtio-blk: Avoid use-after-free on suspend/resume virtio_vdpa: support the arg sizes of find_vqs() vhost-vdpa: Call ida_simple_remove() when failed vDPA: fix 'cast to restricted le16' warnings in vdpa.c vDPA: !FEATURES_OK should not block querying device config space vDPA/ifcvf: support userspace to query features and MQ of a management device vDPA/ifcvf: get_config_size should return a value no greater than dev implementation vhost scsi: Allow user to control num virtqueues vhost-scsi: Fix max number of virtqueues vdpa/mlx5: Support different address spaces for control and data vdpa/mlx5: Implement susupend virtqueue callback ...

Linus Torvalds committed 4 years ago

Commit 7a53e172 parents 999324f 93e530d51 files · +2148 −533Browse files

Unified · Split

51 files

Documentation/devicetree/bindings/virtio/mmio.yaml +4 −0
arch/um/drivers/virtio_uml.c +2 −1
drivers/block/virtio_blk.c +10 −14
drivers/net/virtio_net.c +286 −39
drivers/nvdimm/virtio_pmem.c +8 −1
drivers/platform/mellanox/mlxbf-tmfifo.c +3 −0
drivers/remoteproc/remoteproc_core.c +2 −2
drivers/remoteproc/remoteproc_virtio.c +8 −5
drivers/s390/virtio/virtio_ccw.c +4 −0
drivers/vdpa/ifcvf/ifcvf_base.c +11 −3
drivers/vdpa/ifcvf/ifcvf_base.h +2 −0
drivers/vdpa/ifcvf/ifcvf_main.c +77 −67
drivers/vdpa/mlx5/core/mlx5_vdpa.h +11 −0
drivers/vdpa/mlx5/net/mlx5_vnet.c +158 −15
drivers/vdpa/vdpa.c +3 −11
drivers/vdpa/vdpa_sim/vdpa_sim.c +16 −2
drivers/vdpa/vdpa_sim/vdpa_sim.h +1 −0
drivers/vdpa/vdpa_sim/vdpa_sim_blk.c +147 −29
drivers/vdpa/vdpa_sim/vdpa_sim_net.c +3 −0
drivers/vdpa/vdpa_user/iova_domain.c +89 −13
drivers/vdpa/vdpa_user/iova_domain.h +8 −0
drivers/vdpa/vdpa_user/vduse_dev.c +180 −0
drivers/vhost/scsi.c +61 −24
drivers/vhost/vdpa.c +36 −2
drivers/vhost/vringh.c +56 −22
drivers/virtio/Kconfig +7 −4
drivers/virtio/virtio.c +3 −1
drivers/virtio/virtio_mmio.c +12 −2
drivers/virtio/virtio_pci_common.c +20 −12
drivers/virtio/virtio_pci_common.h +2 −1
drivers/virtio/virtio_pci_legacy.c +7 −1
drivers/virtio/virtio_pci_modern.c +130 −23
drivers/virtio/virtio_pci_modern_dev.c +39 −0
drivers/virtio/virtio_ring.c +559 −211
drivers/virtio/virtio_vdpa.c +12 −6
include/linux/mlx5/mlx5_ifc_vdpa.h +8 −0
include/linux/remoteproc.h +2 −2
include/linux/vdpa.h +4 −0
include/linux/virtio.h +10 −0
include/linux/virtio_config.h +35 −5
include/linux/virtio_pci_modern.h +9 −0
include/linux/virtio_ring.h +0 −10
include/uapi/linux/vduse.h +47 −0
include/uapi/linux/vhost.h +9 −0
include/uapi/linux/vhost_types.h +2 −0
include/uapi/linux/virtio_config.h +6 −1
include/uapi/linux/virtio_net.h +33 −1
include/uapi/linux/virtio_pci.h +2 −0
tools/virtio/linux/kernel.h +1 −1
tools/virtio/linux/vringh.h +1 −0
tools/virtio/virtio_test.c +2 −2

Documentation/devicetree/bindings/virtio/mmio.yaml+4−0

@@ -33,6 +33,10 @@ properties: · 32 unmodified lines
33	33	description: Required for devices making accesses thru an IOMMU.
34	34	maxItems: 1
35	35
	36	+ wakeup-source:
	37	+ type: boolean
	38	+ description: Required for setting irq of a virtio_mmio device as wakeup source.
	39	+
36	40	required:
37	41	- compatible
38	42	- reg

arch/um/drivers/virtio_uml.c+2−1

@@ -958,6 +958,7 @@ static struct virtqueue vu_setup_vq(struct virtio_device vdev, · 957 unmodified lines
958	958	goto error_create;
959	959	}
960	960	vq->priv = info;
	961	+ vq->num_max = num;
961	962	num = virtqueue_get_vring_size(vq);
962	963
963	964	if (vu_dev->protocol_features &
@@ -1010,7 +1011,7 @@ error_kzalloc: · 46 unmodified lines
1010	1011
1011	1012	static int vu_find_vqs(struct virtio_device *vdev, unsigned nvqs,
1012	1013	struct virtqueue vqs[], vq_callback_t callbacks[],
1013		− const char * const names[], const bool *ctx,
	1014	+ const char * const names[], u32 sizes[], const bool *ctx,
1014	1015	struct irq_affinity *desc)
1015	1016	{
1016	1017	struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev);

drivers/block/virtio_blk.c+10−14

@@ -101,6 +101,14 @@ static inline blk_status_t virtblk_result(struct virtblk_req *vbr) · 100 unmodified lines
101	101	}
102	102	}
103	103
	104	+static inline struct virtio_blk_vq get_virtio_blk_vq(struct blk_mq_hw_ctx hctx)
	105	+{
	106	+ struct virtio_blk *vblk = hctx->queue->queuedata;
	107	+ struct virtio_blk_vq *vq = &vblk->vqs[hctx->queue_num];
	108	+
	109	+ return vq;
	110	+}
	111	+
104	112	static int virtblk_add_req(struct virtqueue vq, struct virtblk_req vbr)
105	113	{
106	114	struct scatterlist hdr, status, *sgs[3];
@@ -416,7 +424,7 @@ static void virtio_queue_rqs(struct request **rqlist) · 309 unmodified lines
416	424	struct request *requeue_list = NULL;
417	425
418	426	rq_list_for_each_safe(rqlist, req, next) {
419		− struct virtio_blk_vq *vq = req->mq_hctx->driver_data;
	427	+ struct virtio_blk_vq *vq = get_virtio_blk_vq(req->mq_hctx);
420	428	bool kick;
421	429
422	430	if (!virtblk_prep_rq_batch(req)) {
@@ -837,7 +845,7 @@ static void virtblk_complete_batch(struct io_comp_batch *iob) · 414 unmodified lines
837	845	static int virtblk_poll(struct blk_mq_hw_ctx hctx, struct io_comp_batch iob)
838	846	{
839	847	struct virtio_blk *vblk = hctx->queue->queuedata;
840		− struct virtio_blk_vq *vq = hctx->driver_data;
	848	+ struct virtio_blk_vq *vq = get_virtio_blk_vq(hctx);
841	849	struct virtblk_req *vbr;
842	850	unsigned long flags;
843	851	unsigned int len;
@@ -862,22 +870,10 @@ static int virtblk_poll(struct blk_mq_hw_ctx hctx, struct io_comp_batch iob) · 18 unmodified lines
862	870	return found;
863	871	}
864	872
865		−static int virtblk_init_hctx(struct blk_mq_hw_ctx hctx, void data,
866		− unsigned int hctx_idx)
867		−{
868		− struct virtio_blk *vblk = data;
869		− struct virtio_blk_vq *vq = &vblk->vqs[hctx_idx];
870		−
871		− WARN_ON(vblk->tag_set.tags[hctx_idx] != hctx->tags);
872		− hctx->driver_data = vq;
873		− return 0;
874		−}
875		−
876	873	static const struct blk_mq_ops virtio_mq_ops = {
877	874	.queue_rq = virtio_queue_rq,
878	875	.queue_rqs = virtio_queue_rqs,
879	876	.commit_rqs = virtio_commit_rqs,
880		− .init_hctx = virtblk_init_hctx,
881	877	.complete = virtblk_request_done,
882	878	.map_queues = virtblk_map_queues,
883	879	.poll = virtblk_poll,

drivers/net/virtio_net.c+286−39

@@ -135,6 +135,9 @@ struct send_queue { · 134 unmodified lines
135	135	struct virtnet_sq_stats stats;
136	136
137	137	struct napi_struct napi;
	138	+
	139	+ /* Record whether sq is in reset state. */
	140	+ bool reset;
138	141	};
139	142
140	143	/* Internal representation of a receive virtqueue */
@@ -267,6 +270,12 @@ struct virtnet_info { · 126 unmodified lines
267	270	u8 duplex;
268	271	u32 speed;
269	272
	273	+ /* Interrupt coalescing settings */
	274	+ u32 tx_usecs;
	275	+ u32 rx_usecs;
	276	+ u32 tx_max_packets;
	277	+ u32 rx_max_packets;
	278	+
270	279	unsigned long guest_offloads;
271	280	unsigned long guest_offloads_capable;
272	281
@@ -284,6 +293,9 @@ struct padded_vnet_hdr { · 11 unmodified lines
284	293	char padding[12];
285	294	};
286	295
	296	+static void virtnet_rq_free_unused_buf(struct virtqueue vq, void buf);
	297	+static void virtnet_sq_free_unused_buf(struct virtqueue vq, void buf);
	298	+
287	299	static bool is_xdp_frame(void *ptr)
288	300	{
289	301	return (unsigned long)ptr & VIRTIO_XDP_FLAG;
@@ -1628,6 +1640,11 @@ static void virtnet_poll_cleantx(struct receive_queue *rq) · 1338 unmodified lines
1628	1640	return;
1629	1641
1630	1642	if (__netif_tx_trylock(txq)) {
	1643	+ if (sq->reset) {
	1644	+ __netif_tx_unlock(txq);
	1645	+ return;
	1646	+ }
	1647	+
1631	1648	do {
1632	1649	virtqueue_disable_cb(sq->vq);
1633	1650	free_old_xmit_skbs(sq, true);
@@ -1875,6 +1892,70 @@ static netdev_tx_t start_xmit(struct sk_buff skb, struct net_device dev) · 241 unmodified lines
1875	1892	return NETDEV_TX_OK;
1876	1893	}
1877	1894
	1895	+static int virtnet_rx_resize(struct virtnet_info *vi,
	1896	+ struct receive_queue *rq, u32 ring_num)
	1897	+{
	1898	+ bool running = netif_running(vi->dev);
	1899	+ int err, qindex;
	1900	+
	1901	+ qindex = rq - vi->rq;
	1902	+
	1903	+ if (running)
	1904	+ napi_disable(&rq->napi);
	1905	+
	1906	+ err = virtqueue_resize(rq->vq, ring_num, virtnet_rq_free_unused_buf);
	1907	+ if (err)
	1908	+ netdev_err(vi->dev, "resize rx fail: rx queue index: %d err: %d\n", qindex, err);
	1909	+
	1910	+ if (!try_fill_recv(vi, rq, GFP_KERNEL))
	1911	+ schedule_delayed_work(&vi->refill, 0);
	1912	+
	1913	+ if (running)
	1914	+ virtnet_napi_enable(rq->vq, &rq->napi);
	1915	+ return err;
	1916	+}
	1917	+
	1918	+static int virtnet_tx_resize(struct virtnet_info *vi,
	1919	+ struct send_queue *sq, u32 ring_num)
	1920	+{
	1921	+ bool running = netif_running(vi->dev);
	1922	+ struct netdev_queue *txq;
	1923	+ int err, qindex;
	1924	+
	1925	+ qindex = sq - vi->sq;
	1926	+
	1927	+ if (running)
	1928	+ virtnet_napi_tx_disable(&sq->napi);
	1929	+
	1930	+ txq = netdev_get_tx_queue(vi->dev, qindex);
	1931	+
	1932	+ /* 1. wait all ximt complete
	1933	+ * 2. fix the race of netif_stop_subqueue() vs netif_start_subqueue()
	1934	+ */
	1935	+ __netif_tx_lock_bh(txq);
	1936	+
	1937	+ /* Prevent rx poll from accessing sq. */
	1938	+ sq->reset = true;
	1939	+
	1940	+ /* Prevent the upper layer from trying to send packets. */
	1941	+ netif_stop_subqueue(vi->dev, qindex);
	1942	+
	1943	+ __netif_tx_unlock_bh(txq);
	1944	+
	1945	+ err = virtqueue_resize(sq->vq, ring_num, virtnet_sq_free_unused_buf);
	1946	+ if (err)
	1947	+ netdev_err(vi->dev, "resize tx fail: tx queue index: %d err: %d\n", qindex, err);
	1948	+
	1949	+ __netif_tx_lock_bh(txq);
	1950	+ sq->reset = false;
	1951	+ netif_tx_wake_queue(txq);
	1952	+ __netif_tx_unlock_bh(txq);
	1953	+
	1954	+ if (running)
	1955	+ virtnet_napi_tx_enable(vi, sq->vq, &sq->napi);
	1956	+ return err;
	1957	+}
	1958	+
1878	1959	/*
1879	1960	* Send command via the control virtqueue and check status. Commands
1880	1961	* supported by the hypervisor, as indicated by feature bits, should
@@ -2285,10 +2366,57 @@ static void virtnet_get_ringparam(struct net_device *dev, · 404 unmodified lines
2285	2366	{
2286	2367	struct virtnet_info *vi = netdev_priv(dev);
2287	2368
2288		− ring->rx_max_pending = virtqueue_get_vring_size(vi->rq[0].vq);
2289		− ring->tx_max_pending = virtqueue_get_vring_size(vi->sq[0].vq);
2290		− ring->rx_pending = ring->rx_max_pending;
2291		− ring->tx_pending = ring->tx_max_pending;
	2369	+ ring->rx_max_pending = vi->rq[0].vq->num_max;
	2370	+ ring->tx_max_pending = vi->sq[0].vq->num_max;
	2371	+ ring->rx_pending = virtqueue_get_vring_size(vi->rq[0].vq);
	2372	+ ring->tx_pending = virtqueue_get_vring_size(vi->sq[0].vq);
	2373	+}
	2374	+
	2375	+static int virtnet_set_ringparam(struct net_device *dev,
	2376	+ struct ethtool_ringparam *ring,
	2377	+ struct kernel_ethtool_ringparam *kernel_ring,
	2378	+ struct netlink_ext_ack *extack)
	2379	+{
	2380	+ struct virtnet_info *vi = netdev_priv(dev);
	2381	+ u32 rx_pending, tx_pending;
	2382	+ struct receive_queue *rq;
	2383	+ struct send_queue *sq;
	2384	+ int i, err;
	2385	+
	2386	+ if (ring->rx_mini_pending \|\| ring->rx_jumbo_pending)
	2387	+ return -EINVAL;
	2388	+
	2389	+ rx_pending = virtqueue_get_vring_size(vi->rq[0].vq);
	2390	+ tx_pending = virtqueue_get_vring_size(vi->sq[0].vq);
	2391	+
	2392	+ if (ring->rx_pending == rx_pending &&
	2393	+ ring->tx_pending == tx_pending)
	2394	+ return 0;
	2395	+
	2396	+ if (ring->rx_pending > vi->rq[0].vq->num_max)
	2397	+ return -EINVAL;
	2398	+
	2399	+ if (ring->tx_pending > vi->sq[0].vq->num_max)
	2400	+ return -EINVAL;
	2401	+
	2402	+ for (i = 0; i < vi->max_queue_pairs; i++) {
	2403	+ rq = vi->rq + i;
	2404	+ sq = vi->sq + i;
	2405	+
	2406	+ if (ring->tx_pending != tx_pending) {
	2407	+ err = virtnet_tx_resize(vi, sq, ring->tx_pending);
	2408	+ if (err)
	2409	+ return err;
	2410	+ }
	2411	+
	2412	+ if (ring->rx_pending != rx_pending) {
	2413	+ err = virtnet_rx_resize(vi, rq, ring->rx_pending);
	2414	+ if (err)
	2415	+ return err;
	2416	+ }
	2417	+ }
	2418	+
	2419	+ return 0;
2292	2420	}
2293	2421
2294	2422	static bool virtnet_commit_rss_command(struct virtnet_info *vi)
@@ -2618,27 +2746,89 @@ static int virtnet_get_link_ksettings(struct net_device *dev, · 323 unmodified lines
2618	2746	return 0;
2619	2747	}
2620	2748
	2749	+static int virtnet_send_notf_coal_cmds(struct virtnet_info *vi,
	2750	+ struct ethtool_coalesce *ec)
	2751	+{
	2752	+ struct scatterlist sgs_tx, sgs_rx;
	2753	+ struct virtio_net_ctrl_coal_tx coal_tx;
	2754	+ struct virtio_net_ctrl_coal_rx coal_rx;
	2755	+
	2756	+ coal_tx.tx_usecs = cpu_to_le32(ec->tx_coalesce_usecs);
	2757	+ coal_tx.tx_max_packets = cpu_to_le32(ec->tx_max_coalesced_frames);
	2758	+ sg_init_one(&sgs_tx, &coal_tx, sizeof(coal_tx));
	2759	+
	2760	+ if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_NOTF_COAL,
	2761	+ VIRTIO_NET_CTRL_NOTF_COAL_TX_SET,
	2762	+ &sgs_tx))
	2763	+ return -EINVAL;
	2764	+
	2765	+ /* Save parameters */
	2766	+ vi->tx_usecs = ec->tx_coalesce_usecs;
	2767	+ vi->tx_max_packets = ec->tx_max_coalesced_frames;
	2768	+
	2769	+ coal_rx.rx_usecs = cpu_to_le32(ec->rx_coalesce_usecs);
	2770	+ coal_rx.rx_max_packets = cpu_to_le32(ec->rx_max_coalesced_frames);
	2771	+ sg_init_one(&sgs_rx, &coal_rx, sizeof(coal_rx));
	2772	+
	2773	+ if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_NOTF_COAL,
	2774	+ VIRTIO_NET_CTRL_NOTF_COAL_RX_SET,
	2775	+ &sgs_rx))
	2776	+ return -EINVAL;
	2777	+
	2778	+ /* Save parameters */
	2779	+ vi->rx_usecs = ec->rx_coalesce_usecs;
	2780	+ vi->rx_max_packets = ec->rx_max_coalesced_frames;
	2781	+
	2782	+ return 0;
	2783	+}
	2784	+
	2785	+static int virtnet_coal_params_supported(struct ethtool_coalesce *ec)
	2786	+{
	2787	+ /* usecs coalescing is supported only if VIRTIO_NET_F_NOTF_COAL
	2788	+ * feature is negotiated.
	2789	+ */
	2790	+ if (ec->rx_coalesce_usecs \|\| ec->tx_coalesce_usecs)
	2791	+ return -EOPNOTSUPP;
	2792	+
	2793	+ if (ec->tx_max_coalesced_frames > 1 \|\|
	2794	+ ec->rx_max_coalesced_frames != 1)
	2795	+ return -EINVAL;
	2796	+
	2797	+ return 0;
	2798	+}
	2799	+
2621	2800	static int virtnet_set_coalesce(struct net_device *dev,
2622	2801	struct ethtool_coalesce *ec,
2623	2802	struct kernel_ethtool_coalesce *kernel_coal,
2624	2803	struct netlink_ext_ack *extack)
2625	2804	{
2626	2805	struct virtnet_info *vi = netdev_priv(dev);
2627		− int i, napi_weight;
2628		−
2629		− if (ec->tx_max_coalesced_frames > 1 \|\|
2630		− ec->rx_max_coalesced_frames != 1)
2631		− return -EINVAL;
	2806	+ int ret, i, napi_weight;
	2807	+ bool update_napi = false;
2632	2808
	2809	+ /* Can't change NAPI weight if the link is up */
2633	2810	napi_weight = ec->tx_max_coalesced_frames ? NAPI_POLL_WEIGHT : 0;
2634	2811	if (napi_weight ^ vi->sq[0].napi.weight) {
2635	2812	if (dev->flags & IFF_UP)
2636	2813	return -EBUSY;
	2814	+ else
	2815	+ update_napi = true;
	2816	+ }
	2817	+
	2818	+ if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_NOTF_COAL))
	2819	+ ret = virtnet_send_notf_coal_cmds(vi, ec);
	2820	+ else
	2821	+ ret = virtnet_coal_params_supported(ec);
	2822	+
	2823	+ if (ret)
	2824	+ return ret;
	2825	+
	2826	+ if (update_napi) {
2637	2827	for (i = 0; i < vi->max_queue_pairs; i++)
2638	2828	vi->sq[i].napi.weight = napi_weight;
2639	2829	}
2640	2830
2641		− return 0;
	2831	+ return ret;
2642	2832	}
2643	2833
2644	2834	static int virtnet_get_coalesce(struct net_device *dev,
@@ -2646,16 +2836,19 @@ static int virtnet_get_coalesce(struct net_device *dev, · 1 unmodified line
2646	2836	struct kernel_ethtool_coalesce *kernel_coal,
2647	2837	struct netlink_ext_ack *extack)
2648	2838	{
2649		− struct ethtool_coalesce ec_default = {
2650		− .cmd = ETHTOOL_GCOALESCE,
2651		− .rx_max_coalesced_frames = 1,
2652		− };
2653	2839	struct virtnet_info *vi = netdev_priv(dev);
2654	2840
2655		− memcpy(ec, &ec_default, sizeof(ec_default));
	2841	+ if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_NOTF_COAL)) {
	2842	+ ec->rx_coalesce_usecs = vi->rx_usecs;
	2843	+ ec->tx_coalesce_usecs = vi->tx_usecs;
	2844	+ ec->tx_max_coalesced_frames = vi->tx_max_packets;
	2845	+ ec->rx_max_coalesced_frames = vi->rx_max_packets;
	2846	+ } else {
	2847	+ ec->rx_max_coalesced_frames = 1;
2656	2848
2657		− if (vi->sq[0].napi.weight)
2658		− ec->tx_max_coalesced_frames = 1;
	2849	+ if (vi->sq[0].napi.weight)
	2850	+ ec->tx_max_coalesced_frames = 1;
	2851	+ }
2659	2852
2660	2853	return 0;
2661	2854	}
@@ -2774,10 +2967,12 @@ static int virtnet_set_rxnfc(struct net_device dev, struct ethtool_rxnfc info) · 112 unmodified lines
2774	2967	}
2775	2968
2776	2969	static const struct ethtool_ops virtnet_ethtool_ops = {
2777		− .supported_coalesce_params = ETHTOOL_COALESCE_MAX_FRAMES,
	2970	+ .supported_coalesce_params = ETHTOOL_COALESCE_MAX_FRAMES \|
	2971	+ ETHTOOL_COALESCE_USECS,
2778	2972	.get_drvinfo = virtnet_get_drvinfo,
2779	2973	.get_link = ethtool_op_get_link,
2780	2974	.get_ringparam = virtnet_get_ringparam,
	2975	+ .set_ringparam = virtnet_set_ringparam,
2781	2976	.get_strings = virtnet_get_strings,
2782	2977	.get_sset_count = virtnet_get_sset_count,
2783	2978	.get_ethtool_stats = virtnet_get_ethtool_stats,
@@ -3171,6 +3366,27 @@ static void free_receive_page_frags(struct virtnet_info *vi) · 387 unmodified lines
3171	3366	put_page(vi->rq[i].alloc_frag.page);
3172	3367	}
3173	3368
	3369	+static void virtnet_sq_free_unused_buf(struct virtqueue vq, void buf)
	3370	+{
	3371	+ if (!is_xdp_frame(buf))
	3372	+ dev_kfree_skb(buf);
	3373	+ else
	3374	+ xdp_return_frame(ptr_to_xdp(buf));
	3375	+}
	3376	+
	3377	+static void virtnet_rq_free_unused_buf(struct virtqueue vq, void buf)
	3378	+{
	3379	+ struct virtnet_info *vi = vq->vdev->priv;
	3380	+ int i = vq2rxq(vq);
	3381	+
	3382	+ if (vi->mergeable_rx_bufs)
	3383	+ put_page(virt_to_head_page(buf));
	3384	+ else if (vi->big_packets)
	3385	+ give_pages(&vi->rq[i], buf);
	3386	+ else
	3387	+ put_page(virt_to_head_page(buf));
	3388	+}
	3389	+
3174	3390	static void free_unused_bufs(struct virtnet_info *vi)
3175	3391	{
3176	3392	void *buf;
@@ -3178,26 +3394,14 @@ static void free_unused_bufs(struct virtnet_info *vi) · 1 unmodified line
3178	3394
3179	3395	for (i = 0; i < vi->max_queue_pairs; i++) {
3180	3396	struct virtqueue *vq = vi->sq[i].vq;
3181		− while ((buf = virtqueue_detach_unused_buf(vq)) != NULL) {
3182		− if (!is_xdp_frame(buf))
3183		− dev_kfree_skb(buf);
3184		− else
3185		− xdp_return_frame(ptr_to_xdp(buf));
3186		− }
	3397	+ while ((buf = virtqueue_detach_unused_buf(vq)) != NULL)
	3398	+ virtnet_sq_free_unused_buf(vq, buf);
3187	3399	}
3188	3400
3189	3401	for (i = 0; i < vi->max_queue_pairs; i++) {
3190	3402	struct virtqueue *vq = vi->rq[i].vq;
3191		−
3192		− while ((buf = virtqueue_detach_unused_buf(vq)) != NULL) {
3193		− if (vi->mergeable_rx_bufs) {
3194		− put_page(virt_to_head_page(buf));
3195		− } else if (vi->big_packets) {
3196		− give_pages(&vi->rq[i], buf);
3197		− } else {
3198		− put_page(virt_to_head_page(buf));
3199		− }
3200		− }
	3403	+ while ((buf = virtqueue_detach_unused_buf(vq)) != NULL)
	3404	+ virtnet_rq_free_unused_buf(vq, buf);
3201	3405	}
3202	3406	}
3203	3407
@@ -3228,6 +3432,29 @@ static unsigned int mergeable_min_buf_len(struct virtnet_info *vi, struct virtqu · 24 unmodified lines
3228	3432	(unsigned int)GOOD_PACKET_LEN);
3229	3433	}
3230	3434
	3435	+static void virtnet_config_sizes(struct virtnet_info vi, u32 sizes)
	3436	+{
	3437	+ u32 i, rx_size, tx_size;
	3438	+
	3439	+ if (vi->speed == SPEED_UNKNOWN \|\| vi->speed < SPEED_10000) {
	3440	+ rx_size = 1024;
	3441	+ tx_size = 1024;
	3442	+
	3443	+ } else if (vi->speed < SPEED_40000) {
	3444	+ rx_size = 1024 * 4;
	3445	+ tx_size = 1024 * 4;
	3446	+
	3447	+ } else {
	3448	+ rx_size = 1024 * 8;
	3449	+ tx_size = 1024 * 8;
	3450	+ }
	3451	+
	3452	+ for (i = 0; i < vi->max_queue_pairs; i++) {
	3453	+ sizes[rxq2vq(i)] = rx_size;
	3454	+ sizes[txq2vq(i)] = tx_size;
	3455	+ }
	3456	+}
	3457	+
3231	3458	static int virtnet_find_vqs(struct virtnet_info *vi)
3232	3459	{
3233	3460	vq_callback_t **callbacks;
@@ -3235,6 +3462,7 @@ static int virtnet_find_vqs(struct virtnet_info *vi) · 1 unmodified line
3235	3462	int ret = -ENOMEM;
3236	3463	int i, total_vqs;
3237	3464	const char **names;
	3465	+ u32 *sizes;
3238	3466	bool *ctx;
3239	3467
3240	3468	/* We expect 1 RX virtqueue followed by 1 TX virtqueue, followed by
@@ -3262,10 +3490,15 @@ static int virtnet_find_vqs(struct virtnet_info *vi) · 21 unmodified lines
3262	3490	ctx = NULL;
3263	3491	}
3264	3492
	3493	+ sizes = kmalloc_array(total_vqs, sizeof(*sizes), GFP_KERNEL);
	3494	+ if (!sizes)
	3495	+ goto err_sizes;
	3496	+
3265	3497	/* Parameters for control virtqueue, if any */
3266	3498	if (vi->has_cvq) {
3267	3499	callbacks[total_vqs - 1] = NULL;
3268	3500	names[total_vqs - 1] = "control";
	3501	+ sizes[total_vqs - 1] = 64;
3269	3502	}
3270	3503
3271	3504	/* Allocate/initialize parameters for send/receive virtqueues */
@@ -3280,8 +3513,10 @@ static int virtnet_find_vqs(struct virtnet_info *vi) · 8 unmodified lines
3280	3513	ctx[rxq2vq(i)] = true;
3281	3514	}
3282	3515
3283		− ret = virtio_find_vqs_ctx(vi->vdev, total_vqs, vqs, callbacks,
3284		− names, ctx, NULL);
	3516	+ virtnet_config_sizes(vi, sizes);
	3517	+
	3518	+ ret = virtio_find_vqs_ctx_size(vi->vdev, total_vqs, vqs, callbacks,
	3519	+ names, sizes, ctx, NULL);
3285	3520	if (ret)
3286	3521	goto err_find;
3287	3522
@@ -3301,6 +3536,8 @@ static int virtnet_find_vqs(struct virtnet_info *vi) · 13 unmodified lines
3301	3536
3302	3537
3303	3538	err_find:
	3539	+ kfree(sizes);
	3540	+err_sizes:
3304	3541	kfree(ctx);
3305	3542	err_ctx:
3306	3543	kfree(names);
@@ -3444,6 +3681,8 @@ static bool virtnet_validate_features(struct virtio_device *vdev) · 137 unmodified lines
3444	3681	VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_RSS,
3445	3682	"VIRTIO_NET_F_CTRL_VQ") \|\|
3446	3683	VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_HASH_REPORT,
	3684	+ "VIRTIO_NET_F_CTRL_VQ") \|\|
	3685	+ VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_NOTF_COAL,
3447	3686	"VIRTIO_NET_F_CTRL_VQ"))) {
3448	3687	return false;
3449	3688	}
@@ -3580,6 +3819,13 @@ static int virtnet_probe(struct virtio_device *vdev) · 130 unmodified lines
3580	3819	if (virtio_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF))
3581	3820	vi->mergeable_rx_bufs = true;
3582	3821
	3822	+ if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_NOTF_COAL)) {
	3823	+ vi->rx_usecs = 0;
	3824	+ vi->tx_usecs = 0;
	3825	+ vi->tx_max_packets = 0;
	3826	+ vi->rx_max_packets = 0;
	3827	+ }
	3828	+
3583	3829	if (virtio_has_feature(vdev, VIRTIO_NET_F_HASH_REPORT))
3584	3830	vi->has_rss_hash_report = true;
3585	3831
@@ -3651,6 +3897,9 @@ static int virtnet_probe(struct virtio_device *vdev) · 65 unmodified lines
3651	3897	vi->curr_queue_pairs = num_online_cpus();
3652	3898	vi->max_queue_pairs = max_queue_pairs;
3653	3899
	3900	+ virtnet_init_settings(dev);
	3901	+ virtnet_update_settings(vi);
	3902	+
3654	3903	/* Allocate/initialize the rx/tx queues, and invoke find_vqs */
3655	3904	err = init_vqs(vi);
3656	3905	if (err)
@@ -3663,8 +3912,6 @@ static int virtnet_probe(struct virtio_device *vdev) · 6 unmodified lines
3663	3912	netif_set_real_num_tx_queues(dev, vi->curr_queue_pairs);
3664	3913	netif_set_real_num_rx_queues(dev, vi->curr_queue_pairs);
3665	3914
3666		− virtnet_init_settings(dev);
3667		−
3668	3915	if (virtio_has_feature(vdev, VIRTIO_NET_F_STANDBY)) {
3669	3916	vi->failover = net_failover_create(vi->dev);
3670	3917	if (IS_ERR(vi->failover)) {
@@ -3814,7 +4061,7 @@ static struct virtio_device_id id_table[] = { · 143 unmodified lines
3814	4061	VIRTIO_NET_F_CTRL_MAC_ADDR, \
3815	4062	VIRTIO_NET_F_MTU, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS, \
3816	4063	VIRTIO_NET_F_SPEED_DUPLEX, VIRTIO_NET_F_STANDBY, \
3817		− VIRTIO_NET_F_RSS, VIRTIO_NET_F_HASH_REPORT
	4064	+ VIRTIO_NET_F_RSS, VIRTIO_NET_F_HASH_REPORT, VIRTIO_NET_F_NOTF_COAL
3818	4065
3819	4066	static unsigned int features[] = {
3820	4067	VIRTNET_FEATURES,

drivers/nvdimm/virtio_pmem.c+8−1

@@ -81,17 +81,24 @@ static int virtio_pmem_probe(struct virtio_device *vdev) · 80 unmodified lines
81	81	ndr_desc.res = &res;
82	82	ndr_desc.numa_node = nid;
83	83	ndr_desc.flush = async_pmem_flush;
	84	+ ndr_desc.provider_data = vdev;
84	85	set_bit(ND_REGION_PAGEMAP, &ndr_desc.flags);
85	86	set_bit(ND_REGION_ASYNC, &ndr_desc.flags);
	87	+ /*
	88	+ * The NVDIMM region could be available before the
	89	+ * virtio_device_ready() that is called by
	90	+ * virtio_dev_probe(), so we set device ready here.
	91	+ */
	92	+ virtio_device_ready(vdev);
86	93	nd_region = nvdimm_pmem_region_create(vpmem->nvdimm_bus, &ndr_desc);
87	94	if (!nd_region) {
88	95	dev_err(&vdev->dev, "failed to create nvdimm region\n");
89	96	err = -ENXIO;
90	97	goto out_nd;
91	98	}
92		− nd_region->provider_data = dev_to_virtio(nd_region->dev.parent->parent);
93	99	return 0;
94	100	out_nd:
	101	+ virtio_reset_device(vdev);
95	102	nvdimm_bus_unregister(vpmem->nvdimm_bus);
96	103	out_vq:
97	104	vdev->config->del_vqs(vdev);

drivers/platform/mellanox/mlxbf-tmfifo.c+3−0

@@ -928,6 +928,7 @@ static int mlxbf_tmfifo_virtio_find_vqs(struct virtio_device *vdev, · 927 unmodified lines
928	928	struct virtqueue *vqs[],
929	929	vq_callback_t *callbacks[],
930	930	const char * const names[],
	931	+ u32 sizes[],
931	932	const bool *ctx,
932	933	struct irq_affinity *desc)
933	934	{
@@ -959,6 +960,8 @@ static int mlxbf_tmfifo_virtio_find_vqs(struct virtio_device *vdev, · 25 unmodified lines
959	960	goto error;
960	961	}
961	962
	963	+ vq->num_max = vring->num;
	964	+
962	965	vqs[i] = vq;
963	966	vring->vq = vq;
964	967	vq->priv = vring;

drivers/remoteproc/remoteproc_core.c+2−2

@@ -335,7 +335,7 @@ int rproc_alloc_vring(struct rproc_vdev *rvdev, int i) · 334 unmodified lines
335	335	size_t size;
336	336
337	337	/* actual size of vring (in bytes) */
338		− size = PAGE_ALIGN(vring_size(rvring->len, rvring->align));
	338	+ size = PAGE_ALIGN(vring_size(rvring->num, rvring->align));
339	339
340	340	rsc = (void *)rproc->table_ptr + rvdev->rsc_offset;
341	341
@@ -402,7 +402,7 @@ rproc_parse_vring(struct rproc_vdev rvdev, struct fw_rsc_vdev rsc, int i) · 60 unmodified lines
402	402	return -EINVAL;
403	403	}
404	404
405		− rvring->len = vring->num;
	405	+ rvring->num = vring->num;
406	406	rvring->align = vring->align;
407	407	rvring->rvdev = rvdev;
408	408

drivers/remoteproc/remoteproc_virtio.c+8−5

@@ -87,7 +87,7 @@ static struct virtqueue rp_find_vq(struct virtio_device vdev, · 86 unmodified lines
87	87	struct fw_rsc_vdev *rsc;
88	88	struct virtqueue *vq;
89	89	void *addr;
90		− int len, size;
	90	+ int num, size;
91	91
92	92	/* we're temporarily limited to two virtqueues per rvdev */
93	93	if (id >= ARRAY_SIZE(rvdev->vring))
@@ -104,20 +104,20 @@ static struct virtqueue rp_find_vq(struct virtio_device vdev, · 10 unmodified lines
104	104
105	105	rvring = &rvdev->vring[id];
106	106	addr = mem->va;
107		− len = rvring->len;
	107	+ num = rvring->num;
108	108
109	109	/* zero vring */
110		− size = vring_size(len, rvring->align);
	110	+ size = vring_size(num, rvring->align);
111	111	memset(addr, 0, size);
112	112
113	113	dev_dbg(dev, "vring%d: va %pK qsz %d notifyid %d\n",
114		− id, addr, len, rvring->notifyid);
	114	+ id, addr, num, rvring->notifyid);
115	115
116	116	/*
117	117	* Create the new vq, and tell virtio we're not interested in
118	118	* the 'weak' smp barriers, since we're talking with a real device.
119	119	*/
120		− vq = vring_new_virtqueue(id, len, rvring->align, vdev, false, ctx,
	120	+ vq = vring_new_virtqueue(id, num, rvring->align, vdev, false, ctx,
121	121	addr, rproc_virtio_notify, callback, name);
122	122	if (!vq) {
123	123	dev_err(dev, "vring_new_virtqueue %s failed\n", name);
@@ -125,6 +125,8 @@ static struct virtqueue rp_find_vq(struct virtio_device vdev, · 1 unmodified line
125	125	return ERR_PTR(-ENOMEM);
126	126	}
127	127
	128	+ vq->num_max = num;
	129	+
128	130	rvring->vq = vq;
129	131	vq->priv = rvring;
130	132
@@ -156,6 +158,7 @@ static int rproc_virtio_find_vqs(struct virtio_device *vdev, unsigned int nvqs, · 25 unmodified lines
156	158	struct virtqueue *vqs[],
157	159	vq_callback_t *callbacks[],
158	160	const char * const names[],
	161	+ u32 sizes[],
159	162	const bool * ctx,
160	163	struct irq_affinity *desc)
161	164	{

drivers/s390/virtio/virtio_ccw.c+4−0

@@ -532,6 +532,9 @@ static struct virtqueue virtio_ccw_setup_vq(struct virtio_device vdev, · 531 unmodified lines
532	532	err = -ENOMEM;
533	533	goto out_err;
534	534	}
	535	+
	536	+ vq->num_max = info->num;
	537	+
535	538	/* it may have been reduced */
536	539	info->num = virtqueue_get_vring_size(vq);
537	540
@@ -634,6 +637,7 @@ static int virtio_ccw_find_vqs(struct virtio_device *vdev, unsigned nvqs, · 96 unmodified lines
634	637	struct virtqueue *vqs[],
635	638	vq_callback_t *callbacks[],
636	639	const char * const names[],
	640	+ u32 sizes[],
637	641	const bool *ctx,
638	642	struct irq_affinity *desc)
639	643	{

drivers/vdpa/ifcvf/ifcvf_base.c+11−3

@@ -29,7 +29,6 @@ u16 ifcvf_set_config_vector(struct ifcvf_hw *hw, int vector) · 28 unmodified lines
29	29	{
30	30	struct virtio_pci_common_cfg __iomem *cfg = hw->common_cfg;
31	31
32		− cfg = hw->common_cfg;
33	32	vp_iowrite16(vector, &cfg->msix_config);
34	33
35	34	return vp_ioread16(&cfg->msix_config);
@@ -128,6 +127,7 @@ int ifcvf_init_hw(struct ifcvf_hw hw, struct pci_dev pdev) · 92 unmodified lines
128	127	break;
129	128	case VIRTIO_PCI_CAP_DEVICE_CFG:
130	129	hw->dev_cfg = get_cap_addr(hw, &cap);
	130	+ hw->cap_dev_config_size = le32_to_cpu(cap.length);
131	131	IFCVF_DBG(pdev, "hw->dev_cfg = %p\n", hw->dev_cfg);
132	132	break;
133	133	}
@@ -233,15 +233,23 @@ int ifcvf_verify_min_features(struct ifcvf_hw *hw, u64 features) · 99 unmodified lines
233	233	u32 ifcvf_get_config_size(struct ifcvf_hw *hw)
234	234	{
235	235	struct ifcvf_adapter *adapter;
	236	+ u32 net_config_size = sizeof(struct virtio_net_config);
	237	+ u32 blk_config_size = sizeof(struct virtio_blk_config);
	238	+ u32 cap_size = hw->cap_dev_config_size;
236	239	u32 config_size;
237	240
238	241	adapter = vf_to_adapter(hw);
	242	+ /* If the onboard device config space size is greater than
	243	+ * the size of struct virtio_net/blk_config, only the spec
	244	+ * implementing contents size is returned, this is very
	245	+ * unlikely, defensive programming.
	246	+ */
239	247	switch (hw->dev_type) {
240	248	case VIRTIO_ID_NET:
241		− config_size = sizeof(struct virtio_net_config);
	249	+ config_size = min(cap_size, net_config_size);
242	250	break;
243	251	case VIRTIO_ID_BLOCK:
244		− config_size = sizeof(struct virtio_blk_config);
	252	+ config_size = min(cap_size, blk_config_size);
245	253	break;
246	254	default:
247	255	config_size = 0;

drivers/vdpa/ifcvf/ifcvf_base.h+2−0

@@ -87,6 +87,8 @@ struct ifcvf_hw { · 86 unmodified lines
87	87	int config_irq;
88	88	int vqs_reused_irq;
89	89	u16 nr_vring;
	90	+ /* VIRTIO_PCI_CAP_DEVICE_CFG size */
	91	+ u32 cap_dev_config_size;
90	92	};
91	93
92	94	struct ifcvf_adapter {

drivers/vdpa/ifcvf/ifcvf_main.c+77−67

@@ -685,7 +685,7 @@ static struct vdpa_notification_area ifcvf_get_vq_notification(struct vdpa_devic · 684 unmodified lines
685	685	}
686	686
687	687	/*
688		− * IFCVF currently does't have on-chip IOMMU, so not
	688	+ * IFCVF currently doesn't have on-chip IOMMU, so not
689	689	* implemented set_map()/dma_map()/dma_unmap()
690	690	*/
691	691	static const struct vdpa_config_ops ifc_vdpa_ops = {
@@ -752,59 +752,36 @@ static int ifcvf_vdpa_dev_add(struct vdpa_mgmt_dev mdev, const char name, · 60 unmodified lines
752	752	{
753	753	struct ifcvf_vdpa_mgmt_dev *ifcvf_mgmt_dev;
754	754	struct ifcvf_adapter *adapter;
	755	+ struct vdpa_device *vdpa_dev;
755	756	struct pci_dev *pdev;
756	757	struct ifcvf_hw *vf;
757		− struct device *dev;
758		− int ret, i;
	758	+ int ret;
759	759
760	760	ifcvf_mgmt_dev = container_of(mdev, struct ifcvf_vdpa_mgmt_dev, mdev);
761		− if (ifcvf_mgmt_dev->adapter)
	761	+ if (!ifcvf_mgmt_dev->adapter)
762	762	return -EOPNOTSUPP;
763	763
764		− pdev = ifcvf_mgmt_dev->pdev;
765		− dev = &pdev->dev;
766		− adapter = vdpa_alloc_device(struct ifcvf_adapter, vdpa,
767		− dev, &ifc_vdpa_ops, 1, 1, name, false);
768		− if (IS_ERR(adapter)) {
769		− IFCVF_ERR(pdev, "Failed to allocate vDPA structure");
770		− return PTR_ERR(adapter);
771		− }
772		−
773		− ifcvf_mgmt_dev->adapter = adapter;
774		−
	764	+ adapter = ifcvf_mgmt_dev->adapter;
775	765	vf = &adapter->vf;
776		− vf->dev_type = get_dev_type(pdev);
777		− vf->base = pcim_iomap_table(pdev);
	766	+ pdev = adapter->pdev;
	767	+ vdpa_dev = &adapter->vdpa;
778	768
779		− adapter->pdev = pdev;
780		− adapter->vdpa.dma_dev = &pdev->dev;
781		−
782		− ret = ifcvf_init_hw(vf, pdev);
783		− if (ret) {
784		− IFCVF_ERR(pdev, "Failed to init IFCVF hw\n");
785		− goto err;
786		− }
787		−
788		− for (i = 0; i < vf->nr_vring; i++)
789		− vf->vring[i].irq = -EINVAL;
790		−
791		− vf->hw_features = ifcvf_get_hw_features(vf);
792		− vf->config_size = ifcvf_get_config_size(vf);
	769	+ if (name)
	770	+ ret = dev_set_name(&vdpa_dev->dev, "%s", name);
	771	+ else
	772	+ ret = dev_set_name(&vdpa_dev->dev, "vdpa%u", vdpa_dev->index);
793	773
794		− adapter->vdpa.mdev = &ifcvf_mgmt_dev->mdev;
795	774	ret = _vdpa_register_device(&adapter->vdpa, vf->nr_vring);
796	775	if (ret) {
	776	+ put_device(&adapter->vdpa.dev);
797	777	IFCVF_ERR(pdev, "Failed to register to vDPA bus");
798		− goto err;
	778	+ return ret;
799	779	}
800	780
801	781	return 0;
802		−
803		−err:
804		− put_device(&adapter->vdpa.dev);
805		− return ret;
806	782	}
807	783
	784	+
808	785	static void ifcvf_vdpa_dev_del(struct vdpa_mgmt_dev mdev, struct vdpa_device dev)
809	786	{
810	787	struct ifcvf_vdpa_mgmt_dev *ifcvf_mgmt_dev;
@@ -823,61 +800,94 @@ static int ifcvf_probe(struct pci_dev pdev, const struct pci_device_id id) · 12 unmodified lines
823	800	{
824	801	struct ifcvf_vdpa_mgmt_dev *ifcvf_mgmt_dev;
825	802	struct device *dev = &pdev->dev;
	803	+ struct ifcvf_adapter *adapter;
	804	+ struct ifcvf_hw *vf;
826	805	u32 dev_type;
827		− int ret;
828		−
829		− ifcvf_mgmt_dev = kzalloc(sizeof(struct ifcvf_vdpa_mgmt_dev), GFP_KERNEL);
830		− if (!ifcvf_mgmt_dev) {
831		− IFCVF_ERR(pdev, "Failed to alloc memory for the vDPA management device\n");
832		− return -ENOMEM;
833		− }
834		−
835		− dev_type = get_dev_type(pdev);
836		− switch (dev_type) {
837		− case VIRTIO_ID_NET:
838		− ifcvf_mgmt_dev->mdev.id_table = id_table_net;
839		− break;
840		− case VIRTIO_ID_BLOCK:
841		− ifcvf_mgmt_dev->mdev.id_table = id_table_blk;
842		− break;
843		− default:
844		− IFCVF_ERR(pdev, "VIRTIO ID %u not supported\n", dev_type);
845		− ret = -EOPNOTSUPP;
846		− goto err;
847		− }
848		−
849		− ifcvf_mgmt_dev->mdev.ops = &ifcvf_vdpa_mgmt_dev_ops;
850		− ifcvf_mgmt_dev->mdev.device = dev;
851		− ifcvf_mgmt_dev->pdev = pdev;
	806	+ int ret, i;
852	807
853	808	ret = pcim_enable_device(pdev);
854	809	if (ret) {
855	810	IFCVF_ERR(pdev, "Failed to enable device\n");
856		− goto err;
	811	+ return ret;
857	812	}
858		−
859	813	ret = pcim_iomap_regions(pdev, BIT(0) \| BIT(2) \| BIT(4),
860	814	IFCVF_DRIVER_NAME);
861	815	if (ret) {
862	816	IFCVF_ERR(pdev, "Failed to request MMIO region\n");
863		− goto err;
	817	+ return ret;
864	818	}
865	819
866	820	ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));
867	821	if (ret) {
868	822	IFCVF_ERR(pdev, "No usable DMA configuration\n");
869		− goto err;
	823	+ return ret;
870	824	}
871	825
872	826	ret = devm_add_action_or_reset(dev, ifcvf_free_irq_vectors, pdev);
873	827	if (ret) {
874	828	IFCVF_ERR(pdev,
875	829	"Failed for adding devres for freeing irq vectors\n");
876		− goto err;
	830	+ return ret;
877	831	}
878	832
879	833	pci_set_master(pdev);
880	834
	835	+ adapter = vdpa_alloc_device(struct ifcvf_adapter, vdpa,
	836	+ dev, &ifc_vdpa_ops, 1, 1, NULL, false);
	837	+ if (IS_ERR(adapter)) {
	838	+ IFCVF_ERR(pdev, "Failed to allocate vDPA structure");
	839	+ return PTR_ERR(adapter);
	840	+ }
	841	+
	842	+ vf = &adapter->vf;
	843	+ vf->dev_type = get_dev_type(pdev);
	844	+ vf->base = pcim_iomap_table(pdev);
	845	+
	846	+ adapter->pdev = pdev;
	847	+ adapter->vdpa.dma_dev = &pdev->dev;
	848	+
	849	+ ret = ifcvf_init_hw(vf, pdev);
	850	+ if (ret) {
	851	+ IFCVF_ERR(pdev, "Failed to init IFCVF hw\n");
	852	+ return ret;
	853	+ }
	854	+
	855	+ for (i = 0; i < vf->nr_vring; i++)
	856	+ vf->vring[i].irq = -EINVAL;
	857	+
	858	+ vf->hw_features = ifcvf_get_hw_features(vf);
	859	+ vf->config_size = ifcvf_get_config_size(vf);
	860	+
	861	+ ifcvf_mgmt_dev = kzalloc(sizeof(struct ifcvf_vdpa_mgmt_dev), GFP_KERNEL);
	862	+ if (!ifcvf_mgmt_dev) {
	863	+ IFCVF_ERR(pdev, "Failed to alloc memory for the vDPA management device\n");
	864	+ return -ENOMEM;
	865	+ }
	866	+
	867	+ ifcvf_mgmt_dev->mdev.ops = &ifcvf_vdpa_mgmt_dev_ops;
	868	+ ifcvf_mgmt_dev->mdev.device = dev;
	869	+ ifcvf_mgmt_dev->adapter = adapter;
	870	+
	871	+ dev_type = get_dev_type(pdev);
	872	+ switch (dev_type) {
	873	+ case VIRTIO_ID_NET:
	874	+ ifcvf_mgmt_dev->mdev.id_table = id_table_net;
	875	+ break;
	876	+ case VIRTIO_ID_BLOCK:
	877	+ ifcvf_mgmt_dev->mdev.id_table = id_table_blk;
	878	+ break;
	879	+ default:
	880	+ IFCVF_ERR(pdev, "VIRTIO ID %u not supported\n", dev_type);
	881	+ ret = -EOPNOTSUPP;
	882	+ goto err;
	883	+ }
	884	+
	885	+ ifcvf_mgmt_dev->mdev.max_supported_vqs = vf->nr_vring;
	886	+ ifcvf_mgmt_dev->mdev.supported_features = vf->hw_features;
	887	+
	888	+ adapter->vdpa.mdev = &ifcvf_mgmt_dev->mdev;
	889	+
	890	+
881	891	ret = vdpa_mgmtdev_register(&ifcvf_mgmt_dev->mdev);
882	892	if (ret) {
883	893	IFCVF_ERR(pdev,

drivers/vdpa/mlx5/core/mlx5_vdpa.h+11−0

@@ -70,6 +70,16 @@ struct mlx5_vdpa_wq_ent { · 69 unmodified lines
70	70	struct mlx5_vdpa_dev *mvdev;
71	71	};
72	72
	73	+enum {
	74	+ MLX5_VDPA_DATAVQ_GROUP,
	75	+ MLX5_VDPA_CVQ_GROUP,
	76	+ MLX5_VDPA_NUMVQ_GROUPS
	77	+};
	78	+
	79	+enum {
	80	+ MLX5_VDPA_NUM_AS = MLX5_VDPA_NUMVQ_GROUPS
	81	+};
	82	+
73	83	struct mlx5_vdpa_dev {
74	84	struct vdpa_device vdev;
75	85	struct mlx5_core_dev *mdev;
@@ -85,6 +95,7 @@ struct mlx5_vdpa_dev { · 9 unmodified lines
85	95	struct mlx5_vdpa_mr mr;
86	96	struct mlx5_control_vq cvq;
87	97	struct workqueue_struct *wq;
	98	+ unsigned int group2asid[MLX5_VDPA_NUMVQ_GROUPS];
88	99	};
89	100
90	101	int mlx5_vdpa_alloc_pd(struct mlx5_vdpa_dev dev, u32 pdn, u16 uid);

drivers/vdpa/mlx5/net/mlx5_vnet.c+158−15

@@ -164,6 +164,7 @@ struct mlx5_vdpa_net { · 163 unmodified lines
164	164	bool setup;
165	165	u32 cur_num_vqs;
166	166	u32 rqt_size;
	167	+ bool nb_registered;
167	168	struct notifier_block nb;
168	169	struct vdpa_callback config_cb;
169	170	struct mlx5_vdpa_wq_ent cvq_ent;
@@ -895,6 +896,7 @@ static int create_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtque · 725 unmodified lines
895	896	if (err)
896	897	goto err_cmd;
897	898
	899	+ mvq->fw_state = MLX5_VIRTIO_NET_Q_OBJECT_STATE_INIT;
898	900	kfree(in);
899	901	mvq->virtq_id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);
900	902
@@ -922,6 +924,7 @@ static void destroy_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtq · 21 unmodified lines
922	924	mlx5_vdpa_warn(&ndev->mvdev, "destroy virtqueue 0x%x\n", mvq->virtq_id);
923	925	return;
924	926	}
	927	+ mvq->fw_state = MLX5_VIRTIO_NET_Q_OBJECT_NONE;
925	928	umems_destroy(ndev, mvq);
926	929	}
927	930
@@ -1121,6 +1124,20 @@ err_cmd: · 193 unmodified lines
1121	1124	return err;
1122	1125	}
1123	1126
	1127	+static bool is_valid_state_change(int oldstate, int newstate)
	1128	+{
	1129	+ switch (oldstate) {
	1130	+ case MLX5_VIRTIO_NET_Q_OBJECT_STATE_INIT:
	1131	+ return newstate == MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY;
	1132	+ case MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY:
	1133	+ return newstate == MLX5_VIRTIO_NET_Q_OBJECT_STATE_SUSPEND;
	1134	+ case MLX5_VIRTIO_NET_Q_OBJECT_STATE_SUSPEND:
	1135	+ case MLX5_VIRTIO_NET_Q_OBJECT_STATE_ERR:
	1136	+ default:
	1137	+ return false;
	1138	+ }
	1139	+}
	1140	+
1124	1141	static int modify_virtqueue(struct mlx5_vdpa_net ndev, struct mlx5_vdpa_virtqueue mvq, int state)
1125	1142	{
1126	1143	int inlen = MLX5_ST_SZ_BYTES(modify_virtio_net_q_in);
@@ -1130,6 +1147,12 @@ static int modify_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtque · 3 unmodified lines
1130	1147	void *in;
1131	1148	int err;
1132	1149
	1150	+ if (mvq->fw_state == MLX5_VIRTIO_NET_Q_OBJECT_NONE)
	1151	+ return 0;
	1152	+
	1153	+ if (!is_valid_state_change(mvq->fw_state, state))
	1154	+ return -EINVAL;
	1155	+
1133	1156	in = kzalloc(inlen, GFP_KERNEL);
1134	1157	if (!in)
1135	1158	return -ENOMEM;
@@ -1440,7 +1463,7 @@ static int mlx5_vdpa_add_mac_vlan_rules(struct mlx5_vdpa_net ndev, u8 mac, · 304 unmodified lines
1440	1463	headers_v = MLX5_ADDR_OF(fte_match_param, spec->match_value, outer_headers);
1441	1464	dmac_c = MLX5_ADDR_OF(fte_match_param, headers_c, outer_headers.dmac_47_16);
1442	1465	dmac_v = MLX5_ADDR_OF(fte_match_param, headers_v, outer_headers.dmac_47_16);
1443		− memset(dmac_c, 0xff, ETH_ALEN);
	1466	+ eth_broadcast_addr(dmac_c);
1444	1467	ether_addr_copy(dmac_v, mac);
1445	1468	MLX5_SET(fte_match_set_lyr_2_4, headers_c, cvlan_tag, 1);
1446	1469	if (tagged) {
@@ -1992,6 +2015,7 @@ static void mlx5_vdpa_set_vq_ready(struct vdpa_device *vdev, u16 idx, bool ready · 545 unmodified lines
1992	2015	struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev);
1993	2016	struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev);
1994	2017	struct mlx5_vdpa_virtqueue *mvq;
	2018	+ int err;
1995	2019
1996	2020	if (!mvdev->actual_features)
1997	2021	return;
@@ -2005,8 +2029,16 @@ static void mlx5_vdpa_set_vq_ready(struct vdpa_device *vdev, u16 idx, bool ready · 7 unmodified lines
2005	2029	}
2006	2030
2007	2031	mvq = &ndev->vqs[idx];
2008		− if (!ready)
	2032	+ if (!ready) {
2009	2033	suspend_vq(ndev, mvq);
	2034	+ } else {
	2035	+ err = modify_virtqueue(ndev, mvq, MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY);
	2036	+ if (err) {
	2037	+ mlx5_vdpa_warn(mvdev, "modify VQ %d to ready failed (%d)\n", idx, err);
	2038	+ ready = false;
	2039	+ }
	2040	+ }
	2041	+
2010	2042
2011	2043	mvq->ready = ready;
2012	2044	}
@@ -2095,9 +2127,14 @@ static u32 mlx5_vdpa_get_vq_align(struct vdpa_device *vdev) · 82 unmodified lines
2095	2127	return PAGE_SIZE;
2096	2128	}
2097	2129
2098		−static u32 mlx5_vdpa_get_vq_group(struct vdpa_device *vdpa, u16 idx)
	2130	+static u32 mlx5_vdpa_get_vq_group(struct vdpa_device *vdev, u16 idx)
2099	2131	{
2100		− return 0;
	2132	+ struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev);
	2133	+
	2134	+ if (is_ctrl_vq_idx(mvdev, idx))
	2135	+ return MLX5_VDPA_CVQ_GROUP;
	2136	+
	2137	+ return MLX5_VDPA_DATAVQ_GROUP;
2101	2138	}
2102	2139
2103	2140	enum { MLX5_VIRTIO_NET_F_GUEST_CSUM = 1 << 9,
@@ -2511,6 +2548,15 @@ err_clear: · 407 unmodified lines
2511	2548	up_write(&ndev->reslock);
2512	2549	}
2513	2550
	2551	+static void init_group_to_asid_map(struct mlx5_vdpa_dev *mvdev)
	2552	+{
	2553	+ int i;
	2554	+
	2555	+ /* default mapping all groups are mapped to asid 0 */
	2556	+ for (i = 0; i < MLX5_VDPA_NUMVQ_GROUPS; i++)
	2557	+ mvdev->group2asid[i] = 0;
	2558	+}
	2559	+
2514	2560	static int mlx5_vdpa_reset(struct vdpa_device *vdev)
2515	2561	{
2516	2562	struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev);
@@ -2529,7 +2575,9 @@ static int mlx5_vdpa_reset(struct vdpa_device *vdev) · 12 unmodified lines
2529	2575	ndev->mvdev.cvq.completed_desc = 0;
2530	2576	memset(ndev->event_cbs, 0, sizeof(ndev->event_cbs) (mvdev->max_vqs + 1));
2531	2577	ndev->mvdev.actual_features = 0;
	2578	+ init_group_to_asid_map(mvdev);
2532	2579	++mvdev->generation;
	2580	+
2533	2581	if (MLX5_CAP_GEN(mvdev->mdev, umem_uid_0)) {
2534	2582	if (mlx5_vdpa_create_mr(mvdev, NULL))
2535	2583	mlx5_vdpa_warn(mvdev, "create MR failed\n");
@@ -2567,26 +2615,63 @@ static u32 mlx5_vdpa_get_generation(struct vdpa_device *vdev) · 31 unmodified lines
2567	2615	return mvdev->generation;
2568	2616	}
2569	2617
2570		−static int mlx5_vdpa_set_map(struct vdpa_device *vdev, unsigned int asid,
2571		− struct vhost_iotlb *iotlb)
	2618	+static int set_map_control(struct mlx5_vdpa_dev mvdev, struct vhost_iotlb iotlb)
	2619	+{
	2620	+ u64 start = 0ULL, last = 0ULL - 1;
	2621	+ struct vhost_iotlb_map *map;
	2622	+ int err = 0;
	2623	+
	2624	+ spin_lock(&mvdev->cvq.iommu_lock);
	2625	+ vhost_iotlb_reset(mvdev->cvq.iotlb);
	2626	+
	2627	+ for (map = vhost_iotlb_itree_first(iotlb, start, last); map;
	2628	+ map = vhost_iotlb_itree_next(map, start, last)) {
	2629	+ err = vhost_iotlb_add_range(mvdev->cvq.iotlb, map->start,
	2630	+ map->last, map->addr, map->perm);
	2631	+ if (err)
	2632	+ goto out;
	2633	+ }
	2634	+
	2635	+out:
	2636	+ spin_unlock(&mvdev->cvq.iommu_lock);
	2637	+ return err;
	2638	+}
	2639	+
	2640	+static int set_map_data(struct mlx5_vdpa_dev mvdev, struct vhost_iotlb iotlb)
2572	2641	{
2573		− struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev);
2574		− struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev);
2575	2642	bool change_map;
2576	2643	int err;
2577	2644
2578		− down_write(&ndev->reslock);
2579		−
2580	2645	err = mlx5_vdpa_handle_set_map(mvdev, iotlb, &change_map);
2581	2646	if (err) {
2582	2647	mlx5_vdpa_warn(mvdev, "set map failed(%d)\n", err);
2583		− goto err;
	2648	+ return err;
2584	2649	}
2585	2650
2586	2651	if (change_map)
2587	2652	err = mlx5_vdpa_change_map(mvdev, iotlb);
2588	2653
2589		−err:
	2654	+ return err;
	2655	+}
	2656	+
	2657	+static int mlx5_vdpa_set_map(struct vdpa_device *vdev, unsigned int asid,
	2658	+ struct vhost_iotlb *iotlb)
	2659	+{
	2660	+ struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev);
	2661	+ struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev);
	2662	+ int err = -EINVAL;
	2663	+
	2664	+ down_write(&ndev->reslock);
	2665	+ if (mvdev->group2asid[MLX5_VDPA_DATAVQ_GROUP] == asid) {
	2666	+ err = set_map_data(mvdev, iotlb);
	2667	+ if (err)
	2668	+ goto out;
	2669	+ }
	2670	+
	2671	+ if (mvdev->group2asid[MLX5_VDPA_CVQ_GROUP] == asid)
	2672	+ err = set_map_control(mvdev, iotlb);
	2673	+
	2674	+out:
2590	2675	up_write(&ndev->reslock);
2591	2676	return err;
2592	2677	}
@@ -2733,6 +2818,49 @@ out_err: · 140 unmodified lines
2733	2818	return err;
2734	2819	}
2735	2820
	2821	+static void mlx5_vdpa_cvq_suspend(struct mlx5_vdpa_dev *mvdev)
	2822	+{
	2823	+ struct mlx5_control_vq *cvq;
	2824	+
	2825	+ if (!(mvdev->actual_features & BIT_ULL(VIRTIO_NET_F_CTRL_VQ)))
	2826	+ return;
	2827	+
	2828	+ cvq = &mvdev->cvq;
	2829	+ cvq->ready = false;
	2830	+}
	2831	+
	2832	+static int mlx5_vdpa_suspend(struct vdpa_device *vdev)
	2833	+{
	2834	+ struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev);
	2835	+ struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev);
	2836	+ struct mlx5_vdpa_virtqueue *mvq;
	2837	+ int i;
	2838	+
	2839	+ down_write(&ndev->reslock);
	2840	+ mlx5_notifier_unregister(mvdev->mdev, &ndev->nb);
	2841	+ ndev->nb_registered = false;
	2842	+ flush_workqueue(ndev->mvdev.wq);
	2843	+ for (i = 0; i < ndev->cur_num_vqs; i++) {
	2844	+ mvq = &ndev->vqs[i];
	2845	+ suspend_vq(ndev, mvq);
	2846	+ }
	2847	+ mlx5_vdpa_cvq_suspend(mvdev);
	2848	+ up_write(&ndev->reslock);
	2849	+ return 0;
	2850	+}
	2851	+
	2852	+static int mlx5_set_group_asid(struct vdpa_device *vdev, u32 group,
	2853	+ unsigned int asid)
	2854	+{
	2855	+ struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev);
	2856	+
	2857	+ if (group >= MLX5_VDPA_NUMVQ_GROUPS)
	2858	+ return -EINVAL;
	2859	+
	2860	+ mvdev->group2asid[group] = asid;
	2861	+ return 0;
	2862	+}
	2863	+
2736	2864	static const struct vdpa_config_ops mlx5_vdpa_ops = {
2737	2865	.set_vq_address = mlx5_vdpa_set_vq_address,
2738	2866	.set_vq_num = mlx5_vdpa_set_vq_num,
@@ -2762,7 +2890,9 @@ static const struct vdpa_config_ops mlx5_vdpa_ops = { · 23 unmodified lines
2762	2890	.set_config = mlx5_vdpa_set_config,
2763	2891	.get_generation = mlx5_vdpa_get_generation,
2764	2892	.set_map = mlx5_vdpa_set_map,
	2893	+ .set_group_asid = mlx5_set_group_asid,
2765	2894	.free = mlx5_vdpa_free,
	2895	+ .suspend = mlx5_vdpa_suspend,
2766	2896	};
2767	2897
2768	2898	static int query_mtu(struct mlx5_core_dev mdev, u16 mtu)
@@ -2828,6 +2958,7 @@ static void init_mvqs(struct mlx5_vdpa_net *ndev) · 59 unmodified lines
2828	2958	mvq->index = i;
2829	2959	mvq->ndev = ndev;
2830	2960	mvq->fwqp.fw = true;
	2961	+ mvq->fw_state = MLX5_VIRTIO_NET_Q_OBJECT_NONE;
2831	2962	}
2832	2963	for (; i < ndev->mvdev.max_vqs; i++) {
2833	2964	mvq = &ndev->vqs[i];
@@ -2902,13 +3033,21 @@ static int event_handler(struct notifier_block nb, unsigned long event, void p · 68 unmodified lines
2902	3033	switch (eqe->sub_type) {
2903	3034	case MLX5_PORT_CHANGE_SUBTYPE_DOWN:
2904	3035	case MLX5_PORT_CHANGE_SUBTYPE_ACTIVE:
	3036	+ down_read(&ndev->reslock);
	3037	+ if (!ndev->nb_registered) {
	3038	+ up_read(&ndev->reslock);
	3039	+ return NOTIFY_DONE;
	3040	+ }
2905	3041	wqent = kzalloc(sizeof(*wqent), GFP_ATOMIC);
2906		− if (!wqent)
	3042	+ if (!wqent) {
	3043	+ up_read(&ndev->reslock);
2907	3044	return NOTIFY_DONE;
	3045	+ }
2908	3046
2909	3047	wqent->mvdev = &ndev->mvdev;
2910	3048	INIT_WORK(&wqent->work, update_carrier);
2911	3049	queue_work(ndev->mvdev.wq, &wqent->work);
	3050	+ up_read(&ndev->reslock);
2912	3051	ret = NOTIFY_OK;
2913	3052	break;
2914	3053	default:
@@ -2982,7 +3121,7 @@ static int mlx5_vdpa_dev_add(struct vdpa_mgmt_dev v_mdev, const char name, · 67 unmodified lines
2982	3121	}
2983	3122
2984	3123	ndev = vdpa_alloc_device(struct mlx5_vdpa_net, mvdev.vdev, mdev->device, &mlx5_vdpa_ops,
2985		− 1, 1, name, false);
	3124	+ MLX5_VDPA_NUMVQ_GROUPS, MLX5_VDPA_NUM_AS, name, false);
2986	3125	if (IS_ERR(ndev))
2987	3126	return PTR_ERR(ndev);
2988	3127
@@ -3062,6 +3201,7 @@ static int mlx5_vdpa_dev_add(struct vdpa_mgmt_dev v_mdev, const char name, · 73 unmodified lines
3062	3201
3063	3202	ndev->nb.notifier_call = event_handler;
3064	3203	mlx5_notifier_register(mdev, &ndev->nb);
	3204	+ ndev->nb_registered = true;
3065	3205	mvdev->vdev.mdev = &mgtdev->mgtdev;
3066	3206	err = _vdpa_register_device(&mvdev->vdev, max_vqs + 1);
3067	3207	if (err)
@@ -3093,7 +3233,10 @@ static void mlx5_vdpa_dev_del(struct vdpa_mgmt_dev v_mdev, struct vdpa_device · 25 unmodified lines
3093	3233	struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev);
3094	3234	struct workqueue_struct *wq;
3095	3235
3096		− mlx5_notifier_unregister(mvdev->mdev, &ndev->nb);
	3236	+ if (ndev->nb_registered) {
	3237	+ mlx5_notifier_unregister(mvdev->mdev, &ndev->nb);
	3238	+ ndev->nb_registered = false;
	3239	+ }
3097	3240	wq = mvdev->wq;
3098	3241	mvdev->wq = NULL;
3099	3242	destroy_workqueue(wq);

drivers/vdpa/vdpa.c+3−11

@@ -824,11 +824,11 @@ static int vdpa_dev_net_config_fill(struct vdpa_device vdev, struct sk_buff ms · 823 unmodified lines
824	824	config.mac))
825	825	return -EMSGSIZE;
826	826
827		− val_u16 = le16_to_cpu(config.status);
	827	+ val_u16 = __virtio16_to_cpu(true, config.status);
828	828	if (nla_put_u16(msg, VDPA_ATTR_DEV_NET_STATUS, val_u16))
829	829	return -EMSGSIZE;
830	830
831		− val_u16 = le16_to_cpu(config.mtu);
	831	+ val_u16 = __virtio16_to_cpu(true, config.mtu);
832	832	if (nla_put_u16(msg, VDPA_ATTR_DEV_NET_CFG_MTU, val_u16))
833	833	return -EMSGSIZE;
834	834
@@ -846,17 +846,9 @@ vdpa_dev_config_fill(struct vdpa_device vdev, struct sk_buff msg, u32 portid, · 11 unmodified lines
846	846	{
847	847	u32 device_id;
848	848	void *hdr;
849		− u8 status;
850	849	int err;
851	850
852	851	down_read(&vdev->cf_lock);
853		− status = vdev->config->get_status(vdev);
854		− if (!(status & VIRTIO_CONFIG_S_FEATURES_OK)) {
855		− NL_SET_ERR_MSG_MOD(extack, "Features negotiation not completed");
856		− err = -EAGAIN;
857		− goto out;
858		− }
859		−
860	852	hdr = genlmsg_put(msg, portid, seq, &vdpa_nl_family, flags,
861	853	VDPA_CMD_DEV_CONFIG_GET);
862	854	if (!hdr) {
@@ -913,7 +905,7 @@ static int vdpa_fill_stats_rec(struct vdpa_device vdev, struct sk_buff msg, · 50 unmodified lines
913	905	}
914	906	vdpa_get_config_unlocked(vdev, 0, &config, sizeof(config));
915	907
916		− max_vqp = le16_to_cpu(config.max_virtqueue_pairs);
	908	+ max_vqp = __virtio16_to_cpu(true, config.max_virtqueue_pairs);
917	909	if (nla_put_u16(msg, VDPA_ATTR_DEV_NET_CFG_MAX_VQP, max_vqp))
918	910	return -EMSGSIZE;
919	911

drivers/vdpa/vdpa_sim/vdpa_sim.c+16−2

@@ -33,7 +33,7 @@ MODULE_PARM_DESC(batch_mapping, "Batched mapping 1 -Enable; 0 - Disable"); · 32 unmodified lines
33	33	static int max_iotlb_entries = 2048;
34	34	module_param(max_iotlb_entries, int, 0444);
35	35	MODULE_PARM_DESC(max_iotlb_entries,
36		− "Maximum number of iotlb entries. 0 means unlimited. (default: 2048)");
	36	+ "Maximum number of iotlb entries for each address space. 0 means unlimited. (default: 2048)");
37	37
38	38	#define VDPASIM_QUEUE_ALIGN PAGE_SIZE
39	39	#define VDPASIM_QUEUE_MAX 256
@@ -107,6 +107,7 @@ static void vdpasim_do_reset(struct vdpasim *vdpasim) · 67 unmodified lines
107	107	for (i = 0; i < vdpasim->dev_attr.nas; i++)
108	108	vhost_iotlb_reset(&vdpasim->iommu[i]);
109	109
	110	+ vdpasim->running = true;
110	111	spin_unlock(&vdpasim->iommu_lock);
111	112
112	113	vdpasim->features = 0;
@@ -291,7 +292,7 @@ struct vdpasim vdpasim_create(struct vdpasim_dev_attr dev_attr) · 178 unmodified lines
291	292	goto err_iommu;
292	293
293	294	for (i = 0; i < vdpasim->dev_attr.nas; i++)
294		− vhost_iotlb_init(&vdpasim->iommu[i], 0, 0);
	295	+ vhost_iotlb_init(&vdpasim->iommu[i], max_iotlb_entries, 0);
295	296
296	297	vdpasim->buffer = kvmalloc(dev_attr->buffer_size, GFP_KERNEL);
297	298	if (!vdpasim->buffer)
@@ -505,6 +506,17 @@ static int vdpasim_reset(struct vdpa_device *vdpa) · 207 unmodified lines
505	506	return 0;
506	507	}
507	508
	509	+static int vdpasim_suspend(struct vdpa_device *vdpa)
	510	+{
	511	+ struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
	512	+
	513	+ spin_lock(&vdpasim->lock);
	514	+ vdpasim->running = false;
	515	+ spin_unlock(&vdpasim->lock);
	516	+
	517	+ return 0;
	518	+}
	519	+
508	520	static size_t vdpasim_get_config_size(struct vdpa_device *vdpa)
509	521	{
510	522	struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
@@ -694,6 +706,7 @@ static const struct vdpa_config_ops vdpasim_config_ops = { · 183 unmodified lines
694	706	.get_status = vdpasim_get_status,
695	707	.set_status = vdpasim_set_status,
696	708	.reset = vdpasim_reset,
	709	+ .suspend = vdpasim_suspend,
697	710	.get_config_size = vdpasim_get_config_size,
698	711	.get_config = vdpasim_get_config,
699	712	.set_config = vdpasim_set_config,
@@ -726,6 +739,7 @@ static const struct vdpa_config_ops vdpasim_batch_config_ops = { · 26 unmodified lines
726	739	.get_status = vdpasim_get_status,
727	740	.set_status = vdpasim_set_status,
728	741	.reset = vdpasim_reset,
	742	+ .suspend = vdpasim_suspend,
729	743	.get_config_size = vdpasim_get_config_size,
730	744	.get_config = vdpasim_get_config,
731	745	.set_config = vdpasim_set_config,

drivers/vdpa/vdpa_sim/vdpa_sim.h+1−0

@@ -66,6 +66,7 @@ struct vdpasim { · 65 unmodified lines
66	66	u32 generation;
67	67	u64 features;
68	68	u32 groups;
	69	+ bool running;
69	70	/* spinlock to synchronize iommu table */
70	71	spinlock_t iommu_lock;
71	72	};

drivers/vdpa/vdpa_sim/vdpa_sim_blk.c+147−29

@@ -25,31 +25,49 @@ · 24 unmodified lines
25	25	#define DRV_LICENSE "GPL v2"
26	26
27	27	#define VDPASIM_BLK_FEATURES (VDPASIM_FEATURES \| \
	28	+ (1ULL << VIRTIO_BLK_F_FLUSH) \| \
28	29	(1ULL << VIRTIO_BLK_F_SIZE_MAX) \| \
29	30	(1ULL << VIRTIO_BLK_F_SEG_MAX) \| \
30	31	(1ULL << VIRTIO_BLK_F_BLK_SIZE) \| \
31	32	(1ULL << VIRTIO_BLK_F_TOPOLOGY) \| \
32		− (1ULL << VIRTIO_BLK_F_MQ))
	33	+ (1ULL << VIRTIO_BLK_F_MQ) \| \
	34	+ (1ULL << VIRTIO_BLK_F_DISCARD) \| \
	35	+ (1ULL << VIRTIO_BLK_F_WRITE_ZEROES))
33	36
34	37	#define VDPASIM_BLK_CAPACITY 0x40000
35	38	#define VDPASIM_BLK_SIZE_MAX 0x1000
36	39	#define VDPASIM_BLK_SEG_MAX 32
	40	+#define VDPASIM_BLK_DWZ_MAX_SECTORS UINT_MAX
	41	+
	42	+/* 1 virtqueue, 1 address space, 1 virtqueue group */
37	43	#define VDPASIM_BLK_VQ_NUM 1
	44	+#define VDPASIM_BLK_AS_NUM 1
	45	+#define VDPASIM_BLK_GROUP_NUM 1
38	46
39	47	static char vdpasim_blk_id[VIRTIO_BLK_ID_BYTES] = "vdpa_blk_sim";
40	48
41		−static bool vdpasim_blk_check_range(u64 start_sector, size_t range_size)
	49	+static bool vdpasim_blk_check_range(struct vdpasim *vdpasim, u64 start_sector,
	50	+ u64 num_sectors, u64 max_sectors)
42	51	{
43		− u64 range_sectors = range_size >> SECTOR_SHIFT;
44		−
45		− if (range_size > VDPASIM_BLK_SIZE_MAX * VDPASIM_BLK_SEG_MAX)
46		− return false;
	52	+ if (start_sector > VDPASIM_BLK_CAPACITY) {
	53	+ dev_dbg(&vdpasim->vdpa.dev,
	54	+ "starting sector exceeds the capacity - start: 0x%llx capacity: 0x%x\n",
	55	+ start_sector, VDPASIM_BLK_CAPACITY);
	56	+ }
47	57
48		− if (start_sector > VDPASIM_BLK_CAPACITY)
	58	+ if (num_sectors > max_sectors) {
	59	+ dev_dbg(&vdpasim->vdpa.dev,
	60	+ "number of sectors exceeds the max allowed in a request - num: 0x%llx max: 0x%llx\n",
	61	+ num_sectors, max_sectors);
49	62	return false;
	63	+ }
50	64
51		− if (range_sectors > VDPASIM_BLK_CAPACITY - start_sector)
	65	+ if (num_sectors > VDPASIM_BLK_CAPACITY - start_sector) {
	66	+ dev_dbg(&vdpasim->vdpa.dev,
	67	+ "request exceeds the capacity - start: 0x%llx num: 0x%llx capacity: 0x%x\n",
	68	+ start_sector, num_sectors, VDPASIM_BLK_CAPACITY);
52	69	return false;
	70	+ }
53	71
54	72	return true;
55	73	}
@@ -63,6 +81,7 @@ static bool vdpasim_blk_handle_req(struct vdpasim *vdpasim, · 7 unmodified lines
63	81	{
64	82	size_t pushed = 0, to_pull, to_push;
65	83	struct virtio_blk_outhdr hdr;
	84	+ bool handled = false;
66	85	ssize_t bytes;
67	86	loff_t offset;
68	87	u64 sector;
@@ -76,14 +95,14 @@ static bool vdpasim_blk_handle_req(struct vdpasim *vdpasim, · 7 unmodified lines
76	95	return false;
77	96
78	97	if (vq->out_iov.used < 1 \|\| vq->in_iov.used < 1) {
79		− dev_err(&vdpasim->vdpa.dev, "missing headers - out_iov: %u in_iov %u\n",
	98	+ dev_dbg(&vdpasim->vdpa.dev, "missing headers - out_iov: %u in_iov %u\n",
80	99	vq->out_iov.used, vq->in_iov.used);
81		− return false;
	100	+ goto err;
82	101	}
83	102
84	103	if (vq->in_iov.iov[vq->in_iov.used - 1].iov_len < 1) {
85		− dev_err(&vdpasim->vdpa.dev, "request in header too short\n");
86		− return false;
	104	+ dev_dbg(&vdpasim->vdpa.dev, "request in header too short\n");
	105	+ goto err;
87	106	}
88	107
89	108	/* The last byte is the status and we checked if the last iov has
@@ -96,8 +115,8 @@ static bool vdpasim_blk_handle_req(struct vdpasim *vdpasim, · 6 unmodified lines
96	115	bytes = vringh_iov_pull_iotlb(&vq->vring, &vq->out_iov, &hdr,
97	116	sizeof(hdr));
98	117	if (bytes != sizeof(hdr)) {
99		− dev_err(&vdpasim->vdpa.dev, "request out header too short\n");
100		− return false;
	118	+ dev_dbg(&vdpasim->vdpa.dev, "request out header too short\n");
	119	+ goto err;
101	120	}
102	121
103	122	to_pull -= bytes;
@@ -107,12 +126,20 @@ static bool vdpasim_blk_handle_req(struct vdpasim *vdpasim, · 3 unmodified lines
107	126	offset = sector << SECTOR_SHIFT;
108	127	status = VIRTIO_BLK_S_OK;
109	128
	129	+ if (type != VIRTIO_BLK_T_IN && type != VIRTIO_BLK_T_OUT &&
	130	+ sector != 0) {
	131	+ dev_dbg(&vdpasim->vdpa.dev,
	132	+ "sector must be 0 for %u request - sector: 0x%llx\n",
	133	+ type, sector);
	134	+ status = VIRTIO_BLK_S_IOERR;
	135	+ goto err_status;
	136	+ }
	137	+
110	138	switch (type) {
111	139	case VIRTIO_BLK_T_IN:
112		− if (!vdpasim_blk_check_range(sector, to_push)) {
113		− dev_err(&vdpasim->vdpa.dev,
114		− "reading over the capacity - offset: 0x%llx len: 0x%zx\n",
115		− offset, to_push);
	140	+ if (!vdpasim_blk_check_range(vdpasim, sector,
	141	+ to_push >> SECTOR_SHIFT,
	142	+ VDPASIM_BLK_SIZE_MAX * VDPASIM_BLK_SEG_MAX)) {
116	143	status = VIRTIO_BLK_S_IOERR;
117	144	break;
118	145	}
@@ -121,7 +148,7 @@ static bool vdpasim_blk_handle_req(struct vdpasim *vdpasim, · 2 unmodified lines
121	148	vdpasim->buffer + offset,
122	149	to_push);
123	150	if (bytes < 0) {
124		− dev_err(&vdpasim->vdpa.dev,
	151	+ dev_dbg(&vdpasim->vdpa.dev,
125	152	"vringh_iov_push_iotlb() error: %zd offset: 0x%llx len: 0x%zx\n",
126	153	bytes, offset, to_push);
127	154	status = VIRTIO_BLK_S_IOERR;
@@ -132,10 +159,9 @@ static bool vdpasim_blk_handle_req(struct vdpasim *vdpasim, · 4 unmodified lines
132	159	break;
133	160
134	161	case VIRTIO_BLK_T_OUT:
135		− if (!vdpasim_blk_check_range(sector, to_pull)) {
136		− dev_err(&vdpasim->vdpa.dev,
137		− "writing over the capacity - offset: 0x%llx len: 0x%zx\n",
138		− offset, to_pull);
	162	+ if (!vdpasim_blk_check_range(vdpasim, sector,
	163	+ to_pull >> SECTOR_SHIFT,
	164	+ VDPASIM_BLK_SIZE_MAX * VDPASIM_BLK_SEG_MAX)) {
139	165	status = VIRTIO_BLK_S_IOERR;
140	166	break;
141	167	}
@@ -144,7 +170,7 @@ static bool vdpasim_blk_handle_req(struct vdpasim *vdpasim, · 2 unmodified lines
144	170	vdpasim->buffer + offset,
145	171	to_pull);
146	172	if (bytes < 0) {
147		− dev_err(&vdpasim->vdpa.dev,
	173	+ dev_dbg(&vdpasim->vdpa.dev,
148	174	"vringh_iov_pull_iotlb() error: %zd offset: 0x%llx len: 0x%zx\n",
149	175	bytes, offset, to_pull);
150	176	status = VIRTIO_BLK_S_IOERR;
@@ -157,7 +183,7 @@ static bool vdpasim_blk_handle_req(struct vdpasim *vdpasim, · 6 unmodified lines
157	183	vdpasim_blk_id,
158	184	VIRTIO_BLK_ID_BYTES);
159	185	if (bytes < 0) {
160		− dev_err(&vdpasim->vdpa.dev,
	186	+ dev_dbg(&vdpasim->vdpa.dev,
161	187	"vringh_iov_push_iotlb() error: %zd\n", bytes);
162	188	status = VIRTIO_BLK_S_IOERR;
163	189	break;
@@ -166,13 +192,76 @@ static bool vdpasim_blk_handle_req(struct vdpasim *vdpasim, · 2 unmodified lines
166	192	pushed += bytes;
167	193	break;
168	194
	195	+ case VIRTIO_BLK_T_FLUSH:
	196	+ /* nothing to do */
	197	+ break;
	198	+
	199	+ case VIRTIO_BLK_T_DISCARD:
	200	+ case VIRTIO_BLK_T_WRITE_ZEROES: {
	201	+ struct virtio_blk_discard_write_zeroes range;
	202	+ u32 num_sectors, flags;
	203	+
	204	+ if (to_pull != sizeof(range)) {
	205	+ dev_dbg(&vdpasim->vdpa.dev,
	206	+ "discard/write_zeroes header len: 0x%zx [expected: 0x%zx]\n",
	207	+ to_pull, sizeof(range));
	208	+ status = VIRTIO_BLK_S_IOERR;
	209	+ break;
	210	+ }
	211	+
	212	+ bytes = vringh_iov_pull_iotlb(&vq->vring, &vq->out_iov, &range,
	213	+ to_pull);
	214	+ if (bytes < 0) {
	215	+ dev_dbg(&vdpasim->vdpa.dev,
	216	+ "vringh_iov_pull_iotlb() error: %zd offset: 0x%llx len: 0x%zx\n",
	217	+ bytes, offset, to_pull);
	218	+ status = VIRTIO_BLK_S_IOERR;
	219	+ break;
	220	+ }
	221	+
	222	+ sector = le64_to_cpu(range.sector);
	223	+ offset = sector << SECTOR_SHIFT;
	224	+ num_sectors = le32_to_cpu(range.num_sectors);
	225	+ flags = le32_to_cpu(range.flags);
	226	+
	227	+ if (type == VIRTIO_BLK_T_DISCARD && flags != 0) {
	228	+ dev_dbg(&vdpasim->vdpa.dev,
	229	+ "discard unexpected flags set - flags: 0x%x\n",
	230	+ flags);
	231	+ status = VIRTIO_BLK_S_UNSUPP;
	232	+ break;
	233	+ }
	234	+
	235	+ if (type == VIRTIO_BLK_T_WRITE_ZEROES &&
	236	+ flags & ~VIRTIO_BLK_WRITE_ZEROES_FLAG_UNMAP) {
	237	+ dev_dbg(&vdpasim->vdpa.dev,
	238	+ "write_zeroes unexpected flags set - flags: 0x%x\n",
	239	+ flags);
	240	+ status = VIRTIO_BLK_S_UNSUPP;
	241	+ break;
	242	+ }
	243	+
	244	+ if (!vdpasim_blk_check_range(vdpasim, sector, num_sectors,
	245	+ VDPASIM_BLK_DWZ_MAX_SECTORS)) {
	246	+ status = VIRTIO_BLK_S_IOERR;
	247	+ break;
	248	+ }
	249	+
	250	+ if (type == VIRTIO_BLK_T_WRITE_ZEROES) {
	251	+ memset(vdpasim->buffer + offset, 0,
	252	+ num_sectors << SECTOR_SHIFT);
	253	+ }
	254	+
	255	+ break;
	256	+ }
169	257	default:
170		− dev_warn(&vdpasim->vdpa.dev,
171		− "Unsupported request type %d\n", type);
	258	+ dev_dbg(&vdpasim->vdpa.dev,
	259	+ "Unsupported request type %d\n", type);
172	260	status = VIRTIO_BLK_S_IOERR;
173	261	break;
174	262	}
175	263
	264	+err_status:
176	265	/* If some operations fail, we need to skip the remaining bytes
177	266	* to put the status in the last byte
178	267	*/
@@ -182,21 +271,25 @@ static bool vdpasim_blk_handle_req(struct vdpasim *vdpasim, · 3 unmodified lines
182	271	/* Last byte is the status */
183	272	bytes = vringh_iov_push_iotlb(&vq->vring, &vq->in_iov, &status, 1);
184	273	if (bytes != 1)
185		− return false;
	274	+ goto err;
186	275
187	276	pushed += bytes;
188	277
189	278	/* Make sure data is wrote before advancing index */
190	279	smp_wmb();
191	280
	281	+ handled = true;
	282	+
	283	+err:
192	284	vringh_complete_iotlb(&vq->vring, vq->head, pushed);
193	285
194		− return true;
	286	+ return handled;
195	287	}
196	288
197	289	static void vdpasim_blk_work(struct work_struct *work)
198	290	{
199	291	struct vdpasim *vdpasim = container_of(work, struct vdpasim, work);
	292	+ bool reschedule = false;
200	293	int i;
201	294
202	295	spin_lock(&vdpasim->lock);
@@ -204,8 +297,12 @@ static void vdpasim_blk_work(struct work_struct *work) · 1 unmodified line
204	297	if (!(vdpasim->status & VIRTIO_CONFIG_S_DRIVER_OK))
205	298	goto out;
206	299
	300	+ if (!vdpasim->running)
	301	+ goto out;
	302	+
207	303	for (i = 0; i < VDPASIM_BLK_VQ_NUM; i++) {
208	304	struct vdpasim_virtqueue *vq = &vdpasim->vqs[i];
	305	+ int reqs = 0;
209	306
210	307	if (!vq->ready)
211	308	continue;
@@ -218,10 +315,18 @@ static void vdpasim_blk_work(struct work_struct *work) · 6 unmodified lines
218	315	if (vringh_need_notify_iotlb(&vq->vring) > 0)
219	316	vringh_notify(&vq->vring);
220	317	local_bh_enable();
	318	+
	319	+ if (++reqs > 4) {
	320	+ reschedule = true;
	321	+ break;
	322	+ }
221	323	}
222	324	}
223	325	out:
224	326	spin_unlock(&vdpasim->lock);
	327	+
	328	+ if (reschedule)
	329	+ schedule_work(&vdpasim->work);
225	330	}
226	331
227	332	static void vdpasim_blk_get_config(struct vdpasim vdpasim, void config)
@@ -237,6 +342,17 @@ static void vdpasim_blk_get_config(struct vdpasim vdpasim, void config) · 9 unmodified lines
237	342	blk_config->min_io_size = cpu_to_vdpasim16(vdpasim, 1);
238	343	blk_config->opt_io_size = cpu_to_vdpasim32(vdpasim, 1);
239	344	blk_config->blk_size = cpu_to_vdpasim32(vdpasim, SECTOR_SIZE);
	345	+ /* VIRTIO_BLK_F_DISCARD */
	346	+ blk_config->discard_sector_alignment =
	347	+ cpu_to_vdpasim32(vdpasim, SECTOR_SIZE);
	348	+ blk_config->max_discard_sectors =
	349	+ cpu_to_vdpasim32(vdpasim, VDPASIM_BLK_DWZ_MAX_SECTORS);
	350	+ blk_config->max_discard_seg = cpu_to_vdpasim32(vdpasim, 1);
	351	+ /* VIRTIO_BLK_F_WRITE_ZEROES */
	352	+ blk_config->max_write_zeroes_sectors =
	353	+ cpu_to_vdpasim32(vdpasim, VDPASIM_BLK_DWZ_MAX_SECTORS);
	354	+ blk_config->max_write_zeroes_seg = cpu_to_vdpasim32(vdpasim, 1);
	355	+
240	356	}
241	357
242	358	static void vdpasim_blk_mgmtdev_release(struct device *dev)
@@ -260,6 +376,8 @@ static int vdpasim_blk_dev_add(struct vdpa_mgmt_dev mdev, const char name, · 17 unmodified lines
260	376	dev_attr.id = VIRTIO_ID_BLOCK;
261	377	dev_attr.supported_features = VDPASIM_BLK_FEATURES;
262	378	dev_attr.nvqs = VDPASIM_BLK_VQ_NUM;
	379	+ dev_attr.ngroups = VDPASIM_BLK_GROUP_NUM;
	380	+ dev_attr.nas = VDPASIM_BLK_AS_NUM;
263	381	dev_attr.config_size = sizeof(struct virtio_blk_config);
264	382	dev_attr.get_config = vdpasim_blk_get_config;
265	383	dev_attr.work_fn = vdpasim_blk_work;

drivers/vdpa/vdpa_sim/vdpa_sim_net.c+3−0

@@ -154,6 +154,9 @@ static void vdpasim_net_work(struct work_struct *work) · 153 unmodified lines
154	154
155	155	spin_lock(&vdpasim->lock);
156	156
	157	+ if (!vdpasim->running)
	158	+ goto out;
	159	+
157	160	if (!(vdpasim->status & VIRTIO_CONFIG_S_DRIVER_OK))
158	161	goto out;
159	162

drivers/vdpa/vdpa_user/iova_domain.c+89−13

@@ -138,18 +138,17 @@ static void do_bounce(phys_addr_t orig, void *addr, size_t size, · 137 unmodified lines
138	138	{
139	139	unsigned long pfn = PFN_DOWN(orig);
140	140	unsigned int offset = offset_in_page(orig);
141		− char *buffer;
	141	+ struct page *page;
142	142	unsigned int sz = 0;
143	143
144	144	while (size) {
145	145	sz = min_t(size_t, PAGE_SIZE - offset, size);
146	146
147		− buffer = kmap_atomic(pfn_to_page(pfn));
	147	+ page = pfn_to_page(pfn);
148	148	if (dir == DMA_TO_DEVICE)
149		− memcpy(addr, buffer + offset, sz);
	149	+ memcpy_from_page(addr, page, offset, sz);
150	150	else
151		− memcpy(buffer + offset, addr, sz);
152		− kunmap_atomic(buffer);
	151	+ memcpy_to_page(page, offset, addr, sz);
153	152
154	153	size -= sz;
155	154	pfn++;
@@ -179,8 +178,9 @@ static void vduse_domain_bounce(struct vduse_iova_domain *domain, · 23 unmodified lines
179	178	map->orig_phys == INVALID_PHYS_ADDR))
180	179	return;
181	180
182		− addr = page_address(map->bounce_page) + offset;
183		− do_bounce(map->orig_phys + offset, addr, sz, dir);
	181	+ addr = kmap_local_page(map->bounce_page);
	182	+ do_bounce(map->orig_phys + offset, addr + offset, sz, dir);
	183	+ kunmap_local(addr);
184	184	size -= sz;
185	185	iova += sz;
186	186	}
@@ -213,21 +213,21 @@ vduse_domain_get_bounce_page(struct vduse_iova_domain *domain, u64 iova) · 26 unmodified lines
213	213	struct vduse_bounce_map *map;
214	214	struct page *page = NULL;
215	215
216		− spin_lock(&domain->iotlb_lock);
	216	+ read_lock(&domain->bounce_lock);
217	217	map = &domain->bounce_maps[iova >> PAGE_SHIFT];
218		− if (!map->bounce_page)
	218	+ if (domain->user_bounce_pages \|\| !map->bounce_page)
219	219	goto out;
220	220
221	221	page = map->bounce_page;
222	222	get_page(page);
223	223	out:
224		− spin_unlock(&domain->iotlb_lock);
	224	+ read_unlock(&domain->bounce_lock);
225	225
226	226	return page;
227	227	}
228	228
229	229	static void
230		−vduse_domain_free_bounce_pages(struct vduse_iova_domain *domain)
	230	+vduse_domain_free_kernel_bounce_pages(struct vduse_iova_domain *domain)
231	231	{
232	232	struct vduse_bounce_map *map;
233	233	unsigned long pfn, bounce_pfns;
@@ -247,6 +247,73 @@ vduse_domain_free_bounce_pages(struct vduse_iova_domain *domain) · 13 unmodified lines
247	247	}
248	248	}
249	249
	250	+int vduse_domain_add_user_bounce_pages(struct vduse_iova_domain *domain,
	251	+ struct page **pages, int count)
	252	+{
	253	+ struct vduse_bounce_map *map;
	254	+ int i, ret;
	255	+
	256	+ /* Now we don't support partial mapping */
	257	+ if (count != (domain->bounce_size >> PAGE_SHIFT))
	258	+ return -EINVAL;
	259	+
	260	+ write_lock(&domain->bounce_lock);
	261	+ ret = -EEXIST;
	262	+ if (domain->user_bounce_pages)
	263	+ goto out;
	264	+
	265	+ for (i = 0; i < count; i++) {
	266	+ map = &domain->bounce_maps[i];
	267	+ if (map->bounce_page) {
	268	+ /* Copy kernel page to user page if it's in use */
	269	+ if (map->orig_phys != INVALID_PHYS_ADDR)
	270	+ memcpy_to_page(pages[i], 0,
	271	+ page_address(map->bounce_page),
	272	+ PAGE_SIZE);
	273	+ __free_page(map->bounce_page);
	274	+ }
	275	+ map->bounce_page = pages[i];
	276	+ get_page(pages[i]);
	277	+ }
	278	+ domain->user_bounce_pages = true;
	279	+ ret = 0;
	280	+out:
	281	+ write_unlock(&domain->bounce_lock);
	282	+
	283	+ return ret;
	284	+}
	285	+
	286	+void vduse_domain_remove_user_bounce_pages(struct vduse_iova_domain *domain)
	287	+{
	288	+ struct vduse_bounce_map *map;
	289	+ unsigned long i, count;
	290	+
	291	+ write_lock(&domain->bounce_lock);
	292	+ if (!domain->user_bounce_pages)
	293	+ goto out;
	294	+
	295	+ count = domain->bounce_size >> PAGE_SHIFT;
	296	+ for (i = 0; i < count; i++) {
	297	+ struct page *page = NULL;
	298	+
	299	+ map = &domain->bounce_maps[i];
	300	+ if (WARN_ON(!map->bounce_page))
	301	+ continue;
	302	+
	303	+ /* Copy user page to kernel page if it's in use */
	304	+ if (map->orig_phys != INVALID_PHYS_ADDR) {
	305	+ page = alloc_page(GFP_ATOMIC \| __GFP_NOFAIL);
	306	+ memcpy_from_page(page_address(page),
	307	+ map->bounce_page, 0, PAGE_SIZE);
	308	+ }
	309	+ put_page(map->bounce_page);
	310	+ map->bounce_page = page;
	311	+ }
	312	+ domain->user_bounce_pages = false;
	313	+out:
	314	+ write_unlock(&domain->bounce_lock);
	315	+}
	316	+
250	317	void vduse_domain_reset_bounce_map(struct vduse_iova_domain *domain)
251	318	{
252	319	if (!domain->bounce_map)
@@ -322,13 +389,18 @@ dma_addr_t vduse_domain_map_page(struct vduse_iova_domain *domain, · 69 unmodified lines
322	389	if (vduse_domain_init_bounce_map(domain))
323	390	goto err;
324	391
	392	+ read_lock(&domain->bounce_lock);
325	393	if (vduse_domain_map_bounce_page(domain, (u64)iova, (u64)size, pa))
326		− goto err;
	394	+ goto err_unlock;
327	395
328	396	if (dir == DMA_TO_DEVICE \|\| dir == DMA_BIDIRECTIONAL)
329	397	vduse_domain_bounce(domain, iova, size, DMA_TO_DEVICE);
330	398
	399	+ read_unlock(&domain->bounce_lock);
	400	+
331	401	return iova;
	402	+err_unlock:
	403	+ read_unlock(&domain->bounce_lock);
332	404	err:
333	405	vduse_domain_free_iova(iovad, iova, size);
334	406	return DMA_MAPPING_ERROR;
@@ -340,10 +412,12 @@ void vduse_domain_unmap_page(struct vduse_iova_domain *domain, · 5 unmodified lines
340	412	{
341	413	struct iova_domain *iovad = &domain->stream_iovad;
342	414
	415	+ read_lock(&domain->bounce_lock);
343	416	if (dir == DMA_FROM_DEVICE \|\| dir == DMA_BIDIRECTIONAL)
344	417	vduse_domain_bounce(domain, dma_addr, size, DMA_FROM_DEVICE);
345	418
346	419	vduse_domain_unmap_bounce_page(domain, (u64)dma_addr, (u64)size);
	420	+ read_unlock(&domain->bounce_lock);
347	421	vduse_domain_free_iova(iovad, dma_addr, size);
348	422	}
349	423
@@ -451,7 +525,8 @@ static int vduse_domain_release(struct inode inode, struct file file) · 101 unmodified lines
451	525
452	526	spin_lock(&domain->iotlb_lock);
453	527	vduse_iotlb_del_range(domain, 0, ULLONG_MAX);
454		− vduse_domain_free_bounce_pages(domain);
	528	+ vduse_domain_remove_user_bounce_pages(domain);
	529	+ vduse_domain_free_kernel_bounce_pages(domain);
455	530	spin_unlock(&domain->iotlb_lock);
456	531	put_iova_domain(&domain->stream_iovad);
457	532	put_iova_domain(&domain->consistent_iovad);
@@ -511,6 +586,7 @@ vduse_domain_create(unsigned long iova_limit, size_t bounce_size) · 53 unmodified lines
511	586	goto err_file;
512	587
513	588	domain->file = file;
	589	+ rwlock_init(&domain->bounce_lock);
514	590	spin_lock_init(&domain->iotlb_lock);
515	591	init_iova_domain(&domain->stream_iovad,
516	592	PAGE_SIZE, IOVA_START_PFN);

drivers/vdpa/vdpa_user/iova_domain.h+8−0

@@ -14,6 +14,7 @@ · 13 unmodified lines
14	14	#include <linux/iova.h>
15	15	#include <linux/dma-mapping.h>
16	16	#include <linux/vhost_iotlb.h>
	17	+#include <linux/rwlock.h>
17	18
18	19	#define IOVA_START_PFN 1
19	20
@@ -34,6 +35,8 @@ struct vduse_iova_domain { · 14 unmodified lines
34	35	struct vhost_iotlb *iotlb;
35	36	spinlock_t iotlb_lock;
36	37	struct file *file;
	38	+ bool user_bounce_pages;
	39	+ rwlock_t bounce_lock;
37	40	};
38	41
39	42	int vduse_domain_set_map(struct vduse_iova_domain *domain,
@@ -61,6 +64,11 @@ void vduse_domain_free_coherent(struct vduse_iova_domain *domain, size_t size, · 21 unmodified lines
61	64
62	65	void vduse_domain_reset_bounce_map(struct vduse_iova_domain *domain);
63	66
	67	+int vduse_domain_add_user_bounce_pages(struct vduse_iova_domain *domain,
	68	+ struct page **pages, int count);
	69	+
	70	+void vduse_domain_remove_user_bounce_pages(struct vduse_iova_domain *domain);
	71	+
64	72	void vduse_domain_destroy(struct vduse_iova_domain *domain);
65	73
66	74	struct vduse_iova_domain *vduse_domain_create(unsigned long iova_limit,

drivers/vdpa/vdpa_user/vduse_dev.c+180−0

@@ -21,6 +21,8 @@ · 20 unmodified lines
21	21	#include <linux/uio.h>
22	22	#include <linux/vdpa.h>
23	23	#include <linux/nospec.h>
	24	+#include <linux/vmalloc.h>
	25	+#include <linux/sched/mm.h>
24	26	#include <uapi/linux/vduse.h>
25	27	#include <uapi/linux/vdpa.h>
26	28	#include <uapi/linux/virtio_config.h>
@@ -64,6 +66,13 @@ struct vduse_vdpa { · 37 unmodified lines
64	66	struct vduse_dev *dev;
65	67	};
66	68
	69	+struct vduse_umem {
	70	+ unsigned long iova;
	71	+ unsigned long npages;
	72	+ struct page **pages;
	73	+ struct mm_struct *mm;
	74	+};
	75	+
67	76	struct vduse_dev {
68	77	struct vduse_vdpa *vdev;
69	78	struct device *dev;
@@ -95,6 +104,8 @@ struct vduse_dev { · 25 unmodified lines
95	104	u8 status;
96	105	u32 vq_num;
97	106	u32 vq_align;
	107	+ struct vduse_umem *umem;
	108	+ struct mutex mem_lock;
98	109	};
99	110
100	111	struct vduse_dev_msg {
@@ -917,6 +928,102 @@ unlock: · 816 unmodified lines
917	928	return ret;
918	929	}
919	930
	931	+static int vduse_dev_dereg_umem(struct vduse_dev *dev,
	932	+ u64 iova, u64 size)
	933	+{
	934	+ int ret;
	935	+
	936	+ mutex_lock(&dev->mem_lock);
	937	+ ret = -ENOENT;
	938	+ if (!dev->umem)
	939	+ goto unlock;
	940	+
	941	+ ret = -EINVAL;
	942	+ if (dev->umem->iova != iova \|\| size != dev->domain->bounce_size)
	943	+ goto unlock;
	944	+
	945	+ vduse_domain_remove_user_bounce_pages(dev->domain);
	946	+ unpin_user_pages_dirty_lock(dev->umem->pages,
	947	+ dev->umem->npages, true);
	948	+ atomic64_sub(dev->umem->npages, &dev->umem->mm->pinned_vm);
	949	+ mmdrop(dev->umem->mm);
	950	+ vfree(dev->umem->pages);
	951	+ kfree(dev->umem);
	952	+ dev->umem = NULL;
	953	+ ret = 0;
	954	+unlock:
	955	+ mutex_unlock(&dev->mem_lock);
	956	+ return ret;
	957	+}
	958	+
	959	+static int vduse_dev_reg_umem(struct vduse_dev *dev,
	960	+ u64 iova, u64 uaddr, u64 size)
	961	+{
	962	+ struct page **page_list = NULL;
	963	+ struct vduse_umem *umem = NULL;
	964	+ long pinned = 0;
	965	+ unsigned long npages, lock_limit;
	966	+ int ret;
	967	+
	968	+ if (!dev->domain->bounce_map \|\|
	969	+ size != dev->domain->bounce_size \|\|
	970	+ iova != 0 \|\| uaddr & ~PAGE_MASK)
	971	+ return -EINVAL;
	972	+
	973	+ mutex_lock(&dev->mem_lock);
	974	+ ret = -EEXIST;
	975	+ if (dev->umem)
	976	+ goto unlock;
	977	+
	978	+ ret = -ENOMEM;
	979	+ npages = size >> PAGE_SHIFT;
	980	+ page_list = __vmalloc(array_size(npages, sizeof(struct page *)),
	981	+ GFP_KERNEL_ACCOUNT);
	982	+ umem = kzalloc(sizeof(*umem), GFP_KERNEL);
	983	+ if (!page_list \|\| !umem)
	984	+ goto unlock;
	985	+
	986	+ mmap_read_lock(current->mm);
	987	+
	988	+ lock_limit = PFN_DOWN(rlimit(RLIMIT_MEMLOCK));
	989	+ if (npages + atomic64_read(&current->mm->pinned_vm) > lock_limit)
	990	+ goto out;
	991	+
	992	+ pinned = pin_user_pages(uaddr, npages, FOLL_LONGTERM \| FOLL_WRITE,
	993	+ page_list, NULL);
	994	+ if (pinned != npages) {
	995	+ ret = pinned < 0 ? pinned : -ENOMEM;
	996	+ goto out;
	997	+ }
	998	+
	999	+ ret = vduse_domain_add_user_bounce_pages(dev->domain,
	1000	+ page_list, pinned);
	1001	+ if (ret)
	1002	+ goto out;
	1003	+
	1004	+ atomic64_add(npages, &current->mm->pinned_vm);
	1005	+
	1006	+ umem->pages = page_list;
	1007	+ umem->npages = pinned;
	1008	+ umem->iova = iova;
	1009	+ umem->mm = current->mm;
	1010	+ mmgrab(current->mm);
	1011	+
	1012	+ dev->umem = umem;
	1013	+out:
	1014	+ if (ret && pinned > 0)
	1015	+ unpin_user_pages(page_list, pinned);
	1016	+
	1017	+ mmap_read_unlock(current->mm);
	1018	+unlock:
	1019	+ if (ret) {
	1020	+ vfree(page_list);
	1021	+ kfree(umem);
	1022	+ }
	1023	+ mutex_unlock(&dev->mem_lock);
	1024	+ return ret;
	1025	+}
	1026	+
920	1027	static long vduse_dev_ioctl(struct file *file, unsigned int cmd,
921	1028	unsigned long arg)
922	1029	{
@@ -1089,6 +1196,77 @@ static long vduse_dev_ioctl(struct file *file, unsigned int cmd, · 166 unmodified lines
1089	1196	ret = vduse_dev_queue_irq_work(dev, &dev->vqs[index].inject);
1090	1197	break;
1091	1198	}
	1199	+ case VDUSE_IOTLB_REG_UMEM: {
	1200	+ struct vduse_iova_umem umem;
	1201	+
	1202	+ ret = -EFAULT;
	1203	+ if (copy_from_user(&umem, argp, sizeof(umem)))
	1204	+ break;
	1205	+
	1206	+ ret = -EINVAL;
	1207	+ if (!is_mem_zero((const char *)umem.reserved,
	1208	+ sizeof(umem.reserved)))
	1209	+ break;
	1210	+
	1211	+ ret = vduse_dev_reg_umem(dev, umem.iova,
	1212	+ umem.uaddr, umem.size);
	1213	+ break;
	1214	+ }
	1215	+ case VDUSE_IOTLB_DEREG_UMEM: {
	1216	+ struct vduse_iova_umem umem;
	1217	+
	1218	+ ret = -EFAULT;
	1219	+ if (copy_from_user(&umem, argp, sizeof(umem)))
	1220	+ break;
	1221	+
	1222	+ ret = -EINVAL;
	1223	+ if (!is_mem_zero((const char *)umem.reserved,
	1224	+ sizeof(umem.reserved)))
	1225	+ break;
	1226	+
	1227	+ ret = vduse_dev_dereg_umem(dev, umem.iova,
	1228	+ umem.size);
	1229	+ break;
	1230	+ }
	1231	+ case VDUSE_IOTLB_GET_INFO: {
	1232	+ struct vduse_iova_info info;
	1233	+ struct vhost_iotlb_map *map;
	1234	+ struct vduse_iova_domain *domain = dev->domain;
	1235	+
	1236	+ ret = -EFAULT;
	1237	+ if (copy_from_user(&info, argp, sizeof(info)))
	1238	+ break;
	1239	+
	1240	+ ret = -EINVAL;
	1241	+ if (info.start > info.last)
	1242	+ break;
	1243	+
	1244	+ if (!is_mem_zero((const char *)info.reserved,
	1245	+ sizeof(info.reserved)))
	1246	+ break;
	1247	+
	1248	+ spin_lock(&domain->iotlb_lock);
	1249	+ map = vhost_iotlb_itree_first(domain->iotlb,
	1250	+ info.start, info.last);
	1251	+ if (map) {
	1252	+ info.start = map->start;
	1253	+ info.last = map->last;
	1254	+ info.capability = 0;
	1255	+ if (domain->bounce_map && map->start == 0 &&
	1256	+ map->last == domain->bounce_size - 1)
	1257	+ info.capability \|= VDUSE_IOVA_CAP_UMEM;
	1258	+ }
	1259	+ spin_unlock(&domain->iotlb_lock);
	1260	+ if (!map)
	1261	+ break;
	1262	+
	1263	+ ret = -EFAULT;
	1264	+ if (copy_to_user(argp, &info, sizeof(info)))
	1265	+ break;
	1266	+
	1267	+ ret = 0;
	1268	+ break;
	1269	+ }
1092	1270	default:
1093	1271	ret = -ENOIOCTLCMD;
1094	1272	break;
@@ -1101,6 +1279,7 @@ static int vduse_dev_release(struct inode inode, struct file file) · 6 unmodified lines
1101	1279	{
1102	1280	struct vduse_dev *dev = file->private_data;
1103	1281
	1282	+ vduse_dev_dereg_umem(dev, 0, dev->domain->bounce_size);
1104	1283	spin_lock(&dev->msg_lock);
1105	1284	/* Make sure the inflight messages can processed after reconncection */
1106	1285	list_splice_init(&dev->recv_list, &dev->send_list);
@@ -1163,6 +1342,7 @@ static struct vduse_dev *vduse_dev_create(void) · 56 unmodified lines
1163	1342	return NULL;
1164	1343
1165	1344	mutex_init(&dev->lock);
	1345	+ mutex_init(&dev->mem_lock);
1166	1346	spin_lock_init(&dev->msg_lock);
1167	1347	INIT_LIST_HEAD(&dev->send_list);
1168	1348	INIT_LIST_HEAD(&dev->recv_list);

drivers/vhost/scsi.c+61−24

@@ -159,9 +159,13 @@ enum { · 158 unmodified lines
159	159	};
160	160
161	161	#define VHOST_SCSI_MAX_TARGET 256
162		−#define VHOST_SCSI_MAX_VQ 128
	162	+#define VHOST_SCSI_MAX_IO_VQ 1024
163	163	#define VHOST_SCSI_MAX_EVENT 128
164	164
	165	+static unsigned vhost_scsi_max_io_vqs = 128;
	166	+module_param_named(max_io_vqs, vhost_scsi_max_io_vqs, uint, 0644);
	167	+MODULE_PARM_DESC(max_io_vqs, "Set the max number of IO virtqueues a vhost scsi device can support. The default is 128. The max is 1024.");
	168	+
165	169	struct vhost_scsi_virtqueue {
166	170	struct vhost_virtqueue vq;
167	171	/*
@@ -186,7 +190,9 @@ struct vhost_scsi { · 18 unmodified lines
186	190	char vs_vhost_wwpn[TRANSPORT_IQN_LEN];
187	191
188	192	struct vhost_dev dev;
189		− struct vhost_scsi_virtqueue vqs[VHOST_SCSI_MAX_VQ];
	193	+ struct vhost_scsi_virtqueue *vqs;
	194	+ unsigned long *compl_bitmap;
	195	+ struct vhost_scsi_inflight **old_inflight;
190	196
191	197	struct vhost_work vs_completion_work; /* cmd completion work item */
192	198	struct llist_head vs_completion_list; /* cmd completion queue */
@@ -245,7 +251,7 @@ static void vhost_scsi_init_inflight(struct vhost_scsi *vs, · 52 unmodified lines
245	251	struct vhost_virtqueue *vq;
246	252	int idx, i;
247	253
248		− for (i = 0; i < VHOST_SCSI_MAX_VQ; i++) {
	254	+ for (i = 0; i < vs->dev.nvqs; i++) {
249	255	vq = &vs->vqs[i].vq;
250	256
251	257	mutex_lock(&vq->mutex);
@@ -533,7 +539,6 @@ static void vhost_scsi_complete_cmd_work(struct vhost_work *work) · 281 unmodified lines
533	539	{
534	540	struct vhost_scsi *vs = container_of(work, struct vhost_scsi,
535	541	vs_completion_work);
536		− DECLARE_BITMAP(signal, VHOST_SCSI_MAX_VQ);
537	542	struct virtio_scsi_cmd_resp v_rsp;
538	543	struct vhost_scsi_cmd cmd, t;
539	544	struct llist_node *llnode;
@@ -541,7 +546,7 @@ static void vhost_scsi_complete_cmd_work(struct vhost_work *work) · 1 unmodified line
541	546	struct iov_iter iov_iter;
542	547	int ret, vq;
543	548
544		− bitmap_zero(signal, VHOST_SCSI_MAX_VQ);
	549	+ bitmap_zero(vs->compl_bitmap, vs->dev.nvqs);
545	550	llnode = llist_del_all(&vs->vs_completion_list);
546	551	llist_for_each_entry_safe(cmd, t, llnode, tvc_completion_list) {
547	552	se_cmd = &cmd->tvc_se_cmd;
@@ -566,7 +571,7 @@ static void vhost_scsi_complete_cmd_work(struct vhost_work *work) · 18 unmodified lines
566	571	vhost_add_used(cmd->tvc_vq, cmd->tvc_vq_desc, 0);
567	572	q = container_of(cmd->tvc_vq, struct vhost_scsi_virtqueue, vq);
568	573	vq = q - vs->vqs;
569		− __set_bit(vq, signal);
	574	+ __set_bit(vq, vs->compl_bitmap);
570	575	} else
571	576	pr_err("Faulted on virtio_scsi_cmd_resp\n");
572	577
@@ -574,8 +579,8 @@ static void vhost_scsi_complete_cmd_work(struct vhost_work *work) · 1 unmodified line
574	579	}
575	580
576	581	vq = -1;
577		− while ((vq = find_next_bit(signal, VHOST_SCSI_MAX_VQ, vq + 1))
578		− < VHOST_SCSI_MAX_VQ)
	582	+ while ((vq = find_next_bit(vs->compl_bitmap, vs->dev.nvqs, vq + 1))
	583	+ < vs->dev.nvqs)
579	584	vhost_signal(&vs->dev, &vs->vqs[vq].vq);
580	585	}
581	586
@@ -1419,26 +1424,25 @@ static void vhost_scsi_handle_kick(struct vhost_work *work) · 837 unmodified lines
1419	1424	/* Callers must hold dev mutex */
1420	1425	static void vhost_scsi_flush(struct vhost_scsi *vs)
1421	1426	{
1422		− struct vhost_scsi_inflight *old_inflight[VHOST_SCSI_MAX_VQ];
1423	1427	int i;
1424	1428
1425	1429	/* Init new inflight and remember the old inflight */
1426		− vhost_scsi_init_inflight(vs, old_inflight);
	1430	+ vhost_scsi_init_inflight(vs, vs->old_inflight);
1427	1431
1428	1432	/*
1429	1433	* The inflight->kref was initialized to 1. We decrement it here to
1430	1434	* indicate the start of the flush operation so that it will reach 0
1431	1435	* when all the reqs are finished.
1432	1436	*/
1433		− for (i = 0; i < VHOST_SCSI_MAX_VQ; i++)
1434		− kref_put(&old_inflight[i]->kref, vhost_scsi_done_inflight);
	1437	+ for (i = 0; i < vs->dev.nvqs; i++)
	1438	+ kref_put(&vs->old_inflight[i]->kref, vhost_scsi_done_inflight);
1435	1439
1436	1440	/* Flush both the vhost poll and vhost work */
1437	1441	vhost_dev_flush(&vs->dev);
1438	1442
1439	1443	/* Wait for all reqs issued before the flush to be finished */
1440		− for (i = 0; i < VHOST_SCSI_MAX_VQ; i++)
1441		− wait_for_completion(&old_inflight[i]->comp);
	1444	+ for (i = 0; i < vs->dev.nvqs; i++)
	1445	+ wait_for_completion(&vs->old_inflight[i]->comp);
1442	1446	}
1443	1447
1444	1448	static void vhost_scsi_destroy_vq_cmds(struct vhost_virtqueue *vq)
@@ -1601,7 +1605,7 @@ vhost_scsi_set_endpoint(struct vhost_scsi *vs, · 156 unmodified lines
1601	1605	memcpy(vs->vs_vhost_wwpn, t->vhost_wwpn,
1602	1606	sizeof(vs->vs_vhost_wwpn));
1603	1607
1604		− for (i = VHOST_SCSI_VQ_IO; i < VHOST_SCSI_MAX_VQ; i++) {
	1608	+ for (i = VHOST_SCSI_VQ_IO; i < vs->dev.nvqs; i++) {
1605	1609	vq = &vs->vqs[i].vq;
1606	1610	if (!vhost_vq_is_setup(vq))
1607	1611	continue;
@@ -1611,7 +1615,7 @@ vhost_scsi_set_endpoint(struct vhost_scsi *vs, · 3 unmodified lines
1611	1615	goto destroy_vq_cmds;
1612	1616	}
1613	1617
1614		− for (i = 0; i < VHOST_SCSI_MAX_VQ; i++) {
	1618	+ for (i = 0; i < vs->dev.nvqs; i++) {
1615	1619	vq = &vs->vqs[i].vq;
1616	1620	mutex_lock(&vq->mutex);
1617	1621	vhost_vq_set_backend(vq, vs_tpg);
@@ -1713,7 +1717,7 @@ vhost_scsi_clear_endpoint(struct vhost_scsi *vs, · 95 unmodified lines
1713	1717	target_undepend_item(&se_tpg->tpg_group.cg_item);
1714	1718	}
1715	1719	if (match) {
1716		− for (i = 0; i < VHOST_SCSI_MAX_VQ; i++) {
	1720	+ for (i = 0; i < vs->dev.nvqs; i++) {
1717	1721	vq = &vs->vqs[i].vq;
1718	1722	mutex_lock(&vq->mutex);
1719	1723	vhost_vq_set_backend(vq, NULL);
@@ -1722,7 +1726,7 @@ vhost_scsi_clear_endpoint(struct vhost_scsi *vs, · 2 unmodified lines
1722	1726	/* Make sure cmds are not running before tearing them down. */
1723	1727	vhost_scsi_flush(vs);
1724	1728
1725		− for (i = 0; i < VHOST_SCSI_MAX_VQ; i++) {
	1729	+ for (i = 0; i < vs->dev.nvqs; i++) {
1726	1730	vq = &vs->vqs[i].vq;
1727	1731	vhost_scsi_destroy_vq_cmds(vq);
1728	1732	}
@@ -1762,7 +1766,7 @@ static int vhost_scsi_set_features(struct vhost_scsi *vs, u64 features) · 33 unmodified lines
1762	1766	return -EFAULT;
1763	1767	}
1764	1768
1765		− for (i = 0; i < VHOST_SCSI_MAX_VQ; i++) {
	1769	+ for (i = 0; i < vs->dev.nvqs; i++) {
1766	1770	vq = &vs->vqs[i].vq;
1767	1771	mutex_lock(&vq->mutex);
1768	1772	vq->acked_features = features;
@@ -1776,16 +1780,40 @@ static int vhost_scsi_open(struct inode inode, struct file f) · 7 unmodified lines
1776	1780	{
1777	1781	struct vhost_scsi *vs;
1778	1782	struct vhost_virtqueue **vqs;
1779		− int r = -ENOMEM, i;
	1783	+ int r = -ENOMEM, i, nvqs = vhost_scsi_max_io_vqs;
1780	1784
1781	1785	vs = kvzalloc(sizeof(*vs), GFP_KERNEL);
1782	1786	if (!vs)
1783	1787	goto err_vs;
1784	1788
1785		− vqs = kmalloc_array(VHOST_SCSI_MAX_VQ, sizeof(*vqs), GFP_KERNEL);
1786		− if (!vqs)
	1789	+ if (nvqs > VHOST_SCSI_MAX_IO_VQ) {
	1790	+ pr_err("Invalid max_io_vqs of %d. Using %d.\n", nvqs,
	1791	+ VHOST_SCSI_MAX_IO_VQ);
	1792	+ nvqs = VHOST_SCSI_MAX_IO_VQ;
	1793	+ } else if (nvqs == 0) {
	1794	+ pr_err("Invalid max_io_vqs of %d. Using 1.\n", nvqs);
	1795	+ nvqs = 1;
	1796	+ }
	1797	+ nvqs += VHOST_SCSI_VQ_IO;
	1798	+
	1799	+ vs->compl_bitmap = bitmap_alloc(nvqs, GFP_KERNEL);
	1800	+ if (!vs->compl_bitmap)
	1801	+ goto err_compl_bitmap;
	1802	+
	1803	+ vs->old_inflight = kmalloc_array(nvqs, sizeof(*vs->old_inflight),
	1804	+ GFP_KERNEL \| __GFP_ZERO);
	1805	+ if (!vs->old_inflight)
	1806	+ goto err_inflight;
	1807	+
	1808	+ vs->vqs = kmalloc_array(nvqs, sizeof(*vs->vqs),
	1809	+ GFP_KERNEL \| __GFP_ZERO);
	1810	+ if (!vs->vqs)
1787	1811	goto err_vqs;
1788	1812
	1813	+ vqs = kmalloc_array(nvqs, sizeof(*vqs), GFP_KERNEL);
	1814	+ if (!vqs)
	1815	+ goto err_local_vqs;
	1816	+
1789	1817	vhost_work_init(&vs->vs_completion_work, vhost_scsi_complete_cmd_work);
1790	1818	vhost_work_init(&vs->vs_event_work, vhost_scsi_evt_work);
1791	1819
@@ -1796,11 +1824,11 @@ static int vhost_scsi_open(struct inode inode, struct file f) · 4 unmodified lines
1796	1824	vqs[VHOST_SCSI_VQ_EVT] = &vs->vqs[VHOST_SCSI_VQ_EVT].vq;
1797	1825	vs->vqs[VHOST_SCSI_VQ_CTL].vq.handle_kick = vhost_scsi_ctl_handle_kick;
1798	1826	vs->vqs[VHOST_SCSI_VQ_EVT].vq.handle_kick = vhost_scsi_evt_handle_kick;
1799		− for (i = VHOST_SCSI_VQ_IO; i < VHOST_SCSI_MAX_VQ; i++) {
	1827	+ for (i = VHOST_SCSI_VQ_IO; i < nvqs; i++) {
1800	1828	vqs[i] = &vs->vqs[i].vq;
1801	1829	vs->vqs[i].vq.handle_kick = vhost_scsi_handle_kick;
1802	1830	}
1803		− vhost_dev_init(&vs->dev, vqs, VHOST_SCSI_MAX_VQ, UIO_MAXIOV,
	1831	+ vhost_dev_init(&vs->dev, vqs, nvqs, UIO_MAXIOV,
1804	1832	VHOST_SCSI_WEIGHT, 0, true, NULL);
1805	1833
1806	1834	vhost_scsi_init_inflight(vs, NULL);
@@ -1808,7 +1836,13 @@ static int vhost_scsi_open(struct inode inode, struct file f) · 1 unmodified line
1808	1836	f->private_data = vs;
1809	1837	return 0;
1810	1838
	1839	+err_local_vqs:
	1840	+ kfree(vs->vqs);
1811	1841	err_vqs:
	1842	+ kfree(vs->old_inflight);
	1843	+err_inflight:
	1844	+ bitmap_free(vs->compl_bitmap);
	1845	+err_compl_bitmap:
1812	1846	kvfree(vs);
1813	1847	err_vs:
1814	1848	return r;
@@ -1826,6 +1860,9 @@ static int vhost_scsi_release(struct inode inode, struct file f) · 11 unmodified lines
1826	1860	vhost_dev_stop(&vs->dev);
1827	1861	vhost_dev_cleanup(&vs->dev);
1828	1862	kfree(vs->dev.vqs);
	1863	+ kfree(vs->vqs);
	1864	+ kfree(vs->old_inflight);
	1865	+ bitmap_free(vs->compl_bitmap);
1829	1866	kvfree(vs);
1830	1867	return 0;
1831	1868	}

drivers/vhost/vdpa.c+36−2

@@ -347,6 +347,14 @@ static long vhost_vdpa_set_config(struct vhost_vdpa *v, · 346 unmodified lines
347	347	return 0;
348	348	}
349	349
	350	+static bool vhost_vdpa_can_suspend(const struct vhost_vdpa *v)
	351	+{
	352	+ struct vdpa_device *vdpa = v->vdpa;
	353	+ const struct vdpa_config_ops *ops = vdpa->config;
	354	+
	355	+ return ops->suspend;
	356	+}
	357	+
350	358	static long vhost_vdpa_get_features(struct vhost_vdpa v, u64 __user featurep)
351	359	{
352	360	struct vdpa_device *vdpa = v->vdpa;
@@ -470,6 +478,22 @@ static long vhost_vdpa_get_vqs_count(struct vhost_vdpa v, u32 __user argp) · 117 unmodified lines
470	478	return 0;
471	479	}
472	480
	481	+/* After a successful return of ioctl the device must not process more
	482	+ * virtqueue descriptors. The device can answer to read or writes of config
	483	+ * fields as if it were not suspended. In particular, writing to "queue_enable"
	484	+ * with a value of 1 will not make the device start processing buffers.
	485	+ */
	486	+static long vhost_vdpa_suspend(struct vhost_vdpa *v)
	487	+{
	488	+ struct vdpa_device *vdpa = v->vdpa;
	489	+ const struct vdpa_config_ops *ops = vdpa->config;
	490	+
	491	+ if (!ops->suspend)
	492	+ return -EOPNOTSUPP;
	493	+
	494	+ return ops->suspend(vdpa);
	495	+}
	496	+
473	497	static long vhost_vdpa_vring_ioctl(struct vhost_vdpa *v, unsigned int cmd,
474	498	void __user *argp)
475	499	{
@@ -577,7 +601,11 @@ static long vhost_vdpa_unlocked_ioctl(struct file *filep, · 101 unmodified lines
577	601	if (cmd == VHOST_SET_BACKEND_FEATURES) {
578	602	if (copy_from_user(&features, featurep, sizeof(features)))
579	603	return -EFAULT;
580		− if (features & ~VHOST_VDPA_BACKEND_FEATURES)
	604	+ if (features & ~(VHOST_VDPA_BACKEND_FEATURES \|
	605	+ BIT_ULL(VHOST_BACKEND_F_SUSPEND)))
	606	+ return -EOPNOTSUPP;
	607	+ if ((features & BIT_ULL(VHOST_BACKEND_F_SUSPEND)) &&
	608	+ !vhost_vdpa_can_suspend(v))
581	609	return -EOPNOTSUPP;
582	610	vhost_set_backend_features(&v->vdev, features);
583	611	return 0;
@@ -628,6 +656,8 @@ static long vhost_vdpa_unlocked_ioctl(struct file *filep, · 44 unmodified lines
628	656	break;
629	657	case VHOST_GET_BACKEND_FEATURES:
630	658	features = VHOST_VDPA_BACKEND_FEATURES;
	659	+ if (vhost_vdpa_can_suspend(v))
	660	+ features \|= BIT_ULL(VHOST_BACKEND_F_SUSPEND);
631	661	if (copy_to_user(featurep, &features, sizeof(features)))
632	662	r = -EFAULT;
633	663	break;
@@ -640,6 +670,9 @@ static long vhost_vdpa_unlocked_ioctl(struct file *filep, · 6 unmodified lines
640	670	case VHOST_VDPA_GET_VQS_COUNT:
641	671	r = vhost_vdpa_get_vqs_count(v, argp);
642	672	break;
	673	+ case VHOST_VDPA_SUSPEND:
	674	+ r = vhost_vdpa_suspend(v);
	675	+ break;
643	676	default:
644	677	r = vhost_dev_ioctl(&v->vdev, cmd, argp);
645	678	if (r == -ENOIOCTLCMD)
@@ -1076,7 +1109,7 @@ static int vhost_vdpa_alloc_domain(struct vhost_vdpa *v) · 430 unmodified lines
1076	1109	if (!bus)
1077	1110	return -EFAULT;
1078	1111
1079		− if (!iommu_capable(bus, IOMMU_CAP_CACHE_COHERENCY))
	1112	+ if (!device_iommu_capable(dma_dev, IOMMU_CAP_CACHE_COHERENCY))
1080	1113	return -ENOTSUPP;
1081	1114
1082	1115	v->domain = iommu_domain_alloc(bus);
@@ -1363,6 +1396,7 @@ static int vhost_vdpa_probe(struct vdpa_device *vdpa) · 280 unmodified lines
1363	1396
1364	1397	err:
1365	1398	put_device(&v->dev);
	1399	+ ida_simple_remove(&vhost_vdpa_ida, v->minor);
1366	1400	return r;
1367	1401	}
1368	1402

drivers/vhost/vringh.c+56−22

@@ -1095,7 +1095,8 @@ EXPORT_SYMBOL(vringh_need_notify_kern); · 1094 unmodified lines
1095	1095	#if IS_REACHABLE(CONFIG_VHOST_IOTLB)
1096	1096
1097	1097	static int iotlb_translate(const struct vringh *vrh,
1098		− u64 addr, u64 len, struct bio_vec iov[],
	1098	+ u64 addr, u64 len, u64 *translated,
	1099	+ struct bio_vec iov[],
1099	1100	int iov_size, u32 perm)
1100	1101	{
1101	1102	struct vhost_iotlb_map *map;
@@ -1136,43 +1137,76 @@ static int iotlb_translate(const struct vringh *vrh, · 34 unmodified lines
1136	1137
1137	1138	spin_unlock(vrh->iotlb_lock);
1138	1139
	1140	+ if (translated)
	1141	+ *translated = min(len, s);
	1142	+
1139	1143	return ret;
1140	1144	}
1141	1145
1142	1146	static inline int copy_from_iotlb(const struct vringh vrh, void dst,
1143	1147	void *src, size_t len)
1144	1148	{
1145		− struct iov_iter iter;
1146		− struct bio_vec iov[16];
1147		− int ret;
	1149	+ u64 total_translated = 0;
1148	1150
1149		− ret = iotlb_translate(vrh, (u64)(uintptr_t)src,
1150		− len, iov, 16, VHOST_MAP_RO);
1151		− if (ret < 0)
1152		− return ret;
	1151	+ while (total_translated < len) {
	1152	+ struct bio_vec iov[16];
	1153	+ struct iov_iter iter;
	1154	+ u64 translated;
	1155	+ int ret;
1153	1156
1154		− iov_iter_bvec(&iter, READ, iov, ret, len);
	1157	+ ret = iotlb_translate(vrh, (u64)(uintptr_t)src,
	1158	+ len - total_translated, &translated,
	1159	+ iov, ARRAY_SIZE(iov), VHOST_MAP_RO);
	1160	+ if (ret == -ENOBUFS)
	1161	+ ret = ARRAY_SIZE(iov);
	1162	+ else if (ret < 0)
	1163	+ return ret;
1155	1164
1156		− ret = copy_from_iter(dst, len, &iter);
	1165	+ iov_iter_bvec(&iter, READ, iov, ret, translated);
1157	1166
1158		− return ret;
	1167	+ ret = copy_from_iter(dst, translated, &iter);
	1168	+ if (ret < 0)
	1169	+ return ret;
	1170	+
	1171	+ src += translated;
	1172	+ dst += translated;
	1173	+ total_translated += translated;
	1174	+ }
	1175	+
	1176	+ return total_translated;
1159	1177	}
1160	1178
1161	1179	static inline int copy_to_iotlb(const struct vringh vrh, void dst,
1162	1180	void *src, size_t len)
1163	1181	{
1164		− struct iov_iter iter;
1165		− struct bio_vec iov[16];
1166		− int ret;
	1182	+ u64 total_translated = 0;
1167	1183
1168		− ret = iotlb_translate(vrh, (u64)(uintptr_t)dst,
1169		− len, iov, 16, VHOST_MAP_WO);
1170		− if (ret < 0)
1171		− return ret;
	1184	+ while (total_translated < len) {
	1185	+ struct bio_vec iov[16];
	1186	+ struct iov_iter iter;
	1187	+ u64 translated;
	1188	+ int ret;
	1189	+
	1190	+ ret = iotlb_translate(vrh, (u64)(uintptr_t)dst,
	1191	+ len - total_translated, &translated,
	1192	+ iov, ARRAY_SIZE(iov), VHOST_MAP_WO);
	1193	+ if (ret == -ENOBUFS)
	1194	+ ret = ARRAY_SIZE(iov);
	1195	+ else if (ret < 0)
	1196	+ return ret;
1172	1197
1173		− iov_iter_bvec(&iter, WRITE, iov, ret, len);
	1198	+ iov_iter_bvec(&iter, WRITE, iov, ret, translated);
	1199	+
	1200	+ ret = copy_to_iter(src, translated, &iter);
	1201	+ if (ret < 0)
	1202	+ return ret;
	1203	+
	1204	+ src += translated;
	1205	+ dst += translated;
	1206	+ total_translated += translated;
	1207	+ }
1174	1208
1175		− return copy_to_iter(src, len, &iter);
	1209	+ return total_translated;
1176	1210	}
1177	1211
1178	1212	static inline int getu16_iotlb(const struct vringh *vrh,
@@ -1183,7 +1217,7 @@ static inline int getu16_iotlb(const struct vringh *vrh, · 4 unmodified lines
1183	1217	int ret;
1184	1218
1185	1219	/* Atomic read is needed for getu16 */
1186		− ret = iotlb_translate(vrh, (u64)(uintptr_t)p, sizeof(*p),
	1220	+ ret = iotlb_translate(vrh, (u64)(uintptr_t)p, sizeof(*p), NULL,
1187	1221	&iov, 1, VHOST_MAP_RO);
1188	1222	if (ret < 0)
1189	1223	return ret;
@@ -1204,7 +1238,7 @@ static inline int putu16_iotlb(const struct vringh *vrh, · 14 unmodified lines
1204	1238	int ret;
1205	1239
1206	1240	/* Atomic write is needed for putu16 */
1207		− ret = iotlb_translate(vrh, (u64)(uintptr_t)p, sizeof(*p),
	1241	+ ret = iotlb_translate(vrh, (u64)(uintptr_t)p, sizeof(*p), NULL,
1208	1242	&iov, 1, VHOST_MAP_WO);
1209	1243	if (ret < 0)
1210	1244	return ret;

drivers/virtio/Kconfig+7−4

@@ -35,11 +35,12 @@ if VIRTIO_MENU · 34 unmodified lines
35	35
36	36	config VIRTIO_HARDEN_NOTIFICATION
37	37	bool "Harden virtio notification"
	38	+ depends on BROKEN
38	39	help
39	40	Enable this to harden the device notifications and suppress
40	41	those that happen at a time where notifications are illegal.
41	42
42		− Experimental: Note that several drivers still have bugs that
	43	+ Experimental: Note that several drivers still have issues that
43	44	may cause crashes or hangs when correct handling of
44	45	notifications is enforced; depending on the subset of
45	46	drivers and devices you use, this may or may not work.
@@ -126,9 +127,11 @@ config VIRTIO_MEM · 80 unmodified lines
126	127	This driver provides access to virtio-mem paravirtualized memory
127	128	devices, allowing to hotplug and hotunplug memory.
128	129
129		− This driver was only tested under x86-64 and arm64, but should
130		− theoretically work on all architectures that support memory hotplug
131		− and hotremove.
	130	+ This driver currently only supports x86-64 and arm64. Although it
	131	+ should compile on other architectures that implement memory
	132	+ hot(un)plug, architecture-specific and/or common
	133	+ code changes may be required for virtio-mem, kdump and kexec to work as
	134	+ expected.
132	135
133	136	If unsure, say M.
134	137

drivers/virtio/virtio.c+3−1

@@ -428,7 +428,9 @@ int register_virtio_device(struct virtio_device *dev) · 427 unmodified lines
428	428	goto out;
429	429
430	430	dev->index = err;
431		− dev_set_name(&dev->dev, "virtio%u", dev->index);
	431	+ err = dev_set_name(&dev->dev, "virtio%u", dev->index);
	432	+ if (err)
	433	+ goto out_ida_remove;
432	434
433	435	err = virtio_device_of_init(dev);
434	436	if (err)

drivers/virtio/virtio_mmio.c+12−2

@@ -360,7 +360,7 @@ static void vm_synchronize_cbs(struct virtio_device *vdev) · 359 unmodified lines
360	360
361	361	static struct virtqueue vm_setup_vq(struct virtio_device vdev, unsigned int index,
362	362	void (callback)(struct virtqueue vq),
363		− const char *name, bool ctx)
	363	+ const char *name, u32 size, bool ctx)
364	364	{
365	365	struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev);
366	366	struct virtio_mmio_vq_info *info;
@@ -395,14 +395,19 @@ static struct virtqueue vm_setup_vq(struct virtio_device vdev, unsigned int in · 28 unmodified lines
395	395	goto error_new_virtqueue;
396	396	}
397	397
	398	+ if (!size \|\| size > num)
	399	+ size = num;
	400	+
398	401	/* Create the vring */
399		− vq = vring_create_virtqueue(index, num, VIRTIO_MMIO_VRING_ALIGN, vdev,
	402	+ vq = vring_create_virtqueue(index, size, VIRTIO_MMIO_VRING_ALIGN, vdev,
400	403	true, true, ctx, vm_notify, callback, name);
401	404	if (!vq) {
402	405	err = -ENOMEM;
403	406	goto error_new_virtqueue;
404	407	}
405	408
	409	+ vq->num_max = num;
	410	+
406	411	/* Activate the queue */
407	412	writel(virtqueue_get_vring_size(vq), vm_dev->base + VIRTIO_MMIO_QUEUE_NUM);
408	413	if (vm_dev->version == 1) {
@@ -472,6 +477,7 @@ static int vm_find_vqs(struct virtio_device *vdev, unsigned int nvqs, · 63 unmodified lines
472	477	struct virtqueue *vqs[],
473	478	vq_callback_t *callbacks[],
474	479	const char * const names[],
	480	+ u32 sizes[],
475	481	const bool *ctx,
476	482	struct irq_affinity *desc)
477	483	{
@@ -487,6 +493,9 @@ static int vm_find_vqs(struct virtio_device *vdev, unsigned int nvqs, · 9 unmodified lines
487	493	if (err)
488	494	return err;
489	495
	496	+ if (of_property_read_bool(vm_dev->pdev->dev.of_node, "wakeup-source"))
	497	+ enable_irq_wake(irq);
	498	+
490	499	for (i = 0; i < nvqs; ++i) {
491	500	if (!names[i]) {
492	501	vqs[i] = NULL;
@@ -494,6 +503,7 @@ static int vm_find_vqs(struct virtio_device *vdev, unsigned int nvqs, · 1 unmodified line
494	503	}
495	504
496	505	vqs[i] = vm_setup_vq(vdev, queue_idx++, callbacks[i], names[i],
	506	+ sizes ? sizes[i] : 0,
497	507	ctx ? ctx[i] : false);
498	508	if (IS_ERR(vqs[i])) {
499	509	vm_del_vqs(vdev);

drivers/virtio/virtio_pci_common.c+20−12

@@ -174,6 +174,7 @@ error: · 173 unmodified lines
174	174	static struct virtqueue vp_setup_vq(struct virtio_device vdev, unsigned int index,
175	175	void (callback)(struct virtqueue vq),
176	176	const char *name,
	177	+ u32 size,
177	178	bool ctx,
178	179	u16 msix_vec)
179	180	{
@@ -186,7 +187,7 @@ static struct virtqueue vp_setup_vq(struct virtio_device vdev, unsigned int in · 6 unmodified lines
186	187	if (!info)
187	188	return ERR_PTR(-ENOMEM);
188	189
189		− vq = vp_dev->setup_vq(vp_dev, info, index, callback, name, ctx,
	190	+ vq = vp_dev->setup_vq(vp_dev, info, index, callback, name, size, ctx,
190	191	msix_vec);
191	192	if (IS_ERR(vq))
192	193	goto out_info;
@@ -214,9 +215,15 @@ static void vp_del_vq(struct virtqueue *vq) · 21 unmodified lines
214	215	struct virtio_pci_vq_info *info = vp_dev->vqs[vq->index];
215	216	unsigned long flags;
216	217
217		− spin_lock_irqsave(&vp_dev->lock, flags);
218		− list_del(&info->node);
219		− spin_unlock_irqrestore(&vp_dev->lock, flags);
	218	+ /*
	219	+ * If it fails during re-enable reset vq. This way we won't rejoin
	220	+ * info->node to the queue. Prevent unexpected irqs.
	221	+ */
	222	+ if (!vq->reset) {
	223	+ spin_lock_irqsave(&vp_dev->lock, flags);
	224	+ list_del(&info->node);
	225	+ spin_unlock_irqrestore(&vp_dev->lock, flags);
	226	+ }
220	227
221	228	vp_dev->del_vq(info);
222	229	kfree(info);
@@ -277,7 +284,7 @@ void vp_del_vqs(struct virtio_device *vdev) · 54 unmodified lines
277	284
278	285	static int vp_find_vqs_msix(struct virtio_device *vdev, unsigned int nvqs,
279	286	struct virtqueue vqs[], vq_callback_t callbacks[],
280		− const char * const names[], bool per_vq_vectors,
	287	+ const char * const names[], u32 sizes[], bool per_vq_vectors,
281	288	const bool *ctx,
282	289	struct irq_affinity *desc)
283	290	{
@@ -320,8 +327,8 @@ static int vp_find_vqs_msix(struct virtio_device *vdev, unsigned int nvqs, · 36 unmodified lines
320	327	else
321	328	msix_vec = VP_MSIX_VQ_VECTOR;
322	329	vqs[i] = vp_setup_vq(vdev, queue_idx++, callbacks[i], names[i],
323		− ctx ? ctx[i] : false,
324		− msix_vec);
	330	+ sizes ? sizes[i] : 0,
	331	+ ctx ? ctx[i] : false, msix_vec);
325	332	if (IS_ERR(vqs[i])) {
326	333	err = PTR_ERR(vqs[i]);
327	334	goto error_find;
@@ -351,7 +358,7 @@ error_find: · 23 unmodified lines
351	358
352	359	static int vp_find_vqs_intx(struct virtio_device *vdev, unsigned int nvqs,
353	360	struct virtqueue vqs[], vq_callback_t callbacks[],
354		− const char * const names[], const bool *ctx)
	361	+ const char * const names[], u32 sizes[], const bool *ctx)
355	362	{
356	363	struct virtio_pci_device *vp_dev = to_vp_device(vdev);
357	364	int i, err, queue_idx = 0;
@@ -373,6 +380,7 @@ static int vp_find_vqs_intx(struct virtio_device *vdev, unsigned int nvqs, · 15 unmodified lines
373	380	continue;
374	381	}
375	382	vqs[i] = vp_setup_vq(vdev, queue_idx++, callbacks[i], names[i],
	383	+ sizes ? sizes[i] : 0,
376	384	ctx ? ctx[i] : false,
377	385	VIRTIO_MSI_NO_VECTOR);
378	386	if (IS_ERR(vqs[i])) {
@@ -390,21 +398,21 @@ out_del_vqs: · 11 unmodified lines
390	398	/* the config->find_vqs() implementation */
391	399	int vp_find_vqs(struct virtio_device *vdev, unsigned int nvqs,
392	400	struct virtqueue vqs[], vq_callback_t callbacks[],
393		− const char * const names[], const bool *ctx,
	401	+ const char * const names[], u32 sizes[], const bool *ctx,
394	402	struct irq_affinity *desc)
395	403	{
396	404	int err;
397	405
398	406	/* Try MSI-X with one vector per queue. */
399		− err = vp_find_vqs_msix(vdev, nvqs, vqs, callbacks, names, true, ctx, desc);
	407	+ err = vp_find_vqs_msix(vdev, nvqs, vqs, callbacks, names, sizes, true, ctx, desc);
400	408	if (!err)
401	409	return 0;
402	410	/* Fallback: MSI-X with one vector for config, one shared for queues. */
403		− err = vp_find_vqs_msix(vdev, nvqs, vqs, callbacks, names, false, ctx, desc);
	411	+ err = vp_find_vqs_msix(vdev, nvqs, vqs, callbacks, names, sizes, false, ctx, desc);
404	412	if (!err)
405	413	return 0;
406	414	/* Finally fall back to regular interrupts. */
407		− return vp_find_vqs_intx(vdev, nvqs, vqs, callbacks, names, ctx);
	415	+ return vp_find_vqs_intx(vdev, nvqs, vqs, callbacks, names, sizes, ctx);
408	416	}
409	417
410	418	const char vp_bus_name(struct virtio_device vdev)

drivers/virtio/virtio_pci_common.h+2−1

@@ -80,6 +80,7 @@ struct virtio_pci_device { · 79 unmodified lines
80	80	unsigned int idx,
81	81	void (callback)(struct virtqueue vq),
82	82	const char *name,
	83	+ u32 size,
83	84	bool ctx,
84	85	u16 msix_vec);
85	86	void (del_vq)(struct virtio_pci_vq_info info);
@@ -110,7 +111,7 @@ void vp_del_vqs(struct virtio_device *vdev); · 24 unmodified lines
110	111	/* the config->find_vqs() implementation */
111	112	int vp_find_vqs(struct virtio_device *vdev, unsigned int nvqs,
112	113	struct virtqueue vqs[], vq_callback_t callbacks[],
113		− const char * const names[], const bool *ctx,
	114	+ const char * const names[], u32 sizes[], const bool *ctx,
114	115	struct irq_affinity *desc);
115	116	const char vp_bus_name(struct virtio_device vdev);
116	117

drivers/virtio/virtio_pci_legacy.c+7−1

@@ -112,6 +112,7 @@ static struct virtqueue setup_vq(struct virtio_pci_device vp_dev, · 111 unmodified lines
112	112	unsigned int index,
113	113	void (callback)(struct virtqueue vq),
114	114	const char *name,
	115	+ u32 size,
115	116	bool ctx,
116	117	u16 msix_vec)
117	118	{
@@ -125,16 +126,21 @@ static struct virtqueue setup_vq(struct virtio_pci_device vp_dev, · 7 unmodified lines
125	126	if (!num \|\| vp_legacy_get_queue_enable(&vp_dev->ldev, index))
126	127	return ERR_PTR(-ENOENT);
127	128
	129	+ if (!size \|\| size > num)
	130	+ size = num;
	131	+
128	132	info->msix_vector = msix_vec;
129	133
130	134	/* create the vring */
131		− vq = vring_create_virtqueue(index, num,
	135	+ vq = vring_create_virtqueue(index, size,
132	136	VIRTIO_PCI_VRING_ALIGN, &vp_dev->vdev,
133	137	true, false, ctx,
134	138	vp_notify, callback, name);
135	139	if (!vq)
136	140	return ERR_PTR(-ENOMEM);
137	141
	142	+ vq->num_max = num;
	143	+
138	144	q_pfn = virtqueue_get_desc_addr(vq) >> VIRTIO_PCI_QUEUE_ADDR_SHIFT;
139	145	if (q_pfn >> 32) {
140	146	dev_err(&vp_dev->pci_dev->dev,

drivers/virtio/virtio_pci_modern.c+130−23

@@ -34,6 +34,9 @@ static void vp_transport_features(struct virtio_device *vdev, u64 features) · 33 unmodified lines
34	34	if ((features & BIT_ULL(VIRTIO_F_SR_IOV)) &&
35	35	pci_find_ext_capability(pci_dev, PCI_EXT_CAP_ID_SRIOV))
36	36	__virtio_set_bit(vdev, VIRTIO_F_SR_IOV);
	37	+
	38	+ if (features & BIT_ULL(VIRTIO_F_RING_RESET))
	39	+ __virtio_set_bit(vdev, VIRTIO_F_RING_RESET);
37	40	}
38	41
39	42	/* virtio config->finalize_features() implementation */
@@ -176,6 +179,110 @@ static void vp_reset(struct virtio_device *vdev) · 136 unmodified lines
176	179	vp_synchronize_vectors(vdev);
177	180	}
178	181
	182	+static int vp_active_vq(struct virtqueue *vq, u16 msix_vec)
	183	+{
	184	+ struct virtio_pci_device *vp_dev = to_vp_device(vq->vdev);
	185	+ struct virtio_pci_modern_device *mdev = &vp_dev->mdev;
	186	+ unsigned long index;
	187	+
	188	+ index = vq->index;
	189	+
	190	+ /* activate the queue */
	191	+ vp_modern_set_queue_size(mdev, index, virtqueue_get_vring_size(vq));
	192	+ vp_modern_queue_address(mdev, index, virtqueue_get_desc_addr(vq),
	193	+ virtqueue_get_avail_addr(vq),
	194	+ virtqueue_get_used_addr(vq));
	195	+
	196	+ if (msix_vec != VIRTIO_MSI_NO_VECTOR) {
	197	+ msix_vec = vp_modern_queue_vector(mdev, index, msix_vec);
	198	+ if (msix_vec == VIRTIO_MSI_NO_VECTOR)
	199	+ return -EBUSY;
	200	+ }
	201	+
	202	+ return 0;
	203	+}
	204	+
	205	+static int vp_modern_disable_vq_and_reset(struct virtqueue *vq)
	206	+{
	207	+ struct virtio_pci_device *vp_dev = to_vp_device(vq->vdev);
	208	+ struct virtio_pci_modern_device *mdev = &vp_dev->mdev;
	209	+ struct virtio_pci_vq_info *info;
	210	+ unsigned long flags;
	211	+
	212	+ if (!virtio_has_feature(vq->vdev, VIRTIO_F_RING_RESET))
	213	+ return -ENOENT;
	214	+
	215	+ vp_modern_set_queue_reset(mdev, vq->index);
	216	+
	217	+ info = vp_dev->vqs[vq->index];
	218	+
	219	+ /* delete vq from irq handler */
	220	+ spin_lock_irqsave(&vp_dev->lock, flags);
	221	+ list_del(&info->node);
	222	+ spin_unlock_irqrestore(&vp_dev->lock, flags);
	223	+
	224	+ INIT_LIST_HEAD(&info->node);
	225	+
	226	+#ifdef CONFIG_VIRTIO_HARDEN_NOTIFICATION
	227	+ __virtqueue_break(vq);
	228	+#endif
	229	+
	230	+ /* For the case where vq has an exclusive irq, call synchronize_irq() to
	231	+ * wait for completion.
	232	+ *
	233	+ * note: We can't use disable_irq() since it conflicts with the affinity
	234	+ * managed IRQ that is used by some drivers.
	235	+ */
	236	+ if (vp_dev->per_vq_vectors && info->msix_vector != VIRTIO_MSI_NO_VECTOR)
	237	+ synchronize_irq(pci_irq_vector(vp_dev->pci_dev, info->msix_vector));
	238	+
	239	+ vq->reset = true;
	240	+
	241	+ return 0;
	242	+}
	243	+
	244	+static int vp_modern_enable_vq_after_reset(struct virtqueue *vq)
	245	+{
	246	+ struct virtio_pci_device *vp_dev = to_vp_device(vq->vdev);
	247	+ struct virtio_pci_modern_device *mdev = &vp_dev->mdev;
	248	+ struct virtio_pci_vq_info *info;
	249	+ unsigned long flags, index;
	250	+ int err;
	251	+
	252	+ if (!vq->reset)
	253	+ return -EBUSY;
	254	+
	255	+ index = vq->index;
	256	+ info = vp_dev->vqs[index];
	257	+
	258	+ if (vp_modern_get_queue_reset(mdev, index))
	259	+ return -EBUSY;
	260	+
	261	+ if (vp_modern_get_queue_enable(mdev, index))
	262	+ return -EBUSY;
	263	+
	264	+ err = vp_active_vq(vq, info->msix_vector);
	265	+ if (err)
	266	+ return err;
	267	+
	268	+ if (vq->callback) {
	269	+ spin_lock_irqsave(&vp_dev->lock, flags);
	270	+ list_add(&info->node, &vp_dev->virtqueues);
	271	+ spin_unlock_irqrestore(&vp_dev->lock, flags);
	272	+ } else {
	273	+ INIT_LIST_HEAD(&info->node);
	274	+ }
	275	+
	276	+#ifdef CONFIG_VIRTIO_HARDEN_NOTIFICATION
	277	+ __virtqueue_unbreak(vq);
	278	+#endif
	279	+
	280	+ vp_modern_set_queue_enable(&vp_dev->mdev, index, true);
	281	+ vq->reset = false;
	282	+
	283	+ return 0;
	284	+}
	285	+
179	286	static u16 vp_config_vector(struct virtio_pci_device *vp_dev, u16 vector)
180	287	{
181	288	return vp_modern_config_vector(&vp_dev->mdev, vector);
@@ -186,6 +293,7 @@ static struct virtqueue setup_vq(struct virtio_pci_device vp_dev, · 4 unmodified lines
186	293	unsigned int index,
187	294	void (callback)(struct virtqueue vq),
188	295	const char *name,
	296	+ u32 size,
189	297	bool ctx,
190	298	u16 msix_vec)
191	299	{
@@ -203,47 +311,39 @@ static struct virtqueue setup_vq(struct virtio_pci_device vp_dev, · 11 unmodified lines
203	311	if (!num \|\| vp_modern_get_queue_enable(mdev, index))
204	312	return ERR_PTR(-ENOENT);
205	313
206		− if (num & (num - 1)) {
207		− dev_warn(&vp_dev->pci_dev->dev, "bad queue size %u", num);
	314	+ if (!size \|\| size > num)
	315	+ size = num;
	316	+
	317	+ if (size & (size - 1)) {
	318	+ dev_warn(&vp_dev->pci_dev->dev, "bad queue size %u", size);
208	319	return ERR_PTR(-EINVAL);
209	320	}
210	321
211	322	info->msix_vector = msix_vec;
212	323
213	324	/* create the vring */
214		− vq = vring_create_virtqueue(index, num,
	325	+ vq = vring_create_virtqueue(index, size,
215	326	SMP_CACHE_BYTES, &vp_dev->vdev,
216	327	true, true, ctx,
217	328	vp_notify, callback, name);
218	329	if (!vq)
219	330	return ERR_PTR(-ENOMEM);
220	331
221		− /* activate the queue */
222		− vp_modern_set_queue_size(mdev, index, virtqueue_get_vring_size(vq));
223		− vp_modern_queue_address(mdev, index, virtqueue_get_desc_addr(vq),
224		− virtqueue_get_avail_addr(vq),
225		− virtqueue_get_used_addr(vq));
	332	+ vq->num_max = num;
	333	+
	334	+ err = vp_active_vq(vq, msix_vec);
	335	+ if (err)
	336	+ goto err;
226	337
227	338	vq->priv = (void __force *)vp_modern_map_vq_notify(mdev, index, NULL);
228	339	if (!vq->priv) {
229	340	err = -ENOMEM;
230		− goto err_map_notify;
231		− }
232		−
233		− if (msix_vec != VIRTIO_MSI_NO_VECTOR) {
234		− msix_vec = vp_modern_queue_vector(mdev, index, msix_vec);
235		− if (msix_vec == VIRTIO_MSI_NO_VECTOR) {
236		− err = -EBUSY;
237		− goto err_assign_vector;
238		− }
	341	+ goto err;
239	342	}
240	343
241	344	return vq;
242	345
243		−err_assign_vector:
244		− if (!mdev->notify_base)
245		− pci_iounmap(mdev->pci_dev, (void __iomem __force *)vq->priv);
246		−err_map_notify:
	346	+err:
247	347	vring_del_virtqueue(vq);
248	348	return ERR_PTR(err);
249	349	}
@@ -251,12 +351,15 @@ err_map_notify: · 1 unmodified line
251	351	static int vp_modern_find_vqs(struct virtio_device *vdev, unsigned int nvqs,
252	352	struct virtqueue *vqs[],
253	353	vq_callback_t *callbacks[],
254		− const char * const names[], const bool *ctx,
	354	+ const char * const names[],
	355	+ u32 sizes[],
	356	+ const bool *ctx,
255	357	struct irq_affinity *desc)
256	358	{
257	359	struct virtio_pci_device *vp_dev = to_vp_device(vdev);
258	360	struct virtqueue *vq;
259		− int rc = vp_find_vqs(vdev, nvqs, vqs, callbacks, names, ctx, desc);
	361	+ int rc = vp_find_vqs(vdev, nvqs, vqs, callbacks, names, sizes, ctx,
	362	+ desc);
260	363
261	364	if (rc)
262	365	return rc;
@@ -401,6 +504,8 @@ static const struct virtio_config_ops virtio_pci_config_nodev_ops = { · 138 unmodified lines
401	504	.set_vq_affinity = vp_set_vq_affinity,
402	505	.get_vq_affinity = vp_get_vq_affinity,
403	506	.get_shm_region = vp_get_shm_region,
	507	+ .disable_vq_and_reset = vp_modern_disable_vq_and_reset,
	508	+ .enable_vq_after_reset = vp_modern_enable_vq_after_reset,
404	509	};
405	510
406	511	static const struct virtio_config_ops virtio_pci_config_ops = {
@@ -419,6 +524,8 @@ static const struct virtio_config_ops virtio_pci_config_ops = { · 12 unmodified lines
419	524	.set_vq_affinity = vp_set_vq_affinity,
420	525	.get_vq_affinity = vp_get_vq_affinity,
421	526	.get_shm_region = vp_get_shm_region,
	527	+ .disable_vq_and_reset = vp_modern_disable_vq_and_reset,
	528	+ .enable_vq_after_reset = vp_modern_enable_vq_after_reset,
422	529	};
423	530
424	531	/* the PCI probing function */

drivers/virtio/virtio_pci_modern_dev.c+39−0

@@ -3,6 +3,7 @@ · 2 unmodified lines
3	3	#include <linux/virtio_pci_modern.h>
4	4	#include <linux/module.h>
5	5	#include <linux/pci.h>
	6	+#include <linux/delay.h>
6	7
7	8	/*
8	9	* vp_modern_map_capability - map a part of virtio pci capability
@@ -474,6 +475,44 @@ void vp_modern_set_status(struct virtio_pci_modern_device *mdev, · 465 unmodified lines
474	475	}
475	476	EXPORT_SYMBOL_GPL(vp_modern_set_status);
476	477
	478	+/*
	479	+ * vp_modern_get_queue_reset - get the queue reset status
	480	+ * @mdev: the modern virtio-pci device
	481	+ * @index: queue index
	482	+ */
	483	+int vp_modern_get_queue_reset(struct virtio_pci_modern_device *mdev, u16 index)
	484	+{
	485	+ struct virtio_pci_modern_common_cfg __iomem *cfg;
	486	+
	487	+ cfg = (struct virtio_pci_modern_common_cfg __iomem *)mdev->common;
	488	+
	489	+ vp_iowrite16(index, &cfg->cfg.queue_select);
	490	+ return vp_ioread16(&cfg->queue_reset);
	491	+}
	492	+EXPORT_SYMBOL_GPL(vp_modern_get_queue_reset);
	493	+
	494	+/*
	495	+ * vp_modern_set_queue_reset - reset the queue
	496	+ * @mdev: the modern virtio-pci device
	497	+ * @index: queue index
	498	+ */
	499	+void vp_modern_set_queue_reset(struct virtio_pci_modern_device *mdev, u16 index)
	500	+{
	501	+ struct virtio_pci_modern_common_cfg __iomem *cfg;
	502	+
	503	+ cfg = (struct virtio_pci_modern_common_cfg __iomem *)mdev->common;
	504	+
	505	+ vp_iowrite16(index, &cfg->cfg.queue_select);
	506	+ vp_iowrite16(1, &cfg->queue_reset);
	507	+
	508	+ while (vp_ioread16(&cfg->queue_reset))
	509	+ msleep(1);
	510	+
	511	+ while (vp_ioread16(&cfg->cfg.queue_enable))
	512	+ msleep(1);
	513	+}
	514	+EXPORT_SYMBOL_GPL(vp_modern_set_queue_reset);
	515	+
477	516	/*
478	517	* vp_modern_queue_vector - set the MSIX vector for a specific virtqueue
479	518	* @mdev: the modern virtio-pci device

drivers/virtio/virtio_ring.c+559−211

@@ -85,6 +85,71 @@ struct vring_desc_extra { · 84 unmodified lines
85	85	u16 next; /* The next desc state in a list. */
86	86	};
87	87
	88	+struct vring_virtqueue_split {
	89	+ /* Actual memory layout for this queue. */
	90	+ struct vring vring;
	91	+
	92	+ /* Last written value to avail->flags */
	93	+ u16 avail_flags_shadow;
	94	+
	95	+ /*
	96	+ * Last written value to avail->idx in
	97	+ * guest byte order.
	98	+ */
	99	+ u16 avail_idx_shadow;
	100	+
	101	+ /* Per-descriptor state. */
	102	+ struct vring_desc_state_split *desc_state;
	103	+ struct vring_desc_extra *desc_extra;
	104	+
	105	+ /* DMA address and size information */
	106	+ dma_addr_t queue_dma_addr;
	107	+ size_t queue_size_in_bytes;
	108	+
	109	+ /*
	110	+ * The parameters for creating vrings are reserved for creating new
	111	+ * vring.
	112	+ */
	113	+ u32 vring_align;
	114	+ bool may_reduce_num;
	115	+};
	116	+
	117	+struct vring_virtqueue_packed {
	118	+ /* Actual memory layout for this queue. */
	119	+ struct {
	120	+ unsigned int num;
	121	+ struct vring_packed_desc *desc;
	122	+ struct vring_packed_desc_event *driver;
	123	+ struct vring_packed_desc_event *device;
	124	+ } vring;
	125	+
	126	+ /* Driver ring wrap counter. */
	127	+ bool avail_wrap_counter;
	128	+
	129	+ /* Avail used flags. */
	130	+ u16 avail_used_flags;
	131	+
	132	+ /* Index of the next avail descriptor. */
	133	+ u16 next_avail_idx;
	134	+
	135	+ /*
	136	+ * Last written value to driver->flags in
	137	+ * guest byte order.
	138	+ */
	139	+ u16 event_flags_shadow;
	140	+
	141	+ /* Per-descriptor state. */
	142	+ struct vring_desc_state_packed *desc_state;
	143	+ struct vring_desc_extra *desc_extra;
	144	+
	145	+ /* DMA address and size information */
	146	+ dma_addr_t ring_dma_addr;
	147	+ dma_addr_t driver_event_dma_addr;
	148	+ dma_addr_t device_event_dma_addr;
	149	+ size_t ring_size_in_bytes;
	150	+ size_t event_size_in_bytes;
	151	+};
	152	+
88	153	struct vring_virtqueue {
89	154	struct virtqueue vq;
90	155
@@ -124,64 +189,10 @@ struct vring_virtqueue { · 33 unmodified lines
124	189
125	190	union {
126	191	/* Available for split ring */
127		− struct {
128		− /* Actual memory layout for this queue. */
129		− struct vring vring;
130		−
131		− /* Last written value to avail->flags */
132		− u16 avail_flags_shadow;
133		−
134		− /*
135		− * Last written value to avail->idx in
136		− * guest byte order.
137		− */
138		− u16 avail_idx_shadow;
139		−
140		− /* Per-descriptor state. */
141		− struct vring_desc_state_split *desc_state;
142		− struct vring_desc_extra *desc_extra;
143		−
144		− /* DMA address and size information */
145		− dma_addr_t queue_dma_addr;
146		− size_t queue_size_in_bytes;
147		− } split;
	192	+ struct vring_virtqueue_split split;
148	193
149	194	/* Available for packed ring */
150		− struct {
151		− /* Actual memory layout for this queue. */
152		− struct {
153		− unsigned int num;
154		− struct vring_packed_desc *desc;
155		− struct vring_packed_desc_event *driver;
156		− struct vring_packed_desc_event *device;
157		− } vring;
158		−
159		− /* Driver ring wrap counter. */
160		− bool avail_wrap_counter;
161		−
162		− /* Avail used flags. */
163		− u16 avail_used_flags;
164		−
165		− /* Index of the next avail descriptor. */
166		− u16 next_avail_idx;
167		−
168		− /*
169		− * Last written value to driver->flags in
170		− * guest byte order.
171		− */
172		− u16 event_flags_shadow;
173		−
174		− /* Per-descriptor state. */
175		− struct vring_desc_state_packed *desc_state;
176		− struct vring_desc_extra *desc_extra;
177		−
178		− /* DMA address and size information */
179		− dma_addr_t ring_dma_addr;
180		− dma_addr_t driver_event_dma_addr;
181		− dma_addr_t device_event_dma_addr;
182		− size_t ring_size_in_bytes;
183		− size_t event_size_in_bytes;
184		− } packed;
	195	+ struct vring_virtqueue_packed packed;
185	196	};
186	197
187	198	/* How to notify other side. FIXME: commonalize hcalls! */
@@ -200,6 +211,16 @@ struct vring_virtqueue { · 12 unmodified lines
200	211	#endif
201	212	};
202	213
	214	+static struct virtqueue *__vring_new_virtqueue(unsigned int index,
	215	+ struct vring_virtqueue_split *vring_split,
	216	+ struct virtio_device *vdev,
	217	+ bool weak_barriers,
	218	+ bool context,
	219	+ bool (notify)(struct virtqueue ),
	220	+ void (callback)(struct virtqueue ),
	221	+ const char *name);
	222	+static struct vring_desc_extra *vring_alloc_desc_extra(unsigned int num);
	223	+static void vring_free(struct virtqueue *_vq);
203	224
204	225	/*
205	226	* Helpers.
@@ -364,6 +385,24 @@ static int vring_mapping_error(const struct vring_virtqueue *vq, · 158 unmodified lines
364	385	return dma_mapping_error(vring_dma_dev(vq), addr);
365	386	}
366	387
	388	+static void virtqueue_init(struct vring_virtqueue *vq, u32 num)
	389	+{
	390	+ vq->vq.num_free = num;
	391	+
	392	+ if (vq->packed_ring)
	393	+ vq->last_used_idx = 0 \| (1 << VRING_PACKED_EVENT_F_WRAP_CTR);
	394	+ else
	395	+ vq->last_used_idx = 0;
	396	+
	397	+ vq->event_triggered = false;
	398	+ vq->num_added = 0;
	399	+
	400	+#ifdef DEBUG
	401	+ vq->in_use = false;
	402	+ vq->last_add_time_valid = false;
	403	+#endif
	404	+}
	405	+
367	406
368	407	/*
369	408	* Split ring specific functions - *_split().
@@ -907,28 +946,107 @@ static void virtqueue_detach_unused_buf_split(struct virtqueue _vq) · 537 unmodified lines
907	946	return NULL;
908	947	}
909	948
910		−static struct virtqueue *vring_create_virtqueue_split(
911		− unsigned int index,
912		− unsigned int num,
913		− unsigned int vring_align,
914		− struct virtio_device *vdev,
915		− bool weak_barriers,
916		− bool may_reduce_num,
917		− bool context,
918		− bool (notify)(struct virtqueue ),
919		− void (callback)(struct virtqueue ),
920		− const char *name)
	949	+static void virtqueue_vring_init_split(struct vring_virtqueue_split *vring_split,
	950	+ struct vring_virtqueue *vq)
	951	+{
	952	+ struct virtio_device *vdev;
	953	+
	954	+ vdev = vq->vq.vdev;
	955	+
	956	+ vring_split->avail_flags_shadow = 0;
	957	+ vring_split->avail_idx_shadow = 0;
	958	+
	959	+ /* No callback? Tell other side not to bother us. */
	960	+ if (!vq->vq.callback) {
	961	+ vring_split->avail_flags_shadow \|= VRING_AVAIL_F_NO_INTERRUPT;
	962	+ if (!vq->event)
	963	+ vring_split->vring.avail->flags = cpu_to_virtio16(vdev,
	964	+ vring_split->avail_flags_shadow);
	965	+ }
	966	+}
	967	+
	968	+static void virtqueue_reinit_split(struct vring_virtqueue *vq)
	969	+{
	970	+ int num;
	971	+
	972	+ num = vq->split.vring.num;
	973	+
	974	+ vq->split.vring.avail->flags = 0;
	975	+ vq->split.vring.avail->idx = 0;
	976	+
	977	+ /* reset avail event */
	978	+ vq->split.vring.avail->ring[num] = 0;
	979	+
	980	+ vq->split.vring.used->flags = 0;
	981	+ vq->split.vring.used->idx = 0;
	982	+
	983	+ /* reset used event */
	984	+ (__virtio16 )&(vq->split.vring.used->ring[num]) = 0;
	985	+
	986	+ virtqueue_init(vq, num);
	987	+
	988	+ virtqueue_vring_init_split(&vq->split, vq);
	989	+}
	990	+
	991	+static void virtqueue_vring_attach_split(struct vring_virtqueue *vq,
	992	+ struct vring_virtqueue_split *vring_split)
	993	+{
	994	+ vq->split = *vring_split;
	995	+
	996	+ /* Put everything in free lists. */
	997	+ vq->free_head = 0;
	998	+}
	999	+
	1000	+static int vring_alloc_state_extra_split(struct vring_virtqueue_split *vring_split)
	1001	+{
	1002	+ struct vring_desc_state_split *state;
	1003	+ struct vring_desc_extra *extra;
	1004	+ u32 num = vring_split->vring.num;
	1005	+
	1006	+ state = kmalloc_array(num, sizeof(struct vring_desc_state_split), GFP_KERNEL);
	1007	+ if (!state)
	1008	+ goto err_state;
	1009	+
	1010	+ extra = vring_alloc_desc_extra(num);
	1011	+ if (!extra)
	1012	+ goto err_extra;
	1013	+
	1014	+ memset(state, 0, num * sizeof(struct vring_desc_state_split));
	1015	+
	1016	+ vring_split->desc_state = state;
	1017	+ vring_split->desc_extra = extra;
	1018	+ return 0;
	1019	+
	1020	+err_extra:
	1021	+ kfree(state);
	1022	+err_state:
	1023	+ return -ENOMEM;
	1024	+}
	1025	+
	1026	+static void vring_free_split(struct vring_virtqueue_split *vring_split,
	1027	+ struct virtio_device *vdev)
	1028	+{
	1029	+ vring_free_queue(vdev, vring_split->queue_size_in_bytes,
	1030	+ vring_split->vring.desc,
	1031	+ vring_split->queue_dma_addr);
	1032	+
	1033	+ kfree(vring_split->desc_state);
	1034	+ kfree(vring_split->desc_extra);
	1035	+}
	1036	+
	1037	+static int vring_alloc_queue_split(struct vring_virtqueue_split *vring_split,
	1038	+ struct virtio_device *vdev,
	1039	+ u32 num,
	1040	+ unsigned int vring_align,
	1041	+ bool may_reduce_num)
921	1042	{
922		− struct virtqueue *vq;
923	1043	void *queue = NULL;
924	1044	dma_addr_t dma_addr;
925		− size_t queue_size_in_bytes;
926		− struct vring vring;
927	1045
928	1046	/* We assume num is a power of 2. */
929	1047	if (num & (num - 1)) {
930	1048	dev_warn(&vdev->dev, "Bad virtqueue length %u\n", num);
931		− return NULL;
	1049	+ return -EINVAL;
932	1050	}
933	1051
934	1052	/* TODO: allocate each queue chunk individually */
@@ -939,11 +1057,11 @@ static struct virtqueue *vring_create_virtqueue_split( · 4 unmodified lines
939	1057	if (queue)
940	1058	break;
941	1059	if (!may_reduce_num)
942		− return NULL;
	1060	+ return -ENOMEM;
943	1061	}
944	1062
945	1063	if (!num)
946		− return NULL;
	1064	+ return -ENOMEM;
947	1065
948	1066	if (!queue) {
949	1067	/* Try to get a single page. You are my only hope! */
@@ -951,26 +1069,85 @@ static struct virtqueue *vring_create_virtqueue_split( · 1 unmodified line
951	1069	&dma_addr, GFP_KERNEL\|__GFP_ZERO);
952	1070	}
953	1071	if (!queue)
954		− return NULL;
	1072	+ return -ENOMEM;
	1073	+
	1074	+ vring_init(&vring_split->vring, num, queue, vring_align);
	1075	+
	1076	+ vring_split->queue_dma_addr = dma_addr;
	1077	+ vring_split->queue_size_in_bytes = vring_size(num, vring_align);
955	1078
956		− queue_size_in_bytes = vring_size(num, vring_align);
957		− vring_init(&vring, num, queue, vring_align);
	1079	+ vring_split->vring_align = vring_align;
	1080	+ vring_split->may_reduce_num = may_reduce_num;
958	1081
959		− vq = __vring_new_virtqueue(index, vring, vdev, weak_barriers, context,
960		− notify, callback, name);
	1082	+ return 0;
	1083	+}
	1084	+
	1085	+static struct virtqueue *vring_create_virtqueue_split(
	1086	+ unsigned int index,
	1087	+ unsigned int num,
	1088	+ unsigned int vring_align,
	1089	+ struct virtio_device *vdev,
	1090	+ bool weak_barriers,
	1091	+ bool may_reduce_num,
	1092	+ bool context,
	1093	+ bool (notify)(struct virtqueue ),
	1094	+ void (callback)(struct virtqueue ),
	1095	+ const char *name)
	1096	+{
	1097	+ struct vring_virtqueue_split vring_split = {};
	1098	+ struct virtqueue *vq;
	1099	+ int err;
	1100	+
	1101	+ err = vring_alloc_queue_split(&vring_split, vdev, num, vring_align,
	1102	+ may_reduce_num);
	1103	+ if (err)
	1104	+ return NULL;
	1105	+
	1106	+ vq = __vring_new_virtqueue(index, &vring_split, vdev, weak_barriers,
	1107	+ context, notify, callback, name);
961	1108	if (!vq) {
962		− vring_free_queue(vdev, queue_size_in_bytes, queue,
963		− dma_addr);
	1109	+ vring_free_split(&vring_split, vdev);
964	1110	return NULL;
965	1111	}
966	1112
967		− to_vvq(vq)->split.queue_dma_addr = dma_addr;
968		− to_vvq(vq)->split.queue_size_in_bytes = queue_size_in_bytes;
969	1113	to_vvq(vq)->we_own_ring = true;
970	1114
971	1115	return vq;
972	1116	}
973	1117
	1118	+static int virtqueue_resize_split(struct virtqueue *_vq, u32 num)
	1119	+{
	1120	+ struct vring_virtqueue_split vring_split = {};
	1121	+ struct vring_virtqueue *vq = to_vvq(_vq);
	1122	+ struct virtio_device *vdev = _vq->vdev;
	1123	+ int err;
	1124	+
	1125	+ err = vring_alloc_queue_split(&vring_split, vdev, num,
	1126	+ vq->split.vring_align,
	1127	+ vq->split.may_reduce_num);
	1128	+ if (err)
	1129	+ goto err;
	1130	+
	1131	+ err = vring_alloc_state_extra_split(&vring_split);
	1132	+ if (err)
	1133	+ goto err_state_extra;
	1134	+
	1135	+ vring_free(&vq->vq);
	1136	+
	1137	+ virtqueue_vring_init_split(&vring_split, vq);
	1138	+
	1139	+ virtqueue_init(vq, vring_split.vring.num);
	1140	+ virtqueue_vring_attach_split(vq, &vring_split);
	1141	+
	1142	+ return 0;
	1143	+
	1144	+err_state_extra:
	1145	+ vring_free_split(&vring_split, vdev);
	1146	+err:
	1147	+ virtqueue_reinit_split(vq);
	1148	+ return -ENOMEM;
	1149	+}
	1150	+
974	1151
975	1152	/*
976	1153	* Packed ring specific functions - *_packed().
@@ -1637,8 +1814,7 @@ static void virtqueue_detach_unused_buf_packed(struct virtqueue _vq) · 660 unmodified lines
1637	1814	return NULL;
1638	1815	}
1639	1816
1640		−static struct vring_desc_extra vring_alloc_desc_extra(struct vring_virtqueue vq,
1641		− unsigned int num)
	1817	+static struct vring_desc_extra *vring_alloc_desc_extra(unsigned int num)
1642	1818	{
1643	1819	struct vring_desc_extra *desc_extra;
1644	1820	unsigned int i;
@@ -1656,19 +1832,32 @@ static struct vring_desc_extra vring_alloc_desc_extra(struct vring_virtqueue v · 11 unmodified lines
1656	1832	return desc_extra;
1657	1833	}
1658	1834
1659		−static struct virtqueue *vring_create_virtqueue_packed(
1660		− unsigned int index,
1661		− unsigned int num,
1662		− unsigned int vring_align,
1663		− struct virtio_device *vdev,
1664		− bool weak_barriers,
1665		− bool may_reduce_num,
1666		− bool context,
1667		− bool (notify)(struct virtqueue ),
1668		− void (callback)(struct virtqueue ),
1669		− const char *name)
	1835	+static void vring_free_packed(struct vring_virtqueue_packed *vring_packed,
	1836	+ struct virtio_device *vdev)
	1837	+{
	1838	+ if (vring_packed->vring.desc)
	1839	+ vring_free_queue(vdev, vring_packed->ring_size_in_bytes,
	1840	+ vring_packed->vring.desc,
	1841	+ vring_packed->ring_dma_addr);
	1842	+
	1843	+ if (vring_packed->vring.driver)
	1844	+ vring_free_queue(vdev, vring_packed->event_size_in_bytes,
	1845	+ vring_packed->vring.driver,
	1846	+ vring_packed->driver_event_dma_addr);
	1847	+
	1848	+ if (vring_packed->vring.device)
	1849	+ vring_free_queue(vdev, vring_packed->event_size_in_bytes,
	1850	+ vring_packed->vring.device,
	1851	+ vring_packed->device_event_dma_addr);
	1852	+
	1853	+ kfree(vring_packed->desc_state);
	1854	+ kfree(vring_packed->desc_extra);
	1855	+}
	1856	+
	1857	+static int vring_alloc_queue_packed(struct vring_virtqueue_packed *vring_packed,
	1858	+ struct virtio_device *vdev,
	1859	+ u32 num)
1670	1860	{
1671		− struct vring_virtqueue *vq;
1672	1861	struct vring_packed_desc *ring;
1673	1862	struct vring_packed_desc_event driver, device;
1674	1863	dma_addr_t ring_dma_addr, driver_event_dma_addr, device_event_dma_addr;
@@ -1680,7 +1869,11 @@ static struct virtqueue *vring_create_virtqueue_packed( · 5 unmodified lines
1680	1869	&ring_dma_addr,
1681	1870	GFP_KERNEL\|__GFP_NOWARN\|__GFP_ZERO);
1682	1871	if (!ring)
1683		− goto err_ring;
	1872	+ goto err;
	1873	+
	1874	+ vring_packed->vring.desc = ring;
	1875	+ vring_packed->ring_dma_addr = ring_dma_addr;
	1876	+ vring_packed->ring_size_in_bytes = ring_size_in_bytes;
1684	1877
1685	1878	event_size_in_bytes = sizeof(struct vring_packed_desc_event);
1686	1879
@@ -1688,13 +1881,112 @@ static struct virtqueue *vring_create_virtqueue_packed( · 1 unmodified line
1688	1881	&driver_event_dma_addr,
1689	1882	GFP_KERNEL\|__GFP_NOWARN\|__GFP_ZERO);
1690	1883	if (!driver)
1691		− goto err_driver;
	1884	+ goto err;
	1885	+
	1886	+ vring_packed->vring.driver = driver;
	1887	+ vring_packed->event_size_in_bytes = event_size_in_bytes;
	1888	+ vring_packed->driver_event_dma_addr = driver_event_dma_addr;
1692	1889
1693	1890	device = vring_alloc_queue(vdev, event_size_in_bytes,
1694	1891	&device_event_dma_addr,
1695	1892	GFP_KERNEL\|__GFP_NOWARN\|__GFP_ZERO);
1696	1893	if (!device)
1697		− goto err_device;
	1894	+ goto err;
	1895	+
	1896	+ vring_packed->vring.device = device;
	1897	+ vring_packed->device_event_dma_addr = device_event_dma_addr;
	1898	+
	1899	+ vring_packed->vring.num = num;
	1900	+
	1901	+ return 0;
	1902	+
	1903	+err:
	1904	+ vring_free_packed(vring_packed, vdev);
	1905	+ return -ENOMEM;
	1906	+}
	1907	+
	1908	+static int vring_alloc_state_extra_packed(struct vring_virtqueue_packed *vring_packed)
	1909	+{
	1910	+ struct vring_desc_state_packed *state;
	1911	+ struct vring_desc_extra *extra;
	1912	+ u32 num = vring_packed->vring.num;
	1913	+
	1914	+ state = kmalloc_array(num, sizeof(struct vring_desc_state_packed), GFP_KERNEL);
	1915	+ if (!state)
	1916	+ goto err_desc_state;
	1917	+
	1918	+ memset(state, 0, num * sizeof(struct vring_desc_state_packed));
	1919	+
	1920	+ extra = vring_alloc_desc_extra(num);
	1921	+ if (!extra)
	1922	+ goto err_desc_extra;
	1923	+
	1924	+ vring_packed->desc_state = state;
	1925	+ vring_packed->desc_extra = extra;
	1926	+
	1927	+ return 0;
	1928	+
	1929	+err_desc_extra:
	1930	+ kfree(state);
	1931	+err_desc_state:
	1932	+ return -ENOMEM;
	1933	+}
	1934	+
	1935	+static void virtqueue_vring_init_packed(struct vring_virtqueue_packed *vring_packed,
	1936	+ bool callback)
	1937	+{
	1938	+ vring_packed->next_avail_idx = 0;
	1939	+ vring_packed->avail_wrap_counter = 1;
	1940	+ vring_packed->event_flags_shadow = 0;
	1941	+ vring_packed->avail_used_flags = 1 << VRING_PACKED_DESC_F_AVAIL;
	1942	+
	1943	+ /* No callback? Tell other side not to bother us. */
	1944	+ if (!callback) {
	1945	+ vring_packed->event_flags_shadow = VRING_PACKED_EVENT_FLAG_DISABLE;
	1946	+ vring_packed->vring.driver->flags =
	1947	+ cpu_to_le16(vring_packed->event_flags_shadow);
	1948	+ }
	1949	+}
	1950	+
	1951	+static void virtqueue_vring_attach_packed(struct vring_virtqueue *vq,
	1952	+ struct vring_virtqueue_packed *vring_packed)
	1953	+{
	1954	+ vq->packed = *vring_packed;
	1955	+
	1956	+ /* Put everything in free lists. */
	1957	+ vq->free_head = 0;
	1958	+}
	1959	+
	1960	+static void virtqueue_reinit_packed(struct vring_virtqueue *vq)
	1961	+{
	1962	+ memset(vq->packed.vring.device, 0, vq->packed.event_size_in_bytes);
	1963	+ memset(vq->packed.vring.driver, 0, vq->packed.event_size_in_bytes);
	1964	+
	1965	+ /* we need to reset the desc.flags. For more, see is_used_desc_packed() */
	1966	+ memset(vq->packed.vring.desc, 0, vq->packed.ring_size_in_bytes);
	1967	+
	1968	+ virtqueue_init(vq, vq->packed.vring.num);
	1969	+ virtqueue_vring_init_packed(&vq->packed, !!vq->vq.callback);
	1970	+}
	1971	+
	1972	+static struct virtqueue *vring_create_virtqueue_packed(
	1973	+ unsigned int index,
	1974	+ unsigned int num,
	1975	+ unsigned int vring_align,
	1976	+ struct virtio_device *vdev,
	1977	+ bool weak_barriers,
	1978	+ bool may_reduce_num,
	1979	+ bool context,
	1980	+ bool (notify)(struct virtqueue ),
	1981	+ void (callback)(struct virtqueue ),
	1982	+ const char *name)
	1983	+{
	1984	+ struct vring_virtqueue_packed vring_packed = {};
	1985	+ struct vring_virtqueue *vq;
	1986	+ int err;
	1987	+
	1988	+ if (vring_alloc_queue_packed(&vring_packed, vdev, num))
	1989	+ goto err_ring;
1698	1990
1699	1991	vq = kmalloc(sizeof(*vq), GFP_KERNEL);
1700	1992	if (!vq)
@@ -1703,8 +1995,8 @@ static struct virtqueue *vring_create_virtqueue_packed( · 2 unmodified lines
1703	1995	vq->vq.callback = callback;
1704	1996	vq->vq.vdev = vdev;
1705	1997	vq->vq.name = name;
1706		− vq->vq.num_free = num;
1707	1998	vq->vq.index = index;
	1999	+ vq->vq.reset = false;
1708	2000	vq->we_own_ring = true;
1709	2001	vq->notify = notify;
1710	2002	vq->weak_barriers = weak_barriers;
@@ -1713,15 +2005,8 @@ static struct virtqueue *vring_create_virtqueue_packed( · 2 unmodified lines
1713	2005	#else
1714	2006	vq->broken = false;
1715	2007	#endif
1716		− vq->last_used_idx = 0 \| (1 << VRING_PACKED_EVENT_F_WRAP_CTR);
1717		− vq->event_triggered = false;
1718		− vq->num_added = 0;
1719	2008	vq->packed_ring = true;
1720	2009	vq->use_dma_api = vring_use_dma_api(vdev);
1721		−#ifdef DEBUG
1722		− vq->in_use = false;
1723		− vq->last_add_time_valid = false;
1724		−#endif
1725	2010
1726	2011	vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC) &&
1727	2012	!context;
@@ -1730,65 +2015,58 @@ static struct virtqueue *vring_create_virtqueue_packed( · 2 unmodified lines
1730	2015	if (virtio_has_feature(vdev, VIRTIO_F_ORDER_PLATFORM))
1731	2016	vq->weak_barriers = false;
1732	2017
1733		− vq->packed.ring_dma_addr = ring_dma_addr;
1734		− vq->packed.driver_event_dma_addr = driver_event_dma_addr;
1735		− vq->packed.device_event_dma_addr = device_event_dma_addr;
1736		−
1737		− vq->packed.ring_size_in_bytes = ring_size_in_bytes;
1738		− vq->packed.event_size_in_bytes = event_size_in_bytes;
1739		−
1740		− vq->packed.vring.num = num;
1741		− vq->packed.vring.desc = ring;
1742		− vq->packed.vring.driver = driver;
1743		− vq->packed.vring.device = device;
1744		−
1745		− vq->packed.next_avail_idx = 0;
1746		− vq->packed.avail_wrap_counter = 1;
1747		− vq->packed.event_flags_shadow = 0;
1748		− vq->packed.avail_used_flags = 1 << VRING_PACKED_DESC_F_AVAIL;
1749		−
1750		− vq->packed.desc_state = kmalloc_array(num,
1751		− sizeof(struct vring_desc_state_packed),
1752		− GFP_KERNEL);
1753		− if (!vq->packed.desc_state)
1754		− goto err_desc_state;
1755		−
1756		− memset(vq->packed.desc_state, 0,
1757		− num * sizeof(struct vring_desc_state_packed));
1758		−
1759		− /* Put everything in free lists. */
1760		− vq->free_head = 0;
	2018	+ err = vring_alloc_state_extra_packed(&vring_packed);
	2019	+ if (err)
	2020	+ goto err_state_extra;
1761	2021
1762		− vq->packed.desc_extra = vring_alloc_desc_extra(vq, num);
1763		− if (!vq->packed.desc_extra)
1764		− goto err_desc_extra;
	2022	+ virtqueue_vring_init_packed(&vring_packed, !!callback);
1765	2023
1766		− /* No callback? Tell other side not to bother us. */
1767		− if (!callback) {
1768		− vq->packed.event_flags_shadow = VRING_PACKED_EVENT_FLAG_DISABLE;
1769		− vq->packed.vring.driver->flags =
1770		− cpu_to_le16(vq->packed.event_flags_shadow);
1771		− }
	2024	+ virtqueue_init(vq, num);
	2025	+ virtqueue_vring_attach_packed(vq, &vring_packed);
1772	2026
1773	2027	spin_lock(&vdev->vqs_list_lock);
1774	2028	list_add_tail(&vq->vq.list, &vdev->vqs);
1775	2029	spin_unlock(&vdev->vqs_list_lock);
1776	2030	return &vq->vq;
1777	2031
1778		−err_desc_extra:
1779		− kfree(vq->packed.desc_state);
1780		−err_desc_state:
	2032	+err_state_extra:
1781	2033	kfree(vq);
1782	2034	err_vq:
1783		− vring_free_queue(vdev, event_size_in_bytes, device, device_event_dma_addr);
1784		−err_device:
1785		− vring_free_queue(vdev, event_size_in_bytes, driver, driver_event_dma_addr);
1786		−err_driver:
1787		− vring_free_queue(vdev, ring_size_in_bytes, ring, ring_dma_addr);
	2035	+ vring_free_packed(&vring_packed, vdev);
1788	2036	err_ring:
1789	2037	return NULL;
1790	2038	}
1791	2039
	2040	+static int virtqueue_resize_packed(struct virtqueue *_vq, u32 num)
	2041	+{
	2042	+ struct vring_virtqueue_packed vring_packed = {};
	2043	+ struct vring_virtqueue *vq = to_vvq(_vq);
	2044	+ struct virtio_device *vdev = _vq->vdev;
	2045	+ int err;
	2046	+
	2047	+ if (vring_alloc_queue_packed(&vring_packed, vdev, num))
	2048	+ goto err_ring;
	2049	+
	2050	+ err = vring_alloc_state_extra_packed(&vring_packed);
	2051	+ if (err)
	2052	+ goto err_state_extra;
	2053	+
	2054	+ vring_free(&vq->vq);
	2055	+
	2056	+ virtqueue_vring_init_packed(&vring_packed, !!vq->vq.callback);
	2057	+
	2058	+ virtqueue_init(vq, vring_packed.vring.num);
	2059	+ virtqueue_vring_attach_packed(vq, &vring_packed);
	2060	+
	2061	+ return 0;
	2062	+
	2063	+err_state_extra:
	2064	+ vring_free_packed(&vring_packed, vdev);
	2065	+err_ring:
	2066	+ virtqueue_reinit_packed(vq);
	2067	+ return -ENOMEM;
	2068	+}
	2069	+
1792	2070
1793	2071	/*
1794	2072	* Generic functions and exported symbols.
@@ -2131,8 +2409,8 @@ EXPORT_SYMBOL_GPL(virtqueue_enable_cb_delayed); · 336 unmodified lines
2131	2409	* @_vq: the struct virtqueue we're talking about.
2132	2410	*
2133	2411	* Returns NULL or the "data" token handed to virtqueue_add_*().
2134		− * This is not valid on an active queue; it is useful only for device
2135		− * shutdown.
	2412	+ * This is not valid on an active queue; it is useful for device
	2413	+ * shutdown or the reset queue.
2136	2414	*/
2137	2415	void virtqueue_detach_unused_buf(struct virtqueue _vq)
2138	2416	{
@@ -2180,16 +2458,17 @@ irqreturn_t vring_interrupt(int irq, void *_vq) · 41 unmodified lines
2180	2458	EXPORT_SYMBOL_GPL(vring_interrupt);
2181	2459
2182	2460	/* Only available for split ring */
2183		−struct virtqueue *__vring_new_virtqueue(unsigned int index,
2184		− struct vring vring,
2185		− struct virtio_device *vdev,
2186		− bool weak_barriers,
2187		− bool context,
2188		− bool (notify)(struct virtqueue ),
2189		− void (callback)(struct virtqueue ),
2190		− const char *name)
	2461	+static struct virtqueue *__vring_new_virtqueue(unsigned int index,
	2462	+ struct vring_virtqueue_split *vring_split,
	2463	+ struct virtio_device *vdev,
	2464	+ bool weak_barriers,
	2465	+ bool context,
	2466	+ bool (notify)(struct virtqueue ),
	2467	+ void (callback)(struct virtqueue ),
	2468	+ const char *name)
2191	2469	{
2192	2470	struct vring_virtqueue *vq;
	2471	+ int err;
2193	2472
2194	2473	if (virtio_has_feature(vdev, VIRTIO_F_RING_PACKED))
2195	2474	return NULL;
@@ -2202,8 +2481,8 @@ struct virtqueue *__vring_new_virtqueue(unsigned int index, · 6 unmodified lines
2202	2481	vq->vq.callback = callback;
2203	2482	vq->vq.vdev = vdev;
2204	2483	vq->vq.name = name;
2205		− vq->vq.num_free = vring.num;
2206	2484	vq->vq.index = index;
	2485	+ vq->vq.reset = false;
2207	2486	vq->we_own_ring = false;
2208	2487	vq->notify = notify;
2209	2488	vq->weak_barriers = weak_barriers;
@@ -2212,14 +2491,7 @@ struct virtqueue *__vring_new_virtqueue(unsigned int index, · 2 unmodified lines
2212	2491	#else
2213	2492	vq->broken = false;
2214	2493	#endif
2215		− vq->last_used_idx = 0;
2216		− vq->event_triggered = false;
2217		− vq->num_added = 0;
2218	2494	vq->use_dma_api = vring_use_dma_api(vdev);
2219		−#ifdef DEBUG
2220		− vq->in_use = false;
2221		− vq->last_add_time_valid = false;
2222		−#endif
2223	2495
2224	2496	vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC) &&
2225	2497	!context;
@@ -2228,47 +2500,22 @@ struct virtqueue *__vring_new_virtqueue(unsigned int index, · 2 unmodified lines
2228	2500	if (virtio_has_feature(vdev, VIRTIO_F_ORDER_PLATFORM))
2229	2501	vq->weak_barriers = false;
2230	2502
2231		− vq->split.queue_dma_addr = 0;
2232		− vq->split.queue_size_in_bytes = 0;
2233		−
2234		− vq->split.vring = vring;
2235		− vq->split.avail_flags_shadow = 0;
2236		− vq->split.avail_idx_shadow = 0;
2237		−
2238		− /* No callback? Tell other side not to bother us. */
2239		− if (!callback) {
2240		− vq->split.avail_flags_shadow \|= VRING_AVAIL_F_NO_INTERRUPT;
2241		− if (!vq->event)
2242		− vq->split.vring.avail->flags = cpu_to_virtio16(vdev,
2243		− vq->split.avail_flags_shadow);
	2503	+ err = vring_alloc_state_extra_split(vring_split);
	2504	+ if (err) {
	2505	+ kfree(vq);
	2506	+ return NULL;
2244	2507	}
2245	2508
2246		− vq->split.desc_state = kmalloc_array(vring.num,
2247		− sizeof(struct vring_desc_state_split), GFP_KERNEL);
2248		− if (!vq->split.desc_state)
2249		− goto err_state;
2250		−
2251		− vq->split.desc_extra = vring_alloc_desc_extra(vq, vring.num);
2252		− if (!vq->split.desc_extra)
2253		− goto err_extra;
	2509	+ virtqueue_vring_init_split(vring_split, vq);
2254	2510
2255		− /* Put everything in free lists. */
2256		− vq->free_head = 0;
2257		− memset(vq->split.desc_state, 0, vring.num *
2258		− sizeof(struct vring_desc_state_split));
	2511	+ virtqueue_init(vq, vring_split->vring.num);
	2512	+ virtqueue_vring_attach_split(vq, vring_split);
2259	2513
2260	2514	spin_lock(&vdev->vqs_list_lock);
2261	2515	list_add_tail(&vq->vq.list, &vdev->vqs);
2262	2516	spin_unlock(&vdev->vqs_list_lock);
2263	2517	return &vq->vq;
2264		−
2265		−err_extra:
2266		− kfree(vq->split.desc_state);
2267		−err_state:
2268		− kfree(vq);
2269		− return NULL;
2270	2518	}
2271		−EXPORT_SYMBOL_GPL(__vring_new_virtqueue);
2272	2519
2273	2520	struct virtqueue *vring_create_virtqueue(
2274	2521	unsigned int index,
@@ -2294,6 +2541,75 @@ struct virtqueue *vring_create_virtqueue( · 19 unmodified lines
2294	2541	}
2295	2542	EXPORT_SYMBOL_GPL(vring_create_virtqueue);
2296	2543
	2544	+/**
	2545	+ * virtqueue_resize - resize the vring of vq
	2546	+ * @_vq: the struct virtqueue we're talking about.
	2547	+ * @num: new ring num
	2548	+ * @recycle: callback for recycle the useless buffer
	2549	+ *
	2550	+ * When it is really necessary to create a new vring, it will set the current vq
	2551	+ * into the reset state. Then call the passed callback to recycle the buffer
	2552	+ * that is no longer used. Only after the new vring is successfully created, the
	2553	+ * old vring will be released.
	2554	+ *
	2555	+ * Caller must ensure we don't call this with other virtqueue operations
	2556	+ * at the same time (except where noted).
	2557	+ *
	2558	+ * Returns zero or a negative error.
	2559	+ * 0: success.
	2560	+ * -ENOMEM: Failed to allocate a new ring, fall back to the original ring size.
	2561	+ * vq can still work normally
	2562	+ * -EBUSY: Failed to sync with device, vq may not work properly
	2563	+ * -ENOENT: Transport or device not supported
	2564	+ * -E2BIG/-EINVAL: num error
	2565	+ * -EPERM: Operation not permitted
	2566	+ *
	2567	+ */
	2568	+int virtqueue_resize(struct virtqueue *_vq, u32 num,
	2569	+ void (recycle)(struct virtqueue vq, void *buf))
	2570	+{
	2571	+ struct vring_virtqueue *vq = to_vvq(_vq);
	2572	+ struct virtio_device *vdev = vq->vq.vdev;
	2573	+ void *buf;
	2574	+ int err;
	2575	+
	2576	+ if (!vq->we_own_ring)
	2577	+ return -EPERM;
	2578	+
	2579	+ if (num > vq->vq.num_max)
	2580	+ return -E2BIG;
	2581	+
	2582	+ if (!num)
	2583	+ return -EINVAL;
	2584	+
	2585	+ if ((vq->packed_ring ? vq->packed.vring.num : vq->split.vring.num) == num)
	2586	+ return 0;
	2587	+
	2588	+ if (!vdev->config->disable_vq_and_reset)
	2589	+ return -ENOENT;
	2590	+
	2591	+ if (!vdev->config->enable_vq_after_reset)
	2592	+ return -ENOENT;
	2593	+
	2594	+ err = vdev->config->disable_vq_and_reset(_vq);
	2595	+ if (err)
	2596	+ return err;
	2597	+
	2598	+ while ((buf = virtqueue_detach_unused_buf(_vq)) != NULL)
	2599	+ recycle(_vq, buf);
	2600	+
	2601	+ if (vq->packed_ring)
	2602	+ err = virtqueue_resize_packed(_vq, num);
	2603	+ else
	2604	+ err = virtqueue_resize_split(_vq, num);
	2605	+
	2606	+ if (vdev->config->enable_vq_after_reset(_vq))
	2607	+ return -EBUSY;
	2608	+
	2609	+ return err;
	2610	+}
	2611	+EXPORT_SYMBOL_GPL(virtqueue_resize);
	2612	+
2297	2613	/* Only available for split ring */
2298	2614	struct virtqueue *vring_new_virtqueue(unsigned int index,
2299	2615	unsigned int num,
@@ -2306,25 +2622,21 @@ struct virtqueue *vring_new_virtqueue(unsigned int index, · 6 unmodified lines
2306	2622	void (callback)(struct virtqueue vq),
2307	2623	const char *name)
2308	2624	{
2309		− struct vring vring;
	2625	+ struct vring_virtqueue_split vring_split = {};
2310	2626
2311	2627	if (virtio_has_feature(vdev, VIRTIO_F_RING_PACKED))
2312	2628	return NULL;
2313	2629
2314		− vring_init(&vring, num, pages, vring_align);
2315		− return __vring_new_virtqueue(index, vring, vdev, weak_barriers, context,
2316		− notify, callback, name);
	2630	+ vring_init(&vring_split.vring, num, pages, vring_align);
	2631	+ return __vring_new_virtqueue(index, &vring_split, vdev, weak_barriers,
	2632	+ context, notify, callback, name);
2317	2633	}
2318	2634	EXPORT_SYMBOL_GPL(vring_new_virtqueue);
2319	2635
2320		−void vring_del_virtqueue(struct virtqueue *_vq)
	2636	+static void vring_free(struct virtqueue *_vq)
2321	2637	{
2322	2638	struct vring_virtqueue *vq = to_vvq(_vq);
2323	2639
2324		− spin_lock(&vq->vq.vdev->vqs_list_lock);
2325		− list_del(&_vq->list);
2326		− spin_unlock(&vq->vq.vdev->vqs_list_lock);
2327		−
2328	2640	if (vq->we_own_ring) {
2329	2641	if (vq->packed_ring) {
2330	2642	vring_free_queue(vq->vq.vdev,
@@ -2355,6 +2667,18 @@ void vring_del_virtqueue(struct virtqueue *_vq) · 24 unmodified lines
2355	2667	kfree(vq->split.desc_state);
2356	2668	kfree(vq->split.desc_extra);
2357	2669	}
	2670	+}
	2671	+
	2672	+void vring_del_virtqueue(struct virtqueue *_vq)
	2673	+{
	2674	+ struct vring_virtqueue *vq = to_vvq(_vq);
	2675	+
	2676	+ spin_lock(&vq->vq.vdev->vqs_list_lock);
	2677	+ list_del(&_vq->list);
	2678	+ spin_unlock(&vq->vq.vdev->vqs_list_lock);
	2679	+
	2680	+ vring_free(_vq);
	2681	+
2358	2682	kfree(vq);
2359	2683	}
2360	2684	EXPORT_SYMBOL_GPL(vring_del_virtqueue);
@@ -2402,6 +2726,30 @@ unsigned int virtqueue_get_vring_size(struct virtqueue *_vq) · 41 unmodified lines
2402	2726	}
2403	2727	EXPORT_SYMBOL_GPL(virtqueue_get_vring_size);
2404	2728
	2729	+/*
	2730	+ * This function should only be called by the core, not directly by the driver.
	2731	+ */
	2732	+void __virtqueue_break(struct virtqueue *_vq)
	2733	+{
	2734	+ struct vring_virtqueue *vq = to_vvq(_vq);
	2735	+
	2736	+ /* Pairs with READ_ONCE() in virtqueue_is_broken(). */
	2737	+ WRITE_ONCE(vq->broken, true);
	2738	+}
	2739	+EXPORT_SYMBOL_GPL(__virtqueue_break);
	2740	+
	2741	+/*
	2742	+ * This function should only be called by the core, not directly by the driver.
	2743	+ */
	2744	+void __virtqueue_unbreak(struct virtqueue *_vq)
	2745	+{
	2746	+ struct vring_virtqueue *vq = to_vvq(_vq);
	2747	+
	2748	+ /* Pairs with READ_ONCE() in virtqueue_is_broken(). */
	2749	+ WRITE_ONCE(vq->broken, false);
	2750	+}
	2751	+EXPORT_SYMBOL_GPL(__virtqueue_unbreak);
	2752	+
2405	2753	bool virtqueue_is_broken(struct virtqueue *_vq)
2406	2754	{
2407	2755	struct vring_virtqueue *vq = to_vvq(_vq);

drivers/virtio/virtio_vdpa.c+12−6

@@ -131,7 +131,7 @@ static irqreturn_t virtio_vdpa_virtqueue_cb(void *private) · 130 unmodified lines
131	131	static struct virtqueue *
132	132	virtio_vdpa_setup_vq(struct virtio_device *vdev, unsigned int index,
133	133	void (callback)(struct virtqueue vq),
134		− const char *name, bool ctx)
	134	+ const char *name, u32 size, bool ctx)
135	135	{
136	136	struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vdev);
137	137	struct vdpa_device *vdpa = vd_get_vdpa(vdev);
@@ -168,14 +168,17 @@ virtio_vdpa_setup_vq(struct virtio_device *vdev, unsigned int index, · 30 unmodified lines
168	168	goto error_new_virtqueue;
169	169	}
170	170
	171	+ if (!size \|\| size > max_num)
	172	+ size = max_num;
	173	+
171	174	if (ops->get_vq_num_min)
172	175	min_num = ops->get_vq_num_min(vdpa);
173	176
174		− may_reduce_num = (max_num == min_num) ? false : true;
	177	+ may_reduce_num = (size == min_num) ? false : true;
175	178
176	179	/* Create the vring */
177	180	align = ops->get_vq_align(vdpa);
178		− vq = vring_create_virtqueue(index, max_num, align, vdev,
	181	+ vq = vring_create_virtqueue(index, size, align, vdev,
179	182	true, may_reduce_num, ctx,
180	183	virtio_vdpa_notify, callback, name);
181	184	if (!vq) {
@@ -183,6 +186,8 @@ virtio_vdpa_setup_vq(struct virtio_device *vdev, unsigned int index, · 1 unmodified line
183	186	goto error_new_virtqueue;
184	187	}
185	188
	189	+ vq->num_max = max_num;
	190	+
186	191	/* Setup virtqueue callback */
187	192	cb.callback = callback ? virtio_vdpa_virtqueue_cb : NULL;
188	193	cb.private = info;
@@ -267,6 +272,7 @@ static int virtio_vdpa_find_vqs(struct virtio_device *vdev, unsigned int nvqs, · 78 unmodified lines
267	272	struct virtqueue *vqs[],
268	273	vq_callback_t *callbacks[],
269	274	const char * const names[],
	275	+ u32 sizes[],
270	276	const bool *ctx,
271	277	struct irq_affinity *desc)
272	278	{
@@ -282,9 +288,9 @@ static int virtio_vdpa_find_vqs(struct virtio_device *vdev, unsigned int nvqs, · 9 unmodified lines
282	288	continue;
283	289	}
284	290
285		− vqs[i] = virtio_vdpa_setup_vq(vdev, queue_idx++,
286		− callbacks[i], names[i], ctx ?
287		− ctx[i] : false);
	291	+ vqs[i] = virtio_vdpa_setup_vq(vdev, queue_idx++, callbacks[i],
	292	+ names[i], sizes ? sizes[i] : 0,
	293	+ ctx ? ctx[i] : false);
288	294	if (IS_ERR(vqs[i])) {
289	295	err = PTR_ERR(vqs[i]);
290	296	goto err_setup_vq;

include/linux/mlx5/mlx5_ifc_vdpa.h+8−0

@@ -150,6 +150,14 @@ enum { · 149 unmodified lines
150	150	MLX5_VIRTIO_NET_Q_OBJECT_STATE_ERR = 0x3,
151	151	};
152	152
	153	+/* This indicates that the object was not created or has already
	154	+ * been desroyed. It is very safe to assume that this object will never
	155	+ * have so many states
	156	+ */
	157	+enum {
	158	+ MLX5_VIRTIO_NET_Q_OBJECT_NONE = 0xffffffff
	159	+};
	160	+
153	161	enum {
154	162	MLX5_RQTC_LIST_Q_TYPE_RQ = 0x0,
155	163	MLX5_RQTC_LIST_Q_TYPE_VIRTIO_NET_Q = 0x1,

include/linux/remoteproc.h+2−2

@@ -597,7 +597,7 @@ struct rproc_subdev { · 596 unmodified lines
597	597	/**
598	598	* struct rproc_vring - remoteproc vring state
599	599	* @va: virtual address
600		− * @len: length, in bytes
	600	+ * @num: vring size
601	601	* @da: device address
602	602	* @align: vring alignment
603	603	* @notifyid: rproc-specific unique vring index
@@ -606,7 +606,7 @@ struct rproc_subdev { · 2 unmodified lines
606	606	*/
607	607	struct rproc_vring {
608	608	void *va;
609		− int len;
	609	+ int num;
610	610	u32 da;
611	611	u32 align;
612	612	int notifyid;

include/linux/vdpa.h+4−0

@@ -218,6 +218,9 @@ struct vdpa_map_file { · 217 unmodified lines
218	218	* @reset: Reset device
219	219	* @vdev: vdpa device
220	220	* Returns integer: success (0) or error (< 0)
	221	+ * @suspend: Suspend or resume the device (optional)
	222	+ * @vdev: vdpa device
	223	+ * Returns integer: success (0) or error (< 0)
221	224	* @get_config_size: Get the size of the configuration space includes
222	225	* fields that are conditional on feature bits.
223	226	* @vdev: vdpa device
@@ -319,6 +322,7 @@ struct vdpa_config_ops { · 95 unmodified lines
319	322	u8 (get_status)(struct vdpa_device vdev);
320	323	void (set_status)(struct vdpa_device vdev, u8 status);
321	324	int (reset)(struct vdpa_device vdev);
	325	+ int (suspend)(struct vdpa_device vdev);
322	326	size_t (get_config_size)(struct vdpa_device vdev);
323	327	void (get_config)(struct vdpa_device vdev, unsigned int offset,
324	328	void *buf, unsigned int len);

include/linux/virtio.h+10−0

@@ -19,6 +19,8 @@ · 18 unmodified lines
19	19	* @priv: a pointer for the virtqueue implementation to use.
20	20	* @index: the zero-based ordinal number for this queue.
21	21	* @num_free: number of elements we expect to be able to fit.
	22	+ * @num_max: the maximum number of elements supported by the device.
	23	+ * @reset: vq is in reset state or not.
22	24	*
23	25	* A note on @num_free: with indirect buffers, each buffer needs one
24	26	* element in the queue, otherwise a buffer will need one element per
@@ -31,7 +33,9 @@ struct virtqueue { · 6 unmodified lines
31	33	struct virtio_device *vdev;
32	34	unsigned int index;
33	35	unsigned int num_free;
	36	+ unsigned int num_max;
34	37	void *priv;
	38	+ bool reset;
35	39	};
36	40
37	41	int virtqueue_add_outbuf(struct virtqueue *vq,
@@ -89,6 +93,9 @@ dma_addr_t virtqueue_get_desc_addr(struct virtqueue *vq); · 51 unmodified lines
89	93	dma_addr_t virtqueue_get_avail_addr(struct virtqueue *vq);
90	94	dma_addr_t virtqueue_get_used_addr(struct virtqueue *vq);
91	95
	96	+int virtqueue_resize(struct virtqueue *vq, u32 num,
	97	+ void (recycle)(struct virtqueue vq, void *buf));
	98	+
92	99	/**
93	100	* virtio_device - representation of a device using virtio
94	101	* @index: unique position on the virtio bus
@@ -133,6 +140,9 @@ bool is_virtio_device(struct device *dev); · 38 unmodified lines
133	140	void virtio_break_device(struct virtio_device *dev);
134	141	void __virtio_unbreak_device(struct virtio_device *dev);
135	142
	143	+void __virtqueue_break(struct virtqueue *_vq);
	144	+void __virtqueue_unbreak(struct virtqueue *_vq);
	145	+
136	146	void virtio_config_changed(struct virtio_device *dev);
137	147	#ifdef CONFIG_PM_SLEEP
138	148	int virtio_device_freeze(struct virtio_device *dev);

include/linux/virtio_config.h+35−5

@@ -55,6 +55,7 @@ struct virtio_shm_region { · 54 unmodified lines
55	55	* include a NULL entry for vqs that do not need a callback
56	56	* names: array of virtqueue names (mainly for debugging)
57	57	* include a NULL entry for vqs unused by driver
	58	+ * sizes: array of virtqueue sizes
58	59	* Returns 0 on success or error status
59	60	* @del_vqs: free virtqueues found by find_vqs().
60	61	* @synchronize_cbs: synchronize with the virtqueue callbacks (optional)
@@ -78,6 +79,18 @@ struct virtio_shm_region { · 17 unmodified lines
78	79	* @set_vq_affinity: set the affinity for a virtqueue (optional).
79	80	* @get_vq_affinity: get the affinity for a virtqueue (optional).
80	81	* @get_shm_region: get a shared memory region based on the index.
	82	+ * @disable_vq_and_reset: reset a queue individually (optional).
	83	+ * vq: the virtqueue
	84	+ * Returns 0 on success or error status
	85	+ * disable_vq_and_reset will guarantee that the callbacks are disabled and
	86	+ * synchronized.
	87	+ * Except for the callback, the caller should guarantee that the vring is
	88	+ * not accessed by any functions of virtqueue.
	89	+ * @enable_vq_after_reset: enable a reset queue
	90	+ * vq: the virtqueue
	91	+ * Returns 0 on success or error status
	92	+ * If disable_vq_and_reset is set, then enable_vq_after_reset must also be
	93	+ * set.
81	94	*/
82	95	typedef void vq_callback_t(struct virtqueue *);
83	96	struct virtio_config_ops {
@@ -91,7 +104,9 @@ struct virtio_config_ops { · 7 unmodified lines
91	104	void (reset)(struct virtio_device vdev);
92	105	int (find_vqs)(struct virtio_device , unsigned nvqs,
93	106	struct virtqueue vqs[], vq_callback_t callbacks[],
94		− const char * const names[], const bool *ctx,
	107	+ const char * const names[],
	108	+ u32 sizes[],
	109	+ const bool *ctx,
95	110	struct irq_affinity *desc);
96	111	void (del_vqs)(struct virtio_device );
97	112	void (synchronize_cbs)(struct virtio_device );
@@ -104,6 +119,8 @@ struct virtio_config_ops { · 6 unmodified lines
104	119	int index);
105	120	bool (get_shm_region)(struct virtio_device vdev,
106	121	struct virtio_shm_region *region, u8 id);
	122	+ int (disable_vq_and_reset)(struct virtqueue vq);
	123	+ int (enable_vq_after_reset)(struct virtqueue vq);
107	124	};
108	125
109	126	/* If driver didn't advertise the feature, it will never appear. */
@@ -198,7 +215,7 @@ struct virtqueue virtio_find_single_vq(struct virtio_device vdev, · 88 unmodified lines
198	215	const char *names[] = { n };
199	216	struct virtqueue *vq;
200	217	int err = vdev->config->find_vqs(vdev, 1, &vq, callbacks, names, NULL,
201		− NULL);
	218	+ NULL, NULL);
202	219	if (err < 0)
203	220	return ERR_PTR(err);
204	221	return vq;
@@ -210,7 +227,8 @@ int virtio_find_vqs(struct virtio_device *vdev, unsigned nvqs, · 5 unmodified lines
210	227	const char * const names[],
211	228	struct irq_affinity *desc)
212	229	{
213		− return vdev->config->find_vqs(vdev, nvqs, vqs, callbacks, names, NULL, desc);
	230	+ return vdev->config->find_vqs(vdev, nvqs, vqs, callbacks, names, NULL,
	231	+ NULL, desc);
214	232	}
215	233
216	234	static inline
@@ -219,8 +237,20 @@ int virtio_find_vqs_ctx(struct virtio_device *vdev, unsigned nvqs, · 2 unmodified lines
219	237	const char * const names[], const bool *ctx,
220	238	struct irq_affinity *desc)
221	239	{
222		− return vdev->config->find_vqs(vdev, nvqs, vqs, callbacks, names, ctx,
223		− desc);
	240	+ return vdev->config->find_vqs(vdev, nvqs, vqs, callbacks, names, NULL,
	241	+ ctx, desc);
	242	+}
	243	+
	244	+static inline
	245	+int virtio_find_vqs_ctx_size(struct virtio_device *vdev, u32 nvqs,
	246	+ struct virtqueue *vqs[],
	247	+ vq_callback_t *callbacks[],
	248	+ const char * const names[],
	249	+ u32 sizes[],
	250	+ const bool ctx, struct irq_affinity desc)
	251	+{
	252	+ return vdev->config->find_vqs(vdev, nvqs, vqs, callbacks, names, sizes,
	253	+ ctx, desc);
224	254	}
225	255
226	256	/**

include/linux/virtio_pci_modern.h+9−0

@@ -5,6 +5,13 @@ · 4 unmodified lines
5	5	#include <linux/pci.h>
6	6	#include <linux/virtio_pci.h>
7	7
	8	+struct virtio_pci_modern_common_cfg {
	9	+ struct virtio_pci_common_cfg cfg;
	10	+
	11	+ __le16 queue_notify_data; /* read-write */
	12	+ __le16 queue_reset; /* read-write */
	13	+};
	14	+
8	15	struct virtio_pci_modern_device {
9	16	struct pci_dev *pci_dev;
10	17
@@ -106,4 +113,6 @@ void __iomem * vp_modern_map_vq_notify(struct virtio_pci_modern_device *mdev, · 95 unmodified lines
106	113	u16 index, resource_size_t *pa);
107	114	int vp_modern_probe(struct virtio_pci_modern_device *mdev);
108	115	void vp_modern_remove(struct virtio_pci_modern_device *mdev);
	116	+int vp_modern_get_queue_reset(struct virtio_pci_modern_device *mdev, u16 index);
	117	+void vp_modern_set_queue_reset(struct virtio_pci_modern_device *mdev, u16 index);
109	118	#endif

include/linux/virtio_ring.h+0−10

@@ -76,16 +76,6 @@ struct virtqueue *vring_create_virtqueue(unsigned int index, · 75 unmodified lines
76	76	void (callback)(struct virtqueue vq),
77	77	const char *name);
78	78
79		−/* Creates a virtqueue with a custom layout. */
80		−struct virtqueue *__vring_new_virtqueue(unsigned int index,
81		− struct vring vring,
82		− struct virtio_device *vdev,
83		− bool weak_barriers,
84		− bool ctx,
85		− bool (notify)(struct virtqueue ),
86		− void (callback)(struct virtqueue ),
87		− const char *name);
88		−
89	79	/*
90	80	* Creates a virtqueue with a standard layout but a caller-allocated
91	81	* ring.

include/uapi/linux/vduse.h+47−0

@@ -210,6 +210,53 @@ struct vduse_vq_eventfd { · 209 unmodified lines
210	210	*/
211	211	#define VDUSE_VQ_INJECT_IRQ _IOW(VDUSE_BASE, 0x17, __u32)
212	212
	213	+/**
	214	+ * struct vduse_iova_umem - userspace memory configuration for one IOVA region
	215	+ * @uaddr: start address of userspace memory, it must be aligned to page size
	216	+ * @iova: start of the IOVA region
	217	+ * @size: size of the IOVA region
	218	+ * @reserved: for future use, needs to be initialized to zero
	219	+ *
	220	+ * Structure used by VDUSE_IOTLB_REG_UMEM and VDUSE_IOTLB_DEREG_UMEM
	221	+ * ioctls to register/de-register userspace memory for IOVA regions
	222	+ */
	223	+struct vduse_iova_umem {
	224	+ __u64 uaddr;
	225	+ __u64 iova;
	226	+ __u64 size;
	227	+ __u64 reserved[3];
	228	+};
	229	+
	230	+/* Register userspace memory for IOVA regions */
	231	+#define VDUSE_IOTLB_REG_UMEM _IOW(VDUSE_BASE, 0x18, struct vduse_iova_umem)
	232	+
	233	+/* De-register the userspace memory. Caller should set iova and size field. */
	234	+#define VDUSE_IOTLB_DEREG_UMEM _IOW(VDUSE_BASE, 0x19, struct vduse_iova_umem)
	235	+
	236	+/**
	237	+ * struct vduse_iova_info - information of one IOVA region
	238	+ * @start: start of the IOVA region
	239	+ * @last: last of the IOVA region
	240	+ * @capability: capability of the IOVA regsion
	241	+ * @reserved: for future use, needs to be initialized to zero
	242	+ *
	243	+ * Structure used by VDUSE_IOTLB_GET_INFO ioctl to get information of
	244	+ * one IOVA region.
	245	+ */
	246	+struct vduse_iova_info {
	247	+ __u64 start;
	248	+ __u64 last;
	249	+#define VDUSE_IOVA_CAP_UMEM (1 << 0)
	250	+ __u64 capability;
	251	+ __u64 reserved[3];
	252	+};
	253	+
	254	+/*
	255	+ * Find the first IOVA region that overlaps with the range [start, last]
	256	+ * and return some information on it. Caller should set start and last fields.
	257	+ */
	258	+#define VDUSE_IOTLB_GET_INFO _IOWR(VDUSE_BASE, 0x1a, struct vduse_iova_info)
	259	+
213	260	/* The control messages definition for read(2)/write(2) on /dev/vduse/$NAME */
214	261
215	262	/**

include/uapi/linux/vhost.h+9−0

@@ -171,4 +171,13 @@ · 170 unmodified lines
171	171	#define VHOST_VDPA_SET_GROUP_ASID _IOW(VHOST_VIRTIO, 0x7C, \
172	172	struct vhost_vring_state)
173	173
	174	+/* Suspend a device so it does not process virtqueue requests anymore
	175	+ *
	176	+ * After the return of ioctl the device must preserve all the necessary state
	177	+ * (the virtqueue vring base plus the possible device specific states) that is
	178	+ * required for restoring in the future. The device must not change its
	179	+ * configuration after that point.
	180	+ */
	181	+#define VHOST_VDPA_SUSPEND _IO(VHOST_VIRTIO, 0x7D)
	182	+
174	183	#endif

include/uapi/linux/vhost_types.h+2−0

@@ -161,5 +161,7 @@ struct vhost_vdpa_iova_range { · 160 unmodified lines
161	161	* message
162	162	*/
163	163	#define VHOST_BACKEND_F_IOTLB_ASID 0x3
	164	+/* Device can be suspended */
	165	+#define VHOST_BACKEND_F_SUSPEND 0x4
164	166
165	167	#endif

include/uapi/linux/virtio_config.h+6−1

@@ -52,7 +52,7 @@ · 51 unmodified lines
52	52	* rest are per-device feature bits.
53	53	*/
54	54	#define VIRTIO_TRANSPORT_F_START 28
55		−#define VIRTIO_TRANSPORT_F_END 38
	55	+#define VIRTIO_TRANSPORT_F_END 41
56	56
57	57	#ifndef VIRTIO_CONFIG_NO_LEGACY
58	58	/* Do we get callbacks when the ring is completely used, even if we've
@@ -98,4 +98,9 @@ · 39 unmodified lines
98	98	* Does the device support Single Root I/O Virtualization?
99	99	*/
100	100	#define VIRTIO_F_SR_IOV 37
	101	+
	102	+/*
	103	+ * This feature indicates that the driver can reset a queue individually.
	104	+ */
	105	+#define VIRTIO_F_RING_RESET 40
101	106	#endif /* _UAPI_LINUX_VIRTIO_CONFIG_H */

include/uapi/linux/virtio_net.h+33−1

@@ -56,7 +56,7 @@ · 55 unmodified lines
56	56	#define VIRTIO_NET_F_MQ 22 /* Device supports Receive Flow
57	57	* Steering */
58	58	#define VIRTIO_NET_F_CTRL_MAC_ADDR 23 /* Set MAC address */
59		−
	59	+#define VIRTIO_NET_F_NOTF_COAL 53 /* Guest can handle notifications coalescing */
60	60	#define VIRTIO_NET_F_HASH_REPORT 57 /* Supports hash report */
61	61	#define VIRTIO_NET_F_RSS 60 /* Supports RSS RX steering */
62	62	#define VIRTIO_NET_F_RSC_EXT 61 /* extended coalescing info */
@@ -355,4 +355,36 @@ struct virtio_net_hash_config { · 292 unmodified lines
355	355	#define VIRTIO_NET_CTRL_GUEST_OFFLOADS 5
356	356	#define VIRTIO_NET_CTRL_GUEST_OFFLOADS_SET 0
357	357
	358	+/*
	359	+ * Control notifications coalescing.
	360	+ *
	361	+ * Request the device to change the notifications coalescing parameters.
	362	+ *
	363	+ * Available with the VIRTIO_NET_F_NOTF_COAL feature bit.
	364	+ */
	365	+#define VIRTIO_NET_CTRL_NOTF_COAL 6
	366	+/*
	367	+ * Set the tx-usecs/tx-max-packets patameters.
	368	+ * tx-usecs - Maximum number of usecs to delay a TX notification.
	369	+ * tx-max-packets - Maximum number of packets to send before a TX notification.
	370	+ */
	371	+struct virtio_net_ctrl_coal_tx {
	372	+ __le32 tx_max_packets;
	373	+ __le32 tx_usecs;
	374	+};
	375	+
	376	+#define VIRTIO_NET_CTRL_NOTF_COAL_TX_SET 0
	377	+
	378	+/*
	379	+ * Set the rx-usecs/rx-max-packets patameters.
	380	+ * rx-usecs - Maximum number of usecs to delay a RX notification.
	381	+ * rx-max-frames - Maximum number of packets to receive before a RX notification.
	382	+ */
	383	+struct virtio_net_ctrl_coal_rx {
	384	+ __le32 rx_max_packets;
	385	+ __le32 rx_usecs;
	386	+};
	387	+
	388	+#define VIRTIO_NET_CTRL_NOTF_COAL_RX_SET 1
	389	+
358	390	#endif /* _UAPI_LINUX_VIRTIO_NET_H */

include/uapi/linux/virtio_pci.h+2−0

@@ -202,6 +202,8 @@ struct virtio_pci_cfg_cap { · 201 unmodified lines
202	202	#define VIRTIO_PCI_COMMON_Q_AVAILHI 44
203	203	#define VIRTIO_PCI_COMMON_Q_USEDLO 48
204	204	#define VIRTIO_PCI_COMMON_Q_USEDHI 52
	205	+#define VIRTIO_PCI_COMMON_Q_NDATA 56
	206	+#define VIRTIO_PCI_COMMON_Q_RESET 58
205	207
206	208	#endif /* VIRTIO_PCI_NO_MODERN */
207	209

tools/virtio/linux/kernel.h+1−1

@@ -29,7 +29,6 @@ · 28 unmodified lines
29	29	#define READ 0
30	30	#define WRITE 1
31	31
32		−typedef unsigned long long phys_addr_t;
33	32	typedef unsigned long long dma_addr_t;
34	33	typedef size_t __kernel_size_t;
35	34	typedef unsigned int __wsum;
@@ -136,6 +135,7 @@ static inline void krealloc_array(void p, size_t new_n, size_t new_size, gfp_t · 100 unmodified lines
136	135	#endif
137	136	#define dev_err(dev, format, ...) fprintf (stderr, format, ## __VA_ARGS__)
138	137	#define dev_warn(dev, format, ...) fprintf (stderr, format, ## __VA_ARGS__)
	138	+#define dev_warn_once(dev, format, ...) fprintf (stderr, format, ## __VA_ARGS__)
139	139
140	140	#define min(x, y) ({ \
141	141	typeof(x) _min1 = (x); \

tools/virtio/linux/vringh.h+1−0

@@ -1 +1,2 @@
	1	+#include <limits.h>
1	2	#include "../../../include/linux/vringh.h"

tools/virtio/virtio_test.c+2−2

@@ -102,8 +102,8 @@ static void vq_reset(struct vq_info info, int num, struct virtio_device vdev) · 101 unmodified lines
102	102
103	103	memset(info->ring, 0, vring_size(num, 4096));
104	104	vring_init(&info->vring, num, info->ring, 4096);
105		− info->vq = __vring_new_virtqueue(info->idx, info->vring, vdev, true,
106		− false, vq_notify, vq_callback, "test");
	105	+ info->vq = vring_new_virtqueue(info->idx, num, 4096, vdev, true, false,
	106	+ info->ring, vq_notify, vq_callback, "test");
107	107	assert(info->vq);
108	108	info->vq->priv = info;
109	109	}