// SPDX-License-Identifier: GPL-2.0-or-later/* Virtio ring implementation. * * Copyright 2007 Rusty Russell IBM Corporation */#include <linux/virtio.h>#include <linux/virtio_ring.h>#include <linux/virtio_config.h>#include <linux/device.h>#include <linux/slab.h>#include <linux/module.h>#include <linux/hrtimer.h>#include <linux/dma-mapping.h>#include <linux/kmsan.h>#include <linux/spinlock.h>#include <xen/xen.h> #ifdef DEBUG/* For development, we want to crash whenever the ring is screwed. */#define BAD_RING(_vq, fmt, args...) \ do { \ dev_err(&(_vq)->vq.vdev->dev, \ "%s:"fmt, (_vq)->vq.name, ##args); \ BUG(); \ } while (0)/* Caller is supposed to guarantee no reentry. */#define START_USE(_vq) \ do { \ if ((_vq)->in_use) \ panic("%s:in_use = %i\n", \ (_vq)->vq.name, (_vq)->in_use); \ (_vq)->in_use = __LINE__; \ } while (0)#define END_USE(_vq) \ do { BUG_ON(!(_vq)->in_use); (_vq)->in_use = 0; } while(0)#define LAST_ADD_TIME_UPDATE(_vq) \ do { \ ktime_t now = ktime_get(); \ \ /* No kick or get, with .1 second between? Warn. */ \ if ((_vq)->last_add_time_valid) \ WARN_ON(ktime_to_ms(ktime_sub(now, \ (_vq)->last_add_time)) > 100); \ (_vq)->last_add_time = now; \ (_vq)->last_add_time_valid = true; \ } while (0)#define LAST_ADD_TIME_CHECK(_vq) \ do { \ if ((_vq)->last_add_time_valid) { \ WARN_ON(ktime_to_ms(ktime_sub(ktime_get(), \ (_vq)->last_add_time)) > 100); \ } \ } while (0)#define LAST_ADD_TIME_INVALID(_vq) \ ((_vq)->last_add_time_valid = false)#else#define BAD_RING(_vq, fmt, args...) \ do { \ dev_err(&_vq->vq.vdev->dev, \ "%s:"fmt, (_vq)->vq.name, ##args); \ (_vq)->broken = true; \ } while (0)#define START_USE(vq)#define END_USE(vq)#define LAST_ADD_TIME_UPDATE(vq)#define LAST_ADD_TIME_CHECK(vq)#define LAST_ADD_TIME_INVALID(vq)#endif enum vq_layout { VQ_LAYOUT_SPLIT = 0, VQ_LAYOUT_PACKED, VQ_LAYOUT_SPLIT_IN_ORDER, VQ_LAYOUT_PACKED_IN_ORDER,}; struct vring_desc_state_split { void *data; /* Data for callback. */ /* Indirect desc table and extra table, if any. These two will be * allocated together. So we won't stress more to the memory allocator. */ struct vring_desc *indir_desc; u32 total_in_len;}; struct vring_desc_state_packed { void *data; /* Data for callback. */ /* Indirect desc table and extra table, if any. These two will be * allocated together. So we won't stress more to the memory allocator. */ struct vring_packed_desc *indir_desc; u16 num; /* Descriptor list length. */ u16 last; /* The last desc state in a list. */ u32 total_in_len; /* In length for the skipped buffer. */}; struct vring_desc_extra { dma_addr_t addr; /* Descriptor DMA addr. */ u32 len; /* Descriptor length. */ u16 flags; /* Descriptor flags. */ u16 next; /* The next desc state in a list. */}; struct vring_virtqueue_split { /* Actual memory layout for this queue. */ struct vring vring; /* Last written value to avail->flags */ u16 avail_flags_shadow; /* * Last written value to avail->idx in * guest byte order. */ u16 avail_idx_shadow; /* Per-descriptor state. */ struct vring_desc_state_split *desc_state; struct vring_desc_extra *desc_extra; /* DMA address and size information */ dma_addr_t queue_dma_addr; size_t queue_size_in_bytes; /* * The parameters for creating vrings are reserved for creating new * vring. */ u32 vring_align; bool may_reduce_num;}; struct vring_virtqueue_packed { /* Actual memory layout for this queue. */ struct { unsigned int num; struct vring_packed_desc *desc; struct vring_packed_desc_event *driver; struct vring_packed_desc_event *device; } vring; /* Driver ring wrap counter. */ bool avail_wrap_counter; /* Avail used flags. */ u16 avail_used_flags; /* Index of the next avail descriptor. */ u16 next_avail_idx; /* * Last written value to driver->flags in * guest byte order. */ u16 event_flags_shadow; /* Per-descriptor state. */ struct vring_desc_state_packed *desc_state; struct vring_desc_extra *desc_extra; /* DMA address and size information */ dma_addr_t ring_dma_addr; dma_addr_t driver_event_dma_addr; dma_addr_t device_event_dma_addr; size_t ring_size_in_bytes; size_t event_size_in_bytes;}; struct vring_virtqueue; struct virtqueue_ops { int (*add)(struct vring_virtqueue *vq, struct scatterlist *sgs[], unsigned int total_sg, unsigned int out_sgs, unsigned int in_sgs, void *data, void *ctx, bool premapped, gfp_t gfp, unsigned long attr); void *(*get)(struct vring_virtqueue *vq, unsigned int *len, void **ctx); bool (*kick_prepare)(struct vring_virtqueue *vq); void (*disable_cb)(struct vring_virtqueue *vq); bool (*enable_cb_delayed)(struct vring_virtqueue *vq); unsigned int (*enable_cb_prepare)(struct vring_virtqueue *vq); bool (*poll)(const struct vring_virtqueue *vq, unsigned int last_used_idx); void *(*detach_unused_buf)(struct vring_virtqueue *vq); bool (*more_used)(const struct vring_virtqueue *vq); int (*resize)(struct vring_virtqueue *vq, u32 num); void (*reset)(struct vring_virtqueue *vq);}; struct vring_virtqueue { struct virtqueue vq; /* Is DMA API used? */ bool use_map_api; /* Can we use weak barriers? */ bool weak_barriers; /* Other side has made a mess, don't try any more. */ bool broken; /* Host supports indirect buffers */ bool indirect; /* Host publishes avail event idx */ bool event; enum vq_layout layout; /* * Without IN_ORDER it's the head of free buffer list. With * IN_ORDER and SPLIT, it's the next available buffer * index. With IN_ORDER and PACKED, it's unused. */ unsigned int free_head; /* * With IN_ORDER, once we see an in-order batch, this stores * this last entry, and until we return the last buffer. * After this, id is set to UINT_MAX to mark it invalid. * Unused without IN_ORDER. */ struct used_entry { u32 id; u32 len; } batch_last; /* Number we've added since last sync. */ unsigned int num_added; /* Last used index we've seen. * for split ring, it just contains last used index * for packed ring: * bits up to VRING_PACKED_EVENT_F_WRAP_CTR include the last used index. * bits from VRING_PACKED_EVENT_F_WRAP_CTR include the used wrap counter. */ u16 last_used_idx; /* With IN_ORDER and SPLIT, last descriptor id we used to * detach buffer. */ u16 last_used; /* Hint for event idx: already triggered no need to disable. */ bool event_triggered; union { /* Available for split ring */ struct vring_virtqueue_split split; /* Available for packed ring */ struct vring_virtqueue_packed packed; }; /* How to notify other side. FIXME: commonalize hcalls! */ bool (*notify)(struct virtqueue *vq); /* DMA, allocation, and size information */ bool we_own_ring; union virtio_map map; #ifdef DEBUG /* They're supposed to lock for us. */ unsigned int in_use; /* Figure out if their kicks are too delayed. */ bool last_add_time_valid; ktime_t last_add_time;#endif}; /* * Accessors for device-writable fields in virtio rings. * These fields are concurrently written by the device and read by the driver. * Use READ_ONCE() to prevent compiler optimizations, document the * intentional data race and prevent KCSAN warnings. */static inline u16 vring_read_split_used_idx(const struct vring_virtqueue *vq){ return virtio16_to_cpu(vq->vq.vdev, READ_ONCE(vq->split.vring.used->idx));} static inline u32 vring_read_split_used_id(const struct vring_virtqueue *vq, u16 idx){ return virtio32_to_cpu(vq->vq.vdev, READ_ONCE(vq->split.vring.used->ring[idx].id));} static inline u32 vring_read_split_used_len(const struct vring_virtqueue *vq, u16 idx){ return virtio32_to_cpu(vq->vq.vdev, READ_ONCE(vq->split.vring.used->ring[idx].len));} static inline u16 vring_read_split_avail_event(const struct vring_virtqueue *vq){ return virtio16_to_cpu(vq->vq.vdev, READ_ONCE(vring_avail_event(&vq->split.vring)));} static inline u16 vring_read_packed_desc_flags(const struct vring_virtqueue *vq, u16 idx){ return le16_to_cpu(READ_ONCE(vq->packed.vring.desc[idx].flags));} static inline u16 vring_read_packed_desc_id(const struct vring_virtqueue *vq, u16 idx){ return le16_to_cpu(READ_ONCE(vq->packed.vring.desc[idx].id));} static inline u32 vring_read_packed_desc_len(const struct vring_virtqueue *vq, u16 idx){ return le32_to_cpu(READ_ONCE(vq->packed.vring.desc[idx].len));} static struct vring_desc_extra *vring_alloc_desc_extra(unsigned int num);static void vring_free(struct virtqueue *_vq); /* * Helpers. */ #define to_vvq(_vq) container_of_const(_vq, struct vring_virtqueue, vq) static inline bool virtqueue_is_packed(const struct vring_virtqueue *vq){ return vq->layout == VQ_LAYOUT_PACKED || vq->layout == VQ_LAYOUT_PACKED_IN_ORDER;} static inline bool virtqueue_is_in_order(const struct vring_virtqueue *vq){ return vq->layout == VQ_LAYOUT_SPLIT_IN_ORDER || vq->layout == VQ_LAYOUT_PACKED_IN_ORDER;} static bool virtqueue_use_indirect(const struct vring_virtqueue *vq, unsigned int total_sg){ /* * If the host supports indirect descriptor tables, and we have multiple * buffers, then go indirect. FIXME: tune this threshold */ return (vq->indirect && total_sg > 1 && vq->vq.num_free);} /* * Modern virtio devices have feature bits to specify whether they need a * quirk and bypass the IOMMU. If not there, just use the DMA API. * * If there, the interaction between virtio and DMA API is messy. * * On most systems with virtio, physical addresses match bus addresses, * and it doesn't particularly matter whether we use the DMA API. * * On some systems, including Xen and any system with a physical device * that speaks virtio behind a physical IOMMU, we must use the DMA API * for virtio DMA to work at all. * * On other systems, including SPARC and PPC64, virtio-pci devices are * enumerated as though they are behind an IOMMU, but the virtio host * ignores the IOMMU, so we must either pretend that the IOMMU isn't * there or somehow map everything as the identity. * * For the time being, we preserve historic behavior and bypass the DMA * API. * * TODO: install a per-device DMA ops structure that does the right thing * taking into account all the above quirks, and use the DMA API * unconditionally on data path. */ static bool vring_use_map_api(const struct virtio_device *vdev){ if (!virtio_has_dma_quirk(vdev)) return true; /* Otherwise, we are left to guess. */ /* * In theory, it's possible to have a buggy QEMU-supposed * emulated Q35 IOMMU and Xen enabled at the same time. On * such a configuration, virtio has never worked and will * not work without an even larger kludge. Instead, enable * the DMA API if we're a Xen guest, which at least allows * all of the sensible Xen configurations to work correctly. */ if (xen_domain()) return true; return false;} static bool vring_need_unmap_buffer(const struct vring_virtqueue *vring, const struct vring_desc_extra *extra){ return vring->use_map_api && (extra->addr != DMA_MAPPING_ERROR);} size_t virtio_max_dma_size(const struct virtio_device *vdev){ size_t max_segment_size = SIZE_MAX; if (vring_use_map_api(vdev)) { if (vdev->map) { max_segment_size = vdev->map->max_mapping_size(vdev->vmap); } else max_segment_size = dma_max_mapping_size(vdev->dev.parent); } return max_segment_size;}EXPORT_SYMBOL_GPL(virtio_max_dma_size); static void *vring_alloc_queue(struct virtio_device *vdev, size_t size, dma_addr_t *map_handle, gfp_t flag, union virtio_map map){ if (vring_use_map_api(vdev)) { return virtqueue_map_alloc_coherent(vdev, map, size, map_handle, flag); } else { void *queue = alloc_pages_exact(PAGE_ALIGN(size), flag); if (queue) { phys_addr_t phys_addr = virt_to_phys(queue); *map_handle = (dma_addr_t)phys_addr; /* * Sanity check: make sure we dind't truncate * the address. The only arches I can find that * have 64-bit phys_addr_t but 32-bit dma_addr_t * are certain non-highmem MIPS and x86 * configurations, but these configurations * should never allocate physical pages above 32 * bits, so this is fine. Just in case, throw a * warning and abort if we end up with an * unrepresentable address. */ if (WARN_ON_ONCE(*map_handle != phys_addr)) { free_pages_exact(queue, PAGE_ALIGN(size)); return NULL; } } return queue; }} static void vring_free_queue(struct virtio_device *vdev, size_t size, void *queue, dma_addr_t map_handle, union virtio_map map){ if (vring_use_map_api(vdev)) virtqueue_map_free_coherent(vdev, map, size, queue, map_handle); else free_pages_exact(queue, PAGE_ALIGN(size));} /* * The DMA ops on various arches are rather gnarly right now, and * making all of the arch DMA ops work on the vring device itself * is a mess. */static struct device *vring_dma_dev(const struct vring_virtqueue *vq){ return vq->map.dma_dev;} static int vring_mapping_error(const struct vring_virtqueue *vq, dma_addr_t addr){ struct virtio_device *vdev = vq->vq.vdev; if (!vq->use_map_api) return 0; if (vdev->map) return vdev->map->mapping_error(vq->map, addr); else return dma_mapping_error(vring_dma_dev(vq), addr);} /* Map one sg entry. */static int vring_map_one_sg(const struct vring_virtqueue *vq, struct scatterlist *sg, enum dma_data_direction direction, dma_addr_t *addr, u32 *len, bool premapped, unsigned long attr){ if (premapped) { *addr = sg_dma_address(sg);Lines 1–500 of 4021. Load more
// SPDX-License-Identifier: GPL-2.0-or-later /* Virtio ring implementation. * * Copyright 2007 Rusty Russell IBM Corporation */ #include <linux/virtio.h> #include <linux/virtio_ring.h> #include <linux/virtio_config.h> #include <linux/device.h> #include <linux/slab.h> #include <linux/module.h> #include <linux/hrtimer.h> #include <linux/dma-mapping.h> #include <linux/kmsan.h> #include <linux/spinlock.h> #include <xen/xen.h> #ifdef DEBUG /* For development, we want to crash whenever the ring is screwed. */ #define BAD_RING(_vq, fmt, args...) \ do { \ dev_err(&(_vq)->vq.vdev->dev, \ "%s:"fmt, (_vq)->vq.name, ##args); \ BUG(); \ } while (0) /* Caller is supposed to guarantee no reentry. */ #define START_USE(_vq) \ do { \ if ((_vq)->in_use) \ panic("%s:in_use = %i\n", \ (_vq)->vq.name, (_vq)->in_use); \ (_vq)->in_use = __LINE__; \ } while (0) #define END_USE(_vq) \ do { BUG_ON(!(_vq)->in_use); (_vq)->in_use = 0; } while(0) #define LAST_ADD_TIME_UPDATE(_vq) \ do { \ ktime_t now = ktime_get(); \ \ /* No kick or get, with .1 second between? Warn. */ \ if ((_vq)->last_add_time_valid) \ WARN_ON(ktime_to_ms(ktime_sub(now, \ (_vq)->last_add_time)) > 100); \ (_vq)->last_add_time = now; \ (_vq)->last_add_time_valid = true; \ } while (0) #define LAST_ADD_TIME_CHECK(_vq) \ do { \ if ((_vq)->last_add_time_valid) { \ WARN_ON(ktime_to_ms(ktime_sub(ktime_get(), \ (_vq)->last_add_time)) > 100); \ } \ } while (0) #define LAST_ADD_TIME_INVALID(_vq) \ ((_vq)->last_add_time_valid = false) #else #define BAD_RING(_vq, fmt, args...) \ do { \ dev_err(&_vq->vq.vdev->dev, \ "%s:"fmt, (_vq)->vq.name, ##args); \ (_vq)->broken = true; \ } while (0) #define START_USE(vq) #define END_USE(vq) #define LAST_ADD_TIME_UPDATE(vq) #define LAST_ADD_TIME_CHECK(vq) #define LAST_ADD_TIME_INVALID(vq) #endif enum vq_layout { VQ_LAYOUT_SPLIT = 0, VQ_LAYOUT_PACKED, VQ_LAYOUT_SPLIT_IN_ORDER, VQ_LAYOUT_PACKED_IN_ORDER, }; struct vring_desc_state_split { void *data; /* Data for callback. */ /* Indirect desc table and extra table, if any. These two will be * allocated together. So we won't stress more to the memory allocator. */ struct vring_desc *indir_desc; u32 total_in_len; }; struct vring_desc_state_packed { void *data; /* Data for callback. */ /* Indirect desc table and extra table, if any. These two will be * allocated together. So we won't stress more to the memory allocator. */ struct vring_packed_desc *indir_desc; u16 num; /* Descriptor list length. */ u16 last; /* The last desc state in a list. */ u32 total_in_len; /* In length for the skipped buffer. */ }; struct vring_desc_extra { dma_addr_t addr; /* Descriptor DMA addr. */ u32 len; /* Descriptor length. */ u16 flags; /* Descriptor flags. */ u16 next; /* The next desc state in a list. */ }; struct vring_virtqueue_split { /* Actual memory layout for this queue. */ struct vring vring; /* Last written value to avail->flags */ u16 avail_flags_shadow; /* * Last written value to avail->idx in * guest byte order. */ u16 avail_idx_shadow; /* Per-descriptor state. */ struct vring_desc_state_split *desc_state; struct vring_desc_extra *desc_extra; /* DMA address and size information */ dma_addr_t queue_dma_addr; size_t queue_size_in_bytes; /* * The parameters for creating vrings are reserved for creating new * vring. */ u32 vring_align; bool may_reduce_num; }; struct vring_virtqueue_packed { /* Actual memory layout for this queue. */ struct { unsigned int num; struct vring_packed_desc *desc; struct vring_packed_desc_event *driver; struct vring_packed_desc_event *device; } vring; /* Driver ring wrap counter. */ bool avail_wrap_counter; /* Avail used flags. */ u16 avail_used_flags; /* Index of the next avail descriptor. */ u16 next_avail_idx; /* * Last written value to driver->flags in * guest byte order. */ u16 event_flags_shadow; /* Per-descriptor state. */ struct vring_desc_state_packed *desc_state; struct vring_desc_extra *desc_extra; /* DMA address and size information */ dma_addr_t ring_dma_addr; dma_addr_t driver_event_dma_addr; dma_addr_t device_event_dma_addr; size_t ring_size_in_bytes; size_t event_size_in_bytes; }; struct vring_virtqueue; struct virtqueue_ops { int (*add)(struct vring_virtqueue *vq, struct scatterlist *sgs[], unsigned int total_sg, unsigned int out_sgs, unsigned int in_sgs, void *data, void *ctx, bool premapped, gfp_t gfp, unsigned long attr); void *(*get)(struct vring_virtqueue *vq, unsigned int *len, void **ctx); bool (*kick_prepare)(struct vring_virtqueue *vq); void (*disable_cb)(struct vring_virtqueue *vq); bool (*enable_cb_delayed)(struct vring_virtqueue *vq); unsigned int (*enable_cb_prepare)(struct vring_virtqueue *vq); bool (*poll)(const struct vring_virtqueue *vq, unsigned int last_used_idx); void *(*detach_unused_buf)(struct vring_virtqueue *vq); bool (*more_used)(const struct vring_virtqueue *vq); int (*resize)(struct vring_virtqueue *vq, u32 num); void (*reset)(struct vring_virtqueue *vq); }; struct vring_virtqueue { struct virtqueue vq; /* Is DMA API used? */ bool use_map_api; /* Can we use weak barriers? */ bool weak_barriers; /* Other side has made a mess, don't try any more. */ bool broken; /* Host supports indirect buffers */ bool indirect; /* Host publishes avail event idx */ bool event; enum vq_layout layout; /* * Without IN_ORDER it's the head of free buffer list. With * IN_ORDER and SPLIT, it's the next available buffer * index. With IN_ORDER and PACKED, it's unused. */ unsigned int free_head; /* * With IN_ORDER, once we see an in-order batch, this stores * this last entry, and until we return the last buffer. * After this, id is set to UINT_MAX to mark it invalid. * Unused without IN_ORDER. */ struct used_entry { u32 id; u32 len; } batch_last; /* Number we've added since last sync. */ unsigned int num_added; /* Last used index we've seen. * for split ring, it just contains last used index * for packed ring: * bits up to VRING_PACKED_EVENT_F_WRAP_CTR include the last used index. * bits from VRING_PACKED_EVENT_F_WRAP_CTR include the used wrap counter. */ u16 last_used_idx; /* With IN_ORDER and SPLIT, last descriptor id we used to * detach buffer. */ u16 last_used; /* Hint for event idx: already triggered no need to disable. */ bool event_triggered; union { /* Available for split ring */ struct vring_virtqueue_split split; /* Available for packed ring */ struct vring_virtqueue_packed packed; }; /* How to notify other side. FIXME: commonalize hcalls! */ bool (*notify)(struct virtqueue *vq); /* DMA, allocation, and size information */ bool we_own_ring; union virtio_map map; #ifdef DEBUG /* They're supposed to lock for us. */ unsigned int in_use; /* Figure out if their kicks are too delayed. */ bool last_add_time_valid; ktime_t last_add_time; #endif }; /* * Accessors for device-writable fields in virtio rings. * These fields are concurrently written by the device and read by the driver. * Use READ_ONCE() to prevent compiler optimizations, document the * intentional data race and prevent KCSAN warnings. */ static inline u16 vring_read_split_used_idx(const struct vring_virtqueue *vq) { return virtio16_to_cpu(vq->vq.vdev, READ_ONCE(vq->split.vring.used->idx)); } static inline u32 vring_read_split_used_id(const struct vring_virtqueue *vq, u16 idx) { return virtio32_to_cpu(vq->vq.vdev, READ_ONCE(vq->split.vring.used->ring[idx].id)); } static inline u32 vring_read_split_used_len(const struct vring_virtqueue *vq, u16 idx) { return virtio32_to_cpu(vq->vq.vdev, READ_ONCE(vq->split.vring.used->ring[idx].len)); } static inline u16 vring_read_split_avail_event(const struct vring_virtqueue *vq) { return virtio16_to_cpu(vq->vq.vdev, READ_ONCE(vring_avail_event(&vq->split.vring))); } static inline u16 vring_read_packed_desc_flags(const struct vring_virtqueue *vq, u16 idx) { return le16_to_cpu(READ_ONCE(vq->packed.vring.desc[idx].flags)); } static inline u16 vring_read_packed_desc_id(const struct vring_virtqueue *vq, u16 idx) { return le16_to_cpu(READ_ONCE(vq->packed.vring.desc[idx].id)); } static inline u32 vring_read_packed_desc_len(const struct vring_virtqueue *vq, u16 idx) { return le32_to_cpu(READ_ONCE(vq->packed.vring.desc[idx].len)); } static struct vring_desc_extra *vring_alloc_desc_extra(unsigned int num); static void vring_free(struct virtqueue *_vq); /* * Helpers. */ #define to_vvq(_vq) container_of_const(_vq, struct vring_virtqueue, vq) static inline bool virtqueue_is_packed(const struct vring_virtqueue *vq) { return vq->layout == VQ_LAYOUT_PACKED || vq->layout == VQ_LAYOUT_PACKED_IN_ORDER; } static inline bool virtqueue_is_in_order(const struct vring_virtqueue *vq) { return vq->layout == VQ_LAYOUT_SPLIT_IN_ORDER || vq->layout == VQ_LAYOUT_PACKED_IN_ORDER; } static bool virtqueue_use_indirect(const struct vring_virtqueue *vq, unsigned int total_sg) { /* * If the host supports indirect descriptor tables, and we have multiple * buffers, then go indirect. FIXME: tune this threshold */ return (vq->indirect && total_sg > 1 && vq->vq.num_free); } /* * Modern virtio devices have feature bits to specify whether they need a * quirk and bypass the IOMMU. If not there, just use the DMA API. * * If there, the interaction between virtio and DMA API is messy. * * On most systems with virtio, physical addresses match bus addresses, * and it doesn't particularly matter whether we use the DMA API. * * On some systems, including Xen and any system with a physical device * that speaks virtio behind a physical IOMMU, we must use the DMA API * for virtio DMA to work at all. * * On other systems, including SPARC and PPC64, virtio-pci devices are * enumerated as though they are behind an IOMMU, but the virtio host * ignores the IOMMU, so we must either pretend that the IOMMU isn't * there or somehow map everything as the identity. * * For the time being, we preserve historic behavior and bypass the DMA * API. * * TODO: install a per-device DMA ops structure that does the right thing * taking into account all the above quirks, and use the DMA API * unconditionally on data path. */ static bool vring_use_map_api(const struct virtio_device *vdev) { if (!virtio_has_dma_quirk(vdev)) return true; /* Otherwise, we are left to guess. */ /* * In theory, it's possible to have a buggy QEMU-supposed * emulated Q35 IOMMU and Xen enabled at the same time. On * such a configuration, virtio has never worked and will * not work without an even larger kludge. Instead, enable * the DMA API if we're a Xen guest, which at least allows * all of the sensible Xen configurations to work correctly. */ if (xen_domain()) return true; return false; } static bool vring_need_unmap_buffer(const struct vring_virtqueue *vring, const struct vring_desc_extra *extra) { return vring->use_map_api && (extra->addr != DMA_MAPPING_ERROR); } size_t virtio_max_dma_size(const struct virtio_device *vdev) { size_t max_segment_size = SIZE_MAX; if (vring_use_map_api(vdev)) { if (vdev->map) { max_segment_size = vdev->map->max_mapping_size(vdev->vmap); } else max_segment_size = dma_max_mapping_size(vdev->dev.parent); } return max_segment_size; } EXPORT_SYMBOL_GPL(virtio_max_dma_size); static void *vring_alloc_queue(struct virtio_device *vdev, size_t size, dma_addr_t *map_handle, gfp_t flag, union virtio_map map) { if (vring_use_map_api(vdev)) { return virtqueue_map_alloc_coherent(vdev, map, size, map_handle, flag); } else { void *queue = alloc_pages_exact(PAGE_ALIGN(size), flag); if (queue) { phys_addr_t phys_addr = virt_to_phys(queue); *map_handle = (dma_addr_t)phys_addr; /* * Sanity check: make sure we dind't truncate * the address. The only arches I can find that * have 64-bit phys_addr_t but 32-bit dma_addr_t * are certain non-highmem MIPS and x86 * configurations, but these configurations * should never allocate physical pages above 32 * bits, so this is fine. Just in case, throw a * warning and abort if we end up with an * unrepresentable address. */ if (WARN_ON_ONCE(*map_handle != phys_addr)) { free_pages_exact(queue, PAGE_ALIGN(size)); return NULL; } } return queue; } } static void vring_free_queue(struct virtio_device *vdev, size_t size, void *queue, dma_addr_t map_handle, union virtio_map map) { if (vring_use_map_api(vdev)) virtqueue_map_free_coherent(vdev, map, size, queue, map_handle); else free_pages_exact(queue, PAGE_ALIGN(size)); } /* * The DMA ops on various arches are rather gnarly right now, and * making all of the arch DMA ops work on the vring device itself * is a mess. */ static struct device *vring_dma_dev(const struct vring_virtqueue *vq) { return vq->map.dma_dev; } static int vring_mapping_error(const struct vring_virtqueue *vq, dma_addr_t addr) { struct virtio_device *vdev = vq->vq.vdev; if (!vq->use_map_api) return 0; if (vdev->map) return vdev->map->mapping_error(vq->map, addr); else return dma_mapping_error(vring_dma_dev(vq), addr); } /* Map one sg entry. */ static int vring_map_one_sg(const struct vring_virtqueue *vq, struct scatterlist *sg, enum dma_data_direction direction, dma_addr_t *addr, u32 *len, bool premapped, unsigned long attr) { if (premapped) { *addr = sg_dma_address(sg);