// SPDX-License-Identifier: GPL-2.0-only//#define DEBUG#include <linux/spinlock.h>#include <linux/slab.h>#include <linux/blkdev.h>#include <linux/hdreg.h>#include <linux/module.h>#include <linux/mutex.h>#include <linux/interrupt.h>#include <linux/virtio.h>#include <linux/virtio_blk.h>#include <linux/scatterlist.h>#include <linux/string_helpers.h>#include <linux/idr.h>#include <linux/blk-mq.h>#include <linux/numa.h>#include <linux/vmalloc.h>#include <uapi/linux/virtio_ring.h> #define PART_BITS 4#define VQ_NAME_LEN 16#define MAX_DISCARD_SEGMENTS 256u /* The maximum number of sg elements that fit into a virtqueue */#define VIRTIO_BLK_MAX_SG_ELEMS 32768 #ifdef CONFIG_ARCH_NO_SG_CHAIN#define VIRTIO_BLK_INLINE_SG_CNT 0#else#define VIRTIO_BLK_INLINE_SG_CNT 2#endif static unsigned int num_request_queues;module_param(num_request_queues, uint, 0644);MODULE_PARM_DESC(num_request_queues, "Limit the number of request queues to use for blk device. " "0 for no limit. " "Values > nr_cpu_ids truncated to nr_cpu_ids."); static unsigned int poll_queues;module_param(poll_queues, uint, 0644);MODULE_PARM_DESC(poll_queues, "The number of dedicated virtqueues for polling I/O"); static int major;static DEFINE_IDA(vd_index_ida); static struct workqueue_struct *virtblk_wq; struct virtio_blk_vq { struct virtqueue *vq; spinlock_t lock; char name[VQ_NAME_LEN];} ____cacheline_aligned_in_smp; struct virtio_blk { /* * This mutex must be held by anything that may run after * virtblk_remove() sets vblk->vdev to NULL. * * blk-mq, virtqueue processing, and sysfs attribute code paths are * shut down before vblk->vdev is set to NULL and therefore do not need * to hold this mutex. */ struct mutex vdev_mutex; struct virtio_device *vdev; /* The disk structure for the kernel. */ struct gendisk *disk; /* Block layer tags. */ struct blk_mq_tag_set tag_set; /* Process context for config space updates */ struct work_struct config_work; /* Ida index - used to track minor number allocations. */ int index; /* num of vqs */ int num_vqs; int io_queues[HCTX_MAX_TYPES]; struct virtio_blk_vq *vqs; /* For zoned device */ unsigned int zone_sectors;}; struct virtblk_req { /* Out header */ struct virtio_blk_outhdr out_hdr; /* In header */ union { u8 status; /* * The zone append command has an extended in header. * The status field in zone_append_in_hdr must always * be the last byte. */ struct { __virtio64 sector; u8 status; } zone_append; } in_hdr; size_t in_hdr_len; struct sg_table sg_table; struct scatterlist sg[];}; static inline blk_status_t virtblk_result(u8 status){ switch (status) { case VIRTIO_BLK_S_OK: return BLK_STS_OK; case VIRTIO_BLK_S_UNSUPP: return BLK_STS_NOTSUPP; case VIRTIO_BLK_S_ZONE_OPEN_RESOURCE: return BLK_STS_ZONE_OPEN_RESOURCE; case VIRTIO_BLK_S_ZONE_ACTIVE_RESOURCE: return BLK_STS_ZONE_ACTIVE_RESOURCE; case VIRTIO_BLK_S_IOERR: case VIRTIO_BLK_S_ZONE_UNALIGNED_WP: default: return BLK_STS_IOERR; }} static inline struct virtio_blk_vq *get_virtio_blk_vq(struct blk_mq_hw_ctx *hctx){ struct virtio_blk *vblk = hctx->queue->queuedata; struct virtio_blk_vq *vq = &vblk->vqs[hctx->queue_num]; return vq;} static int virtblk_add_req(struct virtqueue *vq, struct virtblk_req *vbr){ struct scatterlist out_hdr, in_hdr, *sgs[3]; unsigned int num_out = 0, num_in = 0; sg_init_one(&out_hdr, &vbr->out_hdr, sizeof(vbr->out_hdr)); sgs[num_out++] = &out_hdr; if (vbr->sg_table.nents) { if (vbr->out_hdr.type & cpu_to_virtio32(vq->vdev, VIRTIO_BLK_T_OUT)) sgs[num_out++] = vbr->sg_table.sgl; else sgs[num_out + num_in++] = vbr->sg_table.sgl; } sg_init_one(&in_hdr, &vbr->in_hdr.status, vbr->in_hdr_len); sgs[num_out + num_in++] = &in_hdr; return virtqueue_add_sgs(vq, sgs, num_out, num_in, vbr, GFP_ATOMIC);} static int virtblk_setup_discard_write_zeroes_erase(struct request *req, bool unmap){ unsigned short segments = blk_rq_nr_discard_segments(req); unsigned short n = 0; struct virtio_blk_discard_write_zeroes *range; struct bio *bio; u32 flags = 0; if (unmap) flags |= VIRTIO_BLK_WRITE_ZEROES_FLAG_UNMAP; range = kmalloc_objs(*range, segments, GFP_ATOMIC); if (!range) return -ENOMEM; /* * Single max discard segment means multi-range discard isn't * supported, and block layer only runs contiguity merge like * normal RW request. So we can't reply on bio for retrieving * each range info. */ if (queue_max_discard_segments(req->q) == 1) { range[0].flags = cpu_to_le32(flags); range[0].num_sectors = cpu_to_le32(blk_rq_sectors(req)); range[0].sector = cpu_to_le64(blk_rq_pos(req)); n = 1; } else { __rq_for_each_bio(bio, req) { u64 sector = bio->bi_iter.bi_sector; u32 num_sectors = bio->bi_iter.bi_size >> SECTOR_SHIFT; range[n].flags = cpu_to_le32(flags); range[n].num_sectors = cpu_to_le32(num_sectors); range[n].sector = cpu_to_le64(sector); n++; } } WARN_ON_ONCE(n != segments); bvec_set_virt(&req->special_vec, range, sizeof(*range) * segments); req->rq_flags |= RQF_SPECIAL_PAYLOAD; return 0;} static void virtblk_unmap_data(struct request *req, struct virtblk_req *vbr){ if (blk_rq_nr_phys_segments(req)) sg_free_table_chained(&vbr->sg_table, VIRTIO_BLK_INLINE_SG_CNT);} static int virtblk_map_data(struct blk_mq_hw_ctx *hctx, struct request *req, struct virtblk_req *vbr){ int err; if (!blk_rq_nr_phys_segments(req)) return 0; vbr->sg_table.sgl = vbr->sg; err = sg_alloc_table_chained(&vbr->sg_table, blk_rq_nr_phys_segments(req), vbr->sg_table.sgl, VIRTIO_BLK_INLINE_SG_CNT); if (unlikely(err)) return -ENOMEM; return blk_rq_map_sg(req, vbr->sg_table.sgl);} static void virtblk_cleanup_cmd(struct request *req){ if (req->rq_flags & RQF_SPECIAL_PAYLOAD) kfree(bvec_virt(&req->special_vec));} static blk_status_t virtblk_setup_cmd(struct virtio_device *vdev, struct request *req, struct virtblk_req *vbr){ size_t in_hdr_len = sizeof(vbr->in_hdr.status); bool unmap = false; u32 type; u64 sector = 0; if (!IS_ENABLED(CONFIG_BLK_DEV_ZONED) && op_is_zone_mgmt(req_op(req))) return BLK_STS_NOTSUPP; /* Set fields for all request types */ vbr->out_hdr.ioprio = cpu_to_virtio32(vdev, req_get_ioprio(req)); switch (req_op(req)) { case REQ_OP_READ: type = VIRTIO_BLK_T_IN; sector = blk_rq_pos(req); break; case REQ_OP_WRITE: type = VIRTIO_BLK_T_OUT; sector = blk_rq_pos(req); break; case REQ_OP_FLUSH: type = VIRTIO_BLK_T_FLUSH; break; case REQ_OP_DISCARD: type = VIRTIO_BLK_T_DISCARD; break; case REQ_OP_WRITE_ZEROES: type = VIRTIO_BLK_T_WRITE_ZEROES; unmap = !(req->cmd_flags & REQ_NOUNMAP); break; case REQ_OP_SECURE_ERASE: type = VIRTIO_BLK_T_SECURE_ERASE; break; case REQ_OP_ZONE_OPEN: type = VIRTIO_BLK_T_ZONE_OPEN; sector = blk_rq_pos(req); break; case REQ_OP_ZONE_CLOSE: type = VIRTIO_BLK_T_ZONE_CLOSE; sector = blk_rq_pos(req); break; case REQ_OP_ZONE_FINISH: type = VIRTIO_BLK_T_ZONE_FINISH; sector = blk_rq_pos(req); break; case REQ_OP_ZONE_APPEND: type = VIRTIO_BLK_T_ZONE_APPEND; sector = blk_rq_pos(req); in_hdr_len = sizeof(vbr->in_hdr.zone_append); break; case REQ_OP_ZONE_RESET: type = VIRTIO_BLK_T_ZONE_RESET; sector = blk_rq_pos(req); break; case REQ_OP_ZONE_RESET_ALL: type = VIRTIO_BLK_T_ZONE_RESET_ALL; break; case REQ_OP_DRV_IN: /* * Out header has already been prepared by the caller (virtblk_get_id() * or virtblk_submit_zone_report()), nothing to do here. */ return 0; default: WARN_ON_ONCE(1); return BLK_STS_IOERR; } /* Set fields for non-REQ_OP_DRV_IN request types */ vbr->in_hdr_len = in_hdr_len; vbr->out_hdr.type = cpu_to_virtio32(vdev, type); vbr->out_hdr.sector = cpu_to_virtio64(vdev, sector); if (type == VIRTIO_BLK_T_DISCARD || type == VIRTIO_BLK_T_WRITE_ZEROES || type == VIRTIO_BLK_T_SECURE_ERASE) { if (virtblk_setup_discard_write_zeroes_erase(req, unmap)) return BLK_STS_RESOURCE; } return 0;} /* * The status byte is always the last byte of the virtblk request * in-header. This helper fetches its value for all in-header formats * that are currently defined. */static inline u8 virtblk_vbr_status(struct virtblk_req *vbr){ return *((u8 *)&vbr->in_hdr + vbr->in_hdr_len - 1);} static inline void virtblk_request_done(struct request *req){ struct virtblk_req *vbr = blk_mq_rq_to_pdu(req); blk_status_t status = virtblk_result(virtblk_vbr_status(vbr)); struct virtio_blk *vblk = req->mq_hctx->queue->queuedata; virtblk_unmap_data(req, vbr); virtblk_cleanup_cmd(req); if (req_op(req) == REQ_OP_ZONE_APPEND) req->__sector = virtio64_to_cpu(vblk->vdev, vbr->in_hdr.zone_append.sector); blk_mq_end_request(req, status);} static void virtblk_done(struct virtqueue *vq){ struct virtio_blk *vblk = vq->vdev->priv; bool req_done = false; int qid = vq->index; struct virtblk_req *vbr; unsigned long flags; unsigned int len; spin_lock_irqsave(&vblk->vqs[qid].lock, flags); do { virtqueue_disable_cb(vq); while ((vbr = virtqueue_get_buf(vblk->vqs[qid].vq, &len)) != NULL) { struct request *req = blk_mq_rq_from_pdu(vbr); if (likely(!blk_should_fake_timeout(req->q))) blk_mq_complete_request(req); req_done = true; } } while (!virtqueue_enable_cb(vq)); /* In case queue is stopped waiting for more buffers. */ if (req_done) blk_mq_start_stopped_hw_queues(vblk->disk->queue, true); spin_unlock_irqrestore(&vblk->vqs[qid].lock, flags);} static void virtio_commit_rqs(struct blk_mq_hw_ctx *hctx){ struct virtio_blk *vblk = hctx->queue->queuedata; struct virtio_blk_vq *vq = &vblk->vqs[hctx->queue_num]; bool kick; spin_lock_irq(&vq->lock); kick = virtqueue_kick_prepare(vq->vq); spin_unlock_irq(&vq->lock); if (kick) virtqueue_notify(vq->vq);} static blk_status_t virtblk_fail_to_queue(struct request *req, int rc){ virtblk_cleanup_cmd(req); switch (rc) { case -ENOSPC: return BLK_STS_DEV_RESOURCE; case -ENOMEM: return BLK_STS_RESOURCE; default: return BLK_STS_IOERR; }} static blk_status_t virtblk_prep_rq(struct blk_mq_hw_ctx *hctx, struct virtio_blk *vblk, struct request *req, struct virtblk_req *vbr){ blk_status_t status; int num; status = virtblk_setup_cmd(vblk->vdev, req, vbr); if (unlikely(status)) return status; num = virtblk_map_data(hctx, req, vbr); if (unlikely(num < 0)) return virtblk_fail_to_queue(req, -ENOMEM); vbr->sg_table.nents = num; blk_mq_start_request(req); return BLK_STS_OK;} static blk_status_t virtio_queue_rq(struct blk_mq_hw_ctx *hctx, const struct blk_mq_queue_data *bd){ struct virtio_blk *vblk = hctx->queue->queuedata; struct request *req = bd->rq; struct virtblk_req *vbr = blk_mq_rq_to_pdu(req); unsigned long flags; int qid = hctx->queue_num; bool notify = false; blk_status_t status; int err; status = virtblk_prep_rq(hctx, vblk, req, vbr); if (unlikely(status)) return status; spin_lock_irqsave(&vblk->vqs[qid].lock, flags); err = virtblk_add_req(vblk->vqs[qid].vq, vbr); if (err) { virtqueue_kick(vblk->vqs[qid].vq); /* Don't stop the queue if -ENOMEM: we may have failed to * bounce the buffer due to global resource outage. */ if (err == -ENOSPC) blk_mq_stop_hw_queue(hctx); spin_unlock_irqrestore(&vblk->vqs[qid].lock, flags); virtblk_unmap_data(req, vbr); return virtblk_fail_to_queue(req, err); } if (bd->last && virtqueue_kick_prepare(vblk->vqs[qid].vq)) notify = true; spin_unlock_irqrestore(&vblk->vqs[qid].lock, flags); if (notify) virtqueue_notify(vblk->vqs[qid].vq); return BLK_STS_OK;} static bool virtblk_prep_rq_batch(struct request *req){ struct virtio_blk *vblk = req->mq_hctx->queue->queuedata; struct virtblk_req *vbr = blk_mq_rq_to_pdu(req); return virtblk_prep_rq(req->mq_hctx, vblk, req, vbr) == BLK_STS_OK;} static void virtblk_add_req_batch(struct virtio_blk_vq *vq, struct rq_list *rqlist){ struct request *req; unsigned long flags; bool kick; spin_lock_irqsave(&vq->lock, flags); while ((req = rq_list_pop(rqlist))) { struct virtblk_req *vbr = blk_mq_rq_to_pdu(req); int err; err = virtblk_add_req(vq->vq, vbr); if (err) { virtblk_unmap_data(req, vbr); virtblk_cleanup_cmd(req); blk_mq_requeue_request(req, true); } } kick = virtqueue_kick_prepare(vq->vq); spin_unlock_irqrestore(&vq->lock, flags); if (kick) virtqueue_notify(vq->vq);} Lines 1–500 of 1732. Load more
// SPDX-License-Identifier: GPL-2.0-only //#define DEBUG #include <linux/spinlock.h> #include <linux/slab.h> #include <linux/blkdev.h> #include <linux/hdreg.h> #include <linux/module.h> #include <linux/mutex.h> #include <linux/interrupt.h> #include <linux/virtio.h> #include <linux/virtio_blk.h> #include <linux/scatterlist.h> #include <linux/string_helpers.h> #include <linux/idr.h> #include <linux/blk-mq.h> #include <linux/numa.h> #include <linux/vmalloc.h> #include <uapi/linux/virtio_ring.h> #define PART_BITS 4 #define VQ_NAME_LEN 16 #define MAX_DISCARD_SEGMENTS 256u /* The maximum number of sg elements that fit into a virtqueue */ #define VIRTIO_BLK_MAX_SG_ELEMS 32768 #ifdef CONFIG_ARCH_NO_SG_CHAIN #define VIRTIO_BLK_INLINE_SG_CNT 0 #else #define VIRTIO_BLK_INLINE_SG_CNT 2 #endif static unsigned int num_request_queues; module_param(num_request_queues, uint, 0644); MODULE_PARM_DESC(num_request_queues, "Limit the number of request queues to use for blk device. " "0 for no limit. " "Values > nr_cpu_ids truncated to nr_cpu_ids."); static unsigned int poll_queues; module_param(poll_queues, uint, 0644); MODULE_PARM_DESC(poll_queues, "The number of dedicated virtqueues for polling I/O"); static int major; static DEFINE_IDA(vd_index_ida); static struct workqueue_struct *virtblk_wq; struct virtio_blk_vq { struct virtqueue *vq; spinlock_t lock; char name[VQ_NAME_LEN]; } ____cacheline_aligned_in_smp; struct virtio_blk { /* * This mutex must be held by anything that may run after * virtblk_remove() sets vblk->vdev to NULL. * * blk-mq, virtqueue processing, and sysfs attribute code paths are * shut down before vblk->vdev is set to NULL and therefore do not need * to hold this mutex. */ struct mutex vdev_mutex; struct virtio_device *vdev; /* The disk structure for the kernel. */ struct gendisk *disk; /* Block layer tags. */ struct blk_mq_tag_set tag_set; /* Process context for config space updates */ struct work_struct config_work; /* Ida index - used to track minor number allocations. */ int index; /* num of vqs */ int num_vqs; int io_queues[HCTX_MAX_TYPES]; struct virtio_blk_vq *vqs; /* For zoned device */ unsigned int zone_sectors; }; struct virtblk_req { /* Out header */ struct virtio_blk_outhdr out_hdr; /* In header */ union { u8 status; /* * The zone append command has an extended in header. * The status field in zone_append_in_hdr must always * be the last byte. */ struct { __virtio64 sector; u8 status; } zone_append; } in_hdr; size_t in_hdr_len; struct sg_table sg_table; struct scatterlist sg[]; }; static inline blk_status_t virtblk_result(u8 status) { switch (status) { case VIRTIO_BLK_S_OK: return BLK_STS_OK; case VIRTIO_BLK_S_UNSUPP: return BLK_STS_NOTSUPP; case VIRTIO_BLK_S_ZONE_OPEN_RESOURCE: return BLK_STS_ZONE_OPEN_RESOURCE; case VIRTIO_BLK_S_ZONE_ACTIVE_RESOURCE: return BLK_STS_ZONE_ACTIVE_RESOURCE; case VIRTIO_BLK_S_IOERR: case VIRTIO_BLK_S_ZONE_UNALIGNED_WP: default: return BLK_STS_IOERR; } } static inline struct virtio_blk_vq *get_virtio_blk_vq(struct blk_mq_hw_ctx *hctx) { struct virtio_blk *vblk = hctx->queue->queuedata; struct virtio_blk_vq *vq = &vblk->vqs[hctx->queue_num]; return vq; } static int virtblk_add_req(struct virtqueue *vq, struct virtblk_req *vbr) { struct scatterlist out_hdr, in_hdr, *sgs[3]; unsigned int num_out = 0, num_in = 0; sg_init_one(&out_hdr, &vbr->out_hdr, sizeof(vbr->out_hdr)); sgs[num_out++] = &out_hdr; if (vbr->sg_table.nents) { if (vbr->out_hdr.type & cpu_to_virtio32(vq->vdev, VIRTIO_BLK_T_OUT)) sgs[num_out++] = vbr->sg_table.sgl; else sgs[num_out + num_in++] = vbr->sg_table.sgl; } sg_init_one(&in_hdr, &vbr->in_hdr.status, vbr->in_hdr_len); sgs[num_out + num_in++] = &in_hdr; return virtqueue_add_sgs(vq, sgs, num_out, num_in, vbr, GFP_ATOMIC); } static int virtblk_setup_discard_write_zeroes_erase(struct request *req, bool unmap) { unsigned short segments = blk_rq_nr_discard_segments(req); unsigned short n = 0; struct virtio_blk_discard_write_zeroes *range; struct bio *bio; u32 flags = 0; if (unmap) flags |= VIRTIO_BLK_WRITE_ZEROES_FLAG_UNMAP; range = kmalloc_objs(*range, segments, GFP_ATOMIC); if (!range) return -ENOMEM; /* * Single max discard segment means multi-range discard isn't * supported, and block layer only runs contiguity merge like * normal RW request. So we can't reply on bio for retrieving * each range info. */ if (queue_max_discard_segments(req->q) == 1) { range[0].flags = cpu_to_le32(flags); range[0].num_sectors = cpu_to_le32(blk_rq_sectors(req)); range[0].sector = cpu_to_le64(blk_rq_pos(req)); n = 1; } else { __rq_for_each_bio(bio, req) { u64 sector = bio->bi_iter.bi_sector; u32 num_sectors = bio->bi_iter.bi_size >> SECTOR_SHIFT; range[n].flags = cpu_to_le32(flags); range[n].num_sectors = cpu_to_le32(num_sectors); range[n].sector = cpu_to_le64(sector); n++; } } WARN_ON_ONCE(n != segments); bvec_set_virt(&req->special_vec, range, sizeof(*range) * segments); req->rq_flags |= RQF_SPECIAL_PAYLOAD; return 0; } static void virtblk_unmap_data(struct request *req, struct virtblk_req *vbr) { if (blk_rq_nr_phys_segments(req)) sg_free_table_chained(&vbr->sg_table, VIRTIO_BLK_INLINE_SG_CNT); } static int virtblk_map_data(struct blk_mq_hw_ctx *hctx, struct request *req, struct virtblk_req *vbr) { int err; if (!blk_rq_nr_phys_segments(req)) return 0; vbr->sg_table.sgl = vbr->sg; err = sg_alloc_table_chained(&vbr->sg_table, blk_rq_nr_phys_segments(req), vbr->sg_table.sgl, VIRTIO_BLK_INLINE_SG_CNT); if (unlikely(err)) return -ENOMEM; return blk_rq_map_sg(req, vbr->sg_table.sgl); } static void virtblk_cleanup_cmd(struct request *req) { if (req->rq_flags & RQF_SPECIAL_PAYLOAD) kfree(bvec_virt(&req->special_vec)); } static blk_status_t virtblk_setup_cmd(struct virtio_device *vdev, struct request *req, struct virtblk_req *vbr) { size_t in_hdr_len = sizeof(vbr->in_hdr.status); bool unmap = false; u32 type; u64 sector = 0; if (!IS_ENABLED(CONFIG_BLK_DEV_ZONED) && op_is_zone_mgmt(req_op(req))) return BLK_STS_NOTSUPP; /* Set fields for all request types */ vbr->out_hdr.ioprio = cpu_to_virtio32(vdev, req_get_ioprio(req)); switch (req_op(req)) { case REQ_OP_READ: type = VIRTIO_BLK_T_IN; sector = blk_rq_pos(req); break; case REQ_OP_WRITE: type = VIRTIO_BLK_T_OUT; sector = blk_rq_pos(req); break; case REQ_OP_FLUSH: type = VIRTIO_BLK_T_FLUSH; break; case REQ_OP_DISCARD: type = VIRTIO_BLK_T_DISCARD; break; case REQ_OP_WRITE_ZEROES: type = VIRTIO_BLK_T_WRITE_ZEROES; unmap = !(req->cmd_flags & REQ_NOUNMAP); break; case REQ_OP_SECURE_ERASE: type = VIRTIO_BLK_T_SECURE_ERASE; break; case REQ_OP_ZONE_OPEN: type = VIRTIO_BLK_T_ZONE_OPEN; sector = blk_rq_pos(req); break; case REQ_OP_ZONE_CLOSE: type = VIRTIO_BLK_T_ZONE_CLOSE; sector = blk_rq_pos(req); break; case REQ_OP_ZONE_FINISH: type = VIRTIO_BLK_T_ZONE_FINISH; sector = blk_rq_pos(req); break; case REQ_OP_ZONE_APPEND: type = VIRTIO_BLK_T_ZONE_APPEND; sector = blk_rq_pos(req); in_hdr_len = sizeof(vbr->in_hdr.zone_append); break; case REQ_OP_ZONE_RESET: type = VIRTIO_BLK_T_ZONE_RESET; sector = blk_rq_pos(req); break; case REQ_OP_ZONE_RESET_ALL: type = VIRTIO_BLK_T_ZONE_RESET_ALL; break; case REQ_OP_DRV_IN: /* * Out header has already been prepared by the caller (virtblk_get_id() * or virtblk_submit_zone_report()), nothing to do here. */ return 0; default: WARN_ON_ONCE(1); return BLK_STS_IOERR; } /* Set fields for non-REQ_OP_DRV_IN request types */ vbr->in_hdr_len = in_hdr_len; vbr->out_hdr.type = cpu_to_virtio32(vdev, type); vbr->out_hdr.sector = cpu_to_virtio64(vdev, sector); if (type == VIRTIO_BLK_T_DISCARD || type == VIRTIO_BLK_T_WRITE_ZEROES || type == VIRTIO_BLK_T_SECURE_ERASE) { if (virtblk_setup_discard_write_zeroes_erase(req, unmap)) return BLK_STS_RESOURCE; } return 0; } /* * The status byte is always the last byte of the virtblk request * in-header. This helper fetches its value for all in-header formats * that are currently defined. */ static inline u8 virtblk_vbr_status(struct virtblk_req *vbr) { return *((u8 *)&vbr->in_hdr + vbr->in_hdr_len - 1); } static inline void virtblk_request_done(struct request *req) { struct virtblk_req *vbr = blk_mq_rq_to_pdu(req); blk_status_t status = virtblk_result(virtblk_vbr_status(vbr)); struct virtio_blk *vblk = req->mq_hctx->queue->queuedata; virtblk_unmap_data(req, vbr); virtblk_cleanup_cmd(req); if (req_op(req) == REQ_OP_ZONE_APPEND) req->__sector = virtio64_to_cpu(vblk->vdev, vbr->in_hdr.zone_append.sector); blk_mq_end_request(req, status); } static void virtblk_done(struct virtqueue *vq) { struct virtio_blk *vblk = vq->vdev->priv; bool req_done = false; int qid = vq->index; struct virtblk_req *vbr; unsigned long flags; unsigned int len; spin_lock_irqsave(&vblk->vqs[qid].lock, flags); do { virtqueue_disable_cb(vq); while ((vbr = virtqueue_get_buf(vblk->vqs[qid].vq, &len)) != NULL) { struct request *req = blk_mq_rq_from_pdu(vbr); if (likely(!blk_should_fake_timeout(req->q))) blk_mq_complete_request(req); req_done = true; } } while (!virtqueue_enable_cb(vq)); /* In case queue is stopped waiting for more buffers. */ if (req_done) blk_mq_start_stopped_hw_queues(vblk->disk->queue, true); spin_unlock_irqrestore(&vblk->vqs[qid].lock, flags); } static void virtio_commit_rqs(struct blk_mq_hw_ctx *hctx) { struct virtio_blk *vblk = hctx->queue->queuedata; struct virtio_blk_vq *vq = &vblk->vqs[hctx->queue_num]; bool kick; spin_lock_irq(&vq->lock); kick = virtqueue_kick_prepare(vq->vq); spin_unlock_irq(&vq->lock); if (kick) virtqueue_notify(vq->vq); } static blk_status_t virtblk_fail_to_queue(struct request *req, int rc) { virtblk_cleanup_cmd(req); switch (rc) { case -ENOSPC: return BLK_STS_DEV_RESOURCE; case -ENOMEM: return BLK_STS_RESOURCE; default: return BLK_STS_IOERR; } } static blk_status_t virtblk_prep_rq(struct blk_mq_hw_ctx *hctx, struct virtio_blk *vblk, struct request *req, struct virtblk_req *vbr) { blk_status_t status; int num; status = virtblk_setup_cmd(vblk->vdev, req, vbr); if (unlikely(status)) return status; num = virtblk_map_data(hctx, req, vbr); if (unlikely(num < 0)) return virtblk_fail_to_queue(req, -ENOMEM); vbr->sg_table.nents = num; blk_mq_start_request(req); return BLK_STS_OK; } static blk_status_t virtio_queue_rq(struct blk_mq_hw_ctx *hctx, const struct blk_mq_queue_data *bd) { struct virtio_blk *vblk = hctx->queue->queuedata; struct request *req = bd->rq; struct virtblk_req *vbr = blk_mq_rq_to_pdu(req); unsigned long flags; int qid = hctx->queue_num; bool notify = false; blk_status_t status; int err; status = virtblk_prep_rq(hctx, vblk, req, vbr); if (unlikely(status)) return status; spin_lock_irqsave(&vblk->vqs[qid].lock, flags); err = virtblk_add_req(vblk->vqs[qid].vq, vbr); if (err) { virtqueue_kick(vblk->vqs[qid].vq); /* Don't stop the queue if -ENOMEM: we may have failed to * bounce the buffer due to global resource outage. */ if (err == -ENOSPC) blk_mq_stop_hw_queue(hctx); spin_unlock_irqrestore(&vblk->vqs[qid].lock, flags); virtblk_unmap_data(req, vbr); return virtblk_fail_to_queue(req, err); } if (bd->last && virtqueue_kick_prepare(vblk->vqs[qid].vq)) notify = true; spin_unlock_irqrestore(&vblk->vqs[qid].lock, flags); if (notify) virtqueue_notify(vblk->vqs[qid].vq); return BLK_STS_OK; } static bool virtblk_prep_rq_batch(struct request *req) { struct virtio_blk *vblk = req->mq_hctx->queue->queuedata; struct virtblk_req *vbr = blk_mq_rq_to_pdu(req); return virtblk_prep_rq(req->mq_hctx, vblk, req, vbr) == BLK_STS_OK; } static void virtblk_add_req_batch(struct virtio_blk_vq *vq, struct rq_list *rqlist) { struct request *req; unsigned long flags; bool kick; spin_lock_irqsave(&vq->lock, flags); while ((req = rq_list_pop(rqlist))) { struct virtblk_req *vbr = blk_mq_rq_to_pdu(req); int err; err = virtblk_add_req(vq->vq, vbr); if (err) { virtblk_unmap_data(req, vbr); virtblk_cleanup_cmd(req); blk_mq_requeue_request(req, true); } } kick = virtqueue_kick_prepare(vq->vq); spin_unlock_irqrestore(&vq->lock, flags); if (kick) virtqueue_notify(vq->vq); }