// SPDX-License-Identifier: GPL-2.0+/******************************************************************************* * Vhost kernel TCM fabric driver for virtio SCSI initiators * * (C) Copyright 2010-2013 Datera, Inc. * (C) Copyright 2010-2012 IBM Corp. * * Authors: Nicholas A. Bellinger <nab@daterainc.com> * Stefan Hajnoczi <stefanha@linux.vnet.ibm.com> ****************************************************************************/ #include <linux/module.h>#include <linux/moduleparam.h>#include <generated/utsrelease.h>#include <linux/utsname.h>#include <linux/init.h>#include <linux/slab.h>#include <linux/kthread.h>#include <linux/types.h>#include <linux/string.h>#include <linux/configfs.h>#include <linux/ctype.h>#include <linux/compat.h>#include <linux/eventfd.h>#include <linux/fs.h>#include <linux/vmalloc.h>#include <linux/miscdevice.h>#include <linux/blk_types.h>#include <linux/bio.h>#include <linux/unaligned.h>#include <scsi/scsi_common.h>#include <scsi/scsi_proto.h>#include <target/target_core_base.h>#include <target/target_core_fabric.h>#include <linux/vhost.h>#include <linux/virtio_scsi.h>#include <linux/llist.h>#include <linux/bitmap.h> #include "vhost.h" #define VHOST_SCSI_VERSION "v0.1"#define VHOST_SCSI_NAMELEN 256#define VHOST_SCSI_MAX_CDB_SIZE 32#define VHOST_SCSI_PREALLOC_SGLS 2048#define VHOST_SCSI_PREALLOC_UPAGES 2048#define VHOST_SCSI_PREALLOC_PROT_SGLS 2048/* * For the legacy descriptor case we allocate an iov per byte in the * virtio_scsi_cmd_resp struct. */#define VHOST_SCSI_MAX_RESP_IOVS sizeof(struct virtio_scsi_cmd_resp) static unsigned int vhost_scsi_inline_sg_cnt = VHOST_SCSI_PREALLOC_SGLS; #ifdef CONFIG_ARCH_NO_SG_CHAINstatic int vhost_scsi_set_inline_sg_cnt(const char *buf, const struct kernel_param *kp){ pr_err("Setting inline_sg_cnt is not supported.\n"); return -EOPNOTSUPP;}#elsestatic int vhost_scsi_set_inline_sg_cnt(const char *buf, const struct kernel_param *kp){ unsigned int cnt; int ret; ret = kstrtouint(buf, 10, &cnt); if (ret) return ret; if (cnt > VHOST_SCSI_PREALLOC_SGLS) { pr_err("Max inline_sg_cnt is %u\n", VHOST_SCSI_PREALLOC_SGLS); return -EINVAL; } vhost_scsi_inline_sg_cnt = cnt; return 0;}#endif static int vhost_scsi_get_inline_sg_cnt(char *buf, const struct kernel_param *kp){ return sprintf(buf, "%u\n", vhost_scsi_inline_sg_cnt);} static const struct kernel_param_ops vhost_scsi_inline_sg_cnt_op = { .get = vhost_scsi_get_inline_sg_cnt, .set = vhost_scsi_set_inline_sg_cnt,}; module_param_cb(inline_sg_cnt, &vhost_scsi_inline_sg_cnt_op, NULL, 0644);MODULE_PARM_DESC(inline_sg_cnt, "Set the number of scatterlist entries to pre-allocate. The default is 2048."); /* Max number of requests before requeueing the job. * Using this limit prevents one virtqueue from starving others with * request. */#define VHOST_SCSI_WEIGHT 256 struct vhost_scsi_inflight { /* Wait for the flush operation to finish */ struct completion comp; /* Refcount for the inflight reqs */ struct kref kref;}; struct vhost_scsi_cmd { /* Descriptor from vhost_get_vq_desc() for virt_queue segment */ int tvc_vq_desc; /* The number of scatterlists associated with this cmd */ u32 tvc_sgl_count; u32 tvc_prot_sgl_count; u32 copied_iov:1; const void *read_iov; struct iov_iter *read_iter; struct scatterlist *sgl; struct sg_table table; struct scatterlist *prot_sgl; struct sg_table prot_table; /* Fast path response header iovec used when only one vec is needed */ struct iovec tvc_resp_iov; /* Number of iovs for response */ unsigned int tvc_resp_iovs_cnt; /* Pointer to response header iovecs if more than one is needed */ struct iovec *tvc_resp_iovs; /* Pointer to vhost_virtqueue for the cmd */ struct vhost_virtqueue *tvc_vq; /* The TCM I/O descriptor that is accessed via container_of() */ struct se_cmd tvc_se_cmd; /* Sense buffer that will be mapped into outgoing status */ unsigned char tvc_sense_buf[TRANSPORT_SENSE_BUFFER]; /* * Dirty write descriptors of this command. */ struct vhost_log *tvc_log; unsigned int tvc_log_num; /* Completed commands list, serviced from vhost worker thread */ struct llist_node tvc_completion_list; /* Used to track inflight cmd */ struct vhost_scsi_inflight *inflight;}; struct vhost_scsi_nexus { /* Pointer to TCM session for I_T Nexus */ struct se_session *tvn_se_sess;}; struct vhost_scsi_tpg { /* Vhost port target portal group tag for TCM */ u16 tport_tpgt; /* Used to track number of TPG Port/Lun Links wrt to explicit I_T Nexus shutdown */ int tv_tpg_port_count; /* Used for vhost_scsi device reference to tpg_nexus, protected by tv_tpg_mutex */ int tv_tpg_vhost_count; /* Used for enabling T10-PI with legacy devices */ int tv_fabric_prot_type; /* list for vhost_scsi_list */ struct list_head tv_tpg_list; /* Used to protect access for tpg_nexus */ struct mutex tv_tpg_mutex; /* Pointer to the TCM VHost I_T Nexus for this TPG endpoint */ struct vhost_scsi_nexus *tpg_nexus; /* Pointer back to vhost_scsi_tport */ struct vhost_scsi_tport *tport; /* Returned by vhost_scsi_make_tpg() */ struct se_portal_group se_tpg; /* Pointer back to vhost_scsi, protected by tv_tpg_mutex */ struct vhost_scsi *vhost_scsi;}; struct vhost_scsi_tport { /* SCSI protocol the tport is providing */ u8 tport_proto_id; /* Binary World Wide unique Port Name for Vhost Target port */ u64 tport_wwpn; /* ASCII formatted WWPN for Vhost Target port */ char tport_name[VHOST_SCSI_NAMELEN]; /* Returned by vhost_scsi_make_tport() */ struct se_wwn tport_wwn;}; struct vhost_scsi_evt { /* event to be sent to guest */ struct virtio_scsi_event event; /* event list, serviced from vhost worker thread */ struct llist_node list;}; enum { VHOST_SCSI_VQ_CTL = 0, VHOST_SCSI_VQ_EVT = 1, VHOST_SCSI_VQ_IO = 2,}; /* Note: can't set VIRTIO_F_VERSION_1 yet, since that implies ANY_LAYOUT. */static const int vhost_scsi_bits[] = { VHOST_FEATURES, VIRTIO_SCSI_F_HOTPLUG, VIRTIO_SCSI_F_T10_PI}; #define VHOST_SCSI_FEATURES VHOST_FEATURES_U64(vhost_scsi_bits, 0) #define VHOST_SCSI_MAX_TARGET 256#define VHOST_SCSI_MAX_IO_VQ 1024#define VHOST_SCSI_MAX_EVENT 128 static unsigned vhost_scsi_max_io_vqs = 128; static int vhost_scsi_set_max_io_vqs(const char *val, const struct kernel_param *kp){ unsigned int max_io_vqs; int ret; ret = kstrtouint(val, 0, &max_io_vqs); if (ret) return ret; if (max_io_vqs > VHOST_SCSI_MAX_IO_VQ) { pr_err("Invalid max_io_vqs of %u. Using %u.\n", max_io_vqs, VHOST_SCSI_MAX_IO_VQ); max_io_vqs = VHOST_SCSI_MAX_IO_VQ; } else if (!max_io_vqs) { pr_err("Invalid max_io_vqs of 0. Using 1.\n"); max_io_vqs = 1; } WRITE_ONCE(vhost_scsi_max_io_vqs, max_io_vqs); return 0;} static const struct kernel_param_ops vhost_scsi_max_io_vqs_op = { .set = vhost_scsi_set_max_io_vqs, .get = param_get_uint,}; module_param_cb(max_io_vqs, &vhost_scsi_max_io_vqs_op, &vhost_scsi_max_io_vqs, 0644);MODULE_PARM_DESC(max_io_vqs, "Set the max number of IO virtqueues a vhost scsi device can support. The default is 128. The max is 1024."); struct vhost_scsi_virtqueue { struct vhost_virtqueue vq; struct vhost_scsi *vs; /* * Reference counting for inflight reqs, used for flush operation. At * each time, one reference tracks new commands submitted, while we * wait for another one to reach 0. */ struct vhost_scsi_inflight inflights[2]; /* * Indicate current inflight in use, protected by vq->mutex. * Writers must also take dev mutex and flush under it. */ int inflight_idx; struct vhost_scsi_cmd *scsi_cmds; struct sbitmap scsi_tags; int max_cmds; struct page **upages; struct vhost_work completion_work; struct llist_head completion_list;}; struct vhost_scsi { /* Protected by vhost_scsi->dev.mutex */ struct vhost_scsi_tpg **vs_tpg; char vs_vhost_wwpn[TRANSPORT_IQN_LEN]; struct vhost_dev dev; struct vhost_scsi_virtqueue *vqs; struct vhost_scsi_inflight **old_inflight; struct vhost_work vs_event_work; /* evt injection work item */ struct llist_head vs_event_list; /* evt injection queue */ bool vs_events_missed; /* any missed events, protected by vq->mutex */ int vs_events_nr; /* num of pending events, protected by vq->mutex */ unsigned int inline_sg_cnt;}; struct vhost_scsi_tmf { struct vhost_work vwork; struct work_struct flush_work; struct vhost_scsi *vhost; struct vhost_scsi_virtqueue *svq; struct se_cmd se_cmd; u8 scsi_resp; struct vhost_scsi_inflight *inflight; struct iovec resp_iov; int in_iovs; int vq_desc; /* * Dirty write descriptors of this command. */ struct vhost_log *tmf_log; unsigned int tmf_log_num;}; /* * Context for processing request and control queue operations. */struct vhost_scsi_ctx { int head; unsigned int out, in; size_t req_size, rsp_size; size_t out_size, in_size; u8 *target, *lunp; void *req; struct iov_iter out_iter;}; /* * Global mutex to protect vhost_scsi TPG list for vhost IOCTLs and LIO * configfs management operations. */static DEFINE_MUTEX(vhost_scsi_mutex);static LIST_HEAD(vhost_scsi_list); static void vhost_scsi_done_inflight(struct kref *kref){ struct vhost_scsi_inflight *inflight; inflight = container_of(kref, struct vhost_scsi_inflight, kref); complete(&inflight->comp);} static void vhost_scsi_init_inflight(struct vhost_scsi *vs, struct vhost_scsi_inflight *old_inflight[]){ struct vhost_scsi_inflight *new_inflight; struct vhost_virtqueue *vq; int idx, i; for (i = 0; i < vs->dev.nvqs; i++) { vq = &vs->vqs[i].vq; mutex_lock(&vq->mutex); /* store old inflight */ idx = vs->vqs[i].inflight_idx; if (old_inflight) old_inflight[i] = &vs->vqs[i].inflights[idx]; /* setup new inflight */ vs->vqs[i].inflight_idx = idx ^ 1; new_inflight = &vs->vqs[i].inflights[idx ^ 1]; kref_init(&new_inflight->kref); init_completion(&new_inflight->comp); mutex_unlock(&vq->mutex); }} static struct vhost_scsi_inflight *vhost_scsi_get_inflight(struct vhost_virtqueue *vq){ struct vhost_scsi_inflight *inflight; struct vhost_scsi_virtqueue *svq; svq = container_of(vq, struct vhost_scsi_virtqueue, vq); inflight = &svq->inflights[svq->inflight_idx]; kref_get(&inflight->kref); return inflight;} static void vhost_scsi_put_inflight(struct vhost_scsi_inflight *inflight){ kref_put(&inflight->kref, vhost_scsi_done_inflight);} static int vhost_scsi_check_true(struct se_portal_group *se_tpg){ return 1;} static char *vhost_scsi_get_fabric_wwn(struct se_portal_group *se_tpg){ struct vhost_scsi_tpg *tpg = container_of(se_tpg, struct vhost_scsi_tpg, se_tpg); struct vhost_scsi_tport *tport = tpg->tport; return &tport->tport_name[0];} static u16 vhost_scsi_get_tpgt(struct se_portal_group *se_tpg){ struct vhost_scsi_tpg *tpg = container_of(se_tpg, struct vhost_scsi_tpg, se_tpg); return tpg->tport_tpgt;} static int vhost_scsi_check_prot_fabric_only(struct se_portal_group *se_tpg){ struct vhost_scsi_tpg *tpg = container_of(se_tpg, struct vhost_scsi_tpg, se_tpg); return tpg->tv_fabric_prot_type;} static int vhost_scsi_copy_cmd_log(struct vhost_virtqueue *vq, struct vhost_scsi_cmd *cmd, struct vhost_log *log, unsigned int log_num){ if (!cmd->tvc_log) cmd->tvc_log = kmalloc_objs(*cmd->tvc_log, vq->dev->iov_limit); if (unlikely(!cmd->tvc_log)) { vq_err(vq, "Failed to alloc tvc_log\n"); return -ENOMEM; } memcpy(cmd->tvc_log, log, sizeof(*cmd->tvc_log) * log_num); cmd->tvc_log_num = log_num; return 0;} static void vhost_scsi_log_write(struct vhost_virtqueue *vq, struct vhost_log *log, unsigned int log_num){ if (likely(!vhost_has_feature(vq, VHOST_F_LOG_ALL))) return; if (likely(!log_num || !log)) return; /* * vhost-scsi doesn't support VIRTIO_F_ACCESS_PLATFORM. * No requirement for vq->iotlb case. */ WARN_ON_ONCE(unlikely(vq->iotlb)); vhost_log_write(vq, log, log_num, U64_MAX, NULL, 0);} static void vhost_scsi_release_cmd_res(struct se_cmd *se_cmd){ struct vhost_scsi_cmd *tv_cmd = container_of(se_cmd, struct vhost_scsi_cmd, tvc_se_cmd); struct vhost_scsi_virtqueue *svq = container_of(tv_cmd->tvc_vq, struct vhost_scsi_virtqueue, vq); struct vhost_scsi *vs = svq->vs; struct vhost_scsi_inflight *inflight = tv_cmd->inflight; struct scatterlist *sg; struct page *page; int i; if (tv_cmd->tvc_sgl_count) { for_each_sgtable_sg(&tv_cmd->table, sg, i) { page = sg_page(sg); if (!page) continue; if (tv_cmd->copied_iov) __free_page(page); else put_page(page); } kfree(tv_cmd->read_iter); kfree(tv_cmd->read_iov); sg_free_table_chained(&tv_cmd->table, vs->inline_sg_cnt); } if (tv_cmd->tvc_prot_sgl_count) { for_each_sgtable_sg(&tv_cmd->prot_table, sg, i) { page = sg_page(sg); if (page) put_page(page); } sg_free_table_chained(&tv_cmd->prot_table, vs->inline_sg_cnt); } if (tv_cmd->tvc_resp_iovs != &tv_cmd->tvc_resp_iov) kfree(tv_cmd->tvc_resp_iovs); sbitmap_clear_bit(&svq->scsi_tags, se_cmd->map_tag); vhost_scsi_put_inflight(inflight);} static void vhost_scsi_release_tmf_res(struct vhost_scsi_tmf *tmf){ struct vhost_scsi_inflight *inflight = tmf->inflight; /* * tmf->tmf_log is default NULL unless VHOST_F_LOG_ALL is set. */ kfree(tmf->tmf_log); kfree(tmf); vhost_scsi_put_inflight(inflight);} static void vhost_scsi_drop_cmds(struct vhost_scsi_virtqueue *svq)Lines 1–500 of 3034. Load more
// SPDX-License-Identifier: GPL-2.0+ /******************************************************************************* * Vhost kernel TCM fabric driver for virtio SCSI initiators * * (C) Copyright 2010-2013 Datera, Inc. * (C) Copyright 2010-2012 IBM Corp. * * Authors: Nicholas A. Bellinger <nab@daterainc.com> * Stefan Hajnoczi <stefanha@linux.vnet.ibm.com> ****************************************************************************/ #include <linux/module.h> #include <linux/moduleparam.h> #include <generated/utsrelease.h> #include <linux/utsname.h> #include <linux/init.h> #include <linux/slab.h> #include <linux/kthread.h> #include <linux/types.h> #include <linux/string.h> #include <linux/configfs.h> #include <linux/ctype.h> #include <linux/compat.h> #include <linux/eventfd.h> #include <linux/fs.h> #include <linux/vmalloc.h> #include <linux/miscdevice.h> #include <linux/blk_types.h> #include <linux/bio.h> #include <linux/unaligned.h> #include <scsi/scsi_common.h> #include <scsi/scsi_proto.h> #include <target/target_core_base.h> #include <target/target_core_fabric.h> #include <linux/vhost.h> #include <linux/virtio_scsi.h> #include <linux/llist.h> #include <linux/bitmap.h> #include "vhost.h" #define VHOST_SCSI_VERSION "v0.1" #define VHOST_SCSI_NAMELEN 256 #define VHOST_SCSI_MAX_CDB_SIZE 32 #define VHOST_SCSI_PREALLOC_SGLS 2048 #define VHOST_SCSI_PREALLOC_UPAGES 2048 #define VHOST_SCSI_PREALLOC_PROT_SGLS 2048 /* * For the legacy descriptor case we allocate an iov per byte in the * virtio_scsi_cmd_resp struct. */ #define VHOST_SCSI_MAX_RESP_IOVS sizeof(struct virtio_scsi_cmd_resp) static unsigned int vhost_scsi_inline_sg_cnt = VHOST_SCSI_PREALLOC_SGLS; #ifdef CONFIG_ARCH_NO_SG_CHAIN static int vhost_scsi_set_inline_sg_cnt(const char *buf, const struct kernel_param *kp) { pr_err("Setting inline_sg_cnt is not supported.\n"); return -EOPNOTSUPP; } #else static int vhost_scsi_set_inline_sg_cnt(const char *buf, const struct kernel_param *kp) { unsigned int cnt; int ret; ret = kstrtouint(buf, 10, &cnt); if (ret) return ret; if (cnt > VHOST_SCSI_PREALLOC_SGLS) { pr_err("Max inline_sg_cnt is %u\n", VHOST_SCSI_PREALLOC_SGLS); return -EINVAL; } vhost_scsi_inline_sg_cnt = cnt; return 0; } #endif static int vhost_scsi_get_inline_sg_cnt(char *buf, const struct kernel_param *kp) { return sprintf(buf, "%u\n", vhost_scsi_inline_sg_cnt); } static const struct kernel_param_ops vhost_scsi_inline_sg_cnt_op = { .get = vhost_scsi_get_inline_sg_cnt, .set = vhost_scsi_set_inline_sg_cnt, }; module_param_cb(inline_sg_cnt, &vhost_scsi_inline_sg_cnt_op, NULL, 0644); MODULE_PARM_DESC(inline_sg_cnt, "Set the number of scatterlist entries to pre-allocate. The default is 2048."); /* Max number of requests before requeueing the job. * Using this limit prevents one virtqueue from starving others with * request. */ #define VHOST_SCSI_WEIGHT 256 struct vhost_scsi_inflight { /* Wait for the flush operation to finish */ struct completion comp; /* Refcount for the inflight reqs */ struct kref kref; }; struct vhost_scsi_cmd { /* Descriptor from vhost_get_vq_desc() for virt_queue segment */ int tvc_vq_desc; /* The number of scatterlists associated with this cmd */ u32 tvc_sgl_count; u32 tvc_prot_sgl_count; u32 copied_iov:1; const void *read_iov; struct iov_iter *read_iter; struct scatterlist *sgl; struct sg_table table; struct scatterlist *prot_sgl; struct sg_table prot_table; /* Fast path response header iovec used when only one vec is needed */ struct iovec tvc_resp_iov; /* Number of iovs for response */ unsigned int tvc_resp_iovs_cnt; /* Pointer to response header iovecs if more than one is needed */ struct iovec *tvc_resp_iovs; /* Pointer to vhost_virtqueue for the cmd */ struct vhost_virtqueue *tvc_vq; /* The TCM I/O descriptor that is accessed via container_of() */ struct se_cmd tvc_se_cmd; /* Sense buffer that will be mapped into outgoing status */ unsigned char tvc_sense_buf[TRANSPORT_SENSE_BUFFER]; /* * Dirty write descriptors of this command. */ struct vhost_log *tvc_log; unsigned int tvc_log_num; /* Completed commands list, serviced from vhost worker thread */ struct llist_node tvc_completion_list; /* Used to track inflight cmd */ struct vhost_scsi_inflight *inflight; }; struct vhost_scsi_nexus { /* Pointer to TCM session for I_T Nexus */ struct se_session *tvn_se_sess; }; struct vhost_scsi_tpg { /* Vhost port target portal group tag for TCM */ u16 tport_tpgt; /* Used to track number of TPG Port/Lun Links wrt to explicit I_T Nexus shutdown */ int tv_tpg_port_count; /* Used for vhost_scsi device reference to tpg_nexus, protected by tv_tpg_mutex */ int tv_tpg_vhost_count; /* Used for enabling T10-PI with legacy devices */ int tv_fabric_prot_type; /* list for vhost_scsi_list */ struct list_head tv_tpg_list; /* Used to protect access for tpg_nexus */ struct mutex tv_tpg_mutex; /* Pointer to the TCM VHost I_T Nexus for this TPG endpoint */ struct vhost_scsi_nexus *tpg_nexus; /* Pointer back to vhost_scsi_tport */ struct vhost_scsi_tport *tport; /* Returned by vhost_scsi_make_tpg() */ struct se_portal_group se_tpg; /* Pointer back to vhost_scsi, protected by tv_tpg_mutex */ struct vhost_scsi *vhost_scsi; }; struct vhost_scsi_tport { /* SCSI protocol the tport is providing */ u8 tport_proto_id; /* Binary World Wide unique Port Name for Vhost Target port */ u64 tport_wwpn; /* ASCII formatted WWPN for Vhost Target port */ char tport_name[VHOST_SCSI_NAMELEN]; /* Returned by vhost_scsi_make_tport() */ struct se_wwn tport_wwn; }; struct vhost_scsi_evt { /* event to be sent to guest */ struct virtio_scsi_event event; /* event list, serviced from vhost worker thread */ struct llist_node list; }; enum { VHOST_SCSI_VQ_CTL = 0, VHOST_SCSI_VQ_EVT = 1, VHOST_SCSI_VQ_IO = 2, }; /* Note: can't set VIRTIO_F_VERSION_1 yet, since that implies ANY_LAYOUT. */ static const int vhost_scsi_bits[] = { VHOST_FEATURES, VIRTIO_SCSI_F_HOTPLUG, VIRTIO_SCSI_F_T10_PI }; #define VHOST_SCSI_FEATURES VHOST_FEATURES_U64(vhost_scsi_bits, 0) #define VHOST_SCSI_MAX_TARGET 256 #define VHOST_SCSI_MAX_IO_VQ 1024 #define VHOST_SCSI_MAX_EVENT 128 static unsigned vhost_scsi_max_io_vqs = 128; static int vhost_scsi_set_max_io_vqs(const char *val, const struct kernel_param *kp) { unsigned int max_io_vqs; int ret; ret = kstrtouint(val, 0, &max_io_vqs); if (ret) return ret; if (max_io_vqs > VHOST_SCSI_MAX_IO_VQ) { pr_err("Invalid max_io_vqs of %u. Using %u.\n", max_io_vqs, VHOST_SCSI_MAX_IO_VQ); max_io_vqs = VHOST_SCSI_MAX_IO_VQ; } else if (!max_io_vqs) { pr_err("Invalid max_io_vqs of 0. Using 1.\n"); max_io_vqs = 1; } WRITE_ONCE(vhost_scsi_max_io_vqs, max_io_vqs); return 0; } static const struct kernel_param_ops vhost_scsi_max_io_vqs_op = { .set = vhost_scsi_set_max_io_vqs, .get = param_get_uint, }; module_param_cb(max_io_vqs, &vhost_scsi_max_io_vqs_op, &vhost_scsi_max_io_vqs, 0644); MODULE_PARM_DESC(max_io_vqs, "Set the max number of IO virtqueues a vhost scsi device can support. The default is 128. The max is 1024."); struct vhost_scsi_virtqueue { struct vhost_virtqueue vq; struct vhost_scsi *vs; /* * Reference counting for inflight reqs, used for flush operation. At * each time, one reference tracks new commands submitted, while we * wait for another one to reach 0. */ struct vhost_scsi_inflight inflights[2]; /* * Indicate current inflight in use, protected by vq->mutex. * Writers must also take dev mutex and flush under it. */ int inflight_idx; struct vhost_scsi_cmd *scsi_cmds; struct sbitmap scsi_tags; int max_cmds; struct page **upages; struct vhost_work completion_work; struct llist_head completion_list; }; struct vhost_scsi { /* Protected by vhost_scsi->dev.mutex */ struct vhost_scsi_tpg **vs_tpg; char vs_vhost_wwpn[TRANSPORT_IQN_LEN]; struct vhost_dev dev; struct vhost_scsi_virtqueue *vqs; struct vhost_scsi_inflight **old_inflight; struct vhost_work vs_event_work; /* evt injection work item */ struct llist_head vs_event_list; /* evt injection queue */ bool vs_events_missed; /* any missed events, protected by vq->mutex */ int vs_events_nr; /* num of pending events, protected by vq->mutex */ unsigned int inline_sg_cnt; }; struct vhost_scsi_tmf { struct vhost_work vwork; struct work_struct flush_work; struct vhost_scsi *vhost; struct vhost_scsi_virtqueue *svq; struct se_cmd se_cmd; u8 scsi_resp; struct vhost_scsi_inflight *inflight; struct iovec resp_iov; int in_iovs; int vq_desc; /* * Dirty write descriptors of this command. */ struct vhost_log *tmf_log; unsigned int tmf_log_num; }; /* * Context for processing request and control queue operations. */ struct vhost_scsi_ctx { int head; unsigned int out, in; size_t req_size, rsp_size; size_t out_size, in_size; u8 *target, *lunp; void *req; struct iov_iter out_iter; }; /* * Global mutex to protect vhost_scsi TPG list for vhost IOCTLs and LIO * configfs management operations. */ static DEFINE_MUTEX(vhost_scsi_mutex); static LIST_HEAD(vhost_scsi_list); static void vhost_scsi_done_inflight(struct kref *kref) { struct vhost_scsi_inflight *inflight; inflight = container_of(kref, struct vhost_scsi_inflight, kref); complete(&inflight->comp); } static void vhost_scsi_init_inflight(struct vhost_scsi *vs, struct vhost_scsi_inflight *old_inflight[]) { struct vhost_scsi_inflight *new_inflight; struct vhost_virtqueue *vq; int idx, i; for (i = 0; i < vs->dev.nvqs; i++) { vq = &vs->vqs[i].vq; mutex_lock(&vq->mutex); /* store old inflight */ idx = vs->vqs[i].inflight_idx; if (old_inflight) old_inflight[i] = &vs->vqs[i].inflights[idx]; /* setup new inflight */ vs->vqs[i].inflight_idx = idx ^ 1; new_inflight = &vs->vqs[i].inflights[idx ^ 1]; kref_init(&new_inflight->kref); init_completion(&new_inflight->comp); mutex_unlock(&vq->mutex); } } static struct vhost_scsi_inflight * vhost_scsi_get_inflight(struct vhost_virtqueue *vq) { struct vhost_scsi_inflight *inflight; struct vhost_scsi_virtqueue *svq; svq = container_of(vq, struct vhost_scsi_virtqueue, vq); inflight = &svq->inflights[svq->inflight_idx]; kref_get(&inflight->kref); return inflight; } static void vhost_scsi_put_inflight(struct vhost_scsi_inflight *inflight) { kref_put(&inflight->kref, vhost_scsi_done_inflight); } static int vhost_scsi_check_true(struct se_portal_group *se_tpg) { return 1; } static char *vhost_scsi_get_fabric_wwn(struct se_portal_group *se_tpg) { struct vhost_scsi_tpg *tpg = container_of(se_tpg, struct vhost_scsi_tpg, se_tpg); struct vhost_scsi_tport *tport = tpg->tport; return &tport->tport_name[0]; } static u16 vhost_scsi_get_tpgt(struct se_portal_group *se_tpg) { struct vhost_scsi_tpg *tpg = container_of(se_tpg, struct vhost_scsi_tpg, se_tpg); return tpg->tport_tpgt; } static int vhost_scsi_check_prot_fabric_only(struct se_portal_group *se_tpg) { struct vhost_scsi_tpg *tpg = container_of(se_tpg, struct vhost_scsi_tpg, se_tpg); return tpg->tv_fabric_prot_type; } static int vhost_scsi_copy_cmd_log(struct vhost_virtqueue *vq, struct vhost_scsi_cmd *cmd, struct vhost_log *log, unsigned int log_num) { if (!cmd->tvc_log) cmd->tvc_log = kmalloc_objs(*cmd->tvc_log, vq->dev->iov_limit); if (unlikely(!cmd->tvc_log)) { vq_err(vq, "Failed to alloc tvc_log\n"); return -ENOMEM; } memcpy(cmd->tvc_log, log, sizeof(*cmd->tvc_log) * log_num); cmd->tvc_log_num = log_num; return 0; } static void vhost_scsi_log_write(struct vhost_virtqueue *vq, struct vhost_log *log, unsigned int log_num) { if (likely(!vhost_has_feature(vq, VHOST_F_LOG_ALL))) return; if (likely(!log_num || !log)) return; /* * vhost-scsi doesn't support VIRTIO_F_ACCESS_PLATFORM. * No requirement for vq->iotlb case. */ WARN_ON_ONCE(unlikely(vq->iotlb)); vhost_log_write(vq, log, log_num, U64_MAX, NULL, 0); } static void vhost_scsi_release_cmd_res(struct se_cmd *se_cmd) { struct vhost_scsi_cmd *tv_cmd = container_of(se_cmd, struct vhost_scsi_cmd, tvc_se_cmd); struct vhost_scsi_virtqueue *svq = container_of(tv_cmd->tvc_vq, struct vhost_scsi_virtqueue, vq); struct vhost_scsi *vs = svq->vs; struct vhost_scsi_inflight *inflight = tv_cmd->inflight; struct scatterlist *sg; struct page *page; int i; if (tv_cmd->tvc_sgl_count) { for_each_sgtable_sg(&tv_cmd->table, sg, i) { page = sg_page(sg); if (!page) continue; if (tv_cmd->copied_iov) __free_page(page); else put_page(page); } kfree(tv_cmd->read_iter); kfree(tv_cmd->read_iov); sg_free_table_chained(&tv_cmd->table, vs->inline_sg_cnt); } if (tv_cmd->tvc_prot_sgl_count) { for_each_sgtable_sg(&tv_cmd->prot_table, sg, i) { page = sg_page(sg); if (page) put_page(page); } sg_free_table_chained(&tv_cmd->prot_table, vs->inline_sg_cnt); } if (tv_cmd->tvc_resp_iovs != &tv_cmd->tvc_resp_iov) kfree(tv_cmd->tvc_resp_iovs); sbitmap_clear_bit(&svq->scsi_tags, se_cmd->map_tag); vhost_scsi_put_inflight(inflight); } static void vhost_scsi_release_tmf_res(struct vhost_scsi_tmf *tmf) { struct vhost_scsi_inflight *inflight = tmf->inflight; /* * tmf->tmf_log is default NULL unless VHOST_F_LOG_ALL is set. */ kfree(tmf->tmf_log); kfree(tmf); vhost_scsi_put_inflight(inflight); } static void vhost_scsi_drop_cmds(struct vhost_scsi_virtqueue *svq)