/* * Network block device - make block devices work over TCP * * Note that you can not swap over this thing, yet. Seems to work but * deadlocks sometimes - you can not swap over TCP in general. * * Copyright 1997-2000, 2008 Pavel Machek <pavel@ucw.cz> * Parts copyright 2001 Steven Whitehouse <steve@chygwyn.com> * * This file is released under GPLv2 or later. * * (part of code stolen from loop.c) */ #include <linux/major.h> #include <linux/blkdev.h>#include <linux/module.h>#include <linux/init.h>#include <linux/sched.h>#include <linux/sched/mm.h>#include <linux/fs.h>#include <linux/bio.h>#include <linux/stat.h>#include <linux/errno.h>#include <linux/file.h>#include <linux/ioctl.h>#include <linux/mutex.h>#include <linux/compiler.h>#include <linux/err.h>#include <linux/kernel.h>#include <linux/slab.h>#include <net/sock.h>#include <linux/net.h>#include <linux/kthread.h>#include <linux/types.h>#include <linux/debugfs.h>#include <linux/blk-mq.h> #include <linux/uaccess.h>#include <asm/types.h> #include <linux/nbd.h>#include <linux/nbd-netlink.h>#include <net/genetlink.h> static DEFINE_IDR(nbd_index_idr);static DEFINE_MUTEX(nbd_index_mutex);static int nbd_total_devices = 0; struct nbd_sock { struct socket *sock; struct mutex tx_lock; struct request *pending; int sent; bool dead; int fallback_index; int cookie;}; struct recv_thread_args { struct work_struct work; struct nbd_device *nbd; int index;}; struct link_dead_args { struct work_struct work; int index;}; #define NBD_TIMEDOUT 0#define NBD_DISCONNECT_REQUESTED 1#define NBD_DISCONNECTED 2#define NBD_HAS_PID_FILE 3#define NBD_HAS_CONFIG_REF 4#define NBD_BOUND 5#define NBD_DESTROY_ON_DISCONNECT 6 struct nbd_config { u32 flags; unsigned long runtime_flags; u64 dead_conn_timeout; struct nbd_sock **socks; int num_connections; atomic_t live_connections; wait_queue_head_t conn_wait; atomic_t recv_threads; wait_queue_head_t recv_wq; loff_t blksize; loff_t bytesize;#if IS_ENABLED(CONFIG_DEBUG_FS) struct dentry *dbg_dir;#endif}; struct nbd_device { struct blk_mq_tag_set tag_set; int index; refcount_t config_refs; refcount_t refs; struct nbd_config *config; struct mutex config_lock; struct gendisk *disk; struct list_head list; struct task_struct *task_recv; struct task_struct *task_setup;}; struct nbd_cmd { struct nbd_device *nbd; int index; int cookie; struct completion send_complete; blk_status_t status;}; #if IS_ENABLED(CONFIG_DEBUG_FS)static struct dentry *nbd_dbg_dir;#endif #define nbd_name(nbd) ((nbd)->disk->disk_name) #define NBD_MAGIC 0x68797548 static unsigned int nbds_max = 16;static int max_part;static struct workqueue_struct *recv_workqueue;static int part_shift; static int nbd_dev_dbg_init(struct nbd_device *nbd);static void nbd_dev_dbg_close(struct nbd_device *nbd);static void nbd_config_put(struct nbd_device *nbd);static void nbd_connect_reply(struct genl_info *info, int index);static int nbd_genl_status(struct sk_buff *skb, struct genl_info *info);static void nbd_dead_link_work(struct work_struct *work); static inline struct device *nbd_to_dev(struct nbd_device *nbd){ return disk_to_dev(nbd->disk);} static const char *nbdcmd_to_ascii(int cmd){ switch (cmd) { case NBD_CMD_READ: return "read"; case NBD_CMD_WRITE: return "write"; case NBD_CMD_DISC: return "disconnect"; case NBD_CMD_FLUSH: return "flush"; case NBD_CMD_TRIM: return "trim/discard"; } return "invalid";} static ssize_t pid_show(struct device *dev, struct device_attribute *attr, char *buf){ struct gendisk *disk = dev_to_disk(dev); struct nbd_device *nbd = (struct nbd_device *)disk->private_data; return sprintf(buf, "%d\n", task_pid_nr(nbd->task_recv));} static struct device_attribute pid_attr = { .attr = { .name = "pid", .mode = S_IRUGO}, .show = pid_show,}; static void nbd_dev_remove(struct nbd_device *nbd){ struct gendisk *disk = nbd->disk; if (disk) { del_gendisk(disk); blk_cleanup_queue(disk->queue); blk_mq_free_tag_set(&nbd->tag_set); disk->private_data = NULL; put_disk(disk); } kfree(nbd);} static void nbd_put(struct nbd_device *nbd){ if (refcount_dec_and_mutex_lock(&nbd->refs, &nbd_index_mutex)) { idr_remove(&nbd_index_idr, nbd->index); mutex_unlock(&nbd_index_mutex); nbd_dev_remove(nbd); }} static int nbd_disconnected(struct nbd_config *config){ return test_bit(NBD_DISCONNECTED, &config->runtime_flags) || test_bit(NBD_DISCONNECT_REQUESTED, &config->runtime_flags);} static void nbd_mark_nsock_dead(struct nbd_device *nbd, struct nbd_sock *nsock, int notify){ if (!nsock->dead && notify && !nbd_disconnected(nbd->config)) { struct link_dead_args *args; args = kmalloc(sizeof(struct link_dead_args), GFP_NOIO); if (args) { INIT_WORK(&args->work, nbd_dead_link_work); args->index = nbd->index; queue_work(system_wq, &args->work); } } if (!nsock->dead) { kernel_sock_shutdown(nsock->sock, SHUT_RDWR); atomic_dec(&nbd->config->live_connections); } nsock->dead = true; nsock->pending = NULL; nsock->sent = 0;} static void nbd_size_clear(struct nbd_device *nbd){ if (nbd->config->bytesize) { set_capacity(nbd->disk, 0); kobject_uevent(&nbd_to_dev(nbd)->kobj, KOBJ_CHANGE); }} static void nbd_size_update(struct nbd_device *nbd){ struct nbd_config *config = nbd->config; blk_queue_logical_block_size(nbd->disk->queue, config->blksize); blk_queue_physical_block_size(nbd->disk->queue, config->blksize); set_capacity(nbd->disk, config->bytesize >> 9); kobject_uevent(&nbd_to_dev(nbd)->kobj, KOBJ_CHANGE);} static void nbd_size_set(struct nbd_device *nbd, loff_t blocksize, loff_t nr_blocks){ struct nbd_config *config = nbd->config; config->blksize = blocksize; config->bytesize = blocksize * nr_blocks; nbd_size_update(nbd);} static void nbd_complete_rq(struct request *req){ struct nbd_cmd *cmd = blk_mq_rq_to_pdu(req); dev_dbg(nbd_to_dev(cmd->nbd), "request %p: %s\n", cmd, cmd->status ? "failed" : "done"); blk_mq_end_request(req, cmd->status);} /* * Forcibly shutdown the socket causing all listeners to error */static void sock_shutdown(struct nbd_device *nbd){ struct nbd_config *config = nbd->config; int i; if (config->num_connections == 0) return; if (test_and_set_bit(NBD_DISCONNECTED, &config->runtime_flags)) return; for (i = 0; i < config->num_connections; i++) { struct nbd_sock *nsock = config->socks[i]; mutex_lock(&nsock->tx_lock); nbd_mark_nsock_dead(nbd, nsock, 0); mutex_unlock(&nsock->tx_lock); } dev_warn(disk_to_dev(nbd->disk), "shutting down sockets\n");} static enum blk_eh_timer_return nbd_xmit_timeout(struct request *req, bool reserved){ struct nbd_cmd *cmd = blk_mq_rq_to_pdu(req); struct nbd_device *nbd = cmd->nbd; struct nbd_config *config; if (!refcount_inc_not_zero(&nbd->config_refs)) { cmd->status = BLK_STS_TIMEOUT; return BLK_EH_HANDLED; } /* If we are waiting on our dead timer then we could get timeout * callbacks for our request. For this we just want to reset the timer * and let the queue side take care of everything. */ if (!completion_done(&cmd->send_complete)) { nbd_config_put(nbd); return BLK_EH_RESET_TIMER; } config = nbd->config; if (config->num_connections > 1) { dev_err_ratelimited(nbd_to_dev(nbd), "Connection timed out, retrying\n"); /* * Hooray we have more connections, requeue this IO, the submit * path will put it on a real connection. */ if (config->socks && config->num_connections > 1) { if (cmd->index < config->num_connections) { struct nbd_sock *nsock = config->socks[cmd->index]; mutex_lock(&nsock->tx_lock); /* We can have multiple outstanding requests, so * we don't want to mark the nsock dead if we've * already reconnected with a new socket, so * only mark it dead if its the same socket we * were sent out on. */ if (cmd->cookie == nsock->cookie) nbd_mark_nsock_dead(nbd, nsock, 1); mutex_unlock(&nsock->tx_lock); } blk_mq_requeue_request(req, true); nbd_config_put(nbd); return BLK_EH_NOT_HANDLED; } } else { dev_err_ratelimited(nbd_to_dev(nbd), "Connection timed out\n"); } set_bit(NBD_TIMEDOUT, &config->runtime_flags); cmd->status = BLK_STS_IOERR; sock_shutdown(nbd); nbd_config_put(nbd); return BLK_EH_HANDLED;} /* * Send or receive packet. */static int sock_xmit(struct nbd_device *nbd, int index, int send, struct iov_iter *iter, int msg_flags, int *sent){ struct nbd_config *config = nbd->config; struct socket *sock = config->socks[index]->sock; int result; struct msghdr msg; unsigned int noreclaim_flag; if (unlikely(!sock)) { dev_err_ratelimited(disk_to_dev(nbd->disk), "Attempted %s on closed socket in sock_xmit\n", (send ? "send" : "recv")); return -EINVAL; } msg.msg_iter = *iter; noreclaim_flag = memalloc_noreclaim_save(); do { sock->sk->sk_allocation = GFP_NOIO | __GFP_MEMALLOC; msg.msg_name = NULL; msg.msg_namelen = 0; msg.msg_control = NULL; msg.msg_controllen = 0; msg.msg_flags = msg_flags | MSG_NOSIGNAL; if (send) result = sock_sendmsg(sock, &msg); else result = sock_recvmsg(sock, &msg, msg.msg_flags); if (result <= 0) { if (result == 0) result = -EPIPE; /* short read */ break; } if (sent) *sent += result; } while (msg_data_left(&msg)); memalloc_noreclaim_restore(noreclaim_flag); return result;} /* always call with the tx_lock held */static int nbd_send_cmd(struct nbd_device *nbd, struct nbd_cmd *cmd, int index){ struct request *req = blk_mq_rq_from_pdu(cmd); struct nbd_config *config = nbd->config; struct nbd_sock *nsock = config->socks[index]; int result; struct nbd_request request = {.magic = htonl(NBD_REQUEST_MAGIC)}; struct kvec iov = {.iov_base = &request, .iov_len = sizeof(request)}; struct iov_iter from; unsigned long size = blk_rq_bytes(req); struct bio *bio; u32 type; u32 nbd_cmd_flags = 0; u32 tag = blk_mq_unique_tag(req); int sent = nsock->sent, skip = 0; iov_iter_kvec(&from, WRITE | ITER_KVEC, &iov, 1, sizeof(request)); switch (req_op(req)) { case REQ_OP_DISCARD: type = NBD_CMD_TRIM; break; case REQ_OP_FLUSH: type = NBD_CMD_FLUSH; break; case REQ_OP_WRITE: type = NBD_CMD_WRITE; break; case REQ_OP_READ: type = NBD_CMD_READ; break; default: return -EIO; } if (rq_data_dir(req) == WRITE && (config->flags & NBD_FLAG_READ_ONLY)) { dev_err_ratelimited(disk_to_dev(nbd->disk), "Write on read-only\n"); return -EIO; } if (req->cmd_flags & REQ_FUA) nbd_cmd_flags |= NBD_CMD_FLAG_FUA; /* We did a partial send previously, and we at least sent the whole * request struct, so just go and send the rest of the pages in the * request. */ if (sent) { if (sent >= sizeof(request)) { skip = sent - sizeof(request); goto send_pages; } iov_iter_advance(&from, sent); } cmd->index = index; cmd->cookie = nsock->cookie; request.type = htonl(type | nbd_cmd_flags); if (type != NBD_CMD_FLUSH) { request.from = cpu_to_be64((u64)blk_rq_pos(req) << 9); request.len = htonl(size); } memcpy(request.handle, &tag, sizeof(tag)); dev_dbg(nbd_to_dev(nbd), "request %p: sending control (%s@%llu,%uB)\n", cmd, nbdcmd_to_ascii(type), (unsigned long long)blk_rq_pos(req) << 9, blk_rq_bytes(req)); result = sock_xmit(nbd, index, 1, &from, (type == NBD_CMD_WRITE) ? MSG_MORE : 0, &sent); if (result <= 0) { if (result == -ERESTARTSYS) { /* If we havne't sent anything we can just return BUSY, * however if we have sent something we need to make * sure we only allow this req to be sent until we are * completely done. */ if (sent) { nsock->pending = req; nsock->sent = sent; } return BLK_STS_RESOURCE; } dev_err_ratelimited(disk_to_dev(nbd->disk), "Send control failed (result %d)\n", result); return -EAGAIN; }send_pages: if (type != NBD_CMD_WRITE) goto out; bio = req->bio; while (bio) { struct bio *next = bio->bi_next; struct bvec_iter iter; struct bio_vec bvec; bio_for_each_segment(bvec, bio, iter) { bool is_last = !next && bio_iter_last(bvec, iter); int flags = is_last ? 0 : MSG_MORE; dev_dbg(nbd_to_dev(nbd), "request %p: sending %d bytes data\n", cmd, bvec.bv_len); iov_iter_bvec(&from, ITER_BVEC | WRITE, &bvec, 1, bvec.bv_len); if (skip) { if (skip >= iov_iter_count(&from)) { skip -= iov_iter_count(&from); continue; }Lines 1–500 of 2140. Load more
/* * Network block device - make block devices work over TCP * * Note that you can not swap over this thing, yet. Seems to work but * deadlocks sometimes - you can not swap over TCP in general. * * Copyright 1997-2000, 2008 Pavel Machek <pavel@ucw.cz> * Parts copyright 2001 Steven Whitehouse <steve@chygwyn.com> * * This file is released under GPLv2 or later. * * (part of code stolen from loop.c) */ #include <linux/major.h> #include <linux/blkdev.h> #include <linux/module.h> #include <linux/init.h> #include <linux/sched.h> #include <linux/sched/mm.h> #include <linux/fs.h> #include <linux/bio.h> #include <linux/stat.h> #include <linux/errno.h> #include <linux/file.h> #include <linux/ioctl.h> #include <linux/mutex.h> #include <linux/compiler.h> #include <linux/err.h> #include <linux/kernel.h> #include <linux/slab.h> #include <net/sock.h> #include <linux/net.h> #include <linux/kthread.h> #include <linux/types.h> #include <linux/debugfs.h> #include <linux/blk-mq.h> #include <linux/uaccess.h> #include <asm/types.h> #include <linux/nbd.h> #include <linux/nbd-netlink.h> #include <net/genetlink.h> static DEFINE_IDR(nbd_index_idr); static DEFINE_MUTEX(nbd_index_mutex); static int nbd_total_devices = 0; struct nbd_sock { struct socket *sock; struct mutex tx_lock; struct request *pending; int sent; bool dead; int fallback_index; int cookie; }; struct recv_thread_args { struct work_struct work; struct nbd_device *nbd; int index; }; struct link_dead_args { struct work_struct work; int index; }; #define NBD_TIMEDOUT 0 #define NBD_DISCONNECT_REQUESTED 1 #define NBD_DISCONNECTED 2 #define NBD_HAS_PID_FILE 3 #define NBD_HAS_CONFIG_REF 4 #define NBD_BOUND 5 #define NBD_DESTROY_ON_DISCONNECT 6 struct nbd_config { u32 flags; unsigned long runtime_flags; u64 dead_conn_timeout; struct nbd_sock **socks; int num_connections; atomic_t live_connections; wait_queue_head_t conn_wait; atomic_t recv_threads; wait_queue_head_t recv_wq; loff_t blksize; loff_t bytesize; #if IS_ENABLED(CONFIG_DEBUG_FS) struct dentry *dbg_dir; #endif }; struct nbd_device { struct blk_mq_tag_set tag_set; int index; refcount_t config_refs; refcount_t refs; struct nbd_config *config; struct mutex config_lock; struct gendisk *disk; struct list_head list; struct task_struct *task_recv; struct task_struct *task_setup; }; struct nbd_cmd { struct nbd_device *nbd; int index; int cookie; struct completion send_complete; blk_status_t status; }; #if IS_ENABLED(CONFIG_DEBUG_FS) static struct dentry *nbd_dbg_dir; #endif #define nbd_name(nbd) ((nbd)->disk->disk_name) #define NBD_MAGIC 0x68797548 static unsigned int nbds_max = 16; static int max_part; static struct workqueue_struct *recv_workqueue; static int part_shift; static int nbd_dev_dbg_init(struct nbd_device *nbd); static void nbd_dev_dbg_close(struct nbd_device *nbd); static void nbd_config_put(struct nbd_device *nbd); static void nbd_connect_reply(struct genl_info *info, int index); static int nbd_genl_status(struct sk_buff *skb, struct genl_info *info); static void nbd_dead_link_work(struct work_struct *work); static inline struct device *nbd_to_dev(struct nbd_device *nbd) { return disk_to_dev(nbd->disk); } static const char *nbdcmd_to_ascii(int cmd) { switch (cmd) { case NBD_CMD_READ: return "read"; case NBD_CMD_WRITE: return "write"; case NBD_CMD_DISC: return "disconnect"; case NBD_CMD_FLUSH: return "flush"; case NBD_CMD_TRIM: return "trim/discard"; } return "invalid"; } static ssize_t pid_show(struct device *dev, struct device_attribute *attr, char *buf) { struct gendisk *disk = dev_to_disk(dev); struct nbd_device *nbd = (struct nbd_device *)disk->private_data; return sprintf(buf, "%d\n", task_pid_nr(nbd->task_recv)); } static struct device_attribute pid_attr = { .attr = { .name = "pid", .mode = S_IRUGO}, .show = pid_show, }; static void nbd_dev_remove(struct nbd_device *nbd) { struct gendisk *disk = nbd->disk; if (disk) { del_gendisk(disk); blk_cleanup_queue(disk->queue); blk_mq_free_tag_set(&nbd->tag_set); disk->private_data = NULL; put_disk(disk); } kfree(nbd); } static void nbd_put(struct nbd_device *nbd) { if (refcount_dec_and_mutex_lock(&nbd->refs, &nbd_index_mutex)) { idr_remove(&nbd_index_idr, nbd->index); mutex_unlock(&nbd_index_mutex); nbd_dev_remove(nbd); } } static int nbd_disconnected(struct nbd_config *config) { return test_bit(NBD_DISCONNECTED, &config->runtime_flags) || test_bit(NBD_DISCONNECT_REQUESTED, &config->runtime_flags); } static void nbd_mark_nsock_dead(struct nbd_device *nbd, struct nbd_sock *nsock, int notify) { if (!nsock->dead && notify && !nbd_disconnected(nbd->config)) { struct link_dead_args *args; args = kmalloc(sizeof(struct link_dead_args), GFP_NOIO); if (args) { INIT_WORK(&args->work, nbd_dead_link_work); args->index = nbd->index; queue_work(system_wq, &args->work); } } if (!nsock->dead) { kernel_sock_shutdown(nsock->sock, SHUT_RDWR); atomic_dec(&nbd->config->live_connections); } nsock->dead = true; nsock->pending = NULL; nsock->sent = 0; } static void nbd_size_clear(struct nbd_device *nbd) { if (nbd->config->bytesize) { set_capacity(nbd->disk, 0); kobject_uevent(&nbd_to_dev(nbd)->kobj, KOBJ_CHANGE); } } static void nbd_size_update(struct nbd_device *nbd) { struct nbd_config *config = nbd->config; blk_queue_logical_block_size(nbd->disk->queue, config->blksize); blk_queue_physical_block_size(nbd->disk->queue, config->blksize); set_capacity(nbd->disk, config->bytesize >> 9); kobject_uevent(&nbd_to_dev(nbd)->kobj, KOBJ_CHANGE); } static void nbd_size_set(struct nbd_device *nbd, loff_t blocksize, loff_t nr_blocks) { struct nbd_config *config = nbd->config; config->blksize = blocksize; config->bytesize = blocksize * nr_blocks; nbd_size_update(nbd); } static void nbd_complete_rq(struct request *req) { struct nbd_cmd *cmd = blk_mq_rq_to_pdu(req); dev_dbg(nbd_to_dev(cmd->nbd), "request %p: %s\n", cmd, cmd->status ? "failed" : "done"); blk_mq_end_request(req, cmd->status); } /* * Forcibly shutdown the socket causing all listeners to error */ static void sock_shutdown(struct nbd_device *nbd) { struct nbd_config *config = nbd->config; int i; if (config->num_connections == 0) return; if (test_and_set_bit(NBD_DISCONNECTED, &config->runtime_flags)) return; for (i = 0; i < config->num_connections; i++) { struct nbd_sock *nsock = config->socks[i]; mutex_lock(&nsock->tx_lock); nbd_mark_nsock_dead(nbd, nsock, 0); mutex_unlock(&nsock->tx_lock); } dev_warn(disk_to_dev(nbd->disk), "shutting down sockets\n"); } static enum blk_eh_timer_return nbd_xmit_timeout(struct request *req, bool reserved) { struct nbd_cmd *cmd = blk_mq_rq_to_pdu(req); struct nbd_device *nbd = cmd->nbd; struct nbd_config *config; if (!refcount_inc_not_zero(&nbd->config_refs)) { cmd->status = BLK_STS_TIMEOUT; return BLK_EH_HANDLED; } /* If we are waiting on our dead timer then we could get timeout * callbacks for our request. For this we just want to reset the timer * and let the queue side take care of everything. */ if (!completion_done(&cmd->send_complete)) { nbd_config_put(nbd); return BLK_EH_RESET_TIMER; } config = nbd->config; if (config->num_connections > 1) { dev_err_ratelimited(nbd_to_dev(nbd), "Connection timed out, retrying\n"); /* * Hooray we have more connections, requeue this IO, the submit * path will put it on a real connection. */ if (config->socks && config->num_connections > 1) { if (cmd->index < config->num_connections) { struct nbd_sock *nsock = config->socks[cmd->index]; mutex_lock(&nsock->tx_lock); /* We can have multiple outstanding requests, so * we don't want to mark the nsock dead if we've * already reconnected with a new socket, so * only mark it dead if its the same socket we * were sent out on. */ if (cmd->cookie == nsock->cookie) nbd_mark_nsock_dead(nbd, nsock, 1); mutex_unlock(&nsock->tx_lock); } blk_mq_requeue_request(req, true); nbd_config_put(nbd); return BLK_EH_NOT_HANDLED; } } else { dev_err_ratelimited(nbd_to_dev(nbd), "Connection timed out\n"); } set_bit(NBD_TIMEDOUT, &config->runtime_flags); cmd->status = BLK_STS_IOERR; sock_shutdown(nbd); nbd_config_put(nbd); return BLK_EH_HANDLED; } /* * Send or receive packet. */ static int sock_xmit(struct nbd_device *nbd, int index, int send, struct iov_iter *iter, int msg_flags, int *sent) { struct nbd_config *config = nbd->config; struct socket *sock = config->socks[index]->sock; int result; struct msghdr msg; unsigned int noreclaim_flag; if (unlikely(!sock)) { dev_err_ratelimited(disk_to_dev(nbd->disk), "Attempted %s on closed socket in sock_xmit\n", (send ? "send" : "recv")); return -EINVAL; } msg.msg_iter = *iter; noreclaim_flag = memalloc_noreclaim_save(); do { sock->sk->sk_allocation = GFP_NOIO | __GFP_MEMALLOC; msg.msg_name = NULL; msg.msg_namelen = 0; msg.msg_control = NULL; msg.msg_controllen = 0; msg.msg_flags = msg_flags | MSG_NOSIGNAL; if (send) result = sock_sendmsg(sock, &msg); else result = sock_recvmsg(sock, &msg, msg.msg_flags); if (result <= 0) { if (result == 0) result = -EPIPE; /* short read */ break; } if (sent) *sent += result; } while (msg_data_left(&msg)); memalloc_noreclaim_restore(noreclaim_flag); return result; } /* always call with the tx_lock held */ static int nbd_send_cmd(struct nbd_device *nbd, struct nbd_cmd *cmd, int index) { struct request *req = blk_mq_rq_from_pdu(cmd); struct nbd_config *config = nbd->config; struct nbd_sock *nsock = config->socks[index]; int result; struct nbd_request request = {.magic = htonl(NBD_REQUEST_MAGIC)}; struct kvec iov = {.iov_base = &request, .iov_len = sizeof(request)}; struct iov_iter from; unsigned long size = blk_rq_bytes(req); struct bio *bio; u32 type; u32 nbd_cmd_flags = 0; u32 tag = blk_mq_unique_tag(req); int sent = nsock->sent, skip = 0; iov_iter_kvec(&from, WRITE | ITER_KVEC, &iov, 1, sizeof(request)); switch (req_op(req)) { case REQ_OP_DISCARD: type = NBD_CMD_TRIM; break; case REQ_OP_FLUSH: type = NBD_CMD_FLUSH; break; case REQ_OP_WRITE: type = NBD_CMD_WRITE; break; case REQ_OP_READ: type = NBD_CMD_READ; break; default: return -EIO; } if (rq_data_dir(req) == WRITE && (config->flags & NBD_FLAG_READ_ONLY)) { dev_err_ratelimited(disk_to_dev(nbd->disk), "Write on read-only\n"); return -EIO; } if (req->cmd_flags & REQ_FUA) nbd_cmd_flags |= NBD_CMD_FLAG_FUA; /* We did a partial send previously, and we at least sent the whole * request struct, so just go and send the rest of the pages in the * request. */ if (sent) { if (sent >= sizeof(request)) { skip = sent - sizeof(request); goto send_pages; } iov_iter_advance(&from, sent); } cmd->index = index; cmd->cookie = nsock->cookie; request.type = htonl(type | nbd_cmd_flags); if (type != NBD_CMD_FLUSH) { request.from = cpu_to_be64((u64)blk_rq_pos(req) << 9); request.len = htonl(size); } memcpy(request.handle, &tag, sizeof(tag)); dev_dbg(nbd_to_dev(nbd), "request %p: sending control (%s@%llu,%uB)\n", cmd, nbdcmd_to_ascii(type), (unsigned long long)blk_rq_pos(req) << 9, blk_rq_bytes(req)); result = sock_xmit(nbd, index, 1, &from, (type == NBD_CMD_WRITE) ? MSG_MORE : 0, &sent); if (result <= 0) { if (result == -ERESTARTSYS) { /* If we havne't sent anything we can just return BUSY, * however if we have sent something we need to make * sure we only allow this req to be sent until we are * completely done. */ if (sent) { nsock->pending = req; nsock->sent = sent; } return BLK_STS_RESOURCE; } dev_err_ratelimited(disk_to_dev(nbd->disk), "Send control failed (result %d)\n", result); return -EAGAIN; } send_pages: if (type != NBD_CMD_WRITE) goto out; bio = req->bio; while (bio) { struct bio *next = bio->bi_next; struct bvec_iter iter; struct bio_vec bvec; bio_for_each_segment(bvec, bio, iter) { bool is_last = !next && bio_iter_last(bvec, iter); int flags = is_last ? 0 : MSG_MORE; dev_dbg(nbd_to_dev(nbd), "request %p: sending %d bytes data\n", cmd, bvec.bv_len); iov_iter_bvec(&from, ITER_BVEC | WRITE, &bvec, 1, bvec.bv_len); if (skip) { if (skip >= iov_iter_count(&from)) { skip -= iov_iter_count(&from); continue; }