// SPDX-License-Identifier: GPL-2.0-or-later/* * Network block device - make block devices work over TCP * * Note that you can not swap over this thing, yet. Seems to work but * deadlocks sometimes - you can not swap over TCP in general. * * Copyright 1997-2000, 2008 Pavel Machek <pavel@ucw.cz> * Parts copyright 2001 Steven Whitehouse <steve@chygwyn.com> * * (part of code stolen from loop.c) */ #define pr_fmt(fmt) "nbd: " fmt #include <linux/major.h> #include <linux/blkdev.h>#include <linux/module.h>#include <linux/init.h>#include <linux/sched.h>#include <linux/sched/mm.h>#include <linux/fs.h>#include <linux/bio.h>#include <linux/stat.h>#include <linux/errno.h>#include <linux/file.h>#include <linux/ioctl.h>#include <linux/mutex.h>#include <linux/compiler.h>#include <linux/completion.h>#include <linux/err.h>#include <linux/kernel.h>#include <linux/slab.h>#include <net/sock.h>#include <linux/net.h>#include <linux/kthread.h>#include <linux/types.h>#include <linux/debugfs.h>#include <linux/blk-mq.h> #include <linux/uaccess.h>#include <asm/types.h> #include <linux/nbd.h>#include <linux/nbd-netlink.h>#include <net/genetlink.h> #define CREATE_TRACE_POINTS#include <trace/events/nbd.h> static DEFINE_IDR(nbd_index_idr);static DEFINE_MUTEX(nbd_index_mutex);static struct workqueue_struct *nbd_del_wq;static int nbd_total_devices = 0; struct nbd_sock { struct socket *sock; struct mutex tx_lock; struct request *pending; int sent; bool dead; int fallback_index; int cookie; struct work_struct work;}; struct recv_thread_args { struct work_struct work; struct nbd_device *nbd; struct nbd_sock *nsock; int index;}; struct link_dead_args { struct work_struct work; int index;}; #define NBD_RT_TIMEDOUT 0#define NBD_RT_DISCONNECT_REQUESTED 1#define NBD_RT_DISCONNECTED 2#define NBD_RT_HAS_PID_FILE 3#define NBD_RT_HAS_CONFIG_REF 4#define NBD_RT_BOUND 5#define NBD_RT_DISCONNECT_ON_CLOSE 6#define NBD_RT_HAS_BACKEND_FILE 7 #define NBD_DESTROY_ON_DISCONNECT 0#define NBD_DISCONNECT_REQUESTED 1 struct nbd_config { u32 flags; unsigned long runtime_flags; u64 dead_conn_timeout; struct nbd_sock **socks; int num_connections; atomic_t live_connections; wait_queue_head_t conn_wait; atomic_t recv_threads; wait_queue_head_t recv_wq; unsigned int blksize_bits; loff_t bytesize;#if IS_ENABLED(CONFIG_DEBUG_FS) struct dentry *dbg_dir;#endif}; static inline unsigned int nbd_blksize(struct nbd_config *config){ return 1u << config->blksize_bits;} struct nbd_device { struct blk_mq_tag_set tag_set; int index; refcount_t config_refs; refcount_t refs; struct nbd_config *config; struct mutex config_lock; struct gendisk *disk; struct workqueue_struct *recv_workq; struct work_struct remove_work; struct list_head list; struct task_struct *task_setup; unsigned long flags; pid_t pid; /* pid of nbd-client, if attached */ char *backend;}; #define NBD_CMD_REQUEUED 1/* * This flag will be set if nbd_queue_rq() succeed, and will be checked and * cleared in completion. Both setting and clearing of the flag are protected * by cmd->lock. */#define NBD_CMD_INFLIGHT 2 /* Just part of request header or data payload is sent successfully */#define NBD_CMD_PARTIAL_SEND 3 struct nbd_cmd { struct nbd_device *nbd; struct mutex lock; int index; int cookie; int retries; blk_status_t status; unsigned long flags; u32 cmd_cookie;}; #if IS_ENABLED(CONFIG_DEBUG_FS)static struct dentry *nbd_dbg_dir;#endif #define nbd_name(nbd) ((nbd)->disk->disk_name) #define NBD_DEF_BLKSIZE_BITS 10 static unsigned int nbds_max = 16;static int max_part = 16;static int pre_defined_connections = 1;static int part_shift; static int nbd_dev_dbg_init(struct nbd_device *nbd);static void nbd_dev_dbg_close(struct nbd_device *nbd);static void nbd_config_put(struct nbd_device *nbd);static void nbd_connect_reply(struct genl_info *info, int index);static int nbd_genl_status(struct sk_buff *skb, struct genl_info *info);static void nbd_dead_link_work(struct work_struct *work);static void nbd_disconnect_and_put(struct nbd_device *nbd); static inline struct device *nbd_to_dev(struct nbd_device *nbd){ return disk_to_dev(nbd->disk);} static void nbd_requeue_cmd(struct nbd_cmd *cmd){ struct request *req = blk_mq_rq_from_pdu(cmd); lockdep_assert_held(&cmd->lock); /* * Clear INFLIGHT flag so that this cmd won't be completed in * normal completion path * * INFLIGHT flag will be set when the cmd is queued to nbd next * time. */ __clear_bit(NBD_CMD_INFLIGHT, &cmd->flags); if (!test_and_set_bit(NBD_CMD_REQUEUED, &cmd->flags)) blk_mq_requeue_request(req, true);} #define NBD_COOKIE_BITS 32 static u64 nbd_cmd_handle(struct nbd_cmd *cmd){ struct request *req = blk_mq_rq_from_pdu(cmd); u32 tag = blk_mq_unique_tag(req); u64 cookie = cmd->cmd_cookie; return (cookie << NBD_COOKIE_BITS) | tag;} static u32 nbd_handle_to_tag(u64 handle){ return (u32)handle;} static u32 nbd_handle_to_cookie(u64 handle){ return (u32)(handle >> NBD_COOKIE_BITS);} static const char *nbdcmd_to_ascii(int cmd){ switch (cmd) { case NBD_CMD_READ: return "read"; case NBD_CMD_WRITE: return "write"; case NBD_CMD_DISC: return "disconnect"; case NBD_CMD_FLUSH: return "flush"; case NBD_CMD_TRIM: return "trim/discard"; } return "invalid";} static ssize_t pid_show(struct device *dev, struct device_attribute *attr, char *buf){ struct gendisk *disk = dev_to_disk(dev); struct nbd_device *nbd = disk->private_data; return sprintf(buf, "%d\n", nbd->pid);} static const struct device_attribute pid_attr = { .attr = { .name = "pid", .mode = 0444}, .show = pid_show,}; static ssize_t backend_show(struct device *dev, struct device_attribute *attr, char *buf){ struct gendisk *disk = dev_to_disk(dev); struct nbd_device *nbd = disk->private_data; return sprintf(buf, "%s\n", nbd->backend ?: "");} static const struct device_attribute backend_attr = { .attr = { .name = "backend", .mode = 0444}, .show = backend_show,}; static void nbd_dev_remove(struct nbd_device *nbd){ struct gendisk *disk = nbd->disk; del_gendisk(disk); blk_mq_free_tag_set(&nbd->tag_set); /* * Remove from idr after del_gendisk() completes, so if the same ID is * reused, the following add_disk() will succeed. */ mutex_lock(&nbd_index_mutex); idr_remove(&nbd_index_idr, nbd->index); mutex_unlock(&nbd_index_mutex); destroy_workqueue(nbd->recv_workq); put_disk(disk);} static void nbd_dev_remove_work(struct work_struct *work){ nbd_dev_remove(container_of(work, struct nbd_device, remove_work));} static void nbd_put(struct nbd_device *nbd){ if (!refcount_dec_and_test(&nbd->refs)) return; /* Call del_gendisk() asynchrounously to prevent deadlock */ if (test_bit(NBD_DESTROY_ON_DISCONNECT, &nbd->flags)) queue_work(nbd_del_wq, &nbd->remove_work); else nbd_dev_remove(nbd);} static int nbd_disconnected(struct nbd_config *config){ return test_bit(NBD_RT_DISCONNECTED, &config->runtime_flags) || test_bit(NBD_RT_DISCONNECT_REQUESTED, &config->runtime_flags);} static void nbd_mark_nsock_dead(struct nbd_device *nbd, struct nbd_sock *nsock, int notify){ if (!nsock->dead && notify && !nbd_disconnected(nbd->config)) { struct link_dead_args *args; args = kmalloc_obj(struct link_dead_args, GFP_NOIO); if (args) { INIT_WORK(&args->work, nbd_dead_link_work); args->index = nbd->index; queue_work(system_percpu_wq, &args->work); } } if (!nsock->dead) { kernel_sock_shutdown(nsock->sock, SHUT_RDWR); if (atomic_dec_return(&nbd->config->live_connections) == 0) { if (test_and_clear_bit(NBD_RT_DISCONNECT_REQUESTED, &nbd->config->runtime_flags)) { set_bit(NBD_RT_DISCONNECTED, &nbd->config->runtime_flags); dev_info(nbd_to_dev(nbd), "Disconnected due to user request.\n"); } } } nsock->dead = true; nsock->pending = NULL; nsock->sent = 0;} static void nbd_apply_limits(struct queue_limits *lim, u32 flags){ lim->features &= ~(BLK_FEAT_WRITE_CACHE | BLK_FEAT_FUA | BLK_FEAT_ROTATIONAL); lim->max_hw_discard_sectors = 0; lim->max_write_zeroes_sectors = 0; if (flags & NBD_FLAG_SEND_TRIM) lim->max_hw_discard_sectors = UINT_MAX >> SECTOR_SHIFT; if (flags & NBD_FLAG_SEND_FLUSH) { lim->features |= BLK_FEAT_WRITE_CACHE; if (flags & NBD_FLAG_SEND_FUA) lim->features |= BLK_FEAT_FUA; } if (flags & NBD_FLAG_ROTATIONAL) lim->features |= BLK_FEAT_ROTATIONAL; if (flags & NBD_FLAG_SEND_WRITE_ZEROES) lim->max_write_zeroes_sectors = UINT_MAX >> SECTOR_SHIFT;} static int nbd_set_size(struct nbd_device *nbd, loff_t bytesize, loff_t blksize){ struct queue_limits lim; int error; if (!blksize) blksize = 1u << NBD_DEF_BLKSIZE_BITS; if (blk_validate_block_size(blksize)) return -EINVAL; if (bytesize < 0) return -EINVAL; nbd->config->bytesize = bytesize; nbd->config->blksize_bits = __ffs(blksize); if (!nbd->pid) return 0; lim = queue_limits_start_update(nbd->disk->queue); nbd_apply_limits(&lim, nbd->config->flags); lim.logical_block_size = blksize; lim.physical_block_size = blksize; /* No need freeze with 0 capacity and write cache disabled */ if (!get_capacity(nbd->disk) && !blk_queue_write_cache(nbd->disk->queue)) error = queue_limits_commit_update(nbd->disk->queue, &lim); else error = queue_limits_commit_update_frozen(nbd->disk->queue, &lim); if (error) return error; if (max_part) set_bit(GD_NEED_PART_SCAN, &nbd->disk->state); if (!set_capacity_and_notify(nbd->disk, bytesize >> 9)) kobject_uevent(&nbd_to_dev(nbd)->kobj, KOBJ_CHANGE); return 0;} static void nbd_complete_rq(struct request *req){ struct nbd_cmd *cmd = blk_mq_rq_to_pdu(req); dev_dbg(nbd_to_dev(cmd->nbd), "request %p: %s\n", req, cmd->status ? "failed" : "done"); blk_mq_end_request(req, cmd->status);} /* * Forcibly shutdown the socket causing all listeners to error */static void sock_shutdown(struct nbd_device *nbd){ struct nbd_config *config = nbd->config; int i; if (config->num_connections == 0) return; if (test_and_set_bit(NBD_RT_DISCONNECTED, &config->runtime_flags)) return; for (i = 0; i < config->num_connections; i++) { struct nbd_sock *nsock = config->socks[i]; mutex_lock(&nsock->tx_lock); nbd_mark_nsock_dead(nbd, nsock, 0); mutex_unlock(&nsock->tx_lock); } dev_warn(disk_to_dev(nbd->disk), "shutting down sockets\n");} static u32 req_to_nbd_cmd_type(struct request *req){ switch (req_op(req)) { case REQ_OP_DISCARD: return NBD_CMD_TRIM; case REQ_OP_FLUSH: return NBD_CMD_FLUSH; case REQ_OP_WRITE: return NBD_CMD_WRITE; case REQ_OP_READ: return NBD_CMD_READ; case REQ_OP_WRITE_ZEROES: return NBD_CMD_WRITE_ZEROES; default: return U32_MAX; }} static struct nbd_config *nbd_get_config_unlocked(struct nbd_device *nbd){ if (refcount_inc_not_zero(&nbd->config_refs)) { /* * Add smp_mb__after_atomic to ensure that reading nbd->config_refs * and reading nbd->config is ordered. The pair is the barrier in * nbd_alloc_and_init_config(), avoid nbd->config_refs is set * before nbd->config. */ smp_mb__after_atomic(); return nbd->config; } return NULL;} static enum blk_eh_timer_return nbd_xmit_timeout(struct request *req){ struct nbd_cmd *cmd = blk_mq_rq_to_pdu(req); struct nbd_device *nbd = cmd->nbd; struct nbd_config *config; if (!mutex_trylock(&cmd->lock)) return BLK_EH_RESET_TIMER; /* partial send is handled in nbd_sock's work function */ if (test_bit(NBD_CMD_PARTIAL_SEND, &cmd->flags)) { mutex_unlock(&cmd->lock); return BLK_EH_RESET_TIMER; } if (!test_bit(NBD_CMD_INFLIGHT, &cmd->flags)) { mutex_unlock(&cmd->lock); return BLK_EH_DONE; } config = nbd_get_config_unlocked(nbd); if (!config) { cmd->status = BLK_STS_TIMEOUT; __clear_bit(NBD_CMD_INFLIGHT, &cmd->flags); mutex_unlock(&cmd->lock); goto done; } if (config->num_connections > 1 || (config->num_connections == 1 && nbd->tag_set.timeout)) { dev_err_ratelimited(nbd_to_dev(nbd), "Connection timed out, retrying (%d/%d alive)\n", atomic_read(&config->live_connections), config->num_connections); /* * Hooray we have more connections, requeue this IO, the submit * path will put it on a real connection. Or if only one * connection is configured, the submit path will wait util * a new connection is reconfigured or util dead timeout. */ if (config->socks) {Lines 1–500 of 2806. Load more
// SPDX-License-Identifier: GPL-2.0-or-later /* * Network block device - make block devices work over TCP * * Note that you can not swap over this thing, yet. Seems to work but * deadlocks sometimes - you can not swap over TCP in general. * * Copyright 1997-2000, 2008 Pavel Machek <pavel@ucw.cz> * Parts copyright 2001 Steven Whitehouse <steve@chygwyn.com> * * (part of code stolen from loop.c) */ #define pr_fmt(fmt) "nbd: " fmt #include <linux/major.h> #include <linux/blkdev.h> #include <linux/module.h> #include <linux/init.h> #include <linux/sched.h> #include <linux/sched/mm.h> #include <linux/fs.h> #include <linux/bio.h> #include <linux/stat.h> #include <linux/errno.h> #include <linux/file.h> #include <linux/ioctl.h> #include <linux/mutex.h> #include <linux/compiler.h> #include <linux/completion.h> #include <linux/err.h> #include <linux/kernel.h> #include <linux/slab.h> #include <net/sock.h> #include <linux/net.h> #include <linux/kthread.h> #include <linux/types.h> #include <linux/debugfs.h> #include <linux/blk-mq.h> #include <linux/uaccess.h> #include <asm/types.h> #include <linux/nbd.h> #include <linux/nbd-netlink.h> #include <net/genetlink.h> #define CREATE_TRACE_POINTS #include <trace/events/nbd.h> static DEFINE_IDR(nbd_index_idr); static DEFINE_MUTEX(nbd_index_mutex); static struct workqueue_struct *nbd_del_wq; static int nbd_total_devices = 0; struct nbd_sock { struct socket *sock; struct mutex tx_lock; struct request *pending; int sent; bool dead; int fallback_index; int cookie; struct work_struct work; }; struct recv_thread_args { struct work_struct work; struct nbd_device *nbd; struct nbd_sock *nsock; int index; }; struct link_dead_args { struct work_struct work; int index; }; #define NBD_RT_TIMEDOUT 0 #define NBD_RT_DISCONNECT_REQUESTED 1 #define NBD_RT_DISCONNECTED 2 #define NBD_RT_HAS_PID_FILE 3 #define NBD_RT_HAS_CONFIG_REF 4 #define NBD_RT_BOUND 5 #define NBD_RT_DISCONNECT_ON_CLOSE 6 #define NBD_RT_HAS_BACKEND_FILE 7 #define NBD_DESTROY_ON_DISCONNECT 0 #define NBD_DISCONNECT_REQUESTED 1 struct nbd_config { u32 flags; unsigned long runtime_flags; u64 dead_conn_timeout; struct nbd_sock **socks; int num_connections; atomic_t live_connections; wait_queue_head_t conn_wait; atomic_t recv_threads; wait_queue_head_t recv_wq; unsigned int blksize_bits; loff_t bytesize; #if IS_ENABLED(CONFIG_DEBUG_FS) struct dentry *dbg_dir; #endif }; static inline unsigned int nbd_blksize(struct nbd_config *config) { return 1u << config->blksize_bits; } struct nbd_device { struct blk_mq_tag_set tag_set; int index; refcount_t config_refs; refcount_t refs; struct nbd_config *config; struct mutex config_lock; struct gendisk *disk; struct workqueue_struct *recv_workq; struct work_struct remove_work; struct list_head list; struct task_struct *task_setup; unsigned long flags; pid_t pid; /* pid of nbd-client, if attached */ char *backend; }; #define NBD_CMD_REQUEUED 1 /* * This flag will be set if nbd_queue_rq() succeed, and will be checked and * cleared in completion. Both setting and clearing of the flag are protected * by cmd->lock. */ #define NBD_CMD_INFLIGHT 2 /* Just part of request header or data payload is sent successfully */ #define NBD_CMD_PARTIAL_SEND 3 struct nbd_cmd { struct nbd_device *nbd; struct mutex lock; int index; int cookie; int retries; blk_status_t status; unsigned long flags; u32 cmd_cookie; }; #if IS_ENABLED(CONFIG_DEBUG_FS) static struct dentry *nbd_dbg_dir; #endif #define nbd_name(nbd) ((nbd)->disk->disk_name) #define NBD_DEF_BLKSIZE_BITS 10 static unsigned int nbds_max = 16; static int max_part = 16; static int pre_defined_connections = 1; static int part_shift; static int nbd_dev_dbg_init(struct nbd_device *nbd); static void nbd_dev_dbg_close(struct nbd_device *nbd); static void nbd_config_put(struct nbd_device *nbd); static void nbd_connect_reply(struct genl_info *info, int index); static int nbd_genl_status(struct sk_buff *skb, struct genl_info *info); static void nbd_dead_link_work(struct work_struct *work); static void nbd_disconnect_and_put(struct nbd_device *nbd); static inline struct device *nbd_to_dev(struct nbd_device *nbd) { return disk_to_dev(nbd->disk); } static void nbd_requeue_cmd(struct nbd_cmd *cmd) { struct request *req = blk_mq_rq_from_pdu(cmd); lockdep_assert_held(&cmd->lock); /* * Clear INFLIGHT flag so that this cmd won't be completed in * normal completion path * * INFLIGHT flag will be set when the cmd is queued to nbd next * time. */ __clear_bit(NBD_CMD_INFLIGHT, &cmd->flags); if (!test_and_set_bit(NBD_CMD_REQUEUED, &cmd->flags)) blk_mq_requeue_request(req, true); } #define NBD_COOKIE_BITS 32 static u64 nbd_cmd_handle(struct nbd_cmd *cmd) { struct request *req = blk_mq_rq_from_pdu(cmd); u32 tag = blk_mq_unique_tag(req); u64 cookie = cmd->cmd_cookie; return (cookie << NBD_COOKIE_BITS) | tag; } static u32 nbd_handle_to_tag(u64 handle) { return (u32)handle; } static u32 nbd_handle_to_cookie(u64 handle) { return (u32)(handle >> NBD_COOKIE_BITS); } static const char *nbdcmd_to_ascii(int cmd) { switch (cmd) { case NBD_CMD_READ: return "read"; case NBD_CMD_WRITE: return "write"; case NBD_CMD_DISC: return "disconnect"; case NBD_CMD_FLUSH: return "flush"; case NBD_CMD_TRIM: return "trim/discard"; } return "invalid"; } static ssize_t pid_show(struct device *dev, struct device_attribute *attr, char *buf) { struct gendisk *disk = dev_to_disk(dev); struct nbd_device *nbd = disk->private_data; return sprintf(buf, "%d\n", nbd->pid); } static const struct device_attribute pid_attr = { .attr = { .name = "pid", .mode = 0444}, .show = pid_show, }; static ssize_t backend_show(struct device *dev, struct device_attribute *attr, char *buf) { struct gendisk *disk = dev_to_disk(dev); struct nbd_device *nbd = disk->private_data; return sprintf(buf, "%s\n", nbd->backend ?: ""); } static const struct device_attribute backend_attr = { .attr = { .name = "backend", .mode = 0444}, .show = backend_show, }; static void nbd_dev_remove(struct nbd_device *nbd) { struct gendisk *disk = nbd->disk; del_gendisk(disk); blk_mq_free_tag_set(&nbd->tag_set); /* * Remove from idr after del_gendisk() completes, so if the same ID is * reused, the following add_disk() will succeed. */ mutex_lock(&nbd_index_mutex); idr_remove(&nbd_index_idr, nbd->index); mutex_unlock(&nbd_index_mutex); destroy_workqueue(nbd->recv_workq); put_disk(disk); } static void nbd_dev_remove_work(struct work_struct *work) { nbd_dev_remove(container_of(work, struct nbd_device, remove_work)); } static void nbd_put(struct nbd_device *nbd) { if (!refcount_dec_and_test(&nbd->refs)) return; /* Call del_gendisk() asynchrounously to prevent deadlock */ if (test_bit(NBD_DESTROY_ON_DISCONNECT, &nbd->flags)) queue_work(nbd_del_wq, &nbd->remove_work); else nbd_dev_remove(nbd); } static int nbd_disconnected(struct nbd_config *config) { return test_bit(NBD_RT_DISCONNECTED, &config->runtime_flags) || test_bit(NBD_RT_DISCONNECT_REQUESTED, &config->runtime_flags); } static void nbd_mark_nsock_dead(struct nbd_device *nbd, struct nbd_sock *nsock, int notify) { if (!nsock->dead && notify && !nbd_disconnected(nbd->config)) { struct link_dead_args *args; args = kmalloc_obj(struct link_dead_args, GFP_NOIO); if (args) { INIT_WORK(&args->work, nbd_dead_link_work); args->index = nbd->index; queue_work(system_percpu_wq, &args->work); } } if (!nsock->dead) { kernel_sock_shutdown(nsock->sock, SHUT_RDWR); if (atomic_dec_return(&nbd->config->live_connections) == 0) { if (test_and_clear_bit(NBD_RT_DISCONNECT_REQUESTED, &nbd->config->runtime_flags)) { set_bit(NBD_RT_DISCONNECTED, &nbd->config->runtime_flags); dev_info(nbd_to_dev(nbd), "Disconnected due to user request.\n"); } } } nsock->dead = true; nsock->pending = NULL; nsock->sent = 0; } static void nbd_apply_limits(struct queue_limits *lim, u32 flags) { lim->features &= ~(BLK_FEAT_WRITE_CACHE | BLK_FEAT_FUA | BLK_FEAT_ROTATIONAL); lim->max_hw_discard_sectors = 0; lim->max_write_zeroes_sectors = 0; if (flags & NBD_FLAG_SEND_TRIM) lim->max_hw_discard_sectors = UINT_MAX >> SECTOR_SHIFT; if (flags & NBD_FLAG_SEND_FLUSH) { lim->features |= BLK_FEAT_WRITE_CACHE; if (flags & NBD_FLAG_SEND_FUA) lim->features |= BLK_FEAT_FUA; } if (flags & NBD_FLAG_ROTATIONAL) lim->features |= BLK_FEAT_ROTATIONAL; if (flags & NBD_FLAG_SEND_WRITE_ZEROES) lim->max_write_zeroes_sectors = UINT_MAX >> SECTOR_SHIFT; } static int nbd_set_size(struct nbd_device *nbd, loff_t bytesize, loff_t blksize) { struct queue_limits lim; int error; if (!blksize) blksize = 1u << NBD_DEF_BLKSIZE_BITS; if (blk_validate_block_size(blksize)) return -EINVAL; if (bytesize < 0) return -EINVAL; nbd->config->bytesize = bytesize; nbd->config->blksize_bits = __ffs(blksize); if (!nbd->pid) return 0; lim = queue_limits_start_update(nbd->disk->queue); nbd_apply_limits(&lim, nbd->config->flags); lim.logical_block_size = blksize; lim.physical_block_size = blksize; /* No need freeze with 0 capacity and write cache disabled */ if (!get_capacity(nbd->disk) && !blk_queue_write_cache(nbd->disk->queue)) error = queue_limits_commit_update(nbd->disk->queue, &lim); else error = queue_limits_commit_update_frozen(nbd->disk->queue, &lim); if (error) return error; if (max_part) set_bit(GD_NEED_PART_SCAN, &nbd->disk->state); if (!set_capacity_and_notify(nbd->disk, bytesize >> 9)) kobject_uevent(&nbd_to_dev(nbd)->kobj, KOBJ_CHANGE); return 0; } static void nbd_complete_rq(struct request *req) { struct nbd_cmd *cmd = blk_mq_rq_to_pdu(req); dev_dbg(nbd_to_dev(cmd->nbd), "request %p: %s\n", req, cmd->status ? "failed" : "done"); blk_mq_end_request(req, cmd->status); } /* * Forcibly shutdown the socket causing all listeners to error */ static void sock_shutdown(struct nbd_device *nbd) { struct nbd_config *config = nbd->config; int i; if (config->num_connections == 0) return; if (test_and_set_bit(NBD_RT_DISCONNECTED, &config->runtime_flags)) return; for (i = 0; i < config->num_connections; i++) { struct nbd_sock *nsock = config->socks[i]; mutex_lock(&nsock->tx_lock); nbd_mark_nsock_dead(nbd, nsock, 0); mutex_unlock(&nsock->tx_lock); } dev_warn(disk_to_dev(nbd->disk), "shutting down sockets\n"); } static u32 req_to_nbd_cmd_type(struct request *req) { switch (req_op(req)) { case REQ_OP_DISCARD: return NBD_CMD_TRIM; case REQ_OP_FLUSH: return NBD_CMD_FLUSH; case REQ_OP_WRITE: return NBD_CMD_WRITE; case REQ_OP_READ: return NBD_CMD_READ; case REQ_OP_WRITE_ZEROES: return NBD_CMD_WRITE_ZEROES; default: return U32_MAX; } } static struct nbd_config *nbd_get_config_unlocked(struct nbd_device *nbd) { if (refcount_inc_not_zero(&nbd->config_refs)) { /* * Add smp_mb__after_atomic to ensure that reading nbd->config_refs * and reading nbd->config is ordered. The pair is the barrier in * nbd_alloc_and_init_config(), avoid nbd->config_refs is set * before nbd->config. */ smp_mb__after_atomic(); return nbd->config; } return NULL; } static enum blk_eh_timer_return nbd_xmit_timeout(struct request *req) { struct nbd_cmd *cmd = blk_mq_rq_to_pdu(req); struct nbd_device *nbd = cmd->nbd; struct nbd_config *config; if (!mutex_trylock(&cmd->lock)) return BLK_EH_RESET_TIMER; /* partial send is handled in nbd_sock's work function */ if (test_bit(NBD_CMD_PARTIAL_SEND, &cmd->flags)) { mutex_unlock(&cmd->lock); return BLK_EH_RESET_TIMER; } if (!test_bit(NBD_CMD_INFLIGHT, &cmd->flags)) { mutex_unlock(&cmd->lock); return BLK_EH_DONE; } config = nbd_get_config_unlocked(nbd); if (!config) { cmd->status = BLK_STS_TIMEOUT; __clear_bit(NBD_CMD_INFLIGHT, &cmd->flags); mutex_unlock(&cmd->lock); goto done; } if (config->num_connections > 1 || (config->num_connections == 1 && nbd->tag_set.timeout)) { dev_err_ratelimited(nbd_to_dev(nbd), "Connection timed out, retrying (%d/%d alive)\n", atomic_read(&config->live_connections), config->num_connections); /* * Hooray we have more connections, requeue this IO, the submit * path will put it on a real connection. Or if only one * connection is configured, the submit path will wait util * a new connection is reconfigured or util dead timeout. */ if (config->socks) {