2427 lines · 60.4 KB · C
RawDownload
// SPDX-License-Identifier: GPL-2.0-or-later/* * Network block device - make block devices work over TCP * * Note that you can not swap over this thing, yet. Seems to work but * deadlocks sometimes - you can not swap over TCP in general. *  * Copyright 1997-2000, 2008 Pavel Machek <pavel@ucw.cz> * Parts copyright 2001 Steven Whitehouse <steve@chygwyn.com> * * (part of code stolen from loop.c) */ #include <linux/major.h> #include <linux/blkdev.h>#include <linux/module.h>#include <linux/init.h>#include <linux/sched.h>#include <linux/sched/mm.h>#include <linux/fs.h>#include <linux/bio.h>#include <linux/stat.h>#include <linux/errno.h>#include <linux/file.h>#include <linux/ioctl.h>#include <linux/mutex.h>#include <linux/compiler.h>#include <linux/completion.h>#include <linux/err.h>#include <linux/kernel.h>#include <linux/slab.h>#include <net/sock.h>#include <linux/net.h>#include <linux/kthread.h>#include <linux/types.h>#include <linux/debugfs.h>#include <linux/blk-mq.h> #include <linux/uaccess.h>#include <asm/types.h> #include <linux/nbd.h>#include <linux/nbd-netlink.h>#include <net/genetlink.h> #define CREATE_TRACE_POINTS#include <trace/events/nbd.h> static DEFINE_IDR(nbd_index_idr);static DEFINE_MUTEX(nbd_index_mutex);static int nbd_total_devices = 0; struct nbd_sock {	struct socket *sock;	struct mutex tx_lock;	struct request *pending;	int sent;	bool dead;	int fallback_index;	int cookie;}; struct recv_thread_args {	struct work_struct work;	struct nbd_device *nbd;	int index;}; struct link_dead_args {	struct work_struct work;	int index;}; #define NBD_RT_TIMEDOUT			0#define NBD_RT_DISCONNECT_REQUESTED	1#define NBD_RT_DISCONNECTED		2#define NBD_RT_HAS_PID_FILE		3#define NBD_RT_HAS_CONFIG_REF		4#define NBD_RT_BOUND			5#define NBD_RT_DISCONNECT_ON_CLOSE	6 #define NBD_DESTROY_ON_DISCONNECT	0#define NBD_DISCONNECT_REQUESTED	1 struct nbd_config {	u32 flags;	unsigned long runtime_flags;	u64 dead_conn_timeout; 	struct nbd_sock **socks;	int num_connections;	atomic_t live_connections;	wait_queue_head_t conn_wait; 	atomic_t recv_threads;	wait_queue_head_t recv_wq;	loff_t blksize;	loff_t bytesize;#if IS_ENABLED(CONFIG_DEBUG_FS)	struct dentry *dbg_dir;#endif}; struct nbd_device {	struct blk_mq_tag_set tag_set; 	int index;	refcount_t config_refs;	refcount_t refs;	struct nbd_config *config;	struct mutex config_lock;	struct gendisk *disk;	struct workqueue_struct *recv_workq; 	struct list_head list;	struct task_struct *task_recv;	struct task_struct *task_setup; 	struct completion *destroy_complete;	unsigned long flags;}; #define NBD_CMD_REQUEUED	1 struct nbd_cmd {	struct nbd_device *nbd;	struct mutex lock;	int index;	int cookie;	int retries;	blk_status_t status;	unsigned long flags;	u32 cmd_cookie;}; #if IS_ENABLED(CONFIG_DEBUG_FS)static struct dentry *nbd_dbg_dir;#endif #define nbd_name(nbd) ((nbd)->disk->disk_name) #define NBD_MAGIC 0x68797548 #define NBD_DEF_BLKSIZE 1024 static unsigned int nbds_max = 16;static int max_part = 16;static int part_shift; static int nbd_dev_dbg_init(struct nbd_device *nbd);static void nbd_dev_dbg_close(struct nbd_device *nbd);static void nbd_config_put(struct nbd_device *nbd);static void nbd_connect_reply(struct genl_info *info, int index);static int nbd_genl_status(struct sk_buff *skb, struct genl_info *info);static void nbd_dead_link_work(struct work_struct *work);static void nbd_disconnect_and_put(struct nbd_device *nbd); static inline struct device *nbd_to_dev(struct nbd_device *nbd){	return disk_to_dev(nbd->disk);} static void nbd_requeue_cmd(struct nbd_cmd *cmd){	struct request *req = blk_mq_rq_from_pdu(cmd); 	if (!test_and_set_bit(NBD_CMD_REQUEUED, &cmd->flags))		blk_mq_requeue_request(req, true);} #define NBD_COOKIE_BITS 32 static u64 nbd_cmd_handle(struct nbd_cmd *cmd){	struct request *req = blk_mq_rq_from_pdu(cmd);	u32 tag = blk_mq_unique_tag(req);	u64 cookie = cmd->cmd_cookie; 	return (cookie << NBD_COOKIE_BITS) | tag;} static u32 nbd_handle_to_tag(u64 handle){	return (u32)handle;} static u32 nbd_handle_to_cookie(u64 handle){	return (u32)(handle >> NBD_COOKIE_BITS);} static const char *nbdcmd_to_ascii(int cmd){	switch (cmd) {	case  NBD_CMD_READ: return "read";	case NBD_CMD_WRITE: return "write";	case  NBD_CMD_DISC: return "disconnect";	case NBD_CMD_FLUSH: return "flush";	case  NBD_CMD_TRIM: return "trim/discard";	}	return "invalid";} static ssize_t pid_show(struct device *dev,			struct device_attribute *attr, char *buf){	struct gendisk *disk = dev_to_disk(dev);	struct nbd_device *nbd = (struct nbd_device *)disk->private_data; 	return sprintf(buf, "%d\n", task_pid_nr(nbd->task_recv));} static const struct device_attribute pid_attr = {	.attr = { .name = "pid", .mode = 0444},	.show = pid_show,}; static void nbd_dev_remove(struct nbd_device *nbd){	struct gendisk *disk = nbd->disk;	struct request_queue *q; 	if (disk) {		q = disk->queue;		del_gendisk(disk);		blk_cleanup_queue(q);		blk_mq_free_tag_set(&nbd->tag_set);		disk->private_data = NULL;		put_disk(disk);	} 	/*	 * Place this in the last just before the nbd is freed to	 * make sure that the disk and the related kobject are also	 * totally removed to avoid duplicate creation of the same	 * one.	 */	if (test_bit(NBD_DESTROY_ON_DISCONNECT, &nbd->flags) && nbd->destroy_complete)		complete(nbd->destroy_complete); 	kfree(nbd);} static void nbd_put(struct nbd_device *nbd){	if (refcount_dec_and_mutex_lock(&nbd->refs,					&nbd_index_mutex)) {		idr_remove(&nbd_index_idr, nbd->index);		nbd_dev_remove(nbd);		mutex_unlock(&nbd_index_mutex);	}} static int nbd_disconnected(struct nbd_config *config){	return test_bit(NBD_RT_DISCONNECTED, &config->runtime_flags) ||		test_bit(NBD_RT_DISCONNECT_REQUESTED, &config->runtime_flags);} static void nbd_mark_nsock_dead(struct nbd_device *nbd, struct nbd_sock *nsock,				int notify){	if (!nsock->dead && notify && !nbd_disconnected(nbd->config)) {		struct link_dead_args *args;		args = kmalloc(sizeof(struct link_dead_args), GFP_NOIO);		if (args) {			INIT_WORK(&args->work, nbd_dead_link_work);			args->index = nbd->index;			queue_work(system_wq, &args->work);		}	}	if (!nsock->dead) {		kernel_sock_shutdown(nsock->sock, SHUT_RDWR);		if (atomic_dec_return(&nbd->config->live_connections) == 0) {			if (test_and_clear_bit(NBD_RT_DISCONNECT_REQUESTED,					       &nbd->config->runtime_flags)) {				set_bit(NBD_RT_DISCONNECTED,					&nbd->config->runtime_flags);				dev_info(nbd_to_dev(nbd),					"Disconnected due to user request.\n");			}		}	}	nsock->dead = true;	nsock->pending = NULL;	nsock->sent = 0;} static void nbd_size_clear(struct nbd_device *nbd){	if (nbd->config->bytesize) {		set_capacity(nbd->disk, 0);		kobject_uevent(&nbd_to_dev(nbd)->kobj, KOBJ_CHANGE);	}} static int nbd_set_size(struct nbd_device *nbd, loff_t bytesize,		loff_t blksize){	if (!blksize)		blksize = NBD_DEF_BLKSIZE;	if (blksize < 512 || blksize > PAGE_SIZE || !is_power_of_2(blksize))		return -EINVAL; 	nbd->config->bytesize = bytesize;	nbd->config->blksize = blksize; 	if (!nbd->task_recv)		return 0; 	if (nbd->config->flags & NBD_FLAG_SEND_TRIM) {		nbd->disk->queue->limits.discard_granularity = blksize;		nbd->disk->queue->limits.discard_alignment = blksize;		blk_queue_max_discard_sectors(nbd->disk->queue, UINT_MAX);	}	blk_queue_logical_block_size(nbd->disk->queue, blksize);	blk_queue_physical_block_size(nbd->disk->queue, blksize); 	if (max_part)		set_bit(GD_NEED_PART_SCAN, &nbd->disk->state);	if (!set_capacity_and_notify(nbd->disk, bytesize >> 9))		kobject_uevent(&nbd_to_dev(nbd)->kobj, KOBJ_CHANGE);	return 0;} static void nbd_complete_rq(struct request *req){	struct nbd_cmd *cmd = blk_mq_rq_to_pdu(req); 	dev_dbg(nbd_to_dev(cmd->nbd), "request %p: %s\n", req,		cmd->status ? "failed" : "done"); 	blk_mq_end_request(req, cmd->status);} /* * Forcibly shutdown the socket causing all listeners to error */static void sock_shutdown(struct nbd_device *nbd){	struct nbd_config *config = nbd->config;	int i; 	if (config->num_connections == 0)		return;	if (test_and_set_bit(NBD_RT_DISCONNECTED, &config->runtime_flags))		return; 	for (i = 0; i < config->num_connections; i++) {		struct nbd_sock *nsock = config->socks[i];		mutex_lock(&nsock->tx_lock);		nbd_mark_nsock_dead(nbd, nsock, 0);		mutex_unlock(&nsock->tx_lock);	}	dev_warn(disk_to_dev(nbd->disk), "shutting down sockets\n");} static u32 req_to_nbd_cmd_type(struct request *req){	switch (req_op(req)) {	case REQ_OP_DISCARD:		return NBD_CMD_TRIM;	case REQ_OP_FLUSH:		return NBD_CMD_FLUSH;	case REQ_OP_WRITE:		return NBD_CMD_WRITE;	case REQ_OP_READ:		return NBD_CMD_READ;	default:		return U32_MAX;	}} static enum blk_eh_timer_return nbd_xmit_timeout(struct request *req,						 bool reserved){	struct nbd_cmd *cmd = blk_mq_rq_to_pdu(req);	struct nbd_device *nbd = cmd->nbd;	struct nbd_config *config; 	if (!mutex_trylock(&cmd->lock))		return BLK_EH_RESET_TIMER; 	if (!refcount_inc_not_zero(&nbd->config_refs)) {		cmd->status = BLK_STS_TIMEOUT;		mutex_unlock(&cmd->lock);		goto done;	}	config = nbd->config; 	if (config->num_connections > 1 ||	    (config->num_connections == 1 && nbd->tag_set.timeout)) {		dev_err_ratelimited(nbd_to_dev(nbd),				    "Connection timed out, retrying (%d/%d alive)\n",				    atomic_read(&config->live_connections),				    config->num_connections);		/*		 * Hooray we have more connections, requeue this IO, the submit		 * path will put it on a real connection. Or if only one		 * connection is configured, the submit path will wait util		 * a new connection is reconfigured or util dead timeout.		 */		if (config->socks) {			if (cmd->index < config->num_connections) {				struct nbd_sock *nsock =					config->socks[cmd->index];				mutex_lock(&nsock->tx_lock);				/* We can have multiple outstanding requests, so				 * we don't want to mark the nsock dead if we've				 * already reconnected with a new socket, so				 * only mark it dead if its the same socket we				 * were sent out on.				 */				if (cmd->cookie == nsock->cookie)					nbd_mark_nsock_dead(nbd, nsock, 1);				mutex_unlock(&nsock->tx_lock);			}			mutex_unlock(&cmd->lock);			nbd_requeue_cmd(cmd);			nbd_config_put(nbd);			return BLK_EH_DONE;		}	} 	if (!nbd->tag_set.timeout) {		/*		 * Userspace sets timeout=0 to disable socket disconnection,		 * so just warn and reset the timer.		 */		struct nbd_sock *nsock = config->socks[cmd->index];		cmd->retries++;		dev_info(nbd_to_dev(nbd), "Possible stuck request %p: control (%s@%llu,%uB). Runtime %u seconds\n",			req, nbdcmd_to_ascii(req_to_nbd_cmd_type(req)),			(unsigned long long)blk_rq_pos(req) << 9,			blk_rq_bytes(req), (req->timeout / HZ) * cmd->retries); 		mutex_lock(&nsock->tx_lock);		if (cmd->cookie != nsock->cookie) {			nbd_requeue_cmd(cmd);			mutex_unlock(&nsock->tx_lock);			mutex_unlock(&cmd->lock);			nbd_config_put(nbd);			return BLK_EH_DONE;		}		mutex_unlock(&nsock->tx_lock);		mutex_unlock(&cmd->lock);		nbd_config_put(nbd);		return BLK_EH_RESET_TIMER;	} 	dev_err_ratelimited(nbd_to_dev(nbd), "Connection timed out\n");	set_bit(NBD_RT_TIMEDOUT, &config->runtime_flags);	cmd->status = BLK_STS_IOERR;	mutex_unlock(&cmd->lock);	sock_shutdown(nbd);	nbd_config_put(nbd);done:	blk_mq_complete_request(req);	return BLK_EH_DONE;} /* *  Send or receive packet. */static int sock_xmit(struct nbd_device *nbd, int index, int send,		     struct iov_iter *iter, int msg_flags, int *sent){	struct nbd_config *config = nbd->config;	struct socket *sock = config->socks[index]->sock;	int result;	struct msghdr msg;	unsigned int noreclaim_flag; 	if (unlikely(!sock)) {		dev_err_ratelimited(disk_to_dev(nbd->disk),			"Attempted %s on closed socket in sock_xmit\n",			(send ? "send" : "recv"));		return -EINVAL;	} 	msg.msg_iter = *iter; 	noreclaim_flag = memalloc_noreclaim_save();	do {		sock->sk->sk_allocation = GFP_NOIO | __GFP_MEMALLOC;		msg.msg_name = NULL;		msg.msg_namelen = 0;		msg.msg_control = NULL;		msg.msg_controllen = 0;		msg.msg_flags = msg_flags | MSG_NOSIGNAL; 		if (send)			result = sock_sendmsg(sock, &msg);		else			result = sock_recvmsg(sock, &msg, msg.msg_flags); 		if (result <= 0) {			if (result == 0)				result = -EPIPE; /* short read */

Lines 1–500 of 2427.

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t