// SPDX-License-Identifier: GPL-2.0-only/* Copyright (C) 2009 Red Hat, Inc. * Copyright (C) 2006 Rusty Russell IBM Corporation * * Author: Michael S. Tsirkin <mst@redhat.com> * * Inspiration, some code, and most witty comments come from * Documentation/virtual/lguest/lguest.c, by Rusty Russell * * Generic code for virtio server in host kernel. */ #include <linux/eventfd.h>#include <linux/vhost.h>#include <linux/uio.h>#include <linux/mm.h>#include <linux/miscdevice.h>#include <linux/mutex.h>#include <linux/poll.h>#include <linux/file.h>#include <linux/highmem.h>#include <linux/slab.h>#include <linux/vmalloc.h>#include <linux/kthread.h>#include <linux/cgroup.h>#include <linux/module.h>#include <linux/sort.h>#include <linux/sched/mm.h>#include <linux/sched/signal.h>#include <linux/sched/vhost_task.h>#include <linux/interval_tree_generic.h>#include <linux/nospec.h>#include <linux/kcov.h> #include "vhost.h" static ushort max_mem_regions = 64;module_param(max_mem_regions, ushort, 0444);MODULE_PARM_DESC(max_mem_regions,	"Maximum number of memory regions in memory map. (default: 64)");static int max_iotlb_entries = 2048;module_param(max_iotlb_entries, int, 0444);MODULE_PARM_DESC(max_iotlb_entries,	"Maximum number of iotlb entries. (default: 2048)");static bool fork_from_owner_default = VHOST_FORK_OWNER_TASK; #ifdef CONFIG_VHOST_ENABLE_FORK_OWNER_CONTROLmodule_param(fork_from_owner_default, bool, 0444);MODULE_PARM_DESC(fork_from_owner_default,		 "Set task mode as the default(default: Y)");#endif enum {	VHOST_MEMORY_F_LOG = 0x1,}; #define vhost_used_event(vq) ((__virtio16 __user *)&vq->avail->ring[vq->num])#define vhost_avail_event(vq) ((__virtio16 __user *)&vq->used->ring[vq->num]) #ifdef CONFIG_VHOST_CROSS_ENDIAN_LEGACYstatic void vhost_disable_cross_endian(struct vhost_virtqueue *vq){	vq->user_be = !virtio_legacy_is_little_endian();} static void vhost_enable_cross_endian_big(struct vhost_virtqueue *vq){	vq->user_be = true;} static void vhost_enable_cross_endian_little(struct vhost_virtqueue *vq){	vq->user_be = false;} static long vhost_set_vring_endian(struct vhost_virtqueue *vq, int __user *argp){	struct vhost_vring_state s; 	if (vq->private_data)		return -EBUSY; 	if (copy_from_user(&s, argp, sizeof(s)))		return -EFAULT; 	if (s.num != VHOST_VRING_LITTLE_ENDIAN &&	    s.num != VHOST_VRING_BIG_ENDIAN)		return -EINVAL; 	if (s.num == VHOST_VRING_BIG_ENDIAN)		vhost_enable_cross_endian_big(vq);	else		vhost_enable_cross_endian_little(vq); 	return 0;} static long vhost_get_vring_endian(struct vhost_virtqueue *vq, u32 idx,				   int __user *argp){	struct vhost_vring_state s = {		.index = idx,		.num = vq->user_be	}; 	if (copy_to_user(argp, &s, sizeof(s)))		return -EFAULT; 	return 0;} static void vhost_init_is_le(struct vhost_virtqueue *vq){	/* Note for legacy virtio: user_be is initialized at reset time	 * according to the host endianness. If userspace does not set an	 * explicit endianness, the default behavior is native endian, as	 * expected by legacy virtio.	 */	vq->is_le = vhost_has_feature(vq, VIRTIO_F_VERSION_1) || !vq->user_be;}#elsestatic void vhost_disable_cross_endian(struct vhost_virtqueue *vq){} static long vhost_set_vring_endian(struct vhost_virtqueue *vq, int __user *argp){	return -ENOIOCTLCMD;} static long vhost_get_vring_endian(struct vhost_virtqueue *vq, u32 idx,				   int __user *argp){	return -ENOIOCTLCMD;} static void vhost_init_is_le(struct vhost_virtqueue *vq){	vq->is_le = vhost_has_feature(vq, VIRTIO_F_VERSION_1)		|| virtio_legacy_is_little_endian();}#endif /* CONFIG_VHOST_CROSS_ENDIAN_LEGACY */ static void vhost_reset_is_le(struct vhost_virtqueue *vq){	vhost_init_is_le(vq);} struct vhost_flush_struct {	struct vhost_work work;	struct completion wait_event;}; static void vhost_flush_work(struct vhost_work *work){	struct vhost_flush_struct *s; 	s = container_of(work, struct vhost_flush_struct, work);	complete(&s->wait_event);} static void vhost_poll_func(struct file *file, wait_queue_head_t *wqh,			    poll_table *pt){	struct vhost_poll *poll; 	poll = container_of(pt, struct vhost_poll, table);	poll->wqh = wqh;	add_wait_queue(wqh, &poll->wait);} static int vhost_poll_wakeup(wait_queue_entry_t *wait, unsigned mode, int sync,			     void *key){	struct vhost_poll *poll = container_of(wait, struct vhost_poll, wait);	struct vhost_work *work = &poll->work; 	if (!(key_to_poll(key) & poll->mask))		return 0; 	if (!poll->dev->use_worker)		work->fn(work);	else		vhost_poll_queue(poll); 	return 0;} void vhost_work_init(struct vhost_work *work, vhost_work_fn_t fn){	clear_bit(VHOST_WORK_QUEUED, &work->flags);	work->fn = fn;}EXPORT_SYMBOL_GPL(vhost_work_init); /* Init poll structure */void vhost_poll_init(struct vhost_poll *poll, vhost_work_fn_t fn,		     __poll_t mask, struct vhost_dev *dev,		     struct vhost_virtqueue *vq){	init_waitqueue_func_entry(&poll->wait, vhost_poll_wakeup);	init_poll_funcptr(&poll->table, vhost_poll_func);	poll->mask = mask;	poll->dev = dev;	poll->wqh = NULL;	poll->vq = vq; 	vhost_work_init(&poll->work, fn);}EXPORT_SYMBOL_GPL(vhost_poll_init); /* Start polling a file. We add ourselves to file's wait queue. The caller must * keep a reference to a file until after vhost_poll_stop is called. */int vhost_poll_start(struct vhost_poll *poll, struct file *file){	__poll_t mask; 	if (poll->wqh)		return 0; 	mask = vfs_poll(file, &poll->table);	if (mask)		vhost_poll_wakeup(&poll->wait, 0, 0, poll_to_key(mask));	if (mask & EPOLLERR) {		vhost_poll_stop(poll);		return -EINVAL;	} 	return 0;}EXPORT_SYMBOL_GPL(vhost_poll_start); /* Stop polling a file. After this function returns, it becomes safe to drop the * file reference. You must also flush afterwards. */void vhost_poll_stop(struct vhost_poll *poll){	if (poll->wqh) {		remove_wait_queue(poll->wqh, &poll->wait);		poll->wqh = NULL;	}}EXPORT_SYMBOL_GPL(vhost_poll_stop); static void vhost_worker_queue(struct vhost_worker *worker,			       struct vhost_work *work){	if (!test_and_set_bit(VHOST_WORK_QUEUED, &work->flags)) {		/* We can only add the work to the list after we're		 * sure it was not in the list.		 * test_and_set_bit() implies a memory barrier.		 */		llist_add(&work->node, &worker->work_list);		worker->ops->wakeup(worker);	}} bool vhost_vq_work_queue(struct vhost_virtqueue *vq, struct vhost_work *work){	struct vhost_worker *worker;	bool queued = false; 	rcu_read_lock();	worker = rcu_dereference(vq->worker);	if (worker) {		queued = true;		vhost_worker_queue(worker, work);	}	rcu_read_unlock(); 	return queued;}EXPORT_SYMBOL_GPL(vhost_vq_work_queue); /** * __vhost_worker_flush - flush a worker * @worker: worker to flush * * The worker's flush_mutex must be held. */static void __vhost_worker_flush(struct vhost_worker *worker){	struct vhost_flush_struct flush; 	if (!worker->attachment_cnt || worker->killed)		return; 	init_completion(&flush.wait_event);	vhost_work_init(&flush.work, vhost_flush_work); 	vhost_worker_queue(worker, &flush.work);	/*	 * Drop mutex in case our worker is killed and it needs to take the	 * mutex to force cleanup.	 */	mutex_unlock(&worker->mutex);	wait_for_completion(&flush.wait_event);	mutex_lock(&worker->mutex);} static void vhost_worker_flush(struct vhost_worker *worker){	mutex_lock(&worker->mutex);	__vhost_worker_flush(worker);	mutex_unlock(&worker->mutex);} void vhost_dev_flush(struct vhost_dev *dev){	struct vhost_worker *worker;	unsigned long i; 	xa_for_each(&dev->worker_xa, i, worker)		vhost_worker_flush(worker);}EXPORT_SYMBOL_GPL(vhost_dev_flush); /* A lockless hint for busy polling code to exit the loop */bool vhost_vq_has_work(struct vhost_virtqueue *vq){	struct vhost_worker *worker;	bool has_work = false; 	rcu_read_lock();	worker = rcu_dereference(vq->worker);	if (worker && !llist_empty(&worker->work_list))		has_work = true;	rcu_read_unlock(); 	return has_work;}EXPORT_SYMBOL_GPL(vhost_vq_has_work); void vhost_poll_queue(struct vhost_poll *poll){	vhost_vq_work_queue(poll->vq, &poll->work);}EXPORT_SYMBOL_GPL(vhost_poll_queue); static void __vhost_vq_meta_reset(struct vhost_virtqueue *vq){	int j; 	for (j = 0; j < VHOST_NUM_ADDRS; j++)		vq->meta_iotlb[j] = NULL;} /* Caller must hold the virtqueue mutex. */static void vhost_vq_invalidate_access(struct vhost_virtqueue *vq){	vq->desc = NULL;	vq->avail = NULL;	vq->used = NULL;	vq->log_used = false;	vq->log_addr = -1ull;	__vhost_vq_meta_reset(vq);} static void vhost_vq_meta_reset(struct vhost_dev *d){	int i; 	for (i = 0; i < d->nvqs; ++i)		__vhost_vq_meta_reset(d->vqs[i]);} static void vhost_vring_call_reset(struct vhost_vring_call *call_ctx){	call_ctx->ctx = NULL;	memset(&call_ctx->producer, 0x0, sizeof(struct irq_bypass_producer));} bool vhost_vq_is_setup(struct vhost_virtqueue *vq){	return vq->avail && vq->desc && vq->used && vhost_vq_access_ok(vq);}EXPORT_SYMBOL_GPL(vhost_vq_is_setup); static void vhost_vq_reset(struct vhost_dev *dev,			   struct vhost_virtqueue *vq){	vq->num = 1;	vq->desc = NULL;	vq->avail = NULL;	vq->used = NULL;	vq->last_avail_idx = 0;	vq->next_avail_head = 0;	vq->avail_idx = 0;	vq->last_used_idx = 0;	vq->signalled_used = 0;	vq->signalled_used_valid = false;	vq->used_flags = 0;	vq->log_used = false;	vq->log_addr = -1ull;	vq->private_data = NULL;	virtio_features_zero(vq->acked_features_array);	vq->acked_backend_features = 0;	vq->log_base = NULL;	vq->error_ctx = NULL;	vq->kick = NULL;	vq->log_ctx = NULL;	vhost_disable_cross_endian(vq);	vhost_reset_is_le(vq);	vq->busyloop_timeout = 0;	vq->umem = NULL;	vq->iotlb = NULL;	vq->iotlb_miss = NULL;	rcu_assign_pointer(vq->worker, NULL);	vhost_vring_call_reset(&vq->call_ctx);	__vhost_vq_meta_reset(vq);} static int vhost_run_work_kthread_list(void *data){	struct vhost_worker *worker = data;	struct vhost_work *work, *work_next;	struct vhost_dev *dev = worker->dev;	struct llist_node *node; 	kthread_use_mm(dev->mm); 	for (;;) {		/* mb paired w/ kthread_stop */		set_current_state(TASK_INTERRUPTIBLE); 		if (kthread_should_stop()) {			__set_current_state(TASK_RUNNING);			break;		}		node = llist_del_all(&worker->work_list);		if (!node)			schedule(); 		node = llist_reverse_order(node);		/* make sure flag is seen after deletion */		smp_wmb();		llist_for_each_entry_safe(work, work_next, node, node) {			clear_bit(VHOST_WORK_QUEUED, &work->flags);			__set_current_state(TASK_RUNNING);			kcov_remote_start_common(worker->kcov_handle);			work->fn(work);			kcov_remote_stop();			cond_resched();		}	}	kthread_unuse_mm(dev->mm); 	return 0;} static bool vhost_run_work_list(void *data){	struct vhost_worker *worker = data;	struct vhost_work *work, *work_next;	struct llist_node *node; 	node = llist_del_all(&worker->work_list);	if (node) {		__set_current_state(TASK_RUNNING); 		node = llist_reverse_order(node);		/* make sure flag is seen after deletion */		smp_wmb();		llist_for_each_entry_safe(work, work_next, node, node) {			clear_bit(VHOST_WORK_QUEUED, &work->flags);			kcov_remote_start_common(worker->kcov_handle);			work->fn(work);			kcov_remote_stop();			cond_resched();		}	} 	return !!node;} static void vhost_worker_killed(void *data){	struct vhost_worker *worker = data;	struct vhost_dev *dev = worker->dev;	struct vhost_virtqueue *vq;	int i, attach_cnt = 0; 	mutex_lock(&worker->mutex);	worker->killed = true; 	for (i = 0; i < dev->nvqs; i++) {		vq = dev->vqs[i]; 		mutex_lock(&vq->mutex);		if (worker ==		    rcu_dereference_check(vq->worker,					  lockdep_is_held(&vq->mutex))) {			rcu_assign_pointer(vq->worker, NULL);			attach_cnt++;		}		mutex_unlock(&vq->mutex);	} 	worker->attachment_cnt -= attach_cnt;	if (attach_cnt)		synchronize_rcu();

Lines 1–500 of 3433. Load more