// SPDX-License-Identifier: GPL-2.0-only/* * VMware VMCI Driver * * Copyright (C) 2012 VMware, Inc. All rights reserved. */ #include <linux/vmw_vmci_defs.h>#include <linux/vmw_vmci_api.h>#include <linux/highmem.h>#include <linux/kernel.h>#include <linux/mm.h>#include <linux/module.h>#include <linux/mutex.h>#include <linux/pagemap.h>#include <linux/pci.h>#include <linux/sched.h>#include <linux/slab.h>#include <linux/uio.h>#include <linux/wait.h>#include <linux/vmalloc.h>#include <linux/skbuff.h> #include "vmci_handle_array.h"#include "vmci_queue_pair.h"#include "vmci_datagram.h"#include "vmci_resource.h"#include "vmci_context.h"#include "vmci_driver.h"#include "vmci_event.h"#include "vmci_route.h" /* * In the following, we will distinguish between two kinds of VMX processes - * the ones with versions lower than VMCI_VERSION_NOVMVM that use specialized * VMCI page files in the VMX and supporting VM to VM communication and the * newer ones that use the guest memory directly. We will in the following * refer to the older VMX versions as old-style VMX'en, and the newer ones as * new-style VMX'en. * * The state transition datagram is as follows (the VMCIQPB_ prefix has been * removed for readability) - see below for more details on the transtions: * *            --------------  NEW  ------------- *            |                                | *           \_/                              \_/ *     CREATED_NO_MEM <-----------------> CREATED_MEM *            |    |                           | *            |    o-----------------------o   | *            |                            |   | *           \_/                          \_/ \_/ *     ATTACHED_NO_MEM <----------------> ATTACHED_MEM *            |                            |   | *            |     o----------------------o   | *            |     |                          | *           \_/   \_/                        \_/ *     SHUTDOWN_NO_MEM <----------------> SHUTDOWN_MEM *            |                                | *            |                                | *            -------------> gone <------------- * * In more detail. When a VMCI queue pair is first created, it will be in the * VMCIQPB_NEW state. It will then move into one of the following states: * * - VMCIQPB_CREATED_NO_MEM: this state indicates that either: * *     - the created was performed by a host endpoint, in which case there is *       no backing memory yet. * *     - the create was initiated by an old-style VMX, that uses *       vmci_qp_broker_set_page_store to specify the UVAs of the queue pair at *       a later point in time. This state can be distinguished from the one *       above by the context ID of the creator. A host side is not allowed to *       attach until the page store has been set. * * - VMCIQPB_CREATED_MEM: this state is the result when the queue pair *     is created by a VMX using the queue pair device backend that *     sets the UVAs of the queue pair immediately and stores the *     information for later attachers. At this point, it is ready for *     the host side to attach to it. * * Once the queue pair is in one of the created states (with the exception of * the case mentioned for older VMX'en above), it is possible to attach to the * queue pair. Again we have two new states possible: * * - VMCIQPB_ATTACHED_MEM: this state can be reached through the following *   paths: * *     - from VMCIQPB_CREATED_NO_MEM when a new-style VMX allocates a queue *       pair, and attaches to a queue pair previously created by the host side. * *     - from VMCIQPB_CREATED_MEM when the host side attaches to a queue pair *       already created by a guest. * *     - from VMCIQPB_ATTACHED_NO_MEM, when an old-style VMX calls *       vmci_qp_broker_set_page_store (see below). * * - VMCIQPB_ATTACHED_NO_MEM: If the queue pair already was in the *     VMCIQPB_CREATED_NO_MEM due to a host side create, an old-style VMX will *     bring the queue pair into this state. Once vmci_qp_broker_set_page_store *     is called to register the user memory, the VMCIQPB_ATTACH_MEM state *     will be entered. * * From the attached queue pair, the queue pair can enter the shutdown states * when either side of the queue pair detaches. If the guest side detaches * first, the queue pair will enter the VMCIQPB_SHUTDOWN_NO_MEM state, where * the content of the queue pair will no longer be available. If the host * side detaches first, the queue pair will either enter the * VMCIQPB_SHUTDOWN_MEM, if the guest memory is currently mapped, or * VMCIQPB_SHUTDOWN_NO_MEM, if the guest memory is not mapped * (e.g., the host detaches while a guest is stunned). * * New-style VMX'en will also unmap guest memory, if the guest is * quiesced, e.g., during a snapshot operation. In that case, the guest * memory will no longer be available, and the queue pair will transition from * *_MEM state to a *_NO_MEM state. The VMX may later map the memory once more, * in which case the queue pair will transition from the *_NO_MEM state at that * point back to the *_MEM state. Note that the *_NO_MEM state may have changed, * since the peer may have either attached or detached in the meantime. The * values are laid out such that ++ on a state will move from a *_NO_MEM to a * *_MEM state, and vice versa. */ /* The Kernel specific component of the struct vmci_queue structure. */struct vmci_queue_kern_if {	struct mutex __mutex;	/* Protects the queue. */	struct mutex *mutex;	/* Shared by producer and consumer queues. */	size_t num_pages;	/* Number of pages incl. header. */	bool host;		/* Host or guest? */	union {		struct {			dma_addr_t *pas;			void **vas;		} g;		/* Used by the guest. */		struct {			struct page **page;			struct page **header_page;		} h;		/* Used by the host. */	} u;}; /* * This structure is opaque to the clients. */struct vmci_qp {	struct vmci_handle handle;	struct vmci_queue *produce_q;	struct vmci_queue *consume_q;	u64 produce_q_size;	u64 consume_q_size;	u32 peer;	u32 flags;	u32 priv_flags;	bool guest_endpoint;	unsigned int blocked;	unsigned int generation;	wait_queue_head_t event;}; enum qp_broker_state {	VMCIQPB_NEW,	VMCIQPB_CREATED_NO_MEM,	VMCIQPB_CREATED_MEM,	VMCIQPB_ATTACHED_NO_MEM,	VMCIQPB_ATTACHED_MEM,	VMCIQPB_SHUTDOWN_NO_MEM,	VMCIQPB_SHUTDOWN_MEM,	VMCIQPB_GONE}; #define QPBROKERSTATE_HAS_MEM(_qpb) (_qpb->state == VMCIQPB_CREATED_MEM || \				     _qpb->state == VMCIQPB_ATTACHED_MEM || \				     _qpb->state == VMCIQPB_SHUTDOWN_MEM) /* * In the queue pair broker, we always use the guest point of view for * the produce and consume queue values and references, e.g., the * produce queue size stored is the guests produce queue size. The * host endpoint will need to swap these around. The only exception is * the local queue pairs on the host, in which case the host endpoint * that creates the queue pair will have the right orientation, and * the attaching host endpoint will need to swap. */struct qp_entry {	struct list_head list_item;	struct vmci_handle handle;	u32 peer;	u32 flags;	u64 produce_size;	u64 consume_size;	u32 ref_count;}; struct qp_broker_entry {	struct vmci_resource resource;	struct qp_entry qp;	u32 create_id;	u32 attach_id;	enum qp_broker_state state;	bool require_trusted_attach;	bool created_by_trusted;	bool vmci_page_files;	/* Created by VMX using VMCI page files */	struct vmci_queue *produce_q;	struct vmci_queue *consume_q;	struct vmci_queue_header saved_produce_q;	struct vmci_queue_header saved_consume_q;	vmci_event_release_cb wakeup_cb;	void *client_data;	void *local_mem;	/* Kernel memory for local queue pair */}; struct qp_guest_endpoint {	struct vmci_resource resource;	struct qp_entry qp;	u64 num_ppns;	void *produce_q;	void *consume_q;	struct ppn_set ppn_set;}; struct qp_list {	struct list_head head;	struct mutex mutex;	/* Protect queue list. */}; static struct qp_list qp_broker_list = {	.head = LIST_HEAD_INIT(qp_broker_list.head),	.mutex = __MUTEX_INITIALIZER(qp_broker_list.mutex),}; static struct qp_list qp_guest_endpoints = {	.head = LIST_HEAD_INIT(qp_guest_endpoints.head),	.mutex = __MUTEX_INITIALIZER(qp_guest_endpoints.mutex),}; #define INVALID_VMCI_GUEST_MEM_ID  0#define QPE_NUM_PAGES(_QPE) ((u32) \			     (DIV_ROUND_UP(_QPE.produce_size, PAGE_SIZE) + \			      DIV_ROUND_UP(_QPE.consume_size, PAGE_SIZE) + 2))#define QP_SIZES_ARE_VALID(_prod_qsize, _cons_qsize) \	((_prod_qsize) + (_cons_qsize) >= max(_prod_qsize, _cons_qsize) && \	 (_prod_qsize) + (_cons_qsize) <= VMCI_MAX_GUEST_QP_MEMORY) /* * Frees kernel VA space for a given queue and its queue header, and * frees physical data pages. */static void qp_free_queue(void *q, u64 size){	struct vmci_queue *queue = q; 	if (queue) {		u64 i; 		/* Given size does not include header, so add in a page here. */		for (i = 0; i < DIV_ROUND_UP(size, PAGE_SIZE) + 1; i++) {			dma_free_coherent(&vmci_pdev->dev, PAGE_SIZE,					  queue->kernel_if->u.g.vas[i],					  queue->kernel_if->u.g.pas[i]);		} 		vfree(queue);	}} /* * Allocates kernel queue pages of specified size with IOMMU mappings, * plus space for the queue structure/kernel interface and the queue * header. */static void *qp_alloc_queue(u64 size, u32 flags){	u64 i;	struct vmci_queue *queue;	size_t pas_size;	size_t vas_size;	size_t queue_size = sizeof(*queue) + sizeof(*queue->kernel_if);	u64 num_pages; 	if (size > SIZE_MAX - PAGE_SIZE)		return NULL;	num_pages = DIV_ROUND_UP(size, PAGE_SIZE) + 1;	if (num_pages >		 (SIZE_MAX - queue_size) /		 (sizeof(*queue->kernel_if->u.g.pas) +		  sizeof(*queue->kernel_if->u.g.vas)))		return NULL; 	pas_size = num_pages * sizeof(*queue->kernel_if->u.g.pas);	vas_size = num_pages * sizeof(*queue->kernel_if->u.g.vas);	queue_size += pas_size + vas_size; 	queue = vmalloc(queue_size);	if (!queue)		return NULL; 	queue->q_header = NULL;	queue->saved_header = NULL;	queue->kernel_if = (struct vmci_queue_kern_if *)(queue + 1);	queue->kernel_if->mutex = NULL;	queue->kernel_if->num_pages = num_pages;	queue->kernel_if->u.g.pas = (dma_addr_t *)(queue->kernel_if + 1);	queue->kernel_if->u.g.vas =		(void **)((u8 *)queue->kernel_if->u.g.pas + pas_size);	queue->kernel_if->host = false; 	for (i = 0; i < num_pages; i++) {		queue->kernel_if->u.g.vas[i] =			dma_alloc_coherent(&vmci_pdev->dev, PAGE_SIZE,					   &queue->kernel_if->u.g.pas[i],					   GFP_KERNEL);		if (!queue->kernel_if->u.g.vas[i]) {			/* Size excl. the header. */			qp_free_queue(queue, i * PAGE_SIZE);			return NULL;		}	} 	/* Queue header is the first page. */	queue->q_header = queue->kernel_if->u.g.vas[0]; 	return queue;} /* * Copies from a given buffer or iovector to a VMCI Queue.  Uses * kmap_local_page() to dynamically map required portions of the queue * by traversing the offset -> page translation structure for the queue. * Assumes that offset + size does not wrap around in the queue. */static int qp_memcpy_to_queue_iter(struct vmci_queue *queue,				  u64 queue_offset,				  struct iov_iter *from,				  size_t size){	struct vmci_queue_kern_if *kernel_if = queue->kernel_if;	size_t bytes_copied = 0; 	while (bytes_copied < size) {		const u64 page_index =			(queue_offset + bytes_copied) / PAGE_SIZE;		const size_t page_offset =		    (queue_offset + bytes_copied) & (PAGE_SIZE - 1);		void *va;		size_t to_copy; 		if (kernel_if->host)			va = kmap_local_page(kernel_if->u.h.page[page_index]);		else			va = kernel_if->u.g.vas[page_index + 1];			/* Skip header. */ 		if (size - bytes_copied > PAGE_SIZE - page_offset)			/* Enough payload to fill up from this page. */			to_copy = PAGE_SIZE - page_offset;		else			to_copy = size - bytes_copied; 		if (!copy_from_iter_full((u8 *)va + page_offset, to_copy,					 from)) {			if (kernel_if->host)				kunmap_local(va);			return VMCI_ERROR_INVALID_ARGS;		}		bytes_copied += to_copy;		if (kernel_if->host)			kunmap_local(va);	} 	return VMCI_SUCCESS;} /* * Copies to a given buffer or iovector from a VMCI Queue.  Uses * kmap_local_page() to dynamically map required portions of the queue * by traversing the offset -> page translation structure for the queue. * Assumes that offset + size does not wrap around in the queue. */static int qp_memcpy_from_queue_iter(struct iov_iter *to,				    const struct vmci_queue *queue,				    u64 queue_offset, size_t size){	struct vmci_queue_kern_if *kernel_if = queue->kernel_if;	size_t bytes_copied = 0; 	while (bytes_copied < size) {		const u64 page_index =			(queue_offset + bytes_copied) / PAGE_SIZE;		const size_t page_offset =		    (queue_offset + bytes_copied) & (PAGE_SIZE - 1);		void *va;		size_t to_copy;		int err; 		if (kernel_if->host)			va = kmap_local_page(kernel_if->u.h.page[page_index]);		else			va = kernel_if->u.g.vas[page_index + 1];			/* Skip header. */ 		if (size - bytes_copied > PAGE_SIZE - page_offset)			/* Enough payload to fill up this page. */			to_copy = PAGE_SIZE - page_offset;		else			to_copy = size - bytes_copied; 		err = copy_to_iter((u8 *)va + page_offset, to_copy, to);		if (err != to_copy) {			if (kernel_if->host)				kunmap_local(va);			return VMCI_ERROR_INVALID_ARGS;		}		bytes_copied += to_copy;		if (kernel_if->host)			kunmap_local(va);	} 	return VMCI_SUCCESS;} /* * Allocates two list of PPNs --- one for the pages in the produce queue, * and the other for the pages in the consume queue. Initializes the list * of PPNs with the page frame numbers of the KVA for the two queues (and * the queue headers). */static int qp_alloc_ppn_set(void *prod_q,			    u64 num_produce_pages,			    void *cons_q,			    u64 num_consume_pages, struct ppn_set *ppn_set){	u64 *produce_ppns;	u64 *consume_ppns;	struct vmci_queue *produce_q = prod_q;	struct vmci_queue *consume_q = cons_q;	u64 i; 	if (!produce_q || !num_produce_pages || !consume_q ||	    !num_consume_pages || !ppn_set)		return VMCI_ERROR_INVALID_ARGS; 	if (ppn_set->initialized)		return VMCI_ERROR_ALREADY_EXISTS; 	produce_ppns =	    kmalloc_array(num_produce_pages, sizeof(*produce_ppns),			  GFP_KERNEL);	if (!produce_ppns)		return VMCI_ERROR_NO_MEM; 	consume_ppns =	    kmalloc_array(num_consume_pages, sizeof(*consume_ppns),			  GFP_KERNEL);	if (!consume_ppns) {		kfree(produce_ppns);		return VMCI_ERROR_NO_MEM;	} 	for (i = 0; i < num_produce_pages; i++)		produce_ppns[i] =			produce_q->kernel_if->u.g.pas[i] >> PAGE_SHIFT; 	for (i = 0; i < num_consume_pages; i++)		consume_ppns[i] =			consume_q->kernel_if->u.g.pas[i] >> PAGE_SHIFT; 	ppn_set->num_produce_pages = num_produce_pages;	ppn_set->num_consume_pages = num_consume_pages;	ppn_set->produce_ppns = produce_ppns;	ppn_set->consume_ppns = consume_ppns;	ppn_set->initialized = true;	return VMCI_SUCCESS;} /* * Frees the two list of PPNs for a queue pair. */static void qp_free_ppn_set(struct ppn_set *ppn_set){	if (ppn_set->initialized) {		/* Do not call these functions on NULL inputs. */		kfree(ppn_set->produce_ppns);		kfree(ppn_set->consume_ppns);	}	memset(ppn_set, 0, sizeof(*ppn_set));} /* * Populates the list of PPNs in the hypercall structure with the PPNS * of the produce queue and the consume queue. */static int qp_populate_ppn_set(u8 *call_buf, const struct ppn_set *ppn_set){	if (vmci_use_ppn64()) {		memcpy(call_buf, ppn_set->produce_ppns,		       ppn_set->num_produce_pages *		       sizeof(*ppn_set->produce_ppns));		memcpy(call_buf +		       ppn_set->num_produce_pages *		       sizeof(*ppn_set->produce_ppns),

Lines 1–500 of 3146. Load more