virt: acrn: Introduce I/O request management

An I/O request of a User VM, which is constructed by the hypervisor, is distributed by the ACRN Hypervisor Service Module to an I/O client corresponding to the address range of the I/O request. For each User VM, there is a shared 4-KByte memory region used for I/O requests communication between the hypervisor and Service VM. An I/O request is a 256-byte structure buffer, which is 'struct acrn_io_request', that is filled by an I/O handler of the hypervisor when a trapped I/O access happens in a User VM. ACRN userspace in the Service VM first allocates a 4-KByte page and passes the GPA (Guest Physical Address) of the buffer to the hypervisor. The buffer is used as an array of 16 I/O request slots with each I/O request slot being 256 bytes. This array is indexed by vCPU ID. An I/O client, which is 'struct acrn_ioreq_client', is responsible for handling User VM I/O requests whose accessed GPA falls in a certain range. Multiple I/O clients can be associated with each User VM. There is a special client associated with each User VM, called the default client, that handles all I/O requests that do not fit into the range of any other I/O clients. The ACRN userspace acts as the default client for each User VM. The state transitions of a ACRN I/O request are as follows. FREE -> PENDING -> PROCESSING -> COMPLETE -> FREE -> ... FREE: this I/O request slot is empty PENDING: a valid I/O request is pending in this slot PROCESSING: the I/O request is being processed COMPLETE: the I/O request has been processed An I/O request in COMPLETE or FREE state is owned by the hypervisor. HSM and ACRN userspace are in charge of processing the others. The processing flow of I/O requests are listed as following: a) The I/O handler of the hypervisor will fill an I/O request with PENDING state when a trapped I/O access happens in a User VM. b) The hypervisor makes an upcall, which is a notification interrupt, to the Service VM. c) The upcall handler schedules a worker to dispatch I/O requests. d) The worker looks for the PENDING I/O requests, assigns them to different registered clients based on the address of the I/O accesses, updates their state to PROCESSING, and notifies the corresponding client to handle. e) The notified client handles the assigned I/O requests. f) The HSM updates I/O requests states to COMPLETE and notifies the hypervisor of the completion via hypercalls.

Shuo Liu committed 5 years ago

Commit 72f293d1 parent 88f537d7 files · +843 −10Browse files

Unified · Split

7 files

drivers/virt/acrn/Makefile +1 −1
drivers/virt/acrn/acrn_drv.h +82 −0
drivers/virt/acrn/hsm.c +41 −2
drivers/virt/acrn/hypercall.h +28 −0
drivers/virt/acrn/ioreq.c +521 −0
drivers/virt/acrn/vm.c +20 −7
include/uapi/linux/acrn.h +150 −0

drivers/virt/acrn/Makefile+1−1

@@ -1,3 +1,3 @@
1	1	# SPDX-License-Identifier: GPL-2.0
2	2	obj-$(CONFIG_ACRN_HSM) := acrn.o
3		−acrn-y := hsm.o vm.o mm.o
	3	+acrn-y := hsm.o vm.o mm.o ioreq.o

drivers/virt/acrn/acrn_drv.h+82−0

@@ -12,10 +12,15 @@ · 11 unmodified lines
12	12
13	13	extern struct miscdevice acrn_dev;
14	14
	15	+#define ACRN_NAME_LEN 16
15	16	#define ACRN_MEM_MAPPING_MAX 256
16	17
17	18	#define ACRN_MEM_REGION_ADD 0
18	19	#define ACRN_MEM_REGION_DEL 2
	20	+
	21	+struct acrn_vm;
	22	+struct acrn_ioreq_client;
	23	+
19	24	/**
20	25	* struct vm_memory_region_op - Hypervisor memory operation
21	26	* @type: Operation type (ACRN_MEM_REGION_*)
@@ -75,9 +80,63 @@ struct vm_memory_mapping { · 53 unmodified lines
75	80	size_t size;
76	81	};
77	82
	83	+/**
	84	+ * struct acrn_ioreq_buffer - Data for setting the ioreq buffer of User VM
	85	+ * @ioreq_buf: The GPA of the IO request shared buffer of a VM
	86	+ *
	87	+ * The parameter for the HC_SET_IOREQ_BUFFER hypercall used to set up
	88	+ * the shared I/O request buffer between Service VM and ACRN hypervisor.
	89	+ */
	90	+struct acrn_ioreq_buffer {
	91	+ u64 ioreq_buf;
	92	+};
	93	+
	94	+struct acrn_ioreq_range {
	95	+ struct list_head list;
	96	+ u32 type;
	97	+ u64 start;
	98	+ u64 end;
	99	+};
	100	+
	101	+#define ACRN_IOREQ_CLIENT_DESTROYING 0U
	102	+typedef int (ioreq_handler_t)(struct acrn_ioreq_client client,
	103	+ struct acrn_io_request *req);
	104	+/**
	105	+ * struct acrn_ioreq_client - Structure of I/O client.
	106	+ * @name: Client name
	107	+ * @vm: The VM that the client belongs to
	108	+ * @list: List node for this acrn_ioreq_client
	109	+ * @is_default: If this client is the default one
	110	+ * @flags: Flags (ACRN_IOREQ_CLIENT_*)
	111	+ * @range_list: I/O ranges
	112	+ * @range_lock: Lock to protect range_list
	113	+ * @ioreqs_map: The pending I/O requests bitmap.
	114	+ * @handler: I/O requests handler of this client
	115	+ * @thread: The thread which executes the handler
	116	+ * @wq: The wait queue for the handler thread parking
	117	+ * @priv: Data for the thread
	118	+ */
	119	+struct acrn_ioreq_client {
	120	+ char name[ACRN_NAME_LEN];
	121	+ struct acrn_vm *vm;
	122	+ struct list_head list;
	123	+ bool is_default;
	124	+ unsigned long flags;
	125	+ struct list_head range_list;
	126	+ rwlock_t range_lock;
	127	+ DECLARE_BITMAP(ioreqs_map, ACRN_IO_REQUEST_MAX);
	128	+ ioreq_handler_t handler;
	129	+ struct task_struct *thread;
	130	+ wait_queue_head_t wq;
	131	+ void *priv;
	132	+};
	133	+
78	134	#define ACRN_INVALID_VMID (0xffffU)
79	135
80	136	#define ACRN_VM_FLAG_DESTROYED 0U
	137	+#define ACRN_VM_FLAG_CLEARING_IOREQ 1U
	138	+extern struct list_head acrn_vm_list;
	139	+extern rwlock_t acrn_vm_list_lock;
81	140	/**
82	141	* struct acrn_vm - Properties of ACRN User VM.
83	142	* @list: Entry within global list of all VMs.
@@ -90,6 +149,11 @@ struct vm_memory_mapping { · 6 unmodified lines
90	149	* &acrn_vm.regions_mapping_count.
91	150	* @regions_mapping: Memory mappings of this VM.
92	151	* @regions_mapping_count: Number of memory mapping of this VM.
	152	+ * @ioreq_clients_lock: Lock to protect ioreq_clients and default_client
	153	+ * @ioreq_clients: The I/O request clients list of this VM
	154	+ * @default_client: The default I/O request client
	155	+ * @ioreq_buf: I/O request shared buffer
	156	+ * @ioreq_page: The page of the I/O request shared buffer
93	157	*/
94	158	struct acrn_vm {
95	159	struct list_head list;
@@ -99,6 +163,11 @@ struct acrn_vm { · 3 unmodified lines
99	163	struct mutex regions_mapping_lock;
100	164	struct vm_memory_mapping regions_mapping[ACRN_MEM_MAPPING_MAX];
101	165	int regions_mapping_count;
	166	+ spinlock_t ioreq_clients_lock;
	167	+ struct list_head ioreq_clients;
	168	+ struct acrn_ioreq_client *default_client;
	169	+ struct acrn_io_request_buffer *ioreq_buf;
	170	+ struct page *ioreq_page;
102	171	};
103	172
104	173	struct acrn_vm acrn_vm_create(struct acrn_vm vm,
@@ -112,4 +181,17 @@ int acrn_vm_memseg_unmap(struct acrn_vm vm, struct acrn_vm_memmap memmap); · 7 unmodified lines
112	181	int acrn_vm_ram_map(struct acrn_vm vm, struct acrn_vm_memmap memmap);
113	182	void acrn_vm_all_ram_unmap(struct acrn_vm *vm);
114	183
	184	+int acrn_ioreq_init(struct acrn_vm *vm, u64 buf_vma);
	185	+void acrn_ioreq_deinit(struct acrn_vm *vm);
	186	+int acrn_ioreq_intr_setup(void);
	187	+void acrn_ioreq_intr_remove(void);
	188	+void acrn_ioreq_request_clear(struct acrn_vm *vm);
	189	+int acrn_ioreq_client_wait(struct acrn_ioreq_client *client);
	190	+int acrn_ioreq_request_default_complete(struct acrn_vm *vm, u16 vcpu);
	191	+struct acrn_ioreq_client acrn_ioreq_client_create(struct acrn_vm vm,
	192	+ ioreq_handler_t handler,
	193	+ void *data, bool is_default,
	194	+ const char *name);
	195	+void acrn_ioreq_client_destroy(struct acrn_ioreq_client *client);
	196	+
115	197	#endif /* __ACRN_HSM_DRV_H */

drivers/virt/acrn/hsm.c+41−2

@@ -48,6 +48,7 @@ static long acrn_dev_ioctl(struct file *filp, unsigned int cmd, · 47 unmodified lines
48	48	struct acrn_vm *vm = filp->private_data;
49	49	struct acrn_vm_creation *vm_param;
50	50	struct acrn_vcpu_regs *cpu_regs;
	51	+ struct acrn_ioreq_notify notify;
51	52	struct acrn_vm_memmap memmap;
52	53	int i, ret = 0;
53	54
@@ -147,6 +148,35 @@ static long acrn_dev_ioctl(struct file *filp, unsigned int cmd, · 93 unmodified lines
147	148
148	149	ret = acrn_vm_memseg_unmap(vm, &memmap);
149	150	break;
	151	+ case ACRN_IOCTL_CREATE_IOREQ_CLIENT:
	152	+ if (vm->default_client)
	153	+ return -EEXIST;
	154	+ if (!acrn_ioreq_client_create(vm, NULL, NULL, true, "acrndm"))
	155	+ ret = -EINVAL;
	156	+ break;
	157	+ case ACRN_IOCTL_DESTROY_IOREQ_CLIENT:
	158	+ if (vm->default_client)
	159	+ acrn_ioreq_client_destroy(vm->default_client);
	160	+ break;
	161	+ case ACRN_IOCTL_ATTACH_IOREQ_CLIENT:
	162	+ if (vm->default_client)
	163	+ ret = acrn_ioreq_client_wait(vm->default_client);
	164	+ else
	165	+ ret = -ENODEV;
	166	+ break;
	167	+ case ACRN_IOCTL_NOTIFY_REQUEST_FINISH:
	168	+ if (copy_from_user(&notify, (void __user *)ioctl_param,
	169	+ sizeof(struct acrn_ioreq_notify)))
	170	+ return -EFAULT;
	171	+
	172	+ if (notify.reserved != 0)
	173	+ return -EINVAL;
	174	+
	175	+ ret = acrn_ioreq_request_default_complete(vm, notify.vcpu);
	176	+ break;
	177	+ case ACRN_IOCTL_CLEAR_VM_IOREQ:
	178	+ acrn_ioreq_request_clear(vm);
	179	+ break;
150	180	default:
151	181	dev_dbg(acrn_dev.this_device, "Unknown IOCTL 0x%x!\n", cmd);
152	182	ret = -ENOTTY;
@@ -188,14 +218,23 @@ static int __init hsm_init(void) · 35 unmodified lines
188	218	return -EPERM;
189	219
190	220	ret = misc_register(&acrn_dev);
191		− if (ret)
	221	+ if (ret) {
192	222	pr_err("Create misc dev failed!\n");
	223	+ return ret;
	224	+ }
193	225
194		− return ret;
	226	+ ret = acrn_ioreq_intr_setup();
	227	+ if (ret) {
	228	+ pr_err("Setup I/O request handler failed!\n");
	229	+ misc_deregister(&acrn_dev);
	230	+ return ret;
	231	+ }
	232	+ return 0;
195	233	}
196	234
197	235	static void __exit hsm_exit(void)
198	236	{
	237	+ acrn_ioreq_intr_remove();
199	238	misc_deregister(&acrn_dev);
200	239	}
201	240	module_init(hsm_init);

drivers/virt/acrn/hypercall.h+28−0

@@ -21,6 +21,10 @@ · 20 unmodified lines
21	21	#define HC_RESET_VM _HC_ID(HC_ID, HC_ID_VM_BASE + 0x05)
22	22	#define HC_SET_VCPU_REGS _HC_ID(HC_ID, HC_ID_VM_BASE + 0x06)
23	23
	24	+#define HC_ID_IOREQ_BASE 0x30UL
	25	+#define HC_SET_IOREQ_BUFFER _HC_ID(HC_ID, HC_ID_IOREQ_BASE + 0x00)
	26	+#define HC_NOTIFY_REQUEST_FINISH _HC_ID(HC_ID, HC_ID_IOREQ_BASE + 0x01)
	27	+
24	28	#define HC_ID_MEM_BASE 0x40UL
25	29	#define HC_VM_SET_MEMORY_REGIONS _HC_ID(HC_ID, HC_ID_MEM_BASE + 0x02)
26	30
@@ -91,6 +95,30 @@ static inline long hcall_set_vcpu_regs(u64 vmid, u64 regs_state) · 64 unmodified lines
91	95	return acrn_hypercall2(HC_SET_VCPU_REGS, vmid, regs_state);
92	96	}
93	97
	98	+/**
	99	+ * hcall_set_ioreq_buffer() - Set up the shared buffer for I/O Requests.
	100	+ * @vmid: User VM ID
	101	+ * @buffer: Service VM GPA of the shared buffer
	102	+ *
	103	+ * Return: 0 on success, <0 on failure
	104	+ */
	105	+static inline long hcall_set_ioreq_buffer(u64 vmid, u64 buffer)
	106	+{
	107	+ return acrn_hypercall2(HC_SET_IOREQ_BUFFER, vmid, buffer);
	108	+}
	109	+
	110	+/**
	111	+ * hcall_notify_req_finish() - Notify ACRN Hypervisor of I/O request completion.
	112	+ * @vmid: User VM ID
	113	+ * @vcpu: The vCPU which initiated the I/O request
	114	+ *
	115	+ * Return: 0 on success, <0 on failure
	116	+ */
	117	+static inline long hcall_notify_req_finish(u64 vmid, u64 vcpu)
	118	+{
	119	+ return acrn_hypercall2(HC_NOTIFY_REQUEST_FINISH, vmid, vcpu);
	120	+}
	121	+
94	122	/**
95	123	* hcall_set_memory_regions() - Inform the hypervisor to set up EPT mappings
96	124	* @regions_pa: Service VM GPA of &struct vm_memory_region_batch

drivers/virt/acrn/ioreq.c+521−0

@@ -0,0 +1,521 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+/*
	3	+ * ACRN_HSM: Handle I/O requests
	4	+ *
	5	+ * Copyright (C) 2020 Intel Corporation. All rights reserved.
	6	+ *
	7	+ * Authors:
	8	+ * Jason Chen CJ <jason.cj.chen@intel.com>
	9	+ * Fengwei Yin <fengwei.yin@intel.com>
	10	+ */
	11	+
	12	+#include <linux/interrupt.h>
	13	+#include <linux/io.h>
	14	+#include <linux/kthread.h>
	15	+#include <linux/mm.h>
	16	+#include <linux/slab.h>
	17	+
	18	+#include <asm/acrn.h>
	19	+
	20	+#include "acrn_drv.h"
	21	+
	22	+static void ioreq_pause(void);
	23	+static void ioreq_resume(void);
	24	+
	25	+static void ioreq_dispatcher(struct work_struct *work);
	26	+static struct workqueue_struct *ioreq_wq;
	27	+static DECLARE_WORK(ioreq_work, ioreq_dispatcher);
	28	+
	29	+static inline bool has_pending_request(struct acrn_ioreq_client *client)
	30	+{
	31	+ return !bitmap_empty(client->ioreqs_map, ACRN_IO_REQUEST_MAX);
	32	+}
	33	+
	34	+static inline bool is_destroying(struct acrn_ioreq_client *client)
	35	+{
	36	+ return test_bit(ACRN_IOREQ_CLIENT_DESTROYING, &client->flags);
	37	+}
	38	+
	39	+static int ioreq_complete_request(struct acrn_vm *vm, u16 vcpu,
	40	+ struct acrn_io_request *acrn_req)
	41	+{
	42	+ bool polling_mode;
	43	+ int ret = 0;
	44	+
	45	+ polling_mode = acrn_req->completion_polling;
	46	+ /* Add barrier() to make sure the writes are done before completion */
	47	+ smp_store_release(&acrn_req->processed, ACRN_IOREQ_STATE_COMPLETE);
	48	+
	49	+ /*
	50	+ * To fulfill the requirement of real-time in several industry
	51	+ * scenarios, like automotive, ACRN can run under the partition mode,
	52	+ * in which User VMs and Service VM are bound to dedicated CPU cores.
	53	+ * Polling mode of handling the I/O request is introduced to achieve a
	54	+ * faster I/O request handling. In polling mode, the hypervisor polls
	55	+ * I/O request's completion. Once an I/O request is marked as
	56	+ * ACRN_IOREQ_STATE_COMPLETE, hypervisor resumes from the polling point
	57	+ * to continue the I/O request flow. Thus, the completion notification
	58	+ * from HSM of I/O request is not needed. Please note,
	59	+ * completion_polling needs to be read before the I/O request being
	60	+ * marked as ACRN_IOREQ_STATE_COMPLETE to avoid racing with the
	61	+ * hypervisor.
	62	+ */
	63	+ if (!polling_mode) {
	64	+ ret = hcall_notify_req_finish(vm->vmid, vcpu);
	65	+ if (ret < 0)
	66	+ dev_err(acrn_dev.this_device,
	67	+ "Notify I/O request finished failed!\n");
	68	+ }
	69	+
	70	+ return ret;
	71	+}
	72	+
	73	+static int acrn_ioreq_complete_request(struct acrn_ioreq_client *client,
	74	+ u16 vcpu,
	75	+ struct acrn_io_request *acrn_req)
	76	+{
	77	+ int ret;
	78	+
	79	+ if (vcpu >= client->vm->vcpu_num)
	80	+ return -EINVAL;
	81	+
	82	+ clear_bit(vcpu, client->ioreqs_map);
	83	+ if (!acrn_req) {
	84	+ acrn_req = (struct acrn_io_request *)client->vm->ioreq_buf;
	85	+ acrn_req += vcpu;
	86	+ }
	87	+
	88	+ ret = ioreq_complete_request(client->vm, vcpu, acrn_req);
	89	+
	90	+ return ret;
	91	+}
	92	+
	93	+int acrn_ioreq_request_default_complete(struct acrn_vm *vm, u16 vcpu)
	94	+{
	95	+ int ret = 0;
	96	+
	97	+ spin_lock_bh(&vm->ioreq_clients_lock);
	98	+ if (vm->default_client)
	99	+ ret = acrn_ioreq_complete_request(vm->default_client,
	100	+ vcpu, NULL);
	101	+ spin_unlock_bh(&vm->ioreq_clients_lock);
	102	+
	103	+ return ret;
	104	+}
	105	+
	106	+/*
	107	+ * ioreq_task() is the execution entity of handler thread of an I/O client.
	108	+ * The handler callback of the I/O client is called within the handler thread.
	109	+ */
	110	+static int ioreq_task(void *data)
	111	+{
	112	+ struct acrn_ioreq_client *client = data;
	113	+ struct acrn_io_request *req;
	114	+ unsigned long *ioreqs_map;
	115	+ int vcpu, ret;
	116	+
	117	+ /*
	118	+ * Lockless access to ioreqs_map is safe, because
	119	+ * 1) set_bit() and clear_bit() are atomic operations.
	120	+ * 2) I/O requests arrives serialized. The access flow of ioreqs_map is:
	121	+ * set_bit() - in ioreq_work handler
	122	+ * Handler callback handles corresponding I/O request
	123	+ * clear_bit() - in handler thread (include ACRN userspace)
	124	+ * Mark corresponding I/O request completed
	125	+ * Loop again if a new I/O request occurs
	126	+ */
	127	+ ioreqs_map = client->ioreqs_map;
	128	+ while (!kthread_should_stop()) {
	129	+ acrn_ioreq_client_wait(client);
	130	+ while (has_pending_request(client)) {
	131	+ vcpu = find_first_bit(ioreqs_map, client->vm->vcpu_num);
	132	+ req = client->vm->ioreq_buf->req_slot + vcpu;
	133	+ ret = client->handler(client, req);
	134	+ if (ret < 0) {
	135	+ dev_err(acrn_dev.this_device,
	136	+ "IO handle failure: %d\n", ret);
	137	+ break;
	138	+ }
	139	+ acrn_ioreq_complete_request(client, vcpu, req);
	140	+ }
	141	+ }
	142	+
	143	+ return 0;
	144	+}
	145	+
	146	+/*
	147	+ * For the non-default I/O clients, give them chance to complete the current
	148	+ * I/O requests if there are any. For the default I/O client, it is safe to
	149	+ * clear all pending I/O requests because the clearing request is from ACRN
	150	+ * userspace.
	151	+ */
	152	+void acrn_ioreq_request_clear(struct acrn_vm *vm)
	153	+{
	154	+ struct acrn_ioreq_client *client;
	155	+ bool has_pending = false;
	156	+ unsigned long vcpu;
	157	+ int retry = 10;
	158	+
	159	+ /*
	160	+ * IO requests of this VM will be completed directly in
	161	+ * acrn_ioreq_dispatch if ACRN_VM_FLAG_CLEARING_IOREQ flag is set.
	162	+ */
	163	+ set_bit(ACRN_VM_FLAG_CLEARING_IOREQ, &vm->flags);
	164	+
	165	+ /*
	166	+ * acrn_ioreq_request_clear is only called in VM reset case. Simply
	167	+ * wait 100ms in total for the IO requests' completion.
	168	+ */
	169	+ do {
	170	+ spin_lock_bh(&vm->ioreq_clients_lock);
	171	+ list_for_each_entry(client, &vm->ioreq_clients, list) {
	172	+ has_pending = has_pending_request(client);
	173	+ if (has_pending)
	174	+ break;
	175	+ }
	176	+ spin_unlock_bh(&vm->ioreq_clients_lock);
	177	+
	178	+ if (has_pending)
	179	+ schedule_timeout_interruptible(HZ / 100);
	180	+ } while (has_pending && --retry > 0);
	181	+ if (retry == 0)
	182	+ dev_warn(acrn_dev.this_device,
	183	+ "%s cannot flush pending request!\n", client->name);
	184	+
	185	+ /* Clear all ioreqs belonging to the default client */
	186	+ spin_lock_bh(&vm->ioreq_clients_lock);
	187	+ client = vm->default_client;
	188	+ if (client) {
	189	+ vcpu = find_next_bit(client->ioreqs_map,
	190	+ ACRN_IO_REQUEST_MAX, 0);
	191	+ while (vcpu < ACRN_IO_REQUEST_MAX) {
	192	+ acrn_ioreq_complete_request(client, vcpu, NULL);
	193	+ vcpu = find_next_bit(client->ioreqs_map,
	194	+ ACRN_IO_REQUEST_MAX, vcpu + 1);
	195	+ }
	196	+ }
	197	+ spin_unlock_bh(&vm->ioreq_clients_lock);
	198	+
	199	+ /* Clear ACRN_VM_FLAG_CLEARING_IOREQ flag after the clearing */
	200	+ clear_bit(ACRN_VM_FLAG_CLEARING_IOREQ, &vm->flags);
	201	+}
	202	+
	203	+int acrn_ioreq_client_wait(struct acrn_ioreq_client *client)
	204	+{
	205	+ if (client->is_default) {
	206	+ /*
	207	+ * In the default client, a user space thread waits on the
	208	+ * waitqueue. The is_destroying() check is used to notify user
	209	+ * space the client is going to be destroyed.
	210	+ */
	211	+ wait_event_interruptible(client->wq,
	212	+ has_pending_request(client) \|\|
	213	+ is_destroying(client));
	214	+ if (is_destroying(client))
	215	+ return -ENODEV;
	216	+ } else {
	217	+ wait_event_interruptible(client->wq,
	218	+ has_pending_request(client) \|\|
	219	+ kthread_should_stop());
	220	+ }
	221	+
	222	+ return 0;
	223	+}
	224	+
	225	+static bool in_range(struct acrn_ioreq_range *range,
	226	+ struct acrn_io_request *req)
	227	+{
	228	+ bool ret = false;
	229	+
	230	+ if (range->type == req->type) {
	231	+ switch (req->type) {
	232	+ case ACRN_IOREQ_TYPE_MMIO:
	233	+ if (req->reqs.mmio_request.address >= range->start &&
	234	+ (req->reqs.mmio_request.address +
	235	+ req->reqs.mmio_request.size - 1) <= range->end)
	236	+ ret = true;
	237	+ break;
	238	+ case ACRN_IOREQ_TYPE_PORTIO:
	239	+ if (req->reqs.pio_request.address >= range->start &&
	240	+ (req->reqs.pio_request.address +
	241	+ req->reqs.pio_request.size - 1) <= range->end)
	242	+ ret = true;
	243	+ break;
	244	+ default:
	245	+ break;
	246	+ }
	247	+ }
	248	+
	249	+ return ret;
	250	+}
	251	+
	252	+static struct acrn_ioreq_client find_ioreq_client(struct acrn_vm vm,
	253	+ struct acrn_io_request *req)
	254	+{
	255	+ struct acrn_ioreq_client client, found = NULL;
	256	+ struct acrn_ioreq_range *range;
	257	+
	258	+ lockdep_assert_held(&vm->ioreq_clients_lock);
	259	+
	260	+ list_for_each_entry(client, &vm->ioreq_clients, list) {
	261	+ read_lock_bh(&client->range_lock);
	262	+ list_for_each_entry(range, &client->range_list, list) {
	263	+ if (in_range(range, req)) {
	264	+ found = client;
	265	+ break;
	266	+ }
	267	+ }
	268	+ read_unlock_bh(&client->range_lock);
	269	+ if (found)
	270	+ break;
	271	+ }
	272	+ return found ? found : vm->default_client;
	273	+}
	274	+
	275	+/**
	276	+ * acrn_ioreq_client_create() - Create an ioreq client
	277	+ * @vm: The VM that this client belongs to
	278	+ * @handler: The ioreq_handler of ioreq client acrn_hsm will create a kernel
	279	+ * thread and call the handler to handle I/O requests.
	280	+ * @priv: Private data for the handler
	281	+ * @is_default: If it is the default client
	282	+ * @name: The name of ioreq client
	283	+ *
	284	+ * Return: acrn_ioreq_client pointer on success, NULL on error
	285	+ */
	286	+struct acrn_ioreq_client acrn_ioreq_client_create(struct acrn_vm vm,
	287	+ ioreq_handler_t handler,
	288	+ void *priv, bool is_default,
	289	+ const char *name)
	290	+{
	291	+ struct acrn_ioreq_client *client;
	292	+
	293	+ if (!handler && !is_default) {
	294	+ dev_dbg(acrn_dev.this_device,
	295	+ "Cannot create non-default client w/o handler!\n");
	296	+ return NULL;
	297	+ }
	298	+ client = kzalloc(sizeof(*client), GFP_KERNEL);
	299	+ if (!client)
	300	+ return NULL;
	301	+
	302	+ client->handler = handler;
	303	+ client->vm = vm;
	304	+ client->priv = priv;
	305	+ client->is_default = is_default;
	306	+ if (name)
	307	+ strncpy(client->name, name, sizeof(client->name) - 1);
	308	+ rwlock_init(&client->range_lock);
	309	+ INIT_LIST_HEAD(&client->range_list);
	310	+ init_waitqueue_head(&client->wq);
	311	+
	312	+ if (client->handler) {
	313	+ client->thread = kthread_run(ioreq_task, client, "VM%u-%s",
	314	+ client->vm->vmid, client->name);
	315	+ if (IS_ERR(client->thread)) {
	316	+ kfree(client);
	317	+ return NULL;
	318	+ }
	319	+ }
	320	+
	321	+ spin_lock_bh(&vm->ioreq_clients_lock);
	322	+ if (is_default)
	323	+ vm->default_client = client;
	324	+ else
	325	+ list_add(&client->list, &vm->ioreq_clients);
	326	+ spin_unlock_bh(&vm->ioreq_clients_lock);
	327	+
	328	+ dev_dbg(acrn_dev.this_device, "Created ioreq client %s.\n", name);
	329	+ return client;
	330	+}
	331	+
	332	+/**
	333	+ * acrn_ioreq_client_destroy() - Destroy an ioreq client
	334	+ * @client: The ioreq client
	335	+ */
	336	+void acrn_ioreq_client_destroy(struct acrn_ioreq_client *client)
	337	+{
	338	+ struct acrn_ioreq_range range, next;
	339	+ struct acrn_vm *vm = client->vm;
	340	+
	341	+ dev_dbg(acrn_dev.this_device,
	342	+ "Destroy ioreq client %s.\n", client->name);
	343	+ ioreq_pause();
	344	+ set_bit(ACRN_IOREQ_CLIENT_DESTROYING, &client->flags);
	345	+ if (client->is_default)
	346	+ wake_up_interruptible(&client->wq);
	347	+ else
	348	+ kthread_stop(client->thread);
	349	+
	350	+ spin_lock_bh(&vm->ioreq_clients_lock);
	351	+ if (client->is_default)
	352	+ vm->default_client = NULL;
	353	+ else
	354	+ list_del(&client->list);
	355	+ spin_unlock_bh(&vm->ioreq_clients_lock);
	356	+
	357	+ write_lock_bh(&client->range_lock);
	358	+ list_for_each_entry_safe(range, next, &client->range_list, list) {
	359	+ list_del(&range->list);
	360	+ kfree(range);
	361	+ }
	362	+ write_unlock_bh(&client->range_lock);
	363	+ kfree(client);
	364	+
	365	+ ioreq_resume();
	366	+}
	367	+
	368	+static int acrn_ioreq_dispatch(struct acrn_vm *vm)
	369	+{
	370	+ struct acrn_ioreq_client *client;
	371	+ struct acrn_io_request *req;
	372	+ int i;
	373	+
	374	+ for (i = 0; i < vm->vcpu_num; i++) {
	375	+ req = vm->ioreq_buf->req_slot + i;
	376	+
	377	+ /* barrier the read of processed of acrn_io_request */
	378	+ if (smp_load_acquire(&req->processed) ==
	379	+ ACRN_IOREQ_STATE_PENDING) {
	380	+ /* Complete the IO request directly in clearing stage */
	381	+ if (test_bit(ACRN_VM_FLAG_CLEARING_IOREQ, &vm->flags)) {
	382	+ ioreq_complete_request(vm, i, req);
	383	+ continue;
	384	+ }
	385	+
	386	+ spin_lock_bh(&vm->ioreq_clients_lock);
	387	+ client = find_ioreq_client(vm, req);
	388	+ if (!client) {
	389	+ dev_err(acrn_dev.this_device,
	390	+ "Failed to find ioreq client!\n");
	391	+ spin_unlock_bh(&vm->ioreq_clients_lock);
	392	+ return -EINVAL;
	393	+ }
	394	+ if (!client->is_default)
	395	+ req->kernel_handled = 1;
	396	+ else
	397	+ req->kernel_handled = 0;
	398	+ /*
	399	+ * Add barrier() to make sure the writes are done
	400	+ * before setting ACRN_IOREQ_STATE_PROCESSING
	401	+ */
	402	+ smp_store_release(&req->processed,
	403	+ ACRN_IOREQ_STATE_PROCESSING);
	404	+ set_bit(i, client->ioreqs_map);
	405	+ wake_up_interruptible(&client->wq);
	406	+ spin_unlock_bh(&vm->ioreq_clients_lock);
	407	+ }
	408	+ }
	409	+
	410	+ return 0;
	411	+}
	412	+
	413	+static void ioreq_dispatcher(struct work_struct *work)
	414	+{
	415	+ struct acrn_vm *vm;
	416	+
	417	+ read_lock(&acrn_vm_list_lock);
	418	+ list_for_each_entry(vm, &acrn_vm_list, list) {
	419	+ if (!vm->ioreq_buf)
	420	+ break;
	421	+ acrn_ioreq_dispatch(vm);
	422	+ }
	423	+ read_unlock(&acrn_vm_list_lock);
	424	+}
	425	+
	426	+static void ioreq_intr_handler(void)
	427	+{
	428	+ queue_work(ioreq_wq, &ioreq_work);
	429	+}
	430	+
	431	+static void ioreq_pause(void)
	432	+{
	433	+ /* Flush and unarm the handler to ensure no I/O requests pending */
	434	+ acrn_remove_intr_handler();
	435	+ drain_workqueue(ioreq_wq);
	436	+}
	437	+
	438	+static void ioreq_resume(void)
	439	+{
	440	+ /* Schedule after enabling in case other clients miss interrupt */
	441	+ acrn_setup_intr_handler(ioreq_intr_handler);
	442	+ queue_work(ioreq_wq, &ioreq_work);
	443	+}
	444	+
	445	+int acrn_ioreq_intr_setup(void)
	446	+{
	447	+ acrn_setup_intr_handler(ioreq_intr_handler);
	448	+ ioreq_wq = alloc_workqueue("ioreq_wq",
	449	+ WQ_HIGHPRI \| WQ_MEM_RECLAIM \| WQ_UNBOUND, 1);
	450	+ if (!ioreq_wq) {
	451	+ dev_err(acrn_dev.this_device, "Failed to alloc workqueue!\n");
	452	+ acrn_remove_intr_handler();
	453	+ return -ENOMEM;
	454	+ }
	455	+ return 0;
	456	+}
	457	+
	458	+void acrn_ioreq_intr_remove(void)
	459	+{
	460	+ if (ioreq_wq)
	461	+ destroy_workqueue(ioreq_wq);
	462	+ acrn_remove_intr_handler();
	463	+}
	464	+
	465	+int acrn_ioreq_init(struct acrn_vm *vm, u64 buf_vma)
	466	+{
	467	+ struct acrn_ioreq_buffer *set_buffer;
	468	+ struct page *page;
	469	+ int ret;
	470	+
	471	+ if (vm->ioreq_buf)
	472	+ return -EEXIST;
	473	+
	474	+ set_buffer = kzalloc(sizeof(*set_buffer), GFP_KERNEL);
	475	+ if (!set_buffer)
	476	+ return -ENOMEM;
	477	+
	478	+ ret = pin_user_pages_fast(buf_vma, 1,
	479	+ FOLL_WRITE \| FOLL_LONGTERM, &page);
	480	+ if (unlikely(ret != 1) \|\| !page) {
	481	+ dev_err(acrn_dev.this_device, "Failed to pin ioreq page!\n");
	482	+ ret = -EFAULT;
	483	+ goto free_buf;
	484	+ }
	485	+
	486	+ vm->ioreq_buf = page_address(page);
	487	+ vm->ioreq_page = page;
	488	+ set_buffer->ioreq_buf = page_to_phys(page);
	489	+ ret = hcall_set_ioreq_buffer(vm->vmid, virt_to_phys(set_buffer));
	490	+ if (ret < 0) {
	491	+ dev_err(acrn_dev.this_device, "Failed to init ioreq buffer!\n");
	492	+ unpin_user_page(page);
	493	+ vm->ioreq_buf = NULL;
	494	+ goto free_buf;
	495	+ }
	496	+
	497	+ dev_dbg(acrn_dev.this_device,
	498	+ "Init ioreq buffer %pK!\n", vm->ioreq_buf);
	499	+ ret = 0;
	500	+free_buf:
	501	+ kfree(set_buffer);
	502	+ return ret;
	503	+}
	504	+
	505	+void acrn_ioreq_deinit(struct acrn_vm *vm)
	506	+{
	507	+ struct acrn_ioreq_client client, next;
	508	+
	509	+ dev_dbg(acrn_dev.this_device,
	510	+ "Deinit ioreq buffer %pK!\n", vm->ioreq_buf);
	511	+ /* Destroy all clients belonging to this VM */
	512	+ list_for_each_entry_safe(client, next, &vm->ioreq_clients, list)
	513	+ acrn_ioreq_client_destroy(client);
	514	+ if (vm->default_client)
	515	+ acrn_ioreq_client_destroy(vm->default_client);
	516	+
	517	+ if (vm->ioreq_buf && vm->ioreq_page) {
	518	+ unpin_user_page(vm->ioreq_page);
	519	+ vm->ioreq_buf = NULL;
	520	+ }
	521	+}

drivers/virt/acrn/vm.c+20−7

@@ -15,9 +15,12 @@ · 14 unmodified lines
15	15	#include "acrn_drv.h"
16	16
17	17	/* List of VMs */
18		−static LIST_HEAD(acrn_vm_list);
19		−/* To protect acrn_vm_list */
20		−static DEFINE_MUTEX(acrn_vm_list_lock);
	18	+LIST_HEAD(acrn_vm_list);
	19	+/*
	20	+ * acrn_vm_list is read in a worker thread which dispatch I/O requests and
	21	+ * is wrote in VM creation ioctl. Use the rwlock mechanism to protect it.
	22	+ */
	23	+DEFINE_RWLOCK(acrn_vm_list_lock);
21	24
22	25	struct acrn_vm acrn_vm_create(struct acrn_vm vm,
23	26	struct acrn_vm_creation *vm_param)
@@ -32,12 +35,20 @@ struct acrn_vm acrn_vm_create(struct acrn_vm vm, · 8 unmodified lines
32	35	}
33	36
34	37	mutex_init(&vm->regions_mapping_lock);
	38	+ INIT_LIST_HEAD(&vm->ioreq_clients);
	39	+ spin_lock_init(&vm->ioreq_clients_lock);
35	40	vm->vmid = vm_param->vmid;
36	41	vm->vcpu_num = vm_param->vcpu_num;
37	42
38		− mutex_lock(&acrn_vm_list_lock);
	43	+ if (acrn_ioreq_init(vm, vm_param->ioreq_buf) < 0) {
	44	+ hcall_destroy_vm(vm_param->vmid);
	45	+ vm->vmid = ACRN_INVALID_VMID;
	46	+ return NULL;
	47	+ }
	48	+
	49	+ write_lock_bh(&acrn_vm_list_lock);
39	50	list_add(&vm->list, &acrn_vm_list);
40		− mutex_unlock(&acrn_vm_list_lock);
	51	+ write_unlock_bh(&acrn_vm_list_lock);
41	52
42	53	dev_dbg(acrn_dev.this_device, "VM %u created.\n", vm->vmid);
43	54	return vm;
@@ -52,9 +63,11 @@ int acrn_vm_destroy(struct acrn_vm *vm) · 8 unmodified lines
52	63	return 0;
53	64
54	65	/* Remove from global VM list */
55		− mutex_lock(&acrn_vm_list_lock);
	66	+ write_lock_bh(&acrn_vm_list_lock);
56	67	list_del_init(&vm->list);
57		− mutex_unlock(&acrn_vm_list_lock);
	68	+ write_unlock_bh(&acrn_vm_list_lock);
	69	+
	70	+ acrn_ioreq_deinit(vm);
58	71
59	72	ret = hcall_destroy_vm(vm->vmid);
60	73	if (ret < 0) {

include/uapi/linux/acrn.h+150−0

@@ -14,6 +14,145 @@ · 13 unmodified lines
14	14	#include <linux/types.h>
15	15	#include <linux/uuid.h>
16	16
	17	+#define ACRN_IO_REQUEST_MAX 16
	18	+
	19	+#define ACRN_IOREQ_STATE_PENDING 0
	20	+#define ACRN_IOREQ_STATE_COMPLETE 1
	21	+#define ACRN_IOREQ_STATE_PROCESSING 2
	22	+#define ACRN_IOREQ_STATE_FREE 3
	23	+
	24	+#define ACRN_IOREQ_TYPE_PORTIO 0
	25	+#define ACRN_IOREQ_TYPE_MMIO 1
	26	+
	27	+#define ACRN_IOREQ_DIR_READ 0
	28	+#define ACRN_IOREQ_DIR_WRITE 1
	29	+
	30	+/**
	31	+ * struct acrn_mmio_request - Info of a MMIO I/O request
	32	+ * @direction: Access direction of this request (ACRN_IOREQ_DIR_*)
	33	+ * @reserved: Reserved for alignment and should be 0
	34	+ * @address: Access address of this MMIO I/O request
	35	+ * @size: Access size of this MMIO I/O request
	36	+ * @value: Read/write value of this MMIO I/O request
	37	+ */
	38	+struct acrn_mmio_request {
	39	+ __u32 direction;
	40	+ __u32 reserved;
	41	+ __u64 address;
	42	+ __u64 size;
	43	+ __u64 value;
	44	+};
	45	+
	46	+/**
	47	+ * struct acrn_pio_request - Info of a PIO I/O request
	48	+ * @direction: Access direction of this request (ACRN_IOREQ_DIR_*)
	49	+ * @reserved: Reserved for alignment and should be 0
	50	+ * @address: Access address of this PIO I/O request
	51	+ * @size: Access size of this PIO I/O request
	52	+ * @value: Read/write value of this PIO I/O request
	53	+ */
	54	+struct acrn_pio_request {
	55	+ __u32 direction;
	56	+ __u32 reserved;
	57	+ __u64 address;
	58	+ __u64 size;
	59	+ __u32 value;
	60	+};
	61	+
	62	+/**
	63	+ * struct acrn_io_request - 256-byte ACRN I/O request
	64	+ * @type: Type of this request (ACRN_IOREQ_TYPE_*).
	65	+ * @completion_polling: Polling flag. Hypervisor will poll completion of the
	66	+ * I/O request if this flag set.
	67	+ * @reserved0: Reserved fields.
	68	+ * @reqs: Union of different types of request. Byte offset: 64.
	69	+ * @reqs.pio_request: PIO request data of the I/O request.
	70	+ * @reqs.mmio_request: MMIO request data of the I/O request.
	71	+ * @reqs.data: Raw data of the I/O request.
	72	+ * @reserved1: Reserved fields.
	73	+ * @kernel_handled: Flag indicates this request need be handled in kernel.
	74	+ * @processed: The status of this request (ACRN_IOREQ_STATE_*).
	75	+ *
	76	+ * The state transitions of ACRN I/O request:
	77	+ *
	78	+ * FREE -> PENDING -> PROCESSING -> COMPLETE -> FREE -> ...
	79	+ *
	80	+ * An I/O request in COMPLETE or FREE state is owned by the hypervisor. HSM and
	81	+ * ACRN userspace are in charge of processing the others.
	82	+ *
	83	+ * On basis of the states illustrated above, a typical lifecycle of ACRN IO
	84	+ * request would look like:
	85	+ *
	86	+ * Flow (assume the initial state is FREE)
	87	+ * \|
	88	+ * \| Service VM vCPU 0 Service VM vCPU x User vCPU y
	89	+ * \|
	90	+ * \| hypervisor:
	91	+ * \| fills in type, addr, etc.
	92	+ * \| pauses the User VM vCPU y
	93	+ * \| sets the state to PENDING (a)
	94	+ * \| fires an upcall to Service VM
	95	+ * \|
	96	+ * \| HSM:
	97	+ * \| scans for PENDING requests
	98	+ * \| sets the states to PROCESSING (b)
	99	+ * \| assigns the requests to clients (c)
	100	+ * V
	101	+ * \| client:
	102	+ * \| scans for the assigned requests
	103	+ * \| handles the requests (d)
	104	+ * \| HSM:
	105	+ * \| sets states to COMPLETE
	106	+ * \| notifies the hypervisor
	107	+ * \|
	108	+ * \| hypervisor:
	109	+ * \| resumes User VM vCPU y (e)
	110	+ * \|
	111	+ * \| hypervisor:
	112	+ * \| post handling (f)
	113	+ * V sets states to FREE
	114	+ *
	115	+ * Note that the procedures (a) to (f) in the illustration above require to be
	116	+ * strictly processed in the order. One vCPU cannot trigger another request of
	117	+ * I/O emulation before completing the previous one.
	118	+ *
	119	+ * Atomic and barriers are required when HSM and hypervisor accessing the state
	120	+ * of &struct acrn_io_request.
	121	+ *
	122	+ */
	123	+struct acrn_io_request {
	124	+ __u32 type;
	125	+ __u32 completion_polling;
	126	+ __u32 reserved0[14];
	127	+ union {
	128	+ struct acrn_pio_request pio_request;
	129	+ struct acrn_mmio_request mmio_request;
	130	+ __u64 data[8];
	131	+ } reqs;
	132	+ __u32 reserved1;
	133	+ __u32 kernel_handled;
	134	+ __u32 processed;
	135	+} __attribute__((aligned(256)));
	136	+
	137	+struct acrn_io_request_buffer {
	138	+ union {
	139	+ struct acrn_io_request req_slot[ACRN_IO_REQUEST_MAX];
	140	+ __u8 reserved[4096];
	141	+ };
	142	+};
	143	+
	144	+/**
	145	+ * struct acrn_ioreq_notify - The structure of ioreq completion notification
	146	+ * @vmid: User VM ID
	147	+ * @reserved: Reserved and should be 0
	148	+ * @vcpu: vCPU ID
	149	+ */
	150	+struct acrn_ioreq_notify {
	151	+ __u16 vmid;
	152	+ __u16 reserved;
	153	+ __u32 vcpu;
	154	+};
	155	+
17	156	/**
18	157	* struct acrn_vm_creation - Info to create a User VM
19	158	* @vmid: User VM ID returned from the hypervisor
@@ -218,6 +357,17 @@ struct acrn_vm_memmap { · 198 unmodified lines
218	357	#define ACRN_IOCTL_SET_VCPU_REGS \
219	358	_IOW(ACRN_IOCTL_TYPE, 0x16, struct acrn_vcpu_regs)
220	359
	360	+#define ACRN_IOCTL_NOTIFY_REQUEST_FINISH \
	361	+ _IOW(ACRN_IOCTL_TYPE, 0x31, struct acrn_ioreq_notify)
	362	+#define ACRN_IOCTL_CREATE_IOREQ_CLIENT \
	363	+ _IO(ACRN_IOCTL_TYPE, 0x32)
	364	+#define ACRN_IOCTL_ATTACH_IOREQ_CLIENT \
	365	+ _IO(ACRN_IOCTL_TYPE, 0x33)
	366	+#define ACRN_IOCTL_DESTROY_IOREQ_CLIENT \
	367	+ _IO(ACRN_IOCTL_TYPE, 0x34)
	368	+#define ACRN_IOCTL_CLEAR_VM_IOREQ \
	369	+ _IO(ACRN_IOCTL_TYPE, 0x35)
	370	+
221	371	#define ACRN_IOCTL_SET_MEMSEG \
222	372	_IOW(ACRN_IOCTL_TYPE, 0x41, struct acrn_vm_memmap)
223	373	#define ACRN_IOCTL_UNSET_MEMSEG \