Merge tag 'x86_sev_for_v5.19_rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull AMD SEV-SNP support from Borislav Petkov: "The third AMD confidential computing feature called Secure Nested Paging. Add to confidential guests the necessary memory integrity protection against malicious hypervisor-based attacks like data replay, memory remapping and others, thus achieving a stronger isolation from the hypervisor. At the core of the functionality is a new structure called a reverse map table (RMP) with which the guest has a say in which pages get assigned to it and gets notified when a page which it owns, gets accessed/modified under the covers so that the guest can take an appropriate action. In addition, add support for the whole machinery needed to launch a SNP guest, details of which is properly explained in each patch. And last but not least, the series refactors and improves parts of the previous SEV support so that the new code is accomodated properly and not just bolted on" * tag 'x86_sev_for_v5.19_rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (60 commits) x86/entry: Fixup objtool/ibt validation x86/sev: Mark the code returning to user space as syscall gap x86/sev: Annotate stack change in the #VC handler x86/sev: Remove duplicated assignment to variable info x86/sev: Fix address space sparse warning x86/sev: Get the AP jump table address from secrets page x86/sev: Add missing __init annotations to SEV init routines virt: sevguest: Rename the sevguest dir and files to sev-guest virt: sevguest: Change driver name to reflect generic SEV support x86/boot: Put globals that are accessed early into the .data section x86/boot: Add an efi.h header for the decompressor virt: sevguest: Fix bool function returning negative value virt: sevguest: Fix return value check in alloc_shared_pages() x86/sev-es: Replace open-coded hlt-loop with sev_es_terminate() virt: sevguest: Add documentation for SEV-SNP CPUID Enforcement virt: sevguest: Add support to get extended report virt: sevguest: Add support to derive key virt: Add SEV-SNP guest driver x86/sev: Register SEV-SNP guest request platform device x86/sev: Provide support for SNP guest request NAEs ...

Linus Torvalds committed 4 years ago

Commit eb39e372 parents 0be3ff0 ce6565262 files · +3860 −392Browse files

Unified · Split

62 files

Documentation/admin-guide/kernel-parameters.txt +2 −0
Documentation/virt/coco/sev-guest.rst +155 −0
Documentation/virt/index.rst +1 −0
Documentation/x86/x86_64/boot-options.rst +14 −0
Documentation/x86/zero-page.rst +2 −0
arch/x86/boot/compressed/Makefile +1 −0
arch/x86/boot/compressed/acpi.c +35 −141
arch/x86/boot/compressed/early_serial_console.c +2 −1
arch/x86/boot/compressed/efi.c +234 −0
arch/x86/boot/compressed/efi.h +126 −0
arch/x86/boot/compressed/head_64.S +22 −15
arch/x86/boot/compressed/ident_map_64.c +28 −11
arch/x86/boot/compressed/idt_64.c +17 −1
arch/x86/boot/compressed/kaslr.c +1 −2
arch/x86/boot/compressed/mem_encrypt.S +0 −36
arch/x86/boot/compressed/misc.c +4 −1
arch/x86/boot/compressed/misc.h +54 −2
arch/x86/boot/compressed/pgtable_64.c +1 −2
arch/x86/boot/compressed/sev.c +247 −16
arch/x86/boot/cpucheck.c +15 −15
arch/x86/boot/msr.h +26 −0
arch/x86/coco/core.c +3 −0
arch/x86/entry/entry_64.S +6 −0
arch/x86/entry/entry_64_compat.S +5 −0
arch/x86/include/asm/bootparam_utils.h +1 −0
arch/x86/include/asm/cpuid.h +34 −0
arch/x86/include/asm/msr-index.h +2 −0
arch/x86/include/asm/msr.h +1 −10
arch/x86/include/asm/proto.h +4 −0
arch/x86/include/asm/ptrace.h +4 −0
arch/x86/include/asm/setup.h +0 −1
arch/x86/include/asm/sev-common.h +82 −0
arch/x86/include/asm/sev.h +135 −2
arch/x86/include/asm/shared/msr.h +15 −0
arch/x86/include/asm/svm.h +148 −23
arch/x86/include/uapi/asm/bootparam.h +3 −1
arch/x86/include/uapi/asm/svm.h +13 −0
arch/x86/kernel/Makefile +0 −2
arch/x86/kernel/cpu/common.c +4 −0
arch/x86/kernel/head64.c +16 −13
arch/x86/kernel/head_64.S +34 −3
arch/x86/kernel/probe_roms.c +12 −1
arch/x86/kernel/sev-shared.c +497 −37
arch/x86/kernel/sev.c +838 −17
arch/x86/kernel/smpboot.c +3 −0
arch/x86/kvm/cpuid.c +2 −17
arch/x86/kvm/svm/sev.c +11 −11
arch/x86/kvm/svm/svm.c +4 −4
arch/x86/kvm/svm/svm.h +2 −2
arch/x86/mm/mem_encrypt.c +4 −0
arch/x86/mm/mem_encrypt_amd.c +67 −4
arch/x86/mm/mem_encrypt_identity.c +8 −0
arch/x86/realmode/init.c +1 −1
drivers/virt/Kconfig +2 −0
drivers/virt/Makefile +1 −0
drivers/virt/coco/sev-guest/Kconfig +14 −0
drivers/virt/coco/sev-guest/Makefile +2 −0
drivers/virt/coco/sev-guest/sev-guest.c +743 −0
drivers/virt/coco/sev-guest/sev-guest.h +63 −0
include/linux/cc_platform.h +8 −0
include/linux/efi.h +1 −0
include/uapi/linux/sev-guest.h +80 −0

Documentation/admin-guide/kernel-parameters.txt+2−0

@@ -5383,6 +5383,8 @@ · 5382 unmodified lines
5383		5383
5384	serialnumber [BUGS=X86-32]	5384	serialnumber [BUGS=X86-32]
5385		5385
		5386	sev=option[,option...] [X86-64] See Documentation/x86/x86_64/boot-options.rst
		5387
5386	shapers= [NET]	5388	shapers= [NET]
5387	Maximal number of shapers.	5389	Maximal number of shapers.
5388		5390

Documentation/virt/coco/sev-guest.rst+155−0

@@ -0,0 +1,155 @@
	1	.. SPDX-License-Identifier: GPL-2.0
	2
	3	===================================================================
	4	The Definitive SEV Guest API Documentation
	5	===================================================================
	6
	7	1. General description
	8	======================
	9
	10	The SEV API is a set of ioctls that are used by the guest or hypervisor
	11	to get or set a certain aspect of the SEV virtual machine. The ioctls belong
	12	to the following classes:
	13
	14	- Hypervisor ioctls: These query and set global attributes which affect the
	15	whole SEV firmware. These ioctl are used by platform provisioning tools.
	16
	17	- Guest ioctls: These query and set attributes of the SEV virtual machine.
	18
	19	2. API description
	20	==================
	21
	22	This section describes ioctls that is used for querying the SEV guest report
	23	from the SEV firmware. For each ioctl, the following information is provided
	24	along with a description:
	25
	26	Technology:
	27	which SEV technology provides this ioctl. SEV, SEV-ES, SEV-SNP or all.
	28
	29	Type:
	30	hypervisor or guest. The ioctl can be used inside the guest or the
	31	hypervisor.
	32
	33	Parameters:
	34	what parameters are accepted by the ioctl.
	35
	36	Returns:
	37	the return value. General error numbers (-ENOMEM, -EINVAL)
	38	are not detailed, but errors with specific meanings are.
	39
	40	The guest ioctl should be issued on a file descriptor of the /dev/sev-guest device.
	41	The ioctl accepts struct snp_user_guest_request. The input and output structure is
	42	specified through the req_data and resp_data field respectively. If the ioctl fails
	43	to execute due to a firmware error, then fw_err code will be set otherwise the
	44	fw_err will be set to 0x00000000000000ff.
	45
	46	The firmware checks that the message sequence counter is one greater than
	47	the guests message sequence counter. If guest driver fails to increment message
	48	counter (e.g. counter overflow), then -EIO will be returned.
	49
	50	::
	51
	52	struct snp_guest_request_ioctl {
	53	/* Message version number */
	54	__u32 msg_version;
	55
	56	/* Request and response structure address */
	57	__u64 req_data;
	58	__u64 resp_data;
	59
	60	/* firmware error code on failure (see psp-sev.h) */
	61	__u64 fw_err;
	62	};
	63
	64	2.1 SNP_GET_REPORT
	65	------------------
	66
	67	:Technology: sev-snp
	68	:Type: guest ioctl
	69	:Parameters (in): struct snp_report_req
	70	:Returns (out): struct snp_report_resp on success, -negative on error
	71
	72	The SNP_GET_REPORT ioctl can be used to query the attestation report from the
	73	SEV-SNP firmware. The ioctl uses the SNP_GUEST_REQUEST (MSG_REPORT_REQ) command
	74	provided by the SEV-SNP firmware to query the attestation report.
	75
	76	On success, the snp_report_resp.data will contains the report. The report
	77	contain the format described in the SEV-SNP specification. See the SEV-SNP
	78	specification for further details.
	79
	80	2.2 SNP_GET_DERIVED_KEY
	81	-----------------------
	82	:Technology: sev-snp
	83	:Type: guest ioctl
	84	:Parameters (in): struct snp_derived_key_req
	85	:Returns (out): struct snp_derived_key_resp on success, -negative on error
	86
	87	The SNP_GET_DERIVED_KEY ioctl can be used to get a key derive from a root key.
	88	The derived key can be used by the guest for any purpose, such as sealing keys
	89	or communicating with external entities.
	90
	91	The ioctl uses the SNP_GUEST_REQUEST (MSG_KEY_REQ) command provided by the
	92	SEV-SNP firmware to derive the key. See SEV-SNP specification for further details
	93	on the various fields passed in the key derivation request.
	94
	95	On success, the snp_derived_key_resp.data contains the derived key value. See
	96	the SEV-SNP specification for further details.
	97
	98
	99	2.3 SNP_GET_EXT_REPORT
	100	----------------------
	101	:Technology: sev-snp
	102	:Type: guest ioctl
	103	:Parameters (in/out): struct snp_ext_report_req
	104	:Returns (out): struct snp_report_resp on success, -negative on error
	105
	106	The SNP_GET_EXT_REPORT ioctl is similar to the SNP_GET_REPORT. The difference is
	107	related to the additional certificate data that is returned with the report.
	108	The certificate data returned is being provided by the hypervisor through the
	109	SNP_SET_EXT_CONFIG.
	110
	111	The ioctl uses the SNP_GUEST_REQUEST (MSG_REPORT_REQ) command provided by the SEV-SNP
	112	firmware to get the attestation report.
	113
	114	On success, the snp_ext_report_resp.data will contain the attestation report
	115	and snp_ext_report_req.certs_address will contain the certificate blob. If the
	116	length of the blob is smaller than expected then snp_ext_report_req.certs_len will
	117	be updated with the expected value.
	118
	119	See GHCB specification for further detail on how to parse the certificate blob.
	120
	121	3. SEV-SNP CPUID Enforcement
	122	============================
	123
	124	SEV-SNP guests can access a special page that contains a table of CPUID values
	125	that have been validated by the PSP as part of the SNP_LAUNCH_UPDATE firmware
	126	command. It provides the following assurances regarding the validity of CPUID
	127	values:
	128
	129	- Its address is obtained via bootloader/firmware (via CC blob), and those
	130	binaries will be measured as part of the SEV-SNP attestation report.
	131	- Its initial state will be encrypted/pvalidated, so attempts to modify
	132	it during run-time will result in garbage being written, or #VC exceptions
	133	being generated due to changes in validation state if the hypervisor tries
	134	to swap the backing page.
	135	- Attempts to bypass PSP checks by the hypervisor by using a normal page, or
	136	a non-CPUID encrypted page will change the measurement provided by the
	137	SEV-SNP attestation report.
	138	- The CPUID page contents are not measured, but attempts to modify the
	139	expected contents of a CPUID page as part of guest initialization will be
	140	gated by the PSP CPUID enforcement policy checks performed on the page
	141	during SNP_LAUNCH_UPDATE, and noticeable later if the guest owner
	142	implements their own checks of the CPUID values.
	143
	144	It is important to note that this last assurance is only useful if the kernel
	145	has taken care to make use of the SEV-SNP CPUID throughout all stages of boot.
	146	Otherwise, guest owner attestation provides no assurance that the kernel wasn't
	147	fed incorrect values at some point during boot.
	148
	149
	150	Reference
	151	---------
	152
	153	SEV-SNP and GHCB specification: developer.amd.com/sev
	154
	155	The driver is based on SEV-SNP firmware spec 0.9 and GHCB spec version 2.0.

Documentation/virt/index.rst+1−0

@@ -13,6 +13,7 @@ Linux Virtualization Support · 12 unmodified lines
13	guest-halt-polling	13	guest-halt-polling
14	ne_overview	14	ne_overview
15	acrn/index	15	acrn/index
		16	coco/sev-guest
16		17
17	.. only:: html and subproject	18	.. only:: html and subproject
18		19

Documentation/x86/x86_64/boot-options.rst+14−0

@@ -310,3 +310,17 @@ Miscellaneous · 309 unmodified lines
310	Do not use GB pages for kernel direct mappings.	310	Do not use GB pages for kernel direct mappings.
311	gbpages	311	gbpages
312	Use GB pages for kernel direct mappings.	312	Use GB pages for kernel direct mappings.
		313
		314
		315	AMD SEV (Secure Encrypted Virtualization)
		316	=========================================
		317	Options relating to AMD SEV, specified via the following format:
		318
		319	::
		320
		321	sev=option1[,option2]
		322
		323	The available options are:
		324
		325	debug
		326	Enable debug messages.

Documentation/x86/zero-page.rst+2−0

 058/008		ALL	tboot_addr      	Physical address of tboot shared page
 060/010		ALL	ist_info		Intel SpeedStep (IST) BIOS support information
 						(struct ist_info)
+070/008		ALL	acpi_rsdp_addr		Physical address of ACPI RSDP table
 080/010		ALL	hd0_info		hd0 disk parameter, OBSOLETE!!
 090/010		ALL	hd1_info		hd1 disk parameter, OBSOLETE!!
 0A0/010		ALL	sys_desc_table		System description table (struct sys_desc_table),
 0C0/004		ALL	ext_ramdisk_image	ramdisk_image high 32bits
 0C4/004		ALL	ext_ramdisk_size	ramdisk_size high 32bits
 0C8/004		ALL	ext_cmd_line_ptr	cmd_line_ptr high 32bits
+13C/004		ALL	cc_blob_address		Physical address of Confidential Computing blob
 140/080		ALL	edid_info		Video mode setup (struct edid_info)
 1C0/020		ALL	efi_info		EFI 32 information (struct efi_info)
 1E0/004		ALL	alt_mem_k		Alternative mem check, in KB

arch/x86/boot/compressed/Makefile+1−0

@@ -103,6 +103,7 @@ endif · 102 unmodified lines
103	vmlinux-objs-$(CONFIG_ACPI) += $(obj)/acpi.o	103	vmlinux-objs-$(CONFIG_ACPI) += $(obj)/acpi.o
104		104
105	vmlinux-objs-$(CONFIG_EFI_MIXED) += $(obj)/efi_thunk_$(BITS).o	105	vmlinux-objs-$(CONFIG_EFI_MIXED) += $(obj)/efi_thunk_$(BITS).o
		106	vmlinux-objs-$(CONFIG_EFI) += $(obj)/efi.o
106	efi-obj-$(CONFIG_EFI_STUB) = $(objtree)/drivers/firmware/efi/libstub/lib.a	107	efi-obj-$(CONFIG_EFI_STUB) = $(objtree)/drivers/firmware/efi/libstub/lib.a
107		108
108	$(obj)/vmlinux: $(vmlinux-objs-y) $(efi-obj-y) FORCE	109	$(obj)/vmlinux: $(vmlinux-objs-y) $(efi-obj-y) FORCE

arch/x86/boot/compressed/acpi.c+35−141

@@ -3,10 +3,9 @@ · 2 unmodified lines
3	#include "misc.h"	3	#include "misc.h"
4	#include "error.h"	4	#include "error.h"
5	#include "../string.h"	5	#include "../string.h"
		6	#include "efi.h"
6		7
7	#include <linux/numa.h>	8	#include <linux/numa.h>
8	#include <linux/efi.h>
9	#include <asm/efi.h>
10		9
11	/*	10	/*
12	* Longest parameter of 'acpi=' is 'copy_dsdt', plus an extra '\0'	11	* Longest parameter of 'acpi=' is 'copy_dsdt', plus an extra '\0'
@@ -20,153 +19,56 @@ · 7 unmodified lines
20	*/	19	*/
21	struct mem_vector immovable_mem[MAX_NUMNODES*2];	20	struct mem_vector immovable_mem[MAX_NUMNODES*2];
22		21
23	/*
24	* Search EFI system tables for RSDP. If both ACPI_20_TABLE_GUID and
25	* ACPI_TABLE_GUID are found, take the former, which has more features.
26	*/
27	static acpi_physical_address	22	static acpi_physical_address
28	__efi_get_rsdp_addr(unsigned long config_tables, unsigned int nr_tables,	23	__efi_get_rsdp_addr(unsigned long cfg_tbl_pa, unsigned int cfg_tbl_len)
29	bool efi_64)
30	{	24	{
31	acpi_physical_address rsdp_addr = 0;
32
33	#ifdef CONFIG_EFI	25	#ifdef CONFIG_EFI
34	int i;	26	unsigned long rsdp_addr;
35		27	int ret;
36	/* Get EFI tables from systab. */
37	for (i = 0; i < nr_tables; i++) {
38	acpi_physical_address table;
39	efi_guid_t guid;
40
41	if (efi_64) {
42	efi_config_table_64_t tbl = (efi_config_table_64_t )config_tables + i;
43
44	guid = tbl->guid;
45	table = tbl->table;
46
47	if (!IS_ENABLED(CONFIG_X86_64) && table >> 32) {
48	debug_putstr("Error getting RSDP address: EFI config table located above 4GB.\n");
49	return 0;
50	}
51	} else {
52	efi_config_table_32_t tbl = (efi_config_table_32_t )config_tables + i;
53
54	guid = tbl->guid;
55	table = tbl->table;
56	}
57		28
58	if (!(efi_guidcmp(guid, ACPI_TABLE_GUID)))	29	/*
59	rsdp_addr = table;	30	* Search EFI system tables for RSDP. Preferred is ACPI_20_TABLE_GUID to
60	else if (!(efi_guidcmp(guid, ACPI_20_TABLE_GUID)))	31	* ACPI_TABLE_GUID because it has more features.
61	return table;	32	*/
62	}	33	rsdp_addr = efi_find_vendor_table(boot_params, cfg_tbl_pa, cfg_tbl_len,
		34	ACPI_20_TABLE_GUID);
		35	if (rsdp_addr)
		36	return (acpi_physical_address)rsdp_addr;
		37
		38	/* No ACPI_20_TABLE_GUID found, fallback to ACPI_TABLE_GUID. */
		39	rsdp_addr = efi_find_vendor_table(boot_params, cfg_tbl_pa, cfg_tbl_len,
		40	ACPI_TABLE_GUID);
		41	if (rsdp_addr)
		42	return (acpi_physical_address)rsdp_addr;
		43
		44	debug_putstr("Error getting RSDP address.\n");
63	#endif	45	#endif
64	return rsdp_addr;	46	return 0;
65	}
66
67	/* EFI/kexec support is 64-bit only. */
68	#ifdef CONFIG_X86_64
69	static struct efi_setup_data *get_kexec_setup_data_addr(void)
70	{
71	struct setup_data *data;
72	u64 pa_data;
73
74	pa_data = boot_params->hdr.setup_data;
75	while (pa_data) {
76	data = (struct setup_data *)pa_data;
77	if (data->type == SETUP_EFI)
78	return (struct efi_setup_data *)(pa_data + sizeof(struct setup_data));
79
80	pa_data = data->next;
81	}
82	return NULL;
83	}
84
85	static acpi_physical_address kexec_get_rsdp_addr(void)
86	{
87	efi_system_table_64_t *systab;
88	struct efi_setup_data *esd;
89	struct efi_info *ei;
90	char *sig;
91
92	esd = (struct efi_setup_data *)get_kexec_setup_data_addr();
93	if (!esd)
94	return 0;
95
96	if (!esd->tables) {
97	debug_putstr("Wrong kexec SETUP_EFI data.\n");
98	return 0;
99	}
100
101	ei = &boot_params->efi_info;
102	sig = (char *)&ei->efi_loader_signature;
103	if (strncmp(sig, EFI64_LOADER_SIGNATURE, 4)) {
104	debug_putstr("Wrong kexec EFI loader signature.\n");
105	return 0;
106	}
107
108	/* Get systab from boot params. */
109	systab = (efi_system_table_64_t *) (ei->efi_systab \| ((__u64)ei->efi_systab_hi << 32));
110	if (!systab)
111	error("EFI system table not found in kexec boot_params.");
112
113	return __efi_get_rsdp_addr((unsigned long)esd->tables, systab->nr_tables, true);
114	}	47	}
115	#else
116	static acpi_physical_address kexec_get_rsdp_addr(void) { return 0; }
117	#endif /* CONFIG_X86_64 */
118		48
119	static acpi_physical_address efi_get_rsdp_addr(void)	49	static acpi_physical_address efi_get_rsdp_addr(void)
120	{	50	{
121	#ifdef CONFIG_EFI	51	#ifdef CONFIG_EFI
122	unsigned long systab, config_tables;	52	unsigned long cfg_tbl_pa = 0;
		53	unsigned int cfg_tbl_len;
		54	unsigned long systab_pa;
123	unsigned int nr_tables;	55	unsigned int nr_tables;
124	struct efi_info *ei;	56	enum efi_type et;
125	bool efi_64;	57	int ret;
126	char *sig;
127
128	ei = &boot_params->efi_info;
129	sig = (char *)&ei->efi_loader_signature;
130
131	if (!strncmp(sig, EFI64_LOADER_SIGNATURE, 4)) {
132	efi_64 = true;
133	} else if (!strncmp(sig, EFI32_LOADER_SIGNATURE, 4)) {
134	efi_64 = false;
135	} else {
136	debug_putstr("Wrong EFI loader signature.\n");
137	return 0;
138	}
139		58
140	/* Get systab from boot params. */	59	et = efi_get_type(boot_params);
141	#ifdef CONFIG_X86_64	60	if (et == EFI_TYPE_NONE)
142	systab = ei->efi_systab \| ((__u64)ei->efi_systab_hi << 32);
143	#else
144	if (ei->efi_systab_hi \|\| ei->efi_memmap_hi) {
145	debug_putstr("Error getting RSDP address: EFI system table located above 4GB.\n");
146	return 0;	61	return 0;
147	}
148	systab = ei->efi_systab;
149	#endif
150	if (!systab)
151	error("EFI system table not found.");
152		62
153	/* Handle EFI bitness properly */	63	systab_pa = efi_get_system_table(boot_params);
154	if (efi_64) {	64	if (!systab_pa)
155	efi_system_table_64_t stbl = (efi_system_table_64_t )systab;	65	error("EFI support advertised, but unable to locate system table.");
156		66
157	config_tables = stbl->tables;	67	ret = efi_get_conf_table(boot_params, &cfg_tbl_pa, &cfg_tbl_len);
158	nr_tables = stbl->nr_tables;	68	if (ret \|\| !cfg_tbl_pa)
159	} else {	69	error("EFI config table not found.");
160	efi_system_table_32_t stbl = (efi_system_table_32_t )systab;
161		70
162	config_tables = stbl->tables;	71	return __efi_get_rsdp_addr(cfg_tbl_pa, cfg_tbl_len);
163	nr_tables = stbl->nr_tables;
164	}
165
166	if (!config_tables)
167	error("EFI config tables not found.");
168
169	return __efi_get_rsdp_addr(config_tables, nr_tables, efi_64);
170	#else	72	#else
171	return 0;	73	return 0;
172	#endif	74	#endif
@@ -256,14 +158,6 @@ acpi_physical_address get_rsdp_addr(void) · 83 unmodified lines
256		158
257	pa = boot_params->acpi_rsdp_addr;	159	pa = boot_params->acpi_rsdp_addr;
258		160
259	/*
260	* Try to get EFI data from setup_data. This can happen when we're a
261	* kexec'ed kernel and kexec(1) has passed all the required EFI info to
262	* us.
263	*/
264	if (!pa)
265	pa = kexec_get_rsdp_addr();
266
267	if (!pa)	161	if (!pa)
268	pa = efi_get_rsdp_addr();	162	pa = efi_get_rsdp_addr();
269		163

arch/x86/boot/compressed/early_serial_console.c+2−1

@@ -1,5 +1,6 @@
1	#include "misc.h"	1	#include "misc.h"
2		2
3	int early_serial_base;	3	/* This might be accessed before .bss is cleared, so use .data instead. */
		4	int early_serial_base __section(".data");
4		5
5	#include "../early_serial_console.c"	6	#include "../early_serial_console.c"

arch/x86/boot/compressed/efi.c+234−0

@@ -0,0 +1,234 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	/*
	3	* Helpers for early access to EFI configuration table.
	4	*
	5	* Originally derived from arch/x86/boot/compressed/acpi.c
	6	*/
	7
	8	#include "misc.h"
	9
	10	/**
	11	* efi_get_type - Given a pointer to boot_params, determine the type of EFI environment.
	12	*
	13	* @bp: pointer to boot_params
	14	*
	15	* Return: EFI_TYPE_{32,64} for valid EFI environments, EFI_TYPE_NONE otherwise.
	16	*/
	17	enum efi_type efi_get_type(struct boot_params *bp)
	18	{
	19	struct efi_info *ei;
	20	enum efi_type et;
	21	const char *sig;
	22
	23	ei = &bp->efi_info;
	24	sig = (char *)&ei->efi_loader_signature;
	25
	26	if (!strncmp(sig, EFI64_LOADER_SIGNATURE, 4)) {
	27	et = EFI_TYPE_64;
	28	} else if (!strncmp(sig, EFI32_LOADER_SIGNATURE, 4)) {
	29	et = EFI_TYPE_32;
	30	} else {
	31	debug_putstr("No EFI environment detected.\n");
	32	et = EFI_TYPE_NONE;
	33	}
	34
	35	#ifndef CONFIG_X86_64
	36	/*
	37	* Existing callers like acpi.c treat this case as an indicator to
	38	* fall-through to non-EFI, rather than an error, so maintain that
	39	* functionality here as well.
	40	*/
	41	if (ei->efi_systab_hi \|\| ei->efi_memmap_hi) {
	42	debug_putstr("EFI system table is located above 4GB and cannot be accessed.\n");
	43	et = EFI_TYPE_NONE;
	44	}
	45	#endif
	46
	47	return et;
	48	}
	49
	50	/**
	51	* efi_get_system_table - Given a pointer to boot_params, retrieve the physical address
	52	* of the EFI system table.
	53	*
	54	* @bp: pointer to boot_params
	55	*
	56	* Return: EFI system table address on success. On error, return 0.
	57	*/
	58	unsigned long efi_get_system_table(struct boot_params *bp)
	59	{
	60	unsigned long sys_tbl_pa;
	61	struct efi_info *ei;
	62	enum efi_type et;
	63
	64	/* Get systab from boot params. */
	65	ei = &bp->efi_info;
	66	#ifdef CONFIG_X86_64
	67	sys_tbl_pa = ei->efi_systab \| ((__u64)ei->efi_systab_hi << 32);
	68	#else
	69	sys_tbl_pa = ei->efi_systab;
	70	#endif
	71	if (!sys_tbl_pa) {
	72	debug_putstr("EFI system table not found.");
	73	return 0;
	74	}
	75
	76	return sys_tbl_pa;
	77	}
	78
	79	/*
	80	* EFI config table address changes to virtual address after boot, which may
	81	* not be accessible for the kexec'd kernel. To address this, kexec provides
	82	* the initial physical address via a struct setup_data entry, which is
	83	* checked for here, along with some sanity checks.
	84	*/
	85	static struct efi_setup_data get_kexec_setup_data(struct boot_params bp,
	86	enum efi_type et)
	87	{
	88	#ifdef CONFIG_X86_64
	89	struct efi_setup_data *esd = NULL;
	90	struct setup_data *data;
	91	u64 pa_data;
	92
	93	pa_data = bp->hdr.setup_data;
	94	while (pa_data) {
	95	data = (struct setup_data *)pa_data;
	96	if (data->type == SETUP_EFI) {
	97	esd = (struct efi_setup_data *)(pa_data + sizeof(struct setup_data));
	98	break;
	99	}
	100
	101	pa_data = data->next;
	102	}
	103
	104	/*
	105	* Original ACPI code falls back to attempting normal EFI boot in these
	106	* cases, so maintain existing behavior by indicating non-kexec
	107	* environment to the caller, but print them for debugging.
	108	*/
	109	if (esd && !esd->tables) {
	110	debug_putstr("kexec EFI environment missing valid configuration table.\n");
	111	return NULL;
	112	}
	113
	114	return esd;
	115	#endif
	116	return NULL;
	117	}
	118
	119	/**
	120	* efi_get_conf_table - Given a pointer to boot_params, locate and return the physical
	121	* address of EFI configuration table.
	122	*
	123	* @bp: pointer to boot_params
	124	* @cfg_tbl_pa: location to store physical address of config table
	125	* @cfg_tbl_len: location to store number of config table entries
	126	*
	127	* Return: 0 on success. On error, return params are left unchanged.
	128	*/
	129	int efi_get_conf_table(struct boot_params bp, unsigned long cfg_tbl_pa,
	130	unsigned int *cfg_tbl_len)
	131	{
	132	unsigned long sys_tbl_pa;
	133	enum efi_type et;
	134	int ret;
	135
	136	if (!cfg_tbl_pa \|\| !cfg_tbl_len)
	137	return -EINVAL;
	138
	139	sys_tbl_pa = efi_get_system_table(bp);
	140	if (!sys_tbl_pa)
	141	return -EINVAL;
	142
	143	/* Handle EFI bitness properly */
	144	et = efi_get_type(bp);
	145	if (et == EFI_TYPE_64) {
	146	efi_system_table_64_t stbl = (efi_system_table_64_t )sys_tbl_pa;
	147	struct efi_setup_data *esd;
	148
	149	/* kexec provides an alternative EFI conf table, check for it. */
	150	esd = get_kexec_setup_data(bp, et);
	151
	152	*cfg_tbl_pa = esd ? esd->tables : stbl->tables;
	153	*cfg_tbl_len = stbl->nr_tables;
	154	} else if (et == EFI_TYPE_32) {
	155	efi_system_table_32_t stbl = (efi_system_table_32_t )sys_tbl_pa;
	156
	157	*cfg_tbl_pa = stbl->tables;
	158	*cfg_tbl_len = stbl->nr_tables;
	159	} else {
	160	return -EINVAL;
	161	}
	162
	163	return 0;
	164	}
	165
	166	/* Get vendor table address/guid from EFI config table at the given index */
	167	static int get_vendor_table(void *cfg_tbl, unsigned int idx,
	168	unsigned long *vendor_tbl_pa,
	169	efi_guid_t *vendor_tbl_guid,
	170	enum efi_type et)
	171	{
	172	if (et == EFI_TYPE_64) {
	173	efi_config_table_64_t tbl_entry = (efi_config_table_64_t )cfg_tbl + idx;
	174
	175	if (!IS_ENABLED(CONFIG_X86_64) && tbl_entry->table >> 32) {
	176	debug_putstr("Error: EFI config table entry located above 4GB.\n");
	177	return -EINVAL;
	178	}
	179
	180	*vendor_tbl_pa = tbl_entry->table;
	181	*vendor_tbl_guid = tbl_entry->guid;
	182
	183	} else if (et == EFI_TYPE_32) {
	184	efi_config_table_32_t tbl_entry = (efi_config_table_32_t )cfg_tbl + idx;
	185
	186	*vendor_tbl_pa = tbl_entry->table;
	187	*vendor_tbl_guid = tbl_entry->guid;
	188	} else {
	189	return -EINVAL;
	190	}
	191
	192	return 0;
	193	}
	194
	195	/**
	196	* efi_find_vendor_table - Given EFI config table, search it for the physical
	197	* address of the vendor table associated with GUID.
	198	*
	199	* @bp: pointer to boot_params
	200	* @cfg_tbl_pa: pointer to EFI configuration table
	201	* @cfg_tbl_len: number of entries in EFI configuration table
	202	* @guid: GUID of vendor table
	203	*
	204	* Return: vendor table address on success. On error, return 0.
	205	*/
	206	unsigned long efi_find_vendor_table(struct boot_params *bp,
	207	unsigned long cfg_tbl_pa,
	208	unsigned int cfg_tbl_len,
	209	efi_guid_t guid)
	210	{
	211	enum efi_type et;
	212	unsigned int i;
	213
	214	et = efi_get_type(bp);
	215	if (et == EFI_TYPE_NONE)
	216	return 0;
	217
	218	for (i = 0; i < cfg_tbl_len; i++) {
	219	unsigned long vendor_tbl_pa;
	220	efi_guid_t vendor_tbl_guid;
	221	int ret;
	222
	223	ret = get_vendor_table((void *)cfg_tbl_pa, i,
	224	&vendor_tbl_pa,
	225	&vendor_tbl_guid, et);
	226	if (ret)
	227	return 0;
	228
	229	if (!efi_guidcmp(guid, vendor_tbl_guid))
	230	return vendor_tbl_pa;
	231	}
	232
	233	return 0;
	234	}

arch/x86/boot/compressed/efi.h+126−0

@@ -0,0 +1,126 @@
	1	/* SPDX-License-Identifier: GPL-2.0 */
	2	#ifndef BOOT_COMPRESSED_EFI_H
	3	#define BOOT_COMPRESSED_EFI_H
	4
	5	#if defined(_LINUX_EFI_H) \|\| defined(_ASM_X86_EFI_H)
	6	#error Please do not include kernel proper namespace headers
	7	#endif
	8
	9	typedef guid_t efi_guid_t __aligned(__alignof__(u32));
	10
	11	#define EFI_GUID(a, b, c, d...) (efi_guid_t){ { \
	12	(a) & 0xff, ((a) >> 8) & 0xff, ((a) >> 16) & 0xff, ((a) >> 24) & 0xff, \
	13	(b) & 0xff, ((b) >> 8) & 0xff, \
	14	(c) & 0xff, ((c) >> 8) & 0xff, d } }
	15
	16	#define ACPI_TABLE_GUID EFI_GUID(0xeb9d2d30, 0x2d88, 0x11d3, 0x9a, 0x16, 0x00, 0x90, 0x27, 0x3f, 0xc1, 0x4d)
	17	#define ACPI_20_TABLE_GUID EFI_GUID(0x8868e871, 0xe4f1, 0x11d3, 0xbc, 0x22, 0x00, 0x80, 0xc7, 0x3c, 0x88, 0x81)
	18	#define EFI_CC_BLOB_GUID EFI_GUID(0x067b1f5f, 0xcf26, 0x44c5, 0x85, 0x54, 0x93, 0xd7, 0x77, 0x91, 0x2d, 0x42)
	19
	20	#define EFI32_LOADER_SIGNATURE "EL32"
	21	#define EFI64_LOADER_SIGNATURE "EL64"
	22
	23	/*
	24	* Generic EFI table header
	25	*/
	26	typedef struct {
	27	u64 signature;
	28	u32 revision;
	29	u32 headersize;
	30	u32 crc32;
	31	u32 reserved;
	32	} efi_table_hdr_t;
	33
	34	#define EFI_CONVENTIONAL_MEMORY 7
	35
	36	#define EFI_MEMORY_MORE_RELIABLE \
	37	((u64)0x0000000000010000ULL) /* higher reliability */
	38	#define EFI_MEMORY_SP ((u64)0x0000000000040000ULL) /* soft reserved */
	39
	40	#define EFI_PAGE_SHIFT 12
	41
	42	typedef struct {
	43	u32 type;
	44	u32 pad;
	45	u64 phys_addr;
	46	u64 virt_addr;
	47	u64 num_pages;
	48	u64 attribute;
	49	} efi_memory_desc_t;
	50
	51	#define efi_early_memdesc_ptr(map, desc_size, n) \
	52	(efi_memory_desc_t )((void )(map) + ((n) * (desc_size)))
	53
	54	typedef struct {
	55	efi_guid_t guid;
	56	u64 table;
	57	} efi_config_table_64_t;
	58
	59	typedef struct {
	60	efi_guid_t guid;
	61	u32 table;
	62	} efi_config_table_32_t;
	63
	64	typedef struct {
	65	efi_table_hdr_t hdr;
	66	u64 fw_vendor; /* physical addr of CHAR16 vendor string */
	67	u32 fw_revision;
	68	u32 __pad1;
	69	u64 con_in_handle;
	70	u64 con_in;
	71	u64 con_out_handle;
	72	u64 con_out;
	73	u64 stderr_handle;
	74	u64 stderr;
	75	u64 runtime;
	76	u64 boottime;
	77	u32 nr_tables;
	78	u32 __pad2;
	79	u64 tables;
	80	} efi_system_table_64_t;
	81
	82	typedef struct {
	83	efi_table_hdr_t hdr;
	84	u32 fw_vendor; /* physical addr of CHAR16 vendor string */
	85	u32 fw_revision;
	86	u32 con_in_handle;
	87	u32 con_in;
	88	u32 con_out_handle;
	89	u32 con_out;
	90	u32 stderr_handle;
	91	u32 stderr;
	92	u32 runtime;
	93	u32 boottime;
	94	u32 nr_tables;
	95	u32 tables;
	96	} efi_system_table_32_t;
	97
	98	/* kexec external ABI */
	99	struct efi_setup_data {
	100	u64 fw_vendor;
	101	u64 __unused;
	102	u64 tables;
	103	u64 smbios;
	104	u64 reserved[8];
	105	};
	106
	107	static inline int efi_guidcmp (efi_guid_t left, efi_guid_t right)
	108	{
	109	return memcmp(&left, &right, sizeof (efi_guid_t));
	110	}
	111
	112	#ifdef CONFIG_EFI
	113	bool __pure __efi_soft_reserve_enabled(void);
	114
	115	static inline bool __pure efi_soft_reserve_enabled(void)
	116	{
	117	return IS_ENABLED(CONFIG_EFI_SOFT_RESERVE)
	118	&& __efi_soft_reserve_enabled();
	119	}
	120	#else
	121	static inline bool efi_soft_reserve_enabled(void)
	122	{
	123	return false;
	124	}
	125	#endif /* CONFIG_EFI */
	126	#endif /* BOOT_COMPRESSED_EFI_H */

arch/x86/boot/compressed/head_64.S+22−15

 	subl	$32, %eax	/* Encryption bit is always above bit 31 */
 	bts	%eax, %edx	/* Set encryption mask for page tables */
 	/*
-	 * Mark SEV as active in sev_status so that startup32_check_sev_cbit()
+	 * Set MSR_AMD64_SEV_ENABLED_BIT in sev_status so that
-	 * will do a check. The sev_status memory will be fully initialized
+	 * startup32_check_sev_cbit() will do a check. sev_enable() will
-	 * with the contents of MSR_AMD_SEV_STATUS later in
+	 * initialize sev_status with all the bits reported by
-	 * set_sev_encryption_mask(). For now it is sufficient to know that SEV
+	 * MSR_AMD_SEV_STATUS later, but only MSR_AMD64_SEV_ENABLED_BIT
-	 * is active.
+	 * needs to be set for now.
 	 */
 	movl	$1, rva(sev_status)(%ebp)
 1:
 	call	load_stage1_idt
 	popq	%rsi
+#ifdef CONFIG_AMD_MEM_ENCRYPT
+	/*
+	 * Now that the stage1 interrupt handlers are set up, #VC exceptions from
+	 * CPUID instructions can be properly handled for SEV-ES guests.
+	 *
+	 * For SEV-SNP, the CPUID table also needs to be set up in advance of any
+	 * CPUID instructions being issued, so go ahead and do that now via
+	 * sev_enable(), which will also handle the rest of the SEV-related
+	 * detection/setup to ensure that has been done in advance of any dependent
+	 * code.
+	 */
+	pushq	%rsi
+	movq	%rsi, %rdi		/* real mode address */
+	call	sev_enable
+	popq	%rsi
+#endif
 	/*
 	 * paging_prepare() sets up the trampoline and checks if we need to
 	 * enable 5-level paging.
 	shrq	$3, %rcx
 	rep	stosq
-/*
- * If running as an SEV guest, the encryption mask is required in the
- * page-table setup code below. When the guest also has SEV-ES enabled
- * set_sev_encryption_mask() will cause #VC exceptions, but the stage2
- * handler can't map its GHCB because the page-table is not set up yet.
- * So set up the encryption mask here while still on the stage1 #VC
- * handler. Then load stage2 IDT and switch to the kernel's own
- * page-table.
- */
 	pushq	%rsi
-	call	set_sev_encryption_mask
 	call	load_stage2_idt
 	/* Pass boot_params to initialize_identity_maps() */

arch/x86/boot/compressed/ident_map_64.c+28−11

 /*
  * Adds the specified range to the identity mappings.
  */
-static void add_identity_map(unsigned long start, unsigned long end)
+void kernel_add_identity_map(unsigned long start, unsigned long end)
 {
 	int ret;
 	 * explicitly here in case the compressed kernel does not touch them,
 	 * or does not touch all the pages covering them.
 	 */
-	add_identity_map((unsigned long)_head, (unsigned long)_end);
+	kernel_add_identity_map((unsigned long)_head, (unsigned long)_end);
 	boot_params = rmode;
-	add_identity_map((unsigned long)boot_params, (unsigned long)(boot_params + 1));
+	kernel_add_identity_map((unsigned long)boot_params, (unsigned long)(boot_params + 1));
 	cmdline = get_cmd_line_ptr();
-	add_identity_map(cmdline, cmdline + COMMAND_LINE_SIZE);
+	kernel_add_identity_map(cmdline, cmdline + COMMAND_LINE_SIZE);
+	sev_prep_identity_maps(top_level_pgt);
 	/* Load the new page-table. */
-	sev_verify_cbit(top_level_pgt);
 	write_cr3(top_level_pgt);
 }
 	 * It should already exist, but keep things generic.
 	 *
 	 * To map the page just read from it and fault it in if there is no
-	 * mapping yet. add_identity_map() can't be called here because that
+	 * mapping yet. kernel_add_identity_map() can't be called here because
-	 * would unconditionally map the address on PMD level, destroying any
+	 * that would unconditionally map the address on PMD level, destroying
-	 * PTE-level mappings that might already exist. Use assembly here so
+	 * any PTE-level mappings that might already exist. Use assembly here
-	 * the access won't be optimized away.
+	 * so the access won't be optimized away.
 	 */
 	asm volatile("mov %[address], %%r9"
 		     :: [address] "g" (*(unsigned long *)address)
 	 * Changing encryption attributes of a page requires to flush it from
 	 * the caches.
 	 */
-	if ((set | clr) & _PAGE_ENC)
+	if ((set | clr) & _PAGE_ENC) {
 		clflush_page(address);
+		/*
+		 * If the encryption attribute is being cleared, change the page state
+		 * to shared in the RMP table.
+		 */
+		if (clr)
+			snp_set_page_shared(__pa(address & PAGE_MASK));
+	}
 	/* Update PTE */
 	pte = *ptep;
 	pte = pte_set_flags(pte, set);
 	pte = pte_clear_flags(pte, clr);
 	set_pte(ptep, pte);
+	/*
+	 * If the encryption attribute is being set, then change the page state to
+	 * private in the RMP entry. The page state change must be done after the PTE
+	 * is updated.
+	 */
+	if (set & _PAGE_ENC)
+		snp_set_page_private(__pa(address & PAGE_MASK));
 	/* Flush TLB after changing encryption attribute */
 	write_cr3(top_level_pgt);
 	 * Error code is sane - now identity map the 2M region around
 	 * the faulting address.
 	 */
-	add_identity_map(address, end);
+	kernel_add_identity_map(address, end);
 }

arch/x86/boot/compressed/idt_64.c+17−1

@@ -39,7 +39,23 @@ void load_stage1_idt(void) · 38 unmodified lines
39	load_boot_idt(&boot_idt_desc);	39	load_boot_idt(&boot_idt_desc);
40	}	40	}
41		41
42	/* Setup IDT after kernel jumping to .Lrelocated */	42	/*
		43	* Setup IDT after kernel jumping to .Lrelocated.
		44	*
		45	* initialize_identity_maps() needs a #PF handler to be setup
		46	* in order to be able to fault-in identity mapping ranges; see
		47	* do_boot_page_fault().
		48	*
		49	* This #PF handler setup needs to happen in load_stage2_idt() where the
		50	* IDT is loaded and there the #VC IDT entry gets setup too.
		51	*
		52	* In order to be able to handle #VCs, one needs a GHCB which
		53	* gets setup with an already set up pagetable, which is done in
		54	* initialize_identity_maps(). And there's the catch 22: the boot #VC
		55	* handler do_boot_stage2_vc() needs to call early_setup_ghcb() itself
		56	* (and, especially set_page_decrypted()) because the SEV-ES setup code
		57	* cannot initialize a GHCB as there's no #PF handler yet...
		58	*/
43	void load_stage2_idt(void)	59	void load_stage2_idt(void)
44	{	60	{
45	boot_idt_desc.address = (unsigned long)boot_idt;	61	boot_idt_desc.address = (unsigned long)boot_idt;

arch/x86/boot/compressed/kaslr.c+1−2

 #include "misc.h"
 #include "error.h"
 #include "../string.h"
+#include "efi.h"
 #include <generated/compile.h>
 #include <linux/module.h>
 #include <linux/uts.h>
 #include <linux/utsname.h>
 #include <linux/ctype.h>
-#include <linux/efi.h>
 #include <generated/utsrelease.h>
-#include <asm/efi.h>
 #define _SETUP
 #include <asm/setup.h>	/* For COMMAND_LINE_SIZE */

arch/x86/boot/compressed/mem_encrypt.S+0−36

@@ -187,42 +187,6 @@ SYM_CODE_END(startup32_vc_handler) · 186 unmodified lines
187	.code64	187	.code64
188		188
189	#include "../../kernel/sev_verify_cbit.S"	189	#include "../../kernel/sev_verify_cbit.S"
190	SYM_FUNC_START(set_sev_encryption_mask)
191	#ifdef CONFIG_AMD_MEM_ENCRYPT
192	push %rbp
193	push %rdx
194
195	movq %rsp, %rbp /* Save current stack pointer */
196
197	call get_sev_encryption_bit /* Get the encryption bit position */
198	testl %eax, %eax
199	jz .Lno_sev_mask
200
201	bts %rax, sme_me_mask(%rip) /* Create the encryption mask */
202
203	/*
204	* Read MSR_AMD64_SEV again and store it to sev_status. Can't do this in
205	* get_sev_encryption_bit() because this function is 32-bit code and
206	* shared between 64-bit and 32-bit boot path.
207	*/
208	movl $MSR_AMD64_SEV, %ecx /* Read the SEV MSR */
209	rdmsr
210
211	/* Store MSR value in sev_status */
212	shlq $32, %rdx
213	orq %rdx, %rax
214	movq %rax, sev_status(%rip)
215
216	.Lno_sev_mask:
217	movq %rbp, %rsp /* Restore original stack pointer */
218
219	pop %rdx
220	pop %rbp
221	#endif
222
223	xor %rax, %rax
224	RET
225	SYM_FUNC_END(set_sev_encryption_mask)
226		190
227	.data	191	.data
228		192

arch/x86/boot/compressed/misc.c+4−1

@@ -53,7 +53,10 @@ memptr free_mem_end_ptr; · 52 unmodified lines
53		53
54	static char *vidmem;	54	static char *vidmem;
55	static int vidport;	55	static int vidport;
56	static int lines, cols;	56
		57	/* These might be accessed before .bss is cleared, so use .data instead. */
		58	static int lines __section(".data");
		59	static int cols __section(".data");
57		60
58	#ifdef CONFIG_KERNEL_GZIP	61	#ifdef CONFIG_KERNEL_GZIP
59	#include "../../../../lib/decompress_inflate.c"	62	#include "../../../../lib/decompress_inflate.c"

arch/x86/boot/compressed/misc.h+54−2

@@ -34,6 +34,8 @@ · 33 unmodified lines
34	#define BOOT_BOOT_H	34	#define BOOT_BOOT_H
35	#include "../ctype.h"	35	#include "../ctype.h"
36		36
		37	#include "efi.h"
		38
37	#ifdef CONFIG_X86_64	39	#ifdef CONFIG_X86_64
38	#define memptr long	40	#define memptr long
39	#else	41	#else
@@ -120,17 +122,23 @@ static inline void console_init(void) · 80 unmodified lines
120	{ }	122	{ }
121	#endif	123	#endif
122		124
123	void set_sev_encryption_mask(void);
124
125	#ifdef CONFIG_AMD_MEM_ENCRYPT	125	#ifdef CONFIG_AMD_MEM_ENCRYPT
		126	void sev_enable(struct boot_params *bp);
126	void sev_es_shutdown_ghcb(void);	127	void sev_es_shutdown_ghcb(void);
127	extern bool sev_es_check_ghcb_fault(unsigned long address);	128	extern bool sev_es_check_ghcb_fault(unsigned long address);
		129	void snp_set_page_private(unsigned long paddr);
		130	void snp_set_page_shared(unsigned long paddr);
		131	void sev_prep_identity_maps(unsigned long top_level_pgt);
128	#else	132	#else
		133	static inline void sev_enable(struct boot_params *bp) { }
129	static inline void sev_es_shutdown_ghcb(void) { }	134	static inline void sev_es_shutdown_ghcb(void) { }
130	static inline bool sev_es_check_ghcb_fault(unsigned long address)	135	static inline bool sev_es_check_ghcb_fault(unsigned long address)
131	{	136	{
132	return false;	137	return false;
133	}	138	}
		139	static inline void snp_set_page_private(unsigned long paddr) { }
		140	static inline void snp_set_page_shared(unsigned long paddr) { }
		141	static inline void sev_prep_identity_maps(unsigned long top_level_pgt) { }
134	#endif	142	#endif
135		143
136	/* acpi.c */	144	/* acpi.c */
@@ -151,6 +159,7 @@ static inline int count_immovable_mem_regions(void) { return 0; } · 14 unmodified lines
151	#ifdef CONFIG_X86_5LEVEL	159	#ifdef CONFIG_X86_5LEVEL
152	extern unsigned int __pgtable_l5_enabled, pgdir_shift, ptrs_per_p4d;	160	extern unsigned int __pgtable_l5_enabled, pgdir_shift, ptrs_per_p4d;
153	#endif	161	#endif
		162	extern void kernel_add_identity_map(unsigned long start, unsigned long end);
154		163
155	/* Used by PAGE_KERN* macros: */	164	/* Used by PAGE_KERN* macros: */
156	extern pteval_t __default_kernel_pte_mask;	165	extern pteval_t __default_kernel_pte_mask;
@@ -172,4 +181,47 @@ void boot_stage2_vc(void); · 15 unmodified lines
172		181
173	unsigned long sev_verify_cbit(unsigned long cr3);	182	unsigned long sev_verify_cbit(unsigned long cr3);
174		183
		184	enum efi_type {
		185	EFI_TYPE_64,
		186	EFI_TYPE_32,
		187	EFI_TYPE_NONE,
		188	};
		189
		190	#ifdef CONFIG_EFI
		191	/* helpers for early EFI config table access */
		192	enum efi_type efi_get_type(struct boot_params *bp);
		193	unsigned long efi_get_system_table(struct boot_params *bp);
		194	int efi_get_conf_table(struct boot_params bp, unsigned long cfg_tbl_pa,
		195	unsigned int *cfg_tbl_len);
		196	unsigned long efi_find_vendor_table(struct boot_params *bp,
		197	unsigned long cfg_tbl_pa,
		198	unsigned int cfg_tbl_len,
		199	efi_guid_t guid);
		200	#else
		201	static inline enum efi_type efi_get_type(struct boot_params *bp)
		202	{
		203	return EFI_TYPE_NONE;
		204	}
		205
		206	static inline unsigned long efi_get_system_table(struct boot_params *bp)
		207	{
		208	return 0;
		209	}
		210
		211	static inline int efi_get_conf_table(struct boot_params *bp,
		212	unsigned long *cfg_tbl_pa,
		213	unsigned int *cfg_tbl_len)
		214	{
		215	return -ENOENT;
		216	}
		217
		218	static inline unsigned long efi_find_vendor_table(struct boot_params *bp,
		219	unsigned long cfg_tbl_pa,
		220	unsigned int cfg_tbl_len,
		221	efi_guid_t guid)
		222	{
		223	return 0;
		224	}
		225	#endif /* CONFIG_EFI */
		226
175	#endif /* BOOT_COMPRESSED_MISC_H */	227	#endif /* BOOT_COMPRESSED_MISC_H */

arch/x86/boot/compressed/pgtable_64.c+1−2

 // SPDX-License-Identifier: GPL-2.0
 #include "misc.h"
-#include <linux/efi.h>
 #include <asm/e820/types.h>
 #include <asm/processor.h>
-#include <asm/efi.h>
 #include "pgtable.h"
 #include "../string.h"
+#include "efi.h"
 #define BIOS_START_MIN		0x20000U	/* 128K, less than this is insane */
 #define BIOS_START_MAX		0x9f000U	/* 640K, absolute maximum */

arch/x86/boot/compressed/sev.c+247−16

@@ -20,8 +20,10 @@ · 19 unmodified lines
20	#include <asm/fpu/xcr.h>	20	#include <asm/fpu/xcr.h>
21	#include <asm/ptrace.h>	21	#include <asm/ptrace.h>
22	#include <asm/svm.h>	22	#include <asm/svm.h>
		23	#include <asm/cpuid.h>
23		24
24	#include "error.h"	25	#include "error.h"
		26	#include "../msr.h"
25		27
26	struct ghcb boot_ghcb_page __aligned(PAGE_SIZE);	28	struct ghcb boot_ghcb_page __aligned(PAGE_SIZE);
27	struct ghcb *boot_ghcb;	29	struct ghcb *boot_ghcb;
@@ -56,23 +58,19 @@ static unsigned long insn_get_seg_base(struct pt_regs *regs, int seg_reg_idx) · 28 unmodified lines
56		58
57	static inline u64 sev_es_rd_ghcb_msr(void)	59	static inline u64 sev_es_rd_ghcb_msr(void)
58	{	60	{
59	unsigned long low, high;	61	struct msr m;
60		62
61	asm volatile("rdmsr" : "=a" (low), "=d" (high) :	63	boot_rdmsr(MSR_AMD64_SEV_ES_GHCB, &m);
62	"c" (MSR_AMD64_SEV_ES_GHCB));
63		64
64	return ((high << 32) \| low);	65	return m.q;
65	}	66	}
66		67
67	static inline void sev_es_wr_ghcb_msr(u64 val)	68	static inline void sev_es_wr_ghcb_msr(u64 val)
68	{	69	{
69	u32 low, high;	70	struct msr m;
70		71
71	low = val & 0xffffffffUL;	72	m.q = val;
72	high = val >> 32;	73	boot_wrmsr(MSR_AMD64_SEV_ES_GHCB, &m);
73
74	asm volatile("wrmsr" : : "c" (MSR_AMD64_SEV_ES_GHCB),
75	"a"(low), "d" (high) : "memory");
76	}	74	}
77		75
78	static enum es_result vc_decode_insn(struct es_em_ctxt *ctxt)	76	static enum es_result vc_decode_insn(struct es_em_ctxt *ctxt)
@@ -119,11 +117,54 @@ static enum es_result vc_read_mem(struct es_em_ctxt *ctxt, · 40 unmodified lines
119	/* Include code for early handlers */	117	/* Include code for early handlers */
120	#include "../../kernel/sev-shared.c"	118	#include "../../kernel/sev-shared.c"
121		119
122	static bool early_setup_sev_es(void)	120	static inline bool sev_snp_enabled(void)
		121	{
		122	return sev_status & MSR_AMD64_SEV_SNP_ENABLED;
		123	}
		124
		125	static void __page_state_change(unsigned long paddr, enum psc_op op)
		126	{
		127	u64 val;
		128
		129	if (!sev_snp_enabled())
		130	return;
		131
		132	/*
		133	* If private -> shared then invalidate the page before requesting the
		134	* state change in the RMP table.
		135	*/
		136	if (op == SNP_PAGE_STATE_SHARED && pvalidate(paddr, RMP_PG_SIZE_4K, 0))
		137	sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PVALIDATE);
		138
		139	/* Issue VMGEXIT to change the page state in RMP table. */
		140	sev_es_wr_ghcb_msr(GHCB_MSR_PSC_REQ_GFN(paddr >> PAGE_SHIFT, op));
		141	VMGEXIT();
		142
		143	/* Read the response of the VMGEXIT. */
		144	val = sev_es_rd_ghcb_msr();
		145	if ((GHCB_RESP_CODE(val) != GHCB_MSR_PSC_RESP) \|\| GHCB_MSR_PSC_RESP_VAL(val))
		146	sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PSC);
		147
		148	/*
		149	* Now that page state is changed in the RMP table, validate it so that it is
		150	* consistent with the RMP entry.
		151	*/
		152	if (op == SNP_PAGE_STATE_PRIVATE && pvalidate(paddr, RMP_PG_SIZE_4K, 1))
		153	sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PVALIDATE);
		154	}
		155
		156	void snp_set_page_private(unsigned long paddr)
		157	{
		158	__page_state_change(paddr, SNP_PAGE_STATE_PRIVATE);
		159	}
		160
		161	void snp_set_page_shared(unsigned long paddr)
123	{	162	{
124	if (!sev_es_negotiate_protocol())	163	__page_state_change(paddr, SNP_PAGE_STATE_SHARED);
125	sev_es_terminate(GHCB_SEV_ES_PROT_UNSUPPORTED);	164	}
126		165
		166	static bool early_setup_ghcb(void)
		167	{
127	if (set_page_decrypted((unsigned long)&boot_ghcb_page))	168	if (set_page_decrypted((unsigned long)&boot_ghcb_page))
128	return false;	169	return false;
129		170
@@ -135,6 +176,10 @@ static bool early_setup_sev_es(void) · 5 unmodified lines
135	/* Initialize lookup tables for the instruction decoder */	176	/* Initialize lookup tables for the instruction decoder */
136	inat_init_tables();	177	inat_init_tables();
137		178
		179	/* SNP guest requires the GHCB GPA must be registered */
		180	if (sev_snp_enabled())
		181	snp_register_ghcb_early(__pa(&boot_ghcb_page));
		182
138	return true;	183	return true;
139	}	184	}
140		185
@@ -174,8 +219,8 @@ void do_boot_stage2_vc(struct pt_regs *regs, unsigned long exit_code) · 33 unmodified lines
174	struct es_em_ctxt ctxt;	219	struct es_em_ctxt ctxt;
175	enum es_result result;	220	enum es_result result;
176		221
177	if (!boot_ghcb && !early_setup_sev_es())	222	if (!boot_ghcb && !early_setup_ghcb())
178	sev_es_terminate(GHCB_SEV_ES_GEN_REQ);	223	sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_GEN_REQ);
179		224
180	vc_ghcb_invalidate(boot_ghcb);	225	vc_ghcb_invalidate(boot_ghcb);
181	result = vc_init_em_ctxt(&ctxt, regs, exit_code);	226	result = vc_init_em_ctxt(&ctxt, regs, exit_code);
@@ -202,5 +247,191 @@ finish: · 20 unmodified lines
202	if (result == ES_OK)	247	if (result == ES_OK)
203	vc_finish_insn(&ctxt);	248	vc_finish_insn(&ctxt);
204	else if (result != ES_RETRY)	249	else if (result != ES_RETRY)
205	sev_es_terminate(GHCB_SEV_ES_GEN_REQ);	250	sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_GEN_REQ);
		251	}
		252
		253	static void enforce_vmpl0(void)
		254	{
		255	u64 attrs;
		256	int err;
		257
		258	/*
		259	* RMPADJUST modifies RMP permissions of a lesser-privileged (numerically
		260	* higher) privilege level. Here, clear the VMPL1 permission mask of the
		261	* GHCB page. If the guest is not running at VMPL0, this will fail.
		262	*
		263	* If the guest is running at VMPL0, it will succeed. Even if that operation
		264	* modifies permission bits, it is still ok to do so currently because Linux
		265	* SNP guests are supported only on VMPL0 so VMPL1 or higher permission masks
		266	* changing is a don't-care.
		267	*/
		268	attrs = 1;
		269	if (rmpadjust((unsigned long)&boot_ghcb_page, RMP_PG_SIZE_4K, attrs))
		270	sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_NOT_VMPL0);
		271	}
		272
		273	void sev_enable(struct boot_params *bp)
		274	{
		275	unsigned int eax, ebx, ecx, edx;
		276	struct msr m;
		277	bool snp;
		278
		279	/*
		280	* Setup/preliminary detection of SNP. This will be sanity-checked
		281	* against CPUID/MSR values later.
		282	*/
		283	snp = snp_init(bp);
		284
		285	/* Check for the SME/SEV support leaf */
		286	eax = 0x80000000;
		287	ecx = 0;
		288	native_cpuid(&eax, &ebx, &ecx, &edx);
		289	if (eax < 0x8000001f)
		290	return;
		291
		292	/*
		293	* Check for the SME/SEV feature:
		294	* CPUID Fn8000_001F[EAX]
		295	* - Bit 0 - Secure Memory Encryption support
		296	* - Bit 1 - Secure Encrypted Virtualization support
		297	* CPUID Fn8000_001F[EBX]
		298	* - Bits 5:0 - Pagetable bit position used to indicate encryption
		299	*/
		300	eax = 0x8000001f;
		301	ecx = 0;
		302	native_cpuid(&eax, &ebx, &ecx, &edx);
		303	/* Check whether SEV is supported */
		304	if (!(eax & BIT(1))) {
		305	if (snp)
		306	error("SEV-SNP support indicated by CC blob, but not CPUID.");
		307	return;
		308	}
		309
		310	/* Set the SME mask if this is an SEV guest. */
		311	boot_rdmsr(MSR_AMD64_SEV, &m);
		312	sev_status = m.q;
		313	if (!(sev_status & MSR_AMD64_SEV_ENABLED))
		314	return;
		315
		316	/* Negotiate the GHCB protocol version. */
		317	if (sev_status & MSR_AMD64_SEV_ES_ENABLED) {
		318	if (!sev_es_negotiate_protocol())
		319	sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_PROT_UNSUPPORTED);
		320	}
		321
		322	/*
		323	* SNP is supported in v2 of the GHCB spec which mandates support for HV
		324	* features.
		325	*/
		326	if (sev_status & MSR_AMD64_SEV_SNP_ENABLED) {
		327	if (!(get_hv_features() & GHCB_HV_FT_SNP))
		328	sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SNP_UNSUPPORTED);
		329
		330	enforce_vmpl0();
		331	}
		332
		333	if (snp && !(sev_status & MSR_AMD64_SEV_SNP_ENABLED))
		334	error("SEV-SNP supported indicated by CC blob, but not SEV status MSR.");
		335
		336	sme_me_mask = BIT_ULL(ebx & 0x3f);
		337	}
		338
		339	/* Search for Confidential Computing blob in the EFI config table. */
		340	static struct cc_blob_sev_info find_cc_blob_efi(struct boot_params bp)
		341	{
		342	unsigned long cfg_table_pa;
		343	unsigned int cfg_table_len;
		344	int ret;
		345
		346	ret = efi_get_conf_table(bp, &cfg_table_pa, &cfg_table_len);
		347	if (ret)
		348	return NULL;
		349
		350	return (struct cc_blob_sev_info *)efi_find_vendor_table(bp, cfg_table_pa,
		351	cfg_table_len,
		352	EFI_CC_BLOB_GUID);
		353	}
		354
		355	/*
		356	* Initial set up of SNP relies on information provided by the
		357	* Confidential Computing blob, which can be passed to the boot kernel
		358	* by firmware/bootloader in the following ways:
		359	*
		360	* - via an entry in the EFI config table
		361	* - via a setup_data structure, as defined by the Linux Boot Protocol
		362	*
		363	* Scan for the blob in that order.
		364	*/
		365	static struct cc_blob_sev_info find_cc_blob(struct boot_params bp)
		366	{
		367	struct cc_blob_sev_info *cc_info;
		368
		369	cc_info = find_cc_blob_efi(bp);
		370	if (cc_info)
		371	goto found_cc_info;
		372
		373	cc_info = find_cc_blob_setup_data(bp);
		374	if (!cc_info)
		375	return NULL;
		376
		377	found_cc_info:
		378	if (cc_info->magic != CC_BLOB_SEV_HDR_MAGIC)
		379	sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SNP_UNSUPPORTED);
		380
		381	return cc_info;
		382	}
		383
		384	/*
		385	* Indicate SNP based on presence of SNP-specific CC blob. Subsequent checks
		386	* will verify the SNP CPUID/MSR bits.
		387	*/
		388	bool snp_init(struct boot_params *bp)
		389	{
		390	struct cc_blob_sev_info *cc_info;
		391
		392	if (!bp)
		393	return false;
		394
		395	cc_info = find_cc_blob(bp);
		396	if (!cc_info)
		397	return false;
		398
		399	/*
		400	* If a SNP-specific Confidential Computing blob is present, then
		401	* firmware/bootloader have indicated SNP support. Verifying this
		402	* involves CPUID checks which will be more reliable if the SNP
		403	* CPUID table is used. See comments over snp_setup_cpuid_table() for
		404	* more details.
		405	*/
		406	setup_cpuid_table(cc_info);
		407
		408	/*
		409	* Pass run-time kernel a pointer to CC info via boot_params so EFI
		410	* config table doesn't need to be searched again during early startup
		411	* phase.
		412	*/
		413	bp->cc_blob_address = (u32)(unsigned long)cc_info;
		414
		415	return true;
		416	}
		417
		418	void sev_prep_identity_maps(unsigned long top_level_pgt)
		419	{
		420	/*
		421	* The Confidential Computing blob is used very early in uncompressed
		422	* kernel to find the in-memory CPUID table to handle CPUID
		423	* instructions. Make sure an identity-mapping exists so it can be
		424	* accessed after switchover.
		425	*/
		426	if (sev_snp_enabled()) {
		427	unsigned long cc_info_pa = boot_params->cc_blob_address;
		428	struct cc_blob_sev_info *cc_info;
		429
		430	kernel_add_identity_map(cc_info_pa, cc_info_pa + sizeof(*cc_info));
		431
		432	cc_info = (struct cc_blob_sev_info *)cc_info_pa;
		433	kernel_add_identity_map(cc_info->cpuid_phys, cc_info->cpuid_phys + cc_info->cpuid_len);
		434	}
		435
		436	sev_verify_cbit(top_level_pgt);
206	}	437	}

arch/x86/boot/cpucheck.c+15−15

 #include <asm/required-features.h>
 #include <asm/msr-index.h>
 #include "string.h"
+#include "msr.h"
 static u32 err_flags[NCAPINTS];
 		/* If this is an AMD and we're only missing SSE+SSE2, try to
 		   turn them on */
-		u32 ecx = MSR_K7_HWCR;
+		struct msr m;
-		u32 eax, edx;
-		asm("rdmsr" : "=a" (eax), "=d" (edx) : "c" (ecx));
+		boot_rdmsr(MSR_K7_HWCR, &m);
-		eax &= ~(1 << 15);
+		m.l &= ~(1 << 15);
-		asm("wrmsr" : : "a" (eax), "d" (edx), "c" (ecx));
+		boot_wrmsr(MSR_K7_HWCR, &m);
 		get_cpuflags();	/* Make sure it really did something */
 		err = check_cpuflags();
 		/* If this is a VIA C3, we might have to enable CX8
 		   explicitly */
-		u32 ecx = MSR_VIA_FCR;
+		struct msr m;
-		u32 eax, edx;
-		asm("rdmsr" : "=a" (eax), "=d" (edx) : "c" (ecx));
+		boot_rdmsr(MSR_VIA_FCR, &m);
-		eax |= (1<<1)|(1<<7);
+		m.l |= (1 << 1) | (1 << 7);
-		asm("wrmsr" : : "a" (eax), "d" (edx), "c" (ecx));
+		boot_wrmsr(MSR_VIA_FCR, &m);
 		set_bit(X86_FEATURE_CX8, cpu.flags);
 		err = check_cpuflags();
 	} else if (err == 0x01 && is_transmeta()) {
 		/* Transmeta might have masked feature bits in word 0 */
-		u32 ecx = 0x80860004;
+		struct msr m, m_tmp;
-		u32 eax, edx;
 		u32 level = 1;
-		asm("rdmsr" : "=a" (eax), "=d" (edx) : "c" (ecx));
+		boot_rdmsr(0x80860004, &m);
-		asm("wrmsr" : : "a" (~0), "d" (edx), "c" (ecx));
+		m_tmp = m;
+		m_tmp.l = ~0;
+		boot_wrmsr(0x80860004, &m_tmp);
 		asm("cpuid"
 		    : "+a" (level), "=d" (cpu.flags[0])
 		    : : "ecx", "ebx");
-		asm("wrmsr" : : "a" (eax), "d" (edx), "c" (ecx));
+		boot_wrmsr(0x80860004, &m);
 		err = check_cpuflags();
 	} else if (err == 0x01 &&

arch/x86/boot/msr.h+26−0

@@ -0,0 +1,26 @@
	1	/* SPDX-License-Identifier: GPL-2.0-only */
	2	/*
	3	* Helpers/definitions related to MSR access.
	4	*/
	5
	6	#ifndef BOOT_MSR_H
	7	#define BOOT_MSR_H
	8
	9	#include <asm/shared/msr.h>
	10
	11	/*
	12	* The kernel proper already defines rdmsr()/wrmsr(), but they are not for the
	13	* boot kernel since they rely on tracepoint/exception handling infrastructure
	14	* that's not available here.
	15	*/
	16	static inline void boot_rdmsr(unsigned int reg, struct msr *m)
	17	{
	18	asm volatile("rdmsr" : "=a" (m->l), "=d" (m->h) : "c" (reg));
	19	}
	20
	21	static inline void boot_wrmsr(unsigned int reg, const struct msr *m)
	22	{
	23	asm volatile("wrmsr" : : "c" (reg), "a"(m->l), "d" (m->h) : "memory");
	24	}
	25
	26	#endif /* BOOT_MSR_H */

arch/x86/coco/core.c+3−0

@@ -57,6 +57,9 @@ static bool amd_cc_platform_has(enum cc_attr attr) · 56 unmodified lines
57	return (sev_status & MSR_AMD64_SEV_ENABLED) &&	57	return (sev_status & MSR_AMD64_SEV_ENABLED) &&
58	!(sev_status & MSR_AMD64_SEV_ES_ENABLED);	58	!(sev_status & MSR_AMD64_SEV_ES_ENABLED);
59		59
		60	case CC_ATTR_GUEST_SEV_SNP:
		61	return sev_status & MSR_AMD64_SEV_SNP_ENABLED;
		62
60	default:	63	default:
61	return false;	64	return false;
62	}	65	}

arch/x86/entry/entry_64.S+6−0

 	popq	%rdi
 	popq	%rsp
+SYM_INNER_LABEL(entry_SYSRETQ_unsafe_stack, SYM_L_GLOBAL)
+	ANNOTATE_NOENDBR
 	swapgs
 	sysretq
+SYM_INNER_LABEL(entry_SYSRETQ_end, SYM_L_GLOBAL)
+	ANNOTATE_NOENDBR
+	int3
 SYM_CODE_END(entry_SYSCALL_64)
 /*
 	call	vc_switch_off_ist
 	movq	%rax, %rsp		/* Switch to new stack */
+	ENCODE_FRAME_POINTER
 	UNWIND_HINT_REGS
 	/* Update pt_regs */

arch/x86/entry/entry_64_compat.S+5−0

 	 * code.  We zero R8-R10 to avoid info leaks.
          */
 	movq	RSP-ORIG_RAX(%rsp), %rsp
+SYM_INNER_LABEL(entry_SYSRETL_compat_unsafe_stack, SYM_L_GLOBAL)
+	ANNOTATE_NOENDBR
 	/*
 	 * The original userspace %rsp (RSP-ORIG_RAX(%rsp)) is stored
 	xorl	%r10d, %r10d
 	swapgs
 	sysretl
+SYM_INNER_LABEL(entry_SYSRETL_compat_end, SYM_L_GLOBAL)
+	ANNOTATE_NOENDBR
+	int3
 SYM_CODE_END(entry_SYSCALL_compat)
 /*

arch/x86/include/asm/bootparam_utils.h+1−0

@@ -74,6 +74,7 @@ static void sanitize_boot_params(struct boot_params *boot_params) · 73 unmodified lines
74	BOOT_PARAM_PRESERVE(hdr),	74	BOOT_PARAM_PRESERVE(hdr),
75	BOOT_PARAM_PRESERVE(e820_table),	75	BOOT_PARAM_PRESERVE(e820_table),
76	BOOT_PARAM_PRESERVE(eddbuf),	76	BOOT_PARAM_PRESERVE(eddbuf),
		77	BOOT_PARAM_PRESERVE(cc_blob_address),
77	};	78	};
78		79
79	memset(&scratch, 0, sizeof(scratch));	80	memset(&scratch, 0, sizeof(scratch));

arch/x86/include/asm/cpuid.h+34−0

@@ -0,0 +1,34 @@
	1	/* SPDX-License-Identifier: GPL-2.0 */
	2	/*
	3	* CPUID-related helpers/definitions
	4	*
	5	* Derived from arch/x86/kvm/cpuid.c
	6	*/
	7
	8	#ifndef _ASM_X86_CPUID_H
	9	#define _ASM_X86_CPUID_H
	10
	11	static __always_inline bool cpuid_function_is_indexed(u32 function)
	12	{
	13	switch (function) {
	14	case 4:
	15	case 7:
	16	case 0xb:
	17	case 0xd:
	18	case 0xf:
	19	case 0x10:
	20	case 0x12:
	21	case 0x14:
	22	case 0x17:
	23	case 0x18:
	24	case 0x1d:
	25	case 0x1e:
	26	case 0x1f:
	27	case 0x8000001d:
	28	return true;
	29	}
	30
	31	return false;
	32	}
	33
	34	#endif /* _ASM_X86_CPUID_H */

arch/x86/include/asm/msr-index.h+2−0

 #define MSR_AMD64_SEV			0xc0010131
 #define MSR_AMD64_SEV_ENABLED_BIT	0
 #define MSR_AMD64_SEV_ES_ENABLED_BIT	1
+#define MSR_AMD64_SEV_SNP_ENABLED_BIT	2
 #define MSR_AMD64_SEV_ENABLED		BIT_ULL(MSR_AMD64_SEV_ENABLED_BIT)
 #define MSR_AMD64_SEV_ES_ENABLED	BIT_ULL(MSR_AMD64_SEV_ES_ENABLED_BIT)
+#define MSR_AMD64_SEV_SNP_ENABLED	BIT_ULL(MSR_AMD64_SEV_SNP_ENABLED_BIT)
 #define MSR_AMD64_VIRT_SPEC_CTRL	0xc001011f

arch/x86/include/asm/msr.h+1−10

@@ -10,16 +10,7 @@ · 9 unmodified lines
10	#include <asm/errno.h>	10	#include <asm/errno.h>
11	#include <asm/cpumask.h>	11	#include <asm/cpumask.h>
12	#include <uapi/asm/msr.h>	12	#include <uapi/asm/msr.h>
13		13	#include <asm/shared/msr.h>
14	struct msr {
15	union {
16	struct {
17	u32 l;
18	u32 h;
19	};
20	u64 q;
21	};
22	};
23		14
24	struct msr_info {	15	struct msr_info {
25	u32 msr_no;	16	u32 msr_no;

arch/x86/include/asm/proto.h+4−0

 #ifdef CONFIG_X86_64
 void entry_SYSCALL_64(void);
 void entry_SYSCALL_64_safe_stack(void);
+void entry_SYSRETQ_unsafe_stack(void);
+void entry_SYSRETQ_end(void);
 long do_arch_prctl_64(struct task_struct *task, int option, unsigned long arg2);
 #endif
 void __end_entry_SYSENTER_compat(void);
 void entry_SYSCALL_compat(void);
 void entry_SYSCALL_compat_safe_stack(void);
+void entry_SYSRETL_compat_unsafe_stack(void);
+void entry_SYSRETL_compat_end(void);
 void entry_INT80_compat(void);
 #ifdef CONFIG_XEN_PV
 void xen_entry_INT80_compat(void);

arch/x86/include/asm/ptrace.h+4−0

 	bool ret = (regs->ip >= (unsigned long)entry_SYSCALL_64 &&
 		    regs->ip <  (unsigned long)entry_SYSCALL_64_safe_stack);
+	ret = ret || (regs->ip >= (unsigned long)entry_SYSRETQ_unsafe_stack &&
+		      regs->ip <  (unsigned long)entry_SYSRETQ_end);
 #ifdef CONFIG_IA32_EMULATION
 	ret = ret || (regs->ip >= (unsigned long)entry_SYSCALL_compat &&
 		      regs->ip <  (unsigned long)entry_SYSCALL_compat_safe_stack);
+	ret = ret || (regs->ip >= (unsigned long)entry_SYSRETL_compat_unsafe_stack &&
+		      regs->ip <  (unsigned long)entry_SYSRETL_compat_end);
 #endif
 	return ret;

arch/x86/include/asm/setup.h+0−1

@@ -50,7 +50,6 @@ extern unsigned long saved_video_mode; · 49 unmodified lines
50	extern void reserve_standard_io_resources(void);	50	extern void reserve_standard_io_resources(void);
51	extern void i386_reserve_resources(void);	51	extern void i386_reserve_resources(void);
52	extern unsigned long __startup_64(unsigned long physaddr, struct boot_params *bp);	52	extern unsigned long __startup_64(unsigned long physaddr, struct boot_params *bp);
53	extern unsigned long __startup_secondary_64(void);
54	extern void startup_64_setup_env(unsigned long physbase);	53	extern void startup_64_setup_env(unsigned long physbase);
55	extern void early_setup_idt(void);	54	extern void early_setup_idt(void);
56	extern void __init do_early_exception(struct pt_regs *regs, int trapnr);	55	extern void __init do_early_exception(struct pt_regs *regs, int trapnr);

arch/x86/include/asm/sev-common.h+82−0

@@ -57,9 +57,79 @@ · 56 unmodified lines
57	#define GHCB_MSR_AP_RESET_HOLD_REQ 0x006	57	#define GHCB_MSR_AP_RESET_HOLD_REQ 0x006
58	#define GHCB_MSR_AP_RESET_HOLD_RESP 0x007	58	#define GHCB_MSR_AP_RESET_HOLD_RESP 0x007
59		59
		60	/* GHCB GPA Register */
		61	#define GHCB_MSR_REG_GPA_REQ 0x012
		62	#define GHCB_MSR_REG_GPA_REQ_VAL(v) \
		63	/* GHCBData[63:12] */ \
		64	(((u64)((v) & GENMASK_ULL(51, 0)) << 12) \| \
		65	/* GHCBData[11:0] */ \
		66	GHCB_MSR_REG_GPA_REQ)
		67
		68	#define GHCB_MSR_REG_GPA_RESP 0x013
		69	#define GHCB_MSR_REG_GPA_RESP_VAL(v) \
		70	/* GHCBData[63:12] */ \
		71	(((u64)(v) & GENMASK_ULL(63, 12)) >> 12)
		72
		73	/*
		74	* SNP Page State Change Operation
		75	*
		76	* GHCBData[55:52] - Page operation:
		77	* 0x0001 Page assignment, Private
		78	* 0x0002 Page assignment, Shared
		79	*/
		80	enum psc_op {
		81	SNP_PAGE_STATE_PRIVATE = 1,
		82	SNP_PAGE_STATE_SHARED,
		83	};
		84
		85	#define GHCB_MSR_PSC_REQ 0x014
		86	#define GHCB_MSR_PSC_REQ_GFN(gfn, op) \
		87	/* GHCBData[55:52] */ \
		88	(((u64)((op) & 0xf) << 52) \| \
		89	/* GHCBData[51:12] */ \
		90	((u64)((gfn) & GENMASK_ULL(39, 0)) << 12) \| \
		91	/* GHCBData[11:0] */ \
		92	GHCB_MSR_PSC_REQ)
		93
		94	#define GHCB_MSR_PSC_RESP 0x015
		95	#define GHCB_MSR_PSC_RESP_VAL(val) \
		96	/* GHCBData[63:32] */ \
		97	(((u64)(val) & GENMASK_ULL(63, 32)) >> 32)
		98
60	/* GHCB Hypervisor Feature Request/Response */	99	/* GHCB Hypervisor Feature Request/Response */
61	#define GHCB_MSR_HV_FT_REQ 0x080	100	#define GHCB_MSR_HV_FT_REQ 0x080
62	#define GHCB_MSR_HV_FT_RESP 0x081	101	#define GHCB_MSR_HV_FT_RESP 0x081
		102	#define GHCB_MSR_HV_FT_RESP_VAL(v) \
		103	/* GHCBData[63:12] */ \
		104	(((u64)(v) & GENMASK_ULL(63, 12)) >> 12)
		105
		106	#define GHCB_HV_FT_SNP BIT_ULL(0)
		107	#define GHCB_HV_FT_SNP_AP_CREATION BIT_ULL(1)
		108
		109	/* SNP Page State Change NAE event */
		110	#define VMGEXIT_PSC_MAX_ENTRY 253
		111
		112	struct psc_hdr {
		113	u16 cur_entry;
		114	u16 end_entry;
		115	u32 reserved;
		116	} __packed;
		117
		118	struct psc_entry {
		119	u64 cur_page : 12,
		120	gfn : 40,
		121	operation : 4,
		122	pagesize : 1,
		123	reserved : 7;
		124	} __packed;
		125
		126	struct snp_psc_desc {
		127	struct psc_hdr hdr;
		128	struct psc_entry entries[VMGEXIT_PSC_MAX_ENTRY];
		129	} __packed;
		130
		131	/* Guest message request error code */
		132	#define SNP_GUEST_REQ_INVALID_LEN BIT_ULL(32)
63		133
64	#define GHCB_MSR_TERM_REQ 0x100	134	#define GHCB_MSR_TERM_REQ 0x100
65	#define GHCB_MSR_TERM_REASON_SET_POS 12	135	#define GHCB_MSR_TERM_REASON_SET_POS 12
@@ -73,8 +143,20 @@ · 7 unmodified lines
73	/* GHCBData[23:16] */ \	143	/* GHCBData[23:16] */ \
74	((((u64)reason_val) & 0xff) << 16))	144	((((u64)reason_val) & 0xff) << 16))
75		145
		146	/* Error codes from reason set 0 */
		147	#define SEV_TERM_SET_GEN 0
76	#define GHCB_SEV_ES_GEN_REQ 0	148	#define GHCB_SEV_ES_GEN_REQ 0
77	#define GHCB_SEV_ES_PROT_UNSUPPORTED 1	149	#define GHCB_SEV_ES_PROT_UNSUPPORTED 1
		150	#define GHCB_SNP_UNSUPPORTED 2
		151
		152	/* Linux-specific reason codes (used with reason set 1) */
		153	#define SEV_TERM_SET_LINUX 1
		154	#define GHCB_TERM_REGISTER 0 /* GHCB GPA registration failure */
		155	#define GHCB_TERM_PSC 1 /* Page State Change failure */
		156	#define GHCB_TERM_PVALIDATE 2 /* Pvalidate failure */
		157	#define GHCB_TERM_NOT_VMPL0 3 /* SNP guest is not running at VMPL-0 */
		158	#define GHCB_TERM_CPUID 4 /* CPUID-validation failure */
		159	#define GHCB_TERM_CPUID_HV 5 /* CPUID failure during hypervisor fallback */
78		160
79	#define GHCB_RESP_CODE(v) ((v) & GHCB_MSR_INFO_MASK)	161	#define GHCB_RESP_CODE(v) ((v) & GHCB_MSR_INFO_MASK)
80		162

arch/x86/include/asm/sev.h+135−2

@@ -11,9 +11,10 @@ · 10 unmodified lines
11	#include <linux/types.h>	11	#include <linux/types.h>
12	#include <asm/insn.h>	12	#include <asm/insn.h>
13	#include <asm/sev-common.h>	13	#include <asm/sev-common.h>
		14	#include <asm/bootparam.h>
14		15
15	#define GHCB_PROTO_OUR 0x0001UL	16	#define GHCB_PROTOCOL_MIN 1ULL
16	#define GHCB_PROTOCOL_MAX 1ULL	17	#define GHCB_PROTOCOL_MAX 2ULL
17	#define GHCB_DEFAULT_USAGE 0ULL	18	#define GHCB_DEFAULT_USAGE 0ULL
18		19
19	#define VMGEXIT() { asm volatile("rep; vmmcall\n\r"); }	20	#define VMGEXIT() { asm volatile("rep; vmmcall\n\r"); }
@@ -42,6 +43,24 @@ struct es_em_ctxt { · 22 unmodified lines
42	struct es_fault_info fi;	43	struct es_fault_info fi;
43	};	44	};
44		45
		46	/*
		47	* AMD SEV Confidential computing blob structure. The structure is
		48	* defined in OVMF UEFI firmware header:
		49	* https://github.com/tianocore/edk2/blob/master/OvmfPkg/Include/Guid/ConfidentialComputingSevSnpBlob.h
		50	*/
		51	#define CC_BLOB_SEV_HDR_MAGIC 0x45444d41
		52	struct cc_blob_sev_info {
		53	u32 magic;
		54	u16 version;
		55	u16 reserved;
		56	u64 secrets_phys;
		57	u32 secrets_len;
		58	u32 rsvd1;
		59	u64 cpuid_phys;
		60	u32 cpuid_len;
		61	u32 rsvd2;
		62	} __packed;
		63
45	void do_vc_no_ghcb(struct pt_regs *regs, unsigned long exit_code);	64	void do_vc_no_ghcb(struct pt_regs *regs, unsigned long exit_code);
46		65
47	static inline u64 lower_bits(u64 val, unsigned int bits)	66	static inline u64 lower_bits(u64 val, unsigned int bits)
@@ -60,6 +79,61 @@ extern void vc_no_ghcb(void); · 12 unmodified lines
60	extern void vc_boot_ghcb(void);	79	extern void vc_boot_ghcb(void);
61	extern bool handle_vc_boot_ghcb(struct pt_regs *regs);	80	extern bool handle_vc_boot_ghcb(struct pt_regs *regs);
62		81
		82	/* Software defined (when rFlags.CF = 1) */
		83	#define PVALIDATE_FAIL_NOUPDATE 255
		84
		85	/* RMP page size */
		86	#define RMP_PG_SIZE_4K 0
		87
		88	#define RMPADJUST_VMSA_PAGE_BIT BIT(16)
		89
		90	/* SNP Guest message request */
		91	struct snp_req_data {
		92	unsigned long req_gpa;
		93	unsigned long resp_gpa;
		94	unsigned long data_gpa;
		95	unsigned int data_npages;
		96	};
		97
		98	struct sev_guest_platform_data {
		99	u64 secrets_gpa;
		100	};
		101
		102	/*
		103	* The secrets page contains 96-bytes of reserved field that can be used by
		104	* the guest OS. The guest OS uses the area to save the message sequence
		105	* number for each VMPCK.
		106	*
		107	* See the GHCB spec section Secret page layout for the format for this area.
		108	*/
		109	struct secrets_os_area {
		110	u32 msg_seqno_0;
		111	u32 msg_seqno_1;
		112	u32 msg_seqno_2;
		113	u32 msg_seqno_3;
		114	u64 ap_jump_table_pa;
		115	u8 rsvd[40];
		116	u8 guest_usage[32];
		117	} __packed;
		118
		119	#define VMPCK_KEY_LEN 32
		120
		121	/* See the SNP spec version 0.9 for secrets page format */
		122	struct snp_secrets_page_layout {
		123	u32 version;
		124	u32 imien : 1,
		125	rsvd1 : 31;
		126	u32 fms;
		127	u32 rsvd2;
		128	u8 gosvw[16];
		129	u8 vmpck0[VMPCK_KEY_LEN];
		130	u8 vmpck1[VMPCK_KEY_LEN];
		131	u8 vmpck2[VMPCK_KEY_LEN];
		132	u8 vmpck3[VMPCK_KEY_LEN];
		133	struct secrets_os_area os_area;
		134	u8 rsvd3[3840];
		135	} __packed;
		136
63	#ifdef CONFIG_AMD_MEM_ENCRYPT	137	#ifdef CONFIG_AMD_MEM_ENCRYPT
64	extern struct static_key_false sev_es_enable_key;	138	extern struct static_key_false sev_es_enable_key;
65	extern void __sev_es_ist_enter(struct pt_regs *regs);	139	extern void __sev_es_ist_enter(struct pt_regs *regs);
@@ -87,12 +161,71 @@ extern enum es_result sev_es_ghcb_hv_call(struct ghcb *ghcb, · 21 unmodified lines
87	struct es_em_ctxt *ctxt,	161	struct es_em_ctxt *ctxt,
88	u64 exit_code, u64 exit_info_1,	162	u64 exit_code, u64 exit_info_1,
89	u64 exit_info_2);	163	u64 exit_info_2);
		164	static inline int rmpadjust(unsigned long vaddr, bool rmp_psize, unsigned long attrs)
		165	{
		166	int rc;
		167
		168	/* "rmpadjust" mnemonic support in binutils 2.36 and newer */
		169	asm volatile(".byte 0xF3,0x0F,0x01,0xFE\n\t"
		170	: "=a"(rc)
		171	: "a"(vaddr), "c"(rmp_psize), "d"(attrs)
		172	: "memory", "cc");
		173
		174	return rc;
		175	}
		176	static inline int pvalidate(unsigned long vaddr, bool rmp_psize, bool validate)
		177	{
		178	bool no_rmpupdate;
		179	int rc;
		180
		181	/* "pvalidate" mnemonic support in binutils 2.36 and newer */
		182	asm volatile(".byte 0xF2, 0x0F, 0x01, 0xFF\n\t"
		183	CC_SET(c)
		184	: CC_OUT(c) (no_rmpupdate), "=a"(rc)
		185	: "a"(vaddr), "c"(rmp_psize), "d"(validate)
		186	: "memory", "cc");
		187
		188	if (no_rmpupdate)
		189	return PVALIDATE_FAIL_NOUPDATE;
		190
		191	return rc;
		192	}
		193	void setup_ghcb(void);
		194	void __init early_snp_set_memory_private(unsigned long vaddr, unsigned long paddr,
		195	unsigned int npages);
		196	void __init early_snp_set_memory_shared(unsigned long vaddr, unsigned long paddr,
		197	unsigned int npages);
		198	void __init snp_prep_memory(unsigned long paddr, unsigned int sz, enum psc_op op);
		199	void snp_set_memory_shared(unsigned long vaddr, unsigned int npages);
		200	void snp_set_memory_private(unsigned long vaddr, unsigned int npages);
		201	void snp_set_wakeup_secondary_cpu(void);
		202	bool snp_init(struct boot_params *bp);
		203	void snp_abort(void);
		204	int snp_issue_guest_request(u64 exit_code, struct snp_req_data input, unsigned long fw_err);
90	#else	205	#else
91	static inline void sev_es_ist_enter(struct pt_regs *regs) { }	206	static inline void sev_es_ist_enter(struct pt_regs *regs) { }
92	static inline void sev_es_ist_exit(void) { }	207	static inline void sev_es_ist_exit(void) { }
93	static inline int sev_es_setup_ap_jump_table(struct real_mode_header *rmh) { return 0; }	208	static inline int sev_es_setup_ap_jump_table(struct real_mode_header *rmh) { return 0; }
94	static inline void sev_es_nmi_complete(void) { }	209	static inline void sev_es_nmi_complete(void) { }
95	static inline int sev_es_efi_map_ghcbs(pgd_t *pgd) { return 0; }	210	static inline int sev_es_efi_map_ghcbs(pgd_t *pgd) { return 0; }
		211	static inline int pvalidate(unsigned long vaddr, bool rmp_psize, bool validate) { return 0; }
		212	static inline int rmpadjust(unsigned long vaddr, bool rmp_psize, unsigned long attrs) { return 0; }
		213	static inline void setup_ghcb(void) { }
		214	static inline void __init
		215	early_snp_set_memory_private(unsigned long vaddr, unsigned long paddr, unsigned int npages) { }
		216	static inline void __init
		217	early_snp_set_memory_shared(unsigned long vaddr, unsigned long paddr, unsigned int npages) { }
		218	static inline void __init snp_prep_memory(unsigned long paddr, unsigned int sz, enum psc_op op) { }
		219	static inline void snp_set_memory_shared(unsigned long vaddr, unsigned int npages) { }
		220	static inline void snp_set_memory_private(unsigned long vaddr, unsigned int npages) { }
		221	static inline void snp_set_wakeup_secondary_cpu(void) { }
		222	static inline bool snp_init(struct boot_params *bp) { return false; }
		223	static inline void snp_abort(void) { }
		224	static inline int snp_issue_guest_request(u64 exit_code, struct snp_req_data *input,
		225	unsigned long *fw_err)
		226	{
		227	return -ENOTTY;
		228	}
96	#endif	229	#endif
97		230
98	#endif	231	#endif

arch/x86/include/asm/shared/msr.h+15−0

@@ -0,0 +1,15 @@
	1	/* SPDX-License-Identifier: GPL-2.0 */
	2	#ifndef _ASM_X86_SHARED_MSR_H
	3	#define _ASM_X86_SHARED_MSR_H
	4
	5	struct msr {
	6	union {
	7	struct {
	8	u32 l;
	9	u32 h;
	10	};
	11	u64 q;
	12	};
	13	};
	14
	15	#endif /* _ASM_X86_SHARED_MSR_H */

arch/x86/include/asm/svm.h+148−23

@@ -271,6 +271,7 @@ struct vmcb_seg { · 270 unmodified lines
271	u64 base;	271	u64 base;
272	} __packed;	272	} __packed;
273		273
		274	/* Save area definition for legacy and SEV-MEM guests */
274	struct vmcb_save_area {	275	struct vmcb_save_area {
275	struct vmcb_seg es;	276	struct vmcb_seg es;
276	struct vmcb_seg cs;	277	struct vmcb_seg cs;
@@ -282,12 +283,12 @@ struct vmcb_save_area { · 5 unmodified lines
282	struct vmcb_seg ldtr;	283	struct vmcb_seg ldtr;
283	struct vmcb_seg idtr;	284	struct vmcb_seg idtr;
284	struct vmcb_seg tr;	285	struct vmcb_seg tr;
285	u8 reserved_1[43];	286	u8 reserved_1[42];
		287	u8 vmpl;
286	u8 cpl;	288	u8 cpl;
287	u8 reserved_2[4];	289	u8 reserved_2[4];
288	u64 efer;	290	u64 efer;
289	u8 reserved_3[104];	291	u8 reserved_3[112];
290	u64 xss; /* Valid for SEV-ES only */
291	u64 cr4;	292	u64 cr4;
292	u64 cr3;	293	u64 cr3;
293	u64 cr0;	294	u64 cr0;
@@ -297,7 +298,9 @@ struct vmcb_save_area { · 3 unmodified lines
297	u64 rip;	298	u64 rip;
298	u8 reserved_4[88];	299	u8 reserved_4[88];
299	u64 rsp;	300	u64 rsp;
300	u8 reserved_5[24];	301	u64 s_cet;
		302	u64 ssp;
		303	u64 isst_addr;
301	u64 rax;	304	u64 rax;
302	u64 star;	305	u64 star;
303	u64 lstar;	306	u64 lstar;
@@ -308,29 +311,145 @@ struct vmcb_save_area { · 4 unmodified lines
308	u64 sysenter_esp;	311	u64 sysenter_esp;
309	u64 sysenter_eip;	312	u64 sysenter_eip;
310	u64 cr2;	313	u64 cr2;
311	u8 reserved_6[32];	314	u8 reserved_5[32];
312	u64 g_pat;	315	u64 g_pat;
313	u64 dbgctl;	316	u64 dbgctl;
314	u64 br_from;	317	u64 br_from;
315	u64 br_to;	318	u64 br_to;
316	u64 last_excp_from;	319	u64 last_excp_from;
317	u64 last_excp_to;	320	u64 last_excp_to;
318		321	u8 reserved_6[72];
319	/*
320	* The following part of the save area is valid only for
321	* SEV-ES guests when referenced through the GHCB or for
322	* saving to the host save area.
323	*/
324	u8 reserved_7[72];
325	u32 spec_ctrl; /* Guest version of SPEC_CTRL at 0x2E0 */	322	u32 spec_ctrl; /* Guest version of SPEC_CTRL at 0x2E0 */
326	u8 reserved_7b[4];	323	} __packed;
		324
		325	/* Save area definition for SEV-ES and SEV-SNP guests */
		326	struct sev_es_save_area {
		327	struct vmcb_seg es;
		328	struct vmcb_seg cs;
		329	struct vmcb_seg ss;
		330	struct vmcb_seg ds;
		331	struct vmcb_seg fs;
		332	struct vmcb_seg gs;
		333	struct vmcb_seg gdtr;
		334	struct vmcb_seg ldtr;
		335	struct vmcb_seg idtr;
		336	struct vmcb_seg tr;
		337	u64 vmpl0_ssp;
		338	u64 vmpl1_ssp;
		339	u64 vmpl2_ssp;
		340	u64 vmpl3_ssp;
		341	u64 u_cet;
		342	u8 reserved_1[2];
		343	u8 vmpl;
		344	u8 cpl;
		345	u8 reserved_2[4];
		346	u64 efer;
		347	u8 reserved_3[104];
		348	u64 xss;
		349	u64 cr4;
		350	u64 cr3;
		351	u64 cr0;
		352	u64 dr7;
		353	u64 dr6;
		354	u64 rflags;
		355	u64 rip;
		356	u64 dr0;
		357	u64 dr1;
		358	u64 dr2;
		359	u64 dr3;
		360	u64 dr0_addr_mask;
		361	u64 dr1_addr_mask;
		362	u64 dr2_addr_mask;
		363	u64 dr3_addr_mask;
		364	u8 reserved_4[24];
		365	u64 rsp;
		366	u64 s_cet;
		367	u64 ssp;
		368	u64 isst_addr;
		369	u64 rax;
		370	u64 star;
		371	u64 lstar;
		372	u64 cstar;
		373	u64 sfmask;
		374	u64 kernel_gs_base;
		375	u64 sysenter_cs;
		376	u64 sysenter_esp;
		377	u64 sysenter_eip;
		378	u64 cr2;
		379	u8 reserved_5[32];
		380	u64 g_pat;
		381	u64 dbgctl;
		382	u64 br_from;
		383	u64 br_to;
		384	u64 last_excp_from;
		385	u64 last_excp_to;
		386	u8 reserved_7[80];
327	u32 pkru;	387	u32 pkru;
328	u8 reserved_7a[20];	388	u8 reserved_8[20];
329	u64 reserved_8; /* rax already available at 0x01f8 */	389	u64 reserved_9; /* rax already available at 0x01f8 */
		390	u64 rcx;
		391	u64 rdx;
		392	u64 rbx;
		393	u64 reserved_10; /* rsp already available at 0x01d8 */
		394	u64 rbp;
		395	u64 rsi;
		396	u64 rdi;
		397	u64 r8;
		398	u64 r9;
		399	u64 r10;
		400	u64 r11;
		401	u64 r12;
		402	u64 r13;
		403	u64 r14;
		404	u64 r15;
		405	u8 reserved_11[16];
		406	u64 guest_exit_info_1;
		407	u64 guest_exit_info_2;
		408	u64 guest_exit_int_info;
		409	u64 guest_nrip;
		410	u64 sev_features;
		411	u64 vintr_ctrl;
		412	u64 guest_exit_code;
		413	u64 virtual_tom;
		414	u64 tlb_id;
		415	u64 pcpu_id;
		416	u64 event_inj;
		417	u64 xcr0;
		418	u8 reserved_12[16];
		419
		420	/* Floating point area */
		421	u64 x87_dp;
		422	u32 mxcsr;
		423	u16 x87_ftw;
		424	u16 x87_fsw;
		425	u16 x87_fcw;
		426	u16 x87_fop;
		427	u16 x87_ds;
		428	u16 x87_cs;
		429	u64 x87_rip;
		430	u8 fpreg_x87[80];
		431	u8 fpreg_xmm[256];
		432	u8 fpreg_ymm[256];
		433	} __packed;
		434
		435	struct ghcb_save_area {
		436	u8 reserved_1[203];
		437	u8 cpl;
		438	u8 reserved_2[116];
		439	u64 xss;
		440	u8 reserved_3[24];
		441	u64 dr7;
		442	u8 reserved_4[16];
		443	u64 rip;
		444	u8 reserved_5[88];
		445	u64 rsp;
		446	u8 reserved_6[24];
		447	u64 rax;
		448	u8 reserved_7[264];
330	u64 rcx;	449	u64 rcx;
331	u64 rdx;	450	u64 rdx;
332	u64 rbx;	451	u64 rbx;
333	u64 reserved_9; /* rsp already available at 0x01d8 */	452	u8 reserved_8[8];
334	u64 rbp;	453	u64 rbp;
335	u64 rsi;	454	u64 rsi;
336	u64 rdi;	455	u64 rdi;
@@ -342,22 +461,24 @@ struct vmcb_save_area { · 5 unmodified lines
342	u64 r13;	461	u64 r13;
343	u64 r14;	462	u64 r14;
344	u64 r15;	463	u64 r15;
345	u8 reserved_10[16];	464	u8 reserved_9[16];
346	u64 sw_exit_code;	465	u64 sw_exit_code;
347	u64 sw_exit_info_1;	466	u64 sw_exit_info_1;
348	u64 sw_exit_info_2;	467	u64 sw_exit_info_2;
349	u64 sw_scratch;	468	u64 sw_scratch;
350	u8 reserved_11[56];	469	u8 reserved_10[56];
351	u64 xcr0;	470	u64 xcr0;
352	u8 valid_bitmap[16];	471	u8 valid_bitmap[16];
353	u64 x87_state_gpa;	472	u64 x87_state_gpa;
354	} __packed;	473	} __packed;
355		474
		475	#define GHCB_SHARED_BUF_SIZE 2032
		476
356	struct ghcb {	477	struct ghcb {
357	struct vmcb_save_area save;	478	struct ghcb_save_area save;
358	u8 reserved_save[2048 - sizeof(struct vmcb_save_area)];	479	u8 reserved_save[2048 - sizeof(struct ghcb_save_area)];
359		480
360	u8 shared_buffer[2032];	481	u8 shared_buffer[GHCB_SHARED_BUF_SIZE];
361		482
362	u8 reserved_1[10];	483	u8 reserved_1[10];
363	u16 protocol_version; /* negotiated SEV-ES/GHCB protocol version */	484	u16 protocol_version; /* negotiated SEV-ES/GHCB protocol version */
@@ -365,13 +486,17 @@ struct ghcb { · 1 unmodified line
365	} __packed;	486	} __packed;
366		487
367		488
368	#define EXPECTED_VMCB_SAVE_AREA_SIZE 1032	489	#define EXPECTED_VMCB_SAVE_AREA_SIZE 740
		490	#define EXPECTED_GHCB_SAVE_AREA_SIZE 1032
		491	#define EXPECTED_SEV_ES_SAVE_AREA_SIZE 1648
369	#define EXPECTED_VMCB_CONTROL_AREA_SIZE 1024	492	#define EXPECTED_VMCB_CONTROL_AREA_SIZE 1024
370	#define EXPECTED_GHCB_SIZE PAGE_SIZE	493	#define EXPECTED_GHCB_SIZE PAGE_SIZE
371		494
372	static inline void __unused_size_checks(void)	495	static inline void __unused_size_checks(void)
373	{	496	{
374	BUILD_BUG_ON(sizeof(struct vmcb_save_area) != EXPECTED_VMCB_SAVE_AREA_SIZE);	497	BUILD_BUG_ON(sizeof(struct vmcb_save_area) != EXPECTED_VMCB_SAVE_AREA_SIZE);
		498	BUILD_BUG_ON(sizeof(struct ghcb_save_area) != EXPECTED_GHCB_SAVE_AREA_SIZE);
		499	BUILD_BUG_ON(sizeof(struct sev_es_save_area) != EXPECTED_SEV_ES_SAVE_AREA_SIZE);
375	BUILD_BUG_ON(sizeof(struct vmcb_control_area) != EXPECTED_VMCB_CONTROL_AREA_SIZE);	500	BUILD_BUG_ON(sizeof(struct vmcb_control_area) != EXPECTED_VMCB_CONTROL_AREA_SIZE);
376	BUILD_BUG_ON(sizeof(struct ghcb) != EXPECTED_GHCB_SIZE);	501	BUILD_BUG_ON(sizeof(struct ghcb) != EXPECTED_GHCB_SIZE);
377	}	502	}
@@ -441,7 +566,7 @@ struct vmcb { · 63 unmodified lines
441	/* GHCB Accessor functions */	566	/* GHCB Accessor functions */
442		567
443	#define GHCB_BITMAP_IDX(field) \	568	#define GHCB_BITMAP_IDX(field) \
444	(offsetof(struct vmcb_save_area, field) / sizeof(u64))	569	(offsetof(struct ghcb_save_area, field) / sizeof(u64))
445		570
446	#define DEFINE_GHCB_ACCESSORS(field) \	571	#define DEFINE_GHCB_ACCESSORS(field) \
447	static inline bool ghcb_##field##_is_valid(const struct ghcb *ghcb) \	572	static inline bool ghcb_##field##_is_valid(const struct ghcb *ghcb) \

arch/x86/include/uapi/asm/bootparam.h+3−1

 #define SETUP_EFI			4
 #define SETUP_APPLE_PROPERTIES		5
 #define SETUP_JAILHOUSE			6
+#define SETUP_CC_BLOB			7
 #define SETUP_INDIRECT			(1<<31)
 	__u32 ext_ramdisk_image;			/* 0x0c0 */
 	__u32 ext_ramdisk_size;				/* 0x0c4 */
 	__u32 ext_cmd_line_ptr;				/* 0x0c8 */
-	__u8  _pad4[116];				/* 0x0cc */
+	__u8  _pad4[112];				/* 0x0cc */
+	__u32 cc_blob_address;				/* 0x13c */
 	struct edid_info edid_info;			/* 0x140 */
 	struct efi_info efi_info;			/* 0x1c0 */
 	__u32 alt_mem_k;				/* 0x1e0 */

arch/x86/include/uapi/asm/svm.h+13−0

@@ -108,6 +108,14 @@ · 107 unmodified lines
108	#define SVM_VMGEXIT_AP_JUMP_TABLE 0x80000005	108	#define SVM_VMGEXIT_AP_JUMP_TABLE 0x80000005
109	#define SVM_VMGEXIT_SET_AP_JUMP_TABLE 0	109	#define SVM_VMGEXIT_SET_AP_JUMP_TABLE 0
110	#define SVM_VMGEXIT_GET_AP_JUMP_TABLE 1	110	#define SVM_VMGEXIT_GET_AP_JUMP_TABLE 1
		111	#define SVM_VMGEXIT_PSC 0x80000010
		112	#define SVM_VMGEXIT_GUEST_REQUEST 0x80000011
		113	#define SVM_VMGEXIT_EXT_GUEST_REQUEST 0x80000012
		114	#define SVM_VMGEXIT_AP_CREATION 0x80000013
		115	#define SVM_VMGEXIT_AP_CREATE_ON_INIT 0
		116	#define SVM_VMGEXIT_AP_CREATE 1
		117	#define SVM_VMGEXIT_AP_DESTROY 2
		118	#define SVM_VMGEXIT_HV_FEATURES 0x8000fffd
111	#define SVM_VMGEXIT_UNSUPPORTED_EVENT 0x8000ffff	119	#define SVM_VMGEXIT_UNSUPPORTED_EVENT 0x8000ffff
112		120
113	/* Exit code reserved for hypervisor/software use */	121	/* Exit code reserved for hypervisor/software use */
@@ -218,6 +226,11 @@ · 104 unmodified lines
218	{ SVM_VMGEXIT_NMI_COMPLETE, "vmgexit_nmi_complete" }, \	226	{ SVM_VMGEXIT_NMI_COMPLETE, "vmgexit_nmi_complete" }, \
219	{ SVM_VMGEXIT_AP_HLT_LOOP, "vmgexit_ap_hlt_loop" }, \	227	{ SVM_VMGEXIT_AP_HLT_LOOP, "vmgexit_ap_hlt_loop" }, \
220	{ SVM_VMGEXIT_AP_JUMP_TABLE, "vmgexit_ap_jump_table" }, \	228	{ SVM_VMGEXIT_AP_JUMP_TABLE, "vmgexit_ap_jump_table" }, \
		229	{ SVM_VMGEXIT_PSC, "vmgexit_page_state_change" }, \
		230	{ SVM_VMGEXIT_GUEST_REQUEST, "vmgexit_guest_request" }, \
		231	{ SVM_VMGEXIT_EXT_GUEST_REQUEST, "vmgexit_ext_guest_request" }, \
		232	{ SVM_VMGEXIT_AP_CREATION, "vmgexit_ap_creation" }, \
		233	{ SVM_VMGEXIT_HV_FEATURES, "vmgexit_hypervisor_feature" }, \
221	{ SVM_EXIT_ERR, "invalid_guest_state" }	234	{ SVM_EXIT_ERR, "invalid_guest_state" }
222		235
223		236

arch/x86/kernel/Makefile+0−2

@@ -46,8 +46,6 @@ endif · 45 unmodified lines
46	# non-deterministic coverage.	46	# non-deterministic coverage.
47	KCOV_INSTRUMENT := n	47	KCOV_INSTRUMENT := n
48		48
49	CFLAGS_head$(BITS).o += -fno-stack-protector
50
51	CFLAGS_irq.o := -I $(srctree)/$(src)/../include/asm/trace	49	CFLAGS_irq.o := -I $(srctree)/$(src)/../include/asm/trace
52		50
53	obj-y := process_$(BITS).o signal.o	51	obj-y := process_$(BITS).o signal.o

arch/x86/kernel/cpu/common.c+4−0

 #include <asm/uv/uv.h>
 #include <asm/sigframe.h>
 #include <asm/traps.h>
+#include <asm/sev.h>
 #include "cpu.h"
 	load_TR_desc();
+	/* GHCB needs to be setup to handle #VC. */
+	setup_ghcb();
 	/* Finally load the IDT */
 	load_current_idt();
 }

arch/x86/kernel/head64.c+16−13

 	if (sme_get_me_mask()) {
 		vaddr = (unsigned long)__start_bss_decrypted;
 		vaddr_end = (unsigned long)__end_bss_decrypted;
 		for (; vaddr < vaddr_end; vaddr += PMD_SIZE) {
+			/*
+			 * On SNP, transition the page to shared in the RMP table so that
+			 * it is consistent with the page table attribute change.
+			 *
+			 * __start_bss_decrypted has a virtual address in the high range
+			 * mapping (kernel .text). PVALIDATE, by way of
+			 * early_snp_set_memory_shared(), requires a valid virtual
+			 * address but the kernel is currently running off of the identity
+			 * mapping so use __pa() to get a *currently* valid virtual address.
+			 */
+			early_snp_set_memory_shared(__pa(vaddr), __pa(vaddr), PTRS_PER_PMD);
 			i = pmd_index(vaddr);
 			pmd[i] -= sme_get_me_mask();
 		}
 	if (load_delta & ~PMD_PAGE_MASK)
 		for (;;);
-	/* Activate Secure Memory Encryption (SME) if supported and enabled */
-	sme_enable(bp);
 	/* Include the SME encryption mask in the fixup value */
 	load_delta += sme_get_me_mask();
 	return sme_postprocess_startup(bp, pmd);
 }
-unsigned long __startup_secondary_64(void)
-{
-	/*
-	 * Return the SME encryption mask (if SME is active) to be used as a
-	 * modifier for the initial pgdir entry programmed into CR3.
-	 */
-	return sme_get_me_mask();
-}
 /* Wipe all early page tables except for the kernel symbol map */
 static void __init reset_early_page_tables(void)
 {
 void early_setup_idt(void)
 {
 	/* VMM Communication Exception */
-	if (IS_ENABLED(CONFIG_AMD_MEM_ENCRYPT))
+	if (IS_ENABLED(CONFIG_AMD_MEM_ENCRYPT)) {
+		setup_ghcb();
 		set_bringup_idt_handler(bringup_idt_table, X86_TRAP_VC, vc_boot_ghcb);
+	}
 	bringup_idt_descr.address = (unsigned long)bringup_idt_table;
 	native_load_idt(&bringup_idt_descr);

arch/x86/kernel/head_64.S+34−3

@@ -65,10 +65,39 @@ SYM_CODE_START_NOALIGN(startup_64) · 64 unmodified lines
65	leaq (__end_init_task - FRAME_SIZE)(%rip), %rsp	65	leaq (__end_init_task - FRAME_SIZE)(%rip), %rsp
66		66
67	leaq _text(%rip), %rdi	67	leaq _text(%rip), %rdi
		68
		69	/*
		70	* initial_gs points to initial fixed_percpu_data struct with storage for
		71	* the stack protector canary. Global pointer fixups are needed at this
		72	* stage, so apply them as is done in fixup_pointer(), and initialize %gs
		73	* such that the canary can be accessed at %gs:40 for subsequent C calls.
		74	*/
		75	movl $MSR_GS_BASE, %ecx
		76	movq initial_gs(%rip), %rax
		77	movq $_text, %rdx
		78	subq %rdx, %rax
		79	addq %rdi, %rax
		80	movq %rax, %rdx
		81	shrq $32, %rdx
		82	wrmsr
		83
68	pushq %rsi	84	pushq %rsi
69	call startup_64_setup_env	85	call startup_64_setup_env
70	popq %rsi	86	popq %rsi
71		87
		88	#ifdef CONFIG_AMD_MEM_ENCRYPT
		89	/*
		90	* Activate SEV/SME memory encryption if supported/enabled. This needs to
		91	* be done now, since this also includes setup of the SEV-SNP CPUID table,
		92	* which needs to be done before any CPUID instructions are executed in
		93	* subsequent code.
		94	*/
		95	movq %rsi, %rdi
		96	pushq %rsi
		97	call sme_enable
		98	popq %rsi
		99	#endif
		100
72	/* Now switch to __KERNEL_CS so IRET works reliably */	101	/* Now switch to __KERNEL_CS so IRET works reliably */
73	pushq $__KERNEL_CS	102	pushq $__KERNEL_CS
74	leaq .Lon_kernel_cs(%rip), %rax	103	leaq .Lon_kernel_cs(%rip), %rax
@@ -134,9 +163,11 @@ SYM_INNER_LABEL(secondary_startup_64_no_verify, SYM_L_GLOBAL) · 59 unmodified lines
134	* Retrieve the modifier (SME encryption mask if SME is active) to be	163	* Retrieve the modifier (SME encryption mask if SME is active) to be
135	* added to the initial pgdir entry that will be programmed into CR3.	164	* added to the initial pgdir entry that will be programmed into CR3.
136	*/	165	*/
137	pushq %rsi	166	#ifdef CONFIG_AMD_MEM_ENCRYPT
138	call __startup_secondary_64	167	movq sme_me_mask, %rax
139	popq %rsi	168	#else
		169	xorq %rax, %rax
		170	#endif
140		171
141	/* Form the CR3 value being sure to include the CR3 modifier */	172	/* Form the CR3 value being sure to include the CR3 modifier */
142	addq $(init_top_pgt - __START_KERNEL_map), %rax	173	addq $(init_top_pgt - __START_KERNEL_map), %rax

arch/x86/kernel/probe_roms.c+12−1

 #include <asm/sections.h>
 #include <asm/io.h>
 #include <asm/setup_arch.h>
+#include <asm/sev.h>
 static struct resource system_rom_resource = {
 	.name	= "System ROM",
 void __init probe_roms(void)
 {
-	const unsigned char *rom;
 	unsigned long start, length, upper;
+	const unsigned char *rom;
 	unsigned char c;
 	int i;
+	/*
+	 * The ROM memory range is not part of the e820 table and is therefore not
+	 * pre-validated by BIOS. The kernel page table maps the ROM region as encrypted
+	 * memory, and SNP requires encrypted memory to be validated before access.
+	 * Do that here.
+	 */
+	snp_prep_memory(video_rom_resource.start,
+			((system_rom_resource.end + 1) - video_rom_resource.start),
+			SNP_PAGE_STATE_PRIVATE);
 	/* video rom */
 	upper = adapter_rom_resources[0].start;
 	for (start = video_rom_resource.start; start < upper; start += 2048) {

arch/x86/kernel/sev-shared.c+497−37

@@ -14,6 +14,68 @@ · 13 unmodified lines
14	#define has_cpuflag(f) boot_cpu_has(f)	14	#define has_cpuflag(f) boot_cpu_has(f)
15	#endif	15	#endif
16		16
		17	/* I/O parameters for CPUID-related helpers */
		18	struct cpuid_leaf {
		19	u32 fn;
		20	u32 subfn;
		21	u32 eax;
		22	u32 ebx;
		23	u32 ecx;
		24	u32 edx;
		25	};
		26
		27	/*
		28	* Individual entries of the SNP CPUID table, as defined by the SNP
		29	* Firmware ABI, Revision 0.9, Section 7.1, Table 14.
		30	*/
		31	struct snp_cpuid_fn {
		32	u32 eax_in;
		33	u32 ecx_in;
		34	u64 xcr0_in;
		35	u64 xss_in;
		36	u32 eax;
		37	u32 ebx;
		38	u32 ecx;
		39	u32 edx;
		40	u64 __reserved;
		41	} __packed;
		42
		43	/*
		44	* SNP CPUID table, as defined by the SNP Firmware ABI, Revision 0.9,
		45	* Section 8.14.2.6. Also noted there is the SNP firmware-enforced limit
		46	* of 64 entries per CPUID table.
		47	*/
		48	#define SNP_CPUID_COUNT_MAX 64
		49
		50	struct snp_cpuid_table {
		51	u32 count;
		52	u32 __reserved1;
		53	u64 __reserved2;
		54	struct snp_cpuid_fn fn[SNP_CPUID_COUNT_MAX];
		55	} __packed;
		56
		57	/*
		58	* Since feature negotiation related variables are set early in the boot
		59	* process they must reside in the .data section so as not to be zeroed
		60	* out when the .bss section is later cleared.
		61	*
		62	* GHCB protocol version negotiated with the hypervisor.
		63	*/
		64	static u16 ghcb_version __ro_after_init;
		65
		66	/* Copy of the SNP firmware's CPUID page. */
		67	static struct snp_cpuid_table cpuid_table_copy __ro_after_init;
		68
		69	/*
		70	* These will be initialized based on CPUID table so that non-present
		71	* all-zero leaves (for sparse tables) can be differentiated from
		72	* invalid/out-of-range leaves. This is needed since all-zero leaves
		73	* still need to be post-processed.
		74	*/
		75	static u32 cpuid_std_range_max __ro_after_init;
		76	static u32 cpuid_hyp_range_max __ro_after_init;
		77	static u32 cpuid_ext_range_max __ro_after_init;
		78
17	static bool __init sev_es_check_cpu_features(void)	79	static bool __init sev_es_check_cpu_features(void)
18	{	80	{
19	if (!has_cpuflag(X86_FEATURE_RDRAND)) {	81	if (!has_cpuflag(X86_FEATURE_RDRAND)) {
@@ -24,15 +86,12 @@ static bool __init sev_es_check_cpu_features(void) · 4 unmodified lines
24	return true;	86	return true;
25	}	87	}
26		88
27	static void __noreturn sev_es_terminate(unsigned int reason)	89	static void __noreturn sev_es_terminate(unsigned int set, unsigned int reason)
28	{	90	{
29	u64 val = GHCB_MSR_TERM_REQ;	91	u64 val = GHCB_MSR_TERM_REQ;
30		92
31	/*	93	/* Tell the hypervisor what went wrong. */
32	* Tell the hypervisor what went wrong - only reason-set 0 is	94	val \|= GHCB_SEV_TERM_REASON(set, reason);
33	* currently supported.
34	*/
35	val \|= GHCB_SEV_TERM_REASON(0, reason);
36		95
37	/* Request Guest Termination from Hypvervisor */	96	/* Request Guest Termination from Hypvervisor */
38	sev_es_wr_ghcb_msr(val);	97	sev_es_wr_ghcb_msr(val);
@@ -42,6 +101,42 @@ static void __noreturn sev_es_terminate(unsigned int reason) · 3 unmodified lines
42	asm volatile("hlt\n" : : : "memory");	101	asm volatile("hlt\n" : : : "memory");
43	}	102	}
44		103
		104	/*
		105	* The hypervisor features are available from GHCB version 2 onward.
		106	*/
		107	static u64 get_hv_features(void)
		108	{
		109	u64 val;
		110
		111	if (ghcb_version < 2)
		112	return 0;
		113
		114	sev_es_wr_ghcb_msr(GHCB_MSR_HV_FT_REQ);
		115	VMGEXIT();
		116
		117	val = sev_es_rd_ghcb_msr();
		118	if (GHCB_RESP_CODE(val) != GHCB_MSR_HV_FT_RESP)
		119	return 0;
		120
		121	return GHCB_MSR_HV_FT_RESP_VAL(val);
		122	}
		123
		124	static void snp_register_ghcb_early(unsigned long paddr)
		125	{
		126	unsigned long pfn = paddr >> PAGE_SHIFT;
		127	u64 val;
		128
		129	sev_es_wr_ghcb_msr(GHCB_MSR_REG_GPA_REQ_VAL(pfn));
		130	VMGEXIT();
		131
		132	val = sev_es_rd_ghcb_msr();
		133
		134	/* If the response GPA is not ours then abort the guest */
		135	if ((GHCB_RESP_CODE(val) != GHCB_MSR_REG_GPA_RESP) \|\|
		136	(GHCB_MSR_REG_GPA_RESP_VAL(val) != pfn))
		137	sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_REGISTER);
		138	}
		139
45	static bool sev_es_negotiate_protocol(void)	140	static bool sev_es_negotiate_protocol(void)
46	{	141	{
47	u64 val;	142	u64 val;
@@ -54,10 +149,12 @@ static bool sev_es_negotiate_protocol(void) · 6 unmodified lines
54	if (GHCB_MSR_INFO(val) != GHCB_MSR_SEV_INFO_RESP)	149	if (GHCB_MSR_INFO(val) != GHCB_MSR_SEV_INFO_RESP)
55	return false;	150	return false;
56		151
57	if (GHCB_MSR_PROTO_MAX(val) < GHCB_PROTO_OUR \|\|	152	if (GHCB_MSR_PROTO_MAX(val) < GHCB_PROTOCOL_MIN \|\|
58	GHCB_MSR_PROTO_MIN(val) > GHCB_PROTO_OUR)	153	GHCB_MSR_PROTO_MIN(val) > GHCB_PROTOCOL_MAX)
59	return false;	154	return false;
60		155
		156	ghcb_version = min_t(size_t, GHCB_MSR_PROTO_MAX(val), GHCB_PROTOCOL_MAX);
		157
61	return true;	158	return true;
62	}	159	}
63		160
@@ -104,10 +201,7 @@ static enum es_result verify_exception_info(struct ghcb *ghcb, struct es_em_ctxt · 40 unmodified lines
104		201
105	if (ret == 1) {	202	if (ret == 1) {
106	u64 info = ghcb->save.sw_exit_info_2;	203	u64 info = ghcb->save.sw_exit_info_2;
107	unsigned long v;	204	unsigned long v = info & SVM_EVTINJ_VEC_MASK;
108
109	info = ghcb->save.sw_exit_info_2;
110	v = info & SVM_EVTINJ_VEC_MASK;
111		205
112	/* Check if exception information from hypervisor is sane. */	206	/* Check if exception information from hypervisor is sane. */
113	if ((info & SVM_EVTINJ_VALID) &&	207	if ((info & SVM_EVTINJ_VALID) &&
@@ -130,7 +224,7 @@ enum es_result sev_es_ghcb_hv_call(struct ghcb *ghcb, bool set_ghcb_msr, · 16 unmodified lines
130	u64 exit_info_1, u64 exit_info_2)	224	u64 exit_info_1, u64 exit_info_2)
131	{	225	{
132	/* Fill in protocol and format specifiers */	226	/* Fill in protocol and format specifiers */
133	ghcb->protocol_version = GHCB_PROTOCOL_MAX;	227	ghcb->protocol_version = ghcb_version;
134	ghcb->ghcb_usage = GHCB_DEFAULT_USAGE;	228	ghcb->ghcb_usage = GHCB_DEFAULT_USAGE;
135		229
136	ghcb_set_sw_exit_code(ghcb, exit_code);	230	ghcb_set_sw_exit_code(ghcb, exit_code);
@@ -150,6 +244,290 @@ enum es_result sev_es_ghcb_hv_call(struct ghcb *ghcb, bool set_ghcb_msr, · 13 unmodified lines
150	return verify_exception_info(ghcb, ctxt);	244	return verify_exception_info(ghcb, ctxt);
151	}	245	}
152		246
		247	static int __sev_cpuid_hv(u32 fn, int reg_idx, u32 *reg)
		248	{
		249	u64 val;
		250
		251	sev_es_wr_ghcb_msr(GHCB_CPUID_REQ(fn, reg_idx));
		252	VMGEXIT();
		253	val = sev_es_rd_ghcb_msr();
		254	if (GHCB_RESP_CODE(val) != GHCB_MSR_CPUID_RESP)
		255	return -EIO;
		256
		257	*reg = (val >> 32);
		258
		259	return 0;
		260	}
		261
		262	static int sev_cpuid_hv(struct cpuid_leaf *leaf)
		263	{
		264	int ret;
		265
		266	/*
		267	* MSR protocol does not support fetching non-zero subfunctions, but is
		268	* sufficient to handle current early-boot cases. Should that change,
		269	* make sure to report an error rather than ignoring the index and
		270	* grabbing random values. If this issue arises in the future, handling
		271	* can be added here to use GHCB-page protocol for cases that occur late
		272	* enough in boot that GHCB page is available.
		273	*/
		274	if (cpuid_function_is_indexed(leaf->fn) && leaf->subfn)
		275	return -EINVAL;
		276
		277	ret = __sev_cpuid_hv(leaf->fn, GHCB_CPUID_REQ_EAX, &leaf->eax);
		278	ret = ret ? : __sev_cpuid_hv(leaf->fn, GHCB_CPUID_REQ_EBX, &leaf->ebx);
		279	ret = ret ? : __sev_cpuid_hv(leaf->fn, GHCB_CPUID_REQ_ECX, &leaf->ecx);
		280	ret = ret ? : __sev_cpuid_hv(leaf->fn, GHCB_CPUID_REQ_EDX, &leaf->edx);
		281
		282	return ret;
		283	}
		284
		285	/*
		286	* This may be called early while still running on the initial identity
		287	* mapping. Use RIP-relative addressing to obtain the correct address
		288	* while running with the initial identity mapping as well as the
		289	* switch-over to kernel virtual addresses later.
		290	*/
		291	static const struct snp_cpuid_table *snp_cpuid_get_table(void)
		292	{
		293	void *ptr;
		294
		295	asm ("lea cpuid_table_copy(%%rip), %0"
		296	: "=r" (ptr)
		297	: "p" (&cpuid_table_copy));
		298
		299	return ptr;
		300	}
		301
		302	/*
		303	* The SNP Firmware ABI, Revision 0.9, Section 7.1, details the use of
		304	* XCR0_IN and XSS_IN to encode multiple versions of 0xD subfunctions 0
		305	* and 1 based on the corresponding features enabled by a particular
		306	* combination of XCR0 and XSS registers so that a guest can look up the
		307	* version corresponding to the features currently enabled in its XCR0/XSS
		308	* registers. The only values that differ between these versions/table
		309	* entries is the enabled XSAVE area size advertised via EBX.
		310	*
		311	* While hypervisors may choose to make use of this support, it is more
		312	* robust/secure for a guest to simply find the entry corresponding to the
		313	* base/legacy XSAVE area size (XCR0=1 or XCR0=3), and then calculate the
		314	* XSAVE area size using subfunctions 2 through 64, as documented in APM
		315	* Volume 3, Rev 3.31, Appendix E.3.8, which is what is done here.
		316	*
		317	* Since base/legacy XSAVE area size is documented as 0x240, use that value
		318	* directly rather than relying on the base size in the CPUID table.
		319	*
		320	* Return: XSAVE area size on success, 0 otherwise.
		321	*/
		322	static u32 snp_cpuid_calc_xsave_size(u64 xfeatures_en, bool compacted)
		323	{
		324	const struct snp_cpuid_table *cpuid_table = snp_cpuid_get_table();
		325	u64 xfeatures_found = 0;
		326	u32 xsave_size = 0x240;
		327	int i;
		328
		329	for (i = 0; i < cpuid_table->count; i++) {
		330	const struct snp_cpuid_fn *e = &cpuid_table->fn[i];
		331
		332	if (!(e->eax_in == 0xD && e->ecx_in > 1 && e->ecx_in < 64))
		333	continue;
		334	if (!(xfeatures_en & (BIT_ULL(e->ecx_in))))
		335	continue;
		336	if (xfeatures_found & (BIT_ULL(e->ecx_in)))
		337	continue;
		338
		339	xfeatures_found \|= (BIT_ULL(e->ecx_in));
		340
		341	if (compacted)
		342	xsave_size += e->eax;
		343	else
		344	xsave_size = max(xsave_size, e->eax + e->ebx);
		345	}
		346
		347	/*
		348	* Either the guest set unsupported XCR0/XSS bits, or the corresponding
		349	* entries in the CPUID table were not present. This is not a valid
		350	* state to be in.
		351	*/
		352	if (xfeatures_found != (xfeatures_en & GENMASK_ULL(63, 2)))
		353	return 0;
		354
		355	return xsave_size;
		356	}
		357
		358	static bool
		359	snp_cpuid_get_validated_func(struct cpuid_leaf *leaf)
		360	{
		361	const struct snp_cpuid_table *cpuid_table = snp_cpuid_get_table();
		362	int i;
		363
		364	for (i = 0; i < cpuid_table->count; i++) {
		365	const struct snp_cpuid_fn *e = &cpuid_table->fn[i];
		366
		367	if (e->eax_in != leaf->fn)
		368	continue;
		369
		370	if (cpuid_function_is_indexed(leaf->fn) && e->ecx_in != leaf->subfn)
		371	continue;
		372
		373	/*
		374	* For 0xD subfunctions 0 and 1, only use the entry corresponding
		375	* to the base/legacy XSAVE area size (XCR0=1 or XCR0=3, XSS=0).
		376	* See the comments above snp_cpuid_calc_xsave_size() for more
		377	* details.
		378	*/
		379	if (e->eax_in == 0xD && (e->ecx_in == 0 \|\| e->ecx_in == 1))
		380	if (!(e->xcr0_in == 1 \|\| e->xcr0_in == 3) \|\| e->xss_in)
		381	continue;
		382
		383	leaf->eax = e->eax;
		384	leaf->ebx = e->ebx;
		385	leaf->ecx = e->ecx;
		386	leaf->edx = e->edx;
		387
		388	return true;
		389	}
		390
		391	return false;
		392	}
		393
		394	static void snp_cpuid_hv(struct cpuid_leaf *leaf)
		395	{
		396	if (sev_cpuid_hv(leaf))
		397	sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_CPUID_HV);
		398	}
		399
		400	static int snp_cpuid_postprocess(struct cpuid_leaf *leaf)
		401	{
		402	struct cpuid_leaf leaf_hv = *leaf;
		403
		404	switch (leaf->fn) {
		405	case 0x1:
		406	snp_cpuid_hv(&leaf_hv);
		407
		408	/* initial APIC ID */
		409	leaf->ebx = (leaf_hv.ebx & GENMASK(31, 24)) \| (leaf->ebx & GENMASK(23, 0));
		410	/* APIC enabled bit */
		411	leaf->edx = (leaf_hv.edx & BIT(9)) \| (leaf->edx & ~BIT(9));
		412
		413	/* OSXSAVE enabled bit */
		414	if (native_read_cr4() & X86_CR4_OSXSAVE)
		415	leaf->ecx \|= BIT(27);
		416	break;
		417	case 0x7:
		418	/* OSPKE enabled bit */
		419	leaf->ecx &= ~BIT(4);
		420	if (native_read_cr4() & X86_CR4_PKE)
		421	leaf->ecx \|= BIT(4);
		422	break;
		423	case 0xB:
		424	leaf_hv.subfn = 0;
		425	snp_cpuid_hv(&leaf_hv);
		426
		427	/* extended APIC ID */
		428	leaf->edx = leaf_hv.edx;
		429	break;
		430	case 0xD: {
		431	bool compacted = false;
		432	u64 xcr0 = 1, xss = 0;
		433	u32 xsave_size;
		434
		435	if (leaf->subfn != 0 && leaf->subfn != 1)
		436	return 0;
		437
		438	if (native_read_cr4() & X86_CR4_OSXSAVE)
		439	xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK);
		440	if (leaf->subfn == 1) {
		441	/* Get XSS value if XSAVES is enabled. */
		442	if (leaf->eax & BIT(3)) {
		443	unsigned long lo, hi;
		444
		445	asm volatile("rdmsr" : "=a" (lo), "=d" (hi)
		446	: "c" (MSR_IA32_XSS));
		447	xss = (hi << 32) \| lo;
		448	}
		449
		450	/*
		451	* The PPR and APM aren't clear on what size should be
		452	* encoded in 0xD:0x1:EBX when compaction is not enabled
		453	* by either XSAVEC (feature bit 1) or XSAVES (feature
		454	* bit 3) since SNP-capable hardware has these feature
		455	* bits fixed as 1. KVM sets it to 0 in this case, but
		456	* to avoid this becoming an issue it's safer to simply
		457	* treat this as unsupported for SNP guests.
		458	*/
		459	if (!(leaf->eax & (BIT(1) \| BIT(3))))
		460	return -EINVAL;
		461
		462	compacted = true;
		463	}
		464
		465	xsave_size = snp_cpuid_calc_xsave_size(xcr0 \| xss, compacted);
		466	if (!xsave_size)
		467	return -EINVAL;
		468
		469	leaf->ebx = xsave_size;
		470	}
		471	break;
		472	case 0x8000001E:
		473	snp_cpuid_hv(&leaf_hv);
		474
		475	/* extended APIC ID */
		476	leaf->eax = leaf_hv.eax;
		477	/* compute ID */
		478	leaf->ebx = (leaf->ebx & GENMASK(31, 8)) \| (leaf_hv.ebx & GENMASK(7, 0));
		479	/* node ID */
		480	leaf->ecx = (leaf->ecx & GENMASK(31, 8)) \| (leaf_hv.ecx & GENMASK(7, 0));
		481	break;
		482	default:
		483	/* No fix-ups needed, use values as-is. */
		484	break;
		485	}
		486
		487	return 0;
		488	}
		489
		490	/*
		491	* Returns -EOPNOTSUPP if feature not enabled. Any other non-zero return value
		492	* should be treated as fatal by caller.
		493	*/
		494	static int snp_cpuid(struct cpuid_leaf *leaf)
		495	{
		496	const struct snp_cpuid_table *cpuid_table = snp_cpuid_get_table();
		497
		498	if (!cpuid_table->count)
		499	return -EOPNOTSUPP;
		500
		501	if (!snp_cpuid_get_validated_func(leaf)) {
		502	/*
		503	* Some hypervisors will avoid keeping track of CPUID entries
		504	* where all values are zero, since they can be handled the
		505	* same as out-of-range values (all-zero). This is useful here
		506	* as well as it allows virtually all guest configurations to
		507	* work using a single SNP CPUID table.
		508	*
		509	* To allow for this, there is a need to distinguish between
		510	* out-of-range entries and in-range zero entries, since the
		511	* CPUID table entries are only a template that may need to be
		512	* augmented with additional values for things like
		513	* CPU-specific information during post-processing. So if it's
		514	* not in the table, set the values to zero. Then, if they are
		515	* within a valid CPUID range, proceed with post-processing
		516	* using zeros as the initial values. Otherwise, skip
		517	* post-processing and just return zeros immediately.
		518	*/
		519	leaf->eax = leaf->ebx = leaf->ecx = leaf->edx = 0;
		520
		521	/* Skip post-processing for out-of-range zero leafs. */
		522	if (!(leaf->fn <= cpuid_std_range_max \|\|
		523	(leaf->fn >= 0x40000000 && leaf->fn <= cpuid_hyp_range_max) \|\|
		524	(leaf->fn >= 0x80000000 && leaf->fn <= cpuid_ext_range_max)))
		525	return 0;
		526	}
		527
		528	return snp_cpuid_postprocess(leaf);
		529	}
		530
153	/*	531	/*
154	* Boot VC Handler - This is the first VC handler during boot, there is no GHCB	532	* Boot VC Handler - This is the first VC handler during boot, there is no GHCB
155	* page yet, so it only supports the MSR based communication with the	533	* page yet, so it only supports the MSR based communication with the
@@ -157,40 +535,33 @@ enum es_result sev_es_ghcb_hv_call(struct ghcb *ghcb, bool set_ghcb_msr, · 1 unmodified line
157	*/	535	*/
158	void __init do_vc_no_ghcb(struct pt_regs *regs, unsigned long exit_code)	536	void __init do_vc_no_ghcb(struct pt_regs *regs, unsigned long exit_code)
159	{	537	{
		538	unsigned int subfn = lower_bits(regs->cx, 32);
160	unsigned int fn = lower_bits(regs->ax, 32);	539	unsigned int fn = lower_bits(regs->ax, 32);
161	unsigned long val;	540	struct cpuid_leaf leaf;
		541	int ret;
162		542
163	/* Only CPUID is supported via MSR protocol */	543	/* Only CPUID is supported via MSR protocol */
164	if (exit_code != SVM_EXIT_CPUID)	544	if (exit_code != SVM_EXIT_CPUID)
165	goto fail;	545	goto fail;
166		546
167	sev_es_wr_ghcb_msr(GHCB_CPUID_REQ(fn, GHCB_CPUID_REQ_EAX));	547	leaf.fn = fn;
168	VMGEXIT();	548	leaf.subfn = subfn;
169	val = sev_es_rd_ghcb_msr();
170	if (GHCB_RESP_CODE(val) != GHCB_MSR_CPUID_RESP)
171	goto fail;
172	regs->ax = val >> 32;
173		549
174	sev_es_wr_ghcb_msr(GHCB_CPUID_REQ(fn, GHCB_CPUID_REQ_EBX));	550	ret = snp_cpuid(&leaf);
175	VMGEXIT();	551	if (!ret)
176	val = sev_es_rd_ghcb_msr();	552	goto cpuid_done;
177	if (GHCB_RESP_CODE(val) != GHCB_MSR_CPUID_RESP)
178	goto fail;
179	regs->bx = val >> 32;
180		553
181	sev_es_wr_ghcb_msr(GHCB_CPUID_REQ(fn, GHCB_CPUID_REQ_ECX));	554	if (ret != -EOPNOTSUPP)
182	VMGEXIT();
183	val = sev_es_rd_ghcb_msr();
184	if (GHCB_RESP_CODE(val) != GHCB_MSR_CPUID_RESP)
185	goto fail;	555	goto fail;
186	regs->cx = val >> 32;
187		556
188	sev_es_wr_ghcb_msr(GHCB_CPUID_REQ(fn, GHCB_CPUID_REQ_EDX));	557	if (sev_cpuid_hv(&leaf))
189	VMGEXIT();
190	val = sev_es_rd_ghcb_msr();
191	if (GHCB_RESP_CODE(val) != GHCB_MSR_CPUID_RESP)
192	goto fail;	558	goto fail;
193	regs->dx = val >> 32;	559
		560	cpuid_done:
		561	regs->ax = leaf.eax;
		562	regs->bx = leaf.ebx;
		563	regs->cx = leaf.ecx;
		564	regs->dx = leaf.edx;
194		565
195	/*	566	/*
196	* This is a VC handler and the #VC is only raised when SEV-ES is	567	* This is a VC handler and the #VC is only raised when SEV-ES is
@@ -221,7 +592,7 @@ void __init do_vc_no_ghcb(struct pt_regs *regs, unsigned long exit_code) · 24 unmodified lines
221		592
222	fail:	593	fail:
223	/* Terminate the guest */	594	/* Terminate the guest */
224	sev_es_terminate(GHCB_SEV_ES_GEN_REQ);	595	sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_GEN_REQ);
225	}	596	}
226		597
227	static enum es_result vc_insn_string_read(struct es_em_ctxt *ctxt,	598	static enum es_result vc_insn_string_read(struct es_em_ctxt *ctxt,
@@ -481,12 +852,37 @@ static enum es_result vc_handle_ioio(struct ghcb ghcb, struct es_em_ctxt ctxt) · 253 unmodified lines
481	return ret;	852	return ret;
482	}	853	}
483		854
		855	static int vc_handle_cpuid_snp(struct pt_regs *regs)
		856	{
		857	struct cpuid_leaf leaf;
		858	int ret;
		859
		860	leaf.fn = regs->ax;
		861	leaf.subfn = regs->cx;
		862	ret = snp_cpuid(&leaf);
		863	if (!ret) {
		864	regs->ax = leaf.eax;
		865	regs->bx = leaf.ebx;
		866	regs->cx = leaf.ecx;
		867	regs->dx = leaf.edx;
		868	}
		869
		870	return ret;
		871	}
		872
484	static enum es_result vc_handle_cpuid(struct ghcb *ghcb,	873	static enum es_result vc_handle_cpuid(struct ghcb *ghcb,
485	struct es_em_ctxt *ctxt)	874	struct es_em_ctxt *ctxt)
486	{	875	{
487	struct pt_regs *regs = ctxt->regs;	876	struct pt_regs *regs = ctxt->regs;
488	u32 cr4 = native_read_cr4();	877	u32 cr4 = native_read_cr4();
489	enum es_result ret;	878	enum es_result ret;
		879	int snp_cpuid_ret;
		880
		881	snp_cpuid_ret = vc_handle_cpuid_snp(regs);
		882	if (!snp_cpuid_ret)
		883	return ES_OK;
		884	if (snp_cpuid_ret != -EOPNOTSUPP)
		885	return ES_VMM_ERROR;
490		886
491	ghcb_set_rax(ghcb, regs->ax);	887	ghcb_set_rax(ghcb, regs->ax);
492	ghcb_set_rcx(ghcb, regs->cx);	888	ghcb_set_rcx(ghcb, regs->cx);
@@ -538,3 +934,67 @@ static enum es_result vc_handle_rdtsc(struct ghcb *ghcb, · 45 unmodified lines
538		934
539	return ES_OK;	935	return ES_OK;
540	}	936	}
		937
		938	struct cc_setup_data {
		939	struct setup_data header;
		940	u32 cc_blob_address;
		941	};
		942
		943	/*
		944	* Search for a Confidential Computing blob passed in as a setup_data entry
		945	* via the Linux Boot Protocol.
		946	*/
		947	static struct cc_blob_sev_info find_cc_blob_setup_data(struct boot_params bp)
		948	{
		949	struct cc_setup_data *sd = NULL;
		950	struct setup_data *hdr;
		951
		952	hdr = (struct setup_data *)bp->hdr.setup_data;
		953
		954	while (hdr) {
		955	if (hdr->type == SETUP_CC_BLOB) {
		956	sd = (struct cc_setup_data *)hdr;
		957	return (struct cc_blob_sev_info *)(unsigned long)sd->cc_blob_address;
		958	}
		959	hdr = (struct setup_data *)hdr->next;
		960	}
		961
		962	return NULL;
		963	}
		964
		965	/*
		966	* Initialize the kernel's copy of the SNP CPUID table, and set up the
		967	* pointer that will be used to access it.
		968	*
		969	* Maintaining a direct mapping of the SNP CPUID table used by firmware would
		970	* be possible as an alternative, but the approach is brittle since the
		971	* mapping needs to be updated in sync with all the changes to virtual memory
		972	* layout and related mapping facilities throughout the boot process.
		973	*/
		974	static void __init setup_cpuid_table(const struct cc_blob_sev_info *cc_info)
		975	{
		976	const struct snp_cpuid_table cpuid_table_fw, cpuid_table;
		977	int i;
		978
		979	if (!cc_info \|\| !cc_info->cpuid_phys \|\| cc_info->cpuid_len < PAGE_SIZE)
		980	sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_CPUID);
		981
		982	cpuid_table_fw = (const struct snp_cpuid_table *)cc_info->cpuid_phys;
		983	if (!cpuid_table_fw->count \|\| cpuid_table_fw->count > SNP_CPUID_COUNT_MAX)
		984	sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_CPUID);
		985
		986	cpuid_table = snp_cpuid_get_table();
		987	memcpy((void )cpuid_table, cpuid_table_fw, sizeof(cpuid_table));
		988
		989	/* Initialize CPUID ranges for range-checking. */
		990	for (i = 0; i < cpuid_table->count; i++) {
		991	const struct snp_cpuid_fn *fn = &cpuid_table->fn[i];
		992
		993	if (fn->eax_in == 0x0)
		994	cpuid_std_range_max = fn->eax;
		995	else if (fn->eax_in == 0x40000000)
		996	cpuid_hyp_range_max = fn->eax;
		997	else if (fn->eax_in == 0x80000000)
		998	cpuid_ext_range_max = fn->eax;
		999	}
		1000	}

arch/x86/kernel/sev.c+838−17

@@ -18,6 +18,10 @@ · 17 unmodified lines
18	#include <linux/memblock.h>	18	#include <linux/memblock.h>
19	#include <linux/kernel.h>	19	#include <linux/kernel.h>
20	#include <linux/mm.h>	20	#include <linux/mm.h>
		21	#include <linux/cpumask.h>
		22	#include <linux/efi.h>
		23	#include <linux/platform_device.h>
		24	#include <linux/io.h>
21		25
22	#include <asm/cpu_entry_area.h>	26	#include <asm/cpu_entry_area.h>
23	#include <asm/stacktrace.h>	27	#include <asm/stacktrace.h>
@@ -31,9 +35,28 @@ · 7 unmodified lines
31	#include <asm/svm.h>	35	#include <asm/svm.h>
32	#include <asm/smp.h>	36	#include <asm/smp.h>
33	#include <asm/cpu.h>	37	#include <asm/cpu.h>
		38	#include <asm/apic.h>
		39	#include <asm/cpuid.h>
		40	#include <asm/cmdline.h>
34		41
35	#define DR7_RESET_VALUE 0x400	42	#define DR7_RESET_VALUE 0x400
36		43
		44	/* AP INIT values as documented in the APM2 section "Processor Initialization State" */
		45	#define AP_INIT_CS_LIMIT 0xffff
		46	#define AP_INIT_DS_LIMIT 0xffff
		47	#define AP_INIT_LDTR_LIMIT 0xffff
		48	#define AP_INIT_GDTR_LIMIT 0xffff
		49	#define AP_INIT_IDTR_LIMIT 0xffff
		50	#define AP_INIT_TR_LIMIT 0xffff
		51	#define AP_INIT_RFLAGS_DEFAULT 0x2
		52	#define AP_INIT_DR6_DEFAULT 0xffff0ff0
		53	#define AP_INIT_GPAT_DEFAULT 0x0007040600070406ULL
		54	#define AP_INIT_XCR0_DEFAULT 0x1
		55	#define AP_INIT_X87_FTW_DEFAULT 0x5555
		56	#define AP_INIT_X87_FCW_DEFAULT 0x0040
		57	#define AP_INIT_CR0_DEFAULT 0x60000010
		58	#define AP_INIT_MXCSR_DEFAULT 0x1f80
		59
37	/* For early boot hypervisor communication in SEV-ES enabled guests */	60	/* For early boot hypervisor communication in SEV-ES enabled guests */
38	static struct ghcb boot_ghcb_page __bss_decrypted __aligned(PAGE_SIZE);	61	static struct ghcb boot_ghcb_page __bss_decrypted __aligned(PAGE_SIZE);
39		62
@@ -41,7 +64,10 @@ static struct ghcb boot_ghcb_page __bss_decrypted __aligned(PAGE_SIZE); · 1 unmodified line
41	* Needs to be in the .data section because we need it NULL before bss is	64	* Needs to be in the .data section because we need it NULL before bss is
42	* cleared	65	* cleared
43	*/	66	*/
44	static struct ghcb __initdata *boot_ghcb;	67	static struct ghcb *boot_ghcb __section(".data");
		68
		69	/* Bitmap of SEV features supported by the hypervisor */
		70	static u64 sev_hv_features __ro_after_init;
45		71
46	/* #VC handler runtime per-CPU data */	72	/* #VC handler runtime per-CPU data */
47	struct sev_es_runtime_data {	73	struct sev_es_runtime_data {
@@ -87,6 +113,15 @@ struct ghcb_state { · 39 unmodified lines
87	static DEFINE_PER_CPU(struct sev_es_runtime_data*, runtime_data);	113	static DEFINE_PER_CPU(struct sev_es_runtime_data*, runtime_data);
88	DEFINE_STATIC_KEY_FALSE(sev_es_enable_key);	114	DEFINE_STATIC_KEY_FALSE(sev_es_enable_key);
89		115
		116	static DEFINE_PER_CPU(struct sev_es_save_area *, sev_vmsa);
		117
		118	struct sev_config {
		119	__u64 debug : 1,
		120	__reserved : 63;
		121	};
		122
		123	static struct sev_config sev_cfg __read_mostly;
		124
90	static __always_inline bool on_vc_stack(struct pt_regs *regs)	125	static __always_inline bool on_vc_stack(struct pt_regs *regs)
91	{	126	{
92	unsigned long sp = regs->sp;	127	unsigned long sp = regs->sp;
@@ -523,13 +558,68 @@ void noinstr __sev_es_nmi_complete(void) · 430 unmodified lines
523	__sev_put_ghcb(&state);	558	__sev_put_ghcb(&state);
524	}	559	}
525		560
526	static u64 get_jump_table_addr(void)	561	static u64 __init get_secrets_page(void)
		562	{
		563	u64 pa_data = boot_params.cc_blob_address;
		564	struct cc_blob_sev_info info;
		565	void *map;
		566
		567	/*
		568	* The CC blob contains the address of the secrets page, check if the
		569	* blob is present.
		570	*/
		571	if (!pa_data)
		572	return 0;
		573
		574	map = early_memremap(pa_data, sizeof(info));
		575	if (!map) {
		576	pr_err("Unable to locate SNP secrets page: failed to map the Confidential Computing blob.\n");
		577	return 0;
		578	}
		579	memcpy(&info, map, sizeof(info));
		580	early_memunmap(map, sizeof(info));
		581
		582	/* smoke-test the secrets page passed */
		583	if (!info.secrets_phys \|\| info.secrets_len != PAGE_SIZE)
		584	return 0;
		585
		586	return info.secrets_phys;
		587	}
		588
		589	static u64 __init get_snp_jump_table_addr(void)
		590	{
		591	struct snp_secrets_page_layout *layout;
		592	void __iomem *mem;
		593	u64 pa, addr;
		594
		595	pa = get_secrets_page();
		596	if (!pa)
		597	return 0;
		598
		599	mem = ioremap_encrypted(pa, PAGE_SIZE);
		600	if (!mem) {
		601	pr_err("Unable to locate AP jump table address: failed to map the SNP secrets page.\n");
		602	return 0;
		603	}
		604
		605	layout = (__force struct snp_secrets_page_layout *)mem;
		606
		607	addr = layout->os_area.ap_jump_table_pa;
		608	iounmap(mem);
		609
		610	return addr;
		611	}
		612
		613	static u64 __init get_jump_table_addr(void)
527	{	614	{
528	struct ghcb_state state;	615	struct ghcb_state state;
529	unsigned long flags;	616	unsigned long flags;
530	struct ghcb *ghcb;	617	struct ghcb *ghcb;
531	u64 ret = 0;	618	u64 ret = 0;
532		619
		620	if (cc_platform_has(CC_ATTR_GUEST_SEV_SNP))
		621	return get_snp_jump_table_addr();
		622
533	local_irq_save(flags);	623	local_irq_save(flags);
534		624
535	ghcb = __sev_get_ghcb(&state);	625	ghcb = __sev_get_ghcb(&state);
@@ -553,7 +643,496 @@ static u64 get_jump_table_addr(void) · 17 unmodified lines
553	return ret;	643	return ret;
554	}	644	}
555		645
556	int sev_es_setup_ap_jump_table(struct real_mode_header *rmh)	646	static void pvalidate_pages(unsigned long vaddr, unsigned int npages, bool validate)
		647	{
		648	unsigned long vaddr_end;
		649	int rc;
		650
		651	vaddr = vaddr & PAGE_MASK;
		652	vaddr_end = vaddr + (npages << PAGE_SHIFT);
		653
		654	while (vaddr < vaddr_end) {
		655	rc = pvalidate(vaddr, RMP_PG_SIZE_4K, validate);
		656	if (WARN(rc, "Failed to validate address 0x%lx ret %d", vaddr, rc))
		657	sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PVALIDATE);
		658
		659	vaddr = vaddr + PAGE_SIZE;
		660	}
		661	}
		662
		663	static void __init early_set_pages_state(unsigned long paddr, unsigned int npages, enum psc_op op)
		664	{
		665	unsigned long paddr_end;
		666	u64 val;
		667
		668	paddr = paddr & PAGE_MASK;
		669	paddr_end = paddr + (npages << PAGE_SHIFT);
		670
		671	while (paddr < paddr_end) {
		672	/*
		673	* Use the MSR protocol because this function can be called before
		674	* the GHCB is established.
		675	*/
		676	sev_es_wr_ghcb_msr(GHCB_MSR_PSC_REQ_GFN(paddr >> PAGE_SHIFT, op));
		677	VMGEXIT();
		678
		679	val = sev_es_rd_ghcb_msr();
		680
		681	if (WARN(GHCB_RESP_CODE(val) != GHCB_MSR_PSC_RESP,
		682	"Wrong PSC response code: 0x%x\n",
		683	(unsigned int)GHCB_RESP_CODE(val)))
		684	goto e_term;
		685
		686	if (WARN(GHCB_MSR_PSC_RESP_VAL(val),
		687	"Failed to change page state to '%s' paddr 0x%lx error 0x%llx\n",
		688	op == SNP_PAGE_STATE_PRIVATE ? "private" : "shared",
		689	paddr, GHCB_MSR_PSC_RESP_VAL(val)))
		690	goto e_term;
		691
		692	paddr = paddr + PAGE_SIZE;
		693	}
		694
		695	return;
		696
		697	e_term:
		698	sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PSC);
		699	}
		700
		701	void __init early_snp_set_memory_private(unsigned long vaddr, unsigned long paddr,
		702	unsigned int npages)
		703	{
		704	if (!cc_platform_has(CC_ATTR_GUEST_SEV_SNP))
		705	return;
		706
		707	/*
		708	* Ask the hypervisor to mark the memory pages as private in the RMP
		709	* table.
		710	*/
		711	early_set_pages_state(paddr, npages, SNP_PAGE_STATE_PRIVATE);
		712
		713	/* Validate the memory pages after they've been added in the RMP table. */
		714	pvalidate_pages(vaddr, npages, true);
		715	}
		716
		717	void __init early_snp_set_memory_shared(unsigned long vaddr, unsigned long paddr,
		718	unsigned int npages)
		719	{
		720	if (!cc_platform_has(CC_ATTR_GUEST_SEV_SNP))
		721	return;
		722
		723	/* Invalidate the memory pages before they are marked shared in the RMP table. */
		724	pvalidate_pages(vaddr, npages, false);
		725
		726	/* Ask hypervisor to mark the memory pages shared in the RMP table. */
		727	early_set_pages_state(paddr, npages, SNP_PAGE_STATE_SHARED);
		728	}
		729
		730	void __init snp_prep_memory(unsigned long paddr, unsigned int sz, enum psc_op op)
		731	{
		732	unsigned long vaddr, npages;
		733
		734	vaddr = (unsigned long)__va(paddr);
		735	npages = PAGE_ALIGN(sz) >> PAGE_SHIFT;
		736
		737	if (op == SNP_PAGE_STATE_PRIVATE)
		738	early_snp_set_memory_private(vaddr, paddr, npages);
		739	else if (op == SNP_PAGE_STATE_SHARED)
		740	early_snp_set_memory_shared(vaddr, paddr, npages);
		741	else
		742	WARN(1, "invalid memory op %d\n", op);
		743	}
		744
		745	static int vmgexit_psc(struct snp_psc_desc *desc)
		746	{
		747	int cur_entry, end_entry, ret = 0;
		748	struct snp_psc_desc *data;
		749	struct ghcb_state state;
		750	struct es_em_ctxt ctxt;
		751	unsigned long flags;
		752	struct ghcb *ghcb;
		753
		754	/*
		755	* __sev_get_ghcb() needs to run with IRQs disabled because it is using
		756	* a per-CPU GHCB.
		757	*/
		758	local_irq_save(flags);
		759
		760	ghcb = __sev_get_ghcb(&state);
		761	if (!ghcb) {
		762	ret = 1;
		763	goto out_unlock;
		764	}
		765
		766	/* Copy the input desc into GHCB shared buffer */
		767	data = (struct snp_psc_desc *)ghcb->shared_buffer;
		768	memcpy(ghcb->shared_buffer, desc, min_t(int, GHCB_SHARED_BUF_SIZE, sizeof(*desc)));
		769
		770	/*
		771	* As per the GHCB specification, the hypervisor can resume the guest
		772	* before processing all the entries. Check whether all the entries
		773	* are processed. If not, then keep retrying. Note, the hypervisor
		774	* will update the data memory directly to indicate the status, so
		775	* reference the data->hdr everywhere.
		776	*
		777	* The strategy here is to wait for the hypervisor to change the page
		778	* state in the RMP table before guest accesses the memory pages. If the
		779	* page state change was not successful, then later memory access will
		780	* result in a crash.
		781	*/
		782	cur_entry = data->hdr.cur_entry;
		783	end_entry = data->hdr.end_entry;
		784
		785	while (data->hdr.cur_entry <= data->hdr.end_entry) {
		786	ghcb_set_sw_scratch(ghcb, (u64)__pa(data));
		787
		788	/* This will advance the shared buffer data points to. */
		789	ret = sev_es_ghcb_hv_call(ghcb, true, &ctxt, SVM_VMGEXIT_PSC, 0, 0);
		790
		791	/*
		792	* Page State Change VMGEXIT can pass error code through
		793	* exit_info_2.
		794	*/
		795	if (WARN(ret \|\| ghcb->save.sw_exit_info_2,
		796	"SNP: PSC failed ret=%d exit_info_2=%llx\n",
		797	ret, ghcb->save.sw_exit_info_2)) {
		798	ret = 1;
		799	goto out;
		800	}
		801
		802	/* Verify that reserved bit is not set */
		803	if (WARN(data->hdr.reserved, "Reserved bit is set in the PSC header\n")) {
		804	ret = 1;
		805	goto out;
		806	}
		807
		808	/*
		809	* Sanity check that entry processing is not going backwards.
		810	* This will happen only if hypervisor is tricking us.
		811	*/
		812	if (WARN(data->hdr.end_entry > end_entry \|\| cur_entry > data->hdr.cur_entry,
		813	"SNP: PSC processing going backward, end_entry %d (got %d) cur_entry %d (got %d)\n",
		814	end_entry, data->hdr.end_entry, cur_entry, data->hdr.cur_entry)) {
		815	ret = 1;
		816	goto out;
		817	}
		818	}
		819
		820	out:
		821	__sev_put_ghcb(&state);
		822
		823	out_unlock:
		824	local_irq_restore(flags);
		825
		826	return ret;
		827	}
		828
		829	static void __set_pages_state(struct snp_psc_desc *data, unsigned long vaddr,
		830	unsigned long vaddr_end, int op)
		831	{
		832	struct psc_hdr *hdr;
		833	struct psc_entry *e;
		834	unsigned long pfn;
		835	int i;
		836
		837	hdr = &data->hdr;
		838	e = data->entries;
		839
		840	memset(data, 0, sizeof(*data));
		841	i = 0;
		842
		843	while (vaddr < vaddr_end) {
		844	if (is_vmalloc_addr((void *)vaddr))
		845	pfn = vmalloc_to_pfn((void *)vaddr);
		846	else
		847	pfn = __pa(vaddr) >> PAGE_SHIFT;
		848
		849	e->gfn = pfn;
		850	e->operation = op;
		851	hdr->end_entry = i;
		852
		853	/*
		854	* Current SNP implementation doesn't keep track of the RMP page
		855	* size so use 4K for simplicity.
		856	*/
		857	e->pagesize = RMP_PG_SIZE_4K;
		858
		859	vaddr = vaddr + PAGE_SIZE;
		860	e++;
		861	i++;
		862	}
		863
		864	if (vmgexit_psc(data))
		865	sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PSC);
		866	}
		867
		868	static void set_pages_state(unsigned long vaddr, unsigned int npages, int op)
		869	{
		870	unsigned long vaddr_end, next_vaddr;
		871	struct snp_psc_desc *desc;
		872
		873	desc = kmalloc(sizeof(*desc), GFP_KERNEL_ACCOUNT);
		874	if (!desc)
		875	panic("SNP: failed to allocate memory for PSC descriptor\n");
		876
		877	vaddr = vaddr & PAGE_MASK;
		878	vaddr_end = vaddr + (npages << PAGE_SHIFT);
		879
		880	while (vaddr < vaddr_end) {
		881	/* Calculate the last vaddr that fits in one struct snp_psc_desc. */
		882	next_vaddr = min_t(unsigned long, vaddr_end,
		883	(VMGEXIT_PSC_MAX_ENTRY * PAGE_SIZE) + vaddr);
		884
		885	__set_pages_state(desc, vaddr, next_vaddr, op);
		886
		887	vaddr = next_vaddr;
		888	}
		889
		890	kfree(desc);
		891	}
		892
		893	void snp_set_memory_shared(unsigned long vaddr, unsigned int npages)
		894	{
		895	if (!cc_platform_has(CC_ATTR_GUEST_SEV_SNP))
		896	return;
		897
		898	pvalidate_pages(vaddr, npages, false);
		899
		900	set_pages_state(vaddr, npages, SNP_PAGE_STATE_SHARED);
		901	}
		902
		903	void snp_set_memory_private(unsigned long vaddr, unsigned int npages)
		904	{
		905	if (!cc_platform_has(CC_ATTR_GUEST_SEV_SNP))
		906	return;
		907
		908	set_pages_state(vaddr, npages, SNP_PAGE_STATE_PRIVATE);
		909
		910	pvalidate_pages(vaddr, npages, true);
		911	}
		912
		913	static int snp_set_vmsa(void *va, bool vmsa)
		914	{
		915	u64 attrs;
		916
		917	/*
		918	* Running at VMPL0 allows the kernel to change the VMSA bit for a page
		919	* using the RMPADJUST instruction. However, for the instruction to
		920	* succeed it must target the permissions of a lesser privileged
		921	* (higher numbered) VMPL level, so use VMPL1 (refer to the RMPADJUST
		922	* instruction in the AMD64 APM Volume 3).
		923	*/
		924	attrs = 1;
		925	if (vmsa)
		926	attrs \|= RMPADJUST_VMSA_PAGE_BIT;
		927
		928	return rmpadjust((unsigned long)va, RMP_PG_SIZE_4K, attrs);
		929	}
		930
		931	#define __ATTR_BASE (SVM_SELECTOR_P_MASK \| SVM_SELECTOR_S_MASK)
		932	#define INIT_CS_ATTRIBS (__ATTR_BASE \| SVM_SELECTOR_READ_MASK \| SVM_SELECTOR_CODE_MASK)
		933	#define INIT_DS_ATTRIBS (__ATTR_BASE \| SVM_SELECTOR_WRITE_MASK)
		934
		935	#define INIT_LDTR_ATTRIBS (SVM_SELECTOR_P_MASK \| 2)
		936	#define INIT_TR_ATTRIBS (SVM_SELECTOR_P_MASK \| 3)
		937
		938	static void *snp_alloc_vmsa_page(void)
		939	{
		940	struct page *p;
		941
		942	/*
		943	* Allocate VMSA page to work around the SNP erratum where the CPU will
		944	* incorrectly signal an RMP violation #PF if a large page (2MB or 1GB)
		945	* collides with the RMP entry of VMSA page. The recommended workaround
		946	* is to not use a large page.
		947	*
		948	* Allocate an 8k page which is also 8k-aligned.
		949	*/
		950	p = alloc_pages(GFP_KERNEL_ACCOUNT \| __GFP_ZERO, 1);
		951	if (!p)
		952	return NULL;
		953
		954	split_page(p, 1);
		955
		956	/* Free the first 4k. This page may be 2M/1G aligned and cannot be used. */
		957	__free_page(p);
		958
		959	return page_address(p + 1);
		960	}
		961
		962	static void snp_cleanup_vmsa(struct sev_es_save_area *vmsa)
		963	{
		964	int err;
		965
		966	err = snp_set_vmsa(vmsa, false);
		967	if (err)
		968	pr_err("clear VMSA page failed (%u), leaking page\n", err);
		969	else
		970	free_page((unsigned long)vmsa);
		971	}
		972
		973	static int wakeup_cpu_via_vmgexit(int apic_id, unsigned long start_ip)
		974	{
		975	struct sev_es_save_area cur_vmsa, vmsa;
		976	struct ghcb_state state;
		977	unsigned long flags;
		978	struct ghcb *ghcb;
		979	u8 sipi_vector;
		980	int cpu, ret;
		981	u64 cr4;
		982
		983	/*
		984	* The hypervisor SNP feature support check has happened earlier, just check
		985	* the AP_CREATION one here.
		986	*/
		987	if (!(sev_hv_features & GHCB_HV_FT_SNP_AP_CREATION))
		988	return -EOPNOTSUPP;
		989
		990	/*
		991	* Verify the desired start IP against the known trampoline start IP
		992	* to catch any future new trampolines that may be introduced that
		993	* would require a new protected guest entry point.
		994	*/
		995	if (WARN_ONCE(start_ip != real_mode_header->trampoline_start,
		996	"Unsupported SNP start_ip: %lx\n", start_ip))
		997	return -EINVAL;
		998
		999	/* Override start_ip with known protected guest start IP */
		1000	start_ip = real_mode_header->sev_es_trampoline_start;
		1001
		1002	/* Find the logical CPU for the APIC ID */
		1003	for_each_present_cpu(cpu) {
		1004	if (arch_match_cpu_phys_id(cpu, apic_id))
		1005	break;
		1006	}
		1007	if (cpu >= nr_cpu_ids)
		1008	return -EINVAL;
		1009
		1010	cur_vmsa = per_cpu(sev_vmsa, cpu);
		1011
		1012	/*
		1013	* A new VMSA is created each time because there is no guarantee that
		1014	* the current VMSA is the kernels or that the vCPU is not running. If
		1015	* an attempt was done to use the current VMSA with a running vCPU, a
		1016	* #VMEXIT of that vCPU would wipe out all of the settings being done
		1017	* here.
		1018	*/
		1019	vmsa = (struct sev_es_save_area *)snp_alloc_vmsa_page();
		1020	if (!vmsa)
		1021	return -ENOMEM;
		1022
		1023	/* CR4 should maintain the MCE value */
		1024	cr4 = native_read_cr4() & X86_CR4_MCE;
		1025
		1026	/* Set the CS value based on the start_ip converted to a SIPI vector */
		1027	sipi_vector = (start_ip >> 12);
		1028	vmsa->cs.base = sipi_vector << 12;
		1029	vmsa->cs.limit = AP_INIT_CS_LIMIT;
		1030	vmsa->cs.attrib = INIT_CS_ATTRIBS;
		1031	vmsa->cs.selector = sipi_vector << 8;
		1032
		1033	/* Set the RIP value based on start_ip */
		1034	vmsa->rip = start_ip & 0xfff;
		1035
		1036	/* Set AP INIT defaults as documented in the APM */
		1037	vmsa->ds.limit = AP_INIT_DS_LIMIT;
		1038	vmsa->ds.attrib = INIT_DS_ATTRIBS;
		1039	vmsa->es = vmsa->ds;
		1040	vmsa->fs = vmsa->ds;
		1041	vmsa->gs = vmsa->ds;
		1042	vmsa->ss = vmsa->ds;
		1043
		1044	vmsa->gdtr.limit = AP_INIT_GDTR_LIMIT;
		1045	vmsa->ldtr.limit = AP_INIT_LDTR_LIMIT;
		1046	vmsa->ldtr.attrib = INIT_LDTR_ATTRIBS;
		1047	vmsa->idtr.limit = AP_INIT_IDTR_LIMIT;
		1048	vmsa->tr.limit = AP_INIT_TR_LIMIT;
		1049	vmsa->tr.attrib = INIT_TR_ATTRIBS;
		1050
		1051	vmsa->cr4 = cr4;
		1052	vmsa->cr0 = AP_INIT_CR0_DEFAULT;
		1053	vmsa->dr7 = DR7_RESET_VALUE;
		1054	vmsa->dr6 = AP_INIT_DR6_DEFAULT;
		1055	vmsa->rflags = AP_INIT_RFLAGS_DEFAULT;
		1056	vmsa->g_pat = AP_INIT_GPAT_DEFAULT;
		1057	vmsa->xcr0 = AP_INIT_XCR0_DEFAULT;
		1058	vmsa->mxcsr = AP_INIT_MXCSR_DEFAULT;
		1059	vmsa->x87_ftw = AP_INIT_X87_FTW_DEFAULT;
		1060	vmsa->x87_fcw = AP_INIT_X87_FCW_DEFAULT;
		1061
		1062	/* SVME must be set. */
		1063	vmsa->efer = EFER_SVME;
		1064
		1065	/*
		1066	* Set the SNP-specific fields for this VMSA:
		1067	* VMPL level
		1068	* SEV_FEATURES (matches the SEV STATUS MSR right shifted 2 bits)
		1069	*/
		1070	vmsa->vmpl = 0;
		1071	vmsa->sev_features = sev_status >> 2;
		1072
		1073	/* Switch the page over to a VMSA page now that it is initialized */
		1074	ret = snp_set_vmsa(vmsa, true);
		1075	if (ret) {
		1076	pr_err("set VMSA page failed (%u)\n", ret);
		1077	free_page((unsigned long)vmsa);
		1078
		1079	return -EINVAL;
		1080	}
		1081
		1082	/* Issue VMGEXIT AP Creation NAE event */
		1083	local_irq_save(flags);
		1084
		1085	ghcb = __sev_get_ghcb(&state);
		1086
		1087	vc_ghcb_invalidate(ghcb);
		1088	ghcb_set_rax(ghcb, vmsa->sev_features);
		1089	ghcb_set_sw_exit_code(ghcb, SVM_VMGEXIT_AP_CREATION);
		1090	ghcb_set_sw_exit_info_1(ghcb, ((u64)apic_id << 32) \| SVM_VMGEXIT_AP_CREATE);
		1091	ghcb_set_sw_exit_info_2(ghcb, __pa(vmsa));
		1092
		1093	sev_es_wr_ghcb_msr(__pa(ghcb));
		1094	VMGEXIT();
		1095
		1096	if (!ghcb_sw_exit_info_1_is_valid(ghcb) \|\|
		1097	lower_32_bits(ghcb->save.sw_exit_info_1)) {
		1098	pr_err("SNP AP Creation error\n");
		1099	ret = -EINVAL;
		1100	}
		1101
		1102	__sev_put_ghcb(&state);
		1103
		1104	local_irq_restore(flags);
		1105
		1106	/* Perform cleanup if there was an error */
		1107	if (ret) {
		1108	snp_cleanup_vmsa(vmsa);
		1109	vmsa = NULL;
		1110	}
		1111
		1112	/* Free up any previous VMSA page */
		1113	if (cur_vmsa)
		1114	snp_cleanup_vmsa(cur_vmsa);
		1115
		1116	/* Record the current VMSA page */
		1117	per_cpu(sev_vmsa, cpu) = vmsa;
		1118
		1119	return ret;
		1120	}
		1121
		1122	void snp_set_wakeup_secondary_cpu(void)
		1123	{
		1124	if (!cc_platform_has(CC_ATTR_GUEST_SEV_SNP))
		1125	return;
		1126
		1127	/*
		1128	* Always set this override if SNP is enabled. This makes it the
		1129	* required method to start APs under SNP. If the hypervisor does
		1130	* not support AP creation, then no APs will be started.
		1131	*/
		1132	apic->wakeup_secondary_cpu = wakeup_cpu_via_vmgexit;
		1133	}
		1134
		1135	int __init sev_es_setup_ap_jump_table(struct real_mode_header *rmh)
557	{	1136	{
558	u16 startup_cs, startup_ip;	1137	u16 startup_cs, startup_ip;
559	phys_addr_t jump_table_pa;	1138	phys_addr_t jump_table_pa;
@@ -644,15 +1223,39 @@ static enum es_result vc_handle_msr(struct ghcb ghcb, struct es_em_ctxt ctxt) · 84 unmodified lines
644	return ret;	1223	return ret;
645	}	1224	}
646		1225
647	/*	1226	static void snp_register_per_cpu_ghcb(void)
648	* This function runs on the first #VC exception after the kernel	1227	{
649	* switched to virtual addresses.	1228	struct sev_es_runtime_data *data;
650	*/	1229	struct ghcb *ghcb;
651	static bool __init sev_es_setup_ghcb(void)	1230
		1231	data = this_cpu_read(runtime_data);
		1232	ghcb = &data->ghcb_page;
		1233
		1234	snp_register_ghcb_early(__pa(ghcb));
		1235	}
		1236
		1237	void setup_ghcb(void)
652	{	1238	{
		1239	if (!cc_platform_has(CC_ATTR_GUEST_STATE_ENCRYPT))
		1240	return;
		1241
653	/* First make sure the hypervisor talks a supported protocol. */	1242	/* First make sure the hypervisor talks a supported protocol. */
654	if (!sev_es_negotiate_protocol())	1243	if (!sev_es_negotiate_protocol())
655	return false;	1244	sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_GEN_REQ);
		1245
		1246	/*
		1247	* Check whether the runtime #VC exception handler is active. It uses
		1248	* the per-CPU GHCB page which is set up by sev_es_init_vc_handling().
		1249	*
		1250	* If SNP is active, register the per-CPU GHCB page so that the runtime
		1251	* exception handler can use it.
		1252	*/
		1253	if (initial_vc_handler == (unsigned long)kernel_exc_vmm_communication) {
		1254	if (cc_platform_has(CC_ATTR_GUEST_SEV_SNP))
		1255	snp_register_per_cpu_ghcb();
		1256
		1257	return;
		1258	}
656		1259
657	/*	1260	/*
658	* Clear the boot_ghcb. The first exception comes in before the bss	1261	* Clear the boot_ghcb. The first exception comes in before the bss
@@ -663,7 +1266,9 @@ static bool __init sev_es_setup_ghcb(void) · 4 unmodified lines
663	/* Alright - Make the boot-ghcb public */	1266	/* Alright - Make the boot-ghcb public */
664	boot_ghcb = &boot_ghcb_page;	1267	boot_ghcb = &boot_ghcb_page;
665		1268
666	return true;	1269	/* SNP guest requires that GHCB GPA must be registered. */
		1270	if (cc_platform_has(CC_ATTR_GUEST_SEV_SNP))
		1271	snp_register_ghcb_early(__pa(&boot_ghcb_page));
667	}	1272	}
668		1273
669	#ifdef CONFIG_HOTPLUG_CPU	1274	#ifdef CONFIG_HOTPLUG_CPU
@@ -766,6 +1371,17 @@ void __init sev_es_init_vc_handling(void) · 96 unmodified lines
766	if (!sev_es_check_cpu_features())	1371	if (!sev_es_check_cpu_features())
767	panic("SEV-ES CPU Features missing");	1372	panic("SEV-ES CPU Features missing");
768		1373
		1374	/*
		1375	* SNP is supported in v2 of the GHCB spec which mandates support for HV
		1376	* features.
		1377	*/
		1378	if (cc_platform_has(CC_ATTR_GUEST_SEV_SNP)) {
		1379	sev_hv_features = get_hv_features();
		1380
		1381	if (!(sev_hv_features & GHCB_HV_FT_SNP))
		1382	sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SNP_UNSUPPORTED);
		1383	}
		1384
769	/* Enable SEV-ES special handling */	1385	/* Enable SEV-ES special handling */
770	static_branch_enable(&sev_es_enable_key);	1386	static_branch_enable(&sev_es_enable_key);
771		1387
@@ -1337,7 +1953,7 @@ DEFINE_IDTENTRY_VC_KERNEL(exc_vmm_communication) · 565 unmodified lines
1337	show_regs(regs);	1953	show_regs(regs);
1338		1954
1339	/* Ask hypervisor to sev_es_terminate */	1955	/* Ask hypervisor to sev_es_terminate */
1340	sev_es_terminate(GHCB_SEV_ES_GEN_REQ);	1956	sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_GEN_REQ);
1341		1957
1342	/* If that fails and we get here - just panic */	1958	/* If that fails and we get here - just panic */
1343	panic("Returned from Terminate-Request to Hypervisor\n");	1959	panic("Returned from Terminate-Request to Hypervisor\n");
@@ -1383,10 +1999,6 @@ bool __init handle_vc_boot_ghcb(struct pt_regs *regs) · 39 unmodified lines
1383	struct es_em_ctxt ctxt;	1999	struct es_em_ctxt ctxt;
1384	enum es_result result;	2000	enum es_result result;
1385		2001
1386	/* Do initial setup or terminate the guest */
1387	if (unlikely(boot_ghcb == NULL && !sev_es_setup_ghcb()))
1388	sev_es_terminate(GHCB_SEV_ES_GEN_REQ);
1389
1390	vc_ghcb_invalidate(boot_ghcb);	2002	vc_ghcb_invalidate(boot_ghcb);
1391		2003
1392	result = vc_init_em_ctxt(&ctxt, regs, exit_code);	2004	result = vc_init_em_ctxt(&ctxt, regs, exit_code);
@@ -1425,6 +2037,215 @@ bool __init handle_vc_boot_ghcb(struct pt_regs *regs) · 32 unmodified lines
1425	fail:	2037	fail:
1426	show_regs(regs);	2038	show_regs(regs);
1427		2039
1428	while (true)	2040	sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_GEN_REQ);
1429	halt();	2041	}
		2042
		2043	/*
		2044	* Initial set up of SNP relies on information provided by the
		2045	* Confidential Computing blob, which can be passed to the kernel
		2046	* in the following ways, depending on how it is booted:
		2047	*
		2048	* - when booted via the boot/decompress kernel:
		2049	* - via boot_params
		2050	*
		2051	* - when booted directly by firmware/bootloader (e.g. CONFIG_PVH):
		2052	* - via a setup_data entry, as defined by the Linux Boot Protocol
		2053	*
		2054	* Scan for the blob in that order.
		2055	*/
		2056	static __init struct cc_blob_sev_info find_cc_blob(struct boot_params bp)
		2057	{
		2058	struct cc_blob_sev_info *cc_info;
		2059
		2060	/* Boot kernel would have passed the CC blob via boot_params. */
		2061	if (bp->cc_blob_address) {
		2062	cc_info = (struct cc_blob_sev_info *)(unsigned long)bp->cc_blob_address;
		2063	goto found_cc_info;
		2064	}
		2065
		2066	/*
		2067	* If kernel was booted directly, without the use of the
		2068	* boot/decompression kernel, the CC blob may have been passed via
		2069	* setup_data instead.
		2070	*/
		2071	cc_info = find_cc_blob_setup_data(bp);
		2072	if (!cc_info)
		2073	return NULL;
		2074
		2075	found_cc_info:
		2076	if (cc_info->magic != CC_BLOB_SEV_HDR_MAGIC)
		2077	snp_abort();
		2078
		2079	return cc_info;
		2080	}
		2081
		2082	bool __init snp_init(struct boot_params *bp)
		2083	{
		2084	struct cc_blob_sev_info *cc_info;
		2085
		2086	if (!bp)
		2087	return false;
		2088
		2089	cc_info = find_cc_blob(bp);
		2090	if (!cc_info)
		2091	return false;
		2092
		2093	setup_cpuid_table(cc_info);
		2094
		2095	/*
		2096	* The CC blob will be used later to access the secrets page. Cache
		2097	* it here like the boot kernel does.
		2098	*/
		2099	bp->cc_blob_address = (u32)(unsigned long)cc_info;
		2100
		2101	return true;
		2102	}
		2103
		2104	void __init snp_abort(void)
		2105	{
		2106	sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SNP_UNSUPPORTED);
		2107	}
		2108
		2109	static void dump_cpuid_table(void)
		2110	{
		2111	const struct snp_cpuid_table *cpuid_table = snp_cpuid_get_table();
		2112	int i = 0;
		2113
		2114	pr_info("count=%d reserved=0x%x reserved2=0x%llx\n",
		2115	cpuid_table->count, cpuid_table->__reserved1, cpuid_table->__reserved2);
		2116
		2117	for (i = 0; i < SNP_CPUID_COUNT_MAX; i++) {
		2118	const struct snp_cpuid_fn *fn = &cpuid_table->fn[i];
		2119
		2120	pr_info("index=%3d fn=0x%08x subfn=0x%08x: eax=0x%08x ebx=0x%08x ecx=0x%08x edx=0x%08x xcr0_in=0x%016llx xss_in=0x%016llx reserved=0x%016llx\n",
		2121	i, fn->eax_in, fn->ecx_in, fn->eax, fn->ebx, fn->ecx,
		2122	fn->edx, fn->xcr0_in, fn->xss_in, fn->__reserved);
		2123	}
		2124	}
		2125
		2126	/*
		2127	* It is useful from an auditing/testing perspective to provide an easy way
		2128	* for the guest owner to know that the CPUID table has been initialized as
		2129	* expected, but that initialization happens too early in boot to print any
		2130	* sort of indicator, and there's not really any other good place to do it,
		2131	* so do it here.
		2132	*/
		2133	static int __init report_cpuid_table(void)
		2134	{
		2135	const struct snp_cpuid_table *cpuid_table = snp_cpuid_get_table();
		2136
		2137	if (!cpuid_table->count)
		2138	return 0;
		2139
		2140	pr_info("Using SNP CPUID table, %d entries present.\n",
		2141	cpuid_table->count);
		2142
		2143	if (sev_cfg.debug)
		2144	dump_cpuid_table();
		2145
		2146	return 0;
		2147	}
		2148	arch_initcall(report_cpuid_table);
		2149
		2150	static int __init init_sev_config(char *str)
		2151	{
		2152	char *s;
		2153
		2154	while ((s = strsep(&str, ","))) {
		2155	if (!strcmp(s, "debug")) {
		2156	sev_cfg.debug = true;
		2157	continue;
		2158	}
		2159
		2160	pr_info("SEV command-line option '%s' was not recognized\n", s);
		2161	}
		2162
		2163	return 1;
		2164	}
		2165	__setup("sev=", init_sev_config);
		2166
		2167	int snp_issue_guest_request(u64 exit_code, struct snp_req_data input, unsigned long fw_err)
		2168	{
		2169	struct ghcb_state state;
		2170	struct es_em_ctxt ctxt;
		2171	unsigned long flags;
		2172	struct ghcb *ghcb;
		2173	int ret;
		2174
		2175	if (!cc_platform_has(CC_ATTR_GUEST_SEV_SNP))
		2176	return -ENODEV;
		2177
		2178	if (!fw_err)
		2179	return -EINVAL;
		2180
		2181	/*
		2182	* __sev_get_ghcb() needs to run with IRQs disabled because it is using
		2183	* a per-CPU GHCB.
		2184	*/
		2185	local_irq_save(flags);
		2186
		2187	ghcb = __sev_get_ghcb(&state);
		2188	if (!ghcb) {
		2189	ret = -EIO;
		2190	goto e_restore_irq;
		2191	}
		2192
		2193	vc_ghcb_invalidate(ghcb);
		2194
		2195	if (exit_code == SVM_VMGEXIT_EXT_GUEST_REQUEST) {
		2196	ghcb_set_rax(ghcb, input->data_gpa);
		2197	ghcb_set_rbx(ghcb, input->data_npages);
		2198	}
		2199
		2200	ret = sev_es_ghcb_hv_call(ghcb, true, &ctxt, exit_code, input->req_gpa, input->resp_gpa);
		2201	if (ret)
		2202	goto e_put;
		2203
		2204	if (ghcb->save.sw_exit_info_2) {
		2205	/* Number of expected pages are returned in RBX */
		2206	if (exit_code == SVM_VMGEXIT_EXT_GUEST_REQUEST &&
		2207	ghcb->save.sw_exit_info_2 == SNP_GUEST_REQ_INVALID_LEN)
		2208	input->data_npages = ghcb_get_rbx(ghcb);
		2209
		2210	*fw_err = ghcb->save.sw_exit_info_2;
		2211
		2212	ret = -EIO;
		2213	}
		2214
		2215	e_put:
		2216	__sev_put_ghcb(&state);
		2217	e_restore_irq:
		2218	local_irq_restore(flags);
		2219
		2220	return ret;
		2221	}
		2222	EXPORT_SYMBOL_GPL(snp_issue_guest_request);
		2223
		2224	static struct platform_device sev_guest_device = {
		2225	.name = "sev-guest",
		2226	.id = -1,
		2227	};
		2228
		2229	static int __init snp_init_platform_device(void)
		2230	{
		2231	struct sev_guest_platform_data data;
		2232	u64 gpa;
		2233
		2234	if (!cc_platform_has(CC_ATTR_GUEST_SEV_SNP))
		2235	return -ENODEV;
		2236
		2237	gpa = get_secrets_page();
		2238	if (!gpa)
		2239	return -ENODEV;
		2240
		2241	data.secrets_gpa = gpa;
		2242	if (platform_device_add_data(&sev_guest_device, &data, sizeof(data)))
		2243	return -ENODEV;
		2244
		2245	if (platform_device_register(&sev_guest_device))
		2246	return -ENODEV;
		2247
		2248	pr_info("SNP guest platform device initialized.\n");
		2249	return 0;
1430	}	2250	}
		2251	device_initcall(snp_init_platform_device);

arch/x86/kernel/smpboot.c+3−0

 #include <asm/spec-ctrl.h>
 #include <asm/hw_irq.h>
 #include <asm/stackprotector.h>
+#include <asm/sev.h>
 /* representing HT siblings of each logical CPU */
 DEFINE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_sibling_map);
 	smp_quirk_init_udelay();
 	speculative_store_bypass_ht_init();
+	snp_set_wakeup_secondary_cpu();
 }
 void arch_thaw_secondary_cpus_begin(void)

arch/x86/kvm/cpuid.c+2−17

@@ -19,6 +19,7 @@ · 18 unmodified lines
19	#include <asm/user.h>	19	#include <asm/user.h>
20	#include <asm/fpu/xstate.h>	20	#include <asm/fpu/xstate.h>
21	#include <asm/sgx.h>	21	#include <asm/sgx.h>
		22	#include <asm/cpuid.h>
22	#include "cpuid.h"	23	#include "cpuid.h"
23	#include "lapic.h"	24	#include "lapic.h"
24	#include "mmu.h"	25	#include "mmu.h"
@@ -744,24 +745,8 @@ static struct kvm_cpuid_entry2 do_host_cpuid(struct kvm_cpuid_array array, · 719 unmodified lines
744	cpuid_count(entry->function, entry->index,	745	cpuid_count(entry->function, entry->index,
745	&entry->eax, &entry->ebx, &entry->ecx, &entry->edx);	746	&entry->eax, &entry->ebx, &entry->ecx, &entry->edx);
746		747
747	switch (function) {	748	if (cpuid_function_is_indexed(function))
748	case 4:
749	case 7:
750	case 0xb:
751	case 0xd:
752	case 0xf:
753	case 0x10:
754	case 0x12:
755	case 0x14:
756	case 0x17:
757	case 0x18:
758	case 0x1d:
759	case 0x1e:
760	case 0x1f:
761	case 0x8000001d:
762	entry->flags \|= KVM_CPUID_FLAG_SIGNIFCANT_INDEX;	749	entry->flags \|= KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
763	break;
764	}
765		750
766	return entry;	751	return entry;
767	}	752	}

arch/x86/kvm/svm/sev.c+11−11

 static int sev_es_sync_vmsa(struct vcpu_svm *svm)
 {
-	struct vmcb_save_area *save = &svm->vmcb->save;
+	struct sev_es_save_area *save = svm->sev_es.vmsa;
 	/* Check some debug related fields before encrypting the VMSA */
-	if (svm->vcpu.guest_debug || (save->dr7 & ~DR7_FIXED_1))
+	if (svm->vcpu.guest_debug || (svm->vmcb->save.dr7 & ~DR7_FIXED_1))
 		return -EINVAL;
+	/*
+	 * SEV-ES will use a VMSA that is pointed to by the VMCB, not
+	 * the traditional VMSA that is part of the VMCB. Copy the
+	 * traditional VMSA as it has been built so far (in prep
+	 * for LAUNCH_UPDATE_VMSA) to be the initial SEV-ES state.
+	 */
+	memcpy(save, &svm->vmcb->save, sizeof(svm->vmcb->save));
 	/* Sync registgers */
 	save->rax = svm->vcpu.arch.regs[VCPU_REGS_RAX];
 	save->rbx = svm->vcpu.arch.regs[VCPU_REGS_RBX];
 	save->xss  = svm->vcpu.arch.ia32_xss;
 	save->dr6  = svm->vcpu.arch.dr6;
-	/*
-	 * SEV-ES will use a VMSA that is pointed to by the VMCB, not
-	 * the traditional VMSA that is part of the VMCB. Copy the
-	 * traditional VMSA as it has been built so far (in prep
-	 * for LAUNCH_UPDATE_VMSA) to be the initial SEV-ES state.
-	 */
-	memcpy(svm->sev_es.vmsa, save, sizeof(*save));
 	return 0;
 }
 					    sev_enc_bit));
 }
-void sev_es_prepare_switch_to_guest(struct vmcb_save_area *hostsa)
+void sev_es_prepare_switch_to_guest(struct sev_es_save_area *hostsa)
 {
 	/*
 	 * As an SEV-ES guest, hardware will restore the host state on VMEXIT,

arch/x86/kvm/svm/svm.c+4−4

arch/x86/kvm/svm/svm.h+2−2

@@ -181,7 +181,7 @@ struct svm_nested_state { · 180 unmodified lines
181		181
182	struct vcpu_sev_es_state {	182	struct vcpu_sev_es_state {
183	/* SEV-ES support */	183	/* SEV-ES support */
184	struct vmcb_save_area *vmsa;	184	struct sev_es_save_area *vmsa;
185	struct ghcb *ghcb;	185	struct ghcb *ghcb;
186	struct kvm_host_map ghcb_map;	186	struct kvm_host_map ghcb_map;
187	bool received_first_sipi;	187	bool received_first_sipi;
@@ -622,7 +622,7 @@ int sev_es_string_io(struct vcpu_svm *svm, int size, unsigned int port, int in); · 434 unmodified lines
622	void sev_es_init_vmcb(struct vcpu_svm *svm);	622	void sev_es_init_vmcb(struct vcpu_svm *svm);
623	void sev_es_vcpu_reset(struct vcpu_svm *svm);	623	void sev_es_vcpu_reset(struct vcpu_svm *svm);
624	void sev_vcpu_deliver_sipi_vector(struct kvm_vcpu *vcpu, u8 vector);	624	void sev_vcpu_deliver_sipi_vector(struct kvm_vcpu *vcpu, u8 vector);
625	void sev_es_prepare_switch_to_guest(struct vmcb_save_area *hostsa);	625	void sev_es_prepare_switch_to_guest(struct sev_es_save_area *hostsa);
626	void sev_es_unmap_ghcb(struct vcpu_svm *svm);	626	void sev_es_unmap_ghcb(struct vcpu_svm *svm);
627		627
628	/* vmenter.S */	628	/* vmenter.S */

arch/x86/mm/mem_encrypt.c+4−0

@@ -62,6 +62,10 @@ static void print_mem_encrypt_feature_info(void) · 61 unmodified lines
62	if (cc_platform_has(CC_ATTR_GUEST_STATE_ENCRYPT))	62	if (cc_platform_has(CC_ATTR_GUEST_STATE_ENCRYPT))
63	pr_cont(" SEV-ES");	63	pr_cont(" SEV-ES");
64		64
		65	/* Secure Nested Paging */
		66	if (cc_platform_has(CC_ATTR_GUEST_SEV_SNP))
		67	pr_cont(" SEV-SNP");
		68
65	pr_cont("\n");	69	pr_cont("\n");
66	}	70	}
67		71

arch/x86/mm/mem_encrypt_amd.c+67−4

@@ -31,6 +31,7 @@ · 30 unmodified lines
31	#include <asm/processor-flags.h>	31	#include <asm/processor-flags.h>
32	#include <asm/msr.h>	32	#include <asm/msr.h>
33	#include <asm/cmdline.h>	33	#include <asm/cmdline.h>
		34	#include <asm/sev.h>
34		35
35	#include "mm_internal.h"	36	#include "mm_internal.h"
36		37
@@ -47,6 +48,36 @@ EXPORT_SYMBOL(sme_me_mask); · 10 unmodified lines
47	/* Buffer used for early in-place encryption by BSP, no locking needed */	48	/* Buffer used for early in-place encryption by BSP, no locking needed */
48	static char sme_early_buffer[PAGE_SIZE] __initdata __aligned(PAGE_SIZE);	49	static char sme_early_buffer[PAGE_SIZE] __initdata __aligned(PAGE_SIZE);
49		50
		51	/*
		52	* SNP-specific routine which needs to additionally change the page state from
		53	* private to shared before copying the data from the source to destination and
		54	* restore after the copy.
		55	*/
		56	static inline void __init snp_memcpy(void dst, void src, size_t sz,
		57	unsigned long paddr, bool decrypt)
		58	{
		59	unsigned long npages = PAGE_ALIGN(sz) >> PAGE_SHIFT;
		60
		61	if (decrypt) {
		62	/*
		63	* @paddr needs to be accessed decrypted, mark the page shared in
		64	* the RMP table before copying it.
		65	*/
		66	early_snp_set_memory_shared((unsigned long)__va(paddr), paddr, npages);
		67
		68	memcpy(dst, src, sz);
		69
		70	/* Restore the page state after the memcpy. */
		71	early_snp_set_memory_private((unsigned long)__va(paddr), paddr, npages);
		72	} else {
		73	/*
		74	* @paddr need to be accessed encrypted, no need for the page state
		75	* change.
		76	*/
		77	memcpy(dst, src, sz);
		78	}
		79	}
		80
50	/*	81	/*
51	* This routine does not change the underlying encryption setting of the	82	* This routine does not change the underlying encryption setting of the
52	* page(s) that map this memory. It assumes that eventually the memory is	83	* page(s) that map this memory. It assumes that eventually the memory is
@@ -95,8 +126,13 @@ static void __init __sme_early_enc_dec(resource_size_t paddr, · 42 unmodified lines
95	* Use a temporary buffer, of cache-line multiple size, to	126	* Use a temporary buffer, of cache-line multiple size, to
96	* avoid data corruption as documented in the APM.	127	* avoid data corruption as documented in the APM.
97	*/	128	*/
98	memcpy(sme_early_buffer, src, len);	129	if (cc_platform_has(CC_ATTR_GUEST_SEV_SNP)) {
99	memcpy(dst, sme_early_buffer, len);	130	snp_memcpy(sme_early_buffer, src, len, paddr, enc);
		131	snp_memcpy(dst, sme_early_buffer, len, paddr, !enc);
		132	} else {
		133	memcpy(sme_early_buffer, src, len);
		134	memcpy(dst, sme_early_buffer, len);
		135	}
100		136
101	early_memunmap(dst, len);	137	early_memunmap(dst, len);
102	early_memunmap(src, len);	138	early_memunmap(src, len);
@@ -280,11 +316,24 @@ static void enc_dec_hypercall(unsigned long vaddr, int npages, bool enc) · 177 unmodified lines
280		316
281	static void amd_enc_status_change_prepare(unsigned long vaddr, int npages, bool enc)	317	static void amd_enc_status_change_prepare(unsigned long vaddr, int npages, bool enc)
282	{	318	{
		319	/*
		320	* To maintain the security guarantees of SEV-SNP guests, make sure
		321	* to invalidate the memory before encryption attribute is cleared.
		322	*/
		323	if (cc_platform_has(CC_ATTR_GUEST_SEV_SNP) && !enc)
		324	snp_set_memory_shared(vaddr, npages);
283	}	325	}
284		326
285	/* Return true unconditionally: return value doesn't matter for the SEV side */	327	/* Return true unconditionally: return value doesn't matter for the SEV side */
286	static bool amd_enc_status_change_finish(unsigned long vaddr, int npages, bool enc)	328	static bool amd_enc_status_change_finish(unsigned long vaddr, int npages, bool enc)
287	{	329	{
		330	/*
		331	* After memory is mapped encrypted in the page table, validate it
		332	* so that it is consistent with the page table updates.
		333	*/
		334	if (cc_platform_has(CC_ATTR_GUEST_SEV_SNP) && enc)
		335	snp_set_memory_private(vaddr, npages);
		336
288	if (!cc_platform_has(CC_ATTR_HOST_MEM_ENCRYPT))	337	if (!cc_platform_has(CC_ATTR_HOST_MEM_ENCRYPT))
289	enc_dec_hypercall(vaddr, npages, enc);	338	enc_dec_hypercall(vaddr, npages, enc);
290		339
@@ -322,14 +371,28 @@ static void __init __set_clr_pte_enc(pte_t *kpte, int level, bool enc) · 31 unmodified lines
322	clflush_cache_range(__va(pa), size);	371	clflush_cache_range(__va(pa), size);
323		372
324	/* Encrypt/decrypt the contents in-place */	373	/* Encrypt/decrypt the contents in-place */
325	if (enc)	374	if (enc) {
326	sme_early_encrypt(pa, size);	375	sme_early_encrypt(pa, size);
327	else	376	} else {
328	sme_early_decrypt(pa, size);	377	sme_early_decrypt(pa, size);
329		378
		379	/*
		380	* ON SNP, the page state in the RMP table must happen
		381	* before the page table updates.
		382	*/
		383	early_snp_set_memory_shared((unsigned long)__va(pa), pa, 1);
		384	}
		385
330	/* Change the page encryption mask. */	386	/* Change the page encryption mask. */
331	new_pte = pfn_pte(pfn, new_prot);	387	new_pte = pfn_pte(pfn, new_prot);
332	set_pte_atomic(kpte, new_pte);	388	set_pte_atomic(kpte, new_pte);
		389
		390	/*
		391	* If page is set encrypted in the page table, then update the RMP table to
		392	* add this page as private.
		393	*/
		394	if (enc)
		395	early_snp_set_memory_private((unsigned long)__va(pa), pa, 1);
333	}	396	}
334		397
335	static int __init early_set_memory_enc_dec(unsigned long vaddr,	398	static int __init early_set_memory_enc_dec(unsigned long vaddr,

arch/x86/mm/mem_encrypt_identity.c+8−0

 #include <asm/sections.h>
 #include <asm/cmdline.h>
 #include <asm/coco.h>
+#include <asm/sev.h>
 #include "mm_internal.h"
 	bool active_by_default;
 	unsigned long me_mask;
 	char buffer[16];
+	bool snp;
 	u64 msr;
+	snp = snp_init(bp);
 	/* Check for the SME/SEV support leaf */
 	eax = 0x80000000;
 	ecx = 0;
 	sev_status   = __rdmsr(MSR_AMD64_SEV);
 	feature_mask = (sev_status & MSR_AMD64_SEV_ENABLED) ? AMD_SEV_BIT : AMD_SME_BIT;
+	/* The SEV-SNP CC blob should never be present unless SEV-SNP is enabled. */
+	if (snp && !(sev_status & MSR_AMD64_SEV_SNP_ENABLED))
+		snp_abort();
 	/* Check if memory encryption is enabled */
 	if (feature_mask == AMD_SME_BIT) {
 		/*

arch/x86/realmode/init.c+1−1

@@ -67,7 +67,7 @@ void __init reserve_real_mode(void) · 66 unmodified lines
67	memblock_reserve(0, SZ_1M);	67	memblock_reserve(0, SZ_1M);
68	}	68	}
69		69
70	static void sme_sev_setup_real_mode(struct trampoline_header *th)	70	static void __init sme_sev_setup_real_mode(struct trampoline_header *th)
71	{	71	{
72	#ifdef CONFIG_AMD_MEM_ENCRYPT	72	#ifdef CONFIG_AMD_MEM_ENCRYPT
73	if (cc_platform_has(CC_ATTR_HOST_MEM_ENCRYPT))	73	if (cc_platform_has(CC_ATTR_HOST_MEM_ENCRYPT))

drivers/virt/Kconfig+2−0

@@ -50,4 +50,6 @@ source "drivers/virt/acrn/Kconfig" · 49 unmodified lines
50		50
51	source "drivers/virt/coco/efi_secret/Kconfig"	51	source "drivers/virt/coco/efi_secret/Kconfig"
52		52
		53	source "drivers/virt/coco/sev-guest/Kconfig"
		54
53	endif	55	endif

drivers/virt/Makefile+1−0

@@ -10,3 +10,4 @@ obj-y += vboxguest/ · 9 unmodified lines
10	obj-$(CONFIG_NITRO_ENCLAVES) += nitro_enclaves/	10	obj-$(CONFIG_NITRO_ENCLAVES) += nitro_enclaves/
11	obj-$(CONFIG_ACRN_HSM) += acrn/	11	obj-$(CONFIG_ACRN_HSM) += acrn/
12	obj-$(CONFIG_EFI_SECRET) += coco/efi_secret/	12	obj-$(CONFIG_EFI_SECRET) += coco/efi_secret/
		13	obj-$(CONFIG_SEV_GUEST) += coco/sev-guest/

drivers/virt/coco/sev-guest/Kconfig+14−0

@@ -0,0 +1,14 @@
	1	config SEV_GUEST
	2	tristate "AMD SEV Guest driver"
	3	default m
	4	depends on AMD_MEM_ENCRYPT
	5	select CRYPTO_AEAD2
	6	select CRYPTO_GCM
	7	help
	8	SEV-SNP firmware provides the guest a mechanism to communicate with
	9	the PSP without risk from a malicious hypervisor who wishes to read,
	10	alter, drop or replay the messages sent. The driver provides
	11	userspace interface to communicate with the PSP to request the
	12	attestation report and more.
	13
	14	If you choose 'M' here, this module will be called sev-guest.

drivers/virt/coco/sev-guest/Makefile+2−0

@@ -0,0 +1,2 @@
	1	# SPDX-License-Identifier: GPL-2.0-only
	2	obj-$(CONFIG_SEV_GUEST) += sev-guest.o

drivers/virt/coco/sev-guest/sev-guest.c+743−0

@@ -0,0 +1,743 @@
	1	// SPDX-License-Identifier: GPL-2.0-only
	2	/*
	3	* AMD Secure Encrypted Virtualization (SEV) guest driver interface
	4	*
	5	* Copyright (C) 2021 Advanced Micro Devices, Inc.
	6	*
	7	* Author: Brijesh Singh <brijesh.singh@amd.com>
	8	*/
	9
	10	#include <linux/module.h>
	11	#include <linux/kernel.h>
	12	#include <linux/types.h>
	13	#include <linux/mutex.h>
	14	#include <linux/io.h>
	15	#include <linux/platform_device.h>
	16	#include <linux/miscdevice.h>
	17	#include <linux/set_memory.h>
	18	#include <linux/fs.h>
	19	#include <crypto/aead.h>
	20	#include <linux/scatterlist.h>
	21	#include <linux/psp-sev.h>
	22	#include <uapi/linux/sev-guest.h>
	23	#include <uapi/linux/psp-sev.h>
	24
	25	#include <asm/svm.h>
	26	#include <asm/sev.h>
	27
	28	#include "sev-guest.h"
	29
	30	#define DEVICE_NAME "sev-guest"
	31	#define AAD_LEN 48
	32	#define MSG_HDR_VER 1
	33
	34	struct snp_guest_crypto {
	35	struct crypto_aead *tfm;
	36	u8 iv, authtag;
	37	int iv_len, a_len;
	38	};
	39
	40	struct snp_guest_dev {
	41	struct device *dev;
	42	struct miscdevice misc;
	43
	44	void *certs_data;
	45	struct snp_guest_crypto *crypto;
	46	struct snp_guest_msg request, response;
	47	struct snp_secrets_page_layout *layout;
	48	struct snp_req_data input;
	49	u32 *os_area_msg_seqno;
	50	u8 *vmpck;
	51	};
	52
	53	static u32 vmpck_id;
	54	module_param(vmpck_id, uint, 0444);
	55	MODULE_PARM_DESC(vmpck_id, "The VMPCK ID to use when communicating with the PSP.");
	56
	57	/* Mutex to serialize the shared buffer access and command handling. */
	58	static DEFINE_MUTEX(snp_cmd_mutex);
	59
	60	static bool is_vmpck_empty(struct snp_guest_dev *snp_dev)
	61	{
	62	char zero_key[VMPCK_KEY_LEN] = {0};
	63
	64	if (snp_dev->vmpck)
	65	return !memcmp(snp_dev->vmpck, zero_key, VMPCK_KEY_LEN);
	66
	67	return true;
	68	}
	69
	70	static void snp_disable_vmpck(struct snp_guest_dev *snp_dev)
	71	{
	72	memzero_explicit(snp_dev->vmpck, VMPCK_KEY_LEN);
	73	snp_dev->vmpck = NULL;
	74	}
	75
	76	static inline u64 __snp_get_msg_seqno(struct snp_guest_dev *snp_dev)
	77	{
	78	u64 count;
	79
	80	lockdep_assert_held(&snp_cmd_mutex);
	81
	82	/* Read the current message sequence counter from secrets pages */
	83	count = *snp_dev->os_area_msg_seqno;
	84
	85	return count + 1;
	86	}
	87
	88	/* Return a non-zero on success */
	89	static u64 snp_get_msg_seqno(struct snp_guest_dev *snp_dev)
	90	{
	91	u64 count = __snp_get_msg_seqno(snp_dev);
	92
	93	/*
	94	* The message sequence counter for the SNP guest request is a 64-bit
	95	* value but the version 2 of GHCB specification defines a 32-bit storage
	96	* for it. If the counter exceeds the 32-bit value then return zero.
	97	* The caller should check the return value, but if the caller happens to
	98	* not check the value and use it, then the firmware treats zero as an
	99	* invalid number and will fail the message request.
	100	*/
	101	if (count >= UINT_MAX) {
	102	dev_err(snp_dev->dev, "request message sequence counter overflow\n");
	103	return 0;
	104	}
	105
	106	return count;
	107	}
	108
	109	static void snp_inc_msg_seqno(struct snp_guest_dev *snp_dev)
	110	{
	111	/*
	112	* The counter is also incremented by the PSP, so increment it by 2
	113	* and save in secrets page.
	114	*/
	115	*snp_dev->os_area_msg_seqno += 2;
	116	}
	117
	118	static inline struct snp_guest_dev to_snp_dev(struct file file)
	119	{
	120	struct miscdevice *dev = file->private_data;
	121
	122	return container_of(dev, struct snp_guest_dev, misc);
	123	}
	124
	125	static struct snp_guest_crypto init_crypto(struct snp_guest_dev snp_dev, u8 *key, size_t keylen)
	126	{
	127	struct snp_guest_crypto *crypto;
	128
	129	crypto = kzalloc(sizeof(*crypto), GFP_KERNEL_ACCOUNT);
	130	if (!crypto)
	131	return NULL;
	132
	133	crypto->tfm = crypto_alloc_aead("gcm(aes)", 0, 0);
	134	if (IS_ERR(crypto->tfm))
	135	goto e_free;
	136
	137	if (crypto_aead_setkey(crypto->tfm, key, keylen))
	138	goto e_free_crypto;
	139
	140	crypto->iv_len = crypto_aead_ivsize(crypto->tfm);
	141	crypto->iv = kmalloc(crypto->iv_len, GFP_KERNEL_ACCOUNT);
	142	if (!crypto->iv)
	143	goto e_free_crypto;
	144
	145	if (crypto_aead_authsize(crypto->tfm) > MAX_AUTHTAG_LEN) {
	146	if (crypto_aead_setauthsize(crypto->tfm, MAX_AUTHTAG_LEN)) {
	147	dev_err(snp_dev->dev, "failed to set authsize to %d\n", MAX_AUTHTAG_LEN);
	148	goto e_free_iv;
	149	}
	150	}
	151
	152	crypto->a_len = crypto_aead_authsize(crypto->tfm);
	153	crypto->authtag = kmalloc(crypto->a_len, GFP_KERNEL_ACCOUNT);
	154	if (!crypto->authtag)
	155	goto e_free_auth;
	156
	157	return crypto;
	158
	159	e_free_auth:
	160	kfree(crypto->authtag);
	161	e_free_iv:
	162	kfree(crypto->iv);
	163	e_free_crypto:
	164	crypto_free_aead(crypto->tfm);
	165	e_free:
	166	kfree(crypto);
	167
	168	return NULL;
	169	}
	170
	171	static void deinit_crypto(struct snp_guest_crypto *crypto)
	172	{
	173	crypto_free_aead(crypto->tfm);
	174	kfree(crypto->iv);
	175	kfree(crypto->authtag);
	176	kfree(crypto);
	177	}
	178
	179	static int enc_dec_message(struct snp_guest_crypto crypto, struct snp_guest_msg msg,
	180	u8 src_buf, u8 dst_buf, size_t len, bool enc)
	181	{
	182	struct snp_guest_msg_hdr *hdr = &msg->hdr;
	183	struct scatterlist src[3], dst[3];
	184	DECLARE_CRYPTO_WAIT(wait);
	185	struct aead_request *req;
	186	int ret;
	187
	188	req = aead_request_alloc(crypto->tfm, GFP_KERNEL);
	189	if (!req)
	190	return -ENOMEM;
	191
	192	/*
	193	* AEAD memory operations:
	194	* +------ AAD -------+------- DATA -----+---- AUTHTAG----+
	195	* \| msg header \| plaintext \| hdr->authtag \|
	196	* \| bytes 30h - 5Fh \| or \| \|
	197	* \| \| cipher \| \|
	198	* +------------------+------------------+----------------+
	199	*/
	200	sg_init_table(src, 3);
	201	sg_set_buf(&src[0], &hdr->algo, AAD_LEN);
	202	sg_set_buf(&src[1], src_buf, hdr->msg_sz);
	203	sg_set_buf(&src[2], hdr->authtag, crypto->a_len);
	204
	205	sg_init_table(dst, 3);
	206	sg_set_buf(&dst[0], &hdr->algo, AAD_LEN);
	207	sg_set_buf(&dst[1], dst_buf, hdr->msg_sz);
	208	sg_set_buf(&dst[2], hdr->authtag, crypto->a_len);
	209
	210	aead_request_set_ad(req, AAD_LEN);
	211	aead_request_set_tfm(req, crypto->tfm);
	212	aead_request_set_callback(req, 0, crypto_req_done, &wait);
	213
	214	aead_request_set_crypt(req, src, dst, len, crypto->iv);
	215	ret = crypto_wait_req(enc ? crypto_aead_encrypt(req) : crypto_aead_decrypt(req), &wait);
	216
	217	aead_request_free(req);
	218	return ret;
	219	}
	220
	221	static int __enc_payload(struct snp_guest_dev snp_dev, struct snp_guest_msg msg,
	222	void *plaintext, size_t len)
	223	{
	224	struct snp_guest_crypto *crypto = snp_dev->crypto;
	225	struct snp_guest_msg_hdr *hdr = &msg->hdr;
	226
	227	memset(crypto->iv, 0, crypto->iv_len);
	228	memcpy(crypto->iv, &hdr->msg_seqno, sizeof(hdr->msg_seqno));
	229
	230	return enc_dec_message(crypto, msg, plaintext, msg->payload, len, true);
	231	}
	232
	233	static int dec_payload(struct snp_guest_dev snp_dev, struct snp_guest_msg msg,
	234	void *plaintext, size_t len)
	235	{
	236	struct snp_guest_crypto *crypto = snp_dev->crypto;
	237	struct snp_guest_msg_hdr *hdr = &msg->hdr;
	238
	239	/* Build IV with response buffer sequence number */
	240	memset(crypto->iv, 0, crypto->iv_len);
	241	memcpy(crypto->iv, &hdr->msg_seqno, sizeof(hdr->msg_seqno));
	242
	243	return enc_dec_message(crypto, msg, msg->payload, plaintext, len, false);
	244	}
	245
	246	static int verify_and_dec_payload(struct snp_guest_dev snp_dev, void payload, u32 sz)
	247	{
	248	struct snp_guest_crypto *crypto = snp_dev->crypto;
	249	struct snp_guest_msg *resp = snp_dev->response;
	250	struct snp_guest_msg *req = snp_dev->request;
	251	struct snp_guest_msg_hdr *req_hdr = &req->hdr;
	252	struct snp_guest_msg_hdr *resp_hdr = &resp->hdr;
	253
	254	dev_dbg(snp_dev->dev, "response [seqno %lld type %d version %d sz %d]\n",
	255	resp_hdr->msg_seqno, resp_hdr->msg_type, resp_hdr->msg_version, resp_hdr->msg_sz);
	256
	257	/* Verify that the sequence counter is incremented by 1 */
	258	if (unlikely(resp_hdr->msg_seqno != (req_hdr->msg_seqno + 1)))
	259	return -EBADMSG;
	260
	261	/* Verify response message type and version number. */
	262	if (resp_hdr->msg_type != (req_hdr->msg_type + 1) \|\|
	263	resp_hdr->msg_version != req_hdr->msg_version)
	264	return -EBADMSG;
	265
	266	/*
	267	* If the message size is greater than our buffer length then return
	268	* an error.
	269	*/
	270	if (unlikely((resp_hdr->msg_sz + crypto->a_len) > sz))
	271	return -EBADMSG;
	272
	273	/* Decrypt the payload */
	274	return dec_payload(snp_dev, resp, payload, resp_hdr->msg_sz + crypto->a_len);
	275	}
	276
	277	static int enc_payload(struct snp_guest_dev *snp_dev, u64 seqno, int version, u8 type,
	278	void *payload, size_t sz)
	279	{
	280	struct snp_guest_msg *req = snp_dev->request;
	281	struct snp_guest_msg_hdr *hdr = &req->hdr;
	282
	283	memset(req, 0, sizeof(*req));
	284
	285	hdr->algo = SNP_AEAD_AES_256_GCM;
	286	hdr->hdr_version = MSG_HDR_VER;
	287	hdr->hdr_sz = sizeof(*hdr);
	288	hdr->msg_type = type;
	289	hdr->msg_version = version;
	290	hdr->msg_seqno = seqno;
	291	hdr->msg_vmpck = vmpck_id;
	292	hdr->msg_sz = sz;
	293
	294	/* Verify the sequence number is non-zero */
	295	if (!hdr->msg_seqno)
	296	return -ENOSR;
	297
	298	dev_dbg(snp_dev->dev, "request [seqno %lld type %d version %d sz %d]\n",
	299	hdr->msg_seqno, hdr->msg_type, hdr->msg_version, hdr->msg_sz);
	300
	301	return __enc_payload(snp_dev, req, payload, sz);
	302	}
	303
	304	static int handle_guest_request(struct snp_guest_dev *snp_dev, u64 exit_code, int msg_ver,
	305	u8 type, void req_buf, size_t req_sz, void resp_buf,
	306	u32 resp_sz, __u64 *fw_err)
	307	{
	308	unsigned long err;
	309	u64 seqno;
	310	int rc;
	311
	312	/* Get message sequence and verify that its a non-zero */
	313	seqno = snp_get_msg_seqno(snp_dev);
	314	if (!seqno)
	315	return -EIO;
	316
	317	memset(snp_dev->response, 0, sizeof(struct snp_guest_msg));
	318
	319	/* Encrypt the userspace provided payload */
	320	rc = enc_payload(snp_dev, seqno, msg_ver, type, req_buf, req_sz);
	321	if (rc)
	322	return rc;
	323
	324	/* Call firmware to process the request */
	325	rc = snp_issue_guest_request(exit_code, &snp_dev->input, &err);
	326	if (fw_err)
	327	*fw_err = err;
	328
	329	if (rc)
	330	return rc;
	331
	332	/*
	333	* The verify_and_dec_payload() will fail only if the hypervisor is
	334	* actively modifying the message header or corrupting the encrypted payload.
	335	* This hints that hypervisor is acting in a bad faith. Disable the VMPCK so that
	336	* the key cannot be used for any communication. The key is disabled to ensure
	337	* that AES-GCM does not use the same IV while encrypting the request payload.
	338	*/
	339	rc = verify_and_dec_payload(snp_dev, resp_buf, resp_sz);
	340	if (rc) {
	341	dev_alert(snp_dev->dev,
	342	"Detected unexpected decode failure, disabling the vmpck_id %d\n",
	343	vmpck_id);
	344	snp_disable_vmpck(snp_dev);
	345	return rc;
	346	}
	347
	348	/* Increment to new message sequence after payload decryption was successful. */
	349	snp_inc_msg_seqno(snp_dev);
	350
	351	return 0;
	352	}
	353
	354	static int get_report(struct snp_guest_dev snp_dev, struct snp_guest_request_ioctl arg)
	355	{
	356	struct snp_guest_crypto *crypto = snp_dev->crypto;
	357	struct snp_report_resp *resp;
	358	struct snp_report_req req;
	359	int rc, resp_len;
	360
	361	lockdep_assert_held(&snp_cmd_mutex);
	362
	363	if (!arg->req_data \|\| !arg->resp_data)
	364	return -EINVAL;
	365
	366	if (copy_from_user(&req, (void __user *)arg->req_data, sizeof(req)))
	367	return -EFAULT;
	368
	369	/*
	370	* The intermediate response buffer is used while decrypting the
	371	* response payload. Make sure that it has enough space to cover the
	372	* authtag.
	373	*/
	374	resp_len = sizeof(resp->data) + crypto->a_len;
	375	resp = kzalloc(resp_len, GFP_KERNEL_ACCOUNT);
	376	if (!resp)
	377	return -ENOMEM;
	378
	379	rc = handle_guest_request(snp_dev, SVM_VMGEXIT_GUEST_REQUEST, arg->msg_version,
	380	SNP_MSG_REPORT_REQ, &req, sizeof(req), resp->data,
	381	resp_len, &arg->fw_err);
	382	if (rc)
	383	goto e_free;
	384
	385	if (copy_to_user((void __user )arg->resp_data, resp, sizeof(resp)))
	386	rc = -EFAULT;
	387
	388	e_free:
	389	kfree(resp);
	390	return rc;
	391	}
	392
	393	static int get_derived_key(struct snp_guest_dev snp_dev, struct snp_guest_request_ioctl arg)
	394	{
	395	struct snp_guest_crypto *crypto = snp_dev->crypto;
	396	struct snp_derived_key_resp resp = {0};
	397	struct snp_derived_key_req req;
	398	int rc, resp_len;
	399	/* Response data is 64 bytes and max authsize for GCM is 16 bytes. */
	400	u8 buf[64 + 16];
	401
	402	lockdep_assert_held(&snp_cmd_mutex);
	403
	404	if (!arg->req_data \|\| !arg->resp_data)
	405	return -EINVAL;
	406
	407	/*
	408	* The intermediate response buffer is used while decrypting the
	409	* response payload. Make sure that it has enough space to cover the
	410	* authtag.
	411	*/
	412	resp_len = sizeof(resp.data) + crypto->a_len;
	413	if (sizeof(buf) < resp_len)
	414	return -ENOMEM;
	415
	416	if (copy_from_user(&req, (void __user *)arg->req_data, sizeof(req)))
	417	return -EFAULT;
	418
	419	rc = handle_guest_request(snp_dev, SVM_VMGEXIT_GUEST_REQUEST, arg->msg_version,
	420	SNP_MSG_KEY_REQ, &req, sizeof(req), buf, resp_len,
	421	&arg->fw_err);
	422	if (rc)
	423	return rc;
	424
	425	memcpy(resp.data, buf, sizeof(resp.data));
	426	if (copy_to_user((void __user *)arg->resp_data, &resp, sizeof(resp)))
	427	rc = -EFAULT;
	428
	429	/* The response buffer contains the sensitive data, explicitly clear it. */
	430	memzero_explicit(buf, sizeof(buf));
	431	memzero_explicit(&resp, sizeof(resp));
	432	return rc;
	433	}
	434
	435	static int get_ext_report(struct snp_guest_dev snp_dev, struct snp_guest_request_ioctl arg)
	436	{
	437	struct snp_guest_crypto *crypto = snp_dev->crypto;
	438	struct snp_ext_report_req req;
	439	struct snp_report_resp *resp;
	440	int ret, npages = 0, resp_len;
	441
	442	lockdep_assert_held(&snp_cmd_mutex);
	443
	444	if (!arg->req_data \|\| !arg->resp_data)
	445	return -EINVAL;
	446
	447	if (copy_from_user(&req, (void __user *)arg->req_data, sizeof(req)))
	448	return -EFAULT;
	449
	450	/* userspace does not want certificate data */
	451	if (!req.certs_len \|\| !req.certs_address)
	452	goto cmd;
	453
	454	if (req.certs_len > SEV_FW_BLOB_MAX_SIZE \|\|
	455	!IS_ALIGNED(req.certs_len, PAGE_SIZE))
	456	return -EINVAL;
	457
	458	if (!access_ok((const void __user *)req.certs_address, req.certs_len))
	459	return -EFAULT;
	460
	461	/*
	462	* Initialize the intermediate buffer with all zeros. This buffer
	463	* is used in the guest request message to get the certs blob from
	464	* the host. If host does not supply any certs in it, then copy
	465	* zeros to indicate that certificate data was not provided.
	466	*/
	467	memset(snp_dev->certs_data, 0, req.certs_len);
	468	npages = req.certs_len >> PAGE_SHIFT;
	469	cmd:
	470	/*
	471	* The intermediate response buffer is used while decrypting the
	472	* response payload. Make sure that it has enough space to cover the
	473	* authtag.
	474	*/
	475	resp_len = sizeof(resp->data) + crypto->a_len;
	476	resp = kzalloc(resp_len, GFP_KERNEL_ACCOUNT);
	477	if (!resp)
	478	return -ENOMEM;
	479
	480	snp_dev->input.data_npages = npages;
	481	ret = handle_guest_request(snp_dev, SVM_VMGEXIT_EXT_GUEST_REQUEST, arg->msg_version,
	482	SNP_MSG_REPORT_REQ, &req.data,
	483	sizeof(req.data), resp->data, resp_len, &arg->fw_err);
	484
	485	/* If certs length is invalid then copy the returned length */
	486	if (arg->fw_err == SNP_GUEST_REQ_INVALID_LEN) {
	487	req.certs_len = snp_dev->input.data_npages << PAGE_SHIFT;
	488
	489	if (copy_to_user((void __user *)arg->req_data, &req, sizeof(req)))
	490	ret = -EFAULT;
	491	}
	492
	493	if (ret)
	494	goto e_free;
	495
	496	if (npages &&
	497	copy_to_user((void __user *)req.certs_address, snp_dev->certs_data,
	498	req.certs_len)) {
	499	ret = -EFAULT;
	500	goto e_free;
	501	}
	502
	503	if (copy_to_user((void __user )arg->resp_data, resp, sizeof(resp)))
	504	ret = -EFAULT;
	505
	506	e_free:
	507	kfree(resp);
	508	return ret;
	509	}
	510
	511	static long snp_guest_ioctl(struct file *file, unsigned int ioctl, unsigned long arg)
	512	{
	513	struct snp_guest_dev *snp_dev = to_snp_dev(file);
	514	void __user argp = (void __user )arg;
	515	struct snp_guest_request_ioctl input;
	516	int ret = -ENOTTY;
	517
	518	if (copy_from_user(&input, argp, sizeof(input)))
	519	return -EFAULT;
	520
	521	input.fw_err = 0xff;
	522
	523	/* Message version must be non-zero */
	524	if (!input.msg_version)
	525	return -EINVAL;
	526
	527	mutex_lock(&snp_cmd_mutex);
	528
	529	/* Check if the VMPCK is not empty */
	530	if (is_vmpck_empty(snp_dev)) {
	531	dev_err_ratelimited(snp_dev->dev, "VMPCK is disabled\n");
	532	mutex_unlock(&snp_cmd_mutex);
	533	return -ENOTTY;
	534	}
	535
	536	switch (ioctl) {
	537	case SNP_GET_REPORT:
	538	ret = get_report(snp_dev, &input);
	539	break;
	540	case SNP_GET_DERIVED_KEY:
	541	ret = get_derived_key(snp_dev, &input);
	542	break;
	543	case SNP_GET_EXT_REPORT:
	544	ret = get_ext_report(snp_dev, &input);
	545	break;
	546	default:
	547	break;
	548	}
	549
	550	mutex_unlock(&snp_cmd_mutex);
	551
	552	if (input.fw_err && copy_to_user(argp, &input, sizeof(input)))
	553	return -EFAULT;
	554
	555	return ret;
	556	}
	557
	558	static void free_shared_pages(void *buf, size_t sz)
	559	{
	560	unsigned int npages = PAGE_ALIGN(sz) >> PAGE_SHIFT;
	561	int ret;
	562
	563	if (!buf)
	564	return;
	565
	566	ret = set_memory_encrypted((unsigned long)buf, npages);
	567	if (ret) {
	568	WARN_ONCE(ret, "failed to restore encryption mask (leak it)\n");
	569	return;
	570	}
	571
	572	__free_pages(virt_to_page(buf), get_order(sz));
	573	}
	574
	575	static void alloc_shared_pages(struct device dev, size_t sz)
	576	{
	577	unsigned int npages = PAGE_ALIGN(sz) >> PAGE_SHIFT;
	578	struct page *page;
	579	int ret;
	580
	581	page = alloc_pages(GFP_KERNEL_ACCOUNT, get_order(sz));
	582	if (!page)
	583	return NULL;
	584
	585	ret = set_memory_decrypted((unsigned long)page_address(page), npages);
	586	if (ret) {
	587	dev_err(dev, "failed to mark page shared, ret=%d\n", ret);
	588	__free_pages(page, get_order(sz));
	589	return NULL;
	590	}
	591
	592	return page_address(page);
	593	}
	594
	595	static const struct file_operations snp_guest_fops = {
	596	.owner = THIS_MODULE,
	597	.unlocked_ioctl = snp_guest_ioctl,
	598	};
	599
	600	static u8 get_vmpck(int id, struct snp_secrets_page_layout layout, u32 **seqno)
	601	{
	602	u8 *key = NULL;
	603
	604	switch (id) {
	605	case 0:
	606	*seqno = &layout->os_area.msg_seqno_0;
	607	key = layout->vmpck0;
	608	break;
	609	case 1:
	610	*seqno = &layout->os_area.msg_seqno_1;
	611	key = layout->vmpck1;
	612	break;
	613	case 2:
	614	*seqno = &layout->os_area.msg_seqno_2;
	615	key = layout->vmpck2;
	616	break;
	617	case 3:
	618	*seqno = &layout->os_area.msg_seqno_3;
	619	key = layout->vmpck3;
	620	break;
	621	default:
	622	break;
	623	}
	624
	625	return key;
	626	}
	627
	628	static int __init sev_guest_probe(struct platform_device *pdev)
	629	{
	630	struct snp_secrets_page_layout *layout;
	631	struct sev_guest_platform_data *data;
	632	struct device *dev = &pdev->dev;
	633	struct snp_guest_dev *snp_dev;
	634	struct miscdevice *misc;
	635	int ret;
	636
	637	if (!dev->platform_data)
	638	return -ENODEV;
	639
	640	data = (struct sev_guest_platform_data *)dev->platform_data;
	641	layout = (__force void *)ioremap_encrypted(data->secrets_gpa, PAGE_SIZE);
	642	if (!layout)
	643	return -ENODEV;
	644
	645	ret = -ENOMEM;
	646	snp_dev = devm_kzalloc(&pdev->dev, sizeof(struct snp_guest_dev), GFP_KERNEL);
	647	if (!snp_dev)
	648	goto e_unmap;
	649
	650	ret = -EINVAL;
	651	snp_dev->vmpck = get_vmpck(vmpck_id, layout, &snp_dev->os_area_msg_seqno);
	652	if (!snp_dev->vmpck) {
	653	dev_err(dev, "invalid vmpck id %d\n", vmpck_id);
	654	goto e_unmap;
	655	}
	656
	657	/* Verify that VMPCK is not zero. */
	658	if (is_vmpck_empty(snp_dev)) {
	659	dev_err(dev, "vmpck id %d is null\n", vmpck_id);
	660	goto e_unmap;
	661	}
	662
	663	platform_set_drvdata(pdev, snp_dev);
	664	snp_dev->dev = dev;
	665	snp_dev->layout = layout;
	666
	667	/* Allocate the shared page used for the request and response message. */
	668	snp_dev->request = alloc_shared_pages(dev, sizeof(struct snp_guest_msg));
	669	if (!snp_dev->request)
	670	goto e_unmap;
	671
	672	snp_dev->response = alloc_shared_pages(dev, sizeof(struct snp_guest_msg));
	673	if (!snp_dev->response)
	674	goto e_free_request;
	675
	676	snp_dev->certs_data = alloc_shared_pages(dev, SEV_FW_BLOB_MAX_SIZE);
	677	if (!snp_dev->certs_data)
	678	goto e_free_response;
	679
	680	ret = -EIO;
	681	snp_dev->crypto = init_crypto(snp_dev, snp_dev->vmpck, VMPCK_KEY_LEN);
	682	if (!snp_dev->crypto)
	683	goto e_free_cert_data;
	684
	685	misc = &snp_dev->misc;
	686	misc->minor = MISC_DYNAMIC_MINOR;
	687	misc->name = DEVICE_NAME;
	688	misc->fops = &snp_guest_fops;
	689
	690	/* initial the input address for guest request */
	691	snp_dev->input.req_gpa = __pa(snp_dev->request);
	692	snp_dev->input.resp_gpa = __pa(snp_dev->response);
	693	snp_dev->input.data_gpa = __pa(snp_dev->certs_data);
	694
	695	ret = misc_register(misc);
	696	if (ret)
	697	goto e_free_cert_data;
	698
	699	dev_info(dev, "Initialized SEV guest driver (using vmpck_id %d)\n", vmpck_id);
	700	return 0;
	701
	702	e_free_cert_data:
	703	free_shared_pages(snp_dev->certs_data, SEV_FW_BLOB_MAX_SIZE);
	704	e_free_response:
	705	free_shared_pages(snp_dev->response, sizeof(struct snp_guest_msg));
	706	e_free_request:
	707	free_shared_pages(snp_dev->request, sizeof(struct snp_guest_msg));
	708	e_unmap:
	709	iounmap(layout);
	710	return ret;
	711	}
	712
	713	static int __exit sev_guest_remove(struct platform_device *pdev)
	714	{
	715	struct snp_guest_dev *snp_dev = platform_get_drvdata(pdev);
	716
	717	free_shared_pages(snp_dev->certs_data, SEV_FW_BLOB_MAX_SIZE);
	718	free_shared_pages(snp_dev->response, sizeof(struct snp_guest_msg));
	719	free_shared_pages(snp_dev->request, sizeof(struct snp_guest_msg));
	720	deinit_crypto(snp_dev->crypto);
	721	misc_deregister(&snp_dev->misc);
	722
	723	return 0;
	724	}
	725
	726	/*
	727	* This driver is meant to be a common SEV guest interface driver and to
	728	* support any SEV guest API. As such, even though it has been introduced
	729	* with the SEV-SNP support, it is named "sev-guest".
	730	*/
	731	static struct platform_driver sev_guest_driver = {
	732	.remove = __exit_p(sev_guest_remove),
	733	.driver = {
	734	.name = "sev-guest",
	735	},
	736	};
	737
	738	module_platform_driver_probe(sev_guest_driver, sev_guest_probe);
	739
	740	MODULE_AUTHOR("Brijesh Singh <brijesh.singh@amd.com>");
	741	MODULE_LICENSE("GPL");
	742	MODULE_VERSION("1.0.0");
	743	MODULE_DESCRIPTION("AMD SEV Guest Driver");

drivers/virt/coco/sev-guest/sev-guest.h+63−0

@@ -0,0 +1,63 @@
	1	/* SPDX-License-Identifier: GPL-2.0-only */
	2	/*
	3	* Copyright (C) 2021 Advanced Micro Devices, Inc.
	4	*
	5	* Author: Brijesh Singh <brijesh.singh@amd.com>
	6	*
	7	* SEV-SNP API spec is available at https://developer.amd.com/sev
	8	*/
	9
	10	#ifndef __VIRT_SEVGUEST_H__
	11	#define __VIRT_SEVGUEST_H__
	12
	13	#include <linux/types.h>
	14
	15	#define MAX_AUTHTAG_LEN 32
	16
	17	/* See SNP spec SNP_GUEST_REQUEST section for the structure */
	18	enum msg_type {
	19	SNP_MSG_TYPE_INVALID = 0,
	20	SNP_MSG_CPUID_REQ,
	21	SNP_MSG_CPUID_RSP,
	22	SNP_MSG_KEY_REQ,
	23	SNP_MSG_KEY_RSP,
	24	SNP_MSG_REPORT_REQ,
	25	SNP_MSG_REPORT_RSP,
	26	SNP_MSG_EXPORT_REQ,
	27	SNP_MSG_EXPORT_RSP,
	28	SNP_MSG_IMPORT_REQ,
	29	SNP_MSG_IMPORT_RSP,
	30	SNP_MSG_ABSORB_REQ,
	31	SNP_MSG_ABSORB_RSP,
	32	SNP_MSG_VMRK_REQ,
	33	SNP_MSG_VMRK_RSP,
	34
	35	SNP_MSG_TYPE_MAX
	36	};
	37
	38	enum aead_algo {
	39	SNP_AEAD_INVALID,
	40	SNP_AEAD_AES_256_GCM,
	41	};
	42
	43	struct snp_guest_msg_hdr {
	44	u8 authtag[MAX_AUTHTAG_LEN];
	45	u64 msg_seqno;
	46	u8 rsvd1[8];
	47	u8 algo;
	48	u8 hdr_version;
	49	u16 hdr_sz;
	50	u8 msg_type;
	51	u8 msg_version;
	52	u16 msg_sz;
	53	u32 rsvd2;
	54	u8 msg_vmpck;
	55	u8 rsvd3[35];
	56	} __packed;
	57
	58	struct snp_guest_msg {
	59	struct snp_guest_msg_hdr hdr;
	60	u8 payload[4000];
	61	} __packed;
	62
	63	#endif /* __VIRT_SEVGUEST_H__ */

include/linux/cc_platform.h+8−0

@@ -72,6 +72,14 @@ enum cc_attr { · 71 unmodified lines
72	* Examples include TDX guest & SEV.	72	* Examples include TDX guest & SEV.
73	*/	73	*/
74	CC_ATTR_GUEST_UNROLL_STRING_IO,	74	CC_ATTR_GUEST_UNROLL_STRING_IO,
		75
		76	/**
		77	* @CC_ATTR_SEV_SNP: Guest SNP is active.
		78	*
		79	* The platform/OS is running as a guest/virtual machine and actively
		80	* using AMD SEV-SNP features.
		81	*/
		82	CC_ATTR_GUEST_SEV_SNP,
75	};	83	};
76		84
77	#ifdef CONFIG_ARCH_HAS_CC_PLATFORM	85	#ifdef CONFIG_ARCH_HAS_CC_PLATFORM

include/linux/efi.h+1−0

@@ -393,6 +393,7 @@ void efi_native_runtime_setup(void); · 392 unmodified lines
393	#define EFI_CERT_SHA256_GUID EFI_GUID(0xc1c41626, 0x504c, 0x4092, 0xac, 0xa9, 0x41, 0xf9, 0x36, 0x93, 0x43, 0x28)	393	#define EFI_CERT_SHA256_GUID EFI_GUID(0xc1c41626, 0x504c, 0x4092, 0xac, 0xa9, 0x41, 0xf9, 0x36, 0x93, 0x43, 0x28)
394	#define EFI_CERT_X509_GUID EFI_GUID(0xa5c059a1, 0x94e4, 0x4aa7, 0x87, 0xb5, 0xab, 0x15, 0x5c, 0x2b, 0xf0, 0x72)	394	#define EFI_CERT_X509_GUID EFI_GUID(0xa5c059a1, 0x94e4, 0x4aa7, 0x87, 0xb5, 0xab, 0x15, 0x5c, 0x2b, 0xf0, 0x72)
395	#define EFI_CERT_X509_SHA256_GUID EFI_GUID(0x3bd2a492, 0x96c0, 0x4079, 0xb4, 0x20, 0xfc, 0xf9, 0x8e, 0xf1, 0x03, 0xed)	395	#define EFI_CERT_X509_SHA256_GUID EFI_GUID(0x3bd2a492, 0x96c0, 0x4079, 0xb4, 0x20, 0xfc, 0xf9, 0x8e, 0xf1, 0x03, 0xed)
		396	#define EFI_CC_BLOB_GUID EFI_GUID(0x067b1f5f, 0xcf26, 0x44c5, 0x85, 0x54, 0x93, 0xd7, 0x77, 0x91, 0x2d, 0x42)
396		397
397	/*	398	/*
398	* This GUID is used to pass to the kernel proper the struct screen_info	399	* This GUID is used to pass to the kernel proper the struct screen_info

include/uapi/linux/sev-guest.h+80−0

@@ -0,0 +1,80 @@
	1	/* SPDX-License-Identifier: GPL-2.0-only WITH Linux-syscall-note */
	2	/*
	3	* Userspace interface for AMD SEV and SNP guest driver.
	4	*
	5	* Copyright (C) 2021 Advanced Micro Devices, Inc.
	6	*
	7	* Author: Brijesh Singh <brijesh.singh@amd.com>
	8	*
	9	* SEV API specification is available at: https://developer.amd.com/sev/
	10	*/
	11
	12	#ifndef __UAPI_LINUX_SEV_GUEST_H_
	13	#define __UAPI_LINUX_SEV_GUEST_H_
	14
	15	#include <linux/types.h>
	16
	17	struct snp_report_req {
	18	/* user data that should be included in the report */
	19	__u8 user_data[64];
	20
	21	/* The vmpl level to be included in the report */
	22	__u32 vmpl;
	23
	24	/* Must be zero filled */
	25	__u8 rsvd[28];
	26	};
	27
	28	struct snp_report_resp {
	29	/* response data, see SEV-SNP spec for the format */
	30	__u8 data[4000];
	31	};
	32
	33	struct snp_derived_key_req {
	34	__u32 root_key_select;
	35	__u32 rsvd;
	36	__u64 guest_field_select;
	37	__u32 vmpl;
	38	__u32 guest_svn;
	39	__u64 tcb_version;
	40	};
	41
	42	struct snp_derived_key_resp {
	43	/* response data, see SEV-SNP spec for the format */
	44	__u8 data[64];
	45	};
	46
	47	struct snp_guest_request_ioctl {
	48	/* message version number (must be non-zero) */
	49	__u8 msg_version;
	50
	51	/* Request and response structure address */
	52	__u64 req_data;
	53	__u64 resp_data;
	54
	55	/* firmware error code on failure (see psp-sev.h) */
	56	__u64 fw_err;
	57	};
	58
	59	struct snp_ext_report_req {
	60	struct snp_report_req data;
	61
	62	/* where to copy the certificate blob */
	63	__u64 certs_address;
	64
	65	/* length of the certificate blob */
	66	__u32 certs_len;
	67	};
	68
	69	#define SNP_GUEST_REQ_IOC_TYPE 'S'
	70
	71	/* Get SNP attestation report */
	72	#define SNP_GET_REPORT _IOWR(SNP_GUEST_REQ_IOC_TYPE, 0x0, struct snp_guest_request_ioctl)
	73
	74	/* Get a derived key from the root */
	75	#define SNP_GET_DERIVED_KEY _IOWR(SNP_GUEST_REQ_IOC_TYPE, 0x1, struct snp_guest_request_ioctl)
	76
	77	/* Get SNP extended report as defined in the GHCB specification version 2. */
	78	#define SNP_GET_EXT_REPORT _IOWR(SNP_GUEST_REQ_IOC_TYPE, 0x2, struct snp_guest_request_ioctl)
	79
	80	#endif /* __UAPI_LINUX_SEV_GUEST_H_ */

Merge tag 'x86_sev_for_v5.19_rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code

@@ -1270,8 +1270,8 @@ static void svm_prepare_switch_to_guest(struct kvm_vcpu *vcpu) · 1269 unmodified lines
1270	*/	1270	*/
1271	vmsave(__sme_page_pa(sd->save_area));	1271	vmsave(__sme_page_pa(sd->save_area));
1272	if (sev_es_guest(vcpu->kvm)) {	1272	if (sev_es_guest(vcpu->kvm)) {
1273	struct vmcb_save_area *hostsa;	1273	struct sev_es_save_area *hostsa;
1274	hostsa = (struct vmcb_save_area *)(page_address(sd->save_area) + 0x400);	1274	hostsa = (struct sev_es_save_area *)(page_address(sd->save_area) + 0x400);
1275		1275
1276	sev_es_prepare_switch_to_guest(hostsa);	1276	sev_es_prepare_switch_to_guest(hostsa);
1277	}	1277	}
@@ -3117,8 +3117,8 @@ static void dump_vmcb(struct kvm_vcpu *vcpu) · 1839 unmodified lines
3117	"tr:",	3117	"tr:",
3118	save01->tr.selector, save01->tr.attrib,	3118	save01->tr.selector, save01->tr.attrib,
3119	save01->tr.limit, save01->tr.base);	3119	save01->tr.limit, save01->tr.base);
3120	pr_err("cpl: %d efer: %016llx\n",	3120	pr_err("vmpl: %d cpl: %d efer: %016llx\n",
3121	save->cpl, save->efer);	3121	save->vmpl, save->cpl, save->efer);
3122	pr_err("%-15s %016llx %-13s %016llx\n",	3122	pr_err("%-15s %016llx %-13s %016llx\n",
3123	"cr0:", save->cr0, "cr2:", save->cr2);	3123	"cr0:", save->cr0, "cr2:", save->cr2);
3124	pr_err("%-15s %016llx %-13s %016llx\n",	3124	pr_err("%-15s %016llx %-13s %016llx\n",