Merge tag 'arm64-spectre-bhb-for-v5.17-2' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux

Pull arm64 spectre fixes from James Morse: "ARM64 Spectre-BHB mitigations: - Make EL1 vectors per-cpu - Add mitigation sequences to the EL1 and EL2 vectors on vulnerble CPUs - Implement ARCH_WORKAROUND_3 for KVM guests - Report Vulnerable when unprivileged eBPF is enabled" * tag 'arm64-spectre-bhb-for-v5.17-2' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux: arm64: proton-pack: Include unprivileged eBPF status in Spectre v2 mitigation reporting arm64: Use the clearbhb instruction in mitigations KVM: arm64: Allow SMCCC_ARCH_WORKAROUND_3 to be discovered and migrated arm64: Mitigate spectre style branch history side channels arm64: proton-pack: Report Spectre-BHB vulnerabilities as part of Spectre-v2 arm64: Add percpu vectors for EL1 arm64: entry: Add macro for reading symbol addresses from the trampoline arm64: entry: Add vectors that have the bhb mitigation sequences arm64: entry: Add non-kpti __bp_harden_el1_vectors for mitigations arm64: entry: Allow the trampoline text to occupy multiple pages arm64: entry: Make the kpti trampoline's kpti sequence optional arm64: entry: Move trampoline macros out of ifdef'd section arm64: entry: Don't assume tramp_vectors is the start of the vectors arm64: entry: Allow tramp_alias to access symbols after the 4K boundary arm64: entry: Move the trampoline data page before the text page arm64: entry: Free up another register on kpti's tramp_exit path arm64: entry: Make the trampoline cleanup optional KVM: arm64: Allow indirect vectors to be used without SPECTRE_V3A arm64: spectre: Rename spectre_v4_patch_fw_mitigation_conduit arm64: entry.S: Add ventry overflow sanity checks

Linus Torvalds committed 4 years ago

Commit cd22a8b2 parents fc55c23 58c9a5027 files · +830 −76Browse files

Unified · Split

27 files

arch/arm64/Kconfig +9 −0
arch/arm64/include/asm/assembler.h +53 −0
arch/arm64/include/asm/cpufeature.h +29 −0
arch/arm64/include/asm/cputype.h +8 −0
arch/arm64/include/asm/fixmap.h +4 −2
arch/arm64/include/asm/insn.h +1 −0
arch/arm64/include/asm/kvm_host.h +5 −0
arch/arm64/include/asm/sections.h +5 −0
arch/arm64/include/asm/spectre.h +4 −0
arch/arm64/include/asm/sysreg.h +2 −0
arch/arm64/include/asm/vectors.h +73 −0
arch/arm64/include/uapi/asm/kvm.h +5 −0
arch/arm64/kernel/cpu_errata.c +7 −0
arch/arm64/kernel/cpufeature.c +12 −0
arch/arm64/kernel/entry.S +157 −57
arch/arm64/kernel/image-vars.h +4 −0
arch/arm64/kernel/proton-pack.c +386 −5
arch/arm64/kernel/vmlinux.lds.S +1 −1
arch/arm64/kvm/arm.c +1 −4
arch/arm64/kvm/hyp/hyp-entry.S +9 −0
arch/arm64/kvm/hyp/nvhe/mm.c +3 −1
arch/arm64/kvm/hyp/vhe/switch.c +8 −2
arch/arm64/kvm/hypercalls.c +12 −0
arch/arm64/kvm/psci.c +17 −1
arch/arm64/mm/mmu.c +9 −3
arch/arm64/tools/cpucaps +1 −0
include/linux/arm-smccc.h +5 −0

arch/arm64/Kconfig+9−0

@@ -1383,6 +1383,15 @@ config UNMAP_KERNEL_AT_EL0 · 1382 unmodified lines
1383	1383
1384	1384	If unsure, say Y.
1385	1385
	1386	+config MITIGATE_SPECTRE_BRANCH_HISTORY
	1387	+ bool "Mitigate Spectre style attacks against branch history" if EXPERT
	1388	+ default y
	1389	+ help
	1390	+ Speculation attacks against some high-performance processors can
	1391	+ make use of branch history to influence future speculation.
	1392	+ When taking an exception from user-space, a sequence of branches
	1393	+ or a firmware call overwrites the branch history.
	1394	+
1386	1395	config RODATA_FULL_DEFAULT_ENABLED
1387	1396	bool "Apply r/o permissions of VM areas also to their linear aliases"
1388	1397	default y

arch/arm64/include/asm/assembler.h+53−0

@@ -108,6 +108,13 @@ · 107 unmodified lines
108	108	hint #20
109	109	.endm
110	110
	111	+/*
	112	+ * Clear Branch History instruction
	113	+ */
	114	+ .macro clearbhb
	115	+ hint #22
	116	+ .endm
	117	+
111	118	/*
112	119	* Speculation barrier
113	120	*/
@@ -850,4 +857,50 @@ alternative_endif · 736 unmodified lines
850	857
851	858	#endif /* GNU_PROPERTY_AARCH64_FEATURE_1_DEFAULT */
852	859
	860	+ .macro __mitigate_spectre_bhb_loop tmp
	861	+#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY
	862	+alternative_cb spectre_bhb_patch_loop_iter
	863	+ mov \tmp, #32 // Patched to correct the immediate
	864	+alternative_cb_end
	865	+.Lspectre_bhb_loop\@:
	866	+ b . + 4
	867	+ subs \tmp, \tmp, #1
	868	+ b.ne .Lspectre_bhb_loop\@
	869	+ sb
	870	+#endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */
	871	+ .endm
	872	+
	873	+ .macro mitigate_spectre_bhb_loop tmp
	874	+#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY
	875	+alternative_cb spectre_bhb_patch_loop_mitigation_enable
	876	+ b .L_spectre_bhb_loop_done\@ // Patched to NOP
	877	+alternative_cb_end
	878	+ __mitigate_spectre_bhb_loop \tmp
	879	+.L_spectre_bhb_loop_done\@:
	880	+#endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */
	881	+ .endm
	882	+
	883	+ /* Save/restores x0-x3 to the stack */
	884	+ .macro __mitigate_spectre_bhb_fw
	885	+#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY
	886	+ stp x0, x1, [sp, #-16]!
	887	+ stp x2, x3, [sp, #-16]!
	888	+ mov w0, #ARM_SMCCC_ARCH_WORKAROUND_3
	889	+alternative_cb smccc_patch_fw_mitigation_conduit
	890	+ nop // Patched to SMC/HVC #0
	891	+alternative_cb_end
	892	+ ldp x2, x3, [sp], #16
	893	+ ldp x0, x1, [sp], #16
	894	+#endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */
	895	+ .endm
	896	+
	897	+ .macro mitigate_spectre_bhb_clear_insn
	898	+#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY
	899	+alternative_cb spectre_bhb_patch_clearbhb
	900	+ /* Patched to NOP when not supported */
	901	+ clearbhb
	902	+ isb
	903	+alternative_cb_end
	904	+#endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */
	905	+ .endm
853	906	#endif /* __ASM_ASSEMBLER_H */

arch/arm64/include/asm/cpufeature.h+29−0

@@ -637,6 +637,35 @@ static inline bool cpu_supports_mixed_endian_el0(void) · 636 unmodified lines
637	637	return id_aa64mmfr0_mixed_endian_el0(read_cpuid(ID_AA64MMFR0_EL1));
638	638	}
639	639
	640	+
	641	+static inline bool supports_csv2p3(int scope)
	642	+{
	643	+ u64 pfr0;
	644	+ u8 csv2_val;
	645	+
	646	+ if (scope == SCOPE_LOCAL_CPU)
	647	+ pfr0 = read_sysreg_s(SYS_ID_AA64PFR0_EL1);
	648	+ else
	649	+ pfr0 = read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1);
	650	+
	651	+ csv2_val = cpuid_feature_extract_unsigned_field(pfr0,
	652	+ ID_AA64PFR0_CSV2_SHIFT);
	653	+ return csv2_val == 3;
	654	+}
	655	+
	656	+static inline bool supports_clearbhb(int scope)
	657	+{
	658	+ u64 isar2;
	659	+
	660	+ if (scope == SCOPE_LOCAL_CPU)
	661	+ isar2 = read_sysreg_s(SYS_ID_AA64ISAR2_EL1);
	662	+ else
	663	+ isar2 = read_sanitised_ftr_reg(SYS_ID_AA64ISAR2_EL1);
	664	+
	665	+ return cpuid_feature_extract_unsigned_field(isar2,
	666	+ ID_AA64ISAR2_CLEARBHB_SHIFT);
	667	+}
	668	+
640	669	const struct cpumask *system_32bit_el0_cpumask(void);
641	670	DECLARE_STATIC_KEY_FALSE(arm64_mismatched_32bit_el0);
642	671

arch/arm64/include/asm/cputype.h+8−0

@@ -73,10 +73,14 @@ · 72 unmodified lines
73	73	#define ARM_CPU_PART_CORTEX_A76 0xD0B
74	74	#define ARM_CPU_PART_NEOVERSE_N1 0xD0C
75	75	#define ARM_CPU_PART_CORTEX_A77 0xD0D
	76	+#define ARM_CPU_PART_NEOVERSE_V1 0xD40
	77	+#define ARM_CPU_PART_CORTEX_A78 0xD41
	78	+#define ARM_CPU_PART_CORTEX_X1 0xD44
76	79	#define ARM_CPU_PART_CORTEX_A510 0xD46
77	80	#define ARM_CPU_PART_CORTEX_A710 0xD47
78	81	#define ARM_CPU_PART_CORTEX_X2 0xD48
79	82	#define ARM_CPU_PART_NEOVERSE_N2 0xD49
	83	+#define ARM_CPU_PART_CORTEX_A78C 0xD4B
80	84
81	85	#define APM_CPU_PART_POTENZA 0x000
82	86
@@ -117,10 +121,14 @@ · 34 unmodified lines
117	121	#define MIDR_CORTEX_A76 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A76)
118	122	#define MIDR_NEOVERSE_N1 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_NEOVERSE_N1)
119	123	#define MIDR_CORTEX_A77 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A77)
	124	+#define MIDR_NEOVERSE_V1 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_NEOVERSE_V1)
	125	+#define MIDR_CORTEX_A78 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A78)
	126	+#define MIDR_CORTEX_X1 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_X1)
120	127	#define MIDR_CORTEX_A510 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A510)
121	128	#define MIDR_CORTEX_A710 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A710)
122	129	#define MIDR_CORTEX_X2 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_X2)
123	130	#define MIDR_NEOVERSE_N2 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_NEOVERSE_N2)
	131	+#define MIDR_CORTEX_A78C MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A78C)
124	132	#define MIDR_THUNDERX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX)
125	133	#define MIDR_THUNDERX_81XX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX_81XX)
126	134	#define MIDR_THUNDERX_83XX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX_83XX)

arch/arm64/include/asm/fixmap.h+4−2

@@ -62,9 +62,11 @@ enum fixed_addresses { · 61 unmodified lines
62	62	#endif /* CONFIG_ACPI_APEI_GHES */
63	63
64	64	#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
	65	+ FIX_ENTRY_TRAMP_TEXT3,
	66	+ FIX_ENTRY_TRAMP_TEXT2,
	67	+ FIX_ENTRY_TRAMP_TEXT1,
65	68	FIX_ENTRY_TRAMP_DATA,
66		− FIX_ENTRY_TRAMP_TEXT,
67		−#define TRAMP_VALIAS (__fix_to_virt(FIX_ENTRY_TRAMP_TEXT))
	69	+#define TRAMP_VALIAS (__fix_to_virt(FIX_ENTRY_TRAMP_TEXT1))
68	70	#endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */
69	71	__end_of_permanent_fixed_addresses,
70	72

arch/arm64/include/asm/insn.h+1−0

@@ -65,6 +65,7 @@ enum aarch64_insn_hint_cr_op { · 64 unmodified lines
65	65	AARCH64_INSN_HINT_PSB = 0x11 << 5,
66	66	AARCH64_INSN_HINT_TSB = 0x12 << 5,
67	67	AARCH64_INSN_HINT_CSDB = 0x14 << 5,
	68	+ AARCH64_INSN_HINT_CLEARBHB = 0x16 << 5,
68	69
69	70	AARCH64_INSN_HINT_BTI = 0x20 << 5,
70	71	AARCH64_INSN_HINT_BTIC = 0x22 << 5,

arch/arm64/include/asm/kvm_host.h+5−0

@@ -714,6 +714,11 @@ static inline void kvm_init_host_cpu_context(struct kvm_cpu_context *cpu_ctxt) · 713 unmodified lines
714	714	ctxt_sys_reg(cpu_ctxt, MPIDR_EL1) = read_cpuid_mpidr();
715	715	}
716	716
	717	+static inline bool kvm_system_needs_idmapped_vectors(void)
	718	+{
	719	+ return cpus_have_const_cap(ARM64_SPECTRE_V3A);
	720	+}
	721	+
717	722	void kvm_arm_vcpu_ptrauth_trap(struct kvm_vcpu *vcpu);
718	723
719	724	static inline void kvm_arch_hardware_unsetup(void) {}

arch/arm64/include/asm/sections.h+5−0

@@ -23,4 +23,9 @@ extern char __mmuoff_data_start[], __mmuoff_data_end[]; · 22 unmodified lines
23	23	extern char __entry_tramp_text_start[], __entry_tramp_text_end[];
24	24	extern char __relocate_new_kernel_start[], __relocate_new_kernel_end[];
25	25
	26	+static inline size_t entry_tramp_text_size(void)
	27	+{
	28	+ return __entry_tramp_text_end - __entry_tramp_text_start;
	29	+}
	30	+
26	31	#endif /* __ASM_SECTIONS_H */

arch/arm64/include/asm/spectre.h+4−0

@@ -93,5 +93,9 @@ void spectre_v4_enable_task_mitigation(struct task_struct *tsk); · 92 unmodified lines
93	93
94	94	enum mitigation_state arm64_get_meltdown_state(void);
95	95
	96	+enum mitigation_state arm64_get_spectre_bhb_state(void);
	97	+bool is_spectre_bhb_affected(const struct arm64_cpu_capabilities *entry, int scope);
	98	+u8 spectre_bhb_loop_affected(int scope);
	99	+void spectre_bhb_enable_mitigation(const struct arm64_cpu_capabilities *__unused);
96	100	#endif /* __ASSEMBLY__ */
97	101	#endif /* __ASM_SPECTRE_H */

arch/arm64/include/asm/sysreg.h+2−0

@@ -773,6 +773,7 @@ · 772 unmodified lines
773	773	#define ID_AA64ISAR1_GPI_IMP_DEF 0x1
774	774
775	775	/* id_aa64isar2 */
	776	+#define ID_AA64ISAR2_CLEARBHB_SHIFT 28
776	777	#define ID_AA64ISAR2_RPRES_SHIFT 4
777	778	#define ID_AA64ISAR2_WFXT_SHIFT 0
778	779
@@ -904,6 +905,7 @@ · 125 unmodified lines
904	905	#endif
905	906
906	907	/* id_aa64mmfr1 */
	908	+#define ID_AA64MMFR1_ECBHB_SHIFT 60
907	909	#define ID_AA64MMFR1_AFP_SHIFT 44
908	910	#define ID_AA64MMFR1_ETS_SHIFT 36
909	911	#define ID_AA64MMFR1_TWED_SHIFT 32

arch/arm64/include/asm/vectors.h+73−0

@@ -0,0 +1,73 @@
	1	+/* SPDX-License-Identifier: GPL-2.0-only */
	2	+/*
	3	+ * Copyright (C) 2022 ARM Ltd.
	4	+ */
	5	+#ifndef __ASM_VECTORS_H
	6	+#define __ASM_VECTORS_H
	7	+
	8	+#include <linux/bug.h>
	9	+#include <linux/percpu.h>
	10	+
	11	+#include <asm/fixmap.h>
	12	+
	13	+extern char vectors[];
	14	+extern char tramp_vectors[];
	15	+extern char __bp_harden_el1_vectors[];
	16	+
	17	+/*
	18	+ * Note: the order of this enum corresponds to two arrays in entry.S:
	19	+ * tramp_vecs and __bp_harden_el1_vectors. By default the canonical
	20	+ * 'full fat' vectors are used directly.
	21	+ */
	22	+enum arm64_bp_harden_el1_vectors {
	23	+#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY
	24	+ /*
	25	+ * Perform the BHB loop mitigation, before branching to the canonical
	26	+ * vectors.
	27	+ */
	28	+ EL1_VECTOR_BHB_LOOP,
	29	+
	30	+ /*
	31	+ * Make the SMC call for firmware mitigation, before branching to the
	32	+ * canonical vectors.
	33	+ */
	34	+ EL1_VECTOR_BHB_FW,
	35	+
	36	+ /*
	37	+ * Use the ClearBHB instruction, before branching to the canonical
	38	+ * vectors.
	39	+ */
	40	+ EL1_VECTOR_BHB_CLEAR_INSN,
	41	+#endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */
	42	+
	43	+ /*
	44	+ * Remap the kernel before branching to the canonical vectors.
	45	+ */
	46	+ EL1_VECTOR_KPTI,
	47	+};
	48	+
	49	+#ifndef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY
	50	+#define EL1_VECTOR_BHB_LOOP -1
	51	+#define EL1_VECTOR_BHB_FW -1
	52	+#define EL1_VECTOR_BHB_CLEAR_INSN -1
	53	+#endif /* !CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */
	54	+
	55	+/* The vectors to use on return from EL0. e.g. to remap the kernel */
	56	+DECLARE_PER_CPU_READ_MOSTLY(const char *, this_cpu_vector);
	57	+
	58	+#ifndef CONFIG_UNMAP_KERNEL_AT_EL0
	59	+#define TRAMP_VALIAS 0
	60	+#endif
	61	+
	62	+static inline const char *
	63	+arm64_get_bp_hardening_vector(enum arm64_bp_harden_el1_vectors slot)
	64	+{
	65	+ if (arm64_kernel_unmapped_at_el0())
	66	+ return (char )TRAMP_VALIAS + SZ_2K slot;
	67	+
	68	+ WARN_ON_ONCE(slot == EL1_VECTOR_KPTI);
	69	+
	70	+ return __bp_harden_el1_vectors + SZ_2K * slot;
	71	+}
	72	+
	73	+#endif /* __ASM_VECTORS_H */

arch/arm64/include/uapi/asm/kvm.h+5−0

@@ -281,6 +281,11 @@ struct kvm_arm_copy_mte_tags { · 280 unmodified lines
281	281	#define KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_NOT_REQUIRED 3
282	282	#define KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_ENABLED (1U << 4)
283	283
	284	+#define KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3 KVM_REG_ARM_FW_REG(3)
	285	+#define KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3_NOT_AVAIL 0
	286	+#define KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3_AVAIL 1
	287	+#define KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3_NOT_REQUIRED 2
	288	+
284	289	/* SVE registers */
285	290	#define KVM_REG_ARM64_SVE (0x15 << KVM_REG_ARM_COPROC_SHIFT)
286	291

arch/arm64/kernel/cpu_errata.c+7−0

@@ -502,6 +502,13 @@ const struct arm64_cpu_capabilities arm64_errata[] = { · 501 unmodified lines
502	502	.matches = has_spectre_v4,
503	503	.cpu_enable = spectre_v4_enable_mitigation,
504	504	},
	505	+ {
	506	+ .desc = "Spectre-BHB",
	507	+ .capability = ARM64_SPECTRE_BHB,
	508	+ .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,
	509	+ .matches = is_spectre_bhb_affected,
	510	+ .cpu_enable = spectre_bhb_enable_mitigation,
	511	+ },
505	512	#ifdef CONFIG_ARM64_ERRATUM_1418040
506	513	{
507	514	.desc = "ARM erratum 1418040",

arch/arm64/kernel/cpufeature.c+12−0

@@ -73,6 +73,8 @@ · 72 unmodified lines
73	73	#include <linux/mm.h>
74	74	#include <linux/cpu.h>
75	75	#include <linux/kasan.h>
	76	+#include <linux/percpu.h>
	77	+
76	78	#include <asm/cpu.h>
77	79	#include <asm/cpufeature.h>
78	80	#include <asm/cpu_ops.h>
@@ -85,6 +87,7 @@ · 6 unmodified lines
85	87	#include <asm/smp.h>
86	88	#include <asm/sysreg.h>
87	89	#include <asm/traps.h>
	90	+#include <asm/vectors.h>
88	91	#include <asm/virt.h>
89	92
90	93	/* Kernel representation of AT_HWCAP and AT_HWCAP2 */
@@ -110,6 +113,8 @@ DECLARE_BITMAP(boot_capabilities, ARM64_NPATCHABLE); · 19 unmodified lines
110	113	bool arm64_use_ng_mappings = false;
111	114	EXPORT_SYMBOL(arm64_use_ng_mappings);
112	115
	116	+DEFINE_PER_CPU_READ_MOSTLY(const char *, this_cpu_vector) = vectors;
	117	+
113	118	/*
114	119	* Permit PER_LINUX32 and execve() of 32-bit binaries even if not all CPUs
115	120	* support it?
@@ -226,6 +231,7 @@ static const struct arm64_ftr_bits ftr_id_aa64isar1[] = { · 110 unmodified lines
226	231	};
227	232
228	233	static const struct arm64_ftr_bits ftr_id_aa64isar2[] = {
	234	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_HIGHER_SAFE, ID_AA64ISAR2_CLEARBHB_SHIFT, 4, 0),
229	235	ARM64_FTR_BITS(FTR_VISIBLE, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64ISAR2_RPRES_SHIFT, 4, 0),
230	236	ARM64_FTR_END,
231	237	};
@@ -1590,6 +1596,12 @@ kpti_install_ng_mappings(const struct arm64_cpu_capabilities *__unused) · 1358 unmodified lines
1590	1596
1591	1597	int cpu = smp_processor_id();
1592	1598
	1599	+ if (__this_cpu_read(this_cpu_vector) == vectors) {
	1600	+ const char *v = arm64_get_bp_hardening_vector(EL1_VECTOR_KPTI);
	1601	+
	1602	+ __this_cpu_write(this_cpu_vector, v);
	1603	+ }
	1604	+
1593	1605	/*
1594	1606	* We don't need to rewrite the page-tables if either we've done
1595	1607	* it already or we have KASLR enabled and therefore have not

arch/arm64/kernel/entry.S+157−57

@@ -37,18 +37,21 @@ · 36 unmodified lines
37	37
38	38	.macro kernel_ventry, el:req, ht:req, regsize:req, label:req
39	39	.align 7
40		−#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
	40	+.Lventry_start\@:
41	41	.if \el == 0
42		−alternative_if ARM64_UNMAP_KERNEL_AT_EL0
	42	+ /*
	43	+ * This must be the first instruction of the EL0 vector entries. It is
	44	+ * skipped by the trampoline vectors, to trigger the cleanup.
	45	+ */
	46	+ b .Lskip_tramp_vectors_cleanup\@
43	47	.if \regsize == 64
44	48	mrs x30, tpidrro_el0
45	49	msr tpidrro_el0, xzr
46	50	.else
47	51	mov x30, xzr
48	52	.endif
49		−alternative_else_nop_endif
	53	+.Lskip_tramp_vectors_cleanup\@:
50	54	.endif
51		−#endif
52	55
53	56	sub sp, sp, #PT_REGS_SIZE
54	57	#ifdef CONFIG_VMAP_STACK
@@ -95,11 +98,15 @@ alternative_else_nop_endif · 40 unmodified lines
95	98	mrs x0, tpidrro_el0
96	99	#endif
97	100	b el\el\ht\()_\regsize\()_\label
	101	+.org .Lventry_start\@ + 128 // Did we overflow the ventry slot?
98	102	.endm
99	103
100		− .macro tramp_alias, dst, sym
	104	+ .macro tramp_alias, dst, sym, tmp
101	105	mov_q \dst, TRAMP_VALIAS
102		− add \dst, \dst, #(\sym - .entry.tramp.text)
	106	+ adr_l \tmp, \sym
	107	+ add \dst, \dst, \tmp
	108	+ adr_l \tmp, .entry.tramp.text
	109	+ sub \dst, \dst, \tmp
103	110	.endm
104	111
105	112	/*
@@ -116,7 +123,7 @@ alternative_cb_end · 10 unmodified lines
116	123	tbnz \tmp2, #TIF_SSBD, .L__asm_ssbd_skip\@
117	124	mov w0, #ARM_SMCCC_ARCH_WORKAROUND_2
118	125	mov w1, #\state
119		−alternative_cb spectre_v4_patch_fw_mitigation_conduit
	126	+alternative_cb smccc_patch_fw_mitigation_conduit
120	127	nop // Patched to SMC/HVC #0
121	128	alternative_cb_end
122	129	.L__asm_ssbd_skip\@:
@@ -413,21 +420,26 @@ alternative_else_nop_endif · 290 unmodified lines
413	420	ldp x24, x25, [sp, #16 * 12]
414	421	ldp x26, x27, [sp, #16 * 13]
415	422	ldp x28, x29, [sp, #16 * 14]
416		− ldr lr, [sp, #S_LR]
417		− add sp, sp, #PT_REGS_SIZE // restore sp
418	423
419	424	.if \el == 0
420		−alternative_insn eret, nop, ARM64_UNMAP_KERNEL_AT_EL0
	425	+alternative_if_not ARM64_UNMAP_KERNEL_AT_EL0
	426	+ ldr lr, [sp, #S_LR]
	427	+ add sp, sp, #PT_REGS_SIZE // restore sp
	428	+ eret
	429	+alternative_else_nop_endif
421	430	#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
422	431	bne 4f
423		− msr far_el1, x30
424		− tramp_alias x30, tramp_exit_native
	432	+ msr far_el1, x29
	433	+ tramp_alias x30, tramp_exit_native, x29
425	434	br x30
426	435	4:
427		− tramp_alias x30, tramp_exit_compat
	436	+ tramp_alias x30, tramp_exit_compat, x29
428	437	br x30
429	438	#endif
430	439	.else
	440	+ ldr lr, [sp, #S_LR]
	441	+ add sp, sp, #PT_REGS_SIZE // restore sp
	442	+
431	443	/* Ensure any device/NC reads complete */
432	444	alternative_insn nop, "dmb sy", ARM64_WORKAROUND_1508412
433	445
@@ -594,12 +606,6 @@ SYM_CODE_END(ret_to_user) · 160 unmodified lines
594	606
595	607	.popsection // .entry.text
596	608
597		−#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
598		−/*
599		− * Exception vectors trampoline.
600		− */
601		− .pushsection ".entry.tramp.text", "ax"
602		−
603	609	// Move from tramp_pg_dir to swapper_pg_dir
604	610	.macro tramp_map_kernel, tmp
605	611	mrs \tmp, ttbr1_el1
@@ -633,12 +639,47 @@ alternative_else_nop_endif · 27 unmodified lines
633	639	*/
634	640	.endm
635	641
636		− .macro tramp_ventry, regsize = 64
	642	+ .macro tramp_data_page dst
	643	+ adr_l \dst, .entry.tramp.text
	644	+ sub \dst, \dst, PAGE_SIZE
	645	+ .endm
	646	+
	647	+ .macro tramp_data_read_var dst, var
	648	+#ifdef CONFIG_RANDOMIZE_BASE
	649	+ tramp_data_page \dst
	650	+ add \dst, \dst, #:lo12:__entry_tramp_data_\var
	651	+ ldr \dst, [\dst]
	652	+#else
	653	+ ldr \dst, =\var
	654	+#endif
	655	+ .endm
	656	+
	657	+#define BHB_MITIGATION_NONE 0
	658	+#define BHB_MITIGATION_LOOP 1
	659	+#define BHB_MITIGATION_FW 2
	660	+#define BHB_MITIGATION_INSN 3
	661	+
	662	+ .macro tramp_ventry, vector_start, regsize, kpti, bhb
637	663	.align 7
638	664	1:
639	665	.if \regsize == 64
640	666	msr tpidrro_el0, x30 // Restored in kernel_ventry
641	667	.endif
	668	+
	669	+ .if \bhb == BHB_MITIGATION_LOOP
	670	+ /*
	671	+ * This sequence must appear before the first indirect branch. i.e. the
	672	+ * ret out of tramp_ventry. It appears here because x30 is free.
	673	+ */
	674	+ __mitigate_spectre_bhb_loop x30
	675	+ .endif // \bhb == BHB_MITIGATION_LOOP
	676	+
	677	+ .if \bhb == BHB_MITIGATION_INSN
	678	+ clearbhb
	679	+ isb
	680	+ .endif // \bhb == BHB_MITIGATION_INSN
	681	+
	682	+ .if \kpti == 1
642	683	/*
643	684	* Defend against branch aliasing attacks by pushing a dummy
644	685	* entry onto the return stack and using a RET instruction to
@@ -648,46 +689,75 @@ alternative_else_nop_endif · 3 unmodified lines
648	689	b .
649	690	2:
650	691	tramp_map_kernel x30
651		−#ifdef CONFIG_RANDOMIZE_BASE
652		− adr x30, tramp_vectors + PAGE_SIZE
653	692	alternative_insn isb, nop, ARM64_WORKAROUND_QCOM_FALKOR_E1003
654		− ldr x30, [x30]
655		−#else
656		− ldr x30, =vectors
657		−#endif
	693	+ tramp_data_read_var x30, vectors
658	694	alternative_if_not ARM64_WORKAROUND_CAVIUM_TX2_219_PRFM
659		− prfm plil1strm, [x30, #(1b - tramp_vectors)]
	695	+ prfm plil1strm, [x30, #(1b - \vector_start)]
660	696	alternative_else_nop_endif
	697	+
661	698	msr vbar_el1, x30
662		− add x30, x30, #(1b - tramp_vectors)
663	699	isb
	700	+ .else
	701	+ ldr x30, =vectors
	702	+ .endif // \kpti == 1
	703	+
	704	+ .if \bhb == BHB_MITIGATION_FW
	705	+ /*
	706	+ * The firmware sequence must appear before the first indirect branch.
	707	+ * i.e. the ret out of tramp_ventry. But it also needs the stack to be
	708	+ * mapped to save/restore the registers the SMC clobbers.
	709	+ */
	710	+ __mitigate_spectre_bhb_fw
	711	+ .endif // \bhb == BHB_MITIGATION_FW
	712	+
	713	+ add x30, x30, #(1b - \vector_start + 4)
664	714	ret
	715	+.org 1b + 128 // Did we overflow the ventry slot?
665	716	.endm
666	717
667	718	.macro tramp_exit, regsize = 64
668		− adr x30, tramp_vectors
	719	+ tramp_data_read_var x30, this_cpu_vector
	720	+ get_this_cpu_offset x29
	721	+ ldr x30, [x30, x29]
	722	+
669	723	msr vbar_el1, x30
670		− tramp_unmap_kernel x30
	724	+ ldr lr, [sp, #S_LR]
	725	+ tramp_unmap_kernel x29
671	726	.if \regsize == 64
672		− mrs x30, far_el1
	727	+ mrs x29, far_el1
673	728	.endif
	729	+ add sp, sp, #PT_REGS_SIZE // restore sp
674	730	eret
675	731	sb
676	732	.endm
677	733
678		− .align 11
679		−SYM_CODE_START_NOALIGN(tramp_vectors)
	734	+ .macro generate_tramp_vector, kpti, bhb
	735	+.Lvector_start\@:
680	736	.space 0x400
681	737
682		− tramp_ventry
683		− tramp_ventry
684		− tramp_ventry
685		− tramp_ventry
	738	+ .rept 4
	739	+ tramp_ventry .Lvector_start\@, 64, \kpti, \bhb
	740	+ .endr
	741	+ .rept 4
	742	+ tramp_ventry .Lvector_start\@, 32, \kpti, \bhb
	743	+ .endr
	744	+ .endm
686	745
687		− tramp_ventry 32
688		− tramp_ventry 32
689		− tramp_ventry 32
690		− tramp_ventry 32
	746	+#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
	747	+/*
	748	+ * Exception vectors trampoline.
	749	+ * The order must match __bp_harden_el1_vectors and the
	750	+ * arm64_bp_harden_el1_vectors enum.
	751	+ */
	752	+ .pushsection ".entry.tramp.text", "ax"
	753	+ .align 11
	754	+SYM_CODE_START_NOALIGN(tramp_vectors)
	755	+#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY
	756	+ generate_tramp_vector kpti=1, bhb=BHB_MITIGATION_LOOP
	757	+ generate_tramp_vector kpti=1, bhb=BHB_MITIGATION_FW
	758	+ generate_tramp_vector kpti=1, bhb=BHB_MITIGATION_INSN
	759	+#endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */
	760	+ generate_tramp_vector kpti=1, bhb=BHB_MITIGATION_NONE
691	761	SYM_CODE_END(tramp_vectors)
692	762
693	763	SYM_CODE_START(tramp_exit_native)
@@ -704,12 +774,56 @@ SYM_CODE_END(tramp_exit_compat) · 10 unmodified lines
704	774	.pushsection ".rodata", "a"
705	775	.align PAGE_SHIFT
706	776	SYM_DATA_START(__entry_tramp_data_start)
	777	+__entry_tramp_data_vectors:
707	778	.quad vectors
	779	+#ifdef CONFIG_ARM_SDE_INTERFACE
	780	+__entry_tramp_data___sdei_asm_handler:
	781	+ .quad __sdei_asm_handler
	782	+#endif /* CONFIG_ARM_SDE_INTERFACE */
	783	+__entry_tramp_data_this_cpu_vector:
	784	+ .quad this_cpu_vector
708	785	SYM_DATA_END(__entry_tramp_data_start)
709	786	.popsection // .rodata
710	787	#endif /* CONFIG_RANDOMIZE_BASE */
711	788	#endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */
712	789
	790	+/*
	791	+ * Exception vectors for spectre mitigations on entry from EL1 when
	792	+ * kpti is not in use.
	793	+ */
	794	+ .macro generate_el1_vector, bhb
	795	+.Lvector_start\@:
	796	+ kernel_ventry 1, t, 64, sync // Synchronous EL1t
	797	+ kernel_ventry 1, t, 64, irq // IRQ EL1t
	798	+ kernel_ventry 1, t, 64, fiq // FIQ EL1h
	799	+ kernel_ventry 1, t, 64, error // Error EL1t
	800	+
	801	+ kernel_ventry 1, h, 64, sync // Synchronous EL1h
	802	+ kernel_ventry 1, h, 64, irq // IRQ EL1h
	803	+ kernel_ventry 1, h, 64, fiq // FIQ EL1h
	804	+ kernel_ventry 1, h, 64, error // Error EL1h
	805	+
	806	+ .rept 4
	807	+ tramp_ventry .Lvector_start\@, 64, 0, \bhb
	808	+ .endr
	809	+ .rept 4
	810	+ tramp_ventry .Lvector_start\@, 32, 0, \bhb
	811	+ .endr
	812	+ .endm
	813	+
	814	+/* The order must match tramp_vecs and the arm64_bp_harden_el1_vectors enum. */
	815	+ .pushsection ".entry.text", "ax"
	816	+ .align 11
	817	+SYM_CODE_START(__bp_harden_el1_vectors)
	818	+#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY
	819	+ generate_el1_vector bhb=BHB_MITIGATION_LOOP
	820	+ generate_el1_vector bhb=BHB_MITIGATION_FW
	821	+ generate_el1_vector bhb=BHB_MITIGATION_INSN
	822	+#endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */
	823	+SYM_CODE_END(__bp_harden_el1_vectors)
	824	+ .popsection
	825	+
	826	+
713	827	/*
714	828	* Register switch for AArch64. The callee-saved registers need to be saved
715	829	* and restored. On entry:
@@ -835,14 +949,7 @@ SYM_CODE_START(__sdei_asm_entry_trampoline) · 119 unmodified lines
835	949	* Remember whether to unmap the kernel on exit.
836	950	*/
837	951	1: str x4, [x1, #(SDEI_EVENT_INTREGS + S_SDEI_TTBR1)]
838		−
839		−#ifdef CONFIG_RANDOMIZE_BASE
840		− adr x4, tramp_vectors + PAGE_SIZE
841		− add x4, x4, #:lo12:__sdei_asm_trampoline_next_handler
842		− ldr x4, [x4]
843		−#else
844		− ldr x4, =__sdei_asm_handler
845		−#endif
	952	+ tramp_data_read_var x4, __sdei_asm_handler
846	953	br x4
847	954	SYM_CODE_END(__sdei_asm_entry_trampoline)
848	955	NOKPROBE(__sdei_asm_entry_trampoline)
@@ -865,13 +972,6 @@ SYM_CODE_END(__sdei_asm_exit_trampoline) · 16 unmodified lines
865	972	NOKPROBE(__sdei_asm_exit_trampoline)
866	973	.ltorg
867	974	.popsection // .entry.tramp.text
868		−#ifdef CONFIG_RANDOMIZE_BASE
869		−.pushsection ".rodata", "a"
870		−SYM_DATA_START(__sdei_asm_trampoline_next_handler)
871		− .quad __sdei_asm_handler
872		−SYM_DATA_END(__sdei_asm_trampoline_next_handler)
873		−.popsection // .rodata
874		−#endif /* CONFIG_RANDOMIZE_BASE */
875	975	#endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */
876	976
877	977	/*
@@ -981,7 +1081,7 @@ alternative_if_not ARM64_UNMAP_KERNEL_AT_EL0 · 103 unmodified lines
981	1081	alternative_else_nop_endif
982	1082
983	1083	#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
984		− tramp_alias dst=x5, sym=__sdei_asm_exit_trampoline
	1084	+ tramp_alias dst=x5, sym=__sdei_asm_exit_trampoline, tmp=x3
985	1085	br x5
986	1086	#endif
987	1087	SYM_CODE_END(__sdei_asm_handler)

arch/arm64/kernel/image-vars.h+4−0

@@ -66,6 +66,10 @@ KVM_NVHE_ALIAS(kvm_patch_vector_branch); · 65 unmodified lines
66	66	KVM_NVHE_ALIAS(kvm_update_va_mask);
67	67	KVM_NVHE_ALIAS(kvm_get_kimage_voffset);
68	68	KVM_NVHE_ALIAS(kvm_compute_final_ctr_el0);
	69	+KVM_NVHE_ALIAS(spectre_bhb_patch_loop_iter);
	70	+KVM_NVHE_ALIAS(spectre_bhb_patch_loop_mitigation_enable);
	71	+KVM_NVHE_ALIAS(spectre_bhb_patch_wa3);
	72	+KVM_NVHE_ALIAS(spectre_bhb_patch_clearbhb);
69	73
70	74	/* Global kernel state accessed by nVHE hyp code. */
71	75	KVM_NVHE_ALIAS(kvm_vgic_global_state);

arch/arm64/kernel/proton-pack.c+386−5

@@ -18,15 +18,18 @@ · 17 unmodified lines
18	18	*/
19	19
20	20	#include <linux/arm-smccc.h>
	21	+#include <linux/bpf.h>
21	22	#include <linux/cpu.h>
22	23	#include <linux/device.h>
23	24	#include <linux/nospec.h>
24	25	#include <linux/prctl.h>
25	26	#include <linux/sched/task_stack.h>
26	27
	28	+#include <asm/debug-monitors.h>
27	29	#include <asm/insn.h>
28	30	#include <asm/spectre.h>
29	31	#include <asm/traps.h>
	32	+#include <asm/vectors.h>
30	33	#include <asm/virt.h>
31	34
32	35	/*
@@ -96,14 +99,51 @@ static bool spectre_v2_mitigations_off(void) · 63 unmodified lines
96	99	return ret;
97	100	}
98	101
	102	+static const char *get_bhb_affected_string(enum mitigation_state bhb_state)
	103	+{
	104	+ switch (bhb_state) {
	105	+ case SPECTRE_UNAFFECTED:
	106	+ return "";
	107	+ default:
	108	+ case SPECTRE_VULNERABLE:
	109	+ return ", but not BHB";
	110	+ case SPECTRE_MITIGATED:
	111	+ return ", BHB";
	112	+ }
	113	+}
	114	+
	115	+static bool _unprivileged_ebpf_enabled(void)
	116	+{
	117	+#ifdef CONFIG_BPF_SYSCALL
	118	+ return !sysctl_unprivileged_bpf_disabled;
	119	+#else
	120	+ return false;
	121	+#endif
	122	+}
	123	+
99	124	ssize_t cpu_show_spectre_v2(struct device dev, struct device_attribute attr,
100	125	char *buf)
101	126	{
	127	+ enum mitigation_state bhb_state = arm64_get_spectre_bhb_state();
	128	+ const char *bhb_str = get_bhb_affected_string(bhb_state);
	129	+ const char *v2_str = "Branch predictor hardening";
	130	+
102	131	switch (spectre_v2_state) {
103	132	case SPECTRE_UNAFFECTED:
104		− return sprintf(buf, "Not affected\n");
	133	+ if (bhb_state == SPECTRE_UNAFFECTED)
	134	+ return sprintf(buf, "Not affected\n");
	135	+
	136	+ /*
	137	+ * Platforms affected by Spectre-BHB can't report
	138	+ * "Not affected" for Spectre-v2.
	139	+ */
	140	+ v2_str = "CSV2";
	141	+ fallthrough;
105	142	case SPECTRE_MITIGATED:
106		− return sprintf(buf, "Mitigation: Branch predictor hardening\n");
	143	+ if (bhb_state == SPECTRE_MITIGATED && _unprivileged_ebpf_enabled())
	144	+ return sprintf(buf, "Vulnerable: Unprivileged eBPF enabled\n");
	145	+
	146	+ return sprintf(buf, "Mitigation: %s%s\n", v2_str, bhb_str);
107	147	case SPECTRE_VULNERABLE:
108	148	fallthrough;
109	149	default:
@@ -554,9 +594,9 @@ void __init spectre_v4_patch_fw_mitigation_enable(struct alt_instr *alt, · 444 unmodified lines
554	594	* Patch a NOP in the Spectre-v4 mitigation code with an SMC/HVC instruction
555	595	* to call into firmware to adjust the mitigation state.
556	596	*/
557		−void __init spectre_v4_patch_fw_mitigation_conduit(struct alt_instr *alt,
558		− __le32 *origptr,
559		− __le32 *updptr, int nr_inst)
	597	+void __init smccc_patch_fw_mitigation_conduit(struct alt_instr *alt,
	598	+ __le32 *origptr,
	599	+ __le32 *updptr, int nr_inst)
560	600	{
561	601	u32 insn;
562	602
@@ -770,3 +810,344 @@ int arch_prctl_spec_ctrl_get(struct task_struct *task, unsigned long which) · 207 unmodified lines
770	810	return -ENODEV;
771	811	}
772	812	}
	813	+
	814	+/*
	815	+ * Spectre BHB.
	816	+ *
	817	+ * A CPU is either:
	818	+ * - Mitigated by a branchy loop a CPU specific number of times, and listed
	819	+ * in our "loop mitigated list".
	820	+ * - Mitigated in software by the firmware Spectre v2 call.
	821	+ * - Has the ClearBHB instruction to perform the mitigation.
	822	+ * - Has the 'Exception Clears Branch History Buffer' (ECBHB) feature, so no
	823	+ * software mitigation in the vectors is needed.
	824	+ * - Has CSV2.3, so is unaffected.
	825	+ */
	826	+static enum mitigation_state spectre_bhb_state;
	827	+
	828	+enum mitigation_state arm64_get_spectre_bhb_state(void)
	829	+{
	830	+ return spectre_bhb_state;
	831	+}
	832	+
	833	+enum bhb_mitigation_bits {
	834	+ BHB_LOOP,
	835	+ BHB_FW,
	836	+ BHB_HW,
	837	+ BHB_INSN,
	838	+};
	839	+static unsigned long system_bhb_mitigations;
	840	+
	841	+/*
	842	+ * This must be called with SCOPE_LOCAL_CPU for each type of CPU, before any
	843	+ * SCOPE_SYSTEM call will give the right answer.
	844	+ */
	845	+u8 spectre_bhb_loop_affected(int scope)
	846	+{
	847	+ u8 k = 0;
	848	+ static u8 max_bhb_k;
	849	+
	850	+ if (scope == SCOPE_LOCAL_CPU) {
	851	+ static const struct midr_range spectre_bhb_k32_list[] = {
	852	+ MIDR_ALL_VERSIONS(MIDR_CORTEX_A78),
	853	+ MIDR_ALL_VERSIONS(MIDR_CORTEX_A78C),
	854	+ MIDR_ALL_VERSIONS(MIDR_CORTEX_X1),
	855	+ MIDR_ALL_VERSIONS(MIDR_CORTEX_A710),
	856	+ MIDR_ALL_VERSIONS(MIDR_CORTEX_X2),
	857	+ MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N2),
	858	+ MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V1),
	859	+ {},
	860	+ };
	861	+ static const struct midr_range spectre_bhb_k24_list[] = {
	862	+ MIDR_ALL_VERSIONS(MIDR_CORTEX_A76),
	863	+ MIDR_ALL_VERSIONS(MIDR_CORTEX_A77),
	864	+ MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N1),
	865	+ {},
	866	+ };
	867	+ static const struct midr_range spectre_bhb_k8_list[] = {
	868	+ MIDR_ALL_VERSIONS(MIDR_CORTEX_A72),
	869	+ MIDR_ALL_VERSIONS(MIDR_CORTEX_A57),
	870	+ {},
	871	+ };
	872	+
	873	+ if (is_midr_in_range_list(read_cpuid_id(), spectre_bhb_k32_list))
	874	+ k = 32;
	875	+ else if (is_midr_in_range_list(read_cpuid_id(), spectre_bhb_k24_list))
	876	+ k = 24;
	877	+ else if (is_midr_in_range_list(read_cpuid_id(), spectre_bhb_k8_list))
	878	+ k = 8;
	879	+
	880	+ max_bhb_k = max(max_bhb_k, k);
	881	+ } else {
	882	+ k = max_bhb_k;
	883	+ }
	884	+
	885	+ return k;
	886	+}
	887	+
	888	+static enum mitigation_state spectre_bhb_get_cpu_fw_mitigation_state(void)
	889	+{
	890	+ int ret;
	891	+ struct arm_smccc_res res;
	892	+
	893	+ arm_smccc_1_1_invoke(ARM_SMCCC_ARCH_FEATURES_FUNC_ID,
	894	+ ARM_SMCCC_ARCH_WORKAROUND_3, &res);
	895	+
	896	+ ret = res.a0;
	897	+ switch (ret) {
	898	+ case SMCCC_RET_SUCCESS:
	899	+ return SPECTRE_MITIGATED;
	900	+ case SMCCC_ARCH_WORKAROUND_RET_UNAFFECTED:
	901	+ return SPECTRE_UNAFFECTED;
	902	+ default:
	903	+ fallthrough;
	904	+ case SMCCC_RET_NOT_SUPPORTED:
	905	+ return SPECTRE_VULNERABLE;
	906	+ }
	907	+}
	908	+
	909	+static bool is_spectre_bhb_fw_affected(int scope)
	910	+{
	911	+ static bool system_affected;
	912	+ enum mitigation_state fw_state;
	913	+ bool has_smccc = arm_smccc_1_1_get_conduit() != SMCCC_CONDUIT_NONE;
	914	+ static const struct midr_range spectre_bhb_firmware_mitigated_list[] = {
	915	+ MIDR_ALL_VERSIONS(MIDR_CORTEX_A73),
	916	+ MIDR_ALL_VERSIONS(MIDR_CORTEX_A75),
	917	+ {},
	918	+ };
	919	+ bool cpu_in_list = is_midr_in_range_list(read_cpuid_id(),
	920	+ spectre_bhb_firmware_mitigated_list);
	921	+
	922	+ if (scope != SCOPE_LOCAL_CPU)
	923	+ return system_affected;
	924	+
	925	+ fw_state = spectre_bhb_get_cpu_fw_mitigation_state();
	926	+ if (cpu_in_list \|\| (has_smccc && fw_state == SPECTRE_MITIGATED)) {
	927	+ system_affected = true;
	928	+ return true;
	929	+ }
	930	+
	931	+ return false;
	932	+}
	933	+
	934	+static bool supports_ecbhb(int scope)
	935	+{
	936	+ u64 mmfr1;
	937	+
	938	+ if (scope == SCOPE_LOCAL_CPU)
	939	+ mmfr1 = read_sysreg_s(SYS_ID_AA64MMFR1_EL1);
	940	+ else
	941	+ mmfr1 = read_sanitised_ftr_reg(SYS_ID_AA64MMFR1_EL1);
	942	+
	943	+ return cpuid_feature_extract_unsigned_field(mmfr1,
	944	+ ID_AA64MMFR1_ECBHB_SHIFT);
	945	+}
	946	+
	947	+bool is_spectre_bhb_affected(const struct arm64_cpu_capabilities *entry,
	948	+ int scope)
	949	+{
	950	+ WARN_ON(scope != SCOPE_LOCAL_CPU \|\| preemptible());
	951	+
	952	+ if (supports_csv2p3(scope))
	953	+ return false;
	954	+
	955	+ if (supports_clearbhb(scope))
	956	+ return true;
	957	+
	958	+ if (spectre_bhb_loop_affected(scope))
	959	+ return true;
	960	+
	961	+ if (is_spectre_bhb_fw_affected(scope))
	962	+ return true;
	963	+
	964	+ return false;
	965	+}
	966	+
	967	+static void this_cpu_set_vectors(enum arm64_bp_harden_el1_vectors slot)
	968	+{
	969	+ const char *v = arm64_get_bp_hardening_vector(slot);
	970	+
	971	+ if (slot < 0)
	972	+ return;
	973	+
	974	+ __this_cpu_write(this_cpu_vector, v);
	975	+
	976	+ /*
	977	+ * When KPTI is in use, the vectors are switched when exiting to
	978	+ * user-space.
	979	+ */
	980	+ if (arm64_kernel_unmapped_at_el0())
	981	+ return;
	982	+
	983	+ write_sysreg(v, vbar_el1);
	984	+ isb();
	985	+}
	986	+
	987	+void spectre_bhb_enable_mitigation(const struct arm64_cpu_capabilities *entry)
	988	+{
	989	+ bp_hardening_cb_t cpu_cb;
	990	+ enum mitigation_state fw_state, state = SPECTRE_VULNERABLE;
	991	+ struct bp_hardening_data *data = this_cpu_ptr(&bp_hardening_data);
	992	+
	993	+ if (!is_spectre_bhb_affected(entry, SCOPE_LOCAL_CPU))
	994	+ return;
	995	+
	996	+ if (arm64_get_spectre_v2_state() == SPECTRE_VULNERABLE) {
	997	+ /* No point mitigating Spectre-BHB alone. */
	998	+ } else if (!IS_ENABLED(CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY)) {
	999	+ pr_info_once("spectre-bhb mitigation disabled by compile time option\n");
	1000	+ } else if (cpu_mitigations_off()) {
	1001	+ pr_info_once("spectre-bhb mitigation disabled by command line option\n");
	1002	+ } else if (supports_ecbhb(SCOPE_LOCAL_CPU)) {
	1003	+ state = SPECTRE_MITIGATED;
	1004	+ set_bit(BHB_HW, &system_bhb_mitigations);
	1005	+ } else if (supports_clearbhb(SCOPE_LOCAL_CPU)) {
	1006	+ /*
	1007	+ * Ensure KVM uses the indirect vector which will have ClearBHB
	1008	+ * added.
	1009	+ */
	1010	+ if (!data->slot)
	1011	+ data->slot = HYP_VECTOR_INDIRECT;
	1012	+
	1013	+ this_cpu_set_vectors(EL1_VECTOR_BHB_CLEAR_INSN);
	1014	+ state = SPECTRE_MITIGATED;
	1015	+ set_bit(BHB_INSN, &system_bhb_mitigations);
	1016	+ } else if (spectre_bhb_loop_affected(SCOPE_LOCAL_CPU)) {
	1017	+ /*
	1018	+ * Ensure KVM uses the indirect vector which will have the
	1019	+ * branchy-loop added. A57/A72-r0 will already have selected
	1020	+ * the spectre-indirect vector, which is sufficient for BHB
	1021	+ * too.
	1022	+ */
	1023	+ if (!data->slot)
	1024	+ data->slot = HYP_VECTOR_INDIRECT;
	1025	+
	1026	+ this_cpu_set_vectors(EL1_VECTOR_BHB_LOOP);
	1027	+ state = SPECTRE_MITIGATED;
	1028	+ set_bit(BHB_LOOP, &system_bhb_mitigations);
	1029	+ } else if (is_spectre_bhb_fw_affected(SCOPE_LOCAL_CPU)) {
	1030	+ fw_state = spectre_bhb_get_cpu_fw_mitigation_state();
	1031	+ if (fw_state == SPECTRE_MITIGATED) {
	1032	+ /*
	1033	+ * Ensure KVM uses one of the spectre bp_hardening
	1034	+ * vectors. The indirect vector doesn't include the EL3
	1035	+ * call, so needs upgrading to
	1036	+ * HYP_VECTOR_SPECTRE_INDIRECT.
	1037	+ */
	1038	+ if (!data->slot \|\| data->slot == HYP_VECTOR_INDIRECT)
	1039	+ data->slot += 1;
	1040	+
	1041	+ this_cpu_set_vectors(EL1_VECTOR_BHB_FW);
	1042	+
	1043	+ /*
	1044	+ * The WA3 call in the vectors supersedes the WA1 call
	1045	+ * made during context-switch. Uninstall any firmware
	1046	+ * bp_hardening callback.
	1047	+ */
	1048	+ cpu_cb = spectre_v2_get_sw_mitigation_cb();
	1049	+ if (__this_cpu_read(bp_hardening_data.fn) != cpu_cb)
	1050	+ __this_cpu_write(bp_hardening_data.fn, NULL);
	1051	+
	1052	+ state = SPECTRE_MITIGATED;
	1053	+ set_bit(BHB_FW, &system_bhb_mitigations);
	1054	+ }
	1055	+ }
	1056	+
	1057	+ update_mitigation_state(&spectre_bhb_state, state);
	1058	+}
	1059	+
	1060	+/* Patched to NOP when enabled */
	1061	+void noinstr spectre_bhb_patch_loop_mitigation_enable(struct alt_instr *alt,
	1062	+ __le32 *origptr,
	1063	+ __le32 *updptr, int nr_inst)
	1064	+{
	1065	+ BUG_ON(nr_inst != 1);
	1066	+
	1067	+ if (test_bit(BHB_LOOP, &system_bhb_mitigations))
	1068	+ *updptr++ = cpu_to_le32(aarch64_insn_gen_nop());
	1069	+}
	1070	+
	1071	+/* Patched to NOP when enabled */
	1072	+void noinstr spectre_bhb_patch_fw_mitigation_enabled(struct alt_instr *alt,
	1073	+ __le32 *origptr,
	1074	+ __le32 *updptr, int nr_inst)
	1075	+{
	1076	+ BUG_ON(nr_inst != 1);
	1077	+
	1078	+ if (test_bit(BHB_FW, &system_bhb_mitigations))
	1079	+ *updptr++ = cpu_to_le32(aarch64_insn_gen_nop());
	1080	+}
	1081	+
	1082	+/* Patched to correct the immediate */
	1083	+void noinstr spectre_bhb_patch_loop_iter(struct alt_instr *alt,
	1084	+ __le32 origptr, __le32 updptr, int nr_inst)
	1085	+{
	1086	+ u8 rd;
	1087	+ u32 insn;
	1088	+ u16 loop_count = spectre_bhb_loop_affected(SCOPE_SYSTEM);
	1089	+
	1090	+ BUG_ON(nr_inst != 1); /* MOV -> MOV */
	1091	+
	1092	+ if (!IS_ENABLED(CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY))
	1093	+ return;
	1094	+
	1095	+ insn = le32_to_cpu(*origptr);
	1096	+ rd = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RD, insn);
	1097	+ insn = aarch64_insn_gen_movewide(rd, loop_count, 0,
	1098	+ AARCH64_INSN_VARIANT_64BIT,
	1099	+ AARCH64_INSN_MOVEWIDE_ZERO);
	1100	+ *updptr++ = cpu_to_le32(insn);
	1101	+}
	1102	+
	1103	+/* Patched to mov WA3 when supported */
	1104	+void noinstr spectre_bhb_patch_wa3(struct alt_instr *alt,
	1105	+ __le32 origptr, __le32 updptr, int nr_inst)
	1106	+{
	1107	+ u8 rd;
	1108	+ u32 insn;
	1109	+
	1110	+ BUG_ON(nr_inst != 1); /* MOV -> MOV */
	1111	+
	1112	+ if (!IS_ENABLED(CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY) \|\|
	1113	+ !test_bit(BHB_FW, &system_bhb_mitigations))
	1114	+ return;
	1115	+
	1116	+ insn = le32_to_cpu(*origptr);
	1117	+ rd = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RD, insn);
	1118	+
	1119	+ insn = aarch64_insn_gen_logical_immediate(AARCH64_INSN_LOGIC_ORR,
	1120	+ AARCH64_INSN_VARIANT_32BIT,
	1121	+ AARCH64_INSN_REG_ZR, rd,
	1122	+ ARM_SMCCC_ARCH_WORKAROUND_3);
	1123	+ if (WARN_ON_ONCE(insn == AARCH64_BREAK_FAULT))
	1124	+ return;
	1125	+
	1126	+ *updptr++ = cpu_to_le32(insn);
	1127	+}
	1128	+
	1129	+/* Patched to NOP when not supported */
	1130	+void __init spectre_bhb_patch_clearbhb(struct alt_instr *alt,
	1131	+ __le32 origptr, __le32 updptr, int nr_inst)
	1132	+{
	1133	+ BUG_ON(nr_inst != 2);
	1134	+
	1135	+ if (test_bit(BHB_INSN, &system_bhb_mitigations))
	1136	+ return;
	1137	+
	1138	+ *updptr++ = cpu_to_le32(aarch64_insn_gen_nop());
	1139	+ *updptr++ = cpu_to_le32(aarch64_insn_gen_nop());
	1140	+}
	1141	+
	1142	+#ifdef CONFIG_BPF_SYSCALL
	1143	+#define EBPF_WARN "Unprivileged eBPF is enabled, data leaks possible via Spectre v2 BHB attacks!\n"
	1144	+void unpriv_ebpf_notify(int new_state)
	1145	+{
	1146	+ if (spectre_v2_state == SPECTRE_VULNERABLE \|\|
	1147	+ spectre_bhb_state != SPECTRE_MITIGATED)
	1148	+ return;
	1149	+
	1150	+ if (!new_state)
	1151	+ pr_err("WARNING: %s", EBPF_WARN);
	1152	+}
	1153	+#endif

arch/arm64/kernel/vmlinux.lds.S+1−1

@@ -341,7 +341,7 @@ ASSERT(__hibernate_exit_text_end - (__hibernate_exit_text_start & ~(SZ_4K - 1)) · 340 unmodified lines
341	341	<= SZ_4K, "Hibernate exit text too big or misaligned")
342	342	#endif
343	343	#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
344		−ASSERT((__entry_tramp_text_end - __entry_tramp_text_start) == PAGE_SIZE,
	344	+ASSERT((__entry_tramp_text_end - __entry_tramp_text_start) <= 3*PAGE_SIZE,
345	345	"Entry trampoline text too big")
346	346	#endif
347	347	#ifdef CONFIG_KVM

arch/arm64/kvm/arm.c+1−4

@@ -1491,10 +1491,7 @@ static int kvm_init_vector_slots(void) · 1490 unmodified lines
1491	1491	base = kern_hyp_va(kvm_ksym_ref(__bp_harden_hyp_vecs));
1492	1492	kvm_init_vector_slot(base, HYP_VECTOR_SPECTRE_DIRECT);
1493	1493
1494		− if (!cpus_have_const_cap(ARM64_SPECTRE_V3A))
1495		− return 0;
1496		−
1497		− if (!has_vhe()) {
	1494	+ if (kvm_system_needs_idmapped_vectors() && !has_vhe()) {
1498	1495	err = create_hyp_exec_mappings(__pa_symbol(__bp_harden_hyp_vecs),
1499	1496	__BP_HARDEN_HYP_VECS_SZ, &base);
1500	1497	if (err)

arch/arm64/kvm/hyp/hyp-entry.S+9−0

@@ -62,6 +62,10 @@ el1_sync: // Guest trapped into EL2 · 61 unmodified lines
62	62	/* ARM_SMCCC_ARCH_WORKAROUND_2 handling */
63	63	eor w1, w1, #(ARM_SMCCC_ARCH_WORKAROUND_1 ^ \
64	64	ARM_SMCCC_ARCH_WORKAROUND_2)
	65	+ cbz w1, wa_epilogue
	66	+
	67	+ eor w1, w1, #(ARM_SMCCC_ARCH_WORKAROUND_2 ^ \
	68	+ ARM_SMCCC_ARCH_WORKAROUND_3)
65	69	cbnz w1, el1_trap
66	70
67	71	wa_epilogue:
@@ -192,7 +196,10 @@ SYM_CODE_END(__kvm_hyp_vector) · 124 unmodified lines
192	196	sub sp, sp, #(8 * 4)
193	197	stp x2, x3, [sp, #(8 * 0)]
194	198	stp x0, x1, [sp, #(8 * 2)]
	199	+ alternative_cb spectre_bhb_patch_wa3
	200	+ /* Patched to mov WA3 when supported */
195	201	mov w0, #ARM_SMCCC_ARCH_WORKAROUND_1
	202	+ alternative_cb_end
196	203	smc #0
197	204	ldp x2, x3, [sp, #(8 * 0)]
198	205	add sp, sp, #(8 * 2)
@@ -205,6 +212,8 @@ SYM_CODE_END(__kvm_hyp_vector) · 6 unmodified lines
205	212	spectrev2_smccc_wa1_smc
206	213	.else
207	214	stp x0, x1, [sp, #-16]!
	215	+ mitigate_spectre_bhb_loop x0
	216	+ mitigate_spectre_bhb_clear_insn
208	217	.endif
209	218	.if \indirect != 0
210	219	alternative_cb kvm_patch_vector_branch

arch/arm64/kvm/hyp/nvhe/mm.c+3−1

@@ -148,8 +148,10 @@ int hyp_map_vectors(void) · 147 unmodified lines
148	148	phys_addr_t phys;
149	149	void *bp_base;
150	150
151		− if (!cpus_have_const_cap(ARM64_SPECTRE_V3A))
	151	+ if (!kvm_system_needs_idmapped_vectors()) {
	152	+ __hyp_bp_vect_base = __bp_harden_hyp_vecs;
152	153	return 0;
	154	+ }
153	155
154	156	phys = __hyp_pa(__bp_harden_hyp_vecs);
155	157	bp_base = (void *)__pkvm_create_private_mapping(phys,

arch/arm64/kvm/hyp/vhe/switch.c+8−2

@@ -10,6 +10,7 @@ · 9 unmodified lines
10	10	#include <linux/kvm_host.h>
11	11	#include <linux/types.h>
12	12	#include <linux/jump_label.h>
	13	+#include <linux/percpu.h>
13	14	#include <uapi/linux/psci.h>
14	15
15	16	#include <kvm/arm_psci.h>
@@ -24,6 +25,8 @@ · 8 unmodified lines
24	25	#include <asm/fpsimd.h>
25	26	#include <asm/debug-monitors.h>
26	27	#include <asm/processor.h>
	28	+#include <asm/thread_info.h>
	29	+#include <asm/vectors.h>
27	30
28	31	/* VHE specific context */
29	32	DEFINE_PER_CPU(struct kvm_host_data, kvm_host_data);
@@ -67,7 +70,7 @@ NOKPROBE_SYMBOL(__activate_traps); · 37 unmodified lines
67	70
68	71	static void __deactivate_traps(struct kvm_vcpu *vcpu)
69	72	{
70		− extern char vectors[]; /* kernel exception vectors */
	73	+ const char *host_vectors = vectors;
71	74
72	75	___deactivate_traps(vcpu);
73	76
@@ -81,7 +84,10 @@ static void __deactivate_traps(struct kvm_vcpu *vcpu) · 7 unmodified lines
81	84	asm(ALTERNATIVE("nop", "isb", ARM64_WORKAROUND_SPECULATIVE_AT));
82	85
83	86	write_sysreg(CPACR_EL1_DEFAULT, cpacr_el1);
84		− write_sysreg(vectors, vbar_el1);
	87	+
	88	+ if (!arm64_kernel_unmapped_at_el0())
	89	+ host_vectors = __this_cpu_read(this_cpu_vector);
	90	+ write_sysreg(host_vectors, vbar_el1);
85	91	}
86	92	NOKPROBE_SYMBOL(__deactivate_traps);
87	93

arch/arm64/kvm/hypercalls.c+12−0

@@ -107,6 +107,18 @@ int kvm_hvc_call_handler(struct kvm_vcpu *vcpu) · 106 unmodified lines
107	107	break;
108	108	}
109	109	break;
	110	+ case ARM_SMCCC_ARCH_WORKAROUND_3:
	111	+ switch (arm64_get_spectre_bhb_state()) {
	112	+ case SPECTRE_VULNERABLE:
	113	+ break;
	114	+ case SPECTRE_MITIGATED:
	115	+ val[0] = SMCCC_RET_SUCCESS;
	116	+ break;
	117	+ case SPECTRE_UNAFFECTED:
	118	+ val[0] = SMCCC_ARCH_WORKAROUND_RET_UNAFFECTED;
	119	+ break;
	120	+ }
	121	+ break;
110	122	case ARM_SMCCC_HV_PV_TIME_FEATURES:
111	123	val[0] = SMCCC_RET_SUCCESS;
112	124	break;

arch/arm64/kvm/psci.c+17−1

@@ -405,7 +405,7 @@ int kvm_psci_call(struct kvm_vcpu *vcpu) · 404 unmodified lines
405	405
406	406	int kvm_arm_get_fw_num_regs(struct kvm_vcpu *vcpu)
407	407	{
408		− return 3; /* PSCI version and two workaround registers */
	408	+ return 4; /* PSCI version and three workaround registers */
409	409	}
410	410
411	411	int kvm_arm_copy_fw_reg_indices(struct kvm_vcpu vcpu, u64 __user uindices)
@@ -419,6 +419,9 @@ int kvm_arm_copy_fw_reg_indices(struct kvm_vcpu vcpu, u64 __user uindices) · 7 unmodified lines
419	419	if (put_user(KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2, uindices++))
420	420	return -EFAULT;
421	421
	422	+ if (put_user(KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3, uindices++))
	423	+ return -EFAULT;
	424	+
422	425	return 0;
423	426	}
424	427
@@ -458,6 +461,17 @@ static int get_kernel_wa_level(u64 regid) · 33 unmodified lines
458	461	case SPECTRE_VULNERABLE:
459	462	return KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_NOT_AVAIL;
460	463	}
	464	+ break;
	465	+ case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3:
	466	+ switch (arm64_get_spectre_bhb_state()) {
	467	+ case SPECTRE_VULNERABLE:
	468	+ return KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3_NOT_AVAIL;
	469	+ case SPECTRE_MITIGATED:
	470	+ return KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3_AVAIL;
	471	+ case SPECTRE_UNAFFECTED:
	472	+ return KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3_NOT_REQUIRED;
	473	+ }
	474	+ return KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3_NOT_AVAIL;
461	475	}
462	476
463	477	return -EINVAL;
@@ -474,6 +488,7 @@ int kvm_arm_get_fw_reg(struct kvm_vcpu vcpu, const struct kvm_one_reg reg) · 10 unmodified lines
474	488	break;
475	489	case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_1:
476	490	case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2:
	491	+ case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3:
477	492	val = get_kernel_wa_level(reg->id) & KVM_REG_FEATURE_LEVEL_MASK;
478	493	break;
479	494	default:
@@ -519,6 +534,7 @@ int kvm_arm_set_fw_reg(struct kvm_vcpu vcpu, const struct kvm_one_reg reg) · 39 unmodified lines
519	534	}
520	535
521	536	case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_1:
	537	+ case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3:
522	538	if (val & ~KVM_REG_FEATURE_LEVEL_MASK)
523	539	return -EINVAL;
524	540

arch/arm64/mm/mmu.c+9−3

@@ -617,6 +617,8 @@ early_param("rodata", parse_rodata); · 616 unmodified lines
617	617	#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
618	618	static int __init map_entry_trampoline(void)
619	619	{
	620	+ int i;
	621	+
620	622	pgprot_t prot = rodata_enabled ? PAGE_KERNEL_ROX : PAGE_KERNEL_EXEC;
621	623	phys_addr_t pa_start = __pa_symbol(__entry_tramp_text_start);
622	624
@@ -625,11 +627,15 @@ static int __init map_entry_trampoline(void) · 2 unmodified lines
625	627
626	628	/* Map only the text into the trampoline page table */
627	629	memset(tramp_pg_dir, 0, PGD_SIZE);
628		− __create_pgd_mapping(tramp_pg_dir, pa_start, TRAMP_VALIAS, PAGE_SIZE,
629		− prot, __pgd_pgtable_alloc, 0);
	630	+ __create_pgd_mapping(tramp_pg_dir, pa_start, TRAMP_VALIAS,
	631	+ entry_tramp_text_size(), prot,
	632	+ __pgd_pgtable_alloc, NO_BLOCK_MAPPINGS);
630	633
631	634	/* Map both the text and data into the kernel page table */
632		− __set_fixmap(FIX_ENTRY_TRAMP_TEXT, pa_start, prot);
	635	+ for (i = 0; i < DIV_ROUND_UP(entry_tramp_text_size(), PAGE_SIZE); i++)
	636	+ __set_fixmap(FIX_ENTRY_TRAMP_TEXT1 - i,
	637	+ pa_start + i * PAGE_SIZE, prot);
	638	+
633	639	if (IS_ENABLED(CONFIG_RANDOMIZE_BASE)) {
634	640	extern char __entry_tramp_data_start[];
635	641

arch/arm64/tools/cpucaps+1−0

@@ -44,6 +44,7 @@ MTE_ASYMM · 43 unmodified lines
44	44	SPECTRE_V2
45	45	SPECTRE_V3A
46	46	SPECTRE_V4
	47	+SPECTRE_BHB
47	48	SSBS
48	49	SVE
49	50	UNMAP_KERNEL_AT_EL0

include/linux/arm-smccc.h+5−0

@@ -92,6 +92,11 @@ · 91 unmodified lines
92	92	ARM_SMCCC_SMC_32, \
93	93	0, 0x7fff)
94	94
	95	+#define ARM_SMCCC_ARCH_WORKAROUND_3 \
	96	+ ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, \
	97	+ ARM_SMCCC_SMC_32, \
	98	+ 0, 0x3fff)
	99	+
95	100	#define ARM_SMCCC_VENDOR_HYP_CALL_UID_FUNC_ID \
96	101	ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, \
97	102	ARM_SMCCC_SMC_32, \