History

Merge tag 'arm64-spectre-bhb-for-v5.17-2' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux

Pull arm64 spectre fixes from James Morse: "ARM64 Spectre-BHB mitigations: - Make EL1 vectors per-cpu - Add mitigation sequences to the EL1 and EL2 vectors on vulnerble CPUs - Implement ARCH_WORKAROUND_3 for KVM guests - Report Vulnerable when unprivileged eBPF is enabled" * tag 'arm64-spectre-bhb-for-v5.17-2' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux: arm64: proton-pack: Include unprivileged eBPF status in Spectre v2 mitigation reporting arm64: Use the clearbhb instruction in mitigations KVM: arm64: Allow SMCCC_ARCH_WORKAROUND_3 to be discovered and migrated arm64: Mitigate spectre style branch history side channels arm64: proton-pack: Report Spectre-BHB vulnerabilities as part of Spectre-v2 arm64: Add percpu vectors for EL1 arm64: entry: Add macro for reading symbol addresses from the trampoline arm64: entry: Add vectors that have the bhb mitigation sequences arm64: entry: Add non-kpti __bp_harden_el1_vectors for mitigations arm64: entry: Allow the trampoline text to occupy multiple pages arm64: entry: Make the kpti trampoline's kpti sequence optional arm64: entry: Move trampoline macros out of ifdef'd section arm64: entry: Don't assume tramp_vectors is the start of the vectors arm64: entry: Allow tramp_alias to access symbols after the 4K boundary arm64: entry: Move the trampoline data page before the text page arm64: entry: Free up another register on kpti's tramp_exit path arm64: entry: Make the trampoline cleanup optional KVM: arm64: Allow indirect vectors to be used without SPECTRE_V3A arm64: spectre: Rename spectre_v4_patch_fw_mitigation_conduit arm64: entry.S: Add ventry overflow sanity checks

Linus Torvalds committed 4 years ago

Commit cd22a8b2 parents fc55c23 58c9a5027 files · +830 −76Browse files

Unified · Split

27 files
  1. arch/arm64/Kconfig +9 −0
  2. arch/arm64/include/asm/assembler.h +53 −0
  3. arch/arm64/include/asm/cpufeature.h +29 −0
  4. arch/arm64/include/asm/cputype.h +8 −0
  5. arch/arm64/include/asm/fixmap.h +4 −2
  6. arch/arm64/include/asm/insn.h +1 −0
  7. arch/arm64/include/asm/kvm_host.h +5 −0
  8. arch/arm64/include/asm/sections.h +5 −0
  9. arch/arm64/include/asm/spectre.h +4 −0
  10. arch/arm64/include/asm/sysreg.h +2 −0
  11. arch/arm64/include/asm/vectors.h +73 −0
  12. arch/arm64/include/uapi/asm/kvm.h +5 −0
  13. arch/arm64/kernel/cpu_errata.c +7 −0
  14. arch/arm64/kernel/cpufeature.c +12 −0
  15. arch/arm64/kernel/entry.S +157 −57
  16. arch/arm64/kernel/image-vars.h +4 −0
  17. arch/arm64/kernel/proton-pack.c +386 −5
  18. arch/arm64/kernel/vmlinux.lds.S +1 −1
  19. arch/arm64/kvm/arm.c +1 −4
  20. arch/arm64/kvm/hyp/hyp-entry.S +9 −0
  21. arch/arm64/kvm/hyp/nvhe/mm.c +3 −1
  22. arch/arm64/kvm/hyp/vhe/switch.c +8 −2
  23. arch/arm64/kvm/hypercalls.c +12 −0
  24. arch/arm64/kvm/psci.c +17 −1
  25. arch/arm64/mm/mmu.c +9 −3
  26. arch/arm64/tools/cpucaps +1 −0
  27. include/linux/arm-smccc.h +5 −0
arch/arm64/Kconfig+9−0
@@ -1383,6 +1383,15 @@ config UNMAP_KERNEL_AT_EL0 · 1382 unmodified lines
1383 1383
1384 If unsure, say Y.1384 If unsure, say Y.
1385 1385
1386config MITIGATE_SPECTRE_BRANCH_HISTORY
1387 bool "Mitigate Spectre style attacks against branch history" if EXPERT
1388 default y
1389 help
1390 Speculation attacks against some high-performance processors can
1391 make use of branch history to influence future speculation.
1392 When taking an exception from user-space, a sequence of branches
1393 or a firmware call overwrites the branch history.
1394
1386config RODATA_FULL_DEFAULT_ENABLED1395config RODATA_FULL_DEFAULT_ENABLED
1387 bool "Apply r/o permissions of VM areas also to their linear aliases"1396 bool "Apply r/o permissions of VM areas also to their linear aliases"
1388 default y1397 default y
arch/arm64/include/asm/assembler.h+53−0
@@ -108,6 +108,13 @@ · 107 unmodified lines
108 hint #20108 hint #20
109 .endm109 .endm
110 110
111/*
112 * Clear Branch History instruction
113 */
114 .macro clearbhb
115 hint #22
116 .endm
117
111/*118/*
112 * Speculation barrier119 * Speculation barrier
113 */120 */
@@ -850,4 +857,50 @@ alternative_endif · 736 unmodified lines
850 857
851#endif /* GNU_PROPERTY_AARCH64_FEATURE_1_DEFAULT */858#endif /* GNU_PROPERTY_AARCH64_FEATURE_1_DEFAULT */
852 859
860 .macro __mitigate_spectre_bhb_loop tmp
861#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY
862alternative_cb spectre_bhb_patch_loop_iter
863 mov \tmp, #32 // Patched to correct the immediate
864alternative_cb_end
865.Lspectre_bhb_loop\@:
866 b . + 4
867 subs \tmp, \tmp, #1
868 b.ne .Lspectre_bhb_loop\@
869 sb
870#endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */
871 .endm
872
873 .macro mitigate_spectre_bhb_loop tmp
874#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY
875alternative_cb spectre_bhb_patch_loop_mitigation_enable
876 b .L_spectre_bhb_loop_done\@ // Patched to NOP
877alternative_cb_end
878 __mitigate_spectre_bhb_loop \tmp
879.L_spectre_bhb_loop_done\@:
880#endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */
881 .endm
882
883 /* Save/restores x0-x3 to the stack */
884 .macro __mitigate_spectre_bhb_fw
885#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY
886 stp x0, x1, [sp, #-16]!
887 stp x2, x3, [sp, #-16]!
888 mov w0, #ARM_SMCCC_ARCH_WORKAROUND_3
889alternative_cb smccc_patch_fw_mitigation_conduit
890 nop // Patched to SMC/HVC #0
891alternative_cb_end
892 ldp x2, x3, [sp], #16
893 ldp x0, x1, [sp], #16
894#endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */
895 .endm
896
897 .macro mitigate_spectre_bhb_clear_insn
898#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY
899alternative_cb spectre_bhb_patch_clearbhb
900 /* Patched to NOP when not supported */
901 clearbhb
902 isb
903alternative_cb_end
904#endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */
905 .endm
853#endif /* __ASM_ASSEMBLER_H */906#endif /* __ASM_ASSEMBLER_H */
arch/arm64/include/asm/cpufeature.h+29−0
@@ -637,6 +637,35 @@ static inline bool cpu_supports_mixed_endian_el0(void) · 636 unmodified lines
637 return id_aa64mmfr0_mixed_endian_el0(read_cpuid(ID_AA64MMFR0_EL1));637 return id_aa64mmfr0_mixed_endian_el0(read_cpuid(ID_AA64MMFR0_EL1));
638}638}
639 639
640
641static inline bool supports_csv2p3(int scope)
642{
643 u64 pfr0;
644 u8 csv2_val;
645
646 if (scope == SCOPE_LOCAL_CPU)
647 pfr0 = read_sysreg_s(SYS_ID_AA64PFR0_EL1);
648 else
649 pfr0 = read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1);
650
651 csv2_val = cpuid_feature_extract_unsigned_field(pfr0,
652 ID_AA64PFR0_CSV2_SHIFT);
653 return csv2_val == 3;
654}
655
656static inline bool supports_clearbhb(int scope)
657{
658 u64 isar2;
659
660 if (scope == SCOPE_LOCAL_CPU)
661 isar2 = read_sysreg_s(SYS_ID_AA64ISAR2_EL1);
662 else
663 isar2 = read_sanitised_ftr_reg(SYS_ID_AA64ISAR2_EL1);
664
665 return cpuid_feature_extract_unsigned_field(isar2,
666 ID_AA64ISAR2_CLEARBHB_SHIFT);
667}
668
640const struct cpumask *system_32bit_el0_cpumask(void);669const struct cpumask *system_32bit_el0_cpumask(void);
641DECLARE_STATIC_KEY_FALSE(arm64_mismatched_32bit_el0);670DECLARE_STATIC_KEY_FALSE(arm64_mismatched_32bit_el0);
642 671
arch/arm64/include/asm/cputype.h+8−0
@@ -73,10 +73,14 @@ · 72 unmodified lines
73#define ARM_CPU_PART_CORTEX_A76 0xD0B73#define ARM_CPU_PART_CORTEX_A76 0xD0B
74#define ARM_CPU_PART_NEOVERSE_N1 0xD0C74#define ARM_CPU_PART_NEOVERSE_N1 0xD0C
75#define ARM_CPU_PART_CORTEX_A77 0xD0D75#define ARM_CPU_PART_CORTEX_A77 0xD0D
76#define ARM_CPU_PART_NEOVERSE_V1 0xD40
77#define ARM_CPU_PART_CORTEX_A78 0xD41
78#define ARM_CPU_PART_CORTEX_X1 0xD44
76#define ARM_CPU_PART_CORTEX_A510 0xD4679#define ARM_CPU_PART_CORTEX_A510 0xD46
77#define ARM_CPU_PART_CORTEX_A710 0xD4780#define ARM_CPU_PART_CORTEX_A710 0xD47
78#define ARM_CPU_PART_CORTEX_X2 0xD4881#define ARM_CPU_PART_CORTEX_X2 0xD48
79#define ARM_CPU_PART_NEOVERSE_N2 0xD4982#define ARM_CPU_PART_NEOVERSE_N2 0xD49
83#define ARM_CPU_PART_CORTEX_A78C 0xD4B
80 84
81#define APM_CPU_PART_POTENZA 0x00085#define APM_CPU_PART_POTENZA 0x000
82 86
@@ -117,10 +121,14 @@ · 34 unmodified lines
117#define MIDR_CORTEX_A76 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A76)121#define MIDR_CORTEX_A76 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A76)
118#define MIDR_NEOVERSE_N1 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_NEOVERSE_N1)122#define MIDR_NEOVERSE_N1 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_NEOVERSE_N1)
119#define MIDR_CORTEX_A77 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A77)123#define MIDR_CORTEX_A77 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A77)
124#define MIDR_NEOVERSE_V1 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_NEOVERSE_V1)
125#define MIDR_CORTEX_A78 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A78)
126#define MIDR_CORTEX_X1 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_X1)
120#define MIDR_CORTEX_A510 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A510)127#define MIDR_CORTEX_A510 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A510)
121#define MIDR_CORTEX_A710 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A710)128#define MIDR_CORTEX_A710 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A710)
122#define MIDR_CORTEX_X2 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_X2)129#define MIDR_CORTEX_X2 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_X2)
123#define MIDR_NEOVERSE_N2 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_NEOVERSE_N2)130#define MIDR_NEOVERSE_N2 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_NEOVERSE_N2)
131#define MIDR_CORTEX_A78C MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A78C)
124#define MIDR_THUNDERX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX)132#define MIDR_THUNDERX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX)
125#define MIDR_THUNDERX_81XX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX_81XX)133#define MIDR_THUNDERX_81XX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX_81XX)
126#define MIDR_THUNDERX_83XX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX_83XX)134#define MIDR_THUNDERX_83XX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX_83XX)
arch/arm64/include/asm/fixmap.h+4−2
@@ -62,9 +62,11 @@ enum fixed_addresses { · 61 unmodified lines
62#endif /* CONFIG_ACPI_APEI_GHES */62#endif /* CONFIG_ACPI_APEI_GHES */
63 63
64#ifdef CONFIG_UNMAP_KERNEL_AT_EL064#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
65 FIX_ENTRY_TRAMP_TEXT3,
66 FIX_ENTRY_TRAMP_TEXT2,
67 FIX_ENTRY_TRAMP_TEXT1,
65 FIX_ENTRY_TRAMP_DATA,68 FIX_ENTRY_TRAMP_DATA,
66 FIX_ENTRY_TRAMP_TEXT,69#define TRAMP_VALIAS (__fix_to_virt(FIX_ENTRY_TRAMP_TEXT1))
67#define TRAMP_VALIAS (__fix_to_virt(FIX_ENTRY_TRAMP_TEXT))
68#endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */70#endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */
69 __end_of_permanent_fixed_addresses,71 __end_of_permanent_fixed_addresses,
70 72
arch/arm64/include/asm/insn.h+1−0
@@ -65,6 +65,7 @@ enum aarch64_insn_hint_cr_op { · 64 unmodified lines
65 AARCH64_INSN_HINT_PSB = 0x11 << 5,65 AARCH64_INSN_HINT_PSB = 0x11 << 5,
66 AARCH64_INSN_HINT_TSB = 0x12 << 5,66 AARCH64_INSN_HINT_TSB = 0x12 << 5,
67 AARCH64_INSN_HINT_CSDB = 0x14 << 5,67 AARCH64_INSN_HINT_CSDB = 0x14 << 5,
68 AARCH64_INSN_HINT_CLEARBHB = 0x16 << 5,
68 69
69 AARCH64_INSN_HINT_BTI = 0x20 << 5,70 AARCH64_INSN_HINT_BTI = 0x20 << 5,
70 AARCH64_INSN_HINT_BTIC = 0x22 << 5,71 AARCH64_INSN_HINT_BTIC = 0x22 << 5,
arch/arm64/include/asm/kvm_host.h+5−0
@@ -714,6 +714,11 @@ static inline void kvm_init_host_cpu_context(struct kvm_cpu_context *cpu_ctxt) · 713 unmodified lines
714 ctxt_sys_reg(cpu_ctxt, MPIDR_EL1) = read_cpuid_mpidr();714 ctxt_sys_reg(cpu_ctxt, MPIDR_EL1) = read_cpuid_mpidr();
715}715}
716 716
717static inline bool kvm_system_needs_idmapped_vectors(void)
718{
719 return cpus_have_const_cap(ARM64_SPECTRE_V3A);
720}
721
717void kvm_arm_vcpu_ptrauth_trap(struct kvm_vcpu *vcpu);722void kvm_arm_vcpu_ptrauth_trap(struct kvm_vcpu *vcpu);
718 723
719static inline void kvm_arch_hardware_unsetup(void) {}724static inline void kvm_arch_hardware_unsetup(void) {}
arch/arm64/include/asm/sections.h+5−0
@@ -23,4 +23,9 @@ extern char __mmuoff_data_start[], __mmuoff_data_end[]; · 22 unmodified lines
23extern char __entry_tramp_text_start[], __entry_tramp_text_end[];23extern char __entry_tramp_text_start[], __entry_tramp_text_end[];
24extern char __relocate_new_kernel_start[], __relocate_new_kernel_end[];24extern char __relocate_new_kernel_start[], __relocate_new_kernel_end[];
25 25
26static inline size_t entry_tramp_text_size(void)
27{
28 return __entry_tramp_text_end - __entry_tramp_text_start;
29}
30
26#endif /* __ASM_SECTIONS_H */31#endif /* __ASM_SECTIONS_H */
arch/arm64/include/asm/spectre.h+4−0
@@ -93,5 +93,9 @@ void spectre_v4_enable_task_mitigation(struct task_struct *tsk); · 92 unmodified lines
93 93
94enum mitigation_state arm64_get_meltdown_state(void);94enum mitigation_state arm64_get_meltdown_state(void);
95 95
96enum mitigation_state arm64_get_spectre_bhb_state(void);
97bool is_spectre_bhb_affected(const struct arm64_cpu_capabilities *entry, int scope);
98u8 spectre_bhb_loop_affected(int scope);
99void spectre_bhb_enable_mitigation(const struct arm64_cpu_capabilities *__unused);
96#endif /* __ASSEMBLY__ */100#endif /* __ASSEMBLY__ */
97#endif /* __ASM_SPECTRE_H */101#endif /* __ASM_SPECTRE_H */
arch/arm64/include/asm/sysreg.h+2−0
@@ -773,6 +773,7 @@ · 772 unmodified lines
773#define ID_AA64ISAR1_GPI_IMP_DEF 0x1773#define ID_AA64ISAR1_GPI_IMP_DEF 0x1
774 774
775/* id_aa64isar2 */775/* id_aa64isar2 */
776#define ID_AA64ISAR2_CLEARBHB_SHIFT 28
776#define ID_AA64ISAR2_RPRES_SHIFT 4777#define ID_AA64ISAR2_RPRES_SHIFT 4
777#define ID_AA64ISAR2_WFXT_SHIFT 0778#define ID_AA64ISAR2_WFXT_SHIFT 0
778 779
@@ -904,6 +905,7 @@ · 125 unmodified lines
904#endif905#endif
905 906
906/* id_aa64mmfr1 */907/* id_aa64mmfr1 */
908#define ID_AA64MMFR1_ECBHB_SHIFT 60
907#define ID_AA64MMFR1_AFP_SHIFT 44909#define ID_AA64MMFR1_AFP_SHIFT 44
908#define ID_AA64MMFR1_ETS_SHIFT 36910#define ID_AA64MMFR1_ETS_SHIFT 36
909#define ID_AA64MMFR1_TWED_SHIFT 32911#define ID_AA64MMFR1_TWED_SHIFT 32
arch/arm64/include/asm/vectors.h+73−0
@@ -0,0 +1,73 @@
1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 * Copyright (C) 2022 ARM Ltd.
4 */
5#ifndef __ASM_VECTORS_H
6#define __ASM_VECTORS_H
7
8#include <linux/bug.h>
9#include <linux/percpu.h>
10
11#include <asm/fixmap.h>
12
13extern char vectors[];
14extern char tramp_vectors[];
15extern char __bp_harden_el1_vectors[];
16
17/*
18 * Note: the order of this enum corresponds to two arrays in entry.S:
19 * tramp_vecs and __bp_harden_el1_vectors. By default the canonical
20 * 'full fat' vectors are used directly.
21 */
22enum arm64_bp_harden_el1_vectors {
23#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY
24 /*
25 * Perform the BHB loop mitigation, before branching to the canonical
26 * vectors.
27 */
28 EL1_VECTOR_BHB_LOOP,
29
30 /*
31 * Make the SMC call for firmware mitigation, before branching to the
32 * canonical vectors.
33 */
34 EL1_VECTOR_BHB_FW,
35
36 /*
37 * Use the ClearBHB instruction, before branching to the canonical
38 * vectors.
39 */
40 EL1_VECTOR_BHB_CLEAR_INSN,
41#endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */
42
43 /*
44 * Remap the kernel before branching to the canonical vectors.
45 */
46 EL1_VECTOR_KPTI,
47};
48
49#ifndef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY
50#define EL1_VECTOR_BHB_LOOP -1
51#define EL1_VECTOR_BHB_FW -1
52#define EL1_VECTOR_BHB_CLEAR_INSN -1
53#endif /* !CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */
54
55/* The vectors to use on return from EL0. e.g. to remap the kernel */
56DECLARE_PER_CPU_READ_MOSTLY(const char *, this_cpu_vector);
57
58#ifndef CONFIG_UNMAP_KERNEL_AT_EL0
59#define TRAMP_VALIAS 0
60#endif
61
62static inline const char *
63arm64_get_bp_hardening_vector(enum arm64_bp_harden_el1_vectors slot)
64{
65 if (arm64_kernel_unmapped_at_el0())
66 return (char *)TRAMP_VALIAS + SZ_2K * slot;
67
68 WARN_ON_ONCE(slot == EL1_VECTOR_KPTI);
69
70 return __bp_harden_el1_vectors + SZ_2K * slot;
71}
72
73#endif /* __ASM_VECTORS_H */
arch/arm64/include/uapi/asm/kvm.h+5−0
@@ -281,6 +281,11 @@ struct kvm_arm_copy_mte_tags { · 280 unmodified lines
281#define KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_NOT_REQUIRED 3281#define KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_NOT_REQUIRED 3
282#define KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_ENABLED (1U << 4)282#define KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_ENABLED (1U << 4)
283 283
284#define KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3 KVM_REG_ARM_FW_REG(3)
285#define KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3_NOT_AVAIL 0
286#define KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3_AVAIL 1
287#define KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3_NOT_REQUIRED 2
288
284/* SVE registers */289/* SVE registers */
285#define KVM_REG_ARM64_SVE (0x15 << KVM_REG_ARM_COPROC_SHIFT)290#define KVM_REG_ARM64_SVE (0x15 << KVM_REG_ARM_COPROC_SHIFT)
286 291
arch/arm64/kernel/cpu_errata.c+7−0
@@ -502,6 +502,13 @@ const struct arm64_cpu_capabilities arm64_errata[] = { · 501 unmodified lines
502 .matches = has_spectre_v4,502 .matches = has_spectre_v4,
503 .cpu_enable = spectre_v4_enable_mitigation,503 .cpu_enable = spectre_v4_enable_mitigation,
504 },504 },
505 {
506 .desc = "Spectre-BHB",
507 .capability = ARM64_SPECTRE_BHB,
508 .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,
509 .matches = is_spectre_bhb_affected,
510 .cpu_enable = spectre_bhb_enable_mitigation,
511 },
505#ifdef CONFIG_ARM64_ERRATUM_1418040512#ifdef CONFIG_ARM64_ERRATUM_1418040
506 {513 {
507 .desc = "ARM erratum 1418040",514 .desc = "ARM erratum 1418040",
arch/arm64/kernel/cpufeature.c+12−0
@@ -73,6 +73,8 @@ · 72 unmodified lines
73#include <linux/mm.h>73#include <linux/mm.h>
74#include <linux/cpu.h>74#include <linux/cpu.h>
75#include <linux/kasan.h>75#include <linux/kasan.h>
76#include <linux/percpu.h>
77
76#include <asm/cpu.h>78#include <asm/cpu.h>
77#include <asm/cpufeature.h>79#include <asm/cpufeature.h>
78#include <asm/cpu_ops.h>80#include <asm/cpu_ops.h>
@@ -85,6 +87,7 @@ · 6 unmodified lines
85#include <asm/smp.h>87#include <asm/smp.h>
86#include <asm/sysreg.h>88#include <asm/sysreg.h>
87#include <asm/traps.h>89#include <asm/traps.h>
90#include <asm/vectors.h>
88#include <asm/virt.h>91#include <asm/virt.h>
89 92
90/* Kernel representation of AT_HWCAP and AT_HWCAP2 */93/* Kernel representation of AT_HWCAP and AT_HWCAP2 */
@@ -110,6 +113,8 @@ DECLARE_BITMAP(boot_capabilities, ARM64_NPATCHABLE); · 19 unmodified lines
110bool arm64_use_ng_mappings = false;113bool arm64_use_ng_mappings = false;
111EXPORT_SYMBOL(arm64_use_ng_mappings);114EXPORT_SYMBOL(arm64_use_ng_mappings);
112 115
116DEFINE_PER_CPU_READ_MOSTLY(const char *, this_cpu_vector) = vectors;
117
113/*118/*
114 * Permit PER_LINUX32 and execve() of 32-bit binaries even if not all CPUs119 * Permit PER_LINUX32 and execve() of 32-bit binaries even if not all CPUs
115 * support it?120 * support it?
@@ -226,6 +231,7 @@ static const struct arm64_ftr_bits ftr_id_aa64isar1[] = { · 110 unmodified lines
226};231};
227 232
228static const struct arm64_ftr_bits ftr_id_aa64isar2[] = {233static const struct arm64_ftr_bits ftr_id_aa64isar2[] = {
234 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_HIGHER_SAFE, ID_AA64ISAR2_CLEARBHB_SHIFT, 4, 0),
229 ARM64_FTR_BITS(FTR_VISIBLE, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64ISAR2_RPRES_SHIFT, 4, 0),235 ARM64_FTR_BITS(FTR_VISIBLE, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64ISAR2_RPRES_SHIFT, 4, 0),
230 ARM64_FTR_END,236 ARM64_FTR_END,
231};237};
@@ -1590,6 +1596,12 @@ kpti_install_ng_mappings(const struct arm64_cpu_capabilities *__unused) · 1358 unmodified lines
1590 1596
1591 int cpu = smp_processor_id();1597 int cpu = smp_processor_id();
1592 1598
1599 if (__this_cpu_read(this_cpu_vector) == vectors) {
1600 const char *v = arm64_get_bp_hardening_vector(EL1_VECTOR_KPTI);
1601
1602 __this_cpu_write(this_cpu_vector, v);
1603 }
1604
1593 /*1605 /*
1594 * We don't need to rewrite the page-tables if either we've done1606 * We don't need to rewrite the page-tables if either we've done
1595 * it already or we have KASLR enabled and therefore have not1607 * it already or we have KASLR enabled and therefore have not
arch/arm64/kernel/entry.S+157−57
@@ -37,18 +37,21 @@ · 36 unmodified lines
37 37
38 .macro kernel_ventry, el:req, ht:req, regsize:req, label:req38 .macro kernel_ventry, el:req, ht:req, regsize:req, label:req
39 .align 739 .align 7
40#ifdef CONFIG_UNMAP_KERNEL_AT_EL040.Lventry_start\@:
41 .if \el == 041 .if \el == 0
42alternative_if ARM64_UNMAP_KERNEL_AT_EL042 /*
43 * This must be the first instruction of the EL0 vector entries. It is
44 * skipped by the trampoline vectors, to trigger the cleanup.
45 */
46 b .Lskip_tramp_vectors_cleanup\@
43 .if \regsize == 6447 .if \regsize == 64
44 mrs x30, tpidrro_el048 mrs x30, tpidrro_el0
45 msr tpidrro_el0, xzr49 msr tpidrro_el0, xzr
46 .else50 .else
47 mov x30, xzr51 mov x30, xzr
48 .endif52 .endif
49alternative_else_nop_endif53.Lskip_tramp_vectors_cleanup\@:
50 .endif54 .endif
51#endif
52 55
53 sub sp, sp, #PT_REGS_SIZE56 sub sp, sp, #PT_REGS_SIZE
54#ifdef CONFIG_VMAP_STACK57#ifdef CONFIG_VMAP_STACK
@@ -95,11 +98,15 @@ alternative_else_nop_endif · 40 unmodified lines
95 mrs x0, tpidrro_el098 mrs x0, tpidrro_el0
96#endif99#endif
97 b el\el\ht\()_\regsize\()_\label100 b el\el\ht\()_\regsize\()_\label
101.org .Lventry_start\@ + 128 // Did we overflow the ventry slot?
98 .endm102 .endm
99 103
100 .macro tramp_alias, dst, sym104 .macro tramp_alias, dst, sym, tmp
101 mov_q \dst, TRAMP_VALIAS105 mov_q \dst, TRAMP_VALIAS
102 add \dst, \dst, #(\sym - .entry.tramp.text)106 adr_l \tmp, \sym
107 add \dst, \dst, \tmp
108 adr_l \tmp, .entry.tramp.text
109 sub \dst, \dst, \tmp
103 .endm110 .endm
104 111
105 /*112 /*
@@ -116,7 +123,7 @@ alternative_cb_end · 10 unmodified lines
116 tbnz \tmp2, #TIF_SSBD, .L__asm_ssbd_skip\@123 tbnz \tmp2, #TIF_SSBD, .L__asm_ssbd_skip\@
117 mov w0, #ARM_SMCCC_ARCH_WORKAROUND_2124 mov w0, #ARM_SMCCC_ARCH_WORKAROUND_2
118 mov w1, #\state125 mov w1, #\state
119alternative_cb spectre_v4_patch_fw_mitigation_conduit126alternative_cb smccc_patch_fw_mitigation_conduit
120 nop // Patched to SMC/HVC #0127 nop // Patched to SMC/HVC #0
121alternative_cb_end128alternative_cb_end
122.L__asm_ssbd_skip\@:129.L__asm_ssbd_skip\@:
@@ -413,21 +420,26 @@ alternative_else_nop_endif · 290 unmodified lines
413 ldp x24, x25, [sp, #16 * 12]420 ldp x24, x25, [sp, #16 * 12]
414 ldp x26, x27, [sp, #16 * 13]421 ldp x26, x27, [sp, #16 * 13]
415 ldp x28, x29, [sp, #16 * 14]422 ldp x28, x29, [sp, #16 * 14]
416 ldr lr, [sp, #S_LR]
417 add sp, sp, #PT_REGS_SIZE // restore sp
418 423
419 .if \el == 0424 .if \el == 0
420alternative_insn eret, nop, ARM64_UNMAP_KERNEL_AT_EL0425alternative_if_not ARM64_UNMAP_KERNEL_AT_EL0
426 ldr lr, [sp, #S_LR]
427 add sp, sp, #PT_REGS_SIZE // restore sp
428 eret
429alternative_else_nop_endif
421#ifdef CONFIG_UNMAP_KERNEL_AT_EL0430#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
422 bne 4f431 bne 4f
423 msr far_el1, x30432 msr far_el1, x29
424 tramp_alias x30, tramp_exit_native433 tramp_alias x30, tramp_exit_native, x29
425 br x30434 br x30
4264:4354:
427 tramp_alias x30, tramp_exit_compat436 tramp_alias x30, tramp_exit_compat, x29
428 br x30437 br x30
429#endif438#endif
430 .else439 .else
440 ldr lr, [sp, #S_LR]
441 add sp, sp, #PT_REGS_SIZE // restore sp
442
431 /* Ensure any device/NC reads complete */443 /* Ensure any device/NC reads complete */
432 alternative_insn nop, "dmb sy", ARM64_WORKAROUND_1508412444 alternative_insn nop, "dmb sy", ARM64_WORKAROUND_1508412
433 445
@@ -594,12 +606,6 @@ SYM_CODE_END(ret_to_user) · 160 unmodified lines
594 606
595 .popsection // .entry.text607 .popsection // .entry.text
596 608
597#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
598/*
599 * Exception vectors trampoline.
600 */
601 .pushsection ".entry.tramp.text", "ax"
602
603 // Move from tramp_pg_dir to swapper_pg_dir609 // Move from tramp_pg_dir to swapper_pg_dir
604 .macro tramp_map_kernel, tmp610 .macro tramp_map_kernel, tmp
605 mrs \tmp, ttbr1_el1611 mrs \tmp, ttbr1_el1
@@ -633,12 +639,47 @@ alternative_else_nop_endif · 27 unmodified lines
633 */639 */
634 .endm640 .endm
635 641
636 .macro tramp_ventry, regsize = 64642 .macro tramp_data_page dst
643 adr_l \dst, .entry.tramp.text
644 sub \dst, \dst, PAGE_SIZE
645 .endm
646
647 .macro tramp_data_read_var dst, var
648#ifdef CONFIG_RANDOMIZE_BASE
649 tramp_data_page \dst
650 add \dst, \dst, #:lo12:__entry_tramp_data_\var
651 ldr \dst, [\dst]
652#else
653 ldr \dst, =\var
654#endif
655 .endm
656
657#define BHB_MITIGATION_NONE 0
658#define BHB_MITIGATION_LOOP 1
659#define BHB_MITIGATION_FW 2
660#define BHB_MITIGATION_INSN 3
661
662 .macro tramp_ventry, vector_start, regsize, kpti, bhb
637 .align 7663 .align 7
6381:6641:
639 .if \regsize == 64665 .if \regsize == 64
640 msr tpidrro_el0, x30 // Restored in kernel_ventry666 msr tpidrro_el0, x30 // Restored in kernel_ventry
641 .endif667 .endif
668
669 .if \bhb == BHB_MITIGATION_LOOP
670 /*
671 * This sequence must appear before the first indirect branch. i.e. the
672 * ret out of tramp_ventry. It appears here because x30 is free.
673 */
674 __mitigate_spectre_bhb_loop x30
675 .endif // \bhb == BHB_MITIGATION_LOOP
676
677 .if \bhb == BHB_MITIGATION_INSN
678 clearbhb
679 isb
680 .endif // \bhb == BHB_MITIGATION_INSN
681
682 .if \kpti == 1
642 /*683 /*
643 * Defend against branch aliasing attacks by pushing a dummy684 * Defend against branch aliasing attacks by pushing a dummy
644 * entry onto the return stack and using a RET instruction to685 * entry onto the return stack and using a RET instruction to
@@ -648,46 +689,75 @@ alternative_else_nop_endif · 3 unmodified lines
648 b .689 b .
6492:6902:
650 tramp_map_kernel x30691 tramp_map_kernel x30
651#ifdef CONFIG_RANDOMIZE_BASE
652 adr x30, tramp_vectors + PAGE_SIZE
653alternative_insn isb, nop, ARM64_WORKAROUND_QCOM_FALKOR_E1003692alternative_insn isb, nop, ARM64_WORKAROUND_QCOM_FALKOR_E1003
654 ldr x30, [x30]693 tramp_data_read_var x30, vectors
655#else
656 ldr x30, =vectors
657#endif
658alternative_if_not ARM64_WORKAROUND_CAVIUM_TX2_219_PRFM694alternative_if_not ARM64_WORKAROUND_CAVIUM_TX2_219_PRFM
659 prfm plil1strm, [x30, #(1b - tramp_vectors)]695 prfm plil1strm, [x30, #(1b - \vector_start)]
660alternative_else_nop_endif696alternative_else_nop_endif
697
661 msr vbar_el1, x30698 msr vbar_el1, x30
662 add x30, x30, #(1b - tramp_vectors)
663 isb699 isb
700 .else
701 ldr x30, =vectors
702 .endif // \kpti == 1
703
704 .if \bhb == BHB_MITIGATION_FW
705 /*
706 * The firmware sequence must appear before the first indirect branch.
707 * i.e. the ret out of tramp_ventry. But it also needs the stack to be
708 * mapped to save/restore the registers the SMC clobbers.
709 */
710 __mitigate_spectre_bhb_fw
711 .endif // \bhb == BHB_MITIGATION_FW
712
713 add x30, x30, #(1b - \vector_start + 4)
664 ret714 ret
715.org 1b + 128 // Did we overflow the ventry slot?
665 .endm716 .endm
666 717
667 .macro tramp_exit, regsize = 64718 .macro tramp_exit, regsize = 64
668 adr x30, tramp_vectors719 tramp_data_read_var x30, this_cpu_vector
720 get_this_cpu_offset x29
721 ldr x30, [x30, x29]
722
669 msr vbar_el1, x30723 msr vbar_el1, x30
670 tramp_unmap_kernel x30724 ldr lr, [sp, #S_LR]
725 tramp_unmap_kernel x29
671 .if \regsize == 64726 .if \regsize == 64
672 mrs x30, far_el1727 mrs x29, far_el1
673 .endif728 .endif
729 add sp, sp, #PT_REGS_SIZE // restore sp
674 eret730 eret
675 sb731 sb
676 .endm732 .endm
677 733
678 .align 11734 .macro generate_tramp_vector, kpti, bhb
679SYM_CODE_START_NOALIGN(tramp_vectors)735.Lvector_start\@:
680 .space 0x400736 .space 0x400
681 737
682 tramp_ventry738 .rept 4
683 tramp_ventry739 tramp_ventry .Lvector_start\@, 64, \kpti, \bhb
684 tramp_ventry740 .endr
685 tramp_ventry741 .rept 4
742 tramp_ventry .Lvector_start\@, 32, \kpti, \bhb
743 .endr
744 .endm
686 745
687 tramp_ventry 32746#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
688 tramp_ventry 32747/*
689 tramp_ventry 32748 * Exception vectors trampoline.
690 tramp_ventry 32749 * The order must match __bp_harden_el1_vectors and the
750 * arm64_bp_harden_el1_vectors enum.
751 */
752 .pushsection ".entry.tramp.text", "ax"
753 .align 11
754SYM_CODE_START_NOALIGN(tramp_vectors)
755#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY
756 generate_tramp_vector kpti=1, bhb=BHB_MITIGATION_LOOP
757 generate_tramp_vector kpti=1, bhb=BHB_MITIGATION_FW
758 generate_tramp_vector kpti=1, bhb=BHB_MITIGATION_INSN
759#endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */
760 generate_tramp_vector kpti=1, bhb=BHB_MITIGATION_NONE
691SYM_CODE_END(tramp_vectors)761SYM_CODE_END(tramp_vectors)
692 762
693SYM_CODE_START(tramp_exit_native)763SYM_CODE_START(tramp_exit_native)
@@ -704,12 +774,56 @@ SYM_CODE_END(tramp_exit_compat) · 10 unmodified lines
704 .pushsection ".rodata", "a"774 .pushsection ".rodata", "a"
705 .align PAGE_SHIFT775 .align PAGE_SHIFT
706SYM_DATA_START(__entry_tramp_data_start)776SYM_DATA_START(__entry_tramp_data_start)
777__entry_tramp_data_vectors:
707 .quad vectors778 .quad vectors
779#ifdef CONFIG_ARM_SDE_INTERFACE
780__entry_tramp_data___sdei_asm_handler:
781 .quad __sdei_asm_handler
782#endif /* CONFIG_ARM_SDE_INTERFACE */
783__entry_tramp_data_this_cpu_vector:
784 .quad this_cpu_vector
708SYM_DATA_END(__entry_tramp_data_start)785SYM_DATA_END(__entry_tramp_data_start)
709 .popsection // .rodata786 .popsection // .rodata
710#endif /* CONFIG_RANDOMIZE_BASE */787#endif /* CONFIG_RANDOMIZE_BASE */
711#endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */788#endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */
712 789
790/*
791 * Exception vectors for spectre mitigations on entry from EL1 when
792 * kpti is not in use.
793 */
794 .macro generate_el1_vector, bhb
795.Lvector_start\@:
796 kernel_ventry 1, t, 64, sync // Synchronous EL1t
797 kernel_ventry 1, t, 64, irq // IRQ EL1t
798 kernel_ventry 1, t, 64, fiq // FIQ EL1h
799 kernel_ventry 1, t, 64, error // Error EL1t
800
801 kernel_ventry 1, h, 64, sync // Synchronous EL1h
802 kernel_ventry 1, h, 64, irq // IRQ EL1h
803 kernel_ventry 1, h, 64, fiq // FIQ EL1h
804 kernel_ventry 1, h, 64, error // Error EL1h
805
806 .rept 4
807 tramp_ventry .Lvector_start\@, 64, 0, \bhb
808 .endr
809 .rept 4
810 tramp_ventry .Lvector_start\@, 32, 0, \bhb
811 .endr
812 .endm
813
814/* The order must match tramp_vecs and the arm64_bp_harden_el1_vectors enum. */
815 .pushsection ".entry.text", "ax"
816 .align 11
817SYM_CODE_START(__bp_harden_el1_vectors)
818#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY
819 generate_el1_vector bhb=BHB_MITIGATION_LOOP
820 generate_el1_vector bhb=BHB_MITIGATION_FW
821 generate_el1_vector bhb=BHB_MITIGATION_INSN
822#endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */
823SYM_CODE_END(__bp_harden_el1_vectors)
824 .popsection
825
826
713/*827/*
714 * Register switch for AArch64. The callee-saved registers need to be saved828 * Register switch for AArch64. The callee-saved registers need to be saved
715 * and restored. On entry:829 * and restored. On entry:
@@ -835,14 +949,7 @@ SYM_CODE_START(__sdei_asm_entry_trampoline) · 119 unmodified lines
835 * Remember whether to unmap the kernel on exit.949 * Remember whether to unmap the kernel on exit.
836 */950 */
8371: str x4, [x1, #(SDEI_EVENT_INTREGS + S_SDEI_TTBR1)]9511: str x4, [x1, #(SDEI_EVENT_INTREGS + S_SDEI_TTBR1)]
838 952 tramp_data_read_var x4, __sdei_asm_handler
839#ifdef CONFIG_RANDOMIZE_BASE
840 adr x4, tramp_vectors + PAGE_SIZE
841 add x4, x4, #:lo12:__sdei_asm_trampoline_next_handler
842 ldr x4, [x4]
843#else
844 ldr x4, =__sdei_asm_handler
845#endif
846 br x4953 br x4
847SYM_CODE_END(__sdei_asm_entry_trampoline)954SYM_CODE_END(__sdei_asm_entry_trampoline)
848NOKPROBE(__sdei_asm_entry_trampoline)955NOKPROBE(__sdei_asm_entry_trampoline)
@@ -865,13 +972,6 @@ SYM_CODE_END(__sdei_asm_exit_trampoline) · 16 unmodified lines
865NOKPROBE(__sdei_asm_exit_trampoline)972NOKPROBE(__sdei_asm_exit_trampoline)
866 .ltorg973 .ltorg
867.popsection // .entry.tramp.text974.popsection // .entry.tramp.text
868#ifdef CONFIG_RANDOMIZE_BASE
869.pushsection ".rodata", "a"
870SYM_DATA_START(__sdei_asm_trampoline_next_handler)
871 .quad __sdei_asm_handler
872SYM_DATA_END(__sdei_asm_trampoline_next_handler)
873.popsection // .rodata
874#endif /* CONFIG_RANDOMIZE_BASE */
875#endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */975#endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */
876 976
877/*977/*
@@ -981,7 +1081,7 @@ alternative_if_not ARM64_UNMAP_KERNEL_AT_EL0 · 103 unmodified lines
981alternative_else_nop_endif1081alternative_else_nop_endif
982 1082
983#ifdef CONFIG_UNMAP_KERNEL_AT_EL01083#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
984 tramp_alias dst=x5, sym=__sdei_asm_exit_trampoline1084 tramp_alias dst=x5, sym=__sdei_asm_exit_trampoline, tmp=x3
985 br x51085 br x5
986#endif1086#endif
987SYM_CODE_END(__sdei_asm_handler)1087SYM_CODE_END(__sdei_asm_handler)
arch/arm64/kernel/image-vars.h+4−0
@@ -66,6 +66,10 @@ KVM_NVHE_ALIAS(kvm_patch_vector_branch); · 65 unmodified lines
66KVM_NVHE_ALIAS(kvm_update_va_mask);66KVM_NVHE_ALIAS(kvm_update_va_mask);
67KVM_NVHE_ALIAS(kvm_get_kimage_voffset);67KVM_NVHE_ALIAS(kvm_get_kimage_voffset);
68KVM_NVHE_ALIAS(kvm_compute_final_ctr_el0);68KVM_NVHE_ALIAS(kvm_compute_final_ctr_el0);
69KVM_NVHE_ALIAS(spectre_bhb_patch_loop_iter);
70KVM_NVHE_ALIAS(spectre_bhb_patch_loop_mitigation_enable);
71KVM_NVHE_ALIAS(spectre_bhb_patch_wa3);
72KVM_NVHE_ALIAS(spectre_bhb_patch_clearbhb);
69 73
70/* Global kernel state accessed by nVHE hyp code. */74/* Global kernel state accessed by nVHE hyp code. */
71KVM_NVHE_ALIAS(kvm_vgic_global_state);75KVM_NVHE_ALIAS(kvm_vgic_global_state);
arch/arm64/kernel/proton-pack.c+386−5
@@ -18,15 +18,18 @@ · 17 unmodified lines
18 */18 */
19 19
20#include <linux/arm-smccc.h>20#include <linux/arm-smccc.h>
21#include <linux/bpf.h>
21#include <linux/cpu.h>22#include <linux/cpu.h>
22#include <linux/device.h>23#include <linux/device.h>
23#include <linux/nospec.h>24#include <linux/nospec.h>
24#include <linux/prctl.h>25#include <linux/prctl.h>
25#include <linux/sched/task_stack.h>26#include <linux/sched/task_stack.h>
26 27
28#include <asm/debug-monitors.h>
27#include <asm/insn.h>29#include <asm/insn.h>
28#include <asm/spectre.h>30#include <asm/spectre.h>
29#include <asm/traps.h>31#include <asm/traps.h>
32#include <asm/vectors.h>
30#include <asm/virt.h>33#include <asm/virt.h>
31 34
32/*35/*
@@ -96,14 +99,51 @@ static bool spectre_v2_mitigations_off(void) · 63 unmodified lines
96 return ret;99 return ret;
97}100}
98 101
102static const char *get_bhb_affected_string(enum mitigation_state bhb_state)
103{
104 switch (bhb_state) {
105 case SPECTRE_UNAFFECTED:
106 return "";
107 default:
108 case SPECTRE_VULNERABLE:
109 return ", but not BHB";
110 case SPECTRE_MITIGATED:
111 return ", BHB";
112 }
113}
114
115static bool _unprivileged_ebpf_enabled(void)
116{
117#ifdef CONFIG_BPF_SYSCALL
118 return !sysctl_unprivileged_bpf_disabled;
119#else
120 return false;
121#endif
122}
123
99ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr,124ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr,
100 char *buf)125 char *buf)
101{126{
127 enum mitigation_state bhb_state = arm64_get_spectre_bhb_state();
128 const char *bhb_str = get_bhb_affected_string(bhb_state);
129 const char *v2_str = "Branch predictor hardening";
130
102 switch (spectre_v2_state) {131 switch (spectre_v2_state) {
103 case SPECTRE_UNAFFECTED:132 case SPECTRE_UNAFFECTED:
104 return sprintf(buf, "Not affected\n");133 if (bhb_state == SPECTRE_UNAFFECTED)
134 return sprintf(buf, "Not affected\n");
135
136 /*
137 * Platforms affected by Spectre-BHB can't report
138 * "Not affected" for Spectre-v2.
139 */
140 v2_str = "CSV2";
141 fallthrough;
105 case SPECTRE_MITIGATED:142 case SPECTRE_MITIGATED:
106 return sprintf(buf, "Mitigation: Branch predictor hardening\n");143 if (bhb_state == SPECTRE_MITIGATED && _unprivileged_ebpf_enabled())
144 return sprintf(buf, "Vulnerable: Unprivileged eBPF enabled\n");
145
146 return sprintf(buf, "Mitigation: %s%s\n", v2_str, bhb_str);
107 case SPECTRE_VULNERABLE:147 case SPECTRE_VULNERABLE:
108 fallthrough;148 fallthrough;
109 default:149 default:
@@ -554,9 +594,9 @@ void __init spectre_v4_patch_fw_mitigation_enable(struct alt_instr *alt, · 444 unmodified lines
554 * Patch a NOP in the Spectre-v4 mitigation code with an SMC/HVC instruction594 * Patch a NOP in the Spectre-v4 mitigation code with an SMC/HVC instruction
555 * to call into firmware to adjust the mitigation state.595 * to call into firmware to adjust the mitigation state.
556 */596 */
557void __init spectre_v4_patch_fw_mitigation_conduit(struct alt_instr *alt,597void __init smccc_patch_fw_mitigation_conduit(struct alt_instr *alt,
558 __le32 *origptr,598 __le32 *origptr,
559 __le32 *updptr, int nr_inst)599 __le32 *updptr, int nr_inst)
560{600{
561 u32 insn;601 u32 insn;
562 602
@@ -770,3 +810,344 @@ int arch_prctl_spec_ctrl_get(struct task_struct *task, unsigned long which) · 207 unmodified lines
770 return -ENODEV;810 return -ENODEV;
771 }811 }
772}812}
813
814/*
815 * Spectre BHB.
816 *
817 * A CPU is either:
818 * - Mitigated by a branchy loop a CPU specific number of times, and listed
819 * in our "loop mitigated list".
820 * - Mitigated in software by the firmware Spectre v2 call.
821 * - Has the ClearBHB instruction to perform the mitigation.
822 * - Has the 'Exception Clears Branch History Buffer' (ECBHB) feature, so no
823 * software mitigation in the vectors is needed.
824 * - Has CSV2.3, so is unaffected.
825 */
826static enum mitigation_state spectre_bhb_state;
827
828enum mitigation_state arm64_get_spectre_bhb_state(void)
829{
830 return spectre_bhb_state;
831}
832
833enum bhb_mitigation_bits {
834 BHB_LOOP,
835 BHB_FW,
836 BHB_HW,
837 BHB_INSN,
838};
839static unsigned long system_bhb_mitigations;
840
841/*
842 * This must be called with SCOPE_LOCAL_CPU for each type of CPU, before any
843 * SCOPE_SYSTEM call will give the right answer.
844 */
845u8 spectre_bhb_loop_affected(int scope)
846{
847 u8 k = 0;
848 static u8 max_bhb_k;
849
850 if (scope == SCOPE_LOCAL_CPU) {
851 static const struct midr_range spectre_bhb_k32_list[] = {
852 MIDR_ALL_VERSIONS(MIDR_CORTEX_A78),
853 MIDR_ALL_VERSIONS(MIDR_CORTEX_A78C),
854 MIDR_ALL_VERSIONS(MIDR_CORTEX_X1),
855 MIDR_ALL_VERSIONS(MIDR_CORTEX_A710),
856 MIDR_ALL_VERSIONS(MIDR_CORTEX_X2),
857 MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N2),
858 MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V1),
859 {},
860 };
861 static const struct midr_range spectre_bhb_k24_list[] = {
862 MIDR_ALL_VERSIONS(MIDR_CORTEX_A76),
863 MIDR_ALL_VERSIONS(MIDR_CORTEX_A77),
864 MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N1),
865 {},
866 };
867 static const struct midr_range spectre_bhb_k8_list[] = {
868 MIDR_ALL_VERSIONS(MIDR_CORTEX_A72),
869 MIDR_ALL_VERSIONS(MIDR_CORTEX_A57),
870 {},
871 };
872
873 if (is_midr_in_range_list(read_cpuid_id(), spectre_bhb_k32_list))
874 k = 32;
875 else if (is_midr_in_range_list(read_cpuid_id(), spectre_bhb_k24_list))
876 k = 24;
877 else if (is_midr_in_range_list(read_cpuid_id(), spectre_bhb_k8_list))
878 k = 8;
879
880 max_bhb_k = max(max_bhb_k, k);
881 } else {
882 k = max_bhb_k;
883 }
884
885 return k;
886}
887
888static enum mitigation_state spectre_bhb_get_cpu_fw_mitigation_state(void)
889{
890 int ret;
891 struct arm_smccc_res res;
892
893 arm_smccc_1_1_invoke(ARM_SMCCC_ARCH_FEATURES_FUNC_ID,
894 ARM_SMCCC_ARCH_WORKAROUND_3, &res);
895
896 ret = res.a0;
897 switch (ret) {
898 case SMCCC_RET_SUCCESS:
899 return SPECTRE_MITIGATED;
900 case SMCCC_ARCH_WORKAROUND_RET_UNAFFECTED:
901 return SPECTRE_UNAFFECTED;
902 default:
903 fallthrough;
904 case SMCCC_RET_NOT_SUPPORTED:
905 return SPECTRE_VULNERABLE;
906 }
907}
908
909static bool is_spectre_bhb_fw_affected(int scope)
910{
911 static bool system_affected;
912 enum mitigation_state fw_state;
913 bool has_smccc = arm_smccc_1_1_get_conduit() != SMCCC_CONDUIT_NONE;
914 static const struct midr_range spectre_bhb_firmware_mitigated_list[] = {
915 MIDR_ALL_VERSIONS(MIDR_CORTEX_A73),
916 MIDR_ALL_VERSIONS(MIDR_CORTEX_A75),
917 {},
918 };
919 bool cpu_in_list = is_midr_in_range_list(read_cpuid_id(),
920 spectre_bhb_firmware_mitigated_list);
921
922 if (scope != SCOPE_LOCAL_CPU)
923 return system_affected;
924
925 fw_state = spectre_bhb_get_cpu_fw_mitigation_state();
926 if (cpu_in_list || (has_smccc && fw_state == SPECTRE_MITIGATED)) {
927 system_affected = true;
928 return true;
929 }
930
931 return false;
932}
933
934static bool supports_ecbhb(int scope)
935{
936 u64 mmfr1;
937
938 if (scope == SCOPE_LOCAL_CPU)
939 mmfr1 = read_sysreg_s(SYS_ID_AA64MMFR1_EL1);
940 else
941 mmfr1 = read_sanitised_ftr_reg(SYS_ID_AA64MMFR1_EL1);
942
943 return cpuid_feature_extract_unsigned_field(mmfr1,
944 ID_AA64MMFR1_ECBHB_SHIFT);
945}
946
947bool is_spectre_bhb_affected(const struct arm64_cpu_capabilities *entry,
948 int scope)
949{
950 WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible());
951
952 if (supports_csv2p3(scope))
953 return false;
954
955 if (supports_clearbhb(scope))
956 return true;
957
958 if (spectre_bhb_loop_affected(scope))
959 return true;
960
961 if (is_spectre_bhb_fw_affected(scope))
962 return true;
963
964 return false;
965}
966
967static void this_cpu_set_vectors(enum arm64_bp_harden_el1_vectors slot)
968{
969 const char *v = arm64_get_bp_hardening_vector(slot);
970
971 if (slot < 0)
972 return;
973
974 __this_cpu_write(this_cpu_vector, v);
975
976 /*
977 * When KPTI is in use, the vectors are switched when exiting to
978 * user-space.
979 */
980 if (arm64_kernel_unmapped_at_el0())
981 return;
982
983 write_sysreg(v, vbar_el1);
984 isb();
985}
986
987void spectre_bhb_enable_mitigation(const struct arm64_cpu_capabilities *entry)
988{
989 bp_hardening_cb_t cpu_cb;
990 enum mitigation_state fw_state, state = SPECTRE_VULNERABLE;
991 struct bp_hardening_data *data = this_cpu_ptr(&bp_hardening_data);
992
993 if (!is_spectre_bhb_affected(entry, SCOPE_LOCAL_CPU))
994 return;
995
996 if (arm64_get_spectre_v2_state() == SPECTRE_VULNERABLE) {
997 /* No point mitigating Spectre-BHB alone. */
998 } else if (!IS_ENABLED(CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY)) {
999 pr_info_once("spectre-bhb mitigation disabled by compile time option\n");
1000 } else if (cpu_mitigations_off()) {
1001 pr_info_once("spectre-bhb mitigation disabled by command line option\n");
1002 } else if (supports_ecbhb(SCOPE_LOCAL_CPU)) {
1003 state = SPECTRE_MITIGATED;
1004 set_bit(BHB_HW, &system_bhb_mitigations);
1005 } else if (supports_clearbhb(SCOPE_LOCAL_CPU)) {
1006 /*
1007 * Ensure KVM uses the indirect vector which will have ClearBHB
1008 * added.
1009 */
1010 if (!data->slot)
1011 data->slot = HYP_VECTOR_INDIRECT;
1012
1013 this_cpu_set_vectors(EL1_VECTOR_BHB_CLEAR_INSN);
1014 state = SPECTRE_MITIGATED;
1015 set_bit(BHB_INSN, &system_bhb_mitigations);
1016 } else if (spectre_bhb_loop_affected(SCOPE_LOCAL_CPU)) {
1017 /*
1018 * Ensure KVM uses the indirect vector which will have the
1019 * branchy-loop added. A57/A72-r0 will already have selected
1020 * the spectre-indirect vector, which is sufficient for BHB
1021 * too.
1022 */
1023 if (!data->slot)
1024 data->slot = HYP_VECTOR_INDIRECT;
1025
1026 this_cpu_set_vectors(EL1_VECTOR_BHB_LOOP);
1027 state = SPECTRE_MITIGATED;
1028 set_bit(BHB_LOOP, &system_bhb_mitigations);
1029 } else if (is_spectre_bhb_fw_affected(SCOPE_LOCAL_CPU)) {
1030 fw_state = spectre_bhb_get_cpu_fw_mitigation_state();
1031 if (fw_state == SPECTRE_MITIGATED) {
1032 /*
1033 * Ensure KVM uses one of the spectre bp_hardening
1034 * vectors. The indirect vector doesn't include the EL3
1035 * call, so needs upgrading to
1036 * HYP_VECTOR_SPECTRE_INDIRECT.
1037 */
1038 if (!data->slot || data->slot == HYP_VECTOR_INDIRECT)
1039 data->slot += 1;
1040
1041 this_cpu_set_vectors(EL1_VECTOR_BHB_FW);
1042
1043 /*
1044 * The WA3 call in the vectors supersedes the WA1 call
1045 * made during context-switch. Uninstall any firmware
1046 * bp_hardening callback.
1047 */
1048 cpu_cb = spectre_v2_get_sw_mitigation_cb();
1049 if (__this_cpu_read(bp_hardening_data.fn) != cpu_cb)
1050 __this_cpu_write(bp_hardening_data.fn, NULL);
1051
1052 state = SPECTRE_MITIGATED;
1053 set_bit(BHB_FW, &system_bhb_mitigations);
1054 }
1055 }
1056
1057 update_mitigation_state(&spectre_bhb_state, state);
1058}
1059
1060/* Patched to NOP when enabled */
1061void noinstr spectre_bhb_patch_loop_mitigation_enable(struct alt_instr *alt,
1062 __le32 *origptr,
1063 __le32 *updptr, int nr_inst)
1064{
1065 BUG_ON(nr_inst != 1);
1066
1067 if (test_bit(BHB_LOOP, &system_bhb_mitigations))
1068 *updptr++ = cpu_to_le32(aarch64_insn_gen_nop());
1069}
1070
1071/* Patched to NOP when enabled */
1072void noinstr spectre_bhb_patch_fw_mitigation_enabled(struct alt_instr *alt,
1073 __le32 *origptr,
1074 __le32 *updptr, int nr_inst)
1075{
1076 BUG_ON(nr_inst != 1);
1077
1078 if (test_bit(BHB_FW, &system_bhb_mitigations))
1079 *updptr++ = cpu_to_le32(aarch64_insn_gen_nop());
1080}
1081
1082/* Patched to correct the immediate */
1083void noinstr spectre_bhb_patch_loop_iter(struct alt_instr *alt,
1084 __le32 *origptr, __le32 *updptr, int nr_inst)
1085{
1086 u8 rd;
1087 u32 insn;
1088 u16 loop_count = spectre_bhb_loop_affected(SCOPE_SYSTEM);
1089
1090 BUG_ON(nr_inst != 1); /* MOV -> MOV */
1091
1092 if (!IS_ENABLED(CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY))
1093 return;
1094
1095 insn = le32_to_cpu(*origptr);
1096 rd = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RD, insn);
1097 insn = aarch64_insn_gen_movewide(rd, loop_count, 0,
1098 AARCH64_INSN_VARIANT_64BIT,
1099 AARCH64_INSN_MOVEWIDE_ZERO);
1100 *updptr++ = cpu_to_le32(insn);
1101}
1102
1103/* Patched to mov WA3 when supported */
1104void noinstr spectre_bhb_patch_wa3(struct alt_instr *alt,
1105 __le32 *origptr, __le32 *updptr, int nr_inst)
1106{
1107 u8 rd;
1108 u32 insn;
1109
1110 BUG_ON(nr_inst != 1); /* MOV -> MOV */
1111
1112 if (!IS_ENABLED(CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY) ||
1113 !test_bit(BHB_FW, &system_bhb_mitigations))
1114 return;
1115
1116 insn = le32_to_cpu(*origptr);
1117 rd = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RD, insn);
1118
1119 insn = aarch64_insn_gen_logical_immediate(AARCH64_INSN_LOGIC_ORR,
1120 AARCH64_INSN_VARIANT_32BIT,
1121 AARCH64_INSN_REG_ZR, rd,
1122 ARM_SMCCC_ARCH_WORKAROUND_3);
1123 if (WARN_ON_ONCE(insn == AARCH64_BREAK_FAULT))
1124 return;
1125
1126 *updptr++ = cpu_to_le32(insn);
1127}
1128
1129/* Patched to NOP when not supported */
1130void __init spectre_bhb_patch_clearbhb(struct alt_instr *alt,
1131 __le32 *origptr, __le32 *updptr, int nr_inst)
1132{
1133 BUG_ON(nr_inst != 2);
1134
1135 if (test_bit(BHB_INSN, &system_bhb_mitigations))
1136 return;
1137
1138 *updptr++ = cpu_to_le32(aarch64_insn_gen_nop());
1139 *updptr++ = cpu_to_le32(aarch64_insn_gen_nop());
1140}
1141
1142#ifdef CONFIG_BPF_SYSCALL
1143#define EBPF_WARN "Unprivileged eBPF is enabled, data leaks possible via Spectre v2 BHB attacks!\n"
1144void unpriv_ebpf_notify(int new_state)
1145{
1146 if (spectre_v2_state == SPECTRE_VULNERABLE ||
1147 spectre_bhb_state != SPECTRE_MITIGATED)
1148 return;
1149
1150 if (!new_state)
1151 pr_err("WARNING: %s", EBPF_WARN);
1152}
1153#endif
arch/arm64/kernel/vmlinux.lds.S+1−1
@@ -341,7 +341,7 @@ ASSERT(__hibernate_exit_text_end - (__hibernate_exit_text_start & ~(SZ_4K - 1)) · 340 unmodified lines
341 <= SZ_4K, "Hibernate exit text too big or misaligned")341 <= SZ_4K, "Hibernate exit text too big or misaligned")
342#endif342#endif
343#ifdef CONFIG_UNMAP_KERNEL_AT_EL0343#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
344ASSERT((__entry_tramp_text_end - __entry_tramp_text_start) == PAGE_SIZE,344ASSERT((__entry_tramp_text_end - __entry_tramp_text_start) <= 3*PAGE_SIZE,
345 "Entry trampoline text too big")345 "Entry trampoline text too big")
346#endif346#endif
347#ifdef CONFIG_KVM347#ifdef CONFIG_KVM
arch/arm64/kvm/arm.c+1−4
@@ -1491,10 +1491,7 @@ static int kvm_init_vector_slots(void) · 1490 unmodified lines
1491 base = kern_hyp_va(kvm_ksym_ref(__bp_harden_hyp_vecs));1491 base = kern_hyp_va(kvm_ksym_ref(__bp_harden_hyp_vecs));
1492 kvm_init_vector_slot(base, HYP_VECTOR_SPECTRE_DIRECT);1492 kvm_init_vector_slot(base, HYP_VECTOR_SPECTRE_DIRECT);
1493 1493
1494 if (!cpus_have_const_cap(ARM64_SPECTRE_V3A))1494 if (kvm_system_needs_idmapped_vectors() && !has_vhe()) {
1495 return 0;
1496
1497 if (!has_vhe()) {
1498 err = create_hyp_exec_mappings(__pa_symbol(__bp_harden_hyp_vecs),1495 err = create_hyp_exec_mappings(__pa_symbol(__bp_harden_hyp_vecs),
1499 __BP_HARDEN_HYP_VECS_SZ, &base);1496 __BP_HARDEN_HYP_VECS_SZ, &base);
1500 if (err)1497 if (err)
arch/arm64/kvm/hyp/hyp-entry.S+9−0
@@ -62,6 +62,10 @@ el1_sync: // Guest trapped into EL2 · 61 unmodified lines
62 /* ARM_SMCCC_ARCH_WORKAROUND_2 handling */62 /* ARM_SMCCC_ARCH_WORKAROUND_2 handling */
63 eor w1, w1, #(ARM_SMCCC_ARCH_WORKAROUND_1 ^ \63 eor w1, w1, #(ARM_SMCCC_ARCH_WORKAROUND_1 ^ \
64 ARM_SMCCC_ARCH_WORKAROUND_2)64 ARM_SMCCC_ARCH_WORKAROUND_2)
65 cbz w1, wa_epilogue
66
67 eor w1, w1, #(ARM_SMCCC_ARCH_WORKAROUND_2 ^ \
68 ARM_SMCCC_ARCH_WORKAROUND_3)
65 cbnz w1, el1_trap69 cbnz w1, el1_trap
66 70
67wa_epilogue:71wa_epilogue:
@@ -192,7 +196,10 @@ SYM_CODE_END(__kvm_hyp_vector) · 124 unmodified lines
192 sub sp, sp, #(8 * 4)196 sub sp, sp, #(8 * 4)
193 stp x2, x3, [sp, #(8 * 0)]197 stp x2, x3, [sp, #(8 * 0)]
194 stp x0, x1, [sp, #(8 * 2)]198 stp x0, x1, [sp, #(8 * 2)]
199 alternative_cb spectre_bhb_patch_wa3
200 /* Patched to mov WA3 when supported */
195 mov w0, #ARM_SMCCC_ARCH_WORKAROUND_1201 mov w0, #ARM_SMCCC_ARCH_WORKAROUND_1
202 alternative_cb_end
196 smc #0203 smc #0
197 ldp x2, x3, [sp, #(8 * 0)]204 ldp x2, x3, [sp, #(8 * 0)]
198 add sp, sp, #(8 * 2)205 add sp, sp, #(8 * 2)
@@ -205,6 +212,8 @@ SYM_CODE_END(__kvm_hyp_vector) · 6 unmodified lines
205 spectrev2_smccc_wa1_smc212 spectrev2_smccc_wa1_smc
206 .else213 .else
207 stp x0, x1, [sp, #-16]!214 stp x0, x1, [sp, #-16]!
215 mitigate_spectre_bhb_loop x0
216 mitigate_spectre_bhb_clear_insn
208 .endif217 .endif
209 .if \indirect != 0218 .if \indirect != 0
210 alternative_cb kvm_patch_vector_branch219 alternative_cb kvm_patch_vector_branch
arch/arm64/kvm/hyp/nvhe/mm.c+3−1
@@ -148,8 +148,10 @@ int hyp_map_vectors(void) · 147 unmodified lines
148 phys_addr_t phys;148 phys_addr_t phys;
149 void *bp_base;149 void *bp_base;
150 150
151 if (!cpus_have_const_cap(ARM64_SPECTRE_V3A))151 if (!kvm_system_needs_idmapped_vectors()) {
152 __hyp_bp_vect_base = __bp_harden_hyp_vecs;
152 return 0;153 return 0;
154 }
153 155
154 phys = __hyp_pa(__bp_harden_hyp_vecs);156 phys = __hyp_pa(__bp_harden_hyp_vecs);
155 bp_base = (void *)__pkvm_create_private_mapping(phys,157 bp_base = (void *)__pkvm_create_private_mapping(phys,
arch/arm64/kvm/hyp/vhe/switch.c+8−2
@@ -10,6 +10,7 @@ · 9 unmodified lines
10#include <linux/kvm_host.h>10#include <linux/kvm_host.h>
11#include <linux/types.h>11#include <linux/types.h>
12#include <linux/jump_label.h>12#include <linux/jump_label.h>
13#include <linux/percpu.h>
13#include <uapi/linux/psci.h>14#include <uapi/linux/psci.h>
14 15
15#include <kvm/arm_psci.h>16#include <kvm/arm_psci.h>
@@ -24,6 +25,8 @@ · 8 unmodified lines
24#include <asm/fpsimd.h>25#include <asm/fpsimd.h>
25#include <asm/debug-monitors.h>26#include <asm/debug-monitors.h>
26#include <asm/processor.h>27#include <asm/processor.h>
28#include <asm/thread_info.h>
29#include <asm/vectors.h>
27 30
28/* VHE specific context */31/* VHE specific context */
29DEFINE_PER_CPU(struct kvm_host_data, kvm_host_data);32DEFINE_PER_CPU(struct kvm_host_data, kvm_host_data);
@@ -67,7 +70,7 @@ NOKPROBE_SYMBOL(__activate_traps); · 37 unmodified lines
67 70
68static void __deactivate_traps(struct kvm_vcpu *vcpu)71static void __deactivate_traps(struct kvm_vcpu *vcpu)
69{72{
70 extern char vectors[]; /* kernel exception vectors */73 const char *host_vectors = vectors;
71 74
72 ___deactivate_traps(vcpu);75 ___deactivate_traps(vcpu);
73 76
@@ -81,7 +84,10 @@ static void __deactivate_traps(struct kvm_vcpu *vcpu) · 7 unmodified lines
81 asm(ALTERNATIVE("nop", "isb", ARM64_WORKAROUND_SPECULATIVE_AT));84 asm(ALTERNATIVE("nop", "isb", ARM64_WORKAROUND_SPECULATIVE_AT));
82 85
83 write_sysreg(CPACR_EL1_DEFAULT, cpacr_el1);86 write_sysreg(CPACR_EL1_DEFAULT, cpacr_el1);
84 write_sysreg(vectors, vbar_el1);87
88 if (!arm64_kernel_unmapped_at_el0())
89 host_vectors = __this_cpu_read(this_cpu_vector);
90 write_sysreg(host_vectors, vbar_el1);
85}91}
86NOKPROBE_SYMBOL(__deactivate_traps);92NOKPROBE_SYMBOL(__deactivate_traps);
87 93
arch/arm64/kvm/hypercalls.c+12−0
@@ -107,6 +107,18 @@ int kvm_hvc_call_handler(struct kvm_vcpu *vcpu) · 106 unmodified lines
107 break;107 break;
108 }108 }
109 break;109 break;
110 case ARM_SMCCC_ARCH_WORKAROUND_3:
111 switch (arm64_get_spectre_bhb_state()) {
112 case SPECTRE_VULNERABLE:
113 break;
114 case SPECTRE_MITIGATED:
115 val[0] = SMCCC_RET_SUCCESS;
116 break;
117 case SPECTRE_UNAFFECTED:
118 val[0] = SMCCC_ARCH_WORKAROUND_RET_UNAFFECTED;
119 break;
120 }
121 break;
110 case ARM_SMCCC_HV_PV_TIME_FEATURES:122 case ARM_SMCCC_HV_PV_TIME_FEATURES:
111 val[0] = SMCCC_RET_SUCCESS;123 val[0] = SMCCC_RET_SUCCESS;
112 break;124 break;
arch/arm64/kvm/psci.c+17−1
@@ -405,7 +405,7 @@ int kvm_psci_call(struct kvm_vcpu *vcpu) · 404 unmodified lines
405 405
406int kvm_arm_get_fw_num_regs(struct kvm_vcpu *vcpu)406int kvm_arm_get_fw_num_regs(struct kvm_vcpu *vcpu)
407{407{
408 return 3; /* PSCI version and two workaround registers */408 return 4; /* PSCI version and three workaround registers */
409}409}
410 410
411int kvm_arm_copy_fw_reg_indices(struct kvm_vcpu *vcpu, u64 __user *uindices)411int kvm_arm_copy_fw_reg_indices(struct kvm_vcpu *vcpu, u64 __user *uindices)
@@ -419,6 +419,9 @@ int kvm_arm_copy_fw_reg_indices(struct kvm_vcpu *vcpu, u64 __user *uindices) · 7 unmodified lines
419 if (put_user(KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2, uindices++))419 if (put_user(KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2, uindices++))
420 return -EFAULT;420 return -EFAULT;
421 421
422 if (put_user(KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3, uindices++))
423 return -EFAULT;
424
422 return 0;425 return 0;
423}426}
424 427
@@ -458,6 +461,17 @@ static int get_kernel_wa_level(u64 regid) · 33 unmodified lines
458 case SPECTRE_VULNERABLE:461 case SPECTRE_VULNERABLE:
459 return KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_NOT_AVAIL;462 return KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_NOT_AVAIL;
460 }463 }
464 break;
465 case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3:
466 switch (arm64_get_spectre_bhb_state()) {
467 case SPECTRE_VULNERABLE:
468 return KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3_NOT_AVAIL;
469 case SPECTRE_MITIGATED:
470 return KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3_AVAIL;
471 case SPECTRE_UNAFFECTED:
472 return KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3_NOT_REQUIRED;
473 }
474 return KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3_NOT_AVAIL;
461 }475 }
462 476
463 return -EINVAL;477 return -EINVAL;
@@ -474,6 +488,7 @@ int kvm_arm_get_fw_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) · 10 unmodified lines
474 break;488 break;
475 case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_1:489 case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_1:
476 case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2:490 case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2:
491 case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3:
477 val = get_kernel_wa_level(reg->id) & KVM_REG_FEATURE_LEVEL_MASK;492 val = get_kernel_wa_level(reg->id) & KVM_REG_FEATURE_LEVEL_MASK;
478 break;493 break;
479 default:494 default:
@@ -519,6 +534,7 @@ int kvm_arm_set_fw_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) · 39 unmodified lines
519 }534 }
520 535
521 case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_1:536 case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_1:
537 case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3:
522 if (val & ~KVM_REG_FEATURE_LEVEL_MASK)538 if (val & ~KVM_REG_FEATURE_LEVEL_MASK)
523 return -EINVAL;539 return -EINVAL;
524 540
arch/arm64/mm/mmu.c+9−3
@@ -617,6 +617,8 @@ early_param("rodata", parse_rodata); · 616 unmodified lines
617#ifdef CONFIG_UNMAP_KERNEL_AT_EL0617#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
618static int __init map_entry_trampoline(void)618static int __init map_entry_trampoline(void)
619{619{
620 int i;
621
620 pgprot_t prot = rodata_enabled ? PAGE_KERNEL_ROX : PAGE_KERNEL_EXEC;622 pgprot_t prot = rodata_enabled ? PAGE_KERNEL_ROX : PAGE_KERNEL_EXEC;
621 phys_addr_t pa_start = __pa_symbol(__entry_tramp_text_start);623 phys_addr_t pa_start = __pa_symbol(__entry_tramp_text_start);
622 624
@@ -625,11 +627,15 @@ static int __init map_entry_trampoline(void) · 2 unmodified lines
625 627
626 /* Map only the text into the trampoline page table */628 /* Map only the text into the trampoline page table */
627 memset(tramp_pg_dir, 0, PGD_SIZE);629 memset(tramp_pg_dir, 0, PGD_SIZE);
628 __create_pgd_mapping(tramp_pg_dir, pa_start, TRAMP_VALIAS, PAGE_SIZE,630 __create_pgd_mapping(tramp_pg_dir, pa_start, TRAMP_VALIAS,
629 prot, __pgd_pgtable_alloc, 0);631 entry_tramp_text_size(), prot,
632 __pgd_pgtable_alloc, NO_BLOCK_MAPPINGS);
630 633
631 /* Map both the text and data into the kernel page table */634 /* Map both the text and data into the kernel page table */
632 __set_fixmap(FIX_ENTRY_TRAMP_TEXT, pa_start, prot);635 for (i = 0; i < DIV_ROUND_UP(entry_tramp_text_size(), PAGE_SIZE); i++)
636 __set_fixmap(FIX_ENTRY_TRAMP_TEXT1 - i,
637 pa_start + i * PAGE_SIZE, prot);
638
633 if (IS_ENABLED(CONFIG_RANDOMIZE_BASE)) {639 if (IS_ENABLED(CONFIG_RANDOMIZE_BASE)) {
634 extern char __entry_tramp_data_start[];640 extern char __entry_tramp_data_start[];
635 641
arch/arm64/tools/cpucaps+1−0
@@ -44,6 +44,7 @@ MTE_ASYMM · 43 unmodified lines
44SPECTRE_V244SPECTRE_V2
45SPECTRE_V3A45SPECTRE_V3A
46SPECTRE_V446SPECTRE_V4
47SPECTRE_BHB
47SSBS48SSBS
48SVE49SVE
49UNMAP_KERNEL_AT_EL050UNMAP_KERNEL_AT_EL0
include/linux/arm-smccc.h+5−0
@@ -92,6 +92,11 @@ · 91 unmodified lines
92 ARM_SMCCC_SMC_32, \92 ARM_SMCCC_SMC_32, \
93 0, 0x7fff)93 0, 0x7fff)
94 94
95#define ARM_SMCCC_ARCH_WORKAROUND_3 \
96 ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, \
97 ARM_SMCCC_SMC_32, \
98 0, 0x3fff)
99
95#define ARM_SMCCC_VENDOR_HYP_CALL_UID_FUNC_ID \100#define ARM_SMCCC_VENDOR_HYP_CALL_UID_FUNC_ID \
96 ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, \101 ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, \
97 ARM_SMCCC_SMC_32, \102 ARM_SMCCC_SMC_32, \