Pull arm64 updates from Will Deacon: "There's a reasonable amount here and the juicy details are all below. It's worth noting that the MTE/KASAN changes strayed outside of our usual directories due to core mm changes and some associated changes to some other architectures; Andrew asked for us to carry these [1] rather that take them via the -mm tree. Summary: - Optimise SVE switching for CPUs with 128-bit implementations. - Fix output format from SVE selftest. - Add support for versions v1.2 and 1.3 of the SMC calling convention. - Allow Pointer Authentication to be configured independently for kernel and userspace. - PMU driver cleanups for managing IRQ affinity and exposing event attributes via sysfs. - KASAN optimisations for both hardware tagging (MTE) and out-of-line software tagging implementations. - Relax frame record alignment requirements to facilitate 8-byte alignment with KASAN and Clang. - Cleanup of page-table definitions and removal of unused memory types. - Reduction of ARCH_DMA_MINALIGN back to 64 bytes. - Refactoring of our instruction decoding routines and addition of some missing encodings. - Move entry code moved into C and hardened against harmful compiler instrumentation. - Update booting requirements for the FEAT_HCX feature, added to v8.7 of the architecture. - Fix resume from idle when pNMI is being used. - Additional CPU sanity checks for MTE and preparatory changes for systems where not all of the CPUs support 32-bit EL0. - Update our kernel string routines to the latest Cortex Strings implementation. - Big cleanup of our cache maintenance routines, which were confusingly named and inconsistent in their implementations. - Tweak linker flags so that GDB can understand vmlinux when using RELR relocations. - Boot path cleanups to enable early initialisation of per-cpu operations needed by KCSAN. - Non-critical fixes and miscellaneous cleanup" * tag 'arm64-upstream' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux: (150 commits) arm64: tlb: fix the TTL value of tlb_get_level arm64: Restrict undef hook for cpufeature registers arm64/mm: Rename ARM64_SWAPPER_USES_SECTION_MAPS arm64: insn: avoid circular include dependency arm64: smp: Bump debugging information print down to KERN_DEBUG drivers/perf: fix the missed ida_simple_remove() in ddr_perf_probe() perf/arm-cmn: Fix invalid pointer when access dtc object sharing the same IRQ number arm64: suspend: Use cpuidle context helpers in cpu_suspend() PSCI: Use cpuidle context helpers in psci_cpu_suspend_enter() arm64: Convert cpu_do_idle() to using cpuidle context helpers arm64: Add cpuidle context save/restore helpers arm64: head: fix code comments in set_cpu_boot_mode_flag arm64: mm: drop unused __pa(__idmap_text_start) arm64: mm: fix the count comments in compute_indices arm64/mm: Fix ttbr0 values stored in struct thread_info for software-pan arm64: mm: Pass original fault address to handle_mm_fault() arm64/mm: Drop SECTION_[SHIFT|SIZE|MASK] arm64/mm: Use CONT_PMD_SHIFT for ARM64_MEMSTART_SHIFT arm64/mm: Drop SWAPPER_INIT_MAP_SIZE arm64: Conditionally configure PTR_AUTH key of the kernel. ...
Unified · Split
| @@ -277,6 +277,12 @@ Before jumping into the kernel, the following conditions must be met: · 276 unmodified lines | ||
| 277 | 277 | |
| 278 | 278 | - SCR_EL3.FGTEn (bit 27) must be initialised to 0b1. |
| 279 | 279 | |
| 280 | + For CPUs with support for HCRX_EL2 (FEAT_HCX) present: | |
| 281 | + | |
| 282 | + - If EL3 is present and the kernel is entered at EL2: | |
| 283 | + | |
| 284 | + - SCR_EL3.HXEn (bit 38) must be initialised to 0b1. | |
| 285 | + | |
| 280 | 286 | For CPUs with Advanced SIMD and floating point support: |
| 281 | 287 | |
| 282 | 288 | - If EL3 is present: |
| @@ -1039,7 +1039,7 @@ LDFLAGS_vmlinux += $(call ld-option, -X,) · 1038 unmodified lines | ||
| 1039 | 1039 | endif |
| 1040 | 1040 | |
| 1041 | 1041 | ifeq ($(CONFIG_RELR),y) |
| 1042 | −LDFLAGS_vmlinux += --pack-dyn-relocs=relr | |
| 1042 | +LDFLAGS_vmlinux += --pack-dyn-relocs=relr --use-android-relr-tags | |
| 1043 | 1043 | endif |
| 1044 | 1044 | |
| 1045 | 1045 | # We never want expected sections to be placed heuristically by the |
| @@ -17,9 +17,9 @@ · 16 unmodified lines | ||
| 17 | 17 | extern void clear_page(void *page); |
| 18 | 18 | #define clear_user_page(page, vaddr, pg) clear_page(page) |
| 19 | 19 | |
| 20 | −#define __alloc_zeroed_user_highpage(movableflags, vma, vaddr) \ | |
| 21 | − alloc_page_vma(GFP_HIGHUSER | __GFP_ZERO | movableflags, vma, vmaddr) | |
| 22 | −#define __HAVE_ARCH_ALLOC_ZEROED_USER_HIGHPAGE | |
| 20 | +#define alloc_zeroed_user_highpage_movable(vma, vaddr) \ | |
| 21 | + alloc_page_vma(GFP_HIGHUSER_MOVABLE | __GFP_ZERO, vma, vmaddr) | |
| 22 | +#define __HAVE_ARCH_ALLOC_ZEROED_USER_HIGHPAGE_MOVABLE | |
| 23 | 23 | |
| 24 | 24 | extern void copy_page(void * _to, void * _from); |
| 25 | 25 | #define copy_user_page(to, from, vaddr, pg) copy_page(to, from) |
| @@ -50,4 +50,9 @@ extern int arm_cpuidle_suspend(int index); · 49 unmodified lines | ||
| 50 | 50 | |
| 51 | 51 | extern int arm_cpuidle_init(int cpu); |
| 52 | 52 | |
| 53 | +struct arm_cpuidle_irq_context { }; | |
| 54 | + | |
| 55 | +#define arm_cpuidle_save_irq_context(c) (void)c | |
| 56 | +#define arm_cpuidle_restore_irq_context(c) (void)c | |
| 57 | + | |
| 53 | 58 | #endif |
| @@ -773,10 +773,10 @@ static inline void armv7pmu_write_counter(struct perf_event *event, u64 value) · 772 unmodified lines | ||
| 773 | 773 | pr_err("CPU%u writing wrong counter %d\n", |
| 774 | 774 | smp_processor_id(), idx); |
| 775 | 775 | } else if (idx == ARMV7_IDX_CYCLE_COUNTER) { |
| 776 | − asm volatile("mcr p15, 0, %0, c9, c13, 0" : : "r" (value)); | |
| 776 | + asm volatile("mcr p15, 0, %0, c9, c13, 0" : : "r" ((u32)value)); | |
| 777 | 777 | } else { |
| 778 | 778 | armv7_pmnc_select_counter(idx); |
| 779 | − asm volatile("mcr p15, 0, %0, c9, c13, 2" : : "r" (value)); | |
| 779 | + asm volatile("mcr p15, 0, %0, c9, c13, 2" : : "r" ((u32)value)); | |
| 780 | 780 | } |
| 781 | 781 | } |
| 782 | 782 | |
| @@ -1481,12 +1481,6 @@ menu "ARMv8.3 architectural features" · 1480 unmodified lines | ||
| 1481 | 1481 | config ARM64_PTR_AUTH |
| 1482 | 1482 | bool "Enable support for pointer authentication" |
| 1483 | 1483 | default y |
| 1484 | − depends on (CC_HAS_SIGN_RETURN_ADDRESS || CC_HAS_BRANCH_PROT_PAC_RET) && AS_HAS_PAC | |
| 1485 | − # Modern compilers insert a .note.gnu.property section note for PAC | |
| 1486 | − # which is only understood by binutils starting with version 2.33.1. | |
| 1487 | − depends on LD_IS_LLD || LD_VERSION >= 23301 || (CC_IS_GCC && GCC_VERSION < 90100) | |
| 1488 | − depends on !CC_IS_CLANG || AS_HAS_CFI_NEGATE_RA_STATE | |
| 1489 | − depends on (!FUNCTION_GRAPH_TRACER || DYNAMIC_FTRACE_WITH_REGS) | |
| 1490 | 1484 | help |
| 1491 | 1485 | Pointer authentication (part of the ARMv8.3 Extensions) provides |
| 1492 | 1486 | instructions for signing and authenticating pointers against secret |
| @@ -1498,13 +1492,6 @@ config ARM64_PTR_AUTH · 5 unmodified lines | ||
| 1498 | 1492 | for each process at exec() time, with these keys being |
| 1499 | 1493 | context-switched along with the process. |
| 1500 | 1494 | |
| 1501 | − If the compiler supports the -mbranch-protection or | |
| 1502 | − -msign-return-address flag (e.g. GCC 7 or later), then this option | |
| 1503 | − will also cause the kernel itself to be compiled with return address | |
| 1504 | − protection. In this case, and if the target hardware is known to | |
| 1505 | − support pointer authentication, then CONFIG_STACKPROTECTOR can be | |
| 1506 | − disabled with minimal loss of protection. | |
| 1507 | − | |
| 1508 | 1495 | The feature is detected at runtime. If the feature is not present in |
| 1509 | 1496 | hardware it will not be advertised to userspace/KVM guest nor will it |
| 1510 | 1497 | be enabled. |
| @@ -1515,6 +1502,24 @@ config ARM64_PTR_AUTH · 4 unmodified lines | ||
| 1515 | 1502 | but with the feature disabled. On such a system, this option should |
| 1516 | 1503 | not be selected. |
| 1517 | 1504 | |
| 1505 | +config ARM64_PTR_AUTH_KERNEL | |
| 1506 | + bool "Use pointer authentication for kernel" | |
| 1507 | + default y | |
| 1508 | + depends on ARM64_PTR_AUTH | |
| 1509 | + depends on (CC_HAS_SIGN_RETURN_ADDRESS || CC_HAS_BRANCH_PROT_PAC_RET) && AS_HAS_PAC | |
| 1510 | + # Modern compilers insert a .note.gnu.property section note for PAC | |
| 1511 | + # which is only understood by binutils starting with version 2.33.1. | |
| 1512 | + depends on LD_IS_LLD || LD_VERSION >= 23301 || (CC_IS_GCC && GCC_VERSION < 90100) | |
| 1513 | + depends on !CC_IS_CLANG || AS_HAS_CFI_NEGATE_RA_STATE | |
| 1514 | + depends on (!FUNCTION_GRAPH_TRACER || DYNAMIC_FTRACE_WITH_REGS) | |
| 1515 | + help | |
| 1516 | + If the compiler supports the -mbranch-protection or | |
| 1517 | + -msign-return-address flag (e.g. GCC 7 or later), then this option | |
| 1518 | + will cause the kernel itself to be compiled with return address | |
| 1519 | + protection. In this case, and if the target hardware is known to | |
| 1520 | + support pointer authentication, then CONFIG_STACKPROTECTOR can be | |
| 1521 | + disabled with minimal loss of protection. | |
| 1522 | + | |
| 1518 | 1523 | This feature works with FUNCTION_GRAPH_TRACER option only if |
| 1519 | 1524 | DYNAMIC_FTRACE_WITH_REGS is enabled. |
| 1520 | 1525 | |
| @@ -1606,7 +1611,7 @@ config ARM64_BTI_KERNEL · 85 unmodified lines | ||
| 1606 | 1611 | bool "Use Branch Target Identification for kernel" |
| 1607 | 1612 | default y |
| 1608 | 1613 | depends on ARM64_BTI |
| 1609 | − depends on ARM64_PTR_AUTH | |
| 1614 | + depends on ARM64_PTR_AUTH_KERNEL | |
| 1610 | 1615 | depends on CC_HAS_BRANCH_PROT_PAC_RET_BTI |
| 1611 | 1616 | # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=94697 |
| 1612 | 1617 | depends on !CC_IS_GCC || GCC_VERSION >= 100100 |
| @@ -70,7 +70,7 @@ endif · 69 unmodified lines | ||
| 70 | 70 | # off, this will be overridden if we are using branch protection. |
| 71 | 71 | branch-prot-flags-y += $(call cc-option,-mbranch-protection=none) |
| 72 | 72 | |
| 73 | −ifeq ($(CONFIG_ARM64_PTR_AUTH),y) | |
| 73 | +ifeq ($(CONFIG_ARM64_PTR_AUTH_KERNEL),y) | |
| 74 | 74 | branch-prot-flags-$(CONFIG_CC_HAS_SIGN_RETURN_ADDRESS) := -msign-return-address=all |
| 75 | 75 | # We enable additional protection for leaf functions as there is some |
| 76 | 76 | # narrow potential for ROP protection benefits and no substantial |
| @@ -3,12 +3,10 @@ · 2 unmodified lines | ||
| 3 | 3 | #define __ASM_ALTERNATIVE_MACROS_H |
| 4 | 4 | |
| 5 | 5 | #include <asm/cpucaps.h> |
| 6 | +#include <asm/insn-def.h> | |
| 6 | 7 | |
| 7 | 8 | #define ARM64_CB_PATCH ARM64_NCAPS |
| 8 | 9 | |
| 9 | −/* A64 instructions are always 32 bits. */ | |
| 10 | −#define AARCH64_INSN_SIZE 4 | |
| 11 | − | |
| 12 | 10 | #ifndef __ASSEMBLY__ |
| 13 | 11 | |
| 14 | 12 | #include <linux/stringify.h> |
| @@ -197,11 +195,6 @@ alternative_endif · 182 unmodified lines | ||
| 197 | 195 | #define _ALTERNATIVE_CFG(insn1, insn2, cap, cfg, ...) \ |
| 198 | 196 | alternative_insn insn1, insn2, cap, IS_ENABLED(cfg) |
| 199 | 197 | |
| 200 | −.macro user_alt, label, oldinstr, newinstr, cond | |
| 201 | −9999: alternative_insn "\oldinstr", "\newinstr", \cond | |
| 202 | − _asm_extable 9999b, \label | |
| 203 | −.endm | |
| 204 | − | |
| 205 | 198 | #endif /* __ASSEMBLY__ */ |
| 206 | 199 | |
| 207 | 200 | /* |
| @@ -124,7 +124,8 @@ static inline u32 gic_read_rpr(void) · 123 unmodified lines | ||
| 124 | 124 | #define gic_read_lpir(c) readq_relaxed(c) |
| 125 | 125 | #define gic_write_lpir(v, c) writeq_relaxed(v, c) |
| 126 | 126 | |
| 127 | −#define gic_flush_dcache_to_poc(a,l) __flush_dcache_area((a), (l)) | |
| 127 | +#define gic_flush_dcache_to_poc(a,l) \ | |
| 128 | + dcache_clean_inval_poc((unsigned long)(a), (unsigned long)(a)+(l)) | |
| 128 | 129 | |
| 129 | 130 | #define gits_read_baser(c) readq_relaxed(c) |
| 130 | 131 | #define gits_write_baser(v, c) writeq_relaxed(v, c) |
| @@ -23,4 +23,10 @@ long long __ashlti3(long long a, int b); · 22 unmodified lines | ||
| 23 | 23 | long long __ashrti3(long long a, int b); |
| 24 | 24 | long long __lshrti3(long long a, int b); |
| 25 | 25 | |
| 26 | +/* | |
| 27 | + * This function uses a custom calling convention and cannot be called from C so | |
| 28 | + * this prototype is not entirely accurate. | |
| 29 | + */ | |
| 30 | +void __hwasan_tag_mismatch(unsigned long addr, unsigned long access_info); | |
| 31 | + | |
| 26 | 32 | #endif /* __ASM_PROTOTYPES_H */ |
| @@ -7,19 +7,7 @@ · 6 unmodified lines | ||
| 7 | 7 | #include <asm/cpufeature.h> |
| 8 | 8 | #include <asm/sysreg.h> |
| 9 | 9 | |
| 10 | −#ifdef CONFIG_ARM64_PTR_AUTH | |
| 11 | −/* | |
| 12 | − * thread.keys_user.ap* as offset exceeds the #imm offset range | |
| 13 | − * so use the base value of ldp as thread.keys_user and offset as | |
| 14 | − * thread.keys_user.ap*. | |
| 15 | − */ | |
| 16 | − .macro __ptrauth_keys_install_user tsk, tmp1, tmp2, tmp3 | |
| 17 | − mov \tmp1, #THREAD_KEYS_USER | |
| 18 | − add \tmp1, \tsk, \tmp1 | |
| 19 | − ldp \tmp2, \tmp3, [\tmp1, #PTRAUTH_USER_KEY_APIA] | |
| 20 | − msr_s SYS_APIAKEYLO_EL1, \tmp2 | |
| 21 | − msr_s SYS_APIAKEYHI_EL1, \tmp3 | |
| 22 | − .endm | |
| 10 | +#ifdef CONFIG_ARM64_PTR_AUTH_KERNEL | |
| 23 | 11 | |
| 24 | 12 | .macro __ptrauth_keys_install_kernel_nosync tsk, tmp1, tmp2, tmp3 |
| 25 | 13 | mov \tmp1, #THREAD_KEYS_KERNEL |
| @@ -42,6 +30,33 @@ alternative_if ARM64_HAS_ADDRESS_AUTH · 16 unmodified lines | ||
| 42 | 30 | alternative_else_nop_endif |
| 43 | 31 | .endm |
| 44 | 32 | |
| 33 | +#else /* CONFIG_ARM64_PTR_AUTH_KERNEL */ | |
| 34 | + | |
| 35 | + .macro __ptrauth_keys_install_kernel_nosync tsk, tmp1, tmp2, tmp3 | |
| 36 | + .endm | |
| 37 | + | |
| 38 | + .macro ptrauth_keys_install_kernel_nosync tsk, tmp1, tmp2, tmp3 | |
| 39 | + .endm | |
| 40 | + | |
| 41 | + .macro ptrauth_keys_install_kernel tsk, tmp1, tmp2, tmp3 | |
| 42 | + .endm | |
| 43 | + | |
| 44 | +#endif /* CONFIG_ARM64_PTR_AUTH_KERNEL */ | |
| 45 | + | |
| 46 | +#ifdef CONFIG_ARM64_PTR_AUTH | |
| 47 | +/* | |
| 48 | + * thread.keys_user.ap* as offset exceeds the #imm offset range | |
| 49 | + * so use the base value of ldp as thread.keys_user and offset as | |
| 50 | + * thread.keys_user.ap*. | |
| 51 | + */ | |
| 52 | + .macro __ptrauth_keys_install_user tsk, tmp1, tmp2, tmp3 | |
| 53 | + mov \tmp1, #THREAD_KEYS_USER | |
| 54 | + add \tmp1, \tsk, \tmp1 | |
| 55 | + ldp \tmp2, \tmp3, [\tmp1, #PTRAUTH_USER_KEY_APIA] | |
| 56 | + msr_s SYS_APIAKEYLO_EL1, \tmp2 | |
| 57 | + msr_s SYS_APIAKEYHI_EL1, \tmp3 | |
| 58 | + .endm | |
| 59 | + | |
| 45 | 60 | .macro __ptrauth_keys_init_cpu tsk, tmp1, tmp2, tmp3 |
| 46 | 61 | mrs \tmp1, id_aa64isar1_el1 |
| 47 | 62 | ubfx \tmp1, \tmp1, #ID_AA64ISAR1_APA_SHIFT, #8 |
| @@ -64,17 +79,11 @@ alternative_else_nop_endif · 16 unmodified lines | ||
| 64 | 79 | .Lno_addr_auth\@: |
| 65 | 80 | .endm |
| 66 | 81 | |
| 67 | −#else /* CONFIG_ARM64_PTR_AUTH */ | |
| 82 | +#else /* !CONFIG_ARM64_PTR_AUTH */ | |
| 68 | 83 | |
| 69 | 84 | .macro ptrauth_keys_install_user tsk, tmp1, tmp2, tmp3 |
| 70 | 85 | .endm |
| 71 | 86 | |
| 72 | − .macro ptrauth_keys_install_kernel_nosync tsk, tmp1, tmp2, tmp3 | |
| 73 | − .endm | |
| 74 | − | |
| 75 | − .macro ptrauth_keys_install_kernel tsk, tmp1, tmp2, tmp3 | |
| 76 | − .endm | |
| 77 | − | |
| 78 | 87 | #endif /* CONFIG_ARM64_PTR_AUTH */ |
| 79 | 88 | |
| 80 | 89 | #endif /* __ASM_ASM_POINTER_AUTH_H */ |
| @@ -130,15 +130,27 @@ alternative_endif · 129 unmodified lines | ||
| 130 | 130 | .endm |
| 131 | 131 | |
| 132 | 132 | /* |
| 133 | − * Emit an entry into the exception table | |
| 133 | + * Create an exception table entry for `insn`, which will branch to `fixup` | |
| 134 | + * when an unhandled fault is taken. | |
| 134 | 135 | */ |
| 135 | − .macro _asm_extable, from, to | |
| 136 | + .macro _asm_extable, insn, fixup | |
| 136 | 137 | .pushsection __ex_table, "a" |
| 137 | 138 | .align 3 |
| 138 | − .long (\from - .), (\to - .) | |
| 139 | + .long (\insn - .), (\fixup - .) | |
| 139 | 140 | .popsection |
| 140 | 141 | .endm |
| 141 | 142 | |
| 143 | +/* | |
| 144 | + * Create an exception table entry for `insn` if `fixup` is provided. Otherwise | |
| 145 | + * do nothing. | |
| 146 | + */ | |
| 147 | + .macro _cond_extable, insn, fixup | |
| 148 | + .ifnc \fixup, | |
| 149 | + _asm_extable \insn, \fixup | |
| 150 | + .endif | |
| 151 | + .endm | |
| 152 | + | |
| 153 | + | |
| 142 | 154 | #define USER(l, x...) \ |
| 143 | 155 | 9999: x; \ |
| 144 | 156 | _asm_extable 9999b, l |
| @@ -232,15 +244,23 @@ lr .req x30 // link register · 87 unmodified lines | ||
| 232 | 244 | * @dst: destination register |
| 233 | 245 | */ |
| 234 | 246 | #if defined(__KVM_NVHE_HYPERVISOR__) || defined(__KVM_VHE_HYPERVISOR__) |
| 235 | − .macro this_cpu_offset, dst | |
| 247 | + .macro get_this_cpu_offset, dst | |
| 236 | 248 | mrs \dst, tpidr_el2 |
| 237 | 249 | .endm |
| 238 | 250 | #else |
| 239 | − .macro this_cpu_offset, dst | |
| 251 | + .macro get_this_cpu_offset, dst | |
| 240 | 252 | alternative_if_not ARM64_HAS_VIRT_HOST_EXTN |
| 241 | 253 | mrs \dst, tpidr_el1 |
| 242 | 254 | alternative_else |
| 243 | 255 | mrs \dst, tpidr_el2 |
| 256 | +alternative_endif | |
| 257 | + .endm | |
| 258 | + | |
| 259 | + .macro set_this_cpu_offset, src | |
| 260 | +alternative_if_not ARM64_HAS_VIRT_HOST_EXTN | |
| 261 | + msr tpidr_el1, \src | |
| 262 | +alternative_else | |
| 263 | + msr tpidr_el2, \src | |
| 244 | 264 | alternative_endif |
| 245 | 265 | .endm |
| 246 | 266 | #endif |
| @@ -253,7 +273,7 @@ alternative_endif · 6 unmodified lines | ||
| 253 | 273 | .macro adr_this_cpu, dst, sym, tmp |
| 254 | 274 | adrp \tmp, \sym |
| 255 | 275 | add \dst, \tmp, #:lo12:\sym |
| 256 | − this_cpu_offset \tmp | |
| 276 | + get_this_cpu_offset \tmp | |
| 257 | 277 | add \dst, \dst, \tmp |
| 258 | 278 | .endm |
| 259 | 279 | |
| @@ -264,7 +284,7 @@ alternative_endif · 4 unmodified lines | ||
| 264 | 284 | */ |
| 265 | 285 | .macro ldr_this_cpu dst, sym, tmp |
| 266 | 286 | adr_l \dst, \sym |
| 267 | − this_cpu_offset \tmp | |
| 287 | + get_this_cpu_offset \tmp | |
| 268 | 288 | ldr \dst, [\dst, \tmp] |
| 269 | 289 | .endm |
| 270 | 290 | |
| @@ -375,51 +395,53 @@ alternative_cb_end · 104 unmodified lines | ||
| 375 | 395 | bfi \tcr, \tmp0, \pos, #3 |
| 376 | 396 | .endm |
| 377 | 397 | |
| 398 | + .macro __dcache_op_workaround_clean_cache, op, addr | |
| 399 | +alternative_if_not ARM64_WORKAROUND_CLEAN_CACHE | |
| 400 | + dc \op, \addr | |
| 401 | +alternative_else | |
| 402 | + dc civac, \addr | |
| 403 | +alternative_endif | |
| 404 | + .endm | |
| 405 | + | |
| 378 | 406 | /* |
| 379 | 407 | * Macro to perform a data cache maintenance for the interval |
| 380 | − * [kaddr, kaddr + size) | |
| 408 | + * [start, end) | |
| 381 | 409 | * |
| 382 | 410 | * op: operation passed to dc instruction |
| 383 | 411 | * domain: domain used in dsb instruciton |
| 384 | − * kaddr: starting virtual address of the region | |
| 385 | − * size: size of the region | |
| 386 | − * Corrupts: kaddr, size, tmp1, tmp2 | |
| 412 | + * start: starting virtual address of the region | |
| 413 | + * end: end virtual address of the region | |
| 414 | + * fixup: optional label to branch to on user fault | |
| 415 | + * Corrupts: start, end, tmp1, tmp2 | |
| 387 | 416 | */ |
| 388 | − .macro __dcache_op_workaround_clean_cache, op, kaddr | |
| 389 | −alternative_if_not ARM64_WORKAROUND_CLEAN_CACHE | |
| 390 | − dc \op, \kaddr | |
| 391 | −alternative_else | |
| 392 | − dc civac, \kaddr | |
| 393 | −alternative_endif | |
| 394 | − .endm | |
| 395 | − | |
| 396 | − .macro dcache_by_line_op op, domain, kaddr, size, tmp1, tmp2 | |
| 417 | + .macro dcache_by_line_op op, domain, start, end, tmp1, tmp2, fixup | |
| 397 | 418 | dcache_line_size \tmp1, \tmp2 |
| 398 | − add \size, \kaddr, \size | |
| 399 | 419 | sub \tmp2, \tmp1, #1 |
| 400 | − bic \kaddr, \kaddr, \tmp2 | |
| 401 | −9998: | |
| 420 | + bic \start, \start, \tmp2 | |
| 421 | +.Ldcache_op\@: | |
| 402 | 422 | .ifc \op, cvau |
| 403 | − __dcache_op_workaround_clean_cache \op, \kaddr | |
| 423 | + __dcache_op_workaround_clean_cache \op, \start | |
| 404 | 424 | .else |
| 405 | 425 | .ifc \op, cvac |
| 406 | − __dcache_op_workaround_clean_cache \op, \kaddr | |
| 426 | + __dcache_op_workaround_clean_cache \op, \start | |
| 407 | 427 | .else |
| 408 | 428 | .ifc \op, cvap |
| 409 | − sys 3, c7, c12, 1, \kaddr // dc cvap | |
| 429 | + sys 3, c7, c12, 1, \start // dc cvap | |
| 410 | 430 | .else |
| 411 | 431 | .ifc \op, cvadp |
| 412 | − sys 3, c7, c13, 1, \kaddr // dc cvadp | |
| 432 | + sys 3, c7, c13, 1, \start // dc cvadp | |
| 413 | 433 | .else |
| 414 | − dc \op, \kaddr | |
| 434 | + dc \op, \start | |
| 415 | 435 | .endif |
| 416 | 436 | .endif |
| 417 | 437 | .endif |
| 418 | 438 | .endif |
| 419 | − add \kaddr, \kaddr, \tmp1 | |
| 420 | − cmp \kaddr, \size | |
| 421 | − b.lo 9998b | |
| 439 | + add \start, \start, \tmp1 | |
| 440 | + cmp \start, \end | |
| 441 | + b.lo .Ldcache_op\@ | |
| 422 | 442 | dsb \domain |
| 443 | + | |
| 444 | + _cond_extable .Ldcache_op\@, \fixup | |
| 423 | 445 | .endm |
| 424 | 446 | |
| 425 | 447 | /* |
| @@ -427,20 +449,22 @@ alternative_endif · 1 unmodified line | ||
| 427 | 449 | * [start, end) |
| 428 | 450 | * |
| 429 | 451 | * start, end: virtual addresses describing the region |
| 430 | − * label: A label to branch to on user fault. | |
| 452 | + * fixup: optional label to branch to on user fault | |
| 431 | 453 | * Corrupts: tmp1, tmp2 |
| 432 | 454 | */ |
| 433 | − .macro invalidate_icache_by_line start, end, tmp1, tmp2, label | |
| 455 | + .macro invalidate_icache_by_line start, end, tmp1, tmp2, fixup | |
| 434 | 456 | icache_line_size \tmp1, \tmp2 |
| 435 | 457 | sub \tmp2, \tmp1, #1 |
| 436 | 458 | bic \tmp2, \start, \tmp2 |
| 437 | −9997: | |
| 438 | −USER(\label, ic ivau, \tmp2) // invalidate I line PoU | |
| 459 | +.Licache_op\@: | |
| 460 | + ic ivau, \tmp2 // invalidate I line PoU | |
| 439 | 461 | add \tmp2, \tmp2, \tmp1 |
| 440 | 462 | cmp \tmp2, \end |
| 441 | − b.lo 9997b | |
| 463 | + b.lo .Licache_op\@ | |
| 442 | 464 | dsb ish |
| 443 | 465 | isb |
| 466 | + | |
| 467 | + _cond_extable .Licache_op\@, \fixup | |
| 444 | 468 | .endm |
| 445 | 469 | |
| 446 | 470 | /* |
| @@ -745,7 +769,7 @@ USER(\label, ic ivau, \tmp2) // invalidate I line PoU · 298 unmodified lines | ||
| 745 | 769 | cbz \tmp, \lbl |
| 746 | 770 | #endif |
| 747 | 771 | adr_l \tmp, irq_stat + IRQ_CPUSTAT_SOFTIRQ_PENDING |
| 748 | − this_cpu_offset \tmp2 | |
| 772 | + get_this_cpu_offset \tmp2 | |
| 749 | 773 | ldr w\tmp, [\tmp, \tmp2] |
| 750 | 774 | cbnz w\tmp, \lbl // yield on pending softirq in task context |
| 751 | 775 | .Lnoyield_\@: |
| @@ -47,7 +47,7 @@ · 46 unmodified lines | ||
| 47 | 47 | * cache before the transfer is done, causing old data to be seen by |
| 48 | 48 | * the CPU. |
| 49 | 49 | */ |
| 50 | −#define ARCH_DMA_MINALIGN (128) | |
| 50 | +#define ARCH_DMA_MINALIGN L1_CACHE_BYTES | |
| 51 | 51 | |
| 52 | 52 | #ifdef CONFIG_KASAN_SW_TAGS |
| 53 | 53 | #define ARCH_SLAB_MINALIGN (1ULL << KASAN_SHADOW_SCALE_SHIFT) |
| @@ -30,45 +30,58 @@ · 29 unmodified lines | ||
| 30 | 30 | * the implementation assumes non-aliasing VIPT D-cache and (aliasing) |
| 31 | 31 | * VIPT I-cache. |
| 32 | 32 | * |
| 33 | − * flush_icache_range(start, end) | |
| 33 | + * All functions below apply to the interval [start, end) | |
| 34 | + * - start - virtual start address (inclusive) | |
| 35 | + * - end - virtual end address (exclusive) | |
| 34 | 36 | * |
| 35 | − * Ensure coherency between the I-cache and the D-cache in the | |
| 36 | − * region described by start, end. | |
| 37 | − * - start - virtual start address | |
| 38 | − * - end - virtual end address | |
| 37 | + * caches_clean_inval_pou(start, end) | |
| 39 | 38 | * |
| 40 | − * invalidate_icache_range(start, end) | |
| 39 | + * Ensure coherency between the I-cache and the D-cache region to | |
| 40 | + * the Point of Unification. | |
| 41 | 41 | * |
| 42 | − * Invalidate the I-cache in the region described by start, end. | |
| 43 | − * - start - virtual start address | |
| 44 | − * - end - virtual end address | |
| 42 | + * caches_clean_inval_user_pou(start, end) | |
| 45 | 43 | * |
| 46 | − * __flush_cache_user_range(start, end) | |
| 44 | + * Ensure coherency between the I-cache and the D-cache region to | |
| 45 | + * the Point of Unification. | |
| 46 | + * Use only if the region might access user memory. | |
| 47 | 47 | * |
| 48 | − * Ensure coherency between the I-cache and the D-cache in the | |
| 49 | − * region described by start, end. | |
| 50 | − * - start - virtual start address | |
| 51 | − * - end - virtual end address | |
| 48 | + * icache_inval_pou(start, end) | |
| 52 | 49 | * |
| 53 | − * __flush_dcache_area(kaddr, size) | |
| 50 | + * Invalidate I-cache region to the Point of Unification. | |
| 54 | 51 | * |
| 55 | − * Ensure that the data held in page is written back. | |
| 56 | − * - kaddr - page address | |
| 57 | − * - size - region size | |
| 52 | + * dcache_clean_inval_poc(start, end) | |
| 53 | + * | |
| 54 | + * Clean and invalidate D-cache region to the Point of Coherency. | |
| 55 | + * | |
| 56 | + * dcache_inval_poc(start, end) | |
| 57 | + * | |
| 58 | + * Invalidate D-cache region to the Point of Coherency. | |
| 59 | + * | |
| 60 | + * dcache_clean_poc(start, end) | |
| 61 | + * | |
| 62 | + * Clean D-cache region to the Point of Coherency. | |
| 63 | + * | |
| 64 | + * dcache_clean_pop(start, end) | |
| 65 | + * | |
| 66 | + * Clean D-cache region to the Point of Persistence. | |
| 67 | + * | |
| 68 | + * dcache_clean_pou(start, end) | |
| 69 | + * | |
| 70 | + * Clean D-cache region to the Point of Unification. | |
| 58 | 71 | */ |
| 59 | −extern void __flush_icache_range(unsigned long start, unsigned long end); | |
| 60 | −extern int invalidate_icache_range(unsigned long start, unsigned long end); | |
| 61 | −extern void __flush_dcache_area(void *addr, size_t len); | |
| 62 | −extern void __inval_dcache_area(void *addr, size_t len); | |
| 63 | −extern void __clean_dcache_area_poc(void *addr, size_t len); | |
| 64 | −extern void __clean_dcache_area_pop(void *addr, size_t len); | |
| 65 | −extern void __clean_dcache_area_pou(void *addr, size_t len); | |
| 66 | −extern long __flush_cache_user_range(unsigned long start, unsigned long end); | |
| 67 | −extern void sync_icache_aliases(void *kaddr, unsigned long len); | |
| 72 | +extern void caches_clean_inval_pou(unsigned long start, unsigned long end); | |
| 73 | +extern void icache_inval_pou(unsigned long start, unsigned long end); | |
| 74 | +extern void dcache_clean_inval_poc(unsigned long start, unsigned long end); | |
| 75 | +extern void dcache_inval_poc(unsigned long start, unsigned long end); | |
| 76 | +extern void dcache_clean_poc(unsigned long start, unsigned long end); | |
| 77 | +extern void dcache_clean_pop(unsigned long start, unsigned long end); | |
| 78 | +extern void dcache_clean_pou(unsigned long start, unsigned long end); | |
| 79 | +extern long caches_clean_inval_user_pou(unsigned long start, unsigned long end); | |
| 80 | +extern void sync_icache_aliases(unsigned long start, unsigned long end); | |
| 68 | 81 | |
| 69 | 82 | static inline void flush_icache_range(unsigned long start, unsigned long end) |
| 70 | 83 | { |
| 71 | − __flush_icache_range(start, end); | |
| 84 | + caches_clean_inval_pou(start, end); | |
| 72 | 85 | |
| 73 | 86 | /* |
| 74 | 87 | * IPI all online CPUs so that they undergo a context synchronization |
| @@ -122,7 +135,7 @@ extern void copy_to_user_page(struct vm_area_struct *, struct page *, · 47 unmodified lines | ||
| 122 | 135 | #define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE 1 |
| 123 | 136 | extern void flush_dcache_page(struct page *); |
| 124 | 137 | |
| 125 | −static __always_inline void __flush_icache_all(void) | |
| 138 | +static __always_inline void icache_inval_all_pou(void) | |
| 126 | 139 | { |
| 127 | 140 | if (cpus_have_const_cap(ARM64_HAS_CACHE_DIC)) |
| 128 | 141 | return; |
| @@ -12,26 +12,7 @@ · 11 unmodified lines | ||
| 12 | 12 | /* |
| 13 | 13 | * Records attributes of an individual CPU. |
| 14 | 14 | */ |
| 15 | −struct cpuinfo_arm64 { | |
| 16 | − struct cpu cpu; | |
| 17 | − struct kobject kobj; | |
| 18 | − u32 reg_ctr; | |
| 19 | − u32 reg_cntfrq; | |
| 20 | − u32 reg_dczid; | |
| 21 | − u32 reg_midr; | |
| 22 | − u32 reg_revidr; | |
| 23 | − | |
| 24 | − u64 reg_id_aa64dfr0; | |
| 25 | − u64 reg_id_aa64dfr1; | |
| 26 | − u64 reg_id_aa64isar0; | |
| 27 | − u64 reg_id_aa64isar1; | |
| 28 | − u64 reg_id_aa64mmfr0; | |
| 29 | − u64 reg_id_aa64mmfr1; | |
| 30 | − u64 reg_id_aa64mmfr2; | |
| 31 | − u64 reg_id_aa64pfr0; | |
| 32 | − u64 reg_id_aa64pfr1; | |
| 33 | − u64 reg_id_aa64zfr0; | |
| 34 | − | |
| 15 | +struct cpuinfo_32bit { | |
| 35 | 16 | u32 reg_id_dfr0; |
| 36 | 17 | u32 reg_id_dfr1; |
| 37 | 18 | u32 reg_id_isar0; |
| @@ -54,6 +35,30 @@ struct cpuinfo_arm64 { · 16 unmodified lines | ||
| 54 | 35 | u32 reg_mvfr0; |
| 55 | 36 | u32 reg_mvfr1; |
| 56 | 37 | u32 reg_mvfr2; |
| 38 | +}; | |
| 39 | + | |
| 40 | +struct cpuinfo_arm64 { | |
| 41 | + struct cpu cpu; | |
| 42 | + struct kobject kobj; | |
| 43 | + u64 reg_ctr; | |
| 44 | + u64 reg_cntfrq; | |
| 45 | + u64 reg_dczid; | |
| 46 | + u64 reg_midr; | |
| 47 | + u64 reg_revidr; | |
| 48 | + u64 reg_gmid; | |
| 49 | + | |
| 50 | + u64 reg_id_aa64dfr0; | |
| 51 | + u64 reg_id_aa64dfr1; | |
| 52 | + u64 reg_id_aa64isar0; | |
| 53 | + u64 reg_id_aa64isar1; | |
| 54 | + u64 reg_id_aa64mmfr0; | |
| 55 | + u64 reg_id_aa64mmfr1; | |
| 56 | + u64 reg_id_aa64mmfr2; | |
| 57 | + u64 reg_id_aa64pfr0; | |
| 58 | + u64 reg_id_aa64pfr1; | |
| 59 | + u64 reg_id_aa64zfr0; | |
| 60 | + | |
| 61 | + struct cpuinfo_32bit aarch32; | |
| 57 | 62 | |
| 58 | 63 | /* pseudo-ZCR for recording maximum ZCR_EL1 LEN value: */ |
| 59 | 64 | u64 reg_zcr; |
| @@ -619,6 +619,13 @@ static inline bool id_aa64pfr0_sve(u64 pfr0) · 618 unmodified lines | ||
| 619 | 619 | return val > 0; |
| 620 | 620 | } |
| 621 | 621 | |
| 622 | +static inline bool id_aa64pfr1_mte(u64 pfr1) | |
| 623 | +{ | |
| 624 | + u32 val = cpuid_feature_extract_unsigned_field(pfr1, ID_AA64PFR1_MTE_SHIFT); | |
| 625 | + | |
| 626 | + return val >= ID_AA64PFR1_MTE; | |
| 627 | +} | |
| 628 | + | |
| 622 | 629 | void __init setup_cpu_features(void); |
| 623 | 630 | void check_local_cpu_capabilities(void); |
| 624 | 631 | |
| @@ -630,9 +637,15 @@ static inline bool cpu_supports_mixed_endian_el0(void) · 5 unmodified lines | ||
| 630 | 637 | return id_aa64mmfr0_mixed_endian_el0(read_cpuid(ID_AA64MMFR0_EL1)); |
| 631 | 638 | } |
| 632 | 639 | |
| 640 | +const struct cpumask *system_32bit_el0_cpumask(void); | |
| 641 | +DECLARE_STATIC_KEY_FALSE(arm64_mismatched_32bit_el0); | |
| 642 | + | |
| 633 | 643 | static inline bool system_supports_32bit_el0(void) |
| 634 | 644 | { |
| 635 | − return cpus_have_const_cap(ARM64_HAS_32BIT_EL0); | |
| 645 | + u64 pfr0 = read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1); | |
| 646 | + | |
| 647 | + return static_branch_unlikely(&arm64_mismatched_32bit_el0) || | |
| 648 | + id_aa64pfr0_32bit_el0(pfr0); | |
| 636 | 649 | } |
| 637 | 650 | |
| 638 | 651 | static inline bool system_supports_4kb_granule(void) |
| @@ -18,4 +18,39 @@ static inline int arm_cpuidle_suspend(int index) · 17 unmodified lines | ||
| 18 | 18 | return -EOPNOTSUPP; |
| 19 | 19 | } |
| 20 | 20 | #endif |
| 21 | + | |
| 22 | +#ifdef CONFIG_ARM64_PSEUDO_NMI | |
| 23 | +#include <asm/arch_gicv3.h> | |
| 24 | + | |
| 25 | +struct arm_cpuidle_irq_context { | |
| 26 | + unsigned long pmr; | |
| 27 | + unsigned long daif_bits; | |
| 28 | +}; | |
| 29 | + | |
| 30 | +#define arm_cpuidle_save_irq_context(__c) \ | |
| 31 | + do { \ | |
| 32 | + struct arm_cpuidle_irq_context *c = __c; \ | |
| 33 | + if (system_uses_irq_prio_masking()) { \ | |
| 34 | + c->daif_bits = read_sysreg(daif); \ | |
| 35 | + write_sysreg(c->daif_bits | PSR_I_BIT | PSR_F_BIT, \ | |
| 36 | + daif); \ | |
| 37 | + c->pmr = gic_read_pmr(); \ | |
| 38 | + gic_write_pmr(GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET); \ | |
| 39 | + } \ | |
| 40 | + } while (0) | |
| 41 | + | |
| 42 | +#define arm_cpuidle_restore_irq_context(__c) \ | |
| 43 | + do { \ | |
| 44 | + struct arm_cpuidle_irq_context *c = __c; \ | |
| 45 | + if (system_uses_irq_prio_masking()) { \ | |
| 46 | + gic_write_pmr(c->pmr); \ | |
| 47 | + write_sysreg(c->daif_bits, daif); \ | |
| 48 | + } \ | |
| 49 | + } while (0) | |
| 50 | +#else | |
| 51 | +struct arm_cpuidle_irq_context { }; | |
| 52 | + | |
| 53 | +#define arm_cpuidle_save_irq_context(c) (void)c | |
| 54 | +#define arm_cpuidle_restore_irq_context(c) (void)c | |
| 55 | +#endif | |
| 21 | 56 | #endif |
| @@ -137,7 +137,7 @@ void efi_virtmap_unload(void); · 136 unmodified lines | ||
| 137 | 137 | |
| 138 | 138 | static inline void efi_capsule_flush_cache_range(void *addr, int size) |
| 139 | 139 | { |
| 140 | − __flush_dcache_area(addr, size); | |
| 140 | + dcache_clean_inval_poc((unsigned long)addr, (unsigned long)addr + size); | |
| 141 | 141 | } |
| 142 | 142 | |
| 143 | 143 | #endif /* _ASM_EFI_H */ |
| @@ -31,20 +31,35 @@ static inline u32 disr_to_esr(u64 disr) · 30 unmodified lines | ||
| 31 | 31 | return esr; |
| 32 | 32 | } |
| 33 | 33 | |
| 34 | −asmlinkage void el1_sync_handler(struct pt_regs *regs); | |
| 35 | −asmlinkage void el0_sync_handler(struct pt_regs *regs); | |
| 36 | −asmlinkage void el0_sync_compat_handler(struct pt_regs *regs); | |
| 34 | +asmlinkage void handle_bad_stack(struct pt_regs *regs); | |
| 37 | 35 | |
| 38 | −asmlinkage void noinstr enter_el1_irq_or_nmi(struct pt_regs *regs); | |
| 39 | −asmlinkage void noinstr exit_el1_irq_or_nmi(struct pt_regs *regs); | |
| 36 | +asmlinkage void el1t_64_sync_handler(struct pt_regs *regs); | |
| 37 | +asmlinkage void el1t_64_irq_handler(struct pt_regs *regs); | |
| 38 | +asmlinkage void el1t_64_fiq_handler(struct pt_regs *regs); | |
| 39 | +asmlinkage void el1t_64_error_handler(struct pt_regs *regs); | |
| 40 | + | |
| 41 | +asmlinkage void el1h_64_sync_handler(struct pt_regs *regs); | |
| 42 | +asmlinkage void el1h_64_irq_handler(struct pt_regs *regs); | |
| 43 | +asmlinkage void el1h_64_fiq_handler(struct pt_regs *regs); | |
| 44 | +asmlinkage void el1h_64_error_handler(struct pt_regs *regs); | |
| 45 | + | |
| 46 | +asmlinkage void el0t_64_sync_handler(struct pt_regs *regs); | |
| 47 | +asmlinkage void el0t_64_irq_handler(struct pt_regs *regs); | |
| 48 | +asmlinkage void el0t_64_fiq_handler(struct pt_regs *regs); | |
| 49 | +asmlinkage void el0t_64_error_handler(struct pt_regs *regs); | |
| 50 | + | |
| 51 | +asmlinkage void el0t_32_sync_handler(struct pt_regs *regs); | |
| 52 | +asmlinkage void el0t_32_irq_handler(struct pt_regs *regs); | |
| 53 | +asmlinkage void el0t_32_fiq_handler(struct pt_regs *regs); | |
| 54 | +asmlinkage void el0t_32_error_handler(struct pt_regs *regs); | |
| 55 | + | |
| 56 | +asmlinkage void call_on_irq_stack(struct pt_regs *regs, | |
| 57 | + void (*func)(struct pt_regs *)); | |
| 40 | 58 | asmlinkage void enter_from_user_mode(void); |
| 41 | 59 | asmlinkage void exit_to_user_mode(void); |
| 42 | −void arm64_enter_nmi(struct pt_regs *regs); | |
| 43 | −void arm64_exit_nmi(struct pt_regs *regs); | |
| 44 | 60 | void do_mem_abort(unsigned long far, unsigned int esr, struct pt_regs *regs); |
| 45 | 61 | void do_undefinstr(struct pt_regs *regs); |
| 46 | 62 | void do_bti(struct pt_regs *regs); |
| 47 | −asmlinkage void bad_mode(struct pt_regs *regs, int reason, unsigned int esr); | |
| 48 | 63 | void do_debug_exception(unsigned long addr_if_watchpoint, unsigned int esr, |
| 49 | 64 | struct pt_regs *regs); |
| 50 | 65 | void do_fpsimd_acc(unsigned int esr, struct pt_regs *regs); |
| @@ -57,4 +72,7 @@ void do_cp15instr(unsigned int esr, struct pt_regs *regs); · 6 unmodified lines | ||
| 57 | 72 | void do_el0_svc(struct pt_regs *regs); |
| 58 | 73 | void do_el0_svc_compat(struct pt_regs *regs); |
| 59 | 74 | void do_ptrauth_fault(struct pt_regs *regs, unsigned int esr); |
| 75 | +void do_serror(struct pt_regs *regs, unsigned int esr); | |
| 76 | + | |
| 77 | +void panic_bad_stack(struct pt_regs *regs, unsigned int esr, unsigned long far); | |
| 60 | 78 | #endif /* __ASM_EXCEPTION_H */ |
| @@ -69,7 +69,7 @@ static inline void *sve_pffr(struct thread_struct *thread) · 68 unmodified lines | ||
| 69 | 69 | extern void sve_save_state(void *state, u32 *pfpsr); |
| 70 | 70 | extern void sve_load_state(void const *state, u32 const *pfpsr, |
| 71 | 71 | unsigned long vq_minus_1); |
| 72 | −extern void sve_flush_live(void); | |
| 72 | +extern void sve_flush_live(unsigned long vq_minus_1); | |
| 73 | 73 | extern void sve_load_from_fpsimd_state(struct user_fpsimd_state const *state, |
| 74 | 74 | unsigned long vq_minus_1); |
| 75 | 75 | extern unsigned int sve_get_vl(void); |
| @@ -213,8 +213,10 @@ · 212 unmodified lines | ||
| 213 | 213 | mov v\nz\().16b, v\nz\().16b |
| 214 | 214 | .endm |
| 215 | 215 | |
| 216 | −.macro sve_flush | |
| 216 | +.macro sve_flush_z | |
| 217 | 217 | _for n, 0, 31, _sve_flush_z \n |
| 218 | +.endm | |
| 219 | +.macro sve_flush_p_ffr | |
| 218 | 220 | _for n, 0, 15, _sve_pfalse \n |
| 219 | 221 | _sve_wrffr 0 |
| 220 | 222 | .endm |
| @@ -0,0 +1,9 @@ | ||
| 1 | +/* SPDX-License-Identifier: GPL-2.0-only */ | |
| 2 | + | |
| 3 | +#ifndef __ASM_INSN_DEF_H | |
| 4 | +#define __ASM_INSN_DEF_H | |
| 5 | + | |
| 6 | +/* A64 instructions are always 32 bits. */ | |
| 7 | +#define AARCH64_INSN_SIZE 4 | |
| 8 | + | |
| 9 | +#endif /* __ASM_INSN_DEF_H */ | |
| @@ -10,7 +10,7 @@ · 9 unmodified lines | ||
| 10 | 10 | #include <linux/build_bug.h> |
| 11 | 11 | #include <linux/types.h> |
| 12 | 12 | |
| 13 | −#include <asm/alternative.h> | |
| 13 | +#include <asm/insn-def.h> | |
| 14 | 14 | |
| 15 | 15 | #ifndef __ASSEMBLY__ |
| 16 | 16 | /* |
| @@ -30,6 +30,7 @@ · 13 unmodified lines | ||
| 30 | 30 | */ |
| 31 | 31 | enum aarch64_insn_encoding_class { |
| 32 | 32 | AARCH64_INSN_CLS_UNKNOWN, /* UNALLOCATED */ |
| 33 | + AARCH64_INSN_CLS_SVE, /* SVE instructions */ | |
| 33 | 34 | AARCH64_INSN_CLS_DP_IMM, /* Data processing - immediate */ |
| 34 | 35 | AARCH64_INSN_CLS_DP_REG, /* Data processing - register */ |
| 35 | 36 | AARCH64_INSN_CLS_DP_FPSIMD, /* Data processing - SIMD and FP */ |
| @@ -294,6 +295,12 @@ __AARCH64_INSN_FUNCS(adr, 0x9F000000, 0x10000000) · 258 unmodified lines | ||
| 294 | 295 | __AARCH64_INSN_FUNCS(adrp, 0x9F000000, 0x90000000) |
| 295 | 296 | __AARCH64_INSN_FUNCS(prfm, 0x3FC00000, 0x39800000) |
| 296 | 297 | __AARCH64_INSN_FUNCS(prfm_lit, 0xFF000000, 0xD8000000) |
| 298 | +__AARCH64_INSN_FUNCS(store_imm, 0x3FC00000, 0x39000000) | |
| 299 | +__AARCH64_INSN_FUNCS(load_imm, 0x3FC00000, 0x39400000) | |
| 300 | +__AARCH64_INSN_FUNCS(store_pre, 0x3FE00C00, 0x38000C00) | |
| 301 | +__AARCH64_INSN_FUNCS(load_pre, 0x3FE00C00, 0x38400C00) | |
| 302 | +__AARCH64_INSN_FUNCS(store_post, 0x3FE00C00, 0x38000400) | |
| 303 | +__AARCH64_INSN_FUNCS(load_post, 0x3FE00C00, 0x38400400) | |
| 297 | 304 | __AARCH64_INSN_FUNCS(str_reg, 0x3FE0EC00, 0x38206800) |
| 298 | 305 | __AARCH64_INSN_FUNCS(ldadd, 0x3F20FC00, 0x38200000) |
| 299 | 306 | __AARCH64_INSN_FUNCS(ldr_reg, 0x3FE0EC00, 0x38606800) |
| @@ -302,6 +309,8 @@ __AARCH64_INSN_FUNCS(ldrsw_lit, 0xFF000000, 0x98000000) · 2 unmodified lines | ||
| 302 | 309 | __AARCH64_INSN_FUNCS(exclusive, 0x3F800000, 0x08000000) |
| 303 | 310 | __AARCH64_INSN_FUNCS(load_ex, 0x3F400000, 0x08400000) |
| 304 | 311 | __AARCH64_INSN_FUNCS(store_ex, 0x3F400000, 0x08000000) |
| 312 | +__AARCH64_INSN_FUNCS(stp, 0x7FC00000, 0x29000000) | |
| 313 | +__AARCH64_INSN_FUNCS(ldp, 0x7FC00000, 0x29400000) | |
| 305 | 314 | __AARCH64_INSN_FUNCS(stp_post, 0x7FC00000, 0x28800000) |
| 306 | 315 | __AARCH64_INSN_FUNCS(ldp_post, 0x7FC00000, 0x28C00000) |
| 307 | 316 | __AARCH64_INSN_FUNCS(stp_pre, 0x7FC00000, 0x29800000) |
| @@ -334,6 +343,7 @@ __AARCH64_INSN_FUNCS(rev64, 0x7FFFFC00, 0x5AC00C00) · 26 unmodified lines | ||
| 334 | 343 | __AARCH64_INSN_FUNCS(and, 0x7F200000, 0x0A000000) |
| 335 | 344 | __AARCH64_INSN_FUNCS(bic, 0x7F200000, 0x0A200000) |
| 336 | 345 | __AARCH64_INSN_FUNCS(orr, 0x7F200000, 0x2A000000) |
| 346 | +__AARCH64_INSN_FUNCS(mov_reg, 0x7FE0FFE0, 0x2A0003E0) | |
| 337 | 347 | __AARCH64_INSN_FUNCS(orn, 0x7F200000, 0x2A200000) |
| 338 | 348 | __AARCH64_INSN_FUNCS(eor, 0x7F200000, 0x4A000000) |
| 339 | 349 | __AARCH64_INSN_FUNCS(eon, 0x7F200000, 0x4A200000) |
| @@ -368,6 +378,14 @@ __AARCH64_INSN_FUNCS(eret_auth, 0xFFFFFBFF, 0xD69F0BFF) · 28 unmodified lines | ||
| 368 | 378 | __AARCH64_INSN_FUNCS(mrs, 0xFFF00000, 0xD5300000) |
| 369 | 379 | __AARCH64_INSN_FUNCS(msr_imm, 0xFFF8F01F, 0xD500401F) |
| 370 | 380 | __AARCH64_INSN_FUNCS(msr_reg, 0xFFF00000, 0xD5100000) |
| 381 | +__AARCH64_INSN_FUNCS(dmb, 0xFFFFF0FF, 0xD50330BF) | |
| 382 | +__AARCH64_INSN_FUNCS(dsb_base, 0xFFFFF0FF, 0xD503309F) | |
| 383 | +__AARCH64_INSN_FUNCS(dsb_nxs, 0xFFFFF3FF, 0xD503323F) | |
| 384 | +__AARCH64_INSN_FUNCS(isb, 0xFFFFF0FF, 0xD50330DF) | |
| 385 | +__AARCH64_INSN_FUNCS(sb, 0xFFFFFFFF, 0xD50330FF) | |
| 386 | +__AARCH64_INSN_FUNCS(clrex, 0xFFFFF0FF, 0xD503305F) | |
| 387 | +__AARCH64_INSN_FUNCS(ssbb, 0xFFFFFFFF, 0xD503309F) | |
| 388 | +__AARCH64_INSN_FUNCS(pssbb, 0xFFFFFFFF, 0xD503349F) | |
| 371 | 389 | |
| 372 | 390 | #undef __AARCH64_INSN_FUNCS |
| 373 | 391 | |
| @@ -379,8 +397,47 @@ static inline bool aarch64_insn_is_adr_adrp(u32 insn) · 5 unmodified lines | ||
| 379 | 397 | return aarch64_insn_is_adr(insn) || aarch64_insn_is_adrp(insn); |
| 380 | 398 | } |
| 381 | 399 | |
| 382 | −int aarch64_insn_read(void *addr, u32 *insnp); | |
| 383 | −int aarch64_insn_write(void *addr, u32 insn); | |
| 400 | +static inline bool aarch64_insn_is_dsb(u32 insn) | |
| 401 | +{ | |
| 402 | + return aarch64_insn_is_dsb_base(insn) || aarch64_insn_is_dsb_nxs(insn); | |
| 403 | +} | |
| 404 | + | |
| 405 | +static inline bool aarch64_insn_is_barrier(u32 insn) | |
| 406 | +{ | |
| 407 | + return aarch64_insn_is_dmb(insn) || aarch64_insn_is_dsb(insn) || | |
| 408 | + aarch64_insn_is_isb(insn) || aarch64_insn_is_sb(insn) || | |
| 409 | + aarch64_insn_is_clrex(insn) || aarch64_insn_is_ssbb(insn) || | |
| 410 | + aarch64_insn_is_pssbb(insn); | |
| 411 | +} | |
| 412 | + | |
| 413 | +static inline bool aarch64_insn_is_store_single(u32 insn) | |
| 414 | +{ | |
| 415 | + return aarch64_insn_is_store_imm(insn) || | |
| 416 | + aarch64_insn_is_store_pre(insn) || | |
| 417 | + aarch64_insn_is_store_post(insn); | |
| 418 | +} | |
| 419 | + | |
| 420 | +static inline bool aarch64_insn_is_store_pair(u32 insn) | |
| 421 | +{ | |
| 422 | + return aarch64_insn_is_stp(insn) || | |
| 423 | + aarch64_insn_is_stp_pre(insn) || | |
| 424 | + aarch64_insn_is_stp_post(insn); | |
| 425 | +} | |
| 426 | + | |
| 427 | +static inline bool aarch64_insn_is_load_single(u32 insn) | |
| 428 | +{ | |
| 429 | + return aarch64_insn_is_load_imm(insn) || | |
| 430 | + aarch64_insn_is_load_pre(insn) || | |
| 431 | + aarch64_insn_is_load_post(insn); | |
| 432 | +} | |
| 433 | + | |
| 434 | +static inline bool aarch64_insn_is_load_pair(u32 insn) | |
| 435 | +{ | |
| 436 | + return aarch64_insn_is_ldp(insn) || | |
| 437 | + aarch64_insn_is_ldp_pre(insn) || | |
| 438 | + aarch64_insn_is_ldp_post(insn); | |
| 439 | +} | |
| 440 | + | |
| 384 | 441 | enum aarch64_insn_encoding_class aarch64_get_insn_class(u32 insn); |
| 385 | 442 | bool aarch64_insn_uses_literal(u32 insn); |
| 386 | 443 | bool aarch64_insn_is_branch(u32 insn); |
| @@ -487,9 +544,6 @@ u32 aarch64_insn_gen_prefetch(enum aarch64_insn_register base, · 100 unmodified lines | ||
| 487 | 544 | s32 aarch64_get_branch_offset(u32 insn); |
| 488 | 545 | u32 aarch64_set_branch_offset(u32 insn, s32 offset); |
| 489 | 546 | |
| 490 | −int aarch64_insn_patch_text_nosync(void *addr, u32 insn); | |
| 491 | −int aarch64_insn_patch_text(void *addrs[], u32 insns[], int cnt); | |
| 492 | − | |
| 493 | 547 | s32 aarch64_insn_adrp_get_offset(u32 insn); |
| 494 | 548 | u32 aarch64_insn_adrp_set_offset(u32 insn, s32 offset); |
| 495 | 549 | |
| @@ -506,6 +560,7 @@ u32 aarch32_insn_mcr_extract_crm(u32 insn); · 10 unmodified lines | ||
| 506 | 560 | |
| 507 | 561 | typedef bool (pstate_check_t)(unsigned long); |
| 508 | 562 | extern pstate_check_t * const aarch32_opcode_cond_checks[16]; |
| 563 | + | |
| 509 | 564 | #endif /* __ASSEMBLY__ */ |
| 510 | 565 | |
| 511 | 566 | #endif /* __ASM_INSN_H */ |
| @@ -18,9 +18,9 @@ · 17 unmodified lines | ||
| 18 | 18 | * 64K (section size = 512M). |
| 19 | 19 | */ |
| 20 | 20 | #ifdef CONFIG_ARM64_4K_PAGES |
| 21 | −#define ARM64_SWAPPER_USES_SECTION_MAPS 1 | |
| 21 | +#define ARM64_KERNEL_USES_PMD_MAPS 1 | |
| 22 | 22 | #else |
| 23 | −#define ARM64_SWAPPER_USES_SECTION_MAPS 0 | |
| 23 | +#define ARM64_KERNEL_USES_PMD_MAPS 0 | |
| 24 | 24 | #endif |
| 25 | 25 | |
| 26 | 26 | /* |
| @@ -33,7 +33,7 @@ · 6 unmodified lines | ||
| 33 | 33 | * VA range, so pages required to map highest possible PA are reserved in all |
| 34 | 34 | * cases. |
| 35 | 35 | */ |
| 36 | −#if ARM64_SWAPPER_USES_SECTION_MAPS | |
| 36 | +#if ARM64_KERNEL_USES_PMD_MAPS | |
| 37 | 37 | #define SWAPPER_PGTABLE_LEVELS (CONFIG_PGTABLE_LEVELS - 1) |
| 38 | 38 | #define IDMAP_PGTABLE_LEVELS (ARM64_HW_PGTABLE_LEVELS(PHYS_MASK_SHIFT) - 1) |
| 39 | 39 | #else |
| @@ -90,9 +90,9 @@ · 50 unmodified lines | ||
| 90 | 90 | #define IDMAP_DIR_SIZE (IDMAP_PGTABLE_LEVELS * PAGE_SIZE) |
| 91 | 91 | |
| 92 | 92 | /* Initial memory map size */ |
| 93 | −#if ARM64_SWAPPER_USES_SECTION_MAPS | |
| 94 | −#define SWAPPER_BLOCK_SHIFT SECTION_SHIFT | |
| 95 | −#define SWAPPER_BLOCK_SIZE SECTION_SIZE | |
| 93 | +#if ARM64_KERNEL_USES_PMD_MAPS | |
| 94 | +#define SWAPPER_BLOCK_SHIFT PMD_SHIFT | |
| 95 | +#define SWAPPER_BLOCK_SIZE PMD_SIZE | |
| 96 | 96 | #define SWAPPER_TABLE_SHIFT PUD_SHIFT |
| 97 | 97 | #else |
| 98 | 98 | #define SWAPPER_BLOCK_SHIFT PAGE_SHIFT |
| @@ -100,16 +100,13 @@ · 1 unmodified line | ||
| 100 | 100 | #define SWAPPER_TABLE_SHIFT PMD_SHIFT |
| 101 | 101 | #endif |
| 102 | 102 | |
| 103 | −/* The size of the initial kernel direct mapping */ | |
| 104 | −#define SWAPPER_INIT_MAP_SIZE (_AC(1, UL) << SWAPPER_TABLE_SHIFT) | |
| 105 | − | |
| 106 | 103 | /* |
| 107 | 104 | * Initial memory map attributes. |
| 108 | 105 | */ |
| 109 | 106 | #define SWAPPER_PTE_FLAGS (PTE_TYPE_PAGE | PTE_AF | PTE_SHARED) |
| 110 | 107 | #define SWAPPER_PMD_FLAGS (PMD_TYPE_SECT | PMD_SECT_AF | PMD_SECT_S) |
| 111 | 108 | |
| 112 | −#if ARM64_SWAPPER_USES_SECTION_MAPS | |
| 109 | +#if ARM64_KERNEL_USES_PMD_MAPS | |
| 113 | 110 | #define SWAPPER_MM_MMUFLAGS (PMD_ATTRINDX(MT_NORMAL) | SWAPPER_PMD_FLAGS) |
| 114 | 111 | #else |
| 115 | 112 | #define SWAPPER_MM_MMUFLAGS (PTE_ATTRINDX(MT_NORMAL) | SWAPPER_PTE_FLAGS) |
| @@ -125,7 +122,7 @@ · 9 unmodified lines | ||
| 125 | 122 | #if defined(CONFIG_ARM64_4K_PAGES) |
| 126 | 123 | #define ARM64_MEMSTART_SHIFT PUD_SHIFT |
| 127 | 124 | #elif defined(CONFIG_ARM64_16K_PAGES) |
| 128 | −#define ARM64_MEMSTART_SHIFT (PMD_SHIFT + 5) | |
| 125 | +#define ARM64_MEMSTART_SHIFT CONT_PMD_SHIFT | |
| 129 | 126 | #else |
| 130 | 127 | #define ARM64_MEMSTART_SHIFT PMD_SHIFT |
| 131 | 128 | #endif |
| @@ -8,6 +8,7 @@ · 7 unmodified lines | ||
| 8 | 8 | #define __ARM_KVM_ASM_H__ |
| 9 | 9 | |
| 10 | 10 | #include <asm/hyp_image.h> |
| 11 | +#include <asm/insn.h> | |
| 11 | 12 | #include <asm/virt.h> |
| 12 | 13 | |
| 13 | 14 | #define ARM_EXIT_WITH_SERROR_BIT 31 |
| @@ -180,7 +180,8 @@ static inline void *__kvm_vector_slot2addr(void *base, · 179 unmodified lines | ||
| 180 | 180 | |
| 181 | 181 | struct kvm; |
| 182 | 182 | |
| 183 | −#define kvm_flush_dcache_to_poc(a,l) __flush_dcache_area((a), (l)) | |
| 183 | +#define kvm_flush_dcache_to_poc(a,l) \ | |
| 184 | + dcache_clean_inval_poc((unsigned long)(a), (unsigned long)(a)+(l)) | |
| 184 | 185 | |
| 185 | 186 | static inline bool vcpu_has_cache_enabled(struct kvm_vcpu *vcpu) |
| 186 | 187 | { |
| @@ -208,12 +209,12 @@ static inline void __invalidate_icache_guest_page(kvm_pfn_t pfn, · 21 unmodified lines | ||
| 208 | 209 | { |
| 209 | 210 | if (icache_is_aliasing()) { |
| 210 | 211 | /* any kind of VIPT cache */ |
| 211 | − __flush_icache_all(); | |
| 212 | + icache_inval_all_pou(); | |
| 212 | 213 | } else if (is_kernel_in_hyp_mode() || !icache_is_vpipt()) { |
| 213 | 214 | /* PIPT or VPIPT at EL2 (see comment in __kvm_tlb_flush_vmid_ipa) */ |
| 214 | 215 | void *va = page_address(pfn_to_page(pfn)); |
| 215 | 216 | |
| 216 | − invalidate_icache_range((unsigned long)va, | |
| 217 | + icache_inval_pou((unsigned long)va, | |
| 217 | 218 | (unsigned long)va + size); |
| 218 | 219 | } |
| 219 | 220 | } |
| @@ -56,8 +56,16 @@ · 55 unmodified lines | ||
| 56 | 56 | SYM_FUNC_START_ALIAS(__pi_##x); \ |
| 57 | 57 | SYM_FUNC_START_WEAK(x) |
| 58 | 58 | |
| 59 | +#define SYM_FUNC_START_WEAK_ALIAS_PI(x) \ | |
| 60 | + SYM_FUNC_START_ALIAS(__pi_##x); \ | |
| 61 | + SYM_START(x, SYM_L_WEAK, SYM_A_ALIGN) | |
| 62 | + | |
| 59 | 63 | #define SYM_FUNC_END_PI(x) \ |
| 60 | 64 | SYM_FUNC_END(x); \ |
| 61 | 65 | SYM_FUNC_END_ALIAS(__pi_##x) |
| 62 | 66 | |
| 67 | +#define SYM_FUNC_END_ALIAS_PI(x) \ | |
| 68 | + SYM_FUNC_END_ALIAS(x); \ | |
| 69 | + SYM_FUNC_END_ALIAS(__pi_##x) | |
| 70 | + | |
| 63 | 71 | #endif |
| @@ -135,10 +135,8 @@ · 134 unmodified lines | ||
| 135 | 135 | #define MT_NORMAL 0 |
| 136 | 136 | #define MT_NORMAL_TAGGED 1 |
| 137 | 137 | #define MT_NORMAL_NC 2 |
| 138 | −#define MT_NORMAL_WT 3 | |
| 139 | −#define MT_DEVICE_nGnRnE 4 | |
| 140 | −#define MT_DEVICE_nGnRE 5 | |
| 141 | −#define MT_DEVICE_GRE 6 | |
| 138 | +#define MT_DEVICE_nGnRnE 3 | |
| 139 | +#define MT_DEVICE_nGnRE 4 | |
| 142 | 140 | |
| 143 | 141 | /* |
| 144 | 142 | * Memory types for Stage-2 translation |
| @@ -177,9 +177,9 @@ static inline void update_saved_ttbr0(struct task_struct *tsk, · 176 unmodified lines | ||
| 177 | 177 | return; |
| 178 | 178 | |
| 179 | 179 | if (mm == &init_mm) |
| 180 | − ttbr = __pa_symbol(reserved_pg_dir); | |
| 180 | + ttbr = phys_to_ttbr(__pa_symbol(reserved_pg_dir)); | |
| 181 | 181 | else |
| 182 | − ttbr = virt_to_phys(mm->pgd) | ASID(mm) << 48; | |
| 182 | + ttbr = phys_to_ttbr(virt_to_phys(mm->pgd)) | ASID(mm) << 48; | |
| 183 | 183 | |
| 184 | 184 | WRITE_ONCE(task_thread_info(tsk)->ttbr0, ttbr); |
| 185 | 185 | } |
| @@ -1,7 +1,20 @@ | ||
| 1 | −#ifdef CONFIG_ARM64_MODULE_PLTS | |
| 2 | 1 | SECTIONS { |
| 2 | +#ifdef CONFIG_ARM64_MODULE_PLTS | |
| 3 | 3 | .plt 0 (NOLOAD) : { BYTE(0) } |
| 4 | 4 | .init.plt 0 (NOLOAD) : { BYTE(0) } |
| 5 | 5 | .text.ftrace_trampoline 0 (NOLOAD) : { BYTE(0) } |
| 6 | −} | |
| 7 | 6 | #endif |
| 7 | + | |
| 8 | +#ifdef CONFIG_KASAN_SW_TAGS | |
| 9 | + /* | |
| 10 | + * Outlined checks go into comdat-deduplicated sections named .text.hot. | |
| 11 | + * Because they are in comdats they are not combined by the linker and | |
| 12 | + * we otherwise end up with multiple sections with the same .text.hot | |
| 13 | + * name in the .ko file. The kernel module loader warns if it sees | |
| 14 | + * multiple sections with the same name so we use this sections | |
| 15 | + * directive to force them into a single section and silence the | |
| 16 | + * warning. | |
| 17 | + */ | |
| 18 | + .text.hot : { *(.text.hot) } | |
| 19 | +#endif | |
| 20 | +} | |
| @@ -48,43 +48,84 @@ static inline u8 mte_get_random_tag(void) · 47 unmodified lines | ||
| 48 | 48 | return mte_get_ptr_tag(addr); |
| 49 | 49 | } |
| 50 | 50 | |
| 51 | +static inline u64 __stg_post(u64 p) | |
| 52 | +{ | |
| 53 | + asm volatile(__MTE_PREAMBLE "stg %0, [%0], #16" | |
| 54 | + : "+r"(p) | |
| 55 | + : | |
| 56 | + : "memory"); | |
| 57 | + return p; | |
| 58 | +} | |
| 59 | + | |
| 60 | +static inline u64 __stzg_post(u64 p) | |
| 61 | +{ | |
| 62 | + asm volatile(__MTE_PREAMBLE "stzg %0, [%0], #16" | |
| 63 | + : "+r"(p) | |
| 64 | + : | |
| 65 | + : "memory"); | |
| 66 | + return p; | |
| 67 | +} | |
| 68 | + | |
| 69 | +static inline void __dc_gva(u64 p) | |
| 70 | +{ | |
| 71 | + asm volatile(__MTE_PREAMBLE "dc gva, %0" : : "r"(p) : "memory"); | |
| 72 | +} | |
| 73 | + | |
| 74 | +static inline void __dc_gzva(u64 p) | |
| 75 | +{ | |
| 76 | + asm volatile(__MTE_PREAMBLE "dc gzva, %0" : : "r"(p) : "memory"); | |
| 77 | +} | |
| 78 | + | |
| 51 | 79 | /* |
| 52 | 80 | * Assign allocation tags for a region of memory based on the pointer tag. |
| 53 | 81 | * Note: The address must be non-NULL and MTE_GRANULE_SIZE aligned and |
| 54 | − * size must be non-zero and MTE_GRANULE_SIZE aligned. | |
| 82 | + * size must be MTE_GRANULE_SIZE aligned. | |
| 55 | 83 | */ |
| 56 | −static inline void mte_set_mem_tag_range(void *addr, size_t size, | |
| 57 | − u8 tag, bool init) | |
| 84 | +static inline void mte_set_mem_tag_range(void *addr, size_t size, u8 tag, | |
| 85 | + bool init) | |
| 58 | 86 | { |
| 59 | − u64 curr, end; | |
| 87 | + u64 curr, mask, dczid_bs, end1, end2, end3; | |
| 60 | 88 | |
| 61 | − if (!size) | |
| 62 | − return; | |
| 89 | + /* Read DC G(Z)VA block size from the system register. */ | |
| 90 | + dczid_bs = 4ul << (read_cpuid(DCZID_EL0) & 0xf); | |
| 63 | 91 | |
| 64 | 92 | curr = (u64)__tag_set(addr, tag); |
| 65 | − end = curr + size; | |
| 93 | + mask = dczid_bs - 1; | |
| 94 | + /* STG/STZG up to the end of the first block. */ | |
| 95 | + end1 = curr | mask; | |
| 96 | + end3 = curr + size; | |
| 97 | + /* DC GVA / GZVA in [end1, end2) */ | |
| 98 | + end2 = end3 & ~mask; | |
| 66 | 99 | |
| 67 | 100 | /* |
| 68 | − * 'asm volatile' is required to prevent the compiler to move | |
| 69 | − * the statement outside of the loop. | |
| 101 | + * The following code uses STG on the first DC GVA block even if the | |
| 102 | + * start address is aligned - it appears to be faster than an alignment | |
| 103 | + * check + conditional branch. Also, if the range size is at least 2 DC | |
| 104 | + * GVA blocks, the first two loops can use post-condition to save one | |
| 105 | + * branch each. | |
| 70 | 106 | */ |
| 71 | − if (init) { | |
| 72 | − do { | |
| 73 | − asm volatile(__MTE_PREAMBLE "stzg %0, [%0]" | |
| 74 | − : | |
| 75 | − : "r" (curr) | |
| 76 | − : "memory"); | |
| 77 | − curr += MTE_GRANULE_SIZE; | |
| 78 | − } while (curr != end); | |
| 79 | − } else { | |
| 80 | − do { | |
| 81 | − asm volatile(__MTE_PREAMBLE "stg %0, [%0]" | |
| 82 | − : | |
| 83 | − : "r" (curr) | |
| 84 | − : "memory"); | |
| 85 | − curr += MTE_GRANULE_SIZE; | |
| 86 | − } while (curr != end); | |
| 87 | − } | |
| 107 | +#define SET_MEMTAG_RANGE(stg_post, dc_gva) \ | |
| 108 | + do { \ | |
| 109 | + if (size >= 2 * dczid_bs) { \ | |
| 110 | + do { \ | |
| 111 | + curr = stg_post(curr); \ | |
| 112 | + } while (curr < end1); \ | |
| 113 | + \ | |
| 114 | + do { \ | |
| 115 | + dc_gva(curr); \ | |
| 116 | + curr += dczid_bs; \ | |
| 117 | + } while (curr < end2); \ | |
| 118 | + } \ | |
| 119 | + \ | |
| 120 | + while (curr < end3) \ | |
| 121 | + curr = stg_post(curr); \ | |
| 122 | + } while (0) | |
| 123 | + | |
| 124 | + if (init) | |
| 125 | + SET_MEMTAG_RANGE(__stzg_post, __dc_gzva); | |
| 126 | + else | |
| 127 | + SET_MEMTAG_RANGE(__stg_post, __dc_gva); | |
| 128 | +#undef SET_MEMTAG_RANGE | |
| 88 | 129 | } |
| 89 | 130 | |
| 90 | 131 | void mte_enable_kernel_sync(void); |
| @@ -37,6 +37,7 @@ void mte_free_tag_storage(char *storage); · 36 unmodified lines | ||
| 37 | 37 | /* track which pages have valid allocation tags */ |
| 38 | 38 | #define PG_mte_tagged PG_arch_2 |
| 39 | 39 | |
| 40 | +void mte_zero_clear_page_tags(void *addr); | |
| 40 | 41 | void mte_sync_tags(pte_t *ptep, pte_t pte); |
| 41 | 42 | void mte_copy_page_tags(void *kto, const void *kfrom); |
| 42 | 43 | void mte_thread_init_user(void); |
| @@ -53,6 +54,9 @@ int mte_ptrace_copy_tags(struct task_struct *child, long request, · 10 unmodified lines | ||
| 53 | 54 | /* unused if !CONFIG_ARM64_MTE, silence the compiler */ |
| 54 | 55 | #define PG_mte_tagged 0 |
| 55 | 56 | |
| 57 | +static inline void mte_zero_clear_page_tags(void *addr) | |
| 58 | +{ | |
| 59 | +} | |
| 56 | 60 | static inline void mte_sync_tags(pte_t *ptep, pte_t pte) |
| 57 | 61 | { |
| 58 | 62 | } |
| @@ -13,6 +13,7 @@ · 12 unmodified lines | ||
| 13 | 13 | #ifndef __ASSEMBLY__ |
| 14 | 14 | |
| 15 | 15 | #include <linux/personality.h> /* for READ_IMPLIES_EXEC */ |
| 16 | +#include <linux/types.h> /* for gfp_t */ | |
| 16 | 17 | #include <asm/pgtable-types.h> |
| 17 | 18 | |
| 18 | 19 | struct page; |
| @@ -28,9 +29,12 @@ void copy_user_highpage(struct page *to, struct page *from, · 9 unmodified lines | ||
| 28 | 29 | void copy_highpage(struct page *to, struct page *from); |
| 29 | 30 | #define __HAVE_ARCH_COPY_HIGHPAGE |
| 30 | 31 | |
| 31 | −#define __alloc_zeroed_user_highpage(movableflags, vma, vaddr) \ | |
| 32 | − alloc_page_vma(GFP_HIGHUSER | __GFP_ZERO | movableflags, vma, vaddr) | |
| 33 | −#define __HAVE_ARCH_ALLOC_ZEROED_USER_HIGHPAGE | |
| 32 | +struct page *alloc_zeroed_user_highpage_movable(struct vm_area_struct *vma, | |
| 33 | + unsigned long vaddr); | |
| 34 | +#define __HAVE_ARCH_ALLOC_ZEROED_USER_HIGHPAGE_MOVABLE | |
| 35 | + | |
| 36 | +void tag_clear_highpage(struct page *to); | |
| 37 | +#define __HAVE_ARCH_TAG_CLEAR_HIGHPAGE | |
| 34 | 38 | |
| 35 | 39 | #define clear_user_page(page, vaddr, pg) clear_page(page) |
| 36 | 40 | #define copy_user_page(to, from, vaddr, pg) copy_page(to, from) |
| @@ -0,0 +1,13 @@ | ||
| 1 | +/* SPDX-License-Identifier: GPL-2.0-only */ | |
| 2 | +#ifndef __ASM_PATCHING_H | |
| 3 | +#define __ASM_PATCHING_H | |
| 4 | + | |
| 5 | +#include <linux/types.h> | |
| 6 | + | |
| 7 | +int aarch64_insn_read(void *addr, u32 *insnp); | |
| 8 | +int aarch64_insn_write(void *addr, u32 insn); | |
| 9 | + | |
| 10 | +int aarch64_insn_patch_text_nosync(void *addr, u32 insn); | |
| 11 | +int aarch64_insn_patch_text(void *addrs[], u32 insns[], int cnt); | |
| 12 | + | |
| 13 | +#endif /* __ASM_PATCHING_H */ | |
| @@ -239,6 +239,11 @@ · 238 unmodified lines | ||
| 239 | 239 | /* PMMIR_EL1.SLOTS mask */ |
| 240 | 240 | #define ARMV8_PMU_SLOTS_MASK 0xff |
| 241 | 241 | |
| 242 | +#define ARMV8_PMU_BUS_SLOTS_SHIFT 8 | |
| 243 | +#define ARMV8_PMU_BUS_SLOTS_MASK 0xff | |
| 244 | +#define ARMV8_PMU_BUS_WIDTH_SHIFT 16 | |
| 245 | +#define ARMV8_PMU_BUS_WIDTH_MASK 0xf | |
| 246 | + | |
| 242 | 247 | #ifdef CONFIG_PERF_EVENTS |
| 243 | 248 | struct pt_regs; |
| 244 | 249 | extern unsigned long perf_instruction_pointer(struct pt_regs *regs); |
| @@ -71,13 +71,6 @@ · 70 unmodified lines | ||
| 71 | 71 | #define PGDIR_MASK (~(PGDIR_SIZE-1)) |
| 72 | 72 | #define PTRS_PER_PGD (1 << (VA_BITS - PGDIR_SHIFT)) |
| 73 | 73 | |
| 74 | −/* | |
| 75 | − * Section address mask and size definitions. | |
| 76 | − */ | |
| 77 | −#define SECTION_SHIFT PMD_SHIFT | |
| 78 | −#define SECTION_SIZE (_AC(1, UL) << SECTION_SHIFT) | |
| 79 | −#define SECTION_MASK (~(SECTION_SIZE-1)) | |
| 80 | − | |
| 81 | 74 | /* |
| 82 | 75 | * Contiguous page definitions. |
| 83 | 76 | */ |
| @@ -55,7 +55,6 @@ extern bool arm64_use_ng_mappings; · 54 unmodified lines | ||
| 55 | 55 | #define PROT_DEVICE_nGnRnE (PROT_DEFAULT | PTE_PXN | PTE_UXN | PTE_WRITE | PTE_ATTRINDX(MT_DEVICE_nGnRnE)) |
| 56 | 56 | #define PROT_DEVICE_nGnRE (PROT_DEFAULT | PTE_PXN | PTE_UXN | PTE_WRITE | PTE_ATTRINDX(MT_DEVICE_nGnRE)) |
| 57 | 57 | #define PROT_NORMAL_NC (PROT_DEFAULT | PTE_PXN | PTE_UXN | PTE_WRITE | PTE_ATTRINDX(MT_NORMAL_NC)) |
| 58 | −#define PROT_NORMAL_WT (PROT_DEFAULT | PTE_PXN | PTE_UXN | PTE_WRITE | PTE_ATTRINDX(MT_NORMAL_WT)) | |
| 59 | 58 | #define PROT_NORMAL (PROT_DEFAULT | PTE_PXN | PTE_UXN | PTE_WRITE | PTE_ATTRINDX(MT_NORMAL)) |
| 60 | 59 | #define PROT_NORMAL_TAGGED (PROT_DEFAULT | PTE_PXN | PTE_UXN | PTE_WRITE | PTE_ATTRINDX(MT_NORMAL_TAGGED)) |
| 61 | 60 | |
| @@ -511,13 +511,12 @@ extern pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn, · 510 unmodified lines | ||
| 511 | 511 | |
| 512 | 512 | #define pmd_none(pmd) (!pmd_val(pmd)) |
| 513 | 513 | |
| 514 | −#define pmd_bad(pmd) (!(pmd_val(pmd) & PMD_TABLE_BIT)) | |
| 515 | − | |
| 516 | 514 | #define pmd_table(pmd) ((pmd_val(pmd) & PMD_TYPE_MASK) == \ |
| 517 | 515 | PMD_TYPE_TABLE) |
| 518 | 516 | #define pmd_sect(pmd) ((pmd_val(pmd) & PMD_TYPE_MASK) == \ |
| 519 | 517 | PMD_TYPE_SECT) |
| 520 | 518 | #define pmd_leaf(pmd) pmd_sect(pmd) |
| 519 | +#define pmd_bad(pmd) (!pmd_table(pmd)) | |
| 521 | 520 | |
| 522 | 521 | #define pmd_leaf_size(pmd) (pmd_cont(pmd) ? CONT_PMD_SIZE : PMD_SIZE) |
| 523 | 522 | #define pte_leaf_size(pte) (pte_cont(pte) ? CONT_PTE_SIZE : PAGE_SIZE) |
| @@ -604,7 +603,7 @@ static inline unsigned long pmd_page_vaddr(pmd_t pmd) · 80 unmodified lines | ||
| 604 | 603 | pr_err("%s:%d: bad pmd %016llx.\n", __FILE__, __LINE__, pmd_val(e)) |
| 605 | 604 | |
| 606 | 605 | #define pud_none(pud) (!pud_val(pud)) |
| 607 | −#define pud_bad(pud) (!(pud_val(pud) & PUD_TABLE_BIT)) | |
| 606 | +#define pud_bad(pud) (!pud_table(pud)) | |
| 608 | 607 | #define pud_present(pud) pte_present(pud_pte(pud)) |
| 609 | 608 | #define pud_leaf(pud) pud_sect(pud) |
| 610 | 609 | #define pud_valid(pud) pte_valid(pud_pte(pud)) |
| @@ -31,10 +31,6 @@ struct ptrauth_keys_user { · 30 unmodified lines | ||
| 31 | 31 | struct ptrauth_key apga; |
| 32 | 32 | }; |
| 33 | 33 | |
| 34 | −struct ptrauth_keys_kernel { | |
| 35 | − struct ptrauth_key apia; | |
| 36 | −}; | |
| 37 | − | |
| 38 | 34 | #define __ptrauth_key_install_nosync(k, v) \ |
| 39 | 35 | do { \ |
| 40 | 36 | struct ptrauth_key __pki_v = (v); \ |
| @@ -42,6 +38,29 @@ do { \ · 1 unmodified line | ||
| 42 | 38 | write_sysreg_s(__pki_v.hi, SYS_ ## k ## KEYHI_EL1); \ |
| 43 | 39 | } while (0) |
| 44 | 40 | |
| 41 | +#ifdef CONFIG_ARM64_PTR_AUTH_KERNEL | |
| 42 | + | |
| 43 | +struct ptrauth_keys_kernel { | |
| 44 | + struct ptrauth_key apia; | |
| 45 | +}; | |
| 46 | + | |
| 47 | +static __always_inline void ptrauth_keys_init_kernel(struct ptrauth_keys_kernel *keys) | |
| 48 | +{ | |
| 49 | + if (system_supports_address_auth()) | |
| 50 | + get_random_bytes(&keys->apia, sizeof(keys->apia)); | |
| 51 | +} | |
| 52 | + | |
| 53 | +static __always_inline void ptrauth_keys_switch_kernel(struct ptrauth_keys_kernel *keys) | |
| 54 | +{ | |
| 55 | + if (!system_supports_address_auth()) | |
| 56 | + return; | |
| 57 | + | |
| 58 | + __ptrauth_key_install_nosync(APIA, keys->apia); | |
| 59 | + isb(); | |
| 60 | +} | |
| 61 | + | |
| 62 | +#endif /* CONFIG_ARM64_PTR_AUTH_KERNEL */ | |
| 63 | + | |
| 45 | 64 | static inline void ptrauth_keys_install_user(struct ptrauth_keys_user *keys) |
| 46 | 65 | { |
| 47 | 66 | if (system_supports_address_auth()) { |
| @@ -69,21 +88,6 @@ static inline void ptrauth_keys_init_user(struct ptrauth_keys_user *keys) · 21 unmodified lines | ||
| 69 | 88 | ptrauth_keys_install_user(keys); |
| 70 | 89 | } |
| 71 | 90 | |
| 72 | −static __always_inline void ptrauth_keys_init_kernel(struct ptrauth_keys_kernel *keys) | |
| 73 | −{ | |
| 74 | − if (system_supports_address_auth()) | |
| 75 | − get_random_bytes(&keys->apia, sizeof(keys->apia)); | |
| 76 | −} | |
| 77 | − | |
| 78 | −static __always_inline void ptrauth_keys_switch_kernel(struct ptrauth_keys_kernel *keys) | |
| 79 | −{ | |
| 80 | − if (!system_supports_address_auth()) | |
| 81 | − return; | |
| 82 | − | |
| 83 | − __ptrauth_key_install_nosync(APIA, keys->apia); | |
| 84 | − isb(); | |
| 85 | −} | |
| 86 | − | |
| 87 | 91 | extern int ptrauth_prctl_reset_keys(struct task_struct *tsk, unsigned long arg); |
| 88 | 92 | |
| 89 | 93 | extern int ptrauth_set_enabled_keys(struct task_struct *tsk, unsigned long keys, |
| @@ -121,11 +125,6 @@ static __always_inline void ptrauth_enable(void) · 31 unmodified lines | ||
| 121 | 125 | #define ptrauth_thread_switch_user(tsk) \ |
| 122 | 126 | ptrauth_keys_install_user(&(tsk)->thread.keys_user) |
| 123 | 127 | |
| 124 | −#define ptrauth_thread_init_kernel(tsk) \ | |
| 125 | − ptrauth_keys_init_kernel(&(tsk)->thread.keys_kernel) | |
| 126 | −#define ptrauth_thread_switch_kernel(tsk) \ | |
| 127 | − ptrauth_keys_switch_kernel(&(tsk)->thread.keys_kernel) | |
| 128 | − | |
| 129 | 128 | #else /* CONFIG_ARM64_PTR_AUTH */ |
| 130 | 129 | #define ptrauth_enable() |
| 131 | 130 | #define ptrauth_prctl_reset_keys(tsk, arg) (-EINVAL) |
| @@ -134,11 +133,19 @@ static __always_inline void ptrauth_enable(void) · 2 unmodified lines | ||
| 134 | 133 | #define ptrauth_strip_insn_pac(lr) (lr) |
| 135 | 134 | #define ptrauth_suspend_exit() |
| 136 | 135 | #define ptrauth_thread_init_user() |
| 137 | −#define ptrauth_thread_init_kernel(tsk) | |
| 138 | 136 | #define ptrauth_thread_switch_user(tsk) |
| 139 | −#define ptrauth_thread_switch_kernel(tsk) | |
| 140 | 137 | #endif /* CONFIG_ARM64_PTR_AUTH */ |
| 141 | 138 | |
| 139 | +#ifdef CONFIG_ARM64_PTR_AUTH_KERNEL | |
| 140 | +#define ptrauth_thread_init_kernel(tsk) \ | |
| 141 | + ptrauth_keys_init_kernel(&(tsk)->thread.keys_kernel) | |
| 142 | +#define ptrauth_thread_switch_kernel(tsk) \ | |
| 143 | + ptrauth_keys_switch_kernel(&(tsk)->thread.keys_kernel) | |
| 144 | +#else | |
| 145 | +#define ptrauth_thread_init_kernel(tsk) | |
| 146 | +#define ptrauth_thread_switch_kernel(tsk) | |
| 147 | +#endif /* CONFIG_ARM64_PTR_AUTH_KERNEL */ | |
| 148 | + | |
| 142 | 149 | #define PR_PAC_ENABLED_KEYS_MASK \ |
| 143 | 150 | (PR_PAC_APIAKEY | PR_PAC_APIBKEY | PR_PAC_APDAKEY | PR_PAC_APDBKEY) |
| 144 | 151 | |
| @@ -148,8 +148,10 @@ struct thread_struct { · 147 unmodified lines | ||
| 148 | 148 | struct debug_info debug; /* debugging */ |
| 149 | 149 | #ifdef CONFIG_ARM64_PTR_AUTH |
| 150 | 150 | struct ptrauth_keys_user keys_user; |
| 151 | +#ifdef CONFIG_ARM64_PTR_AUTH_KERNEL | |
| 151 | 152 | struct ptrauth_keys_kernel keys_kernel; |
| 152 | 153 | #endif |
| 154 | +#endif | |
| 153 | 155 | #ifdef CONFIG_ARM64_MTE |
| 154 | 156 | u64 gcr_user_excl; |
| 155 | 157 | #endif |
| @@ -257,8 +259,6 @@ void set_task_sctlr_el1(u64 sctlr); · 101 unmodified lines | ||
| 257 | 259 | extern struct task_struct *cpu_switch_to(struct task_struct *prev, |
| 258 | 260 | struct task_struct *next); |
| 259 | 261 | |
| 260 | −asmlinkage void arm64_preempt_schedule_irq(void); | |
| 261 | − | |
| 262 | 262 | #define task_pt_regs(p) \ |
| 263 | 263 | ((struct pt_regs *)(THREAD_SIZE + task_stack_page(p)) - 1) |
| 264 | 264 | |
| @@ -329,13 +329,13 @@ long get_tagged_addr_ctrl(struct task_struct *task); · 64 unmodified lines | ||
| 329 | 329 | * of header definitions for the use of task_stack_page. |
| 330 | 330 | */ |
| 331 | 331 | |
| 332 | −#define current_top_of_stack() \ | |
| 333 | −({ \ | |
| 334 | − struct stack_info _info; \ | |
| 335 | − BUG_ON(!on_accessible_stack(current, current_stack_pointer, &_info)); \ | |
| 336 | − _info.high; \ | |
| 332 | +#define current_top_of_stack() \ | |
| 333 | +({ \ | |
| 334 | + struct stack_info _info; \ | |
| 335 | + BUG_ON(!on_accessible_stack(current, current_stack_pointer, 1, &_info)); \ | |
| 336 | + _info.high; \ | |
| 337 | 337 | }) |
| 338 | −#define on_thread_stack() (on_task_stack(current, current_stack_pointer, NULL)) | |
| 338 | +#define on_thread_stack() (on_task_stack(current, current_stack_pointer, 1, NULL)) | |
| 339 | 339 | |
| 340 | 340 | #endif /* __ASSEMBLY__ */ |
| 341 | 341 | #endif /* __ASM_PROCESSOR_H */ |
| @@ -9,18 +9,18 @@ · 8 unmodified lines | ||
| 9 | 9 | #ifdef CONFIG_SHADOW_CALL_STACK |
| 10 | 10 | scs_sp .req x18 |
| 11 | 11 | |
| 12 | − .macro scs_load tsk, tmp | |
| 12 | + .macro scs_load tsk | |
| 13 | 13 | ldr scs_sp, [\tsk, #TSK_TI_SCS_SP] |
| 14 | 14 | .endm |
| 15 | 15 | |
| 16 | − .macro scs_save tsk, tmp | |
| 16 | + .macro scs_save tsk | |
| 17 | 17 | str scs_sp, [\tsk, #TSK_TI_SCS_SP] |
| 18 | 18 | .endm |
| 19 | 19 | #else |
| 20 | − .macro scs_load tsk, tmp | |
| 20 | + .macro scs_load tsk | |
| 21 | 21 | .endm |
| 22 | 22 | |
| 23 | − .macro scs_save tsk, tmp | |
| 23 | + .macro scs_save tsk | |
| 24 | 24 | .endm |
| 25 | 25 | #endif /* CONFIG_SHADOW_CALL_STACK */ |
| 26 | 26 | |
| @@ -37,13 +37,17 @@ struct sdei_registered_event; · 36 unmodified lines | ||
| 37 | 37 | asmlinkage unsigned long __sdei_handler(struct pt_regs *regs, |
| 38 | 38 | struct sdei_registered_event *arg); |
| 39 | 39 | |
| 40 | +unsigned long do_sdei_event(struct pt_regs *regs, | |
| 41 | + struct sdei_registered_event *arg); | |
| 42 | + | |
| 40 | 43 | unsigned long sdei_arch_get_entry_point(int conduit); |
| 41 | 44 | #define sdei_arch_get_entry_point(x) sdei_arch_get_entry_point(x) |
| 42 | 45 | |
| 43 | 46 | struct stack_info; |
| 44 | 47 | |
| 45 | −bool _on_sdei_stack(unsigned long sp, struct stack_info *info); | |
| 46 | −static inline bool on_sdei_stack(unsigned long sp, | |
| 48 | +bool _on_sdei_stack(unsigned long sp, unsigned long size, | |
| 49 | + struct stack_info *info); | |
| 50 | +static inline bool on_sdei_stack(unsigned long sp, unsigned long size, | |
| 47 | 51 | struct stack_info *info) |
| 48 | 52 | { |
| 49 | 53 | if (!IS_ENABLED(CONFIG_VMAP_STACK)) |
| @@ -51,7 +55,7 @@ static inline bool on_sdei_stack(unsigned long sp, · 1 unmodified line | ||
| 51 | 55 | if (!IS_ENABLED(CONFIG_ARM_SDE_INTERFACE)) |
| 52 | 56 | return false; |
| 53 | 57 | if (in_nmi()) |
| 54 | − return _on_sdei_stack(sp, info); | |
| 58 | + return _on_sdei_stack(sp, size, info); | |
| 55 | 59 | |
| 56 | 60 | return false; |
| 57 | 61 | } |
| @@ -73,12 +73,10 @@ asmlinkage void secondary_start_kernel(void); · 72 unmodified lines | ||
| 73 | 73 | |
| 74 | 74 | /* |
| 75 | 75 | * Initial data for bringing up a secondary CPU. |
| 76 | − * @stack - sp for the secondary CPU | |
| 77 | 76 | * @status - Result passed back from the secondary CPU to |
| 78 | 77 | * indicate failure. |
| 79 | 78 | */ |
| 80 | 79 | struct secondary_data { |
| 81 | − void *stack; | |
| 82 | 80 | struct task_struct *task; |
| 83 | 81 | long status; |
| 84 | 82 | }; |
| @@ -69,14 +69,14 @@ extern void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk, · 68 unmodified lines | ||
| 69 | 69 | |
| 70 | 70 | DECLARE_PER_CPU(unsigned long *, irq_stack_ptr); |
| 71 | 71 | |
| 72 | −static inline bool on_stack(unsigned long sp, unsigned long low, | |
| 73 | − unsigned long high, enum stack_type type, | |
| 74 | − struct stack_info *info) | |
| 72 | +static inline bool on_stack(unsigned long sp, unsigned long size, | |
| 73 | + unsigned long low, unsigned long high, | |
| 74 | + enum stack_type type, struct stack_info *info) | |
| 75 | 75 | { |
| 76 | 76 | if (!low) |
| 77 | 77 | return false; |
| 78 | 78 | |
| 79 | − if (sp < low || sp >= high) | |
| 79 | + if (sp < low || sp + size < sp || sp + size > high) | |
| 80 | 80 | return false; |
| 81 | 81 | |
| 82 | 82 | if (info) { |
| @@ -87,38 +87,38 @@ static inline bool on_stack(unsigned long sp, unsigned long low, · 4 unmodified lines | ||
| 87 | 87 | return true; |
| 88 | 88 | } |
| 89 | 89 | |
| 90 | −static inline bool on_irq_stack(unsigned long sp, | |
| 90 | +static inline bool on_irq_stack(unsigned long sp, unsigned long size, | |
| 91 | 91 | struct stack_info *info) |
| 92 | 92 | { |
| 93 | 93 | unsigned long low = (unsigned long)raw_cpu_read(irq_stack_ptr); |
| 94 | 94 | unsigned long high = low + IRQ_STACK_SIZE; |
| 95 | 95 | |
| 96 | − return on_stack(sp, low, high, STACK_TYPE_IRQ, info); | |
| 96 | + return on_stack(sp, size, low, high, STACK_TYPE_IRQ, info); | |
| 97 | 97 | } |
| 98 | 98 | |
| 99 | 99 | static inline bool on_task_stack(const struct task_struct *tsk, |
| 100 | − unsigned long sp, | |
| 100 | + unsigned long sp, unsigned long size, | |
| 101 | 101 | struct stack_info *info) |
| 102 | 102 | { |
| 103 | 103 | unsigned long low = (unsigned long)task_stack_page(tsk); |
| 104 | 104 | unsigned long high = low + THREAD_SIZE; |
| 105 | 105 | |
| 106 | − return on_stack(sp, low, high, STACK_TYPE_TASK, info); | |
| 106 | + return on_stack(sp, size, low, high, STACK_TYPE_TASK, info); | |
| 107 | 107 | } |
| 108 | 108 | |
| 109 | 109 | #ifdef CONFIG_VMAP_STACK |
| 110 | 110 | DECLARE_PER_CPU(unsigned long [OVERFLOW_STACK_SIZE/sizeof(long)], overflow_stack); |
| 111 | 111 | |
| 112 | −static inline bool on_overflow_stack(unsigned long sp, | |
| 112 | +static inline bool on_overflow_stack(unsigned long sp, unsigned long size, | |
| 113 | 113 | struct stack_info *info) |
| 114 | 114 | { |
| 115 | 115 | unsigned long low = (unsigned long)raw_cpu_ptr(overflow_stack); |
| 116 | 116 | unsigned long high = low + OVERFLOW_STACK_SIZE; |
| 117 | 117 | |
| 118 | − return on_stack(sp, low, high, STACK_TYPE_OVERFLOW, info); | |
| 118 | + return on_stack(sp, size, low, high, STACK_TYPE_OVERFLOW, info); | |
| 119 | 119 | } |
| 120 | 120 | #else |
| 121 | −static inline bool on_overflow_stack(unsigned long sp, | |
| 121 | +static inline bool on_overflow_stack(unsigned long sp, unsigned long size, | |
| 122 | 122 | struct stack_info *info) { return false; } |
| 123 | 123 | #endif |
| 124 | 124 | |
| @@ -128,21 +128,21 @@ static inline bool on_overflow_stack(unsigned long sp, · 3 unmodified lines | ||
| 128 | 128 | * context. |
| 129 | 129 | */ |
| 130 | 130 | static inline bool on_accessible_stack(const struct task_struct *tsk, |
| 131 | − unsigned long sp, | |
| 131 | + unsigned long sp, unsigned long size, | |
| 132 | 132 | struct stack_info *info) |
| 133 | 133 | { |
| 134 | 134 | if (info) |
| 135 | 135 | info->type = STACK_TYPE_UNKNOWN; |
| 136 | 136 | |
| 137 | − if (on_task_stack(tsk, sp, info)) | |
| 137 | + if (on_task_stack(tsk, sp, size, info)) | |
| 138 | 138 | return true; |
| 139 | 139 | if (tsk != current || preemptible()) |
| 140 | 140 | return false; |
| 141 | − if (on_irq_stack(sp, info)) | |
| 141 | + if (on_irq_stack(sp, size, info)) | |
| 142 | 142 | return true; |
| 143 | − if (on_overflow_stack(sp, info)) | |
| 143 | + if (on_overflow_stack(sp, size, info)) | |
| 144 | 144 | return true; |
| 145 | − if (on_sdei_stack(sp, info)) | |
| 145 | + if (on_sdei_stack(sp, size, info)) | |
| 146 | 146 | return true; |
| 147 | 147 | |
| 148 | 148 | return false; |
| @@ -703,9 +703,7 @@ · 702 unmodified lines | ||
| 703 | 703 | /* MAIR_ELx memory attributes (used by Linux) */ |
| 704 | 704 | #define MAIR_ATTR_DEVICE_nGnRnE UL(0x00) |
| 705 | 705 | #define MAIR_ATTR_DEVICE_nGnRE UL(0x04) |
| 706 | −#define MAIR_ATTR_DEVICE_GRE UL(0x0c) | |
| 707 | 706 | #define MAIR_ATTR_NORMAL_NC UL(0x44) |
| 708 | −#define MAIR_ATTR_NORMAL_WT UL(0xbb) | |
| 709 | 707 | #define MAIR_ATTR_NORMAL_TAGGED UL(0xf0) |
| 710 | 708 | #define MAIR_ATTR_NORMAL UL(0xff) |
| 711 | 709 | #define MAIR_ATTR_MASK UL(0xff) |
| @@ -28,6 +28,10 @@ static void tlb_flush(struct mmu_gather *tlb); · 27 unmodified lines | ||
| 28 | 28 | */ |
| 29 | 29 | static inline int tlb_get_level(struct mmu_gather *tlb) |
| 30 | 30 | { |
| 31 | + /* The TTL field is only valid for the leaf entry. */ | |
| 32 | + if (tlb->freed_tables) | |
| 33 | + return 0; | |
| 34 | + | |
| 31 | 35 | if (tlb->cleared_ptes && !(tlb->cleared_pmds || |
| 32 | 36 | tlb->cleared_puds || |
| 33 | 37 | tlb->cleared_p4ds)) |
| @@ -14,15 +14,22 @@ CFLAGS_REMOVE_return_address.o = $(CC_FLAGS_FTRACE) · 13 unmodified lines | ||
| 14 | 14 | CFLAGS_REMOVE_syscall.o = -fstack-protector -fstack-protector-strong |
| 15 | 15 | CFLAGS_syscall.o += -fno-stack-protector |
| 16 | 16 | |
| 17 | +# It's not safe to invoke KCOV when portions of the kernel environment aren't | |
| 18 | +# available or are out-of-sync with HW state. Since `noinstr` doesn't always | |
| 19 | +# inhibit KCOV instrumentation, disable it for the entire compilation unit. | |
| 20 | +KCOV_INSTRUMENT_entry.o := n | |
| 21 | +KCOV_INSTRUMENT_idle.o := n | |
| 22 | + | |
| 17 | 23 | # Object file lists. |
| 18 | 24 | obj-y := debug-monitors.o entry.o irq.o fpsimd.o \ |
| 19 | 25 | entry-common.o entry-fpsimd.o process.o ptrace.o \ |
| 20 | 26 | setup.o signal.o sys.o stacktrace.o time.o traps.o \ |
| 21 | − io.o vdso.o hyp-stub.o psci.o cpu_ops.o insn.o \ | |
| 27 | + io.o vdso.o hyp-stub.o psci.o cpu_ops.o \ | |
| 22 | 28 | return_address.o cpuinfo.o cpu_errata.o \ |
| 23 | 29 | cpufeature.o alternative.o cacheinfo.o \ |
| 24 | 30 | smp.o smp_spin_table.o topology.o smccc-call.o \ |
| 25 | − syscall.o proton-pack.o idreg-override.o | |
| 31 | + syscall.o proton-pack.o idreg-override.o idle.o \ | |
| 32 | + patching.o | |
| 26 | 33 | |
| 27 | 34 | targets += efi-entry.o |
| 28 | 35 | |
| @@ -239,6 +239,18 @@ done: · 238 unmodified lines | ||
| 239 | 239 | } |
| 240 | 240 | } |
| 241 | 241 | |
| 242 | +static pgprot_t __acpi_get_writethrough_mem_attribute(void) | |
| 243 | +{ | |
| 244 | + /* | |
| 245 | + * Although UEFI specifies the use of Normal Write-through for | |
| 246 | + * EFI_MEMORY_WT, it is seldom used in practice and not implemented | |
| 247 | + * by most (all?) CPUs. Rather than allocate a MAIR just for this | |
| 248 | + * purpose, emit a warning and use Normal Non-cacheable instead. | |
| 249 | + */ | |
| 250 | + pr_warn_once("No MAIR allocation for EFI_MEMORY_WT; treating as Normal Non-cacheable\n"); | |
| 251 | + return __pgprot(PROT_NORMAL_NC); | |
| 252 | +} | |
| 253 | + | |
| 242 | 254 | pgprot_t __acpi_get_mem_attribute(phys_addr_t addr) |
| 243 | 255 | { |
| 244 | 256 | /* |
| @@ -246,7 +258,7 @@ pgprot_t __acpi_get_mem_attribute(phys_addr_t addr) · 1 unmodified line | ||
| 246 | 258 | * types" of UEFI 2.5 section 2.3.6.1, each EFI memory type is |
| 247 | 259 | * mapped to a corresponding MAIR attribute encoding. |
| 248 | 260 | * The EFI memory attribute advises all possible capabilities |
| 249 | − * of a memory region. We use the most efficient capability. | |
| 261 | + * of a memory region. | |
| 250 | 262 | */ |
| 251 | 263 | |
| 252 | 264 | u64 attr; |
| @@ -254,10 +266,10 @@ pgprot_t __acpi_get_mem_attribute(phys_addr_t addr) · 1 unmodified line | ||
| 254 | 266 | attr = efi_mem_attributes(addr); |
| 255 | 267 | if (attr & EFI_MEMORY_WB) |
| 256 | 268 | return PAGE_KERNEL; |
| 257 | − if (attr & EFI_MEMORY_WT) | |
| 258 | − return __pgprot(PROT_NORMAL_WT); | |
| 259 | 269 | if (attr & EFI_MEMORY_WC) |
| 260 | 270 | return __pgprot(PROT_NORMAL_NC); |
| 271 | + if (attr & EFI_MEMORY_WT) | |
| 272 | + return __acpi_get_writethrough_mem_attribute(); | |
| 261 | 273 | return __pgprot(PROT_DEVICE_nGnRnE); |
| 262 | 274 | } |
| 263 | 275 | |
| @@ -340,10 +352,10 @@ void __iomem *acpi_os_ioremap(acpi_physical_address phys, acpi_size size) · 76 unmodified lines | ||
| 340 | 352 | default: |
| 341 | 353 | if (region->attribute & EFI_MEMORY_WB) |
| 342 | 354 | prot = PAGE_KERNEL; |
| 343 | − else if (region->attribute & EFI_MEMORY_WT) | |
| 344 | − prot = __pgprot(PROT_NORMAL_WT); | |
| 345 | 355 | else if (region->attribute & EFI_MEMORY_WC) |
| 346 | 356 | prot = __pgprot(PROT_NORMAL_NC); |
| 357 | + else if (region->attribute & EFI_MEMORY_WT) | |
| 358 | + prot = __acpi_get_writethrough_mem_attribute(); | |
| 347 | 359 | } |
| 348 | 360 | } |
| 349 | 361 | return __ioremap(phys, size, prot); |
| @@ -181,7 +181,7 @@ static void __nocfi __apply_alternatives(struct alt_region *region, bool is_modu · 180 unmodified lines | ||
| 181 | 181 | */ |
| 182 | 182 | if (!is_module) { |
| 183 | 183 | dsb(ish); |
| 184 | − __flush_icache_all(); | |
| 184 | + icache_inval_all_pou(); | |
| 185 | 185 | isb(); |
| 186 | 186 | |
| 187 | 187 | /* Ignore ARM64_CB bit from feature mask */ |
| @@ -27,6 +27,7 @@ · 26 unmodified lines | ||
| 27 | 27 | int main(void) |
| 28 | 28 | { |
| 29 | 29 | DEFINE(TSK_ACTIVE_MM, offsetof(struct task_struct, active_mm)); |
| 30 | + DEFINE(TSK_CPU, offsetof(struct task_struct, cpu)); | |
| 30 | 31 | BLANK(); |
| 31 | 32 | DEFINE(TSK_TI_FLAGS, offsetof(struct task_struct, thread_info.flags)); |
| 32 | 33 | DEFINE(TSK_TI_PREEMPT, offsetof(struct task_struct, thread_info.preempt_count)); |
| @@ -46,6 +47,8 @@ int main(void) · 13 unmodified lines | ||
| 46 | 47 | DEFINE(THREAD_SCTLR_USER, offsetof(struct task_struct, thread.sctlr_user)); |
| 47 | 48 | #ifdef CONFIG_ARM64_PTR_AUTH |
| 48 | 49 | DEFINE(THREAD_KEYS_USER, offsetof(struct task_struct, thread.keys_user)); |
| 50 | +#endif | |
| 51 | +#ifdef CONFIG_ARM64_PTR_AUTH_KERNEL | |
| 49 | 52 | DEFINE(THREAD_KEYS_KERNEL, offsetof(struct task_struct, thread.keys_kernel)); |
| 50 | 53 | #endif |
| 51 | 54 | #ifdef CONFIG_ARM64_MTE |
| @@ -99,7 +102,6 @@ int main(void) · 47 unmodified lines | ||
| 99 | 102 | DEFINE(SOFTIRQ_SHIFT, SOFTIRQ_SHIFT); |
| 100 | 103 | DEFINE(IRQ_CPUSTAT_SOFTIRQ_PENDING, offsetof(irq_cpustat_t, __softirq_pending)); |
| 101 | 104 | BLANK(); |
| 102 | − DEFINE(CPU_BOOT_STACK, offsetof(struct secondary_data, stack)); | |
| 103 | 105 | DEFINE(CPU_BOOT_TASK, offsetof(struct secondary_data, task)); |
| 104 | 106 | BLANK(); |
| 105 | 107 | DEFINE(FTR_OVR_VAL_OFFSET, offsetof(struct arm64_ftr_override, val)); |
| @@ -138,6 +140,15 @@ int main(void) · 32 unmodified lines | ||
| 138 | 140 | DEFINE(ARM_SMCCC_RES_X2_OFFS, offsetof(struct arm_smccc_res, a2)); |
| 139 | 141 | DEFINE(ARM_SMCCC_QUIRK_ID_OFFS, offsetof(struct arm_smccc_quirk, id)); |
| 140 | 142 | DEFINE(ARM_SMCCC_QUIRK_STATE_OFFS, offsetof(struct arm_smccc_quirk, state)); |
| 143 | + DEFINE(ARM_SMCCC_1_2_REGS_X0_OFFS, offsetof(struct arm_smccc_1_2_regs, a0)); | |
| 144 | + DEFINE(ARM_SMCCC_1_2_REGS_X2_OFFS, offsetof(struct arm_smccc_1_2_regs, a2)); | |
| 145 | + DEFINE(ARM_SMCCC_1_2_REGS_X4_OFFS, offsetof(struct arm_smccc_1_2_regs, a4)); | |
| 146 | + DEFINE(ARM_SMCCC_1_2_REGS_X6_OFFS, offsetof(struct arm_smccc_1_2_regs, a6)); | |
| 147 | + DEFINE(ARM_SMCCC_1_2_REGS_X8_OFFS, offsetof(struct arm_smccc_1_2_regs, a8)); | |
| 148 | + DEFINE(ARM_SMCCC_1_2_REGS_X10_OFFS, offsetof(struct arm_smccc_1_2_regs, a10)); | |
| 149 | + DEFINE(ARM_SMCCC_1_2_REGS_X12_OFFS, offsetof(struct arm_smccc_1_2_regs, a12)); | |
| 150 | + DEFINE(ARM_SMCCC_1_2_REGS_X14_OFFS, offsetof(struct arm_smccc_1_2_regs, a14)); | |
| 151 | + DEFINE(ARM_SMCCC_1_2_REGS_X16_OFFS, offsetof(struct arm_smccc_1_2_regs, a16)); | |
| 141 | 152 | BLANK(); |
| 142 | 153 | DEFINE(HIBERN_PBE_ORIG, offsetof(struct pbe, orig_address)); |
| 143 | 154 | DEFINE(HIBERN_PBE_ADDR, offsetof(struct pbe, address)); |
| @@ -153,7 +164,9 @@ int main(void) · 9 unmodified lines | ||
| 153 | 164 | #endif |
| 154 | 165 | #ifdef CONFIG_ARM64_PTR_AUTH |
| 155 | 166 | DEFINE(PTRAUTH_USER_KEY_APIA, offsetof(struct ptrauth_keys_user, apia)); |
| 167 | +#ifdef CONFIG_ARM64_PTR_AUTH_KERNEL | |
| 156 | 168 | DEFINE(PTRAUTH_KERNEL_KEY_APIA, offsetof(struct ptrauth_keys_kernel, apia)); |
| 169 | +#endif | |
| 157 | 170 | BLANK(); |
| 158 | 171 | #endif |
| 159 | 172 | return 0; |
| @@ -76,6 +76,7 @@ · 75 unmodified lines | ||
| 76 | 76 | #include <asm/cpufeature.h> |
| 77 | 77 | #include <asm/cpu_ops.h> |
| 78 | 78 | #include <asm/fpsimd.h> |
| 79 | +#include <asm/insn.h> | |
| 79 | 80 | #include <asm/kvm_host.h> |
| 80 | 81 | #include <asm/mmu_context.h> |
| 81 | 82 | #include <asm/mte.h> |
| @@ -107,6 +108,24 @@ DECLARE_BITMAP(boot_capabilities, ARM64_NPATCHABLE); · 25 unmodified lines | ||
| 107 | 108 | bool arm64_use_ng_mappings = false; |
| 108 | 109 | EXPORT_SYMBOL(arm64_use_ng_mappings); |
| 109 | 110 | |
| 111 | +/* | |
| 112 | + * Permit PER_LINUX32 and execve() of 32-bit binaries even if not all CPUs | |
| 113 | + * support it? | |
| 114 | + */ | |
| 115 | +static bool __read_mostly allow_mismatched_32bit_el0; | |
| 116 | + | |
| 117 | +/* | |
| 118 | + * Static branch enabled only if allow_mismatched_32bit_el0 is set and we have | |
| 119 | + * seen at least one CPU capable of 32-bit EL0. | |
| 120 | + */ | |
| 121 | +DEFINE_STATIC_KEY_FALSE(arm64_mismatched_32bit_el0); | |
| 122 | + | |
| 123 | +/* | |
| 124 | + * Mask of CPUs supporting 32-bit EL0. | |
| 125 | + * Only valid if arm64_mismatched_32bit_el0 is enabled. | |
| 126 | + */ | |
| 127 | +static cpumask_var_t cpu_32bit_el0_mask __cpumask_var_read_mostly; | |
| 128 | + | |
| 110 | 129 | /* |
| 111 | 130 | * Flag to indicate if we have computed the system wide |
| 112 | 131 | * capabilities based on the boot time active CPUs. This |
| @@ -400,6 +419,11 @@ static const struct arm64_ftr_bits ftr_dczid[] = { · 287 unmodified lines | ||
| 400 | 419 | ARM64_FTR_END, |
| 401 | 420 | }; |
| 402 | 421 | |
| 422 | +static const struct arm64_ftr_bits ftr_gmid[] = { | |
| 423 | + ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, SYS_GMID_EL1_BS_SHIFT, 4, 0), | |
| 424 | + ARM64_FTR_END, | |
| 425 | +}; | |
| 426 | + | |
| 403 | 427 | static const struct arm64_ftr_bits ftr_id_isar0[] = { |
| 404 | 428 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_DIVIDE_SHIFT, 4, 0), |
| 405 | 429 | ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_DEBUG_SHIFT, 4, 0), |
| @@ -617,6 +641,9 @@ static const struct __ftr_reg_entry { · 211 unmodified lines | ||
| 617 | 641 | /* Op1 = 0, CRn = 1, CRm = 2 */ |
| 618 | 642 | ARM64_FTR_REG(SYS_ZCR_EL1, ftr_zcr), |
| 619 | 643 | |
| 644 | + /* Op1 = 1, CRn = 0, CRm = 0 */ | |
| 645 | + ARM64_FTR_REG(SYS_GMID_EL1, ftr_gmid), | |
| 646 | + | |
| 620 | 647 | /* Op1 = 3, CRn = 0, CRm = 0 */ |
| 621 | 648 | { SYS_CTR_EL0, &arm64_ftr_reg_ctrel0 }, |
| 622 | 649 | ARM64_FTR_REG(SYS_DCZID_EL0, ftr_dczid), |
| @@ -767,7 +794,7 @@ static void __init sort_ftr_regs(void) · 144 unmodified lines | ||
| 767 | 794 | * Any bits that are not covered by an arm64_ftr_bits entry are considered |
| 768 | 795 | * RES0 for the system-wide value, and must strictly match. |
| 769 | 796 | */ |
| 770 | −static void __init init_cpu_ftr_reg(u32 sys_reg, u64 new) | |
| 797 | +static void init_cpu_ftr_reg(u32 sys_reg, u64 new) | |
| 771 | 798 | { |
| 772 | 799 | u64 val = 0; |
| 773 | 800 | u64 strict_mask = ~0x0ULL; |
| @@ -863,6 +890,31 @@ static void __init init_cpu_hwcaps_indirect_list(void) · 89 unmodified lines | ||
| 863 | 890 | |
| 864 | 891 | static void __init setup_boot_cpu_capabilities(void); |
| 865 | 892 | |
| 893 | +static void init_32bit_cpu_features(struct cpuinfo_32bit *info) | |
| 894 | +{ | |
| 895 | + init_cpu_ftr_reg(SYS_ID_DFR0_EL1, info->reg_id_dfr0); | |
| 896 | + init_cpu_ftr_reg(SYS_ID_DFR1_EL1, info->reg_id_dfr1); | |
| 897 | + init_cpu_ftr_reg(SYS_ID_ISAR0_EL1, info->reg_id_isar0); | |
| 898 | + init_cpu_ftr_reg(SYS_ID_ISAR1_EL1, info->reg_id_isar1); | |
| 899 | + init_cpu_ftr_reg(SYS_ID_ISAR2_EL1, info->reg_id_isar2); | |
| 900 | + init_cpu_ftr_reg(SYS_ID_ISAR3_EL1, info->reg_id_isar3); | |
| 901 | + init_cpu_ftr_reg(SYS_ID_ISAR4_EL1, info->reg_id_isar4); | |
| 902 | + init_cpu_ftr_reg(SYS_ID_ISAR5_EL1, info->reg_id_isar5); | |
| 903 | + init_cpu_ftr_reg(SYS_ID_ISAR6_EL1, info->reg_id_isar6); | |
| 904 | + init_cpu_ftr_reg(SYS_ID_MMFR0_EL1, info->reg_id_mmfr0); | |
| 905 | + init_cpu_ftr_reg(SYS_ID_MMFR1_EL1, info->reg_id_mmfr1); | |
| 906 | + init_cpu_ftr_reg(SYS_ID_MMFR2_EL1, info->reg_id_mmfr2); | |
| 907 | + init_cpu_ftr_reg(SYS_ID_MMFR3_EL1, info->reg_id_mmfr3); | |
| 908 | + init_cpu_ftr_reg(SYS_ID_MMFR4_EL1, info->reg_id_mmfr4); | |
| 909 | + init_cpu_ftr_reg(SYS_ID_MMFR5_EL1, info->reg_id_mmfr5); | |
| 910 | + init_cpu_ftr_reg(SYS_ID_PFR0_EL1, info->reg_id_pfr0); | |
| 911 | + init_cpu_ftr_reg(SYS_ID_PFR1_EL1, info->reg_id_pfr1); | |
| 912 | + init_cpu_ftr_reg(SYS_ID_PFR2_EL1, info->reg_id_pfr2); | |
| 913 | + init_cpu_ftr_reg(SYS_MVFR0_EL1, info->reg_mvfr0); | |
| 914 | + init_cpu_ftr_reg(SYS_MVFR1_EL1, info->reg_mvfr1); | |
| 915 | + init_cpu_ftr_reg(SYS_MVFR2_EL1, info->reg_mvfr2); | |
| 916 | +} | |
| 917 | + | |
| 866 | 918 | void __init init_cpu_features(struct cpuinfo_arm64 *info) |
| 867 | 919 | { |
| 868 | 920 | /* Before we start using the tables, make sure it is sorted */ |
| @@ -882,35 +934,17 @@ void __init init_cpu_features(struct cpuinfo_arm64 *info) · 13 unmodified lines | ||
| 882 | 934 | init_cpu_ftr_reg(SYS_ID_AA64PFR1_EL1, info->reg_id_aa64pfr1); |
| 883 | 935 | init_cpu_ftr_reg(SYS_ID_AA64ZFR0_EL1, info->reg_id_aa64zfr0); |
| 884 | 936 | |
| 885 | − if (id_aa64pfr0_32bit_el0(info->reg_id_aa64pfr0)) { | |
| 886 | − init_cpu_ftr_reg(SYS_ID_DFR0_EL1, info->reg_id_dfr0); | |
| 887 | − init_cpu_ftr_reg(SYS_ID_DFR1_EL1, info->reg_id_dfr1); | |
| 888 | − init_cpu_ftr_reg(SYS_ID_ISAR0_EL1, info->reg_id_isar0); | |
| 889 | − init_cpu_ftr_reg(SYS_ID_ISAR1_EL1, info->reg_id_isar1); | |
| 890 | − init_cpu_ftr_reg(SYS_ID_ISAR2_EL1, info->reg_id_isar2); | |
| 891 | − init_cpu_ftr_reg(SYS_ID_ISAR3_EL1, info->reg_id_isar3); | |
| 892 | − init_cpu_ftr_reg(SYS_ID_ISAR4_EL1, info->reg_id_isar4); | |
| 893 | − init_cpu_ftr_reg(SYS_ID_ISAR5_EL1, info->reg_id_isar5); | |
| 894 | − init_cpu_ftr_reg(SYS_ID_ISAR6_EL1, info->reg_id_isar6); | |
| 895 | − init_cpu_ftr_reg(SYS_ID_MMFR0_EL1, info->reg_id_mmfr0); | |
| 896 | − init_cpu_ftr_reg(SYS_ID_MMFR1_EL1, info->reg_id_mmfr1); | |
| 897 | − init_cpu_ftr_reg(SYS_ID_MMFR2_EL1, info->reg_id_mmfr2); | |
| 898 | − init_cpu_ftr_reg(SYS_ID_MMFR3_EL1, info->reg_id_mmfr3); | |
| 899 | − init_cpu_ftr_reg(SYS_ID_MMFR4_EL1, info->reg_id_mmfr4); | |
| 900 | − init_cpu_ftr_reg(SYS_ID_MMFR5_EL1, info->reg_id_mmfr5); | |
| 901 | − init_cpu_ftr_reg(SYS_ID_PFR0_EL1, info->reg_id_pfr0); | |
| 902 | − init_cpu_ftr_reg(SYS_ID_PFR1_EL1, info->reg_id_pfr1); | |
| 903 | − init_cpu_ftr_reg(SYS_ID_PFR2_EL1, info->reg_id_pfr2); | |
| 904 | − init_cpu_ftr_reg(SYS_MVFR0_EL1, info->reg_mvfr0); | |
| 905 | − init_cpu_ftr_reg(SYS_MVFR1_EL1, info->reg_mvfr1); | |
| 906 | − init_cpu_ftr_reg(SYS_MVFR2_EL1, info->reg_mvfr2); | |
| 907 | − } | |
| 937 | + if (id_aa64pfr0_32bit_el0(info->reg_id_aa64pfr0)) | |
| 938 | + init_32bit_cpu_features(&info->aarch32); | |
| 908 | 939 | |
| 909 | 940 | if (id_aa64pfr0_sve(info->reg_id_aa64pfr0)) { |
| 910 | 941 | init_cpu_ftr_reg(SYS_ZCR_EL1, info->reg_zcr); |
| 911 | 942 | sve_init_vq_map(); |
| 912 | 943 | } |
| 913 | 944 | |
| 945 | + if (id_aa64pfr1_mte(info->reg_id_aa64pfr1)) | |
| 946 | + init_cpu_ftr_reg(SYS_GMID_EL1, info->reg_gmid); | |
| 947 | + | |
| 914 | 948 | /* |
| 915 | 949 | * Initialize the indirect array of CPU hwcaps capabilities pointers |
| 916 | 950 | * before we handle the boot CPU below. |
| @@ -975,20 +1009,28 @@ static void relax_cpu_ftr_reg(u32 sys_id, int field) · 58 unmodified lines | ||
| 975 | 1009 | WARN_ON(!ftrp->width); |
| 976 | 1010 | } |
| 977 | 1011 | |
| 978 | −static int update_32bit_cpu_features(int cpu, struct cpuinfo_arm64 *info, | |
| 979 | − struct cpuinfo_arm64 *boot) | |
| 1012 | +static void lazy_init_32bit_cpu_features(struct cpuinfo_arm64 *info, | |
| 1013 | + struct cpuinfo_arm64 *boot) | |
| 1014 | +{ | |
| 1015 | + static bool boot_cpu_32bit_regs_overridden = false; | |
| 1016 | + | |
| 1017 | + if (!allow_mismatched_32bit_el0 || boot_cpu_32bit_regs_overridden) | |
| 1018 | + return; | |
| 1019 | + | |
| 1020 | + if (id_aa64pfr0_32bit_el0(boot->reg_id_aa64pfr0)) | |
| 1021 | + return; | |
| 1022 | + | |
| 1023 | + boot->aarch32 = info->aarch32; | |
| 1024 | + init_32bit_cpu_features(&boot->aarch32); | |
| 1025 | + boot_cpu_32bit_regs_overridden = true; | |
| 1026 | +} | |
| 1027 | + | |
| 1028 | +static int update_32bit_cpu_features(int cpu, struct cpuinfo_32bit *info, | |
| 1029 | + struct cpuinfo_32bit *boot) | |
| 980 | 1030 | { |
| 981 | 1031 | int taint = 0; |
| 982 | 1032 | u64 pfr0 = read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1); |
| 983 | 1033 | |
| 984 | − /* | |
| 985 | − * If we don't have AArch32 at all then skip the checks entirely | |
| 986 | − * as the register values may be UNKNOWN and we're not going to be | |
| 987 | − * using them for anything. | |
| 988 | − */ | |
| 989 | − if (!id_aa64pfr0_32bit_el0(pfr0)) | |
| 990 | − return taint; | |
| 991 | − | |
| 992 | 1034 | /* |
| 993 | 1035 | * If we don't have AArch32 at EL1, then relax the strictness of |
| 994 | 1036 | * EL1-dependent register fields to avoid spurious sanity check fails. |
| @@ -1135,10 +1177,29 @@ void update_cpu_features(int cpu, · 140 unmodified lines | ||
| 1135 | 1177 | } |
| 1136 | 1178 | |
| 1137 | 1179 | /* |
| 1180 | + * The kernel uses the LDGM/STGM instructions and the number of tags | |
| 1181 | + * they read/write depends on the GMID_EL1.BS field. Check that the | |
| 1182 | + * value is the same on all CPUs. | |
| 1183 | + */ | |
| 1184 | + if (IS_ENABLED(CONFIG_ARM64_MTE) && | |
| 1185 | + id_aa64pfr1_mte(info->reg_id_aa64pfr1)) { | |
| 1186 | + taint |= check_update_ftr_reg(SYS_GMID_EL1, cpu, | |
| 1187 | + info->reg_gmid, boot->reg_gmid); | |
| 1188 | + } | |
| 1189 | + | |
| 1190 | + /* | |
| 1191 | + * If we don't have AArch32 at all then skip the checks entirely | |
| 1192 | + * as the register values may be UNKNOWN and we're not going to be | |
| 1193 | + * using them for anything. | |
| 1194 | + * | |
| 1138 | 1195 | * This relies on a sanitised view of the AArch64 ID registers |
| 1139 | 1196 | * (e.g. SYS_ID_AA64PFR0_EL1), so we call it last. |
| 1140 | 1197 | */ |
| 1141 | − taint |= update_32bit_cpu_features(cpu, info, boot); | |
| 1198 | + if (id_aa64pfr0_32bit_el0(info->reg_id_aa64pfr0)) { | |
| 1199 | + lazy_init_32bit_cpu_features(info, boot); | |
| 1200 | + taint |= update_32bit_cpu_features(cpu, &info->aarch32, | |
| 1201 | + &boot->aarch32); | |
| 1202 | + } | |
| 1142 | 1203 | |
| 1143 | 1204 | /* |
| 1144 | 1205 | * Mismatched CPU features are a recipe for disaster. Don't even |
| @@ -1248,6 +1309,28 @@ has_cpuid_feature(const struct arm64_cpu_capabilities *entry, int scope) · 103 unmodified lines | ||
| 1248 | 1309 | return feature_matches(val, entry); |
| 1249 | 1310 | } |
| 1250 | 1311 | |
| 1312 | +const struct cpumask *system_32bit_el0_cpumask(void) | |
| 1313 | +{ | |
| 1314 | + if (!system_supports_32bit_el0()) | |
| 1315 | + return cpu_none_mask; | |
| 1316 | + | |
| 1317 | + if (static_branch_unlikely(&arm64_mismatched_32bit_el0)) | |
| 1318 | + return cpu_32bit_el0_mask; | |
| 1319 | + | |
| 1320 | + return cpu_possible_mask; | |
| 1321 | +} | |
| 1322 | + | |
| 1323 | +static bool has_32bit_el0(const struct arm64_cpu_capabilities *entry, int scope) | |
| 1324 | +{ | |
| 1325 | + if (!has_cpuid_feature(entry, scope)) | |
| 1326 | + return allow_mismatched_32bit_el0; | |
| 1327 | + | |
| 1328 | + if (scope == SCOPE_SYSTEM) | |
| 1329 | + pr_info("detected: 32-bit EL0 Support\n"); | |
| 1330 | + | |
| 1331 | + return true; | |
| 1332 | +} | |
| 1333 | + | |
| 1251 | 1334 | static bool has_useable_gicv3_cpuif(const struct arm64_cpu_capabilities *entry, int scope) |
| 1252 | 1335 | { |
| 1253 | 1336 | bool has_sre; |
| @@ -1866,10 +1949,9 @@ static const struct arm64_cpu_capabilities arm64_features[] = { · 612 unmodified lines | ||
| 1866 | 1949 | .cpu_enable = cpu_copy_el2regs, |
| 1867 | 1950 | }, |
| 1868 | 1951 | { |
| 1869 | − .desc = "32-bit EL0 Support", | |
| 1870 | − .capability = ARM64_HAS_32BIT_EL0, | |
| 1952 | + .capability = ARM64_HAS_32BIT_EL0_DO_NOT_USE, | |
| 1871 | 1953 | .type = ARM64_CPUCAP_SYSTEM_FEATURE, |
| 1872 | − .matches = has_cpuid_feature, | |
| 1954 | + .matches = has_32bit_el0, | |
| 1873 | 1955 | .sys_reg = SYS_ID_AA64PFR0_EL1, |
| 1874 | 1956 | .sign = FTR_UNSIGNED, |
| 1875 | 1957 | .field_pos = ID_AA64PFR0_EL0_SHIFT, |
| @@ -2378,7 +2460,7 @@ static const struct arm64_cpu_capabilities compat_elf_hwcaps[] = { · 502 unmodified lines | ||
| 2378 | 2460 | {}, |
| 2379 | 2461 | }; |
| 2380 | 2462 | |
| 2381 | −static void __init cap_set_elf_hwcap(const struct arm64_cpu_capabilities *cap) | |
| 2463 | +static void cap_set_elf_hwcap(const struct arm64_cpu_capabilities *cap) | |
| 2382 | 2464 | { |
| 2383 | 2465 | switch (cap->hwcap_type) { |
| 2384 | 2466 | case CAP_HWCAP: |
| @@ -2423,7 +2505,7 @@ static bool cpus_have_elf_hwcap(const struct arm64_cpu_capabilities *cap) · 38 unmodified lines | ||
| 2423 | 2505 | return rc; |
| 2424 | 2506 | } |
| 2425 | 2507 | |
| 2426 | −static void __init setup_elf_hwcaps(const struct arm64_cpu_capabilities *hwcaps) | |
| 2508 | +static void setup_elf_hwcaps(const struct arm64_cpu_capabilities *hwcaps) | |
| 2427 | 2509 | { |
| 2428 | 2510 | /* We support emulation of accesses to CPU ID feature registers */ |
| 2429 | 2511 | cpu_set_named_feature(CPUID); |
| @@ -2598,7 +2680,7 @@ static void check_early_cpu_features(void) · 168 unmodified lines | ||
| 2598 | 2680 | } |
| 2599 | 2681 | |
| 2600 | 2682 | static void |
| 2601 | −verify_local_elf_hwcaps(const struct arm64_cpu_capabilities *caps) | |
| 2683 | +__verify_local_elf_hwcaps(const struct arm64_cpu_capabilities *caps) | |
| 2602 | 2684 | { |
| 2603 | 2685 | |
| 2604 | 2686 | for (; caps->matches; caps++) |
| @@ -2609,6 +2691,14 @@ verify_local_elf_hwcaps(const struct arm64_cpu_capabilities *caps) · 4 unmodified lines | ||
| 2609 | 2691 | } |
| 2610 | 2692 | } |
| 2611 | 2693 | |
| 2694 | +static void verify_local_elf_hwcaps(void) | |
| 2695 | +{ | |
| 2696 | + __verify_local_elf_hwcaps(arm64_elf_hwcaps); | |
| 2697 | + | |
| 2698 | + if (id_aa64pfr0_32bit_el0(read_cpuid(ID_AA64PFR0_EL1))) | |
| 2699 | + __verify_local_elf_hwcaps(compat_elf_hwcaps); | |
| 2700 | +} | |
| 2701 | + | |
| 2612 | 2702 | static void verify_sve_features(void) |
| 2613 | 2703 | { |
| 2614 | 2704 | u64 safe_zcr = read_sanitised_ftr_reg(SYS_ZCR_EL1); |
| @@ -2673,11 +2763,7 @@ static void verify_local_cpu_capabilities(void) · 58 unmodified lines | ||
| 2673 | 2763 | * on all secondary CPUs. |
| 2674 | 2764 | */ |
| 2675 | 2765 | verify_local_cpu_caps(SCOPE_ALL & ~SCOPE_BOOT_CPU); |
| 2676 | − | |
| 2677 | − verify_local_elf_hwcaps(arm64_elf_hwcaps); | |
| 2678 | − | |
| 2679 | − if (system_supports_32bit_el0()) | |
| 2680 | − verify_local_elf_hwcaps(compat_elf_hwcaps); | |
| 2766 | + verify_local_elf_hwcaps(); | |
| 2681 | 2767 | |
| 2682 | 2768 | if (system_supports_sve()) |
| 2683 | 2769 | verify_sve_features(); |
| @@ -2812,6 +2898,34 @@ void __init setup_cpu_features(void) · 128 unmodified lines | ||
| 2812 | 2898 | ARCH_DMA_MINALIGN); |
| 2813 | 2899 | } |
| 2814 | 2900 | |
| 2901 | +static int enable_mismatched_32bit_el0(unsigned int cpu) | |
| 2902 | +{ | |
| 2903 | + struct cpuinfo_arm64 *info = &per_cpu(cpu_data, cpu); | |
| 2904 | + bool cpu_32bit = id_aa64pfr0_32bit_el0(info->reg_id_aa64pfr0); | |
| 2905 | + | |
| 2906 | + if (cpu_32bit) { | |
| 2907 | + cpumask_set_cpu(cpu, cpu_32bit_el0_mask); | |
| 2908 | + static_branch_enable_cpuslocked(&arm64_mismatched_32bit_el0); | |
| 2909 | + setup_elf_hwcaps(compat_elf_hwcaps); | |
| 2910 | + } | |
| 2911 | + | |
| 2912 | + return 0; | |
| 2913 | +} | |
| 2914 | + | |
| 2915 | +static int __init init_32bit_el0_mask(void) | |
| 2916 | +{ | |
| 2917 | + if (!allow_mismatched_32bit_el0) | |
| 2918 | + return 0; | |
| 2919 | + | |
| 2920 | + if (!zalloc_cpumask_var(&cpu_32bit_el0_mask, GFP_KERNEL)) | |
| 2921 | + return -ENOMEM; | |
| 2922 | + | |
| 2923 | + return cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, | |
| 2924 | + "arm64/mismatched_32bit_el0:online", | |
| 2925 | + enable_mismatched_32bit_el0, NULL); | |
| 2926 | +} | |
| 2927 | +subsys_initcall_sync(init_32bit_el0_mask); | |
| 2928 | + | |
| 2815 | 2929 | static void __maybe_unused cpu_enable_cnp(struct arm64_cpu_capabilities const *cap) |
| 2816 | 2930 | { |
| 2817 | 2931 | cpu_replace_ttbr1(lm_alias(swapper_pg_dir)); |
| @@ -2905,8 +3019,8 @@ static int emulate_mrs(struct pt_regs *regs, u32 insn) · 87 unmodified lines | ||
| 2905 | 3019 | } |
| 2906 | 3020 | |
| 2907 | 3021 | static struct undef_hook mrs_hook = { |
| 2908 | − .instr_mask = 0xfff00000, | |
| 2909 | − .instr_val = 0xd5300000, | |
| 3022 | + .instr_mask = 0xffff0000, | |
| 3023 | + .instr_val = 0xd5380000, | |
| 2910 | 3024 | .pstate_mask = PSR_AA32_MODE_MASK, |
| 2911 | 3025 | .pstate_val = PSR_MODE_EL0t, |
| 2912 | 3026 | .fn = emulate_mrs, |
| @@ -246,7 +246,7 @@ static struct kobj_type cpuregs_kobj_type = { · 245 unmodified lines | ||
| 246 | 246 | struct cpuinfo_arm64 *info = kobj_to_cpuinfo(kobj); \ |
| 247 | 247 | \ |
| 248 | 248 | if (info->reg_midr) \ |
| 249 | − return sprintf(buf, "0x%016x\n", info->reg_##_field); \ | |
| 249 | + return sprintf(buf, "0x%016llx\n", info->reg_##_field); \ | |
| 250 | 250 | else \ |
| 251 | 251 | return 0; \ |
| 252 | 252 | } \ |
| @@ -344,6 +344,32 @@ static void cpuinfo_detect_icache_policy(struct cpuinfo_arm64 *info) · 91 unmodified lines | ||
| 344 | 344 | pr_info("Detected %s I-cache on CPU%d\n", icache_policy_str[l1ip], cpu); |
| 345 | 345 | } |
| 346 | 346 | |
| 347 | +static void __cpuinfo_store_cpu_32bit(struct cpuinfo_32bit *info) | |
| 348 | +{ | |
| 349 | + info->reg_id_dfr0 = read_cpuid(ID_DFR0_EL1); | |
| 350 | + info->reg_id_dfr1 = read_cpuid(ID_DFR1_EL1); | |
| 351 | + info->reg_id_isar0 = read_cpuid(ID_ISAR0_EL1); | |
| 352 | + info->reg_id_isar1 = read_cpuid(ID_ISAR1_EL1); | |
| 353 | + info->reg_id_isar2 = read_cpuid(ID_ISAR2_EL1); | |
| 354 | + info->reg_id_isar3 = read_cpuid(ID_ISAR3_EL1); | |
| 355 | + info->reg_id_isar4 = read_cpuid(ID_ISAR4_EL1); | |
| 356 | + info->reg_id_isar5 = read_cpuid(ID_ISAR5_EL1); | |
| 357 | + info->reg_id_isar6 = read_cpuid(ID_ISAR6_EL1); | |
| 358 | + info->reg_id_mmfr0 = read_cpuid(ID_MMFR0_EL1); | |
| 359 | + info->reg_id_mmfr1 = read_cpuid(ID_MMFR1_EL1); | |
| 360 | + info->reg_id_mmfr2 = read_cpuid(ID_MMFR2_EL1); | |
| 361 | + info->reg_id_mmfr3 = read_cpuid(ID_MMFR3_EL1); | |
| 362 | + info->reg_id_mmfr4 = read_cpuid(ID_MMFR4_EL1); | |
| 363 | + info->reg_id_mmfr5 = read_cpuid(ID_MMFR5_EL1); | |
| 364 | + info->reg_id_pfr0 = read_cpuid(ID_PFR0_EL1); | |
| 365 | + info->reg_id_pfr1 = read_cpuid(ID_PFR1_EL1); | |
| 366 | + info->reg_id_pfr2 = read_cpuid(ID_PFR2_EL1); | |
| 367 | + | |
| 368 | + info->reg_mvfr0 = read_cpuid(MVFR0_EL1); | |
| 369 | + info->reg_mvfr1 = read_cpuid(MVFR1_EL1); | |
| 370 | + info->reg_mvfr2 = read_cpuid(MVFR2_EL1); | |
| 371 | +} | |
| 372 | + | |
| 347 | 373 | static void __cpuinfo_store_cpu(struct cpuinfo_arm64 *info) |
| 348 | 374 | { |
| 349 | 375 | info->reg_cntfrq = arch_timer_get_cntfrq(); |
| @@ -371,31 +397,11 @@ static void __cpuinfo_store_cpu(struct cpuinfo_arm64 *info) · 21 unmodified lines | ||
| 371 | 397 | info->reg_id_aa64pfr1 = read_cpuid(ID_AA64PFR1_EL1); |
| 372 | 398 | info->reg_id_aa64zfr0 = read_cpuid(ID_AA64ZFR0_EL1); |
| 373 | 399 | |
| 374 | − /* Update the 32bit ID registers only if AArch32 is implemented */ | |
| 375 | − if (id_aa64pfr0_32bit_el0(info->reg_id_aa64pfr0)) { | |
| 376 | − info->reg_id_dfr0 = read_cpuid(ID_DFR0_EL1); | |
| 377 | − info->reg_id_dfr1 = read_cpuid(ID_DFR1_EL1); | |
| 378 | − info->reg_id_isar0 = read_cpuid(ID_ISAR0_EL1); | |
| 379 | − info->reg_id_isar1 = read_cpuid(ID_ISAR1_EL1); | |
| 380 | − info->reg_id_isar2 = read_cpuid(ID_ISAR2_EL1); | |
| 381 | − info->reg_id_isar3 = read_cpuid(ID_ISAR3_EL1); | |
| 382 | − info->reg_id_isar4 = read_cpuid(ID_ISAR4_EL1); | |
| 383 | − info->reg_id_isar5 = read_cpuid(ID_ISAR5_EL1); | |
| 384 | − info->reg_id_isar6 = read_cpuid(ID_ISAR6_EL1); | |
| 385 | − info->reg_id_mmfr0 = read_cpuid(ID_MMFR0_EL1); | |
| 386 | − info->reg_id_mmfr1 = read_cpuid(ID_MMFR1_EL1); | |
| 387 | − info->reg_id_mmfr2 = read_cpuid(ID_MMFR2_EL1); | |
| 388 | − info->reg_id_mmfr3 = read_cpuid(ID_MMFR3_EL1); | |
| 389 | − info->reg_id_mmfr4 = read_cpuid(ID_MMFR4_EL1); | |
| 390 | − info->reg_id_mmfr5 = read_cpuid(ID_MMFR5_EL1); | |
| 391 | − info->reg_id_pfr0 = read_cpuid(ID_PFR0_EL1); | |
| 392 | − info->reg_id_pfr1 = read_cpuid(ID_PFR1_EL1); | |
| 393 | − info->reg_id_pfr2 = read_cpuid(ID_PFR2_EL1); | |
| 394 | − | |
| 395 | − info->reg_mvfr0 = read_cpuid(MVFR0_EL1); | |
| 396 | − info->reg_mvfr1 = read_cpuid(MVFR1_EL1); | |
| 397 | − info->reg_mvfr2 = read_cpuid(MVFR2_EL1); | |
| 398 | − } | |
| 400 | + if (id_aa64pfr1_mte(info->reg_id_aa64pfr1)) | |
| 401 | + info->reg_gmid = read_cpuid(GMID_EL1); | |
| 402 | + | |
| 403 | + if (id_aa64pfr0_32bit_el0(info->reg_id_aa64pfr0)) | |
| 404 | + __cpuinfo_store_cpu_32bit(&info->aarch32); | |
| 399 | 405 | |
| 400 | 406 | if (IS_ENABLED(CONFIG_ARM64_SVE) && |
| 401 | 407 | id_aa64pfr0_sve(info->reg_id_aa64pfr0)) |
| @@ -28,7 +28,8 @@ SYM_CODE_START(efi_enter_kernel) · 27 unmodified lines | ||
| 28 | 28 | * stale icache entries from before relocation. |
| 29 | 29 | */ |
| 30 | 30 | ldr w1, =kernel_size |
| 31 | − bl __clean_dcache_area_poc | |
| 31 | + add x1, x0, x1 | |
| 32 | + bl dcache_clean_poc | |
| 32 | 33 | ic ialluis |
| 33 | 34 | |
| 34 | 35 | /* |
| @@ -36,8 +37,8 @@ SYM_CODE_START(efi_enter_kernel) · 1 unmodified line | ||
| 36 | 37 | * so that we can safely disable the MMU and caches. |
| 37 | 38 | */ |
| 38 | 39 | adr x0, 0f |
| 39 | − ldr w1, 3f | |
| 40 | − bl __clean_dcache_area_poc | |
| 40 | + adr x1, 3f | |
| 41 | + bl dcache_clean_poc | |
| 41 | 42 | 0: |
| 42 | 43 | /* Turn off Dcache and MMU */ |
| 43 | 44 | mrs x0, CurrentEL |
| @@ -64,5 +65,5 @@ SYM_CODE_START(efi_enter_kernel) · 20 unmodified lines | ||
| 64 | 65 | mov x2, xzr |
| 65 | 66 | mov x3, xzr |
| 66 | 67 | br x19 |
| 68 | +3: | |
| 67 | 69 | SYM_CODE_END(efi_enter_kernel) |
| 68 | −3: .long . - 0b | |
| @@ -6,7 +6,11 @@ · 5 unmodified lines | ||
| 6 | 6 | */ |
| 7 | 7 | |
| 8 | 8 | #include <linux/context_tracking.h> |
| 9 | +#include <linux/linkage.h> | |
| 10 | +#include <linux/lockdep.h> | |
| 9 | 11 | #include <linux/ptrace.h> |
| 12 | +#include <linux/sched.h> | |
| 13 | +#include <linux/sched/debug.h> | |
| 10 | 14 | #include <linux/thread_info.h> |
| 11 | 15 | |
| 12 | 16 | #include <asm/cpufeature.h> |
| @@ -15,7 +19,11 @@ · 2 unmodified lines | ||
| 15 | 19 | #include <asm/exception.h> |
| 16 | 20 | #include <asm/kprobes.h> |
| 17 | 21 | #include <asm/mmu.h> |
| 22 | +#include <asm/processor.h> | |
| 23 | +#include <asm/sdei.h> | |
| 24 | +#include <asm/stacktrace.h> | |
| 18 | 25 | #include <asm/sysreg.h> |
| 26 | +#include <asm/system_misc.h> | |
| 19 | 27 | |
| 20 | 28 | /* |
| 21 | 29 | * This is intended to match the logic in irqentry_enter(), handling the kernel |
| @@ -67,7 +75,7 @@ static void noinstr exit_to_kernel_mode(struct pt_regs *regs) · 45 unmodified lines | ||
| 67 | 75 | } |
| 68 | 76 | } |
| 69 | 77 | |
| 70 | −void noinstr arm64_enter_nmi(struct pt_regs *regs) | |
| 78 | +static void noinstr arm64_enter_nmi(struct pt_regs *regs) | |
| 71 | 79 | { |
| 72 | 80 | regs->lockdep_hardirqs = lockdep_hardirqs_enabled(); |
| 73 | 81 | |
| @@ -80,7 +88,7 @@ void noinstr arm64_enter_nmi(struct pt_regs *regs) · 6 unmodified lines | ||
| 80 | 88 | ftrace_nmi_enter(); |
| 81 | 89 | } |
| 82 | 90 | |
| 83 | −void noinstr arm64_exit_nmi(struct pt_regs *regs) | |
| 91 | +static void noinstr arm64_exit_nmi(struct pt_regs *regs) | |
| 84 | 92 | { |
| 85 | 93 | bool restore = regs->lockdep_hardirqs; |
| 86 | 94 | |
| @@ -97,7 +105,7 @@ void noinstr arm64_exit_nmi(struct pt_regs *regs) · 10 unmodified lines | ||
| 97 | 105 | __nmi_exit(); |
| 98 | 106 | } |
| 99 | 107 | |
| 100 | −asmlinkage void noinstr enter_el1_irq_or_nmi(struct pt_regs *regs) | |
| 108 | +static void noinstr enter_el1_irq_or_nmi(struct pt_regs *regs) | |
| 101 | 109 | { |
| 102 | 110 | if (IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) && !interrupts_enabled(regs)) |
| 103 | 111 | arm64_enter_nmi(regs); |
| @@ -105,7 +113,7 @@ asmlinkage void noinstr enter_el1_irq_or_nmi(struct pt_regs *regs) · 1 unmodified line | ||
| 105 | 113 | enter_from_kernel_mode(regs); |
| 106 | 114 | } |
| 107 | 115 | |
| 108 | −asmlinkage void noinstr exit_el1_irq_or_nmi(struct pt_regs *regs) | |
| 116 | +static void noinstr exit_el1_irq_or_nmi(struct pt_regs *regs) | |
| 109 | 117 | { |
| 110 | 118 | if (IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) && !interrupts_enabled(regs)) |
| 111 | 119 | arm64_exit_nmi(regs); |
| @@ -113,6 +121,65 @@ asmlinkage void noinstr exit_el1_irq_or_nmi(struct pt_regs *regs) · 1 unmodified line | ||
| 113 | 121 | exit_to_kernel_mode(regs); |
| 114 | 122 | } |
| 115 | 123 | |
| 124 | +static void __sched arm64_preempt_schedule_irq(void) | |
| 125 | +{ | |
| 126 | + lockdep_assert_irqs_disabled(); | |
| 127 | + | |
| 128 | + /* | |
| 129 | + * DAIF.DA are cleared at the start of IRQ/FIQ handling, and when GIC | |
| 130 | + * priority masking is used the GIC irqchip driver will clear DAIF.IF | |
| 131 | + * using gic_arch_enable_irqs() for normal IRQs. If anything is set in | |
| 132 | + * DAIF we must have handled an NMI, so skip preemption. | |
| 133 | + */ | |
| 134 | + if (system_uses_irq_prio_masking() && read_sysreg(daif)) | |
| 135 | + return; | |
| 136 | + | |
| 137 | + /* | |
| 138 | + * Preempting a task from an IRQ means we leave copies of PSTATE | |
| 139 | + * on the stack. cpufeature's enable calls may modify PSTATE, but | |
| 140 | + * resuming one of these preempted tasks would undo those changes. | |
| 141 | + * | |
| 142 | + * Only allow a task to be preempted once cpufeatures have been | |
| 143 | + * enabled. | |
| 144 | + */ | |
| 145 | + if (system_capabilities_finalized()) | |
| 146 | + preempt_schedule_irq(); | |
| 147 | +} | |
| 148 | + | |
| 149 | +static void do_interrupt_handler(struct pt_regs *regs, | |
| 150 | + void (*handler)(struct pt_regs *)) | |
| 151 | +{ | |
| 152 | + if (on_thread_stack()) | |
| 153 | + call_on_irq_stack(regs, handler); | |
| 154 | + else | |
| 155 | + handler(regs); | |
| 156 | +} | |
| 157 | + | |
| 158 | +extern void (*handle_arch_irq)(struct pt_regs *); | |
| 159 | +extern void (*handle_arch_fiq)(struct pt_regs *); | |
| 160 | + | |
| 161 | +static void noinstr __panic_unhandled(struct pt_regs *regs, const char *vector, | |
| 162 | + unsigned int esr) | |
| 163 | +{ | |
| 164 | + arm64_enter_nmi(regs); | |
| 165 | + | |
| 166 | + console_verbose(); | |
| 167 | + | |
| 168 | + pr_crit("Unhandled %s exception on CPU%d, ESR 0x%08x -- %s\n", | |
| 169 | + vector, smp_processor_id(), esr, | |
| 170 | + esr_get_class_string(esr)); | |
| 171 | + | |
| 172 | + __show_regs(regs); | |
| 173 | + panic("Unhandled exception"); | |
| 174 | +} | |
| 175 | + | |
| 176 | +#define UNHANDLED(el, regsize, vector) \ | |
| 177 | +asmlinkage void noinstr el##_##regsize##_##vector##_handler(struct pt_regs *regs) \ | |
| 178 | +{ \ | |
| 179 | + const char *desc = #regsize "-bit " #el " " #vector; \ | |
| 180 | + __panic_unhandled(regs, desc, read_sysreg(esr_el1)); \ | |
| 181 | +} | |
| 182 | + | |
| 116 | 183 | #ifdef CONFIG_ARM64_ERRATUM_1463225 |
| 117 | 184 | static DEFINE_PER_CPU(int, __in_cortex_a76_erratum_1463225_wa); |
| 118 | 185 | |
| @@ -162,6 +229,11 @@ static bool cortex_a76_erratum_1463225_debug_handler(struct pt_regs *regs) · 43 unmodified lines | ||
| 162 | 229 | } |
| 163 | 230 | #endif /* CONFIG_ARM64_ERRATUM_1463225 */ |
| 164 | 231 | |
| 232 | +UNHANDLED(el1t, 64, sync) | |
| 233 | +UNHANDLED(el1t, 64, irq) | |
| 234 | +UNHANDLED(el1t, 64, fiq) | |
| 235 | +UNHANDLED(el1t, 64, error) | |
| 236 | + | |
| 165 | 237 | static void noinstr el1_abort(struct pt_regs *regs, unsigned long esr) |
| 166 | 238 | { |
| 167 | 239 | unsigned long far = read_sysreg(far_el1); |
| @@ -193,15 +265,6 @@ static void noinstr el1_undef(struct pt_regs *regs) · 25 unmodified lines | ||
| 193 | 265 | exit_to_kernel_mode(regs); |
| 194 | 266 | } |
| 195 | 267 | |
| 196 | −static void noinstr el1_inv(struct pt_regs *regs, unsigned long esr) | |
| 197 | −{ | |
| 198 | − enter_from_kernel_mode(regs); | |
| 199 | − local_daif_inherit(regs); | |
| 200 | − bad_mode(regs, 0, esr); | |
| 201 | − local_daif_mask(); | |
| 202 | − exit_to_kernel_mode(regs); | |
| 203 | −} | |
| 204 | − | |
| 205 | 268 | static void noinstr arm64_enter_el1_dbg(struct pt_regs *regs) |
| 206 | 269 | { |
| 207 | 270 | regs->lockdep_hardirqs = lockdep_hardirqs_enabled(); |
| @@ -245,7 +308,7 @@ static void noinstr el1_fpac(struct pt_regs *regs, unsigned long esr) · 37 unmodified lines | ||
| 245 | 308 | exit_to_kernel_mode(regs); |
| 246 | 309 | } |
| 247 | 310 | |
| 248 | −asmlinkage void noinstr el1_sync_handler(struct pt_regs *regs) | |
| 311 | +asmlinkage void noinstr el1h_64_sync_handler(struct pt_regs *regs) | |
| 249 | 312 | { |
| 250 | 313 | unsigned long esr = read_sysreg(esr_el1); |
| 251 | 314 | |
| @@ -275,10 +338,50 @@ asmlinkage void noinstr el1_sync_handler(struct pt_regs *regs) · 23 unmodified lines | ||
| 275 | 338 | el1_fpac(regs, esr); |
| 276 | 339 | break; |
| 277 | 340 | default: |
| 278 | − el1_inv(regs, esr); | |
| 341 | + __panic_unhandled(regs, "64-bit el1h sync", esr); | |
| 279 | 342 | } |
| 280 | 343 | } |
| 281 | 344 | |
| 345 | +static void noinstr el1_interrupt(struct pt_regs *regs, | |
| 346 | + void (*handler)(struct pt_regs *)) | |
| 347 | +{ | |
| 348 | + write_sysreg(DAIF_PROCCTX_NOIRQ, daif); | |
| 349 | + | |
| 350 | + enter_el1_irq_or_nmi(regs); | |
| 351 | + do_interrupt_handler(regs, handler); | |
| 352 | + | |
| 353 | + /* | |
| 354 | + * Note: thread_info::preempt_count includes both thread_info::count | |
| 355 | + * and thread_info::need_resched, and is not equivalent to | |
| 356 | + * preempt_count(). | |
| 357 | + */ | |
| 358 | + if (IS_ENABLED(CONFIG_PREEMPTION) && | |
| 359 | + READ_ONCE(current_thread_info()->preempt_count) == 0) | |
| 360 | + arm64_preempt_schedule_irq(); | |
| 361 | + | |
| 362 | + exit_el1_irq_or_nmi(regs); | |
| 363 | +} | |
| 364 | + | |
| 365 | +asmlinkage void noinstr el1h_64_irq_handler(struct pt_regs *regs) | |
| 366 | +{ | |
| 367 | + el1_interrupt(regs, handle_arch_irq); | |
| 368 | +} | |
| 369 | + | |
| 370 | +asmlinkage void noinstr el1h_64_fiq_handler(struct pt_regs *regs) | |
| 371 | +{ | |
| 372 | + el1_interrupt(regs, handle_arch_fiq); | |
| 373 | +} | |
| 374 | + | |
| 375 | +asmlinkage void noinstr el1h_64_error_handler(struct pt_regs *regs) | |
| 376 | +{ | |
| 377 | + unsigned long esr = read_sysreg(esr_el1); | |
| 378 | + | |
| 379 | + local_daif_restore(DAIF_ERRCTX); | |
| 380 | + arm64_enter_nmi(regs); | |
| 381 | + do_serror(regs, esr); | |
| 382 | + arm64_exit_nmi(regs); | |
| 383 | +} | |
| 384 | + | |
| 282 | 385 | asmlinkage void noinstr enter_from_user_mode(void) |
| 283 | 386 | { |
| 284 | 387 | lockdep_hardirqs_off(CALLER_ADDR0); |
| @@ -398,7 +501,7 @@ static void noinstr el0_dbg(struct pt_regs *regs, unsigned long esr) · 113 unmodified lines | ||
| 398 | 501 | |
| 399 | 502 | enter_from_user_mode(); |
| 400 | 503 | do_debug_exception(far, esr, regs); |
| 401 | − local_daif_restore(DAIF_PROCCTX_NOIRQ); | |
| 504 | + local_daif_restore(DAIF_PROCCTX); | |
| 402 | 505 | } |
| 403 | 506 | |
| 404 | 507 | static void noinstr el0_svc(struct pt_regs *regs) |
| @@ -415,7 +518,7 @@ static void noinstr el0_fpac(struct pt_regs *regs, unsigned long esr) · 10 unmodified lines | ||
| 415 | 518 | do_ptrauth_fault(regs, esr); |
| 416 | 519 | } |
| 417 | 520 | |
| 418 | −asmlinkage void noinstr el0_sync_handler(struct pt_regs *regs) | |
| 521 | +asmlinkage void noinstr el0t_64_sync_handler(struct pt_regs *regs) | |
| 419 | 522 | { |
| 420 | 523 | unsigned long esr = read_sysreg(esr_el1); |
| 421 | 524 | |
| @@ -468,6 +571,56 @@ asmlinkage void noinstr el0_sync_handler(struct pt_regs *regs) · 46 unmodified lines | ||
| 468 | 571 | } |
| 469 | 572 | } |
| 470 | 573 | |
| 574 | +static void noinstr el0_interrupt(struct pt_regs *regs, | |
| 575 | + void (*handler)(struct pt_regs *)) | |
| 576 | +{ | |
| 577 | + enter_from_user_mode(); | |
| 578 | + | |
| 579 | + write_sysreg(DAIF_PROCCTX_NOIRQ, daif); | |
| 580 | + | |
| 581 | + if (regs->pc & BIT(55)) | |
| 582 | + arm64_apply_bp_hardening(); | |
| 583 | + | |
| 584 | + do_interrupt_handler(regs, handler); | |
| 585 | +} | |
| 586 | + | |
| 587 | +static void noinstr __el0_irq_handler_common(struct pt_regs *regs) | |
| 588 | +{ | |
| 589 | + el0_interrupt(regs, handle_arch_irq); | |
| 590 | +} | |
| 591 | + | |
| 592 | +asmlinkage void noinstr el0t_64_irq_handler(struct pt_regs *regs) | |
| 593 | +{ | |
| 594 | + __el0_irq_handler_common(regs); | |
| 595 | +} | |
| 596 | + | |
| 597 | +static void noinstr __el0_fiq_handler_common(struct pt_regs *regs) | |
| 598 | +{ | |
| 599 | + el0_interrupt(regs, handle_arch_fiq); | |
| 600 | +} | |
| 601 | + | |
| 602 | +asmlinkage void noinstr el0t_64_fiq_handler(struct pt_regs *regs) | |
| 603 | +{ | |
| 604 | + __el0_fiq_handler_common(regs); | |
| 605 | +} | |
| 606 | + | |
| 607 | +static void __el0_error_handler_common(struct pt_regs *regs) | |
| 608 | +{ | |
| 609 | + unsigned long esr = read_sysreg(esr_el1); | |
| 610 | + | |
| 611 | + enter_from_user_mode(); | |
| 612 | + local_daif_restore(DAIF_ERRCTX); | |
| 613 | + arm64_enter_nmi(regs); | |
| 614 | + do_serror(regs, esr); | |
| 615 | + arm64_exit_nmi(regs); | |
| 616 | + local_daif_restore(DAIF_PROCCTX); | |
| 617 | +} | |
| 618 | + | |
| 619 | +asmlinkage void noinstr el0t_64_error_handler(struct pt_regs *regs) | |
| 620 | +{ | |
| 621 | + __el0_error_handler_common(regs); | |
| 622 | +} | |
| 623 | + | |
| 471 | 624 | #ifdef CONFIG_COMPAT |
| 472 | 625 | static void noinstr el0_cp15(struct pt_regs *regs, unsigned long esr) |
| 473 | 626 | { |
| @@ -483,7 +636,7 @@ static void noinstr el0_svc_compat(struct pt_regs *regs) · 9 unmodified lines | ||
| 483 | 636 | do_el0_svc_compat(regs); |
| 484 | 637 | } |
| 485 | 638 | |
| 486 | −asmlinkage void noinstr el0_sync_compat_handler(struct pt_regs *regs) | |
| 639 | +asmlinkage void noinstr el0t_32_sync_handler(struct pt_regs *regs) | |
| 487 | 640 | { |
| 488 | 641 | unsigned long esr = read_sysreg(esr_el1); |
| 489 | 642 | |
| @@ -526,4 +679,71 @@ asmlinkage void noinstr el0_sync_compat_handler(struct pt_regs *regs) · 36 unmodified lines | ||
| 526 | 679 | el0_inv(regs, esr); |
| 527 | 680 | } |
| 528 | 681 | } |
| 682 | + | |
| 683 | +asmlinkage void noinstr el0t_32_irq_handler(struct pt_regs *regs) | |
| 684 | +{ | |
| 685 | + __el0_irq_handler_common(regs); | |
| 686 | +} | |
| 687 | + | |
| 688 | +asmlinkage void noinstr el0t_32_fiq_handler(struct pt_regs *regs) | |
| 689 | +{ | |
| 690 | + __el0_fiq_handler_common(regs); | |
| 691 | +} | |
| 692 | + | |
| 693 | +asmlinkage void noinstr el0t_32_error_handler(struct pt_regs *regs) | |
| 694 | +{ | |
| 695 | + __el0_error_handler_common(regs); | |
| 696 | +} | |
| 697 | +#else /* CONFIG_COMPAT */ | |
| 698 | +UNHANDLED(el0t, 32, sync) | |
| 699 | +UNHANDLED(el0t, 32, irq) | |
| 700 | +UNHANDLED(el0t, 32, fiq) | |
| 701 | +UNHANDLED(el0t, 32, error) | |
| 529 | 702 | #endif /* CONFIG_COMPAT */ |
| 703 | + | |
| 704 | +#ifdef CONFIG_VMAP_STACK | |
| 705 | +asmlinkage void noinstr handle_bad_stack(struct pt_regs *regs) | |
| 706 | +{ | |
| 707 | + unsigned int esr = read_sysreg(esr_el1); | |
| 708 | + unsigned long far = read_sysreg(far_el1); | |
| 709 | + | |
| 710 | + arm64_enter_nmi(regs); | |
| 711 | + panic_bad_stack(regs, esr, far); | |
| 712 | +} | |
| 713 | +#endif /* CONFIG_VMAP_STACK */ | |
| 714 | + | |
| 715 | +#ifdef CONFIG_ARM_SDE_INTERFACE | |
| 716 | +asmlinkage noinstr unsigned long | |
| 717 | +__sdei_handler(struct pt_regs *regs, struct sdei_registered_event *arg) | |
| 718 | +{ | |
| 719 | + unsigned long ret; | |
| 720 | + | |
| 721 | + /* | |
| 722 | + * We didn't take an exception to get here, so the HW hasn't | |
| 723 | + * set/cleared bits in PSTATE that we may rely on. | |
| 724 | + * | |
| 725 | + * The original SDEI spec (ARM DEN 0054A) can be read ambiguously as to | |
| 726 | + * whether PSTATE bits are inherited unchanged or generated from | |
| 727 | + * scratch, and the TF-A implementation always clears PAN and always | |
| 728 | + * clears UAO. There are no other known implementations. | |
| 729 | + * | |
| 730 | + * Subsequent revisions (ARM DEN 0054B) follow the usual rules for how | |
| 731 | + * PSTATE is modified upon architectural exceptions, and so PAN is | |
| 732 | + * either inherited or set per SCTLR_ELx.SPAN, and UAO is always | |
| 733 | + * cleared. | |
| 734 | + * | |
| 735 | + * We must explicitly reset PAN to the expected state, including | |
| 736 | + * clearing it when the host isn't using it, in case a VM had it set. | |
| 737 | + */ | |
| 738 | + if (system_uses_hw_pan()) | |
| 739 | + set_pstate_pan(1); | |
| 740 | + else if (cpu_has_pan()) | |
| 741 | + set_pstate_pan(0); | |
| 742 | + | |
| 743 | + arm64_enter_nmi(regs); | |
| 744 | + ret = do_sdei_event(regs, arg); | |
| 745 | + arm64_exit_nmi(regs); | |
| 746 | + | |
| 747 | + return ret; | |
| 748 | +} | |
| 749 | +#endif /* CONFIG_ARM_SDE_INTERFACE */ | |
| @@ -63,16 +63,24 @@ SYM_FUNC_END(sve_set_vq) · 62 unmodified lines | ||
| 63 | 63 | * and the rest zeroed. All the other SVE registers will be zeroed. |
| 64 | 64 | */ |
| 65 | 65 | SYM_FUNC_START(sve_load_from_fpsimd_state) |
| 66 | − sve_load_vq x1, x2, x3 | |
| 67 | − fpsimd_restore x0, 8 | |
| 68 | − _for n, 0, 15, _sve_pfalse \n | |
| 69 | − _sve_wrffr 0 | |
| 70 | − ret | |
| 66 | + sve_load_vq x1, x2, x3 | |
| 67 | + fpsimd_restore x0, 8 | |
| 68 | + sve_flush_p_ffr | |
| 69 | + ret | |
| 71 | 70 | SYM_FUNC_END(sve_load_from_fpsimd_state) |
| 72 | 71 | |
| 73 | −/* Zero all SVE registers but the first 128-bits of each vector */ | |
| 72 | +/* | |
| 73 | + * Zero all SVE registers but the first 128-bits of each vector | |
| 74 | + * | |
| 75 | + * VQ must already be configured by caller, any further updates of VQ | |
| 76 | + * will need to ensure that the register state remains valid. | |
| 77 | + * | |
| 78 | + * x0 = VQ - 1 | |
| 79 | + */ | |
| 74 | 80 | SYM_FUNC_START(sve_flush_live) |
| 75 | − sve_flush | |
| 81 | + cbz x0, 1f // A VQ-1 of 0 is 128 bits so no extra Z state | |
| 82 | + sve_flush_z | |
| 83 | +1: sve_flush_p_ffr | |
| 76 | 84 | ret |
| 77 | 85 | SYM_FUNC_END(sve_flush_live) |
| 78 | 86 | |
| @@ -33,12 +33,6 @@ · 32 unmodified lines | ||
| 33 | 33 | * Context tracking and irqflag tracing need to instrument transitions between |
| 34 | 34 | * user and kernel mode. |
| 35 | 35 | */ |
| 36 | − .macro user_exit_irqoff | |
| 37 | −#if defined(CONFIG_CONTEXT_TRACKING) || defined(CONFIG_TRACE_IRQFLAGS) | |
| 38 | − bl enter_from_user_mode | |
| 39 | −#endif | |
| 40 | − .endm | |
| 41 | − | |
| 42 | 36 | .macro user_enter_irqoff |
| 43 | 37 | #if defined(CONFIG_CONTEXT_TRACKING) || defined(CONFIG_TRACE_IRQFLAGS) |
| 44 | 38 | bl exit_to_user_mode |
| @@ -51,16 +45,7 @@ · 6 unmodified lines | ||
| 51 | 45 | .endr |
| 52 | 46 | .endm |
| 53 | 47 | |
| 54 | −/* | |
| 55 | − * Bad Abort numbers | |
| 56 | − *----------------- | |
| 57 | − */ | |
| 58 | −#define BAD_SYNC 0 | |
| 59 | −#define BAD_IRQ 1 | |
| 60 | −#define BAD_FIQ 2 | |
| 61 | −#define BAD_ERROR 3 | |
| 62 | − | |
| 63 | − .macro kernel_ventry, el, label, regsize = 64 | |
| 48 | + .macro kernel_ventry, el:req, ht:req, regsize:req, label:req | |
| 64 | 49 | .align 7 |
| 65 | 50 | #ifdef CONFIG_UNMAP_KERNEL_AT_EL0 |
| 66 | 51 | .if \el == 0 |
| @@ -87,7 +72,7 @@ alternative_else_nop_endif · 20 unmodified lines | ||
| 87 | 72 | tbnz x0, #THREAD_SHIFT, 0f |
| 88 | 73 | sub x0, sp, x0 // x0'' = sp' - x0' = (sp + x0) - sp = x0 |
| 89 | 74 | sub sp, sp, x0 // sp'' = sp' - x0 = (sp + x0) - x0 = sp |
| 90 | − b el\()\el\()_\label | |
| 75 | + b el\el\ht\()_\regsize\()_\label | |
| 91 | 76 | |
| 92 | 77 | 0: |
| 93 | 78 | /* |
| @@ -119,7 +104,7 @@ alternative_else_nop_endif · 25 unmodified lines | ||
| 119 | 104 | sub sp, sp, x0 |
| 120 | 105 | mrs x0, tpidrro_el0 |
| 121 | 106 | #endif |
| 122 | − b el\()\el\()_\label | |
| 107 | + b el\el\ht\()_\regsize\()_\label | |
| 123 | 108 | .endm |
| 124 | 109 | |
| 125 | 110 | .macro tramp_alias, dst, sym |
| @@ -275,7 +260,7 @@ alternative_else_nop_endif · 149 unmodified lines | ||
| 275 | 260 | |
| 276 | 261 | mte_set_kernel_gcr x22, x23 |
| 277 | 262 | |
| 278 | − scs_load tsk, x20 | |
| 263 | + scs_load tsk | |
| 279 | 264 | .else |
| 280 | 265 | add x21, sp, #PT_REGS_SIZE |
| 281 | 266 | get_current_task tsk |
| @@ -285,7 +270,7 @@ alternative_else_nop_endif · 3 unmodified lines | ||
| 285 | 270 | stp lr, x21, [sp, #S_LR] |
| 286 | 271 | |
| 287 | 272 | /* |
| 288 | − * For exceptions from EL0, create a terminal frame record. | |
| 273 | + * For exceptions from EL0, create a final frame record. | |
| 289 | 274 | * For exceptions from EL1, create a synthetic frame record so the |
| 290 | 275 | * interrupted code shows up in the backtrace. |
| 291 | 276 | */ |
| @@ -375,7 +360,7 @@ alternative_if ARM64_WORKAROUND_845719 · 83 unmodified lines | ||
| 375 | 360 | alternative_else_nop_endif |
| 376 | 361 | #endif |
| 377 | 362 | 3: |
| 378 | − scs_save tsk, x0 | |
| 363 | + scs_save tsk | |
| 379 | 364 | |
| 380 | 365 | #ifdef CONFIG_ARM64_PTR_AUTH |
| 381 | 366 | alternative_if ARM64_HAS_ADDRESS_AUTH |
| @@ -486,63 +471,12 @@ SYM_CODE_START_LOCAL(__swpan_exit_el0) · 104 unmodified lines | ||
| 486 | 471 | SYM_CODE_END(__swpan_exit_el0) |
| 487 | 472 | #endif |
| 488 | 473 | |
| 489 | − .macro irq_stack_entry | |
| 490 | − mov x19, sp // preserve the original sp | |
| 491 | −#ifdef CONFIG_SHADOW_CALL_STACK | |
| 492 | − mov x24, scs_sp // preserve the original shadow stack | |
| 493 | −#endif | |
| 494 | − | |
| 495 | − /* | |
| 496 | − * Compare sp with the base of the task stack. | |
| 497 | − * If the top ~(THREAD_SIZE - 1) bits match, we are on a task stack, | |
| 498 | − * and should switch to the irq stack. | |
| 499 | − */ | |
| 500 | − ldr x25, [tsk, TSK_STACK] | |
| 501 | − eor x25, x25, x19 | |
| 502 | − and x25, x25, #~(THREAD_SIZE - 1) | |
| 503 | − cbnz x25, 9998f | |
| 504 | − | |
| 505 | − ldr_this_cpu x25, irq_stack_ptr, x26 | |
| 506 | − mov x26, #IRQ_STACK_SIZE | |
| 507 | − add x26, x25, x26 | |
| 508 | − | |
| 509 | − /* switch to the irq stack */ | |
| 510 | − mov sp, x26 | |
| 511 | − | |
| 512 | −#ifdef CONFIG_SHADOW_CALL_STACK | |
| 513 | − /* also switch to the irq shadow stack */ | |
| 514 | − ldr_this_cpu scs_sp, irq_shadow_call_stack_ptr, x26 | |
| 515 | −#endif | |
| 516 | − | |
| 517 | −9998: | |
| 518 | − .endm | |
| 519 | − | |
| 520 | − /* | |
| 521 | − * The callee-saved regs (x19-x29) should be preserved between | |
| 522 | − * irq_stack_entry and irq_stack_exit, but note that kernel_entry | |
| 523 | − * uses x20-x23 to store data for later use. | |
| 524 | − */ | |
| 525 | − .macro irq_stack_exit | |
| 526 | − mov sp, x19 | |
| 527 | −#ifdef CONFIG_SHADOW_CALL_STACK | |
| 528 | − mov scs_sp, x24 | |
| 529 | −#endif | |
| 530 | − .endm | |
| 531 | − | |
| 532 | 474 | /* GPRs used by entry code */ |
| 533 | 475 | tsk .req x28 // current thread_info |
| 534 | 476 | |
| 535 | 477 | /* |
| 536 | 478 | * Interrupt handling. |
| 537 | 479 | */ |
| 538 | − .macro irq_handler, handler:req | |
| 539 | − ldr_l x1, \handler | |
| 540 | − mov x0, sp | |
| 541 | − irq_stack_entry | |
| 542 | − blr x1 | |
| 543 | − irq_stack_exit | |
| 544 | − .endm | |
| 545 | − | |
| 546 | 480 | .macro gic_prio_kentry_setup, tmp:req |
| 547 | 481 | #ifdef CONFIG_ARM64_PSEUDO_NMI |
| 548 | 482 | alternative_if ARM64_HAS_IRQ_PRIO_MASKING |
| @@ -552,45 +486,6 @@ tsk .req x28 // current thread_info · 3 unmodified lines | ||
| 552 | 486 | #endif |
| 553 | 487 | .endm |
| 554 | 488 | |
| 555 | − .macro el1_interrupt_handler, handler:req | |
| 556 | − enable_da | |
| 557 | − | |
| 558 | − mov x0, sp | |
| 559 | − bl enter_el1_irq_or_nmi | |
| 560 | − | |
| 561 | − irq_handler \handler | |
| 562 | − | |
| 563 | −#ifdef CONFIG_PREEMPTION | |
| 564 | − ldr x24, [tsk, #TSK_TI_PREEMPT] // get preempt count | |
| 565 | −alternative_if ARM64_HAS_IRQ_PRIO_MASKING | |
| 566 | − /* | |
| 567 | − * DA were cleared at start of handling, and IF are cleared by | |
| 568 | − * the GIC irqchip driver using gic_arch_enable_irqs() for | |
| 569 | − * normal IRQs. If anything is set, it means we come back from | |
| 570 | − * an NMI instead of a normal IRQ, so skip preemption | |
| 571 | − */ | |
| 572 | − mrs x0, daif | |
| 573 | − orr x24, x24, x0 | |
| 574 | −alternative_else_nop_endif | |
| 575 | − cbnz x24, 1f // preempt count != 0 || NMI return path | |
| 576 | − bl arm64_preempt_schedule_irq // irq en/disable is done inside | |
| 577 | −1: | |
| 578 | −#endif | |
| 579 | − | |
| 580 | − mov x0, sp | |
| 581 | − bl exit_el1_irq_or_nmi | |
| 582 | − .endm | |
| 583 | − | |
| 584 | − .macro el0_interrupt_handler, handler:req | |
| 585 | − user_exit_irqoff | |
| 586 | − enable_da | |
| 587 | − | |
| 588 | − tbz x22, #55, 1f | |
| 589 | − bl do_el0_irq_bp_hardening | |
| 590 | −1: | |
| 591 | − irq_handler \handler | |
| 592 | − .endm | |
| 593 | − | |
| 594 | 489 | .text |
| 595 | 490 | |
| 596 | 491 | /* |
| @@ -600,32 +495,25 @@ alternative_else_nop_endif · 3 unmodified lines | ||
| 600 | 495 | |
| 601 | 496 | .align 11 |
| 602 | 497 | SYM_CODE_START(vectors) |
| 603 | − kernel_ventry 1, sync_invalid // Synchronous EL1t | |
| 604 | − kernel_ventry 1, irq_invalid // IRQ EL1t | |
| 605 | − kernel_ventry 1, fiq_invalid // FIQ EL1t | |
| 606 | − kernel_ventry 1, error_invalid // Error EL1t | |
| 607 | − | |
| 608 | − kernel_ventry 1, sync // Synchronous EL1h | |
| 609 | − kernel_ventry 1, irq // IRQ EL1h | |
| 610 | − kernel_ventry 1, fiq // FIQ EL1h | |
| 611 | − kernel_ventry 1, error // Error EL1h | |
| 612 | − | |
| 613 | − kernel_ventry 0, sync // Synchronous 64-bit EL0 | |
| 614 | − kernel_ventry 0, irq // IRQ 64-bit EL0 | |
| 615 | − kernel_ventry 0, fiq // FIQ 64-bit EL0 | |
| 616 | − kernel_ventry 0, error // Error 64-bit EL0 | |
| 617 | − | |
| 618 | −#ifdef CONFIG_COMPAT | |
| 619 | − kernel_ventry 0, sync_compat, 32 // Synchronous 32-bit EL0 | |
| 620 | − kernel_ventry 0, irq_compat, 32 // IRQ 32-bit EL0 | |
| 621 | − kernel_ventry 0, fiq_compat, 32 // FIQ 32-bit EL0 | |
| 622 | − kernel_ventry 0, error_compat, 32 // Error 32-bit EL0 | |
| 623 | −#else | |
| 624 | − kernel_ventry 0, sync_invalid, 32 // Synchronous 32-bit EL0 | |
| 625 | − kernel_ventry 0, irq_invalid, 32 // IRQ 32-bit EL0 | |
| 626 | − kernel_ventry 0, fiq_invalid, 32 // FIQ 32-bit EL0 | |
| 627 | − kernel_ventry 0, error_invalid, 32 // Error 32-bit EL0 | |
| 628 | −#endif | |
| 498 | + kernel_ventry 1, t, 64, sync // Synchronous EL1t | |
| 499 | + kernel_ventry 1, t, 64, irq // IRQ EL1t | |
| 500 | + kernel_ventry 1, t, 64, fiq // FIQ EL1h | |
| 501 | + kernel_ventry 1, t, 64, error // Error EL1t | |
| 502 | + | |
| 503 | + kernel_ventry 1, h, 64, sync // Synchronous EL1h | |
| 504 | + kernel_ventry 1, h, 64, irq // IRQ EL1h | |
| 505 | + kernel_ventry 1, h, 64, fiq // FIQ EL1h | |
| 506 | + kernel_ventry 1, h, 64, error // Error EL1h | |
| 507 | + | |
| 508 | + kernel_ventry 0, t, 64, sync // Synchronous 64-bit EL0 | |
| 509 | + kernel_ventry 0, t, 64, irq // IRQ 64-bit EL0 | |
| 510 | + kernel_ventry 0, t, 64, fiq // FIQ 64-bit EL0 | |
| 511 | + kernel_ventry 0, t, 64, error // Error 64-bit EL0 | |
| 512 | + | |
| 513 | + kernel_ventry 0, t, 32, sync // Synchronous 32-bit EL0 | |
| 514 | + kernel_ventry 0, t, 32, irq // IRQ 32-bit EL0 | |
| 515 | + kernel_ventry 0, t, 32, fiq // FIQ 32-bit EL0 | |
| 516 | + kernel_ventry 0, t, 32, error // Error 32-bit EL0 | |
| 629 | 517 | SYM_CODE_END(vectors) |
| 630 | 518 | |
| 631 | 519 | #ifdef CONFIG_VMAP_STACK |
| @@ -656,147 +544,46 @@ __bad_stack: · 24 unmodified lines | ||
| 656 | 544 | ASM_BUG() |
| 657 | 545 | #endif /* CONFIG_VMAP_STACK */ |
| 658 | 546 | |
| 659 | −/* | |
| 660 | − * Invalid mode handlers | |
| 661 | − */ | |
| 662 | − .macro inv_entry, el, reason, regsize = 64 | |
| 547 | + | |
| 548 | + .macro entry_handler el:req, ht:req, regsize:req, label:req | |
| 549 | +SYM_CODE_START_LOCAL(el\el\ht\()_\regsize\()_\label) | |
| 663 | 550 | kernel_entry \el, \regsize |
| 664 | 551 | mov x0, sp |
| 665 | − mov x1, #\reason | |
| 666 | − mrs x2, esr_el1 | |
| 667 | − bl bad_mode | |
| 668 | − ASM_BUG() | |
| 552 | + bl el\el\ht\()_\regsize\()_\label\()_handler | |
| 553 | + .if \el == 0 | |
| 554 | + b ret_to_user | |
| 555 | + .else | |
| 556 | + b ret_to_kernel | |
| 557 | + .endif | |
| 558 | +SYM_CODE_END(el\el\ht\()_\regsize\()_\label) | |
| 669 | 559 | .endm |
| 670 | 560 | |
| 671 | −SYM_CODE_START_LOCAL(el0_sync_invalid) | |
| 672 | − inv_entry 0, BAD_SYNC | |
| 673 | −SYM_CODE_END(el0_sync_invalid) | |
| 674 | − | |
| 675 | −SYM_CODE_START_LOCAL(el0_irq_invalid) | |
| 676 | − inv_entry 0, BAD_IRQ | |
| 677 | −SYM_CODE_END(el0_irq_invalid) | |
| 678 | − | |
| 679 | −SYM_CODE_START_LOCAL(el0_fiq_invalid) | |
| 680 | − inv_entry 0, BAD_FIQ | |
| 681 | −SYM_CODE_END(el0_fiq_invalid) | |
| 682 | − | |
| 683 | −SYM_CODE_START_LOCAL(el0_error_invalid) | |
| 684 | − inv_entry 0, BAD_ERROR | |
| 685 | −SYM_CODE_END(el0_error_invalid) | |
| 686 | − | |
| 687 | −SYM_CODE_START_LOCAL(el1_sync_invalid) | |
| 688 | − inv_entry 1, BAD_SYNC | |
| 689 | −SYM_CODE_END(el1_sync_invalid) | |
| 690 | − | |
| 691 | −SYM_CODE_START_LOCAL(el1_irq_invalid) | |
| 692 | − inv_entry 1, BAD_IRQ | |
| 693 | −SYM_CODE_END(el1_irq_invalid) | |
| 694 | − | |
| 695 | −SYM_CODE_START_LOCAL(el1_fiq_invalid) | |
| 696 | − inv_entry 1, BAD_FIQ | |
| 697 | −SYM_CODE_END(el1_fiq_invalid) | |
| 698 | − | |
| 699 | −SYM_CODE_START_LOCAL(el1_error_invalid) | |
| 700 | − inv_entry 1, BAD_ERROR | |
| 701 | −SYM_CODE_END(el1_error_invalid) | |
| 702 | − | |
| 703 | 561 | /* |
| 704 | − * EL1 mode handlers. | |
| 562 | + * Early exception handlers | |
| 705 | 563 | */ |
| 706 | − .align 6 | |
| 707 | −SYM_CODE_START_LOCAL_NOALIGN(el1_sync) | |
| 708 | − kernel_entry 1 | |
| 709 | − mov x0, sp | |
| 710 | − bl el1_sync_handler | |
| 711 | − kernel_exit 1 | |
| 712 | −SYM_CODE_END(el1_sync) | |
| 713 | − | |
| 714 | − .align 6 | |
| 715 | −SYM_CODE_START_LOCAL_NOALIGN(el1_irq) | |
| 716 | − kernel_entry 1 | |
| 717 | − el1_interrupt_handler handle_arch_irq | |
| 718 | − kernel_exit 1 | |
| 719 | −SYM_CODE_END(el1_irq) | |
| 720 | − | |
| 721 | −SYM_CODE_START_LOCAL_NOALIGN(el1_fiq) | |
| 722 | − kernel_entry 1 | |
| 723 | − el1_interrupt_handler handle_arch_fiq | |
| 724 | − kernel_exit 1 | |
| 725 | −SYM_CODE_END(el1_fiq) | |
| 726 | − | |
| 727 | −/* | |
| 728 | − * EL0 mode handlers. | |
| 729 | − */ | |
| 730 | − .align 6 | |
| 731 | −SYM_CODE_START_LOCAL_NOALIGN(el0_sync) | |
| 732 | − kernel_entry 0 | |
| 733 | − mov x0, sp | |
| 734 | − bl el0_sync_handler | |
| 735 | − b ret_to_user | |
| 736 | −SYM_CODE_END(el0_sync) | |
| 737 | − | |
| 738 | −#ifdef CONFIG_COMPAT | |
| 739 | − .align 6 | |
| 740 | −SYM_CODE_START_LOCAL_NOALIGN(el0_sync_compat) | |
| 741 | − kernel_entry 0, 32 | |
| 742 | − mov x0, sp | |
| 743 | − bl el0_sync_compat_handler | |
| 744 | − b ret_to_user | |
| 745 | −SYM_CODE_END(el0_sync_compat) | |
| 746 | − | |
| 747 | − .align 6 | |
| 748 | −SYM_CODE_START_LOCAL_NOALIGN(el0_irq_compat) | |
| 749 | − kernel_entry 0, 32 | |
| 750 | − b el0_irq_naked | |
| 751 | −SYM_CODE_END(el0_irq_compat) | |
| 752 | − | |
| 753 | −SYM_CODE_START_LOCAL_NOALIGN(el0_fiq_compat) | |
| 754 | − kernel_entry 0, 32 | |
| 755 | − b el0_fiq_naked | |
| 756 | −SYM_CODE_END(el0_fiq_compat) | |
| 757 | − | |
| 758 | −SYM_CODE_START_LOCAL_NOALIGN(el0_error_compat) | |
| 759 | − kernel_entry 0, 32 | |
| 760 | − b el0_error_naked | |
| 761 | −SYM_CODE_END(el0_error_compat) | |
| 762 | −#endif | |
| 763 | − | |
| 764 | − .align 6 | |
| 765 | −SYM_CODE_START_LOCAL_NOALIGN(el0_irq) | |
| 766 | − kernel_entry 0 | |
| 767 | −el0_irq_naked: | |
| 768 | − el0_interrupt_handler handle_arch_irq | |
| 769 | − b ret_to_user | |
| 770 | −SYM_CODE_END(el0_irq) | |
| 771 | − | |
| 772 | −SYM_CODE_START_LOCAL_NOALIGN(el0_fiq) | |
| 773 | − kernel_entry 0 | |
| 774 | −el0_fiq_naked: | |
| 775 | − el0_interrupt_handler handle_arch_fiq | |
| 776 | − b ret_to_user | |
| 777 | −SYM_CODE_END(el0_fiq) | |
| 778 | − | |
| 779 | −SYM_CODE_START_LOCAL(el1_error) | |
| 780 | − kernel_entry 1 | |
| 781 | − mrs x1, esr_el1 | |
| 782 | − enable_dbg | |
| 783 | − mov x0, sp | |
| 784 | − bl do_serror | |
| 564 | + entry_handler 1, t, 64, sync | |
| 565 | + entry_handler 1, t, 64, irq | |
| 566 | + entry_handler 1, t, 64, fiq | |
| 567 | + entry_handler 1, t, 64, error | |
| 568 | + | |
| 569 | + entry_handler 1, h, 64, sync | |
| 570 | + entry_handler 1, h, 64, irq | |
| 571 | + entry_handler 1, h, 64, fiq | |
| 572 | + entry_handler 1, h, 64, error | |
| 573 | + | |
| 574 | + entry_handler 0, t, 64, sync | |
| 575 | + entry_handler 0, t, 64, irq | |
| 576 | + entry_handler 0, t, 64, fiq | |
| 577 | + entry_handler 0, t, 64, error | |
| 578 | + | |
| 579 | + entry_handler 0, t, 32, sync | |
| 580 | + entry_handler 0, t, 32, irq | |
| 581 | + entry_handler 0, t, 32, fiq | |
| 582 | + entry_handler 0, t, 32, error | |
| 583 | + | |
| 584 | +SYM_CODE_START_LOCAL(ret_to_kernel) | |
| 785 | 585 | kernel_exit 1 |
| 786 | −SYM_CODE_END(el1_error) | |
| 787 | − | |
| 788 | −SYM_CODE_START_LOCAL(el0_error) | |
| 789 | − kernel_entry 0 | |
| 790 | −el0_error_naked: | |
| 791 | − mrs x25, esr_el1 | |
| 792 | − user_exit_irqoff | |
| 793 | − enable_dbg | |
| 794 | − mov x0, sp | |
| 795 | − mov x1, x25 | |
| 796 | − bl do_serror | |
| 797 | − enable_da | |
| 798 | − b ret_to_user | |
| 799 | −SYM_CODE_END(el0_error) | |
| 586 | +SYM_CODE_END(ret_to_kernel) | |
| 800 | 587 | |
| 801 | 588 | /* |
| 802 | 589 | * "slow" syscall return path. |
| @@ -979,8 +766,8 @@ SYM_FUNC_START(cpu_switch_to) · 176 unmodified lines | ||
| 979 | 766 | mov sp, x9 |
| 980 | 767 | msr sp_el0, x1 |
| 981 | 768 | ptrauth_keys_install_kernel x1, x8, x9, x10 |
| 982 | − scs_save x0, x8 | |
| 983 | − scs_load x1, x8 | |
| 769 | + scs_save x0 | |
| 770 | + scs_load x1 | |
| 984 | 771 | ret |
| 985 | 772 | SYM_FUNC_END(cpu_switch_to) |
| 986 | 773 | NOKPROBE(cpu_switch_to) |
| @@ -998,6 +785,42 @@ SYM_CODE_START(ret_from_fork) · 11 unmodified lines | ||
| 998 | 785 | SYM_CODE_END(ret_from_fork) |
| 999 | 786 | NOKPROBE(ret_from_fork) |
| 1000 | 787 | |
| 788 | +/* | |
| 789 | + * void call_on_irq_stack(struct pt_regs *regs, | |
| 790 | + * void (*func)(struct pt_regs *)); | |
| 791 | + * | |
| 792 | + * Calls func(regs) using this CPU's irq stack and shadow irq stack. | |
| 793 | + */ | |
| 794 | +SYM_FUNC_START(call_on_irq_stack) | |
| 795 | +#ifdef CONFIG_SHADOW_CALL_STACK | |
| 796 | + stp scs_sp, xzr, [sp, #-16]! | |
| 797 | + ldr_this_cpu scs_sp, irq_shadow_call_stack_ptr, x17 | |
| 798 | +#endif | |
| 799 | + /* Create a frame record to save our LR and SP (implicit in FP) */ | |
| 800 | + stp x29, x30, [sp, #-16]! | |
| 801 | + mov x29, sp | |
| 802 | + | |
| 803 | + ldr_this_cpu x16, irq_stack_ptr, x17 | |
| 804 | + mov x15, #IRQ_STACK_SIZE | |
| 805 | + add x16, x16, x15 | |
| 806 | + | |
| 807 | + /* Move to the new stack and call the function there */ | |
| 808 | + mov sp, x16 | |
| 809 | + blr x1 | |
| 810 | + | |
| 811 | + /* | |
| 812 | + * Restore the SP from the FP, and restore the FP and LR from the frame | |
| 813 | + * record. | |
| 814 | + */ | |
| 815 | + mov sp, x29 | |
| 816 | + ldp x29, x30, [sp], #16 | |
| 817 | +#ifdef CONFIG_SHADOW_CALL_STACK | |
| 818 | + ldp scs_sp, xzr, [sp], #16 | |
| 819 | +#endif | |
| 820 | + ret | |
| 821 | +SYM_FUNC_END(call_on_irq_stack) | |
| 822 | +NOKPROBE(call_on_irq_stack) | |
| 823 | + | |
| 1001 | 824 | #ifdef CONFIG_ARM_SDE_INTERFACE |
| 1002 | 825 | |
| 1003 | 826 | #include <asm/sdei.h> |
| @@ -957,8 +957,10 @@ void do_sve_acc(unsigned int esr, struct pt_regs *regs) · 956 unmodified lines | ||
| 957 | 957 | * disabling the trap, otherwise update our in-memory copy. |
| 958 | 958 | */ |
| 959 | 959 | if (!test_thread_flag(TIF_FOREIGN_FPSTATE)) { |
| 960 | − sve_set_vq(sve_vq_from_vl(current->thread.sve_vl) - 1); | |
| 961 | − sve_flush_live(); | |
| 960 | + unsigned long vq_minus_one = | |
| 961 | + sve_vq_from_vl(current->thread.sve_vl) - 1; | |
| 962 | + sve_set_vq(vq_minus_one); | |
| 963 | + sve_flush_live(vq_minus_one); | |
| 962 | 964 | fpsimd_bind_task_to_cpu(); |
| 963 | 965 | } else { |
| 964 | 966 | fpsimd_to_sve(current); |
| @@ -15,6 +15,7 @@ · 14 unmodified lines | ||
| 15 | 15 | #include <asm/debug-monitors.h> |
| 16 | 16 | #include <asm/ftrace.h> |
| 17 | 17 | #include <asm/insn.h> |
| 18 | +#include <asm/patching.h> | |
| 18 | 19 | |
| 19 | 20 | #ifdef CONFIG_DYNAMIC_FTRACE |
| 20 | 21 | /* |
| @@ -16,6 +16,7 @@ · 15 unmodified lines | ||
| 16 | 16 | #include <asm/asm_pointer_auth.h> |
| 17 | 17 | #include <asm/assembler.h> |
| 18 | 18 | #include <asm/boot.h> |
| 19 | +#include <asm/bug.h> | |
| 19 | 20 | #include <asm/ptrace.h> |
| 20 | 21 | #include <asm/asm-offsets.h> |
| 21 | 22 | #include <asm/cache.h> |
| @@ -117,8 +118,8 @@ SYM_CODE_START_LOCAL(preserve_boot_args) · 95 unmodified lines | ||
| 117 | 118 | dmb sy // needed before dc ivac with |
| 118 | 119 | // MMU off |
| 119 | 120 | |
| 120 | − mov x1, #0x20 // 4 x 8 bytes | |
| 121 | − b __inval_dcache_area // tail call | |
| 121 | + add x1, x0, #0x20 // 4 x 8 bytes | |
| 122 | + b dcache_inval_poc // tail call | |
| 122 | 123 | SYM_CODE_END(preserve_boot_args) |
| 123 | 124 | |
| 124 | 125 | /* |
| @@ -195,7 +196,7 @@ SYM_CODE_END(preserve_boot_args) · 70 unmodified lines | ||
| 195 | 196 | and \iend, \iend, \istart // iend = (vend >> shift) & (ptrs - 1) |
| 196 | 197 | mov \istart, \ptrs |
| 197 | 198 | mul \istart, \istart, \count |
| 198 | − add \iend, \iend, \istart // iend += (count - 1) * ptrs | |
| 199 | + add \iend, \iend, \istart // iend += count * ptrs | |
| 199 | 200 | // our entries span multiple tables |
| 200 | 201 | |
| 201 | 202 | lsr \istart, \vstart, \shift |
| @@ -268,8 +269,7 @@ SYM_FUNC_START_LOCAL(__create_page_tables) · 66 unmodified lines | ||
| 268 | 269 | */ |
| 269 | 270 | adrp x0, init_pg_dir |
| 270 | 271 | adrp x1, init_pg_end |
| 271 | − sub x1, x1, x0 | |
| 272 | − bl __inval_dcache_area | |
| 272 | + bl dcache_inval_poc | |
| 273 | 273 | |
| 274 | 274 | /* |
| 275 | 275 | * Clear the init page tables. |
| @@ -354,7 +354,6 @@ SYM_FUNC_START_LOCAL(__create_page_tables) · 78 unmodified lines | ||
| 354 | 354 | #endif |
| 355 | 355 | 1: |
| 356 | 356 | ldr_l x4, idmap_ptrs_per_pgd |
| 357 | − mov x5, x3 // __pa(__idmap_text_start) | |
| 358 | 357 | adr_l x6, __idmap_text_end // __pa(__idmap_text_end) |
| 359 | 358 | |
| 360 | 359 | map_memory x0, x1, x3, x6, x7, x3, x4, x10, x11, x12, x13, x14 |
| @@ -382,39 +381,57 @@ SYM_FUNC_START_LOCAL(__create_page_tables) · 21 unmodified lines | ||
| 382 | 381 | |
| 383 | 382 | adrp x0, idmap_pg_dir |
| 384 | 383 | adrp x1, idmap_pg_end |
| 385 | − sub x1, x1, x0 | |
| 386 | − bl __inval_dcache_area | |
| 384 | + bl dcache_inval_poc | |
| 387 | 385 | |
| 388 | 386 | adrp x0, init_pg_dir |
| 389 | 387 | adrp x1, init_pg_end |
| 390 | − sub x1, x1, x0 | |
| 391 | − bl __inval_dcache_area | |
| 388 | + bl dcache_inval_poc | |
| 392 | 389 | |
| 393 | 390 | ret x28 |
| 394 | 391 | SYM_FUNC_END(__create_page_tables) |
| 395 | 392 | |
| 393 | + /* | |
| 394 | + * Initialize CPU registers with task-specific and cpu-specific context. | |
| 395 | + * | |
| 396 | + * Create a final frame record at task_pt_regs(current)->stackframe, so | |
| 397 | + * that the unwinder can identify the final frame record of any task by | |
| 398 | + * its location in the task stack. We reserve the entire pt_regs space | |
| 399 | + * for consistency with user tasks and kthreads. | |
| 400 | + */ | |
| 401 | + .macro init_cpu_task tsk, tmp1, tmp2 | |
| 402 | + msr sp_el0, \tsk | |
| 403 | + | |
| 404 | + ldr \tmp1, [\tsk, #TSK_STACK] | |
| 405 | + add sp, \tmp1, #THREAD_SIZE | |
| 406 | + sub sp, sp, #PT_REGS_SIZE | |
| 407 | + | |
| 408 | + stp xzr, xzr, [sp, #S_STACKFRAME] | |
| 409 | + add x29, sp, #S_STACKFRAME | |
| 410 | + | |
| 411 | + scs_load \tsk | |
| 412 | + | |
| 413 | + adr_l \tmp1, __per_cpu_offset | |
| 414 | + ldr w\tmp2, [\tsk, #TSK_CPU] | |
| 415 | + ldr \tmp1, [\tmp1, \tmp2, lsl #3] | |
| 416 | + set_this_cpu_offset \tmp1 | |
| 417 | + .endm | |
| 418 | + | |
| 396 | 419 | /* |
| 397 | 420 | * The following fragment of code is executed with the MMU enabled. |
| 398 | 421 | * |
| 399 | 422 | * x0 = __PHYS_OFFSET |
| 400 | 423 | */ |
| 401 | 424 | SYM_FUNC_START_LOCAL(__primary_switched) |
| 402 | − adrp x4, init_thread_union | |
| 403 | − add sp, x4, #THREAD_SIZE | |
| 404 | − adr_l x5, init_task | |
| 405 | − msr sp_el0, x5 // Save thread_info | |
| 425 | + adr_l x4, init_task | |
| 426 | + init_cpu_task x4, x5, x6 | |
| 406 | 427 | |
| 407 | 428 | adr_l x8, vectors // load VBAR_EL1 with virtual |
| 408 | 429 | msr vbar_el1, x8 // vector table address |
| 409 | 430 | isb |
| 410 | 431 | |
| 411 | − stp xzr, x30, [sp, #-16]! | |
| 432 | + stp x29, x30, [sp, #-16]! | |
| 412 | 433 | mov x29, sp |
| 413 | 434 | |
| 414 | −#ifdef CONFIG_SHADOW_CALL_STACK | |
| 415 | − adr_l scs_sp, init_shadow_call_stack // Set shadow call stack | |
| 416 | −#endif | |
| 417 | − | |
| 418 | 435 | str_l x21, __fdt_pointer, x5 // Save FDT pointer |
| 419 | 436 | |
| 420 | 437 | ldr_l x4, kimage_vaddr // Save the offset between |
| @@ -446,10 +463,9 @@ SYM_FUNC_START_LOCAL(__primary_switched) · 25 unmodified lines | ||
| 446 | 463 | 0: |
| 447 | 464 | #endif |
| 448 | 465 | bl switch_to_vhe // Prefer VHE if possible |
| 449 | − add sp, sp, #16 | |
| 450 | − mov x29, #0 | |
| 451 | − mov x30, #0 | |
| 452 | − b start_kernel | |
| 466 | + ldp x29, x30, [sp], #16 | |
| 467 | + bl start_kernel | |
| 468 | + ASM_BUG() | |
| 453 | 469 | SYM_FUNC_END(__primary_switched) |
| 454 | 470 | |
| 455 | 471 | .pushsection ".rodata", "a" |
| @@ -551,7 +567,7 @@ SYM_FUNC_START_LOCAL(set_cpu_boot_mode_flag) · 95 unmodified lines | ||
| 551 | 567 | cmp w0, #BOOT_CPU_MODE_EL2 |
| 552 | 568 | b.ne 1f |
| 553 | 569 | add x1, x1, #4 |
| 554 | −1: str w0, [x1] // This CPU has booted in EL1 | |
| 570 | +1: str w0, [x1] // Save CPU boot mode | |
| 555 | 571 | dmb sy |
| 556 | 572 | dc ivac, x1 // Invalidate potentially stale cache line |
| 557 | 573 | ret |
| @@ -632,21 +648,17 @@ SYM_FUNC_START_LOCAL(__secondary_switched) · 74 unmodified lines | ||
| 632 | 648 | isb |
| 633 | 649 | |
| 634 | 650 | adr_l x0, secondary_data |
| 635 | − ldr x1, [x0, #CPU_BOOT_STACK] // get secondary_data.stack | |
| 636 | − cbz x1, __secondary_too_slow | |
| 637 | − mov sp, x1 | |
| 638 | 651 | ldr x2, [x0, #CPU_BOOT_TASK] |
| 639 | 652 | cbz x2, __secondary_too_slow |
| 640 | − msr sp_el0, x2 | |
| 641 | − scs_load x2, x3 | |
| 642 | − mov x29, #0 | |
| 643 | − mov x30, #0 | |
| 653 | + | |
| 654 | + init_cpu_task x2, x1, x3 | |
| 644 | 655 | |
| 645 | 656 | #ifdef CONFIG_ARM64_PTR_AUTH |
| 646 | 657 | ptrauth_keys_init_cpu x2, x3, x4, x5 |
| 647 | 658 | #endif |
| 648 | 659 | |
| 649 | − b secondary_start_kernel | |
| 660 | + bl secondary_start_kernel | |
| 661 | + ASM_BUG() | |
| 650 | 662 | SYM_FUNC_END(__secondary_switched) |
| 651 | 663 | |
| 652 | 664 | SYM_FUNC_START_LOCAL(__secondary_too_slow) |
| @@ -45,7 +45,7 @@ · 44 unmodified lines | ||
| 45 | 45 | * Because this code has to be copied to a 'safe' page, it can't call out to |
| 46 | 46 | * other functions by PC-relative address. Also remember that it may be |
| 47 | 47 | * mid-way through over-writing other functions. For this reason it contains |
| 48 | − * code from flush_icache_range() and uses the copy_page() macro. | |
| 48 | + * code from caches_clean_inval_pou() and uses the copy_page() macro. | |
| 49 | 49 | * |
| 50 | 50 | * This 'safe' page is mapped via ttbr0, and executed from there. This function |
| 51 | 51 | * switches to a copy of the linear map in ttbr1, performs the restore, then |
| @@ -87,11 +87,12 @@ SYM_CODE_START(swsusp_arch_suspend_exit) · 35 unmodified lines | ||
| 87 | 87 | copy_page x0, x1, x2, x3, x4, x5, x6, x7, x8, x9 |
| 88 | 88 | |
| 89 | 89 | add x1, x10, #PAGE_SIZE |
| 90 | − /* Clean the copied page to PoU - based on flush_icache_range() */ | |
| 90 | + /* Clean the copied page to PoU - based on caches_clean_inval_pou() */ | |
| 91 | 91 | raw_dcache_line_size x2, x3 |
| 92 | 92 | sub x3, x2, #1 |
| 93 | 93 | bic x4, x10, x3 |
| 94 | −2: dc cvau, x4 /* clean D line / unified line */ | |
| 94 | +2: /* clean D line / unified line */ | |
| 95 | +alternative_insn "dc cvau, x4", "dc civac, x4", ARM64_WORKAROUND_CLEAN_CACHE | |
| 95 | 96 | add x4, x4, x2 |
| 96 | 97 | cmp x4, x1 |
| 97 | 98 | b.lo 2b |
| @@ -210,7 +210,7 @@ static int create_safe_exec_page(void *src_start, size_t length, · 209 unmodified lines | ||
| 210 | 210 | return -ENOMEM; |
| 211 | 211 | |
| 212 | 212 | memcpy(page, src_start, length); |
| 213 | − __flush_icache_range((unsigned long)page, (unsigned long)page + length); | |
| 213 | + caches_clean_inval_pou((unsigned long)page, (unsigned long)page + length); | |
| 214 | 214 | rc = trans_pgd_idmap_page(&trans_info, &trans_ttbr0, &t0sz, page); |
| 215 | 215 | if (rc) |
| 216 | 216 | return rc; |
| @@ -240,8 +240,6 @@ static int create_safe_exec_page(void *src_start, size_t length, · 23 unmodified lines | ||
| 240 | 240 | return 0; |
| 241 | 241 | } |
| 242 | 242 | |
| 243 | −#define dcache_clean_range(start, end) __flush_dcache_area(start, (end - start)) | |
| 244 | − | |
| 245 | 243 | #ifdef CONFIG_ARM64_MTE |
| 246 | 244 | |
| 247 | 245 | static DEFINE_XARRAY(mte_pages); |
| @@ -383,13 +381,18 @@ int swsusp_arch_suspend(void) · 135 unmodified lines | ||
| 383 | 381 | ret = swsusp_save(); |
| 384 | 382 | } else { |
| 385 | 383 | /* Clean kernel core startup/idle code to PoC*/ |
| 386 | − dcache_clean_range(__mmuoff_data_start, __mmuoff_data_end); | |
| 387 | − dcache_clean_range(__idmap_text_start, __idmap_text_end); | |
| 384 | + dcache_clean_inval_poc((unsigned long)__mmuoff_data_start, | |
| 385 | + (unsigned long)__mmuoff_data_end); | |
| 386 | + dcache_clean_inval_poc((unsigned long)__idmap_text_start, | |
| 387 | + (unsigned long)__idmap_text_end); | |
| 388 | 388 | |
| 389 | 389 | /* Clean kvm setup code to PoC? */ |
| 390 | 390 | if (el2_reset_needed()) { |
| 391 | − dcache_clean_range(__hyp_idmap_text_start, __hyp_idmap_text_end); | |
| 392 | − dcache_clean_range(__hyp_text_start, __hyp_text_end); | |
| 391 | + dcache_clean_inval_poc( | |
| 392 | + (unsigned long)__hyp_idmap_text_start, | |
| 393 | + (unsigned long)__hyp_idmap_text_end); | |
| 394 | + dcache_clean_inval_poc((unsigned long)__hyp_text_start, | |
| 395 | + (unsigned long)__hyp_text_end); | |
| 393 | 396 | } |
| 394 | 397 | |
| 395 | 398 | swsusp_mte_restore_tags(); |
| @@ -474,7 +477,8 @@ int swsusp_arch_resume(void) · 78 unmodified lines | ||
| 474 | 477 | * The hibernate exit text contains a set of el2 vectors, that will |
| 475 | 478 | * be executed at el2 with the mmu off in order to reload hyp-stub. |
| 476 | 479 | */ |
| 477 | − __flush_dcache_area(hibernate_exit, exit_size); | |
| 480 | + dcache_clean_inval_poc((unsigned long)hibernate_exit, | |
| 481 | + (unsigned long)hibernate_exit + exit_size); | |
| 478 | 482 | |
| 479 | 483 | /* |
| 480 | 484 | * KASLR will cause the el2 vectors to be in a different location in |
| @@ -0,0 +1,46 @@ | ||
| 1 | +// SPDX-License-Identifier: GPL-2.0-only | |
| 2 | +/* | |
| 3 | + * Low-level idle sequences | |
| 4 | + */ | |
| 5 | + | |
| 6 | +#include <linux/cpu.h> | |
| 7 | +#include <linux/irqflags.h> | |
| 8 | + | |
| 9 | +#include <asm/barrier.h> | |
| 10 | +#include <asm/cpuidle.h> | |
| 11 | +#include <asm/cpufeature.h> | |
| 12 | +#include <asm/sysreg.h> | |
| 13 | + | |
| 14 | +/* | |
| 15 | + * cpu_do_idle() | |
| 16 | + * | |
| 17 | + * Idle the processor (wait for interrupt). | |
| 18 | + * | |
| 19 | + * If the CPU supports priority masking we must do additional work to | |
| 20 | + * ensure that interrupts are not masked at the PMR (because the core will | |
| 21 | + * not wake up if we block the wake up signal in the interrupt controller). | |
| 22 | + */ | |
| 23 | +void noinstr cpu_do_idle(void) | |
| 24 | +{ | |
| 25 | + struct arm_cpuidle_irq_context context; | |
| 26 | + | |
| 27 | + arm_cpuidle_save_irq_context(&context); | |
| 28 | + | |
| 29 | + dsb(sy); | |
| 30 | + wfi(); | |
| 31 | + | |
| 32 | + arm_cpuidle_restore_irq_context(&context); | |
| 33 | +} | |
| 34 | + | |
| 35 | +/* | |
| 36 | + * This is our default idle handler. | |
| 37 | + */ | |
| 38 | +void noinstr arch_cpu_idle(void) | |
| 39 | +{ | |
| 40 | + /* | |
| 41 | + * This should do all the clock switching and wait for interrupt | |
| 42 | + * tricks | |
| 43 | + */ | |
| 44 | + cpu_do_idle(); | |
| 45 | + raw_local_irq_enable(); | |
| 46 | +} | |
| @@ -237,7 +237,8 @@ asmlinkage void __init init_feature_override(void) · 236 unmodified lines | ||
| 237 | 237 | |
| 238 | 238 | for (i = 0; i < ARRAY_SIZE(regs); i++) { |
| 239 | 239 | if (regs[i]->override) |
| 240 | − __flush_dcache_area(regs[i]->override, | |
| 240 | + dcache_clean_inval_poc((unsigned long)regs[i]->override, | |
| 241 | + (unsigned long)regs[i]->override + | |
| 241 | 242 | sizeof(*regs[i]->override)); |
| 242 | 243 | } |
| 243 | 244 | } |
| @@ -35,7 +35,7 @@ __efistub_strnlen = __pi_strnlen; · 34 unmodified lines | ||
| 35 | 35 | __efistub_strcmp = __pi_strcmp; |
| 36 | 36 | __efistub_strncmp = __pi_strncmp; |
| 37 | 37 | __efistub_strrchr = __pi_strrchr; |
| 38 | −__efistub___clean_dcache_area_poc = __pi___clean_dcache_area_poc; | |
| 38 | +__efistub_dcache_clean_poc = __pi_dcache_clean_poc; | |
| 39 | 39 | |
| 40 | 40 | #if defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS) |
| 41 | 41 | __efistub___memcpy = __pi_memcpy; |
| @@ -8,6 +8,7 @@ · 7 unmodified lines | ||
| 8 | 8 | #include <linux/kernel.h> |
| 9 | 9 | #include <linux/jump_label.h> |
| 10 | 10 | #include <asm/insn.h> |
| 11 | +#include <asm/patching.h> | |
| 11 | 12 | |
| 12 | 13 | void arch_jump_label_transform(struct jump_entry *entry, |
| 13 | 14 | enum jump_label_type type) |
| @@ -72,7 +72,9 @@ u64 __init kaslr_early_init(void) · 71 unmodified lines | ||
| 72 | 72 | * we end up running with module randomization disabled. |
| 73 | 73 | */ |
| 74 | 74 | module_alloc_base = (u64)_etext - MODULES_VSIZE; |
| 75 | − __flush_dcache_area(&module_alloc_base, sizeof(module_alloc_base)); | |
| 75 | + dcache_clean_inval_poc((unsigned long)&module_alloc_base, | |
| 76 | + (unsigned long)&module_alloc_base + | |
| 77 | + sizeof(module_alloc_base)); | |
| 76 | 78 | |
| 77 | 79 | /* |
| 78 | 80 | * Try to map the FDT early. If this fails, we simply bail, |
| @@ -170,8 +172,12 @@ u64 __init kaslr_early_init(void) · 91 unmodified lines | ||
| 170 | 172 | module_alloc_base += (module_range * (seed & ((1 << 21) - 1))) >> 21; |
| 171 | 173 | module_alloc_base &= PAGE_MASK; |
| 172 | 174 | |
| 173 | − __flush_dcache_area(&module_alloc_base, sizeof(module_alloc_base)); | |
| 174 | − __flush_dcache_area(&memstart_offset_seed, sizeof(memstart_offset_seed)); | |
| 175 | + dcache_clean_inval_poc((unsigned long)&module_alloc_base, | |
| 176 | + (unsigned long)&module_alloc_base + | |
| 177 | + sizeof(module_alloc_base)); | |
| 178 | + dcache_clean_inval_poc((unsigned long)&memstart_offset_seed, | |
| 179 | + (unsigned long)&memstart_offset_seed + | |
| 180 | + sizeof(memstart_offset_seed)); | |
| 175 | 181 | |
| 176 | 182 | return offset; |
| 177 | 183 | } |
| @@ -17,6 +17,7 @@ · 16 unmodified lines | ||
| 17 | 17 | |
| 18 | 18 | #include <asm/debug-monitors.h> |
| 19 | 19 | #include <asm/insn.h> |
| 20 | +#include <asm/patching.h> | |
| 20 | 21 | #include <asm/traps.h> |
| 21 | 22 | |
| 22 | 23 | struct dbg_reg_def_t dbg_reg_def[DBG_MAX_REG_NUM] = { |
| @@ -68,10 +68,16 @@ int machine_kexec_post_load(struct kimage *kimage) · 67 unmodified lines | ||
| 68 | 68 | kimage->arch.kern_reloc = __pa(reloc_code); |
| 69 | 69 | kexec_image_info(kimage); |
| 70 | 70 | |
| 71 | − /* Flush the reloc_code in preparation for its execution. */ | |
| 72 | − __flush_dcache_area(reloc_code, arm64_relocate_new_kernel_size); | |
| 73 | − flush_icache_range((uintptr_t)reloc_code, (uintptr_t)reloc_code + | |
| 74 | − arm64_relocate_new_kernel_size); | |
| 71 | + /* | |
| 72 | + * For execution with the MMU off, reloc_code needs to be cleaned to the | |
| 73 | + * PoC and invalidated from the I-cache. | |
| 74 | + */ | |
| 75 | + dcache_clean_inval_poc((unsigned long)reloc_code, | |
| 76 | + (unsigned long)reloc_code + | |
| 77 | + arm64_relocate_new_kernel_size); | |
| 78 | + icache_inval_pou((uintptr_t)reloc_code, | |
| 79 | + (uintptr_t)reloc_code + | |
| 80 | + arm64_relocate_new_kernel_size); | |
| 75 | 81 | |
| 76 | 82 | return 0; |
| 77 | 83 | } |
| @@ -102,16 +108,18 @@ static void kexec_list_flush(struct kimage *kimage) · 24 unmodified lines | ||
| 102 | 108 | |
| 103 | 109 | for (entry = &kimage->head; ; entry++) { |
| 104 | 110 | unsigned int flag; |
| 105 | − void *addr; | |
| 111 | + unsigned long addr; | |
| 106 | 112 | |
| 107 | 113 | /* flush the list entries. */ |
| 108 | − __flush_dcache_area(entry, sizeof(kimage_entry_t)); | |
| 114 | + dcache_clean_inval_poc((unsigned long)entry, | |
| 115 | + (unsigned long)entry + | |
| 116 | + sizeof(kimage_entry_t)); | |
| 109 | 117 | |
| 110 | 118 | flag = *entry & IND_FLAGS; |
| 111 | 119 | if (flag == IND_DONE) |
| 112 | 120 | break; |
| 113 | 121 | |
| 114 | − addr = phys_to_virt(*entry & PAGE_MASK); | |
| 122 | + addr = (unsigned long)phys_to_virt(*entry & PAGE_MASK); | |
| 115 | 123 | |
| 116 | 124 | switch (flag) { |
| 117 | 125 | case IND_INDIRECTION: |
| @@ -120,7 +128,7 @@ static void kexec_list_flush(struct kimage *kimage) · 2 unmodified lines | ||
| 120 | 128 | break; |
| 121 | 129 | case IND_SOURCE: |
| 122 | 130 | /* flush the source pages. */ |
| 123 | − __flush_dcache_area(addr, PAGE_SIZE); | |
| 131 | + dcache_clean_inval_poc(addr, addr + PAGE_SIZE); | |
| 124 | 132 | break; |
| 125 | 133 | case IND_DESTINATION: |
| 126 | 134 | break; |
| @@ -147,8 +155,10 @@ static void kexec_segment_flush(const struct kimage *kimage) · 20 unmodified lines | ||
| 147 | 155 | kimage->segment[i].memsz, |
| 148 | 156 | kimage->segment[i].memsz / PAGE_SIZE); |
| 149 | 157 | |
| 150 | − __flush_dcache_area(phys_to_virt(kimage->segment[i].mem), | |
| 151 | − kimage->segment[i].memsz); | |
| 158 | + dcache_clean_inval_poc( | |
| 159 | + (unsigned long)phys_to_virt(kimage->segment[i].mem), | |
| 160 | + (unsigned long)phys_to_virt(kimage->segment[i].mem) + | |
| 161 | + kimage->segment[i].memsz); | |
| 152 | 162 | } |
| 153 | 163 | } |
| 154 | 164 | |
| @@ -0,0 +1,150 @@ | ||
| 1 | +// SPDX-License-Identifier: GPL-2.0-only | |
| 2 | +#include <linux/kernel.h> | |
| 3 | +#include <linux/mm.h> | |
| 4 | +#include <linux/smp.h> | |
| 5 | +#include <linux/spinlock.h> | |
| 6 | +#include <linux/stop_machine.h> | |
| 7 | +#include <linux/uaccess.h> | |
| 8 | + | |
| 9 | +#include <asm/cacheflush.h> | |
| 10 | +#include <asm/fixmap.h> | |
| 11 | +#include <asm/insn.h> | |
| 12 | +#include <asm/kprobes.h> | |
| 13 | +#include <asm/patching.h> | |
| 14 | +#include <asm/sections.h> | |
| 15 | + | |
| 16 | +static DEFINE_RAW_SPINLOCK(patch_lock); | |
| 17 | + | |
| 18 | +static bool is_exit_text(unsigned long addr) | |
| 19 | +{ | |
| 20 | + /* discarded with init text/data */ | |
| 21 | + return system_state < SYSTEM_RUNNING && | |
| 22 | + addr >= (unsigned long)__exittext_begin && | |
| 23 | + addr < (unsigned long)__exittext_end; | |
| 24 | +} | |
| 25 | + | |
| 26 | +static bool is_image_text(unsigned long addr) | |
| 27 | +{ | |
| 28 | + return core_kernel_text(addr) || is_exit_text(addr); | |
| 29 | +} | |
| 30 | + | |
| 31 | +static void __kprobes *patch_map(void *addr, int fixmap) | |
| 32 | +{ | |
| 33 | + unsigned long uintaddr = (uintptr_t) addr; | |
| 34 | + bool image = is_image_text(uintaddr); | |
| 35 | + struct page *page; | |
| 36 | + | |
| 37 | + if (image) | |
| 38 | + page = phys_to_page(__pa_symbol(addr)); | |
| 39 | + else if (IS_ENABLED(CONFIG_STRICT_MODULE_RWX)) | |
| 40 | + page = vmalloc_to_page(addr); | |
| 41 | + else | |
| 42 | + return addr; | |
| 43 | + | |
| 44 | + BUG_ON(!page); | |
| 45 | + return (void *)set_fixmap_offset(fixmap, page_to_phys(page) + | |
| 46 | + (uintaddr & ~PAGE_MASK)); | |
| 47 | +} | |
| 48 | + | |
| 49 | +static void __kprobes patch_unmap(int fixmap) | |
| 50 | +{ | |
| 51 | + clear_fixmap(fixmap); | |
| 52 | +} | |
| 53 | +/* | |
| 54 | + * In ARMv8-A, A64 instructions have a fixed length of 32 bits and are always | |
| 55 | + * little-endian. | |
| 56 | + */ | |
| 57 | +int __kprobes aarch64_insn_read(void *addr, u32 *insnp) | |
| 58 | +{ | |
| 59 | + int ret; | |
| 60 | + __le32 val; | |
| 61 | + | |
| 62 | + ret = copy_from_kernel_nofault(&val, addr, AARCH64_INSN_SIZE); | |
| 63 | + if (!ret) | |
| 64 | + *insnp = le32_to_cpu(val); | |
| 65 | + | |
| 66 | + return ret; | |
| 67 | +} | |
| 68 | + | |
| 69 | +static int __kprobes __aarch64_insn_write(void *addr, __le32 insn) | |
| 70 | +{ | |
| 71 | + void *waddr = addr; | |
| 72 | + unsigned long flags = 0; | |
| 73 | + int ret; | |
| 74 | + | |
| 75 | + raw_spin_lock_irqsave(&patch_lock, flags); | |
| 76 | + waddr = patch_map(addr, FIX_TEXT_POKE0); | |
| 77 | + | |
| 78 | + ret = copy_to_kernel_nofault(waddr, &insn, AARCH64_INSN_SIZE); | |
| 79 | + | |
| 80 | + patch_unmap(FIX_TEXT_POKE0); | |
| 81 | + raw_spin_unlock_irqrestore(&patch_lock, flags); | |
| 82 | + | |
| 83 | + return ret; | |
| 84 | +} | |
| 85 | + | |
| 86 | +int __kprobes aarch64_insn_write(void *addr, u32 insn) | |
| 87 | +{ | |
| 88 | + return __aarch64_insn_write(addr, cpu_to_le32(insn)); | |
| 89 | +} | |
| 90 | + | |
| 91 | +int __kprobes aarch64_insn_patch_text_nosync(void *addr, u32 insn) | |
| 92 | +{ | |
| 93 | + u32 *tp = addr; | |
| 94 | + int ret; | |
| 95 | + | |
| 96 | + /* A64 instructions must be word aligned */ | |
| 97 | + if ((uintptr_t)tp & 0x3) | |
| 98 | + return -EINVAL; | |
| 99 | + | |
| 100 | + ret = aarch64_insn_write(tp, insn); | |
| 101 | + if (ret == 0) | |
| 102 | + caches_clean_inval_pou((uintptr_t)tp, | |
| 103 | + (uintptr_t)tp + AARCH64_INSN_SIZE); | |
| 104 | + | |
| 105 | + return ret; | |
| 106 | +} | |
| 107 | + | |
| 108 | +struct aarch64_insn_patch { | |
| 109 | + void **text_addrs; | |
| 110 | + u32 *new_insns; | |
| 111 | + int insn_cnt; | |
| 112 | + atomic_t cpu_count; | |
| 113 | +}; | |
| 114 | + | |
| 115 | +static int __kprobes aarch64_insn_patch_text_cb(void *arg) | |
| 116 | +{ | |
| 117 | + int i, ret = 0; | |
| 118 | + struct aarch64_insn_patch *pp = arg; | |
| 119 | + | |
| 120 | + /* The first CPU becomes master */ | |
| 121 | + if (atomic_inc_return(&pp->cpu_count) == 1) { | |
| 122 | + for (i = 0; ret == 0 && i < pp->insn_cnt; i++) | |
| 123 | + ret = aarch64_insn_patch_text_nosync(pp->text_addrs[i], | |
| 124 | + pp->new_insns[i]); | |
| 125 | + /* Notify other processors with an additional increment. */ | |
| 126 | + atomic_inc(&pp->cpu_count); | |
| 127 | + } else { | |
| 128 | + while (atomic_read(&pp->cpu_count) <= num_online_cpus()) | |
| 129 | + cpu_relax(); | |
| 130 | + isb(); | |
| 131 | + } | |
| 132 | + | |
| 133 | + return ret; | |
| 134 | +} | |
| 135 | + | |
| 136 | +int __kprobes aarch64_insn_patch_text(void *addrs[], u32 insns[], int cnt) | |
| 137 | +{ | |
| 138 | + struct aarch64_insn_patch patch = { | |
| 139 | + .text_addrs = addrs, | |
| 140 | + .new_insns = insns, | |
| 141 | + .insn_cnt = cnt, | |
| 142 | + .cpu_count = ATOMIC_INIT(0), | |
| 143 | + }; | |
| 144 | + | |
| 145 | + if (cnt <= 0) | |
| 146 | + return -EINVAL; | |
| 147 | + | |
| 148 | + return stop_machine_cpuslocked(aarch64_insn_patch_text_cb, &patch, | |
| 149 | + cpu_online_mask); | |
| 150 | +} | |
| @@ -116,7 +116,7 @@ void perf_callchain_user(struct perf_callchain_entry_ctx *entry, · 115 unmodified lines | ||
| 116 | 116 | tail = (struct frame_tail __user *)regs->regs[29]; |
| 117 | 117 | |
| 118 | 118 | while (entry->nr < entry->max_stack && |
| 119 | − tail && !((unsigned long)tail & 0xf)) | |
| 119 | + tail && !((unsigned long)tail & 0x7)) | |
| 120 | 120 | tail = user_backtrace(tail, entry); |
| 121 | 121 | } else { |
| 122 | 122 | #ifdef CONFIG_COMPAT |
| @@ -165,10 +165,7 @@ armv8pmu_events_sysfs_show(struct device *dev, · 164 unmodified lines | ||
| 165 | 165 | } |
| 166 | 166 | |
| 167 | 167 | #define ARMV8_EVENT_ATTR(name, config) \ |
| 168 | − (&((struct perf_pmu_events_attr) { \ | |
| 169 | − .attr = __ATTR(name, 0444, armv8pmu_events_sysfs_show, NULL), \ | |
| 170 | − .id = config, \ | |
| 171 | − }).attr.attr) | |
| 168 | + PMU_EVENT_ATTR_ID(name, armv8pmu_events_sysfs_show, config) | |
| 172 | 169 | |
| 173 | 170 | static struct attribute *armv8_pmuv3_event_attrs[] = { |
| 174 | 171 | ARMV8_EVENT_ATTR(sw_incr, ARMV8_PMUV3_PERFCTR_SW_INCR), |
| @@ -312,13 +309,46 @@ static ssize_t slots_show(struct device *dev, struct device_attribute *attr, · 137 unmodified lines | ||
| 312 | 309 | struct arm_pmu *cpu_pmu = container_of(pmu, struct arm_pmu, pmu); |
| 313 | 310 | u32 slots = cpu_pmu->reg_pmmir & ARMV8_PMU_SLOTS_MASK; |
| 314 | 311 | |
| 315 | − return snprintf(page, PAGE_SIZE, "0x%08x\n", slots); | |
| 312 | + return sysfs_emit(page, "0x%08x\n", slots); | |
| 316 | 313 | } |
| 317 | 314 | |
| 318 | 315 | static DEVICE_ATTR_RO(slots); |
| 319 | 316 | |
| 317 | +static ssize_t bus_slots_show(struct device *dev, struct device_attribute *attr, | |
| 318 | + char *page) | |
| 319 | +{ | |
| 320 | + struct pmu *pmu = dev_get_drvdata(dev); | |
| 321 | + struct arm_pmu *cpu_pmu = container_of(pmu, struct arm_pmu, pmu); | |
| 322 | + u32 bus_slots = (cpu_pmu->reg_pmmir >> ARMV8_PMU_BUS_SLOTS_SHIFT) | |
| 323 | + & ARMV8_PMU_BUS_SLOTS_MASK; | |
| 324 | + | |
| 325 | + return sysfs_emit(page, "0x%08x\n", bus_slots); | |
| 326 | +} | |
| 327 | + | |
| 328 | +static DEVICE_ATTR_RO(bus_slots); | |
| 329 | + | |
| 330 | +static ssize_t bus_width_show(struct device *dev, struct device_attribute *attr, | |
| 331 | + char *page) | |
| 332 | +{ | |
| 333 | + struct pmu *pmu = dev_get_drvdata(dev); | |
| 334 | + struct arm_pmu *cpu_pmu = container_of(pmu, struct arm_pmu, pmu); | |
| 335 | + u32 bus_width = (cpu_pmu->reg_pmmir >> ARMV8_PMU_BUS_WIDTH_SHIFT) | |
| 336 | + & ARMV8_PMU_BUS_WIDTH_MASK; | |
| 337 | + u32 val = 0; | |
| 338 | + | |
| 339 | + /* Encoded as Log2(number of bytes), plus one */ | |
| 340 | + if (bus_width > 2 && bus_width < 13) | |
| 341 | + val = 1 << (bus_width - 1); | |
| 342 | + | |
| 343 | + return sysfs_emit(page, "0x%08x\n", val); | |
| 344 | +} | |
| 345 | + | |
| 346 | +static DEVICE_ATTR_RO(bus_width); | |
| 347 | + | |
| 320 | 348 | static struct attribute *armv8_pmuv3_caps_attrs[] = { |
| 321 | 349 | &dev_attr_slots.attr, |
| 350 | + &dev_attr_bus_slots.attr, | |
| 351 | + &dev_attr_bus_width.attr, | |
| 322 | 352 | NULL, |
| 323 | 353 | }; |
| 324 | 354 | |
| @@ -7,26 +7,28 @@ · 6 unmodified lines | ||
| 7 | 7 | * Copyright (C) 2013 Linaro Limited. |
| 8 | 8 | * Author: Sandeepa Prabhu <sandeepa.prabhu@linaro.org> |
| 9 | 9 | */ |
| 10 | +#include <linux/extable.h> | |
| 10 | 11 | #include <linux/kasan.h> |
| 11 | 12 | #include <linux/kernel.h> |
| 12 | 13 | #include <linux/kprobes.h> |
| 13 | −#include <linux/extable.h> | |
| 14 | −#include <linux/slab.h> | |
| 15 | −#include <linux/stop_machine.h> | |
| 16 | 14 | #include <linux/sched/debug.h> |
| 17 | 15 | #include <linux/set_memory.h> |
| 16 | +#include <linux/slab.h> | |
| 17 | +#include <linux/stop_machine.h> | |
| 18 | 18 | #include <linux/stringify.h> |
| 19 | +#include <linux/uaccess.h> | |
| 19 | 20 | #include <linux/vmalloc.h> |
| 20 | −#include <asm/traps.h> | |
| 21 | −#include <asm/ptrace.h> | |
| 21 | + | |
| 22 | 22 | #include <asm/cacheflush.h> |
| 23 | −#include <asm/debug-monitors.h> | |
| 24 | 23 | #include <asm/daifflags.h> |
| 25 | −#include <asm/system_misc.h> | |
| 24 | +#include <asm/debug-monitors.h> | |
| 26 | 25 | #include <asm/insn.h> |
| 27 | −#include <linux/uaccess.h> | |
| 28 | 26 | #include <asm/irq.h> |
| 27 | +#include <asm/patching.h> | |
| 28 | +#include <asm/ptrace.h> | |
| 29 | 29 | #include <asm/sections.h> |
| 30 | +#include <asm/system_misc.h> | |
| 31 | +#include <asm/traps.h> | |
| 30 | 32 | |
| 31 | 33 | #include "decode-insn.h" |
| 32 | 34 | |
| @@ -10,6 +10,7 @@ · 9 unmodified lines | ||
| 10 | 10 | #include <linux/kprobes.h> |
| 11 | 11 | |
| 12 | 12 | #include <asm/ptrace.h> |
| 13 | +#include <asm/traps.h> | |
| 13 | 14 | |
| 14 | 15 | #include "simulate-insn.h" |
| 15 | 16 | |
| @@ -21,7 +21,7 @@ void arch_uprobe_copy_ixol(struct page *page, unsigned long vaddr, · 20 unmodified lines | ||
| 21 | 21 | memcpy(dst, src, len); |
| 22 | 22 | |
| 23 | 23 | /* flush caches (dcache/icache) */ |
| 24 | − sync_icache_aliases(dst, len); | |
| 24 | + sync_icache_aliases((unsigned long)dst, (unsigned long)dst + len); | |
| 25 | 25 | |
| 26 | 26 | kunmap_atomic(xol_page_kaddr); |
| 27 | 27 | } |
| @@ -18,7 +18,6 @@ · 17 unmodified lines | ||
| 18 | 18 | #include <linux/sched/task.h> |
| 19 | 19 | #include <linux/sched/task_stack.h> |
| 20 | 20 | #include <linux/kernel.h> |
| 21 | −#include <linux/lockdep.h> | |
| 22 | 21 | #include <linux/mman.h> |
| 23 | 22 | #include <linux/mm.h> |
| 24 | 23 | #include <linux/nospec.h> |
| @@ -46,7 +45,6 @@ · 21 unmodified lines | ||
| 46 | 45 | #include <linux/prctl.h> |
| 47 | 46 | |
| 48 | 47 | #include <asm/alternative.h> |
| 49 | −#include <asm/arch_gicv3.h> | |
| 50 | 48 | #include <asm/compat.h> |
| 51 | 49 | #include <asm/cpufeature.h> |
| 52 | 50 | #include <asm/cacheflush.h> |
| @@ -74,63 +72,6 @@ EXPORT_SYMBOL_GPL(pm_power_off); · 21 unmodified lines | ||
| 74 | 72 | |
| 75 | 73 | void (*arm_pm_restart)(enum reboot_mode reboot_mode, const char *cmd); |
| 76 | 74 | |
| 77 | −static void noinstr __cpu_do_idle(void) | |
| 78 | −{ | |
| 79 | − dsb(sy); | |
| 80 | − wfi(); | |
| 81 | −} | |
| 82 | − | |
| 83 | −static void noinstr __cpu_do_idle_irqprio(void) | |
| 84 | −{ | |
| 85 | − unsigned long pmr; | |
| 86 | − unsigned long daif_bits; | |
| 87 | − | |
| 88 | − daif_bits = read_sysreg(daif); | |
| 89 | − write_sysreg(daif_bits | PSR_I_BIT | PSR_F_BIT, daif); | |
| 90 | − | |
| 91 | − /* | |
| 92 | − * Unmask PMR before going idle to make sure interrupts can | |
| 93 | − * be raised. | |
| 94 | − */ | |
| 95 | − pmr = gic_read_pmr(); | |
| 96 | − gic_write_pmr(GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET); | |
| 97 | − | |
| 98 | − __cpu_do_idle(); | |
| 99 | − | |
| 100 | − gic_write_pmr(pmr); | |
| 101 | − write_sysreg(daif_bits, daif); | |
| 102 | −} | |
| 103 | − | |
| 104 | −/* | |
| 105 | − * cpu_do_idle() | |
| 106 | − * | |
| 107 | − * Idle the processor (wait for interrupt). | |
| 108 | − * | |
| 109 | − * If the CPU supports priority masking we must do additional work to | |
| 110 | − * ensure that interrupts are not masked at the PMR (because the core will | |
| 111 | − * not wake up if we block the wake up signal in the interrupt controller). | |
| 112 | − */ | |
| 113 | −void noinstr cpu_do_idle(void) | |
| 114 | −{ | |
| 115 | − if (system_uses_irq_prio_masking()) | |
| 116 | − __cpu_do_idle_irqprio(); | |
| 117 | − else | |
| 118 | − __cpu_do_idle(); | |
| 119 | −} | |
| 120 | − | |
| 121 | −/* | |
| 122 | − * This is our default idle handler. | |
| 123 | − */ | |
| 124 | −void noinstr arch_cpu_idle(void) | |
| 125 | −{ | |
| 126 | − /* | |
| 127 | − * This should do all the clock switching and wait for interrupt | |
| 128 | − * tricks | |
| 129 | − */ | |
| 130 | − cpu_do_idle(); | |
| 131 | − raw_local_irq_enable(); | |
| 132 | −} | |
| 133 | − | |
| 134 | 75 | #ifdef CONFIG_HOTPLUG_CPU |
| 135 | 76 | void arch_cpu_idle_dead(void) |
| 136 | 77 | { |
| @@ -435,6 +376,11 @@ int copy_thread(unsigned long clone_flags, unsigned long stack_start, · 298 unmodified lines | ||
| 435 | 376 | } |
| 436 | 377 | p->thread.cpu_context.pc = (unsigned long)ret_from_fork; |
| 437 | 378 | p->thread.cpu_context.sp = (unsigned long)childregs; |
| 379 | + /* | |
| 380 | + * For the benefit of the unwinder, set up childregs->stackframe | |
| 381 | + * as the final frame for the new task. | |
| 382 | + */ | |
| 383 | + p->thread.cpu_context.fp = (unsigned long)childregs->stackframe; | |
| 438 | 384 | |
| 439 | 385 | ptrace_hw_copy_thread(p); |
| 440 | 386 | |
| @@ -527,6 +473,15 @@ static void erratum_1418040_thread_switch(struct task_struct *prev, · 86 unmodified lines | ||
| 527 | 473 | write_sysreg(val, cntkctl_el1); |
| 528 | 474 | } |
| 529 | 475 | |
| 476 | +static void compat_thread_switch(struct task_struct *next) | |
| 477 | +{ | |
| 478 | + if (!is_compat_thread(task_thread_info(next))) | |
| 479 | + return; | |
| 480 | + | |
| 481 | + if (static_branch_unlikely(&arm64_mismatched_32bit_el0)) | |
| 482 | + set_tsk_thread_flag(next, TIF_NOTIFY_RESUME); | |
| 483 | +} | |
| 484 | + | |
| 530 | 485 | static void update_sctlr_el1(u64 sctlr) |
| 531 | 486 | { |
| 532 | 487 | /* |
| @@ -568,6 +523,7 @@ __notrace_funcgraph struct task_struct *__switch_to(struct task_struct *prev, · 35 unmodified lines | ||
| 568 | 523 | ssbs_thread_switch(next); |
| 569 | 524 | erratum_1418040_thread_switch(prev, next); |
| 570 | 525 | ptrauth_thread_switch_user(next); |
| 526 | + compat_thread_switch(next); | |
| 571 | 527 | |
| 572 | 528 | /* |
| 573 | 529 | * Complete any pending TLB or cache maintenance on this CPU in case |
| @@ -633,8 +589,15 @@ unsigned long arch_align_stack(unsigned long sp) · 59 unmodified lines | ||
| 633 | 589 | */ |
| 634 | 590 | void arch_setup_new_exec(void) |
| 635 | 591 | { |
| 636 | − current->mm->context.flags = is_compat_task() ? MMCF_AARCH32 : 0; | |
| 592 | + unsigned long mmflags = 0; | |
| 593 | + | |
| 594 | + if (is_compat_task()) { | |
| 595 | + mmflags = MMCF_AARCH32; | |
| 596 | + if (static_branch_unlikely(&arm64_mismatched_32bit_el0)) | |
| 597 | + set_tsk_thread_flag(current, TIF_NOTIFY_RESUME); | |
| 598 | + } | |
| 637 | 599 | |
| 600 | + current->mm->context.flags = mmflags; | |
| 638 | 601 | ptrauth_thread_init_user(); |
| 639 | 602 | mte_thread_init_user(); |
| 640 | 603 | |
| @@ -724,22 +687,6 @@ static int __init tagged_addr_init(void) · 83 unmodified lines | ||
| 724 | 687 | core_initcall(tagged_addr_init); |
| 725 | 688 | #endif /* CONFIG_ARM64_TAGGED_ADDR_ABI */ |
| 726 | 689 | |
| 727 | −asmlinkage void __sched arm64_preempt_schedule_irq(void) | |
| 728 | −{ | |
| 729 | − lockdep_assert_irqs_disabled(); | |
| 730 | − | |
| 731 | − /* | |
| 732 | − * Preempting a task from an IRQ means we leave copies of PSTATE | |
| 733 | − * on the stack. cpufeature's enable calls may modify PSTATE, but | |
| 734 | − * resuming one of these preempted tasks would undo those changes. | |
| 735 | − * | |
| 736 | − * Only allow a task to be preempted once cpufeatures have been | |
| 737 | − * enabled. | |
| 738 | − */ | |
| 739 | − if (system_capabilities_finalized()) | |
| 740 | − preempt_schedule_irq(); | |
| 741 | −} | |
| 742 | − | |
| 743 | 690 | #ifdef CONFIG_BINFMT_ELF |
| 744 | 691 | int arch_elf_adjust_prot(int prot, const struct arch_elf_state *state, |
| 745 | 692 | bool has_interp, bool is_interp) |
| @@ -122,7 +122,7 @@ static bool regs_within_kernel_stack(struct pt_regs *regs, unsigned long addr) · 121 unmodified lines | ||
| 122 | 122 | { |
| 123 | 123 | return ((addr & ~(THREAD_SIZE - 1)) == |
| 124 | 124 | (kernel_stack_pointer(regs) & ~(THREAD_SIZE - 1))) || |
| 125 | − on_irq_stack(addr, NULL); | |
| 125 | + on_irq_stack(addr, sizeof(unsigned long), NULL); | |
| 126 | 126 | } |
| 127 | 127 | |
| 128 | 128 | /** |
| @@ -162,31 +162,33 @@ static int init_sdei_scs(void) · 161 unmodified lines | ||
| 162 | 162 | return err; |
| 163 | 163 | } |
| 164 | 164 | |
| 165 | −static bool on_sdei_normal_stack(unsigned long sp, struct stack_info *info) | |
| 165 | +static bool on_sdei_normal_stack(unsigned long sp, unsigned long size, | |
| 166 | + struct stack_info *info) | |
| 166 | 167 | { |
| 167 | 168 | unsigned long low = (unsigned long)raw_cpu_read(sdei_stack_normal_ptr); |
| 168 | 169 | unsigned long high = low + SDEI_STACK_SIZE; |
| 169 | 170 | |
| 170 | − return on_stack(sp, low, high, STACK_TYPE_SDEI_NORMAL, info); | |
| 171 | + return on_stack(sp, size, low, high, STACK_TYPE_SDEI_NORMAL, info); | |
| 171 | 172 | } |
| 172 | 173 | |
| 173 | −static bool on_sdei_critical_stack(unsigned long sp, struct stack_info *info) | |
| 174 | +static bool on_sdei_critical_stack(unsigned long sp, unsigned long size, | |
| 175 | + struct stack_info *info) | |
| 174 | 176 | { |
| 175 | 177 | unsigned long low = (unsigned long)raw_cpu_read(sdei_stack_critical_ptr); |
| 176 | 178 | unsigned long high = low + SDEI_STACK_SIZE; |
| 177 | 179 | |
| 178 | − return on_stack(sp, low, high, STACK_TYPE_SDEI_CRITICAL, info); | |
| 180 | + return on_stack(sp, size, low, high, STACK_TYPE_SDEI_CRITICAL, info); | |
| 179 | 181 | } |
| 180 | 182 | |
| 181 | −bool _on_sdei_stack(unsigned long sp, struct stack_info *info) | |
| 183 | +bool _on_sdei_stack(unsigned long sp, unsigned long size, struct stack_info *info) | |
| 182 | 184 | { |
| 183 | 185 | if (!IS_ENABLED(CONFIG_VMAP_STACK)) |
| 184 | 186 | return false; |
| 185 | 187 | |
| 186 | − if (on_sdei_critical_stack(sp, info)) | |
| 188 | + if (on_sdei_critical_stack(sp, size, info)) | |
| 187 | 189 | return true; |
| 188 | 190 | |
| 189 | − if (on_sdei_normal_stack(sp, info)) | |
| 191 | + if (on_sdei_normal_stack(sp, size, info)) | |
| 190 | 192 | return true; |
| 191 | 193 | |
| 192 | 194 | return false; |
| @@ -231,13 +233,13 @@ out_err: · 38 unmodified lines | ||
| 231 | 233 | } |
| 232 | 234 | |
| 233 | 235 | /* |
| 234 | − * __sdei_handler() returns one of: | |
| 236 | + * do_sdei_event() returns one of: | |
| 235 | 237 | * SDEI_EV_HANDLED - success, return to the interrupted context. |
| 236 | 238 | * SDEI_EV_FAILED - failure, return this error code to firmare. |
| 237 | 239 | * virtual-address - success, return to this address. |
| 238 | 240 | */ |
| 239 | −static __kprobes unsigned long _sdei_handler(struct pt_regs *regs, | |
| 240 | − struct sdei_registered_event *arg) | |
| 241 | +unsigned long __kprobes do_sdei_event(struct pt_regs *regs, | |
| 242 | + struct sdei_registered_event *arg) | |
| 241 | 243 | { |
| 242 | 244 | u32 mode; |
| 243 | 245 | int i, err = 0; |
| @@ -292,45 +294,3 @@ static __kprobes unsigned long _sdei_handler(struct pt_regs *regs, · 48 unmodified lines | ||
| 292 | 294 | |
| 293 | 295 | return vbar + 0x480; |
| 294 | 296 | } |
| 295 | − | |
| 296 | −static void __kprobes notrace __sdei_pstate_entry(void) | |
| 297 | −{ | |
| 298 | − /* | |
| 299 | − * The original SDEI spec (ARM DEN 0054A) can be read ambiguously as to | |
| 300 | − * whether PSTATE bits are inherited unchanged or generated from | |
| 301 | − * scratch, and the TF-A implementation always clears PAN and always | |
| 302 | − * clears UAO. There are no other known implementations. | |
| 303 | − * | |
| 304 | − * Subsequent revisions (ARM DEN 0054B) follow the usual rules for how | |
| 305 | − * PSTATE is modified upon architectural exceptions, and so PAN is | |
| 306 | − * either inherited or set per SCTLR_ELx.SPAN, and UAO is always | |
| 307 | − * cleared. | |
| 308 | − * | |
| 309 | − * We must explicitly reset PAN to the expected state, including | |
| 310 | − * clearing it when the host isn't using it, in case a VM had it set. | |
| 311 | − */ | |
| 312 | − if (system_uses_hw_pan()) | |
| 313 | − set_pstate_pan(1); | |
| 314 | − else if (cpu_has_pan()) | |
| 315 | − set_pstate_pan(0); | |
| 316 | −} | |
| 317 | − | |
| 318 | −asmlinkage noinstr unsigned long | |
| 319 | −__sdei_handler(struct pt_regs *regs, struct sdei_registered_event *arg) | |
| 320 | −{ | |
| 321 | − unsigned long ret; | |
| 322 | − | |
| 323 | − /* | |
| 324 | − * We didn't take an exception to get here, so the HW hasn't | |
| 325 | − * set/cleared bits in PSTATE that we may rely on. Initialize PAN. | |
| 326 | − */ | |
| 327 | − __sdei_pstate_entry(); | |
| 328 | − | |
| 329 | − arm64_enter_nmi(regs); | |
| 330 | − | |
| 331 | − ret = _sdei_handler(regs, arg); | |
| 332 | − | |
| 333 | − arm64_exit_nmi(regs); | |
| 334 | − | |
| 335 | − return ret; | |
| 336 | −} | |
| @@ -87,12 +87,6 @@ void __init smp_setup_processor_id(void) · 86 unmodified lines | ||
| 87 | 87 | u64 mpidr = read_cpuid_mpidr() & MPIDR_HWID_BITMASK; |
| 88 | 88 | set_cpu_logical_map(0, mpidr); |
| 89 | 89 | |
| 90 | − /* | |
| 91 | − * clear __my_cpu_offset on boot CPU to avoid hang caused by | |
| 92 | − * using percpu variable early, for example, lockdep will | |
| 93 | − * access percpu variable inside lock_release | |
| 94 | − */ | |
| 95 | − set_my_cpu_offset(0); | |
| 96 | 90 | pr_info("Booting Linux on physical CPU 0x%010lx [0x%08x]\n", |
| 97 | 91 | (unsigned long)mpidr, read_cpuid_id()); |
| 98 | 92 | } |
| @@ -381,7 +375,7 @@ void __init __no_sanitize_address setup_arch(char **cmdline_p) · 282 unmodified lines | ||
| 381 | 375 | * faults in case uaccess_enable() is inadvertently called by the init |
| 382 | 376 | * thread. |
| 383 | 377 | */ |
| 384 | − init_task.thread_info.ttbr0 = __pa_symbol(reserved_pg_dir); | |
| 378 | + init_task.thread_info.ttbr0 = phys_to_ttbr(__pa_symbol(reserved_pg_dir)); | |
| 385 | 379 | #endif |
| 386 | 380 | |
| 387 | 381 | if (boot_args[1] || boot_args[2] || boot_args[3]) { |
| @@ -911,6 +911,19 @@ static void do_signal(struct pt_regs *regs) · 910 unmodified lines | ||
| 911 | 911 | restore_saved_sigmask(); |
| 912 | 912 | } |
| 913 | 913 | |
| 914 | +static bool cpu_affinity_invalid(struct pt_regs *regs) | |
| 915 | +{ | |
| 916 | + if (!compat_user_mode(regs)) | |
| 917 | + return false; | |
| 918 | + | |
| 919 | + /* | |
| 920 | + * We're preemptible, but a reschedule will cause us to check the | |
| 921 | + * affinity again. | |
| 922 | + */ | |
| 923 | + return !cpumask_test_cpu(raw_smp_processor_id(), | |
| 924 | + system_32bit_el0_cpumask()); | |
| 925 | +} | |
| 926 | + | |
| 914 | 927 | asmlinkage void do_notify_resume(struct pt_regs *regs, |
| 915 | 928 | unsigned long thread_flags) |
| 916 | 929 | { |
| @@ -938,6 +951,19 @@ asmlinkage void do_notify_resume(struct pt_regs *regs, · 21 unmodified lines | ||
| 938 | 951 | if (thread_flags & _TIF_NOTIFY_RESUME) { |
| 939 | 952 | tracehook_notify_resume(regs); |
| 940 | 953 | rseq_handle_notify_resume(NULL, regs); |
| 954 | + | |
| 955 | + /* | |
| 956 | + * If we reschedule after checking the affinity | |
| 957 | + * then we must ensure that TIF_NOTIFY_RESUME | |
| 958 | + * is set so that we check the affinity again. | |
| 959 | + * Since tracehook_notify_resume() clears the | |
| 960 | + * flag, ensure that the compiler doesn't move | |
| 961 | + * it after the affinity check. | |
| 962 | + */ | |
| 963 | + barrier(); | |
| 964 | + | |
| 965 | + if (cpu_affinity_invalid(regs)) | |
| 966 | + force_sig(SIGKILL); | |
| 941 | 967 | } |
| 942 | 968 | |
| 943 | 969 | if (thread_flags & _TIF_FOREIGN_FPSTATE) |
| @@ -7,8 +7,34 @@ · 6 unmodified lines | ||
| 7 | 7 | |
| 8 | 8 | #include <asm/asm-offsets.h> |
| 9 | 9 | #include <asm/assembler.h> |
| 10 | +#include <asm/thread_info.h> | |
| 11 | + | |
| 12 | +/* | |
| 13 | + * If we have SMCCC v1.3 and (as is likely) no SVE state in | |
| 14 | + * the registers then set the SMCCC hint bit to say there's no | |
| 15 | + * need to preserve it. Do this by directly adjusting the SMCCC | |
| 16 | + * function value which is already stored in x0 ready to be called. | |
| 17 | + */ | |
| 18 | +SYM_FUNC_START(__arm_smccc_sve_check) | |
| 19 | + | |
| 20 | + ldr_l x16, smccc_has_sve_hint | |
| 21 | + cbz x16, 2f | |
| 22 | + | |
| 23 | + get_current_task x16 | |
| 24 | + ldr x16, [x16, #TSK_TI_FLAGS] | |
| 25 | + tbnz x16, #TIF_FOREIGN_FPSTATE, 1f // Any live FP state? | |
| 26 | + tbnz x16, #TIF_SVE, 2f // Does that state include SVE? | |
| 27 | + | |
| 28 | +1: orr x0, x0, ARM_SMCCC_1_3_SVE_HINT | |
| 29 | + | |
| 30 | +2: ret | |
| 31 | +SYM_FUNC_END(__arm_smccc_sve_check) | |
| 32 | +EXPORT_SYMBOL(__arm_smccc_sve_check) | |
| 10 | 33 | |
| 11 | 34 | .macro SMCCC instr |
| 35 | +alternative_if ARM64_SVE | |
| 36 | + bl __arm_smccc_sve_check | |
| 37 | +alternative_else_nop_endif | |
| 12 | 38 | \instr #0 |
| 13 | 39 | ldr x4, [sp] |
| 14 | 40 | stp x0, x1, [x4, #ARM_SMCCC_RES_X0_OFFS] |
| @@ -43,3 +69,60 @@ SYM_FUNC_START(__arm_smccc_hvc) · 28 unmodified lines | ||
| 43 | 69 | SMCCC hvc |
| 44 | 70 | SYM_FUNC_END(__arm_smccc_hvc) |
| 45 | 71 | EXPORT_SYMBOL(__arm_smccc_hvc) |
| 72 | + | |
| 73 | + .macro SMCCC_1_2 instr | |
| 74 | + /* Save `res` and free a GPR that won't be clobbered */ | |
| 75 | + stp x1, x19, [sp, #-16]! | |
| 76 | + | |
| 77 | + /* Ensure `args` won't be clobbered while loading regs in next step */ | |
| 78 | + mov x19, x0 | |
| 79 | + | |
| 80 | + /* Load the registers x0 - x17 from the struct arm_smccc_1_2_regs */ | |
| 81 | + ldp x0, x1, [x19, #ARM_SMCCC_1_2_REGS_X0_OFFS] | |
| 82 | + ldp x2, x3, [x19, #ARM_SMCCC_1_2_REGS_X2_OFFS] | |
| 83 | + ldp x4, x5, [x19, #ARM_SMCCC_1_2_REGS_X4_OFFS] | |
| 84 | + ldp x6, x7, [x19, #ARM_SMCCC_1_2_REGS_X6_OFFS] | |
| 85 | + ldp x8, x9, [x19, #ARM_SMCCC_1_2_REGS_X8_OFFS] | |
| 86 | + ldp x10, x11, [x19, #ARM_SMCCC_1_2_REGS_X10_OFFS] | |
| 87 | + ldp x12, x13, [x19, #ARM_SMCCC_1_2_REGS_X12_OFFS] | |
| 88 | + ldp x14, x15, [x19, #ARM_SMCCC_1_2_REGS_X14_OFFS] | |
| 89 | + ldp x16, x17, [x19, #ARM_SMCCC_1_2_REGS_X16_OFFS] | |
| 90 | + | |
| 91 | + \instr #0 | |
| 92 | + | |
| 93 | + /* Load the `res` from the stack */ | |
| 94 | + ldr x19, [sp] | |
| 95 | + | |
| 96 | + /* Store the registers x0 - x17 into the result structure */ | |
| 97 | + stp x0, x1, [x19, #ARM_SMCCC_1_2_REGS_X0_OFFS] | |
| 98 | + stp x2, x3, [x19, #ARM_SMCCC_1_2_REGS_X2_OFFS] | |
| 99 | + stp x4, x5, [x19, #ARM_SMCCC_1_2_REGS_X4_OFFS] | |
| 100 | + stp x6, x7, [x19, #ARM_SMCCC_1_2_REGS_X6_OFFS] | |
| 101 | + stp x8, x9, [x19, #ARM_SMCCC_1_2_REGS_X8_OFFS] | |
| 102 | + stp x10, x11, [x19, #ARM_SMCCC_1_2_REGS_X10_OFFS] | |
| 103 | + stp x12, x13, [x19, #ARM_SMCCC_1_2_REGS_X12_OFFS] | |
| 104 | + stp x14, x15, [x19, #ARM_SMCCC_1_2_REGS_X14_OFFS] | |
| 105 | + stp x16, x17, [x19, #ARM_SMCCC_1_2_REGS_X16_OFFS] | |
| 106 | + | |
| 107 | + /* Restore original x19 */ | |
| 108 | + ldp xzr, x19, [sp], #16 | |
| 109 | + ret | |
| 110 | +.endm | |
| 111 | + | |
| 112 | +/* | |
| 113 | + * void arm_smccc_1_2_hvc(const struct arm_smccc_1_2_regs *args, | |
| 114 | + * struct arm_smccc_1_2_regs *res); | |
| 115 | + */ | |
| 116 | +SYM_FUNC_START(arm_smccc_1_2_hvc) | |
| 117 | + SMCCC_1_2 hvc | |
| 118 | +SYM_FUNC_END(arm_smccc_1_2_hvc) | |
| 119 | +EXPORT_SYMBOL(arm_smccc_1_2_hvc) | |
| 120 | + | |
| 121 | +/* | |
| 122 | + * void arm_smccc_1_2_smc(const struct arm_smccc_1_2_regs *args, | |
| 123 | + * struct arm_smccc_1_2_regs *res); | |
| 124 | + */ | |
| 125 | +SYM_FUNC_START(arm_smccc_1_2_smc) | |
| 126 | + SMCCC_1_2 smc | |
| 127 | +SYM_FUNC_END(arm_smccc_1_2_smc) | |
| 128 | +EXPORT_SYMBOL(arm_smccc_1_2_smc) | |
| @@ -120,9 +120,7 @@ int __cpu_up(unsigned int cpu, struct task_struct *idle) · 119 unmodified lines | ||
| 120 | 120 | * page tables. |
| 121 | 121 | */ |
| 122 | 122 | secondary_data.task = idle; |
| 123 | − secondary_data.stack = task_stack_page(idle) + THREAD_SIZE; | |
| 124 | 123 | update_cpu_boot_status(CPU_MMU_OFF); |
| 125 | − __flush_dcache_area(&secondary_data, sizeof(secondary_data)); | |
| 126 | 124 | |
| 127 | 125 | /* Now bring the CPU into our world */ |
| 128 | 126 | ret = boot_secondary(cpu, idle); |
| @@ -142,8 +140,6 @@ int __cpu_up(unsigned int cpu, struct task_struct *idle) · 13 unmodified lines | ||
| 142 | 140 | |
| 143 | 141 | pr_crit("CPU%u: failed to come online\n", cpu); |
| 144 | 142 | secondary_data.task = NULL; |
| 145 | − secondary_data.stack = NULL; | |
| 146 | − __flush_dcache_area(&secondary_data, sizeof(secondary_data)); | |
| 147 | 143 | status = READ_ONCE(secondary_data.status); |
| 148 | 144 | if (status == CPU_MMU_OFF) |
| 149 | 145 | status = READ_ONCE(__early_cpu_boot_status); |
| @@ -202,10 +198,7 @@ asmlinkage notrace void secondary_start_kernel(void) · 52 unmodified lines | ||
| 202 | 198 | u64 mpidr = read_cpuid_mpidr() & MPIDR_HWID_BITMASK; |
| 203 | 199 | struct mm_struct *mm = &init_mm; |
| 204 | 200 | const struct cpu_operations *ops; |
| 205 | − unsigned int cpu; | |
| 206 | − | |
| 207 | − cpu = task_cpu(current); | |
| 208 | − set_my_cpu_offset(per_cpu_offset(cpu)); | |
| 201 | + unsigned int cpu = smp_processor_id(); | |
| 209 | 202 | |
| 210 | 203 | /* |
| 211 | 204 | * All kernel threads share the same mm context; grab a |
| @@ -351,7 +344,7 @@ void __cpu_die(unsigned int cpu) · 139 unmodified lines | ||
| 351 | 344 | pr_crit("CPU%u: cpu didn't die\n", cpu); |
| 352 | 345 | return; |
| 353 | 346 | } |
| 354 | − pr_notice("CPU%u: shutdown\n", cpu); | |
| 347 | + pr_debug("CPU%u: shutdown\n", cpu); | |
| 355 | 348 | |
| 356 | 349 | /* |
| 357 | 350 | * Now that the dying CPU is beyond the point of no return w.r.t. |
| @@ -451,6 +444,11 @@ void __init smp_cpus_done(unsigned int max_cpus) · 93 unmodified lines | ||
| 451 | 444 | |
| 452 | 445 | void __init smp_prepare_boot_cpu(void) |
| 453 | 446 | { |
| 447 | + /* | |
| 448 | + * The runtime per-cpu areas have been allocated by | |
| 449 | + * setup_per_cpu_areas(), and CPU0's boot time per-cpu area will be | |
| 450 | + * freed shortly, so we must move over to the runtime per-cpu area. | |
| 451 | + */ | |
| 454 | 452 | set_my_cpu_offset(per_cpu_offset(smp_processor_id())); |
| 455 | 453 | cpuinfo_store_boot_cpu(); |
| 456 | 454 | |
| @@ -36,7 +36,7 @@ static void write_pen_release(u64 val) · 35 unmodified lines | ||
| 36 | 36 | unsigned long size = sizeof(secondary_holding_pen_release); |
| 37 | 37 | |
| 38 | 38 | secondary_holding_pen_release = val; |
| 39 | − __flush_dcache_area(start, size); | |
| 39 | + dcache_clean_inval_poc((unsigned long)start, (unsigned long)start + size); | |
| 40 | 40 | } |
| 41 | 41 | |
| 42 | 42 | |
| @@ -90,8 +90,9 @@ static int smp_spin_table_cpu_prepare(unsigned int cpu) · 47 unmodified lines | ||
| 90 | 90 | * the boot protocol. |
| 91 | 91 | */ |
| 92 | 92 | writeq_relaxed(pa_holding_pen, release_addr); |
| 93 | − __flush_dcache_area((__force void *)release_addr, | |
| 94 | − sizeof(*release_addr)); | |
| 93 | + dcache_clean_inval_poc((__force unsigned long)release_addr, | |
| 94 | + (__force unsigned long)release_addr + | |
| 95 | + sizeof(*release_addr)); | |
| 95 | 96 | |
| 96 | 97 | /* |
| 97 | 98 | * Send an event to wake up the secondary CPU. |
| @@ -68,13 +68,17 @@ int notrace unwind_frame(struct task_struct *tsk, struct stackframe *frame) · 67 unmodified lines | ||
| 68 | 68 | unsigned long fp = frame->fp; |
| 69 | 69 | struct stack_info info; |
| 70 | 70 | |
| 71 | − if (fp & 0xf) | |
| 72 | − return -EINVAL; | |
| 73 | − | |
| 74 | 71 | if (!tsk) |
| 75 | 72 | tsk = current; |
| 76 | 73 | |
| 77 | − if (!on_accessible_stack(tsk, fp, &info)) | |
| 74 | + /* Final frame; nothing to unwind */ | |
| 75 | + if (fp == (unsigned long)task_pt_regs(tsk)->stackframe) | |
| 76 | + return -ENOENT; | |
| 77 | + | |
| 78 | + if (fp & 0x7) | |
| 79 | + return -EINVAL; | |
| 80 | + | |
| 81 | + if (!on_accessible_stack(tsk, fp, 16, &info)) | |
| 78 | 82 | return -EINVAL; |
| 79 | 83 | |
| 80 | 84 | if (test_bit(info.type, frame->stacks_done)) |
| @@ -128,12 +132,6 @@ int notrace unwind_frame(struct task_struct *tsk, struct stackframe *frame) · 47 unmodified lines | ||
| 128 | 132 | |
| 129 | 133 | frame->pc = ptrauth_strip_insn_pac(frame->pc); |
| 130 | 134 | |
| 131 | − /* | |
| 132 | − * This is a terminal record, so we have finished unwinding. | |
| 133 | − */ | |
| 134 | − if (!frame->fp && !frame->pc) | |
| 135 | − return -ENOENT; | |
| 136 | − | |
| 137 | 135 | return 0; |
| 138 | 136 | } |
| 139 | 137 | NOKPROBE_SYMBOL(unwind_frame); |
| @@ -7,6 +7,7 @@ · 6 unmodified lines | ||
| 7 | 7 | #include <asm/alternative.h> |
| 8 | 8 | #include <asm/cacheflush.h> |
| 9 | 9 | #include <asm/cpufeature.h> |
| 10 | +#include <asm/cpuidle.h> | |
| 10 | 11 | #include <asm/daifflags.h> |
| 11 | 12 | #include <asm/debug-monitors.h> |
| 12 | 13 | #include <asm/exec.h> |
| @@ -91,6 +92,7 @@ int cpu_suspend(unsigned long arg, int (*fn)(unsigned long)) · 78 unmodified lines | ||
| 91 | 92 | int ret = 0; |
| 92 | 93 | unsigned long flags; |
| 93 | 94 | struct sleep_stack_data state; |
| 95 | + struct arm_cpuidle_irq_context context; | |
| 94 | 96 | |
| 95 | 97 | /* Report any MTE async fault before going to suspend */ |
| 96 | 98 | mte_suspend_enter(); |
| @@ -103,12 +105,18 @@ int cpu_suspend(unsigned long arg, int (*fn)(unsigned long)) · 6 unmodified lines | ||
| 103 | 105 | flags = local_daif_save(); |
| 104 | 106 | |
| 105 | 107 | /* |
| 106 | − * Function graph tracer state gets incosistent when the kernel | |
| 108 | + * Function graph tracer state gets inconsistent when the kernel | |
| 107 | 109 | * calls functions that never return (aka suspend finishers) hence |
| 108 | 110 | * disable graph tracing during their execution. |
| 109 | 111 | */ |
| 110 | 112 | pause_graph_tracing(); |
| 111 | 113 | |
| 114 | + /* | |
| 115 | + * Switch to using DAIF.IF instead of PMR in order to reliably | |
| 116 | + * resume if we're using pseudo-NMIs. | |
| 117 | + */ | |
| 118 | + arm_cpuidle_save_irq_context(&context); | |
| 119 | + | |
| 112 | 120 | if (__cpu_suspend_enter(&state)) { |
| 113 | 121 | /* Call the suspend finisher */ |
| 114 | 122 | ret = fn(arg); |
| @@ -126,6 +134,8 @@ int cpu_suspend(unsigned long arg, int (*fn)(unsigned long)) · 11 unmodified lines | ||
| 126 | 134 | RCU_NONIDLE(__cpu_suspend_exit()); |
| 127 | 135 | } |
| 128 | 136 | |
| 137 | + arm_cpuidle_restore_irq_context(&context); | |
| 138 | + | |
| 129 | 139 | unpause_graph_tracing(); |
| 130 | 140 | |
| 131 | 141 | /* |
| @@ -41,7 +41,7 @@ __do_compat_cache_op(unsigned long start, unsigned long end) · 40 unmodified lines | ||
| 41 | 41 | dsb(ish); |
| 42 | 42 | } |
| 43 | 43 | |
| 44 | − ret = __flush_cache_user_range(start, start + chunk); | |
| 44 | + ret = caches_clean_inval_user_pou(start, start + chunk); | |
| 45 | 45 | if (ret) |
| 46 | 46 | return ret; |
| 47 | 47 | |
| @@ -38,6 +38,7 @@ · 37 unmodified lines | ||
| 38 | 38 | #include <asm/extable.h> |
| 39 | 39 | #include <asm/insn.h> |
| 40 | 40 | #include <asm/kprobes.h> |
| 41 | +#include <asm/patching.h> | |
| 41 | 42 | #include <asm/traps.h> |
| 42 | 43 | #include <asm/smp.h> |
| 43 | 44 | #include <asm/stack_pointer.h> |
| @@ -45,11 +46,102 @@ · 1 unmodified line | ||
| 45 | 46 | #include <asm/system_misc.h> |
| 46 | 47 | #include <asm/sysreg.h> |
| 47 | 48 | |
| 48 | −static const char *handler[] = { | |
| 49 | − "Synchronous Abort", | |
| 50 | − "IRQ", | |
| 51 | − "FIQ", | |
| 52 | − "Error" | |
| 49 | +static bool __kprobes __check_eq(unsigned long pstate) | |
| 50 | +{ | |
| 51 | + return (pstate & PSR_Z_BIT) != 0; | |
| 52 | +} | |
| 53 | + | |
| 54 | +static bool __kprobes __check_ne(unsigned long pstate) | |
| 55 | +{ | |
| 56 | + return (pstate & PSR_Z_BIT) == 0; | |
| 57 | +} | |
| 58 | + | |
| 59 | +static bool __kprobes __check_cs(unsigned long pstate) | |
| 60 | +{ | |
| 61 | + return (pstate & PSR_C_BIT) != 0; | |
| 62 | +} | |
| 63 | + | |
| 64 | +static bool __kprobes __check_cc(unsigned long pstate) | |
| 65 | +{ | |
| 66 | + return (pstate & PSR_C_BIT) == 0; | |
| 67 | +} | |
| 68 | + | |
| 69 | +static bool __kprobes __check_mi(unsigned long pstate) | |
| 70 | +{ | |
| 71 | + return (pstate & PSR_N_BIT) != 0; | |
| 72 | +} | |
| 73 | + | |
| 74 | +static bool __kprobes __check_pl(unsigned long pstate) | |
| 75 | +{ | |
| 76 | + return (pstate & PSR_N_BIT) == 0; | |
| 77 | +} | |
| 78 | + | |
| 79 | +static bool __kprobes __check_vs(unsigned long pstate) | |
| 80 | +{ | |
| 81 | + return (pstate & PSR_V_BIT) != 0; | |
| 82 | +} | |
| 83 | + | |
| 84 | +static bool __kprobes __check_vc(unsigned long pstate) | |
| 85 | +{ | |
| 86 | + return (pstate & PSR_V_BIT) == 0; | |
| 87 | +} | |
| 88 | + | |
| 89 | +static bool __kprobes __check_hi(unsigned long pstate) | |
| 90 | +{ | |
| 91 | + pstate &= ~(pstate >> 1); /* PSR_C_BIT &= ~PSR_Z_BIT */ | |
| 92 | + return (pstate & PSR_C_BIT) != 0; | |
| 93 | +} | |
| 94 | + | |
| 95 | +static bool __kprobes __check_ls(unsigned long pstate) | |
| 96 | +{ | |
| 97 | + pstate &= ~(pstate >> 1); /* PSR_C_BIT &= ~PSR_Z_BIT */ | |
| 98 | + return (pstate & PSR_C_BIT) == 0; | |
| 99 | +} | |
| 100 | + | |
| 101 | +static bool __kprobes __check_ge(unsigned long pstate) | |
| 102 | +{ | |
| 103 | + pstate ^= (pstate << 3); /* PSR_N_BIT ^= PSR_V_BIT */ | |
| 104 | + return (pstate & PSR_N_BIT) == 0; | |
| 105 | +} | |
| 106 | + | |
| 107 | +static bool __kprobes __check_lt(unsigned long pstate) | |
| 108 | +{ | |
| 109 | + pstate ^= (pstate << 3); /* PSR_N_BIT ^= PSR_V_BIT */ | |
| 110 | + return (pstate & PSR_N_BIT) != 0; | |
| 111 | +} | |
| 112 | + | |
| 113 | +static bool __kprobes __check_gt(unsigned long pstate) | |
| 114 | +{ | |
| 115 | + /*PSR_N_BIT ^= PSR_V_BIT */ | |
| 116 | + unsigned long temp = pstate ^ (pstate << 3); | |
| 117 | + | |
| 118 | + temp |= (pstate << 1); /*PSR_N_BIT |= PSR_Z_BIT */ | |
| 119 | + return (temp & PSR_N_BIT) == 0; | |
| 120 | +} | |
| 121 | + | |
| 122 | +static bool __kprobes __check_le(unsigned long pstate) | |
| 123 | +{ | |
| 124 | + /*PSR_N_BIT ^= PSR_V_BIT */ | |
| 125 | + unsigned long temp = pstate ^ (pstate << 3); | |
| 126 | + | |
| 127 | + temp |= (pstate << 1); /*PSR_N_BIT |= PSR_Z_BIT */ | |
| 128 | + return (temp & PSR_N_BIT) != 0; | |
| 129 | +} | |
| 130 | + | |
| 131 | +static bool __kprobes __check_al(unsigned long pstate) | |
| 132 | +{ | |
| 133 | + return true; | |
| 134 | +} | |
| 135 | + | |
| 136 | +/* | |
| 137 | + * Note that the ARMv8 ARM calls condition code 0b1111 "nv", but states that | |
| 138 | + * it behaves identically to 0b1110 ("al"). | |
| 139 | + */ | |
| 140 | +pstate_check_t * const aarch32_opcode_cond_checks[16] = { | |
| 141 | + __check_eq, __check_ne, __check_cs, __check_cc, | |
| 142 | + __check_mi, __check_pl, __check_vs, __check_vc, | |
| 143 | + __check_hi, __check_ls, __check_ge, __check_lt, | |
| 144 | + __check_gt, __check_le, __check_al, __check_al | |
| 53 | 145 | }; |
| 54 | 146 | |
| 55 | 147 | int show_unhandled_signals = 0; |
| @@ -750,28 +842,9 @@ const char *esr_get_class_string(u32 esr) · 694 unmodified lines | ||
| 750 | 842 | return esr_class_str[ESR_ELx_EC(esr)]; |
| 751 | 843 | } |
| 752 | 844 | |
| 753 | −/* | |
| 754 | − * bad_mode handles the impossible case in the exception vector. This is always | |
| 755 | − * fatal. | |
| 756 | − */ | |
| 757 | −asmlinkage void notrace bad_mode(struct pt_regs *regs, int reason, unsigned int esr) | |
| 758 | −{ | |
| 759 | − arm64_enter_nmi(regs); | |
| 760 | − | |
| 761 | − console_verbose(); | |
| 762 | − | |
| 763 | − pr_crit("Bad mode in %s handler detected on CPU%d, code 0x%08x -- %s\n", | |
| 764 | − handler[reason], smp_processor_id(), esr, | |
| 765 | − esr_get_class_string(esr)); | |
| 766 | − | |
| 767 | − __show_regs(regs); | |
| 768 | − local_daif_mask(); | |
| 769 | − panic("bad mode"); | |
| 770 | −} | |
| 771 | − | |
| 772 | 845 | /* |
| 773 | 846 | * bad_el0_sync handles unexpected, but potentially recoverable synchronous |
| 774 | − * exceptions taken from EL0. Unlike bad_mode, this returns. | |
| 847 | + * exceptions taken from EL0. | |
| 775 | 848 | */ |
| 776 | 849 | void bad_el0_sync(struct pt_regs *regs, int reason, unsigned int esr) |
| 777 | 850 | { |
| @@ -789,15 +862,11 @@ void bad_el0_sync(struct pt_regs *regs, int reason, unsigned int esr) · 11 unmodified lines | ||
| 789 | 862 | DEFINE_PER_CPU(unsigned long [OVERFLOW_STACK_SIZE/sizeof(long)], overflow_stack) |
| 790 | 863 | __aligned(16); |
| 791 | 864 | |
| 792 | −asmlinkage void noinstr handle_bad_stack(struct pt_regs *regs) | |
| 865 | +void panic_bad_stack(struct pt_regs *regs, unsigned int esr, unsigned long far) | |
| 793 | 866 | { |
| 794 | 867 | unsigned long tsk_stk = (unsigned long)current->stack; |
| 795 | 868 | unsigned long irq_stk = (unsigned long)this_cpu_read(irq_stack_ptr); |
| 796 | 869 | unsigned long ovf_stk = (unsigned long)this_cpu_ptr(overflow_stack); |
| 797 | − unsigned int esr = read_sysreg(esr_el1); | |
| 798 | − unsigned long far = read_sysreg(far_el1); | |
| 799 | − | |
| 800 | − arm64_enter_nmi(regs); | |
| 801 | 870 | |
| 802 | 871 | console_verbose(); |
| 803 | 872 | pr_emerg("Insufficient stack space to handle exception!"); |
| @@ -870,15 +939,11 @@ bool arm64_is_fatal_ras_serror(struct pt_regs *regs, unsigned int esr) · 66 unmodified lines | ||
| 870 | 939 | } |
| 871 | 940 | } |
| 872 | 941 | |
| 873 | −asmlinkage void noinstr do_serror(struct pt_regs *regs, unsigned int esr) | |
| 942 | +void do_serror(struct pt_regs *regs, unsigned int esr) | |
| 874 | 943 | { |
| 875 | − arm64_enter_nmi(regs); | |
| 876 | − | |
| 877 | 944 | /* non-RAS errors are not containable */ |
| 878 | 945 | if (!arm64_is_ras_serror(esr) || arm64_is_fatal_ras_serror(regs, esr)) |
| 879 | 946 | arm64_serror_panic(regs, esr); |
| 880 | − | |
| 881 | − arm64_exit_nmi(regs); | |
| 882 | 947 | } |
| 883 | 948 | |
| 884 | 949 | /* GENERIC_BUG traps */ |
| @@ -692,6 +692,15 @@ static void check_vcpu_requests(struct kvm_vcpu *vcpu) · 691 unmodified lines | ||
| 692 | 692 | } |
| 693 | 693 | } |
| 694 | 694 | |
| 695 | +static bool vcpu_mode_is_bad_32bit(struct kvm_vcpu *vcpu) | |
| 696 | +{ | |
| 697 | + if (likely(!vcpu_mode_is_32bit(vcpu))) | |
| 698 | + return false; | |
| 699 | + | |
| 700 | + return !system_supports_32bit_el0() || | |
| 701 | + static_branch_unlikely(&arm64_mismatched_32bit_el0); | |
| 702 | +} | |
| 703 | + | |
| 695 | 704 | /** |
| 696 | 705 | * kvm_arch_vcpu_ioctl_run - the main VCPU run function to execute guest code |
| 697 | 706 | * @vcpu: The VCPU pointer |
| @@ -877,7 +886,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) · 179 unmodified lines | ||
| 877 | 886 | * with the asymmetric AArch32 case), return to userspace with |
| 878 | 887 | * a fatal error. |
| 879 | 888 | */ |
| 880 | − if (!system_supports_32bit_el0() && vcpu_mode_is_32bit(vcpu)) { | |
| 889 | + if (vcpu_mode_is_bad_32bit(vcpu)) { | |
| 881 | 890 | /* |
| 882 | 891 | * As we have caught the guest red-handed, decide that |
| 883 | 892 | * it isn't fit for purpose anymore by making the vcpu |
| @@ -1078,7 +1087,7 @@ static int kvm_arch_vcpu_ioctl_vcpu_init(struct kvm_vcpu *vcpu, · 194 unmodified lines | ||
| 1078 | 1087 | if (!cpus_have_final_cap(ARM64_HAS_STAGE2_FWB)) |
| 1079 | 1088 | stage2_unmap_vm(vcpu->kvm); |
| 1080 | 1089 | else |
| 1081 | − __flush_icache_all(); | |
| 1090 | + icache_inval_all_pou(); | |
| 1082 | 1091 | } |
| 1083 | 1092 | |
| 1084 | 1093 | vcpu_reset_hcr(vcpu); |
| @@ -7,7 +7,7 @@ · 6 unmodified lines | ||
| 7 | 7 | #include <asm/assembler.h> |
| 8 | 8 | #include <asm/alternative.h> |
| 9 | 9 | |
| 10 | −SYM_FUNC_START_PI(__flush_dcache_area) | |
| 10 | +SYM_FUNC_START_PI(dcache_clean_inval_poc) | |
| 11 | 11 | dcache_by_line_op civac, sy, x0, x1, x2, x3 |
| 12 | 12 | ret |
| 13 | −SYM_FUNC_END_PI(__flush_dcache_area) | |
| 13 | +SYM_FUNC_END_PI(dcache_clean_inval_poc) | |
| @@ -134,7 +134,8 @@ static void update_nvhe_init_params(void) · 133 unmodified lines | ||
| 134 | 134 | for (i = 0; i < hyp_nr_cpus; i++) { |
| 135 | 135 | params = per_cpu_ptr(&kvm_init_params, i); |
| 136 | 136 | params->pgd_pa = __hyp_pa(pkvm_pgtable.pgd); |
| 137 | − __flush_dcache_area(params, sizeof(*params)); | |
| 137 | + dcache_clean_inval_poc((unsigned long)params, | |
| 138 | + (unsigned long)params + sizeof(*params)); | |
| 138 | 139 | } |
| 139 | 140 | } |
| 140 | 141 | |
| @@ -104,7 +104,7 @@ void __kvm_tlb_flush_vmid_ipa(struct kvm_s2_mmu *mmu, · 103 unmodified lines | ||
| 104 | 104 | * you should be running with VHE enabled. |
| 105 | 105 | */ |
| 106 | 106 | if (icache_is_vpipt()) |
| 107 | − __flush_icache_all(); | |
| 107 | + icache_inval_all_pou(); | |
| 108 | 108 | |
| 109 | 109 | __tlb_switch_to_host(&cxt); |
| 110 | 110 | } |
| @@ -839,8 +839,11 @@ static int stage2_unmap_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep, · 838 unmodified lines | ||
| 839 | 839 | stage2_put_pte(ptep, mmu, addr, level, mm_ops); |
| 840 | 840 | |
| 841 | 841 | if (need_flush) { |
| 842 | − __flush_dcache_area(kvm_pte_follow(pte, mm_ops), | |
| 843 | − kvm_granule_size(level)); | |
| 842 | + kvm_pte_t *pte_follow = kvm_pte_follow(pte, mm_ops); | |
| 843 | + | |
| 844 | + dcache_clean_inval_poc((unsigned long)pte_follow, | |
| 845 | + (unsigned long)pte_follow + | |
| 846 | + kvm_granule_size(level)); | |
| 844 | 847 | } |
| 845 | 848 | |
| 846 | 849 | if (childp) |
| @@ -988,11 +991,15 @@ static int stage2_flush_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep, · 141 unmodified lines | ||
| 988 | 991 | struct kvm_pgtable *pgt = arg; |
| 989 | 992 | struct kvm_pgtable_mm_ops *mm_ops = pgt->mm_ops; |
| 990 | 993 | kvm_pte_t pte = *ptep; |
| 994 | + kvm_pte_t *pte_follow; | |
| 991 | 995 | |
| 992 | 996 | if (!kvm_pte_valid(pte) || !stage2_pte_cacheable(pgt, pte)) |
| 993 | 997 | return 0; |
| 994 | 998 | |
| 995 | − __flush_dcache_area(kvm_pte_follow(pte, mm_ops), kvm_granule_size(level)); | |
| 999 | + pte_follow = kvm_pte_follow(pte, mm_ops); | |
| 1000 | + dcache_clean_inval_poc((unsigned long)pte_follow, | |
| 1001 | + (unsigned long)pte_follow + | |
| 1002 | + kvm_granule_size(level)); | |
| 996 | 1003 | return 0; |
| 997 | 1004 | } |
| 998 | 1005 | |
| @@ -1,7 +1,7 @@ | ||
| 1 | 1 | # SPDX-License-Identifier: GPL-2.0 |
| 2 | 2 | lib-y := clear_user.o delay.o copy_from_user.o \ |
| 3 | 3 | copy_to_user.o copy_in_user.o copy_page.o \ |
| 4 | − clear_page.o csum.o memchr.o memcpy.o memmove.o \ | |
| 4 | + clear_page.o csum.o insn.o memchr.o memcpy.o \ | |
| 5 | 5 | memset.o memcmp.o strcmp.o strncmp.o strlen.o \ |
| 6 | 6 | strnlen.o strchr.o strrchr.o tishift.o |
| 7 | 7 | |
| @@ -18,3 +18,5 @@ obj-$(CONFIG_CRC32) += crc32.o · 10 unmodified lines | ||
| 18 | 18 | obj-$(CONFIG_FUNCTION_ERROR_INJECTION) += error-inject.o |
| 19 | 19 | |
| 20 | 20 | obj-$(CONFIG_ARM64_MTE) += mte.o |
| 21 | + | |
| 22 | +obj-$(CONFIG_KASAN_SW_TAGS) += kasan_sw_tags.o | |
| @@ -1,12 +1,9 @@ | ||
| 1 | 1 | /* SPDX-License-Identifier: GPL-2.0-only */ |
| 2 | 2 | /* |
| 3 | − * Based on arch/arm/lib/clear_user.S | |
| 4 | − * | |
| 5 | − * Copyright (C) 2012 ARM Ltd. | |
| 3 | + * Copyright (C) 2021 Arm Ltd. | |
| 6 | 4 | */ |
| 7 | −#include <linux/linkage.h> | |
| 8 | 5 | |
| 9 | −#include <asm/asm-uaccess.h> | |
| 6 | +#include <linux/linkage.h> | |
| 10 | 7 | #include <asm/assembler.h> |
| 11 | 8 | |
| 12 | 9 | .text |
| @@ -19,25 +16,33 @@ · 6 unmodified lines | ||
| 19 | 16 | * |
| 20 | 17 | * Alignment fixed up by hardware. |
| 21 | 18 | */ |
| 19 | + | |
| 20 | + .p2align 4 | |
| 21 | + // Alignment is for the loop, but since the prologue (including BTI) | |
| 22 | + // is also 16 bytes we can keep any padding outside the function | |
| 22 | 23 | SYM_FUNC_START(__arch_clear_user) |
| 23 | − mov x2, x1 // save the size for fixup return | |
| 24 | + add x2, x0, x1 | |
| 24 | 25 | subs x1, x1, #8 |
| 25 | 26 | b.mi 2f |
| 26 | 27 | 1: |
| 27 | −user_ldst 9f, sttr, xzr, x0, 8 | |
| 28 | +USER(9f, sttr xzr, [x0]) | |
| 29 | + add x0, x0, #8 | |
| 28 | 30 | subs x1, x1, #8 |
| 29 | − b.pl 1b | |
| 30 | −2: adds x1, x1, #4 | |
| 31 | − b.mi 3f | |
| 32 | −user_ldst 9f, sttr, wzr, x0, 4 | |
| 33 | − sub x1, x1, #4 | |
| 34 | −3: adds x1, x1, #2 | |
| 35 | − b.mi 4f | |
| 36 | −user_ldst 9f, sttrh, wzr, x0, 2 | |
| 37 | − sub x1, x1, #2 | |
| 38 | −4: adds x1, x1, #1 | |
| 39 | − b.mi 5f | |
| 40 | −user_ldst 9f, sttrb, wzr, x0, 0 | |
| 31 | + b.hi 1b | |
| 32 | +USER(9f, sttr xzr, [x2, #-8]) | |
| 33 | + mov x0, #0 | |
| 34 | + ret | |
| 35 | + | |
| 36 | +2: tbz x1, #2, 3f | |
| 37 | +USER(9f, sttr wzr, [x0]) | |
| 38 | +USER(8f, sttr wzr, [x2, #-4]) | |
| 39 | + mov x0, #0 | |
| 40 | + ret | |
| 41 | + | |
| 42 | +3: tbz x1, #1, 4f | |
| 43 | +USER(9f, sttrh wzr, [x0]) | |
| 44 | +4: tbz x1, #0, 5f | |
| 45 | +USER(7f, sttrb wzr, [x2, #-1]) | |
| 41 | 46 | 5: mov x0, #0 |
| 42 | 47 | ret |
| 43 | 48 | SYM_FUNC_END(__arch_clear_user) |
| @@ -45,6 +50,8 @@ EXPORT_SYMBOL(__arch_clear_user) · 1 unmodified line | ||
| 45 | 50 | |
| 46 | 51 | .section .fixup,"ax" |
| 47 | 52 | .align 2 |
| 48 | −9: mov x0, x2 // return the original size | |
| 53 | +7: sub x0, x2, #5 // Adjust for faulting on the final byte... | |
| 54 | +8: add x0, x0, #4 // ...or the second word of the 4-7 byte case | |
| 55 | +9: sub x0, x2, x0 | |
| 49 | 56 | ret |
| 50 | 57 | .previous |
| @@ -7,21 +7,14 @@ · 6 unmodified lines | ||
| 7 | 7 | */ |
| 8 | 8 | #include <linux/bitops.h> |
| 9 | 9 | #include <linux/bug.h> |
| 10 | −#include <linux/compiler.h> | |
| 11 | −#include <linux/kernel.h> | |
| 12 | −#include <linux/mm.h> | |
| 13 | −#include <linux/smp.h> | |
| 14 | −#include <linux/spinlock.h> | |
| 15 | −#include <linux/stop_machine.h> | |
| 10 | +#include <linux/printk.h> | |
| 11 | +#include <linux/sizes.h> | |
| 16 | 12 | #include <linux/types.h> |
| 17 | −#include <linux/uaccess.h> | |
| 18 | 13 | |
| 19 | −#include <asm/cacheflush.h> | |
| 20 | 14 | #include <asm/debug-monitors.h> |
| 21 | −#include <asm/fixmap.h> | |
| 15 | +#include <asm/errno.h> | |
| 22 | 16 | #include <asm/insn.h> |
| 23 | 17 | #include <asm/kprobes.h> |
| 24 | −#include <asm/sections.h> | |
| 25 | 18 | |
| 26 | 19 | #define AARCH64_INSN_SF_BIT BIT(31) |
| 27 | 20 | #define AARCH64_INSN_N_BIT BIT(22) |
| @@ -30,7 +23,7 @@ · 2 unmodified lines | ||
| 30 | 23 | static const int aarch64_insn_encoding_class[] = { |
| 31 | 24 | AARCH64_INSN_CLS_UNKNOWN, |
| 32 | 25 | AARCH64_INSN_CLS_UNKNOWN, |
| 33 | − AARCH64_INSN_CLS_UNKNOWN, | |
| 26 | + AARCH64_INSN_CLS_SVE, | |
| 34 | 27 | AARCH64_INSN_CLS_UNKNOWN, |
| 35 | 28 | AARCH64_INSN_CLS_LDST, |
| 36 | 29 | AARCH64_INSN_CLS_DP_REG, |
| @@ -83,81 +76,6 @@ bool aarch64_insn_is_branch_imm(u32 insn) · 46 unmodified lines | ||
| 83 | 76 | aarch64_insn_is_bcond(insn)); |
| 84 | 77 | } |
| 85 | 78 | |
| 86 | −static DEFINE_RAW_SPINLOCK(patch_lock); | |
| 87 | − | |
| 88 | −static bool is_exit_text(unsigned long addr) | |
| 89 | −{ | |
| 90 | − /* discarded with init text/data */ | |
| 91 | − return system_state < SYSTEM_RUNNING && | |
| 92 | − addr >= (unsigned long)__exittext_begin && | |
| 93 | − addr < (unsigned long)__exittext_end; | |
| 94 | −} | |
| 95 | − | |
| 96 | −static bool is_image_text(unsigned long addr) | |
| 97 | −{ | |
| 98 | − return core_kernel_text(addr) || is_exit_text(addr); | |
| 99 | −} | |
| 100 | − | |
| 101 | −static void __kprobes *patch_map(void *addr, int fixmap) | |
| 102 | −{ | |
| 103 | − unsigned long uintaddr = (uintptr_t) addr; | |
| 104 | − bool image = is_image_text(uintaddr); | |
| 105 | − struct page *page; | |
| 106 | − | |
| 107 | − if (image) | |
| 108 | − page = phys_to_page(__pa_symbol(addr)); | |
| 109 | − else if (IS_ENABLED(CONFIG_STRICT_MODULE_RWX)) | |
| 110 | − page = vmalloc_to_page(addr); | |
| 111 | − else | |
| 112 | − return addr; | |
| 113 | − | |
| 114 | − BUG_ON(!page); | |
| 115 | − return (void *)set_fixmap_offset(fixmap, page_to_phys(page) + | |
| 116 | − (uintaddr & ~PAGE_MASK)); | |
| 117 | −} | |
| 118 | − | |
| 119 | −static void __kprobes patch_unmap(int fixmap) | |
| 120 | −{ | |
| 121 | − clear_fixmap(fixmap); | |
| 122 | −} | |
| 123 | −/* | |
| 124 | − * In ARMv8-A, A64 instructions have a fixed length of 32 bits and are always | |
| 125 | − * little-endian. | |
| 126 | − */ | |
| 127 | −int __kprobes aarch64_insn_read(void *addr, u32 *insnp) | |
| 128 | −{ | |
| 129 | − int ret; | |
| 130 | − __le32 val; | |
| 131 | − | |
| 132 | − ret = copy_from_kernel_nofault(&val, addr, AARCH64_INSN_SIZE); | |
| 133 | − if (!ret) | |
| 134 | − *insnp = le32_to_cpu(val); | |
| 135 | − | |
| 136 | − return ret; | |
| 137 | −} | |
| 138 | − | |
| 139 | −static int __kprobes __aarch64_insn_write(void *addr, __le32 insn) | |
| 140 | −{ | |
| 141 | − void *waddr = addr; | |
| 142 | − unsigned long flags = 0; | |
| 143 | − int ret; | |
| 144 | − | |
| 145 | − raw_spin_lock_irqsave(&patch_lock, flags); | |
| 146 | − waddr = patch_map(addr, FIX_TEXT_POKE0); | |
| 147 | − | |
| 148 | − ret = copy_to_kernel_nofault(waddr, &insn, AARCH64_INSN_SIZE); | |
| 149 | − | |
| 150 | − patch_unmap(FIX_TEXT_POKE0); | |
| 151 | − raw_spin_unlock_irqrestore(&patch_lock, flags); | |
| 152 | − | |
| 153 | − return ret; | |
| 154 | −} | |
| 155 | − | |
| 156 | −int __kprobes aarch64_insn_write(void *addr, u32 insn) | |
| 157 | −{ | |
| 158 | − return __aarch64_insn_write(addr, cpu_to_le32(insn)); | |
| 159 | −} | |
| 160 | − | |
| 161 | 79 | bool __kprobes aarch64_insn_uses_literal(u32 insn) |
| 162 | 80 | { |
| 163 | 81 | /* ldr/ldrsw (literal), prfm */ |
| @@ -187,67 +105,6 @@ bool __kprobes aarch64_insn_is_branch(u32 insn) · 23 unmodified lines | ||
| 187 | 105 | aarch64_insn_is_bcond(insn); |
| 188 | 106 | } |
| 189 | 107 | |
| 190 | −int __kprobes aarch64_insn_patch_text_nosync(void *addr, u32 insn) | |
| 191 | −{ | |
| 192 | − u32 *tp = addr; | |
| 193 | − int ret; | |
| 194 | − | |
| 195 | − /* A64 instructions must be word aligned */ | |
| 196 | − if ((uintptr_t)tp & 0x3) | |
| 197 | − return -EINVAL; | |
| 198 | − | |
| 199 | − ret = aarch64_insn_write(tp, insn); | |
| 200 | − if (ret == 0) | |
| 201 | − __flush_icache_range((uintptr_t)tp, | |
| 202 | − (uintptr_t)tp + AARCH64_INSN_SIZE); | |
| 203 | − | |
| 204 | − return ret; | |
| 205 | −} | |
| 206 | − | |
| 207 | −struct aarch64_insn_patch { | |
| 208 | − void **text_addrs; | |
| 209 | − u32 *new_insns; | |
| 210 | − int insn_cnt; | |
| 211 | − atomic_t cpu_count; | |
| 212 | −}; | |
| 213 | − | |
| 214 | −static int __kprobes aarch64_insn_patch_text_cb(void *arg) | |
| 215 | −{ | |
| 216 | − int i, ret = 0; | |
| 217 | − struct aarch64_insn_patch *pp = arg; | |
| 218 | − | |
| 219 | − /* The first CPU becomes master */ | |
| 220 | − if (atomic_inc_return(&pp->cpu_count) == 1) { | |
| 221 | − for (i = 0; ret == 0 && i < pp->insn_cnt; i++) | |
| 222 | − ret = aarch64_insn_patch_text_nosync(pp->text_addrs[i], | |
| 223 | − pp->new_insns[i]); | |
| 224 | − /* Notify other processors with an additional increment. */ | |
| 225 | − atomic_inc(&pp->cpu_count); | |
| 226 | − } else { | |
| 227 | − while (atomic_read(&pp->cpu_count) <= num_online_cpus()) | |
| 228 | − cpu_relax(); | |
| 229 | − isb(); | |
| 230 | − } | |
| 231 | − | |
| 232 | − return ret; | |
| 233 | −} | |
| 234 | − | |
| 235 | −int __kprobes aarch64_insn_patch_text(void *addrs[], u32 insns[], int cnt) | |
| 236 | −{ | |
| 237 | − struct aarch64_insn_patch patch = { | |
| 238 | − .text_addrs = addrs, | |
| 239 | − .new_insns = insns, | |
| 240 | − .insn_cnt = cnt, | |
| 241 | − .cpu_count = ATOMIC_INIT(0), | |
| 242 | − }; | |
| 243 | − | |
| 244 | − if (cnt <= 0) | |
| 245 | − return -EINVAL; | |
| 246 | − | |
| 247 | − return stop_machine_cpuslocked(aarch64_insn_patch_text_cb, &patch, | |
| 248 | − cpu_online_mask); | |
| 249 | −} | |
| 250 | − | |
| 251 | 108 | static int __kprobes aarch64_get_imm_shift_mask(enum aarch64_insn_imm_type type, |
| 252 | 109 | u32 *maskp, int *shiftp) |
| 253 | 110 | { |
| @@ -1432,104 +1289,6 @@ u32 aarch32_insn_mcr_extract_crm(u32 insn) · 1178 unmodified lines | ||
| 1432 | 1289 | return insn & CRM_MASK; |
| 1433 | 1290 | } |
| 1434 | 1291 | |
| 1435 | −static bool __kprobes __check_eq(unsigned long pstate) | |
| 1436 | −{ | |
| 1437 | − return (pstate & PSR_Z_BIT) != 0; | |
| 1438 | −} | |
| 1439 | − | |
| 1440 | −static bool __kprobes __check_ne(unsigned long pstate) | |
| 1441 | −{ | |
| 1442 | − return (pstate & PSR_Z_BIT) == 0; | |
| 1443 | −} | |
| 1444 | − | |
| 1445 | −static bool __kprobes __check_cs(unsigned long pstate) | |
| 1446 | −{ | |
| 1447 | − return (pstate & PSR_C_BIT) != 0; | |
| 1448 | −} | |
| 1449 | − | |
| 1450 | −static bool __kprobes __check_cc(unsigned long pstate) | |
| 1451 | −{ | |
| 1452 | − return (pstate & PSR_C_BIT) == 0; | |
| 1453 | −} | |
| 1454 | − | |
| 1455 | −static bool __kprobes __check_mi(unsigned long pstate) | |
| 1456 | −{ | |
| 1457 | − return (pstate & PSR_N_BIT) != 0; | |
| 1458 | −} | |
| 1459 | − | |
| 1460 | −static bool __kprobes __check_pl(unsigned long pstate) | |
| 1461 | −{ | |
| 1462 | − return (pstate & PSR_N_BIT) == 0; | |
| 1463 | −} | |
| 1464 | − | |
| 1465 | −static bool __kprobes __check_vs(unsigned long pstate) | |
| 1466 | −{ | |
| 1467 | − return (pstate & PSR_V_BIT) != 0; | |
| 1468 | −} | |
| 1469 | − | |
| 1470 | −static bool __kprobes __check_vc(unsigned long pstate) | |
| 1471 | −{ | |
| 1472 | − return (pstate & PSR_V_BIT) == 0; | |
| 1473 | −} | |
| 1474 | − | |
| 1475 | −static bool __kprobes __check_hi(unsigned long pstate) | |
| 1476 | −{ | |
| 1477 | − pstate &= ~(pstate >> 1); /* PSR_C_BIT &= ~PSR_Z_BIT */ | |
| 1478 | − return (pstate & PSR_C_BIT) != 0; | |
| 1479 | −} | |
| 1480 | − | |
| 1481 | −static bool __kprobes __check_ls(unsigned long pstate) | |
| 1482 | −{ | |
| 1483 | − pstate &= ~(pstate >> 1); /* PSR_C_BIT &= ~PSR_Z_BIT */ | |
| 1484 | − return (pstate & PSR_C_BIT) == 0; | |
| 1485 | −} | |
| 1486 | − | |
| 1487 | −static bool __kprobes __check_ge(unsigned long pstate) | |
| 1488 | −{ | |
| 1489 | − pstate ^= (pstate << 3); /* PSR_N_BIT ^= PSR_V_BIT */ | |
| 1490 | − return (pstate & PSR_N_BIT) == 0; | |
| 1491 | −} | |
| 1492 | − | |
| 1493 | −static bool __kprobes __check_lt(unsigned long pstate) | |
| 1494 | −{ | |
| 1495 | − pstate ^= (pstate << 3); /* PSR_N_BIT ^= PSR_V_BIT */ | |
| 1496 | − return (pstate & PSR_N_BIT) != 0; | |
| 1497 | −} | |
| 1498 | − | |
| 1499 | −static bool __kprobes __check_gt(unsigned long pstate) | |
| 1500 | −{ | |
| 1501 | − /*PSR_N_BIT ^= PSR_V_BIT */ | |
| 1502 | − unsigned long temp = pstate ^ (pstate << 3); | |
| 1503 | − | |
| 1504 | − temp |= (pstate << 1); /*PSR_N_BIT |= PSR_Z_BIT */ | |
| 1505 | − return (temp & PSR_N_BIT) == 0; | |
| 1506 | −} | |
| 1507 | − | |
| 1508 | −static bool __kprobes __check_le(unsigned long pstate) | |
| 1509 | −{ | |
| 1510 | − /*PSR_N_BIT ^= PSR_V_BIT */ | |
| 1511 | − unsigned long temp = pstate ^ (pstate << 3); | |
| 1512 | − | |
| 1513 | − temp |= (pstate << 1); /*PSR_N_BIT |= PSR_Z_BIT */ | |
| 1514 | − return (temp & PSR_N_BIT) != 0; | |
| 1515 | −} | |
| 1516 | − | |
| 1517 | −static bool __kprobes __check_al(unsigned long pstate) | |
| 1518 | −{ | |
| 1519 | − return true; | |
| 1520 | −} | |
| 1521 | − | |
| 1522 | −/* | |
| 1523 | − * Note that the ARMv8 ARM calls condition code 0b1111 "nv", but states that | |
| 1524 | − * it behaves identically to 0b1110 ("al"). | |
| 1525 | − */ | |
| 1526 | −pstate_check_t * const aarch32_opcode_cond_checks[16] = { | |
| 1527 | − __check_eq, __check_ne, __check_cs, __check_cc, | |
| 1528 | − __check_mi, __check_pl, __check_vs, __check_vc, | |
| 1529 | − __check_hi, __check_ls, __check_ge, __check_lt, | |
| 1530 | − __check_gt, __check_le, __check_al, __check_al | |
| 1531 | −}; | |
| 1532 | − | |
| 1533 | 1292 | static bool range_of_ones(u64 val) |
| 1534 | 1293 | { |
| 1535 | 1294 | /* Doesn't handle full ones or full zeroes */ |
| @@ -0,0 +1,76 @@ | ||
| 1 | +/* SPDX-License-Identifier: GPL-2.0-only */ | |
| 2 | +/* | |
| 3 | + * Copyright (C) 2020 Google LLC | |
| 4 | + */ | |
| 5 | + | |
| 6 | +#include <linux/linkage.h> | |
| 7 | +#include <asm/assembler.h> | |
| 8 | + | |
| 9 | +/* | |
| 10 | + * Report a tag mismatch detected by tag-based KASAN. | |
| 11 | + * | |
| 12 | + * A compiler-generated thunk calls this with a non-AAPCS calling | |
| 13 | + * convention. Upon entry to this function, registers are as follows: | |
| 14 | + * | |
| 15 | + * x0: fault address (see below for restore) | |
| 16 | + * x1: fault description (see below for restore) | |
| 17 | + * x2 to x15: callee-saved | |
| 18 | + * x16 to x17: safe to clobber | |
| 19 | + * x18 to x30: callee-saved | |
| 20 | + * sp: pre-decremented by 256 bytes (see below for restore) | |
| 21 | + * | |
| 22 | + * The caller has decremented the SP by 256 bytes, and created a | |
| 23 | + * structure on the stack as follows: | |
| 24 | + * | |
| 25 | + * sp + 0..15: x0 and x1 to be restored | |
| 26 | + * sp + 16..231: free for use | |
| 27 | + * sp + 232..247: x29 and x30 (same as in GPRs) | |
| 28 | + * sp + 248..255: free for use | |
| 29 | + * | |
| 30 | + * Note that this is not a struct pt_regs. | |
| 31 | + * | |
| 32 | + * To call a regular AAPCS function we must save x2 to x15 (which we can | |
| 33 | + * store in the gaps), and create a frame record (for which we can use | |
| 34 | + * x29 and x30 spilled by the caller as those match the GPRs). | |
| 35 | + * | |
| 36 | + * The caller expects x0 and x1 to be restored from the structure, and | |
| 37 | + * for the structure to be removed from the stack (i.e. the SP must be | |
| 38 | + * incremented by 256 prior to return). | |
| 39 | + */ | |
| 40 | +SYM_CODE_START(__hwasan_tag_mismatch) | |
| 41 | +#ifdef BTI_C | |
| 42 | + BTI_C | |
| 43 | +#endif | |
| 44 | + add x29, sp, #232 | |
| 45 | + stp x2, x3, [sp, #8 * 2] | |
| 46 | + stp x4, x5, [sp, #8 * 4] | |
| 47 | + stp x6, x7, [sp, #8 * 6] | |
| 48 | + stp x8, x9, [sp, #8 * 8] | |
| 49 | + stp x10, x11, [sp, #8 * 10] | |
| 50 | + stp x12, x13, [sp, #8 * 12] | |
| 51 | + stp x14, x15, [sp, #8 * 14] | |
| 52 | +#ifndef CONFIG_SHADOW_CALL_STACK | |
| 53 | + str x18, [sp, #8 * 18] | |
| 54 | +#endif | |
| 55 | + | |
| 56 | + mov x2, x30 | |
| 57 | + bl kasan_tag_mismatch | |
| 58 | + | |
| 59 | + ldp x0, x1, [sp] | |
| 60 | + ldp x2, x3, [sp, #8 * 2] | |
| 61 | + ldp x4, x5, [sp, #8 * 4] | |
| 62 | + ldp x6, x7, [sp, #8 * 6] | |
| 63 | + ldp x8, x9, [sp, #8 * 8] | |
| 64 | + ldp x10, x11, [sp, #8 * 10] | |
| 65 | + ldp x12, x13, [sp, #8 * 12] | |
| 66 | + ldp x14, x15, [sp, #8 * 14] | |
| 67 | +#ifndef CONFIG_SHADOW_CALL_STACK | |
| 68 | + ldr x18, [sp, #8 * 18] | |
| 69 | +#endif | |
| 70 | + ldp x29, x30, [sp, #8 * 29] | |
| 71 | + | |
| 72 | + /* remove the structure from the stack */ | |
| 73 | + add sp, sp, #256 | |
| 74 | + ret | |
| 75 | +SYM_CODE_END(__hwasan_tag_mismatch) | |
| 76 | +EXPORT_SYMBOL(__hwasan_tag_mismatch) | |
| @@ -1,9 +1,6 @@ | ||
| 1 | 1 | /* SPDX-License-Identifier: GPL-2.0-only */ |
| 2 | 2 | /* |
| 3 | − * Based on arch/arm/lib/memchr.S | |
| 4 | − * | |
| 5 | − * Copyright (C) 1995-2000 Russell King | |
| 6 | − * Copyright (C) 2013 ARM Ltd. | |
| 3 | + * Copyright (C) 2021 Arm Ltd. | |
| 7 | 4 | */ |
| 8 | 5 | |
| 9 | 6 | #include <linux/linkage.h> |
| @@ -19,16 +16,60 @@ · 9 unmodified lines | ||
| 19 | 16 | * Returns: |
| 20 | 17 | * x0 - address of first occurrence of 'c' or 0 |
| 21 | 18 | */ |
| 19 | + | |
| 20 | +#define L(label) .L ## label | |
| 21 | + | |
| 22 | +#define REP8_01 0x0101010101010101 | |
| 23 | +#define REP8_7f 0x7f7f7f7f7f7f7f7f | |
| 24 | + | |
| 25 | +#define srcin x0 | |
| 26 | +#define chrin w1 | |
| 27 | +#define cntin x2 | |
| 28 | + | |
| 29 | +#define result x0 | |
| 30 | + | |
| 31 | +#define wordcnt x3 | |
| 32 | +#define rep01 x4 | |
| 33 | +#define repchr x5 | |
| 34 | +#define cur_word x6 | |
| 35 | +#define cur_byte w6 | |
| 36 | +#define tmp x7 | |
| 37 | +#define tmp2 x8 | |
| 38 | + | |
| 39 | + .p2align 4 | |
| 40 | + nop | |
| 22 | 41 | SYM_FUNC_START_WEAK_PI(memchr) |
| 23 | − and w1, w1, #0xff | |
| 24 | −1: subs x2, x2, #1 | |
| 25 | − b.mi 2f | |
| 26 | − ldrb w3, [x0], #1 | |
| 27 | − cmp w3, w1 | |
| 28 | − b.ne 1b | |
| 29 | − sub x0, x0, #1 | |
| 42 | + and chrin, chrin, #0xff | |
| 43 | + lsr wordcnt, cntin, #3 | |
| 44 | + cbz wordcnt, L(byte_loop) | |
| 45 | + mov rep01, #REP8_01 | |
| 46 | + mul repchr, x1, rep01 | |
| 47 | + and cntin, cntin, #7 | |
| 48 | +L(word_loop): | |
| 49 | + ldr cur_word, [srcin], #8 | |
| 50 | + sub wordcnt, wordcnt, #1 | |
| 51 | + eor cur_word, cur_word, repchr | |
| 52 | + sub tmp, cur_word, rep01 | |
| 53 | + orr tmp2, cur_word, #REP8_7f | |
| 54 | + bics tmp, tmp, tmp2 | |
| 55 | + b.ne L(found_word) | |
| 56 | + cbnz wordcnt, L(word_loop) | |
| 57 | +L(byte_loop): | |
| 58 | + cbz cntin, L(not_found) | |
| 59 | + ldrb cur_byte, [srcin], #1 | |
| 60 | + sub cntin, cntin, #1 | |
| 61 | + cmp cur_byte, chrin | |
| 62 | + b.ne L(byte_loop) | |
| 63 | + sub srcin, srcin, #1 | |
| 64 | + ret | |
| 65 | +L(found_word): | |
| 66 | +CPU_LE( rev tmp, tmp) | |
| 67 | + clz tmp, tmp | |
| 68 | + sub tmp, tmp, #64 | |
| 69 | + add result, srcin, tmp, asr #3 | |
| 30 | 70 | ret |
| 31 | −2: mov x0, #0 | |
| 71 | +L(not_found): | |
| 72 | + mov result, #0 | |
| 32 | 73 | ret |
| 33 | 74 | SYM_FUNC_END_PI(memchr) |
| 34 | 75 | EXPORT_SYMBOL_NOKASAN(memchr) |
| @@ -1,247 +1,139 @@ | ||
| 1 | 1 | /* SPDX-License-Identifier: GPL-2.0-only */ |
| 2 | 2 | /* |
| 3 | − * Copyright (C) 2013 ARM Ltd. | |
| 4 | − * Copyright (C) 2013 Linaro. | |
| 3 | + * Copyright (c) 2013-2021, Arm Limited. | |
| 5 | 4 | * |
| 6 | − * This code is based on glibc cortex strings work originally authored by Linaro | |
| 7 | − * be found @ | |
| 8 | − * | |
| 9 | − * http://bazaar.launchpad.net/~linaro-toolchain-dev/cortex-strings/trunk/ | |
| 10 | − * files/head:/src/aarch64/ | |
| 5 | + * Adapted from the original at: | |
| 6 | + * https://github.com/ARM-software/optimized-routines/blob/e823e3abf5f89ecb/string/aarch64/memcmp.S | |
| 11 | 7 | */ |
| 12 | 8 | |
| 13 | 9 | #include <linux/linkage.h> |
| 14 | 10 | #include <asm/assembler.h> |
| 15 | 11 | |
| 16 | −/* | |
| 17 | −* compare memory areas(when two memory areas' offset are different, | |
| 18 | −* alignment handled by the hardware) | |
| 19 | −* | |
| 20 | −* Parameters: | |
| 21 | −* x0 - const memory area 1 pointer | |
| 22 | −* x1 - const memory area 2 pointer | |
| 23 | −* x2 - the maximal compare byte length | |
| 24 | −* Returns: | |
| 25 | −* x0 - a compare result, maybe less than, equal to, or greater than ZERO | |
| 26 | −*/ | |
| 12 | +/* Assumptions: | |
| 13 | + * | |
| 14 | + * ARMv8-a, AArch64, unaligned accesses. | |
| 15 | + */ | |
| 16 | + | |
| 17 | +#define L(label) .L ## label | |
| 27 | 18 | |
| 28 | 19 | /* Parameters and result. */ |
| 29 | −src1 .req x0 | |
| 30 | −src2 .req x1 | |
| 31 | −limit .req x2 | |
| 32 | −result .req x0 | |
| 20 | +#define src1 x0 | |
| 21 | +#define src2 x1 | |
| 22 | +#define limit x2 | |
| 23 | +#define result w0 | |
| 33 | 24 | |
| 34 | 25 | /* Internal variables. */ |
| 35 | −data1 .req x3 | |
| 36 | −data1w .req w3 | |
| 37 | −data2 .req x4 | |
| 38 | −data2w .req w4 | |
| 39 | −has_nul .req x5 | |
| 40 | −diff .req x6 | |
| 41 | −endloop .req x7 | |
| 42 | −tmp1 .req x8 | |
| 43 | −tmp2 .req x9 | |
| 44 | −tmp3 .req x10 | |
| 45 | −pos .req x11 | |
| 46 | −limit_wd .req x12 | |
| 47 | −mask .req x13 | |
| 26 | +#define data1 x3 | |
| 27 | +#define data1w w3 | |
| 28 | +#define data1h x4 | |
| 29 | +#define data2 x5 | |
| 30 | +#define data2w w5 | |
| 31 | +#define data2h x6 | |
| 32 | +#define tmp1 x7 | |
| 33 | +#define tmp2 x8 | |
| 48 | 34 | |
| 49 | 35 | SYM_FUNC_START_WEAK_PI(memcmp) |
| 50 | − cbz limit, .Lret0 | |
| 51 | − eor tmp1, src1, src2 | |
| 52 | − tst tmp1, #7 | |
| 53 | − b.ne .Lmisaligned8 | |
| 54 | − ands tmp1, src1, #7 | |
| 55 | − b.ne .Lmutual_align | |
| 56 | − sub limit_wd, limit, #1 /* limit != 0, so no underflow. */ | |
| 57 | − lsr limit_wd, limit_wd, #3 /* Convert to Dwords. */ | |
| 58 | − /* | |
| 59 | − * The input source addresses are at alignment boundary. | |
| 60 | − * Directly compare eight bytes each time. | |
| 61 | − */ | |
| 62 | −.Lloop_aligned: | |
| 63 | − ldr data1, [src1], #8 | |
| 64 | − ldr data2, [src2], #8 | |
| 65 | −.Lstart_realigned: | |
| 66 | − subs limit_wd, limit_wd, #1 | |
| 67 | − eor diff, data1, data2 /* Non-zero if differences found. */ | |
| 68 | − csinv endloop, diff, xzr, cs /* Last Dword or differences. */ | |
| 69 | − cbz endloop, .Lloop_aligned | |
| 70 | − | |
| 71 | − /* Not reached the limit, must have found a diff. */ | |
| 72 | − tbz limit_wd, #63, .Lnot_limit | |
| 73 | − | |
| 74 | − /* Limit % 8 == 0 => the diff is in the last 8 bytes. */ | |
| 75 | − ands limit, limit, #7 | |
| 76 | − b.eq .Lnot_limit | |
| 77 | − /* | |
| 78 | − * The remained bytes less than 8. It is needed to extract valid data | |
| 79 | − * from last eight bytes of the intended memory range. | |
| 80 | − */ | |
| 81 | − lsl limit, limit, #3 /* bytes-> bits. */ | |
| 82 | − mov mask, #~0 | |
| 83 | −CPU_BE( lsr mask, mask, limit ) | |
| 84 | −CPU_LE( lsl mask, mask, limit ) | |
| 85 | − bic data1, data1, mask | |
| 86 | − bic data2, data2, mask | |
| 87 | − | |
| 88 | − orr diff, diff, mask | |
| 89 | − b .Lnot_limit | |
| 90 | − | |
| 91 | −.Lmutual_align: | |
| 92 | − /* | |
| 93 | − * Sources are mutually aligned, but are not currently at an | |
| 94 | − * alignment boundary. Round down the addresses and then mask off | |
| 95 | − * the bytes that precede the start point. | |
| 96 | − */ | |
| 97 | − bic src1, src1, #7 | |
| 98 | − bic src2, src2, #7 | |
| 99 | − ldr data1, [src1], #8 | |
| 100 | − ldr data2, [src2], #8 | |
| 101 | − /* | |
| 102 | − * We can not add limit with alignment offset(tmp1) here. Since the | |
| 103 | − * addition probably make the limit overflown. | |
| 104 | − */ | |
| 105 | − sub limit_wd, limit, #1/*limit != 0, so no underflow.*/ | |
| 106 | − and tmp3, limit_wd, #7 | |
| 107 | − lsr limit_wd, limit_wd, #3 | |
| 108 | − add tmp3, tmp3, tmp1 | |
| 109 | − add limit_wd, limit_wd, tmp3, lsr #3 | |
| 110 | − add limit, limit, tmp1/* Adjust the limit for the extra. */ | |
| 111 | − | |
| 112 | − lsl tmp1, tmp1, #3/* Bytes beyond alignment -> bits.*/ | |
| 113 | − neg tmp1, tmp1/* Bits to alignment -64. */ | |
| 114 | − mov tmp2, #~0 | |
| 115 | − /*mask off the non-intended bytes before the start address.*/ | |
| 116 | −CPU_BE( lsl tmp2, tmp2, tmp1 )/*Big-endian.Early bytes are at MSB*/ | |
| 117 | − /* Little-endian. Early bytes are at LSB. */ | |
| 118 | −CPU_LE( lsr tmp2, tmp2, tmp1 ) | |
| 119 | − | |
| 120 | − orr data1, data1, tmp2 | |
| 121 | − orr data2, data2, tmp2 | |
| 122 | − b .Lstart_realigned | |
| 123 | − | |
| 124 | − /*src1 and src2 have different alignment offset.*/ | |
| 125 | −.Lmisaligned8: | |
| 126 | − cmp limit, #8 | |
| 127 | − b.lo .Ltiny8proc /*limit < 8: compare byte by byte*/ | |
| 128 | − | |
| 129 | − and tmp1, src1, #7 | |
| 130 | − neg tmp1, tmp1 | |
| 131 | − add tmp1, tmp1, #8/*valid length in the first 8 bytes of src1*/ | |
| 132 | − and tmp2, src2, #7 | |
| 133 | − neg tmp2, tmp2 | |
| 134 | − add tmp2, tmp2, #8/*valid length in the first 8 bytes of src2*/ | |
| 135 | − subs tmp3, tmp1, tmp2 | |
| 136 | − csel pos, tmp1, tmp2, hi /*Choose the maximum.*/ | |
| 137 | − | |
| 138 | − sub limit, limit, pos | |
| 139 | − /*compare the proceeding bytes in the first 8 byte segment.*/ | |
| 140 | −.Ltinycmp: | |
| 141 | − ldrb data1w, [src1], #1 | |
| 142 | − ldrb data2w, [src2], #1 | |
| 143 | − subs pos, pos, #1 | |
| 144 | − ccmp data1w, data2w, #0, ne /* NZCV = 0b0000. */ | |
| 145 | − b.eq .Ltinycmp | |
| 146 | − cbnz pos, 1f /*diff occurred before the last byte.*/ | |
| 147 | − cmp data1w, data2w | |
| 148 | − b.eq .Lstart_align | |
| 149 | −1: | |
| 150 | − sub result, data1, data2 | |
| 36 | + subs limit, limit, 8 | |
| 37 | + b.lo L(less8) | |
| 38 | + | |
| 39 | + ldr data1, [src1], 8 | |
| 40 | + ldr data2, [src2], 8 | |
| 41 | + cmp data1, data2 | |
| 42 | + b.ne L(return) | |
| 43 | + | |
| 44 | + subs limit, limit, 8 | |
| 45 | + b.gt L(more16) | |
| 46 | + | |
| 47 | + ldr data1, [src1, limit] | |
| 48 | + ldr data2, [src2, limit] | |
| 49 | + b L(return) | |
| 50 | + | |
| 51 | +L(more16): | |
| 52 | + ldr data1, [src1], 8 | |
| 53 | + ldr data2, [src2], 8 | |
| 54 | + cmp data1, data2 | |
| 55 | + bne L(return) | |
| 56 | + | |
| 57 | + /* Jump directly to comparing the last 16 bytes for 32 byte (or less) | |
| 58 | + strings. */ | |
| 59 | + subs limit, limit, 16 | |
| 60 | + b.ls L(last_bytes) | |
| 61 | + | |
| 62 | + /* We overlap loads between 0-32 bytes at either side of SRC1 when we | |
| 63 | + try to align, so limit it only to strings larger than 128 bytes. */ | |
| 64 | + cmp limit, 96 | |
| 65 | + b.ls L(loop16) | |
| 66 | + | |
| 67 | + /* Align src1 and adjust src2 with bytes not yet done. */ | |
| 68 | + and tmp1, src1, 15 | |
| 69 | + add limit, limit, tmp1 | |
| 70 | + sub src1, src1, tmp1 | |
| 71 | + sub src2, src2, tmp1 | |
| 72 | + | |
| 73 | + /* Loop performing 16 bytes per iteration using aligned src1. | |
| 74 | + Limit is pre-decremented by 16 and must be larger than zero. | |
| 75 | + Exit if <= 16 bytes left to do or if the data is not equal. */ | |
| 76 | + .p2align 4 | |
| 77 | +L(loop16): | |
| 78 | + ldp data1, data1h, [src1], 16 | |
| 79 | + ldp data2, data2h, [src2], 16 | |
| 80 | + subs limit, limit, 16 | |
| 81 | + ccmp data1, data2, 0, hi | |
| 82 | + ccmp data1h, data2h, 0, eq | |
| 83 | + b.eq L(loop16) | |
| 84 | + | |
| 85 | + cmp data1, data2 | |
| 86 | + bne L(return) | |
| 87 | + mov data1, data1h | |
| 88 | + mov data2, data2h | |
| 89 | + cmp data1, data2 | |
| 90 | + bne L(return) | |
| 91 | + | |
| 92 | + /* Compare last 1-16 bytes using unaligned access. */ | |
| 93 | +L(last_bytes): | |
| 94 | + add src1, src1, limit | |
| 95 | + add src2, src2, limit | |
| 96 | + ldp data1, data1h, [src1] | |
| 97 | + ldp data2, data2h, [src2] | |
| 98 | + cmp data1, data2 | |
| 99 | + bne L(return) | |
| 100 | + mov data1, data1h | |
| 101 | + mov data2, data2h | |
| 102 | + cmp data1, data2 | |
| 103 | + | |
| 104 | + /* Compare data bytes and set return value to 0, -1 or 1. */ | |
| 105 | +L(return): | |
| 106 | +#ifndef __AARCH64EB__ | |
| 107 | + rev data1, data1 | |
| 108 | + rev data2, data2 | |
| 109 | +#endif | |
| 110 | + cmp data1, data2 | |
| 111 | +L(ret_eq): | |
| 112 | + cset result, ne | |
| 113 | + cneg result, result, lo | |
| 151 | 114 | ret |
| 152 | 115 | |
| 153 | −.Lstart_align: | |
| 154 | − lsr limit_wd, limit, #3 | |
| 155 | − cbz limit_wd, .Lremain8 | |
| 156 | − | |
| 157 | − ands xzr, src1, #7 | |
| 158 | − b.eq .Lrecal_offset | |
| 159 | − /*process more leading bytes to make src1 aligned...*/ | |
| 160 | − add src1, src1, tmp3 /*backwards src1 to alignment boundary*/ | |
| 161 | − add src2, src2, tmp3 | |
| 162 | − sub limit, limit, tmp3 | |
| 163 | − lsr limit_wd, limit, #3 | |
| 164 | − cbz limit_wd, .Lremain8 | |
| 165 | − /*load 8 bytes from aligned SRC1..*/ | |
| 166 | − ldr data1, [src1], #8 | |
| 167 | − ldr data2, [src2], #8 | |
| 168 | − | |
| 169 | − subs limit_wd, limit_wd, #1 | |
| 170 | − eor diff, data1, data2 /*Non-zero if differences found.*/ | |
| 171 | − csinv endloop, diff, xzr, ne | |
| 172 | − cbnz endloop, .Lunequal_proc | |
| 173 | − /*How far is the current SRC2 from the alignment boundary...*/ | |
| 174 | − and tmp3, tmp3, #7 | |
| 175 | − | |
| 176 | −.Lrecal_offset:/*src1 is aligned now..*/ | |
| 177 | − neg pos, tmp3 | |
| 178 | −.Lloopcmp_proc: | |
| 179 | − /* | |
| 180 | − * Divide the eight bytes into two parts. First,backwards the src2 | |
| 181 | − * to an alignment boundary,load eight bytes and compare from | |
| 182 | − * the SRC2 alignment boundary. If all 8 bytes are equal,then start | |
| 183 | − * the second part's comparison. Otherwise finish the comparison. | |
| 184 | − * This special handle can garantee all the accesses are in the | |
| 185 | − * thread/task space in avoid to overrange access. | |
| 186 | − */ | |
| 187 | − ldr data1, [src1,pos] | |
| 188 | − ldr data2, [src2,pos] | |
| 189 | − eor diff, data1, data2 /* Non-zero if differences found. */ | |
| 190 | − cbnz diff, .Lnot_limit | |
| 191 | − | |
| 192 | − /*The second part process*/ | |
| 193 | − ldr data1, [src1], #8 | |
| 194 | − ldr data2, [src2], #8 | |
| 195 | − eor diff, data1, data2 /* Non-zero if differences found. */ | |
| 196 | − subs limit_wd, limit_wd, #1 | |
| 197 | − csinv endloop, diff, xzr, ne/*if limit_wd is 0,will finish the cmp*/ | |
| 198 | − cbz endloop, .Lloopcmp_proc | |
| 199 | −.Lunequal_proc: | |
| 200 | − cbz diff, .Lremain8 | |
| 201 | − | |
| 202 | −/* There is difference occurred in the latest comparison. */ | |
| 203 | −.Lnot_limit: | |
| 204 | −/* | |
| 205 | −* For little endian,reverse the low significant equal bits into MSB,then | |
| 206 | −* following CLZ can find how many equal bits exist. | |
| 207 | −*/ | |
| 208 | −CPU_LE( rev diff, diff ) | |
| 209 | −CPU_LE( rev data1, data1 ) | |
| 210 | −CPU_LE( rev data2, data2 ) | |
| 211 | − | |
| 212 | − /* | |
| 213 | − * The MS-non-zero bit of DIFF marks either the first bit | |
| 214 | − * that is different, or the end of the significant data. | |
| 215 | − * Shifting left now will bring the critical information into the | |
| 216 | − * top bits. | |
| 217 | − */ | |
| 218 | − clz pos, diff | |
| 219 | − lsl data1, data1, pos | |
| 220 | − lsl data2, data2, pos | |
| 221 | − /* | |
| 222 | − * We need to zero-extend (char is unsigned) the value and then | |
| 223 | − * perform a signed subtraction. | |
| 224 | − */ | |
| 225 | − lsr data1, data1, #56 | |
| 226 | − sub result, data1, data2, lsr #56 | |
| 116 | + .p2align 4 | |
| 117 | + /* Compare up to 8 bytes. Limit is [-8..-1]. */ | |
| 118 | +L(less8): | |
| 119 | + adds limit, limit, 4 | |
| 120 | + b.lo L(less4) | |
| 121 | + ldr data1w, [src1], 4 | |
| 122 | + ldr data2w, [src2], 4 | |
| 123 | + cmp data1w, data2w | |
| 124 | + b.ne L(return) | |
| 125 | + sub limit, limit, 4 | |
| 126 | +L(less4): | |
| 127 | + adds limit, limit, 4 | |
| 128 | + beq L(ret_eq) | |
| 129 | +L(byte_loop): | |
| 130 | + ldrb data1w, [src1], 1 | |
| 131 | + ldrb data2w, [src2], 1 | |
| 132 | + subs limit, limit, 1 | |
| 133 | + ccmp data1w, data2w, 0, ne /* NZCV = 0b0000. */ | |
| 134 | + b.eq L(byte_loop) | |
| 135 | + sub result, data1w, data2w | |
| 227 | 136 | ret |
| 228 | 137 | |
| 229 | −.Lremain8: | |
| 230 | − /* Limit % 8 == 0 =>. all data are equal.*/ | |
| 231 | − ands limit, limit, #7 | |
| 232 | − b.eq .Lret0 | |
| 233 | − | |
| 234 | −.Ltiny8proc: | |
| 235 | − ldrb data1w, [src1], #1 | |
| 236 | − ldrb data2w, [src2], #1 | |
| 237 | − subs limit, limit, #1 | |
| 238 | − | |
| 239 | − ccmp data1w, data2w, #0, ne /* NZCV = 0b0000. */ | |
| 240 | − b.eq .Ltiny8proc | |
| 241 | − sub result, data1, data2 | |
| 242 | − ret | |
| 243 | −.Lret0: | |
| 244 | − mov result, #0 | |
| 245 | − ret | |
| 246 | 138 | SYM_FUNC_END_PI(memcmp) |
| 247 | 139 | EXPORT_SYMBOL_NOKASAN(memcmp) |
| @@ -1,66 +1,252 @@ | ||
| 1 | 1 | /* SPDX-License-Identifier: GPL-2.0-only */ |
| 2 | 2 | /* |
| 3 | − * Copyright (C) 2013 ARM Ltd. | |
| 4 | − * Copyright (C) 2013 Linaro. | |
| 3 | + * Copyright (c) 2012-2021, Arm Limited. | |
| 5 | 4 | * |
| 6 | − * This code is based on glibc cortex strings work originally authored by Linaro | |
| 7 | − * be found @ | |
| 8 | − * | |
| 9 | − * http://bazaar.launchpad.net/~linaro-toolchain-dev/cortex-strings/trunk/ | |
| 10 | − * files/head:/src/aarch64/ | |
| 5 | + * Adapted from the original at: | |
| 6 | + * https://github.com/ARM-software/optimized-routines/blob/afd6244a1f8d9229/string/aarch64/memcpy.S | |
| 11 | 7 | */ |
| 12 | 8 | |
| 13 | 9 | #include <linux/linkage.h> |
| 14 | 10 | #include <asm/assembler.h> |
| 15 | −#include <asm/cache.h> | |
| 16 | 11 | |
| 17 | −/* | |
| 18 | − * Copy a buffer from src to dest (alignment handled by the hardware) | |
| 12 | +/* Assumptions: | |
| 13 | + * | |
| 14 | + * ARMv8-a, AArch64, unaligned accesses. | |
| 19 | 15 | * |
| 20 | − * Parameters: | |
| 21 | − * x0 - dest | |
| 22 | − * x1 - src | |
| 23 | − * x2 - n | |
| 24 | − * Returns: | |
| 25 | − * x0 - dest | |
| 26 | 16 | */ |
| 27 | − .macro ldrb1 reg, ptr, val | |
| 28 | − ldrb \reg, [\ptr], \val | |
| 29 | − .endm | |
| 30 | − | |
| 31 | − .macro strb1 reg, ptr, val | |
| 32 | − strb \reg, [\ptr], \val | |
| 33 | − .endm | |
| 34 | 17 | |
| 35 | − .macro ldrh1 reg, ptr, val | |
| 36 | − ldrh \reg, [\ptr], \val | |
| 37 | − .endm | |
| 18 | +#define L(label) .L ## label | |
| 38 | 19 | |
| 39 | − .macro strh1 reg, ptr, val | |
| 40 | − strh \reg, [\ptr], \val | |
| 41 | − .endm | |
| 20 | +#define dstin x0 | |
| 21 | +#define src x1 | |
| 22 | +#define count x2 | |
| 23 | +#define dst x3 | |
| 24 | +#define srcend x4 | |
| 25 | +#define dstend x5 | |
| 26 | +#define A_l x6 | |
| 27 | +#define A_lw w6 | |
| 28 | +#define A_h x7 | |
| 29 | +#define B_l x8 | |
| 30 | +#define B_lw w8 | |
| 31 | +#define B_h x9 | |
| 32 | +#define C_l x10 | |
| 33 | +#define C_lw w10 | |
| 34 | +#define C_h x11 | |
| 35 | +#define D_l x12 | |
| 36 | +#define D_h x13 | |
| 37 | +#define E_l x14 | |
| 38 | +#define E_h x15 | |
| 39 | +#define F_l x16 | |
| 40 | +#define F_h x17 | |
| 41 | +#define G_l count | |
| 42 | +#define G_h dst | |
| 43 | +#define H_l src | |
| 44 | +#define H_h srcend | |
| 45 | +#define tmp1 x14 | |
| 42 | 46 | |
| 43 | − .macro ldr1 reg, ptr, val | |
| 44 | − ldr \reg, [\ptr], \val | |
| 45 | − .endm | |
| 47 | +/* This implementation handles overlaps and supports both memcpy and memmove | |
| 48 | + from a single entry point. It uses unaligned accesses and branchless | |
| 49 | + sequences to keep the code small, simple and improve performance. | |
| 46 | 50 | |
| 47 | − .macro str1 reg, ptr, val | |
| 48 | − str \reg, [\ptr], \val | |
| 49 | − .endm | |
| 51 | + Copies are split into 3 main cases: small copies of up to 32 bytes, medium | |
| 52 | + copies of up to 128 bytes, and large copies. The overhead of the overlap | |
| 53 | + check is negligible since it is only required for large copies. | |
| 50 | 54 | |
| 51 | − .macro ldp1 reg1, reg2, ptr, val | |
| 52 | − ldp \reg1, \reg2, [\ptr], \val | |
| 53 | − .endm | |
| 54 | − | |
| 55 | − .macro stp1 reg1, reg2, ptr, val | |
| 56 | − stp \reg1, \reg2, [\ptr], \val | |
| 57 | − .endm | |
| 55 | + Large copies use a software pipelined loop processing 64 bytes per iteration. | |
| 56 | + The destination pointer is 16-byte aligned to minimize unaligned accesses. | |
| 57 | + The loop tail is handled by always copying 64 bytes from the end. | |
| 58 | +*/ | |
| 58 | 59 | |
| 60 | +SYM_FUNC_START_ALIAS(__memmove) | |
| 61 | +SYM_FUNC_START_WEAK_ALIAS_PI(memmove) | |
| 59 | 62 | SYM_FUNC_START_ALIAS(__memcpy) |
| 60 | 63 | SYM_FUNC_START_WEAK_PI(memcpy) |
| 61 | −#include "copy_template.S" | |
| 64 | + add srcend, src, count | |
| 65 | + add dstend, dstin, count | |
| 66 | + cmp count, 128 | |
| 67 | + b.hi L(copy_long) | |
| 68 | + cmp count, 32 | |
| 69 | + b.hi L(copy32_128) | |
| 70 | + | |
| 71 | + /* Small copies: 0..32 bytes. */ | |
| 72 | + cmp count, 16 | |
| 73 | + b.lo L(copy16) | |
| 74 | + ldp A_l, A_h, [src] | |
| 75 | + ldp D_l, D_h, [srcend, -16] | |
| 76 | + stp A_l, A_h, [dstin] | |
| 77 | + stp D_l, D_h, [dstend, -16] | |
| 78 | + ret | |
| 79 | + | |
| 80 | + /* Copy 8-15 bytes. */ | |
| 81 | +L(copy16): | |
| 82 | + tbz count, 3, L(copy8) | |
| 83 | + ldr A_l, [src] | |
| 84 | + ldr A_h, [srcend, -8] | |
| 85 | + str A_l, [dstin] | |
| 86 | + str A_h, [dstend, -8] | |
| 87 | + ret | |
| 88 | + | |
| 89 | + .p2align 3 | |
| 90 | + /* Copy 4-7 bytes. */ | |
| 91 | +L(copy8): | |
| 92 | + tbz count, 2, L(copy4) | |
| 93 | + ldr A_lw, [src] | |
| 94 | + ldr B_lw, [srcend, -4] | |
| 95 | + str A_lw, [dstin] | |
| 96 | + str B_lw, [dstend, -4] | |
| 97 | + ret | |
| 98 | + | |
| 99 | + /* Copy 0..3 bytes using a branchless sequence. */ | |
| 100 | +L(copy4): | |
| 101 | + cbz count, L(copy0) | |
| 102 | + lsr tmp1, count, 1 | |
| 103 | + ldrb A_lw, [src] | |
| 104 | + ldrb C_lw, [srcend, -1] | |
| 105 | + ldrb B_lw, [src, tmp1] | |
| 106 | + strb A_lw, [dstin] | |
| 107 | + strb B_lw, [dstin, tmp1] | |
| 108 | + strb C_lw, [dstend, -1] | |
| 109 | +L(copy0): | |
| 110 | + ret | |
| 111 | + | |
| 112 | + .p2align 4 | |
| 113 | + /* Medium copies: 33..128 bytes. */ | |
| 114 | +L(copy32_128): | |
| 115 | + ldp A_l, A_h, [src] | |
| 116 | + ldp B_l, B_h, [src, 16] | |
| 117 | + ldp C_l, C_h, [srcend, -32] | |
| 118 | + ldp D_l, D_h, [srcend, -16] | |
| 119 | + cmp count, 64 | |
| 120 | + b.hi L(copy128) | |
| 121 | + stp A_l, A_h, [dstin] | |
| 122 | + stp B_l, B_h, [dstin, 16] | |
| 123 | + stp C_l, C_h, [dstend, -32] | |
| 124 | + stp D_l, D_h, [dstend, -16] | |
| 62 | 125 | ret |
| 126 | + | |
| 127 | + .p2align 4 | |
| 128 | + /* Copy 65..128 bytes. */ | |
| 129 | +L(copy128): | |
| 130 | + ldp E_l, E_h, [src, 32] | |
| 131 | + ldp F_l, F_h, [src, 48] | |
| 132 | + cmp count, 96 | |
| 133 | + b.ls L(copy96) | |
| 134 | + ldp G_l, G_h, [srcend, -64] | |
| 135 | + ldp H_l, H_h, [srcend, -48] | |
| 136 | + stp G_l, G_h, [dstend, -64] | |
| 137 | + stp H_l, H_h, [dstend, -48] | |
| 138 | +L(copy96): | |
| 139 | + stp A_l, A_h, [dstin] | |
| 140 | + stp B_l, B_h, [dstin, 16] | |
| 141 | + stp E_l, E_h, [dstin, 32] | |
| 142 | + stp F_l, F_h, [dstin, 48] | |
| 143 | + stp C_l, C_h, [dstend, -32] | |
| 144 | + stp D_l, D_h, [dstend, -16] | |
| 145 | + ret | |
| 146 | + | |
| 147 | + .p2align 4 | |
| 148 | + /* Copy more than 128 bytes. */ | |
| 149 | +L(copy_long): | |
| 150 | + /* Use backwards copy if there is an overlap. */ | |
| 151 | + sub tmp1, dstin, src | |
| 152 | + cbz tmp1, L(copy0) | |
| 153 | + cmp tmp1, count | |
| 154 | + b.lo L(copy_long_backwards) | |
| 155 | + | |
| 156 | + /* Copy 16 bytes and then align dst to 16-byte alignment. */ | |
| 157 | + | |
| 158 | + ldp D_l, D_h, [src] | |
| 159 | + and tmp1, dstin, 15 | |
| 160 | + bic dst, dstin, 15 | |
| 161 | + sub src, src, tmp1 | |
| 162 | + add count, count, tmp1 /* Count is now 16 too large. */ | |
| 163 | + ldp A_l, A_h, [src, 16] | |
| 164 | + stp D_l, D_h, [dstin] | |
| 165 | + ldp B_l, B_h, [src, 32] | |
| 166 | + ldp C_l, C_h, [src, 48] | |
| 167 | + ldp D_l, D_h, [src, 64]! | |
| 168 | + subs count, count, 128 + 16 /* Test and readjust count. */ | |
| 169 | + b.ls L(copy64_from_end) | |
| 170 | + | |
| 171 | +L(loop64): | |
| 172 | + stp A_l, A_h, [dst, 16] | |
| 173 | + ldp A_l, A_h, [src, 16] | |
| 174 | + stp B_l, B_h, [dst, 32] | |
| 175 | + ldp B_l, B_h, [src, 32] | |
| 176 | + stp C_l, C_h, [dst, 48] | |
| 177 | + ldp C_l, C_h, [src, 48] | |
| 178 | + stp D_l, D_h, [dst, 64]! | |
| 179 | + ldp D_l, D_h, [src, 64]! | |
| 180 | + subs count, count, 64 | |
| 181 | + b.hi L(loop64) | |
| 182 | + | |
| 183 | + /* Write the last iteration and copy 64 bytes from the end. */ | |
| 184 | +L(copy64_from_end): | |
| 185 | + ldp E_l, E_h, [srcend, -64] | |
| 186 | + stp A_l, A_h, [dst, 16] | |
| 187 | + ldp A_l, A_h, [srcend, -48] | |
| 188 | + stp B_l, B_h, [dst, 32] | |
| 189 | + ldp B_l, B_h, [srcend, -32] | |
| 190 | + stp C_l, C_h, [dst, 48] | |
| 191 | + ldp C_l, C_h, [srcend, -16] | |
| 192 | + stp D_l, D_h, [dst, 64] | |
| 193 | + stp E_l, E_h, [dstend, -64] | |
| 194 | + stp A_l, A_h, [dstend, -48] | |
| 195 | + stp B_l, B_h, [dstend, -32] | |
| 196 | + stp C_l, C_h, [dstend, -16] | |
| 197 | + ret | |
| 198 | + | |
| 199 | + .p2align 4 | |
| 200 | + | |
| 201 | + /* Large backwards copy for overlapping copies. | |
| 202 | + Copy 16 bytes and then align dst to 16-byte alignment. */ | |
| 203 | +L(copy_long_backwards): | |
| 204 | + ldp D_l, D_h, [srcend, -16] | |
| 205 | + and tmp1, dstend, 15 | |
| 206 | + sub srcend, srcend, tmp1 | |
| 207 | + sub count, count, tmp1 | |
| 208 | + ldp A_l, A_h, [srcend, -16] | |
| 209 | + stp D_l, D_h, [dstend, -16] | |
| 210 | + ldp B_l, B_h, [srcend, -32] | |
| 211 | + ldp C_l, C_h, [srcend, -48] | |
| 212 | + ldp D_l, D_h, [srcend, -64]! | |
| 213 | + sub dstend, dstend, tmp1 | |
| 214 | + subs count, count, 128 | |
| 215 | + b.ls L(copy64_from_start) | |
| 216 | + | |
| 217 | +L(loop64_backwards): | |
| 218 | + stp A_l, A_h, [dstend, -16] | |
| 219 | + ldp A_l, A_h, [srcend, -16] | |
| 220 | + stp B_l, B_h, [dstend, -32] | |
| 221 | + ldp B_l, B_h, [srcend, -32] | |
| 222 | + stp C_l, C_h, [dstend, -48] | |
| 223 | + ldp C_l, C_h, [srcend, -48] | |
| 224 | + stp D_l, D_h, [dstend, -64]! | |
| 225 | + ldp D_l, D_h, [srcend, -64]! | |
| 226 | + subs count, count, 64 | |
| 227 | + b.hi L(loop64_backwards) | |
| 228 | + | |
| 229 | + /* Write the last iteration and copy 64 bytes from the start. */ | |
| 230 | +L(copy64_from_start): | |
| 231 | + ldp G_l, G_h, [src, 48] | |
| 232 | + stp A_l, A_h, [dstend, -16] | |
| 233 | + ldp A_l, A_h, [src, 32] | |
| 234 | + stp B_l, B_h, [dstend, -32] | |
| 235 | + ldp B_l, B_h, [src, 16] | |
| 236 | + stp C_l, C_h, [dstend, -48] | |
| 237 | + ldp C_l, C_h, [src] | |
| 238 | + stp D_l, D_h, [dstend, -64] | |
| 239 | + stp G_l, G_h, [dstin, 48] | |
| 240 | + stp A_l, A_h, [dstin, 32] | |
| 241 | + stp B_l, B_h, [dstin, 16] | |
| 242 | + stp C_l, C_h, [dstin] | |
| 243 | + ret | |
| 244 | + | |
| 63 | 245 | SYM_FUNC_END_PI(memcpy) |
| 64 | 246 | EXPORT_SYMBOL(memcpy) |
| 65 | 247 | SYM_FUNC_END_ALIAS(__memcpy) |
| 66 | 248 | EXPORT_SYMBOL(__memcpy) |
| 249 | +SYM_FUNC_END_ALIAS_PI(memmove) | |
| 250 | +EXPORT_SYMBOL(memmove) | |
| 251 | +SYM_FUNC_END_ALIAS(__memmove) | |
| 252 | +EXPORT_SYMBOL(__memmove) | |
| @@ -1,189 +0,0 @@ | ||
| 1 | −/* SPDX-License-Identifier: GPL-2.0-only */ | |
| 2 | −/* | |
| 3 | − * Copyright (C) 2013 ARM Ltd. | |
| 4 | − * Copyright (C) 2013 Linaro. | |
| 5 | − * | |
| 6 | − * This code is based on glibc cortex strings work originally authored by Linaro | |
| 7 | − * be found @ | |
| 8 | − * | |
| 9 | − * http://bazaar.launchpad.net/~linaro-toolchain-dev/cortex-strings/trunk/ | |
| 10 | − * files/head:/src/aarch64/ | |
| 11 | − */ | |
| 12 | − | |
| 13 | −#include <linux/linkage.h> | |
| 14 | −#include <asm/assembler.h> | |
| 15 | −#include <asm/cache.h> | |
| 16 | − | |
| 17 | −/* | |
| 18 | − * Move a buffer from src to test (alignment handled by the hardware). | |
| 19 | − * If dest <= src, call memcpy, otherwise copy in reverse order. | |
| 20 | − * | |
| 21 | − * Parameters: | |
| 22 | − * x0 - dest | |
| 23 | − * x1 - src | |
| 24 | − * x2 - n | |
| 25 | − * Returns: | |
| 26 | − * x0 - dest | |
| 27 | − */ | |
| 28 | −dstin .req x0 | |
| 29 | −src .req x1 | |
| 30 | −count .req x2 | |
| 31 | −tmp1 .req x3 | |
| 32 | −tmp1w .req w3 | |
| 33 | −tmp2 .req x4 | |
| 34 | −tmp2w .req w4 | |
| 35 | −tmp3 .req x5 | |
| 36 | −tmp3w .req w5 | |
| 37 | −dst .req x6 | |
| 38 | − | |
| 39 | −A_l .req x7 | |
| 40 | −A_h .req x8 | |
| 41 | −B_l .req x9 | |
| 42 | −B_h .req x10 | |
| 43 | −C_l .req x11 | |
| 44 | −C_h .req x12 | |
| 45 | −D_l .req x13 | |
| 46 | −D_h .req x14 | |
| 47 | − | |
| 48 | −SYM_FUNC_START_ALIAS(__memmove) | |
| 49 | −SYM_FUNC_START_WEAK_PI(memmove) | |
| 50 | − cmp dstin, src | |
| 51 | − b.lo __memcpy | |
| 52 | − add tmp1, src, count | |
| 53 | − cmp dstin, tmp1 | |
| 54 | − b.hs __memcpy /* No overlap. */ | |
| 55 | − | |
| 56 | − add dst, dstin, count | |
| 57 | − add src, src, count | |
| 58 | − cmp count, #16 | |
| 59 | − b.lo .Ltail15 /*probably non-alignment accesses.*/ | |
| 60 | − | |
| 61 | − ands tmp2, src, #15 /* Bytes to reach alignment. */ | |
| 62 | − b.eq .LSrcAligned | |
| 63 | − sub count, count, tmp2 | |
| 64 | − /* | |
| 65 | − * process the aligned offset length to make the src aligned firstly. | |
| 66 | − * those extra instructions' cost is acceptable. It also make the | |
| 67 | − * coming accesses are based on aligned address. | |
| 68 | − */ | |
| 69 | − tbz tmp2, #0, 1f | |
| 70 | − ldrb tmp1w, [src, #-1]! | |
| 71 | − strb tmp1w, [dst, #-1]! | |
| 72 | −1: | |
| 73 | − tbz tmp2, #1, 2f | |
| 74 | − ldrh tmp1w, [src, #-2]! | |
| 75 | − strh tmp1w, [dst, #-2]! | |
| 76 | −2: | |
| 77 | − tbz tmp2, #2, 3f | |
| 78 | − ldr tmp1w, [src, #-4]! | |
| 79 | − str tmp1w, [dst, #-4]! | |
| 80 | −3: | |
| 81 | − tbz tmp2, #3, .LSrcAligned | |
| 82 | − ldr tmp1, [src, #-8]! | |
| 83 | − str tmp1, [dst, #-8]! | |
| 84 | − | |
| 85 | −.LSrcAligned: | |
| 86 | − cmp count, #64 | |
| 87 | − b.ge .Lcpy_over64 | |
| 88 | − | |
| 89 | − /* | |
| 90 | − * Deal with small copies quickly by dropping straight into the | |
| 91 | − * exit block. | |
| 92 | − */ | |
| 93 | −.Ltail63: | |
| 94 | − /* | |
| 95 | − * Copy up to 48 bytes of data. At this point we only need the | |
| 96 | − * bottom 6 bits of count to be accurate. | |
| 97 | − */ | |
| 98 | − ands tmp1, count, #0x30 | |
| 99 | − b.eq .Ltail15 | |
| 100 | − cmp tmp1w, #0x20 | |
| 101 | − b.eq 1f | |
| 102 | − b.lt 2f | |
| 103 | − ldp A_l, A_h, [src, #-16]! | |
| 104 | − stp A_l, A_h, [dst, #-16]! | |
| 105 | −1: | |
| 106 | − ldp A_l, A_h, [src, #-16]! | |
| 107 | − stp A_l, A_h, [dst, #-16]! | |
| 108 | −2: | |
| 109 | − ldp A_l, A_h, [src, #-16]! | |
| 110 | − stp A_l, A_h, [dst, #-16]! | |
| 111 | − | |
| 112 | −.Ltail15: | |
| 113 | − tbz count, #3, 1f | |
| 114 | − ldr tmp1, [src, #-8]! | |
| 115 | − str tmp1, [dst, #-8]! | |
| 116 | −1: | |
| 117 | − tbz count, #2, 2f | |
| 118 | − ldr tmp1w, [src, #-4]! | |
| 119 | − str tmp1w, [dst, #-4]! | |
| 120 | −2: | |
| 121 | − tbz count, #1, 3f | |
| 122 | − ldrh tmp1w, [src, #-2]! | |
| 123 | − strh tmp1w, [dst, #-2]! | |
| 124 | −3: | |
| 125 | − tbz count, #0, .Lexitfunc | |
| 126 | − ldrb tmp1w, [src, #-1] | |
| 127 | − strb tmp1w, [dst, #-1] | |
| 128 | − | |
| 129 | −.Lexitfunc: | |
| 130 | − ret | |
| 131 | − | |
| 132 | −.Lcpy_over64: | |
| 133 | − subs count, count, #128 | |
| 134 | − b.ge .Lcpy_body_large | |
| 135 | − /* | |
| 136 | − * Less than 128 bytes to copy, so handle 64 bytes here and then jump | |
| 137 | − * to the tail. | |
| 138 | − */ | |
| 139 | − ldp A_l, A_h, [src, #-16] | |
| 140 | − stp A_l, A_h, [dst, #-16] | |
| 141 | − ldp B_l, B_h, [src, #-32] | |
| 142 | − ldp C_l, C_h, [src, #-48] | |
| 143 | − stp B_l, B_h, [dst, #-32] | |
| 144 | − stp C_l, C_h, [dst, #-48] | |
| 145 | − ldp D_l, D_h, [src, #-64]! | |
| 146 | − stp D_l, D_h, [dst, #-64]! | |
| 147 | − | |
| 148 | − tst count, #0x3f | |
| 149 | − b.ne .Ltail63 | |
| 150 | − ret | |
| 151 | − | |
| 152 | − /* | |
| 153 | − * Critical loop. Start at a new cache line boundary. Assuming | |
| 154 | − * 64 bytes per line this ensures the entire loop is in one line. | |
| 155 | − */ | |
| 156 | − .p2align L1_CACHE_SHIFT | |
| 157 | −.Lcpy_body_large: | |
| 158 | − /* pre-load 64 bytes data. */ | |
| 159 | − ldp A_l, A_h, [src, #-16] | |
| 160 | − ldp B_l, B_h, [src, #-32] | |
| 161 | − ldp C_l, C_h, [src, #-48] | |
| 162 | − ldp D_l, D_h, [src, #-64]! | |
| 163 | −1: | |
| 164 | − /* | |
| 165 | − * interlace the load of next 64 bytes data block with store of the last | |
| 166 | − * loaded 64 bytes data. | |
| 167 | − */ | |
| 168 | − stp A_l, A_h, [dst, #-16] | |
| 169 | − ldp A_l, A_h, [src, #-16] | |
| 170 | − stp B_l, B_h, [dst, #-32] | |
| 171 | − ldp B_l, B_h, [src, #-32] | |
| 172 | − stp C_l, C_h, [dst, #-48] | |
| 173 | − ldp C_l, C_h, [src, #-48] | |
| 174 | − stp D_l, D_h, [dst, #-64]! | |
| 175 | − ldp D_l, D_h, [src, #-64]! | |
| 176 | − subs count, count, #64 | |
| 177 | − b.ge 1b | |
| 178 | − stp A_l, A_h, [dst, #-16] | |
| 179 | − stp B_l, B_h, [dst, #-32] | |
| 180 | − stp C_l, C_h, [dst, #-48] | |
| 181 | − stp D_l, D_h, [dst, #-64]! | |
| 182 | − | |
| 183 | − tst count, #0x3f | |
| 184 | − b.ne .Ltail63 | |
| 185 | − ret | |
| 186 | −SYM_FUNC_END_PI(memmove) | |
| 187 | −EXPORT_SYMBOL(memmove) | |
| 188 | −SYM_FUNC_END_ALIAS(__memmove) | |
| 189 | −EXPORT_SYMBOL(__memmove) | |
| @@ -36,6 +36,26 @@ SYM_FUNC_START(mte_clear_page_tags) · 35 unmodified lines | ||
| 36 | 36 | ret |
| 37 | 37 | SYM_FUNC_END(mte_clear_page_tags) |
| 38 | 38 | |
| 39 | +/* | |
| 40 | + * Zero the page and tags at the same time | |
| 41 | + * | |
| 42 | + * Parameters: | |
| 43 | + * x0 - address to the beginning of the page | |
| 44 | + */ | |
| 45 | +SYM_FUNC_START(mte_zero_clear_page_tags) | |
| 46 | + mrs x1, dczid_el0 | |
| 47 | + and w1, w1, #0xf | |
| 48 | + mov x2, #4 | |
| 49 | + lsl x1, x2, x1 | |
| 50 | + and x0, x0, #(1 << MTE_TAG_SHIFT) - 1 // clear the tag | |
| 51 | + | |
| 52 | +1: dc gzva, x0 | |
| 53 | + add x0, x0, x1 | |
| 54 | + tst x0, #(PAGE_SIZE - 1) | |
| 55 | + b.ne 1b | |
| 56 | + ret | |
| 57 | +SYM_FUNC_END(mte_zero_clear_page_tags) | |
| 58 | + | |
| 39 | 59 | /* |
| 40 | 60 | * Copy the tags from the source page to the destination one |
| 41 | 61 | * x0 - address of the destination page |
| @@ -1,84 +1,123 @@ | ||
| 1 | 1 | /* SPDX-License-Identifier: GPL-2.0-only */ |
| 2 | 2 | /* |
| 3 | − * Copyright (C) 2013 ARM Ltd. | |
| 4 | − * Copyright (C) 2013 Linaro. | |
| 3 | + * Copyright (c) 2012-2021, Arm Limited. | |
| 5 | 4 | * |
| 6 | − * This code is based on glibc cortex strings work originally authored by Linaro | |
| 7 | − * be found @ | |
| 8 | − * | |
| 9 | − * http://bazaar.launchpad.net/~linaro-toolchain-dev/cortex-strings/trunk/ | |
| 10 | − * files/head:/src/aarch64/ | |
| 5 | + * Adapted from the original at: | |
| 6 | + * https://github.com/ARM-software/optimized-routines/blob/afd6244a1f8d9229/string/aarch64/strcmp.S | |
| 11 | 7 | */ |
| 12 | 8 | |
| 13 | 9 | #include <linux/linkage.h> |
| 14 | 10 | #include <asm/assembler.h> |
| 15 | 11 | |
| 16 | −/* | |
| 17 | − * compare two strings | |
| 12 | +/* Assumptions: | |
| 18 | 13 | * |
| 19 | − * Parameters: | |
| 20 | − * x0 - const string 1 pointer | |
| 21 | − * x1 - const string 2 pointer | |
| 22 | − * Returns: | |
| 23 | − * x0 - an integer less than, equal to, or greater than zero | |
| 24 | − * if s1 is found, respectively, to be less than, to match, | |
| 25 | − * or be greater than s2. | |
| 14 | + * ARMv8-a, AArch64 | |
| 26 | 15 | */ |
| 27 | 16 | |
| 17 | +#define L(label) .L ## label | |
| 18 | + | |
| 28 | 19 | #define REP8_01 0x0101010101010101 |
| 29 | 20 | #define REP8_7f 0x7f7f7f7f7f7f7f7f |
| 30 | 21 | #define REP8_80 0x8080808080808080 |
| 31 | 22 | |
| 32 | 23 | /* Parameters and result. */ |
| 33 | −src1 .req x0 | |
| 34 | −src2 .req x1 | |
| 35 | −result .req x0 | |
| 24 | +#define src1 x0 | |
| 25 | +#define src2 x1 | |
| 26 | +#define result x0 | |
| 36 | 27 | |
| 37 | 28 | /* Internal variables. */ |
| 38 | −data1 .req x2 | |
| 39 | −data1w .req w2 | |
| 40 | −data2 .req x3 | |
| 41 | −data2w .req w3 | |
| 42 | −has_nul .req x4 | |
| 43 | −diff .req x5 | |
| 44 | −syndrome .req x6 | |
| 45 | −tmp1 .req x7 | |
| 46 | −tmp2 .req x8 | |
| 47 | −tmp3 .req x9 | |
| 48 | −zeroones .req x10 | |
| 49 | −pos .req x11 | |
| 50 | − | |
| 29 | +#define data1 x2 | |
| 30 | +#define data1w w2 | |
| 31 | +#define data2 x3 | |
| 32 | +#define data2w w3 | |
| 33 | +#define has_nul x4 | |
| 34 | +#define diff x5 | |
| 35 | +#define syndrome x6 | |
| 36 | +#define tmp1 x7 | |
| 37 | +#define tmp2 x8 | |
| 38 | +#define tmp3 x9 | |
| 39 | +#define zeroones x10 | |
| 40 | +#define pos x11 | |
| 41 | + | |
| 42 | + /* Start of performance-critical section -- one 64B cache line. */ | |
| 43 | + .align 6 | |
| 51 | 44 | SYM_FUNC_START_WEAK_PI(strcmp) |
| 52 | 45 | eor tmp1, src1, src2 |
| 53 | 46 | mov zeroones, #REP8_01 |
| 54 | 47 | tst tmp1, #7 |
| 55 | − b.ne .Lmisaligned8 | |
| 48 | + b.ne L(misaligned8) | |
| 56 | 49 | ands tmp1, src1, #7 |
| 57 | − b.ne .Lmutual_align | |
| 58 | − | |
| 59 | − /* | |
| 60 | − * NUL detection works on the principle that (X - 1) & (~X) & 0x80 | |
| 61 | − * (=> (X - 1) & ~(X | 0x7f)) is non-zero iff a byte is zero, and | |
| 62 | − * can be done in parallel across the entire word. | |
| 63 | − */ | |
| 64 | −.Lloop_aligned: | |
| 50 | + b.ne L(mutual_align) | |
| 51 | + /* NUL detection works on the principle that (X - 1) & (~X) & 0x80 | |
| 52 | + (=> (X - 1) & ~(X | 0x7f)) is non-zero iff a byte is zero, and | |
| 53 | + can be done in parallel across the entire word. */ | |
| 54 | +L(loop_aligned): | |
| 65 | 55 | ldr data1, [src1], #8 |
| 66 | 56 | ldr data2, [src2], #8 |
| 67 | −.Lstart_realigned: | |
| 57 | +L(start_realigned): | |
| 68 | 58 | sub tmp1, data1, zeroones |
| 69 | 59 | orr tmp2, data1, #REP8_7f |
| 70 | 60 | eor diff, data1, data2 /* Non-zero if differences found. */ |
| 71 | 61 | bic has_nul, tmp1, tmp2 /* Non-zero if NUL terminator. */ |
| 72 | 62 | orr syndrome, diff, has_nul |
| 73 | − cbz syndrome, .Lloop_aligned | |
| 74 | − b .Lcal_cmpresult | |
| 63 | + cbz syndrome, L(loop_aligned) | |
| 64 | + /* End of performance-critical section -- one 64B cache line. */ | |
| 65 | + | |
| 66 | +L(end): | |
| 67 | +#ifndef __AARCH64EB__ | |
| 68 | + rev syndrome, syndrome | |
| 69 | + rev data1, data1 | |
| 70 | + /* The MS-non-zero bit of the syndrome marks either the first bit | |
| 71 | + that is different, or the top bit of the first zero byte. | |
| 72 | + Shifting left now will bring the critical information into the | |
| 73 | + top bits. */ | |
| 74 | + clz pos, syndrome | |
| 75 | + rev data2, data2 | |
| 76 | + lsl data1, data1, pos | |
| 77 | + lsl data2, data2, pos | |
| 78 | + /* But we need to zero-extend (char is unsigned) the value and then | |
| 79 | + perform a signed 32-bit subtraction. */ | |
| 80 | + lsr data1, data1, #56 | |
| 81 | + sub result, data1, data2, lsr #56 | |
| 82 | + ret | |
| 83 | +#else | |
| 84 | + /* For big-endian we cannot use the trick with the syndrome value | |
| 85 | + as carry-propagation can corrupt the upper bits if the trailing | |
| 86 | + bytes in the string contain 0x01. */ | |
| 87 | + /* However, if there is no NUL byte in the dword, we can generate | |
| 88 | + the result directly. We can't just subtract the bytes as the | |
| 89 | + MSB might be significant. */ | |
| 90 | + cbnz has_nul, 1f | |
| 91 | + cmp data1, data2 | |
| 92 | + cset result, ne | |
| 93 | + cneg result, result, lo | |
| 94 | + ret | |
| 95 | +1: | |
| 96 | + /* Re-compute the NUL-byte detection, using a byte-reversed value. */ | |
| 97 | + rev tmp3, data1 | |
| 98 | + sub tmp1, tmp3, zeroones | |
| 99 | + orr tmp2, tmp3, #REP8_7f | |
| 100 | + bic has_nul, tmp1, tmp2 | |
| 101 | + rev has_nul, has_nul | |
| 102 | + orr syndrome, diff, has_nul | |
| 103 | + clz pos, syndrome | |
| 104 | + /* The MS-non-zero bit of the syndrome marks either the first bit | |
| 105 | + that is different, or the top bit of the first zero byte. | |
| 106 | + Shifting left now will bring the critical information into the | |
| 107 | + top bits. */ | |
| 108 | + lsl data1, data1, pos | |
| 109 | + lsl data2, data2, pos | |
| 110 | + /* But we need to zero-extend (char is unsigned) the value and then | |
| 111 | + perform a signed 32-bit subtraction. */ | |
| 112 | + lsr data1, data1, #56 | |
| 113 | + sub result, data1, data2, lsr #56 | |
| 114 | + ret | |
| 115 | +#endif | |
| 75 | 116 | |
| 76 | −.Lmutual_align: | |
| 77 | − /* | |
| 78 | − * Sources are mutually aligned, but are not currently at an | |
| 79 | − * alignment boundary. Round down the addresses and then mask off | |
| 80 | − * the bytes that preceed the start point. | |
| 81 | − */ | |
| 117 | +L(mutual_align): | |
| 118 | + /* Sources are mutually aligned, but are not currently at an | |
| 119 | + alignment boundary. Round down the addresses and then mask off | |
| 120 | + the bytes that preceed the start point. */ | |
| 82 | 121 | bic src1, src1, #7 |
| 83 | 122 | bic src2, src2, #7 |
| 84 | 123 | lsl tmp1, tmp1, #3 /* Bytes beyond alignment -> bits. */ |
| @@ -86,138 +125,52 @@ SYM_FUNC_START_WEAK_PI(strcmp) · 1 unmodified line | ||
| 86 | 125 | neg tmp1, tmp1 /* Bits to alignment -64. */ |
| 87 | 126 | ldr data2, [src2], #8 |
| 88 | 127 | mov tmp2, #~0 |
| 128 | +#ifdef __AARCH64EB__ | |
| 89 | 129 | /* Big-endian. Early bytes are at MSB. */ |
| 90 | −CPU_BE( lsl tmp2, tmp2, tmp1 ) /* Shift (tmp1 & 63). */ | |
| 130 | + lsl tmp2, tmp2, tmp1 /* Shift (tmp1 & 63). */ | |
| 131 | +#else | |
| 91 | 132 | /* Little-endian. Early bytes are at LSB. */ |
| 92 | −CPU_LE( lsr tmp2, tmp2, tmp1 ) /* Shift (tmp1 & 63). */ | |
| 93 | − | |
| 133 | + lsr tmp2, tmp2, tmp1 /* Shift (tmp1 & 63). */ | |
| 134 | +#endif | |
| 94 | 135 | orr data1, data1, tmp2 |
| 95 | 136 | orr data2, data2, tmp2 |
| 96 | − b .Lstart_realigned | |
| 97 | − | |
| 98 | −.Lmisaligned8: | |
| 99 | − /* | |
| 100 | − * Get the align offset length to compare per byte first. | |
| 101 | − * After this process, one string's address will be aligned. | |
| 102 | − */ | |
| 103 | − and tmp1, src1, #7 | |
| 104 | − neg tmp1, tmp1 | |
| 105 | − add tmp1, tmp1, #8 | |
| 106 | − and tmp2, src2, #7 | |
| 107 | − neg tmp2, tmp2 | |
| 108 | − add tmp2, tmp2, #8 | |
| 109 | − subs tmp3, tmp1, tmp2 | |
| 110 | − csel pos, tmp1, tmp2, hi /*Choose the maximum. */ | |
| 111 | −.Ltinycmp: | |
| 137 | + b L(start_realigned) | |
| 138 | + | |
| 139 | +L(misaligned8): | |
| 140 | + /* Align SRC1 to 8 bytes and then compare 8 bytes at a time, always | |
| 141 | + checking to make sure that we don't access beyond page boundary in | |
| 142 | + SRC2. */ | |
| 143 | + tst src1, #7 | |
| 144 | + b.eq L(loop_misaligned) | |
| 145 | +L(do_misaligned): | |
| 112 | 146 | ldrb data1w, [src1], #1 |
| 113 | 147 | ldrb data2w, [src2], #1 |
| 114 | − subs pos, pos, #1 | |
| 115 | − ccmp data1w, #1, #0, ne /* NZCV = 0b0000. */ | |
| 116 | − ccmp data1w, data2w, #0, cs /* NZCV = 0b0000. */ | |
| 117 | − b.eq .Ltinycmp | |
| 118 | − cbnz pos, 1f /*find the null or unequal...*/ | |
| 119 | 148 | cmp data1w, #1 |
| 120 | − ccmp data1w, data2w, #0, cs | |
| 121 | − b.eq .Lstart_align /*the last bytes are equal....*/ | |
| 122 | −1: | |
| 123 | − sub result, data1, data2 | |
| 124 | − ret | |
| 125 | − | |
| 126 | −.Lstart_align: | |
| 127 | − ands xzr, src1, #7 | |
| 128 | − b.eq .Lrecal_offset | |
| 129 | − /*process more leading bytes to make str1 aligned...*/ | |
| 130 | − add src1, src1, tmp3 | |
| 131 | − add src2, src2, tmp3 | |
| 132 | − /*load 8 bytes from aligned str1 and non-aligned str2..*/ | |
| 149 | + ccmp data1w, data2w, #0, cs /* NZCV = 0b0000. */ | |
| 150 | + b.ne L(done) | |
| 151 | + tst src1, #7 | |
| 152 | + b.ne L(do_misaligned) | |
| 153 | + | |
| 154 | +L(loop_misaligned): | |
| 155 | + /* Test if we are within the last dword of the end of a 4K page. If | |
| 156 | + yes then jump back to the misaligned loop to copy a byte at a time. */ | |
| 157 | + and tmp1, src2, #0xff8 | |
| 158 | + eor tmp1, tmp1, #0xff8 | |
| 159 | + cbz tmp1, L(do_misaligned) | |
| 133 | 160 | ldr data1, [src1], #8 |
| 134 | 161 | ldr data2, [src2], #8 |
| 135 | 162 | |
| 136 | 163 | sub tmp1, data1, zeroones |
| 137 | 164 | orr tmp2, data1, #REP8_7f |
| 138 | − bic has_nul, tmp1, tmp2 | |
| 139 | − eor diff, data1, data2 /* Non-zero if differences found. */ | |
| 140 | − orr syndrome, diff, has_nul | |
| 141 | − cbnz syndrome, .Lcal_cmpresult | |
| 142 | − /*How far is the current str2 from the alignment boundary...*/ | |
| 143 | − and tmp3, tmp3, #7 | |
| 144 | −.Lrecal_offset: | |
| 145 | − neg pos, tmp3 | |
| 146 | −.Lloopcmp_proc: | |
| 147 | − /* | |
| 148 | − * Divide the eight bytes into two parts. First,backwards the src2 | |
| 149 | − * to an alignment boundary,load eight bytes from the SRC2 alignment | |
| 150 | − * boundary,then compare with the relative bytes from SRC1. | |
| 151 | − * If all 8 bytes are equal,then start the second part's comparison. | |
| 152 | − * Otherwise finish the comparison. | |
| 153 | − * This special handle can garantee all the accesses are in the | |
| 154 | − * thread/task space in avoid to overrange access. | |
| 155 | − */ | |
| 156 | − ldr data1, [src1,pos] | |
| 157 | − ldr data2, [src2,pos] | |
| 158 | − sub tmp1, data1, zeroones | |
| 159 | − orr tmp2, data1, #REP8_7f | |
| 160 | − bic has_nul, tmp1, tmp2 | |
| 161 | − eor diff, data1, data2 /* Non-zero if differences found. */ | |
| 162 | − orr syndrome, diff, has_nul | |
| 163 | − cbnz syndrome, .Lcal_cmpresult | |
| 164 | − | |
| 165 | − /*The second part process*/ | |
| 166 | − ldr data1, [src1], #8 | |
| 167 | − ldr data2, [src2], #8 | |
| 168 | − sub tmp1, data1, zeroones | |
| 169 | − orr tmp2, data1, #REP8_7f | |
| 170 | − bic has_nul, tmp1, tmp2 | |
| 171 | − eor diff, data1, data2 /* Non-zero if differences found. */ | |
| 165 | + eor diff, data1, data2 /* Non-zero if differences found. */ | |
| 166 | + bic has_nul, tmp1, tmp2 /* Non-zero if NUL terminator. */ | |
| 172 | 167 | orr syndrome, diff, has_nul |
| 173 | − cbz syndrome, .Lloopcmp_proc | |
| 168 | + cbz syndrome, L(loop_misaligned) | |
| 169 | + b L(end) | |
| 174 | 170 | |
| 175 | −.Lcal_cmpresult: | |
| 176 | − /* | |
| 177 | − * reversed the byte-order as big-endian,then CLZ can find the most | |
| 178 | − * significant zero bits. | |
| 179 | − */ | |
| 180 | −CPU_LE( rev syndrome, syndrome ) | |
| 181 | −CPU_LE( rev data1, data1 ) | |
| 182 | −CPU_LE( rev data2, data2 ) | |
| 183 | − | |
| 184 | − /* | |
| 185 | − * For big-endian we cannot use the trick with the syndrome value | |
| 186 | − * as carry-propagation can corrupt the upper bits if the trailing | |
| 187 | − * bytes in the string contain 0x01. | |
| 188 | − * However, if there is no NUL byte in the dword, we can generate | |
| 189 | − * the result directly. We cannot just subtract the bytes as the | |
| 190 | − * MSB might be significant. | |
| 191 | − */ | |
| 192 | −CPU_BE( cbnz has_nul, 1f ) | |
| 193 | −CPU_BE( cmp data1, data2 ) | |
| 194 | −CPU_BE( cset result, ne ) | |
| 195 | −CPU_BE( cneg result, result, lo ) | |
| 196 | −CPU_BE( ret ) | |
| 197 | −CPU_BE( 1: ) | |
| 198 | − /*Re-compute the NUL-byte detection, using a byte-reversed value. */ | |
| 199 | −CPU_BE( rev tmp3, data1 ) | |
| 200 | −CPU_BE( sub tmp1, tmp3, zeroones ) | |
| 201 | −CPU_BE( orr tmp2, tmp3, #REP8_7f ) | |
| 202 | −CPU_BE( bic has_nul, tmp1, tmp2 ) | |
| 203 | −CPU_BE( rev has_nul, has_nul ) | |
| 204 | −CPU_BE( orr syndrome, diff, has_nul ) | |
| 205 | − | |
| 206 | − clz pos, syndrome | |
| 207 | − /* | |
| 208 | − * The MS-non-zero bit of the syndrome marks either the first bit | |
| 209 | − * that is different, or the top bit of the first zero byte. | |
| 210 | − * Shifting left now will bring the critical information into the | |
| 211 | − * top bits. | |
| 212 | − */ | |
| 213 | − lsl data1, data1, pos | |
| 214 | − lsl data2, data2, pos | |
| 215 | − /* | |
| 216 | − * But we need to zero-extend (char is unsigned) the value and then | |
| 217 | − * perform a signed 32-bit subtraction. | |
| 218 | − */ | |
| 219 | − lsr data1, data1, #56 | |
| 220 | − sub result, data1, data2, lsr #56 | |
| 171 | +L(done): | |
| 172 | + sub result, data1, data2 | |
| 221 | 173 | ret |
| 174 | + | |
| 222 | 175 | SYM_FUNC_END_PI(strcmp) |
| 223 | 176 | EXPORT_SYMBOL_NOKASAN(strcmp) |
| @@ -1,115 +1,203 @@ | ||
| 1 | 1 | /* SPDX-License-Identifier: GPL-2.0-only */ |
| 2 | 2 | /* |
| 3 | − * Copyright (C) 2013 ARM Ltd. | |
| 4 | − * Copyright (C) 2013 Linaro. | |
| 3 | + * Copyright (c) 2013-2021, Arm Limited. | |
| 5 | 4 | * |
| 6 | − * This code is based on glibc cortex strings work originally authored by Linaro | |
| 7 | − * be found @ | |
| 8 | − * | |
| 9 | − * http://bazaar.launchpad.net/~linaro-toolchain-dev/cortex-strings/trunk/ | |
| 10 | − * files/head:/src/aarch64/ | |
| 5 | + * Adapted from the original at: | |
| 6 | + * https://github.com/ARM-software/optimized-routines/blob/98e4d6a5c13c8e54/string/aarch64/strlen.S | |
| 11 | 7 | */ |
| 12 | 8 | |
| 13 | 9 | #include <linux/linkage.h> |
| 14 | 10 | #include <asm/assembler.h> |
| 15 | 11 | |
| 16 | −/* | |
| 17 | − * calculate the length of a string | |
| 12 | +/* Assumptions: | |
| 18 | 13 | * |
| 19 | − * Parameters: | |
| 20 | − * x0 - const string pointer | |
| 21 | − * Returns: | |
| 22 | − * x0 - the return length of specific string | |
| 14 | + * ARMv8-a, AArch64, unaligned accesses, min page size 4k. | |
| 23 | 15 | */ |
| 24 | 16 | |
| 17 | +#define L(label) .L ## label | |
| 18 | + | |
| 25 | 19 | /* Arguments and results. */ |
| 26 | −srcin .req x0 | |
| 27 | −len .req x0 | |
| 20 | +#define srcin x0 | |
| 21 | +#define len x0 | |
| 28 | 22 | |
| 29 | 23 | /* Locals and temporaries. */ |
| 30 | −src .req x1 | |
| 31 | −data1 .req x2 | |
| 32 | −data2 .req x3 | |
| 33 | −data2a .req x4 | |
| 34 | −has_nul1 .req x5 | |
| 35 | −has_nul2 .req x6 | |
| 36 | −tmp1 .req x7 | |
| 37 | −tmp2 .req x8 | |
| 38 | −tmp3 .req x9 | |
| 39 | −tmp4 .req x10 | |
| 40 | −zeroones .req x11 | |
| 41 | −pos .req x12 | |
| 24 | +#define src x1 | |
| 25 | +#define data1 x2 | |
| 26 | +#define data2 x3 | |
| 27 | +#define has_nul1 x4 | |
| 28 | +#define has_nul2 x5 | |
| 29 | +#define tmp1 x4 | |
| 30 | +#define tmp2 x5 | |
| 31 | +#define tmp3 x6 | |
| 32 | +#define tmp4 x7 | |
| 33 | +#define zeroones x8 | |
| 34 | + | |
| 35 | + /* NUL detection works on the principle that (X - 1) & (~X) & 0x80 | |
| 36 | + (=> (X - 1) & ~(X | 0x7f)) is non-zero iff a byte is zero, and | |
| 37 | + can be done in parallel across the entire word. A faster check | |
| 38 | + (X - 1) & 0x80 is zero for non-NUL ASCII characters, but gives | |
| 39 | + false hits for characters 129..255. */ | |
| 42 | 40 | |
| 43 | 41 | #define REP8_01 0x0101010101010101 |
| 44 | 42 | #define REP8_7f 0x7f7f7f7f7f7f7f7f |
| 45 | 43 | #define REP8_80 0x8080808080808080 |
| 46 | 44 | |
| 45 | +#define MIN_PAGE_SIZE 4096 | |
| 46 | + | |
| 47 | + /* Since strings are short on average, we check the first 16 bytes | |
| 48 | + of the string for a NUL character. In order to do an unaligned ldp | |
| 49 | + safely we have to do a page cross check first. If there is a NUL | |
| 50 | + byte we calculate the length from the 2 8-byte words using | |
| 51 | + conditional select to reduce branch mispredictions (it is unlikely | |
| 52 | + strlen will be repeatedly called on strings with the same length). | |
| 53 | + | |
| 54 | + If the string is longer than 16 bytes, we align src so don't need | |
| 55 | + further page cross checks, and process 32 bytes per iteration | |
| 56 | + using the fast NUL check. If we encounter non-ASCII characters, | |
| 57 | + fallback to a second loop using the full NUL check. | |
| 58 | + | |
| 59 | + If the page cross check fails, we read 16 bytes from an aligned | |
| 60 | + address, remove any characters before the string, and continue | |
| 61 | + in the main loop using aligned loads. Since strings crossing a | |
| 62 | + page in the first 16 bytes are rare (probability of | |
| 63 | + 16/MIN_PAGE_SIZE ~= 0.4%), this case does not need to be optimized. | |
| 64 | + | |
| 65 | + AArch64 systems have a minimum page size of 4k. We don't bother | |
| 66 | + checking for larger page sizes - the cost of setting up the correct | |
| 67 | + page size is just not worth the extra gain from a small reduction in | |
| 68 | + the cases taking the slow path. Note that we only care about | |
| 69 | + whether the first fetch, which may be misaligned, crosses a page | |
| 70 | + boundary. */ | |
| 71 | + | |
| 47 | 72 | SYM_FUNC_START_WEAK_PI(strlen) |
| 48 | − mov zeroones, #REP8_01 | |
| 49 | − bic src, srcin, #15 | |
| 50 | − ands tmp1, srcin, #15 | |
| 51 | − b.ne .Lmisaligned | |
| 52 | − /* | |
| 53 | − * NUL detection works on the principle that (X - 1) & (~X) & 0x80 | |
| 54 | − * (=> (X - 1) & ~(X | 0x7f)) is non-zero iff a byte is zero, and | |
| 55 | − * can be done in parallel across the entire word. | |
| 56 | − */ | |
| 57 | − /* | |
| 58 | − * The inner loop deals with two Dwords at a time. This has a | |
| 59 | − * slightly higher start-up cost, but we should win quite quickly, | |
| 60 | − * especially on cores with a high number of issue slots per | |
| 61 | − * cycle, as we get much better parallelism out of the operations. | |
| 62 | − */ | |
| 63 | −.Lloop: | |
| 64 | − ldp data1, data2, [src], #16 | |
| 65 | −.Lrealigned: | |
| 73 | + and tmp1, srcin, MIN_PAGE_SIZE - 1 | |
| 74 | + mov zeroones, REP8_01 | |
| 75 | + cmp tmp1, MIN_PAGE_SIZE - 16 | |
| 76 | + b.gt L(page_cross) | |
| 77 | + ldp data1, data2, [srcin] | |
| 78 | +#ifdef __AARCH64EB__ | |
| 79 | + /* For big-endian, carry propagation (if the final byte in the | |
| 80 | + string is 0x01) means we cannot use has_nul1/2 directly. | |
| 81 | + Since we expect strings to be small and early-exit, | |
| 82 | + byte-swap the data now so has_null1/2 will be correct. */ | |
| 83 | + rev data1, data1 | |
| 84 | + rev data2, data2 | |
| 85 | +#endif | |
| 66 | 86 | sub tmp1, data1, zeroones |
| 67 | − orr tmp2, data1, #REP8_7f | |
| 87 | + orr tmp2, data1, REP8_7f | |
| 68 | 88 | sub tmp3, data2, zeroones |
| 69 | − orr tmp4, data2, #REP8_7f | |
| 70 | − bic has_nul1, tmp1, tmp2 | |
| 71 | − bics has_nul2, tmp3, tmp4 | |
| 72 | − ccmp has_nul1, #0, #0, eq /* NZCV = 0000 */ | |
| 73 | − b.eq .Lloop | |
| 89 | + orr tmp4, data2, REP8_7f | |
| 90 | + bics has_nul1, tmp1, tmp2 | |
| 91 | + bic has_nul2, tmp3, tmp4 | |
| 92 | + ccmp has_nul2, 0, 0, eq | |
| 93 | + beq L(main_loop_entry) | |
| 94 | + | |
| 95 | + /* Enter with C = has_nul1 == 0. */ | |
| 96 | + csel has_nul1, has_nul1, has_nul2, cc | |
| 97 | + mov len, 8 | |
| 98 | + rev has_nul1, has_nul1 | |
| 99 | + clz tmp1, has_nul1 | |
| 100 | + csel len, xzr, len, cc | |
| 101 | + add len, len, tmp1, lsr 3 | |
| 102 | + ret | |
| 74 | 103 | |
| 104 | + /* The inner loop processes 32 bytes per iteration and uses the fast | |
| 105 | + NUL check. If we encounter non-ASCII characters, use a second | |
| 106 | + loop with the accurate NUL check. */ | |
| 107 | + .p2align 4 | |
| 108 | +L(main_loop_entry): | |
| 109 | + bic src, srcin, 15 | |
| 110 | + sub src, src, 16 | |
| 111 | +L(main_loop): | |
| 112 | + ldp data1, data2, [src, 32]! | |
| 113 | +L(page_cross_entry): | |
| 114 | + sub tmp1, data1, zeroones | |
| 115 | + sub tmp3, data2, zeroones | |
| 116 | + orr tmp2, tmp1, tmp3 | |
| 117 | + tst tmp2, zeroones, lsl 7 | |
| 118 | + bne 1f | |
| 119 | + ldp data1, data2, [src, 16] | |
| 120 | + sub tmp1, data1, zeroones | |
| 121 | + sub tmp3, data2, zeroones | |
| 122 | + orr tmp2, tmp1, tmp3 | |
| 123 | + tst tmp2, zeroones, lsl 7 | |
| 124 | + beq L(main_loop) | |
| 125 | + add src, src, 16 | |
| 126 | +1: | |
| 127 | + /* The fast check failed, so do the slower, accurate NUL check. */ | |
| 128 | + orr tmp2, data1, REP8_7f | |
| 129 | + orr tmp4, data2, REP8_7f | |
| 130 | + bics has_nul1, tmp1, tmp2 | |
| 131 | + bic has_nul2, tmp3, tmp4 | |
| 132 | + ccmp has_nul2, 0, 0, eq | |
| 133 | + beq L(nonascii_loop) | |
| 134 | + | |
| 135 | + /* Enter with C = has_nul1 == 0. */ | |
| 136 | +L(tail): | |
| 137 | +#ifdef __AARCH64EB__ | |
| 138 | + /* For big-endian, carry propagation (if the final byte in the | |
| 139 | + string is 0x01) means we cannot use has_nul1/2 directly. The | |
| 140 | + easiest way to get the correct byte is to byte-swap the data | |
| 141 | + and calculate the syndrome a second time. */ | |
| 142 | + csel data1, data1, data2, cc | |
| 143 | + rev data1, data1 | |
| 144 | + sub tmp1, data1, zeroones | |
| 145 | + orr tmp2, data1, REP8_7f | |
| 146 | + bic has_nul1, tmp1, tmp2 | |
| 147 | +#else | |
| 148 | + csel has_nul1, has_nul1, has_nul2, cc | |
| 149 | +#endif | |
| 75 | 150 | sub len, src, srcin |
| 76 | − cbz has_nul1, .Lnul_in_data2 | |
| 77 | −CPU_BE( mov data2, data1 ) /*prepare data to re-calculate the syndrome*/ | |
| 78 | − sub len, len, #8 | |
| 79 | − mov has_nul2, has_nul1 | |
| 80 | −.Lnul_in_data2: | |
| 81 | − /* | |
| 82 | − * For big-endian, carry propagation (if the final byte in the | |
| 83 | − * string is 0x01) means we cannot use has_nul directly. The | |
| 84 | − * easiest way to get the correct byte is to byte-swap the data | |
| 85 | − * and calculate the syndrome a second time. | |
| 86 | − */ | |
| 87 | −CPU_BE( rev data2, data2 ) | |
| 88 | −CPU_BE( sub tmp1, data2, zeroones ) | |
| 89 | −CPU_BE( orr tmp2, data2, #REP8_7f ) | |
| 90 | −CPU_BE( bic has_nul2, tmp1, tmp2 ) | |
| 91 | − | |
| 92 | − sub len, len, #8 | |
| 93 | − rev has_nul2, has_nul2 | |
| 94 | − clz pos, has_nul2 | |
| 95 | − add len, len, pos, lsr #3 /* Bits to bytes. */ | |
| 151 | + rev has_nul1, has_nul1 | |
| 152 | + add tmp2, len, 8 | |
| 153 | + clz tmp1, has_nul1 | |
| 154 | + csel len, len, tmp2, cc | |
| 155 | + add len, len, tmp1, lsr 3 | |
| 96 | 156 | ret |
| 97 | 157 | |
| 98 | −.Lmisaligned: | |
| 99 | − cmp tmp1, #8 | |
| 100 | − neg tmp1, tmp1 | |
| 101 | − ldp data1, data2, [src], #16 | |
| 102 | − lsl tmp1, tmp1, #3 /* Bytes beyond alignment -> bits. */ | |
| 103 | − mov tmp2, #~0 | |
| 104 | − /* Big-endian. Early bytes are at MSB. */ | |
| 105 | −CPU_BE( lsl tmp2, tmp2, tmp1 ) /* Shift (tmp1 & 63). */ | |
| 158 | +L(nonascii_loop): | |
| 159 | + ldp data1, data2, [src, 16]! | |
| 160 | + sub tmp1, data1, zeroones | |
| 161 | + orr tmp2, data1, REP8_7f | |
| 162 | + sub tmp3, data2, zeroones | |
| 163 | + orr tmp4, data2, REP8_7f | |
| 164 | + bics has_nul1, tmp1, tmp2 | |
| 165 | + bic has_nul2, tmp3, tmp4 | |
| 166 | + ccmp has_nul2, 0, 0, eq | |
| 167 | + bne L(tail) | |
| 168 | + ldp data1, data2, [src, 16]! | |
| 169 | + sub tmp1, data1, zeroones | |
| 170 | + orr tmp2, data1, REP8_7f | |
| 171 | + sub tmp3, data2, zeroones | |
| 172 | + orr tmp4, data2, REP8_7f | |
| 173 | + bics has_nul1, tmp1, tmp2 | |
| 174 | + bic has_nul2, tmp3, tmp4 | |
| 175 | + ccmp has_nul2, 0, 0, eq | |
| 176 | + beq L(nonascii_loop) | |
| 177 | + b L(tail) | |
| 178 | + | |
| 179 | + /* Load 16 bytes from [srcin & ~15] and force the bytes that precede | |
| 180 | + srcin to 0x7f, so we ignore any NUL bytes before the string. | |
| 181 | + Then continue in the aligned loop. */ | |
| 182 | +L(page_cross): | |
| 183 | + bic src, srcin, 15 | |
| 184 | + ldp data1, data2, [src] | |
| 185 | + lsl tmp1, srcin, 3 | |
| 186 | + mov tmp4, -1 | |
| 187 | +#ifdef __AARCH64EB__ | |
| 188 | + /* Big-endian. Early bytes are at MSB. */ | |
| 189 | + lsr tmp1, tmp4, tmp1 /* Shift (tmp1 & 63). */ | |
| 190 | +#else | |
| 106 | 191 | /* Little-endian. Early bytes are at LSB. */ |
| 107 | −CPU_LE( lsr tmp2, tmp2, tmp1 ) /* Shift (tmp1 & 63). */ | |
| 192 | + lsl tmp1, tmp4, tmp1 /* Shift (tmp1 & 63). */ | |
| 193 | +#endif | |
| 194 | + orr tmp1, tmp1, REP8_80 | |
| 195 | + orn data1, data1, tmp1 | |
| 196 | + orn tmp2, data2, tmp1 | |
| 197 | + tst srcin, 8 | |
| 198 | + csel data1, data1, tmp4, eq | |
| 199 | + csel data2, data2, tmp2, eq | |
| 200 | + b L(page_cross_entry) | |
| 108 | 201 | |
| 109 | − orr data1, data1, tmp2 | |
| 110 | − orr data2a, data2, tmp2 | |
| 111 | − csinv data1, data1, xzr, le | |
| 112 | − csel data2, data2, data2a, le | |
| 113 | − b .Lrealigned | |
| 114 | 202 | SYM_FUNC_END_PI(strlen) |
| 115 | 203 | EXPORT_SYMBOL_NOKASAN(strlen) |
| @@ -1,299 +1,261 @@ | ||
| 1 | 1 | /* SPDX-License-Identifier: GPL-2.0-only */ |
| 2 | 2 | /* |
| 3 | − * Copyright (C) 2013 ARM Ltd. | |
| 4 | − * Copyright (C) 2013 Linaro. | |
| 3 | + * Copyright (c) 2013-2021, Arm Limited. | |
| 5 | 4 | * |
| 6 | − * This code is based on glibc cortex strings work originally authored by Linaro | |
| 7 | − * be found @ | |
| 8 | − * | |
| 9 | − * http://bazaar.launchpad.net/~linaro-toolchain-dev/cortex-strings/trunk/ | |
| 10 | − * files/head:/src/aarch64/ | |
| 5 | + * Adapted from the original at: | |
| 6 | + * https://github.com/ARM-software/optimized-routines/blob/e823e3abf5f89ecb/string/aarch64/strncmp.S | |
| 11 | 7 | */ |
| 12 | 8 | |
| 13 | 9 | #include <linux/linkage.h> |
| 14 | 10 | #include <asm/assembler.h> |
| 15 | 11 | |
| 16 | −/* | |
| 17 | − * compare two strings | |
| 12 | +/* Assumptions: | |
| 18 | 13 | * |
| 19 | − * Parameters: | |
| 20 | − * x0 - const string 1 pointer | |
| 21 | − * x1 - const string 2 pointer | |
| 22 | − * x2 - the maximal length to be compared | |
| 23 | − * Returns: | |
| 24 | − * x0 - an integer less than, equal to, or greater than zero if s1 is found, | |
| 25 | − * respectively, to be less than, to match, or be greater than s2. | |
| 14 | + * ARMv8-a, AArch64 | |
| 26 | 15 | */ |
| 27 | 16 | |
| 17 | +#define L(label) .L ## label | |
| 18 | + | |
| 28 | 19 | #define REP8_01 0x0101010101010101 |
| 29 | 20 | #define REP8_7f 0x7f7f7f7f7f7f7f7f |
| 30 | 21 | #define REP8_80 0x8080808080808080 |
| 31 | 22 | |
| 32 | 23 | /* Parameters and result. */ |
| 33 | −src1 .req x0 | |
| 34 | −src2 .req x1 | |
| 35 | −limit .req x2 | |
| 36 | −result .req x0 | |
| 24 | +#define src1 x0 | |
| 25 | +#define src2 x1 | |
| 26 | +#define limit x2 | |
| 27 | +#define result x0 | |
| 37 | 28 | |
| 38 | 29 | /* Internal variables. */ |
| 39 | −data1 .req x3 | |
| 40 | −data1w .req w3 | |
| 41 | −data2 .req x4 | |
| 42 | −data2w .req w4 | |
| 43 | −has_nul .req x5 | |
| 44 | −diff .req x6 | |
| 45 | −syndrome .req x7 | |
| 46 | −tmp1 .req x8 | |
| 47 | −tmp2 .req x9 | |
| 48 | −tmp3 .req x10 | |
| 49 | −zeroones .req x11 | |
| 50 | −pos .req x12 | |
| 51 | −limit_wd .req x13 | |
| 52 | −mask .req x14 | |
| 53 | −endloop .req x15 | |
| 30 | +#define data1 x3 | |
| 31 | +#define data1w w3 | |
| 32 | +#define data2 x4 | |
| 33 | +#define data2w w4 | |
| 34 | +#define has_nul x5 | |
| 35 | +#define diff x6 | |
| 36 | +#define syndrome x7 | |
| 37 | +#define tmp1 x8 | |
| 38 | +#define tmp2 x9 | |
| 39 | +#define tmp3 x10 | |
| 40 | +#define zeroones x11 | |
| 41 | +#define pos x12 | |
| 42 | +#define limit_wd x13 | |
| 43 | +#define mask x14 | |
| 44 | +#define endloop x15 | |
| 45 | +#define count mask | |
| 54 | 46 | |
| 55 | 47 | SYM_FUNC_START_WEAK_PI(strncmp) |
| 56 | − cbz limit, .Lret0 | |
| 48 | + cbz limit, L(ret0) | |
| 57 | 49 | eor tmp1, src1, src2 |
| 58 | 50 | mov zeroones, #REP8_01 |
| 59 | 51 | tst tmp1, #7 |
| 60 | − b.ne .Lmisaligned8 | |
| 61 | − ands tmp1, src1, #7 | |
| 62 | − b.ne .Lmutual_align | |
| 52 | + and count, src1, #7 | |
| 53 | + b.ne L(misaligned8) | |
| 54 | + cbnz count, L(mutual_align) | |
| 63 | 55 | /* Calculate the number of full and partial words -1. */ |
| 64 | − /* | |
| 65 | − * when limit is mulitply of 8, if not sub 1, | |
| 66 | − * the judgement of last dword will wrong. | |
| 67 | − */ | |
| 68 | − sub limit_wd, limit, #1 /* limit != 0, so no underflow. */ | |
| 69 | − lsr limit_wd, limit_wd, #3 /* Convert to Dwords. */ | |
| 56 | + sub limit_wd, limit, #1 /* limit != 0, so no underflow. */ | |
| 57 | + lsr limit_wd, limit_wd, #3 /* Convert to Dwords. */ | |
| 70 | 58 | |
| 71 | − /* | |
| 72 | − * NUL detection works on the principle that (X - 1) & (~X) & 0x80 | |
| 73 | − * (=> (X - 1) & ~(X | 0x7f)) is non-zero iff a byte is zero, and | |
| 74 | − * can be done in parallel across the entire word. | |
| 75 | − */ | |
| 76 | −.Lloop_aligned: | |
| 59 | + /* NUL detection works on the principle that (X - 1) & (~X) & 0x80 | |
| 60 | + (=> (X - 1) & ~(X | 0x7f)) is non-zero iff a byte is zero, and | |
| 61 | + can be done in parallel across the entire word. */ | |
| 62 | + .p2align 4 | |
| 63 | +L(loop_aligned): | |
| 77 | 64 | ldr data1, [src1], #8 |
| 78 | 65 | ldr data2, [src2], #8 |
| 79 | −.Lstart_realigned: | |
| 66 | +L(start_realigned): | |
| 80 | 67 | subs limit_wd, limit_wd, #1 |
| 81 | 68 | sub tmp1, data1, zeroones |
| 82 | 69 | orr tmp2, data1, #REP8_7f |
| 83 | − eor diff, data1, data2 /* Non-zero if differences found. */ | |
| 84 | − csinv endloop, diff, xzr, pl /* Last Dword or differences.*/ | |
| 85 | − bics has_nul, tmp1, tmp2 /* Non-zero if NUL terminator. */ | |
| 70 | + eor diff, data1, data2 /* Non-zero if differences found. */ | |
| 71 | + csinv endloop, diff, xzr, pl /* Last Dword or differences. */ | |
| 72 | + bics has_nul, tmp1, tmp2 /* Non-zero if NUL terminator. */ | |
| 86 | 73 | ccmp endloop, #0, #0, eq |
| 87 | − b.eq .Lloop_aligned | |
| 74 | + b.eq L(loop_aligned) | |
| 75 | + /* End of main loop */ | |
| 88 | 76 | |
| 89 | − /*Not reached the limit, must have found the end or a diff. */ | |
| 90 | − tbz limit_wd, #63, .Lnot_limit | |
| 77 | + /* Not reached the limit, must have found the end or a diff. */ | |
| 78 | + tbz limit_wd, #63, L(not_limit) | |
| 91 | 79 | |
| 92 | 80 | /* Limit % 8 == 0 => all bytes significant. */ |
| 93 | 81 | ands limit, limit, #7 |
| 94 | − b.eq .Lnot_limit | |
| 82 | + b.eq L(not_limit) | |
| 95 | 83 | |
| 96 | − lsl limit, limit, #3 /* Bits -> bytes. */ | |
| 84 | + lsl limit, limit, #3 /* Bits -> bytes. */ | |
| 97 | 85 | mov mask, #~0 |
| 98 | −CPU_BE( lsr mask, mask, limit ) | |
| 99 | −CPU_LE( lsl mask, mask, limit ) | |
| 86 | +#ifdef __AARCH64EB__ | |
| 87 | + lsr mask, mask, limit | |
| 88 | +#else | |
| 89 | + lsl mask, mask, limit | |
| 90 | +#endif | |
| 100 | 91 | bic data1, data1, mask |
| 101 | 92 | bic data2, data2, mask |
| 102 | 93 | |
| 103 | 94 | /* Make sure that the NUL byte is marked in the syndrome. */ |
| 104 | 95 | orr has_nul, has_nul, mask |
| 105 | 96 | |
| 106 | −.Lnot_limit: | |
| 97 | +L(not_limit): | |
| 107 | 98 | orr syndrome, diff, has_nul |
| 108 | − b .Lcal_cmpresult | |
| 109 | 99 | |
| 110 | −.Lmutual_align: | |
| 111 | − /* | |
| 112 | − * Sources are mutually aligned, but are not currently at an | |
| 113 | − * alignment boundary. Round down the addresses and then mask off | |
| 114 | − * the bytes that precede the start point. | |
| 115 | − * We also need to adjust the limit calculations, but without | |
| 116 | − * overflowing if the limit is near ULONG_MAX. | |
| 117 | − */ | |
| 100 | +#ifndef __AARCH64EB__ | |
| 101 | + rev syndrome, syndrome | |
| 102 | + rev data1, data1 | |
| 103 | + /* The MS-non-zero bit of the syndrome marks either the first bit | |
| 104 | + that is different, or the top bit of the first zero byte. | |
| 105 | + Shifting left now will bring the critical information into the | |
| 106 | + top bits. */ | |
| 107 | + clz pos, syndrome | |
| 108 | + rev data2, data2 | |
| 109 | + lsl data1, data1, pos | |
| 110 | + lsl data2, data2, pos | |
| 111 | + /* But we need to zero-extend (char is unsigned) the value and then | |
| 112 | + perform a signed 32-bit subtraction. */ | |
| 113 | + lsr data1, data1, #56 | |
| 114 | + sub result, data1, data2, lsr #56 | |
| 115 | + ret | |
| 116 | +#else | |
| 117 | + /* For big-endian we cannot use the trick with the syndrome value | |
| 118 | + as carry-propagation can corrupt the upper bits if the trailing | |
| 119 | + bytes in the string contain 0x01. */ | |
| 120 | + /* However, if there is no NUL byte in the dword, we can generate | |
| 121 | + the result directly. We can't just subtract the bytes as the | |
| 122 | + MSB might be significant. */ | |
| 123 | + cbnz has_nul, 1f | |
| 124 | + cmp data1, data2 | |
| 125 | + cset result, ne | |
| 126 | + cneg result, result, lo | |
| 127 | + ret | |
| 128 | +1: | |
| 129 | + /* Re-compute the NUL-byte detection, using a byte-reversed value. */ | |
| 130 | + rev tmp3, data1 | |
| 131 | + sub tmp1, tmp3, zeroones | |
| 132 | + orr tmp2, tmp3, #REP8_7f | |
| 133 | + bic has_nul, tmp1, tmp2 | |
| 134 | + rev has_nul, has_nul | |
| 135 | + orr syndrome, diff, has_nul | |
| 136 | + clz pos, syndrome | |
| 137 | + /* The MS-non-zero bit of the syndrome marks either the first bit | |
| 138 | + that is different, or the top bit of the first zero byte. | |
| 139 | + Shifting left now will bring the critical information into the | |
| 140 | + top bits. */ | |
| 141 | + lsl data1, data1, pos | |
| 142 | + lsl data2, data2, pos | |
| 143 | + /* But we need to zero-extend (char is unsigned) the value and then | |
| 144 | + perform a signed 32-bit subtraction. */ | |
| 145 | + lsr data1, data1, #56 | |
| 146 | + sub result, data1, data2, lsr #56 | |
| 147 | + ret | |
| 148 | +#endif | |
| 149 | + | |
| 150 | +L(mutual_align): | |
| 151 | + /* Sources are mutually aligned, but are not currently at an | |
| 152 | + alignment boundary. Round down the addresses and then mask off | |
| 153 | + the bytes that precede the start point. | |
| 154 | + We also need to adjust the limit calculations, but without | |
| 155 | + overflowing if the limit is near ULONG_MAX. */ | |
| 118 | 156 | bic src1, src1, #7 |
| 119 | 157 | bic src2, src2, #7 |
| 120 | 158 | ldr data1, [src1], #8 |
| 121 | − neg tmp3, tmp1, lsl #3 /* 64 - bits(bytes beyond align). */ | |
| 159 | + neg tmp3, count, lsl #3 /* 64 - bits(bytes beyond align). */ | |
| 122 | 160 | ldr data2, [src2], #8 |
| 123 | 161 | mov tmp2, #~0 |
| 124 | − sub limit_wd, limit, #1 /* limit != 0, so no underflow. */ | |
| 162 | + sub limit_wd, limit, #1 /* limit != 0, so no underflow. */ | |
| 163 | +#ifdef __AARCH64EB__ | |
| 125 | 164 | /* Big-endian. Early bytes are at MSB. */ |
| 126 | −CPU_BE( lsl tmp2, tmp2, tmp3 ) /* Shift (tmp1 & 63). */ | |
| 165 | + lsl tmp2, tmp2, tmp3 /* Shift (count & 63). */ | |
| 166 | +#else | |
| 127 | 167 | /* Little-endian. Early bytes are at LSB. */ |
| 128 | −CPU_LE( lsr tmp2, tmp2, tmp3 ) /* Shift (tmp1 & 63). */ | |
| 129 | − | |
| 168 | + lsr tmp2, tmp2, tmp3 /* Shift (count & 63). */ | |
| 169 | +#endif | |
| 130 | 170 | and tmp3, limit_wd, #7 |
| 131 | 171 | lsr limit_wd, limit_wd, #3 |
| 132 | − /* Adjust the limit. Only low 3 bits used, so overflow irrelevant.*/ | |
| 133 | − add limit, limit, tmp1 | |
| 134 | − add tmp3, tmp3, tmp1 | |
| 172 | + /* Adjust the limit. Only low 3 bits used, so overflow irrelevant. */ | |
| 173 | + add limit, limit, count | |
| 174 | + add tmp3, tmp3, count | |
| 135 | 175 | orr data1, data1, tmp2 |
| 136 | 176 | orr data2, data2, tmp2 |
| 137 | 177 | add limit_wd, limit_wd, tmp3, lsr #3 |
| 138 | − b .Lstart_realigned | |
| 178 | + b L(start_realigned) | |
| 179 | + | |
| 180 | + .p2align 4 | |
| 181 | + /* Don't bother with dwords for up to 16 bytes. */ | |
| 182 | +L(misaligned8): | |
| 183 | + cmp limit, #16 | |
| 184 | + b.hs L(try_misaligned_words) | |
| 139 | 185 | |
| 140 | −/*when src1 offset is not equal to src2 offset...*/ | |
| 141 | −.Lmisaligned8: | |
| 142 | − cmp limit, #8 | |
| 143 | − b.lo .Ltiny8proc /*limit < 8... */ | |
| 144 | − /* | |
| 145 | − * Get the align offset length to compare per byte first. | |
| 146 | − * After this process, one string's address will be aligned.*/ | |
| 147 | − and tmp1, src1, #7 | |
| 148 | − neg tmp1, tmp1 | |
| 149 | − add tmp1, tmp1, #8 | |
| 150 | − and tmp2, src2, #7 | |
| 151 | − neg tmp2, tmp2 | |
| 152 | − add tmp2, tmp2, #8 | |
| 153 | − subs tmp3, tmp1, tmp2 | |
| 154 | − csel pos, tmp1, tmp2, hi /*Choose the maximum. */ | |
| 155 | − /* | |
| 156 | − * Here, limit is not less than 8, so directly run .Ltinycmp | |
| 157 | − * without checking the limit.*/ | |
| 158 | − sub limit, limit, pos | |
| 159 | −.Ltinycmp: | |
| 186 | +L(byte_loop): | |
| 187 | + /* Perhaps we can do better than this. */ | |
| 160 | 188 | ldrb data1w, [src1], #1 |
| 161 | 189 | ldrb data2w, [src2], #1 |
| 162 | − subs pos, pos, #1 | |
| 163 | − ccmp data1w, #1, #0, ne /* NZCV = 0b0000. */ | |
| 164 | − ccmp data1w, data2w, #0, cs /* NZCV = 0b0000. */ | |
| 165 | − b.eq .Ltinycmp | |
| 166 | − cbnz pos, 1f /*find the null or unequal...*/ | |
| 167 | − cmp data1w, #1 | |
| 168 | − ccmp data1w, data2w, #0, cs | |
| 169 | − b.eq .Lstart_align /*the last bytes are equal....*/ | |
| 170 | −1: | |
| 190 | + subs limit, limit, #1 | |
| 191 | + ccmp data1w, #1, #0, hi /* NZCV = 0b0000. */ | |
| 192 | + ccmp data1w, data2w, #0, cs /* NZCV = 0b0000. */ | |
| 193 | + b.eq L(byte_loop) | |
| 194 | +L(done): | |
| 171 | 195 | sub result, data1, data2 |
| 172 | 196 | ret |
| 173 | − | |
| 174 | −.Lstart_align: | |
| 197 | + /* Align the SRC1 to a dword by doing a bytewise compare and then do | |
| 198 | + the dword loop. */ | |
| 199 | +L(try_misaligned_words): | |
| 175 | 200 | lsr limit_wd, limit, #3 |
| 176 | − cbz limit_wd, .Lremain8 | |
| 177 | − /*process more leading bytes to make str1 aligned...*/ | |
| 178 | − ands xzr, src1, #7 | |
| 179 | − b.eq .Lrecal_offset | |
| 180 | − add src1, src1, tmp3 /*tmp3 is positive in this branch.*/ | |
| 181 | − add src2, src2, tmp3 | |
| 182 | − ldr data1, [src1], #8 | |
| 183 | − ldr data2, [src2], #8 | |
| 201 | + cbz count, L(do_misaligned) | |
| 184 | 202 | |
| 185 | − sub limit, limit, tmp3 | |
| 203 | + neg count, count | |
| 204 | + and count, count, #7 | |
| 205 | + sub limit, limit, count | |
| 186 | 206 | lsr limit_wd, limit, #3 |
| 187 | − subs limit_wd, limit_wd, #1 | |
| 188 | 207 | |
| 189 | − sub tmp1, data1, zeroones | |
| 190 | − orr tmp2, data1, #REP8_7f | |
| 191 | − eor diff, data1, data2 /* Non-zero if differences found. */ | |
| 192 | − csinv endloop, diff, xzr, ne/*if limit_wd is 0,will finish the cmp*/ | |
| 193 | − bics has_nul, tmp1, tmp2 | |
| 194 | − ccmp endloop, #0, #0, eq /*has_null is ZERO: no null byte*/ | |
| 195 | − b.ne .Lunequal_proc | |
| 196 | − /*How far is the current str2 from the alignment boundary...*/ | |
| 197 | − and tmp3, tmp3, #7 | |
| 198 | −.Lrecal_offset: | |
| 199 | − neg pos, tmp3 | |
| 200 | −.Lloopcmp_proc: | |
| 201 | − /* | |
| 202 | − * Divide the eight bytes into two parts. First,backwards the src2 | |
| 203 | − * to an alignment boundary,load eight bytes from the SRC2 alignment | |
| 204 | − * boundary,then compare with the relative bytes from SRC1. | |
| 205 | − * If all 8 bytes are equal,then start the second part's comparison. | |
| 206 | − * Otherwise finish the comparison. | |
| 207 | − * This special handle can garantee all the accesses are in the | |
| 208 | − * thread/task space in avoid to overrange access. | |
| 209 | − */ | |
| 210 | − ldr data1, [src1,pos] | |
| 211 | − ldr data2, [src2,pos] | |
| 212 | − sub tmp1, data1, zeroones | |
| 213 | − orr tmp2, data1, #REP8_7f | |
| 214 | − bics has_nul, tmp1, tmp2 /* Non-zero if NUL terminator. */ | |
| 215 | − eor diff, data1, data2 /* Non-zero if differences found. */ | |
| 216 | − csinv endloop, diff, xzr, eq | |
| 217 | − cbnz endloop, .Lunequal_proc | |
| 208 | +L(page_end_loop): | |
| 209 | + ldrb data1w, [src1], #1 | |
| 210 | + ldrb data2w, [src2], #1 | |
| 211 | + cmp data1w, #1 | |
| 212 | + ccmp data1w, data2w, #0, cs /* NZCV = 0b0000. */ | |
| 213 | + b.ne L(done) | |
| 214 | + subs count, count, #1 | |
| 215 | + b.hi L(page_end_loop) | |
| 216 | + | |
| 217 | +L(do_misaligned): | |
| 218 | + /* Prepare ourselves for the next page crossing. Unlike the aligned | |
| 219 | + loop, we fetch 1 less dword because we risk crossing bounds on | |
| 220 | + SRC2. */ | |
| 221 | + mov count, #8 | |
| 222 | + subs limit_wd, limit_wd, #1 | |
| 223 | + b.lo L(done_loop) | |
| 224 | +L(loop_misaligned): | |
| 225 | + and tmp2, src2, #0xff8 | |
| 226 | + eor tmp2, tmp2, #0xff8 | |
| 227 | + cbz tmp2, L(page_end_loop) | |
| 218 | 228 | |
| 219 | − /*The second part process*/ | |
| 220 | 229 | ldr data1, [src1], #8 |
| 221 | 230 | ldr data2, [src2], #8 |
| 222 | − subs limit_wd, limit_wd, #1 | |
| 223 | 231 | sub tmp1, data1, zeroones |
| 224 | 232 | orr tmp2, data1, #REP8_7f |
| 225 | − eor diff, data1, data2 /* Non-zero if differences found. */ | |
| 226 | − csinv endloop, diff, xzr, ne/*if limit_wd is 0,will finish the cmp*/ | |
| 227 | − bics has_nul, tmp1, tmp2 | |
| 228 | − ccmp endloop, #0, #0, eq /*has_null is ZERO: no null byte*/ | |
| 229 | − b.eq .Lloopcmp_proc | |
| 230 | − | |
| 231 | −.Lunequal_proc: | |
| 232 | − orr syndrome, diff, has_nul | |
| 233 | − cbz syndrome, .Lremain8 | |
| 234 | −.Lcal_cmpresult: | |
| 235 | − /* | |
| 236 | − * reversed the byte-order as big-endian,then CLZ can find the most | |
| 237 | − * significant zero bits. | |
| 238 | − */ | |
| 239 | −CPU_LE( rev syndrome, syndrome ) | |
| 240 | −CPU_LE( rev data1, data1 ) | |
| 241 | −CPU_LE( rev data2, data2 ) | |
| 242 | − /* | |
| 243 | − * For big-endian we cannot use the trick with the syndrome value | |
| 244 | − * as carry-propagation can corrupt the upper bits if the trailing | |
| 245 | − * bytes in the string contain 0x01. | |
| 246 | − * However, if there is no NUL byte in the dword, we can generate | |
| 247 | − * the result directly. We can't just subtract the bytes as the | |
| 248 | − * MSB might be significant. | |
| 249 | − */ | |
| 250 | −CPU_BE( cbnz has_nul, 1f ) | |
| 251 | −CPU_BE( cmp data1, data2 ) | |
| 252 | −CPU_BE( cset result, ne ) | |
| 253 | −CPU_BE( cneg result, result, lo ) | |
| 254 | −CPU_BE( ret ) | |
| 255 | −CPU_BE( 1: ) | |
| 256 | − /* Re-compute the NUL-byte detection, using a byte-reversed value.*/ | |
| 257 | −CPU_BE( rev tmp3, data1 ) | |
| 258 | −CPU_BE( sub tmp1, tmp3, zeroones ) | |
| 259 | −CPU_BE( orr tmp2, tmp3, #REP8_7f ) | |
| 260 | −CPU_BE( bic has_nul, tmp1, tmp2 ) | |
| 261 | −CPU_BE( rev has_nul, has_nul ) | |
| 262 | −CPU_BE( orr syndrome, diff, has_nul ) | |
| 263 | − /* | |
| 264 | − * The MS-non-zero bit of the syndrome marks either the first bit | |
| 265 | − * that is different, or the top bit of the first zero byte. | |
| 266 | − * Shifting left now will bring the critical information into the | |
| 267 | − * top bits. | |
| 268 | − */ | |
| 269 | − clz pos, syndrome | |
| 270 | − lsl data1, data1, pos | |
| 271 | − lsl data2, data2, pos | |
| 272 | − /* | |
| 273 | − * But we need to zero-extend (char is unsigned) the value and then | |
| 274 | − * perform a signed 32-bit subtraction. | |
| 275 | − */ | |
| 276 | − lsr data1, data1, #56 | |
| 277 | − sub result, data1, data2, lsr #56 | |
| 278 | − ret | |
| 279 | − | |
| 280 | −.Lremain8: | |
| 281 | − /* Limit % 8 == 0 => all bytes significant. */ | |
| 282 | − ands limit, limit, #7 | |
| 283 | − b.eq .Lret0 | |
| 284 | −.Ltiny8proc: | |
| 285 | − ldrb data1w, [src1], #1 | |
| 286 | − ldrb data2w, [src2], #1 | |
| 287 | − subs limit, limit, #1 | |
| 233 | + eor diff, data1, data2 /* Non-zero if differences found. */ | |
| 234 | + bics has_nul, tmp1, tmp2 /* Non-zero if NUL terminator. */ | |
| 235 | + ccmp diff, #0, #0, eq | |
| 236 | + b.ne L(not_limit) | |
| 237 | + subs limit_wd, limit_wd, #1 | |
| 238 | + b.pl L(loop_misaligned) | |
| 288 | 239 | |
| 289 | − ccmp data1w, #1, #0, ne /* NZCV = 0b0000. */ | |
| 290 | − ccmp data1w, data2w, #0, cs /* NZCV = 0b0000. */ | |
| 291 | − b.eq .Ltiny8proc | |
| 292 | − sub result, data1, data2 | |
| 293 | − ret | |
| 240 | +L(done_loop): | |
| 241 | + /* We found a difference or a NULL before the limit was reached. */ | |
| 242 | + and limit, limit, #7 | |
| 243 | + cbz limit, L(not_limit) | |
| 244 | + /* Read the last word. */ | |
| 245 | + sub src1, src1, 8 | |
| 246 | + sub src2, src2, 8 | |
| 247 | + ldr data1, [src1, limit] | |
| 248 | + ldr data2, [src2, limit] | |
| 249 | + sub tmp1, data1, zeroones | |
| 250 | + orr tmp2, data1, #REP8_7f | |
| 251 | + eor diff, data1, data2 /* Non-zero if differences found. */ | |
| 252 | + bics has_nul, tmp1, tmp2 /* Non-zero if NUL terminator. */ | |
| 253 | + ccmp diff, #0, #0, eq | |
| 254 | + b.ne L(not_limit) | |
| 294 | 255 | |
| 295 | −.Lret0: | |
| 256 | +L(ret0): | |
| 296 | 257 | mov result, #0 |
| 297 | 258 | ret |
| 259 | + | |
| 298 | 260 | SYM_FUNC_END_PI(strncmp) |
| 299 | 261 | EXPORT_SYMBOL_NOKASAN(strncmp) |
| @@ -15,7 +15,7 @@ void memcpy_flushcache(void *dst, const void *src, size_t cnt) · 14 unmodified lines | ||
| 15 | 15 | * barrier to order the cache maintenance against the memcpy. |
| 16 | 16 | */ |
| 17 | 17 | memcpy(dst, src, cnt); |
| 18 | − __clean_dcache_area_pop(dst, cnt); | |
| 18 | + dcache_clean_pop((unsigned long)dst, (unsigned long)dst + cnt); | |
| 19 | 19 | } |
| 20 | 20 | EXPORT_SYMBOL_GPL(memcpy_flushcache); |
| 21 | 21 | |
| @@ -33,6 +33,6 @@ unsigned long __copy_user_flushcache(void *to, const void __user *from, · 11 unmodified lines | ||
| 33 | 33 | rc = raw_copy_from_user(to, from, n); |
| 34 | 34 | |
| 35 | 35 | /* See above */ |
| 36 | − __clean_dcache_area_pop(to, n - rc); | |
| 36 | + dcache_clean_pop((unsigned long)to, (unsigned long)to + n - rc); | |
| 37 | 37 | return rc; |
| 38 | 38 | } |
| @@ -15,7 +15,7 @@ · 14 unmodified lines | ||
| 15 | 15 | #include <asm/asm-uaccess.h> |
| 16 | 16 | |
| 17 | 17 | /* |
| 18 | − * flush_icache_range(start,end) | |
| 18 | + * caches_clean_inval_pou_macro(start,end) [fixup] | |
| 19 | 19 | * |
| 20 | 20 | * Ensure that the I and D caches are coherent within specified region. |
| 21 | 21 | * This is typically used when code has been written to a memory region, |
| @@ -23,12 +23,27 @@ · 1 unmodified line | ||
| 23 | 23 | * |
| 24 | 24 | * - start - virtual start address of region |
| 25 | 25 | * - end - virtual end address of region |
| 26 | + * - fixup - optional label to branch to on user fault | |
| 26 | 27 | */ |
| 27 | −SYM_FUNC_START(__flush_icache_range) | |
| 28 | − /* FALLTHROUGH */ | |
| 28 | +.macro caches_clean_inval_pou_macro, fixup | |
| 29 | +alternative_if ARM64_HAS_CACHE_IDC | |
| 30 | + dsb ishst | |
| 31 | + b .Ldc_skip_\@ | |
| 32 | +alternative_else_nop_endif | |
| 33 | + mov x2, x0 | |
| 34 | + mov x3, x1 | |
| 35 | + dcache_by_line_op cvau, ish, x2, x3, x4, x5, \fixup | |
| 36 | +.Ldc_skip_\@: | |
| 37 | +alternative_if ARM64_HAS_CACHE_DIC | |
| 38 | + isb | |
| 39 | + b .Lic_skip_\@ | |
| 40 | +alternative_else_nop_endif | |
| 41 | + invalidate_icache_by_line x0, x1, x2, x3, \fixup | |
| 42 | +.Lic_skip_\@: | |
| 43 | +.endm | |
| 29 | 44 | |
| 30 | 45 | /* |
| 31 | − * __flush_cache_user_range(start,end) | |
| 46 | + * caches_clean_inval_pou(start,end) | |
| 32 | 47 | * |
| 33 | 48 | * Ensure that the I and D caches are coherent within specified region. |
| 34 | 49 | * This is typically used when code has been written to a memory region, |
| @@ -37,117 +52,103 @@ SYM_FUNC_START(__flush_icache_range) · 2 unmodified lines | ||
| 37 | 52 | * - start - virtual start address of region |
| 38 | 53 | * - end - virtual end address of region |
| 39 | 54 | */ |
| 40 | −SYM_FUNC_START(__flush_cache_user_range) | |
| 55 | +SYM_FUNC_START(caches_clean_inval_pou) | |
| 56 | + caches_clean_inval_pou_macro | |
| 57 | + ret | |
| 58 | +SYM_FUNC_END(caches_clean_inval_pou) | |
| 59 | + | |
| 60 | +/* | |
| 61 | + * caches_clean_inval_user_pou(start,end) | |
| 62 | + * | |
| 63 | + * Ensure that the I and D caches are coherent within specified region. | |
| 64 | + * This is typically used when code has been written to a memory region, | |
| 65 | + * and will be executed. | |
| 66 | + * | |
| 67 | + * - start - virtual start address of region | |
| 68 | + * - end - virtual end address of region | |
| 69 | + */ | |
| 70 | +SYM_FUNC_START(caches_clean_inval_user_pou) | |
| 41 | 71 | uaccess_ttbr0_enable x2, x3, x4 |
| 42 | −alternative_if ARM64_HAS_CACHE_IDC | |
| 43 | − dsb ishst | |
| 44 | − b 7f | |
| 45 | −alternative_else_nop_endif | |
| 46 | − dcache_line_size x2, x3 | |
| 47 | − sub x3, x2, #1 | |
| 48 | − bic x4, x0, x3 | |
| 49 | −1: | |
| 50 | −user_alt 9f, "dc cvau, x4", "dc civac, x4", ARM64_WORKAROUND_CLEAN_CACHE | |
| 51 | − add x4, x4, x2 | |
| 52 | − cmp x4, x1 | |
| 53 | − b.lo 1b | |
| 54 | − dsb ish | |
| 55 | 72 | |
| 56 | −7: | |
| 57 | −alternative_if ARM64_HAS_CACHE_DIC | |
| 58 | − isb | |
| 59 | − b 8f | |
| 60 | −alternative_else_nop_endif | |
| 61 | − invalidate_icache_by_line x0, x1, x2, x3, 9f | |
| 62 | −8: mov x0, #0 | |
| 73 | + caches_clean_inval_pou_macro 2f | |
| 74 | + mov x0, xzr | |
| 63 | 75 | 1: |
| 64 | 76 | uaccess_ttbr0_disable x1, x2 |
| 65 | 77 | ret |
| 66 | −9: | |
| 78 | +2: | |
| 67 | 79 | mov x0, #-EFAULT |
| 68 | 80 | b 1b |
| 69 | −SYM_FUNC_END(__flush_icache_range) | |
| 70 | −SYM_FUNC_END(__flush_cache_user_range) | |
| 81 | +SYM_FUNC_END(caches_clean_inval_user_pou) | |
| 71 | 82 | |
| 72 | 83 | /* |
| 73 | − * invalidate_icache_range(start,end) | |
| 84 | + * icache_inval_pou(start,end) | |
| 74 | 85 | * |
| 75 | 86 | * Ensure that the I cache is invalid within specified region. |
| 76 | 87 | * |
| 77 | 88 | * - start - virtual start address of region |
| 78 | 89 | * - end - virtual end address of region |
| 79 | 90 | */ |
| 80 | −SYM_FUNC_START(invalidate_icache_range) | |
| 91 | +SYM_FUNC_START(icache_inval_pou) | |
| 81 | 92 | alternative_if ARM64_HAS_CACHE_DIC |
| 82 | − mov x0, xzr | |
| 83 | 93 | isb |
| 84 | 94 | ret |
| 85 | 95 | alternative_else_nop_endif |
| 86 | 96 | |
| 87 | − uaccess_ttbr0_enable x2, x3, x4 | |
| 88 | − | |
| 89 | − invalidate_icache_by_line x0, x1, x2, x3, 2f | |
| 90 | − mov x0, xzr | |
| 91 | −1: | |
| 92 | − uaccess_ttbr0_disable x1, x2 | |
| 97 | + invalidate_icache_by_line x0, x1, x2, x3 | |
| 93 | 98 | ret |
| 94 | −2: | |
| 95 | − mov x0, #-EFAULT | |
| 96 | − b 1b | |
| 97 | −SYM_FUNC_END(invalidate_icache_range) | |
| 99 | +SYM_FUNC_END(icache_inval_pou) | |
| 98 | 100 | |
| 99 | 101 | /* |
| 100 | − * __flush_dcache_area(kaddr, size) | |
| 102 | + * dcache_clean_inval_poc(start, end) | |
| 101 | 103 | * |
| 102 | − * Ensure that any D-cache lines for the interval [kaddr, kaddr+size) | |
| 104 | + * Ensure that any D-cache lines for the interval [start, end) | |
| 103 | 105 | * are cleaned and invalidated to the PoC. |
| 104 | 106 | * |
| 105 | − * - kaddr - kernel address | |
| 106 | − * - size - size in question | |
| 107 | + * - start - virtual start address of region | |
| 108 | + * - end - virtual end address of region | |
| 107 | 109 | */ |
| 108 | −SYM_FUNC_START_PI(__flush_dcache_area) | |
| 110 | +SYM_FUNC_START_PI(dcache_clean_inval_poc) | |
| 109 | 111 | dcache_by_line_op civac, sy, x0, x1, x2, x3 |
| 110 | 112 | ret |
| 111 | −SYM_FUNC_END_PI(__flush_dcache_area) | |
| 113 | +SYM_FUNC_END_PI(dcache_clean_inval_poc) | |
| 112 | 114 | |
| 113 | 115 | /* |
| 114 | − * __clean_dcache_area_pou(kaddr, size) | |
| 116 | + * dcache_clean_pou(start, end) | |
| 115 | 117 | * |
| 116 | − * Ensure that any D-cache lines for the interval [kaddr, kaddr+size) | |
| 118 | + * Ensure that any D-cache lines for the interval [start, end) | |
| 117 | 119 | * are cleaned to the PoU. |
| 118 | 120 | * |
| 119 | − * - kaddr - kernel address | |
| 120 | − * - size - size in question | |
| 121 | + * - start - virtual start address of region | |
| 122 | + * - end - virtual end address of region | |
| 121 | 123 | */ |
| 122 | −SYM_FUNC_START(__clean_dcache_area_pou) | |
| 124 | +SYM_FUNC_START(dcache_clean_pou) | |
| 123 | 125 | alternative_if ARM64_HAS_CACHE_IDC |
| 124 | 126 | dsb ishst |
| 125 | 127 | ret |
| 126 | 128 | alternative_else_nop_endif |
| 127 | 129 | dcache_by_line_op cvau, ish, x0, x1, x2, x3 |
| 128 | 130 | ret |
| 129 | −SYM_FUNC_END(__clean_dcache_area_pou) | |
| 131 | +SYM_FUNC_END(dcache_clean_pou) | |
| 130 | 132 | |
| 131 | 133 | /* |
| 132 | − * __inval_dcache_area(kaddr, size) | |
| 134 | + * dcache_inval_poc(start, end) | |
| 133 | 135 | * |
| 134 | − * Ensure that any D-cache lines for the interval [kaddr, kaddr+size) | |
| 136 | + * Ensure that any D-cache lines for the interval [start, end) | |
| 135 | 137 | * are invalidated. Any partial lines at the ends of the interval are |
| 136 | 138 | * also cleaned to PoC to prevent data loss. |
| 137 | 139 | * |
| 138 | − * - kaddr - kernel address | |
| 139 | − * - size - size in question | |
| 140 | + * - start - kernel start address of region | |
| 141 | + * - end - kernel end address of region | |
| 140 | 142 | */ |
| 141 | 143 | SYM_FUNC_START_LOCAL(__dma_inv_area) |
| 142 | −SYM_FUNC_START_PI(__inval_dcache_area) | |
| 144 | +SYM_FUNC_START_PI(dcache_inval_poc) | |
| 143 | 145 | /* FALLTHROUGH */ |
| 144 | 146 | |
| 145 | 147 | /* |
| 146 | − * __dma_inv_area(start, size) | |
| 148 | + * __dma_inv_area(start, end) | |
| 147 | 149 | * - start - virtual start address of region |
| 148 | − * - size - size in question | |
| 150 | + * - end - virtual end address of region | |
| 149 | 151 | */ |
| 150 | − add x1, x1, x0 | |
| 151 | 152 | dcache_line_size x2, x3 |
| 152 | 153 | sub x3, x2, #1 |
| 153 | 154 | tst x1, x3 // end cache line aligned? |
| @@ -165,48 +166,48 @@ SYM_FUNC_START_PI(__inval_dcache_area) · 11 unmodified lines | ||
| 165 | 166 | b.lo 2b |
| 166 | 167 | dsb sy |
| 167 | 168 | ret |
| 168 | −SYM_FUNC_END_PI(__inval_dcache_area) | |
| 169 | +SYM_FUNC_END_PI(dcache_inval_poc) | |
| 169 | 170 | SYM_FUNC_END(__dma_inv_area) |
| 170 | 171 | |
| 171 | 172 | /* |
| 172 | − * __clean_dcache_area_poc(kaddr, size) | |
| 173 | + * dcache_clean_poc(start, end) | |
| 173 | 174 | * |
| 174 | − * Ensure that any D-cache lines for the interval [kaddr, kaddr+size) | |
| 175 | + * Ensure that any D-cache lines for the interval [start, end) | |
| 175 | 176 | * are cleaned to the PoC. |
| 176 | 177 | * |
| 177 | − * - kaddr - kernel address | |
| 178 | − * - size - size in question | |
| 178 | + * - start - virtual start address of region | |
| 179 | + * - end - virtual end address of region | |
| 179 | 180 | */ |
| 180 | 181 | SYM_FUNC_START_LOCAL(__dma_clean_area) |
| 181 | −SYM_FUNC_START_PI(__clean_dcache_area_poc) | |
| 182 | +SYM_FUNC_START_PI(dcache_clean_poc) | |
| 182 | 183 | /* FALLTHROUGH */ |
| 183 | 184 | |
| 184 | 185 | /* |
| 185 | − * __dma_clean_area(start, size) | |
| 186 | + * __dma_clean_area(start, end) | |
| 186 | 187 | * - start - virtual start address of region |
| 187 | − * - size - size in question | |
| 188 | + * - end - virtual end address of region | |
| 188 | 189 | */ |
| 189 | 190 | dcache_by_line_op cvac, sy, x0, x1, x2, x3 |
| 190 | 191 | ret |
| 191 | −SYM_FUNC_END_PI(__clean_dcache_area_poc) | |
| 192 | +SYM_FUNC_END_PI(dcache_clean_poc) | |
| 192 | 193 | SYM_FUNC_END(__dma_clean_area) |
| 193 | 194 | |
| 194 | 195 | /* |
| 195 | − * __clean_dcache_area_pop(kaddr, size) | |
| 196 | + * dcache_clean_pop(start, end) | |
| 196 | 197 | * |
| 197 | − * Ensure that any D-cache lines for the interval [kaddr, kaddr+size) | |
| 198 | + * Ensure that any D-cache lines for the interval [start, end) | |
| 198 | 199 | * are cleaned to the PoP. |
| 199 | 200 | * |
| 200 | − * - kaddr - kernel address | |
| 201 | − * - size - size in question | |
| 201 | + * - start - virtual start address of region | |
| 202 | + * - end - virtual end address of region | |
| 202 | 203 | */ |
| 203 | −SYM_FUNC_START_PI(__clean_dcache_area_pop) | |
| 204 | +SYM_FUNC_START_PI(dcache_clean_pop) | |
| 204 | 205 | alternative_if_not ARM64_HAS_DCPOP |
| 205 | − b __clean_dcache_area_poc | |
| 206 | + b dcache_clean_poc | |
| 206 | 207 | alternative_else_nop_endif |
| 207 | 208 | dcache_by_line_op cvap, sy, x0, x1, x2, x3 |
| 208 | 209 | ret |
| 209 | −SYM_FUNC_END_PI(__clean_dcache_area_pop) | |
| 210 | +SYM_FUNC_END_PI(dcache_clean_pop) | |
| 210 | 211 | |
| 211 | 212 | /* |
| 212 | 213 | * __dma_flush_area(start, size) |
| @@ -217,6 +218,7 @@ SYM_FUNC_END_PI(__clean_dcache_area_pop) · 4 unmodified lines | ||
| 217 | 218 | * - size - size in question |
| 218 | 219 | */ |
| 219 | 220 | SYM_FUNC_START_PI(__dma_flush_area) |
| 221 | + add x1, x0, x1 | |
| 220 | 222 | dcache_by_line_op civac, sy, x0, x1, x2, x3 |
| 221 | 223 | ret |
| 222 | 224 | SYM_FUNC_END_PI(__dma_flush_area) |
| @@ -228,6 +230,7 @@ SYM_FUNC_END_PI(__dma_flush_area) · 5 unmodified lines | ||
| 228 | 230 | * - dir - DMA direction |
| 229 | 231 | */ |
| 230 | 232 | SYM_FUNC_START_PI(__dma_map_area) |
| 233 | + add x1, x0, x1 | |
| 231 | 234 | cmp w2, #DMA_FROM_DEVICE |
| 232 | 235 | b.eq __dma_inv_area |
| 233 | 236 | b __dma_clean_area |
| @@ -240,6 +243,7 @@ SYM_FUNC_END_PI(__dma_map_area) · 6 unmodified lines | ||
| 240 | 243 | * - dir - DMA direction |
| 241 | 244 | */ |
| 242 | 245 | SYM_FUNC_START_PI(__dma_unmap_area) |
| 246 | + add x1, x0, x1 | |
| 243 | 247 | cmp w2, #DMA_TO_DEVICE |
| 244 | 248 | b.ne __dma_inv_area |
| 245 | 249 | ret |
| @@ -402,14 +402,12 @@ static int asids_init(void) · 401 unmodified lines | ||
| 402 | 402 | { |
| 403 | 403 | asid_bits = get_cpu_asid_bits(); |
| 404 | 404 | atomic64_set(&asid_generation, ASID_FIRST_VERSION); |
| 405 | − asid_map = kcalloc(BITS_TO_LONGS(NUM_USER_ASIDS), sizeof(*asid_map), | |
| 406 | − GFP_KERNEL); | |
| 405 | + asid_map = bitmap_zalloc(NUM_USER_ASIDS, GFP_KERNEL); | |
| 407 | 406 | if (!asid_map) |
| 408 | 407 | panic("Failed to allocate bitmap for %lu ASIDs\n", |
| 409 | 408 | NUM_USER_ASIDS); |
| 410 | 409 | |
| 411 | − pinned_asid_map = kcalloc(BITS_TO_LONGS(NUM_USER_ASIDS), | |
| 412 | − sizeof(*pinned_asid_map), GFP_KERNEL); | |
| 410 | + pinned_asid_map = bitmap_zalloc(NUM_USER_ASIDS, GFP_KERNEL); | |
| 413 | 411 | nr_pinned_asids = 0; |
| 414 | 412 | |
| 415 | 413 | /* |
| @@ -99,6 +99,8 @@ static void mem_abort_decode(unsigned int esr) · 98 unmodified lines | ||
| 99 | 99 | pr_alert(" EA = %lu, S1PTW = %lu\n", |
| 100 | 100 | (esr & ESR_ELx_EA) >> ESR_ELx_EA_SHIFT, |
| 101 | 101 | (esr & ESR_ELx_S1PTW) >> ESR_ELx_S1PTW_SHIFT); |
| 102 | + pr_alert(" FSC = 0x%02x: %s\n", (esr & ESR_ELx_FSC), | |
| 103 | + esr_to_fault_info(esr)->name); | |
| 102 | 104 | |
| 103 | 105 | if (esr_is_data_abort(esr)) |
| 104 | 106 | data_abort_decode(esr); |
| @@ -232,13 +234,17 @@ static bool is_el1_instruction_abort(unsigned int esr) · 127 unmodified lines | ||
| 232 | 234 | return ESR_ELx_EC(esr) == ESR_ELx_EC_IABT_CUR; |
| 233 | 235 | } |
| 234 | 236 | |
| 237 | +static bool is_el1_data_abort(unsigned int esr) | |
| 238 | +{ | |
| 239 | + return ESR_ELx_EC(esr) == ESR_ELx_EC_DABT_CUR; | |
| 240 | +} | |
| 241 | + | |
| 235 | 242 | static inline bool is_el1_permission_fault(unsigned long addr, unsigned int esr, |
| 236 | 243 | struct pt_regs *regs) |
| 237 | 244 | { |
| 238 | − unsigned int ec = ESR_ELx_EC(esr); | |
| 239 | 245 | unsigned int fsc_type = esr & ESR_ELx_FSC_TYPE; |
| 240 | 246 | |
| 241 | − if (ec != ESR_ELx_EC_DABT_CUR && ec != ESR_ELx_EC_IABT_CUR) | |
| 247 | + if (!is_el1_data_abort(esr) && !is_el1_instruction_abort(esr)) | |
| 242 | 248 | return false; |
| 243 | 249 | |
| 244 | 250 | if (fsc_type == ESR_ELx_FSC_PERM) |
| @@ -258,7 +264,7 @@ static bool __kprobes is_spurious_el1_translation_fault(unsigned long addr, · 13 unmodified lines | ||
| 258 | 264 | unsigned long flags; |
| 259 | 265 | u64 par, dfsc; |
| 260 | 266 | |
| 261 | − if (ESR_ELx_EC(esr) != ESR_ELx_EC_DABT_CUR || | |
| 267 | + if (!is_el1_data_abort(esr) || | |
| 262 | 268 | (esr & ESR_ELx_FSC_TYPE) != ESR_ELx_FSC_FAULT) |
| 263 | 269 | return false; |
| 264 | 270 | |
| @@ -346,10 +352,9 @@ static void do_tag_recovery(unsigned long addr, unsigned int esr, · 81 unmodified lines | ||
| 346 | 352 | |
| 347 | 353 | static bool is_el1_mte_sync_tag_check_fault(unsigned int esr) |
| 348 | 354 | { |
| 349 | − unsigned int ec = ESR_ELx_EC(esr); | |
| 350 | 355 | unsigned int fsc = esr & ESR_ELx_FSC; |
| 351 | 356 | |
| 352 | − if (ec != ESR_ELx_EC_DABT_CUR) | |
| 357 | + if (!is_el1_data_abort(esr)) | |
| 353 | 358 | return false; |
| 354 | 359 | |
| 355 | 360 | if (fsc == ESR_ELx_FSC_MTE) |
| @@ -504,7 +509,7 @@ static vm_fault_t __do_page_fault(struct mm_struct *mm, unsigned long addr, · 148 unmodified lines | ||
| 504 | 509 | */ |
| 505 | 510 | if (!(vma->vm_flags & vm_flags)) |
| 506 | 511 | return VM_FAULT_BADACCESS; |
| 507 | − return handle_mm_fault(vma, addr & PAGE_MASK, mm_flags, regs); | |
| 512 | + return handle_mm_fault(vma, addr, mm_flags, regs); | |
| 508 | 513 | } |
| 509 | 514 | |
| 510 | 515 | static bool is_el0_instruction_abort(unsigned int esr) |
| @@ -836,13 +841,6 @@ void do_mem_abort(unsigned long far, unsigned int esr, struct pt_regs *regs) · 325 unmodified lines | ||
| 836 | 841 | } |
| 837 | 842 | NOKPROBE_SYMBOL(do_mem_abort); |
| 838 | 843 | |
| 839 | −void do_el0_irq_bp_hardening(void) | |
| 840 | −{ | |
| 841 | − /* PC has already been checked in entry.S */ | |
| 842 | − arm64_apply_bp_hardening(); | |
| 843 | −} | |
| 844 | −NOKPROBE_SYMBOL(do_el0_irq_bp_hardening); | |
| 845 | − | |
| 846 | 844 | void do_sp_pc_abort(unsigned long addr, unsigned int esr, struct pt_regs *regs) |
| 847 | 845 | { |
| 848 | 846 | arm64_notify_die("SP/PC alignment exception", regs, SIGBUS, BUS_ADRALN, |
| @@ -921,3 +919,29 @@ void do_debug_exception(unsigned long addr_if_watchpoint, unsigned int esr, · 72 unmodified lines | ||
| 921 | 919 | debug_exception_exit(regs); |
| 922 | 920 | } |
| 923 | 921 | NOKPROBE_SYMBOL(do_debug_exception); |
| 922 | + | |
| 923 | +/* | |
| 924 | + * Used during anonymous page fault handling. | |
| 925 | + */ | |
| 926 | +struct page *alloc_zeroed_user_highpage_movable(struct vm_area_struct *vma, | |
| 927 | + unsigned long vaddr) | |
| 928 | +{ | |
| 929 | + gfp_t flags = GFP_HIGHUSER_MOVABLE | __GFP_ZERO; | |
| 930 | + | |
| 931 | + /* | |
| 932 | + * If the page is mapped with PROT_MTE, initialise the tags at the | |
| 933 | + * point of allocation and page zeroing as this is usually faster than | |
| 934 | + * separate DC ZVA and STGM. | |
| 935 | + */ | |
| 936 | + if (vma->vm_flags & VM_MTE) | |
| 937 | + flags |= __GFP_ZEROTAGS; | |
| 938 | + | |
| 939 | + return alloc_page_vma(flags, vma, vaddr); | |
| 940 | +} | |
| 941 | + | |
| 942 | +void tag_clear_highpage(struct page *page) | |
| 943 | +{ | |
| 944 | + mte_zero_clear_page_tags(page_address(page)); | |
| 945 | + page_kasan_tag_reset(page); | |
| 946 | + set_bit(PG_mte_tagged, &page->flags); | |
| 947 | +} | |
| @@ -14,28 +14,25 @@ · 13 unmodified lines | ||
| 14 | 14 | #include <asm/cache.h> |
| 15 | 15 | #include <asm/tlbflush.h> |
| 16 | 16 | |
| 17 | −void sync_icache_aliases(void *kaddr, unsigned long len) | |
| 17 | +void sync_icache_aliases(unsigned long start, unsigned long end) | |
| 18 | 18 | { |
| 19 | − unsigned long addr = (unsigned long)kaddr; | |
| 20 | − | |
| 21 | 19 | if (icache_is_aliasing()) { |
| 22 | − __clean_dcache_area_pou(kaddr, len); | |
| 23 | − __flush_icache_all(); | |
| 20 | + dcache_clean_pou(start, end); | |
| 21 | + icache_inval_all_pou(); | |
| 24 | 22 | } else { |
| 25 | 23 | /* |
| 26 | 24 | * Don't issue kick_all_cpus_sync() after I-cache invalidation |
| 27 | 25 | * for user mappings. |
| 28 | 26 | */ |
| 29 | − __flush_icache_range(addr, addr + len); | |
| 27 | + caches_clean_inval_pou(start, end); | |
| 30 | 28 | } |
| 31 | 29 | } |
| 32 | 30 | |
| 33 | −static void flush_ptrace_access(struct vm_area_struct *vma, struct page *page, | |
| 34 | − unsigned long uaddr, void *kaddr, | |
| 35 | − unsigned long len) | |
| 31 | +static void flush_ptrace_access(struct vm_area_struct *vma, unsigned long start, | |
| 32 | + unsigned long end) | |
| 36 | 33 | { |
| 37 | 34 | if (vma->vm_flags & VM_EXEC) |
| 38 | − sync_icache_aliases(kaddr, len); | |
| 35 | + sync_icache_aliases(start, end); | |
| 39 | 36 | } |
| 40 | 37 | |
| 41 | 38 | /* |
| @@ -48,7 +45,7 @@ void copy_to_user_page(struct vm_area_struct *vma, struct page *page, · 6 unmodified lines | ||
| 48 | 45 | unsigned long len) |
| 49 | 46 | { |
| 50 | 47 | memcpy(dst, src, len); |
| 51 | − flush_ptrace_access(vma, page, uaddr, dst, len); | |
| 48 | + flush_ptrace_access(vma, (unsigned long)dst, (unsigned long)dst + len); | |
| 52 | 49 | } |
| 53 | 50 | |
| 54 | 51 | void __sync_icache_dcache(pte_t pte) |
| @@ -56,7 +53,9 @@ void __sync_icache_dcache(pte_t pte) · 1 unmodified line | ||
| 56 | 53 | struct page *page = pte_page(pte); |
| 57 | 54 | |
| 58 | 55 | if (!test_bit(PG_dcache_clean, &page->flags)) { |
| 59 | − sync_icache_aliases(page_address(page), page_size(page)); | |
| 56 | + sync_icache_aliases((unsigned long)page_address(page), | |
| 57 | + (unsigned long)page_address(page) + | |
| 58 | + page_size(page)); | |
| 60 | 59 | set_bit(PG_dcache_clean, &page->flags); |
| 61 | 60 | } |
| 62 | 61 | } |
| @@ -77,20 +76,20 @@ EXPORT_SYMBOL(flush_dcache_page); · 14 unmodified lines | ||
| 77 | 76 | /* |
| 78 | 77 | * Additional functions defined in assembly. |
| 79 | 78 | */ |
| 80 | −EXPORT_SYMBOL(__flush_icache_range); | |
| 79 | +EXPORT_SYMBOL(caches_clean_inval_pou); | |
| 81 | 80 | |
| 82 | 81 | #ifdef CONFIG_ARCH_HAS_PMEM_API |
| 83 | 82 | void arch_wb_cache_pmem(void *addr, size_t size) |
| 84 | 83 | { |
| 85 | 84 | /* Ensure order against any prior non-cacheable writes */ |
| 86 | 85 | dmb(osh); |
| 87 | − __clean_dcache_area_pop(addr, size); | |
| 86 | + dcache_clean_pop((unsigned long)addr, (unsigned long)addr + size); | |
| 88 | 87 | } |
| 89 | 88 | EXPORT_SYMBOL_GPL(arch_wb_cache_pmem); |
| 90 | 89 | |
| 91 | 90 | void arch_invalidate_pmem(void *addr, size_t size) |
| 92 | 91 | { |
| 93 | − __inval_dcache_area(addr, size); | |
| 92 | + dcache_inval_poc((unsigned long)addr, (unsigned long)addr + size); | |
| 94 | 93 | } |
| 95 | 94 | EXPORT_SYMBOL_GPL(arch_invalidate_pmem); |
| 96 | 95 | #endif |
| @@ -499,6 +499,13 @@ void __init mem_init(void) · 498 unmodified lines | ||
| 499 | 499 | BUILD_BUG_ON(TASK_SIZE_32 > DEFAULT_MAP_WINDOW_64); |
| 500 | 500 | #endif |
| 501 | 501 | |
| 502 | + /* | |
| 503 | + * Selected page table levels should match when derived from | |
| 504 | + * scratch using the virtual address range and page size. | |
| 505 | + */ | |
| 506 | + BUILD_BUG_ON(ARM64_HW_PGTABLE_LEVELS(CONFIG_ARM64_VA_BITS) != | |
| 507 | + CONFIG_PGTABLE_LEVELS); | |
| 508 | + | |
| 502 | 509 | if (PAGE_SIZE >= 16384 && get_num_physpages() <= 128) { |
| 503 | 510 | extern int sysctl_overcommit_memory; |
| 504 | 511 | /* |
| @@ -228,7 +228,7 @@ static void init_pmd(pud_t *pudp, unsigned long addr, unsigned long end, · 227 unmodified lines | ||
| 228 | 228 | next = pmd_addr_end(addr, end); |
| 229 | 229 | |
| 230 | 230 | /* try section mapping first */ |
| 231 | − if (((addr | next | phys) & ~SECTION_MASK) == 0 && | |
| 231 | + if (((addr | next | phys) & ~PMD_MASK) == 0 && | |
| 232 | 232 | (flags & NO_BLOCK_MAPPINGS) == 0) { |
| 233 | 233 | pmd_set_huge(pmdp, phys, prot); |
| 234 | 234 | |
| @@ -1114,14 +1114,14 @@ static void free_empty_tables(unsigned long addr, unsigned long end, · 879 unmodified lines | ||
| 1114 | 1114 | } |
| 1115 | 1115 | #endif |
| 1116 | 1116 | |
| 1117 | −#if !ARM64_SWAPPER_USES_SECTION_MAPS | |
| 1117 | +#if !ARM64_KERNEL_USES_PMD_MAPS | |
| 1118 | 1118 | int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node, |
| 1119 | 1119 | struct vmem_altmap *altmap) |
| 1120 | 1120 | { |
| 1121 | 1121 | WARN_ON((start < VMEMMAP_START) || (end > VMEMMAP_END)); |
| 1122 | 1122 | return vmemmap_populate_basepages(start, end, node, altmap); |
| 1123 | 1123 | } |
| 1124 | −#else /* !ARM64_SWAPPER_USES_SECTION_MAPS */ | |
| 1124 | +#else /* !ARM64_KERNEL_USES_PMD_MAPS */ | |
| 1125 | 1125 | int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node, |
| 1126 | 1126 | struct vmem_altmap *altmap) |
| 1127 | 1127 | { |
| @@ -1166,17 +1166,18 @@ int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node, · 38 unmodified lines | ||
| 1166 | 1166 | |
| 1167 | 1167 | return 0; |
| 1168 | 1168 | } |
| 1169 | −#endif /* !ARM64_SWAPPER_USES_SECTION_MAPS */ | |
| 1169 | +#endif /* !ARM64_KERNEL_USES_PMD_MAPS */ | |
| 1170 | + | |
| 1171 | +#ifdef CONFIG_MEMORY_HOTPLUG | |
| 1170 | 1172 | void vmemmap_free(unsigned long start, unsigned long end, |
| 1171 | 1173 | struct vmem_altmap *altmap) |
| 1172 | 1174 | { |
| 1173 | −#ifdef CONFIG_MEMORY_HOTPLUG | |
| 1174 | 1175 | WARN_ON((start < VMEMMAP_START) || (end > VMEMMAP_END)); |
| 1175 | 1176 | |
| 1176 | 1177 | unmap_hotplug_range(start, end, true, altmap); |
| 1177 | 1178 | free_empty_tables(start, end, VMEMMAP_START, VMEMMAP_END); |
| 1178 | −#endif | |
| 1179 | 1179 | } |
| 1180 | +#endif /* CONFIG_MEMORY_HOTPLUG */ | |
| 1180 | 1181 | |
| 1181 | 1182 | static inline pud_t *fixmap_pud(unsigned long addr) |
| 1182 | 1183 | { |
| @@ -46,9 +46,13 @@ · 45 unmodified lines | ||
| 46 | 46 | #endif |
| 47 | 47 | |
| 48 | 48 | #ifdef CONFIG_KASAN_HW_TAGS |
| 49 | −#define TCR_KASAN_HW_FLAGS SYS_TCR_EL1_TCMA1 | TCR_TBI1 | TCR_TBID1 | |
| 49 | +#define TCR_MTE_FLAGS SYS_TCR_EL1_TCMA1 | TCR_TBI1 | TCR_TBID1 | |
| 50 | 50 | #else |
| 51 | −#define TCR_KASAN_HW_FLAGS 0 | |
| 51 | +/* | |
| 52 | + * The mte_zero_clear_page_tags() implementation uses DC GZVA, which relies on | |
| 53 | + * TBI being enabled at EL1. | |
| 54 | + */ | |
| 55 | +#define TCR_MTE_FLAGS TCR_TBI1 | TCR_TBID1 | |
| 52 | 56 | #endif |
| 53 | 57 | |
| 54 | 58 | /* |
| @@ -58,10 +62,8 @@ · 3 unmodified lines | ||
| 58 | 62 | #define MAIR_EL1_SET \ |
| 59 | 63 | (MAIR_ATTRIDX(MAIR_ATTR_DEVICE_nGnRnE, MT_DEVICE_nGnRnE) | \ |
| 60 | 64 | MAIR_ATTRIDX(MAIR_ATTR_DEVICE_nGnRE, MT_DEVICE_nGnRE) | \ |
| 61 | − MAIR_ATTRIDX(MAIR_ATTR_DEVICE_GRE, MT_DEVICE_GRE) | \ | |
| 62 | 65 | MAIR_ATTRIDX(MAIR_ATTR_NORMAL_NC, MT_NORMAL_NC) | \ |
| 63 | 66 | MAIR_ATTRIDX(MAIR_ATTR_NORMAL, MT_NORMAL) | \ |
| 64 | − MAIR_ATTRIDX(MAIR_ATTR_NORMAL_WT, MT_NORMAL_WT) | \ | |
| 65 | 67 | MAIR_ATTRIDX(MAIR_ATTR_NORMAL, MT_NORMAL_TAGGED)) |
| 66 | 68 | |
| 67 | 69 | #ifdef CONFIG_CPU_PM |
| @@ -83,11 +85,7 @@ SYM_FUNC_START(cpu_do_suspend) · 15 unmodified lines | ||
| 83 | 85 | mrs x9, mdscr_el1 |
| 84 | 86 | mrs x10, oslsr_el1 |
| 85 | 87 | mrs x11, sctlr_el1 |
| 86 | −alternative_if_not ARM64_HAS_VIRT_HOST_EXTN | |
| 87 | − mrs x12, tpidr_el1 | |
| 88 | −alternative_else | |
| 89 | − mrs x12, tpidr_el2 | |
| 90 | −alternative_endif | |
| 88 | + get_this_cpu_offset x12 | |
| 91 | 89 | mrs x13, sp_el0 |
| 92 | 90 | stp x2, x3, [x0] |
| 93 | 91 | stp x4, x5, [x0, #16] |
| @@ -145,11 +143,7 @@ SYM_FUNC_START(cpu_do_resume) · 51 unmodified lines | ||
| 145 | 143 | msr mdscr_el1, x10 |
| 146 | 144 | |
| 147 | 145 | msr sctlr_el1, x12 |
| 148 | −alternative_if_not ARM64_HAS_VIRT_HOST_EXTN | |
| 149 | − msr tpidr_el1, x13 | |
| 150 | −alternative_else | |
| 151 | − msr tpidr_el2, x13 | |
| 152 | −alternative_endif | |
| 146 | + set_this_cpu_offset x13 | |
| 153 | 147 | msr sp_el0, x14 |
| 154 | 148 | /* |
| 155 | 149 | * Restore oslsr_el1 by writing oslar_el1 |
| @@ -464,7 +458,7 @@ SYM_FUNC_START(__cpu_setup) · 308 unmodified lines | ||
| 464 | 458 | msr_s SYS_TFSRE0_EL1, xzr |
| 465 | 459 | |
| 466 | 460 | /* set the TCR_EL1 bits */ |
| 467 | − mov_q x10, TCR_KASAN_HW_FLAGS | |
| 461 | + mov_q x10, TCR_MTE_FLAGS | |
| 468 | 462 | orr tcr, tcr, x10 |
| 469 | 463 | 1: |
| 470 | 464 | #endif |
| @@ -157,10 +157,6 @@ static const struct prot_bits pte_bits[] = { · 156 unmodified lines | ||
| 157 | 157 | .mask = PTE_ATTRINDX_MASK, |
| 158 | 158 | .val = PTE_ATTRINDX(MT_DEVICE_nGnRE), |
| 159 | 159 | .set = "DEVICE/nGnRE", |
| 160 | − }, { | |
| 161 | − .mask = PTE_ATTRINDX_MASK, | |
| 162 | − .val = PTE_ATTRINDX(MT_DEVICE_GRE), | |
| 163 | − .set = "DEVICE/GRE", | |
| 164 | 160 | }, { |
| 165 | 161 | .mask = PTE_ATTRINDX_MASK, |
| 166 | 162 | .val = PTE_ATTRINDX(MT_NORMAL_NC), |
| @@ -16,6 +16,7 @@ · 15 unmodified lines | ||
| 16 | 16 | #include <asm/byteorder.h> |
| 17 | 17 | #include <asm/cacheflush.h> |
| 18 | 18 | #include <asm/debug-monitors.h> |
| 19 | +#include <asm/insn.h> | |
| 19 | 20 | #include <asm/set_memory.h> |
| 20 | 21 | |
| 21 | 22 | #include "bpf_jit.h" |
| @@ -3,7 +3,8 @@ · 2 unmodified lines | ||
| 3 | 3 | # Internal CPU capabilities constants, keep this list sorted |
| 4 | 4 | |
| 5 | 5 | BTI |
| 6 | −HAS_32BIT_EL0 | |
| 6 | +# Unreliable: use system_supports_32bit_el0() instead. | |
| 7 | +HAS_32BIT_EL0_DO_NOT_USE | |
| 7 | 8 | HAS_32BIT_EL1 |
| 8 | 9 | HAS_ADDRESS_AUTH |
| 9 | 10 | HAS_ADDRESS_AUTH_ARCH |
| @@ -82,16 +82,16 @@ do { \ · 81 unmodified lines | ||
| 82 | 82 | } while (0) |
| 83 | 83 | |
| 84 | 84 | |
| 85 | −#define __alloc_zeroed_user_highpage(movableflags, vma, vaddr) \ | |
| 85 | +#define alloc_zeroed_user_highpage_movable(vma, vaddr) \ | |
| 86 | 86 | ({ \ |
| 87 | 87 | struct page *page = alloc_page_vma( \ |
| 88 | − GFP_HIGHUSER | __GFP_ZERO | movableflags, vma, vaddr); \ | |
| 88 | + GFP_HIGHUSER_MOVABLE | __GFP_ZERO, vma, vaddr); \ | |
| 89 | 89 | if (page) \ |
| 90 | 90 | flush_dcache_page(page); \ |
| 91 | 91 | page; \ |
| 92 | 92 | }) |
| 93 | 93 | |
| 94 | −#define __HAVE_ARCH_ALLOC_ZEROED_USER_HIGHPAGE | |
| 94 | +#define __HAVE_ARCH_ALLOC_ZEROED_USER_HIGHPAGE_MOVABLE | |
| 95 | 95 | |
| 96 | 96 | #define virt_addr_valid(kaddr) pfn_valid(__pa(kaddr) >> PAGE_SHIFT) |
| 97 | 97 | |
| @@ -13,9 +13,9 @@ extern unsigned long memory_end; · 12 unmodified lines | ||
| 13 | 13 | #define clear_user_page(page, vaddr, pg) clear_page(page) |
| 14 | 14 | #define copy_user_page(to, from, vaddr, pg) copy_page(to, from) |
| 15 | 15 | |
| 16 | −#define __alloc_zeroed_user_highpage(movableflags, vma, vaddr) \ | |
| 17 | − alloc_page_vma(GFP_HIGHUSER | __GFP_ZERO | movableflags, vma, vaddr) | |
| 18 | −#define __HAVE_ARCH_ALLOC_ZEROED_USER_HIGHPAGE | |
| 16 | +#define alloc_zeroed_user_highpage_movable(vma, vaddr) \ | |
| 17 | + alloc_page_vma(GFP_HIGHUSER_MOVABLE | __GFP_ZERO, vma, vaddr) | |
| 18 | +#define __HAVE_ARCH_ALLOC_ZEROED_USER_HIGHPAGE_MOVABLE | |
| 19 | 19 | |
| 20 | 20 | #define __pa(vaddr) ((unsigned long)(vaddr)) |
| 21 | 21 | #define __va(paddr) ((void *)((unsigned long)(paddr))) |
| @@ -68,9 +68,9 @@ static inline void copy_page(void *to, void *from) · 67 unmodified lines | ||
| 68 | 68 | #define clear_user_page(page, vaddr, pg) clear_page(page) |
| 69 | 69 | #define copy_user_page(to, from, vaddr, pg) copy_page(to, from) |
| 70 | 70 | |
| 71 | −#define __alloc_zeroed_user_highpage(movableflags, vma, vaddr) \ | |
| 72 | − alloc_page_vma(GFP_HIGHUSER | __GFP_ZERO | movableflags, vma, vaddr) | |
| 73 | −#define __HAVE_ARCH_ALLOC_ZEROED_USER_HIGHPAGE | |
| 71 | +#define alloc_zeroed_user_highpage_movable(vma, vaddr) \ | |
| 72 | + alloc_page_vma(GFP_HIGHUSER_MOVABLE | __GFP_ZERO, vma, vaddr) | |
| 73 | +#define __HAVE_ARCH_ALLOC_ZEROED_USER_HIGHPAGE_MOVABLE | |
| 74 | 74 | |
| 75 | 75 | /* |
| 76 | 76 | * These are used to make use of C type-checking.. |
| @@ -34,9 +34,9 @@ static inline void copy_user_page(void *to, void *from, unsigned long vaddr, · 33 unmodified lines | ||
| 34 | 34 | copy_page(to, from); |
| 35 | 35 | } |
| 36 | 36 | |
| 37 | −#define __alloc_zeroed_user_highpage(movableflags, vma, vaddr) \ | |
| 38 | − alloc_page_vma(GFP_HIGHUSER | __GFP_ZERO | movableflags, vma, vaddr) | |
| 39 | −#define __HAVE_ARCH_ALLOC_ZEROED_USER_HIGHPAGE | |
| 37 | +#define alloc_zeroed_user_highpage_movable(vma, vaddr) \ | |
| 38 | + alloc_page_vma(GFP_HIGHUSER_MOVABLE | __GFP_ZERO, vma, vaddr) | |
| 39 | +#define __HAVE_ARCH_ALLOC_ZEROED_USER_HIGHPAGE_MOVABLE | |
| 40 | 40 | |
| 41 | 41 | #ifndef __pa |
| 42 | 42 | #define __pa(x) __phys_addr((unsigned long)(x)) |
| @@ -335,10 +335,15 @@ int psci_cpu_suspend_enter(u32 state) · 334 unmodified lines | ||
| 335 | 335 | { |
| 336 | 336 | int ret; |
| 337 | 337 | |
| 338 | − if (!psci_power_state_loses_context(state)) | |
| 338 | + if (!psci_power_state_loses_context(state)) { | |
| 339 | + struct arm_cpuidle_irq_context context; | |
| 340 | + | |
| 341 | + arm_cpuidle_save_irq_context(&context); | |
| 339 | 342 | ret = psci_ops.cpu_suspend(state, 0); |
| 340 | − else | |
| 343 | + arm_cpuidle_restore_irq_context(&context); | |
| 344 | + } else { | |
| 341 | 345 | ret = cpu_suspend(state, psci_suspend_finisher); |
| 346 | + } | |
| 342 | 347 | |
| 343 | 348 | return ret; |
| 344 | 349 | } |
| @@ -15,6 +15,7 @@ static u32 smccc_version = ARM_SMCCC_VERSION_1_0; · 14 unmodified lines | ||
| 15 | 15 | static enum arm_smccc_conduit smccc_conduit = SMCCC_CONDUIT_NONE; |
| 16 | 16 | |
| 17 | 17 | bool __ro_after_init smccc_trng_available = false; |
| 18 | +u64 __ro_after_init smccc_has_sve_hint = false; | |
| 18 | 19 | |
| 19 | 20 | void __init arm_smccc_version_init(u32 version, enum arm_smccc_conduit conduit) |
| 20 | 21 | { |
| @@ -22,6 +23,9 @@ void __init arm_smccc_version_init(u32 version, enum arm_smccc_conduit conduit) · 1 unmodified line | ||
| 22 | 23 | smccc_conduit = conduit; |
| 23 | 24 | |
| 24 | 25 | smccc_trng_available = smccc_probe_trng(); |
| 26 | + if (IS_ENABLED(CONFIG_ARM64_SVE) && | |
| 27 | + smccc_version >= ARM_SMCCC_VERSION_1_3) | |
| 28 | + smccc_has_sve_hint = true; | |
| 25 | 29 | } |
| 26 | 30 | |
| 27 | 31 | enum arm_smccc_conduit arm_smccc_1_1_get_conduit(void) |
| @@ -463,7 +463,7 @@ void lkdtm_DOUBLE_FAULT(void) · 462 unmodified lines | ||
| 463 | 463 | #ifdef CONFIG_ARM64 |
| 464 | 464 | static noinline void change_pac_parameters(void) |
| 465 | 465 | { |
| 466 | − if (IS_ENABLED(CONFIG_ARM64_PTR_AUTH)) { | |
| 466 | + if (IS_ENABLED(CONFIG_ARM64_PTR_AUTH_KERNEL)) { | |
| 467 | 467 | /* Reset the keys of current task */ |
| 468 | 468 | ptrauth_thread_init_kernel(current); |
| 469 | 469 | ptrauth_thread_switch_kernel(current); |
| @@ -477,8 +477,8 @@ noinline void lkdtm_CORRUPT_PAC(void) · 7 unmodified lines | ||
| 477 | 477 | #define CORRUPT_PAC_ITERATE 10 |
| 478 | 478 | int i; |
| 479 | 479 | |
| 480 | − if (!IS_ENABLED(CONFIG_ARM64_PTR_AUTH)) | |
| 481 | − pr_err("FAIL: kernel not built with CONFIG_ARM64_PTR_AUTH\n"); | |
| 480 | + if (!IS_ENABLED(CONFIG_ARM64_PTR_AUTH_KERNEL)) | |
| 481 | + pr_err("FAIL: kernel not built with CONFIG_ARM64_PTR_AUTH_KERNEL\n"); | |
| 482 | 482 | |
| 483 | 483 | if (!system_supports_address_auth()) { |
| 484 | 484 | pr_err("FAIL: CPU lacks pointer authentication feature\n"); |
| @@ -37,7 +37,7 @@ · 36 unmodified lines | ||
| 37 | 37 | |
| 38 | 38 | #define CCI_PMU_CNTR_SIZE(model) ((model)->cntr_size) |
| 39 | 39 | #define CCI_PMU_CNTR_BASE(model, idx) ((idx) * CCI_PMU_CNTR_SIZE(model)) |
| 40 | −#define CCI_PMU_CNTR_MASK ((1ULL << 32) -1) | |
| 40 | +#define CCI_PMU_CNTR_MASK ((1ULL << 32) - 1) | |
| 41 | 41 | #define CCI_PMU_CNTR_LAST(cci_pmu) (cci_pmu->num_cntrs - 1) |
| 42 | 42 | |
| 43 | 43 | #define CCI_PMU_MAX_HW_CNTRS(model) \ |
| @@ -806,7 +806,7 @@ static int pmu_get_event_idx(struct cci_pmu_hw_events *hw, struct perf_event *ev · 762 unmodified lines | ||
| 806 | 806 | return cci_pmu->model->get_event_idx(cci_pmu, hw, cci_event); |
| 807 | 807 | |
| 808 | 808 | /* Generic code to find an unused idx from the mask */ |
| 809 | − for(idx = 0; idx <= CCI_PMU_CNTR_LAST(cci_pmu); idx++) | |
| 809 | + for (idx = 0; idx <= CCI_PMU_CNTR_LAST(cci_pmu); idx++) | |
| 810 | 810 | if (!test_and_set_bit(idx, hw->used_mask)) |
| 811 | 811 | return idx; |
| 812 | 812 | |
| @@ -1211,7 +1211,7 @@ static int arm_ccn_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node) · 1210 unmodified lines | ||
| 1211 | 1211 | perf_pmu_migrate_context(&dt->pmu, cpu, target); |
| 1212 | 1212 | dt->cpu = target; |
| 1213 | 1213 | if (ccn->irq) |
| 1214 | − WARN_ON(irq_set_affinity_hint(ccn->irq, cpumask_of(dt->cpu))); | |
| 1214 | + WARN_ON(irq_set_affinity(ccn->irq, cpumask_of(dt->cpu))); | |
| 1215 | 1215 | return 0; |
| 1216 | 1216 | } |
| 1217 | 1217 | |
| @@ -1291,7 +1291,7 @@ static int arm_ccn_pmu_init(struct arm_ccn *ccn) · 73 unmodified lines | ||
| 1291 | 1291 | |
| 1292 | 1292 | /* Also make sure that the overflow interrupt is handled by this CPU */ |
| 1293 | 1293 | if (ccn->irq) { |
| 1294 | − err = irq_set_affinity_hint(ccn->irq, cpumask_of(ccn->dt.cpu)); | |
| 1294 | + err = irq_set_affinity(ccn->irq, cpumask_of(ccn->dt.cpu)); | |
| 1295 | 1295 | if (err) { |
| 1296 | 1296 | dev_err(ccn->dev, "Failed to set interrupt affinity!\n"); |
| 1297 | 1297 | goto error_set_affinity; |
| @@ -1325,8 +1325,6 @@ static void arm_ccn_pmu_cleanup(struct arm_ccn *ccn) · 27 unmodified lines | ||
| 1325 | 1325 | |
| 1326 | 1326 | cpuhp_state_remove_instance_nocalls(CPUHP_AP_PERF_ARM_CCN_ONLINE, |
| 1327 | 1327 | &ccn->dt.node); |
| 1328 | − if (ccn->irq) | |
| 1329 | − irq_set_affinity_hint(ccn->irq, NULL); | |
| 1330 | 1328 | for (i = 0; i < ccn->num_xps; i++) |
| 1331 | 1329 | writel(0, ccn->xp[i].base + CCN_XP_DT_CONTROL); |
| 1332 | 1330 | writel(0, ccn->dt.base + CCN_DT_PMCR); |
| @@ -31,7 +31,7 @@ · 30 unmodified lines | ||
| 31 | 31 | #define CMN_CI_CHILD_COUNT GENMASK_ULL(15, 0) |
| 32 | 32 | #define CMN_CI_CHILD_PTR_OFFSET GENMASK_ULL(31, 16) |
| 33 | 33 | |
| 34 | −#define CMN_CHILD_NODE_ADDR GENMASK(27,0) | |
| 34 | +#define CMN_CHILD_NODE_ADDR GENMASK(27, 0) | |
| 35 | 35 | #define CMN_CHILD_NODE_EXTERNAL BIT(31) |
| 36 | 36 | |
| 37 | 37 | #define CMN_ADDR_NODE_PTR GENMASK(27, 14) |
| @@ -1162,7 +1162,7 @@ static int arm_cmn_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node) · 1124 unmodified lines | ||
| 1162 | 1162 | |
| 1163 | 1163 | perf_pmu_migrate_context(&cmn->pmu, cpu, target); |
| 1164 | 1164 | for (i = 0; i < cmn->num_dtcs; i++) |
| 1165 | − irq_set_affinity_hint(cmn->dtc[i].irq, cpumask_of(target)); | |
| 1165 | + irq_set_affinity(cmn->dtc[i].irq, cpumask_of(target)); | |
| 1166 | 1166 | cmn->cpu = target; |
| 1167 | 1167 | return 0; |
| 1168 | 1168 | } |
| @@ -1212,7 +1212,7 @@ static int arm_cmn_init_irqs(struct arm_cmn *cmn) · 43 unmodified lines | ||
| 1212 | 1212 | irq = cmn->dtc[i].irq; |
| 1213 | 1213 | for (j = i; j--; ) { |
| 1214 | 1214 | if (cmn->dtc[j].irq == irq) { |
| 1215 | − cmn->dtc[j].irq_friend = j - i; | |
| 1215 | + cmn->dtc[j].irq_friend = i - j; | |
| 1216 | 1216 | goto next; |
| 1217 | 1217 | } |
| 1218 | 1218 | } |
| @@ -1222,7 +1222,7 @@ static int arm_cmn_init_irqs(struct arm_cmn *cmn) · 3 unmodified lines | ||
| 1222 | 1222 | if (err) |
| 1223 | 1223 | return err; |
| 1224 | 1224 | |
| 1225 | − err = irq_set_affinity_hint(irq, cpumask_of(cmn->cpu)); | |
| 1225 | + err = irq_set_affinity(irq, cpumask_of(cmn->cpu)); | |
| 1226 | 1226 | if (err) |
| 1227 | 1227 | return err; |
| 1228 | 1228 | next: |
| @@ -1568,16 +1568,11 @@ static int arm_cmn_probe(struct platform_device *pdev) · 339 unmodified lines | ||
| 1568 | 1568 | static int arm_cmn_remove(struct platform_device *pdev) |
| 1569 | 1569 | { |
| 1570 | 1570 | struct arm_cmn *cmn = platform_get_drvdata(pdev); |
| 1571 | − int i; | |
| 1572 | 1571 | |
| 1573 | 1572 | writel_relaxed(0, cmn->dtc[0].base + CMN_DT_DTC_CTL); |
| 1574 | 1573 | |
| 1575 | 1574 | perf_pmu_unregister(&cmn->pmu); |
| 1576 | 1575 | cpuhp_state_remove_instance(arm_cmn_hp_state, &cmn->cpuhp_node); |
| 1577 | − | |
| 1578 | − for (i = 0; i < cmn->num_dtcs; i++) | |
| 1579 | − irq_set_affinity_hint(cmn->dtc[i].irq, NULL); | |
| 1580 | − | |
| 1581 | 1576 | return 0; |
| 1582 | 1577 | } |
| 1583 | 1578 | |
| @@ -421,7 +421,7 @@ static struct dmc620_pmu_irq *__dmc620_pmu_get_irq(int irq_num) · 420 unmodified lines | ||
| 421 | 421 | if (ret) |
| 422 | 422 | goto out_free_aff; |
| 423 | 423 | |
| 424 | − ret = irq_set_affinity_hint(irq_num, cpumask_of(irq->cpu)); | |
| 424 | + ret = irq_set_affinity(irq_num, cpumask_of(irq->cpu)); | |
| 425 | 425 | if (ret) |
| 426 | 426 | goto out_free_irq; |
| 427 | 427 | |
| @@ -475,7 +475,6 @@ static void dmc620_pmu_put_irq(struct dmc620_pmu *dmc620_pmu) · 47 unmodified lines | ||
| 475 | 475 | list_del(&irq->irqs_node); |
| 476 | 476 | mutex_unlock(&dmc620_pmu_irqs_lock); |
| 477 | 477 | |
| 478 | − WARN_ON(irq_set_affinity_hint(irq->irq_num, NULL)); | |
| 479 | 478 | free_irq(irq->irq_num, irq); |
| 480 | 479 | cpuhp_state_remove_instance_nocalls(cpuhp_state_num, &irq->node); |
| 481 | 480 | kfree(irq); |
| @@ -622,7 +621,7 @@ static int dmc620_pmu_cpu_teardown(unsigned int cpu, · 140 unmodified lines | ||
| 622 | 621 | perf_pmu_migrate_context(&dmc620_pmu->pmu, irq->cpu, target); |
| 623 | 622 | mutex_unlock(&dmc620_pmu_irqs_lock); |
| 624 | 623 | |
| 625 | − WARN_ON(irq_set_affinity_hint(irq->irq_num, cpumask_of(target))); | |
| 624 | + WARN_ON(irq_set_affinity(irq->irq_num, cpumask_of(target))); | |
| 626 | 625 | irq->cpu = target; |
| 627 | 626 | |
| 628 | 627 | return 0; |
| @@ -687,7 +687,7 @@ static void dsu_pmu_probe_pmu(struct dsu_pmu *dsu_pmu) · 686 unmodified lines | ||
| 687 | 687 | static void dsu_pmu_set_active_cpu(int cpu, struct dsu_pmu *dsu_pmu) |
| 688 | 688 | { |
| 689 | 689 | cpumask_set_cpu(cpu, &dsu_pmu->active_cpu); |
| 690 | − if (irq_set_affinity_hint(dsu_pmu->irq, &dsu_pmu->active_cpu)) | |
| 690 | + if (irq_set_affinity(dsu_pmu->irq, &dsu_pmu->active_cpu)) | |
| 691 | 691 | pr_warn("Failed to set irq affinity to %d\n", cpu); |
| 692 | 692 | } |
| 693 | 693 | |
| @@ -769,7 +769,6 @@ static int dsu_pmu_device_probe(struct platform_device *pdev) · 75 unmodified lines | ||
| 769 | 769 | if (rc) { |
| 770 | 770 | cpuhp_state_remove_instance(dsu_pmu_cpuhp_state, |
| 771 | 771 | &dsu_pmu->cpuhp_node); |
| 772 | − irq_set_affinity_hint(dsu_pmu->irq, NULL); | |
| 773 | 772 | } |
| 774 | 773 | |
| 775 | 774 | return rc; |
| @@ -781,7 +780,6 @@ static int dsu_pmu_device_remove(struct platform_device *pdev) · 5 unmodified lines | ||
| 781 | 780 | |
| 782 | 781 | perf_pmu_unregister(&dsu_pmu->pmu); |
| 783 | 782 | cpuhp_state_remove_instance(dsu_pmu_cpuhp_state, &dsu_pmu->cpuhp_node); |
| 784 | − irq_set_affinity_hint(dsu_pmu->irq, NULL); | |
| 785 | 783 | |
| 786 | 784 | return 0; |
| 787 | 785 | } |
| @@ -840,10 +838,8 @@ static int dsu_pmu_cpu_teardown(unsigned int cpu, struct hlist_node *node) · 52 unmodified lines | ||
| 840 | 838 | |
| 841 | 839 | dst = dsu_pmu_get_online_cpu_any_but(dsu_pmu, cpu); |
| 842 | 840 | /* If there are no active CPUs in the DSU, leave IRQ disabled */ |
| 843 | − if (dst >= nr_cpu_ids) { | |
| 844 | − irq_set_affinity_hint(dsu_pmu->irq, NULL); | |
| 841 | + if (dst >= nr_cpu_ids) | |
| 845 | 842 | return 0; |
| 846 | − } | |
| 847 | 843 | |
| 848 | 844 | perf_pmu_migrate_context(&dsu_pmu->pmu, cpu, dst); |
| 849 | 845 | dsu_pmu_set_active_cpu(dst, dsu_pmu); |
| @@ -563,14 +563,14 @@ static int armpmu_filter_match(struct perf_event *event) · 562 unmodified lines | ||
| 563 | 563 | return ret; |
| 564 | 564 | } |
| 565 | 565 | |
| 566 | −static ssize_t armpmu_cpumask_show(struct device *dev, | |
| 567 | − struct device_attribute *attr, char *buf) | |
| 566 | +static ssize_t cpus_show(struct device *dev, | |
| 567 | + struct device_attribute *attr, char *buf) | |
| 568 | 568 | { |
| 569 | 569 | struct arm_pmu *armpmu = to_arm_pmu(dev_get_drvdata(dev)); |
| 570 | 570 | return cpumap_print_to_pagebuf(true, buf, &armpmu->supported_cpus); |
| 571 | 571 | } |
| 572 | 572 | |
| 573 | −static DEVICE_ATTR(cpus, S_IRUGO, armpmu_cpumask_show, NULL); | |
| 573 | +static DEVICE_ATTR_RO(cpus); | |
| 574 | 574 | |
| 575 | 575 | static struct attribute *armpmu_common_attrs[] = { |
| 576 | 576 | &dev_attr_cpus.attr, |
| @@ -644,11 +644,9 @@ int armpmu_request_irq(int irq, int cpu) · 67 unmodified lines | ||
| 644 | 644 | } |
| 645 | 645 | |
| 646 | 646 | irq_flags = IRQF_PERCPU | |
| 647 | − IRQF_NOBALANCING | | |
| 647 | + IRQF_NOBALANCING | IRQF_NO_AUTOEN | | |
| 648 | 648 | IRQF_NO_THREAD; |
| 649 | 649 | |
| 650 | − irq_set_status_flags(irq, IRQ_NOAUTOEN); | |
| 651 | − | |
| 652 | 650 | err = request_nmi(irq, handler, irq_flags, "arm-pmu", |
| 653 | 651 | per_cpu_ptr(&cpu_armpmu, cpu)); |
| 654 | 652 | |
| @@ -670,7 +668,7 @@ int armpmu_request_irq(int irq, int cpu) · 15 unmodified lines | ||
| 670 | 668 | &cpu_armpmu); |
| 671 | 669 | irq_ops = &percpu_pmuirq_ops; |
| 672 | 670 | } else { |
| 673 | − has_nmi= true; | |
| 671 | + has_nmi = true; | |
| 674 | 672 | irq_ops = &percpu_pmunmi_ops; |
| 675 | 673 | } |
| 676 | 674 | } else { |
| @@ -869,10 +867,8 @@ static struct arm_pmu *__armpmu_alloc(gfp_t flags) · 192 unmodified lines | ||
| 869 | 867 | int cpu; |
| 870 | 868 | |
| 871 | 869 | pmu = kzalloc(sizeof(*pmu), flags); |
| 872 | − if (!pmu) { | |
| 873 | − pr_info("failed to allocate PMU device!\n"); | |
| 870 | + if (!pmu) | |
| 874 | 871 | goto out; |
| 875 | − } | |
| 876 | 872 | |
| 877 | 873 | pmu->hw_events = alloc_percpu_gfp(struct pmu_hw_events, flags); |
| 878 | 874 | if (!pmu->hw_events) { |
| @@ -277,7 +277,7 @@ static int smmu_pmu_apply_event_filter(struct smmu_pmu *smmu_pmu, · 276 unmodified lines | ||
| 277 | 277 | struct perf_event *event, int idx) |
| 278 | 278 | { |
| 279 | 279 | u32 span, sid; |
| 280 | − unsigned int num_ctrs = smmu_pmu->num_counters; | |
| 280 | + unsigned int cur_idx, num_ctrs = smmu_pmu->num_counters; | |
| 281 | 281 | bool filter_en = !!get_filter_enable(event); |
| 282 | 282 | |
| 283 | 283 | span = filter_en ? get_filter_span(event) : |
| @@ -285,17 +285,19 @@ static int smmu_pmu_apply_event_filter(struct smmu_pmu *smmu_pmu, · 1 unmodified line | ||
| 285 | 285 | sid = filter_en ? get_filter_stream_id(event) : |
| 286 | 286 | SMMU_PMCG_DEFAULT_FILTER_SID; |
| 287 | 287 | |
| 288 | − /* Support individual filter settings */ | |
| 289 | − if (!smmu_pmu->global_filter) { | |
| 288 | + cur_idx = find_first_bit(smmu_pmu->used_counters, num_ctrs); | |
| 289 | + /* | |
| 290 | + * Per-counter filtering, or scheduling the first globally-filtered | |
| 291 | + * event into an empty PMU so idx == 0 and it works out equivalent. | |
| 292 | + */ | |
| 293 | + if (!smmu_pmu->global_filter || cur_idx == num_ctrs) { | |
| 290 | 294 | smmu_pmu_set_event_filter(event, idx, span, sid); |
| 291 | 295 | return 0; |
| 292 | 296 | } |
| 293 | 297 | |
| 294 | − /* Requested settings same as current global settings*/ | |
| 295 | − idx = find_first_bit(smmu_pmu->used_counters, num_ctrs); | |
| 296 | − if (idx == num_ctrs || | |
| 297 | − smmu_pmu_check_global_filter(smmu_pmu->events[idx], event)) { | |
| 298 | − smmu_pmu_set_event_filter(event, 0, span, sid); | |
| 298 | + /* Otherwise, must match whatever's currently scheduled */ | |
| 299 | + if (smmu_pmu_check_global_filter(smmu_pmu->events[cur_idx], event)) { | |
| 300 | + smmu_pmu_set_evtyper(smmu_pmu, idx, get_event(event)); | |
| 299 | 301 | return 0; |
| 300 | 302 | } |
| 301 | 303 | |
| @@ -509,11 +511,8 @@ static ssize_t smmu_pmu_event_show(struct device *dev, · 207 unmodified lines | ||
| 509 | 511 | return sysfs_emit(page, "event=0x%02llx\n", pmu_attr->id); |
| 510 | 512 | } |
| 511 | 513 | |
| 512 | −#define SMMU_EVENT_ATTR(name, config) \ | |
| 513 | − (&((struct perf_pmu_events_attr) { \ | |
| 514 | − .attr = __ATTR(name, 0444, smmu_pmu_event_show, NULL), \ | |
| 515 | − .id = config, \ | |
| 516 | − }).attr.attr) | |
| 514 | +#define SMMU_EVENT_ATTR(name, config) \ | |
| 515 | + PMU_EVENT_ATTR_ID(name, smmu_pmu_event_show, config) | |
| 517 | 516 | |
| 518 | 517 | static struct attribute *smmu_pmu_events[] = { |
| 519 | 518 | SMMU_EVENT_ATTR(cycles, 0), |
| @@ -628,7 +627,7 @@ static int smmu_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node) · 108 unmodified lines | ||
| 628 | 627 | |
| 629 | 628 | perf_pmu_migrate_context(&smmu_pmu->pmu, cpu, target); |
| 630 | 629 | smmu_pmu->on_cpu = target; |
| 631 | − WARN_ON(irq_set_affinity_hint(smmu_pmu->irq, cpumask_of(target))); | |
| 630 | + WARN_ON(irq_set_affinity(smmu_pmu->irq, cpumask_of(target))); | |
| 632 | 631 | |
| 633 | 632 | return 0; |
| 634 | 633 | } |
| @@ -839,15 +838,14 @@ static int smmu_pmu_probe(struct platform_device *pdev) · 204 unmodified lines | ||
| 839 | 838 | |
| 840 | 839 | /* Pick one CPU to be the preferred one to use */ |
| 841 | 840 | smmu_pmu->on_cpu = raw_smp_processor_id(); |
| 842 | − WARN_ON(irq_set_affinity_hint(smmu_pmu->irq, | |
| 843 | − cpumask_of(smmu_pmu->on_cpu))); | |
| 841 | + WARN_ON(irq_set_affinity(smmu_pmu->irq, cpumask_of(smmu_pmu->on_cpu))); | |
| 844 | 842 | |
| 845 | 843 | err = cpuhp_state_add_instance_nocalls(cpuhp_state_num, |
| 846 | 844 | &smmu_pmu->node); |
| 847 | 845 | if (err) { |
| 848 | 846 | dev_err(dev, "Error %d registering hotplug, PMU @%pa\n", |
| 849 | 847 | err, &res_0->start); |
| 850 | − goto out_clear_affinity; | |
| 848 | + return err; | |
| 851 | 849 | } |
| 852 | 850 | |
| 853 | 851 | err = perf_pmu_register(&smmu_pmu->pmu, name, -1); |
| @@ -866,8 +864,6 @@ static int smmu_pmu_probe(struct platform_device *pdev) · 12 unmodified lines | ||
| 866 | 864 | |
| 867 | 865 | out_unregister: |
| 868 | 866 | cpuhp_state_remove_instance_nocalls(cpuhp_state_num, &smmu_pmu->node); |
| 869 | −out_clear_affinity: | |
| 870 | − irq_set_affinity_hint(smmu_pmu->irq, NULL); | |
| 871 | 867 | return err; |
| 872 | 868 | } |
| 873 | 869 | |
| @@ -877,7 +873,6 @@ static int smmu_pmu_remove(struct platform_device *pdev) · 3 unmodified lines | ||
| 877 | 873 | |
| 878 | 874 | perf_pmu_unregister(&smmu_pmu->pmu); |
| 879 | 875 | cpuhp_state_remove_instance_nocalls(cpuhp_state_num, &smmu_pmu->node); |
| 880 | − irq_set_affinity_hint(smmu_pmu->irq, NULL); | |
| 881 | 876 | |
| 882 | 877 | return 0; |
| 883 | 878 | } |
| @@ -231,15 +231,14 @@ static const struct attribute_group arm_spe_pmu_format_group = { · 230 unmodified lines | ||
| 231 | 231 | .attrs = arm_spe_pmu_formats_attr, |
| 232 | 232 | }; |
| 233 | 233 | |
| 234 | −static ssize_t arm_spe_pmu_get_attr_cpumask(struct device *dev, | |
| 235 | − struct device_attribute *attr, | |
| 236 | − char *buf) | |
| 234 | +static ssize_t cpumask_show(struct device *dev, | |
| 235 | + struct device_attribute *attr, char *buf) | |
| 237 | 236 | { |
| 238 | 237 | struct arm_spe_pmu *spe_pmu = dev_get_drvdata(dev); |
| 239 | 238 | |
| 240 | 239 | return cpumap_print_to_pagebuf(true, buf, &spe_pmu->supported_cpus); |
| 241 | 240 | } |
| 242 | −static DEVICE_ATTR(cpumask, S_IRUGO, arm_spe_pmu_get_attr_cpumask, NULL); | |
| 241 | +static DEVICE_ATTR_RO(cpumask); | |
| 243 | 242 | |
| 244 | 243 | static struct attribute *arm_spe_pmu_attrs[] = { |
| 245 | 244 | &dev_attr_cpumask.attr, |
| @@ -1044,7 +1043,6 @@ static void __arm_spe_pmu_dev_probe(void *info) · 798 unmodified lines | ||
| 1044 | 1043 | spe_pmu->max_record_sz, spe_pmu->align, spe_pmu->features); |
| 1045 | 1044 | |
| 1046 | 1045 | spe_pmu->features |= SPE_PMU_FEAT_DEV_PROBED; |
| 1047 | − return; | |
| 1048 | 1046 | } |
| 1049 | 1047 | |
| 1050 | 1048 | static void __arm_spe_pmu_reset_local(void) |
| @@ -1190,10 +1188,8 @@ static int arm_spe_pmu_device_probe(struct platform_device *pdev) · 139 unmodified lines | ||
| 1190 | 1188 | } |
| 1191 | 1189 | |
| 1192 | 1190 | spe_pmu = devm_kzalloc(dev, sizeof(*spe_pmu), GFP_KERNEL); |
| 1193 | − if (!spe_pmu) { | |
| 1194 | − dev_err(dev, "failed to allocate spe_pmu\n"); | |
| 1191 | + if (!spe_pmu) | |
| 1195 | 1192 | return -ENOMEM; |
| 1196 | − } | |
| 1197 | 1193 | |
| 1198 | 1194 | spe_pmu->handle = alloc_percpu(typeof(*spe_pmu->handle)); |
| 1199 | 1195 | if (!spe_pmu->handle) |
| @@ -222,11 +222,8 @@ ddr_pmu_event_show(struct device *dev, struct device_attribute *attr, · 221 unmodified lines | ||
| 222 | 222 | return sysfs_emit(page, "event=0x%02llx\n", pmu_attr->id); |
| 223 | 223 | } |
| 224 | 224 | |
| 225 | −#define IMX8_DDR_PMU_EVENT_ATTR(_name, _id) \ | |
| 226 | − (&((struct perf_pmu_events_attr[]) { \ | |
| 227 | − { .attr = __ATTR(_name, 0444, ddr_pmu_event_show, NULL),\ | |
| 228 | − .id = _id, } \ | |
| 229 | − })[0].attr.attr) | |
| 225 | +#define IMX8_DDR_PMU_EVENT_ATTR(_name, _id) \ | |
| 226 | + PMU_EVENT_ATTR_ID(_name, ddr_pmu_event_show, _id) | |
| 230 | 227 | |
| 231 | 228 | static struct attribute *ddr_perf_events_attrs[] = { |
| 232 | 229 | IMX8_DDR_PMU_EVENT_ATTR(cycles, EVENT_CYCLES_ID), |
| @@ -674,7 +671,7 @@ static int ddr_perf_offline_cpu(unsigned int cpu, struct hlist_node *node) · 441 unmodified lines | ||
| 674 | 671 | perf_pmu_migrate_context(&pmu->pmu, cpu, target); |
| 675 | 672 | pmu->cpu = target; |
| 676 | 673 | |
| 677 | − WARN_ON(irq_set_affinity_hint(pmu->irq, cpumask_of(pmu->cpu))); | |
| 674 | + WARN_ON(irq_set_affinity(pmu->irq, cpumask_of(pmu->cpu))); | |
| 678 | 675 | |
| 679 | 676 | return 0; |
| 680 | 677 | } |
| @@ -705,8 +702,10 @@ static int ddr_perf_probe(struct platform_device *pdev) · 24 unmodified lines | ||
| 705 | 702 | |
| 706 | 703 | name = devm_kasprintf(&pdev->dev, GFP_KERNEL, DDR_PERF_DEV_NAME "%d", |
| 707 | 704 | num); |
| 708 | − if (!name) | |
| 709 | − return -ENOMEM; | |
| 705 | + if (!name) { | |
| 706 | + ret = -ENOMEM; | |
| 707 | + goto cpuhp_state_err; | |
| 708 | + } | |
| 710 | 709 | |
| 711 | 710 | pmu->devtype_data = of_device_get_match_data(&pdev->dev); |
| 712 | 711 | |
| @@ -749,7 +748,7 @@ static int ddr_perf_probe(struct platform_device *pdev) · 36 unmodified lines | ||
| 749 | 748 | } |
| 750 | 749 | |
| 751 | 750 | pmu->irq = irq; |
| 752 | − ret = irq_set_affinity_hint(pmu->irq, cpumask_of(pmu->cpu)); | |
| 751 | + ret = irq_set_affinity(pmu->irq, cpumask_of(pmu->cpu)); | |
| 753 | 752 | if (ret) { |
| 754 | 753 | dev_err(pmu->dev, "Failed to set interrupt affinity!\n"); |
| 755 | 754 | goto ddr_perf_err; |
| @@ -777,7 +776,6 @@ static int ddr_perf_remove(struct platform_device *pdev) · 21 unmodified lines | ||
| 777 | 776 | |
| 778 | 777 | cpuhp_state_remove_instance_nocalls(pmu->cpuhp_state, &pmu->node); |
| 779 | 778 | cpuhp_remove_multi_state(pmu->cpuhp_state); |
| 780 | − irq_set_affinity_hint(pmu->irq, NULL); | |
| 781 | 779 | |
| 782 | 780 | perf_pmu_unregister(&pmu->pmu); |
| 783 | 781 | |
| @@ -2,7 +2,7 @@ · 1 unmodified line | ||
| 2 | 2 | /* |
| 3 | 3 | * HiSilicon SoC DDRC uncore Hardware event counters support |
| 4 | 4 | * |
| 5 | − * Copyright (C) 2017 Hisilicon Limited | |
| 5 | + * Copyright (C) 2017 HiSilicon Limited | |
| 6 | 6 | * Author: Shaokun Zhang <zhangshaokun@hisilicon.com> |
| 7 | 7 | * Anurup M <anurup.m@huawei.com> |
| 8 | 8 | * |
| @@ -537,7 +537,6 @@ static int hisi_ddrc_pmu_probe(struct platform_device *pdev) · 528 unmodified lines | ||
| 537 | 537 | dev_err(ddrc_pmu->dev, "DDRC PMU register failed!\n"); |
| 538 | 538 | cpuhp_state_remove_instance_nocalls( |
| 539 | 539 | CPUHP_AP_PERF_ARM_HISI_DDRC_ONLINE, &ddrc_pmu->node); |
| 540 | − irq_set_affinity_hint(ddrc_pmu->irq, NULL); | |
| 541 | 540 | } |
| 542 | 541 | |
| 543 | 542 | return ret; |
| @@ -550,8 +549,6 @@ static int hisi_ddrc_pmu_remove(struct platform_device *pdev) · 6 unmodified lines | ||
| 550 | 549 | perf_pmu_unregister(&ddrc_pmu->pmu); |
| 551 | 550 | cpuhp_state_remove_instance_nocalls(CPUHP_AP_PERF_ARM_HISI_DDRC_ONLINE, |
| 552 | 551 | &ddrc_pmu->node); |
| 553 | − irq_set_affinity_hint(ddrc_pmu->irq, NULL); | |
| 554 | − | |
| 555 | 552 | return 0; |
| 556 | 553 | } |
| 557 | 554 | |
| @@ -2,7 +2,7 @@ · 1 unmodified line | ||
| 2 | 2 | /* |
| 3 | 3 | * HiSilicon SoC HHA uncore Hardware event counters support |
| 4 | 4 | * |
| 5 | − * Copyright (C) 2017 Hisilicon Limited | |
| 5 | + * Copyright (C) 2017 HiSilicon Limited | |
| 6 | 6 | * Author: Shaokun Zhang <zhangshaokun@hisilicon.com> |
| 7 | 7 | * Anurup M <anurup.m@huawei.com> |
| 8 | 8 | * |
| @@ -90,7 +90,7 @@ static void hisi_hha_pmu_config_ds(struct perf_event *event) · 81 unmodified lines | ||
| 90 | 90 | |
| 91 | 91 | val = readl(hha_pmu->base + HHA_DATSRC_CTRL); |
| 92 | 92 | val |= HHA_DATSRC_SKT_EN; |
| 93 | − writel(ds_skt, hha_pmu->base + HHA_DATSRC_CTRL); | |
| 93 | + writel(val, hha_pmu->base + HHA_DATSRC_CTRL); | |
| 94 | 94 | } |
| 95 | 95 | } |
| 96 | 96 | |
| @@ -104,7 +104,7 @@ static void hisi_hha_pmu_clear_ds(struct perf_event *event) · 7 unmodified lines | ||
| 104 | 104 | |
| 105 | 105 | val = readl(hha_pmu->base + HHA_DATSRC_CTRL); |
| 106 | 106 | val &= ~HHA_DATSRC_SKT_EN; |
| 107 | − writel(ds_skt, hha_pmu->base + HHA_DATSRC_CTRL); | |
| 107 | + writel(val, hha_pmu->base + HHA_DATSRC_CTRL); | |
| 108 | 108 | } |
| 109 | 109 | } |
| 110 | 110 | |
| @@ -540,7 +540,6 @@ static int hisi_hha_pmu_probe(struct platform_device *pdev) · 429 unmodified lines | ||
| 540 | 540 | dev_err(hha_pmu->dev, "HHA PMU register failed!\n"); |
| 541 | 541 | cpuhp_state_remove_instance_nocalls( |
| 542 | 542 | CPUHP_AP_PERF_ARM_HISI_HHA_ONLINE, &hha_pmu->node); |
| 543 | − irq_set_affinity_hint(hha_pmu->irq, NULL); | |
| 544 | 543 | } |
| 545 | 544 | |
| 546 | 545 | return ret; |
| @@ -553,8 +552,6 @@ static int hisi_hha_pmu_remove(struct platform_device *pdev) · 6 unmodified lines | ||
| 553 | 552 | perf_pmu_unregister(&hha_pmu->pmu); |
| 554 | 553 | cpuhp_state_remove_instance_nocalls(CPUHP_AP_PERF_ARM_HISI_HHA_ONLINE, |
| 555 | 554 | &hha_pmu->node); |
| 556 | − irq_set_affinity_hint(hha_pmu->irq, NULL); | |
| 557 | − | |
| 558 | 555 | return 0; |
| 559 | 556 | } |
| 560 | 557 | |
| @@ -2,7 +2,7 @@ · 1 unmodified line | ||
| 2 | 2 | /* |
| 3 | 3 | * HiSilicon SoC L3C uncore Hardware event counters support |
| 4 | 4 | * |
| 5 | − * Copyright (C) 2017 Hisilicon Limited | |
| 5 | + * Copyright (C) 2017 HiSilicon Limited | |
| 6 | 6 | * Author: Anurup M <anurup.m@huawei.com> |
| 7 | 7 | * Shaokun Zhang <zhangshaokun@hisilicon.com> |
| 8 | 8 | * |
| @@ -578,7 +578,6 @@ static int hisi_l3c_pmu_probe(struct platform_device *pdev) · 569 unmodified lines | ||
| 578 | 578 | dev_err(l3c_pmu->dev, "L3C PMU register failed!\n"); |
| 579 | 579 | cpuhp_state_remove_instance_nocalls( |
| 580 | 580 | CPUHP_AP_PERF_ARM_HISI_L3_ONLINE, &l3c_pmu->node); |
| 581 | − irq_set_affinity_hint(l3c_pmu->irq, NULL); | |
| 582 | 581 | } |
| 583 | 582 | |
| 584 | 583 | return ret; |
| @@ -591,8 +590,6 @@ static int hisi_l3c_pmu_remove(struct platform_device *pdev) · 6 unmodified lines | ||
| 591 | 590 | perf_pmu_unregister(&l3c_pmu->pmu); |
| 592 | 591 | cpuhp_state_remove_instance_nocalls(CPUHP_AP_PERF_ARM_HISI_L3_ONLINE, |
| 593 | 592 | &l3c_pmu->node); |
| 594 | − irq_set_affinity_hint(l3c_pmu->irq, NULL); | |
| 595 | − | |
| 596 | 593 | return 0; |
| 597 | 594 | } |
| 598 | 595 | |
| @@ -333,7 +333,7 @@ static struct attribute *hisi_pa_pmu_identifier_attrs[] = { · 332 unmodified lines | ||
| 333 | 333 | NULL |
| 334 | 334 | }; |
| 335 | 335 | |
| 336 | −static struct attribute_group hisi_pa_pmu_identifier_group = { | |
| 336 | +static const struct attribute_group hisi_pa_pmu_identifier_group = { | |
| 337 | 337 | .attrs = hisi_pa_pmu_identifier_attrs, |
| 338 | 338 | }; |
| 339 | 339 | |
| @@ -436,7 +436,6 @@ static int hisi_pa_pmu_probe(struct platform_device *pdev) · 96 unmodified lines | ||
| 436 | 436 | dev_err(pa_pmu->dev, "PMU register failed, ret = %d\n", ret); |
| 437 | 437 | cpuhp_state_remove_instance(CPUHP_AP_PERF_ARM_HISI_PA_ONLINE, |
| 438 | 438 | &pa_pmu->node); |
| 439 | − irq_set_affinity_hint(pa_pmu->irq, NULL); | |
| 440 | 439 | return ret; |
| 441 | 440 | } |
| 442 | 441 | |
| @@ -451,8 +450,6 @@ static int hisi_pa_pmu_remove(struct platform_device *pdev) · 8 unmodified lines | ||
| 451 | 450 | perf_pmu_unregister(&pa_pmu->pmu); |
| 452 | 451 | cpuhp_state_remove_instance_nocalls(CPUHP_AP_PERF_ARM_HISI_PA_ONLINE, |
| 453 | 452 | &pa_pmu->node); |
| 454 | − irq_set_affinity_hint(pa_pmu->irq, NULL); | |
| 455 | − | |
| 456 | 453 | return 0; |
| 457 | 454 | } |
| 458 | 455 | |
| @@ -2,7 +2,7 @@ · 1 unmodified line | ||
| 2 | 2 | /* |
| 3 | 3 | * HiSilicon SoC Hardware event counters support |
| 4 | 4 | * |
| 5 | − * Copyright (C) 2017 Hisilicon Limited | |
| 5 | + * Copyright (C) 2017 HiSilicon Limited | |
| 6 | 6 | * Author: Anurup M <anurup.m@huawei.com> |
| 7 | 7 | * Shaokun Zhang <zhangshaokun@hisilicon.com> |
| 8 | 8 | * |
| @@ -488,7 +488,7 @@ int hisi_uncore_pmu_online_cpu(unsigned int cpu, struct hlist_node *node) · 479 unmodified lines | ||
| 488 | 488 | hisi_pmu->on_cpu = cpu; |
| 489 | 489 | |
| 490 | 490 | /* Overflow interrupt also should use the same CPU */ |
| 491 | − WARN_ON(irq_set_affinity_hint(hisi_pmu->irq, cpumask_of(cpu))); | |
| 491 | + WARN_ON(irq_set_affinity(hisi_pmu->irq, cpumask_of(cpu))); | |
| 492 | 492 | |
| 493 | 493 | return 0; |
| 494 | 494 | } |
| @@ -521,7 +521,7 @@ int hisi_uncore_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node) · 26 unmodified lines | ||
| 521 | 521 | perf_pmu_migrate_context(&hisi_pmu->pmu, cpu, target); |
| 522 | 522 | /* Use this CPU for event counting */ |
| 523 | 523 | hisi_pmu->on_cpu = target; |
| 524 | − WARN_ON(irq_set_affinity_hint(hisi_pmu->irq, cpumask_of(target))); | |
| 524 | + WARN_ON(irq_set_affinity(hisi_pmu->irq, cpumask_of(target))); | |
| 525 | 525 | |
| 526 | 526 | return 0; |
| 527 | 527 | } |
| @@ -2,7 +2,7 @@ · 1 unmodified line | ||
| 2 | 2 | /* |
| 3 | 3 | * HiSilicon SoC Hardware event counters support |
| 4 | 4 | * |
| 5 | − * Copyright (C) 2017 Hisilicon Limited | |
| 5 | + * Copyright (C) 2017 HiSilicon Limited | |
| 6 | 6 | * Author: Anurup M <anurup.m@huawei.com> |
| 7 | 7 | * Shaokun Zhang <zhangshaokun@hisilicon.com> |
| 8 | 8 | * |
| @@ -2,7 +2,7 @@ · 1 unmodified line | ||
| 2 | 2 | /* |
| 3 | 3 | * HiSilicon SLLC uncore Hardware event counters support |
| 4 | 4 | * |
| 5 | − * Copyright (C) 2020 Hisilicon Limited | |
| 5 | + * Copyright (C) 2020 HiSilicon Limited | |
| 6 | 6 | * Author: Shaokun Zhang <zhangshaokun@hisilicon.com> |
| 7 | 7 | * |
| 8 | 8 | * This code is based on the uncore PMUs like arm-cci and arm-ccn. |
| @@ -366,7 +366,7 @@ static struct attribute *hisi_sllc_pmu_identifier_attrs[] = { · 357 unmodified lines | ||
| 366 | 366 | NULL |
| 367 | 367 | }; |
| 368 | 368 | |
| 369 | −static struct attribute_group hisi_sllc_pmu_identifier_group = { | |
| 369 | +static const struct attribute_group hisi_sllc_pmu_identifier_group = { | |
| 370 | 370 | .attrs = hisi_sllc_pmu_identifier_attrs, |
| 371 | 371 | }; |
| 372 | 372 | |
| @@ -465,7 +465,6 @@ static int hisi_sllc_pmu_probe(struct platform_device *pdev) · 92 unmodified lines | ||
| 465 | 465 | dev_err(sllc_pmu->dev, "PMU register failed, ret = %d\n", ret); |
| 466 | 466 | cpuhp_state_remove_instance(CPUHP_AP_PERF_ARM_HISI_SLLC_ONLINE, |
| 467 | 467 | &sllc_pmu->node); |
| 468 | − irq_set_affinity_hint(sllc_pmu->irq, NULL); | |
| 469 | 468 | return ret; |
| 470 | 469 | } |
| 471 | 470 | |
| @@ -481,8 +480,6 @@ static int hisi_sllc_pmu_remove(struct platform_device *pdev) · 9 unmodified lines | ||
| 481 | 480 | perf_pmu_unregister(&sllc_pmu->pmu); |
| 482 | 481 | cpuhp_state_remove_instance_nocalls(CPUHP_AP_PERF_ARM_HISI_SLLC_ONLINE, |
| 483 | 482 | &sllc_pmu->node); |
| 484 | − irq_set_affinity_hint(sllc_pmu->irq, NULL); | |
| 485 | − | |
| 486 | 483 | return 0; |
| 487 | 484 | } |
| 488 | 485 | |
| @@ -679,11 +679,8 @@ static ssize_t l2cache_pmu_event_show(struct device *dev, · 678 unmodified lines | ||
| 679 | 679 | return sysfs_emit(page, "event=0x%02llx\n", pmu_attr->id); |
| 680 | 680 | } |
| 681 | 681 | |
| 682 | −#define L2CACHE_EVENT_ATTR(_name, _id) \ | |
| 683 | − (&((struct perf_pmu_events_attr[]) { \ | |
| 684 | − { .attr = __ATTR(_name, 0444, l2cache_pmu_event_show, NULL), \ | |
| 685 | − .id = _id, } \ | |
| 686 | − })[0].attr.attr) | |
| 682 | +#define L2CACHE_EVENT_ATTR(_name, _id) \ | |
| 683 | + PMU_EVENT_ATTR_ID(_name, l2cache_pmu_event_show, _id) | |
| 687 | 684 | |
| 688 | 685 | static struct attribute *l2_cache_pmu_events[] = { |
| 689 | 686 | L2CACHE_EVENT_ATTR(cycles, L2_EVENT_CYCLES), |
| @@ -869,14 +866,14 @@ static int l2_cache_pmu_probe_cluster(struct device *dev, void *data) · 179 unmodified lines | ||
| 869 | 866 | irq = platform_get_irq(sdev, 0); |
| 870 | 867 | if (irq < 0) |
| 871 | 868 | return irq; |
| 872 | − irq_set_status_flags(irq, IRQ_NOAUTOEN); | |
| 873 | 869 | cluster->irq = irq; |
| 874 | 870 | |
| 875 | 871 | cluster->l2cache_pmu = l2cache_pmu; |
| 876 | 872 | cluster->on_cpu = -1; |
| 877 | 873 | |
| 878 | 874 | err = devm_request_irq(&pdev->dev, irq, l2_cache_handle_irq, |
| 879 | − IRQF_NOBALANCING | IRQF_NO_THREAD, | |
| 875 | + IRQF_NOBALANCING | IRQF_NO_THREAD | | |
| 876 | + IRQF_NO_AUTOEN, | |
| 880 | 877 | "l2-cache-pmu", cluster); |
| 881 | 878 | if (err) { |
| 882 | 879 | dev_err(&pdev->dev, |
| @@ -647,10 +647,7 @@ static ssize_t l3cache_pmu_event_show(struct device *dev, · 646 unmodified lines | ||
| 647 | 647 | } |
| 648 | 648 | |
| 649 | 649 | #define L3CACHE_EVENT_ATTR(_name, _id) \ |
| 650 | − (&((struct perf_pmu_events_attr[]) { \ | |
| 651 | − { .attr = __ATTR(_name, 0444, l3cache_pmu_event_show, NULL), \ | |
| 652 | − .id = _id, } \ | |
| 653 | − })[0].attr.attr) | |
| 650 | + PMU_EVENT_ATTR_ID(_name, l3cache_pmu_event_show, _id) | |
| 654 | 651 | |
| 655 | 652 | static struct attribute *qcom_l3_cache_pmu_events[] = { |
| 656 | 653 | L3CACHE_EVENT_ATTR(cycles, L3_EVENT_CYCLES), |
| @@ -670,15 +667,15 @@ static const struct attribute_group qcom_l3_cache_pmu_events_group = { · 13 unmodified lines | ||
| 670 | 667 | |
| 671 | 668 | /* cpumask */ |
| 672 | 669 | |
| 673 | −static ssize_t qcom_l3_cache_pmu_cpumask_show(struct device *dev, | |
| 674 | − struct device_attribute *attr, char *buf) | |
| 670 | +static ssize_t cpumask_show(struct device *dev, | |
| 671 | + struct device_attribute *attr, char *buf) | |
| 675 | 672 | { |
| 676 | 673 | struct l3cache_pmu *l3pmu = to_l3cache_pmu(dev_get_drvdata(dev)); |
| 677 | 674 | |
| 678 | 675 | return cpumap_print_to_pagebuf(true, buf, &l3pmu->cpumask); |
| 679 | 676 | } |
| 680 | 677 | |
| 681 | −static DEVICE_ATTR(cpumask, 0444, qcom_l3_cache_pmu_cpumask_show, NULL); | |
| 678 | +static DEVICE_ATTR_RO(cpumask); | |
| 682 | 679 | |
| 683 | 680 | static struct attribute *qcom_l3_cache_pmu_cpumask_attrs[] = { |
| 684 | 681 | &dev_attr_cpumask.attr, |
| @@ -767,10 +764,8 @@ static int qcom_l3_cache_pmu_probe(struct platform_device *pdev) · 82 unmodified lines | ||
| 767 | 764 | |
| 768 | 765 | memrc = platform_get_resource(pdev, IORESOURCE_MEM, 0); |
| 769 | 766 | l3pmu->regs = devm_ioremap_resource(&pdev->dev, memrc); |
| 770 | − if (IS_ERR(l3pmu->regs)) { | |
| 771 | − dev_err(&pdev->dev, "Can't map PMU @%pa\n", &memrc->start); | |
| 767 | + if (IS_ERR(l3pmu->regs)) | |
| 772 | 768 | return PTR_ERR(l3pmu->regs); |
| 773 | − } | |
| 774 | 769 | |
| 775 | 770 | qcom_l3_cache__init(l3pmu); |
| 776 | 771 | |
| @@ -817,10 +817,8 @@ static struct tx2_uncore_pmu *tx2_uncore_pmu_init_dev(struct device *dev, · 816 unmodified lines | ||
| 817 | 817 | } |
| 818 | 818 | |
| 819 | 819 | base = devm_ioremap_resource(dev, &res); |
| 820 | − if (IS_ERR(base)) { | |
| 821 | − dev_err(dev, "PMU type %d: Fail to map resource\n", type); | |
| 820 | + if (IS_ERR(base)) | |
| 822 | 821 | return NULL; |
| 823 | − } | |
| 824 | 822 | |
| 825 | 823 | tx2_pmu = devm_kzalloc(dev, sizeof(*tx2_pmu), GFP_KERNEL); |
| 826 | 824 | if (!tx2_pmu) |
| @@ -278,17 +278,14 @@ static const struct attribute_group mc_pmu_v3_format_attr_group = { · 277 unmodified lines | ||
| 278 | 278 | static ssize_t xgene_pmu_event_show(struct device *dev, |
| 279 | 279 | struct device_attribute *attr, char *buf) |
| 280 | 280 | { |
| 281 | − struct dev_ext_attribute *eattr; | |
| 281 | + struct perf_pmu_events_attr *pmu_attr = | |
| 282 | + container_of(attr, struct perf_pmu_events_attr, attr); | |
| 282 | 283 | |
| 283 | − eattr = container_of(attr, struct dev_ext_attribute, attr); | |
| 284 | − return sysfs_emit(buf, "config=0x%lx\n", (unsigned long) eattr->var); | |
| 284 | + return sysfs_emit(buf, "config=0x%llx\n", pmu_attr->id); | |
| 285 | 285 | } |
| 286 | 286 | |
| 287 | 287 | #define XGENE_PMU_EVENT_ATTR(_name, _config) \ |
| 288 | − (&((struct dev_ext_attribute[]) { \ | |
| 289 | − { .attr = __ATTR(_name, S_IRUGO, xgene_pmu_event_show, NULL), \ | |
| 290 | − .var = (void *) _config, } \ | |
| 291 | − })[0].attr.attr) | |
| 288 | + PMU_EVENT_ATTR_ID(_name, xgene_pmu_event_show, _config) | |
| 292 | 289 | |
| 293 | 290 | static struct attribute *l3c_pmu_events_attrs[] = { |
| 294 | 291 | XGENE_PMU_EVENT_ATTR(cycle-count, 0x00), |
| @@ -604,15 +601,15 @@ static const struct attribute_group mc_pmu_v3_events_attr_group = { · 309 unmodified lines | ||
| 604 | 601 | /* |
| 605 | 602 | * sysfs cpumask attributes |
| 606 | 603 | */ |
| 607 | −static ssize_t xgene_pmu_cpumask_show(struct device *dev, | |
| 608 | − struct device_attribute *attr, char *buf) | |
| 604 | +static ssize_t cpumask_show(struct device *dev, | |
| 605 | + struct device_attribute *attr, char *buf) | |
| 609 | 606 | { |
| 610 | 607 | struct xgene_pmu_dev *pmu_dev = to_pmu_dev(dev_get_drvdata(dev)); |
| 611 | 608 | |
| 612 | 609 | return cpumap_print_to_pagebuf(true, buf, &pmu_dev->parent->cpu); |
| 613 | 610 | } |
| 614 | 611 | |
| 615 | −static DEVICE_ATTR(cpumask, S_IRUGO, xgene_pmu_cpumask_show, NULL); | |
| 612 | +static DEVICE_ATTR_RO(cpumask); | |
| 616 | 613 | |
| 617 | 614 | static struct attribute *xgene_pmu_cpumask_attrs[] = { |
| 618 | 615 | &dev_attr_cpumask.attr, |
| @@ -63,6 +63,9 @@ · 62 unmodified lines | ||
| 63 | 63 | #define ARM_SMCCC_VERSION_1_0 0x10000 |
| 64 | 64 | #define ARM_SMCCC_VERSION_1_1 0x10001 |
| 65 | 65 | #define ARM_SMCCC_VERSION_1_2 0x10002 |
| 66 | +#define ARM_SMCCC_VERSION_1_3 0x10003 | |
| 67 | + | |
| 68 | +#define ARM_SMCCC_1_3_SVE_HINT 0x10000 | |
| 66 | 69 | |
| 67 | 70 | #define ARM_SMCCC_VERSION_FUNC_ID \ |
| 68 | 71 | ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, \ |
| @@ -216,6 +219,8 @@ u32 arm_smccc_get_version(void); · 147 unmodified lines | ||
| 216 | 219 | |
| 217 | 220 | void __init arm_smccc_version_init(u32 version, enum arm_smccc_conduit conduit); |
| 218 | 221 | |
| 222 | +extern u64 smccc_has_sve_hint; | |
| 223 | + | |
| 219 | 224 | /** |
| 220 | 225 | * struct arm_smccc_res - Result from SMC/HVC call |
| 221 | 226 | * @a0-a3 result values from registers 0 to 3 |
| @@ -227,6 +232,61 @@ struct arm_smccc_res { · 5 unmodified lines | ||
| 227 | 232 | unsigned long a3; |
| 228 | 233 | }; |
| 229 | 234 | |
| 235 | +#ifdef CONFIG_ARM64 | |
| 236 | +/** | |
| 237 | + * struct arm_smccc_1_2_regs - Arguments for or Results from SMC/HVC call | |
| 238 | + * @a0-a17 argument values from registers 0 to 17 | |
| 239 | + */ | |
| 240 | +struct arm_smccc_1_2_regs { | |
| 241 | + unsigned long a0; | |
| 242 | + unsigned long a1; | |
| 243 | + unsigned long a2; | |
| 244 | + unsigned long a3; | |
| 245 | + unsigned long a4; | |
| 246 | + unsigned long a5; | |
| 247 | + unsigned long a6; | |
| 248 | + unsigned long a7; | |
| 249 | + unsigned long a8; | |
| 250 | + unsigned long a9; | |
| 251 | + unsigned long a10; | |
| 252 | + unsigned long a11; | |
| 253 | + unsigned long a12; | |
| 254 | + unsigned long a13; | |
| 255 | + unsigned long a14; | |
| 256 | + unsigned long a15; | |
| 257 | + unsigned long a16; | |
| 258 | + unsigned long a17; | |
| 259 | +}; | |
| 260 | + | |
| 261 | +/** | |
| 262 | + * arm_smccc_1_2_hvc() - make HVC calls | |
| 263 | + * @args: arguments passed via struct arm_smccc_1_2_regs | |
| 264 | + * @res: result values via struct arm_smccc_1_2_regs | |
| 265 | + * | |
| 266 | + * This function is used to make HVC calls following SMC Calling Convention | |
| 267 | + * v1.2 or above. The content of the supplied param are copied from the | |
| 268 | + * structure to registers prior to the HVC instruction. The return values | |
| 269 | + * are updated with the content from registers on return from the HVC | |
| 270 | + * instruction. | |
| 271 | + */ | |
| 272 | +asmlinkage void arm_smccc_1_2_hvc(const struct arm_smccc_1_2_regs *args, | |
| 273 | + struct arm_smccc_1_2_regs *res); | |
| 274 | + | |
| 275 | +/** | |
| 276 | + * arm_smccc_1_2_smc() - make SMC calls | |
| 277 | + * @args: arguments passed via struct arm_smccc_1_2_regs | |
| 278 | + * @res: result values via struct arm_smccc_1_2_regs | |
| 279 | + * | |
| 280 | + * This function is used to make SMC calls following SMC Calling Convention | |
| 281 | + * v1.2 or above. The content of the supplied param are copied from the | |
| 282 | + * structure to registers prior to the SMC instruction. The return values | |
| 283 | + * are updated with the content from registers on return from the SMC | |
| 284 | + * instruction. | |
| 285 | + */ | |
| 286 | +asmlinkage void arm_smccc_1_2_smc(const struct arm_smccc_1_2_regs *args, | |
| 287 | + struct arm_smccc_1_2_regs *res); | |
| 288 | +#endif | |
| 289 | + | |
| 230 | 290 | /** |
| 231 | 291 | * struct arm_smccc_quirk - Contains quirk information |
| 232 | 292 | * @id: quirk identification |
| @@ -240,6 +300,15 @@ struct arm_smccc_quirk { · 7 unmodified lines | ||
| 240 | 300 | } state; |
| 241 | 301 | }; |
| 242 | 302 | |
| 303 | +/** | |
| 304 | + * __arm_smccc_sve_check() - Set the SVE hint bit when doing SMC calls | |
| 305 | + * | |
| 306 | + * Sets the SMCCC hint bit to indicate if there is live state in the SVE | |
| 307 | + * registers, this modifies x0 in place and should never be called from C | |
| 308 | + * code. | |
| 309 | + */ | |
| 310 | +asmlinkage unsigned long __arm_smccc_sve_check(unsigned long x0); | |
| 311 | + | |
| 243 | 312 | /** |
| 244 | 313 | * __arm_smccc_smc() - make SMC calls |
| 245 | 314 | * @a0-a7: arguments passed in registers 0 to 7 |
| @@ -297,6 +366,20 @@ asmlinkage void __arm_smccc_hvc(unsigned long a0, unsigned long a1, · 51 unmodified lines | ||
| 297 | 366 | |
| 298 | 367 | #endif |
| 299 | 368 | |
| 369 | +/* nVHE hypervisor doesn't have a current thread so needs separate checks */ | |
| 370 | +#if defined(CONFIG_ARM64_SVE) && !defined(__KVM_NVHE_HYPERVISOR__) | |
| 371 | + | |
| 372 | +#define SMCCC_SVE_CHECK ALTERNATIVE("nop \n", "bl __arm_smccc_sve_check \n", \ | |
| 373 | + ARM64_SVE) | |
| 374 | +#define smccc_sve_clobbers "x16", "x30", "cc", | |
| 375 | + | |
| 376 | +#else | |
| 377 | + | |
| 378 | +#define SMCCC_SVE_CHECK | |
| 379 | +#define smccc_sve_clobbers | |
| 380 | + | |
| 381 | +#endif | |
| 382 | + | |
| 300 | 383 | #define ___count_args(_0, _1, _2, _3, _4, _5, _6, _7, _8, x, ...) x |
| 301 | 384 | |
| 302 | 385 | #define __count_args(...) \ |
| @@ -364,7 +447,7 @@ asmlinkage void __arm_smccc_hvc(unsigned long a0, unsigned long a1, · 61 unmodified lines | ||
| 364 | 447 | |
| 365 | 448 | #define ___constraints(count) \ |
| 366 | 449 | : __constraint_read_ ## count \ |
| 367 | − : "memory" | |
| 450 | + : smccc_sve_clobbers "memory" | |
| 368 | 451 | #define __constraints(count) ___constraints(count) |
| 369 | 452 | |
| 370 | 453 | /* |
| @@ -379,7 +462,8 @@ asmlinkage void __arm_smccc_hvc(unsigned long a0, unsigned long a1, · 8 unmodified lines | ||
| 379 | 462 | register unsigned long r2 asm("r2"); \ |
| 380 | 463 | register unsigned long r3 asm("r3"); \ |
| 381 | 464 | __declare_args(__count_args(__VA_ARGS__), __VA_ARGS__); \ |
| 382 | − asm volatile(inst "\n" : \ | |
| 465 | + asm volatile(SMCCC_SVE_CHECK \ | |
| 466 | + inst "\n" : \ | |
| 383 | 467 | "=r" (r0), "=r" (r1), "=r" (r2), "=r" (r3) \ |
| 384 | 468 | __constraints(__count_args(__VA_ARGS__))); \ |
| 385 | 469 | if (___res) \ |
| @@ -53,8 +53,10 @@ struct vm_area_struct; · 52 unmodified lines | ||
| 53 | 53 | #define ___GFP_HARDWALL 0x100000u |
| 54 | 54 | #define ___GFP_THISNODE 0x200000u |
| 55 | 55 | #define ___GFP_ACCOUNT 0x400000u |
| 56 | +#define ___GFP_ZEROTAGS 0x800000u | |
| 57 | +#define ___GFP_SKIP_KASAN_POISON 0x1000000u | |
| 56 | 58 | #ifdef CONFIG_LOCKDEP |
| 57 | −#define ___GFP_NOLOCKDEP 0x800000u | |
| 59 | +#define ___GFP_NOLOCKDEP 0x2000000u | |
| 58 | 60 | #else |
| 59 | 61 | #define ___GFP_NOLOCKDEP 0 |
| 60 | 62 | #endif |
| @@ -229,16 +231,25 @@ struct vm_area_struct; · 168 unmodified lines | ||
| 229 | 231 | * %__GFP_COMP address compound page metadata. |
| 230 | 232 | * |
| 231 | 233 | * %__GFP_ZERO returns a zeroed page on success. |
| 234 | + * | |
| 235 | + * %__GFP_ZEROTAGS returns a page with zeroed memory tags on success, if | |
| 236 | + * __GFP_ZERO is set. | |
| 237 | + * | |
| 238 | + * %__GFP_SKIP_KASAN_POISON returns a page which does not need to be poisoned | |
| 239 | + * on deallocation. Typically used for userspace pages. Currently only has an | |
| 240 | + * effect in HW tags mode. | |
| 232 | 241 | */ |
| 233 | 242 | #define __GFP_NOWARN ((__force gfp_t)___GFP_NOWARN) |
| 234 | 243 | #define __GFP_COMP ((__force gfp_t)___GFP_COMP) |
| 235 | 244 | #define __GFP_ZERO ((__force gfp_t)___GFP_ZERO) |
| 245 | +#define __GFP_ZEROTAGS ((__force gfp_t)___GFP_ZEROTAGS) | |
| 246 | +#define __GFP_SKIP_KASAN_POISON ((__force gfp_t)___GFP_SKIP_KASAN_POISON) | |
| 236 | 247 | |
| 237 | 248 | /* Disable lockdep for GFP context tracking */ |
| 238 | 249 | #define __GFP_NOLOCKDEP ((__force gfp_t)___GFP_NOLOCKDEP) |
| 239 | 250 | |
| 240 | 251 | /* Room for N __GFP_FOO bits */ |
| 241 | −#define __GFP_BITS_SHIFT (23 + IS_ENABLED(CONFIG_LOCKDEP)) | |
| 252 | +#define __GFP_BITS_SHIFT (25 + IS_ENABLED(CONFIG_LOCKDEP)) | |
| 242 | 253 | #define __GFP_BITS_MASK ((__force gfp_t)((1 << __GFP_BITS_SHIFT) - 1)) |
| 243 | 254 | |
| 244 | 255 | /** |
| @@ -319,7 +330,8 @@ struct vm_area_struct; · 74 unmodified lines | ||
| 319 | 330 | #define GFP_DMA __GFP_DMA |
| 320 | 331 | #define GFP_DMA32 __GFP_DMA32 |
| 321 | 332 | #define GFP_HIGHUSER (GFP_USER | __GFP_HIGHMEM) |
| 322 | −#define GFP_HIGHUSER_MOVABLE (GFP_HIGHUSER | __GFP_MOVABLE) | |
| 333 | +#define GFP_HIGHUSER_MOVABLE (GFP_HIGHUSER | __GFP_MOVABLE | \ | |
| 334 | + __GFP_SKIP_KASAN_POISON) | |
| 323 | 335 | #define GFP_TRANSHUGE_LIGHT ((GFP_HIGHUSER_MOVABLE | __GFP_COMP | \ |
| 324 | 336 | __GFP_NOMEMALLOC | __GFP_NOWARN) & ~__GFP_RECLAIM) |
| 325 | 337 | #define GFP_TRANSHUGE (GFP_TRANSHUGE_LIGHT | __GFP_DIRECT_RECLAIM) |
| @@ -152,28 +152,24 @@ static inline void clear_user_highpage(struct page *page, unsigned long vaddr) · 151 unmodified lines | ||
| 152 | 152 | } |
| 153 | 153 | #endif |
| 154 | 154 | |
| 155 | −#ifndef __HAVE_ARCH_ALLOC_ZEROED_USER_HIGHPAGE | |
| 155 | +#ifndef __HAVE_ARCH_ALLOC_ZEROED_USER_HIGHPAGE_MOVABLE | |
| 156 | 156 | /** |
| 157 | − * __alloc_zeroed_user_highpage - Allocate a zeroed HIGHMEM page for a VMA with caller-specified movable GFP flags | |
| 158 | − * @movableflags: The GFP flags related to the pages future ability to move like __GFP_MOVABLE | |
| 157 | + * alloc_zeroed_user_highpage_movable - Allocate a zeroed HIGHMEM page for a VMA that the caller knows can move | |
| 159 | 158 | * @vma: The VMA the page is to be allocated for |
| 160 | 159 | * @vaddr: The virtual address the page will be inserted into |
| 161 | 160 | * |
| 162 | − * This function will allocate a page for a VMA but the caller is expected | |
| 163 | − * to specify via movableflags whether the page will be movable in the | |
| 164 | − * future or not | |
| 161 | + * This function will allocate a page for a VMA that the caller knows will | |
| 162 | + * be able to migrate in the future using move_pages() or reclaimed | |
| 165 | 163 | * |
| 166 | 164 | * An architecture may override this function by defining |
| 167 | − * __HAVE_ARCH_ALLOC_ZEROED_USER_HIGHPAGE and providing their own | |
| 165 | + * __HAVE_ARCH_ALLOC_ZEROED_USER_HIGHPAGE_MOVABLE and providing their own | |
| 168 | 166 | * implementation. |
| 169 | 167 | */ |
| 170 | 168 | static inline struct page * |
| 171 | −__alloc_zeroed_user_highpage(gfp_t movableflags, | |
| 172 | − struct vm_area_struct *vma, | |
| 173 | − unsigned long vaddr) | |
| 169 | +alloc_zeroed_user_highpage_movable(struct vm_area_struct *vma, | |
| 170 | + unsigned long vaddr) | |
| 174 | 171 | { |
| 175 | − struct page *page = alloc_page_vma(GFP_HIGHUSER | movableflags, | |
| 176 | − vma, vaddr); | |
| 172 | + struct page *page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, vaddr); | |
| 177 | 173 | |
| 178 | 174 | if (page) |
| 179 | 175 | clear_user_highpage(page, vaddr); |
| @@ -182,21 +178,6 @@ __alloc_zeroed_user_highpage(gfp_t movableflags, · 2 unmodified lines | ||
| 182 | 178 | } |
| 183 | 179 | #endif |
| 184 | 180 | |
| 185 | −/** | |
| 186 | − * alloc_zeroed_user_highpage_movable - Allocate a zeroed HIGHMEM page for a VMA that the caller knows can move | |
| 187 | − * @vma: The VMA the page is to be allocated for | |
| 188 | − * @vaddr: The virtual address the page will be inserted into | |
| 189 | − * | |
| 190 | − * This function will allocate a page for a VMA that the caller knows will | |
| 191 | − * be able to migrate in the future using move_pages() or reclaimed | |
| 192 | − */ | |
| 193 | −static inline struct page * | |
| 194 | −alloc_zeroed_user_highpage_movable(struct vm_area_struct *vma, | |
| 195 | − unsigned long vaddr) | |
| 196 | −{ | |
| 197 | − return __alloc_zeroed_user_highpage(__GFP_MOVABLE, vma, vaddr); | |
| 198 | −} | |
| 199 | − | |
| 200 | 181 | static inline void clear_highpage(struct page *page) |
| 201 | 182 | { |
| 202 | 183 | void *kaddr = kmap_atomic(page); |
| @@ -204,6 +185,14 @@ static inline void clear_highpage(struct page *page) · 1 unmodified line | ||
| 204 | 185 | kunmap_atomic(kaddr); |
| 205 | 186 | } |
| 206 | 187 | |
| 188 | +#ifndef __HAVE_ARCH_TAG_CLEAR_HIGHPAGE | |
| 189 | + | |
| 190 | +static inline void tag_clear_highpage(struct page *page) | |
| 191 | +{ | |
| 192 | +} | |
| 193 | + | |
| 194 | +#endif | |
| 195 | + | |
| 207 | 196 | /* |
| 208 | 197 | * If we pass in a base or tail page, we can zero up to PAGE_SIZE. |
| 209 | 198 | * If we pass in a head page, we can zero up to the size of the compound page. |
| @@ -319,39 +319,8 @@ struct irq_affinity_desc { · 318 unmodified lines | ||
| 319 | 319 | |
| 320 | 320 | extern cpumask_var_t irq_default_affinity; |
| 321 | 321 | |
| 322 | −/* Internal implementation. Use the helpers below */ | |
| 323 | −extern int __irq_set_affinity(unsigned int irq, const struct cpumask *cpumask, | |
| 324 | − bool force); | |
| 325 | − | |
| 326 | −/** | |
| 327 | − * irq_set_affinity - Set the irq affinity of a given irq | |
| 328 | − * @irq: Interrupt to set affinity | |
| 329 | − * @cpumask: cpumask | |
| 330 | − * | |
| 331 | − * Fails if cpumask does not contain an online CPU | |
| 332 | − */ | |
| 333 | −static inline int | |
| 334 | −irq_set_affinity(unsigned int irq, const struct cpumask *cpumask) | |
| 335 | −{ | |
| 336 | − return __irq_set_affinity(irq, cpumask, false); | |
| 337 | −} | |
| 338 | − | |
| 339 | −/** | |
| 340 | − * irq_force_affinity - Force the irq affinity of a given irq | |
| 341 | − * @irq: Interrupt to set affinity | |
| 342 | − * @cpumask: cpumask | |
| 343 | − * | |
| 344 | − * Same as irq_set_affinity, but without checking the mask against | |
| 345 | − * online cpus. | |
| 346 | − * | |
| 347 | − * Solely for low level cpu hotplug code, where we need to make per | |
| 348 | − * cpu interrupts affine before the cpu becomes online. | |
| 349 | − */ | |
| 350 | −static inline int | |
| 351 | −irq_force_affinity(unsigned int irq, const struct cpumask *cpumask) | |
| 352 | −{ | |
| 353 | − return __irq_set_affinity(irq, cpumask, true); | |
| 354 | −} | |
| 322 | +extern int irq_set_affinity(unsigned int irq, const struct cpumask *cpumask); | |
| 323 | +extern int irq_force_affinity(unsigned int irq, const struct cpumask *cpumask); | |
| 355 | 324 | |
| 356 | 325 | extern int irq_can_set_affinity(unsigned int irq); |
| 357 | 326 | extern int irq_select_affinity(unsigned int irq); |
| @@ -2,6 +2,7 @@ · 1 unmodified line | ||
| 2 | 2 | #ifndef _LINUX_KASAN_H |
| 3 | 3 | #define _LINUX_KASAN_H |
| 4 | 4 | |
| 5 | +#include <linux/bug.h> | |
| 5 | 6 | #include <linux/static_key.h> |
| 6 | 7 | #include <linux/types.h> |
| 7 | 8 | |
| @@ -79,14 +80,6 @@ static inline void kasan_disable_current(void) {} · 71 unmodified lines | ||
| 79 | 80 | |
| 80 | 81 | #endif /* CONFIG_KASAN_GENERIC || CONFIG_KASAN_SW_TAGS */ |
| 81 | 82 | |
| 82 | −#ifdef CONFIG_KASAN | |
| 83 | − | |
| 84 | −struct kasan_cache { | |
| 85 | − int alloc_meta_offset; | |
| 86 | − int free_meta_offset; | |
| 87 | − bool is_kmalloc; | |
| 88 | −}; | |
| 89 | − | |
| 90 | 83 | #ifdef CONFIG_KASAN_HW_TAGS |
| 91 | 84 | |
| 92 | 85 | DECLARE_STATIC_KEY_FALSE(kasan_flag_enabled); |
| @@ -101,11 +94,14 @@ static inline bool kasan_has_integrated_init(void) · 8 unmodified lines | ||
| 101 | 94 | return kasan_enabled(); |
| 102 | 95 | } |
| 103 | 96 | |
| 97 | +void kasan_alloc_pages(struct page *page, unsigned int order, gfp_t flags); | |
| 98 | +void kasan_free_pages(struct page *page, unsigned int order); | |
| 99 | + | |
| 104 | 100 | #else /* CONFIG_KASAN_HW_TAGS */ |
| 105 | 101 | |
| 106 | 102 | static inline bool kasan_enabled(void) |
| 107 | 103 | { |
| 108 | − return true; | |
| 104 | + return IS_ENABLED(CONFIG_KASAN); | |
| 109 | 105 | } |
| 110 | 106 | |
| 111 | 107 | static inline bool kasan_has_integrated_init(void) |
| @@ -113,8 +109,30 @@ static inline bool kasan_has_integrated_init(void) · 1 unmodified line | ||
| 113 | 109 | return false; |
| 114 | 110 | } |
| 115 | 111 | |
| 112 | +static __always_inline void kasan_alloc_pages(struct page *page, | |
| 113 | + unsigned int order, gfp_t flags) | |
| 114 | +{ | |
| 115 | + /* Only available for integrated init. */ | |
| 116 | + BUILD_BUG(); | |
| 117 | +} | |
| 118 | + | |
| 119 | +static __always_inline void kasan_free_pages(struct page *page, | |
| 120 | + unsigned int order) | |
| 121 | +{ | |
| 122 | + /* Only available for integrated init. */ | |
| 123 | + BUILD_BUG(); | |
| 124 | +} | |
| 125 | + | |
| 116 | 126 | #endif /* CONFIG_KASAN_HW_TAGS */ |
| 117 | 127 | |
| 128 | +#ifdef CONFIG_KASAN | |
| 129 | + | |
| 130 | +struct kasan_cache { | |
| 131 | + int alloc_meta_offset; | |
| 132 | + int free_meta_offset; | |
| 133 | + bool is_kmalloc; | |
| 134 | +}; | |
| 135 | + | |
| 118 | 136 | slab_flags_t __kasan_never_merge(void); |
| 119 | 137 | static __always_inline slab_flags_t kasan_never_merge(void) |
| 120 | 138 | { |
| @@ -130,20 +148,20 @@ static __always_inline void kasan_unpoison_range(const void *addr, size_t size) · 9 unmodified lines | ||
| 130 | 148 | __kasan_unpoison_range(addr, size); |
| 131 | 149 | } |
| 132 | 150 | |
| 133 | −void __kasan_alloc_pages(struct page *page, unsigned int order, bool init); | |
| 134 | −static __always_inline void kasan_alloc_pages(struct page *page, | |
| 151 | +void __kasan_poison_pages(struct page *page, unsigned int order, bool init); | |
| 152 | +static __always_inline void kasan_poison_pages(struct page *page, | |
| 135 | 153 | unsigned int order, bool init) |
| 136 | 154 | { |
| 137 | 155 | if (kasan_enabled()) |
| 138 | − __kasan_alloc_pages(page, order, init); | |
| 156 | + __kasan_poison_pages(page, order, init); | |
| 139 | 157 | } |
| 140 | 158 | |
| 141 | −void __kasan_free_pages(struct page *page, unsigned int order, bool init); | |
| 142 | −static __always_inline void kasan_free_pages(struct page *page, | |
| 143 | − unsigned int order, bool init) | |
| 159 | +void __kasan_unpoison_pages(struct page *page, unsigned int order, bool init); | |
| 160 | +static __always_inline void kasan_unpoison_pages(struct page *page, | |
| 161 | + unsigned int order, bool init) | |
| 144 | 162 | { |
| 145 | 163 | if (kasan_enabled()) |
| 146 | − __kasan_free_pages(page, order, init); | |
| 164 | + __kasan_unpoison_pages(page, order, init); | |
| 147 | 165 | } |
| 148 | 166 | |
| 149 | 167 | void __kasan_cache_create(struct kmem_cache *cache, unsigned int *size, |
| @@ -285,21 +303,15 @@ void kasan_restore_multi_shot(bool enabled); · 135 unmodified lines | ||
| 285 | 303 | |
| 286 | 304 | #else /* CONFIG_KASAN */ |
| 287 | 305 | |
| 288 | −static inline bool kasan_enabled(void) | |
| 289 | −{ | |
| 290 | − return false; | |
| 291 | −} | |
| 292 | −static inline bool kasan_has_integrated_init(void) | |
| 293 | −{ | |
| 294 | − return false; | |
| 295 | −} | |
| 296 | 306 | static inline slab_flags_t kasan_never_merge(void) |
| 297 | 307 | { |
| 298 | 308 | return 0; |
| 299 | 309 | } |
| 300 | 310 | static inline void kasan_unpoison_range(const void *address, size_t size) {} |
| 301 | −static inline void kasan_alloc_pages(struct page *page, unsigned int order, bool init) {} | |
| 302 | −static inline void kasan_free_pages(struct page *page, unsigned int order, bool init) {} | |
| 311 | +static inline void kasan_poison_pages(struct page *page, unsigned int order, | |
| 312 | + bool init) {} | |
| 313 | +static inline void kasan_unpoison_pages(struct page *page, unsigned int order, | |
| 314 | + bool init) {} | |
| 303 | 315 | static inline void kasan_cache_create(struct kmem_cache *cache, |
| 304 | 316 | unsigned int *size, |
| 305 | 317 | slab_flags_t *flags) {} |
| @@ -137,6 +137,9 @@ enum pageflags { · 136 unmodified lines | ||
| 137 | 137 | #endif |
| 138 | 138 | #ifdef CONFIG_64BIT |
| 139 | 139 | PG_arch_2, |
| 140 | +#endif | |
| 141 | +#ifdef CONFIG_KASAN_HW_TAGS | |
| 142 | + PG_skip_kasan_poison, | |
| 140 | 143 | #endif |
| 141 | 144 | __NR_PAGEFLAGS, |
| 142 | 145 | |
| @@ -443,6 +446,12 @@ TESTCLEARFLAG(Young, young, PF_ANY) · 300 unmodified lines | ||
| 443 | 446 | PAGEFLAG(Idle, idle, PF_ANY) |
| 444 | 447 | #endif |
| 445 | 448 | |
| 449 | +#ifdef CONFIG_KASAN_HW_TAGS | |
| 450 | +PAGEFLAG(SkipKASanPoison, skip_kasan_poison, PF_HEAD) | |
| 451 | +#else | |
| 452 | +PAGEFLAG_FALSE(SkipKASanPoison) | |
| 453 | +#endif | |
| 454 | + | |
| 446 | 455 | /* |
| 447 | 456 | * PageReported() is used to track reported free pages within the Buddy |
| 448 | 457 | * allocator. We can use the non-atomic version of the test and set |
| @@ -1576,6 +1576,12 @@ static struct perf_pmu_events_attr _var = { \ · 1575 unmodified lines | ||
| 1576 | 1576 | .event_str = _str, \ |
| 1577 | 1577 | }; |
| 1578 | 1578 | |
| 1579 | +#define PMU_EVENT_ATTR_ID(_name, _show, _id) \ | |
| 1580 | + (&((struct perf_pmu_events_attr[]) { \ | |
| 1581 | + { .attr = __ATTR(_name, 0444, _show, NULL), \ | |
| 1582 | + .id = _id, } \ | |
| 1583 | + })[0].attr.attr) | |
| 1584 | + | |
| 1579 | 1585 | #define PMU_FORMAT_ATTR(_name, _format) \ |
| 1580 | 1586 | static ssize_t \ |
| 1581 | 1587 | _name##_show(struct device *dev, \ |
| @@ -85,6 +85,12 @@ · 84 unmodified lines | ||
| 85 | 85 | #define IF_HAVE_PG_ARCH_2(flag,string) |
| 86 | 86 | #endif |
| 87 | 87 | |
| 88 | +#ifdef CONFIG_KASAN_HW_TAGS | |
| 89 | +#define IF_HAVE_PG_SKIP_KASAN_POISON(flag,string) ,{1UL << flag, string} | |
| 90 | +#else | |
| 91 | +#define IF_HAVE_PG_SKIP_KASAN_POISON(flag,string) | |
| 92 | +#endif | |
| 93 | + | |
| 88 | 94 | #define __def_pageflag_names \ |
| 89 | 95 | {1UL << PG_locked, "locked" }, \ |
| 90 | 96 | {1UL << PG_waiters, "waiters" }, \ |
| @@ -112,7 +118,8 @@ IF_HAVE_PG_UNCACHED(PG_uncached, "uncached" ) \ · 21 unmodified lines | ||
| 112 | 118 | IF_HAVE_PG_HWPOISON(PG_hwpoison, "hwpoison" ) \ |
| 113 | 119 | IF_HAVE_PG_IDLE(PG_young, "young" ) \ |
| 114 | 120 | IF_HAVE_PG_IDLE(PG_idle, "idle" ) \ |
| 115 | −IF_HAVE_PG_ARCH_2(PG_arch_2, "arch_2" ) | |
| 121 | +IF_HAVE_PG_ARCH_2(PG_arch_2, "arch_2" ) \ | |
| 122 | +IF_HAVE_PG_SKIP_KASAN_POISON(PG_skip_kasan_poison, "skip_kasan_poison") | |
| 116 | 123 | |
| 117 | 124 | #define show_page_flags(flags) \ |
| 118 | 125 | (flags) ? __print_flags(flags, "|", \ |
| @@ -441,7 +441,8 @@ out_unlock: · 440 unmodified lines | ||
| 441 | 441 | return ret; |
| 442 | 442 | } |
| 443 | 443 | |
| 444 | −int __irq_set_affinity(unsigned int irq, const struct cpumask *mask, bool force) | |
| 444 | +static int __irq_set_affinity(unsigned int irq, const struct cpumask *mask, | |
| 445 | + bool force) | |
| 445 | 446 | { |
| 446 | 447 | struct irq_desc *desc = irq_to_desc(irq); |
| 447 | 448 | unsigned long flags; |
| @@ -456,6 +457,36 @@ int __irq_set_affinity(unsigned int irq, const struct cpumask *mask, bool force) · 8 unmodified lines | ||
| 456 | 457 | return ret; |
| 457 | 458 | } |
| 458 | 459 | |
| 460 | +/** | |
| 461 | + * irq_set_affinity - Set the irq affinity of a given irq | |
| 462 | + * @irq: Interrupt to set affinity | |
| 463 | + * @cpumask: cpumask | |
| 464 | + * | |
| 465 | + * Fails if cpumask does not contain an online CPU | |
| 466 | + */ | |
| 467 | +int irq_set_affinity(unsigned int irq, const struct cpumask *cpumask) | |
| 468 | +{ | |
| 469 | + return __irq_set_affinity(irq, cpumask, false); | |
| 470 | +} | |
| 471 | +EXPORT_SYMBOL_GPL(irq_set_affinity); | |
| 472 | + | |
| 473 | +/** | |
| 474 | + * irq_force_affinity - Force the irq affinity of a given irq | |
| 475 | + * @irq: Interrupt to set affinity | |
| 476 | + * @cpumask: cpumask | |
| 477 | + * | |
| 478 | + * Same as irq_set_affinity, but without checking the mask against | |
| 479 | + * online cpus. | |
| 480 | + * | |
| 481 | + * Solely for low level cpu hotplug code, where we need to make per | |
| 482 | + * cpu interrupts affine before the cpu becomes online. | |
| 483 | + */ | |
| 484 | +int irq_force_affinity(unsigned int irq, const struct cpumask *cpumask) | |
| 485 | +{ | |
| 486 | + return __irq_set_affinity(irq, cpumask, true); | |
| 487 | +} | |
| 488 | +EXPORT_SYMBOL_GPL(irq_force_affinity); | |
| 489 | + | |
| 459 | 490 | int irq_set_affinity_hint(unsigned int irq, const struct cpumask *m) |
| 460 | 491 | { |
| 461 | 492 | unsigned long flags; |
| @@ -97,7 +97,7 @@ slab_flags_t __kasan_never_merge(void) · 96 unmodified lines | ||
| 97 | 97 | return 0; |
| 98 | 98 | } |
| 99 | 99 | |
| 100 | −void __kasan_alloc_pages(struct page *page, unsigned int order, bool init) | |
| 100 | +void __kasan_unpoison_pages(struct page *page, unsigned int order, bool init) | |
| 101 | 101 | { |
| 102 | 102 | u8 tag; |
| 103 | 103 | unsigned long i; |
| @@ -111,7 +111,7 @@ void __kasan_alloc_pages(struct page *page, unsigned int order, bool init) · 7 unmodified lines | ||
| 111 | 111 | kasan_unpoison(page_address(page), PAGE_SIZE << order, init); |
| 112 | 112 | } |
| 113 | 113 | |
| 114 | −void __kasan_free_pages(struct page *page, unsigned int order, bool init) | |
| 114 | +void __kasan_poison_pages(struct page *page, unsigned int order, bool init) | |
| 115 | 115 | { |
| 116 | 116 | if (likely(!PageHighMem(page))) |
| 117 | 117 | kasan_poison(page_address(page), PAGE_SIZE << order, |
| @@ -238,6 +238,38 @@ struct kasan_track *kasan_get_free_track(struct kmem_cache *cache, · 237 unmodified lines | ||
| 238 | 238 | return &alloc_meta->free_track[0]; |
| 239 | 239 | } |
| 240 | 240 | |
| 241 | +void kasan_alloc_pages(struct page *page, unsigned int order, gfp_t flags) | |
| 242 | +{ | |
| 243 | + /* | |
| 244 | + * This condition should match the one in post_alloc_hook() in | |
| 245 | + * page_alloc.c. | |
| 246 | + */ | |
| 247 | + bool init = !want_init_on_free() && want_init_on_alloc(flags); | |
| 248 | + | |
| 249 | + if (flags & __GFP_SKIP_KASAN_POISON) | |
| 250 | + SetPageSkipKASanPoison(page); | |
| 251 | + | |
| 252 | + if (flags & __GFP_ZEROTAGS) { | |
| 253 | + int i; | |
| 254 | + | |
| 255 | + for (i = 0; i != 1 << order; ++i) | |
| 256 | + tag_clear_highpage(page + i); | |
| 257 | + } else { | |
| 258 | + kasan_unpoison_pages(page, order, init); | |
| 259 | + } | |
| 260 | +} | |
| 261 | + | |
| 262 | +void kasan_free_pages(struct page *page, unsigned int order) | |
| 263 | +{ | |
| 264 | + /* | |
| 265 | + * This condition should match the one in free_pages_prepare() in | |
| 266 | + * page_alloc.c. | |
| 267 | + */ | |
| 268 | + bool init = want_init_on_free(); | |
| 269 | + | |
| 270 | + kasan_poison_pages(page, order, init); | |
| 271 | +} | |
| 272 | + | |
| 241 | 273 | #if IS_ENABLED(CONFIG_KASAN_KUNIT_TEST) |
| 242 | 274 | |
| 243 | 275 | void kasan_set_tagging_report_once(bool state) |
| @@ -207,3 +207,10 @@ struct kasan_track *kasan_get_free_track(struct kmem_cache *cache, · 206 unmodified lines | ||
| 207 | 207 | |
| 208 | 208 | return &alloc_meta->free_track[i]; |
| 209 | 209 | } |
| 210 | + | |
| 211 | +void kasan_tag_mismatch(unsigned long addr, unsigned long access_info, | |
| 212 | + unsigned long ret_ip) | |
| 213 | +{ | |
| 214 | + kasan_report(addr, 1 << (access_info & 0xf), access_info & 0x10, | |
| 215 | + ret_ip); | |
| 216 | +} | |
| @@ -106,7 +106,8 @@ static __always_inline void kasan_poison_element(mempool_t *pool, void *element) · 105 unmodified lines | ||
| 106 | 106 | if (pool->alloc == mempool_alloc_slab || pool->alloc == mempool_kmalloc) |
| 107 | 107 | kasan_slab_free_mempool(element); |
| 108 | 108 | else if (pool->alloc == mempool_alloc_pages) |
| 109 | − kasan_free_pages(element, (unsigned long)pool->pool_data, false); | |
| 109 | + kasan_poison_pages(element, (unsigned long)pool->pool_data, | |
| 110 | + false); | |
| 110 | 111 | } |
| 111 | 112 | |
| 112 | 113 | static void kasan_unpoison_element(mempool_t *pool, void *element) |
| @@ -114,7 +115,8 @@ static void kasan_unpoison_element(mempool_t *pool, void *element) · 1 unmodified line | ||
| 114 | 115 | if (pool->alloc == mempool_alloc_slab || pool->alloc == mempool_kmalloc) |
| 115 | 116 | kasan_unpoison_range(element, __ksize(element)); |
| 116 | 117 | else if (pool->alloc == mempool_alloc_pages) |
| 117 | − kasan_alloc_pages(element, (unsigned long)pool->pool_data, false); | |
| 118 | + kasan_unpoison_pages(element, (unsigned long)pool->pool_data, | |
| 119 | + false); | |
| 118 | 120 | } |
| 119 | 121 | |
| 120 | 122 | static __always_inline void add_element(mempool_t *pool, void *element) |
| @@ -382,7 +382,7 @@ int page_group_by_mobility_disabled __read_mostly; · 381 unmodified lines | ||
| 382 | 382 | static DEFINE_STATIC_KEY_TRUE(deferred_pages); |
| 383 | 383 | |
| 384 | 384 | /* |
| 385 | − * Calling kasan_free_pages() only after deferred memory initialization | |
| 385 | + * Calling kasan_poison_pages() only after deferred memory initialization | |
| 386 | 386 | * has completed. Poisoning pages during deferred memory init will greatly |
| 387 | 387 | * lengthen the process and cause problem in large memory systems as the |
| 388 | 388 | * deferred pages initialization is done with interrupt disabled. |
| @@ -394,15 +394,12 @@ static DEFINE_STATIC_KEY_TRUE(deferred_pages); · 5 unmodified lines | ||
| 394 | 394 | * on-demand allocation and then freed again before the deferred pages |
| 395 | 395 | * initialization is done, but this is not likely to happen. |
| 396 | 396 | */ |
| 397 | −static inline void kasan_free_nondeferred_pages(struct page *page, int order, | |
| 398 | − bool init, fpi_t fpi_flags) | |
| 397 | +static inline bool should_skip_kasan_poison(struct page *page, fpi_t fpi_flags) | |
| 399 | 398 | { |
| 400 | − if (static_branch_unlikely(&deferred_pages)) | |
| 401 | − return; | |
| 402 | − if (!IS_ENABLED(CONFIG_KASAN_GENERIC) && | |
| 403 | − (fpi_flags & FPI_SKIP_KASAN_POISON)) | |
| 404 | − return; | |
| 405 | − kasan_free_pages(page, order, init); | |
| 399 | + return static_branch_unlikely(&deferred_pages) || | |
| 400 | + (!IS_ENABLED(CONFIG_KASAN_GENERIC) && | |
| 401 | + (fpi_flags & FPI_SKIP_KASAN_POISON)) || | |
| 402 | + PageSkipKASanPoison(page); | |
| 406 | 403 | } |
| 407 | 404 | |
| 408 | 405 | /* Returns true if the struct page for the pfn is uninitialised */ |
| @@ -453,13 +450,11 @@ defer_init(int nid, unsigned long pfn, unsigned long end_pfn) · 44 unmodified lines | ||
| 453 | 450 | return false; |
| 454 | 451 | } |
| 455 | 452 | #else |
| 456 | −static inline void kasan_free_nondeferred_pages(struct page *page, int order, | |
| 457 | − bool init, fpi_t fpi_flags) | |
| 453 | +static inline bool should_skip_kasan_poison(struct page *page, fpi_t fpi_flags) | |
| 458 | 454 | { |
| 459 | − if (!IS_ENABLED(CONFIG_KASAN_GENERIC) && | |
| 460 | − (fpi_flags & FPI_SKIP_KASAN_POISON)) | |
| 461 | − return; | |
| 462 | − kasan_free_pages(page, order, init); | |
| 455 | + return (!IS_ENABLED(CONFIG_KASAN_GENERIC) && | |
| 456 | + (fpi_flags & FPI_SKIP_KASAN_POISON)) || | |
| 457 | + PageSkipKASanPoison(page); | |
| 463 | 458 | } |
| 464 | 459 | |
| 465 | 460 | static inline bool early_page_uninitialised(unsigned long pfn) |
| @@ -1226,10 +1221,16 @@ out: · 760 unmodified lines | ||
| 1226 | 1221 | return ret; |
| 1227 | 1222 | } |
| 1228 | 1223 | |
| 1229 | −static void kernel_init_free_pages(struct page *page, int numpages) | |
| 1224 | +static void kernel_init_free_pages(struct page *page, int numpages, bool zero_tags) | |
| 1230 | 1225 | { |
| 1231 | 1226 | int i; |
| 1232 | 1227 | |
| 1228 | + if (zero_tags) { | |
| 1229 | + for (i = 0; i < numpages; i++) | |
| 1230 | + tag_clear_highpage(page + i); | |
| 1231 | + return; | |
| 1232 | + } | |
| 1233 | + | |
| 1233 | 1234 | /* s390's use of memset() could override KASAN redzones. */ |
| 1234 | 1235 | kasan_disable_current(); |
| 1235 | 1236 | for (i = 0; i < numpages; i++) { |
| @@ -1245,7 +1246,7 @@ static __always_inline bool free_pages_prepare(struct page *page, · 9 unmodified lines | ||
| 1245 | 1246 | unsigned int order, bool check_free, fpi_t fpi_flags) |
| 1246 | 1247 | { |
| 1247 | 1248 | int bad = 0; |
| 1248 | − bool init; | |
| 1249 | + bool skip_kasan_poison = should_skip_kasan_poison(page, fpi_flags); | |
| 1249 | 1250 | |
| 1250 | 1251 | VM_BUG_ON_PAGE(PageTail(page), page); |
| 1251 | 1252 | |
| @@ -1314,10 +1315,17 @@ static __always_inline bool free_pages_prepare(struct page *page, · 62 unmodified lines | ||
| 1314 | 1315 | * With hardware tag-based KASAN, memory tags must be set before the |
| 1315 | 1316 | * page becomes unavailable via debug_pagealloc or arch_free_page. |
| 1316 | 1317 | */ |
| 1317 | − init = want_init_on_free(); | |
| 1318 | − if (init && !kasan_has_integrated_init()) | |
| 1319 | − kernel_init_free_pages(page, 1 << order); | |
| 1320 | − kasan_free_nondeferred_pages(page, order, init, fpi_flags); | |
| 1318 | + if (kasan_has_integrated_init()) { | |
| 1319 | + if (!skip_kasan_poison) | |
| 1320 | + kasan_free_pages(page, order); | |
| 1321 | + } else { | |
| 1322 | + bool init = want_init_on_free(); | |
| 1323 | + | |
| 1324 | + if (init) | |
| 1325 | + kernel_init_free_pages(page, 1 << order, false); | |
| 1326 | + if (!skip_kasan_poison) | |
| 1327 | + kasan_poison_pages(page, order, init); | |
| 1328 | + } | |
| 1321 | 1329 | |
| 1322 | 1330 | /* |
| 1323 | 1331 | * arch_free_page() can make the page's contents inaccessible. s390 |
| @@ -2324,8 +2332,6 @@ static bool check_new_pages(struct page *page, unsigned int order) · 1000 unmodified lines | ||
| 2324 | 2332 | inline void post_alloc_hook(struct page *page, unsigned int order, |
| 2325 | 2333 | gfp_t gfp_flags) |
| 2326 | 2334 | { |
| 2327 | − bool init; | |
| 2328 | − | |
| 2329 | 2335 | set_page_private(page, 0); |
| 2330 | 2336 | set_page_refcounted(page); |
| 2331 | 2337 | |
| @@ -2344,10 +2350,16 @@ inline void post_alloc_hook(struct page *page, unsigned int order, · 12 unmodified lines | ||
| 2344 | 2350 | * kasan_alloc_pages and kernel_init_free_pages must be |
| 2345 | 2351 | * kept together to avoid discrepancies in behavior. |
| 2346 | 2352 | */ |
| 2347 | − init = !want_init_on_free() && want_init_on_alloc(gfp_flags); | |
| 2348 | − kasan_alloc_pages(page, order, init); | |
| 2349 | − if (init && !kasan_has_integrated_init()) | |
| 2350 | − kernel_init_free_pages(page, 1 << order); | |
| 2353 | + if (kasan_has_integrated_init()) { | |
| 2354 | + kasan_alloc_pages(page, order, gfp_flags); | |
| 2355 | + } else { | |
| 2356 | + bool init = !want_init_on_free() && want_init_on_alloc(gfp_flags); | |
| 2357 | + | |
| 2358 | + kasan_unpoison_pages(page, order, init); | |
| 2359 | + if (init) | |
| 2360 | + kernel_init_free_pages(page, 1 << order, | |
| 2361 | + gfp_flags & __GFP_ZEROTAGS); | |
| 2362 | + } | |
| 2351 | 2363 | |
| 2352 | 2364 | set_page_owner(page, order, gfp_flags); |
| 2353 | 2365 | } |
| @@ -50,6 +50,7 @@ endif · 49 unmodified lines | ||
| 50 | 50 | CFLAGS_KASAN := -fsanitize=kernel-hwaddress \ |
| 51 | 51 | $(call cc-param,hwasan-instrument-stack=$(stack_enable)) \ |
| 52 | 52 | $(call cc-param,hwasan-use-short-granules=0) \ |
| 53 | + $(call cc-param,hwasan-inline-all-checks=0) \ | |
| 53 | 54 | $(instrumentation_flags) |
| 54 | 55 | |
| 55 | 56 | endif # CONFIG_KASAN_SW_TAGS |
| @@ -7,7 +7,8 @@ trap "rm -f $tmp_file.o $tmp_file $tmp_file.bin" EXIT · 6 unmodified lines | ||
| 7 | 7 | cat << "END" | $CC -c -x c - -o $tmp_file.o >/dev/null 2>&1 |
| 8 | 8 | void *p = &p; |
| 9 | 9 | END |
| 10 | −$LD $tmp_file.o -shared -Bsymbolic --pack-dyn-relocs=relr -o $tmp_file | |
| 10 | +$LD $tmp_file.o -shared -Bsymbolic --pack-dyn-relocs=relr \ | |
| 11 | + --use-android-relr-tags -o $tmp_file | |
| 11 | 12 | |
| 12 | 13 | # Despite printing an error message, GNU nm still exits with exit code 0 if it |
| 13 | 14 | # sees a relr section. So we need to check that nothing is printed to stderr. |
| @@ -25,7 +25,7 @@ int main(int argc, char **argv) · 24 unmodified lines | ||
| 25 | 25 | ksft_set_plan(2); |
| 26 | 26 | |
| 27 | 27 | if (!(getauxval(AT_HWCAP) & HWCAP_SVE)) |
| 28 | − ksft_exit_skip("SVE not available"); | |
| 28 | + ksft_exit_skip("SVE not available\n"); | |
| 29 | 29 | |
| 30 | 30 | /* |
| 31 | 31 | * Enumerate up to SVE_VQ_MAX vector lengths |