Pull arm64 updates from Catalin Marinas: - VMAP_STACK support, allowing the kernel stacks to be allocated in the vmalloc space with a guard page for trapping stack overflows. One of the patches introduces THREAD_ALIGN and changes the generic alloc_thread_stack_node() to use this instead of THREAD_SIZE (no functional change for other architectures) - Contiguous PTE hugetlb support re-enabled (after being reverted a couple of times). We now have the semantics agreed in the generic mm layer together with API improvements so that the architecture code can detect between contiguous and non-contiguous huge PTEs - Initial support for persistent memory on ARM: DC CVAP instruction exposed to user space (HWCAP) and the in-kernel pmem API implemented - raid6 improvements for arm64: faster algorithm for the delta syndrome and implementation of the recovery routines using Neon - FP/SIMD refactoring and removal of support for Neon in interrupt context. This is in preparation for full SVE support - PTE accessors converted from inline asm to cmpxchg so that we can use LSE atomics if available (ARMv8.1) - Perf support for Cortex-A35 and A73 - Non-urgent fixes and cleanups * tag 'arm64-upstream' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux: (75 commits) arm64: cleanup {COMPAT_,}SET_PERSONALITY() macro arm64: introduce separated bits for mm_context_t flags arm64: hugetlb: Cleanup setup_hugepagesz arm64: Re-enable support for contiguous hugepages arm64: hugetlb: Override set_huge_swap_pte_at() to support contiguous hugepages arm64: hugetlb: Override huge_pte_clear() to support contiguous hugepages arm64: hugetlb: Handle swap entries in huge_pte_offset() for contiguous hugepages arm64: hugetlb: Add break-before-make logic for contiguous entries arm64: hugetlb: Spring clean huge pte accessors arm64: hugetlb: Introduce pte_pgprot helper arm64: hugetlb: set_huge_pte_at Add WARN_ON on !pte_present arm64: kexec: have own crash_smp_send_stop() for crash dump for nonpanic cores arm64: dma-mapping: Mark atomic_pool as __ro_after_init arm64: dma-mapping: Do not pass data to gen_pool_set_algo() arm64: Remove the !CONFIG_ARM64_HW_AFDBM alternative code paths arm64: Ignore hardware dirty bit updates in ptep_set_wrprotect() arm64: Move PTE_RDONLY bit handling out of set_pte_at() kvm: arm64: Convert kvm_set_s2pte_readonly() from inline asm to cmpxchg() arm64: Convert pte handling from inline asm to using (cmp)xchg arm64: neon/efi: Make EFI fpsimd save/restore variables static ...
Unified · Split
| @@ -179,6 +179,8 @@ infrastructure: · 178 unmodified lines | |||
| 179 | | FCMA | [19-16] | y | | 179 | | FCMA | [19-16] | y | |
| 180 | |--------------------------------------------------| | 180 | |--------------------------------------------------| |
| 181 | | JSCVT | [15-12] | y | | 181 | | JSCVT | [15-12] | y | |
| 182 | |--------------------------------------------------| | ||
| 183 | | DPB | [3-0] | y | | ||
| 182 | x--------------------------------------------------x | 184 | x--------------------------------------------------x |
| 183 | 185 | ||
| 184 | Appendix I: Example | 186 | Appendix I: Example |
| @@ -9,9 +9,11 @@ Required properties: · 8 unmodified lines | |||
| 9 | - compatible : should be one of | 9 | - compatible : should be one of |
| 10 | "apm,potenza-pmu" | 10 | "apm,potenza-pmu" |
| 11 | "arm,armv8-pmuv3" | 11 | "arm,armv8-pmuv3" |
| 12 | "arm,cortex-a73-pmu" | ||
| 12 | "arm,cortex-a72-pmu" | 13 | "arm,cortex-a72-pmu" |
| 13 | "arm,cortex-a57-pmu" | 14 | "arm,cortex-a57-pmu" |
| 14 | "arm,cortex-a53-pmu" | 15 | "arm,cortex-a53-pmu" |
| 16 | "arm,cortex-a35-pmu" | ||
| 15 | "arm,cortex-a17-pmu" | 17 | "arm,cortex-a17-pmu" |
| 16 | "arm,cortex-a15-pmu" | 18 | "arm,cortex-a15-pmu" |
| 17 | "arm,cortex-a12-pmu" | 19 | "arm,cortex-a12-pmu" |
| @@ -75,6 +75,7 @@ config ARM64 · 74 unmodified lines | |||
| 75 | select HAVE_ARCH_SECCOMP_FILTER | 75 | select HAVE_ARCH_SECCOMP_FILTER |
| 76 | select HAVE_ARCH_TRACEHOOK | 76 | select HAVE_ARCH_TRACEHOOK |
| 77 | select HAVE_ARCH_TRANSPARENT_HUGEPAGE | 77 | select HAVE_ARCH_TRANSPARENT_HUGEPAGE |
| 78 | select HAVE_ARCH_VMAP_STACK | ||
| 78 | select HAVE_ARM_SMCCC | 79 | select HAVE_ARM_SMCCC |
| 79 | select HAVE_EBPF_JIT | 80 | select HAVE_EBPF_JIT |
| 80 | select HAVE_C_RECORDMCOUNT | 81 | select HAVE_C_RECORDMCOUNT |
| @@ -960,6 +961,18 @@ config ARM64_UAO · 879 unmodified lines | |||
| 960 | regular load/store instructions if the cpu does not implement the | 961 | regular load/store instructions if the cpu does not implement the |
| 961 | feature. | 962 | feature. |
| 962 | 963 | ||
| 964 | config ARM64_PMEM | ||
| 965 | bool "Enable support for persistent memory" | ||
| 966 | select ARCH_HAS_PMEM_API | ||
| 967 | select ARCH_HAS_UACCESS_FLUSHCACHE | ||
| 968 | help | ||
| 969 | Say Y to enable support for the persistent memory API based on the | ||
| 970 | ARMv8.2 DCPoP feature. | ||
| 971 | |||
| 972 | The feature is detected at runtime, and the kernel will use DC CVAC | ||
| 973 | operations if DC CVAP is not supported (following the behaviour of | ||
| 974 | DC CVAP itself if the system does not define a point of persistence). | ||
| 975 | |||
| 963 | endmenu | 976 | endmenu |
| 964 | 977 | ||
| 965 | config ARM64_MODULE_CMODEL_LARGE | 978 | config ARM64_MODULE_CMODEL_LARGE |
| @@ -20,7 +20,6 @@ generic-y += rwsem.h · 19 unmodified lines | |||
| 20 | generic-y += segment.h | 20 | generic-y += segment.h |
| 21 | generic-y += serial.h | 21 | generic-y += serial.h |
| 22 | generic-y += set_memory.h | 22 | generic-y += set_memory.h |
| 23 | generic-y += simd.h | ||
| 24 | generic-y += sizes.h | 23 | generic-y += sizes.h |
| 25 | generic-y += switch_to.h | 24 | generic-y += switch_to.h |
| 26 | generic-y += trace_clock.h | 25 | generic-y += trace_clock.h |
| @@ -0,0 +1,54 @@ | |||
| 1 | #ifndef __ASM_ASM_BUG_H | ||
| 2 | /* | ||
| 3 | * Copyright (C) 2017 ARM Limited | ||
| 4 | * | ||
| 5 | * This program is free software; you can redistribute it and/or modify | ||
| 6 | * it under the terms of the GNU General Public License version 2 as | ||
| 7 | * published by the Free Software Foundation. | ||
| 8 | * | ||
| 9 | * This program is distributed in the hope that it will be useful, | ||
| 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
| 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
| 12 | * GNU General Public License for more details. | ||
| 13 | * | ||
| 14 | * You should have received a copy of the GNU General Public License | ||
| 15 | * along with this program. If not, see <http://www.gnu.org/licenses/>. | ||
| 16 | */ | ||
| 17 | #define __ASM_ASM_BUG_H | ||
| 18 | |||
| 19 | #include <asm/brk-imm.h> | ||
| 20 | |||
| 21 | #ifdef CONFIG_DEBUG_BUGVERBOSE | ||
| 22 | #define _BUGVERBOSE_LOCATION(file, line) __BUGVERBOSE_LOCATION(file, line) | ||
| 23 | #define __BUGVERBOSE_LOCATION(file, line) \ | ||
| 24 | .pushsection .rodata.str,"aMS",@progbits,1; \ | ||
| 25 | 2: .string file; \ | ||
| 26 | .popsection; \ | ||
| 27 | \ | ||
| 28 | .long 2b - 0b; \ | ||
| 29 | .short line; | ||
| 30 | #else | ||
| 31 | #define _BUGVERBOSE_LOCATION(file, line) | ||
| 32 | #endif | ||
| 33 | |||
| 34 | #ifdef CONFIG_GENERIC_BUG | ||
| 35 | |||
| 36 | #define __BUG_ENTRY(flags) \ | ||
| 37 | .pushsection __bug_table,"aw"; \ | ||
| 38 | .align 2; \ | ||
| 39 | 0: .long 1f - 0b; \ | ||
| 40 | _BUGVERBOSE_LOCATION(__FILE__, __LINE__) \ | ||
| 41 | .short flags; \ | ||
| 42 | .popsection; \ | ||
| 43 | 1: | ||
| 44 | #else | ||
| 45 | #define __BUG_ENTRY(flags) | ||
| 46 | #endif | ||
| 47 | |||
| 48 | #define ASM_BUG_FLAGS(flags) \ | ||
| 49 | __BUG_ENTRY(flags) \ | ||
| 50 | brk BUG_BRK_IMM | ||
| 51 | |||
| 52 | #define ASM_BUG() ASM_BUG_FLAGS(0) | ||
| 53 | |||
| 54 | #endif /* __ASM_ASM_BUG_H */ | ||
| @@ -230,12 +230,18 @@ lr .req x30 // link register · 229 unmodified lines | |||
| 230 | .endm | 230 | .endm |
| 231 | 231 | ||
| 232 | /* | 232 | /* |
| 233 | * @dst: Result of per_cpu(sym, smp_processor_id()) | 233 | * @dst: Result of per_cpu(sym, smp_processor_id()), can be SP for |
| 234 | * non-module code | ||
| 234 | * @sym: The name of the per-cpu variable | 235 | * @sym: The name of the per-cpu variable |
| 235 | * @tmp: scratch register | 236 | * @tmp: scratch register |
| 236 | */ | 237 | */ |
| 237 | .macro adr_this_cpu, dst, sym, tmp | 238 | .macro adr_this_cpu, dst, sym, tmp |
| 239 | #ifndef MODULE | ||
| 240 | adrp \tmp, \sym | ||
| 241 | add \dst, \tmp, #:lo12:\sym | ||
| 242 | #else | ||
| 238 | adr_l \dst, \sym | 243 | adr_l \dst, \sym |
| 244 | #endif | ||
| 239 | mrs \tmp, tpidr_el1 | 245 | mrs \tmp, tpidr_el1 |
| 240 | add \dst, \dst, \tmp | 246 | add \dst, \dst, \tmp |
| 241 | .endm | 247 | .endm |
| @@ -352,6 +358,12 @@ alternative_if_not ARM64_WORKAROUND_CLEAN_CACHE · 110 unmodified lines | |||
| 352 | dc \op, \kaddr | 358 | dc \op, \kaddr |
| 353 | alternative_else | 359 | alternative_else |
| 354 | dc civac, \kaddr | 360 | dc civac, \kaddr |
| 361 | alternative_endif | ||
| 362 | .elseif (\op == cvap) | ||
| 363 | alternative_if ARM64_HAS_DCPOP | ||
| 364 | sys 3, c7, c12, 1, \kaddr // dc cvap | ||
| 365 | alternative_else | ||
| 366 | dc cvac, \kaddr | ||
| 355 | alternative_endif | 367 | alternative_endif |
| 356 | .else | 368 | .else |
| 357 | dc \op, \kaddr | 369 | dc \op, \kaddr |
| @@ -403,6 +415,17 @@ alternative_endif · 45 unmodified lines | |||
| 403 | .size __pi_##x, . - x; \ | 415 | .size __pi_##x, . - x; \ |
| 404 | ENDPROC(x) | 416 | ENDPROC(x) |
| 405 | 417 | ||
| 418 | /* | ||
| 419 | * Annotate a function as being unsuitable for kprobes. | ||
| 420 | */ | ||
| 421 | #ifdef CONFIG_KPROBES | ||
| 422 | #define NOKPROBE(x) \ | ||
| 423 | .pushsection "_kprobe_blacklist", "aw"; \ | ||
| 424 | .quad x; \ | ||
| 425 | .popsection; | ||
| 426 | #else | ||
| 427 | #define NOKPROBE(x) | ||
| 428 | #endif | ||
| 406 | /* | 429 | /* |
| 407 | * Emit a 64-bit absolute little endian symbol reference in a way that | 430 | * Emit a 64-bit absolute little endian symbol reference in a way that |
| 408 | * ensures that it will be resolved at build time, even when building a | 431 | * ensures that it will be resolved at build time, even when building a |
| @@ -18,41 +18,12 @@ · 17 unmodified lines | |||
| 18 | #ifndef _ARCH_ARM64_ASM_BUG_H | 18 | #ifndef _ARCH_ARM64_ASM_BUG_H |
| 19 | #define _ARCH_ARM64_ASM_BUG_H | 19 | #define _ARCH_ARM64_ASM_BUG_H |
| 20 | 20 | ||
| 21 | #include <asm/brk-imm.h> | 21 | #include <linux/stringify.h> |
| 22 | 22 | ||
| 23 | #ifdef CONFIG_DEBUG_BUGVERBOSE | 23 | #include <asm/asm-bug.h> |
| 24 | #define _BUGVERBOSE_LOCATION(file, line) __BUGVERBOSE_LOCATION(file, line) | ||
| 25 | #define __BUGVERBOSE_LOCATION(file, line) \ | ||
| 26 | ".pushsection .rodata.str,\"aMS\",@progbits,1\n" \ | ||
| 27 | "2: .string \"" file "\"\n\t" \ | ||
| 28 | ".popsection\n\t" \ | ||
| 29 | \ | ||
| 30 | ".long 2b - 0b\n\t" \ | ||
| 31 | ".short " #line "\n\t" | ||
| 32 | #else | ||
| 33 | #define _BUGVERBOSE_LOCATION(file, line) | ||
| 34 | #endif | ||
| 35 | |||
| 36 | #ifdef CONFIG_GENERIC_BUG | ||
| 37 | |||
| 38 | #define __BUG_ENTRY(flags) \ | ||
| 39 | ".pushsection __bug_table,\"aw\"\n\t" \ | ||
| 40 | ".align 2\n\t" \ | ||
| 41 | "0: .long 1f - 0b\n\t" \ | ||
| 42 | _BUGVERBOSE_LOCATION(__FILE__, __LINE__) \ | ||
| 43 | ".short " #flags "\n\t" \ | ||
| 44 | ".popsection\n" \ | ||
| 45 | "1: " | ||
| 46 | #else | ||
| 47 | #define __BUG_ENTRY(flags) "" | ||
| 48 | #endif | ||
| 49 | 24 | ||
| 50 | #define __BUG_FLAGS(flags) \ | 25 | #define __BUG_FLAGS(flags) \ |
| 51 | asm volatile ( \ | 26 | asm volatile (__stringify(ASM_BUG_FLAGS(flags))); |
| 52 | __BUG_ENTRY(flags) \ | ||
| 53 | "brk %[imm]" :: [imm] "i" (BUG_BRK_IMM) \ | ||
| 54 | ); | ||
| 55 | |||
| 56 | 27 | ||
| 57 | #define BUG() do { \ | 28 | #define BUG() do { \ |
| 58 | __BUG_FLAGS(0); \ | 29 | __BUG_FLAGS(0); \ |
| @@ -67,7 +67,9 @@ · 66 unmodified lines | |||
| 67 | */ | 67 | */ |
| 68 | extern void flush_icache_range(unsigned long start, unsigned long end); | 68 | extern void flush_icache_range(unsigned long start, unsigned long end); |
| 69 | extern void __flush_dcache_area(void *addr, size_t len); | 69 | extern void __flush_dcache_area(void *addr, size_t len); |
| 70 | extern void __inval_dcache_area(void *addr, size_t len); | ||
| 70 | extern void __clean_dcache_area_poc(void *addr, size_t len); | 71 | extern void __clean_dcache_area_poc(void *addr, size_t len); |
| 72 | extern void __clean_dcache_area_pop(void *addr, size_t len); | ||
| 71 | extern void __clean_dcache_area_pou(void *addr, size_t len); | 73 | extern void __clean_dcache_area_pou(void *addr, size_t len); |
| 72 | extern long __flush_cache_user_range(unsigned long start, unsigned long end); | 74 | extern long __flush_cache_user_range(unsigned long start, unsigned long end); |
| 73 | extern void sync_icache_aliases(void *kaddr, unsigned long len); | 75 | extern void sync_icache_aliases(void *kaddr, unsigned long len); |
| @@ -150,6 +152,6 @@ static inline void flush_cache_vunmap(unsigned long start, unsigned long end) · 76 unmodified lines | |||
| 150 | { | 152 | { |
| 151 | } | 153 | } |
| 152 | 154 | ||
| 153 | int set_memory_valid(unsigned long addr, unsigned long size, int enable); | 155 | int set_memory_valid(unsigned long addr, int numpages, int enable); |
| 154 | 156 | ||
| 155 | #endif | 157 | #endif |
| @@ -39,7 +39,8 @@ · 38 unmodified lines | |||
| 39 | #define ARM64_WORKAROUND_QCOM_FALKOR_E1003 18 | 39 | #define ARM64_WORKAROUND_QCOM_FALKOR_E1003 18 |
| 40 | #define ARM64_WORKAROUND_858921 19 | 40 | #define ARM64_WORKAROUND_858921 19 |
| 41 | #define ARM64_WORKAROUND_CAVIUM_30115 20 | 41 | #define ARM64_WORKAROUND_CAVIUM_30115 20 |
| 42 | #define ARM64_HAS_DCPOP 21 | ||
| 42 | 43 | ||
| 43 | #define ARM64_NCAPS 21 | 44 | #define ARM64_NCAPS 22 |
| 44 | 45 | ||
| 45 | #endif /* __ASM_CPUCAPS_H */ | 46 | #endif /* __ASM_CPUCAPS_H */ |
| @@ -3,7 +3,9 @@ · 2 unmodified lines | |||
| 3 | 3 | ||
| 4 | #include <asm/boot.h> | 4 | #include <asm/boot.h> |
| 5 | #include <asm/cpufeature.h> | 5 | #include <asm/cpufeature.h> |
| 6 | #include <asm/fpsimd.h> | ||
| 6 | #include <asm/io.h> | 7 | #include <asm/io.h> |
| 8 | #include <asm/memory.h> | ||
| 7 | #include <asm/mmu_context.h> | 9 | #include <asm/mmu_context.h> |
| 8 | #include <asm/neon.h> | 10 | #include <asm/neon.h> |
| 9 | #include <asm/ptrace.h> | 11 | #include <asm/ptrace.h> |
| @@ -20,8 +22,8 @@ int efi_set_mapping_permissions(struct mm_struct *mm, efi_memory_desc_t *md); · 10 unmodified lines | |||
| 20 | 22 | ||
| 21 | #define arch_efi_call_virt_setup() \ | 23 | #define arch_efi_call_virt_setup() \ |
| 22 | ({ \ | 24 | ({ \ |
| 23 | kernel_neon_begin(); \ | ||
| 24 | efi_virtmap_load(); \ | 25 | efi_virtmap_load(); \ |
| 26 | __efi_fpsimd_begin(); \ | ||
| 25 | }) | 27 | }) |
| 26 | 28 | ||
| 27 | #define arch_efi_call_virt(p, f, args...) \ | 29 | #define arch_efi_call_virt(p, f, args...) \ |
| @@ -33,8 +35,8 @@ int efi_set_mapping_permissions(struct mm_struct *mm, efi_memory_desc_t *md); · 5 unmodified lines | |||
| 33 | 35 | ||
| 34 | #define arch_efi_call_virt_teardown() \ | 36 | #define arch_efi_call_virt_teardown() \ |
| 35 | ({ \ | 37 | ({ \ |
| 38 | __efi_fpsimd_end(); \ | ||
| 36 | efi_virtmap_unload(); \ | 39 | efi_virtmap_unload(); \ |
| 37 | kernel_neon_end(); \ | ||
| 38 | }) | 40 | }) |
| 39 | 41 | ||
| 40 | #define ARCH_EFI_IRQ_FLAGS_MASK (PSR_D_BIT | PSR_A_BIT | PSR_I_BIT | PSR_F_BIT) | 42 | #define ARCH_EFI_IRQ_FLAGS_MASK (PSR_D_BIT | PSR_A_BIT | PSR_I_BIT | PSR_F_BIT) |
| @@ -48,6 +50,13 @@ int efi_set_mapping_permissions(struct mm_struct *mm, efi_memory_desc_t *md); · 7 unmodified lines | |||
| 48 | */ | 50 | */ |
| 49 | #define EFI_FDT_ALIGN SZ_2M /* used by allocate_new_fdt_and_exit_boot() */ | 51 | #define EFI_FDT_ALIGN SZ_2M /* used by allocate_new_fdt_and_exit_boot() */ |
| 50 | 52 | ||
| 53 | /* | ||
| 54 | * In some configurations (e.g. VMAP_STACK && 64K pages), stacks built into the | ||
| 55 | * kernel need greater alignment than we require the segments to be padded to. | ||
| 56 | */ | ||
| 57 | #define EFI_KIMG_ALIGN \ | ||
| 58 | (SEGMENT_ALIGN > THREAD_ALIGN ? SEGMENT_ALIGN : THREAD_ALIGN) | ||
| 59 | |||
| 51 | /* on arm64, the FDT may be located anywhere in system RAM */ | 60 | /* on arm64, the FDT may be located anywhere in system RAM */ |
| 52 | static inline unsigned long efi_get_max_fdt_addr(unsigned long dram_base) | 61 | static inline unsigned long efi_get_max_fdt_addr(unsigned long dram_base) |
| 53 | { | 62 | { |
| @@ -139,7 +139,6 @@ typedef struct user_fpsimd_state elf_fpregset_t; · 138 unmodified lines | |||
| 139 | 139 | ||
| 140 | #define SET_PERSONALITY(ex) \ | 140 | #define SET_PERSONALITY(ex) \ |
| 141 | ({ \ | 141 | ({ \ |
| 142 | clear_bit(TIF_32BIT, ¤t->mm->context.flags); \ | ||
| 143 | clear_thread_flag(TIF_32BIT); \ | 142 | clear_thread_flag(TIF_32BIT); \ |
| 144 | current->personality &= ~READ_IMPLIES_EXEC; \ | 143 | current->personality &= ~READ_IMPLIES_EXEC; \ |
| 145 | }) | 144 | }) |
| @@ -195,7 +194,6 @@ typedef compat_elf_greg_t compat_elf_gregset_t[COMPAT_ELF_NGREG]; · 49 unmodified lines | |||
| 195 | */ | 194 | */ |
| 196 | #define COMPAT_SET_PERSONALITY(ex) \ | 195 | #define COMPAT_SET_PERSONALITY(ex) \ |
| 197 | ({ \ | 196 | ({ \ |
| 198 | set_bit(TIF_32BIT, ¤t->mm->context.flags); \ | ||
| 199 | set_thread_flag(TIF_32BIT); \ | 197 | set_thread_flag(TIF_32BIT); \ |
| 200 | }) | 198 | }) |
| 201 | #define COMPAT_ARCH_DLINFO | 199 | #define COMPAT_ARCH_DLINFO |
| @@ -77,16 +77,23 @@ · 76 unmodified lines | |||
| 77 | #define ESR_ELx_EC_MASK (UL(0x3F) << ESR_ELx_EC_SHIFT) | 77 | #define ESR_ELx_EC_MASK (UL(0x3F) << ESR_ELx_EC_SHIFT) |
| 78 | #define ESR_ELx_EC(esr) (((esr) & ESR_ELx_EC_MASK) >> ESR_ELx_EC_SHIFT) | 78 | #define ESR_ELx_EC(esr) (((esr) & ESR_ELx_EC_MASK) >> ESR_ELx_EC_SHIFT) |
| 79 | 79 | ||
| 80 | #define ESR_ELx_IL (UL(1) << 25) | 80 | #define ESR_ELx_IL_SHIFT (25) |
| 81 | #define ESR_ELx_IL (UL(1) << ESR_ELx_IL_SHIFT) | ||
| 81 | #define ESR_ELx_ISS_MASK (ESR_ELx_IL - 1) | 82 | #define ESR_ELx_ISS_MASK (ESR_ELx_IL - 1) |
| 82 | 83 | ||
| 83 | /* ISS field definitions shared by different classes */ | 84 | /* ISS field definitions shared by different classes */ |
| 84 | #define ESR_ELx_WNR (UL(1) << 6) | 85 | #define ESR_ELx_WNR_SHIFT (6) |
| 86 | #define ESR_ELx_WNR (UL(1) << ESR_ELx_WNR_SHIFT) | ||
| 85 | 87 | ||
| 86 | /* Shared ISS field definitions for Data/Instruction aborts */ | 88 | /* Shared ISS field definitions for Data/Instruction aborts */ |
| 87 | #define ESR_ELx_FnV (UL(1) << 10) | 89 | #define ESR_ELx_SET_SHIFT (11) |
| 88 | #define ESR_ELx_EA (UL(1) << 9) | 90 | #define ESR_ELx_SET_MASK (UL(3) << ESR_ELx_SET_SHIFT) |
| 89 | #define ESR_ELx_S1PTW (UL(1) << 7) | 91 | #define ESR_ELx_FnV_SHIFT (10) |
| 92 | #define ESR_ELx_FnV (UL(1) << ESR_ELx_FnV_SHIFT) | ||
| 93 | #define ESR_ELx_EA_SHIFT (9) | ||
| 94 | #define ESR_ELx_EA (UL(1) << ESR_ELx_EA_SHIFT) | ||
| 95 | #define ESR_ELx_S1PTW_SHIFT (7) | ||
| 96 | #define ESR_ELx_S1PTW (UL(1) << ESR_ELx_S1PTW_SHIFT) | ||
| 90 | 97 | ||
| 91 | /* Shared ISS fault status code(IFSC/DFSC) for Data/Instruction aborts */ | 98 | /* Shared ISS fault status code(IFSC/DFSC) for Data/Instruction aborts */ |
| 92 | #define ESR_ELx_FSC (0x3F) | 99 | #define ESR_ELx_FSC (0x3F) |
| @@ -97,15 +104,20 @@ · 4 unmodified lines | |||
| 97 | #define ESR_ELx_FSC_PERM (0x0C) | 104 | #define ESR_ELx_FSC_PERM (0x0C) |
| 98 | 105 | ||
| 99 | /* ISS field definitions for Data Aborts */ | 106 | /* ISS field definitions for Data Aborts */ |
| 100 | #define ESR_ELx_ISV (UL(1) << 24) | 107 | #define ESR_ELx_ISV_SHIFT (24) |
| 108 | #define ESR_ELx_ISV (UL(1) << ESR_ELx_ISV_SHIFT) | ||
| 101 | #define ESR_ELx_SAS_SHIFT (22) | 109 | #define ESR_ELx_SAS_SHIFT (22) |
| 102 | #define ESR_ELx_SAS (UL(3) << ESR_ELx_SAS_SHIFT) | 110 | #define ESR_ELx_SAS (UL(3) << ESR_ELx_SAS_SHIFT) |
| 103 | #define ESR_ELx_SSE (UL(1) << 21) | 111 | #define ESR_ELx_SSE_SHIFT (21) |
| 112 | #define ESR_ELx_SSE (UL(1) << ESR_ELx_SSE_SHIFT) | ||
| 104 | #define ESR_ELx_SRT_SHIFT (16) | 113 | #define ESR_ELx_SRT_SHIFT (16) |
| 105 | #define ESR_ELx_SRT_MASK (UL(0x1F) << ESR_ELx_SRT_SHIFT) | 114 | #define ESR_ELx_SRT_MASK (UL(0x1F) << ESR_ELx_SRT_SHIFT) |
| 106 | #define ESR_ELx_SF (UL(1) << 15) | 115 | #define ESR_ELx_SF_SHIFT (15) |
| 107 | #define ESR_ELx_AR (UL(1) << 14) | 116 | #define ESR_ELx_SF (UL(1) << ESR_ELx_SF_SHIFT) |
| 108 | #define ESR_ELx_CM (UL(1) << 8) | 117 | #define ESR_ELx_AR_SHIFT (14) |
| 118 | #define ESR_ELx_AR (UL(1) << ESR_ELx_AR_SHIFT) | ||
| 119 | #define ESR_ELx_CM_SHIFT (8) | ||
| 120 | #define ESR_ELx_CM (UL(1) << ESR_ELx_CM_SHIFT) | ||
| 109 | 121 | ||
| 110 | /* ISS field definitions for exceptions taken in to Hyp */ | 122 | /* ISS field definitions for exceptions taken in to Hyp */ |
| 111 | #define ESR_ELx_CV (UL(1) << 24) | 123 | #define ESR_ELx_CV (UL(1) << 24) |
| @@ -157,9 +169,10 @@ · 45 unmodified lines | |||
| 157 | /* | 169 | /* |
| 158 | * User space cache operations have the following sysreg encoding | 170 | * User space cache operations have the following sysreg encoding |
| 159 | * in System instructions. | 171 | * in System instructions. |
| 160 | * op0=1, op1=3, op2=1, crn=7, crm={ 5, 10, 11, 14 }, WRITE (L=0) | 172 | * op0=1, op1=3, op2=1, crn=7, crm={ 5, 10, 11, 12, 14 }, WRITE (L=0) |
| 161 | */ | 173 | */ |
| 162 | #define ESR_ELx_SYS64_ISS_CRM_DC_CIVAC 14 | 174 | #define ESR_ELx_SYS64_ISS_CRM_DC_CIVAC 14 |
| 175 | #define ESR_ELx_SYS64_ISS_CRM_DC_CVAP 12 | ||
| 163 | #define ESR_ELx_SYS64_ISS_CRM_DC_CVAU 11 | 176 | #define ESR_ELx_SYS64_ISS_CRM_DC_CVAU 11 |
| 164 | #define ESR_ELx_SYS64_ISS_CRM_DC_CVAC 10 | 177 | #define ESR_ELx_SYS64_ISS_CRM_DC_CVAC 10 |
| 165 | #define ESR_ELx_SYS64_ISS_CRM_IC_IVAU 5 | 178 | #define ESR_ELx_SYS64_ISS_CRM_IC_IVAU 5 |
| @@ -209,6 +222,13 @@ · 43 unmodified lines | |||
| 209 | #ifndef __ASSEMBLY__ | 222 | #ifndef __ASSEMBLY__ |
| 210 | #include <asm/types.h> | 223 | #include <asm/types.h> |
| 211 | 224 | ||
| 225 | static inline bool esr_is_data_abort(u32 esr) | ||
| 226 | { | ||
| 227 | const u32 ec = ESR_ELx_EC(esr); | ||
| 228 | |||
| 229 | return ec == ESR_ELx_EC_DABT_LOW || ec == ESR_ELx_EC_DABT_CUR; | ||
| 230 | } | ||
| 231 | |||
| 212 | const char *esr_get_class_string(u32 esr); | 232 | const char *esr_get_class_string(u32 esr); |
| 213 | #endif /* __ASSEMBLY */ | 233 | #endif /* __ASSEMBLY */ |
| 214 | 234 | ||
| @@ -41,16 +41,6 @@ struct fpsimd_state { · 40 unmodified lines | |||
| 41 | unsigned int cpu; | 41 | unsigned int cpu; |
| 42 | }; | 42 | }; |
| 43 | 43 | ||
| 44 | /* | ||
| 45 | * Struct for stacking the bottom 'n' FP/SIMD registers. | ||
| 46 | */ | ||
| 47 | struct fpsimd_partial_state { | ||
| 48 | u32 fpsr; | ||
| 49 | u32 fpcr; | ||
| 50 | u32 num_regs; | ||
| 51 | __uint128_t vregs[32]; | ||
| 52 | }; | ||
| 53 | |||
| 54 | 44 | ||
| 55 | #if defined(__KERNEL__) && defined(CONFIG_COMPAT) | 45 | #if defined(__KERNEL__) && defined(CONFIG_COMPAT) |
| 56 | /* Masks for extracting the FPSR and FPCR from the FPSCR */ | 46 | /* Masks for extracting the FPSR and FPCR from the FPSCR */ |
| @@ -77,9 +67,9 @@ extern void fpsimd_update_current_state(struct fpsimd_state *state); · 20 unmodified lines | |||
| 77 | 67 | ||
| 78 | extern void fpsimd_flush_task_state(struct task_struct *target); | 68 | extern void fpsimd_flush_task_state(struct task_struct *target); |
| 79 | 69 | ||
| 80 | extern void fpsimd_save_partial_state(struct fpsimd_partial_state *state, | 70 | /* For use by EFI runtime services calls only */ |
| 81 | u32 num_regs); | 71 | extern void __efi_fpsimd_begin(void); |
| 82 | extern void fpsimd_load_partial_state(struct fpsimd_partial_state *state); | 72 | extern void __efi_fpsimd_end(void); |
| 83 | 73 | ||
| 84 | #endif | 74 | #endif |
| 85 | 75 | ||
| @@ -75,59 +75,3 @@ · 74 unmodified lines | |||
| 75 | ldr w\tmpnr, [\state, #16 * 2 + 4] | 75 | ldr w\tmpnr, [\state, #16 * 2 + 4] |
| 76 | fpsimd_restore_fpcr x\tmpnr, \state | 76 | fpsimd_restore_fpcr x\tmpnr, \state |
| 77 | .endm | 77 | .endm |
| 78 | |||
| 79 | .macro fpsimd_save_partial state, numnr, tmpnr1, tmpnr2 | ||
| 80 | mrs x\tmpnr1, fpsr | ||
| 81 | str w\numnr, [\state, #8] | ||
| 82 | mrs x\tmpnr2, fpcr | ||
| 83 | stp w\tmpnr1, w\tmpnr2, [\state] | ||
| 84 | adr x\tmpnr1, 0f | ||
| 85 | add \state, \state, x\numnr, lsl #4 | ||
| 86 | sub x\tmpnr1, x\tmpnr1, x\numnr, lsl #1 | ||
| 87 | br x\tmpnr1 | ||
| 88 | stp q30, q31, [\state, #-16 * 30 - 16] | ||
| 89 | stp q28, q29, [\state, #-16 * 28 - 16] | ||
| 90 | stp q26, q27, [\state, #-16 * 26 - 16] | ||
| 91 | stp q24, q25, [\state, #-16 * 24 - 16] | ||
| 92 | stp q22, q23, [\state, #-16 * 22 - 16] | ||
| 93 | stp q20, q21, [\state, #-16 * 20 - 16] | ||
| 94 | stp q18, q19, [\state, #-16 * 18 - 16] | ||
| 95 | stp q16, q17, [\state, #-16 * 16 - 16] | ||
| 96 | stp q14, q15, [\state, #-16 * 14 - 16] | ||
| 97 | stp q12, q13, [\state, #-16 * 12 - 16] | ||
| 98 | stp q10, q11, [\state, #-16 * 10 - 16] | ||
| 99 | stp q8, q9, [\state, #-16 * 8 - 16] | ||
| 100 | stp q6, q7, [\state, #-16 * 6 - 16] | ||
| 101 | stp q4, q5, [\state, #-16 * 4 - 16] | ||
| 102 | stp q2, q3, [\state, #-16 * 2 - 16] | ||
| 103 | stp q0, q1, [\state, #-16 * 0 - 16] | ||
| 104 | 0: | ||
| 105 | .endm | ||
| 106 | |||
| 107 | .macro fpsimd_restore_partial state, tmpnr1, tmpnr2 | ||
| 108 | ldp w\tmpnr1, w\tmpnr2, [\state] | ||
| 109 | msr fpsr, x\tmpnr1 | ||
| 110 | fpsimd_restore_fpcr x\tmpnr2, x\tmpnr1 | ||
| 111 | adr x\tmpnr1, 0f | ||
| 112 | ldr w\tmpnr2, [\state, #8] | ||
| 113 | add \state, \state, x\tmpnr2, lsl #4 | ||
| 114 | sub x\tmpnr1, x\tmpnr1, x\tmpnr2, lsl #1 | ||
| 115 | br x\tmpnr1 | ||
| 116 | ldp q30, q31, [\state, #-16 * 30 - 16] | ||
| 117 | ldp q28, q29, [\state, #-16 * 28 - 16] | ||
| 118 | ldp q26, q27, [\state, #-16 * 26 - 16] | ||
| 119 | ldp q24, q25, [\state, #-16 * 24 - 16] | ||
| 120 | ldp q22, q23, [\state, #-16 * 22 - 16] | ||
| 121 | ldp q20, q21, [\state, #-16 * 20 - 16] | ||
| 122 | ldp q18, q19, [\state, #-16 * 18 - 16] | ||
| 123 | ldp q16, q17, [\state, #-16 * 16 - 16] | ||
| 124 | ldp q14, q15, [\state, #-16 * 14 - 16] | ||
| 125 | ldp q12, q13, [\state, #-16 * 12 - 16] | ||
| 126 | ldp q10, q11, [\state, #-16 * 10 - 16] | ||
| 127 | ldp q8, q9, [\state, #-16 * 8 - 16] | ||
| 128 | ldp q6, q7, [\state, #-16 * 6 - 16] | ||
| 129 | ldp q4, q5, [\state, #-16 * 4 - 16] | ||
| 130 | ldp q2, q3, [\state, #-16 * 2 - 16] | ||
| 131 | ldp q0, q1, [\state, #-16 * 0 - 16] | ||
| 132 | 0: | ||
| 133 | .endm | ||
| @@ -18,7 +18,6 @@ · 17 unmodified lines | |||
| 18 | #ifndef __ASM_HUGETLB_H | 18 | #ifndef __ASM_HUGETLB_H |
| 19 | #define __ASM_HUGETLB_H | 19 | #define __ASM_HUGETLB_H |
| 20 | 20 | ||
| 21 | #include <asm-generic/hugetlb.h> | ||
| 22 | #include <asm/page.h> | 21 | #include <asm/page.h> |
| 23 | 22 | ||
| 24 | static inline pte_t huge_ptep_get(pte_t *ptep) | 23 | static inline pte_t huge_ptep_get(pte_t *ptep) |
| @@ -82,6 +81,14 @@ extern void huge_ptep_set_wrprotect(struct mm_struct *mm, · 57 unmodified lines | |||
| 82 | unsigned long addr, pte_t *ptep); | 81 | unsigned long addr, pte_t *ptep); |
| 83 | extern void huge_ptep_clear_flush(struct vm_area_struct *vma, | 82 | extern void huge_ptep_clear_flush(struct vm_area_struct *vma, |
| 84 | unsigned long addr, pte_t *ptep); | 83 | unsigned long addr, pte_t *ptep); |
| 84 | extern void huge_pte_clear(struct mm_struct *mm, unsigned long addr, | ||
| 85 | pte_t *ptep, unsigned long sz); | ||
| 86 | #define huge_pte_clear huge_pte_clear | ||
| 87 | extern void set_huge_swap_pte_at(struct mm_struct *mm, unsigned long addr, | ||
| 88 | pte_t *ptep, pte_t pte, unsigned long sz); | ||
| 89 | #define set_huge_swap_pte_at set_huge_swap_pte_at | ||
| 90 | |||
| 91 | #include <asm-generic/hugetlb.h> | ||
| 85 | 92 | ||
| 86 | #ifdef CONFIG_ARCH_HAS_GIGANTIC_PAGE | 93 | #ifdef CONFIG_ARCH_HAS_GIGANTIC_PAGE |
| 87 | static inline bool gigantic_page_supported(void) { return true; } | 94 | static inline bool gigantic_page_supported(void) { return true; } |
| @@ -1,45 +1,12 @@ | |||
| 1 | #ifndef __ASM_IRQ_H | 1 | #ifndef __ASM_IRQ_H |
| 2 | #define __ASM_IRQ_H | 2 | #define __ASM_IRQ_H |
| 3 | 3 | ||
| 4 | #define IRQ_STACK_SIZE THREAD_SIZE | ||
| 5 | #define IRQ_STACK_START_SP THREAD_START_SP | ||
| 6 | |||
| 7 | #ifndef __ASSEMBLER__ | 4 | #ifndef __ASSEMBLER__ |
| 8 | 5 | ||
| 9 | #include <linux/percpu.h> | ||
| 10 | |||
| 11 | #include <asm-generic/irq.h> | 6 | #include <asm-generic/irq.h> |
| 12 | #include <asm/thread_info.h> | ||
| 13 | 7 | ||
| 14 | struct pt_regs; | 8 | struct pt_regs; |
| 15 | 9 | ||
| 16 | DECLARE_PER_CPU(unsigned long [IRQ_STACK_SIZE/sizeof(long)], irq_stack); | ||
| 17 | |||
| 18 | /* | ||
| 19 | * The highest address on the stack, and the first to be used. Used to | ||
| 20 | * find the dummy-stack frame put down by el?_irq() in entry.S, which | ||
| 21 | * is structured as follows: | ||
| 22 | * | ||
| 23 | * ------------ | ||
| 24 | * | | <- irq_stack_ptr | ||
| 25 | * top ------------ | ||
| 26 | * | x19 | <- irq_stack_ptr - 0x08 | ||
| 27 | * ------------ | ||
| 28 | * | x29 | <- irq_stack_ptr - 0x10 | ||
| 29 | * ------------ | ||
| 30 | * | ||
| 31 | * where x19 holds a copy of the task stack pointer where the struct pt_regs | ||
| 32 | * from kernel_entry can be found. | ||
| 33 | * | ||
| 34 | */ | ||
| 35 | #define IRQ_STACK_PTR(cpu) ((unsigned long)per_cpu(irq_stack, cpu) + IRQ_STACK_START_SP) | ||
| 36 | |||
| 37 | /* | ||
| 38 | * The offset from irq_stack_ptr where entry.S will store the original | ||
| 39 | * stack pointer. Used by unwind_frame() and dump_backtrace(). | ||
| 40 | */ | ||
| 41 | #define IRQ_STACK_TO_TASK_STACK(ptr) (*((unsigned long *)((ptr) - 0x08))) | ||
| 42 | |||
| 43 | extern void set_handle_irq(void (*handle_irq)(struct pt_regs *)); | 10 | extern void set_handle_irq(void (*handle_irq)(struct pt_regs *)); |
| 44 | 11 | ||
| 45 | static inline int nr_legacy_irqs(void) | 12 | static inline int nr_legacy_irqs(void) |
| @@ -47,14 +14,5 @@ static inline int nr_legacy_irqs(void) · 1 unmodified line | |||
| 47 | return 0; | 14 | return 0; |
| 48 | } | 15 | } |
| 49 | 16 | ||
| 50 | static inline bool on_irq_stack(unsigned long sp, int cpu) | ||
| 51 | { | ||
| 52 | /* variable names the same as kernel/stacktrace.c */ | ||
| 53 | unsigned long low = (unsigned long)per_cpu(irq_stack, cpu); | ||
| 54 | unsigned long high = low + IRQ_STACK_START_SP; | ||
| 55 | |||
| 56 | return (low <= sp && sp <= high); | ||
| 57 | } | ||
| 58 | |||
| 59 | #endif /* !__ASSEMBLER__ */ | 17 | #endif /* !__ASSEMBLER__ */ |
| 60 | #endif | 18 | #endif |
| @@ -175,18 +175,15 @@ static inline pmd_t kvm_s2pmd_mkwrite(pmd_t pmd) · 174 unmodified lines | |||
| 175 | 175 | ||
| 176 | static inline void kvm_set_s2pte_readonly(pte_t *pte) | 176 | static inline void kvm_set_s2pte_readonly(pte_t *pte) |
| 177 | { | 177 | { |
| 178 | pteval_t pteval; | 178 | pteval_t old_pteval, pteval; |
| 179 | unsigned long tmp; | 179 | |
| 180 | 180 | pteval = READ_ONCE(pte_val(*pte)); | |
| 181 | asm volatile("// kvm_set_s2pte_readonly\n" | 181 | do { |
| 182 | " prfm pstl1strm, %2\n" | 182 | old_pteval = pteval; |
| 183 | "1: ldxr %0, %2\n" | 183 | pteval &= ~PTE_S2_RDWR; |
| 184 | " and %0, %0, %3 // clear PTE_S2_RDWR\n" | 184 | pteval |= PTE_S2_RDONLY; |
| 185 | " orr %0, %0, %4 // set PTE_S2_RDONLY\n" | 185 | pteval = cmpxchg_relaxed(&pte_val(*pte), old_pteval, pteval); |
| 186 | " stxr %w1, %0, %2\n" | 186 | } while (pteval != old_pteval); |
| 187 | " cbnz %w1, 1b\n" | ||
| 188 | : "=&r" (pteval), "=&r" (tmp), "+Q" (pte_val(*pte)) | ||
| 189 | : "L" (~PTE_S2_RDWR), "L" (PTE_S2_RDONLY)); | ||
| 190 | } | 187 | } |
| 191 | 188 | ||
| 192 | static inline bool kvm_s2pte_readonly(pte_t *pte) | 189 | static inline bool kvm_s2pte_readonly(pte_t *pte) |
| @@ -25,6 +25,7 @@ · 24 unmodified lines | |||
| 25 | #include <linux/const.h> | 25 | #include <linux/const.h> |
| 26 | #include <linux/types.h> | 26 | #include <linux/types.h> |
| 27 | #include <asm/bug.h> | 27 | #include <asm/bug.h> |
| 28 | #include <asm/page-def.h> | ||
| 28 | #include <asm/sizes.h> | 29 | #include <asm/sizes.h> |
| 29 | 30 | ||
| 30 | /* | 31 | /* |
| @@ -103,6 +104,58 @@ · 72 unmodified lines | |||
| 103 | #define KASAN_SHADOW_SIZE (0) | 104 | #define KASAN_SHADOW_SIZE (0) |
| 104 | #endif | 105 | #endif |
| 105 | 106 | ||
| 107 | #define MIN_THREAD_SHIFT 14 | ||
| 108 | |||
| 109 | /* | ||
| 110 | * VMAP'd stacks are allocated at page granularity, so we must ensure that such | ||
| 111 | * stacks are a multiple of page size. | ||
| 112 | */ | ||
| 113 | #if defined(CONFIG_VMAP_STACK) && (MIN_THREAD_SHIFT < PAGE_SHIFT) | ||
| 114 | #define THREAD_SHIFT PAGE_SHIFT | ||
| 115 | #else | ||
| 116 | #define THREAD_SHIFT MIN_THREAD_SHIFT | ||
| 117 | #endif | ||
| 118 | |||
| 119 | #if THREAD_SHIFT >= PAGE_SHIFT | ||
| 120 | #define THREAD_SIZE_ORDER (THREAD_SHIFT - PAGE_SHIFT) | ||
| 121 | #endif | ||
| 122 | |||
| 123 | #define THREAD_SIZE (UL(1) << THREAD_SHIFT) | ||
| 124 | |||
| 125 | /* | ||
| 126 | * By aligning VMAP'd stacks to 2 * THREAD_SIZE, we can detect overflow by | ||
| 127 | * checking sp & (1 << THREAD_SHIFT), which we can do cheaply in the entry | ||
| 128 | * assembly. | ||
| 129 | */ | ||
| 130 | #ifdef CONFIG_VMAP_STACK | ||
| 131 | #define THREAD_ALIGN (2 * THREAD_SIZE) | ||
| 132 | #else | ||
| 133 | #define THREAD_ALIGN THREAD_SIZE | ||
| 134 | #endif | ||
| 135 | |||
| 136 | #define IRQ_STACK_SIZE THREAD_SIZE | ||
| 137 | |||
| 138 | #define OVERFLOW_STACK_SIZE SZ_4K | ||
| 139 | |||
| 140 | /* | ||
| 141 | * Alignment of kernel segments (e.g. .text, .data). | ||
| 142 | */ | ||
| 143 | #if defined(CONFIG_DEBUG_ALIGN_RODATA) | ||
| 144 | /* | ||
| 145 | * 4 KB granule: 1 level 2 entry | ||
| 146 | * 16 KB granule: 128 level 3 entries, with contiguous bit | ||
| 147 | * 64 KB granule: 32 level 3 entries, with contiguous bit | ||
| 148 | */ | ||
| 149 | #define SEGMENT_ALIGN SZ_2M | ||
| 150 | #else | ||
| 151 | /* | ||
| 152 | * 4 KB granule: 16 level 3 entries, with contiguous bit | ||
| 153 | * 16 KB granule: 4 level 3 entries, without contiguous bit | ||
| 154 | * 64 KB granule: 1 level 3 entry | ||
| 155 | */ | ||
| 156 | #define SEGMENT_ALIGN SZ_64K | ||
| 157 | #endif | ||
| 158 | |||
| 106 | /* | 159 | /* |
| 107 | * Memory types available. | 160 | * Memory types available. |
| 108 | */ | 161 | */ |
| @@ -16,6 +16,8 @@ · 15 unmodified lines | |||
| 16 | #ifndef __ASM_MMU_H | 16 | #ifndef __ASM_MMU_H |
| 17 | #define __ASM_MMU_H | 17 | #define __ASM_MMU_H |
| 18 | 18 | ||
| 19 | #define MMCF_AARCH32 0x1 /* mm context flag for AArch32 executables */ | ||
| 20 | |||
| 19 | typedef struct { | 21 | typedef struct { |
| 20 | atomic64_t id; | 22 | atomic64_t id; |
| 21 | void *vdso; | 23 | void *vdso; |
| @@ -8,12 +8,22 @@ · 7 unmodified lines | |||
| 8 | * published by the Free Software Foundation. | 8 | * published by the Free Software Foundation. |
| 9 | */ | 9 | */ |
| 10 | 10 | ||
| 11 | #ifndef __ASM_NEON_H | ||
| 12 | #define __ASM_NEON_H | ||
| 13 | |||
| 11 | #include <linux/types.h> | 14 | #include <linux/types.h> |
| 12 | #include <asm/fpsimd.h> | 15 | #include <asm/fpsimd.h> |
| 13 | 16 | ||
| 14 | #define cpu_has_neon() system_supports_fpsimd() | 17 | #define cpu_has_neon() system_supports_fpsimd() |
| 15 | 18 | ||
| 16 | #define kernel_neon_begin() kernel_neon_begin_partial(32) | 19 | void kernel_neon_begin(void); |
| 17 | |||
| 18 | void kernel_neon_begin_partial(u32 num_regs); | ||
| 19 | void kernel_neon_end(void); | 20 | void kernel_neon_end(void); |
| 21 | |||
| 22 | /* | ||
| 23 | * Temporary macro to allow the crypto code to compile. Note that the | ||
| 24 | * semantics of kernel_neon_begin_partial() are now different from the | ||
| 25 | * original as it does not allow being called in an interrupt context. | ||
| 26 | */ | ||
| 27 | #define kernel_neon_begin_partial(num_regs) kernel_neon_begin() | ||
| 28 | |||
| 29 | #endif /* ! __ASM_NEON_H */ | ||
| @@ -7,9 +7,6 @@ · 6 unmodified lines | |||
| 7 | 7 | ||
| 8 | #define NR_NODE_MEMBLKS (MAX_NUMNODES * 2) | 8 | #define NR_NODE_MEMBLKS (MAX_NUMNODES * 2) |
| 9 | 9 | ||
| 10 | /* currently, arm64 implements flat NUMA topology */ | ||
| 11 | #define parent_node(node) (node) | ||
| 12 | |||
| 13 | int __node_distance(int from, int to); | 10 | int __node_distance(int from, int to); |
| 14 | #define node_distance(a, b) __node_distance(a, b) | 11 | #define node_distance(a, b) __node_distance(a, b) |
| 15 | 12 | ||
| @@ -0,0 +1,34 @@ | |||
| 1 | /* | ||
| 2 | * Based on arch/arm/include/asm/page.h | ||
| 3 | * | ||
| 4 | * Copyright (C) 1995-2003 Russell King | ||
| 5 | * Copyright (C) 2017 ARM Ltd. | ||
| 6 | * | ||
| 7 | * This program is free software; you can redistribute it and/or modify | ||
| 8 | * it under the terms of the GNU General Public License version 2 as | ||
| 9 | * published by the Free Software Foundation. | ||
| 10 | * | ||
| 11 | * This program is distributed in the hope that it will be useful, | ||
| 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
| 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
| 14 | * GNU General Public License for more details. | ||
| 15 | * | ||
| 16 | * You should have received a copy of the GNU General Public License | ||
| 17 | * along with this program. If not, see <http://www.gnu.org/licenses/>. | ||
| 18 | */ | ||
| 19 | #ifndef __ASM_PAGE_DEF_H | ||
| 20 | #define __ASM_PAGE_DEF_H | ||
| 21 | |||
| 22 | #include <linux/const.h> | ||
| 23 | |||
| 24 | /* PAGE_SHIFT determines the page size */ | ||
| 25 | /* CONT_SHIFT determines the number of pages which can be tracked together */ | ||
| 26 | #define PAGE_SHIFT CONFIG_ARM64_PAGE_SHIFT | ||
| 27 | #define CONT_SHIFT CONFIG_ARM64_CONT_SHIFT | ||
| 28 | #define PAGE_SIZE (_AC(1, UL) << PAGE_SHIFT) | ||
| 29 | #define PAGE_MASK (~(PAGE_SIZE-1)) | ||
| 30 | |||
| 31 | #define CONT_SIZE (_AC(1, UL) << (CONT_SHIFT + PAGE_SHIFT)) | ||
| 32 | #define CONT_MASK (~(CONT_SIZE-1)) | ||
| 33 | |||
| 34 | #endif /* __ASM_PAGE_DEF_H */ | ||
| @@ -19,17 +19,7 @@ · 18 unmodified lines | |||
| 19 | #ifndef __ASM_PAGE_H | 19 | #ifndef __ASM_PAGE_H |
| 20 | #define __ASM_PAGE_H | 20 | #define __ASM_PAGE_H |
| 21 | 21 | ||
| 22 | #include <linux/const.h> | 22 | #include <asm/page-def.h> |
| 23 | |||
| 24 | /* PAGE_SHIFT determines the page size */ | ||
| 25 | /* CONT_SHIFT determines the number of pages which can be tracked together */ | ||
| 26 | #define PAGE_SHIFT CONFIG_ARM64_PAGE_SHIFT | ||
| 27 | #define CONT_SHIFT CONFIG_ARM64_CONT_SHIFT | ||
| 28 | #define PAGE_SIZE (_AC(1, UL) << PAGE_SHIFT) | ||
| 29 | #define PAGE_MASK (~(PAGE_SIZE-1)) | ||
| 30 | |||
| 31 | #define CONT_SIZE (_AC(1, UL) << (CONT_SHIFT + PAGE_SHIFT)) | ||
| 32 | #define CONT_MASK (~(CONT_SIZE-1)) | ||
| 33 | 23 | ||
| 34 | #ifndef __ASSEMBLY__ | 24 | #ifndef __ASSEMBLY__ |
| 35 | 25 | ||
| @@ -63,23 +63,21 @@ · 62 unmodified lines | |||
| 63 | #define PAGE_S2 __pgprot(PROT_DEFAULT | PTE_S2_MEMATTR(MT_S2_NORMAL) | PTE_S2_RDONLY) | 63 | #define PAGE_S2 __pgprot(PROT_DEFAULT | PTE_S2_MEMATTR(MT_S2_NORMAL) | PTE_S2_RDONLY) |
| 64 | #define PAGE_S2_DEVICE __pgprot(PROT_DEFAULT | PTE_S2_MEMATTR(MT_S2_DEVICE_nGnRE) | PTE_S2_RDONLY | PTE_UXN) | 64 | #define PAGE_S2_DEVICE __pgprot(PROT_DEFAULT | PTE_S2_MEMATTR(MT_S2_DEVICE_nGnRE) | PTE_S2_RDONLY | PTE_UXN) |
| 65 | 65 | ||
| 66 | #define PAGE_NONE __pgprot(((_PAGE_DEFAULT) & ~PTE_VALID) | PTE_PROT_NONE | PTE_PXN | PTE_UXN) | 66 | #define PAGE_NONE __pgprot(((_PAGE_DEFAULT) & ~PTE_VALID) | PTE_PROT_NONE | PTE_RDONLY | PTE_PXN | PTE_UXN) |
| 67 | #define PAGE_SHARED __pgprot(_PAGE_DEFAULT | PTE_USER | PTE_NG | PTE_PXN | PTE_UXN | PTE_WRITE) | 67 | #define PAGE_SHARED __pgprot(_PAGE_DEFAULT | PTE_USER | PTE_NG | PTE_PXN | PTE_UXN | PTE_WRITE) |
| 68 | #define PAGE_SHARED_EXEC __pgprot(_PAGE_DEFAULT | PTE_USER | PTE_NG | PTE_PXN | PTE_WRITE) | 68 | #define PAGE_SHARED_EXEC __pgprot(_PAGE_DEFAULT | PTE_USER | PTE_NG | PTE_PXN | PTE_WRITE) |
| 69 | #define PAGE_COPY __pgprot(_PAGE_DEFAULT | PTE_USER | PTE_NG | PTE_PXN | PTE_UXN) | 69 | #define PAGE_READONLY __pgprot(_PAGE_DEFAULT | PTE_USER | PTE_RDONLY | PTE_NG | PTE_PXN | PTE_UXN) |
| 70 | #define PAGE_COPY_EXEC __pgprot(_PAGE_DEFAULT | PTE_USER | PTE_NG | PTE_PXN) | 70 | #define PAGE_READONLY_EXEC __pgprot(_PAGE_DEFAULT | PTE_USER | PTE_RDONLY | PTE_NG | PTE_PXN) |
| 71 | #define PAGE_READONLY __pgprot(_PAGE_DEFAULT | PTE_USER | PTE_NG | PTE_PXN | PTE_UXN) | 71 | #define PAGE_EXECONLY __pgprot(_PAGE_DEFAULT | PTE_RDONLY | PTE_NG | PTE_PXN) |
| 72 | #define PAGE_READONLY_EXEC __pgprot(_PAGE_DEFAULT | PTE_USER | PTE_NG | PTE_PXN) | ||
| 73 | #define PAGE_EXECONLY __pgprot(_PAGE_DEFAULT | PTE_NG | PTE_PXN) | ||
| 74 | 72 | ||
| 75 | #define __P000 PAGE_NONE | 73 | #define __P000 PAGE_NONE |
| 76 | #define __P001 PAGE_READONLY | 74 | #define __P001 PAGE_READONLY |
| 77 | #define __P010 PAGE_COPY | 75 | #define __P010 PAGE_READONLY |
| 78 | #define __P011 PAGE_COPY | 76 | #define __P011 PAGE_READONLY |
| 79 | #define __P100 PAGE_EXECONLY | 77 | #define __P100 PAGE_EXECONLY |
| 80 | #define __P101 PAGE_READONLY_EXEC | 78 | #define __P101 PAGE_READONLY_EXEC |
| 81 | #define __P110 PAGE_COPY_EXEC | 79 | #define __P110 PAGE_READONLY_EXEC |
| 82 | #define __P111 PAGE_COPY_EXEC | 80 | #define __P111 PAGE_READONLY_EXEC |
| 83 | 81 | ||
| 84 | #define __S000 PAGE_NONE | 82 | #define __S000 PAGE_NONE |
| 85 | #define __S001 PAGE_READONLY | 83 | #define __S001 PAGE_READONLY |
| @@ -39,6 +39,7 @@ · 38 unmodified lines | |||
| 39 | 39 | ||
| 40 | #ifndef __ASSEMBLY__ | 40 | #ifndef __ASSEMBLY__ |
| 41 | 41 | ||
| 42 | #include <asm/cmpxchg.h> | ||
| 42 | #include <asm/fixmap.h> | 43 | #include <asm/fixmap.h> |
| 43 | #include <linux/mmdebug.h> | 44 | #include <linux/mmdebug.h> |
| 44 | 45 | ||
| @@ -84,11 +85,7 @@ extern unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)]; · 39 unmodified lines | |||
| 84 | (__boundary - 1 < (end) - 1) ? __boundary : (end); \ | 85 | (__boundary - 1 < (end) - 1) ? __boundary : (end); \ |
| 85 | }) | 86 | }) |
| 86 | 87 | ||
| 87 | #ifdef CONFIG_ARM64_HW_AFDBM | ||
| 88 | #define pte_hw_dirty(pte) (pte_write(pte) && !(pte_val(pte) & PTE_RDONLY)) | 88 | #define pte_hw_dirty(pte) (pte_write(pte) && !(pte_val(pte) & PTE_RDONLY)) |
| 89 | #else | ||
| 90 | #define pte_hw_dirty(pte) (0) | ||
| 91 | #endif | ||
| 92 | #define pte_sw_dirty(pte) (!!(pte_val(pte) & PTE_DIRTY)) | 89 | #define pte_sw_dirty(pte) (!!(pte_val(pte) & PTE_DIRTY)) |
| 93 | #define pte_dirty(pte) (pte_sw_dirty(pte) || pte_hw_dirty(pte)) | 90 | #define pte_dirty(pte) (pte_sw_dirty(pte) || pte_hw_dirty(pte)) |
| 94 | 91 | ||
| @@ -124,12 +121,16 @@ static inline pte_t set_pte_bit(pte_t pte, pgprot_t prot) · 29 unmodified lines | |||
| 124 | 121 | ||
| 125 | static inline pte_t pte_wrprotect(pte_t pte) | 122 | static inline pte_t pte_wrprotect(pte_t pte) |
| 126 | { | 123 | { |
| 127 | return clear_pte_bit(pte, __pgprot(PTE_WRITE)); | 124 | pte = clear_pte_bit(pte, __pgprot(PTE_WRITE)); |
| 125 | pte = set_pte_bit(pte, __pgprot(PTE_RDONLY)); | ||
| 126 | return pte; | ||
| 128 | } | 127 | } |
| 129 | 128 | ||
| 130 | static inline pte_t pte_mkwrite(pte_t pte) | 129 | static inline pte_t pte_mkwrite(pte_t pte) |
| 131 | { | 130 | { |
| 132 | return set_pte_bit(pte, __pgprot(PTE_WRITE)); | 131 | pte = set_pte_bit(pte, __pgprot(PTE_WRITE)); |
| 132 | pte = clear_pte_bit(pte, __pgprot(PTE_RDONLY)); | ||
| 133 | return pte; | ||
| 133 | } | 134 | } |
| 134 | 135 | ||
| 135 | static inline pte_t pte_mkclean(pte_t pte) | 136 | static inline pte_t pte_mkclean(pte_t pte) |
| @@ -168,11 +169,6 @@ static inline pte_t pte_mknoncont(pte_t pte) · 32 unmodified lines | |||
| 168 | return clear_pte_bit(pte, __pgprot(PTE_CONT)); | 169 | return clear_pte_bit(pte, __pgprot(PTE_CONT)); |
| 169 | } | 170 | } |
| 170 | 171 | ||
| 171 | static inline pte_t pte_clear_rdonly(pte_t pte) | ||
| 172 | { | ||
| 173 | return clear_pte_bit(pte, __pgprot(PTE_RDONLY)); | ||
| 174 | } | ||
| 175 | |||
| 176 | static inline pte_t pte_mkpresent(pte_t pte) | 172 | static inline pte_t pte_mkpresent(pte_t pte) |
| 177 | { | 173 | { |
| 178 | return set_pte_bit(pte, __pgprot(PTE_VALID)); | 174 | return set_pte_bit(pte, __pgprot(PTE_VALID)); |
| @@ -220,22 +216,15 @@ extern void __sync_icache_dcache(pte_t pteval, unsigned long addr); · 41 unmodified lines | |||
| 220 | static inline void set_pte_at(struct mm_struct *mm, unsigned long addr, | 216 | static inline void set_pte_at(struct mm_struct *mm, unsigned long addr, |
| 221 | pte_t *ptep, pte_t pte) | 217 | pte_t *ptep, pte_t pte) |
| 222 | { | 218 | { |
| 223 | if (pte_present(pte)) { | 219 | if (pte_present(pte) && pte_user_exec(pte) && !pte_special(pte)) |
| 224 | if (pte_sw_dirty(pte) && pte_write(pte)) | 220 | __sync_icache_dcache(pte, addr); |
| 225 | pte_val(pte) &= ~PTE_RDONLY; | ||
| 226 | else | ||
| 227 | pte_val(pte) |= PTE_RDONLY; | ||
| 228 | if (pte_user_exec(pte) && !pte_special(pte)) | ||
| 229 | __sync_icache_dcache(pte, addr); | ||
| 230 | } | ||
| 231 | 221 | ||
| 232 | /* | 222 | /* |
| 233 | * If the existing pte is valid, check for potential race with | 223 | * If the existing pte is valid, check for potential race with |
| 234 | * hardware updates of the pte (ptep_set_access_flags safely changes | 224 | * hardware updates of the pte (ptep_set_access_flags safely changes |
| 235 | * valid ptes without going through an invalid entry). | 225 | * valid ptes without going through an invalid entry). |
| 236 | */ | 226 | */ |
| 237 | if (IS_ENABLED(CONFIG_ARM64_HW_AFDBM) && | 227 | if (pte_valid(*ptep) && pte_valid(pte)) { |
| 238 | pte_valid(*ptep) && pte_valid(pte)) { | ||
| 239 | VM_WARN_ONCE(!pte_young(pte), | 228 | VM_WARN_ONCE(!pte_young(pte), |
| 240 | "%s: racy access flag clearing: 0x%016llx -> 0x%016llx", | 229 | "%s: racy access flag clearing: 0x%016llx -> 0x%016llx", |
| 241 | __func__, pte_val(*ptep), pte_val(pte)); | 230 | __func__, pte_val(*ptep), pte_val(pte)); |
| @@ -571,7 +560,6 @@ static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot) · 329 unmodified lines | |||
| 571 | return pte_pmd(pte_modify(pmd_pte(pmd), newprot)); | 560 | return pte_pmd(pte_modify(pmd_pte(pmd), newprot)); |
| 572 | } | 561 | } |
| 573 | 562 | ||
| 574 | #ifdef CONFIG_ARM64_HW_AFDBM | ||
| 575 | #define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS | 563 | #define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS |
| 576 | extern int ptep_set_access_flags(struct vm_area_struct *vma, | 564 | extern int ptep_set_access_flags(struct vm_area_struct *vma, |
| 577 | unsigned long address, pte_t *ptep, | 565 | unsigned long address, pte_t *ptep, |
| @@ -593,20 +581,17 @@ static inline int pmdp_set_access_flags(struct vm_area_struct *vma, · 15 unmodified lines | |||
| 593 | #define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG | 581 | #define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG |
| 594 | static inline int __ptep_test_and_clear_young(pte_t *ptep) | 582 | static inline int __ptep_test_and_clear_young(pte_t *ptep) |
| 595 | { | 583 | { |
| 596 | pteval_t pteval; | 584 | pte_t old_pte, pte; |
| 597 | unsigned int tmp, res; | ||
| 598 | 585 | ||
| 599 | asm volatile("// __ptep_test_and_clear_young\n" | 586 | pte = READ_ONCE(*ptep); |
| 600 | " prfm pstl1strm, %2\n" | 587 | do { |
| 601 | "1: ldxr %0, %2\n" | 588 | old_pte = pte; |
| 602 | " ubfx %w3, %w0, %5, #1 // extract PTE_AF (young)\n" | 589 | pte = pte_mkold(pte); |
| 603 | " and %0, %0, %4 // clear PTE_AF\n" | 590 | pte_val(pte) = cmpxchg_relaxed(&pte_val(*ptep), |
| 604 | " stxr %w1, %0, %2\n" | 591 | pte_val(old_pte), pte_val(pte)); |
| 605 | " cbnz %w1, 1b\n" | 592 | } while (pte_val(pte) != pte_val(old_pte)); |
| 606 | : "=&r" (pteval), "=&r" (tmp), "+Q" (pte_val(*ptep)), "=&r" (res) | ||
| 607 | : "L" (~PTE_AF), "I" (ilog2(PTE_AF))); | ||
| 608 | 593 | ||
| 609 | return res; | 594 | return pte_young(pte); |
| 610 | } | 595 | } |
| 611 | 596 | ||
| 612 | static inline int ptep_test_and_clear_young(struct vm_area_struct *vma, | 597 | static inline int ptep_test_and_clear_young(struct vm_area_struct *vma, |
| @@ -630,17 +615,7 @@ static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma, · 17 unmodified lines | |||
| 630 | static inline pte_t ptep_get_and_clear(struct mm_struct *mm, | 615 | static inline pte_t ptep_get_and_clear(struct mm_struct *mm, |
| 631 | unsigned long address, pte_t *ptep) | 616 | unsigned long address, pte_t *ptep) |
| 632 | { | 617 | { |
| 633 | pteval_t old_pteval; | 618 | return __pte(xchg_relaxed(&pte_val(*ptep), 0)); |
| 634 | unsigned int tmp; | ||
| 635 | |||
| 636 | asm volatile("// ptep_get_and_clear\n" | ||
| 637 | " prfm pstl1strm, %2\n" | ||
| 638 | "1: ldxr %0, %2\n" | ||
| 639 | " stxr %w1, xzr, %2\n" | ||
| 640 | " cbnz %w1, 1b\n" | ||
| 641 | : "=&r" (old_pteval), "=&r" (tmp), "+Q" (pte_val(*ptep))); | ||
| 642 | |||
| 643 | return __pte(old_pteval); | ||
| 644 | } | 619 | } |
| 645 | 620 | ||
| 646 | #ifdef CONFIG_TRANSPARENT_HUGEPAGE | 621 | #ifdef CONFIG_TRANSPARENT_HUGEPAGE |
| @@ -653,27 +628,32 @@ static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm, · 6 unmodified lines | |||
| 653 | #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ | 628 | #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ |
| 654 | 629 | ||
| 655 | /* | 630 | /* |
| 656 | * ptep_set_wrprotect - mark read-only while trasferring potential hardware | 631 | * ptep_set_wrprotect - mark read-only while preserving the hardware update of |
| 657 | * dirty status (PTE_DBM && !PTE_RDONLY) to the software PTE_DIRTY bit. | 632 | * the Access Flag. |
| 658 | */ | 633 | */ |
| 659 | #define __HAVE_ARCH_PTEP_SET_WRPROTECT | 634 | #define __HAVE_ARCH_PTEP_SET_WRPROTECT |
| 660 | static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long address, pte_t *ptep) | 635 | static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long address, pte_t *ptep) |
| 661 | { | 636 | { |
| 662 | pteval_t pteval; | 637 | pte_t old_pte, pte; |
| 663 | unsigned long tmp; | ||
| 664 | 638 | ||
| 665 | asm volatile("// ptep_set_wrprotect\n" | 639 | /* |
| 666 | " prfm pstl1strm, %2\n" | 640 | * ptep_set_wrprotect() is only called on CoW mappings which are |
| 667 | "1: ldxr %0, %2\n" | 641 | * private (!VM_SHARED) with the pte either read-only (!PTE_WRITE && |
| 668 | " tst %0, %4 // check for hw dirty (!PTE_RDONLY)\n" | 642 | * PTE_RDONLY) or writable and software-dirty (PTE_WRITE && |
| 669 | " csel %1, %3, xzr, eq // set PTE_DIRTY|PTE_RDONLY if dirty\n" | 643 | * !PTE_RDONLY && PTE_DIRTY); see is_cow_mapping() and |
| 670 | " orr %0, %0, %1 // if !dirty, PTE_RDONLY is already set\n" | 644 | * protection_map[]. There is no race with the hardware update of the |
| 671 | " and %0, %0, %5 // clear PTE_WRITE/PTE_DBM\n" | 645 | * dirty state: clearing of PTE_RDONLY when PTE_WRITE (a.k.a. PTE_DBM) |
| 672 | " stxr %w1, %0, %2\n" | 646 | * is set. |
| 673 | " cbnz %w1, 1b\n" | 647 | */ |
| 674 | : "=&r" (pteval), "=&r" (tmp), "+Q" (pte_val(*ptep)) | 648 | VM_WARN_ONCE(pte_write(*ptep) && !pte_dirty(*ptep), |
| 675 | : "r" (PTE_DIRTY|PTE_RDONLY), "L" (PTE_RDONLY), "L" (~PTE_WRITE) | 649 | "%s: potential race with hardware DBM", __func__); |
| 676 | : "cc"); | 650 | pte = READ_ONCE(*ptep); |
| 651 | do { | ||
| 652 | old_pte = pte; | ||
| 653 | pte = pte_wrprotect(pte); | ||
| 654 | pte_val(pte) = cmpxchg_relaxed(&pte_val(*ptep), | ||
| 655 | pte_val(old_pte), pte_val(pte)); | ||
| 656 | } while (pte_val(pte) != pte_val(old_pte)); | ||
| 677 | } | 657 | } |
| 678 | 658 | ||
| 679 | #ifdef CONFIG_TRANSPARENT_HUGEPAGE | 659 | #ifdef CONFIG_TRANSPARENT_HUGEPAGE |
| @@ -684,7 +664,6 @@ static inline void pmdp_set_wrprotect(struct mm_struct *mm, · 4 unmodified lines | |||
| 684 | ptep_set_wrprotect(mm, address, (pte_t *)pmdp); | 664 | ptep_set_wrprotect(mm, address, (pte_t *)pmdp); |
| 685 | } | 665 | } |
| 686 | #endif | 666 | #endif |
| 687 | #endif /* CONFIG_ARM64_HW_AFDBM */ | ||
| 688 | 667 | ||
| 689 | extern pgd_t swapper_pg_dir[PTRS_PER_PGD]; | 668 | extern pgd_t swapper_pg_dir[PTRS_PER_PGD]; |
| 690 | extern pgd_t idmap_pg_dir[PTRS_PER_PGD]; | 669 | extern pgd_t idmap_pg_dir[PTRS_PER_PGD]; |
| @@ -112,7 +112,7 @@ void tls_preserve_current_state(void); · 111 unmodified lines | |||
| 112 | static inline void start_thread_common(struct pt_regs *regs, unsigned long pc) | 112 | static inline void start_thread_common(struct pt_regs *regs, unsigned long pc) |
| 113 | { | 113 | { |
| 114 | memset(regs, 0, sizeof(*regs)); | 114 | memset(regs, 0, sizeof(*regs)); |
| 115 | regs->syscallno = ~0UL; | 115 | forget_syscall(regs); |
| 116 | regs->pc = pc; | 116 | regs->pc = pc; |
| 117 | } | 117 | } |
| 118 | 118 | ||
| @@ -159,7 +159,7 @@ extern struct task_struct *cpu_switch_to(struct task_struct *prev, · 40 unmodified lines | |||
| 159 | struct task_struct *next); | 159 | struct task_struct *next); |
| 160 | 160 | ||
| 161 | #define task_pt_regs(p) \ | 161 | #define task_pt_regs(p) \ |
| 162 | ((struct pt_regs *)(THREAD_START_SP + task_stack_page(p)) - 1) | 162 | ((struct pt_regs *)(THREAD_SIZE + task_stack_page(p)) - 1) |
| 163 | 163 | ||
| 164 | #define KSTK_EIP(tsk) ((unsigned long)task_pt_regs(tsk)->pc) | 164 | #define KSTK_EIP(tsk) ((unsigned long)task_pt_regs(tsk)->pc) |
| 165 | #define KSTK_ESP(tsk) user_stack_pointer(task_pt_regs(tsk)) | 165 | #define KSTK_ESP(tsk) user_stack_pointer(task_pt_regs(tsk)) |
| @@ -72,8 +72,19 @@ · 71 unmodified lines | |||
| 72 | #define COMPAT_PT_TEXT_ADDR 0x10000 | 72 | #define COMPAT_PT_TEXT_ADDR 0x10000 |
| 73 | #define COMPAT_PT_DATA_ADDR 0x10004 | 73 | #define COMPAT_PT_DATA_ADDR 0x10004 |
| 74 | #define COMPAT_PT_TEXT_END_ADDR 0x10008 | 74 | #define COMPAT_PT_TEXT_END_ADDR 0x10008 |
| 75 | |||
| 76 | /* | ||
| 77 | * If pt_regs.syscallno == NO_SYSCALL, then the thread is not executing | ||
| 78 | * a syscall -- i.e., its most recent entry into the kernel from | ||
| 79 | * userspace was not via SVC, or otherwise a tracer cancelled the syscall. | ||
| 80 | * | ||
| 81 | * This must have the value -1, for ABI compatibility with ptrace etc. | ||
| 82 | */ | ||
| 83 | #define NO_SYSCALL (-1) | ||
| 84 | |||
| 75 | #ifndef __ASSEMBLY__ | 85 | #ifndef __ASSEMBLY__ |
| 76 | #include <linux/bug.h> | 86 | #include <linux/bug.h> |
| 87 | #include <linux/types.h> | ||
| 77 | 88 | ||
| 78 | /* sizeof(struct user) for AArch32 */ | 89 | /* sizeof(struct user) for AArch32 */ |
| 79 | #define COMPAT_USER_SZ 296 | 90 | #define COMPAT_USER_SZ 296 |
| @@ -116,11 +127,29 @@ struct pt_regs { · 36 unmodified lines | |||
| 116 | }; | 127 | }; |
| 117 | }; | 128 | }; |
| 118 | u64 orig_x0; | 129 | u64 orig_x0; |
| 119 | u64 syscallno; | 130 | #ifdef __AARCH64EB__ |
| 131 | u32 unused2; | ||
| 132 | s32 syscallno; | ||
| 133 | #else | ||
| 134 | s32 syscallno; | ||
| 135 | u32 unused2; | ||
| 136 | #endif | ||
| 137 | |||
| 120 | u64 orig_addr_limit; | 138 | u64 orig_addr_limit; |
| 121 | u64 unused; // maintain 16 byte alignment | 139 | u64 unused; // maintain 16 byte alignment |
| 140 | u64 stackframe[2]; | ||
| 122 | }; | 141 | }; |
| 123 | 142 | ||
| 143 | static inline bool in_syscall(struct pt_regs const *regs) | ||
| 144 | { | ||
| 145 | return regs->syscallno != NO_SYSCALL; | ||
| 146 | } | ||
| 147 | |||
| 148 | static inline void forget_syscall(struct pt_regs *regs) | ||
| 149 | { | ||
| 150 | regs->syscallno = NO_SYSCALL; | ||
| 151 | } | ||
| 152 | |||
| 124 | #define MAX_REG_OFFSET offsetof(struct pt_regs, pstate) | 153 | #define MAX_REG_OFFSET offsetof(struct pt_regs, pstate) |
| 125 | 154 | ||
| 126 | #define arch_has_single_step() (1) | 155 | #define arch_has_single_step() (1) |
| @@ -22,8 +22,6 @@ · 21 unmodified lines | |||
| 22 | 22 | ||
| 23 | #define AARCH32_KERN_SIGRET_CODE_OFFSET 0x500 | 23 | #define AARCH32_KERN_SIGRET_CODE_OFFSET 0x500 |
| 24 | 24 | ||
| 25 | extern const compat_ulong_t aarch32_sigret_code[6]; | ||
| 26 | |||
| 27 | int compat_setup_frame(int usig, struct ksignal *ksig, sigset_t *set, | 25 | int compat_setup_frame(int usig, struct ksignal *ksig, sigset_t *set, |
| 28 | struct pt_regs *regs); | 26 | struct pt_regs *regs); |
| 29 | int compat_setup_rt_frame(int usig, struct ksignal *ksig, sigset_t *set, | 27 | int compat_setup_rt_frame(int usig, struct ksignal *ksig, sigset_t *set, |
| @@ -0,0 +1,56 @@ | |||
| 1 | /* | ||
| 2 | * Copyright (C) 2017 Linaro Ltd. <ard.biesheuvel@linaro.org> | ||
| 3 | * | ||
| 4 | * This program is free software; you can redistribute it and/or modify it | ||
| 5 | * under the terms of the GNU General Public License version 2 as published | ||
| 6 | * by the Free Software Foundation. | ||
| 7 | */ | ||
| 8 | |||
| 9 | #ifndef __ASM_SIMD_H | ||
| 10 | #define __ASM_SIMD_H | ||
| 11 | |||
| 12 | #include <linux/compiler.h> | ||
| 13 | #include <linux/irqflags.h> | ||
| 14 | #include <linux/percpu.h> | ||
| 15 | #include <linux/preempt.h> | ||
| 16 | #include <linux/types.h> | ||
| 17 | |||
| 18 | #ifdef CONFIG_KERNEL_MODE_NEON | ||
| 19 | |||
| 20 | DECLARE_PER_CPU(bool, kernel_neon_busy); | ||
| 21 | |||
| 22 | /* | ||
| 23 | * may_use_simd - whether it is allowable at this time to issue SIMD | ||
| 24 | * instructions or access the SIMD register file | ||
| 25 | * | ||
| 26 | * Callers must not assume that the result remains true beyond the next | ||
| 27 | * preempt_enable() or return from softirq context. | ||
| 28 | */ | ||
| 29 | static __must_check inline bool may_use_simd(void) | ||
| 30 | { | ||
| 31 | /* | ||
| 32 | * The raw_cpu_read() is racy if called with preemption enabled. | ||
| 33 | * This is not a bug: kernel_neon_busy is only set when | ||
| 34 | * preemption is disabled, so we cannot migrate to another CPU | ||
| 35 | * while it is set, nor can we migrate to a CPU where it is set. | ||
| 36 | * So, if we find it clear on some CPU then we're guaranteed to | ||
| 37 | * find it clear on any CPU we could migrate to. | ||
| 38 | * | ||
| 39 | * If we are in between kernel_neon_begin()...kernel_neon_end(), | ||
| 40 | * the flag will be set, but preemption is also disabled, so we | ||
| 41 | * can't migrate to another CPU and spuriously see it become | ||
| 42 | * false. | ||
| 43 | */ | ||
| 44 | return !in_irq() && !irqs_disabled() && !in_nmi() && | ||
| 45 | !raw_cpu_read(kernel_neon_busy); | ||
| 46 | } | ||
| 47 | |||
| 48 | #else /* ! CONFIG_KERNEL_MODE_NEON */ | ||
| 49 | |||
| 50 | static __must_check inline bool may_use_simd(void) { | ||
| 51 | return false; | ||
| 52 | } | ||
| 53 | |||
| 54 | #endif /* ! CONFIG_KERNEL_MODE_NEON */ | ||
| 55 | |||
| 56 | #endif | ||
| @@ -148,7 +148,7 @@ static inline void cpu_panic_kernel(void) · 147 unmodified lines | |||
| 148 | */ | 148 | */ |
| 149 | bool cpus_are_stuck_in_kernel(void); | 149 | bool cpus_are_stuck_in_kernel(void); |
| 150 | 150 | ||
| 151 | extern void smp_send_crash_stop(void); | 151 | extern void crash_smp_send_stop(void); |
| 152 | extern bool smp_crash_stop_failed(void); | 152 | extern bool smp_crash_stop_failed(void); |
| 153 | 153 | ||
| 154 | #endif /* ifndef __ASSEMBLY__ */ | 154 | #endif /* ifndef __ASSEMBLY__ */ |
| @@ -16,11 +16,15 @@ · 15 unmodified lines | |||
| 16 | #ifndef __ASM_STACKTRACE_H | 16 | #ifndef __ASM_STACKTRACE_H |
| 17 | #define __ASM_STACKTRACE_H | 17 | #define __ASM_STACKTRACE_H |
| 18 | 18 | ||
| 19 | struct task_struct; | 19 | #include <linux/percpu.h> |
| 20 | #include <linux/sched.h> | ||
| 21 | #include <linux/sched/task_stack.h> | ||
| 22 | |||
| 23 | #include <asm/memory.h> | ||
| 24 | #include <asm/ptrace.h> | ||
| 20 | 25 | ||
| 21 | struct stackframe { | 26 | struct stackframe { |
| 22 | unsigned long fp; | 27 | unsigned long fp; |
| 23 | unsigned long sp; | ||
| 24 | unsigned long pc; | 28 | unsigned long pc; |
| 25 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER | 29 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER |
| 26 | unsigned int graph; | 30 | unsigned int graph; |
| @@ -32,4 +36,57 @@ extern void walk_stackframe(struct task_struct *tsk, struct stackframe *frame, · 5 unmodified lines | |||
| 32 | int (*fn)(struct stackframe *, void *), void *data); | 36 | int (*fn)(struct stackframe *, void *), void *data); |
| 33 | extern void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk); | 37 | extern void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk); |
| 34 | 38 | ||
| 39 | DECLARE_PER_CPU(unsigned long *, irq_stack_ptr); | ||
| 40 | |||
| 41 | static inline bool on_irq_stack(unsigned long sp) | ||
| 42 | { | ||
| 43 | unsigned long low = (unsigned long)raw_cpu_read(irq_stack_ptr); | ||
| 44 | unsigned long high = low + IRQ_STACK_SIZE; | ||
| 45 | |||
| 46 | if (!low) | ||
| 47 | return false; | ||
| 48 | |||
| 49 | return (low <= sp && sp < high); | ||
| 50 | } | ||
| 51 | |||
| 52 | static inline bool on_task_stack(struct task_struct *tsk, unsigned long sp) | ||
| 53 | { | ||
| 54 | unsigned long low = (unsigned long)task_stack_page(tsk); | ||
| 55 | unsigned long high = low + THREAD_SIZE; | ||
| 56 | |||
| 57 | return (low <= sp && sp < high); | ||
| 58 | } | ||
| 59 | |||
| 60 | #ifdef CONFIG_VMAP_STACK | ||
| 61 | DECLARE_PER_CPU(unsigned long [OVERFLOW_STACK_SIZE/sizeof(long)], overflow_stack); | ||
| 62 | |||
| 63 | static inline bool on_overflow_stack(unsigned long sp) | ||
| 64 | { | ||
| 65 | unsigned long low = (unsigned long)raw_cpu_ptr(overflow_stack); | ||
| 66 | unsigned long high = low + OVERFLOW_STACK_SIZE; | ||
| 67 | |||
| 68 | return (low <= sp && sp < high); | ||
| 69 | } | ||
| 70 | #else | ||
| 71 | static inline bool on_overflow_stack(unsigned long sp) { return false; } | ||
| 72 | #endif | ||
| 73 | |||
| 74 | /* | ||
| 75 | * We can only safely access per-cpu stacks from current in a non-preemptible | ||
| 76 | * context. | ||
| 77 | */ | ||
| 78 | static inline bool on_accessible_stack(struct task_struct *tsk, unsigned long sp) | ||
| 79 | { | ||
| 80 | if (on_task_stack(tsk, sp)) | ||
| 81 | return true; | ||
| 82 | if (tsk != current || preemptible()) | ||
| 83 | return false; | ||
| 84 | if (on_irq_stack(sp)) | ||
| 85 | return true; | ||
| 86 | if (on_overflow_stack(sp)) | ||
| 87 | return true; | ||
| 88 | |||
| 89 | return false; | ||
| 90 | } | ||
| 91 | |||
| 35 | #endif /* __ASM_STACKTRACE_H */ | 92 | #endif /* __ASM_STACKTRACE_H */ |
| @@ -52,6 +52,10 @@ extern void *__memset(void *, int, __kernel_size_t); · 51 unmodified lines | |||
| 52 | #define __HAVE_ARCH_MEMCMP | 52 | #define __HAVE_ARCH_MEMCMP |
| 53 | extern int memcmp(const void *, const void *, size_t); | 53 | extern int memcmp(const void *, const void *, size_t); |
| 54 | 54 | ||
| 55 | #ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE | ||
| 56 | #define __HAVE_ARCH_MEMCPY_FLUSHCACHE | ||
| 57 | void memcpy_flushcache(void *dst, const void *src, size_t cnt); | ||
| 58 | #endif | ||
| 55 | 59 | ||
| 56 | #if defined(CONFIG_KASAN) && !defined(__SANITIZE_ADDRESS__) | 60 | #if defined(CONFIG_KASAN) && !defined(__SANITIZE_ADDRESS__) |
| 57 | 61 | ||
| @@ -329,6 +329,7 @@ · 328 unmodified lines | |||
| 329 | #define ID_AA64ISAR1_LRCPC_SHIFT 20 | 329 | #define ID_AA64ISAR1_LRCPC_SHIFT 20 |
| 330 | #define ID_AA64ISAR1_FCMA_SHIFT 16 | 330 | #define ID_AA64ISAR1_FCMA_SHIFT 16 |
| 331 | #define ID_AA64ISAR1_JSCVT_SHIFT 12 | 331 | #define ID_AA64ISAR1_JSCVT_SHIFT 12 |
| 332 | #define ID_AA64ISAR1_DPB_SHIFT 0 | ||
| 332 | 333 | ||
| 333 | /* id_aa64pfr0 */ | 334 | /* id_aa64pfr0 */ |
| 334 | #define ID_AA64PFR0_GIC_SHIFT 24 | 335 | #define ID_AA64PFR0_GIC_SHIFT 24 |
| @@ -23,19 +23,11 @@ · 22 unmodified lines | |||
| 23 | 23 | ||
| 24 | #include <linux/compiler.h> | 24 | #include <linux/compiler.h> |
| 25 | 25 | ||
| 26 | #ifdef CONFIG_ARM64_4K_PAGES | ||
| 27 | #define THREAD_SIZE_ORDER 2 | ||
| 28 | #elif defined(CONFIG_ARM64_16K_PAGES) | ||
| 29 | #define THREAD_SIZE_ORDER 0 | ||
| 30 | #endif | ||
| 31 | |||
| 32 | #define THREAD_SIZE 16384 | ||
| 33 | #define THREAD_START_SP (THREAD_SIZE - 16) | ||
| 34 | |||
| 35 | #ifndef __ASSEMBLY__ | 26 | #ifndef __ASSEMBLY__ |
| 36 | 27 | ||
| 37 | struct task_struct; | 28 | struct task_struct; |
| 38 | 29 | ||
| 30 | #include <asm/memory.h> | ||
| 39 | #include <asm/stack_pointer.h> | 31 | #include <asm/stack_pointer.h> |
| 40 | #include <asm/types.h> | 32 | #include <asm/types.h> |
| 41 | 33 | ||
| @@ -68,6 +60,9 @@ struct thread_info { · 26 unmodified lines | |||
| 68 | #define thread_saved_fp(tsk) \ | 60 | #define thread_saved_fp(tsk) \ |
| 69 | ((unsigned long)(tsk->thread.cpu_context.fp)) | 61 | ((unsigned long)(tsk->thread.cpu_context.fp)) |
| 70 | 62 | ||
| 63 | void arch_setup_new_exec(void); | ||
| 64 | #define arch_setup_new_exec arch_setup_new_exec | ||
| 65 | |||
| 71 | #endif | 66 | #endif |
| 72 | 67 | ||
| 73 | /* | 68 | /* |
| @@ -53,4 +53,9 @@ static inline int in_exception_text(unsigned long ptr) · 52 unmodified lines | |||
| 53 | return in ? : __in_irqentry_text(ptr); | 53 | return in ? : __in_irqentry_text(ptr); |
| 54 | } | 54 | } |
| 55 | 55 | ||
| 56 | static inline int in_entry_text(unsigned long ptr) | ||
| 57 | { | ||
| 58 | return ptr >= (unsigned long)&__entry_text_start && | ||
| 59 | ptr < (unsigned long)&__entry_text_end; | ||
| 60 | } | ||
| 56 | #endif | 61 | #endif |
| @@ -350,4 +350,16 @@ extern long strncpy_from_user(char *dest, const char __user *src, long count); · 349 unmodified lines | |||
| 350 | 350 | ||
| 351 | extern __must_check long strnlen_user(const char __user *str, long n); | 351 | extern __must_check long strnlen_user(const char __user *str, long n); |
| 352 | 352 | ||
| 353 | #ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE | ||
| 354 | struct page; | ||
| 355 | void memcpy_page_flushcache(char *to, struct page *page, size_t offset, size_t len); | ||
| 356 | extern unsigned long __must_check __copy_user_flushcache(void *to, const void __user *from, unsigned long n); | ||
| 357 | |||
| 358 | static inline int __copy_from_user_flushcache(void *dst, const void __user *src, unsigned size) | ||
| 359 | { | ||
| 360 | kasan_check_write(dst, size); | ||
| 361 | return __copy_user_flushcache(dst, src, size); | ||
| 362 | } | ||
| 363 | #endif | ||
| 364 | |||
| 353 | #endif /* __ASM_UACCESS_H */ | 365 | #endif /* __ASM_UACCESS_H */ |
| @@ -35,5 +35,6 @@ · 34 unmodified lines | |||
| 35 | #define HWCAP_JSCVT (1 << 13) | 35 | #define HWCAP_JSCVT (1 << 13) |
| 36 | #define HWCAP_FCMA (1 << 14) | 36 | #define HWCAP_FCMA (1 << 14) |
| 37 | #define HWCAP_LRCPC (1 << 15) | 37 | #define HWCAP_LRCPC (1 << 15) |
| 38 | #define HWCAP_DCPOP (1 << 16) | ||
| 38 | 39 | ||
| 39 | #endif /* _UAPI__ASM_HWCAP_H */ | 40 | #endif /* _UAPI__ASM_HWCAP_H */ |
| @@ -75,6 +75,7 @@ int main(void) · 74 unmodified lines | |||
| 75 | DEFINE(S_ORIG_X0, offsetof(struct pt_regs, orig_x0)); | 75 | DEFINE(S_ORIG_X0, offsetof(struct pt_regs, orig_x0)); |
| 76 | DEFINE(S_SYSCALLNO, offsetof(struct pt_regs, syscallno)); | 76 | DEFINE(S_SYSCALLNO, offsetof(struct pt_regs, syscallno)); |
| 77 | DEFINE(S_ORIG_ADDR_LIMIT, offsetof(struct pt_regs, orig_addr_limit)); | 77 | DEFINE(S_ORIG_ADDR_LIMIT, offsetof(struct pt_regs, orig_addr_limit)); |
| 78 | DEFINE(S_STACKFRAME, offsetof(struct pt_regs, stackframe)); | ||
| 78 | DEFINE(S_FRAME_SIZE, sizeof(struct pt_regs)); | 79 | DEFINE(S_FRAME_SIZE, sizeof(struct pt_regs)); |
| 79 | BLANK(); | 80 | BLANK(); |
| 80 | DEFINE(MM_CONTEXT_ID, offsetof(struct mm_struct, context.id.counter)); | 81 | DEFINE(MM_CONTEXT_ID, offsetof(struct mm_struct, context.id.counter)); |
| @@ -120,6 +120,7 @@ static const struct arm64_ftr_bits ftr_id_aa64isar1[] = { · 119 unmodified lines | |||
| 120 | ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64ISAR1_LRCPC_SHIFT, 4, 0), | 120 | ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64ISAR1_LRCPC_SHIFT, 4, 0), |
| 121 | ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64ISAR1_FCMA_SHIFT, 4, 0), | 121 | ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64ISAR1_FCMA_SHIFT, 4, 0), |
| 122 | ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64ISAR1_JSCVT_SHIFT, 4, 0), | 122 | ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64ISAR1_JSCVT_SHIFT, 4, 0), |
| 123 | ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64ISAR1_DPB_SHIFT, 4, 0), | ||
| 123 | ARM64_FTR_END, | 124 | ARM64_FTR_END, |
| 124 | }; | 125 | }; |
| 125 | 126 | ||
| @@ -888,6 +889,17 @@ static const struct arm64_cpu_capabilities arm64_features[] = { · 762 unmodified lines | |||
| 888 | .min_field_value = 0, | 889 | .min_field_value = 0, |
| 889 | .matches = has_no_fpsimd, | 890 | .matches = has_no_fpsimd, |
| 890 | }, | 891 | }, |
| 892 | #ifdef CONFIG_ARM64_PMEM | ||
| 893 | { | ||
| 894 | .desc = "Data cache clean to Point of Persistence", | ||
| 895 | .capability = ARM64_HAS_DCPOP, | ||
| 896 | .def_scope = SCOPE_SYSTEM, | ||
| 897 | .matches = has_cpuid_feature, | ||
| 898 | .sys_reg = SYS_ID_AA64ISAR1_EL1, | ||
| 899 | .field_pos = ID_AA64ISAR1_DPB_SHIFT, | ||
| 900 | .min_field_value = 1, | ||
| 901 | }, | ||
| 902 | #endif | ||
| 891 | {}, | 903 | {}, |
| 892 | }; | 904 | }; |
| 893 | 905 | ||
| @@ -916,6 +928,7 @@ static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = { · 22 unmodified lines | |||
| 916 | HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_FP_SHIFT, FTR_SIGNED, 1, CAP_HWCAP, HWCAP_FPHP), | 928 | HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_FP_SHIFT, FTR_SIGNED, 1, CAP_HWCAP, HWCAP_FPHP), |
| 917 | HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_ASIMD_SHIFT, FTR_SIGNED, 0, CAP_HWCAP, HWCAP_ASIMD), | 929 | HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_ASIMD_SHIFT, FTR_SIGNED, 0, CAP_HWCAP, HWCAP_ASIMD), |
| 918 | HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_ASIMD_SHIFT, FTR_SIGNED, 1, CAP_HWCAP, HWCAP_ASIMDHP), | 930 | HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_ASIMD_SHIFT, FTR_SIGNED, 1, CAP_HWCAP, HWCAP_ASIMDHP), |
| 931 | HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_DPB_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_DCPOP), | ||
| 919 | HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_JSCVT_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_JSCVT), | 932 | HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_JSCVT_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_JSCVT), |
| 920 | HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_FCMA_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_FCMA), | 933 | HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_FCMA_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_FCMA), |
| 921 | HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_LRCPC_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_LRCPC), | 934 | HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_LRCPC_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_LRCPC), |
| @@ -68,6 +68,7 @@ static const char *const hwcap_str[] = { · 67 unmodified lines | |||
| 68 | "jscvt", | 68 | "jscvt", |
| 69 | "fcma", | 69 | "fcma", |
| 70 | "lrcpc", | 70 | "lrcpc", |
| 71 | "dcpop", | ||
| 71 | NULL | 72 | NULL |
| 72 | }; | 73 | }; |
| 73 | 74 | ||
| @@ -41,27 +41,3 @@ ENTRY(fpsimd_load_state) · 40 unmodified lines | |||
| 41 | fpsimd_restore x0, 8 | 41 | fpsimd_restore x0, 8 |
| 42 | ret | 42 | ret |
| 43 | ENDPROC(fpsimd_load_state) | 43 | ENDPROC(fpsimd_load_state) |
| 44 | |||
| 45 | #ifdef CONFIG_KERNEL_MODE_NEON | ||
| 46 | |||
| 47 | /* | ||
| 48 | * Save the bottom n FP registers. | ||
| 49 | * | ||
| 50 | * x0 - pointer to struct fpsimd_partial_state | ||
| 51 | */ | ||
| 52 | ENTRY(fpsimd_save_partial_state) | ||
| 53 | fpsimd_save_partial x0, 1, 8, 9 | ||
| 54 | ret | ||
| 55 | ENDPROC(fpsimd_save_partial_state) | ||
| 56 | |||
| 57 | /* | ||
| 58 | * Load the bottom n FP registers. | ||
| 59 | * | ||
| 60 | * x0 - pointer to struct fpsimd_partial_state | ||
| 61 | */ | ||
| 62 | ENTRY(fpsimd_load_partial_state) | ||
| 63 | fpsimd_restore_partial x0, 8, 9 | ||
| 64 | ret | ||
| 65 | ENDPROC(fpsimd_load_partial_state) | ||
| 66 | |||
| 67 | #endif | ||
| @@ -69,8 +69,55 @@ · 68 unmodified lines | |||
| 69 | #define BAD_FIQ 2 | 69 | #define BAD_FIQ 2 |
| 70 | #define BAD_ERROR 3 | 70 | #define BAD_ERROR 3 |
| 71 | 71 | ||
| 72 | .macro kernel_entry, el, regsize = 64 | 72 | .macro kernel_ventry label |
| 73 | .align 7 | ||
| 73 | sub sp, sp, #S_FRAME_SIZE | 74 | sub sp, sp, #S_FRAME_SIZE |
| 75 | #ifdef CONFIG_VMAP_STACK | ||
| 76 | /* | ||
| 77 | * Test whether the SP has overflowed, without corrupting a GPR. | ||
| 78 | * Task and IRQ stacks are aligned to (1 << THREAD_SHIFT). | ||
| 79 | */ | ||
| 80 | add sp, sp, x0 // sp' = sp + x0 | ||
| 81 | sub x0, sp, x0 // x0' = sp' - x0 = (sp + x0) - x0 = sp | ||
| 82 | tbnz x0, #THREAD_SHIFT, 0f | ||
| 83 | sub x0, sp, x0 // x0'' = sp' - x0' = (sp + x0) - sp = x0 | ||
| 84 | sub sp, sp, x0 // sp'' = sp' - x0 = (sp + x0) - x0 = sp | ||
| 85 | b \label | ||
| 86 | |||
| 87 | 0: | ||
| 88 | /* | ||
| 89 | * Either we've just detected an overflow, or we've taken an exception | ||
| 90 | * while on the overflow stack. Either way, we won't return to | ||
| 91 | * userspace, and can clobber EL0 registers to free up GPRs. | ||
| 92 | */ | ||
| 93 | |||
| 94 | /* Stash the original SP (minus S_FRAME_SIZE) in tpidr_el0. */ | ||
| 95 | msr tpidr_el0, x0 | ||
| 96 | |||
| 97 | /* Recover the original x0 value and stash it in tpidrro_el0 */ | ||
| 98 | sub x0, sp, x0 | ||
| 99 | msr tpidrro_el0, x0 | ||
| 100 | |||
| 101 | /* Switch to the overflow stack */ | ||
| 102 | adr_this_cpu sp, overflow_stack + OVERFLOW_STACK_SIZE, x0 | ||
| 103 | |||
| 104 | /* | ||
| 105 | * Check whether we were already on the overflow stack. This may happen | ||
| 106 | * after panic() re-enables interrupts. | ||
| 107 | */ | ||
| 108 | mrs x0, tpidr_el0 // sp of interrupted context | ||
| 109 | sub x0, sp, x0 // delta with top of overflow stack | ||
| 110 | tst x0, #~(OVERFLOW_STACK_SIZE - 1) // within range? | ||
| 111 | b.ne __bad_stack // no? -> bad stack pointer | ||
| 112 | |||
| 113 | /* We were already on the overflow stack. Restore sp/x0 and carry on. */ | ||
| 114 | sub sp, sp, x0 | ||
| 115 | mrs x0, tpidrro_el0 | ||
| 116 | #endif | ||
| 117 | b \label | ||
| 118 | .endm | ||
| 119 | |||
| 120 | .macro kernel_entry, el, regsize = 64 | ||
| 74 | .if \regsize == 32 | 121 | .if \regsize == 32 |
| 75 | mov w0, w0 // zero upper 32 bits of x0 | 122 | mov w0, w0 // zero upper 32 bits of x0 |
| 76 | .endif | 123 | .endif |
| @@ -111,6 +158,18 @@ · 34 unmodified lines | |||
| 111 | mrs x23, spsr_el1 | 158 | mrs x23, spsr_el1 |
| 112 | stp lr, x21, [sp, #S_LR] | 159 | stp lr, x21, [sp, #S_LR] |
| 113 | 160 | ||
| 161 | /* | ||
| 162 | * In order to be able to dump the contents of struct pt_regs at the | ||
| 163 | * time the exception was taken (in case we attempt to walk the call | ||
| 164 | * stack later), chain it together with the stack frames. | ||
| 165 | */ | ||
| 166 | .if \el == 0 | ||
| 167 | stp xzr, xzr, [sp, #S_STACKFRAME] | ||
| 168 | .else | ||
| 169 | stp x29, x22, [sp, #S_STACKFRAME] | ||
| 170 | .endif | ||
| 171 | add x29, sp, #S_STACKFRAME | ||
| 172 | |||
| 114 | #ifdef CONFIG_ARM64_SW_TTBR0_PAN | 173 | #ifdef CONFIG_ARM64_SW_TTBR0_PAN |
| 115 | /* | 174 | /* |
| 116 | * Set the TTBR0 PAN bit in SPSR. When the exception is taken from | 175 | * Set the TTBR0 PAN bit in SPSR. When the exception is taken from |
| @@ -138,12 +197,10 @@ alternative_else_nop_endif · 21 unmodified lines | |||
| 138 | 197 | ||
| 139 | stp x22, x23, [sp, #S_PC] | 198 | stp x22, x23, [sp, #S_PC] |
| 140 | 199 | ||
| 141 | /* | 200 | /* Not in a syscall by default (el0_svc overwrites for real syscall) */ |
| 142 | * Set syscallno to -1 by default (overridden later if real syscall). | ||
| 143 | */ | ||
| 144 | .if \el == 0 | 201 | .if \el == 0 |
| 145 | mvn x21, xzr | 202 | mov w21, #NO_SYSCALL |
| 146 | str x21, [sp, #S_SYSCALLNO] | 203 | str w21, [sp, #S_SYSCALLNO] |
| 147 | .endif | 204 | .endif |
| 148 | 205 | ||
| 149 | /* | 206 | /* |
| @@ -259,20 +316,12 @@ alternative_else_nop_endif · 109 unmodified lines | |||
| 259 | and x25, x25, #~(THREAD_SIZE - 1) | 316 | and x25, x25, #~(THREAD_SIZE - 1) |
| 260 | cbnz x25, 9998f | 317 | cbnz x25, 9998f |
| 261 | 318 | ||
| 262 | adr_this_cpu x25, irq_stack, x26 | 319 | ldr_this_cpu x25, irq_stack_ptr, x26 |
| 263 | mov x26, #IRQ_STACK_START_SP | 320 | mov x26, #IRQ_STACK_SIZE |
| 264 | add x26, x25, x26 | 321 | add x26, x25, x26 |
| 265 | 322 | ||
| 266 | /* switch to the irq stack */ | 323 | /* switch to the irq stack */ |
| 267 | mov sp, x26 | 324 | mov sp, x26 |
| 268 | |||
| 269 | /* | ||
| 270 | * Add a dummy stack frame, this non-standard format is fixed up | ||
| 271 | * by unwind_frame() | ||
| 272 | */ | ||
| 273 | stp x29, x19, [sp, #-16]! | ||
| 274 | mov x29, sp | ||
| 275 | |||
| 276 | 9998: | 325 | 9998: |
| 277 | .endm | 326 | .endm |
| 278 | 327 | ||
| @@ -290,8 +339,9 @@ alternative_else_nop_endif · 11 unmodified lines | |||
| 290 | * | 339 | * |
| 291 | * x7 is reserved for the system call number in 32-bit mode. | 340 | * x7 is reserved for the system call number in 32-bit mode. |
| 292 | */ | 341 | */ |
| 293 | sc_nr .req x25 // number of system calls | 342 | wsc_nr .req w25 // number of system calls |
| 294 | scno .req x26 // syscall number | 343 | wscno .req w26 // syscall number |
| 344 | xscno .req x26 // syscall number (zero-extended) | ||
| 295 | stbl .req x27 // syscall table pointer | 345 | stbl .req x27 // syscall table pointer |
| 296 | tsk .req x28 // current thread_info | 346 | tsk .req x28 // current thread_info |
| 297 | 347 | ||
| @@ -315,34 +365,62 @@ tsk .req x28 // current thread_info · 17 unmodified lines | |||
| 315 | 365 | ||
| 316 | .align 11 | 366 | .align 11 |
| 317 | ENTRY(vectors) | 367 | ENTRY(vectors) |
| 318 | ventry el1_sync_invalid // Synchronous EL1t | 368 | kernel_ventry el1_sync_invalid // Synchronous EL1t |
| 319 | ventry el1_irq_invalid // IRQ EL1t | 369 | kernel_ventry el1_irq_invalid // IRQ EL1t |
| 320 | ventry el1_fiq_invalid // FIQ EL1t | 370 | kernel_ventry el1_fiq_invalid // FIQ EL1t |
| 321 | ventry el1_error_invalid // Error EL1t | 371 | kernel_ventry el1_error_invalid // Error EL1t |
| 322 | 372 | ||
| 323 | ventry el1_sync // Synchronous EL1h | 373 | kernel_ventry el1_sync // Synchronous EL1h |
| 324 | ventry el1_irq // IRQ EL1h | 374 | kernel_ventry el1_irq // IRQ EL1h |
| 325 | ventry el1_fiq_invalid // FIQ EL1h | 375 | kernel_ventry el1_fiq_invalid // FIQ EL1h |
| 326 | ventry el1_error_invalid // Error EL1h | 376 | kernel_ventry el1_error_invalid // Error EL1h |
| 327 | 377 | ||
| 328 | ventry el0_sync // Synchronous 64-bit EL0 | 378 | kernel_ventry el0_sync // Synchronous 64-bit EL0 |
| 329 | ventry el0_irq // IRQ 64-bit EL0 | 379 | kernel_ventry el0_irq // IRQ 64-bit EL0 |
| 330 | ventry el0_fiq_invalid // FIQ 64-bit EL0 | 380 | kernel_ventry el0_fiq_invalid // FIQ 64-bit EL0 |
| 331 | ventry el0_error_invalid // Error 64-bit EL0 | 381 | kernel_ventry el0_error_invalid // Error 64-bit EL0 |
| 332 | 382 | ||
| 333 | #ifdef CONFIG_COMPAT | 383 | #ifdef CONFIG_COMPAT |
| 334 | ventry el0_sync_compat // Synchronous 32-bit EL0 | 384 | kernel_ventry el0_sync_compat // Synchronous 32-bit EL0 |
| 335 | ventry el0_irq_compat // IRQ 32-bit EL0 | 385 | kernel_ventry el0_irq_compat // IRQ 32-bit EL0 |
| 336 | ventry el0_fiq_invalid_compat // FIQ 32-bit EL0 | 386 | kernel_ventry el0_fiq_invalid_compat // FIQ 32-bit EL0 |
| 337 | ventry el0_error_invalid_compat // Error 32-bit EL0 | 387 | kernel_ventry el0_error_invalid_compat // Error 32-bit EL0 |
| 338 | #else | 388 | #else |
| 339 | ventry el0_sync_invalid // Synchronous 32-bit EL0 | 389 | kernel_ventry el0_sync_invalid // Synchronous 32-bit EL0 |
| 340 | ventry el0_irq_invalid // IRQ 32-bit EL0 | 390 | kernel_ventry el0_irq_invalid // IRQ 32-bit EL0 |
| 341 | ventry el0_fiq_invalid // FIQ 32-bit EL0 | 391 | kernel_ventry el0_fiq_invalid // FIQ 32-bit EL0 |
| 342 | ventry el0_error_invalid // Error 32-bit EL0 | 392 | kernel_ventry el0_error_invalid // Error 32-bit EL0 |
| 343 | #endif | 393 | #endif |
| 344 | END(vectors) | 394 | END(vectors) |
| 345 | 395 | ||
| 396 | #ifdef CONFIG_VMAP_STACK | ||
| 397 | /* | ||
| 398 | * We detected an overflow in kernel_ventry, which switched to the | ||
| 399 | * overflow stack. Stash the exception regs, and head to our overflow | ||
| 400 | * handler. | ||
| 401 | */ | ||
| 402 | __bad_stack: | ||
| 403 | /* Restore the original x0 value */ | ||
| 404 | mrs x0, tpidrro_el0 | ||
| 405 | |||
| 406 | /* | ||
| 407 | * Store the original GPRs to the new stack. The orginal SP (minus | ||
| 408 | * S_FRAME_SIZE) was stashed in tpidr_el0 by kernel_ventry. | ||
| 409 | */ | ||
| 410 | sub sp, sp, #S_FRAME_SIZE | ||
| 411 | kernel_entry 1 | ||
| 412 | mrs x0, tpidr_el0 | ||
| 413 | add x0, x0, #S_FRAME_SIZE | ||
| 414 | str x0, [sp, #S_SP] | ||
| 415 | |||
| 416 | /* Stash the regs for handle_bad_stack */ | ||
| 417 | mov x0, sp | ||
| 418 | |||
| 419 | /* Time to die */ | ||
| 420 | bl handle_bad_stack | ||
| 421 | ASM_BUG() | ||
| 422 | #endif /* CONFIG_VMAP_STACK */ | ||
| 423 | |||
| 346 | /* | 424 | /* |
| 347 | * Invalid mode handlers | 425 | * Invalid mode handlers |
| 348 | */ | 426 | */ |
| @@ -351,7 +429,8 @@ END(vectors) · 2 unmodified lines | |||
| 351 | mov x0, sp | 429 | mov x0, sp |
| 352 | mov x1, #\reason | 430 | mov x1, #\reason |
| 353 | mrs x2, esr_el1 | 431 | mrs x2, esr_el1 |
| 354 | b bad_mode | 432 | bl bad_mode |
| 433 | ASM_BUG() | ||
| 355 | .endm | 434 | .endm |
| 356 | 435 | ||
| 357 | el0_sync_invalid: | 436 | el0_sync_invalid: |
| @@ -448,14 +527,16 @@ el1_sp_pc: · 90 unmodified lines | |||
| 448 | mrs x0, far_el1 | 527 | mrs x0, far_el1 |
| 449 | enable_dbg | 528 | enable_dbg |
| 450 | mov x2, sp | 529 | mov x2, sp |
| 451 | b do_sp_pc_abort | 530 | bl do_sp_pc_abort |
| 531 | ASM_BUG() | ||
| 452 | el1_undef: | 532 | el1_undef: |
| 453 | /* | 533 | /* |
| 454 | * Undefined instruction | 534 | * Undefined instruction |
| 455 | */ | 535 | */ |
| 456 | enable_dbg | 536 | enable_dbg |
| 457 | mov x0, sp | 537 | mov x0, sp |
| 458 | b do_undefinstr | 538 | bl do_undefinstr |
| 539 | ASM_BUG() | ||
| 459 | el1_dbg: | 540 | el1_dbg: |
| 460 | /* | 541 | /* |
| 461 | * Debug exception handling | 542 | * Debug exception handling |
| @@ -473,7 +554,8 @@ el1_inv: · 11 unmodified lines | |||
| 473 | mov x0, sp | 554 | mov x0, sp |
| 474 | mov x2, x1 | 555 | mov x2, x1 |
| 475 | mov x1, #BAD_SYNC | 556 | mov x1, #BAD_SYNC |
| 476 | b bad_mode | 557 | bl bad_mode |
| 558 | ASM_BUG() | ||
| 477 | ENDPROC(el1_sync) | 559 | ENDPROC(el1_sync) |
| 478 | 560 | ||
| 479 | .align 6 | 561 | .align 6 |
| @@ -577,8 +659,8 @@ el0_svc_compat: · 97 unmodified lines | |||
| 577 | * AArch32 syscall handling | 659 | * AArch32 syscall handling |
| 578 | */ | 660 | */ |
| 579 | adrp stbl, compat_sys_call_table // load compat syscall table pointer | 661 | adrp stbl, compat_sys_call_table // load compat syscall table pointer |
| 580 | uxtw scno, w7 // syscall number in w7 (r7) | 662 | mov wscno, w7 // syscall number in w7 (r7) |
| 581 | mov sc_nr, #__NR_compat_syscalls | 663 | mov wsc_nr, #__NR_compat_syscalls |
| 582 | b el0_svc_naked | 664 | b el0_svc_naked |
| 583 | 665 | ||
| 584 | .align 6 | 666 | .align 6 |
| @@ -706,38 +788,6 @@ el0_irq_naked: · 121 unmodified lines | |||
| 706 | b ret_to_user | 788 | b ret_to_user |
| 707 | ENDPROC(el0_irq) | 789 | ENDPROC(el0_irq) |
| 708 | 790 | ||
| 709 | /* | ||
| 710 | * Register switch for AArch64. The callee-saved registers need to be saved | ||
| 711 | * and restored. On entry: | ||
| 712 | * x0 = previous task_struct (must be preserved across the switch) | ||
| 713 | * x1 = next task_struct | ||
| 714 | * Previous and next are guaranteed not to be the same. | ||
| 715 | * | ||
| 716 | */ | ||
| 717 | ENTRY(cpu_switch_to) | ||
| 718 | mov x10, #THREAD_CPU_CONTEXT | ||
| 719 | add x8, x0, x10 | ||
| 720 | mov x9, sp | ||
| 721 | stp x19, x20, [x8], #16 // store callee-saved registers | ||
| 722 | stp x21, x22, [x8], #16 | ||
| 723 | stp x23, x24, [x8], #16 | ||
| 724 | stp x25, x26, [x8], #16 | ||
| 725 | stp x27, x28, [x8], #16 | ||
| 726 | stp x29, x9, [x8], #16 | ||
| 727 | str lr, [x8] | ||
| 728 | add x8, x1, x10 | ||
| 729 | ldp x19, x20, [x8], #16 // restore callee-saved registers | ||
| 730 | ldp x21, x22, [x8], #16 | ||
| 731 | ldp x23, x24, [x8], #16 | ||
| 732 | ldp x25, x26, [x8], #16 | ||
| 733 | ldp x27, x28, [x8], #16 | ||
| 734 | ldp x29, x9, [x8], #16 | ||
| 735 | ldr lr, [x8] | ||
| 736 | mov sp, x9 | ||
| 737 | msr sp_el0, x1 | ||
| 738 | ret | ||
| 739 | ENDPROC(cpu_switch_to) | ||
| 740 | |||
| 741 | /* | 791 | /* |
| 742 | * This is the fast syscall return path. We do as little as possible here, | 792 | * This is the fast syscall return path. We do as little as possible here, |
| 743 | * and this includes saving x0 back into the kernel stack. | 793 | * and this includes saving x0 back into the kernel stack. |
| @@ -780,37 +830,25 @@ finish_ret_to_user: · 36 unmodified lines | |||
| 780 | kernel_exit 0 | 830 | kernel_exit 0 |
| 781 | ENDPROC(ret_to_user) | 831 | ENDPROC(ret_to_user) |
| 782 | 832 | ||
| 783 | /* | ||
| 784 | * This is how we return from a fork. | ||
| 785 | */ | ||
| 786 | ENTRY(ret_from_fork) | ||
| 787 | bl schedule_tail | ||
| 788 | cbz x19, 1f // not a kernel thread | ||
| 789 | mov x0, x20 | ||
| 790 | blr x19 | ||
| 791 | 1: get_thread_info tsk | ||
| 792 | b ret_to_user | ||
| 793 | ENDPROC(ret_from_fork) | ||
| 794 | |||
| 795 | /* | 833 | /* |
| 796 | * SVC handler. | 834 | * SVC handler. |
| 797 | */ | 835 | */ |
| 798 | .align 6 | 836 | .align 6 |
| 799 | el0_svc: | 837 | el0_svc: |
| 800 | adrp stbl, sys_call_table // load syscall table pointer | 838 | adrp stbl, sys_call_table // load syscall table pointer |
| 801 | uxtw scno, w8 // syscall number in w8 | 839 | mov wscno, w8 // syscall number in w8 |
| 802 | mov sc_nr, #__NR_syscalls | 840 | mov wsc_nr, #__NR_syscalls |
| 803 | el0_svc_naked: // compat entry point | 841 | el0_svc_naked: // compat entry point |
| 804 | stp x0, scno, [sp, #S_ORIG_X0] // save the original x0 and syscall number | 842 | stp x0, xscno, [sp, #S_ORIG_X0] // save the original x0 and syscall number |
| 805 | enable_dbg_and_irq | 843 | enable_dbg_and_irq |
| 806 | ct_user_exit 1 | 844 | ct_user_exit 1 |
| 807 | 845 | ||
| 808 | ldr x16, [tsk, #TSK_TI_FLAGS] // check for syscall hooks | 846 | ldr x16, [tsk, #TSK_TI_FLAGS] // check for syscall hooks |
| 809 | tst x16, #_TIF_SYSCALL_WORK | 847 | tst x16, #_TIF_SYSCALL_WORK |
| 810 | b.ne __sys_trace | 848 | b.ne __sys_trace |
| 811 | cmp scno, sc_nr // check upper syscall limit | 849 | cmp wscno, wsc_nr // check upper syscall limit |
| 812 | b.hs ni_sys | 850 | b.hs ni_sys |
| 813 | ldr x16, [stbl, scno, lsl #3] // address in the syscall table | 851 | ldr x16, [stbl, xscno, lsl #3] // address in the syscall table |
| 814 | blr x16 // call sys_* routine | 852 | blr x16 // call sys_* routine |
| 815 | b ret_fast_syscall | 853 | b ret_fast_syscall |
| 816 | ni_sys: | 854 | ni_sys: |
| @@ -824,24 +862,23 @@ ENDPROC(el0_svc) · 7 unmodified lines | |||
| 824 | * switches, and waiting for our parent to respond. | 862 | * switches, and waiting for our parent to respond. |
| 825 | */ | 863 | */ |
| 826 | __sys_trace: | 864 | __sys_trace: |
| 827 | mov w0, #-1 // set default errno for | 865 | cmp wscno, #NO_SYSCALL // user-issued syscall(-1)? |
| 828 | cmp scno, x0 // user-issued syscall(-1) | ||
| 829 | b.ne 1f | 866 | b.ne 1f |
| 830 | mov x0, #-ENOSYS | 867 | mov x0, #-ENOSYS // set default errno if so |
| 831 | str x0, [sp, #S_X0] | 868 | str x0, [sp, #S_X0] |
| 832 | 1: mov x0, sp | 869 | 1: mov x0, sp |
| 833 | bl syscall_trace_enter | 870 | bl syscall_trace_enter |
| 834 | cmp w0, #-1 // skip the syscall? | 871 | cmp w0, #NO_SYSCALL // skip the syscall? |
| 835 | b.eq __sys_trace_return_skipped | 872 | b.eq __sys_trace_return_skipped |
| 836 | uxtw scno, w0 // syscall number (possibly new) | 873 | mov wscno, w0 // syscall number (possibly new) |
| 837 | mov x1, sp // pointer to regs | 874 | mov x1, sp // pointer to regs |
| 838 | cmp scno, sc_nr // check upper syscall limit | 875 | cmp wscno, wsc_nr // check upper syscall limit |
| 839 | b.hs __ni_sys_trace | 876 | b.hs __ni_sys_trace |
| 840 | ldp x0, x1, [sp] // restore the syscall args | 877 | ldp x0, x1, [sp] // restore the syscall args |
| 841 | ldp x2, x3, [sp, #S_X2] | 878 | ldp x2, x3, [sp, #S_X2] |
| 842 | ldp x4, x5, [sp, #S_X4] | 879 | ldp x4, x5, [sp, #S_X4] |
| 843 | ldp x6, x7, [sp, #S_X6] | 880 | ldp x6, x7, [sp, #S_X6] |
| 844 | ldr x16, [stbl, scno, lsl #3] // address in the syscall table | 881 | ldr x16, [stbl, xscno, lsl #3] // address in the syscall table |
| 845 | blr x16 // call sys_* routine | 882 | blr x16 // call sys_* routine |
| 846 | 883 | ||
| 847 | __sys_trace_return: | 884 | __sys_trace_return: |
| @@ -865,3 +902,49 @@ ENTRY(sys_rt_sigreturn_wrapper) · 17 unmodified lines | |||
| 865 | mov x0, sp | 902 | mov x0, sp |
| 866 | b sys_rt_sigreturn | 903 | b sys_rt_sigreturn |
| 867 | ENDPROC(sys_rt_sigreturn_wrapper) | 904 | ENDPROC(sys_rt_sigreturn_wrapper) |
| 905 | |||
| 906 | /* | ||
| 907 | * Register switch for AArch64. The callee-saved registers need to be saved | ||
| 908 | * and restored. On entry: | ||
| 909 | * x0 = previous task_struct (must be preserved across the switch) | ||
| 910 | * x1 = next task_struct | ||
| 911 | * Previous and next are guaranteed not to be the same. | ||
| 912 | * | ||
| 913 | */ | ||
| 914 | ENTRY(cpu_switch_to) | ||
| 915 | mov x10, #THREAD_CPU_CONTEXT | ||
| 916 | add x8, x0, x10 | ||
| 917 | mov x9, sp | ||
| 918 | stp x19, x20, [x8], #16 // store callee-saved registers | ||
| 919 | stp x21, x22, [x8], #16 | ||
| 920 | stp x23, x24, [x8], #16 | ||
| 921 | stp x25, x26, [x8], #16 | ||
| 922 | stp x27, x28, [x8], #16 | ||
| 923 | stp x29, x9, [x8], #16 | ||
| 924 | str lr, [x8] | ||
| 925 | add x8, x1, x10 | ||
| 926 | ldp x19, x20, [x8], #16 // restore callee-saved registers | ||
| 927 | ldp x21, x22, [x8], #16 | ||
| 928 | ldp x23, x24, [x8], #16 | ||
| 929 | ldp x25, x26, [x8], #16 | ||
| 930 | ldp x27, x28, [x8], #16 | ||
| 931 | ldp x29, x9, [x8], #16 | ||
| 932 | ldr lr, [x8] | ||
| 933 | mov sp, x9 | ||
| 934 | msr sp_el0, x1 | ||
| 935 | ret | ||
| 936 | ENDPROC(cpu_switch_to) | ||
| 937 | NOKPROBE(cpu_switch_to) | ||
| 938 | |||
| 939 | /* | ||
| 940 | * This is how we return from a fork. | ||
| 941 | */ | ||
| 942 | ENTRY(ret_from_fork) | ||
| 943 | bl schedule_tail | ||
| 944 | cbz x19, 1f // not a kernel thread | ||
| 945 | mov x0, x20 | ||
| 946 | blr x19 | ||
| 947 | 1: get_thread_info tsk | ||
| 948 | b ret_to_user | ||
| 949 | ENDPROC(ret_from_fork) | ||
| 950 | NOKPROBE(ret_from_fork) | ||
| @@ -17,16 +17,19 @@ · 16 unmodified lines | |||
| 17 | * along with this program. If not, see <http://www.gnu.org/licenses/>. | 17 | * along with this program. If not, see <http://www.gnu.org/licenses/>. |
| 18 | */ | 18 | */ |
| 19 | 19 | ||
| 20 | #include <linux/bottom_half.h> | ||
| 20 | #include <linux/cpu.h> | 21 | #include <linux/cpu.h> |
| 21 | #include <linux/cpu_pm.h> | 22 | #include <linux/cpu_pm.h> |
| 22 | #include <linux/kernel.h> | 23 | #include <linux/kernel.h> |
| 23 | #include <linux/init.h> | 24 | #include <linux/init.h> |
| 25 | #include <linux/percpu.h> | ||
| 26 | #include <linux/preempt.h> | ||
| 24 | #include <linux/sched/signal.h> | 27 | #include <linux/sched/signal.h> |
| 25 | #include <linux/signal.h> | 28 | #include <linux/signal.h> |
| 26 | #include <linux/hardirq.h> | ||
| 27 | 29 | ||
| 28 | #include <asm/fpsimd.h> | 30 | #include <asm/fpsimd.h> |
| 29 | #include <asm/cputype.h> | 31 | #include <asm/cputype.h> |
| 32 | #include <asm/simd.h> | ||
| 30 | 33 | ||
| 31 | #define FPEXC_IOF (1 << 0) | 34 | #define FPEXC_IOF (1 << 0) |
| 32 | #define FPEXC_DZF (1 << 1) | 35 | #define FPEXC_DZF (1 << 1) |
| @@ -62,6 +65,13 @@ · 29 unmodified lines | |||
| 62 | * CPU currently contain the most recent userland FPSIMD state of the current | 65 | * CPU currently contain the most recent userland FPSIMD state of the current |
| 63 | * task. | 66 | * task. |
| 64 | * | 67 | * |
| 68 | * In order to allow softirq handlers to use FPSIMD, kernel_neon_begin() may | ||
| 69 | * save the task's FPSIMD context back to task_struct from softirq context. | ||
| 70 | * To prevent this from racing with the manipulation of the task's FPSIMD state | ||
| 71 | * from task context and thereby corrupting the state, it is necessary to | ||
| 72 | * protect any manipulation of a task's fpsimd_state or TIF_FOREIGN_FPSTATE | ||
| 73 | * flag with local_bh_disable() unless softirqs are already masked. | ||
| 74 | * | ||
| 65 | * For a certain task, the sequence may look something like this: | 75 | * For a certain task, the sequence may look something like this: |
| 66 | * - the task gets scheduled in; if both the task's fpsimd_state.cpu field | 76 | * - the task gets scheduled in; if both the task's fpsimd_state.cpu field |
| 67 | * contains the id of the current CPU, and the CPU's fpsimd_last_state per-cpu | 77 | * contains the id of the current CPU, and the CPU's fpsimd_last_state per-cpu |
| @@ -161,11 +171,14 @@ void fpsimd_flush_thread(void) · 93 unmodified lines | |||
| 161 | { | 171 | { |
| 162 | if (!system_supports_fpsimd()) | 172 | if (!system_supports_fpsimd()) |
| 163 | return; | 173 | return; |
| 164 | preempt_disable(); | 174 | |
| 175 | local_bh_disable(); | ||
| 176 | |||
| 165 | memset(¤t->thread.fpsimd_state, 0, sizeof(struct fpsimd_state)); | 177 | memset(¤t->thread.fpsimd_state, 0, sizeof(struct fpsimd_state)); |
| 166 | fpsimd_flush_task_state(current); | 178 | fpsimd_flush_task_state(current); |
| 167 | set_thread_flag(TIF_FOREIGN_FPSTATE); | 179 | set_thread_flag(TIF_FOREIGN_FPSTATE); |
| 168 | preempt_enable(); | 180 | |
| 181 | local_bh_enable(); | ||
| 169 | } | 182 | } |
| 170 | 183 | ||
| 171 | /* | 184 | /* |
| @@ -176,10 +189,13 @@ void fpsimd_preserve_current_state(void) · 4 unmodified lines | |||
| 176 | { | 189 | { |
| 177 | if (!system_supports_fpsimd()) | 190 | if (!system_supports_fpsimd()) |
| 178 | return; | 191 | return; |
| 179 | preempt_disable(); | 192 | |
| 193 | local_bh_disable(); | ||
| 194 | |||
| 180 | if (!test_thread_flag(TIF_FOREIGN_FPSTATE)) | 195 | if (!test_thread_flag(TIF_FOREIGN_FPSTATE)) |
| 181 | fpsimd_save_state(¤t->thread.fpsimd_state); | 196 | fpsimd_save_state(¤t->thread.fpsimd_state); |
| 182 | preempt_enable(); | 197 | |
| 198 | local_bh_enable(); | ||
| 183 | } | 199 | } |
| 184 | 200 | ||
| 185 | /* | 201 | /* |
| @@ -191,15 +207,18 @@ void fpsimd_restore_current_state(void) · 5 unmodified lines | |||
| 191 | { | 207 | { |
| 192 | if (!system_supports_fpsimd()) | 208 | if (!system_supports_fpsimd()) |
| 193 | return; | 209 | return; |
| 194 | preempt_disable(); | 210 | |
| 211 | local_bh_disable(); | ||
| 212 | |||
| 195 | if (test_and_clear_thread_flag(TIF_FOREIGN_FPSTATE)) { | 213 | if (test_and_clear_thread_flag(TIF_FOREIGN_FPSTATE)) { |
| 196 | struct fpsimd_state *st = ¤t->thread.fpsimd_state; | 214 | struct fpsimd_state *st = ¤t->thread.fpsimd_state; |
| 197 | 215 | ||
| 198 | fpsimd_load_state(st); | 216 | fpsimd_load_state(st); |
| 199 | this_cpu_write(fpsimd_last_state, st); | 217 | __this_cpu_write(fpsimd_last_state, st); |
| 200 | st->cpu = smp_processor_id(); | 218 | st->cpu = smp_processor_id(); |
| 201 | } | 219 | } |
| 202 | preempt_enable(); | 220 | |
| 221 | local_bh_enable(); | ||
| 203 | } | 222 | } |
| 204 | 223 | ||
| 205 | /* | 224 | /* |
| @@ -211,15 +230,18 @@ void fpsimd_update_current_state(struct fpsimd_state *state) · 5 unmodified lines | |||
| 211 | { | 230 | { |
| 212 | if (!system_supports_fpsimd()) | 231 | if (!system_supports_fpsimd()) |
| 213 | return; | 232 | return; |
| 214 | preempt_disable(); | 233 | |
| 234 | local_bh_disable(); | ||
| 235 | |||
| 215 | fpsimd_load_state(state); | 236 | fpsimd_load_state(state); |
| 216 | if (test_and_clear_thread_flag(TIF_FOREIGN_FPSTATE)) { | 237 | if (test_and_clear_thread_flag(TIF_FOREIGN_FPSTATE)) { |
| 217 | struct fpsimd_state *st = ¤t->thread.fpsimd_state; | 238 | struct fpsimd_state *st = ¤t->thread.fpsimd_state; |
| 218 | 239 | ||
| 219 | this_cpu_write(fpsimd_last_state, st); | 240 | __this_cpu_write(fpsimd_last_state, st); |
| 220 | st->cpu = smp_processor_id(); | 241 | st->cpu = smp_processor_id(); |
| 221 | } | 242 | } |
| 222 | preempt_enable(); | 243 | |
| 244 | local_bh_enable(); | ||
| 223 | } | 245 | } |
| 224 | 246 | ||
| 225 | /* | 247 | /* |
| @@ -232,52 +254,122 @@ void fpsimd_flush_task_state(struct task_struct *t) · 6 unmodified lines | |||
| 232 | 254 | ||
| 233 | #ifdef CONFIG_KERNEL_MODE_NEON | 255 | #ifdef CONFIG_KERNEL_MODE_NEON |
| 234 | 256 | ||
| 235 | static DEFINE_PER_CPU(struct fpsimd_partial_state, hardirq_fpsimdstate); | 257 | DEFINE_PER_CPU(bool, kernel_neon_busy); |
| 236 | static DEFINE_PER_CPU(struct fpsimd_partial_state, softirq_fpsimdstate); | 258 | EXPORT_PER_CPU_SYMBOL(kernel_neon_busy); |
| 237 | 259 | ||
| 238 | /* | 260 | /* |
| 239 | * Kernel-side NEON support functions | 261 | * Kernel-side NEON support functions |
| 240 | */ | 262 | */ |
| 241 | void kernel_neon_begin_partial(u32 num_regs) | 263 | |
| 264 | /* | ||
| 265 | * kernel_neon_begin(): obtain the CPU FPSIMD registers for use by the calling | ||
| 266 | * context | ||
| 267 | * | ||
| 268 | * Must not be called unless may_use_simd() returns true. | ||
| 269 | * Task context in the FPSIMD registers is saved back to memory as necessary. | ||
| 270 | * | ||
| 271 | * A matching call to kernel_neon_end() must be made before returning from the | ||
| 272 | * calling context. | ||
| 273 | * | ||
| 274 | * The caller may freely use the FPSIMD registers until kernel_neon_end() is | ||
| 275 | * called. | ||
| 276 | */ | ||
| 277 | void kernel_neon_begin(void) | ||
| 242 | { | 278 | { |
| 243 | if (WARN_ON(!system_supports_fpsimd())) | 279 | if (WARN_ON(!system_supports_fpsimd())) |
| 244 | return; | 280 | return; |
| 245 | if (in_interrupt()) { | ||
| 246 | struct fpsimd_partial_state *s = this_cpu_ptr( | ||
| 247 | in_irq() ? &hardirq_fpsimdstate : &softirq_fpsimdstate); | ||
| 248 | 281 | ||
| 249 | BUG_ON(num_regs > 32); | 282 | BUG_ON(!may_use_simd()); |
| 250 | fpsimd_save_partial_state(s, roundup(num_regs, 2)); | 283 | |
| 251 | } else { | 284 | local_bh_disable(); |
| 252 | /* | 285 | |
| 253 | * Save the userland FPSIMD state if we have one and if we | 286 | __this_cpu_write(kernel_neon_busy, true); |
| 254 | * haven't done so already. Clear fpsimd_last_state to indicate | 287 | |
| 255 | * that there is no longer userland FPSIMD state in the | 288 | /* Save unsaved task fpsimd state, if any: */ |
| 256 | * registers. | 289 | if (current->mm && !test_and_set_thread_flag(TIF_FOREIGN_FPSTATE)) |
| 257 | */ | 290 | fpsimd_save_state(¤t->thread.fpsimd_state); |
| 258 | preempt_disable(); | 291 | |
| 259 | if (current->mm && | 292 | /* Invalidate any task state remaining in the fpsimd regs: */ |
| 260 | !test_and_set_thread_flag(TIF_FOREIGN_FPSTATE)) | 293 | __this_cpu_write(fpsimd_last_state, NULL); |
| 261 | fpsimd_save_state(¤t->thread.fpsimd_state); | 294 | |
| 262 | this_cpu_write(fpsimd_last_state, NULL); | 295 | preempt_disable(); |
| 263 | } | 296 | |
| 297 | local_bh_enable(); | ||
| 264 | } | 298 | } |
| 265 | EXPORT_SYMBOL(kernel_neon_begin_partial); | 299 | EXPORT_SYMBOL(kernel_neon_begin); |
| 266 | 300 | ||
| 301 | /* | ||
| 302 | * kernel_neon_end(): give the CPU FPSIMD registers back to the current task | ||
| 303 | * | ||
| 304 | * Must be called from a context in which kernel_neon_begin() was previously | ||
| 305 | * called, with no call to kernel_neon_end() in the meantime. | ||
| 306 | * | ||
| 307 | * The caller must not use the FPSIMD registers after this function is called, | ||
| 308 | * unless kernel_neon_begin() is called again in the meantime. | ||
| 309 | */ | ||
| 267 | void kernel_neon_end(void) | 310 | void kernel_neon_end(void) |
| 268 | { | 311 | { |
| 312 | bool busy; | ||
| 313 | |||
| 269 | if (!system_supports_fpsimd()) | 314 | if (!system_supports_fpsimd()) |
| 270 | return; | 315 | return; |
| 271 | if (in_interrupt()) { | 316 | |
| 272 | struct fpsimd_partial_state *s = this_cpu_ptr( | 317 | busy = __this_cpu_xchg(kernel_neon_busy, false); |
| 273 | in_irq() ? &hardirq_fpsimdstate : &softirq_fpsimdstate); | 318 | WARN_ON(!busy); /* No matching kernel_neon_begin()? */ |
| 274 | fpsimd_load_partial_state(s); | 319 | |
| 275 | } else { | 320 | preempt_enable(); |
| 276 | preempt_enable(); | ||
| 277 | } | ||
| 278 | } | 321 | } |
| 279 | EXPORT_SYMBOL(kernel_neon_end); | 322 | EXPORT_SYMBOL(kernel_neon_end); |
| 280 | 323 | ||
| 324 | static DEFINE_PER_CPU(struct fpsimd_state, efi_fpsimd_state); | ||
| 325 | static DEFINE_PER_CPU(bool, efi_fpsimd_state_used); | ||
| 326 | |||
| 327 | /* | ||
| 328 | * EFI runtime services support functions | ||
| 329 | * | ||
| 330 | * The ABI for EFI runtime services allows EFI to use FPSIMD during the call. | ||
| 331 | * This means that for EFI (and only for EFI), we have to assume that FPSIMD | ||
| 332 | * is always used rather than being an optional accelerator. | ||
| 333 | * | ||
| 334 | * These functions provide the necessary support for ensuring FPSIMD | ||
| 335 | * save/restore in the contexts from which EFI is used. | ||
| 336 | * | ||
| 337 | * Do not use them for any other purpose -- if tempted to do so, you are | ||
| 338 | * either doing something wrong or you need to propose some refactoring. | ||
| 339 | */ | ||
| 340 | |||
| 341 | /* | ||
| 342 | * __efi_fpsimd_begin(): prepare FPSIMD for making an EFI runtime services call | ||
| 343 | */ | ||
| 344 | void __efi_fpsimd_begin(void) | ||
| 345 | { | ||
| 346 | if (!system_supports_fpsimd()) | ||
| 347 | return; | ||
| 348 | |||
| 349 | WARN_ON(preemptible()); | ||
| 350 | |||
| 351 | if (may_use_simd()) | ||
| 352 | kernel_neon_begin(); | ||
| 353 | else { | ||
| 354 | fpsimd_save_state(this_cpu_ptr(&efi_fpsimd_state)); | ||
| 355 | __this_cpu_write(efi_fpsimd_state_used, true); | ||
| 356 | } | ||
| 357 | } | ||
| 358 | |||
| 359 | /* | ||
| 360 | * __efi_fpsimd_end(): clean up FPSIMD after an EFI runtime services call | ||
| 361 | */ | ||
| 362 | void __efi_fpsimd_end(void) | ||
| 363 | { | ||
| 364 | if (!system_supports_fpsimd()) | ||
| 365 | return; | ||
| 366 | |||
| 367 | if (__this_cpu_xchg(efi_fpsimd_state_used, false)) | ||
| 368 | fpsimd_load_state(this_cpu_ptr(&efi_fpsimd_state)); | ||
| 369 | else | ||
| 370 | kernel_neon_end(); | ||
| 371 | } | ||
| 372 | |||
| 281 | #endif /* CONFIG_KERNEL_MODE_NEON */ | 373 | #endif /* CONFIG_KERNEL_MODE_NEON */ |
| 282 | 374 | ||
| 283 | #ifdef CONFIG_CPU_PM | 375 | #ifdef CONFIG_CPU_PM |
| @@ -143,8 +143,8 @@ preserve_boot_args: · 142 unmodified lines | |||
| 143 | dmb sy // needed before dc ivac with | 143 | dmb sy // needed before dc ivac with |
| 144 | // MMU off | 144 | // MMU off |
| 145 | 145 | ||
| 146 | add x1, x0, #0x20 // 4 x 8 bytes | 146 | mov x1, #0x20 // 4 x 8 bytes |
| 147 | b __inval_cache_range // tail call | 147 | b __inval_dcache_area // tail call |
| 148 | ENDPROC(preserve_boot_args) | 148 | ENDPROC(preserve_boot_args) |
| 149 | 149 | ||
| 150 | /* | 150 | /* |
| @@ -221,20 +221,20 @@ __create_page_tables: · 70 unmodified lines | |||
| 221 | * dirty cache lines being evicted. | 221 | * dirty cache lines being evicted. |
| 222 | */ | 222 | */ |
| 223 | adrp x0, idmap_pg_dir | 223 | adrp x0, idmap_pg_dir |
| 224 | adrp x1, swapper_pg_dir + SWAPPER_DIR_SIZE + RESERVED_TTBR0_SIZE | 224 | ldr x1, =(IDMAP_DIR_SIZE + SWAPPER_DIR_SIZE + RESERVED_TTBR0_SIZE) |
| 225 | bl __inval_cache_range | 225 | bl __inval_dcache_area |
| 226 | 226 | ||
| 227 | /* | 227 | /* |
| 228 | * Clear the idmap and swapper page tables. | 228 | * Clear the idmap and swapper page tables. |
| 229 | */ | 229 | */ |
| 230 | adrp x0, idmap_pg_dir | 230 | adrp x0, idmap_pg_dir |
| 231 | adrp x6, swapper_pg_dir + SWAPPER_DIR_SIZE + RESERVED_TTBR0_SIZE | 231 | ldr x1, =(IDMAP_DIR_SIZE + SWAPPER_DIR_SIZE + RESERVED_TTBR0_SIZE) |
| 232 | 1: stp xzr, xzr, [x0], #16 | 232 | 1: stp xzr, xzr, [x0], #16 |
| 233 | stp xzr, xzr, [x0], #16 | 233 | stp xzr, xzr, [x0], #16 |
| 234 | stp xzr, xzr, [x0], #16 | 234 | stp xzr, xzr, [x0], #16 |
| 235 | stp xzr, xzr, [x0], #16 | 235 | stp xzr, xzr, [x0], #16 |
| 236 | cmp x0, x6 | 236 | subs x1, x1, #64 |
| 237 | b.lo 1b | 237 | b.ne 1b |
| 238 | 238 | ||
| 239 | mov x7, SWAPPER_MM_MMUFLAGS | 239 | mov x7, SWAPPER_MM_MMUFLAGS |
| 240 | 240 | ||
| @@ -307,9 +307,9 @@ __create_page_tables: · 66 unmodified lines | |||
| 307 | * tables again to remove any speculatively loaded cache lines. | 307 | * tables again to remove any speculatively loaded cache lines. |
| 308 | */ | 308 | */ |
| 309 | adrp x0, idmap_pg_dir | 309 | adrp x0, idmap_pg_dir |
| 310 | adrp x1, swapper_pg_dir + SWAPPER_DIR_SIZE + RESERVED_TTBR0_SIZE | 310 | ldr x1, =(IDMAP_DIR_SIZE + SWAPPER_DIR_SIZE + RESERVED_TTBR0_SIZE) |
| 311 | dmb sy | 311 | dmb sy |
| 312 | bl __inval_cache_range | 312 | bl __inval_dcache_area |
| 313 | 313 | ||
| 314 | ret x28 | 314 | ret x28 |
| 315 | ENDPROC(__create_page_tables) | 315 | ENDPROC(__create_page_tables) |
| @@ -361,6 +361,9 @@ __primary_switched: · 45 unmodified lines | |||
| 361 | ret // to __primary_switch() | 361 | ret // to __primary_switch() |
| 362 | 0: | 362 | 0: |
| 363 | #endif | 363 | #endif |
| 364 | add sp, sp, #16 | ||
| 365 | mov x29, #0 | ||
| 366 | mov x30, #0 | ||
| 364 | b start_kernel | 367 | b start_kernel |
| 365 | ENDPROC(__primary_switched) | 368 | ENDPROC(__primary_switched) |
| 366 | 369 | ||
| @@ -616,6 +619,7 @@ __secondary_switched: · 249 unmodified lines | |||
| 616 | ldr x2, [x0, #CPU_BOOT_TASK] | 619 | ldr x2, [x0, #CPU_BOOT_TASK] |
| 617 | msr sp_el0, x2 | 620 | msr sp_el0, x2 |
| 618 | mov x29, #0 | 621 | mov x29, #0 |
| 622 | mov x30, #0 | ||
| 619 | b secondary_start_kernel | 623 | b secondary_start_kernel |
| 620 | ENDPROC(__secondary_switched) | 624 | ENDPROC(__secondary_switched) |
| 621 | 625 | ||
| @@ -330,7 +330,7 @@ static void _copy_pte(pte_t *dst_pte, pte_t *src_pte, unsigned long addr) · 329 unmodified lines | |||
| 330 | * read only (code, rodata). Clear the RDONLY bit from | 330 | * read only (code, rodata). Clear the RDONLY bit from |
| 331 | * the temporary mappings we use during restore. | 331 | * the temporary mappings we use during restore. |
| 332 | */ | 332 | */ |
| 333 | set_pte(dst_pte, pte_clear_rdonly(pte)); | 333 | set_pte(dst_pte, pte_mkwrite(pte)); |
| 334 | } else if (debug_pagealloc_enabled() && !pte_none(pte)) { | 334 | } else if (debug_pagealloc_enabled() && !pte_none(pte)) { |
| 335 | /* | 335 | /* |
| 336 | * debug_pagealloc will removed the PTE_VALID bit if | 336 | * debug_pagealloc will removed the PTE_VALID bit if |
| @@ -343,7 +343,7 @@ static void _copy_pte(pte_t *dst_pte, pte_t *src_pte, unsigned long addr) · 6 unmodified lines | |||
| 343 | */ | 343 | */ |
| 344 | BUG_ON(!pfn_valid(pte_pfn(pte))); | 344 | BUG_ON(!pfn_valid(pte_pfn(pte))); |
| 345 | 345 | ||
| 346 | set_pte(dst_pte, pte_mkpresent(pte_clear_rdonly(pte))); | 346 | set_pte(dst_pte, pte_mkpresent(pte_mkwrite(pte))); |
| 347 | } | 347 | } |
| 348 | } | 348 | } |
| 349 | 349 | ||
| @@ -23,15 +23,16 @@ · 22 unmodified lines | |||
| 23 | 23 | ||
| 24 | #include <linux/kernel_stat.h> | 24 | #include <linux/kernel_stat.h> |
| 25 | #include <linux/irq.h> | 25 | #include <linux/irq.h> |
| 26 | #include <linux/memory.h> | ||
| 26 | #include <linux/smp.h> | 27 | #include <linux/smp.h> |
| 27 | #include <linux/init.h> | 28 | #include <linux/init.h> |
| 28 | #include <linux/irqchip.h> | 29 | #include <linux/irqchip.h> |
| 29 | #include <linux/seq_file.h> | 30 | #include <linux/seq_file.h> |
| 31 | #include <linux/vmalloc.h> | ||
| 30 | 32 | ||
| 31 | unsigned long irq_err_count; | 33 | unsigned long irq_err_count; |
| 32 | 34 | ||
| 33 | /* irq stack only needs to be 16 byte aligned - not IRQ_STACK_SIZE aligned. */ | 35 | DEFINE_PER_CPU(unsigned long *, irq_stack_ptr); |
| 34 | DEFINE_PER_CPU(unsigned long [IRQ_STACK_SIZE/sizeof(long)], irq_stack) __aligned(16); | ||
| 35 | 36 | ||
| 36 | int arch_show_interrupts(struct seq_file *p, int prec) | 37 | int arch_show_interrupts(struct seq_file *p, int prec) |
| 37 | { | 38 | { |
| @@ -50,8 +51,43 @@ void __init set_handle_irq(void (*handle_irq)(struct pt_regs *)) · 12 unmodified lines | |||
| 50 | handle_arch_irq = handle_irq; | 51 | handle_arch_irq = handle_irq; |
| 51 | } | 52 | } |
| 52 | 53 | ||
| 54 | #ifdef CONFIG_VMAP_STACK | ||
| 55 | static void init_irq_stacks(void) | ||
| 56 | { | ||
| 57 | int cpu; | ||
| 58 | unsigned long *p; | ||
| 59 | |||
| 60 | for_each_possible_cpu(cpu) { | ||
| 61 | /* | ||
| 62 | * To ensure that VMAP'd stack overflow detection works | ||
| 63 | * correctly, the IRQ stacks need to have the same | ||
| 64 | * alignment as other stacks. | ||
| 65 | */ | ||
| 66 | p = __vmalloc_node_range(IRQ_STACK_SIZE, THREAD_ALIGN, | ||
| 67 | VMALLOC_START, VMALLOC_END, | ||
| 68 | THREADINFO_GFP, PAGE_KERNEL, | ||
| 69 | 0, cpu_to_node(cpu), | ||
| 70 | __builtin_return_address(0)); | ||
| 71 | |||
| 72 | per_cpu(irq_stack_ptr, cpu) = p; | ||
| 73 | } | ||
| 74 | } | ||
| 75 | #else | ||
| 76 | /* irq stack only needs to be 16 byte aligned - not IRQ_STACK_SIZE aligned. */ | ||
| 77 | DEFINE_PER_CPU_ALIGNED(unsigned long [IRQ_STACK_SIZE/sizeof(long)], irq_stack); | ||
| 78 | |||
| 79 | static void init_irq_stacks(void) | ||
| 80 | { | ||
| 81 | int cpu; | ||
| 82 | |||
| 83 | for_each_possible_cpu(cpu) | ||
| 84 | per_cpu(irq_stack_ptr, cpu) = per_cpu(irq_stack, cpu); | ||
| 85 | } | ||
| 86 | #endif | ||
| 87 | |||
| 53 | void __init init_IRQ(void) | 88 | void __init init_IRQ(void) |
| 54 | { | 89 | { |
| 90 | init_irq_stacks(); | ||
| 55 | irqchip_init(); | 91 | irqchip_init(); |
| 56 | if (!handle_arch_irq) | 92 | if (!handle_arch_irq) |
| 57 | panic("No interrupt controller found."); | 93 | panic("No interrupt controller found."); |
| @@ -252,7 +252,7 @@ void machine_crash_shutdown(struct pt_regs *regs) · 251 unmodified lines | |||
| 252 | local_irq_disable(); | 252 | local_irq_disable(); |
| 253 | 253 | ||
| 254 | /* shutdown non-crashing cpus */ | 254 | /* shutdown non-crashing cpus */ |
| 255 | smp_send_crash_stop(); | 255 | crash_smp_send_stop(); |
| 256 | 256 | ||
| 257 | /* for crashing cpu */ | 257 | /* for crashing cpu */ |
| 258 | crash_save_cpu(regs, smp_processor_id()); | 258 | crash_save_cpu(regs, smp_processor_id()); |
| @@ -162,7 +162,6 @@ void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, · 161 unmodified lines | |||
| 162 | } | 162 | } |
| 163 | 163 | ||
| 164 | frame.fp = regs->regs[29]; | 164 | frame.fp = regs->regs[29]; |
| 165 | frame.sp = regs->sp; | ||
| 166 | frame.pc = regs->pc; | 165 | frame.pc = regs->pc; |
| 167 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER | 166 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER |
| 168 | frame.graph = current->curr_ret_stack; | 167 | frame.graph = current->curr_ret_stack; |
| @@ -202,55 +202,6 @@ static const unsigned armv8_pmuv3_perf_map[PERF_COUNT_HW_MAX] = { · 201 unmodified lines | |||
| 202 | [PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = ARMV8_PMUV3_PERFCTR_STALL_BACKEND, | 202 | [PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = ARMV8_PMUV3_PERFCTR_STALL_BACKEND, |
| 203 | }; | 203 | }; |
| 204 | 204 | ||
| 205 | /* ARM Cortex-A53 HW events mapping. */ | ||
| 206 | static const unsigned armv8_a53_perf_map[PERF_COUNT_HW_MAX] = { | ||
| 207 | PERF_MAP_ALL_UNSUPPORTED, | ||
| 208 | [PERF_COUNT_HW_CPU_CYCLES] = ARMV8_PMUV3_PERFCTR_CPU_CYCLES, | ||
| 209 | [PERF_COUNT_HW_INSTRUCTIONS] = ARMV8_PMUV3_PERFCTR_INST_RETIRED, | ||
| 210 | [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV8_PMUV3_PERFCTR_L1D_CACHE, | ||
| 211 | [PERF_COUNT_HW_CACHE_MISSES] = ARMV8_PMUV3_PERFCTR_L1D_CACHE_REFILL, | ||
| 212 | [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV8_PMUV3_PERFCTR_PC_WRITE_RETIRED, | ||
| 213 | [PERF_COUNT_HW_BRANCH_MISSES] = ARMV8_PMUV3_PERFCTR_BR_MIS_PRED, | ||
| 214 | [PERF_COUNT_HW_BUS_CYCLES] = ARMV8_PMUV3_PERFCTR_BUS_CYCLES, | ||
| 215 | }; | ||
| 216 | |||
| 217 | /* ARM Cortex-A57 and Cortex-A72 events mapping. */ | ||
| 218 | static const unsigned armv8_a57_perf_map[PERF_COUNT_HW_MAX] = { | ||
| 219 | PERF_MAP_ALL_UNSUPPORTED, | ||
| 220 | [PERF_COUNT_HW_CPU_CYCLES] = ARMV8_PMUV3_PERFCTR_CPU_CYCLES, | ||
| 221 | [PERF_COUNT_HW_INSTRUCTIONS] = ARMV8_PMUV3_PERFCTR_INST_RETIRED, | ||
| 222 | [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV8_PMUV3_PERFCTR_L1D_CACHE, | ||
| 223 | [PERF_COUNT_HW_CACHE_MISSES] = ARMV8_PMUV3_PERFCTR_L1D_CACHE_REFILL, | ||
| 224 | [PERF_COUNT_HW_BRANCH_MISSES] = ARMV8_PMUV3_PERFCTR_BR_MIS_PRED, | ||
| 225 | [PERF_COUNT_HW_BUS_CYCLES] = ARMV8_PMUV3_PERFCTR_BUS_CYCLES, | ||
| 226 | }; | ||
| 227 | |||
| 228 | static const unsigned armv8_thunder_perf_map[PERF_COUNT_HW_MAX] = { | ||
| 229 | PERF_MAP_ALL_UNSUPPORTED, | ||
| 230 | [PERF_COUNT_HW_CPU_CYCLES] = ARMV8_PMUV3_PERFCTR_CPU_CYCLES, | ||
| 231 | [PERF_COUNT_HW_INSTRUCTIONS] = ARMV8_PMUV3_PERFCTR_INST_RETIRED, | ||
| 232 | [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV8_PMUV3_PERFCTR_L1D_CACHE, | ||
| 233 | [PERF_COUNT_HW_CACHE_MISSES] = ARMV8_PMUV3_PERFCTR_L1D_CACHE_REFILL, | ||
| 234 | [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV8_PMUV3_PERFCTR_PC_WRITE_RETIRED, | ||
| 235 | [PERF_COUNT_HW_BRANCH_MISSES] = ARMV8_PMUV3_PERFCTR_BR_MIS_PRED, | ||
| 236 | [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = ARMV8_PMUV3_PERFCTR_STALL_FRONTEND, | ||
| 237 | [PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = ARMV8_PMUV3_PERFCTR_STALL_BACKEND, | ||
| 238 | }; | ||
| 239 | |||
| 240 | /* Broadcom Vulcan events mapping */ | ||
| 241 | static const unsigned armv8_vulcan_perf_map[PERF_COUNT_HW_MAX] = { | ||
| 242 | PERF_MAP_ALL_UNSUPPORTED, | ||
| 243 | [PERF_COUNT_HW_CPU_CYCLES] = ARMV8_PMUV3_PERFCTR_CPU_CYCLES, | ||
| 244 | [PERF_COUNT_HW_INSTRUCTIONS] = ARMV8_PMUV3_PERFCTR_INST_RETIRED, | ||
| 245 | [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV8_PMUV3_PERFCTR_L1D_CACHE, | ||
| 246 | [PERF_COUNT_HW_CACHE_MISSES] = ARMV8_PMUV3_PERFCTR_L1D_CACHE_REFILL, | ||
| 247 | [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV8_PMUV3_PERFCTR_BR_RETIRED, | ||
| 248 | [PERF_COUNT_HW_BRANCH_MISSES] = ARMV8_PMUV3_PERFCTR_BR_MIS_PRED, | ||
| 249 | [PERF_COUNT_HW_BUS_CYCLES] = ARMV8_PMUV3_PERFCTR_BUS_CYCLES, | ||
| 250 | [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = ARMV8_PMUV3_PERFCTR_STALL_FRONTEND, | ||
| 251 | [PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = ARMV8_PMUV3_PERFCTR_STALL_BACKEND, | ||
| 252 | }; | ||
| 253 | |||
| 254 | static const unsigned armv8_pmuv3_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] | 205 | static const unsigned armv8_pmuv3_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] |
| 255 | [PERF_COUNT_HW_CACHE_OP_MAX] | 206 | [PERF_COUNT_HW_CACHE_OP_MAX] |
| 256 | [PERF_COUNT_HW_CACHE_RESULT_MAX] = { | 207 | [PERF_COUNT_HW_CACHE_RESULT_MAX] = { |
| @@ -281,27 +232,10 @@ static const unsigned armv8_a53_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] · 24 unmodified lines | |||
| 281 | [PERF_COUNT_HW_CACHE_RESULT_MAX] = { | 232 | [PERF_COUNT_HW_CACHE_RESULT_MAX] = { |
| 282 | PERF_CACHE_MAP_ALL_UNSUPPORTED, | 233 | PERF_CACHE_MAP_ALL_UNSUPPORTED, |
| 283 | 234 | ||
| 284 | [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_L1D_CACHE, | ||
| 285 | [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L1D_CACHE_REFILL, | ||
| 286 | [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_L1D_CACHE, | ||
| 287 | [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L1D_CACHE_REFILL, | ||
| 288 | [C(L1D)][C(OP_PREFETCH)][C(RESULT_MISS)] = ARMV8_A53_PERFCTR_PREF_LINEFILL, | 235 | [C(L1D)][C(OP_PREFETCH)][C(RESULT_MISS)] = ARMV8_A53_PERFCTR_PREF_LINEFILL, |
| 289 | 236 | ||
| 290 | [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_L1I_CACHE, | 237 | [C(NODE)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_RD, |
| 291 | [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L1I_CACHE_REFILL, | 238 | [C(NODE)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_WR, |
| 292 | |||
| 293 | [C(LL)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_L2D_CACHE, | ||
| 294 | [C(LL)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L2D_CACHE_REFILL, | ||
| 295 | [C(LL)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_L2D_CACHE, | ||
| 296 | [C(LL)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L2D_CACHE_REFILL, | ||
| 297 | |||
| 298 | [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L1D_TLB_REFILL, | ||
| 299 | [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L1I_TLB_REFILL, | ||
| 300 | |||
| 301 | [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_BR_PRED, | ||
| 302 | [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_BR_MIS_PRED, | ||
| 303 | [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_BR_PRED, | ||
| 304 | [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_BR_MIS_PRED, | ||
| 305 | }; | 239 | }; |
| 306 | 240 | ||
| 307 | static const unsigned armv8_a57_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] | 241 | static const unsigned armv8_a57_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] |
| @@ -314,18 +248,26 @@ static const unsigned armv8_a57_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] · 6 unmodified lines | |||
| 314 | [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_L1D_CACHE_WR, | 248 | [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_L1D_CACHE_WR, |
| 315 | [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV8_IMPDEF_PERFCTR_L1D_CACHE_REFILL_WR, | 249 | [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV8_IMPDEF_PERFCTR_L1D_CACHE_REFILL_WR, |
| 316 | 250 | ||
| 317 | [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_L1I_CACHE, | ||
| 318 | [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L1I_CACHE_REFILL, | ||
| 319 | |||
| 320 | [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_REFILL_RD, | 251 | [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_REFILL_RD, |
| 321 | [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_REFILL_WR, | 252 | [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_REFILL_WR, |
| 322 | 253 | ||
| 323 | [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L1I_TLB_REFILL, | 254 | [C(NODE)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_RD, |
| 255 | [C(NODE)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_WR, | ||
| 256 | }; | ||
| 324 | 257 | ||
| 325 | [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_BR_PRED, | 258 | static const unsigned armv8_a73_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] |
| 326 | [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_BR_MIS_PRED, | 259 | [PERF_COUNT_HW_CACHE_OP_MAX] |
| 327 | [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_BR_PRED, | 260 | [PERF_COUNT_HW_CACHE_RESULT_MAX] = { |
| 328 | [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_BR_MIS_PRED, | 261 | PERF_CACHE_MAP_ALL_UNSUPPORTED, |
| 262 | |||
| 263 | [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_L1D_CACHE_RD, | ||
| 264 | [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_L1D_CACHE_WR, | ||
| 265 | |||
| 266 | [C(NODE)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_RD, | ||
| 267 | [C(NODE)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_WR, | ||
| 268 | |||
| 269 | [C(NODE)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_RD, | ||
| 270 | [C(NODE)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_WR, | ||
| 329 | }; | 271 | }; |
| 330 | 272 | ||
| 331 | static const unsigned armv8_thunder_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] | 273 | static const unsigned armv8_thunder_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] |
| @@ -340,8 +282,6 @@ static const unsigned armv8_thunder_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] · 8 unmodified lines | |||
| 340 | [C(L1D)][C(OP_PREFETCH)][C(RESULT_ACCESS)] = ARMV8_THUNDER_PERFCTR_L1D_CACHE_PREF_ACCESS, | 282 | [C(L1D)][C(OP_PREFETCH)][C(RESULT_ACCESS)] = ARMV8_THUNDER_PERFCTR_L1D_CACHE_PREF_ACCESS, |
| 341 | [C(L1D)][C(OP_PREFETCH)][C(RESULT_MISS)] = ARMV8_THUNDER_PERFCTR_L1D_CACHE_PREF_MISS, | 283 | [C(L1D)][C(OP_PREFETCH)][C(RESULT_MISS)] = ARMV8_THUNDER_PERFCTR_L1D_CACHE_PREF_MISS, |
| 342 | 284 | ||
| 343 | [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_L1I_CACHE, | ||
| 344 | [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L1I_CACHE_REFILL, | ||
| 345 | [C(L1I)][C(OP_PREFETCH)][C(RESULT_ACCESS)] = ARMV8_THUNDER_PERFCTR_L1I_CACHE_PREF_ACCESS, | 285 | [C(L1I)][C(OP_PREFETCH)][C(RESULT_ACCESS)] = ARMV8_THUNDER_PERFCTR_L1I_CACHE_PREF_ACCESS, |
| 346 | [C(L1I)][C(OP_PREFETCH)][C(RESULT_MISS)] = ARMV8_THUNDER_PERFCTR_L1I_CACHE_PREF_MISS, | 286 | [C(L1I)][C(OP_PREFETCH)][C(RESULT_MISS)] = ARMV8_THUNDER_PERFCTR_L1I_CACHE_PREF_MISS, |
| 347 | 287 | ||
| @@ -349,13 +289,6 @@ static const unsigned armv8_thunder_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] · 1 unmodified line | |||
| 349 | [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_REFILL_RD, | 289 | [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_REFILL_RD, |
| 350 | [C(DTLB)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_WR, | 290 | [C(DTLB)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_WR, |
| 351 | [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_REFILL_WR, | 291 | [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_REFILL_WR, |
| 352 | |||
| 353 | [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L1I_TLB_REFILL, | ||
| 354 | |||
| 355 | [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_BR_PRED, | ||
| 356 | [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_BR_MIS_PRED, | ||
| 357 | [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_BR_PRED, | ||
| 358 | [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_BR_MIS_PRED, | ||
| 359 | }; | 292 | }; |
| 360 | 293 | ||
| 361 | static const unsigned armv8_vulcan_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] | 294 | static const unsigned armv8_vulcan_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] |
| @@ -368,22 +301,11 @@ static const unsigned armv8_vulcan_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] · 6 unmodified lines | |||
| 368 | [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_L1D_CACHE_WR, | 301 | [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_L1D_CACHE_WR, |
| 369 | [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV8_IMPDEF_PERFCTR_L1D_CACHE_REFILL_WR, | 302 | [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV8_IMPDEF_PERFCTR_L1D_CACHE_REFILL_WR, |
| 370 | 303 | ||
| 371 | [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_L1I_CACHE, | ||
| 372 | [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L1I_CACHE_REFILL, | ||
| 373 | |||
| 374 | [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L1I_TLB_REFILL, | ||
| 375 | [C(ITLB)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_L1I_TLB, | ||
| 376 | |||
| 377 | [C(DTLB)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_RD, | 304 | [C(DTLB)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_RD, |
| 378 | [C(DTLB)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_WR, | 305 | [C(DTLB)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_WR, |
| 379 | [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_REFILL_RD, | 306 | [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_REFILL_RD, |
| 380 | [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_REFILL_WR, | 307 | [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_REFILL_WR, |
| 381 | 308 | ||
| 382 | [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_BR_PRED, | ||
| 383 | [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_BR_MIS_PRED, | ||
| 384 | [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_BR_PRED, | ||
| 385 | [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_BR_MIS_PRED, | ||
| 386 | |||
| 387 | [C(NODE)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_RD, | 309 | [C(NODE)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_RD, |
| 388 | [C(NODE)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_WR, | 310 | [C(NODE)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_WR, |
| 389 | }; | 311 | }; |
| @@ -846,17 +768,14 @@ static int armv8pmu_get_event_idx(struct pmu_hw_events *cpuc, · 456 unmodified lines | |||
| 846 | struct hw_perf_event *hwc = &event->hw; | 768 | struct hw_perf_event *hwc = &event->hw; |
| 847 | unsigned long evtype = hwc->config_base & ARMV8_PMU_EVTYPE_EVENT; | 769 | unsigned long evtype = hwc->config_base & ARMV8_PMU_EVTYPE_EVENT; |
| 848 | 770 | ||
| 849 | /* Always place a cycle counter into the cycle counter. */ | 771 | /* Always prefer to place a cycle counter into the cycle counter. */ |
| 850 | if (evtype == ARMV8_PMUV3_PERFCTR_CPU_CYCLES) { | 772 | if (evtype == ARMV8_PMUV3_PERFCTR_CPU_CYCLES) { |
| 851 | if (test_and_set_bit(ARMV8_IDX_CYCLE_COUNTER, cpuc->used_mask)) | 773 | if (!test_and_set_bit(ARMV8_IDX_CYCLE_COUNTER, cpuc->used_mask)) |
| 852 | return -EAGAIN; | 774 | return ARMV8_IDX_CYCLE_COUNTER; |
| 853 | |||
| 854 | return ARMV8_IDX_CYCLE_COUNTER; | ||
| 855 | } | 775 | } |
| 856 | 776 | ||
| 857 | /* | 777 | /* |
| 858 | * For anything other than a cycle counter, try and use | 778 | * Otherwise use events counters |
| 859 | * the events counters | ||
| 860 | */ | 779 | */ |
| 861 | for (idx = ARMV8_IDX_COUNTER0; idx < cpu_pmu->num_events; ++idx) { | 780 | for (idx = ARMV8_IDX_COUNTER0; idx < cpu_pmu->num_events; ++idx) { |
| 862 | if (!test_and_set_bit(idx, cpuc->used_mask)) | 781 | if (!test_and_set_bit(idx, cpuc->used_mask)) |
| @@ -924,7 +843,13 @@ static void armv8pmu_reset(void *info) · 61 unmodified lines | |||
| 924 | ARMV8_PMU_PMCR_LC); | 843 | ARMV8_PMU_PMCR_LC); |
| 925 | } | 844 | } |
| 926 | 845 | ||
| 927 | static int armv8_pmuv3_map_event(struct perf_event *event) | 846 | static int __armv8_pmuv3_map_event(struct perf_event *event, |
| 847 | const unsigned (*extra_event_map) | ||
| 848 | [PERF_COUNT_HW_MAX], | ||
| 849 | const unsigned (*extra_cache_map) | ||
| 850 | [PERF_COUNT_HW_CACHE_MAX] | ||
| 851 | [PERF_COUNT_HW_CACHE_OP_MAX] | ||
| 852 | [PERF_COUNT_HW_CACHE_RESULT_MAX]) | ||
| 928 | { | 853 | { |
| 929 | int hw_event_id; | 854 | int hw_event_id; |
| 930 | struct arm_pmu *armpmu = to_arm_pmu(event->pmu); | 855 | struct arm_pmu *armpmu = to_arm_pmu(event->pmu); |
| @@ -932,44 +857,47 @@ static int armv8_pmuv3_map_event(struct perf_event *event) · 1 unmodified line | |||
| 932 | hw_event_id = armpmu_map_event(event, &armv8_pmuv3_perf_map, | 857 | hw_event_id = armpmu_map_event(event, &armv8_pmuv3_perf_map, |
| 933 | &armv8_pmuv3_perf_cache_map, | 858 | &armv8_pmuv3_perf_cache_map, |
| 934 | ARMV8_PMU_EVTYPE_EVENT); | 859 | ARMV8_PMU_EVTYPE_EVENT); |
| 935 | if (hw_event_id < 0) | ||
| 936 | return hw_event_id; | ||
| 937 | 860 | ||
| 938 | /* disable micro/arch events not supported by this PMU */ | 861 | /* Onl expose micro/arch events supported by this PMU */ |
| 939 | if ((hw_event_id < ARMV8_PMUV3_MAX_COMMON_EVENTS) && | 862 | if ((hw_event_id > 0) && (hw_event_id < ARMV8_PMUV3_MAX_COMMON_EVENTS) |
| 940 | !test_bit(hw_event_id, armpmu->pmceid_bitmap)) { | 863 | && test_bit(hw_event_id, armpmu->pmceid_bitmap)) { |
| 941 | return -EOPNOTSUPP; | 864 | return hw_event_id; |
| 942 | } | 865 | } |
| 943 | 866 | ||
| 944 | return hw_event_id; | 867 | return armpmu_map_event(event, extra_event_map, extra_cache_map, |
| 868 | ARMV8_PMU_EVTYPE_EVENT); | ||
| 869 | } | ||
| 870 | |||
| 871 | static int armv8_pmuv3_map_event(struct perf_event *event) | ||
| 872 | { | ||
| 873 | return __armv8_pmuv3_map_event(event, NULL, NULL); | ||
| 945 | } | 874 | } |
| 946 | 875 | ||
| 947 | static int armv8_a53_map_event(struct perf_event *event) | 876 | static int armv8_a53_map_event(struct perf_event *event) |
| 948 | { | 877 | { |
| 949 | return armpmu_map_event(event, &armv8_a53_perf_map, | 878 | return __armv8_pmuv3_map_event(event, NULL, &armv8_a53_perf_cache_map); |
| 950 | &armv8_a53_perf_cache_map, | ||
| 951 | ARMV8_PMU_EVTYPE_EVENT); | ||
| 952 | } | 879 | } |
| 953 | 880 | ||
| 954 | static int armv8_a57_map_event(struct perf_event *event) | 881 | static int armv8_a57_map_event(struct perf_event *event) |
| 955 | { | 882 | { |
| 956 | return armpmu_map_event(event, &armv8_a57_perf_map, | 883 | return __armv8_pmuv3_map_event(event, NULL, &armv8_a57_perf_cache_map); |
| 957 | &armv8_a57_perf_cache_map, | 884 | } |
| 958 | ARMV8_PMU_EVTYPE_EVENT); | 885 | |
| 886 | static int armv8_a73_map_event(struct perf_event *event) | ||
| 887 | { | ||
| 888 | return __armv8_pmuv3_map_event(event, NULL, &armv8_a73_perf_cache_map); | ||
| 959 | } | 889 | } |
| 960 | 890 | ||
| 961 | static int armv8_thunder_map_event(struct perf_event *event) | 891 | static int armv8_thunder_map_event(struct perf_event *event) |
| 962 | { | 892 | { |
| 963 | return armpmu_map_event(event, &armv8_thunder_perf_map, | 893 | return __armv8_pmuv3_map_event(event, NULL, |
| 964 | &armv8_thunder_perf_cache_map, | 894 | &armv8_thunder_perf_cache_map); |
| 965 | ARMV8_PMU_EVTYPE_EVENT); | ||
| 966 | } | 895 | } |
| 967 | 896 | ||
| 968 | static int armv8_vulcan_map_event(struct perf_event *event) | 897 | static int armv8_vulcan_map_event(struct perf_event *event) |
| 969 | { | 898 | { |
| 970 | return armpmu_map_event(event, &armv8_vulcan_perf_map, | 899 | return __armv8_pmuv3_map_event(event, NULL, |
| 971 | &armv8_vulcan_perf_cache_map, | 900 | &armv8_vulcan_perf_cache_map); |
| 972 | ARMV8_PMU_EVTYPE_EVENT); | ||
| 973 | } | 901 | } |
| 974 | 902 | ||
| 975 | struct armv8pmu_probe_info { | 903 | struct armv8pmu_probe_info { |
| @@ -1062,6 +990,22 @@ static int armv8_pmuv3_init(struct arm_pmu *cpu_pmu) · 86 unmodified lines | |||
| 1062 | return 0; | 990 | return 0; |
| 1063 | } | 991 | } |
| 1064 | 992 | ||
| 993 | static int armv8_a35_pmu_init(struct arm_pmu *cpu_pmu) | ||
| 994 | { | ||
| 995 | int ret = armv8_pmu_init(cpu_pmu); | ||
| 996 | if (ret) | ||
| 997 | return ret; | ||
| 998 | |||
| 999 | cpu_pmu->name = "armv8_cortex_a35"; | ||
| 1000 | cpu_pmu->map_event = armv8_a53_map_event; | ||
| 1001 | cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] = | ||
| 1002 | &armv8_pmuv3_events_attr_group; | ||
| 1003 | cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] = | ||
| 1004 | &armv8_pmuv3_format_attr_group; | ||
| 1005 | |||
| 1006 | return 0; | ||
| 1007 | } | ||
| 1008 | |||
| 1065 | static int armv8_a53_pmu_init(struct arm_pmu *cpu_pmu) | 1009 | static int armv8_a53_pmu_init(struct arm_pmu *cpu_pmu) |
| 1066 | { | 1010 | { |
| 1067 | int ret = armv8_pmu_init(cpu_pmu); | 1011 | int ret = armv8_pmu_init(cpu_pmu); |
| @@ -1110,6 +1054,22 @@ static int armv8_a72_pmu_init(struct arm_pmu *cpu_pmu) · 42 unmodified lines | |||
| 1110 | return 0; | 1054 | return 0; |
| 1111 | } | 1055 | } |
| 1112 | 1056 | ||
| 1057 | static int armv8_a73_pmu_init(struct arm_pmu *cpu_pmu) | ||
| 1058 | { | ||
| 1059 | int ret = armv8_pmu_init(cpu_pmu); | ||
| 1060 | if (ret) | ||
| 1061 | return ret; | ||
| 1062 | |||
| 1063 | cpu_pmu->name = "armv8_cortex_a73"; | ||
| 1064 | cpu_pmu->map_event = armv8_a73_map_event; | ||
| 1065 | cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] = | ||
| 1066 | &armv8_pmuv3_events_attr_group; | ||
| 1067 | cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] = | ||
| 1068 | &armv8_pmuv3_format_attr_group; | ||
| 1069 | |||
| 1070 | return 0; | ||
| 1071 | } | ||
| 1072 | |||
| 1113 | static int armv8_thunder_pmu_init(struct arm_pmu *cpu_pmu) | 1073 | static int armv8_thunder_pmu_init(struct arm_pmu *cpu_pmu) |
| 1114 | { | 1074 | { |
| 1115 | int ret = armv8_pmu_init(cpu_pmu); | 1075 | int ret = armv8_pmu_init(cpu_pmu); |
| @@ -1144,9 +1104,11 @@ static int armv8_vulcan_pmu_init(struct arm_pmu *cpu_pmu) · 28 unmodified lines | |||
| 1144 | 1104 | ||
| 1145 | static const struct of_device_id armv8_pmu_of_device_ids[] = { | 1105 | static const struct of_device_id armv8_pmu_of_device_ids[] = { |
| 1146 | {.compatible = "arm,armv8-pmuv3", .data = armv8_pmuv3_init}, | 1106 | {.compatible = "arm,armv8-pmuv3", .data = armv8_pmuv3_init}, |
| 1107 | {.compatible = "arm,cortex-a35-pmu", .data = armv8_a35_pmu_init}, | ||
| 1147 | {.compatible = "arm,cortex-a53-pmu", .data = armv8_a53_pmu_init}, | 1108 | {.compatible = "arm,cortex-a53-pmu", .data = armv8_a53_pmu_init}, |
| 1148 | {.compatible = "arm,cortex-a57-pmu", .data = armv8_a57_pmu_init}, | 1109 | {.compatible = "arm,cortex-a57-pmu", .data = armv8_a57_pmu_init}, |
| 1149 | {.compatible = "arm,cortex-a72-pmu", .data = armv8_a72_pmu_init}, | 1110 | {.compatible = "arm,cortex-a72-pmu", .data = armv8_a72_pmu_init}, |
| 1111 | {.compatible = "arm,cortex-a73-pmu", .data = armv8_a73_pmu_init}, | ||
| 1150 | {.compatible = "cavium,thunder-pmu", .data = armv8_thunder_pmu_init}, | 1112 | {.compatible = "cavium,thunder-pmu", .data = armv8_thunder_pmu_init}, |
| 1151 | {.compatible = "brcm,vulcan-pmu", .data = armv8_vulcan_pmu_init}, | 1113 | {.compatible = "brcm,vulcan-pmu", .data = armv8_vulcan_pmu_init}, |
| 1152 | {}, | 1114 | {}, |
| @@ -40,7 +40,7 @@ int arch_uprobe_analyze_insn(struct arch_uprobe *auprobe, struct mm_struct *mm, · 39 unmodified lines | |||
| 40 | probe_opcode_t insn; | 40 | probe_opcode_t insn; |
| 41 | 41 | ||
| 42 | /* TODO: Currently we do not support AARCH32 instruction probing */ | 42 | /* TODO: Currently we do not support AARCH32 instruction probing */ |
| 43 | if (test_bit(TIF_32BIT, &mm->context.flags)) | 43 | if (mm->context.flags & MMCF_AARCH32) |
| 44 | return -ENOTSUPP; | 44 | return -ENOTSUPP; |
| 45 | else if (!IS_ALIGNED(addr, AARCH64_INSN_SIZE)) | 45 | else if (!IS_ALIGNED(addr, AARCH64_INSN_SIZE)) |
| 46 | return -EINVAL; | 46 | return -EINVAL; |
| @@ -384,15 +384,12 @@ unsigned long get_wchan(struct task_struct *p) · 383 unmodified lines | |||
| 384 | return 0; | 384 | return 0; |
| 385 | 385 | ||
| 386 | frame.fp = thread_saved_fp(p); | 386 | frame.fp = thread_saved_fp(p); |
| 387 | frame.sp = thread_saved_sp(p); | ||
| 388 | frame.pc = thread_saved_pc(p); | 387 | frame.pc = thread_saved_pc(p); |
| 389 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER | 388 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER |
| 390 | frame.graph = p->curr_ret_stack; | 389 | frame.graph = p->curr_ret_stack; |
| 391 | #endif | 390 | #endif |
| 392 | do { | 391 | do { |
| 393 | if (frame.sp < stack_page || | 392 | if (unwind_frame(p, &frame)) |
| 394 | frame.sp >= stack_page + THREAD_SIZE || | ||
| 395 | unwind_frame(p, &frame)) | ||
| 396 | goto out; | 393 | goto out; |
| 397 | if (!in_sched_functions(frame.pc)) { | 394 | if (!in_sched_functions(frame.pc)) { |
| 398 | ret = frame.pc; | 395 | ret = frame.pc; |
| @@ -419,3 +416,11 @@ unsigned long arch_randomize_brk(struct mm_struct *mm) · 20 unmodified lines | |||
| 419 | else | 416 | else |
| 420 | return randomize_page(mm->brk, SZ_1G); | 417 | return randomize_page(mm->brk, SZ_1G); |
| 421 | } | 418 | } |
| 419 | |||
| 420 | /* | ||
| 421 | * Called from setup_new_exec() after (COMPAT_)SET_PERSONALITY. | ||
| 422 | */ | ||
| 423 | void arch_setup_new_exec(void) | ||
| 424 | { | ||
| 425 | current->mm->context.flags = is_compat_task() ? MMCF_AARCH32 : 0; | ||
| 426 | } | ||
| @@ -42,6 +42,7 @@ · 41 unmodified lines | |||
| 42 | #include <asm/compat.h> | 42 | #include <asm/compat.h> |
| 43 | #include <asm/debug-monitors.h> | 43 | #include <asm/debug-monitors.h> |
| 44 | #include <asm/pgtable.h> | 44 | #include <asm/pgtable.h> |
| 45 | #include <asm/stacktrace.h> | ||
| 45 | #include <asm/syscall.h> | 46 | #include <asm/syscall.h> |
| 46 | #include <asm/traps.h> | 47 | #include <asm/traps.h> |
| 47 | #include <asm/system_misc.h> | 48 | #include <asm/system_misc.h> |
| @@ -127,7 +128,7 @@ static bool regs_within_kernel_stack(struct pt_regs *regs, unsigned long addr) · 79 unmodified lines | |||
| 127 | { | 128 | { |
| 128 | return ((addr & ~(THREAD_SIZE - 1)) == | 129 | return ((addr & ~(THREAD_SIZE - 1)) == |
| 129 | (kernel_stack_pointer(regs) & ~(THREAD_SIZE - 1))) || | 130 | (kernel_stack_pointer(regs) & ~(THREAD_SIZE - 1))) || |
| 130 | on_irq_stack(addr, raw_smp_processor_id()); | 131 | on_irq_stack(addr); |
| 131 | } | 132 | } |
| 132 | 133 | ||
| 133 | /** | 134 | /** |
| @@ -1363,7 +1364,7 @@ static void tracehook_report_syscall(struct pt_regs *regs, · 1229 unmodified lines | |||
| 1363 | if (dir == PTRACE_SYSCALL_EXIT) | 1364 | if (dir == PTRACE_SYSCALL_EXIT) |
| 1364 | tracehook_report_syscall_exit(regs, 0); | 1365 | tracehook_report_syscall_exit(regs, 0); |
| 1365 | else if (tracehook_report_syscall_entry(regs)) | 1366 | else if (tracehook_report_syscall_entry(regs)) |
| 1366 | regs->syscallno = ~0UL; | 1367 | forget_syscall(regs); |
| 1367 | 1368 | ||
| 1368 | regs->regs[regno] = saved_reg; | 1369 | regs->regs[regno] = saved_reg; |
| 1369 | } | 1370 | } |
| @@ -42,7 +42,6 @@ void *return_address(unsigned int level) · 41 unmodified lines | |||
| 42 | data.addr = NULL; | 42 | data.addr = NULL; |
| 43 | 43 | ||
| 44 | frame.fp = (unsigned long)__builtin_frame_address(0); | 44 | frame.fp = (unsigned long)__builtin_frame_address(0); |
| 45 | frame.sp = current_stack_pointer; | ||
| 46 | frame.pc = (unsigned long)return_address; /* dummy */ | 45 | frame.pc = (unsigned long)return_address; /* dummy */ |
| 47 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER | 46 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER |
| 48 | frame.graph = current->curr_ret_stack; | 47 | frame.graph = current->curr_ret_stack; |
| @@ -37,6 +37,7 @@ · 36 unmodified lines | |||
| 37 | #include <asm/ucontext.h> | 37 | #include <asm/ucontext.h> |
| 38 | #include <asm/unistd.h> | 38 | #include <asm/unistd.h> |
| 39 | #include <asm/fpsimd.h> | 39 | #include <asm/fpsimd.h> |
| 40 | #include <asm/ptrace.h> | ||
| 40 | #include <asm/signal32.h> | 41 | #include <asm/signal32.h> |
| 41 | #include <asm/vdso.h> | 42 | #include <asm/vdso.h> |
| 42 | 43 | ||
| @@ -388,7 +389,7 @@ static int restore_sigframe(struct pt_regs *regs, · 345 unmodified lines | |||
| 388 | /* | 389 | /* |
| 389 | * Avoid sys_rt_sigreturn() restarting. | 390 | * Avoid sys_rt_sigreturn() restarting. |
| 390 | */ | 391 | */ |
| 391 | regs->syscallno = ~0UL; | 392 | forget_syscall(regs); |
| 392 | 393 | ||
| 393 | err |= !valid_user_regs(®s->user_regs, current); | 394 | err |= !valid_user_regs(®s->user_regs, current); |
| 394 | if (err == 0) | 395 | if (err == 0) |
| @@ -674,13 +675,12 @@ static void do_signal(struct pt_regs *regs) · 279 unmodified lines | |||
| 674 | { | 675 | { |
| 675 | unsigned long continue_addr = 0, restart_addr = 0; | 676 | unsigned long continue_addr = 0, restart_addr = 0; |
| 676 | int retval = 0; | 677 | int retval = 0; |
| 677 | int syscall = (int)regs->syscallno; | ||
| 678 | struct ksignal ksig; | 678 | struct ksignal ksig; |
| 679 | 679 | ||
| 680 | /* | 680 | /* |
| 681 | * If we were from a system call, check for system call restarting... | 681 | * If we were from a system call, check for system call restarting... |
| 682 | */ | 682 | */ |
| 683 | if (syscall >= 0) { | 683 | if (in_syscall(regs)) { |
| 684 | continue_addr = regs->pc; | 684 | continue_addr = regs->pc; |
| 685 | restart_addr = continue_addr - (compat_thumb_mode(regs) ? 2 : 4); | 685 | restart_addr = continue_addr - (compat_thumb_mode(regs) ? 2 : 4); |
| 686 | retval = regs->regs[0]; | 686 | retval = regs->regs[0]; |
| @@ -688,7 +688,7 @@ static void do_signal(struct pt_regs *regs) · 1 unmodified line | |||
| 688 | /* | 688 | /* |
| 689 | * Avoid additional syscall restarting via ret_to_user. | 689 | * Avoid additional syscall restarting via ret_to_user. |
| 690 | */ | 690 | */ |
| 691 | regs->syscallno = ~0UL; | 691 | forget_syscall(regs); |
| 692 | 692 | ||
| 693 | /* | 693 | /* |
| 694 | * Prepare for system call restart. We do this here so that a | 694 | * Prepare for system call restart. We do this here so that a |
| @@ -732,7 +732,7 @@ static void do_signal(struct pt_regs *regs) · 37 unmodified lines | |||
| 732 | * Handle restarting a different system call. As above, if a debugger | 732 | * Handle restarting a different system call. As above, if a debugger |
| 733 | * has chosen to restart at a different PC, ignore the restart. | 733 | * has chosen to restart at a different PC, ignore the restart. |
| 734 | */ | 734 | */ |
| 735 | if (syscall >= 0 && regs->pc == restart_addr) { | 735 | if (in_syscall(regs) && regs->pc == restart_addr) { |
| 736 | if (retval == -ERESTART_RESTARTBLOCK) | 736 | if (retval == -ERESTART_RESTARTBLOCK) |
| 737 | setup_restart_syscall(regs); | 737 | setup_restart_syscall(regs); |
| 738 | user_rewind_single_step(current); | 738 | user_rewind_single_step(current); |
| @@ -354,7 +354,7 @@ static int compat_restore_sigframe(struct pt_regs *regs, · 353 unmodified lines | |||
| 354 | /* | 354 | /* |
| 355 | * Avoid compat_sys_sigreturn() restarting. | 355 | * Avoid compat_sys_sigreturn() restarting. |
| 356 | */ | 356 | */ |
| 357 | regs->syscallno = ~0UL; | 357 | forget_syscall(regs); |
| 358 | 358 | ||
| 359 | err |= !valid_user_regs(®s->user_regs, current); | 359 | err |= !valid_user_regs(®s->user_regs, current); |
| 360 | 360 | ||
| @@ -154,7 +154,7 @@ int __cpu_up(unsigned int cpu, struct task_struct *idle) · 153 unmodified lines | |||
| 154 | * page tables. | 154 | * page tables. |
| 155 | */ | 155 | */ |
| 156 | secondary_data.task = idle; | 156 | secondary_data.task = idle; |
| 157 | secondary_data.stack = task_stack_page(idle) + THREAD_START_SP; | 157 | secondary_data.stack = task_stack_page(idle) + THREAD_SIZE; |
| 158 | update_cpu_boot_status(CPU_MMU_OFF); | 158 | update_cpu_boot_status(CPU_MMU_OFF); |
| 159 | __flush_dcache_area(&secondary_data, sizeof(secondary_data)); | 159 | __flush_dcache_area(&secondary_data, sizeof(secondary_data)); |
| 160 | 160 | ||
| @@ -977,11 +977,21 @@ void smp_send_stop(void) · 816 unmodified lines | |||
| 977 | } | 977 | } |
| 978 | 978 | ||
| 979 | #ifdef CONFIG_KEXEC_CORE | 979 | #ifdef CONFIG_KEXEC_CORE |
| 980 | void smp_send_crash_stop(void) | 980 | void crash_smp_send_stop(void) |
| 981 | { | 981 | { |
| 982 | static int cpus_stopped; | ||
| 982 | cpumask_t mask; | 983 | cpumask_t mask; |
| 983 | unsigned long timeout; | 984 | unsigned long timeout; |
| 984 | 985 | ||
| 986 | /* | ||
| 987 | * This function can be called twice in panic path, but obviously | ||
| 988 | * we execute this only once. | ||
| 989 | */ | ||
| 990 | if (cpus_stopped) | ||
| 991 | return; | ||
| 992 | |||
| 993 | cpus_stopped = 1; | ||
| 994 | |||
| 985 | if (num_online_cpus() == 1) | 995 | if (num_online_cpus() == 1) |
| 986 | return; | 996 | return; |
| 987 | 997 | ||
| @@ -42,33 +42,17 @@ · 41 unmodified lines | |||
| 42 | */ | 42 | */ |
| 43 | int notrace unwind_frame(struct task_struct *tsk, struct stackframe *frame) | 43 | int notrace unwind_frame(struct task_struct *tsk, struct stackframe *frame) |
| 44 | { | 44 | { |
| 45 | unsigned long high, low; | ||
| 46 | unsigned long fp = frame->fp; | 45 | unsigned long fp = frame->fp; |
| 47 | unsigned long irq_stack_ptr; | 46 | |
| 47 | if (fp & 0xf) | ||
| 48 | return -EINVAL; | ||
| 48 | 49 | ||
| 49 | if (!tsk) | 50 | if (!tsk) |
| 50 | tsk = current; | 51 | tsk = current; |
| 51 | 52 | ||
| 52 | /* | 53 | if (!on_accessible_stack(tsk, fp)) |
| 53 | * Switching between stacks is valid when tracing current and in | ||
| 54 | * non-preemptible context. | ||
| 55 | */ | ||
| 56 | if (tsk == current && !preemptible()) | ||
| 57 | irq_stack_ptr = IRQ_STACK_PTR(smp_processor_id()); | ||
| 58 | else | ||
| 59 | irq_stack_ptr = 0; | ||
| 60 | |||
| 61 | low = frame->sp; | ||
| 62 | /* irq stacks are not THREAD_SIZE aligned */ | ||
| 63 | if (on_irq_stack(frame->sp, raw_smp_processor_id())) | ||
| 64 | high = irq_stack_ptr; | ||
| 65 | else | ||
| 66 | high = ALIGN(low, THREAD_SIZE) - 0x20; | ||
| 67 | |||
| 68 | if (fp < low || fp > high || fp & 0xf) | ||
| 69 | return -EINVAL; | 54 | return -EINVAL; |
| 70 | 55 | ||
| 71 | frame->sp = fp + 0x10; | ||
| 72 | frame->fp = READ_ONCE_NOCHECK(*(unsigned long *)(fp)); | 56 | frame->fp = READ_ONCE_NOCHECK(*(unsigned long *)(fp)); |
| 73 | frame->pc = READ_ONCE_NOCHECK(*(unsigned long *)(fp + 8)); | 57 | frame->pc = READ_ONCE_NOCHECK(*(unsigned long *)(fp + 8)); |
| 74 | 58 | ||
| @@ -86,34 +70,13 @@ int notrace unwind_frame(struct task_struct *tsk, struct stackframe *frame) · 11 unmodified lines | |||
| 86 | #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ | 70 | #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ |
| 87 | 71 | ||
| 88 | /* | 72 | /* |
| 89 | * Check whether we are going to walk through from interrupt stack | 73 | * Frames created upon entry from EL0 have NULL FP and PC values, so |
| 90 | * to task stack. | 74 | * don't bother reporting these. Frames created by __noreturn functions |
| 91 | * If we reach the end of the stack - and its an interrupt stack, | 75 | * might have a valid FP even if PC is bogus, so only terminate where |
| 92 | * unpack the dummy frame to find the original elr. | 76 | * both are NULL. |
| 93 | * | ||
| 94 | * Check the frame->fp we read from the bottom of the irq_stack, | ||
| 95 | * and the original task stack pointer are both in current->stack. | ||
| 96 | */ | 77 | */ |
| 97 | if (frame->sp == irq_stack_ptr) { | 78 | if (!frame->fp && !frame->pc) |
| 98 | struct pt_regs *irq_args; | 79 | return -EINVAL; |
| 99 | unsigned long orig_sp = IRQ_STACK_TO_TASK_STACK(irq_stack_ptr); | ||
| 100 | |||
| 101 | if (object_is_on_stack((void *)orig_sp) && | ||
| 102 | object_is_on_stack((void *)frame->fp)) { | ||
| 103 | frame->sp = orig_sp; | ||
| 104 | |||
| 105 | /* orig_sp is the saved pt_regs, find the elr */ | ||
| 106 | irq_args = (struct pt_regs *)orig_sp; | ||
| 107 | frame->pc = irq_args->pc; | ||
| 108 | } else { | ||
| 109 | /* | ||
| 110 | * This frame has a non-standard format, and we | ||
| 111 | * didn't fix it, because the data looked wrong. | ||
| 112 | * Refuse to output this frame. | ||
| 113 | */ | ||
| 114 | return -EINVAL; | ||
| 115 | } | ||
| 116 | } | ||
| 117 | 80 | ||
| 118 | return 0; | 81 | return 0; |
| 119 | } | 82 | } |
| @@ -167,7 +130,6 @@ void save_stack_trace_regs(struct pt_regs *regs, struct stack_trace *trace) · 47 unmodified lines | |||
| 167 | data.no_sched_functions = 0; | 130 | data.no_sched_functions = 0; |
| 168 | 131 | ||
| 169 | frame.fp = regs->regs[29]; | 132 | frame.fp = regs->regs[29]; |
| 170 | frame.sp = regs->sp; | ||
| 171 | frame.pc = regs->pc; | 133 | frame.pc = regs->pc; |
| 172 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER | 134 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER |
| 173 | frame.graph = current->curr_ret_stack; | 135 | frame.graph = current->curr_ret_stack; |
| @@ -192,12 +154,10 @@ void save_stack_trace_tsk(struct task_struct *tsk, struct stack_trace *trace) · 18 unmodified lines | |||
| 192 | if (tsk != current) { | 154 | if (tsk != current) { |
| 193 | data.no_sched_functions = 1; | 155 | data.no_sched_functions = 1; |
| 194 | frame.fp = thread_saved_fp(tsk); | 156 | frame.fp = thread_saved_fp(tsk); |
| 195 | frame.sp = thread_saved_sp(tsk); | ||
| 196 | frame.pc = thread_saved_pc(tsk); | 157 | frame.pc = thread_saved_pc(tsk); |
| 197 | } else { | 158 | } else { |
| 198 | data.no_sched_functions = 0; | 159 | data.no_sched_functions = 0; |
| 199 | frame.fp = (unsigned long)__builtin_frame_address(0); | 160 | frame.fp = (unsigned long)__builtin_frame_address(0); |
| 200 | frame.sp = current_stack_pointer; | ||
| 201 | frame.pc = (unsigned long)save_stack_trace_tsk; | 161 | frame.pc = (unsigned long)save_stack_trace_tsk; |
| 202 | } | 162 | } |
| 203 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER | 163 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER |
| @@ -50,7 +50,6 @@ unsigned long profile_pc(struct pt_regs *regs) · 49 unmodified lines | |||
| 50 | return regs->pc; | 50 | return regs->pc; |
| 51 | 51 | ||
| 52 | frame.fp = regs->regs[29]; | 52 | frame.fp = regs->regs[29]; |
| 53 | frame.sp = regs->sp; | ||
| 54 | frame.pc = regs->pc; | 53 | frame.pc = regs->pc; |
| 55 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER | 54 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER |
| 56 | frame.graph = -1; /* no task info */ | 55 | frame.graph = -1; /* no task info */ |
| @@ -32,6 +32,7 @@ · 31 unmodified lines | |||
| 32 | #include <linux/sched/signal.h> | 32 | #include <linux/sched/signal.h> |
| 33 | #include <linux/sched/debug.h> | 33 | #include <linux/sched/debug.h> |
| 34 | #include <linux/sched/task_stack.h> | 34 | #include <linux/sched/task_stack.h> |
| 35 | #include <linux/sizes.h> | ||
| 35 | #include <linux/syscalls.h> | 36 | #include <linux/syscalls.h> |
| 36 | #include <linux/mm_types.h> | 37 | #include <linux/mm_types.h> |
| 37 | 38 | ||
| @@ -41,6 +42,7 @@ · 3 unmodified lines | |||
| 41 | #include <asm/esr.h> | 42 | #include <asm/esr.h> |
| 42 | #include <asm/insn.h> | 43 | #include <asm/insn.h> |
| 43 | #include <asm/traps.h> | 44 | #include <asm/traps.h> |
| 45 | #include <asm/smp.h> | ||
| 44 | #include <asm/stack_pointer.h> | 46 | #include <asm/stack_pointer.h> |
| 45 | #include <asm/stacktrace.h> | 47 | #include <asm/stacktrace.h> |
| 46 | #include <asm/exception.h> | 48 | #include <asm/exception.h> |
| @@ -143,7 +145,6 @@ static void dump_instr(const char *lvl, struct pt_regs *regs) · 96 unmodified lines | |||
| 143 | void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk) | 145 | void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk) |
| 144 | { | 146 | { |
| 145 | struct stackframe frame; | 147 | struct stackframe frame; |
| 146 | unsigned long irq_stack_ptr; | ||
| 147 | int skip; | 148 | int skip; |
| 148 | 149 | ||
| 149 | pr_debug("%s(regs = %p tsk = %p)\n", __func__, regs, tsk); | 150 | pr_debug("%s(regs = %p tsk = %p)\n", __func__, regs, tsk); |
| @@ -154,25 +155,14 @@ void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk) · 4 unmodified lines | |||
| 154 | if (!try_get_task_stack(tsk)) | 155 | if (!try_get_task_stack(tsk)) |
| 155 | return; | 156 | return; |
| 156 | 157 | ||
| 157 | /* | ||
| 158 | * Switching between stacks is valid when tracing current and in | ||
| 159 | * non-preemptible context. | ||
| 160 | */ | ||
| 161 | if (tsk == current && !preemptible()) | ||
| 162 | irq_stack_ptr = IRQ_STACK_PTR(smp_processor_id()); | ||
| 163 | else | ||
| 164 | irq_stack_ptr = 0; | ||
| 165 | |||
| 166 | if (tsk == current) { | 158 | if (tsk == current) { |
| 167 | frame.fp = (unsigned long)__builtin_frame_address(0); | 159 | frame.fp = (unsigned long)__builtin_frame_address(0); |
| 168 | frame.sp = current_stack_pointer; | ||
| 169 | frame.pc = (unsigned long)dump_backtrace; | 160 | frame.pc = (unsigned long)dump_backtrace; |
| 170 | } else { | 161 | } else { |
| 171 | /* | 162 | /* |
| 172 | * task blocked in __switch_to | 163 | * task blocked in __switch_to |
| 173 | */ | 164 | */ |
| 174 | frame.fp = thread_saved_fp(tsk); | 165 | frame.fp = thread_saved_fp(tsk); |
| 175 | frame.sp = thread_saved_sp(tsk); | ||
| 176 | frame.pc = thread_saved_pc(tsk); | 166 | frame.pc = thread_saved_pc(tsk); |
| 177 | } | 167 | } |
| 178 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER | 168 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER |
| @@ -182,13 +172,12 @@ void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk) · 3 unmodified lines | |||
| 182 | skip = !!regs; | 172 | skip = !!regs; |
| 183 | printk("Call trace:\n"); | 173 | printk("Call trace:\n"); |
| 184 | while (1) { | 174 | while (1) { |
| 185 | unsigned long where = frame.pc; | ||
| 186 | unsigned long stack; | 175 | unsigned long stack; |
| 187 | int ret; | 176 | int ret; |
| 188 | 177 | ||
| 189 | /* skip until specified stack frame */ | 178 | /* skip until specified stack frame */ |
| 190 | if (!skip) { | 179 | if (!skip) { |
| 191 | dump_backtrace_entry(where); | 180 | dump_backtrace_entry(frame.pc); |
| 192 | } else if (frame.fp == regs->regs[29]) { | 181 | } else if (frame.fp == regs->regs[29]) { |
| 193 | skip = 0; | 182 | skip = 0; |
| 194 | /* | 183 | /* |
| @@ -203,20 +192,12 @@ void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk) · 8 unmodified lines | |||
| 203 | ret = unwind_frame(tsk, &frame); | 192 | ret = unwind_frame(tsk, &frame); |
| 204 | if (ret < 0) | 193 | if (ret < 0) |
| 205 | break; | 194 | break; |
| 206 | stack = frame.sp; | 195 | if (in_entry_text(frame.pc)) { |
| 207 | if (in_exception_text(where)) { | 196 | stack = frame.fp - offsetof(struct pt_regs, stackframe); |
| 208 | /* | ||
| 209 | * If we switched to the irq_stack before calling this | ||
| 210 | * exception handler, then the pt_regs will be on the | ||
| 211 | * task stack. The easiest way to tell is if the large | ||
| 212 | * pt_regs would overlap with the end of the irq_stack. | ||
| 213 | */ | ||
| 214 | if (stack < irq_stack_ptr && | ||
| 215 | (stack + sizeof(struct pt_regs)) > irq_stack_ptr) | ||
| 216 | stack = IRQ_STACK_TO_TASK_STACK(irq_stack_ptr); | ||
| 217 | 197 | ||
| 218 | dump_mem("", "Exception stack", stack, | 198 | if (on_accessible_stack(tsk, stack)) |
| 219 | stack + sizeof(struct pt_regs)); | 199 | dump_mem("", "Exception stack", stack, |
| 200 | stack + sizeof(struct pt_regs)); | ||
| 220 | } | 201 | } |
| 221 | } | 202 | } |
| 222 | 203 | ||
| @@ -257,8 +238,6 @@ static int __die(const char *str, int err, struct pt_regs *regs) · 34 unmodified lines | |||
| 257 | end_of_stack(tsk)); | 238 | end_of_stack(tsk)); |
| 258 | 239 | ||
| 259 | if (!user_mode(regs)) { | 240 | if (!user_mode(regs)) { |
| 260 | dump_mem(KERN_EMERG, "Stack: ", regs->sp, | ||
| 261 | THREAD_SIZE + (unsigned long)task_stack_page(tsk)); | ||
| 262 | dump_backtrace(regs, tsk); | 241 | dump_backtrace(regs, tsk); |
| 263 | dump_instr(KERN_EMERG, regs); | 242 | dump_instr(KERN_EMERG, regs); |
| 264 | } | 243 | } |
| @@ -484,6 +463,9 @@ static void user_cache_maint_handler(unsigned int esr, struct pt_regs *regs) · 219 unmodified lines | |||
| 484 | case ESR_ELx_SYS64_ISS_CRM_DC_CVAC: /* DC CVAC, gets promoted */ | 463 | case ESR_ELx_SYS64_ISS_CRM_DC_CVAC: /* DC CVAC, gets promoted */ |
| 485 | __user_cache_maint("dc civac", address, ret); | 464 | __user_cache_maint("dc civac", address, ret); |
| 486 | break; | 465 | break; |
| 466 | case ESR_ELx_SYS64_ISS_CRM_DC_CVAP: /* DC CVAP */ | ||
| 467 | __user_cache_maint("sys 3, c7, c12, 1", address, ret); | ||
| 468 | break; | ||
| 487 | case ESR_ELx_SYS64_ISS_CRM_DC_CIVAC: /* DC CIVAC */ | 469 | case ESR_ELx_SYS64_ISS_CRM_DC_CIVAC: /* DC CIVAC */ |
| 488 | __user_cache_maint("dc civac", address, ret); | 470 | __user_cache_maint("dc civac", address, ret); |
| 489 | break; | 471 | break; |
| @@ -593,7 +575,7 @@ asmlinkage long do_ni_syscall(struct pt_regs *regs) · 103 unmodified lines | |||
| 593 | 575 | ||
| 594 | if (show_unhandled_signals_ratelimited()) { | 576 | if (show_unhandled_signals_ratelimited()) { |
| 595 | pr_info("%s[%d]: syscall %d\n", current->comm, | 577 | pr_info("%s[%d]: syscall %d\n", current->comm, |
| 596 | task_pid_nr(current), (int)regs->syscallno); | 578 | task_pid_nr(current), regs->syscallno); |
| 597 | dump_instr("", regs); | 579 | dump_instr("", regs); |
| 598 | if (user_mode(regs)) | 580 | if (user_mode(regs)) |
| 599 | __show_regs(regs); | 581 | __show_regs(regs); |
| @@ -689,6 +671,43 @@ asmlinkage void bad_el0_sync(struct pt_regs *regs, int reason, unsigned int esr) · 89 unmodified lines | |||
| 689 | force_sig_info(info.si_signo, &info, current); | 671 | force_sig_info(info.si_signo, &info, current); |
| 690 | } | 672 | } |
| 691 | 673 | ||
| 674 | #ifdef CONFIG_VMAP_STACK | ||
| 675 | |||
| 676 | DEFINE_PER_CPU(unsigned long [OVERFLOW_STACK_SIZE/sizeof(long)], overflow_stack) | ||
| 677 | __aligned(16); | ||
| 678 | |||
| 679 | asmlinkage void handle_bad_stack(struct pt_regs *regs) | ||
| 680 | { | ||
| 681 | unsigned long tsk_stk = (unsigned long)current->stack; | ||
| 682 | unsigned long irq_stk = (unsigned long)this_cpu_read(irq_stack_ptr); | ||
| 683 | unsigned long ovf_stk = (unsigned long)this_cpu_ptr(overflow_stack); | ||
| 684 | unsigned int esr = read_sysreg(esr_el1); | ||
| 685 | unsigned long far = read_sysreg(far_el1); | ||
| 686 | |||
| 687 | console_verbose(); | ||
| 688 | pr_emerg("Insufficient stack space to handle exception!"); | ||
| 689 | |||
| 690 | pr_emerg("ESR: 0x%08x -- %s\n", esr, esr_get_class_string(esr)); | ||
| 691 | pr_emerg("FAR: 0x%016lx\n", far); | ||
| 692 | |||
| 693 | pr_emerg("Task stack: [0x%016lx..0x%016lx]\n", | ||
| 694 | tsk_stk, tsk_stk + THREAD_SIZE); | ||
| 695 | pr_emerg("IRQ stack: [0x%016lx..0x%016lx]\n", | ||
| 696 | irq_stk, irq_stk + THREAD_SIZE); | ||
| 697 | pr_emerg("Overflow stack: [0x%016lx..0x%016lx]\n", | ||
| 698 | ovf_stk, ovf_stk + OVERFLOW_STACK_SIZE); | ||
| 699 | |||
| 700 | __show_regs(regs); | ||
| 701 | |||
| 702 | /* | ||
| 703 | * We use nmi_panic to limit the potential for recusive overflows, and | ||
| 704 | * to get a better stack trace. | ||
| 705 | */ | ||
| 706 | nmi_panic(NULL, "kernel stack overflow"); | ||
| 707 | cpu_park_loop(); | ||
| 708 | } | ||
| 709 | #endif | ||
| 710 | |||
| 692 | void __pte_error(const char *file, int line, unsigned long val) | 711 | void __pte_error(const char *file, int line, unsigned long val) |
| 693 | { | 712 | { |
| 694 | pr_err("%s:%d: bad pte %016lx.\n", file, line, val); | 713 | pr_err("%s:%d: bad pte %016lx.\n", file, line, val); |
| @@ -110,12 +110,27 @@ int aarch32_setup_vectors_page(struct linux_binprm *bprm, int uses_interp) · 109 unmodified lines | |||
| 110 | } | 110 | } |
| 111 | #endif /* CONFIG_COMPAT */ | 111 | #endif /* CONFIG_COMPAT */ |
| 112 | 112 | ||
| 113 | static int vdso_mremap(const struct vm_special_mapping *sm, | ||
| 114 | struct vm_area_struct *new_vma) | ||
| 115 | { | ||
| 116 | unsigned long new_size = new_vma->vm_end - new_vma->vm_start; | ||
| 117 | unsigned long vdso_size = vdso_end - vdso_start; | ||
| 118 | |||
| 119 | if (vdso_size != new_size) | ||
| 120 | return -EINVAL; | ||
| 121 | |||
| 122 | current->mm->context.vdso = (void *)new_vma->vm_start; | ||
| 123 | |||
| 124 | return 0; | ||
| 125 | } | ||
| 126 | |||
| 113 | static struct vm_special_mapping vdso_spec[2] __ro_after_init = { | 127 | static struct vm_special_mapping vdso_spec[2] __ro_after_init = { |
| 114 | { | 128 | { |
| 115 | .name = "[vvar]", | 129 | .name = "[vvar]", |
| 116 | }, | 130 | }, |
| 117 | { | 131 | { |
| 118 | .name = "[vdso]", | 132 | .name = "[vdso]", |
| 133 | .mremap = vdso_mremap, | ||
| 119 | }, | 134 | }, |
| 120 | }; | 135 | }; |
| 121 | 136 | ||
| @@ -72,22 +72,6 @@ PECOFF_FILE_ALIGNMENT = 0x200; · 71 unmodified lines | |||
| 72 | #define PECOFF_EDATA_PADDING | 72 | #define PECOFF_EDATA_PADDING |
| 73 | #endif | 73 | #endif |
| 74 | 74 | ||
| 75 | #if defined(CONFIG_DEBUG_ALIGN_RODATA) | ||
| 76 | /* | ||
| 77 | * 4 KB granule: 1 level 2 entry | ||
| 78 | * 16 KB granule: 128 level 3 entries, with contiguous bit | ||
| 79 | * 64 KB granule: 32 level 3 entries, with contiguous bit | ||
| 80 | */ | ||
| 81 | #define SEGMENT_ALIGN SZ_2M | ||
| 82 | #else | ||
| 83 | /* | ||
| 84 | * 4 KB granule: 16 level 3 entries, with contiguous bit | ||
| 85 | * 16 KB granule: 4 level 3 entries, without contiguous bit | ||
| 86 | * 64 KB granule: 1 level 3 entry | ||
| 87 | */ | ||
| 88 | #define SEGMENT_ALIGN SZ_64K | ||
| 89 | #endif | ||
| 90 | |||
| 91 | SECTIONS | 75 | SECTIONS |
| 92 | { | 76 | { |
| 93 | /* | 77 | /* |
| @@ -192,7 +176,7 @@ SECTIONS · 98 unmodified lines | |||
| 192 | 176 | ||
| 193 | _data = .; | 177 | _data = .; |
| 194 | _sdata = .; | 178 | _sdata = .; |
| 195 | RW_DATA_SECTION(L1_CACHE_BYTES, PAGE_SIZE, THREAD_SIZE) | 179 | RW_DATA_SECTION(L1_CACHE_BYTES, PAGE_SIZE, THREAD_ALIGN) |
| 196 | 180 | ||
| 197 | /* | 181 | /* |
| 198 | * Data written with the MMU off but read with the MMU on requires | 182 | * Data written with the MMU off but read with the MMU on requires |
| @@ -70,7 +70,7 @@ u32 __hyp_text __init_stage2_translation(void) · 69 unmodified lines | |||
| 70 | * Management in ID_AA64MMFR1_EL1 and enable the feature in VTCR_EL2. | 70 | * Management in ID_AA64MMFR1_EL1 and enable the feature in VTCR_EL2. |
| 71 | */ | 71 | */ |
| 72 | tmp = (read_sysreg(id_aa64mmfr1_el1) >> ID_AA64MMFR1_HADBS_SHIFT) & 0xf; | 72 | tmp = (read_sysreg(id_aa64mmfr1_el1) >> ID_AA64MMFR1_HADBS_SHIFT) & 0xf; |
| 73 | if (IS_ENABLED(CONFIG_ARM64_HW_AFDBM) && tmp) | 73 | if (tmp) |
| 74 | val |= VTCR_EL2_HA; | 74 | val |= VTCR_EL2_HA; |
| 75 | 75 | ||
| 76 | /* | 76 | /* |
| @@ -17,3 +17,5 @@ CFLAGS_atomic_ll_sc.o := -fcall-used-x0 -ffixed-x1 -ffixed-x2 \ · 16 unmodified lines | |||
| 17 | -fcall-saved-x10 -fcall-saved-x11 -fcall-saved-x12 \ | 17 | -fcall-saved-x10 -fcall-saved-x11 -fcall-saved-x12 \ |
| 18 | -fcall-saved-x13 -fcall-saved-x14 -fcall-saved-x15 \ | 18 | -fcall-saved-x13 -fcall-saved-x14 -fcall-saved-x15 \ |
| 19 | -fcall-saved-x18 | 19 | -fcall-saved-x18 |
| 20 | |||
| 21 | lib-$(CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE) += uaccess_flushcache.o | ||
| @@ -0,0 +1,47 @@ | |||
| 1 | /* | ||
| 2 | * Copyright (C) 2017 ARM Ltd. | ||
| 3 | * | ||
| 4 | * This program is free software; you can redistribute it and/or modify | ||
| 5 | * it under the terms of the GNU General Public License version 2 as | ||
| 6 | * published by the Free Software Foundation. | ||
| 7 | * | ||
| 8 | * This program is distributed in the hope that it will be useful, | ||
| 9 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
| 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
| 11 | * GNU General Public License for more details. | ||
| 12 | * | ||
| 13 | * You should have received a copy of the GNU General Public License | ||
| 14 | * along with this program. If not, see <http://www.gnu.org/licenses/>. | ||
| 15 | */ | ||
| 16 | |||
| 17 | #include <linux/uaccess.h> | ||
| 18 | #include <asm/barrier.h> | ||
| 19 | #include <asm/cacheflush.h> | ||
| 20 | |||
| 21 | void memcpy_flushcache(void *dst, const void *src, size_t cnt) | ||
| 22 | { | ||
| 23 | /* | ||
| 24 | * We assume this should not be called with @dst pointing to | ||
| 25 | * non-cacheable memory, such that we don't need an explicit | ||
| 26 | * barrier to order the cache maintenance against the memcpy. | ||
| 27 | */ | ||
| 28 | memcpy(dst, src, cnt); | ||
| 29 | __clean_dcache_area_pop(dst, cnt); | ||
| 30 | } | ||
| 31 | EXPORT_SYMBOL_GPL(memcpy_flushcache); | ||
| 32 | |||
| 33 | void memcpy_page_flushcache(char *to, struct page *page, size_t offset, | ||
| 34 | size_t len) | ||
| 35 | { | ||
| 36 | memcpy_flushcache(to, page_address(page) + offset, len); | ||
| 37 | } | ||
| 38 | |||
| 39 | unsigned long __copy_user_flushcache(void *to, const void __user *from, | ||
| 40 | unsigned long n) | ||
| 41 | { | ||
| 42 | unsigned long rc = __arch_copy_from_user(to, from, n); | ||
| 43 | |||
| 44 | /* See above */ | ||
| 45 | __clean_dcache_area_pop(to, n - rc); | ||
| 46 | return rc; | ||
| 47 | } | ||
| @@ -109,20 +109,25 @@ ENTRY(__clean_dcache_area_pou) · 108 unmodified lines | |||
| 109 | ENDPROC(__clean_dcache_area_pou) | 109 | ENDPROC(__clean_dcache_area_pou) |
| 110 | 110 | ||
| 111 | /* | 111 | /* |
| 112 | * __dma_inv_area(start, size) | 112 | * __inval_dcache_area(kaddr, size) |
| 113 | * - start - virtual start address of region | 113 | * |
| 114 | * Ensure that any D-cache lines for the interval [kaddr, kaddr+size) | ||
| 115 | * are invalidated. Any partial lines at the ends of the interval are | ||
| 116 | * also cleaned to PoC to prevent data loss. | ||
| 117 | * | ||
| 118 | * - kaddr - kernel address | ||
| 114 | * - size - size in question | 119 | * - size - size in question |
| 115 | */ | 120 | */ |
| 116 | __dma_inv_area: | 121 | ENTRY(__inval_dcache_area) |
| 117 | add x1, x1, x0 | ||
| 118 | /* FALLTHROUGH */ | 122 | /* FALLTHROUGH */ |
| 119 | 123 | ||
| 120 | /* | 124 | /* |
| 121 | * __inval_cache_range(start, end) | 125 | * __dma_inv_area(start, size) |
| 122 | * - start - start address of region | 126 | * - start - virtual start address of region |
| 123 | * - end - end address of region | 127 | * - size - size in question |
| 124 | */ | 128 | */ |
| 125 | ENTRY(__inval_cache_range) | 129 | __dma_inv_area: |
| 130 | add x1, x1, x0 | ||
| 126 | dcache_line_size x2, x3 | 131 | dcache_line_size x2, x3 |
| 127 | sub x3, x2, #1 | 132 | sub x3, x2, #1 |
| 128 | tst x1, x3 // end cache line aligned? | 133 | tst x1, x3 // end cache line aligned? |
| @@ -140,7 +145,7 @@ ENTRY(__inval_cache_range) · 11 unmodified lines | |||
| 140 | b.lo 2b | 145 | b.lo 2b |
| 141 | dsb sy | 146 | dsb sy |
| 142 | ret | 147 | ret |
| 143 | ENDPIPROC(__inval_cache_range) | 148 | ENDPIPROC(__inval_dcache_area) |
| 144 | ENDPROC(__dma_inv_area) | 149 | ENDPROC(__dma_inv_area) |
| 145 | 150 | ||
| 146 | /* | 151 | /* |
| @@ -166,6 +171,20 @@ __dma_clean_area: · 19 unmodified lines | |||
| 166 | ENDPIPROC(__clean_dcache_area_poc) | 171 | ENDPIPROC(__clean_dcache_area_poc) |
| 167 | ENDPROC(__dma_clean_area) | 172 | ENDPROC(__dma_clean_area) |
| 168 | 173 | ||
| 174 | /* | ||
| 175 | * __clean_dcache_area_pop(kaddr, size) | ||
| 176 | * | ||
| 177 | * Ensure that any D-cache lines for the interval [kaddr, kaddr+size) | ||
| 178 | * are cleaned to the PoP. | ||
| 179 | * | ||
| 180 | * - kaddr - kernel address | ||
| 181 | * - size - size in question | ||
| 182 | */ | ||
| 183 | ENTRY(__clean_dcache_area_pop) | ||
| 184 | dcache_by_line_op cvap, sy, x0, x1, x2, x3 | ||
| 185 | ret | ||
| 186 | ENDPIPROC(__clean_dcache_area_pop) | ||
| 187 | |||
| 169 | /* | 188 | /* |
| 170 | * __dma_flush_area(start, size) | 189 | * __dma_flush_area(start, size) |
| 171 | * | 190 | * |
| @@ -42,7 +42,7 @@ static pgprot_t __get_dma_pgprot(unsigned long attrs, pgprot_t prot, · 41 unmodified lines | |||
| 42 | return prot; | 42 | return prot; |
| 43 | } | 43 | } |
| 44 | 44 | ||
| 45 | static struct gen_pool *atomic_pool; | 45 | static struct gen_pool *atomic_pool __ro_after_init; |
| 46 | 46 | ||
| 47 | #define DEFAULT_DMA_COHERENT_POOL_SIZE SZ_256K | 47 | #define DEFAULT_DMA_COHERENT_POOL_SIZE SZ_256K |
| 48 | static size_t atomic_pool_size __initdata = DEFAULT_DMA_COHERENT_POOL_SIZE; | 48 | static size_t atomic_pool_size __initdata = DEFAULT_DMA_COHERENT_POOL_SIZE; |
| @@ -425,7 +425,7 @@ static int __init atomic_pool_init(void) · 376 unmodified lines | |||
| 425 | 425 | ||
| 426 | gen_pool_set_algo(atomic_pool, | 426 | gen_pool_set_algo(atomic_pool, |
| 427 | gen_pool_first_fit_order_align, | 427 | gen_pool_first_fit_order_align, |
| 428 | (void *)PAGE_SHIFT); | 428 | NULL); |
| 429 | 429 | ||
| 430 | pr_info("DMA: preallocated %zu KiB pool for atomic allocations\n", | 430 | pr_info("DMA: preallocated %zu KiB pool for atomic allocations\n", |
| 431 | atomic_pool_size / 1024); | 431 | atomic_pool_size / 1024); |
| @@ -34,6 +34,7 @@ · 33 unmodified lines | |||
| 34 | #include <linux/hugetlb.h> | 34 | #include <linux/hugetlb.h> |
| 35 | 35 | ||
| 36 | #include <asm/bug.h> | 36 | #include <asm/bug.h> |
| 37 | #include <asm/cmpxchg.h> | ||
| 37 | #include <asm/cpufeature.h> | 38 | #include <asm/cpufeature.h> |
| 38 | #include <asm/exception.h> | 39 | #include <asm/exception.h> |
| 39 | #include <asm/debug-monitors.h> | 40 | #include <asm/debug-monitors.h> |
| @@ -82,6 +83,49 @@ static inline int notify_page_fault(struct pt_regs *regs, unsigned int esr) · 42 unmodified lines | |||
| 82 | } | 83 | } |
| 83 | #endif | 84 | #endif |
| 84 | 85 | ||
| 86 | static void data_abort_decode(unsigned int esr) | ||
| 87 | { | ||
| 88 | pr_alert("Data abort info:\n"); | ||
| 89 | |||
| 90 | if (esr & ESR_ELx_ISV) { | ||
| 91 | pr_alert(" Access size = %u byte(s)\n", | ||
| 92 | 1U << ((esr & ESR_ELx_SAS) >> ESR_ELx_SAS_SHIFT)); | ||
| 93 | pr_alert(" SSE = %lu, SRT = %lu\n", | ||
| 94 | (esr & ESR_ELx_SSE) >> ESR_ELx_SSE_SHIFT, | ||
| 95 | (esr & ESR_ELx_SRT_MASK) >> ESR_ELx_SRT_SHIFT); | ||
| 96 | pr_alert(" SF = %lu, AR = %lu\n", | ||
| 97 | (esr & ESR_ELx_SF) >> ESR_ELx_SF_SHIFT, | ||
| 98 | (esr & ESR_ELx_AR) >> ESR_ELx_AR_SHIFT); | ||
| 99 | } else { | ||
| 100 | pr_alert(" ISV = 0, ISS = 0x%08lu\n", esr & ESR_ELx_ISS_MASK); | ||
| 101 | } | ||
| 102 | |||
| 103 | pr_alert(" CM = %lu, WnR = %lu\n", | ||
| 104 | (esr & ESR_ELx_CM) >> ESR_ELx_CM_SHIFT, | ||
| 105 | (esr & ESR_ELx_WNR) >> ESR_ELx_WNR_SHIFT); | ||
| 106 | } | ||
| 107 | |||
| 108 | /* | ||
| 109 | * Decode mem abort information | ||
| 110 | */ | ||
| 111 | static void mem_abort_decode(unsigned int esr) | ||
| 112 | { | ||
| 113 | pr_alert("Mem abort info:\n"); | ||
| 114 | |||
| 115 | pr_alert(" Exception class = %s, IL = %u bits\n", | ||
| 116 | esr_get_class_string(esr), | ||
| 117 | (esr & ESR_ELx_IL) ? 32 : 16); | ||
| 118 | pr_alert(" SET = %lu, FnV = %lu\n", | ||
| 119 | (esr & ESR_ELx_SET_MASK) >> ESR_ELx_SET_SHIFT, | ||
| 120 | (esr & ESR_ELx_FnV) >> ESR_ELx_FnV_SHIFT); | ||
| 121 | pr_alert(" EA = %lu, S1PTW = %lu\n", | ||
| 122 | (esr & ESR_ELx_EA) >> ESR_ELx_EA_SHIFT, | ||
| 123 | (esr & ESR_ELx_S1PTW) >> ESR_ELx_S1PTW_SHIFT); | ||
| 124 | |||
| 125 | if (esr_is_data_abort(esr)) | ||
| 126 | data_abort_decode(esr); | ||
| 127 | } | ||
| 128 | |||
| 85 | /* | 129 | /* |
| 86 | * Dump out the page tables associated with 'addr' in the currently active mm. | 130 | * Dump out the page tables associated with 'addr' in the currently active mm. |
| 87 | */ | 131 | */ |
| @@ -139,7 +183,6 @@ void show_pte(unsigned long addr) · 51 unmodified lines | |||
| 139 | pr_cont("\n"); | 183 | pr_cont("\n"); |
| 140 | } | 184 | } |
| 141 | 185 | ||
| 142 | #ifdef CONFIG_ARM64_HW_AFDBM | ||
| 143 | /* | 186 | /* |
| 144 | * This function sets the access flags (dirty, accessed), as well as write | 187 | * This function sets the access flags (dirty, accessed), as well as write |
| 145 | * permission, and only to a more permissive setting. | 188 | * permission, and only to a more permissive setting. |
| @@ -154,18 +197,13 @@ int ptep_set_access_flags(struct vm_area_struct *vma, · 8 unmodified lines | |||
| 154 | unsigned long address, pte_t *ptep, | 197 | unsigned long address, pte_t *ptep, |
| 155 | pte_t entry, int dirty) | 198 | pte_t entry, int dirty) |
| 156 | { | 199 | { |
| 157 | pteval_t old_pteval; | 200 | pteval_t old_pteval, pteval; |
| 158 | unsigned int tmp; | ||
| 159 | 201 | ||
| 160 | if (pte_same(*ptep, entry)) | 202 | if (pte_same(*ptep, entry)) |
| 161 | return 0; | 203 | return 0; |
| 162 | 204 | ||
| 163 | /* only preserve the access flags and write permission */ | 205 | /* only preserve the access flags and write permission */ |
| 164 | pte_val(entry) &= PTE_AF | PTE_WRITE | PTE_DIRTY; | 206 | pte_val(entry) &= PTE_RDONLY | PTE_AF | PTE_WRITE | PTE_DIRTY; |
| 165 | |||
| 166 | /* set PTE_RDONLY if actual read-only or clean PTE */ | ||
| 167 | if (!pte_write(entry) || !pte_sw_dirty(entry)) | ||
| 168 | pte_val(entry) |= PTE_RDONLY; | ||
| 169 | 207 | ||
| 170 | /* | 208 | /* |
| 171 | * Setting the flags must be done atomically to avoid racing with the | 209 | * Setting the flags must be done atomically to avoid racing with the |
| @@ -174,21 +212,18 @@ int ptep_set_access_flags(struct vm_area_struct *vma, · 2 unmodified lines | |||
| 174 | * (calculated as: a & b == ~(~a | ~b)). | 212 | * (calculated as: a & b == ~(~a | ~b)). |
| 175 | */ | 213 | */ |
| 176 | pte_val(entry) ^= PTE_RDONLY; | 214 | pte_val(entry) ^= PTE_RDONLY; |
| 177 | asm volatile("// ptep_set_access_flags\n" | 215 | pteval = READ_ONCE(pte_val(*ptep)); |
| 178 | " prfm pstl1strm, %2\n" | 216 | do { |
| 179 | "1: ldxr %0, %2\n" | 217 | old_pteval = pteval; |
| 180 | " eor %0, %0, %3 // negate PTE_RDONLY in *ptep\n" | 218 | pteval ^= PTE_RDONLY; |
| 181 | " orr %0, %0, %4 // set flags\n" | 219 | pteval |= pte_val(entry); |
| 182 | " eor %0, %0, %3 // negate final PTE_RDONLY\n" | 220 | pteval ^= PTE_RDONLY; |
| 183 | " stxr %w1, %0, %2\n" | 221 | pteval = cmpxchg_relaxed(&pte_val(*ptep), old_pteval, pteval); |
| 184 | " cbnz %w1, 1b\n" | 222 | } while (pteval != old_pteval); |
| 185 | : "=&r" (old_pteval), "=&r" (tmp), "+Q" (pte_val(*ptep)) | ||
| 186 | : "L" (PTE_RDONLY), "r" (pte_val(entry))); | ||
| 187 | 223 | ||
| 188 | flush_tlb_fix_spurious_fault(vma, address); | 224 | flush_tlb_fix_spurious_fault(vma, address); |
| 189 | return 1; | 225 | return 1; |
| 190 | } | 226 | } |
| 191 | #endif | ||
| 192 | 227 | ||
| 193 | static bool is_el1_instruction_abort(unsigned int esr) | 228 | static bool is_el1_instruction_abort(unsigned int esr) |
| 194 | { | 229 | { |
| @@ -248,6 +283,8 @@ static void __do_kernel_fault(unsigned long addr, unsigned int esr, · 53 unmodified lines | |||
| 248 | pr_alert("Unable to handle kernel %s at virtual address %08lx\n", msg, | 283 | pr_alert("Unable to handle kernel %s at virtual address %08lx\n", msg, |
| 249 | addr); | 284 | addr); |
| 250 | 285 | ||
| 286 | mem_abort_decode(esr); | ||
| 287 | |||
| 251 | show_pte(addr); | 288 | show_pte(addr); |
| 252 | die("Oops", regs, esr); | 289 | die("Oops", regs, esr); |
| 253 | bust_spinlocks(0); | 290 | bust_spinlocks(0); |
| @@ -705,6 +742,8 @@ asmlinkage void __exception do_mem_abort(unsigned long addr, unsigned int esr, · 451 unmodified lines | |||
| 705 | pr_alert("Unhandled fault: %s (0x%08x) at 0x%016lx\n", | 742 | pr_alert("Unhandled fault: %s (0x%08x) at 0x%016lx\n", |
| 706 | inf->name, esr, addr); | 743 | inf->name, esr, addr); |
| 707 | 744 | ||
| 745 | mem_abort_decode(esr); | ||
| 746 | |||
| 708 | info.si_signo = inf->sig; | 747 | info.si_signo = inf->sig; |
| 709 | info.si_errno = 0; | 748 | info.si_errno = 0; |
| 710 | info.si_code = inf->code; | 749 | info.si_code = inf->code; |
| @@ -83,3 +83,19 @@ EXPORT_SYMBOL(flush_dcache_page); · 82 unmodified lines | |||
| 83 | * Additional functions defined in assembly. | 83 | * Additional functions defined in assembly. |
| 84 | */ | 84 | */ |
| 85 | EXPORT_SYMBOL(flush_icache_range); | 85 | EXPORT_SYMBOL(flush_icache_range); |
| 86 | |||
| 87 | #ifdef CONFIG_ARCH_HAS_PMEM_API | ||
| 88 | void arch_wb_cache_pmem(void *addr, size_t size) | ||
| 89 | { | ||
| 90 | /* Ensure order against any prior non-cacheable writes */ | ||
| 91 | dmb(osh); | ||
| 92 | __clean_dcache_area_pop(addr, size); | ||
| 93 | } | ||
| 94 | EXPORT_SYMBOL_GPL(arch_wb_cache_pmem); | ||
| 95 | |||
| 96 | void arch_invalidate_pmem(void *addr, size_t size) | ||
| 97 | { | ||
| 98 | __inval_dcache_area(addr, size); | ||
| 99 | } | ||
| 100 | EXPORT_SYMBOL_GPL(arch_invalidate_pmem); | ||
| 101 | #endif | ||
| @@ -41,6 +41,16 @@ int pud_huge(pud_t pud) · 40 unmodified lines | |||
| 41 | #endif | 41 | #endif |
| 42 | } | 42 | } |
| 43 | 43 | ||
| 44 | /* | ||
| 45 | * Select all bits except the pfn | ||
| 46 | */ | ||
| 47 | static inline pgprot_t pte_pgprot(pte_t pte) | ||
| 48 | { | ||
| 49 | unsigned long pfn = pte_pfn(pte); | ||
| 50 | |||
| 51 | return __pgprot(pte_val(pfn_pte(pfn, __pgprot(0))) ^ pte_val(pte)); | ||
| 52 | } | ||
| 53 | |||
| 44 | static int find_num_contig(struct mm_struct *mm, unsigned long addr, | 54 | static int find_num_contig(struct mm_struct *mm, unsigned long addr, |
| 45 | pte_t *ptep, size_t *pgsize) | 55 | pte_t *ptep, size_t *pgsize) |
| 46 | { | 56 | { |
| @@ -58,15 +68,107 @@ static int find_num_contig(struct mm_struct *mm, unsigned long addr, · 11 unmodified lines | |||
| 58 | return CONT_PTES; | 68 | return CONT_PTES; |
| 59 | } | 69 | } |
| 60 | 70 | ||
| 71 | static inline int num_contig_ptes(unsigned long size, size_t *pgsize) | ||
| 72 | { | ||
| 73 | int contig_ptes = 0; | ||
| 74 | |||
| 75 | *pgsize = size; | ||
| 76 | |||
| 77 | switch (size) { | ||
| 78 | #ifdef CONFIG_ARM64_4K_PAGES | ||
| 79 | case PUD_SIZE: | ||
| 80 | #endif | ||
| 81 | case PMD_SIZE: | ||
| 82 | contig_ptes = 1; | ||
| 83 | break; | ||
| 84 | case CONT_PMD_SIZE: | ||
| 85 | *pgsize = PMD_SIZE; | ||
| 86 | contig_ptes = CONT_PMDS; | ||
| 87 | break; | ||
| 88 | case CONT_PTE_SIZE: | ||
| 89 | *pgsize = PAGE_SIZE; | ||
| 90 | contig_ptes = CONT_PTES; | ||
| 91 | break; | ||
| 92 | } | ||
| 93 | |||
| 94 | return contig_ptes; | ||
| 95 | } | ||
| 96 | |||
| 97 | /* | ||
| 98 | * Changing some bits of contiguous entries requires us to follow a | ||
| 99 | * Break-Before-Make approach, breaking the whole contiguous set | ||
| 100 | * before we can change any entries. See ARM DDI 0487A.k_iss10775, | ||
| 101 | * "Misprogramming of the Contiguous bit", page D4-1762. | ||
| 102 | * | ||
| 103 | * This helper performs the break step. | ||
| 104 | */ | ||
| 105 | static pte_t get_clear_flush(struct mm_struct *mm, | ||
| 106 | unsigned long addr, | ||
| 107 | pte_t *ptep, | ||
| 108 | unsigned long pgsize, | ||
| 109 | unsigned long ncontig) | ||
| 110 | { | ||
| 111 | struct vm_area_struct vma = { .vm_mm = mm }; | ||
| 112 | pte_t orig_pte = huge_ptep_get(ptep); | ||
| 113 | bool valid = pte_valid(orig_pte); | ||
| 114 | unsigned long i, saddr = addr; | ||
| 115 | |||
| 116 | for (i = 0; i < ncontig; i++, addr += pgsize, ptep++) { | ||
| 117 | pte_t pte = ptep_get_and_clear(mm, addr, ptep); | ||
| 118 | |||
| 119 | /* | ||
| 120 | * If HW_AFDBM is enabled, then the HW could turn on | ||
| 121 | * the dirty bit for any page in the set, so check | ||
| 122 | * them all. All hugetlb entries are already young. | ||
| 123 | */ | ||
| 124 | if (pte_dirty(pte)) | ||
| 125 | orig_pte = pte_mkdirty(orig_pte); | ||
| 126 | } | ||
| 127 | |||
| 128 | if (valid) | ||
| 129 | flush_tlb_range(&vma, saddr, addr); | ||
| 130 | return orig_pte; | ||
| 131 | } | ||
| 132 | |||
| 133 | /* | ||
| 134 | * Changing some bits of contiguous entries requires us to follow a | ||
| 135 | * Break-Before-Make approach, breaking the whole contiguous set | ||
| 136 | * before we can change any entries. See ARM DDI 0487A.k_iss10775, | ||
| 137 | * "Misprogramming of the Contiguous bit", page D4-1762. | ||
| 138 | * | ||
| 139 | * This helper performs the break step for use cases where the | ||
| 140 | * original pte is not needed. | ||
| 141 | */ | ||
| 142 | static void clear_flush(struct mm_struct *mm, | ||
| 143 | unsigned long addr, | ||
| 144 | pte_t *ptep, | ||
| 145 | unsigned long pgsize, | ||
| 146 | unsigned long ncontig) | ||
| 147 | { | ||
| 148 | struct vm_area_struct vma = { .vm_mm = mm }; | ||
| 149 | unsigned long i, saddr = addr; | ||
| 150 | |||
| 151 | for (i = 0; i < ncontig; i++, addr += pgsize, ptep++) | ||
| 152 | pte_clear(mm, addr, ptep); | ||
| 153 | |||
| 154 | flush_tlb_range(&vma, saddr, addr); | ||
| 155 | } | ||
| 156 | |||
| 61 | void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, | 157 | void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, |
| 62 | pte_t *ptep, pte_t pte) | 158 | pte_t *ptep, pte_t pte) |
| 63 | { | 159 | { |
| 64 | size_t pgsize; | 160 | size_t pgsize; |
| 65 | int i; | 161 | int i; |
| 66 | int ncontig; | 162 | int ncontig; |
| 67 | unsigned long pfn; | 163 | unsigned long pfn, dpfn; |
| 68 | pgprot_t hugeprot; | 164 | pgprot_t hugeprot; |
| 69 | 165 | ||
| 166 | /* | ||
| 167 | * Code needs to be expanded to handle huge swap and migration | ||
| 168 | * entries. Needed for HUGETLB and MEMORY_FAILURE. | ||
| 169 | */ | ||
| 170 | WARN_ON(!pte_present(pte)); | ||
| 171 | |||
| 70 | if (!pte_cont(pte)) { | 172 | if (!pte_cont(pte)) { |
| 71 | set_pte_at(mm, addr, ptep, pte); | 173 | set_pte_at(mm, addr, ptep, pte); |
| 72 | return; | 174 | return; |
| @@ -74,17 +176,30 @@ void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, · 1 unmodified line | |||
| 74 | 176 | ||
| 75 | ncontig = find_num_contig(mm, addr, ptep, &pgsize); | 177 | ncontig = find_num_contig(mm, addr, ptep, &pgsize); |
| 76 | pfn = pte_pfn(pte); | 178 | pfn = pte_pfn(pte); |
| 77 | hugeprot = __pgprot(pte_val(pfn_pte(pfn, __pgprot(0))) ^ pte_val(pte)); | 179 | dpfn = pgsize >> PAGE_SHIFT; |
| 78 | for (i = 0; i < ncontig; i++) { | 180 | hugeprot = pte_pgprot(pte); |
| 181 | |||
| 182 | clear_flush(mm, addr, ptep, pgsize, ncontig); | ||
| 183 | |||
| 184 | for (i = 0; i < ncontig; i++, ptep++, addr += pgsize, pfn += dpfn) { | ||
| 79 | pr_debug("%s: set pte %p to 0x%llx\n", __func__, ptep, | 185 | pr_debug("%s: set pte %p to 0x%llx\n", __func__, ptep, |
| 80 | pte_val(pfn_pte(pfn, hugeprot))); | 186 | pte_val(pfn_pte(pfn, hugeprot))); |
| 81 | set_pte_at(mm, addr, ptep, pfn_pte(pfn, hugeprot)); | 187 | set_pte_at(mm, addr, ptep, pfn_pte(pfn, hugeprot)); |
| 82 | ptep++; | ||
| 83 | pfn += pgsize >> PAGE_SHIFT; | ||
| 84 | addr += pgsize; | ||
| 85 | } | 188 | } |
| 86 | } | 189 | } |
| 87 | 190 | ||
| 191 | void set_huge_swap_pte_at(struct mm_struct *mm, unsigned long addr, | ||
| 192 | pte_t *ptep, pte_t pte, unsigned long sz) | ||
| 193 | { | ||
| 194 | int i, ncontig; | ||
| 195 | size_t pgsize; | ||
| 196 | |||
| 197 | ncontig = num_contig_ptes(sz, &pgsize); | ||
| 198 | |||
| 199 | for (i = 0; i < ncontig; i++, ptep++) | ||
| 200 | set_pte(ptep, pte); | ||
| 201 | } | ||
| 202 | |||
| 88 | pte_t *huge_pte_alloc(struct mm_struct *mm, | 203 | pte_t *huge_pte_alloc(struct mm_struct *mm, |
| 89 | unsigned long addr, unsigned long sz) | 204 | unsigned long addr, unsigned long sz) |
| 90 | { | 205 | { |
| @@ -144,19 +259,28 @@ pte_t *huge_pte_offset(struct mm_struct *mm, · 53 unmodified lines | |||
| 144 | return NULL; | 259 | return NULL; |
| 145 | 260 | ||
| 146 | pud = pud_offset(pgd, addr); | 261 | pud = pud_offset(pgd, addr); |
| 147 | if (pud_none(*pud)) | 262 | if (sz != PUD_SIZE && pud_none(*pud)) |
| 148 | return NULL; | 263 | return NULL; |
| 149 | /* swap or huge page */ | 264 | /* hugepage or swap? */ |
| 150 | if (!pud_present(*pud) || pud_huge(*pud)) | 265 | if (pud_huge(*pud) || !pud_present(*pud)) |
| 151 | return (pte_t *)pud; | 266 | return (pte_t *)pud; |
| 152 | /* table; check the next level */ | 267 | /* table; check the next level */ |
| 153 | 268 | ||
| 269 | if (sz == CONT_PMD_SIZE) | ||
| 270 | addr &= CONT_PMD_MASK; | ||
| 271 | |||
| 154 | pmd = pmd_offset(pud, addr); | 272 | pmd = pmd_offset(pud, addr); |
| 155 | if (pmd_none(*pmd)) | 273 | if (!(sz == PMD_SIZE || sz == CONT_PMD_SIZE) && |
| 274 | pmd_none(*pmd)) | ||
| 156 | return NULL; | 275 | return NULL; |
| 157 | if (!pmd_present(*pmd) || pmd_huge(*pmd)) | 276 | if (pmd_huge(*pmd) || !pmd_present(*pmd)) |
| 158 | return (pte_t *)pmd; | 277 | return (pte_t *)pmd; |
| 159 | 278 | ||
| 279 | if (sz == CONT_PTE_SIZE) { | ||
| 280 | pte_t *pte = pte_offset_kernel(pmd, (addr & CONT_PTE_MASK)); | ||
| 281 | return pte; | ||
| 282 | } | ||
| 283 | |||
| 160 | return NULL; | 284 | return NULL; |
| 161 | } | 285 | } |
| 162 | 286 | ||
| @@ -176,111 +300,133 @@ pte_t arch_make_huge_pte(pte_t entry, struct vm_area_struct *vma, · 13 unmodified lines | |||
| 176 | return entry; | 300 | return entry; |
| 177 | } | 301 | } |
| 178 | 302 | ||
| 303 | void huge_pte_clear(struct mm_struct *mm, unsigned long addr, | ||
| 304 | pte_t *ptep, unsigned long sz) | ||
| 305 | { | ||
| 306 | int i, ncontig; | ||
| 307 | size_t pgsize; | ||
| 308 | |||
| 309 | ncontig = num_contig_ptes(sz, &pgsize); | ||
| 310 | |||
| 311 | for (i = 0; i < ncontig; i++, addr += pgsize, ptep++) | ||
| 312 | pte_clear(mm, addr, ptep); | ||
| 313 | } | ||
| 314 | |||
| 179 | pte_t huge_ptep_get_and_clear(struct mm_struct *mm, | 315 | pte_t huge_ptep_get_and_clear(struct mm_struct *mm, |
| 180 | unsigned long addr, pte_t *ptep) | 316 | unsigned long addr, pte_t *ptep) |
| 181 | { | 317 | { |
| 182 | pte_t pte; | 318 | int ncontig; |
| 319 | size_t pgsize; | ||
| 320 | pte_t orig_pte = huge_ptep_get(ptep); | ||
| 183 | 321 | ||
| 184 | if (pte_cont(*ptep)) { | 322 | if (!pte_cont(orig_pte)) |
| 185 | int ncontig, i; | ||
| 186 | size_t pgsize; | ||
| 187 | bool is_dirty = false; | ||
| 188 | |||
| 189 | ncontig = find_num_contig(mm, addr, ptep, &pgsize); | ||
| 190 | /* save the 1st pte to return */ | ||
| 191 | pte = ptep_get_and_clear(mm, addr, ptep); | ||
| 192 | for (i = 1, addr += pgsize; i < ncontig; ++i, addr += pgsize) { | ||
| 193 | /* | ||
| 194 | * If HW_AFDBM is enabled, then the HW could | ||
| 195 | * turn on the dirty bit for any of the page | ||
| 196 | * in the set, so check them all. | ||
| 197 | */ | ||
| 198 | ++ptep; | ||
| 199 | if (pte_dirty(ptep_get_and_clear(mm, addr, ptep))) | ||
| 200 | is_dirty = true; | ||
| 201 | } | ||
| 202 | if (is_dirty) | ||
| 203 | return pte_mkdirty(pte); | ||
| 204 | else | ||
| 205 | return pte; | ||
| 206 | } else { | ||
| 207 | return ptep_get_and_clear(mm, addr, ptep); | 323 | return ptep_get_and_clear(mm, addr, ptep); |
| 208 | } | 324 | |
| 325 | ncontig = find_num_contig(mm, addr, ptep, &pgsize); | ||
| 326 | |||
| 327 | return get_clear_flush(mm, addr, ptep, pgsize, ncontig); | ||
| 209 | } | 328 | } |
| 210 | 329 | ||
| 211 | int huge_ptep_set_access_flags(struct vm_area_struct *vma, | 330 | int huge_ptep_set_access_flags(struct vm_area_struct *vma, |
| 212 | unsigned long addr, pte_t *ptep, | 331 | unsigned long addr, pte_t *ptep, |
| 213 | pte_t pte, int dirty) | 332 | pte_t pte, int dirty) |
| 214 | { | 333 | { |
| 215 | if (pte_cont(pte)) { | 334 | int ncontig, i, changed = 0; |
| 216 | int ncontig, i, changed = 0; | 335 | size_t pgsize = 0; |
| 217 | size_t pgsize = 0; | 336 | unsigned long pfn = pte_pfn(pte), dpfn; |
| 218 | unsigned long pfn = pte_pfn(pte); | 337 | pgprot_t hugeprot; |
| 219 | /* Select all bits except the pfn */ | 338 | pte_t orig_pte; |
| 220 | pgprot_t hugeprot = | 339 | |
| 221 | __pgprot(pte_val(pfn_pte(pfn, __pgprot(0))) ^ | 340 | if (!pte_cont(pte)) |
| 222 | pte_val(pte)); | ||
| 223 | |||
| 224 | pfn = pte_pfn(pte); | ||
| 225 | ncontig = find_num_contig(vma->vm_mm, addr, ptep, | ||
| 226 | &pgsize); | ||
| 227 | for (i = 0; i < ncontig; ++i, ++ptep, addr += pgsize) { | ||
| 228 | changed |= ptep_set_access_flags(vma, addr, ptep, | ||
| 229 | pfn_pte(pfn, | ||
| 230 | hugeprot), | ||
| 231 | dirty); | ||
| 232 | pfn += pgsize >> PAGE_SHIFT; | ||
| 233 | } | ||
| 234 | return changed; | ||
| 235 | } else { | ||
| 236 | return ptep_set_access_flags(vma, addr, ptep, pte, dirty); | 341 | return ptep_set_access_flags(vma, addr, ptep, pte, dirty); |
| 237 | } | 342 | |
| 343 | ncontig = find_num_contig(vma->vm_mm, addr, ptep, &pgsize); | ||
| 344 | dpfn = pgsize >> PAGE_SHIFT; | ||
| 345 | |||
| 346 | orig_pte = get_clear_flush(vma->vm_mm, addr, ptep, pgsize, ncontig); | ||
| 347 | if (!pte_same(orig_pte, pte)) | ||
| 348 | changed = 1; | ||
| 349 | |||
| 350 | /* Make sure we don't lose the dirty state */ | ||
| 351 | if (pte_dirty(orig_pte)) | ||
| 352 | pte = pte_mkdirty(pte); | ||
| 353 | |||
| 354 | hugeprot = pte_pgprot(pte); | ||
| 355 | for (i = 0; i < ncontig; i++, ptep++, addr += pgsize, pfn += dpfn) | ||
| 356 | set_pte_at(vma->vm_mm, addr, ptep, pfn_pte(pfn, hugeprot)); | ||
| 357 | |||
| 358 | return changed; | ||
| 238 | } | 359 | } |
| 239 | 360 | ||
| 240 | void huge_ptep_set_wrprotect(struct mm_struct *mm, | 361 | void huge_ptep_set_wrprotect(struct mm_struct *mm, |
| 241 | unsigned long addr, pte_t *ptep) | 362 | unsigned long addr, pte_t *ptep) |
| 242 | { | 363 | { |
| 243 | if (pte_cont(*ptep)) { | 364 | unsigned long pfn, dpfn; |
| 244 | int ncontig, i; | 365 | pgprot_t hugeprot; |
| 245 | size_t pgsize = 0; | 366 | int ncontig, i; |
| 246 | 367 | size_t pgsize; | |
| 247 | ncontig = find_num_contig(mm, addr, ptep, &pgsize); | 368 | pte_t pte; |
| 248 | for (i = 0; i < ncontig; ++i, ++ptep, addr += pgsize) | 369 | |
| 249 | ptep_set_wrprotect(mm, addr, ptep); | 370 | if (!pte_cont(*ptep)) { |
| 250 | } else { | ||
| 251 | ptep_set_wrprotect(mm, addr, ptep); | 371 | ptep_set_wrprotect(mm, addr, ptep); |
| 372 | return; | ||
| 252 | } | 373 | } |
| 374 | |||
| 375 | ncontig = find_num_contig(mm, addr, ptep, &pgsize); | ||
| 376 | dpfn = pgsize >> PAGE_SHIFT; | ||
| 377 | |||
| 378 | pte = get_clear_flush(mm, addr, ptep, pgsize, ncontig); | ||
| 379 | pte = pte_wrprotect(pte); | ||
| 380 | |||
| 381 | hugeprot = pte_pgprot(pte); | ||
| 382 | pfn = pte_pfn(pte); | ||
| 383 | |||
| 384 | for (i = 0; i < ncontig; i++, ptep++, addr += pgsize, pfn += dpfn) | ||
| 385 | set_pte_at(mm, addr, ptep, pfn_pte(pfn, hugeprot)); | ||
| 253 | } | 386 | } |
| 254 | 387 | ||
| 255 | void huge_ptep_clear_flush(struct vm_area_struct *vma, | 388 | void huge_ptep_clear_flush(struct vm_area_struct *vma, |
| 256 | unsigned long addr, pte_t *ptep) | 389 | unsigned long addr, pte_t *ptep) |
| 257 | { | 390 | { |
| 258 | if (pte_cont(*ptep)) { | 391 | size_t pgsize; |
| 259 | int ncontig, i; | 392 | int ncontig; |
| 260 | size_t pgsize = 0; | 393 | |
| 261 | 394 | if (!pte_cont(*ptep)) { | |
| 262 | ncontig = find_num_contig(vma->vm_mm, addr, ptep, | ||
| 263 | &pgsize); | ||
| 264 | for (i = 0; i < ncontig; ++i, ++ptep, addr += pgsize) | ||
| 265 | ptep_clear_flush(vma, addr, ptep); | ||
| 266 | } else { | ||
| 267 | ptep_clear_flush(vma, addr, ptep); | 395 | ptep_clear_flush(vma, addr, ptep); |
| 396 | return; | ||
| 268 | } | 397 | } |
| 398 | |||
| 399 | ncontig = find_num_contig(vma->vm_mm, addr, ptep, &pgsize); | ||
| 400 | clear_flush(vma->vm_mm, addr, ptep, pgsize, ncontig); | ||
| 269 | } | 401 | } |
| 270 | 402 | ||
| 271 | static __init int setup_hugepagesz(char *opt) | 403 | static __init int setup_hugepagesz(char *opt) |
| 272 | { | 404 | { |
| 273 | unsigned long ps = memparse(opt, &opt); | 405 | unsigned long ps = memparse(opt, &opt); |
| 274 | 406 | ||
| 275 | if (ps == PMD_SIZE) { | 407 | switch (ps) { |
| 276 | hugetlb_add_hstate(PMD_SHIFT - PAGE_SHIFT); | 408 | #ifdef CONFIG_ARM64_4K_PAGES |
| 277 | } else if (ps == PUD_SIZE) { | 409 | case PUD_SIZE: |
| 278 | hugetlb_add_hstate(PUD_SHIFT - PAGE_SHIFT); | 410 | #endif |
| 279 | } else { | 411 | case PMD_SIZE * CONT_PMDS: |
| 280 | hugetlb_bad_size(); | 412 | case PMD_SIZE: |
| 281 | pr_err("hugepagesz: Unsupported page size %lu K\n", ps >> 10); | 413 | case PAGE_SIZE * CONT_PTES: |
| 282 | return 0; | 414 | hugetlb_add_hstate(ilog2(ps) - PAGE_SHIFT); |
| 415 | return 1; | ||
| 283 | } | 416 | } |
| 284 | return 1; | 417 | |
| 418 | hugetlb_bad_size(); | ||
| 419 | pr_err("hugepagesz: Unsupported page size %lu K\n", ps >> 10); | ||
| 420 | return 0; | ||
| 285 | } | 421 | } |
| 286 | __setup("hugepagesz=", setup_hugepagesz); | 422 | __setup("hugepagesz=", setup_hugepagesz); |
| 423 | |||
| 424 | #ifdef CONFIG_ARM64_64K_PAGES | ||
| 425 | static __init int add_default_hugepagesz(void) | ||
| 426 | { | ||
| 427 | if (size_to_hstate(CONT_PTES * PAGE_SIZE) == NULL) | ||
| 428 | hugetlb_add_hstate(CONT_PTE_SHIFT); | ||
| 429 | return 0; | ||
| 430 | } | ||
| 431 | arch_initcall(add_default_hugepagesz); | ||
| 432 | #endif | ||
| @@ -588,7 +588,8 @@ void acpi_configure_pmsi_domain(struct device *dev) · 587 unmodified lines | |||
| 588 | dev_set_msi_domain(dev, msi_domain); | 588 | dev_set_msi_domain(dev, msi_domain); |
| 589 | } | 589 | } |
| 590 | 590 | ||
| 591 | static int __get_pci_rid(struct pci_dev *pdev, u16 alias, void *data) | 591 | static int __maybe_unused __get_pci_rid(struct pci_dev *pdev, u16 alias, |
| 592 | void *data) | ||
| 592 | { | 593 | { |
| 593 | u32 *rid = data; | 594 | u32 *rid = data; |
| 594 | 595 | ||
| @@ -633,8 +634,7 @@ int iort_add_device_replay(const struct iommu_ops *ops, struct device *dev) · 38 unmodified lines | |||
| 633 | { | 634 | { |
| 634 | int err = 0; | 635 | int err = 0; |
| 635 | 636 | ||
| 636 | if (!IS_ERR_OR_NULL(ops) && ops->add_device && dev->bus && | 637 | if (ops->add_device && dev->bus && !dev->iommu_group) |
| 637 | !dev->iommu_group) | ||
| 638 | err = ops->add_device(dev); | 638 | err = ops->add_device(dev); |
| 639 | 639 | ||
| 640 | return err; | 640 | return err; |
| @@ -648,36 +648,49 @@ int iort_add_device_replay(const struct iommu_ops *ops, struct device *dev) · 7 unmodified lines | |||
| 648 | { return 0; } | 648 | { return 0; } |
| 649 | #endif | 649 | #endif |
| 650 | 650 | ||
| 651 | static const struct iommu_ops *iort_iommu_xlate(struct device *dev, | 651 | static int iort_iommu_xlate(struct device *dev, struct acpi_iort_node *node, |
| 652 | struct acpi_iort_node *node, | 652 | u32 streamid) |
| 653 | u32 streamid) | ||
| 654 | { | 653 | { |
| 655 | const struct iommu_ops *ops = NULL; | 654 | const struct iommu_ops *ops; |
| 656 | int ret = -ENODEV; | ||
| 657 | struct fwnode_handle *iort_fwnode; | 655 | struct fwnode_handle *iort_fwnode; |
| 658 | 656 | ||
| 659 | if (node) { | 657 | if (!node) |
| 660 | iort_fwnode = iort_get_fwnode(node); | 658 | return -ENODEV; |
| 661 | if (!iort_fwnode) | ||
| 662 | return NULL; | ||
| 663 | 659 | ||
| 664 | ops = iommu_ops_from_fwnode(iort_fwnode); | 660 | iort_fwnode = iort_get_fwnode(node); |
| 665 | /* | 661 | if (!iort_fwnode) |
| 666 | * If the ops look-up fails, this means that either | 662 | return -ENODEV; |
| 667 | * the SMMU drivers have not been probed yet or that | ||
| 668 | * the SMMU drivers are not built in the kernel; | ||
| 669 | * Depending on whether the SMMU drivers are built-in | ||
| 670 | * in the kernel or not, defer the IOMMU configuration | ||
| 671 | * or just abort it. | ||
| 672 | */ | ||
| 673 | if (!ops) | ||
| 674 | return iort_iommu_driver_enabled(node->type) ? | ||
| 675 | ERR_PTR(-EPROBE_DEFER) : NULL; | ||
| 676 | 663 | ||
| 677 | ret = arm_smmu_iort_xlate(dev, streamid, iort_fwnode, ops); | 664 | /* |
| 678 | } | 665 | * If the ops look-up fails, this means that either |
| 666 | * the SMMU drivers have not been probed yet or that | ||
| 667 | * the SMMU drivers are not built in the kernel; | ||
| 668 | * Depending on whether the SMMU drivers are built-in | ||
| 669 | * in the kernel or not, defer the IOMMU configuration | ||
| 670 | * or just abort it. | ||
| 671 | */ | ||
| 672 | ops = iommu_ops_from_fwnode(iort_fwnode); | ||
| 673 | if (!ops) | ||
| 674 | return iort_iommu_driver_enabled(node->type) ? | ||
| 675 | -EPROBE_DEFER : -ENODEV; | ||
| 679 | 676 | ||
| 680 | return ret ? NULL : ops; | 677 | return arm_smmu_iort_xlate(dev, streamid, iort_fwnode, ops); |
| 678 | } | ||
| 679 | |||
| 680 | struct iort_pci_alias_info { | ||
| 681 | struct device *dev; | ||
| 682 | struct acpi_iort_node *node; | ||
| 683 | }; | ||
| 684 | |||
| 685 | static int iort_pci_iommu_init(struct pci_dev *pdev, u16 alias, void *data) | ||
| 686 | { | ||
| 687 | struct iort_pci_alias_info *info = data; | ||
| 688 | struct acpi_iort_node *parent; | ||
| 689 | u32 streamid; | ||
| 690 | |||
| 691 | parent = iort_node_map_id(info->node, alias, &streamid, | ||
| 692 | IORT_IOMMU_TYPE); | ||
| 693 | return iort_iommu_xlate(info->dev, parent, streamid); | ||
| 681 | } | 694 | } |
| 682 | 695 | ||
| 683 | /** | 696 | /** |
| @@ -713,9 +726,9 @@ void iort_set_dma_mask(struct device *dev) · 29 unmodified lines | |||
| 713 | const struct iommu_ops *iort_iommu_configure(struct device *dev) | 726 | const struct iommu_ops *iort_iommu_configure(struct device *dev) |
| 714 | { | 727 | { |
| 715 | struct acpi_iort_node *node, *parent; | 728 | struct acpi_iort_node *node, *parent; |
| 716 | const struct iommu_ops *ops = NULL; | 729 | const struct iommu_ops *ops; |
| 717 | u32 streamid = 0; | 730 | u32 streamid = 0; |
| 718 | int err; | 731 | int err = -ENODEV; |
| 719 | 732 | ||
| 720 | /* | 733 | /* |
| 721 | * If we already translated the fwspec there | 734 | * If we already translated the fwspec there |
| @@ -727,21 +740,16 @@ const struct iommu_ops *iort_iommu_configure(struct device *dev) · 5 unmodified lines | |||
| 727 | 740 | ||
| 728 | if (dev_is_pci(dev)) { | 741 | if (dev_is_pci(dev)) { |
| 729 | struct pci_bus *bus = to_pci_dev(dev)->bus; | 742 | struct pci_bus *bus = to_pci_dev(dev)->bus; |
| 730 | u32 rid; | 743 | struct iort_pci_alias_info info = { .dev = dev }; |
| 731 | |||
| 732 | pci_for_each_dma_alias(to_pci_dev(dev), __get_pci_rid, | ||
| 733 | &rid); | ||
| 734 | 744 | ||
| 735 | node = iort_scan_node(ACPI_IORT_NODE_PCI_ROOT_COMPLEX, | 745 | node = iort_scan_node(ACPI_IORT_NODE_PCI_ROOT_COMPLEX, |
| 736 | iort_match_node_callback, &bus->dev); | 746 | iort_match_node_callback, &bus->dev); |
| 737 | if (!node) | 747 | if (!node) |
| 738 | return NULL; | 748 | return NULL; |
| 739 | 749 | ||
| 740 | parent = iort_node_map_id(node, rid, &streamid, | 750 | info.node = node; |
| 741 | IORT_IOMMU_TYPE); | 751 | err = pci_for_each_dma_alias(to_pci_dev(dev), |
| 742 | 752 | iort_pci_iommu_init, &info); | |
| 743 | ops = iort_iommu_xlate(dev, parent, streamid); | ||
| 744 | |||
| 745 | } else { | 753 | } else { |
| 746 | int i = 0; | 754 | int i = 0; |
| 747 | 755 | ||
| @@ -750,31 +758,30 @@ const struct iommu_ops *iort_iommu_configure(struct device *dev) · 2 unmodified lines | |||
| 750 | if (!node) | 758 | if (!node) |
| 751 | return NULL; | 759 | return NULL; |
| 752 | 760 | ||
| 753 | parent = iort_node_map_platform_id(node, &streamid, | 761 | do { |
| 754 | IORT_IOMMU_TYPE, i++); | ||
| 755 | |||
| 756 | while (parent) { | ||
| 757 | ops = iort_iommu_xlate(dev, parent, streamid); | ||
| 758 | if (IS_ERR_OR_NULL(ops)) | ||
| 759 | return ops; | ||
| 760 | |||
| 761 | parent = iort_node_map_platform_id(node, &streamid, | 762 | parent = iort_node_map_platform_id(node, &streamid, |
| 762 | IORT_IOMMU_TYPE, | 763 | IORT_IOMMU_TYPE, |
| 763 | i++); | 764 | i++); |
| 764 | } | 765 | |
| 766 | if (parent) | ||
| 767 | err = iort_iommu_xlate(dev, parent, streamid); | ||
| 768 | } while (parent && !err); | ||
| 765 | } | 769 | } |
| 766 | 770 | ||
| 767 | /* | 771 | /* |
| 768 | * If we have reason to believe the IOMMU driver missed the initial | 772 | * If we have reason to believe the IOMMU driver missed the initial |
| 769 | * add_device callback for dev, replay it to get things in order. | 773 | * add_device callback for dev, replay it to get things in order. |
| 770 | */ | 774 | */ |
| 771 | err = iort_add_device_replay(ops, dev); | 775 | if (!err) { |
| 772 | if (err) | 776 | ops = iort_fwspec_iommu_ops(dev->iommu_fwspec); |
| 773 | ops = ERR_PTR(err); | 777 | err = iort_add_device_replay(ops, dev); |
| 778 | } | ||
| 774 | 779 | ||
| 775 | /* Ignore all other errors apart from EPROBE_DEFER */ | 780 | /* Ignore all other errors apart from EPROBE_DEFER */ |
| 776 | if (IS_ERR(ops) && (PTR_ERR(ops) != -EPROBE_DEFER)) { | 781 | if (err == -EPROBE_DEFER) { |
| 777 | dev_dbg(dev, "Adding to IOMMU failed: %ld\n", PTR_ERR(ops)); | 782 | ops = ERR_PTR(err); |
| 783 | } else if (err) { | ||
| 784 | dev_dbg(dev, "Adding to IOMMU failed: %d\n", err); | ||
| 778 | ops = NULL; | 785 | ops = NULL; |
| 779 | } | 786 | } |
| 780 | 787 | ||
| @@ -908,6 +915,27 @@ static bool __init arm_smmu_v3_is_coherent(struct acpi_iort_node *node) · 127 unmodified lines | |||
| 908 | return smmu->flags & ACPI_IORT_SMMU_V3_COHACC_OVERRIDE; | 915 | return smmu->flags & ACPI_IORT_SMMU_V3_COHACC_OVERRIDE; |
| 909 | } | 916 | } |
| 910 | 917 | ||
| 918 | #if defined(CONFIG_ACPI_NUMA) && defined(ACPI_IORT_SMMU_V3_PXM_VALID) | ||
| 919 | /* | ||
| 920 | * set numa proximity domain for smmuv3 device | ||
| 921 | */ | ||
| 922 | static void __init arm_smmu_v3_set_proximity(struct device *dev, | ||
| 923 | struct acpi_iort_node *node) | ||
| 924 | { | ||
| 925 | struct acpi_iort_smmu_v3 *smmu; | ||
| 926 | |||
| 927 | smmu = (struct acpi_iort_smmu_v3 *)node->node_data; | ||
| 928 | if (smmu->flags & ACPI_IORT_SMMU_V3_PXM_VALID) { | ||
| 929 | set_dev_node(dev, acpi_map_pxm_to_node(smmu->pxm)); | ||
| 930 | pr_info("SMMU-v3[%llx] Mapped to Proximity domain %d\n", | ||
| 931 | smmu->base_address, | ||
| 932 | smmu->pxm); | ||
| 933 | } | ||
| 934 | } | ||
| 935 | #else | ||
| 936 | #define arm_smmu_v3_set_proximity NULL | ||
| 937 | #endif | ||
| 938 | |||
| 911 | static int __init arm_smmu_count_resources(struct acpi_iort_node *node) | 939 | static int __init arm_smmu_count_resources(struct acpi_iort_node *node) |
| 912 | { | 940 | { |
| 913 | struct acpi_iort_smmu *smmu; | 941 | struct acpi_iort_smmu *smmu; |
| @@ -977,13 +1005,16 @@ struct iort_iommu_config { · 63 unmodified lines | |||
| 977 | int (*iommu_count_resources)(struct acpi_iort_node *node); | 1005 | int (*iommu_count_resources)(struct acpi_iort_node *node); |
| 978 | void (*iommu_init_resources)(struct resource *res, | 1006 | void (*iommu_init_resources)(struct resource *res, |
| 979 | struct acpi_iort_node *node); | 1007 | struct acpi_iort_node *node); |
| 1008 | void (*iommu_set_proximity)(struct device *dev, | ||
| 1009 | struct acpi_iort_node *node); | ||
| 980 | }; | 1010 | }; |
| 981 | 1011 | ||
| 982 | static const struct iort_iommu_config iort_arm_smmu_v3_cfg __initconst = { | 1012 | static const struct iort_iommu_config iort_arm_smmu_v3_cfg __initconst = { |
| 983 | .name = "arm-smmu-v3", | 1013 | .name = "arm-smmu-v3", |
| 984 | .iommu_is_coherent = arm_smmu_v3_is_coherent, | 1014 | .iommu_is_coherent = arm_smmu_v3_is_coherent, |
| 985 | .iommu_count_resources = arm_smmu_v3_count_resources, | 1015 | .iommu_count_resources = arm_smmu_v3_count_resources, |
| 986 | .iommu_init_resources = arm_smmu_v3_init_resources | 1016 | .iommu_init_resources = arm_smmu_v3_init_resources, |
| 1017 | .iommu_set_proximity = arm_smmu_v3_set_proximity, | ||
| 987 | }; | 1018 | }; |
| 988 | 1019 | ||
| 989 | static const struct iort_iommu_config iort_arm_smmu_cfg __initconst = { | 1020 | static const struct iort_iommu_config iort_arm_smmu_cfg __initconst = { |
| @@ -1028,6 +1059,9 @@ static int __init iort_add_smmu_platform_device(struct acpi_iort_node *node) · 38 unmodified lines | |||
| 1028 | if (!pdev) | 1059 | if (!pdev) |
| 1029 | return -ENOMEM; | 1060 | return -ENOMEM; |
| 1030 | 1061 | ||
| 1062 | if (ops->iommu_set_proximity) | ||
| 1063 | ops->iommu_set_proximity(&pdev->dev, node); | ||
| 1064 | |||
| 1031 | count = ops->iommu_count_resources(node); | 1065 | count = ops->iommu_count_resources(node); |
| 1032 | 1066 | ||
| 1033 | r = kcalloc(count, sizeof(*r), GFP_KERNEL); | 1067 | r = kcalloc(count, sizeof(*r), GFP_KERNEL); |
| @@ -11,6 +11,7 @@ · 10 unmodified lines | |||
| 11 | */ | 11 | */ |
| 12 | #include <linux/efi.h> | 12 | #include <linux/efi.h> |
| 13 | #include <asm/efi.h> | 13 | #include <asm/efi.h> |
| 14 | #include <asm/memory.h> | ||
| 14 | #include <asm/sections.h> | 15 | #include <asm/sections.h> |
| 15 | #include <asm/sysreg.h> | 16 | #include <asm/sysreg.h> |
| 16 | 17 | ||
| @@ -81,9 +82,10 @@ efi_status_t handle_kernel_image(efi_system_table_t *sys_table_arg, · 64 unmodified lines | |||
| 81 | /* | 82 | /* |
| 82 | * If CONFIG_DEBUG_ALIGN_RODATA is not set, produce a | 83 | * If CONFIG_DEBUG_ALIGN_RODATA is not set, produce a |
| 83 | * displacement in the interval [0, MIN_KIMG_ALIGN) that | 84 | * displacement in the interval [0, MIN_KIMG_ALIGN) that |
| 84 | * is a multiple of the minimal segment alignment (SZ_64K) | 85 | * doesn't violate this kernel's de-facto alignment |
| 86 | * constraints. | ||
| 85 | */ | 87 | */ |
| 86 | u32 mask = (MIN_KIMG_ALIGN - 1) & ~(SZ_64K - 1); | 88 | u32 mask = (MIN_KIMG_ALIGN - 1) & ~(EFI_KIMG_ALIGN - 1); |
| 87 | u32 offset = !IS_ENABLED(CONFIG_DEBUG_ALIGN_RODATA) ? | 89 | u32 offset = !IS_ENABLED(CONFIG_DEBUG_ALIGN_RODATA) ? |
| 88 | (phys_seed >> 32) & mask : TEXT_OFFSET; | 90 | (phys_seed >> 32) & mask : TEXT_OFFSET; |
| 89 | 91 | ||
| @@ -47,6 +47,9 @@ armpmu_map_cache_event(const unsigned (*cache_map) · 46 unmodified lines | |||
| 47 | if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX) | 47 | if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX) |
| 48 | return -EINVAL; | 48 | return -EINVAL; |
| 49 | 49 | ||
| 50 | if (!cache_map) | ||
| 51 | return -ENOENT; | ||
| 52 | |||
| 50 | ret = (int)(*cache_map)[cache_type][cache_op][cache_result]; | 53 | ret = (int)(*cache_map)[cache_type][cache_op][cache_result]; |
| 51 | 54 | ||
| 52 | if (ret == CACHE_OP_UNSUPPORTED) | 55 | if (ret == CACHE_OP_UNSUPPORTED) |
| @@ -63,6 +66,9 @@ armpmu_map_hw_event(const unsigned (*event_map)[PERF_COUNT_HW_MAX], u64 config) · 10 unmodified lines | |||
| 63 | if (config >= PERF_COUNT_HW_MAX) | 66 | if (config >= PERF_COUNT_HW_MAX) |
| 64 | return -EINVAL; | 67 | return -EINVAL; |
| 65 | 68 | ||
| 69 | if (!event_map) | ||
| 70 | return -ENOENT; | ||
| 71 | |||
| 66 | mapping = (*event_map)[config]; | 72 | mapping = (*event_map)[config]; |
| 67 | return mapping == HW_OP_UNSUPPORTED ? -ENOENT : mapping; | 73 | return mapping == HW_OP_UNSUPPORTED ? -ENOENT : mapping; |
| 68 | } | 74 | } |
| @@ -1147,7 +1147,6 @@ xgene_pmu_dev_add(struct xgene_pmu *xgene_pmu, struct xgene_pmu_dev_ctx *ctx) · 1146 unmodified lines | |||
| 1147 | { | 1147 | { |
| 1148 | struct device *dev = xgene_pmu->dev; | 1148 | struct device *dev = xgene_pmu->dev; |
| 1149 | struct xgene_pmu_dev *pmu; | 1149 | struct xgene_pmu_dev *pmu; |
| 1150 | int rc; | ||
| 1151 | 1150 | ||
| 1152 | pmu = devm_kzalloc(dev, sizeof(*pmu), GFP_KERNEL); | 1151 | pmu = devm_kzalloc(dev, sizeof(*pmu), GFP_KERNEL); |
| 1153 | if (!pmu) | 1152 | if (!pmu) |
| @@ -1159,7 +1158,7 @@ xgene_pmu_dev_add(struct xgene_pmu *xgene_pmu, struct xgene_pmu_dev_ctx *ctx) · 5 unmodified lines | |||
| 1159 | switch (pmu->inf->type) { | 1158 | switch (pmu->inf->type) { |
| 1160 | case PMU_TYPE_L3C: | 1159 | case PMU_TYPE_L3C: |
| 1161 | if (!(xgene_pmu->l3c_active_mask & pmu->inf->enable_mask)) | 1160 | if (!(xgene_pmu->l3c_active_mask & pmu->inf->enable_mask)) |
| 1162 | goto dev_err; | 1161 | return -ENODEV; |
| 1163 | if (xgene_pmu->version == PCP_PMU_V3) | 1162 | if (xgene_pmu->version == PCP_PMU_V3) |
| 1164 | pmu->attr_groups = l3c_pmu_v3_attr_groups; | 1163 | pmu->attr_groups = l3c_pmu_v3_attr_groups; |
| 1165 | else | 1164 | else |
| @@ -1177,7 +1176,7 @@ xgene_pmu_dev_add(struct xgene_pmu *xgene_pmu, struct xgene_pmu_dev_ctx *ctx) · 11 unmodified lines | |||
| 1177 | break; | 1176 | break; |
| 1178 | case PMU_TYPE_MCB: | 1177 | case PMU_TYPE_MCB: |
| 1179 | if (!(xgene_pmu->mcb_active_mask & pmu->inf->enable_mask)) | 1178 | if (!(xgene_pmu->mcb_active_mask & pmu->inf->enable_mask)) |
| 1180 | goto dev_err; | 1179 | return -ENODEV; |
| 1181 | if (xgene_pmu->version == PCP_PMU_V3) | 1180 | if (xgene_pmu->version == PCP_PMU_V3) |
| 1182 | pmu->attr_groups = mcb_pmu_v3_attr_groups; | 1181 | pmu->attr_groups = mcb_pmu_v3_attr_groups; |
| 1183 | else | 1182 | else |
| @@ -1185,7 +1184,7 @@ xgene_pmu_dev_add(struct xgene_pmu *xgene_pmu, struct xgene_pmu_dev_ctx *ctx) · 1 unmodified line | |||
| 1185 | break; | 1184 | break; |
| 1186 | case PMU_TYPE_MC: | 1185 | case PMU_TYPE_MC: |
| 1187 | if (!(xgene_pmu->mc_active_mask & pmu->inf->enable_mask)) | 1186 | if (!(xgene_pmu->mc_active_mask & pmu->inf->enable_mask)) |
| 1188 | goto dev_err; | 1187 | return -ENODEV; |
| 1189 | if (xgene_pmu->version == PCP_PMU_V3) | 1188 | if (xgene_pmu->version == PCP_PMU_V3) |
| 1190 | pmu->attr_groups = mc_pmu_v3_attr_groups; | 1189 | pmu->attr_groups = mc_pmu_v3_attr_groups; |
| 1191 | else | 1190 | else |
| @@ -1195,19 +1194,14 @@ xgene_pmu_dev_add(struct xgene_pmu *xgene_pmu, struct xgene_pmu_dev_ctx *ctx) · 3 unmodified lines | |||
| 1195 | return -EINVAL; | 1194 | return -EINVAL; |
| 1196 | } | 1195 | } |
| 1197 | 1196 | ||
| 1198 | rc = xgene_init_perf(pmu, ctx->name); | 1197 | if (xgene_init_perf(pmu, ctx->name)) { |
| 1199 | if (rc) { | ||
| 1200 | dev_err(dev, "%s PMU: Failed to init perf driver\n", ctx->name); | 1198 | dev_err(dev, "%s PMU: Failed to init perf driver\n", ctx->name); |
| 1201 | goto dev_err; | 1199 | return -ENODEV; |
| 1202 | } | 1200 | } |
| 1203 | 1201 | ||
| 1204 | dev_info(dev, "%s PMU registered\n", ctx->name); | 1202 | dev_info(dev, "%s PMU registered\n", ctx->name); |
| 1205 | 1203 | ||
| 1206 | return rc; | 1204 | return 0; |
| 1207 | |||
| 1208 | dev_err: | ||
| 1209 | devm_kfree(dev, pmu); | ||
| 1210 | return -ENODEV; | ||
| 1211 | } | 1205 | } |
| 1212 | 1206 | ||
| 1213 | static void _xgene_pmu_isr(int irq, struct xgene_pmu_dev *pmu_dev) | 1207 | static void _xgene_pmu_isr(int irq, struct xgene_pmu_dev *pmu_dev) |
| @@ -1515,13 +1509,13 @@ xgene_pmu_dev_ctx *acpi_get_pmu_hw_inf(struct xgene_pmu *xgene_pmu, · 301 unmodified lines | |||
| 1515 | acpi_dev_free_resource_list(&resource_list); | 1509 | acpi_dev_free_resource_list(&resource_list); |
| 1516 | if (rc < 0) { | 1510 | if (rc < 0) { |
| 1517 | dev_err(dev, "PMU type %d: No resource address found\n", type); | 1511 | dev_err(dev, "PMU type %d: No resource address found\n", type); |
| 1518 | goto err; | 1512 | return NULL; |
| 1519 | } | 1513 | } |
| 1520 | 1514 | ||
| 1521 | dev_csr = devm_ioremap_resource(dev, &res); | 1515 | dev_csr = devm_ioremap_resource(dev, &res); |
| 1522 | if (IS_ERR(dev_csr)) { | 1516 | if (IS_ERR(dev_csr)) { |
| 1523 | dev_err(dev, "PMU type %d: Fail to map resource\n", type); | 1517 | dev_err(dev, "PMU type %d: Fail to map resource\n", type); |
| 1524 | goto err; | 1518 | return NULL; |
| 1525 | } | 1519 | } |
| 1526 | 1520 | ||
| 1527 | /* A PMU device node without enable-bit-index is always enabled */ | 1521 | /* A PMU device node without enable-bit-index is always enabled */ |
| @@ -1535,7 +1529,7 @@ xgene_pmu_dev_ctx *acpi_get_pmu_hw_inf(struct xgene_pmu *xgene_pmu, · 7 unmodified lines | |||
| 1535 | ctx->name = xgene_pmu_dev_name(dev, type, enable_bit); | 1529 | ctx->name = xgene_pmu_dev_name(dev, type, enable_bit); |
| 1536 | if (!ctx->name) { | 1530 | if (!ctx->name) { |
| 1537 | dev_err(dev, "PMU type %d: Fail to get device name\n", type); | 1531 | dev_err(dev, "PMU type %d: Fail to get device name\n", type); |
| 1538 | goto err; | 1532 | return NULL; |
| 1539 | } | 1533 | } |
| 1540 | inf = &ctx->inf; | 1534 | inf = &ctx->inf; |
| 1541 | inf->type = type; | 1535 | inf->type = type; |
| @@ -1543,9 +1537,6 @@ xgene_pmu_dev_ctx *acpi_get_pmu_hw_inf(struct xgene_pmu *xgene_pmu, · 1 unmodified line | |||
| 1543 | inf->enable_mask = 1 << enable_bit; | 1537 | inf->enable_mask = 1 << enable_bit; |
| 1544 | 1538 | ||
| 1545 | return ctx; | 1539 | return ctx; |
| 1546 | err: | ||
| 1547 | devm_kfree(dev, ctx); | ||
| 1548 | return NULL; | ||
| 1549 | } | 1540 | } |
| 1550 | 1541 | ||
| 1551 | static const struct acpi_device_id xgene_pmu_acpi_type_match[] = { | 1542 | static const struct acpi_device_id xgene_pmu_acpi_type_match[] = { |
| @@ -1663,20 +1654,20 @@ xgene_pmu_dev_ctx *fdt_get_pmu_hw_inf(struct xgene_pmu *xgene_pmu, · 111 unmodified lines | |||
| 1663 | void __iomem *dev_csr; | 1654 | void __iomem *dev_csr; |
| 1664 | struct resource res; | 1655 | struct resource res; |
| 1665 | int enable_bit; | 1656 | int enable_bit; |
| 1666 | int rc; | ||
| 1667 | 1657 | ||
| 1668 | ctx = devm_kzalloc(dev, sizeof(*ctx), GFP_KERNEL); | 1658 | ctx = devm_kzalloc(dev, sizeof(*ctx), GFP_KERNEL); |
| 1669 | if (!ctx) | 1659 | if (!ctx) |
| 1670 | return NULL; | 1660 | return NULL; |
| 1671 | rc = of_address_to_resource(np, 0, &res); | 1661 | |
| 1672 | if (rc < 0) { | 1662 | if (of_address_to_resource(np, 0, &res) < 0) { |
| 1673 | dev_err(dev, "PMU type %d: No resource address found\n", type); | 1663 | dev_err(dev, "PMU type %d: No resource address found\n", type); |
| 1674 | goto err; | 1664 | return NULL; |
| 1675 | } | 1665 | } |
| 1666 | |||
| 1676 | dev_csr = devm_ioremap_resource(dev, &res); | 1667 | dev_csr = devm_ioremap_resource(dev, &res); |
| 1677 | if (IS_ERR(dev_csr)) { | 1668 | if (IS_ERR(dev_csr)) { |
| 1678 | dev_err(dev, "PMU type %d: Fail to map resource\n", type); | 1669 | dev_err(dev, "PMU type %d: Fail to map resource\n", type); |
| 1679 | goto err; | 1670 | return NULL; |
| 1680 | } | 1671 | } |
| 1681 | 1672 | ||
| 1682 | /* A PMU device node without enable-bit-index is always enabled */ | 1673 | /* A PMU device node without enable-bit-index is always enabled */ |
| @@ -1686,17 +1677,15 @@ xgene_pmu_dev_ctx *fdt_get_pmu_hw_inf(struct xgene_pmu *xgene_pmu, · 3 unmodified lines | |||
| 1686 | ctx->name = xgene_pmu_dev_name(dev, type, enable_bit); | 1677 | ctx->name = xgene_pmu_dev_name(dev, type, enable_bit); |
| 1687 | if (!ctx->name) { | 1678 | if (!ctx->name) { |
| 1688 | dev_err(dev, "PMU type %d: Fail to get device name\n", type); | 1679 | dev_err(dev, "PMU type %d: Fail to get device name\n", type); |
| 1689 | goto err; | 1680 | return NULL; |
| 1690 | } | 1681 | } |
| 1682 | |||
| 1691 | inf = &ctx->inf; | 1683 | inf = &ctx->inf; |
| 1692 | inf->type = type; | 1684 | inf->type = type; |
| 1693 | inf->csr = dev_csr; | 1685 | inf->csr = dev_csr; |
| 1694 | inf->enable_mask = 1 << enable_bit; | 1686 | inf->enable_mask = 1 << enable_bit; |
| 1695 | 1687 | ||
| 1696 | return ctx; | 1688 | return ctx; |
| 1697 | err: | ||
| 1698 | devm_kfree(dev, ctx); | ||
| 1699 | return NULL; | ||
| 1700 | } | 1689 | } |
| 1701 | 1690 | ||
| 1702 | static int fdt_pmu_probe_pmu_dev(struct xgene_pmu *xgene_pmu, | 1691 | static int fdt_pmu_probe_pmu_dev(struct xgene_pmu *xgene_pmu, |
| @@ -1868,22 +1857,20 @@ static int xgene_pmu_probe(struct platform_device *pdev) · 165 unmodified lines | |||
| 1868 | xgene_pmu->pcppmu_csr = devm_ioremap_resource(&pdev->dev, res); | 1857 | xgene_pmu->pcppmu_csr = devm_ioremap_resource(&pdev->dev, res); |
| 1869 | if (IS_ERR(xgene_pmu->pcppmu_csr)) { | 1858 | if (IS_ERR(xgene_pmu->pcppmu_csr)) { |
| 1870 | dev_err(&pdev->dev, "ioremap failed for PCP PMU resource\n"); | 1859 | dev_err(&pdev->dev, "ioremap failed for PCP PMU resource\n"); |
| 1871 | rc = PTR_ERR(xgene_pmu->pcppmu_csr); | 1860 | return PTR_ERR(xgene_pmu->pcppmu_csr); |
| 1872 | goto err; | ||
| 1873 | } | 1861 | } |
| 1874 | 1862 | ||
| 1875 | irq = platform_get_irq(pdev, 0); | 1863 | irq = platform_get_irq(pdev, 0); |
| 1876 | if (irq < 0) { | 1864 | if (irq < 0) { |
| 1877 | dev_err(&pdev->dev, "No IRQ resource\n"); | 1865 | dev_err(&pdev->dev, "No IRQ resource\n"); |
| 1878 | rc = -EINVAL; | 1866 | return -EINVAL; |
| 1879 | goto err; | ||
| 1880 | } | 1867 | } |
| 1881 | rc = devm_request_irq(&pdev->dev, irq, xgene_pmu_isr, | 1868 | rc = devm_request_irq(&pdev->dev, irq, xgene_pmu_isr, |
| 1882 | IRQF_NOBALANCING | IRQF_NO_THREAD, | 1869 | IRQF_NOBALANCING | IRQF_NO_THREAD, |
| 1883 | dev_name(&pdev->dev), xgene_pmu); | 1870 | dev_name(&pdev->dev), xgene_pmu); |
| 1884 | if (rc) { | 1871 | if (rc) { |
| 1885 | dev_err(&pdev->dev, "Could not request IRQ %d\n", irq); | 1872 | dev_err(&pdev->dev, "Could not request IRQ %d\n", irq); |
| 1886 | goto err; | 1873 | return rc; |
| 1887 | } | 1874 | } |
| 1888 | 1875 | ||
| 1889 | raw_spin_lock_init(&xgene_pmu->lock); | 1876 | raw_spin_lock_init(&xgene_pmu->lock); |
| @@ -1903,42 +1890,29 @@ static int xgene_pmu_probe(struct platform_device *pdev) · 13 unmodified lines | |||
| 1903 | rc = irq_set_affinity(irq, &xgene_pmu->cpu); | 1890 | rc = irq_set_affinity(irq, &xgene_pmu->cpu); |
| 1904 | if (rc) { | 1891 | if (rc) { |
| 1905 | dev_err(&pdev->dev, "Failed to set interrupt affinity!\n"); | 1892 | dev_err(&pdev->dev, "Failed to set interrupt affinity!\n"); |
| 1906 | goto err; | 1893 | return rc; |
| 1907 | } | 1894 | } |
| 1908 | 1895 | ||
| 1909 | /* Walk through the tree for all PMU perf devices */ | 1896 | /* Walk through the tree for all PMU perf devices */ |
| 1910 | rc = xgene_pmu_probe_pmu_dev(xgene_pmu, pdev); | 1897 | rc = xgene_pmu_probe_pmu_dev(xgene_pmu, pdev); |
| 1911 | if (rc) { | 1898 | if (rc) { |
| 1912 | dev_err(&pdev->dev, "No PMU perf devices found!\n"); | 1899 | dev_err(&pdev->dev, "No PMU perf devices found!\n"); |
| 1913 | goto err; | 1900 | return rc; |
| 1914 | } | 1901 | } |
| 1915 | 1902 | ||
| 1916 | /* Enable interrupt */ | 1903 | /* Enable interrupt */ |
| 1917 | xgene_pmu->ops->unmask_int(xgene_pmu); | 1904 | xgene_pmu->ops->unmask_int(xgene_pmu); |
| 1918 | 1905 | ||
| 1919 | return 0; | 1906 | return 0; |
| 1920 | |||
| 1921 | err: | ||
| 1922 | if (xgene_pmu->pcppmu_csr) | ||
| 1923 | devm_iounmap(&pdev->dev, xgene_pmu->pcppmu_csr); | ||
| 1924 | devm_kfree(&pdev->dev, xgene_pmu); | ||
| 1925 | |||
| 1926 | return rc; | ||
| 1927 | } | 1907 | } |
| 1928 | 1908 | ||
| 1929 | static void | 1909 | static void |
| 1930 | xgene_pmu_dev_cleanup(struct xgene_pmu *xgene_pmu, struct list_head *pmus) | 1910 | xgene_pmu_dev_cleanup(struct xgene_pmu *xgene_pmu, struct list_head *pmus) |
| 1931 | { | 1911 | { |
| 1932 | struct xgene_pmu_dev_ctx *ctx; | 1912 | struct xgene_pmu_dev_ctx *ctx; |
| 1933 | struct device *dev = xgene_pmu->dev; | ||
| 1934 | struct xgene_pmu_dev *pmu_dev; | ||
| 1935 | 1913 | ||
| 1936 | list_for_each_entry(ctx, pmus, next) { | 1914 | list_for_each_entry(ctx, pmus, next) { |
| 1937 | pmu_dev = ctx->pmu_dev; | 1915 | perf_pmu_unregister(&ctx->pmu_dev->pmu); |
| 1938 | if (pmu_dev->inf->csr) | ||
| 1939 | devm_iounmap(dev, pmu_dev->inf->csr); | ||
| 1940 | devm_kfree(dev, ctx); | ||
| 1941 | devm_kfree(dev, pmu_dev); | ||
| 1942 | } | 1916 | } |
| 1943 | } | 1917 | } |
| 1944 | 1918 | ||
| @@ -1951,10 +1925,6 @@ static int xgene_pmu_remove(struct platform_device *pdev) · 6 unmodified lines | |||
| 1951 | xgene_pmu_dev_cleanup(xgene_pmu, &xgene_pmu->mcbpmus); | 1925 | xgene_pmu_dev_cleanup(xgene_pmu, &xgene_pmu->mcbpmus); |
| 1952 | xgene_pmu_dev_cleanup(xgene_pmu, &xgene_pmu->mcpmus); | 1926 | xgene_pmu_dev_cleanup(xgene_pmu, &xgene_pmu->mcpmus); |
| 1953 | 1927 | ||
| 1954 | if (xgene_pmu->pcppmu_csr) | ||
| 1955 | devm_iounmap(&pdev->dev, xgene_pmu->pcppmu_csr); | ||
| 1956 | devm_kfree(&pdev->dev, xgene_pmu); | ||
| 1957 | |||
| 1958 | return 0; | 1928 | return 0; |
| 1959 | } | 1929 | } |
| 1960 | 1930 | ||
| @@ -121,6 +121,7 @@ extern const struct raid6_recov_calls raid6_recov_ssse3; · 120 unmodified lines | |||
| 121 | extern const struct raid6_recov_calls raid6_recov_avx2; | 121 | extern const struct raid6_recov_calls raid6_recov_avx2; |
| 122 | extern const struct raid6_recov_calls raid6_recov_avx512; | 122 | extern const struct raid6_recov_calls raid6_recov_avx512; |
| 123 | extern const struct raid6_recov_calls raid6_recov_s390xc; | 123 | extern const struct raid6_recov_calls raid6_recov_s390xc; |
| 124 | extern const struct raid6_recov_calls raid6_recov_neon; | ||
| 124 | 125 | ||
| 125 | extern const struct raid6_calls raid6_neonx1; | 126 | extern const struct raid6_calls raid6_neonx1; |
| 126 | extern const struct raid6_calls raid6_neonx2; | 127 | extern const struct raid6_calls raid6_neonx2; |
| @@ -38,6 +38,10 @@ enum { · 37 unmodified lines | |||
| 38 | 38 | ||
| 39 | #ifdef __KERNEL__ | 39 | #ifdef __KERNEL__ |
| 40 | 40 | ||
| 41 | #ifndef THREAD_ALIGN | ||
| 42 | #define THREAD_ALIGN THREAD_SIZE | ||
| 43 | #endif | ||
| 44 | |||
| 41 | #ifdef CONFIG_DEBUG_STACK_USAGE | 45 | #ifdef CONFIG_DEBUG_STACK_USAGE |
| 42 | # define THREADINFO_GFP (GFP_KERNEL_ACCOUNT | __GFP_NOTRACK | \ | 46 | # define THREADINFO_GFP (GFP_KERNEL_ACCOUNT | __GFP_NOTRACK | \ |
| 43 | __GFP_ZERO) | 47 | __GFP_ZERO) |
| @@ -88,6 +88,7 @@ · 87 unmodified lines | |||
| 88 | #include <linux/sysctl.h> | 88 | #include <linux/sysctl.h> |
| 89 | #include <linux/kcov.h> | 89 | #include <linux/kcov.h> |
| 90 | #include <linux/livepatch.h> | 90 | #include <linux/livepatch.h> |
| 91 | #include <linux/thread_info.h> | ||
| 91 | 92 | ||
| 92 | #include <asm/pgtable.h> | 93 | #include <asm/pgtable.h> |
| 93 | #include <asm/pgalloc.h> | 94 | #include <asm/pgalloc.h> |
| @@ -217,7 +218,7 @@ static unsigned long *alloc_thread_stack_node(struct task_struct *tsk, int node) · 123 unmodified lines | |||
| 217 | return s->addr; | 218 | return s->addr; |
| 218 | } | 219 | } |
| 219 | 220 | ||
| 220 | stack = __vmalloc_node_range(THREAD_SIZE, THREAD_SIZE, | 221 | stack = __vmalloc_node_range(THREAD_SIZE, THREAD_ALIGN, |
| 221 | VMALLOC_START, VMALLOC_END, | 222 | VMALLOC_START, VMALLOC_END, |
| 222 | THREADINFO_GFP, | 223 | THREADINFO_GFP, |
| 223 | PAGE_KERNEL, | 224 | PAGE_KERNEL, |
| @@ -5,7 +5,7 @@ raid6_pq-y += algos.o recov.o tables.o int1.o int2.o int4.o \ · 4 unmodified lines | |||
| 5 | 5 | ||
| 6 | raid6_pq-$(CONFIG_X86) += recov_ssse3.o recov_avx2.o mmx.o sse1.o sse2.o avx2.o avx512.o recov_avx512.o | 6 | raid6_pq-$(CONFIG_X86) += recov_ssse3.o recov_avx2.o mmx.o sse1.o sse2.o avx2.o avx512.o recov_avx512.o |
| 7 | raid6_pq-$(CONFIG_ALTIVEC) += altivec1.o altivec2.o altivec4.o altivec8.o | 7 | raid6_pq-$(CONFIG_ALTIVEC) += altivec1.o altivec2.o altivec4.o altivec8.o |
| 8 | raid6_pq-$(CONFIG_KERNEL_MODE_NEON) += neon.o neon1.o neon2.o neon4.o neon8.o | 8 | raid6_pq-$(CONFIG_KERNEL_MODE_NEON) += neon.o neon1.o neon2.o neon4.o neon8.o recov_neon.o recov_neon_inner.o |
| 9 | raid6_pq-$(CONFIG_TILEGX) += tilegx8.o | 9 | raid6_pq-$(CONFIG_TILEGX) += tilegx8.o |
| 10 | raid6_pq-$(CONFIG_S390) += s390vx8.o recov_s390xc.o | 10 | raid6_pq-$(CONFIG_S390) += s390vx8.o recov_s390xc.o |
| 11 | 11 | ||
| @@ -26,7 +26,9 @@ NEON_FLAGS := -ffreestanding · 14 unmodified lines | |||
| 26 | ifeq ($(ARCH),arm) | 26 | ifeq ($(ARCH),arm) |
| 27 | NEON_FLAGS += -mfloat-abi=softfp -mfpu=neon | 27 | NEON_FLAGS += -mfloat-abi=softfp -mfpu=neon |
| 28 | endif | 28 | endif |
| 29 | CFLAGS_recov_neon_inner.o += $(NEON_FLAGS) | ||
| 29 | ifeq ($(ARCH),arm64) | 30 | ifeq ($(ARCH),arm64) |
| 31 | CFLAGS_REMOVE_recov_neon_inner.o += -mgeneral-regs-only | ||
| 30 | CFLAGS_REMOVE_neon1.o += -mgeneral-regs-only | 32 | CFLAGS_REMOVE_neon1.o += -mgeneral-regs-only |
| 31 | CFLAGS_REMOVE_neon2.o += -mgeneral-regs-only | 33 | CFLAGS_REMOVE_neon2.o += -mgeneral-regs-only |
| 32 | CFLAGS_REMOVE_neon4.o += -mgeneral-regs-only | 34 | CFLAGS_REMOVE_neon4.o += -mgeneral-regs-only |
| @@ -112,6 +112,9 @@ const struct raid6_recov_calls *const raid6_recov_algos[] = { · 111 unmodified lines | |||
| 112 | #endif | 112 | #endif |
| 113 | #ifdef CONFIG_S390 | 113 | #ifdef CONFIG_S390 |
| 114 | &raid6_recov_s390xc, | 114 | &raid6_recov_s390xc, |
| 115 | #endif | ||
| 116 | #if defined(CONFIG_KERNEL_MODE_NEON) | ||
| 117 | &raid6_recov_neon, | ||
| 115 | #endif | 118 | #endif |
| 116 | &raid6_recov_intx1, | 119 | &raid6_recov_intx1, |
| 117 | NULL | 120 | NULL |
| @@ -46,8 +46,12 @@ static inline unative_t SHLBYTE(unative_t v) · 45 unmodified lines | |||
| 46 | */ | 46 | */ |
| 47 | static inline unative_t MASK(unative_t v) | 47 | static inline unative_t MASK(unative_t v) |
| 48 | { | 48 | { |
| 49 | const uint8x16_t temp = NBYTES(0); | 49 | return (unative_t)vshrq_n_s8((int8x16_t)v, 7); |
| 50 | return (unative_t)vcltq_s8((int8x16_t)v, (int8x16_t)temp); | 50 | } |
| 51 | |||
| 52 | static inline unative_t PMUL(unative_t v, unative_t u) | ||
| 53 | { | ||
| 54 | return (unative_t)vmulq_p8((poly8x16_t)v, (poly8x16_t)u); | ||
| 51 | } | 55 | } |
| 52 | 56 | ||
| 53 | void raid6_neon$#_gen_syndrome_real(int disks, unsigned long bytes, void **ptrs) | 57 | void raid6_neon$#_gen_syndrome_real(int disks, unsigned long bytes, void **ptrs) |
| @@ -110,7 +114,30 @@ void raid6_neon$#_xor_syndrome_real(int disks, int start, int stop, · 56 unmodified lines | |||
| 110 | wq$$ = veorq_u8(w1$$, wd$$); | 114 | wq$$ = veorq_u8(w1$$, wd$$); |
| 111 | } | 115 | } |
| 112 | /* P/Q left side optimization */ | 116 | /* P/Q left side optimization */ |
| 113 | for ( z = start-1 ; z >= 0 ; z-- ) { | 117 | for ( z = start-1 ; z >= 3 ; z -= 4 ) { |
| 118 | w2$$ = vshrq_n_u8(wq$$, 4); | ||
| 119 | w1$$ = vshlq_n_u8(wq$$, 4); | ||
| 120 | |||
| 121 | w2$$ = PMUL(w2$$, x1d); | ||
| 122 | wq$$ = veorq_u8(w1$$, w2$$); | ||
| 123 | } | ||
| 124 | |||
| 125 | switch (z) { | ||
| 126 | case 2: | ||
| 127 | w2$$ = vshrq_n_u8(wq$$, 5); | ||
| 128 | w1$$ = vshlq_n_u8(wq$$, 3); | ||
| 129 | |||
| 130 | w2$$ = PMUL(w2$$, x1d); | ||
| 131 | wq$$ = veorq_u8(w1$$, w2$$); | ||
| 132 | break; | ||
| 133 | case 1: | ||
| 134 | w2$$ = vshrq_n_u8(wq$$, 6); | ||
| 135 | w1$$ = vshlq_n_u8(wq$$, 2); | ||
| 136 | |||
| 137 | w2$$ = PMUL(w2$$, x1d); | ||
| 138 | wq$$ = veorq_u8(w1$$, w2$$); | ||
| 139 | break; | ||
| 140 | case 0: | ||
| 114 | w2$$ = MASK(wq$$); | 141 | w2$$ = MASK(wq$$); |
| 115 | w1$$ = SHLBYTE(wq$$); | 142 | w1$$ = SHLBYTE(wq$$); |
| 116 | 143 | ||
| @@ -0,0 +1,110 @@ | |||
| 1 | /* | ||
| 2 | * Copyright (C) 2012 Intel Corporation | ||
| 3 | * Copyright (C) 2017 Linaro Ltd. <ard.biesheuvel@linaro.org> | ||
| 4 | * | ||
| 5 | * This program is free software; you can redistribute it and/or | ||
| 6 | * modify it under the terms of the GNU General Public License | ||
| 7 | * as published by the Free Software Foundation; version 2 | ||
| 8 | * of the License. | ||
| 9 | */ | ||
| 10 | |||
| 11 | #include <linux/raid/pq.h> | ||
| 12 | |||
| 13 | #ifdef __KERNEL__ | ||
| 14 | #include <asm/neon.h> | ||
| 15 | #else | ||
| 16 | #define kernel_neon_begin() | ||
| 17 | #define kernel_neon_end() | ||
| 18 | #define cpu_has_neon() (1) | ||
| 19 | #endif | ||
| 20 | |||
| 21 | static int raid6_has_neon(void) | ||
| 22 | { | ||
| 23 | return cpu_has_neon(); | ||
| 24 | } | ||
| 25 | |||
| 26 | void __raid6_2data_recov_neon(int bytes, uint8_t *p, uint8_t *q, uint8_t *dp, | ||
| 27 | uint8_t *dq, const uint8_t *pbmul, | ||
| 28 | const uint8_t *qmul); | ||
| 29 | |||
| 30 | void __raid6_datap_recov_neon(int bytes, uint8_t *p, uint8_t *q, uint8_t *dq, | ||
| 31 | const uint8_t *qmul); | ||
| 32 | |||
| 33 | static void raid6_2data_recov_neon(int disks, size_t bytes, int faila, | ||
| 34 | int failb, void **ptrs) | ||
| 35 | { | ||
| 36 | u8 *p, *q, *dp, *dq; | ||
| 37 | const u8 *pbmul; /* P multiplier table for B data */ | ||
| 38 | const u8 *qmul; /* Q multiplier table (for both) */ | ||
| 39 | |||
| 40 | p = (u8 *)ptrs[disks - 2]; | ||
| 41 | q = (u8 *)ptrs[disks - 1]; | ||
| 42 | |||
| 43 | /* | ||
| 44 | * Compute syndrome with zero for the missing data pages | ||
| 45 | * Use the dead data pages as temporary storage for | ||
| 46 | * delta p and delta q | ||
| 47 | */ | ||
| 48 | dp = (u8 *)ptrs[faila]; | ||
| 49 | ptrs[faila] = (void *)raid6_empty_zero_page; | ||
| 50 | ptrs[disks - 2] = dp; | ||
| 51 | dq = (u8 *)ptrs[failb]; | ||
| 52 | ptrs[failb] = (void *)raid6_empty_zero_page; | ||
| 53 | ptrs[disks - 1] = dq; | ||
| 54 | |||
| 55 | raid6_call.gen_syndrome(disks, bytes, ptrs); | ||
| 56 | |||
| 57 | /* Restore pointer table */ | ||
| 58 | ptrs[faila] = dp; | ||
| 59 | ptrs[failb] = dq; | ||
| 60 | ptrs[disks - 2] = p; | ||
| 61 | ptrs[disks - 1] = q; | ||
| 62 | |||
| 63 | /* Now, pick the proper data tables */ | ||
| 64 | pbmul = raid6_vgfmul[raid6_gfexi[failb-faila]]; | ||
| 65 | qmul = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila] ^ | ||
| 66 | raid6_gfexp[failb]]]; | ||
| 67 | |||
| 68 | kernel_neon_begin(); | ||
| 69 | __raid6_2data_recov_neon(bytes, p, q, dp, dq, pbmul, qmul); | ||
| 70 | kernel_neon_end(); | ||
| 71 | } | ||
| 72 | |||
| 73 | static void raid6_datap_recov_neon(int disks, size_t bytes, int faila, | ||
| 74 | void **ptrs) | ||
| 75 | { | ||
| 76 | u8 *p, *q, *dq; | ||
| 77 | const u8 *qmul; /* Q multiplier table */ | ||
| 78 | |||
| 79 | p = (u8 *)ptrs[disks - 2]; | ||
| 80 | q = (u8 *)ptrs[disks - 1]; | ||
| 81 | |||
| 82 | /* | ||
| 83 | * Compute syndrome with zero for the missing data page | ||
| 84 | * Use the dead data page as temporary storage for delta q | ||
| 85 | */ | ||
| 86 | dq = (u8 *)ptrs[faila]; | ||
| 87 | ptrs[faila] = (void *)raid6_empty_zero_page; | ||
| 88 | ptrs[disks - 1] = dq; | ||
| 89 | |||
| 90 | raid6_call.gen_syndrome(disks, bytes, ptrs); | ||
| 91 | |||
| 92 | /* Restore pointer table */ | ||
| 93 | ptrs[faila] = dq; | ||
| 94 | ptrs[disks - 1] = q; | ||
| 95 | |||
| 96 | /* Now, pick the proper data tables */ | ||
| 97 | qmul = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila]]]; | ||
| 98 | |||
| 99 | kernel_neon_begin(); | ||
| 100 | __raid6_datap_recov_neon(bytes, p, q, dq, qmul); | ||
| 101 | kernel_neon_end(); | ||
| 102 | } | ||
| 103 | |||
| 104 | const struct raid6_recov_calls raid6_recov_neon = { | ||
| 105 | .data2 = raid6_2data_recov_neon, | ||
| 106 | .datap = raid6_datap_recov_neon, | ||
| 107 | .valid = raid6_has_neon, | ||
| 108 | .name = "neon", | ||
| 109 | .priority = 10, | ||
| 110 | }; | ||
| @@ -0,0 +1,117 @@ | |||
| 1 | /* | ||
| 2 | * Copyright (C) 2012 Intel Corporation | ||
| 3 | * Copyright (C) 2017 Linaro Ltd. <ard.biesheuvel@linaro.org> | ||
| 4 | * | ||
| 5 | * This program is free software; you can redistribute it and/or | ||
| 6 | * modify it under the terms of the GNU General Public License | ||
| 7 | * as published by the Free Software Foundation; version 2 | ||
| 8 | * of the License. | ||
| 9 | */ | ||
| 10 | |||
| 11 | #include <arm_neon.h> | ||
| 12 | |||
| 13 | static const uint8x16_t x0f = { | ||
| 14 | 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, | ||
| 15 | 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, | ||
| 16 | }; | ||
| 17 | |||
| 18 | #ifdef CONFIG_ARM | ||
| 19 | /* | ||
| 20 | * AArch32 does not provide this intrinsic natively because it does not | ||
| 21 | * implement the underlying instruction. AArch32 only provides a 64-bit | ||
| 22 | * wide vtbl.8 instruction, so use that instead. | ||
| 23 | */ | ||
| 24 | static uint8x16_t vqtbl1q_u8(uint8x16_t a, uint8x16_t b) | ||
| 25 | { | ||
| 26 | union { | ||
| 27 | uint8x16_t val; | ||
| 28 | uint8x8x2_t pair; | ||
| 29 | } __a = { a }; | ||
| 30 | |||
| 31 | return vcombine_u8(vtbl2_u8(__a.pair, vget_low_u8(b)), | ||
| 32 | vtbl2_u8(__a.pair, vget_high_u8(b))); | ||
| 33 | } | ||
| 34 | #endif | ||
| 35 | |||
| 36 | void __raid6_2data_recov_neon(int bytes, uint8_t *p, uint8_t *q, uint8_t *dp, | ||
| 37 | uint8_t *dq, const uint8_t *pbmul, | ||
| 38 | const uint8_t *qmul) | ||
| 39 | { | ||
| 40 | uint8x16_t pm0 = vld1q_u8(pbmul); | ||
| 41 | uint8x16_t pm1 = vld1q_u8(pbmul + 16); | ||
| 42 | uint8x16_t qm0 = vld1q_u8(qmul); | ||
| 43 | uint8x16_t qm1 = vld1q_u8(qmul + 16); | ||
| 44 | |||
| 45 | /* | ||
| 46 | * while ( bytes-- ) { | ||
| 47 | * uint8_t px, qx, db; | ||
| 48 | * | ||
| 49 | * px = *p ^ *dp; | ||
| 50 | * qx = qmul[*q ^ *dq]; | ||
| 51 | * *dq++ = db = pbmul[px] ^ qx; | ||
| 52 | * *dp++ = db ^ px; | ||
| 53 | * p++; q++; | ||
| 54 | * } | ||
| 55 | */ | ||
| 56 | |||
| 57 | while (bytes) { | ||
| 58 | uint8x16_t vx, vy, px, qx, db; | ||
| 59 | |||
| 60 | px = veorq_u8(vld1q_u8(p), vld1q_u8(dp)); | ||
| 61 | vx = veorq_u8(vld1q_u8(q), vld1q_u8(dq)); | ||
| 62 | |||
| 63 | vy = (uint8x16_t)vshrq_n_s16((int16x8_t)vx, 4); | ||
| 64 | vx = vqtbl1q_u8(qm0, vandq_u8(vx, x0f)); | ||
| 65 | vy = vqtbl1q_u8(qm1, vandq_u8(vy, x0f)); | ||
| 66 | qx = veorq_u8(vx, vy); | ||
| 67 | |||
| 68 | vy = (uint8x16_t)vshrq_n_s16((int16x8_t)px, 4); | ||
| 69 | vx = vqtbl1q_u8(pm0, vandq_u8(px, x0f)); | ||
| 70 | vy = vqtbl1q_u8(pm1, vandq_u8(vy, x0f)); | ||
| 71 | vx = veorq_u8(vx, vy); | ||
| 72 | db = veorq_u8(vx, qx); | ||
| 73 | |||
| 74 | vst1q_u8(dq, db); | ||
| 75 | vst1q_u8(dp, veorq_u8(db, px)); | ||
| 76 | |||
| 77 | bytes -= 16; | ||
| 78 | p += 16; | ||
| 79 | q += 16; | ||
| 80 | dp += 16; | ||
| 81 | dq += 16; | ||
| 82 | } | ||
| 83 | } | ||
| 84 | |||
| 85 | void __raid6_datap_recov_neon(int bytes, uint8_t *p, uint8_t *q, uint8_t *dq, | ||
| 86 | const uint8_t *qmul) | ||
| 87 | { | ||
| 88 | uint8x16_t qm0 = vld1q_u8(qmul); | ||
| 89 | uint8x16_t qm1 = vld1q_u8(qmul + 16); | ||
| 90 | |||
| 91 | /* | ||
| 92 | * while (bytes--) { | ||
| 93 | * *p++ ^= *dq = qmul[*q ^ *dq]; | ||
| 94 | * q++; dq++; | ||
| 95 | * } | ||
| 96 | */ | ||
| 97 | |||
| 98 | while (bytes) { | ||
| 99 | uint8x16_t vx, vy; | ||
| 100 | |||
| 101 | vx = veorq_u8(vld1q_u8(q), vld1q_u8(dq)); | ||
| 102 | |||
| 103 | vy = (uint8x16_t)vshrq_n_s16((int16x8_t)vx, 4); | ||
| 104 | vx = vqtbl1q_u8(qm0, vandq_u8(vx, x0f)); | ||
| 105 | vy = vqtbl1q_u8(qm1, vandq_u8(vy, x0f)); | ||
| 106 | vx = veorq_u8(vx, vy); | ||
| 107 | vy = veorq_u8(vx, vld1q_u8(p)); | ||
| 108 | |||
| 109 | vst1q_u8(dq, vx); | ||
| 110 | vst1q_u8(p, vy); | ||
| 111 | |||
| 112 | bytes -= 16; | ||
| 113 | p += 16; | ||
| 114 | q += 16; | ||
| 115 | dq += 16; | ||
| 116 | } | ||
| 117 | } | ||