Pull arm64 updates from Catalin Marinas: - VMAP_STACK support, allowing the kernel stacks to be allocated in the vmalloc space with a guard page for trapping stack overflows. One of the patches introduces THREAD_ALIGN and changes the generic alloc_thread_stack_node() to use this instead of THREAD_SIZE (no functional change for other architectures) - Contiguous PTE hugetlb support re-enabled (after being reverted a couple of times). We now have the semantics agreed in the generic mm layer together with API improvements so that the architecture code can detect between contiguous and non-contiguous huge PTEs - Initial support for persistent memory on ARM: DC CVAP instruction exposed to user space (HWCAP) and the in-kernel pmem API implemented - raid6 improvements for arm64: faster algorithm for the delta syndrome and implementation of the recovery routines using Neon - FP/SIMD refactoring and removal of support for Neon in interrupt context. This is in preparation for full SVE support - PTE accessors converted from inline asm to cmpxchg so that we can use LSE atomics if available (ARMv8.1) - Perf support for Cortex-A35 and A73 - Non-urgent fixes and cleanups * tag 'arm64-upstream' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux: (75 commits) arm64: cleanup {COMPAT_,}SET_PERSONALITY() macro arm64: introduce separated bits for mm_context_t flags arm64: hugetlb: Cleanup setup_hugepagesz arm64: Re-enable support for contiguous hugepages arm64: hugetlb: Override set_huge_swap_pte_at() to support contiguous hugepages arm64: hugetlb: Override huge_pte_clear() to support contiguous hugepages arm64: hugetlb: Handle swap entries in huge_pte_offset() for contiguous hugepages arm64: hugetlb: Add break-before-make logic for contiguous entries arm64: hugetlb: Spring clean huge pte accessors arm64: hugetlb: Introduce pte_pgprot helper arm64: hugetlb: set_huge_pte_at Add WARN_ON on !pte_present arm64: kexec: have own crash_smp_send_stop() for crash dump for nonpanic cores arm64: dma-mapping: Mark atomic_pool as __ro_after_init arm64: dma-mapping: Do not pass data to gen_pool_set_algo() arm64: Remove the !CONFIG_ARM64_HW_AFDBM alternative code paths arm64: Ignore hardware dirty bit updates in ptep_set_wrprotect() arm64: Move PTE_RDONLY bit handling out of set_pte_at() kvm: arm64: Convert kvm_set_s2pte_readonly() from inline asm to cmpxchg() arm64: Convert pte handling from inline asm to using (cmp)xchg arm64: neon/efi: Make EFI fpsimd save/restore variables static ...
Unified · Split
| @@ -179,6 +179,8 @@ infrastructure: · 178 unmodified lines | ||
| 179 | 179 | | FCMA | [19-16] | y | |
| 180 | 180 | |--------------------------------------------------| |
| 181 | 181 | | JSCVT | [15-12] | y | |
| 182 | + |--------------------------------------------------| | |
| 183 | + | DPB | [3-0] | y | | |
| 182 | 184 | x--------------------------------------------------x |
| 183 | 185 | |
| 184 | 186 | Appendix I: Example |
| @@ -9,9 +9,11 @@ Required properties: · 8 unmodified lines | ||
| 9 | 9 | - compatible : should be one of |
| 10 | 10 | "apm,potenza-pmu" |
| 11 | 11 | "arm,armv8-pmuv3" |
| 12 | + "arm,cortex-a73-pmu" | |
| 12 | 13 | "arm,cortex-a72-pmu" |
| 13 | 14 | "arm,cortex-a57-pmu" |
| 14 | 15 | "arm,cortex-a53-pmu" |
| 16 | + "arm,cortex-a35-pmu" | |
| 15 | 17 | "arm,cortex-a17-pmu" |
| 16 | 18 | "arm,cortex-a15-pmu" |
| 17 | 19 | "arm,cortex-a12-pmu" |
| @@ -75,6 +75,7 @@ config ARM64 · 74 unmodified lines | ||
| 75 | 75 | select HAVE_ARCH_SECCOMP_FILTER |
| 76 | 76 | select HAVE_ARCH_TRACEHOOK |
| 77 | 77 | select HAVE_ARCH_TRANSPARENT_HUGEPAGE |
| 78 | + select HAVE_ARCH_VMAP_STACK | |
| 78 | 79 | select HAVE_ARM_SMCCC |
| 79 | 80 | select HAVE_EBPF_JIT |
| 80 | 81 | select HAVE_C_RECORDMCOUNT |
| @@ -960,6 +961,18 @@ config ARM64_UAO · 879 unmodified lines | ||
| 960 | 961 | regular load/store instructions if the cpu does not implement the |
| 961 | 962 | feature. |
| 962 | 963 | |
| 964 | +config ARM64_PMEM | |
| 965 | + bool "Enable support for persistent memory" | |
| 966 | + select ARCH_HAS_PMEM_API | |
| 967 | + select ARCH_HAS_UACCESS_FLUSHCACHE | |
| 968 | + help | |
| 969 | + Say Y to enable support for the persistent memory API based on the | |
| 970 | + ARMv8.2 DCPoP feature. | |
| 971 | + | |
| 972 | + The feature is detected at runtime, and the kernel will use DC CVAC | |
| 973 | + operations if DC CVAP is not supported (following the behaviour of | |
| 974 | + DC CVAP itself if the system does not define a point of persistence). | |
| 975 | + | |
| 963 | 976 | endmenu |
| 964 | 977 | |
| 965 | 978 | config ARM64_MODULE_CMODEL_LARGE |
| @@ -20,7 +20,6 @@ generic-y += rwsem.h · 19 unmodified lines | ||
| 20 | 20 | generic-y += segment.h |
| 21 | 21 | generic-y += serial.h |
| 22 | 22 | generic-y += set_memory.h |
| 23 | −generic-y += simd.h | |
| 24 | 23 | generic-y += sizes.h |
| 25 | 24 | generic-y += switch_to.h |
| 26 | 25 | generic-y += trace_clock.h |
| @@ -0,0 +1,54 @@ | ||
| 1 | +#ifndef __ASM_ASM_BUG_H | |
| 2 | +/* | |
| 3 | + * Copyright (C) 2017 ARM Limited | |
| 4 | + * | |
| 5 | + * This program is free software; you can redistribute it and/or modify | |
| 6 | + * it under the terms of the GNU General Public License version 2 as | |
| 7 | + * published by the Free Software Foundation. | |
| 8 | + * | |
| 9 | + * This program is distributed in the hope that it will be useful, | |
| 10 | + * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
| 11 | + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
| 12 | + * GNU General Public License for more details. | |
| 13 | + * | |
| 14 | + * You should have received a copy of the GNU General Public License | |
| 15 | + * along with this program. If not, see <http://www.gnu.org/licenses/>. | |
| 16 | + */ | |
| 17 | +#define __ASM_ASM_BUG_H | |
| 18 | + | |
| 19 | +#include <asm/brk-imm.h> | |
| 20 | + | |
| 21 | +#ifdef CONFIG_DEBUG_BUGVERBOSE | |
| 22 | +#define _BUGVERBOSE_LOCATION(file, line) __BUGVERBOSE_LOCATION(file, line) | |
| 23 | +#define __BUGVERBOSE_LOCATION(file, line) \ | |
| 24 | + .pushsection .rodata.str,"aMS",@progbits,1; \ | |
| 25 | + 2: .string file; \ | |
| 26 | + .popsection; \ | |
| 27 | + \ | |
| 28 | + .long 2b - 0b; \ | |
| 29 | + .short line; | |
| 30 | +#else | |
| 31 | +#define _BUGVERBOSE_LOCATION(file, line) | |
| 32 | +#endif | |
| 33 | + | |
| 34 | +#ifdef CONFIG_GENERIC_BUG | |
| 35 | + | |
| 36 | +#define __BUG_ENTRY(flags) \ | |
| 37 | + .pushsection __bug_table,"aw"; \ | |
| 38 | + .align 2; \ | |
| 39 | + 0: .long 1f - 0b; \ | |
| 40 | +_BUGVERBOSE_LOCATION(__FILE__, __LINE__) \ | |
| 41 | + .short flags; \ | |
| 42 | + .popsection; \ | |
| 43 | + 1: | |
| 44 | +#else | |
| 45 | +#define __BUG_ENTRY(flags) | |
| 46 | +#endif | |
| 47 | + | |
| 48 | +#define ASM_BUG_FLAGS(flags) \ | |
| 49 | + __BUG_ENTRY(flags) \ | |
| 50 | + brk BUG_BRK_IMM | |
| 51 | + | |
| 52 | +#define ASM_BUG() ASM_BUG_FLAGS(0) | |
| 53 | + | |
| 54 | +#endif /* __ASM_ASM_BUG_H */ | |
| @@ -230,12 +230,18 @@ lr .req x30 // link register · 229 unmodified lines | ||
| 230 | 230 | .endm |
| 231 | 231 | |
| 232 | 232 | /* |
| 233 | − * @dst: Result of per_cpu(sym, smp_processor_id()) | |
| 233 | + * @dst: Result of per_cpu(sym, smp_processor_id()), can be SP for | |
| 234 | + * non-module code | |
| 234 | 235 | * @sym: The name of the per-cpu variable |
| 235 | 236 | * @tmp: scratch register |
| 236 | 237 | */ |
| 237 | 238 | .macro adr_this_cpu, dst, sym, tmp |
| 239 | +#ifndef MODULE | |
| 240 | + adrp \tmp, \sym | |
| 241 | + add \dst, \tmp, #:lo12:\sym | |
| 242 | +#else | |
| 238 | 243 | adr_l \dst, \sym |
| 244 | +#endif | |
| 239 | 245 | mrs \tmp, tpidr_el1 |
| 240 | 246 | add \dst, \dst, \tmp |
| 241 | 247 | .endm |
| @@ -352,6 +358,12 @@ alternative_if_not ARM64_WORKAROUND_CLEAN_CACHE · 110 unmodified lines | ||
| 352 | 358 | dc \op, \kaddr |
| 353 | 359 | alternative_else |
| 354 | 360 | dc civac, \kaddr |
| 361 | +alternative_endif | |
| 362 | + .elseif (\op == cvap) | |
| 363 | +alternative_if ARM64_HAS_DCPOP | |
| 364 | + sys 3, c7, c12, 1, \kaddr // dc cvap | |
| 365 | +alternative_else | |
| 366 | + dc cvac, \kaddr | |
| 355 | 367 | alternative_endif |
| 356 | 368 | .else |
| 357 | 369 | dc \op, \kaddr |
| @@ -403,6 +415,17 @@ alternative_endif · 45 unmodified lines | ||
| 403 | 415 | .size __pi_##x, . - x; \ |
| 404 | 416 | ENDPROC(x) |
| 405 | 417 | |
| 418 | +/* | |
| 419 | + * Annotate a function as being unsuitable for kprobes. | |
| 420 | + */ | |
| 421 | +#ifdef CONFIG_KPROBES | |
| 422 | +#define NOKPROBE(x) \ | |
| 423 | + .pushsection "_kprobe_blacklist", "aw"; \ | |
| 424 | + .quad x; \ | |
| 425 | + .popsection; | |
| 426 | +#else | |
| 427 | +#define NOKPROBE(x) | |
| 428 | +#endif | |
| 406 | 429 | /* |
| 407 | 430 | * Emit a 64-bit absolute little endian symbol reference in a way that |
| 408 | 431 | * ensures that it will be resolved at build time, even when building a |
| @@ -18,41 +18,12 @@ · 17 unmodified lines | ||
| 18 | 18 | #ifndef _ARCH_ARM64_ASM_BUG_H |
| 19 | 19 | #define _ARCH_ARM64_ASM_BUG_H |
| 20 | 20 | |
| 21 | −#include <asm/brk-imm.h> | |
| 21 | +#include <linux/stringify.h> | |
| 22 | 22 | |
| 23 | −#ifdef CONFIG_DEBUG_BUGVERBOSE | |
| 24 | −#define _BUGVERBOSE_LOCATION(file, line) __BUGVERBOSE_LOCATION(file, line) | |
| 25 | −#define __BUGVERBOSE_LOCATION(file, line) \ | |
| 26 | − ".pushsection .rodata.str,\"aMS\",@progbits,1\n" \ | |
| 27 | − "2: .string \"" file "\"\n\t" \ | |
| 28 | − ".popsection\n\t" \ | |
| 29 | − \ | |
| 30 | − ".long 2b - 0b\n\t" \ | |
| 31 | − ".short " #line "\n\t" | |
| 32 | −#else | |
| 33 | −#define _BUGVERBOSE_LOCATION(file, line) | |
| 34 | −#endif | |
| 35 | − | |
| 36 | −#ifdef CONFIG_GENERIC_BUG | |
| 37 | − | |
| 38 | −#define __BUG_ENTRY(flags) \ | |
| 39 | − ".pushsection __bug_table,\"aw\"\n\t" \ | |
| 40 | − ".align 2\n\t" \ | |
| 41 | − "0: .long 1f - 0b\n\t" \ | |
| 42 | −_BUGVERBOSE_LOCATION(__FILE__, __LINE__) \ | |
| 43 | − ".short " #flags "\n\t" \ | |
| 44 | − ".popsection\n" \ | |
| 45 | − "1: " | |
| 46 | −#else | |
| 47 | −#define __BUG_ENTRY(flags) "" | |
| 48 | −#endif | |
| 23 | +#include <asm/asm-bug.h> | |
| 49 | 24 | |
| 50 | 25 | #define __BUG_FLAGS(flags) \ |
| 51 | − asm volatile ( \ | |
| 52 | − __BUG_ENTRY(flags) \ | |
| 53 | − "brk %[imm]" :: [imm] "i" (BUG_BRK_IMM) \ | |
| 54 | − ); | |
| 55 | − | |
| 26 | + asm volatile (__stringify(ASM_BUG_FLAGS(flags))); | |
| 56 | 27 | |
| 57 | 28 | #define BUG() do { \ |
| 58 | 29 | __BUG_FLAGS(0); \ |
| @@ -67,7 +67,9 @@ · 66 unmodified lines | ||
| 67 | 67 | */ |
| 68 | 68 | extern void flush_icache_range(unsigned long start, unsigned long end); |
| 69 | 69 | extern void __flush_dcache_area(void *addr, size_t len); |
| 70 | +extern void __inval_dcache_area(void *addr, size_t len); | |
| 70 | 71 | extern void __clean_dcache_area_poc(void *addr, size_t len); |
| 72 | +extern void __clean_dcache_area_pop(void *addr, size_t len); | |
| 71 | 73 | extern void __clean_dcache_area_pou(void *addr, size_t len); |
| 72 | 74 | extern long __flush_cache_user_range(unsigned long start, unsigned long end); |
| 73 | 75 | extern void sync_icache_aliases(void *kaddr, unsigned long len); |
| @@ -150,6 +152,6 @@ static inline void flush_cache_vunmap(unsigned long start, unsigned long end) · 76 unmodified lines | ||
| 150 | 152 | { |
| 151 | 153 | } |
| 152 | 154 | |
| 153 | −int set_memory_valid(unsigned long addr, unsigned long size, int enable); | |
| 155 | +int set_memory_valid(unsigned long addr, int numpages, int enable); | |
| 154 | 156 | |
| 155 | 157 | #endif |
| @@ -39,7 +39,8 @@ · 38 unmodified lines | ||
| 39 | 39 | #define ARM64_WORKAROUND_QCOM_FALKOR_E1003 18 |
| 40 | 40 | #define ARM64_WORKAROUND_858921 19 |
| 41 | 41 | #define ARM64_WORKAROUND_CAVIUM_30115 20 |
| 42 | +#define ARM64_HAS_DCPOP 21 | |
| 42 | 43 | |
| 43 | −#define ARM64_NCAPS 21 | |
| 44 | +#define ARM64_NCAPS 22 | |
| 44 | 45 | |
| 45 | 46 | #endif /* __ASM_CPUCAPS_H */ |
| @@ -3,7 +3,9 @@ · 2 unmodified lines | ||
| 3 | 3 | |
| 4 | 4 | #include <asm/boot.h> |
| 5 | 5 | #include <asm/cpufeature.h> |
| 6 | +#include <asm/fpsimd.h> | |
| 6 | 7 | #include <asm/io.h> |
| 8 | +#include <asm/memory.h> | |
| 7 | 9 | #include <asm/mmu_context.h> |
| 8 | 10 | #include <asm/neon.h> |
| 9 | 11 | #include <asm/ptrace.h> |
| @@ -20,8 +22,8 @@ int efi_set_mapping_permissions(struct mm_struct *mm, efi_memory_desc_t *md); · 10 unmodified lines | ||
| 20 | 22 | |
| 21 | 23 | #define arch_efi_call_virt_setup() \ |
| 22 | 24 | ({ \ |
| 23 | − kernel_neon_begin(); \ | |
| 24 | 25 | efi_virtmap_load(); \ |
| 26 | + __efi_fpsimd_begin(); \ | |
| 25 | 27 | }) |
| 26 | 28 | |
| 27 | 29 | #define arch_efi_call_virt(p, f, args...) \ |
| @@ -33,8 +35,8 @@ int efi_set_mapping_permissions(struct mm_struct *mm, efi_memory_desc_t *md); · 5 unmodified lines | ||
| 33 | 35 | |
| 34 | 36 | #define arch_efi_call_virt_teardown() \ |
| 35 | 37 | ({ \ |
| 38 | + __efi_fpsimd_end(); \ | |
| 36 | 39 | efi_virtmap_unload(); \ |
| 37 | − kernel_neon_end(); \ | |
| 38 | 40 | }) |
| 39 | 41 | |
| 40 | 42 | #define ARCH_EFI_IRQ_FLAGS_MASK (PSR_D_BIT | PSR_A_BIT | PSR_I_BIT | PSR_F_BIT) |
| @@ -48,6 +50,13 @@ int efi_set_mapping_permissions(struct mm_struct *mm, efi_memory_desc_t *md); · 7 unmodified lines | ||
| 48 | 50 | */ |
| 49 | 51 | #define EFI_FDT_ALIGN SZ_2M /* used by allocate_new_fdt_and_exit_boot() */ |
| 50 | 52 | |
| 53 | +/* | |
| 54 | + * In some configurations (e.g. VMAP_STACK && 64K pages), stacks built into the | |
| 55 | + * kernel need greater alignment than we require the segments to be padded to. | |
| 56 | + */ | |
| 57 | +#define EFI_KIMG_ALIGN \ | |
| 58 | + (SEGMENT_ALIGN > THREAD_ALIGN ? SEGMENT_ALIGN : THREAD_ALIGN) | |
| 59 | + | |
| 51 | 60 | /* on arm64, the FDT may be located anywhere in system RAM */ |
| 52 | 61 | static inline unsigned long efi_get_max_fdt_addr(unsigned long dram_base) |
| 53 | 62 | { |
| @@ -139,7 +139,6 @@ typedef struct user_fpsimd_state elf_fpregset_t; · 138 unmodified lines | ||
| 139 | 139 | |
| 140 | 140 | #define SET_PERSONALITY(ex) \ |
| 141 | 141 | ({ \ |
| 142 | − clear_bit(TIF_32BIT, ¤t->mm->context.flags); \ | |
| 143 | 142 | clear_thread_flag(TIF_32BIT); \ |
| 144 | 143 | current->personality &= ~READ_IMPLIES_EXEC; \ |
| 145 | 144 | }) |
| @@ -195,7 +194,6 @@ typedef compat_elf_greg_t compat_elf_gregset_t[COMPAT_ELF_NGREG]; · 49 unmodified lines | ||
| 195 | 194 | */ |
| 196 | 195 | #define COMPAT_SET_PERSONALITY(ex) \ |
| 197 | 196 | ({ \ |
| 198 | − set_bit(TIF_32BIT, ¤t->mm->context.flags); \ | |
| 199 | 197 | set_thread_flag(TIF_32BIT); \ |
| 200 | 198 | }) |
| 201 | 199 | #define COMPAT_ARCH_DLINFO |
| @@ -77,16 +77,23 @@ · 76 unmodified lines | ||
| 77 | 77 | #define ESR_ELx_EC_MASK (UL(0x3F) << ESR_ELx_EC_SHIFT) |
| 78 | 78 | #define ESR_ELx_EC(esr) (((esr) & ESR_ELx_EC_MASK) >> ESR_ELx_EC_SHIFT) |
| 79 | 79 | |
| 80 | −#define ESR_ELx_IL (UL(1) << 25) | |
| 80 | +#define ESR_ELx_IL_SHIFT (25) | |
| 81 | +#define ESR_ELx_IL (UL(1) << ESR_ELx_IL_SHIFT) | |
| 81 | 82 | #define ESR_ELx_ISS_MASK (ESR_ELx_IL - 1) |
| 82 | 83 | |
| 83 | 84 | /* ISS field definitions shared by different classes */ |
| 84 | −#define ESR_ELx_WNR (UL(1) << 6) | |
| 85 | +#define ESR_ELx_WNR_SHIFT (6) | |
| 86 | +#define ESR_ELx_WNR (UL(1) << ESR_ELx_WNR_SHIFT) | |
| 85 | 87 | |
| 86 | 88 | /* Shared ISS field definitions for Data/Instruction aborts */ |
| 87 | −#define ESR_ELx_FnV (UL(1) << 10) | |
| 88 | −#define ESR_ELx_EA (UL(1) << 9) | |
| 89 | −#define ESR_ELx_S1PTW (UL(1) << 7) | |
| 89 | +#define ESR_ELx_SET_SHIFT (11) | |
| 90 | +#define ESR_ELx_SET_MASK (UL(3) << ESR_ELx_SET_SHIFT) | |
| 91 | +#define ESR_ELx_FnV_SHIFT (10) | |
| 92 | +#define ESR_ELx_FnV (UL(1) << ESR_ELx_FnV_SHIFT) | |
| 93 | +#define ESR_ELx_EA_SHIFT (9) | |
| 94 | +#define ESR_ELx_EA (UL(1) << ESR_ELx_EA_SHIFT) | |
| 95 | +#define ESR_ELx_S1PTW_SHIFT (7) | |
| 96 | +#define ESR_ELx_S1PTW (UL(1) << ESR_ELx_S1PTW_SHIFT) | |
| 90 | 97 | |
| 91 | 98 | /* Shared ISS fault status code(IFSC/DFSC) for Data/Instruction aborts */ |
| 92 | 99 | #define ESR_ELx_FSC (0x3F) |
| @@ -97,15 +104,20 @@ · 4 unmodified lines | ||
| 97 | 104 | #define ESR_ELx_FSC_PERM (0x0C) |
| 98 | 105 | |
| 99 | 106 | /* ISS field definitions for Data Aborts */ |
| 100 | −#define ESR_ELx_ISV (UL(1) << 24) | |
| 107 | +#define ESR_ELx_ISV_SHIFT (24) | |
| 108 | +#define ESR_ELx_ISV (UL(1) << ESR_ELx_ISV_SHIFT) | |
| 101 | 109 | #define ESR_ELx_SAS_SHIFT (22) |
| 102 | 110 | #define ESR_ELx_SAS (UL(3) << ESR_ELx_SAS_SHIFT) |
| 103 | −#define ESR_ELx_SSE (UL(1) << 21) | |
| 111 | +#define ESR_ELx_SSE_SHIFT (21) | |
| 112 | +#define ESR_ELx_SSE (UL(1) << ESR_ELx_SSE_SHIFT) | |
| 104 | 113 | #define ESR_ELx_SRT_SHIFT (16) |
| 105 | 114 | #define ESR_ELx_SRT_MASK (UL(0x1F) << ESR_ELx_SRT_SHIFT) |
| 106 | −#define ESR_ELx_SF (UL(1) << 15) | |
| 107 | −#define ESR_ELx_AR (UL(1) << 14) | |
| 108 | −#define ESR_ELx_CM (UL(1) << 8) | |
| 115 | +#define ESR_ELx_SF_SHIFT (15) | |
| 116 | +#define ESR_ELx_SF (UL(1) << ESR_ELx_SF_SHIFT) | |
| 117 | +#define ESR_ELx_AR_SHIFT (14) | |
| 118 | +#define ESR_ELx_AR (UL(1) << ESR_ELx_AR_SHIFT) | |
| 119 | +#define ESR_ELx_CM_SHIFT (8) | |
| 120 | +#define ESR_ELx_CM (UL(1) << ESR_ELx_CM_SHIFT) | |
| 109 | 121 | |
| 110 | 122 | /* ISS field definitions for exceptions taken in to Hyp */ |
| 111 | 123 | #define ESR_ELx_CV (UL(1) << 24) |
| @@ -157,9 +169,10 @@ · 45 unmodified lines | ||
| 157 | 169 | /* |
| 158 | 170 | * User space cache operations have the following sysreg encoding |
| 159 | 171 | * in System instructions. |
| 160 | − * op0=1, op1=3, op2=1, crn=7, crm={ 5, 10, 11, 14 }, WRITE (L=0) | |
| 172 | + * op0=1, op1=3, op2=1, crn=7, crm={ 5, 10, 11, 12, 14 }, WRITE (L=0) | |
| 161 | 173 | */ |
| 162 | 174 | #define ESR_ELx_SYS64_ISS_CRM_DC_CIVAC 14 |
| 175 | +#define ESR_ELx_SYS64_ISS_CRM_DC_CVAP 12 | |
| 163 | 176 | #define ESR_ELx_SYS64_ISS_CRM_DC_CVAU 11 |
| 164 | 177 | #define ESR_ELx_SYS64_ISS_CRM_DC_CVAC 10 |
| 165 | 178 | #define ESR_ELx_SYS64_ISS_CRM_IC_IVAU 5 |
| @@ -209,6 +222,13 @@ · 43 unmodified lines | ||
| 209 | 222 | #ifndef __ASSEMBLY__ |
| 210 | 223 | #include <asm/types.h> |
| 211 | 224 | |
| 225 | +static inline bool esr_is_data_abort(u32 esr) | |
| 226 | +{ | |
| 227 | + const u32 ec = ESR_ELx_EC(esr); | |
| 228 | + | |
| 229 | + return ec == ESR_ELx_EC_DABT_LOW || ec == ESR_ELx_EC_DABT_CUR; | |
| 230 | +} | |
| 231 | + | |
| 212 | 232 | const char *esr_get_class_string(u32 esr); |
| 213 | 233 | #endif /* __ASSEMBLY */ |
| 214 | 234 | |
| @@ -41,16 +41,6 @@ struct fpsimd_state { · 40 unmodified lines | ||
| 41 | 41 | unsigned int cpu; |
| 42 | 42 | }; |
| 43 | 43 | |
| 44 | −/* | |
| 45 | − * Struct for stacking the bottom 'n' FP/SIMD registers. | |
| 46 | − */ | |
| 47 | −struct fpsimd_partial_state { | |
| 48 | − u32 fpsr; | |
| 49 | − u32 fpcr; | |
| 50 | − u32 num_regs; | |
| 51 | − __uint128_t vregs[32]; | |
| 52 | −}; | |
| 53 | − | |
| 54 | 44 | |
| 55 | 45 | #if defined(__KERNEL__) && defined(CONFIG_COMPAT) |
| 56 | 46 | /* Masks for extracting the FPSR and FPCR from the FPSCR */ |
| @@ -77,9 +67,9 @@ extern void fpsimd_update_current_state(struct fpsimd_state *state); · 20 unmodified lines | ||
| 77 | 67 | |
| 78 | 68 | extern void fpsimd_flush_task_state(struct task_struct *target); |
| 79 | 69 | |
| 80 | −extern void fpsimd_save_partial_state(struct fpsimd_partial_state *state, | |
| 81 | − u32 num_regs); | |
| 82 | −extern void fpsimd_load_partial_state(struct fpsimd_partial_state *state); | |
| 70 | +/* For use by EFI runtime services calls only */ | |
| 71 | +extern void __efi_fpsimd_begin(void); | |
| 72 | +extern void __efi_fpsimd_end(void); | |
| 83 | 73 | |
| 84 | 74 | #endif |
| 85 | 75 | |
| @@ -75,59 +75,3 @@ · 74 unmodified lines | ||
| 75 | 75 | ldr w\tmpnr, [\state, #16 * 2 + 4] |
| 76 | 76 | fpsimd_restore_fpcr x\tmpnr, \state |
| 77 | 77 | .endm |
| 78 | − | |
| 79 | −.macro fpsimd_save_partial state, numnr, tmpnr1, tmpnr2 | |
| 80 | − mrs x\tmpnr1, fpsr | |
| 81 | − str w\numnr, [\state, #8] | |
| 82 | − mrs x\tmpnr2, fpcr | |
| 83 | − stp w\tmpnr1, w\tmpnr2, [\state] | |
| 84 | − adr x\tmpnr1, 0f | |
| 85 | − add \state, \state, x\numnr, lsl #4 | |
| 86 | − sub x\tmpnr1, x\tmpnr1, x\numnr, lsl #1 | |
| 87 | − br x\tmpnr1 | |
| 88 | − stp q30, q31, [\state, #-16 * 30 - 16] | |
| 89 | − stp q28, q29, [\state, #-16 * 28 - 16] | |
| 90 | − stp q26, q27, [\state, #-16 * 26 - 16] | |
| 91 | − stp q24, q25, [\state, #-16 * 24 - 16] | |
| 92 | − stp q22, q23, [\state, #-16 * 22 - 16] | |
| 93 | − stp q20, q21, [\state, #-16 * 20 - 16] | |
| 94 | − stp q18, q19, [\state, #-16 * 18 - 16] | |
| 95 | − stp q16, q17, [\state, #-16 * 16 - 16] | |
| 96 | − stp q14, q15, [\state, #-16 * 14 - 16] | |
| 97 | − stp q12, q13, [\state, #-16 * 12 - 16] | |
| 98 | − stp q10, q11, [\state, #-16 * 10 - 16] | |
| 99 | − stp q8, q9, [\state, #-16 * 8 - 16] | |
| 100 | − stp q6, q7, [\state, #-16 * 6 - 16] | |
| 101 | − stp q4, q5, [\state, #-16 * 4 - 16] | |
| 102 | − stp q2, q3, [\state, #-16 * 2 - 16] | |
| 103 | − stp q0, q1, [\state, #-16 * 0 - 16] | |
| 104 | −0: | |
| 105 | −.endm | |
| 106 | − | |
| 107 | −.macro fpsimd_restore_partial state, tmpnr1, tmpnr2 | |
| 108 | − ldp w\tmpnr1, w\tmpnr2, [\state] | |
| 109 | − msr fpsr, x\tmpnr1 | |
| 110 | − fpsimd_restore_fpcr x\tmpnr2, x\tmpnr1 | |
| 111 | − adr x\tmpnr1, 0f | |
| 112 | − ldr w\tmpnr2, [\state, #8] | |
| 113 | − add \state, \state, x\tmpnr2, lsl #4 | |
| 114 | − sub x\tmpnr1, x\tmpnr1, x\tmpnr2, lsl #1 | |
| 115 | − br x\tmpnr1 | |
| 116 | − ldp q30, q31, [\state, #-16 * 30 - 16] | |
| 117 | − ldp q28, q29, [\state, #-16 * 28 - 16] | |
| 118 | − ldp q26, q27, [\state, #-16 * 26 - 16] | |
| 119 | − ldp q24, q25, [\state, #-16 * 24 - 16] | |
| 120 | − ldp q22, q23, [\state, #-16 * 22 - 16] | |
| 121 | − ldp q20, q21, [\state, #-16 * 20 - 16] | |
| 122 | − ldp q18, q19, [\state, #-16 * 18 - 16] | |
| 123 | − ldp q16, q17, [\state, #-16 * 16 - 16] | |
| 124 | − ldp q14, q15, [\state, #-16 * 14 - 16] | |
| 125 | − ldp q12, q13, [\state, #-16 * 12 - 16] | |
| 126 | − ldp q10, q11, [\state, #-16 * 10 - 16] | |
| 127 | − ldp q8, q9, [\state, #-16 * 8 - 16] | |
| 128 | − ldp q6, q7, [\state, #-16 * 6 - 16] | |
| 129 | − ldp q4, q5, [\state, #-16 * 4 - 16] | |
| 130 | − ldp q2, q3, [\state, #-16 * 2 - 16] | |
| 131 | − ldp q0, q1, [\state, #-16 * 0 - 16] | |
| 132 | −0: | |
| 133 | −.endm | |
| @@ -18,7 +18,6 @@ · 17 unmodified lines | ||
| 18 | 18 | #ifndef __ASM_HUGETLB_H |
| 19 | 19 | #define __ASM_HUGETLB_H |
| 20 | 20 | |
| 21 | −#include <asm-generic/hugetlb.h> | |
| 22 | 21 | #include <asm/page.h> |
| 23 | 22 | |
| 24 | 23 | static inline pte_t huge_ptep_get(pte_t *ptep) |
| @@ -82,6 +81,14 @@ extern void huge_ptep_set_wrprotect(struct mm_struct *mm, · 57 unmodified lines | ||
| 82 | 81 | unsigned long addr, pte_t *ptep); |
| 83 | 82 | extern void huge_ptep_clear_flush(struct vm_area_struct *vma, |
| 84 | 83 | unsigned long addr, pte_t *ptep); |
| 84 | +extern void huge_pte_clear(struct mm_struct *mm, unsigned long addr, | |
| 85 | + pte_t *ptep, unsigned long sz); | |
| 86 | +#define huge_pte_clear huge_pte_clear | |
| 87 | +extern void set_huge_swap_pte_at(struct mm_struct *mm, unsigned long addr, | |
| 88 | + pte_t *ptep, pte_t pte, unsigned long sz); | |
| 89 | +#define set_huge_swap_pte_at set_huge_swap_pte_at | |
| 90 | + | |
| 91 | +#include <asm-generic/hugetlb.h> | |
| 85 | 92 | |
| 86 | 93 | #ifdef CONFIG_ARCH_HAS_GIGANTIC_PAGE |
| 87 | 94 | static inline bool gigantic_page_supported(void) { return true; } |
| @@ -1,45 +1,12 @@ | ||
| 1 | 1 | #ifndef __ASM_IRQ_H |
| 2 | 2 | #define __ASM_IRQ_H |
| 3 | 3 | |
| 4 | −#define IRQ_STACK_SIZE THREAD_SIZE | |
| 5 | −#define IRQ_STACK_START_SP THREAD_START_SP | |
| 6 | − | |
| 7 | 4 | #ifndef __ASSEMBLER__ |
| 8 | 5 | |
| 9 | −#include <linux/percpu.h> | |
| 10 | − | |
| 11 | 6 | #include <asm-generic/irq.h> |
| 12 | −#include <asm/thread_info.h> | |
| 13 | 7 | |
| 14 | 8 | struct pt_regs; |
| 15 | 9 | |
| 16 | −DECLARE_PER_CPU(unsigned long [IRQ_STACK_SIZE/sizeof(long)], irq_stack); | |
| 17 | − | |
| 18 | −/* | |
| 19 | − * The highest address on the stack, and the first to be used. Used to | |
| 20 | − * find the dummy-stack frame put down by el?_irq() in entry.S, which | |
| 21 | − * is structured as follows: | |
| 22 | − * | |
| 23 | − * ------------ | |
| 24 | − * | | <- irq_stack_ptr | |
| 25 | − * top ------------ | |
| 26 | − * | x19 | <- irq_stack_ptr - 0x08 | |
| 27 | − * ------------ | |
| 28 | − * | x29 | <- irq_stack_ptr - 0x10 | |
| 29 | − * ------------ | |
| 30 | − * | |
| 31 | − * where x19 holds a copy of the task stack pointer where the struct pt_regs | |
| 32 | − * from kernel_entry can be found. | |
| 33 | − * | |
| 34 | − */ | |
| 35 | −#define IRQ_STACK_PTR(cpu) ((unsigned long)per_cpu(irq_stack, cpu) + IRQ_STACK_START_SP) | |
| 36 | − | |
| 37 | −/* | |
| 38 | − * The offset from irq_stack_ptr where entry.S will store the original | |
| 39 | − * stack pointer. Used by unwind_frame() and dump_backtrace(). | |
| 40 | − */ | |
| 41 | −#define IRQ_STACK_TO_TASK_STACK(ptr) (*((unsigned long *)((ptr) - 0x08))) | |
| 42 | − | |
| 43 | 10 | extern void set_handle_irq(void (*handle_irq)(struct pt_regs *)); |
| 44 | 11 | |
| 45 | 12 | static inline int nr_legacy_irqs(void) |
| @@ -47,14 +14,5 @@ static inline int nr_legacy_irqs(void) · 1 unmodified line | ||
| 47 | 14 | return 0; |
| 48 | 15 | } |
| 49 | 16 | |
| 50 | −static inline bool on_irq_stack(unsigned long sp, int cpu) | |
| 51 | −{ | |
| 52 | − /* variable names the same as kernel/stacktrace.c */ | |
| 53 | − unsigned long low = (unsigned long)per_cpu(irq_stack, cpu); | |
| 54 | − unsigned long high = low + IRQ_STACK_START_SP; | |
| 55 | − | |
| 56 | − return (low <= sp && sp <= high); | |
| 57 | −} | |
| 58 | − | |
| 59 | 17 | #endif /* !__ASSEMBLER__ */ |
| 60 | 18 | #endif |
| @@ -175,18 +175,15 @@ static inline pmd_t kvm_s2pmd_mkwrite(pmd_t pmd) · 174 unmodified lines | ||
| 175 | 175 | |
| 176 | 176 | static inline void kvm_set_s2pte_readonly(pte_t *pte) |
| 177 | 177 | { |
| 178 | − pteval_t pteval; | |
| 179 | − unsigned long tmp; | |
| 180 | − | |
| 181 | − asm volatile("// kvm_set_s2pte_readonly\n" | |
| 182 | − " prfm pstl1strm, %2\n" | |
| 183 | − "1: ldxr %0, %2\n" | |
| 184 | − " and %0, %0, %3 // clear PTE_S2_RDWR\n" | |
| 185 | − " orr %0, %0, %4 // set PTE_S2_RDONLY\n" | |
| 186 | − " stxr %w1, %0, %2\n" | |
| 187 | − " cbnz %w1, 1b\n" | |
| 188 | − : "=&r" (pteval), "=&r" (tmp), "+Q" (pte_val(*pte)) | |
| 189 | − : "L" (~PTE_S2_RDWR), "L" (PTE_S2_RDONLY)); | |
| 178 | + pteval_t old_pteval, pteval; | |
| 179 | + | |
| 180 | + pteval = READ_ONCE(pte_val(*pte)); | |
| 181 | + do { | |
| 182 | + old_pteval = pteval; | |
| 183 | + pteval &= ~PTE_S2_RDWR; | |
| 184 | + pteval |= PTE_S2_RDONLY; | |
| 185 | + pteval = cmpxchg_relaxed(&pte_val(*pte), old_pteval, pteval); | |
| 186 | + } while (pteval != old_pteval); | |
| 190 | 187 | } |
| 191 | 188 | |
| 192 | 189 | static inline bool kvm_s2pte_readonly(pte_t *pte) |
| @@ -25,6 +25,7 @@ · 24 unmodified lines | ||
| 25 | 25 | #include <linux/const.h> |
| 26 | 26 | #include <linux/types.h> |
| 27 | 27 | #include <asm/bug.h> |
| 28 | +#include <asm/page-def.h> | |
| 28 | 29 | #include <asm/sizes.h> |
| 29 | 30 | |
| 30 | 31 | /* |
| @@ -103,6 +104,58 @@ · 72 unmodified lines | ||
| 103 | 104 | #define KASAN_SHADOW_SIZE (0) |
| 104 | 105 | #endif |
| 105 | 106 | |
| 107 | +#define MIN_THREAD_SHIFT 14 | |
| 108 | + | |
| 109 | +/* | |
| 110 | + * VMAP'd stacks are allocated at page granularity, so we must ensure that such | |
| 111 | + * stacks are a multiple of page size. | |
| 112 | + */ | |
| 113 | +#if defined(CONFIG_VMAP_STACK) && (MIN_THREAD_SHIFT < PAGE_SHIFT) | |
| 114 | +#define THREAD_SHIFT PAGE_SHIFT | |
| 115 | +#else | |
| 116 | +#define THREAD_SHIFT MIN_THREAD_SHIFT | |
| 117 | +#endif | |
| 118 | + | |
| 119 | +#if THREAD_SHIFT >= PAGE_SHIFT | |
| 120 | +#define THREAD_SIZE_ORDER (THREAD_SHIFT - PAGE_SHIFT) | |
| 121 | +#endif | |
| 122 | + | |
| 123 | +#define THREAD_SIZE (UL(1) << THREAD_SHIFT) | |
| 124 | + | |
| 125 | +/* | |
| 126 | + * By aligning VMAP'd stacks to 2 * THREAD_SIZE, we can detect overflow by | |
| 127 | + * checking sp & (1 << THREAD_SHIFT), which we can do cheaply in the entry | |
| 128 | + * assembly. | |
| 129 | + */ | |
| 130 | +#ifdef CONFIG_VMAP_STACK | |
| 131 | +#define THREAD_ALIGN (2 * THREAD_SIZE) | |
| 132 | +#else | |
| 133 | +#define THREAD_ALIGN THREAD_SIZE | |
| 134 | +#endif | |
| 135 | + | |
| 136 | +#define IRQ_STACK_SIZE THREAD_SIZE | |
| 137 | + | |
| 138 | +#define OVERFLOW_STACK_SIZE SZ_4K | |
| 139 | + | |
| 140 | +/* | |
| 141 | + * Alignment of kernel segments (e.g. .text, .data). | |
| 142 | + */ | |
| 143 | +#if defined(CONFIG_DEBUG_ALIGN_RODATA) | |
| 144 | +/* | |
| 145 | + * 4 KB granule: 1 level 2 entry | |
| 146 | + * 16 KB granule: 128 level 3 entries, with contiguous bit | |
| 147 | + * 64 KB granule: 32 level 3 entries, with contiguous bit | |
| 148 | + */ | |
| 149 | +#define SEGMENT_ALIGN SZ_2M | |
| 150 | +#else | |
| 151 | +/* | |
| 152 | + * 4 KB granule: 16 level 3 entries, with contiguous bit | |
| 153 | + * 16 KB granule: 4 level 3 entries, without contiguous bit | |
| 154 | + * 64 KB granule: 1 level 3 entry | |
| 155 | + */ | |
| 156 | +#define SEGMENT_ALIGN SZ_64K | |
| 157 | +#endif | |
| 158 | + | |
| 106 | 159 | /* |
| 107 | 160 | * Memory types available. |
| 108 | 161 | */ |
| @@ -16,6 +16,8 @@ · 15 unmodified lines | ||
| 16 | 16 | #ifndef __ASM_MMU_H |
| 17 | 17 | #define __ASM_MMU_H |
| 18 | 18 | |
| 19 | +#define MMCF_AARCH32 0x1 /* mm context flag for AArch32 executables */ | |
| 20 | + | |
| 19 | 21 | typedef struct { |
| 20 | 22 | atomic64_t id; |
| 21 | 23 | void *vdso; |
| @@ -8,12 +8,22 @@ · 7 unmodified lines | ||
| 8 | 8 | * published by the Free Software Foundation. |
| 9 | 9 | */ |
| 10 | 10 | |
| 11 | +#ifndef __ASM_NEON_H | |
| 12 | +#define __ASM_NEON_H | |
| 13 | + | |
| 11 | 14 | #include <linux/types.h> |
| 12 | 15 | #include <asm/fpsimd.h> |
| 13 | 16 | |
| 14 | 17 | #define cpu_has_neon() system_supports_fpsimd() |
| 15 | 18 | |
| 16 | −#define kernel_neon_begin() kernel_neon_begin_partial(32) | |
| 17 | − | |
| 18 | −void kernel_neon_begin_partial(u32 num_regs); | |
| 19 | +void kernel_neon_begin(void); | |
| 19 | 20 | void kernel_neon_end(void); |
| 21 | + | |
| 22 | +/* | |
| 23 | + * Temporary macro to allow the crypto code to compile. Note that the | |
| 24 | + * semantics of kernel_neon_begin_partial() are now different from the | |
| 25 | + * original as it does not allow being called in an interrupt context. | |
| 26 | + */ | |
| 27 | +#define kernel_neon_begin_partial(num_regs) kernel_neon_begin() | |
| 28 | + | |
| 29 | +#endif /* ! __ASM_NEON_H */ | |
| @@ -7,9 +7,6 @@ · 6 unmodified lines | ||
| 7 | 7 | |
| 8 | 8 | #define NR_NODE_MEMBLKS (MAX_NUMNODES * 2) |
| 9 | 9 | |
| 10 | −/* currently, arm64 implements flat NUMA topology */ | |
| 11 | −#define parent_node(node) (node) | |
| 12 | − | |
| 13 | 10 | int __node_distance(int from, int to); |
| 14 | 11 | #define node_distance(a, b) __node_distance(a, b) |
| 15 | 12 | |
| @@ -0,0 +1,34 @@ | ||
| 1 | +/* | |
| 2 | + * Based on arch/arm/include/asm/page.h | |
| 3 | + * | |
| 4 | + * Copyright (C) 1995-2003 Russell King | |
| 5 | + * Copyright (C) 2017 ARM Ltd. | |
| 6 | + * | |
| 7 | + * This program is free software; you can redistribute it and/or modify | |
| 8 | + * it under the terms of the GNU General Public License version 2 as | |
| 9 | + * published by the Free Software Foundation. | |
| 10 | + * | |
| 11 | + * This program is distributed in the hope that it will be useful, | |
| 12 | + * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
| 13 | + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
| 14 | + * GNU General Public License for more details. | |
| 15 | + * | |
| 16 | + * You should have received a copy of the GNU General Public License | |
| 17 | + * along with this program. If not, see <http://www.gnu.org/licenses/>. | |
| 18 | + */ | |
| 19 | +#ifndef __ASM_PAGE_DEF_H | |
| 20 | +#define __ASM_PAGE_DEF_H | |
| 21 | + | |
| 22 | +#include <linux/const.h> | |
| 23 | + | |
| 24 | +/* PAGE_SHIFT determines the page size */ | |
| 25 | +/* CONT_SHIFT determines the number of pages which can be tracked together */ | |
| 26 | +#define PAGE_SHIFT CONFIG_ARM64_PAGE_SHIFT | |
| 27 | +#define CONT_SHIFT CONFIG_ARM64_CONT_SHIFT | |
| 28 | +#define PAGE_SIZE (_AC(1, UL) << PAGE_SHIFT) | |
| 29 | +#define PAGE_MASK (~(PAGE_SIZE-1)) | |
| 30 | + | |
| 31 | +#define CONT_SIZE (_AC(1, UL) << (CONT_SHIFT + PAGE_SHIFT)) | |
| 32 | +#define CONT_MASK (~(CONT_SIZE-1)) | |
| 33 | + | |
| 34 | +#endif /* __ASM_PAGE_DEF_H */ | |
| @@ -19,17 +19,7 @@ · 18 unmodified lines | ||
| 19 | 19 | #ifndef __ASM_PAGE_H |
| 20 | 20 | #define __ASM_PAGE_H |
| 21 | 21 | |
| 22 | −#include <linux/const.h> | |
| 23 | − | |
| 24 | −/* PAGE_SHIFT determines the page size */ | |
| 25 | −/* CONT_SHIFT determines the number of pages which can be tracked together */ | |
| 26 | −#define PAGE_SHIFT CONFIG_ARM64_PAGE_SHIFT | |
| 27 | −#define CONT_SHIFT CONFIG_ARM64_CONT_SHIFT | |
| 28 | −#define PAGE_SIZE (_AC(1, UL) << PAGE_SHIFT) | |
| 29 | −#define PAGE_MASK (~(PAGE_SIZE-1)) | |
| 30 | − | |
| 31 | −#define CONT_SIZE (_AC(1, UL) << (CONT_SHIFT + PAGE_SHIFT)) | |
| 32 | −#define CONT_MASK (~(CONT_SIZE-1)) | |
| 22 | +#include <asm/page-def.h> | |
| 33 | 23 | |
| 34 | 24 | #ifndef __ASSEMBLY__ |
| 35 | 25 | |
| @@ -63,23 +63,21 @@ · 62 unmodified lines | ||
| 63 | 63 | #define PAGE_S2 __pgprot(PROT_DEFAULT | PTE_S2_MEMATTR(MT_S2_NORMAL) | PTE_S2_RDONLY) |
| 64 | 64 | #define PAGE_S2_DEVICE __pgprot(PROT_DEFAULT | PTE_S2_MEMATTR(MT_S2_DEVICE_nGnRE) | PTE_S2_RDONLY | PTE_UXN) |
| 65 | 65 | |
| 66 | −#define PAGE_NONE __pgprot(((_PAGE_DEFAULT) & ~PTE_VALID) | PTE_PROT_NONE | PTE_PXN | PTE_UXN) | |
| 66 | +#define PAGE_NONE __pgprot(((_PAGE_DEFAULT) & ~PTE_VALID) | PTE_PROT_NONE | PTE_RDONLY | PTE_PXN | PTE_UXN) | |
| 67 | 67 | #define PAGE_SHARED __pgprot(_PAGE_DEFAULT | PTE_USER | PTE_NG | PTE_PXN | PTE_UXN | PTE_WRITE) |
| 68 | 68 | #define PAGE_SHARED_EXEC __pgprot(_PAGE_DEFAULT | PTE_USER | PTE_NG | PTE_PXN | PTE_WRITE) |
| 69 | −#define PAGE_COPY __pgprot(_PAGE_DEFAULT | PTE_USER | PTE_NG | PTE_PXN | PTE_UXN) | |
| 70 | −#define PAGE_COPY_EXEC __pgprot(_PAGE_DEFAULT | PTE_USER | PTE_NG | PTE_PXN) | |
| 71 | −#define PAGE_READONLY __pgprot(_PAGE_DEFAULT | PTE_USER | PTE_NG | PTE_PXN | PTE_UXN) | |
| 72 | −#define PAGE_READONLY_EXEC __pgprot(_PAGE_DEFAULT | PTE_USER | PTE_NG | PTE_PXN) | |
| 73 | −#define PAGE_EXECONLY __pgprot(_PAGE_DEFAULT | PTE_NG | PTE_PXN) | |
| 69 | +#define PAGE_READONLY __pgprot(_PAGE_DEFAULT | PTE_USER | PTE_RDONLY | PTE_NG | PTE_PXN | PTE_UXN) | |
| 70 | +#define PAGE_READONLY_EXEC __pgprot(_PAGE_DEFAULT | PTE_USER | PTE_RDONLY | PTE_NG | PTE_PXN) | |
| 71 | +#define PAGE_EXECONLY __pgprot(_PAGE_DEFAULT | PTE_RDONLY | PTE_NG | PTE_PXN) | |
| 74 | 72 | |
| 75 | 73 | #define __P000 PAGE_NONE |
| 76 | 74 | #define __P001 PAGE_READONLY |
| 77 | −#define __P010 PAGE_COPY | |
| 78 | −#define __P011 PAGE_COPY | |
| 75 | +#define __P010 PAGE_READONLY | |
| 76 | +#define __P011 PAGE_READONLY | |
| 79 | 77 | #define __P100 PAGE_EXECONLY |
| 80 | 78 | #define __P101 PAGE_READONLY_EXEC |
| 81 | −#define __P110 PAGE_COPY_EXEC | |
| 82 | −#define __P111 PAGE_COPY_EXEC | |
| 79 | +#define __P110 PAGE_READONLY_EXEC | |
| 80 | +#define __P111 PAGE_READONLY_EXEC | |
| 83 | 81 | |
| 84 | 82 | #define __S000 PAGE_NONE |
| 85 | 83 | #define __S001 PAGE_READONLY |
| @@ -39,6 +39,7 @@ · 38 unmodified lines | ||
| 39 | 39 | |
| 40 | 40 | #ifndef __ASSEMBLY__ |
| 41 | 41 | |
| 42 | +#include <asm/cmpxchg.h> | |
| 42 | 43 | #include <asm/fixmap.h> |
| 43 | 44 | #include <linux/mmdebug.h> |
| 44 | 45 | |
| @@ -84,11 +85,7 @@ extern unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)]; · 39 unmodified lines | ||
| 84 | 85 | (__boundary - 1 < (end) - 1) ? __boundary : (end); \ |
| 85 | 86 | }) |
| 86 | 87 | |
| 87 | −#ifdef CONFIG_ARM64_HW_AFDBM | |
| 88 | 88 | #define pte_hw_dirty(pte) (pte_write(pte) && !(pte_val(pte) & PTE_RDONLY)) |
| 89 | −#else | |
| 90 | −#define pte_hw_dirty(pte) (0) | |
| 91 | −#endif | |
| 92 | 89 | #define pte_sw_dirty(pte) (!!(pte_val(pte) & PTE_DIRTY)) |
| 93 | 90 | #define pte_dirty(pte) (pte_sw_dirty(pte) || pte_hw_dirty(pte)) |
| 94 | 91 | |
| @@ -124,12 +121,16 @@ static inline pte_t set_pte_bit(pte_t pte, pgprot_t prot) · 29 unmodified lines | ||
| 124 | 121 | |
| 125 | 122 | static inline pte_t pte_wrprotect(pte_t pte) |
| 126 | 123 | { |
| 127 | − return clear_pte_bit(pte, __pgprot(PTE_WRITE)); | |
| 124 | + pte = clear_pte_bit(pte, __pgprot(PTE_WRITE)); | |
| 125 | + pte = set_pte_bit(pte, __pgprot(PTE_RDONLY)); | |
| 126 | + return pte; | |
| 128 | 127 | } |
| 129 | 128 | |
| 130 | 129 | static inline pte_t pte_mkwrite(pte_t pte) |
| 131 | 130 | { |
| 132 | − return set_pte_bit(pte, __pgprot(PTE_WRITE)); | |
| 131 | + pte = set_pte_bit(pte, __pgprot(PTE_WRITE)); | |
| 132 | + pte = clear_pte_bit(pte, __pgprot(PTE_RDONLY)); | |
| 133 | + return pte; | |
| 133 | 134 | } |
| 134 | 135 | |
| 135 | 136 | static inline pte_t pte_mkclean(pte_t pte) |
| @@ -168,11 +169,6 @@ static inline pte_t pte_mknoncont(pte_t pte) · 32 unmodified lines | ||
| 168 | 169 | return clear_pte_bit(pte, __pgprot(PTE_CONT)); |
| 169 | 170 | } |
| 170 | 171 | |
| 171 | −static inline pte_t pte_clear_rdonly(pte_t pte) | |
| 172 | −{ | |
| 173 | − return clear_pte_bit(pte, __pgprot(PTE_RDONLY)); | |
| 174 | −} | |
| 175 | − | |
| 176 | 172 | static inline pte_t pte_mkpresent(pte_t pte) |
| 177 | 173 | { |
| 178 | 174 | return set_pte_bit(pte, __pgprot(PTE_VALID)); |
| @@ -220,22 +216,15 @@ extern void __sync_icache_dcache(pte_t pteval, unsigned long addr); · 41 unmodified lines | ||
| 220 | 216 | static inline void set_pte_at(struct mm_struct *mm, unsigned long addr, |
| 221 | 217 | pte_t *ptep, pte_t pte) |
| 222 | 218 | { |
| 223 | − if (pte_present(pte)) { | |
| 224 | − if (pte_sw_dirty(pte) && pte_write(pte)) | |
| 225 | − pte_val(pte) &= ~PTE_RDONLY; | |
| 226 | − else | |
| 227 | − pte_val(pte) |= PTE_RDONLY; | |
| 228 | − if (pte_user_exec(pte) && !pte_special(pte)) | |
| 229 | − __sync_icache_dcache(pte, addr); | |
| 230 | − } | |
| 219 | + if (pte_present(pte) && pte_user_exec(pte) && !pte_special(pte)) | |
| 220 | + __sync_icache_dcache(pte, addr); | |
| 231 | 221 | |
| 232 | 222 | /* |
| 233 | 223 | * If the existing pte is valid, check for potential race with |
| 234 | 224 | * hardware updates of the pte (ptep_set_access_flags safely changes |
| 235 | 225 | * valid ptes without going through an invalid entry). |
| 236 | 226 | */ |
| 237 | − if (IS_ENABLED(CONFIG_ARM64_HW_AFDBM) && | |
| 238 | − pte_valid(*ptep) && pte_valid(pte)) { | |
| 227 | + if (pte_valid(*ptep) && pte_valid(pte)) { | |
| 239 | 228 | VM_WARN_ONCE(!pte_young(pte), |
| 240 | 229 | "%s: racy access flag clearing: 0x%016llx -> 0x%016llx", |
| 241 | 230 | __func__, pte_val(*ptep), pte_val(pte)); |
| @@ -571,7 +560,6 @@ static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot) · 329 unmodified lines | ||
| 571 | 560 | return pte_pmd(pte_modify(pmd_pte(pmd), newprot)); |
| 572 | 561 | } |
| 573 | 562 | |
| 574 | −#ifdef CONFIG_ARM64_HW_AFDBM | |
| 575 | 563 | #define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS |
| 576 | 564 | extern int ptep_set_access_flags(struct vm_area_struct *vma, |
| 577 | 565 | unsigned long address, pte_t *ptep, |
| @@ -593,20 +581,17 @@ static inline int pmdp_set_access_flags(struct vm_area_struct *vma, · 15 unmodified lines | ||
| 593 | 581 | #define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG |
| 594 | 582 | static inline int __ptep_test_and_clear_young(pte_t *ptep) |
| 595 | 583 | { |
| 596 | − pteval_t pteval; | |
| 597 | − unsigned int tmp, res; | |
| 584 | + pte_t old_pte, pte; | |
| 598 | 585 | |
| 599 | − asm volatile("// __ptep_test_and_clear_young\n" | |
| 600 | − " prfm pstl1strm, %2\n" | |
| 601 | − "1: ldxr %0, %2\n" | |
| 602 | − " ubfx %w3, %w0, %5, #1 // extract PTE_AF (young)\n" | |
| 603 | − " and %0, %0, %4 // clear PTE_AF\n" | |
| 604 | − " stxr %w1, %0, %2\n" | |
| 605 | − " cbnz %w1, 1b\n" | |
| 606 | − : "=&r" (pteval), "=&r" (tmp), "+Q" (pte_val(*ptep)), "=&r" (res) | |
| 607 | − : "L" (~PTE_AF), "I" (ilog2(PTE_AF))); | |
| 586 | + pte = READ_ONCE(*ptep); | |
| 587 | + do { | |
| 588 | + old_pte = pte; | |
| 589 | + pte = pte_mkold(pte); | |
| 590 | + pte_val(pte) = cmpxchg_relaxed(&pte_val(*ptep), | |
| 591 | + pte_val(old_pte), pte_val(pte)); | |
| 592 | + } while (pte_val(pte) != pte_val(old_pte)); | |
| 608 | 593 | |
| 609 | − return res; | |
| 594 | + return pte_young(pte); | |
| 610 | 595 | } |
| 611 | 596 | |
| 612 | 597 | static inline int ptep_test_and_clear_young(struct vm_area_struct *vma, |
| @@ -630,17 +615,7 @@ static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma, · 17 unmodified lines | ||
| 630 | 615 | static inline pte_t ptep_get_and_clear(struct mm_struct *mm, |
| 631 | 616 | unsigned long address, pte_t *ptep) |
| 632 | 617 | { |
| 633 | − pteval_t old_pteval; | |
| 634 | − unsigned int tmp; | |
| 635 | − | |
| 636 | − asm volatile("// ptep_get_and_clear\n" | |
| 637 | − " prfm pstl1strm, %2\n" | |
| 638 | − "1: ldxr %0, %2\n" | |
| 639 | − " stxr %w1, xzr, %2\n" | |
| 640 | − " cbnz %w1, 1b\n" | |
| 641 | − : "=&r" (old_pteval), "=&r" (tmp), "+Q" (pte_val(*ptep))); | |
| 642 | − | |
| 643 | − return __pte(old_pteval); | |
| 618 | + return __pte(xchg_relaxed(&pte_val(*ptep), 0)); | |
| 644 | 619 | } |
| 645 | 620 | |
| 646 | 621 | #ifdef CONFIG_TRANSPARENT_HUGEPAGE |
| @@ -653,27 +628,32 @@ static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm, · 6 unmodified lines | ||
| 653 | 628 | #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ |
| 654 | 629 | |
| 655 | 630 | /* |
| 656 | − * ptep_set_wrprotect - mark read-only while trasferring potential hardware | |
| 657 | − * dirty status (PTE_DBM && !PTE_RDONLY) to the software PTE_DIRTY bit. | |
| 631 | + * ptep_set_wrprotect - mark read-only while preserving the hardware update of | |
| 632 | + * the Access Flag. | |
| 658 | 633 | */ |
| 659 | 634 | #define __HAVE_ARCH_PTEP_SET_WRPROTECT |
| 660 | 635 | static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long address, pte_t *ptep) |
| 661 | 636 | { |
| 662 | − pteval_t pteval; | |
| 663 | − unsigned long tmp; | |
| 637 | + pte_t old_pte, pte; | |
| 664 | 638 | |
| 665 | − asm volatile("// ptep_set_wrprotect\n" | |
| 666 | − " prfm pstl1strm, %2\n" | |
| 667 | − "1: ldxr %0, %2\n" | |
| 668 | − " tst %0, %4 // check for hw dirty (!PTE_RDONLY)\n" | |
| 669 | − " csel %1, %3, xzr, eq // set PTE_DIRTY|PTE_RDONLY if dirty\n" | |
| 670 | − " orr %0, %0, %1 // if !dirty, PTE_RDONLY is already set\n" | |
| 671 | − " and %0, %0, %5 // clear PTE_WRITE/PTE_DBM\n" | |
| 672 | − " stxr %w1, %0, %2\n" | |
| 673 | − " cbnz %w1, 1b\n" | |
| 674 | − : "=&r" (pteval), "=&r" (tmp), "+Q" (pte_val(*ptep)) | |
| 675 | − : "r" (PTE_DIRTY|PTE_RDONLY), "L" (PTE_RDONLY), "L" (~PTE_WRITE) | |
| 676 | − : "cc"); | |
| 639 | + /* | |
| 640 | + * ptep_set_wrprotect() is only called on CoW mappings which are | |
| 641 | + * private (!VM_SHARED) with the pte either read-only (!PTE_WRITE && | |
| 642 | + * PTE_RDONLY) or writable and software-dirty (PTE_WRITE && | |
| 643 | + * !PTE_RDONLY && PTE_DIRTY); see is_cow_mapping() and | |
| 644 | + * protection_map[]. There is no race with the hardware update of the | |
| 645 | + * dirty state: clearing of PTE_RDONLY when PTE_WRITE (a.k.a. PTE_DBM) | |
| 646 | + * is set. | |
| 647 | + */ | |
| 648 | + VM_WARN_ONCE(pte_write(*ptep) && !pte_dirty(*ptep), | |
| 649 | + "%s: potential race with hardware DBM", __func__); | |
| 650 | + pte = READ_ONCE(*ptep); | |
| 651 | + do { | |
| 652 | + old_pte = pte; | |
| 653 | + pte = pte_wrprotect(pte); | |
| 654 | + pte_val(pte) = cmpxchg_relaxed(&pte_val(*ptep), | |
| 655 | + pte_val(old_pte), pte_val(pte)); | |
| 656 | + } while (pte_val(pte) != pte_val(old_pte)); | |
| 677 | 657 | } |
| 678 | 658 | |
| 679 | 659 | #ifdef CONFIG_TRANSPARENT_HUGEPAGE |
| @@ -684,7 +664,6 @@ static inline void pmdp_set_wrprotect(struct mm_struct *mm, · 4 unmodified lines | ||
| 684 | 664 | ptep_set_wrprotect(mm, address, (pte_t *)pmdp); |
| 685 | 665 | } |
| 686 | 666 | #endif |
| 687 | −#endif /* CONFIG_ARM64_HW_AFDBM */ | |
| 688 | 667 | |
| 689 | 668 | extern pgd_t swapper_pg_dir[PTRS_PER_PGD]; |
| 690 | 669 | extern pgd_t idmap_pg_dir[PTRS_PER_PGD]; |
| @@ -112,7 +112,7 @@ void tls_preserve_current_state(void); · 111 unmodified lines | ||
| 112 | 112 | static inline void start_thread_common(struct pt_regs *regs, unsigned long pc) |
| 113 | 113 | { |
| 114 | 114 | memset(regs, 0, sizeof(*regs)); |
| 115 | − regs->syscallno = ~0UL; | |
| 115 | + forget_syscall(regs); | |
| 116 | 116 | regs->pc = pc; |
| 117 | 117 | } |
| 118 | 118 | |
| @@ -159,7 +159,7 @@ extern struct task_struct *cpu_switch_to(struct task_struct *prev, · 40 unmodified lines | ||
| 159 | 159 | struct task_struct *next); |
| 160 | 160 | |
| 161 | 161 | #define task_pt_regs(p) \ |
| 162 | − ((struct pt_regs *)(THREAD_START_SP + task_stack_page(p)) - 1) | |
| 162 | + ((struct pt_regs *)(THREAD_SIZE + task_stack_page(p)) - 1) | |
| 163 | 163 | |
| 164 | 164 | #define KSTK_EIP(tsk) ((unsigned long)task_pt_regs(tsk)->pc) |
| 165 | 165 | #define KSTK_ESP(tsk) user_stack_pointer(task_pt_regs(tsk)) |
| @@ -72,8 +72,19 @@ · 71 unmodified lines | ||
| 72 | 72 | #define COMPAT_PT_TEXT_ADDR 0x10000 |
| 73 | 73 | #define COMPAT_PT_DATA_ADDR 0x10004 |
| 74 | 74 | #define COMPAT_PT_TEXT_END_ADDR 0x10008 |
| 75 | + | |
| 76 | +/* | |
| 77 | + * If pt_regs.syscallno == NO_SYSCALL, then the thread is not executing | |
| 78 | + * a syscall -- i.e., its most recent entry into the kernel from | |
| 79 | + * userspace was not via SVC, or otherwise a tracer cancelled the syscall. | |
| 80 | + * | |
| 81 | + * This must have the value -1, for ABI compatibility with ptrace etc. | |
| 82 | + */ | |
| 83 | +#define NO_SYSCALL (-1) | |
| 84 | + | |
| 75 | 85 | #ifndef __ASSEMBLY__ |
| 76 | 86 | #include <linux/bug.h> |
| 87 | +#include <linux/types.h> | |
| 77 | 88 | |
| 78 | 89 | /* sizeof(struct user) for AArch32 */ |
| 79 | 90 | #define COMPAT_USER_SZ 296 |
| @@ -116,11 +127,29 @@ struct pt_regs { · 36 unmodified lines | ||
| 116 | 127 | }; |
| 117 | 128 | }; |
| 118 | 129 | u64 orig_x0; |
| 119 | − u64 syscallno; | |
| 130 | +#ifdef __AARCH64EB__ | |
| 131 | + u32 unused2; | |
| 132 | + s32 syscallno; | |
| 133 | +#else | |
| 134 | + s32 syscallno; | |
| 135 | + u32 unused2; | |
| 136 | +#endif | |
| 137 | + | |
| 120 | 138 | u64 orig_addr_limit; |
| 121 | 139 | u64 unused; // maintain 16 byte alignment |
| 140 | + u64 stackframe[2]; | |
| 122 | 141 | }; |
| 123 | 142 | |
| 143 | +static inline bool in_syscall(struct pt_regs const *regs) | |
| 144 | +{ | |
| 145 | + return regs->syscallno != NO_SYSCALL; | |
| 146 | +} | |
| 147 | + | |
| 148 | +static inline void forget_syscall(struct pt_regs *regs) | |
| 149 | +{ | |
| 150 | + regs->syscallno = NO_SYSCALL; | |
| 151 | +} | |
| 152 | + | |
| 124 | 153 | #define MAX_REG_OFFSET offsetof(struct pt_regs, pstate) |
| 125 | 154 | |
| 126 | 155 | #define arch_has_single_step() (1) |
| @@ -22,8 +22,6 @@ · 21 unmodified lines | ||
| 22 | 22 | |
| 23 | 23 | #define AARCH32_KERN_SIGRET_CODE_OFFSET 0x500 |
| 24 | 24 | |
| 25 | −extern const compat_ulong_t aarch32_sigret_code[6]; | |
| 26 | − | |
| 27 | 25 | int compat_setup_frame(int usig, struct ksignal *ksig, sigset_t *set, |
| 28 | 26 | struct pt_regs *regs); |
| 29 | 27 | int compat_setup_rt_frame(int usig, struct ksignal *ksig, sigset_t *set, |
| @@ -0,0 +1,56 @@ | ||
| 1 | +/* | |
| 2 | + * Copyright (C) 2017 Linaro Ltd. <ard.biesheuvel@linaro.org> | |
| 3 | + * | |
| 4 | + * This program is free software; you can redistribute it and/or modify it | |
| 5 | + * under the terms of the GNU General Public License version 2 as published | |
| 6 | + * by the Free Software Foundation. | |
| 7 | + */ | |
| 8 | + | |
| 9 | +#ifndef __ASM_SIMD_H | |
| 10 | +#define __ASM_SIMD_H | |
| 11 | + | |
| 12 | +#include <linux/compiler.h> | |
| 13 | +#include <linux/irqflags.h> | |
| 14 | +#include <linux/percpu.h> | |
| 15 | +#include <linux/preempt.h> | |
| 16 | +#include <linux/types.h> | |
| 17 | + | |
| 18 | +#ifdef CONFIG_KERNEL_MODE_NEON | |
| 19 | + | |
| 20 | +DECLARE_PER_CPU(bool, kernel_neon_busy); | |
| 21 | + | |
| 22 | +/* | |
| 23 | + * may_use_simd - whether it is allowable at this time to issue SIMD | |
| 24 | + * instructions or access the SIMD register file | |
| 25 | + * | |
| 26 | + * Callers must not assume that the result remains true beyond the next | |
| 27 | + * preempt_enable() or return from softirq context. | |
| 28 | + */ | |
| 29 | +static __must_check inline bool may_use_simd(void) | |
| 30 | +{ | |
| 31 | + /* | |
| 32 | + * The raw_cpu_read() is racy if called with preemption enabled. | |
| 33 | + * This is not a bug: kernel_neon_busy is only set when | |
| 34 | + * preemption is disabled, so we cannot migrate to another CPU | |
| 35 | + * while it is set, nor can we migrate to a CPU where it is set. | |
| 36 | + * So, if we find it clear on some CPU then we're guaranteed to | |
| 37 | + * find it clear on any CPU we could migrate to. | |
| 38 | + * | |
| 39 | + * If we are in between kernel_neon_begin()...kernel_neon_end(), | |
| 40 | + * the flag will be set, but preemption is also disabled, so we | |
| 41 | + * can't migrate to another CPU and spuriously see it become | |
| 42 | + * false. | |
| 43 | + */ | |
| 44 | + return !in_irq() && !irqs_disabled() && !in_nmi() && | |
| 45 | + !raw_cpu_read(kernel_neon_busy); | |
| 46 | +} | |
| 47 | + | |
| 48 | +#else /* ! CONFIG_KERNEL_MODE_NEON */ | |
| 49 | + | |
| 50 | +static __must_check inline bool may_use_simd(void) { | |
| 51 | + return false; | |
| 52 | +} | |
| 53 | + | |
| 54 | +#endif /* ! CONFIG_KERNEL_MODE_NEON */ | |
| 55 | + | |
| 56 | +#endif | |
| @@ -148,7 +148,7 @@ static inline void cpu_panic_kernel(void) · 147 unmodified lines | ||
| 148 | 148 | */ |
| 149 | 149 | bool cpus_are_stuck_in_kernel(void); |
| 150 | 150 | |
| 151 | −extern void smp_send_crash_stop(void); | |
| 151 | +extern void crash_smp_send_stop(void); | |
| 152 | 152 | extern bool smp_crash_stop_failed(void); |
| 153 | 153 | |
| 154 | 154 | #endif /* ifndef __ASSEMBLY__ */ |
| @@ -16,11 +16,15 @@ · 15 unmodified lines | ||
| 16 | 16 | #ifndef __ASM_STACKTRACE_H |
| 17 | 17 | #define __ASM_STACKTRACE_H |
| 18 | 18 | |
| 19 | −struct task_struct; | |
| 19 | +#include <linux/percpu.h> | |
| 20 | +#include <linux/sched.h> | |
| 21 | +#include <linux/sched/task_stack.h> | |
| 22 | + | |
| 23 | +#include <asm/memory.h> | |
| 24 | +#include <asm/ptrace.h> | |
| 20 | 25 | |
| 21 | 26 | struct stackframe { |
| 22 | 27 | unsigned long fp; |
| 23 | − unsigned long sp; | |
| 24 | 28 | unsigned long pc; |
| 25 | 29 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER |
| 26 | 30 | unsigned int graph; |
| @@ -32,4 +36,57 @@ extern void walk_stackframe(struct task_struct *tsk, struct stackframe *frame, · 5 unmodified lines | ||
| 32 | 36 | int (*fn)(struct stackframe *, void *), void *data); |
| 33 | 37 | extern void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk); |
| 34 | 38 | |
| 39 | +DECLARE_PER_CPU(unsigned long *, irq_stack_ptr); | |
| 40 | + | |
| 41 | +static inline bool on_irq_stack(unsigned long sp) | |
| 42 | +{ | |
| 43 | + unsigned long low = (unsigned long)raw_cpu_read(irq_stack_ptr); | |
| 44 | + unsigned long high = low + IRQ_STACK_SIZE; | |
| 45 | + | |
| 46 | + if (!low) | |
| 47 | + return false; | |
| 48 | + | |
| 49 | + return (low <= sp && sp < high); | |
| 50 | +} | |
| 51 | + | |
| 52 | +static inline bool on_task_stack(struct task_struct *tsk, unsigned long sp) | |
| 53 | +{ | |
| 54 | + unsigned long low = (unsigned long)task_stack_page(tsk); | |
| 55 | + unsigned long high = low + THREAD_SIZE; | |
| 56 | + | |
| 57 | + return (low <= sp && sp < high); | |
| 58 | +} | |
| 59 | + | |
| 60 | +#ifdef CONFIG_VMAP_STACK | |
| 61 | +DECLARE_PER_CPU(unsigned long [OVERFLOW_STACK_SIZE/sizeof(long)], overflow_stack); | |
| 62 | + | |
| 63 | +static inline bool on_overflow_stack(unsigned long sp) | |
| 64 | +{ | |
| 65 | + unsigned long low = (unsigned long)raw_cpu_ptr(overflow_stack); | |
| 66 | + unsigned long high = low + OVERFLOW_STACK_SIZE; | |
| 67 | + | |
| 68 | + return (low <= sp && sp < high); | |
| 69 | +} | |
| 70 | +#else | |
| 71 | +static inline bool on_overflow_stack(unsigned long sp) { return false; } | |
| 72 | +#endif | |
| 73 | + | |
| 74 | +/* | |
| 75 | + * We can only safely access per-cpu stacks from current in a non-preemptible | |
| 76 | + * context. | |
| 77 | + */ | |
| 78 | +static inline bool on_accessible_stack(struct task_struct *tsk, unsigned long sp) | |
| 79 | +{ | |
| 80 | + if (on_task_stack(tsk, sp)) | |
| 81 | + return true; | |
| 82 | + if (tsk != current || preemptible()) | |
| 83 | + return false; | |
| 84 | + if (on_irq_stack(sp)) | |
| 85 | + return true; | |
| 86 | + if (on_overflow_stack(sp)) | |
| 87 | + return true; | |
| 88 | + | |
| 89 | + return false; | |
| 90 | +} | |
| 91 | + | |
| 35 | 92 | #endif /* __ASM_STACKTRACE_H */ |
| @@ -52,6 +52,10 @@ extern void *__memset(void *, int, __kernel_size_t); · 51 unmodified lines | ||
| 52 | 52 | #define __HAVE_ARCH_MEMCMP |
| 53 | 53 | extern int memcmp(const void *, const void *, size_t); |
| 54 | 54 | |
| 55 | +#ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE | |
| 56 | +#define __HAVE_ARCH_MEMCPY_FLUSHCACHE | |
| 57 | +void memcpy_flushcache(void *dst, const void *src, size_t cnt); | |
| 58 | +#endif | |
| 55 | 59 | |
| 56 | 60 | #if defined(CONFIG_KASAN) && !defined(__SANITIZE_ADDRESS__) |
| 57 | 61 | |
| @@ -329,6 +329,7 @@ · 328 unmodified lines | ||
| 329 | 329 | #define ID_AA64ISAR1_LRCPC_SHIFT 20 |
| 330 | 330 | #define ID_AA64ISAR1_FCMA_SHIFT 16 |
| 331 | 331 | #define ID_AA64ISAR1_JSCVT_SHIFT 12 |
| 332 | +#define ID_AA64ISAR1_DPB_SHIFT 0 | |
| 332 | 333 | |
| 333 | 334 | /* id_aa64pfr0 */ |
| 334 | 335 | #define ID_AA64PFR0_GIC_SHIFT 24 |
| @@ -23,19 +23,11 @@ · 22 unmodified lines | ||
| 23 | 23 | |
| 24 | 24 | #include <linux/compiler.h> |
| 25 | 25 | |
| 26 | −#ifdef CONFIG_ARM64_4K_PAGES | |
| 27 | −#define THREAD_SIZE_ORDER 2 | |
| 28 | −#elif defined(CONFIG_ARM64_16K_PAGES) | |
| 29 | −#define THREAD_SIZE_ORDER 0 | |
| 30 | −#endif | |
| 31 | − | |
| 32 | −#define THREAD_SIZE 16384 | |
| 33 | −#define THREAD_START_SP (THREAD_SIZE - 16) | |
| 34 | − | |
| 35 | 26 | #ifndef __ASSEMBLY__ |
| 36 | 27 | |
| 37 | 28 | struct task_struct; |
| 38 | 29 | |
| 30 | +#include <asm/memory.h> | |
| 39 | 31 | #include <asm/stack_pointer.h> |
| 40 | 32 | #include <asm/types.h> |
| 41 | 33 | |
| @@ -68,6 +60,9 @@ struct thread_info { · 26 unmodified lines | ||
| 68 | 60 | #define thread_saved_fp(tsk) \ |
| 69 | 61 | ((unsigned long)(tsk->thread.cpu_context.fp)) |
| 70 | 62 | |
| 63 | +void arch_setup_new_exec(void); | |
| 64 | +#define arch_setup_new_exec arch_setup_new_exec | |
| 65 | + | |
| 71 | 66 | #endif |
| 72 | 67 | |
| 73 | 68 | /* |
| @@ -53,4 +53,9 @@ static inline int in_exception_text(unsigned long ptr) · 52 unmodified lines | ||
| 53 | 53 | return in ? : __in_irqentry_text(ptr); |
| 54 | 54 | } |
| 55 | 55 | |
| 56 | +static inline int in_entry_text(unsigned long ptr) | |
| 57 | +{ | |
| 58 | + return ptr >= (unsigned long)&__entry_text_start && | |
| 59 | + ptr < (unsigned long)&__entry_text_end; | |
| 60 | +} | |
| 56 | 61 | #endif |
| @@ -350,4 +350,16 @@ extern long strncpy_from_user(char *dest, const char __user *src, long count); · 349 unmodified lines | ||
| 350 | 350 | |
| 351 | 351 | extern __must_check long strnlen_user(const char __user *str, long n); |
| 352 | 352 | |
| 353 | +#ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE | |
| 354 | +struct page; | |
| 355 | +void memcpy_page_flushcache(char *to, struct page *page, size_t offset, size_t len); | |
| 356 | +extern unsigned long __must_check __copy_user_flushcache(void *to, const void __user *from, unsigned long n); | |
| 357 | + | |
| 358 | +static inline int __copy_from_user_flushcache(void *dst, const void __user *src, unsigned size) | |
| 359 | +{ | |
| 360 | + kasan_check_write(dst, size); | |
| 361 | + return __copy_user_flushcache(dst, src, size); | |
| 362 | +} | |
| 363 | +#endif | |
| 364 | + | |
| 353 | 365 | #endif /* __ASM_UACCESS_H */ |
| @@ -35,5 +35,6 @@ · 34 unmodified lines | ||
| 35 | 35 | #define HWCAP_JSCVT (1 << 13) |
| 36 | 36 | #define HWCAP_FCMA (1 << 14) |
| 37 | 37 | #define HWCAP_LRCPC (1 << 15) |
| 38 | +#define HWCAP_DCPOP (1 << 16) | |
| 38 | 39 | |
| 39 | 40 | #endif /* _UAPI__ASM_HWCAP_H */ |
| @@ -75,6 +75,7 @@ int main(void) · 74 unmodified lines | ||
| 75 | 75 | DEFINE(S_ORIG_X0, offsetof(struct pt_regs, orig_x0)); |
| 76 | 76 | DEFINE(S_SYSCALLNO, offsetof(struct pt_regs, syscallno)); |
| 77 | 77 | DEFINE(S_ORIG_ADDR_LIMIT, offsetof(struct pt_regs, orig_addr_limit)); |
| 78 | + DEFINE(S_STACKFRAME, offsetof(struct pt_regs, stackframe)); | |
| 78 | 79 | DEFINE(S_FRAME_SIZE, sizeof(struct pt_regs)); |
| 79 | 80 | BLANK(); |
| 80 | 81 | DEFINE(MM_CONTEXT_ID, offsetof(struct mm_struct, context.id.counter)); |
| @@ -120,6 +120,7 @@ static const struct arm64_ftr_bits ftr_id_aa64isar1[] = { · 119 unmodified lines | ||
| 120 | 120 | ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64ISAR1_LRCPC_SHIFT, 4, 0), |
| 121 | 121 | ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64ISAR1_FCMA_SHIFT, 4, 0), |
| 122 | 122 | ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64ISAR1_JSCVT_SHIFT, 4, 0), |
| 123 | + ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64ISAR1_DPB_SHIFT, 4, 0), | |
| 123 | 124 | ARM64_FTR_END, |
| 124 | 125 | }; |
| 125 | 126 | |
| @@ -888,6 +889,17 @@ static const struct arm64_cpu_capabilities arm64_features[] = { · 762 unmodified lines | ||
| 888 | 889 | .min_field_value = 0, |
| 889 | 890 | .matches = has_no_fpsimd, |
| 890 | 891 | }, |
| 892 | +#ifdef CONFIG_ARM64_PMEM | |
| 893 | + { | |
| 894 | + .desc = "Data cache clean to Point of Persistence", | |
| 895 | + .capability = ARM64_HAS_DCPOP, | |
| 896 | + .def_scope = SCOPE_SYSTEM, | |
| 897 | + .matches = has_cpuid_feature, | |
| 898 | + .sys_reg = SYS_ID_AA64ISAR1_EL1, | |
| 899 | + .field_pos = ID_AA64ISAR1_DPB_SHIFT, | |
| 900 | + .min_field_value = 1, | |
| 901 | + }, | |
| 902 | +#endif | |
| 891 | 903 | {}, |
| 892 | 904 | }; |
| 893 | 905 | |
| @@ -916,6 +928,7 @@ static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = { · 22 unmodified lines | ||
| 916 | 928 | HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_FP_SHIFT, FTR_SIGNED, 1, CAP_HWCAP, HWCAP_FPHP), |
| 917 | 929 | HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_ASIMD_SHIFT, FTR_SIGNED, 0, CAP_HWCAP, HWCAP_ASIMD), |
| 918 | 930 | HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_ASIMD_SHIFT, FTR_SIGNED, 1, CAP_HWCAP, HWCAP_ASIMDHP), |
| 931 | + HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_DPB_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_DCPOP), | |
| 919 | 932 | HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_JSCVT_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_JSCVT), |
| 920 | 933 | HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_FCMA_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_FCMA), |
| 921 | 934 | HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_LRCPC_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_LRCPC), |
| @@ -68,6 +68,7 @@ static const char *const hwcap_str[] = { · 67 unmodified lines | ||
| 68 | 68 | "jscvt", |
| 69 | 69 | "fcma", |
| 70 | 70 | "lrcpc", |
| 71 | + "dcpop", | |
| 71 | 72 | NULL |
| 72 | 73 | }; |
| 73 | 74 | |
| @@ -41,27 +41,3 @@ ENTRY(fpsimd_load_state) · 40 unmodified lines | ||
| 41 | 41 | fpsimd_restore x0, 8 |
| 42 | 42 | ret |
| 43 | 43 | ENDPROC(fpsimd_load_state) |
| 44 | − | |
| 45 | −#ifdef CONFIG_KERNEL_MODE_NEON | |
| 46 | − | |
| 47 | −/* | |
| 48 | − * Save the bottom n FP registers. | |
| 49 | − * | |
| 50 | − * x0 - pointer to struct fpsimd_partial_state | |
| 51 | − */ | |
| 52 | −ENTRY(fpsimd_save_partial_state) | |
| 53 | − fpsimd_save_partial x0, 1, 8, 9 | |
| 54 | − ret | |
| 55 | −ENDPROC(fpsimd_save_partial_state) | |
| 56 | − | |
| 57 | −/* | |
| 58 | − * Load the bottom n FP registers. | |
| 59 | − * | |
| 60 | − * x0 - pointer to struct fpsimd_partial_state | |
| 61 | − */ | |
| 62 | −ENTRY(fpsimd_load_partial_state) | |
| 63 | − fpsimd_restore_partial x0, 8, 9 | |
| 64 | − ret | |
| 65 | −ENDPROC(fpsimd_load_partial_state) | |
| 66 | − | |
| 67 | −#endif | |
| @@ -69,8 +69,55 @@ · 68 unmodified lines | ||
| 69 | 69 | #define BAD_FIQ 2 |
| 70 | 70 | #define BAD_ERROR 3 |
| 71 | 71 | |
| 72 | − .macro kernel_entry, el, regsize = 64 | |
| 72 | + .macro kernel_ventry label | |
| 73 | + .align 7 | |
| 73 | 74 | sub sp, sp, #S_FRAME_SIZE |
| 75 | +#ifdef CONFIG_VMAP_STACK | |
| 76 | + /* | |
| 77 | + * Test whether the SP has overflowed, without corrupting a GPR. | |
| 78 | + * Task and IRQ stacks are aligned to (1 << THREAD_SHIFT). | |
| 79 | + */ | |
| 80 | + add sp, sp, x0 // sp' = sp + x0 | |
| 81 | + sub x0, sp, x0 // x0' = sp' - x0 = (sp + x0) - x0 = sp | |
| 82 | + tbnz x0, #THREAD_SHIFT, 0f | |
| 83 | + sub x0, sp, x0 // x0'' = sp' - x0' = (sp + x0) - sp = x0 | |
| 84 | + sub sp, sp, x0 // sp'' = sp' - x0 = (sp + x0) - x0 = sp | |
| 85 | + b \label | |
| 86 | + | |
| 87 | +0: | |
| 88 | + /* | |
| 89 | + * Either we've just detected an overflow, or we've taken an exception | |
| 90 | + * while on the overflow stack. Either way, we won't return to | |
| 91 | + * userspace, and can clobber EL0 registers to free up GPRs. | |
| 92 | + */ | |
| 93 | + | |
| 94 | + /* Stash the original SP (minus S_FRAME_SIZE) in tpidr_el0. */ | |
| 95 | + msr tpidr_el0, x0 | |
| 96 | + | |
| 97 | + /* Recover the original x0 value and stash it in tpidrro_el0 */ | |
| 98 | + sub x0, sp, x0 | |
| 99 | + msr tpidrro_el0, x0 | |
| 100 | + | |
| 101 | + /* Switch to the overflow stack */ | |
| 102 | + adr_this_cpu sp, overflow_stack + OVERFLOW_STACK_SIZE, x0 | |
| 103 | + | |
| 104 | + /* | |
| 105 | + * Check whether we were already on the overflow stack. This may happen | |
| 106 | + * after panic() re-enables interrupts. | |
| 107 | + */ | |
| 108 | + mrs x0, tpidr_el0 // sp of interrupted context | |
| 109 | + sub x0, sp, x0 // delta with top of overflow stack | |
| 110 | + tst x0, #~(OVERFLOW_STACK_SIZE - 1) // within range? | |
| 111 | + b.ne __bad_stack // no? -> bad stack pointer | |
| 112 | + | |
| 113 | + /* We were already on the overflow stack. Restore sp/x0 and carry on. */ | |
| 114 | + sub sp, sp, x0 | |
| 115 | + mrs x0, tpidrro_el0 | |
| 116 | +#endif | |
| 117 | + b \label | |
| 118 | + .endm | |
| 119 | + | |
| 120 | + .macro kernel_entry, el, regsize = 64 | |
| 74 | 121 | .if \regsize == 32 |
| 75 | 122 | mov w0, w0 // zero upper 32 bits of x0 |
| 76 | 123 | .endif |
| @@ -111,6 +158,18 @@ · 34 unmodified lines | ||
| 111 | 158 | mrs x23, spsr_el1 |
| 112 | 159 | stp lr, x21, [sp, #S_LR] |
| 113 | 160 | |
| 161 | + /* | |
| 162 | + * In order to be able to dump the contents of struct pt_regs at the | |
| 163 | + * time the exception was taken (in case we attempt to walk the call | |
| 164 | + * stack later), chain it together with the stack frames. | |
| 165 | + */ | |
| 166 | + .if \el == 0 | |
| 167 | + stp xzr, xzr, [sp, #S_STACKFRAME] | |
| 168 | + .else | |
| 169 | + stp x29, x22, [sp, #S_STACKFRAME] | |
| 170 | + .endif | |
| 171 | + add x29, sp, #S_STACKFRAME | |
| 172 | + | |
| 114 | 173 | #ifdef CONFIG_ARM64_SW_TTBR0_PAN |
| 115 | 174 | /* |
| 116 | 175 | * Set the TTBR0 PAN bit in SPSR. When the exception is taken from |
| @@ -138,12 +197,10 @@ alternative_else_nop_endif · 21 unmodified lines | ||
| 138 | 197 | |
| 139 | 198 | stp x22, x23, [sp, #S_PC] |
| 140 | 199 | |
| 141 | − /* | |
| 142 | − * Set syscallno to -1 by default (overridden later if real syscall). | |
| 143 | − */ | |
| 200 | + /* Not in a syscall by default (el0_svc overwrites for real syscall) */ | |
| 144 | 201 | .if \el == 0 |
| 145 | − mvn x21, xzr | |
| 146 | − str x21, [sp, #S_SYSCALLNO] | |
| 202 | + mov w21, #NO_SYSCALL | |
| 203 | + str w21, [sp, #S_SYSCALLNO] | |
| 147 | 204 | .endif |
| 148 | 205 | |
| 149 | 206 | /* |
| @@ -259,20 +316,12 @@ alternative_else_nop_endif · 109 unmodified lines | ||
| 259 | 316 | and x25, x25, #~(THREAD_SIZE - 1) |
| 260 | 317 | cbnz x25, 9998f |
| 261 | 318 | |
| 262 | − adr_this_cpu x25, irq_stack, x26 | |
| 263 | − mov x26, #IRQ_STACK_START_SP | |
| 319 | + ldr_this_cpu x25, irq_stack_ptr, x26 | |
| 320 | + mov x26, #IRQ_STACK_SIZE | |
| 264 | 321 | add x26, x25, x26 |
| 265 | 322 | |
| 266 | 323 | /* switch to the irq stack */ |
| 267 | 324 | mov sp, x26 |
| 268 | − | |
| 269 | − /* | |
| 270 | − * Add a dummy stack frame, this non-standard format is fixed up | |
| 271 | − * by unwind_frame() | |
| 272 | − */ | |
| 273 | − stp x29, x19, [sp, #-16]! | |
| 274 | − mov x29, sp | |
| 275 | − | |
| 276 | 325 | 9998: |
| 277 | 326 | .endm |
| 278 | 327 | |
| @@ -290,8 +339,9 @@ alternative_else_nop_endif · 11 unmodified lines | ||
| 290 | 339 | * |
| 291 | 340 | * x7 is reserved for the system call number in 32-bit mode. |
| 292 | 341 | */ |
| 293 | −sc_nr .req x25 // number of system calls | |
| 294 | −scno .req x26 // syscall number | |
| 342 | +wsc_nr .req w25 // number of system calls | |
| 343 | +wscno .req w26 // syscall number | |
| 344 | +xscno .req x26 // syscall number (zero-extended) | |
| 295 | 345 | stbl .req x27 // syscall table pointer |
| 296 | 346 | tsk .req x28 // current thread_info |
| 297 | 347 | |
| @@ -315,34 +365,62 @@ tsk .req x28 // current thread_info · 17 unmodified lines | ||
| 315 | 365 | |
| 316 | 366 | .align 11 |
| 317 | 367 | ENTRY(vectors) |
| 318 | − ventry el1_sync_invalid // Synchronous EL1t | |
| 319 | − ventry el1_irq_invalid // IRQ EL1t | |
| 320 | − ventry el1_fiq_invalid // FIQ EL1t | |
| 321 | − ventry el1_error_invalid // Error EL1t | |
| 368 | + kernel_ventry el1_sync_invalid // Synchronous EL1t | |
| 369 | + kernel_ventry el1_irq_invalid // IRQ EL1t | |
| 370 | + kernel_ventry el1_fiq_invalid // FIQ EL1t | |
| 371 | + kernel_ventry el1_error_invalid // Error EL1t | |
| 322 | 372 | |
| 323 | − ventry el1_sync // Synchronous EL1h | |
| 324 | − ventry el1_irq // IRQ EL1h | |
| 325 | − ventry el1_fiq_invalid // FIQ EL1h | |
| 326 | − ventry el1_error_invalid // Error EL1h | |
| 373 | + kernel_ventry el1_sync // Synchronous EL1h | |
| 374 | + kernel_ventry el1_irq // IRQ EL1h | |
| 375 | + kernel_ventry el1_fiq_invalid // FIQ EL1h | |
| 376 | + kernel_ventry el1_error_invalid // Error EL1h | |
| 327 | 377 | |
| 328 | − ventry el0_sync // Synchronous 64-bit EL0 | |
| 329 | − ventry el0_irq // IRQ 64-bit EL0 | |
| 330 | − ventry el0_fiq_invalid // FIQ 64-bit EL0 | |
| 331 | − ventry el0_error_invalid // Error 64-bit EL0 | |
| 378 | + kernel_ventry el0_sync // Synchronous 64-bit EL0 | |
| 379 | + kernel_ventry el0_irq // IRQ 64-bit EL0 | |
| 380 | + kernel_ventry el0_fiq_invalid // FIQ 64-bit EL0 | |
| 381 | + kernel_ventry el0_error_invalid // Error 64-bit EL0 | |
| 332 | 382 | |
| 333 | 383 | #ifdef CONFIG_COMPAT |
| 334 | − ventry el0_sync_compat // Synchronous 32-bit EL0 | |
| 335 | − ventry el0_irq_compat // IRQ 32-bit EL0 | |
| 336 | − ventry el0_fiq_invalid_compat // FIQ 32-bit EL0 | |
| 337 | − ventry el0_error_invalid_compat // Error 32-bit EL0 | |
| 384 | + kernel_ventry el0_sync_compat // Synchronous 32-bit EL0 | |
| 385 | + kernel_ventry el0_irq_compat // IRQ 32-bit EL0 | |
| 386 | + kernel_ventry el0_fiq_invalid_compat // FIQ 32-bit EL0 | |
| 387 | + kernel_ventry el0_error_invalid_compat // Error 32-bit EL0 | |
| 338 | 388 | #else |
| 339 | − ventry el0_sync_invalid // Synchronous 32-bit EL0 | |
| 340 | − ventry el0_irq_invalid // IRQ 32-bit EL0 | |
| 341 | − ventry el0_fiq_invalid // FIQ 32-bit EL0 | |
| 342 | − ventry el0_error_invalid // Error 32-bit EL0 | |
| 389 | + kernel_ventry el0_sync_invalid // Synchronous 32-bit EL0 | |
| 390 | + kernel_ventry el0_irq_invalid // IRQ 32-bit EL0 | |
| 391 | + kernel_ventry el0_fiq_invalid // FIQ 32-bit EL0 | |
| 392 | + kernel_ventry el0_error_invalid // Error 32-bit EL0 | |
| 343 | 393 | #endif |
| 344 | 394 | END(vectors) |
| 345 | 395 | |
| 396 | +#ifdef CONFIG_VMAP_STACK | |
| 397 | + /* | |
| 398 | + * We detected an overflow in kernel_ventry, which switched to the | |
| 399 | + * overflow stack. Stash the exception regs, and head to our overflow | |
| 400 | + * handler. | |
| 401 | + */ | |
| 402 | +__bad_stack: | |
| 403 | + /* Restore the original x0 value */ | |
| 404 | + mrs x0, tpidrro_el0 | |
| 405 | + | |
| 406 | + /* | |
| 407 | + * Store the original GPRs to the new stack. The orginal SP (minus | |
| 408 | + * S_FRAME_SIZE) was stashed in tpidr_el0 by kernel_ventry. | |
| 409 | + */ | |
| 410 | + sub sp, sp, #S_FRAME_SIZE | |
| 411 | + kernel_entry 1 | |
| 412 | + mrs x0, tpidr_el0 | |
| 413 | + add x0, x0, #S_FRAME_SIZE | |
| 414 | + str x0, [sp, #S_SP] | |
| 415 | + | |
| 416 | + /* Stash the regs for handle_bad_stack */ | |
| 417 | + mov x0, sp | |
| 418 | + | |
| 419 | + /* Time to die */ | |
| 420 | + bl handle_bad_stack | |
| 421 | + ASM_BUG() | |
| 422 | +#endif /* CONFIG_VMAP_STACK */ | |
| 423 | + | |
| 346 | 424 | /* |
| 347 | 425 | * Invalid mode handlers |
| 348 | 426 | */ |
| @@ -351,7 +429,8 @@ END(vectors) · 2 unmodified lines | ||
| 351 | 429 | mov x0, sp |
| 352 | 430 | mov x1, #\reason |
| 353 | 431 | mrs x2, esr_el1 |
| 354 | − b bad_mode | |
| 432 | + bl bad_mode | |
| 433 | + ASM_BUG() | |
| 355 | 434 | .endm |
| 356 | 435 | |
| 357 | 436 | el0_sync_invalid: |
| @@ -448,14 +527,16 @@ el1_sp_pc: · 90 unmodified lines | ||
| 448 | 527 | mrs x0, far_el1 |
| 449 | 528 | enable_dbg |
| 450 | 529 | mov x2, sp |
| 451 | − b do_sp_pc_abort | |
| 530 | + bl do_sp_pc_abort | |
| 531 | + ASM_BUG() | |
| 452 | 532 | el1_undef: |
| 453 | 533 | /* |
| 454 | 534 | * Undefined instruction |
| 455 | 535 | */ |
| 456 | 536 | enable_dbg |
| 457 | 537 | mov x0, sp |
| 458 | − b do_undefinstr | |
| 538 | + bl do_undefinstr | |
| 539 | + ASM_BUG() | |
| 459 | 540 | el1_dbg: |
| 460 | 541 | /* |
| 461 | 542 | * Debug exception handling |
| @@ -473,7 +554,8 @@ el1_inv: · 11 unmodified lines | ||
| 473 | 554 | mov x0, sp |
| 474 | 555 | mov x2, x1 |
| 475 | 556 | mov x1, #BAD_SYNC |
| 476 | − b bad_mode | |
| 557 | + bl bad_mode | |
| 558 | + ASM_BUG() | |
| 477 | 559 | ENDPROC(el1_sync) |
| 478 | 560 | |
| 479 | 561 | .align 6 |
| @@ -577,8 +659,8 @@ el0_svc_compat: · 97 unmodified lines | ||
| 577 | 659 | * AArch32 syscall handling |
| 578 | 660 | */ |
| 579 | 661 | adrp stbl, compat_sys_call_table // load compat syscall table pointer |
| 580 | − uxtw scno, w7 // syscall number in w7 (r7) | |
| 581 | − mov sc_nr, #__NR_compat_syscalls | |
| 662 | + mov wscno, w7 // syscall number in w7 (r7) | |
| 663 | + mov wsc_nr, #__NR_compat_syscalls | |
| 582 | 664 | b el0_svc_naked |
| 583 | 665 | |
| 584 | 666 | .align 6 |
| @@ -706,38 +788,6 @@ el0_irq_naked: · 121 unmodified lines | ||
| 706 | 788 | b ret_to_user |
| 707 | 789 | ENDPROC(el0_irq) |
| 708 | 790 | |
| 709 | −/* | |
| 710 | − * Register switch for AArch64. The callee-saved registers need to be saved | |
| 711 | − * and restored. On entry: | |
| 712 | − * x0 = previous task_struct (must be preserved across the switch) | |
| 713 | − * x1 = next task_struct | |
| 714 | − * Previous and next are guaranteed not to be the same. | |
| 715 | − * | |
| 716 | − */ | |
| 717 | −ENTRY(cpu_switch_to) | |
| 718 | − mov x10, #THREAD_CPU_CONTEXT | |
| 719 | − add x8, x0, x10 | |
| 720 | − mov x9, sp | |
| 721 | − stp x19, x20, [x8], #16 // store callee-saved registers | |
| 722 | − stp x21, x22, [x8], #16 | |
| 723 | − stp x23, x24, [x8], #16 | |
| 724 | − stp x25, x26, [x8], #16 | |
| 725 | − stp x27, x28, [x8], #16 | |
| 726 | − stp x29, x9, [x8], #16 | |
| 727 | − str lr, [x8] | |
| 728 | − add x8, x1, x10 | |
| 729 | − ldp x19, x20, [x8], #16 // restore callee-saved registers | |
| 730 | − ldp x21, x22, [x8], #16 | |
| 731 | − ldp x23, x24, [x8], #16 | |
| 732 | − ldp x25, x26, [x8], #16 | |
| 733 | − ldp x27, x28, [x8], #16 | |
| 734 | − ldp x29, x9, [x8], #16 | |
| 735 | − ldr lr, [x8] | |
| 736 | − mov sp, x9 | |
| 737 | − msr sp_el0, x1 | |
| 738 | − ret | |
| 739 | −ENDPROC(cpu_switch_to) | |
| 740 | − | |
| 741 | 791 | /* |
| 742 | 792 | * This is the fast syscall return path. We do as little as possible here, |
| 743 | 793 | * and this includes saving x0 back into the kernel stack. |
| @@ -780,37 +830,25 @@ finish_ret_to_user: · 36 unmodified lines | ||
| 780 | 830 | kernel_exit 0 |
| 781 | 831 | ENDPROC(ret_to_user) |
| 782 | 832 | |
| 783 | −/* | |
| 784 | − * This is how we return from a fork. | |
| 785 | − */ | |
| 786 | −ENTRY(ret_from_fork) | |
| 787 | − bl schedule_tail | |
| 788 | − cbz x19, 1f // not a kernel thread | |
| 789 | − mov x0, x20 | |
| 790 | − blr x19 | |
| 791 | −1: get_thread_info tsk | |
| 792 | − b ret_to_user | |
| 793 | −ENDPROC(ret_from_fork) | |
| 794 | − | |
| 795 | 833 | /* |
| 796 | 834 | * SVC handler. |
| 797 | 835 | */ |
| 798 | 836 | .align 6 |
| 799 | 837 | el0_svc: |
| 800 | 838 | adrp stbl, sys_call_table // load syscall table pointer |
| 801 | − uxtw scno, w8 // syscall number in w8 | |
| 802 | − mov sc_nr, #__NR_syscalls | |
| 839 | + mov wscno, w8 // syscall number in w8 | |
| 840 | + mov wsc_nr, #__NR_syscalls | |
| 803 | 841 | el0_svc_naked: // compat entry point |
| 804 | − stp x0, scno, [sp, #S_ORIG_X0] // save the original x0 and syscall number | |
| 842 | + stp x0, xscno, [sp, #S_ORIG_X0] // save the original x0 and syscall number | |
| 805 | 843 | enable_dbg_and_irq |
| 806 | 844 | ct_user_exit 1 |
| 807 | 845 | |
| 808 | 846 | ldr x16, [tsk, #TSK_TI_FLAGS] // check for syscall hooks |
| 809 | 847 | tst x16, #_TIF_SYSCALL_WORK |
| 810 | 848 | b.ne __sys_trace |
| 811 | − cmp scno, sc_nr // check upper syscall limit | |
| 849 | + cmp wscno, wsc_nr // check upper syscall limit | |
| 812 | 850 | b.hs ni_sys |
| 813 | − ldr x16, [stbl, scno, lsl #3] // address in the syscall table | |
| 851 | + ldr x16, [stbl, xscno, lsl #3] // address in the syscall table | |
| 814 | 852 | blr x16 // call sys_* routine |
| 815 | 853 | b ret_fast_syscall |
| 816 | 854 | ni_sys: |
| @@ -824,24 +862,23 @@ ENDPROC(el0_svc) · 7 unmodified lines | ||
| 824 | 862 | * switches, and waiting for our parent to respond. |
| 825 | 863 | */ |
| 826 | 864 | __sys_trace: |
| 827 | − mov w0, #-1 // set default errno for | |
| 828 | − cmp scno, x0 // user-issued syscall(-1) | |
| 865 | + cmp wscno, #NO_SYSCALL // user-issued syscall(-1)? | |
| 829 | 866 | b.ne 1f |
| 830 | − mov x0, #-ENOSYS | |
| 867 | + mov x0, #-ENOSYS // set default errno if so | |
| 831 | 868 | str x0, [sp, #S_X0] |
| 832 | 869 | 1: mov x0, sp |
| 833 | 870 | bl syscall_trace_enter |
| 834 | − cmp w0, #-1 // skip the syscall? | |
| 871 | + cmp w0, #NO_SYSCALL // skip the syscall? | |
| 835 | 872 | b.eq __sys_trace_return_skipped |
| 836 | − uxtw scno, w0 // syscall number (possibly new) | |
| 873 | + mov wscno, w0 // syscall number (possibly new) | |
| 837 | 874 | mov x1, sp // pointer to regs |
| 838 | − cmp scno, sc_nr // check upper syscall limit | |
| 875 | + cmp wscno, wsc_nr // check upper syscall limit | |
| 839 | 876 | b.hs __ni_sys_trace |
| 840 | 877 | ldp x0, x1, [sp] // restore the syscall args |
| 841 | 878 | ldp x2, x3, [sp, #S_X2] |
| 842 | 879 | ldp x4, x5, [sp, #S_X4] |
| 843 | 880 | ldp x6, x7, [sp, #S_X6] |
| 844 | − ldr x16, [stbl, scno, lsl #3] // address in the syscall table | |
| 881 | + ldr x16, [stbl, xscno, lsl #3] // address in the syscall table | |
| 845 | 882 | blr x16 // call sys_* routine |
| 846 | 883 | |
| 847 | 884 | __sys_trace_return: |
| @@ -865,3 +902,49 @@ ENTRY(sys_rt_sigreturn_wrapper) · 17 unmodified lines | ||
| 865 | 902 | mov x0, sp |
| 866 | 903 | b sys_rt_sigreturn |
| 867 | 904 | ENDPROC(sys_rt_sigreturn_wrapper) |
| 905 | + | |
| 906 | +/* | |
| 907 | + * Register switch for AArch64. The callee-saved registers need to be saved | |
| 908 | + * and restored. On entry: | |
| 909 | + * x0 = previous task_struct (must be preserved across the switch) | |
| 910 | + * x1 = next task_struct | |
| 911 | + * Previous and next are guaranteed not to be the same. | |
| 912 | + * | |
| 913 | + */ | |
| 914 | +ENTRY(cpu_switch_to) | |
| 915 | + mov x10, #THREAD_CPU_CONTEXT | |
| 916 | + add x8, x0, x10 | |
| 917 | + mov x9, sp | |
| 918 | + stp x19, x20, [x8], #16 // store callee-saved registers | |
| 919 | + stp x21, x22, [x8], #16 | |
| 920 | + stp x23, x24, [x8], #16 | |
| 921 | + stp x25, x26, [x8], #16 | |
| 922 | + stp x27, x28, [x8], #16 | |
| 923 | + stp x29, x9, [x8], #16 | |
| 924 | + str lr, [x8] | |
| 925 | + add x8, x1, x10 | |
| 926 | + ldp x19, x20, [x8], #16 // restore callee-saved registers | |
| 927 | + ldp x21, x22, [x8], #16 | |
| 928 | + ldp x23, x24, [x8], #16 | |
| 929 | + ldp x25, x26, [x8], #16 | |
| 930 | + ldp x27, x28, [x8], #16 | |
| 931 | + ldp x29, x9, [x8], #16 | |
| 932 | + ldr lr, [x8] | |
| 933 | + mov sp, x9 | |
| 934 | + msr sp_el0, x1 | |
| 935 | + ret | |
| 936 | +ENDPROC(cpu_switch_to) | |
| 937 | +NOKPROBE(cpu_switch_to) | |
| 938 | + | |
| 939 | +/* | |
| 940 | + * This is how we return from a fork. | |
| 941 | + */ | |
| 942 | +ENTRY(ret_from_fork) | |
| 943 | + bl schedule_tail | |
| 944 | + cbz x19, 1f // not a kernel thread | |
| 945 | + mov x0, x20 | |
| 946 | + blr x19 | |
| 947 | +1: get_thread_info tsk | |
| 948 | + b ret_to_user | |
| 949 | +ENDPROC(ret_from_fork) | |
| 950 | +NOKPROBE(ret_from_fork) | |
| @@ -17,16 +17,19 @@ · 16 unmodified lines | ||
| 17 | 17 | * along with this program. If not, see <http://www.gnu.org/licenses/>. |
| 18 | 18 | */ |
| 19 | 19 | |
| 20 | +#include <linux/bottom_half.h> | |
| 20 | 21 | #include <linux/cpu.h> |
| 21 | 22 | #include <linux/cpu_pm.h> |
| 22 | 23 | #include <linux/kernel.h> |
| 23 | 24 | #include <linux/init.h> |
| 25 | +#include <linux/percpu.h> | |
| 26 | +#include <linux/preempt.h> | |
| 24 | 27 | #include <linux/sched/signal.h> |
| 25 | 28 | #include <linux/signal.h> |
| 26 | −#include <linux/hardirq.h> | |
| 27 | 29 | |
| 28 | 30 | #include <asm/fpsimd.h> |
| 29 | 31 | #include <asm/cputype.h> |
| 32 | +#include <asm/simd.h> | |
| 30 | 33 | |
| 31 | 34 | #define FPEXC_IOF (1 << 0) |
| 32 | 35 | #define FPEXC_DZF (1 << 1) |
| @@ -62,6 +65,13 @@ · 29 unmodified lines | ||
| 62 | 65 | * CPU currently contain the most recent userland FPSIMD state of the current |
| 63 | 66 | * task. |
| 64 | 67 | * |
| 68 | + * In order to allow softirq handlers to use FPSIMD, kernel_neon_begin() may | |
| 69 | + * save the task's FPSIMD context back to task_struct from softirq context. | |
| 70 | + * To prevent this from racing with the manipulation of the task's FPSIMD state | |
| 71 | + * from task context and thereby corrupting the state, it is necessary to | |
| 72 | + * protect any manipulation of a task's fpsimd_state or TIF_FOREIGN_FPSTATE | |
| 73 | + * flag with local_bh_disable() unless softirqs are already masked. | |
| 74 | + * | |
| 65 | 75 | * For a certain task, the sequence may look something like this: |
| 66 | 76 | * - the task gets scheduled in; if both the task's fpsimd_state.cpu field |
| 67 | 77 | * contains the id of the current CPU, and the CPU's fpsimd_last_state per-cpu |
| @@ -161,11 +171,14 @@ void fpsimd_flush_thread(void) · 93 unmodified lines | ||
| 161 | 171 | { |
| 162 | 172 | if (!system_supports_fpsimd()) |
| 163 | 173 | return; |
| 164 | − preempt_disable(); | |
| 174 | + | |
| 175 | + local_bh_disable(); | |
| 176 | + | |
| 165 | 177 | memset(¤t->thread.fpsimd_state, 0, sizeof(struct fpsimd_state)); |
| 166 | 178 | fpsimd_flush_task_state(current); |
| 167 | 179 | set_thread_flag(TIF_FOREIGN_FPSTATE); |
| 168 | − preempt_enable(); | |
| 180 | + | |
| 181 | + local_bh_enable(); | |
| 169 | 182 | } |
| 170 | 183 | |
| 171 | 184 | /* |
| @@ -176,10 +189,13 @@ void fpsimd_preserve_current_state(void) · 4 unmodified lines | ||
| 176 | 189 | { |
| 177 | 190 | if (!system_supports_fpsimd()) |
| 178 | 191 | return; |
| 179 | − preempt_disable(); | |
| 192 | + | |
| 193 | + local_bh_disable(); | |
| 194 | + | |
| 180 | 195 | if (!test_thread_flag(TIF_FOREIGN_FPSTATE)) |
| 181 | 196 | fpsimd_save_state(¤t->thread.fpsimd_state); |
| 182 | − preempt_enable(); | |
| 197 | + | |
| 198 | + local_bh_enable(); | |
| 183 | 199 | } |
| 184 | 200 | |
| 185 | 201 | /* |
| @@ -191,15 +207,18 @@ void fpsimd_restore_current_state(void) · 5 unmodified lines | ||
| 191 | 207 | { |
| 192 | 208 | if (!system_supports_fpsimd()) |
| 193 | 209 | return; |
| 194 | − preempt_disable(); | |
| 210 | + | |
| 211 | + local_bh_disable(); | |
| 212 | + | |
| 195 | 213 | if (test_and_clear_thread_flag(TIF_FOREIGN_FPSTATE)) { |
| 196 | 214 | struct fpsimd_state *st = ¤t->thread.fpsimd_state; |
| 197 | 215 | |
| 198 | 216 | fpsimd_load_state(st); |
| 199 | − this_cpu_write(fpsimd_last_state, st); | |
| 217 | + __this_cpu_write(fpsimd_last_state, st); | |
| 200 | 218 | st->cpu = smp_processor_id(); |
| 201 | 219 | } |
| 202 | − preempt_enable(); | |
| 220 | + | |
| 221 | + local_bh_enable(); | |
| 203 | 222 | } |
| 204 | 223 | |
| 205 | 224 | /* |
| @@ -211,15 +230,18 @@ void fpsimd_update_current_state(struct fpsimd_state *state) · 5 unmodified lines | ||
| 211 | 230 | { |
| 212 | 231 | if (!system_supports_fpsimd()) |
| 213 | 232 | return; |
| 214 | − preempt_disable(); | |
| 233 | + | |
| 234 | + local_bh_disable(); | |
| 235 | + | |
| 215 | 236 | fpsimd_load_state(state); |
| 216 | 237 | if (test_and_clear_thread_flag(TIF_FOREIGN_FPSTATE)) { |
| 217 | 238 | struct fpsimd_state *st = ¤t->thread.fpsimd_state; |
| 218 | 239 | |
| 219 | − this_cpu_write(fpsimd_last_state, st); | |
| 240 | + __this_cpu_write(fpsimd_last_state, st); | |
| 220 | 241 | st->cpu = smp_processor_id(); |
| 221 | 242 | } |
| 222 | − preempt_enable(); | |
| 243 | + | |
| 244 | + local_bh_enable(); | |
| 223 | 245 | } |
| 224 | 246 | |
| 225 | 247 | /* |
| @@ -232,52 +254,122 @@ void fpsimd_flush_task_state(struct task_struct *t) · 6 unmodified lines | ||
| 232 | 254 | |
| 233 | 255 | #ifdef CONFIG_KERNEL_MODE_NEON |
| 234 | 256 | |
| 235 | −static DEFINE_PER_CPU(struct fpsimd_partial_state, hardirq_fpsimdstate); | |
| 236 | −static DEFINE_PER_CPU(struct fpsimd_partial_state, softirq_fpsimdstate); | |
| 257 | +DEFINE_PER_CPU(bool, kernel_neon_busy); | |
| 258 | +EXPORT_PER_CPU_SYMBOL(kernel_neon_busy); | |
| 237 | 259 | |
| 238 | 260 | /* |
| 239 | 261 | * Kernel-side NEON support functions |
| 240 | 262 | */ |
| 241 | −void kernel_neon_begin_partial(u32 num_regs) | |
| 263 | + | |
| 264 | +/* | |
| 265 | + * kernel_neon_begin(): obtain the CPU FPSIMD registers for use by the calling | |
| 266 | + * context | |
| 267 | + * | |
| 268 | + * Must not be called unless may_use_simd() returns true. | |
| 269 | + * Task context in the FPSIMD registers is saved back to memory as necessary. | |
| 270 | + * | |
| 271 | + * A matching call to kernel_neon_end() must be made before returning from the | |
| 272 | + * calling context. | |
| 273 | + * | |
| 274 | + * The caller may freely use the FPSIMD registers until kernel_neon_end() is | |
| 275 | + * called. | |
| 276 | + */ | |
| 277 | +void kernel_neon_begin(void) | |
| 242 | 278 | { |
| 243 | 279 | if (WARN_ON(!system_supports_fpsimd())) |
| 244 | 280 | return; |
| 245 | − if (in_interrupt()) { | |
| 246 | − struct fpsimd_partial_state *s = this_cpu_ptr( | |
| 247 | − in_irq() ? &hardirq_fpsimdstate : &softirq_fpsimdstate); | |
| 248 | 281 | |
| 249 | − BUG_ON(num_regs > 32); | |
| 250 | − fpsimd_save_partial_state(s, roundup(num_regs, 2)); | |
| 251 | − } else { | |
| 252 | − /* | |
| 253 | − * Save the userland FPSIMD state if we have one and if we | |
| 254 | − * haven't done so already. Clear fpsimd_last_state to indicate | |
| 255 | − * that there is no longer userland FPSIMD state in the | |
| 256 | − * registers. | |
| 257 | − */ | |
| 258 | − preempt_disable(); | |
| 259 | − if (current->mm && | |
| 260 | − !test_and_set_thread_flag(TIF_FOREIGN_FPSTATE)) | |
| 261 | − fpsimd_save_state(¤t->thread.fpsimd_state); | |
| 262 | − this_cpu_write(fpsimd_last_state, NULL); | |
| 263 | − } | |
| 282 | + BUG_ON(!may_use_simd()); | |
| 283 | + | |
| 284 | + local_bh_disable(); | |
| 285 | + | |
| 286 | + __this_cpu_write(kernel_neon_busy, true); | |
| 287 | + | |
| 288 | + /* Save unsaved task fpsimd state, if any: */ | |
| 289 | + if (current->mm && !test_and_set_thread_flag(TIF_FOREIGN_FPSTATE)) | |
| 290 | + fpsimd_save_state(¤t->thread.fpsimd_state); | |
| 291 | + | |
| 292 | + /* Invalidate any task state remaining in the fpsimd regs: */ | |
| 293 | + __this_cpu_write(fpsimd_last_state, NULL); | |
| 294 | + | |
| 295 | + preempt_disable(); | |
| 296 | + | |
| 297 | + local_bh_enable(); | |
| 264 | 298 | } |
| 265 | −EXPORT_SYMBOL(kernel_neon_begin_partial); | |
| 299 | +EXPORT_SYMBOL(kernel_neon_begin); | |
| 266 | 300 | |
| 301 | +/* | |
| 302 | + * kernel_neon_end(): give the CPU FPSIMD registers back to the current task | |
| 303 | + * | |
| 304 | + * Must be called from a context in which kernel_neon_begin() was previously | |
| 305 | + * called, with no call to kernel_neon_end() in the meantime. | |
| 306 | + * | |
| 307 | + * The caller must not use the FPSIMD registers after this function is called, | |
| 308 | + * unless kernel_neon_begin() is called again in the meantime. | |
| 309 | + */ | |
| 267 | 310 | void kernel_neon_end(void) |
| 268 | 311 | { |
| 312 | + bool busy; | |
| 313 | + | |
| 269 | 314 | if (!system_supports_fpsimd()) |
| 270 | 315 | return; |
| 271 | − if (in_interrupt()) { | |
| 272 | − struct fpsimd_partial_state *s = this_cpu_ptr( | |
| 273 | − in_irq() ? &hardirq_fpsimdstate : &softirq_fpsimdstate); | |
| 274 | − fpsimd_load_partial_state(s); | |
| 275 | − } else { | |
| 276 | − preempt_enable(); | |
| 277 | − } | |
| 316 | + | |
| 317 | + busy = __this_cpu_xchg(kernel_neon_busy, false); | |
| 318 | + WARN_ON(!busy); /* No matching kernel_neon_begin()? */ | |
| 319 | + | |
| 320 | + preempt_enable(); | |
| 278 | 321 | } |
| 279 | 322 | EXPORT_SYMBOL(kernel_neon_end); |
| 280 | 323 | |
| 324 | +static DEFINE_PER_CPU(struct fpsimd_state, efi_fpsimd_state); | |
| 325 | +static DEFINE_PER_CPU(bool, efi_fpsimd_state_used); | |
| 326 | + | |
| 327 | +/* | |
| 328 | + * EFI runtime services support functions | |
| 329 | + * | |
| 330 | + * The ABI for EFI runtime services allows EFI to use FPSIMD during the call. | |
| 331 | + * This means that for EFI (and only for EFI), we have to assume that FPSIMD | |
| 332 | + * is always used rather than being an optional accelerator. | |
| 333 | + * | |
| 334 | + * These functions provide the necessary support for ensuring FPSIMD | |
| 335 | + * save/restore in the contexts from which EFI is used. | |
| 336 | + * | |
| 337 | + * Do not use them for any other purpose -- if tempted to do so, you are | |
| 338 | + * either doing something wrong or you need to propose some refactoring. | |
| 339 | + */ | |
| 340 | + | |
| 341 | +/* | |
| 342 | + * __efi_fpsimd_begin(): prepare FPSIMD for making an EFI runtime services call | |
| 343 | + */ | |
| 344 | +void __efi_fpsimd_begin(void) | |
| 345 | +{ | |
| 346 | + if (!system_supports_fpsimd()) | |
| 347 | + return; | |
| 348 | + | |
| 349 | + WARN_ON(preemptible()); | |
| 350 | + | |
| 351 | + if (may_use_simd()) | |
| 352 | + kernel_neon_begin(); | |
| 353 | + else { | |
| 354 | + fpsimd_save_state(this_cpu_ptr(&efi_fpsimd_state)); | |
| 355 | + __this_cpu_write(efi_fpsimd_state_used, true); | |
| 356 | + } | |
| 357 | +} | |
| 358 | + | |
| 359 | +/* | |
| 360 | + * __efi_fpsimd_end(): clean up FPSIMD after an EFI runtime services call | |
| 361 | + */ | |
| 362 | +void __efi_fpsimd_end(void) | |
| 363 | +{ | |
| 364 | + if (!system_supports_fpsimd()) | |
| 365 | + return; | |
| 366 | + | |
| 367 | + if (__this_cpu_xchg(efi_fpsimd_state_used, false)) | |
| 368 | + fpsimd_load_state(this_cpu_ptr(&efi_fpsimd_state)); | |
| 369 | + else | |
| 370 | + kernel_neon_end(); | |
| 371 | +} | |
| 372 | + | |
| 281 | 373 | #endif /* CONFIG_KERNEL_MODE_NEON */ |
| 282 | 374 | |
| 283 | 375 | #ifdef CONFIG_CPU_PM |
| @@ -143,8 +143,8 @@ preserve_boot_args: · 142 unmodified lines | ||
| 143 | 143 | dmb sy // needed before dc ivac with |
| 144 | 144 | // MMU off |
| 145 | 145 | |
| 146 | − add x1, x0, #0x20 // 4 x 8 bytes | |
| 147 | − b __inval_cache_range // tail call | |
| 146 | + mov x1, #0x20 // 4 x 8 bytes | |
| 147 | + b __inval_dcache_area // tail call | |
| 148 | 148 | ENDPROC(preserve_boot_args) |
| 149 | 149 | |
| 150 | 150 | /* |
| @@ -221,20 +221,20 @@ __create_page_tables: · 70 unmodified lines | ||
| 221 | 221 | * dirty cache lines being evicted. |
| 222 | 222 | */ |
| 223 | 223 | adrp x0, idmap_pg_dir |
| 224 | − adrp x1, swapper_pg_dir + SWAPPER_DIR_SIZE + RESERVED_TTBR0_SIZE | |
| 225 | − bl __inval_cache_range | |
| 224 | + ldr x1, =(IDMAP_DIR_SIZE + SWAPPER_DIR_SIZE + RESERVED_TTBR0_SIZE) | |
| 225 | + bl __inval_dcache_area | |
| 226 | 226 | |
| 227 | 227 | /* |
| 228 | 228 | * Clear the idmap and swapper page tables. |
| 229 | 229 | */ |
| 230 | 230 | adrp x0, idmap_pg_dir |
| 231 | − adrp x6, swapper_pg_dir + SWAPPER_DIR_SIZE + RESERVED_TTBR0_SIZE | |
| 231 | + ldr x1, =(IDMAP_DIR_SIZE + SWAPPER_DIR_SIZE + RESERVED_TTBR0_SIZE) | |
| 232 | 232 | 1: stp xzr, xzr, [x0], #16 |
| 233 | 233 | stp xzr, xzr, [x0], #16 |
| 234 | 234 | stp xzr, xzr, [x0], #16 |
| 235 | 235 | stp xzr, xzr, [x0], #16 |
| 236 | − cmp x0, x6 | |
| 237 | − b.lo 1b | |
| 236 | + subs x1, x1, #64 | |
| 237 | + b.ne 1b | |
| 238 | 238 | |
| 239 | 239 | mov x7, SWAPPER_MM_MMUFLAGS |
| 240 | 240 | |
| @@ -307,9 +307,9 @@ __create_page_tables: · 66 unmodified lines | ||
| 307 | 307 | * tables again to remove any speculatively loaded cache lines. |
| 308 | 308 | */ |
| 309 | 309 | adrp x0, idmap_pg_dir |
| 310 | − adrp x1, swapper_pg_dir + SWAPPER_DIR_SIZE + RESERVED_TTBR0_SIZE | |
| 310 | + ldr x1, =(IDMAP_DIR_SIZE + SWAPPER_DIR_SIZE + RESERVED_TTBR0_SIZE) | |
| 311 | 311 | dmb sy |
| 312 | − bl __inval_cache_range | |
| 312 | + bl __inval_dcache_area | |
| 313 | 313 | |
| 314 | 314 | ret x28 |
| 315 | 315 | ENDPROC(__create_page_tables) |
| @@ -361,6 +361,9 @@ __primary_switched: · 45 unmodified lines | ||
| 361 | 361 | ret // to __primary_switch() |
| 362 | 362 | 0: |
| 363 | 363 | #endif |
| 364 | + add sp, sp, #16 | |
| 365 | + mov x29, #0 | |
| 366 | + mov x30, #0 | |
| 364 | 367 | b start_kernel |
| 365 | 368 | ENDPROC(__primary_switched) |
| 366 | 369 | |
| @@ -616,6 +619,7 @@ __secondary_switched: · 249 unmodified lines | ||
| 616 | 619 | ldr x2, [x0, #CPU_BOOT_TASK] |
| 617 | 620 | msr sp_el0, x2 |
| 618 | 621 | mov x29, #0 |
| 622 | + mov x30, #0 | |
| 619 | 623 | b secondary_start_kernel |
| 620 | 624 | ENDPROC(__secondary_switched) |
| 621 | 625 | |
| @@ -330,7 +330,7 @@ static void _copy_pte(pte_t *dst_pte, pte_t *src_pte, unsigned long addr) · 329 unmodified lines | ||
| 330 | 330 | * read only (code, rodata). Clear the RDONLY bit from |
| 331 | 331 | * the temporary mappings we use during restore. |
| 332 | 332 | */ |
| 333 | − set_pte(dst_pte, pte_clear_rdonly(pte)); | |
| 333 | + set_pte(dst_pte, pte_mkwrite(pte)); | |
| 334 | 334 | } else if (debug_pagealloc_enabled() && !pte_none(pte)) { |
| 335 | 335 | /* |
| 336 | 336 | * debug_pagealloc will removed the PTE_VALID bit if |
| @@ -343,7 +343,7 @@ static void _copy_pte(pte_t *dst_pte, pte_t *src_pte, unsigned long addr) · 6 unmodified lines | ||
| 343 | 343 | */ |
| 344 | 344 | BUG_ON(!pfn_valid(pte_pfn(pte))); |
| 345 | 345 | |
| 346 | − set_pte(dst_pte, pte_mkpresent(pte_clear_rdonly(pte))); | |
| 346 | + set_pte(dst_pte, pte_mkpresent(pte_mkwrite(pte))); | |
| 347 | 347 | } |
| 348 | 348 | } |
| 349 | 349 | |
| @@ -23,15 +23,16 @@ · 22 unmodified lines | ||
| 23 | 23 | |
| 24 | 24 | #include <linux/kernel_stat.h> |
| 25 | 25 | #include <linux/irq.h> |
| 26 | +#include <linux/memory.h> | |
| 26 | 27 | #include <linux/smp.h> |
| 27 | 28 | #include <linux/init.h> |
| 28 | 29 | #include <linux/irqchip.h> |
| 29 | 30 | #include <linux/seq_file.h> |
| 31 | +#include <linux/vmalloc.h> | |
| 30 | 32 | |
| 31 | 33 | unsigned long irq_err_count; |
| 32 | 34 | |
| 33 | −/* irq stack only needs to be 16 byte aligned - not IRQ_STACK_SIZE aligned. */ | |
| 34 | −DEFINE_PER_CPU(unsigned long [IRQ_STACK_SIZE/sizeof(long)], irq_stack) __aligned(16); | |
| 35 | +DEFINE_PER_CPU(unsigned long *, irq_stack_ptr); | |
| 35 | 36 | |
| 36 | 37 | int arch_show_interrupts(struct seq_file *p, int prec) |
| 37 | 38 | { |
| @@ -50,8 +51,43 @@ void __init set_handle_irq(void (*handle_irq)(struct pt_regs *)) · 12 unmodified lines | ||
| 50 | 51 | handle_arch_irq = handle_irq; |
| 51 | 52 | } |
| 52 | 53 | |
| 54 | +#ifdef CONFIG_VMAP_STACK | |
| 55 | +static void init_irq_stacks(void) | |
| 56 | +{ | |
| 57 | + int cpu; | |
| 58 | + unsigned long *p; | |
| 59 | + | |
| 60 | + for_each_possible_cpu(cpu) { | |
| 61 | + /* | |
| 62 | + * To ensure that VMAP'd stack overflow detection works | |
| 63 | + * correctly, the IRQ stacks need to have the same | |
| 64 | + * alignment as other stacks. | |
| 65 | + */ | |
| 66 | + p = __vmalloc_node_range(IRQ_STACK_SIZE, THREAD_ALIGN, | |
| 67 | + VMALLOC_START, VMALLOC_END, | |
| 68 | + THREADINFO_GFP, PAGE_KERNEL, | |
| 69 | + 0, cpu_to_node(cpu), | |
| 70 | + __builtin_return_address(0)); | |
| 71 | + | |
| 72 | + per_cpu(irq_stack_ptr, cpu) = p; | |
| 73 | + } | |
| 74 | +} | |
| 75 | +#else | |
| 76 | +/* irq stack only needs to be 16 byte aligned - not IRQ_STACK_SIZE aligned. */ | |
| 77 | +DEFINE_PER_CPU_ALIGNED(unsigned long [IRQ_STACK_SIZE/sizeof(long)], irq_stack); | |
| 78 | + | |
| 79 | +static void init_irq_stacks(void) | |
| 80 | +{ | |
| 81 | + int cpu; | |
| 82 | + | |
| 83 | + for_each_possible_cpu(cpu) | |
| 84 | + per_cpu(irq_stack_ptr, cpu) = per_cpu(irq_stack, cpu); | |
| 85 | +} | |
| 86 | +#endif | |
| 87 | + | |
| 53 | 88 | void __init init_IRQ(void) |
| 54 | 89 | { |
| 90 | + init_irq_stacks(); | |
| 55 | 91 | irqchip_init(); |
| 56 | 92 | if (!handle_arch_irq) |
| 57 | 93 | panic("No interrupt controller found."); |
| @@ -252,7 +252,7 @@ void machine_crash_shutdown(struct pt_regs *regs) · 251 unmodified lines | ||
| 252 | 252 | local_irq_disable(); |
| 253 | 253 | |
| 254 | 254 | /* shutdown non-crashing cpus */ |
| 255 | − smp_send_crash_stop(); | |
| 255 | + crash_smp_send_stop(); | |
| 256 | 256 | |
| 257 | 257 | /* for crashing cpu */ |
| 258 | 258 | crash_save_cpu(regs, smp_processor_id()); |
| @@ -162,7 +162,6 @@ void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, · 161 unmodified lines | ||
| 162 | 162 | } |
| 163 | 163 | |
| 164 | 164 | frame.fp = regs->regs[29]; |
| 165 | − frame.sp = regs->sp; | |
| 166 | 165 | frame.pc = regs->pc; |
| 167 | 166 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER |
| 168 | 167 | frame.graph = current->curr_ret_stack; |
| @@ -202,55 +202,6 @@ static const unsigned armv8_pmuv3_perf_map[PERF_COUNT_HW_MAX] = { · 201 unmodified lines | ||
| 202 | 202 | [PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = ARMV8_PMUV3_PERFCTR_STALL_BACKEND, |
| 203 | 203 | }; |
| 204 | 204 | |
| 205 | −/* ARM Cortex-A53 HW events mapping. */ | |
| 206 | −static const unsigned armv8_a53_perf_map[PERF_COUNT_HW_MAX] = { | |
| 207 | − PERF_MAP_ALL_UNSUPPORTED, | |
| 208 | − [PERF_COUNT_HW_CPU_CYCLES] = ARMV8_PMUV3_PERFCTR_CPU_CYCLES, | |
| 209 | − [PERF_COUNT_HW_INSTRUCTIONS] = ARMV8_PMUV3_PERFCTR_INST_RETIRED, | |
| 210 | − [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV8_PMUV3_PERFCTR_L1D_CACHE, | |
| 211 | − [PERF_COUNT_HW_CACHE_MISSES] = ARMV8_PMUV3_PERFCTR_L1D_CACHE_REFILL, | |
| 212 | − [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV8_PMUV3_PERFCTR_PC_WRITE_RETIRED, | |
| 213 | − [PERF_COUNT_HW_BRANCH_MISSES] = ARMV8_PMUV3_PERFCTR_BR_MIS_PRED, | |
| 214 | − [PERF_COUNT_HW_BUS_CYCLES] = ARMV8_PMUV3_PERFCTR_BUS_CYCLES, | |
| 215 | −}; | |
| 216 | − | |
| 217 | −/* ARM Cortex-A57 and Cortex-A72 events mapping. */ | |
| 218 | −static const unsigned armv8_a57_perf_map[PERF_COUNT_HW_MAX] = { | |
| 219 | − PERF_MAP_ALL_UNSUPPORTED, | |
| 220 | − [PERF_COUNT_HW_CPU_CYCLES] = ARMV8_PMUV3_PERFCTR_CPU_CYCLES, | |
| 221 | − [PERF_COUNT_HW_INSTRUCTIONS] = ARMV8_PMUV3_PERFCTR_INST_RETIRED, | |
| 222 | − [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV8_PMUV3_PERFCTR_L1D_CACHE, | |
| 223 | − [PERF_COUNT_HW_CACHE_MISSES] = ARMV8_PMUV3_PERFCTR_L1D_CACHE_REFILL, | |
| 224 | − [PERF_COUNT_HW_BRANCH_MISSES] = ARMV8_PMUV3_PERFCTR_BR_MIS_PRED, | |
| 225 | − [PERF_COUNT_HW_BUS_CYCLES] = ARMV8_PMUV3_PERFCTR_BUS_CYCLES, | |
| 226 | −}; | |
| 227 | − | |
| 228 | −static const unsigned armv8_thunder_perf_map[PERF_COUNT_HW_MAX] = { | |
| 229 | − PERF_MAP_ALL_UNSUPPORTED, | |
| 230 | − [PERF_COUNT_HW_CPU_CYCLES] = ARMV8_PMUV3_PERFCTR_CPU_CYCLES, | |
| 231 | − [PERF_COUNT_HW_INSTRUCTIONS] = ARMV8_PMUV3_PERFCTR_INST_RETIRED, | |
| 232 | − [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV8_PMUV3_PERFCTR_L1D_CACHE, | |
| 233 | − [PERF_COUNT_HW_CACHE_MISSES] = ARMV8_PMUV3_PERFCTR_L1D_CACHE_REFILL, | |
| 234 | − [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV8_PMUV3_PERFCTR_PC_WRITE_RETIRED, | |
| 235 | − [PERF_COUNT_HW_BRANCH_MISSES] = ARMV8_PMUV3_PERFCTR_BR_MIS_PRED, | |
| 236 | − [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = ARMV8_PMUV3_PERFCTR_STALL_FRONTEND, | |
| 237 | − [PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = ARMV8_PMUV3_PERFCTR_STALL_BACKEND, | |
| 238 | −}; | |
| 239 | − | |
| 240 | −/* Broadcom Vulcan events mapping */ | |
| 241 | −static const unsigned armv8_vulcan_perf_map[PERF_COUNT_HW_MAX] = { | |
| 242 | − PERF_MAP_ALL_UNSUPPORTED, | |
| 243 | − [PERF_COUNT_HW_CPU_CYCLES] = ARMV8_PMUV3_PERFCTR_CPU_CYCLES, | |
| 244 | − [PERF_COUNT_HW_INSTRUCTIONS] = ARMV8_PMUV3_PERFCTR_INST_RETIRED, | |
| 245 | − [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV8_PMUV3_PERFCTR_L1D_CACHE, | |
| 246 | − [PERF_COUNT_HW_CACHE_MISSES] = ARMV8_PMUV3_PERFCTR_L1D_CACHE_REFILL, | |
| 247 | − [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV8_PMUV3_PERFCTR_BR_RETIRED, | |
| 248 | − [PERF_COUNT_HW_BRANCH_MISSES] = ARMV8_PMUV3_PERFCTR_BR_MIS_PRED, | |
| 249 | − [PERF_COUNT_HW_BUS_CYCLES] = ARMV8_PMUV3_PERFCTR_BUS_CYCLES, | |
| 250 | − [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = ARMV8_PMUV3_PERFCTR_STALL_FRONTEND, | |
| 251 | − [PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = ARMV8_PMUV3_PERFCTR_STALL_BACKEND, | |
| 252 | −}; | |
| 253 | − | |
| 254 | 205 | static const unsigned armv8_pmuv3_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] |
| 255 | 206 | [PERF_COUNT_HW_CACHE_OP_MAX] |
| 256 | 207 | [PERF_COUNT_HW_CACHE_RESULT_MAX] = { |
| @@ -281,27 +232,10 @@ static const unsigned armv8_a53_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] · 24 unmodified lines | ||
| 281 | 232 | [PERF_COUNT_HW_CACHE_RESULT_MAX] = { |
| 282 | 233 | PERF_CACHE_MAP_ALL_UNSUPPORTED, |
| 283 | 234 | |
| 284 | − [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_L1D_CACHE, | |
| 285 | − [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L1D_CACHE_REFILL, | |
| 286 | − [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_L1D_CACHE, | |
| 287 | − [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L1D_CACHE_REFILL, | |
| 288 | 235 | [C(L1D)][C(OP_PREFETCH)][C(RESULT_MISS)] = ARMV8_A53_PERFCTR_PREF_LINEFILL, |
| 289 | 236 | |
| 290 | − [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_L1I_CACHE, | |
| 291 | − [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L1I_CACHE_REFILL, | |
| 292 | − | |
| 293 | − [C(LL)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_L2D_CACHE, | |
| 294 | − [C(LL)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L2D_CACHE_REFILL, | |
| 295 | − [C(LL)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_L2D_CACHE, | |
| 296 | − [C(LL)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L2D_CACHE_REFILL, | |
| 297 | − | |
| 298 | − [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L1D_TLB_REFILL, | |
| 299 | − [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L1I_TLB_REFILL, | |
| 300 | − | |
| 301 | − [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_BR_PRED, | |
| 302 | − [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_BR_MIS_PRED, | |
| 303 | − [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_BR_PRED, | |
| 304 | − [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_BR_MIS_PRED, | |
| 237 | + [C(NODE)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_RD, | |
| 238 | + [C(NODE)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_WR, | |
| 305 | 239 | }; |
| 306 | 240 | |
| 307 | 241 | static const unsigned armv8_a57_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] |
| @@ -314,18 +248,26 @@ static const unsigned armv8_a57_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] · 6 unmodified lines | ||
| 314 | 248 | [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_L1D_CACHE_WR, |
| 315 | 249 | [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV8_IMPDEF_PERFCTR_L1D_CACHE_REFILL_WR, |
| 316 | 250 | |
| 317 | − [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_L1I_CACHE, | |
| 318 | − [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L1I_CACHE_REFILL, | |
| 319 | − | |
| 320 | 251 | [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_REFILL_RD, |
| 321 | 252 | [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_REFILL_WR, |
| 322 | 253 | |
| 323 | − [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L1I_TLB_REFILL, | |
| 254 | + [C(NODE)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_RD, | |
| 255 | + [C(NODE)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_WR, | |
| 256 | +}; | |
| 324 | 257 | |
| 325 | − [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_BR_PRED, | |
| 326 | − [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_BR_MIS_PRED, | |
| 327 | − [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_BR_PRED, | |
| 328 | − [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_BR_MIS_PRED, | |
| 258 | +static const unsigned armv8_a73_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] | |
| 259 | + [PERF_COUNT_HW_CACHE_OP_MAX] | |
| 260 | + [PERF_COUNT_HW_CACHE_RESULT_MAX] = { | |
| 261 | + PERF_CACHE_MAP_ALL_UNSUPPORTED, | |
| 262 | + | |
| 263 | + [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_L1D_CACHE_RD, | |
| 264 | + [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_L1D_CACHE_WR, | |
| 265 | + | |
| 266 | + [C(NODE)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_RD, | |
| 267 | + [C(NODE)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_WR, | |
| 268 | + | |
| 269 | + [C(NODE)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_RD, | |
| 270 | + [C(NODE)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_WR, | |
| 329 | 271 | }; |
| 330 | 272 | |
| 331 | 273 | static const unsigned armv8_thunder_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] |
| @@ -340,8 +282,6 @@ static const unsigned armv8_thunder_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] · 8 unmodified lines | ||
| 340 | 282 | [C(L1D)][C(OP_PREFETCH)][C(RESULT_ACCESS)] = ARMV8_THUNDER_PERFCTR_L1D_CACHE_PREF_ACCESS, |
| 341 | 283 | [C(L1D)][C(OP_PREFETCH)][C(RESULT_MISS)] = ARMV8_THUNDER_PERFCTR_L1D_CACHE_PREF_MISS, |
| 342 | 284 | |
| 343 | − [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_L1I_CACHE, | |
| 344 | − [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L1I_CACHE_REFILL, | |
| 345 | 285 | [C(L1I)][C(OP_PREFETCH)][C(RESULT_ACCESS)] = ARMV8_THUNDER_PERFCTR_L1I_CACHE_PREF_ACCESS, |
| 346 | 286 | [C(L1I)][C(OP_PREFETCH)][C(RESULT_MISS)] = ARMV8_THUNDER_PERFCTR_L1I_CACHE_PREF_MISS, |
| 347 | 287 | |
| @@ -349,13 +289,6 @@ static const unsigned armv8_thunder_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] · 1 unmodified line | ||
| 349 | 289 | [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_REFILL_RD, |
| 350 | 290 | [C(DTLB)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_WR, |
| 351 | 291 | [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_REFILL_WR, |
| 352 | − | |
| 353 | − [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L1I_TLB_REFILL, | |
| 354 | − | |
| 355 | − [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_BR_PRED, | |
| 356 | − [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_BR_MIS_PRED, | |
| 357 | − [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_BR_PRED, | |
| 358 | − [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_BR_MIS_PRED, | |
| 359 | 292 | }; |
| 360 | 293 | |
| 361 | 294 | static const unsigned armv8_vulcan_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] |
| @@ -368,22 +301,11 @@ static const unsigned armv8_vulcan_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] · 6 unmodified lines | ||
| 368 | 301 | [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_L1D_CACHE_WR, |
| 369 | 302 | [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV8_IMPDEF_PERFCTR_L1D_CACHE_REFILL_WR, |
| 370 | 303 | |
| 371 | − [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_L1I_CACHE, | |
| 372 | − [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L1I_CACHE_REFILL, | |
| 373 | − | |
| 374 | − [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L1I_TLB_REFILL, | |
| 375 | − [C(ITLB)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_L1I_TLB, | |
| 376 | − | |
| 377 | 304 | [C(DTLB)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_RD, |
| 378 | 305 | [C(DTLB)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_WR, |
| 379 | 306 | [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_REFILL_RD, |
| 380 | 307 | [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_REFILL_WR, |
| 381 | 308 | |
| 382 | − [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_BR_PRED, | |
| 383 | − [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_BR_MIS_PRED, | |
| 384 | − [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_BR_PRED, | |
| 385 | − [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_BR_MIS_PRED, | |
| 386 | − | |
| 387 | 309 | [C(NODE)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_RD, |
| 388 | 310 | [C(NODE)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_WR, |
| 389 | 311 | }; |
| @@ -846,17 +768,14 @@ static int armv8pmu_get_event_idx(struct pmu_hw_events *cpuc, · 456 unmodified lines | ||
| 846 | 768 | struct hw_perf_event *hwc = &event->hw; |
| 847 | 769 | unsigned long evtype = hwc->config_base & ARMV8_PMU_EVTYPE_EVENT; |
| 848 | 770 | |
| 849 | − /* Always place a cycle counter into the cycle counter. */ | |
| 771 | + /* Always prefer to place a cycle counter into the cycle counter. */ | |
| 850 | 772 | if (evtype == ARMV8_PMUV3_PERFCTR_CPU_CYCLES) { |
| 851 | − if (test_and_set_bit(ARMV8_IDX_CYCLE_COUNTER, cpuc->used_mask)) | |
| 852 | − return -EAGAIN; | |
| 853 | − | |
| 854 | − return ARMV8_IDX_CYCLE_COUNTER; | |
| 773 | + if (!test_and_set_bit(ARMV8_IDX_CYCLE_COUNTER, cpuc->used_mask)) | |
| 774 | + return ARMV8_IDX_CYCLE_COUNTER; | |
| 855 | 775 | } |
| 856 | 776 | |
| 857 | 777 | /* |
| 858 | − * For anything other than a cycle counter, try and use | |
| 859 | − * the events counters | |
| 778 | + * Otherwise use events counters | |
| 860 | 779 | */ |
| 861 | 780 | for (idx = ARMV8_IDX_COUNTER0; idx < cpu_pmu->num_events; ++idx) { |
| 862 | 781 | if (!test_and_set_bit(idx, cpuc->used_mask)) |
| @@ -924,7 +843,13 @@ static void armv8pmu_reset(void *info) · 61 unmodified lines | ||
| 924 | 843 | ARMV8_PMU_PMCR_LC); |
| 925 | 844 | } |
| 926 | 845 | |
| 927 | −static int armv8_pmuv3_map_event(struct perf_event *event) | |
| 846 | +static int __armv8_pmuv3_map_event(struct perf_event *event, | |
| 847 | + const unsigned (*extra_event_map) | |
| 848 | + [PERF_COUNT_HW_MAX], | |
| 849 | + const unsigned (*extra_cache_map) | |
| 850 | + [PERF_COUNT_HW_CACHE_MAX] | |
| 851 | + [PERF_COUNT_HW_CACHE_OP_MAX] | |
| 852 | + [PERF_COUNT_HW_CACHE_RESULT_MAX]) | |
| 928 | 853 | { |
| 929 | 854 | int hw_event_id; |
| 930 | 855 | struct arm_pmu *armpmu = to_arm_pmu(event->pmu); |
| @@ -932,44 +857,47 @@ static int armv8_pmuv3_map_event(struct perf_event *event) · 1 unmodified line | ||
| 932 | 857 | hw_event_id = armpmu_map_event(event, &armv8_pmuv3_perf_map, |
| 933 | 858 | &armv8_pmuv3_perf_cache_map, |
| 934 | 859 | ARMV8_PMU_EVTYPE_EVENT); |
| 935 | − if (hw_event_id < 0) | |
| 936 | − return hw_event_id; | |
| 937 | 860 | |
| 938 | − /* disable micro/arch events not supported by this PMU */ | |
| 939 | − if ((hw_event_id < ARMV8_PMUV3_MAX_COMMON_EVENTS) && | |
| 940 | − !test_bit(hw_event_id, armpmu->pmceid_bitmap)) { | |
| 941 | − return -EOPNOTSUPP; | |
| 861 | + /* Onl expose micro/arch events supported by this PMU */ | |
| 862 | + if ((hw_event_id > 0) && (hw_event_id < ARMV8_PMUV3_MAX_COMMON_EVENTS) | |
| 863 | + && test_bit(hw_event_id, armpmu->pmceid_bitmap)) { | |
| 864 | + return hw_event_id; | |
| 942 | 865 | } |
| 943 | 866 | |
| 944 | − return hw_event_id; | |
| 867 | + return armpmu_map_event(event, extra_event_map, extra_cache_map, | |
| 868 | + ARMV8_PMU_EVTYPE_EVENT); | |
| 869 | +} | |
| 870 | + | |
| 871 | +static int armv8_pmuv3_map_event(struct perf_event *event) | |
| 872 | +{ | |
| 873 | + return __armv8_pmuv3_map_event(event, NULL, NULL); | |
| 945 | 874 | } |
| 946 | 875 | |
| 947 | 876 | static int armv8_a53_map_event(struct perf_event *event) |
| 948 | 877 | { |
| 949 | − return armpmu_map_event(event, &armv8_a53_perf_map, | |
| 950 | − &armv8_a53_perf_cache_map, | |
| 951 | − ARMV8_PMU_EVTYPE_EVENT); | |
| 878 | + return __armv8_pmuv3_map_event(event, NULL, &armv8_a53_perf_cache_map); | |
| 952 | 879 | } |
| 953 | 880 | |
| 954 | 881 | static int armv8_a57_map_event(struct perf_event *event) |
| 955 | 882 | { |
| 956 | − return armpmu_map_event(event, &armv8_a57_perf_map, | |
| 957 | − &armv8_a57_perf_cache_map, | |
| 958 | − ARMV8_PMU_EVTYPE_EVENT); | |
| 883 | + return __armv8_pmuv3_map_event(event, NULL, &armv8_a57_perf_cache_map); | |
| 884 | +} | |
| 885 | + | |
| 886 | +static int armv8_a73_map_event(struct perf_event *event) | |
| 887 | +{ | |
| 888 | + return __armv8_pmuv3_map_event(event, NULL, &armv8_a73_perf_cache_map); | |
| 959 | 889 | } |
| 960 | 890 | |
| 961 | 891 | static int armv8_thunder_map_event(struct perf_event *event) |
| 962 | 892 | { |
| 963 | − return armpmu_map_event(event, &armv8_thunder_perf_map, | |
| 964 | − &armv8_thunder_perf_cache_map, | |
| 965 | − ARMV8_PMU_EVTYPE_EVENT); | |
| 893 | + return __armv8_pmuv3_map_event(event, NULL, | |
| 894 | + &armv8_thunder_perf_cache_map); | |
| 966 | 895 | } |
| 967 | 896 | |
| 968 | 897 | static int armv8_vulcan_map_event(struct perf_event *event) |
| 969 | 898 | { |
| 970 | − return armpmu_map_event(event, &armv8_vulcan_perf_map, | |
| 971 | − &armv8_vulcan_perf_cache_map, | |
| 972 | − ARMV8_PMU_EVTYPE_EVENT); | |
| 899 | + return __armv8_pmuv3_map_event(event, NULL, | |
| 900 | + &armv8_vulcan_perf_cache_map); | |
| 973 | 901 | } |
| 974 | 902 | |
| 975 | 903 | struct armv8pmu_probe_info { |
| @@ -1062,6 +990,22 @@ static int armv8_pmuv3_init(struct arm_pmu *cpu_pmu) · 86 unmodified lines | ||
| 1062 | 990 | return 0; |
| 1063 | 991 | } |
| 1064 | 992 | |
| 993 | +static int armv8_a35_pmu_init(struct arm_pmu *cpu_pmu) | |
| 994 | +{ | |
| 995 | + int ret = armv8_pmu_init(cpu_pmu); | |
| 996 | + if (ret) | |
| 997 | + return ret; | |
| 998 | + | |
| 999 | + cpu_pmu->name = "armv8_cortex_a35"; | |
| 1000 | + cpu_pmu->map_event = armv8_a53_map_event; | |
| 1001 | + cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] = | |
| 1002 | + &armv8_pmuv3_events_attr_group; | |
| 1003 | + cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] = | |
| 1004 | + &armv8_pmuv3_format_attr_group; | |
| 1005 | + | |
| 1006 | + return 0; | |
| 1007 | +} | |
| 1008 | + | |
| 1065 | 1009 | static int armv8_a53_pmu_init(struct arm_pmu *cpu_pmu) |
| 1066 | 1010 | { |
| 1067 | 1011 | int ret = armv8_pmu_init(cpu_pmu); |
| @@ -1110,6 +1054,22 @@ static int armv8_a72_pmu_init(struct arm_pmu *cpu_pmu) · 42 unmodified lines | ||
| 1110 | 1054 | return 0; |
| 1111 | 1055 | } |
| 1112 | 1056 | |
| 1057 | +static int armv8_a73_pmu_init(struct arm_pmu *cpu_pmu) | |
| 1058 | +{ | |
| 1059 | + int ret = armv8_pmu_init(cpu_pmu); | |
| 1060 | + if (ret) | |
| 1061 | + return ret; | |
| 1062 | + | |
| 1063 | + cpu_pmu->name = "armv8_cortex_a73"; | |
| 1064 | + cpu_pmu->map_event = armv8_a73_map_event; | |
| 1065 | + cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] = | |
| 1066 | + &armv8_pmuv3_events_attr_group; | |
| 1067 | + cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] = | |
| 1068 | + &armv8_pmuv3_format_attr_group; | |
| 1069 | + | |
| 1070 | + return 0; | |
| 1071 | +} | |
| 1072 | + | |
| 1113 | 1073 | static int armv8_thunder_pmu_init(struct arm_pmu *cpu_pmu) |
| 1114 | 1074 | { |
| 1115 | 1075 | int ret = armv8_pmu_init(cpu_pmu); |
| @@ -1144,9 +1104,11 @@ static int armv8_vulcan_pmu_init(struct arm_pmu *cpu_pmu) · 28 unmodified lines | ||
| 1144 | 1104 | |
| 1145 | 1105 | static const struct of_device_id armv8_pmu_of_device_ids[] = { |
| 1146 | 1106 | {.compatible = "arm,armv8-pmuv3", .data = armv8_pmuv3_init}, |
| 1107 | + {.compatible = "arm,cortex-a35-pmu", .data = armv8_a35_pmu_init}, | |
| 1147 | 1108 | {.compatible = "arm,cortex-a53-pmu", .data = armv8_a53_pmu_init}, |
| 1148 | 1109 | {.compatible = "arm,cortex-a57-pmu", .data = armv8_a57_pmu_init}, |
| 1149 | 1110 | {.compatible = "arm,cortex-a72-pmu", .data = armv8_a72_pmu_init}, |
| 1111 | + {.compatible = "arm,cortex-a73-pmu", .data = armv8_a73_pmu_init}, | |
| 1150 | 1112 | {.compatible = "cavium,thunder-pmu", .data = armv8_thunder_pmu_init}, |
| 1151 | 1113 | {.compatible = "brcm,vulcan-pmu", .data = armv8_vulcan_pmu_init}, |
| 1152 | 1114 | {}, |
| @@ -40,7 +40,7 @@ int arch_uprobe_analyze_insn(struct arch_uprobe *auprobe, struct mm_struct *mm, · 39 unmodified lines | ||
| 40 | 40 | probe_opcode_t insn; |
| 41 | 41 | |
| 42 | 42 | /* TODO: Currently we do not support AARCH32 instruction probing */ |
| 43 | − if (test_bit(TIF_32BIT, &mm->context.flags)) | |
| 43 | + if (mm->context.flags & MMCF_AARCH32) | |
| 44 | 44 | return -ENOTSUPP; |
| 45 | 45 | else if (!IS_ALIGNED(addr, AARCH64_INSN_SIZE)) |
| 46 | 46 | return -EINVAL; |
| @@ -384,15 +384,12 @@ unsigned long get_wchan(struct task_struct *p) · 383 unmodified lines | ||
| 384 | 384 | return 0; |
| 385 | 385 | |
| 386 | 386 | frame.fp = thread_saved_fp(p); |
| 387 | − frame.sp = thread_saved_sp(p); | |
| 388 | 387 | frame.pc = thread_saved_pc(p); |
| 389 | 388 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER |
| 390 | 389 | frame.graph = p->curr_ret_stack; |
| 391 | 390 | #endif |
| 392 | 391 | do { |
| 393 | − if (frame.sp < stack_page || | |
| 394 | − frame.sp >= stack_page + THREAD_SIZE || | |
| 395 | − unwind_frame(p, &frame)) | |
| 392 | + if (unwind_frame(p, &frame)) | |
| 396 | 393 | goto out; |
| 397 | 394 | if (!in_sched_functions(frame.pc)) { |
| 398 | 395 | ret = frame.pc; |
| @@ -419,3 +416,11 @@ unsigned long arch_randomize_brk(struct mm_struct *mm) · 20 unmodified lines | ||
| 419 | 416 | else |
| 420 | 417 | return randomize_page(mm->brk, SZ_1G); |
| 421 | 418 | } |
| 419 | + | |
| 420 | +/* | |
| 421 | + * Called from setup_new_exec() after (COMPAT_)SET_PERSONALITY. | |
| 422 | + */ | |
| 423 | +void arch_setup_new_exec(void) | |
| 424 | +{ | |
| 425 | + current->mm->context.flags = is_compat_task() ? MMCF_AARCH32 : 0; | |
| 426 | +} | |
| @@ -42,6 +42,7 @@ · 41 unmodified lines | ||
| 42 | 42 | #include <asm/compat.h> |
| 43 | 43 | #include <asm/debug-monitors.h> |
| 44 | 44 | #include <asm/pgtable.h> |
| 45 | +#include <asm/stacktrace.h> | |
| 45 | 46 | #include <asm/syscall.h> |
| 46 | 47 | #include <asm/traps.h> |
| 47 | 48 | #include <asm/system_misc.h> |
| @@ -127,7 +128,7 @@ static bool regs_within_kernel_stack(struct pt_regs *regs, unsigned long addr) · 79 unmodified lines | ||
| 127 | 128 | { |
| 128 | 129 | return ((addr & ~(THREAD_SIZE - 1)) == |
| 129 | 130 | (kernel_stack_pointer(regs) & ~(THREAD_SIZE - 1))) || |
| 130 | − on_irq_stack(addr, raw_smp_processor_id()); | |
| 131 | + on_irq_stack(addr); | |
| 131 | 132 | } |
| 132 | 133 | |
| 133 | 134 | /** |
| @@ -1363,7 +1364,7 @@ static void tracehook_report_syscall(struct pt_regs *regs, · 1229 unmodified lines | ||
| 1363 | 1364 | if (dir == PTRACE_SYSCALL_EXIT) |
| 1364 | 1365 | tracehook_report_syscall_exit(regs, 0); |
| 1365 | 1366 | else if (tracehook_report_syscall_entry(regs)) |
| 1366 | − regs->syscallno = ~0UL; | |
| 1367 | + forget_syscall(regs); | |
| 1367 | 1368 | |
| 1368 | 1369 | regs->regs[regno] = saved_reg; |
| 1369 | 1370 | } |
| @@ -42,7 +42,6 @@ void *return_address(unsigned int level) · 41 unmodified lines | ||
| 42 | 42 | data.addr = NULL; |
| 43 | 43 | |
| 44 | 44 | frame.fp = (unsigned long)__builtin_frame_address(0); |
| 45 | − frame.sp = current_stack_pointer; | |
| 46 | 45 | frame.pc = (unsigned long)return_address; /* dummy */ |
| 47 | 46 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER |
| 48 | 47 | frame.graph = current->curr_ret_stack; |
| @@ -37,6 +37,7 @@ · 36 unmodified lines | ||
| 37 | 37 | #include <asm/ucontext.h> |
| 38 | 38 | #include <asm/unistd.h> |
| 39 | 39 | #include <asm/fpsimd.h> |
| 40 | +#include <asm/ptrace.h> | |
| 40 | 41 | #include <asm/signal32.h> |
| 41 | 42 | #include <asm/vdso.h> |
| 42 | 43 | |
| @@ -388,7 +389,7 @@ static int restore_sigframe(struct pt_regs *regs, · 345 unmodified lines | ||
| 388 | 389 | /* |
| 389 | 390 | * Avoid sys_rt_sigreturn() restarting. |
| 390 | 391 | */ |
| 391 | − regs->syscallno = ~0UL; | |
| 392 | + forget_syscall(regs); | |
| 392 | 393 | |
| 393 | 394 | err |= !valid_user_regs(®s->user_regs, current); |
| 394 | 395 | if (err == 0) |
| @@ -674,13 +675,12 @@ static void do_signal(struct pt_regs *regs) · 279 unmodified lines | ||
| 674 | 675 | { |
| 675 | 676 | unsigned long continue_addr = 0, restart_addr = 0; |
| 676 | 677 | int retval = 0; |
| 677 | − int syscall = (int)regs->syscallno; | |
| 678 | 678 | struct ksignal ksig; |
| 679 | 679 | |
| 680 | 680 | /* |
| 681 | 681 | * If we were from a system call, check for system call restarting... |
| 682 | 682 | */ |
| 683 | − if (syscall >= 0) { | |
| 683 | + if (in_syscall(regs)) { | |
| 684 | 684 | continue_addr = regs->pc; |
| 685 | 685 | restart_addr = continue_addr - (compat_thumb_mode(regs) ? 2 : 4); |
| 686 | 686 | retval = regs->regs[0]; |
| @@ -688,7 +688,7 @@ static void do_signal(struct pt_regs *regs) · 1 unmodified line | ||
| 688 | 688 | /* |
| 689 | 689 | * Avoid additional syscall restarting via ret_to_user. |
| 690 | 690 | */ |
| 691 | − regs->syscallno = ~0UL; | |
| 691 | + forget_syscall(regs); | |
| 692 | 692 | |
| 693 | 693 | /* |
| 694 | 694 | * Prepare for system call restart. We do this here so that a |
| @@ -732,7 +732,7 @@ static void do_signal(struct pt_regs *regs) · 37 unmodified lines | ||
| 732 | 732 | * Handle restarting a different system call. As above, if a debugger |
| 733 | 733 | * has chosen to restart at a different PC, ignore the restart. |
| 734 | 734 | */ |
| 735 | − if (syscall >= 0 && regs->pc == restart_addr) { | |
| 735 | + if (in_syscall(regs) && regs->pc == restart_addr) { | |
| 736 | 736 | if (retval == -ERESTART_RESTARTBLOCK) |
| 737 | 737 | setup_restart_syscall(regs); |
| 738 | 738 | user_rewind_single_step(current); |
| @@ -354,7 +354,7 @@ static int compat_restore_sigframe(struct pt_regs *regs, · 353 unmodified lines | ||
| 354 | 354 | /* |
| 355 | 355 | * Avoid compat_sys_sigreturn() restarting. |
| 356 | 356 | */ |
| 357 | − regs->syscallno = ~0UL; | |
| 357 | + forget_syscall(regs); | |
| 358 | 358 | |
| 359 | 359 | err |= !valid_user_regs(®s->user_regs, current); |
| 360 | 360 | |
| @@ -154,7 +154,7 @@ int __cpu_up(unsigned int cpu, struct task_struct *idle) · 153 unmodified lines | ||
| 154 | 154 | * page tables. |
| 155 | 155 | */ |
| 156 | 156 | secondary_data.task = idle; |
| 157 | − secondary_data.stack = task_stack_page(idle) + THREAD_START_SP; | |
| 157 | + secondary_data.stack = task_stack_page(idle) + THREAD_SIZE; | |
| 158 | 158 | update_cpu_boot_status(CPU_MMU_OFF); |
| 159 | 159 | __flush_dcache_area(&secondary_data, sizeof(secondary_data)); |
| 160 | 160 | |
| @@ -977,11 +977,21 @@ void smp_send_stop(void) · 816 unmodified lines | ||
| 977 | 977 | } |
| 978 | 978 | |
| 979 | 979 | #ifdef CONFIG_KEXEC_CORE |
| 980 | −void smp_send_crash_stop(void) | |
| 980 | +void crash_smp_send_stop(void) | |
| 981 | 981 | { |
| 982 | + static int cpus_stopped; | |
| 982 | 983 | cpumask_t mask; |
| 983 | 984 | unsigned long timeout; |
| 984 | 985 | |
| 986 | + /* | |
| 987 | + * This function can be called twice in panic path, but obviously | |
| 988 | + * we execute this only once. | |
| 989 | + */ | |
| 990 | + if (cpus_stopped) | |
| 991 | + return; | |
| 992 | + | |
| 993 | + cpus_stopped = 1; | |
| 994 | + | |
| 985 | 995 | if (num_online_cpus() == 1) |
| 986 | 996 | return; |
| 987 | 997 | |
| @@ -42,33 +42,17 @@ · 41 unmodified lines | ||
| 42 | 42 | */ |
| 43 | 43 | int notrace unwind_frame(struct task_struct *tsk, struct stackframe *frame) |
| 44 | 44 | { |
| 45 | − unsigned long high, low; | |
| 46 | 45 | unsigned long fp = frame->fp; |
| 47 | − unsigned long irq_stack_ptr; | |
| 46 | + | |
| 47 | + if (fp & 0xf) | |
| 48 | + return -EINVAL; | |
| 48 | 49 | |
| 49 | 50 | if (!tsk) |
| 50 | 51 | tsk = current; |
| 51 | 52 | |
| 52 | − /* | |
| 53 | − * Switching between stacks is valid when tracing current and in | |
| 54 | − * non-preemptible context. | |
| 55 | − */ | |
| 56 | − if (tsk == current && !preemptible()) | |
| 57 | − irq_stack_ptr = IRQ_STACK_PTR(smp_processor_id()); | |
| 58 | − else | |
| 59 | − irq_stack_ptr = 0; | |
| 60 | − | |
| 61 | − low = frame->sp; | |
| 62 | − /* irq stacks are not THREAD_SIZE aligned */ | |
| 63 | − if (on_irq_stack(frame->sp, raw_smp_processor_id())) | |
| 64 | − high = irq_stack_ptr; | |
| 65 | − else | |
| 66 | − high = ALIGN(low, THREAD_SIZE) - 0x20; | |
| 67 | − | |
| 68 | − if (fp < low || fp > high || fp & 0xf) | |
| 53 | + if (!on_accessible_stack(tsk, fp)) | |
| 69 | 54 | return -EINVAL; |
| 70 | 55 | |
| 71 | − frame->sp = fp + 0x10; | |
| 72 | 56 | frame->fp = READ_ONCE_NOCHECK(*(unsigned long *)(fp)); |
| 73 | 57 | frame->pc = READ_ONCE_NOCHECK(*(unsigned long *)(fp + 8)); |
| 74 | 58 | |
| @@ -86,34 +70,13 @@ int notrace unwind_frame(struct task_struct *tsk, struct stackframe *frame) · 11 unmodified lines | ||
| 86 | 70 | #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ |
| 87 | 71 | |
| 88 | 72 | /* |
| 89 | − * Check whether we are going to walk through from interrupt stack | |
| 90 | − * to task stack. | |
| 91 | − * If we reach the end of the stack - and its an interrupt stack, | |
| 92 | − * unpack the dummy frame to find the original elr. | |
| 93 | − * | |
| 94 | − * Check the frame->fp we read from the bottom of the irq_stack, | |
| 95 | − * and the original task stack pointer are both in current->stack. | |
| 73 | + * Frames created upon entry from EL0 have NULL FP and PC values, so | |
| 74 | + * don't bother reporting these. Frames created by __noreturn functions | |
| 75 | + * might have a valid FP even if PC is bogus, so only terminate where | |
| 76 | + * both are NULL. | |
| 96 | 77 | */ |
| 97 | − if (frame->sp == irq_stack_ptr) { | |
| 98 | − struct pt_regs *irq_args; | |
| 99 | − unsigned long orig_sp = IRQ_STACK_TO_TASK_STACK(irq_stack_ptr); | |
| 100 | − | |
| 101 | − if (object_is_on_stack((void *)orig_sp) && | |
| 102 | − object_is_on_stack((void *)frame->fp)) { | |
| 103 | − frame->sp = orig_sp; | |
| 104 | − | |
| 105 | − /* orig_sp is the saved pt_regs, find the elr */ | |
| 106 | − irq_args = (struct pt_regs *)orig_sp; | |
| 107 | − frame->pc = irq_args->pc; | |
| 108 | − } else { | |
| 109 | − /* | |
| 110 | − * This frame has a non-standard format, and we | |
| 111 | − * didn't fix it, because the data looked wrong. | |
| 112 | − * Refuse to output this frame. | |
| 113 | − */ | |
| 114 | − return -EINVAL; | |
| 115 | − } | |
| 116 | − } | |
| 78 | + if (!frame->fp && !frame->pc) | |
| 79 | + return -EINVAL; | |
| 117 | 80 | |
| 118 | 81 | return 0; |
| 119 | 82 | } |
| @@ -167,7 +130,6 @@ void save_stack_trace_regs(struct pt_regs *regs, struct stack_trace *trace) · 47 unmodified lines | ||
| 167 | 130 | data.no_sched_functions = 0; |
| 168 | 131 | |
| 169 | 132 | frame.fp = regs->regs[29]; |
| 170 | − frame.sp = regs->sp; | |
| 171 | 133 | frame.pc = regs->pc; |
| 172 | 134 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER |
| 173 | 135 | frame.graph = current->curr_ret_stack; |
| @@ -192,12 +154,10 @@ void save_stack_trace_tsk(struct task_struct *tsk, struct stack_trace *trace) · 18 unmodified lines | ||
| 192 | 154 | if (tsk != current) { |
| 193 | 155 | data.no_sched_functions = 1; |
| 194 | 156 | frame.fp = thread_saved_fp(tsk); |
| 195 | − frame.sp = thread_saved_sp(tsk); | |
| 196 | 157 | frame.pc = thread_saved_pc(tsk); |
| 197 | 158 | } else { |
| 198 | 159 | data.no_sched_functions = 0; |
| 199 | 160 | frame.fp = (unsigned long)__builtin_frame_address(0); |
| 200 | − frame.sp = current_stack_pointer; | |
| 201 | 161 | frame.pc = (unsigned long)save_stack_trace_tsk; |
| 202 | 162 | } |
| 203 | 163 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER |
| @@ -50,7 +50,6 @@ unsigned long profile_pc(struct pt_regs *regs) · 49 unmodified lines | ||
| 50 | 50 | return regs->pc; |
| 51 | 51 | |
| 52 | 52 | frame.fp = regs->regs[29]; |
| 53 | − frame.sp = regs->sp; | |
| 54 | 53 | frame.pc = regs->pc; |
| 55 | 54 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER |
| 56 | 55 | frame.graph = -1; /* no task info */ |
| @@ -32,6 +32,7 @@ · 31 unmodified lines | ||
| 32 | 32 | #include <linux/sched/signal.h> |
| 33 | 33 | #include <linux/sched/debug.h> |
| 34 | 34 | #include <linux/sched/task_stack.h> |
| 35 | +#include <linux/sizes.h> | |
| 35 | 36 | #include <linux/syscalls.h> |
| 36 | 37 | #include <linux/mm_types.h> |
| 37 | 38 | |
| @@ -41,6 +42,7 @@ · 3 unmodified lines | ||
| 41 | 42 | #include <asm/esr.h> |
| 42 | 43 | #include <asm/insn.h> |
| 43 | 44 | #include <asm/traps.h> |
| 45 | +#include <asm/smp.h> | |
| 44 | 46 | #include <asm/stack_pointer.h> |
| 45 | 47 | #include <asm/stacktrace.h> |
| 46 | 48 | #include <asm/exception.h> |
| @@ -143,7 +145,6 @@ static void dump_instr(const char *lvl, struct pt_regs *regs) · 96 unmodified lines | ||
| 143 | 145 | void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk) |
| 144 | 146 | { |
| 145 | 147 | struct stackframe frame; |
| 146 | − unsigned long irq_stack_ptr; | |
| 147 | 148 | int skip; |
| 148 | 149 | |
| 149 | 150 | pr_debug("%s(regs = %p tsk = %p)\n", __func__, regs, tsk); |
| @@ -154,25 +155,14 @@ void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk) · 4 unmodified lines | ||
| 154 | 155 | if (!try_get_task_stack(tsk)) |
| 155 | 156 | return; |
| 156 | 157 | |
| 157 | − /* | |
| 158 | − * Switching between stacks is valid when tracing current and in | |
| 159 | − * non-preemptible context. | |
| 160 | − */ | |
| 161 | − if (tsk == current && !preemptible()) | |
| 162 | − irq_stack_ptr = IRQ_STACK_PTR(smp_processor_id()); | |
| 163 | − else | |
| 164 | − irq_stack_ptr = 0; | |
| 165 | − | |
| 166 | 158 | if (tsk == current) { |
| 167 | 159 | frame.fp = (unsigned long)__builtin_frame_address(0); |
| 168 | − frame.sp = current_stack_pointer; | |
| 169 | 160 | frame.pc = (unsigned long)dump_backtrace; |
| 170 | 161 | } else { |
| 171 | 162 | /* |
| 172 | 163 | * task blocked in __switch_to |
| 173 | 164 | */ |
| 174 | 165 | frame.fp = thread_saved_fp(tsk); |
| 175 | − frame.sp = thread_saved_sp(tsk); | |
| 176 | 166 | frame.pc = thread_saved_pc(tsk); |
| 177 | 167 | } |
| 178 | 168 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER |
| @@ -182,13 +172,12 @@ void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk) · 3 unmodified lines | ||
| 182 | 172 | skip = !!regs; |
| 183 | 173 | printk("Call trace:\n"); |
| 184 | 174 | while (1) { |
| 185 | − unsigned long where = frame.pc; | |
| 186 | 175 | unsigned long stack; |
| 187 | 176 | int ret; |
| 188 | 177 | |
| 189 | 178 | /* skip until specified stack frame */ |
| 190 | 179 | if (!skip) { |
| 191 | − dump_backtrace_entry(where); | |
| 180 | + dump_backtrace_entry(frame.pc); | |
| 192 | 181 | } else if (frame.fp == regs->regs[29]) { |
| 193 | 182 | skip = 0; |
| 194 | 183 | /* |
| @@ -203,20 +192,12 @@ void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk) · 8 unmodified lines | ||
| 203 | 192 | ret = unwind_frame(tsk, &frame); |
| 204 | 193 | if (ret < 0) |
| 205 | 194 | break; |
| 206 | − stack = frame.sp; | |
| 207 | − if (in_exception_text(where)) { | |
| 208 | − /* | |
| 209 | − * If we switched to the irq_stack before calling this | |
| 210 | − * exception handler, then the pt_regs will be on the | |
| 211 | − * task stack. The easiest way to tell is if the large | |
| 212 | − * pt_regs would overlap with the end of the irq_stack. | |
| 213 | − */ | |
| 214 | − if (stack < irq_stack_ptr && | |
| 215 | − (stack + sizeof(struct pt_regs)) > irq_stack_ptr) | |
| 216 | − stack = IRQ_STACK_TO_TASK_STACK(irq_stack_ptr); | |
| 195 | + if (in_entry_text(frame.pc)) { | |
| 196 | + stack = frame.fp - offsetof(struct pt_regs, stackframe); | |
| 217 | 197 | |
| 218 | − dump_mem("", "Exception stack", stack, | |
| 219 | − stack + sizeof(struct pt_regs)); | |
| 198 | + if (on_accessible_stack(tsk, stack)) | |
| 199 | + dump_mem("", "Exception stack", stack, | |
| 200 | + stack + sizeof(struct pt_regs)); | |
| 220 | 201 | } |
| 221 | 202 | } |
| 222 | 203 | |
| @@ -257,8 +238,6 @@ static int __die(const char *str, int err, struct pt_regs *regs) · 34 unmodified lines | ||
| 257 | 238 | end_of_stack(tsk)); |
| 258 | 239 | |
| 259 | 240 | if (!user_mode(regs)) { |
| 260 | − dump_mem(KERN_EMERG, "Stack: ", regs->sp, | |
| 261 | − THREAD_SIZE + (unsigned long)task_stack_page(tsk)); | |
| 262 | 241 | dump_backtrace(regs, tsk); |
| 263 | 242 | dump_instr(KERN_EMERG, regs); |
| 264 | 243 | } |
| @@ -484,6 +463,9 @@ static void user_cache_maint_handler(unsigned int esr, struct pt_regs *regs) · 219 unmodified lines | ||
| 484 | 463 | case ESR_ELx_SYS64_ISS_CRM_DC_CVAC: /* DC CVAC, gets promoted */ |
| 485 | 464 | __user_cache_maint("dc civac", address, ret); |
| 486 | 465 | break; |
| 466 | + case ESR_ELx_SYS64_ISS_CRM_DC_CVAP: /* DC CVAP */ | |
| 467 | + __user_cache_maint("sys 3, c7, c12, 1", address, ret); | |
| 468 | + break; | |
| 487 | 469 | case ESR_ELx_SYS64_ISS_CRM_DC_CIVAC: /* DC CIVAC */ |
| 488 | 470 | __user_cache_maint("dc civac", address, ret); |
| 489 | 471 | break; |
| @@ -593,7 +575,7 @@ asmlinkage long do_ni_syscall(struct pt_regs *regs) · 103 unmodified lines | ||
| 593 | 575 | |
| 594 | 576 | if (show_unhandled_signals_ratelimited()) { |
| 595 | 577 | pr_info("%s[%d]: syscall %d\n", current->comm, |
| 596 | − task_pid_nr(current), (int)regs->syscallno); | |
| 578 | + task_pid_nr(current), regs->syscallno); | |
| 597 | 579 | dump_instr("", regs); |
| 598 | 580 | if (user_mode(regs)) |
| 599 | 581 | __show_regs(regs); |
| @@ -689,6 +671,43 @@ asmlinkage void bad_el0_sync(struct pt_regs *regs, int reason, unsigned int esr) · 89 unmodified lines | ||
| 689 | 671 | force_sig_info(info.si_signo, &info, current); |
| 690 | 672 | } |
| 691 | 673 | |
| 674 | +#ifdef CONFIG_VMAP_STACK | |
| 675 | + | |
| 676 | +DEFINE_PER_CPU(unsigned long [OVERFLOW_STACK_SIZE/sizeof(long)], overflow_stack) | |
| 677 | + __aligned(16); | |
| 678 | + | |
| 679 | +asmlinkage void handle_bad_stack(struct pt_regs *regs) | |
| 680 | +{ | |
| 681 | + unsigned long tsk_stk = (unsigned long)current->stack; | |
| 682 | + unsigned long irq_stk = (unsigned long)this_cpu_read(irq_stack_ptr); | |
| 683 | + unsigned long ovf_stk = (unsigned long)this_cpu_ptr(overflow_stack); | |
| 684 | + unsigned int esr = read_sysreg(esr_el1); | |
| 685 | + unsigned long far = read_sysreg(far_el1); | |
| 686 | + | |
| 687 | + console_verbose(); | |
| 688 | + pr_emerg("Insufficient stack space to handle exception!"); | |
| 689 | + | |
| 690 | + pr_emerg("ESR: 0x%08x -- %s\n", esr, esr_get_class_string(esr)); | |
| 691 | + pr_emerg("FAR: 0x%016lx\n", far); | |
| 692 | + | |
| 693 | + pr_emerg("Task stack: [0x%016lx..0x%016lx]\n", | |
| 694 | + tsk_stk, tsk_stk + THREAD_SIZE); | |
| 695 | + pr_emerg("IRQ stack: [0x%016lx..0x%016lx]\n", | |
| 696 | + irq_stk, irq_stk + THREAD_SIZE); | |
| 697 | + pr_emerg("Overflow stack: [0x%016lx..0x%016lx]\n", | |
| 698 | + ovf_stk, ovf_stk + OVERFLOW_STACK_SIZE); | |
| 699 | + | |
| 700 | + __show_regs(regs); | |
| 701 | + | |
| 702 | + /* | |
| 703 | + * We use nmi_panic to limit the potential for recusive overflows, and | |
| 704 | + * to get a better stack trace. | |
| 705 | + */ | |
| 706 | + nmi_panic(NULL, "kernel stack overflow"); | |
| 707 | + cpu_park_loop(); | |
| 708 | +} | |
| 709 | +#endif | |
| 710 | + | |
| 692 | 711 | void __pte_error(const char *file, int line, unsigned long val) |
| 693 | 712 | { |
| 694 | 713 | pr_err("%s:%d: bad pte %016lx.\n", file, line, val); |
| @@ -110,12 +110,27 @@ int aarch32_setup_vectors_page(struct linux_binprm *bprm, int uses_interp) · 109 unmodified lines | ||
| 110 | 110 | } |
| 111 | 111 | #endif /* CONFIG_COMPAT */ |
| 112 | 112 | |
| 113 | +static int vdso_mremap(const struct vm_special_mapping *sm, | |
| 114 | + struct vm_area_struct *new_vma) | |
| 115 | +{ | |
| 116 | + unsigned long new_size = new_vma->vm_end - new_vma->vm_start; | |
| 117 | + unsigned long vdso_size = vdso_end - vdso_start; | |
| 118 | + | |
| 119 | + if (vdso_size != new_size) | |
| 120 | + return -EINVAL; | |
| 121 | + | |
| 122 | + current->mm->context.vdso = (void *)new_vma->vm_start; | |
| 123 | + | |
| 124 | + return 0; | |
| 125 | +} | |
| 126 | + | |
| 113 | 127 | static struct vm_special_mapping vdso_spec[2] __ro_after_init = { |
| 114 | 128 | { |
| 115 | 129 | .name = "[vvar]", |
| 116 | 130 | }, |
| 117 | 131 | { |
| 118 | 132 | .name = "[vdso]", |
| 133 | + .mremap = vdso_mremap, | |
| 119 | 134 | }, |
| 120 | 135 | }; |
| 121 | 136 | |
| @@ -72,22 +72,6 @@ PECOFF_FILE_ALIGNMENT = 0x200; · 71 unmodified lines | ||
| 72 | 72 | #define PECOFF_EDATA_PADDING |
| 73 | 73 | #endif |
| 74 | 74 | |
| 75 | −#if defined(CONFIG_DEBUG_ALIGN_RODATA) | |
| 76 | −/* | |
| 77 | − * 4 KB granule: 1 level 2 entry | |
| 78 | − * 16 KB granule: 128 level 3 entries, with contiguous bit | |
| 79 | − * 64 KB granule: 32 level 3 entries, with contiguous bit | |
| 80 | − */ | |
| 81 | −#define SEGMENT_ALIGN SZ_2M | |
| 82 | −#else | |
| 83 | −/* | |
| 84 | − * 4 KB granule: 16 level 3 entries, with contiguous bit | |
| 85 | − * 16 KB granule: 4 level 3 entries, without contiguous bit | |
| 86 | − * 64 KB granule: 1 level 3 entry | |
| 87 | − */ | |
| 88 | −#define SEGMENT_ALIGN SZ_64K | |
| 89 | −#endif | |
| 90 | − | |
| 91 | 75 | SECTIONS |
| 92 | 76 | { |
| 93 | 77 | /* |
| @@ -192,7 +176,7 @@ SECTIONS · 98 unmodified lines | ||
| 192 | 176 | |
| 193 | 177 | _data = .; |
| 194 | 178 | _sdata = .; |
| 195 | − RW_DATA_SECTION(L1_CACHE_BYTES, PAGE_SIZE, THREAD_SIZE) | |
| 179 | + RW_DATA_SECTION(L1_CACHE_BYTES, PAGE_SIZE, THREAD_ALIGN) | |
| 196 | 180 | |
| 197 | 181 | /* |
| 198 | 182 | * Data written with the MMU off but read with the MMU on requires |
| @@ -70,7 +70,7 @@ u32 __hyp_text __init_stage2_translation(void) · 69 unmodified lines | ||
| 70 | 70 | * Management in ID_AA64MMFR1_EL1 and enable the feature in VTCR_EL2. |
| 71 | 71 | */ |
| 72 | 72 | tmp = (read_sysreg(id_aa64mmfr1_el1) >> ID_AA64MMFR1_HADBS_SHIFT) & 0xf; |
| 73 | − if (IS_ENABLED(CONFIG_ARM64_HW_AFDBM) && tmp) | |
| 73 | + if (tmp) | |
| 74 | 74 | val |= VTCR_EL2_HA; |
| 75 | 75 | |
| 76 | 76 | /* |
| @@ -17,3 +17,5 @@ CFLAGS_atomic_ll_sc.o := -fcall-used-x0 -ffixed-x1 -ffixed-x2 \ · 16 unmodified lines | ||
| 17 | 17 | -fcall-saved-x10 -fcall-saved-x11 -fcall-saved-x12 \ |
| 18 | 18 | -fcall-saved-x13 -fcall-saved-x14 -fcall-saved-x15 \ |
| 19 | 19 | -fcall-saved-x18 |
| 20 | + | |
| 21 | +lib-$(CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE) += uaccess_flushcache.o | |
| @@ -0,0 +1,47 @@ | ||
| 1 | +/* | |
| 2 | + * Copyright (C) 2017 ARM Ltd. | |
| 3 | + * | |
| 4 | + * This program is free software; you can redistribute it and/or modify | |
| 5 | + * it under the terms of the GNU General Public License version 2 as | |
| 6 | + * published by the Free Software Foundation. | |
| 7 | + * | |
| 8 | + * This program is distributed in the hope that it will be useful, | |
| 9 | + * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
| 10 | + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
| 11 | + * GNU General Public License for more details. | |
| 12 | + * | |
| 13 | + * You should have received a copy of the GNU General Public License | |
| 14 | + * along with this program. If not, see <http://www.gnu.org/licenses/>. | |
| 15 | + */ | |
| 16 | + | |
| 17 | +#include <linux/uaccess.h> | |
| 18 | +#include <asm/barrier.h> | |
| 19 | +#include <asm/cacheflush.h> | |
| 20 | + | |
| 21 | +void memcpy_flushcache(void *dst, const void *src, size_t cnt) | |
| 22 | +{ | |
| 23 | + /* | |
| 24 | + * We assume this should not be called with @dst pointing to | |
| 25 | + * non-cacheable memory, such that we don't need an explicit | |
| 26 | + * barrier to order the cache maintenance against the memcpy. | |
| 27 | + */ | |
| 28 | + memcpy(dst, src, cnt); | |
| 29 | + __clean_dcache_area_pop(dst, cnt); | |
| 30 | +} | |
| 31 | +EXPORT_SYMBOL_GPL(memcpy_flushcache); | |
| 32 | + | |
| 33 | +void memcpy_page_flushcache(char *to, struct page *page, size_t offset, | |
| 34 | + size_t len) | |
| 35 | +{ | |
| 36 | + memcpy_flushcache(to, page_address(page) + offset, len); | |
| 37 | +} | |
| 38 | + | |
| 39 | +unsigned long __copy_user_flushcache(void *to, const void __user *from, | |
| 40 | + unsigned long n) | |
| 41 | +{ | |
| 42 | + unsigned long rc = __arch_copy_from_user(to, from, n); | |
| 43 | + | |
| 44 | + /* See above */ | |
| 45 | + __clean_dcache_area_pop(to, n - rc); | |
| 46 | + return rc; | |
| 47 | +} | |
| @@ -109,20 +109,25 @@ ENTRY(__clean_dcache_area_pou) · 108 unmodified lines | ||
| 109 | 109 | ENDPROC(__clean_dcache_area_pou) |
| 110 | 110 | |
| 111 | 111 | /* |
| 112 | − * __dma_inv_area(start, size) | |
| 113 | − * - start - virtual start address of region | |
| 112 | + * __inval_dcache_area(kaddr, size) | |
| 113 | + * | |
| 114 | + * Ensure that any D-cache lines for the interval [kaddr, kaddr+size) | |
| 115 | + * are invalidated. Any partial lines at the ends of the interval are | |
| 116 | + * also cleaned to PoC to prevent data loss. | |
| 117 | + * | |
| 118 | + * - kaddr - kernel address | |
| 114 | 119 | * - size - size in question |
| 115 | 120 | */ |
| 116 | −__dma_inv_area: | |
| 117 | − add x1, x1, x0 | |
| 121 | +ENTRY(__inval_dcache_area) | |
| 118 | 122 | /* FALLTHROUGH */ |
| 119 | 123 | |
| 120 | 124 | /* |
| 121 | − * __inval_cache_range(start, end) | |
| 122 | − * - start - start address of region | |
| 123 | − * - end - end address of region | |
| 125 | + * __dma_inv_area(start, size) | |
| 126 | + * - start - virtual start address of region | |
| 127 | + * - size - size in question | |
| 124 | 128 | */ |
| 125 | −ENTRY(__inval_cache_range) | |
| 129 | +__dma_inv_area: | |
| 130 | + add x1, x1, x0 | |
| 126 | 131 | dcache_line_size x2, x3 |
| 127 | 132 | sub x3, x2, #1 |
| 128 | 133 | tst x1, x3 // end cache line aligned? |
| @@ -140,7 +145,7 @@ ENTRY(__inval_cache_range) · 11 unmodified lines | ||
| 140 | 145 | b.lo 2b |
| 141 | 146 | dsb sy |
| 142 | 147 | ret |
| 143 | −ENDPIPROC(__inval_cache_range) | |
| 148 | +ENDPIPROC(__inval_dcache_area) | |
| 144 | 149 | ENDPROC(__dma_inv_area) |
| 145 | 150 | |
| 146 | 151 | /* |
| @@ -166,6 +171,20 @@ __dma_clean_area: · 19 unmodified lines | ||
| 166 | 171 | ENDPIPROC(__clean_dcache_area_poc) |
| 167 | 172 | ENDPROC(__dma_clean_area) |
| 168 | 173 | |
| 174 | +/* | |
| 175 | + * __clean_dcache_area_pop(kaddr, size) | |
| 176 | + * | |
| 177 | + * Ensure that any D-cache lines for the interval [kaddr, kaddr+size) | |
| 178 | + * are cleaned to the PoP. | |
| 179 | + * | |
| 180 | + * - kaddr - kernel address | |
| 181 | + * - size - size in question | |
| 182 | + */ | |
| 183 | +ENTRY(__clean_dcache_area_pop) | |
| 184 | + dcache_by_line_op cvap, sy, x0, x1, x2, x3 | |
| 185 | + ret | |
| 186 | +ENDPIPROC(__clean_dcache_area_pop) | |
| 187 | + | |
| 169 | 188 | /* |
| 170 | 189 | * __dma_flush_area(start, size) |
| 171 | 190 | * |
| @@ -42,7 +42,7 @@ static pgprot_t __get_dma_pgprot(unsigned long attrs, pgprot_t prot, · 41 unmodified lines | ||
| 42 | 42 | return prot; |
| 43 | 43 | } |
| 44 | 44 | |
| 45 | −static struct gen_pool *atomic_pool; | |
| 45 | +static struct gen_pool *atomic_pool __ro_after_init; | |
| 46 | 46 | |
| 47 | 47 | #define DEFAULT_DMA_COHERENT_POOL_SIZE SZ_256K |
| 48 | 48 | static size_t atomic_pool_size __initdata = DEFAULT_DMA_COHERENT_POOL_SIZE; |
| @@ -425,7 +425,7 @@ static int __init atomic_pool_init(void) · 376 unmodified lines | ||
| 425 | 425 | |
| 426 | 426 | gen_pool_set_algo(atomic_pool, |
| 427 | 427 | gen_pool_first_fit_order_align, |
| 428 | − (void *)PAGE_SHIFT); | |
| 428 | + NULL); | |
| 429 | 429 | |
| 430 | 430 | pr_info("DMA: preallocated %zu KiB pool for atomic allocations\n", |
| 431 | 431 | atomic_pool_size / 1024); |
| @@ -34,6 +34,7 @@ · 33 unmodified lines | ||
| 34 | 34 | #include <linux/hugetlb.h> |
| 35 | 35 | |
| 36 | 36 | #include <asm/bug.h> |
| 37 | +#include <asm/cmpxchg.h> | |
| 37 | 38 | #include <asm/cpufeature.h> |
| 38 | 39 | #include <asm/exception.h> |
| 39 | 40 | #include <asm/debug-monitors.h> |
| @@ -82,6 +83,49 @@ static inline int notify_page_fault(struct pt_regs *regs, unsigned int esr) · 42 unmodified lines | ||
| 82 | 83 | } |
| 83 | 84 | #endif |
| 84 | 85 | |
| 86 | +static void data_abort_decode(unsigned int esr) | |
| 87 | +{ | |
| 88 | + pr_alert("Data abort info:\n"); | |
| 89 | + | |
| 90 | + if (esr & ESR_ELx_ISV) { | |
| 91 | + pr_alert(" Access size = %u byte(s)\n", | |
| 92 | + 1U << ((esr & ESR_ELx_SAS) >> ESR_ELx_SAS_SHIFT)); | |
| 93 | + pr_alert(" SSE = %lu, SRT = %lu\n", | |
| 94 | + (esr & ESR_ELx_SSE) >> ESR_ELx_SSE_SHIFT, | |
| 95 | + (esr & ESR_ELx_SRT_MASK) >> ESR_ELx_SRT_SHIFT); | |
| 96 | + pr_alert(" SF = %lu, AR = %lu\n", | |
| 97 | + (esr & ESR_ELx_SF) >> ESR_ELx_SF_SHIFT, | |
| 98 | + (esr & ESR_ELx_AR) >> ESR_ELx_AR_SHIFT); | |
| 99 | + } else { | |
| 100 | + pr_alert(" ISV = 0, ISS = 0x%08lu\n", esr & ESR_ELx_ISS_MASK); | |
| 101 | + } | |
| 102 | + | |
| 103 | + pr_alert(" CM = %lu, WnR = %lu\n", | |
| 104 | + (esr & ESR_ELx_CM) >> ESR_ELx_CM_SHIFT, | |
| 105 | + (esr & ESR_ELx_WNR) >> ESR_ELx_WNR_SHIFT); | |
| 106 | +} | |
| 107 | + | |
| 108 | +/* | |
| 109 | + * Decode mem abort information | |
| 110 | + */ | |
| 111 | +static void mem_abort_decode(unsigned int esr) | |
| 112 | +{ | |
| 113 | + pr_alert("Mem abort info:\n"); | |
| 114 | + | |
| 115 | + pr_alert(" Exception class = %s, IL = %u bits\n", | |
| 116 | + esr_get_class_string(esr), | |
| 117 | + (esr & ESR_ELx_IL) ? 32 : 16); | |
| 118 | + pr_alert(" SET = %lu, FnV = %lu\n", | |
| 119 | + (esr & ESR_ELx_SET_MASK) >> ESR_ELx_SET_SHIFT, | |
| 120 | + (esr & ESR_ELx_FnV) >> ESR_ELx_FnV_SHIFT); | |
| 121 | + pr_alert(" EA = %lu, S1PTW = %lu\n", | |
| 122 | + (esr & ESR_ELx_EA) >> ESR_ELx_EA_SHIFT, | |
| 123 | + (esr & ESR_ELx_S1PTW) >> ESR_ELx_S1PTW_SHIFT); | |
| 124 | + | |
| 125 | + if (esr_is_data_abort(esr)) | |
| 126 | + data_abort_decode(esr); | |
| 127 | +} | |
| 128 | + | |
| 85 | 129 | /* |
| 86 | 130 | * Dump out the page tables associated with 'addr' in the currently active mm. |
| 87 | 131 | */ |
| @@ -139,7 +183,6 @@ void show_pte(unsigned long addr) · 51 unmodified lines | ||
| 139 | 183 | pr_cont("\n"); |
| 140 | 184 | } |
| 141 | 185 | |
| 142 | −#ifdef CONFIG_ARM64_HW_AFDBM | |
| 143 | 186 | /* |
| 144 | 187 | * This function sets the access flags (dirty, accessed), as well as write |
| 145 | 188 | * permission, and only to a more permissive setting. |
| @@ -154,18 +197,13 @@ int ptep_set_access_flags(struct vm_area_struct *vma, · 8 unmodified lines | ||
| 154 | 197 | unsigned long address, pte_t *ptep, |
| 155 | 198 | pte_t entry, int dirty) |
| 156 | 199 | { |
| 157 | − pteval_t old_pteval; | |
| 158 | − unsigned int tmp; | |
| 200 | + pteval_t old_pteval, pteval; | |
| 159 | 201 | |
| 160 | 202 | if (pte_same(*ptep, entry)) |
| 161 | 203 | return 0; |
| 162 | 204 | |
| 163 | 205 | /* only preserve the access flags and write permission */ |
| 164 | − pte_val(entry) &= PTE_AF | PTE_WRITE | PTE_DIRTY; | |
| 165 | − | |
| 166 | − /* set PTE_RDONLY if actual read-only or clean PTE */ | |
| 167 | − if (!pte_write(entry) || !pte_sw_dirty(entry)) | |
| 168 | − pte_val(entry) |= PTE_RDONLY; | |
| 206 | + pte_val(entry) &= PTE_RDONLY | PTE_AF | PTE_WRITE | PTE_DIRTY; | |
| 169 | 207 | |
| 170 | 208 | /* |
| 171 | 209 | * Setting the flags must be done atomically to avoid racing with the |
| @@ -174,21 +212,18 @@ int ptep_set_access_flags(struct vm_area_struct *vma, · 2 unmodified lines | ||
| 174 | 212 | * (calculated as: a & b == ~(~a | ~b)). |
| 175 | 213 | */ |
| 176 | 214 | pte_val(entry) ^= PTE_RDONLY; |
| 177 | − asm volatile("// ptep_set_access_flags\n" | |
| 178 | − " prfm pstl1strm, %2\n" | |
| 179 | − "1: ldxr %0, %2\n" | |
| 180 | − " eor %0, %0, %3 // negate PTE_RDONLY in *ptep\n" | |
| 181 | − " orr %0, %0, %4 // set flags\n" | |
| 182 | − " eor %0, %0, %3 // negate final PTE_RDONLY\n" | |
| 183 | − " stxr %w1, %0, %2\n" | |
| 184 | − " cbnz %w1, 1b\n" | |
| 185 | − : "=&r" (old_pteval), "=&r" (tmp), "+Q" (pte_val(*ptep)) | |
| 186 | − : "L" (PTE_RDONLY), "r" (pte_val(entry))); | |
| 215 | + pteval = READ_ONCE(pte_val(*ptep)); | |
| 216 | + do { | |
| 217 | + old_pteval = pteval; | |
| 218 | + pteval ^= PTE_RDONLY; | |
| 219 | + pteval |= pte_val(entry); | |
| 220 | + pteval ^= PTE_RDONLY; | |
| 221 | + pteval = cmpxchg_relaxed(&pte_val(*ptep), old_pteval, pteval); | |
| 222 | + } while (pteval != old_pteval); | |
| 187 | 223 | |
| 188 | 224 | flush_tlb_fix_spurious_fault(vma, address); |
| 189 | 225 | return 1; |
| 190 | 226 | } |
| 191 | −#endif | |
| 192 | 227 | |
| 193 | 228 | static bool is_el1_instruction_abort(unsigned int esr) |
| 194 | 229 | { |
| @@ -248,6 +283,8 @@ static void __do_kernel_fault(unsigned long addr, unsigned int esr, · 53 unmodified lines | ||
| 248 | 283 | pr_alert("Unable to handle kernel %s at virtual address %08lx\n", msg, |
| 249 | 284 | addr); |
| 250 | 285 | |
| 286 | + mem_abort_decode(esr); | |
| 287 | + | |
| 251 | 288 | show_pte(addr); |
| 252 | 289 | die("Oops", regs, esr); |
| 253 | 290 | bust_spinlocks(0); |
| @@ -705,6 +742,8 @@ asmlinkage void __exception do_mem_abort(unsigned long addr, unsigned int esr, · 451 unmodified lines | ||
| 705 | 742 | pr_alert("Unhandled fault: %s (0x%08x) at 0x%016lx\n", |
| 706 | 743 | inf->name, esr, addr); |
| 707 | 744 | |
| 745 | + mem_abort_decode(esr); | |
| 746 | + | |
| 708 | 747 | info.si_signo = inf->sig; |
| 709 | 748 | info.si_errno = 0; |
| 710 | 749 | info.si_code = inf->code; |
| @@ -83,3 +83,19 @@ EXPORT_SYMBOL(flush_dcache_page); · 82 unmodified lines | ||
| 83 | 83 | * Additional functions defined in assembly. |
| 84 | 84 | */ |
| 85 | 85 | EXPORT_SYMBOL(flush_icache_range); |
| 86 | + | |
| 87 | +#ifdef CONFIG_ARCH_HAS_PMEM_API | |
| 88 | +void arch_wb_cache_pmem(void *addr, size_t size) | |
| 89 | +{ | |
| 90 | + /* Ensure order against any prior non-cacheable writes */ | |
| 91 | + dmb(osh); | |
| 92 | + __clean_dcache_area_pop(addr, size); | |
| 93 | +} | |
| 94 | +EXPORT_SYMBOL_GPL(arch_wb_cache_pmem); | |
| 95 | + | |
| 96 | +void arch_invalidate_pmem(void *addr, size_t size) | |
| 97 | +{ | |
| 98 | + __inval_dcache_area(addr, size); | |
| 99 | +} | |
| 100 | +EXPORT_SYMBOL_GPL(arch_invalidate_pmem); | |
| 101 | +#endif | |
| @@ -41,6 +41,16 @@ int pud_huge(pud_t pud) · 40 unmodified lines | ||
| 41 | 41 | #endif |
| 42 | 42 | } |
| 43 | 43 | |
| 44 | +/* | |
| 45 | + * Select all bits except the pfn | |
| 46 | + */ | |
| 47 | +static inline pgprot_t pte_pgprot(pte_t pte) | |
| 48 | +{ | |
| 49 | + unsigned long pfn = pte_pfn(pte); | |
| 50 | + | |
| 51 | + return __pgprot(pte_val(pfn_pte(pfn, __pgprot(0))) ^ pte_val(pte)); | |
| 52 | +} | |
| 53 | + | |
| 44 | 54 | static int find_num_contig(struct mm_struct *mm, unsigned long addr, |
| 45 | 55 | pte_t *ptep, size_t *pgsize) |
| 46 | 56 | { |
| @@ -58,15 +68,107 @@ static int find_num_contig(struct mm_struct *mm, unsigned long addr, · 11 unmodified lines | ||
| 58 | 68 | return CONT_PTES; |
| 59 | 69 | } |
| 60 | 70 | |
| 71 | +static inline int num_contig_ptes(unsigned long size, size_t *pgsize) | |
| 72 | +{ | |
| 73 | + int contig_ptes = 0; | |
| 74 | + | |
| 75 | + *pgsize = size; | |
| 76 | + | |
| 77 | + switch (size) { | |
| 78 | +#ifdef CONFIG_ARM64_4K_PAGES | |
| 79 | + case PUD_SIZE: | |
| 80 | +#endif | |
| 81 | + case PMD_SIZE: | |
| 82 | + contig_ptes = 1; | |
| 83 | + break; | |
| 84 | + case CONT_PMD_SIZE: | |
| 85 | + *pgsize = PMD_SIZE; | |
| 86 | + contig_ptes = CONT_PMDS; | |
| 87 | + break; | |
| 88 | + case CONT_PTE_SIZE: | |
| 89 | + *pgsize = PAGE_SIZE; | |
| 90 | + contig_ptes = CONT_PTES; | |
| 91 | + break; | |
| 92 | + } | |
| 93 | + | |
| 94 | + return contig_ptes; | |
| 95 | +} | |
| 96 | + | |
| 97 | +/* | |
| 98 | + * Changing some bits of contiguous entries requires us to follow a | |
| 99 | + * Break-Before-Make approach, breaking the whole contiguous set | |
| 100 | + * before we can change any entries. See ARM DDI 0487A.k_iss10775, | |
| 101 | + * "Misprogramming of the Contiguous bit", page D4-1762. | |
| 102 | + * | |
| 103 | + * This helper performs the break step. | |
| 104 | + */ | |
| 105 | +static pte_t get_clear_flush(struct mm_struct *mm, | |
| 106 | + unsigned long addr, | |
| 107 | + pte_t *ptep, | |
| 108 | + unsigned long pgsize, | |
| 109 | + unsigned long ncontig) | |
| 110 | +{ | |
| 111 | + struct vm_area_struct vma = { .vm_mm = mm }; | |
| 112 | + pte_t orig_pte = huge_ptep_get(ptep); | |
| 113 | + bool valid = pte_valid(orig_pte); | |
| 114 | + unsigned long i, saddr = addr; | |
| 115 | + | |
| 116 | + for (i = 0; i < ncontig; i++, addr += pgsize, ptep++) { | |
| 117 | + pte_t pte = ptep_get_and_clear(mm, addr, ptep); | |
| 118 | + | |
| 119 | + /* | |
| 120 | + * If HW_AFDBM is enabled, then the HW could turn on | |
| 121 | + * the dirty bit for any page in the set, so check | |
| 122 | + * them all. All hugetlb entries are already young. | |
| 123 | + */ | |
| 124 | + if (pte_dirty(pte)) | |
| 125 | + orig_pte = pte_mkdirty(orig_pte); | |
| 126 | + } | |
| 127 | + | |
| 128 | + if (valid) | |
| 129 | + flush_tlb_range(&vma, saddr, addr); | |
| 130 | + return orig_pte; | |
| 131 | +} | |
| 132 | + | |
| 133 | +/* | |
| 134 | + * Changing some bits of contiguous entries requires us to follow a | |
| 135 | + * Break-Before-Make approach, breaking the whole contiguous set | |
| 136 | + * before we can change any entries. See ARM DDI 0487A.k_iss10775, | |
| 137 | + * "Misprogramming of the Contiguous bit", page D4-1762. | |
| 138 | + * | |
| 139 | + * This helper performs the break step for use cases where the | |
| 140 | + * original pte is not needed. | |
| 141 | + */ | |
| 142 | +static void clear_flush(struct mm_struct *mm, | |
| 143 | + unsigned long addr, | |
| 144 | + pte_t *ptep, | |
| 145 | + unsigned long pgsize, | |
| 146 | + unsigned long ncontig) | |
| 147 | +{ | |
| 148 | + struct vm_area_struct vma = { .vm_mm = mm }; | |
| 149 | + unsigned long i, saddr = addr; | |
| 150 | + | |
| 151 | + for (i = 0; i < ncontig; i++, addr += pgsize, ptep++) | |
| 152 | + pte_clear(mm, addr, ptep); | |
| 153 | + | |
| 154 | + flush_tlb_range(&vma, saddr, addr); | |
| 155 | +} | |
| 156 | + | |
| 61 | 157 | void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, |
| 62 | 158 | pte_t *ptep, pte_t pte) |
| 63 | 159 | { |
| 64 | 160 | size_t pgsize; |
| 65 | 161 | int i; |
| 66 | 162 | int ncontig; |
| 67 | − unsigned long pfn; | |
| 163 | + unsigned long pfn, dpfn; | |
| 68 | 164 | pgprot_t hugeprot; |
| 69 | 165 | |
| 166 | + /* | |
| 167 | + * Code needs to be expanded to handle huge swap and migration | |
| 168 | + * entries. Needed for HUGETLB and MEMORY_FAILURE. | |
| 169 | + */ | |
| 170 | + WARN_ON(!pte_present(pte)); | |
| 171 | + | |
| 70 | 172 | if (!pte_cont(pte)) { |
| 71 | 173 | set_pte_at(mm, addr, ptep, pte); |
| 72 | 174 | return; |
| @@ -74,17 +176,30 @@ void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, · 1 unmodified line | ||
| 74 | 176 | |
| 75 | 177 | ncontig = find_num_contig(mm, addr, ptep, &pgsize); |
| 76 | 178 | pfn = pte_pfn(pte); |
| 77 | − hugeprot = __pgprot(pte_val(pfn_pte(pfn, __pgprot(0))) ^ pte_val(pte)); | |
| 78 | − for (i = 0; i < ncontig; i++) { | |
| 179 | + dpfn = pgsize >> PAGE_SHIFT; | |
| 180 | + hugeprot = pte_pgprot(pte); | |
| 181 | + | |
| 182 | + clear_flush(mm, addr, ptep, pgsize, ncontig); | |
| 183 | + | |
| 184 | + for (i = 0; i < ncontig; i++, ptep++, addr += pgsize, pfn += dpfn) { | |
| 79 | 185 | pr_debug("%s: set pte %p to 0x%llx\n", __func__, ptep, |
| 80 | 186 | pte_val(pfn_pte(pfn, hugeprot))); |
| 81 | 187 | set_pte_at(mm, addr, ptep, pfn_pte(pfn, hugeprot)); |
| 82 | − ptep++; | |
| 83 | − pfn += pgsize >> PAGE_SHIFT; | |
| 84 | − addr += pgsize; | |
| 85 | 188 | } |
| 86 | 189 | } |
| 87 | 190 | |
| 191 | +void set_huge_swap_pte_at(struct mm_struct *mm, unsigned long addr, | |
| 192 | + pte_t *ptep, pte_t pte, unsigned long sz) | |
| 193 | +{ | |
| 194 | + int i, ncontig; | |
| 195 | + size_t pgsize; | |
| 196 | + | |
| 197 | + ncontig = num_contig_ptes(sz, &pgsize); | |
| 198 | + | |
| 199 | + for (i = 0; i < ncontig; i++, ptep++) | |
| 200 | + set_pte(ptep, pte); | |
| 201 | +} | |
| 202 | + | |
| 88 | 203 | pte_t *huge_pte_alloc(struct mm_struct *mm, |
| 89 | 204 | unsigned long addr, unsigned long sz) |
| 90 | 205 | { |
| @@ -144,19 +259,28 @@ pte_t *huge_pte_offset(struct mm_struct *mm, · 53 unmodified lines | ||
| 144 | 259 | return NULL; |
| 145 | 260 | |
| 146 | 261 | pud = pud_offset(pgd, addr); |
| 147 | − if (pud_none(*pud)) | |
| 262 | + if (sz != PUD_SIZE && pud_none(*pud)) | |
| 148 | 263 | return NULL; |
| 149 | − /* swap or huge page */ | |
| 150 | − if (!pud_present(*pud) || pud_huge(*pud)) | |
| 264 | + /* hugepage or swap? */ | |
| 265 | + if (pud_huge(*pud) || !pud_present(*pud)) | |
| 151 | 266 | return (pte_t *)pud; |
| 152 | 267 | /* table; check the next level */ |
| 153 | 268 | |
| 269 | + if (sz == CONT_PMD_SIZE) | |
| 270 | + addr &= CONT_PMD_MASK; | |
| 271 | + | |
| 154 | 272 | pmd = pmd_offset(pud, addr); |
| 155 | − if (pmd_none(*pmd)) | |
| 273 | + if (!(sz == PMD_SIZE || sz == CONT_PMD_SIZE) && | |
| 274 | + pmd_none(*pmd)) | |
| 156 | 275 | return NULL; |
| 157 | − if (!pmd_present(*pmd) || pmd_huge(*pmd)) | |
| 276 | + if (pmd_huge(*pmd) || !pmd_present(*pmd)) | |
| 158 | 277 | return (pte_t *)pmd; |
| 159 | 278 | |
| 279 | + if (sz == CONT_PTE_SIZE) { | |
| 280 | + pte_t *pte = pte_offset_kernel(pmd, (addr & CONT_PTE_MASK)); | |
| 281 | + return pte; | |
| 282 | + } | |
| 283 | + | |
| 160 | 284 | return NULL; |
| 161 | 285 | } |
| 162 | 286 | |
| @@ -176,111 +300,133 @@ pte_t arch_make_huge_pte(pte_t entry, struct vm_area_struct *vma, · 13 unmodified lines | ||
| 176 | 300 | return entry; |
| 177 | 301 | } |
| 178 | 302 | |
| 303 | +void huge_pte_clear(struct mm_struct *mm, unsigned long addr, | |
| 304 | + pte_t *ptep, unsigned long sz) | |
| 305 | +{ | |
| 306 | + int i, ncontig; | |
| 307 | + size_t pgsize; | |
| 308 | + | |
| 309 | + ncontig = num_contig_ptes(sz, &pgsize); | |
| 310 | + | |
| 311 | + for (i = 0; i < ncontig; i++, addr += pgsize, ptep++) | |
| 312 | + pte_clear(mm, addr, ptep); | |
| 313 | +} | |
| 314 | + | |
| 179 | 315 | pte_t huge_ptep_get_and_clear(struct mm_struct *mm, |
| 180 | 316 | unsigned long addr, pte_t *ptep) |
| 181 | 317 | { |
| 182 | − pte_t pte; | |
| 318 | + int ncontig; | |
| 319 | + size_t pgsize; | |
| 320 | + pte_t orig_pte = huge_ptep_get(ptep); | |
| 183 | 321 | |
| 184 | − if (pte_cont(*ptep)) { | |
| 185 | − int ncontig, i; | |
| 186 | − size_t pgsize; | |
| 187 | − bool is_dirty = false; | |
| 188 | − | |
| 189 | − ncontig = find_num_contig(mm, addr, ptep, &pgsize); | |
| 190 | − /* save the 1st pte to return */ | |
| 191 | − pte = ptep_get_and_clear(mm, addr, ptep); | |
| 192 | − for (i = 1, addr += pgsize; i < ncontig; ++i, addr += pgsize) { | |
| 193 | − /* | |
| 194 | − * If HW_AFDBM is enabled, then the HW could | |
| 195 | − * turn on the dirty bit for any of the page | |
| 196 | − * in the set, so check them all. | |
| 197 | − */ | |
| 198 | − ++ptep; | |
| 199 | − if (pte_dirty(ptep_get_and_clear(mm, addr, ptep))) | |
| 200 | − is_dirty = true; | |
| 201 | − } | |
| 202 | − if (is_dirty) | |
| 203 | − return pte_mkdirty(pte); | |
| 204 | − else | |
| 205 | − return pte; | |
| 206 | − } else { | |
| 322 | + if (!pte_cont(orig_pte)) | |
| 207 | 323 | return ptep_get_and_clear(mm, addr, ptep); |
| 208 | − } | |
| 324 | + | |
| 325 | + ncontig = find_num_contig(mm, addr, ptep, &pgsize); | |
| 326 | + | |
| 327 | + return get_clear_flush(mm, addr, ptep, pgsize, ncontig); | |
| 209 | 328 | } |
| 210 | 329 | |
| 211 | 330 | int huge_ptep_set_access_flags(struct vm_area_struct *vma, |
| 212 | 331 | unsigned long addr, pte_t *ptep, |
| 213 | 332 | pte_t pte, int dirty) |
| 214 | 333 | { |
| 215 | − if (pte_cont(pte)) { | |
| 216 | − int ncontig, i, changed = 0; | |
| 217 | − size_t pgsize = 0; | |
| 218 | − unsigned long pfn = pte_pfn(pte); | |
| 219 | − /* Select all bits except the pfn */ | |
| 220 | − pgprot_t hugeprot = | |
| 221 | − __pgprot(pte_val(pfn_pte(pfn, __pgprot(0))) ^ | |
| 222 | − pte_val(pte)); | |
| 223 | − | |
| 224 | − pfn = pte_pfn(pte); | |
| 225 | − ncontig = find_num_contig(vma->vm_mm, addr, ptep, | |
| 226 | − &pgsize); | |
| 227 | − for (i = 0; i < ncontig; ++i, ++ptep, addr += pgsize) { | |
| 228 | − changed |= ptep_set_access_flags(vma, addr, ptep, | |
| 229 | − pfn_pte(pfn, | |
| 230 | − hugeprot), | |
| 231 | − dirty); | |
| 232 | − pfn += pgsize >> PAGE_SHIFT; | |
| 233 | − } | |
| 234 | − return changed; | |
| 235 | − } else { | |
| 334 | + int ncontig, i, changed = 0; | |
| 335 | + size_t pgsize = 0; | |
| 336 | + unsigned long pfn = pte_pfn(pte), dpfn; | |
| 337 | + pgprot_t hugeprot; | |
| 338 | + pte_t orig_pte; | |
| 339 | + | |
| 340 | + if (!pte_cont(pte)) | |
| 236 | 341 | return ptep_set_access_flags(vma, addr, ptep, pte, dirty); |
| 237 | − } | |
| 342 | + | |
| 343 | + ncontig = find_num_contig(vma->vm_mm, addr, ptep, &pgsize); | |
| 344 | + dpfn = pgsize >> PAGE_SHIFT; | |
| 345 | + | |
| 346 | + orig_pte = get_clear_flush(vma->vm_mm, addr, ptep, pgsize, ncontig); | |
| 347 | + if (!pte_same(orig_pte, pte)) | |
| 348 | + changed = 1; | |
| 349 | + | |
| 350 | + /* Make sure we don't lose the dirty state */ | |
| 351 | + if (pte_dirty(orig_pte)) | |
| 352 | + pte = pte_mkdirty(pte); | |
| 353 | + | |
| 354 | + hugeprot = pte_pgprot(pte); | |
| 355 | + for (i = 0; i < ncontig; i++, ptep++, addr += pgsize, pfn += dpfn) | |
| 356 | + set_pte_at(vma->vm_mm, addr, ptep, pfn_pte(pfn, hugeprot)); | |
| 357 | + | |
| 358 | + return changed; | |
| 238 | 359 | } |
| 239 | 360 | |
| 240 | 361 | void huge_ptep_set_wrprotect(struct mm_struct *mm, |
| 241 | 362 | unsigned long addr, pte_t *ptep) |
| 242 | 363 | { |
| 243 | − if (pte_cont(*ptep)) { | |
| 244 | − int ncontig, i; | |
| 245 | − size_t pgsize = 0; | |
| 246 | − | |
| 247 | − ncontig = find_num_contig(mm, addr, ptep, &pgsize); | |
| 248 | − for (i = 0; i < ncontig; ++i, ++ptep, addr += pgsize) | |
| 249 | − ptep_set_wrprotect(mm, addr, ptep); | |
| 250 | − } else { | |
| 364 | + unsigned long pfn, dpfn; | |
| 365 | + pgprot_t hugeprot; | |
| 366 | + int ncontig, i; | |
| 367 | + size_t pgsize; | |
| 368 | + pte_t pte; | |
| 369 | + | |
| 370 | + if (!pte_cont(*ptep)) { | |
| 251 | 371 | ptep_set_wrprotect(mm, addr, ptep); |
| 372 | + return; | |
| 252 | 373 | } |
| 374 | + | |
| 375 | + ncontig = find_num_contig(mm, addr, ptep, &pgsize); | |
| 376 | + dpfn = pgsize >> PAGE_SHIFT; | |
| 377 | + | |
| 378 | + pte = get_clear_flush(mm, addr, ptep, pgsize, ncontig); | |
| 379 | + pte = pte_wrprotect(pte); | |
| 380 | + | |
| 381 | + hugeprot = pte_pgprot(pte); | |
| 382 | + pfn = pte_pfn(pte); | |
| 383 | + | |
| 384 | + for (i = 0; i < ncontig; i++, ptep++, addr += pgsize, pfn += dpfn) | |
| 385 | + set_pte_at(mm, addr, ptep, pfn_pte(pfn, hugeprot)); | |
| 253 | 386 | } |
| 254 | 387 | |
| 255 | 388 | void huge_ptep_clear_flush(struct vm_area_struct *vma, |
| 256 | 389 | unsigned long addr, pte_t *ptep) |
| 257 | 390 | { |
| 258 | − if (pte_cont(*ptep)) { | |
| 259 | − int ncontig, i; | |
| 260 | − size_t pgsize = 0; | |
| 261 | − | |
| 262 | − ncontig = find_num_contig(vma->vm_mm, addr, ptep, | |
| 263 | − &pgsize); | |
| 264 | − for (i = 0; i < ncontig; ++i, ++ptep, addr += pgsize) | |
| 265 | − ptep_clear_flush(vma, addr, ptep); | |
| 266 | − } else { | |
| 391 | + size_t pgsize; | |
| 392 | + int ncontig; | |
| 393 | + | |
| 394 | + if (!pte_cont(*ptep)) { | |
| 267 | 395 | ptep_clear_flush(vma, addr, ptep); |
| 396 | + return; | |
| 268 | 397 | } |
| 398 | + | |
| 399 | + ncontig = find_num_contig(vma->vm_mm, addr, ptep, &pgsize); | |
| 400 | + clear_flush(vma->vm_mm, addr, ptep, pgsize, ncontig); | |
| 269 | 401 | } |
| 270 | 402 | |
| 271 | 403 | static __init int setup_hugepagesz(char *opt) |
| 272 | 404 | { |
| 273 | 405 | unsigned long ps = memparse(opt, &opt); |
| 274 | 406 | |
| 275 | − if (ps == PMD_SIZE) { | |
| 276 | − hugetlb_add_hstate(PMD_SHIFT - PAGE_SHIFT); | |
| 277 | − } else if (ps == PUD_SIZE) { | |
| 278 | − hugetlb_add_hstate(PUD_SHIFT - PAGE_SHIFT); | |
| 279 | − } else { | |
| 280 | − hugetlb_bad_size(); | |
| 281 | − pr_err("hugepagesz: Unsupported page size %lu K\n", ps >> 10); | |
| 282 | − return 0; | |
| 407 | + switch (ps) { | |
| 408 | +#ifdef CONFIG_ARM64_4K_PAGES | |
| 409 | + case PUD_SIZE: | |
| 410 | +#endif | |
| 411 | + case PMD_SIZE * CONT_PMDS: | |
| 412 | + case PMD_SIZE: | |
| 413 | + case PAGE_SIZE * CONT_PTES: | |
| 414 | + hugetlb_add_hstate(ilog2(ps) - PAGE_SHIFT); | |
| 415 | + return 1; | |
| 283 | 416 | } |
| 284 | − return 1; | |
| 417 | + | |
| 418 | + hugetlb_bad_size(); | |
| 419 | + pr_err("hugepagesz: Unsupported page size %lu K\n", ps >> 10); | |
| 420 | + return 0; | |
| 285 | 421 | } |
| 286 | 422 | __setup("hugepagesz=", setup_hugepagesz); |
| 423 | + | |
| 424 | +#ifdef CONFIG_ARM64_64K_PAGES | |
| 425 | +static __init int add_default_hugepagesz(void) | |
| 426 | +{ | |
| 427 | + if (size_to_hstate(CONT_PTES * PAGE_SIZE) == NULL) | |
| 428 | + hugetlb_add_hstate(CONT_PTE_SHIFT); | |
| 429 | + return 0; | |
| 430 | +} | |
| 431 | +arch_initcall(add_default_hugepagesz); | |
| 432 | +#endif | |
| @@ -588,7 +588,8 @@ void acpi_configure_pmsi_domain(struct device *dev) · 587 unmodified lines | ||
| 588 | 588 | dev_set_msi_domain(dev, msi_domain); |
| 589 | 589 | } |
| 590 | 590 | |
| 591 | −static int __get_pci_rid(struct pci_dev *pdev, u16 alias, void *data) | |
| 591 | +static int __maybe_unused __get_pci_rid(struct pci_dev *pdev, u16 alias, | |
| 592 | + void *data) | |
| 592 | 593 | { |
| 593 | 594 | u32 *rid = data; |
| 594 | 595 | |
| @@ -633,8 +634,7 @@ int iort_add_device_replay(const struct iommu_ops *ops, struct device *dev) · 38 unmodified lines | ||
| 633 | 634 | { |
| 634 | 635 | int err = 0; |
| 635 | 636 | |
| 636 | − if (!IS_ERR_OR_NULL(ops) && ops->add_device && dev->bus && | |
| 637 | − !dev->iommu_group) | |
| 637 | + if (ops->add_device && dev->bus && !dev->iommu_group) | |
| 638 | 638 | err = ops->add_device(dev); |
| 639 | 639 | |
| 640 | 640 | return err; |
| @@ -648,36 +648,49 @@ int iort_add_device_replay(const struct iommu_ops *ops, struct device *dev) · 7 unmodified lines | ||
| 648 | 648 | { return 0; } |
| 649 | 649 | #endif |
| 650 | 650 | |
| 651 | −static const struct iommu_ops *iort_iommu_xlate(struct device *dev, | |
| 652 | − struct acpi_iort_node *node, | |
| 653 | − u32 streamid) | |
| 651 | +static int iort_iommu_xlate(struct device *dev, struct acpi_iort_node *node, | |
| 652 | + u32 streamid) | |
| 654 | 653 | { |
| 655 | − const struct iommu_ops *ops = NULL; | |
| 656 | − int ret = -ENODEV; | |
| 654 | + const struct iommu_ops *ops; | |
| 657 | 655 | struct fwnode_handle *iort_fwnode; |
| 658 | 656 | |
| 659 | − if (node) { | |
| 660 | − iort_fwnode = iort_get_fwnode(node); | |
| 661 | − if (!iort_fwnode) | |
| 662 | − return NULL; | |
| 657 | + if (!node) | |
| 658 | + return -ENODEV; | |
| 663 | 659 | |
| 664 | − ops = iommu_ops_from_fwnode(iort_fwnode); | |
| 665 | − /* | |
| 666 | − * If the ops look-up fails, this means that either | |
| 667 | − * the SMMU drivers have not been probed yet or that | |
| 668 | − * the SMMU drivers are not built in the kernel; | |
| 669 | − * Depending on whether the SMMU drivers are built-in | |
| 670 | − * in the kernel or not, defer the IOMMU configuration | |
| 671 | − * or just abort it. | |
| 672 | − */ | |
| 673 | − if (!ops) | |
| 674 | − return iort_iommu_driver_enabled(node->type) ? | |
| 675 | − ERR_PTR(-EPROBE_DEFER) : NULL; | |
| 660 | + iort_fwnode = iort_get_fwnode(node); | |
| 661 | + if (!iort_fwnode) | |
| 662 | + return -ENODEV; | |
| 676 | 663 | |
| 677 | − ret = arm_smmu_iort_xlate(dev, streamid, iort_fwnode, ops); | |
| 678 | − } | |
| 664 | + /* | |
| 665 | + * If the ops look-up fails, this means that either | |
| 666 | + * the SMMU drivers have not been probed yet or that | |
| 667 | + * the SMMU drivers are not built in the kernel; | |
| 668 | + * Depending on whether the SMMU drivers are built-in | |
| 669 | + * in the kernel or not, defer the IOMMU configuration | |
| 670 | + * or just abort it. | |
| 671 | + */ | |
| 672 | + ops = iommu_ops_from_fwnode(iort_fwnode); | |
| 673 | + if (!ops) | |
| 674 | + return iort_iommu_driver_enabled(node->type) ? | |
| 675 | + -EPROBE_DEFER : -ENODEV; | |
| 679 | 676 | |
| 680 | − return ret ? NULL : ops; | |
| 677 | + return arm_smmu_iort_xlate(dev, streamid, iort_fwnode, ops); | |
| 678 | +} | |
| 679 | + | |
| 680 | +struct iort_pci_alias_info { | |
| 681 | + struct device *dev; | |
| 682 | + struct acpi_iort_node *node; | |
| 683 | +}; | |
| 684 | + | |
| 685 | +static int iort_pci_iommu_init(struct pci_dev *pdev, u16 alias, void *data) | |
| 686 | +{ | |
| 687 | + struct iort_pci_alias_info *info = data; | |
| 688 | + struct acpi_iort_node *parent; | |
| 689 | + u32 streamid; | |
| 690 | + | |
| 691 | + parent = iort_node_map_id(info->node, alias, &streamid, | |
| 692 | + IORT_IOMMU_TYPE); | |
| 693 | + return iort_iommu_xlate(info->dev, parent, streamid); | |
| 681 | 694 | } |
| 682 | 695 | |
| 683 | 696 | /** |
| @@ -713,9 +726,9 @@ void iort_set_dma_mask(struct device *dev) · 29 unmodified lines | ||
| 713 | 726 | const struct iommu_ops *iort_iommu_configure(struct device *dev) |
| 714 | 727 | { |
| 715 | 728 | struct acpi_iort_node *node, *parent; |
| 716 | − const struct iommu_ops *ops = NULL; | |
| 729 | + const struct iommu_ops *ops; | |
| 717 | 730 | u32 streamid = 0; |
| 718 | − int err; | |
| 731 | + int err = -ENODEV; | |
| 719 | 732 | |
| 720 | 733 | /* |
| 721 | 734 | * If we already translated the fwspec there |
| @@ -727,21 +740,16 @@ const struct iommu_ops *iort_iommu_configure(struct device *dev) · 5 unmodified lines | ||
| 727 | 740 | |
| 728 | 741 | if (dev_is_pci(dev)) { |
| 729 | 742 | struct pci_bus *bus = to_pci_dev(dev)->bus; |
| 730 | − u32 rid; | |
| 731 | − | |
| 732 | − pci_for_each_dma_alias(to_pci_dev(dev), __get_pci_rid, | |
| 733 | − &rid); | |
| 743 | + struct iort_pci_alias_info info = { .dev = dev }; | |
| 734 | 744 | |
| 735 | 745 | node = iort_scan_node(ACPI_IORT_NODE_PCI_ROOT_COMPLEX, |
| 736 | 746 | iort_match_node_callback, &bus->dev); |
| 737 | 747 | if (!node) |
| 738 | 748 | return NULL; |
| 739 | 749 | |
| 740 | − parent = iort_node_map_id(node, rid, &streamid, | |
| 741 | − IORT_IOMMU_TYPE); | |
| 742 | − | |
| 743 | − ops = iort_iommu_xlate(dev, parent, streamid); | |
| 744 | − | |
| 750 | + info.node = node; | |
| 751 | + err = pci_for_each_dma_alias(to_pci_dev(dev), | |
| 752 | + iort_pci_iommu_init, &info); | |
| 745 | 753 | } else { |
| 746 | 754 | int i = 0; |
| 747 | 755 | |
| @@ -750,31 +758,30 @@ const struct iommu_ops *iort_iommu_configure(struct device *dev) · 2 unmodified lines | ||
| 750 | 758 | if (!node) |
| 751 | 759 | return NULL; |
| 752 | 760 | |
| 753 | − parent = iort_node_map_platform_id(node, &streamid, | |
| 754 | − IORT_IOMMU_TYPE, i++); | |
| 755 | − | |
| 756 | − while (parent) { | |
| 757 | − ops = iort_iommu_xlate(dev, parent, streamid); | |
| 758 | − if (IS_ERR_OR_NULL(ops)) | |
| 759 | − return ops; | |
| 760 | − | |
| 761 | + do { | |
| 761 | 762 | parent = iort_node_map_platform_id(node, &streamid, |
| 762 | 763 | IORT_IOMMU_TYPE, |
| 763 | 764 | i++); |
| 764 | − } | |
| 765 | + | |
| 766 | + if (parent) | |
| 767 | + err = iort_iommu_xlate(dev, parent, streamid); | |
| 768 | + } while (parent && !err); | |
| 765 | 769 | } |
| 766 | 770 | |
| 767 | 771 | /* |
| 768 | 772 | * If we have reason to believe the IOMMU driver missed the initial |
| 769 | 773 | * add_device callback for dev, replay it to get things in order. |
| 770 | 774 | */ |
| 771 | − err = iort_add_device_replay(ops, dev); | |
| 772 | − if (err) | |
| 773 | − ops = ERR_PTR(err); | |
| 775 | + if (!err) { | |
| 776 | + ops = iort_fwspec_iommu_ops(dev->iommu_fwspec); | |
| 777 | + err = iort_add_device_replay(ops, dev); | |
| 778 | + } | |
| 774 | 779 | |
| 775 | 780 | /* Ignore all other errors apart from EPROBE_DEFER */ |
| 776 | − if (IS_ERR(ops) && (PTR_ERR(ops) != -EPROBE_DEFER)) { | |
| 777 | − dev_dbg(dev, "Adding to IOMMU failed: %ld\n", PTR_ERR(ops)); | |
| 781 | + if (err == -EPROBE_DEFER) { | |
| 782 | + ops = ERR_PTR(err); | |
| 783 | + } else if (err) { | |
| 784 | + dev_dbg(dev, "Adding to IOMMU failed: %d\n", err); | |
| 778 | 785 | ops = NULL; |
| 779 | 786 | } |
| 780 | 787 | |
| @@ -908,6 +915,27 @@ static bool __init arm_smmu_v3_is_coherent(struct acpi_iort_node *node) · 127 unmodified lines | ||
| 908 | 915 | return smmu->flags & ACPI_IORT_SMMU_V3_COHACC_OVERRIDE; |
| 909 | 916 | } |
| 910 | 917 | |
| 918 | +#if defined(CONFIG_ACPI_NUMA) && defined(ACPI_IORT_SMMU_V3_PXM_VALID) | |
| 919 | +/* | |
| 920 | + * set numa proximity domain for smmuv3 device | |
| 921 | + */ | |
| 922 | +static void __init arm_smmu_v3_set_proximity(struct device *dev, | |
| 923 | + struct acpi_iort_node *node) | |
| 924 | +{ | |
| 925 | + struct acpi_iort_smmu_v3 *smmu; | |
| 926 | + | |
| 927 | + smmu = (struct acpi_iort_smmu_v3 *)node->node_data; | |
| 928 | + if (smmu->flags & ACPI_IORT_SMMU_V3_PXM_VALID) { | |
| 929 | + set_dev_node(dev, acpi_map_pxm_to_node(smmu->pxm)); | |
| 930 | + pr_info("SMMU-v3[%llx] Mapped to Proximity domain %d\n", | |
| 931 | + smmu->base_address, | |
| 932 | + smmu->pxm); | |
| 933 | + } | |
| 934 | +} | |
| 935 | +#else | |
| 936 | +#define arm_smmu_v3_set_proximity NULL | |
| 937 | +#endif | |
| 938 | + | |
| 911 | 939 | static int __init arm_smmu_count_resources(struct acpi_iort_node *node) |
| 912 | 940 | { |
| 913 | 941 | struct acpi_iort_smmu *smmu; |
| @@ -977,13 +1005,16 @@ struct iort_iommu_config { · 63 unmodified lines | ||
| 977 | 1005 | int (*iommu_count_resources)(struct acpi_iort_node *node); |
| 978 | 1006 | void (*iommu_init_resources)(struct resource *res, |
| 979 | 1007 | struct acpi_iort_node *node); |
| 1008 | + void (*iommu_set_proximity)(struct device *dev, | |
| 1009 | + struct acpi_iort_node *node); | |
| 980 | 1010 | }; |
| 981 | 1011 | |
| 982 | 1012 | static const struct iort_iommu_config iort_arm_smmu_v3_cfg __initconst = { |
| 983 | 1013 | .name = "arm-smmu-v3", |
| 984 | 1014 | .iommu_is_coherent = arm_smmu_v3_is_coherent, |
| 985 | 1015 | .iommu_count_resources = arm_smmu_v3_count_resources, |
| 986 | − .iommu_init_resources = arm_smmu_v3_init_resources | |
| 1016 | + .iommu_init_resources = arm_smmu_v3_init_resources, | |
| 1017 | + .iommu_set_proximity = arm_smmu_v3_set_proximity, | |
| 987 | 1018 | }; |
| 988 | 1019 | |
| 989 | 1020 | static const struct iort_iommu_config iort_arm_smmu_cfg __initconst = { |
| @@ -1028,6 +1059,9 @@ static int __init iort_add_smmu_platform_device(struct acpi_iort_node *node) · 38 unmodified lines | ||
| 1028 | 1059 | if (!pdev) |
| 1029 | 1060 | return -ENOMEM; |
| 1030 | 1061 | |
| 1062 | + if (ops->iommu_set_proximity) | |
| 1063 | + ops->iommu_set_proximity(&pdev->dev, node); | |
| 1064 | + | |
| 1031 | 1065 | count = ops->iommu_count_resources(node); |
| 1032 | 1066 | |
| 1033 | 1067 | r = kcalloc(count, sizeof(*r), GFP_KERNEL); |
| @@ -11,6 +11,7 @@ · 10 unmodified lines | ||
| 11 | 11 | */ |
| 12 | 12 | #include <linux/efi.h> |
| 13 | 13 | #include <asm/efi.h> |
| 14 | +#include <asm/memory.h> | |
| 14 | 15 | #include <asm/sections.h> |
| 15 | 16 | #include <asm/sysreg.h> |
| 16 | 17 | |
| @@ -81,9 +82,10 @@ efi_status_t handle_kernel_image(efi_system_table_t *sys_table_arg, · 64 unmodified lines | ||
| 81 | 82 | /* |
| 82 | 83 | * If CONFIG_DEBUG_ALIGN_RODATA is not set, produce a |
| 83 | 84 | * displacement in the interval [0, MIN_KIMG_ALIGN) that |
| 84 | − * is a multiple of the minimal segment alignment (SZ_64K) | |
| 85 | + * doesn't violate this kernel's de-facto alignment | |
| 86 | + * constraints. | |
| 85 | 87 | */ |
| 86 | − u32 mask = (MIN_KIMG_ALIGN - 1) & ~(SZ_64K - 1); | |
| 88 | + u32 mask = (MIN_KIMG_ALIGN - 1) & ~(EFI_KIMG_ALIGN - 1); | |
| 87 | 89 | u32 offset = !IS_ENABLED(CONFIG_DEBUG_ALIGN_RODATA) ? |
| 88 | 90 | (phys_seed >> 32) & mask : TEXT_OFFSET; |
| 89 | 91 | |
| @@ -47,6 +47,9 @@ armpmu_map_cache_event(const unsigned (*cache_map) · 46 unmodified lines | ||
| 47 | 47 | if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX) |
| 48 | 48 | return -EINVAL; |
| 49 | 49 | |
| 50 | + if (!cache_map) | |
| 51 | + return -ENOENT; | |
| 52 | + | |
| 50 | 53 | ret = (int)(*cache_map)[cache_type][cache_op][cache_result]; |
| 51 | 54 | |
| 52 | 55 | if (ret == CACHE_OP_UNSUPPORTED) |
| @@ -63,6 +66,9 @@ armpmu_map_hw_event(const unsigned (*event_map)[PERF_COUNT_HW_MAX], u64 config) · 10 unmodified lines | ||
| 63 | 66 | if (config >= PERF_COUNT_HW_MAX) |
| 64 | 67 | return -EINVAL; |
| 65 | 68 | |
| 69 | + if (!event_map) | |
| 70 | + return -ENOENT; | |
| 71 | + | |
| 66 | 72 | mapping = (*event_map)[config]; |
| 67 | 73 | return mapping == HW_OP_UNSUPPORTED ? -ENOENT : mapping; |
| 68 | 74 | } |
| @@ -1147,7 +1147,6 @@ xgene_pmu_dev_add(struct xgene_pmu *xgene_pmu, struct xgene_pmu_dev_ctx *ctx) · 1146 unmodified lines | ||
| 1147 | 1147 | { |
| 1148 | 1148 | struct device *dev = xgene_pmu->dev; |
| 1149 | 1149 | struct xgene_pmu_dev *pmu; |
| 1150 | − int rc; | |
| 1151 | 1150 | |
| 1152 | 1151 | pmu = devm_kzalloc(dev, sizeof(*pmu), GFP_KERNEL); |
| 1153 | 1152 | if (!pmu) |
| @@ -1159,7 +1158,7 @@ xgene_pmu_dev_add(struct xgene_pmu *xgene_pmu, struct xgene_pmu_dev_ctx *ctx) · 5 unmodified lines | ||
| 1159 | 1158 | switch (pmu->inf->type) { |
| 1160 | 1159 | case PMU_TYPE_L3C: |
| 1161 | 1160 | if (!(xgene_pmu->l3c_active_mask & pmu->inf->enable_mask)) |
| 1162 | − goto dev_err; | |
| 1161 | + return -ENODEV; | |
| 1163 | 1162 | if (xgene_pmu->version == PCP_PMU_V3) |
| 1164 | 1163 | pmu->attr_groups = l3c_pmu_v3_attr_groups; |
| 1165 | 1164 | else |
| @@ -1177,7 +1176,7 @@ xgene_pmu_dev_add(struct xgene_pmu *xgene_pmu, struct xgene_pmu_dev_ctx *ctx) · 11 unmodified lines | ||
| 1177 | 1176 | break; |
| 1178 | 1177 | case PMU_TYPE_MCB: |
| 1179 | 1178 | if (!(xgene_pmu->mcb_active_mask & pmu->inf->enable_mask)) |
| 1180 | − goto dev_err; | |
| 1179 | + return -ENODEV; | |
| 1181 | 1180 | if (xgene_pmu->version == PCP_PMU_V3) |
| 1182 | 1181 | pmu->attr_groups = mcb_pmu_v3_attr_groups; |
| 1183 | 1182 | else |
| @@ -1185,7 +1184,7 @@ xgene_pmu_dev_add(struct xgene_pmu *xgene_pmu, struct xgene_pmu_dev_ctx *ctx) · 1 unmodified line | ||
| 1185 | 1184 | break; |
| 1186 | 1185 | case PMU_TYPE_MC: |
| 1187 | 1186 | if (!(xgene_pmu->mc_active_mask & pmu->inf->enable_mask)) |
| 1188 | − goto dev_err; | |
| 1187 | + return -ENODEV; | |
| 1189 | 1188 | if (xgene_pmu->version == PCP_PMU_V3) |
| 1190 | 1189 | pmu->attr_groups = mc_pmu_v3_attr_groups; |
| 1191 | 1190 | else |
| @@ -1195,19 +1194,14 @@ xgene_pmu_dev_add(struct xgene_pmu *xgene_pmu, struct xgene_pmu_dev_ctx *ctx) · 3 unmodified lines | ||
| 1195 | 1194 | return -EINVAL; |
| 1196 | 1195 | } |
| 1197 | 1196 | |
| 1198 | − rc = xgene_init_perf(pmu, ctx->name); | |
| 1199 | − if (rc) { | |
| 1197 | + if (xgene_init_perf(pmu, ctx->name)) { | |
| 1200 | 1198 | dev_err(dev, "%s PMU: Failed to init perf driver\n", ctx->name); |
| 1201 | − goto dev_err; | |
| 1199 | + return -ENODEV; | |
| 1202 | 1200 | } |
| 1203 | 1201 | |
| 1204 | 1202 | dev_info(dev, "%s PMU registered\n", ctx->name); |
| 1205 | 1203 | |
| 1206 | − return rc; | |
| 1207 | − | |
| 1208 | −dev_err: | |
| 1209 | − devm_kfree(dev, pmu); | |
| 1210 | − return -ENODEV; | |
| 1204 | + return 0; | |
| 1211 | 1205 | } |
| 1212 | 1206 | |
| 1213 | 1207 | static void _xgene_pmu_isr(int irq, struct xgene_pmu_dev *pmu_dev) |
| @@ -1515,13 +1509,13 @@ xgene_pmu_dev_ctx *acpi_get_pmu_hw_inf(struct xgene_pmu *xgene_pmu, · 301 unmodified lines | ||
| 1515 | 1509 | acpi_dev_free_resource_list(&resource_list); |
| 1516 | 1510 | if (rc < 0) { |
| 1517 | 1511 | dev_err(dev, "PMU type %d: No resource address found\n", type); |
| 1518 | − goto err; | |
| 1512 | + return NULL; | |
| 1519 | 1513 | } |
| 1520 | 1514 | |
| 1521 | 1515 | dev_csr = devm_ioremap_resource(dev, &res); |
| 1522 | 1516 | if (IS_ERR(dev_csr)) { |
| 1523 | 1517 | dev_err(dev, "PMU type %d: Fail to map resource\n", type); |
| 1524 | − goto err; | |
| 1518 | + return NULL; | |
| 1525 | 1519 | } |
| 1526 | 1520 | |
| 1527 | 1521 | /* A PMU device node without enable-bit-index is always enabled */ |
| @@ -1535,7 +1529,7 @@ xgene_pmu_dev_ctx *acpi_get_pmu_hw_inf(struct xgene_pmu *xgene_pmu, · 7 unmodified lines | ||
| 1535 | 1529 | ctx->name = xgene_pmu_dev_name(dev, type, enable_bit); |
| 1536 | 1530 | if (!ctx->name) { |
| 1537 | 1531 | dev_err(dev, "PMU type %d: Fail to get device name\n", type); |
| 1538 | − goto err; | |
| 1532 | + return NULL; | |
| 1539 | 1533 | } |
| 1540 | 1534 | inf = &ctx->inf; |
| 1541 | 1535 | inf->type = type; |
| @@ -1543,9 +1537,6 @@ xgene_pmu_dev_ctx *acpi_get_pmu_hw_inf(struct xgene_pmu *xgene_pmu, · 1 unmodified line | ||
| 1543 | 1537 | inf->enable_mask = 1 << enable_bit; |
| 1544 | 1538 | |
| 1545 | 1539 | return ctx; |
| 1546 | −err: | |
| 1547 | − devm_kfree(dev, ctx); | |
| 1548 | − return NULL; | |
| 1549 | 1540 | } |
| 1550 | 1541 | |
| 1551 | 1542 | static const struct acpi_device_id xgene_pmu_acpi_type_match[] = { |
| @@ -1663,20 +1654,20 @@ xgene_pmu_dev_ctx *fdt_get_pmu_hw_inf(struct xgene_pmu *xgene_pmu, · 111 unmodified lines | ||
| 1663 | 1654 | void __iomem *dev_csr; |
| 1664 | 1655 | struct resource res; |
| 1665 | 1656 | int enable_bit; |
| 1666 | − int rc; | |
| 1667 | 1657 | |
| 1668 | 1658 | ctx = devm_kzalloc(dev, sizeof(*ctx), GFP_KERNEL); |
| 1669 | 1659 | if (!ctx) |
| 1670 | 1660 | return NULL; |
| 1671 | − rc = of_address_to_resource(np, 0, &res); | |
| 1672 | − if (rc < 0) { | |
| 1661 | + | |
| 1662 | + if (of_address_to_resource(np, 0, &res) < 0) { | |
| 1673 | 1663 | dev_err(dev, "PMU type %d: No resource address found\n", type); |
| 1674 | − goto err; | |
| 1664 | + return NULL; | |
| 1675 | 1665 | } |
| 1666 | + | |
| 1676 | 1667 | dev_csr = devm_ioremap_resource(dev, &res); |
| 1677 | 1668 | if (IS_ERR(dev_csr)) { |
| 1678 | 1669 | dev_err(dev, "PMU type %d: Fail to map resource\n", type); |
| 1679 | − goto err; | |
| 1670 | + return NULL; | |
| 1680 | 1671 | } |
| 1681 | 1672 | |
| 1682 | 1673 | /* A PMU device node without enable-bit-index is always enabled */ |
| @@ -1686,17 +1677,15 @@ xgene_pmu_dev_ctx *fdt_get_pmu_hw_inf(struct xgene_pmu *xgene_pmu, · 3 unmodified lines | ||
| 1686 | 1677 | ctx->name = xgene_pmu_dev_name(dev, type, enable_bit); |
| 1687 | 1678 | if (!ctx->name) { |
| 1688 | 1679 | dev_err(dev, "PMU type %d: Fail to get device name\n", type); |
| 1689 | − goto err; | |
| 1680 | + return NULL; | |
| 1690 | 1681 | } |
| 1682 | + | |
| 1691 | 1683 | inf = &ctx->inf; |
| 1692 | 1684 | inf->type = type; |
| 1693 | 1685 | inf->csr = dev_csr; |
| 1694 | 1686 | inf->enable_mask = 1 << enable_bit; |
| 1695 | 1687 | |
| 1696 | 1688 | return ctx; |
| 1697 | −err: | |
| 1698 | − devm_kfree(dev, ctx); | |
| 1699 | − return NULL; | |
| 1700 | 1689 | } |
| 1701 | 1690 | |
| 1702 | 1691 | static int fdt_pmu_probe_pmu_dev(struct xgene_pmu *xgene_pmu, |
| @@ -1868,22 +1857,20 @@ static int xgene_pmu_probe(struct platform_device *pdev) · 165 unmodified lines | ||
| 1868 | 1857 | xgene_pmu->pcppmu_csr = devm_ioremap_resource(&pdev->dev, res); |
| 1869 | 1858 | if (IS_ERR(xgene_pmu->pcppmu_csr)) { |
| 1870 | 1859 | dev_err(&pdev->dev, "ioremap failed for PCP PMU resource\n"); |
| 1871 | − rc = PTR_ERR(xgene_pmu->pcppmu_csr); | |
| 1872 | − goto err; | |
| 1860 | + return PTR_ERR(xgene_pmu->pcppmu_csr); | |
| 1873 | 1861 | } |
| 1874 | 1862 | |
| 1875 | 1863 | irq = platform_get_irq(pdev, 0); |
| 1876 | 1864 | if (irq < 0) { |
| 1877 | 1865 | dev_err(&pdev->dev, "No IRQ resource\n"); |
| 1878 | − rc = -EINVAL; | |
| 1879 | − goto err; | |
| 1866 | + return -EINVAL; | |
| 1880 | 1867 | } |
| 1881 | 1868 | rc = devm_request_irq(&pdev->dev, irq, xgene_pmu_isr, |
| 1882 | 1869 | IRQF_NOBALANCING | IRQF_NO_THREAD, |
| 1883 | 1870 | dev_name(&pdev->dev), xgene_pmu); |
| 1884 | 1871 | if (rc) { |
| 1885 | 1872 | dev_err(&pdev->dev, "Could not request IRQ %d\n", irq); |
| 1886 | − goto err; | |
| 1873 | + return rc; | |
| 1887 | 1874 | } |
| 1888 | 1875 | |
| 1889 | 1876 | raw_spin_lock_init(&xgene_pmu->lock); |
| @@ -1903,42 +1890,29 @@ static int xgene_pmu_probe(struct platform_device *pdev) · 13 unmodified lines | ||
| 1903 | 1890 | rc = irq_set_affinity(irq, &xgene_pmu->cpu); |
| 1904 | 1891 | if (rc) { |
| 1905 | 1892 | dev_err(&pdev->dev, "Failed to set interrupt affinity!\n"); |
| 1906 | − goto err; | |
| 1893 | + return rc; | |
| 1907 | 1894 | } |
| 1908 | 1895 | |
| 1909 | 1896 | /* Walk through the tree for all PMU perf devices */ |
| 1910 | 1897 | rc = xgene_pmu_probe_pmu_dev(xgene_pmu, pdev); |
| 1911 | 1898 | if (rc) { |
| 1912 | 1899 | dev_err(&pdev->dev, "No PMU perf devices found!\n"); |
| 1913 | − goto err; | |
| 1900 | + return rc; | |
| 1914 | 1901 | } |
| 1915 | 1902 | |
| 1916 | 1903 | /* Enable interrupt */ |
| 1917 | 1904 | xgene_pmu->ops->unmask_int(xgene_pmu); |
| 1918 | 1905 | |
| 1919 | 1906 | return 0; |
| 1920 | − | |
| 1921 | −err: | |
| 1922 | − if (xgene_pmu->pcppmu_csr) | |
| 1923 | − devm_iounmap(&pdev->dev, xgene_pmu->pcppmu_csr); | |
| 1924 | − devm_kfree(&pdev->dev, xgene_pmu); | |
| 1925 | − | |
| 1926 | − return rc; | |
| 1927 | 1907 | } |
| 1928 | 1908 | |
| 1929 | 1909 | static void |
| 1930 | 1910 | xgene_pmu_dev_cleanup(struct xgene_pmu *xgene_pmu, struct list_head *pmus) |
| 1931 | 1911 | { |
| 1932 | 1912 | struct xgene_pmu_dev_ctx *ctx; |
| 1933 | − struct device *dev = xgene_pmu->dev; | |
| 1934 | − struct xgene_pmu_dev *pmu_dev; | |
| 1935 | 1913 | |
| 1936 | 1914 | list_for_each_entry(ctx, pmus, next) { |
| 1937 | − pmu_dev = ctx->pmu_dev; | |
| 1938 | − if (pmu_dev->inf->csr) | |
| 1939 | − devm_iounmap(dev, pmu_dev->inf->csr); | |
| 1940 | − devm_kfree(dev, ctx); | |
| 1941 | − devm_kfree(dev, pmu_dev); | |
| 1915 | + perf_pmu_unregister(&ctx->pmu_dev->pmu); | |
| 1942 | 1916 | } |
| 1943 | 1917 | } |
| 1944 | 1918 | |
| @@ -1951,10 +1925,6 @@ static int xgene_pmu_remove(struct platform_device *pdev) · 6 unmodified lines | ||
| 1951 | 1925 | xgene_pmu_dev_cleanup(xgene_pmu, &xgene_pmu->mcbpmus); |
| 1952 | 1926 | xgene_pmu_dev_cleanup(xgene_pmu, &xgene_pmu->mcpmus); |
| 1953 | 1927 | |
| 1954 | − if (xgene_pmu->pcppmu_csr) | |
| 1955 | − devm_iounmap(&pdev->dev, xgene_pmu->pcppmu_csr); | |
| 1956 | − devm_kfree(&pdev->dev, xgene_pmu); | |
| 1957 | − | |
| 1958 | 1928 | return 0; |
| 1959 | 1929 | } |
| 1960 | 1930 | |
| @@ -121,6 +121,7 @@ extern const struct raid6_recov_calls raid6_recov_ssse3; · 120 unmodified lines | ||
| 121 | 121 | extern const struct raid6_recov_calls raid6_recov_avx2; |
| 122 | 122 | extern const struct raid6_recov_calls raid6_recov_avx512; |
| 123 | 123 | extern const struct raid6_recov_calls raid6_recov_s390xc; |
| 124 | +extern const struct raid6_recov_calls raid6_recov_neon; | |
| 124 | 125 | |
| 125 | 126 | extern const struct raid6_calls raid6_neonx1; |
| 126 | 127 | extern const struct raid6_calls raid6_neonx2; |
| @@ -38,6 +38,10 @@ enum { · 37 unmodified lines | ||
| 38 | 38 | |
| 39 | 39 | #ifdef __KERNEL__ |
| 40 | 40 | |
| 41 | +#ifndef THREAD_ALIGN | |
| 42 | +#define THREAD_ALIGN THREAD_SIZE | |
| 43 | +#endif | |
| 44 | + | |
| 41 | 45 | #ifdef CONFIG_DEBUG_STACK_USAGE |
| 42 | 46 | # define THREADINFO_GFP (GFP_KERNEL_ACCOUNT | __GFP_NOTRACK | \ |
| 43 | 47 | __GFP_ZERO) |
| @@ -88,6 +88,7 @@ · 87 unmodified lines | ||
| 88 | 88 | #include <linux/sysctl.h> |
| 89 | 89 | #include <linux/kcov.h> |
| 90 | 90 | #include <linux/livepatch.h> |
| 91 | +#include <linux/thread_info.h> | |
| 91 | 92 | |
| 92 | 93 | #include <asm/pgtable.h> |
| 93 | 94 | #include <asm/pgalloc.h> |
| @@ -217,7 +218,7 @@ static unsigned long *alloc_thread_stack_node(struct task_struct *tsk, int node) · 123 unmodified lines | ||
| 217 | 218 | return s->addr; |
| 218 | 219 | } |
| 219 | 220 | |
| 220 | − stack = __vmalloc_node_range(THREAD_SIZE, THREAD_SIZE, | |
| 221 | + stack = __vmalloc_node_range(THREAD_SIZE, THREAD_ALIGN, | |
| 221 | 222 | VMALLOC_START, VMALLOC_END, |
| 222 | 223 | THREADINFO_GFP, |
| 223 | 224 | PAGE_KERNEL, |
| @@ -5,7 +5,7 @@ raid6_pq-y += algos.o recov.o tables.o int1.o int2.o int4.o \ · 4 unmodified lines | ||
| 5 | 5 | |
| 6 | 6 | raid6_pq-$(CONFIG_X86) += recov_ssse3.o recov_avx2.o mmx.o sse1.o sse2.o avx2.o avx512.o recov_avx512.o |
| 7 | 7 | raid6_pq-$(CONFIG_ALTIVEC) += altivec1.o altivec2.o altivec4.o altivec8.o |
| 8 | −raid6_pq-$(CONFIG_KERNEL_MODE_NEON) += neon.o neon1.o neon2.o neon4.o neon8.o | |
| 8 | +raid6_pq-$(CONFIG_KERNEL_MODE_NEON) += neon.o neon1.o neon2.o neon4.o neon8.o recov_neon.o recov_neon_inner.o | |
| 9 | 9 | raid6_pq-$(CONFIG_TILEGX) += tilegx8.o |
| 10 | 10 | raid6_pq-$(CONFIG_S390) += s390vx8.o recov_s390xc.o |
| 11 | 11 | |
| @@ -26,7 +26,9 @@ NEON_FLAGS := -ffreestanding · 14 unmodified lines | ||
| 26 | 26 | ifeq ($(ARCH),arm) |
| 27 | 27 | NEON_FLAGS += -mfloat-abi=softfp -mfpu=neon |
| 28 | 28 | endif |
| 29 | +CFLAGS_recov_neon_inner.o += $(NEON_FLAGS) | |
| 29 | 30 | ifeq ($(ARCH),arm64) |
| 31 | +CFLAGS_REMOVE_recov_neon_inner.o += -mgeneral-regs-only | |
| 30 | 32 | CFLAGS_REMOVE_neon1.o += -mgeneral-regs-only |
| 31 | 33 | CFLAGS_REMOVE_neon2.o += -mgeneral-regs-only |
| 32 | 34 | CFLAGS_REMOVE_neon4.o += -mgeneral-regs-only |
| @@ -112,6 +112,9 @@ const struct raid6_recov_calls *const raid6_recov_algos[] = { · 111 unmodified lines | ||
| 112 | 112 | #endif |
| 113 | 113 | #ifdef CONFIG_S390 |
| 114 | 114 | &raid6_recov_s390xc, |
| 115 | +#endif | |
| 116 | +#if defined(CONFIG_KERNEL_MODE_NEON) | |
| 117 | + &raid6_recov_neon, | |
| 115 | 118 | #endif |
| 116 | 119 | &raid6_recov_intx1, |
| 117 | 120 | NULL |
| @@ -46,8 +46,12 @@ static inline unative_t SHLBYTE(unative_t v) · 45 unmodified lines | ||
| 46 | 46 | */ |
| 47 | 47 | static inline unative_t MASK(unative_t v) |
| 48 | 48 | { |
| 49 | − const uint8x16_t temp = NBYTES(0); | |
| 50 | − return (unative_t)vcltq_s8((int8x16_t)v, (int8x16_t)temp); | |
| 49 | + return (unative_t)vshrq_n_s8((int8x16_t)v, 7); | |
| 50 | +} | |
| 51 | + | |
| 52 | +static inline unative_t PMUL(unative_t v, unative_t u) | |
| 53 | +{ | |
| 54 | + return (unative_t)vmulq_p8((poly8x16_t)v, (poly8x16_t)u); | |
| 51 | 55 | } |
| 52 | 56 | |
| 53 | 57 | void raid6_neon$#_gen_syndrome_real(int disks, unsigned long bytes, void **ptrs) |
| @@ -110,7 +114,30 @@ void raid6_neon$#_xor_syndrome_real(int disks, int start, int stop, · 56 unmodified lines | ||
| 110 | 114 | wq$$ = veorq_u8(w1$$, wd$$); |
| 111 | 115 | } |
| 112 | 116 | /* P/Q left side optimization */ |
| 113 | − for ( z = start-1 ; z >= 0 ; z-- ) { | |
| 117 | + for ( z = start-1 ; z >= 3 ; z -= 4 ) { | |
| 118 | + w2$$ = vshrq_n_u8(wq$$, 4); | |
| 119 | + w1$$ = vshlq_n_u8(wq$$, 4); | |
| 120 | + | |
| 121 | + w2$$ = PMUL(w2$$, x1d); | |
| 122 | + wq$$ = veorq_u8(w1$$, w2$$); | |
| 123 | + } | |
| 124 | + | |
| 125 | + switch (z) { | |
| 126 | + case 2: | |
| 127 | + w2$$ = vshrq_n_u8(wq$$, 5); | |
| 128 | + w1$$ = vshlq_n_u8(wq$$, 3); | |
| 129 | + | |
| 130 | + w2$$ = PMUL(w2$$, x1d); | |
| 131 | + wq$$ = veorq_u8(w1$$, w2$$); | |
| 132 | + break; | |
| 133 | + case 1: | |
| 134 | + w2$$ = vshrq_n_u8(wq$$, 6); | |
| 135 | + w1$$ = vshlq_n_u8(wq$$, 2); | |
| 136 | + | |
| 137 | + w2$$ = PMUL(w2$$, x1d); | |
| 138 | + wq$$ = veorq_u8(w1$$, w2$$); | |
| 139 | + break; | |
| 140 | + case 0: | |
| 114 | 141 | w2$$ = MASK(wq$$); |
| 115 | 142 | w1$$ = SHLBYTE(wq$$); |
| 116 | 143 | |
| @@ -0,0 +1,110 @@ | ||
| 1 | +/* | |
| 2 | + * Copyright (C) 2012 Intel Corporation | |
| 3 | + * Copyright (C) 2017 Linaro Ltd. <ard.biesheuvel@linaro.org> | |
| 4 | + * | |
| 5 | + * This program is free software; you can redistribute it and/or | |
| 6 | + * modify it under the terms of the GNU General Public License | |
| 7 | + * as published by the Free Software Foundation; version 2 | |
| 8 | + * of the License. | |
| 9 | + */ | |
| 10 | + | |
| 11 | +#include <linux/raid/pq.h> | |
| 12 | + | |
| 13 | +#ifdef __KERNEL__ | |
| 14 | +#include <asm/neon.h> | |
| 15 | +#else | |
| 16 | +#define kernel_neon_begin() | |
| 17 | +#define kernel_neon_end() | |
| 18 | +#define cpu_has_neon() (1) | |
| 19 | +#endif | |
| 20 | + | |
| 21 | +static int raid6_has_neon(void) | |
| 22 | +{ | |
| 23 | + return cpu_has_neon(); | |
| 24 | +} | |
| 25 | + | |
| 26 | +void __raid6_2data_recov_neon(int bytes, uint8_t *p, uint8_t *q, uint8_t *dp, | |
| 27 | + uint8_t *dq, const uint8_t *pbmul, | |
| 28 | + const uint8_t *qmul); | |
| 29 | + | |
| 30 | +void __raid6_datap_recov_neon(int bytes, uint8_t *p, uint8_t *q, uint8_t *dq, | |
| 31 | + const uint8_t *qmul); | |
| 32 | + | |
| 33 | +static void raid6_2data_recov_neon(int disks, size_t bytes, int faila, | |
| 34 | + int failb, void **ptrs) | |
| 35 | +{ | |
| 36 | + u8 *p, *q, *dp, *dq; | |
| 37 | + const u8 *pbmul; /* P multiplier table for B data */ | |
| 38 | + const u8 *qmul; /* Q multiplier table (for both) */ | |
| 39 | + | |
| 40 | + p = (u8 *)ptrs[disks - 2]; | |
| 41 | + q = (u8 *)ptrs[disks - 1]; | |
| 42 | + | |
| 43 | + /* | |
| 44 | + * Compute syndrome with zero for the missing data pages | |
| 45 | + * Use the dead data pages as temporary storage for | |
| 46 | + * delta p and delta q | |
| 47 | + */ | |
| 48 | + dp = (u8 *)ptrs[faila]; | |
| 49 | + ptrs[faila] = (void *)raid6_empty_zero_page; | |
| 50 | + ptrs[disks - 2] = dp; | |
| 51 | + dq = (u8 *)ptrs[failb]; | |
| 52 | + ptrs[failb] = (void *)raid6_empty_zero_page; | |
| 53 | + ptrs[disks - 1] = dq; | |
| 54 | + | |
| 55 | + raid6_call.gen_syndrome(disks, bytes, ptrs); | |
| 56 | + | |
| 57 | + /* Restore pointer table */ | |
| 58 | + ptrs[faila] = dp; | |
| 59 | + ptrs[failb] = dq; | |
| 60 | + ptrs[disks - 2] = p; | |
| 61 | + ptrs[disks - 1] = q; | |
| 62 | + | |
| 63 | + /* Now, pick the proper data tables */ | |
| 64 | + pbmul = raid6_vgfmul[raid6_gfexi[failb-faila]]; | |
| 65 | + qmul = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila] ^ | |
| 66 | + raid6_gfexp[failb]]]; | |
| 67 | + | |
| 68 | + kernel_neon_begin(); | |
| 69 | + __raid6_2data_recov_neon(bytes, p, q, dp, dq, pbmul, qmul); | |
| 70 | + kernel_neon_end(); | |
| 71 | +} | |
| 72 | + | |
| 73 | +static void raid6_datap_recov_neon(int disks, size_t bytes, int faila, | |
| 74 | + void **ptrs) | |
| 75 | +{ | |
| 76 | + u8 *p, *q, *dq; | |
| 77 | + const u8 *qmul; /* Q multiplier table */ | |
| 78 | + | |
| 79 | + p = (u8 *)ptrs[disks - 2]; | |
| 80 | + q = (u8 *)ptrs[disks - 1]; | |
| 81 | + | |
| 82 | + /* | |
| 83 | + * Compute syndrome with zero for the missing data page | |
| 84 | + * Use the dead data page as temporary storage for delta q | |
| 85 | + */ | |
| 86 | + dq = (u8 *)ptrs[faila]; | |
| 87 | + ptrs[faila] = (void *)raid6_empty_zero_page; | |
| 88 | + ptrs[disks - 1] = dq; | |
| 89 | + | |
| 90 | + raid6_call.gen_syndrome(disks, bytes, ptrs); | |
| 91 | + | |
| 92 | + /* Restore pointer table */ | |
| 93 | + ptrs[faila] = dq; | |
| 94 | + ptrs[disks - 1] = q; | |
| 95 | + | |
| 96 | + /* Now, pick the proper data tables */ | |
| 97 | + qmul = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila]]]; | |
| 98 | + | |
| 99 | + kernel_neon_begin(); | |
| 100 | + __raid6_datap_recov_neon(bytes, p, q, dq, qmul); | |
| 101 | + kernel_neon_end(); | |
| 102 | +} | |
| 103 | + | |
| 104 | +const struct raid6_recov_calls raid6_recov_neon = { | |
| 105 | + .data2 = raid6_2data_recov_neon, | |
| 106 | + .datap = raid6_datap_recov_neon, | |
| 107 | + .valid = raid6_has_neon, | |
| 108 | + .name = "neon", | |
| 109 | + .priority = 10, | |
| 110 | +}; | |
| @@ -0,0 +1,117 @@ | ||
| 1 | +/* | |
| 2 | + * Copyright (C) 2012 Intel Corporation | |
| 3 | + * Copyright (C) 2017 Linaro Ltd. <ard.biesheuvel@linaro.org> | |
| 4 | + * | |
| 5 | + * This program is free software; you can redistribute it and/or | |
| 6 | + * modify it under the terms of the GNU General Public License | |
| 7 | + * as published by the Free Software Foundation; version 2 | |
| 8 | + * of the License. | |
| 9 | + */ | |
| 10 | + | |
| 11 | +#include <arm_neon.h> | |
| 12 | + | |
| 13 | +static const uint8x16_t x0f = { | |
| 14 | + 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, | |
| 15 | + 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, | |
| 16 | +}; | |
| 17 | + | |
| 18 | +#ifdef CONFIG_ARM | |
| 19 | +/* | |
| 20 | + * AArch32 does not provide this intrinsic natively because it does not | |
| 21 | + * implement the underlying instruction. AArch32 only provides a 64-bit | |
| 22 | + * wide vtbl.8 instruction, so use that instead. | |
| 23 | + */ | |
| 24 | +static uint8x16_t vqtbl1q_u8(uint8x16_t a, uint8x16_t b) | |
| 25 | +{ | |
| 26 | + union { | |
| 27 | + uint8x16_t val; | |
| 28 | + uint8x8x2_t pair; | |
| 29 | + } __a = { a }; | |
| 30 | + | |
| 31 | + return vcombine_u8(vtbl2_u8(__a.pair, vget_low_u8(b)), | |
| 32 | + vtbl2_u8(__a.pair, vget_high_u8(b))); | |
| 33 | +} | |
| 34 | +#endif | |
| 35 | + | |
| 36 | +void __raid6_2data_recov_neon(int bytes, uint8_t *p, uint8_t *q, uint8_t *dp, | |
| 37 | + uint8_t *dq, const uint8_t *pbmul, | |
| 38 | + const uint8_t *qmul) | |
| 39 | +{ | |
| 40 | + uint8x16_t pm0 = vld1q_u8(pbmul); | |
| 41 | + uint8x16_t pm1 = vld1q_u8(pbmul + 16); | |
| 42 | + uint8x16_t qm0 = vld1q_u8(qmul); | |
| 43 | + uint8x16_t qm1 = vld1q_u8(qmul + 16); | |
| 44 | + | |
| 45 | + /* | |
| 46 | + * while ( bytes-- ) { | |
| 47 | + * uint8_t px, qx, db; | |
| 48 | + * | |
| 49 | + * px = *p ^ *dp; | |
| 50 | + * qx = qmul[*q ^ *dq]; | |
| 51 | + * *dq++ = db = pbmul[px] ^ qx; | |
| 52 | + * *dp++ = db ^ px; | |
| 53 | + * p++; q++; | |
| 54 | + * } | |
| 55 | + */ | |
| 56 | + | |
| 57 | + while (bytes) { | |
| 58 | + uint8x16_t vx, vy, px, qx, db; | |
| 59 | + | |
| 60 | + px = veorq_u8(vld1q_u8(p), vld1q_u8(dp)); | |
| 61 | + vx = veorq_u8(vld1q_u8(q), vld1q_u8(dq)); | |
| 62 | + | |
| 63 | + vy = (uint8x16_t)vshrq_n_s16((int16x8_t)vx, 4); | |
| 64 | + vx = vqtbl1q_u8(qm0, vandq_u8(vx, x0f)); | |
| 65 | + vy = vqtbl1q_u8(qm1, vandq_u8(vy, x0f)); | |
| 66 | + qx = veorq_u8(vx, vy); | |
| 67 | + | |
| 68 | + vy = (uint8x16_t)vshrq_n_s16((int16x8_t)px, 4); | |
| 69 | + vx = vqtbl1q_u8(pm0, vandq_u8(px, x0f)); | |
| 70 | + vy = vqtbl1q_u8(pm1, vandq_u8(vy, x0f)); | |
| 71 | + vx = veorq_u8(vx, vy); | |
| 72 | + db = veorq_u8(vx, qx); | |
| 73 | + | |
| 74 | + vst1q_u8(dq, db); | |
| 75 | + vst1q_u8(dp, veorq_u8(db, px)); | |
| 76 | + | |
| 77 | + bytes -= 16; | |
| 78 | + p += 16; | |
| 79 | + q += 16; | |
| 80 | + dp += 16; | |
| 81 | + dq += 16; | |
| 82 | + } | |
| 83 | +} | |
| 84 | + | |
| 85 | +void __raid6_datap_recov_neon(int bytes, uint8_t *p, uint8_t *q, uint8_t *dq, | |
| 86 | + const uint8_t *qmul) | |
| 87 | +{ | |
| 88 | + uint8x16_t qm0 = vld1q_u8(qmul); | |
| 89 | + uint8x16_t qm1 = vld1q_u8(qmul + 16); | |
| 90 | + | |
| 91 | + /* | |
| 92 | + * while (bytes--) { | |
| 93 | + * *p++ ^= *dq = qmul[*q ^ *dq]; | |
| 94 | + * q++; dq++; | |
| 95 | + * } | |
| 96 | + */ | |
| 97 | + | |
| 98 | + while (bytes) { | |
| 99 | + uint8x16_t vx, vy; | |
| 100 | + | |
| 101 | + vx = veorq_u8(vld1q_u8(q), vld1q_u8(dq)); | |
| 102 | + | |
| 103 | + vy = (uint8x16_t)vshrq_n_s16((int16x8_t)vx, 4); | |
| 104 | + vx = vqtbl1q_u8(qm0, vandq_u8(vx, x0f)); | |
| 105 | + vy = vqtbl1q_u8(qm1, vandq_u8(vy, x0f)); | |
| 106 | + vx = veorq_u8(vx, vy); | |
| 107 | + vy = veorq_u8(vx, vld1q_u8(p)); | |
| 108 | + | |
| 109 | + vst1q_u8(dq, vx); | |
| 110 | + vst1q_u8(p, vy); | |
| 111 | + | |
| 112 | + bytes -= 16; | |
| 113 | + p += 16; | |
| 114 | + q += 16; | |
| 115 | + dq += 16; | |
| 116 | + } | |
| 117 | +} | |