History

Merge tag 'arm64-upstream' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux

Pull arm64 updates from Catalin Marinas: - VMAP_STACK support, allowing the kernel stacks to be allocated in the vmalloc space with a guard page for trapping stack overflows. One of the patches introduces THREAD_ALIGN and changes the generic alloc_thread_stack_node() to use this instead of THREAD_SIZE (no functional change for other architectures) - Contiguous PTE hugetlb support re-enabled (after being reverted a couple of times). We now have the semantics agreed in the generic mm layer together with API improvements so that the architecture code can detect between contiguous and non-contiguous huge PTEs - Initial support for persistent memory on ARM: DC CVAP instruction exposed to user space (HWCAP) and the in-kernel pmem API implemented - raid6 improvements for arm64: faster algorithm for the delta syndrome and implementation of the recovery routines using Neon - FP/SIMD refactoring and removal of support for Neon in interrupt context. This is in preparation for full SVE support - PTE accessors converted from inline asm to cmpxchg so that we can use LSE atomics if available (ARMv8.1) - Perf support for Cortex-A35 and A73 - Non-urgent fixes and cleanups * tag 'arm64-upstream' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux: (75 commits) arm64: cleanup {COMPAT_,}SET_PERSONALITY() macro arm64: introduce separated bits for mm_context_t flags arm64: hugetlb: Cleanup setup_hugepagesz arm64: Re-enable support for contiguous hugepages arm64: hugetlb: Override set_huge_swap_pte_at() to support contiguous hugepages arm64: hugetlb: Override huge_pte_clear() to support contiguous hugepages arm64: hugetlb: Handle swap entries in huge_pte_offset() for contiguous hugepages arm64: hugetlb: Add break-before-make logic for contiguous entries arm64: hugetlb: Spring clean huge pte accessors arm64: hugetlb: Introduce pte_pgprot helper arm64: hugetlb: set_huge_pte_at Add WARN_ON on !pte_present arm64: kexec: have own crash_smp_send_stop() for crash dump for nonpanic cores arm64: dma-mapping: Mark atomic_pool as __ro_after_init arm64: dma-mapping: Do not pass data to gen_pool_set_algo() arm64: Remove the !CONFIG_ARM64_HW_AFDBM alternative code paths arm64: Ignore hardware dirty bit updates in ptep_set_wrprotect() arm64: Move PTE_RDONLY bit handling out of set_pte_at() kvm: arm64: Convert kvm_set_s2pte_readonly() from inline asm to cmpxchg() arm64: Convert pte handling from inline asm to using (cmp)xchg arm64: neon/efi: Make EFI fpsimd save/restore variables static ...

Linus Torvalds committed 9 years ago

Commit 04759192 parents 9e85ae6 d1be5c981 files · +1840 −924Browse files

Unified · Split

81 files
  1. Documentation/arm64/cpu-feature-registers.txt +2 −0
  2. Documentation/devicetree/bindings/arm/pmu.txt +2 −0
  3. arch/arm64/Kconfig +13 −0
  4. arch/arm64/include/asm/Kbuild +0 −1
  5. arch/arm64/include/asm/asm-bug.h +54 −0
  6. arch/arm64/include/asm/assembler.h +24 −1
  7. arch/arm64/include/asm/bug.h +3 −32
  8. arch/arm64/include/asm/cacheflush.h +3 −1
  9. arch/arm64/include/asm/cpucaps.h +2 −1
  10. arch/arm64/include/asm/efi.h +11 −2
  11. arch/arm64/include/asm/elf.h +0 −2
  12. arch/arm64/include/asm/esr.h +31 −11
  13. arch/arm64/include/asm/fpsimd.h +3 −13
  14. arch/arm64/include/asm/fpsimdmacros.h +0 −56
  15. arch/arm64/include/asm/hugetlb.h +8 −1
  16. arch/arm64/include/asm/irq.h +0 −42
  17. arch/arm64/include/asm/kvm_mmu.h +9 −12
  18. arch/arm64/include/asm/memory.h +53 −0
  19. arch/arm64/include/asm/mmu.h +2 −0
  20. arch/arm64/include/asm/neon.h +13 −3
  21. arch/arm64/include/asm/numa.h +0 −3
  22. arch/arm64/include/asm/page-def.h +34 −0
  23. arch/arm64/include/asm/page.h +1 −11
  24. arch/arm64/include/asm/pgtable-prot.h +8 −10
  25. arch/arm64/include/asm/pgtable.h +41 −62
  26. arch/arm64/include/asm/processor.h +2 −2
  27. arch/arm64/include/asm/ptrace.h +30 −1
  28. arch/arm64/include/asm/signal32.h +0 −2
  29. arch/arm64/include/asm/simd.h +56 −0
  30. arch/arm64/include/asm/smp.h +1 −1
  31. arch/arm64/include/asm/stacktrace.h +59 −2
  32. arch/arm64/include/asm/string.h +4 −0
  33. arch/arm64/include/asm/sysreg.h +1 −0
  34. arch/arm64/include/asm/thread_info.h +4 −9
  35. arch/arm64/include/asm/traps.h +5 −0
  36. arch/arm64/include/asm/uaccess.h +12 −0
  37. arch/arm64/include/uapi/asm/hwcap.h +1 −0
  38. arch/arm64/kernel/asm-offsets.c +1 −0
  39. arch/arm64/kernel/cpufeature.c +13 −0
  40. arch/arm64/kernel/cpuinfo.c +1 −0
  41. arch/arm64/kernel/entry-fpsimd.S +0 −24
  42. arch/arm64/kernel/entry.S +183 −100
  43. arch/arm64/kernel/fpsimd.c +132 −40
  44. arch/arm64/kernel/head.S +13 −9
  45. arch/arm64/kernel/hibernate.c +2 −2
  46. arch/arm64/kernel/irq.c +38 −2
  47. arch/arm64/kernel/machine_kexec.c +1 −1
  48. arch/arm64/kernel/perf_callchain.c +0 −1
  49. arch/arm64/kernel/perf_event.c +85 −123
  50. arch/arm64/kernel/probes/uprobes.c +1 −1
  51. arch/arm64/kernel/process.c +9 −4
  52. arch/arm64/kernel/ptrace.c +3 −2
  53. arch/arm64/kernel/return_address.c +0 −1
  54. arch/arm64/kernel/signal.c +5 −5
  55. arch/arm64/kernel/signal32.c +1 −1
  56. arch/arm64/kernel/smp.c +12 −2
  57. arch/arm64/kernel/stacktrace.c +10 −50
  58. arch/arm64/kernel/time.c +0 −1
  59. arch/arm64/kernel/traps.c +49 −30
  60. arch/arm64/kernel/vdso.c +15 −0
  61. arch/arm64/kernel/vmlinux.lds.S +1 −17
  62. arch/arm64/kvm/hyp/s2-setup.c +1 −1
  63. arch/arm64/lib/Makefile +2 −0
  64. arch/arm64/lib/uaccess_flushcache.c +47 −0
  65. arch/arm64/mm/cache.S +28 −9
  66. arch/arm64/mm/dma-mapping.c +2 −2
  67. arch/arm64/mm/fault.c +58 −19
  68. arch/arm64/mm/flush.c +16 −0
  69. arch/arm64/mm/hugetlbpage.c +230 −84
  70. drivers/acpi/arm64/iort.c +87 −53
  71. drivers/firmware/efi/libstub/arm64-stub.c +4 −2
  72. drivers/perf/arm_pmu.c +6 −0
  73. drivers/perf/xgene_pmu.c +22 −52
  74. include/linux/raid/pq.h +1 −0
  75. include/linux/thread_info.h +4 −0
  76. kernel/fork.c +2 −1
  77. lib/raid6/Makefile +3 −1
  78. lib/raid6/algos.c +3 −0
  79. lib/raid6/neon.uc +30 −3
  80. lib/raid6/recov_neon.c +110 −0
  81. lib/raid6/recov_neon_inner.c +117 −0
Documentation/arm64/cpu-feature-registers.txt+2−0
@@ -179,6 +179,8 @@ infrastructure: · 178 unmodified lines
179179 | FCMA | [19-16] | y |
180180 |--------------------------------------------------|
181181 | JSCVT | [15-12] | y |
182+ |--------------------------------------------------|
183+ | DPB | [3-0] | y |
182184 x--------------------------------------------------x
183185
184186 Appendix I: Example
Documentation/devicetree/bindings/arm/pmu.txt+2−0
@@ -9,9 +9,11 @@ Required properties: · 8 unmodified lines
99 - compatible : should be one of
1010 "apm,potenza-pmu"
1111 "arm,armv8-pmuv3"
12+ "arm,cortex-a73-pmu"
1213 "arm,cortex-a72-pmu"
1314 "arm,cortex-a57-pmu"
1415 "arm,cortex-a53-pmu"
16+ "arm,cortex-a35-pmu"
1517 "arm,cortex-a17-pmu"
1618 "arm,cortex-a15-pmu"
1719 "arm,cortex-a12-pmu"
arch/arm64/Kconfig+13−0
@@ -75,6 +75,7 @@ config ARM64 · 74 unmodified lines
7575 select HAVE_ARCH_SECCOMP_FILTER
7676 select HAVE_ARCH_TRACEHOOK
7777 select HAVE_ARCH_TRANSPARENT_HUGEPAGE
78+ select HAVE_ARCH_VMAP_STACK
7879 select HAVE_ARM_SMCCC
7980 select HAVE_EBPF_JIT
8081 select HAVE_C_RECORDMCOUNT
@@ -960,6 +961,18 @@ config ARM64_UAO · 879 unmodified lines
960961 regular load/store instructions if the cpu does not implement the
961962 feature.
962963
964+config ARM64_PMEM
965+ bool "Enable support for persistent memory"
966+ select ARCH_HAS_PMEM_API
967+ select ARCH_HAS_UACCESS_FLUSHCACHE
968+ help
969+ Say Y to enable support for the persistent memory API based on the
970+ ARMv8.2 DCPoP feature.
971+
972+ The feature is detected at runtime, and the kernel will use DC CVAC
973+ operations if DC CVAP is not supported (following the behaviour of
974+ DC CVAP itself if the system does not define a point of persistence).
975+
963976 endmenu
964977
965978 config ARM64_MODULE_CMODEL_LARGE
arch/arm64/include/asm/Kbuild+0−1
@@ -20,7 +20,6 @@ generic-y += rwsem.h · 19 unmodified lines
2020 generic-y += segment.h
2121 generic-y += serial.h
2222 generic-y += set_memory.h
23−generic-y += simd.h
2423 generic-y += sizes.h
2524 generic-y += switch_to.h
2625 generic-y += trace_clock.h
arch/arm64/include/asm/asm-bug.h+54−0
@@ -0,0 +1,54 @@
1+#ifndef __ASM_ASM_BUG_H
2+/*
3+ * Copyright (C) 2017 ARM Limited
4+ *
5+ * This program is free software; you can redistribute it and/or modify
6+ * it under the terms of the GNU General Public License version 2 as
7+ * published by the Free Software Foundation.
8+ *
9+ * This program is distributed in the hope that it will be useful,
10+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
11+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+ * GNU General Public License for more details.
13+ *
14+ * You should have received a copy of the GNU General Public License
15+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
16+ */
17+#define __ASM_ASM_BUG_H
18+
19+#include <asm/brk-imm.h>
20+
21+#ifdef CONFIG_DEBUG_BUGVERBOSE
22+#define _BUGVERBOSE_LOCATION(file, line) __BUGVERBOSE_LOCATION(file, line)
23+#define __BUGVERBOSE_LOCATION(file, line) \
24+ .pushsection .rodata.str,"aMS",@progbits,1; \
25+ 2: .string file; \
26+ .popsection; \
27+ \
28+ .long 2b - 0b; \
29+ .short line;
30+#else
31+#define _BUGVERBOSE_LOCATION(file, line)
32+#endif
33+
34+#ifdef CONFIG_GENERIC_BUG
35+
36+#define __BUG_ENTRY(flags) \
37+ .pushsection __bug_table,"aw"; \
38+ .align 2; \
39+ 0: .long 1f - 0b; \
40+_BUGVERBOSE_LOCATION(__FILE__, __LINE__) \
41+ .short flags; \
42+ .popsection; \
43+ 1:
44+#else
45+#define __BUG_ENTRY(flags)
46+#endif
47+
48+#define ASM_BUG_FLAGS(flags) \
49+ __BUG_ENTRY(flags) \
50+ brk BUG_BRK_IMM
51+
52+#define ASM_BUG() ASM_BUG_FLAGS(0)
53+
54+#endif /* __ASM_ASM_BUG_H */
arch/arm64/include/asm/assembler.h+24−1
@@ -230,12 +230,18 @@ lr .req x30 // link register · 229 unmodified lines
230230 .endm
231231
232232 /*
233− * @dst: Result of per_cpu(sym, smp_processor_id())
233+ * @dst: Result of per_cpu(sym, smp_processor_id()), can be SP for
234+ * non-module code
234235 * @sym: The name of the per-cpu variable
235236 * @tmp: scratch register
236237 */
237238 .macro adr_this_cpu, dst, sym, tmp
239+#ifndef MODULE
240+ adrp \tmp, \sym
241+ add \dst, \tmp, #:lo12:\sym
242+#else
238243 adr_l \dst, \sym
244+#endif
239245 mrs \tmp, tpidr_el1
240246 add \dst, \dst, \tmp
241247 .endm
@@ -352,6 +358,12 @@ alternative_if_not ARM64_WORKAROUND_CLEAN_CACHE · 110 unmodified lines
352358 dc \op, \kaddr
353359 alternative_else
354360 dc civac, \kaddr
361+alternative_endif
362+ .elseif (\op == cvap)
363+alternative_if ARM64_HAS_DCPOP
364+ sys 3, c7, c12, 1, \kaddr // dc cvap
365+alternative_else
366+ dc cvac, \kaddr
355367 alternative_endif
356368 .else
357369 dc \op, \kaddr
@@ -403,6 +415,17 @@ alternative_endif · 45 unmodified lines
403415 .size __pi_##x, . - x; \
404416 ENDPROC(x)
405417
418+/*
419+ * Annotate a function as being unsuitable for kprobes.
420+ */
421+#ifdef CONFIG_KPROBES
422+#define NOKPROBE(x) \
423+ .pushsection "_kprobe_blacklist", "aw"; \
424+ .quad x; \
425+ .popsection;
426+#else
427+#define NOKPROBE(x)
428+#endif
406429 /*
407430 * Emit a 64-bit absolute little endian symbol reference in a way that
408431 * ensures that it will be resolved at build time, even when building a
arch/arm64/include/asm/bug.h+3−32
@@ -18,41 +18,12 @@ · 17 unmodified lines
1818 #ifndef _ARCH_ARM64_ASM_BUG_H
1919 #define _ARCH_ARM64_ASM_BUG_H
2020
21−#include <asm/brk-imm.h>
21+#include <linux/stringify.h>
2222
23−#ifdef CONFIG_DEBUG_BUGVERBOSE
24−#define _BUGVERBOSE_LOCATION(file, line) __BUGVERBOSE_LOCATION(file, line)
25−#define __BUGVERBOSE_LOCATION(file, line) \
26− ".pushsection .rodata.str,\"aMS\",@progbits,1\n" \
27− "2: .string \"" file "\"\n\t" \
28− ".popsection\n\t" \
29− \
30− ".long 2b - 0b\n\t" \
31− ".short " #line "\n\t"
32−#else
33−#define _BUGVERBOSE_LOCATION(file, line)
34−#endif
35−
36−#ifdef CONFIG_GENERIC_BUG
37−
38−#define __BUG_ENTRY(flags) \
39− ".pushsection __bug_table,\"aw\"\n\t" \
40− ".align 2\n\t" \
41− "0: .long 1f - 0b\n\t" \
42−_BUGVERBOSE_LOCATION(__FILE__, __LINE__) \
43− ".short " #flags "\n\t" \
44− ".popsection\n" \
45− "1: "
46−#else
47−#define __BUG_ENTRY(flags) ""
48−#endif
23+#include <asm/asm-bug.h>
4924
5025 #define __BUG_FLAGS(flags) \
51− asm volatile ( \
52− __BUG_ENTRY(flags) \
53− "brk %[imm]" :: [imm] "i" (BUG_BRK_IMM) \
54− );
55−
26+ asm volatile (__stringify(ASM_BUG_FLAGS(flags)));
5627
5728 #define BUG() do { \
5829 __BUG_FLAGS(0); \
arch/arm64/include/asm/cacheflush.h+3−1
@@ -67,7 +67,9 @@ · 66 unmodified lines
6767 */
6868 extern void flush_icache_range(unsigned long start, unsigned long end);
6969 extern void __flush_dcache_area(void *addr, size_t len);
70+extern void __inval_dcache_area(void *addr, size_t len);
7071 extern void __clean_dcache_area_poc(void *addr, size_t len);
72+extern void __clean_dcache_area_pop(void *addr, size_t len);
7173 extern void __clean_dcache_area_pou(void *addr, size_t len);
7274 extern long __flush_cache_user_range(unsigned long start, unsigned long end);
7375 extern void sync_icache_aliases(void *kaddr, unsigned long len);
@@ -150,6 +152,6 @@ static inline void flush_cache_vunmap(unsigned long start, unsigned long end) · 76 unmodified lines
150152 {
151153 }
152154
153−int set_memory_valid(unsigned long addr, unsigned long size, int enable);
155+int set_memory_valid(unsigned long addr, int numpages, int enable);
154156
155157 #endif
arch/arm64/include/asm/cpucaps.h+2−1
@@ -39,7 +39,8 @@ · 38 unmodified lines
3939 #define ARM64_WORKAROUND_QCOM_FALKOR_E1003 18
4040 #define ARM64_WORKAROUND_858921 19
4141 #define ARM64_WORKAROUND_CAVIUM_30115 20
42+#define ARM64_HAS_DCPOP 21
4243
43−#define ARM64_NCAPS 21
44+#define ARM64_NCAPS 22
4445
4546 #endif /* __ASM_CPUCAPS_H */
arch/arm64/include/asm/efi.h+11−2
@@ -3,7 +3,9 @@ · 2 unmodified lines
33
44 #include <asm/boot.h>
55 #include <asm/cpufeature.h>
6+#include <asm/fpsimd.h>
67 #include <asm/io.h>
8+#include <asm/memory.h>
79 #include <asm/mmu_context.h>
810 #include <asm/neon.h>
911 #include <asm/ptrace.h>
@@ -20,8 +22,8 @@ int efi_set_mapping_permissions(struct mm_struct *mm, efi_memory_desc_t *md); · 10 unmodified lines
2022
2123 #define arch_efi_call_virt_setup() \
2224 ({ \
23− kernel_neon_begin(); \
2425 efi_virtmap_load(); \
26+ __efi_fpsimd_begin(); \
2527 })
2628
2729 #define arch_efi_call_virt(p, f, args...) \
@@ -33,8 +35,8 @@ int efi_set_mapping_permissions(struct mm_struct *mm, efi_memory_desc_t *md); · 5 unmodified lines
3335
3436 #define arch_efi_call_virt_teardown() \
3537 ({ \
38+ __efi_fpsimd_end(); \
3639 efi_virtmap_unload(); \
37− kernel_neon_end(); \
3840 })
3941
4042 #define ARCH_EFI_IRQ_FLAGS_MASK (PSR_D_BIT | PSR_A_BIT | PSR_I_BIT | PSR_F_BIT)
@@ -48,6 +50,13 @@ int efi_set_mapping_permissions(struct mm_struct *mm, efi_memory_desc_t *md); · 7 unmodified lines
4850 */
4951 #define EFI_FDT_ALIGN SZ_2M /* used by allocate_new_fdt_and_exit_boot() */
5052
53+/*
54+ * In some configurations (e.g. VMAP_STACK && 64K pages), stacks built into the
55+ * kernel need greater alignment than we require the segments to be padded to.
56+ */
57+#define EFI_KIMG_ALIGN \
58+ (SEGMENT_ALIGN > THREAD_ALIGN ? SEGMENT_ALIGN : THREAD_ALIGN)
59+
5160 /* on arm64, the FDT may be located anywhere in system RAM */
5261 static inline unsigned long efi_get_max_fdt_addr(unsigned long dram_base)
5362 {
arch/arm64/include/asm/elf.h+0−2
@@ -139,7 +139,6 @@ typedef struct user_fpsimd_state elf_fpregset_t; · 138 unmodified lines
139139
140140 #define SET_PERSONALITY(ex) \
141141 ({ \
142− clear_bit(TIF_32BIT, &current->mm->context.flags); \
143142 clear_thread_flag(TIF_32BIT); \
144143 current->personality &= ~READ_IMPLIES_EXEC; \
145144 })
@@ -195,7 +194,6 @@ typedef compat_elf_greg_t compat_elf_gregset_t[COMPAT_ELF_NGREG]; · 49 unmodified lines
195194 */
196195 #define COMPAT_SET_PERSONALITY(ex) \
197196 ({ \
198− set_bit(TIF_32BIT, &current->mm->context.flags); \
199197 set_thread_flag(TIF_32BIT); \
200198 })
201199 #define COMPAT_ARCH_DLINFO
arch/arm64/include/asm/esr.h+31−11
@@ -77,16 +77,23 @@ · 76 unmodified lines
7777 #define ESR_ELx_EC_MASK (UL(0x3F) << ESR_ELx_EC_SHIFT)
7878 #define ESR_ELx_EC(esr) (((esr) & ESR_ELx_EC_MASK) >> ESR_ELx_EC_SHIFT)
7979
80−#define ESR_ELx_IL (UL(1) << 25)
80+#define ESR_ELx_IL_SHIFT (25)
81+#define ESR_ELx_IL (UL(1) << ESR_ELx_IL_SHIFT)
8182 #define ESR_ELx_ISS_MASK (ESR_ELx_IL - 1)
8283
8384 /* ISS field definitions shared by different classes */
84−#define ESR_ELx_WNR (UL(1) << 6)
85+#define ESR_ELx_WNR_SHIFT (6)
86+#define ESR_ELx_WNR (UL(1) << ESR_ELx_WNR_SHIFT)
8587
8688 /* Shared ISS field definitions for Data/Instruction aborts */
87−#define ESR_ELx_FnV (UL(1) << 10)
88−#define ESR_ELx_EA (UL(1) << 9)
89−#define ESR_ELx_S1PTW (UL(1) << 7)
89+#define ESR_ELx_SET_SHIFT (11)
90+#define ESR_ELx_SET_MASK (UL(3) << ESR_ELx_SET_SHIFT)
91+#define ESR_ELx_FnV_SHIFT (10)
92+#define ESR_ELx_FnV (UL(1) << ESR_ELx_FnV_SHIFT)
93+#define ESR_ELx_EA_SHIFT (9)
94+#define ESR_ELx_EA (UL(1) << ESR_ELx_EA_SHIFT)
95+#define ESR_ELx_S1PTW_SHIFT (7)
96+#define ESR_ELx_S1PTW (UL(1) << ESR_ELx_S1PTW_SHIFT)
9097
9198 /* Shared ISS fault status code(IFSC/DFSC) for Data/Instruction aborts */
9299 #define ESR_ELx_FSC (0x3F)
@@ -97,15 +104,20 @@ · 4 unmodified lines
97104 #define ESR_ELx_FSC_PERM (0x0C)
98105
99106 /* ISS field definitions for Data Aborts */
100−#define ESR_ELx_ISV (UL(1) << 24)
107+#define ESR_ELx_ISV_SHIFT (24)
108+#define ESR_ELx_ISV (UL(1) << ESR_ELx_ISV_SHIFT)
101109 #define ESR_ELx_SAS_SHIFT (22)
102110 #define ESR_ELx_SAS (UL(3) << ESR_ELx_SAS_SHIFT)
103−#define ESR_ELx_SSE (UL(1) << 21)
111+#define ESR_ELx_SSE_SHIFT (21)
112+#define ESR_ELx_SSE (UL(1) << ESR_ELx_SSE_SHIFT)
104113 #define ESR_ELx_SRT_SHIFT (16)
105114 #define ESR_ELx_SRT_MASK (UL(0x1F) << ESR_ELx_SRT_SHIFT)
106−#define ESR_ELx_SF (UL(1) << 15)
107−#define ESR_ELx_AR (UL(1) << 14)
108−#define ESR_ELx_CM (UL(1) << 8)
115+#define ESR_ELx_SF_SHIFT (15)
116+#define ESR_ELx_SF (UL(1) << ESR_ELx_SF_SHIFT)
117+#define ESR_ELx_AR_SHIFT (14)
118+#define ESR_ELx_AR (UL(1) << ESR_ELx_AR_SHIFT)
119+#define ESR_ELx_CM_SHIFT (8)
120+#define ESR_ELx_CM (UL(1) << ESR_ELx_CM_SHIFT)
109121
110122 /* ISS field definitions for exceptions taken in to Hyp */
111123 #define ESR_ELx_CV (UL(1) << 24)
@@ -157,9 +169,10 @@ · 45 unmodified lines
157169 /*
158170 * User space cache operations have the following sysreg encoding
159171 * in System instructions.
160− * op0=1, op1=3, op2=1, crn=7, crm={ 5, 10, 11, 14 }, WRITE (L=0)
172+ * op0=1, op1=3, op2=1, crn=7, crm={ 5, 10, 11, 12, 14 }, WRITE (L=0)
161173 */
162174 #define ESR_ELx_SYS64_ISS_CRM_DC_CIVAC 14
175+#define ESR_ELx_SYS64_ISS_CRM_DC_CVAP 12
163176 #define ESR_ELx_SYS64_ISS_CRM_DC_CVAU 11
164177 #define ESR_ELx_SYS64_ISS_CRM_DC_CVAC 10
165178 #define ESR_ELx_SYS64_ISS_CRM_IC_IVAU 5
@@ -209,6 +222,13 @@ · 43 unmodified lines
209222 #ifndef __ASSEMBLY__
210223 #include <asm/types.h>
211224
225+static inline bool esr_is_data_abort(u32 esr)
226+{
227+ const u32 ec = ESR_ELx_EC(esr);
228+
229+ return ec == ESR_ELx_EC_DABT_LOW || ec == ESR_ELx_EC_DABT_CUR;
230+}
231+
212232 const char *esr_get_class_string(u32 esr);
213233 #endif /* __ASSEMBLY */
214234
arch/arm64/include/asm/fpsimd.h+3−13
@@ -41,16 +41,6 @@ struct fpsimd_state { · 40 unmodified lines
4141 unsigned int cpu;
4242 };
4343
44−/*
45− * Struct for stacking the bottom 'n' FP/SIMD registers.
46− */
47−struct fpsimd_partial_state {
48− u32 fpsr;
49− u32 fpcr;
50− u32 num_regs;
51− __uint128_t vregs[32];
52−};
53−
5444
5545 #if defined(__KERNEL__) && defined(CONFIG_COMPAT)
5646 /* Masks for extracting the FPSR and FPCR from the FPSCR */
@@ -77,9 +67,9 @@ extern void fpsimd_update_current_state(struct fpsimd_state *state); · 20 unmodified lines
7767
7868 extern void fpsimd_flush_task_state(struct task_struct *target);
7969
80−extern void fpsimd_save_partial_state(struct fpsimd_partial_state *state,
81− u32 num_regs);
82−extern void fpsimd_load_partial_state(struct fpsimd_partial_state *state);
70+/* For use by EFI runtime services calls only */
71+extern void __efi_fpsimd_begin(void);
72+extern void __efi_fpsimd_end(void);
8373
8474 #endif
8575
arch/arm64/include/asm/fpsimdmacros.h+0−56
@@ -75,59 +75,3 @@ · 74 unmodified lines
7575 ldr w\tmpnr, [\state, #16 * 2 + 4]
7676 fpsimd_restore_fpcr x\tmpnr, \state
7777 .endm
78−
79−.macro fpsimd_save_partial state, numnr, tmpnr1, tmpnr2
80− mrs x\tmpnr1, fpsr
81− str w\numnr, [\state, #8]
82− mrs x\tmpnr2, fpcr
83− stp w\tmpnr1, w\tmpnr2, [\state]
84− adr x\tmpnr1, 0f
85− add \state, \state, x\numnr, lsl #4
86− sub x\tmpnr1, x\tmpnr1, x\numnr, lsl #1
87− br x\tmpnr1
88− stp q30, q31, [\state, #-16 * 30 - 16]
89− stp q28, q29, [\state, #-16 * 28 - 16]
90− stp q26, q27, [\state, #-16 * 26 - 16]
91− stp q24, q25, [\state, #-16 * 24 - 16]
92− stp q22, q23, [\state, #-16 * 22 - 16]
93− stp q20, q21, [\state, #-16 * 20 - 16]
94− stp q18, q19, [\state, #-16 * 18 - 16]
95− stp q16, q17, [\state, #-16 * 16 - 16]
96− stp q14, q15, [\state, #-16 * 14 - 16]
97− stp q12, q13, [\state, #-16 * 12 - 16]
98− stp q10, q11, [\state, #-16 * 10 - 16]
99− stp q8, q9, [\state, #-16 * 8 - 16]
100− stp q6, q7, [\state, #-16 * 6 - 16]
101− stp q4, q5, [\state, #-16 * 4 - 16]
102− stp q2, q3, [\state, #-16 * 2 - 16]
103− stp q0, q1, [\state, #-16 * 0 - 16]
104−0:
105−.endm
106−
107−.macro fpsimd_restore_partial state, tmpnr1, tmpnr2
108− ldp w\tmpnr1, w\tmpnr2, [\state]
109− msr fpsr, x\tmpnr1
110− fpsimd_restore_fpcr x\tmpnr2, x\tmpnr1
111− adr x\tmpnr1, 0f
112− ldr w\tmpnr2, [\state, #8]
113− add \state, \state, x\tmpnr2, lsl #4
114− sub x\tmpnr1, x\tmpnr1, x\tmpnr2, lsl #1
115− br x\tmpnr1
116− ldp q30, q31, [\state, #-16 * 30 - 16]
117− ldp q28, q29, [\state, #-16 * 28 - 16]
118− ldp q26, q27, [\state, #-16 * 26 - 16]
119− ldp q24, q25, [\state, #-16 * 24 - 16]
120− ldp q22, q23, [\state, #-16 * 22 - 16]
121− ldp q20, q21, [\state, #-16 * 20 - 16]
122− ldp q18, q19, [\state, #-16 * 18 - 16]
123− ldp q16, q17, [\state, #-16 * 16 - 16]
124− ldp q14, q15, [\state, #-16 * 14 - 16]
125− ldp q12, q13, [\state, #-16 * 12 - 16]
126− ldp q10, q11, [\state, #-16 * 10 - 16]
127− ldp q8, q9, [\state, #-16 * 8 - 16]
128− ldp q6, q7, [\state, #-16 * 6 - 16]
129− ldp q4, q5, [\state, #-16 * 4 - 16]
130− ldp q2, q3, [\state, #-16 * 2 - 16]
131− ldp q0, q1, [\state, #-16 * 0 - 16]
132−0:
133−.endm
arch/arm64/include/asm/hugetlb.h+8−1
@@ -18,7 +18,6 @@ · 17 unmodified lines
1818 #ifndef __ASM_HUGETLB_H
1919 #define __ASM_HUGETLB_H
2020
21−#include <asm-generic/hugetlb.h>
2221 #include <asm/page.h>
2322
2423 static inline pte_t huge_ptep_get(pte_t *ptep)
@@ -82,6 +81,14 @@ extern void huge_ptep_set_wrprotect(struct mm_struct *mm, · 57 unmodified lines
8281 unsigned long addr, pte_t *ptep);
8382 extern void huge_ptep_clear_flush(struct vm_area_struct *vma,
8483 unsigned long addr, pte_t *ptep);
84+extern void huge_pte_clear(struct mm_struct *mm, unsigned long addr,
85+ pte_t *ptep, unsigned long sz);
86+#define huge_pte_clear huge_pte_clear
87+extern void set_huge_swap_pte_at(struct mm_struct *mm, unsigned long addr,
88+ pte_t *ptep, pte_t pte, unsigned long sz);
89+#define set_huge_swap_pte_at set_huge_swap_pte_at
90+
91+#include <asm-generic/hugetlb.h>
8592
8693 #ifdef CONFIG_ARCH_HAS_GIGANTIC_PAGE
8794 static inline bool gigantic_page_supported(void) { return true; }
arch/arm64/include/asm/irq.h+0−42
@@ -1,45 +1,12 @@
11 #ifndef __ASM_IRQ_H
22 #define __ASM_IRQ_H
33
4−#define IRQ_STACK_SIZE THREAD_SIZE
5−#define IRQ_STACK_START_SP THREAD_START_SP
6−
74 #ifndef __ASSEMBLER__
85
9−#include <linux/percpu.h>
10−
116 #include <asm-generic/irq.h>
12−#include <asm/thread_info.h>
137
148 struct pt_regs;
159
16−DECLARE_PER_CPU(unsigned long [IRQ_STACK_SIZE/sizeof(long)], irq_stack);
17−
18−/*
19− * The highest address on the stack, and the first to be used. Used to
20− * find the dummy-stack frame put down by el?_irq() in entry.S, which
21− * is structured as follows:
22− *
23− * ------------
24− * | | <- irq_stack_ptr
25− * top ------------
26− * | x19 | <- irq_stack_ptr - 0x08
27− * ------------
28− * | x29 | <- irq_stack_ptr - 0x10
29− * ------------
30− *
31− * where x19 holds a copy of the task stack pointer where the struct pt_regs
32− * from kernel_entry can be found.
33− *
34− */
35−#define IRQ_STACK_PTR(cpu) ((unsigned long)per_cpu(irq_stack, cpu) + IRQ_STACK_START_SP)
36−
37−/*
38− * The offset from irq_stack_ptr where entry.S will store the original
39− * stack pointer. Used by unwind_frame() and dump_backtrace().
40− */
41−#define IRQ_STACK_TO_TASK_STACK(ptr) (*((unsigned long *)((ptr) - 0x08)))
42−
4310 extern void set_handle_irq(void (*handle_irq)(struct pt_regs *));
4411
4512 static inline int nr_legacy_irqs(void)
@@ -47,14 +14,5 @@ static inline int nr_legacy_irqs(void) · 1 unmodified line
4714 return 0;
4815 }
4916
50−static inline bool on_irq_stack(unsigned long sp, int cpu)
51−{
52− /* variable names the same as kernel/stacktrace.c */
53− unsigned long low = (unsigned long)per_cpu(irq_stack, cpu);
54− unsigned long high = low + IRQ_STACK_START_SP;
55−
56− return (low <= sp && sp <= high);
57−}
58−
5917 #endif /* !__ASSEMBLER__ */
6018 #endif
arch/arm64/include/asm/kvm_mmu.h+9−12
@@ -175,18 +175,15 @@ static inline pmd_t kvm_s2pmd_mkwrite(pmd_t pmd) · 174 unmodified lines
175175
176176 static inline void kvm_set_s2pte_readonly(pte_t *pte)
177177 {
178− pteval_t pteval;
179− unsigned long tmp;
180−
181− asm volatile("// kvm_set_s2pte_readonly\n"
182− " prfm pstl1strm, %2\n"
183− "1: ldxr %0, %2\n"
184− " and %0, %0, %3 // clear PTE_S2_RDWR\n"
185− " orr %0, %0, %4 // set PTE_S2_RDONLY\n"
186− " stxr %w1, %0, %2\n"
187− " cbnz %w1, 1b\n"
188− : "=&r" (pteval), "=&r" (tmp), "+Q" (pte_val(*pte))
189− : "L" (~PTE_S2_RDWR), "L" (PTE_S2_RDONLY));
178+ pteval_t old_pteval, pteval;
179+
180+ pteval = READ_ONCE(pte_val(*pte));
181+ do {
182+ old_pteval = pteval;
183+ pteval &= ~PTE_S2_RDWR;
184+ pteval |= PTE_S2_RDONLY;
185+ pteval = cmpxchg_relaxed(&pte_val(*pte), old_pteval, pteval);
186+ } while (pteval != old_pteval);
190187 }
191188
192189 static inline bool kvm_s2pte_readonly(pte_t *pte)
arch/arm64/include/asm/memory.h+53−0
@@ -25,6 +25,7 @@ · 24 unmodified lines
2525 #include <linux/const.h>
2626 #include <linux/types.h>
2727 #include <asm/bug.h>
28+#include <asm/page-def.h>
2829 #include <asm/sizes.h>
2930
3031 /*
@@ -103,6 +104,58 @@ · 72 unmodified lines
103104 #define KASAN_SHADOW_SIZE (0)
104105 #endif
105106
107+#define MIN_THREAD_SHIFT 14
108+
109+/*
110+ * VMAP'd stacks are allocated at page granularity, so we must ensure that such
111+ * stacks are a multiple of page size.
112+ */
113+#if defined(CONFIG_VMAP_STACK) && (MIN_THREAD_SHIFT < PAGE_SHIFT)
114+#define THREAD_SHIFT PAGE_SHIFT
115+#else
116+#define THREAD_SHIFT MIN_THREAD_SHIFT
117+#endif
118+
119+#if THREAD_SHIFT >= PAGE_SHIFT
120+#define THREAD_SIZE_ORDER (THREAD_SHIFT - PAGE_SHIFT)
121+#endif
122+
123+#define THREAD_SIZE (UL(1) << THREAD_SHIFT)
124+
125+/*
126+ * By aligning VMAP'd stacks to 2 * THREAD_SIZE, we can detect overflow by
127+ * checking sp & (1 << THREAD_SHIFT), which we can do cheaply in the entry
128+ * assembly.
129+ */
130+#ifdef CONFIG_VMAP_STACK
131+#define THREAD_ALIGN (2 * THREAD_SIZE)
132+#else
133+#define THREAD_ALIGN THREAD_SIZE
134+#endif
135+
136+#define IRQ_STACK_SIZE THREAD_SIZE
137+
138+#define OVERFLOW_STACK_SIZE SZ_4K
139+
140+/*
141+ * Alignment of kernel segments (e.g. .text, .data).
142+ */
143+#if defined(CONFIG_DEBUG_ALIGN_RODATA)
144+/*
145+ * 4 KB granule: 1 level 2 entry
146+ * 16 KB granule: 128 level 3 entries, with contiguous bit
147+ * 64 KB granule: 32 level 3 entries, with contiguous bit
148+ */
149+#define SEGMENT_ALIGN SZ_2M
150+#else
151+/*
152+ * 4 KB granule: 16 level 3 entries, with contiguous bit
153+ * 16 KB granule: 4 level 3 entries, without contiguous bit
154+ * 64 KB granule: 1 level 3 entry
155+ */
156+#define SEGMENT_ALIGN SZ_64K
157+#endif
158+
106159 /*
107160 * Memory types available.
108161 */
arch/arm64/include/asm/mmu.h+2−0
@@ -16,6 +16,8 @@ · 15 unmodified lines
1616 #ifndef __ASM_MMU_H
1717 #define __ASM_MMU_H
1818
19+#define MMCF_AARCH32 0x1 /* mm context flag for AArch32 executables */
20+
1921 typedef struct {
2022 atomic64_t id;
2123 void *vdso;
arch/arm64/include/asm/neon.h+13−3
@@ -8,12 +8,22 @@ · 7 unmodified lines
88 * published by the Free Software Foundation.
99 */
1010
11+#ifndef __ASM_NEON_H
12+#define __ASM_NEON_H
13+
1114 #include <linux/types.h>
1215 #include <asm/fpsimd.h>
1316
1417 #define cpu_has_neon() system_supports_fpsimd()
1518
16−#define kernel_neon_begin() kernel_neon_begin_partial(32)
17−
18−void kernel_neon_begin_partial(u32 num_regs);
19+void kernel_neon_begin(void);
1920 void kernel_neon_end(void);
21+
22+/*
23+ * Temporary macro to allow the crypto code to compile. Note that the
24+ * semantics of kernel_neon_begin_partial() are now different from the
25+ * original as it does not allow being called in an interrupt context.
26+ */
27+#define kernel_neon_begin_partial(num_regs) kernel_neon_begin()
28+
29+#endif /* ! __ASM_NEON_H */
arch/arm64/include/asm/numa.h+0−3
@@ -7,9 +7,6 @@ · 6 unmodified lines
77
88 #define NR_NODE_MEMBLKS (MAX_NUMNODES * 2)
99
10−/* currently, arm64 implements flat NUMA topology */
11−#define parent_node(node) (node)
12−
1310 int __node_distance(int from, int to);
1411 #define node_distance(a, b) __node_distance(a, b)
1512
arch/arm64/include/asm/page-def.h+34−0
@@ -0,0 +1,34 @@
1+/*
2+ * Based on arch/arm/include/asm/page.h
3+ *
4+ * Copyright (C) 1995-2003 Russell King
5+ * Copyright (C) 2017 ARM Ltd.
6+ *
7+ * This program is free software; you can redistribute it and/or modify
8+ * it under the terms of the GNU General Public License version 2 as
9+ * published by the Free Software Foundation.
10+ *
11+ * This program is distributed in the hope that it will be useful,
12+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
13+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14+ * GNU General Public License for more details.
15+ *
16+ * You should have received a copy of the GNU General Public License
17+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
18+ */
19+#ifndef __ASM_PAGE_DEF_H
20+#define __ASM_PAGE_DEF_H
21+
22+#include <linux/const.h>
23+
24+/* PAGE_SHIFT determines the page size */
25+/* CONT_SHIFT determines the number of pages which can be tracked together */
26+#define PAGE_SHIFT CONFIG_ARM64_PAGE_SHIFT
27+#define CONT_SHIFT CONFIG_ARM64_CONT_SHIFT
28+#define PAGE_SIZE (_AC(1, UL) << PAGE_SHIFT)
29+#define PAGE_MASK (~(PAGE_SIZE-1))
30+
31+#define CONT_SIZE (_AC(1, UL) << (CONT_SHIFT + PAGE_SHIFT))
32+#define CONT_MASK (~(CONT_SIZE-1))
33+
34+#endif /* __ASM_PAGE_DEF_H */
arch/arm64/include/asm/page.h+1−11
@@ -19,17 +19,7 @@ · 18 unmodified lines
1919 #ifndef __ASM_PAGE_H
2020 #define __ASM_PAGE_H
2121
22−#include <linux/const.h>
23−
24−/* PAGE_SHIFT determines the page size */
25−/* CONT_SHIFT determines the number of pages which can be tracked together */
26−#define PAGE_SHIFT CONFIG_ARM64_PAGE_SHIFT
27−#define CONT_SHIFT CONFIG_ARM64_CONT_SHIFT
28−#define PAGE_SIZE (_AC(1, UL) << PAGE_SHIFT)
29−#define PAGE_MASK (~(PAGE_SIZE-1))
30−
31−#define CONT_SIZE (_AC(1, UL) << (CONT_SHIFT + PAGE_SHIFT))
32−#define CONT_MASK (~(CONT_SIZE-1))
22+#include <asm/page-def.h>
3323
3424 #ifndef __ASSEMBLY__
3525
arch/arm64/include/asm/pgtable-prot.h+8−10
@@ -63,23 +63,21 @@ · 62 unmodified lines
6363 #define PAGE_S2 __pgprot(PROT_DEFAULT | PTE_S2_MEMATTR(MT_S2_NORMAL) | PTE_S2_RDONLY)
6464 #define PAGE_S2_DEVICE __pgprot(PROT_DEFAULT | PTE_S2_MEMATTR(MT_S2_DEVICE_nGnRE) | PTE_S2_RDONLY | PTE_UXN)
6565
66−#define PAGE_NONE __pgprot(((_PAGE_DEFAULT) & ~PTE_VALID) | PTE_PROT_NONE | PTE_PXN | PTE_UXN)
66+#define PAGE_NONE __pgprot(((_PAGE_DEFAULT) & ~PTE_VALID) | PTE_PROT_NONE | PTE_RDONLY | PTE_PXN | PTE_UXN)
6767 #define PAGE_SHARED __pgprot(_PAGE_DEFAULT | PTE_USER | PTE_NG | PTE_PXN | PTE_UXN | PTE_WRITE)
6868 #define PAGE_SHARED_EXEC __pgprot(_PAGE_DEFAULT | PTE_USER | PTE_NG | PTE_PXN | PTE_WRITE)
69−#define PAGE_COPY __pgprot(_PAGE_DEFAULT | PTE_USER | PTE_NG | PTE_PXN | PTE_UXN)
70−#define PAGE_COPY_EXEC __pgprot(_PAGE_DEFAULT | PTE_USER | PTE_NG | PTE_PXN)
71−#define PAGE_READONLY __pgprot(_PAGE_DEFAULT | PTE_USER | PTE_NG | PTE_PXN | PTE_UXN)
72−#define PAGE_READONLY_EXEC __pgprot(_PAGE_DEFAULT | PTE_USER | PTE_NG | PTE_PXN)
73−#define PAGE_EXECONLY __pgprot(_PAGE_DEFAULT | PTE_NG | PTE_PXN)
69+#define PAGE_READONLY __pgprot(_PAGE_DEFAULT | PTE_USER | PTE_RDONLY | PTE_NG | PTE_PXN | PTE_UXN)
70+#define PAGE_READONLY_EXEC __pgprot(_PAGE_DEFAULT | PTE_USER | PTE_RDONLY | PTE_NG | PTE_PXN)
71+#define PAGE_EXECONLY __pgprot(_PAGE_DEFAULT | PTE_RDONLY | PTE_NG | PTE_PXN)
7472
7573 #define __P000 PAGE_NONE
7674 #define __P001 PAGE_READONLY
77−#define __P010 PAGE_COPY
78−#define __P011 PAGE_COPY
75+#define __P010 PAGE_READONLY
76+#define __P011 PAGE_READONLY
7977 #define __P100 PAGE_EXECONLY
8078 #define __P101 PAGE_READONLY_EXEC
81−#define __P110 PAGE_COPY_EXEC
82−#define __P111 PAGE_COPY_EXEC
79+#define __P110 PAGE_READONLY_EXEC
80+#define __P111 PAGE_READONLY_EXEC
8381
8482 #define __S000 PAGE_NONE
8583 #define __S001 PAGE_READONLY
arch/arm64/include/asm/pgtable.h+41−62
@@ -39,6 +39,7 @@ · 38 unmodified lines
3939
4040 #ifndef __ASSEMBLY__
4141
42+#include <asm/cmpxchg.h>
4243 #include <asm/fixmap.h>
4344 #include <linux/mmdebug.h>
4445
@@ -84,11 +85,7 @@ extern unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)]; · 39 unmodified lines
8485 (__boundary - 1 < (end) - 1) ? __boundary : (end); \
8586 })
8687
87−#ifdef CONFIG_ARM64_HW_AFDBM
8888 #define pte_hw_dirty(pte) (pte_write(pte) && !(pte_val(pte) & PTE_RDONLY))
89−#else
90−#define pte_hw_dirty(pte) (0)
91−#endif
9289 #define pte_sw_dirty(pte) (!!(pte_val(pte) & PTE_DIRTY))
9390 #define pte_dirty(pte) (pte_sw_dirty(pte) || pte_hw_dirty(pte))
9491
@@ -124,12 +121,16 @@ static inline pte_t set_pte_bit(pte_t pte, pgprot_t prot) · 29 unmodified lines
124121
125122 static inline pte_t pte_wrprotect(pte_t pte)
126123 {
127− return clear_pte_bit(pte, __pgprot(PTE_WRITE));
124+ pte = clear_pte_bit(pte, __pgprot(PTE_WRITE));
125+ pte = set_pte_bit(pte, __pgprot(PTE_RDONLY));
126+ return pte;
128127 }
129128
130129 static inline pte_t pte_mkwrite(pte_t pte)
131130 {
132− return set_pte_bit(pte, __pgprot(PTE_WRITE));
131+ pte = set_pte_bit(pte, __pgprot(PTE_WRITE));
132+ pte = clear_pte_bit(pte, __pgprot(PTE_RDONLY));
133+ return pte;
133134 }
134135
135136 static inline pte_t pte_mkclean(pte_t pte)
@@ -168,11 +169,6 @@ static inline pte_t pte_mknoncont(pte_t pte) · 32 unmodified lines
168169 return clear_pte_bit(pte, __pgprot(PTE_CONT));
169170 }
170171
171−static inline pte_t pte_clear_rdonly(pte_t pte)
172−{
173− return clear_pte_bit(pte, __pgprot(PTE_RDONLY));
174−}
175−
176172 static inline pte_t pte_mkpresent(pte_t pte)
177173 {
178174 return set_pte_bit(pte, __pgprot(PTE_VALID));
@@ -220,22 +216,15 @@ extern void __sync_icache_dcache(pte_t pteval, unsigned long addr); · 41 unmodified lines
220216 static inline void set_pte_at(struct mm_struct *mm, unsigned long addr,
221217 pte_t *ptep, pte_t pte)
222218 {
223− if (pte_present(pte)) {
224− if (pte_sw_dirty(pte) && pte_write(pte))
225− pte_val(pte) &= ~PTE_RDONLY;
226− else
227− pte_val(pte) |= PTE_RDONLY;
228− if (pte_user_exec(pte) && !pte_special(pte))
229− __sync_icache_dcache(pte, addr);
230− }
219+ if (pte_present(pte) && pte_user_exec(pte) && !pte_special(pte))
220+ __sync_icache_dcache(pte, addr);
231221
232222 /*
233223 * If the existing pte is valid, check for potential race with
234224 * hardware updates of the pte (ptep_set_access_flags safely changes
235225 * valid ptes without going through an invalid entry).
236226 */
237− if (IS_ENABLED(CONFIG_ARM64_HW_AFDBM) &&
238− pte_valid(*ptep) && pte_valid(pte)) {
227+ if (pte_valid(*ptep) && pte_valid(pte)) {
239228 VM_WARN_ONCE(!pte_young(pte),
240229 "%s: racy access flag clearing: 0x%016llx -> 0x%016llx",
241230 __func__, pte_val(*ptep), pte_val(pte));
@@ -571,7 +560,6 @@ static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot) · 329 unmodified lines
571560 return pte_pmd(pte_modify(pmd_pte(pmd), newprot));
572561 }
573562
574−#ifdef CONFIG_ARM64_HW_AFDBM
575563 #define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS
576564 extern int ptep_set_access_flags(struct vm_area_struct *vma,
577565 unsigned long address, pte_t *ptep,
@@ -593,20 +581,17 @@ static inline int pmdp_set_access_flags(struct vm_area_struct *vma, · 15 unmodified lines
593581 #define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
594582 static inline int __ptep_test_and_clear_young(pte_t *ptep)
595583 {
596− pteval_t pteval;
597− unsigned int tmp, res;
584+ pte_t old_pte, pte;
598585
599− asm volatile("// __ptep_test_and_clear_young\n"
600− " prfm pstl1strm, %2\n"
601− "1: ldxr %0, %2\n"
602− " ubfx %w3, %w0, %5, #1 // extract PTE_AF (young)\n"
603− " and %0, %0, %4 // clear PTE_AF\n"
604− " stxr %w1, %0, %2\n"
605− " cbnz %w1, 1b\n"
606− : "=&r" (pteval), "=&r" (tmp), "+Q" (pte_val(*ptep)), "=&r" (res)
607− : "L" (~PTE_AF), "I" (ilog2(PTE_AF)));
586+ pte = READ_ONCE(*ptep);
587+ do {
588+ old_pte = pte;
589+ pte = pte_mkold(pte);
590+ pte_val(pte) = cmpxchg_relaxed(&pte_val(*ptep),
591+ pte_val(old_pte), pte_val(pte));
592+ } while (pte_val(pte) != pte_val(old_pte));
608593
609− return res;
594+ return pte_young(pte);
610595 }
611596
612597 static inline int ptep_test_and_clear_young(struct vm_area_struct *vma,
@@ -630,17 +615,7 @@ static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma, · 17 unmodified lines
630615 static inline pte_t ptep_get_and_clear(struct mm_struct *mm,
631616 unsigned long address, pte_t *ptep)
632617 {
633− pteval_t old_pteval;
634− unsigned int tmp;
635−
636− asm volatile("// ptep_get_and_clear\n"
637− " prfm pstl1strm, %2\n"
638− "1: ldxr %0, %2\n"
639− " stxr %w1, xzr, %2\n"
640− " cbnz %w1, 1b\n"
641− : "=&r" (old_pteval), "=&r" (tmp), "+Q" (pte_val(*ptep)));
642−
643− return __pte(old_pteval);
618+ return __pte(xchg_relaxed(&pte_val(*ptep), 0));
644619 }
645620
646621 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
@@ -653,27 +628,32 @@ static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm, · 6 unmodified lines
653628 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
654629
655630 /*
656− * ptep_set_wrprotect - mark read-only while trasferring potential hardware
657− * dirty status (PTE_DBM && !PTE_RDONLY) to the software PTE_DIRTY bit.
631+ * ptep_set_wrprotect - mark read-only while preserving the hardware update of
632+ * the Access Flag.
658633 */
659634 #define __HAVE_ARCH_PTEP_SET_WRPROTECT
660635 static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long address, pte_t *ptep)
661636 {
662− pteval_t pteval;
663− unsigned long tmp;
637+ pte_t old_pte, pte;
664638
665− asm volatile("// ptep_set_wrprotect\n"
666− " prfm pstl1strm, %2\n"
667− "1: ldxr %0, %2\n"
668− " tst %0, %4 // check for hw dirty (!PTE_RDONLY)\n"
669− " csel %1, %3, xzr, eq // set PTE_DIRTY|PTE_RDONLY if dirty\n"
670− " orr %0, %0, %1 // if !dirty, PTE_RDONLY is already set\n"
671− " and %0, %0, %5 // clear PTE_WRITE/PTE_DBM\n"
672− " stxr %w1, %0, %2\n"
673− " cbnz %w1, 1b\n"
674− : "=&r" (pteval), "=&r" (tmp), "+Q" (pte_val(*ptep))
675− : "r" (PTE_DIRTY|PTE_RDONLY), "L" (PTE_RDONLY), "L" (~PTE_WRITE)
676− : "cc");
639+ /*
640+ * ptep_set_wrprotect() is only called on CoW mappings which are
641+ * private (!VM_SHARED) with the pte either read-only (!PTE_WRITE &&
642+ * PTE_RDONLY) or writable and software-dirty (PTE_WRITE &&
643+ * !PTE_RDONLY && PTE_DIRTY); see is_cow_mapping() and
644+ * protection_map[]. There is no race with the hardware update of the
645+ * dirty state: clearing of PTE_RDONLY when PTE_WRITE (a.k.a. PTE_DBM)
646+ * is set.
647+ */
648+ VM_WARN_ONCE(pte_write(*ptep) && !pte_dirty(*ptep),
649+ "%s: potential race with hardware DBM", __func__);
650+ pte = READ_ONCE(*ptep);
651+ do {
652+ old_pte = pte;
653+ pte = pte_wrprotect(pte);
654+ pte_val(pte) = cmpxchg_relaxed(&pte_val(*ptep),
655+ pte_val(old_pte), pte_val(pte));
656+ } while (pte_val(pte) != pte_val(old_pte));
677657 }
678658
679659 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
@@ -684,7 +664,6 @@ static inline void pmdp_set_wrprotect(struct mm_struct *mm, · 4 unmodified lines
684664 ptep_set_wrprotect(mm, address, (pte_t *)pmdp);
685665 }
686666 #endif
687−#endif /* CONFIG_ARM64_HW_AFDBM */
688667
689668 extern pgd_t swapper_pg_dir[PTRS_PER_PGD];
690669 extern pgd_t idmap_pg_dir[PTRS_PER_PGD];
arch/arm64/include/asm/processor.h+2−2
@@ -112,7 +112,7 @@ void tls_preserve_current_state(void); · 111 unmodified lines
112112 static inline void start_thread_common(struct pt_regs *regs, unsigned long pc)
113113 {
114114 memset(regs, 0, sizeof(*regs));
115− regs->syscallno = ~0UL;
115+ forget_syscall(regs);
116116 regs->pc = pc;
117117 }
118118
@@ -159,7 +159,7 @@ extern struct task_struct *cpu_switch_to(struct task_struct *prev, · 40 unmodified lines
159159 struct task_struct *next);
160160
161161 #define task_pt_regs(p) \
162− ((struct pt_regs *)(THREAD_START_SP + task_stack_page(p)) - 1)
162+ ((struct pt_regs *)(THREAD_SIZE + task_stack_page(p)) - 1)
163163
164164 #define KSTK_EIP(tsk) ((unsigned long)task_pt_regs(tsk)->pc)
165165 #define KSTK_ESP(tsk) user_stack_pointer(task_pt_regs(tsk))
arch/arm64/include/asm/ptrace.h+30−1
@@ -72,8 +72,19 @@ · 71 unmodified lines
7272 #define COMPAT_PT_TEXT_ADDR 0x10000
7373 #define COMPAT_PT_DATA_ADDR 0x10004
7474 #define COMPAT_PT_TEXT_END_ADDR 0x10008
75+
76+/*
77+ * If pt_regs.syscallno == NO_SYSCALL, then the thread is not executing
78+ * a syscall -- i.e., its most recent entry into the kernel from
79+ * userspace was not via SVC, or otherwise a tracer cancelled the syscall.
80+ *
81+ * This must have the value -1, for ABI compatibility with ptrace etc.
82+ */
83+#define NO_SYSCALL (-1)
84+
7585 #ifndef __ASSEMBLY__
7686 #include <linux/bug.h>
87+#include <linux/types.h>
7788
7889 /* sizeof(struct user) for AArch32 */
7990 #define COMPAT_USER_SZ 296
@@ -116,11 +127,29 @@ struct pt_regs { · 36 unmodified lines
116127 };
117128 };
118129 u64 orig_x0;
119− u64 syscallno;
130+#ifdef __AARCH64EB__
131+ u32 unused2;
132+ s32 syscallno;
133+#else
134+ s32 syscallno;
135+ u32 unused2;
136+#endif
137+
120138 u64 orig_addr_limit;
121139 u64 unused; // maintain 16 byte alignment
140+ u64 stackframe[2];
122141 };
123142
143+static inline bool in_syscall(struct pt_regs const *regs)
144+{
145+ return regs->syscallno != NO_SYSCALL;
146+}
147+
148+static inline void forget_syscall(struct pt_regs *regs)
149+{
150+ regs->syscallno = NO_SYSCALL;
151+}
152+
124153 #define MAX_REG_OFFSET offsetof(struct pt_regs, pstate)
125154
126155 #define arch_has_single_step() (1)
arch/arm64/include/asm/signal32.h+0−2
@@ -22,8 +22,6 @@ · 21 unmodified lines
2222
2323 #define AARCH32_KERN_SIGRET_CODE_OFFSET 0x500
2424
25−extern const compat_ulong_t aarch32_sigret_code[6];
26−
2725 int compat_setup_frame(int usig, struct ksignal *ksig, sigset_t *set,
2826 struct pt_regs *regs);
2927 int compat_setup_rt_frame(int usig, struct ksignal *ksig, sigset_t *set,
arch/arm64/include/asm/simd.h+56−0
@@ -0,0 +1,56 @@
1+/*
2+ * Copyright (C) 2017 Linaro Ltd. <ard.biesheuvel@linaro.org>
3+ *
4+ * This program is free software; you can redistribute it and/or modify it
5+ * under the terms of the GNU General Public License version 2 as published
6+ * by the Free Software Foundation.
7+ */
8+
9+#ifndef __ASM_SIMD_H
10+#define __ASM_SIMD_H
11+
12+#include <linux/compiler.h>
13+#include <linux/irqflags.h>
14+#include <linux/percpu.h>
15+#include <linux/preempt.h>
16+#include <linux/types.h>
17+
18+#ifdef CONFIG_KERNEL_MODE_NEON
19+
20+DECLARE_PER_CPU(bool, kernel_neon_busy);
21+
22+/*
23+ * may_use_simd - whether it is allowable at this time to issue SIMD
24+ * instructions or access the SIMD register file
25+ *
26+ * Callers must not assume that the result remains true beyond the next
27+ * preempt_enable() or return from softirq context.
28+ */
29+static __must_check inline bool may_use_simd(void)
30+{
31+ /*
32+ * The raw_cpu_read() is racy if called with preemption enabled.
33+ * This is not a bug: kernel_neon_busy is only set when
34+ * preemption is disabled, so we cannot migrate to another CPU
35+ * while it is set, nor can we migrate to a CPU where it is set.
36+ * So, if we find it clear on some CPU then we're guaranteed to
37+ * find it clear on any CPU we could migrate to.
38+ *
39+ * If we are in between kernel_neon_begin()...kernel_neon_end(),
40+ * the flag will be set, but preemption is also disabled, so we
41+ * can't migrate to another CPU and spuriously see it become
42+ * false.
43+ */
44+ return !in_irq() && !irqs_disabled() && !in_nmi() &&
45+ !raw_cpu_read(kernel_neon_busy);
46+}
47+
48+#else /* ! CONFIG_KERNEL_MODE_NEON */
49+
50+static __must_check inline bool may_use_simd(void) {
51+ return false;
52+}
53+
54+#endif /* ! CONFIG_KERNEL_MODE_NEON */
55+
56+#endif
arch/arm64/include/asm/smp.h+1−1
@@ -148,7 +148,7 @@ static inline void cpu_panic_kernel(void) · 147 unmodified lines
148148 */
149149 bool cpus_are_stuck_in_kernel(void);
150150
151−extern void smp_send_crash_stop(void);
151+extern void crash_smp_send_stop(void);
152152 extern bool smp_crash_stop_failed(void);
153153
154154 #endif /* ifndef __ASSEMBLY__ */
arch/arm64/include/asm/stacktrace.h+59−2
@@ -16,11 +16,15 @@ · 15 unmodified lines
1616 #ifndef __ASM_STACKTRACE_H
1717 #define __ASM_STACKTRACE_H
1818
19−struct task_struct;
19+#include <linux/percpu.h>
20+#include <linux/sched.h>
21+#include <linux/sched/task_stack.h>
22+
23+#include <asm/memory.h>
24+#include <asm/ptrace.h>
2025
2126 struct stackframe {
2227 unsigned long fp;
23− unsigned long sp;
2428 unsigned long pc;
2529 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
2630 unsigned int graph;
@@ -32,4 +36,57 @@ extern void walk_stackframe(struct task_struct *tsk, struct stackframe *frame, · 5 unmodified lines
3236 int (*fn)(struct stackframe *, void *), void *data);
3337 extern void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk);
3438
39+DECLARE_PER_CPU(unsigned long *, irq_stack_ptr);
40+
41+static inline bool on_irq_stack(unsigned long sp)
42+{
43+ unsigned long low = (unsigned long)raw_cpu_read(irq_stack_ptr);
44+ unsigned long high = low + IRQ_STACK_SIZE;
45+
46+ if (!low)
47+ return false;
48+
49+ return (low <= sp && sp < high);
50+}
51+
52+static inline bool on_task_stack(struct task_struct *tsk, unsigned long sp)
53+{
54+ unsigned long low = (unsigned long)task_stack_page(tsk);
55+ unsigned long high = low + THREAD_SIZE;
56+
57+ return (low <= sp && sp < high);
58+}
59+
60+#ifdef CONFIG_VMAP_STACK
61+DECLARE_PER_CPU(unsigned long [OVERFLOW_STACK_SIZE/sizeof(long)], overflow_stack);
62+
63+static inline bool on_overflow_stack(unsigned long sp)
64+{
65+ unsigned long low = (unsigned long)raw_cpu_ptr(overflow_stack);
66+ unsigned long high = low + OVERFLOW_STACK_SIZE;
67+
68+ return (low <= sp && sp < high);
69+}
70+#else
71+static inline bool on_overflow_stack(unsigned long sp) { return false; }
72+#endif
73+
74+/*
75+ * We can only safely access per-cpu stacks from current in a non-preemptible
76+ * context.
77+ */
78+static inline bool on_accessible_stack(struct task_struct *tsk, unsigned long sp)
79+{
80+ if (on_task_stack(tsk, sp))
81+ return true;
82+ if (tsk != current || preemptible())
83+ return false;
84+ if (on_irq_stack(sp))
85+ return true;
86+ if (on_overflow_stack(sp))
87+ return true;
88+
89+ return false;
90+}
91+
3592 #endif /* __ASM_STACKTRACE_H */
arch/arm64/include/asm/string.h+4−0
@@ -52,6 +52,10 @@ extern void *__memset(void *, int, __kernel_size_t); · 51 unmodified lines
5252 #define __HAVE_ARCH_MEMCMP
5353 extern int memcmp(const void *, const void *, size_t);
5454
55+#ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE
56+#define __HAVE_ARCH_MEMCPY_FLUSHCACHE
57+void memcpy_flushcache(void *dst, const void *src, size_t cnt);
58+#endif
5559
5660 #if defined(CONFIG_KASAN) && !defined(__SANITIZE_ADDRESS__)
5761
arch/arm64/include/asm/sysreg.h+1−0
@@ -329,6 +329,7 @@ · 328 unmodified lines
329329 #define ID_AA64ISAR1_LRCPC_SHIFT 20
330330 #define ID_AA64ISAR1_FCMA_SHIFT 16
331331 #define ID_AA64ISAR1_JSCVT_SHIFT 12
332+#define ID_AA64ISAR1_DPB_SHIFT 0
332333
333334 /* id_aa64pfr0 */
334335 #define ID_AA64PFR0_GIC_SHIFT 24
arch/arm64/include/asm/thread_info.h+4−9
@@ -23,19 +23,11 @@ · 22 unmodified lines
2323
2424 #include <linux/compiler.h>
2525
26−#ifdef CONFIG_ARM64_4K_PAGES
27−#define THREAD_SIZE_ORDER 2
28−#elif defined(CONFIG_ARM64_16K_PAGES)
29−#define THREAD_SIZE_ORDER 0
30−#endif
31−
32−#define THREAD_SIZE 16384
33−#define THREAD_START_SP (THREAD_SIZE - 16)
34−
3526 #ifndef __ASSEMBLY__
3627
3728 struct task_struct;
3829
30+#include <asm/memory.h>
3931 #include <asm/stack_pointer.h>
4032 #include <asm/types.h>
4133
@@ -68,6 +60,9 @@ struct thread_info { · 26 unmodified lines
6860 #define thread_saved_fp(tsk) \
6961 ((unsigned long)(tsk->thread.cpu_context.fp))
7062
63+void arch_setup_new_exec(void);
64+#define arch_setup_new_exec arch_setup_new_exec
65+
7166 #endif
7267
7368 /*
arch/arm64/include/asm/traps.h+5−0
@@ -53,4 +53,9 @@ static inline int in_exception_text(unsigned long ptr) · 52 unmodified lines
5353 return in ? : __in_irqentry_text(ptr);
5454 }
5555
56+static inline int in_entry_text(unsigned long ptr)
57+{
58+ return ptr >= (unsigned long)&__entry_text_start &&
59+ ptr < (unsigned long)&__entry_text_end;
60+}
5661 #endif
arch/arm64/include/asm/uaccess.h+12−0
@@ -350,4 +350,16 @@ extern long strncpy_from_user(char *dest, const char __user *src, long count); · 349 unmodified lines
350350
351351 extern __must_check long strnlen_user(const char __user *str, long n);
352352
353+#ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE
354+struct page;
355+void memcpy_page_flushcache(char *to, struct page *page, size_t offset, size_t len);
356+extern unsigned long __must_check __copy_user_flushcache(void *to, const void __user *from, unsigned long n);
357+
358+static inline int __copy_from_user_flushcache(void *dst, const void __user *src, unsigned size)
359+{
360+ kasan_check_write(dst, size);
361+ return __copy_user_flushcache(dst, src, size);
362+}
363+#endif
364+
353365 #endif /* __ASM_UACCESS_H */
arch/arm64/include/uapi/asm/hwcap.h+1−0
@@ -35,5 +35,6 @@ · 34 unmodified lines
3535 #define HWCAP_JSCVT (1 << 13)
3636 #define HWCAP_FCMA (1 << 14)
3737 #define HWCAP_LRCPC (1 << 15)
38+#define HWCAP_DCPOP (1 << 16)
3839
3940 #endif /* _UAPI__ASM_HWCAP_H */
arch/arm64/kernel/asm-offsets.c+1−0
@@ -75,6 +75,7 @@ int main(void) · 74 unmodified lines
7575 DEFINE(S_ORIG_X0, offsetof(struct pt_regs, orig_x0));
7676 DEFINE(S_SYSCALLNO, offsetof(struct pt_regs, syscallno));
7777 DEFINE(S_ORIG_ADDR_LIMIT, offsetof(struct pt_regs, orig_addr_limit));
78+ DEFINE(S_STACKFRAME, offsetof(struct pt_regs, stackframe));
7879 DEFINE(S_FRAME_SIZE, sizeof(struct pt_regs));
7980 BLANK();
8081 DEFINE(MM_CONTEXT_ID, offsetof(struct mm_struct, context.id.counter));
arch/arm64/kernel/cpufeature.c+13−0
@@ -120,6 +120,7 @@ static const struct arm64_ftr_bits ftr_id_aa64isar1[] = { · 119 unmodified lines
120120 ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64ISAR1_LRCPC_SHIFT, 4, 0),
121121 ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64ISAR1_FCMA_SHIFT, 4, 0),
122122 ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64ISAR1_JSCVT_SHIFT, 4, 0),
123+ ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64ISAR1_DPB_SHIFT, 4, 0),
123124 ARM64_FTR_END,
124125 };
125126
@@ -888,6 +889,17 @@ static const struct arm64_cpu_capabilities arm64_features[] = { · 762 unmodified lines
888889 .min_field_value = 0,
889890 .matches = has_no_fpsimd,
890891 },
892+#ifdef CONFIG_ARM64_PMEM
893+ {
894+ .desc = "Data cache clean to Point of Persistence",
895+ .capability = ARM64_HAS_DCPOP,
896+ .def_scope = SCOPE_SYSTEM,
897+ .matches = has_cpuid_feature,
898+ .sys_reg = SYS_ID_AA64ISAR1_EL1,
899+ .field_pos = ID_AA64ISAR1_DPB_SHIFT,
900+ .min_field_value = 1,
901+ },
902+#endif
891903 {},
892904 };
893905
@@ -916,6 +928,7 @@ static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = { · 22 unmodified lines
916928 HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_FP_SHIFT, FTR_SIGNED, 1, CAP_HWCAP, HWCAP_FPHP),
917929 HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_ASIMD_SHIFT, FTR_SIGNED, 0, CAP_HWCAP, HWCAP_ASIMD),
918930 HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_ASIMD_SHIFT, FTR_SIGNED, 1, CAP_HWCAP, HWCAP_ASIMDHP),
931+ HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_DPB_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_DCPOP),
919932 HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_JSCVT_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_JSCVT),
920933 HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_FCMA_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_FCMA),
921934 HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_LRCPC_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_LRCPC),
arch/arm64/kernel/cpuinfo.c+1−0
@@ -68,6 +68,7 @@ static const char *const hwcap_str[] = { · 67 unmodified lines
6868 "jscvt",
6969 "fcma",
7070 "lrcpc",
71+ "dcpop",
7172 NULL
7273 };
7374
arch/arm64/kernel/entry-fpsimd.S+0−24
@@ -41,27 +41,3 @@ ENTRY(fpsimd_load_state) · 40 unmodified lines
4141 fpsimd_restore x0, 8
4242 ret
4343 ENDPROC(fpsimd_load_state)
44−
45−#ifdef CONFIG_KERNEL_MODE_NEON
46−
47−/*
48− * Save the bottom n FP registers.
49− *
50− * x0 - pointer to struct fpsimd_partial_state
51− */
52−ENTRY(fpsimd_save_partial_state)
53− fpsimd_save_partial x0, 1, 8, 9
54− ret
55−ENDPROC(fpsimd_save_partial_state)
56−
57−/*
58− * Load the bottom n FP registers.
59− *
60− * x0 - pointer to struct fpsimd_partial_state
61− */
62−ENTRY(fpsimd_load_partial_state)
63− fpsimd_restore_partial x0, 8, 9
64− ret
65−ENDPROC(fpsimd_load_partial_state)
66−
67−#endif
arch/arm64/kernel/entry.S+183−100
@@ -69,8 +69,55 @@ · 68 unmodified lines
6969 #define BAD_FIQ 2
7070 #define BAD_ERROR 3
7171
72− .macro kernel_entry, el, regsize = 64
72+ .macro kernel_ventry label
73+ .align 7
7374 sub sp, sp, #S_FRAME_SIZE
75+#ifdef CONFIG_VMAP_STACK
76+ /*
77+ * Test whether the SP has overflowed, without corrupting a GPR.
78+ * Task and IRQ stacks are aligned to (1 << THREAD_SHIFT).
79+ */
80+ add sp, sp, x0 // sp' = sp + x0
81+ sub x0, sp, x0 // x0' = sp' - x0 = (sp + x0) - x0 = sp
82+ tbnz x0, #THREAD_SHIFT, 0f
83+ sub x0, sp, x0 // x0'' = sp' - x0' = (sp + x0) - sp = x0
84+ sub sp, sp, x0 // sp'' = sp' - x0 = (sp + x0) - x0 = sp
85+ b \label
86+
87+0:
88+ /*
89+ * Either we've just detected an overflow, or we've taken an exception
90+ * while on the overflow stack. Either way, we won't return to
91+ * userspace, and can clobber EL0 registers to free up GPRs.
92+ */
93+
94+ /* Stash the original SP (minus S_FRAME_SIZE) in tpidr_el0. */
95+ msr tpidr_el0, x0
96+
97+ /* Recover the original x0 value and stash it in tpidrro_el0 */
98+ sub x0, sp, x0
99+ msr tpidrro_el0, x0
100+
101+ /* Switch to the overflow stack */
102+ adr_this_cpu sp, overflow_stack + OVERFLOW_STACK_SIZE, x0
103+
104+ /*
105+ * Check whether we were already on the overflow stack. This may happen
106+ * after panic() re-enables interrupts.
107+ */
108+ mrs x0, tpidr_el0 // sp of interrupted context
109+ sub x0, sp, x0 // delta with top of overflow stack
110+ tst x0, #~(OVERFLOW_STACK_SIZE - 1) // within range?
111+ b.ne __bad_stack // no? -> bad stack pointer
112+
113+ /* We were already on the overflow stack. Restore sp/x0 and carry on. */
114+ sub sp, sp, x0
115+ mrs x0, tpidrro_el0
116+#endif
117+ b \label
118+ .endm
119+
120+ .macro kernel_entry, el, regsize = 64
74121 .if \regsize == 32
75122 mov w0, w0 // zero upper 32 bits of x0
76123 .endif
@@ -111,6 +158,18 @@ · 34 unmodified lines
111158 mrs x23, spsr_el1
112159 stp lr, x21, [sp, #S_LR]
113160
161+ /*
162+ * In order to be able to dump the contents of struct pt_regs at the
163+ * time the exception was taken (in case we attempt to walk the call
164+ * stack later), chain it together with the stack frames.
165+ */
166+ .if \el == 0
167+ stp xzr, xzr, [sp, #S_STACKFRAME]
168+ .else
169+ stp x29, x22, [sp, #S_STACKFRAME]
170+ .endif
171+ add x29, sp, #S_STACKFRAME
172+
114173 #ifdef CONFIG_ARM64_SW_TTBR0_PAN
115174 /*
116175 * Set the TTBR0 PAN bit in SPSR. When the exception is taken from
@@ -138,12 +197,10 @@ alternative_else_nop_endif · 21 unmodified lines
138197
139198 stp x22, x23, [sp, #S_PC]
140199
141− /*
142− * Set syscallno to -1 by default (overridden later if real syscall).
143− */
200+ /* Not in a syscall by default (el0_svc overwrites for real syscall) */
144201 .if \el == 0
145− mvn x21, xzr
146− str x21, [sp, #S_SYSCALLNO]
202+ mov w21, #NO_SYSCALL
203+ str w21, [sp, #S_SYSCALLNO]
147204 .endif
148205
149206 /*
@@ -259,20 +316,12 @@ alternative_else_nop_endif · 109 unmodified lines
259316 and x25, x25, #~(THREAD_SIZE - 1)
260317 cbnz x25, 9998f
261318
262− adr_this_cpu x25, irq_stack, x26
263− mov x26, #IRQ_STACK_START_SP
319+ ldr_this_cpu x25, irq_stack_ptr, x26
320+ mov x26, #IRQ_STACK_SIZE
264321 add x26, x25, x26
265322
266323 /* switch to the irq stack */
267324 mov sp, x26
268−
269− /*
270− * Add a dummy stack frame, this non-standard format is fixed up
271− * by unwind_frame()
272− */
273− stp x29, x19, [sp, #-16]!
274− mov x29, sp
275−
276325 9998:
277326 .endm
278327
@@ -290,8 +339,9 @@ alternative_else_nop_endif · 11 unmodified lines
290339 *
291340 * x7 is reserved for the system call number in 32-bit mode.
292341 */
293−sc_nr .req x25 // number of system calls
294−scno .req x26 // syscall number
342+wsc_nr .req w25 // number of system calls
343+wscno .req w26 // syscall number
344+xscno .req x26 // syscall number (zero-extended)
295345 stbl .req x27 // syscall table pointer
296346 tsk .req x28 // current thread_info
297347
@@ -315,34 +365,62 @@ tsk .req x28 // current thread_info · 17 unmodified lines
315365
316366 .align 11
317367 ENTRY(vectors)
318− ventry el1_sync_invalid // Synchronous EL1t
319− ventry el1_irq_invalid // IRQ EL1t
320− ventry el1_fiq_invalid // FIQ EL1t
321− ventry el1_error_invalid // Error EL1t
368+ kernel_ventry el1_sync_invalid // Synchronous EL1t
369+ kernel_ventry el1_irq_invalid // IRQ EL1t
370+ kernel_ventry el1_fiq_invalid // FIQ EL1t
371+ kernel_ventry el1_error_invalid // Error EL1t
322372
323− ventry el1_sync // Synchronous EL1h
324− ventry el1_irq // IRQ EL1h
325− ventry el1_fiq_invalid // FIQ EL1h
326− ventry el1_error_invalid // Error EL1h
373+ kernel_ventry el1_sync // Synchronous EL1h
374+ kernel_ventry el1_irq // IRQ EL1h
375+ kernel_ventry el1_fiq_invalid // FIQ EL1h
376+ kernel_ventry el1_error_invalid // Error EL1h
327377
328− ventry el0_sync // Synchronous 64-bit EL0
329− ventry el0_irq // IRQ 64-bit EL0
330− ventry el0_fiq_invalid // FIQ 64-bit EL0
331− ventry el0_error_invalid // Error 64-bit EL0
378+ kernel_ventry el0_sync // Synchronous 64-bit EL0
379+ kernel_ventry el0_irq // IRQ 64-bit EL0
380+ kernel_ventry el0_fiq_invalid // FIQ 64-bit EL0
381+ kernel_ventry el0_error_invalid // Error 64-bit EL0
332382
333383 #ifdef CONFIG_COMPAT
334− ventry el0_sync_compat // Synchronous 32-bit EL0
335− ventry el0_irq_compat // IRQ 32-bit EL0
336− ventry el0_fiq_invalid_compat // FIQ 32-bit EL0
337− ventry el0_error_invalid_compat // Error 32-bit EL0
384+ kernel_ventry el0_sync_compat // Synchronous 32-bit EL0
385+ kernel_ventry el0_irq_compat // IRQ 32-bit EL0
386+ kernel_ventry el0_fiq_invalid_compat // FIQ 32-bit EL0
387+ kernel_ventry el0_error_invalid_compat // Error 32-bit EL0
338388 #else
339− ventry el0_sync_invalid // Synchronous 32-bit EL0
340− ventry el0_irq_invalid // IRQ 32-bit EL0
341− ventry el0_fiq_invalid // FIQ 32-bit EL0
342− ventry el0_error_invalid // Error 32-bit EL0
389+ kernel_ventry el0_sync_invalid // Synchronous 32-bit EL0
390+ kernel_ventry el0_irq_invalid // IRQ 32-bit EL0
391+ kernel_ventry el0_fiq_invalid // FIQ 32-bit EL0
392+ kernel_ventry el0_error_invalid // Error 32-bit EL0
343393 #endif
344394 END(vectors)
345395
396+#ifdef CONFIG_VMAP_STACK
397+ /*
398+ * We detected an overflow in kernel_ventry, which switched to the
399+ * overflow stack. Stash the exception regs, and head to our overflow
400+ * handler.
401+ */
402+__bad_stack:
403+ /* Restore the original x0 value */
404+ mrs x0, tpidrro_el0
405+
406+ /*
407+ * Store the original GPRs to the new stack. The orginal SP (minus
408+ * S_FRAME_SIZE) was stashed in tpidr_el0 by kernel_ventry.
409+ */
410+ sub sp, sp, #S_FRAME_SIZE
411+ kernel_entry 1
412+ mrs x0, tpidr_el0
413+ add x0, x0, #S_FRAME_SIZE
414+ str x0, [sp, #S_SP]
415+
416+ /* Stash the regs for handle_bad_stack */
417+ mov x0, sp
418+
419+ /* Time to die */
420+ bl handle_bad_stack
421+ ASM_BUG()
422+#endif /* CONFIG_VMAP_STACK */
423+
346424 /*
347425 * Invalid mode handlers
348426 */
@@ -351,7 +429,8 @@ END(vectors) · 2 unmodified lines
351429 mov x0, sp
352430 mov x1, #\reason
353431 mrs x2, esr_el1
354− b bad_mode
432+ bl bad_mode
433+ ASM_BUG()
355434 .endm
356435
357436 el0_sync_invalid:
@@ -448,14 +527,16 @@ el1_sp_pc: · 90 unmodified lines
448527 mrs x0, far_el1
449528 enable_dbg
450529 mov x2, sp
451− b do_sp_pc_abort
530+ bl do_sp_pc_abort
531+ ASM_BUG()
452532 el1_undef:
453533 /*
454534 * Undefined instruction
455535 */
456536 enable_dbg
457537 mov x0, sp
458− b do_undefinstr
538+ bl do_undefinstr
539+ ASM_BUG()
459540 el1_dbg:
460541 /*
461542 * Debug exception handling
@@ -473,7 +554,8 @@ el1_inv: · 11 unmodified lines
473554 mov x0, sp
474555 mov x2, x1
475556 mov x1, #BAD_SYNC
476− b bad_mode
557+ bl bad_mode
558+ ASM_BUG()
477559 ENDPROC(el1_sync)
478560
479561 .align 6
@@ -577,8 +659,8 @@ el0_svc_compat: · 97 unmodified lines
577659 * AArch32 syscall handling
578660 */
579661 adrp stbl, compat_sys_call_table // load compat syscall table pointer
580− uxtw scno, w7 // syscall number in w7 (r7)
581− mov sc_nr, #__NR_compat_syscalls
662+ mov wscno, w7 // syscall number in w7 (r7)
663+ mov wsc_nr, #__NR_compat_syscalls
582664 b el0_svc_naked
583665
584666 .align 6
@@ -706,38 +788,6 @@ el0_irq_naked: · 121 unmodified lines
706788 b ret_to_user
707789 ENDPROC(el0_irq)
708790
709−/*
710− * Register switch for AArch64. The callee-saved registers need to be saved
711− * and restored. On entry:
712− * x0 = previous task_struct (must be preserved across the switch)
713− * x1 = next task_struct
714− * Previous and next are guaranteed not to be the same.
715− *
716− */
717−ENTRY(cpu_switch_to)
718− mov x10, #THREAD_CPU_CONTEXT
719− add x8, x0, x10
720− mov x9, sp
721− stp x19, x20, [x8], #16 // store callee-saved registers
722− stp x21, x22, [x8], #16
723− stp x23, x24, [x8], #16
724− stp x25, x26, [x8], #16
725− stp x27, x28, [x8], #16
726− stp x29, x9, [x8], #16
727− str lr, [x8]
728− add x8, x1, x10
729− ldp x19, x20, [x8], #16 // restore callee-saved registers
730− ldp x21, x22, [x8], #16
731− ldp x23, x24, [x8], #16
732− ldp x25, x26, [x8], #16
733− ldp x27, x28, [x8], #16
734− ldp x29, x9, [x8], #16
735− ldr lr, [x8]
736− mov sp, x9
737− msr sp_el0, x1
738− ret
739−ENDPROC(cpu_switch_to)
740−
741791 /*
742792 * This is the fast syscall return path. We do as little as possible here,
743793 * and this includes saving x0 back into the kernel stack.
@@ -780,37 +830,25 @@ finish_ret_to_user: · 36 unmodified lines
780830 kernel_exit 0
781831 ENDPROC(ret_to_user)
782832
783−/*
784− * This is how we return from a fork.
785− */
786−ENTRY(ret_from_fork)
787− bl schedule_tail
788− cbz x19, 1f // not a kernel thread
789− mov x0, x20
790− blr x19
791−1: get_thread_info tsk
792− b ret_to_user
793−ENDPROC(ret_from_fork)
794−
795833 /*
796834 * SVC handler.
797835 */
798836 .align 6
799837 el0_svc:
800838 adrp stbl, sys_call_table // load syscall table pointer
801− uxtw scno, w8 // syscall number in w8
802− mov sc_nr, #__NR_syscalls
839+ mov wscno, w8 // syscall number in w8
840+ mov wsc_nr, #__NR_syscalls
803841 el0_svc_naked: // compat entry point
804− stp x0, scno, [sp, #S_ORIG_X0] // save the original x0 and syscall number
842+ stp x0, xscno, [sp, #S_ORIG_X0] // save the original x0 and syscall number
805843 enable_dbg_and_irq
806844 ct_user_exit 1
807845
808846 ldr x16, [tsk, #TSK_TI_FLAGS] // check for syscall hooks
809847 tst x16, #_TIF_SYSCALL_WORK
810848 b.ne __sys_trace
811− cmp scno, sc_nr // check upper syscall limit
849+ cmp wscno, wsc_nr // check upper syscall limit
812850 b.hs ni_sys
813− ldr x16, [stbl, scno, lsl #3] // address in the syscall table
851+ ldr x16, [stbl, xscno, lsl #3] // address in the syscall table
814852 blr x16 // call sys_* routine
815853 b ret_fast_syscall
816854 ni_sys:
@@ -824,24 +862,23 @@ ENDPROC(el0_svc) · 7 unmodified lines
824862 * switches, and waiting for our parent to respond.
825863 */
826864 __sys_trace:
827− mov w0, #-1 // set default errno for
828− cmp scno, x0 // user-issued syscall(-1)
865+ cmp wscno, #NO_SYSCALL // user-issued syscall(-1)?
829866 b.ne 1f
830− mov x0, #-ENOSYS
867+ mov x0, #-ENOSYS // set default errno if so
831868 str x0, [sp, #S_X0]
832869 1: mov x0, sp
833870 bl syscall_trace_enter
834− cmp w0, #-1 // skip the syscall?
871+ cmp w0, #NO_SYSCALL // skip the syscall?
835872 b.eq __sys_trace_return_skipped
836− uxtw scno, w0 // syscall number (possibly new)
873+ mov wscno, w0 // syscall number (possibly new)
837874 mov x1, sp // pointer to regs
838− cmp scno, sc_nr // check upper syscall limit
875+ cmp wscno, wsc_nr // check upper syscall limit
839876 b.hs __ni_sys_trace
840877 ldp x0, x1, [sp] // restore the syscall args
841878 ldp x2, x3, [sp, #S_X2]
842879 ldp x4, x5, [sp, #S_X4]
843880 ldp x6, x7, [sp, #S_X6]
844− ldr x16, [stbl, scno, lsl #3] // address in the syscall table
881+ ldr x16, [stbl, xscno, lsl #3] // address in the syscall table
845882 blr x16 // call sys_* routine
846883
847884 __sys_trace_return:
@@ -865,3 +902,49 @@ ENTRY(sys_rt_sigreturn_wrapper) · 17 unmodified lines
865902 mov x0, sp
866903 b sys_rt_sigreturn
867904 ENDPROC(sys_rt_sigreturn_wrapper)
905+
906+/*
907+ * Register switch for AArch64. The callee-saved registers need to be saved
908+ * and restored. On entry:
909+ * x0 = previous task_struct (must be preserved across the switch)
910+ * x1 = next task_struct
911+ * Previous and next are guaranteed not to be the same.
912+ *
913+ */
914+ENTRY(cpu_switch_to)
915+ mov x10, #THREAD_CPU_CONTEXT
916+ add x8, x0, x10
917+ mov x9, sp
918+ stp x19, x20, [x8], #16 // store callee-saved registers
919+ stp x21, x22, [x8], #16
920+ stp x23, x24, [x8], #16
921+ stp x25, x26, [x8], #16
922+ stp x27, x28, [x8], #16
923+ stp x29, x9, [x8], #16
924+ str lr, [x8]
925+ add x8, x1, x10
926+ ldp x19, x20, [x8], #16 // restore callee-saved registers
927+ ldp x21, x22, [x8], #16
928+ ldp x23, x24, [x8], #16
929+ ldp x25, x26, [x8], #16
930+ ldp x27, x28, [x8], #16
931+ ldp x29, x9, [x8], #16
932+ ldr lr, [x8]
933+ mov sp, x9
934+ msr sp_el0, x1
935+ ret
936+ENDPROC(cpu_switch_to)
937+NOKPROBE(cpu_switch_to)
938+
939+/*
940+ * This is how we return from a fork.
941+ */
942+ENTRY(ret_from_fork)
943+ bl schedule_tail
944+ cbz x19, 1f // not a kernel thread
945+ mov x0, x20
946+ blr x19
947+1: get_thread_info tsk
948+ b ret_to_user
949+ENDPROC(ret_from_fork)
950+NOKPROBE(ret_from_fork)
arch/arm64/kernel/fpsimd.c+132−40
@@ -17,16 +17,19 @@ · 16 unmodified lines
1717 * along with this program. If not, see <http://www.gnu.org/licenses/>.
1818 */
1919
20+#include <linux/bottom_half.h>
2021 #include <linux/cpu.h>
2122 #include <linux/cpu_pm.h>
2223 #include <linux/kernel.h>
2324 #include <linux/init.h>
25+#include <linux/percpu.h>
26+#include <linux/preempt.h>
2427 #include <linux/sched/signal.h>
2528 #include <linux/signal.h>
26−#include <linux/hardirq.h>
2729
2830 #include <asm/fpsimd.h>
2931 #include <asm/cputype.h>
32+#include <asm/simd.h>
3033
3134 #define FPEXC_IOF (1 << 0)
3235 #define FPEXC_DZF (1 << 1)
@@ -62,6 +65,13 @@ · 29 unmodified lines
6265 * CPU currently contain the most recent userland FPSIMD state of the current
6366 * task.
6467 *
68+ * In order to allow softirq handlers to use FPSIMD, kernel_neon_begin() may
69+ * save the task's FPSIMD context back to task_struct from softirq context.
70+ * To prevent this from racing with the manipulation of the task's FPSIMD state
71+ * from task context and thereby corrupting the state, it is necessary to
72+ * protect any manipulation of a task's fpsimd_state or TIF_FOREIGN_FPSTATE
73+ * flag with local_bh_disable() unless softirqs are already masked.
74+ *
6575 * For a certain task, the sequence may look something like this:
6676 * - the task gets scheduled in; if both the task's fpsimd_state.cpu field
6777 * contains the id of the current CPU, and the CPU's fpsimd_last_state per-cpu
@@ -161,11 +171,14 @@ void fpsimd_flush_thread(void) · 93 unmodified lines
161171 {
162172 if (!system_supports_fpsimd())
163173 return;
164− preempt_disable();
174+
175+ local_bh_disable();
176+
165177 memset(&current->thread.fpsimd_state, 0, sizeof(struct fpsimd_state));
166178 fpsimd_flush_task_state(current);
167179 set_thread_flag(TIF_FOREIGN_FPSTATE);
168− preempt_enable();
180+
181+ local_bh_enable();
169182 }
170183
171184 /*
@@ -176,10 +189,13 @@ void fpsimd_preserve_current_state(void) · 4 unmodified lines
176189 {
177190 if (!system_supports_fpsimd())
178191 return;
179− preempt_disable();
192+
193+ local_bh_disable();
194+
180195 if (!test_thread_flag(TIF_FOREIGN_FPSTATE))
181196 fpsimd_save_state(&current->thread.fpsimd_state);
182− preempt_enable();
197+
198+ local_bh_enable();
183199 }
184200
185201 /*
@@ -191,15 +207,18 @@ void fpsimd_restore_current_state(void) · 5 unmodified lines
191207 {
192208 if (!system_supports_fpsimd())
193209 return;
194− preempt_disable();
210+
211+ local_bh_disable();
212+
195213 if (test_and_clear_thread_flag(TIF_FOREIGN_FPSTATE)) {
196214 struct fpsimd_state *st = &current->thread.fpsimd_state;
197215
198216 fpsimd_load_state(st);
199− this_cpu_write(fpsimd_last_state, st);
217+ __this_cpu_write(fpsimd_last_state, st);
200218 st->cpu = smp_processor_id();
201219 }
202− preempt_enable();
220+
221+ local_bh_enable();
203222 }
204223
205224 /*
@@ -211,15 +230,18 @@ void fpsimd_update_current_state(struct fpsimd_state *state) · 5 unmodified lines
211230 {
212231 if (!system_supports_fpsimd())
213232 return;
214− preempt_disable();
233+
234+ local_bh_disable();
235+
215236 fpsimd_load_state(state);
216237 if (test_and_clear_thread_flag(TIF_FOREIGN_FPSTATE)) {
217238 struct fpsimd_state *st = &current->thread.fpsimd_state;
218239
219− this_cpu_write(fpsimd_last_state, st);
240+ __this_cpu_write(fpsimd_last_state, st);
220241 st->cpu = smp_processor_id();
221242 }
222− preempt_enable();
243+
244+ local_bh_enable();
223245 }
224246
225247 /*
@@ -232,52 +254,122 @@ void fpsimd_flush_task_state(struct task_struct *t) · 6 unmodified lines
232254
233255 #ifdef CONFIG_KERNEL_MODE_NEON
234256
235−static DEFINE_PER_CPU(struct fpsimd_partial_state, hardirq_fpsimdstate);
236−static DEFINE_PER_CPU(struct fpsimd_partial_state, softirq_fpsimdstate);
257+DEFINE_PER_CPU(bool, kernel_neon_busy);
258+EXPORT_PER_CPU_SYMBOL(kernel_neon_busy);
237259
238260 /*
239261 * Kernel-side NEON support functions
240262 */
241−void kernel_neon_begin_partial(u32 num_regs)
263+
264+/*
265+ * kernel_neon_begin(): obtain the CPU FPSIMD registers for use by the calling
266+ * context
267+ *
268+ * Must not be called unless may_use_simd() returns true.
269+ * Task context in the FPSIMD registers is saved back to memory as necessary.
270+ *
271+ * A matching call to kernel_neon_end() must be made before returning from the
272+ * calling context.
273+ *
274+ * The caller may freely use the FPSIMD registers until kernel_neon_end() is
275+ * called.
276+ */
277+void kernel_neon_begin(void)
242278 {
243279 if (WARN_ON(!system_supports_fpsimd()))
244280 return;
245− if (in_interrupt()) {
246− struct fpsimd_partial_state *s = this_cpu_ptr(
247− in_irq() ? &hardirq_fpsimdstate : &softirq_fpsimdstate);
248281
249− BUG_ON(num_regs > 32);
250− fpsimd_save_partial_state(s, roundup(num_regs, 2));
251− } else {
252− /*
253− * Save the userland FPSIMD state if we have one and if we
254− * haven't done so already. Clear fpsimd_last_state to indicate
255− * that there is no longer userland FPSIMD state in the
256− * registers.
257− */
258− preempt_disable();
259− if (current->mm &&
260− !test_and_set_thread_flag(TIF_FOREIGN_FPSTATE))
261− fpsimd_save_state(&current->thread.fpsimd_state);
262− this_cpu_write(fpsimd_last_state, NULL);
263− }
282+ BUG_ON(!may_use_simd());
283+
284+ local_bh_disable();
285+
286+ __this_cpu_write(kernel_neon_busy, true);
287+
288+ /* Save unsaved task fpsimd state, if any: */
289+ if (current->mm && !test_and_set_thread_flag(TIF_FOREIGN_FPSTATE))
290+ fpsimd_save_state(&current->thread.fpsimd_state);
291+
292+ /* Invalidate any task state remaining in the fpsimd regs: */
293+ __this_cpu_write(fpsimd_last_state, NULL);
294+
295+ preempt_disable();
296+
297+ local_bh_enable();
264298 }
265−EXPORT_SYMBOL(kernel_neon_begin_partial);
299+EXPORT_SYMBOL(kernel_neon_begin);
266300
301+/*
302+ * kernel_neon_end(): give the CPU FPSIMD registers back to the current task
303+ *
304+ * Must be called from a context in which kernel_neon_begin() was previously
305+ * called, with no call to kernel_neon_end() in the meantime.
306+ *
307+ * The caller must not use the FPSIMD registers after this function is called,
308+ * unless kernel_neon_begin() is called again in the meantime.
309+ */
267310 void kernel_neon_end(void)
268311 {
312+ bool busy;
313+
269314 if (!system_supports_fpsimd())
270315 return;
271− if (in_interrupt()) {
272− struct fpsimd_partial_state *s = this_cpu_ptr(
273− in_irq() ? &hardirq_fpsimdstate : &softirq_fpsimdstate);
274− fpsimd_load_partial_state(s);
275− } else {
276− preempt_enable();
277− }
316+
317+ busy = __this_cpu_xchg(kernel_neon_busy, false);
318+ WARN_ON(!busy); /* No matching kernel_neon_begin()? */
319+
320+ preempt_enable();
278321 }
279322 EXPORT_SYMBOL(kernel_neon_end);
280323
324+static DEFINE_PER_CPU(struct fpsimd_state, efi_fpsimd_state);
325+static DEFINE_PER_CPU(bool, efi_fpsimd_state_used);
326+
327+/*
328+ * EFI runtime services support functions
329+ *
330+ * The ABI for EFI runtime services allows EFI to use FPSIMD during the call.
331+ * This means that for EFI (and only for EFI), we have to assume that FPSIMD
332+ * is always used rather than being an optional accelerator.
333+ *
334+ * These functions provide the necessary support for ensuring FPSIMD
335+ * save/restore in the contexts from which EFI is used.
336+ *
337+ * Do not use them for any other purpose -- if tempted to do so, you are
338+ * either doing something wrong or you need to propose some refactoring.
339+ */
340+
341+/*
342+ * __efi_fpsimd_begin(): prepare FPSIMD for making an EFI runtime services call
343+ */
344+void __efi_fpsimd_begin(void)
345+{
346+ if (!system_supports_fpsimd())
347+ return;
348+
349+ WARN_ON(preemptible());
350+
351+ if (may_use_simd())
352+ kernel_neon_begin();
353+ else {
354+ fpsimd_save_state(this_cpu_ptr(&efi_fpsimd_state));
355+ __this_cpu_write(efi_fpsimd_state_used, true);
356+ }
357+}
358+
359+/*
360+ * __efi_fpsimd_end(): clean up FPSIMD after an EFI runtime services call
361+ */
362+void __efi_fpsimd_end(void)
363+{
364+ if (!system_supports_fpsimd())
365+ return;
366+
367+ if (__this_cpu_xchg(efi_fpsimd_state_used, false))
368+ fpsimd_load_state(this_cpu_ptr(&efi_fpsimd_state));
369+ else
370+ kernel_neon_end();
371+}
372+
281373 #endif /* CONFIG_KERNEL_MODE_NEON */
282374
283375 #ifdef CONFIG_CPU_PM
arch/arm64/kernel/head.S+13−9
@@ -143,8 +143,8 @@ preserve_boot_args: · 142 unmodified lines
143143 dmb sy // needed before dc ivac with
144144 // MMU off
145145
146− add x1, x0, #0x20 // 4 x 8 bytes
147− b __inval_cache_range // tail call
146+ mov x1, #0x20 // 4 x 8 bytes
147+ b __inval_dcache_area // tail call
148148 ENDPROC(preserve_boot_args)
149149
150150 /*
@@ -221,20 +221,20 @@ __create_page_tables: · 70 unmodified lines
221221 * dirty cache lines being evicted.
222222 */
223223 adrp x0, idmap_pg_dir
224− adrp x1, swapper_pg_dir + SWAPPER_DIR_SIZE + RESERVED_TTBR0_SIZE
225− bl __inval_cache_range
224+ ldr x1, =(IDMAP_DIR_SIZE + SWAPPER_DIR_SIZE + RESERVED_TTBR0_SIZE)
225+ bl __inval_dcache_area
226226
227227 /*
228228 * Clear the idmap and swapper page tables.
229229 */
230230 adrp x0, idmap_pg_dir
231− adrp x6, swapper_pg_dir + SWAPPER_DIR_SIZE + RESERVED_TTBR0_SIZE
231+ ldr x1, =(IDMAP_DIR_SIZE + SWAPPER_DIR_SIZE + RESERVED_TTBR0_SIZE)
232232 1: stp xzr, xzr, [x0], #16
233233 stp xzr, xzr, [x0], #16
234234 stp xzr, xzr, [x0], #16
235235 stp xzr, xzr, [x0], #16
236− cmp x0, x6
237− b.lo 1b
236+ subs x1, x1, #64
237+ b.ne 1b
238238
239239 mov x7, SWAPPER_MM_MMUFLAGS
240240
@@ -307,9 +307,9 @@ __create_page_tables: · 66 unmodified lines
307307 * tables again to remove any speculatively loaded cache lines.
308308 */
309309 adrp x0, idmap_pg_dir
310− adrp x1, swapper_pg_dir + SWAPPER_DIR_SIZE + RESERVED_TTBR0_SIZE
310+ ldr x1, =(IDMAP_DIR_SIZE + SWAPPER_DIR_SIZE + RESERVED_TTBR0_SIZE)
311311 dmb sy
312− bl __inval_cache_range
312+ bl __inval_dcache_area
313313
314314 ret x28
315315 ENDPROC(__create_page_tables)
@@ -361,6 +361,9 @@ __primary_switched: · 45 unmodified lines
361361 ret // to __primary_switch()
362362 0:
363363 #endif
364+ add sp, sp, #16
365+ mov x29, #0
366+ mov x30, #0
364367 b start_kernel
365368 ENDPROC(__primary_switched)
366369
@@ -616,6 +619,7 @@ __secondary_switched: · 249 unmodified lines
616619 ldr x2, [x0, #CPU_BOOT_TASK]
617620 msr sp_el0, x2
618621 mov x29, #0
622+ mov x30, #0
619623 b secondary_start_kernel
620624 ENDPROC(__secondary_switched)
621625
arch/arm64/kernel/hibernate.c+2−2
@@ -330,7 +330,7 @@ static void _copy_pte(pte_t *dst_pte, pte_t *src_pte, unsigned long addr) · 329 unmodified lines
330330 * read only (code, rodata). Clear the RDONLY bit from
331331 * the temporary mappings we use during restore.
332332 */
333− set_pte(dst_pte, pte_clear_rdonly(pte));
333+ set_pte(dst_pte, pte_mkwrite(pte));
334334 } else if (debug_pagealloc_enabled() && !pte_none(pte)) {
335335 /*
336336 * debug_pagealloc will removed the PTE_VALID bit if
@@ -343,7 +343,7 @@ static void _copy_pte(pte_t *dst_pte, pte_t *src_pte, unsigned long addr) · 6 unmodified lines
343343 */
344344 BUG_ON(!pfn_valid(pte_pfn(pte)));
345345
346− set_pte(dst_pte, pte_mkpresent(pte_clear_rdonly(pte)));
346+ set_pte(dst_pte, pte_mkpresent(pte_mkwrite(pte)));
347347 }
348348 }
349349
arch/arm64/kernel/irq.c+38−2
@@ -23,15 +23,16 @@ · 22 unmodified lines
2323
2424 #include <linux/kernel_stat.h>
2525 #include <linux/irq.h>
26+#include <linux/memory.h>
2627 #include <linux/smp.h>
2728 #include <linux/init.h>
2829 #include <linux/irqchip.h>
2930 #include <linux/seq_file.h>
31+#include <linux/vmalloc.h>
3032
3133 unsigned long irq_err_count;
3234
33−/* irq stack only needs to be 16 byte aligned - not IRQ_STACK_SIZE aligned. */
34−DEFINE_PER_CPU(unsigned long [IRQ_STACK_SIZE/sizeof(long)], irq_stack) __aligned(16);
35+DEFINE_PER_CPU(unsigned long *, irq_stack_ptr);
3536
3637 int arch_show_interrupts(struct seq_file *p, int prec)
3738 {
@@ -50,8 +51,43 @@ void __init set_handle_irq(void (*handle_irq)(struct pt_regs *)) · 12 unmodified lines
5051 handle_arch_irq = handle_irq;
5152 }
5253
54+#ifdef CONFIG_VMAP_STACK
55+static void init_irq_stacks(void)
56+{
57+ int cpu;
58+ unsigned long *p;
59+
60+ for_each_possible_cpu(cpu) {
61+ /*
62+ * To ensure that VMAP'd stack overflow detection works
63+ * correctly, the IRQ stacks need to have the same
64+ * alignment as other stacks.
65+ */
66+ p = __vmalloc_node_range(IRQ_STACK_SIZE, THREAD_ALIGN,
67+ VMALLOC_START, VMALLOC_END,
68+ THREADINFO_GFP, PAGE_KERNEL,
69+ 0, cpu_to_node(cpu),
70+ __builtin_return_address(0));
71+
72+ per_cpu(irq_stack_ptr, cpu) = p;
73+ }
74+}
75+#else
76+/* irq stack only needs to be 16 byte aligned - not IRQ_STACK_SIZE aligned. */
77+DEFINE_PER_CPU_ALIGNED(unsigned long [IRQ_STACK_SIZE/sizeof(long)], irq_stack);
78+
79+static void init_irq_stacks(void)
80+{
81+ int cpu;
82+
83+ for_each_possible_cpu(cpu)
84+ per_cpu(irq_stack_ptr, cpu) = per_cpu(irq_stack, cpu);
85+}
86+#endif
87+
5388 void __init init_IRQ(void)
5489 {
90+ init_irq_stacks();
5591 irqchip_init();
5692 if (!handle_arch_irq)
5793 panic("No interrupt controller found.");
arch/arm64/kernel/machine_kexec.c+1−1
@@ -252,7 +252,7 @@ void machine_crash_shutdown(struct pt_regs *regs) · 251 unmodified lines
252252 local_irq_disable();
253253
254254 /* shutdown non-crashing cpus */
255− smp_send_crash_stop();
255+ crash_smp_send_stop();
256256
257257 /* for crashing cpu */
258258 crash_save_cpu(regs, smp_processor_id());
arch/arm64/kernel/perf_callchain.c+0−1
@@ -162,7 +162,6 @@ void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, · 161 unmodified lines
162162 }
163163
164164 frame.fp = regs->regs[29];
165− frame.sp = regs->sp;
166165 frame.pc = regs->pc;
167166 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
168167 frame.graph = current->curr_ret_stack;
arch/arm64/kernel/perf_event.c+85−123
@@ -202,55 +202,6 @@ static const unsigned armv8_pmuv3_perf_map[PERF_COUNT_HW_MAX] = { · 201 unmodified lines
202202 [PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = ARMV8_PMUV3_PERFCTR_STALL_BACKEND,
203203 };
204204
205−/* ARM Cortex-A53 HW events mapping. */
206−static const unsigned armv8_a53_perf_map[PERF_COUNT_HW_MAX] = {
207− PERF_MAP_ALL_UNSUPPORTED,
208− [PERF_COUNT_HW_CPU_CYCLES] = ARMV8_PMUV3_PERFCTR_CPU_CYCLES,
209− [PERF_COUNT_HW_INSTRUCTIONS] = ARMV8_PMUV3_PERFCTR_INST_RETIRED,
210− [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV8_PMUV3_PERFCTR_L1D_CACHE,
211− [PERF_COUNT_HW_CACHE_MISSES] = ARMV8_PMUV3_PERFCTR_L1D_CACHE_REFILL,
212− [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV8_PMUV3_PERFCTR_PC_WRITE_RETIRED,
213− [PERF_COUNT_HW_BRANCH_MISSES] = ARMV8_PMUV3_PERFCTR_BR_MIS_PRED,
214− [PERF_COUNT_HW_BUS_CYCLES] = ARMV8_PMUV3_PERFCTR_BUS_CYCLES,
215−};
216−
217−/* ARM Cortex-A57 and Cortex-A72 events mapping. */
218−static const unsigned armv8_a57_perf_map[PERF_COUNT_HW_MAX] = {
219− PERF_MAP_ALL_UNSUPPORTED,
220− [PERF_COUNT_HW_CPU_CYCLES] = ARMV8_PMUV3_PERFCTR_CPU_CYCLES,
221− [PERF_COUNT_HW_INSTRUCTIONS] = ARMV8_PMUV3_PERFCTR_INST_RETIRED,
222− [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV8_PMUV3_PERFCTR_L1D_CACHE,
223− [PERF_COUNT_HW_CACHE_MISSES] = ARMV8_PMUV3_PERFCTR_L1D_CACHE_REFILL,
224− [PERF_COUNT_HW_BRANCH_MISSES] = ARMV8_PMUV3_PERFCTR_BR_MIS_PRED,
225− [PERF_COUNT_HW_BUS_CYCLES] = ARMV8_PMUV3_PERFCTR_BUS_CYCLES,
226−};
227−
228−static const unsigned armv8_thunder_perf_map[PERF_COUNT_HW_MAX] = {
229− PERF_MAP_ALL_UNSUPPORTED,
230− [PERF_COUNT_HW_CPU_CYCLES] = ARMV8_PMUV3_PERFCTR_CPU_CYCLES,
231− [PERF_COUNT_HW_INSTRUCTIONS] = ARMV8_PMUV3_PERFCTR_INST_RETIRED,
232− [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV8_PMUV3_PERFCTR_L1D_CACHE,
233− [PERF_COUNT_HW_CACHE_MISSES] = ARMV8_PMUV3_PERFCTR_L1D_CACHE_REFILL,
234− [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV8_PMUV3_PERFCTR_PC_WRITE_RETIRED,
235− [PERF_COUNT_HW_BRANCH_MISSES] = ARMV8_PMUV3_PERFCTR_BR_MIS_PRED,
236− [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = ARMV8_PMUV3_PERFCTR_STALL_FRONTEND,
237− [PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = ARMV8_PMUV3_PERFCTR_STALL_BACKEND,
238−};
239−
240−/* Broadcom Vulcan events mapping */
241−static const unsigned armv8_vulcan_perf_map[PERF_COUNT_HW_MAX] = {
242− PERF_MAP_ALL_UNSUPPORTED,
243− [PERF_COUNT_HW_CPU_CYCLES] = ARMV8_PMUV3_PERFCTR_CPU_CYCLES,
244− [PERF_COUNT_HW_INSTRUCTIONS] = ARMV8_PMUV3_PERFCTR_INST_RETIRED,
245− [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV8_PMUV3_PERFCTR_L1D_CACHE,
246− [PERF_COUNT_HW_CACHE_MISSES] = ARMV8_PMUV3_PERFCTR_L1D_CACHE_REFILL,
247− [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV8_PMUV3_PERFCTR_BR_RETIRED,
248− [PERF_COUNT_HW_BRANCH_MISSES] = ARMV8_PMUV3_PERFCTR_BR_MIS_PRED,
249− [PERF_COUNT_HW_BUS_CYCLES] = ARMV8_PMUV3_PERFCTR_BUS_CYCLES,
250− [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = ARMV8_PMUV3_PERFCTR_STALL_FRONTEND,
251− [PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = ARMV8_PMUV3_PERFCTR_STALL_BACKEND,
252−};
253−
254205 static const unsigned armv8_pmuv3_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
255206 [PERF_COUNT_HW_CACHE_OP_MAX]
256207 [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
@@ -281,27 +232,10 @@ static const unsigned armv8_a53_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] · 24 unmodified lines
281232 [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
282233 PERF_CACHE_MAP_ALL_UNSUPPORTED,
283234
284− [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_L1D_CACHE,
285− [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L1D_CACHE_REFILL,
286− [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_L1D_CACHE,
287− [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L1D_CACHE_REFILL,
288235 [C(L1D)][C(OP_PREFETCH)][C(RESULT_MISS)] = ARMV8_A53_PERFCTR_PREF_LINEFILL,
289236
290− [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_L1I_CACHE,
291− [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L1I_CACHE_REFILL,
292−
293− [C(LL)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_L2D_CACHE,
294− [C(LL)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L2D_CACHE_REFILL,
295− [C(LL)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_L2D_CACHE,
296− [C(LL)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L2D_CACHE_REFILL,
297−
298− [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L1D_TLB_REFILL,
299− [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L1I_TLB_REFILL,
300−
301− [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_BR_PRED,
302− [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_BR_MIS_PRED,
303− [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_BR_PRED,
304− [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_BR_MIS_PRED,
237+ [C(NODE)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_RD,
238+ [C(NODE)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_WR,
305239 };
306240
307241 static const unsigned armv8_a57_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
@@ -314,18 +248,26 @@ static const unsigned armv8_a57_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] · 6 unmodified lines
314248 [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_L1D_CACHE_WR,
315249 [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV8_IMPDEF_PERFCTR_L1D_CACHE_REFILL_WR,
316250
317− [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_L1I_CACHE,
318− [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L1I_CACHE_REFILL,
319−
320251 [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_REFILL_RD,
321252 [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_REFILL_WR,
322253
323− [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L1I_TLB_REFILL,
254+ [C(NODE)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_RD,
255+ [C(NODE)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_WR,
256+};
324257
325− [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_BR_PRED,
326− [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_BR_MIS_PRED,
327− [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_BR_PRED,
328− [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_BR_MIS_PRED,
258+static const unsigned armv8_a73_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
259+ [PERF_COUNT_HW_CACHE_OP_MAX]
260+ [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
261+ PERF_CACHE_MAP_ALL_UNSUPPORTED,
262+
263+ [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_L1D_CACHE_RD,
264+ [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_L1D_CACHE_WR,
265+
266+ [C(NODE)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_RD,
267+ [C(NODE)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_WR,
268+
269+ [C(NODE)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_RD,
270+ [C(NODE)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_WR,
329271 };
330272
331273 static const unsigned armv8_thunder_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
@@ -340,8 +282,6 @@ static const unsigned armv8_thunder_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] · 8 unmodified lines
340282 [C(L1D)][C(OP_PREFETCH)][C(RESULT_ACCESS)] = ARMV8_THUNDER_PERFCTR_L1D_CACHE_PREF_ACCESS,
341283 [C(L1D)][C(OP_PREFETCH)][C(RESULT_MISS)] = ARMV8_THUNDER_PERFCTR_L1D_CACHE_PREF_MISS,
342284
343− [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_L1I_CACHE,
344− [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L1I_CACHE_REFILL,
345285 [C(L1I)][C(OP_PREFETCH)][C(RESULT_ACCESS)] = ARMV8_THUNDER_PERFCTR_L1I_CACHE_PREF_ACCESS,
346286 [C(L1I)][C(OP_PREFETCH)][C(RESULT_MISS)] = ARMV8_THUNDER_PERFCTR_L1I_CACHE_PREF_MISS,
347287
@@ -349,13 +289,6 @@ static const unsigned armv8_thunder_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] · 1 unmodified line
349289 [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_REFILL_RD,
350290 [C(DTLB)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_WR,
351291 [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_REFILL_WR,
352−
353− [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L1I_TLB_REFILL,
354−
355− [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_BR_PRED,
356− [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_BR_MIS_PRED,
357− [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_BR_PRED,
358− [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_BR_MIS_PRED,
359292 };
360293
361294 static const unsigned armv8_vulcan_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
@@ -368,22 +301,11 @@ static const unsigned armv8_vulcan_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] · 6 unmodified lines
368301 [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_L1D_CACHE_WR,
369302 [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV8_IMPDEF_PERFCTR_L1D_CACHE_REFILL_WR,
370303
371− [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_L1I_CACHE,
372− [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L1I_CACHE_REFILL,
373−
374− [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L1I_TLB_REFILL,
375− [C(ITLB)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_L1I_TLB,
376−
377304 [C(DTLB)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_RD,
378305 [C(DTLB)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_WR,
379306 [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_REFILL_RD,
380307 [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_REFILL_WR,
381308
382− [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_BR_PRED,
383− [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_BR_MIS_PRED,
384− [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_BR_PRED,
385− [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_BR_MIS_PRED,
386−
387309 [C(NODE)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_RD,
388310 [C(NODE)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_WR,
389311 };
@@ -846,17 +768,14 @@ static int armv8pmu_get_event_idx(struct pmu_hw_events *cpuc, · 456 unmodified lines
846768 struct hw_perf_event *hwc = &event->hw;
847769 unsigned long evtype = hwc->config_base & ARMV8_PMU_EVTYPE_EVENT;
848770
849− /* Always place a cycle counter into the cycle counter. */
771+ /* Always prefer to place a cycle counter into the cycle counter. */
850772 if (evtype == ARMV8_PMUV3_PERFCTR_CPU_CYCLES) {
851− if (test_and_set_bit(ARMV8_IDX_CYCLE_COUNTER, cpuc->used_mask))
852− return -EAGAIN;
853−
854− return ARMV8_IDX_CYCLE_COUNTER;
773+ if (!test_and_set_bit(ARMV8_IDX_CYCLE_COUNTER, cpuc->used_mask))
774+ return ARMV8_IDX_CYCLE_COUNTER;
855775 }
856776
857777 /*
858− * For anything other than a cycle counter, try and use
859− * the events counters
778+ * Otherwise use events counters
860779 */
861780 for (idx = ARMV8_IDX_COUNTER0; idx < cpu_pmu->num_events; ++idx) {
862781 if (!test_and_set_bit(idx, cpuc->used_mask))
@@ -924,7 +843,13 @@ static void armv8pmu_reset(void *info) · 61 unmodified lines
924843 ARMV8_PMU_PMCR_LC);
925844 }
926845
927−static int armv8_pmuv3_map_event(struct perf_event *event)
846+static int __armv8_pmuv3_map_event(struct perf_event *event,
847+ const unsigned (*extra_event_map)
848+ [PERF_COUNT_HW_MAX],
849+ const unsigned (*extra_cache_map)
850+ [PERF_COUNT_HW_CACHE_MAX]
851+ [PERF_COUNT_HW_CACHE_OP_MAX]
852+ [PERF_COUNT_HW_CACHE_RESULT_MAX])
928853 {
929854 int hw_event_id;
930855 struct arm_pmu *armpmu = to_arm_pmu(event->pmu);
@@ -932,44 +857,47 @@ static int armv8_pmuv3_map_event(struct perf_event *event) · 1 unmodified line
932857 hw_event_id = armpmu_map_event(event, &armv8_pmuv3_perf_map,
933858 &armv8_pmuv3_perf_cache_map,
934859 ARMV8_PMU_EVTYPE_EVENT);
935− if (hw_event_id < 0)
936− return hw_event_id;
937860
938− /* disable micro/arch events not supported by this PMU */
939− if ((hw_event_id < ARMV8_PMUV3_MAX_COMMON_EVENTS) &&
940− !test_bit(hw_event_id, armpmu->pmceid_bitmap)) {
941− return -EOPNOTSUPP;
861+ /* Onl expose micro/arch events supported by this PMU */
862+ if ((hw_event_id > 0) && (hw_event_id < ARMV8_PMUV3_MAX_COMMON_EVENTS)
863+ && test_bit(hw_event_id, armpmu->pmceid_bitmap)) {
864+ return hw_event_id;
942865 }
943866
944− return hw_event_id;
867+ return armpmu_map_event(event, extra_event_map, extra_cache_map,
868+ ARMV8_PMU_EVTYPE_EVENT);
869+}
870+
871+static int armv8_pmuv3_map_event(struct perf_event *event)
872+{
873+ return __armv8_pmuv3_map_event(event, NULL, NULL);
945874 }
946875
947876 static int armv8_a53_map_event(struct perf_event *event)
948877 {
949− return armpmu_map_event(event, &armv8_a53_perf_map,
950− &armv8_a53_perf_cache_map,
951− ARMV8_PMU_EVTYPE_EVENT);
878+ return __armv8_pmuv3_map_event(event, NULL, &armv8_a53_perf_cache_map);
952879 }
953880
954881 static int armv8_a57_map_event(struct perf_event *event)
955882 {
956− return armpmu_map_event(event, &armv8_a57_perf_map,
957− &armv8_a57_perf_cache_map,
958− ARMV8_PMU_EVTYPE_EVENT);
883+ return __armv8_pmuv3_map_event(event, NULL, &armv8_a57_perf_cache_map);
884+}
885+
886+static int armv8_a73_map_event(struct perf_event *event)
887+{
888+ return __armv8_pmuv3_map_event(event, NULL, &armv8_a73_perf_cache_map);
959889 }
960890
961891 static int armv8_thunder_map_event(struct perf_event *event)
962892 {
963− return armpmu_map_event(event, &armv8_thunder_perf_map,
964− &armv8_thunder_perf_cache_map,
965− ARMV8_PMU_EVTYPE_EVENT);
893+ return __armv8_pmuv3_map_event(event, NULL,
894+ &armv8_thunder_perf_cache_map);
966895 }
967896
968897 static int armv8_vulcan_map_event(struct perf_event *event)
969898 {
970− return armpmu_map_event(event, &armv8_vulcan_perf_map,
971− &armv8_vulcan_perf_cache_map,
972− ARMV8_PMU_EVTYPE_EVENT);
899+ return __armv8_pmuv3_map_event(event, NULL,
900+ &armv8_vulcan_perf_cache_map);
973901 }
974902
975903 struct armv8pmu_probe_info {
@@ -1062,6 +990,22 @@ static int armv8_pmuv3_init(struct arm_pmu *cpu_pmu) · 86 unmodified lines
1062990 return 0;
1063991 }
1064992
993+static int armv8_a35_pmu_init(struct arm_pmu *cpu_pmu)
994+{
995+ int ret = armv8_pmu_init(cpu_pmu);
996+ if (ret)
997+ return ret;
998+
999+ cpu_pmu->name = "armv8_cortex_a35";
1000+ cpu_pmu->map_event = armv8_a53_map_event;
1001+ cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] =
1002+ &armv8_pmuv3_events_attr_group;
1003+ cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] =
1004+ &armv8_pmuv3_format_attr_group;
1005+
1006+ return 0;
1007+}
1008+
10651009 static int armv8_a53_pmu_init(struct arm_pmu *cpu_pmu)
10661010 {
10671011 int ret = armv8_pmu_init(cpu_pmu);
@@ -1110,6 +1054,22 @@ static int armv8_a72_pmu_init(struct arm_pmu *cpu_pmu) · 42 unmodified lines
11101054 return 0;
11111055 }
11121056
1057+static int armv8_a73_pmu_init(struct arm_pmu *cpu_pmu)
1058+{
1059+ int ret = armv8_pmu_init(cpu_pmu);
1060+ if (ret)
1061+ return ret;
1062+
1063+ cpu_pmu->name = "armv8_cortex_a73";
1064+ cpu_pmu->map_event = armv8_a73_map_event;
1065+ cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] =
1066+ &armv8_pmuv3_events_attr_group;
1067+ cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] =
1068+ &armv8_pmuv3_format_attr_group;
1069+
1070+ return 0;
1071+}
1072+
11131073 static int armv8_thunder_pmu_init(struct arm_pmu *cpu_pmu)
11141074 {
11151075 int ret = armv8_pmu_init(cpu_pmu);
@@ -1144,9 +1104,11 @@ static int armv8_vulcan_pmu_init(struct arm_pmu *cpu_pmu) · 28 unmodified lines
11441104
11451105 static const struct of_device_id armv8_pmu_of_device_ids[] = {
11461106 {.compatible = "arm,armv8-pmuv3", .data = armv8_pmuv3_init},
1107+ {.compatible = "arm,cortex-a35-pmu", .data = armv8_a35_pmu_init},
11471108 {.compatible = "arm,cortex-a53-pmu", .data = armv8_a53_pmu_init},
11481109 {.compatible = "arm,cortex-a57-pmu", .data = armv8_a57_pmu_init},
11491110 {.compatible = "arm,cortex-a72-pmu", .data = armv8_a72_pmu_init},
1111+ {.compatible = "arm,cortex-a73-pmu", .data = armv8_a73_pmu_init},
11501112 {.compatible = "cavium,thunder-pmu", .data = armv8_thunder_pmu_init},
11511113 {.compatible = "brcm,vulcan-pmu", .data = armv8_vulcan_pmu_init},
11521114 {},
arch/arm64/kernel/probes/uprobes.c+1−1
@@ -40,7 +40,7 @@ int arch_uprobe_analyze_insn(struct arch_uprobe *auprobe, struct mm_struct *mm, · 39 unmodified lines
4040 probe_opcode_t insn;
4141
4242 /* TODO: Currently we do not support AARCH32 instruction probing */
43− if (test_bit(TIF_32BIT, &mm->context.flags))
43+ if (mm->context.flags & MMCF_AARCH32)
4444 return -ENOTSUPP;
4545 else if (!IS_ALIGNED(addr, AARCH64_INSN_SIZE))
4646 return -EINVAL;
arch/arm64/kernel/process.c+9−4
@@ -384,15 +384,12 @@ unsigned long get_wchan(struct task_struct *p) · 383 unmodified lines
384384 return 0;
385385
386386 frame.fp = thread_saved_fp(p);
387− frame.sp = thread_saved_sp(p);
388387 frame.pc = thread_saved_pc(p);
389388 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
390389 frame.graph = p->curr_ret_stack;
391390 #endif
392391 do {
393− if (frame.sp < stack_page ||
394− frame.sp >= stack_page + THREAD_SIZE ||
395− unwind_frame(p, &frame))
392+ if (unwind_frame(p, &frame))
396393 goto out;
397394 if (!in_sched_functions(frame.pc)) {
398395 ret = frame.pc;
@@ -419,3 +416,11 @@ unsigned long arch_randomize_brk(struct mm_struct *mm) · 20 unmodified lines
419416 else
420417 return randomize_page(mm->brk, SZ_1G);
421418 }
419+
420+/*
421+ * Called from setup_new_exec() after (COMPAT_)SET_PERSONALITY.
422+ */
423+void arch_setup_new_exec(void)
424+{
425+ current->mm->context.flags = is_compat_task() ? MMCF_AARCH32 : 0;
426+}
arch/arm64/kernel/ptrace.c+3−2
@@ -42,6 +42,7 @@ · 41 unmodified lines
4242 #include <asm/compat.h>
4343 #include <asm/debug-monitors.h>
4444 #include <asm/pgtable.h>
45+#include <asm/stacktrace.h>
4546 #include <asm/syscall.h>
4647 #include <asm/traps.h>
4748 #include <asm/system_misc.h>
@@ -127,7 +128,7 @@ static bool regs_within_kernel_stack(struct pt_regs *regs, unsigned long addr) · 79 unmodified lines
127128 {
128129 return ((addr & ~(THREAD_SIZE - 1)) ==
129130 (kernel_stack_pointer(regs) & ~(THREAD_SIZE - 1))) ||
130− on_irq_stack(addr, raw_smp_processor_id());
131+ on_irq_stack(addr);
131132 }
132133
133134 /**
@@ -1363,7 +1364,7 @@ static void tracehook_report_syscall(struct pt_regs *regs, · 1229 unmodified lines
13631364 if (dir == PTRACE_SYSCALL_EXIT)
13641365 tracehook_report_syscall_exit(regs, 0);
13651366 else if (tracehook_report_syscall_entry(regs))
1366− regs->syscallno = ~0UL;
1367+ forget_syscall(regs);
13671368
13681369 regs->regs[regno] = saved_reg;
13691370 }
arch/arm64/kernel/return_address.c+0−1
@@ -42,7 +42,6 @@ void *return_address(unsigned int level) · 41 unmodified lines
4242 data.addr = NULL;
4343
4444 frame.fp = (unsigned long)__builtin_frame_address(0);
45− frame.sp = current_stack_pointer;
4645 frame.pc = (unsigned long)return_address; /* dummy */
4746 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
4847 frame.graph = current->curr_ret_stack;
arch/arm64/kernel/signal.c+5−5
@@ -37,6 +37,7 @@ · 36 unmodified lines
3737 #include <asm/ucontext.h>
3838 #include <asm/unistd.h>
3939 #include <asm/fpsimd.h>
40+#include <asm/ptrace.h>
4041 #include <asm/signal32.h>
4142 #include <asm/vdso.h>
4243
@@ -388,7 +389,7 @@ static int restore_sigframe(struct pt_regs *regs, · 345 unmodified lines
388389 /*
389390 * Avoid sys_rt_sigreturn() restarting.
390391 */
391− regs->syscallno = ~0UL;
392+ forget_syscall(regs);
392393
393394 err |= !valid_user_regs(&regs->user_regs, current);
394395 if (err == 0)
@@ -674,13 +675,12 @@ static void do_signal(struct pt_regs *regs) · 279 unmodified lines
674675 {
675676 unsigned long continue_addr = 0, restart_addr = 0;
676677 int retval = 0;
677− int syscall = (int)regs->syscallno;
678678 struct ksignal ksig;
679679
680680 /*
681681 * If we were from a system call, check for system call restarting...
682682 */
683− if (syscall >= 0) {
683+ if (in_syscall(regs)) {
684684 continue_addr = regs->pc;
685685 restart_addr = continue_addr - (compat_thumb_mode(regs) ? 2 : 4);
686686 retval = regs->regs[0];
@@ -688,7 +688,7 @@ static void do_signal(struct pt_regs *regs) · 1 unmodified line
688688 /*
689689 * Avoid additional syscall restarting via ret_to_user.
690690 */
691− regs->syscallno = ~0UL;
691+ forget_syscall(regs);
692692
693693 /*
694694 * Prepare for system call restart. We do this here so that a
@@ -732,7 +732,7 @@ static void do_signal(struct pt_regs *regs) · 37 unmodified lines
732732 * Handle restarting a different system call. As above, if a debugger
733733 * has chosen to restart at a different PC, ignore the restart.
734734 */
735− if (syscall >= 0 && regs->pc == restart_addr) {
735+ if (in_syscall(regs) && regs->pc == restart_addr) {
736736 if (retval == -ERESTART_RESTARTBLOCK)
737737 setup_restart_syscall(regs);
738738 user_rewind_single_step(current);
arch/arm64/kernel/signal32.c+1−1
@@ -354,7 +354,7 @@ static int compat_restore_sigframe(struct pt_regs *regs, · 353 unmodified lines
354354 /*
355355 * Avoid compat_sys_sigreturn() restarting.
356356 */
357− regs->syscallno = ~0UL;
357+ forget_syscall(regs);
358358
359359 err |= !valid_user_regs(&regs->user_regs, current);
360360
arch/arm64/kernel/smp.c+12−2
@@ -154,7 +154,7 @@ int __cpu_up(unsigned int cpu, struct task_struct *idle) · 153 unmodified lines
154154 * page tables.
155155 */
156156 secondary_data.task = idle;
157− secondary_data.stack = task_stack_page(idle) + THREAD_START_SP;
157+ secondary_data.stack = task_stack_page(idle) + THREAD_SIZE;
158158 update_cpu_boot_status(CPU_MMU_OFF);
159159 __flush_dcache_area(&secondary_data, sizeof(secondary_data));
160160
@@ -977,11 +977,21 @@ void smp_send_stop(void) · 816 unmodified lines
977977 }
978978
979979 #ifdef CONFIG_KEXEC_CORE
980−void smp_send_crash_stop(void)
980+void crash_smp_send_stop(void)
981981 {
982+ static int cpus_stopped;
982983 cpumask_t mask;
983984 unsigned long timeout;
984985
986+ /*
987+ * This function can be called twice in panic path, but obviously
988+ * we execute this only once.
989+ */
990+ if (cpus_stopped)
991+ return;
992+
993+ cpus_stopped = 1;
994+
985995 if (num_online_cpus() == 1)
986996 return;
987997
arch/arm64/kernel/stacktrace.c+10−50
@@ -42,33 +42,17 @@ · 41 unmodified lines
4242 */
4343 int notrace unwind_frame(struct task_struct *tsk, struct stackframe *frame)
4444 {
45− unsigned long high, low;
4645 unsigned long fp = frame->fp;
47− unsigned long irq_stack_ptr;
46+
47+ if (fp & 0xf)
48+ return -EINVAL;
4849
4950 if (!tsk)
5051 tsk = current;
5152
52− /*
53− * Switching between stacks is valid when tracing current and in
54− * non-preemptible context.
55− */
56− if (tsk == current && !preemptible())
57− irq_stack_ptr = IRQ_STACK_PTR(smp_processor_id());
58− else
59− irq_stack_ptr = 0;
60−
61− low = frame->sp;
62− /* irq stacks are not THREAD_SIZE aligned */
63− if (on_irq_stack(frame->sp, raw_smp_processor_id()))
64− high = irq_stack_ptr;
65− else
66− high = ALIGN(low, THREAD_SIZE) - 0x20;
67−
68− if (fp < low || fp > high || fp & 0xf)
53+ if (!on_accessible_stack(tsk, fp))
6954 return -EINVAL;
7055
71− frame->sp = fp + 0x10;
7256 frame->fp = READ_ONCE_NOCHECK(*(unsigned long *)(fp));
7357 frame->pc = READ_ONCE_NOCHECK(*(unsigned long *)(fp + 8));
7458
@@ -86,34 +70,13 @@ int notrace unwind_frame(struct task_struct *tsk, struct stackframe *frame) · 11 unmodified lines
8670 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
8771
8872 /*
89− * Check whether we are going to walk through from interrupt stack
90− * to task stack.
91− * If we reach the end of the stack - and its an interrupt stack,
92− * unpack the dummy frame to find the original elr.
93− *
94− * Check the frame->fp we read from the bottom of the irq_stack,
95− * and the original task stack pointer are both in current->stack.
73+ * Frames created upon entry from EL0 have NULL FP and PC values, so
74+ * don't bother reporting these. Frames created by __noreturn functions
75+ * might have a valid FP even if PC is bogus, so only terminate where
76+ * both are NULL.
9677 */
97− if (frame->sp == irq_stack_ptr) {
98− struct pt_regs *irq_args;
99− unsigned long orig_sp = IRQ_STACK_TO_TASK_STACK(irq_stack_ptr);
100−
101− if (object_is_on_stack((void *)orig_sp) &&
102− object_is_on_stack((void *)frame->fp)) {
103− frame->sp = orig_sp;
104−
105− /* orig_sp is the saved pt_regs, find the elr */
106− irq_args = (struct pt_regs *)orig_sp;
107− frame->pc = irq_args->pc;
108− } else {
109− /*
110− * This frame has a non-standard format, and we
111− * didn't fix it, because the data looked wrong.
112− * Refuse to output this frame.
113− */
114− return -EINVAL;
115− }
116− }
78+ if (!frame->fp && !frame->pc)
79+ return -EINVAL;
11780
11881 return 0;
11982 }
@@ -167,7 +130,6 @@ void save_stack_trace_regs(struct pt_regs *regs, struct stack_trace *trace) · 47 unmodified lines
167130 data.no_sched_functions = 0;
168131
169132 frame.fp = regs->regs[29];
170− frame.sp = regs->sp;
171133 frame.pc = regs->pc;
172134 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
173135 frame.graph = current->curr_ret_stack;
@@ -192,12 +154,10 @@ void save_stack_trace_tsk(struct task_struct *tsk, struct stack_trace *trace) · 18 unmodified lines
192154 if (tsk != current) {
193155 data.no_sched_functions = 1;
194156 frame.fp = thread_saved_fp(tsk);
195− frame.sp = thread_saved_sp(tsk);
196157 frame.pc = thread_saved_pc(tsk);
197158 } else {
198159 data.no_sched_functions = 0;
199160 frame.fp = (unsigned long)__builtin_frame_address(0);
200− frame.sp = current_stack_pointer;
201161 frame.pc = (unsigned long)save_stack_trace_tsk;
202162 }
203163 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
arch/arm64/kernel/time.c+0−1
@@ -50,7 +50,6 @@ unsigned long profile_pc(struct pt_regs *regs) · 49 unmodified lines
5050 return regs->pc;
5151
5252 frame.fp = regs->regs[29];
53− frame.sp = regs->sp;
5453 frame.pc = regs->pc;
5554 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
5655 frame.graph = -1; /* no task info */
arch/arm64/kernel/traps.c+49−30
@@ -32,6 +32,7 @@ · 31 unmodified lines
3232 #include <linux/sched/signal.h>
3333 #include <linux/sched/debug.h>
3434 #include <linux/sched/task_stack.h>
35+#include <linux/sizes.h>
3536 #include <linux/syscalls.h>
3637 #include <linux/mm_types.h>
3738
@@ -41,6 +42,7 @@ · 3 unmodified lines
4142 #include <asm/esr.h>
4243 #include <asm/insn.h>
4344 #include <asm/traps.h>
45+#include <asm/smp.h>
4446 #include <asm/stack_pointer.h>
4547 #include <asm/stacktrace.h>
4648 #include <asm/exception.h>
@@ -143,7 +145,6 @@ static void dump_instr(const char *lvl, struct pt_regs *regs) · 96 unmodified lines
143145 void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk)
144146 {
145147 struct stackframe frame;
146− unsigned long irq_stack_ptr;
147148 int skip;
148149
149150 pr_debug("%s(regs = %p tsk = %p)\n", __func__, regs, tsk);
@@ -154,25 +155,14 @@ void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk) · 4 unmodified lines
154155 if (!try_get_task_stack(tsk))
155156 return;
156157
157− /*
158− * Switching between stacks is valid when tracing current and in
159− * non-preemptible context.
160− */
161− if (tsk == current && !preemptible())
162− irq_stack_ptr = IRQ_STACK_PTR(smp_processor_id());
163− else
164− irq_stack_ptr = 0;
165−
166158 if (tsk == current) {
167159 frame.fp = (unsigned long)__builtin_frame_address(0);
168− frame.sp = current_stack_pointer;
169160 frame.pc = (unsigned long)dump_backtrace;
170161 } else {
171162 /*
172163 * task blocked in __switch_to
173164 */
174165 frame.fp = thread_saved_fp(tsk);
175− frame.sp = thread_saved_sp(tsk);
176166 frame.pc = thread_saved_pc(tsk);
177167 }
178168 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
@@ -182,13 +172,12 @@ void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk) · 3 unmodified lines
182172 skip = !!regs;
183173 printk("Call trace:\n");
184174 while (1) {
185− unsigned long where = frame.pc;
186175 unsigned long stack;
187176 int ret;
188177
189178 /* skip until specified stack frame */
190179 if (!skip) {
191− dump_backtrace_entry(where);
180+ dump_backtrace_entry(frame.pc);
192181 } else if (frame.fp == regs->regs[29]) {
193182 skip = 0;
194183 /*
@@ -203,20 +192,12 @@ void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk) · 8 unmodified lines
203192 ret = unwind_frame(tsk, &frame);
204193 if (ret < 0)
205194 break;
206− stack = frame.sp;
207− if (in_exception_text(where)) {
208− /*
209− * If we switched to the irq_stack before calling this
210− * exception handler, then the pt_regs will be on the
211− * task stack. The easiest way to tell is if the large
212− * pt_regs would overlap with the end of the irq_stack.
213− */
214− if (stack < irq_stack_ptr &&
215− (stack + sizeof(struct pt_regs)) > irq_stack_ptr)
216− stack = IRQ_STACK_TO_TASK_STACK(irq_stack_ptr);
195+ if (in_entry_text(frame.pc)) {
196+ stack = frame.fp - offsetof(struct pt_regs, stackframe);
217197
218− dump_mem("", "Exception stack", stack,
219− stack + sizeof(struct pt_regs));
198+ if (on_accessible_stack(tsk, stack))
199+ dump_mem("", "Exception stack", stack,
200+ stack + sizeof(struct pt_regs));
220201 }
221202 }
222203
@@ -257,8 +238,6 @@ static int __die(const char *str, int err, struct pt_regs *regs) · 34 unmodified lines
257238 end_of_stack(tsk));
258239
259240 if (!user_mode(regs)) {
260− dump_mem(KERN_EMERG, "Stack: ", regs->sp,
261− THREAD_SIZE + (unsigned long)task_stack_page(tsk));
262241 dump_backtrace(regs, tsk);
263242 dump_instr(KERN_EMERG, regs);
264243 }
@@ -484,6 +463,9 @@ static void user_cache_maint_handler(unsigned int esr, struct pt_regs *regs) · 219 unmodified lines
484463 case ESR_ELx_SYS64_ISS_CRM_DC_CVAC: /* DC CVAC, gets promoted */
485464 __user_cache_maint("dc civac", address, ret);
486465 break;
466+ case ESR_ELx_SYS64_ISS_CRM_DC_CVAP: /* DC CVAP */
467+ __user_cache_maint("sys 3, c7, c12, 1", address, ret);
468+ break;
487469 case ESR_ELx_SYS64_ISS_CRM_DC_CIVAC: /* DC CIVAC */
488470 __user_cache_maint("dc civac", address, ret);
489471 break;
@@ -593,7 +575,7 @@ asmlinkage long do_ni_syscall(struct pt_regs *regs) · 103 unmodified lines
593575
594576 if (show_unhandled_signals_ratelimited()) {
595577 pr_info("%s[%d]: syscall %d\n", current->comm,
596− task_pid_nr(current), (int)regs->syscallno);
578+ task_pid_nr(current), regs->syscallno);
597579 dump_instr("", regs);
598580 if (user_mode(regs))
599581 __show_regs(regs);
@@ -689,6 +671,43 @@ asmlinkage void bad_el0_sync(struct pt_regs *regs, int reason, unsigned int esr) · 89 unmodified lines
689671 force_sig_info(info.si_signo, &info, current);
690672 }
691673
674+#ifdef CONFIG_VMAP_STACK
675+
676+DEFINE_PER_CPU(unsigned long [OVERFLOW_STACK_SIZE/sizeof(long)], overflow_stack)
677+ __aligned(16);
678+
679+asmlinkage void handle_bad_stack(struct pt_regs *regs)
680+{
681+ unsigned long tsk_stk = (unsigned long)current->stack;
682+ unsigned long irq_stk = (unsigned long)this_cpu_read(irq_stack_ptr);
683+ unsigned long ovf_stk = (unsigned long)this_cpu_ptr(overflow_stack);
684+ unsigned int esr = read_sysreg(esr_el1);
685+ unsigned long far = read_sysreg(far_el1);
686+
687+ console_verbose();
688+ pr_emerg("Insufficient stack space to handle exception!");
689+
690+ pr_emerg("ESR: 0x%08x -- %s\n", esr, esr_get_class_string(esr));
691+ pr_emerg("FAR: 0x%016lx\n", far);
692+
693+ pr_emerg("Task stack: [0x%016lx..0x%016lx]\n",
694+ tsk_stk, tsk_stk + THREAD_SIZE);
695+ pr_emerg("IRQ stack: [0x%016lx..0x%016lx]\n",
696+ irq_stk, irq_stk + THREAD_SIZE);
697+ pr_emerg("Overflow stack: [0x%016lx..0x%016lx]\n",
698+ ovf_stk, ovf_stk + OVERFLOW_STACK_SIZE);
699+
700+ __show_regs(regs);
701+
702+ /*
703+ * We use nmi_panic to limit the potential for recusive overflows, and
704+ * to get a better stack trace.
705+ */
706+ nmi_panic(NULL, "kernel stack overflow");
707+ cpu_park_loop();
708+}
709+#endif
710+
692711 void __pte_error(const char *file, int line, unsigned long val)
693712 {
694713 pr_err("%s:%d: bad pte %016lx.\n", file, line, val);
arch/arm64/kernel/vdso.c+15−0
@@ -110,12 +110,27 @@ int aarch32_setup_vectors_page(struct linux_binprm *bprm, int uses_interp) · 109 unmodified lines
110110 }
111111 #endif /* CONFIG_COMPAT */
112112
113+static int vdso_mremap(const struct vm_special_mapping *sm,
114+ struct vm_area_struct *new_vma)
115+{
116+ unsigned long new_size = new_vma->vm_end - new_vma->vm_start;
117+ unsigned long vdso_size = vdso_end - vdso_start;
118+
119+ if (vdso_size != new_size)
120+ return -EINVAL;
121+
122+ current->mm->context.vdso = (void *)new_vma->vm_start;
123+
124+ return 0;
125+}
126+
113127 static struct vm_special_mapping vdso_spec[2] __ro_after_init = {
114128 {
115129 .name = "[vvar]",
116130 },
117131 {
118132 .name = "[vdso]",
133+ .mremap = vdso_mremap,
119134 },
120135 };
121136
arch/arm64/kernel/vmlinux.lds.S+1−17
@@ -72,22 +72,6 @@ PECOFF_FILE_ALIGNMENT = 0x200; · 71 unmodified lines
7272 #define PECOFF_EDATA_PADDING
7373 #endif
7474
75−#if defined(CONFIG_DEBUG_ALIGN_RODATA)
76−/*
77− * 4 KB granule: 1 level 2 entry
78− * 16 KB granule: 128 level 3 entries, with contiguous bit
79− * 64 KB granule: 32 level 3 entries, with contiguous bit
80− */
81−#define SEGMENT_ALIGN SZ_2M
82−#else
83−/*
84− * 4 KB granule: 16 level 3 entries, with contiguous bit
85− * 16 KB granule: 4 level 3 entries, without contiguous bit
86− * 64 KB granule: 1 level 3 entry
87− */
88−#define SEGMENT_ALIGN SZ_64K
89−#endif
90−
9175 SECTIONS
9276 {
9377 /*
@@ -192,7 +176,7 @@ SECTIONS · 98 unmodified lines
192176
193177 _data = .;
194178 _sdata = .;
195− RW_DATA_SECTION(L1_CACHE_BYTES, PAGE_SIZE, THREAD_SIZE)
179+ RW_DATA_SECTION(L1_CACHE_BYTES, PAGE_SIZE, THREAD_ALIGN)
196180
197181 /*
198182 * Data written with the MMU off but read with the MMU on requires
arch/arm64/kvm/hyp/s2-setup.c+1−1
@@ -70,7 +70,7 @@ u32 __hyp_text __init_stage2_translation(void) · 69 unmodified lines
7070 * Management in ID_AA64MMFR1_EL1 and enable the feature in VTCR_EL2.
7171 */
7272 tmp = (read_sysreg(id_aa64mmfr1_el1) >> ID_AA64MMFR1_HADBS_SHIFT) & 0xf;
73− if (IS_ENABLED(CONFIG_ARM64_HW_AFDBM) && tmp)
73+ if (tmp)
7474 val |= VTCR_EL2_HA;
7575
7676 /*
arch/arm64/lib/Makefile+2−0
@@ -17,3 +17,5 @@ CFLAGS_atomic_ll_sc.o := -fcall-used-x0 -ffixed-x1 -ffixed-x2 \ · 16 unmodified lines
1717 -fcall-saved-x10 -fcall-saved-x11 -fcall-saved-x12 \
1818 -fcall-saved-x13 -fcall-saved-x14 -fcall-saved-x15 \
1919 -fcall-saved-x18
20+
21+lib-$(CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE) += uaccess_flushcache.o
arch/arm64/lib/uaccess_flushcache.c+47−0
@@ -0,0 +1,47 @@
1+/*
2+ * Copyright (C) 2017 ARM Ltd.
3+ *
4+ * This program is free software; you can redistribute it and/or modify
5+ * it under the terms of the GNU General Public License version 2 as
6+ * published by the Free Software Foundation.
7+ *
8+ * This program is distributed in the hope that it will be useful,
9+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
10+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11+ * GNU General Public License for more details.
12+ *
13+ * You should have received a copy of the GNU General Public License
14+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
15+ */
16+
17+#include <linux/uaccess.h>
18+#include <asm/barrier.h>
19+#include <asm/cacheflush.h>
20+
21+void memcpy_flushcache(void *dst, const void *src, size_t cnt)
22+{
23+ /*
24+ * We assume this should not be called with @dst pointing to
25+ * non-cacheable memory, such that we don't need an explicit
26+ * barrier to order the cache maintenance against the memcpy.
27+ */
28+ memcpy(dst, src, cnt);
29+ __clean_dcache_area_pop(dst, cnt);
30+}
31+EXPORT_SYMBOL_GPL(memcpy_flushcache);
32+
33+void memcpy_page_flushcache(char *to, struct page *page, size_t offset,
34+ size_t len)
35+{
36+ memcpy_flushcache(to, page_address(page) + offset, len);
37+}
38+
39+unsigned long __copy_user_flushcache(void *to, const void __user *from,
40+ unsigned long n)
41+{
42+ unsigned long rc = __arch_copy_from_user(to, from, n);
43+
44+ /* See above */
45+ __clean_dcache_area_pop(to, n - rc);
46+ return rc;
47+}
arch/arm64/mm/cache.S+28−9
@@ -109,20 +109,25 @@ ENTRY(__clean_dcache_area_pou) · 108 unmodified lines
109109 ENDPROC(__clean_dcache_area_pou)
110110
111111 /*
112− * __dma_inv_area(start, size)
113− * - start - virtual start address of region
112+ * __inval_dcache_area(kaddr, size)
113+ *
114+ * Ensure that any D-cache lines for the interval [kaddr, kaddr+size)
115+ * are invalidated. Any partial lines at the ends of the interval are
116+ * also cleaned to PoC to prevent data loss.
117+ *
118+ * - kaddr - kernel address
114119 * - size - size in question
115120 */
116−__dma_inv_area:
117− add x1, x1, x0
121+ENTRY(__inval_dcache_area)
118122 /* FALLTHROUGH */
119123
120124 /*
121− * __inval_cache_range(start, end)
122− * - start - start address of region
123− * - end - end address of region
125+ * __dma_inv_area(start, size)
126+ * - start - virtual start address of region
127+ * - size - size in question
124128 */
125−ENTRY(__inval_cache_range)
129+__dma_inv_area:
130+ add x1, x1, x0
126131 dcache_line_size x2, x3
127132 sub x3, x2, #1
128133 tst x1, x3 // end cache line aligned?
@@ -140,7 +145,7 @@ ENTRY(__inval_cache_range) · 11 unmodified lines
140145 b.lo 2b
141146 dsb sy
142147 ret
143−ENDPIPROC(__inval_cache_range)
148+ENDPIPROC(__inval_dcache_area)
144149 ENDPROC(__dma_inv_area)
145150
146151 /*
@@ -166,6 +171,20 @@ __dma_clean_area: · 19 unmodified lines
166171 ENDPIPROC(__clean_dcache_area_poc)
167172 ENDPROC(__dma_clean_area)
168173
174+/*
175+ * __clean_dcache_area_pop(kaddr, size)
176+ *
177+ * Ensure that any D-cache lines for the interval [kaddr, kaddr+size)
178+ * are cleaned to the PoP.
179+ *
180+ * - kaddr - kernel address
181+ * - size - size in question
182+ */
183+ENTRY(__clean_dcache_area_pop)
184+ dcache_by_line_op cvap, sy, x0, x1, x2, x3
185+ ret
186+ENDPIPROC(__clean_dcache_area_pop)
187+
169188 /*
170189 * __dma_flush_area(start, size)
171190 *
arch/arm64/mm/dma-mapping.c+2−2
@@ -42,7 +42,7 @@ static pgprot_t __get_dma_pgprot(unsigned long attrs, pgprot_t prot, · 41 unmodified lines
4242 return prot;
4343 }
4444
45−static struct gen_pool *atomic_pool;
45+static struct gen_pool *atomic_pool __ro_after_init;
4646
4747 #define DEFAULT_DMA_COHERENT_POOL_SIZE SZ_256K
4848 static size_t atomic_pool_size __initdata = DEFAULT_DMA_COHERENT_POOL_SIZE;
@@ -425,7 +425,7 @@ static int __init atomic_pool_init(void) · 376 unmodified lines
425425
426426 gen_pool_set_algo(atomic_pool,
427427 gen_pool_first_fit_order_align,
428− (void *)PAGE_SHIFT);
428+ NULL);
429429
430430 pr_info("DMA: preallocated %zu KiB pool for atomic allocations\n",
431431 atomic_pool_size / 1024);
arch/arm64/mm/fault.c+58−19
@@ -34,6 +34,7 @@ · 33 unmodified lines
3434 #include <linux/hugetlb.h>
3535
3636 #include <asm/bug.h>
37+#include <asm/cmpxchg.h>
3738 #include <asm/cpufeature.h>
3839 #include <asm/exception.h>
3940 #include <asm/debug-monitors.h>
@@ -82,6 +83,49 @@ static inline int notify_page_fault(struct pt_regs *regs, unsigned int esr) · 42 unmodified lines
8283 }
8384 #endif
8485
86+static void data_abort_decode(unsigned int esr)
87+{
88+ pr_alert("Data abort info:\n");
89+
90+ if (esr & ESR_ELx_ISV) {
91+ pr_alert(" Access size = %u byte(s)\n",
92+ 1U << ((esr & ESR_ELx_SAS) >> ESR_ELx_SAS_SHIFT));
93+ pr_alert(" SSE = %lu, SRT = %lu\n",
94+ (esr & ESR_ELx_SSE) >> ESR_ELx_SSE_SHIFT,
95+ (esr & ESR_ELx_SRT_MASK) >> ESR_ELx_SRT_SHIFT);
96+ pr_alert(" SF = %lu, AR = %lu\n",
97+ (esr & ESR_ELx_SF) >> ESR_ELx_SF_SHIFT,
98+ (esr & ESR_ELx_AR) >> ESR_ELx_AR_SHIFT);
99+ } else {
100+ pr_alert(" ISV = 0, ISS = 0x%08lu\n", esr & ESR_ELx_ISS_MASK);
101+ }
102+
103+ pr_alert(" CM = %lu, WnR = %lu\n",
104+ (esr & ESR_ELx_CM) >> ESR_ELx_CM_SHIFT,
105+ (esr & ESR_ELx_WNR) >> ESR_ELx_WNR_SHIFT);
106+}
107+
108+/*
109+ * Decode mem abort information
110+ */
111+static void mem_abort_decode(unsigned int esr)
112+{
113+ pr_alert("Mem abort info:\n");
114+
115+ pr_alert(" Exception class = %s, IL = %u bits\n",
116+ esr_get_class_string(esr),
117+ (esr & ESR_ELx_IL) ? 32 : 16);
118+ pr_alert(" SET = %lu, FnV = %lu\n",
119+ (esr & ESR_ELx_SET_MASK) >> ESR_ELx_SET_SHIFT,
120+ (esr & ESR_ELx_FnV) >> ESR_ELx_FnV_SHIFT);
121+ pr_alert(" EA = %lu, S1PTW = %lu\n",
122+ (esr & ESR_ELx_EA) >> ESR_ELx_EA_SHIFT,
123+ (esr & ESR_ELx_S1PTW) >> ESR_ELx_S1PTW_SHIFT);
124+
125+ if (esr_is_data_abort(esr))
126+ data_abort_decode(esr);
127+}
128+
85129 /*
86130 * Dump out the page tables associated with 'addr' in the currently active mm.
87131 */
@@ -139,7 +183,6 @@ void show_pte(unsigned long addr) · 51 unmodified lines
139183 pr_cont("\n");
140184 }
141185
142−#ifdef CONFIG_ARM64_HW_AFDBM
143186 /*
144187 * This function sets the access flags (dirty, accessed), as well as write
145188 * permission, and only to a more permissive setting.
@@ -154,18 +197,13 @@ int ptep_set_access_flags(struct vm_area_struct *vma, · 8 unmodified lines
154197 unsigned long address, pte_t *ptep,
155198 pte_t entry, int dirty)
156199 {
157− pteval_t old_pteval;
158− unsigned int tmp;
200+ pteval_t old_pteval, pteval;
159201
160202 if (pte_same(*ptep, entry))
161203 return 0;
162204
163205 /* only preserve the access flags and write permission */
164− pte_val(entry) &= PTE_AF | PTE_WRITE | PTE_DIRTY;
165−
166− /* set PTE_RDONLY if actual read-only or clean PTE */
167− if (!pte_write(entry) || !pte_sw_dirty(entry))
168− pte_val(entry) |= PTE_RDONLY;
206+ pte_val(entry) &= PTE_RDONLY | PTE_AF | PTE_WRITE | PTE_DIRTY;
169207
170208 /*
171209 * Setting the flags must be done atomically to avoid racing with the
@@ -174,21 +212,18 @@ int ptep_set_access_flags(struct vm_area_struct *vma, · 2 unmodified lines
174212 * (calculated as: a & b == ~(~a | ~b)).
175213 */
176214 pte_val(entry) ^= PTE_RDONLY;
177− asm volatile("// ptep_set_access_flags\n"
178− " prfm pstl1strm, %2\n"
179− "1: ldxr %0, %2\n"
180− " eor %0, %0, %3 // negate PTE_RDONLY in *ptep\n"
181− " orr %0, %0, %4 // set flags\n"
182− " eor %0, %0, %3 // negate final PTE_RDONLY\n"
183− " stxr %w1, %0, %2\n"
184− " cbnz %w1, 1b\n"
185− : "=&r" (old_pteval), "=&r" (tmp), "+Q" (pte_val(*ptep))
186− : "L" (PTE_RDONLY), "r" (pte_val(entry)));
215+ pteval = READ_ONCE(pte_val(*ptep));
216+ do {
217+ old_pteval = pteval;
218+ pteval ^= PTE_RDONLY;
219+ pteval |= pte_val(entry);
220+ pteval ^= PTE_RDONLY;
221+ pteval = cmpxchg_relaxed(&pte_val(*ptep), old_pteval, pteval);
222+ } while (pteval != old_pteval);
187223
188224 flush_tlb_fix_spurious_fault(vma, address);
189225 return 1;
190226 }
191−#endif
192227
193228 static bool is_el1_instruction_abort(unsigned int esr)
194229 {
@@ -248,6 +283,8 @@ static void __do_kernel_fault(unsigned long addr, unsigned int esr, · 53 unmodified lines
248283 pr_alert("Unable to handle kernel %s at virtual address %08lx\n", msg,
249284 addr);
250285
286+ mem_abort_decode(esr);
287+
251288 show_pte(addr);
252289 die("Oops", regs, esr);
253290 bust_spinlocks(0);
@@ -705,6 +742,8 @@ asmlinkage void __exception do_mem_abort(unsigned long addr, unsigned int esr, · 451 unmodified lines
705742 pr_alert("Unhandled fault: %s (0x%08x) at 0x%016lx\n",
706743 inf->name, esr, addr);
707744
745+ mem_abort_decode(esr);
746+
708747 info.si_signo = inf->sig;
709748 info.si_errno = 0;
710749 info.si_code = inf->code;
arch/arm64/mm/flush.c+16−0
@@ -83,3 +83,19 @@ EXPORT_SYMBOL(flush_dcache_page); · 82 unmodified lines
8383 * Additional functions defined in assembly.
8484 */
8585 EXPORT_SYMBOL(flush_icache_range);
86+
87+#ifdef CONFIG_ARCH_HAS_PMEM_API
88+void arch_wb_cache_pmem(void *addr, size_t size)
89+{
90+ /* Ensure order against any prior non-cacheable writes */
91+ dmb(osh);
92+ __clean_dcache_area_pop(addr, size);
93+}
94+EXPORT_SYMBOL_GPL(arch_wb_cache_pmem);
95+
96+void arch_invalidate_pmem(void *addr, size_t size)
97+{
98+ __inval_dcache_area(addr, size);
99+}
100+EXPORT_SYMBOL_GPL(arch_invalidate_pmem);
101+#endif
arch/arm64/mm/hugetlbpage.c+230−84
@@ -41,6 +41,16 @@ int pud_huge(pud_t pud) · 40 unmodified lines
4141 #endif
4242 }
4343
44+/*
45+ * Select all bits except the pfn
46+ */
47+static inline pgprot_t pte_pgprot(pte_t pte)
48+{
49+ unsigned long pfn = pte_pfn(pte);
50+
51+ return __pgprot(pte_val(pfn_pte(pfn, __pgprot(0))) ^ pte_val(pte));
52+}
53+
4454 static int find_num_contig(struct mm_struct *mm, unsigned long addr,
4555 pte_t *ptep, size_t *pgsize)
4656 {
@@ -58,15 +68,107 @@ static int find_num_contig(struct mm_struct *mm, unsigned long addr, · 11 unmodified lines
5868 return CONT_PTES;
5969 }
6070
71+static inline int num_contig_ptes(unsigned long size, size_t *pgsize)
72+{
73+ int contig_ptes = 0;
74+
75+ *pgsize = size;
76+
77+ switch (size) {
78+#ifdef CONFIG_ARM64_4K_PAGES
79+ case PUD_SIZE:
80+#endif
81+ case PMD_SIZE:
82+ contig_ptes = 1;
83+ break;
84+ case CONT_PMD_SIZE:
85+ *pgsize = PMD_SIZE;
86+ contig_ptes = CONT_PMDS;
87+ break;
88+ case CONT_PTE_SIZE:
89+ *pgsize = PAGE_SIZE;
90+ contig_ptes = CONT_PTES;
91+ break;
92+ }
93+
94+ return contig_ptes;
95+}
96+
97+/*
98+ * Changing some bits of contiguous entries requires us to follow a
99+ * Break-Before-Make approach, breaking the whole contiguous set
100+ * before we can change any entries. See ARM DDI 0487A.k_iss10775,
101+ * "Misprogramming of the Contiguous bit", page D4-1762.
102+ *
103+ * This helper performs the break step.
104+ */
105+static pte_t get_clear_flush(struct mm_struct *mm,
106+ unsigned long addr,
107+ pte_t *ptep,
108+ unsigned long pgsize,
109+ unsigned long ncontig)
110+{
111+ struct vm_area_struct vma = { .vm_mm = mm };
112+ pte_t orig_pte = huge_ptep_get(ptep);
113+ bool valid = pte_valid(orig_pte);
114+ unsigned long i, saddr = addr;
115+
116+ for (i = 0; i < ncontig; i++, addr += pgsize, ptep++) {
117+ pte_t pte = ptep_get_and_clear(mm, addr, ptep);
118+
119+ /*
120+ * If HW_AFDBM is enabled, then the HW could turn on
121+ * the dirty bit for any page in the set, so check
122+ * them all. All hugetlb entries are already young.
123+ */
124+ if (pte_dirty(pte))
125+ orig_pte = pte_mkdirty(orig_pte);
126+ }
127+
128+ if (valid)
129+ flush_tlb_range(&vma, saddr, addr);
130+ return orig_pte;
131+}
132+
133+/*
134+ * Changing some bits of contiguous entries requires us to follow a
135+ * Break-Before-Make approach, breaking the whole contiguous set
136+ * before we can change any entries. See ARM DDI 0487A.k_iss10775,
137+ * "Misprogramming of the Contiguous bit", page D4-1762.
138+ *
139+ * This helper performs the break step for use cases where the
140+ * original pte is not needed.
141+ */
142+static void clear_flush(struct mm_struct *mm,
143+ unsigned long addr,
144+ pte_t *ptep,
145+ unsigned long pgsize,
146+ unsigned long ncontig)
147+{
148+ struct vm_area_struct vma = { .vm_mm = mm };
149+ unsigned long i, saddr = addr;
150+
151+ for (i = 0; i < ncontig; i++, addr += pgsize, ptep++)
152+ pte_clear(mm, addr, ptep);
153+
154+ flush_tlb_range(&vma, saddr, addr);
155+}
156+
61157 void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
62158 pte_t *ptep, pte_t pte)
63159 {
64160 size_t pgsize;
65161 int i;
66162 int ncontig;
67− unsigned long pfn;
163+ unsigned long pfn, dpfn;
68164 pgprot_t hugeprot;
69165
166+ /*
167+ * Code needs to be expanded to handle huge swap and migration
168+ * entries. Needed for HUGETLB and MEMORY_FAILURE.
169+ */
170+ WARN_ON(!pte_present(pte));
171+
70172 if (!pte_cont(pte)) {
71173 set_pte_at(mm, addr, ptep, pte);
72174 return;
@@ -74,17 +176,30 @@ void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, · 1 unmodified line
74176
75177 ncontig = find_num_contig(mm, addr, ptep, &pgsize);
76178 pfn = pte_pfn(pte);
77− hugeprot = __pgprot(pte_val(pfn_pte(pfn, __pgprot(0))) ^ pte_val(pte));
78− for (i = 0; i < ncontig; i++) {
179+ dpfn = pgsize >> PAGE_SHIFT;
180+ hugeprot = pte_pgprot(pte);
181+
182+ clear_flush(mm, addr, ptep, pgsize, ncontig);
183+
184+ for (i = 0; i < ncontig; i++, ptep++, addr += pgsize, pfn += dpfn) {
79185 pr_debug("%s: set pte %p to 0x%llx\n", __func__, ptep,
80186 pte_val(pfn_pte(pfn, hugeprot)));
81187 set_pte_at(mm, addr, ptep, pfn_pte(pfn, hugeprot));
82− ptep++;
83− pfn += pgsize >> PAGE_SHIFT;
84− addr += pgsize;
85188 }
86189 }
87190
191+void set_huge_swap_pte_at(struct mm_struct *mm, unsigned long addr,
192+ pte_t *ptep, pte_t pte, unsigned long sz)
193+{
194+ int i, ncontig;
195+ size_t pgsize;
196+
197+ ncontig = num_contig_ptes(sz, &pgsize);
198+
199+ for (i = 0; i < ncontig; i++, ptep++)
200+ set_pte(ptep, pte);
201+}
202+
88203 pte_t *huge_pte_alloc(struct mm_struct *mm,
89204 unsigned long addr, unsigned long sz)
90205 {
@@ -144,19 +259,28 @@ pte_t *huge_pte_offset(struct mm_struct *mm, · 53 unmodified lines
144259 return NULL;
145260
146261 pud = pud_offset(pgd, addr);
147− if (pud_none(*pud))
262+ if (sz != PUD_SIZE && pud_none(*pud))
148263 return NULL;
149− /* swap or huge page */
150− if (!pud_present(*pud) || pud_huge(*pud))
264+ /* hugepage or swap? */
265+ if (pud_huge(*pud) || !pud_present(*pud))
151266 return (pte_t *)pud;
152267 /* table; check the next level */
153268
269+ if (sz == CONT_PMD_SIZE)
270+ addr &= CONT_PMD_MASK;
271+
154272 pmd = pmd_offset(pud, addr);
155− if (pmd_none(*pmd))
273+ if (!(sz == PMD_SIZE || sz == CONT_PMD_SIZE) &&
274+ pmd_none(*pmd))
156275 return NULL;
157− if (!pmd_present(*pmd) || pmd_huge(*pmd))
276+ if (pmd_huge(*pmd) || !pmd_present(*pmd))
158277 return (pte_t *)pmd;
159278
279+ if (sz == CONT_PTE_SIZE) {
280+ pte_t *pte = pte_offset_kernel(pmd, (addr & CONT_PTE_MASK));
281+ return pte;
282+ }
283+
160284 return NULL;
161285 }
162286
@@ -176,111 +300,133 @@ pte_t arch_make_huge_pte(pte_t entry, struct vm_area_struct *vma, · 13 unmodified lines
176300 return entry;
177301 }
178302
303+void huge_pte_clear(struct mm_struct *mm, unsigned long addr,
304+ pte_t *ptep, unsigned long sz)
305+{
306+ int i, ncontig;
307+ size_t pgsize;
308+
309+ ncontig = num_contig_ptes(sz, &pgsize);
310+
311+ for (i = 0; i < ncontig; i++, addr += pgsize, ptep++)
312+ pte_clear(mm, addr, ptep);
313+}
314+
179315 pte_t huge_ptep_get_and_clear(struct mm_struct *mm,
180316 unsigned long addr, pte_t *ptep)
181317 {
182− pte_t pte;
318+ int ncontig;
319+ size_t pgsize;
320+ pte_t orig_pte = huge_ptep_get(ptep);
183321
184− if (pte_cont(*ptep)) {
185− int ncontig, i;
186− size_t pgsize;
187− bool is_dirty = false;
188−
189− ncontig = find_num_contig(mm, addr, ptep, &pgsize);
190− /* save the 1st pte to return */
191− pte = ptep_get_and_clear(mm, addr, ptep);
192− for (i = 1, addr += pgsize; i < ncontig; ++i, addr += pgsize) {
193− /*
194− * If HW_AFDBM is enabled, then the HW could
195− * turn on the dirty bit for any of the page
196− * in the set, so check them all.
197− */
198− ++ptep;
199− if (pte_dirty(ptep_get_and_clear(mm, addr, ptep)))
200− is_dirty = true;
201− }
202− if (is_dirty)
203− return pte_mkdirty(pte);
204− else
205− return pte;
206− } else {
322+ if (!pte_cont(orig_pte))
207323 return ptep_get_and_clear(mm, addr, ptep);
208− }
324+
325+ ncontig = find_num_contig(mm, addr, ptep, &pgsize);
326+
327+ return get_clear_flush(mm, addr, ptep, pgsize, ncontig);
209328 }
210329
211330 int huge_ptep_set_access_flags(struct vm_area_struct *vma,
212331 unsigned long addr, pte_t *ptep,
213332 pte_t pte, int dirty)
214333 {
215− if (pte_cont(pte)) {
216− int ncontig, i, changed = 0;
217− size_t pgsize = 0;
218− unsigned long pfn = pte_pfn(pte);
219− /* Select all bits except the pfn */
220− pgprot_t hugeprot =
221− __pgprot(pte_val(pfn_pte(pfn, __pgprot(0))) ^
222− pte_val(pte));
223−
224− pfn = pte_pfn(pte);
225− ncontig = find_num_contig(vma->vm_mm, addr, ptep,
226− &pgsize);
227− for (i = 0; i < ncontig; ++i, ++ptep, addr += pgsize) {
228− changed |= ptep_set_access_flags(vma, addr, ptep,
229− pfn_pte(pfn,
230− hugeprot),
231− dirty);
232− pfn += pgsize >> PAGE_SHIFT;
233− }
234− return changed;
235− } else {
334+ int ncontig, i, changed = 0;
335+ size_t pgsize = 0;
336+ unsigned long pfn = pte_pfn(pte), dpfn;
337+ pgprot_t hugeprot;
338+ pte_t orig_pte;
339+
340+ if (!pte_cont(pte))
236341 return ptep_set_access_flags(vma, addr, ptep, pte, dirty);
237− }
342+
343+ ncontig = find_num_contig(vma->vm_mm, addr, ptep, &pgsize);
344+ dpfn = pgsize >> PAGE_SHIFT;
345+
346+ orig_pte = get_clear_flush(vma->vm_mm, addr, ptep, pgsize, ncontig);
347+ if (!pte_same(orig_pte, pte))
348+ changed = 1;
349+
350+ /* Make sure we don't lose the dirty state */
351+ if (pte_dirty(orig_pte))
352+ pte = pte_mkdirty(pte);
353+
354+ hugeprot = pte_pgprot(pte);
355+ for (i = 0; i < ncontig; i++, ptep++, addr += pgsize, pfn += dpfn)
356+ set_pte_at(vma->vm_mm, addr, ptep, pfn_pte(pfn, hugeprot));
357+
358+ return changed;
238359 }
239360
240361 void huge_ptep_set_wrprotect(struct mm_struct *mm,
241362 unsigned long addr, pte_t *ptep)
242363 {
243− if (pte_cont(*ptep)) {
244− int ncontig, i;
245− size_t pgsize = 0;
246−
247− ncontig = find_num_contig(mm, addr, ptep, &pgsize);
248− for (i = 0; i < ncontig; ++i, ++ptep, addr += pgsize)
249− ptep_set_wrprotect(mm, addr, ptep);
250− } else {
364+ unsigned long pfn, dpfn;
365+ pgprot_t hugeprot;
366+ int ncontig, i;
367+ size_t pgsize;
368+ pte_t pte;
369+
370+ if (!pte_cont(*ptep)) {
251371 ptep_set_wrprotect(mm, addr, ptep);
372+ return;
252373 }
374+
375+ ncontig = find_num_contig(mm, addr, ptep, &pgsize);
376+ dpfn = pgsize >> PAGE_SHIFT;
377+
378+ pte = get_clear_flush(mm, addr, ptep, pgsize, ncontig);
379+ pte = pte_wrprotect(pte);
380+
381+ hugeprot = pte_pgprot(pte);
382+ pfn = pte_pfn(pte);
383+
384+ for (i = 0; i < ncontig; i++, ptep++, addr += pgsize, pfn += dpfn)
385+ set_pte_at(mm, addr, ptep, pfn_pte(pfn, hugeprot));
253386 }
254387
255388 void huge_ptep_clear_flush(struct vm_area_struct *vma,
256389 unsigned long addr, pte_t *ptep)
257390 {
258− if (pte_cont(*ptep)) {
259− int ncontig, i;
260− size_t pgsize = 0;
261−
262− ncontig = find_num_contig(vma->vm_mm, addr, ptep,
263− &pgsize);
264− for (i = 0; i < ncontig; ++i, ++ptep, addr += pgsize)
265− ptep_clear_flush(vma, addr, ptep);
266− } else {
391+ size_t pgsize;
392+ int ncontig;
393+
394+ if (!pte_cont(*ptep)) {
267395 ptep_clear_flush(vma, addr, ptep);
396+ return;
268397 }
398+
399+ ncontig = find_num_contig(vma->vm_mm, addr, ptep, &pgsize);
400+ clear_flush(vma->vm_mm, addr, ptep, pgsize, ncontig);
269401 }
270402
271403 static __init int setup_hugepagesz(char *opt)
272404 {
273405 unsigned long ps = memparse(opt, &opt);
274406
275− if (ps == PMD_SIZE) {
276− hugetlb_add_hstate(PMD_SHIFT - PAGE_SHIFT);
277− } else if (ps == PUD_SIZE) {
278− hugetlb_add_hstate(PUD_SHIFT - PAGE_SHIFT);
279− } else {
280− hugetlb_bad_size();
281− pr_err("hugepagesz: Unsupported page size %lu K\n", ps >> 10);
282− return 0;
407+ switch (ps) {
408+#ifdef CONFIG_ARM64_4K_PAGES
409+ case PUD_SIZE:
410+#endif
411+ case PMD_SIZE * CONT_PMDS:
412+ case PMD_SIZE:
413+ case PAGE_SIZE * CONT_PTES:
414+ hugetlb_add_hstate(ilog2(ps) - PAGE_SHIFT);
415+ return 1;
283416 }
284− return 1;
417+
418+ hugetlb_bad_size();
419+ pr_err("hugepagesz: Unsupported page size %lu K\n", ps >> 10);
420+ return 0;
285421 }
286422 __setup("hugepagesz=", setup_hugepagesz);
423+
424+#ifdef CONFIG_ARM64_64K_PAGES
425+static __init int add_default_hugepagesz(void)
426+{
427+ if (size_to_hstate(CONT_PTES * PAGE_SIZE) == NULL)
428+ hugetlb_add_hstate(CONT_PTE_SHIFT);
429+ return 0;
430+}
431+arch_initcall(add_default_hugepagesz);
432+#endif
drivers/acpi/arm64/iort.c+87−53
@@ -588,7 +588,8 @@ void acpi_configure_pmsi_domain(struct device *dev) · 587 unmodified lines
588588 dev_set_msi_domain(dev, msi_domain);
589589 }
590590
591−static int __get_pci_rid(struct pci_dev *pdev, u16 alias, void *data)
591+static int __maybe_unused __get_pci_rid(struct pci_dev *pdev, u16 alias,
592+ void *data)
592593 {
593594 u32 *rid = data;
594595
@@ -633,8 +634,7 @@ int iort_add_device_replay(const struct iommu_ops *ops, struct device *dev) · 38 unmodified lines
633634 {
634635 int err = 0;
635636
636− if (!IS_ERR_OR_NULL(ops) && ops->add_device && dev->bus &&
637− !dev->iommu_group)
637+ if (ops->add_device && dev->bus && !dev->iommu_group)
638638 err = ops->add_device(dev);
639639
640640 return err;
@@ -648,36 +648,49 @@ int iort_add_device_replay(const struct iommu_ops *ops, struct device *dev) · 7 unmodified lines
648648 { return 0; }
649649 #endif
650650
651−static const struct iommu_ops *iort_iommu_xlate(struct device *dev,
652− struct acpi_iort_node *node,
653− u32 streamid)
651+static int iort_iommu_xlate(struct device *dev, struct acpi_iort_node *node,
652+ u32 streamid)
654653 {
655− const struct iommu_ops *ops = NULL;
656− int ret = -ENODEV;
654+ const struct iommu_ops *ops;
657655 struct fwnode_handle *iort_fwnode;
658656
659− if (node) {
660− iort_fwnode = iort_get_fwnode(node);
661− if (!iort_fwnode)
662− return NULL;
657+ if (!node)
658+ return -ENODEV;
663659
664− ops = iommu_ops_from_fwnode(iort_fwnode);
665− /*
666− * If the ops look-up fails, this means that either
667− * the SMMU drivers have not been probed yet or that
668− * the SMMU drivers are not built in the kernel;
669− * Depending on whether the SMMU drivers are built-in
670− * in the kernel or not, defer the IOMMU configuration
671− * or just abort it.
672− */
673− if (!ops)
674− return iort_iommu_driver_enabled(node->type) ?
675− ERR_PTR(-EPROBE_DEFER) : NULL;
660+ iort_fwnode = iort_get_fwnode(node);
661+ if (!iort_fwnode)
662+ return -ENODEV;
676663
677− ret = arm_smmu_iort_xlate(dev, streamid, iort_fwnode, ops);
678− }
664+ /*
665+ * If the ops look-up fails, this means that either
666+ * the SMMU drivers have not been probed yet or that
667+ * the SMMU drivers are not built in the kernel;
668+ * Depending on whether the SMMU drivers are built-in
669+ * in the kernel or not, defer the IOMMU configuration
670+ * or just abort it.
671+ */
672+ ops = iommu_ops_from_fwnode(iort_fwnode);
673+ if (!ops)
674+ return iort_iommu_driver_enabled(node->type) ?
675+ -EPROBE_DEFER : -ENODEV;
679676
680− return ret ? NULL : ops;
677+ return arm_smmu_iort_xlate(dev, streamid, iort_fwnode, ops);
678+}
679+
680+struct iort_pci_alias_info {
681+ struct device *dev;
682+ struct acpi_iort_node *node;
683+};
684+
685+static int iort_pci_iommu_init(struct pci_dev *pdev, u16 alias, void *data)
686+{
687+ struct iort_pci_alias_info *info = data;
688+ struct acpi_iort_node *parent;
689+ u32 streamid;
690+
691+ parent = iort_node_map_id(info->node, alias, &streamid,
692+ IORT_IOMMU_TYPE);
693+ return iort_iommu_xlate(info->dev, parent, streamid);
681694 }
682695
683696 /**
@@ -713,9 +726,9 @@ void iort_set_dma_mask(struct device *dev) · 29 unmodified lines
713726 const struct iommu_ops *iort_iommu_configure(struct device *dev)
714727 {
715728 struct acpi_iort_node *node, *parent;
716− const struct iommu_ops *ops = NULL;
729+ const struct iommu_ops *ops;
717730 u32 streamid = 0;
718− int err;
731+ int err = -ENODEV;
719732
720733 /*
721734 * If we already translated the fwspec there
@@ -727,21 +740,16 @@ const struct iommu_ops *iort_iommu_configure(struct device *dev) · 5 unmodified lines
727740
728741 if (dev_is_pci(dev)) {
729742 struct pci_bus *bus = to_pci_dev(dev)->bus;
730− u32 rid;
731−
732− pci_for_each_dma_alias(to_pci_dev(dev), __get_pci_rid,
733− &rid);
743+ struct iort_pci_alias_info info = { .dev = dev };
734744
735745 node = iort_scan_node(ACPI_IORT_NODE_PCI_ROOT_COMPLEX,
736746 iort_match_node_callback, &bus->dev);
737747 if (!node)
738748 return NULL;
739749
740− parent = iort_node_map_id(node, rid, &streamid,
741− IORT_IOMMU_TYPE);
742−
743− ops = iort_iommu_xlate(dev, parent, streamid);
744−
750+ info.node = node;
751+ err = pci_for_each_dma_alias(to_pci_dev(dev),
752+ iort_pci_iommu_init, &info);
745753 } else {
746754 int i = 0;
747755
@@ -750,31 +758,30 @@ const struct iommu_ops *iort_iommu_configure(struct device *dev) · 2 unmodified lines
750758 if (!node)
751759 return NULL;
752760
753− parent = iort_node_map_platform_id(node, &streamid,
754− IORT_IOMMU_TYPE, i++);
755−
756− while (parent) {
757− ops = iort_iommu_xlate(dev, parent, streamid);
758− if (IS_ERR_OR_NULL(ops))
759− return ops;
760−
761+ do {
761762 parent = iort_node_map_platform_id(node, &streamid,
762763 IORT_IOMMU_TYPE,
763764 i++);
764− }
765+
766+ if (parent)
767+ err = iort_iommu_xlate(dev, parent, streamid);
768+ } while (parent && !err);
765769 }
766770
767771 /*
768772 * If we have reason to believe the IOMMU driver missed the initial
769773 * add_device callback for dev, replay it to get things in order.
770774 */
771− err = iort_add_device_replay(ops, dev);
772− if (err)
773− ops = ERR_PTR(err);
775+ if (!err) {
776+ ops = iort_fwspec_iommu_ops(dev->iommu_fwspec);
777+ err = iort_add_device_replay(ops, dev);
778+ }
774779
775780 /* Ignore all other errors apart from EPROBE_DEFER */
776− if (IS_ERR(ops) && (PTR_ERR(ops) != -EPROBE_DEFER)) {
777− dev_dbg(dev, "Adding to IOMMU failed: %ld\n", PTR_ERR(ops));
781+ if (err == -EPROBE_DEFER) {
782+ ops = ERR_PTR(err);
783+ } else if (err) {
784+ dev_dbg(dev, "Adding to IOMMU failed: %d\n", err);
778785 ops = NULL;
779786 }
780787
@@ -908,6 +915,27 @@ static bool __init arm_smmu_v3_is_coherent(struct acpi_iort_node *node) · 127 unmodified lines
908915 return smmu->flags & ACPI_IORT_SMMU_V3_COHACC_OVERRIDE;
909916 }
910917
918+#if defined(CONFIG_ACPI_NUMA) && defined(ACPI_IORT_SMMU_V3_PXM_VALID)
919+/*
920+ * set numa proximity domain for smmuv3 device
921+ */
922+static void __init arm_smmu_v3_set_proximity(struct device *dev,
923+ struct acpi_iort_node *node)
924+{
925+ struct acpi_iort_smmu_v3 *smmu;
926+
927+ smmu = (struct acpi_iort_smmu_v3 *)node->node_data;
928+ if (smmu->flags & ACPI_IORT_SMMU_V3_PXM_VALID) {
929+ set_dev_node(dev, acpi_map_pxm_to_node(smmu->pxm));
930+ pr_info("SMMU-v3[%llx] Mapped to Proximity domain %d\n",
931+ smmu->base_address,
932+ smmu->pxm);
933+ }
934+}
935+#else
936+#define arm_smmu_v3_set_proximity NULL
937+#endif
938+
911939 static int __init arm_smmu_count_resources(struct acpi_iort_node *node)
912940 {
913941 struct acpi_iort_smmu *smmu;
@@ -977,13 +1005,16 @@ struct iort_iommu_config { · 63 unmodified lines
9771005 int (*iommu_count_resources)(struct acpi_iort_node *node);
9781006 void (*iommu_init_resources)(struct resource *res,
9791007 struct acpi_iort_node *node);
1008+ void (*iommu_set_proximity)(struct device *dev,
1009+ struct acpi_iort_node *node);
9801010 };
9811011
9821012 static const struct iort_iommu_config iort_arm_smmu_v3_cfg __initconst = {
9831013 .name = "arm-smmu-v3",
9841014 .iommu_is_coherent = arm_smmu_v3_is_coherent,
9851015 .iommu_count_resources = arm_smmu_v3_count_resources,
986− .iommu_init_resources = arm_smmu_v3_init_resources
1016+ .iommu_init_resources = arm_smmu_v3_init_resources,
1017+ .iommu_set_proximity = arm_smmu_v3_set_proximity,
9871018 };
9881019
9891020 static const struct iort_iommu_config iort_arm_smmu_cfg __initconst = {
@@ -1028,6 +1059,9 @@ static int __init iort_add_smmu_platform_device(struct acpi_iort_node *node) · 38 unmodified lines
10281059 if (!pdev)
10291060 return -ENOMEM;
10301061
1062+ if (ops->iommu_set_proximity)
1063+ ops->iommu_set_proximity(&pdev->dev, node);
1064+
10311065 count = ops->iommu_count_resources(node);
10321066
10331067 r = kcalloc(count, sizeof(*r), GFP_KERNEL);
drivers/firmware/efi/libstub/arm64-stub.c+4−2
@@ -11,6 +11,7 @@ · 10 unmodified lines
1111 */
1212 #include <linux/efi.h>
1313 #include <asm/efi.h>
14+#include <asm/memory.h>
1415 #include <asm/sections.h>
1516 #include <asm/sysreg.h>
1617
@@ -81,9 +82,10 @@ efi_status_t handle_kernel_image(efi_system_table_t *sys_table_arg, · 64 unmodified lines
8182 /*
8283 * If CONFIG_DEBUG_ALIGN_RODATA is not set, produce a
8384 * displacement in the interval [0, MIN_KIMG_ALIGN) that
84− * is a multiple of the minimal segment alignment (SZ_64K)
85+ * doesn't violate this kernel's de-facto alignment
86+ * constraints.
8587 */
86− u32 mask = (MIN_KIMG_ALIGN - 1) & ~(SZ_64K - 1);
88+ u32 mask = (MIN_KIMG_ALIGN - 1) & ~(EFI_KIMG_ALIGN - 1);
8789 u32 offset = !IS_ENABLED(CONFIG_DEBUG_ALIGN_RODATA) ?
8890 (phys_seed >> 32) & mask : TEXT_OFFSET;
8991
drivers/perf/arm_pmu.c+6−0
@@ -47,6 +47,9 @@ armpmu_map_cache_event(const unsigned (*cache_map) · 46 unmodified lines
4747 if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
4848 return -EINVAL;
4949
50+ if (!cache_map)
51+ return -ENOENT;
52+
5053 ret = (int)(*cache_map)[cache_type][cache_op][cache_result];
5154
5255 if (ret == CACHE_OP_UNSUPPORTED)
@@ -63,6 +66,9 @@ armpmu_map_hw_event(const unsigned (*event_map)[PERF_COUNT_HW_MAX], u64 config) · 10 unmodified lines
6366 if (config >= PERF_COUNT_HW_MAX)
6467 return -EINVAL;
6568
69+ if (!event_map)
70+ return -ENOENT;
71+
6672 mapping = (*event_map)[config];
6773 return mapping == HW_OP_UNSUPPORTED ? -ENOENT : mapping;
6874 }
drivers/perf/xgene_pmu.c+22−52
@@ -1147,7 +1147,6 @@ xgene_pmu_dev_add(struct xgene_pmu *xgene_pmu, struct xgene_pmu_dev_ctx *ctx) · 1146 unmodified lines
11471147 {
11481148 struct device *dev = xgene_pmu->dev;
11491149 struct xgene_pmu_dev *pmu;
1150− int rc;
11511150
11521151 pmu = devm_kzalloc(dev, sizeof(*pmu), GFP_KERNEL);
11531152 if (!pmu)
@@ -1159,7 +1158,7 @@ xgene_pmu_dev_add(struct xgene_pmu *xgene_pmu, struct xgene_pmu_dev_ctx *ctx) · 5 unmodified lines
11591158 switch (pmu->inf->type) {
11601159 case PMU_TYPE_L3C:
11611160 if (!(xgene_pmu->l3c_active_mask & pmu->inf->enable_mask))
1162− goto dev_err;
1161+ return -ENODEV;
11631162 if (xgene_pmu->version == PCP_PMU_V3)
11641163 pmu->attr_groups = l3c_pmu_v3_attr_groups;
11651164 else
@@ -1177,7 +1176,7 @@ xgene_pmu_dev_add(struct xgene_pmu *xgene_pmu, struct xgene_pmu_dev_ctx *ctx) · 11 unmodified lines
11771176 break;
11781177 case PMU_TYPE_MCB:
11791178 if (!(xgene_pmu->mcb_active_mask & pmu->inf->enable_mask))
1180− goto dev_err;
1179+ return -ENODEV;
11811180 if (xgene_pmu->version == PCP_PMU_V3)
11821181 pmu->attr_groups = mcb_pmu_v3_attr_groups;
11831182 else
@@ -1185,7 +1184,7 @@ xgene_pmu_dev_add(struct xgene_pmu *xgene_pmu, struct xgene_pmu_dev_ctx *ctx) · 1 unmodified line
11851184 break;
11861185 case PMU_TYPE_MC:
11871186 if (!(xgene_pmu->mc_active_mask & pmu->inf->enable_mask))
1188− goto dev_err;
1187+ return -ENODEV;
11891188 if (xgene_pmu->version == PCP_PMU_V3)
11901189 pmu->attr_groups = mc_pmu_v3_attr_groups;
11911190 else
@@ -1195,19 +1194,14 @@ xgene_pmu_dev_add(struct xgene_pmu *xgene_pmu, struct xgene_pmu_dev_ctx *ctx) · 3 unmodified lines
11951194 return -EINVAL;
11961195 }
11971196
1198− rc = xgene_init_perf(pmu, ctx->name);
1199− if (rc) {
1197+ if (xgene_init_perf(pmu, ctx->name)) {
12001198 dev_err(dev, "%s PMU: Failed to init perf driver\n", ctx->name);
1201− goto dev_err;
1199+ return -ENODEV;
12021200 }
12031201
12041202 dev_info(dev, "%s PMU registered\n", ctx->name);
12051203
1206− return rc;
1207−
1208−dev_err:
1209− devm_kfree(dev, pmu);
1210− return -ENODEV;
1204+ return 0;
12111205 }
12121206
12131207 static void _xgene_pmu_isr(int irq, struct xgene_pmu_dev *pmu_dev)
@@ -1515,13 +1509,13 @@ xgene_pmu_dev_ctx *acpi_get_pmu_hw_inf(struct xgene_pmu *xgene_pmu, · 301 unmodified lines
15151509 acpi_dev_free_resource_list(&resource_list);
15161510 if (rc < 0) {
15171511 dev_err(dev, "PMU type %d: No resource address found\n", type);
1518− goto err;
1512+ return NULL;
15191513 }
15201514
15211515 dev_csr = devm_ioremap_resource(dev, &res);
15221516 if (IS_ERR(dev_csr)) {
15231517 dev_err(dev, "PMU type %d: Fail to map resource\n", type);
1524− goto err;
1518+ return NULL;
15251519 }
15261520
15271521 /* A PMU device node without enable-bit-index is always enabled */
@@ -1535,7 +1529,7 @@ xgene_pmu_dev_ctx *acpi_get_pmu_hw_inf(struct xgene_pmu *xgene_pmu, · 7 unmodified lines
15351529 ctx->name = xgene_pmu_dev_name(dev, type, enable_bit);
15361530 if (!ctx->name) {
15371531 dev_err(dev, "PMU type %d: Fail to get device name\n", type);
1538− goto err;
1532+ return NULL;
15391533 }
15401534 inf = &ctx->inf;
15411535 inf->type = type;
@@ -1543,9 +1537,6 @@ xgene_pmu_dev_ctx *acpi_get_pmu_hw_inf(struct xgene_pmu *xgene_pmu, · 1 unmodified line
15431537 inf->enable_mask = 1 << enable_bit;
15441538
15451539 return ctx;
1546−err:
1547− devm_kfree(dev, ctx);
1548− return NULL;
15491540 }
15501541
15511542 static const struct acpi_device_id xgene_pmu_acpi_type_match[] = {
@@ -1663,20 +1654,20 @@ xgene_pmu_dev_ctx *fdt_get_pmu_hw_inf(struct xgene_pmu *xgene_pmu, · 111 unmodified lines
16631654 void __iomem *dev_csr;
16641655 struct resource res;
16651656 int enable_bit;
1666− int rc;
16671657
16681658 ctx = devm_kzalloc(dev, sizeof(*ctx), GFP_KERNEL);
16691659 if (!ctx)
16701660 return NULL;
1671− rc = of_address_to_resource(np, 0, &res);
1672− if (rc < 0) {
1661+
1662+ if (of_address_to_resource(np, 0, &res) < 0) {
16731663 dev_err(dev, "PMU type %d: No resource address found\n", type);
1674− goto err;
1664+ return NULL;
16751665 }
1666+
16761667 dev_csr = devm_ioremap_resource(dev, &res);
16771668 if (IS_ERR(dev_csr)) {
16781669 dev_err(dev, "PMU type %d: Fail to map resource\n", type);
1679− goto err;
1670+ return NULL;
16801671 }
16811672
16821673 /* A PMU device node without enable-bit-index is always enabled */
@@ -1686,17 +1677,15 @@ xgene_pmu_dev_ctx *fdt_get_pmu_hw_inf(struct xgene_pmu *xgene_pmu, · 3 unmodified lines
16861677 ctx->name = xgene_pmu_dev_name(dev, type, enable_bit);
16871678 if (!ctx->name) {
16881679 dev_err(dev, "PMU type %d: Fail to get device name\n", type);
1689− goto err;
1680+ return NULL;
16901681 }
1682+
16911683 inf = &ctx->inf;
16921684 inf->type = type;
16931685 inf->csr = dev_csr;
16941686 inf->enable_mask = 1 << enable_bit;
16951687
16961688 return ctx;
1697−err:
1698− devm_kfree(dev, ctx);
1699− return NULL;
17001689 }
17011690
17021691 static int fdt_pmu_probe_pmu_dev(struct xgene_pmu *xgene_pmu,
@@ -1868,22 +1857,20 @@ static int xgene_pmu_probe(struct platform_device *pdev) · 165 unmodified lines
18681857 xgene_pmu->pcppmu_csr = devm_ioremap_resource(&pdev->dev, res);
18691858 if (IS_ERR(xgene_pmu->pcppmu_csr)) {
18701859 dev_err(&pdev->dev, "ioremap failed for PCP PMU resource\n");
1871− rc = PTR_ERR(xgene_pmu->pcppmu_csr);
1872− goto err;
1860+ return PTR_ERR(xgene_pmu->pcppmu_csr);
18731861 }
18741862
18751863 irq = platform_get_irq(pdev, 0);
18761864 if (irq < 0) {
18771865 dev_err(&pdev->dev, "No IRQ resource\n");
1878− rc = -EINVAL;
1879− goto err;
1866+ return -EINVAL;
18801867 }
18811868 rc = devm_request_irq(&pdev->dev, irq, xgene_pmu_isr,
18821869 IRQF_NOBALANCING | IRQF_NO_THREAD,
18831870 dev_name(&pdev->dev), xgene_pmu);
18841871 if (rc) {
18851872 dev_err(&pdev->dev, "Could not request IRQ %d\n", irq);
1886− goto err;
1873+ return rc;
18871874 }
18881875
18891876 raw_spin_lock_init(&xgene_pmu->lock);
@@ -1903,42 +1890,29 @@ static int xgene_pmu_probe(struct platform_device *pdev) · 13 unmodified lines
19031890 rc = irq_set_affinity(irq, &xgene_pmu->cpu);
19041891 if (rc) {
19051892 dev_err(&pdev->dev, "Failed to set interrupt affinity!\n");
1906− goto err;
1893+ return rc;
19071894 }
19081895
19091896 /* Walk through the tree for all PMU perf devices */
19101897 rc = xgene_pmu_probe_pmu_dev(xgene_pmu, pdev);
19111898 if (rc) {
19121899 dev_err(&pdev->dev, "No PMU perf devices found!\n");
1913− goto err;
1900+ return rc;
19141901 }
19151902
19161903 /* Enable interrupt */
19171904 xgene_pmu->ops->unmask_int(xgene_pmu);
19181905
19191906 return 0;
1920−
1921−err:
1922− if (xgene_pmu->pcppmu_csr)
1923− devm_iounmap(&pdev->dev, xgene_pmu->pcppmu_csr);
1924− devm_kfree(&pdev->dev, xgene_pmu);
1925−
1926− return rc;
19271907 }
19281908
19291909 static void
19301910 xgene_pmu_dev_cleanup(struct xgene_pmu *xgene_pmu, struct list_head *pmus)
19311911 {
19321912 struct xgene_pmu_dev_ctx *ctx;
1933− struct device *dev = xgene_pmu->dev;
1934− struct xgene_pmu_dev *pmu_dev;
19351913
19361914 list_for_each_entry(ctx, pmus, next) {
1937− pmu_dev = ctx->pmu_dev;
1938− if (pmu_dev->inf->csr)
1939− devm_iounmap(dev, pmu_dev->inf->csr);
1940− devm_kfree(dev, ctx);
1941− devm_kfree(dev, pmu_dev);
1915+ perf_pmu_unregister(&ctx->pmu_dev->pmu);
19421916 }
19431917 }
19441918
@@ -1951,10 +1925,6 @@ static int xgene_pmu_remove(struct platform_device *pdev) · 6 unmodified lines
19511925 xgene_pmu_dev_cleanup(xgene_pmu, &xgene_pmu->mcbpmus);
19521926 xgene_pmu_dev_cleanup(xgene_pmu, &xgene_pmu->mcpmus);
19531927
1954− if (xgene_pmu->pcppmu_csr)
1955− devm_iounmap(&pdev->dev, xgene_pmu->pcppmu_csr);
1956− devm_kfree(&pdev->dev, xgene_pmu);
1957−
19581928 return 0;
19591929 }
19601930
include/linux/raid/pq.h+1−0
@@ -121,6 +121,7 @@ extern const struct raid6_recov_calls raid6_recov_ssse3; · 120 unmodified lines
121121 extern const struct raid6_recov_calls raid6_recov_avx2;
122122 extern const struct raid6_recov_calls raid6_recov_avx512;
123123 extern const struct raid6_recov_calls raid6_recov_s390xc;
124+extern const struct raid6_recov_calls raid6_recov_neon;
124125
125126 extern const struct raid6_calls raid6_neonx1;
126127 extern const struct raid6_calls raid6_neonx2;
include/linux/thread_info.h+4−0
@@ -38,6 +38,10 @@ enum { · 37 unmodified lines
3838
3939 #ifdef __KERNEL__
4040
41+#ifndef THREAD_ALIGN
42+#define THREAD_ALIGN THREAD_SIZE
43+#endif
44+
4145 #ifdef CONFIG_DEBUG_STACK_USAGE
4246 # define THREADINFO_GFP (GFP_KERNEL_ACCOUNT | __GFP_NOTRACK | \
4347 __GFP_ZERO)
kernel/fork.c+2−1
@@ -88,6 +88,7 @@ · 87 unmodified lines
8888 #include <linux/sysctl.h>
8989 #include <linux/kcov.h>
9090 #include <linux/livepatch.h>
91+#include <linux/thread_info.h>
9192
9293 #include <asm/pgtable.h>
9394 #include <asm/pgalloc.h>
@@ -217,7 +218,7 @@ static unsigned long *alloc_thread_stack_node(struct task_struct *tsk, int node) · 123 unmodified lines
217218 return s->addr;
218219 }
219220
220− stack = __vmalloc_node_range(THREAD_SIZE, THREAD_SIZE,
221+ stack = __vmalloc_node_range(THREAD_SIZE, THREAD_ALIGN,
221222 VMALLOC_START, VMALLOC_END,
222223 THREADINFO_GFP,
223224 PAGE_KERNEL,
lib/raid6/Makefile+3−1
@@ -5,7 +5,7 @@ raid6_pq-y += algos.o recov.o tables.o int1.o int2.o int4.o \ · 4 unmodified lines
55
66 raid6_pq-$(CONFIG_X86) += recov_ssse3.o recov_avx2.o mmx.o sse1.o sse2.o avx2.o avx512.o recov_avx512.o
77 raid6_pq-$(CONFIG_ALTIVEC) += altivec1.o altivec2.o altivec4.o altivec8.o
8−raid6_pq-$(CONFIG_KERNEL_MODE_NEON) += neon.o neon1.o neon2.o neon4.o neon8.o
8+raid6_pq-$(CONFIG_KERNEL_MODE_NEON) += neon.o neon1.o neon2.o neon4.o neon8.o recov_neon.o recov_neon_inner.o
99 raid6_pq-$(CONFIG_TILEGX) += tilegx8.o
1010 raid6_pq-$(CONFIG_S390) += s390vx8.o recov_s390xc.o
1111
@@ -26,7 +26,9 @@ NEON_FLAGS := -ffreestanding · 14 unmodified lines
2626 ifeq ($(ARCH),arm)
2727 NEON_FLAGS += -mfloat-abi=softfp -mfpu=neon
2828 endif
29+CFLAGS_recov_neon_inner.o += $(NEON_FLAGS)
2930 ifeq ($(ARCH),arm64)
31+CFLAGS_REMOVE_recov_neon_inner.o += -mgeneral-regs-only
3032 CFLAGS_REMOVE_neon1.o += -mgeneral-regs-only
3133 CFLAGS_REMOVE_neon2.o += -mgeneral-regs-only
3234 CFLAGS_REMOVE_neon4.o += -mgeneral-regs-only
lib/raid6/algos.c+3−0
@@ -112,6 +112,9 @@ const struct raid6_recov_calls *const raid6_recov_algos[] = { · 111 unmodified lines
112112 #endif
113113 #ifdef CONFIG_S390
114114 &raid6_recov_s390xc,
115+#endif
116+#if defined(CONFIG_KERNEL_MODE_NEON)
117+ &raid6_recov_neon,
115118 #endif
116119 &raid6_recov_intx1,
117120 NULL
lib/raid6/neon.uc+30−3
@@ -46,8 +46,12 @@ static inline unative_t SHLBYTE(unative_t v) · 45 unmodified lines
4646 */
4747 static inline unative_t MASK(unative_t v)
4848 {
49− const uint8x16_t temp = NBYTES(0);
50− return (unative_t)vcltq_s8((int8x16_t)v, (int8x16_t)temp);
49+ return (unative_t)vshrq_n_s8((int8x16_t)v, 7);
50+}
51+
52+static inline unative_t PMUL(unative_t v, unative_t u)
53+{
54+ return (unative_t)vmulq_p8((poly8x16_t)v, (poly8x16_t)u);
5155 }
5256
5357 void raid6_neon$#_gen_syndrome_real(int disks, unsigned long bytes, void **ptrs)
@@ -110,7 +114,30 @@ void raid6_neon$#_xor_syndrome_real(int disks, int start, int stop, · 56 unmodified lines
110114 wq$$ = veorq_u8(w1$$, wd$$);
111115 }
112116 /* P/Q left side optimization */
113− for ( z = start-1 ; z >= 0 ; z-- ) {
117+ for ( z = start-1 ; z >= 3 ; z -= 4 ) {
118+ w2$$ = vshrq_n_u8(wq$$, 4);
119+ w1$$ = vshlq_n_u8(wq$$, 4);
120+
121+ w2$$ = PMUL(w2$$, x1d);
122+ wq$$ = veorq_u8(w1$$, w2$$);
123+ }
124+
125+ switch (z) {
126+ case 2:
127+ w2$$ = vshrq_n_u8(wq$$, 5);
128+ w1$$ = vshlq_n_u8(wq$$, 3);
129+
130+ w2$$ = PMUL(w2$$, x1d);
131+ wq$$ = veorq_u8(w1$$, w2$$);
132+ break;
133+ case 1:
134+ w2$$ = vshrq_n_u8(wq$$, 6);
135+ w1$$ = vshlq_n_u8(wq$$, 2);
136+
137+ w2$$ = PMUL(w2$$, x1d);
138+ wq$$ = veorq_u8(w1$$, w2$$);
139+ break;
140+ case 0:
114141 w2$$ = MASK(wq$$);
115142 w1$$ = SHLBYTE(wq$$);
116143
lib/raid6/recov_neon.c+110−0
@@ -0,0 +1,110 @@
1+/*
2+ * Copyright (C) 2012 Intel Corporation
3+ * Copyright (C) 2017 Linaro Ltd. <ard.biesheuvel@linaro.org>
4+ *
5+ * This program is free software; you can redistribute it and/or
6+ * modify it under the terms of the GNU General Public License
7+ * as published by the Free Software Foundation; version 2
8+ * of the License.
9+ */
10+
11+#include <linux/raid/pq.h>
12+
13+#ifdef __KERNEL__
14+#include <asm/neon.h>
15+#else
16+#define kernel_neon_begin()
17+#define kernel_neon_end()
18+#define cpu_has_neon() (1)
19+#endif
20+
21+static int raid6_has_neon(void)
22+{
23+ return cpu_has_neon();
24+}
25+
26+void __raid6_2data_recov_neon(int bytes, uint8_t *p, uint8_t *q, uint8_t *dp,
27+ uint8_t *dq, const uint8_t *pbmul,
28+ const uint8_t *qmul);
29+
30+void __raid6_datap_recov_neon(int bytes, uint8_t *p, uint8_t *q, uint8_t *dq,
31+ const uint8_t *qmul);
32+
33+static void raid6_2data_recov_neon(int disks, size_t bytes, int faila,
34+ int failb, void **ptrs)
35+{
36+ u8 *p, *q, *dp, *dq;
37+ const u8 *pbmul; /* P multiplier table for B data */
38+ const u8 *qmul; /* Q multiplier table (for both) */
39+
40+ p = (u8 *)ptrs[disks - 2];
41+ q = (u8 *)ptrs[disks - 1];
42+
43+ /*
44+ * Compute syndrome with zero for the missing data pages
45+ * Use the dead data pages as temporary storage for
46+ * delta p and delta q
47+ */
48+ dp = (u8 *)ptrs[faila];
49+ ptrs[faila] = (void *)raid6_empty_zero_page;
50+ ptrs[disks - 2] = dp;
51+ dq = (u8 *)ptrs[failb];
52+ ptrs[failb] = (void *)raid6_empty_zero_page;
53+ ptrs[disks - 1] = dq;
54+
55+ raid6_call.gen_syndrome(disks, bytes, ptrs);
56+
57+ /* Restore pointer table */
58+ ptrs[faila] = dp;
59+ ptrs[failb] = dq;
60+ ptrs[disks - 2] = p;
61+ ptrs[disks - 1] = q;
62+
63+ /* Now, pick the proper data tables */
64+ pbmul = raid6_vgfmul[raid6_gfexi[failb-faila]];
65+ qmul = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila] ^
66+ raid6_gfexp[failb]]];
67+
68+ kernel_neon_begin();
69+ __raid6_2data_recov_neon(bytes, p, q, dp, dq, pbmul, qmul);
70+ kernel_neon_end();
71+}
72+
73+static void raid6_datap_recov_neon(int disks, size_t bytes, int faila,
74+ void **ptrs)
75+{
76+ u8 *p, *q, *dq;
77+ const u8 *qmul; /* Q multiplier table */
78+
79+ p = (u8 *)ptrs[disks - 2];
80+ q = (u8 *)ptrs[disks - 1];
81+
82+ /*
83+ * Compute syndrome with zero for the missing data page
84+ * Use the dead data page as temporary storage for delta q
85+ */
86+ dq = (u8 *)ptrs[faila];
87+ ptrs[faila] = (void *)raid6_empty_zero_page;
88+ ptrs[disks - 1] = dq;
89+
90+ raid6_call.gen_syndrome(disks, bytes, ptrs);
91+
92+ /* Restore pointer table */
93+ ptrs[faila] = dq;
94+ ptrs[disks - 1] = q;
95+
96+ /* Now, pick the proper data tables */
97+ qmul = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila]]];
98+
99+ kernel_neon_begin();
100+ __raid6_datap_recov_neon(bytes, p, q, dq, qmul);
101+ kernel_neon_end();
102+}
103+
104+const struct raid6_recov_calls raid6_recov_neon = {
105+ .data2 = raid6_2data_recov_neon,
106+ .datap = raid6_datap_recov_neon,
107+ .valid = raid6_has_neon,
108+ .name = "neon",
109+ .priority = 10,
110+};
lib/raid6/recov_neon_inner.c+117−0
@@ -0,0 +1,117 @@
1+/*
2+ * Copyright (C) 2012 Intel Corporation
3+ * Copyright (C) 2017 Linaro Ltd. <ard.biesheuvel@linaro.org>
4+ *
5+ * This program is free software; you can redistribute it and/or
6+ * modify it under the terms of the GNU General Public License
7+ * as published by the Free Software Foundation; version 2
8+ * of the License.
9+ */
10+
11+#include <arm_neon.h>
12+
13+static const uint8x16_t x0f = {
14+ 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f,
15+ 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f,
16+};
17+
18+#ifdef CONFIG_ARM
19+/*
20+ * AArch32 does not provide this intrinsic natively because it does not
21+ * implement the underlying instruction. AArch32 only provides a 64-bit
22+ * wide vtbl.8 instruction, so use that instead.
23+ */
24+static uint8x16_t vqtbl1q_u8(uint8x16_t a, uint8x16_t b)
25+{
26+ union {
27+ uint8x16_t val;
28+ uint8x8x2_t pair;
29+ } __a = { a };
30+
31+ return vcombine_u8(vtbl2_u8(__a.pair, vget_low_u8(b)),
32+ vtbl2_u8(__a.pair, vget_high_u8(b)));
33+}
34+#endif
35+
36+void __raid6_2data_recov_neon(int bytes, uint8_t *p, uint8_t *q, uint8_t *dp,
37+ uint8_t *dq, const uint8_t *pbmul,
38+ const uint8_t *qmul)
39+{
40+ uint8x16_t pm0 = vld1q_u8(pbmul);
41+ uint8x16_t pm1 = vld1q_u8(pbmul + 16);
42+ uint8x16_t qm0 = vld1q_u8(qmul);
43+ uint8x16_t qm1 = vld1q_u8(qmul + 16);
44+
45+ /*
46+ * while ( bytes-- ) {
47+ * uint8_t px, qx, db;
48+ *
49+ * px = *p ^ *dp;
50+ * qx = qmul[*q ^ *dq];
51+ * *dq++ = db = pbmul[px] ^ qx;
52+ * *dp++ = db ^ px;
53+ * p++; q++;
54+ * }
55+ */
56+
57+ while (bytes) {
58+ uint8x16_t vx, vy, px, qx, db;
59+
60+ px = veorq_u8(vld1q_u8(p), vld1q_u8(dp));
61+ vx = veorq_u8(vld1q_u8(q), vld1q_u8(dq));
62+
63+ vy = (uint8x16_t)vshrq_n_s16((int16x8_t)vx, 4);
64+ vx = vqtbl1q_u8(qm0, vandq_u8(vx, x0f));
65+ vy = vqtbl1q_u8(qm1, vandq_u8(vy, x0f));
66+ qx = veorq_u8(vx, vy);
67+
68+ vy = (uint8x16_t)vshrq_n_s16((int16x8_t)px, 4);
69+ vx = vqtbl1q_u8(pm0, vandq_u8(px, x0f));
70+ vy = vqtbl1q_u8(pm1, vandq_u8(vy, x0f));
71+ vx = veorq_u8(vx, vy);
72+ db = veorq_u8(vx, qx);
73+
74+ vst1q_u8(dq, db);
75+ vst1q_u8(dp, veorq_u8(db, px));
76+
77+ bytes -= 16;
78+ p += 16;
79+ q += 16;
80+ dp += 16;
81+ dq += 16;
82+ }
83+}
84+
85+void __raid6_datap_recov_neon(int bytes, uint8_t *p, uint8_t *q, uint8_t *dq,
86+ const uint8_t *qmul)
87+{
88+ uint8x16_t qm0 = vld1q_u8(qmul);
89+ uint8x16_t qm1 = vld1q_u8(qmul + 16);
90+
91+ /*
92+ * while (bytes--) {
93+ * *p++ ^= *dq = qmul[*q ^ *dq];
94+ * q++; dq++;
95+ * }
96+ */
97+
98+ while (bytes) {
99+ uint8x16_t vx, vy;
100+
101+ vx = veorq_u8(vld1q_u8(q), vld1q_u8(dq));
102+
103+ vy = (uint8x16_t)vshrq_n_s16((int16x8_t)vx, 4);
104+ vx = vqtbl1q_u8(qm0, vandq_u8(vx, x0f));
105+ vy = vqtbl1q_u8(qm1, vandq_u8(vy, x0f));
106+ vx = veorq_u8(vx, vy);
107+ vy = veorq_u8(vx, vld1q_u8(p));
108+
109+ vst1q_u8(dq, vx);
110+ vst1q_u8(p, vy);
111+
112+ bytes -= 16;
113+ p += 16;
114+ q += 16;
115+ dq += 16;
116+ }
117+}

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t