History

Merge tag 'arm64-upstream' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux

Pull arm64 updates from Catalin Marinas: - VMAP_STACK support, allowing the kernel stacks to be allocated in the vmalloc space with a guard page for trapping stack overflows. One of the patches introduces THREAD_ALIGN and changes the generic alloc_thread_stack_node() to use this instead of THREAD_SIZE (no functional change for other architectures) - Contiguous PTE hugetlb support re-enabled (after being reverted a couple of times). We now have the semantics agreed in the generic mm layer together with API improvements so that the architecture code can detect between contiguous and non-contiguous huge PTEs - Initial support for persistent memory on ARM: DC CVAP instruction exposed to user space (HWCAP) and the in-kernel pmem API implemented - raid6 improvements for arm64: faster algorithm for the delta syndrome and implementation of the recovery routines using Neon - FP/SIMD refactoring and removal of support for Neon in interrupt context. This is in preparation for full SVE support - PTE accessors converted from inline asm to cmpxchg so that we can use LSE atomics if available (ARMv8.1) - Perf support for Cortex-A35 and A73 - Non-urgent fixes and cleanups * tag 'arm64-upstream' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux: (75 commits) arm64: cleanup {COMPAT_,}SET_PERSONALITY() macro arm64: introduce separated bits for mm_context_t flags arm64: hugetlb: Cleanup setup_hugepagesz arm64: Re-enable support for contiguous hugepages arm64: hugetlb: Override set_huge_swap_pte_at() to support contiguous hugepages arm64: hugetlb: Override huge_pte_clear() to support contiguous hugepages arm64: hugetlb: Handle swap entries in huge_pte_offset() for contiguous hugepages arm64: hugetlb: Add break-before-make logic for contiguous entries arm64: hugetlb: Spring clean huge pte accessors arm64: hugetlb: Introduce pte_pgprot helper arm64: hugetlb: set_huge_pte_at Add WARN_ON on !pte_present arm64: kexec: have own crash_smp_send_stop() for crash dump for nonpanic cores arm64: dma-mapping: Mark atomic_pool as __ro_after_init arm64: dma-mapping: Do not pass data to gen_pool_set_algo() arm64: Remove the !CONFIG_ARM64_HW_AFDBM alternative code paths arm64: Ignore hardware dirty bit updates in ptep_set_wrprotect() arm64: Move PTE_RDONLY bit handling out of set_pte_at() kvm: arm64: Convert kvm_set_s2pte_readonly() from inline asm to cmpxchg() arm64: Convert pte handling from inline asm to using (cmp)xchg arm64: neon/efi: Make EFI fpsimd save/restore variables static ...

Linus Torvalds committed 9 years ago

Commit 04759192 parents 9e85ae6 d1be5c981 files · +1840 −924Browse files

Unified · Split

81 files
  1. Documentation/arm64/cpu-feature-registers.txt +2 −0
  2. Documentation/devicetree/bindings/arm/pmu.txt +2 −0
  3. arch/arm64/Kconfig +13 −0
  4. arch/arm64/include/asm/Kbuild +0 −1
  5. arch/arm64/include/asm/asm-bug.h +54 −0
  6. arch/arm64/include/asm/assembler.h +24 −1
  7. arch/arm64/include/asm/bug.h +3 −32
  8. arch/arm64/include/asm/cacheflush.h +3 −1
  9. arch/arm64/include/asm/cpucaps.h +2 −1
  10. arch/arm64/include/asm/efi.h +11 −2
  11. arch/arm64/include/asm/elf.h +0 −2
  12. arch/arm64/include/asm/esr.h +31 −11
  13. arch/arm64/include/asm/fpsimd.h +3 −13
  14. arch/arm64/include/asm/fpsimdmacros.h +0 −56
  15. arch/arm64/include/asm/hugetlb.h +8 −1
  16. arch/arm64/include/asm/irq.h +0 −42
  17. arch/arm64/include/asm/kvm_mmu.h +9 −12
  18. arch/arm64/include/asm/memory.h +53 −0
  19. arch/arm64/include/asm/mmu.h +2 −0
  20. arch/arm64/include/asm/neon.h +13 −3
  21. arch/arm64/include/asm/numa.h +0 −3
  22. arch/arm64/include/asm/page-def.h +34 −0
  23. arch/arm64/include/asm/page.h +1 −11
  24. arch/arm64/include/asm/pgtable-prot.h +8 −10
  25. arch/arm64/include/asm/pgtable.h +41 −62
  26. arch/arm64/include/asm/processor.h +2 −2
  27. arch/arm64/include/asm/ptrace.h +30 −1
  28. arch/arm64/include/asm/signal32.h +0 −2
  29. arch/arm64/include/asm/simd.h +56 −0
  30. arch/arm64/include/asm/smp.h +1 −1
  31. arch/arm64/include/asm/stacktrace.h +59 −2
  32. arch/arm64/include/asm/string.h +4 −0
  33. arch/arm64/include/asm/sysreg.h +1 −0
  34. arch/arm64/include/asm/thread_info.h +4 −9
  35. arch/arm64/include/asm/traps.h +5 −0
  36. arch/arm64/include/asm/uaccess.h +12 −0
  37. arch/arm64/include/uapi/asm/hwcap.h +1 −0
  38. arch/arm64/kernel/asm-offsets.c +1 −0
  39. arch/arm64/kernel/cpufeature.c +13 −0
  40. arch/arm64/kernel/cpuinfo.c +1 −0
  41. arch/arm64/kernel/entry-fpsimd.S +0 −24
  42. arch/arm64/kernel/entry.S +183 −100
  43. arch/arm64/kernel/fpsimd.c +132 −40
  44. arch/arm64/kernel/head.S +13 −9
  45. arch/arm64/kernel/hibernate.c +2 −2
  46. arch/arm64/kernel/irq.c +38 −2
  47. arch/arm64/kernel/machine_kexec.c +1 −1
  48. arch/arm64/kernel/perf_callchain.c +0 −1
  49. arch/arm64/kernel/perf_event.c +85 −123
  50. arch/arm64/kernel/probes/uprobes.c +1 −1
  51. arch/arm64/kernel/process.c +9 −4
  52. arch/arm64/kernel/ptrace.c +3 −2
  53. arch/arm64/kernel/return_address.c +0 −1
  54. arch/arm64/kernel/signal.c +5 −5
  55. arch/arm64/kernel/signal32.c +1 −1
  56. arch/arm64/kernel/smp.c +12 −2
  57. arch/arm64/kernel/stacktrace.c +10 −50
  58. arch/arm64/kernel/time.c +0 −1
  59. arch/arm64/kernel/traps.c +49 −30
  60. arch/arm64/kernel/vdso.c +15 −0
  61. arch/arm64/kernel/vmlinux.lds.S +1 −17
  62. arch/arm64/kvm/hyp/s2-setup.c +1 −1
  63. arch/arm64/lib/Makefile +2 −0
  64. arch/arm64/lib/uaccess_flushcache.c +47 −0
  65. arch/arm64/mm/cache.S +28 −9
  66. arch/arm64/mm/dma-mapping.c +2 −2
  67. arch/arm64/mm/fault.c +58 −19
  68. arch/arm64/mm/flush.c +16 −0
  69. arch/arm64/mm/hugetlbpage.c +230 −84
  70. drivers/acpi/arm64/iort.c +87 −53
  71. drivers/firmware/efi/libstub/arm64-stub.c +4 −2
  72. drivers/perf/arm_pmu.c +6 −0
  73. drivers/perf/xgene_pmu.c +22 −52
  74. include/linux/raid/pq.h +1 −0
  75. include/linux/thread_info.h +4 −0
  76. kernel/fork.c +2 −1
  77. lib/raid6/Makefile +3 −1
  78. lib/raid6/algos.c +3 −0
  79. lib/raid6/neon.uc +30 −3
  80. lib/raid6/recov_neon.c +110 −0
  81. lib/raid6/recov_neon_inner.c +117 −0
Documentation/arm64/cpu-feature-registers.txt+2−0
@@ -179,6 +179,8 @@ infrastructure: · 178 unmodified lines
179 | FCMA | [19-16] | y |179 | FCMA | [19-16] | y |
180 |--------------------------------------------------|180 |--------------------------------------------------|
181 | JSCVT | [15-12] | y |181 | JSCVT | [15-12] | y |
182 |--------------------------------------------------|
183 | DPB | [3-0] | y |
182 x--------------------------------------------------x184 x--------------------------------------------------x
183 185
184Appendix I: Example186Appendix I: Example
Documentation/devicetree/bindings/arm/pmu.txt+2−0
@@ -9,9 +9,11 @@ Required properties: · 8 unmodified lines
9- compatible : should be one of9- compatible : should be one of
10 "apm,potenza-pmu"10 "apm,potenza-pmu"
11 "arm,armv8-pmuv3"11 "arm,armv8-pmuv3"
12 "arm,cortex-a73-pmu"
12 "arm,cortex-a72-pmu"13 "arm,cortex-a72-pmu"
13 "arm,cortex-a57-pmu"14 "arm,cortex-a57-pmu"
14 "arm,cortex-a53-pmu"15 "arm,cortex-a53-pmu"
16 "arm,cortex-a35-pmu"
15 "arm,cortex-a17-pmu"17 "arm,cortex-a17-pmu"
16 "arm,cortex-a15-pmu"18 "arm,cortex-a15-pmu"
17 "arm,cortex-a12-pmu"19 "arm,cortex-a12-pmu"
arch/arm64/Kconfig+13−0
@@ -75,6 +75,7 @@ config ARM64 · 74 unmodified lines
75 select HAVE_ARCH_SECCOMP_FILTER75 select HAVE_ARCH_SECCOMP_FILTER
76 select HAVE_ARCH_TRACEHOOK76 select HAVE_ARCH_TRACEHOOK
77 select HAVE_ARCH_TRANSPARENT_HUGEPAGE77 select HAVE_ARCH_TRANSPARENT_HUGEPAGE
78 select HAVE_ARCH_VMAP_STACK
78 select HAVE_ARM_SMCCC79 select HAVE_ARM_SMCCC
79 select HAVE_EBPF_JIT80 select HAVE_EBPF_JIT
80 select HAVE_C_RECORDMCOUNT81 select HAVE_C_RECORDMCOUNT
@@ -960,6 +961,18 @@ config ARM64_UAO · 879 unmodified lines
960 regular load/store instructions if the cpu does not implement the961 regular load/store instructions if the cpu does not implement the
961 feature.962 feature.
962 963
964config ARM64_PMEM
965 bool "Enable support for persistent memory"
966 select ARCH_HAS_PMEM_API
967 select ARCH_HAS_UACCESS_FLUSHCACHE
968 help
969 Say Y to enable support for the persistent memory API based on the
970 ARMv8.2 DCPoP feature.
971
972 The feature is detected at runtime, and the kernel will use DC CVAC
973 operations if DC CVAP is not supported (following the behaviour of
974 DC CVAP itself if the system does not define a point of persistence).
975
963endmenu976endmenu
964 977
965config ARM64_MODULE_CMODEL_LARGE978config ARM64_MODULE_CMODEL_LARGE
arch/arm64/include/asm/Kbuild+0−1
@@ -20,7 +20,6 @@ generic-y += rwsem.h · 19 unmodified lines
20generic-y += segment.h20generic-y += segment.h
21generic-y += serial.h21generic-y += serial.h
22generic-y += set_memory.h22generic-y += set_memory.h
23generic-y += simd.h
24generic-y += sizes.h23generic-y += sizes.h
25generic-y += switch_to.h24generic-y += switch_to.h
26generic-y += trace_clock.h25generic-y += trace_clock.h
arch/arm64/include/asm/asm-bug.h+54−0
@@ -0,0 +1,54 @@
1#ifndef __ASM_ASM_BUG_H
2/*
3 * Copyright (C) 2017 ARM Limited
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License version 2 as
7 * published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program. If not, see <http://www.gnu.org/licenses/>.
16 */
17#define __ASM_ASM_BUG_H
18
19#include <asm/brk-imm.h>
20
21#ifdef CONFIG_DEBUG_BUGVERBOSE
22#define _BUGVERBOSE_LOCATION(file, line) __BUGVERBOSE_LOCATION(file, line)
23#define __BUGVERBOSE_LOCATION(file, line) \
24 .pushsection .rodata.str,"aMS",@progbits,1; \
25 2: .string file; \
26 .popsection; \
27 \
28 .long 2b - 0b; \
29 .short line;
30#else
31#define _BUGVERBOSE_LOCATION(file, line)
32#endif
33
34#ifdef CONFIG_GENERIC_BUG
35
36#define __BUG_ENTRY(flags) \
37 .pushsection __bug_table,"aw"; \
38 .align 2; \
39 0: .long 1f - 0b; \
40_BUGVERBOSE_LOCATION(__FILE__, __LINE__) \
41 .short flags; \
42 .popsection; \
43 1:
44#else
45#define __BUG_ENTRY(flags)
46#endif
47
48#define ASM_BUG_FLAGS(flags) \
49 __BUG_ENTRY(flags) \
50 brk BUG_BRK_IMM
51
52#define ASM_BUG() ASM_BUG_FLAGS(0)
53
54#endif /* __ASM_ASM_BUG_H */
arch/arm64/include/asm/assembler.h+24−1
@@ -230,12 +230,18 @@ lr .req x30 // link register · 229 unmodified lines
230 .endm230 .endm
231 231
232 /*232 /*
233 * @dst: Result of per_cpu(sym, smp_processor_id())233 * @dst: Result of per_cpu(sym, smp_processor_id()), can be SP for
234 * non-module code
234 * @sym: The name of the per-cpu variable235 * @sym: The name of the per-cpu variable
235 * @tmp: scratch register236 * @tmp: scratch register
236 */237 */
237 .macro adr_this_cpu, dst, sym, tmp238 .macro adr_this_cpu, dst, sym, tmp
239#ifndef MODULE
240 adrp \tmp, \sym
241 add \dst, \tmp, #:lo12:\sym
242#else
238 adr_l \dst, \sym243 adr_l \dst, \sym
244#endif
239 mrs \tmp, tpidr_el1245 mrs \tmp, tpidr_el1
240 add \dst, \dst, \tmp246 add \dst, \dst, \tmp
241 .endm247 .endm
@@ -352,6 +358,12 @@ alternative_if_not ARM64_WORKAROUND_CLEAN_CACHE · 110 unmodified lines
352 dc \op, \kaddr358 dc \op, \kaddr
353alternative_else359alternative_else
354 dc civac, \kaddr360 dc civac, \kaddr
361alternative_endif
362 .elseif (\op == cvap)
363alternative_if ARM64_HAS_DCPOP
364 sys 3, c7, c12, 1, \kaddr // dc cvap
365alternative_else
366 dc cvac, \kaddr
355alternative_endif367alternative_endif
356 .else368 .else
357 dc \op, \kaddr369 dc \op, \kaddr
@@ -403,6 +415,17 @@ alternative_endif · 45 unmodified lines
403 .size __pi_##x, . - x; \415 .size __pi_##x, . - x; \
404 ENDPROC(x)416 ENDPROC(x)
405 417
418/*
419 * Annotate a function as being unsuitable for kprobes.
420 */
421#ifdef CONFIG_KPROBES
422#define NOKPROBE(x) \
423 .pushsection "_kprobe_blacklist", "aw"; \
424 .quad x; \
425 .popsection;
426#else
427#define NOKPROBE(x)
428#endif
406 /*429 /*
407 * Emit a 64-bit absolute little endian symbol reference in a way that430 * Emit a 64-bit absolute little endian symbol reference in a way that
408 * ensures that it will be resolved at build time, even when building a431 * ensures that it will be resolved at build time, even when building a
arch/arm64/include/asm/bug.h+3−32
@@ -18,41 +18,12 @@ · 17 unmodified lines
18#ifndef _ARCH_ARM64_ASM_BUG_H18#ifndef _ARCH_ARM64_ASM_BUG_H
19#define _ARCH_ARM64_ASM_BUG_H19#define _ARCH_ARM64_ASM_BUG_H
20 20
21#include <asm/brk-imm.h>21#include <linux/stringify.h>
22 22
23#ifdef CONFIG_DEBUG_BUGVERBOSE23#include <asm/asm-bug.h>
24#define _BUGVERBOSE_LOCATION(file, line) __BUGVERBOSE_LOCATION(file, line)
25#define __BUGVERBOSE_LOCATION(file, line) \
26 ".pushsection .rodata.str,\"aMS\",@progbits,1\n" \
27 "2: .string \"" file "\"\n\t" \
28 ".popsection\n\t" \
29 \
30 ".long 2b - 0b\n\t" \
31 ".short " #line "\n\t"
32#else
33#define _BUGVERBOSE_LOCATION(file, line)
34#endif
35
36#ifdef CONFIG_GENERIC_BUG
37
38#define __BUG_ENTRY(flags) \
39 ".pushsection __bug_table,\"aw\"\n\t" \
40 ".align 2\n\t" \
41 "0: .long 1f - 0b\n\t" \
42_BUGVERBOSE_LOCATION(__FILE__, __LINE__) \
43 ".short " #flags "\n\t" \
44 ".popsection\n" \
45 "1: "
46#else
47#define __BUG_ENTRY(flags) ""
48#endif
49 24
50#define __BUG_FLAGS(flags) \25#define __BUG_FLAGS(flags) \
51 asm volatile ( \26 asm volatile (__stringify(ASM_BUG_FLAGS(flags)));
52 __BUG_ENTRY(flags) \
53 "brk %[imm]" :: [imm] "i" (BUG_BRK_IMM) \
54 );
55
56 27
57#define BUG() do { \28#define BUG() do { \
58 __BUG_FLAGS(0); \29 __BUG_FLAGS(0); \
arch/arm64/include/asm/cacheflush.h+3−1
@@ -67,7 +67,9 @@ · 66 unmodified lines
67 */67 */
68extern void flush_icache_range(unsigned long start, unsigned long end);68extern void flush_icache_range(unsigned long start, unsigned long end);
69extern void __flush_dcache_area(void *addr, size_t len);69extern void __flush_dcache_area(void *addr, size_t len);
70extern void __inval_dcache_area(void *addr, size_t len);
70extern void __clean_dcache_area_poc(void *addr, size_t len);71extern void __clean_dcache_area_poc(void *addr, size_t len);
72extern void __clean_dcache_area_pop(void *addr, size_t len);
71extern void __clean_dcache_area_pou(void *addr, size_t len);73extern void __clean_dcache_area_pou(void *addr, size_t len);
72extern long __flush_cache_user_range(unsigned long start, unsigned long end);74extern long __flush_cache_user_range(unsigned long start, unsigned long end);
73extern void sync_icache_aliases(void *kaddr, unsigned long len);75extern void sync_icache_aliases(void *kaddr, unsigned long len);
@@ -150,6 +152,6 @@ static inline void flush_cache_vunmap(unsigned long start, unsigned long end) · 76 unmodified lines
150{152{
151}153}
152 154
153int set_memory_valid(unsigned long addr, unsigned long size, int enable);155int set_memory_valid(unsigned long addr, int numpages, int enable);
154 156
155#endif157#endif
arch/arm64/include/asm/cpucaps.h+2−1
@@ -39,7 +39,8 @@ · 38 unmodified lines
39#define ARM64_WORKAROUND_QCOM_FALKOR_E1003 1839#define ARM64_WORKAROUND_QCOM_FALKOR_E1003 18
40#define ARM64_WORKAROUND_858921 1940#define ARM64_WORKAROUND_858921 19
41#define ARM64_WORKAROUND_CAVIUM_30115 2041#define ARM64_WORKAROUND_CAVIUM_30115 20
42#define ARM64_HAS_DCPOP 21
42 43
43#define ARM64_NCAPS 2144#define ARM64_NCAPS 22
44 45
45#endif /* __ASM_CPUCAPS_H */46#endif /* __ASM_CPUCAPS_H */
arch/arm64/include/asm/efi.h+11−2
@@ -3,7 +3,9 @@ · 2 unmodified lines
3 3
4#include <asm/boot.h>4#include <asm/boot.h>
5#include <asm/cpufeature.h>5#include <asm/cpufeature.h>
6#include <asm/fpsimd.h>
6#include <asm/io.h>7#include <asm/io.h>
8#include <asm/memory.h>
7#include <asm/mmu_context.h>9#include <asm/mmu_context.h>
8#include <asm/neon.h>10#include <asm/neon.h>
9#include <asm/ptrace.h>11#include <asm/ptrace.h>
@@ -20,8 +22,8 @@ int efi_set_mapping_permissions(struct mm_struct *mm, efi_memory_desc_t *md); · 10 unmodified lines
20 22
21#define arch_efi_call_virt_setup() \23#define arch_efi_call_virt_setup() \
22({ \24({ \
23 kernel_neon_begin(); \
24 efi_virtmap_load(); \25 efi_virtmap_load(); \
26 __efi_fpsimd_begin(); \
25})27})
26 28
27#define arch_efi_call_virt(p, f, args...) \29#define arch_efi_call_virt(p, f, args...) \
@@ -33,8 +35,8 @@ int efi_set_mapping_permissions(struct mm_struct *mm, efi_memory_desc_t *md); · 5 unmodified lines
33 35
34#define arch_efi_call_virt_teardown() \36#define arch_efi_call_virt_teardown() \
35({ \37({ \
38 __efi_fpsimd_end(); \
36 efi_virtmap_unload(); \39 efi_virtmap_unload(); \
37 kernel_neon_end(); \
38})40})
39 41
40#define ARCH_EFI_IRQ_FLAGS_MASK (PSR_D_BIT | PSR_A_BIT | PSR_I_BIT | PSR_F_BIT)42#define ARCH_EFI_IRQ_FLAGS_MASK (PSR_D_BIT | PSR_A_BIT | PSR_I_BIT | PSR_F_BIT)
@@ -48,6 +50,13 @@ int efi_set_mapping_permissions(struct mm_struct *mm, efi_memory_desc_t *md); · 7 unmodified lines
48 */50 */
49#define EFI_FDT_ALIGN SZ_2M /* used by allocate_new_fdt_and_exit_boot() */51#define EFI_FDT_ALIGN SZ_2M /* used by allocate_new_fdt_and_exit_boot() */
50 52
53/*
54 * In some configurations (e.g. VMAP_STACK && 64K pages), stacks built into the
55 * kernel need greater alignment than we require the segments to be padded to.
56 */
57#define EFI_KIMG_ALIGN \
58 (SEGMENT_ALIGN > THREAD_ALIGN ? SEGMENT_ALIGN : THREAD_ALIGN)
59
51/* on arm64, the FDT may be located anywhere in system RAM */60/* on arm64, the FDT may be located anywhere in system RAM */
52static inline unsigned long efi_get_max_fdt_addr(unsigned long dram_base)61static inline unsigned long efi_get_max_fdt_addr(unsigned long dram_base)
53{62{
arch/arm64/include/asm/elf.h+0−2
@@ -139,7 +139,6 @@ typedef struct user_fpsimd_state elf_fpregset_t; · 138 unmodified lines
139 139
140#define SET_PERSONALITY(ex) \140#define SET_PERSONALITY(ex) \
141({ \141({ \
142 clear_bit(TIF_32BIT, &current->mm->context.flags); \
143 clear_thread_flag(TIF_32BIT); \142 clear_thread_flag(TIF_32BIT); \
144 current->personality &= ~READ_IMPLIES_EXEC; \143 current->personality &= ~READ_IMPLIES_EXEC; \
145})144})
@@ -195,7 +194,6 @@ typedef compat_elf_greg_t compat_elf_gregset_t[COMPAT_ELF_NGREG]; · 49 unmodified lines
195 */194 */
196#define COMPAT_SET_PERSONALITY(ex) \195#define COMPAT_SET_PERSONALITY(ex) \
197({ \196({ \
198 set_bit(TIF_32BIT, &current->mm->context.flags); \
199 set_thread_flag(TIF_32BIT); \197 set_thread_flag(TIF_32BIT); \
200 })198 })
201#define COMPAT_ARCH_DLINFO199#define COMPAT_ARCH_DLINFO
arch/arm64/include/asm/esr.h+31−11
@@ -77,16 +77,23 @@ · 76 unmodified lines
77#define ESR_ELx_EC_MASK (UL(0x3F) << ESR_ELx_EC_SHIFT)77#define ESR_ELx_EC_MASK (UL(0x3F) << ESR_ELx_EC_SHIFT)
78#define ESR_ELx_EC(esr) (((esr) & ESR_ELx_EC_MASK) >> ESR_ELx_EC_SHIFT)78#define ESR_ELx_EC(esr) (((esr) & ESR_ELx_EC_MASK) >> ESR_ELx_EC_SHIFT)
79 79
80#define ESR_ELx_IL (UL(1) << 25)80#define ESR_ELx_IL_SHIFT (25)
81#define ESR_ELx_IL (UL(1) << ESR_ELx_IL_SHIFT)
81#define ESR_ELx_ISS_MASK (ESR_ELx_IL - 1)82#define ESR_ELx_ISS_MASK (ESR_ELx_IL - 1)
82 83
83/* ISS field definitions shared by different classes */84/* ISS field definitions shared by different classes */
84#define ESR_ELx_WNR (UL(1) << 6)85#define ESR_ELx_WNR_SHIFT (6)
86#define ESR_ELx_WNR (UL(1) << ESR_ELx_WNR_SHIFT)
85 87
86/* Shared ISS field definitions for Data/Instruction aborts */88/* Shared ISS field definitions for Data/Instruction aborts */
87#define ESR_ELx_FnV (UL(1) << 10)89#define ESR_ELx_SET_SHIFT (11)
88#define ESR_ELx_EA (UL(1) << 9)90#define ESR_ELx_SET_MASK (UL(3) << ESR_ELx_SET_SHIFT)
89#define ESR_ELx_S1PTW (UL(1) << 7)91#define ESR_ELx_FnV_SHIFT (10)
92#define ESR_ELx_FnV (UL(1) << ESR_ELx_FnV_SHIFT)
93#define ESR_ELx_EA_SHIFT (9)
94#define ESR_ELx_EA (UL(1) << ESR_ELx_EA_SHIFT)
95#define ESR_ELx_S1PTW_SHIFT (7)
96#define ESR_ELx_S1PTW (UL(1) << ESR_ELx_S1PTW_SHIFT)
90 97
91/* Shared ISS fault status code(IFSC/DFSC) for Data/Instruction aborts */98/* Shared ISS fault status code(IFSC/DFSC) for Data/Instruction aborts */
92#define ESR_ELx_FSC (0x3F)99#define ESR_ELx_FSC (0x3F)
@@ -97,15 +104,20 @@ · 4 unmodified lines
97#define ESR_ELx_FSC_PERM (0x0C)104#define ESR_ELx_FSC_PERM (0x0C)
98 105
99/* ISS field definitions for Data Aborts */106/* ISS field definitions for Data Aborts */
100#define ESR_ELx_ISV (UL(1) << 24)107#define ESR_ELx_ISV_SHIFT (24)
108#define ESR_ELx_ISV (UL(1) << ESR_ELx_ISV_SHIFT)
101#define ESR_ELx_SAS_SHIFT (22)109#define ESR_ELx_SAS_SHIFT (22)
102#define ESR_ELx_SAS (UL(3) << ESR_ELx_SAS_SHIFT)110#define ESR_ELx_SAS (UL(3) << ESR_ELx_SAS_SHIFT)
103#define ESR_ELx_SSE (UL(1) << 21)111#define ESR_ELx_SSE_SHIFT (21)
112#define ESR_ELx_SSE (UL(1) << ESR_ELx_SSE_SHIFT)
104#define ESR_ELx_SRT_SHIFT (16)113#define ESR_ELx_SRT_SHIFT (16)
105#define ESR_ELx_SRT_MASK (UL(0x1F) << ESR_ELx_SRT_SHIFT)114#define ESR_ELx_SRT_MASK (UL(0x1F) << ESR_ELx_SRT_SHIFT)
106#define ESR_ELx_SF (UL(1) << 15)115#define ESR_ELx_SF_SHIFT (15)
107#define ESR_ELx_AR (UL(1) << 14)116#define ESR_ELx_SF (UL(1) << ESR_ELx_SF_SHIFT)
108#define ESR_ELx_CM (UL(1) << 8)117#define ESR_ELx_AR_SHIFT (14)
118#define ESR_ELx_AR (UL(1) << ESR_ELx_AR_SHIFT)
119#define ESR_ELx_CM_SHIFT (8)
120#define ESR_ELx_CM (UL(1) << ESR_ELx_CM_SHIFT)
109 121
110/* ISS field definitions for exceptions taken in to Hyp */122/* ISS field definitions for exceptions taken in to Hyp */
111#define ESR_ELx_CV (UL(1) << 24)123#define ESR_ELx_CV (UL(1) << 24)
@@ -157,9 +169,10 @@ · 45 unmodified lines
157/*169/*
158 * User space cache operations have the following sysreg encoding170 * User space cache operations have the following sysreg encoding
159 * in System instructions.171 * in System instructions.
160 * op0=1, op1=3, op2=1, crn=7, crm={ 5, 10, 11, 14 }, WRITE (L=0)172 * op0=1, op1=3, op2=1, crn=7, crm={ 5, 10, 11, 12, 14 }, WRITE (L=0)
161 */173 */
162#define ESR_ELx_SYS64_ISS_CRM_DC_CIVAC 14174#define ESR_ELx_SYS64_ISS_CRM_DC_CIVAC 14
175#define ESR_ELx_SYS64_ISS_CRM_DC_CVAP 12
163#define ESR_ELx_SYS64_ISS_CRM_DC_CVAU 11176#define ESR_ELx_SYS64_ISS_CRM_DC_CVAU 11
164#define ESR_ELx_SYS64_ISS_CRM_DC_CVAC 10177#define ESR_ELx_SYS64_ISS_CRM_DC_CVAC 10
165#define ESR_ELx_SYS64_ISS_CRM_IC_IVAU 5178#define ESR_ELx_SYS64_ISS_CRM_IC_IVAU 5
@@ -209,6 +222,13 @@ · 43 unmodified lines
209#ifndef __ASSEMBLY__222#ifndef __ASSEMBLY__
210#include <asm/types.h>223#include <asm/types.h>
211 224
225static inline bool esr_is_data_abort(u32 esr)
226{
227 const u32 ec = ESR_ELx_EC(esr);
228
229 return ec == ESR_ELx_EC_DABT_LOW || ec == ESR_ELx_EC_DABT_CUR;
230}
231
212const char *esr_get_class_string(u32 esr);232const char *esr_get_class_string(u32 esr);
213#endif /* __ASSEMBLY */233#endif /* __ASSEMBLY */
214 234
arch/arm64/include/asm/fpsimd.h+3−13
@@ -41,16 +41,6 @@ struct fpsimd_state { · 40 unmodified lines
41 unsigned int cpu;41 unsigned int cpu;
42};42};
43 43
44/*
45 * Struct for stacking the bottom 'n' FP/SIMD registers.
46 */
47struct fpsimd_partial_state {
48 u32 fpsr;
49 u32 fpcr;
50 u32 num_regs;
51 __uint128_t vregs[32];
52};
53
54 44
55#if defined(__KERNEL__) && defined(CONFIG_COMPAT)45#if defined(__KERNEL__) && defined(CONFIG_COMPAT)
56/* Masks for extracting the FPSR and FPCR from the FPSCR */46/* Masks for extracting the FPSR and FPCR from the FPSCR */
@@ -77,9 +67,9 @@ extern void fpsimd_update_current_state(struct fpsimd_state *state); · 20 unmodified lines
77 67
78extern void fpsimd_flush_task_state(struct task_struct *target);68extern void fpsimd_flush_task_state(struct task_struct *target);
79 69
80extern void fpsimd_save_partial_state(struct fpsimd_partial_state *state,70/* For use by EFI runtime services calls only */
81 u32 num_regs);71extern void __efi_fpsimd_begin(void);
82extern void fpsimd_load_partial_state(struct fpsimd_partial_state *state);72extern void __efi_fpsimd_end(void);
83 73
84#endif74#endif
85 75
arch/arm64/include/asm/fpsimdmacros.h+0−56
@@ -75,59 +75,3 @@ · 74 unmodified lines
75 ldr w\tmpnr, [\state, #16 * 2 + 4]75 ldr w\tmpnr, [\state, #16 * 2 + 4]
76 fpsimd_restore_fpcr x\tmpnr, \state76 fpsimd_restore_fpcr x\tmpnr, \state
77.endm77.endm
78
79.macro fpsimd_save_partial state, numnr, tmpnr1, tmpnr2
80 mrs x\tmpnr1, fpsr
81 str w\numnr, [\state, #8]
82 mrs x\tmpnr2, fpcr
83 stp w\tmpnr1, w\tmpnr2, [\state]
84 adr x\tmpnr1, 0f
85 add \state, \state, x\numnr, lsl #4
86 sub x\tmpnr1, x\tmpnr1, x\numnr, lsl #1
87 br x\tmpnr1
88 stp q30, q31, [\state, #-16 * 30 - 16]
89 stp q28, q29, [\state, #-16 * 28 - 16]
90 stp q26, q27, [\state, #-16 * 26 - 16]
91 stp q24, q25, [\state, #-16 * 24 - 16]
92 stp q22, q23, [\state, #-16 * 22 - 16]
93 stp q20, q21, [\state, #-16 * 20 - 16]
94 stp q18, q19, [\state, #-16 * 18 - 16]
95 stp q16, q17, [\state, #-16 * 16 - 16]
96 stp q14, q15, [\state, #-16 * 14 - 16]
97 stp q12, q13, [\state, #-16 * 12 - 16]
98 stp q10, q11, [\state, #-16 * 10 - 16]
99 stp q8, q9, [\state, #-16 * 8 - 16]
100 stp q6, q7, [\state, #-16 * 6 - 16]
101 stp q4, q5, [\state, #-16 * 4 - 16]
102 stp q2, q3, [\state, #-16 * 2 - 16]
103 stp q0, q1, [\state, #-16 * 0 - 16]
1040:
105.endm
106
107.macro fpsimd_restore_partial state, tmpnr1, tmpnr2
108 ldp w\tmpnr1, w\tmpnr2, [\state]
109 msr fpsr, x\tmpnr1
110 fpsimd_restore_fpcr x\tmpnr2, x\tmpnr1
111 adr x\tmpnr1, 0f
112 ldr w\tmpnr2, [\state, #8]
113 add \state, \state, x\tmpnr2, lsl #4
114 sub x\tmpnr1, x\tmpnr1, x\tmpnr2, lsl #1
115 br x\tmpnr1
116 ldp q30, q31, [\state, #-16 * 30 - 16]
117 ldp q28, q29, [\state, #-16 * 28 - 16]
118 ldp q26, q27, [\state, #-16 * 26 - 16]
119 ldp q24, q25, [\state, #-16 * 24 - 16]
120 ldp q22, q23, [\state, #-16 * 22 - 16]
121 ldp q20, q21, [\state, #-16 * 20 - 16]
122 ldp q18, q19, [\state, #-16 * 18 - 16]
123 ldp q16, q17, [\state, #-16 * 16 - 16]
124 ldp q14, q15, [\state, #-16 * 14 - 16]
125 ldp q12, q13, [\state, #-16 * 12 - 16]
126 ldp q10, q11, [\state, #-16 * 10 - 16]
127 ldp q8, q9, [\state, #-16 * 8 - 16]
128 ldp q6, q7, [\state, #-16 * 6 - 16]
129 ldp q4, q5, [\state, #-16 * 4 - 16]
130 ldp q2, q3, [\state, #-16 * 2 - 16]
131 ldp q0, q1, [\state, #-16 * 0 - 16]
1320:
133.endm
arch/arm64/include/asm/hugetlb.h+8−1
@@ -18,7 +18,6 @@ · 17 unmodified lines
18#ifndef __ASM_HUGETLB_H18#ifndef __ASM_HUGETLB_H
19#define __ASM_HUGETLB_H19#define __ASM_HUGETLB_H
20 20
21#include <asm-generic/hugetlb.h>
22#include <asm/page.h>21#include <asm/page.h>
23 22
24static inline pte_t huge_ptep_get(pte_t *ptep)23static inline pte_t huge_ptep_get(pte_t *ptep)
@@ -82,6 +81,14 @@ extern void huge_ptep_set_wrprotect(struct mm_struct *mm, · 57 unmodified lines
82 unsigned long addr, pte_t *ptep);81 unsigned long addr, pte_t *ptep);
83extern void huge_ptep_clear_flush(struct vm_area_struct *vma,82extern void huge_ptep_clear_flush(struct vm_area_struct *vma,
84 unsigned long addr, pte_t *ptep);83 unsigned long addr, pte_t *ptep);
84extern void huge_pte_clear(struct mm_struct *mm, unsigned long addr,
85 pte_t *ptep, unsigned long sz);
86#define huge_pte_clear huge_pte_clear
87extern void set_huge_swap_pte_at(struct mm_struct *mm, unsigned long addr,
88 pte_t *ptep, pte_t pte, unsigned long sz);
89#define set_huge_swap_pte_at set_huge_swap_pte_at
90
91#include <asm-generic/hugetlb.h>
85 92
86#ifdef CONFIG_ARCH_HAS_GIGANTIC_PAGE93#ifdef CONFIG_ARCH_HAS_GIGANTIC_PAGE
87static inline bool gigantic_page_supported(void) { return true; }94static inline bool gigantic_page_supported(void) { return true; }
arch/arm64/include/asm/irq.h+0−42
@@ -1,45 +1,12 @@
1#ifndef __ASM_IRQ_H1#ifndef __ASM_IRQ_H
2#define __ASM_IRQ_H2#define __ASM_IRQ_H
3 3
4#define IRQ_STACK_SIZE THREAD_SIZE
5#define IRQ_STACK_START_SP THREAD_START_SP
6
7#ifndef __ASSEMBLER__4#ifndef __ASSEMBLER__
8 5
9#include <linux/percpu.h>
10
11#include <asm-generic/irq.h>6#include <asm-generic/irq.h>
12#include <asm/thread_info.h>
13 7
14struct pt_regs;8struct pt_regs;
15 9
16DECLARE_PER_CPU(unsigned long [IRQ_STACK_SIZE/sizeof(long)], irq_stack);
17
18/*
19 * The highest address on the stack, and the first to be used. Used to
20 * find the dummy-stack frame put down by el?_irq() in entry.S, which
21 * is structured as follows:
22 *
23 * ------------
24 * | | <- irq_stack_ptr
25 * top ------------
26 * | x19 | <- irq_stack_ptr - 0x08
27 * ------------
28 * | x29 | <- irq_stack_ptr - 0x10
29 * ------------
30 *
31 * where x19 holds a copy of the task stack pointer where the struct pt_regs
32 * from kernel_entry can be found.
33 *
34 */
35#define IRQ_STACK_PTR(cpu) ((unsigned long)per_cpu(irq_stack, cpu) + IRQ_STACK_START_SP)
36
37/*
38 * The offset from irq_stack_ptr where entry.S will store the original
39 * stack pointer. Used by unwind_frame() and dump_backtrace().
40 */
41#define IRQ_STACK_TO_TASK_STACK(ptr) (*((unsigned long *)((ptr) - 0x08)))
42
43extern void set_handle_irq(void (*handle_irq)(struct pt_regs *));10extern void set_handle_irq(void (*handle_irq)(struct pt_regs *));
44 11
45static inline int nr_legacy_irqs(void)12static inline int nr_legacy_irqs(void)
@@ -47,14 +14,5 @@ static inline int nr_legacy_irqs(void) · 1 unmodified line
47 return 0;14 return 0;
48}15}
49 16
50static inline bool on_irq_stack(unsigned long sp, int cpu)
51{
52 /* variable names the same as kernel/stacktrace.c */
53 unsigned long low = (unsigned long)per_cpu(irq_stack, cpu);
54 unsigned long high = low + IRQ_STACK_START_SP;
55
56 return (low <= sp && sp <= high);
57}
58
59#endif /* !__ASSEMBLER__ */17#endif /* !__ASSEMBLER__ */
60#endif18#endif
arch/arm64/include/asm/kvm_mmu.h+9−12
@@ -175,18 +175,15 @@ static inline pmd_t kvm_s2pmd_mkwrite(pmd_t pmd) · 174 unmodified lines
175 175
176static inline void kvm_set_s2pte_readonly(pte_t *pte)176static inline void kvm_set_s2pte_readonly(pte_t *pte)
177{177{
178 pteval_t pteval;178 pteval_t old_pteval, pteval;
179 unsigned long tmp;179
180 180 pteval = READ_ONCE(pte_val(*pte));
181 asm volatile("// kvm_set_s2pte_readonly\n"181 do {
182 " prfm pstl1strm, %2\n"182 old_pteval = pteval;
183 "1: ldxr %0, %2\n"183 pteval &= ~PTE_S2_RDWR;
184 " and %0, %0, %3 // clear PTE_S2_RDWR\n"184 pteval |= PTE_S2_RDONLY;
185 " orr %0, %0, %4 // set PTE_S2_RDONLY\n"185 pteval = cmpxchg_relaxed(&pte_val(*pte), old_pteval, pteval);
186 " stxr %w1, %0, %2\n"186 } while (pteval != old_pteval);
187 " cbnz %w1, 1b\n"
188 : "=&r" (pteval), "=&r" (tmp), "+Q" (pte_val(*pte))
189 : "L" (~PTE_S2_RDWR), "L" (PTE_S2_RDONLY));
190}187}
191 188
192static inline bool kvm_s2pte_readonly(pte_t *pte)189static inline bool kvm_s2pte_readonly(pte_t *pte)
arch/arm64/include/asm/memory.h+53−0
@@ -25,6 +25,7 @@ · 24 unmodified lines
25#include <linux/const.h>25#include <linux/const.h>
26#include <linux/types.h>26#include <linux/types.h>
27#include <asm/bug.h>27#include <asm/bug.h>
28#include <asm/page-def.h>
28#include <asm/sizes.h>29#include <asm/sizes.h>
29 30
30/*31/*
@@ -103,6 +104,58 @@ · 72 unmodified lines
103#define KASAN_SHADOW_SIZE (0)104#define KASAN_SHADOW_SIZE (0)
104#endif105#endif
105 106
107#define MIN_THREAD_SHIFT 14
108
109/*
110 * VMAP'd stacks are allocated at page granularity, so we must ensure that such
111 * stacks are a multiple of page size.
112 */
113#if defined(CONFIG_VMAP_STACK) && (MIN_THREAD_SHIFT < PAGE_SHIFT)
114#define THREAD_SHIFT PAGE_SHIFT
115#else
116#define THREAD_SHIFT MIN_THREAD_SHIFT
117#endif
118
119#if THREAD_SHIFT >= PAGE_SHIFT
120#define THREAD_SIZE_ORDER (THREAD_SHIFT - PAGE_SHIFT)
121#endif
122
123#define THREAD_SIZE (UL(1) << THREAD_SHIFT)
124
125/*
126 * By aligning VMAP'd stacks to 2 * THREAD_SIZE, we can detect overflow by
127 * checking sp & (1 << THREAD_SHIFT), which we can do cheaply in the entry
128 * assembly.
129 */
130#ifdef CONFIG_VMAP_STACK
131#define THREAD_ALIGN (2 * THREAD_SIZE)
132#else
133#define THREAD_ALIGN THREAD_SIZE
134#endif
135
136#define IRQ_STACK_SIZE THREAD_SIZE
137
138#define OVERFLOW_STACK_SIZE SZ_4K
139
140/*
141 * Alignment of kernel segments (e.g. .text, .data).
142 */
143#if defined(CONFIG_DEBUG_ALIGN_RODATA)
144/*
145 * 4 KB granule: 1 level 2 entry
146 * 16 KB granule: 128 level 3 entries, with contiguous bit
147 * 64 KB granule: 32 level 3 entries, with contiguous bit
148 */
149#define SEGMENT_ALIGN SZ_2M
150#else
151/*
152 * 4 KB granule: 16 level 3 entries, with contiguous bit
153 * 16 KB granule: 4 level 3 entries, without contiguous bit
154 * 64 KB granule: 1 level 3 entry
155 */
156#define SEGMENT_ALIGN SZ_64K
157#endif
158
106/*159/*
107 * Memory types available.160 * Memory types available.
108 */161 */
arch/arm64/include/asm/mmu.h+2−0
@@ -16,6 +16,8 @@ · 15 unmodified lines
16#ifndef __ASM_MMU_H16#ifndef __ASM_MMU_H
17#define __ASM_MMU_H17#define __ASM_MMU_H
18 18
19#define MMCF_AARCH32 0x1 /* mm context flag for AArch32 executables */
20
19typedef struct {21typedef struct {
20 atomic64_t id;22 atomic64_t id;
21 void *vdso;23 void *vdso;
arch/arm64/include/asm/neon.h+13−3
@@ -8,12 +8,22 @@ · 7 unmodified lines
8 * published by the Free Software Foundation.8 * published by the Free Software Foundation.
9 */9 */
10 10
11#ifndef __ASM_NEON_H
12#define __ASM_NEON_H
13
11#include <linux/types.h>14#include <linux/types.h>
12#include <asm/fpsimd.h>15#include <asm/fpsimd.h>
13 16
14#define cpu_has_neon() system_supports_fpsimd()17#define cpu_has_neon() system_supports_fpsimd()
15 18
16#define kernel_neon_begin() kernel_neon_begin_partial(32)19void kernel_neon_begin(void);
17
18void kernel_neon_begin_partial(u32 num_regs);
19void kernel_neon_end(void);20void kernel_neon_end(void);
21
22/*
23 * Temporary macro to allow the crypto code to compile. Note that the
24 * semantics of kernel_neon_begin_partial() are now different from the
25 * original as it does not allow being called in an interrupt context.
26 */
27#define kernel_neon_begin_partial(num_regs) kernel_neon_begin()
28
29#endif /* ! __ASM_NEON_H */
arch/arm64/include/asm/numa.h+0−3
@@ -7,9 +7,6 @@ · 6 unmodified lines
7 7
8#define NR_NODE_MEMBLKS (MAX_NUMNODES * 2)8#define NR_NODE_MEMBLKS (MAX_NUMNODES * 2)
9 9
10/* currently, arm64 implements flat NUMA topology */
11#define parent_node(node) (node)
12
13int __node_distance(int from, int to);10int __node_distance(int from, int to);
14#define node_distance(a, b) __node_distance(a, b)11#define node_distance(a, b) __node_distance(a, b)
15 12
arch/arm64/include/asm/page-def.h+34−0
@@ -0,0 +1,34 @@
1/*
2 * Based on arch/arm/include/asm/page.h
3 *
4 * Copyright (C) 1995-2003 Russell King
5 * Copyright (C) 2017 ARM Ltd.
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program. If not, see <http://www.gnu.org/licenses/>.
18 */
19#ifndef __ASM_PAGE_DEF_H
20#define __ASM_PAGE_DEF_H
21
22#include <linux/const.h>
23
24/* PAGE_SHIFT determines the page size */
25/* CONT_SHIFT determines the number of pages which can be tracked together */
26#define PAGE_SHIFT CONFIG_ARM64_PAGE_SHIFT
27#define CONT_SHIFT CONFIG_ARM64_CONT_SHIFT
28#define PAGE_SIZE (_AC(1, UL) << PAGE_SHIFT)
29#define PAGE_MASK (~(PAGE_SIZE-1))
30
31#define CONT_SIZE (_AC(1, UL) << (CONT_SHIFT + PAGE_SHIFT))
32#define CONT_MASK (~(CONT_SIZE-1))
33
34#endif /* __ASM_PAGE_DEF_H */
arch/arm64/include/asm/page.h+1−11
@@ -19,17 +19,7 @@ · 18 unmodified lines
19#ifndef __ASM_PAGE_H19#ifndef __ASM_PAGE_H
20#define __ASM_PAGE_H20#define __ASM_PAGE_H
21 21
22#include <linux/const.h>22#include <asm/page-def.h>
23
24/* PAGE_SHIFT determines the page size */
25/* CONT_SHIFT determines the number of pages which can be tracked together */
26#define PAGE_SHIFT CONFIG_ARM64_PAGE_SHIFT
27#define CONT_SHIFT CONFIG_ARM64_CONT_SHIFT
28#define PAGE_SIZE (_AC(1, UL) << PAGE_SHIFT)
29#define PAGE_MASK (~(PAGE_SIZE-1))
30
31#define CONT_SIZE (_AC(1, UL) << (CONT_SHIFT + PAGE_SHIFT))
32#define CONT_MASK (~(CONT_SIZE-1))
33 23
34#ifndef __ASSEMBLY__24#ifndef __ASSEMBLY__
35 25
arch/arm64/include/asm/pgtable-prot.h+8−10
@@ -63,23 +63,21 @@ · 62 unmodified lines
63#define PAGE_S2 __pgprot(PROT_DEFAULT | PTE_S2_MEMATTR(MT_S2_NORMAL) | PTE_S2_RDONLY)63#define PAGE_S2 __pgprot(PROT_DEFAULT | PTE_S2_MEMATTR(MT_S2_NORMAL) | PTE_S2_RDONLY)
64#define PAGE_S2_DEVICE __pgprot(PROT_DEFAULT | PTE_S2_MEMATTR(MT_S2_DEVICE_nGnRE) | PTE_S2_RDONLY | PTE_UXN)64#define PAGE_S2_DEVICE __pgprot(PROT_DEFAULT | PTE_S2_MEMATTR(MT_S2_DEVICE_nGnRE) | PTE_S2_RDONLY | PTE_UXN)
65 65
66#define PAGE_NONE __pgprot(((_PAGE_DEFAULT) & ~PTE_VALID) | PTE_PROT_NONE | PTE_PXN | PTE_UXN)66#define PAGE_NONE __pgprot(((_PAGE_DEFAULT) & ~PTE_VALID) | PTE_PROT_NONE | PTE_RDONLY | PTE_PXN | PTE_UXN)
67#define PAGE_SHARED __pgprot(_PAGE_DEFAULT | PTE_USER | PTE_NG | PTE_PXN | PTE_UXN | PTE_WRITE)67#define PAGE_SHARED __pgprot(_PAGE_DEFAULT | PTE_USER | PTE_NG | PTE_PXN | PTE_UXN | PTE_WRITE)
68#define PAGE_SHARED_EXEC __pgprot(_PAGE_DEFAULT | PTE_USER | PTE_NG | PTE_PXN | PTE_WRITE)68#define PAGE_SHARED_EXEC __pgprot(_PAGE_DEFAULT | PTE_USER | PTE_NG | PTE_PXN | PTE_WRITE)
69#define PAGE_COPY __pgprot(_PAGE_DEFAULT | PTE_USER | PTE_NG | PTE_PXN | PTE_UXN)69#define PAGE_READONLY __pgprot(_PAGE_DEFAULT | PTE_USER | PTE_RDONLY | PTE_NG | PTE_PXN | PTE_UXN)
70#define PAGE_COPY_EXEC __pgprot(_PAGE_DEFAULT | PTE_USER | PTE_NG | PTE_PXN)70#define PAGE_READONLY_EXEC __pgprot(_PAGE_DEFAULT | PTE_USER | PTE_RDONLY | PTE_NG | PTE_PXN)
71#define PAGE_READONLY __pgprot(_PAGE_DEFAULT | PTE_USER | PTE_NG | PTE_PXN | PTE_UXN)71#define PAGE_EXECONLY __pgprot(_PAGE_DEFAULT | PTE_RDONLY | PTE_NG | PTE_PXN)
72#define PAGE_READONLY_EXEC __pgprot(_PAGE_DEFAULT | PTE_USER | PTE_NG | PTE_PXN)
73#define PAGE_EXECONLY __pgprot(_PAGE_DEFAULT | PTE_NG | PTE_PXN)
74 72
75#define __P000 PAGE_NONE73#define __P000 PAGE_NONE
76#define __P001 PAGE_READONLY74#define __P001 PAGE_READONLY
77#define __P010 PAGE_COPY75#define __P010 PAGE_READONLY
78#define __P011 PAGE_COPY76#define __P011 PAGE_READONLY
79#define __P100 PAGE_EXECONLY77#define __P100 PAGE_EXECONLY
80#define __P101 PAGE_READONLY_EXEC78#define __P101 PAGE_READONLY_EXEC
81#define __P110 PAGE_COPY_EXEC79#define __P110 PAGE_READONLY_EXEC
82#define __P111 PAGE_COPY_EXEC80#define __P111 PAGE_READONLY_EXEC
83 81
84#define __S000 PAGE_NONE82#define __S000 PAGE_NONE
85#define __S001 PAGE_READONLY83#define __S001 PAGE_READONLY
arch/arm64/include/asm/pgtable.h+41−62
@@ -39,6 +39,7 @@ · 38 unmodified lines
39 39
40#ifndef __ASSEMBLY__40#ifndef __ASSEMBLY__
41 41
42#include <asm/cmpxchg.h>
42#include <asm/fixmap.h>43#include <asm/fixmap.h>
43#include <linux/mmdebug.h>44#include <linux/mmdebug.h>
44 45
@@ -84,11 +85,7 @@ extern unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)]; · 39 unmodified lines
84 (__boundary - 1 < (end) - 1) ? __boundary : (end); \85 (__boundary - 1 < (end) - 1) ? __boundary : (end); \
85})86})
86 87
87#ifdef CONFIG_ARM64_HW_AFDBM
88#define pte_hw_dirty(pte) (pte_write(pte) && !(pte_val(pte) & PTE_RDONLY))88#define pte_hw_dirty(pte) (pte_write(pte) && !(pte_val(pte) & PTE_RDONLY))
89#else
90#define pte_hw_dirty(pte) (0)
91#endif
92#define pte_sw_dirty(pte) (!!(pte_val(pte) & PTE_DIRTY))89#define pte_sw_dirty(pte) (!!(pte_val(pte) & PTE_DIRTY))
93#define pte_dirty(pte) (pte_sw_dirty(pte) || pte_hw_dirty(pte))90#define pte_dirty(pte) (pte_sw_dirty(pte) || pte_hw_dirty(pte))
94 91
@@ -124,12 +121,16 @@ static inline pte_t set_pte_bit(pte_t pte, pgprot_t prot) · 29 unmodified lines
124 121
125static inline pte_t pte_wrprotect(pte_t pte)122static inline pte_t pte_wrprotect(pte_t pte)
126{123{
127 return clear_pte_bit(pte, __pgprot(PTE_WRITE));124 pte = clear_pte_bit(pte, __pgprot(PTE_WRITE));
125 pte = set_pte_bit(pte, __pgprot(PTE_RDONLY));
126 return pte;
128}127}
129 128
130static inline pte_t pte_mkwrite(pte_t pte)129static inline pte_t pte_mkwrite(pte_t pte)
131{130{
132 return set_pte_bit(pte, __pgprot(PTE_WRITE));131 pte = set_pte_bit(pte, __pgprot(PTE_WRITE));
132 pte = clear_pte_bit(pte, __pgprot(PTE_RDONLY));
133 return pte;
133}134}
134 135
135static inline pte_t pte_mkclean(pte_t pte)136static inline pte_t pte_mkclean(pte_t pte)
@@ -168,11 +169,6 @@ static inline pte_t pte_mknoncont(pte_t pte) · 32 unmodified lines
168 return clear_pte_bit(pte, __pgprot(PTE_CONT));169 return clear_pte_bit(pte, __pgprot(PTE_CONT));
169}170}
170 171
171static inline pte_t pte_clear_rdonly(pte_t pte)
172{
173 return clear_pte_bit(pte, __pgprot(PTE_RDONLY));
174}
175
176static inline pte_t pte_mkpresent(pte_t pte)172static inline pte_t pte_mkpresent(pte_t pte)
177{173{
178 return set_pte_bit(pte, __pgprot(PTE_VALID));174 return set_pte_bit(pte, __pgprot(PTE_VALID));
@@ -220,22 +216,15 @@ extern void __sync_icache_dcache(pte_t pteval, unsigned long addr); · 41 unmodified lines
220static inline void set_pte_at(struct mm_struct *mm, unsigned long addr,216static inline void set_pte_at(struct mm_struct *mm, unsigned long addr,
221 pte_t *ptep, pte_t pte)217 pte_t *ptep, pte_t pte)
222{218{
223 if (pte_present(pte)) {219 if (pte_present(pte) && pte_user_exec(pte) && !pte_special(pte))
224 if (pte_sw_dirty(pte) && pte_write(pte))220 __sync_icache_dcache(pte, addr);
225 pte_val(pte) &= ~PTE_RDONLY;
226 else
227 pte_val(pte) |= PTE_RDONLY;
228 if (pte_user_exec(pte) && !pte_special(pte))
229 __sync_icache_dcache(pte, addr);
230 }
231 221
232 /*222 /*
233 * If the existing pte is valid, check for potential race with223 * If the existing pte is valid, check for potential race with
234 * hardware updates of the pte (ptep_set_access_flags safely changes224 * hardware updates of the pte (ptep_set_access_flags safely changes
235 * valid ptes without going through an invalid entry).225 * valid ptes without going through an invalid entry).
236 */226 */
237 if (IS_ENABLED(CONFIG_ARM64_HW_AFDBM) &&227 if (pte_valid(*ptep) && pte_valid(pte)) {
238 pte_valid(*ptep) && pte_valid(pte)) {
239 VM_WARN_ONCE(!pte_young(pte),228 VM_WARN_ONCE(!pte_young(pte),
240 "%s: racy access flag clearing: 0x%016llx -> 0x%016llx",229 "%s: racy access flag clearing: 0x%016llx -> 0x%016llx",
241 __func__, pte_val(*ptep), pte_val(pte));230 __func__, pte_val(*ptep), pte_val(pte));
@@ -571,7 +560,6 @@ static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot) · 329 unmodified lines
571 return pte_pmd(pte_modify(pmd_pte(pmd), newprot));560 return pte_pmd(pte_modify(pmd_pte(pmd), newprot));
572}561}
573 562
574#ifdef CONFIG_ARM64_HW_AFDBM
575#define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS563#define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS
576extern int ptep_set_access_flags(struct vm_area_struct *vma,564extern int ptep_set_access_flags(struct vm_area_struct *vma,
577 unsigned long address, pte_t *ptep,565 unsigned long address, pte_t *ptep,
@@ -593,20 +581,17 @@ static inline int pmdp_set_access_flags(struct vm_area_struct *vma, · 15 unmodified lines
593#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG581#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
594static inline int __ptep_test_and_clear_young(pte_t *ptep)582static inline int __ptep_test_and_clear_young(pte_t *ptep)
595{583{
596 pteval_t pteval;584 pte_t old_pte, pte;
597 unsigned int tmp, res;
598 585
599 asm volatile("// __ptep_test_and_clear_young\n"586 pte = READ_ONCE(*ptep);
600 " prfm pstl1strm, %2\n"587 do {
601 "1: ldxr %0, %2\n"588 old_pte = pte;
602 " ubfx %w3, %w0, %5, #1 // extract PTE_AF (young)\n"589 pte = pte_mkold(pte);
603 " and %0, %0, %4 // clear PTE_AF\n"590 pte_val(pte) = cmpxchg_relaxed(&pte_val(*ptep),
604 " stxr %w1, %0, %2\n"591 pte_val(old_pte), pte_val(pte));
605 " cbnz %w1, 1b\n"592 } while (pte_val(pte) != pte_val(old_pte));
606 : "=&r" (pteval), "=&r" (tmp), "+Q" (pte_val(*ptep)), "=&r" (res)
607 : "L" (~PTE_AF), "I" (ilog2(PTE_AF)));
608 593
609 return res;594 return pte_young(pte);
610}595}
611 596
612static inline int ptep_test_and_clear_young(struct vm_area_struct *vma,597static inline int ptep_test_and_clear_young(struct vm_area_struct *vma,
@@ -630,17 +615,7 @@ static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma, · 17 unmodified lines
630static inline pte_t ptep_get_and_clear(struct mm_struct *mm,615static inline pte_t ptep_get_and_clear(struct mm_struct *mm,
631 unsigned long address, pte_t *ptep)616 unsigned long address, pte_t *ptep)
632{617{
633 pteval_t old_pteval;618 return __pte(xchg_relaxed(&pte_val(*ptep), 0));
634 unsigned int tmp;
635
636 asm volatile("// ptep_get_and_clear\n"
637 " prfm pstl1strm, %2\n"
638 "1: ldxr %0, %2\n"
639 " stxr %w1, xzr, %2\n"
640 " cbnz %w1, 1b\n"
641 : "=&r" (old_pteval), "=&r" (tmp), "+Q" (pte_val(*ptep)));
642
643 return __pte(old_pteval);
644}619}
645 620
646#ifdef CONFIG_TRANSPARENT_HUGEPAGE621#ifdef CONFIG_TRANSPARENT_HUGEPAGE
@@ -653,27 +628,32 @@ static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm, · 6 unmodified lines
653#endif /* CONFIG_TRANSPARENT_HUGEPAGE */628#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
654 629
655/*630/*
656 * ptep_set_wrprotect - mark read-only while trasferring potential hardware631 * ptep_set_wrprotect - mark read-only while preserving the hardware update of
657 * dirty status (PTE_DBM && !PTE_RDONLY) to the software PTE_DIRTY bit.632 * the Access Flag.
658 */633 */
659#define __HAVE_ARCH_PTEP_SET_WRPROTECT634#define __HAVE_ARCH_PTEP_SET_WRPROTECT
660static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long address, pte_t *ptep)635static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long address, pte_t *ptep)
661{636{
662 pteval_t pteval;637 pte_t old_pte, pte;
663 unsigned long tmp;
664 638
665 asm volatile("// ptep_set_wrprotect\n"639 /*
666 " prfm pstl1strm, %2\n"640 * ptep_set_wrprotect() is only called on CoW mappings which are
667 "1: ldxr %0, %2\n"641 * private (!VM_SHARED) with the pte either read-only (!PTE_WRITE &&
668 " tst %0, %4 // check for hw dirty (!PTE_RDONLY)\n"642 * PTE_RDONLY) or writable and software-dirty (PTE_WRITE &&
669 " csel %1, %3, xzr, eq // set PTE_DIRTY|PTE_RDONLY if dirty\n"643 * !PTE_RDONLY && PTE_DIRTY); see is_cow_mapping() and
670 " orr %0, %0, %1 // if !dirty, PTE_RDONLY is already set\n"644 * protection_map[]. There is no race with the hardware update of the
671 " and %0, %0, %5 // clear PTE_WRITE/PTE_DBM\n"645 * dirty state: clearing of PTE_RDONLY when PTE_WRITE (a.k.a. PTE_DBM)
672 " stxr %w1, %0, %2\n"646 * is set.
673 " cbnz %w1, 1b\n"647 */
674 : "=&r" (pteval), "=&r" (tmp), "+Q" (pte_val(*ptep))648 VM_WARN_ONCE(pte_write(*ptep) && !pte_dirty(*ptep),
675 : "r" (PTE_DIRTY|PTE_RDONLY), "L" (PTE_RDONLY), "L" (~PTE_WRITE)649 "%s: potential race with hardware DBM", __func__);
676 : "cc");650 pte = READ_ONCE(*ptep);
651 do {
652 old_pte = pte;
653 pte = pte_wrprotect(pte);
654 pte_val(pte) = cmpxchg_relaxed(&pte_val(*ptep),
655 pte_val(old_pte), pte_val(pte));
656 } while (pte_val(pte) != pte_val(old_pte));
677}657}
678 658
679#ifdef CONFIG_TRANSPARENT_HUGEPAGE659#ifdef CONFIG_TRANSPARENT_HUGEPAGE
@@ -684,7 +664,6 @@ static inline void pmdp_set_wrprotect(struct mm_struct *mm, · 4 unmodified lines
684 ptep_set_wrprotect(mm, address, (pte_t *)pmdp);664 ptep_set_wrprotect(mm, address, (pte_t *)pmdp);
685}665}
686#endif666#endif
687#endif /* CONFIG_ARM64_HW_AFDBM */
688 667
689extern pgd_t swapper_pg_dir[PTRS_PER_PGD];668extern pgd_t swapper_pg_dir[PTRS_PER_PGD];
690extern pgd_t idmap_pg_dir[PTRS_PER_PGD];669extern pgd_t idmap_pg_dir[PTRS_PER_PGD];
arch/arm64/include/asm/processor.h+2−2
@@ -112,7 +112,7 @@ void tls_preserve_current_state(void); · 111 unmodified lines
112static inline void start_thread_common(struct pt_regs *regs, unsigned long pc)112static inline void start_thread_common(struct pt_regs *regs, unsigned long pc)
113{113{
114 memset(regs, 0, sizeof(*regs));114 memset(regs, 0, sizeof(*regs));
115 regs->syscallno = ~0UL;115 forget_syscall(regs);
116 regs->pc = pc;116 regs->pc = pc;
117}117}
118 118
@@ -159,7 +159,7 @@ extern struct task_struct *cpu_switch_to(struct task_struct *prev, · 40 unmodified lines
159 struct task_struct *next);159 struct task_struct *next);
160 160
161#define task_pt_regs(p) \161#define task_pt_regs(p) \
162 ((struct pt_regs *)(THREAD_START_SP + task_stack_page(p)) - 1)162 ((struct pt_regs *)(THREAD_SIZE + task_stack_page(p)) - 1)
163 163
164#define KSTK_EIP(tsk) ((unsigned long)task_pt_regs(tsk)->pc)164#define KSTK_EIP(tsk) ((unsigned long)task_pt_regs(tsk)->pc)
165#define KSTK_ESP(tsk) user_stack_pointer(task_pt_regs(tsk))165#define KSTK_ESP(tsk) user_stack_pointer(task_pt_regs(tsk))
arch/arm64/include/asm/ptrace.h+30−1
@@ -72,8 +72,19 @@ · 71 unmodified lines
72#define COMPAT_PT_TEXT_ADDR 0x1000072#define COMPAT_PT_TEXT_ADDR 0x10000
73#define COMPAT_PT_DATA_ADDR 0x1000473#define COMPAT_PT_DATA_ADDR 0x10004
74#define COMPAT_PT_TEXT_END_ADDR 0x1000874#define COMPAT_PT_TEXT_END_ADDR 0x10008
75
76/*
77 * If pt_regs.syscallno == NO_SYSCALL, then the thread is not executing
78 * a syscall -- i.e., its most recent entry into the kernel from
79 * userspace was not via SVC, or otherwise a tracer cancelled the syscall.
80 *
81 * This must have the value -1, for ABI compatibility with ptrace etc.
82 */
83#define NO_SYSCALL (-1)
84
75#ifndef __ASSEMBLY__85#ifndef __ASSEMBLY__
76#include <linux/bug.h>86#include <linux/bug.h>
87#include <linux/types.h>
77 88
78/* sizeof(struct user) for AArch32 */89/* sizeof(struct user) for AArch32 */
79#define COMPAT_USER_SZ 29690#define COMPAT_USER_SZ 296
@@ -116,11 +127,29 @@ struct pt_regs { · 36 unmodified lines
116 };127 };
117 };128 };
118 u64 orig_x0;129 u64 orig_x0;
119 u64 syscallno;130#ifdef __AARCH64EB__
131 u32 unused2;
132 s32 syscallno;
133#else
134 s32 syscallno;
135 u32 unused2;
136#endif
137
120 u64 orig_addr_limit;138 u64 orig_addr_limit;
121 u64 unused; // maintain 16 byte alignment139 u64 unused; // maintain 16 byte alignment
140 u64 stackframe[2];
122};141};
123 142
143static inline bool in_syscall(struct pt_regs const *regs)
144{
145 return regs->syscallno != NO_SYSCALL;
146}
147
148static inline void forget_syscall(struct pt_regs *regs)
149{
150 regs->syscallno = NO_SYSCALL;
151}
152
124#define MAX_REG_OFFSET offsetof(struct pt_regs, pstate)153#define MAX_REG_OFFSET offsetof(struct pt_regs, pstate)
125 154
126#define arch_has_single_step() (1)155#define arch_has_single_step() (1)
arch/arm64/include/asm/signal32.h+0−2
@@ -22,8 +22,6 @@ · 21 unmodified lines
22 22
23#define AARCH32_KERN_SIGRET_CODE_OFFSET 0x50023#define AARCH32_KERN_SIGRET_CODE_OFFSET 0x500
24 24
25extern const compat_ulong_t aarch32_sigret_code[6];
26
27int compat_setup_frame(int usig, struct ksignal *ksig, sigset_t *set,25int compat_setup_frame(int usig, struct ksignal *ksig, sigset_t *set,
28 struct pt_regs *regs);26 struct pt_regs *regs);
29int compat_setup_rt_frame(int usig, struct ksignal *ksig, sigset_t *set,27int compat_setup_rt_frame(int usig, struct ksignal *ksig, sigset_t *set,
arch/arm64/include/asm/simd.h+56−0
@@ -0,0 +1,56 @@
1/*
2 * Copyright (C) 2017 Linaro Ltd. <ard.biesheuvel@linaro.org>
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License version 2 as published
6 * by the Free Software Foundation.
7 */
8
9#ifndef __ASM_SIMD_H
10#define __ASM_SIMD_H
11
12#include <linux/compiler.h>
13#include <linux/irqflags.h>
14#include <linux/percpu.h>
15#include <linux/preempt.h>
16#include <linux/types.h>
17
18#ifdef CONFIG_KERNEL_MODE_NEON
19
20DECLARE_PER_CPU(bool, kernel_neon_busy);
21
22/*
23 * may_use_simd - whether it is allowable at this time to issue SIMD
24 * instructions or access the SIMD register file
25 *
26 * Callers must not assume that the result remains true beyond the next
27 * preempt_enable() or return from softirq context.
28 */
29static __must_check inline bool may_use_simd(void)
30{
31 /*
32 * The raw_cpu_read() is racy if called with preemption enabled.
33 * This is not a bug: kernel_neon_busy is only set when
34 * preemption is disabled, so we cannot migrate to another CPU
35 * while it is set, nor can we migrate to a CPU where it is set.
36 * So, if we find it clear on some CPU then we're guaranteed to
37 * find it clear on any CPU we could migrate to.
38 *
39 * If we are in between kernel_neon_begin()...kernel_neon_end(),
40 * the flag will be set, but preemption is also disabled, so we
41 * can't migrate to another CPU and spuriously see it become
42 * false.
43 */
44 return !in_irq() && !irqs_disabled() && !in_nmi() &&
45 !raw_cpu_read(kernel_neon_busy);
46}
47
48#else /* ! CONFIG_KERNEL_MODE_NEON */
49
50static __must_check inline bool may_use_simd(void) {
51 return false;
52}
53
54#endif /* ! CONFIG_KERNEL_MODE_NEON */
55
56#endif
arch/arm64/include/asm/smp.h+1−1
@@ -148,7 +148,7 @@ static inline void cpu_panic_kernel(void) · 147 unmodified lines
148 */148 */
149bool cpus_are_stuck_in_kernel(void);149bool cpus_are_stuck_in_kernel(void);
150 150
151extern void smp_send_crash_stop(void);151extern void crash_smp_send_stop(void);
152extern bool smp_crash_stop_failed(void);152extern bool smp_crash_stop_failed(void);
153 153
154#endif /* ifndef __ASSEMBLY__ */154#endif /* ifndef __ASSEMBLY__ */
arch/arm64/include/asm/stacktrace.h+59−2
@@ -16,11 +16,15 @@ · 15 unmodified lines
16#ifndef __ASM_STACKTRACE_H16#ifndef __ASM_STACKTRACE_H
17#define __ASM_STACKTRACE_H17#define __ASM_STACKTRACE_H
18 18
19struct task_struct;19#include <linux/percpu.h>
20#include <linux/sched.h>
21#include <linux/sched/task_stack.h>
22
23#include <asm/memory.h>
24#include <asm/ptrace.h>
20 25
21struct stackframe {26struct stackframe {
22 unsigned long fp;27 unsigned long fp;
23 unsigned long sp;
24 unsigned long pc;28 unsigned long pc;
25#ifdef CONFIG_FUNCTION_GRAPH_TRACER29#ifdef CONFIG_FUNCTION_GRAPH_TRACER
26 unsigned int graph;30 unsigned int graph;
@@ -32,4 +36,57 @@ extern void walk_stackframe(struct task_struct *tsk, struct stackframe *frame, · 5 unmodified lines
32 int (*fn)(struct stackframe *, void *), void *data);36 int (*fn)(struct stackframe *, void *), void *data);
33extern void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk);37extern void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk);
34 38
39DECLARE_PER_CPU(unsigned long *, irq_stack_ptr);
40
41static inline bool on_irq_stack(unsigned long sp)
42{
43 unsigned long low = (unsigned long)raw_cpu_read(irq_stack_ptr);
44 unsigned long high = low + IRQ_STACK_SIZE;
45
46 if (!low)
47 return false;
48
49 return (low <= sp && sp < high);
50}
51
52static inline bool on_task_stack(struct task_struct *tsk, unsigned long sp)
53{
54 unsigned long low = (unsigned long)task_stack_page(tsk);
55 unsigned long high = low + THREAD_SIZE;
56
57 return (low <= sp && sp < high);
58}
59
60#ifdef CONFIG_VMAP_STACK
61DECLARE_PER_CPU(unsigned long [OVERFLOW_STACK_SIZE/sizeof(long)], overflow_stack);
62
63static inline bool on_overflow_stack(unsigned long sp)
64{
65 unsigned long low = (unsigned long)raw_cpu_ptr(overflow_stack);
66 unsigned long high = low + OVERFLOW_STACK_SIZE;
67
68 return (low <= sp && sp < high);
69}
70#else
71static inline bool on_overflow_stack(unsigned long sp) { return false; }
72#endif
73
74/*
75 * We can only safely access per-cpu stacks from current in a non-preemptible
76 * context.
77 */
78static inline bool on_accessible_stack(struct task_struct *tsk, unsigned long sp)
79{
80 if (on_task_stack(tsk, sp))
81 return true;
82 if (tsk != current || preemptible())
83 return false;
84 if (on_irq_stack(sp))
85 return true;
86 if (on_overflow_stack(sp))
87 return true;
88
89 return false;
90}
91
35#endif /* __ASM_STACKTRACE_H */92#endif /* __ASM_STACKTRACE_H */
arch/arm64/include/asm/string.h+4−0
@@ -52,6 +52,10 @@ extern void *__memset(void *, int, __kernel_size_t); · 51 unmodified lines
52#define __HAVE_ARCH_MEMCMP52#define __HAVE_ARCH_MEMCMP
53extern int memcmp(const void *, const void *, size_t);53extern int memcmp(const void *, const void *, size_t);
54 54
55#ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE
56#define __HAVE_ARCH_MEMCPY_FLUSHCACHE
57void memcpy_flushcache(void *dst, const void *src, size_t cnt);
58#endif
55 59
56#if defined(CONFIG_KASAN) && !defined(__SANITIZE_ADDRESS__)60#if defined(CONFIG_KASAN) && !defined(__SANITIZE_ADDRESS__)
57 61
arch/arm64/include/asm/sysreg.h+1−0
@@ -329,6 +329,7 @@ · 328 unmodified lines
329#define ID_AA64ISAR1_LRCPC_SHIFT 20329#define ID_AA64ISAR1_LRCPC_SHIFT 20
330#define ID_AA64ISAR1_FCMA_SHIFT 16330#define ID_AA64ISAR1_FCMA_SHIFT 16
331#define ID_AA64ISAR1_JSCVT_SHIFT 12331#define ID_AA64ISAR1_JSCVT_SHIFT 12
332#define ID_AA64ISAR1_DPB_SHIFT 0
332 333
333/* id_aa64pfr0 */334/* id_aa64pfr0 */
334#define ID_AA64PFR0_GIC_SHIFT 24335#define ID_AA64PFR0_GIC_SHIFT 24
arch/arm64/include/asm/thread_info.h+4−9
@@ -23,19 +23,11 @@ · 22 unmodified lines
23 23
24#include <linux/compiler.h>24#include <linux/compiler.h>
25 25
26#ifdef CONFIG_ARM64_4K_PAGES
27#define THREAD_SIZE_ORDER 2
28#elif defined(CONFIG_ARM64_16K_PAGES)
29#define THREAD_SIZE_ORDER 0
30#endif
31
32#define THREAD_SIZE 16384
33#define THREAD_START_SP (THREAD_SIZE - 16)
34
35#ifndef __ASSEMBLY__26#ifndef __ASSEMBLY__
36 27
37struct task_struct;28struct task_struct;
38 29
30#include <asm/memory.h>
39#include <asm/stack_pointer.h>31#include <asm/stack_pointer.h>
40#include <asm/types.h>32#include <asm/types.h>
41 33
@@ -68,6 +60,9 @@ struct thread_info { · 26 unmodified lines
68#define thread_saved_fp(tsk) \60#define thread_saved_fp(tsk) \
69 ((unsigned long)(tsk->thread.cpu_context.fp))61 ((unsigned long)(tsk->thread.cpu_context.fp))
70 62
63void arch_setup_new_exec(void);
64#define arch_setup_new_exec arch_setup_new_exec
65
71#endif66#endif
72 67
73/*68/*
arch/arm64/include/asm/traps.h+5−0
@@ -53,4 +53,9 @@ static inline int in_exception_text(unsigned long ptr) · 52 unmodified lines
53 return in ? : __in_irqentry_text(ptr);53 return in ? : __in_irqentry_text(ptr);
54}54}
55 55
56static inline int in_entry_text(unsigned long ptr)
57{
58 return ptr >= (unsigned long)&__entry_text_start &&
59 ptr < (unsigned long)&__entry_text_end;
60}
56#endif61#endif
arch/arm64/include/asm/uaccess.h+12−0
@@ -350,4 +350,16 @@ extern long strncpy_from_user(char *dest, const char __user *src, long count); · 349 unmodified lines
350 350
351extern __must_check long strnlen_user(const char __user *str, long n);351extern __must_check long strnlen_user(const char __user *str, long n);
352 352
353#ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE
354struct page;
355void memcpy_page_flushcache(char *to, struct page *page, size_t offset, size_t len);
356extern unsigned long __must_check __copy_user_flushcache(void *to, const void __user *from, unsigned long n);
357
358static inline int __copy_from_user_flushcache(void *dst, const void __user *src, unsigned size)
359{
360 kasan_check_write(dst, size);
361 return __copy_user_flushcache(dst, src, size);
362}
363#endif
364
353#endif /* __ASM_UACCESS_H */365#endif /* __ASM_UACCESS_H */
arch/arm64/include/uapi/asm/hwcap.h+1−0
@@ -35,5 +35,6 @@ · 34 unmodified lines
35#define HWCAP_JSCVT (1 << 13)35#define HWCAP_JSCVT (1 << 13)
36#define HWCAP_FCMA (1 << 14)36#define HWCAP_FCMA (1 << 14)
37#define HWCAP_LRCPC (1 << 15)37#define HWCAP_LRCPC (1 << 15)
38#define HWCAP_DCPOP (1 << 16)
38 39
39#endif /* _UAPI__ASM_HWCAP_H */40#endif /* _UAPI__ASM_HWCAP_H */
arch/arm64/kernel/asm-offsets.c+1−0
@@ -75,6 +75,7 @@ int main(void) · 74 unmodified lines
75 DEFINE(S_ORIG_X0, offsetof(struct pt_regs, orig_x0));75 DEFINE(S_ORIG_X0, offsetof(struct pt_regs, orig_x0));
76 DEFINE(S_SYSCALLNO, offsetof(struct pt_regs, syscallno));76 DEFINE(S_SYSCALLNO, offsetof(struct pt_regs, syscallno));
77 DEFINE(S_ORIG_ADDR_LIMIT, offsetof(struct pt_regs, orig_addr_limit));77 DEFINE(S_ORIG_ADDR_LIMIT, offsetof(struct pt_regs, orig_addr_limit));
78 DEFINE(S_STACKFRAME, offsetof(struct pt_regs, stackframe));
78 DEFINE(S_FRAME_SIZE, sizeof(struct pt_regs));79 DEFINE(S_FRAME_SIZE, sizeof(struct pt_regs));
79 BLANK();80 BLANK();
80 DEFINE(MM_CONTEXT_ID, offsetof(struct mm_struct, context.id.counter));81 DEFINE(MM_CONTEXT_ID, offsetof(struct mm_struct, context.id.counter));
arch/arm64/kernel/cpufeature.c+13−0
@@ -120,6 +120,7 @@ static const struct arm64_ftr_bits ftr_id_aa64isar1[] = { · 119 unmodified lines
120 ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64ISAR1_LRCPC_SHIFT, 4, 0),120 ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64ISAR1_LRCPC_SHIFT, 4, 0),
121 ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64ISAR1_FCMA_SHIFT, 4, 0),121 ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64ISAR1_FCMA_SHIFT, 4, 0),
122 ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64ISAR1_JSCVT_SHIFT, 4, 0),122 ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64ISAR1_JSCVT_SHIFT, 4, 0),
123 ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64ISAR1_DPB_SHIFT, 4, 0),
123 ARM64_FTR_END,124 ARM64_FTR_END,
124};125};
125 126
@@ -888,6 +889,17 @@ static const struct arm64_cpu_capabilities arm64_features[] = { · 762 unmodified lines
888 .min_field_value = 0,889 .min_field_value = 0,
889 .matches = has_no_fpsimd,890 .matches = has_no_fpsimd,
890 },891 },
892#ifdef CONFIG_ARM64_PMEM
893 {
894 .desc = "Data cache clean to Point of Persistence",
895 .capability = ARM64_HAS_DCPOP,
896 .def_scope = SCOPE_SYSTEM,
897 .matches = has_cpuid_feature,
898 .sys_reg = SYS_ID_AA64ISAR1_EL1,
899 .field_pos = ID_AA64ISAR1_DPB_SHIFT,
900 .min_field_value = 1,
901 },
902#endif
891 {},903 {},
892};904};
893 905
@@ -916,6 +928,7 @@ static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = { · 22 unmodified lines
916 HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_FP_SHIFT, FTR_SIGNED, 1, CAP_HWCAP, HWCAP_FPHP),928 HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_FP_SHIFT, FTR_SIGNED, 1, CAP_HWCAP, HWCAP_FPHP),
917 HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_ASIMD_SHIFT, FTR_SIGNED, 0, CAP_HWCAP, HWCAP_ASIMD),929 HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_ASIMD_SHIFT, FTR_SIGNED, 0, CAP_HWCAP, HWCAP_ASIMD),
918 HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_ASIMD_SHIFT, FTR_SIGNED, 1, CAP_HWCAP, HWCAP_ASIMDHP),930 HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_ASIMD_SHIFT, FTR_SIGNED, 1, CAP_HWCAP, HWCAP_ASIMDHP),
931 HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_DPB_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_DCPOP),
919 HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_JSCVT_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_JSCVT),932 HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_JSCVT_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_JSCVT),
920 HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_FCMA_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_FCMA),933 HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_FCMA_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_FCMA),
921 HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_LRCPC_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_LRCPC),934 HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_LRCPC_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_LRCPC),
arch/arm64/kernel/cpuinfo.c+1−0
@@ -68,6 +68,7 @@ static const char *const hwcap_str[] = { · 67 unmodified lines
68 "jscvt",68 "jscvt",
69 "fcma",69 "fcma",
70 "lrcpc",70 "lrcpc",
71 "dcpop",
71 NULL72 NULL
72};73};
73 74
arch/arm64/kernel/entry-fpsimd.S+0−24
@@ -41,27 +41,3 @@ ENTRY(fpsimd_load_state) · 40 unmodified lines
41 fpsimd_restore x0, 841 fpsimd_restore x0, 8
42 ret42 ret
43ENDPROC(fpsimd_load_state)43ENDPROC(fpsimd_load_state)
44
45#ifdef CONFIG_KERNEL_MODE_NEON
46
47/*
48 * Save the bottom n FP registers.
49 *
50 * x0 - pointer to struct fpsimd_partial_state
51 */
52ENTRY(fpsimd_save_partial_state)
53 fpsimd_save_partial x0, 1, 8, 9
54 ret
55ENDPROC(fpsimd_save_partial_state)
56
57/*
58 * Load the bottom n FP registers.
59 *
60 * x0 - pointer to struct fpsimd_partial_state
61 */
62ENTRY(fpsimd_load_partial_state)
63 fpsimd_restore_partial x0, 8, 9
64 ret
65ENDPROC(fpsimd_load_partial_state)
66
67#endif
arch/arm64/kernel/entry.S+183−100
@@ -69,8 +69,55 @@ · 68 unmodified lines
69#define BAD_FIQ 269#define BAD_FIQ 2
70#define BAD_ERROR 370#define BAD_ERROR 3
71 71
72 .macro kernel_entry, el, regsize = 6472 .macro kernel_ventry label
73 .align 7
73 sub sp, sp, #S_FRAME_SIZE74 sub sp, sp, #S_FRAME_SIZE
75#ifdef CONFIG_VMAP_STACK
76 /*
77 * Test whether the SP has overflowed, without corrupting a GPR.
78 * Task and IRQ stacks are aligned to (1 << THREAD_SHIFT).
79 */
80 add sp, sp, x0 // sp' = sp + x0
81 sub x0, sp, x0 // x0' = sp' - x0 = (sp + x0) - x0 = sp
82 tbnz x0, #THREAD_SHIFT, 0f
83 sub x0, sp, x0 // x0'' = sp' - x0' = (sp + x0) - sp = x0
84 sub sp, sp, x0 // sp'' = sp' - x0 = (sp + x0) - x0 = sp
85 b \label
86
870:
88 /*
89 * Either we've just detected an overflow, or we've taken an exception
90 * while on the overflow stack. Either way, we won't return to
91 * userspace, and can clobber EL0 registers to free up GPRs.
92 */
93
94 /* Stash the original SP (minus S_FRAME_SIZE) in tpidr_el0. */
95 msr tpidr_el0, x0
96
97 /* Recover the original x0 value and stash it in tpidrro_el0 */
98 sub x0, sp, x0
99 msr tpidrro_el0, x0
100
101 /* Switch to the overflow stack */
102 adr_this_cpu sp, overflow_stack + OVERFLOW_STACK_SIZE, x0
103
104 /*
105 * Check whether we were already on the overflow stack. This may happen
106 * after panic() re-enables interrupts.
107 */
108 mrs x0, tpidr_el0 // sp of interrupted context
109 sub x0, sp, x0 // delta with top of overflow stack
110 tst x0, #~(OVERFLOW_STACK_SIZE - 1) // within range?
111 b.ne __bad_stack // no? -> bad stack pointer
112
113 /* We were already on the overflow stack. Restore sp/x0 and carry on. */
114 sub sp, sp, x0
115 mrs x0, tpidrro_el0
116#endif
117 b \label
118 .endm
119
120 .macro kernel_entry, el, regsize = 64
74 .if \regsize == 32121 .if \regsize == 32
75 mov w0, w0 // zero upper 32 bits of x0122 mov w0, w0 // zero upper 32 bits of x0
76 .endif123 .endif
@@ -111,6 +158,18 @@ · 34 unmodified lines
111 mrs x23, spsr_el1158 mrs x23, spsr_el1
112 stp lr, x21, [sp, #S_LR]159 stp lr, x21, [sp, #S_LR]
113 160
161 /*
162 * In order to be able to dump the contents of struct pt_regs at the
163 * time the exception was taken (in case we attempt to walk the call
164 * stack later), chain it together with the stack frames.
165 */
166 .if \el == 0
167 stp xzr, xzr, [sp, #S_STACKFRAME]
168 .else
169 stp x29, x22, [sp, #S_STACKFRAME]
170 .endif
171 add x29, sp, #S_STACKFRAME
172
114#ifdef CONFIG_ARM64_SW_TTBR0_PAN173#ifdef CONFIG_ARM64_SW_TTBR0_PAN
115 /*174 /*
116 * Set the TTBR0 PAN bit in SPSR. When the exception is taken from175 * Set the TTBR0 PAN bit in SPSR. When the exception is taken from
@@ -138,12 +197,10 @@ alternative_else_nop_endif · 21 unmodified lines
138 197
139 stp x22, x23, [sp, #S_PC]198 stp x22, x23, [sp, #S_PC]
140 199
141 /*200 /* Not in a syscall by default (el0_svc overwrites for real syscall) */
142 * Set syscallno to -1 by default (overridden later if real syscall).
143 */
144 .if \el == 0201 .if \el == 0
145 mvn x21, xzr202 mov w21, #NO_SYSCALL
146 str x21, [sp, #S_SYSCALLNO]203 str w21, [sp, #S_SYSCALLNO]
147 .endif204 .endif
148 205
149 /*206 /*
@@ -259,20 +316,12 @@ alternative_else_nop_endif · 109 unmodified lines
259 and x25, x25, #~(THREAD_SIZE - 1)316 and x25, x25, #~(THREAD_SIZE - 1)
260 cbnz x25, 9998f317 cbnz x25, 9998f
261 318
262 adr_this_cpu x25, irq_stack, x26319 ldr_this_cpu x25, irq_stack_ptr, x26
263 mov x26, #IRQ_STACK_START_SP320 mov x26, #IRQ_STACK_SIZE
264 add x26, x25, x26321 add x26, x25, x26
265 322
266 /* switch to the irq stack */323 /* switch to the irq stack */
267 mov sp, x26324 mov sp, x26
268
269 /*
270 * Add a dummy stack frame, this non-standard format is fixed up
271 * by unwind_frame()
272 */
273 stp x29, x19, [sp, #-16]!
274 mov x29, sp
275
2769998:3259998:
277 .endm326 .endm
278 327
@@ -290,8 +339,9 @@ alternative_else_nop_endif · 11 unmodified lines
290 *339 *
291 * x7 is reserved for the system call number in 32-bit mode.340 * x7 is reserved for the system call number in 32-bit mode.
292 */341 */
293sc_nr .req x25 // number of system calls342wsc_nr .req w25 // number of system calls
294scno .req x26 // syscall number343wscno .req w26 // syscall number
344xscno .req x26 // syscall number (zero-extended)
295stbl .req x27 // syscall table pointer345stbl .req x27 // syscall table pointer
296tsk .req x28 // current thread_info346tsk .req x28 // current thread_info
297 347
@@ -315,34 +365,62 @@ tsk .req x28 // current thread_info · 17 unmodified lines
315 365
316 .align 11366 .align 11
317ENTRY(vectors)367ENTRY(vectors)
318 ventry el1_sync_invalid // Synchronous EL1t368 kernel_ventry el1_sync_invalid // Synchronous EL1t
319 ventry el1_irq_invalid // IRQ EL1t369 kernel_ventry el1_irq_invalid // IRQ EL1t
320 ventry el1_fiq_invalid // FIQ EL1t370 kernel_ventry el1_fiq_invalid // FIQ EL1t
321 ventry el1_error_invalid // Error EL1t371 kernel_ventry el1_error_invalid // Error EL1t
322 372
323 ventry el1_sync // Synchronous EL1h373 kernel_ventry el1_sync // Synchronous EL1h
324 ventry el1_irq // IRQ EL1h374 kernel_ventry el1_irq // IRQ EL1h
325 ventry el1_fiq_invalid // FIQ EL1h375 kernel_ventry el1_fiq_invalid // FIQ EL1h
326 ventry el1_error_invalid // Error EL1h376 kernel_ventry el1_error_invalid // Error EL1h
327 377
328 ventry el0_sync // Synchronous 64-bit EL0378 kernel_ventry el0_sync // Synchronous 64-bit EL0
329 ventry el0_irq // IRQ 64-bit EL0379 kernel_ventry el0_irq // IRQ 64-bit EL0
330 ventry el0_fiq_invalid // FIQ 64-bit EL0380 kernel_ventry el0_fiq_invalid // FIQ 64-bit EL0
331 ventry el0_error_invalid // Error 64-bit EL0381 kernel_ventry el0_error_invalid // Error 64-bit EL0
332 382
333#ifdef CONFIG_COMPAT383#ifdef CONFIG_COMPAT
334 ventry el0_sync_compat // Synchronous 32-bit EL0384 kernel_ventry el0_sync_compat // Synchronous 32-bit EL0
335 ventry el0_irq_compat // IRQ 32-bit EL0385 kernel_ventry el0_irq_compat // IRQ 32-bit EL0
336 ventry el0_fiq_invalid_compat // FIQ 32-bit EL0386 kernel_ventry el0_fiq_invalid_compat // FIQ 32-bit EL0
337 ventry el0_error_invalid_compat // Error 32-bit EL0387 kernel_ventry el0_error_invalid_compat // Error 32-bit EL0
338#else388#else
339 ventry el0_sync_invalid // Synchronous 32-bit EL0389 kernel_ventry el0_sync_invalid // Synchronous 32-bit EL0
340 ventry el0_irq_invalid // IRQ 32-bit EL0390 kernel_ventry el0_irq_invalid // IRQ 32-bit EL0
341 ventry el0_fiq_invalid // FIQ 32-bit EL0391 kernel_ventry el0_fiq_invalid // FIQ 32-bit EL0
342 ventry el0_error_invalid // Error 32-bit EL0392 kernel_ventry el0_error_invalid // Error 32-bit EL0
343#endif393#endif
344END(vectors)394END(vectors)
345 395
396#ifdef CONFIG_VMAP_STACK
397 /*
398 * We detected an overflow in kernel_ventry, which switched to the
399 * overflow stack. Stash the exception regs, and head to our overflow
400 * handler.
401 */
402__bad_stack:
403 /* Restore the original x0 value */
404 mrs x0, tpidrro_el0
405
406 /*
407 * Store the original GPRs to the new stack. The orginal SP (minus
408 * S_FRAME_SIZE) was stashed in tpidr_el0 by kernel_ventry.
409 */
410 sub sp, sp, #S_FRAME_SIZE
411 kernel_entry 1
412 mrs x0, tpidr_el0
413 add x0, x0, #S_FRAME_SIZE
414 str x0, [sp, #S_SP]
415
416 /* Stash the regs for handle_bad_stack */
417 mov x0, sp
418
419 /* Time to die */
420 bl handle_bad_stack
421 ASM_BUG()
422#endif /* CONFIG_VMAP_STACK */
423
346/*424/*
347 * Invalid mode handlers425 * Invalid mode handlers
348 */426 */
@@ -351,7 +429,8 @@ END(vectors) · 2 unmodified lines
351 mov x0, sp429 mov x0, sp
352 mov x1, #\reason430 mov x1, #\reason
353 mrs x2, esr_el1431 mrs x2, esr_el1
354 b bad_mode432 bl bad_mode
433 ASM_BUG()
355 .endm434 .endm
356 435
357el0_sync_invalid:436el0_sync_invalid:
@@ -448,14 +527,16 @@ el1_sp_pc: · 90 unmodified lines
448 mrs x0, far_el1527 mrs x0, far_el1
449 enable_dbg528 enable_dbg
450 mov x2, sp529 mov x2, sp
451 b do_sp_pc_abort530 bl do_sp_pc_abort
531 ASM_BUG()
452el1_undef:532el1_undef:
453 /*533 /*
454 * Undefined instruction534 * Undefined instruction
455 */535 */
456 enable_dbg536 enable_dbg
457 mov x0, sp537 mov x0, sp
458 b do_undefinstr538 bl do_undefinstr
539 ASM_BUG()
459el1_dbg:540el1_dbg:
460 /*541 /*
461 * Debug exception handling542 * Debug exception handling
@@ -473,7 +554,8 @@ el1_inv: · 11 unmodified lines
473 mov x0, sp554 mov x0, sp
474 mov x2, x1555 mov x2, x1
475 mov x1, #BAD_SYNC556 mov x1, #BAD_SYNC
476 b bad_mode557 bl bad_mode
558 ASM_BUG()
477ENDPROC(el1_sync)559ENDPROC(el1_sync)
478 560
479 .align 6561 .align 6
@@ -577,8 +659,8 @@ el0_svc_compat: · 97 unmodified lines
577 * AArch32 syscall handling659 * AArch32 syscall handling
578 */660 */
579 adrp stbl, compat_sys_call_table // load compat syscall table pointer661 adrp stbl, compat_sys_call_table // load compat syscall table pointer
580 uxtw scno, w7 // syscall number in w7 (r7)662 mov wscno, w7 // syscall number in w7 (r7)
581 mov sc_nr, #__NR_compat_syscalls663 mov wsc_nr, #__NR_compat_syscalls
582 b el0_svc_naked664 b el0_svc_naked
583 665
584 .align 6666 .align 6
@@ -706,38 +788,6 @@ el0_irq_naked: · 121 unmodified lines
706 b ret_to_user788 b ret_to_user
707ENDPROC(el0_irq)789ENDPROC(el0_irq)
708 790
709/*
710 * Register switch for AArch64. The callee-saved registers need to be saved
711 * and restored. On entry:
712 * x0 = previous task_struct (must be preserved across the switch)
713 * x1 = next task_struct
714 * Previous and next are guaranteed not to be the same.
715 *
716 */
717ENTRY(cpu_switch_to)
718 mov x10, #THREAD_CPU_CONTEXT
719 add x8, x0, x10
720 mov x9, sp
721 stp x19, x20, [x8], #16 // store callee-saved registers
722 stp x21, x22, [x8], #16
723 stp x23, x24, [x8], #16
724 stp x25, x26, [x8], #16
725 stp x27, x28, [x8], #16
726 stp x29, x9, [x8], #16
727 str lr, [x8]
728 add x8, x1, x10
729 ldp x19, x20, [x8], #16 // restore callee-saved registers
730 ldp x21, x22, [x8], #16
731 ldp x23, x24, [x8], #16
732 ldp x25, x26, [x8], #16
733 ldp x27, x28, [x8], #16
734 ldp x29, x9, [x8], #16
735 ldr lr, [x8]
736 mov sp, x9
737 msr sp_el0, x1
738 ret
739ENDPROC(cpu_switch_to)
740
741/*791/*
742 * This is the fast syscall return path. We do as little as possible here,792 * This is the fast syscall return path. We do as little as possible here,
743 * and this includes saving x0 back into the kernel stack.793 * and this includes saving x0 back into the kernel stack.
@@ -780,37 +830,25 @@ finish_ret_to_user: · 36 unmodified lines
780 kernel_exit 0830 kernel_exit 0
781ENDPROC(ret_to_user)831ENDPROC(ret_to_user)
782 832
783/*
784 * This is how we return from a fork.
785 */
786ENTRY(ret_from_fork)
787 bl schedule_tail
788 cbz x19, 1f // not a kernel thread
789 mov x0, x20
790 blr x19
7911: get_thread_info tsk
792 b ret_to_user
793ENDPROC(ret_from_fork)
794
795/*833/*
796 * SVC handler.834 * SVC handler.
797 */835 */
798 .align 6836 .align 6
799el0_svc:837el0_svc:
800 adrp stbl, sys_call_table // load syscall table pointer838 adrp stbl, sys_call_table // load syscall table pointer
801 uxtw scno, w8 // syscall number in w8839 mov wscno, w8 // syscall number in w8
802 mov sc_nr, #__NR_syscalls840 mov wsc_nr, #__NR_syscalls
803el0_svc_naked: // compat entry point841el0_svc_naked: // compat entry point
804 stp x0, scno, [sp, #S_ORIG_X0] // save the original x0 and syscall number842 stp x0, xscno, [sp, #S_ORIG_X0] // save the original x0 and syscall number
805 enable_dbg_and_irq843 enable_dbg_and_irq
806 ct_user_exit 1844 ct_user_exit 1
807 845
808 ldr x16, [tsk, #TSK_TI_FLAGS] // check for syscall hooks846 ldr x16, [tsk, #TSK_TI_FLAGS] // check for syscall hooks
809 tst x16, #_TIF_SYSCALL_WORK847 tst x16, #_TIF_SYSCALL_WORK
810 b.ne __sys_trace848 b.ne __sys_trace
811 cmp scno, sc_nr // check upper syscall limit849 cmp wscno, wsc_nr // check upper syscall limit
812 b.hs ni_sys850 b.hs ni_sys
813 ldr x16, [stbl, scno, lsl #3] // address in the syscall table851 ldr x16, [stbl, xscno, lsl #3] // address in the syscall table
814 blr x16 // call sys_* routine852 blr x16 // call sys_* routine
815 b ret_fast_syscall853 b ret_fast_syscall
816ni_sys:854ni_sys:
@@ -824,24 +862,23 @@ ENDPROC(el0_svc) · 7 unmodified lines
824 * switches, and waiting for our parent to respond.862 * switches, and waiting for our parent to respond.
825 */863 */
826__sys_trace:864__sys_trace:
827 mov w0, #-1 // set default errno for865 cmp wscno, #NO_SYSCALL // user-issued syscall(-1)?
828 cmp scno, x0 // user-issued syscall(-1)
829 b.ne 1f866 b.ne 1f
830 mov x0, #-ENOSYS867 mov x0, #-ENOSYS // set default errno if so
831 str x0, [sp, #S_X0]868 str x0, [sp, #S_X0]
8321: mov x0, sp8691: mov x0, sp
833 bl syscall_trace_enter870 bl syscall_trace_enter
834 cmp w0, #-1 // skip the syscall?871 cmp w0, #NO_SYSCALL // skip the syscall?
835 b.eq __sys_trace_return_skipped872 b.eq __sys_trace_return_skipped
836 uxtw scno, w0 // syscall number (possibly new)873 mov wscno, w0 // syscall number (possibly new)
837 mov x1, sp // pointer to regs874 mov x1, sp // pointer to regs
838 cmp scno, sc_nr // check upper syscall limit875 cmp wscno, wsc_nr // check upper syscall limit
839 b.hs __ni_sys_trace876 b.hs __ni_sys_trace
840 ldp x0, x1, [sp] // restore the syscall args877 ldp x0, x1, [sp] // restore the syscall args
841 ldp x2, x3, [sp, #S_X2]878 ldp x2, x3, [sp, #S_X2]
842 ldp x4, x5, [sp, #S_X4]879 ldp x4, x5, [sp, #S_X4]
843 ldp x6, x7, [sp, #S_X6]880 ldp x6, x7, [sp, #S_X6]
844 ldr x16, [stbl, scno, lsl #3] // address in the syscall table881 ldr x16, [stbl, xscno, lsl #3] // address in the syscall table
845 blr x16 // call sys_* routine882 blr x16 // call sys_* routine
846 883
847__sys_trace_return:884__sys_trace_return:
@@ -865,3 +902,49 @@ ENTRY(sys_rt_sigreturn_wrapper) · 17 unmodified lines
865 mov x0, sp902 mov x0, sp
866 b sys_rt_sigreturn903 b sys_rt_sigreturn
867ENDPROC(sys_rt_sigreturn_wrapper)904ENDPROC(sys_rt_sigreturn_wrapper)
905
906/*
907 * Register switch for AArch64. The callee-saved registers need to be saved
908 * and restored. On entry:
909 * x0 = previous task_struct (must be preserved across the switch)
910 * x1 = next task_struct
911 * Previous and next are guaranteed not to be the same.
912 *
913 */
914ENTRY(cpu_switch_to)
915 mov x10, #THREAD_CPU_CONTEXT
916 add x8, x0, x10
917 mov x9, sp
918 stp x19, x20, [x8], #16 // store callee-saved registers
919 stp x21, x22, [x8], #16
920 stp x23, x24, [x8], #16
921 stp x25, x26, [x8], #16
922 stp x27, x28, [x8], #16
923 stp x29, x9, [x8], #16
924 str lr, [x8]
925 add x8, x1, x10
926 ldp x19, x20, [x8], #16 // restore callee-saved registers
927 ldp x21, x22, [x8], #16
928 ldp x23, x24, [x8], #16
929 ldp x25, x26, [x8], #16
930 ldp x27, x28, [x8], #16
931 ldp x29, x9, [x8], #16
932 ldr lr, [x8]
933 mov sp, x9
934 msr sp_el0, x1
935 ret
936ENDPROC(cpu_switch_to)
937NOKPROBE(cpu_switch_to)
938
939/*
940 * This is how we return from a fork.
941 */
942ENTRY(ret_from_fork)
943 bl schedule_tail
944 cbz x19, 1f // not a kernel thread
945 mov x0, x20
946 blr x19
9471: get_thread_info tsk
948 b ret_to_user
949ENDPROC(ret_from_fork)
950NOKPROBE(ret_from_fork)
arch/arm64/kernel/fpsimd.c+132−40
@@ -17,16 +17,19 @@ · 16 unmodified lines
17 * along with this program. If not, see <http://www.gnu.org/licenses/>.17 * along with this program. If not, see <http://www.gnu.org/licenses/>.
18 */18 */
19 19
20#include <linux/bottom_half.h>
20#include <linux/cpu.h>21#include <linux/cpu.h>
21#include <linux/cpu_pm.h>22#include <linux/cpu_pm.h>
22#include <linux/kernel.h>23#include <linux/kernel.h>
23#include <linux/init.h>24#include <linux/init.h>
25#include <linux/percpu.h>
26#include <linux/preempt.h>
24#include <linux/sched/signal.h>27#include <linux/sched/signal.h>
25#include <linux/signal.h>28#include <linux/signal.h>
26#include <linux/hardirq.h>
27 29
28#include <asm/fpsimd.h>30#include <asm/fpsimd.h>
29#include <asm/cputype.h>31#include <asm/cputype.h>
32#include <asm/simd.h>
30 33
31#define FPEXC_IOF (1 << 0)34#define FPEXC_IOF (1 << 0)
32#define FPEXC_DZF (1 << 1)35#define FPEXC_DZF (1 << 1)
@@ -62,6 +65,13 @@ · 29 unmodified lines
62 * CPU currently contain the most recent userland FPSIMD state of the current65 * CPU currently contain the most recent userland FPSIMD state of the current
63 * task.66 * task.
64 *67 *
68 * In order to allow softirq handlers to use FPSIMD, kernel_neon_begin() may
69 * save the task's FPSIMD context back to task_struct from softirq context.
70 * To prevent this from racing with the manipulation of the task's FPSIMD state
71 * from task context and thereby corrupting the state, it is necessary to
72 * protect any manipulation of a task's fpsimd_state or TIF_FOREIGN_FPSTATE
73 * flag with local_bh_disable() unless softirqs are already masked.
74 *
65 * For a certain task, the sequence may look something like this:75 * For a certain task, the sequence may look something like this:
66 * - the task gets scheduled in; if both the task's fpsimd_state.cpu field76 * - the task gets scheduled in; if both the task's fpsimd_state.cpu field
67 * contains the id of the current CPU, and the CPU's fpsimd_last_state per-cpu77 * contains the id of the current CPU, and the CPU's fpsimd_last_state per-cpu
@@ -161,11 +171,14 @@ void fpsimd_flush_thread(void) · 93 unmodified lines
161{171{
162 if (!system_supports_fpsimd())172 if (!system_supports_fpsimd())
163 return;173 return;
164 preempt_disable();174
175 local_bh_disable();
176
165 memset(&current->thread.fpsimd_state, 0, sizeof(struct fpsimd_state));177 memset(&current->thread.fpsimd_state, 0, sizeof(struct fpsimd_state));
166 fpsimd_flush_task_state(current);178 fpsimd_flush_task_state(current);
167 set_thread_flag(TIF_FOREIGN_FPSTATE);179 set_thread_flag(TIF_FOREIGN_FPSTATE);
168 preempt_enable();180
181 local_bh_enable();
169}182}
170 183
171/*184/*
@@ -176,10 +189,13 @@ void fpsimd_preserve_current_state(void) · 4 unmodified lines
176{189{
177 if (!system_supports_fpsimd())190 if (!system_supports_fpsimd())
178 return;191 return;
179 preempt_disable();192
193 local_bh_disable();
194
180 if (!test_thread_flag(TIF_FOREIGN_FPSTATE))195 if (!test_thread_flag(TIF_FOREIGN_FPSTATE))
181 fpsimd_save_state(&current->thread.fpsimd_state);196 fpsimd_save_state(&current->thread.fpsimd_state);
182 preempt_enable();197
198 local_bh_enable();
183}199}
184 200
185/*201/*
@@ -191,15 +207,18 @@ void fpsimd_restore_current_state(void) · 5 unmodified lines
191{207{
192 if (!system_supports_fpsimd())208 if (!system_supports_fpsimd())
193 return;209 return;
194 preempt_disable();210
211 local_bh_disable();
212
195 if (test_and_clear_thread_flag(TIF_FOREIGN_FPSTATE)) {213 if (test_and_clear_thread_flag(TIF_FOREIGN_FPSTATE)) {
196 struct fpsimd_state *st = &current->thread.fpsimd_state;214 struct fpsimd_state *st = &current->thread.fpsimd_state;
197 215
198 fpsimd_load_state(st);216 fpsimd_load_state(st);
199 this_cpu_write(fpsimd_last_state, st);217 __this_cpu_write(fpsimd_last_state, st);
200 st->cpu = smp_processor_id();218 st->cpu = smp_processor_id();
201 }219 }
202 preempt_enable();220
221 local_bh_enable();
203}222}
204 223
205/*224/*
@@ -211,15 +230,18 @@ void fpsimd_update_current_state(struct fpsimd_state *state) · 5 unmodified lines
211{230{
212 if (!system_supports_fpsimd())231 if (!system_supports_fpsimd())
213 return;232 return;
214 preempt_disable();233
234 local_bh_disable();
235
215 fpsimd_load_state(state);236 fpsimd_load_state(state);
216 if (test_and_clear_thread_flag(TIF_FOREIGN_FPSTATE)) {237 if (test_and_clear_thread_flag(TIF_FOREIGN_FPSTATE)) {
217 struct fpsimd_state *st = &current->thread.fpsimd_state;238 struct fpsimd_state *st = &current->thread.fpsimd_state;
218 239
219 this_cpu_write(fpsimd_last_state, st);240 __this_cpu_write(fpsimd_last_state, st);
220 st->cpu = smp_processor_id();241 st->cpu = smp_processor_id();
221 }242 }
222 preempt_enable();243
244 local_bh_enable();
223}245}
224 246
225/*247/*
@@ -232,52 +254,122 @@ void fpsimd_flush_task_state(struct task_struct *t) · 6 unmodified lines
232 254
233#ifdef CONFIG_KERNEL_MODE_NEON255#ifdef CONFIG_KERNEL_MODE_NEON
234 256
235static DEFINE_PER_CPU(struct fpsimd_partial_state, hardirq_fpsimdstate);257DEFINE_PER_CPU(bool, kernel_neon_busy);
236static DEFINE_PER_CPU(struct fpsimd_partial_state, softirq_fpsimdstate);258EXPORT_PER_CPU_SYMBOL(kernel_neon_busy);
237 259
238/*260/*
239 * Kernel-side NEON support functions261 * Kernel-side NEON support functions
240 */262 */
241void kernel_neon_begin_partial(u32 num_regs)263
264/*
265 * kernel_neon_begin(): obtain the CPU FPSIMD registers for use by the calling
266 * context
267 *
268 * Must not be called unless may_use_simd() returns true.
269 * Task context in the FPSIMD registers is saved back to memory as necessary.
270 *
271 * A matching call to kernel_neon_end() must be made before returning from the
272 * calling context.
273 *
274 * The caller may freely use the FPSIMD registers until kernel_neon_end() is
275 * called.
276 */
277void kernel_neon_begin(void)
242{278{
243 if (WARN_ON(!system_supports_fpsimd()))279 if (WARN_ON(!system_supports_fpsimd()))
244 return;280 return;
245 if (in_interrupt()) {
246 struct fpsimd_partial_state *s = this_cpu_ptr(
247 in_irq() ? &hardirq_fpsimdstate : &softirq_fpsimdstate);
248 281
249 BUG_ON(num_regs > 32);282 BUG_ON(!may_use_simd());
250 fpsimd_save_partial_state(s, roundup(num_regs, 2));283
251 } else {284 local_bh_disable();
252 /*285
253 * Save the userland FPSIMD state if we have one and if we286 __this_cpu_write(kernel_neon_busy, true);
254 * haven't done so already. Clear fpsimd_last_state to indicate287
255 * that there is no longer userland FPSIMD state in the288 /* Save unsaved task fpsimd state, if any: */
256 * registers.289 if (current->mm && !test_and_set_thread_flag(TIF_FOREIGN_FPSTATE))
257 */290 fpsimd_save_state(&current->thread.fpsimd_state);
258 preempt_disable();291
259 if (current->mm &&292 /* Invalidate any task state remaining in the fpsimd regs: */
260 !test_and_set_thread_flag(TIF_FOREIGN_FPSTATE))293 __this_cpu_write(fpsimd_last_state, NULL);
261 fpsimd_save_state(&current->thread.fpsimd_state);294
262 this_cpu_write(fpsimd_last_state, NULL);295 preempt_disable();
263 }296
297 local_bh_enable();
264}298}
265EXPORT_SYMBOL(kernel_neon_begin_partial);299EXPORT_SYMBOL(kernel_neon_begin);
266 300
301/*
302 * kernel_neon_end(): give the CPU FPSIMD registers back to the current task
303 *
304 * Must be called from a context in which kernel_neon_begin() was previously
305 * called, with no call to kernel_neon_end() in the meantime.
306 *
307 * The caller must not use the FPSIMD registers after this function is called,
308 * unless kernel_neon_begin() is called again in the meantime.
309 */
267void kernel_neon_end(void)310void kernel_neon_end(void)
268{311{
312 bool busy;
313
269 if (!system_supports_fpsimd())314 if (!system_supports_fpsimd())
270 return;315 return;
271 if (in_interrupt()) {316
272 struct fpsimd_partial_state *s = this_cpu_ptr(317 busy = __this_cpu_xchg(kernel_neon_busy, false);
273 in_irq() ? &hardirq_fpsimdstate : &softirq_fpsimdstate);318 WARN_ON(!busy); /* No matching kernel_neon_begin()? */
274 fpsimd_load_partial_state(s);319
275 } else {320 preempt_enable();
276 preempt_enable();
277 }
278}321}
279EXPORT_SYMBOL(kernel_neon_end);322EXPORT_SYMBOL(kernel_neon_end);
280 323
324static DEFINE_PER_CPU(struct fpsimd_state, efi_fpsimd_state);
325static DEFINE_PER_CPU(bool, efi_fpsimd_state_used);
326
327/*
328 * EFI runtime services support functions
329 *
330 * The ABI for EFI runtime services allows EFI to use FPSIMD during the call.
331 * This means that for EFI (and only for EFI), we have to assume that FPSIMD
332 * is always used rather than being an optional accelerator.
333 *
334 * These functions provide the necessary support for ensuring FPSIMD
335 * save/restore in the contexts from which EFI is used.
336 *
337 * Do not use them for any other purpose -- if tempted to do so, you are
338 * either doing something wrong or you need to propose some refactoring.
339 */
340
341/*
342 * __efi_fpsimd_begin(): prepare FPSIMD for making an EFI runtime services call
343 */
344void __efi_fpsimd_begin(void)
345{
346 if (!system_supports_fpsimd())
347 return;
348
349 WARN_ON(preemptible());
350
351 if (may_use_simd())
352 kernel_neon_begin();
353 else {
354 fpsimd_save_state(this_cpu_ptr(&efi_fpsimd_state));
355 __this_cpu_write(efi_fpsimd_state_used, true);
356 }
357}
358
359/*
360 * __efi_fpsimd_end(): clean up FPSIMD after an EFI runtime services call
361 */
362void __efi_fpsimd_end(void)
363{
364 if (!system_supports_fpsimd())
365 return;
366
367 if (__this_cpu_xchg(efi_fpsimd_state_used, false))
368 fpsimd_load_state(this_cpu_ptr(&efi_fpsimd_state));
369 else
370 kernel_neon_end();
371}
372
281#endif /* CONFIG_KERNEL_MODE_NEON */373#endif /* CONFIG_KERNEL_MODE_NEON */
282 374
283#ifdef CONFIG_CPU_PM375#ifdef CONFIG_CPU_PM
arch/arm64/kernel/head.S+13−9
@@ -143,8 +143,8 @@ preserve_boot_args: · 142 unmodified lines
143 dmb sy // needed before dc ivac with143 dmb sy // needed before dc ivac with
144 // MMU off144 // MMU off
145 145
146 add x1, x0, #0x20 // 4 x 8 bytes146 mov x1, #0x20 // 4 x 8 bytes
147 b __inval_cache_range // tail call147 b __inval_dcache_area // tail call
148ENDPROC(preserve_boot_args)148ENDPROC(preserve_boot_args)
149 149
150/*150/*
@@ -221,20 +221,20 @@ __create_page_tables: · 70 unmodified lines
221 * dirty cache lines being evicted.221 * dirty cache lines being evicted.
222 */222 */
223 adrp x0, idmap_pg_dir223 adrp x0, idmap_pg_dir
224 adrp x1, swapper_pg_dir + SWAPPER_DIR_SIZE + RESERVED_TTBR0_SIZE224 ldr x1, =(IDMAP_DIR_SIZE + SWAPPER_DIR_SIZE + RESERVED_TTBR0_SIZE)
225 bl __inval_cache_range225 bl __inval_dcache_area
226 226
227 /*227 /*
228 * Clear the idmap and swapper page tables.228 * Clear the idmap and swapper page tables.
229 */229 */
230 adrp x0, idmap_pg_dir230 adrp x0, idmap_pg_dir
231 adrp x6, swapper_pg_dir + SWAPPER_DIR_SIZE + RESERVED_TTBR0_SIZE231 ldr x1, =(IDMAP_DIR_SIZE + SWAPPER_DIR_SIZE + RESERVED_TTBR0_SIZE)
2321: stp xzr, xzr, [x0], #162321: stp xzr, xzr, [x0], #16
233 stp xzr, xzr, [x0], #16233 stp xzr, xzr, [x0], #16
234 stp xzr, xzr, [x0], #16234 stp xzr, xzr, [x0], #16
235 stp xzr, xzr, [x0], #16235 stp xzr, xzr, [x0], #16
236 cmp x0, x6236 subs x1, x1, #64
237 b.lo 1b237 b.ne 1b
238 238
239 mov x7, SWAPPER_MM_MMUFLAGS239 mov x7, SWAPPER_MM_MMUFLAGS
240 240
@@ -307,9 +307,9 @@ __create_page_tables: · 66 unmodified lines
307 * tables again to remove any speculatively loaded cache lines.307 * tables again to remove any speculatively loaded cache lines.
308 */308 */
309 adrp x0, idmap_pg_dir309 adrp x0, idmap_pg_dir
310 adrp x1, swapper_pg_dir + SWAPPER_DIR_SIZE + RESERVED_TTBR0_SIZE310 ldr x1, =(IDMAP_DIR_SIZE + SWAPPER_DIR_SIZE + RESERVED_TTBR0_SIZE)
311 dmb sy311 dmb sy
312 bl __inval_cache_range312 bl __inval_dcache_area
313 313
314 ret x28314 ret x28
315ENDPROC(__create_page_tables)315ENDPROC(__create_page_tables)
@@ -361,6 +361,9 @@ __primary_switched: · 45 unmodified lines
361 ret // to __primary_switch()361 ret // to __primary_switch()
3620:3620:
363#endif363#endif
364 add sp, sp, #16
365 mov x29, #0
366 mov x30, #0
364 b start_kernel367 b start_kernel
365ENDPROC(__primary_switched)368ENDPROC(__primary_switched)
366 369
@@ -616,6 +619,7 @@ __secondary_switched: · 249 unmodified lines
616 ldr x2, [x0, #CPU_BOOT_TASK]619 ldr x2, [x0, #CPU_BOOT_TASK]
617 msr sp_el0, x2620 msr sp_el0, x2
618 mov x29, #0621 mov x29, #0
622 mov x30, #0
619 b secondary_start_kernel623 b secondary_start_kernel
620ENDPROC(__secondary_switched)624ENDPROC(__secondary_switched)
621 625
arch/arm64/kernel/hibernate.c+2−2
@@ -330,7 +330,7 @@ static void _copy_pte(pte_t *dst_pte, pte_t *src_pte, unsigned long addr) · 329 unmodified lines
330 * read only (code, rodata). Clear the RDONLY bit from330 * read only (code, rodata). Clear the RDONLY bit from
331 * the temporary mappings we use during restore.331 * the temporary mappings we use during restore.
332 */332 */
333 set_pte(dst_pte, pte_clear_rdonly(pte));333 set_pte(dst_pte, pte_mkwrite(pte));
334 } else if (debug_pagealloc_enabled() && !pte_none(pte)) {334 } else if (debug_pagealloc_enabled() && !pte_none(pte)) {
335 /*335 /*
336 * debug_pagealloc will removed the PTE_VALID bit if336 * debug_pagealloc will removed the PTE_VALID bit if
@@ -343,7 +343,7 @@ static void _copy_pte(pte_t *dst_pte, pte_t *src_pte, unsigned long addr) · 6 unmodified lines
343 */343 */
344 BUG_ON(!pfn_valid(pte_pfn(pte)));344 BUG_ON(!pfn_valid(pte_pfn(pte)));
345 345
346 set_pte(dst_pte, pte_mkpresent(pte_clear_rdonly(pte)));346 set_pte(dst_pte, pte_mkpresent(pte_mkwrite(pte)));
347 }347 }
348}348}
349 349
arch/arm64/kernel/irq.c+38−2
@@ -23,15 +23,16 @@ · 22 unmodified lines
23 23
24#include <linux/kernel_stat.h>24#include <linux/kernel_stat.h>
25#include <linux/irq.h>25#include <linux/irq.h>
26#include <linux/memory.h>
26#include <linux/smp.h>27#include <linux/smp.h>
27#include <linux/init.h>28#include <linux/init.h>
28#include <linux/irqchip.h>29#include <linux/irqchip.h>
29#include <linux/seq_file.h>30#include <linux/seq_file.h>
31#include <linux/vmalloc.h>
30 32
31unsigned long irq_err_count;33unsigned long irq_err_count;
32 34
33/* irq stack only needs to be 16 byte aligned - not IRQ_STACK_SIZE aligned. */35DEFINE_PER_CPU(unsigned long *, irq_stack_ptr);
34DEFINE_PER_CPU(unsigned long [IRQ_STACK_SIZE/sizeof(long)], irq_stack) __aligned(16);
35 36
36int arch_show_interrupts(struct seq_file *p, int prec)37int arch_show_interrupts(struct seq_file *p, int prec)
37{38{
@@ -50,8 +51,43 @@ void __init set_handle_irq(void (*handle_irq)(struct pt_regs *)) · 12 unmodified lines
50 handle_arch_irq = handle_irq;51 handle_arch_irq = handle_irq;
51}52}
52 53
54#ifdef CONFIG_VMAP_STACK
55static void init_irq_stacks(void)
56{
57 int cpu;
58 unsigned long *p;
59
60 for_each_possible_cpu(cpu) {
61 /*
62 * To ensure that VMAP'd stack overflow detection works
63 * correctly, the IRQ stacks need to have the same
64 * alignment as other stacks.
65 */
66 p = __vmalloc_node_range(IRQ_STACK_SIZE, THREAD_ALIGN,
67 VMALLOC_START, VMALLOC_END,
68 THREADINFO_GFP, PAGE_KERNEL,
69 0, cpu_to_node(cpu),
70 __builtin_return_address(0));
71
72 per_cpu(irq_stack_ptr, cpu) = p;
73 }
74}
75#else
76/* irq stack only needs to be 16 byte aligned - not IRQ_STACK_SIZE aligned. */
77DEFINE_PER_CPU_ALIGNED(unsigned long [IRQ_STACK_SIZE/sizeof(long)], irq_stack);
78
79static void init_irq_stacks(void)
80{
81 int cpu;
82
83 for_each_possible_cpu(cpu)
84 per_cpu(irq_stack_ptr, cpu) = per_cpu(irq_stack, cpu);
85}
86#endif
87
53void __init init_IRQ(void)88void __init init_IRQ(void)
54{89{
90 init_irq_stacks();
55 irqchip_init();91 irqchip_init();
56 if (!handle_arch_irq)92 if (!handle_arch_irq)
57 panic("No interrupt controller found.");93 panic("No interrupt controller found.");
arch/arm64/kernel/machine_kexec.c+1−1
@@ -252,7 +252,7 @@ void machine_crash_shutdown(struct pt_regs *regs) · 251 unmodified lines
252 local_irq_disable();252 local_irq_disable();
253 253
254 /* shutdown non-crashing cpus */254 /* shutdown non-crashing cpus */
255 smp_send_crash_stop();255 crash_smp_send_stop();
256 256
257 /* for crashing cpu */257 /* for crashing cpu */
258 crash_save_cpu(regs, smp_processor_id());258 crash_save_cpu(regs, smp_processor_id());
arch/arm64/kernel/perf_callchain.c+0−1
@@ -162,7 +162,6 @@ void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, · 161 unmodified lines
162 }162 }
163 163
164 frame.fp = regs->regs[29];164 frame.fp = regs->regs[29];
165 frame.sp = regs->sp;
166 frame.pc = regs->pc;165 frame.pc = regs->pc;
167#ifdef CONFIG_FUNCTION_GRAPH_TRACER166#ifdef CONFIG_FUNCTION_GRAPH_TRACER
168 frame.graph = current->curr_ret_stack;167 frame.graph = current->curr_ret_stack;
arch/arm64/kernel/perf_event.c+85−123
@@ -202,55 +202,6 @@ static const unsigned armv8_pmuv3_perf_map[PERF_COUNT_HW_MAX] = { · 201 unmodified lines
202 [PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = ARMV8_PMUV3_PERFCTR_STALL_BACKEND,202 [PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = ARMV8_PMUV3_PERFCTR_STALL_BACKEND,
203};203};
204 204
205/* ARM Cortex-A53 HW events mapping. */
206static const unsigned armv8_a53_perf_map[PERF_COUNT_HW_MAX] = {
207 PERF_MAP_ALL_UNSUPPORTED,
208 [PERF_COUNT_HW_CPU_CYCLES] = ARMV8_PMUV3_PERFCTR_CPU_CYCLES,
209 [PERF_COUNT_HW_INSTRUCTIONS] = ARMV8_PMUV3_PERFCTR_INST_RETIRED,
210 [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV8_PMUV3_PERFCTR_L1D_CACHE,
211 [PERF_COUNT_HW_CACHE_MISSES] = ARMV8_PMUV3_PERFCTR_L1D_CACHE_REFILL,
212 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV8_PMUV3_PERFCTR_PC_WRITE_RETIRED,
213 [PERF_COUNT_HW_BRANCH_MISSES] = ARMV8_PMUV3_PERFCTR_BR_MIS_PRED,
214 [PERF_COUNT_HW_BUS_CYCLES] = ARMV8_PMUV3_PERFCTR_BUS_CYCLES,
215};
216
217/* ARM Cortex-A57 and Cortex-A72 events mapping. */
218static const unsigned armv8_a57_perf_map[PERF_COUNT_HW_MAX] = {
219 PERF_MAP_ALL_UNSUPPORTED,
220 [PERF_COUNT_HW_CPU_CYCLES] = ARMV8_PMUV3_PERFCTR_CPU_CYCLES,
221 [PERF_COUNT_HW_INSTRUCTIONS] = ARMV8_PMUV3_PERFCTR_INST_RETIRED,
222 [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV8_PMUV3_PERFCTR_L1D_CACHE,
223 [PERF_COUNT_HW_CACHE_MISSES] = ARMV8_PMUV3_PERFCTR_L1D_CACHE_REFILL,
224 [PERF_COUNT_HW_BRANCH_MISSES] = ARMV8_PMUV3_PERFCTR_BR_MIS_PRED,
225 [PERF_COUNT_HW_BUS_CYCLES] = ARMV8_PMUV3_PERFCTR_BUS_CYCLES,
226};
227
228static const unsigned armv8_thunder_perf_map[PERF_COUNT_HW_MAX] = {
229 PERF_MAP_ALL_UNSUPPORTED,
230 [PERF_COUNT_HW_CPU_CYCLES] = ARMV8_PMUV3_PERFCTR_CPU_CYCLES,
231 [PERF_COUNT_HW_INSTRUCTIONS] = ARMV8_PMUV3_PERFCTR_INST_RETIRED,
232 [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV8_PMUV3_PERFCTR_L1D_CACHE,
233 [PERF_COUNT_HW_CACHE_MISSES] = ARMV8_PMUV3_PERFCTR_L1D_CACHE_REFILL,
234 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV8_PMUV3_PERFCTR_PC_WRITE_RETIRED,
235 [PERF_COUNT_HW_BRANCH_MISSES] = ARMV8_PMUV3_PERFCTR_BR_MIS_PRED,
236 [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = ARMV8_PMUV3_PERFCTR_STALL_FRONTEND,
237 [PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = ARMV8_PMUV3_PERFCTR_STALL_BACKEND,
238};
239
240/* Broadcom Vulcan events mapping */
241static const unsigned armv8_vulcan_perf_map[PERF_COUNT_HW_MAX] = {
242 PERF_MAP_ALL_UNSUPPORTED,
243 [PERF_COUNT_HW_CPU_CYCLES] = ARMV8_PMUV3_PERFCTR_CPU_CYCLES,
244 [PERF_COUNT_HW_INSTRUCTIONS] = ARMV8_PMUV3_PERFCTR_INST_RETIRED,
245 [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV8_PMUV3_PERFCTR_L1D_CACHE,
246 [PERF_COUNT_HW_CACHE_MISSES] = ARMV8_PMUV3_PERFCTR_L1D_CACHE_REFILL,
247 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV8_PMUV3_PERFCTR_BR_RETIRED,
248 [PERF_COUNT_HW_BRANCH_MISSES] = ARMV8_PMUV3_PERFCTR_BR_MIS_PRED,
249 [PERF_COUNT_HW_BUS_CYCLES] = ARMV8_PMUV3_PERFCTR_BUS_CYCLES,
250 [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = ARMV8_PMUV3_PERFCTR_STALL_FRONTEND,
251 [PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = ARMV8_PMUV3_PERFCTR_STALL_BACKEND,
252};
253
254static const unsigned armv8_pmuv3_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]205static const unsigned armv8_pmuv3_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
255 [PERF_COUNT_HW_CACHE_OP_MAX]206 [PERF_COUNT_HW_CACHE_OP_MAX]
256 [PERF_COUNT_HW_CACHE_RESULT_MAX] = {207 [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
@@ -281,27 +232,10 @@ static const unsigned armv8_a53_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] · 24 unmodified lines
281 [PERF_COUNT_HW_CACHE_RESULT_MAX] = {232 [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
282 PERF_CACHE_MAP_ALL_UNSUPPORTED,233 PERF_CACHE_MAP_ALL_UNSUPPORTED,
283 234
284 [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_L1D_CACHE,
285 [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L1D_CACHE_REFILL,
286 [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_L1D_CACHE,
287 [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L1D_CACHE_REFILL,
288 [C(L1D)][C(OP_PREFETCH)][C(RESULT_MISS)] = ARMV8_A53_PERFCTR_PREF_LINEFILL,235 [C(L1D)][C(OP_PREFETCH)][C(RESULT_MISS)] = ARMV8_A53_PERFCTR_PREF_LINEFILL,
289 236
290 [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_L1I_CACHE,237 [C(NODE)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_RD,
291 [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L1I_CACHE_REFILL,238 [C(NODE)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_WR,
292
293 [C(LL)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_L2D_CACHE,
294 [C(LL)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L2D_CACHE_REFILL,
295 [C(LL)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_L2D_CACHE,
296 [C(LL)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L2D_CACHE_REFILL,
297
298 [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L1D_TLB_REFILL,
299 [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L1I_TLB_REFILL,
300
301 [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_BR_PRED,
302 [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_BR_MIS_PRED,
303 [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_BR_PRED,
304 [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_BR_MIS_PRED,
305};239};
306 240
307static const unsigned armv8_a57_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]241static const unsigned armv8_a57_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
@@ -314,18 +248,26 @@ static const unsigned armv8_a57_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] · 6 unmodified lines
314 [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_L1D_CACHE_WR,248 [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_L1D_CACHE_WR,
315 [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV8_IMPDEF_PERFCTR_L1D_CACHE_REFILL_WR,249 [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV8_IMPDEF_PERFCTR_L1D_CACHE_REFILL_WR,
316 250
317 [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_L1I_CACHE,
318 [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L1I_CACHE_REFILL,
319
320 [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_REFILL_RD,251 [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_REFILL_RD,
321 [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_REFILL_WR,252 [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_REFILL_WR,
322 253
323 [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L1I_TLB_REFILL,254 [C(NODE)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_RD,
255 [C(NODE)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_WR,
256};
324 257
325 [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_BR_PRED,258static const unsigned armv8_a73_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
326 [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_BR_MIS_PRED,259 [PERF_COUNT_HW_CACHE_OP_MAX]
327 [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_BR_PRED,260 [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
328 [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_BR_MIS_PRED,261 PERF_CACHE_MAP_ALL_UNSUPPORTED,
262
263 [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_L1D_CACHE_RD,
264 [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_L1D_CACHE_WR,
265
266 [C(NODE)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_RD,
267 [C(NODE)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_WR,
268
269 [C(NODE)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_RD,
270 [C(NODE)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_WR,
329};271};
330 272
331static const unsigned armv8_thunder_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]273static const unsigned armv8_thunder_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
@@ -340,8 +282,6 @@ static const unsigned armv8_thunder_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] · 8 unmodified lines
340 [C(L1D)][C(OP_PREFETCH)][C(RESULT_ACCESS)] = ARMV8_THUNDER_PERFCTR_L1D_CACHE_PREF_ACCESS,282 [C(L1D)][C(OP_PREFETCH)][C(RESULT_ACCESS)] = ARMV8_THUNDER_PERFCTR_L1D_CACHE_PREF_ACCESS,
341 [C(L1D)][C(OP_PREFETCH)][C(RESULT_MISS)] = ARMV8_THUNDER_PERFCTR_L1D_CACHE_PREF_MISS,283 [C(L1D)][C(OP_PREFETCH)][C(RESULT_MISS)] = ARMV8_THUNDER_PERFCTR_L1D_CACHE_PREF_MISS,
342 284
343 [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_L1I_CACHE,
344 [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L1I_CACHE_REFILL,
345 [C(L1I)][C(OP_PREFETCH)][C(RESULT_ACCESS)] = ARMV8_THUNDER_PERFCTR_L1I_CACHE_PREF_ACCESS,285 [C(L1I)][C(OP_PREFETCH)][C(RESULT_ACCESS)] = ARMV8_THUNDER_PERFCTR_L1I_CACHE_PREF_ACCESS,
346 [C(L1I)][C(OP_PREFETCH)][C(RESULT_MISS)] = ARMV8_THUNDER_PERFCTR_L1I_CACHE_PREF_MISS,286 [C(L1I)][C(OP_PREFETCH)][C(RESULT_MISS)] = ARMV8_THUNDER_PERFCTR_L1I_CACHE_PREF_MISS,
347 287
@@ -349,13 +289,6 @@ static const unsigned armv8_thunder_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] · 1 unmodified line
349 [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_REFILL_RD,289 [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_REFILL_RD,
350 [C(DTLB)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_WR,290 [C(DTLB)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_WR,
351 [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_REFILL_WR,291 [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_REFILL_WR,
352
353 [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L1I_TLB_REFILL,
354
355 [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_BR_PRED,
356 [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_BR_MIS_PRED,
357 [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_BR_PRED,
358 [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_BR_MIS_PRED,
359};292};
360 293
361static const unsigned armv8_vulcan_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]294static const unsigned armv8_vulcan_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
@@ -368,22 +301,11 @@ static const unsigned armv8_vulcan_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] · 6 unmodified lines
368 [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_L1D_CACHE_WR,301 [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_L1D_CACHE_WR,
369 [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV8_IMPDEF_PERFCTR_L1D_CACHE_REFILL_WR,302 [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV8_IMPDEF_PERFCTR_L1D_CACHE_REFILL_WR,
370 303
371 [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_L1I_CACHE,
372 [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L1I_CACHE_REFILL,
373
374 [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L1I_TLB_REFILL,
375 [C(ITLB)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_L1I_TLB,
376
377 [C(DTLB)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_RD,304 [C(DTLB)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_RD,
378 [C(DTLB)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_WR,305 [C(DTLB)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_WR,
379 [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_REFILL_RD,306 [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_REFILL_RD,
380 [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_REFILL_WR,307 [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_REFILL_WR,
381 308
382 [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_BR_PRED,
383 [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_BR_MIS_PRED,
384 [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_BR_PRED,
385 [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_BR_MIS_PRED,
386
387 [C(NODE)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_RD,309 [C(NODE)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_RD,
388 [C(NODE)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_WR,310 [C(NODE)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_WR,
389};311};
@@ -846,17 +768,14 @@ static int armv8pmu_get_event_idx(struct pmu_hw_events *cpuc, · 456 unmodified lines
846 struct hw_perf_event *hwc = &event->hw;768 struct hw_perf_event *hwc = &event->hw;
847 unsigned long evtype = hwc->config_base & ARMV8_PMU_EVTYPE_EVENT;769 unsigned long evtype = hwc->config_base & ARMV8_PMU_EVTYPE_EVENT;
848 770
849 /* Always place a cycle counter into the cycle counter. */771 /* Always prefer to place a cycle counter into the cycle counter. */
850 if (evtype == ARMV8_PMUV3_PERFCTR_CPU_CYCLES) {772 if (evtype == ARMV8_PMUV3_PERFCTR_CPU_CYCLES) {
851 if (test_and_set_bit(ARMV8_IDX_CYCLE_COUNTER, cpuc->used_mask))773 if (!test_and_set_bit(ARMV8_IDX_CYCLE_COUNTER, cpuc->used_mask))
852 return -EAGAIN;774 return ARMV8_IDX_CYCLE_COUNTER;
853
854 return ARMV8_IDX_CYCLE_COUNTER;
855 }775 }
856 776
857 /*777 /*
858 * For anything other than a cycle counter, try and use778 * Otherwise use events counters
859 * the events counters
860 */779 */
861 for (idx = ARMV8_IDX_COUNTER0; idx < cpu_pmu->num_events; ++idx) {780 for (idx = ARMV8_IDX_COUNTER0; idx < cpu_pmu->num_events; ++idx) {
862 if (!test_and_set_bit(idx, cpuc->used_mask))781 if (!test_and_set_bit(idx, cpuc->used_mask))
@@ -924,7 +843,13 @@ static void armv8pmu_reset(void *info) · 61 unmodified lines
924 ARMV8_PMU_PMCR_LC);843 ARMV8_PMU_PMCR_LC);
925}844}
926 845
927static int armv8_pmuv3_map_event(struct perf_event *event)846static int __armv8_pmuv3_map_event(struct perf_event *event,
847 const unsigned (*extra_event_map)
848 [PERF_COUNT_HW_MAX],
849 const unsigned (*extra_cache_map)
850 [PERF_COUNT_HW_CACHE_MAX]
851 [PERF_COUNT_HW_CACHE_OP_MAX]
852 [PERF_COUNT_HW_CACHE_RESULT_MAX])
928{853{
929 int hw_event_id;854 int hw_event_id;
930 struct arm_pmu *armpmu = to_arm_pmu(event->pmu);855 struct arm_pmu *armpmu = to_arm_pmu(event->pmu);
@@ -932,44 +857,47 @@ static int armv8_pmuv3_map_event(struct perf_event *event) · 1 unmodified line
932 hw_event_id = armpmu_map_event(event, &armv8_pmuv3_perf_map,857 hw_event_id = armpmu_map_event(event, &armv8_pmuv3_perf_map,
933 &armv8_pmuv3_perf_cache_map,858 &armv8_pmuv3_perf_cache_map,
934 ARMV8_PMU_EVTYPE_EVENT);859 ARMV8_PMU_EVTYPE_EVENT);
935 if (hw_event_id < 0)
936 return hw_event_id;
937 860
938 /* disable micro/arch events not supported by this PMU */861 /* Onl expose micro/arch events supported by this PMU */
939 if ((hw_event_id < ARMV8_PMUV3_MAX_COMMON_EVENTS) &&862 if ((hw_event_id > 0) && (hw_event_id < ARMV8_PMUV3_MAX_COMMON_EVENTS)
940 !test_bit(hw_event_id, armpmu->pmceid_bitmap)) {863 && test_bit(hw_event_id, armpmu->pmceid_bitmap)) {
941 return -EOPNOTSUPP;864 return hw_event_id;
942 }865 }
943 866
944 return hw_event_id;867 return armpmu_map_event(event, extra_event_map, extra_cache_map,
868 ARMV8_PMU_EVTYPE_EVENT);
869}
870
871static int armv8_pmuv3_map_event(struct perf_event *event)
872{
873 return __armv8_pmuv3_map_event(event, NULL, NULL);
945}874}
946 875
947static int armv8_a53_map_event(struct perf_event *event)876static int armv8_a53_map_event(struct perf_event *event)
948{877{
949 return armpmu_map_event(event, &armv8_a53_perf_map,878 return __armv8_pmuv3_map_event(event, NULL, &armv8_a53_perf_cache_map);
950 &armv8_a53_perf_cache_map,
951 ARMV8_PMU_EVTYPE_EVENT);
952}879}
953 880
954static int armv8_a57_map_event(struct perf_event *event)881static int armv8_a57_map_event(struct perf_event *event)
955{882{
956 return armpmu_map_event(event, &armv8_a57_perf_map,883 return __armv8_pmuv3_map_event(event, NULL, &armv8_a57_perf_cache_map);
957 &armv8_a57_perf_cache_map,884}
958 ARMV8_PMU_EVTYPE_EVENT);885
886static int armv8_a73_map_event(struct perf_event *event)
887{
888 return __armv8_pmuv3_map_event(event, NULL, &armv8_a73_perf_cache_map);
959}889}
960 890
961static int armv8_thunder_map_event(struct perf_event *event)891static int armv8_thunder_map_event(struct perf_event *event)
962{892{
963 return armpmu_map_event(event, &armv8_thunder_perf_map,893 return __armv8_pmuv3_map_event(event, NULL,
964 &armv8_thunder_perf_cache_map,894 &armv8_thunder_perf_cache_map);
965 ARMV8_PMU_EVTYPE_EVENT);
966}895}
967 896
968static int armv8_vulcan_map_event(struct perf_event *event)897static int armv8_vulcan_map_event(struct perf_event *event)
969{898{
970 return armpmu_map_event(event, &armv8_vulcan_perf_map,899 return __armv8_pmuv3_map_event(event, NULL,
971 &armv8_vulcan_perf_cache_map,900 &armv8_vulcan_perf_cache_map);
972 ARMV8_PMU_EVTYPE_EVENT);
973}901}
974 902
975struct armv8pmu_probe_info {903struct armv8pmu_probe_info {
@@ -1062,6 +990,22 @@ static int armv8_pmuv3_init(struct arm_pmu *cpu_pmu) · 86 unmodified lines
1062 return 0;990 return 0;
1063}991}
1064 992
993static int armv8_a35_pmu_init(struct arm_pmu *cpu_pmu)
994{
995 int ret = armv8_pmu_init(cpu_pmu);
996 if (ret)
997 return ret;
998
999 cpu_pmu->name = "armv8_cortex_a35";
1000 cpu_pmu->map_event = armv8_a53_map_event;
1001 cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] =
1002 &armv8_pmuv3_events_attr_group;
1003 cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] =
1004 &armv8_pmuv3_format_attr_group;
1005
1006 return 0;
1007}
1008
1065static int armv8_a53_pmu_init(struct arm_pmu *cpu_pmu)1009static int armv8_a53_pmu_init(struct arm_pmu *cpu_pmu)
1066{1010{
1067 int ret = armv8_pmu_init(cpu_pmu);1011 int ret = armv8_pmu_init(cpu_pmu);
@@ -1110,6 +1054,22 @@ static int armv8_a72_pmu_init(struct arm_pmu *cpu_pmu) · 42 unmodified lines
1110 return 0;1054 return 0;
1111}1055}
1112 1056
1057static int armv8_a73_pmu_init(struct arm_pmu *cpu_pmu)
1058{
1059 int ret = armv8_pmu_init(cpu_pmu);
1060 if (ret)
1061 return ret;
1062
1063 cpu_pmu->name = "armv8_cortex_a73";
1064 cpu_pmu->map_event = armv8_a73_map_event;
1065 cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] =
1066 &armv8_pmuv3_events_attr_group;
1067 cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] =
1068 &armv8_pmuv3_format_attr_group;
1069
1070 return 0;
1071}
1072
1113static int armv8_thunder_pmu_init(struct arm_pmu *cpu_pmu)1073static int armv8_thunder_pmu_init(struct arm_pmu *cpu_pmu)
1114{1074{
1115 int ret = armv8_pmu_init(cpu_pmu);1075 int ret = armv8_pmu_init(cpu_pmu);
@@ -1144,9 +1104,11 @@ static int armv8_vulcan_pmu_init(struct arm_pmu *cpu_pmu) · 28 unmodified lines
1144 1104
1145static const struct of_device_id armv8_pmu_of_device_ids[] = {1105static const struct of_device_id armv8_pmu_of_device_ids[] = {
1146 {.compatible = "arm,armv8-pmuv3", .data = armv8_pmuv3_init},1106 {.compatible = "arm,armv8-pmuv3", .data = armv8_pmuv3_init},
1107 {.compatible = "arm,cortex-a35-pmu", .data = armv8_a35_pmu_init},
1147 {.compatible = "arm,cortex-a53-pmu", .data = armv8_a53_pmu_init},1108 {.compatible = "arm,cortex-a53-pmu", .data = armv8_a53_pmu_init},
1148 {.compatible = "arm,cortex-a57-pmu", .data = armv8_a57_pmu_init},1109 {.compatible = "arm,cortex-a57-pmu", .data = armv8_a57_pmu_init},
1149 {.compatible = "arm,cortex-a72-pmu", .data = armv8_a72_pmu_init},1110 {.compatible = "arm,cortex-a72-pmu", .data = armv8_a72_pmu_init},
1111 {.compatible = "arm,cortex-a73-pmu", .data = armv8_a73_pmu_init},
1150 {.compatible = "cavium,thunder-pmu", .data = armv8_thunder_pmu_init},1112 {.compatible = "cavium,thunder-pmu", .data = armv8_thunder_pmu_init},
1151 {.compatible = "brcm,vulcan-pmu", .data = armv8_vulcan_pmu_init},1113 {.compatible = "brcm,vulcan-pmu", .data = armv8_vulcan_pmu_init},
1152 {},1114 {},
arch/arm64/kernel/probes/uprobes.c+1−1
@@ -40,7 +40,7 @@ int arch_uprobe_analyze_insn(struct arch_uprobe *auprobe, struct mm_struct *mm, · 39 unmodified lines
40 probe_opcode_t insn;40 probe_opcode_t insn;
41 41
42 /* TODO: Currently we do not support AARCH32 instruction probing */42 /* TODO: Currently we do not support AARCH32 instruction probing */
43 if (test_bit(TIF_32BIT, &mm->context.flags))43 if (mm->context.flags & MMCF_AARCH32)
44 return -ENOTSUPP;44 return -ENOTSUPP;
45 else if (!IS_ALIGNED(addr, AARCH64_INSN_SIZE))45 else if (!IS_ALIGNED(addr, AARCH64_INSN_SIZE))
46 return -EINVAL;46 return -EINVAL;
arch/arm64/kernel/process.c+9−4
@@ -384,15 +384,12 @@ unsigned long get_wchan(struct task_struct *p) · 383 unmodified lines
384 return 0;384 return 0;
385 385
386 frame.fp = thread_saved_fp(p);386 frame.fp = thread_saved_fp(p);
387 frame.sp = thread_saved_sp(p);
388 frame.pc = thread_saved_pc(p);387 frame.pc = thread_saved_pc(p);
389#ifdef CONFIG_FUNCTION_GRAPH_TRACER388#ifdef CONFIG_FUNCTION_GRAPH_TRACER
390 frame.graph = p->curr_ret_stack;389 frame.graph = p->curr_ret_stack;
391#endif390#endif
392 do {391 do {
393 if (frame.sp < stack_page ||392 if (unwind_frame(p, &frame))
394 frame.sp >= stack_page + THREAD_SIZE ||
395 unwind_frame(p, &frame))
396 goto out;393 goto out;
397 if (!in_sched_functions(frame.pc)) {394 if (!in_sched_functions(frame.pc)) {
398 ret = frame.pc;395 ret = frame.pc;
@@ -419,3 +416,11 @@ unsigned long arch_randomize_brk(struct mm_struct *mm) · 20 unmodified lines
419 else416 else
420 return randomize_page(mm->brk, SZ_1G);417 return randomize_page(mm->brk, SZ_1G);
421}418}
419
420/*
421 * Called from setup_new_exec() after (COMPAT_)SET_PERSONALITY.
422 */
423void arch_setup_new_exec(void)
424{
425 current->mm->context.flags = is_compat_task() ? MMCF_AARCH32 : 0;
426}
arch/arm64/kernel/ptrace.c+3−2
@@ -42,6 +42,7 @@ · 41 unmodified lines
42#include <asm/compat.h>42#include <asm/compat.h>
43#include <asm/debug-monitors.h>43#include <asm/debug-monitors.h>
44#include <asm/pgtable.h>44#include <asm/pgtable.h>
45#include <asm/stacktrace.h>
45#include <asm/syscall.h>46#include <asm/syscall.h>
46#include <asm/traps.h>47#include <asm/traps.h>
47#include <asm/system_misc.h>48#include <asm/system_misc.h>
@@ -127,7 +128,7 @@ static bool regs_within_kernel_stack(struct pt_regs *regs, unsigned long addr) · 79 unmodified lines
127{128{
128 return ((addr & ~(THREAD_SIZE - 1)) ==129 return ((addr & ~(THREAD_SIZE - 1)) ==
129 (kernel_stack_pointer(regs) & ~(THREAD_SIZE - 1))) ||130 (kernel_stack_pointer(regs) & ~(THREAD_SIZE - 1))) ||
130 on_irq_stack(addr, raw_smp_processor_id());131 on_irq_stack(addr);
131}132}
132 133
133/**134/**
@@ -1363,7 +1364,7 @@ static void tracehook_report_syscall(struct pt_regs *regs, · 1229 unmodified lines
1363 if (dir == PTRACE_SYSCALL_EXIT)1364 if (dir == PTRACE_SYSCALL_EXIT)
1364 tracehook_report_syscall_exit(regs, 0);1365 tracehook_report_syscall_exit(regs, 0);
1365 else if (tracehook_report_syscall_entry(regs))1366 else if (tracehook_report_syscall_entry(regs))
1366 regs->syscallno = ~0UL;1367 forget_syscall(regs);
1367 1368
1368 regs->regs[regno] = saved_reg;1369 regs->regs[regno] = saved_reg;
1369}1370}
arch/arm64/kernel/return_address.c+0−1
@@ -42,7 +42,6 @@ void *return_address(unsigned int level) · 41 unmodified lines
42 data.addr = NULL;42 data.addr = NULL;
43 43
44 frame.fp = (unsigned long)__builtin_frame_address(0);44 frame.fp = (unsigned long)__builtin_frame_address(0);
45 frame.sp = current_stack_pointer;
46 frame.pc = (unsigned long)return_address; /* dummy */45 frame.pc = (unsigned long)return_address; /* dummy */
47#ifdef CONFIG_FUNCTION_GRAPH_TRACER46#ifdef CONFIG_FUNCTION_GRAPH_TRACER
48 frame.graph = current->curr_ret_stack;47 frame.graph = current->curr_ret_stack;
arch/arm64/kernel/signal.c+5−5
@@ -37,6 +37,7 @@ · 36 unmodified lines
37#include <asm/ucontext.h>37#include <asm/ucontext.h>
38#include <asm/unistd.h>38#include <asm/unistd.h>
39#include <asm/fpsimd.h>39#include <asm/fpsimd.h>
40#include <asm/ptrace.h>
40#include <asm/signal32.h>41#include <asm/signal32.h>
41#include <asm/vdso.h>42#include <asm/vdso.h>
42 43
@@ -388,7 +389,7 @@ static int restore_sigframe(struct pt_regs *regs, · 345 unmodified lines
388 /*389 /*
389 * Avoid sys_rt_sigreturn() restarting.390 * Avoid sys_rt_sigreturn() restarting.
390 */391 */
391 regs->syscallno = ~0UL;392 forget_syscall(regs);
392 393
393 err |= !valid_user_regs(&regs->user_regs, current);394 err |= !valid_user_regs(&regs->user_regs, current);
394 if (err == 0)395 if (err == 0)
@@ -674,13 +675,12 @@ static void do_signal(struct pt_regs *regs) · 279 unmodified lines
674{675{
675 unsigned long continue_addr = 0, restart_addr = 0;676 unsigned long continue_addr = 0, restart_addr = 0;
676 int retval = 0;677 int retval = 0;
677 int syscall = (int)regs->syscallno;
678 struct ksignal ksig;678 struct ksignal ksig;
679 679
680 /*680 /*
681 * If we were from a system call, check for system call restarting...681 * If we were from a system call, check for system call restarting...
682 */682 */
683 if (syscall >= 0) {683 if (in_syscall(regs)) {
684 continue_addr = regs->pc;684 continue_addr = regs->pc;
685 restart_addr = continue_addr - (compat_thumb_mode(regs) ? 2 : 4);685 restart_addr = continue_addr - (compat_thumb_mode(regs) ? 2 : 4);
686 retval = regs->regs[0];686 retval = regs->regs[0];
@@ -688,7 +688,7 @@ static void do_signal(struct pt_regs *regs) · 1 unmodified line
688 /*688 /*
689 * Avoid additional syscall restarting via ret_to_user.689 * Avoid additional syscall restarting via ret_to_user.
690 */690 */
691 regs->syscallno = ~0UL;691 forget_syscall(regs);
692 692
693 /*693 /*
694 * Prepare for system call restart. We do this here so that a694 * Prepare for system call restart. We do this here so that a
@@ -732,7 +732,7 @@ static void do_signal(struct pt_regs *regs) · 37 unmodified lines
732 * Handle restarting a different system call. As above, if a debugger732 * Handle restarting a different system call. As above, if a debugger
733 * has chosen to restart at a different PC, ignore the restart.733 * has chosen to restart at a different PC, ignore the restart.
734 */734 */
735 if (syscall >= 0 && regs->pc == restart_addr) {735 if (in_syscall(regs) && regs->pc == restart_addr) {
736 if (retval == -ERESTART_RESTARTBLOCK)736 if (retval == -ERESTART_RESTARTBLOCK)
737 setup_restart_syscall(regs);737 setup_restart_syscall(regs);
738 user_rewind_single_step(current);738 user_rewind_single_step(current);
arch/arm64/kernel/signal32.c+1−1
@@ -354,7 +354,7 @@ static int compat_restore_sigframe(struct pt_regs *regs, · 353 unmodified lines
354 /*354 /*
355 * Avoid compat_sys_sigreturn() restarting.355 * Avoid compat_sys_sigreturn() restarting.
356 */356 */
357 regs->syscallno = ~0UL;357 forget_syscall(regs);
358 358
359 err |= !valid_user_regs(&regs->user_regs, current);359 err |= !valid_user_regs(&regs->user_regs, current);
360 360
arch/arm64/kernel/smp.c+12−2
@@ -154,7 +154,7 @@ int __cpu_up(unsigned int cpu, struct task_struct *idle) · 153 unmodified lines
154 * page tables.154 * page tables.
155 */155 */
156 secondary_data.task = idle;156 secondary_data.task = idle;
157 secondary_data.stack = task_stack_page(idle) + THREAD_START_SP;157 secondary_data.stack = task_stack_page(idle) + THREAD_SIZE;
158 update_cpu_boot_status(CPU_MMU_OFF);158 update_cpu_boot_status(CPU_MMU_OFF);
159 __flush_dcache_area(&secondary_data, sizeof(secondary_data));159 __flush_dcache_area(&secondary_data, sizeof(secondary_data));
160 160
@@ -977,11 +977,21 @@ void smp_send_stop(void) · 816 unmodified lines
977}977}
978 978
979#ifdef CONFIG_KEXEC_CORE979#ifdef CONFIG_KEXEC_CORE
980void smp_send_crash_stop(void)980void crash_smp_send_stop(void)
981{981{
982 static int cpus_stopped;
982 cpumask_t mask;983 cpumask_t mask;
983 unsigned long timeout;984 unsigned long timeout;
984 985
986 /*
987 * This function can be called twice in panic path, but obviously
988 * we execute this only once.
989 */
990 if (cpus_stopped)
991 return;
992
993 cpus_stopped = 1;
994
985 if (num_online_cpus() == 1)995 if (num_online_cpus() == 1)
986 return;996 return;
987 997
arch/arm64/kernel/stacktrace.c+10−50
@@ -42,33 +42,17 @@ · 41 unmodified lines
42 */42 */
43int notrace unwind_frame(struct task_struct *tsk, struct stackframe *frame)43int notrace unwind_frame(struct task_struct *tsk, struct stackframe *frame)
44{44{
45 unsigned long high, low;
46 unsigned long fp = frame->fp;45 unsigned long fp = frame->fp;
47 unsigned long irq_stack_ptr;46
47 if (fp & 0xf)
48 return -EINVAL;
48 49
49 if (!tsk)50 if (!tsk)
50 tsk = current;51 tsk = current;
51 52
52 /*53 if (!on_accessible_stack(tsk, fp))
53 * Switching between stacks is valid when tracing current and in
54 * non-preemptible context.
55 */
56 if (tsk == current && !preemptible())
57 irq_stack_ptr = IRQ_STACK_PTR(smp_processor_id());
58 else
59 irq_stack_ptr = 0;
60
61 low = frame->sp;
62 /* irq stacks are not THREAD_SIZE aligned */
63 if (on_irq_stack(frame->sp, raw_smp_processor_id()))
64 high = irq_stack_ptr;
65 else
66 high = ALIGN(low, THREAD_SIZE) - 0x20;
67
68 if (fp < low || fp > high || fp & 0xf)
69 return -EINVAL;54 return -EINVAL;
70 55
71 frame->sp = fp + 0x10;
72 frame->fp = READ_ONCE_NOCHECK(*(unsigned long *)(fp));56 frame->fp = READ_ONCE_NOCHECK(*(unsigned long *)(fp));
73 frame->pc = READ_ONCE_NOCHECK(*(unsigned long *)(fp + 8));57 frame->pc = READ_ONCE_NOCHECK(*(unsigned long *)(fp + 8));
74 58
@@ -86,34 +70,13 @@ int notrace unwind_frame(struct task_struct *tsk, struct stackframe *frame) · 11 unmodified lines
86#endif /* CONFIG_FUNCTION_GRAPH_TRACER */70#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
87 71
88 /*72 /*
89 * Check whether we are going to walk through from interrupt stack73 * Frames created upon entry from EL0 have NULL FP and PC values, so
90 * to task stack.74 * don't bother reporting these. Frames created by __noreturn functions
91 * If we reach the end of the stack - and its an interrupt stack,75 * might have a valid FP even if PC is bogus, so only terminate where
92 * unpack the dummy frame to find the original elr.76 * both are NULL.
93 *
94 * Check the frame->fp we read from the bottom of the irq_stack,
95 * and the original task stack pointer are both in current->stack.
96 */77 */
97 if (frame->sp == irq_stack_ptr) {78 if (!frame->fp && !frame->pc)
98 struct pt_regs *irq_args;79 return -EINVAL;
99 unsigned long orig_sp = IRQ_STACK_TO_TASK_STACK(irq_stack_ptr);
100
101 if (object_is_on_stack((void *)orig_sp) &&
102 object_is_on_stack((void *)frame->fp)) {
103 frame->sp = orig_sp;
104
105 /* orig_sp is the saved pt_regs, find the elr */
106 irq_args = (struct pt_regs *)orig_sp;
107 frame->pc = irq_args->pc;
108 } else {
109 /*
110 * This frame has a non-standard format, and we
111 * didn't fix it, because the data looked wrong.
112 * Refuse to output this frame.
113 */
114 return -EINVAL;
115 }
116 }
117 80
118 return 0;81 return 0;
119}82}
@@ -167,7 +130,6 @@ void save_stack_trace_regs(struct pt_regs *regs, struct stack_trace *trace) · 47 unmodified lines
167 data.no_sched_functions = 0;130 data.no_sched_functions = 0;
168 131
169 frame.fp = regs->regs[29];132 frame.fp = regs->regs[29];
170 frame.sp = regs->sp;
171 frame.pc = regs->pc;133 frame.pc = regs->pc;
172#ifdef CONFIG_FUNCTION_GRAPH_TRACER134#ifdef CONFIG_FUNCTION_GRAPH_TRACER
173 frame.graph = current->curr_ret_stack;135 frame.graph = current->curr_ret_stack;
@@ -192,12 +154,10 @@ void save_stack_trace_tsk(struct task_struct *tsk, struct stack_trace *trace) · 18 unmodified lines
192 if (tsk != current) {154 if (tsk != current) {
193 data.no_sched_functions = 1;155 data.no_sched_functions = 1;
194 frame.fp = thread_saved_fp(tsk);156 frame.fp = thread_saved_fp(tsk);
195 frame.sp = thread_saved_sp(tsk);
196 frame.pc = thread_saved_pc(tsk);157 frame.pc = thread_saved_pc(tsk);
197 } else {158 } else {
198 data.no_sched_functions = 0;159 data.no_sched_functions = 0;
199 frame.fp = (unsigned long)__builtin_frame_address(0);160 frame.fp = (unsigned long)__builtin_frame_address(0);
200 frame.sp = current_stack_pointer;
201 frame.pc = (unsigned long)save_stack_trace_tsk;161 frame.pc = (unsigned long)save_stack_trace_tsk;
202 }162 }
203#ifdef CONFIG_FUNCTION_GRAPH_TRACER163#ifdef CONFIG_FUNCTION_GRAPH_TRACER
arch/arm64/kernel/time.c+0−1
@@ -50,7 +50,6 @@ unsigned long profile_pc(struct pt_regs *regs) · 49 unmodified lines
50 return regs->pc;50 return regs->pc;
51 51
52 frame.fp = regs->regs[29];52 frame.fp = regs->regs[29];
53 frame.sp = regs->sp;
54 frame.pc = regs->pc;53 frame.pc = regs->pc;
55#ifdef CONFIG_FUNCTION_GRAPH_TRACER54#ifdef CONFIG_FUNCTION_GRAPH_TRACER
56 frame.graph = -1; /* no task info */55 frame.graph = -1; /* no task info */
arch/arm64/kernel/traps.c+49−30
@@ -32,6 +32,7 @@ · 31 unmodified lines
32#include <linux/sched/signal.h>32#include <linux/sched/signal.h>
33#include <linux/sched/debug.h>33#include <linux/sched/debug.h>
34#include <linux/sched/task_stack.h>34#include <linux/sched/task_stack.h>
35#include <linux/sizes.h>
35#include <linux/syscalls.h>36#include <linux/syscalls.h>
36#include <linux/mm_types.h>37#include <linux/mm_types.h>
37 38
@@ -41,6 +42,7 @@ · 3 unmodified lines
41#include <asm/esr.h>42#include <asm/esr.h>
42#include <asm/insn.h>43#include <asm/insn.h>
43#include <asm/traps.h>44#include <asm/traps.h>
45#include <asm/smp.h>
44#include <asm/stack_pointer.h>46#include <asm/stack_pointer.h>
45#include <asm/stacktrace.h>47#include <asm/stacktrace.h>
46#include <asm/exception.h>48#include <asm/exception.h>
@@ -143,7 +145,6 @@ static void dump_instr(const char *lvl, struct pt_regs *regs) · 96 unmodified lines
143void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk)145void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk)
144{146{
145 struct stackframe frame;147 struct stackframe frame;
146 unsigned long irq_stack_ptr;
147 int skip;148 int skip;
148 149
149 pr_debug("%s(regs = %p tsk = %p)\n", __func__, regs, tsk);150 pr_debug("%s(regs = %p tsk = %p)\n", __func__, regs, tsk);
@@ -154,25 +155,14 @@ void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk) · 4 unmodified lines
154 if (!try_get_task_stack(tsk))155 if (!try_get_task_stack(tsk))
155 return;156 return;
156 157
157 /*
158 * Switching between stacks is valid when tracing current and in
159 * non-preemptible context.
160 */
161 if (tsk == current && !preemptible())
162 irq_stack_ptr = IRQ_STACK_PTR(smp_processor_id());
163 else
164 irq_stack_ptr = 0;
165
166 if (tsk == current) {158 if (tsk == current) {
167 frame.fp = (unsigned long)__builtin_frame_address(0);159 frame.fp = (unsigned long)__builtin_frame_address(0);
168 frame.sp = current_stack_pointer;
169 frame.pc = (unsigned long)dump_backtrace;160 frame.pc = (unsigned long)dump_backtrace;
170 } else {161 } else {
171 /*162 /*
172 * task blocked in __switch_to163 * task blocked in __switch_to
173 */164 */
174 frame.fp = thread_saved_fp(tsk);165 frame.fp = thread_saved_fp(tsk);
175 frame.sp = thread_saved_sp(tsk);
176 frame.pc = thread_saved_pc(tsk);166 frame.pc = thread_saved_pc(tsk);
177 }167 }
178#ifdef CONFIG_FUNCTION_GRAPH_TRACER168#ifdef CONFIG_FUNCTION_GRAPH_TRACER
@@ -182,13 +172,12 @@ void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk) · 3 unmodified lines
182 skip = !!regs;172 skip = !!regs;
183 printk("Call trace:\n");173 printk("Call trace:\n");
184 while (1) {174 while (1) {
185 unsigned long where = frame.pc;
186 unsigned long stack;175 unsigned long stack;
187 int ret;176 int ret;
188 177
189 /* skip until specified stack frame */178 /* skip until specified stack frame */
190 if (!skip) {179 if (!skip) {
191 dump_backtrace_entry(where);180 dump_backtrace_entry(frame.pc);
192 } else if (frame.fp == regs->regs[29]) {181 } else if (frame.fp == regs->regs[29]) {
193 skip = 0;182 skip = 0;
194 /*183 /*
@@ -203,20 +192,12 @@ void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk) · 8 unmodified lines
203 ret = unwind_frame(tsk, &frame);192 ret = unwind_frame(tsk, &frame);
204 if (ret < 0)193 if (ret < 0)
205 break;194 break;
206 stack = frame.sp;195 if (in_entry_text(frame.pc)) {
207 if (in_exception_text(where)) {196 stack = frame.fp - offsetof(struct pt_regs, stackframe);
208 /*
209 * If we switched to the irq_stack before calling this
210 * exception handler, then the pt_regs will be on the
211 * task stack. The easiest way to tell is if the large
212 * pt_regs would overlap with the end of the irq_stack.
213 */
214 if (stack < irq_stack_ptr &&
215 (stack + sizeof(struct pt_regs)) > irq_stack_ptr)
216 stack = IRQ_STACK_TO_TASK_STACK(irq_stack_ptr);
217 197
218 dump_mem("", "Exception stack", stack,198 if (on_accessible_stack(tsk, stack))
219 stack + sizeof(struct pt_regs));199 dump_mem("", "Exception stack", stack,
200 stack + sizeof(struct pt_regs));
220 }201 }
221 }202 }
222 203
@@ -257,8 +238,6 @@ static int __die(const char *str, int err, struct pt_regs *regs) · 34 unmodified lines
257 end_of_stack(tsk));238 end_of_stack(tsk));
258 239
259 if (!user_mode(regs)) {240 if (!user_mode(regs)) {
260 dump_mem(KERN_EMERG, "Stack: ", regs->sp,
261 THREAD_SIZE + (unsigned long)task_stack_page(tsk));
262 dump_backtrace(regs, tsk);241 dump_backtrace(regs, tsk);
263 dump_instr(KERN_EMERG, regs);242 dump_instr(KERN_EMERG, regs);
264 }243 }
@@ -484,6 +463,9 @@ static void user_cache_maint_handler(unsigned int esr, struct pt_regs *regs) · 219 unmodified lines
484 case ESR_ELx_SYS64_ISS_CRM_DC_CVAC: /* DC CVAC, gets promoted */463 case ESR_ELx_SYS64_ISS_CRM_DC_CVAC: /* DC CVAC, gets promoted */
485 __user_cache_maint("dc civac", address, ret);464 __user_cache_maint("dc civac", address, ret);
486 break;465 break;
466 case ESR_ELx_SYS64_ISS_CRM_DC_CVAP: /* DC CVAP */
467 __user_cache_maint("sys 3, c7, c12, 1", address, ret);
468 break;
487 case ESR_ELx_SYS64_ISS_CRM_DC_CIVAC: /* DC CIVAC */469 case ESR_ELx_SYS64_ISS_CRM_DC_CIVAC: /* DC CIVAC */
488 __user_cache_maint("dc civac", address, ret);470 __user_cache_maint("dc civac", address, ret);
489 break;471 break;
@@ -593,7 +575,7 @@ asmlinkage long do_ni_syscall(struct pt_regs *regs) · 103 unmodified lines
593 575
594 if (show_unhandled_signals_ratelimited()) {576 if (show_unhandled_signals_ratelimited()) {
595 pr_info("%s[%d]: syscall %d\n", current->comm,577 pr_info("%s[%d]: syscall %d\n", current->comm,
596 task_pid_nr(current), (int)regs->syscallno);578 task_pid_nr(current), regs->syscallno);
597 dump_instr("", regs);579 dump_instr("", regs);
598 if (user_mode(regs))580 if (user_mode(regs))
599 __show_regs(regs);581 __show_regs(regs);
@@ -689,6 +671,43 @@ asmlinkage void bad_el0_sync(struct pt_regs *regs, int reason, unsigned int esr) · 89 unmodified lines
689 force_sig_info(info.si_signo, &info, current);671 force_sig_info(info.si_signo, &info, current);
690}672}
691 673
674#ifdef CONFIG_VMAP_STACK
675
676DEFINE_PER_CPU(unsigned long [OVERFLOW_STACK_SIZE/sizeof(long)], overflow_stack)
677 __aligned(16);
678
679asmlinkage void handle_bad_stack(struct pt_regs *regs)
680{
681 unsigned long tsk_stk = (unsigned long)current->stack;
682 unsigned long irq_stk = (unsigned long)this_cpu_read(irq_stack_ptr);
683 unsigned long ovf_stk = (unsigned long)this_cpu_ptr(overflow_stack);
684 unsigned int esr = read_sysreg(esr_el1);
685 unsigned long far = read_sysreg(far_el1);
686
687 console_verbose();
688 pr_emerg("Insufficient stack space to handle exception!");
689
690 pr_emerg("ESR: 0x%08x -- %s\n", esr, esr_get_class_string(esr));
691 pr_emerg("FAR: 0x%016lx\n", far);
692
693 pr_emerg("Task stack: [0x%016lx..0x%016lx]\n",
694 tsk_stk, tsk_stk + THREAD_SIZE);
695 pr_emerg("IRQ stack: [0x%016lx..0x%016lx]\n",
696 irq_stk, irq_stk + THREAD_SIZE);
697 pr_emerg("Overflow stack: [0x%016lx..0x%016lx]\n",
698 ovf_stk, ovf_stk + OVERFLOW_STACK_SIZE);
699
700 __show_regs(regs);
701
702 /*
703 * We use nmi_panic to limit the potential for recusive overflows, and
704 * to get a better stack trace.
705 */
706 nmi_panic(NULL, "kernel stack overflow");
707 cpu_park_loop();
708}
709#endif
710
692void __pte_error(const char *file, int line, unsigned long val)711void __pte_error(const char *file, int line, unsigned long val)
693{712{
694 pr_err("%s:%d: bad pte %016lx.\n", file, line, val);713 pr_err("%s:%d: bad pte %016lx.\n", file, line, val);
arch/arm64/kernel/vdso.c+15−0
@@ -110,12 +110,27 @@ int aarch32_setup_vectors_page(struct linux_binprm *bprm, int uses_interp) · 109 unmodified lines
110}110}
111#endif /* CONFIG_COMPAT */111#endif /* CONFIG_COMPAT */
112 112
113static int vdso_mremap(const struct vm_special_mapping *sm,
114 struct vm_area_struct *new_vma)
115{
116 unsigned long new_size = new_vma->vm_end - new_vma->vm_start;
117 unsigned long vdso_size = vdso_end - vdso_start;
118
119 if (vdso_size != new_size)
120 return -EINVAL;
121
122 current->mm->context.vdso = (void *)new_vma->vm_start;
123
124 return 0;
125}
126
113static struct vm_special_mapping vdso_spec[2] __ro_after_init = {127static struct vm_special_mapping vdso_spec[2] __ro_after_init = {
114 {128 {
115 .name = "[vvar]",129 .name = "[vvar]",
116 },130 },
117 {131 {
118 .name = "[vdso]",132 .name = "[vdso]",
133 .mremap = vdso_mremap,
119 },134 },
120};135};
121 136
arch/arm64/kernel/vmlinux.lds.S+1−17
@@ -72,22 +72,6 @@ PECOFF_FILE_ALIGNMENT = 0x200; · 71 unmodified lines
72#define PECOFF_EDATA_PADDING72#define PECOFF_EDATA_PADDING
73#endif73#endif
74 74
75#if defined(CONFIG_DEBUG_ALIGN_RODATA)
76/*
77 * 4 KB granule: 1 level 2 entry
78 * 16 KB granule: 128 level 3 entries, with contiguous bit
79 * 64 KB granule: 32 level 3 entries, with contiguous bit
80 */
81#define SEGMENT_ALIGN SZ_2M
82#else
83/*
84 * 4 KB granule: 16 level 3 entries, with contiguous bit
85 * 16 KB granule: 4 level 3 entries, without contiguous bit
86 * 64 KB granule: 1 level 3 entry
87 */
88#define SEGMENT_ALIGN SZ_64K
89#endif
90
91SECTIONS75SECTIONS
92{76{
93 /*77 /*
@@ -192,7 +176,7 @@ SECTIONS · 98 unmodified lines
192 176
193 _data = .;177 _data = .;
194 _sdata = .;178 _sdata = .;
195 RW_DATA_SECTION(L1_CACHE_BYTES, PAGE_SIZE, THREAD_SIZE)179 RW_DATA_SECTION(L1_CACHE_BYTES, PAGE_SIZE, THREAD_ALIGN)
196 180
197 /*181 /*
198 * Data written with the MMU off but read with the MMU on requires182 * Data written with the MMU off but read with the MMU on requires
arch/arm64/kvm/hyp/s2-setup.c+1−1
@@ -70,7 +70,7 @@ u32 __hyp_text __init_stage2_translation(void) · 69 unmodified lines
70 * Management in ID_AA64MMFR1_EL1 and enable the feature in VTCR_EL2.70 * Management in ID_AA64MMFR1_EL1 and enable the feature in VTCR_EL2.
71 */71 */
72 tmp = (read_sysreg(id_aa64mmfr1_el1) >> ID_AA64MMFR1_HADBS_SHIFT) & 0xf;72 tmp = (read_sysreg(id_aa64mmfr1_el1) >> ID_AA64MMFR1_HADBS_SHIFT) & 0xf;
73 if (IS_ENABLED(CONFIG_ARM64_HW_AFDBM) && tmp)73 if (tmp)
74 val |= VTCR_EL2_HA;74 val |= VTCR_EL2_HA;
75 75
76 /*76 /*
arch/arm64/lib/Makefile+2−0
@@ -17,3 +17,5 @@ CFLAGS_atomic_ll_sc.o := -fcall-used-x0 -ffixed-x1 -ffixed-x2 \ · 16 unmodified lines
17 -fcall-saved-x10 -fcall-saved-x11 -fcall-saved-x12 \17 -fcall-saved-x10 -fcall-saved-x11 -fcall-saved-x12 \
18 -fcall-saved-x13 -fcall-saved-x14 -fcall-saved-x15 \18 -fcall-saved-x13 -fcall-saved-x14 -fcall-saved-x15 \
19 -fcall-saved-x1819 -fcall-saved-x18
20
21lib-$(CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE) += uaccess_flushcache.o
arch/arm64/lib/uaccess_flushcache.c+47−0
@@ -0,0 +1,47 @@
1/*
2 * Copyright (C) 2017 ARM Ltd.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 * GNU General Public License for more details.
12 *
13 * You should have received a copy of the GNU General Public License
14 * along with this program. If not, see <http://www.gnu.org/licenses/>.
15 */
16
17#include <linux/uaccess.h>
18#include <asm/barrier.h>
19#include <asm/cacheflush.h>
20
21void memcpy_flushcache(void *dst, const void *src, size_t cnt)
22{
23 /*
24 * We assume this should not be called with @dst pointing to
25 * non-cacheable memory, such that we don't need an explicit
26 * barrier to order the cache maintenance against the memcpy.
27 */
28 memcpy(dst, src, cnt);
29 __clean_dcache_area_pop(dst, cnt);
30}
31EXPORT_SYMBOL_GPL(memcpy_flushcache);
32
33void memcpy_page_flushcache(char *to, struct page *page, size_t offset,
34 size_t len)
35{
36 memcpy_flushcache(to, page_address(page) + offset, len);
37}
38
39unsigned long __copy_user_flushcache(void *to, const void __user *from,
40 unsigned long n)
41{
42 unsigned long rc = __arch_copy_from_user(to, from, n);
43
44 /* See above */
45 __clean_dcache_area_pop(to, n - rc);
46 return rc;
47}
arch/arm64/mm/cache.S+28−9
@@ -109,20 +109,25 @@ ENTRY(__clean_dcache_area_pou) · 108 unmodified lines
109ENDPROC(__clean_dcache_area_pou)109ENDPROC(__clean_dcache_area_pou)
110 110
111/*111/*
112 * __dma_inv_area(start, size)112 * __inval_dcache_area(kaddr, size)
113 * - start - virtual start address of region113 *
114 * Ensure that any D-cache lines for the interval [kaddr, kaddr+size)
115 * are invalidated. Any partial lines at the ends of the interval are
116 * also cleaned to PoC to prevent data loss.
117 *
118 * - kaddr - kernel address
114 * - size - size in question119 * - size - size in question
115 */120 */
116__dma_inv_area:121ENTRY(__inval_dcache_area)
117 add x1, x1, x0
118 /* FALLTHROUGH */122 /* FALLTHROUGH */
119 123
120/*124/*
121 * __inval_cache_range(start, end)125 * __dma_inv_area(start, size)
122 * - start - start address of region126 * - start - virtual start address of region
123 * - end - end address of region127 * - size - size in question
124 */128 */
125ENTRY(__inval_cache_range)129__dma_inv_area:
130 add x1, x1, x0
126 dcache_line_size x2, x3131 dcache_line_size x2, x3
127 sub x3, x2, #1132 sub x3, x2, #1
128 tst x1, x3 // end cache line aligned?133 tst x1, x3 // end cache line aligned?
@@ -140,7 +145,7 @@ ENTRY(__inval_cache_range) · 11 unmodified lines
140 b.lo 2b145 b.lo 2b
141 dsb sy146 dsb sy
142 ret147 ret
143ENDPIPROC(__inval_cache_range)148ENDPIPROC(__inval_dcache_area)
144ENDPROC(__dma_inv_area)149ENDPROC(__dma_inv_area)
145 150
146/*151/*
@@ -166,6 +171,20 @@ __dma_clean_area: · 19 unmodified lines
166ENDPIPROC(__clean_dcache_area_poc)171ENDPIPROC(__clean_dcache_area_poc)
167ENDPROC(__dma_clean_area)172ENDPROC(__dma_clean_area)
168 173
174/*
175 * __clean_dcache_area_pop(kaddr, size)
176 *
177 * Ensure that any D-cache lines for the interval [kaddr, kaddr+size)
178 * are cleaned to the PoP.
179 *
180 * - kaddr - kernel address
181 * - size - size in question
182 */
183ENTRY(__clean_dcache_area_pop)
184 dcache_by_line_op cvap, sy, x0, x1, x2, x3
185 ret
186ENDPIPROC(__clean_dcache_area_pop)
187
169/*188/*
170 * __dma_flush_area(start, size)189 * __dma_flush_area(start, size)
171 *190 *
arch/arm64/mm/dma-mapping.c+2−2
@@ -42,7 +42,7 @@ static pgprot_t __get_dma_pgprot(unsigned long attrs, pgprot_t prot, · 41 unmodified lines
42 return prot;42 return prot;
43}43}
44 44
45static struct gen_pool *atomic_pool;45static struct gen_pool *atomic_pool __ro_after_init;
46 46
47#define DEFAULT_DMA_COHERENT_POOL_SIZE SZ_256K47#define DEFAULT_DMA_COHERENT_POOL_SIZE SZ_256K
48static size_t atomic_pool_size __initdata = DEFAULT_DMA_COHERENT_POOL_SIZE;48static size_t atomic_pool_size __initdata = DEFAULT_DMA_COHERENT_POOL_SIZE;
@@ -425,7 +425,7 @@ static int __init atomic_pool_init(void) · 376 unmodified lines
425 425
426 gen_pool_set_algo(atomic_pool,426 gen_pool_set_algo(atomic_pool,
427 gen_pool_first_fit_order_align,427 gen_pool_first_fit_order_align,
428 (void *)PAGE_SHIFT);428 NULL);
429 429
430 pr_info("DMA: preallocated %zu KiB pool for atomic allocations\n",430 pr_info("DMA: preallocated %zu KiB pool for atomic allocations\n",
431 atomic_pool_size / 1024);431 atomic_pool_size / 1024);
arch/arm64/mm/fault.c+58−19
@@ -34,6 +34,7 @@ · 33 unmodified lines
34#include <linux/hugetlb.h>34#include <linux/hugetlb.h>
35 35
36#include <asm/bug.h>36#include <asm/bug.h>
37#include <asm/cmpxchg.h>
37#include <asm/cpufeature.h>38#include <asm/cpufeature.h>
38#include <asm/exception.h>39#include <asm/exception.h>
39#include <asm/debug-monitors.h>40#include <asm/debug-monitors.h>
@@ -82,6 +83,49 @@ static inline int notify_page_fault(struct pt_regs *regs, unsigned int esr) · 42 unmodified lines
82}83}
83#endif84#endif
84 85
86static void data_abort_decode(unsigned int esr)
87{
88 pr_alert("Data abort info:\n");
89
90 if (esr & ESR_ELx_ISV) {
91 pr_alert(" Access size = %u byte(s)\n",
92 1U << ((esr & ESR_ELx_SAS) >> ESR_ELx_SAS_SHIFT));
93 pr_alert(" SSE = %lu, SRT = %lu\n",
94 (esr & ESR_ELx_SSE) >> ESR_ELx_SSE_SHIFT,
95 (esr & ESR_ELx_SRT_MASK) >> ESR_ELx_SRT_SHIFT);
96 pr_alert(" SF = %lu, AR = %lu\n",
97 (esr & ESR_ELx_SF) >> ESR_ELx_SF_SHIFT,
98 (esr & ESR_ELx_AR) >> ESR_ELx_AR_SHIFT);
99 } else {
100 pr_alert(" ISV = 0, ISS = 0x%08lu\n", esr & ESR_ELx_ISS_MASK);
101 }
102
103 pr_alert(" CM = %lu, WnR = %lu\n",
104 (esr & ESR_ELx_CM) >> ESR_ELx_CM_SHIFT,
105 (esr & ESR_ELx_WNR) >> ESR_ELx_WNR_SHIFT);
106}
107
108/*
109 * Decode mem abort information
110 */
111static void mem_abort_decode(unsigned int esr)
112{
113 pr_alert("Mem abort info:\n");
114
115 pr_alert(" Exception class = %s, IL = %u bits\n",
116 esr_get_class_string(esr),
117 (esr & ESR_ELx_IL) ? 32 : 16);
118 pr_alert(" SET = %lu, FnV = %lu\n",
119 (esr & ESR_ELx_SET_MASK) >> ESR_ELx_SET_SHIFT,
120 (esr & ESR_ELx_FnV) >> ESR_ELx_FnV_SHIFT);
121 pr_alert(" EA = %lu, S1PTW = %lu\n",
122 (esr & ESR_ELx_EA) >> ESR_ELx_EA_SHIFT,
123 (esr & ESR_ELx_S1PTW) >> ESR_ELx_S1PTW_SHIFT);
124
125 if (esr_is_data_abort(esr))
126 data_abort_decode(esr);
127}
128
85/*129/*
86 * Dump out the page tables associated with 'addr' in the currently active mm.130 * Dump out the page tables associated with 'addr' in the currently active mm.
87 */131 */
@@ -139,7 +183,6 @@ void show_pte(unsigned long addr) · 51 unmodified lines
139 pr_cont("\n");183 pr_cont("\n");
140}184}
141 185
142#ifdef CONFIG_ARM64_HW_AFDBM
143/*186/*
144 * This function sets the access flags (dirty, accessed), as well as write187 * This function sets the access flags (dirty, accessed), as well as write
145 * permission, and only to a more permissive setting.188 * permission, and only to a more permissive setting.
@@ -154,18 +197,13 @@ int ptep_set_access_flags(struct vm_area_struct *vma, · 8 unmodified lines
154 unsigned long address, pte_t *ptep,197 unsigned long address, pte_t *ptep,
155 pte_t entry, int dirty)198 pte_t entry, int dirty)
156{199{
157 pteval_t old_pteval;200 pteval_t old_pteval, pteval;
158 unsigned int tmp;
159 201
160 if (pte_same(*ptep, entry))202 if (pte_same(*ptep, entry))
161 return 0;203 return 0;
162 204
163 /* only preserve the access flags and write permission */205 /* only preserve the access flags and write permission */
164 pte_val(entry) &= PTE_AF | PTE_WRITE | PTE_DIRTY;206 pte_val(entry) &= PTE_RDONLY | PTE_AF | PTE_WRITE | PTE_DIRTY;
165
166 /* set PTE_RDONLY if actual read-only or clean PTE */
167 if (!pte_write(entry) || !pte_sw_dirty(entry))
168 pte_val(entry) |= PTE_RDONLY;
169 207
170 /*208 /*
171 * Setting the flags must be done atomically to avoid racing with the209 * Setting the flags must be done atomically to avoid racing with the
@@ -174,21 +212,18 @@ int ptep_set_access_flags(struct vm_area_struct *vma, · 2 unmodified lines
174 * (calculated as: a & b == ~(~a | ~b)).212 * (calculated as: a & b == ~(~a | ~b)).
175 */213 */
176 pte_val(entry) ^= PTE_RDONLY;214 pte_val(entry) ^= PTE_RDONLY;
177 asm volatile("// ptep_set_access_flags\n"215 pteval = READ_ONCE(pte_val(*ptep));
178 " prfm pstl1strm, %2\n"216 do {
179 "1: ldxr %0, %2\n"217 old_pteval = pteval;
180 " eor %0, %0, %3 // negate PTE_RDONLY in *ptep\n"218 pteval ^= PTE_RDONLY;
181 " orr %0, %0, %4 // set flags\n"219 pteval |= pte_val(entry);
182 " eor %0, %0, %3 // negate final PTE_RDONLY\n"220 pteval ^= PTE_RDONLY;
183 " stxr %w1, %0, %2\n"221 pteval = cmpxchg_relaxed(&pte_val(*ptep), old_pteval, pteval);
184 " cbnz %w1, 1b\n"222 } while (pteval != old_pteval);
185 : "=&r" (old_pteval), "=&r" (tmp), "+Q" (pte_val(*ptep))
186 : "L" (PTE_RDONLY), "r" (pte_val(entry)));
187 223
188 flush_tlb_fix_spurious_fault(vma, address);224 flush_tlb_fix_spurious_fault(vma, address);
189 return 1;225 return 1;
190}226}
191#endif
192 227
193static bool is_el1_instruction_abort(unsigned int esr)228static bool is_el1_instruction_abort(unsigned int esr)
194{229{
@@ -248,6 +283,8 @@ static void __do_kernel_fault(unsigned long addr, unsigned int esr, · 53 unmodified lines
248 pr_alert("Unable to handle kernel %s at virtual address %08lx\n", msg,283 pr_alert("Unable to handle kernel %s at virtual address %08lx\n", msg,
249 addr);284 addr);
250 285
286 mem_abort_decode(esr);
287
251 show_pte(addr);288 show_pte(addr);
252 die("Oops", regs, esr);289 die("Oops", regs, esr);
253 bust_spinlocks(0);290 bust_spinlocks(0);
@@ -705,6 +742,8 @@ asmlinkage void __exception do_mem_abort(unsigned long addr, unsigned int esr, · 451 unmodified lines
705 pr_alert("Unhandled fault: %s (0x%08x) at 0x%016lx\n",742 pr_alert("Unhandled fault: %s (0x%08x) at 0x%016lx\n",
706 inf->name, esr, addr);743 inf->name, esr, addr);
707 744
745 mem_abort_decode(esr);
746
708 info.si_signo = inf->sig;747 info.si_signo = inf->sig;
709 info.si_errno = 0;748 info.si_errno = 0;
710 info.si_code = inf->code;749 info.si_code = inf->code;
arch/arm64/mm/flush.c+16−0
@@ -83,3 +83,19 @@ EXPORT_SYMBOL(flush_dcache_page); · 82 unmodified lines
83 * Additional functions defined in assembly.83 * Additional functions defined in assembly.
84 */84 */
85EXPORT_SYMBOL(flush_icache_range);85EXPORT_SYMBOL(flush_icache_range);
86
87#ifdef CONFIG_ARCH_HAS_PMEM_API
88void arch_wb_cache_pmem(void *addr, size_t size)
89{
90 /* Ensure order against any prior non-cacheable writes */
91 dmb(osh);
92 __clean_dcache_area_pop(addr, size);
93}
94EXPORT_SYMBOL_GPL(arch_wb_cache_pmem);
95
96void arch_invalidate_pmem(void *addr, size_t size)
97{
98 __inval_dcache_area(addr, size);
99}
100EXPORT_SYMBOL_GPL(arch_invalidate_pmem);
101#endif
arch/arm64/mm/hugetlbpage.c+230−84
@@ -41,6 +41,16 @@ int pud_huge(pud_t pud) · 40 unmodified lines
41#endif41#endif
42}42}
43 43
44/*
45 * Select all bits except the pfn
46 */
47static inline pgprot_t pte_pgprot(pte_t pte)
48{
49 unsigned long pfn = pte_pfn(pte);
50
51 return __pgprot(pte_val(pfn_pte(pfn, __pgprot(0))) ^ pte_val(pte));
52}
53
44static int find_num_contig(struct mm_struct *mm, unsigned long addr,54static int find_num_contig(struct mm_struct *mm, unsigned long addr,
45 pte_t *ptep, size_t *pgsize)55 pte_t *ptep, size_t *pgsize)
46{56{
@@ -58,15 +68,107 @@ static int find_num_contig(struct mm_struct *mm, unsigned long addr, · 11 unmodified lines
58 return CONT_PTES;68 return CONT_PTES;
59}69}
60 70
71static inline int num_contig_ptes(unsigned long size, size_t *pgsize)
72{
73 int contig_ptes = 0;
74
75 *pgsize = size;
76
77 switch (size) {
78#ifdef CONFIG_ARM64_4K_PAGES
79 case PUD_SIZE:
80#endif
81 case PMD_SIZE:
82 contig_ptes = 1;
83 break;
84 case CONT_PMD_SIZE:
85 *pgsize = PMD_SIZE;
86 contig_ptes = CONT_PMDS;
87 break;
88 case CONT_PTE_SIZE:
89 *pgsize = PAGE_SIZE;
90 contig_ptes = CONT_PTES;
91 break;
92 }
93
94 return contig_ptes;
95}
96
97/*
98 * Changing some bits of contiguous entries requires us to follow a
99 * Break-Before-Make approach, breaking the whole contiguous set
100 * before we can change any entries. See ARM DDI 0487A.k_iss10775,
101 * "Misprogramming of the Contiguous bit", page D4-1762.
102 *
103 * This helper performs the break step.
104 */
105static pte_t get_clear_flush(struct mm_struct *mm,
106 unsigned long addr,
107 pte_t *ptep,
108 unsigned long pgsize,
109 unsigned long ncontig)
110{
111 struct vm_area_struct vma = { .vm_mm = mm };
112 pte_t orig_pte = huge_ptep_get(ptep);
113 bool valid = pte_valid(orig_pte);
114 unsigned long i, saddr = addr;
115
116 for (i = 0; i < ncontig; i++, addr += pgsize, ptep++) {
117 pte_t pte = ptep_get_and_clear(mm, addr, ptep);
118
119 /*
120 * If HW_AFDBM is enabled, then the HW could turn on
121 * the dirty bit for any page in the set, so check
122 * them all. All hugetlb entries are already young.
123 */
124 if (pte_dirty(pte))
125 orig_pte = pte_mkdirty(orig_pte);
126 }
127
128 if (valid)
129 flush_tlb_range(&vma, saddr, addr);
130 return orig_pte;
131}
132
133/*
134 * Changing some bits of contiguous entries requires us to follow a
135 * Break-Before-Make approach, breaking the whole contiguous set
136 * before we can change any entries. See ARM DDI 0487A.k_iss10775,
137 * "Misprogramming of the Contiguous bit", page D4-1762.
138 *
139 * This helper performs the break step for use cases where the
140 * original pte is not needed.
141 */
142static void clear_flush(struct mm_struct *mm,
143 unsigned long addr,
144 pte_t *ptep,
145 unsigned long pgsize,
146 unsigned long ncontig)
147{
148 struct vm_area_struct vma = { .vm_mm = mm };
149 unsigned long i, saddr = addr;
150
151 for (i = 0; i < ncontig; i++, addr += pgsize, ptep++)
152 pte_clear(mm, addr, ptep);
153
154 flush_tlb_range(&vma, saddr, addr);
155}
156
61void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,157void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
62 pte_t *ptep, pte_t pte)158 pte_t *ptep, pte_t pte)
63{159{
64 size_t pgsize;160 size_t pgsize;
65 int i;161 int i;
66 int ncontig;162 int ncontig;
67 unsigned long pfn;163 unsigned long pfn, dpfn;
68 pgprot_t hugeprot;164 pgprot_t hugeprot;
69 165
166 /*
167 * Code needs to be expanded to handle huge swap and migration
168 * entries. Needed for HUGETLB and MEMORY_FAILURE.
169 */
170 WARN_ON(!pte_present(pte));
171
70 if (!pte_cont(pte)) {172 if (!pte_cont(pte)) {
71 set_pte_at(mm, addr, ptep, pte);173 set_pte_at(mm, addr, ptep, pte);
72 return;174 return;
@@ -74,17 +176,30 @@ void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, · 1 unmodified line
74 176
75 ncontig = find_num_contig(mm, addr, ptep, &pgsize);177 ncontig = find_num_contig(mm, addr, ptep, &pgsize);
76 pfn = pte_pfn(pte);178 pfn = pte_pfn(pte);
77 hugeprot = __pgprot(pte_val(pfn_pte(pfn, __pgprot(0))) ^ pte_val(pte));179 dpfn = pgsize >> PAGE_SHIFT;
78 for (i = 0; i < ncontig; i++) {180 hugeprot = pte_pgprot(pte);
181
182 clear_flush(mm, addr, ptep, pgsize, ncontig);
183
184 for (i = 0; i < ncontig; i++, ptep++, addr += pgsize, pfn += dpfn) {
79 pr_debug("%s: set pte %p to 0x%llx\n", __func__, ptep,185 pr_debug("%s: set pte %p to 0x%llx\n", __func__, ptep,
80 pte_val(pfn_pte(pfn, hugeprot)));186 pte_val(pfn_pte(pfn, hugeprot)));
81 set_pte_at(mm, addr, ptep, pfn_pte(pfn, hugeprot));187 set_pte_at(mm, addr, ptep, pfn_pte(pfn, hugeprot));
82 ptep++;
83 pfn += pgsize >> PAGE_SHIFT;
84 addr += pgsize;
85 }188 }
86}189}
87 190
191void set_huge_swap_pte_at(struct mm_struct *mm, unsigned long addr,
192 pte_t *ptep, pte_t pte, unsigned long sz)
193{
194 int i, ncontig;
195 size_t pgsize;
196
197 ncontig = num_contig_ptes(sz, &pgsize);
198
199 for (i = 0; i < ncontig; i++, ptep++)
200 set_pte(ptep, pte);
201}
202
88pte_t *huge_pte_alloc(struct mm_struct *mm,203pte_t *huge_pte_alloc(struct mm_struct *mm,
89 unsigned long addr, unsigned long sz)204 unsigned long addr, unsigned long sz)
90{205{
@@ -144,19 +259,28 @@ pte_t *huge_pte_offset(struct mm_struct *mm, · 53 unmodified lines
144 return NULL;259 return NULL;
145 260
146 pud = pud_offset(pgd, addr);261 pud = pud_offset(pgd, addr);
147 if (pud_none(*pud))262 if (sz != PUD_SIZE && pud_none(*pud))
148 return NULL;263 return NULL;
149 /* swap or huge page */264 /* hugepage or swap? */
150 if (!pud_present(*pud) || pud_huge(*pud))265 if (pud_huge(*pud) || !pud_present(*pud))
151 return (pte_t *)pud;266 return (pte_t *)pud;
152 /* table; check the next level */267 /* table; check the next level */
153 268
269 if (sz == CONT_PMD_SIZE)
270 addr &= CONT_PMD_MASK;
271
154 pmd = pmd_offset(pud, addr);272 pmd = pmd_offset(pud, addr);
155 if (pmd_none(*pmd))273 if (!(sz == PMD_SIZE || sz == CONT_PMD_SIZE) &&
274 pmd_none(*pmd))
156 return NULL;275 return NULL;
157 if (!pmd_present(*pmd) || pmd_huge(*pmd))276 if (pmd_huge(*pmd) || !pmd_present(*pmd))
158 return (pte_t *)pmd;277 return (pte_t *)pmd;
159 278
279 if (sz == CONT_PTE_SIZE) {
280 pte_t *pte = pte_offset_kernel(pmd, (addr & CONT_PTE_MASK));
281 return pte;
282 }
283
160 return NULL;284 return NULL;
161}285}
162 286
@@ -176,111 +300,133 @@ pte_t arch_make_huge_pte(pte_t entry, struct vm_area_struct *vma, · 13 unmodified lines
176 return entry;300 return entry;
177}301}
178 302
303void huge_pte_clear(struct mm_struct *mm, unsigned long addr,
304 pte_t *ptep, unsigned long sz)
305{
306 int i, ncontig;
307 size_t pgsize;
308
309 ncontig = num_contig_ptes(sz, &pgsize);
310
311 for (i = 0; i < ncontig; i++, addr += pgsize, ptep++)
312 pte_clear(mm, addr, ptep);
313}
314
179pte_t huge_ptep_get_and_clear(struct mm_struct *mm,315pte_t huge_ptep_get_and_clear(struct mm_struct *mm,
180 unsigned long addr, pte_t *ptep)316 unsigned long addr, pte_t *ptep)
181{317{
182 pte_t pte;318 int ncontig;
319 size_t pgsize;
320 pte_t orig_pte = huge_ptep_get(ptep);
183 321
184 if (pte_cont(*ptep)) {322 if (!pte_cont(orig_pte))
185 int ncontig, i;
186 size_t pgsize;
187 bool is_dirty = false;
188
189 ncontig = find_num_contig(mm, addr, ptep, &pgsize);
190 /* save the 1st pte to return */
191 pte = ptep_get_and_clear(mm, addr, ptep);
192 for (i = 1, addr += pgsize; i < ncontig; ++i, addr += pgsize) {
193 /*
194 * If HW_AFDBM is enabled, then the HW could
195 * turn on the dirty bit for any of the page
196 * in the set, so check them all.
197 */
198 ++ptep;
199 if (pte_dirty(ptep_get_and_clear(mm, addr, ptep)))
200 is_dirty = true;
201 }
202 if (is_dirty)
203 return pte_mkdirty(pte);
204 else
205 return pte;
206 } else {
207 return ptep_get_and_clear(mm, addr, ptep);323 return ptep_get_and_clear(mm, addr, ptep);
208 }324
325 ncontig = find_num_contig(mm, addr, ptep, &pgsize);
326
327 return get_clear_flush(mm, addr, ptep, pgsize, ncontig);
209}328}
210 329
211int huge_ptep_set_access_flags(struct vm_area_struct *vma,330int huge_ptep_set_access_flags(struct vm_area_struct *vma,
212 unsigned long addr, pte_t *ptep,331 unsigned long addr, pte_t *ptep,
213 pte_t pte, int dirty)332 pte_t pte, int dirty)
214{333{
215 if (pte_cont(pte)) {334 int ncontig, i, changed = 0;
216 int ncontig, i, changed = 0;335 size_t pgsize = 0;
217 size_t pgsize = 0;336 unsigned long pfn = pte_pfn(pte), dpfn;
218 unsigned long pfn = pte_pfn(pte);337 pgprot_t hugeprot;
219 /* Select all bits except the pfn */338 pte_t orig_pte;
220 pgprot_t hugeprot =339
221 __pgprot(pte_val(pfn_pte(pfn, __pgprot(0))) ^340 if (!pte_cont(pte))
222 pte_val(pte));
223
224 pfn = pte_pfn(pte);
225 ncontig = find_num_contig(vma->vm_mm, addr, ptep,
226 &pgsize);
227 for (i = 0; i < ncontig; ++i, ++ptep, addr += pgsize) {
228 changed |= ptep_set_access_flags(vma, addr, ptep,
229 pfn_pte(pfn,
230 hugeprot),
231 dirty);
232 pfn += pgsize >> PAGE_SHIFT;
233 }
234 return changed;
235 } else {
236 return ptep_set_access_flags(vma, addr, ptep, pte, dirty);341 return ptep_set_access_flags(vma, addr, ptep, pte, dirty);
237 }342
343 ncontig = find_num_contig(vma->vm_mm, addr, ptep, &pgsize);
344 dpfn = pgsize >> PAGE_SHIFT;
345
346 orig_pte = get_clear_flush(vma->vm_mm, addr, ptep, pgsize, ncontig);
347 if (!pte_same(orig_pte, pte))
348 changed = 1;
349
350 /* Make sure we don't lose the dirty state */
351 if (pte_dirty(orig_pte))
352 pte = pte_mkdirty(pte);
353
354 hugeprot = pte_pgprot(pte);
355 for (i = 0; i < ncontig; i++, ptep++, addr += pgsize, pfn += dpfn)
356 set_pte_at(vma->vm_mm, addr, ptep, pfn_pte(pfn, hugeprot));
357
358 return changed;
238}359}
239 360
240void huge_ptep_set_wrprotect(struct mm_struct *mm,361void huge_ptep_set_wrprotect(struct mm_struct *mm,
241 unsigned long addr, pte_t *ptep)362 unsigned long addr, pte_t *ptep)
242{363{
243 if (pte_cont(*ptep)) {364 unsigned long pfn, dpfn;
244 int ncontig, i;365 pgprot_t hugeprot;
245 size_t pgsize = 0;366 int ncontig, i;
246 367 size_t pgsize;
247 ncontig = find_num_contig(mm, addr, ptep, &pgsize);368 pte_t pte;
248 for (i = 0; i < ncontig; ++i, ++ptep, addr += pgsize)369
249 ptep_set_wrprotect(mm, addr, ptep);370 if (!pte_cont(*ptep)) {
250 } else {
251 ptep_set_wrprotect(mm, addr, ptep);371 ptep_set_wrprotect(mm, addr, ptep);
372 return;
252 }373 }
374
375 ncontig = find_num_contig(mm, addr, ptep, &pgsize);
376 dpfn = pgsize >> PAGE_SHIFT;
377
378 pte = get_clear_flush(mm, addr, ptep, pgsize, ncontig);
379 pte = pte_wrprotect(pte);
380
381 hugeprot = pte_pgprot(pte);
382 pfn = pte_pfn(pte);
383
384 for (i = 0; i < ncontig; i++, ptep++, addr += pgsize, pfn += dpfn)
385 set_pte_at(mm, addr, ptep, pfn_pte(pfn, hugeprot));
253}386}
254 387
255void huge_ptep_clear_flush(struct vm_area_struct *vma,388void huge_ptep_clear_flush(struct vm_area_struct *vma,
256 unsigned long addr, pte_t *ptep)389 unsigned long addr, pte_t *ptep)
257{390{
258 if (pte_cont(*ptep)) {391 size_t pgsize;
259 int ncontig, i;392 int ncontig;
260 size_t pgsize = 0;393
261 394 if (!pte_cont(*ptep)) {
262 ncontig = find_num_contig(vma->vm_mm, addr, ptep,
263 &pgsize);
264 for (i = 0; i < ncontig; ++i, ++ptep, addr += pgsize)
265 ptep_clear_flush(vma, addr, ptep);
266 } else {
267 ptep_clear_flush(vma, addr, ptep);395 ptep_clear_flush(vma, addr, ptep);
396 return;
268 }397 }
398
399 ncontig = find_num_contig(vma->vm_mm, addr, ptep, &pgsize);
400 clear_flush(vma->vm_mm, addr, ptep, pgsize, ncontig);
269}401}
270 402
271static __init int setup_hugepagesz(char *opt)403static __init int setup_hugepagesz(char *opt)
272{404{
273 unsigned long ps = memparse(opt, &opt);405 unsigned long ps = memparse(opt, &opt);
274 406
275 if (ps == PMD_SIZE) {407 switch (ps) {
276 hugetlb_add_hstate(PMD_SHIFT - PAGE_SHIFT);408#ifdef CONFIG_ARM64_4K_PAGES
277 } else if (ps == PUD_SIZE) {409 case PUD_SIZE:
278 hugetlb_add_hstate(PUD_SHIFT - PAGE_SHIFT);410#endif
279 } else {411 case PMD_SIZE * CONT_PMDS:
280 hugetlb_bad_size();412 case PMD_SIZE:
281 pr_err("hugepagesz: Unsupported page size %lu K\n", ps >> 10);413 case PAGE_SIZE * CONT_PTES:
282 return 0;414 hugetlb_add_hstate(ilog2(ps) - PAGE_SHIFT);
415 return 1;
283 }416 }
284 return 1;417
418 hugetlb_bad_size();
419 pr_err("hugepagesz: Unsupported page size %lu K\n", ps >> 10);
420 return 0;
285}421}
286__setup("hugepagesz=", setup_hugepagesz);422__setup("hugepagesz=", setup_hugepagesz);
423
424#ifdef CONFIG_ARM64_64K_PAGES
425static __init int add_default_hugepagesz(void)
426{
427 if (size_to_hstate(CONT_PTES * PAGE_SIZE) == NULL)
428 hugetlb_add_hstate(CONT_PTE_SHIFT);
429 return 0;
430}
431arch_initcall(add_default_hugepagesz);
432#endif
drivers/acpi/arm64/iort.c+87−53
@@ -588,7 +588,8 @@ void acpi_configure_pmsi_domain(struct device *dev) · 587 unmodified lines
588 dev_set_msi_domain(dev, msi_domain);588 dev_set_msi_domain(dev, msi_domain);
589}589}
590 590
591static int __get_pci_rid(struct pci_dev *pdev, u16 alias, void *data)591static int __maybe_unused __get_pci_rid(struct pci_dev *pdev, u16 alias,
592 void *data)
592{593{
593 u32 *rid = data;594 u32 *rid = data;
594 595
@@ -633,8 +634,7 @@ int iort_add_device_replay(const struct iommu_ops *ops, struct device *dev) · 38 unmodified lines
633{634{
634 int err = 0;635 int err = 0;
635 636
636 if (!IS_ERR_OR_NULL(ops) && ops->add_device && dev->bus &&637 if (ops->add_device && dev->bus && !dev->iommu_group)
637 !dev->iommu_group)
638 err = ops->add_device(dev);638 err = ops->add_device(dev);
639 639
640 return err;640 return err;
@@ -648,36 +648,49 @@ int iort_add_device_replay(const struct iommu_ops *ops, struct device *dev) · 7 unmodified lines
648{ return 0; }648{ return 0; }
649#endif649#endif
650 650
651static const struct iommu_ops *iort_iommu_xlate(struct device *dev,651static int iort_iommu_xlate(struct device *dev, struct acpi_iort_node *node,
652 struct acpi_iort_node *node,652 u32 streamid)
653 u32 streamid)
654{653{
655 const struct iommu_ops *ops = NULL;654 const struct iommu_ops *ops;
656 int ret = -ENODEV;
657 struct fwnode_handle *iort_fwnode;655 struct fwnode_handle *iort_fwnode;
658 656
659 if (node) {657 if (!node)
660 iort_fwnode = iort_get_fwnode(node);658 return -ENODEV;
661 if (!iort_fwnode)
662 return NULL;
663 659
664 ops = iommu_ops_from_fwnode(iort_fwnode);660 iort_fwnode = iort_get_fwnode(node);
665 /*661 if (!iort_fwnode)
666 * If the ops look-up fails, this means that either662 return -ENODEV;
667 * the SMMU drivers have not been probed yet or that
668 * the SMMU drivers are not built in the kernel;
669 * Depending on whether the SMMU drivers are built-in
670 * in the kernel or not, defer the IOMMU configuration
671 * or just abort it.
672 */
673 if (!ops)
674 return iort_iommu_driver_enabled(node->type) ?
675 ERR_PTR(-EPROBE_DEFER) : NULL;
676 663
677 ret = arm_smmu_iort_xlate(dev, streamid, iort_fwnode, ops);664 /*
678 }665 * If the ops look-up fails, this means that either
666 * the SMMU drivers have not been probed yet or that
667 * the SMMU drivers are not built in the kernel;
668 * Depending on whether the SMMU drivers are built-in
669 * in the kernel or not, defer the IOMMU configuration
670 * or just abort it.
671 */
672 ops = iommu_ops_from_fwnode(iort_fwnode);
673 if (!ops)
674 return iort_iommu_driver_enabled(node->type) ?
675 -EPROBE_DEFER : -ENODEV;
679 676
680 return ret ? NULL : ops;677 return arm_smmu_iort_xlate(dev, streamid, iort_fwnode, ops);
678}
679
680struct iort_pci_alias_info {
681 struct device *dev;
682 struct acpi_iort_node *node;
683};
684
685static int iort_pci_iommu_init(struct pci_dev *pdev, u16 alias, void *data)
686{
687 struct iort_pci_alias_info *info = data;
688 struct acpi_iort_node *parent;
689 u32 streamid;
690
691 parent = iort_node_map_id(info->node, alias, &streamid,
692 IORT_IOMMU_TYPE);
693 return iort_iommu_xlate(info->dev, parent, streamid);
681}694}
682 695
683/**696/**
@@ -713,9 +726,9 @@ void iort_set_dma_mask(struct device *dev) · 29 unmodified lines
713const struct iommu_ops *iort_iommu_configure(struct device *dev)726const struct iommu_ops *iort_iommu_configure(struct device *dev)
714{727{
715 struct acpi_iort_node *node, *parent;728 struct acpi_iort_node *node, *parent;
716 const struct iommu_ops *ops = NULL;729 const struct iommu_ops *ops;
717 u32 streamid = 0;730 u32 streamid = 0;
718 int err;731 int err = -ENODEV;
719 732
720 /*733 /*
721 * If we already translated the fwspec there734 * If we already translated the fwspec there
@@ -727,21 +740,16 @@ const struct iommu_ops *iort_iommu_configure(struct device *dev) · 5 unmodified lines
727 740
728 if (dev_is_pci(dev)) {741 if (dev_is_pci(dev)) {
729 struct pci_bus *bus = to_pci_dev(dev)->bus;742 struct pci_bus *bus = to_pci_dev(dev)->bus;
730 u32 rid;743 struct iort_pci_alias_info info = { .dev = dev };
731
732 pci_for_each_dma_alias(to_pci_dev(dev), __get_pci_rid,
733 &rid);
734 744
735 node = iort_scan_node(ACPI_IORT_NODE_PCI_ROOT_COMPLEX,745 node = iort_scan_node(ACPI_IORT_NODE_PCI_ROOT_COMPLEX,
736 iort_match_node_callback, &bus->dev);746 iort_match_node_callback, &bus->dev);
737 if (!node)747 if (!node)
738 return NULL;748 return NULL;
739 749
740 parent = iort_node_map_id(node, rid, &streamid,750 info.node = node;
741 IORT_IOMMU_TYPE);751 err = pci_for_each_dma_alias(to_pci_dev(dev),
742 752 iort_pci_iommu_init, &info);
743 ops = iort_iommu_xlate(dev, parent, streamid);
744
745 } else {753 } else {
746 int i = 0;754 int i = 0;
747 755
@@ -750,31 +758,30 @@ const struct iommu_ops *iort_iommu_configure(struct device *dev) · 2 unmodified lines
750 if (!node)758 if (!node)
751 return NULL;759 return NULL;
752 760
753 parent = iort_node_map_platform_id(node, &streamid,761 do {
754 IORT_IOMMU_TYPE, i++);
755
756 while (parent) {
757 ops = iort_iommu_xlate(dev, parent, streamid);
758 if (IS_ERR_OR_NULL(ops))
759 return ops;
760
761 parent = iort_node_map_platform_id(node, &streamid,762 parent = iort_node_map_platform_id(node, &streamid,
762 IORT_IOMMU_TYPE,763 IORT_IOMMU_TYPE,
763 i++);764 i++);
764 }765
766 if (parent)
767 err = iort_iommu_xlate(dev, parent, streamid);
768 } while (parent && !err);
765 }769 }
766 770
767 /*771 /*
768 * If we have reason to believe the IOMMU driver missed the initial772 * If we have reason to believe the IOMMU driver missed the initial
769 * add_device callback for dev, replay it to get things in order.773 * add_device callback for dev, replay it to get things in order.
770 */774 */
771 err = iort_add_device_replay(ops, dev);775 if (!err) {
772 if (err)776 ops = iort_fwspec_iommu_ops(dev->iommu_fwspec);
773 ops = ERR_PTR(err);777 err = iort_add_device_replay(ops, dev);
778 }
774 779
775 /* Ignore all other errors apart from EPROBE_DEFER */780 /* Ignore all other errors apart from EPROBE_DEFER */
776 if (IS_ERR(ops) && (PTR_ERR(ops) != -EPROBE_DEFER)) {781 if (err == -EPROBE_DEFER) {
777 dev_dbg(dev, "Adding to IOMMU failed: %ld\n", PTR_ERR(ops));782 ops = ERR_PTR(err);
783 } else if (err) {
784 dev_dbg(dev, "Adding to IOMMU failed: %d\n", err);
778 ops = NULL;785 ops = NULL;
779 }786 }
780 787
@@ -908,6 +915,27 @@ static bool __init arm_smmu_v3_is_coherent(struct acpi_iort_node *node) · 127 unmodified lines
908 return smmu->flags & ACPI_IORT_SMMU_V3_COHACC_OVERRIDE;915 return smmu->flags & ACPI_IORT_SMMU_V3_COHACC_OVERRIDE;
909}916}
910 917
918#if defined(CONFIG_ACPI_NUMA) && defined(ACPI_IORT_SMMU_V3_PXM_VALID)
919/*
920 * set numa proximity domain for smmuv3 device
921 */
922static void __init arm_smmu_v3_set_proximity(struct device *dev,
923 struct acpi_iort_node *node)
924{
925 struct acpi_iort_smmu_v3 *smmu;
926
927 smmu = (struct acpi_iort_smmu_v3 *)node->node_data;
928 if (smmu->flags & ACPI_IORT_SMMU_V3_PXM_VALID) {
929 set_dev_node(dev, acpi_map_pxm_to_node(smmu->pxm));
930 pr_info("SMMU-v3[%llx] Mapped to Proximity domain %d\n",
931 smmu->base_address,
932 smmu->pxm);
933 }
934}
935#else
936#define arm_smmu_v3_set_proximity NULL
937#endif
938
911static int __init arm_smmu_count_resources(struct acpi_iort_node *node)939static int __init arm_smmu_count_resources(struct acpi_iort_node *node)
912{940{
913 struct acpi_iort_smmu *smmu;941 struct acpi_iort_smmu *smmu;
@@ -977,13 +1005,16 @@ struct iort_iommu_config { · 63 unmodified lines
977 int (*iommu_count_resources)(struct acpi_iort_node *node);1005 int (*iommu_count_resources)(struct acpi_iort_node *node);
978 void (*iommu_init_resources)(struct resource *res,1006 void (*iommu_init_resources)(struct resource *res,
979 struct acpi_iort_node *node);1007 struct acpi_iort_node *node);
1008 void (*iommu_set_proximity)(struct device *dev,
1009 struct acpi_iort_node *node);
980};1010};
981 1011
982static const struct iort_iommu_config iort_arm_smmu_v3_cfg __initconst = {1012static const struct iort_iommu_config iort_arm_smmu_v3_cfg __initconst = {
983 .name = "arm-smmu-v3",1013 .name = "arm-smmu-v3",
984 .iommu_is_coherent = arm_smmu_v3_is_coherent,1014 .iommu_is_coherent = arm_smmu_v3_is_coherent,
985 .iommu_count_resources = arm_smmu_v3_count_resources,1015 .iommu_count_resources = arm_smmu_v3_count_resources,
986 .iommu_init_resources = arm_smmu_v3_init_resources1016 .iommu_init_resources = arm_smmu_v3_init_resources,
1017 .iommu_set_proximity = arm_smmu_v3_set_proximity,
987};1018};
988 1019
989static const struct iort_iommu_config iort_arm_smmu_cfg __initconst = {1020static const struct iort_iommu_config iort_arm_smmu_cfg __initconst = {
@@ -1028,6 +1059,9 @@ static int __init iort_add_smmu_platform_device(struct acpi_iort_node *node) · 38 unmodified lines
1028 if (!pdev)1059 if (!pdev)
1029 return -ENOMEM;1060 return -ENOMEM;
1030 1061
1062 if (ops->iommu_set_proximity)
1063 ops->iommu_set_proximity(&pdev->dev, node);
1064
1031 count = ops->iommu_count_resources(node);1065 count = ops->iommu_count_resources(node);
1032 1066
1033 r = kcalloc(count, sizeof(*r), GFP_KERNEL);1067 r = kcalloc(count, sizeof(*r), GFP_KERNEL);
drivers/firmware/efi/libstub/arm64-stub.c+4−2
@@ -11,6 +11,7 @@ · 10 unmodified lines
11 */11 */
12#include <linux/efi.h>12#include <linux/efi.h>
13#include <asm/efi.h>13#include <asm/efi.h>
14#include <asm/memory.h>
14#include <asm/sections.h>15#include <asm/sections.h>
15#include <asm/sysreg.h>16#include <asm/sysreg.h>
16 17
@@ -81,9 +82,10 @@ efi_status_t handle_kernel_image(efi_system_table_t *sys_table_arg, · 64 unmodified lines
81 /*82 /*
82 * If CONFIG_DEBUG_ALIGN_RODATA is not set, produce a83 * If CONFIG_DEBUG_ALIGN_RODATA is not set, produce a
83 * displacement in the interval [0, MIN_KIMG_ALIGN) that84 * displacement in the interval [0, MIN_KIMG_ALIGN) that
84 * is a multiple of the minimal segment alignment (SZ_64K)85 * doesn't violate this kernel's de-facto alignment
86 * constraints.
85 */87 */
86 u32 mask = (MIN_KIMG_ALIGN - 1) & ~(SZ_64K - 1);88 u32 mask = (MIN_KIMG_ALIGN - 1) & ~(EFI_KIMG_ALIGN - 1);
87 u32 offset = !IS_ENABLED(CONFIG_DEBUG_ALIGN_RODATA) ?89 u32 offset = !IS_ENABLED(CONFIG_DEBUG_ALIGN_RODATA) ?
88 (phys_seed >> 32) & mask : TEXT_OFFSET;90 (phys_seed >> 32) & mask : TEXT_OFFSET;
89 91
drivers/perf/arm_pmu.c+6−0
@@ -47,6 +47,9 @@ armpmu_map_cache_event(const unsigned (*cache_map) · 46 unmodified lines
47 if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX)47 if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
48 return -EINVAL;48 return -EINVAL;
49 49
50 if (!cache_map)
51 return -ENOENT;
52
50 ret = (int)(*cache_map)[cache_type][cache_op][cache_result];53 ret = (int)(*cache_map)[cache_type][cache_op][cache_result];
51 54
52 if (ret == CACHE_OP_UNSUPPORTED)55 if (ret == CACHE_OP_UNSUPPORTED)
@@ -63,6 +66,9 @@ armpmu_map_hw_event(const unsigned (*event_map)[PERF_COUNT_HW_MAX], u64 config) · 10 unmodified lines
63 if (config >= PERF_COUNT_HW_MAX)66 if (config >= PERF_COUNT_HW_MAX)
64 return -EINVAL;67 return -EINVAL;
65 68
69 if (!event_map)
70 return -ENOENT;
71
66 mapping = (*event_map)[config];72 mapping = (*event_map)[config];
67 return mapping == HW_OP_UNSUPPORTED ? -ENOENT : mapping;73 return mapping == HW_OP_UNSUPPORTED ? -ENOENT : mapping;
68}74}
drivers/perf/xgene_pmu.c+22−52
@@ -1147,7 +1147,6 @@ xgene_pmu_dev_add(struct xgene_pmu *xgene_pmu, struct xgene_pmu_dev_ctx *ctx) · 1146 unmodified lines
1147{1147{
1148 struct device *dev = xgene_pmu->dev;1148 struct device *dev = xgene_pmu->dev;
1149 struct xgene_pmu_dev *pmu;1149 struct xgene_pmu_dev *pmu;
1150 int rc;
1151 1150
1152 pmu = devm_kzalloc(dev, sizeof(*pmu), GFP_KERNEL);1151 pmu = devm_kzalloc(dev, sizeof(*pmu), GFP_KERNEL);
1153 if (!pmu)1152 if (!pmu)
@@ -1159,7 +1158,7 @@ xgene_pmu_dev_add(struct xgene_pmu *xgene_pmu, struct xgene_pmu_dev_ctx *ctx) · 5 unmodified lines
1159 switch (pmu->inf->type) {1158 switch (pmu->inf->type) {
1160 case PMU_TYPE_L3C:1159 case PMU_TYPE_L3C:
1161 if (!(xgene_pmu->l3c_active_mask & pmu->inf->enable_mask))1160 if (!(xgene_pmu->l3c_active_mask & pmu->inf->enable_mask))
1162 goto dev_err;1161 return -ENODEV;
1163 if (xgene_pmu->version == PCP_PMU_V3)1162 if (xgene_pmu->version == PCP_PMU_V3)
1164 pmu->attr_groups = l3c_pmu_v3_attr_groups;1163 pmu->attr_groups = l3c_pmu_v3_attr_groups;
1165 else1164 else
@@ -1177,7 +1176,7 @@ xgene_pmu_dev_add(struct xgene_pmu *xgene_pmu, struct xgene_pmu_dev_ctx *ctx) · 11 unmodified lines
1177 break;1176 break;
1178 case PMU_TYPE_MCB:1177 case PMU_TYPE_MCB:
1179 if (!(xgene_pmu->mcb_active_mask & pmu->inf->enable_mask))1178 if (!(xgene_pmu->mcb_active_mask & pmu->inf->enable_mask))
1180 goto dev_err;1179 return -ENODEV;
1181 if (xgene_pmu->version == PCP_PMU_V3)1180 if (xgene_pmu->version == PCP_PMU_V3)
1182 pmu->attr_groups = mcb_pmu_v3_attr_groups;1181 pmu->attr_groups = mcb_pmu_v3_attr_groups;
1183 else1182 else
@@ -1185,7 +1184,7 @@ xgene_pmu_dev_add(struct xgene_pmu *xgene_pmu, struct xgene_pmu_dev_ctx *ctx) · 1 unmodified line
1185 break;1184 break;
1186 case PMU_TYPE_MC:1185 case PMU_TYPE_MC:
1187 if (!(xgene_pmu->mc_active_mask & pmu->inf->enable_mask))1186 if (!(xgene_pmu->mc_active_mask & pmu->inf->enable_mask))
1188 goto dev_err;1187 return -ENODEV;
1189 if (xgene_pmu->version == PCP_PMU_V3)1188 if (xgene_pmu->version == PCP_PMU_V3)
1190 pmu->attr_groups = mc_pmu_v3_attr_groups;1189 pmu->attr_groups = mc_pmu_v3_attr_groups;
1191 else1190 else
@@ -1195,19 +1194,14 @@ xgene_pmu_dev_add(struct xgene_pmu *xgene_pmu, struct xgene_pmu_dev_ctx *ctx) · 3 unmodified lines
1195 return -EINVAL;1194 return -EINVAL;
1196 }1195 }
1197 1196
1198 rc = xgene_init_perf(pmu, ctx->name);1197 if (xgene_init_perf(pmu, ctx->name)) {
1199 if (rc) {
1200 dev_err(dev, "%s PMU: Failed to init perf driver\n", ctx->name);1198 dev_err(dev, "%s PMU: Failed to init perf driver\n", ctx->name);
1201 goto dev_err;1199 return -ENODEV;
1202 }1200 }
1203 1201
1204 dev_info(dev, "%s PMU registered\n", ctx->name);1202 dev_info(dev, "%s PMU registered\n", ctx->name);
1205 1203
1206 return rc;1204 return 0;
1207
1208dev_err:
1209 devm_kfree(dev, pmu);
1210 return -ENODEV;
1211}1205}
1212 1206
1213static void _xgene_pmu_isr(int irq, struct xgene_pmu_dev *pmu_dev)1207static void _xgene_pmu_isr(int irq, struct xgene_pmu_dev *pmu_dev)
@@ -1515,13 +1509,13 @@ xgene_pmu_dev_ctx *acpi_get_pmu_hw_inf(struct xgene_pmu *xgene_pmu, · 301 unmodified lines
1515 acpi_dev_free_resource_list(&resource_list);1509 acpi_dev_free_resource_list(&resource_list);
1516 if (rc < 0) {1510 if (rc < 0) {
1517 dev_err(dev, "PMU type %d: No resource address found\n", type);1511 dev_err(dev, "PMU type %d: No resource address found\n", type);
1518 goto err;1512 return NULL;
1519 }1513 }
1520 1514
1521 dev_csr = devm_ioremap_resource(dev, &res);1515 dev_csr = devm_ioremap_resource(dev, &res);
1522 if (IS_ERR(dev_csr)) {1516 if (IS_ERR(dev_csr)) {
1523 dev_err(dev, "PMU type %d: Fail to map resource\n", type);1517 dev_err(dev, "PMU type %d: Fail to map resource\n", type);
1524 goto err;1518 return NULL;
1525 }1519 }
1526 1520
1527 /* A PMU device node without enable-bit-index is always enabled */1521 /* A PMU device node without enable-bit-index is always enabled */
@@ -1535,7 +1529,7 @@ xgene_pmu_dev_ctx *acpi_get_pmu_hw_inf(struct xgene_pmu *xgene_pmu, · 7 unmodified lines
1535 ctx->name = xgene_pmu_dev_name(dev, type, enable_bit);1529 ctx->name = xgene_pmu_dev_name(dev, type, enable_bit);
1536 if (!ctx->name) {1530 if (!ctx->name) {
1537 dev_err(dev, "PMU type %d: Fail to get device name\n", type);1531 dev_err(dev, "PMU type %d: Fail to get device name\n", type);
1538 goto err;1532 return NULL;
1539 }1533 }
1540 inf = &ctx->inf;1534 inf = &ctx->inf;
1541 inf->type = type;1535 inf->type = type;
@@ -1543,9 +1537,6 @@ xgene_pmu_dev_ctx *acpi_get_pmu_hw_inf(struct xgene_pmu *xgene_pmu, · 1 unmodified line
1543 inf->enable_mask = 1 << enable_bit;1537 inf->enable_mask = 1 << enable_bit;
1544 1538
1545 return ctx;1539 return ctx;
1546err:
1547 devm_kfree(dev, ctx);
1548 return NULL;
1549}1540}
1550 1541
1551static const struct acpi_device_id xgene_pmu_acpi_type_match[] = {1542static const struct acpi_device_id xgene_pmu_acpi_type_match[] = {
@@ -1663,20 +1654,20 @@ xgene_pmu_dev_ctx *fdt_get_pmu_hw_inf(struct xgene_pmu *xgene_pmu, · 111 unmodified lines
1663 void __iomem *dev_csr;1654 void __iomem *dev_csr;
1664 struct resource res;1655 struct resource res;
1665 int enable_bit;1656 int enable_bit;
1666 int rc;
1667 1657
1668 ctx = devm_kzalloc(dev, sizeof(*ctx), GFP_KERNEL);1658 ctx = devm_kzalloc(dev, sizeof(*ctx), GFP_KERNEL);
1669 if (!ctx)1659 if (!ctx)
1670 return NULL;1660 return NULL;
1671 rc = of_address_to_resource(np, 0, &res);1661
1672 if (rc < 0) {1662 if (of_address_to_resource(np, 0, &res) < 0) {
1673 dev_err(dev, "PMU type %d: No resource address found\n", type);1663 dev_err(dev, "PMU type %d: No resource address found\n", type);
1674 goto err;1664 return NULL;
1675 }1665 }
1666
1676 dev_csr = devm_ioremap_resource(dev, &res);1667 dev_csr = devm_ioremap_resource(dev, &res);
1677 if (IS_ERR(dev_csr)) {1668 if (IS_ERR(dev_csr)) {
1678 dev_err(dev, "PMU type %d: Fail to map resource\n", type);1669 dev_err(dev, "PMU type %d: Fail to map resource\n", type);
1679 goto err;1670 return NULL;
1680 }1671 }
1681 1672
1682 /* A PMU device node without enable-bit-index is always enabled */1673 /* A PMU device node without enable-bit-index is always enabled */
@@ -1686,17 +1677,15 @@ xgene_pmu_dev_ctx *fdt_get_pmu_hw_inf(struct xgene_pmu *xgene_pmu, · 3 unmodified lines
1686 ctx->name = xgene_pmu_dev_name(dev, type, enable_bit);1677 ctx->name = xgene_pmu_dev_name(dev, type, enable_bit);
1687 if (!ctx->name) {1678 if (!ctx->name) {
1688 dev_err(dev, "PMU type %d: Fail to get device name\n", type);1679 dev_err(dev, "PMU type %d: Fail to get device name\n", type);
1689 goto err;1680 return NULL;
1690 }1681 }
1682
1691 inf = &ctx->inf;1683 inf = &ctx->inf;
1692 inf->type = type;1684 inf->type = type;
1693 inf->csr = dev_csr;1685 inf->csr = dev_csr;
1694 inf->enable_mask = 1 << enable_bit;1686 inf->enable_mask = 1 << enable_bit;
1695 1687
1696 return ctx;1688 return ctx;
1697err:
1698 devm_kfree(dev, ctx);
1699 return NULL;
1700}1689}
1701 1690
1702static int fdt_pmu_probe_pmu_dev(struct xgene_pmu *xgene_pmu,1691static int fdt_pmu_probe_pmu_dev(struct xgene_pmu *xgene_pmu,
@@ -1868,22 +1857,20 @@ static int xgene_pmu_probe(struct platform_device *pdev) · 165 unmodified lines
1868 xgene_pmu->pcppmu_csr = devm_ioremap_resource(&pdev->dev, res);1857 xgene_pmu->pcppmu_csr = devm_ioremap_resource(&pdev->dev, res);
1869 if (IS_ERR(xgene_pmu->pcppmu_csr)) {1858 if (IS_ERR(xgene_pmu->pcppmu_csr)) {
1870 dev_err(&pdev->dev, "ioremap failed for PCP PMU resource\n");1859 dev_err(&pdev->dev, "ioremap failed for PCP PMU resource\n");
1871 rc = PTR_ERR(xgene_pmu->pcppmu_csr);1860 return PTR_ERR(xgene_pmu->pcppmu_csr);
1872 goto err;
1873 }1861 }
1874 1862
1875 irq = platform_get_irq(pdev, 0);1863 irq = platform_get_irq(pdev, 0);
1876 if (irq < 0) {1864 if (irq < 0) {
1877 dev_err(&pdev->dev, "No IRQ resource\n");1865 dev_err(&pdev->dev, "No IRQ resource\n");
1878 rc = -EINVAL;1866 return -EINVAL;
1879 goto err;
1880 }1867 }
1881 rc = devm_request_irq(&pdev->dev, irq, xgene_pmu_isr,1868 rc = devm_request_irq(&pdev->dev, irq, xgene_pmu_isr,
1882 IRQF_NOBALANCING | IRQF_NO_THREAD,1869 IRQF_NOBALANCING | IRQF_NO_THREAD,
1883 dev_name(&pdev->dev), xgene_pmu);1870 dev_name(&pdev->dev), xgene_pmu);
1884 if (rc) {1871 if (rc) {
1885 dev_err(&pdev->dev, "Could not request IRQ %d\n", irq);1872 dev_err(&pdev->dev, "Could not request IRQ %d\n", irq);
1886 goto err;1873 return rc;
1887 }1874 }
1888 1875
1889 raw_spin_lock_init(&xgene_pmu->lock);1876 raw_spin_lock_init(&xgene_pmu->lock);
@@ -1903,42 +1890,29 @@ static int xgene_pmu_probe(struct platform_device *pdev) · 13 unmodified lines
1903 rc = irq_set_affinity(irq, &xgene_pmu->cpu);1890 rc = irq_set_affinity(irq, &xgene_pmu->cpu);
1904 if (rc) {1891 if (rc) {
1905 dev_err(&pdev->dev, "Failed to set interrupt affinity!\n");1892 dev_err(&pdev->dev, "Failed to set interrupt affinity!\n");
1906 goto err;1893 return rc;
1907 }1894 }
1908 1895
1909 /* Walk through the tree for all PMU perf devices */1896 /* Walk through the tree for all PMU perf devices */
1910 rc = xgene_pmu_probe_pmu_dev(xgene_pmu, pdev);1897 rc = xgene_pmu_probe_pmu_dev(xgene_pmu, pdev);
1911 if (rc) {1898 if (rc) {
1912 dev_err(&pdev->dev, "No PMU perf devices found!\n");1899 dev_err(&pdev->dev, "No PMU perf devices found!\n");
1913 goto err;1900 return rc;
1914 }1901 }
1915 1902
1916 /* Enable interrupt */1903 /* Enable interrupt */
1917 xgene_pmu->ops->unmask_int(xgene_pmu);1904 xgene_pmu->ops->unmask_int(xgene_pmu);
1918 1905
1919 return 0;1906 return 0;
1920
1921err:
1922 if (xgene_pmu->pcppmu_csr)
1923 devm_iounmap(&pdev->dev, xgene_pmu->pcppmu_csr);
1924 devm_kfree(&pdev->dev, xgene_pmu);
1925
1926 return rc;
1927}1907}
1928 1908
1929static void1909static void
1930xgene_pmu_dev_cleanup(struct xgene_pmu *xgene_pmu, struct list_head *pmus)1910xgene_pmu_dev_cleanup(struct xgene_pmu *xgene_pmu, struct list_head *pmus)
1931{1911{
1932 struct xgene_pmu_dev_ctx *ctx;1912 struct xgene_pmu_dev_ctx *ctx;
1933 struct device *dev = xgene_pmu->dev;
1934 struct xgene_pmu_dev *pmu_dev;
1935 1913
1936 list_for_each_entry(ctx, pmus, next) {1914 list_for_each_entry(ctx, pmus, next) {
1937 pmu_dev = ctx->pmu_dev;1915 perf_pmu_unregister(&ctx->pmu_dev->pmu);
1938 if (pmu_dev->inf->csr)
1939 devm_iounmap(dev, pmu_dev->inf->csr);
1940 devm_kfree(dev, ctx);
1941 devm_kfree(dev, pmu_dev);
1942 }1916 }
1943}1917}
1944 1918
@@ -1951,10 +1925,6 @@ static int xgene_pmu_remove(struct platform_device *pdev) · 6 unmodified lines
1951 xgene_pmu_dev_cleanup(xgene_pmu, &xgene_pmu->mcbpmus);1925 xgene_pmu_dev_cleanup(xgene_pmu, &xgene_pmu->mcbpmus);
1952 xgene_pmu_dev_cleanup(xgene_pmu, &xgene_pmu->mcpmus);1926 xgene_pmu_dev_cleanup(xgene_pmu, &xgene_pmu->mcpmus);
1953 1927
1954 if (xgene_pmu->pcppmu_csr)
1955 devm_iounmap(&pdev->dev, xgene_pmu->pcppmu_csr);
1956 devm_kfree(&pdev->dev, xgene_pmu);
1957
1958 return 0;1928 return 0;
1959}1929}
1960 1930
include/linux/raid/pq.h+1−0
@@ -121,6 +121,7 @@ extern const struct raid6_recov_calls raid6_recov_ssse3; · 120 unmodified lines
121extern const struct raid6_recov_calls raid6_recov_avx2;121extern const struct raid6_recov_calls raid6_recov_avx2;
122extern const struct raid6_recov_calls raid6_recov_avx512;122extern const struct raid6_recov_calls raid6_recov_avx512;
123extern const struct raid6_recov_calls raid6_recov_s390xc;123extern const struct raid6_recov_calls raid6_recov_s390xc;
124extern const struct raid6_recov_calls raid6_recov_neon;
124 125
125extern const struct raid6_calls raid6_neonx1;126extern const struct raid6_calls raid6_neonx1;
126extern const struct raid6_calls raid6_neonx2;127extern const struct raid6_calls raid6_neonx2;
include/linux/thread_info.h+4−0
@@ -38,6 +38,10 @@ enum { · 37 unmodified lines
38 38
39#ifdef __KERNEL__39#ifdef __KERNEL__
40 40
41#ifndef THREAD_ALIGN
42#define THREAD_ALIGN THREAD_SIZE
43#endif
44
41#ifdef CONFIG_DEBUG_STACK_USAGE45#ifdef CONFIG_DEBUG_STACK_USAGE
42# define THREADINFO_GFP (GFP_KERNEL_ACCOUNT | __GFP_NOTRACK | \46# define THREADINFO_GFP (GFP_KERNEL_ACCOUNT | __GFP_NOTRACK | \
43 __GFP_ZERO)47 __GFP_ZERO)
kernel/fork.c+2−1
@@ -88,6 +88,7 @@ · 87 unmodified lines
88#include <linux/sysctl.h>88#include <linux/sysctl.h>
89#include <linux/kcov.h>89#include <linux/kcov.h>
90#include <linux/livepatch.h>90#include <linux/livepatch.h>
91#include <linux/thread_info.h>
91 92
92#include <asm/pgtable.h>93#include <asm/pgtable.h>
93#include <asm/pgalloc.h>94#include <asm/pgalloc.h>
@@ -217,7 +218,7 @@ static unsigned long *alloc_thread_stack_node(struct task_struct *tsk, int node) · 123 unmodified lines
217 return s->addr;218 return s->addr;
218 }219 }
219 220
220 stack = __vmalloc_node_range(THREAD_SIZE, THREAD_SIZE,221 stack = __vmalloc_node_range(THREAD_SIZE, THREAD_ALIGN,
221 VMALLOC_START, VMALLOC_END,222 VMALLOC_START, VMALLOC_END,
222 THREADINFO_GFP,223 THREADINFO_GFP,
223 PAGE_KERNEL,224 PAGE_KERNEL,
lib/raid6/Makefile+3−1
@@ -5,7 +5,7 @@ raid6_pq-y += algos.o recov.o tables.o int1.o int2.o int4.o \ · 4 unmodified lines
5 5
6raid6_pq-$(CONFIG_X86) += recov_ssse3.o recov_avx2.o mmx.o sse1.o sse2.o avx2.o avx512.o recov_avx512.o6raid6_pq-$(CONFIG_X86) += recov_ssse3.o recov_avx2.o mmx.o sse1.o sse2.o avx2.o avx512.o recov_avx512.o
7raid6_pq-$(CONFIG_ALTIVEC) += altivec1.o altivec2.o altivec4.o altivec8.o7raid6_pq-$(CONFIG_ALTIVEC) += altivec1.o altivec2.o altivec4.o altivec8.o
8raid6_pq-$(CONFIG_KERNEL_MODE_NEON) += neon.o neon1.o neon2.o neon4.o neon8.o8raid6_pq-$(CONFIG_KERNEL_MODE_NEON) += neon.o neon1.o neon2.o neon4.o neon8.o recov_neon.o recov_neon_inner.o
9raid6_pq-$(CONFIG_TILEGX) += tilegx8.o9raid6_pq-$(CONFIG_TILEGX) += tilegx8.o
10raid6_pq-$(CONFIG_S390) += s390vx8.o recov_s390xc.o10raid6_pq-$(CONFIG_S390) += s390vx8.o recov_s390xc.o
11 11
@@ -26,7 +26,9 @@ NEON_FLAGS := -ffreestanding · 14 unmodified lines
26ifeq ($(ARCH),arm)26ifeq ($(ARCH),arm)
27NEON_FLAGS += -mfloat-abi=softfp -mfpu=neon27NEON_FLAGS += -mfloat-abi=softfp -mfpu=neon
28endif28endif
29CFLAGS_recov_neon_inner.o += $(NEON_FLAGS)
29ifeq ($(ARCH),arm64)30ifeq ($(ARCH),arm64)
31CFLAGS_REMOVE_recov_neon_inner.o += -mgeneral-regs-only
30CFLAGS_REMOVE_neon1.o += -mgeneral-regs-only32CFLAGS_REMOVE_neon1.o += -mgeneral-regs-only
31CFLAGS_REMOVE_neon2.o += -mgeneral-regs-only33CFLAGS_REMOVE_neon2.o += -mgeneral-regs-only
32CFLAGS_REMOVE_neon4.o += -mgeneral-regs-only34CFLAGS_REMOVE_neon4.o += -mgeneral-regs-only
lib/raid6/algos.c+3−0
@@ -112,6 +112,9 @@ const struct raid6_recov_calls *const raid6_recov_algos[] = { · 111 unmodified lines
112#endif112#endif
113#ifdef CONFIG_S390113#ifdef CONFIG_S390
114 &raid6_recov_s390xc,114 &raid6_recov_s390xc,
115#endif
116#if defined(CONFIG_KERNEL_MODE_NEON)
117 &raid6_recov_neon,
115#endif118#endif
116 &raid6_recov_intx1,119 &raid6_recov_intx1,
117 NULL120 NULL
lib/raid6/neon.uc+30−3
@@ -46,8 +46,12 @@ static inline unative_t SHLBYTE(unative_t v) · 45 unmodified lines
46 */46 */
47static inline unative_t MASK(unative_t v)47static inline unative_t MASK(unative_t v)
48{48{
49 const uint8x16_t temp = NBYTES(0);49 return (unative_t)vshrq_n_s8((int8x16_t)v, 7);
50 return (unative_t)vcltq_s8((int8x16_t)v, (int8x16_t)temp);50}
51
52static inline unative_t PMUL(unative_t v, unative_t u)
53{
54 return (unative_t)vmulq_p8((poly8x16_t)v, (poly8x16_t)u);
51}55}
52 56
53void raid6_neon$#_gen_syndrome_real(int disks, unsigned long bytes, void **ptrs)57void raid6_neon$#_gen_syndrome_real(int disks, unsigned long bytes, void **ptrs)
@@ -110,7 +114,30 @@ void raid6_neon$#_xor_syndrome_real(int disks, int start, int stop, · 56 unmodified lines
110 wq$$ = veorq_u8(w1$$, wd$$);114 wq$$ = veorq_u8(w1$$, wd$$);
111 }115 }
112 /* P/Q left side optimization */116 /* P/Q left side optimization */
113 for ( z = start-1 ; z >= 0 ; z-- ) {117 for ( z = start-1 ; z >= 3 ; z -= 4 ) {
118 w2$$ = vshrq_n_u8(wq$$, 4);
119 w1$$ = vshlq_n_u8(wq$$, 4);
120
121 w2$$ = PMUL(w2$$, x1d);
122 wq$$ = veorq_u8(w1$$, w2$$);
123 }
124
125 switch (z) {
126 case 2:
127 w2$$ = vshrq_n_u8(wq$$, 5);
128 w1$$ = vshlq_n_u8(wq$$, 3);
129
130 w2$$ = PMUL(w2$$, x1d);
131 wq$$ = veorq_u8(w1$$, w2$$);
132 break;
133 case 1:
134 w2$$ = vshrq_n_u8(wq$$, 6);
135 w1$$ = vshlq_n_u8(wq$$, 2);
136
137 w2$$ = PMUL(w2$$, x1d);
138 wq$$ = veorq_u8(w1$$, w2$$);
139 break;
140 case 0:
114 w2$$ = MASK(wq$$);141 w2$$ = MASK(wq$$);
115 w1$$ = SHLBYTE(wq$$);142 w1$$ = SHLBYTE(wq$$);
116 143
lib/raid6/recov_neon.c+110−0
@@ -0,0 +1,110 @@
1/*
2 * Copyright (C) 2012 Intel Corporation
3 * Copyright (C) 2017 Linaro Ltd. <ard.biesheuvel@linaro.org>
4 *
5 * This program is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU General Public License
7 * as published by the Free Software Foundation; version 2
8 * of the License.
9 */
10
11#include <linux/raid/pq.h>
12
13#ifdef __KERNEL__
14#include <asm/neon.h>
15#else
16#define kernel_neon_begin()
17#define kernel_neon_end()
18#define cpu_has_neon() (1)
19#endif
20
21static int raid6_has_neon(void)
22{
23 return cpu_has_neon();
24}
25
26void __raid6_2data_recov_neon(int bytes, uint8_t *p, uint8_t *q, uint8_t *dp,
27 uint8_t *dq, const uint8_t *pbmul,
28 const uint8_t *qmul);
29
30void __raid6_datap_recov_neon(int bytes, uint8_t *p, uint8_t *q, uint8_t *dq,
31 const uint8_t *qmul);
32
33static void raid6_2data_recov_neon(int disks, size_t bytes, int faila,
34 int failb, void **ptrs)
35{
36 u8 *p, *q, *dp, *dq;
37 const u8 *pbmul; /* P multiplier table for B data */
38 const u8 *qmul; /* Q multiplier table (for both) */
39
40 p = (u8 *)ptrs[disks - 2];
41 q = (u8 *)ptrs[disks - 1];
42
43 /*
44 * Compute syndrome with zero for the missing data pages
45 * Use the dead data pages as temporary storage for
46 * delta p and delta q
47 */
48 dp = (u8 *)ptrs[faila];
49 ptrs[faila] = (void *)raid6_empty_zero_page;
50 ptrs[disks - 2] = dp;
51 dq = (u8 *)ptrs[failb];
52 ptrs[failb] = (void *)raid6_empty_zero_page;
53 ptrs[disks - 1] = dq;
54
55 raid6_call.gen_syndrome(disks, bytes, ptrs);
56
57 /* Restore pointer table */
58 ptrs[faila] = dp;
59 ptrs[failb] = dq;
60 ptrs[disks - 2] = p;
61 ptrs[disks - 1] = q;
62
63 /* Now, pick the proper data tables */
64 pbmul = raid6_vgfmul[raid6_gfexi[failb-faila]];
65 qmul = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila] ^
66 raid6_gfexp[failb]]];
67
68 kernel_neon_begin();
69 __raid6_2data_recov_neon(bytes, p, q, dp, dq, pbmul, qmul);
70 kernel_neon_end();
71}
72
73static void raid6_datap_recov_neon(int disks, size_t bytes, int faila,
74 void **ptrs)
75{
76 u8 *p, *q, *dq;
77 const u8 *qmul; /* Q multiplier table */
78
79 p = (u8 *)ptrs[disks - 2];
80 q = (u8 *)ptrs[disks - 1];
81
82 /*
83 * Compute syndrome with zero for the missing data page
84 * Use the dead data page as temporary storage for delta q
85 */
86 dq = (u8 *)ptrs[faila];
87 ptrs[faila] = (void *)raid6_empty_zero_page;
88 ptrs[disks - 1] = dq;
89
90 raid6_call.gen_syndrome(disks, bytes, ptrs);
91
92 /* Restore pointer table */
93 ptrs[faila] = dq;
94 ptrs[disks - 1] = q;
95
96 /* Now, pick the proper data tables */
97 qmul = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila]]];
98
99 kernel_neon_begin();
100 __raid6_datap_recov_neon(bytes, p, q, dq, qmul);
101 kernel_neon_end();
102}
103
104const struct raid6_recov_calls raid6_recov_neon = {
105 .data2 = raid6_2data_recov_neon,
106 .datap = raid6_datap_recov_neon,
107 .valid = raid6_has_neon,
108 .name = "neon",
109 .priority = 10,
110};
lib/raid6/recov_neon_inner.c+117−0
@@ -0,0 +1,117 @@
1/*
2 * Copyright (C) 2012 Intel Corporation
3 * Copyright (C) 2017 Linaro Ltd. <ard.biesheuvel@linaro.org>
4 *
5 * This program is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU General Public License
7 * as published by the Free Software Foundation; version 2
8 * of the License.
9 */
10
11#include <arm_neon.h>
12
13static const uint8x16_t x0f = {
14 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f,
15 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f,
16};
17
18#ifdef CONFIG_ARM
19/*
20 * AArch32 does not provide this intrinsic natively because it does not
21 * implement the underlying instruction. AArch32 only provides a 64-bit
22 * wide vtbl.8 instruction, so use that instead.
23 */
24static uint8x16_t vqtbl1q_u8(uint8x16_t a, uint8x16_t b)
25{
26 union {
27 uint8x16_t val;
28 uint8x8x2_t pair;
29 } __a = { a };
30
31 return vcombine_u8(vtbl2_u8(__a.pair, vget_low_u8(b)),
32 vtbl2_u8(__a.pair, vget_high_u8(b)));
33}
34#endif
35
36void __raid6_2data_recov_neon(int bytes, uint8_t *p, uint8_t *q, uint8_t *dp,
37 uint8_t *dq, const uint8_t *pbmul,
38 const uint8_t *qmul)
39{
40 uint8x16_t pm0 = vld1q_u8(pbmul);
41 uint8x16_t pm1 = vld1q_u8(pbmul + 16);
42 uint8x16_t qm0 = vld1q_u8(qmul);
43 uint8x16_t qm1 = vld1q_u8(qmul + 16);
44
45 /*
46 * while ( bytes-- ) {
47 * uint8_t px, qx, db;
48 *
49 * px = *p ^ *dp;
50 * qx = qmul[*q ^ *dq];
51 * *dq++ = db = pbmul[px] ^ qx;
52 * *dp++ = db ^ px;
53 * p++; q++;
54 * }
55 */
56
57 while (bytes) {
58 uint8x16_t vx, vy, px, qx, db;
59
60 px = veorq_u8(vld1q_u8(p), vld1q_u8(dp));
61 vx = veorq_u8(vld1q_u8(q), vld1q_u8(dq));
62
63 vy = (uint8x16_t)vshrq_n_s16((int16x8_t)vx, 4);
64 vx = vqtbl1q_u8(qm0, vandq_u8(vx, x0f));
65 vy = vqtbl1q_u8(qm1, vandq_u8(vy, x0f));
66 qx = veorq_u8(vx, vy);
67
68 vy = (uint8x16_t)vshrq_n_s16((int16x8_t)px, 4);
69 vx = vqtbl1q_u8(pm0, vandq_u8(px, x0f));
70 vy = vqtbl1q_u8(pm1, vandq_u8(vy, x0f));
71 vx = veorq_u8(vx, vy);
72 db = veorq_u8(vx, qx);
73
74 vst1q_u8(dq, db);
75 vst1q_u8(dp, veorq_u8(db, px));
76
77 bytes -= 16;
78 p += 16;
79 q += 16;
80 dp += 16;
81 dq += 16;
82 }
83}
84
85void __raid6_datap_recov_neon(int bytes, uint8_t *p, uint8_t *q, uint8_t *dq,
86 const uint8_t *qmul)
87{
88 uint8x16_t qm0 = vld1q_u8(qmul);
89 uint8x16_t qm1 = vld1q_u8(qmul + 16);
90
91 /*
92 * while (bytes--) {
93 * *p++ ^= *dq = qmul[*q ^ *dq];
94 * q++; dq++;
95 * }
96 */
97
98 while (bytes) {
99 uint8x16_t vx, vy;
100
101 vx = veorq_u8(vld1q_u8(q), vld1q_u8(dq));
102
103 vy = (uint8x16_t)vshrq_n_s16((int16x8_t)vx, 4);
104 vx = vqtbl1q_u8(qm0, vandq_u8(vx, x0f));
105 vy = vqtbl1q_u8(qm1, vandq_u8(vy, x0f));
106 vx = veorq_u8(vx, vy);
107 vy = veorq_u8(vx, vld1q_u8(p));
108
109 vst1q_u8(dq, vx);
110 vst1q_u8(p, vy);
111
112 bytes -= 16;
113 p += 16;
114 q += 16;
115 dq += 16;
116 }
117}

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t