Merge tag 'arm64-upstream' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux

Pull arm64 updates from Will Deacon: - Support for new architectural features introduced in ARMv8.1: * Privileged Access Never (PAN) to catch user pointer dereferences in the kernel * Large System Extension (LSE) for building scalable atomics and locks (depends on locking/arch-atomic from tip, which is included here) * Hardware Dirty Bit Management (DBM) for updating clean PTEs automatically - Move our PSCI implementation out into drivers/firmware/, where it can be shared with arch/arm/. RMK has also pulled this component branch and has additional patches moving arch/arm/ over. MAINTAINERS is updated accordingly. - Better BUG implementation based on the BRK instruction for trapping - Leaf TLB invalidation for unmapping user pages - Support for PROBE_ONLY PCI configurations - Various cleanups and non-critical fixes, including: * Always flush FP/SIMD state over exec() * Restrict memblock additions based on range of linear mapping * Ensure *(LIST_POISON) generates a fatal fault * Context-tracking syscall return no longer corrupts return value when not forced on. * Alternatives patching synchronisation/stability improvements * Signed sub-word cmpxchg compare fix (tickled by HAVE_CMPXCHG_LOCAL) * Force SMP=y * Hide direct DCC access from userspace * Fix EFI stub memory allocation when DRAM starts at 0x0 * tag 'arm64-upstream' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux: (92 commits) arm64: flush FP/SIMD state correctly after execve() arm64: makefile: fix perf_callchain.o kconfig dependency arm64: set MAX_MEMBLOCK_ADDR according to linear region size of/fdt: make memblock maximum physical address arch configurable arm64: Fix source code file path in comments arm64: entry: always restore x0 from the stack on syscall return arm64: mdscr_el1: avoid exposing DCC to userspace arm64: kconfig: Move LIST_POISON to a safe value arm64: Add __exception_irq_entry definition for function graph arm64: mm: ensure patched kernel text is fetched from PoU arm64: alternatives: ensure secondary CPUs execute ISB after patching arm64: make ll/sc __cmpxchg_case_##name asm consistent arm64: dma-mapping: Simplify pgprot handling arm64: restore cpu suspend/resume functionality ARM64: PCI: do not enable resources on PROBE_ONLY systems arm64: cmpxchg: truncate sub-word signed types before comparison arm64: alternative: put secondary CPUs into polling loop during patch arm64/Documentation: clarify wording regarding memory below the Image arm64: lse: fix lse cmpxchg code indentation arm64: remove redundant object file list ...

Linus Torvalds committed 11 years ago

Commit a4fdb2a2 parents 807249d 674c24286 files · +2321 −1145Browse files

Unified · Split

86 files

Documentation/arm64/booting.txt +7 −4
arch/arm64/Kconfig +64 −22
arch/arm64/Makefile +16 −2
arch/arm64/boot/Makefile +12 −0
arch/arm64/include/asm/alternative.h +74 −4
arch/arm64/include/asm/assembler.h +0 −14
arch/arm64/include/asm/atomic.h +52 −227
arch/arm64/include/asm/atomic_ll_sc.h +247 −0
arch/arm64/include/asm/atomic_lse.h +391 −0
arch/arm64/include/asm/barrier.h +0 −24
arch/arm64/include/asm/bug.h +64 −0
arch/arm64/include/asm/cmpxchg.h +74 −118
arch/arm64/include/asm/cpufeature.h +15 −3
arch/arm64/include/asm/cputype.h +0 −3
arch/arm64/include/asm/debug-monitors.h +16 −28
arch/arm64/include/asm/esr.h +9 −0
arch/arm64/include/asm/exception.h +6 −0
arch/arm64/include/asm/fixmap.h +1 −1
arch/arm64/include/asm/futex.h +10 −0
arch/arm64/include/asm/hardirq.h +0 −4
arch/arm64/include/asm/hugetlb.h +0 −4
arch/arm64/include/asm/irq_work.h +0 −11
arch/arm64/include/asm/lse.h +53 −0
arch/arm64/include/asm/memory.h +8 −0
arch/arm64/include/asm/mmu.h +0 −1
arch/arm64/include/asm/percpu.h +0 −8
arch/arm64/include/asm/perf_event.h +1 −1
arch/arm64/include/asm/pgtable-hwdef.h +3 −0
arch/arm64/include/asm/pgtable.h +160 −7
arch/arm64/include/asm/processor.h +2 −0
arch/arm64/include/asm/ptrace.h +0 −4
arch/arm64/include/asm/smp.h +0 −4
arch/arm64/include/asm/smp_plat.h +0 −2
arch/arm64/include/asm/spinlock.h +114 −33
arch/arm64/include/asm/spinlock_types.h +2 −0
arch/arm64/include/asm/sysreg.h +35 −5
arch/arm64/include/asm/tlb.h +6 −1
arch/arm64/include/asm/tlbflush.h +34 −42
arch/arm64/include/asm/topology.h +0 −9
arch/arm64/include/asm/traps.h +21 −2
arch/arm64/include/asm/uaccess.h +11 −0
arch/arm64/include/uapi/asm/hwcap.h +1 −0
arch/arm64/include/uapi/asm/ptrace.h +1 −0
arch/arm64/kernel/Makefile +3 −3
arch/arm64/kernel/alternative.c +25 −5
arch/arm64/kernel/armv8_deprecated.c +8 −11
arch/arm64/kernel/cpu_ops.c +0 −2
arch/arm64/kernel/cpufeature.c +46 −7
arch/arm64/kernel/debug-monitors.c +2 −2
arch/arm64/kernel/efi-stub.c +32 −9
arch/arm64/kernel/entry.S +19 −27
arch/arm64/kernel/fpsimd.c +1 −0
arch/arm64/kernel/head.S +8 −7
arch/arm64/kernel/hw_breakpoint.c +3 −3
arch/arm64/kernel/insn.c +2 −3
arch/arm64/kernel/irq.c +0 −2
arch/arm64/kernel/kgdb.c +6 −6
arch/arm64/kernel/pci.c +13 −0
arch/arm64/kernel/perf_callchain.c +196 −0
arch/arm64/kernel/perf_event.c +25 −285
arch/arm64/kernel/psci.c +0 −5
arch/arm64/kernel/ptrace.c +91 −1
arch/arm64/kernel/setup.c +34 −51
arch/arm64/kernel/sleep.S +8 −6
arch/arm64/kernel/smp.c +14 −1
arch/arm64/kernel/time.c +0 −2
arch/arm64/kernel/topology.c +1 −1
arch/arm64/kernel/traps.c +80 −14
arch/arm64/kvm/hyp.S +10 −2
arch/arm64/lib/Makefile +13 −0
arch/arm64/lib/atomic_ll_sc.c +3 −0
arch/arm64/lib/bitops.S +26 −19
arch/arm64/lib/clear_user.S +8 −0
arch/arm64/lib/copy_from_user.S +19 −6
arch/arm64/lib/copy_in_user.S +19 −6
arch/arm64/lib/copy_to_user.S +19 −6
arch/arm64/mm/cache.S +6 −1
arch/arm64/mm/context.c +0 −16
arch/arm64/mm/dma-mapping.c +20 −13
arch/arm64/mm/fault.c +26 −2
arch/arm64/mm/flush.c +0 −4
arch/arm64/mm/hugetlbpage.c +0 −4
arch/arm64/mm/init.c +2 −2
arch/arm64/mm/mmu.c +1 −12
arch/arm64/mm/proc.S +15 −6
drivers/of/fdt.c +7 −5

Documentation/arm64/booting.txt+7−4

@@ -81,7 +81,7 @@ The decompressed kernel image contains a 64-byte header as follows: · 80 unmodified lines
81	81	u64 res3 = 0; /* reserved */
82	82	u64 res4 = 0; /* reserved */
83	83	u32 magic = 0x644d5241; /* Magic number, little endian, "ARM\x64" */
84		− u32 res5; /* reserved (used for PE COFF offset) */
	84	+ u32 res5; /* reserved (used for PE COFF offset) */
85	85
86	86
87	87	Header notes:
@@ -103,7 +103,7 @@ Header notes: · 15 unmodified lines
103	103
104	104	- The flags field (introduced in v3.17) is a little-endian 64-bit field
105	105	composed as follows:
106		− Bit 0: Kernel endianness. 1 if BE, 0 if LE.
	106	+ Bit 0: Kernel endianness. 1 if BE, 0 if LE.
107	107	Bits 1-63: Reserved.
108	108
109	109	- When image_size is zero, a bootloader should attempt to keep as much
@@ -115,11 +115,14 @@ The Image must be placed text_offset bytes from a 2MB aligned base · 5 unmodified lines
115	115	address near the start of usable system RAM and called there. Memory
116	116	below that base address is currently unusable by Linux, and therefore it
117	117	is strongly recommended that this location is the start of system RAM.
	118	+The region between the 2 MB aligned base address and the start of the
	119	+image has no special significance to the kernel, and may be used for
	120	+other purposes.
118	121	At least image_size bytes from the start of the image must be free for
119	122	use by the kernel.
120	123
121		−Any memory described to the kernel (even that below the 2MB aligned base
122		−address) which is not marked as reserved from the kernel e.g. with a
	124	+Any memory described to the kernel (even that below the start of the
	125	+image) which is not marked as reserved from the kernel (e.g., with a
123	126	memreserve region in the device tree) will be considered as available to
124	127	the kernel.
125	128

arch/arm64/Kconfig+64−22

@@ -29,7 +29,7 @@ config ARM64 · 28 unmodified lines
29	29	select EDAC_SUPPORT
30	30	select GENERIC_ALLOCATOR
31	31	select GENERIC_CLOCKEVENTS
32		− select GENERIC_CLOCKEVENTS_BROADCAST if SMP
	32	+ select GENERIC_CLOCKEVENTS_BROADCAST
33	33	select GENERIC_CPU_AUTOPROBE
34	34	select GENERIC_EARLY_IOREMAP
35	35	select GENERIC_IRQ_PROBE
@@ -54,6 +54,7 @@ config ARM64 · 18 unmodified lines
54	54	select HAVE_C_RECORDMCOUNT
55	55	select HAVE_CC_STACKPROTECTOR
56	56	select HAVE_CMPXCHG_DOUBLE
	57	+ select HAVE_CMPXCHG_LOCAL
57	58	select HAVE_DEBUG_BUGVERBOSE
58	59	select HAVE_DEBUG_KMEMLEAK
59	60	select HAVE_DMA_API_DEBUG
@@ -105,6 +106,10 @@ config NO_IOPORT_MAP · 45 unmodified lines
105	106	config STACKTRACE_SUPPORT
106	107	def_bool y
107	108
	109	+config ILLEGAL_POINTER_VALUE
	110	+ hex
	111	+ default 0xdead000000000000
	112	+
108	113	config LOCKDEP_SUPPORT
109	114	def_bool y
110	115
@@ -114,6 +119,14 @@ config TRACE_IRQFLAGS_SUPPORT · 3 unmodified lines
114	119	config RWSEM_XCHGADD_ALGORITHM
115	120	def_bool y
116	121
	122	+config GENERIC_BUG
	123	+ def_bool y
	124	+ depends on BUG
	125	+
	126	+config GENERIC_BUG_RELATIVE_POINTERS
	127	+ def_bool y
	128	+ depends on GENERIC_BUG
	129	+
117	130	config GENERIC_HWEIGHT
118	131	def_bool y
119	132
@@ -138,6 +151,9 @@ config NEED_DMA_MAP_STATE · 18 unmodified lines
138	151	config NEED_SG_DMA_LENGTH
139	152	def_bool y
140	153
	154	+config SMP
	155	+ def_bool y
	156	+
141	157	config SWIOTLB
142	158	def_bool y
143	159
@@ -372,22 +388,8 @@ config CPU_BIG_ENDIAN · 228 unmodified lines
372	388	help
373	389	Say Y if you plan on running a kernel in big-endian mode.
374	390
375		−config SMP
376		− bool "Symmetric Multi-Processing"
377		− help
378		− This enables support for systems with more than one CPU. If
379		− you say N here, the kernel will run on single and
380		− multiprocessor machines, but will use only one CPU of a
381		− multiprocessor machine. If you say Y here, the kernel will run
382		− on many, but not all, single processor machines. On a single
383		− processor machine, the kernel will run faster if you say N
384		− here.
385		−
386		− If you don't know what to do here, say N.
387		−
388	391	config SCHED_MC
389	392	bool "Multi-core scheduler support"
390		− depends on SMP
391	393	help
392	394	Multi-core scheduler support improves the CPU scheduler's decision
393	395	making when dealing with multi-core CPU chips at a cost of slightly
@@ -395,7 +397,6 @@ config SCHED_MC · 1 unmodified line
395	397
396	398	config SCHED_SMT
397	399	bool "SMT scheduler support"
398		− depends on SMP
399	400	help
400	401	Improves the CPU scheduler's decision making when dealing with
401	402	MultiThreading at a cost of slightly increased overhead in some
@@ -404,23 +405,17 @@ config SCHED_SMT · 2 unmodified lines
404	405	config NR_CPUS
405	406	int "Maximum number of CPUs (2-4096)"
406	407	range 2 4096
407		− depends on SMP
408	408	# These have to remain sorted largest to smallest
409	409	default "64"
410	410
411	411	config HOTPLUG_CPU
412	412	bool "Support for hot-pluggable CPUs"
413		− depends on SMP
414	413	help
415	414	Say Y here to experiment with turning CPUs off and on. CPUs
416	415	can be controlled through /sys/devices/system/cpu.
417	416
418	417	source kernel/Kconfig.preempt
419	418
420		−config UP_LATE_INIT
421		− def_bool y
422		− depends on !SMP
423		−
424	419	config HZ
425	420	int
426	421	default 100
@@ -562,6 +557,53 @@ config SETEND_EMULATION · 135 unmodified lines
562	557	If unsure, say Y
563	558	endif
564	559
	560	+menu "ARMv8.1 architectural features"
	561	+
	562	+config ARM64_HW_AFDBM
	563	+ bool "Support for hardware updates of the Access and Dirty page flags"
	564	+ default y
	565	+ help
	566	+ The ARMv8.1 architecture extensions introduce support for
	567	+ hardware updates of the access and dirty information in page
	568	+ table entries. When enabled in TCR_EL1 (HA and HD bits) on
	569	+ capable processors, accesses to pages with PTE_AF cleared will
	570	+ set this bit instead of raising an access flag fault.
	571	+ Similarly, writes to read-only pages with the DBM bit set will
	572	+ clear the read-only bit (AP[2]) instead of raising a
	573	+ permission fault.
	574	+
	575	+ Kernels built with this configuration option enabled continue
	576	+ to work on pre-ARMv8.1 hardware and the performance impact is
	577	+ minimal. If unsure, say Y.
	578	+
	579	+config ARM64_PAN
	580	+ bool "Enable support for Privileged Access Never (PAN)"
	581	+ default y
	582	+ help
	583	+ Privileged Access Never (PAN; part of the ARMv8.1 Extensions)
	584	+ prevents the kernel or hypervisor from accessing user-space (EL0)
	585	+ memory directly.
	586	+
	587	+ Choosing this option will cause any unprotected (not using
	588	+ copy_to_user et al) memory access to fail with a permission fault.
	589	+
	590	+ The feature is detected at runtime, and will remain as a 'nop'
	591	+ instruction if the cpu does not implement the feature.
	592	+
	593	+config ARM64_LSE_ATOMICS
	594	+ bool "Atomic instructions"
	595	+ help
	596	+ As part of the Large System Extensions, ARMv8.1 introduces new
	597	+ atomic instructions that are designed specifically to scale in
	598	+ very large systems.
	599	+
	600	+ Say Y here to make use of these instructions for the in-kernel
	601	+ atomic routines. This incurs a small overhead on CPUs that do
	602	+ not support these instructions and requires the kernel to be
	603	+ built with binutils >= 2.25.
	604	+
	605	+endmenu
	606	+
565	607	endmenu
566	608
567	609	menu "Boot options"

arch/arm64/Makefile+16−2

@@ -17,7 +17,18 @@ GZFLAGS :=-9 · 16 unmodified lines
17	17
18	18	KBUILD_DEFCONFIG := defconfig
19	19
20		−KBUILD_CFLAGS += -mgeneral-regs-only
	20	+# Check for binutils support for specific extensions
	21	+lseinstr := $(call as-instr,.arch_extension lse,-DCONFIG_AS_LSE=1)
	22	+
	23	+ifeq ($(CONFIG_ARM64_LSE_ATOMICS), y)
	24	+ ifeq ($(lseinstr),)
	25	+$(warning LSE atomics not supported by binutils)
	26	+ endif
	27	+endif
	28	+
	29	+KBUILD_CFLAGS += -mgeneral-regs-only $(lseinstr)
	30	+KBUILD_AFLAGS += $(lseinstr)
	31	+
21	32	ifeq ($(CONFIG_CPU_BIG_ENDIAN), y)
22	33	KBUILD_CPPFLAGS += -mbig-endian
23	34	AS += -EB
@@ -58,7 +69,10 @@ all: $(KBUILD_IMAGE) $(KBUILD_DTBS) · 34 unmodified lines
58	69
59	70	boot := arch/arm64/boot
60	71
61		−Image Image.gz: vmlinux
	72	+Image: vmlinux
	73	+ $(Q)$(MAKE) $(build)=$(boot) $(boot)/$@
	74	+
	75	+Image.%: vmlinux
62	76	$(Q)$(MAKE) $(build)=$(boot) $(boot)/$@
63	77
64	78	zinstall install: vmlinux

arch/arm64/boot/Makefile+12−0

@@ -19,9 +19,21 @@ targets := Image Image.gz · 18 unmodified lines
19	19	$(obj)/Image: vmlinux FORCE
20	20	$(call if_changed,objcopy)
21	21
	22	+$(obj)/Image.bz2: $(obj)/Image FORCE
	23	+ $(call if_changed,bzip2)
	24	+
22	25	$(obj)/Image.gz: $(obj)/Image FORCE
23	26	$(call if_changed,gzip)
24	27
	28	+$(obj)/Image.lz4: $(obj)/Image FORCE
	29	+ $(call if_changed,lz4)
	30	+
	31	+$(obj)/Image.lzma: $(obj)/Image FORCE
	32	+ $(call if_changed,lzma)
	33	+
	34	+$(obj)/Image.lzo: $(obj)/Image FORCE
	35	+ $(call if_changed,lzo)
	36	+
25	37	install: $(obj)/Image
26	38	$(CONFIG_SHELL) $(srctree)/$(src)/install.sh $(KERNELRELEASE) \
27	39	$(obj)/Image System.map "$(INSTALL_PATH)"

arch/arm64/include/asm/alternative.h+74−4

@@ -3,6 +3,8 @@ · 2 unmodified lines
3	3
4	4	#ifndef __ASSEMBLY__
5	5
	6	+#include <linux/init.h>
	7	+#include <linux/kconfig.h>
6	8	#include <linux/types.h>
7	9	#include <linux/stddef.h>
8	10	#include <linux/stringify.h>
@@ -15,7 +17,7 @@ struct alt_instr { · 6 unmodified lines
15	17	u8 alt_len; /* size of new instruction(s), <= orig_len */
16	18	};
17	19
18		−void apply_alternatives_all(void);
	20	+void __init apply_alternatives_all(void);
19	21	void apply_alternatives(void *start, size_t length);
20	22	void free_alternatives_memory(void);
21	23
@@ -40,7 +42,8 @@ void free_alternatives_memory(void); · 18 unmodified lines
40	42	* be fixed in a binutils release posterior to 2.25.51.0.2 (anything
41	43	* containing commit 4e4d08cf7399b606 or c1baaddf8861).
42	44	*/
43		−#define ALTERNATIVE(oldinstr, newinstr, feature) \
	45	+#define __ALTERNATIVE_CFG(oldinstr, newinstr, feature, cfg_enabled) \
	46	+ ".if "__stringify(cfg_enabled)" == 1\n" \
44	47	"661:\n\t" \
45	48	oldinstr "\n" \
46	49	"662:\n" \
@@ -53,7 +56,11 @@ void free_alternatives_memory(void); · 6 unmodified lines
53	56	"664:\n\t" \
54	57	".popsection\n\t" \
55	58	".org . - (664b-663b) + (662b-661b)\n\t" \
56		− ".org . - (662b-661b) + (664b-663b)\n"
	59	+ ".org . - (662b-661b) + (664b-663b)\n" \
	60	+ ".endif\n"
	61	+
	62	+#define _ALTERNATIVE_CFG(oldinstr, newinstr, feature, cfg, ...) \
	63	+ __ALTERNATIVE_CFG(oldinstr, newinstr, feature, IS_ENABLED(cfg))
57	64
58	65	#else
59	66
@@ -65,7 +72,8 @@ void free_alternatives_memory(void); · 5 unmodified lines
65	72	.byte \alt_len
66	73	.endm
67	74
68		−.macro alternative_insn insn1 insn2 cap
	75	+.macro alternative_insn insn1, insn2, cap, enable = 1
	76	+ .if \enable
69	77	661: \insn1
70	78	662: .pushsection .altinstructions, "a"
71	79	altinstruction_entry 661b, 663f, \cap, 662b-661b, 664f-663f
@@ -75,8 +83,70 @@ void free_alternatives_memory(void); · 3 unmodified lines
75	83	664: .popsection
76	84	.org . - (664b-663b) + (662b-661b)
77	85	.org . - (662b-661b) + (664b-663b)
	86	+ .endif
	87	+.endm
	88	+
	89	+/*
	90	+ * Begin an alternative code sequence.
	91	+ *
	92	+ * The code that follows this macro will be assembled and linked as
	93	+ * normal. There are no restrictions on this code.
	94	+ */
	95	+.macro alternative_if_not cap, enable = 1
	96	+ .if \enable
	97	+ .pushsection .altinstructions, "a"
	98	+ altinstruction_entry 661f, 663f, \cap, 662f-661f, 664f-663f
	99	+ .popsection
	100	+661:
	101	+ .endif
	102	+.endm
	103	+
	104	+/*
	105	+ * Provide the alternative code sequence.
	106	+ *
	107	+ * The code that follows this macro is assembled into a special
	108	+ * section to be used for dynamic patching. Code that follows this
	109	+ * macro must:
	110	+ *
	111	+ * 1. Be exactly the same length (in bytes) as the default code
	112	+ * sequence.
	113	+ *
	114	+ * 2. Not contain a branch target that is used outside of the
	115	+ * alternative sequence it is defined in (branches into an
	116	+ * alternative sequence are not fixed up).
	117	+ */
	118	+.macro alternative_else, enable = 1
	119	+ .if \enable
	120	+662: .pushsection .altinstr_replacement, "ax"
	121	+663:
	122	+ .endif
78	123	.endm
79	124
	125	+/*
	126	+ * Complete an alternative code sequence.
	127	+ */
	128	+.macro alternative_endif, enable = 1
	129	+ .if \enable
	130	+664: .popsection
	131	+ .org . - (664b-663b) + (662b-661b)
	132	+ .org . - (662b-661b) + (664b-663b)
	133	+ .endif
	134	+.endm
	135	+
	136	+#define _ALTERNATIVE_CFG(insn1, insn2, cap, cfg, ...) \
	137	+ alternative_insn insn1, insn2, cap, IS_ENABLED(cfg)
	138	+
	139	+
80	140	#endif /* __ASSEMBLY__ */
81	141
	142	+/*
	143	+ * Usage: asm(ALTERNATIVE(oldinstr, newinstr, feature));
	144	+ *
	145	+ * Usage: asm(ALTERNATIVE(oldinstr, newinstr, feature, CONFIG_FOO));
	146	+ * N.B. If CONFIG_FOO is specified, but not selected, the whole block
	147	+ * will be omitted, including oldinstr.
	148	+ */
	149	+#define ALTERNATIVE(oldinstr, newinstr, ...) \
	150	+ _ALTERNATIVE_CFG(oldinstr, newinstr, __VA_ARGS__, 1)
	151	+
82	152	#endif /* __ASM_ALTERNATIVE_H */

arch/arm64/include/asm/assembler.h+0−14

@@ -49,18 +49,6 @@ · 48 unmodified lines
49	49	msr daifclr, #2
50	50	.endm
51	51
52		−/*
53		− * Save/disable and restore interrupts.
54		− */
55		− .macro save_and_disable_irqs, olddaif
56		− mrs \olddaif, daif
57		− disable_irq
58		− .endm
59		−
60		− .macro restore_irqs, olddaif
61		− msr daif, \olddaif
62		− .endm
63		−
64	52	/*
65	53	* Enable and disable debug exceptions.
66	54	*/
@@ -103,9 +91,7 @@ · 36 unmodified lines
103	91	* SMP data memory barrier
104	92	*/
105	93	.macro smp_dmb, opt
106		−#ifdef CONFIG_SMP
107	94	dmb \opt
108		−#endif
109	95	.endm
110	96
111	97	#define USER(l, x...) \

arch/arm64/include/asm/atomic.h+52−227

@@ -24,247 +24,72 @@ · 23 unmodified lines
24	24	#include <linux/types.h>
25	25
26	26	#include <asm/barrier.h>
27		−#include <asm/cmpxchg.h>
28		−
29		−#define ATOMIC_INIT(i) { (i) }
	27	+#include <asm/lse.h>
30	28
31	29	#ifdef __KERNEL__
32	30
33		−/*
34		− * On ARM, ordinary assignment (str instruction) doesn't clear the local
35		− * strex/ldrex monitor on some implementations. The reason we can use it for
36		− * atomic_set() is the clrex or dummy strex done on every exception return.
37		− */
38		−#define atomic_read(v) ACCESS_ONCE((v)->counter)
39		−#define atomic_set(v,i) (((v)->counter) = (i))
40		−
41		−/*
42		− * AArch64 UP and SMP safe atomic ops. We use load exclusive and
43		− * store exclusive to ensure that these are atomic. We may loop
44		− * to ensure that the update happens.
45		− */
46		−
47		−#define ATOMIC_OP(op, asm_op) \
48		−static inline void atomic_##op(int i, atomic_t *v) \
49		−{ \
50		− unsigned long tmp; \
51		− int result; \
52		− \
53		− asm volatile("// atomic_" #op "\n" \
54		−"1: ldxr %w0, %2\n" \
55		−" " #asm_op " %w0, %w0, %w3\n" \
56		−" stxr %w1, %w0, %2\n" \
57		−" cbnz %w1, 1b" \
58		− : "=&r" (result), "=&r" (tmp), "+Q" (v->counter) \
59		− : "Ir" (i)); \
60		−} \
61		−
62		−#define ATOMIC_OP_RETURN(op, asm_op) \
63		−static inline int atomic_##op##_return(int i, atomic_t *v) \
64		−{ \
65		− unsigned long tmp; \
66		− int result; \
67		− \
68		− asm volatile("// atomic_" #op "_return\n" \
69		−"1: ldxr %w0, %2\n" \
70		−" " #asm_op " %w0, %w0, %w3\n" \
71		−" stlxr %w1, %w0, %2\n" \
72		−" cbnz %w1, 1b" \
73		− : "=&r" (result), "=&r" (tmp), "+Q" (v->counter) \
74		− : "Ir" (i) \
75		− : "memory"); \
76		− \
77		− smp_mb(); \
78		− return result; \
79		−}
	31	+#define __ARM64_IN_ATOMIC_IMPL
80	32
81		−#define ATOMIC_OPS(op, asm_op) \
82		− ATOMIC_OP(op, asm_op) \
83		− ATOMIC_OP_RETURN(op, asm_op)
84		−
85		−ATOMIC_OPS(add, add)
86		−ATOMIC_OPS(sub, sub)
87		−
88		−#define atomic_andnot atomic_andnot
89		−
90		−ATOMIC_OP(and, and)
91		−ATOMIC_OP(andnot, bic)
92		−ATOMIC_OP(or, orr)
93		−ATOMIC_OP(xor, eor)
94		−
95		−#undef ATOMIC_OPS
96		−#undef ATOMIC_OP_RETURN
97		−#undef ATOMIC_OP
98		−
99		−static inline int atomic_cmpxchg(atomic_t *ptr, int old, int new)
100		−{
101		− unsigned long tmp;
102		− int oldval;
103		−
104		− smp_mb();
105		−
106		− asm volatile("// atomic_cmpxchg\n"
107		−"1: ldxr %w1, %2\n"
108		−" cmp %w1, %w3\n"
109		−" b.ne 2f\n"
110		−" stxr %w0, %w4, %2\n"
111		−" cbnz %w0, 1b\n"
112		−"2:"
113		− : "=&r" (tmp), "=&r" (oldval), "+Q" (ptr->counter)
114		− : "Ir" (old), "r" (new)
115		− : "cc");
116		−
117		− smp_mb();
118		− return oldval;
119		−}
120		−
121		−#define atomic_xchg(v, new) (xchg(&((v)->counter), new))
	33	+#if defined(CONFIG_ARM64_LSE_ATOMICS) && defined(CONFIG_AS_LSE)
	34	+#include <asm/atomic_lse.h>
	35	+#else
	36	+#include <asm/atomic_ll_sc.h>
	37	+#endif
122	38
123		−static inline int __atomic_add_unless(atomic_t *v, int a, int u)
124		−{
125		− int c, old;
	39	+#undef __ARM64_IN_ATOMIC_IMPL
126	40
127		− c = atomic_read(v);
128		− while (c != u && (old = atomic_cmpxchg((v), c, c + a)) != c)
129		− c = old;
130		− return c;
131		−}
	41	+#include <asm/cmpxchg.h>
132	42
133		−#define atomic_inc(v) atomic_add(1, v)
134		−#define atomic_dec(v) atomic_sub(1, v)
	43	+#define ___atomic_add_unless(v, a, u, sfx) \
	44	+({ \
	45	+ typeof((v)->counter) c, old; \
	46	+ \
	47	+ c = atomic##sfx##_read(v); \
	48	+ while (c != (u) && \
	49	+ (old = atomic##sfx##_cmpxchg((v), c, c + (a))) != c) \
	50	+ c = old; \
	51	+ c; \
	52	+ })
135	53
136		−#define atomic_inc_and_test(v) (atomic_add_return(1, v) == 0)
137		−#define atomic_dec_and_test(v) (atomic_sub_return(1, v) == 0)
138		−#define atomic_inc_return(v) (atomic_add_return(1, v))
139		−#define atomic_dec_return(v) (atomic_sub_return(1, v))
140		−#define atomic_sub_and_test(i, v) (atomic_sub_return(i, v) == 0)
	54	+#define ATOMIC_INIT(i) { (i) }
141	55
142		−#define atomic_add_negative(i,v) (atomic_add_return(i, v) < 0)
	56	+#define atomic_read(v) READ_ONCE((v)->counter)
	57	+#define atomic_set(v, i) (((v)->counter) = (i))
	58	+#define atomic_xchg(v, new) xchg(&((v)->counter), (new))
	59	+#define atomic_cmpxchg(v, old, new) cmpxchg(&((v)->counter), (old), (new))
	60	+
	61	+#define atomic_inc(v) atomic_add(1, (v))
	62	+#define atomic_dec(v) atomic_sub(1, (v))
	63	+#define atomic_inc_return(v) atomic_add_return(1, (v))
	64	+#define atomic_dec_return(v) atomic_sub_return(1, (v))
	65	+#define atomic_inc_and_test(v) (atomic_inc_return(v) == 0)
	66	+#define atomic_dec_and_test(v) (atomic_dec_return(v) == 0)
	67	+#define atomic_sub_and_test(i, v) (atomic_sub_return((i), (v)) == 0)
	68	+#define atomic_add_negative(i, v) (atomic_add_return((i), (v)) < 0)
	69	+#define __atomic_add_unless(v, a, u) ___atomic_add_unless(v, a, u,)
	70	+#define atomic_andnot atomic_andnot
143	71
144	72	/*
145	73	* 64-bit atomic operations.
146	74	*/
147		−#define ATOMIC64_INIT(i) { (i) }
148		−
149		−#define atomic64_read(v) ACCESS_ONCE((v)->counter)
150		−#define atomic64_set(v,i) (((v)->counter) = (i))
151		−
152		−#define ATOMIC64_OP(op, asm_op) \
153		−static inline void atomic64_##op(long i, atomic64_t *v) \
154		−{ \
155		− long result; \
156		− unsigned long tmp; \
157		− \
158		− asm volatile("// atomic64_" #op "\n" \
159		−"1: ldxr %0, %2\n" \
160		−" " #asm_op " %0, %0, %3\n" \
161		−" stxr %w1, %0, %2\n" \
162		−" cbnz %w1, 1b" \
163		− : "=&r" (result), "=&r" (tmp), "+Q" (v->counter) \
164		− : "Ir" (i)); \
165		−} \
166		−
167		−#define ATOMIC64_OP_RETURN(op, asm_op) \
168		−static inline long atomic64_##op##_return(long i, atomic64_t *v) \
169		−{ \
170		− long result; \
171		− unsigned long tmp; \
172		− \
173		− asm volatile("// atomic64_" #op "_return\n" \
174		−"1: ldxr %0, %2\n" \
175		−" " #asm_op " %0, %0, %3\n" \
176		−" stlxr %w1, %0, %2\n" \
177		−" cbnz %w1, 1b" \
178		− : "=&r" (result), "=&r" (tmp), "+Q" (v->counter) \
179		− : "Ir" (i) \
180		− : "memory"); \
181		− \
182		− smp_mb(); \
183		− return result; \
184		−}
185		−
186		−#define ATOMIC64_OPS(op, asm_op) \
187		− ATOMIC64_OP(op, asm_op) \
188		− ATOMIC64_OP_RETURN(op, asm_op)
189		−
190		−ATOMIC64_OPS(add, add)
191		−ATOMIC64_OPS(sub, sub)
192		−
193		−#define atomic64_andnot atomic64_andnot
194		−
195		−ATOMIC64_OP(and, and)
196		−ATOMIC64_OP(andnot, bic)
197		−ATOMIC64_OP(or, orr)
198		−ATOMIC64_OP(xor, eor)
199		−
200		−#undef ATOMIC64_OPS
201		−#undef ATOMIC64_OP_RETURN
202		−#undef ATOMIC64_OP
203		−
204		−static inline long atomic64_cmpxchg(atomic64_t *ptr, long old, long new)
205		−{
206		− long oldval;
207		− unsigned long res;
208		−
209		− smp_mb();
210		−
211		− asm volatile("// atomic64_cmpxchg\n"
212		−"1: ldxr %1, %2\n"
213		−" cmp %1, %3\n"
214		−" b.ne 2f\n"
215		−" stxr %w0, %4, %2\n"
216		−" cbnz %w0, 1b\n"
217		−"2:"
218		− : "=&r" (res), "=&r" (oldval), "+Q" (ptr->counter)
219		− : "Ir" (old), "r" (new)
220		− : "cc");
221		−
222		− smp_mb();
223		− return oldval;
224		−}
225		−
226		−#define atomic64_xchg(v, new) (xchg(&((v)->counter), new))
227		−
228		−static inline long atomic64_dec_if_positive(atomic64_t *v)
229		−{
230		− long result;
231		− unsigned long tmp;
232		−
233		− asm volatile("// atomic64_dec_if_positive\n"
234		−"1: ldxr %0, %2\n"
235		−" subs %0, %0, #1\n"
236		−" b.mi 2f\n"
237		−" stlxr %w1, %0, %2\n"
238		−" cbnz %w1, 1b\n"
239		−" dmb ish\n"
240		−"2:"
241		− : "=&r" (result), "=&r" (tmp), "+Q" (v->counter)
242		− :
243		− : "cc", "memory");
244		−
245		− return result;
246		−}
247		−
248		−static inline int atomic64_add_unless(atomic64_t *v, long a, long u)
249		−{
250		− long c, old;
251		−
252		− c = atomic64_read(v);
253		− while (c != u && (old = atomic64_cmpxchg((v), c, c + a)) != c)
254		− c = old;
255		−
256		− return c != u;
257		−}
258		−
259		−#define atomic64_add_negative(a, v) (atomic64_add_return((a), (v)) < 0)
260		−#define atomic64_inc(v) atomic64_add(1LL, (v))
261		−#define atomic64_inc_return(v) atomic64_add_return(1LL, (v))
	75	+#define ATOMIC64_INIT ATOMIC_INIT
	76	+#define atomic64_read atomic_read
	77	+#define atomic64_set atomic_set
	78	+#define atomic64_xchg atomic_xchg
	79	+#define atomic64_cmpxchg atomic_cmpxchg
	80	+
	81	+#define atomic64_inc(v) atomic64_add(1, (v))
	82	+#define atomic64_dec(v) atomic64_sub(1, (v))
	83	+#define atomic64_inc_return(v) atomic64_add_return(1, (v))
	84	+#define atomic64_dec_return(v) atomic64_sub_return(1, (v))
262	85	#define atomic64_inc_and_test(v) (atomic64_inc_return(v) == 0)
263		−#define atomic64_sub_and_test(a, v) (atomic64_sub_return((a), (v)) == 0)
264		−#define atomic64_dec(v) atomic64_sub(1LL, (v))
265		−#define atomic64_dec_return(v) atomic64_sub_return(1LL, (v))
266		−#define atomic64_dec_and_test(v) (atomic64_dec_return((v)) == 0)
267		−#define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1LL, 0LL)
	86	+#define atomic64_dec_and_test(v) (atomic64_dec_return(v) == 0)
	87	+#define atomic64_sub_and_test(i, v) (atomic64_sub_return((i), (v)) == 0)
	88	+#define atomic64_add_negative(i, v) (atomic64_add_return((i), (v)) < 0)
	89	+#define atomic64_add_unless(v, a, u) (___atomic_add_unless(v, a, u, 64) != u)
	90	+#define atomic64_andnot atomic64_andnot
	91	+
	92	+#define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1, 0)
268	93
269	94	#endif
270	95	#endif

arch/arm64/include/asm/atomic_ll_sc.h+247−0

@@ -0,0 +1,247 @@
	1	+/*
	2	+ * Based on arch/arm/include/asm/atomic.h
	3	+ *
	4	+ * Copyright (C) 1996 Russell King.
	5	+ * Copyright (C) 2002 Deep Blue Solutions Ltd.
	6	+ * Copyright (C) 2012 ARM Ltd.
	7	+ *
	8	+ * This program is free software; you can redistribute it and/or modify
	9	+ * it under the terms of the GNU General Public License version 2 as
	10	+ * published by the Free Software Foundation.
	11	+ *
	12	+ * This program is distributed in the hope that it will be useful,
	13	+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
	14	+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	15	+ * GNU General Public License for more details.
	16	+ *
	17	+ * You should have received a copy of the GNU General Public License
	18	+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
	19	+ */
	20	+
	21	+#ifndef __ASM_ATOMIC_LL_SC_H
	22	+#define __ASM_ATOMIC_LL_SC_H
	23	+
	24	+#ifndef __ARM64_IN_ATOMIC_IMPL
	25	+#error "please don't include this file directly"
	26	+#endif
	27	+
	28	+/*
	29	+ * AArch64 UP and SMP safe atomic ops. We use load exclusive and
	30	+ * store exclusive to ensure that these are atomic. We may loop
	31	+ * to ensure that the update happens.
	32	+ *
	33	+ * NOTE: these functions do not follow the PCS and must explicitly
	34	+ * save any clobbered registers other than x0 (regardless of return
	35	+ * value). This is achieved through -fcall-saved-* compiler flags for
	36	+ * this file, which unfortunately don't work on a per-function basis
	37	+ * (the optimize attribute silently ignores these options).
	38	+ */
	39	+
	40	+#define ATOMIC_OP(op, asm_op) \
	41	+__LL_SC_INLINE void \
	42	+__LL_SC_PREFIX(atomic_##op(int i, atomic_t *v)) \
	43	+{ \
	44	+ unsigned long tmp; \
	45	+ int result; \
	46	+ \
	47	+ asm volatile("// atomic_" #op "\n" \
	48	+" prfm pstl1strm, %2\n" \
	49	+"1: ldxr %w0, %2\n" \
	50	+" " #asm_op " %w0, %w0, %w3\n" \
	51	+" stxr %w1, %w0, %2\n" \
	52	+" cbnz %w1, 1b" \
	53	+ : "=&r" (result), "=&r" (tmp), "+Q" (v->counter) \
	54	+ : "Ir" (i)); \
	55	+} \
	56	+__LL_SC_EXPORT(atomic_##op);
	57	+
	58	+#define ATOMIC_OP_RETURN(op, asm_op) \
	59	+__LL_SC_INLINE int \
	60	+__LL_SC_PREFIX(atomic_##op##_return(int i, atomic_t *v)) \
	61	+{ \
	62	+ unsigned long tmp; \
	63	+ int result; \
	64	+ \
	65	+ asm volatile("// atomic_" #op "_return\n" \
	66	+" prfm pstl1strm, %2\n" \
	67	+"1: ldxr %w0, %2\n" \
	68	+" " #asm_op " %w0, %w0, %w3\n" \
	69	+" stlxr %w1, %w0, %2\n" \
	70	+" cbnz %w1, 1b" \
	71	+ : "=&r" (result), "=&r" (tmp), "+Q" (v->counter) \
	72	+ : "Ir" (i) \
	73	+ : "memory"); \
	74	+ \
	75	+ smp_mb(); \
	76	+ return result; \
	77	+} \
	78	+__LL_SC_EXPORT(atomic_##op##_return);
	79	+
	80	+#define ATOMIC_OPS(op, asm_op) \
	81	+ ATOMIC_OP(op, asm_op) \
	82	+ ATOMIC_OP_RETURN(op, asm_op)
	83	+
	84	+ATOMIC_OPS(add, add)
	85	+ATOMIC_OPS(sub, sub)
	86	+
	87	+ATOMIC_OP(and, and)
	88	+ATOMIC_OP(andnot, bic)
	89	+ATOMIC_OP(or, orr)
	90	+ATOMIC_OP(xor, eor)
	91	+
	92	+#undef ATOMIC_OPS
	93	+#undef ATOMIC_OP_RETURN
	94	+#undef ATOMIC_OP
	95	+
	96	+#define ATOMIC64_OP(op, asm_op) \
	97	+__LL_SC_INLINE void \
	98	+__LL_SC_PREFIX(atomic64_##op(long i, atomic64_t *v)) \
	99	+{ \
	100	+ long result; \
	101	+ unsigned long tmp; \
	102	+ \
	103	+ asm volatile("// atomic64_" #op "\n" \
	104	+" prfm pstl1strm, %2\n" \
	105	+"1: ldxr %0, %2\n" \
	106	+" " #asm_op " %0, %0, %3\n" \
	107	+" stxr %w1, %0, %2\n" \
	108	+" cbnz %w1, 1b" \
	109	+ : "=&r" (result), "=&r" (tmp), "+Q" (v->counter) \
	110	+ : "Ir" (i)); \
	111	+} \
	112	+__LL_SC_EXPORT(atomic64_##op);
	113	+
	114	+#define ATOMIC64_OP_RETURN(op, asm_op) \
	115	+__LL_SC_INLINE long \
	116	+__LL_SC_PREFIX(atomic64_##op##_return(long i, atomic64_t *v)) \
	117	+{ \
	118	+ long result; \
	119	+ unsigned long tmp; \
	120	+ \
	121	+ asm volatile("// atomic64_" #op "_return\n" \
	122	+" prfm pstl1strm, %2\n" \
	123	+"1: ldxr %0, %2\n" \
	124	+" " #asm_op " %0, %0, %3\n" \
	125	+" stlxr %w1, %0, %2\n" \
	126	+" cbnz %w1, 1b" \
	127	+ : "=&r" (result), "=&r" (tmp), "+Q" (v->counter) \
	128	+ : "Ir" (i) \
	129	+ : "memory"); \
	130	+ \
	131	+ smp_mb(); \
	132	+ return result; \
	133	+} \
	134	+__LL_SC_EXPORT(atomic64_##op##_return);
	135	+
	136	+#define ATOMIC64_OPS(op, asm_op) \
	137	+ ATOMIC64_OP(op, asm_op) \
	138	+ ATOMIC64_OP_RETURN(op, asm_op)
	139	+
	140	+ATOMIC64_OPS(add, add)
	141	+ATOMIC64_OPS(sub, sub)
	142	+
	143	+ATOMIC64_OP(and, and)
	144	+ATOMIC64_OP(andnot, bic)
	145	+ATOMIC64_OP(or, orr)
	146	+ATOMIC64_OP(xor, eor)
	147	+
	148	+#undef ATOMIC64_OPS
	149	+#undef ATOMIC64_OP_RETURN
	150	+#undef ATOMIC64_OP
	151	+
	152	+__LL_SC_INLINE long
	153	+__LL_SC_PREFIX(atomic64_dec_if_positive(atomic64_t *v))
	154	+{
	155	+ long result;
	156	+ unsigned long tmp;
	157	+
	158	+ asm volatile("// atomic64_dec_if_positive\n"
	159	+" prfm pstl1strm, %2\n"
	160	+"1: ldxr %0, %2\n"
	161	+" subs %0, %0, #1\n"
	162	+" b.lt 2f\n"
	163	+" stlxr %w1, %0, %2\n"
	164	+" cbnz %w1, 1b\n"
	165	+" dmb ish\n"
	166	+"2:"
	167	+ : "=&r" (result), "=&r" (tmp), "+Q" (v->counter)
	168	+ :
	169	+ : "cc", "memory");
	170	+
	171	+ return result;
	172	+}
	173	+__LL_SC_EXPORT(atomic64_dec_if_positive);
	174	+
	175	+#define __CMPXCHG_CASE(w, sz, name, mb, rel, cl) \
	176	+__LL_SC_INLINE unsigned long \
	177	+__LL_SC_PREFIX(__cmpxchg_case_##name(volatile void *ptr, \
	178	+ unsigned long old, \
	179	+ unsigned long new)) \
	180	+{ \
	181	+ unsigned long tmp, oldval; \
	182	+ \
	183	+ asm volatile( \
	184	+ " prfm pstl1strm, %[v]\n" \
	185	+ "1: ldxr" #sz "\t%" #w "[oldval], %[v]\n" \
	186	+ " eor %" #w "[tmp], %" #w "[oldval], %" #w "[old]\n" \
	187	+ " cbnz %" #w "[tmp], 2f\n" \
	188	+ " st" #rel "xr" #sz "\t%w[tmp], %" #w "[new], %[v]\n" \
	189	+ " cbnz %w[tmp], 1b\n" \
	190	+ " " #mb "\n" \
	191	+ " mov %" #w "[oldval], %" #w "[old]\n" \
	192	+ "2:" \
	193	+ : [tmp] "=&r" (tmp), [oldval] "=&r" (oldval), \
	194	+ [v] "+Q" ((unsigned long )ptr) \
	195	+ : [old] "Lr" (old), [new] "r" (new) \
	196	+ : cl); \
	197	+ \
	198	+ return oldval; \
	199	+} \
	200	+__LL_SC_EXPORT(__cmpxchg_case_##name);
	201	+
	202	+__CMPXCHG_CASE(w, b, 1, , , )
	203	+__CMPXCHG_CASE(w, h, 2, , , )
	204	+__CMPXCHG_CASE(w, , 4, , , )
	205	+__CMPXCHG_CASE( , , 8, , , )
	206	+__CMPXCHG_CASE(w, b, mb_1, dmb ish, l, "memory")
	207	+__CMPXCHG_CASE(w, h, mb_2, dmb ish, l, "memory")
	208	+__CMPXCHG_CASE(w, , mb_4, dmb ish, l, "memory")
	209	+__CMPXCHG_CASE( , , mb_8, dmb ish, l, "memory")
	210	+
	211	+#undef __CMPXCHG_CASE
	212	+
	213	+#define __CMPXCHG_DBL(name, mb, rel, cl) \
	214	+__LL_SC_INLINE int \
	215	+__LL_SC_PREFIX(__cmpxchg_double##name(unsigned long old1, \
	216	+ unsigned long old2, \
	217	+ unsigned long new1, \
	218	+ unsigned long new2, \
	219	+ volatile void *ptr)) \
	220	+{ \
	221	+ unsigned long tmp, ret; \
	222	+ \
	223	+ asm volatile("// __cmpxchg_double" #name "\n" \
	224	+ " prfm pstl1strm, %2\n" \
	225	+ "1: ldxp %0, %1, %2\n" \
	226	+ " eor %0, %0, %3\n" \
	227	+ " eor %1, %1, %4\n" \
	228	+ " orr %1, %0, %1\n" \
	229	+ " cbnz %1, 2f\n" \
	230	+ " st" #rel "xp %w0, %5, %6, %2\n" \
	231	+ " cbnz %w0, 1b\n" \
	232	+ " " #mb "\n" \
	233	+ "2:" \
	234	+ : "=&r" (tmp), "=&r" (ret), "+Q" ((unsigned long )ptr) \
	235	+ : "r" (old1), "r" (old2), "r" (new1), "r" (new2) \
	236	+ : cl); \
	237	+ \
	238	+ return ret; \
	239	+} \
	240	+__LL_SC_EXPORT(__cmpxchg_double##name);
	241	+
	242	+__CMPXCHG_DBL( , , , )
	243	+__CMPXCHG_DBL(_mb, dmb ish, l, "memory")
	244	+
	245	+#undef __CMPXCHG_DBL
	246	+
	247	+#endif /* __ASM_ATOMIC_LL_SC_H */

arch/arm64/include/asm/atomic_lse.h+391−0

@@ -0,0 +1,391 @@
	1	+/*
	2	+ * Based on arch/arm/include/asm/atomic.h
	3	+ *
	4	+ * Copyright (C) 1996 Russell King.
	5	+ * Copyright (C) 2002 Deep Blue Solutions Ltd.
	6	+ * Copyright (C) 2012 ARM Ltd.
	7	+ *
	8	+ * This program is free software; you can redistribute it and/or modify
	9	+ * it under the terms of the GNU General Public License version 2 as
	10	+ * published by the Free Software Foundation.
	11	+ *
	12	+ * This program is distributed in the hope that it will be useful,
	13	+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
	14	+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	15	+ * GNU General Public License for more details.
	16	+ *
	17	+ * You should have received a copy of the GNU General Public License
	18	+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
	19	+ */
	20	+
	21	+#ifndef __ASM_ATOMIC_LSE_H
	22	+#define __ASM_ATOMIC_LSE_H
	23	+
	24	+#ifndef __ARM64_IN_ATOMIC_IMPL
	25	+#error "please don't include this file directly"
	26	+#endif
	27	+
	28	+#define __LL_SC_ATOMIC(op) __LL_SC_CALL(atomic_##op)
	29	+
	30	+static inline void atomic_andnot(int i, atomic_t *v)
	31	+{
	32	+ register int w0 asm ("w0") = i;
	33	+ register atomic_t *x1 asm ("x1") = v;
	34	+
	35	+ asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC(andnot),
	36	+ " stclr %w[i], %[v]\n")
	37	+ : [i] "+r" (w0), [v] "+Q" (v->counter)
	38	+ : "r" (x1)
	39	+ : "x30");
	40	+}
	41	+
	42	+static inline void atomic_or(int i, atomic_t *v)
	43	+{
	44	+ register int w0 asm ("w0") = i;
	45	+ register atomic_t *x1 asm ("x1") = v;
	46	+
	47	+ asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC(or),
	48	+ " stset %w[i], %[v]\n")
	49	+ : [i] "+r" (w0), [v] "+Q" (v->counter)
	50	+ : "r" (x1)
	51	+ : "x30");
	52	+}
	53	+
	54	+static inline void atomic_xor(int i, atomic_t *v)
	55	+{
	56	+ register int w0 asm ("w0") = i;
	57	+ register atomic_t *x1 asm ("x1") = v;
	58	+
	59	+ asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC(xor),
	60	+ " steor %w[i], %[v]\n")
	61	+ : [i] "+r" (w0), [v] "+Q" (v->counter)
	62	+ : "r" (x1)
	63	+ : "x30");
	64	+}
	65	+
	66	+static inline void atomic_add(int i, atomic_t *v)
	67	+{
	68	+ register int w0 asm ("w0") = i;
	69	+ register atomic_t *x1 asm ("x1") = v;
	70	+
	71	+ asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC(add),
	72	+ " stadd %w[i], %[v]\n")
	73	+ : [i] "+r" (w0), [v] "+Q" (v->counter)
	74	+ : "r" (x1)
	75	+ : "x30");
	76	+}
	77	+
	78	+static inline int atomic_add_return(int i, atomic_t *v)
	79	+{
	80	+ register int w0 asm ("w0") = i;
	81	+ register atomic_t *x1 asm ("x1") = v;
	82	+
	83	+ asm volatile(ARM64_LSE_ATOMIC_INSN(
	84	+ /* LL/SC */
	85	+ " nop\n"
	86	+ __LL_SC_ATOMIC(add_return),
	87	+ /* LSE atomics */
	88	+ " ldaddal %w[i], w30, %[v]\n"
	89	+ " add %w[i], %w[i], w30")
	90	+ : [i] "+r" (w0), [v] "+Q" (v->counter)
	91	+ : "r" (x1)
	92	+ : "x30", "memory");
	93	+
	94	+ return w0;
	95	+}
	96	+
	97	+static inline void atomic_and(int i, atomic_t *v)
	98	+{
	99	+ register int w0 asm ("w0") = i;
	100	+ register atomic_t *x1 asm ("x1") = v;
	101	+
	102	+ asm volatile(ARM64_LSE_ATOMIC_INSN(
	103	+ /* LL/SC */
	104	+ " nop\n"
	105	+ __LL_SC_ATOMIC(and),
	106	+ /* LSE atomics */
	107	+ " mvn %w[i], %w[i]\n"
	108	+ " stclr %w[i], %[v]")
	109	+ : [i] "+r" (w0), [v] "+Q" (v->counter)
	110	+ : "r" (x1)
	111	+ : "x30");
	112	+}
	113	+
	114	+static inline void atomic_sub(int i, atomic_t *v)
	115	+{
	116	+ register int w0 asm ("w0") = i;
	117	+ register atomic_t *x1 asm ("x1") = v;
	118	+
	119	+ asm volatile(ARM64_LSE_ATOMIC_INSN(
	120	+ /* LL/SC */
	121	+ " nop\n"
	122	+ __LL_SC_ATOMIC(sub),
	123	+ /* LSE atomics */
	124	+ " neg %w[i], %w[i]\n"
	125	+ " stadd %w[i], %[v]")
	126	+ : [i] "+r" (w0), [v] "+Q" (v->counter)
	127	+ : "r" (x1)
	128	+ : "x30");
	129	+}
	130	+
	131	+static inline int atomic_sub_return(int i, atomic_t *v)
	132	+{
	133	+ register int w0 asm ("w0") = i;
	134	+ register atomic_t *x1 asm ("x1") = v;
	135	+
	136	+ asm volatile(ARM64_LSE_ATOMIC_INSN(
	137	+ /* LL/SC */
	138	+ " nop\n"
	139	+ __LL_SC_ATOMIC(sub_return)
	140	+ " nop",
	141	+ /* LSE atomics */
	142	+ " neg %w[i], %w[i]\n"
	143	+ " ldaddal %w[i], w30, %[v]\n"
	144	+ " add %w[i], %w[i], w30")
	145	+ : [i] "+r" (w0), [v] "+Q" (v->counter)
	146	+ : "r" (x1)
	147	+ : "x30", "memory");
	148	+
	149	+ return w0;
	150	+}
	151	+
	152	+#undef __LL_SC_ATOMIC
	153	+
	154	+#define __LL_SC_ATOMIC64(op) __LL_SC_CALL(atomic64_##op)
	155	+
	156	+static inline void atomic64_andnot(long i, atomic64_t *v)
	157	+{
	158	+ register long x0 asm ("x0") = i;
	159	+ register atomic64_t *x1 asm ("x1") = v;
	160	+
	161	+ asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC64(andnot),
	162	+ " stclr %[i], %[v]\n")
	163	+ : [i] "+r" (x0), [v] "+Q" (v->counter)
	164	+ : "r" (x1)
	165	+ : "x30");
	166	+}
	167	+
	168	+static inline void atomic64_or(long i, atomic64_t *v)
	169	+{
	170	+ register long x0 asm ("x0") = i;
	171	+ register atomic64_t *x1 asm ("x1") = v;
	172	+
	173	+ asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC64(or),
	174	+ " stset %[i], %[v]\n")
	175	+ : [i] "+r" (x0), [v] "+Q" (v->counter)
	176	+ : "r" (x1)
	177	+ : "x30");
	178	+}
	179	+
	180	+static inline void atomic64_xor(long i, atomic64_t *v)
	181	+{
	182	+ register long x0 asm ("x0") = i;
	183	+ register atomic64_t *x1 asm ("x1") = v;
	184	+
	185	+ asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC64(xor),
	186	+ " steor %[i], %[v]\n")
	187	+ : [i] "+r" (x0), [v] "+Q" (v->counter)
	188	+ : "r" (x1)
	189	+ : "x30");
	190	+}
	191	+
	192	+static inline void atomic64_add(long i, atomic64_t *v)
	193	+{
	194	+ register long x0 asm ("x0") = i;
	195	+ register atomic64_t *x1 asm ("x1") = v;
	196	+
	197	+ asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC64(add),
	198	+ " stadd %[i], %[v]\n")
	199	+ : [i] "+r" (x0), [v] "+Q" (v->counter)
	200	+ : "r" (x1)
	201	+ : "x30");
	202	+}
	203	+
	204	+static inline long atomic64_add_return(long i, atomic64_t *v)
	205	+{
	206	+ register long x0 asm ("x0") = i;
	207	+ register atomic64_t *x1 asm ("x1") = v;
	208	+
	209	+ asm volatile(ARM64_LSE_ATOMIC_INSN(
	210	+ /* LL/SC */
	211	+ " nop\n"
	212	+ __LL_SC_ATOMIC64(add_return),
	213	+ /* LSE atomics */
	214	+ " ldaddal %[i], x30, %[v]\n"
	215	+ " add %[i], %[i], x30")
	216	+ : [i] "+r" (x0), [v] "+Q" (v->counter)
	217	+ : "r" (x1)
	218	+ : "x30", "memory");
	219	+
	220	+ return x0;
	221	+}
	222	+
	223	+static inline void atomic64_and(long i, atomic64_t *v)
	224	+{
	225	+ register long x0 asm ("x0") = i;
	226	+ register atomic64_t *x1 asm ("x1") = v;
	227	+
	228	+ asm volatile(ARM64_LSE_ATOMIC_INSN(
	229	+ /* LL/SC */
	230	+ " nop\n"
	231	+ __LL_SC_ATOMIC64(and),
	232	+ /* LSE atomics */
	233	+ " mvn %[i], %[i]\n"
	234	+ " stclr %[i], %[v]")
	235	+ : [i] "+r" (x0), [v] "+Q" (v->counter)
	236	+ : "r" (x1)
	237	+ : "x30");
	238	+}
	239	+
	240	+static inline void atomic64_sub(long i, atomic64_t *v)
	241	+{
	242	+ register long x0 asm ("x0") = i;
	243	+ register atomic64_t *x1 asm ("x1") = v;
	244	+
	245	+ asm volatile(ARM64_LSE_ATOMIC_INSN(
	246	+ /* LL/SC */
	247	+ " nop\n"
	248	+ __LL_SC_ATOMIC64(sub),
	249	+ /* LSE atomics */
	250	+ " neg %[i], %[i]\n"
	251	+ " stadd %[i], %[v]")
	252	+ : [i] "+r" (x0), [v] "+Q" (v->counter)
	253	+ : "r" (x1)
	254	+ : "x30");
	255	+}
	256	+
	257	+static inline long atomic64_sub_return(long i, atomic64_t *v)
	258	+{
	259	+ register long x0 asm ("x0") = i;
	260	+ register atomic64_t *x1 asm ("x1") = v;
	261	+
	262	+ asm volatile(ARM64_LSE_ATOMIC_INSN(
	263	+ /* LL/SC */
	264	+ " nop\n"
	265	+ __LL_SC_ATOMIC64(sub_return)
	266	+ " nop",
	267	+ /* LSE atomics */
	268	+ " neg %[i], %[i]\n"
	269	+ " ldaddal %[i], x30, %[v]\n"
	270	+ " add %[i], %[i], x30")
	271	+ : [i] "+r" (x0), [v] "+Q" (v->counter)
	272	+ : "r" (x1)
	273	+ : "x30", "memory");
	274	+
	275	+ return x0;
	276	+}
	277	+
	278	+static inline long atomic64_dec_if_positive(atomic64_t *v)
	279	+{
	280	+ register long x0 asm ("x0") = (long)v;
	281	+
	282	+ asm volatile(ARM64_LSE_ATOMIC_INSN(
	283	+ /* LL/SC */
	284	+ " nop\n"
	285	+ __LL_SC_ATOMIC64(dec_if_positive)
	286	+ " nop\n"
	287	+ " nop\n"
	288	+ " nop\n"
	289	+ " nop\n"
	290	+ " nop",
	291	+ /* LSE atomics */
	292	+ "1: ldr x30, %[v]\n"
	293	+ " subs %[ret], x30, #1\n"
	294	+ " b.lt 2f\n"
	295	+ " casal x30, %[ret], %[v]\n"
	296	+ " sub x30, x30, #1\n"
	297	+ " sub x30, x30, %[ret]\n"
	298	+ " cbnz x30, 1b\n"
	299	+ "2:")
	300	+ : [ret] "+&r" (x0), [v] "+Q" (v->counter)
	301	+ :
	302	+ : "x30", "cc", "memory");
	303	+
	304	+ return x0;
	305	+}
	306	+
	307	+#undef __LL_SC_ATOMIC64
	308	+
	309	+#define __LL_SC_CMPXCHG(op) __LL_SC_CALL(__cmpxchg_case_##op)
	310	+
	311	+#define __CMPXCHG_CASE(w, sz, name, mb, cl...) \
	312	+static inline unsigned long __cmpxchg_case_##name(volatile void *ptr, \
	313	+ unsigned long old, \
	314	+ unsigned long new) \
	315	+{ \
	316	+ register unsigned long x0 asm ("x0") = (unsigned long)ptr; \
	317	+ register unsigned long x1 asm ("x1") = old; \
	318	+ register unsigned long x2 asm ("x2") = new; \
	319	+ \
	320	+ asm volatile(ARM64_LSE_ATOMIC_INSN( \
	321	+ /* LL/SC */ \
	322	+ " nop\n" \
	323	+ __LL_SC_CMPXCHG(name) \
	324	+ " nop", \
	325	+ /* LSE atomics */ \
	326	+ " mov " #w "30, %" #w "[old]\n" \
	327	+ " cas" #mb #sz "\t" #w "30, %" #w "[new], %[v]\n" \
	328	+ " mov %" #w "[ret], " #w "30") \
	329	+ : [ret] "+r" (x0), [v] "+Q" ((unsigned long )ptr) \
	330	+ : [old] "r" (x1), [new] "r" (x2) \
	331	+ : "x30" , ##cl); \
	332	+ \
	333	+ return x0; \
	334	+}
	335	+
	336	+__CMPXCHG_CASE(w, b, 1, )
	337	+__CMPXCHG_CASE(w, h, 2, )
	338	+__CMPXCHG_CASE(w, , 4, )
	339	+__CMPXCHG_CASE(x, , 8, )
	340	+__CMPXCHG_CASE(w, b, mb_1, al, "memory")
	341	+__CMPXCHG_CASE(w, h, mb_2, al, "memory")
	342	+__CMPXCHG_CASE(w, , mb_4, al, "memory")
	343	+__CMPXCHG_CASE(x, , mb_8, al, "memory")
	344	+
	345	+#undef __LL_SC_CMPXCHG
	346	+#undef __CMPXCHG_CASE
	347	+
	348	+#define __LL_SC_CMPXCHG_DBL(op) __LL_SC_CALL(__cmpxchg_double##op)
	349	+
	350	+#define __CMPXCHG_DBL(name, mb, cl...) \
	351	+static inline int __cmpxchg_double##name(unsigned long old1, \
	352	+ unsigned long old2, \
	353	+ unsigned long new1, \
	354	+ unsigned long new2, \
	355	+ volatile void *ptr) \
	356	+{ \
	357	+ unsigned long oldval1 = old1; \
	358	+ unsigned long oldval2 = old2; \
	359	+ register unsigned long x0 asm ("x0") = old1; \
	360	+ register unsigned long x1 asm ("x1") = old2; \
	361	+ register unsigned long x2 asm ("x2") = new1; \
	362	+ register unsigned long x3 asm ("x3") = new2; \
	363	+ register unsigned long x4 asm ("x4") = (unsigned long)ptr; \
	364	+ \
	365	+ asm volatile(ARM64_LSE_ATOMIC_INSN( \
	366	+ /* LL/SC */ \
	367	+ " nop\n" \
	368	+ " nop\n" \
	369	+ " nop\n" \
	370	+ __LL_SC_CMPXCHG_DBL(name), \
	371	+ /* LSE atomics */ \
	372	+ " casp" #mb "\t%[old1], %[old2], %[new1], %[new2], %[v]\n"\
	373	+ " eor %[old1], %[old1], %[oldval1]\n" \
	374	+ " eor %[old2], %[old2], %[oldval2]\n" \
	375	+ " orr %[old1], %[old1], %[old2]") \
	376	+ : [old1] "+r" (x0), [old2] "+r" (x1), \
	377	+ [v] "+Q" ((unsigned long )ptr) \
	378	+ : [new1] "r" (x2), [new2] "r" (x3), [ptr] "r" (x4), \
	379	+ [oldval1] "r" (oldval1), [oldval2] "r" (oldval2) \
	380	+ : "x30" , ##cl); \
	381	+ \
	382	+ return x0; \
	383	+}
	384	+
	385	+__CMPXCHG_DBL( , )
	386	+__CMPXCHG_DBL(_mb, al, "memory")
	387	+
	388	+#undef __LL_SC_CMPXCHG_DBL
	389	+#undef __CMPXCHG_DBL
	390	+
	391	+#endif /* __ASM_ATOMIC_LSE_H */

arch/arm64/include/asm/barrier.h+0−24

@@ -35,28 +35,6 @@ · 34 unmodified lines
35	35	#define dma_rmb() dmb(oshld)
36	36	#define dma_wmb() dmb(oshst)
37	37
38		−#ifndef CONFIG_SMP
39		−#define smp_mb() barrier()
40		−#define smp_rmb() barrier()
41		−#define smp_wmb() barrier()
42		−
43		−#define smp_store_release(p, v) \
44		−do { \
45		− compiletime_assert_atomic_type(*p); \
46		− barrier(); \
47		− WRITE_ONCE(*p, v); \
48		−} while (0)
49		−
50		−#define smp_load_acquire(p) \
51		−({ \
52		− typeof(p) ___p1 = READ_ONCE(p); \
53		− compiletime_assert_atomic_type(*p); \
54		− barrier(); \
55		− ___p1; \
56		−})
57		−
58		−#else
59		−
60	38	#define smp_mb() dmb(ish)
61	39	#define smp_rmb() dmb(ishld)
62	40	#define smp_wmb() dmb(ishst)
@@ -109,8 +87,6 @@ do { \ · 46 unmodified lines
109	87	___p1; \
110	88	})
111	89
112		−#endif
113		−
114	90	#define read_barrier_depends() do { } while(0)
115	91	#define smp_read_barrier_depends() do { } while(0)
116	92

arch/arm64/include/asm/bug.h+64−0

@@ -0,0 +1,64 @@
	1	+/*
	2	+ * Copyright (C) 2015 ARM Limited
	3	+ * Author: Dave Martin <Dave.Martin@arm.com>
	4	+ *
	5	+ * This program is free software; you can redistribute it and/or modify
	6	+ * it under the terms of the GNU General Public License version 2 as
	7	+ * published by the Free Software Foundation.
	8	+ *
	9	+ * This program is distributed in the hope that it will be useful,
	10	+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
	11	+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	12	+ * GNU General Public License for more details.
	13	+ *
	14	+ * You should have received a copy of the GNU General Public License
	15	+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
	16	+ */
	17	+
	18	+#ifndef _ARCH_ARM64_ASM_BUG_H
	19	+#define _ARCH_ARM64_ASM_BUG_H
	20	+
	21	+#include <asm/debug-monitors.h>
	22	+
	23	+#ifdef CONFIG_GENERIC_BUG
	24	+#define HAVE_ARCH_BUG
	25	+
	26	+#ifdef CONFIG_DEBUG_BUGVERBOSE
	27	+#define _BUGVERBOSE_LOCATION(file, line) __BUGVERBOSE_LOCATION(file, line)
	28	+#define __BUGVERBOSE_LOCATION(file, line) \
	29	+ ".pushsection .rodata.str,\"aMS\",@progbits,1\n" \
	30	+ "2: .string \"" file "\"\n\t" \
	31	+ ".popsection\n\t" \
	32	+ \
	33	+ ".long 2b - 0b\n\t" \
	34	+ ".short " #line "\n\t"
	35	+#else
	36	+#define _BUGVERBOSE_LOCATION(file, line)
	37	+#endif
	38	+
	39	+#define _BUG_FLAGS(flags) __BUG_FLAGS(flags)
	40	+
	41	+#define __BUG_FLAGS(flags) asm volatile ( \
	42	+ ".pushsection __bug_table,\"a\"\n\t" \
	43	+ ".align 2\n\t" \
	44	+ "0: .long 1f - 0b\n\t" \
	45	+_BUGVERBOSE_LOCATION(__FILE__, __LINE__) \
	46	+ ".short " #flags "\n\t" \
	47	+ ".popsection\n" \
	48	+ \
	49	+ "1: brk %[imm]" \
	50	+ :: [imm] "i" (BUG_BRK_IMM) \
	51	+)
	52	+
	53	+#define BUG() do { \
	54	+ _BUG_FLAGS(0); \
	55	+ unreachable(); \
	56	+} while (0)
	57	+
	58	+#define __WARN_TAINT(taint) _BUG_FLAGS(BUGFLAG_TAINT(taint))
	59	+
	60	+#endif /* ! CONFIG_GENERIC_BUG */
	61	+
	62	+#include <asm-generic/bug.h>
	63	+
	64	+#endif /* ! _ARCH_ARM64_ASM_BUG_H */

arch/arm64/include/asm/cmpxchg.h+74−118

@@ -21,7 +21,9 @@ · 20 unmodified lines
21	21	#include <linux/bug.h>
22	22	#include <linux/mmdebug.h>
23	23
	24	+#include <asm/atomic.h>
24	25	#include <asm/barrier.h>
	26	+#include <asm/lse.h>
25	27
26	28	static inline unsigned long __xchg(unsigned long x, volatile void *ptr, int size)
27	29	{
@@ -29,37 +31,73 @@ static inline unsigned long __xchg(unsigned long x, volatile void *ptr, int size · 1 unmodified line
29	31
30	32	switch (size) {
31	33	case 1:
32		− asm volatile("// __xchg1\n"
	34	+ asm volatile(ARM64_LSE_ATOMIC_INSN(
	35	+ /* LL/SC */
	36	+ " prfm pstl1strm, %2\n"
33	37	"1: ldxrb %w0, %2\n"
34	38	" stlxrb %w1, %w3, %2\n"
35	39	" cbnz %w1, 1b\n"
	40	+ " dmb ish",
	41	+ /* LSE atomics */
	42	+ " nop\n"
	43	+ " nop\n"
	44	+ " swpalb %w3, %w0, %2\n"
	45	+ " nop\n"
	46	+ " nop")
36	47	: "=&r" (ret), "=&r" (tmp), "+Q" ((u8 )ptr)
37	48	: "r" (x)
38	49	: "memory");
39	50	break;
40	51	case 2:
41		− asm volatile("// __xchg2\n"
	52	+ asm volatile(ARM64_LSE_ATOMIC_INSN(
	53	+ /* LL/SC */
	54	+ " prfm pstl1strm, %2\n"
42	55	"1: ldxrh %w0, %2\n"
43	56	" stlxrh %w1, %w3, %2\n"
44	57	" cbnz %w1, 1b\n"
	58	+ " dmb ish",
	59	+ /* LSE atomics */
	60	+ " nop\n"
	61	+ " nop\n"
	62	+ " swpalh %w3, %w0, %2\n"
	63	+ " nop\n"
	64	+ " nop")
45	65	: "=&r" (ret), "=&r" (tmp), "+Q" ((u16 )ptr)
46	66	: "r" (x)
47	67	: "memory");
48	68	break;
49	69	case 4:
50		− asm volatile("// __xchg4\n"
	70	+ asm volatile(ARM64_LSE_ATOMIC_INSN(
	71	+ /* LL/SC */
	72	+ " prfm pstl1strm, %2\n"
51	73	"1: ldxr %w0, %2\n"
52	74	" stlxr %w1, %w3, %2\n"
53	75	" cbnz %w1, 1b\n"
	76	+ " dmb ish",
	77	+ /* LSE atomics */
	78	+ " nop\n"
	79	+ " nop\n"
	80	+ " swpal %w3, %w0, %2\n"
	81	+ " nop\n"
	82	+ " nop")
54	83	: "=&r" (ret), "=&r" (tmp), "+Q" ((u32 )ptr)
55	84	: "r" (x)
56	85	: "memory");
57	86	break;
58	87	case 8:
59		− asm volatile("// __xchg8\n"
	88	+ asm volatile(ARM64_LSE_ATOMIC_INSN(
	89	+ /* LL/SC */
	90	+ " prfm pstl1strm, %2\n"
60	91	"1: ldxr %0, %2\n"
61	92	" stlxr %w1, %3, %2\n"
62	93	" cbnz %w1, 1b\n"
	94	+ " dmb ish",
	95	+ /* LSE atomics */
	96	+ " nop\n"
	97	+ " nop\n"
	98	+ " swpal %3, %0, %2\n"
	99	+ " nop\n"
	100	+ " nop")
63	101	: "=&r" (ret), "=&r" (tmp), "+Q" ((u64 )ptr)
64	102	: "r" (x)
65	103	: "memory");
@@ -68,7 +106,6 @@ static inline unsigned long __xchg(unsigned long x, volatile void *ptr, int size · 2 unmodified lines
68	106	BUILD_BUG();
69	107	}
70	108
71		− smp_mb();
72	109	return ret;
73	110	}
74	111
@@ -83,131 +120,39 @@ static inline unsigned long __xchg(unsigned long x, volatile void *ptr, int size · 8 unmodified lines
83	120	static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old,
84	121	unsigned long new, int size)
85	122	{
86		− unsigned long oldval = 0, res;
87		−
88	123	switch (size) {
89	124	case 1:
90		− do {
91		− asm volatile("// __cmpxchg1\n"
92		− " ldxrb %w1, %2\n"
93		− " mov %w0, #0\n"
94		− " cmp %w1, %w3\n"
95		− " b.ne 1f\n"
96		− " stxrb %w0, %w4, %2\n"
97		− "1:\n"
98		− : "=&r" (res), "=&r" (oldval), "+Q" ((u8 )ptr)
99		− : "Ir" (old), "r" (new)
100		− : "cc");
101		− } while (res);
102		− break;
103		−
	125	+ return __cmpxchg_case_1(ptr, (u8)old, new);
104	126	case 2:
105		− do {
106		− asm volatile("// __cmpxchg2\n"
107		− " ldxrh %w1, %2\n"
108		− " mov %w0, #0\n"
109		− " cmp %w1, %w3\n"
110		− " b.ne 1f\n"
111		− " stxrh %w0, %w4, %2\n"
112		− "1:\n"
113		− : "=&r" (res), "=&r" (oldval), "+Q" ((u16 )ptr)
114		− : "Ir" (old), "r" (new)
115		− : "cc");
116		− } while (res);
117		− break;
118		−
	127	+ return __cmpxchg_case_2(ptr, (u16)old, new);
119	128	case 4:
120		− do {
121		− asm volatile("// __cmpxchg4\n"
122		− " ldxr %w1, %2\n"
123		− " mov %w0, #0\n"
124		− " cmp %w1, %w3\n"
125		− " b.ne 1f\n"
126		− " stxr %w0, %w4, %2\n"
127		− "1:\n"
128		− : "=&r" (res), "=&r" (oldval), "+Q" ((u32 )ptr)
129		− : "Ir" (old), "r" (new)
130		− : "cc");
131		− } while (res);
132		− break;
133		−
	129	+ return __cmpxchg_case_4(ptr, old, new);
134	130	case 8:
135		− do {
136		− asm volatile("// __cmpxchg8\n"
137		− " ldxr %1, %2\n"
138		− " mov %w0, #0\n"
139		− " cmp %1, %3\n"
140		− " b.ne 1f\n"
141		− " stxr %w0, %4, %2\n"
142		− "1:\n"
143		− : "=&r" (res), "=&r" (oldval), "+Q" ((u64 )ptr)
144		− : "Ir" (old), "r" (new)
145		− : "cc");
146		− } while (res);
147		− break;
148		−
	131	+ return __cmpxchg_case_8(ptr, old, new);
149	132	default:
150	133	BUILD_BUG();
151	134	}
152	135
153		− return oldval;
	136	+ unreachable();
154	137	}
155	138
156		−#define system_has_cmpxchg_double() 1
157		−
158		−static inline int __cmpxchg_double(volatile void ptr1, volatile void ptr2,
159		− unsigned long old1, unsigned long old2,
160		− unsigned long new1, unsigned long new2, int size)
	139	+static inline unsigned long __cmpxchg_mb(volatile void *ptr, unsigned long old,
	140	+ unsigned long new, int size)
161	141	{
162		− unsigned long loop, lost;
163		−
164	142	switch (size) {
	143	+ case 1:
	144	+ return __cmpxchg_case_mb_1(ptr, (u8)old, new);
	145	+ case 2:
	146	+ return __cmpxchg_case_mb_2(ptr, (u16)old, new);
	147	+ case 4:
	148	+ return __cmpxchg_case_mb_4(ptr, old, new);
165	149	case 8:
166		− VM_BUG_ON((unsigned long )ptr2 - (unsigned long )ptr1 != 1);
167		− do {
168		− asm volatile("// __cmpxchg_double8\n"
169		− " ldxp %0, %1, %2\n"
170		− " eor %0, %0, %3\n"
171		− " eor %1, %1, %4\n"
172		− " orr %1, %0, %1\n"
173		− " mov %w0, #0\n"
174		− " cbnz %1, 1f\n"
175		− " stxp %w0, %5, %6, %2\n"
176		− "1:\n"
177		− : "=&r"(loop), "=&r"(lost), "+Q" ((u64 )ptr1)
178		− : "r" (old1), "r"(old2), "r"(new1), "r"(new2));
179		− } while (loop);
180		− break;
	150	+ return __cmpxchg_case_mb_8(ptr, old, new);
181	151	default:
182	152	BUILD_BUG();
183	153	}
184	154
185		− return !lost;
186		−}
187		−
188		−static inline int __cmpxchg_double_mb(volatile void ptr1, volatile void ptr2,
189		− unsigned long old1, unsigned long old2,
190		− unsigned long new1, unsigned long new2, int size)
191		−{
192		− int ret;
193		−
194		− smp_mb();
195		− ret = __cmpxchg_double(ptr1, ptr2, old1, old2, new1, new2, size);
196		− smp_mb();
197		−
198		− return ret;
199		−}
200		−
201		−static inline unsigned long __cmpxchg_mb(volatile void *ptr, unsigned long old,
202		− unsigned long new, int size)
203		−{
204		− unsigned long ret;
205		−
206		− smp_mb();
207		− ret = __cmpxchg(ptr, old, new, size);
208		− smp_mb();
209		−
210		− return ret;
	155	+ unreachable();
211	156	}
212	157
213	158	#define cmpxchg(ptr, o, n) \
@@ -228,21 +173,32 @@ static inline unsigned long __cmpxchg_mb(volatile void *ptr, unsigned long old, · 14 unmodified lines
228	173	__ret; \
229	174	})
230	175
	176	+#define system_has_cmpxchg_double() 1
	177	+
	178	+#define __cmpxchg_double_check(ptr1, ptr2) \
	179	+({ \
	180	+ if (sizeof(*(ptr1)) != 8) \
	181	+ BUILD_BUG(); \
	182	+ VM_BUG_ON((unsigned long )(ptr2) - (unsigned long )(ptr1) != 1); \
	183	+})
	184	+
231	185	#define cmpxchg_double(ptr1, ptr2, o1, o2, n1, n2) \
232	186	({\
233	187	int __ret;\
234		− __ret = __cmpxchg_double_mb((ptr1), (ptr2), (unsigned long)(o1), \
235		− (unsigned long)(o2), (unsigned long)(n1), \
236		− (unsigned long)(n2), sizeof(*(ptr1)));\
	188	+ __cmpxchg_double_check(ptr1, ptr2); \
	189	+ __ret = !__cmpxchg_double_mb((unsigned long)(o1), (unsigned long)(o2), \
	190	+ (unsigned long)(n1), (unsigned long)(n2), \
	191	+ ptr1); \
237	192	__ret; \
238	193	})
239	194
240	195	#define cmpxchg_double_local(ptr1, ptr2, o1, o2, n1, n2) \
241	196	({\
242	197	int __ret;\
243		− __ret = __cmpxchg_double((ptr1), (ptr2), (unsigned long)(o1), \
244		− (unsigned long)(o2), (unsigned long)(n1), \
245		− (unsigned long)(n2), sizeof(*(ptr1)));\
	198	+ __cmpxchg_double_check(ptr1, ptr2); \
	199	+ __ret = !__cmpxchg_double((unsigned long)(o1), (unsigned long)(o2), \
	200	+ (unsigned long)(n1), (unsigned long)(n2), \
	201	+ ptr1); \
246	202	__ret; \
247	203	})
248	204

arch/arm64/include/asm/cpufeature.h+15−3

@@ -25,15 +25,20 @@ · 24 unmodified lines
25	25	#define ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE 1
26	26	#define ARM64_WORKAROUND_845719 2
27	27	#define ARM64_HAS_SYSREG_GIC_CPUIF 3
	28	+#define ARM64_HAS_PAN 4
	29	+#define ARM64_HAS_LSE_ATOMICS 5
28	30
29		−#define ARM64_NCAPS 4
	31	+#define ARM64_NCAPS 6
30	32
31	33	#ifndef __ASSEMBLY__
32	34
	35	+#include <linux/kernel.h>
	36	+
33	37	struct arm64_cpu_capabilities {
34	38	const char *desc;
35	39	u16 capability;
36	40	bool (matches)(const struct arm64_cpu_capabilities );
	41	+ void (*enable)(void);
37	42	union {
38	43	struct { /* To be used for erratum handling only */
39	44	u32 midr_model;
@@ -41,8 +46,8 @@ struct arm64_cpu_capabilities { · 1 unmodified line
41	46	};
42	47
43	48	struct { /* Feature register checking */
44		− u64 register_mask;
45		− u64 register_value;
	49	+ int field_pos;
	50	+ int min_field_value;
46	51	};
47	52	};
48	53	};
@@ -70,6 +75,13 @@ static inline void cpus_set_cap(unsigned int num) · 21 unmodified lines
70	75	__set_bit(num, cpu_hwcaps);
71	76	}
72	77
	78	+static inline int __attribute_const__ cpuid_feature_extract_field(u64 features,
	79	+ int field)
	80	+{
	81	+ return (s64)(features << (64 - 4 - field)) >> (64 - 4);
	82	+}
	83	+
	84	+
73	85	void check_cpu_capabilities(const struct arm64_cpu_capabilities *caps,
74	86	const char *info);
75	87	void check_local_cpu_errata(void);

arch/arm64/include/asm/cputype.h+0−3

@@ -81,9 +81,6 @@ · 80 unmodified lines
81	81	#define ID_AA64MMFR0_BIGEND(mmfr0) \
82	82	(((mmfr0) & ID_AA64MMFR0_BIGEND_MASK) >> ID_AA64MMFR0_BIGEND_SHIFT)
83	83
84		−#define SCTLR_EL1_CP15BEN (0x1 << 5)
85		−#define SCTLR_EL1_SED (0x1 << 8)
86		−
87	84	#ifndef __ASSEMBLY__
88	85
89	86	/*

arch/arm64/include/asm/debug-monitors.h+16−28

@@ -18,6 +18,12 @@ · 17 unmodified lines
18	18
19	19	#ifdef __KERNEL__
20	20
	21	+#include <linux/errno.h>
	22	+#include <linux/types.h>
	23	+#include <asm/esr.h>
	24	+#include <asm/insn.h>
	25	+#include <asm/ptrace.h>
	26	+
21	27	/* Low-level stepping controls. */
22	28	#define DBG_MDSCR_SS (1 << 0)
23	29	#define DBG_SPSR_SS (1 << 21)
@@ -38,12 +44,7 @@ · 14 unmodified lines
38	44	/*
39	45	* Break point instruction encoding
40	46	*/
41		−#define BREAK_INSTR_SIZE 4
42		−
43		−/*
44		− * ESR values expected for dynamic and compile time BRK instruction
45		− */
46		−#define DBG_ESR_VAL_BRK(x) (0xf2000000 \| ((x) & 0xfffff))
	47	+#define BREAK_INSTR_SIZE AARCH64_INSN_SIZE
47	48
48	49	/*
49	50	* #imm16 values used for BRK instruction generation
@@ -51,10 +52,12 @@ · 1 unmodified line
51	52	* 0x100: for triggering a fault on purpose (reserved)
52	53	* 0x400: for dynamic BRK instruction
53	54	* 0x401: for compile time BRK instruction
	55	+ * 0x800: kernel-mode BUG() and WARN() traps
54	56	*/
55	57	#define FAULT_BRK_IMM 0x100
56	58	#define KGDB_DYN_DBG_BRK_IMM 0x400
57	59	#define KGDB_COMPILED_DBG_BRK_IMM 0x401
	60	+#define BUG_BRK_IMM 0x800
58	61
59	62	/*
60	63	* BRK instruction encoding
@@ -68,25 +71,10 @@ · 7 unmodified lines
68	71	*/
69	72	#define AARCH64_BREAK_FAULT (AARCH64_BREAK_MON \| (FAULT_BRK_IMM << 5))
70	73
71		−/*
72		− * Extract byte from BRK instruction
73		− */
74		−#define KGDB_DYN_DBG_BRK_INS_BYTE(x) \
75		− ((((AARCH64_BREAK_MON) & 0xffe0001f) >> (x * 8)) & 0xff)
76		−
77		−/*
78		− * Extract byte from BRK #imm16
79		− */
80		−#define KGBD_DYN_DBG_BRK_IMM_BYTE(x) \
81		− (((((KGDB_DYN_DBG_BRK_IMM) & 0xffff) << 5) >> (x * 8)) & 0xff)
82		−
83		−#define KGDB_DYN_DBG_BRK_BYTE(x) \
84		− (KGDB_DYN_DBG_BRK_INS_BYTE(x) \| KGBD_DYN_DBG_BRK_IMM_BYTE(x))
85		−
86		−#define KGDB_DYN_BRK_INS_BYTE0 KGDB_DYN_DBG_BRK_BYTE(0)
87		−#define KGDB_DYN_BRK_INS_BYTE1 KGDB_DYN_DBG_BRK_BYTE(1)
88		−#define KGDB_DYN_BRK_INS_BYTE2 KGDB_DYN_DBG_BRK_BYTE(2)
89		−#define KGDB_DYN_BRK_INS_BYTE3 KGDB_DYN_DBG_BRK_BYTE(3)
	74	+#define AARCH64_BREAK_KGDB_DYN_DBG \
	75	+ (AARCH64_BREAK_MON \| (KGDB_DYN_DBG_BRK_IMM << 5))
	76	+#define KGDB_DYN_BRK_INS_BYTE(x) \
	77	+ ((AARCH64_BREAK_KGDB_DYN_DBG >> (8 * (x))) & 0xff)
90	78
91	79	#define CACHE_FLUSH_IS_SAFE 1
92	80
@@ -127,13 +115,13 @@ void unregister_break_hook(struct break_hook *hook); · 34 unmodified lines
127	115
128	116	u8 debug_monitors_arch(void);
129	117
130		−enum debug_el {
	118	+enum dbg_active_el {
131	119	DBG_ACTIVE_EL0 = 0,
132	120	DBG_ACTIVE_EL1,
133	121	};
134	122
135		−void enable_debug_monitors(enum debug_el el);
136		−void disable_debug_monitors(enum debug_el el);
	123	+void enable_debug_monitors(enum dbg_active_el el);
	124	+void disable_debug_monitors(enum dbg_active_el el);
137	125
138	126	void user_rewind_single_step(struct task_struct *task);
139	127	void user_fastforward_single_step(struct task_struct *task);

arch/arm64/include/asm/esr.h+9−0

@@ -18,6 +18,8 @@ · 17 unmodified lines
18	18	#ifndef __ASM_ESR_H
19	19	#define __ASM_ESR_H
20	20
	21	+#include <asm/memory.h>
	22	+
21	23	#define ESR_ELx_EC_UNKNOWN (0x00)
22	24	#define ESR_ELx_EC_WFx (0x01)
23	25	/* Unallocated EC: 0x02 */
@@ -99,6 +101,13 @@ · 75 unmodified lines
99	101	#define ESR_ELx_WFx_ISS_WFE (UL(1) << 0)
100	102	#define ESR_ELx_xVC_IMM_MASK ((1UL << 16) - 1)
101	103
	104	+/* ESR value templates for specific events */
	105	+
	106	+/* BRK instruction trap from AArch64 state */
	107	+#define ESR_ELx_VAL_BRK64(imm) \
	108	+ ((ESR_ELx_EC_BRK64 << ESR_ELx_EC_SHIFT) \| ESR_ELx_IL \| \
	109	+ ((imm) & 0xffff))
	110	+
102	111	#ifndef __ASSEMBLY__
103	112	#include <asm/types.h>
104	113

arch/arm64/include/asm/exception.h+6−0

@@ -18,7 +18,13 @@ · 17 unmodified lines
18	18	#ifndef __ASM_EXCEPTION_H
19	19	#define __ASM_EXCEPTION_H
20	20
	21	+#include <linux/ftrace.h>
	22	+
21	23	#define __exception __attribute__((section(".exception.text")))
	24	+#ifdef CONFIG_FUNCTION_GRAPH_TRACER
	25	+#define __exception_irq_entry __irq_entry
	26	+#else
22	27	#define __exception_irq_entry __exception
	28	+#endif
23	29
24	30	#endif /* __ASM_EXCEPTION_H */

arch/arm64/include/asm/fixmap.h+1−1

@@ -8,7 +8,7 @@ · 7 unmodified lines
8	8	* Copyright (C) 1998 Ingo Molnar
9	9	* Copyright (C) 2013 Mark Salter <msalter@redhat.com>
10	10	*
11		− * Adapted from arch/x86_64 version.
	11	+ * Adapted from arch/x86 version.
12	12	*
13	13	*/
14	14

arch/arm64/include/asm/futex.h+10−0

@@ -20,10 +20,17 @@ · 19 unmodified lines
20	20
21	21	#include <linux/futex.h>
22	22	#include <linux/uaccess.h>
	23	+
	24	+#include <asm/alternative.h>
	25	+#include <asm/cpufeature.h>
23	26	#include <asm/errno.h>
	27	+#include <asm/sysreg.h>
24	28
25	29	#define __futex_atomic_op(insn, ret, oldval, uaddr, tmp, oparg) \
26	30	asm volatile( \
	31	+ ALTERNATIVE("nop", SET_PSTATE_PAN(0), ARM64_HAS_PAN, \
	32	+ CONFIG_ARM64_PAN) \
	33	+" prfm pstl1strm, %2\n" \
27	34	"1: ldxr %w1, %2\n" \
28	35	insn "\n" \
29	36	"2: stlxr %w3, %w0, %2\n" \
@@ -39,6 +46,8 @@ · 9 unmodified lines
39	46	" .align 3\n" \
40	47	" .quad 1b, 4b, 2b, 4b\n" \
41	48	" .popsection\n" \
	49	+ ALTERNATIVE("nop", SET_PSTATE_PAN(1), ARM64_HAS_PAN, \
	50	+ CONFIG_ARM64_PAN) \
42	51	: "=&r" (ret), "=&r" (oldval), "+Q" (*uaddr), "=&r" (tmp) \
43	52	: "r" (oparg), "Ir" (-EFAULT) \
44	53	: "memory")
@@ -112,6 +121,7 @@ futex_atomic_cmpxchg_inatomic(u32 uval, u32 __user uaddr, · 67 unmodified lines
112	121	return -EFAULT;
113	122
114	123	asm volatile("// futex_atomic_cmpxchg_inatomic\n"
	124	+" prfm pstl1strm, %2\n"
115	125	"1: ldxr %w1, %2\n"
116	126	" sub %w3, %w1, %w4\n"
117	127	" cbnz %w3, 3f\n"

arch/arm64/include/asm/hardirq.h+0−4

@@ -24,9 +24,7 @@ · 23 unmodified lines
24	24
25	25	typedef struct {
26	26	unsigned int __softirq_pending;
27		−#ifdef CONFIG_SMP
28	27	unsigned int ipi_irqs[NR_IPI];
29		−#endif
30	28	} ____cacheline_aligned irq_cpustat_t;
31	29
32	30	#include <linux/irq_cpustat.h> /* Standard mappings for irq_cpustat_t above */
@@ -34,10 +32,8 @@ typedef struct { · 1 unmodified line
34	32	#define __inc_irq_stat(cpu, member) __IRQ_STAT(cpu, member)++
35	33	#define __get_irq_stat(cpu, member) __IRQ_STAT(cpu, member)
36	34
37		−#ifdef CONFIG_SMP
38	35	u64 smp_irq_stat_cpu(unsigned int cpu);
39	36	#define arch_irq_stat_cpu smp_irq_stat_cpu
40		−#endif
41	37
42	38	#define __ARCH_IRQ_EXIT_IRQS_DISABLED 1
43	39

arch/arm64/include/asm/hugetlb.h+0−4

@@ -13,10 +13,6 @@ · 12 unmodified lines
13	13	* but WITHOUT ANY WARRANTY; without even the implied warranty of
14	14	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15	15	* GNU General Public License for more details.
16		− *
17		− * You should have received a copy of the GNU General Public License
18		− * along with this program; if not, write to the Free Software
19		− * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
20	16	*/
21	17
22	18	#ifndef __ASM_HUGETLB_H

arch/arm64/include/asm/irq_work.h+0−11

@@ -1,8 +1,6 @@
1	1	#ifndef __ASM_IRQ_WORK_H
2	2	#define __ASM_IRQ_WORK_H
3	3
4		−#ifdef CONFIG_SMP
5		−
6	4	#include <asm/smp.h>
7	5
8	6	static inline bool arch_irq_work_has_interrupt(void)
@@ -10,13 +8,4 @@ static inline bool arch_irq_work_has_interrupt(void) · 1 unmodified line
10	8	return !!__smp_cross_call;
11	9	}
12	10
13		−#else
14		−
15		−static inline bool arch_irq_work_has_interrupt(void)
16		−{
17		− return false;
18		−}
19		−
20		−#endif
21		−
22	11	#endif /* __ASM_IRQ_WORK_H */

arch/arm64/include/asm/lse.h+53−0

@@ -0,0 +1,53 @@
	1	+#ifndef __ASM_LSE_H
	2	+#define __ASM_LSE_H
	3	+
	4	+#if defined(CONFIG_AS_LSE) && defined(CONFIG_ARM64_LSE_ATOMICS)
	5	+
	6	+#include <linux/stringify.h>
	7	+#include <asm/alternative.h>
	8	+#include <asm/cpufeature.h>
	9	+
	10	+#ifdef __ASSEMBLER__
	11	+
	12	+.arch_extension lse
	13	+
	14	+.macro alt_lse, llsc, lse
	15	+ alternative_insn "\llsc", "\lse", ARM64_HAS_LSE_ATOMICS
	16	+.endm
	17	+
	18	+#else /* __ASSEMBLER__ */
	19	+
	20	+__asm__(".arch_extension lse");
	21	+
	22	+/* Move the ll/sc atomics out-of-line */
	23	+#define __LL_SC_INLINE
	24	+#define __LL_SC_PREFIX(x) __ll_sc_##x
	25	+#define __LL_SC_EXPORT(x) EXPORT_SYMBOL(__LL_SC_PREFIX(x))
	26	+
	27	+/* Macro for constructing calls to out-of-line ll/sc atomics */
	28	+#define __LL_SC_CALL(op) "bl\t" __stringify(__LL_SC_PREFIX(op)) "\n"
	29	+
	30	+/* In-line patching at runtime */
	31	+#define ARM64_LSE_ATOMIC_INSN(llsc, lse) \
	32	+ ALTERNATIVE(llsc, lse, ARM64_HAS_LSE_ATOMICS)
	33	+
	34	+#endif /* __ASSEMBLER__ */
	35	+#else /* CONFIG_AS_LSE && CONFIG_ARM64_LSE_ATOMICS */
	36	+
	37	+#ifdef __ASSEMBLER__
	38	+
	39	+.macro alt_lse, llsc, lse
	40	+ \llsc
	41	+.endm
	42	+
	43	+#else /* __ASSEMBLER__ */
	44	+
	45	+#define __LL_SC_INLINE static inline
	46	+#define __LL_SC_PREFIX(x) x
	47	+#define __LL_SC_EXPORT(x)
	48	+
	49	+#define ARM64_LSE_ATOMIC_INSN(llsc, lse) llsc
	50	+
	51	+#endif /* __ASSEMBLER__ */
	52	+#endif /* CONFIG_AS_LSE && CONFIG_ARM64_LSE_ATOMICS */
	53	+#endif /* __ASM_LSE_H */

arch/arm64/include/asm/memory.h+8−0

@@ -113,6 +113,14 @@ extern phys_addr_t memstart_addr; · 112 unmodified lines
113	113	/* PHYS_OFFSET - the physical address of the start of memory. */
114	114	#define PHYS_OFFSET ({ memstart_addr; })
115	115
	116	+/*
	117	+ * The maximum physical address that the linear direct mapping
	118	+ * of system RAM can cover. (PAGE_OFFSET can be interpreted as
	119	+ * a 2's complement signed quantity and negated to derive the
	120	+ * maximum size of the linear mapping.)
	121	+ */
	122	+#define MAX_MEMBLOCK_ADDR ({ memstart_addr - PAGE_OFFSET - 1; })
	123	+
116	124	/*
117	125	* PFNs are used to describe any physical page; this means
118	126	* PFN 0 == physical address 0.

arch/arm64/include/asm/mmu.h+0−1

@@ -28,7 +28,6 @@ typedef struct { · 27 unmodified lines
28	28	#define ASID(mm) ((mm)->context.id & 0xffff)
29	29
30	30	extern void paging_init(void);
31		−extern void setup_mm_for_reboot(void);
32	31	extern void __iomem *early_io_map(phys_addr_t phys, unsigned long virt);
33	32	extern void init_mem_pgprot(void);
34	33	extern void create_pgd_mapping(struct mm_struct *mm, phys_addr_t phys,

arch/arm64/include/asm/percpu.h+0−8

@@ -16,8 +16,6 @@ · 15 unmodified lines
16	16	#ifndef __ASM_PERCPU_H
17	17	#define __ASM_PERCPU_H
18	18
19		−#ifdef CONFIG_SMP
20		−
21	19	static inline void set_my_cpu_offset(unsigned long off)
22	20	{
23	21	asm volatile("msr tpidr_el1, %0" :: "r" (off) : "memory");
@@ -38,12 +36,6 @@ static inline unsigned long __my_cpu_offset(void) · 14 unmodified lines
38	36	}
39	37	#define __my_cpu_offset __my_cpu_offset()
40	38
41		−#else /* !CONFIG_SMP */
42		−
43		−#define set_my_cpu_offset(x) do { } while (0)
44		−
45		−#endif /* CONFIG_SMP */
46		−
47	39	#define PERCPU_OP(op, asm_op) \
48	40	static inline unsigned long __percpu_##op(void *ptr, \
49	41	unsigned long val, int size) \

arch/arm64/include/asm/perf_event.h+1−1

@@ -17,7 +17,7 @@ · 16 unmodified lines
17	17	#ifndef __ASM_PERF_EVENT_H
18	18	#define __ASM_PERF_EVENT_H
19	19
20		−#ifdef CONFIG_HW_PERF_EVENTS
	20	+#ifdef CONFIG_PERF_EVENTS
21	21	struct pt_regs;
22	22	extern unsigned long perf_instruction_pointer(struct pt_regs *regs);
23	23	extern unsigned long perf_misc_flags(struct pt_regs *regs);

arch/arm64/include/asm/pgtable-hwdef.h+3−0

@@ -104,6 +104,7 @@ · 103 unmodified lines
104	104	#define PTE_SHARED (_AT(pteval_t, 3) << 8) /* SH[1:0], inner shareable */
105	105	#define PTE_AF (_AT(pteval_t, 1) << 10) /* Access Flag */
106	106	#define PTE_NG (_AT(pteval_t, 1) << 11) /* nG */
	107	+#define PTE_DBM (_AT(pteval_t, 1) << 51) /* Dirty Bit Management */
107	108	#define PTE_PXN (_AT(pteval_t, 1) << 53) /* Privileged XN */
108	109	#define PTE_UXN (_AT(pteval_t, 1) << 54) /* User XN */
109	110
@@ -168,5 +169,7 @@ · 58 unmodified lines
168	169	#define TCR_TG1_64K (UL(3) << 30)
169	170	#define TCR_ASID16 (UL(1) << 36)
170	171	#define TCR_TBI0 (UL(1) << 37)
	172	+#define TCR_HA (UL(1) << 39)
	173	+#define TCR_HD (UL(1) << 40)
171	174
172	175	#endif

arch/arm64/include/asm/pgtable.h+160−7

@@ -16,6 +16,7 @@ · 15 unmodified lines
16	16	#ifndef __ASM_PGTABLE_H
17	17	#define __ASM_PGTABLE_H
18	18
	19	+#include <asm/bug.h>
19	20	#include <asm/proc-fns.h>
20	21
21	22	#include <asm/memory.h>
@@ -27,7 +28,11 @@ · 5 unmodified lines
27	28	#define PTE_VALID (_AT(pteval_t, 1) << 0)
28	29	#define PTE_DIRTY (_AT(pteval_t, 1) << 55)
29	30	#define PTE_SPECIAL (_AT(pteval_t, 1) << 56)
	31	+#ifdef CONFIG_ARM64_HW_AFDBM
	32	+#define PTE_WRITE (PTE_DBM) /* same as DBM */
	33	+#else
30	34	#define PTE_WRITE (_AT(pteval_t, 1) << 57)
	35	+#endif
31	36	#define PTE_PROT_NONE (_AT(pteval_t, 1) << 58) /* only when !PTE_VALID */
32	37
33	38	/*
@@ -48,18 +53,16 @@ · 14 unmodified lines
48	53	#define FIRST_USER_ADDRESS 0UL
49	54
50	55	#ifndef __ASSEMBLY__
	56	+
	57	+#include <linux/mmdebug.h>
	58	+
51	59	extern void __pte_error(const char *file, int line, unsigned long val);
52	60	extern void __pmd_error(const char *file, int line, unsigned long val);
53	61	extern void __pud_error(const char *file, int line, unsigned long val);
54	62	extern void __pgd_error(const char *file, int line, unsigned long val);
55	63
56		−#ifdef CONFIG_SMP
57	64	#define PROT_DEFAULT (PTE_TYPE_PAGE \| PTE_AF \| PTE_SHARED)
58	65	#define PROT_SECT_DEFAULT (PMD_TYPE_SECT \| PMD_SECT_AF \| PMD_SECT_S)
59		−#else
60		−#define PROT_DEFAULT (PTE_TYPE_PAGE \| PTE_AF)
61		−#define PROT_SECT_DEFAULT (PMD_TYPE_SECT \| PMD_SECT_AF)
62		−#endif
63	66
64	67	#define PROT_DEVICE_nGnRE (PROT_DEFAULT \| PTE_PXN \| PTE_UXN \| PTE_ATTRINDX(MT_DEVICE_nGnRE))
65	68	#define PROT_NORMAL_NC (PROT_DEFAULT \| PTE_PXN \| PTE_UXN \| PTE_ATTRINDX(MT_NORMAL_NC))
@@ -137,12 +140,20 @@ extern struct page *empty_zero_page; · 71 unmodified lines
137	140	* The following only work if pte_present(). Undefined behaviour otherwise.
138	141	*/
139	142	#define pte_present(pte) (!!(pte_val(pte) & (PTE_VALID \| PTE_PROT_NONE)))
140		−#define pte_dirty(pte) (!!(pte_val(pte) & PTE_DIRTY))
141	143	#define pte_young(pte) (!!(pte_val(pte) & PTE_AF))
142	144	#define pte_special(pte) (!!(pte_val(pte) & PTE_SPECIAL))
143	145	#define pte_write(pte) (!!(pte_val(pte) & PTE_WRITE))
144	146	#define pte_exec(pte) (!(pte_val(pte) & PTE_UXN))
145	147
	148	+#ifdef CONFIG_ARM64_HW_AFDBM
	149	+#define pte_hw_dirty(pte) (!(pte_val(pte) & PTE_RDONLY))
	150	+#else
	151	+#define pte_hw_dirty(pte) (0)
	152	+#endif
	153	+#define pte_sw_dirty(pte) (!!(pte_val(pte) & PTE_DIRTY))
	154	+#define pte_dirty(pte) (pte_sw_dirty(pte) \|\| pte_hw_dirty(pte))
	155	+
	156	+#define pte_valid(pte) (!!(pte_val(pte) & PTE_VALID))
146	157	#define pte_valid_user(pte) \
147	158	((pte_val(pte) & (PTE_VALID \| PTE_USER)) == (PTE_VALID \| PTE_USER))
148	159	#define pte_valid_not_user(pte) \
@@ -209,20 +220,49 @@ static inline void set_pte(pte_t *ptep, pte_t pte) · 60 unmodified lines
209	220	}
210	221	}
211	222
	223	+struct mm_struct;
	224	+struct vm_area_struct;
	225	+
212	226	extern void __sync_icache_dcache(pte_t pteval, unsigned long addr);
213	227
	228	+/*
	229	+ * PTE bits configuration in the presence of hardware Dirty Bit Management
	230	+ * (PTE_WRITE == PTE_DBM):
	231	+ *
	232	+ * Dirty Writable \| PTE_RDONLY PTE_WRITE PTE_DIRTY (sw)
	233	+ * 0 0 \| 1 0 0
	234	+ * 0 1 \| 1 1 0
	235	+ * 1 0 \| 1 0 1
	236	+ * 1 1 \| 0 1 x
	237	+ *
	238	+ * When hardware DBM is not present, the sofware PTE_DIRTY bit is updated via
	239	+ * the page fault mechanism. Checking the dirty status of a pte becomes:
	240	+ *
	241	+ * PTE_DIRTY \|\| !PTE_RDONLY
	242	+ */
214	243	static inline void set_pte_at(struct mm_struct *mm, unsigned long addr,
215	244	pte_t *ptep, pte_t pte)
216	245	{
217	246	if (pte_valid_user(pte)) {
218	247	if (!pte_special(pte) && pte_exec(pte))
219	248	__sync_icache_dcache(pte, addr);
220		− if (pte_dirty(pte) && pte_write(pte))
	249	+ if (pte_sw_dirty(pte) && pte_write(pte))
221	250	pte_val(pte) &= ~PTE_RDONLY;
222	251	else
223	252	pte_val(pte) \|= PTE_RDONLY;
224	253	}
225	254
	255	+ /*
	256	+ * If the existing pte is valid, check for potential race with
	257	+ * hardware updates of the pte (ptep_set_access_flags safely changes
	258	+ * valid ptes without going through an invalid entry).
	259	+ */
	260	+ if (IS_ENABLED(CONFIG_DEBUG_VM) && IS_ENABLED(CONFIG_ARM64_HW_AFDBM) &&
	261	+ pte_valid(*ptep)) {
	262	+ BUG_ON(!pte_young(pte));
	263	+ BUG_ON(pte_write(*ptep) && !pte_dirty(pte));
	264	+ }
	265	+
226	266	set_pte(ptep, pte);
227	267	}
228	268
@@ -461,6 +501,9 @@ static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) · 232 unmodified lines
461	501	{
462	502	const pteval_t mask = PTE_USER \| PTE_PXN \| PTE_UXN \| PTE_RDONLY \|
463	503	PTE_PROT_NONE \| PTE_WRITE \| PTE_TYPE_MASK;
	504	+ /* preserve the hardware dirty information */
	505	+ if (pte_hw_dirty(pte))
	506	+ newprot \|= PTE_DIRTY;
464	507	pte_val(pte) = (pte_val(pte) & ~mask) \| (pgprot_val(newprot) & mask);
465	508	return pte;
466	509	}
@@ -470,6 +513,101 @@ static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot) · 3 unmodified lines
470	513	return pte_pmd(pte_modify(pmd_pte(pmd), newprot));
471	514	}
472	515
	516	+#ifdef CONFIG_ARM64_HW_AFDBM
	517	+/*
	518	+ * Atomic pte/pmd modifications.
	519	+ */
	520	+#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
	521	+static inline int ptep_test_and_clear_young(struct vm_area_struct *vma,
	522	+ unsigned long address,
	523	+ pte_t *ptep)
	524	+{
	525	+ pteval_t pteval;
	526	+ unsigned int tmp, res;
	527	+
	528	+ asm volatile("// ptep_test_and_clear_young\n"
	529	+ " prfm pstl1strm, %2\n"
	530	+ "1: ldxr %0, %2\n"
	531	+ " ubfx %w3, %w0, %5, #1 // extract PTE_AF (young)\n"
	532	+ " and %0, %0, %4 // clear PTE_AF\n"
	533	+ " stxr %w1, %0, %2\n"
	534	+ " cbnz %w1, 1b\n"
	535	+ : "=&r" (pteval), "=&r" (tmp), "+Q" (pte_val(*ptep)), "=&r" (res)
	536	+ : "L" (~PTE_AF), "I" (ilog2(PTE_AF)));
	537	+
	538	+ return res;
	539	+}
	540	+
	541	+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
	542	+#define __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG
	543	+static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma,
	544	+ unsigned long address,
	545	+ pmd_t *pmdp)
	546	+{
	547	+ return ptep_test_and_clear_young(vma, address, (pte_t *)pmdp);
	548	+}
	549	+#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
	550	+
	551	+#define __HAVE_ARCH_PTEP_GET_AND_CLEAR
	552	+static inline pte_t ptep_get_and_clear(struct mm_struct *mm,
	553	+ unsigned long address, pte_t *ptep)
	554	+{
	555	+ pteval_t old_pteval;
	556	+ unsigned int tmp;
	557	+
	558	+ asm volatile("// ptep_get_and_clear\n"
	559	+ " prfm pstl1strm, %2\n"
	560	+ "1: ldxr %0, %2\n"
	561	+ " stxr %w1, xzr, %2\n"
	562	+ " cbnz %w1, 1b\n"
	563	+ : "=&r" (old_pteval), "=&r" (tmp), "+Q" (pte_val(*ptep)));
	564	+
	565	+ return __pte(old_pteval);
	566	+}
	567	+
	568	+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
	569	+#define __HAVE_ARCH_PMDP_GET_AND_CLEAR
	570	+static inline pmd_t pmdp_get_and_clear(struct mm_struct *mm,
	571	+ unsigned long address, pmd_t *pmdp)
	572	+{
	573	+ return pte_pmd(ptep_get_and_clear(mm, address, (pte_t *)pmdp));
	574	+}
	575	+#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
	576	+
	577	+/*
	578	+ * ptep_set_wrprotect - mark read-only while trasferring potential hardware
	579	+ * dirty status (PTE_DBM && !PTE_RDONLY) to the software PTE_DIRTY bit.
	580	+ */
	581	+#define __HAVE_ARCH_PTEP_SET_WRPROTECT
	582	+static inline void ptep_set_wrprotect(struct mm_struct mm, unsigned long address, pte_t ptep)
	583	+{
	584	+ pteval_t pteval;
	585	+ unsigned long tmp;
	586	+
	587	+ asm volatile("// ptep_set_wrprotect\n"
	588	+ " prfm pstl1strm, %2\n"
	589	+ "1: ldxr %0, %2\n"
	590	+ " tst %0, %4 // check for hw dirty (!PTE_RDONLY)\n"
	591	+ " csel %1, %3, xzr, eq // set PTE_DIRTY\|PTE_RDONLY if dirty\n"
	592	+ " orr %0, %0, %1 // if !dirty, PTE_RDONLY is already set\n"
	593	+ " and %0, %0, %5 // clear PTE_WRITE/PTE_DBM\n"
	594	+ " stxr %w1, %0, %2\n"
	595	+ " cbnz %w1, 1b\n"
	596	+ : "=&r" (pteval), "=&r" (tmp), "+Q" (pte_val(*ptep))
	597	+ : "r" (PTE_DIRTY\|PTE_RDONLY), "L" (PTE_RDONLY), "L" (~PTE_WRITE)
	598	+ : "cc");
	599	+}
	600	+
	601	+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
	602	+#define __HAVE_ARCH_PMDP_SET_WRPROTECT
	603	+static inline void pmdp_set_wrprotect(struct mm_struct *mm,
	604	+ unsigned long address, pmd_t *pmdp)
	605	+{
	606	+ ptep_set_wrprotect(mm, address, (pte_t *)pmdp);
	607	+}
	608	+#endif
	609	+#endif /* CONFIG_ARM64_HW_AFDBM */
	610	+
473	611	extern pgd_t swapper_pg_dir[PTRS_PER_PGD];
474	612	extern pgd_t idmap_pg_dir[PTRS_PER_PGD];
475	613
@@ -505,6 +643,21 @@ extern int kern_addr_valid(unsigned long addr); · 29 unmodified lines
505	643
506	644	#define pgtable_cache_init() do { } while (0)
507	645
	646	+/*
	647	+ * On AArch64, the cache coherency is handled via the set_pte_at() function.
	648	+ */
	649	+static inline void update_mmu_cache(struct vm_area_struct *vma,
	650	+ unsigned long addr, pte_t *ptep)
	651	+{
	652	+ /*
	653	+ * set_pte() does not have a DSB for user mappings, so make sure that
	654	+ * the page table write is visible.
	655	+ */
	656	+ dsb(ishst);
	657	+}
	658	+
	659	+#define update_mmu_cache_pmd(vma, address, pmd) do { } while (0)
	660	+
508	661	#endif /* !__ASSEMBLY__ */
509	662
510	663	#endif /* __ASM_PGTABLE_H */

arch/arm64/include/asm/processor.h+2−0

@@ -186,4 +186,6 @@ static inline void spin_lock_prefetch(const void *x) · 185 unmodified lines
186	186
187	187	#endif
188	188
	189	+void cpu_enable_pan(void);
	190	+
189	191	#endif /* __ASM_PROCESSOR_H */

arch/arm64/include/asm/ptrace.h+0−4

@@ -183,11 +183,7 @@ static inline int valid_user_regs(struct user_pt_regs *regs) · 182 unmodified lines
183	183
184	184	#define instruction_pointer(regs) ((unsigned long)(regs)->pc)
185	185
186		−#ifdef CONFIG_SMP
187	186	extern unsigned long profile_pc(struct pt_regs *regs);
188		−#else
189		−#define profile_pc(regs) instruction_pointer(regs)
190		−#endif
191	187
192	188	#endif /* __ASSEMBLY__ */
193	189	#endif

arch/arm64/include/asm/smp.h+0−4

@@ -20,10 +20,6 @@ · 19 unmodified lines
20	20	#include <linux/cpumask.h>
21	21	#include <linux/thread_info.h>
22	22
23		−#ifndef CONFIG_SMP
24		−# error "<asm/smp.h> included in non-SMP build"
25		−#endif
26		−
27	23	#define raw_smp_processor_id() (current_thread_info()->cpu)
28	24
29	25	struct seq_file;

arch/arm64/include/asm/smp_plat.h+0−2

@@ -56,6 +56,4 @@ static inline int get_logical_index(u64 mpidr) · 55 unmodified lines
56	56	return -EINVAL;
57	57	}
58	58
59		−void __init do_post_cpus_up_work(void);
60		−
61	59	#endif /* __ASM_SMP_PLAT_H */

arch/arm64/include/asm/spinlock.h+114−33

@@ -16,6 +16,7 @@ · 15 unmodified lines
16	16	#ifndef __ASM_SPINLOCK_H
17	17	#define __ASM_SPINLOCK_H
18	18
	19	+#include <asm/lse.h>
19	20	#include <asm/spinlock_types.h>
20	21	#include <asm/processor.h>
21	22
@@ -38,11 +39,21 @@ static inline void arch_spin_lock(arch_spinlock_t *lock) · 16 unmodified lines
38	39
39	40	asm volatile(
40	41	/* Atomically increment the next ticket. */
	42	+ ARM64_LSE_ATOMIC_INSN(
	43	+ /* LL/SC */
41	44	" prfm pstl1strm, %3\n"
42	45	"1: ldaxr %w0, %3\n"
43	46	" add %w1, %w0, %w5\n"
44	47	" stxr %w2, %w1, %3\n"
45		−" cbnz %w2, 1b\n"
	48	+" cbnz %w2, 1b\n",
	49	+ /* LSE atomics */
	50	+" mov %w2, %w5\n"
	51	+" ldadda %w2, %w0, %3\n"
	52	+" nop\n"
	53	+" nop\n"
	54	+" nop\n"
	55	+ )
	56	+
46	57	/* Did we get the lock? */
47	58	" eor %w1, %w0, %w0, ror #16\n"
48	59	" cbz %w1, 3f\n"
@@ -67,15 +78,25 @@ static inline int arch_spin_trylock(arch_spinlock_t *lock) · 18 unmodified lines
67	78	unsigned int tmp;
68	79	arch_spinlock_t lockval;
69	80
70		− asm volatile(
71		−" prfm pstl1strm, %2\n"
72		−"1: ldaxr %w0, %2\n"
73		−" eor %w1, %w0, %w0, ror #16\n"
74		−" cbnz %w1, 2f\n"
75		−" add %w0, %w0, %3\n"
76		−" stxr %w1, %w0, %2\n"
77		−" cbnz %w1, 1b\n"
78		−"2:"
	81	+ asm volatile(ARM64_LSE_ATOMIC_INSN(
	82	+ /* LL/SC */
	83	+ " prfm pstl1strm, %2\n"
	84	+ "1: ldaxr %w0, %2\n"
	85	+ " eor %w1, %w0, %w0, ror #16\n"
	86	+ " cbnz %w1, 2f\n"
	87	+ " add %w0, %w0, %3\n"
	88	+ " stxr %w1, %w0, %2\n"
	89	+ " cbnz %w1, 1b\n"
	90	+ "2:",
	91	+ /* LSE atomics */
	92	+ " ldr %w0, %2\n"
	93	+ " eor %w1, %w0, %w0, ror #16\n"
	94	+ " cbnz %w1, 1f\n"
	95	+ " add %w1, %w0, %3\n"
	96	+ " casa %w0, %w1, %2\n"
	97	+ " and %w1, %w1, #0xffff\n"
	98	+ " eor %w1, %w1, %w0, lsr #16\n"
	99	+ "1:")
79	100	: "=&r" (lockval), "=&r" (tmp), "+Q" (*lock)
80	101	: "I" (1 << TICKET_SHIFT)
81	102	: "memory");
@@ -85,10 +106,19 @@ static inline int arch_spin_trylock(arch_spinlock_t *lock) · 3 unmodified lines
85	106
86	107	static inline void arch_spin_unlock(arch_spinlock_t *lock)
87	108	{
88		− asm volatile(
89		−" stlrh %w1, %0\n"
90		− : "=Q" (lock->owner)
91		− : "r" (lock->owner + 1)
	109	+ unsigned long tmp;
	110	+
	111	+ asm volatile(ARM64_LSE_ATOMIC_INSN(
	112	+ /* LL/SC */
	113	+ " ldrh %w1, %0\n"
	114	+ " add %w1, %w1, #1\n"
	115	+ " stlrh %w1, %0",
	116	+ /* LSE atomics */
	117	+ " mov %w1, #1\n"
	118	+ " nop\n"
	119	+ " staddlh %w1, %0")
	120	+ : "=Q" (lock->owner), "=&r" (tmp)
	121	+ :
92	122	: "memory");
93	123	}
94	124
@@ -123,13 +153,24 @@ static inline void arch_write_lock(arch_rwlock_t *rw) · 28 unmodified lines
123	153	{
124	154	unsigned int tmp;
125	155
126		− asm volatile(
	156	+ asm volatile(ARM64_LSE_ATOMIC_INSN(
	157	+ /* LL/SC */
127	158	" sevl\n"
128	159	"1: wfe\n"
129	160	"2: ldaxr %w0, %1\n"
130	161	" cbnz %w0, 1b\n"
131	162	" stxr %w0, %w2, %1\n"
132	163	" cbnz %w0, 2b\n"
	164	+ " nop",
	165	+ /* LSE atomics */
	166	+ "1: mov %w0, wzr\n"
	167	+ "2: casa %w0, %w2, %1\n"
	168	+ " cbz %w0, 3f\n"
	169	+ " ldxr %w0, %1\n"
	170	+ " cbz %w0, 2b\n"
	171	+ " wfe\n"
	172	+ " b 1b\n"
	173	+ "3:")
133	174	: "=&r" (tmp), "+Q" (rw->lock)
134	175	: "r" (0x80000000)
135	176	: "memory");
@@ -139,11 +180,18 @@ static inline int arch_write_trylock(arch_rwlock_t *rw) · 3 unmodified lines
139	180	{
140	181	unsigned int tmp;
141	182
142		− asm volatile(
143		− " ldaxr %w0, %1\n"
144		− " cbnz %w0, 1f\n"
	183	+ asm volatile(ARM64_LSE_ATOMIC_INSN(
	184	+ /* LL/SC */
	185	+ "1: ldaxr %w0, %1\n"
	186	+ " cbnz %w0, 2f\n"
145	187	" stxr %w0, %w2, %1\n"
146		− "1:\n"
	188	+ " cbnz %w0, 1b\n"
	189	+ "2:",
	190	+ /* LSE atomics */
	191	+ " mov %w0, wzr\n"
	192	+ " casa %w0, %w2, %1\n"
	193	+ " nop\n"
	194	+ " nop")
147	195	: "=&r" (tmp), "+Q" (rw->lock)
148	196	: "r" (0x80000000)
149	197	: "memory");
@@ -153,9 +201,10 @@ static inline int arch_write_trylock(arch_rwlock_t *rw) · 3 unmodified lines
153	201
154	202	static inline void arch_write_unlock(arch_rwlock_t *rw)
155	203	{
156		− asm volatile(
157		− " stlr %w1, %0\n"
158		− : "=Q" (rw->lock) : "r" (0) : "memory");
	204	+ asm volatile(ARM64_LSE_ATOMIC_INSN(
	205	+ " stlr wzr, %0",
	206	+ " swpl wzr, wzr, %0")
	207	+ : "=Q" (rw->lock) :: "memory");
159	208	}
160	209
161	210	/* write_can_lock - would write_trylock() succeed? */
@@ -172,6 +221,10 @@ static inline void arch_write_unlock(arch_rwlock_t *rw) · 10 unmodified lines
172	221	*
173	222	* The memory barriers are implicit with the load-acquire and store-release
174	223	* instructions.
	224	+ *
	225	+ * Note that in UNDEFINED cases, such as unlocking a lock twice, the LL/SC
	226	+ * and LSE implementations may exhibit different behaviour (although this
	227	+ * will have no effect on lockdep).
175	228	*/
176	229	static inline void arch_read_lock(arch_rwlock_t *rw)
177	230	{
@@ -179,26 +232,43 @@ static inline void arch_read_lock(arch_rwlock_t *rw) · 1 unmodified line
179	232
180	233	asm volatile(
181	234	" sevl\n"
	235	+ ARM64_LSE_ATOMIC_INSN(
	236	+ /* LL/SC */
182	237	"1: wfe\n"
183	238	"2: ldaxr %w0, %2\n"
184	239	" add %w0, %w0, #1\n"
185	240	" tbnz %w0, #31, 1b\n"
186	241	" stxr %w1, %w0, %2\n"
187		− " cbnz %w1, 2b\n"
	242	+ " nop\n"
	243	+ " cbnz %w1, 2b",
	244	+ /* LSE atomics */
	245	+ "1: wfe\n"
	246	+ "2: ldxr %w0, %2\n"
	247	+ " adds %w1, %w0, #1\n"
	248	+ " tbnz %w1, #31, 1b\n"
	249	+ " casa %w0, %w1, %2\n"
	250	+ " sbc %w0, %w1, %w0\n"
	251	+ " cbnz %w0, 2b")
188	252	: "=&r" (tmp), "=&r" (tmp2), "+Q" (rw->lock)
189	253	:
190		− : "memory");
	254	+ : "cc", "memory");
191	255	}
192	256
193	257	static inline void arch_read_unlock(arch_rwlock_t *rw)
194	258	{
195	259	unsigned int tmp, tmp2;
196	260
197		− asm volatile(
	261	+ asm volatile(ARM64_LSE_ATOMIC_INSN(
	262	+ /* LL/SC */
198	263	"1: ldxr %w0, %2\n"
199	264	" sub %w0, %w0, #1\n"
200	265	" stlxr %w1, %w0, %2\n"
201		− " cbnz %w1, 1b\n"
	266	+ " cbnz %w1, 1b",
	267	+ /* LSE atomics */
	268	+ " movn %w0, #0\n"
	269	+ " nop\n"
	270	+ " nop\n"
	271	+ " staddl %w0, %2")
202	272	: "=&r" (tmp), "=&r" (tmp2), "+Q" (rw->lock)
203	273	:
204	274	: "memory");
@@ -206,17 +276,28 @@ static inline void arch_read_unlock(arch_rwlock_t *rw) · 1 unmodified line
206	276
207	277	static inline int arch_read_trylock(arch_rwlock_t *rw)
208	278	{
209		− unsigned int tmp, tmp2 = 1;
	279	+ unsigned int tmp, tmp2;
210	280
211		− asm volatile(
212		− " ldaxr %w0, %2\n"
	281	+ asm volatile(ARM64_LSE_ATOMIC_INSN(
	282	+ /* LL/SC */
	283	+ " mov %w1, #1\n"
	284	+ "1: ldaxr %w0, %2\n"
213	285	" add %w0, %w0, #1\n"
214		− " tbnz %w0, #31, 1f\n"
	286	+ " tbnz %w0, #31, 2f\n"
215	287	" stxr %w1, %w0, %2\n"
216		− "1:\n"
217		− : "=&r" (tmp), "+r" (tmp2), "+Q" (rw->lock)
	288	+ " cbnz %w1, 1b\n"
	289	+ "2:",
	290	+ /* LSE atomics */
	291	+ " ldr %w0, %2\n"
	292	+ " adds %w1, %w0, #1\n"
	293	+ " tbnz %w1, #31, 1f\n"
	294	+ " casa %w0, %w1, %2\n"
	295	+ " sbc %w1, %w1, %w0\n"
	296	+ " nop\n"
	297	+ "1:")
	298	+ : "=&r" (tmp), "=&r" (tmp2), "+Q" (rw->lock)
218	299	:
219		− : "memory");
	300	+ : "cc", "memory");
220	301
221	302	return !tmp2;
222	303	}

arch/arm64/include/asm/spinlock_types.h+2−0

@@ -20,6 +20,8 @@ · 19 unmodified lines
20	20	# error "please don't include this file directly"
21	21	#endif
22	22
	23	+#include <linux/types.h>
	24	+
23	25	#define TICKET_SHIFT 16
24	26
25	27	typedef struct {

arch/arm64/include/asm/sysreg.h+35−5

@@ -20,8 +20,29 @@ · 19 unmodified lines
20	20	#ifndef __ASM_SYSREG_H
21	21	#define __ASM_SYSREG_H
22	22
	23	+#include <asm/opcodes.h>
	24	+
	25	+#define SCTLR_EL1_CP15BEN (0x1 << 5)
	26	+#define SCTLR_EL1_SED (0x1 << 8)
	27	+
	28	+/*
	29	+ * ARMv8 ARM reserves the following encoding for system registers:
	30	+ * (Ref: ARMv8 ARM, Section: "System instruction class encoding overview",
	31	+ * C5.2, version:ARM DDI 0487A.f)
	32	+ * [20-19] : Op0
	33	+ * [18-16] : Op1
	34	+ * [15-12] : CRn
	35	+ * [11-8] : CRm
	36	+ * [7-5] : Op2
	37	+ */
23	38	#define sys_reg(op0, op1, crn, crm, op2) \
24		− ((((op0)-2)<<19)\|((op1)<<16)\|((crn)<<12)\|((crm)<<8)\|((op2)<<5))
	39	+ ((((op0)&3)<<19)\|((op1)<<16)\|((crn)<<12)\|((crm)<<8)\|((op2)<<5))
	40	+
	41	+#define REG_PSTATE_PAN_IMM sys_reg(0, 0, 4, 0, 4)
	42	+#define SCTLR_EL1_SPAN (1 << 23)
	43	+
	44	+#define SET_PSTATE_PAN(x) __inst_arm(0xd5000000 \| REG_PSTATE_PAN_IMM \|\
	45	+ (!!x)<<8 \| 0x1f)
25	46
26	47	#ifdef __ASSEMBLY__
27	48
@@ -31,11 +52,11 @@ · 3 unmodified lines
31	52	.equ __reg_num_xzr, 31
32	53
33	54	.macro mrs_s, rt, sreg
34		− .inst 0xd5300000\|(\sreg)\|(__reg_num_\rt)
	55	+ .inst 0xd5200000\|(\sreg)\|(__reg_num_\rt)
35	56	.endm
36	57
37	58	.macro msr_s, sreg, rt
38		− .inst 0xd5100000\|(\sreg)\|(__reg_num_\rt)
	59	+ .inst 0xd5000000\|(\sreg)\|(__reg_num_\rt)
39	60	.endm
40	61
41	62	#else
@@ -47,14 +68,23 @@ asm( · 5 unmodified lines
47	68	" .equ __reg_num_xzr, 31\n"
48	69	"\n"
49	70	" .macro mrs_s, rt, sreg\n"
50		−" .inst 0xd5300000\|(\\sreg)\|(__reg_num_\\rt)\n"
	71	+" .inst 0xd5200000\|(\\sreg)\|(__reg_num_\\rt)\n"
51	72	" .endm\n"
52	73	"\n"
53	74	" .macro msr_s, sreg, rt\n"
54		−" .inst 0xd5100000\|(\\sreg)\|(__reg_num_\\rt)\n"
	75	+" .inst 0xd5000000\|(\\sreg)\|(__reg_num_\\rt)\n"
55	76	" .endm\n"
56	77	);
57	78
	79	+static inline void config_sctlr_el1(u32 clear, u32 set)
	80	+{
	81	+ u32 val;
	82	+
	83	+ asm volatile("mrs %0, sctlr_el1" : "=r" (val));
	84	+ val &= ~clear;
	85	+ val \|= set;
	86	+ asm volatile("msr sctlr_el1, %0" : : "r" (val));
	87	+}
58	88	#endif
59	89
60	90	#endif /* __ASM_SYSREG_H */

arch/arm64/include/asm/tlb.h+6−1

@@ -41,7 +41,12 @@ static inline void tlb_flush(struct mmu_gather *tlb) · 40 unmodified lines
41	41	flush_tlb_mm(tlb->mm);
42	42	} else {
43	43	struct vm_area_struct vma = { .vm_mm = tlb->mm, };
44		− flush_tlb_range(&vma, tlb->start, tlb->end);
	44	+ /*
	45	+ * The intermediate page table levels are already handled by
	46	+ * the __(pte\|pmd\|pud)_free_tlb() functions, so last level
	47	+ * TLBI is sufficient here.
	48	+ */
	49	+ __flush_tlb_range(&vma, tlb->start, tlb->end, true);
45	50	}
46	51	}
47	52

arch/arm64/include/asm/tlbflush.h+34−42

@@ -87,27 +87,56 @@ static inline void flush_tlb_page(struct vm_area_struct *vma, · 86 unmodified lines
87	87	((unsigned long)ASID(vma->vm_mm) << 48);
88	88
89	89	dsb(ishst);
90		− asm("tlbi vae1is, %0" : : "r" (addr));
	90	+ asm("tlbi vale1is, %0" : : "r" (addr));
91	91	dsb(ish);
92	92	}
93	93
	94	+/*
	95	+ * This is meant to avoid soft lock-ups on large TLB flushing ranges and not
	96	+ * necessarily a performance improvement.
	97	+ */
	98	+#define MAX_TLB_RANGE (1024UL << PAGE_SHIFT)
	99	+
94	100	static inline void __flush_tlb_range(struct vm_area_struct *vma,
95		− unsigned long start, unsigned long end)
	101	+ unsigned long start, unsigned long end,
	102	+ bool last_level)
96	103	{
97	104	unsigned long asid = (unsigned long)ASID(vma->vm_mm) << 48;
98	105	unsigned long addr;
	106	+
	107	+ if ((end - start) > MAX_TLB_RANGE) {
	108	+ flush_tlb_mm(vma->vm_mm);
	109	+ return;
	110	+ }
	111	+
99	112	start = asid \| (start >> 12);
100	113	end = asid \| (end >> 12);
101	114
102	115	dsb(ishst);
103		− for (addr = start; addr < end; addr += 1 << (PAGE_SHIFT - 12))
104		− asm("tlbi vae1is, %0" : : "r"(addr));
	116	+ for (addr = start; addr < end; addr += 1 << (PAGE_SHIFT - 12)) {
	117	+ if (last_level)
	118	+ asm("tlbi vale1is, %0" : : "r"(addr));
	119	+ else
	120	+ asm("tlbi vae1is, %0" : : "r"(addr));
	121	+ }
105	122	dsb(ish);
106	123	}
107	124
108		−static inline void __flush_tlb_kernel_range(unsigned long start, unsigned long end)
	125	+static inline void flush_tlb_range(struct vm_area_struct *vma,
	126	+ unsigned long start, unsigned long end)
	127	+{
	128	+ __flush_tlb_range(vma, start, end, false);
	129	+}
	130	+
	131	+static inline void flush_tlb_kernel_range(unsigned long start, unsigned long end)
109	132	{
110	133	unsigned long addr;
	134	+
	135	+ if ((end - start) > MAX_TLB_RANGE) {
	136	+ flush_tlb_all();
	137	+ return;
	138	+ }
	139	+
111	140	start >>= 12;
112	141	end >>= 12;
113	142
@@ -118,29 +147,6 @@ static inline void __flush_tlb_kernel_range(unsigned long start, unsigned long e · 4 unmodified lines
118	147	isb();
119	148	}
120	149
121		−/*
122		− * This is meant to avoid soft lock-ups on large TLB flushing ranges and not
123		− * necessarily a performance improvement.
124		− */
125		−#define MAX_TLB_RANGE (1024UL << PAGE_SHIFT)
126		−
127		−static inline void flush_tlb_range(struct vm_area_struct *vma,
128		− unsigned long start, unsigned long end)
129		−{
130		− if ((end - start) <= MAX_TLB_RANGE)
131		− __flush_tlb_range(vma, start, end);
132		− else
133		− flush_tlb_mm(vma->vm_mm);
134		−}
135		−
136		−static inline void flush_tlb_kernel_range(unsigned long start, unsigned long end)
137		−{
138		− if ((end - start) <= MAX_TLB_RANGE)
139		− __flush_tlb_kernel_range(start, end);
140		− else
141		− flush_tlb_all();
142		−}
143		−
144	150	/*
145	151	* Used to invalidate the TLB (walk caches) corresponding to intermediate page
146	152	* table levels (pgd/pud/pmd).
@@ -154,20 +160,6 @@ static inline void __flush_tlb_pgtable(struct mm_struct *mm, · 7 unmodified lines
154	160	asm("tlbi vae1is, %0" : : "r" (addr));
155	161	dsb(ish);
156	162	}
157		−/*
158		− * On AArch64, the cache coherency is handled via the set_pte_at() function.
159		− */
160		−static inline void update_mmu_cache(struct vm_area_struct *vma,
161		− unsigned long addr, pte_t *ptep)
162		−{
163		− /*
164		− * set_pte() does not have a DSB for user mappings, so make sure that
165		− * the page table write is visible.
166		− */
167		− dsb(ishst);
168		−}
169		−
170		−#define update_mmu_cache_pmd(vma, address, pmd) do { } while (0)
171	163
172	164	#endif
173	165

arch/arm64/include/asm/topology.h+0−9

@@ -1,8 +1,6 @@
1	1	#ifndef __ASM_TOPOLOGY_H
2	2	#define __ASM_TOPOLOGY_H
3	3
4		−#ifdef CONFIG_SMP
5		−
6	4	#include <linux/cpumask.h>
7	5
8	6	struct cpu_topology {
@@ -24,13 +22,6 @@ void init_cpu_topology(void); · 15 unmodified lines
24	22	void store_cpu_topology(unsigned int cpuid);
25	23	const struct cpumask *cpu_coregroup_mask(int cpu);
26	24
27		−#else
28		−
29		−static inline void init_cpu_topology(void) { }
30		−static inline void store_cpu_topology(unsigned int cpuid) { }
31		−
32		−#endif
33		−
34	25	#include <asm-generic/topology.h>
35	26
36	27	#endif /* _ASM_ARM_TOPOLOGY_H */

arch/arm64/include/asm/traps.h+21−2

@@ -34,13 +34,32 @@ struct undef_hook { · 33 unmodified lines
34	34	void register_undef_hook(struct undef_hook *hook);
35	35	void unregister_undef_hook(struct undef_hook *hook);
36	36
	37	+#ifdef CONFIG_FUNCTION_GRAPH_TRACER
	38	+static inline int __in_irqentry_text(unsigned long ptr)
	39	+{
	40	+ extern char __irqentry_text_start[];
	41	+ extern char __irqentry_text_end[];
	42	+
	43	+ return ptr >= (unsigned long)&__irqentry_text_start &&
	44	+ ptr < (unsigned long)&__irqentry_text_end;
	45	+}
	46	+#else
	47	+static inline int __in_irqentry_text(unsigned long ptr)
	48	+{
	49	+ return 0;
	50	+}
	51	+#endif
	52	+
37	53	static inline int in_exception_text(unsigned long ptr)
38	54	{
39	55	extern char __exception_text_start[];
40	56	extern char __exception_text_end[];
	57	+ int in;
	58	+
	59	+ in = ptr >= (unsigned long)&__exception_text_start &&
	60	+ ptr < (unsigned long)&__exception_text_end;
41	61
42		− return ptr >= (unsigned long)&__exception_text_start &&
43		− ptr < (unsigned long)&__exception_text_end;
	62	+ return in ? : __in_irqentry_text(ptr);
44	63	}
45	64
46	65	#endif

arch/arm64/include/asm/uaccess.h+11−0

@@ -24,7 +24,10 @@ · 23 unmodified lines
24	24	#include <linux/string.h>
25	25	#include <linux/thread_info.h>
26	26
	27	+#include <asm/alternative.h>
	28	+#include <asm/cpufeature.h>
27	29	#include <asm/ptrace.h>
	30	+#include <asm/sysreg.h>
28	31	#include <asm/errno.h>
29	32	#include <asm/memory.h>
30	33	#include <asm/compiler.h>
@@ -131,6 +134,8 @@ static inline void set_fs(mm_segment_t fs) · 100 unmodified lines
131	134	do { \
132	135	unsigned long __gu_val; \
133	136	__chk_user_ptr(ptr); \
	137	+ asm(ALTERNATIVE("nop", SET_PSTATE_PAN(0), ARM64_HAS_PAN, \
	138	+ CONFIG_ARM64_PAN)); \
134	139	switch (sizeof(*(ptr))) { \
135	140	case 1: \
136	141	__get_user_asm("ldrb", "%w", __gu_val, (ptr), (err)); \
@@ -148,6 +153,8 @@ do { \ · 11 unmodified lines
148	153	BUILD_BUG(); \
149	154	} \
150	155	(x) = (__force __typeof__(*(ptr)))__gu_val; \
	156	+ asm(ALTERNATIVE("nop", SET_PSTATE_PAN(1), ARM64_HAS_PAN, \
	157	+ CONFIG_ARM64_PAN)); \
151	158	} while (0)
152	159
153	160	#define __get_user(x, ptr) \
@@ -194,6 +201,8 @@ do { \ · 40 unmodified lines
194	201	do { \
195	202	__typeof__(*(ptr)) __pu_val = (x); \
196	203	__chk_user_ptr(ptr); \
	204	+ asm(ALTERNATIVE("nop", SET_PSTATE_PAN(0), ARM64_HAS_PAN, \
	205	+ CONFIG_ARM64_PAN)); \
197	206	switch (sizeof(*(ptr))) { \
198	207	case 1: \
199	208	__put_user_asm("strb", "%w", __pu_val, (ptr), (err)); \
@@ -210,6 +219,8 @@ do { \ · 10 unmodified lines
210	219	default: \
211	220	BUILD_BUG(); \
212	221	} \
	222	+ asm(ALTERNATIVE("nop", SET_PSTATE_PAN(1), ARM64_HAS_PAN, \
	223	+ CONFIG_ARM64_PAN)); \
213	224	} while (0)
214	225
215	226	#define __put_user(x, ptr) \

arch/arm64/include/uapi/asm/hwcap.h+1−0

@@ -27,5 +27,6 @@ · 26 unmodified lines
27	27	#define HWCAP_SHA1 (1 << 5)
28	28	#define HWCAP_SHA2 (1 << 6)
29	29	#define HWCAP_CRC32 (1 << 7)
	30	+#define HWCAP_ATOMICS (1 << 8)
30	31
31	32	#endif /* _UAPI__ASM_HWCAP_H */

arch/arm64/include/uapi/asm/ptrace.h+1−0

@@ -44,6 +44,7 @@ · 43 unmodified lines
44	44	#define PSR_I_BIT 0x00000080
45	45	#define PSR_A_BIT 0x00000100
46	46	#define PSR_D_BIT 0x00000200
	47	+#define PSR_PAN_BIT 0x00400000
47	48	#define PSR_Q_BIT 0x08000000
48	49	#define PSR_V_BIT 0x10000000
49	50	#define PSR_C_BIT 0x20000000

arch/arm64/kernel/Makefile+3−3

@@ -17,15 +17,15 @@ arm64-obj-y := debug-monitors.o entry.o irq.o fpsimd.o \ · 16 unmodified lines
17	17	sys.o stacktrace.o time.o traps.o io.o vdso.o \
18	18	hyp-stub.o psci.o psci-call.o cpu_ops.o insn.o \
19	19	return_address.o cpuinfo.o cpu_errata.o \
20		− cpufeature.o alternative.o cacheinfo.o
	20	+ cpufeature.o alternative.o cacheinfo.o \
	21	+ smp.o smp_spin_table.o topology.o
21	22
22	23	arm64-obj-$(CONFIG_COMPAT) += sys32.o kuser32.o signal32.o \
23	24	sys_compat.o entry32.o \
24	25	../../arm/kernel/opcodes.o
25	26	arm64-obj-$(CONFIG_FUNCTION_TRACER) += ftrace.o entry-ftrace.o
26	27	arm64-obj-$(CONFIG_MODULES) += arm64ksyms.o module.o
27		−arm64-obj-$(CONFIG_SMP) += smp.o smp_spin_table.o topology.o
28		−arm64-obj-$(CONFIG_PERF_EVENTS) += perf_regs.o
	28	+arm64-obj-$(CONFIG_PERF_EVENTS) += perf_regs.o perf_callchain.o
29	29	arm64-obj-$(CONFIG_HW_PERF_EVENTS) += perf_event.o
30	30	arm64-obj-$(CONFIG_HAVE_HW_BREAKPOINT) += hw_breakpoint.o
31	31	arm64-obj-$(CONFIG_CPU_PM) += sleep.o suspend.o

arch/arm64/kernel/alternative.c+25−5

@@ -85,7 +85,7 @@ static u32 get_alt_insn(struct alt_instr alt, u32 insnptr, u32 *altinsnptr) · 84 unmodified lines
85	85	return insn;
86	86	}
87	87
88		−static int __apply_alternatives(void *alt_region)
	88	+static void __apply_alternatives(void *alt_region)
89	89	{
90	90	struct alt_instr *alt;
91	91	struct alt_region *region = alt_region;
@@ -114,19 +114,39 @@ static int __apply_alternatives(void *alt_region) · 22 unmodified lines
114	114	flush_icache_range((uintptr_t)origptr,
115	115	(uintptr_t)(origptr + nr_inst));
116	116	}
117		−
118		− return 0;
119	117	}
120	118
121		−void apply_alternatives_all(void)
	119	+/*
	120	+ * We might be patching the stop_machine state machine, so implement a
	121	+ * really simple polling protocol here.
	122	+ */
	123	+static int __apply_alternatives_multi_stop(void *unused)
122	124	{
	125	+ static int patched = 0;
123	126	struct alt_region region = {
124	127	.begin = __alt_instructions,
125	128	.end = __alt_instructions_end,
126	129	};
127	130
	131	+ /* We always have a CPU 0 at this point (__init) */
	132	+ if (smp_processor_id()) {
	133	+ while (!READ_ONCE(patched))
	134	+ cpu_relax();
	135	+ isb();
	136	+ } else {
	137	+ BUG_ON(patched);
	138	+ __apply_alternatives(&region);
	139	+ /* Barriers provided by the cache flushing */
	140	+ WRITE_ONCE(patched, 1);
	141	+ }
	142	+
	143	+ return 0;
	144	+}
	145	+
	146	+void __init apply_alternatives_all(void)
	147	+{
128	148	/* better not try code patching on a live SMP system */
129		− stop_machine(__apply_alternatives, &region, NULL);
	149	+ stop_machine(__apply_alternatives_multi_stop, NULL, cpu_online_mask);
130	150	}
131	151
132	152	void apply_alternatives(void *start, size_t length)

arch/arm64/kernel/armv8_deprecated.c+8−11

@@ -14,8 +14,11 @@ · 13 unmodified lines
14	14	#include <linux/slab.h>
15	15	#include <linux/sysctl.h>
16	16
	17	+#include <asm/alternative.h>
	18	+#include <asm/cpufeature.h>
17	19	#include <asm/insn.h>
18	20	#include <asm/opcodes.h>
	21	+#include <asm/sysreg.h>
19	22	#include <asm/system_misc.h>
20	23	#include <asm/traps.h>
21	24	#include <asm/uaccess.h>
@@ -279,6 +282,8 @@ static void register_insn_emulation_sysctl(struct ctl_table *table) · 257 unmodified lines
279	282	*/
280	283	#define __user_swpX_asm(data, addr, res, temp, B) \
281	284	__asm__ __volatile__( \
	285	+ ALTERNATIVE("nop", SET_PSTATE_PAN(0), ARM64_HAS_PAN, \
	286	+ CONFIG_ARM64_PAN) \
282	287	" mov %w2, %w1\n" \
283	288	"0: ldxr"B" %w1, [%3]\n" \
284	289	"1: stxr"B" %w0, %w2, [%3]\n" \
@@ -294,7 +299,9 @@ static void register_insn_emulation_sysctl(struct ctl_table *table) · 9 unmodified lines
294	299	" .align 3\n" \
295	300	" .quad 0b, 3b\n" \
296	301	" .quad 1b, 3b\n" \
297		− " .popsection" \
	302	+ " .popsection\n" \
	303	+ ALTERNATIVE("nop", SET_PSTATE_PAN(1), ARM64_HAS_PAN, \
	304	+ CONFIG_ARM64_PAN) \
298	305	: "=&r" (res), "+r" (data), "=&r" (temp) \
299	306	: "r" (addr), "i" (-EAGAIN), "i" (-EFAULT) \
300	307	: "memory")
@@ -504,16 +511,6 @@ ret: · 203 unmodified lines
504	511	return 0;
505	512	}
506	513
507		−static inline void config_sctlr_el1(u32 clear, u32 set)
508		−{
509		− u32 val;
510		−
511		− asm volatile("mrs %0, sctlr_el1" : "=r" (val));
512		− val &= ~clear;
513		− val \|= set;
514		− asm volatile("msr sctlr_el1, %0" : : "r" (val));
515		−}
516		−
517	514	static int cp15_barrier_set_hw_mode(bool enable)
518	515	{
519	516	if (enable)

arch/arm64/kernel/cpu_ops.c+0−2

@@ -30,9 +30,7 @@ extern const struct cpu_operations cpu_psci_ops; · 29 unmodified lines
30	30	const struct cpu_operations *cpu_ops[NR_CPUS];
31	31
32	32	static const struct cpu_operations *supported_cpu_ops[] __initconst = {
33		−#ifdef CONFIG_SMP
34	33	&smp_spin_table_ops,
35		−#endif
36	34	&cpu_psci_ops,
37	35	NULL,
38	36	};

arch/arm64/kernel/cpufeature.c+46−7

@@ -21,24 +21,57 @@ · 20 unmodified lines
21	21	#include <linux/types.h>
22	22	#include <asm/cpu.h>
23	23	#include <asm/cpufeature.h>
	24	+#include <asm/processor.h>
24	25
25	26	static bool
26		−has_id_aa64pfr0_feature(const struct arm64_cpu_capabilities *entry)
	27	+feature_matches(u64 reg, const struct arm64_cpu_capabilities *entry)
27	28	{
28		− u64 val;
	29	+ int val = cpuid_feature_extract_field(reg, entry->field_pos);
29	30
30		− val = read_cpuid(id_aa64pfr0_el1);
31		− return (val & entry->register_mask) == entry->register_value;
	31	+ return val >= entry->min_field_value;
32	32	}
33	33
	34	+#define __ID_FEAT_CHK(reg) \
	35	+static bool __maybe_unused \
	36	+has_##reg##_feature(const struct arm64_cpu_capabilities *entry) \
	37	+{ \
	38	+ u64 val; \
	39	+ \
	40	+ val = read_cpuid(reg##_el1); \
	41	+ return feature_matches(val, entry); \
	42	+}
	43	+
	44	+__ID_FEAT_CHK(id_aa64pfr0);
	45	+__ID_FEAT_CHK(id_aa64mmfr1);
	46	+__ID_FEAT_CHK(id_aa64isar0);
	47	+
34	48	static const struct arm64_cpu_capabilities arm64_features[] = {
35	49	{
36	50	.desc = "GIC system register CPU interface",
37	51	.capability = ARM64_HAS_SYSREG_GIC_CPUIF,
38	52	.matches = has_id_aa64pfr0_feature,
39		− .register_mask = (0xf << 24),
40		− .register_value = (1 << 24),
	53	+ .field_pos = 24,
	54	+ .min_field_value = 1,
	55	+ },
	56	+#ifdef CONFIG_ARM64_PAN
	57	+ {
	58	+ .desc = "Privileged Access Never",
	59	+ .capability = ARM64_HAS_PAN,
	60	+ .matches = has_id_aa64mmfr1_feature,
	61	+ .field_pos = 20,
	62	+ .min_field_value = 1,
	63	+ .enable = cpu_enable_pan,
41	64	},
	65	+#endif /* CONFIG_ARM64_PAN */
	66	+#if defined(CONFIG_AS_LSE) && defined(CONFIG_ARM64_LSE_ATOMICS)
	67	+ {
	68	+ .desc = "LSE atomic instructions",
	69	+ .capability = ARM64_HAS_LSE_ATOMICS,
	70	+ .matches = has_id_aa64isar0_feature,
	71	+ .field_pos = 20,
	72	+ .min_field_value = 2,
	73	+ },
	74	+#endif /* CONFIG_AS_LSE && CONFIG_ARM64_LSE_ATOMICS */
42	75	{},
43	76	};
44	77
@@ -55,9 +88,15 @@ void check_cpu_capabilities(const struct arm64_cpu_capabilities *caps, · 10 unmodified lines
55	88	pr_info("%s %s\n", info, caps[i].desc);
56	89	cpus_set_cap(caps[i].capability);
57	90	}
	91	+
	92	+ /* second pass allows enable() to consider interacting capabilities */
	93	+ for (i = 0; caps[i].desc; i++) {
	94	+ if (cpus_have_cap(caps[i].capability) && caps[i].enable)
	95	+ caps[i].enable();
	96	+ }
58	97	}
59	98
60	99	void check_local_cpu_features(void)
61	100	{
62		− check_cpu_capabilities(arm64_features, "detected feature");
	101	+ check_cpu_capabilities(arm64_features, "detected feature:");
63	102	}

arch/arm64/kernel/debug-monitors.c+2−2

@@ -82,7 +82,7 @@ early_param("nodebugmon", early_debug_disable); · 81 unmodified lines
82	82	static DEFINE_PER_CPU(int, mde_ref_count);
83	83	static DEFINE_PER_CPU(int, kde_ref_count);
84	84
85		−void enable_debug_monitors(enum debug_el el)
	85	+void enable_debug_monitors(enum dbg_active_el el)
86	86	{
87	87	u32 mdscr, enable = 0;
88	88
@@ -102,7 +102,7 @@ void enable_debug_monitors(enum debug_el el) · 13 unmodified lines
102	102	}
103	103	}
104	104
105		−void disable_debug_monitors(enum debug_el el)
	105	+void disable_debug_monitors(enum dbg_active_el el)
106	106	{
107	107	u32 mdscr, disable = 0;
108	108

arch/arm64/kernel/efi-stub.c+32−9

@@ -13,7 +13,7 @@ · 12 unmodified lines
13	13	#include <asm/efi.h>
14	14	#include <asm/sections.h>
15	15
16		−efi_status_t __init handle_kernel_image(efi_system_table_t *sys_table,
	16	+efi_status_t __init handle_kernel_image(efi_system_table_t *sys_table_arg,
17	17	unsigned long *image_addr,
18	18	unsigned long *image_size,
19	19	unsigned long *reserve_addr,
@@ -23,21 +23,44 @@ efi_status_t __init handle_kernel_image(efi_system_table_t *sys_table, · 3 unmodified lines
23	23	{
24	24	efi_status_t status;
25	25	unsigned long kernel_size, kernel_memsize = 0;
	26	+ unsigned long nr_pages;
	27	+ void old_image_addr = (void )*image_addr;
26	28
27	29	/* Relocate the image, if required. */
28	30	kernel_size = _edata - _text;
29	31	if (*image_addr != (dram_base + TEXT_OFFSET)) {
30	32	kernel_memsize = kernel_size + (_end - _edata);
31		− status = efi_low_alloc(sys_table, kernel_memsize + TEXT_OFFSET,
32		− SZ_2M, reserve_addr);
	33	+
	34	+ /*
	35	+ * First, try a straight allocation at the preferred offset.
	36	+ * This will work around the issue where, if dram_base == 0x0,
	37	+ * efi_low_alloc() refuses to allocate at 0x0 (to prevent the
	38	+ * address of the allocation to be mistaken for a FAIL return
	39	+ * value or a NULL pointer). It will also ensure that, on
	40	+ * platforms where the [dram_base, dram_base + TEXT_OFFSET)
	41	+ * interval is partially occupied by the firmware (like on APM
	42	+ * Mustang), we can still place the kernel at the address
	43	+ * 'dram_base + TEXT_OFFSET'.
	44	+ */
	45	+ image_addr = reserve_addr = dram_base + TEXT_OFFSET;
	46	+ nr_pages = round_up(kernel_memsize, EFI_ALLOC_ALIGN) /
	47	+ EFI_PAGE_SIZE;
	48	+ status = efi_call_early(allocate_pages, EFI_ALLOCATE_ADDRESS,
	49	+ EFI_LOADER_DATA, nr_pages,
	50	+ (efi_physical_addr_t *)reserve_addr);
33	51	if (status != EFI_SUCCESS) {
34		− pr_efi_err(sys_table, "Failed to relocate kernel\n");
35		− return status;
	52	+ kernel_memsize += TEXT_OFFSET;
	53	+ status = efi_low_alloc(sys_table_arg, kernel_memsize,
	54	+ SZ_2M, reserve_addr);
	55	+
	56	+ if (status != EFI_SUCCESS) {
	57	+ pr_efi_err(sys_table_arg, "Failed to relocate kernel\n");
	58	+ return status;
	59	+ }
	60	+ image_addr = reserve_addr + TEXT_OFFSET;
36	61	}
37		− memcpy((void )reserve_addr + TEXT_OFFSET, (void )image_addr,
38		− kernel_size);
39		− image_addr = reserve_addr + TEXT_OFFSET;
40		− *reserve_size = kernel_memsize + TEXT_OFFSET;
	62	+ memcpy((void )image_addr, old_image_addr, kernel_size);
	63	+ *reserve_size = kernel_memsize;
41	64	}
42	65
43	66

arch/arm64/kernel/entry.S+19−27

@@ -116,41 +116,34 @@ · 115 unmodified lines
116	116	*/
117	117	.endm
118	118
119		− .macro kernel_exit, el, ret = 0
	119	+ .macro kernel_exit, el
120	120	ldp x21, x22, [sp, #S_PC] // load ELR, SPSR
121	121	.if \el == 0
122	122	ct_user_enter
123	123	ldr x23, [sp, #S_SP] // load return stack pointer
124	124	msr sp_el0, x23
125		−
126	125	#ifdef CONFIG_ARM64_ERRATUM_845719
127		−
128		−#undef SEQUENCE_ORG
129		−#undef SEQUENCE_ALT
130		−
	126	+alternative_if_not ARM64_WORKAROUND_845719
	127	+ nop
	128	+ nop
131	129	#ifdef CONFIG_PID_IN_CONTEXTIDR
132		−
133		−#define SEQUENCE_ORG "nop ; nop ; nop"
134		−#define SEQUENCE_ALT "tbz x22, #4, 1f ; mrs x29, contextidr_el1; msr contextidr_el1, x29; 1:"
135		−
	130	+ nop
	131	+#endif
	132	+alternative_else
	133	+ tbz x22, #4, 1f
	134	+#ifdef CONFIG_PID_IN_CONTEXTIDR
	135	+ mrs x29, contextidr_el1
	136	+ msr contextidr_el1, x29
136	137	#else
137		−
138		−#define SEQUENCE_ORG "nop ; nop"
139		−#define SEQUENCE_ALT "tbz x22, #4, 1f ; msr contextidr_el1, xzr; 1:"
140		−
	138	+ msr contextidr_el1, xzr
141	139	#endif
142		−
143		− alternative_insn SEQUENCE_ORG, SEQUENCE_ALT, ARM64_WORKAROUND_845719
144		−
	140	+1:
	141	+alternative_endif
145	142	#endif
146	143	.endif
147	144	msr elr_el1, x21 // set up the return data
148	145	msr spsr_el1, x22
149		− .if \ret
150		− ldr x1, [sp, #S_X1] // preserve x0 (syscall return)
151		− .else
152	146	ldp x0, x1, [sp, #16 * 0]
153		− .endif
154	147	ldp x2, x3, [sp, #16 * 1]
155	148	ldp x4, x5, [sp, #16 * 2]
156	149	ldp x6, x7, [sp, #16 * 3]
@@ -613,22 +606,21 @@ ENDPROC(cpu_switch_to) · 456 unmodified lines
613	606	*/
614	607	ret_fast_syscall:
615	608	disable_irq // disable interrupts
	609	+ str x0, [sp, #S_X0] // returned x0
616	610	ldr x1, [tsk, #TI_FLAGS] // re-check for syscall tracing
617	611	and x2, x1, #_TIF_SYSCALL_WORK
618	612	cbnz x2, ret_fast_syscall_trace
619	613	and x2, x1, #_TIF_WORK_MASK
620		− cbnz x2, fast_work_pending
	614	+ cbnz x2, work_pending
621	615	enable_step_tsk x1, x2
622		− kernel_exit 0, ret = 1
	616	+ kernel_exit 0
623	617	ret_fast_syscall_trace:
624	618	enable_irq // enable interrupts
625		− b __sys_trace_return
	619	+ b __sys_trace_return_skipped // we already saved x0
626	620
627	621	/*
628	622	* Ok, we need to do extra processing, enter the slow path.
629	623	*/
630		−fast_work_pending:
631		− str x0, [sp, #S_X0] // returned x0
632	624	work_pending:
633	625	tbnz x1, #TIF_NEED_RESCHED, work_resched
634	626	/* TIF_SIGPENDING, TIF_NOTIFY_RESUME or TIF_FOREIGN_FPSTATE case */
@@ -652,7 +644,7 @@ ret_to_user: · 17 unmodified lines
652	644	cbnz x2, work_pending
653	645	enable_step_tsk x1, x2
654	646	no_work_pending:
655		− kernel_exit 0, ret = 0
	647	+ kernel_exit 0
656	648	ENDPROC(ret_to_user)
657	649
658	650	/*

arch/arm64/kernel/fpsimd.c+1−0

@@ -158,6 +158,7 @@ void fpsimd_thread_switch(struct task_struct *next) · 157 unmodified lines
158	158	void fpsimd_flush_thread(void)
159	159	{
160	160	memset(&current->thread.fpsimd_state, 0, sizeof(struct fpsimd_state));
	161	+ fpsimd_flush_task_state(current);
161	162	set_thread_flag(TIF_FOREIGN_FPSTATE);
162	163	}
163	164

arch/arm64/kernel/head.S+8−7

@@ -62,13 +62,8 @@ · 61 unmodified lines
62	62	/*
63	63	* Initial memory map attributes.
64	64	*/
65		−#ifndef CONFIG_SMP
66		−#define PTE_FLAGS PTE_TYPE_PAGE \| PTE_AF
67		−#define PMD_FLAGS PMD_TYPE_SECT \| PMD_SECT_AF
68		−#else
69	65	#define PTE_FLAGS PTE_TYPE_PAGE \| PTE_AF \| PTE_SHARED
70	66	#define PMD_FLAGS PMD_TYPE_SECT \| PMD_SECT_AF \| PMD_SECT_S
71		−#endif
72	67
73	68	#ifdef CONFIG_ARM64_64K_PAGES
74	69	#define MM_MMUFLAGS PTE_ATTRINDX(MT_NORMAL) \| PTE_FLAGS
@@ -574,7 +569,6 @@ ENTRY(__boot_cpu_mode) · 499 unmodified lines
574	569	.long BOOT_CPU_MODE_EL1
575	570	.popsection
576	571
577		−#ifdef CONFIG_SMP
578	572	/*
579	573	* This provides a "holding pen" for platforms to hold all secondary
580	574	* cores are held until we're ready for them to initialise.
@@ -622,7 +616,6 @@ ENTRY(__secondary_switched) · 41 unmodified lines
622	616	mov x29, #0
623	617	b secondary_start_kernel
624	618	ENDPROC(__secondary_switched)
625		−#endif /* CONFIG_SMP */
626	619
627	620	/*
628	621	* Enable the MMU.
@@ -641,5 +634,13 @@ __enable_mmu: · 12 unmodified lines
641	634	isb
642	635	msr sctlr_el1, x0
643	636	isb
	637	+ /*
	638	+ * Invalidate the local I-cache so that any instructions fetched
	639	+ * speculatively from the PoC are discarded, since they may have
	640	+ * been dynamically patched at the PoU.
	641	+ */
	642	+ ic iallu
	643	+ dsb nsh
	644	+ isb
644	645	br x27
645	646	ENDPROC(__enable_mmu)

arch/arm64/kernel/hw_breakpoint.c+3−3

@@ -156,7 +156,7 @@ static void write_wb_reg(int reg, int n, u64 val) · 155 unmodified lines
156	156	* Convert a breakpoint privilege level to the corresponding exception
157	157	* level.
158	158	*/
159		−static enum debug_el debug_exception_level(int privilege)
	159	+static enum dbg_active_el debug_exception_level(int privilege)
160	160	{
161	161	switch (privilege) {
162	162	case AARCH64_BREAKPOINT_EL0:
@@ -230,7 +230,7 @@ static int hw_breakpoint_control(struct perf_event *bp, · 67 unmodified lines
230	230	struct perf_event **slots;
231	231	struct debug_info *debug_info = &current->thread.debug;
232	232	int i, max_slots, ctrl_reg, val_reg, reg_enable;
233		− enum debug_el dbg_el = debug_exception_level(info->ctrl.privilege);
	233	+ enum dbg_active_el dbg_el = debug_exception_level(info->ctrl.privilege);
234	234	u32 ctrl;
235	235
236	236	if (info->ctrl.type == ARM_BREAKPOINT_EXECUTE) {
@@ -537,7 +537,7 @@ int arch_validate_hwbkpt_settings(struct perf_event *bp) · 300 unmodified lines
537	537	* exception level at the register level.
538	538	* This is used when single-stepping after a breakpoint exception.
539	539	*/
540		−static void toggle_bp_registers(int reg, enum debug_el el, int enable)
	540	+static void toggle_bp_registers(int reg, enum dbg_active_el el, int enable)
541	541	{
542	542	int i, max_slots, privilege;
543	543	u32 ctrl;

arch/arm64/kernel/insn.c+2−3

@@ -101,9 +101,8 @@ static void __kprobes patch_map(void addr, int fixmap) · 100 unmodified lines
101	101	return addr;
102	102
103	103	BUG_ON(!page);
104		− set_fixmap(fixmap, page_to_phys(page));
105		−
106		− return (void *) (__fix_to_virt(fixmap) + (uintaddr & ~PAGE_MASK));
	104	+ return (void *)set_fixmap_offset(fixmap, page_to_phys(page) +
	105	+ (uintaddr & ~PAGE_MASK));
107	106	}
108	107
109	108	static void __kprobes patch_unmap(int fixmap)

arch/arm64/kernel/irq.c+0−2

@@ -33,9 +33,7 @@ unsigned long irq_err_count; · 32 unmodified lines
33	33
34	34	int arch_show_interrupts(struct seq_file *p, int prec)
35	35	{
36		−#ifdef CONFIG_SMP
37	36	show_ipi_list(p, prec);
38		−#endif
39	37	seq_printf(p, "%*s: %10lu\n", prec, "Err", irq_err_count);
40	38	return 0;
41	39	}

arch/arm64/kernel/kgdb.c+6−6

@@ -235,13 +235,13 @@ static int kgdb_step_brk_fn(struct pt_regs *regs, unsigned int esr) · 234 unmodified lines
235	235
236	236	static struct break_hook kgdb_brkpt_hook = {
237	237	.esr_mask = 0xffffffff,
238		− .esr_val = DBG_ESR_VAL_BRK(KGDB_DYN_DBG_BRK_IMM),
	238	+ .esr_val = (u32)ESR_ELx_VAL_BRK64(KGDB_DYN_DBG_BRK_IMM),
239	239	.fn = kgdb_brk_fn
240	240	};
241	241
242	242	static struct break_hook kgdb_compiled_brkpt_hook = {
243	243	.esr_mask = 0xffffffff,
244		− .esr_val = DBG_ESR_VAL_BRK(KGDB_COMPILED_DBG_BRK_IMM),
	244	+ .esr_val = (u32)ESR_ELx_VAL_BRK64(KGDB_COMPILED_DBG_BRK_IMM),
245	245	.fn = kgdb_compiled_brk_fn
246	246	};
247	247
@@ -328,9 +328,9 @@ void kgdb_arch_exit(void) · 80 unmodified lines
328	328	*/
329	329	struct kgdb_arch arch_kgdb_ops = {
330	330	.gdb_bpt_instr = {
331		− KGDB_DYN_BRK_INS_BYTE0,
332		− KGDB_DYN_BRK_INS_BYTE1,
333		− KGDB_DYN_BRK_INS_BYTE2,
334		− KGDB_DYN_BRK_INS_BYTE3,
	331	+ KGDB_DYN_BRK_INS_BYTE(0),
	332	+ KGDB_DYN_BRK_INS_BYTE(1),
	333	+ KGDB_DYN_BRK_INS_BYTE(2),
	334	+ KGDB_DYN_BRK_INS_BYTE(3),
335	335	}
336	336	};

arch/arm64/kernel/pci.c+13−0

@@ -38,6 +38,19 @@ resource_size_t pcibios_align_resource(void data, const struct resource res, · 37 unmodified lines
38	38	return res->start;
39	39	}
40	40
	41	+/**
	42	+ * pcibios_enable_device - Enable I/O and memory.
	43	+ * @dev: PCI device to be enabled
	44	+ * @mask: bitmask of BARs to enable
	45	+ */
	46	+int pcibios_enable_device(struct pci_dev *dev, int mask)
	47	+{
	48	+ if (pci_has_flag(PCI_PROBE_ONLY))
	49	+ return 0;
	50	+
	51	+ return pci_enable_resources(dev, mask);
	52	+}
	53	+
41	54	/*
42	55	* Try to assign the IRQ number from DT when adding a new device
43	56	*/

arch/arm64/kernel/perf_callchain.c+196−0

@@ -0,0 +1,196 @@
	1	+/*
	2	+ * arm64 callchain support
	3	+ *
	4	+ * Copyright (C) 2015 ARM Limited
	5	+ *
	6	+ * This program is free software; you can redistribute it and/or modify
	7	+ * it under the terms of the GNU General Public License version 2 as
	8	+ * published by the Free Software Foundation.
	9	+ *
	10	+ * This program is distributed in the hope that it will be useful,
	11	+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
	12	+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	13	+ * GNU General Public License for more details.
	14	+ *
	15	+ * You should have received a copy of the GNU General Public License
	16	+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
	17	+ */
	18	+#include <linux/perf_event.h>
	19	+#include <linux/uaccess.h>
	20	+
	21	+#include <asm/stacktrace.h>
	22	+
	23	+struct frame_tail {
	24	+ struct frame_tail __user *fp;
	25	+ unsigned long lr;
	26	+} __attribute__((packed));
	27	+
	28	+/*
	29	+ * Get the return address for a single stackframe and return a pointer to the
	30	+ * next frame tail.
	31	+ */
	32	+static struct frame_tail __user *
	33	+user_backtrace(struct frame_tail __user *tail,
	34	+ struct perf_callchain_entry *entry)
	35	+{
	36	+ struct frame_tail buftail;
	37	+ unsigned long err;
	38	+
	39	+ /* Also check accessibility of one struct frame_tail beyond */
	40	+ if (!access_ok(VERIFY_READ, tail, sizeof(buftail)))
	41	+ return NULL;
	42	+
	43	+ pagefault_disable();
	44	+ err = __copy_from_user_inatomic(&buftail, tail, sizeof(buftail));
	45	+ pagefault_enable();
	46	+
	47	+ if (err)
	48	+ return NULL;
	49	+
	50	+ perf_callchain_store(entry, buftail.lr);
	51	+
	52	+ /*
	53	+ * Frame pointers should strictly progress back up the stack
	54	+ * (towards higher addresses).
	55	+ */
	56	+ if (tail >= buftail.fp)
	57	+ return NULL;
	58	+
	59	+ return buftail.fp;
	60	+}
	61	+
	62	+#ifdef CONFIG_COMPAT
	63	+/*
	64	+ * The registers we're interested in are at the end of the variable
	65	+ * length saved register structure. The fp points at the end of this
	66	+ * structure so the address of this struct is:
	67	+ * (struct compat_frame_tail *)(xxx->fp)-1
	68	+ *
	69	+ * This code has been adapted from the ARM OProfile support.
	70	+ */
	71	+struct compat_frame_tail {
	72	+ compat_uptr_t fp; /* a (struct compat_frame_tail ) in compat mode /
	73	+ u32 sp;
	74	+ u32 lr;
	75	+} __attribute__((packed));
	76	+
	77	+static struct compat_frame_tail __user *
	78	+compat_user_backtrace(struct compat_frame_tail __user *tail,
	79	+ struct perf_callchain_entry *entry)
	80	+{
	81	+ struct compat_frame_tail buftail;
	82	+ unsigned long err;
	83	+
	84	+ /* Also check accessibility of one struct frame_tail beyond */
	85	+ if (!access_ok(VERIFY_READ, tail, sizeof(buftail)))
	86	+ return NULL;
	87	+
	88	+ pagefault_disable();
	89	+ err = __copy_from_user_inatomic(&buftail, tail, sizeof(buftail));
	90	+ pagefault_enable();
	91	+
	92	+ if (err)
	93	+ return NULL;
	94	+
	95	+ perf_callchain_store(entry, buftail.lr);
	96	+
	97	+ /*
	98	+ * Frame pointers should strictly progress back up the stack
	99	+ * (towards higher addresses).
	100	+ */
	101	+ if (tail + 1 >= (struct compat_frame_tail __user *)
	102	+ compat_ptr(buftail.fp))
	103	+ return NULL;
	104	+
	105	+ return (struct compat_frame_tail __user *)compat_ptr(buftail.fp) - 1;
	106	+}
	107	+#endif /* CONFIG_COMPAT */
	108	+
	109	+void perf_callchain_user(struct perf_callchain_entry *entry,
	110	+ struct pt_regs *regs)
	111	+{
	112	+ if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
	113	+ /* We don't support guest os callchain now */
	114	+ return;
	115	+ }
	116	+
	117	+ perf_callchain_store(entry, regs->pc);
	118	+
	119	+ if (!compat_user_mode(regs)) {
	120	+ /* AARCH64 mode */
	121	+ struct frame_tail __user *tail;
	122	+
	123	+ tail = (struct frame_tail __user *)regs->regs[29];
	124	+
	125	+ while (entry->nr < PERF_MAX_STACK_DEPTH &&
	126	+ tail && !((unsigned long)tail & 0xf))
	127	+ tail = user_backtrace(tail, entry);
	128	+ } else {
	129	+#ifdef CONFIG_COMPAT
	130	+ /* AARCH32 compat mode */
	131	+ struct compat_frame_tail __user *tail;
	132	+
	133	+ tail = (struct compat_frame_tail __user *)regs->compat_fp - 1;
	134	+
	135	+ while ((entry->nr < PERF_MAX_STACK_DEPTH) &&
	136	+ tail && !((unsigned long)tail & 0x3))
	137	+ tail = compat_user_backtrace(tail, entry);
	138	+#endif
	139	+ }
	140	+}
	141	+
	142	+/*
	143	+ * Gets called by walk_stackframe() for every stackframe. This will be called
	144	+ * whist unwinding the stackframe and is like a subroutine return so we use
	145	+ * the PC.
	146	+ */
	147	+static int callchain_trace(struct stackframe frame, void data)
	148	+{
	149	+ struct perf_callchain_entry *entry = data;
	150	+ perf_callchain_store(entry, frame->pc);
	151	+ return 0;
	152	+}
	153	+
	154	+void perf_callchain_kernel(struct perf_callchain_entry *entry,
	155	+ struct pt_regs *regs)
	156	+{
	157	+ struct stackframe frame;
	158	+
	159	+ if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
	160	+ /* We don't support guest os callchain now */
	161	+ return;
	162	+ }
	163	+
	164	+ frame.fp = regs->regs[29];
	165	+ frame.sp = regs->sp;
	166	+ frame.pc = regs->pc;
	167	+
	168	+ walk_stackframe(&frame, callchain_trace, entry);
	169	+}
	170	+
	171	+unsigned long perf_instruction_pointer(struct pt_regs *regs)
	172	+{
	173	+ if (perf_guest_cbs && perf_guest_cbs->is_in_guest())
	174	+ return perf_guest_cbs->get_guest_ip();
	175	+
	176	+ return instruction_pointer(regs);
	177	+}
	178	+
	179	+unsigned long perf_misc_flags(struct pt_regs *regs)
	180	+{
	181	+ int misc = 0;
	182	+
	183	+ if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
	184	+ if (perf_guest_cbs->is_user_mode())
	185	+ misc \|= PERF_RECORD_MISC_GUEST_USER;
	186	+ else
	187	+ misc \|= PERF_RECORD_MISC_GUEST_KERNEL;
	188	+ } else {
	189	+ if (user_mode(regs))
	190	+ misc \|= PERF_RECORD_MISC_USER;
	191	+ else
	192	+ misc \|= PERF_RECORD_MISC_KERNEL;
	193	+ }
	194	+
	195	+ return misc;
	196	+}

arch/arm64/kernel/perf_event.c+25−285

@@ -25,7 +25,7 @@ · 24 unmodified lines
25	25	#include <linux/irq.h>
26	26	#include <linux/kernel.h>
27	27	#include <linux/export.h>
28		−#include <linux/of.h>
	28	+#include <linux/of_device.h>
29	29	#include <linux/perf_event.h>
30	30	#include <linux/platform_device.h>
31	31	#include <linux/slab.h>
@@ -36,7 +36,6 @@ · 4 unmodified lines
36	36	#include <asm/irq.h>
37	37	#include <asm/irq_regs.h>
38	38	#include <asm/pmu.h>
39		−#include <asm/stacktrace.h>
40	39
41	40	/*
42	41	* ARMv8 supports a maximum of 32 events.
@@ -78,6 +77,16 @@ EXPORT_SYMBOL_GPL(perf_num_counters); · 35 unmodified lines
78	77
79	78	#define CACHE_OP_UNSUPPORTED 0xFFFF
80	79
	80	+#define PERF_MAP_ALL_UNSUPPORTED \
	81	+ [0 ... PERF_COUNT_HW_MAX - 1] = HW_OP_UNSUPPORTED
	82	+
	83	+#define PERF_CACHE_MAP_ALL_UNSUPPORTED \
	84	+[0 ... C(MAX) - 1] = { \
	85	+ [0 ... C(OP_MAX) - 1] = { \
	86	+ [0 ... C(RESULT_MAX) - 1] = CACHE_OP_UNSUPPORTED, \
	87	+ }, \
	88	+}
	89	+
81	90	static int
82	91	armpmu_map_cache_event(const unsigned (*cache_map)
83	92	[PERF_COUNT_HW_CACHE_MAX]
@@ -435,10 +444,8 @@ armpmu_reserve_hardware(struct arm_pmu *armpmu) · 351 unmodified lines
435	444	unsigned int i, irqs;
436	445	struct platform_device *pmu_device = armpmu->plat_device;
437	446
438		− if (!pmu_device) {
439		− pr_err("no PMU device registered\n");
	447	+ if (!pmu_device)
440	448	return -ENODEV;
441		− }
442	449
443	450	irqs = min(pmu_device->num_resources, num_possible_cpus());
444	451	if (!irqs) {
@@ -703,118 +710,28 @@ enum armv8_pmuv3_perf_types { · 258 unmodified lines
703	710
704	711	/* PMUv3 HW events mapping. */
705	712	static const unsigned armv8_pmuv3_perf_map[PERF_COUNT_HW_MAX] = {
	713	+ PERF_MAP_ALL_UNSUPPORTED,
706	714	[PERF_COUNT_HW_CPU_CYCLES] = ARMV8_PMUV3_PERFCTR_CLOCK_CYCLES,
707	715	[PERF_COUNT_HW_INSTRUCTIONS] = ARMV8_PMUV3_PERFCTR_INSTR_EXECUTED,
708	716	[PERF_COUNT_HW_CACHE_REFERENCES] = ARMV8_PMUV3_PERFCTR_L1_DCACHE_ACCESS,
709	717	[PERF_COUNT_HW_CACHE_MISSES] = ARMV8_PMUV3_PERFCTR_L1_DCACHE_REFILL,
710		− [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = HW_OP_UNSUPPORTED,
711	718	[PERF_COUNT_HW_BRANCH_MISSES] = ARMV8_PMUV3_PERFCTR_PC_BRANCH_MIS_PRED,
712		− [PERF_COUNT_HW_BUS_CYCLES] = HW_OP_UNSUPPORTED,
713		− [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = HW_OP_UNSUPPORTED,
714		− [PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = HW_OP_UNSUPPORTED,
715	719	};
716	720
717	721	static const unsigned armv8_pmuv3_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
718	722	[PERF_COUNT_HW_CACHE_OP_MAX]
719	723	[PERF_COUNT_HW_CACHE_RESULT_MAX] = {
720		− [C(L1D)] = {
721		− [C(OP_READ)] = {
722		− [C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_L1_DCACHE_ACCESS,
723		− [C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L1_DCACHE_REFILL,
724		− },
725		− [C(OP_WRITE)] = {
726		− [C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_L1_DCACHE_ACCESS,
727		− [C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L1_DCACHE_REFILL,
728		− },
729		− [C(OP_PREFETCH)] = {
730		− [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
731		− [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
732		− },
733		− },
734		− [C(L1I)] = {
735		− [C(OP_READ)] = {
736		− [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
737		− [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
738		− },
739		− [C(OP_WRITE)] = {
740		− [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
741		− [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
742		− },
743		− [C(OP_PREFETCH)] = {
744		− [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
745		− [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
746		− },
747		− },
748		− [C(LL)] = {
749		− [C(OP_READ)] = {
750		− [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
751		− [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
752		− },
753		− [C(OP_WRITE)] = {
754		− [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
755		− [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
756		− },
757		− [C(OP_PREFETCH)] = {
758		− [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
759		− [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
760		− },
761		− },
762		− [C(DTLB)] = {
763		− [C(OP_READ)] = {
764		− [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
765		− [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
766		− },
767		− [C(OP_WRITE)] = {
768		− [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
769		− [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
770		− },
771		− [C(OP_PREFETCH)] = {
772		− [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
773		− [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
774		− },
775		− },
776		− [C(ITLB)] = {
777		− [C(OP_READ)] = {
778		− [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
779		− [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
780		− },
781		− [C(OP_WRITE)] = {
782		− [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
783		− [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
784		− },
785		− [C(OP_PREFETCH)] = {
786		− [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
787		− [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
788		− },
789		− },
790		− [C(BPU)] = {
791		− [C(OP_READ)] = {
792		− [C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_PC_BRANCH_PRED,
793		− [C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_PC_BRANCH_MIS_PRED,
794		− },
795		− [C(OP_WRITE)] = {
796		− [C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_PC_BRANCH_PRED,
797		− [C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_PC_BRANCH_MIS_PRED,
798		− },
799		− [C(OP_PREFETCH)] = {
800		− [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
801		− [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
802		− },
803		− },
804		− [C(NODE)] = {
805		− [C(OP_READ)] = {
806		− [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
807		− [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
808		− },
809		− [C(OP_WRITE)] = {
810		− [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
811		− [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
812		− },
813		− [C(OP_PREFETCH)] = {
814		− [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
815		− [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
816		− },
817		− },
	724	+ PERF_CACHE_MAP_ALL_UNSUPPORTED,
	725	+
	726	+ [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_L1_DCACHE_ACCESS,
	727	+ [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L1_DCACHE_REFILL,
	728	+ [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_L1_DCACHE_ACCESS,
	729	+ [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L1_DCACHE_REFILL,
	730	+
	731	+ [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_PC_BRANCH_PRED,
	732	+ [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_PC_BRANCH_MIS_PRED,
	733	+ [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_PC_BRANCH_PRED,
	734	+ [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_PC_BRANCH_MIS_PRED,
818	735	};
819	736
820	737	/*
@@ -1337,7 +1254,7 @@ static int armpmu_device_probe(struct platform_device *pdev) · 516 unmodified lines
1337	1254	}
1338	1255
1339	1256	for_each_possible_cpu(cpu)
1340		− if (arch_find_n_match_cpu_physical_id(dn, cpu, NULL))
	1257	+ if (dn == of_cpu_device_node_get(cpu))
1341	1258	break;
1342	1259
1343	1260	if (cpu >= nr_cpu_ids) {
@@ -1415,180 +1332,3 @@ static int __init init_hw_perf_events(void) · 71 unmodified lines
1415	1332	}
1416	1333	early_initcall(init_hw_perf_events);
1417	1334
1418		−/*
1419		− * Callchain handling code.
1420		− */
1421		−struct frame_tail {
1422		− struct frame_tail __user *fp;
1423		− unsigned long lr;
1424		−} __attribute__((packed));
1425		−
1426		−/*
1427		− * Get the return address for a single stackframe and return a pointer to the
1428		− * next frame tail.
1429		− */
1430		−static struct frame_tail __user *
1431		−user_backtrace(struct frame_tail __user *tail,
1432		− struct perf_callchain_entry *entry)
1433		−{
1434		− struct frame_tail buftail;
1435		− unsigned long err;
1436		−
1437		− /* Also check accessibility of one struct frame_tail beyond */
1438		− if (!access_ok(VERIFY_READ, tail, sizeof(buftail)))
1439		− return NULL;
1440		−
1441		− pagefault_disable();
1442		− err = __copy_from_user_inatomic(&buftail, tail, sizeof(buftail));
1443		− pagefault_enable();
1444		−
1445		− if (err)
1446		− return NULL;
1447		−
1448		− perf_callchain_store(entry, buftail.lr);
1449		−
1450		− /*
1451		− * Frame pointers should strictly progress back up the stack
1452		− * (towards higher addresses).
1453		− */
1454		− if (tail >= buftail.fp)
1455		− return NULL;
1456		−
1457		− return buftail.fp;
1458		−}
1459		−
1460		−#ifdef CONFIG_COMPAT
1461		−/*
1462		− * The registers we're interested in are at the end of the variable
1463		− * length saved register structure. The fp points at the end of this
1464		− * structure so the address of this struct is:
1465		− * (struct compat_frame_tail *)(xxx->fp)-1
1466		− *
1467		− * This code has been adapted from the ARM OProfile support.
1468		− */
1469		−struct compat_frame_tail {
1470		− compat_uptr_t fp; /* a (struct compat_frame_tail ) in compat mode /
1471		− u32 sp;
1472		− u32 lr;
1473		−} __attribute__((packed));
1474		−
1475		−static struct compat_frame_tail __user *
1476		−compat_user_backtrace(struct compat_frame_tail __user *tail,
1477		− struct perf_callchain_entry *entry)
1478		−{
1479		− struct compat_frame_tail buftail;
1480		− unsigned long err;
1481		−
1482		− /* Also check accessibility of one struct frame_tail beyond */
1483		− if (!access_ok(VERIFY_READ, tail, sizeof(buftail)))
1484		− return NULL;
1485		−
1486		− pagefault_disable();
1487		− err = __copy_from_user_inatomic(&buftail, tail, sizeof(buftail));
1488		− pagefault_enable();
1489		−
1490		− if (err)
1491		− return NULL;
1492		−
1493		− perf_callchain_store(entry, buftail.lr);
1494		−
1495		− /*
1496		− * Frame pointers should strictly progress back up the stack
1497		− * (towards higher addresses).
1498		− */
1499		− if (tail + 1 >= (struct compat_frame_tail __user *)
1500		− compat_ptr(buftail.fp))
1501		− return NULL;
1502		−
1503		− return (struct compat_frame_tail __user *)compat_ptr(buftail.fp) - 1;
1504		−}
1505		−#endif /* CONFIG_COMPAT */
1506		−
1507		−void perf_callchain_user(struct perf_callchain_entry *entry,
1508		− struct pt_regs *regs)
1509		−{
1510		− if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
1511		− /* We don't support guest os callchain now */
1512		− return;
1513		− }
1514		−
1515		− perf_callchain_store(entry, regs->pc);
1516		−
1517		− if (!compat_user_mode(regs)) {
1518		− /* AARCH64 mode */
1519		− struct frame_tail __user *tail;
1520		−
1521		− tail = (struct frame_tail __user *)regs->regs[29];
1522		−
1523		− while (entry->nr < PERF_MAX_STACK_DEPTH &&
1524		− tail && !((unsigned long)tail & 0xf))
1525		− tail = user_backtrace(tail, entry);
1526		− } else {
1527		−#ifdef CONFIG_COMPAT
1528		− /* AARCH32 compat mode */
1529		− struct compat_frame_tail __user *tail;
1530		−
1531		− tail = (struct compat_frame_tail __user *)regs->compat_fp - 1;
1532		−
1533		− while ((entry->nr < PERF_MAX_STACK_DEPTH) &&
1534		− tail && !((unsigned long)tail & 0x3))
1535		− tail = compat_user_backtrace(tail, entry);
1536		−#endif
1537		− }
1538		−}
1539		−
1540		−/*
1541		− * Gets called by walk_stackframe() for every stackframe. This will be called
1542		− * whist unwinding the stackframe and is like a subroutine return so we use
1543		− * the PC.
1544		− */
1545		−static int callchain_trace(struct stackframe frame, void data)
1546		−{
1547		− struct perf_callchain_entry *entry = data;
1548		− perf_callchain_store(entry, frame->pc);
1549		− return 0;
1550		−}
1551		−
1552		−void perf_callchain_kernel(struct perf_callchain_entry *entry,
1553		− struct pt_regs *regs)
1554		−{
1555		− struct stackframe frame;
1556		−
1557		− if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
1558		− /* We don't support guest os callchain now */
1559		− return;
1560		− }
1561		−
1562		− frame.fp = regs->regs[29];
1563		− frame.sp = regs->sp;
1564		− frame.pc = regs->pc;
1565		−
1566		− walk_stackframe(&frame, callchain_trace, entry);
1567		−}
1568		−
1569		−unsigned long perf_instruction_pointer(struct pt_regs *regs)
1570		−{
1571		− if (perf_guest_cbs && perf_guest_cbs->is_in_guest())
1572		− return perf_guest_cbs->get_guest_ip();
1573		−
1574		− return instruction_pointer(regs);
1575		−}
1576		−
1577		−unsigned long perf_misc_flags(struct pt_regs *regs)
1578		−{
1579		− int misc = 0;
1580		−
1581		− if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
1582		− if (perf_guest_cbs->is_user_mode())
1583		− misc \|= PERF_RECORD_MISC_GUEST_USER;
1584		− else
1585		− misc \|= PERF_RECORD_MISC_GUEST_KERNEL;
1586		− } else {
1587		− if (user_mode(regs))
1588		− misc \|= PERF_RECORD_MISC_USER;
1589		− else
1590		− misc \|= PERF_RECORD_MISC_KERNEL;
1591		− }
1592		−
1593		− return misc;
1594		−}

arch/arm64/kernel/psci.c+0−5

@@ -110,8 +110,6 @@ free_mem: · 109 unmodified lines
110	110	return ret;
111	111	}
112	112
113		−#ifdef CONFIG_SMP
114		−
115	113	static int __init cpu_psci_cpu_init(unsigned int cpu)
116	114	{
117	115	return 0;
@@ -193,7 +191,6 @@ static int cpu_psci_cpu_kill(unsigned int cpu) · 75 unmodified lines
193	191	return -ETIMEDOUT;
194	192	}
195	193	#endif
196		−#endif
197	194
198	195	static int psci_suspend_finisher(unsigned long index)
199	196	{
@@ -228,7 +225,6 @@ const struct cpu_operations cpu_psci_ops = { · 28 unmodified lines
228	225	.cpu_init_idle = cpu_psci_cpu_init_idle,
229	226	.cpu_suspend = cpu_psci_cpu_suspend,
230	227	#endif
231		−#ifdef CONFIG_SMP
232	228	.cpu_init = cpu_psci_cpu_init,
233	229	.cpu_prepare = cpu_psci_cpu_prepare,
234	230	.cpu_boot = cpu_psci_cpu_boot,
@@ -237,6 +233,5 @@ const struct cpu_operations cpu_psci_ops = { · 2 unmodified lines
237	233	.cpu_die = cpu_psci_cpu_die,
238	234	.cpu_kill = cpu_psci_cpu_kill,
239	235	#endif
240		−#endif
241	236	};
242	237

arch/arm64/kernel/ptrace.c+91−1

@@ -826,6 +826,30 @@ static int compat_vfp_set(struct task_struct *target, · 825 unmodified lines
826	826	return ret;
827	827	}
828	828
	829	+static int compat_tls_get(struct task_struct *target,
	830	+ const struct user_regset *regset, unsigned int pos,
	831	+ unsigned int count, void kbuf, void __user ubuf)
	832	+{
	833	+ compat_ulong_t tls = (compat_ulong_t)target->thread.tp_value;
	834	+ return user_regset_copyout(&pos, &count, &kbuf, &ubuf, &tls, 0, -1);
	835	+}
	836	+
	837	+static int compat_tls_set(struct task_struct *target,
	838	+ const struct user_regset *regset, unsigned int pos,
	839	+ unsigned int count, const void *kbuf,
	840	+ const void __user *ubuf)
	841	+{
	842	+ int ret;
	843	+ compat_ulong_t tls;
	844	+
	845	+ ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &tls, 0, -1);
	846	+ if (ret)
	847	+ return ret;
	848	+
	849	+ target->thread.tp_value = tls;
	850	+ return ret;
	851	+}
	852	+
829	853	static const struct user_regset aarch32_regsets[] = {
830	854	[REGSET_COMPAT_GPR] = {
831	855	.core_note_type = NT_PRSTATUS,
@@ -850,6 +874,64 @@ static const struct user_regset_view user_aarch32_view = { · 18 unmodified lines
850	874	.regsets = aarch32_regsets, .n = ARRAY_SIZE(aarch32_regsets)
851	875	};
852	876
	877	+static const struct user_regset aarch32_ptrace_regsets[] = {
	878	+ [REGSET_GPR] = {
	879	+ .core_note_type = NT_PRSTATUS,
	880	+ .n = COMPAT_ELF_NGREG,
	881	+ .size = sizeof(compat_elf_greg_t),
	882	+ .align = sizeof(compat_elf_greg_t),
	883	+ .get = compat_gpr_get,
	884	+ .set = compat_gpr_set
	885	+ },
	886	+ [REGSET_FPR] = {
	887	+ .core_note_type = NT_ARM_VFP,
	888	+ .n = VFP_STATE_SIZE / sizeof(compat_ulong_t),
	889	+ .size = sizeof(compat_ulong_t),
	890	+ .align = sizeof(compat_ulong_t),
	891	+ .get = compat_vfp_get,
	892	+ .set = compat_vfp_set
	893	+ },
	894	+ [REGSET_TLS] = {
	895	+ .core_note_type = NT_ARM_TLS,
	896	+ .n = 1,
	897	+ .size = sizeof(compat_ulong_t),
	898	+ .align = sizeof(compat_ulong_t),
	899	+ .get = compat_tls_get,
	900	+ .set = compat_tls_set,
	901	+ },
	902	+#ifdef CONFIG_HAVE_HW_BREAKPOINT
	903	+ [REGSET_HW_BREAK] = {
	904	+ .core_note_type = NT_ARM_HW_BREAK,
	905	+ .n = sizeof(struct user_hwdebug_state) / sizeof(u32),
	906	+ .size = sizeof(u32),
	907	+ .align = sizeof(u32),
	908	+ .get = hw_break_get,
	909	+ .set = hw_break_set,
	910	+ },
	911	+ [REGSET_HW_WATCH] = {
	912	+ .core_note_type = NT_ARM_HW_WATCH,
	913	+ .n = sizeof(struct user_hwdebug_state) / sizeof(u32),
	914	+ .size = sizeof(u32),
	915	+ .align = sizeof(u32),
	916	+ .get = hw_break_get,
	917	+ .set = hw_break_set,
	918	+ },
	919	+#endif
	920	+ [REGSET_SYSTEM_CALL] = {
	921	+ .core_note_type = NT_ARM_SYSTEM_CALL,
	922	+ .n = 1,
	923	+ .size = sizeof(int),
	924	+ .align = sizeof(int),
	925	+ .get = system_call_get,
	926	+ .set = system_call_set,
	927	+ },
	928	+};
	929	+
	930	+static const struct user_regset_view user_aarch32_ptrace_view = {
	931	+ .name = "aarch32", .e_machine = EM_ARM,
	932	+ .regsets = aarch32_ptrace_regsets, .n = ARRAY_SIZE(aarch32_ptrace_regsets)
	933	+};
	934	+
853	935	static int compat_ptrace_read_user(struct task_struct *tsk, compat_ulong_t off,
854	936	compat_ulong_t __user *ret)
855	937	{
@@ -1109,8 +1191,16 @@ long compat_arch_ptrace(struct task_struct *child, compat_long_t request, · 253 unmodified lines
1109	1191	const struct user_regset_view task_user_regset_view(struct task_struct task)
1110	1192	{
1111	1193	#ifdef CONFIG_COMPAT
1112		− if (is_compat_thread(task_thread_info(task)))
	1194	+ /*
	1195	+ * Core dumping of 32-bit tasks or compat ptrace requests must use the
	1196	+ * user_aarch32_view compatible with arm32. Native ptrace requests on
	1197	+ * 32-bit children use an extended user_aarch32_ptrace_view to allow
	1198	+ * access to the TLS register.
	1199	+ */
	1200	+ if (is_compat_task())
1113	1201	return &user_aarch32_view;
	1202	+ else if (is_compat_thread(task_thread_info(task)))
	1203	+ return &user_aarch32_ptrace_view;
1114	1204	#endif
1115	1205	return &user_aarch64_view;
1116	1206	}

arch/arm64/kernel/setup.c+34−51

@@ -62,7 +62,6 @@ · 61 unmodified lines
62	62	#include <asm/traps.h>
63	63	#include <asm/memblock.h>
64	64	#include <asm/efi.h>
65		−#include <asm/virt.h>
66	65	#include <asm/xen/hypervisor.h>
67	66
68	67	unsigned long elf_hwcap __read_mostly;
@@ -130,7 +129,6 @@ bool arch_match_cpu_phys_id(int cpu, u64 phys_id) · 61 unmodified lines
130	129	}
131	130
132	131	struct mpidr_hash mpidr_hash;
133		−#ifdef CONFIG_SMP
134	132	/**
135	133	* smp_build_mpidr_hash - Pre-compute shifts required at each affinity
136	134	* level in order to build a linear index from an
@@ -196,35 +194,11 @@ static void __init smp_build_mpidr_hash(void) · 59 unmodified lines
196	194	pr_warn("Large number of MPIDR hash buckets detected\n");
197	195	__flush_dcache_area(&mpidr_hash, sizeof(struct mpidr_hash));
198	196	}
199		−#endif
200		−
201		−static void __init hyp_mode_check(void)
202		−{
203		− if (is_hyp_mode_available())
204		− pr_info("CPU: All CPU(s) started at EL2\n");
205		− else if (is_hyp_mode_mismatched())
206		− WARN_TAINT(1, TAINT_CPU_OUT_OF_SPEC,
207		− "CPU: CPUs started in inconsistent modes");
208		− else
209		− pr_info("CPU: All CPU(s) started at EL1\n");
210		−}
211		−
212		−void __init do_post_cpus_up_work(void)
213		−{
214		− hyp_mode_check();
215		− apply_alternatives_all();
216		−}
217		−
218		−#ifdef CONFIG_UP_LATE_INIT
219		−void __init up_late_init(void)
220		−{
221		− do_post_cpus_up_work();
222		−}
223		−#endif /* CONFIG_UP_LATE_INIT */
224	197
225	198	static void __init setup_processor(void)
226	199	{
227		− u64 features, block;
	200	+ u64 features;
	201	+ s64 block;
228	202	u32 cwg;
229	203	int cls;
230	204
@@ -254,8 +228,8 @@ static void __init setup_processor(void) · 23 unmodified lines
254	228	* for non-negative values. Negative values are reserved.
255	229	*/
256	230	features = read_cpuid(ID_AA64ISAR0_EL1);
257		− block = (features >> 4) & 0xf;
258		− if (!(block & 0x8)) {
	231	+ block = cpuid_feature_extract_field(features, 4);
	232	+ if (block > 0) {
259	233	switch (block) {
260	234	default:
261	235	case 2:
@@ -267,26 +241,36 @@ static void __init setup_processor(void) · 5 unmodified lines
267	241	}
268	242	}
269	243
270		− block = (features >> 8) & 0xf;
271		− if (block && !(block & 0x8))
	244	+ if (cpuid_feature_extract_field(features, 8) > 0)
272	245	elf_hwcap \|= HWCAP_SHA1;
273	246
274		− block = (features >> 12) & 0xf;
275		− if (block && !(block & 0x8))
	247	+ if (cpuid_feature_extract_field(features, 12) > 0)
276	248	elf_hwcap \|= HWCAP_SHA2;
277	249
278		− block = (features >> 16) & 0xf;
279		− if (block && !(block & 0x8))
	250	+ if (cpuid_feature_extract_field(features, 16) > 0)
280	251	elf_hwcap \|= HWCAP_CRC32;
281	252
	253	+ block = cpuid_feature_extract_field(features, 20);
	254	+ if (block > 0) {
	255	+ switch (block) {
	256	+ default:
	257	+ case 2:
	258	+ elf_hwcap \|= HWCAP_ATOMICS;
	259	+ case 1:
	260	+ /* RESERVED */
	261	+ case 0:
	262	+ break;
	263	+ }
	264	+ }
	265	+
282	266	#ifdef CONFIG_COMPAT
283	267	/*
284	268	* ID_ISAR5_EL1 carries similar information as above, but pertaining to
285		− * the Aarch32 32-bit execution state.
	269	+ * the AArch32 32-bit execution state.
286	270	*/
287	271	features = read_cpuid(ID_ISAR5_EL1);
288		− block = (features >> 4) & 0xf;
289		− if (!(block & 0x8)) {
	272	+ block = cpuid_feature_extract_field(features, 4);
	273	+ if (block > 0) {
290	274	switch (block) {
291	275	default:
292	276	case 2:
@@ -298,16 +282,13 @@ static void __init setup_processor(void) · 5 unmodified lines
298	282	}
299	283	}
300	284
301		− block = (features >> 8) & 0xf;
302		− if (block && !(block & 0x8))
	285	+ if (cpuid_feature_extract_field(features, 8) > 0)
303	286	compat_elf_hwcap2 \|= COMPAT_HWCAP2_SHA1;
304	287
305		− block = (features >> 12) & 0xf;
306		− if (block && !(block & 0x8))
	288	+ if (cpuid_feature_extract_field(features, 12) > 0)
307	289	compat_elf_hwcap2 \|= COMPAT_HWCAP2_SHA2;
308	290
309		− block = (features >> 16) & 0xf;
310		− if (block && !(block & 0x8))
	291	+ if (cpuid_feature_extract_field(features, 16) > 0)
311	292	compat_elf_hwcap2 \|= COMPAT_HWCAP2_CRC32;
312	293	#endif
313	294	}
@@ -404,10 +385,8 @@ void __init setup_arch(char **cmdline_p) · 90 unmodified lines
404	385	xen_early_init();
405	386
406	387	cpu_read_bootcpu_ops();
407		−#ifdef CONFIG_SMP
408	388	smp_init_cpus();
409	389	smp_build_mpidr_hash();
410		−#endif
411	390
412	391	#ifdef CONFIG_VT
413	392	#if defined(CONFIG_VGA_CONSOLE)
@@ -426,8 +405,13 @@ void __init setup_arch(char **cmdline_p) · 12 unmodified lines
426	405
427	406	static int __init arm64_device_init(void)
428	407	{
429		− of_iommu_init();
430		− of_platform_populate(NULL, of_default_bus_match_table, NULL, NULL);
	408	+ if (of_have_populated_dt()) {
	409	+ of_iommu_init();
	410	+ of_platform_populate(NULL, of_default_bus_match_table,
	411	+ NULL, NULL);
	412	+ } else if (acpi_disabled) {
	413	+ pr_crit("Device tree not populated\n");
	414	+ }
431	415	return 0;
432	416	}
433	417	arch_initcall_sync(arm64_device_init);
@@ -455,6 +439,7 @@ static const char *hwcap_str[] = { · 21 unmodified lines
455	439	"sha1",
456	440	"sha2",
457	441	"crc32",
	442	+ "atomics",
458	443	NULL
459	444	};
460	445
@@ -507,9 +492,7 @@ static int c_show(struct seq_file m, void v) · 46 unmodified lines
507	492	* online processors, looking for lines beginning with
508	493	* "processor". Give glibc what it expects.
509	494	*/
510		−#ifdef CONFIG_SMP
511	495	seq_printf(m, "processor\t: %d\n", i);
512		−#endif
513	496
514	497	/*
515	498	* Dump out the common processor features in a single line.

arch/arm64/kernel/sleep.S+8−6

@@ -82,7 +82,6 @@ ENTRY(__cpu_suspend_enter) · 81 unmodified lines
82	82	str x2, [x0, #CPU_CTX_SP]
83	83	ldr x1, =sleep_save_sp
84	84	ldr x1, [x1, #SLEEP_SAVE_SP_VIRT]
85		−#ifdef CONFIG_SMP
86	85	mrs x7, mpidr_el1
87	86	ldr x9, =mpidr_hash
88	87	ldr x10, [x9, #MPIDR_HASH_MASK]
@@ -94,7 +93,6 @@ ENTRY(__cpu_suspend_enter) · 5 unmodified lines
94	93	ldp w5, w6, [x9, #(MPIDR_HASH_SHIFTS + 8)]
95	94	compute_mpidr_hash x8, x3, x4, x5, x6, x7, x10
96	95	add x1, x1, x8, lsl #3
97		−#endif
98	96	bl __cpu_suspend_save
99	97	/*
100	98	* Grab suspend finisher in x20 and its argument in x19
@@ -135,6 +133,14 @@ ENTRY(cpu_resume_mmu) · 34 unmodified lines
135	133	ldr x3, =cpu_resume_after_mmu
136	134	msr sctlr_el1, x0 // restore sctlr_el1
137	135	isb
	136	+ /*
	137	+ * Invalidate the local I-cache so that any instructions fetched
	138	+ * speculatively from the PoC are discarded, since they may have
	139	+ * been dynamically patched at the PoU.
	140	+ */
	141	+ ic iallu
	142	+ dsb nsh
	143	+ isb
138	144	br x3 // global jump to virtual address
139	145	ENDPROC(cpu_resume_mmu)
140	146	.popsection
@@ -151,7 +157,6 @@ ENDPROC(cpu_resume_after_mmu) · 10 unmodified lines
151	157
152	158	ENTRY(cpu_resume)
153	159	bl el2_setup // if in EL2 drop to EL1 cleanly
154		−#ifdef CONFIG_SMP
155	160	mrs x1, mpidr_el1
156	161	adrp x8, mpidr_hash
157	162	add x8, x8, #:lo12:mpidr_hash // x8 = struct mpidr_hash phys address
@@ -161,9 +166,6 @@ ENTRY(cpu_resume) · 3 unmodified lines
161	166	ldp w5, w6, [x8, #(MPIDR_HASH_SHIFTS + 8)]
162	167	compute_mpidr_hash x7, x3, x4, x5, x6, x1, x2
163	168	/* x7 contains hash index, let's use it to grab context pointer */
164		−#else
165		− mov x7, xzr
166		−#endif
167	169	ldr_l x0, sleep_save_sp + SLEEP_SAVE_SP_PHYS
168	170	ldr x0, [x0, x7, lsl #3]
169	171	/* load sp from context */

arch/arm64/kernel/smp.c+14−1

@@ -52,6 +52,7 @@ · 51 unmodified lines
52	52	#include <asm/sections.h>
53	53	#include <asm/tlbflush.h>
54	54	#include <asm/ptrace.h>
	55	+#include <asm/virt.h>
55	56
56	57	#define CREATE_TRACE_POINTS
57	58	#include <trace/events/ipi.h>
@@ -310,10 +311,22 @@ void cpu_die(void) · 252 unmodified lines
310	311	}
311	312	#endif
312	313
	314	+static void __init hyp_mode_check(void)
	315	+{
	316	+ if (is_hyp_mode_available())
	317	+ pr_info("CPU: All CPU(s) started at EL2\n");
	318	+ else if (is_hyp_mode_mismatched())
	319	+ WARN_TAINT(1, TAINT_CPU_OUT_OF_SPEC,
	320	+ "CPU: CPUs started in inconsistent modes");
	321	+ else
	322	+ pr_info("CPU: All CPU(s) started at EL1\n");
	323	+}
	324	+
313	325	void __init smp_cpus_done(unsigned int max_cpus)
314	326	{
315	327	pr_info("SMP: Total of %d processors activated.\n", num_online_cpus());
316		− do_post_cpus_up_work();
	328	+ hyp_mode_check();
	329	+ apply_alternatives_all();
317	330	}
318	331
319	332	void __init smp_prepare_boot_cpu(void)

arch/arm64/kernel/time.c+0−2

@@ -42,7 +42,6 @@ · 41 unmodified lines
42	42	#include <asm/thread_info.h>
43	43	#include <asm/stacktrace.h>
44	44
45		−#ifdef CONFIG_SMP
46	45	unsigned long profile_pc(struct pt_regs *regs)
47	46	{
48	47	struct stackframe frame;
@@ -62,7 +61,6 @@ unsigned long profile_pc(struct pt_regs *regs) · 13 unmodified lines
62	61	return frame.pc;
63	62	}
64	63	EXPORT_SYMBOL(profile_pc);
65		−#endif
66	64
67	65	void __init time_init(void)
68	66	{

arch/arm64/kernel/topology.c+1−1

@@ -300,6 +300,6 @@ void __init init_cpu_topology(void) · 299 unmodified lines
300	300	* Discard anything that was parsed if we hit an error so we
301	301	* don't use partial information.
302	302	*/
303		− if (parse_dt_topology())
	303	+ if (of_have_populated_dt() && parse_dt_topology())
304	304	reset_cpu_topology();
305	305	}

arch/arm64/kernel/traps.c+80−14

@@ -17,6 +17,7 @@ · 16 unmodified lines
17	17	* along with this program. If not, see <http://www.gnu.org/licenses/>.
18	18	*/
19	19
	20	+#include <linux/bug.h>
20	21	#include <linux/signal.h>
21	22	#include <linux/personality.h>
22	23	#include <linux/kallsyms.h>
@@ -32,8 +33,10 @@ · 9 unmodified lines
32	33	#include <linux/syscalls.h>
33	34
34	35	#include <asm/atomic.h>
	36	+#include <asm/bug.h>
35	37	#include <asm/debug-monitors.h>
36	38	#include <asm/esr.h>
	39	+#include <asm/insn.h>
37	40	#include <asm/traps.h>
38	41	#include <asm/stacktrace.h>
39	42	#include <asm/exception.h>
@@ -52,11 +55,12 @@ int show_unhandled_signals = 1; · 12 unmodified lines
52	55	* Dump out the contents of some memory nicely...
53	56	*/
54	57	static void dump_mem(const char lvl, const char str, unsigned long bottom,
55		− unsigned long top)
	58	+ unsigned long top, bool compat)
56	59	{
57	60	unsigned long first;
58	61	mm_segment_t fs;
59	62	int i;
	63	+ unsigned int width = compat ? 4 : 8;
60	64
61	65	/*
62	66	* We need to switch to kernel mode so that we can use __get_user
@@ -75,13 +79,22 @@ static void dump_mem(const char lvl, const char str, unsigned long bottom, · 12 unmodified lines
75	79	memset(str, ' ', sizeof(str));
76	80	str[sizeof(str) - 1] = '\0';
77	81
78		− for (p = first, i = 0; i < 8 && p < top; i++, p += 4) {
	82	+ for (p = first, i = 0; i < (32 / width)
	83	+ && p < top; i++, p += width) {
79	84	if (p >= bottom && p < top) {
80		− unsigned int val;
81		− if (__get_user(val, (unsigned int *)p) == 0)
82		− sprintf(str + i * 9, " %08x", val);
83		− else
84		− sprintf(str + i * 9, " ????????");
	85	+ unsigned long val;
	86	+
	87	+ if (width == 8) {
	88	+ if (__get_user(val, (unsigned long *)p) == 0)
	89	+ sprintf(str + i * 17, " %016lx", val);
	90	+ else
	91	+ sprintf(str + i * 17, " ????????????????");
	92	+ } else {
	93	+ if (__get_user(val, (unsigned int *)p) == 0)
	94	+ sprintf(str + i * 9, " %08lx", val);
	95	+ else
	96	+ sprintf(str + i * 9, " ????????");
	97	+ }
85	98	}
86	99	}
87	100	printk("%s%04lx:%s\n", lvl, first & 0xffff, str);
@@ -95,7 +108,7 @@ static void dump_backtrace_entry(unsigned long where, unsigned long stack) · 7 unmodified lines
95	108	print_ip_sym(where);
96	109	if (in_exception_text(where))
97	110	dump_mem("", "Exception stack", stack,
98		− stack + sizeof(struct pt_regs));
	111	+ stack + sizeof(struct pt_regs), false);
99	112	}
100	113
101	114	static void dump_instr(const char lvl, struct pt_regs regs)
@@ -179,11 +192,7 @@ void show_stack(struct task_struct tsk, unsigned long sp) · 77 unmodified lines
179	192	#else
180	193	#define S_PREEMPT ""
181	194	#endif
182		−#ifdef CONFIG_SMP
183	195	#define S_SMP " SMP"
184		−#else
185		−#define S_SMP ""
186		−#endif
187	196
188	197	static int __die(const char str, int err, struct thread_info thread,
189	198	struct pt_regs *regs)
@@ -207,7 +216,8 @@ static int __die(const char str, int err, struct thread_info thread, · 17 unmodified lines
207	216
208	217	if (!user_mode(regs) \|\| in_interrupt()) {
209	218	dump_mem(KERN_EMERG, "Stack: ", regs->sp,
210		− THREAD_SIZE + (unsigned long)task_stack_page(tsk));
	219	+ THREAD_SIZE + (unsigned long)task_stack_page(tsk),
	220	+ compat_user_mode(regs));
211	221	dump_backtrace(regs, tsk);
212	222	dump_instr(KERN_EMERG, regs);
213	223	}
@@ -459,7 +469,63 @@ void __pgd_error(const char *file, int line, unsigned long val) · 245 unmodified lines
459	469	pr_crit("%s:%d: bad pgd %016lx.\n", file, line, val);
460	470	}
461	471
	472	+/* GENERIC_BUG traps */
	473	+
	474	+int is_valid_bugaddr(unsigned long addr)
	475	+{
	476	+ /*
	477	+ * bug_handler() only called for BRK #BUG_BRK_IMM.
	478	+ * So the answer is trivial -- any spurious instances with no
	479	+ * bug table entry will be rejected by report_bug() and passed
	480	+ * back to the debug-monitors code and handled as a fatal
	481	+ * unexpected debug exception.
	482	+ */
	483	+ return 1;
	484	+}
	485	+
	486	+static int bug_handler(struct pt_regs *regs, unsigned int esr)
	487	+{
	488	+ if (user_mode(regs))
	489	+ return DBG_HOOK_ERROR;
	490	+
	491	+ switch (report_bug(regs->pc, regs)) {
	492	+ case BUG_TRAP_TYPE_BUG:
	493	+ die("Oops - BUG", regs, 0);
	494	+ break;
	495	+
	496	+ case BUG_TRAP_TYPE_WARN:
	497	+ /* Ideally, report_bug() should backtrace for us... but no. */
	498	+ dump_backtrace(regs, NULL);
	499	+ break;
	500	+
	501	+ default:
	502	+ /* unknown/unrecognised bug trap type */
	503	+ return DBG_HOOK_ERROR;
	504	+ }
	505	+
	506	+ /* If thread survives, skip over the BUG instruction and continue: */
	507	+ regs->pc += AARCH64_INSN_SIZE; /* skip BRK and resume */
	508	+ return DBG_HOOK_HANDLED;
	509	+}
	510	+
	511	+static struct break_hook bug_break_hook = {
	512	+ .esr_val = 0xf2000000 \| BUG_BRK_IMM,
	513	+ .esr_mask = 0xffffffff,
	514	+ .fn = bug_handler,
	515	+};
	516	+
	517	+/*
	518	+ * Initial handler for AArch64 BRK exceptions
	519	+ * This handler only used until debug_traps_init().
	520	+ */
	521	+int __init early_brk64(unsigned long addr, unsigned int esr,
	522	+ struct pt_regs *regs)
	523	+{
	524	+ return bug_handler(regs, esr) != DBG_HOOK_HANDLED;
	525	+}
	526	+
	527	+/* This registration must happen early, before debug_traps_init(). */
462	528	void __init trap_init(void)
463	529	{
464		− return;
	530	+ register_break_hook(&bug_break_hook);
465	531	}

arch/arm64/kvm/hyp.S+10−2

@@ -810,7 +810,11 @@ · 809 unmodified lines
810	810	* Call into the vgic backend for state saving
811	811	*/
812	812	.macro save_vgic_state
813		− alternative_insn "bl __save_vgic_v2_state", "bl __save_vgic_v3_state", ARM64_HAS_SYSREG_GIC_CPUIF
	813	+alternative_if_not ARM64_HAS_SYSREG_GIC_CPUIF
	814	+ bl __save_vgic_v2_state
	815	+alternative_else
	816	+ bl __save_vgic_v3_state
	817	+alternative_endif
814	818	mrs x24, hcr_el2
815	819	mov x25, #HCR_INT_OVERRIDE
816	820	neg x25, x25
@@ -827,7 +831,11 @@ · 10 unmodified lines
827	831	orr x24, x24, #HCR_INT_OVERRIDE
828	832	orr x24, x24, x25
829	833	msr hcr_el2, x24
830		− alternative_insn "bl __restore_vgic_v2_state", "bl __restore_vgic_v3_state", ARM64_HAS_SYSREG_GIC_CPUIF
	834	+alternative_if_not ARM64_HAS_SYSREG_GIC_CPUIF
	835	+ bl __restore_vgic_v2_state
	836	+alternative_else
	837	+ bl __restore_vgic_v3_state
	838	+alternative_endif
831	839	.endm
832	840
833	841	.macro save_timer_state

arch/arm64/lib/Makefile+13−0

@@ -3,3 +3,16 @@ lib-y := bitops.o clear_user.o delay.o copy_from_user.o \ · 2 unmodified lines
3	3	clear_page.o memchr.o memcpy.o memmove.o memset.o \
4	4	memcmp.o strcmp.o strncmp.o strlen.o strnlen.o \
5	5	strchr.o strrchr.o
	6	+
	7	+# Tell the compiler to treat all general purpose registers as
	8	+# callee-saved, which allows for efficient runtime patching of the bl
	9	+# instruction in the caller with an atomic instruction when supported by
	10	+# the CPU. Result and argument registers are handled correctly, based on
	11	+# the function prototype.
	12	+lib-$(CONFIG_ARM64_LSE_ATOMICS) += atomic_ll_sc.o
	13	+CFLAGS_atomic_ll_sc.o := -fcall-used-x0 -ffixed-x1 -ffixed-x2 \
	14	+ -ffixed-x3 -ffixed-x4 -ffixed-x5 -ffixed-x6 \
	15	+ -ffixed-x7 -fcall-saved-x8 -fcall-saved-x9 \
	16	+ -fcall-saved-x10 -fcall-saved-x11 -fcall-saved-x12 \
	17	+ -fcall-saved-x13 -fcall-saved-x14 -fcall-saved-x15 \
	18	+ -fcall-saved-x16 -fcall-saved-x17 -fcall-saved-x18

arch/arm64/lib/atomic_ll_sc.c+3−0

@@ -0,0 +1,3 @@
	1	+#include <asm/atomic.h>
	2	+#define __ARM64_IN_ATOMIC_IMPL
	3	+#include <asm/atomic_ll_sc.h>

arch/arm64/lib/bitops.S+26−19

@@ -18,52 +18,59 @@ · 17 unmodified lines
18	18
19	19	#include <linux/linkage.h>
20	20	#include <asm/assembler.h>
	21	+#include <asm/lse.h>
21	22
22	23	/*
23	24	* x0: bits 5:0 bit offset
24	25	* bits 31:6 word offset
25	26	* x1: address
26	27	*/
27		− .macro bitop, name, instr
	28	+ .macro bitop, name, llsc, lse
28	29	ENTRY( \name )
29	30	and w3, w0, #63 // Get bit offset
30	31	eor w0, w0, w3 // Clear low bits
31	32	mov x2, #1
32	33	add x1, x1, x0, lsr #3 // Get word offset
	34	+alt_lse " prfm pstl1strm, [x1]", "nop"
33	35	lsl x3, x2, x3 // Create mask
34		−1: ldxr x2, [x1]
35		− \instr x2, x2, x3
36		− stxr w0, x2, [x1]
37		− cbnz w0, 1b
	36	+
	37	+alt_lse "1: ldxr x2, [x1]", "\lse x3, [x1]"
	38	+alt_lse " \llsc x2, x2, x3", "nop"
	39	+alt_lse " stxr w0, x2, [x1]", "nop"
	40	+alt_lse " cbnz w0, 1b", "nop"
	41	+
38	42	ret
39	43	ENDPROC(\name )
40	44	.endm
41	45
42		− .macro testop, name, instr
	46	+ .macro testop, name, llsc, lse
43	47	ENTRY( \name )
44	48	and w3, w0, #63 // Get bit offset
45	49	eor w0, w0, w3 // Clear low bits
46	50	mov x2, #1
47	51	add x1, x1, x0, lsr #3 // Get word offset
	52	+alt_lse " prfm pstl1strm, [x1]", "nop"
48	53	lsl x4, x2, x3 // Create mask
49		−1: ldxr x2, [x1]
50		− lsr x0, x2, x3 // Save old value of bit
51		− \instr x2, x2, x4 // toggle bit
52		− stlxr w5, x2, [x1]
53		− cbnz w5, 1b
54		− dmb ish
	54	+
	55	+alt_lse "1: ldxr x2, [x1]", "\lse x4, x2, [x1]"
	56	+ lsr x0, x2, x3
	57	+alt_lse " \llsc x2, x2, x4", "nop"
	58	+alt_lse " stlxr w5, x2, [x1]", "nop"
	59	+alt_lse " cbnz w5, 1b", "nop"
	60	+alt_lse " dmb ish", "nop"
	61	+
55	62	and x0, x0, #1
56		−3: ret
	63	+ ret
57	64	ENDPROC(\name )
58	65	.endm
59	66
60	67	/*
61	68	* Atomic bit operations.
62	69	*/
63		− bitop change_bit, eor
64		− bitop clear_bit, bic
65		− bitop set_bit, orr
	70	+ bitop change_bit, eor, steor
	71	+ bitop clear_bit, bic, stclr
	72	+ bitop set_bit, orr, stset
66	73
67		− testop test_and_change_bit, eor
68		− testop test_and_clear_bit, bic
69		− testop test_and_set_bit, orr
	74	+ testop test_and_change_bit, eor, ldeoral
	75	+ testop test_and_clear_bit, bic, ldclral
	76	+ testop test_and_set_bit, orr, ldsetal

arch/arm64/lib/clear_user.S+8−0

@@ -16,7 +16,11 @@ · 15 unmodified lines
16	16	* along with this program. If not, see <http://www.gnu.org/licenses/>.
17	17	*/
18	18	#include <linux/linkage.h>
	19	+
	20	+#include <asm/alternative.h>
19	21	#include <asm/assembler.h>
	22	+#include <asm/cpufeature.h>
	23	+#include <asm/sysreg.h>
20	24
21	25	.text
22	26
@@ -29,6 +33,8 @@ · 6 unmodified lines
29	33	* Alignment fixed up by hardware.
30	34	*/
31	35	ENTRY(__clear_user)
	36	+ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(0)), ARM64_HAS_PAN, \
	37	+ CONFIG_ARM64_PAN)
32	38	mov x2, x1 // save the size for fixup return
33	39	subs x1, x1, #8
34	40	b.mi 2f
@@ -48,6 +54,8 @@ USER(9f, strh wzr, [x0], #2 ) · 13 unmodified lines
48	54	b.mi 5f
49	55	USER(9f, strb wzr, [x0] )
50	56	5: mov x0, #0
	57	+ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(1)), ARM64_HAS_PAN, \
	58	+ CONFIG_ARM64_PAN)
51	59	ret
52	60	ENDPROC(__clear_user)
53	61

arch/arm64/lib/copy_from_user.S+19−6

@@ -15,7 +15,11 @@ · 14 unmodified lines
15	15	*/
16	16
17	17	#include <linux/linkage.h>
	18	+
	19	+#include <asm/alternative.h>
18	20	#include <asm/assembler.h>
	21	+#include <asm/cpufeature.h>
	22	+#include <asm/sysreg.h>
19	23
20	24	/*
21	25	* Copy from user space to a kernel buffer (alignment handled by the hardware)
@@ -28,14 +32,21 @@ · 6 unmodified lines
28	32	* x0 - bytes not copied
29	33	*/
30	34	ENTRY(__copy_from_user)
31		− add x4, x1, x2 // upper user buffer boundary
32		− subs x2, x2, #8
	35	+ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(0)), ARM64_HAS_PAN, \
	36	+ CONFIG_ARM64_PAN)
	37	+ add x5, x1, x2 // upper user buffer boundary
	38	+ subs x2, x2, #16
	39	+ b.mi 1f
	40	+0:
	41	+USER(9f, ldp x3, x4, [x1], #16)
	42	+ subs x2, x2, #16
	43	+ stp x3, x4, [x0], #16
	44	+ b.pl 0b
	45	+1: adds x2, x2, #8
33	46	b.mi 2f
34		−1:
35	47	USER(9f, ldr x3, [x1], #8 )
36		− subs x2, x2, #8
	48	+ sub x2, x2, #8
37	49	str x3, [x0], #8
38		− b.pl 1b
39	50	2: adds x2, x2, #4
40	51	b.mi 3f
41	52	USER(9f, ldr w3, [x1], #4 )
@@ -51,12 +62,14 @@ USER(9f, ldrh w3, [x1], #2 ) · 9 unmodified lines
51	62	USER(9f, ldrb w3, [x1] )
52	63	strb w3, [x0]
53	64	5: mov x0, #0
	65	+ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(1)), ARM64_HAS_PAN, \
	66	+ CONFIG_ARM64_PAN)
54	67	ret
55	68	ENDPROC(__copy_from_user)
56	69
57	70	.section .fixup,"ax"
58	71	.align 2
59		−9: sub x2, x4, x1
	72	+9: sub x2, x5, x1
60	73	mov x3, x2
61	74	10: strb wzr, [x0], #1 // zero remaining buffer space
62	75	subs x3, x3, #1

arch/arm64/lib/copy_in_user.S+19−6

@@ -17,7 +17,11 @@ · 16 unmodified lines
17	17	*/
18	18
19	19	#include <linux/linkage.h>
	20	+
	21	+#include <asm/alternative.h>
20	22	#include <asm/assembler.h>
	23	+#include <asm/cpufeature.h>
	24	+#include <asm/sysreg.h>
21	25
22	26	/*
23	27	* Copy from user space to user space (alignment handled by the hardware)
@@ -30,14 +34,21 @@ · 6 unmodified lines
30	34	* x0 - bytes not copied
31	35	*/
32	36	ENTRY(__copy_in_user)
33		− add x4, x0, x2 // upper user buffer boundary
34		− subs x2, x2, #8
	37	+ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(0)), ARM64_HAS_PAN, \
	38	+ CONFIG_ARM64_PAN)
	39	+ add x5, x0, x2 // upper user buffer boundary
	40	+ subs x2, x2, #16
	41	+ b.mi 1f
	42	+0:
	43	+USER(9f, ldp x3, x4, [x1], #16)
	44	+ subs x2, x2, #16
	45	+USER(9f, stp x3, x4, [x0], #16)
	46	+ b.pl 0b
	47	+1: adds x2, x2, #8
35	48	b.mi 2f
36		−1:
37	49	USER(9f, ldr x3, [x1], #8 )
38		− subs x2, x2, #8
	50	+ sub x2, x2, #8
39	51	USER(9f, str x3, [x0], #8 )
40		− b.pl 1b
41	52	2: adds x2, x2, #4
42	53	b.mi 3f
43	54	USER(9f, ldr w3, [x1], #4 )
@@ -53,11 +64,13 @@ USER(9f, strh w3, [x0], #2 ) · 9 unmodified lines
53	64	USER(9f, ldrb w3, [x1] )
54	65	USER(9f, strb w3, [x0] )
55	66	5: mov x0, #0
	67	+ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(1)), ARM64_HAS_PAN, \
	68	+ CONFIG_ARM64_PAN)
56	69	ret
57	70	ENDPROC(__copy_in_user)
58	71
59	72	.section .fixup,"ax"
60	73	.align 2
61		−9: sub x0, x4, x0 // bytes not copied
	74	+9: sub x0, x5, x0 // bytes not copied
62	75	ret
63	76	.previous

arch/arm64/lib/copy_to_user.S+19−6

@@ -15,7 +15,11 @@ · 14 unmodified lines
15	15	*/
16	16
17	17	#include <linux/linkage.h>
	18	+
	19	+#include <asm/alternative.h>
18	20	#include <asm/assembler.h>
	21	+#include <asm/cpufeature.h>
	22	+#include <asm/sysreg.h>
19	23
20	24	/*
21	25	* Copy to user space from a kernel buffer (alignment handled by the hardware)
@@ -28,14 +32,21 @@ · 6 unmodified lines
28	32	* x0 - bytes not copied
29	33	*/
30	34	ENTRY(__copy_to_user)
31		− add x4, x0, x2 // upper user buffer boundary
32		− subs x2, x2, #8
	35	+ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(0)), ARM64_HAS_PAN, \
	36	+ CONFIG_ARM64_PAN)
	37	+ add x5, x0, x2 // upper user buffer boundary
	38	+ subs x2, x2, #16
	39	+ b.mi 1f
	40	+0:
	41	+ ldp x3, x4, [x1], #16
	42	+ subs x2, x2, #16
	43	+USER(9f, stp x3, x4, [x0], #16)
	44	+ b.pl 0b
	45	+1: adds x2, x2, #8
33	46	b.mi 2f
34		−1:
35	47	ldr x3, [x1], #8
36		− subs x2, x2, #8
	48	+ sub x2, x2, #8
37	49	USER(9f, str x3, [x0], #8 )
38		− b.pl 1b
39	50	2: adds x2, x2, #4
40	51	b.mi 3f
41	52	ldr w3, [x1], #4
@@ -51,11 +62,13 @@ USER(9f, strh w3, [x0], #2 ) · 9 unmodified lines
51	62	ldrb w3, [x1]
52	63	USER(9f, strb w3, [x0] )
53	64	5: mov x0, #0
	65	+ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(1)), ARM64_HAS_PAN, \
	66	+ CONFIG_ARM64_PAN)
54	67	ret
55	68	ENDPROC(__copy_to_user)
56	69
57	70	.section .fixup,"ax"
58	71	.align 2
59		−9: sub x0, x4, x0 // bytes not copied
	72	+9: sub x0, x5, x0 // bytes not copied
60	73	ret
61	74	.previous

arch/arm64/mm/cache.S+6−1

@@ -143,7 +143,12 @@ __dma_clean_range: · 142 unmodified lines
143	143	dcache_line_size x2, x3
144	144	sub x3, x2, #1
145	145	bic x0, x0, x3
146		−1: alternative_insn "dc cvac, x0", "dc civac, x0", ARM64_WORKAROUND_CLEAN_CACHE
	146	+1:
	147	+alternative_if_not ARM64_WORKAROUND_CLEAN_CACHE
	148	+ dc cvac, x0
	149	+alternative_else
	150	+ dc civac, x0
	151	+alternative_endif
147	152	add x0, x0, x2
148	153	cmp x0, x1
149	154	b.lo 1b

arch/arm64/mm/context.c+0−16

@@ -53,8 +53,6 @@ static void flush_context(void) · 52 unmodified lines
53	53	__flush_icache_all();
54	54	}
55	55
56		−#ifdef CONFIG_SMP
57		−
58	56	static void set_mm_context(struct mm_struct *mm, unsigned int asid)
59	57	{
60	58	unsigned long flags;
@@ -110,23 +108,12 @@ static void reset_context(void *info) · 49 unmodified lines
110	108	cpu_switch_mm(mm->pgd, mm);
111	109	}
112	110
113		−#else
114		−
115		−static inline void set_mm_context(struct mm_struct *mm, unsigned int asid)
116		−{
117		− mm->context.id = asid;
118		− cpumask_copy(mm_cpumask(mm), cpumask_of(smp_processor_id()));
119		−}
120		−
121		−#endif
122		−
123	111	void __new_context(struct mm_struct *mm)
124	112	{
125	113	unsigned int asid;
126	114	unsigned int bits = asid_bits();
127	115
128	116	raw_spin_lock(&cpu_asid_lock);
129		−#ifdef CONFIG_SMP
130	117	/*
131	118	* Check the ASID again, in case the change was broadcast from another
132	119	* CPU before we acquired the lock.
@@ -136,7 +123,6 @@ void __new_context(struct mm_struct *mm) · 3 unmodified lines
136	123	raw_spin_unlock(&cpu_asid_lock);
137	124	return;
138	125	}
139		−#endif
140	126	/*
141	127	* At this point, it is guaranteed that the current mm (with an old
142	128	* ASID) isn't active on any other CPU since the ASIDs are changed
@@ -155,10 +141,8 @@ void __new_context(struct mm_struct *mm) · 12 unmodified lines
155	141	cpu_last_asid = ASID_FIRST_VERSION;
156	142	asid = cpu_last_asid + smp_processor_id();
157	143	flush_context();
158		−#ifdef CONFIG_SMP
159	144	smp_wmb();
160	145	smp_call_function(reset_context, NULL, 1);
161		−#endif
162	146	cpu_last_asid += NR_CPUS - 1;
163	147	}
164	148

arch/arm64/mm/dma-mapping.c+20−13

@@ -144,6 +144,7 @@ static void __dma_alloc(struct device dev, size_t size, · 143 unmodified lines
144	144	struct page *page;
145	145	void ptr, coherent_ptr;
146	146	bool coherent = is_device_dma_coherent(dev);
	147	+ pgprot_t prot = __get_dma_pgprot(attrs, PAGE_KERNEL, false);
147	148
148	149	size = PAGE_ALIGN(size);
149	150
@@ -171,9 +172,7 @@ static void __dma_alloc(struct device dev, size_t size, · 21 unmodified lines
171	172	/* create a coherent mapping */
172	173	page = virt_to_page(ptr);
173	174	coherent_ptr = dma_common_contiguous_remap(page, size, VM_USERMAP,
174		− __get_dma_pgprot(attrs,
175		− __pgprot(PROT_NORMAL_NC), false),
176		− NULL);
	175	+ prot, NULL);
177	176	if (!coherent_ptr)
178	177	goto no_map;
179	178
@@ -303,9 +302,10 @@ static void __swiotlb_sync_sg_for_device(struct device *dev, · 123 unmodified lines
303	302	sg->length, dir);
304	303	}
305	304
306		−/* vma->vm_page_prot must be set appropriately before calling this function */
307		−static int __dma_common_mmap(struct device dev, struct vm_area_struct vma,
308		− void *cpu_addr, dma_addr_t dma_addr, size_t size)
	305	+static int __swiotlb_mmap(struct device *dev,
	306	+ struct vm_area_struct *vma,
	307	+ void *cpu_addr, dma_addr_t dma_addr, size_t size,
	308	+ struct dma_attrs *attrs)
309	309	{
310	310	int ret = -ENXIO;
311	311	unsigned long nr_vma_pages = (vma->vm_end - vma->vm_start) >>
@@ -314,6 +314,9 @@ static int __dma_common_mmap(struct device dev, struct vm_area_struct vma, · 2 unmodified lines
314	314	unsigned long pfn = dma_to_phys(dev, dma_addr) >> PAGE_SHIFT;
315	315	unsigned long off = vma->vm_pgoff;
316	316
	317	+ vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot,
	318	+ is_device_dma_coherent(dev));
	319	+
317	320	if (dma_mmap_from_coherent(dev, vma, cpu_addr, size, &ret))
318	321	return ret;
319	322
@@ -327,20 +330,24 @@ static int __dma_common_mmap(struct device dev, struct vm_area_struct vma, · 7 unmodified lines
327	330	return ret;
328	331	}
329	332
330		−static int __swiotlb_mmap(struct device *dev,
331		− struct vm_area_struct *vma,
332		− void *cpu_addr, dma_addr_t dma_addr, size_t size,
333		− struct dma_attrs *attrs)
	333	+static int __swiotlb_get_sgtable(struct device dev, struct sg_table sgt,
	334	+ void *cpu_addr, dma_addr_t handle, size_t size,
	335	+ struct dma_attrs *attrs)
334	336	{
335		− vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot,
336		− is_device_dma_coherent(dev));
337		− return __dma_common_mmap(dev, vma, cpu_addr, dma_addr, size);
	337	+ int ret = sg_alloc_table(sgt, 1, GFP_KERNEL);
	338	+
	339	+ if (!ret)
	340	+ sg_set_page(sgt->sgl, phys_to_page(dma_to_phys(dev, handle)),
	341	+ PAGE_ALIGN(size), 0);
	342	+
	343	+ return ret;
338	344	}
339	345
340	346	static struct dma_map_ops swiotlb_dma_ops = {
341	347	.alloc = __dma_alloc,
342	348	.free = __dma_free,
343	349	.mmap = __swiotlb_mmap,
	350	+ .get_sgtable = __swiotlb_get_sgtable,
344	351	.map_page = __swiotlb_map_page,
345	352	.unmap_page = __swiotlb_unmap_page,
346	353	.map_sg = __swiotlb_map_sg_attrs,

arch/arm64/mm/fault.c+26−2

@@ -30,9 +30,11 @@ · 29 unmodified lines
30	30	#include <linux/highmem.h>
31	31	#include <linux/perf_event.h>
32	32
	33	+#include <asm/cpufeature.h>
33	34	#include <asm/exception.h>
34	35	#include <asm/debug-monitors.h>
35	36	#include <asm/esr.h>
	37	+#include <asm/sysreg.h>
36	38	#include <asm/system_misc.h>
37	39	#include <asm/pgtable.h>
38	40	#include <asm/tlbflush.h>
@@ -223,6 +225,13 @@ static int __kprobes do_page_fault(unsigned long addr, unsigned int esr, · 184 unmodified lines
223	225	mm_flags \|= FAULT_FLAG_WRITE;
224	226	}
225	227
	228	+ /*
	229	+ * PAN bit set implies the fault happened in kernel space, but not
	230	+ * in the arch's user access functions.
	231	+ */
	232	+ if (IS_ENABLED(CONFIG_ARM64_PAN) && (regs->pstate & PSR_PAN_BIT))
	233	+ goto no_context;
	234	+
226	235	/*
227	236	* As per x86, we may deadlock here. However, since the kernel only
228	237	* validly references user space from well defined areas of the code,
@@ -492,14 +501,22 @@ asmlinkage void __exception do_sp_pc_abort(unsigned long addr, · 263 unmodified lines
492	501	arm64_notify_die("Oops - SP/PC alignment exception", regs, &info, esr);
493	502	}
494	503
495		−static struct fault_info debug_fault_info[] = {
	504	+int __init early_brk64(unsigned long addr, unsigned int esr,
	505	+ struct pt_regs *regs);
	506	+
	507	+/*
	508	+ * __refdata because early_brk64 is __init, but the reference to it is
	509	+ * clobbered at arch_initcall time.
	510	+ * See traps.c and debug-monitors.c:debug_traps_init().
	511	+ */
	512	+static struct fault_info __refdata debug_fault_info[] = {
496	513	{ do_bad, SIGTRAP, TRAP_HWBKPT, "hardware breakpoint" },
497	514	{ do_bad, SIGTRAP, TRAP_HWBKPT, "hardware single-step" },
498	515	{ do_bad, SIGTRAP, TRAP_HWBKPT, "hardware watchpoint" },
499	516	{ do_bad, SIGBUS, 0, "unknown 3" },
500	517	{ do_bad, SIGTRAP, TRAP_BRKPT, "aarch32 BKPT" },
501	518	{ do_bad, SIGTRAP, 0, "aarch32 vector catch" },
502		− { do_bad, SIGTRAP, TRAP_BRKPT, "aarch64 BRK" },
	519	+ { early_brk64, SIGTRAP, TRAP_BRKPT, "aarch64 BRK" },
503	520	{ do_bad, SIGBUS, 0, "unknown 7" },
504	521	};
505	522
@@ -536,3 +553,10 @@ asmlinkage int __exception do_debug_exception(unsigned long addr, · 30 unmodified lines
536	553
537	554	return 0;
538	555	}
	556	+
	557	+#ifdef CONFIG_ARM64_PAN
	558	+void cpu_enable_pan(void)
	559	+{
	560	+ config_sctlr_el1(SCTLR_EL1_SPAN, 0);
	561	+}
	562	+#endif /* CONFIG_ARM64_PAN */

arch/arm64/mm/flush.c+0−4

@@ -60,14 +60,10 @@ void copy_to_user_page(struct vm_area_struct vma, struct page page, · 59 unmodified lines
60	60	unsigned long uaddr, void dst, const void src,
61	61	unsigned long len)
62	62	{
63		−#ifdef CONFIG_SMP
64	63	preempt_disable();
65		−#endif
66	64	memcpy(dst, src, len);
67	65	flush_ptrace_access(vma, page, uaddr, dst, len);
68		−#ifdef CONFIG_SMP
69	66	preempt_enable();
70		−#endif
71	67	}
72	68
73	69	void __sync_icache_dcache(pte_t pte, unsigned long addr)

arch/arm64/mm/hugetlbpage.c+0−4

@@ -13,10 +13,6 @@ · 12 unmodified lines
13	13	* but WITHOUT ANY WARRANTY; without even the implied warranty of
14	14	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15	15	* GNU General Public License for more details.
16		− *
17		− * You should have received a copy of the GNU General Public License
18		− * along with this program; if not, write to the Free Software
19		− * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
20	16	*/
21	17
22	18	#include <linux/init.h>

arch/arm64/mm/init.c+2−2

@@ -358,9 +358,9 @@ void free_initmem(void) · 357 unmodified lines
358	358
359	359	#ifdef CONFIG_BLK_DEV_INITRD
360	360
361		−static int keep_initrd;
	361	+static int keep_initrd __initdata;
362	362
363		−void free_initrd_mem(unsigned long start, unsigned long end)
	363	+void __init free_initrd_mem(unsigned long start, unsigned long end)
364	364	{
365	365	if (!keep_initrd)
366	366	free_reserved_area((void )start, (void )end, 0, "initrd");

arch/arm64/mm/mmu.c+1−12

@@ -267,7 +267,7 @@ static void *late_alloc(unsigned long size) · 266 unmodified lines
267	267	return ptr;
268	268	}
269	269
270		−static void __ref create_mapping(phys_addr_t phys, unsigned long virt,
	270	+static void __init create_mapping(phys_addr_t phys, unsigned long virt,
271	271	phys_addr_t size, pgprot_t prot)
272	272	{
273	273	if (virt < VMALLOC_START) {
@@ -460,17 +460,6 @@ void __init paging_init(void) · 186 unmodified lines
460	460	cpu_set_default_tcr_t0sz();
461	461	}
462	462
463		−/*
464		− * Enable the identity mapping to allow the MMU disabling.
465		− */
466		−void setup_mm_for_reboot(void)
467		−{
468		− cpu_set_reserved_ttbr0();
469		− flush_tlb_all();
470		− cpu_set_idmap_tcr_t0sz();
471		− cpu_switch_mm(idmap_pg_dir, &init_mm);
472		−}
473		−
474	463	/*
475	464	* Check whether a kernel address is valid (derived from arch/x86/).
476	465	*/

arch/arm64/mm/proc.S+15−6

@@ -34,11 +34,7 @@ · 33 unmodified lines
34	34	#define TCR_TG_FLAGS TCR_TG0_4K \| TCR_TG1_4K
35	35	#endif
36	36
37		−#ifdef CONFIG_SMP
38	37	#define TCR_SMP_FLAGS TCR_SHARED
39		−#else
40		−#define TCR_SMP_FLAGS 0
41		−#endif
42	38
43	39	/* PTWs cacheable, inner/outer WBWA */
44	40	#define TCR_CACHE_FLAGS TCR_IRGN_WBWA \| TCR_ORGN_WBWA
@@ -150,13 +146,13 @@ ENDPROC(cpu_do_switch_mm) · 105 unmodified lines
150	146	* value of the SCTLR_EL1 register.
151	147	*/
152	148	ENTRY(__cpu_setup)
153		− ic iallu // I+BTB cache invalidate
154	149	tlbi vmalle1is // invalidate I + D TLBs
155	150	dsb ish
156	151
157	152	mov x0, #3 << 20
158	153	msr cpacr_el1, x0 // Enable FP/ASIMD
159		− msr mdscr_el1, xzr // Reset mdscr_el1
	154	+ mov x0, #1 << 12 // Reset mdscr_el1 and disable
	155	+ msr mdscr_el1, x0 // access to the DCC from EL0
160	156	/*
161	157	* Memory region attributes for LPAE:
162	158	*
@@ -196,6 +192,19 @@ ENTRY(__cpu_setup) · 33 unmodified lines
196	192	*/
197	193	mrs x9, ID_AA64MMFR0_EL1
198	194	bfi x10, x9, #32, #3
	195	+#ifdef CONFIG_ARM64_HW_AFDBM
	196	+ /*
	197	+ * Hardware update of the Access and Dirty bits.
	198	+ */
	199	+ mrs x9, ID_AA64MMFR1_EL1
	200	+ and x9, x9, #0xf
	201	+ cbz x9, 2f
	202	+ cmp x9, #2
	203	+ b.lt 1f
	204	+ orr x10, x10, #TCR_HD // hardware Dirty flag update
	205	+1: orr x10, x10, #TCR_HA // hardware Access flag update
	206	+2:
	207	+#endif /* CONFIG_ARM64_HW_AFDBM */
199	208	msr tcr_el1, x10
200	209	ret // return to head.S
201	210	ENDPROC(__cpu_setup)

drivers/of/fdt.c+7−5

@@ -967,7 +967,9 @@ int __init early_init_dt_scan_chosen(unsigned long node, const char *uname, · 966 unmodified lines
967	967	}
968	968
969	969	#ifdef CONFIG_HAVE_MEMBLOCK
970		−#define MAX_PHYS_ADDR ((phys_addr_t)~0)
	970	+#ifndef MAX_MEMBLOCK_ADDR
	971	+#define MAX_MEMBLOCK_ADDR ((phys_addr_t)~0)
	972	+#endif
971	973
972	974	void __init __weak early_init_dt_add_memory_arch(u64 base, u64 size)
973	975	{
@@ -984,16 +986,16 @@ void __init __weak early_init_dt_add_memory_arch(u64 base, u64 size) · 10 unmodified lines
984	986	}
985	987	size &= PAGE_MASK;
986	988
987		− if (base > MAX_PHYS_ADDR) {
	989	+ if (base > MAX_MEMBLOCK_ADDR) {
988	990	pr_warning("Ignoring memory block 0x%llx - 0x%llx\n",
989	991	base, base + size);
990	992	return;
991	993	}
992	994
993		− if (base + size - 1 > MAX_PHYS_ADDR) {
	995	+ if (base + size - 1 > MAX_MEMBLOCK_ADDR) {
994	996	pr_warning("Ignoring memory range 0x%llx - 0x%llx\n",
995		− ((u64)MAX_PHYS_ADDR) + 1, base + size);
996		− size = MAX_PHYS_ADDR - base + 1;
	997	+ ((u64)MAX_MEMBLOCK_ADDR) + 1, base + size);
	998	+ size = MAX_MEMBLOCK_ADDR - base + 1;
997	999	}
998	1000
999	1001	if (base + size < phys_offset) {

Merge tag 'arm64-upstream' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code