lael-labs/linux

Documentation/arm64/booting.txt+7−4

   u64 res3	= 0;		/* reserved */
   u64 res4	= 0;		/* reserved */
   u32 magic	= 0x644d5241;	/* Magic number, little endian, "ARM\x64" */
-  u32 res5;      		/* reserved (used for PE COFF offset) */
+  u32 res5;			/* reserved (used for PE COFF offset) */
 Header notes:
 - The flags field (introduced in v3.17) is a little-endian 64-bit field
   composed as follows:
-  Bit 0: 	Kernel endianness.  1 if BE, 0 if LE.
+  Bit 0:	Kernel endianness.  1 if BE, 0 if LE.
   Bits 1-63:	Reserved.
 - When image_size is zero, a bootloader should attempt to keep as much
 address near the start of usable system RAM and called there. Memory
 below that base address is currently unusable by Linux, and therefore it
 is strongly recommended that this location is the start of system RAM.
+The region between the 2 MB aligned base address and the start of the
+image has no special significance to the kernel, and may be used for
+other purposes.
 At least image_size bytes from the start of the image must be free for
 use by the kernel.
-Any memory described to the kernel (even that below the 2MB aligned base
+Any memory described to the kernel (even that below the start of the
-address) which is not marked as reserved from the kernel e.g. with a
+image) which is not marked as reserved from the kernel (e.g., with a
 memreserve region in the device tree) will be considered as available to
 the kernel.

arch/arm64/Kconfig+64−22

 	select EDAC_SUPPORT
 	select GENERIC_ALLOCATOR
 	select GENERIC_CLOCKEVENTS
-	select GENERIC_CLOCKEVENTS_BROADCAST if SMP
+	select GENERIC_CLOCKEVENTS_BROADCAST
 	select GENERIC_CPU_AUTOPROBE
 	select GENERIC_EARLY_IOREMAP
 	select GENERIC_IRQ_PROBE
 	select HAVE_C_RECORDMCOUNT
 	select HAVE_CC_STACKPROTECTOR
 	select HAVE_CMPXCHG_DOUBLE
+	select HAVE_CMPXCHG_LOCAL
 	select HAVE_DEBUG_BUGVERBOSE
 	select HAVE_DEBUG_KMEMLEAK
 	select HAVE_DMA_API_DEBUG
 config STACKTRACE_SUPPORT
 	def_bool y
+config ILLEGAL_POINTER_VALUE
+	hex
+	default 0xdead000000000000
 config LOCKDEP_SUPPORT
 	def_bool y
 config RWSEM_XCHGADD_ALGORITHM
 	def_bool y
+config GENERIC_BUG
+	def_bool y
+	depends on BUG
+config GENERIC_BUG_RELATIVE_POINTERS
+	def_bool y
+	depends on GENERIC_BUG
 config GENERIC_HWEIGHT
 	def_bool y
 config NEED_SG_DMA_LENGTH
 	def_bool y
+config SMP
+	def_bool y
 config SWIOTLB
 	def_bool y
        help
          Say Y if you plan on running a kernel in big-endian mode.
-config SMP
-	bool "Symmetric Multi-Processing"
-	help
-	  This enables support for systems with more than one CPU.  If
-	  you say N here, the kernel will run on single and
-	  multiprocessor machines, but will use only one CPU of a
-	  multiprocessor machine. If you say Y here, the kernel will run
-	  on many, but not all, single processor machines. On a single
-	  processor machine, the kernel will run faster if you say N
-	  here.
-	  If you don't know what to do here, say N.
 config SCHED_MC
 	bool "Multi-core scheduler support"
-	depends on SMP
 	help
 	  Multi-core scheduler support improves the CPU scheduler's decision
 	  making when dealing with multi-core CPU chips at a cost of slightly
 config SCHED_SMT
 	bool "SMT scheduler support"
-	depends on SMP
 	help
 	  Improves the CPU scheduler's decision making when dealing with
 	  MultiThreading at a cost of slightly increased overhead in some
 config NR_CPUS
 	int "Maximum number of CPUs (2-4096)"
 	range 2 4096
-	depends on SMP
 	# These have to remain sorted largest to smallest
 	default "64"
 config HOTPLUG_CPU
 	bool "Support for hot-pluggable CPUs"
-	depends on SMP
 	help
 	  Say Y here to experiment with turning CPUs off and on.  CPUs
 	  can be controlled through /sys/devices/system/cpu.
 source kernel/Kconfig.preempt
-config UP_LATE_INIT
-       def_bool y
-       depends on !SMP
 config HZ
 	int
 	default 100
 	  If unsure, say Y
 endif
+menu "ARMv8.1 architectural features"
+config ARM64_HW_AFDBM
+	bool "Support for hardware updates of the Access and Dirty page flags"
+	default y
+	help
+	  The ARMv8.1 architecture extensions introduce support for
+	  hardware updates of the access and dirty information in page
+	  table entries. When enabled in TCR_EL1 (HA and HD bits) on
+	  capable processors, accesses to pages with PTE_AF cleared will
+	  set this bit instead of raising an access flag fault.
+	  Similarly, writes to read-only pages with the DBM bit set will
+	  clear the read-only bit (AP[2]) instead of raising a
+	  permission fault.
+	  Kernels built with this configuration option enabled continue
+	  to work on pre-ARMv8.1 hardware and the performance impact is
+	  minimal. If unsure, say Y.
+config ARM64_PAN
+	bool "Enable support for Privileged Access Never (PAN)"
+	default y
+	help
+	 Privileged Access Never (PAN; part of the ARMv8.1 Extensions)
+	 prevents the kernel or hypervisor from accessing user-space (EL0)
+	 memory directly.
+	 Choosing this option will cause any unprotected (not using
+	 copy_to_user et al) memory access to fail with a permission fault.
+	 The feature is detected at runtime, and will remain as a 'nop'
+	 instruction if the cpu does not implement the feature.
+config ARM64_LSE_ATOMICS
+	bool "Atomic instructions"
+	help
+	  As part of the Large System Extensions, ARMv8.1 introduces new
+	  atomic instructions that are designed specifically to scale in
+	  very large systems.
+	  Say Y here to make use of these instructions for the in-kernel
+	  atomic routines. This incurs a small overhead on CPUs that do
+	  not support these instructions and requires the kernel to be
+	  built with binutils >= 2.25.
+endmenu
 endmenu
 menu "Boot options"

arch/arm64/Makefile+16−2

 KBUILD_DEFCONFIG := defconfig
-KBUILD_CFLAGS	+= -mgeneral-regs-only
+# Check for binutils support for specific extensions
+lseinstr := $(call as-instr,.arch_extension lse,-DCONFIG_AS_LSE=1)
+ifeq ($(CONFIG_ARM64_LSE_ATOMICS), y)
+  ifeq ($(lseinstr),)
+$(warning LSE atomics not supported by binutils)
+  endif
+endif
+KBUILD_CFLAGS	+= -mgeneral-regs-only $(lseinstr)
+KBUILD_AFLAGS	+= $(lseinstr)
 ifeq ($(CONFIG_CPU_BIG_ENDIAN), y)
 KBUILD_CPPFLAGS	+= -mbig-endian
 AS		+= -EB
 boot := arch/arm64/boot
-Image Image.gz: vmlinux
+Image: vmlinux
+	$(Q)$(MAKE) $(build)=$(boot) $(boot)/$@
+Image.%: vmlinux
 	$(Q)$(MAKE) $(build)=$(boot) $(boot)/$@
 zinstall install: vmlinux

arch/arm64/boot/Makefile+12−0

@@ -19,9 +19,21 @@ targets := Image Image.gz · 18 unmodified lines
19	$(obj)/Image: vmlinux FORCE	19	$(obj)/Image: vmlinux FORCE
20	$(call if_changed,objcopy)	20	$(call if_changed,objcopy)
21		21
		22	$(obj)/Image.bz2: $(obj)/Image FORCE
		23	$(call if_changed,bzip2)
		24
22	$(obj)/Image.gz: $(obj)/Image FORCE	25	$(obj)/Image.gz: $(obj)/Image FORCE
23	$(call if_changed,gzip)	26	$(call if_changed,gzip)
24		27
		28	$(obj)/Image.lz4: $(obj)/Image FORCE
		29	$(call if_changed,lz4)
		30
		31	$(obj)/Image.lzma: $(obj)/Image FORCE
		32	$(call if_changed,lzma)
		33
		34	$(obj)/Image.lzo: $(obj)/Image FORCE
		35	$(call if_changed,lzo)
		36
25	install: $(obj)/Image	37	install: $(obj)/Image
26	$(CONFIG_SHELL) $(srctree)/$(src)/install.sh $(KERNELRELEASE) \	38	$(CONFIG_SHELL) $(srctree)/$(src)/install.sh $(KERNELRELEASE) \
27	$(obj)/Image System.map "$(INSTALL_PATH)"	39	$(obj)/Image System.map "$(INSTALL_PATH)"

arch/arm64/include/asm/alternative.h+74−4

@@ -3,6 +3,8 @@ · 2 unmodified lines
3		3
4	#ifndef __ASSEMBLY__	4	#ifndef __ASSEMBLY__
5		5
		6	#include <linux/init.h>
		7	#include <linux/kconfig.h>
6	#include <linux/types.h>	8	#include <linux/types.h>
7	#include <linux/stddef.h>	9	#include <linux/stddef.h>
8	#include <linux/stringify.h>	10	#include <linux/stringify.h>
@@ -15,7 +17,7 @@ struct alt_instr { · 6 unmodified lines
15	u8 alt_len; /* size of new instruction(s), <= orig_len */	17	u8 alt_len; /* size of new instruction(s), <= orig_len */
16	};	18	};
17		19
18	void apply_alternatives_all(void);	20	void __init apply_alternatives_all(void);
19	void apply_alternatives(void *start, size_t length);	21	void apply_alternatives(void *start, size_t length);
20	void free_alternatives_memory(void);	22	void free_alternatives_memory(void);
21		23
@@ -40,7 +42,8 @@ void free_alternatives_memory(void); · 18 unmodified lines
40	* be fixed in a binutils release posterior to 2.25.51.0.2 (anything	42	* be fixed in a binutils release posterior to 2.25.51.0.2 (anything
41	* containing commit 4e4d08cf7399b606 or c1baaddf8861).	43	* containing commit 4e4d08cf7399b606 or c1baaddf8861).
42	*/	44	*/
43	#define ALTERNATIVE(oldinstr, newinstr, feature) \	45	#define __ALTERNATIVE_CFG(oldinstr, newinstr, feature, cfg_enabled) \
		46	".if "__stringify(cfg_enabled)" == 1\n" \
44	"661:\n\t" \	47	"661:\n\t" \
45	oldinstr "\n" \	48	oldinstr "\n" \
46	"662:\n" \	49	"662:\n" \
@@ -53,7 +56,11 @@ void free_alternatives_memory(void); · 6 unmodified lines
53	"664:\n\t" \	56	"664:\n\t" \
54	".popsection\n\t" \	57	".popsection\n\t" \
55	".org . - (664b-663b) + (662b-661b)\n\t" \	58	".org . - (664b-663b) + (662b-661b)\n\t" \
56	".org . - (662b-661b) + (664b-663b)\n"	59	".org . - (662b-661b) + (664b-663b)\n" \
		60	".endif\n"
		61
		62	#define _ALTERNATIVE_CFG(oldinstr, newinstr, feature, cfg, ...) \
		63	__ALTERNATIVE_CFG(oldinstr, newinstr, feature, IS_ENABLED(cfg))
57		64
58	#else	65	#else
59		66
@@ -65,7 +72,8 @@ void free_alternatives_memory(void); · 5 unmodified lines
65	.byte \alt_len	72	.byte \alt_len
66	.endm	73	.endm
67		74
68	.macro alternative_insn insn1 insn2 cap	75	.macro alternative_insn insn1, insn2, cap, enable = 1
		76	.if \enable
69	661: \insn1	77	661: \insn1
70	662: .pushsection .altinstructions, "a"	78	662: .pushsection .altinstructions, "a"
71	altinstruction_entry 661b, 663f, \cap, 662b-661b, 664f-663f	79	altinstruction_entry 661b, 663f, \cap, 662b-661b, 664f-663f
@@ -75,8 +83,70 @@ void free_alternatives_memory(void); · 3 unmodified lines
75	664: .popsection	83	664: .popsection
76	.org . - (664b-663b) + (662b-661b)	84	.org . - (664b-663b) + (662b-661b)
77	.org . - (662b-661b) + (664b-663b)	85	.org . - (662b-661b) + (664b-663b)
		86	.endif
		87	.endm
		88
		89	/*
		90	* Begin an alternative code sequence.
		91	*
		92	* The code that follows this macro will be assembled and linked as
		93	* normal. There are no restrictions on this code.
		94	*/
		95	.macro alternative_if_not cap, enable = 1
		96	.if \enable
		97	.pushsection .altinstructions, "a"
		98	altinstruction_entry 661f, 663f, \cap, 662f-661f, 664f-663f
		99	.popsection
		100	661:
		101	.endif
		102	.endm
		103
		104	/*
		105	* Provide the alternative code sequence.
		106	*
		107	* The code that follows this macro is assembled into a special
		108	* section to be used for dynamic patching. Code that follows this
		109	* macro must:
		110	*
		111	* 1. Be exactly the same length (in bytes) as the default code
		112	* sequence.
		113	*
		114	* 2. Not contain a branch target that is used outside of the
		115	* alternative sequence it is defined in (branches into an
		116	* alternative sequence are not fixed up).
		117	*/
		118	.macro alternative_else, enable = 1
		119	.if \enable
		120	662: .pushsection .altinstr_replacement, "ax"
		121	663:
		122	.endif
78	.endm	123	.endm
79		124
		125	/*
		126	* Complete an alternative code sequence.
		127	*/
		128	.macro alternative_endif, enable = 1
		129	.if \enable
		130	664: .popsection
		131	.org . - (664b-663b) + (662b-661b)
		132	.org . - (662b-661b) + (664b-663b)
		133	.endif
		134	.endm
		135
		136	#define _ALTERNATIVE_CFG(insn1, insn2, cap, cfg, ...) \
		137	alternative_insn insn1, insn2, cap, IS_ENABLED(cfg)
		138
		139
80	#endif /* __ASSEMBLY__ */	140	#endif /* __ASSEMBLY__ */
81		141
		142	/*
		143	* Usage: asm(ALTERNATIVE(oldinstr, newinstr, feature));
		144	*
		145	* Usage: asm(ALTERNATIVE(oldinstr, newinstr, feature, CONFIG_FOO));
		146	* N.B. If CONFIG_FOO is specified, but not selected, the whole block
		147	* will be omitted, including oldinstr.
		148	*/
		149	#define ALTERNATIVE(oldinstr, newinstr, ...) \
		150	_ALTERNATIVE_CFG(oldinstr, newinstr, __VA_ARGS__, 1)
		151
82	#endif /* __ASM_ALTERNATIVE_H */	152	#endif /* __ASM_ALTERNATIVE_H */

arch/arm64/include/asm/assembler.h+0−14

@@ -49,18 +49,6 @@ · 48 unmodified lines
49	msr daifclr, #2	49	msr daifclr, #2
50	.endm	50	.endm
51		51
52	/*
53	* Save/disable and restore interrupts.
54	*/
55	.macro save_and_disable_irqs, olddaif
56	mrs \olddaif, daif
57	disable_irq
58	.endm
59
60	.macro restore_irqs, olddaif
61	msr daif, \olddaif
62	.endm
63
64	/*	52	/*
65	* Enable and disable debug exceptions.	53	* Enable and disable debug exceptions.
66	*/	54	*/
@@ -103,9 +91,7 @@ · 36 unmodified lines
103	* SMP data memory barrier	91	* SMP data memory barrier
104	*/	92	*/
105	.macro smp_dmb, opt	93	.macro smp_dmb, opt
106	#ifdef CONFIG_SMP
107	dmb \opt	94	dmb \opt
108	#endif
109	.endm	95	.endm
110		96
111	#define USER(l, x...) \	97	#define USER(l, x...) \

arch/arm64/include/asm/atomic.h+52−227

@@ -24,247 +24,72 @@ · 23 unmodified lines
24	#include <linux/types.h>	24	#include <linux/types.h>
25		25
26	#include <asm/barrier.h>	26	#include <asm/barrier.h>
27	#include <asm/cmpxchg.h>	27	#include <asm/lse.h>
28
29	#define ATOMIC_INIT(i) { (i) }
30		28
31	#ifdef __KERNEL__	29	#ifdef __KERNEL__
32		30
33	/*	31	#define __ARM64_IN_ATOMIC_IMPL
34	* On ARM, ordinary assignment (str instruction) doesn't clear the local
35	* strex/ldrex monitor on some implementations. The reason we can use it for
36	* atomic_set() is the clrex or dummy strex done on every exception return.
37	*/
38	#define atomic_read(v) ACCESS_ONCE((v)->counter)
39	#define atomic_set(v,i) (((v)->counter) = (i))
40
41	/*
42	* AArch64 UP and SMP safe atomic ops. We use load exclusive and
43	* store exclusive to ensure that these are atomic. We may loop
44	* to ensure that the update happens.
45	*/
46
47	#define ATOMIC_OP(op, asm_op) \
48	static inline void atomic_##op(int i, atomic_t *v) \
49	{ \
50	unsigned long tmp; \
51	int result; \
52	\
53	asm volatile("// atomic_" #op "\n" \
54	"1: ldxr %w0, %2\n" \
55	" " #asm_op " %w0, %w0, %w3\n" \
56	" stxr %w1, %w0, %2\n" \
57	" cbnz %w1, 1b" \
58	: "=&r" (result), "=&r" (tmp), "+Q" (v->counter) \
59	: "Ir" (i)); \
60	} \
61
62	#define ATOMIC_OP_RETURN(op, asm_op) \
63	static inline int atomic_##op##_return(int i, atomic_t *v) \
64	{ \
65	unsigned long tmp; \
66	int result; \
67	\
68	asm volatile("// atomic_" #op "_return\n" \
69	"1: ldxr %w0, %2\n" \
70	" " #asm_op " %w0, %w0, %w3\n" \
71	" stlxr %w1, %w0, %2\n" \
72	" cbnz %w1, 1b" \
73	: "=&r" (result), "=&r" (tmp), "+Q" (v->counter) \
74	: "Ir" (i) \
75	: "memory"); \
76	\
77	smp_mb(); \
78	return result; \
79	}
80		32
81	#define ATOMIC_OPS(op, asm_op) \	33	#if defined(CONFIG_ARM64_LSE_ATOMICS) && defined(CONFIG_AS_LSE)
82	ATOMIC_OP(op, asm_op) \	34	#include <asm/atomic_lse.h>
83	ATOMIC_OP_RETURN(op, asm_op)	35	#else
84		36	#include <asm/atomic_ll_sc.h>
85	ATOMIC_OPS(add, add)	37	#endif
86	ATOMIC_OPS(sub, sub)
87
88	#define atomic_andnot atomic_andnot
89
90	ATOMIC_OP(and, and)
91	ATOMIC_OP(andnot, bic)
92	ATOMIC_OP(or, orr)
93	ATOMIC_OP(xor, eor)
94
95	#undef ATOMIC_OPS
96	#undef ATOMIC_OP_RETURN
97	#undef ATOMIC_OP
98
99	static inline int atomic_cmpxchg(atomic_t *ptr, int old, int new)
100	{
101	unsigned long tmp;
102	int oldval;
103
104	smp_mb();
105
106	asm volatile("// atomic_cmpxchg\n"
107	"1: ldxr %w1, %2\n"
108	" cmp %w1, %w3\n"
109	" b.ne 2f\n"
110	" stxr %w0, %w4, %2\n"
111	" cbnz %w0, 1b\n"
112	"2:"
113	: "=&r" (tmp), "=&r" (oldval), "+Q" (ptr->counter)
114	: "Ir" (old), "r" (new)
115	: "cc");
116
117	smp_mb();
118	return oldval;
119	}
120
121	#define atomic_xchg(v, new) (xchg(&((v)->counter), new))
122		38
123	static inline int __atomic_add_unless(atomic_t *v, int a, int u)	39	#undef __ARM64_IN_ATOMIC_IMPL
124	{
125	int c, old;
126		40
127	c = atomic_read(v);	41	#include <asm/cmpxchg.h>
128	while (c != u && (old = atomic_cmpxchg((v), c, c + a)) != c)
129	c = old;
130	return c;
131	}
132		42
133	#define atomic_inc(v) atomic_add(1, v)	43	#define ___atomic_add_unless(v, a, u, sfx) \
134	#define atomic_dec(v) atomic_sub(1, v)	44	({ \
		45	typeof((v)->counter) c, old; \
		46	\
		47	c = atomic##sfx##_read(v); \
		48	while (c != (u) && \
		49	(old = atomic##sfx##_cmpxchg((v), c, c + (a))) != c) \
		50	c = old; \
		51	c; \
		52	})
135		53
136	#define atomic_inc_and_test(v) (atomic_add_return(1, v) == 0)	54	#define ATOMIC_INIT(i) { (i) }
137	#define atomic_dec_and_test(v) (atomic_sub_return(1, v) == 0)
138	#define atomic_inc_return(v) (atomic_add_return(1, v))
139	#define atomic_dec_return(v) (atomic_sub_return(1, v))
140	#define atomic_sub_and_test(i, v) (atomic_sub_return(i, v) == 0)
141		55
142	#define atomic_add_negative(i,v) (atomic_add_return(i, v) < 0)	56	#define atomic_read(v) READ_ONCE((v)->counter)
		57	#define atomic_set(v, i) (((v)->counter) = (i))
		58	#define atomic_xchg(v, new) xchg(&((v)->counter), (new))
		59	#define atomic_cmpxchg(v, old, new) cmpxchg(&((v)->counter), (old), (new))
		60
		61	#define atomic_inc(v) atomic_add(1, (v))
		62	#define atomic_dec(v) atomic_sub(1, (v))
		63	#define atomic_inc_return(v) atomic_add_return(1, (v))
		64	#define atomic_dec_return(v) atomic_sub_return(1, (v))
		65	#define atomic_inc_and_test(v) (atomic_inc_return(v) == 0)
		66	#define atomic_dec_and_test(v) (atomic_dec_return(v) == 0)
		67	#define atomic_sub_and_test(i, v) (atomic_sub_return((i), (v)) == 0)
		68	#define atomic_add_negative(i, v) (atomic_add_return((i), (v)) < 0)
		69	#define __atomic_add_unless(v, a, u) ___atomic_add_unless(v, a, u,)
		70	#define atomic_andnot atomic_andnot
143		71
144	/*	72	/*
145	* 64-bit atomic operations.	73	* 64-bit atomic operations.
146	*/	74	*/
147	#define ATOMIC64_INIT(i) { (i) }	75	#define ATOMIC64_INIT ATOMIC_INIT
148		76	#define atomic64_read atomic_read
149	#define atomic64_read(v) ACCESS_ONCE((v)->counter)	77	#define atomic64_set atomic_set
150	#define atomic64_set(v,i) (((v)->counter) = (i))	78	#define atomic64_xchg atomic_xchg
151		79	#define atomic64_cmpxchg atomic_cmpxchg
152	#define ATOMIC64_OP(op, asm_op) \	80
153	static inline void atomic64_##op(long i, atomic64_t *v) \	81	#define atomic64_inc(v) atomic64_add(1, (v))
154	{ \	82	#define atomic64_dec(v) atomic64_sub(1, (v))
155	long result; \	83	#define atomic64_inc_return(v) atomic64_add_return(1, (v))
156	unsigned long tmp; \	84	#define atomic64_dec_return(v) atomic64_sub_return(1, (v))
157	\
158	asm volatile("// atomic64_" #op "\n" \
159	"1: ldxr %0, %2\n" \
160	" " #asm_op " %0, %0, %3\n" \
161	" stxr %w1, %0, %2\n" \
162	" cbnz %w1, 1b" \
163	: "=&r" (result), "=&r" (tmp), "+Q" (v->counter) \
164	: "Ir" (i)); \
165	} \
166
167	#define ATOMIC64_OP_RETURN(op, asm_op) \
168	static inline long atomic64_##op##_return(long i, atomic64_t *v) \
169	{ \
170	long result; \
171	unsigned long tmp; \
172	\
173	asm volatile("// atomic64_" #op "_return\n" \
174	"1: ldxr %0, %2\n" \
175	" " #asm_op " %0, %0, %3\n" \
176	" stlxr %w1, %0, %2\n" \
177	" cbnz %w1, 1b" \
178	: "=&r" (result), "=&r" (tmp), "+Q" (v->counter) \
179	: "Ir" (i) \
180	: "memory"); \
181	\
182	smp_mb(); \
183	return result; \
184	}
185
186	#define ATOMIC64_OPS(op, asm_op) \
187	ATOMIC64_OP(op, asm_op) \
188	ATOMIC64_OP_RETURN(op, asm_op)
189
190	ATOMIC64_OPS(add, add)
191	ATOMIC64_OPS(sub, sub)
192
193	#define atomic64_andnot atomic64_andnot
194
195	ATOMIC64_OP(and, and)
196	ATOMIC64_OP(andnot, bic)
197	ATOMIC64_OP(or, orr)
198	ATOMIC64_OP(xor, eor)
199
200	#undef ATOMIC64_OPS
201	#undef ATOMIC64_OP_RETURN
202	#undef ATOMIC64_OP
203
204	static inline long atomic64_cmpxchg(atomic64_t *ptr, long old, long new)
205	{
206	long oldval;
207	unsigned long res;
208
209	smp_mb();
210
211	asm volatile("// atomic64_cmpxchg\n"
212	"1: ldxr %1, %2\n"
213	" cmp %1, %3\n"
214	" b.ne 2f\n"
215	" stxr %w0, %4, %2\n"
216	" cbnz %w0, 1b\n"
217	"2:"
218	: "=&r" (res), "=&r" (oldval), "+Q" (ptr->counter)
219	: "Ir" (old), "r" (new)
220	: "cc");
221
222	smp_mb();
223	return oldval;
224	}
225
226	#define atomic64_xchg(v, new) (xchg(&((v)->counter), new))
227
228	static inline long atomic64_dec_if_positive(atomic64_t *v)
229	{
230	long result;
231	unsigned long tmp;
232
233	asm volatile("// atomic64_dec_if_positive\n"
234	"1: ldxr %0, %2\n"
235	" subs %0, %0, #1\n"
236	" b.mi 2f\n"
237	" stlxr %w1, %0, %2\n"
238	" cbnz %w1, 1b\n"
239	" dmb ish\n"
240	"2:"
241	: "=&r" (result), "=&r" (tmp), "+Q" (v->counter)
242	:
243	: "cc", "memory");
244
245	return result;
246	}
247
248	static inline int atomic64_add_unless(atomic64_t *v, long a, long u)
249	{
250	long c, old;
251
252	c = atomic64_read(v);
253	while (c != u && (old = atomic64_cmpxchg((v), c, c + a)) != c)
254	c = old;
255
256	return c != u;
257	}
258
259	#define atomic64_add_negative(a, v) (atomic64_add_return((a), (v)) < 0)
260	#define atomic64_inc(v) atomic64_add(1LL, (v))
261	#define atomic64_inc_return(v) atomic64_add_return(1LL, (v))
262	#define atomic64_inc_and_test(v) (atomic64_inc_return(v) == 0)	85	#define atomic64_inc_and_test(v) (atomic64_inc_return(v) == 0)
263	#define atomic64_sub_and_test(a, v) (atomic64_sub_return((a), (v)) == 0)	86	#define atomic64_dec_and_test(v) (atomic64_dec_return(v) == 0)
264	#define atomic64_dec(v) atomic64_sub(1LL, (v))	87	#define atomic64_sub_and_test(i, v) (atomic64_sub_return((i), (v)) == 0)
265	#define atomic64_dec_return(v) atomic64_sub_return(1LL, (v))	88	#define atomic64_add_negative(i, v) (atomic64_add_return((i), (v)) < 0)
266	#define atomic64_dec_and_test(v) (atomic64_dec_return((v)) == 0)	89	#define atomic64_add_unless(v, a, u) (___atomic_add_unless(v, a, u, 64) != u)
267	#define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1LL, 0LL)	90	#define atomic64_andnot atomic64_andnot
		91
		92	#define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1, 0)
268		93
269	#endif	94	#endif
270	#endif	95	#endif

arch/arm64/include/asm/atomic_ll_sc.h+247−0

@@ -0,0 +1,247 @@
	1	/*
	2	* Based on arch/arm/include/asm/atomic.h
	3	*
	4	* Copyright (C) 1996 Russell King.
	5	* Copyright (C) 2002 Deep Blue Solutions Ltd.
	6	* Copyright (C) 2012 ARM Ltd.
	7	*
	8	* This program is free software; you can redistribute it and/or modify
	9	* it under the terms of the GNU General Public License version 2 as
	10	* published by the Free Software Foundation.
	11	*
	12	* This program is distributed in the hope that it will be useful,
	13	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	14	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	15	* GNU General Public License for more details.
	16	*
	17	* You should have received a copy of the GNU General Public License
	18	* along with this program. If not, see <http://www.gnu.org/licenses/>.
	19	*/
	20
	21	#ifndef __ASM_ATOMIC_LL_SC_H
	22	#define __ASM_ATOMIC_LL_SC_H
	23
	24	#ifndef __ARM64_IN_ATOMIC_IMPL
	25	#error "please don't include this file directly"
	26	#endif
	27
	28	/*
	29	* AArch64 UP and SMP safe atomic ops. We use load exclusive and
	30	* store exclusive to ensure that these are atomic. We may loop
	31	* to ensure that the update happens.
	32	*
	33	* NOTE: these functions do not follow the PCS and must explicitly
	34	* save any clobbered registers other than x0 (regardless of return
	35	* value). This is achieved through -fcall-saved-* compiler flags for
	36	* this file, which unfortunately don't work on a per-function basis
	37	* (the optimize attribute silently ignores these options).
	38	*/
	39
	40	#define ATOMIC_OP(op, asm_op) \
	41	__LL_SC_INLINE void \
	42	__LL_SC_PREFIX(atomic_##op(int i, atomic_t *v)) \
	43	{ \
	44	unsigned long tmp; \
	45	int result; \
	46	\
	47	asm volatile("// atomic_" #op "\n" \
	48	" prfm pstl1strm, %2\n" \
	49	"1: ldxr %w0, %2\n" \
	50	" " #asm_op " %w0, %w0, %w3\n" \
	51	" stxr %w1, %w0, %2\n" \
	52	" cbnz %w1, 1b" \
	53	: "=&r" (result), "=&r" (tmp), "+Q" (v->counter) \
	54	: "Ir" (i)); \
	55	} \
	56	__LL_SC_EXPORT(atomic_##op);
	57
	58	#define ATOMIC_OP_RETURN(op, asm_op) \
	59	__LL_SC_INLINE int \
	60	__LL_SC_PREFIX(atomic_##op##_return(int i, atomic_t *v)) \
	61	{ \
	62	unsigned long tmp; \
	63	int result; \
	64	\
	65	asm volatile("// atomic_" #op "_return\n" \
	66	" prfm pstl1strm, %2\n" \
	67	"1: ldxr %w0, %2\n" \
	68	" " #asm_op " %w0, %w0, %w3\n" \
	69	" stlxr %w1, %w0, %2\n" \
	70	" cbnz %w1, 1b" \
	71	: "=&r" (result), "=&r" (tmp), "+Q" (v->counter) \
	72	: "Ir" (i) \
	73	: "memory"); \
	74	\
	75	smp_mb(); \
	76	return result; \
	77	} \
	78	__LL_SC_EXPORT(atomic_##op##_return);
	79
	80	#define ATOMIC_OPS(op, asm_op) \
	81	ATOMIC_OP(op, asm_op) \
	82	ATOMIC_OP_RETURN(op, asm_op)
	83
	84	ATOMIC_OPS(add, add)
	85	ATOMIC_OPS(sub, sub)
	86
	87	ATOMIC_OP(and, and)
	88	ATOMIC_OP(andnot, bic)
	89	ATOMIC_OP(or, orr)
	90	ATOMIC_OP(xor, eor)
	91
	92	#undef ATOMIC_OPS
	93	#undef ATOMIC_OP_RETURN
	94	#undef ATOMIC_OP
	95
	96	#define ATOMIC64_OP(op, asm_op) \
	97	__LL_SC_INLINE void \
	98	__LL_SC_PREFIX(atomic64_##op(long i, atomic64_t *v)) \
	99	{ \
	100	long result; \
	101	unsigned long tmp; \
	102	\
	103	asm volatile("// atomic64_" #op "\n" \
	104	" prfm pstl1strm, %2\n" \
	105	"1: ldxr %0, %2\n" \
	106	" " #asm_op " %0, %0, %3\n" \
	107	" stxr %w1, %0, %2\n" \
	108	" cbnz %w1, 1b" \
	109	: "=&r" (result), "=&r" (tmp), "+Q" (v->counter) \
	110	: "Ir" (i)); \
	111	} \
	112	__LL_SC_EXPORT(atomic64_##op);
	113
	114	#define ATOMIC64_OP_RETURN(op, asm_op) \
	115	__LL_SC_INLINE long \
	116	__LL_SC_PREFIX(atomic64_##op##_return(long i, atomic64_t *v)) \
	117	{ \
	118	long result; \
	119	unsigned long tmp; \
	120	\
	121	asm volatile("// atomic64_" #op "_return\n" \
	122	" prfm pstl1strm, %2\n" \
	123	"1: ldxr %0, %2\n" \
	124	" " #asm_op " %0, %0, %3\n" \
	125	" stlxr %w1, %0, %2\n" \
	126	" cbnz %w1, 1b" \
	127	: "=&r" (result), "=&r" (tmp), "+Q" (v->counter) \
	128	: "Ir" (i) \
	129	: "memory"); \
	130	\
	131	smp_mb(); \
	132	return result; \
	133	} \
	134	__LL_SC_EXPORT(atomic64_##op##_return);
	135
	136	#define ATOMIC64_OPS(op, asm_op) \
	137	ATOMIC64_OP(op, asm_op) \
	138	ATOMIC64_OP_RETURN(op, asm_op)
	139
	140	ATOMIC64_OPS(add, add)
	141	ATOMIC64_OPS(sub, sub)
	142
	143	ATOMIC64_OP(and, and)
	144	ATOMIC64_OP(andnot, bic)
	145	ATOMIC64_OP(or, orr)
	146	ATOMIC64_OP(xor, eor)
	147
	148	#undef ATOMIC64_OPS
	149	#undef ATOMIC64_OP_RETURN
	150	#undef ATOMIC64_OP
	151
	152	__LL_SC_INLINE long
	153	__LL_SC_PREFIX(atomic64_dec_if_positive(atomic64_t *v))
	154	{
	155	long result;
	156	unsigned long tmp;
	157
	158	asm volatile("// atomic64_dec_if_positive\n"
	159	" prfm pstl1strm, %2\n"
	160	"1: ldxr %0, %2\n"
	161	" subs %0, %0, #1\n"
	162	" b.lt 2f\n"
	163	" stlxr %w1, %0, %2\n"
	164	" cbnz %w1, 1b\n"
	165	" dmb ish\n"
	166	"2:"
	167	: "=&r" (result), "=&r" (tmp), "+Q" (v->counter)
	168	:
	169	: "cc", "memory");
	170
	171	return result;
	172	}
	173	__LL_SC_EXPORT(atomic64_dec_if_positive);
	174
	175	#define __CMPXCHG_CASE(w, sz, name, mb, rel, cl) \
	176	__LL_SC_INLINE unsigned long \
	177	__LL_SC_PREFIX(__cmpxchg_case_##name(volatile void *ptr, \
	178	unsigned long old, \
	179	unsigned long new)) \
	180	{ \
	181	unsigned long tmp, oldval; \
	182	\
	183	asm volatile( \
	184	" prfm pstl1strm, %[v]\n" \
	185	"1: ldxr" #sz "\t%" #w "[oldval], %[v]\n" \
	186	" eor %" #w "[tmp], %" #w "[oldval], %" #w "[old]\n" \
	187	" cbnz %" #w "[tmp], 2f\n" \
	188	" st" #rel "xr" #sz "\t%w[tmp], %" #w "[new], %[v]\n" \
	189	" cbnz %w[tmp], 1b\n" \
	190	" " #mb "\n" \
	191	" mov %" #w "[oldval], %" #w "[old]\n" \
	192	"2:" \
	193	: [tmp] "=&r" (tmp), [oldval] "=&r" (oldval), \
	194	[v] "+Q" ((unsigned long )ptr) \
	195	: [old] "Lr" (old), [new] "r" (new) \
	196	: cl); \
	197	\
	198	return oldval; \
	199	} \
	200	__LL_SC_EXPORT(__cmpxchg_case_##name);
	201
	202	__CMPXCHG_CASE(w, b, 1, , , )
	203	__CMPXCHG_CASE(w, h, 2, , , )
	204	__CMPXCHG_CASE(w, , 4, , , )
	205	__CMPXCHG_CASE( , , 8, , , )
	206	__CMPXCHG_CASE(w, b, mb_1, dmb ish, l, "memory")
	207	__CMPXCHG_CASE(w, h, mb_2, dmb ish, l, "memory")
	208	__CMPXCHG_CASE(w, , mb_4, dmb ish, l, "memory")
	209	__CMPXCHG_CASE( , , mb_8, dmb ish, l, "memory")
	210
	211	#undef __CMPXCHG_CASE
	212
	213	#define __CMPXCHG_DBL(name, mb, rel, cl) \
	214	__LL_SC_INLINE int \
	215	__LL_SC_PREFIX(__cmpxchg_double##name(unsigned long old1, \
	216	unsigned long old2, \
	217	unsigned long new1, \
	218	unsigned long new2, \
	219	volatile void *ptr)) \
	220	{ \
	221	unsigned long tmp, ret; \
	222	\
	223	asm volatile("// __cmpxchg_double" #name "\n" \
	224	" prfm pstl1strm, %2\n" \
	225	"1: ldxp %0, %1, %2\n" \
	226	" eor %0, %0, %3\n" \
	227	" eor %1, %1, %4\n" \
	228	" orr %1, %0, %1\n" \
	229	" cbnz %1, 2f\n" \
	230	" st" #rel "xp %w0, %5, %6, %2\n" \
	231	" cbnz %w0, 1b\n" \
	232	" " #mb "\n" \
	233	"2:" \
	234	: "=&r" (tmp), "=&r" (ret), "+Q" ((unsigned long )ptr) \
	235	: "r" (old1), "r" (old2), "r" (new1), "r" (new2) \
	236	: cl); \
	237	\
	238	return ret; \
	239	} \
	240	__LL_SC_EXPORT(__cmpxchg_double##name);
	241
	242	__CMPXCHG_DBL( , , , )
	243	__CMPXCHG_DBL(_mb, dmb ish, l, "memory")
	244
	245	#undef __CMPXCHG_DBL
	246
	247	#endif /* __ASM_ATOMIC_LL_SC_H */

arch/arm64/include/asm/atomic_lse.h+391−0

@@ -0,0 +1,391 @@
	1	/*
	2	* Based on arch/arm/include/asm/atomic.h
	3	*
	4	* Copyright (C) 1996 Russell King.
	5	* Copyright (C) 2002 Deep Blue Solutions Ltd.
	6	* Copyright (C) 2012 ARM Ltd.
	7	*
	8	* This program is free software; you can redistribute it and/or modify
	9	* it under the terms of the GNU General Public License version 2 as
	10	* published by the Free Software Foundation.
	11	*
	12	* This program is distributed in the hope that it will be useful,
	13	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	14	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	15	* GNU General Public License for more details.
	16	*
	17	* You should have received a copy of the GNU General Public License
	18	* along with this program. If not, see <http://www.gnu.org/licenses/>.
	19	*/
	20
	21	#ifndef __ASM_ATOMIC_LSE_H
	22	#define __ASM_ATOMIC_LSE_H
	23
	24	#ifndef __ARM64_IN_ATOMIC_IMPL
	25	#error "please don't include this file directly"
	26	#endif
	27
	28	#define __LL_SC_ATOMIC(op) __LL_SC_CALL(atomic_##op)
	29
	30	static inline void atomic_andnot(int i, atomic_t *v)
	31	{
	32	register int w0 asm ("w0") = i;
	33	register atomic_t *x1 asm ("x1") = v;
	34
	35	asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC(andnot),
	36	" stclr %w[i], %[v]\n")
	37	: [i] "+r" (w0), [v] "+Q" (v->counter)
	38	: "r" (x1)
	39	: "x30");
	40	}
	41
	42	static inline void atomic_or(int i, atomic_t *v)
	43	{
	44	register int w0 asm ("w0") = i;
	45	register atomic_t *x1 asm ("x1") = v;
	46
	47	asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC(or),
	48	" stset %w[i], %[v]\n")
	49	: [i] "+r" (w0), [v] "+Q" (v->counter)
	50	: "r" (x1)
	51	: "x30");
	52	}
	53
	54	static inline void atomic_xor(int i, atomic_t *v)
	55	{
	56	register int w0 asm ("w0") = i;
	57	register atomic_t *x1 asm ("x1") = v;
	58
	59	asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC(xor),
	60	" steor %w[i], %[v]\n")
	61	: [i] "+r" (w0), [v] "+Q" (v->counter)
	62	: "r" (x1)
	63	: "x30");
	64	}
	65
	66	static inline void atomic_add(int i, atomic_t *v)
	67	{
	68	register int w0 asm ("w0") = i;
	69	register atomic_t *x1 asm ("x1") = v;
	70
	71	asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC(add),
	72	" stadd %w[i], %[v]\n")
	73	: [i] "+r" (w0), [v] "+Q" (v->counter)
	74	: "r" (x1)
	75	: "x30");
	76	}
	77
	78	static inline int atomic_add_return(int i, atomic_t *v)
	79	{
	80	register int w0 asm ("w0") = i;
	81	register atomic_t *x1 asm ("x1") = v;
	82
	83	asm volatile(ARM64_LSE_ATOMIC_INSN(
	84	/* LL/SC */
	85	" nop\n"
	86	__LL_SC_ATOMIC(add_return),
	87	/* LSE atomics */
	88	" ldaddal %w[i], w30, %[v]\n"
	89	" add %w[i], %w[i], w30")
	90	: [i] "+r" (w0), [v] "+Q" (v->counter)
	91	: "r" (x1)
	92	: "x30", "memory");
	93
	94	return w0;
	95	}
	96
	97	static inline void atomic_and(int i, atomic_t *v)
	98	{
	99	register int w0 asm ("w0") = i;
	100	register atomic_t *x1 asm ("x1") = v;
	101
	102	asm volatile(ARM64_LSE_ATOMIC_INSN(
	103	/* LL/SC */
	104	" nop\n"
	105	__LL_SC_ATOMIC(and),
	106	/* LSE atomics */
	107	" mvn %w[i], %w[i]\n"
	108	" stclr %w[i], %[v]")
	109	: [i] "+r" (w0), [v] "+Q" (v->counter)
	110	: "r" (x1)
	111	: "x30");
	112	}
	113
	114	static inline void atomic_sub(int i, atomic_t *v)
	115	{
	116	register int w0 asm ("w0") = i;
	117	register atomic_t *x1 asm ("x1") = v;
	118
	119	asm volatile(ARM64_LSE_ATOMIC_INSN(
	120	/* LL/SC */
	121	" nop\n"
	122	__LL_SC_ATOMIC(sub),
	123	/* LSE atomics */
	124	" neg %w[i], %w[i]\n"
	125	" stadd %w[i], %[v]")
	126	: [i] "+r" (w0), [v] "+Q" (v->counter)
	127	: "r" (x1)
	128	: "x30");
	129	}
	130
	131	static inline int atomic_sub_return(int i, atomic_t *v)
	132	{
	133	register int w0 asm ("w0") = i;
	134	register atomic_t *x1 asm ("x1") = v;
	135
	136	asm volatile(ARM64_LSE_ATOMIC_INSN(
	137	/* LL/SC */
	138	" nop\n"
	139	__LL_SC_ATOMIC(sub_return)
	140	" nop",
	141	/* LSE atomics */
	142	" neg %w[i], %w[i]\n"
	143	" ldaddal %w[i], w30, %[v]\n"
	144	" add %w[i], %w[i], w30")
	145	: [i] "+r" (w0), [v] "+Q" (v->counter)
	146	: "r" (x1)
	147	: "x30", "memory");
	148
	149	return w0;
	150	}
	151
	152	#undef __LL_SC_ATOMIC
	153
	154	#define __LL_SC_ATOMIC64(op) __LL_SC_CALL(atomic64_##op)
	155
	156	static inline void atomic64_andnot(long i, atomic64_t *v)
	157	{
	158	register long x0 asm ("x0") = i;
	159	register atomic64_t *x1 asm ("x1") = v;
	160
	161	asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC64(andnot),
	162	" stclr %[i], %[v]\n")
	163	: [i] "+r" (x0), [v] "+Q" (v->counter)
	164	: "r" (x1)
	165	: "x30");
	166	}
	167
	168	static inline void atomic64_or(long i, atomic64_t *v)
	169	{
	170	register long x0 asm ("x0") = i;
	171	register atomic64_t *x1 asm ("x1") = v;
	172
	173	asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC64(or),
	174	" stset %[i], %[v]\n")
	175	: [i] "+r" (x0), [v] "+Q" (v->counter)
	176	: "r" (x1)
	177	: "x30");
	178	}
	179
	180	static inline void atomic64_xor(long i, atomic64_t *v)
	181	{
	182	register long x0 asm ("x0") = i;
	183	register atomic64_t *x1 asm ("x1") = v;
	184
	185	asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC64(xor),
	186	" steor %[i], %[v]\n")
	187	: [i] "+r" (x0), [v] "+Q" (v->counter)
	188	: "r" (x1)
	189	: "x30");
	190	}
	191
	192	static inline void atomic64_add(long i, atomic64_t *v)
	193	{
	194	register long x0 asm ("x0") = i;
	195	register atomic64_t *x1 asm ("x1") = v;
	196
	197	asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC64(add),
	198	" stadd %[i], %[v]\n")
	199	: [i] "+r" (x0), [v] "+Q" (v->counter)
	200	: "r" (x1)
	201	: "x30");
	202	}
	203
	204	static inline long atomic64_add_return(long i, atomic64_t *v)
	205	{
	206	register long x0 asm ("x0") = i;
	207	register atomic64_t *x1 asm ("x1") = v;
	208
	209	asm volatile(ARM64_LSE_ATOMIC_INSN(
	210	/* LL/SC */
	211	" nop\n"
	212	__LL_SC_ATOMIC64(add_return),
	213	/* LSE atomics */
	214	" ldaddal %[i], x30, %[v]\n"
	215	" add %[i], %[i], x30")
	216	: [i] "+r" (x0), [v] "+Q" (v->counter)
	217	: "r" (x1)
	218	: "x30", "memory");
	219
	220	return x0;
	221	}
	222
	223	static inline void atomic64_and(long i, atomic64_t *v)
	224	{
	225	register long x0 asm ("x0") = i;
	226	register atomic64_t *x1 asm ("x1") = v;
	227
	228	asm volatile(ARM64_LSE_ATOMIC_INSN(
	229	/* LL/SC */
	230	" nop\n"
	231	__LL_SC_ATOMIC64(and),
	232	/* LSE atomics */
	233	" mvn %[i], %[i]\n"
	234	" stclr %[i], %[v]")
	235	: [i] "+r" (x0), [v] "+Q" (v->counter)
	236	: "r" (x1)
	237	: "x30");
	238	}
	239
	240	static inline void atomic64_sub(long i, atomic64_t *v)
	241	{
	242	register long x0 asm ("x0") = i;
	243	register atomic64_t *x1 asm ("x1") = v;
	244
	245	asm volatile(ARM64_LSE_ATOMIC_INSN(
	246	/* LL/SC */
	247	" nop\n"
	248	__LL_SC_ATOMIC64(sub),
	249	/* LSE atomics */
	250	" neg %[i], %[i]\n"
	251	" stadd %[i], %[v]")
	252	: [i] "+r" (x0), [v] "+Q" (v->counter)
	253	: "r" (x1)
	254	: "x30");
	255	}
	256
	257	static inline long atomic64_sub_return(long i, atomic64_t *v)
	258	{
	259	register long x0 asm ("x0") = i;
	260	register atomic64_t *x1 asm ("x1") = v;
	261
	262	asm volatile(ARM64_LSE_ATOMIC_INSN(
	263	/* LL/SC */
	264	" nop\n"
	265	__LL_SC_ATOMIC64(sub_return)
	266	" nop",
	267	/* LSE atomics */
	268	" neg %[i], %[i]\n"
	269	" ldaddal %[i], x30, %[v]\n"
	270	" add %[i], %[i], x30")
	271	: [i] "+r" (x0), [v] "+Q" (v->counter)
	272	: "r" (x1)
	273	: "x30", "memory");
	274
	275	return x0;
	276	}
	277
	278	static inline long atomic64_dec_if_positive(atomic64_t *v)
	279	{
	280	register long x0 asm ("x0") = (long)v;
	281
	282	asm volatile(ARM64_LSE_ATOMIC_INSN(
	283	/* LL/SC */
	284	" nop\n"
	285	__LL_SC_ATOMIC64(dec_if_positive)
	286	" nop\n"
	287	" nop\n"
	288	" nop\n"
	289	" nop\n"
	290	" nop",
	291	/* LSE atomics */
	292	"1: ldr x30, %[v]\n"
	293	" subs %[ret], x30, #1\n"
	294	" b.lt 2f\n"
	295	" casal x30, %[ret], %[v]\n"
	296	" sub x30, x30, #1\n"
	297	" sub x30, x30, %[ret]\n"
	298	" cbnz x30, 1b\n"
	299	"2:")
	300	: [ret] "+&r" (x0), [v] "+Q" (v->counter)
	301	:
	302	: "x30", "cc", "memory");
	303
	304	return x0;
	305	}
	306
	307	#undef __LL_SC_ATOMIC64
	308
	309	#define __LL_SC_CMPXCHG(op) __LL_SC_CALL(__cmpxchg_case_##op)
	310
	311	#define __CMPXCHG_CASE(w, sz, name, mb, cl...) \
	312	static inline unsigned long __cmpxchg_case_##name(volatile void *ptr, \
	313	unsigned long old, \
	314	unsigned long new) \
	315	{ \
	316	register unsigned long x0 asm ("x0") = (unsigned long)ptr; \
	317	register unsigned long x1 asm ("x1") = old; \
	318	register unsigned long x2 asm ("x2") = new; \
	319	\
	320	asm volatile(ARM64_LSE_ATOMIC_INSN( \
	321	/* LL/SC */ \
	322	" nop\n" \
	323	__LL_SC_CMPXCHG(name) \
	324	" nop", \
	325	/* LSE atomics */ \
	326	" mov " #w "30, %" #w "[old]\n" \
	327	" cas" #mb #sz "\t" #w "30, %" #w "[new], %[v]\n" \
	328	" mov %" #w "[ret], " #w "30") \
	329	: [ret] "+r" (x0), [v] "+Q" ((unsigned long )ptr) \
	330	: [old] "r" (x1), [new] "r" (x2) \
	331	: "x30" , ##cl); \
	332	\
	333	return x0; \
	334	}
	335
	336	__CMPXCHG_CASE(w, b, 1, )
	337	__CMPXCHG_CASE(w, h, 2, )
	338	__CMPXCHG_CASE(w, , 4, )
	339	__CMPXCHG_CASE(x, , 8, )
	340	__CMPXCHG_CASE(w, b, mb_1, al, "memory")
	341	__CMPXCHG_CASE(w, h, mb_2, al, "memory")
	342	__CMPXCHG_CASE(w, , mb_4, al, "memory")
	343	__CMPXCHG_CASE(x, , mb_8, al, "memory")
	344
	345	#undef __LL_SC_CMPXCHG
	346	#undef __CMPXCHG_CASE
	347
	348	#define __LL_SC_CMPXCHG_DBL(op) __LL_SC_CALL(__cmpxchg_double##op)
	349
	350	#define __CMPXCHG_DBL(name, mb, cl...) \
	351	static inline int __cmpxchg_double##name(unsigned long old1, \
	352	unsigned long old2, \
	353	unsigned long new1, \
	354	unsigned long new2, \
	355	volatile void *ptr) \
	356	{ \
	357	unsigned long oldval1 = old1; \
	358	unsigned long oldval2 = old2; \
	359	register unsigned long x0 asm ("x0") = old1; \
	360	register unsigned long x1 asm ("x1") = old2; \
	361	register unsigned long x2 asm ("x2") = new1; \
	362	register unsigned long x3 asm ("x3") = new2; \
	363	register unsigned long x4 asm ("x4") = (unsigned long)ptr; \
	364	\
	365	asm volatile(ARM64_LSE_ATOMIC_INSN( \
	366	/* LL/SC */ \
	367	" nop\n" \
	368	" nop\n" \
	369	" nop\n" \
	370	__LL_SC_CMPXCHG_DBL(name), \
	371	/* LSE atomics */ \
	372	" casp" #mb "\t%[old1], %[old2], %[new1], %[new2], %[v]\n"\
	373	" eor %[old1], %[old1], %[oldval1]\n" \
	374	" eor %[old2], %[old2], %[oldval2]\n" \
	375	" orr %[old1], %[old1], %[old2]") \
	376	: [old1] "+r" (x0), [old2] "+r" (x1), \
	377	[v] "+Q" ((unsigned long )ptr) \
	378	: [new1] "r" (x2), [new2] "r" (x3), [ptr] "r" (x4), \
	379	[oldval1] "r" (oldval1), [oldval2] "r" (oldval2) \
	380	: "x30" , ##cl); \
	381	\
	382	return x0; \
	383	}
	384
	385	__CMPXCHG_DBL( , )
	386	__CMPXCHG_DBL(_mb, al, "memory")
	387
	388	#undef __LL_SC_CMPXCHG_DBL
	389	#undef __CMPXCHG_DBL
	390
	391	#endif /* __ASM_ATOMIC_LSE_H */

arch/arm64/include/asm/barrier.h+0−24

@@ -35,28 +35,6 @@ · 34 unmodified lines
35	#define dma_rmb() dmb(oshld)	35	#define dma_rmb() dmb(oshld)
36	#define dma_wmb() dmb(oshst)	36	#define dma_wmb() dmb(oshst)
37		37
38	#ifndef CONFIG_SMP
39	#define smp_mb() barrier()
40	#define smp_rmb() barrier()
41	#define smp_wmb() barrier()
42
43	#define smp_store_release(p, v) \
44	do { \
45	compiletime_assert_atomic_type(*p); \
46	barrier(); \
47	WRITE_ONCE(*p, v); \
48	} while (0)
49
50	#define smp_load_acquire(p) \
51	({ \
52	typeof(p) ___p1 = READ_ONCE(p); \
53	compiletime_assert_atomic_type(*p); \
54	barrier(); \
55	___p1; \
56	})
57
58	#else
59
60	#define smp_mb() dmb(ish)	38	#define smp_mb() dmb(ish)
61	#define smp_rmb() dmb(ishld)	39	#define smp_rmb() dmb(ishld)
62	#define smp_wmb() dmb(ishst)	40	#define smp_wmb() dmb(ishst)
@@ -109,8 +87,6 @@ do { \ · 46 unmodified lines
109	___p1; \	87	___p1; \
110	})	88	})
111		89
112	#endif
113
114	#define read_barrier_depends() do { } while(0)	90	#define read_barrier_depends() do { } while(0)
115	#define smp_read_barrier_depends() do { } while(0)	91	#define smp_read_barrier_depends() do { } while(0)
116		92

arch/arm64/include/asm/bug.h+64−0

@@ -0,0 +1,64 @@
	1	/*
	2	* Copyright (C) 2015 ARM Limited
	3	* Author: Dave Martin <Dave.Martin@arm.com>
	4	*
	5	* This program is free software; you can redistribute it and/or modify
	6	* it under the terms of the GNU General Public License version 2 as
	7	* published by the Free Software Foundation.
	8	*
	9	* This program is distributed in the hope that it will be useful,
	10	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	11	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	12	* GNU General Public License for more details.
	13	*
	14	* You should have received a copy of the GNU General Public License
	15	* along with this program. If not, see <http://www.gnu.org/licenses/>.
	16	*/
	17
	18	#ifndef _ARCH_ARM64_ASM_BUG_H
	19	#define _ARCH_ARM64_ASM_BUG_H
	20
	21	#include <asm/debug-monitors.h>
	22
	23	#ifdef CONFIG_GENERIC_BUG
	24	#define HAVE_ARCH_BUG
	25
	26	#ifdef CONFIG_DEBUG_BUGVERBOSE
	27	#define _BUGVERBOSE_LOCATION(file, line) __BUGVERBOSE_LOCATION(file, line)
	28	#define __BUGVERBOSE_LOCATION(file, line) \
	29	".pushsection .rodata.str,\"aMS\",@progbits,1\n" \
	30	"2: .string \"" file "\"\n\t" \
	31	".popsection\n\t" \
	32	\
	33	".long 2b - 0b\n\t" \
	34	".short " #line "\n\t"
	35	#else
	36	#define _BUGVERBOSE_LOCATION(file, line)
	37	#endif
	38
	39	#define _BUG_FLAGS(flags) __BUG_FLAGS(flags)
	40
	41	#define __BUG_FLAGS(flags) asm volatile ( \
	42	".pushsection __bug_table,\"a\"\n\t" \
	43	".align 2\n\t" \
	44	"0: .long 1f - 0b\n\t" \
	45	_BUGVERBOSE_LOCATION(__FILE__, __LINE__) \
	46	".short " #flags "\n\t" \
	47	".popsection\n" \
	48	\
	49	"1: brk %[imm]" \
	50	:: [imm] "i" (BUG_BRK_IMM) \
	51	)
	52
	53	#define BUG() do { \
	54	_BUG_FLAGS(0); \
	55	unreachable(); \
	56	} while (0)
	57
	58	#define __WARN_TAINT(taint) _BUG_FLAGS(BUGFLAG_TAINT(taint))
	59
	60	#endif /* ! CONFIG_GENERIC_BUG */
	61
	62	#include <asm-generic/bug.h>
	63
	64	#endif /* ! _ARCH_ARM64_ASM_BUG_H */

arch/arm64/include/asm/cmpxchg.h+74−118

@@ -21,7 +21,9 @@ · 20 unmodified lines
21	#include <linux/bug.h>	21	#include <linux/bug.h>
22	#include <linux/mmdebug.h>	22	#include <linux/mmdebug.h>
23		23
		24	#include <asm/atomic.h>
24	#include <asm/barrier.h>	25	#include <asm/barrier.h>
		26	#include <asm/lse.h>
25		27
26	static inline unsigned long __xchg(unsigned long x, volatile void *ptr, int size)	28	static inline unsigned long __xchg(unsigned long x, volatile void *ptr, int size)
27	{	29	{
@@ -29,37 +31,73 @@ static inline unsigned long __xchg(unsigned long x, volatile void *ptr, int size · 1 unmodified line
29		31
30	switch (size) {	32	switch (size) {
31	case 1:	33	case 1:
32	asm volatile("// __xchg1\n"	34	asm volatile(ARM64_LSE_ATOMIC_INSN(
		35	/* LL/SC */
		36	" prfm pstl1strm, %2\n"
33	"1: ldxrb %w0, %2\n"	37	"1: ldxrb %w0, %2\n"
34	" stlxrb %w1, %w3, %2\n"	38	" stlxrb %w1, %w3, %2\n"
35	" cbnz %w1, 1b\n"	39	" cbnz %w1, 1b\n"
		40	" dmb ish",
		41	/* LSE atomics */
		42	" nop\n"
		43	" nop\n"
		44	" swpalb %w3, %w0, %2\n"
		45	" nop\n"
		46	" nop")
36	: "=&r" (ret), "=&r" (tmp), "+Q" ((u8 )ptr)	47	: "=&r" (ret), "=&r" (tmp), "+Q" ((u8 )ptr)
37	: "r" (x)	48	: "r" (x)
38	: "memory");	49	: "memory");
39	break;	50	break;
40	case 2:	51	case 2:
41	asm volatile("// __xchg2\n"	52	asm volatile(ARM64_LSE_ATOMIC_INSN(
		53	/* LL/SC */
		54	" prfm pstl1strm, %2\n"
42	"1: ldxrh %w0, %2\n"	55	"1: ldxrh %w0, %2\n"
43	" stlxrh %w1, %w3, %2\n"	56	" stlxrh %w1, %w3, %2\n"
44	" cbnz %w1, 1b\n"	57	" cbnz %w1, 1b\n"
		58	" dmb ish",
		59	/* LSE atomics */
		60	" nop\n"
		61	" nop\n"
		62	" swpalh %w3, %w0, %2\n"
		63	" nop\n"
		64	" nop")
45	: "=&r" (ret), "=&r" (tmp), "+Q" ((u16 )ptr)	65	: "=&r" (ret), "=&r" (tmp), "+Q" ((u16 )ptr)
46	: "r" (x)	66	: "r" (x)
47	: "memory");	67	: "memory");
48	break;	68	break;
49	case 4:	69	case 4:
50	asm volatile("// __xchg4\n"	70	asm volatile(ARM64_LSE_ATOMIC_INSN(
		71	/* LL/SC */
		72	" prfm pstl1strm, %2\n"
51	"1: ldxr %w0, %2\n"	73	"1: ldxr %w0, %2\n"
52	" stlxr %w1, %w3, %2\n"	74	" stlxr %w1, %w3, %2\n"
53	" cbnz %w1, 1b\n"	75	" cbnz %w1, 1b\n"
		76	" dmb ish",
		77	/* LSE atomics */
		78	" nop\n"
		79	" nop\n"
		80	" swpal %w3, %w0, %2\n"
		81	" nop\n"
		82	" nop")
54	: "=&r" (ret), "=&r" (tmp), "+Q" ((u32 )ptr)	83	: "=&r" (ret), "=&r" (tmp), "+Q" ((u32 )ptr)
55	: "r" (x)	84	: "r" (x)
56	: "memory");	85	: "memory");
57	break;	86	break;
58	case 8:	87	case 8:
59	asm volatile("// __xchg8\n"	88	asm volatile(ARM64_LSE_ATOMIC_INSN(
		89	/* LL/SC */
		90	" prfm pstl1strm, %2\n"
60	"1: ldxr %0, %2\n"	91	"1: ldxr %0, %2\n"
61	" stlxr %w1, %3, %2\n"	92	" stlxr %w1, %3, %2\n"
62	" cbnz %w1, 1b\n"	93	" cbnz %w1, 1b\n"
		94	" dmb ish",
		95	/* LSE atomics */
		96	" nop\n"
		97	" nop\n"
		98	" swpal %3, %0, %2\n"
		99	" nop\n"
		100	" nop")
63	: "=&r" (ret), "=&r" (tmp), "+Q" ((u64 )ptr)	101	: "=&r" (ret), "=&r" (tmp), "+Q" ((u64 )ptr)
64	: "r" (x)	102	: "r" (x)
65	: "memory");	103	: "memory");
@@ -68,7 +106,6 @@ static inline unsigned long __xchg(unsigned long x, volatile void *ptr, int size · 2 unmodified lines
68	BUILD_BUG();	106	BUILD_BUG();
69	}	107	}
70		108
71	smp_mb();
72	return ret;	109	return ret;
73	}	110	}
74		111
@@ -83,131 +120,39 @@ static inline unsigned long __xchg(unsigned long x, volatile void *ptr, int size · 8 unmodified lines
83	static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old,	120	static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old,
84	unsigned long new, int size)	121	unsigned long new, int size)
85	{	122	{
86	unsigned long oldval = 0, res;
87
88	switch (size) {	123	switch (size) {
89	case 1:	124	case 1:
90	do {	125	return __cmpxchg_case_1(ptr, (u8)old, new);
91	asm volatile("// __cmpxchg1\n"
92	" ldxrb %w1, %2\n"
93	" mov %w0, #0\n"
94	" cmp %w1, %w3\n"
95	" b.ne 1f\n"
96	" stxrb %w0, %w4, %2\n"
97	"1:\n"
98	: "=&r" (res), "=&r" (oldval), "+Q" ((u8 )ptr)
99	: "Ir" (old), "r" (new)
100	: "cc");
101	} while (res);
102	break;
103
104	case 2:	126	case 2:
105	do {	127	return __cmpxchg_case_2(ptr, (u16)old, new);
106	asm volatile("// __cmpxchg2\n"
107	" ldxrh %w1, %2\n"
108	" mov %w0, #0\n"
109	" cmp %w1, %w3\n"
110	" b.ne 1f\n"
111	" stxrh %w0, %w4, %2\n"
112	"1:\n"
113	: "=&r" (res), "=&r" (oldval), "+Q" ((u16 )ptr)
114	: "Ir" (old), "r" (new)
115	: "cc");
116	} while (res);
117	break;
118
119	case 4:	128	case 4:
120	do {	129	return __cmpxchg_case_4(ptr, old, new);
121	asm volatile("// __cmpxchg4\n"
122	" ldxr %w1, %2\n"
123	" mov %w0, #0\n"
124	" cmp %w1, %w3\n"
125	" b.ne 1f\n"
126	" stxr %w0, %w4, %2\n"
127	"1:\n"
128	: "=&r" (res), "=&r" (oldval), "+Q" ((u32 )ptr)
129	: "Ir" (old), "r" (new)
130	: "cc");
131	} while (res);
132	break;
133
134	case 8:	130	case 8:
135	do {	131	return __cmpxchg_case_8(ptr, old, new);
136	asm volatile("// __cmpxchg8\n"
137	" ldxr %1, %2\n"
138	" mov %w0, #0\n"
139	" cmp %1, %3\n"
140	" b.ne 1f\n"
141	" stxr %w0, %4, %2\n"
142	"1:\n"
143	: "=&r" (res), "=&r" (oldval), "+Q" ((u64 )ptr)
144	: "Ir" (old), "r" (new)
145	: "cc");
146	} while (res);
147	break;
148
149	default:	132	default:
150	BUILD_BUG();	133	BUILD_BUG();
151	}	134	}
152		135
153	return oldval;	136	unreachable();
154	}	137	}
155		138
156	#define system_has_cmpxchg_double() 1	139	static inline unsigned long __cmpxchg_mb(volatile void *ptr, unsigned long old,
157		140	unsigned long new, int size)
158	static inline int __cmpxchg_double(volatile void ptr1, volatile void ptr2,
159	unsigned long old1, unsigned long old2,
160	unsigned long new1, unsigned long new2, int size)
161	{	141	{
162	unsigned long loop, lost;
163
164	switch (size) {	142	switch (size) {
		143	case 1:
		144	return __cmpxchg_case_mb_1(ptr, (u8)old, new);
		145	case 2:
		146	return __cmpxchg_case_mb_2(ptr, (u16)old, new);
		147	case 4:
		148	return __cmpxchg_case_mb_4(ptr, old, new);
165	case 8:	149	case 8:
166	VM_BUG_ON((unsigned long )ptr2 - (unsigned long )ptr1 != 1);	150	return __cmpxchg_case_mb_8(ptr, old, new);
167	do {
168	asm volatile("// __cmpxchg_double8\n"
169	" ldxp %0, %1, %2\n"
170	" eor %0, %0, %3\n"
171	" eor %1, %1, %4\n"
172	" orr %1, %0, %1\n"
173	" mov %w0, #0\n"
174	" cbnz %1, 1f\n"
175	" stxp %w0, %5, %6, %2\n"
176	"1:\n"
177	: "=&r"(loop), "=&r"(lost), "+Q" ((u64 )ptr1)
178	: "r" (old1), "r"(old2), "r"(new1), "r"(new2));
179	} while (loop);
180	break;
181	default:	151	default:
182	BUILD_BUG();	152	BUILD_BUG();
183	}	153	}
184		154
185	return !lost;	155	unreachable();
186	}
187
188	static inline int __cmpxchg_double_mb(volatile void ptr1, volatile void ptr2,
189	unsigned long old1, unsigned long old2,
190	unsigned long new1, unsigned long new2, int size)
191	{
192	int ret;
193
194	smp_mb();
195	ret = __cmpxchg_double(ptr1, ptr2, old1, old2, new1, new2, size);
196	smp_mb();
197
198	return ret;
199	}
200
201	static inline unsigned long __cmpxchg_mb(volatile void *ptr, unsigned long old,
202	unsigned long new, int size)
203	{
204	unsigned long ret;
205
206	smp_mb();
207	ret = __cmpxchg(ptr, old, new, size);
208	smp_mb();
209
210	return ret;
211	}	156	}
212		157
213	#define cmpxchg(ptr, o, n) \	158	#define cmpxchg(ptr, o, n) \
@@ -228,21 +173,32 @@ static inline unsigned long __cmpxchg_mb(volatile void *ptr, unsigned long old, · 14 unmodified lines
228	__ret; \	173	__ret; \
229	})	174	})
230		175
		176	#define system_has_cmpxchg_double() 1
		177
		178	#define __cmpxchg_double_check(ptr1, ptr2) \
		179	({ \
		180	if (sizeof(*(ptr1)) != 8) \
		181	BUILD_BUG(); \
		182	VM_BUG_ON((unsigned long )(ptr2) - (unsigned long )(ptr1) != 1); \
		183	})
		184
231	#define cmpxchg_double(ptr1, ptr2, o1, o2, n1, n2) \	185	#define cmpxchg_double(ptr1, ptr2, o1, o2, n1, n2) \
232	({\	186	({\
233	int __ret;\	187	int __ret;\
234	__ret = __cmpxchg_double_mb((ptr1), (ptr2), (unsigned long)(o1), \	188	__cmpxchg_double_check(ptr1, ptr2); \
235	(unsigned long)(o2), (unsigned long)(n1), \	189	__ret = !__cmpxchg_double_mb((unsigned long)(o1), (unsigned long)(o2), \
236	(unsigned long)(n2), sizeof(*(ptr1)));\	190	(unsigned long)(n1), (unsigned long)(n2), \
		191	ptr1); \
237	__ret; \	192	__ret; \
238	})	193	})
239		194
240	#define cmpxchg_double_local(ptr1, ptr2, o1, o2, n1, n2) \	195	#define cmpxchg_double_local(ptr1, ptr2, o1, o2, n1, n2) \
241	({\	196	({\
242	int __ret;\	197	int __ret;\
243	__ret = __cmpxchg_double((ptr1), (ptr2), (unsigned long)(o1), \	198	__cmpxchg_double_check(ptr1, ptr2); \
244	(unsigned long)(o2), (unsigned long)(n1), \	199	__ret = !__cmpxchg_double((unsigned long)(o1), (unsigned long)(o2), \
245	(unsigned long)(n2), sizeof(*(ptr1)));\	200	(unsigned long)(n1), (unsigned long)(n2), \
		201	ptr1); \
246	__ret; \	202	__ret; \
247	})	203	})
248		204

arch/arm64/include/asm/cpufeature.h+15−3

 #define ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE	1
 #define ARM64_WORKAROUND_845719			2
 #define ARM64_HAS_SYSREG_GIC_CPUIF		3
+#define ARM64_HAS_PAN				4
+#define ARM64_HAS_LSE_ATOMICS			5
-#define ARM64_NCAPS				4
+#define ARM64_NCAPS				6
 #ifndef __ASSEMBLY__
+#include <linux/kernel.h>
 struct arm64_cpu_capabilities {
 	const char *desc;
 	u16 capability;
 	bool (*matches)(const struct arm64_cpu_capabilities *);
+	void (*enable)(void);
 	union {
 		struct {	/* To be used for erratum handling only */
 			u32 midr_model;
 		};
 		struct {	/* Feature register checking */
-			u64 register_mask;
+			int field_pos;
-			u64 register_value;
+			int min_field_value;
 		};
 	};
 };
 		__set_bit(num, cpu_hwcaps);
 }
+static inline int __attribute_const__ cpuid_feature_extract_field(u64 features,
+								  int field)
+{
+	return (s64)(features << (64 - 4 - field)) >> (64 - 4);
+}
 void check_cpu_capabilities(const struct arm64_cpu_capabilities *caps,
 			    const char *info);
 void check_local_cpu_errata(void);

arch/arm64/include/asm/cputype.h+0−3

@@ -81,9 +81,6 @@ · 80 unmodified lines
81	#define ID_AA64MMFR0_BIGEND(mmfr0) \	81	#define ID_AA64MMFR0_BIGEND(mmfr0) \
82	(((mmfr0) & ID_AA64MMFR0_BIGEND_MASK) >> ID_AA64MMFR0_BIGEND_SHIFT)	82	(((mmfr0) & ID_AA64MMFR0_BIGEND_MASK) >> ID_AA64MMFR0_BIGEND_SHIFT)
83		83
84	#define SCTLR_EL1_CP15BEN (0x1 << 5)
85	#define SCTLR_EL1_SED (0x1 << 8)
86
87	#ifndef __ASSEMBLY__	84	#ifndef __ASSEMBLY__
88		85
89	/*	86	/*

arch/arm64/include/asm/debug-monitors.h+16−28

 #ifdef __KERNEL__
+#include <linux/errno.h>
+#include <linux/types.h>
+#include <asm/esr.h>
+#include <asm/insn.h>
+#include <asm/ptrace.h>
 /* Low-level stepping controls. */
 #define DBG_MDSCR_SS		(1 << 0)
 #define DBG_SPSR_SS		(1 << 21)
 /*
  * Break point instruction encoding
  */
-#define BREAK_INSTR_SIZE		4
+#define BREAK_INSTR_SIZE		AARCH64_INSN_SIZE
-/*
- * ESR values expected for dynamic and compile time BRK instruction
- */
-#define DBG_ESR_VAL_BRK(x)	(0xf2000000 | ((x) & 0xfffff))
 /*
  * #imm16 values used for BRK instruction generation
  * 0x100: for triggering a fault on purpose (reserved)
  * 0x400: for dynamic BRK instruction
  * 0x401: for compile time BRK instruction
+ * 0x800: kernel-mode BUG() and WARN() traps
  */
 #define FAULT_BRK_IMM			0x100
 #define KGDB_DYN_DBG_BRK_IMM		0x400
 #define KGDB_COMPILED_DBG_BRK_IMM	0x401
+#define BUG_BRK_IMM			0x800
 /*
  * BRK instruction encoding
  */
 #define AARCH64_BREAK_FAULT	(AARCH64_BREAK_MON | (FAULT_BRK_IMM << 5))
-/*
+#define AARCH64_BREAK_KGDB_DYN_DBG	\
- * Extract byte from BRK instruction
+	(AARCH64_BREAK_MON | (KGDB_DYN_DBG_BRK_IMM << 5))
- */
+#define KGDB_DYN_BRK_INS_BYTE(x)	\
-#define KGDB_DYN_DBG_BRK_INS_BYTE(x) \
+	((AARCH64_BREAK_KGDB_DYN_DBG >> (8 * (x))) & 0xff)
-	((((AARCH64_BREAK_MON) & 0xffe0001f) >> (x * 8)) & 0xff)
-/*
- * Extract byte from BRK #imm16
- */
-#define KGBD_DYN_DBG_BRK_IMM_BYTE(x) \
-	(((((KGDB_DYN_DBG_BRK_IMM) & 0xffff) << 5) >> (x * 8)) & 0xff)
-#define KGDB_DYN_DBG_BRK_BYTE(x) \
-	(KGDB_DYN_DBG_BRK_INS_BYTE(x) | KGBD_DYN_DBG_BRK_IMM_BYTE(x))
-#define  KGDB_DYN_BRK_INS_BYTE0  KGDB_DYN_DBG_BRK_BYTE(0)
-#define  KGDB_DYN_BRK_INS_BYTE1  KGDB_DYN_DBG_BRK_BYTE(1)
-#define  KGDB_DYN_BRK_INS_BYTE2  KGDB_DYN_DBG_BRK_BYTE(2)
-#define  KGDB_DYN_BRK_INS_BYTE3  KGDB_DYN_DBG_BRK_BYTE(3)
 #define CACHE_FLUSH_IS_SAFE		1
 u8 debug_monitors_arch(void);
-enum debug_el {
+enum dbg_active_el {
 	DBG_ACTIVE_EL0 = 0,
 	DBG_ACTIVE_EL1,
 };
-void enable_debug_monitors(enum debug_el el);
+void enable_debug_monitors(enum dbg_active_el el);
-void disable_debug_monitors(enum debug_el el);
+void disable_debug_monitors(enum dbg_active_el el);
 void user_rewind_single_step(struct task_struct *task);
 void user_fastforward_single_step(struct task_struct *task);

arch/arm64/include/asm/esr.h+9−0

 #ifndef __ASM_ESR_H
 #define __ASM_ESR_H
+#include <asm/memory.h>
 #define ESR_ELx_EC_UNKNOWN	(0x00)
 #define ESR_ELx_EC_WFx		(0x01)
 /* Unallocated EC: 0x02 */
 #define ESR_ELx_WFx_ISS_WFE	(UL(1) << 0)
 #define ESR_ELx_xVC_IMM_MASK	((1UL << 16) - 1)
+/* ESR value templates for specific events */
+/* BRK instruction trap from AArch64 state */
+#define ESR_ELx_VAL_BRK64(imm)					\
+	((ESR_ELx_EC_BRK64 << ESR_ELx_EC_SHIFT) | ESR_ELx_IL |	\
+	 ((imm) & 0xffff))
 #ifndef __ASSEMBLY__
 #include <asm/types.h>

arch/arm64/include/asm/exception.h+6−0

 #ifndef __ASM_EXCEPTION_H
 #define __ASM_EXCEPTION_H
+#include <linux/ftrace.h>
 #define __exception	__attribute__((section(".exception.text")))
+#ifdef CONFIG_FUNCTION_GRAPH_TRACER
+#define __exception_irq_entry	__irq_entry
+#else
 #define __exception_irq_entry	__exception
+#endif
 #endif	/* __ASM_EXCEPTION_H */

arch/arm64/include/asm/fixmap.h+1−1

@@ -8,7 +8,7 @@ · 7 unmodified lines
8	* Copyright (C) 1998 Ingo Molnar	8	* Copyright (C) 1998 Ingo Molnar
9	* Copyright (C) 2013 Mark Salter <msalter@redhat.com>	9	* Copyright (C) 2013 Mark Salter <msalter@redhat.com>
10	*	10	*
11	* Adapted from arch/x86_64 version.	11	* Adapted from arch/x86 version.
12	*	12	*
13	*/	13	*/
14		14

arch/arm64/include/asm/futex.h+10−0

 #include <linux/futex.h>
 #include <linux/uaccess.h>
+#include <asm/alternative.h>
+#include <asm/cpufeature.h>
 #include <asm/errno.h>
+#include <asm/sysreg.h>
 #define __futex_atomic_op(insn, ret, oldval, uaddr, tmp, oparg)		\
 	asm volatile(							\
+	ALTERNATIVE("nop", SET_PSTATE_PAN(0), ARM64_HAS_PAN,		\
+		    CONFIG_ARM64_PAN)					\
+"	prfm	pstl1strm, %2\n"					\
 "1:	ldxr	%w1, %2\n"						\
 	insn "\n"							\
 "2:	stlxr	%w3, %w0, %2\n"						\
 "	.align	3\n"							\
 "	.quad	1b, 4b, 2b, 4b\n"					\
 "	.popsection\n"							\
+	ALTERNATIVE("nop", SET_PSTATE_PAN(1), ARM64_HAS_PAN,		\
+		    CONFIG_ARM64_PAN)					\
 	: "=&r" (ret), "=&r" (oldval), "+Q" (*uaddr), "=&r" (tmp)	\
 	: "r" (oparg), "Ir" (-EFAULT)					\
 	: "memory")
 		return -EFAULT;
 	asm volatile("// futex_atomic_cmpxchg_inatomic\n"
+"	prfm	pstl1strm, %2\n"
 "1:	ldxr	%w1, %2\n"
 "	sub	%w3, %w1, %w4\n"
 "	cbnz	%w3, 3f\n"

arch/arm64/include/asm/hardirq.h+0−4

 typedef struct {
 	unsigned int __softirq_pending;
-#ifdef CONFIG_SMP
 	unsigned int ipi_irqs[NR_IPI];
-#endif
 } ____cacheline_aligned irq_cpustat_t;
 #include <linux/irq_cpustat.h>	/* Standard mappings for irq_cpustat_t above */
 #define __inc_irq_stat(cpu, member)	__IRQ_STAT(cpu, member)++
 #define __get_irq_stat(cpu, member)	__IRQ_STAT(cpu, member)
-#ifdef CONFIG_SMP
 u64 smp_irq_stat_cpu(unsigned int cpu);
 #define arch_irq_stat_cpu	smp_irq_stat_cpu
-#endif
 #define __ARCH_IRQ_EXIT_IRQS_DISABLED	1

arch/arm64/include/asm/hugetlb.h+0−4

  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
  */
 #ifndef __ASM_HUGETLB_H

arch/arm64/include/asm/irq_work.h+0−11

@@ -1,8 +1,6 @@
1	#ifndef __ASM_IRQ_WORK_H	1	#ifndef __ASM_IRQ_WORK_H
2	#define __ASM_IRQ_WORK_H	2	#define __ASM_IRQ_WORK_H
3		3
4	#ifdef CONFIG_SMP
5
6	#include <asm/smp.h>	4	#include <asm/smp.h>
7		5
8	static inline bool arch_irq_work_has_interrupt(void)	6	static inline bool arch_irq_work_has_interrupt(void)
@@ -10,13 +8,4 @@ static inline bool arch_irq_work_has_interrupt(void) · 1 unmodified line
10	return !!__smp_cross_call;	8	return !!__smp_cross_call;
11	}	9	}
12		10
13	#else
14
15	static inline bool arch_irq_work_has_interrupt(void)
16	{
17	return false;
18	}
19
20	#endif
21
22	#endif /* __ASM_IRQ_WORK_H */	11	#endif /* __ASM_IRQ_WORK_H */

arch/arm64/include/asm/lse.h+53−0

@@ -0,0 +1,53 @@
	1	#ifndef __ASM_LSE_H
	2	#define __ASM_LSE_H
	3
	4	#if defined(CONFIG_AS_LSE) && defined(CONFIG_ARM64_LSE_ATOMICS)
	5
	6	#include <linux/stringify.h>
	7	#include <asm/alternative.h>
	8	#include <asm/cpufeature.h>
	9
	10	#ifdef __ASSEMBLER__
	11
	12	.arch_extension lse
	13
	14	.macro alt_lse, llsc, lse
	15	alternative_insn "\llsc", "\lse", ARM64_HAS_LSE_ATOMICS
	16	.endm
	17
	18	#else /* __ASSEMBLER__ */
	19
	20	__asm__(".arch_extension lse");
	21
	22	/* Move the ll/sc atomics out-of-line */
	23	#define __LL_SC_INLINE
	24	#define __LL_SC_PREFIX(x) __ll_sc_##x
	25	#define __LL_SC_EXPORT(x) EXPORT_SYMBOL(__LL_SC_PREFIX(x))
	26
	27	/* Macro for constructing calls to out-of-line ll/sc atomics */
	28	#define __LL_SC_CALL(op) "bl\t" __stringify(__LL_SC_PREFIX(op)) "\n"
	29
	30	/* In-line patching at runtime */
	31	#define ARM64_LSE_ATOMIC_INSN(llsc, lse) \
	32	ALTERNATIVE(llsc, lse, ARM64_HAS_LSE_ATOMICS)
	33
	34	#endif /* __ASSEMBLER__ */
	35	#else /* CONFIG_AS_LSE && CONFIG_ARM64_LSE_ATOMICS */
	36
	37	#ifdef __ASSEMBLER__
	38
	39	.macro alt_lse, llsc, lse
	40	\llsc
	41	.endm
	42
	43	#else /* __ASSEMBLER__ */
	44
	45	#define __LL_SC_INLINE static inline
	46	#define __LL_SC_PREFIX(x) x
	47	#define __LL_SC_EXPORT(x)
	48
	49	#define ARM64_LSE_ATOMIC_INSN(llsc, lse) llsc
	50
	51	#endif /* __ASSEMBLER__ */
	52	#endif /* CONFIG_AS_LSE && CONFIG_ARM64_LSE_ATOMICS */
	53	#endif /* __ASM_LSE_H */

arch/arm64/include/asm/memory.h+8−0

@@ -113,6 +113,14 @@ extern phys_addr_t memstart_addr; · 112 unmodified lines
113	/* PHYS_OFFSET - the physical address of the start of memory. */	113	/* PHYS_OFFSET - the physical address of the start of memory. */
114	#define PHYS_OFFSET ({ memstart_addr; })	114	#define PHYS_OFFSET ({ memstart_addr; })
115		115
		116	/*
		117	* The maximum physical address that the linear direct mapping
		118	* of system RAM can cover. (PAGE_OFFSET can be interpreted as
		119	* a 2's complement signed quantity and negated to derive the
		120	* maximum size of the linear mapping.)
		121	*/
		122	#define MAX_MEMBLOCK_ADDR ({ memstart_addr - PAGE_OFFSET - 1; })
		123
116	/*	124	/*
117	* PFNs are used to describe any physical page; this means	125	* PFNs are used to describe any physical page; this means
118	* PFN 0 == physical address 0.	126	* PFN 0 == physical address 0.

arch/arm64/include/asm/mmu.h+0−1

@@ -28,7 +28,6 @@ typedef struct { · 27 unmodified lines
28	#define ASID(mm) ((mm)->context.id & 0xffff)	28	#define ASID(mm) ((mm)->context.id & 0xffff)
29		29
30	extern void paging_init(void);	30	extern void paging_init(void);
31	extern void setup_mm_for_reboot(void);
32	extern void __iomem *early_io_map(phys_addr_t phys, unsigned long virt);	31	extern void __iomem *early_io_map(phys_addr_t phys, unsigned long virt);
33	extern void init_mem_pgprot(void);	32	extern void init_mem_pgprot(void);
34	extern void create_pgd_mapping(struct mm_struct *mm, phys_addr_t phys,	33	extern void create_pgd_mapping(struct mm_struct *mm, phys_addr_t phys,

arch/arm64/include/asm/percpu.h+0−8

 #ifndef __ASM_PERCPU_H
 #define __ASM_PERCPU_H
-#ifdef CONFIG_SMP
 static inline void set_my_cpu_offset(unsigned long off)
 {
 	asm volatile("msr tpidr_el1, %0" :: "r" (off) : "memory");
 }
 #define __my_cpu_offset __my_cpu_offset()
-#else	/* !CONFIG_SMP */
-#define set_my_cpu_offset(x)	do { } while (0)
-#endif /* CONFIG_SMP */
 #define PERCPU_OP(op, asm_op)						\
 static inline unsigned long __percpu_##op(void *ptr,			\
 			unsigned long val, int size)			\

arch/arm64/include/asm/perf_event.h+1−1

@@ -17,7 +17,7 @@ · 16 unmodified lines
17	#ifndef __ASM_PERF_EVENT_H	17	#ifndef __ASM_PERF_EVENT_H
18	#define __ASM_PERF_EVENT_H	18	#define __ASM_PERF_EVENT_H
19		19
20	#ifdef CONFIG_HW_PERF_EVENTS	20	#ifdef CONFIG_PERF_EVENTS
21	struct pt_regs;	21	struct pt_regs;
22	extern unsigned long perf_instruction_pointer(struct pt_regs *regs);	22	extern unsigned long perf_instruction_pointer(struct pt_regs *regs);
23	extern unsigned long perf_misc_flags(struct pt_regs *regs);	23	extern unsigned long perf_misc_flags(struct pt_regs *regs);

arch/arm64/include/asm/pgtable-hwdef.h+3−0

 #define PTE_SHARED		(_AT(pteval_t, 3) << 8)		/* SH[1:0], inner shareable */
 #define PTE_AF			(_AT(pteval_t, 1) << 10)	/* Access Flag */
 #define PTE_NG			(_AT(pteval_t, 1) << 11)	/* nG */
+#define PTE_DBM			(_AT(pteval_t, 1) << 51)	/* Dirty Bit Management */
 #define PTE_PXN			(_AT(pteval_t, 1) << 53)	/* Privileged XN */
 #define PTE_UXN			(_AT(pteval_t, 1) << 54)	/* User XN */
 #define TCR_TG1_64K		(UL(3) << 30)
 #define TCR_ASID16		(UL(1) << 36)
 #define TCR_TBI0		(UL(1) << 37)
+#define TCR_HA			(UL(1) << 39)
+#define TCR_HD			(UL(1) << 40)
 #endif

arch/arm64/include/asm/pgtable.h+160−7

@@ -16,6 +16,7 @@ · 15 unmodified lines
16	#ifndef __ASM_PGTABLE_H	16	#ifndef __ASM_PGTABLE_H
17	#define __ASM_PGTABLE_H	17	#define __ASM_PGTABLE_H
18		18
		19	#include <asm/bug.h>
19	#include <asm/proc-fns.h>	20	#include <asm/proc-fns.h>
20		21
21	#include <asm/memory.h>	22	#include <asm/memory.h>
@@ -27,7 +28,11 @@ · 5 unmodified lines
27	#define PTE_VALID (_AT(pteval_t, 1) << 0)	28	#define PTE_VALID (_AT(pteval_t, 1) << 0)
28	#define PTE_DIRTY (_AT(pteval_t, 1) << 55)	29	#define PTE_DIRTY (_AT(pteval_t, 1) << 55)
29	#define PTE_SPECIAL (_AT(pteval_t, 1) << 56)	30	#define PTE_SPECIAL (_AT(pteval_t, 1) << 56)
		31	#ifdef CONFIG_ARM64_HW_AFDBM
		32	#define PTE_WRITE (PTE_DBM) /* same as DBM */
		33	#else
30	#define PTE_WRITE (_AT(pteval_t, 1) << 57)	34	#define PTE_WRITE (_AT(pteval_t, 1) << 57)
		35	#endif
31	#define PTE_PROT_NONE (_AT(pteval_t, 1) << 58) /* only when !PTE_VALID */	36	#define PTE_PROT_NONE (_AT(pteval_t, 1) << 58) /* only when !PTE_VALID */
32		37
33	/*	38	/*
@@ -48,18 +53,16 @@ · 14 unmodified lines
48	#define FIRST_USER_ADDRESS 0UL	53	#define FIRST_USER_ADDRESS 0UL
49		54
50	#ifndef __ASSEMBLY__	55	#ifndef __ASSEMBLY__
		56
		57	#include <linux/mmdebug.h>
		58
51	extern void __pte_error(const char *file, int line, unsigned long val);	59	extern void __pte_error(const char *file, int line, unsigned long val);
52	extern void __pmd_error(const char *file, int line, unsigned long val);	60	extern void __pmd_error(const char *file, int line, unsigned long val);
53	extern void __pud_error(const char *file, int line, unsigned long val);	61	extern void __pud_error(const char *file, int line, unsigned long val);
54	extern void __pgd_error(const char *file, int line, unsigned long val);	62	extern void __pgd_error(const char *file, int line, unsigned long val);
55		63
56	#ifdef CONFIG_SMP
57	#define PROT_DEFAULT (PTE_TYPE_PAGE \| PTE_AF \| PTE_SHARED)	64	#define PROT_DEFAULT (PTE_TYPE_PAGE \| PTE_AF \| PTE_SHARED)
58	#define PROT_SECT_DEFAULT (PMD_TYPE_SECT \| PMD_SECT_AF \| PMD_SECT_S)	65	#define PROT_SECT_DEFAULT (PMD_TYPE_SECT \| PMD_SECT_AF \| PMD_SECT_S)
59	#else
60	#define PROT_DEFAULT (PTE_TYPE_PAGE \| PTE_AF)
61	#define PROT_SECT_DEFAULT (PMD_TYPE_SECT \| PMD_SECT_AF)
62	#endif
63		66
64	#define PROT_DEVICE_nGnRE (PROT_DEFAULT \| PTE_PXN \| PTE_UXN \| PTE_ATTRINDX(MT_DEVICE_nGnRE))	67	#define PROT_DEVICE_nGnRE (PROT_DEFAULT \| PTE_PXN \| PTE_UXN \| PTE_ATTRINDX(MT_DEVICE_nGnRE))
65	#define PROT_NORMAL_NC (PROT_DEFAULT \| PTE_PXN \| PTE_UXN \| PTE_ATTRINDX(MT_NORMAL_NC))	68	#define PROT_NORMAL_NC (PROT_DEFAULT \| PTE_PXN \| PTE_UXN \| PTE_ATTRINDX(MT_NORMAL_NC))
@@ -137,12 +140,20 @@ extern struct page *empty_zero_page; · 71 unmodified lines
137	* The following only work if pte_present(). Undefined behaviour otherwise.	140	* The following only work if pte_present(). Undefined behaviour otherwise.
138	*/	141	*/
139	#define pte_present(pte) (!!(pte_val(pte) & (PTE_VALID \| PTE_PROT_NONE)))	142	#define pte_present(pte) (!!(pte_val(pte) & (PTE_VALID \| PTE_PROT_NONE)))
140	#define pte_dirty(pte) (!!(pte_val(pte) & PTE_DIRTY))
141	#define pte_young(pte) (!!(pte_val(pte) & PTE_AF))	143	#define pte_young(pte) (!!(pte_val(pte) & PTE_AF))
142	#define pte_special(pte) (!!(pte_val(pte) & PTE_SPECIAL))	144	#define pte_special(pte) (!!(pte_val(pte) & PTE_SPECIAL))
143	#define pte_write(pte) (!!(pte_val(pte) & PTE_WRITE))	145	#define pte_write(pte) (!!(pte_val(pte) & PTE_WRITE))
144	#define pte_exec(pte) (!(pte_val(pte) & PTE_UXN))	146	#define pte_exec(pte) (!(pte_val(pte) & PTE_UXN))
145		147
		148	#ifdef CONFIG_ARM64_HW_AFDBM
		149	#define pte_hw_dirty(pte) (!(pte_val(pte) & PTE_RDONLY))
		150	#else
		151	#define pte_hw_dirty(pte) (0)
		152	#endif
		153	#define pte_sw_dirty(pte) (!!(pte_val(pte) & PTE_DIRTY))
		154	#define pte_dirty(pte) (pte_sw_dirty(pte) \|\| pte_hw_dirty(pte))
		155
		156	#define pte_valid(pte) (!!(pte_val(pte) & PTE_VALID))
146	#define pte_valid_user(pte) \	157	#define pte_valid_user(pte) \
147	((pte_val(pte) & (PTE_VALID \| PTE_USER)) == (PTE_VALID \| PTE_USER))	158	((pte_val(pte) & (PTE_VALID \| PTE_USER)) == (PTE_VALID \| PTE_USER))
148	#define pte_valid_not_user(pte) \	159	#define pte_valid_not_user(pte) \
@@ -209,20 +220,49 @@ static inline void set_pte(pte_t *ptep, pte_t pte) · 60 unmodified lines
209	}	220	}
210	}	221	}
211		222
		223	struct mm_struct;
		224	struct vm_area_struct;
		225
212	extern void __sync_icache_dcache(pte_t pteval, unsigned long addr);	226	extern void __sync_icache_dcache(pte_t pteval, unsigned long addr);
213		227
		228	/*
		229	* PTE bits configuration in the presence of hardware Dirty Bit Management
		230	* (PTE_WRITE == PTE_DBM):
		231	*
		232	* Dirty Writable \| PTE_RDONLY PTE_WRITE PTE_DIRTY (sw)
		233	* 0 0 \| 1 0 0
		234	* 0 1 \| 1 1 0
		235	* 1 0 \| 1 0 1
		236	* 1 1 \| 0 1 x
		237	*
		238	* When hardware DBM is not present, the sofware PTE_DIRTY bit is updated via
		239	* the page fault mechanism. Checking the dirty status of a pte becomes:
		240	*
		241	* PTE_DIRTY \|\| !PTE_RDONLY
		242	*/
214	static inline void set_pte_at(struct mm_struct *mm, unsigned long addr,	243	static inline void set_pte_at(struct mm_struct *mm, unsigned long addr,
215	pte_t *ptep, pte_t pte)	244	pte_t *ptep, pte_t pte)
216	{	245	{
217	if (pte_valid_user(pte)) {	246	if (pte_valid_user(pte)) {
218	if (!pte_special(pte) && pte_exec(pte))	247	if (!pte_special(pte) && pte_exec(pte))
219	__sync_icache_dcache(pte, addr);	248	__sync_icache_dcache(pte, addr);
220	if (pte_dirty(pte) && pte_write(pte))	249	if (pte_sw_dirty(pte) && pte_write(pte))
221	pte_val(pte) &= ~PTE_RDONLY;	250	pte_val(pte) &= ~PTE_RDONLY;
222	else	251	else
223	pte_val(pte) \|= PTE_RDONLY;	252	pte_val(pte) \|= PTE_RDONLY;
224	}	253	}
225		254
		255	/*
		256	* If the existing pte is valid, check for potential race with
		257	* hardware updates of the pte (ptep_set_access_flags safely changes
		258	* valid ptes without going through an invalid entry).
		259	*/
		260	if (IS_ENABLED(CONFIG_DEBUG_VM) && IS_ENABLED(CONFIG_ARM64_HW_AFDBM) &&
		261	pte_valid(*ptep)) {
		262	BUG_ON(!pte_young(pte));
		263	BUG_ON(pte_write(*ptep) && !pte_dirty(pte));
		264	}
		265
226	set_pte(ptep, pte);	266	set_pte(ptep, pte);
227	}	267	}
228		268
@@ -461,6 +501,9 @@ static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) · 232 unmodified lines
461	{	501	{
462	const pteval_t mask = PTE_USER \| PTE_PXN \| PTE_UXN \| PTE_RDONLY \|	502	const pteval_t mask = PTE_USER \| PTE_PXN \| PTE_UXN \| PTE_RDONLY \|
463	PTE_PROT_NONE \| PTE_WRITE \| PTE_TYPE_MASK;	503	PTE_PROT_NONE \| PTE_WRITE \| PTE_TYPE_MASK;
		504	/* preserve the hardware dirty information */
		505	if (pte_hw_dirty(pte))
		506	newprot \|= PTE_DIRTY;
464	pte_val(pte) = (pte_val(pte) & ~mask) \| (pgprot_val(newprot) & mask);	507	pte_val(pte) = (pte_val(pte) & ~mask) \| (pgprot_val(newprot) & mask);
465	return pte;	508	return pte;
466	}	509	}
@@ -470,6 +513,101 @@ static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot) · 3 unmodified lines
470	return pte_pmd(pte_modify(pmd_pte(pmd), newprot));	513	return pte_pmd(pte_modify(pmd_pte(pmd), newprot));
471	}	514	}
472		515
		516	#ifdef CONFIG_ARM64_HW_AFDBM
		517	/*
		518	* Atomic pte/pmd modifications.
		519	*/
		520	#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
		521	static inline int ptep_test_and_clear_young(struct vm_area_struct *vma,
		522	unsigned long address,
		523	pte_t *ptep)
		524	{
		525	pteval_t pteval;
		526	unsigned int tmp, res;
		527
		528	asm volatile("// ptep_test_and_clear_young\n"
		529	" prfm pstl1strm, %2\n"
		530	"1: ldxr %0, %2\n"
		531	" ubfx %w3, %w0, %5, #1 // extract PTE_AF (young)\n"
		532	" and %0, %0, %4 // clear PTE_AF\n"
		533	" stxr %w1, %0, %2\n"
		534	" cbnz %w1, 1b\n"
		535	: "=&r" (pteval), "=&r" (tmp), "+Q" (pte_val(*ptep)), "=&r" (res)
		536	: "L" (~PTE_AF), "I" (ilog2(PTE_AF)));
		537
		538	return res;
		539	}
		540
		541	#ifdef CONFIG_TRANSPARENT_HUGEPAGE
		542	#define __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG
		543	static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma,
		544	unsigned long address,
		545	pmd_t *pmdp)
		546	{
		547	return ptep_test_and_clear_young(vma, address, (pte_t *)pmdp);
		548	}
		549	#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
		550
		551	#define __HAVE_ARCH_PTEP_GET_AND_CLEAR
		552	static inline pte_t ptep_get_and_clear(struct mm_struct *mm,
		553	unsigned long address, pte_t *ptep)
		554	{
		555	pteval_t old_pteval;
		556	unsigned int tmp;
		557
		558	asm volatile("// ptep_get_and_clear\n"
		559	" prfm pstl1strm, %2\n"
		560	"1: ldxr %0, %2\n"
		561	" stxr %w1, xzr, %2\n"
		562	" cbnz %w1, 1b\n"
		563	: "=&r" (old_pteval), "=&r" (tmp), "+Q" (pte_val(*ptep)));
		564
		565	return __pte(old_pteval);
		566	}
		567
		568	#ifdef CONFIG_TRANSPARENT_HUGEPAGE
		569	#define __HAVE_ARCH_PMDP_GET_AND_CLEAR
		570	static inline pmd_t pmdp_get_and_clear(struct mm_struct *mm,
		571	unsigned long address, pmd_t *pmdp)
		572	{
		573	return pte_pmd(ptep_get_and_clear(mm, address, (pte_t *)pmdp));
		574	}
		575	#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
		576
		577	/*
		578	* ptep_set_wrprotect - mark read-only while trasferring potential hardware
		579	* dirty status (PTE_DBM && !PTE_RDONLY) to the software PTE_DIRTY bit.
		580	*/
		581	#define __HAVE_ARCH_PTEP_SET_WRPROTECT
		582	static inline void ptep_set_wrprotect(struct mm_struct mm, unsigned long address, pte_t ptep)
		583	{
		584	pteval_t pteval;
		585	unsigned long tmp;
		586
		587	asm volatile("// ptep_set_wrprotect\n"
		588	" prfm pstl1strm, %2\n"
		589	"1: ldxr %0, %2\n"
		590	" tst %0, %4 // check for hw dirty (!PTE_RDONLY)\n"
		591	" csel %1, %3, xzr, eq // set PTE_DIRTY\|PTE_RDONLY if dirty\n"
		592	" orr %0, %0, %1 // if !dirty, PTE_RDONLY is already set\n"
		593	" and %0, %0, %5 // clear PTE_WRITE/PTE_DBM\n"
		594	" stxr %w1, %0, %2\n"
		595	" cbnz %w1, 1b\n"
		596	: "=&r" (pteval), "=&r" (tmp), "+Q" (pte_val(*ptep))
		597	: "r" (PTE_DIRTY\|PTE_RDONLY), "L" (PTE_RDONLY), "L" (~PTE_WRITE)
		598	: "cc");
		599	}
		600
		601	#ifdef CONFIG_TRANSPARENT_HUGEPAGE
		602	#define __HAVE_ARCH_PMDP_SET_WRPROTECT
		603	static inline void pmdp_set_wrprotect(struct mm_struct *mm,
		604	unsigned long address, pmd_t *pmdp)
		605	{
		606	ptep_set_wrprotect(mm, address, (pte_t *)pmdp);
		607	}
		608	#endif
		609	#endif /* CONFIG_ARM64_HW_AFDBM */
		610
473	extern pgd_t swapper_pg_dir[PTRS_PER_PGD];	611	extern pgd_t swapper_pg_dir[PTRS_PER_PGD];
474	extern pgd_t idmap_pg_dir[PTRS_PER_PGD];	612	extern pgd_t idmap_pg_dir[PTRS_PER_PGD];
475		613
@@ -505,6 +643,21 @@ extern int kern_addr_valid(unsigned long addr); · 29 unmodified lines
505		643
506	#define pgtable_cache_init() do { } while (0)	644	#define pgtable_cache_init() do { } while (0)
507		645
		646	/*
		647	* On AArch64, the cache coherency is handled via the set_pte_at() function.
		648	*/
		649	static inline void update_mmu_cache(struct vm_area_struct *vma,
		650	unsigned long addr, pte_t *ptep)
		651	{
		652	/*
		653	* set_pte() does not have a DSB for user mappings, so make sure that
		654	* the page table write is visible.
		655	*/
		656	dsb(ishst);
		657	}
		658
		659	#define update_mmu_cache_pmd(vma, address, pmd) do { } while (0)
		660
508	#endif /* !__ASSEMBLY__ */	661	#endif /* !__ASSEMBLY__ */
509		662
510	#endif /* __ASM_PGTABLE_H */	663	#endif /* __ASM_PGTABLE_H */

arch/arm64/include/asm/processor.h+2−0

@@ -186,4 +186,6 @@ static inline void spin_lock_prefetch(const void *x) · 185 unmodified lines
186		186
187	#endif	187	#endif
188		188
		189	void cpu_enable_pan(void);
		190
189	#endif /* __ASM_PROCESSOR_H */	191	#endif /* __ASM_PROCESSOR_H */

arch/arm64/include/asm/ptrace.h+0−4

 #define instruction_pointer(regs)	((unsigned long)(regs)->pc)
-#ifdef CONFIG_SMP
 extern unsigned long profile_pc(struct pt_regs *regs);
-#else
-#define profile_pc(regs) instruction_pointer(regs)
-#endif
 #endif /* __ASSEMBLY__ */
 #endif

arch/arm64/include/asm/smp.h+0−4

@@ -20,10 +20,6 @@ · 19 unmodified lines
20	#include <linux/cpumask.h>	20	#include <linux/cpumask.h>
21	#include <linux/thread_info.h>	21	#include <linux/thread_info.h>
22		22
23	#ifndef CONFIG_SMP
24	# error "<asm/smp.h> included in non-SMP build"
25	#endif
26
27	#define raw_smp_processor_id() (current_thread_info()->cpu)	23	#define raw_smp_processor_id() (current_thread_info()->cpu)
28		24
29	struct seq_file;	25	struct seq_file;

arch/arm64/include/asm/smp_plat.h+0−2

@@ -56,6 +56,4 @@ static inline int get_logical_index(u64 mpidr) · 55 unmodified lines
56	return -EINVAL;	56	return -EINVAL;
57	}	57	}
58		58
59	void __init do_post_cpus_up_work(void);
60
61	#endif /* __ASM_SMP_PLAT_H */	59	#endif /* __ASM_SMP_PLAT_H */

arch/arm64/include/asm/spinlock.h+114−33

@@ -16,6 +16,7 @@ · 15 unmodified lines
16	#ifndef __ASM_SPINLOCK_H	16	#ifndef __ASM_SPINLOCK_H
17	#define __ASM_SPINLOCK_H	17	#define __ASM_SPINLOCK_H
18		18
		19	#include <asm/lse.h>
19	#include <asm/spinlock_types.h>	20	#include <asm/spinlock_types.h>
20	#include <asm/processor.h>	21	#include <asm/processor.h>
21		22
@@ -38,11 +39,21 @@ static inline void arch_spin_lock(arch_spinlock_t *lock) · 16 unmodified lines
38		39
39	asm volatile(	40	asm volatile(
40	/* Atomically increment the next ticket. */	41	/* Atomically increment the next ticket. */
		42	ARM64_LSE_ATOMIC_INSN(
		43	/* LL/SC */
41	" prfm pstl1strm, %3\n"	44	" prfm pstl1strm, %3\n"
42	"1: ldaxr %w0, %3\n"	45	"1: ldaxr %w0, %3\n"
43	" add %w1, %w0, %w5\n"	46	" add %w1, %w0, %w5\n"
44	" stxr %w2, %w1, %3\n"	47	" stxr %w2, %w1, %3\n"
45	" cbnz %w2, 1b\n"	48	" cbnz %w2, 1b\n",
		49	/* LSE atomics */
		50	" mov %w2, %w5\n"
		51	" ldadda %w2, %w0, %3\n"
		52	" nop\n"
		53	" nop\n"
		54	" nop\n"
		55	)
		56
46	/* Did we get the lock? */	57	/* Did we get the lock? */
47	" eor %w1, %w0, %w0, ror #16\n"	58	" eor %w1, %w0, %w0, ror #16\n"
48	" cbz %w1, 3f\n"	59	" cbz %w1, 3f\n"
@@ -67,15 +78,25 @@ static inline int arch_spin_trylock(arch_spinlock_t *lock) · 18 unmodified lines
67	unsigned int tmp;	78	unsigned int tmp;
68	arch_spinlock_t lockval;	79	arch_spinlock_t lockval;
69		80
70	asm volatile(	81	asm volatile(ARM64_LSE_ATOMIC_INSN(
71	" prfm pstl1strm, %2\n"	82	/* LL/SC */
72	"1: ldaxr %w0, %2\n"	83	" prfm pstl1strm, %2\n"
73	" eor %w1, %w0, %w0, ror #16\n"	84	"1: ldaxr %w0, %2\n"
74	" cbnz %w1, 2f\n"	85	" eor %w1, %w0, %w0, ror #16\n"
75	" add %w0, %w0, %3\n"	86	" cbnz %w1, 2f\n"
76	" stxr %w1, %w0, %2\n"	87	" add %w0, %w0, %3\n"
77	" cbnz %w1, 1b\n"	88	" stxr %w1, %w0, %2\n"
78	"2:"	89	" cbnz %w1, 1b\n"
		90	"2:",
		91	/* LSE atomics */
		92	" ldr %w0, %2\n"
		93	" eor %w1, %w0, %w0, ror #16\n"
		94	" cbnz %w1, 1f\n"
		95	" add %w1, %w0, %3\n"
		96	" casa %w0, %w1, %2\n"
		97	" and %w1, %w1, #0xffff\n"
		98	" eor %w1, %w1, %w0, lsr #16\n"
		99	"1:")
79	: "=&r" (lockval), "=&r" (tmp), "+Q" (*lock)	100	: "=&r" (lockval), "=&r" (tmp), "+Q" (*lock)
80	: "I" (1 << TICKET_SHIFT)	101	: "I" (1 << TICKET_SHIFT)
81	: "memory");	102	: "memory");
@@ -85,10 +106,19 @@ static inline int arch_spin_trylock(arch_spinlock_t *lock) · 3 unmodified lines
85		106
86	static inline void arch_spin_unlock(arch_spinlock_t *lock)	107	static inline void arch_spin_unlock(arch_spinlock_t *lock)
87	{	108	{
88	asm volatile(	109	unsigned long tmp;
89	" stlrh %w1, %0\n"	110
90	: "=Q" (lock->owner)	111	asm volatile(ARM64_LSE_ATOMIC_INSN(
91	: "r" (lock->owner + 1)	112	/* LL/SC */
		113	" ldrh %w1, %0\n"
		114	" add %w1, %w1, #1\n"
		115	" stlrh %w1, %0",
		116	/* LSE atomics */
		117	" mov %w1, #1\n"
		118	" nop\n"
		119	" staddlh %w1, %0")
		120	: "=Q" (lock->owner), "=&r" (tmp)
		121	:
92	: "memory");	122	: "memory");
93	}	123	}
94		124
@@ -123,13 +153,24 @@ static inline void arch_write_lock(arch_rwlock_t *rw) · 28 unmodified lines
123	{	153	{
124	unsigned int tmp;	154	unsigned int tmp;
125		155
126	asm volatile(	156	asm volatile(ARM64_LSE_ATOMIC_INSN(
		157	/* LL/SC */
127	" sevl\n"	158	" sevl\n"
128	"1: wfe\n"	159	"1: wfe\n"
129	"2: ldaxr %w0, %1\n"	160	"2: ldaxr %w0, %1\n"
130	" cbnz %w0, 1b\n"	161	" cbnz %w0, 1b\n"
131	" stxr %w0, %w2, %1\n"	162	" stxr %w0, %w2, %1\n"
132	" cbnz %w0, 2b\n"	163	" cbnz %w0, 2b\n"
		164	" nop",
		165	/* LSE atomics */
		166	"1: mov %w0, wzr\n"
		167	"2: casa %w0, %w2, %1\n"
		168	" cbz %w0, 3f\n"
		169	" ldxr %w0, %1\n"
		170	" cbz %w0, 2b\n"
		171	" wfe\n"
		172	" b 1b\n"
		173	"3:")
133	: "=&r" (tmp), "+Q" (rw->lock)	174	: "=&r" (tmp), "+Q" (rw->lock)
134	: "r" (0x80000000)	175	: "r" (0x80000000)
135	: "memory");	176	: "memory");
@@ -139,11 +180,18 @@ static inline int arch_write_trylock(arch_rwlock_t *rw) · 3 unmodified lines
139	{	180	{
140	unsigned int tmp;	181	unsigned int tmp;
141		182
142	asm volatile(	183	asm volatile(ARM64_LSE_ATOMIC_INSN(
143	" ldaxr %w0, %1\n"	184	/* LL/SC */
144	" cbnz %w0, 1f\n"	185	"1: ldaxr %w0, %1\n"
		186	" cbnz %w0, 2f\n"
145	" stxr %w0, %w2, %1\n"	187	" stxr %w0, %w2, %1\n"
146	"1:\n"	188	" cbnz %w0, 1b\n"
		189	"2:",
		190	/* LSE atomics */
		191	" mov %w0, wzr\n"
		192	" casa %w0, %w2, %1\n"
		193	" nop\n"
		194	" nop")
147	: "=&r" (tmp), "+Q" (rw->lock)	195	: "=&r" (tmp), "+Q" (rw->lock)
148	: "r" (0x80000000)	196	: "r" (0x80000000)
149	: "memory");	197	: "memory");
@@ -153,9 +201,10 @@ static inline int arch_write_trylock(arch_rwlock_t *rw) · 3 unmodified lines
153		201
154	static inline void arch_write_unlock(arch_rwlock_t *rw)	202	static inline void arch_write_unlock(arch_rwlock_t *rw)
155	{	203	{
156	asm volatile(	204	asm volatile(ARM64_LSE_ATOMIC_INSN(
157	" stlr %w1, %0\n"	205	" stlr wzr, %0",
158	: "=Q" (rw->lock) : "r" (0) : "memory");	206	" swpl wzr, wzr, %0")
		207	: "=Q" (rw->lock) :: "memory");
159	}	208	}
160		209
161	/* write_can_lock - would write_trylock() succeed? */	210	/* write_can_lock - would write_trylock() succeed? */
@@ -172,6 +221,10 @@ static inline void arch_write_unlock(arch_rwlock_t *rw) · 10 unmodified lines
172	*	221	*
173	* The memory barriers are implicit with the load-acquire and store-release	222	* The memory barriers are implicit with the load-acquire and store-release
174	* instructions.	223	* instructions.
		224	*
		225	* Note that in UNDEFINED cases, such as unlocking a lock twice, the LL/SC
		226	* and LSE implementations may exhibit different behaviour (although this
		227	* will have no effect on lockdep).
175	*/	228	*/
176	static inline void arch_read_lock(arch_rwlock_t *rw)	229	static inline void arch_read_lock(arch_rwlock_t *rw)
177	{	230	{
@@ -179,26 +232,43 @@ static inline void arch_read_lock(arch_rwlock_t *rw) · 1 unmodified line
179		232
180	asm volatile(	233	asm volatile(
181	" sevl\n"	234	" sevl\n"
		235	ARM64_LSE_ATOMIC_INSN(
		236	/* LL/SC */
182	"1: wfe\n"	237	"1: wfe\n"
183	"2: ldaxr %w0, %2\n"	238	"2: ldaxr %w0, %2\n"
184	" add %w0, %w0, #1\n"	239	" add %w0, %w0, #1\n"
185	" tbnz %w0, #31, 1b\n"	240	" tbnz %w0, #31, 1b\n"
186	" stxr %w1, %w0, %2\n"	241	" stxr %w1, %w0, %2\n"
187	" cbnz %w1, 2b\n"	242	" nop\n"
		243	" cbnz %w1, 2b",
		244	/* LSE atomics */
		245	"1: wfe\n"
		246	"2: ldxr %w0, %2\n"
		247	" adds %w1, %w0, #1\n"
		248	" tbnz %w1, #31, 1b\n"
		249	" casa %w0, %w1, %2\n"
		250	" sbc %w0, %w1, %w0\n"
		251	" cbnz %w0, 2b")
188	: "=&r" (tmp), "=&r" (tmp2), "+Q" (rw->lock)	252	: "=&r" (tmp), "=&r" (tmp2), "+Q" (rw->lock)
189	:	253	:
190	: "memory");	254	: "cc", "memory");
191	}	255	}
192		256
193	static inline void arch_read_unlock(arch_rwlock_t *rw)	257	static inline void arch_read_unlock(arch_rwlock_t *rw)
194	{	258	{
195	unsigned int tmp, tmp2;	259	unsigned int tmp, tmp2;
196		260
197	asm volatile(	261	asm volatile(ARM64_LSE_ATOMIC_INSN(
		262	/* LL/SC */
198	"1: ldxr %w0, %2\n"	263	"1: ldxr %w0, %2\n"
199	" sub %w0, %w0, #1\n"	264	" sub %w0, %w0, #1\n"
200	" stlxr %w1, %w0, %2\n"	265	" stlxr %w1, %w0, %2\n"
201	" cbnz %w1, 1b\n"	266	" cbnz %w1, 1b",
		267	/* LSE atomics */
		268	" movn %w0, #0\n"
		269	" nop\n"
		270	" nop\n"
		271	" staddl %w0, %2")
202	: "=&r" (tmp), "=&r" (tmp2), "+Q" (rw->lock)	272	: "=&r" (tmp), "=&r" (tmp2), "+Q" (rw->lock)
203	:	273	:
204	: "memory");	274	: "memory");
@@ -206,17 +276,28 @@ static inline void arch_read_unlock(arch_rwlock_t *rw) · 1 unmodified line
206		276
207	static inline int arch_read_trylock(arch_rwlock_t *rw)	277	static inline int arch_read_trylock(arch_rwlock_t *rw)
208	{	278	{
209	unsigned int tmp, tmp2 = 1;	279	unsigned int tmp, tmp2;
210		280
211	asm volatile(	281	asm volatile(ARM64_LSE_ATOMIC_INSN(
212	" ldaxr %w0, %2\n"	282	/* LL/SC */
		283	" mov %w1, #1\n"
		284	"1: ldaxr %w0, %2\n"
213	" add %w0, %w0, #1\n"	285	" add %w0, %w0, #1\n"
214	" tbnz %w0, #31, 1f\n"	286	" tbnz %w0, #31, 2f\n"
215	" stxr %w1, %w0, %2\n"	287	" stxr %w1, %w0, %2\n"
216	"1:\n"	288	" cbnz %w1, 1b\n"
217	: "=&r" (tmp), "+r" (tmp2), "+Q" (rw->lock)	289	"2:",
		290	/* LSE atomics */
		291	" ldr %w0, %2\n"
		292	" adds %w1, %w0, #1\n"
		293	" tbnz %w1, #31, 1f\n"
		294	" casa %w0, %w1, %2\n"
		295	" sbc %w1, %w1, %w0\n"
		296	" nop\n"
		297	"1:")
		298	: "=&r" (tmp), "=&r" (tmp2), "+Q" (rw->lock)
218	:	299	:
219	: "memory");	300	: "cc", "memory");
220		301
221	return !tmp2;	302	return !tmp2;
222	}	303	}

arch/arm64/include/asm/spinlock_types.h+2−0

@@ -20,6 +20,8 @@ · 19 unmodified lines
20	# error "please don't include this file directly"	20	# error "please don't include this file directly"
21	#endif	21	#endif
22		22
		23	#include <linux/types.h>
		24
23	#define TICKET_SHIFT 16	25	#define TICKET_SHIFT 16
24		26
25	typedef struct {	27	typedef struct {

arch/arm64/include/asm/sysreg.h+35−5

 #ifndef __ASM_SYSREG_H
 #define __ASM_SYSREG_H
+#include <asm/opcodes.h>
+#define SCTLR_EL1_CP15BEN	(0x1 << 5)
+#define SCTLR_EL1_SED		(0x1 << 8)
+/*
+ * ARMv8 ARM reserves the following encoding for system registers:
+ * (Ref: ARMv8 ARM, Section: "System instruction class encoding overview",
+ *  C5.2, version:ARM DDI 0487A.f)
+ *	[20-19] : Op0
+ *	[18-16] : Op1
+ *	[15-12] : CRn
+ *	[11-8]  : CRm
+ *	[7-5]   : Op2
+ */
 #define sys_reg(op0, op1, crn, crm, op2) \
-	((((op0)-2)<<19)|((op1)<<16)|((crn)<<12)|((crm)<<8)|((op2)<<5))
+	((((op0)&3)<<19)|((op1)<<16)|((crn)<<12)|((crm)<<8)|((op2)<<5))
+#define REG_PSTATE_PAN_IMM                     sys_reg(0, 0, 4, 0, 4)
+#define SCTLR_EL1_SPAN                         (1 << 23)
+#define SET_PSTATE_PAN(x) __inst_arm(0xd5000000 | REG_PSTATE_PAN_IMM |\
+				     (!!x)<<8 | 0x1f)
 #ifdef __ASSEMBLY__
 	.equ	__reg_num_xzr, 31
 	.macro	mrs_s, rt, sreg
-	.inst	0xd5300000|(\sreg)|(__reg_num_\rt)
+	.inst	0xd5200000|(\sreg)|(__reg_num_\rt)
 	.endm
 	.macro	msr_s, sreg, rt
-	.inst	0xd5100000|(\sreg)|(__reg_num_\rt)
+	.inst	0xd5000000|(\sreg)|(__reg_num_\rt)
 	.endm
 #else
 "	.equ	__reg_num_xzr, 31\n"
 "\n"
 "	.macro	mrs_s, rt, sreg\n"
-"	.inst	0xd5300000|(\\sreg)|(__reg_num_\\rt)\n"
+"	.inst	0xd5200000|(\\sreg)|(__reg_num_\\rt)\n"
 "	.endm\n"
 "\n"
 "	.macro	msr_s, sreg, rt\n"
-"	.inst	0xd5100000|(\\sreg)|(__reg_num_\\rt)\n"
+"	.inst	0xd5000000|(\\sreg)|(__reg_num_\\rt)\n"
 "	.endm\n"
 );
+static inline void config_sctlr_el1(u32 clear, u32 set)
+{
+	u32 val;
+	asm volatile("mrs %0, sctlr_el1" : "=r" (val));
+	val &= ~clear;
+	val |= set;
+	asm volatile("msr sctlr_el1, %0" : : "r" (val));
+}
 #endif
 #endif	/* __ASM_SYSREG_H */

arch/arm64/include/asm/tlb.h+6−1

 		flush_tlb_mm(tlb->mm);
 	} else {
 		struct vm_area_struct vma = { .vm_mm = tlb->mm, };
-		flush_tlb_range(&vma, tlb->start, tlb->end);
+		/*
+		 * The intermediate page table levels are already handled by
+		 * the __(pte|pmd|pud)_free_tlb() functions, so last level
+		 * TLBI is sufficient here.
+		 */
+		__flush_tlb_range(&vma, tlb->start, tlb->end, true);
 	}
 }

arch/arm64/include/asm/tlbflush.h+34−42

 		((unsigned long)ASID(vma->vm_mm) << 48);
 	dsb(ishst);
-	asm("tlbi	vae1is, %0" : : "r" (addr));
+	asm("tlbi	vale1is, %0" : : "r" (addr));
 	dsb(ish);
 }
+/*
+ * This is meant to avoid soft lock-ups on large TLB flushing ranges and not
+ * necessarily a performance improvement.
+ */
+#define MAX_TLB_RANGE	(1024UL << PAGE_SHIFT)
 static inline void __flush_tlb_range(struct vm_area_struct *vma,
-				     unsigned long start, unsigned long end)
+				     unsigned long start, unsigned long end,
+				     bool last_level)
 {
 	unsigned long asid = (unsigned long)ASID(vma->vm_mm) << 48;
 	unsigned long addr;
+	if ((end - start) > MAX_TLB_RANGE) {
+		flush_tlb_mm(vma->vm_mm);
+		return;
+	}
 	start = asid | (start >> 12);
 	end = asid | (end >> 12);
 	dsb(ishst);
-	for (addr = start; addr < end; addr += 1 << (PAGE_SHIFT - 12))
+	for (addr = start; addr < end; addr += 1 << (PAGE_SHIFT - 12)) {
-		asm("tlbi vae1is, %0" : : "r"(addr));
+		if (last_level)
+			asm("tlbi vale1is, %0" : : "r"(addr));
+		else
+			asm("tlbi vae1is, %0" : : "r"(addr));
+	}
 	dsb(ish);
 }
-static inline void __flush_tlb_kernel_range(unsigned long start, unsigned long end)
+static inline void flush_tlb_range(struct vm_area_struct *vma,
+				   unsigned long start, unsigned long end)
+{
+	__flush_tlb_range(vma, start, end, false);
+}
+static inline void flush_tlb_kernel_range(unsigned long start, unsigned long end)
 {
 	unsigned long addr;
+	if ((end - start) > MAX_TLB_RANGE) {
+		flush_tlb_all();
+		return;
+	}
 	start >>= 12;
 	end >>= 12;
 	isb();
 }
-/*
- * This is meant to avoid soft lock-ups on large TLB flushing ranges and not
- * necessarily a performance improvement.
- */
-#define MAX_TLB_RANGE	(1024UL << PAGE_SHIFT)
-static inline void flush_tlb_range(struct vm_area_struct *vma,
-				   unsigned long start, unsigned long end)
-{
-	if ((end - start) <= MAX_TLB_RANGE)
-		__flush_tlb_range(vma, start, end);
-	else
-		flush_tlb_mm(vma->vm_mm);
-}
-static inline void flush_tlb_kernel_range(unsigned long start, unsigned long end)
-{
-	if ((end - start) <= MAX_TLB_RANGE)
-		__flush_tlb_kernel_range(start, end);
-	else
-		flush_tlb_all();
-}
 /*
  * Used to invalidate the TLB (walk caches) corresponding to intermediate page
  * table levels (pgd/pud/pmd).
 	asm("tlbi	vae1is, %0" : : "r" (addr));
 	dsb(ish);
 }
-/*
- * On AArch64, the cache coherency is handled via the set_pte_at() function.
- */
-static inline void update_mmu_cache(struct vm_area_struct *vma,
-				    unsigned long addr, pte_t *ptep)
-{
-	/*
-	 * set_pte() does not have a DSB for user mappings, so make sure that
-	 * the page table write is visible.
-	 */
-	dsb(ishst);
-}
-#define update_mmu_cache_pmd(vma, address, pmd) do { } while (0)
 #endif

arch/arm64/include/asm/topology.h+0−9

 #ifndef __ASM_TOPOLOGY_H
 #define __ASM_TOPOLOGY_H
-#ifdef CONFIG_SMP
 #include <linux/cpumask.h>
 struct cpu_topology {
 void store_cpu_topology(unsigned int cpuid);
 const struct cpumask *cpu_coregroup_mask(int cpu);
-#else
-static inline void init_cpu_topology(void) { }
-static inline void store_cpu_topology(unsigned int cpuid) { }
-#endif
 #include <asm-generic/topology.h>
 #endif /* _ASM_ARM_TOPOLOGY_H */

arch/arm64/include/asm/traps.h+21−2

@@ -34,13 +34,32 @@ struct undef_hook { · 33 unmodified lines
34	void register_undef_hook(struct undef_hook *hook);	34	void register_undef_hook(struct undef_hook *hook);
35	void unregister_undef_hook(struct undef_hook *hook);	35	void unregister_undef_hook(struct undef_hook *hook);
36		36
		37	#ifdef CONFIG_FUNCTION_GRAPH_TRACER
		38	static inline int __in_irqentry_text(unsigned long ptr)
		39	{
		40	extern char __irqentry_text_start[];
		41	extern char __irqentry_text_end[];
		42
		43	return ptr >= (unsigned long)&__irqentry_text_start &&
		44	ptr < (unsigned long)&__irqentry_text_end;
		45	}
		46	#else
		47	static inline int __in_irqentry_text(unsigned long ptr)
		48	{
		49	return 0;
		50	}
		51	#endif
		52
37	static inline int in_exception_text(unsigned long ptr)	53	static inline int in_exception_text(unsigned long ptr)
38	{	54	{
39	extern char __exception_text_start[];	55	extern char __exception_text_start[];
40	extern char __exception_text_end[];	56	extern char __exception_text_end[];
		57	int in;
		58
		59	in = ptr >= (unsigned long)&__exception_text_start &&
		60	ptr < (unsigned long)&__exception_text_end;
41		61
42	return ptr >= (unsigned long)&__exception_text_start &&	62	return in ? : __in_irqentry_text(ptr);
43	ptr < (unsigned long)&__exception_text_end;
44	}	63	}
45		64
46	#endif	65	#endif

arch/arm64/include/asm/uaccess.h+11−0

 #include <linux/string.h>
 #include <linux/thread_info.h>
+#include <asm/alternative.h>
+#include <asm/cpufeature.h>
 #include <asm/ptrace.h>
+#include <asm/sysreg.h>
 #include <asm/errno.h>
 #include <asm/memory.h>
 #include <asm/compiler.h>
 do {									\
 	unsigned long __gu_val;						\
 	__chk_user_ptr(ptr);						\
+	asm(ALTERNATIVE("nop", SET_PSTATE_PAN(0), ARM64_HAS_PAN,	\
+			CONFIG_ARM64_PAN));				\
 	switch (sizeof(*(ptr))) {					\
 	case 1:								\
 		__get_user_asm("ldrb", "%w", __gu_val, (ptr), (err));	\
 		BUILD_BUG();						\
 	}								\
 	(x) = (__force __typeof__(*(ptr)))__gu_val;			\
+	asm(ALTERNATIVE("nop", SET_PSTATE_PAN(1), ARM64_HAS_PAN,	\
+			CONFIG_ARM64_PAN));				\
 } while (0)
 #define __get_user(x, ptr)						\
 do {									\
 	__typeof__(*(ptr)) __pu_val = (x);				\
 	__chk_user_ptr(ptr);						\
+	asm(ALTERNATIVE("nop", SET_PSTATE_PAN(0), ARM64_HAS_PAN,	\
+			CONFIG_ARM64_PAN));				\
 	switch (sizeof(*(ptr))) {					\
 	case 1:								\
 		__put_user_asm("strb", "%w", __pu_val, (ptr), (err));	\
 	default:							\
 		BUILD_BUG();						\
 	}								\
+	asm(ALTERNATIVE("nop", SET_PSTATE_PAN(1), ARM64_HAS_PAN,	\
+			CONFIG_ARM64_PAN));				\
 } while (0)
 #define __put_user(x, ptr)						\

arch/arm64/include/uapi/asm/hwcap.h+1−0

@@ -27,5 +27,6 @@ · 26 unmodified lines
27	#define HWCAP_SHA1 (1 << 5)	27	#define HWCAP_SHA1 (1 << 5)
28	#define HWCAP_SHA2 (1 << 6)	28	#define HWCAP_SHA2 (1 << 6)
29	#define HWCAP_CRC32 (1 << 7)	29	#define HWCAP_CRC32 (1 << 7)
		30	#define HWCAP_ATOMICS (1 << 8)
30		31
31	#endif /* _UAPI__ASM_HWCAP_H */	32	#endif /* _UAPI__ASM_HWCAP_H */

arch/arm64/include/uapi/asm/ptrace.h+1−0

@@ -44,6 +44,7 @@ · 43 unmodified lines
44	#define PSR_I_BIT 0x00000080	44	#define PSR_I_BIT 0x00000080
45	#define PSR_A_BIT 0x00000100	45	#define PSR_A_BIT 0x00000100
46	#define PSR_D_BIT 0x00000200	46	#define PSR_D_BIT 0x00000200
		47	#define PSR_PAN_BIT 0x00400000
47	#define PSR_Q_BIT 0x08000000	48	#define PSR_Q_BIT 0x08000000
48	#define PSR_V_BIT 0x10000000	49	#define PSR_V_BIT 0x10000000
49	#define PSR_C_BIT 0x20000000	50	#define PSR_C_BIT 0x20000000

arch/arm64/kernel/Makefile+3−3

 			   sys.o stacktrace.o time.o traps.o io.o vdso.o	\
 			   hyp-stub.o psci.o psci-call.o cpu_ops.o insn.o	\
 			   return_address.o cpuinfo.o cpu_errata.o		\
-			   cpufeature.o alternative.o cacheinfo.o
+			   cpufeature.o alternative.o cacheinfo.o		\
+			   smp.o smp_spin_table.o topology.o
 arm64-obj-$(CONFIG_COMPAT)		+= sys32.o kuser32.o signal32.o 	\
 					   sys_compat.o entry32.o		\
 					   ../../arm/kernel/opcodes.o
 arm64-obj-$(CONFIG_FUNCTION_TRACER)	+= ftrace.o entry-ftrace.o
 arm64-obj-$(CONFIG_MODULES)		+= arm64ksyms.o module.o
-arm64-obj-$(CONFIG_SMP)			+= smp.o smp_spin_table.o topology.o
+arm64-obj-$(CONFIG_PERF_EVENTS)		+= perf_regs.o perf_callchain.o
-arm64-obj-$(CONFIG_PERF_EVENTS)		+= perf_regs.o
 arm64-obj-$(CONFIG_HW_PERF_EVENTS)	+= perf_event.o
 arm64-obj-$(CONFIG_HAVE_HW_BREAKPOINT)	+= hw_breakpoint.o
 arm64-obj-$(CONFIG_CPU_PM)		+= sleep.o suspend.o

arch/arm64/kernel/alternative.c+25−5

 	return insn;
 }
-static int __apply_alternatives(void *alt_region)
+static void __apply_alternatives(void *alt_region)
 {
 	struct alt_instr *alt;
 	struct alt_region *region = alt_region;
 		flush_icache_range((uintptr_t)origptr,
 				   (uintptr_t)(origptr + nr_inst));
 	}
-	return 0;
 }
-void apply_alternatives_all(void)
+/*
+ * We might be patching the stop_machine state machine, so implement a
+ * really simple polling protocol here.
+ */
+static int __apply_alternatives_multi_stop(void *unused)
 {
+	static int patched = 0;
 	struct alt_region region = {
 		.begin	= __alt_instructions,
 		.end	= __alt_instructions_end,
 	};
+	/* We always have a CPU 0 at this point (__init) */
+	if (smp_processor_id()) {
+		while (!READ_ONCE(patched))
+			cpu_relax();
+		isb();
+	} else {
+		BUG_ON(patched);
+		__apply_alternatives(&region);
+		/* Barriers provided by the cache flushing */
+		WRITE_ONCE(patched, 1);
+	}
+	return 0;
+}
+void __init apply_alternatives_all(void)
+{
 	/* better not try code patching on a live SMP system */
-	stop_machine(__apply_alternatives, &region, NULL);
+	stop_machine(__apply_alternatives_multi_stop, NULL, cpu_online_mask);
 }
 void apply_alternatives(void *start, size_t length)

arch/arm64/kernel/armv8_deprecated.c+8−11

 #include <linux/slab.h>
 #include <linux/sysctl.h>
+#include <asm/alternative.h>
+#include <asm/cpufeature.h>
 #include <asm/insn.h>
 #include <asm/opcodes.h>
+#include <asm/sysreg.h>
 #include <asm/system_misc.h>
 #include <asm/traps.h>
 #include <asm/uaccess.h>
  */
 #define __user_swpX_asm(data, addr, res, temp, B)		\
 	__asm__ __volatile__(					\
+	ALTERNATIVE("nop", SET_PSTATE_PAN(0), ARM64_HAS_PAN,	\
+		    CONFIG_ARM64_PAN)				\
 	"	mov		%w2, %w1\n"			\
 	"0:	ldxr"B"		%w1, [%3]\n"			\
 	"1:	stxr"B"		%w0, %w2, [%3]\n"		\
 	"	.align		3\n"				\
 	"	.quad		0b, 3b\n"			\
 	"	.quad		1b, 3b\n"			\
-	"	.popsection"					\
+	"	.popsection\n"					\
+	ALTERNATIVE("nop", SET_PSTATE_PAN(1), ARM64_HAS_PAN,	\
+		CONFIG_ARM64_PAN)				\
 	: "=&r" (res), "+r" (data), "=&r" (temp)		\
 	: "r" (addr), "i" (-EAGAIN), "i" (-EFAULT)		\
 	: "memory")
 	return 0;
 }
-static inline void config_sctlr_el1(u32 clear, u32 set)
-{
-	u32 val;
-	asm volatile("mrs %0, sctlr_el1" : "=r" (val));
-	val &= ~clear;
-	val |= set;
-	asm volatile("msr sctlr_el1, %0" : : "r" (val));
-}
 static int cp15_barrier_set_hw_mode(bool enable)
 {
 	if (enable)

arch/arm64/kernel/cpu_ops.c+0−2

 const struct cpu_operations *cpu_ops[NR_CPUS];
 static const struct cpu_operations *supported_cpu_ops[] __initconst = {
-#ifdef CONFIG_SMP
 	&smp_spin_table_ops,
-#endif
 	&cpu_psci_ops,
 	NULL,
 };

arch/arm64/kernel/cpufeature.c+46−7

@@ -21,24 +21,57 @@ · 20 unmodified lines
21	#include <linux/types.h>	21	#include <linux/types.h>
22	#include <asm/cpu.h>	22	#include <asm/cpu.h>
23	#include <asm/cpufeature.h>	23	#include <asm/cpufeature.h>
		24	#include <asm/processor.h>
24		25
25	static bool	26	static bool
26	has_id_aa64pfr0_feature(const struct arm64_cpu_capabilities *entry)	27	feature_matches(u64 reg, const struct arm64_cpu_capabilities *entry)
27	{	28	{
28	u64 val;	29	int val = cpuid_feature_extract_field(reg, entry->field_pos);
29		30
30	val = read_cpuid(id_aa64pfr0_el1);	31	return val >= entry->min_field_value;
31	return (val & entry->register_mask) == entry->register_value;
32	}	32	}
33		33
		34	#define __ID_FEAT_CHK(reg) \
		35	static bool __maybe_unused \
		36	has_##reg##_feature(const struct arm64_cpu_capabilities *entry) \
		37	{ \
		38	u64 val; \
		39	\
		40	val = read_cpuid(reg##_el1); \
		41	return feature_matches(val, entry); \
		42	}
		43
		44	__ID_FEAT_CHK(id_aa64pfr0);
		45	__ID_FEAT_CHK(id_aa64mmfr1);
		46	__ID_FEAT_CHK(id_aa64isar0);
		47
34	static const struct arm64_cpu_capabilities arm64_features[] = {	48	static const struct arm64_cpu_capabilities arm64_features[] = {
35	{	49	{
36	.desc = "GIC system register CPU interface",	50	.desc = "GIC system register CPU interface",
37	.capability = ARM64_HAS_SYSREG_GIC_CPUIF,	51	.capability = ARM64_HAS_SYSREG_GIC_CPUIF,
38	.matches = has_id_aa64pfr0_feature,	52	.matches = has_id_aa64pfr0_feature,
39	.register_mask = (0xf << 24),	53	.field_pos = 24,
40	.register_value = (1 << 24),	54	.min_field_value = 1,
		55	},
		56	#ifdef CONFIG_ARM64_PAN
		57	{
		58	.desc = "Privileged Access Never",
		59	.capability = ARM64_HAS_PAN,
		60	.matches = has_id_aa64mmfr1_feature,
		61	.field_pos = 20,
		62	.min_field_value = 1,
		63	.enable = cpu_enable_pan,
41	},	64	},
		65	#endif /* CONFIG_ARM64_PAN */
		66	#if defined(CONFIG_AS_LSE) && defined(CONFIG_ARM64_LSE_ATOMICS)
		67	{
		68	.desc = "LSE atomic instructions",
		69	.capability = ARM64_HAS_LSE_ATOMICS,
		70	.matches = has_id_aa64isar0_feature,
		71	.field_pos = 20,
		72	.min_field_value = 2,
		73	},
		74	#endif /* CONFIG_AS_LSE && CONFIG_ARM64_LSE_ATOMICS */
42	{},	75	{},
43	};	76	};
44		77
@@ -55,9 +88,15 @@ void check_cpu_capabilities(const struct arm64_cpu_capabilities *caps, · 10 unmodified lines
55	pr_info("%s %s\n", info, caps[i].desc);	88	pr_info("%s %s\n", info, caps[i].desc);
56	cpus_set_cap(caps[i].capability);	89	cpus_set_cap(caps[i].capability);
57	}	90	}
		91
		92	/* second pass allows enable() to consider interacting capabilities */
		93	for (i = 0; caps[i].desc; i++) {
		94	if (cpus_have_cap(caps[i].capability) && caps[i].enable)
		95	caps[i].enable();
		96	}
58	}	97	}
59		98
60	void check_local_cpu_features(void)	99	void check_local_cpu_features(void)
61	{	100	{
62	check_cpu_capabilities(arm64_features, "detected feature");	101	check_cpu_capabilities(arm64_features, "detected feature:");
63	}	102	}

arch/arm64/kernel/debug-monitors.c+2−2

@@ -82,7 +82,7 @@ early_param("nodebugmon", early_debug_disable); · 81 unmodified lines
82	static DEFINE_PER_CPU(int, mde_ref_count);	82	static DEFINE_PER_CPU(int, mde_ref_count);
83	static DEFINE_PER_CPU(int, kde_ref_count);	83	static DEFINE_PER_CPU(int, kde_ref_count);
84		84
85	void enable_debug_monitors(enum debug_el el)	85	void enable_debug_monitors(enum dbg_active_el el)
86	{	86	{
87	u32 mdscr, enable = 0;	87	u32 mdscr, enable = 0;
88		88
@@ -102,7 +102,7 @@ void enable_debug_monitors(enum debug_el el) · 13 unmodified lines
102	}	102	}
103	}	103	}
104		104
105	void disable_debug_monitors(enum debug_el el)	105	void disable_debug_monitors(enum dbg_active_el el)
106	{	106	{
107	u32 mdscr, disable = 0;	107	u32 mdscr, disable = 0;
108		108

arch/arm64/kernel/efi-stub.c+32−9

@@ -13,7 +13,7 @@ · 12 unmodified lines
13	#include <asm/efi.h>	13	#include <asm/efi.h>
14	#include <asm/sections.h>	14	#include <asm/sections.h>
15		15
16	efi_status_t __init handle_kernel_image(efi_system_table_t *sys_table,	16	efi_status_t __init handle_kernel_image(efi_system_table_t *sys_table_arg,
17	unsigned long *image_addr,	17	unsigned long *image_addr,
18	unsigned long *image_size,	18	unsigned long *image_size,
19	unsigned long *reserve_addr,	19	unsigned long *reserve_addr,
@@ -23,21 +23,44 @@ efi_status_t __init handle_kernel_image(efi_system_table_t *sys_table, · 3 unmodified lines
23	{	23	{
24	efi_status_t status;	24	efi_status_t status;
25	unsigned long kernel_size, kernel_memsize = 0;	25	unsigned long kernel_size, kernel_memsize = 0;
		26	unsigned long nr_pages;
		27	void old_image_addr = (void )*image_addr;
26		28
27	/* Relocate the image, if required. */	29	/* Relocate the image, if required. */
28	kernel_size = _edata - _text;	30	kernel_size = _edata - _text;
29	if (*image_addr != (dram_base + TEXT_OFFSET)) {	31	if (*image_addr != (dram_base + TEXT_OFFSET)) {
30	kernel_memsize = kernel_size + (_end - _edata);	32	kernel_memsize = kernel_size + (_end - _edata);
31	status = efi_low_alloc(sys_table, kernel_memsize + TEXT_OFFSET,	33
32	SZ_2M, reserve_addr);	34	/*
		35	* First, try a straight allocation at the preferred offset.
		36	* This will work around the issue where, if dram_base == 0x0,
		37	* efi_low_alloc() refuses to allocate at 0x0 (to prevent the
		38	* address of the allocation to be mistaken for a FAIL return
		39	* value or a NULL pointer). It will also ensure that, on
		40	* platforms where the [dram_base, dram_base + TEXT_OFFSET)
		41	* interval is partially occupied by the firmware (like on APM
		42	* Mustang), we can still place the kernel at the address
		43	* 'dram_base + TEXT_OFFSET'.
		44	*/
		45	image_addr = reserve_addr = dram_base + TEXT_OFFSET;
		46	nr_pages = round_up(kernel_memsize, EFI_ALLOC_ALIGN) /
		47	EFI_PAGE_SIZE;
		48	status = efi_call_early(allocate_pages, EFI_ALLOCATE_ADDRESS,
		49	EFI_LOADER_DATA, nr_pages,
		50	(efi_physical_addr_t *)reserve_addr);
33	if (status != EFI_SUCCESS) {	51	if (status != EFI_SUCCESS) {
34	pr_efi_err(sys_table, "Failed to relocate kernel\n");	52	kernel_memsize += TEXT_OFFSET;
35	return status;	53	status = efi_low_alloc(sys_table_arg, kernel_memsize,
		54	SZ_2M, reserve_addr);
		55
		56	if (status != EFI_SUCCESS) {
		57	pr_efi_err(sys_table_arg, "Failed to relocate kernel\n");
		58	return status;
		59	}
		60	image_addr = reserve_addr + TEXT_OFFSET;
36	}	61	}
37	memcpy((void )reserve_addr + TEXT_OFFSET, (void )image_addr,	62	memcpy((void )image_addr, old_image_addr, kernel_size);
38	kernel_size);	63	*reserve_size = kernel_memsize;
39	image_addr = reserve_addr + TEXT_OFFSET;
40	*reserve_size = kernel_memsize + TEXT_OFFSET;
41	}	64	}
42		65
43		66

arch/arm64/kernel/entry.S+19−27

@@ -116,41 +116,34 @@ · 115 unmodified lines
116	*/	116	*/
117	.endm	117	.endm
118		118
119	.macro kernel_exit, el, ret = 0	119	.macro kernel_exit, el
120	ldp x21, x22, [sp, #S_PC] // load ELR, SPSR	120	ldp x21, x22, [sp, #S_PC] // load ELR, SPSR
121	.if \el == 0	121	.if \el == 0
122	ct_user_enter	122	ct_user_enter
123	ldr x23, [sp, #S_SP] // load return stack pointer	123	ldr x23, [sp, #S_SP] // load return stack pointer
124	msr sp_el0, x23	124	msr sp_el0, x23
125
126	#ifdef CONFIG_ARM64_ERRATUM_845719	125	#ifdef CONFIG_ARM64_ERRATUM_845719
127		126	alternative_if_not ARM64_WORKAROUND_845719
128	#undef SEQUENCE_ORG	127	nop
129	#undef SEQUENCE_ALT	128	nop
130
131	#ifdef CONFIG_PID_IN_CONTEXTIDR	129	#ifdef CONFIG_PID_IN_CONTEXTIDR
132		130	nop
133	#define SEQUENCE_ORG "nop ; nop ; nop"	131	#endif
134	#define SEQUENCE_ALT "tbz x22, #4, 1f ; mrs x29, contextidr_el1; msr contextidr_el1, x29; 1:"	132	alternative_else
135		133	tbz x22, #4, 1f
		134	#ifdef CONFIG_PID_IN_CONTEXTIDR
		135	mrs x29, contextidr_el1
		136	msr contextidr_el1, x29
136	#else	137	#else
137		138	msr contextidr_el1, xzr
138	#define SEQUENCE_ORG "nop ; nop"
139	#define SEQUENCE_ALT "tbz x22, #4, 1f ; msr contextidr_el1, xzr; 1:"
140
141	#endif	139	#endif
142		140	1:
143	alternative_insn SEQUENCE_ORG, SEQUENCE_ALT, ARM64_WORKAROUND_845719	141	alternative_endif
144
145	#endif	142	#endif
146	.endif	143	.endif
147	msr elr_el1, x21 // set up the return data	144	msr elr_el1, x21 // set up the return data
148	msr spsr_el1, x22	145	msr spsr_el1, x22
149	.if \ret
150	ldr x1, [sp, #S_X1] // preserve x0 (syscall return)
151	.else
152	ldp x0, x1, [sp, #16 * 0]	146	ldp x0, x1, [sp, #16 * 0]
153	.endif
154	ldp x2, x3, [sp, #16 * 1]	147	ldp x2, x3, [sp, #16 * 1]
155	ldp x4, x5, [sp, #16 * 2]	148	ldp x4, x5, [sp, #16 * 2]
156	ldp x6, x7, [sp, #16 * 3]	149	ldp x6, x7, [sp, #16 * 3]
@@ -613,22 +606,21 @@ ENDPROC(cpu_switch_to) · 456 unmodified lines
613	*/	606	*/
614	ret_fast_syscall:	607	ret_fast_syscall:
615	disable_irq // disable interrupts	608	disable_irq // disable interrupts
		609	str x0, [sp, #S_X0] // returned x0
616	ldr x1, [tsk, #TI_FLAGS] // re-check for syscall tracing	610	ldr x1, [tsk, #TI_FLAGS] // re-check for syscall tracing
617	and x2, x1, #_TIF_SYSCALL_WORK	611	and x2, x1, #_TIF_SYSCALL_WORK
618	cbnz x2, ret_fast_syscall_trace	612	cbnz x2, ret_fast_syscall_trace
619	and x2, x1, #_TIF_WORK_MASK	613	and x2, x1, #_TIF_WORK_MASK
620	cbnz x2, fast_work_pending	614	cbnz x2, work_pending
621	enable_step_tsk x1, x2	615	enable_step_tsk x1, x2
622	kernel_exit 0, ret = 1	616	kernel_exit 0
623	ret_fast_syscall_trace:	617	ret_fast_syscall_trace:
624	enable_irq // enable interrupts	618	enable_irq // enable interrupts
625	b __sys_trace_return	619	b __sys_trace_return_skipped // we already saved x0
626		620
627	/*	621	/*
628	* Ok, we need to do extra processing, enter the slow path.	622	* Ok, we need to do extra processing, enter the slow path.
629	*/	623	*/
630	fast_work_pending:
631	str x0, [sp, #S_X0] // returned x0
632	work_pending:	624	work_pending:
633	tbnz x1, #TIF_NEED_RESCHED, work_resched	625	tbnz x1, #TIF_NEED_RESCHED, work_resched
634	/* TIF_SIGPENDING, TIF_NOTIFY_RESUME or TIF_FOREIGN_FPSTATE case */	626	/* TIF_SIGPENDING, TIF_NOTIFY_RESUME or TIF_FOREIGN_FPSTATE case */
@@ -652,7 +644,7 @@ ret_to_user: · 17 unmodified lines
652	cbnz x2, work_pending	644	cbnz x2, work_pending
653	enable_step_tsk x1, x2	645	enable_step_tsk x1, x2
654	no_work_pending:	646	no_work_pending:
655	kernel_exit 0, ret = 0	647	kernel_exit 0
656	ENDPROC(ret_to_user)	648	ENDPROC(ret_to_user)
657		649
658	/*	650	/*

arch/arm64/kernel/fpsimd.c+1−0

@@ -158,6 +158,7 @@ void fpsimd_thread_switch(struct task_struct *next) · 157 unmodified lines
158	void fpsimd_flush_thread(void)	158	void fpsimd_flush_thread(void)
159	{	159	{
160	memset(&current->thread.fpsimd_state, 0, sizeof(struct fpsimd_state));	160	memset(&current->thread.fpsimd_state, 0, sizeof(struct fpsimd_state));
		161	fpsimd_flush_task_state(current);
161	set_thread_flag(TIF_FOREIGN_FPSTATE);	162	set_thread_flag(TIF_FOREIGN_FPSTATE);
162	}	163	}
163		164

arch/arm64/kernel/head.S+8−7

 /*
  * Initial memory map attributes.
  */
-#ifndef CONFIG_SMP
-#define PTE_FLAGS	PTE_TYPE_PAGE | PTE_AF
-#define PMD_FLAGS	PMD_TYPE_SECT | PMD_SECT_AF
-#else
 #define PTE_FLAGS	PTE_TYPE_PAGE | PTE_AF | PTE_SHARED
 #define PMD_FLAGS	PMD_TYPE_SECT | PMD_SECT_AF | PMD_SECT_S
-#endif
 #ifdef CONFIG_ARM64_64K_PAGES
 #define MM_MMUFLAGS	PTE_ATTRINDX(MT_NORMAL) | PTE_FLAGS
 	.long	BOOT_CPU_MODE_EL1
 	.popsection
-#ifdef CONFIG_SMP
 	/*
 	 * This provides a "holding pen" for platforms to hold all secondary
 	 * cores are held until we're ready for them to initialise.
 	mov	x29, #0
 	b	secondary_start_kernel
 ENDPROC(__secondary_switched)
-#endif	/* CONFIG_SMP */
 /*
  * Enable the MMU.
 	isb
 	msr	sctlr_el1, x0
 	isb
+	/*
+	 * Invalidate the local I-cache so that any instructions fetched
+	 * speculatively from the PoC are discarded, since they may have
+	 * been dynamically patched at the PoU.
+	 */
+	ic	iallu
+	dsb	nsh
+	isb
 	br	x27
 ENDPROC(__enable_mmu)

arch/arm64/kernel/hw_breakpoint.c+3−3

@@ -156,7 +156,7 @@ static void write_wb_reg(int reg, int n, u64 val) · 155 unmodified lines
156	* Convert a breakpoint privilege level to the corresponding exception	156	* Convert a breakpoint privilege level to the corresponding exception
157	* level.	157	* level.
158	*/	158	*/
159	static enum debug_el debug_exception_level(int privilege)	159	static enum dbg_active_el debug_exception_level(int privilege)
160	{	160	{
161	switch (privilege) {	161	switch (privilege) {
162	case AARCH64_BREAKPOINT_EL0:	162	case AARCH64_BREAKPOINT_EL0:
@@ -230,7 +230,7 @@ static int hw_breakpoint_control(struct perf_event *bp, · 67 unmodified lines
230	struct perf_event **slots;	230	struct perf_event **slots;
231	struct debug_info *debug_info = &current->thread.debug;	231	struct debug_info *debug_info = &current->thread.debug;
232	int i, max_slots, ctrl_reg, val_reg, reg_enable;	232	int i, max_slots, ctrl_reg, val_reg, reg_enable;
233	enum debug_el dbg_el = debug_exception_level(info->ctrl.privilege);	233	enum dbg_active_el dbg_el = debug_exception_level(info->ctrl.privilege);
234	u32 ctrl;	234	u32 ctrl;
235		235
236	if (info->ctrl.type == ARM_BREAKPOINT_EXECUTE) {	236	if (info->ctrl.type == ARM_BREAKPOINT_EXECUTE) {
@@ -537,7 +537,7 @@ int arch_validate_hwbkpt_settings(struct perf_event *bp) · 300 unmodified lines
537	* exception level at the register level.	537	* exception level at the register level.
538	* This is used when single-stepping after a breakpoint exception.	538	* This is used when single-stepping after a breakpoint exception.
539	*/	539	*/
540	static void toggle_bp_registers(int reg, enum debug_el el, int enable)	540	static void toggle_bp_registers(int reg, enum dbg_active_el el, int enable)
541	{	541	{
542	int i, max_slots, privilege;	542	int i, max_slots, privilege;
543	u32 ctrl;	543	u32 ctrl;

arch/arm64/kernel/insn.c+2−3

@@ -101,9 +101,8 @@ static void __kprobes patch_map(void addr, int fixmap) · 100 unmodified lines
101	return addr;	101	return addr;
102		102
103	BUG_ON(!page);	103	BUG_ON(!page);
104	set_fixmap(fixmap, page_to_phys(page));	104	return (void *)set_fixmap_offset(fixmap, page_to_phys(page) +
105		105	(uintaddr & ~PAGE_MASK));
106	return (void *) (__fix_to_virt(fixmap) + (uintaddr & ~PAGE_MASK));
107	}	106	}
108		107
109	static void __kprobes patch_unmap(int fixmap)	108	static void __kprobes patch_unmap(int fixmap)

arch/arm64/kernel/irq.c+0−2

 int arch_show_interrupts(struct seq_file *p, int prec)
 {
-#ifdef CONFIG_SMP
 	show_ipi_list(p, prec);
-#endif
 	seq_printf(p, "%*s: %10lu\n", prec, "Err", irq_err_count);
 	return 0;
 }

arch/arm64/kernel/kgdb.c+6−6

@@ -235,13 +235,13 @@ static int kgdb_step_brk_fn(struct pt_regs *regs, unsigned int esr) · 234 unmodified lines
235		235
236	static struct break_hook kgdb_brkpt_hook = {	236	static struct break_hook kgdb_brkpt_hook = {
237	.esr_mask = 0xffffffff,	237	.esr_mask = 0xffffffff,
238	.esr_val = DBG_ESR_VAL_BRK(KGDB_DYN_DBG_BRK_IMM),	238	.esr_val = (u32)ESR_ELx_VAL_BRK64(KGDB_DYN_DBG_BRK_IMM),
239	.fn = kgdb_brk_fn	239	.fn = kgdb_brk_fn
240	};	240	};
241		241
242	static struct break_hook kgdb_compiled_brkpt_hook = {	242	static struct break_hook kgdb_compiled_brkpt_hook = {
243	.esr_mask = 0xffffffff,	243	.esr_mask = 0xffffffff,
244	.esr_val = DBG_ESR_VAL_BRK(KGDB_COMPILED_DBG_BRK_IMM),	244	.esr_val = (u32)ESR_ELx_VAL_BRK64(KGDB_COMPILED_DBG_BRK_IMM),
245	.fn = kgdb_compiled_brk_fn	245	.fn = kgdb_compiled_brk_fn
246	};	246	};
247		247
@@ -328,9 +328,9 @@ void kgdb_arch_exit(void) · 80 unmodified lines
328	*/	328	*/
329	struct kgdb_arch arch_kgdb_ops = {	329	struct kgdb_arch arch_kgdb_ops = {
330	.gdb_bpt_instr = {	330	.gdb_bpt_instr = {
331	KGDB_DYN_BRK_INS_BYTE0,	331	KGDB_DYN_BRK_INS_BYTE(0),
332	KGDB_DYN_BRK_INS_BYTE1,	332	KGDB_DYN_BRK_INS_BYTE(1),
333	KGDB_DYN_BRK_INS_BYTE2,	333	KGDB_DYN_BRK_INS_BYTE(2),
334	KGDB_DYN_BRK_INS_BYTE3,	334	KGDB_DYN_BRK_INS_BYTE(3),
335	}	335	}
336	};	336	};

arch/arm64/kernel/pci.c+13−0

@@ -38,6 +38,19 @@ resource_size_t pcibios_align_resource(void data, const struct resource res, · 37 unmodified lines
38	return res->start;	38	return res->start;
39	}	39	}
40		40
		41	/**
		42	* pcibios_enable_device - Enable I/O and memory.
		43	* @dev: PCI device to be enabled
		44	* @mask: bitmask of BARs to enable
		45	*/
		46	int pcibios_enable_device(struct pci_dev *dev, int mask)
		47	{
		48	if (pci_has_flag(PCI_PROBE_ONLY))
		49	return 0;
		50
		51	return pci_enable_resources(dev, mask);
		52	}
		53
41	/*	54	/*
42	* Try to assign the IRQ number from DT when adding a new device	55	* Try to assign the IRQ number from DT when adding a new device
43	*/	56	*/

arch/arm64/kernel/perf_callchain.c+196−0

@@ -0,0 +1,196 @@
	1	/*
	2	* arm64 callchain support
	3	*
	4	* Copyright (C) 2015 ARM Limited
	5	*
	6	* This program is free software; you can redistribute it and/or modify
	7	* it under the terms of the GNU General Public License version 2 as
	8	* published by the Free Software Foundation.
	9	*
	10	* This program is distributed in the hope that it will be useful,
	11	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	12	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	13	* GNU General Public License for more details.
	14	*
	15	* You should have received a copy of the GNU General Public License
	16	* along with this program. If not, see <http://www.gnu.org/licenses/>.
	17	*/
	18	#include <linux/perf_event.h>
	19	#include <linux/uaccess.h>
	20
	21	#include <asm/stacktrace.h>
	22
	23	struct frame_tail {
	24	struct frame_tail __user *fp;
	25	unsigned long lr;
	26	} __attribute__((packed));
	27
	28	/*
	29	* Get the return address for a single stackframe and return a pointer to the
	30	* next frame tail.
	31	*/
	32	static struct frame_tail __user *
	33	user_backtrace(struct frame_tail __user *tail,
	34	struct perf_callchain_entry *entry)
	35	{
	36	struct frame_tail buftail;
	37	unsigned long err;
	38
	39	/* Also check accessibility of one struct frame_tail beyond */
	40	if (!access_ok(VERIFY_READ, tail, sizeof(buftail)))
	41	return NULL;
	42
	43	pagefault_disable();
	44	err = __copy_from_user_inatomic(&buftail, tail, sizeof(buftail));
	45	pagefault_enable();
	46
	47	if (err)
	48	return NULL;
	49
	50	perf_callchain_store(entry, buftail.lr);
	51
	52	/*
	53	* Frame pointers should strictly progress back up the stack
	54	* (towards higher addresses).
	55	*/
	56	if (tail >= buftail.fp)
	57	return NULL;
	58
	59	return buftail.fp;
	60	}
	61
	62	#ifdef CONFIG_COMPAT
	63	/*
	64	* The registers we're interested in are at the end of the variable
	65	* length saved register structure. The fp points at the end of this
	66	* structure so the address of this struct is:
	67	* (struct compat_frame_tail *)(xxx->fp)-1
	68	*
	69	* This code has been adapted from the ARM OProfile support.
	70	*/
	71	struct compat_frame_tail {
	72	compat_uptr_t fp; /* a (struct compat_frame_tail ) in compat mode /
	73	u32 sp;
	74	u32 lr;
	75	} __attribute__((packed));
	76
	77	static struct compat_frame_tail __user *
	78	compat_user_backtrace(struct compat_frame_tail __user *tail,
	79	struct perf_callchain_entry *entry)
	80	{
	81	struct compat_frame_tail buftail;
	82	unsigned long err;
	83
	84	/* Also check accessibility of one struct frame_tail beyond */
	85	if (!access_ok(VERIFY_READ, tail, sizeof(buftail)))
	86	return NULL;
	87
	88	pagefault_disable();
	89	err = __copy_from_user_inatomic(&buftail, tail, sizeof(buftail));
	90	pagefault_enable();
	91
	92	if (err)
	93	return NULL;
	94
	95	perf_callchain_store(entry, buftail.lr);
	96
	97	/*
	98	* Frame pointers should strictly progress back up the stack
	99	* (towards higher addresses).
	100	*/
	101	if (tail + 1 >= (struct compat_frame_tail __user *)
	102	compat_ptr(buftail.fp))
	103	return NULL;
	104
	105	return (struct compat_frame_tail __user *)compat_ptr(buftail.fp) - 1;
	106	}
	107	#endif /* CONFIG_COMPAT */
	108
	109	void perf_callchain_user(struct perf_callchain_entry *entry,
	110	struct pt_regs *regs)
	111	{
	112	if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
	113	/* We don't support guest os callchain now */
	114	return;
	115	}
	116
	117	perf_callchain_store(entry, regs->pc);
	118
	119	if (!compat_user_mode(regs)) {
	120	/* AARCH64 mode */
	121	struct frame_tail __user *tail;
	122
	123	tail = (struct frame_tail __user *)regs->regs[29];
	124
	125	while (entry->nr < PERF_MAX_STACK_DEPTH &&
	126	tail && !((unsigned long)tail & 0xf))
	127	tail = user_backtrace(tail, entry);
	128	} else {
	129	#ifdef CONFIG_COMPAT
	130	/* AARCH32 compat mode */
	131	struct compat_frame_tail __user *tail;
	132
	133	tail = (struct compat_frame_tail __user *)regs->compat_fp - 1;
	134
	135	while ((entry->nr < PERF_MAX_STACK_DEPTH) &&
	136	tail && !((unsigned long)tail & 0x3))
	137	tail = compat_user_backtrace(tail, entry);
	138	#endif
	139	}
	140	}
	141
	142	/*
	143	* Gets called by walk_stackframe() for every stackframe. This will be called
	144	* whist unwinding the stackframe and is like a subroutine return so we use
	145	* the PC.
	146	*/
	147	static int callchain_trace(struct stackframe frame, void data)
	148	{
	149	struct perf_callchain_entry *entry = data;
	150	perf_callchain_store(entry, frame->pc);
	151	return 0;
	152	}
	153
	154	void perf_callchain_kernel(struct perf_callchain_entry *entry,
	155	struct pt_regs *regs)
	156	{
	157	struct stackframe frame;
	158
	159	if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
	160	/* We don't support guest os callchain now */
	161	return;
	162	}
	163
	164	frame.fp = regs->regs[29];
	165	frame.sp = regs->sp;
	166	frame.pc = regs->pc;
	167
	168	walk_stackframe(&frame, callchain_trace, entry);
	169	}
	170
	171	unsigned long perf_instruction_pointer(struct pt_regs *regs)
	172	{
	173	if (perf_guest_cbs && perf_guest_cbs->is_in_guest())
	174	return perf_guest_cbs->get_guest_ip();
	175
	176	return instruction_pointer(regs);
	177	}
	178
	179	unsigned long perf_misc_flags(struct pt_regs *regs)
	180	{
	181	int misc = 0;
	182
	183	if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
	184	if (perf_guest_cbs->is_user_mode())
	185	misc \|= PERF_RECORD_MISC_GUEST_USER;
	186	else
	187	misc \|= PERF_RECORD_MISC_GUEST_KERNEL;
	188	} else {
	189	if (user_mode(regs))
	190	misc \|= PERF_RECORD_MISC_USER;
	191	else
	192	misc \|= PERF_RECORD_MISC_KERNEL;
	193	}
	194
	195	return misc;
	196	}

arch/arm64/kernel/perf_event.c+25−285

@@ -25,7 +25,7 @@ · 24 unmodified lines
25	#include <linux/irq.h>	25	#include <linux/irq.h>
26	#include <linux/kernel.h>	26	#include <linux/kernel.h>
27	#include <linux/export.h>	27	#include <linux/export.h>
28	#include <linux/of.h>	28	#include <linux/of_device.h>
29	#include <linux/perf_event.h>	29	#include <linux/perf_event.h>
30	#include <linux/platform_device.h>	30	#include <linux/platform_device.h>
31	#include <linux/slab.h>	31	#include <linux/slab.h>
@@ -36,7 +36,6 @@ · 4 unmodified lines
36	#include <asm/irq.h>	36	#include <asm/irq.h>
37	#include <asm/irq_regs.h>	37	#include <asm/irq_regs.h>
38	#include <asm/pmu.h>	38	#include <asm/pmu.h>
39	#include <asm/stacktrace.h>
40		39
41	/*	40	/*
42	* ARMv8 supports a maximum of 32 events.	41	* ARMv8 supports a maximum of 32 events.
@@ -78,6 +77,16 @@ EXPORT_SYMBOL_GPL(perf_num_counters); · 35 unmodified lines
78		77
79	#define CACHE_OP_UNSUPPORTED 0xFFFF	78	#define CACHE_OP_UNSUPPORTED 0xFFFF
80		79
		80	#define PERF_MAP_ALL_UNSUPPORTED \
		81	[0 ... PERF_COUNT_HW_MAX - 1] = HW_OP_UNSUPPORTED
		82
		83	#define PERF_CACHE_MAP_ALL_UNSUPPORTED \
		84	[0 ... C(MAX) - 1] = { \
		85	[0 ... C(OP_MAX) - 1] = { \
		86	[0 ... C(RESULT_MAX) - 1] = CACHE_OP_UNSUPPORTED, \
		87	}, \
		88	}
		89
81	static int	90	static int
82	armpmu_map_cache_event(const unsigned (*cache_map)	91	armpmu_map_cache_event(const unsigned (*cache_map)
83	[PERF_COUNT_HW_CACHE_MAX]	92	[PERF_COUNT_HW_CACHE_MAX]
@@ -435,10 +444,8 @@ armpmu_reserve_hardware(struct arm_pmu *armpmu) · 351 unmodified lines
435	unsigned int i, irqs;	444	unsigned int i, irqs;
436	struct platform_device *pmu_device = armpmu->plat_device;	445	struct platform_device *pmu_device = armpmu->plat_device;
437		446
438	if (!pmu_device) {	447	if (!pmu_device)
439	pr_err("no PMU device registered\n");
440	return -ENODEV;	448	return -ENODEV;
441	}
442		449
443	irqs = min(pmu_device->num_resources, num_possible_cpus());	450	irqs = min(pmu_device->num_resources, num_possible_cpus());
444	if (!irqs) {	451	if (!irqs) {
@@ -703,118 +710,28 @@ enum armv8_pmuv3_perf_types { · 258 unmodified lines
703		710
704	/* PMUv3 HW events mapping. */	711	/* PMUv3 HW events mapping. */
705	static const unsigned armv8_pmuv3_perf_map[PERF_COUNT_HW_MAX] = {	712	static const unsigned armv8_pmuv3_perf_map[PERF_COUNT_HW_MAX] = {
		713	PERF_MAP_ALL_UNSUPPORTED,
706	[PERF_COUNT_HW_CPU_CYCLES] = ARMV8_PMUV3_PERFCTR_CLOCK_CYCLES,	714	[PERF_COUNT_HW_CPU_CYCLES] = ARMV8_PMUV3_PERFCTR_CLOCK_CYCLES,
707	[PERF_COUNT_HW_INSTRUCTIONS] = ARMV8_PMUV3_PERFCTR_INSTR_EXECUTED,	715	[PERF_COUNT_HW_INSTRUCTIONS] = ARMV8_PMUV3_PERFCTR_INSTR_EXECUTED,
708	[PERF_COUNT_HW_CACHE_REFERENCES] = ARMV8_PMUV3_PERFCTR_L1_DCACHE_ACCESS,	716	[PERF_COUNT_HW_CACHE_REFERENCES] = ARMV8_PMUV3_PERFCTR_L1_DCACHE_ACCESS,
709	[PERF_COUNT_HW_CACHE_MISSES] = ARMV8_PMUV3_PERFCTR_L1_DCACHE_REFILL,	717	[PERF_COUNT_HW_CACHE_MISSES] = ARMV8_PMUV3_PERFCTR_L1_DCACHE_REFILL,
710	[PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = HW_OP_UNSUPPORTED,
711	[PERF_COUNT_HW_BRANCH_MISSES] = ARMV8_PMUV3_PERFCTR_PC_BRANCH_MIS_PRED,	718	[PERF_COUNT_HW_BRANCH_MISSES] = ARMV8_PMUV3_PERFCTR_PC_BRANCH_MIS_PRED,
712	[PERF_COUNT_HW_BUS_CYCLES] = HW_OP_UNSUPPORTED,
713	[PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = HW_OP_UNSUPPORTED,
714	[PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = HW_OP_UNSUPPORTED,
715	};	719	};
716		720
717	static const unsigned armv8_pmuv3_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]	721	static const unsigned armv8_pmuv3_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
718	[PERF_COUNT_HW_CACHE_OP_MAX]	722	[PERF_COUNT_HW_CACHE_OP_MAX]
719	[PERF_COUNT_HW_CACHE_RESULT_MAX] = {	723	[PERF_COUNT_HW_CACHE_RESULT_MAX] = {
720	[C(L1D)] = {	724	PERF_CACHE_MAP_ALL_UNSUPPORTED,
721	[C(OP_READ)] = {	725
722	[C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_L1_DCACHE_ACCESS,	726	[C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_L1_DCACHE_ACCESS,
723	[C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L1_DCACHE_REFILL,	727	[C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L1_DCACHE_REFILL,
724	},	728	[C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_L1_DCACHE_ACCESS,
725	[C(OP_WRITE)] = {	729	[C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L1_DCACHE_REFILL,
726	[C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_L1_DCACHE_ACCESS,	730
727	[C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L1_DCACHE_REFILL,	731	[C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_PC_BRANCH_PRED,
728	},	732	[C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_PC_BRANCH_MIS_PRED,
729	[C(OP_PREFETCH)] = {	733	[C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_PC_BRANCH_PRED,
730	[C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,	734	[C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_PC_BRANCH_MIS_PRED,
731	[C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
732	},
733	},
734	[C(L1I)] = {
735	[C(OP_READ)] = {
736	[C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
737	[C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
738	},
739	[C(OP_WRITE)] = {
740	[C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
741	[C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
742	},
743	[C(OP_PREFETCH)] = {
744	[C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
745	[C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
746	},
747	},
748	[C(LL)] = {
749	[C(OP_READ)] = {
750	[C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
751	[C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
752	},
753	[C(OP_WRITE)] = {
754	[C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
755	[C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
756	},
757	[C(OP_PREFETCH)] = {
758	[C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
759	[C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
760	},
761	},
762	[C(DTLB)] = {
763	[C(OP_READ)] = {
764	[C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
765	[C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
766	},
767	[C(OP_WRITE)] = {
768	[C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
769	[C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
770	},
771	[C(OP_PREFETCH)] = {
772	[C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
773	[C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
774	},
775	},
776	[C(ITLB)] = {
777	[C(OP_READ)] = {
778	[C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
779	[C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
780	},
781	[C(OP_WRITE)] = {
782	[C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
783	[C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
784	},
785	[C(OP_PREFETCH)] = {
786	[C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
787	[C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
788	},
789	},
790	[C(BPU)] = {
791	[C(OP_READ)] = {
792	[C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_PC_BRANCH_PRED,
793	[C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_PC_BRANCH_MIS_PRED,
794	},
795	[C(OP_WRITE)] = {
796	[C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_PC_BRANCH_PRED,
797	[C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_PC_BRANCH_MIS_PRED,
798	},
799	[C(OP_PREFETCH)] = {
800	[C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
801	[C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
802	},
803	},
804	[C(NODE)] = {
805	[C(OP_READ)] = {
806	[C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
807	[C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
808	},
809	[C(OP_WRITE)] = {
810	[C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
811	[C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
812	},
813	[C(OP_PREFETCH)] = {
814	[C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
815	[C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
816	},
817	},
818	};	735	};
819		736
820	/*	737	/*
@@ -1337,7 +1254,7 @@ static int armpmu_device_probe(struct platform_device *pdev) · 516 unmodified lines
1337	}	1254	}
1338		1255
1339	for_each_possible_cpu(cpu)	1256	for_each_possible_cpu(cpu)
1340	if (arch_find_n_match_cpu_physical_id(dn, cpu, NULL))	1257	if (dn == of_cpu_device_node_get(cpu))
1341	break;	1258	break;
1342		1259
1343	if (cpu >= nr_cpu_ids) {	1260	if (cpu >= nr_cpu_ids) {
@@ -1415,180 +1332,3 @@ static int __init init_hw_perf_events(void) · 71 unmodified lines
1415	}	1332	}
1416	early_initcall(init_hw_perf_events);	1333	early_initcall(init_hw_perf_events);
1417		1334
1418	/*
1419	* Callchain handling code.
1420	*/
1421	struct frame_tail {
1422	struct frame_tail __user *fp;
1423	unsigned long lr;
1424	} __attribute__((packed));
1425
1426	/*
1427	* Get the return address for a single stackframe and return a pointer to the
1428	* next frame tail.
1429	*/
1430	static struct frame_tail __user *
1431	user_backtrace(struct frame_tail __user *tail,
1432	struct perf_callchain_entry *entry)
1433	{
1434	struct frame_tail buftail;
1435	unsigned long err;
1436
1437	/* Also check accessibility of one struct frame_tail beyond */
1438	if (!access_ok(VERIFY_READ, tail, sizeof(buftail)))
1439	return NULL;
1440
1441	pagefault_disable();
1442	err = __copy_from_user_inatomic(&buftail, tail, sizeof(buftail));
1443	pagefault_enable();
1444
1445	if (err)
1446	return NULL;
1447
1448	perf_callchain_store(entry, buftail.lr);
1449
1450	/*
1451	* Frame pointers should strictly progress back up the stack
1452	* (towards higher addresses).
1453	*/
1454	if (tail >= buftail.fp)
1455	return NULL;
1456
1457	return buftail.fp;
1458	}
1459
1460	#ifdef CONFIG_COMPAT
1461	/*
1462	* The registers we're interested in are at the end of the variable
1463	* length saved register structure. The fp points at the end of this
1464	* structure so the address of this struct is:
1465	* (struct compat_frame_tail *)(xxx->fp)-1
1466	*
1467	* This code has been adapted from the ARM OProfile support.
1468	*/
1469	struct compat_frame_tail {
1470	compat_uptr_t fp; /* a (struct compat_frame_tail ) in compat mode /
1471	u32 sp;
1472	u32 lr;
1473	} __attribute__((packed));
1474
1475	static struct compat_frame_tail __user *
1476	compat_user_backtrace(struct compat_frame_tail __user *tail,
1477	struct perf_callchain_entry *entry)
1478	{
1479	struct compat_frame_tail buftail;
1480	unsigned long err;
1481
1482	/* Also check accessibility of one struct frame_tail beyond */
1483	if (!access_ok(VERIFY_READ, tail, sizeof(buftail)))
1484	return NULL;
1485
1486	pagefault_disable();
1487	err = __copy_from_user_inatomic(&buftail, tail, sizeof(buftail));
1488	pagefault_enable();
1489
1490	if (err)
1491	return NULL;
1492
1493	perf_callchain_store(entry, buftail.lr);
1494
1495	/*
1496	* Frame pointers should strictly progress back up the stack
1497	* (towards higher addresses).
1498	*/
1499	if (tail + 1 >= (struct compat_frame_tail __user *)
1500	compat_ptr(buftail.fp))
1501	return NULL;
1502
1503	return (struct compat_frame_tail __user *)compat_ptr(buftail.fp) - 1;
1504	}
1505	#endif /* CONFIG_COMPAT */
1506
1507	void perf_callchain_user(struct perf_callchain_entry *entry,
1508	struct pt_regs *regs)
1509	{
1510	if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
1511	/* We don't support guest os callchain now */
1512	return;
1513	}
1514
1515	perf_callchain_store(entry, regs->pc);
1516
1517	if (!compat_user_mode(regs)) {
1518	/* AARCH64 mode */
1519	struct frame_tail __user *tail;
1520
1521	tail = (struct frame_tail __user *)regs->regs[29];
1522
1523	while (entry->nr < PERF_MAX_STACK_DEPTH &&
1524	tail && !((unsigned long)tail & 0xf))
1525	tail = user_backtrace(tail, entry);
1526	} else {
1527	#ifdef CONFIG_COMPAT
1528	/* AARCH32 compat mode */
1529	struct compat_frame_tail __user *tail;
1530
1531	tail = (struct compat_frame_tail __user *)regs->compat_fp - 1;
1532
1533	while ((entry->nr < PERF_MAX_STACK_DEPTH) &&
1534	tail && !((unsigned long)tail & 0x3))
1535	tail = compat_user_backtrace(tail, entry);
1536	#endif
1537	}
1538	}
1539
1540	/*
1541	* Gets called by walk_stackframe() for every stackframe. This will be called
1542	* whist unwinding the stackframe and is like a subroutine return so we use
1543	* the PC.
1544	*/
1545	static int callchain_trace(struct stackframe frame, void data)
1546	{
1547	struct perf_callchain_entry *entry = data;
1548	perf_callchain_store(entry, frame->pc);
1549	return 0;
1550	}
1551
1552	void perf_callchain_kernel(struct perf_callchain_entry *entry,
1553	struct pt_regs *regs)
1554	{
1555	struct stackframe frame;
1556
1557	if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
1558	/* We don't support guest os callchain now */
1559	return;
1560	}
1561
1562	frame.fp = regs->regs[29];
1563	frame.sp = regs->sp;
1564	frame.pc = regs->pc;
1565
1566	walk_stackframe(&frame, callchain_trace, entry);
1567	}
1568
1569	unsigned long perf_instruction_pointer(struct pt_regs *regs)
1570	{
1571	if (perf_guest_cbs && perf_guest_cbs->is_in_guest())
1572	return perf_guest_cbs->get_guest_ip();
1573
1574	return instruction_pointer(regs);
1575	}
1576
1577	unsigned long perf_misc_flags(struct pt_regs *regs)
1578	{
1579	int misc = 0;
1580
1581	if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
1582	if (perf_guest_cbs->is_user_mode())
1583	misc \|= PERF_RECORD_MISC_GUEST_USER;
1584	else
1585	misc \|= PERF_RECORD_MISC_GUEST_KERNEL;
1586	} else {
1587	if (user_mode(regs))
1588	misc \|= PERF_RECORD_MISC_USER;
1589	else
1590	misc \|= PERF_RECORD_MISC_KERNEL;
1591	}
1592
1593	return misc;
1594	}

arch/arm64/kernel/psci.c+0−5

 	return ret;
 }
-#ifdef CONFIG_SMP
 static int __init cpu_psci_cpu_init(unsigned int cpu)
 {
 	return 0;
 	return -ETIMEDOUT;
 }
 #endif
-#endif
 static int psci_suspend_finisher(unsigned long index)
 {
 	.cpu_init_idle	= cpu_psci_cpu_init_idle,
 	.cpu_suspend	= cpu_psci_cpu_suspend,
 #endif
-#ifdef CONFIG_SMP
 	.cpu_init	= cpu_psci_cpu_init,
 	.cpu_prepare	= cpu_psci_cpu_prepare,
 	.cpu_boot	= cpu_psci_cpu_boot,
 	.cpu_die	= cpu_psci_cpu_die,
 	.cpu_kill	= cpu_psci_cpu_kill,
 #endif
-#endif
 };

arch/arm64/kernel/ptrace.c+91−1

@@ -826,6 +826,30 @@ static int compat_vfp_set(struct task_struct *target, · 825 unmodified lines
826	return ret;	826	return ret;
827	}	827	}
828		828
		829	static int compat_tls_get(struct task_struct *target,
		830	const struct user_regset *regset, unsigned int pos,
		831	unsigned int count, void kbuf, void __user ubuf)
		832	{
		833	compat_ulong_t tls = (compat_ulong_t)target->thread.tp_value;
		834	return user_regset_copyout(&pos, &count, &kbuf, &ubuf, &tls, 0, -1);
		835	}
		836
		837	static int compat_tls_set(struct task_struct *target,
		838	const struct user_regset *regset, unsigned int pos,
		839	unsigned int count, const void *kbuf,
		840	const void __user *ubuf)
		841	{
		842	int ret;
		843	compat_ulong_t tls;
		844
		845	ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &tls, 0, -1);
		846	if (ret)
		847	return ret;
		848
		849	target->thread.tp_value = tls;
		850	return ret;
		851	}
		852
829	static const struct user_regset aarch32_regsets[] = {	853	static const struct user_regset aarch32_regsets[] = {
830	[REGSET_COMPAT_GPR] = {	854	[REGSET_COMPAT_GPR] = {
831	.core_note_type = NT_PRSTATUS,	855	.core_note_type = NT_PRSTATUS,
@@ -850,6 +874,64 @@ static const struct user_regset_view user_aarch32_view = { · 18 unmodified lines
850	.regsets = aarch32_regsets, .n = ARRAY_SIZE(aarch32_regsets)	874	.regsets = aarch32_regsets, .n = ARRAY_SIZE(aarch32_regsets)
851	};	875	};
852		876
		877	static const struct user_regset aarch32_ptrace_regsets[] = {
		878	[REGSET_GPR] = {
		879	.core_note_type = NT_PRSTATUS,
		880	.n = COMPAT_ELF_NGREG,
		881	.size = sizeof(compat_elf_greg_t),
		882	.align = sizeof(compat_elf_greg_t),
		883	.get = compat_gpr_get,
		884	.set = compat_gpr_set
		885	},
		886	[REGSET_FPR] = {
		887	.core_note_type = NT_ARM_VFP,
		888	.n = VFP_STATE_SIZE / sizeof(compat_ulong_t),
		889	.size = sizeof(compat_ulong_t),
		890	.align = sizeof(compat_ulong_t),
		891	.get = compat_vfp_get,
		892	.set = compat_vfp_set
		893	},
		894	[REGSET_TLS] = {
		895	.core_note_type = NT_ARM_TLS,
		896	.n = 1,
		897	.size = sizeof(compat_ulong_t),
		898	.align = sizeof(compat_ulong_t),
		899	.get = compat_tls_get,
		900	.set = compat_tls_set,
		901	},
		902	#ifdef CONFIG_HAVE_HW_BREAKPOINT
		903	[REGSET_HW_BREAK] = {
		904	.core_note_type = NT_ARM_HW_BREAK,
		905	.n = sizeof(struct user_hwdebug_state) / sizeof(u32),
		906	.size = sizeof(u32),
		907	.align = sizeof(u32),
		908	.get = hw_break_get,
		909	.set = hw_break_set,
		910	},
		911	[REGSET_HW_WATCH] = {
		912	.core_note_type = NT_ARM_HW_WATCH,
		913	.n = sizeof(struct user_hwdebug_state) / sizeof(u32),
		914	.size = sizeof(u32),
		915	.align = sizeof(u32),
		916	.get = hw_break_get,
		917	.set = hw_break_set,
		918	},
		919	#endif
		920	[REGSET_SYSTEM_CALL] = {
		921	.core_note_type = NT_ARM_SYSTEM_CALL,
		922	.n = 1,
		923	.size = sizeof(int),
		924	.align = sizeof(int),
		925	.get = system_call_get,
		926	.set = system_call_set,
		927	},
		928	};
		929
		930	static const struct user_regset_view user_aarch32_ptrace_view = {
		931	.name = "aarch32", .e_machine = EM_ARM,
		932	.regsets = aarch32_ptrace_regsets, .n = ARRAY_SIZE(aarch32_ptrace_regsets)
		933	};
		934
853	static int compat_ptrace_read_user(struct task_struct *tsk, compat_ulong_t off,	935	static int compat_ptrace_read_user(struct task_struct *tsk, compat_ulong_t off,
854	compat_ulong_t __user *ret)	936	compat_ulong_t __user *ret)
855	{	937	{
@@ -1109,8 +1191,16 @@ long compat_arch_ptrace(struct task_struct *child, compat_long_t request, · 253 unmodified lines
1109	const struct user_regset_view task_user_regset_view(struct task_struct task)	1191	const struct user_regset_view task_user_regset_view(struct task_struct task)
1110	{	1192	{
1111	#ifdef CONFIG_COMPAT	1193	#ifdef CONFIG_COMPAT
1112	if (is_compat_thread(task_thread_info(task)))	1194	/*
		1195	* Core dumping of 32-bit tasks or compat ptrace requests must use the
		1196	* user_aarch32_view compatible with arm32. Native ptrace requests on
		1197	* 32-bit children use an extended user_aarch32_ptrace_view to allow
		1198	* access to the TLS register.
		1199	*/
		1200	if (is_compat_task())
1113	return &user_aarch32_view;	1201	return &user_aarch32_view;
		1202	else if (is_compat_thread(task_thread_info(task)))
		1203	return &user_aarch32_ptrace_view;
1114	#endif	1204	#endif
1115	return &user_aarch64_view;	1205	return &user_aarch64_view;
1116	}	1206	}

arch/arm64/kernel/setup.c+34−51

 #include <asm/traps.h>
 #include <asm/memblock.h>
 #include <asm/efi.h>
-#include <asm/virt.h>
 #include <asm/xen/hypervisor.h>
 unsigned long elf_hwcap __read_mostly;
 }
 struct mpidr_hash mpidr_hash;
-#ifdef CONFIG_SMP
 /**
  * smp_build_mpidr_hash - Pre-compute shifts required at each affinity
  *			  level in order to build a linear index from an
 		pr_warn("Large number of MPIDR hash buckets detected\n");
 	__flush_dcache_area(&mpidr_hash, sizeof(struct mpidr_hash));
 }
-#endif
-static void __init hyp_mode_check(void)
-{
-	if (is_hyp_mode_available())
-		pr_info("CPU: All CPU(s) started at EL2\n");
-	else if (is_hyp_mode_mismatched())
-		WARN_TAINT(1, TAINT_CPU_OUT_OF_SPEC,
-			   "CPU: CPUs started in inconsistent modes");
-	else
-		pr_info("CPU: All CPU(s) started at EL1\n");
-}
-void __init do_post_cpus_up_work(void)
-{
-	hyp_mode_check();
-	apply_alternatives_all();
-}
-#ifdef CONFIG_UP_LATE_INIT
-void __init up_late_init(void)
-{
-	do_post_cpus_up_work();
-}
-#endif /* CONFIG_UP_LATE_INIT */
 static void __init setup_processor(void)
 {
-	u64 features, block;
+	u64 features;
+	s64 block;
 	u32 cwg;
 	int cls;
 	 * for non-negative values. Negative values are reserved.
 	 */
 	features = read_cpuid(ID_AA64ISAR0_EL1);
-	block = (features >> 4) & 0xf;
+	block = cpuid_feature_extract_field(features, 4);
-	if (!(block & 0x8)) {
+	if (block > 0) {
 		switch (block) {
 		default:
 		case 2:
 		}
 	}
-	block = (features >> 8) & 0xf;
+	if (cpuid_feature_extract_field(features, 8) > 0)
-	if (block && !(block & 0x8))
 		elf_hwcap |= HWCAP_SHA1;
-	block = (features >> 12) & 0xf;
+	if (cpuid_feature_extract_field(features, 12) > 0)
-	if (block && !(block & 0x8))
 		elf_hwcap |= HWCAP_SHA2;
-	block = (features >> 16) & 0xf;
+	if (cpuid_feature_extract_field(features, 16) > 0)
-	if (block && !(block & 0x8))
 		elf_hwcap |= HWCAP_CRC32;
+	block = cpuid_feature_extract_field(features, 20);
+	if (block > 0) {
+		switch (block) {
+		default:
+		case 2:
+			elf_hwcap |= HWCAP_ATOMICS;
+		case 1:
+			/* RESERVED */
+		case 0:
+			break;
+		}
+	}
 #ifdef CONFIG_COMPAT
 	/*
 	 * ID_ISAR5_EL1 carries similar information as above, but pertaining to
-	 * the Aarch32 32-bit execution state.
+	 * the AArch32 32-bit execution state.
 	 */
 	features = read_cpuid(ID_ISAR5_EL1);
-	block = (features >> 4) & 0xf;
+	block = cpuid_feature_extract_field(features, 4);
-	if (!(block & 0x8)) {
+	if (block > 0) {
 		switch (block) {
 		default:
 		case 2:
 		}
 	}
-	block = (features >> 8) & 0xf;
+	if (cpuid_feature_extract_field(features, 8) > 0)
-	if (block && !(block & 0x8))
 		compat_elf_hwcap2 |= COMPAT_HWCAP2_SHA1;
-	block = (features >> 12) & 0xf;
+	if (cpuid_feature_extract_field(features, 12) > 0)
-	if (block && !(block & 0x8))
 		compat_elf_hwcap2 |= COMPAT_HWCAP2_SHA2;
-	block = (features >> 16) & 0xf;
+	if (cpuid_feature_extract_field(features, 16) > 0)
-	if (block && !(block & 0x8))
 		compat_elf_hwcap2 |= COMPAT_HWCAP2_CRC32;
 #endif
 }
 	xen_early_init();
 	cpu_read_bootcpu_ops();
-#ifdef CONFIG_SMP
 	smp_init_cpus();
 	smp_build_mpidr_hash();
-#endif
 #ifdef CONFIG_VT
 #if defined(CONFIG_VGA_CONSOLE)
 static int __init arm64_device_init(void)
 {
-	of_iommu_init();
+	if (of_have_populated_dt()) {
-	of_platform_populate(NULL, of_default_bus_match_table, NULL, NULL);
+		of_iommu_init();
+		of_platform_populate(NULL, of_default_bus_match_table,
+				     NULL, NULL);
+	} else if (acpi_disabled) {
+		pr_crit("Device tree not populated\n");
+	}
 	return 0;
 }
 arch_initcall_sync(arm64_device_init);
 	"sha1",
 	"sha2",
 	"crc32",
+	"atomics",
 	NULL
 };
 		 * online processors, looking for lines beginning with
 		 * "processor".  Give glibc what it expects.
 		 */
-#ifdef CONFIG_SMP
 		seq_printf(m, "processor\t: %d\n", i);
-#endif
 		/*
 		 * Dump out the common processor features in a single line.

arch/arm64/kernel/sleep.S+8−6

 	str	x2, [x0, #CPU_CTX_SP]
 	ldr	x1, =sleep_save_sp
 	ldr	x1, [x1, #SLEEP_SAVE_SP_VIRT]
-#ifdef CONFIG_SMP
 	mrs	x7, mpidr_el1
 	ldr	x9, =mpidr_hash
 	ldr	x10, [x9, #MPIDR_HASH_MASK]
 	ldp	w5, w6, [x9, #(MPIDR_HASH_SHIFTS + 8)]
 	compute_mpidr_hash x8, x3, x4, x5, x6, x7, x10
 	add	x1, x1, x8, lsl #3
-#endif
 	bl	__cpu_suspend_save
 	/*
 	 * Grab suspend finisher in x20 and its argument in x19
 	ldr	x3, =cpu_resume_after_mmu
 	msr	sctlr_el1, x0		// restore sctlr_el1
 	isb
+	/*
+	 * Invalidate the local I-cache so that any instructions fetched
+	 * speculatively from the PoC are discarded, since they may have
+	 * been dynamically patched at the PoU.
+	 */
+	ic	iallu
+	dsb	nsh
+	isb
 	br	x3			// global jump to virtual address
 ENDPROC(cpu_resume_mmu)
 	.popsection
 ENTRY(cpu_resume)
 	bl	el2_setup		// if in EL2 drop to EL1 cleanly
-#ifdef CONFIG_SMP
 	mrs	x1, mpidr_el1
 	adrp	x8, mpidr_hash
 	add x8, x8, #:lo12:mpidr_hash // x8 = struct mpidr_hash phys address
 	ldp	w5, w6, [x8, #(MPIDR_HASH_SHIFTS + 8)]
 	compute_mpidr_hash x7, x3, x4, x5, x6, x1, x2
         /* x7 contains hash index, let's use it to grab context pointer */
-#else
-	mov	x7, xzr
-#endif
 	ldr_l	x0, sleep_save_sp + SLEEP_SAVE_SP_PHYS
 	ldr	x0, [x0, x7, lsl #3]
 	/* load sp from context */

arch/arm64/kernel/smp.c+14−1

 #include <asm/sections.h>
 #include <asm/tlbflush.h>
 #include <asm/ptrace.h>
+#include <asm/virt.h>
 #define CREATE_TRACE_POINTS
 #include <trace/events/ipi.h>
 }
 #endif
+static void __init hyp_mode_check(void)
+{
+	if (is_hyp_mode_available())
+		pr_info("CPU: All CPU(s) started at EL2\n");
+	else if (is_hyp_mode_mismatched())
+		WARN_TAINT(1, TAINT_CPU_OUT_OF_SPEC,
+			   "CPU: CPUs started in inconsistent modes");
+	else
+		pr_info("CPU: All CPU(s) started at EL1\n");
+}
 void __init smp_cpus_done(unsigned int max_cpus)
 {
 	pr_info("SMP: Total of %d processors activated.\n", num_online_cpus());
-	do_post_cpus_up_work();
+	hyp_mode_check();
+	apply_alternatives_all();
 }
 void __init smp_prepare_boot_cpu(void)

arch/arm64/kernel/time.c+0−2

 #include <asm/thread_info.h>
 #include <asm/stacktrace.h>
-#ifdef CONFIG_SMP
 unsigned long profile_pc(struct pt_regs *regs)
 {
 	struct stackframe frame;
 	return frame.pc;
 }
 EXPORT_SYMBOL(profile_pc);
-#endif
 void __init time_init(void)
 {

arch/arm64/kernel/topology.c+1−1

@@ -300,6 +300,6 @@ void __init init_cpu_topology(void) · 299 unmodified lines
300	* Discard anything that was parsed if we hit an error so we	300	* Discard anything that was parsed if we hit an error so we
301	* don't use partial information.	301	* don't use partial information.
302	*/	302	*/
303	if (parse_dt_topology())	303	if (of_have_populated_dt() && parse_dt_topology())
304	reset_cpu_topology();	304	reset_cpu_topology();
305	}	305	}

arch/arm64/kernel/traps.c+80−14

@@ -17,6 +17,7 @@ · 16 unmodified lines
17	* along with this program. If not, see <http://www.gnu.org/licenses/>.	17	* along with this program. If not, see <http://www.gnu.org/licenses/>.
18	*/	18	*/
19		19
		20	#include <linux/bug.h>
20	#include <linux/signal.h>	21	#include <linux/signal.h>
21	#include <linux/personality.h>	22	#include <linux/personality.h>
22	#include <linux/kallsyms.h>	23	#include <linux/kallsyms.h>
@@ -32,8 +33,10 @@ · 9 unmodified lines
32	#include <linux/syscalls.h>	33	#include <linux/syscalls.h>
33		34
34	#include <asm/atomic.h>	35	#include <asm/atomic.h>
		36	#include <asm/bug.h>
35	#include <asm/debug-monitors.h>	37	#include <asm/debug-monitors.h>
36	#include <asm/esr.h>	38	#include <asm/esr.h>
		39	#include <asm/insn.h>
37	#include <asm/traps.h>	40	#include <asm/traps.h>
38	#include <asm/stacktrace.h>	41	#include <asm/stacktrace.h>
39	#include <asm/exception.h>	42	#include <asm/exception.h>
@@ -52,11 +55,12 @@ int show_unhandled_signals = 1; · 12 unmodified lines
52	* Dump out the contents of some memory nicely...	55	* Dump out the contents of some memory nicely...
53	*/	56	*/
54	static void dump_mem(const char lvl, const char str, unsigned long bottom,	57	static void dump_mem(const char lvl, const char str, unsigned long bottom,
55	unsigned long top)	58	unsigned long top, bool compat)
56	{	59	{
57	unsigned long first;	60	unsigned long first;
58	mm_segment_t fs;	61	mm_segment_t fs;
59	int i;	62	int i;
		63	unsigned int width = compat ? 4 : 8;
60		64
61	/*	65	/*
62	* We need to switch to kernel mode so that we can use __get_user	66	* We need to switch to kernel mode so that we can use __get_user
@@ -75,13 +79,22 @@ static void dump_mem(const char lvl, const char str, unsigned long bottom, · 12 unmodified lines
75	memset(str, ' ', sizeof(str));	79	memset(str, ' ', sizeof(str));
76	str[sizeof(str) - 1] = '\0';	80	str[sizeof(str) - 1] = '\0';
77		81
78	for (p = first, i = 0; i < 8 && p < top; i++, p += 4) {	82	for (p = first, i = 0; i < (32 / width)
		83	&& p < top; i++, p += width) {
79	if (p >= bottom && p < top) {	84	if (p >= bottom && p < top) {
80	unsigned int val;	85	unsigned long val;
81	if (__get_user(val, (unsigned int *)p) == 0)	86
82	sprintf(str + i * 9, " %08x", val);	87	if (width == 8) {
83	else	88	if (__get_user(val, (unsigned long *)p) == 0)
84	sprintf(str + i * 9, " ????????");	89	sprintf(str + i * 17, " %016lx", val);
		90	else
		91	sprintf(str + i * 17, " ????????????????");
		92	} else {
		93	if (__get_user(val, (unsigned int *)p) == 0)
		94	sprintf(str + i * 9, " %08lx", val);
		95	else
		96	sprintf(str + i * 9, " ????????");
		97	}
85	}	98	}
86	}	99	}
87	printk("%s%04lx:%s\n", lvl, first & 0xffff, str);	100	printk("%s%04lx:%s\n", lvl, first & 0xffff, str);
@@ -95,7 +108,7 @@ static void dump_backtrace_entry(unsigned long where, unsigned long stack) · 7 unmodified lines
95	print_ip_sym(where);	108	print_ip_sym(where);
96	if (in_exception_text(where))	109	if (in_exception_text(where))
97	dump_mem("", "Exception stack", stack,	110	dump_mem("", "Exception stack", stack,
98	stack + sizeof(struct pt_regs));	111	stack + sizeof(struct pt_regs), false);
99	}	112	}
100		113
101	static void dump_instr(const char lvl, struct pt_regs regs)	114	static void dump_instr(const char lvl, struct pt_regs regs)
@@ -179,11 +192,7 @@ void show_stack(struct task_struct tsk, unsigned long sp) · 77 unmodified lines
179	#else	192	#else
180	#define S_PREEMPT ""	193	#define S_PREEMPT ""
181	#endif	194	#endif
182	#ifdef CONFIG_SMP
183	#define S_SMP " SMP"	195	#define S_SMP " SMP"
184	#else
185	#define S_SMP ""
186	#endif
187		196
188	static int __die(const char str, int err, struct thread_info thread,	197	static int __die(const char str, int err, struct thread_info thread,
189	struct pt_regs *regs)	198	struct pt_regs *regs)
@@ -207,7 +216,8 @@ static int __die(const char str, int err, struct thread_info thread, · 17 unmodified lines
207		216
208	if (!user_mode(regs) \|\| in_interrupt()) {	217	if (!user_mode(regs) \|\| in_interrupt()) {
209	dump_mem(KERN_EMERG, "Stack: ", regs->sp,	218	dump_mem(KERN_EMERG, "Stack: ", regs->sp,
210	THREAD_SIZE + (unsigned long)task_stack_page(tsk));	219	THREAD_SIZE + (unsigned long)task_stack_page(tsk),
		220	compat_user_mode(regs));
211	dump_backtrace(regs, tsk);	221	dump_backtrace(regs, tsk);
212	dump_instr(KERN_EMERG, regs);	222	dump_instr(KERN_EMERG, regs);
213	}	223	}
@@ -459,7 +469,63 @@ void __pgd_error(const char *file, int line, unsigned long val) · 245 unmodified lines
459	pr_crit("%s:%d: bad pgd %016lx.\n", file, line, val);	469	pr_crit("%s:%d: bad pgd %016lx.\n", file, line, val);
460	}	470	}
461		471
		472	/* GENERIC_BUG traps */
		473
		474	int is_valid_bugaddr(unsigned long addr)
		475	{
		476	/*
		477	* bug_handler() only called for BRK #BUG_BRK_IMM.
		478	* So the answer is trivial -- any spurious instances with no
		479	* bug table entry will be rejected by report_bug() and passed
		480	* back to the debug-monitors code and handled as a fatal
		481	* unexpected debug exception.
		482	*/
		483	return 1;
		484	}
		485
		486	static int bug_handler(struct pt_regs *regs, unsigned int esr)
		487	{
		488	if (user_mode(regs))
		489	return DBG_HOOK_ERROR;
		490
		491	switch (report_bug(regs->pc, regs)) {
		492	case BUG_TRAP_TYPE_BUG:
		493	die("Oops - BUG", regs, 0);
		494	break;
		495
		496	case BUG_TRAP_TYPE_WARN:
		497	/* Ideally, report_bug() should backtrace for us... but no. */
		498	dump_backtrace(regs, NULL);
		499	break;
		500
		501	default:
		502	/* unknown/unrecognised bug trap type */
		503	return DBG_HOOK_ERROR;
		504	}
		505
		506	/* If thread survives, skip over the BUG instruction and continue: */
		507	regs->pc += AARCH64_INSN_SIZE; /* skip BRK and resume */
		508	return DBG_HOOK_HANDLED;
		509	}
		510
		511	static struct break_hook bug_break_hook = {
		512	.esr_val = 0xf2000000 \| BUG_BRK_IMM,
		513	.esr_mask = 0xffffffff,
		514	.fn = bug_handler,
		515	};
		516
		517	/*
		518	* Initial handler for AArch64 BRK exceptions
		519	* This handler only used until debug_traps_init().
		520	*/
		521	int __init early_brk64(unsigned long addr, unsigned int esr,
		522	struct pt_regs *regs)
		523	{
		524	return bug_handler(regs, esr) != DBG_HOOK_HANDLED;
		525	}
		526
		527	/* This registration must happen early, before debug_traps_init(). */
462	void __init trap_init(void)	528	void __init trap_init(void)
463	{	529	{
464	return;	530	register_break_hook(&bug_break_hook);
465	}	531	}

arch/arm64/kvm/hyp.S+10−2

  * Call into the vgic backend for state saving
  */
 .macro save_vgic_state
-	alternative_insn "bl __save_vgic_v2_state", "bl __save_vgic_v3_state", ARM64_HAS_SYSREG_GIC_CPUIF
+alternative_if_not ARM64_HAS_SYSREG_GIC_CPUIF
+	bl	__save_vgic_v2_state
+alternative_else
+	bl	__save_vgic_v3_state
+alternative_endif
 	mrs	x24, hcr_el2
 	mov	x25, #HCR_INT_OVERRIDE
 	neg	x25, x25
 	orr	x24, x24, #HCR_INT_OVERRIDE
 	orr	x24, x24, x25
 	msr	hcr_el2, x24
-	alternative_insn "bl __restore_vgic_v2_state", "bl __restore_vgic_v3_state", ARM64_HAS_SYSREG_GIC_CPUIF
+alternative_if_not ARM64_HAS_SYSREG_GIC_CPUIF
+	bl	__restore_vgic_v2_state
+alternative_else
+	bl	__restore_vgic_v3_state
+alternative_endif
 .endm
 .macro save_timer_state

arch/arm64/lib/Makefile+13−0

@@ -3,3 +3,16 @@ lib-y := bitops.o clear_user.o delay.o copy_from_user.o \ · 2 unmodified lines
3	clear_page.o memchr.o memcpy.o memmove.o memset.o \	3	clear_page.o memchr.o memcpy.o memmove.o memset.o \
4	memcmp.o strcmp.o strncmp.o strlen.o strnlen.o \	4	memcmp.o strcmp.o strncmp.o strlen.o strnlen.o \
5	strchr.o strrchr.o	5	strchr.o strrchr.o
		6
		7	# Tell the compiler to treat all general purpose registers as
		8	# callee-saved, which allows for efficient runtime patching of the bl
		9	# instruction in the caller with an atomic instruction when supported by
		10	# the CPU. Result and argument registers are handled correctly, based on
		11	# the function prototype.
		12	lib-$(CONFIG_ARM64_LSE_ATOMICS) += atomic_ll_sc.o
		13	CFLAGS_atomic_ll_sc.o := -fcall-used-x0 -ffixed-x1 -ffixed-x2 \
		14	-ffixed-x3 -ffixed-x4 -ffixed-x5 -ffixed-x6 \
		15	-ffixed-x7 -fcall-saved-x8 -fcall-saved-x9 \
		16	-fcall-saved-x10 -fcall-saved-x11 -fcall-saved-x12 \
		17	-fcall-saved-x13 -fcall-saved-x14 -fcall-saved-x15 \
		18	-fcall-saved-x16 -fcall-saved-x17 -fcall-saved-x18

arch/arm64/lib/atomic_ll_sc.c+3−0

@@ -0,0 +1,3 @@
	1	#include <asm/atomic.h>
	2	#define __ARM64_IN_ATOMIC_IMPL
	3	#include <asm/atomic_ll_sc.h>

arch/arm64/lib/bitops.S+26−19

@@ -18,52 +18,59 @@ · 17 unmodified lines
18		18
19	#include <linux/linkage.h>	19	#include <linux/linkage.h>
20	#include <asm/assembler.h>	20	#include <asm/assembler.h>
		21	#include <asm/lse.h>
21		22
22	/*	23	/*
23	* x0: bits 5:0 bit offset	24	* x0: bits 5:0 bit offset
24	* bits 31:6 word offset	25	* bits 31:6 word offset
25	* x1: address	26	* x1: address
26	*/	27	*/
27	.macro bitop, name, instr	28	.macro bitop, name, llsc, lse
28	ENTRY( \name )	29	ENTRY( \name )
29	and w3, w0, #63 // Get bit offset	30	and w3, w0, #63 // Get bit offset
30	eor w0, w0, w3 // Clear low bits	31	eor w0, w0, w3 // Clear low bits
31	mov x2, #1	32	mov x2, #1
32	add x1, x1, x0, lsr #3 // Get word offset	33	add x1, x1, x0, lsr #3 // Get word offset
		34	alt_lse " prfm pstl1strm, [x1]", "nop"
33	lsl x3, x2, x3 // Create mask	35	lsl x3, x2, x3 // Create mask
34	1: ldxr x2, [x1]	36
35	\instr x2, x2, x3	37	alt_lse "1: ldxr x2, [x1]", "\lse x3, [x1]"
36	stxr w0, x2, [x1]	38	alt_lse " \llsc x2, x2, x3", "nop"
37	cbnz w0, 1b	39	alt_lse " stxr w0, x2, [x1]", "nop"
		40	alt_lse " cbnz w0, 1b", "nop"
		41
38	ret	42	ret
39	ENDPROC(\name )	43	ENDPROC(\name )
40	.endm	44	.endm
41		45
42	.macro testop, name, instr	46	.macro testop, name, llsc, lse
43	ENTRY( \name )	47	ENTRY( \name )
44	and w3, w0, #63 // Get bit offset	48	and w3, w0, #63 // Get bit offset
45	eor w0, w0, w3 // Clear low bits	49	eor w0, w0, w3 // Clear low bits
46	mov x2, #1	50	mov x2, #1
47	add x1, x1, x0, lsr #3 // Get word offset	51	add x1, x1, x0, lsr #3 // Get word offset
		52	alt_lse " prfm pstl1strm, [x1]", "nop"
48	lsl x4, x2, x3 // Create mask	53	lsl x4, x2, x3 // Create mask
49	1: ldxr x2, [x1]	54
50	lsr x0, x2, x3 // Save old value of bit	55	alt_lse "1: ldxr x2, [x1]", "\lse x4, x2, [x1]"
51	\instr x2, x2, x4 // toggle bit	56	lsr x0, x2, x3
52	stlxr w5, x2, [x1]	57	alt_lse " \llsc x2, x2, x4", "nop"
53	cbnz w5, 1b	58	alt_lse " stlxr w5, x2, [x1]", "nop"
54	dmb ish	59	alt_lse " cbnz w5, 1b", "nop"
		60	alt_lse " dmb ish", "nop"
		61
55	and x0, x0, #1	62	and x0, x0, #1
56	3: ret	63	ret
57	ENDPROC(\name )	64	ENDPROC(\name )
58	.endm	65	.endm
59		66
60	/*	67	/*
61	* Atomic bit operations.	68	* Atomic bit operations.
62	*/	69	*/
63	bitop change_bit, eor	70	bitop change_bit, eor, steor
64	bitop clear_bit, bic	71	bitop clear_bit, bic, stclr
65	bitop set_bit, orr	72	bitop set_bit, orr, stset
66		73
67	testop test_and_change_bit, eor	74	testop test_and_change_bit, eor, ldeoral
68	testop test_and_clear_bit, bic	75	testop test_and_clear_bit, bic, ldclral
69	testop test_and_set_bit, orr	76	testop test_and_set_bit, orr, ldsetal

arch/arm64/lib/clear_user.S+8−0

  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 #include <linux/linkage.h>
+#include <asm/alternative.h>
 #include <asm/assembler.h>
+#include <asm/cpufeature.h>
+#include <asm/sysreg.h>
 	.text
  * Alignment fixed up by hardware.
  */
 ENTRY(__clear_user)
+ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(0)), ARM64_HAS_PAN, \
+	    CONFIG_ARM64_PAN)
 	mov	x2, x1			// save the size for fixup return
 	subs	x1, x1, #8
 	b.mi	2f
 	b.mi	5f
 USER(9f, strb	wzr, [x0]	)
 5:	mov	x0, #0
+ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(1)), ARM64_HAS_PAN, \
+	    CONFIG_ARM64_PAN)
 	ret
 ENDPROC(__clear_user)

arch/arm64/lib/copy_from_user.S+19−6

  */
 #include <linux/linkage.h>
+#include <asm/alternative.h>
 #include <asm/assembler.h>
+#include <asm/cpufeature.h>
+#include <asm/sysreg.h>
 /*
  * Copy from user space to a kernel buffer (alignment handled by the hardware)
  *	x0 - bytes not copied
  */
 ENTRY(__copy_from_user)
-	add	x4, x1, x2			// upper user buffer boundary
+ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(0)), ARM64_HAS_PAN, \
-	subs	x2, x2, #8
+	    CONFIG_ARM64_PAN)
+	add	x5, x1, x2			// upper user buffer boundary
+	subs	x2, x2, #16
+	b.mi	1f
+0:
+USER(9f, ldp	x3, x4, [x1], #16)
+	subs	x2, x2, #16
+	stp	x3, x4, [x0], #16
+	b.pl	0b
+1:	adds	x2, x2, #8
 	b.mi	2f
-1:
 USER(9f, ldr	x3, [x1], #8	)
-	subs	x2, x2, #8
+	sub	x2, x2, #8
 	str	x3, [x0], #8
-	b.pl	1b
 2:	adds	x2, x2, #4
 	b.mi	3f
 USER(9f, ldr	w3, [x1], #4	)
 USER(9f, ldrb	w3, [x1]	)
 	strb	w3, [x0]
 5:	mov	x0, #0
+ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(1)), ARM64_HAS_PAN, \
+	    CONFIG_ARM64_PAN)
 	ret
 ENDPROC(__copy_from_user)
 	.section .fixup,"ax"
 	.align	2
-9:	sub	x2, x4, x1
+9:	sub	x2, x5, x1
 	mov	x3, x2
 10:	strb	wzr, [x0], #1			// zero remaining buffer space
 	subs	x3, x3, #1

arch/arm64/lib/copy_in_user.S+19−6

  */
 #include <linux/linkage.h>
+#include <asm/alternative.h>
 #include <asm/assembler.h>
+#include <asm/cpufeature.h>
+#include <asm/sysreg.h>
 /*
  * Copy from user space to user space (alignment handled by the hardware)
  *	x0 - bytes not copied
  */
 ENTRY(__copy_in_user)
-	add	x4, x0, x2			// upper user buffer boundary
+ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(0)), ARM64_HAS_PAN, \
-	subs	x2, x2, #8
+	    CONFIG_ARM64_PAN)
+	add	x5, x0, x2			// upper user buffer boundary
+	subs	x2, x2, #16
+	b.mi	1f
+0:
+USER(9f, ldp	x3, x4, [x1], #16)
+	subs	x2, x2, #16
+USER(9f, stp	x3, x4, [x0], #16)
+	b.pl	0b
+1:	adds	x2, x2, #8
 	b.mi	2f
-1:
 USER(9f, ldr	x3, [x1], #8	)
-	subs	x2, x2, #8
+	sub	x2, x2, #8
 USER(9f, str	x3, [x0], #8	)
-	b.pl	1b
 2:	adds	x2, x2, #4
 	b.mi	3f
 USER(9f, ldr	w3, [x1], #4	)
 USER(9f, ldrb	w3, [x1]	)
 USER(9f, strb	w3, [x0]	)
 5:	mov	x0, #0
+ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(1)), ARM64_HAS_PAN, \
+	    CONFIG_ARM64_PAN)
 	ret
 ENDPROC(__copy_in_user)
 	.section .fixup,"ax"
 	.align	2
-9:	sub	x0, x4, x0			// bytes not copied
+9:	sub	x0, x5, x0			// bytes not copied
 	ret
 	.previous

arch/arm64/lib/copy_to_user.S+19−6

  */
 #include <linux/linkage.h>
+#include <asm/alternative.h>
 #include <asm/assembler.h>
+#include <asm/cpufeature.h>
+#include <asm/sysreg.h>
 /*
  * Copy to user space from a kernel buffer (alignment handled by the hardware)
  *	x0 - bytes not copied
  */
 ENTRY(__copy_to_user)
-	add	x4, x0, x2			// upper user buffer boundary
+ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(0)), ARM64_HAS_PAN, \
-	subs	x2, x2, #8
+	    CONFIG_ARM64_PAN)
+	add	x5, x0, x2			// upper user buffer boundary
+	subs	x2, x2, #16
+	b.mi	1f
+0:
+	ldp	x3, x4, [x1], #16
+	subs	x2, x2, #16
+USER(9f, stp	x3, x4, [x0], #16)
+	b.pl	0b
+1:	adds	x2, x2, #8
 	b.mi	2f
-1:
 	ldr	x3, [x1], #8
-	subs	x2, x2, #8
+	sub	x2, x2, #8
 USER(9f, str	x3, [x0], #8	)
-	b.pl	1b
 2:	adds	x2, x2, #4
 	b.mi	3f
 	ldr	w3, [x1], #4
 	ldrb	w3, [x1]
 USER(9f, strb	w3, [x0]	)
 5:	mov	x0, #0
+ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(1)), ARM64_HAS_PAN, \
+	    CONFIG_ARM64_PAN)
 	ret
 ENDPROC(__copy_to_user)
 	.section .fixup,"ax"
 	.align	2
-9:	sub	x0, x4, x0			// bytes not copied
+9:	sub	x0, x5, x0			// bytes not copied
 	ret
 	.previous

arch/arm64/mm/cache.S+6−1

 	dcache_line_size x2, x3
 	sub	x3, x2, #1
 	bic	x0, x0, x3
-1:	alternative_insn "dc cvac, x0", "dc civac, x0", ARM64_WORKAROUND_CLEAN_CACHE
+1:
+alternative_if_not ARM64_WORKAROUND_CLEAN_CACHE
+	dc	cvac, x0
+alternative_else
+	dc	civac, x0
+alternative_endif
 	add	x0, x0, x2
 	cmp	x0, x1
 	b.lo	1b

arch/arm64/mm/context.c+0−16

 		__flush_icache_all();
 }
-#ifdef CONFIG_SMP
 static void set_mm_context(struct mm_struct *mm, unsigned int asid)
 {
 	unsigned long flags;
 	cpu_switch_mm(mm->pgd, mm);
 }
-#else
-static inline void set_mm_context(struct mm_struct *mm, unsigned int asid)
-{
-	mm->context.id = asid;
-	cpumask_copy(mm_cpumask(mm), cpumask_of(smp_processor_id()));
-}
-#endif
 void __new_context(struct mm_struct *mm)
 {
 	unsigned int asid;
 	unsigned int bits = asid_bits();
 	raw_spin_lock(&cpu_asid_lock);
-#ifdef CONFIG_SMP
 	/*
 	 * Check the ASID again, in case the change was broadcast from another
 	 * CPU before we acquired the lock.
 		raw_spin_unlock(&cpu_asid_lock);
 		return;
 	}
-#endif
 	/*
 	 * At this point, it is guaranteed that the current mm (with an old
 	 * ASID) isn't active on any other CPU since the ASIDs are changed
 			cpu_last_asid = ASID_FIRST_VERSION;
 		asid = cpu_last_asid + smp_processor_id();
 		flush_context();
-#ifdef CONFIG_SMP
 		smp_wmb();
 		smp_call_function(reset_context, NULL, 1);
-#endif
 		cpu_last_asid += NR_CPUS - 1;
 	}

arch/arm64/mm/dma-mapping.c+20−13

@@ -144,6 +144,7 @@ static void __dma_alloc(struct device dev, size_t size, · 143 unmodified lines
144	struct page *page;	144	struct page *page;
145	void ptr, coherent_ptr;	145	void ptr, coherent_ptr;
146	bool coherent = is_device_dma_coherent(dev);	146	bool coherent = is_device_dma_coherent(dev);
		147	pgprot_t prot = __get_dma_pgprot(attrs, PAGE_KERNEL, false);
147		148
148	size = PAGE_ALIGN(size);	149	size = PAGE_ALIGN(size);
149		150
@@ -171,9 +172,7 @@ static void __dma_alloc(struct device dev, size_t size, · 21 unmodified lines
171	/* create a coherent mapping */	172	/* create a coherent mapping */
172	page = virt_to_page(ptr);	173	page = virt_to_page(ptr);
173	coherent_ptr = dma_common_contiguous_remap(page, size, VM_USERMAP,	174	coherent_ptr = dma_common_contiguous_remap(page, size, VM_USERMAP,
174	__get_dma_pgprot(attrs,	175	prot, NULL);
175	__pgprot(PROT_NORMAL_NC), false),
176	NULL);
177	if (!coherent_ptr)	176	if (!coherent_ptr)
178	goto no_map;	177	goto no_map;
179		178
@@ -303,9 +302,10 @@ static void __swiotlb_sync_sg_for_device(struct device *dev, · 123 unmodified lines
303	sg->length, dir);	302	sg->length, dir);
304	}	303	}
305		304
306	/* vma->vm_page_prot must be set appropriately before calling this function */	305	static int __swiotlb_mmap(struct device *dev,
307	static int __dma_common_mmap(struct device dev, struct vm_area_struct vma,	306	struct vm_area_struct *vma,
308	void *cpu_addr, dma_addr_t dma_addr, size_t size)	307	void *cpu_addr, dma_addr_t dma_addr, size_t size,
		308	struct dma_attrs *attrs)
309	{	309	{
310	int ret = -ENXIO;	310	int ret = -ENXIO;
311	unsigned long nr_vma_pages = (vma->vm_end - vma->vm_start) >>	311	unsigned long nr_vma_pages = (vma->vm_end - vma->vm_start) >>
@@ -314,6 +314,9 @@ static int __dma_common_mmap(struct device dev, struct vm_area_struct vma, · 2 unmodified lines
314	unsigned long pfn = dma_to_phys(dev, dma_addr) >> PAGE_SHIFT;	314	unsigned long pfn = dma_to_phys(dev, dma_addr) >> PAGE_SHIFT;
315	unsigned long off = vma->vm_pgoff;	315	unsigned long off = vma->vm_pgoff;
316		316
		317	vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot,
		318	is_device_dma_coherent(dev));
		319
317	if (dma_mmap_from_coherent(dev, vma, cpu_addr, size, &ret))	320	if (dma_mmap_from_coherent(dev, vma, cpu_addr, size, &ret))
318	return ret;	321	return ret;
319		322
@@ -327,20 +330,24 @@ static int __dma_common_mmap(struct device dev, struct vm_area_struct vma, · 7 unmodified lines
327	return ret;	330	return ret;
328	}	331	}
329		332
330	static int __swiotlb_mmap(struct device *dev,	333	static int __swiotlb_get_sgtable(struct device dev, struct sg_table sgt,
331	struct vm_area_struct *vma,	334	void *cpu_addr, dma_addr_t handle, size_t size,
332	void *cpu_addr, dma_addr_t dma_addr, size_t size,	335	struct dma_attrs *attrs)
333	struct dma_attrs *attrs)
334	{	336	{
335	vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot,	337	int ret = sg_alloc_table(sgt, 1, GFP_KERNEL);
336	is_device_dma_coherent(dev));	338
337	return __dma_common_mmap(dev, vma, cpu_addr, dma_addr, size);	339	if (!ret)
		340	sg_set_page(sgt->sgl, phys_to_page(dma_to_phys(dev, handle)),
		341	PAGE_ALIGN(size), 0);
		342
		343	return ret;
338	}	344	}
339		345
340	static struct dma_map_ops swiotlb_dma_ops = {	346	static struct dma_map_ops swiotlb_dma_ops = {
341	.alloc = __dma_alloc,	347	.alloc = __dma_alloc,
342	.free = __dma_free,	348	.free = __dma_free,
343	.mmap = __swiotlb_mmap,	349	.mmap = __swiotlb_mmap,
		350	.get_sgtable = __swiotlb_get_sgtable,
344	.map_page = __swiotlb_map_page,	351	.map_page = __swiotlb_map_page,
345	.unmap_page = __swiotlb_unmap_page,	352	.unmap_page = __swiotlb_unmap_page,
346	.map_sg = __swiotlb_map_sg_attrs,	353	.map_sg = __swiotlb_map_sg_attrs,

arch/arm64/mm/fault.c+26−2

 #include <linux/highmem.h>
 #include <linux/perf_event.h>
+#include <asm/cpufeature.h>
 #include <asm/exception.h>
 #include <asm/debug-monitors.h>
 #include <asm/esr.h>
+#include <asm/sysreg.h>
 #include <asm/system_misc.h>
 #include <asm/pgtable.h>
 #include <asm/tlbflush.h>
 		mm_flags |= FAULT_FLAG_WRITE;
 	}
+	/*
+	 * PAN bit set implies the fault happened in kernel space, but not
+	 * in the arch's user access functions.
+	 */
+	if (IS_ENABLED(CONFIG_ARM64_PAN) && (regs->pstate & PSR_PAN_BIT))
+		goto no_context;
 	/*
 	 * As per x86, we may deadlock here. However, since the kernel only
 	 * validly references user space from well defined areas of the code,
 	arm64_notify_die("Oops - SP/PC alignment exception", regs, &info, esr);
 }
-static struct fault_info debug_fault_info[] = {
+int __init early_brk64(unsigned long addr, unsigned int esr,
+		       struct pt_regs *regs);
+/*
+ * __refdata because early_brk64 is __init, but the reference to it is
+ * clobbered at arch_initcall time.
+ * See traps.c and debug-monitors.c:debug_traps_init().
+ */
+static struct fault_info __refdata debug_fault_info[] = {
 	{ do_bad,	SIGTRAP,	TRAP_HWBKPT,	"hardware breakpoint"	},
 	{ do_bad,	SIGTRAP,	TRAP_HWBKPT,	"hardware single-step"	},
 	{ do_bad,	SIGTRAP,	TRAP_HWBKPT,	"hardware watchpoint"	},
 	{ do_bad,	SIGBUS,		0,		"unknown 3"		},
 	{ do_bad,	SIGTRAP,	TRAP_BRKPT,	"aarch32 BKPT"		},
 	{ do_bad,	SIGTRAP,	0,		"aarch32 vector catch"	},
-	{ do_bad,	SIGTRAP,	TRAP_BRKPT,	"aarch64 BRK"		},
+	{ early_brk64,	SIGTRAP,	TRAP_BRKPT,	"aarch64 BRK"		},
 	{ do_bad,	SIGBUS,		0,		"unknown 7"		},
 };
 	return 0;
 }
+#ifdef CONFIG_ARM64_PAN
+void cpu_enable_pan(void)
+{
+	config_sctlr_el1(SCTLR_EL1_SPAN, 0);
+}
+#endif /* CONFIG_ARM64_PAN */

arch/arm64/mm/flush.c+0−4

 		       unsigned long uaddr, void *dst, const void *src,
 		       unsigned long len)
 {
-#ifdef CONFIG_SMP
 	preempt_disable();
-#endif
 	memcpy(dst, src, len);
 	flush_ptrace_access(vma, page, uaddr, dst, len);
-#ifdef CONFIG_SMP
 	preempt_enable();
-#endif
 }
 void __sync_icache_dcache(pte_t pte, unsigned long addr)

arch/arm64/mm/hugetlbpage.c+0−4

  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
  */
 #include <linux/init.h>

arch/arm64/mm/init.c+2−2

@@ -358,9 +358,9 @@ void free_initmem(void) · 357 unmodified lines
358		358
359	#ifdef CONFIG_BLK_DEV_INITRD	359	#ifdef CONFIG_BLK_DEV_INITRD
360		360
361	static int keep_initrd;	361	static int keep_initrd __initdata;
362		362
363	void free_initrd_mem(unsigned long start, unsigned long end)	363	void __init free_initrd_mem(unsigned long start, unsigned long end)
364	{	364	{
365	if (!keep_initrd)	365	if (!keep_initrd)
366	free_reserved_area((void )start, (void )end, 0, "initrd");	366	free_reserved_area((void )start, (void )end, 0, "initrd");

arch/arm64/mm/mmu.c+1−12

 	return ptr;
 }
-static void __ref create_mapping(phys_addr_t phys, unsigned long virt,
+static void __init create_mapping(phys_addr_t phys, unsigned long virt,
 				  phys_addr_t size, pgprot_t prot)
 {
 	if (virt < VMALLOC_START) {
 	cpu_set_default_tcr_t0sz();
 }
-/*
- * Enable the identity mapping to allow the MMU disabling.
- */
-void setup_mm_for_reboot(void)
-{
-	cpu_set_reserved_ttbr0();
-	flush_tlb_all();
-	cpu_set_idmap_tcr_t0sz();
-	cpu_switch_mm(idmap_pg_dir, &init_mm);
-}
 /*
  * Check whether a kernel address is valid (derived from arch/x86/).
  */

arch/arm64/mm/proc.S+15−6

 #define TCR_TG_FLAGS	TCR_TG0_4K | TCR_TG1_4K
 #endif
-#ifdef CONFIG_SMP
 #define TCR_SMP_FLAGS	TCR_SHARED
-#else
-#define TCR_SMP_FLAGS	0
-#endif
 /* PTWs cacheable, inner/outer WBWA */
 #define TCR_CACHE_FLAGS	TCR_IRGN_WBWA | TCR_ORGN_WBWA
  *	value of the SCTLR_EL1 register.
  */
 ENTRY(__cpu_setup)
-	ic	iallu				// I+BTB cache invalidate
 	tlbi	vmalle1is			// invalidate I + D TLBs
 	dsb	ish
 	mov	x0, #3 << 20
 	msr	cpacr_el1, x0			// Enable FP/ASIMD
-	msr	mdscr_el1, xzr			// Reset mdscr_el1
+	mov	x0, #1 << 12			// Reset mdscr_el1 and disable
+	msr	mdscr_el1, x0			// access to the DCC from EL0
 	/*
 	 * Memory region attributes for LPAE:
 	 *
 	 */
 	mrs	x9, ID_AA64MMFR0_EL1
 	bfi	x10, x9, #32, #3
+#ifdef CONFIG_ARM64_HW_AFDBM
+	/*
+	 * Hardware update of the Access and Dirty bits.
+	 */
+	mrs	x9, ID_AA64MMFR1_EL1
+	and	x9, x9, #0xf
+	cbz	x9, 2f
+	cmp	x9, #2
+	b.lt	1f
+	orr	x10, x10, #TCR_HD		// hardware Dirty flag update
+1:	orr	x10, x10, #TCR_HA		// hardware Access flag update
+2:
+#endif	/* CONFIG_ARM64_HW_AFDBM */
 	msr	tcr_el1, x10
 	ret					// return to head.S
 ENDPROC(__cpu_setup)

drivers/of/fdt.c+7−5

 }
 #ifdef CONFIG_HAVE_MEMBLOCK
-#define MAX_PHYS_ADDR	((phys_addr_t)~0)
+#ifndef MAX_MEMBLOCK_ADDR
+#define MAX_MEMBLOCK_ADDR	((phys_addr_t)~0)
+#endif
 void __init __weak early_init_dt_add_memory_arch(u64 base, u64 size)
 {
 	}
 	size &= PAGE_MASK;
-	if (base > MAX_PHYS_ADDR) {
+	if (base > MAX_MEMBLOCK_ADDR) {
 		pr_warning("Ignoring memory block 0x%llx - 0x%llx\n",
 				base, base + size);
 		return;
 	}
-	if (base + size - 1 > MAX_PHYS_ADDR) {
+	if (base + size - 1 > MAX_MEMBLOCK_ADDR) {
 		pr_warning("Ignoring memory range 0x%llx - 0x%llx\n",
-				((u64)MAX_PHYS_ADDR) + 1, base + size);
+				((u64)MAX_MEMBLOCK_ADDR) + 1, base + size);
-		size = MAX_PHYS_ADDR - base + 1;
+		size = MAX_MEMBLOCK_ADDR - base + 1;
 	}
 	if (base + size < phys_offset) {

Merge tag 'arm64-upstream' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code