lael-labs/linux

Documentation/admin-guide/sysctl/net.rst+1−1

   - arm64
   - arm32
   - ppc64
+  - ppc32
   - sparc64
   - mips64
   - s390x
 And the older cBPF JIT supported on the following archs:
   - mips
-  - ppc
   - sparc
 eBPF JITs are a superset of cBPF JITs, meaning the kernel will

Documentation/features/debug/debug-vm-pgtable/arch-support.txt+1−1

@@ -21,7 +21,7 @@ · 20 unmodified lines
21	\| nios2: \| TODO \|	21	\| nios2: \| TODO \|
22	\| openrisc: \| TODO \|	22	\| openrisc: \| TODO \|
23	\| parisc: \| TODO \|	23	\| parisc: \| TODO \|
24	\| powerpc: \| TODO \|	24	\| powerpc: \| ok \|
25	\| riscv: \| ok \|	25	\| riscv: \| ok \|
26	\| s390: \| ok \|	26	\| s390: \| ok \|
27	\| sh: \| TODO \|	27	\| sh: \| TODO \|

Documentation/powerpc/papr_hcalls.rst+14−0

@@ -275,6 +275,20 @@ Health Bitmap Flags: · 274 unmodified lines
275	Given a DRC Index collect the performance statistics for NVDIMM and copy them	275	Given a DRC Index collect the performance statistics for NVDIMM and copy them
276	to the resultBuffer.	276	to the resultBuffer.
277		277
		278	H_SCM_FLUSH
		279
		280	\| Input: drcIndex, continue-token
		281	\| Out: continue-token
		282	\| Return Value: H_SUCCESS, H_Parameter, H_P2, H_BUSY
		283
		284	Given a DRC Index Flush the data to backend NVDIMM device.
		285
		286	The hcall returns H_BUSY when the flush takes longer time and the hcall needs
		287	to be issued multiple times in order to be completely serviced. The
		288	continue-token from the output to be passed in the argument list of
		289	subsequent hcalls to the hypervisor until the hcall is completely serviced
		290	at which point H_SUCCESS or other error is returned by the hypervisor.
		291
278	References	292	References
279	==========	293	==========
280	.. [1] "Power Architecture Platform Reference"	294	.. [1] "Power Architecture Platform Reference"

Documentation/powerpc/vas-api.rst+2−2

@@ -254,7 +254,7 @@ using this window. the signal will be issued to the thread group leader · 253 unmodified lines
254	signals.	254	signals.
255		255
256	NX-GZIP User's Manual:	256	NX-GZIP User's Manual:
257	https://github.com/libnxz/power-gzip/blob/master/power_nx_gzip_um.pdf	257	https://github.com/libnxz/power-gzip/blob/master/doc/power_nx_gzip_um.pdf
258		258
259	Simple example	259	Simple example
260	==============	260	==============
@@ -301,5 +301,5 @@ Simple example · 40 unmodified lines
301	close(fd) or window can be closed upon process exit	301	close(fd) or window can be closed upon process exit
302	}	302	}
303		303
304	Refer https://github.com/abalib/power-gzip for tests or more	304	Refer https://github.com/libnxz/power-gzip for tests or more
305	use cases.	305	use cases.

arch/arm64/include/asm/vdso/compat_gettimeofday.h+2−1

@@ -155,7 +155,8 @@ static __always_inline const struct vdso_data *__arch_get_vdso_data(void) · 154 unmodified lines
155	}	155	}
156		156
157	#ifdef CONFIG_TIME_NS	157	#ifdef CONFIG_TIME_NS
158	static __always_inline const struct vdso_data *__arch_get_timens_vdso_data(void)	158	static __always_inline
		159	const struct vdso_data __arch_get_timens_vdso_data(const struct vdso_data vd)
159	{	160	{
160	const struct vdso_data *ret;	161	const struct vdso_data *ret;
161		162

arch/arm64/include/asm/vdso/gettimeofday.h+1−1

@@ -96,7 +96,7 @@ const struct vdso_data *__arch_get_vdso_data(void) · 95 unmodified lines
96		96
97	#ifdef CONFIG_TIME_NS	97	#ifdef CONFIG_TIME_NS
98	static __always_inline	98	static __always_inline
99	const struct vdso_data *__arch_get_timens_vdso_data(void)	99	const struct vdso_data __arch_get_timens_vdso_data(const struct vdso_data vd)
100	{	100	{
101	return _timens_data;	101	return _timens_data;
102	}	102	}

arch/powerpc/Kconfig+18−14

 	#
 	select ARCH_32BIT_OFF_T if PPC32
 	select ARCH_HAS_DEBUG_VIRTUAL
+	select ARCH_HAS_DEBUG_VM_PGTABLE
 	select ARCH_HAS_DEVMEM_IS_ALLOWED
 	select ARCH_HAS_ELF_RANDOMIZE
 	select ARCH_HAS_FORTIFY_SOURCE
 	select ARCH_HAS_MEMBARRIER_CALLBACKS
 	select ARCH_HAS_MEMBARRIER_SYNC_CORE
 	select ARCH_HAS_SCALED_CPUTIME		if VIRT_CPU_ACCOUNTING_NATIVE && PPC_BOOK3S_64
-	select ARCH_HAS_STRICT_KERNEL_RWX	if (PPC32 && !HIBERNATION)
+	select ARCH_HAS_STRICT_KERNEL_RWX	if ((PPC_BOOK3S_64 || PPC32) && !HIBERNATION)
 	select ARCH_HAS_TICK_BROADCAST		if GENERIC_CLOCKEVENTS_BROADCAST
 	select ARCH_HAS_UACCESS_FLUSHCACHE
 	select ARCH_HAS_COPY_MC			if PPC64
 	select ARCH_MIGHT_HAVE_PC_PARPORT
 	select ARCH_MIGHT_HAVE_PC_SERIO
 	select ARCH_OPTIONAL_KERNEL_RWX		if ARCH_HAS_STRICT_KERNEL_RWX
+	select ARCH_STACKWALK
 	select ARCH_SUPPORTS_ATOMIC_RMW
 	select ARCH_SUPPORTS_DEBUG_PAGEALLOC	if PPC32 || PPC_BOOK3S_64
 	select ARCH_USE_BUILTIN_BSWAP
 	select GENERIC_CPU_AUTOPROBE
 	select GENERIC_CPU_VULNERABILITIES	if PPC_BARRIER_NOSPEC
 	select GENERIC_EARLY_IOREMAP
+	select GENERIC_GETTIMEOFDAY
 	select GENERIC_IRQ_SHOW
 	select GENERIC_IRQ_SHOW_LEVEL
 	select GENERIC_PCI_IOMAP		if PCI
 	select GENERIC_STRNCPY_FROM_USER
 	select GENERIC_STRNLEN_USER
 	select GENERIC_TIME_VSYSCALL
-	select GENERIC_GETTIMEOFDAY
+	select GENERIC_VDSO_TIME_NS
 	select HAVE_ARCH_AUDITSYSCALL
 	select HAVE_ARCH_HUGE_VMAP		if PPC_BOOK3S_64 && PPC_RADIX_MMU
 	select HAVE_ARCH_JUMP_LABEL
+	select HAVE_ARCH_JUMP_LABEL_RELATIVE
 	select HAVE_ARCH_KASAN			if PPC32 && PPC_PAGE_SHIFT <= 14
 	select HAVE_ARCH_KASAN_VMALLOC		if PPC32 && PPC_PAGE_SHIFT <= 14
 	select HAVE_ARCH_KGDB
+	select HAVE_ARCH_KFENCE			if PPC32
 	select HAVE_ARCH_MMAP_RND_BITS
 	select HAVE_ARCH_MMAP_RND_COMPAT_BITS	if COMPAT
 	select HAVE_ARCH_NVRAM_OPS
 	select HAVE_ARCH_TRACEHOOK
 	select HAVE_ASM_MODVERSIONS
 	select HAVE_C_RECORDMCOUNT
-	select HAVE_CBPF_JIT			if !PPC64
 	select HAVE_STACKPROTECTOR		if PPC64 && $(cc-option,-mstack-protector-guard=tls -mstack-protector-guard-reg=r13)
 	select HAVE_STACKPROTECTOR		if PPC32 && $(cc-option,-mstack-protector-guard=tls -mstack-protector-guard-reg=r2)
 	select HAVE_CONTEXT_TRACKING		if PPC64
 	select HAVE_DEBUG_STACKOVERFLOW
 	select HAVE_DYNAMIC_FTRACE
 	select HAVE_DYNAMIC_FTRACE_WITH_REGS	if MPROFILE_KERNEL
-	select HAVE_EBPF_JIT			if PPC64
+	select HAVE_EBPF_JIT
 	select HAVE_EFFICIENT_UNALIGNED_ACCESS	if !(CPU_LITTLE_ENDIAN && POWER7_CPU)
 	select HAVE_FAST_GUP
 	select HAVE_FTRACE_MCOUNT_RECORD
 	select HAVE_LIVEPATCH			if HAVE_DYNAMIC_FTRACE_WITH_REGS
 	select HAVE_MOD_ARCH_SPECIFIC
 	select HAVE_NMI				if PERF_EVENTS || (PPC64 && PPC_BOOK3S)
-	select HAVE_HARDLOCKUP_DETECTOR_ARCH	if (PPC64 && PPC_BOOK3S)
+	select HAVE_HARDLOCKUP_DETECTOR_ARCH	if PPC64 && PPC_BOOK3S && SMP
-	select HAVE_OPTPROBES			if PPC64
+	select HAVE_OPTPROBES
 	select HAVE_PERF_EVENTS
 	select HAVE_PERF_EVENTS_NMI		if PPC64
 	select HAVE_HARDLOCKUP_DETECTOR_PERF	if PERF_EVENTS && HAVE_PERF_EVENTS_NMI && !HAVE_HARDLOCKUP_DETECTOR_ARCH
 	select MMU_GATHER_RCU_TABLE_FREE
 	select MMU_GATHER_PAGE_SIZE
 	select HAVE_REGS_AND_STACK_ACCESS_API
-	select HAVE_RELIABLE_STACKTRACE		if PPC_BOOK3S_64 && CPU_LITTLE_ENDIAN
+	select HAVE_RELIABLE_STACKTRACE
 	select HAVE_SOFTIRQ_ON_OWN_STACK
 	select HAVE_SYSCALL_TRACEPOINTS
 	select HAVE_VIRT_CPU_ACCOUNTING
 config DATA_SHIFT_BOOL
 	bool "Set custom data alignment"
 	depends on ADVANCED_OPTIONS
-	depends on STRICT_KERNEL_RWX || DEBUG_PAGEALLOC
+	depends on STRICT_KERNEL_RWX || DEBUG_PAGEALLOC || KFENCE
 	depends on PPC_BOOK3S_32 || (PPC_8xx && !PIN_TLB_DATA && !STRICT_KERNEL_RWX)
 	help
 	  This option allows you to set the kernel data alignment. When
 config DATA_SHIFT
 	int "Data shift" if DATA_SHIFT_BOOL
 	default 24 if STRICT_KERNEL_RWX && PPC64
-	range 17 28 if (STRICT_KERNEL_RWX || DEBUG_PAGEALLOC) && PPC_BOOK3S_32
+	range 17 28 if (STRICT_KERNEL_RWX || DEBUG_PAGEALLOC || KFENCE) && PPC_BOOK3S_32
-	range 19 23 if (STRICT_KERNEL_RWX || DEBUG_PAGEALLOC) && PPC_8xx
+	range 19 23 if (STRICT_KERNEL_RWX || DEBUG_PAGEALLOC || KFENCE) && PPC_8xx
 	default 22 if STRICT_KERNEL_RWX && PPC_BOOK3S_32
-	default 18 if DEBUG_PAGEALLOC && PPC_BOOK3S_32
+	default 18 if (DEBUG_PAGEALLOC || KFENCE) && PPC_BOOK3S_32
 	default 23 if STRICT_KERNEL_RWX && PPC_8xx
-	default 23 if DEBUG_PAGEALLOC && PPC_8xx && PIN_TLB_DATA
+	default 23 if (DEBUG_PAGEALLOC || KFENCE) && PPC_8xx && PIN_TLB_DATA
-	default 19 if DEBUG_PAGEALLOC && PPC_8xx
+	default 19 if (DEBUG_PAGEALLOC || KFENCE) && PPC_8xx
 	default PPC_PAGE_SHIFT
 	help
 	  On Book3S 32 (603+), DBATs are used to map kernel text and rodata RO.
 config TASK_SIZE
 	hex "Size of user task space" if TASK_SIZE_BOOL
 	default "0x80000000" if PPC_8xx
-	default "0xb0000000" if PPC_BOOK3S_32 && STRICT_KERNEL_RWX
+	default "0xb0000000" if PPC_BOOK3S_32
 	default "0xc0000000"
 endmenu

arch/powerpc/Kconfig.debug+1−0

@@ -353,6 +353,7 @@ config PPC_EARLY_DEBUG_CPM_ADDR · 352 unmodified lines
353	config FAIL_IOMMU	353	config FAIL_IOMMU
354	bool "Fault-injection capability for IOMMU"	354	bool "Fault-injection capability for IOMMU"
355	depends on FAULT_INJECTION	355	depends on FAULT_INJECTION
		356	depends on PCI \|\| IBMVIO
356	help	357	help
357	Provide fault-injection capability for IOMMU. Each device can	358	Provide fault-injection capability for IOMMU. Each device can
358	be selectively enabled via the fail_iommu property.	359	be selectively enabled via the fail_iommu property.

arch/powerpc/Makefile+4−7

@@ -181,12 +181,6 @@ CC_FLAGS_FTRACE := -pg · 180 unmodified lines
181	ifdef CONFIG_MPROFILE_KERNEL	181	ifdef CONFIG_MPROFILE_KERNEL
182	CC_FLAGS_FTRACE += -mprofile-kernel	182	CC_FLAGS_FTRACE += -mprofile-kernel
183	endif	183	endif
184	# Work around gcc code-gen bugs with -pg / -fno-omit-frame-pointer in gcc <= 4.8
185	# https://gcc.gnu.org/bugzilla/show_bug.cgi?id=44199
186	# https://gcc.gnu.org/bugzilla/show_bug.cgi?id=52828
187	ifndef CONFIG_CC_IS_CLANG
188	CC_FLAGS_FTRACE += $(call cc-ifversion, -lt, 0409, -mno-sched-epilog)
189	endif
190	endif	184	endif
191		185
192	CFLAGS-$(CONFIG_TARGET_CPU_BOOL) += $(call cc-option,-mcpu=$(CONFIG_TARGET_CPU))	186	CFLAGS-$(CONFIG_TARGET_CPU_BOOL) += $(call cc-option,-mcpu=$(CONFIG_TARGET_CPU))
@@ -444,12 +438,15 @@ endif · 251 unmodified lines
444	endif	438	endif
445		439
446	ifdef CONFIG_SMP	440	ifdef CONFIG_SMP
		441	ifdef CONFIG_PPC32
447	prepare: task_cpu_prepare	442	prepare: task_cpu_prepare
448		443
449	PHONY += task_cpu_prepare	444	PHONY += task_cpu_prepare
450	task_cpu_prepare: prepare0	445	task_cpu_prepare: prepare0
451	$(eval KBUILD_CFLAGS += -D_TASK_CPU=$(shell awk '{if ($$2 == "TASK_CPU") print $$3;}' include/generated/asm-offsets.h))	446	$(eval KBUILD_CFLAGS += -D_TASK_CPU=$(shell awk '{if ($$2 == "TASK_CPU") print $$3;}' include/generated/asm-offsets.h))
452	endif	447
		448	endif # CONFIG_PPC32
		449	endif # CONFIG_SMP
453		450
454	PHONY += checkbin	451	PHONY += checkbin
455	# Check toolchain versions:	452	# Check toolchain versions:

arch/powerpc/configs/ppc64_defconfig+2−0

 CONFIG_KEXEC=y
 CONFIG_KEXEC_FILE=y
 CONFIG_CRASH_DUMP=y
+CONFIG_FA_DUMP=y
 CONFIG_IRQ_ALL_CPUS=y
 CONFIG_PPC_64K_PAGES=y
 CONFIG_SCHED_SMT=y
 CONFIG_BE2NET=m
 CONFIG_IBMVETH=m
 CONFIG_EHEA=m
+CONFIG_IBMVNIC=m
 CONFIG_E100=y
 CONFIG_E1000=y
 CONFIG_E1000E=y

arch/powerpc/configs/pseries_defconfig+2−0

 CONFIG_SCANLOG=m
 CONFIG_PPC_SMLPAR=y
 CONFIG_IBMEBUS=y
+CONFIG_PAPR_SCM=m
 CONFIG_PPC_SVM=y
 # CONFIG_PPC_PMAC is not set
 CONFIG_RTAS_FLASH=m
 CONFIG_S2IO=m
 CONFIG_IBMVETH=y
 CONFIG_EHEA=y
+CONFIG_IBMVNIC=y
 CONFIG_E100=y
 CONFIG_E1000=y
 CONFIG_E1000E=y

arch/powerpc/include/asm/Kbuild+0−1

@@ -1,7 +1,6 @@
1	# SPDX-License-Identifier: GPL-2.0	1	# SPDX-License-Identifier: GPL-2.0
2	generated-y += syscall_table_32.h	2	generated-y += syscall_table_32.h
3	generated-y += syscall_table_64.h	3	generated-y += syscall_table_64.h
4	generated-y += syscall_table_c32.h
5	generated-y += syscall_table_spu.h	4	generated-y += syscall_table_spu.h
6	generic-y += export.h	5	generic-y += export.h
7	generic-y += kvm_types.h	6	generic-y += kvm_types.h

arch/powerpc/include/asm/asm-prototypes.h+0−2

@@ -77,8 +77,6 @@ notrace unsigned long interrupt_exit_kernel_prepare(struct pt_regs *regs, unsign · 76 unmodified lines
77	long ppc_fadvise64_64(int fd, int advice, u32 offset_high, u32 offset_low,	77	long ppc_fadvise64_64(int fd, int advice, u32 offset_high, u32 offset_low,
78	u32 len_high, u32 len_low);	78	u32 len_high, u32 len_low);
79	long sys_switch_endian(void);	79	long sys_switch_endian(void);
80	notrace unsigned int __check_irq_replay(void);
81	void notrace restore_interrupts(void);
82		80
83	/* prom_init (OpenFirmware) */	81	/* prom_init (OpenFirmware) */
84	unsigned long __init prom_init(unsigned long r3, unsigned long r4,	82	unsigned long __init prom_init(unsigned long r3, unsigned long r4,

arch/powerpc/include/asm/barrier.h+0−16

@@ -80,22 +80,6 @@ do { \ · 79 unmodified lines
80	___p1; \	80	___p1; \
81	})	81	})
82		82
83	#ifdef CONFIG_PPC64
84	#define smp_cond_load_relaxed(ptr, cond_expr) ({ \
85	typeof(ptr) __PTR = (ptr); \
86	__unqual_scalar_typeof(*ptr) VAL; \
87	VAL = READ_ONCE(*__PTR); \
88	if (unlikely(!(cond_expr))) { \
89	spin_begin(); \
90	do { \
91	VAL = READ_ONCE(*__PTR); \
92	} while (!(cond_expr)); \
93	spin_end(); \
94	} \
95	(typeof(*ptr))VAL; \
96	})
97	#endif
98
99	#ifdef CONFIG_PPC_BOOK3S_64	83	#ifdef CONFIG_PPC_BOOK3S_64
100	#define NOSPEC_BARRIER_SLOT nop	84	#define NOSPEC_BARRIER_SLOT nop
101	#elif defined(CONFIG_PPC_FSL_BOOK3E)	85	#elif defined(CONFIG_PPC_FSL_BOOK3E)

arch/powerpc/include/asm/book3s/32/kup.h+46−80

@@ -5,86 +5,7 @@ · 4 unmodified lines
5	#include <asm/bug.h>	5	#include <asm/bug.h>
6	#include <asm/book3s/32/mmu-hash.h>	6	#include <asm/book3s/32/mmu-hash.h>
7		7
8	#ifdef __ASSEMBLY__	8	#ifndef __ASSEMBLY__
9
10	.macro kuep_update_sr gpr1, gpr2 /* NEVER use r0 as gpr2 due to addis */
11	101: mtsrin \gpr1, \gpr2
12	addi \gpr1, \gpr1, 0x111 /* next VSID */
13	rlwinm \gpr1, \gpr1, 0, 0xf0ffffff /* clear VSID overflow */
14	addis \gpr2, \gpr2, 0x1000 /* address of next segment */
15	bdnz 101b
16	isync
17	.endm
18
19	.macro kuep_lock gpr1, gpr2
20	#ifdef CONFIG_PPC_KUEP
21	li \gpr1, NUM_USER_SEGMENTS
22	li \gpr2, 0
23	mtctr \gpr1
24	mfsrin \gpr1, \gpr2
25	oris \gpr1, \gpr1, SR_NX@h /* set Nx */
26	kuep_update_sr \gpr1, \gpr2
27	#endif
28	.endm
29
30	.macro kuep_unlock gpr1, gpr2
31	#ifdef CONFIG_PPC_KUEP
32	li \gpr1, NUM_USER_SEGMENTS
33	li \gpr2, 0
34	mtctr \gpr1
35	mfsrin \gpr1, \gpr2
36	rlwinm \gpr1, \gpr1, 0, ~SR_NX /* Clear Nx */
37	kuep_update_sr \gpr1, \gpr2
38	#endif
39	.endm
40
41	#ifdef CONFIG_PPC_KUAP
42
43	.macro kuap_update_sr gpr1, gpr2, gpr3 /* NEVER use r0 as gpr2 due to addis */
44	101: mtsrin \gpr1, \gpr2
45	addi \gpr1, \gpr1, 0x111 /* next VSID */
46	rlwinm \gpr1, \gpr1, 0, 0xf0ffffff /* clear VSID overflow */
47	addis \gpr2, \gpr2, 0x1000 /* address of next segment */
48	cmplw \gpr2, \gpr3
49	blt- 101b
50	isync
51	.endm
52
53	.macro kuap_save_and_lock sp, thread, gpr1, gpr2, gpr3
54	lwz \gpr2, KUAP(\thread)
55	rlwinm. \gpr3, \gpr2, 28, 0xf0000000
56	stw \gpr2, STACK_REGS_KUAP(\sp)
57	beq+ 102f
58	li \gpr1, 0
59	stw \gpr1, KUAP(\thread)
60	mfsrin \gpr1, \gpr2
61	oris \gpr1, \gpr1, SR_KS@h /* set Ks */
62	kuap_update_sr \gpr1, \gpr2, \gpr3
63	102:
64	.endm
65
66	.macro kuap_restore sp, current, gpr1, gpr2, gpr3
67	lwz \gpr2, STACK_REGS_KUAP(\sp)
68	rlwinm. \gpr3, \gpr2, 28, 0xf0000000
69	stw \gpr2, THREAD + KUAP(\current)
70	beq+ 102f
71	mfsrin \gpr1, \gpr2
72	rlwinm \gpr1, \gpr1, 0, ~SR_KS /* Clear Ks */
73	kuap_update_sr \gpr1, \gpr2, \gpr3
74	102:
75	.endm
76
77	.macro kuap_check current, gpr
78	#ifdef CONFIG_PPC_KUAP_DEBUG
79	lwz \gpr, THREAD + KUAP(\current)
80	999: twnei \gpr, 0
81	EMIT_BUG_ENTRY 999b, __FILE__, __LINE__, (BUGFLAG_WARNING \| BUGFLAG_ONCE)
82	#endif
83	.endm
84
85	#endif /* CONFIG_PPC_KUAP */
86
87	#else /* !__ASSEMBLY__ */
88		9
89	#ifdef CONFIG_PPC_KUAP	10	#ifdef CONFIG_PPC_KUAP
90		11
@@ -103,6 +24,51 @@ static inline void kuap_update_sr(u32 sr, u32 addr, u32 end) · 12 unmodified lines
103	isync(); /* Context sync required after mtsr() */	24	isync(); /* Context sync required after mtsr() */
104	}	25	}
105		26
		27	static inline void kuap_save_and_lock(struct pt_regs *regs)
		28	{
		29	unsigned long kuap = current->thread.kuap;
		30	u32 addr = kuap & 0xf0000000;
		31	u32 end = kuap << 28;
		32
		33	regs->kuap = kuap;
		34	if (unlikely(!kuap))
		35	return;
		36
		37	current->thread.kuap = 0;
		38	kuap_update_sr(mfsr(addr) \| SR_KS, addr, end); /* Set Ks */
		39	}
		40
		41	static inline void kuap_user_restore(struct pt_regs *regs)
		42	{
		43	}
		44
		45	static inline void kuap_kernel_restore(struct pt_regs *regs, unsigned long kuap)
		46	{
		47	u32 addr = regs->kuap & 0xf0000000;
		48	u32 end = regs->kuap << 28;
		49
		50	current->thread.kuap = regs->kuap;
		51
		52	if (unlikely(regs->kuap == kuap))
		53	return;
		54
		55	kuap_update_sr(mfsr(addr) & ~SR_KS, addr, end); /* Clear Ks */
		56	}
		57
		58	static inline unsigned long kuap_get_and_assert_locked(void)
		59	{
		60	unsigned long kuap = current->thread.kuap;
		61
		62	WARN_ON_ONCE(IS_ENABLED(CONFIG_PPC_KUAP_DEBUG) && kuap != 0);
		63
		64	return kuap;
		65	}
		66
		67	static inline void kuap_assert_locked(void)
		68	{
		69	kuap_get_and_assert_locked();
		70	}
		71
106	static __always_inline void allow_user_access(void __user to, const void __user from,	72	static __always_inline void allow_user_access(void __user to, const void __user from,
107	u32 size, unsigned long dir)	73	u32 size, unsigned long dir)
108	{	74	{

arch/powerpc/include/asm/book3s/32/pgtable.h+0−2

 #define VMALLOC_END	ioremap_bot
 #endif
-#ifdef CONFIG_STRICT_KERNEL_RWX
 #define MODULES_END	ALIGN_DOWN(PAGE_OFFSET, SZ_256M)
 #define MODULES_VADDR	(MODULES_END - SZ_256M)
-#endif
 #ifndef __ASSEMBLY__
 #include <linux/sched.h>

arch/powerpc/include/asm/book3s/32/tlbflush.h+1−1

@@ -79,4 +79,4 @@ static inline void local_flush_tlb_mm(struct mm_struct *mm) · 78 unmodified lines
79	flush_tlb_mm(mm);	79	flush_tlb_mm(mm);
80	}	80	}
81		81
82	#endif /* _ASM_POWERPC_TLBFLUSH_H */	82	#endif /* _ASM_POWERPC_BOOK3S_32_TLBFLUSH_H */

arch/powerpc/include/asm/book3s/64/kup.h+2−22

@@ -287,7 +287,7 @@ static inline void kuap_kernel_restore(struct pt_regs *regs, · 286 unmodified lines
287	*/	287	*/
288	}	288	}
289		289
290	static inline unsigned long kuap_get_and_check_amr(void)	290	static inline unsigned long kuap_get_and_assert_locked(void)
291	{	291	{
292	if (mmu_has_feature(MMU_FTR_BOOK3S_KUAP)) {	292	if (mmu_has_feature(MMU_FTR_BOOK3S_KUAP)) {
293	unsigned long amr = mfspr(SPRN_AMR);	293	unsigned long amr = mfspr(SPRN_AMR);
@@ -298,27 +298,7 @@ static inline unsigned long kuap_get_and_check_amr(void) · 4 unmodified lines
298	return 0;	298	return 0;
299	}	299	}
300		300
301	#else /* CONFIG_PPC_PKEY */	301	static inline void kuap_assert_locked(void)
302
303	static inline void kuap_user_restore(struct pt_regs *regs)
304	{
305	}
306
307	static inline void kuap_kernel_restore(struct pt_regs *regs, unsigned long amr)
308	{
309	}
310
311	static inline unsigned long kuap_get_and_check_amr(void)
312	{
313	return 0;
314	}
315
316	#endif /* CONFIG_PPC_PKEY */
317
318
319	#ifdef CONFIG_PPC_KUAP
320
321	static inline void kuap_check_amr(void)
322	{	302	{
323	if (IS_ENABLED(CONFIG_PPC_KUAP_DEBUG) && mmu_has_feature(MMU_FTR_BOOK3S_KUAP))	303	if (IS_ENABLED(CONFIG_PPC_KUAP_DEBUG) && mmu_has_feature(MMU_FTR_BOOK3S_KUAP))
324	WARN_ON_ONCE(mfspr(SPRN_AMR) != AMR_KUAP_BLOCKED);	304	WARN_ON_ONCE(mfspr(SPRN_AMR) != AMR_KUAP_BLOCKED);

arch/powerpc/include/asm/book3s/64/mmu-hash.h+0−1

@@ -18,7 +18,6 @@ · 17 unmodified lines
18	* complete pgtable.h but only a portion of it.	18	* complete pgtable.h but only a portion of it.
19	*/	19	*/
20	#include <asm/book3s/64/pgtable.h>	20	#include <asm/book3s/64/pgtable.h>
21	#include <asm/bug.h>
22	#include <asm/task_size_64.h>	21	#include <asm/task_size_64.h>
23	#include <asm/cpu_has_feature.h>	22	#include <asm/cpu_has_feature.h>
24		23

arch/powerpc/include/asm/book3s/64/pgtable.h+4−1

 #ifndef __ASSEMBLY__
 #include <linux/mmdebug.h>
 #include <linux/bug.h>
+#include <linux/sizes.h>
 #endif
 /*
  */
 #define _PAGE_KERNEL_RW		(_PAGE_PRIVILEGED | _PAGE_RW | _PAGE_DIRTY)
 #define _PAGE_KERNEL_RO		 (_PAGE_PRIVILEGED | _PAGE_READ)
+#define _PAGE_KERNEL_ROX	 (_PAGE_PRIVILEGED | _PAGE_READ | _PAGE_EXEC)
 #define _PAGE_KERNEL_RWX	(_PAGE_PRIVILEGED | _PAGE_DIRTY |	\
 				 _PAGE_RW | _PAGE_EXEC)
 /*
 #define  PHB_IO_END	(KERN_IO_START + FULL_IO_SIZE)
 #define IOREMAP_BASE	(PHB_IO_END)
 #define IOREMAP_START	(ioremap_bot)
-#define IOREMAP_END	(KERN_IO_END)
+#define IOREMAP_END	(KERN_IO_END - FIXADDR_SIZE)
+#define FIXADDR_SIZE	SZ_32M
 /* Advertise special mapping type for AGP */
 #define HAVE_PAGE_AGP

arch/powerpc/include/asm/book3s/64/radix.h+4−2

 	 * from ptesync, it should probably go into update_mmu_cache, rather
 	 * than set_pte_at (which is used to set ptes unrelated to faults).
 	 *
-	 * Spurious faults to vmalloc region are not tolerated, so there is
+	 * Spurious faults from the kernel memory are not tolerated, so there
-	 * a ptesync in flush_cache_vmap.
+	 * is a ptesync in flush_cache_vmap, and __map_kernel_page() follows
+	 * the pte update sequence from ISA Book III 6.10 Translation Table
+	 * Update Synchronization Requirements.
 	 */
 }

arch/powerpc/include/asm/bug.h+1−4

 #ifndef __ASSEMBLY__
 struct pt_regs;
-long do_page_fault(struct pt_regs *);
+void hash__do_page_fault(struct pt_regs *);
-long hash__do_page_fault(struct pt_regs *);
 void bad_page_fault(struct pt_regs *, int);
-void __bad_page_fault(struct pt_regs *regs, int sig);
-void do_bad_page_fault_segv(struct pt_regs *regs);
 extern void _exception(int, struct pt_regs *, int, unsigned long);
 extern void _exception_pkey(struct pt_regs *, unsigned long, int);
 extern void die(const char *, struct pt_regs *, long);

arch/powerpc/include/asm/cacheflush.h+13−2

 #endif /* CONFIG_PPC_BOOK3S_64 */
 #define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE 1
-extern void flush_dcache_page(struct page *page);
+/*
+ * This is called when a page has been modified by the kernel.
+ * It just marks the page as not i-cache clean.  We do the i-cache
+ * flush later when the page is given to a user process, if necessary.
+ */
+static inline void flush_dcache_page(struct page *page)
+{
+	if (cpu_has_feature(CPU_FTR_COHERENT_ICACHE))
+		return;
+	/* avoid an atomic op if possible */
+	if (test_bit(PG_dcache_clean, &page->flags))
+		clear_bit(PG_dcache_clean, &page->flags);
+}
 void flush_icache_range(unsigned long start, unsigned long stop);
 #define flush_icache_range flush_icache_range
 #define flush_icache_user_page flush_icache_user_page
 void flush_dcache_icache_page(struct page *page);
-void __flush_dcache_icache(void *page);
 /**
  * flush_dcache_range(): Write any modified data cache blocks out to memory and

arch/powerpc/include/asm/cpm2.h+1−1

@@ -594,7 +594,7 @@ typedef struct fcc_enet { · 593 unmodified lines
594	uint fen_p256c; /* Total packets 256 < bytes <= 511 */	594	uint fen_p256c; /* Total packets 256 < bytes <= 511 */
595	uint fen_p512c; /* Total packets 512 < bytes <= 1023 */	595	uint fen_p512c; /* Total packets 512 < bytes <= 1023 */
596	uint fen_p1024c; /* Total packets 1024 < bytes <= 1518 */	596	uint fen_p1024c; /* Total packets 1024 < bytes <= 1518 */
597	uint fen_cambuf; /* Internal CAM buffer poiner */	597	uint fen_cambuf; /* Internal CAM buffer pointer */
598	ushort fen_rfthr; /* Received frames threshold */	598	ushort fen_rfthr; /* Received frames threshold */
599	ushort fen_rfcnt; /* Received frames count */	599	ushort fen_rfcnt; /* Received frames count */
600	} fcc_enet_t;	600	} fcc_enet_t;

arch/powerpc/include/asm/fixmap.h+9−0

 #include <asm/kmap_size.h>
 #endif
+#ifdef CONFIG_PPC64
+#define FIXADDR_TOP	(IOREMAP_END + FIXADDR_SIZE)
+#else
+#define FIXADDR_SIZE	0
 #ifdef CONFIG_KASAN
 #include <asm/kasan.h>
 #define FIXADDR_TOP	(KASAN_SHADOW_START - PAGE_SIZE)
 #else
 #define FIXADDR_TOP	((unsigned long)(-PAGE_SIZE))
 #endif
+#endif
 /*
  * Here we define all the compile-time 'special' virtual
  */
 enum fixed_addresses {
 	FIX_HOLE,
+#ifdef CONFIG_PPC32
 	/* reserve the top 128K for early debugging purposes */
 	FIX_EARLY_DEBUG_TOP = FIX_HOLE,
 	FIX_EARLY_DEBUG_BASE = FIX_EARLY_DEBUG_TOP+(ALIGN(SZ_128K, PAGE_SIZE)/PAGE_SIZE)-1,
 		       FIX_IMMR_SIZE,
 #endif
 	/* FIX_PCIE_MCFG, */
+#endif /* CONFIG_PPC32 */
 	__end_of_permanent_fixed_addresses,
 #define NR_FIX_BTMAPS		(SZ_256K / PAGE_SIZE)
 static inline void __set_fixmap(enum fixed_addresses idx,
 				phys_addr_t phys, pgprot_t flags)
 {
+	BUILD_BUG_ON(IS_ENABLED(CONFIG_PPC64) && __FIXADDR_SIZE > FIXADDR_SIZE);
 	if (__builtin_constant_p(idx))
 		BUILD_BUG_ON(idx >= __end_of_fixed_addresses);
 	else if (WARN_ON(idx >= __end_of_fixed_addresses))

arch/powerpc/include/asm/futex.h+5−7

 {
 	int oldval = 0, ret;
-	if (!access_ok(uaddr, sizeof(u32)))
+	if (!user_access_begin(uaddr, sizeof(u32)))
 		return -EFAULT;
-	allow_read_write_user(uaddr, uaddr, sizeof(*uaddr));
 	switch (op) {
 	case FUTEX_OP_SET:
 	default:
 		ret = -ENOSYS;
 	}
+	user_access_end();
 	*oval = oldval;
-	prevent_read_write_user(uaddr, uaddr, sizeof(*uaddr));
 	return ret;
 }
 	int ret = 0;
 	u32 prev;
-	if (!access_ok(uaddr, sizeof(u32)))
+	if (!user_access_begin(uaddr, sizeof(u32)))
 		return -EFAULT;
-	allow_read_write_user(uaddr, uaddr, sizeof(*uaddr));
         __asm__ __volatile__ (
         PPC_ATOMIC_ENTRY_BARRIER
 "1:     lwarx   %1,0,%3         # futex_atomic_cmpxchg_inatomic\n\
         : "r" (uaddr), "r" (oldval), "r" (newval), "i" (-EFAULT)
         : "cc", "memory");
+	user_access_end();
 	*uval = prev;
-	prevent_read_write_user(uaddr, uaddr, sizeof(*uaddr));
         return ret;
 }

arch/powerpc/include/asm/hvcall.h+3−1

 #define H_SCM_HEALTH            0x400
 #define H_SCM_PERFORMANCE_STATS 0x418
 #define H_RPT_INVALIDATE	0x448
-#define MAX_HCALL_OPCODE	H_RPT_INVALIDATE
+#define H_SCM_FLUSH		0x44C
+#define MAX_HCALL_OPCODE	H_SCM_FLUSH
 /* Scope args for H_SCM_UNBIND_ALL */
 #define H_UNBIND_SCOPE_ALL (0x1)
 #define H_CPU_BEHAV_FAVOUR_SECURITY	(1ull << 63) // IBM bit 0
 #define H_CPU_BEHAV_L1D_FLUSH_PR	(1ull << 62) // IBM bit 1
 #define H_CPU_BEHAV_BNDS_CHK_SPEC_BAR	(1ull << 61) // IBM bit 2
+#define H_CPU_BEHAV_FAVOUR_SECURITY_H	(1ull << 60) // IBM bit 3
 #define H_CPU_BEHAV_FLUSH_COUNT_CACHE	(1ull << 58) // IBM bit 5
 #define H_CPU_BEHAV_FLUSH_LINK_STACK	(1ull << 57) // IBM bit 6

arch/powerpc/include/asm/hvconsole.h+3−0

@@ -24,5 +24,8 @@ · 23 unmodified lines
24	extern int hvc_get_chars(uint32_t vtermno, char *buf, int count);	24	extern int hvc_get_chars(uint32_t vtermno, char *buf, int count);
25	extern int hvc_put_chars(uint32_t vtermno, const char *buf, int count);	25	extern int hvc_put_chars(uint32_t vtermno, const char *buf, int count);
26		26
		27	/* Provided by HVC VIO */
		28	void hvc_vio_init_early(void);
		29
27	#endif /* __KERNEL__ */	30	#endif /* __KERNEL__ */
28	#endif /* _PPC64_HVCONSOLE_H */	31	#endif /* _PPC64_HVCONSOLE_H */

arch/powerpc/include/asm/hydra.h+0−2

@@ -94,8 +94,6 @@ extern volatile struct Hydra __iomem *Hydra; · 93 unmodified lines
94	#define HYDRA_INT_EXT7 18 /* Power Off Request */	94	#define HYDRA_INT_EXT7 18 /* Power Off Request */
95	#define HYDRA_INT_SPARE 19	95	#define HYDRA_INT_SPARE 19
96		96
97	extern int hydra_init(void);
98
99	#endif /* __KERNEL__ */	97	#endif /* __KERNEL__ */
100		98
101	#endif /* _ASMPPC_HYDRA_H */	99	#endif /* _ASMPPC_HYDRA_H */

arch/powerpc/include/asm/inst.h+44−11

@@ -4,6 +4,40 @@ · 3 unmodified lines
4		4
5	#include <asm/ppc-opcode.h>	5	#include <asm/ppc-opcode.h>
6		6
		7	#ifdef CONFIG_PPC64
		8
		9	#define ___get_user_instr(gu_op, dest, ptr) \
		10	({ \
		11	long __gui_ret = 0; \
		12	unsigned long __gui_ptr = (unsigned long)ptr; \
		13	struct ppc_inst __gui_inst; \
		14	unsigned int __prefix, __suffix; \
		15	__gui_ret = gu_op(__prefix, (unsigned int __user *)__gui_ptr); \
		16	if (__gui_ret == 0) { \
		17	if ((__prefix >> 26) == OP_PREFIX) { \
		18	__gui_ret = gu_op(__suffix, \
		19	(unsigned int __user *)__gui_ptr + 1); \
		20	__gui_inst = ppc_inst_prefix(__prefix, \
		21	__suffix); \
		22	} else { \
		23	__gui_inst = ppc_inst(__prefix); \
		24	} \
		25	if (__gui_ret == 0) \
		26	(dest) = __gui_inst; \
		27	} \
		28	__gui_ret; \
		29	})
		30	#else /* !CONFIG_PPC64 */
		31	#define ___get_user_instr(gu_op, dest, ptr) \
		32	gu_op((dest).val, (u32 __user *)(ptr))
		33	#endif /* CONFIG_PPC64 */
		34
		35	#define get_user_instr(x, ptr) \
		36	___get_user_instr(get_user, x, ptr)
		37
		38	#define __get_user_instr(x, ptr) \
		39	___get_user_instr(__get_user, x, ptr)
		40
7	/*	41	/*
8	* Instruction data type for POWER	42	* Instruction data type for POWER
9	*/	43	*/
@@ -68,6 +102,8 @@ static inline bool ppc_inst_equal(struct ppc_inst x, struct ppc_inst y) · 58 unmodified lines
68		102
69	#define ppc_inst(x) ((struct ppc_inst){ .val = x })	103	#define ppc_inst(x) ((struct ppc_inst){ .val = x })
70		104
		105	#define ppc_inst_prefix(x, y) ppc_inst(x)
		106
71	static inline bool ppc_inst_prefixed(struct ppc_inst x)	107	static inline bool ppc_inst_prefixed(struct ppc_inst x)
72	{	108	{
73	return false;	109	return false;
@@ -113,13 +149,14 @@ static inline struct ppc_inst ppc_inst_next(void location, struct ppc_inst *va · 39 unmodified lines
113	return location + ppc_inst_len(tmp);	149	return location + ppc_inst_len(tmp);
114	}	150	}
115		151
116	static inline u64 ppc_inst_as_u64(struct ppc_inst x)	152	static inline unsigned long ppc_inst_as_ulong(struct ppc_inst x)
117	{	153	{
118	#ifdef CONFIG_CPU_LITTLE_ENDIAN	154	if (IS_ENABLED(CONFIG_PPC32))
119	return (u64)ppc_inst_suffix(x) << 32 \| ppc_inst_val(x);	155	return ppc_inst_val(x);
120	#else	156	else if (IS_ENABLED(CONFIG_CPU_LITTLE_ENDIAN))
121	return (u64)ppc_inst_val(x) << 32 \| ppc_inst_suffix(x);	157	return (u64)ppc_inst_suffix(x) << 32 \| ppc_inst_val(x);
122	#endif	158	else
		159	return (u64)ppc_inst_val(x) << 32 \| ppc_inst_suffix(x);
123	}	160	}
124		161
125	#define PPC_INST_STR_LEN sizeof("00000000 00000000")	162	#define PPC_INST_STR_LEN sizeof("00000000 00000000")
@@ -141,10 +178,6 @@ static inline char *__ppc_inst_as_str(char str[PPC_INST_STR_LEN], struct ppc_ins · 15 unmodified lines
141	__str; \	178	__str; \
142	})	179	})
143		180
144	int probe_user_read_inst(struct ppc_inst *inst,	181	int copy_inst_from_kernel_nofault(struct ppc_inst inst, struct ppc_inst src);
145	struct ppc_inst __user *nip);
146
147	int probe_kernel_read_inst(struct ppc_inst *inst,
148	struct ppc_inst *src);
149		182
150	#endif /* _ASM_POWERPC_INST_H */	183	#endif /* _ASM_POWERPC_INST_H */

arch/powerpc/include/asm/interrupt.h+131−42

@@ -2,6 +2,70 @@ · 1 unmodified line
2	#ifndef _ASM_POWERPC_INTERRUPT_H	2	#ifndef _ASM_POWERPC_INTERRUPT_H
3	#define _ASM_POWERPC_INTERRUPT_H	3	#define _ASM_POWERPC_INTERRUPT_H
4		4
		5	/* BookE/4xx */
		6	#define INTERRUPT_CRITICAL_INPUT 0x100
		7
		8	/* BookE */
		9	#define INTERRUPT_DEBUG 0xd00
		10	#ifdef CONFIG_BOOKE
		11	#define INTERRUPT_PERFMON 0x260
		12	#define INTERRUPT_DOORBELL 0x280
		13	#endif
		14
		15	/* BookS/4xx/8xx */
		16	#define INTERRUPT_MACHINE_CHECK 0x200
		17
		18	/* BookS/8xx */
		19	#define INTERRUPT_SYSTEM_RESET 0x100
		20
		21	/* BookS */
		22	#define INTERRUPT_DATA_SEGMENT 0x380
		23	#define INTERRUPT_INST_SEGMENT 0x480
		24	#define INTERRUPT_TRACE 0xd00
		25	#define INTERRUPT_H_DATA_STORAGE 0xe00
		26	#define INTERRUPT_HMI 0xe60
		27	#define INTERRUPT_H_FAC_UNAVAIL 0xf80
		28	#ifdef CONFIG_PPC_BOOK3S
		29	#define INTERRUPT_DOORBELL 0xa00
		30	#define INTERRUPT_PERFMON 0xf00
		31	#define INTERRUPT_ALTIVEC_UNAVAIL 0xf20
		32	#endif
		33
		34	/* BookE/BookS/4xx/8xx */
		35	#define INTERRUPT_DATA_STORAGE 0x300
		36	#define INTERRUPT_INST_STORAGE 0x400
		37	#define INTERRUPT_EXTERNAL 0x500
		38	#define INTERRUPT_ALIGNMENT 0x600
		39	#define INTERRUPT_PROGRAM 0x700
		40	#define INTERRUPT_SYSCALL 0xc00
		41	#define INTERRUPT_TRACE 0xd00
		42
		43	/* BookE/BookS/44x */
		44	#define INTERRUPT_FP_UNAVAIL 0x800
		45
		46	/* BookE/BookS/44x/8xx */
		47	#define INTERRUPT_DECREMENTER 0x900
		48
		49	#ifndef INTERRUPT_PERFMON
		50	#define INTERRUPT_PERFMON 0x0
		51	#endif
		52
		53	/* 8xx */
		54	#define INTERRUPT_SOFT_EMU_8xx 0x1000
		55	#define INTERRUPT_INST_TLB_MISS_8xx 0x1100
		56	#define INTERRUPT_DATA_TLB_MISS_8xx 0x1200
		57	#define INTERRUPT_INST_TLB_ERROR_8xx 0x1300
		58	#define INTERRUPT_DATA_TLB_ERROR_8xx 0x1400
		59	#define INTERRUPT_DATA_BREAKPOINT_8xx 0x1c00
		60	#define INTERRUPT_INST_BREAKPOINT_8xx 0x1d00
		61
		62	/* 603 */
		63	#define INTERRUPT_INST_TLB_MISS_603 0x1000
		64	#define INTERRUPT_DATA_LOAD_TLB_MISS_603 0x1100
		65	#define INTERRUPT_DATA_STORE_TLB_MISS_603 0x1200
		66
		67	#ifndef __ASSEMBLY__
		68
5	#include <linux/context_tracking.h>	69	#include <linux/context_tracking.h>
6	#include <linux/hardirq.h>	70	#include <linux/hardirq.h>
7	#include <asm/cputime.h>	71	#include <asm/cputime.h>
@@ -9,10 +73,18 @@ · 1 unmodified line
9	#include <asm/kprobes.h>	73	#include <asm/kprobes.h>
10	#include <asm/runlatch.h>	74	#include <asm/runlatch.h>
11		75
12	struct interrupt_state {	76	static inline void nap_adjust_return(struct pt_regs *regs)
13	#ifdef CONFIG_PPC_BOOK3E_64	77	{
14	enum ctx_state ctx_state;	78	#ifdef CONFIG_PPC_970_NAP
		79	if (unlikely(test_thread_local_flags(_TLF_NAPPING))) {
		80	/* Can avoid a test-and-clear because NMIs do not call this */
		81	clear_thread_local_flags(_TLF_NAPPING);
		82	regs->nip = (unsigned long)power4_idle_nap_return;
		83	}
15	#endif	84	#endif
		85	}
		86
		87	struct interrupt_state {
16	};	88	};
17		89
18	static inline void booke_restore_dbcr0(void)	90	static inline void booke_restore_dbcr0(void)
@@ -29,10 +101,19 @@ static inline void booke_restore_dbcr0(void) · 10 unmodified lines
29		101
30	static inline void interrupt_enter_prepare(struct pt_regs regs, struct interrupt_state state)	102	static inline void interrupt_enter_prepare(struct pt_regs regs, struct interrupt_state state)
31	{	103	{
32	/*	104	#ifdef CONFIG_PPC32
33	* Book3E reconciles irq soft mask in asm	105	if (!arch_irq_disabled_regs(regs))
34	*/	106	trace_hardirqs_off();
35	#ifdef CONFIG_PPC_BOOK3S_64	107
		108	if (user_mode(regs)) {
		109	kuep_lock();
		110	account_cpu_user_entry();
		111	} else {
		112	kuap_save_and_lock(regs);
		113	}
		114	#endif
		115
		116	#ifdef CONFIG_PPC64
36	if (irq_soft_mask_set_return(IRQS_ALL_DISABLED) == IRQS_ENABLED)	117	if (irq_soft_mask_set_return(IRQS_ALL_DISABLED) == IRQS_ENABLED)
37	trace_hardirqs_off();	118	trace_hardirqs_off();
38	local_paca->irq_happened \|= PACA_IRQ_HARD_DIS;	119	local_paca->irq_happened \|= PACA_IRQ_HARD_DIS;
@@ -48,16 +129,12 @@ static inline void interrupt_enter_prepare(struct pt_regs *regs, struct interrup · 9 unmodified lines
48	* CT_WARN_ON comes here via program_check_exception,	129	* CT_WARN_ON comes here via program_check_exception,
49	* so avoid recursion.	130	* so avoid recursion.
50	*/	131	*/
51	if (TRAP(regs) != 0x700)	132	if (TRAP(regs) != INTERRUPT_PROGRAM)
52	CT_WARN_ON(ct_state() != CONTEXT_KERNEL);	133	CT_WARN_ON(ct_state() != CONTEXT_KERNEL);
53	}	134	}
54	#endif	135	#endif
55		136
56	#ifdef CONFIG_PPC_BOOK3E_64	137	booke_restore_dbcr0();
57	state->ctx_state = exception_enter();
58	if (user_mode(regs))
59	account_cpu_user_entry();
60	#endif
61	}	138	}
62		139
63	/*	140	/*
@@ -76,23 +153,8 @@ static inline void interrupt_enter_prepare(struct pt_regs *regs, struct interrup · 12 unmodified lines
76	*/	153	*/
77	static inline void interrupt_exit_prepare(struct pt_regs regs, struct interrupt_state state)	154	static inline void interrupt_exit_prepare(struct pt_regs regs, struct interrupt_state state)
78	{	155	{
79	#ifdef CONFIG_PPC_BOOK3E_64	156	if (user_mode(regs))
80	exception_exit(state->ctx_state);	157	kuep_unlock();
81	#endif
82
83	/*
84	* Book3S exits to user via interrupt_exit_user_prepare(), which does
85	* context tracking, which is a cleaner way to handle PREEMPT=y
86	* and avoid context entry/exit in e.g., preempt_schedule_irq()),
87	* which is likely to be where the core code wants to end up.
88	*
89	* The above comment explains why we can't do the
90	*
91	* if (user_mode(regs))
92	* user_exit_irqoff();
93	*
94	* sequence here.
95	*/
96	}	158	}
97		159
98	static inline void interrupt_async_enter_prepare(struct pt_regs regs, struct interrupt_state state)	160	static inline void interrupt_async_enter_prepare(struct pt_regs regs, struct interrupt_state state)
@@ -109,24 +171,46 @@ static inline void interrupt_async_enter_prepare(struct pt_regs *regs, struct in · 10 unmodified lines
109		171
110	static inline void interrupt_async_exit_prepare(struct pt_regs regs, struct interrupt_state state)	172	static inline void interrupt_async_exit_prepare(struct pt_regs regs, struct interrupt_state state)
111	{	173	{
		174	/*
		175	* Adjust at exit so the main handler sees the true NIA. This must
		176	* come before irq_exit() because irq_exit can enable interrupts, and
		177	* if another interrupt is taken before nap_adjust_return has run
		178	* here, then that interrupt would return directly to idle nap return.
		179	*/
		180	nap_adjust_return(regs);
		181
112	irq_exit();	182	irq_exit();
113	interrupt_exit_prepare(regs, state);	183	interrupt_exit_prepare(regs, state);
114	}	184	}
115		185
116	struct interrupt_nmi_state {	186	struct interrupt_nmi_state {
117	#ifdef CONFIG_PPC64	187	#ifdef CONFIG_PPC64
118	#ifdef CONFIG_PPC_BOOK3S_64
119	u8 irq_soft_mask;	188	u8 irq_soft_mask;
120	u8 irq_happened;	189	u8 irq_happened;
121	#endif
122	u8 ftrace_enabled;	190	u8 ftrace_enabled;
123	#endif	191	#endif
124	};	192	};
125		193
		194	static inline bool nmi_disables_ftrace(struct pt_regs *regs)
		195	{
		196	/* Allow DEC and PMI to be traced when they are soft-NMI */
		197	if (IS_ENABLED(CONFIG_PPC_BOOK3S_64)) {
		198	if (TRAP(regs) == INTERRUPT_DECREMENTER)
		199	return false;
		200	if (TRAP(regs) == INTERRUPT_PERFMON)
		201	return false;
		202	}
		203	if (IS_ENABLED(CONFIG_PPC_BOOK3E)) {
		204	if (TRAP(regs) == INTERRUPT_PERFMON)
		205	return false;
		206	}
		207
		208	return true;
		209	}
		210
126	static inline void interrupt_nmi_enter_prepare(struct pt_regs regs, struct interrupt_nmi_state state)	211	static inline void interrupt_nmi_enter_prepare(struct pt_regs regs, struct interrupt_nmi_state state)
127	{	212	{
128	#ifdef CONFIG_PPC64	213	#ifdef CONFIG_PPC64
129	#ifdef CONFIG_PPC_BOOK3S_64
130	state->irq_soft_mask = local_paca->irq_soft_mask;	214	state->irq_soft_mask = local_paca->irq_soft_mask;
131	state->irq_happened = local_paca->irq_happened;	215	state->irq_happened = local_paca->irq_happened;
132		216
@@ -139,9 +223,8 @@ static inline void interrupt_nmi_enter_prepare(struct pt_regs *regs, struct inte · 6 unmodified lines
139	local_paca->irq_happened \|= PACA_IRQ_HARD_DIS;	223	local_paca->irq_happened \|= PACA_IRQ_HARD_DIS;
140		224
141	/* Don't do any per-CPU operations until interrupt state is fixed */	225	/* Don't do any per-CPU operations until interrupt state is fixed */
142	#endif	226
143	/* Allow DEC and PMI to be traced when they are soft-NMI */	227	if (nmi_disables_ftrace(regs)) {
144	if (TRAP(regs) != 0x900 && TRAP(regs) != 0xf00 && TRAP(regs) != 0x260) {
145	state->ftrace_enabled = this_cpu_get_ftrace_enabled();	228	state->ftrace_enabled = this_cpu_get_ftrace_enabled();
146	this_cpu_set_ftrace_enabled(0);	229	this_cpu_set_ftrace_enabled(0);
147	}	230	}
@@ -164,17 +247,20 @@ static inline void interrupt_nmi_exit_prepare(struct pt_regs *regs, struct inter · 16 unmodified lines
164	radix_enabled() \|\| (mfmsr() & MSR_DR))	247	radix_enabled() \|\| (mfmsr() & MSR_DR))
165	nmi_exit();	248	nmi_exit();
166		249
		250	/*
		251	* nmi does not call nap_adjust_return because nmi should not create
		252	* new work to do (must use irq_work for that).
		253	*/
		254
167	#ifdef CONFIG_PPC64	255	#ifdef CONFIG_PPC64
168	if (TRAP(regs) != 0x900 && TRAP(regs) != 0xf00 && TRAP(regs) != 0x260)	256	if (nmi_disables_ftrace(regs))
169	this_cpu_set_ftrace_enabled(state->ftrace_enabled);	257	this_cpu_set_ftrace_enabled(state->ftrace_enabled);
170		258
171	#ifdef CONFIG_PPC_BOOK3S_64
172	/* Check we didn't change the pending interrupt mask. */	259	/* Check we didn't change the pending interrupt mask. */
173	WARN_ON_ONCE((state->irq_happened \| PACA_IRQ_HARD_DIS) != local_paca->irq_happened);	260	WARN_ON_ONCE((state->irq_happened \| PACA_IRQ_HARD_DIS) != local_paca->irq_happened);
174	local_paca->irq_happened = state->irq_happened;	261	local_paca->irq_happened = state->irq_happened;
175	local_paca->irq_soft_mask = state->irq_soft_mask;	262	local_paca->irq_soft_mask = state->irq_soft_mask;
176	#endif	263	#endif
177	#endif
178	}	264	}
179		265
180	/*	266	/*
@@ -387,6 +473,7 @@ DECLARE_INTERRUPT_HANDLER(SMIException); · 206 unmodified lines
387	DECLARE_INTERRUPT_HANDLER(handle_hmi_exception);	473	DECLARE_INTERRUPT_HANDLER(handle_hmi_exception);
388	DECLARE_INTERRUPT_HANDLER(unknown_exception);	474	DECLARE_INTERRUPT_HANDLER(unknown_exception);
389	DECLARE_INTERRUPT_HANDLER_ASYNC(unknown_async_exception);	475	DECLARE_INTERRUPT_HANDLER_ASYNC(unknown_async_exception);
		476	DECLARE_INTERRUPT_HANDLER_NMI(unknown_nmi_exception);
390	DECLARE_INTERRUPT_HANDLER(instruction_breakpoint_exception);	477	DECLARE_INTERRUPT_HANDLER(instruction_breakpoint_exception);
391	DECLARE_INTERRUPT_HANDLER(RunModeException);	478	DECLARE_INTERRUPT_HANDLER(RunModeException);
392	DECLARE_INTERRUPT_HANDLER(single_step_exception);	479	DECLARE_INTERRUPT_HANDLER(single_step_exception);
@@ -410,7 +497,7 @@ DECLARE_INTERRUPT_HANDLER(altivec_assist_exception); · 17 unmodified lines
410	DECLARE_INTERRUPT_HANDLER(CacheLockingException);	497	DECLARE_INTERRUPT_HANDLER(CacheLockingException);
411	DECLARE_INTERRUPT_HANDLER(SPEFloatingPointException);	498	DECLARE_INTERRUPT_HANDLER(SPEFloatingPointException);
412	DECLARE_INTERRUPT_HANDLER(SPEFloatingPointRoundException);	499	DECLARE_INTERRUPT_HANDLER(SPEFloatingPointRoundException);
413	DECLARE_INTERRUPT_HANDLER(WatchdogException);	500	DECLARE_INTERRUPT_HANDLER_NMI(WatchdogException);
414	DECLARE_INTERRUPT_HANDLER(kernel_bad_stack);	501	DECLARE_INTERRUPT_HANDLER(kernel_bad_stack);
415		502
416	/* slb.c */	503	/* slb.c */
@@ -421,7 +508,7 @@ DECLARE_INTERRUPT_HANDLER(do_bad_slb_fault); · 4 unmodified lines
421	DECLARE_INTERRUPT_HANDLER_RAW(do_hash_fault);	508	DECLARE_INTERRUPT_HANDLER_RAW(do_hash_fault);
422		509
423	/* fault.c */	510	/* fault.c */
424	DECLARE_INTERRUPT_HANDLER_RET(do_page_fault);	511	DECLARE_INTERRUPT_HANDLER(do_page_fault);
425	DECLARE_INTERRUPT_HANDLER(do_bad_page_fault_segv);	512	DECLARE_INTERRUPT_HANDLER(do_bad_page_fault_segv);
426		513
427	/* process.c */	514	/* process.c */
@@ -436,7 +523,7 @@ DECLARE_INTERRUPT_HANDLER_NMI(hmi_exception_realmode); · 8 unmodified lines
436		523
437	DECLARE_INTERRUPT_HANDLER_ASYNC(TAUException);	524	DECLARE_INTERRUPT_HANDLER_ASYNC(TAUException);
438		525
439	void unrecoverable_exception(struct pt_regs *regs);	526	void __noreturn unrecoverable_exception(struct pt_regs *regs);
440		527
441	void replay_system_reset(void);	528	void replay_system_reset(void);
442	void replay_soft_interrupts(void);	529	void replay_soft_interrupts(void);
@@ -447,4 +534,6 @@ static inline void interrupt_cond_local_irq_enable(struct pt_regs *regs) · 4 unmodified lines
447	local_irq_enable();	534	local_irq_enable();
448	}	535	}
449		536
		537	#endif /* __ASSEMBLY__ */
		538
450	#endif /* _ASM_POWERPC_INTERRUPT_H */	539	#endif /* _ASM_POWERPC_INTERRUPT_H */

arch/powerpc/include/asm/irq.h+0−2

@@ -53,8 +53,6 @@ extern void *mcheckirq_ctx[NR_CPUS]; · 52 unmodified lines
53	extern void *hardirq_ctx[NR_CPUS];	53	extern void *hardirq_ctx[NR_CPUS];
54	extern void *softirq_ctx[NR_CPUS];	54	extern void *softirq_ctx[NR_CPUS];
55		55
56	void call_do_softirq(void *sp);
57	void call_do_irq(struct pt_regs regs, void sp);
58	extern void do_IRQ(struct pt_regs *regs);	56	extern void do_IRQ(struct pt_regs *regs);
59	extern void __init init_IRQ(void);	57	extern void __init init_IRQ(void);
60	extern void __do_irq(struct pt_regs *regs);	58	extern void __do_irq(struct pt_regs *regs);

arch/powerpc/include/asm/jump_label.h+6−15

 	asm_volatile_goto("1:\n\t"
 		 "nop # arch_static_branch\n\t"
 		 ".pushsection __jump_table,  \"aw\"\n\t"
-		 JUMP_ENTRY_TYPE "1b, %l[l_yes], %c0\n\t"
+		 ".long 1b - ., %l[l_yes] - .\n\t"
+		 JUMP_ENTRY_TYPE "%c0 - .\n\t"
 		 ".popsection \n\t"
 		 : :  "i" (&((char *)key)[branch]) : : l_yes);
 	asm_volatile_goto("1:\n\t"
 		 "b %l[l_yes] # arch_static_branch_jump\n\t"
 		 ".pushsection __jump_table,  \"aw\"\n\t"
-		 JUMP_ENTRY_TYPE "1b, %l[l_yes], %c0\n\t"
+		 ".long 1b - ., %l[l_yes] - .\n\t"
+		 JUMP_ENTRY_TYPE "%c0 - .\n\t"
 		 ".popsection \n\t"
 		 : :  "i" (&((char *)key)[branch]) : : l_yes);
 	return true;
 }
-#ifdef CONFIG_PPC64
-typedef u64 jump_label_t;
-#else
-typedef u32 jump_label_t;
-#endif
-struct jump_entry {
-	jump_label_t code;
-	jump_label_t target;
-	jump_label_t key;
-};
 #else
 #define ARCH_STATIC_BRANCH(LABEL, KEY)		\
 1098:	nop;					\
 	.pushsection __jump_table, "aw";	\
-	FTR_ENTRY_LONG 1098b, LABEL, KEY;	\
+	.long 1098b - ., LABEL - .;		\
+	FTR_ENTRY_LONG KEY;			\
 	.popsection
 #endif

arch/powerpc/include/asm/kasan.h+1−1

@@ -19,7 +19,7 @@ · 18 unmodified lines
19		19
20	#define KASAN_SHADOW_SCALE_SHIFT 3	20	#define KASAN_SHADOW_SCALE_SHIFT 3
21		21
22	#if defined(CONFIG_PPC_BOOK3S_32) && defined(CONFIG_MODULES) && defined(CONFIG_STRICT_KERNEL_RWX)	22	#ifdef CONFIG_MODULES
23	#define KASAN_KERN_START ALIGN_DOWN(PAGE_OFFSET - SZ_256M, SZ_256M)	23	#define KASAN_KERN_START ALIGN_DOWN(PAGE_OFFSET - SZ_256M, SZ_256M)
24	#else	24	#else
25	#define KASAN_KERN_START PAGE_OFFSET	25	#define KASAN_KERN_START PAGE_OFFSET

arch/powerpc/include/asm/kfence.h+33−0

@@ -0,0 +1,33 @@
	1	/* SPDX-License-Identifier: GPL-2.0 */
	2	/*
	3	* powerpc KFENCE support.
	4	*
	5	* Copyright (C) 2020 CS GROUP France
	6	*/
	7
	8	#ifndef __ASM_POWERPC_KFENCE_H
	9	#define __ASM_POWERPC_KFENCE_H
	10
	11	#include <linux/mm.h>
	12	#include <asm/pgtable.h>
	13
	14	static inline bool arch_kfence_init_pool(void)
	15	{
	16	return true;
	17	}
	18
	19	static inline bool kfence_protect_page(unsigned long addr, bool protect)
	20	{
	21	pte_t *kpte = virt_to_kpte(addr);
	22
	23	if (protect) {
	24	pte_update(&init_mm, addr, kpte, _PAGE_PRESENT, 0, 0);
	25	flush_tlb_kernel_range(addr, addr + PAGE_SIZE);
	26	} else {
	27	pte_update(&init_mm, addr, kpte, 0, _PAGE_PRESENT, 0);
	28	}
	29
	30	return true;
	31	}
	32
	33	#endif /* __ASM_POWERPC_KFENCE_H */

arch/powerpc/include/asm/kup.h+17−10

 #ifdef __ASSEMBLY__
 #ifndef CONFIG_PPC_KUAP
-.macro kuap_save_and_lock	sp, thread, gpr1, gpr2, gpr3
-.endm
-.macro kuap_restore	sp, current, gpr1, gpr2, gpr3
-.endm
-.macro kuap_check	current, gpr
-.endm
 .macro kuap_check_amr	gpr1, gpr2
 .endm
 static inline void setup_kuep(bool disabled) { }
 #endif /* CONFIG_PPC_KUEP */
+#if defined(CONFIG_PPC_KUEP) && defined(CONFIG_PPC_BOOK3S_32)
+void kuep_lock(void);
+void kuep_unlock(void);
+#else
+static inline void kuep_lock(void) { }
+static inline void kuep_unlock(void) { }
+#endif
 #ifdef CONFIG_PPC_KUAP
 void setup_kuap(bool disabled);
 #else
 	return false;
 }
-static inline void kuap_check_amr(void) { }
+static inline void kuap_assert_locked(void) { }
+static inline void kuap_save_and_lock(struct pt_regs *regs) { }
+static inline void kuap_user_restore(struct pt_regs *regs) { }
+static inline void kuap_kernel_restore(struct pt_regs *regs, unsigned long amr) { }
+static inline unsigned long kuap_get_and_assert_locked(void)
+{
+	return 0;
+}
 /*
  * book3s/64/kup-radix.h defines these functions for the !KUAP case to flush

arch/powerpc/include/asm/kvm_book3s.h+2−0

 extern void kvmppc_harvest_vpa_dirty(struct kvmppc_vpa *vpa,
 			struct kvm_memory_slot *memslot,
 			unsigned long *map);
+extern unsigned long kvmppc_filter_lpcr_hv(struct kvm *kvm,
+			unsigned long lpcr);
 extern void kvmppc_update_lpcr(struct kvm *kvm, unsigned long lpcr,
 			unsigned long mask);
 extern void kvmppc_set_fscr(struct kvm_vcpu *vcpu, u64 fscr);

arch/powerpc/include/asm/kvm_ppc.h+1−2

                      unsigned long pte_index, unsigned long avpn);
 long kvmppc_h_bulk_remove(struct kvm_vcpu *vcpu);
 long kvmppc_h_protect(struct kvm_vcpu *vcpu, unsigned long flags,
-                      unsigned long pte_index, unsigned long avpn,
+                      unsigned long pte_index, unsigned long avpn);
-                      unsigned long va);
 long kvmppc_h_read(struct kvm_vcpu *vcpu, unsigned long flags,
                    unsigned long pte_index);
 long kvmppc_h_clear_ref(struct kvm_vcpu *vcpu, unsigned long flags,

arch/powerpc/include/asm/mmu_context.h+1−1

@@ -263,7 +263,7 @@ extern void arch_exit_mmap(struct mm_struct *mm); · 262 unmodified lines
263	static inline void arch_unmap(struct mm_struct *mm,	263	static inline void arch_unmap(struct mm_struct *mm,
264	unsigned long start, unsigned long end)	264	unsigned long start, unsigned long end)
265	{	265	{
266	unsigned long vdso_base = (unsigned long)mm->context.vdso - PAGE_SIZE;	266	unsigned long vdso_base = (unsigned long)mm->context.vdso;
267		267
268	if (start <= vdso_base && vdso_base < end)	268	if (start <= vdso_base && vdso_base < end)
269	mm->context.vdso = NULL;	269	mm->context.vdso = NULL;

arch/powerpc/include/asm/nohash/32/kup-8xx.h+32−24

 #ifdef CONFIG_PPC_KUAP
-#ifdef __ASSEMBLY__
+#ifndef __ASSEMBLY__
-.macro kuap_save_and_lock	sp, thread, gpr1, gpr2, gpr3
-	lis	\gpr2, MD_APG_KUAP@h	/* only APG0 and APG1 are used */
-	mfspr	\gpr1, SPRN_MD_AP
-	mtspr	SPRN_MD_AP, \gpr2
-	stw	\gpr1, STACK_REGS_KUAP(\sp)
-.endm
-.macro kuap_restore	sp, current, gpr1, gpr2, gpr3
-	lwz	\gpr1, STACK_REGS_KUAP(\sp)
-	mtspr	SPRN_MD_AP, \gpr1
-.endm
-.macro kuap_check	current, gpr
-#ifdef CONFIG_PPC_KUAP_DEBUG
-	mfspr	\gpr, SPRN_MD_AP
-	rlwinm	\gpr, \gpr, 16, 0xffff
-999:	twnei	\gpr, MD_APG_KUAP@h
-	EMIT_BUG_ENTRY 999b, __FILE__, __LINE__, (BUGFLAG_WARNING | BUGFLAG_ONCE)
-#endif
-.endm
-#else /* !__ASSEMBLY__ */
 #include <asm/reg.h>
+static inline void kuap_save_and_lock(struct pt_regs *regs)
+{
+	regs->kuap = mfspr(SPRN_MD_AP);
+	mtspr(SPRN_MD_AP, MD_APG_KUAP);
+}
+static inline void kuap_user_restore(struct pt_regs *regs)
+{
+}
+static inline void kuap_kernel_restore(struct pt_regs *regs, unsigned long kuap)
+{
+	mtspr(SPRN_MD_AP, regs->kuap);
+}
+static inline unsigned long kuap_get_and_assert_locked(void)
+{
+	unsigned long kuap = mfspr(SPRN_MD_AP);
+	if (IS_ENABLED(CONFIG_PPC_KUAP_DEBUG))
+		WARN_ON_ONCE(kuap >> 16 != MD_APG_KUAP >> 16);
+	return kuap;
+}
+static inline void kuap_assert_locked(void)
+{
+	if (IS_ENABLED(CONFIG_PPC_KUAP_DEBUG))
+		kuap_get_and_assert_locked();
+}
 static inline void allow_user_access(void __user *to, const void __user *from,
 				     unsigned long size, unsigned long dir)
 {

arch/powerpc/include/asm/nohash/32/mmu-8xx.h+3−0

@@ -172,6 +172,9 @@ · 171 unmodified lines
172		172
173	#define mmu_linear_psize MMU_PAGE_8M	173	#define mmu_linear_psize MMU_PAGE_8M
174		174
		175	#define MODULES_VADDR (PAGE_OFFSET - SZ_256M)
		176	#define MODULES_END PAGE_OFFSET
		177
175	#ifndef __ASSEMBLY__	178	#ifndef __ASSEMBLY__
176		179
177	#include <linux/mmdebug.h>	180	#include <linux/mmdebug.h>

arch/powerpc/include/asm/nohash/64/pgtable.h+4−1

  * the ppc64 non-hashed page table.
  */
+#include <linux/sizes.h>
 #include <asm/nohash/64/pgtable-4k.h>
 #include <asm/barrier.h>
 #include <asm/asm-const.h>
 #define  PHB_IO_END	(KERN_IO_START + FULL_IO_SIZE)
 #define IOREMAP_BASE	(PHB_IO_END)
 #define IOREMAP_START	(ioremap_bot)
-#define IOREMAP_END	(KERN_VIRT_START + KERN_VIRT_SIZE)
+#define IOREMAP_END	(KERN_VIRT_START + KERN_VIRT_SIZE - FIXADDR_SIZE)
+#define FIXADDR_SIZE	SZ_32M
 /*

arch/powerpc/include/asm/opal.h+1−1

@@ -307,7 +307,7 @@ int opal_secvar_enqueue_update(const char key, uint64_t key_len, u8 data, · 306 unmodified lines
307		307
308	s64 opal_mpipl_update(enum opal_mpipl_ops op, u64 src, u64 dest, u64 size);	308	s64 opal_mpipl_update(enum opal_mpipl_ops op, u64 src, u64 dest, u64 size);
309	s64 opal_mpipl_register_tag(enum opal_mpipl_tags tag, u64 addr);	309	s64 opal_mpipl_register_tag(enum opal_mpipl_tags tag, u64 addr);
310	s64 opal_mpipl_query_tag(enum opal_mpipl_tags tag, u64 *addr);	310	s64 opal_mpipl_query_tag(enum opal_mpipl_tags tag, __be64 *addr);
311		311
312	s64 opal_signal_system_reset(s32 cpu);	312	s64 opal_signal_system_reset(s32 cpu);
313	s64 opal_quiesce(u64 shutdown_type, s32 cpu);	313	s64 opal_quiesce(u64 shutdown_type, s32 cpu);

arch/powerpc/include/asm/perf_event_server.h+7−1

 				u64 alt[]);
 	void		(*get_mem_data_src)(union perf_mem_data_src *dsrc,
 				u32 flags, struct pt_regs *regs);
-	void		(*get_mem_weight)(u64 *weight);
+	void		(*get_mem_weight)(u64 *weight, u64 type);
 	unsigned long	group_constraint_mask;
 	unsigned long	group_constraint_val;
 	u64             (*bhrb_filter_map)(u64 branch_sample_type);
 	 * the pmu supports extended perf regs capability
 	 */
 	int		capabilities;
+	/*
+	 * Function to check event code for values which are
+	 * reserved. Function takes struct perf_event as input,
+	 * since event code could be spread in attr.config*
+	 */
+	int		(*check_attr_config)(struct perf_event *ev);
 };
 /*

arch/powerpc/include/asm/pgtable.h+0−2

@@ -41,8 +41,6 @@ struct mm_struct; · 40 unmodified lines
41		41
42	#ifndef __ASSEMBLY__	42	#ifndef __ASSEMBLY__
43		43
44	#include <asm/tlbflush.h>
45
46	/* Keep these as a macros to avoid include dependency mess */	44	/* Keep these as a macros to avoid include dependency mess */
47	#define pte_page(x) pfn_to_page(pte_pfn(x))	45	#define pte_page(x) pfn_to_page(pte_pfn(x))
48	#define mk_pte(page, pgprot) pfn_pte(page_to_pfn(page), (pgprot))	46	#define mk_pte(page, pgprot) pfn_pte(page_to_pfn(page), (pgprot))

arch/powerpc/include/asm/ppc-opcode.h+13−0

 #define PPC_INST_ORI			0x60000000
 #define PPC_INST_ORIS			0x64000000
 #define PPC_INST_BRANCH			0x48000000
+#define PPC_INST_BL			0x48000001
 #define PPC_INST_BRANCH_COND		0x40800000
 /* Prefixes */
 #define PPC_RAW_STFDX(s, a, b)		(0x7c0005ae | ___PPC_RS(s) | ___PPC_RA(a) | ___PPC_RB(b))
 #define PPC_RAW_LVX(t, a, b)		(0x7c0000ce | ___PPC_RT(t) | ___PPC_RA(a) | ___PPC_RB(b))
 #define PPC_RAW_STVX(s, a, b)		(0x7c0001ce | ___PPC_RS(s) | ___PPC_RA(a) | ___PPC_RB(b))
+#define PPC_RAW_ADDE(t, a, b)		(0x7c000114 | ___PPC_RT(t) | ___PPC_RA(a) | ___PPC_RB(b))
+#define PPC_RAW_ADDZE(t, a)		(0x7c000194 | ___PPC_RT(t) | ___PPC_RA(a))
+#define PPC_RAW_ADDME(t, a)		(0x7c0001d4 | ___PPC_RT(t) | ___PPC_RA(a))
 #define PPC_RAW_ADD(t, a, b)		(PPC_INST_ADD | ___PPC_RT(t) | ___PPC_RA(a) | ___PPC_RB(b))
 #define PPC_RAW_ADD_DOT(t, a, b)	(PPC_INST_ADD | ___PPC_RT(t) | ___PPC_RA(a) | ___PPC_RB(b) | 0x1)
 #define PPC_RAW_ADDC(t, a, b)		(0x7c000014 | ___PPC_RT(t) | ___PPC_RA(a) | ___PPC_RB(b))
 #define PPC_RAW_BLR()			(PPC_INST_BLR)
 #define PPC_RAW_BLRL()			(0x4e800021)
 #define PPC_RAW_MTLR(r)			(0x7c0803a6 | ___PPC_RT(r))
+#define PPC_RAW_MFLR(t)			(PPC_INST_MFLR | ___PPC_RT(t))
 #define PPC_RAW_BCTR()			(PPC_INST_BCTR)
 #define PPC_RAW_MTCTR(r)		(PPC_INST_MTCTR | ___PPC_RT(r))
 #define PPC_RAW_ADDI(d, a, i)		(PPC_INST_ADDI | ___PPC_RT(d) | ___PPC_RA(a) | IMM_L(i))
 #define PPC_RAW_LI(r, i)		PPC_RAW_ADDI(r, 0, i)
 #define PPC_RAW_ADDIS(d, a, i)		(PPC_INST_ADDIS | ___PPC_RT(d) | ___PPC_RA(a) | IMM_L(i))
+#define PPC_RAW_ADDIC(d, a, i)		(0x30000000 | ___PPC_RT(d) | ___PPC_RA(a) | IMM_L(i))
+#define PPC_RAW_ADDIC_DOT(d, a, i)	(0x34000000 | ___PPC_RT(d) | ___PPC_RA(a) | IMM_L(i))
 #define PPC_RAW_LIS(r, i)		PPC_RAW_ADDIS(r, 0, i)
 #define PPC_RAW_STDX(r, base, b)	(0x7c00012a | ___PPC_RS(r) | ___PPC_RA(base) | ___PPC_RB(b))
 #define PPC_RAW_STDU(r, base, i)	(0xf8000001 | ___PPC_RS(r) | ___PPC_RA(base) | ((i) & 0xfffc))
 #define PPC_RAW_CMPLW(a, b)		(0x7c000040 | ___PPC_RA(a) | ___PPC_RB(b))
 #define PPC_RAW_CMPLD(a, b)		(0x7c200040 | ___PPC_RA(a) | ___PPC_RB(b))
 #define PPC_RAW_SUB(d, a, b)		(0x7c000050 | ___PPC_RT(d) | ___PPC_RB(a) | ___PPC_RA(b))
+#define PPC_RAW_SUBFC(d, a, b)		(0x7c000010 | ___PPC_RT(d) | ___PPC_RA(a) | ___PPC_RB(b))
+#define PPC_RAW_SUBFE(d, a, b)		(0x7c000110 | ___PPC_RT(d) | ___PPC_RA(a) | ___PPC_RB(b))
+#define PPC_RAW_SUBFIC(d, a, i)		(0x20000000 | ___PPC_RT(d) | ___PPC_RA(a) | IMM_L(i))
+#define PPC_RAW_SUBFZE(d, a)		(0x7c000190 | ___PPC_RT(d) | ___PPC_RA(a))
 #define PPC_RAW_MULD(d, a, b)		(0x7c0001d2 | ___PPC_RT(d) | ___PPC_RA(a) | ___PPC_RB(b))
 #define PPC_RAW_MULW(d, a, b)		(0x7c0001d6 | ___PPC_RT(d) | ___PPC_RA(a) | ___PPC_RB(b))
 #define PPC_RAW_MULHWU(d, a, b)		(0x7c000016 | ___PPC_RT(d) | ___PPC_RA(a) | ___PPC_RB(b))
 #define PPC_RAW_DIVDEU_DOT(t, a, b)	(0x7c000312 | ___PPC_RT(t) | ___PPC_RA(a) | ___PPC_RB(b) | 0x1)
 #define PPC_RAW_AND(d, a, b)		(0x7c000038 | ___PPC_RA(d) | ___PPC_RS(a) | ___PPC_RB(b))
 #define PPC_RAW_ANDI(d, a, i)		(0x70000000 | ___PPC_RA(d) | ___PPC_RS(a) | IMM_L(i))
+#define PPC_RAW_ANDIS(d, a, i)		(0x74000000 | ___PPC_RA(d) | ___PPC_RS(a) | IMM_L(i))
 #define PPC_RAW_AND_DOT(d, a, b)	(0x7c000039 | ___PPC_RA(d) | ___PPC_RS(a) | ___PPC_RB(b))
 #define PPC_RAW_OR(d, a, b)		(0x7c000378 | ___PPC_RA(d) | ___PPC_RS(a) | ___PPC_RB(b))
 #define PPC_RAW_MR(d, a)		PPC_RAW_OR(d, a, a)
 #define PPC_RAW_ORI(d, a, i)		(PPC_INST_ORI | ___PPC_RA(d) | ___PPC_RS(a) | IMM_L(i))
 #define PPC_RAW_ORIS(d, a, i)		(PPC_INST_ORIS | ___PPC_RA(d) | ___PPC_RS(a) | IMM_L(i))
+#define PPC_RAW_NOR(d, a, b)		(0x7c0000f8 | ___PPC_RA(d) | ___PPC_RS(a) | ___PPC_RB(b))
 #define PPC_RAW_XOR(d, a, b)		(0x7c000278 | ___PPC_RA(d) | ___PPC_RS(a) | ___PPC_RB(b))
 #define PPC_RAW_XORI(d, a, i)		(0x68000000 | ___PPC_RA(d) | ___PPC_RS(a) | IMM_L(i))
 #define PPC_RAW_XORIS(d, a, i)		(0x6c000000 | ___PPC_RA(d) | ___PPC_RS(a) | IMM_L(i))

arch/powerpc/include/asm/ppc_asm.h+0−30

@@ -15,36 +15,6 @@ · 14 unmodified lines
15		15
16	#define SZL (BITS_PER_LONG/8)	16	#define SZL (BITS_PER_LONG/8)
17		17
18	/*
19	* Stuff for accurate CPU time accounting.
20	* These macros handle transitions between user and system state
21	* in exception entry and exit and accumulate time to the
22	* user_time and system_time fields in the paca.
23	*/
24
25	#ifndef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
26	#define ACCOUNT_CPU_USER_ENTRY(ptr, ra, rb)
27	#define ACCOUNT_CPU_USER_EXIT(ptr, ra, rb)
28	#else
29	#define ACCOUNT_CPU_USER_ENTRY(ptr, ra, rb) \
30	MFTB(ra); /* get timebase */ \
31	PPC_LL rb, ACCOUNT_STARTTIME_USER(ptr); \
32	PPC_STL ra, ACCOUNT_STARTTIME(ptr); \
33	subf rb,rb,ra; /* subtract start value */ \
34	PPC_LL ra, ACCOUNT_USER_TIME(ptr); \
35	add ra,ra,rb; /* add on to user time */ \
36	PPC_STL ra, ACCOUNT_USER_TIME(ptr); \
37
38	#define ACCOUNT_CPU_USER_EXIT(ptr, ra, rb) \
39	MFTB(ra); /* get timebase */ \
40	PPC_LL rb, ACCOUNT_STARTTIME(ptr); \
41	PPC_STL ra, ACCOUNT_STARTTIME_USER(ptr); \
42	subf rb,rb,ra; /* subtract start value */ \
43	PPC_LL ra, ACCOUNT_SYSTEM_TIME(ptr); \
44	add ra,ra,rb; /* add on to system time */ \
45	PPC_STL ra, ACCOUNT_SYSTEM_TIME(ptr)
46	#endif /* CONFIG_VIRT_CPU_ACCOUNTING_NATIVE */
47
48	/*	18	/*
49	* Macros for storing registers into and loading registers from	19	* Macros for storing registers into and loading registers from
50	* exception frames.	20	* exception frames.

arch/powerpc/include/asm/processor.h+4−5

 #endif
 #ifdef CONFIG_PPC32
 	void		*pgdir;		/* root of page-table tree */
-	unsigned long	ksp_limit;	/* if ksp <= ksp_limit stack overflow */
 #ifdef CONFIG_PPC_RTAS
 	unsigned long	rtas_sp;	/* stack pointer for when in RTAS */
 #endif
-#endif
 #if defined(CONFIG_PPC_BOOK3S_32) && defined(CONFIG_PPC_KUAP)
 	unsigned long	kuap;		/* opened segments for user access */
 #endif
-#ifdef CONFIG_VMAP_STACK
 	unsigned long	srr0;
 	unsigned long	srr1;
 	unsigned long	dar;
 	unsigned long	r0, r3, r4, r5, r6, r8, r9, r11;
 	unsigned long	lr, ctr;
 #endif
-#endif
+#endif /* CONFIG_PPC32 */
 	/* Debug Registers */
 	struct debug_reg debug;
 #ifdef CONFIG_PPC_FPU_REGS
 #ifdef CONFIG_PPC32
 #define INIT_THREAD { \
 	.ksp = INIT_SP, \
-	.ksp_limit = INIT_SP_LIMIT, \
 	.pgdir = swapper_pg_dir, \
 	.fpexc_mode = MSR_FE0 | MSR_FE1, \
 	SPEFSCR_INIT \
 extern unsigned long isa206_idle_insn_mayloss(unsigned long type);
 #ifdef CONFIG_PPC_970_NAP
 extern void power4_idle_nap(void);
+void power4_idle_nap_return(void);
 #endif
 extern unsigned long cpuidle_disable;
 #define NET_IP_ALIGN	0
 #endif
+int do_mathemu(struct pt_regs *regs);
 #endif /* __KERNEL__ */
 #endif /* __ASSEMBLY__ */
 #endif /* _ASM_POWERPC_PROCESSOR_H */

arch/powerpc/include/asm/ptrace.h+14−31

 #define current_pt_regs() \
 	((struct pt_regs *)((unsigned long)task_stack_page(current) + THREAD_SIZE) - 1)
+/*
+ * The 4 low bits (0xf) are available as flags to overload the trap word,
+ * because interrupt vectors have minimum alignment of 0x10. TRAP_FLAGS_MASK
+ * must cover the bits used as flags, including bit 0 which is used as the
+ * "norestart" bit.
+ */
 #ifdef __powerpc64__
-#ifdef CONFIG_PPC_BOOK3S
+#define TRAP_FLAGS_MASK		0x1
-#define TRAP_FLAGS_MASK		0x10
-#define TRAP(regs)		((regs)->trap & ~TRAP_FLAGS_MASK)
-#define FULL_REGS(regs)		true
-#define SET_FULL_REGS(regs)	do { } while (0)
-#else
-#define TRAP_FLAGS_MASK		0x11
-#define TRAP(regs)		((regs)->trap & ~TRAP_FLAGS_MASK)
-#define FULL_REGS(regs)		(((regs)->trap & 1) == 0)
-#define SET_FULL_REGS(regs)	((regs)->trap &= ~1)
-#endif
-#define CHECK_FULL_REGS(regs)	BUG_ON(!FULL_REGS(regs))
-#define NV_REG_POISON		0xdeadbeefdeadbeefUL
 #else
 /*
- * We use the least-significant bit of the trap field to indicate
+ * On 4xx we use bit 1 in the trap word to indicate whether the exception
- * whether we have saved the full set of registers, or only a
- * partial set.  A 1 there means the partial set.
- * On 4xx we use the next bit to indicate whether the exception
  * is a critical exception (1 means it is).
  */
-#define TRAP_FLAGS_MASK		0x1F
+#define TRAP_FLAGS_MASK		0xf
-#define TRAP(regs)		((regs)->trap & ~TRAP_FLAGS_MASK)
-#define FULL_REGS(regs)		(((regs)->trap & 1) == 0)
-#define SET_FULL_REGS(regs)	((regs)->trap &= ~1)
 #define IS_CRITICAL_EXC(regs)	(((regs)->trap & 2) != 0)
 #define IS_MCHECK_EXC(regs)	(((regs)->trap & 4) != 0)
 #define IS_DEBUG_EXC(regs)	(((regs)->trap & 8) != 0)
-#define NV_REG_POISON		0xdeadbeef
-#define CHECK_FULL_REGS(regs)						      \
-do {									      \
-	if ((regs)->trap & 1)						      \
-		printk(KERN_CRIT "%s: partial register set\n", __func__); \
-} while (0)
 #endif /* __powerpc64__ */
+#define TRAP(regs)		((regs)->trap & ~TRAP_FLAGS_MASK)
-static inline void set_trap(struct pt_regs *regs, unsigned long val)
+static __always_inline void set_trap(struct pt_regs *regs, unsigned long val)
 {
 	regs->trap = (regs->trap & TRAP_FLAGS_MASK) | (val & ~TRAP_FLAGS_MASK);
 }
 static inline bool trap_norestart(struct pt_regs *regs)
 {
-	return regs->trap & 0x10;
+	return regs->trap & 0x1;
 }
-static inline void set_trap_norestart(struct pt_regs *regs)
+static __always_inline void set_trap_norestart(struct pt_regs *regs)
 {
-	regs->trap |= 0x10;
+	regs->trap |= 0x1;
 }
 #define arch_has_single_step()	(1)

arch/powerpc/include/asm/qspinlock.h+7−14

 }
 #define queued_spin_lock queued_spin_lock
-#define smp_mb__after_spinlock()   smp_mb()
-static __always_inline int queued_spin_is_locked(struct qspinlock *lock)
-{
-	/*
-	 * This barrier was added to simple spinlocks by commit 51d7d5205d338,
-	 * but it should now be possible to remove it, asm arm64 has done with
-	 * commit c6f5d02b6a0f.
-	 */
-	smp_mb();
-	return atomic_read(&lock->val);
-}
-#define queued_spin_is_locked queued_spin_is_locked
 #ifdef CONFIG_PARAVIRT_SPINLOCKS
 #define SPIN_THRESHOLD (1<<15) /* not tuned */
 #endif
+/*
+ * Queued spinlocks rely heavily on smp_cond_load_relaxed() to busy-wait,
+ * which was found to have performance problems if implemented with
+ * the preferred spin_begin()/spin_end() SMT priority pattern. Use the
+ * generic version instead.
+ */
 #include <asm-generic/qspinlock.h>
 #endif /* _ASM_POWERPC_QSPINLOCK_H */

arch/powerpc/include/asm/reg.h+3−3

 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
 #define MSR_TM_ACTIVE(x) (((x) & MSR_TS_MASK) != 0) /* Transaction active? */
 #else
-#define MSR_TM_ACTIVE(x) 0
+#define MSR_TM_ACTIVE(x) ((void)(x), 0)
 #endif
 #if defined(CONFIG_PPC_BOOK3S_64)
 #define   LPCR_VRMA_LP1		ASM_CONST(0x0000800000000000)
 #define   LPCR_RMLS		0x1C000000	/* Implementation dependent RMO limit sel */
 #define   LPCR_RMLS_SH		26
+#define   LPCR_HAIL		ASM_CONST(0x0000000004000000)   /* HV AIL (ISAv3.1) */
 #define   LPCR_ILE		ASM_CONST(0x0000000002000000)   /* !HV irqs set MSR:LE */
 #define   LPCR_AIL		ASM_CONST(0x0000000001800000)	/* Alternate interrupt location */
 #define   LPCR_AIL_0		ASM_CONST(0x0000000000000000)	/* MMU off exception offset 0x0 */
 				     : "r" ((unsigned long)(v)) \
 				     : "memory")
 #endif
-#define wrtspr(rn)	asm volatile("mtspr " __stringify(rn) ",0" : \
+#define wrtspr(rn)	asm volatile("mtspr " __stringify(rn) ",2" : : : "memory")
-				     : : "memory")
 static inline void wrtee(unsigned long val)
 {

arch/powerpc/include/asm/rtas.h+3−3

@@ -19,8 +19,8 @@ · 18 unmodified lines
19	#define RTAS_UNKNOWN_SERVICE (-1)	19	#define RTAS_UNKNOWN_SERVICE (-1)
20	#define RTAS_INSTANTIATE_MAX (1ULL<<30) /* Don't instantiate rtas at/above this value */	20	#define RTAS_INSTANTIATE_MAX (1ULL<<30) /* Don't instantiate rtas at/above this value */
21		21
22	/* Buffer size for ppc_rtas system call. */	22	/* Memory set aside for sys_rtas to use with calls that need a work area. */
23	#define RTAS_RMOBUF_MAX (64 * 1024)	23	#define RTAS_USER_REGION_SIZE (64 * 1024)
24		24
25	/* RTAS return status codes */	25	/* RTAS return status codes */
26	#define RTAS_BUSY -2 /* RTAS Busy */	26	#define RTAS_BUSY -2 /* RTAS Busy */
@@ -357,7 +357,7 @@ extern void rtas_take_timebase(void); · 330 unmodified lines
357	static inline int page_is_rtas_user_buf(unsigned long pfn)	357	static inline int page_is_rtas_user_buf(unsigned long pfn)
358	{	358	{
359	unsigned long paddr = (pfn << PAGE_SHIFT);	359	unsigned long paddr = (pfn << PAGE_SHIFT);
360	if (paddr >= rtas_rmo_buf && paddr < (rtas_rmo_buf + RTAS_RMOBUF_MAX))	360	if (paddr >= rtas_rmo_buf && paddr < (rtas_rmo_buf + RTAS_USER_REGION_SIZE))
361	return 1;	361	return 1;
362	return 0;	362	return 0;
363	}	363	}

arch/powerpc/include/asm/simple_spinlock.h+1−5

 static inline int arch_spin_is_locked(arch_spinlock_t *lock)
 {
-	smp_mb();
+	return !arch_spin_value_unlocked(READ_ONCE(*lock));
-	return !arch_spin_value_unlocked(*lock);
 }
 /*
 #define arch_read_relax(lock)	rw_yield(lock)
 #define arch_write_relax(lock)	rw_yield(lock)
-/* See include/linux/spinlock.h */
-#define smp_mb__after_spinlock()   smp_mb()
 #endif /* _ASM_POWERPC_SIMPLE_SPINLOCK_H */

arch/powerpc/include/asm/smp.h+6−0

 extern bool coregroup_enabled;
 extern int cpu_to_chip_id(int cpu);
+extern int *chip_id_lookup_table;
 #ifdef CONFIG_SMP
 	return per_cpu(cpu_sibling_map, cpu);
 }
+static inline struct cpumask *cpu_core_mask(int cpu)
+{
+	return per_cpu(cpu_core_map, cpu);
+}
 static inline struct cpumask *cpu_l2_cache_mask(int cpu)
 {
 	return per_cpu(cpu_l2_cache_map, cpu);

arch/powerpc/include/asm/spinlock.h+3−0

@@ -10,6 +10,9 @@ · 9 unmodified lines
10	#include <asm/simple_spinlock.h>	10	#include <asm/simple_spinlock.h>
11	#endif	11	#endif
12		12
		13	/* See include/linux/spinlock.h */
		14	#define smp_mb__after_spinlock() smp_mb()
		15
13	#ifndef CONFIG_PARAVIRT_SPINLOCKS	16	#ifndef CONFIG_PARAVIRT_SPINLOCKS
14	static inline void pv_spinlocks_init(void) { }	17	static inline void pv_spinlocks_init(void) { }
15	#endif	18	#endif

arch/powerpc/include/asm/thread_info.h+6−1

 #ifndef __ASSEMBLY__
 #include <linux/cache.h>
 #include <asm/processor.h>
-#include <asm/page.h>
 #include <asm/accounting.h>
 #define SLB_PRELOAD_NR	16U
 #ifndef __ASSEMBLY__
+static inline void clear_thread_local_flags(unsigned int flags)
+{
+	struct thread_info *ti = current_thread_info();
+	ti->local_flags &= ~flags;
+}
 static inline bool test_thread_local_flags(unsigned int flags)
 {
 	struct thread_info *ti = current_thread_info();

arch/powerpc/include/asm/topology.h+1−1

@@ -126,7 +126,7 @@ static inline int cpu_to_coregroup_id(int cpu) · 125 unmodified lines
126	#define topology_physical_package_id(cpu) (cpu_to_chip_id(cpu))	126	#define topology_physical_package_id(cpu) (cpu_to_chip_id(cpu))
127		127
128	#define topology_sibling_cpumask(cpu) (per_cpu(cpu_sibling_map, cpu))	128	#define topology_sibling_cpumask(cpu) (per_cpu(cpu_sibling_map, cpu))
129	#define topology_core_cpumask(cpu) (cpu_cpu_mask(cpu))	129	#define topology_core_cpumask(cpu) (per_cpu(cpu_core_map, cpu))
130	#define topology_core_id(cpu) (cpu_to_core_id(cpu))	130	#define topology_core_id(cpu) (cpu_to_core_id(cpu))
131		131
132	#endif	132	#endif

arch/powerpc/include/asm/uaccess.h+171−218

  * exception handling means that it's no longer "just"...)
  *
  */
-#define get_user(x, ptr) \
+#define __put_user(x, ptr)					\
-	__get_user_check((x), (ptr), sizeof(*(ptr)))
-#define put_user(x, ptr) \
-	__put_user_check((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr)))
-#define __get_user(x, ptr) \
-	__get_user_nocheck((x), (ptr), sizeof(*(ptr)), true)
-#define __put_user(x, ptr) \
-	__put_user_nocheck((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr)))
-#define __get_user_allowed(x, ptr) \
-	__get_user_nocheck((x), (ptr), sizeof(*(ptr)), false)
-#define __get_user_inatomic(x, ptr) \
-	__get_user_nosleep((x), (ptr), sizeof(*(ptr)))
-#define __put_user_inatomic(x, ptr) \
-	__put_user_nosleep((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr)))
-#ifdef CONFIG_PPC64
-#define ___get_user_instr(gu_op, dest, ptr)				\
-({									\
-	long __gui_ret = 0;						\
-	unsigned long __gui_ptr = (unsigned long)ptr;			\
-	struct ppc_inst __gui_inst;					\
-	unsigned int __prefix, __suffix;				\
-	__gui_ret = gu_op(__prefix, (unsigned int __user *)__gui_ptr);	\
-	if (__gui_ret == 0) {						\
-		if ((__prefix >> 26) == OP_PREFIX) {			\
-			__gui_ret = gu_op(__suffix,			\
-				(unsigned int __user *)__gui_ptr + 1);	\
-			__gui_inst = ppc_inst_prefix(__prefix,		\
-						     __suffix);		\
-		} else {						\
-			__gui_inst = ppc_inst(__prefix);		\
-		}							\
-		if (__gui_ret == 0)					\
-			(dest) = __gui_inst;				\
-	}								\
-	__gui_ret;							\
-})
-#define get_user_instr(x, ptr) \
-	___get_user_instr(get_user, x, ptr)
-#define __get_user_instr(x, ptr) \
-	___get_user_instr(__get_user, x, ptr)
-#define __get_user_instr_inatomic(x, ptr) \
-	___get_user_instr(__get_user_inatomic, x, ptr)
-#else /* !CONFIG_PPC64 */
-#define get_user_instr(x, ptr) \
-	get_user((x).val, (u32 __user *)(ptr))
-#define __get_user_instr(x, ptr) \
-	__get_user_nocheck((x).val, (u32 __user *)(ptr), sizeof(u32), true)
-#define __get_user_instr_inatomic(x, ptr) \
-	__get_user_nosleep((x).val, (u32 __user *)(ptr), sizeof(u32))
-#endif /* CONFIG_PPC64 */
-extern long __put_user_bad(void);
-#define __put_user_size(x, ptr, size, retval)			\
-do {								\
-	__label__ __pu_failed;					\
-								\
-	retval = 0;						\
-	allow_write_to_user(ptr, size);				\
-	__put_user_size_goto(x, ptr, size, __pu_failed);	\
-	prevent_write_to_user(ptr, size);			\
-	break;							\
-								\
-__pu_failed:							\
-	retval = -EFAULT;					\
-	prevent_write_to_user(ptr, size);			\
-} while (0)
-#define __put_user_nocheck(x, ptr, size)			\
 ({								\
 	long __pu_err;						\
 	__typeof__(*(ptr)) __user *__pu_addr = (ptr);		\
-	__typeof__(*(ptr)) __pu_val = (x);			\
+	__typeof__(*(ptr)) __pu_val = (__typeof__(*(ptr)))(x);	\
-	__typeof__(size) __pu_size = (size);			\
+	__typeof__(sizeof(*(ptr))) __pu_size = sizeof(*(ptr));	\
 								\
-	if (!is_kernel_addr((unsigned long)__pu_addr))		\
+	might_fault();						\
-		might_fault();					\
+	do {							\
-	__chk_user_ptr(__pu_addr);				\
+		__label__ __pu_failed;				\
-	__put_user_size(__pu_val, __pu_addr, __pu_size, __pu_err);	\
+								\
+		allow_write_to_user(__pu_addr, __pu_size);	\
+		__put_user_size_goto(__pu_val, __pu_addr, __pu_size, __pu_failed);	\
+		prevent_write_to_user(__pu_addr, __pu_size);	\
+		__pu_err = 0;					\
+		break;						\
+								\
+__pu_failed:							\
+		prevent_write_to_user(__pu_addr, __pu_size);	\
+		__pu_err = -EFAULT;				\
+	} while (0);						\
 								\
 	__pu_err;						\
 })
-#define __put_user_check(x, ptr, size)					\
+#define put_user(x, ptr)						\
 ({									\
-	long __pu_err = -EFAULT;					\
+	__typeof__(*(ptr)) __user *_pu_addr = (ptr);			\
-	__typeof__(*(ptr)) __user *__pu_addr = (ptr);			\
-	__typeof__(*(ptr)) __pu_val = (x);				\
-	__typeof__(size) __pu_size = (size);				\
-									\
-	might_fault();							\
-	if (access_ok(__pu_addr, __pu_size))				\
-		__put_user_size(__pu_val, __pu_addr, __pu_size, __pu_err); \
 									\
-	__pu_err;							\
+	access_ok(_pu_addr, sizeof(*(ptr))) ?				\
+		  __put_user(x, _pu_addr) : -EFAULT;			\
 })
-#define __put_user_nosleep(x, ptr, size)			\
-({								\
-	long __pu_err;						\
-	__typeof__(*(ptr)) __user *__pu_addr = (ptr);		\
-	__typeof__(*(ptr)) __pu_val = (x);			\
-	__typeof__(size) __pu_size = (size);			\
-								\
-	__chk_user_ptr(__pu_addr);				\
-	__put_user_size(__pu_val, __pu_addr, __pu_size, __pu_err); \
-								\
-	__pu_err;						\
-})
 /*
  * We don't tell gcc that we are accessing memory, but this is OK
  * because we do not write to any memory gcc knows about, so there
 #define __put_user_size_goto(x, ptr, size, label)		\
 do {								\
+	__typeof__(*(ptr)) __user *__pus_addr = (ptr);		\
+								\
 	switch (size) {						\
-	case 1: __put_user_asm_goto(x, ptr, label, "stb"); break;	\
+	case 1: __put_user_asm_goto(x, __pus_addr, label, "stb"); break;	\
-	case 2: __put_user_asm_goto(x, ptr, label, "sth"); break;	\
+	case 2: __put_user_asm_goto(x, __pus_addr, label, "sth"); break;	\
-	case 4: __put_user_asm_goto(x, ptr, label, "stw"); break;	\
+	case 4: __put_user_asm_goto(x, __pus_addr, label, "stw"); break;	\
-	case 8: __put_user_asm2_goto(x, ptr, label); break;	\
+	case 8: __put_user_asm2_goto(x, __pus_addr, label); break;		\
-	default: __put_user_bad();				\
+	default: BUILD_BUG();					\
 	}							\
 } while (0)
-#define __unsafe_put_user_goto(x, ptr, size, label)		\
-do {								\
-	__typeof__(*(ptr)) __user *__pu_addr = (ptr);		\
-	__chk_user_ptr(ptr);					\
-	__put_user_size_goto((x), __pu_addr, (size), label);	\
-} while (0)
-extern long __get_user_bad(void);
 /*
  * This does an atomic 128 byte aligned load from userspace.
  * Upto caller to do enable_kernel_vmx() before calling!
 		: "=r" (err)			\
 		: "b" (uaddr), "b" (kaddr), "i" (-EFAULT), "0" (err))
+#ifdef CONFIG_CC_HAS_ASM_GOTO_OUTPUT
+#define __get_user_asm_goto(x, addr, label, op)			\
+	asm_volatile_goto(					\
+		"1:	"op"%U1%X1 %0, %1	# get_user\n"	\
+		EX_TABLE(1b, %l2)				\
+		: "=r" (x)					\
+		: "m"UPD_CONSTR (*addr)				\
+		:						\
+		: label)
+#ifdef __powerpc64__
+#define __get_user_asm2_goto(x, addr, label)			\
+	__get_user_asm_goto(x, addr, label, "ld")
+#else /* __powerpc64__ */
+#define __get_user_asm2_goto(x, addr, label)			\
+	asm_volatile_goto(					\
+		"1:	lwz%X1 %0, %1\n"			\
+		"2:	lwz%X1 %L0, %L1\n"			\
+		EX_TABLE(1b, %l2)				\
+		EX_TABLE(2b, %l2)				\
+		: "=r" (x)					\
+		: "m" (*addr)					\
+		:						\
+		: label)
+#endif /* __powerpc64__ */
+#define __get_user_size_goto(x, ptr, size, label)				\
+do {										\
+	BUILD_BUG_ON(size > sizeof(x));						\
+	switch (size) {								\
+	case 1: __get_user_asm_goto(x, (u8 __user *)ptr, label, "lbz"); break;	\
+	case 2: __get_user_asm_goto(x, (u16 __user *)ptr, label, "lhz"); break;	\
+	case 4: __get_user_asm_goto(x, (u32 __user *)ptr, label, "lwz"); break;	\
+	case 8: __get_user_asm2_goto(x, (u64 __user *)ptr, label);  break;	\
+	default: x = 0; BUILD_BUG();						\
+	}									\
+} while (0)
+#define __get_user_size_allowed(x, ptr, size, retval)			\
+do {									\
+		__label__ __gus_failed;					\
+									\
+		__get_user_size_goto(x, ptr, size, __gus_failed);	\
+		retval = 0;						\
+		break;							\
+__gus_failed:								\
+		x = 0;							\
+		retval = -EFAULT;					\
+} while (0)
+#else /* CONFIG_CC_HAS_ASM_GOTO_OUTPUT */
 #define __get_user_asm(x, addr, err, op)		\
 	__asm__ __volatile__(				\
 		"1:	"op"%U2%X2 %1, %2	# get_user\n"	\
 #define __get_user_size_allowed(x, ptr, size, retval)		\
 do {								\
 	retval = 0;						\
-	__chk_user_ptr(ptr);					\
+	BUILD_BUG_ON(size > sizeof(x));				\
-	if (size > sizeof(x))					\
-		(x) = __get_user_bad();				\
 	switch (size) {						\
 	case 1: __get_user_asm(x, (u8 __user *)ptr, retval, "lbz"); break;	\
 	case 2: __get_user_asm(x, (u16 __user *)ptr, retval, "lhz"); break;	\
 	case 4: __get_user_asm(x, (u32 __user *)ptr, retval, "lwz"); break;	\
 	case 8: __get_user_asm2(x, (u64 __user *)ptr, retval);  break;	\
-	default: (x) = __get_user_bad();			\
+	default: x = 0; BUILD_BUG();				\
 	}							\
 } while (0)
-#define __get_user_size(x, ptr, size, retval)			\
+#define __get_user_size_goto(x, ptr, size, label)		\
 do {								\
-	allow_read_from_user(ptr, size);			\
+	long __gus_retval;					\
-	__get_user_size_allowed(x, ptr, size, retval);		\
+								\
-	prevent_read_from_user(ptr, size);			\
+	__get_user_size_allowed(x, ptr, size, __gus_retval);	\
+	if (__gus_retval)					\
+		goto label;					\
 } while (0)
+#endif /* CONFIG_CC_HAS_ASM_GOTO_OUTPUT */
 /*
  * This is a type: either unsigned long, if the argument fits into
  * that type, or otherwise unsigned long long.
 #define __long_type(x) \
 	__typeof__(__builtin_choose_expr(sizeof(x) > sizeof(0UL), 0ULL, 0UL))
-#define __get_user_nocheck(x, ptr, size, do_allow)			\
+#define __get_user(x, ptr)					\
 ({								\
 	long __gu_err;						\
 	__long_type(*(ptr)) __gu_val;				\
 	__typeof__(*(ptr)) __user *__gu_addr = (ptr);	\
-	__typeof__(size) __gu_size = (size);			\
+	__typeof__(sizeof(*(ptr))) __gu_size = sizeof(*(ptr));	\
 								\
-	__chk_user_ptr(__gu_addr);				\
+	might_fault();					\
-	if (do_allow && !is_kernel_addr((unsigned long)__gu_addr)) \
+	allow_read_from_user(__gu_addr, __gu_size);		\
-		might_fault();					\
+	__get_user_size_allowed(__gu_val, __gu_addr, __gu_size, __gu_err);	\
-	if (do_allow)								\
+	prevent_read_from_user(__gu_addr, __gu_size);		\
-		__get_user_size(__gu_val, __gu_addr, __gu_size, __gu_err);	\
-	else									\
-		__get_user_size_allowed(__gu_val, __gu_addr, __gu_size, __gu_err); \
 	(x) = (__typeof__(*(ptr)))__gu_val;			\
 								\
 	__gu_err;						\
 })
-#define __get_user_check(x, ptr, size)					\
+#define get_user(x, ptr)						\
 ({									\
-	long __gu_err = -EFAULT;					\
+	__typeof__(*(ptr)) __user *_gu_addr = (ptr);			\
-	__long_type(*(ptr)) __gu_val = 0;				\
-	__typeof__(*(ptr)) __user *__gu_addr = (ptr);		\
-	__typeof__(size) __gu_size = (size);				\
-									\
-	might_fault();							\
-	if (access_ok(__gu_addr, __gu_size))				\
-		__get_user_size(__gu_val, __gu_addr, __gu_size, __gu_err); \
-	(x) = (__force __typeof__(*(ptr)))__gu_val;				\
 									\
-	__gu_err;							\
+	access_ok(_gu_addr, sizeof(*(ptr))) ?				\
+		  __get_user(x, _gu_addr) :				\
+		  ((x) = (__force __typeof__(*(ptr)))0, -EFAULT);	\
 })
-#define __get_user_nosleep(x, ptr, size)			\
-({								\
-	long __gu_err;						\
-	__long_type(*(ptr)) __gu_val;				\
-	__typeof__(*(ptr)) __user *__gu_addr = (ptr);	\
-	__typeof__(size) __gu_size = (size);			\
-								\
-	__chk_user_ptr(__gu_addr);				\
-	__get_user_size(__gu_val, __gu_addr, __gu_size, __gu_err); \
-	(x) = (__force __typeof__(*(ptr)))__gu_val;			\
-								\
-	__gu_err;						\
-})
 /* more complex routines */
 extern unsigned long __copy_tofrom_user(void __user *to,
 		const void __user *from, unsigned long size);
-#ifdef CONFIG_ARCH_HAS_COPY_MC
-unsigned long __must_check
-copy_mc_generic(void *to, const void *from, unsigned long size);
-static inline unsigned long __must_check
-copy_mc_to_kernel(void *to, const void *from, unsigned long size)
-{
-	return copy_mc_generic(to, from, size);
-}
-#define copy_mc_to_kernel copy_mc_to_kernel
-static inline unsigned long __must_check
-copy_mc_to_user(void __user *to, const void *from, unsigned long n)
-{
-	if (likely(check_copy_size(from, n, true))) {
-		if (access_ok(to, n)) {
-			allow_write_to_user(to, n);
-			n = copy_mc_generic((void *)to, from, n);
-			prevent_write_to_user(to, n);
-		}
-	}
-	return n;
-}
-#endif
 #ifdef __powerpc64__
 static inline unsigned long
 raw_copy_in_user(void __user *to, const void __user *from, unsigned long n)
 unsigned long __arch_clear_user(void __user *addr, unsigned long size);
-static inline unsigned long clear_user(void __user *addr, unsigned long size)
+static inline unsigned long __clear_user(void __user *addr, unsigned long size)
 {
-	unsigned long ret = size;
+	unsigned long ret;
 	might_fault();
-	if (likely(access_ok(addr, size))) {
+	allow_write_to_user(addr, size);
-		allow_write_to_user(addr, size);
+	ret = __arch_clear_user(addr, size);
-		ret = __arch_clear_user(addr, size);
+	prevent_write_to_user(addr, size);
-		prevent_write_to_user(addr, size);
-	}
 	return ret;
 }
-static inline unsigned long __clear_user(void __user *addr, unsigned long size)
+static inline unsigned long clear_user(void __user *addr, unsigned long size)
 {
-	return clear_user(addr, size);
+	return likely(access_ok(addr, size)) ? __clear_user(addr, size) : size;
 }
 extern long strncpy_from_user(char *dst, const char __user *src, long count);
 extern __must_check long strnlen_user(const char __user *str, long n);
+#ifdef CONFIG_ARCH_HAS_COPY_MC
+unsigned long __must_check
+copy_mc_generic(void *to, const void *from, unsigned long size);
+static inline unsigned long __must_check
+copy_mc_to_kernel(void *to, const void *from, unsigned long size)
+{
+	return copy_mc_generic(to, from, size);
+}
+#define copy_mc_to_kernel copy_mc_to_kernel
+static inline unsigned long __must_check
+copy_mc_to_user(void __user *to, const void *from, unsigned long n)
+{
+	if (likely(check_copy_size(from, n, true))) {
+		if (access_ok(to, n)) {
+			allow_write_to_user(to, n);
+			n = copy_mc_generic((void *)to, from, n);
+			prevent_write_to_user(to, n);
+		}
+	}
+	return n;
+}
+#endif
 extern long __copy_from_user_flushcache(void *dst, const void __user *src,
 		unsigned size);
 extern void memcpy_page_flushcache(char *to, struct page *page, size_t offset,
 #define user_write_access_begin	user_write_access_begin
 #define user_write_access_end		prevent_current_write_to_user
-#define unsafe_op_wrap(op, err) do { if (unlikely(op)) goto err; } while (0)
+#define unsafe_get_user(x, p, e) do {					\
-#define unsafe_get_user(x, p, e) unsafe_op_wrap(__get_user_allowed(x, p), e)
+	__long_type(*(p)) __gu_val;				\
+	__typeof__(*(p)) __user *__gu_addr = (p);		\
+								\
+	__get_user_size_goto(__gu_val, __gu_addr, sizeof(*(p)), e); \
+	(x) = (__typeof__(*(p)))__gu_val;			\
+} while (0)
 #define unsafe_put_user(x, p, e) \
-	__unsafe_put_user_goto((__typeof__(*(p)))(x), (p), sizeof(*(p)), e)
+	__put_user_size_goto((__typeof__(*(p)))(x), (p), sizeof(*(p)), e)
+#define unsafe_copy_from_user(d, s, l, e) \
+do {											\
+	u8 *_dst = (u8 *)(d);								\
+	const u8 __user *_src = (const u8 __user *)(s);					\
+	size_t _len = (l);								\
+	int _i;										\
+											\
+	for (_i = 0; _i < (_len & ~(sizeof(u64) - 1)); _i += sizeof(u64))		\
+		unsafe_get_user(*(u64 *)(_dst + _i), (u64 __user *)(_src + _i), e);	\
+	if (_len & 4) {									\
+		unsafe_get_user(*(u32 *)(_dst + _i), (u32 __user *)(_src + _i), e);	\
+		_i += 4;								\
+	}										\
+	if (_len & 2) {									\
+		unsafe_get_user(*(u16 *)(_dst + _i), (u16 __user *)(_src + _i), e);	\
+		_i += 2;								\
+	}										\
+	if (_len & 1)									\
+		unsafe_get_user(*(u8 *)(_dst + _i), (u8 __user *)(_src + _i), e);	\
+} while (0)
 #define unsafe_copy_to_user(d, s, l, e) \
 do {									\
 	size_t _len = (l);						\
 	int _i;								\
 									\
-	for (_i = 0; _i < (_len & ~(sizeof(long) - 1)); _i += sizeof(long))		\
+	for (_i = 0; _i < (_len & ~(sizeof(u64) - 1)); _i += sizeof(u64))	\
-		unsafe_put_user(*(long*)(_src + _i), (long __user *)(_dst + _i), e); \
+		unsafe_put_user(*(u64 *)(_src + _i), (u64 __user *)(_dst + _i), e); \
-	if (IS_ENABLED(CONFIG_PPC64) && (_len & 4)) {			\
+	if (_len & 4) {							\
 		unsafe_put_user(*(u32*)(_src + _i), (u32 __user *)(_dst + _i), e); \
 		_i += 4;						\
 	}								\
 #define HAVE_GET_KERNEL_NOFAULT
 #define __get_kernel_nofault(dst, src, type, err_label)			\
-do {									\
+	__get_user_size_goto(*((type *)(dst)),				\
-	int __kr_err;							\
+		(__force type __user *)(src), sizeof(type), err_label)
-									\
-	__get_user_size_allowed(*((type *)(dst)), (__force type __user *)(src),\
-			sizeof(type), __kr_err);			\
-	if (unlikely(__kr_err))						\
-		goto err_label;						\
-} while (0)
 #define __put_kernel_nofault(dst, src, type, err_label)			\
 	__put_user_size_goto(*((type *)(src)),				\

arch/powerpc/include/asm/unistd.h+1−0

@@ -40,6 +40,7 @@ · 39 unmodified lines
40	#define __ARCH_WANT_SYS_SIGPROCMASK	40	#define __ARCH_WANT_SYS_SIGPROCMASK
41	#ifdef CONFIG_PPC32	41	#ifdef CONFIG_PPC32
42	#define __ARCH_WANT_OLD_STAT	42	#define __ARCH_WANT_OLD_STAT
		43	#define __ARCH_WANT_SYS_OLD_SELECT
43	#endif	44	#endif
44	#ifdef CONFIG_PPC64	45	#ifdef CONFIG_PPC64
45	#define __ARCH_WANT_SYS_TIME	46	#define __ARCH_WANT_SYS_TIME

arch/powerpc/include/asm/vdso/gettimeofday.h+10−0

 #ifndef _ASM_POWERPC_VDSO_GETTIMEOFDAY_H
 #define _ASM_POWERPC_VDSO_GETTIMEOFDAY_H
+#include <asm/page.h>
 #ifdef __ASSEMBLY__
 #include <asm/ppc_asm.h>
 const struct vdso_data *__arch_get_vdso_data(void);
+#ifdef CONFIG_TIME_NS
+static __always_inline
+const struct vdso_data *__arch_get_timens_vdso_data(const struct vdso_data *vd)
+{
+	return (void *)vd + PAGE_SIZE;
+}
+#endif
 static inline bool vdso_clocksource_ok(const struct vdso_data *vd)
 {
 	return true;

arch/powerpc/include/asm/vdso_datapage.h+0−2

 	bcl	20, 31, .+4
 999:
 	mflr	\ptr
-#if CONFIG_PPC_PAGE_SHIFT > 14
 	addis	\ptr, \ptr, (_vdso_datapage - 999b)@ha
-#endif
 	addi	\ptr, \ptr, (_vdso_datapage - 999b)@l
 .endm

arch/powerpc/include/asm/vio.h+1−0

@@ -114,6 +114,7 @@ struct vio_driver { · 113 unmodified lines
114	const struct vio_device_id *id_table;	114	const struct vio_device_id *id_table;
115	int (probe)(struct vio_dev dev, const struct vio_device_id *id);	115	int (probe)(struct vio_dev dev, const struct vio_device_id *id);
116	void (remove)(struct vio_dev dev);	116	void (remove)(struct vio_dev dev);
		117	void (shutdown)(struct vio_dev dev);
117	/* A driver must have a get_desired_dma() function to	118	/* A driver must have a get_desired_dma() function to
118	* be loaded in a CMO environment if it uses DMA.	119	* be loaded in a CMO environment if it uses DMA.
119	*/	120	*/

arch/powerpc/include/asm/xive.h+1−0

@@ -102,6 +102,7 @@ void xive_flush_interrupt(void); · 101 unmodified lines
102	/* xmon hook */	102	/* xmon hook */
103	void xmon_xive_do_dump(int cpu);	103	void xmon_xive_do_dump(int cpu);
104	int xmon_xive_get_irq_config(u32 hw_irq, struct irq_data *d);	104	int xmon_xive_get_irq_config(u32 hw_irq, struct irq_data *d);
		105	void xmon_xive_get_irq_all(void);
105		106
106	/* APIs used by KVM */	107	/* APIs used by KVM */
107	u32 xive_native_default_eq_shift(void);	108	u32 xive_native_default_eq_shift(void);

arch/powerpc/include/uapi/asm/errno.h+1−0

@@ -2,6 +2,7 @@ · 1 unmodified line
2	#ifndef _ASM_POWERPC_ERRNO_H	2	#ifndef _ASM_POWERPC_ERRNO_H
3	#define _ASM_POWERPC_ERRNO_H	3	#define _ASM_POWERPC_ERRNO_H
4		4
		5	#undef EDEADLOCK
5	#include <asm-generic/errno.h>	6	#include <asm-generic/errno.h>
6		7
7	#undef EDEADLOCK	8	#undef EDEADLOCK

arch/powerpc/include/uapi/asm/posix_types.h+0−5

 typedef unsigned long	__kernel_old_dev_t;
 #define __kernel_old_dev_t __kernel_old_dev_t
 #else
-typedef unsigned int	__kernel_size_t;
-typedef int		__kernel_ssize_t;
-typedef long		__kernel_ptrdiff_t;
-#define __kernel_size_t __kernel_size_t
 typedef short		__kernel_ipc_pid_t;
 #define __kernel_ipc_pid_t __kernel_ipc_pid_t
 #endif

arch/powerpc/kernel/align.c+38−34

 static int emulate_spe(struct pt_regs *regs, unsigned int reg,
 		       struct ppc_inst ppc_instr)
 {
-	int ret;
 	union {
 		u64 ll;
 		u32 w[2];
 	nb = spe_aligninfo[instr].len;
 	flags = spe_aligninfo[instr].flags;
-	/* Verify the address of the operand */
-	if (unlikely(user_mode(regs) &&
-		     !access_ok(addr, nb)))
-		return -EFAULT;
 	/* userland only */
 	if (unlikely(!user_mode(regs)))
 		return 0;
 		}
 	} else {
 		temp.ll = data.ll = 0;
-		ret = 0;
 		p = addr;
+		if (!user_read_access_begin(addr, nb))
+			return -EFAULT;
 		switch (nb) {
 		case 8:
-			ret |= __get_user_inatomic(temp.v[0], p++);
+			unsafe_get_user(temp.v[0], p++, Efault_read);
-			ret |= __get_user_inatomic(temp.v[1], p++);
+			unsafe_get_user(temp.v[1], p++, Efault_read);
-			ret |= __get_user_inatomic(temp.v[2], p++);
+			unsafe_get_user(temp.v[2], p++, Efault_read);
-			ret |= __get_user_inatomic(temp.v[3], p++);
+			unsafe_get_user(temp.v[3], p++, Efault_read);
 			fallthrough;
 		case 4:
-			ret |= __get_user_inatomic(temp.v[4], p++);
+			unsafe_get_user(temp.v[4], p++, Efault_read);
-			ret |= __get_user_inatomic(temp.v[5], p++);
+			unsafe_get_user(temp.v[5], p++, Efault_read);
 			fallthrough;
 		case 2:
-			ret |= __get_user_inatomic(temp.v[6], p++);
+			unsafe_get_user(temp.v[6], p++, Efault_read);
-			ret |= __get_user_inatomic(temp.v[7], p++);
+			unsafe_get_user(temp.v[7], p++, Efault_read);
-			if (unlikely(ret))
-				return -EFAULT;
 		}
+		user_read_access_end();
 		switch (instr) {
 		case EVLDD:
 	/* Store result to memory or update registers */
 	if (flags & ST) {
-		ret = 0;
 		p = addr;
+		if (!user_write_access_begin(addr, nb))
+			return -EFAULT;
 		switch (nb) {
 		case 8:
-			ret |= __put_user_inatomic(data.v[0], p++);
+			unsafe_put_user(data.v[0], p++, Efault_write);
-			ret |= __put_user_inatomic(data.v[1], p++);
+			unsafe_put_user(data.v[1], p++, Efault_write);
-			ret |= __put_user_inatomic(data.v[2], p++);
+			unsafe_put_user(data.v[2], p++, Efault_write);
-			ret |= __put_user_inatomic(data.v[3], p++);
+			unsafe_put_user(data.v[3], p++, Efault_write);
 			fallthrough;
 		case 4:
-			ret |= __put_user_inatomic(data.v[4], p++);
+			unsafe_put_user(data.v[4], p++, Efault_write);
-			ret |= __put_user_inatomic(data.v[5], p++);
+			unsafe_put_user(data.v[5], p++, Efault_write);
 			fallthrough;
 		case 2:
-			ret |= __put_user_inatomic(data.v[6], p++);
+			unsafe_put_user(data.v[6], p++, Efault_write);
-			ret |= __put_user_inatomic(data.v[7], p++);
+			unsafe_put_user(data.v[7], p++, Efault_write);
 		}
-		if (unlikely(ret))
+		user_write_access_end();
-			return -EFAULT;
 	} else {
 		*evr = data.w[0];
 		regs->gpr[reg] = data.w[1];
 	}
 	return 1;
+Efault_read:
+	user_read_access_end();
+	return -EFAULT;
+Efault_write:
+	user_write_access_end();
+	return -EFAULT;
 }
 #endif /* CONFIG_SPE */
 	struct instruction_op op;
 	int r, type;
-	/*
+	if (is_kernel_addr(regs->nip))
-	 * We require a complete register set, if not, then our assembly
+		r = copy_inst_from_kernel_nofault(&instr, (void *)regs->nip);
-	 * is broken
+	else
-	 */
+		r = __get_user_instr(instr, (void __user *)regs->nip);
-	CHECK_FULL_REGS(regs);
-	if (unlikely(__get_user_instr(instr, (void __user *)regs->nip)))
+	if (unlikely(r))
 		return -EFAULT;
 	if ((regs->msr & MSR_LE) != (MSR_KERNEL & MSR_LE)) {
 		/* We don't handle PPC little-endian any more... */

arch/powerpc/kernel/asm-offsets.c+0−17

 	DEFINE(SIGSEGV, SIGSEGV);
 	DEFINE(NMI_MASK, NMI_MASK);
 #else
-	OFFSET(KSP_LIMIT, thread_struct, ksp_limit);
 #ifdef CONFIG_PPC_RTAS
 	OFFSET(RTAS_SP, thread_struct, rtas_sp);
 #endif
 	OFFSET(KSP_VSID, thread_struct, ksp_vsid);
 #else /* CONFIG_PPC64 */
 	OFFSET(PGDIR, thread_struct, pgdir);
-#ifdef CONFIG_VMAP_STACK
 	OFFSET(SRR0, thread_struct, srr0);
 	OFFSET(SRR1, thread_struct, srr1);
 	OFFSET(DAR, thread_struct, dar);
 	OFFSET(THLR, thread_struct, lr);
 	OFFSET(THCTR, thread_struct, ctr);
 #endif
-#endif
 #ifdef CONFIG_SPE
 	OFFSET(THREAD_EVR0, thread_struct, evr[0]);
 	OFFSET(THREAD_ACC, thread_struct, acc);
 	OFFSET(PACAHWCPUID, paca_struct, hw_cpu_id);
 	OFFSET(PACAKEXECSTATE, paca_struct, kexec_state);
 	OFFSET(PACA_DSCR_DEFAULT, paca_struct, dscr_default);
-	OFFSET(ACCOUNT_STARTTIME, paca_struct, accounting.starttime);
-	OFFSET(ACCOUNT_STARTTIME_USER, paca_struct, accounting.starttime_user);
-	OFFSET(ACCOUNT_USER_TIME, paca_struct, accounting.utime);
-	OFFSET(ACCOUNT_SYSTEM_TIME, paca_struct, accounting.stime);
 #ifdef CONFIG_PPC_BOOK3E
 	OFFSET(PACA_TRAP_SAVE, paca_struct, trap_save);
 #endif
 	OFFSET(PACA_SPRG_VDSO, paca_struct, sprg_vdso);
 #else /* CONFIG_PPC64 */
-#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
-	OFFSET(ACCOUNT_STARTTIME, thread_info, accounting.starttime);
-	OFFSET(ACCOUNT_STARTTIME_USER, thread_info, accounting.starttime_user);
-	OFFSET(ACCOUNT_USER_TIME, thread_info, accounting.utime);
-	OFFSET(ACCOUNT_SYSTEM_TIME, thread_info, accounting.stime);
-#endif
 #endif /* CONFIG_PPC64 */
 	/* RTAS */
 	STACK_PT_REGS_OFFSET(GPR11, gpr[11]);
 	STACK_PT_REGS_OFFSET(GPR12, gpr[12]);
 	STACK_PT_REGS_OFFSET(GPR13, gpr[13]);
-#ifndef CONFIG_PPC64
-	STACK_PT_REGS_OFFSET(GPR14, gpr[14]);
-#endif /* CONFIG_PPC64 */
 	/*
 	 * Note: these symbols include _ because they overlap with special
 	 * register names
 	DEFINE(_CSRR1, STACK_INT_FRAME_SIZE+offsetof(struct exception_regs, csrr1));
 	DEFINE(_DSRR0, STACK_INT_FRAME_SIZE+offsetof(struct exception_regs, dsrr0));
 	DEFINE(_DSRR1, STACK_INT_FRAME_SIZE+offsetof(struct exception_regs, dsrr1));
-	DEFINE(SAVED_KSP_LIMIT, STACK_INT_FRAME_SIZE+offsetof(struct exception_regs, saved_ksp_limit));
 #endif
 #endif

arch/powerpc/kernel/eeh.c+7−8

 	pa = pte_pfn(*ptep);
 	/* On radix we can do hugepage mappings for io, so handle that */
-	if (hugepage_shift) {
+	if (!hugepage_shift)
-		pa <<= hugepage_shift;
+		hugepage_shift = PAGE_SHIFT;
-		pa |= token & ((1ul << hugepage_shift) - 1);
-	} else {
-		pa <<= PAGE_SHIFT;
-		pa |= token & (PAGE_SIZE - 1);
-	}
+	pa <<= PAGE_SHIFT;
+	pa |= token & ((1ul << hugepage_shift) - 1);
 	return pa;
 }
 	default:
 		eeh_pe_state_clear(pe, EEH_PE_ISOLATED | EEH_PE_CFG_BLOCKED, true);
 		return -EINVAL;
-	};
+	}
 	return 0;
 }
 }
 EXPORT_SYMBOL_GPL(eeh_pe_inject_err);
+#ifdef CONFIG_PROC_FS
 static int proc_eeh_show(struct seq_file *m, void *v)
 {
 	if (!eeh_enabled()) {
 	return 0;
 }
+#endif /* CONFIG_PROC_FS */
 #ifdef CONFIG_DEBUG_FS

arch/powerpc/kernel/entry_32.S+137−659

@@ -48,195 +48,16 @@ · 47 unmodified lines
48	*/	48	*/
49	.align 12	49	.align 12
50		50
51	#ifdef CONFIG_BOOKE
52	.globl mcheck_transfer_to_handler
53	mcheck_transfer_to_handler:
54	mfspr r0,SPRN_DSRR0
55	stw r0,_DSRR0(r11)
56	mfspr r0,SPRN_DSRR1
57	stw r0,_DSRR1(r11)
58	/* fall through */
59	_ASM_NOKPROBE_SYMBOL(mcheck_transfer_to_handler)
60
61	.globl debug_transfer_to_handler
62	debug_transfer_to_handler:
63	mfspr r0,SPRN_CSRR0
64	stw r0,_CSRR0(r11)
65	mfspr r0,SPRN_CSRR1
66	stw r0,_CSRR1(r11)
67	/* fall through */
68	_ASM_NOKPROBE_SYMBOL(debug_transfer_to_handler)
69
70	.globl crit_transfer_to_handler
71	crit_transfer_to_handler:
72	#ifdef CONFIG_PPC_BOOK3E_MMU
73	mfspr r0,SPRN_MAS0
74	stw r0,MAS0(r11)
75	mfspr r0,SPRN_MAS1
76	stw r0,MAS1(r11)
77	mfspr r0,SPRN_MAS2
78	stw r0,MAS2(r11)
79	mfspr r0,SPRN_MAS3
80	stw r0,MAS3(r11)
81	mfspr r0,SPRN_MAS6
82	stw r0,MAS6(r11)
83	#ifdef CONFIG_PHYS_64BIT
84	mfspr r0,SPRN_MAS7
85	stw r0,MAS7(r11)
86	#endif /* CONFIG_PHYS_64BIT */
87	#endif /* CONFIG_PPC_BOOK3E_MMU */
88	#ifdef CONFIG_44x
89	mfspr r0,SPRN_MMUCR
90	stw r0,MMUCR(r11)
91	#endif
92	mfspr r0,SPRN_SRR0
93	stw r0,_SRR0(r11)
94	mfspr r0,SPRN_SRR1
95	stw r0,_SRR1(r11)
96
97	/* set the stack limit to the current stack */
98	mfspr r8,SPRN_SPRG_THREAD
99	lwz r0,KSP_LIMIT(r8)
100	stw r0,SAVED_KSP_LIMIT(r11)
101	rlwinm r0,r1,0,0,(31 - THREAD_SHIFT)
102	stw r0,KSP_LIMIT(r8)
103	/* fall through */
104	_ASM_NOKPROBE_SYMBOL(crit_transfer_to_handler)
105	#endif
106
107	#ifdef CONFIG_40x
108	.globl crit_transfer_to_handler
109	crit_transfer_to_handler:
110	lwz r0,crit_r10@l(0)
111	stw r0,GPR10(r11)
112	lwz r0,crit_r11@l(0)
113	stw r0,GPR11(r11)
114	mfspr r0,SPRN_SRR0
115	stw r0,crit_srr0@l(0)
116	mfspr r0,SPRN_SRR1
117	stw r0,crit_srr1@l(0)
118
119	/* set the stack limit to the current stack */
120	mfspr r8,SPRN_SPRG_THREAD
121	lwz r0,KSP_LIMIT(r8)
122	stw r0,saved_ksp_limit@l(0)
123	rlwinm r0,r1,0,0,(31 - THREAD_SHIFT)
124	stw r0,KSP_LIMIT(r8)
125	/* fall through */
126	_ASM_NOKPROBE_SYMBOL(crit_transfer_to_handler)
127	#endif
128
129	/*
130	* This code finishes saving the registers to the exception frame
131	* and jumps to the appropriate handler for the exception, turning
132	* on address translation.
133	* Note that we rely on the caller having set cr0.eq iff the exception
134	* occurred in kernel mode (i.e. MSR:PR = 0).
135	*/
136	.globl transfer_to_handler_full
137	transfer_to_handler_full:
138	SAVE_NVGPRS(r11)
139	_ASM_NOKPROBE_SYMBOL(transfer_to_handler_full)
140	/* fall through */
141
142	.globl transfer_to_handler
143	transfer_to_handler:
144	stw r2,GPR2(r11)
145	stw r12,_NIP(r11)
146	stw r9,_MSR(r11)
147	andi. r2,r9,MSR_PR
148	mfctr r12
149	mfspr r2,SPRN_XER
150	stw r12,_CTR(r11)
151	stw r2,_XER(r11)
152	mfspr r12,SPRN_SPRG_THREAD
153	tovirt_vmstack r12, r12
154	beq 2f /* if from user, fix up THREAD.regs */
155	addi r2, r12, -THREAD
156	addi r11,r1,STACK_FRAME_OVERHEAD
157	stw r11,PT_REGS(r12)
158	#if defined(CONFIG_40x) \|\| defined(CONFIG_BOOKE)
159	/* Check to see if the dbcr0 register is set up to debug. Use the
160	internal debug mode bit to do this. */
161	lwz r12,THREAD_DBCR0(r12)
162	andis. r12,r12,DBCR0_IDM@h
163	#endif
164	ACCOUNT_CPU_USER_ENTRY(r2, r11, r12)
165	#ifdef CONFIG_PPC_BOOK3S_32
166	kuep_lock r11, r12
167	#endif
168	#if defined(CONFIG_40x) \|\| defined(CONFIG_BOOKE)
169	beq+ 3f
170	/* From user and task is ptraced - load up global dbcr0 */
171	li r12,-1 /* clear all pending debug events */
172	mtspr SPRN_DBSR,r12
173	lis r11,global_dbcr0@ha
174	tophys(r11,r11)
175	addi r11,r11,global_dbcr0@l
176	#ifdef CONFIG_SMP
177	lwz r9,TASK_CPU(r2)
178	slwi r9,r9,2
179	add r11,r11,r9
180	#endif
181	lwz r12,0(r11)
182	mtspr SPRN_DBCR0,r12
183	#endif
184
185	b 3f
186
187	2: /* if from kernel, check interrupted DOZE/NAP mode and
188	* check for stack overflow
189	*/
190	kuap_save_and_lock r11, r12, r9, r2, r6
191	addi r2, r12, -THREAD
192	#ifndef CONFIG_VMAP_STACK
193	lwz r9,KSP_LIMIT(r12)
194	cmplw r1,r9 /* if r1 <= ksp_limit */
195	ble- stack_ovf /* then the kernel stack overflowed */
196	#endif
197	5:
198	#if defined(CONFIG_PPC_BOOK3S_32) \|\| defined(CONFIG_E500)	51	#if defined(CONFIG_PPC_BOOK3S_32) \|\| defined(CONFIG_E500)
		52	.globl prepare_transfer_to_handler
		53	prepare_transfer_to_handler:
		54	/* if from kernel, check interrupted DOZE/NAP mode */
199	lwz r12,TI_LOCAL_FLAGS(r2)	55	lwz r12,TI_LOCAL_FLAGS(r2)
200	mtcrf 0x01,r12	56	mtcrf 0x01,r12
201	bt- 31-TLF_NAPPING,4f	57	bt- 31-TLF_NAPPING,4f
202	bt- 31-TLF_SLEEPING,7f	58	bt- 31-TLF_SLEEPING,7f
203	#endif /* CONFIG_PPC_BOOK3S_32 \|\| CONFIG_E500 */	59	blr
204	.globl transfer_to_handler_cont
205	transfer_to_handler_cont:
206	3:
207	mflr r9
208	tovirt_novmstack r2, r2 /* set r2 to current */
209	tovirt_vmstack r9, r9
210	lwz r11,0(r9) /* virtual address of handler */
211	lwz r9,4(r9) /* where to go when done */
212	#if defined(CONFIG_PPC_8xx) && defined(CONFIG_PERF_EVENTS)
213	mtspr SPRN_NRI, r0
214	#endif
215	#ifdef CONFIG_TRACE_IRQFLAGS
216	/*
217	* When tracing IRQ state (lockdep) we enable the MMU before we call
218	* the IRQ tracing functions as they might access vmalloc space or
219	* perform IOs for console output.
220	*
221	* To speed up the syscall path where interrupts stay on, let's check
222	* first if we are changing the MSR value at all.
223	*/
224	tophys_novmstack r12, r1
225	lwz r12,_MSR(r12)
226	andi. r12,r12,MSR_EE
227	bne 1f
228
229	/* MSR isn't changing, just transition directly */
230	#endif
231	mtspr SPRN_SRR0,r11
232	mtspr SPRN_SRR1,r10
233	mtlr r9
234	rfi /* jump to handler, enable MMU */
235	#ifdef CONFIG_40x
236	b . /* Prevent prefetch past rfi */
237	#endif
238		60
239	#if defined (CONFIG_PPC_BOOK3S_32) \|\| defined(CONFIG_E500)
240	4: rlwinm r12,r12,0,~_TLF_NAPPING	61	4: rlwinm r12,r12,0,~_TLF_NAPPING
241	stw r12,TI_LOCAL_FLAGS(r2)	62	stw r12,TI_LOCAL_FLAGS(r2)
242	b power_save_ppc32_restore	63	b power_save_ppc32_restore
@@ -246,97 +67,18 @@ transfer_to_handler_cont: · 3 unmodified lines
246	lwz r9,_MSR(r11) /* if sleeping, clear MSR.EE */	67	lwz r9,_MSR(r11) /* if sleeping, clear MSR.EE */
247	rlwinm r9,r9,0,~MSR_EE	68	rlwinm r9,r9,0,~MSR_EE
248	lwz r12,_LINK(r11) /* and return to address in LR */	69	lwz r12,_LINK(r11) /* and return to address in LR */
249	kuap_restore r11, r2, r3, r4, r5
250	lwz r2, GPR2(r11)	70	lwz r2, GPR2(r11)
251	b fast_exception_return	71	b fast_exception_return
252	#endif	72	_ASM_NOKPROBE_SYMBOL(prepare_transfer_to_handler)
253	_ASM_NOKPROBE_SYMBOL(transfer_to_handler)	73	#endif /* CONFIG_PPC_BOOK3S_32 \|\| CONFIG_E500 */
254	_ASM_NOKPROBE_SYMBOL(transfer_to_handler_cont)
255
256	#ifdef CONFIG_TRACE_IRQFLAGS
257	1: /* MSR is changing, re-enable MMU so we can notify lockdep. We need to
258	* keep interrupts disabled at this point otherwise we might risk
259	* taking an interrupt before we tell lockdep they are enabled.
260	*/
261	lis r12,reenable_mmu@h
262	ori r12,r12,reenable_mmu@l
263	LOAD_REG_IMMEDIATE(r0, MSR_KERNEL)
264	mtspr SPRN_SRR0,r12
265	mtspr SPRN_SRR1,r0
266	rfi
267	#ifdef CONFIG_40x
268	b . /* Prevent prefetch past rfi */
269	#endif
270
271	reenable_mmu:
272	/*
273	* We save a bunch of GPRs,
274	* r3 can be different from GPR3(r1) at this point, r9 and r11
275	* contains the old MSR and handler address respectively,
276	* r0, r4-r8, r12, CCR, CTR, XER etc... are left
277	* clobbered as they aren't useful past this point.
278	*/
279
280	stwu r1,-32(r1)
281	stw r9,8(r1)
282	stw r11,12(r1)
283	stw r3,16(r1)
284
285	/* If we are disabling interrupts (normal case), simply log it with
286	* lockdep
287	*/
288	1: bl trace_hardirqs_off
289	lwz r3,16(r1)
290	lwz r11,12(r1)
291	lwz r9,8(r1)
292	addi r1,r1,32
293	mtctr r11
294	mtlr r9
295	bctr /* jump to handler */
296	#endif /* CONFIG_TRACE_IRQFLAGS */
297
298	#ifndef CONFIG_VMAP_STACK
299	/*
300	* On kernel stack overflow, load up an initial stack pointer
301	* and call StackOverflow(regs), which should not return.
302	*/
303	stack_ovf:
304	/* sometimes we use a statically-allocated stack, which is OK. */
305	lis r12,_end@h
306	ori r12,r12,_end@l
307	cmplw r1,r12
308	ble 5b /* r1 <= &_end is OK */
309	SAVE_NVGPRS(r11)
310	addi r3,r1,STACK_FRAME_OVERHEAD
311	lis r1,init_thread_union@ha
312	addi r1,r1,init_thread_union@l
313	addi r1,r1,THREAD_SIZE-STACK_FRAME_OVERHEAD
314	lis r9,StackOverflow@ha
315	addi r9,r9,StackOverflow@l
316	LOAD_REG_IMMEDIATE(r10,MSR_KERNEL)
317	#if defined(CONFIG_PPC_8xx) && defined(CONFIG_PERF_EVENTS)
318	mtspr SPRN_NRI, r0
319	#endif
320	mtspr SPRN_SRR0,r9
321	mtspr SPRN_SRR1,r10
322	rfi
323	#ifdef CONFIG_40x
324	b . /* Prevent prefetch past rfi */
325	#endif
326	_ASM_NOKPROBE_SYMBOL(stack_ovf)
327	#endif
328		74
329	.globl transfer_to_syscall	75	.globl transfer_to_syscall
330	transfer_to_syscall:	76	transfer_to_syscall:
331	SAVE_NVGPRS(r1)	77	SAVE_NVGPRS(r1)
332	#ifdef CONFIG_PPC_BOOK3S_32
333	kuep_lock r11, r12
334	#endif
335		78
336	/* Calling convention has r9 = orig r0, r10 = regs */	79	/* Calling convention has r9 = orig r0, r10 = regs */
337	addi r10,r1,STACK_FRAME_OVERHEAD	80	addi r10,r1,STACK_FRAME_OVERHEAD
338	mr r9,r0	81	mr r9,r0
339	stw r10,THREAD+PT_REGS(r2)
340	bl system_call_exception	82	bl system_call_exception
341		83
342	ret_from_syscall:	84	ret_from_syscall:
@@ -349,10 +91,6 @@ ret_from_syscall: · 6 unmodified lines
349	cmplwi cr0,r5,0	91	cmplwi cr0,r5,0
350	bne- 2f	92	bne- 2f
351	#endif /* CONFIG_PPC_47x */	93	#endif /* CONFIG_PPC_47x */
352	#ifdef CONFIG_PPC_BOOK3S_32
353	kuep_unlock r5, r7
354	#endif
355	kuap_check r2, r4
356	lwz r4,_LINK(r1)	94	lwz r4,_LINK(r1)
357	lwz r5,_CCR(r1)	95	lwz r5,_CCR(r1)
358	mtlr r4	96	mtlr r4
@@ -411,27 +149,6 @@ ret_from_kernel_thread: · 52 unmodified lines
411	li r3,0	149	li r3,0
412	b ret_from_syscall	150	b ret_from_syscall
413		151
414	/*
415	* Top-level page fault handling.
416	* This is in assembler because if do_page_fault tells us that
417	* it is a bad kernel page fault, we want to save the non-volatile
418	* registers before calling bad_page_fault.
419	*/
420	.globl handle_page_fault
421	handle_page_fault:
422	addi r3,r1,STACK_FRAME_OVERHEAD
423	bl do_page_fault
424	cmpwi r3,0
425	beq+ ret_from_except
426	SAVE_NVGPRS(r1)
427	lwz r0,_TRAP(r1)
428	clrrwi r0,r0,1
429	stw r0,_TRAP(r1)
430	mr r4,r3 /* err arg for bad_page_fault */
431	addi r3,r1,STACK_FRAME_OVERHEAD
432	bl __bad_page_fault
433	b ret_from_except_full
434
435	/*	152	/*
436	* This routine switches between two different tasks. The process	153	* This routine switches between two different tasks. The process
437	* state of one is saved on its kernel stack. Then the state	154	* state of one is saved on its kernel stack. Then the state
@@ -485,7 +202,6 @@ END_FTR_SECTION_IFSET(CPU_FTR_SPE) · 47 unmodified lines
485	stw r10,_CCR(r1)	202	stw r10,_CCR(r1)
486	stw r1,KSP(r3) /* Set old stack pointer */	203	stw r1,KSP(r3) /* Set old stack pointer */
487		204
488	kuap_check r2, r0
489	#ifdef CONFIG_SMP	205	#ifdef CONFIG_SMP
490	/* We need a sync somewhere here to make sure that if the	206	/* We need a sync somewhere here to make sure that if the
491	* previous task gets rescheduled on another CPU, it sees all	207	* previous task gets rescheduled on another CPU, it sees all
@@ -529,12 +245,12 @@ END_FTR_SECTION_IFSET(CPU_FTR_SPE) · 37 unmodified lines
529	fast_exception_return:	245	fast_exception_return:
530	#if !(defined(CONFIG_4xx) \|\| defined(CONFIG_BOOKE))	246	#if !(defined(CONFIG_4xx) \|\| defined(CONFIG_BOOKE))
531	andi. r10,r9,MSR_RI /* check for recoverable interrupt */	247	andi. r10,r9,MSR_RI /* check for recoverable interrupt */
532	beq 1f /* if not, we've got problems */	248	beq 3f /* if not, we've got problems */
533	#endif	249	#endif
534		250
535	2: REST_4GPRS(3, r11)	251	2: REST_4GPRS(3, r11)
536	lwz r10,_CCR(r11)	252	lwz r10,_CCR(r11)
537	REST_GPR(1, r11)	253	REST_2GPRS(1, r11)
538	mtcr r10	254	mtcr r10
539	lwz r10,_LINK(r11)	255	lwz r10,_LINK(r11)
540	mtlr r10	256	mtlr r10
@@ -556,257 +272,147 @@ fast_exception_return: · 15 unmodified lines
556	#endif	272	#endif
557	_ASM_NOKPROBE_SYMBOL(fast_exception_return)	273	_ASM_NOKPROBE_SYMBOL(fast_exception_return)
558		274
559	#if !(defined(CONFIG_4xx) \|\| defined(CONFIG_BOOKE))
560	/* check if the exception happened in a restartable section */
561	1: lis r3,exc_exit_restart_end@ha
562	addi r3,r3,exc_exit_restart_end@l
563	cmplw r12,r3
564	bge 3f
565	lis r4,exc_exit_restart@ha
566	addi r4,r4,exc_exit_restart@l
567	cmplw r12,r4
568	blt 3f
569	lis r3,fee_restarts@ha
570	tophys(r3,r3)
571	lwz r5,fee_restarts@l(r3)
572	addi r5,r5,1
573	stw r5,fee_restarts@l(r3)
574	mr r12,r4 /* restart at exc_exit_restart */
575	b 2b
576
577	.section .bss
578	.align 2
579	fee_restarts:
580	.space 4
581	.previous
582
583	/* aargh, a nonrecoverable interrupt, panic */	275	/* aargh, a nonrecoverable interrupt, panic */
584	/* aargh, we don't know which trap this is */	276	/* aargh, we don't know which trap this is */
585	3:	277	3:
586	li r10,-1	278	li r10,-1
587	stw r10,_TRAP(r11)	279	stw r10,_TRAP(r11)
588	addi r3,r1,STACK_FRAME_OVERHEAD	280	prepare_transfer_to_handler
589	lis r10,MSR_KERNEL@h	281	bl unrecoverable_exception
590	ori r10,r10,MSR_KERNEL@l	282	trap /* should not get here */
591	bl transfer_to_handler_full
592	.long unrecoverable_exception
593	.long ret_from_except
594	#endif
595
596	.globl ret_from_except_full
597	ret_from_except_full:
598	REST_NVGPRS(r1)
599	/* fall through */
600
601	.globl ret_from_except
602	ret_from_except:
603	/* Hard-disable interrupts so that current_thread_info()->flags
604	* can't change between when we test it and when we return
605	* from the interrupt. */
606	/* Note: We don't bother telling lockdep about it */
607	LOAD_REG_IMMEDIATE(r10,MSR_KERNEL)
608	mtmsr r10 /* disable interrupts */
609
610	lwz r3,_MSR(r1) /* Returning to user mode? */
611	andi. r0,r3,MSR_PR
612	beq resume_kernel
613
614	user_exc_return: /* r10 contains MSR_KERNEL here */
615	/* Check current_thread_info()->flags */
616	lwz r9,TI_FLAGS(r2)
617	andi. r0,r9,_TIF_USER_WORK_MASK
618	bne do_work
619		283
620	restore_user:	284	.globl interrupt_return
621	#if defined(CONFIG_4xx) \|\| defined(CONFIG_BOOKE)	285	interrupt_return:
622	/* Check whether this process has its own DBCR0 value. The internal	286	lwz r4,_MSR(r1)
623	debug mode bit tells us that dbcr0 should be loaded. */	287	addi r3,r1,STACK_FRAME_OVERHEAD
624	lwz r0,THREAD+THREAD_DBCR0(r2)	288	andi. r0,r4,MSR_PR
625	andis. r10,r0,DBCR0_IDM@h	289	beq .Lkernel_interrupt_return
626	bnel- load_dbcr0	290	bl interrupt_exit_user_prepare
627	#endif	291	cmpwi r3,0
628	ACCOUNT_CPU_USER_EXIT(r2, r10, r11)	292	bne- .Lrestore_nvgprs
629	#ifdef CONFIG_PPC_BOOK3S_32
630	kuep_unlock r10, r11
631	#endif
632		293
633	b restore	294	.Lfast_user_interrupt_return:
		295	lwz r11,_NIP(r1)
		296	lwz r12,_MSR(r1)
		297	mtspr SPRN_SRR0,r11
		298	mtspr SPRN_SRR1,r12
634		299
635	/* N.B. the only way to get here is from the beq following ret_from_except. */	300	BEGIN_FTR_SECTION
636	resume_kernel:	301	stwcx. r0,0,r1 /* to clear the reservation */
637	/* check current_thread_info, _TIF_EMULATE_STACK_STORE */	302	FTR_SECTION_ELSE
638	lwz r8,TI_FLAGS(r2)	303	lwarx r0,0,r1
639	andis. r0,r8,_TIF_EMULATE_STACK_STORE@h	304	ALT_FTR_SECTION_END_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
640	beq+ 1f
641		305
642	addi r8,r1,INT_FRAME_SIZE /* Get the kprobed function entry */	306	lwz r3,_CCR(r1)
		307	lwz r4,_LINK(r1)
		308	lwz r5,_CTR(r1)
		309	lwz r6,_XER(r1)
		310	li r0,0
643		311
644	lwz r3,GPR1(r1)	312	/*
645	subi r3,r3,INT_FRAME_SIZE /* dst: Allocate a trampoline exception frame */	313	* Leaving a stale exception_marker on the stack can confuse
646	mr r4,r1 /* src: current exception frame */	314	* the reliable stack unwinder later on. Clear it.
647	mr r1,r3 /* Reroute the trampoline frame to r1 */	315	*/
		316	stw r0,8(r1)
		317	REST_4GPRS(7, r1)
		318	REST_2GPRS(11, r1)
648		319
649	/* Copy from the original to the trampoline. */	320	mtcr r3
650	li r5,INT_FRAME_SIZE/4 /* size: INT_FRAME_SIZE */	321	mtlr r4
651	li r6,0 /* start offset: 0 */
652	mtctr r5	322	mtctr r5
653	2: lwzx r0,r6,r4	323	mtspr SPRN_XER,r6
654	stwx r0,r6,r3
655	addi r6,r6,4
656	bdnz 2b
657
658	/* Do real store operation to complete stwu */
659	lwz r5,GPR1(r1)
660	stw r8,0(r5)
661		324
662	/* Clear _TIF_EMULATE_STACK_STORE flag */	325	REST_4GPRS(2, r1)
663	lis r11,_TIF_EMULATE_STACK_STORE@h	326	REST_GPR(6, r1)
664	addi r5,r2,TI_FLAGS	327	REST_GPR(0, r1)
665	0: lwarx r8,0,r5	328	REST_GPR(1, r1)
666	andc r8,r8,r11	329	rfi
667	stwcx. r8,0,r5	330	#ifdef CONFIG_40x
668	bne- 0b	331	b . /* Prevent prefetch past rfi */
669	1:
670
671	#ifdef CONFIG_PREEMPTION
672	/* check current_thread_info->preempt_count */
673	lwz r0,TI_PREEMPT(r2)
674	cmpwi 0,r0,0 /* if non-zero, just restore regs and return */
675	bne restore_kuap
676	andi. r8,r8,_TIF_NEED_RESCHED
677	beq+ restore_kuap
678	lwz r3,_MSR(r1)
679	andi. r0,r3,MSR_EE /* interrupts off? */
680	beq restore_kuap /* don't schedule if so */
681	#ifdef CONFIG_TRACE_IRQFLAGS
682	/* Lockdep thinks irqs are enabled, we need to call
683	* preempt_schedule_irq with IRQs off, so we inform lockdep
684	* now that we -did- turn them off already
685	*/
686	bl trace_hardirqs_off
687	#endif
688	bl preempt_schedule_irq
689	#ifdef CONFIG_TRACE_IRQFLAGS
690	/* And now, to properly rebalance the above, we tell lockdep they
691	* are being turned back on, which will happen when we return
692	*/
693	bl trace_hardirqs_on
694	#endif	332	#endif
695	#endif /* CONFIG_PREEMPTION */
696	restore_kuap:
697	kuap_restore r1, r2, r9, r10, r0
698
699	/* interrupts are hard-disabled at this point */
700	restore:
701	#if defined(CONFIG_44x) && !defined(CONFIG_PPC_47x)
702	lis r4,icache_44x_need_flush@ha
703	lwz r5,icache_44x_need_flush@l(r4)
704	cmplwi cr0,r5,0
705	beq+ 1f
706	li r6,0
707	iccci r0,r0
708	stw r6,icache_44x_need_flush@l(r4)
709	1:
710	#endif /* CONFIG_44x */
711		333
712	lwz r9,_MSR(r1)	334	.Lrestore_nvgprs:
713	#ifdef CONFIG_TRACE_IRQFLAGS	335	REST_NVGPRS(r1)
714	/* Lockdep doesn't know about the fact that IRQs are temporarily turned	336	b .Lfast_user_interrupt_return
715	* off in this assembly code while peeking at TI_FLAGS() and such. However
716	* we need to inform it if the exception turned interrupts off, and we
717	* are about to trun them back on.
718	*/
719	andi. r10,r9,MSR_EE
720	beq 1f
721	stwu r1,-32(r1)
722	mflr r0
723	stw r0,4(r1)
724	bl trace_hardirqs_on
725	addi r1, r1, 32
726	lwz r9,_MSR(r1)
727	1:
728	#endif /* CONFIG_TRACE_IRQFLAGS */
729		337
730	lwz r0,GPR0(r1)	338	.Lkernel_interrupt_return:
731	lwz r2,GPR2(r1)	339	bl interrupt_exit_kernel_prepare
732	REST_4GPRS(3, r1)
733	REST_2GPRS(7, r1)
734		340
735	lwz r10,_XER(r1)	341	.Lfast_kernel_interrupt_return:
736	lwz r11,_CTR(r1)	342	cmpwi cr1,r3,0
737	mtspr SPRN_XER,r10	343	lwz r11,_NIP(r1)
738	mtctr r11	344	lwz r12,_MSR(r1)
		345	mtspr SPRN_SRR0,r11
		346	mtspr SPRN_SRR1,r12
739		347
740	BEGIN_FTR_SECTION	348	BEGIN_FTR_SECTION
741	lwarx r11,0,r1	349	stwcx. r0,0,r1 /* to clear the reservation */
742	END_FTR_SECTION_IFSET(CPU_FTR_NEED_PAIRED_STWCX)	350	FTR_SECTION_ELSE
743	stwcx. r0,0,r1 /* to clear the reservation */	351	lwarx r0,0,r1
		352	ALT_FTR_SECTION_END_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
744		353
745	#if !(defined(CONFIG_4xx) \|\| defined(CONFIG_BOOKE))	354	lwz r3,_LINK(r1)
746	andi. r10,r9,MSR_RI /* check if this exception occurred */	355	lwz r4,_CTR(r1)
747	beql nonrecoverable /* at a bad place (MSR:RI = 0) */	356	lwz r5,_XER(r1)
		357	lwz r6,_CCR(r1)
		358	li r0,0
		359
		360	REST_4GPRS(7, r1)
		361	REST_2GPRS(11, r1)
748		362
749	lwz r10,_CCR(r1)	363	mtlr r3
750	lwz r11,_LINK(r1)	364	mtctr r4
751	mtcrf 0xFF,r10	365	mtspr SPRN_XER,r5
752	mtlr r11
753		366
754	/* Clear the exception_marker on the stack to avoid confusing stacktrace */
755	li r10, 0
756	stw r10, 8(r1)
757	/*	367	/*
758	* Once we put values in SRR0 and SRR1, we are in a state	368	* Leaving a stale exception_marker on the stack can confuse
759	* where exceptions are not recoverable, since taking an	369	* the reliable stack unwinder later on. Clear it.
760	* exception will trash SRR0 and SRR1. Therefore we clear the
761	* MSR:RI bit to indicate this. If we do take an exception,
762	* we can't return to the point of the exception but we
763	* can restart the exception exit path at the label
764	* exc_exit_restart below. -- paulus
765	*/	370	*/
766	LOAD_REG_IMMEDIATE(r10,MSR_KERNEL & ~MSR_RI)	371	stw r0,8(r1)
767	mtmsr r10 /* clear the RI bit */	372
768	.globl exc_exit_restart	373	REST_4GPRS(2, r1)
769	exc_exit_restart:	374
770	lwz r12,_NIP(r1)	375	bne- cr1,1f /* emulate stack store */
771	mtspr SPRN_SRR0,r12	376	mtcr r6
772	mtspr SPRN_SRR1,r9	377	REST_GPR(6, r1)
773	REST_4GPRS(9, r1)	378	REST_GPR(0, r1)
774	lwz r1,GPR1(r1)	379	REST_GPR(1, r1)
775	.globl exc_exit_restart_end
776	exc_exit_restart_end:
777	rfi	380	rfi
778	_ASM_NOKPROBE_SYMBOL(exc_exit_restart)	381	#ifdef CONFIG_40x
779	_ASM_NOKPROBE_SYMBOL(exc_exit_restart_end)	382	b . /* Prevent prefetch past rfi */
		383	#endif
780		384
781	#else /* !(CONFIG_4xx \|\| CONFIG_BOOKE) */	385	1: /*
782	/*	386	* Emulate stack store with update. New r1 value was already calculated
783	* This is a bit different on 4xx/Book-E because it doesn't have	387	* and updated in our interrupt regs by emulate_loadstore, but we can't
784	* the RI bit in the MSR.	388	* store the previous value of r1 to the stack before re-loading our
785	* The TLB miss handler checks if we have interrupted	389	* registers from it, otherwise they could be clobbered. Use
786	* the exception exit path and restarts it if so	390	* SPRG Scratch0 as temporary storage to hold the store
787	* (well maybe one day it will... :).	391	* data, as interrupts are disabled here so it won't be clobbered.
788	*/	392	*/
789	lwz r11,_LINK(r1)	393	mtcr r6
790	mtlr r11	394	#ifdef CONFIG_BOOKE
791	lwz r10,_CCR(r1)	395	mtspr SPRN_SPRG_WSCRATCH0, r9
792	mtcrf 0xff,r10	396	#else
793	/* Clear the exception_marker on the stack to avoid confusing stacktrace */	397	mtspr SPRN_SPRG_SCRATCH0, r9
794	li r10, 0	398	#endif
795	stw r10, 8(r1)	399	addi r9,r1,INT_FRAME_SIZE /* get original r1 */
796	REST_2GPRS(9, r1)	400	REST_GPR(6, r1)
797	.globl exc_exit_restart	401	REST_GPR(0, r1)
798	exc_exit_restart:	402	REST_GPR(1, r1)
799	lwz r11,_NIP(r1)	403	stw r9,0(r1) /* perform store component of stwu */
800	lwz r12,_MSR(r1)	404	#ifdef CONFIG_BOOKE
801	mtspr SPRN_SRR0,r11	405	mfspr r9, SPRN_SPRG_RSCRATCH0
802	mtspr SPRN_SRR1,r12	406	#else
803	REST_2GPRS(11, r1)	407	mfspr r9, SPRN_SPRG_SCRATCH0
804	lwz r1,GPR1(r1)	408	#endif
805	.globl exc_exit_restart_end
806	exc_exit_restart_end:
807	rfi	409	rfi
808	b . /* prevent prefetch past rfi */	410	#ifdef CONFIG_40x
809	_ASM_NOKPROBE_SYMBOL(exc_exit_restart)	411	b . /* Prevent prefetch past rfi */
		412	#endif
		413	_ASM_NOKPROBE_SYMBOL(interrupt_return)
		414
		415	#if defined(CONFIG_4xx) \|\| defined(CONFIG_BOOKE)
810		416
811	/*	417	/*
812	* Returning from a critical interrupt in user mode doesn't need	418	* Returning from a critical interrupt in user mode doesn't need
@@ -837,8 +443,7 @@ _ASM_NOKPROBE_SYMBOL(exc_exit_restart) · 24 unmodified lines
837	REST_NVGPRS(r1); \	443	REST_NVGPRS(r1); \
838	lwz r3,_MSR(r1); \	444	lwz r3,_MSR(r1); \
839	andi. r3,r3,MSR_PR; \	445	andi. r3,r3,MSR_PR; \
840	LOAD_REG_IMMEDIATE(r10,MSR_KERNEL); \	446	bne interrupt_return; \
841	bne user_exc_return; \
842	lwz r0,GPR0(r1); \	447	lwz r0,GPR0(r1); \
843	lwz r2,GPR2(r1); \	448	lwz r2,GPR2(r1); \
844	REST_4GPRS(3, r1); \	449	REST_4GPRS(3, r1); \
@@ -906,11 +511,6 @@ _ASM_NOKPROBE_SYMBOL(exc_exit_restart) · 61 unmodified lines
906	#ifdef CONFIG_40x	511	#ifdef CONFIG_40x
907	.globl ret_from_crit_exc	512	.globl ret_from_crit_exc
908	ret_from_crit_exc:	513	ret_from_crit_exc:
909	mfspr r9,SPRN_SPRG_THREAD
910	lis r10,saved_ksp_limit@ha;
911	lwz r10,saved_ksp_limit@l(r10);
912	tovirt(r9,r9);
913	stw r10,KSP_LIMIT(r9)
914	lis r9,crit_srr0@ha;	514	lis r9,crit_srr0@ha;
915	lwz r9,crit_srr0@l(r9);	515	lwz r9,crit_srr0@l(r9);
916	lis r10,crit_srr1@ha;	516	lis r10,crit_srr1@ha;
@@ -924,9 +524,6 @@ _ASM_NOKPROBE_SYMBOL(ret_from_crit_exc) · 7 unmodified lines
924	#ifdef CONFIG_BOOKE	524	#ifdef CONFIG_BOOKE
925	.globl ret_from_crit_exc	525	.globl ret_from_crit_exc
926	ret_from_crit_exc:	526	ret_from_crit_exc:
927	mfspr r9,SPRN_SPRG_THREAD
928	lwz r10,SAVED_KSP_LIMIT(r1)
929	stw r10,KSP_LIMIT(r9)
930	RESTORE_xSRR(SRR0,SRR1);	527	RESTORE_xSRR(SRR0,SRR1);
931	RESTORE_MMU_REGS;	528	RESTORE_MMU_REGS;
932	RET_FROM_EXC_LEVEL(SPRN_CSRR0, SPRN_CSRR1, PPC_RFCI)	529	RET_FROM_EXC_LEVEL(SPRN_CSRR0, SPRN_CSRR1, PPC_RFCI)
@@ -934,9 +531,6 @@ _ASM_NOKPROBE_SYMBOL(ret_from_crit_exc) · 1 unmodified line
934		531
935	.globl ret_from_debug_exc	532	.globl ret_from_debug_exc
936	ret_from_debug_exc:	533	ret_from_debug_exc:
937	mfspr r9,SPRN_SPRG_THREAD
938	lwz r10,SAVED_KSP_LIMIT(r1)
939	stw r10,KSP_LIMIT(r9)
940	RESTORE_xSRR(SRR0,SRR1);	534	RESTORE_xSRR(SRR0,SRR1);
941	RESTORE_xSRR(CSRR0,CSRR1);	535	RESTORE_xSRR(CSRR0,CSRR1);
942	RESTORE_MMU_REGS;	536	RESTORE_MMU_REGS;
@@ -945,9 +539,6 @@ _ASM_NOKPROBE_SYMBOL(ret_from_debug_exc) · 2 unmodified lines
945		539
946	.globl ret_from_mcheck_exc	540	.globl ret_from_mcheck_exc
947	ret_from_mcheck_exc:	541	ret_from_mcheck_exc:
948	mfspr r9,SPRN_SPRG_THREAD
949	lwz r10,SAVED_KSP_LIMIT(r1)
950	stw r10,KSP_LIMIT(r9)
951	RESTORE_xSRR(SRR0,SRR1);	542	RESTORE_xSRR(SRR0,SRR1);
952	RESTORE_xSRR(CSRR0,CSRR1);	543	RESTORE_xSRR(CSRR0,CSRR1);
953	RESTORE_xSRR(DSRR0,DSRR1);	544	RESTORE_xSRR(DSRR0,DSRR1);
@@ -955,121 +546,8 @@ ret_from_mcheck_exc: · 1 unmodified line
955	RET_FROM_EXC_LEVEL(SPRN_MCSRR0, SPRN_MCSRR1, PPC_RFMCI)	546	RET_FROM_EXC_LEVEL(SPRN_MCSRR0, SPRN_MCSRR1, PPC_RFMCI)
956	_ASM_NOKPROBE_SYMBOL(ret_from_mcheck_exc)	547	_ASM_NOKPROBE_SYMBOL(ret_from_mcheck_exc)
957	#endif /* CONFIG_BOOKE */	548	#endif /* CONFIG_BOOKE */
958
959	/*
960	* Load the DBCR0 value for a task that is being ptraced,
961	* having first saved away the global DBCR0. Note that r0
962	* has the dbcr0 value to set upon entry to this.
963	*/
964	load_dbcr0:
965	mfmsr r10 /* first disable debug exceptions */
966	rlwinm r10,r10,0,~MSR_DE
967	mtmsr r10
968	isync
969	mfspr r10,SPRN_DBCR0
970	lis r11,global_dbcr0@ha
971	addi r11,r11,global_dbcr0@l
972	#ifdef CONFIG_SMP
973	lwz r9,TASK_CPU(r2)
974	slwi r9,r9,2
975	add r11,r11,r9
976	#endif
977	stw r10,0(r11)
978	mtspr SPRN_DBCR0,r0
979	li r11,-1
980	mtspr SPRN_DBSR,r11 /* clear all pending debug events */
981	blr
982
983	.section .bss
984	.align 4
985	.global global_dbcr0
986	global_dbcr0:
987	.space 4*NR_CPUS
988	.previous
989	#endif /* !(CONFIG_4xx \|\| CONFIG_BOOKE) */	549	#endif /* !(CONFIG_4xx \|\| CONFIG_BOOKE) */
990		550
991	do_work: /* r10 contains MSR_KERNEL here */
992	andi. r0,r9,_TIF_NEED_RESCHED
993	beq do_user_signal
994
995	do_resched: /* r10 contains MSR_KERNEL here */
996	#ifdef CONFIG_TRACE_IRQFLAGS
997	bl trace_hardirqs_on
998	mfmsr r10
999	#endif
1000	ori r10,r10,MSR_EE
1001	mtmsr r10 /* hard-enable interrupts */
1002	bl schedule
1003	recheck:
1004	/* Note: And we don't tell it we are disabling them again
1005	* neither. Those disable/enable cycles used to peek at
1006	* TI_FLAGS aren't advertised.
1007	*/
1008	LOAD_REG_IMMEDIATE(r10,MSR_KERNEL)
1009	mtmsr r10 /* disable interrupts */
1010	lwz r9,TI_FLAGS(r2)
1011	andi. r0,r9,_TIF_NEED_RESCHED
1012	bne- do_resched
1013	andi. r0,r9,_TIF_USER_WORK_MASK
1014	beq restore_user
1015	do_user_signal: /* r10 contains MSR_KERNEL here */
1016	ori r10,r10,MSR_EE
1017	mtmsr r10 /* hard-enable interrupts */
1018	/* save r13-r31 in the exception frame, if not already done */
1019	lwz r3,_TRAP(r1)
1020	andi. r0,r3,1
1021	beq 2f
1022	SAVE_NVGPRS(r1)
1023	rlwinm r3,r3,0,0,30
1024	stw r3,_TRAP(r1)
1025	2: addi r3,r1,STACK_FRAME_OVERHEAD
1026	mr r4,r9
1027	bl do_notify_resume
1028	REST_NVGPRS(r1)
1029	b recheck
1030
1031	/*
1032	* We come here when we are at the end of handling an exception
1033	* that occurred at a place where taking an exception will lose
1034	* state information, such as the contents of SRR0 and SRR1.
1035	*/
1036	nonrecoverable:
1037	lis r10,exc_exit_restart_end@ha
1038	addi r10,r10,exc_exit_restart_end@l
1039	cmplw r12,r10
1040	bge 3f
1041	lis r11,exc_exit_restart@ha
1042	addi r11,r11,exc_exit_restart@l
1043	cmplw r12,r11
1044	blt 3f
1045	lis r10,ee_restarts@ha
1046	lwz r12,ee_restarts@l(r10)
1047	addi r12,r12,1
1048	stw r12,ee_restarts@l(r10)
1049	mr r12,r11 /* restart at exc_exit_restart */
1050	blr
1051	3: /* OK, we can't recover, kill this process */
1052	lwz r3,_TRAP(r1)
1053	andi. r0,r3,1
1054	beq 5f
1055	SAVE_NVGPRS(r1)
1056	rlwinm r3,r3,0,0,30
1057	stw r3,_TRAP(r1)
1058	5: mfspr r2,SPRN_SPRG_THREAD
1059	addi r2,r2,-THREAD
1060	tovirt(r2,r2) /* set back r2 to current */
1061	4: addi r3,r1,STACK_FRAME_OVERHEAD
1062	bl unrecoverable_exception
1063	/* shouldn't return */
1064	b 4b
1065	_ASM_NOKPROBE_SYMBOL(nonrecoverable)
1066
1067	.section .bss
1068	.align 2
1069	ee_restarts:
1070	.space 4
1071	.previous
1072
1073	/*	551	/*
1074	* PROM code for specific machines follows. Put it	552	* PROM code for specific machines follows. Put it
1075	* here so it's easy to add arch-specific sections later.	553	* here so it's easy to add arch-specific sections later.
@@ -1088,7 +566,6 @@ _GLOBAL(enter_rtas) · 12 unmodified lines
1088	lis r6,1f@ha /* physical return address for rtas */	566	lis r6,1f@ha /* physical return address for rtas */
1089	addi r6,r6,1f@l	567	addi r6,r6,1f@l
1090	tophys(r6,r6)	568	tophys(r6,r6)
1091	tophys_novmstack r7, r1
1092	lwz r8,RTASENTRY(r4)	569	lwz r8,RTASENTRY(r4)
1093	lwz r4,RTASBASE(r4)	570	lwz r4,RTASBASE(r4)
1094	mfmsr r9	571	mfmsr r9
@@ -1097,24 +574,25 @@ _GLOBAL(enter_rtas) · 2 unmodified lines
1097	mtmsr r0 /* disable interrupts so SRR0/1 don't get trashed */	574	mtmsr r0 /* disable interrupts so SRR0/1 don't get trashed */
1098	li r9,MSR_KERNEL & ~(MSR_IR\|MSR_DR)	575	li r9,MSR_KERNEL & ~(MSR_IR\|MSR_DR)
1099	mtlr r6	576	mtlr r6
1100	stw r7, THREAD + RTAS_SP(r2)	577	stw r1, THREAD + RTAS_SP(r2)
1101	mtspr SPRN_SRR0,r8	578	mtspr SPRN_SRR0,r8
1102	mtspr SPRN_SRR1,r9	579	mtspr SPRN_SRR1,r9
1103	rfi	580	rfi
1104	1: tophys_novmstack r9, r1	581	1:
1105	#ifdef CONFIG_VMAP_STACK	582	lis r8, 1f@h
1106	li r0, MSR_KERNEL & ~MSR_IR /* can take DTLB miss */	583	ori r8, r8, 1f@l
1107	mtmsr r0	584	LOAD_REG_IMMEDIATE(r9,MSR_KERNEL)
1108	isync
1109	#endif
1110	lwz r8,INT_FRAME_SIZE+4(r9) /* get return address */
1111	lwz r9,8(r9) /* original msr value */
1112	addi r1,r1,INT_FRAME_SIZE
1113	li r0,0
1114	tophys_novmstack r7, r2
1115	stw r0, THREAD + RTAS_SP(r7)
1116	mtspr SPRN_SRR0,r8	585	mtspr SPRN_SRR0,r8
1117	mtspr SPRN_SRR1,r9	586	mtspr SPRN_SRR1,r9
1118	rfi /* return to caller */	587	rfi /* Reactivate MMU translation */
		588	1:
		589	lwz r8,INT_FRAME_SIZE+4(r1) /* get return address */
		590	lwz r9,8(r1) /* original msr value */
		591	addi r1,r1,INT_FRAME_SIZE
		592	li r0,0
		593	stw r0, THREAD + RTAS_SP(r2)
		594	mtlr r8
		595	mtmsr r9
		596	blr /* return to caller */
1119	_ASM_NOKPROBE_SYMBOL(enter_rtas)	597	_ASM_NOKPROBE_SYMBOL(enter_rtas)
1120	#endif /* CONFIG_PPC_RTAS */	598	#endif /* CONFIG_PPC_RTAS */

arch/powerpc/kernel/entry_64.S+15−25

 END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
 	/*
-	 * RECONCILE_IRQ_STATE without calling trace_hardirqs_off(), which
+	 * scv enters with MSR[EE]=1 and is immediately considered soft-masked.
-	 * would clobber syscall parameters. Also we always enter with IRQs
+	 * The entry vector already sets PACAIRQSOFTMASK to IRQS_ALL_DISABLED,
-	 * enabled and nothing pending. system_call_exception() will call
+	 * and interrupts may be masked and pending already.
-	 * trace_hardirqs_off().
+	 * system_call_exception() will call trace_hardirqs_off() which means
-	 *
+	 * interrupts could already have been blocked before trace_hardirqs_off,
-	 * scv enters with MSR[EE]=1, so don't set PACA_IRQ_HARD_DIS. The
+	 * but this is the best we can do.
-	 * entry vector already sets PACAIRQSOFTMASK to IRQS_ALL_DISABLED.
 	 */
 	/* Calling convention has r9 = orig r0, r10 = regs */
 	std	r11,-16(r10)		/* "regshere" marker */
 	/*
-	 * RECONCILE_IRQ_STATE without calling trace_hardirqs_off(), which
+	 * We always enter kernel from userspace with irq soft-mask enabled and
-	 * would clobber syscall parameters. Also we always enter with IRQs
+	 * nothing pending. system_call_exception() will call
-	 * enabled and nothing pending. system_call_exception() will call
 	 * trace_hardirqs_off().
 	 */
 	li	r11,IRQS_ALL_DISABLED
 	li	r3,0
 	b	.Lsyscall_exit
-#ifdef CONFIG_PPC_BOOK3E
-/* Save non-volatile GPRs, if not already saved. */
-_GLOBAL(save_nvgprs)
-	ld	r11,_TRAP(r1)
-	andi.	r0,r11,1
-	beqlr-
-	SAVE_NVGPRS(r1)
-	clrrdi	r0,r11,1
-	std	r0,_TRAP(r1)
-	blr
-_ASM_NOKPROBE_SYMBOL(save_nvgprs);
-#endif
 #ifdef CONFIG_PPC_BOOK3S_64
 #define FLUSH_COUNT_CACHE	\
 	addi	r1,r1,SWITCH_FRAME_SIZE
 	blr
-#ifdef CONFIG_PPC_BOOK3S
 	/*
 	 * If MSR EE/RI was never enabled, IRQs not reconciled, NVGPRs not
 	 * touched, no exit work created, then this can be used.
 	kuap_check_amr r3, r4
 	ld	r5,_MSR(r1)
 	andi.	r0,r5,MSR_PR
+#ifdef CONFIG_PPC_BOOK3S
 	bne	.Lfast_user_interrupt_return_amr
 	kuap_kernel_restore r3, r4
 	andi.	r0,r5,MSR_RI
 	addi	r3,r1,STACK_FRAME_OVERHEAD
 	bl	unrecoverable_exception
 	b	. /* should not get here */
+#else
+	bne	.Lfast_user_interrupt_return
+	b	.Lfast_kernel_interrupt_return
+#endif
 	.balign IFETCH_ALIGN_BYTES
 	.globl interrupt_return
 	cmpdi	r3,0
 	bne-	.Lrestore_nvgprs
+#ifdef CONFIG_PPC_BOOK3S
 .Lfast_user_interrupt_return_amr:
 	kuap_user_restore r3, r4
+#endif
 .Lfast_user_interrupt_return:
 	ld	r11,_NIP(r1)
 	ld	r12,_MSR(r1)
 	RFI_TO_KERNEL
 	b	.	/* prevent speculative execution */
-#endif /* CONFIG_PPC_BOOK3S */
 #ifdef CONFIG_PPC_RTAS
 /*

arch/powerpc/kernel/exceptions-64e.S+31−394

@@ -63,9 +63,6 @@ · 62 unmodified lines
63	ld reg, (SPECIAL_EXC_##name * 8 + SPECIAL_EXC_FRAME_OFFS)(r1)	63	ld reg, (SPECIAL_EXC_##name * 8 + SPECIAL_EXC_FRAME_OFFS)(r1)
64		64
65	special_reg_save:	65	special_reg_save:
66	lbz r9,PACAIRQHAPPENED(r13)
67	RECONCILE_IRQ_STATE(r3,r4)
68
69	/*	66	/*
70	* We only need (or have stack space) to save this stuff if	67	* We only need (or have stack space) to save this stuff if
71	* we interrupted the kernel.	68	* we interrupted the kernel.
@@ -119,15 +116,11 @@ BEGIN_FTR_SECTION · 47 unmodified lines
119	mtspr SPRN_MAS5,r10	116	mtspr SPRN_MAS5,r10
120	mtspr SPRN_MAS8,r10	117	mtspr SPRN_MAS8,r10
121	END_FTR_SECTION_IFSET(CPU_FTR_EMB_HV)	118	END_FTR_SECTION_IFSET(CPU_FTR_EMB_HV)
122	SPECIAL_EXC_STORE(r9,IRQHAPPENED)
123
124	mfspr r10,SPRN_DEAR	119	mfspr r10,SPRN_DEAR
125	SPECIAL_EXC_STORE(r10,DEAR)	120	SPECIAL_EXC_STORE(r10,DEAR)
126	mfspr r10,SPRN_ESR	121	mfspr r10,SPRN_ESR
127	SPECIAL_EXC_STORE(r10,ESR)	122	SPECIAL_EXC_STORE(r10,ESR)
128		123
129	lbz r10,PACAIRQSOFTMASK(r13)
130	SPECIAL_EXC_STORE(r10,SOFTE)
131	ld r10,_NIP(r1)	124	ld r10,_NIP(r1)
132	SPECIAL_EXC_STORE(r10,CSRR0)	125	SPECIAL_EXC_STORE(r10,CSRR0)
133	ld r10,_MSR(r1)	126	ld r10,_MSR(r1)
@@ -139,7 +132,8 @@ ret_from_level_except: · 5 unmodified lines
139	ld r3,_MSR(r1)	132	ld r3,_MSR(r1)
140	andi. r3,r3,MSR_PR	133	andi. r3,r3,MSR_PR
141	beq 1f	134	beq 1f
142	b ret_from_except	135	REST_NVGPRS(r1)
		136	b interrupt_return
143	1:	137	1:
144		138
145	LOAD_REG_ADDR(r11,extlb_level_exc)	139	LOAD_REG_ADDR(r11,extlb_level_exc)
@@ -193,27 +187,6 @@ BEGIN_FTR_SECTION · 47 unmodified lines
193	mtspr SPRN_MAS8,r10	187	mtspr SPRN_MAS8,r10
194	END_FTR_SECTION_IFSET(CPU_FTR_EMB_HV)	188	END_FTR_SECTION_IFSET(CPU_FTR_EMB_HV)
195		189
196	lbz r6,PACAIRQSOFTMASK(r13)
197	ld r5,SOFTE(r1)
198
199	/* Interrupts had better not already be enabled... */
200	tweqi r6,IRQS_ENABLED
201
202	andi. r6,r5,IRQS_DISABLED
203	bne 1f
204
205	TRACE_ENABLE_INTS
206	stb r5,PACAIRQSOFTMASK(r13)
207	1:
208	/*
209	* Restore PACAIRQHAPPENED rather than setting it based on
210	* the return MSR[EE], since we could have interrupted
211	* __check_irq_replay() or other inconsistent transitory
212	* states that must remain that way.
213	*/
214	SPECIAL_EXC_LOAD(r10,IRQHAPPENED)
215	stb r10,PACAIRQHAPPENED(r13)
216
217	SPECIAL_EXC_LOAD(r10,DEAR)	190	SPECIAL_EXC_LOAD(r10,DEAR)
218	mtspr SPRN_DEAR,r10	191	mtspr SPRN_DEAR,r10
219	SPECIAL_EXC_LOAD(r10,ESR)	192	SPECIAL_EXC_LOAD(r10,ESR)
@@ -417,14 +390,15 @@ exc_##n##_common: \ · 197 unmodified lines
417	std r6,_LINK(r1); \	390	std r6,_LINK(r1); \
418	std r7,_CTR(r1); \	391	std r7,_CTR(r1); \
419	std r8,_XER(r1); \	392	std r8,_XER(r1); \
420	li r3,(n)+1; /* indicate partial regs in trap */ \	393	li r3,(n); /* regs.trap vector */ \
421	std r9,0(r1); /* store stack frame back link */ \	394	std r9,0(r1); /* store stack frame back link */ \
422	std r10,_CCR(r1); /* store orig CR in stackframe */ \	395	std r10,_CCR(r1); /* store orig CR in stackframe */ \
423	std r9,GPR1(r1); /* store stack frame back link */ \	396	std r9,GPR1(r1); /* store stack frame back link */ \
424	std r11,SOFTE(r1); /* and save it to stackframe */ \	397	std r11,SOFTE(r1); /* and save it to stackframe */ \
425	std r12,STACK_FRAME_OVERHEAD-16(r1); /* mark the frame */ \	398	std r12,STACK_FRAME_OVERHEAD-16(r1); /* mark the frame */ \
426	std r3,_TRAP(r1); /* set trap number */ \	399	std r3,_TRAP(r1); /* set trap number */ \
427	std r0,RESULT(r1); /* clear regs->result */	400	std r0,RESULT(r1); /* clear regs->result */ \
		401	SAVE_NVGPRS(r1);
428		402
429	#define EXCEPTION_COMMON(n) \	403	#define EXCEPTION_COMMON(n) \
430	EXCEPTION_COMMON_LVL(n, SPRN_SPRG_GEN_SCRATCH, PACA_EXGEN)	404	EXCEPTION_COMMON_LVL(n, SPRN_SPRG_GEN_SCRATCH, PACA_EXGEN)
@@ -435,28 +409,6 @@ exc_##n##_common: \ · 4 unmodified lines
435	#define EXCEPTION_COMMON_DBG(n) \	409	#define EXCEPTION_COMMON_DBG(n) \
436	EXCEPTION_COMMON_LVL(n, SPRN_SPRG_DBG_SCRATCH, PACA_EXDBG)	410	EXCEPTION_COMMON_LVL(n, SPRN_SPRG_DBG_SCRATCH, PACA_EXDBG)
437		411
438	/*
439	* This is meant for exceptions that don't immediately hard-enable. We
440	* set a bit in paca->irq_happened to ensure that a subsequent call to
441	* arch_local_irq_restore() will properly hard-enable and avoid the
442	* fast-path, and then reconcile irq state.
443	*/
444	#define INTS_DISABLE RECONCILE_IRQ_STATE(r3,r4)
445
446	/*
447	* This is called by exceptions that don't use INTS_DISABLE (that did not
448	* touch irq indicators in the PACA). This will restore MSR:EE to it's
449	* previous value
450	*
451	* XXX In the long run, we may want to open-code it in order to separate the
452	* load from the wrtee, thus limiting the latency caused by the dependency
453	* but at this point, I'll favor code clarity until we have a near to final
454	* implementation
455	*/
456	#define INTS_RESTORE_HARD \
457	ld r11,_MSR(r1); \
458	wrtee r11;
459
460	/* XXX FIXME: Restore r14/r15 when necessary */	412	/* XXX FIXME: Restore r14/r15 when necessary */
461	#define BAD_STACK_TRAMPOLINE(n) \	413	#define BAD_STACK_TRAMPOLINE(n) \
462	exc_##n##_bad_stack: \	414	exc_##n##_bad_stack: \
@@ -505,12 +457,11 @@ exc_##n##_bad_stack: \ · 42 unmodified lines
505	START_EXCEPTION(label); \	457	START_EXCEPTION(label); \
506	NORMAL_EXCEPTION_PROLOG(trapnum, intnum, PROLOG_ADDITION_MASKABLE)\	458	NORMAL_EXCEPTION_PROLOG(trapnum, intnum, PROLOG_ADDITION_MASKABLE)\
507	EXCEPTION_COMMON(trapnum) \	459	EXCEPTION_COMMON(trapnum) \
508	INTS_DISABLE; \
509	ack(r8); \	460	ack(r8); \
510	CHECK_NAPPING(); \	461	CHECK_NAPPING(); \
511	addi r3,r1,STACK_FRAME_OVERHEAD; \	462	addi r3,r1,STACK_FRAME_OVERHEAD; \
512	bl hdlr; \	463	bl hdlr; \
513	b ret_from_except_lite;	464	b interrupt_return
514		465
515	/* This value is used to mark exception frames on the stack. */	466	/* This value is used to mark exception frames on the stack. */
516	.section ".toc","aw"	467	.section ".toc","aw"
@@ -561,11 +512,10 @@ __end_interrupts: · 44 unmodified lines
561	CRIT_EXCEPTION_PROLOG(0x100, BOOKE_INTERRUPT_CRITICAL,	512	CRIT_EXCEPTION_PROLOG(0x100, BOOKE_INTERRUPT_CRITICAL,
562	PROLOG_ADDITION_NONE)	513	PROLOG_ADDITION_NONE)
563	EXCEPTION_COMMON_CRIT(0x100)	514	EXCEPTION_COMMON_CRIT(0x100)
564	bl save_nvgprs
565	bl special_reg_save	515	bl special_reg_save
566	CHECK_NAPPING();	516	CHECK_NAPPING();
567	addi r3,r1,STACK_FRAME_OVERHEAD	517	addi r3,r1,STACK_FRAME_OVERHEAD
568	bl unknown_exception	518	bl unknown_nmi_exception
569	b ret_from_crit_except	519	b ret_from_crit_except
570		520
571	/* Machine Check Interrupt */	521	/* Machine Check Interrupt */
@@ -573,7 +523,6 @@ __end_interrupts: · 1 unmodified line
573	MC_EXCEPTION_PROLOG(0x000, BOOKE_INTERRUPT_MACHINE_CHECK,	523	MC_EXCEPTION_PROLOG(0x000, BOOKE_INTERRUPT_MACHINE_CHECK,
574	PROLOG_ADDITION_NONE)	524	PROLOG_ADDITION_NONE)
575	EXCEPTION_COMMON_MC(0x000)	525	EXCEPTION_COMMON_MC(0x000)
576	bl save_nvgprs
577	bl special_reg_save	526	bl special_reg_save
578	CHECK_NAPPING();	527	CHECK_NAPPING();
579	addi r3,r1,STACK_FRAME_OVERHEAD	528	addi r3,r1,STACK_FRAME_OVERHEAD
@@ -587,7 +536,6 @@ __end_interrupts: · 7 unmodified lines
587	mfspr r14,SPRN_DEAR	536	mfspr r14,SPRN_DEAR
588	mfspr r15,SPRN_ESR	537	mfspr r15,SPRN_ESR
589	EXCEPTION_COMMON(0x300)	538	EXCEPTION_COMMON(0x300)
590	INTS_DISABLE
591	b storage_fault_common	539	b storage_fault_common
592		540
593	/* Instruction Storage Interrupt */	541	/* Instruction Storage Interrupt */
@@ -597,7 +545,6 @@ __end_interrupts: · 3 unmodified lines
597	li r15,0	545	li r15,0
598	mr r14,r10	546	mr r14,r10
599	EXCEPTION_COMMON(0x400)	547	EXCEPTION_COMMON(0x400)
600	INTS_DISABLE
601	b storage_fault_common	548	b storage_fault_common
602		549
603	/* External Input Interrupt */	550	/* External Input Interrupt */
@@ -619,13 +566,12 @@ __end_interrupts: · 15 unmodified lines
619	PROLOG_ADDITION_1REG)	566	PROLOG_ADDITION_1REG)
620	mfspr r14,SPRN_ESR	567	mfspr r14,SPRN_ESR
621	EXCEPTION_COMMON(0x700)	568	EXCEPTION_COMMON(0x700)
622	INTS_DISABLE
623	std r14,_DSISR(r1)	569	std r14,_DSISR(r1)
624	addi r3,r1,STACK_FRAME_OVERHEAD	570	addi r3,r1,STACK_FRAME_OVERHEAD
625	ld r14,PACA_EXGEN+EX_R14(r13)	571	ld r14,PACA_EXGEN+EX_R14(r13)
626	bl save_nvgprs
627	bl program_check_exception	572	bl program_check_exception
628	b ret_from_except	573	REST_NVGPRS(r1)
		574	b interrupt_return
629		575
630	/* Floating Point Unavailable Interrupt */	576	/* Floating Point Unavailable Interrupt */
631	START_EXCEPTION(fp_unavailable);	577	START_EXCEPTION(fp_unavailable);
@@ -637,12 +583,10 @@ __end_interrupts: · 5 unmodified lines
637	andi. r0,r12,MSR_PR;	583	andi. r0,r12,MSR_PR;
638	beq- 1f	584	beq- 1f
639	bl load_up_fpu	585	bl load_up_fpu
640	b fast_exception_return	586	b fast_interrupt_return
641	1: INTS_DISABLE	587	1: addi r3,r1,STACK_FRAME_OVERHEAD
642	bl save_nvgprs
643	addi r3,r1,STACK_FRAME_OVERHEAD
644	bl kernel_fp_unavailable_exception	588	bl kernel_fp_unavailable_exception
645	b ret_from_except	589	b interrupt_return
646		590
647	/* Altivec Unavailable Interrupt */	591	/* Altivec Unavailable Interrupt */
648	START_EXCEPTION(altivec_unavailable);	592	START_EXCEPTION(altivec_unavailable);
@@ -656,15 +600,13 @@ BEGIN_FTR_SECTION · 7 unmodified lines
656	andi. r0,r12,MSR_PR;	600	andi. r0,r12,MSR_PR;
657	beq- 1f	601	beq- 1f
658	bl load_up_altivec	602	bl load_up_altivec
659	b fast_exception_return	603	b fast_interrupt_return
660	1:	604	1:
661	END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)	605	END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
662	#endif	606	#endif
663	INTS_DISABLE
664	bl save_nvgprs
665	addi r3,r1,STACK_FRAME_OVERHEAD	607	addi r3,r1,STACK_FRAME_OVERHEAD
666	bl altivec_unavailable_exception	608	bl altivec_unavailable_exception
667	b ret_from_except	609	b interrupt_return
668		610
669	/* AltiVec Assist */	611	/* AltiVec Assist */
670	START_EXCEPTION(altivec_assist);	612	START_EXCEPTION(altivec_assist);
@@ -672,17 +614,16 @@ END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) · 1 unmodified line
672	BOOKE_INTERRUPT_ALTIVEC_ASSIST,	614	BOOKE_INTERRUPT_ALTIVEC_ASSIST,
673	PROLOG_ADDITION_NONE)	615	PROLOG_ADDITION_NONE)
674	EXCEPTION_COMMON(0x220)	616	EXCEPTION_COMMON(0x220)
675	INTS_DISABLE
676	bl save_nvgprs
677	addi r3,r1,STACK_FRAME_OVERHEAD	617	addi r3,r1,STACK_FRAME_OVERHEAD
678	#ifdef CONFIG_ALTIVEC	618	#ifdef CONFIG_ALTIVEC
679	BEGIN_FTR_SECTION	619	BEGIN_FTR_SECTION
680	bl altivec_assist_exception	620	bl altivec_assist_exception
681	END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)	621	END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
		622	REST_NVGPRS(r1)
682	#else	623	#else
683	bl unknown_exception	624	bl unknown_exception
684	#endif	625	#endif
685	b ret_from_except	626	b interrupt_return
686		627
687		628
688	/* Decrementer Interrupt */	629	/* Decrementer Interrupt */
@@ -698,14 +639,13 @@ END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) · 9 unmodified lines
698	CRIT_EXCEPTION_PROLOG(0x9f0, BOOKE_INTERRUPT_WATCHDOG,	639	CRIT_EXCEPTION_PROLOG(0x9f0, BOOKE_INTERRUPT_WATCHDOG,
699	PROLOG_ADDITION_NONE)	640	PROLOG_ADDITION_NONE)
700	EXCEPTION_COMMON_CRIT(0x9f0)	641	EXCEPTION_COMMON_CRIT(0x9f0)
701	bl save_nvgprs
702	bl special_reg_save	642	bl special_reg_save
703	CHECK_NAPPING();	643	CHECK_NAPPING();
704	addi r3,r1,STACK_FRAME_OVERHEAD	644	addi r3,r1,STACK_FRAME_OVERHEAD
705	#ifdef CONFIG_BOOKE_WDT	645	#ifdef CONFIG_BOOKE_WDT
706	bl WatchdogException	646	bl WatchdogException
707	#else	647	#else
708	bl unknown_exception	648	bl unknown_nmi_exception
709	#endif	649	#endif
710	b ret_from_crit_except	650	b ret_from_crit_except
711		651
@@ -722,11 +662,9 @@ END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) · 10 unmodified lines
722	NORMAL_EXCEPTION_PROLOG(0xf20, BOOKE_INTERRUPT_AP_UNAVAIL,	662	NORMAL_EXCEPTION_PROLOG(0xf20, BOOKE_INTERRUPT_AP_UNAVAIL,
723	PROLOG_ADDITION_NONE)	663	PROLOG_ADDITION_NONE)
724	EXCEPTION_COMMON(0xf20)	664	EXCEPTION_COMMON(0xf20)
725	INTS_DISABLE
726	bl save_nvgprs
727	addi r3,r1,STACK_FRAME_OVERHEAD	665	addi r3,r1,STACK_FRAME_OVERHEAD
728	bl unknown_exception	666	bl unknown_exception
729	b ret_from_except	667	b interrupt_return
730		668
731	/* Debug exception as a critical interrupt*/	669	/* Debug exception as a critical interrupt*/
732	START_EXCEPTION(debug_crit);	670	START_EXCEPTION(debug_crit);
@@ -792,9 +730,9 @@ END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) · 59 unmodified lines
792	addi r3,r1,STACK_FRAME_OVERHEAD	730	addi r3,r1,STACK_FRAME_OVERHEAD
793	ld r14,PACA_EXCRIT+EX_R14(r13)	731	ld r14,PACA_EXCRIT+EX_R14(r13)
794	ld r15,PACA_EXCRIT+EX_R15(r13)	732	ld r15,PACA_EXCRIT+EX_R15(r13)
795	bl save_nvgprs
796	bl DebugException	733	bl DebugException
797	b ret_from_except	734	REST_NVGPRS(r1)
		735	b interrupt_return
798		736
799	kernel_dbg_exc:	737	kernel_dbg_exc:
800	b . /* NYI */	738	b . /* NYI */
@@ -859,24 +797,22 @@ kernel_dbg_exc: · 58 unmodified lines
859	*/	797	*/
860	mfspr r14,SPRN_DBSR	798	mfspr r14,SPRN_DBSR
861	EXCEPTION_COMMON_DBG(0xd08)	799	EXCEPTION_COMMON_DBG(0xd08)
862	INTS_DISABLE
863	std r14,_DSISR(r1)	800	std r14,_DSISR(r1)
864	addi r3,r1,STACK_FRAME_OVERHEAD	801	addi r3,r1,STACK_FRAME_OVERHEAD
865	ld r14,PACA_EXDBG+EX_R14(r13)	802	ld r14,PACA_EXDBG+EX_R14(r13)
866	ld r15,PACA_EXDBG+EX_R15(r13)	803	ld r15,PACA_EXDBG+EX_R15(r13)
867	bl save_nvgprs
868	bl DebugException	804	bl DebugException
869	b ret_from_except	805	REST_NVGPRS(r1)
		806	b interrupt_return
870		807
871	START_EXCEPTION(perfmon);	808	START_EXCEPTION(perfmon);
872	NORMAL_EXCEPTION_PROLOG(0x260, BOOKE_INTERRUPT_PERFORMANCE_MONITOR,	809	NORMAL_EXCEPTION_PROLOG(0x260, BOOKE_INTERRUPT_PERFORMANCE_MONITOR,
873	PROLOG_ADDITION_NONE)	810	PROLOG_ADDITION_NONE)
874	EXCEPTION_COMMON(0x260)	811	EXCEPTION_COMMON(0x260)
875	INTS_DISABLE
876	CHECK_NAPPING()	812	CHECK_NAPPING()
877	addi r3,r1,STACK_FRAME_OVERHEAD	813	addi r3,r1,STACK_FRAME_OVERHEAD
878	bl performance_monitor_exception	814	bl performance_monitor_exception
879	b ret_from_except_lite	815	b interrupt_return
880		816
881	/* Doorbell interrupt */	817	/* Doorbell interrupt */
882	MASKABLE_EXCEPTION(0x280, BOOKE_INTERRUPT_DOORBELL,	818	MASKABLE_EXCEPTION(0x280, BOOKE_INTERRUPT_DOORBELL,
@@ -887,11 +823,10 @@ kernel_dbg_exc: · 4 unmodified lines
887	CRIT_EXCEPTION_PROLOG(0x2a0, BOOKE_INTERRUPT_DOORBELL_CRITICAL,	823	CRIT_EXCEPTION_PROLOG(0x2a0, BOOKE_INTERRUPT_DOORBELL_CRITICAL,
888	PROLOG_ADDITION_NONE)	824	PROLOG_ADDITION_NONE)
889	EXCEPTION_COMMON_CRIT(0x2a0)	825	EXCEPTION_COMMON_CRIT(0x2a0)
890	bl save_nvgprs
891	bl special_reg_save	826	bl special_reg_save
892	CHECK_NAPPING();	827	CHECK_NAPPING();
893	addi r3,r1,STACK_FRAME_OVERHEAD	828	addi r3,r1,STACK_FRAME_OVERHEAD
894	bl unknown_exception	829	bl unknown_nmi_exception
895	b ret_from_crit_except	830	b ret_from_crit_except
896		831
897	/*	832	/*
@@ -903,21 +838,18 @@ kernel_dbg_exc: · 5 unmodified lines
903	PROLOG_ADDITION_NONE)	838	PROLOG_ADDITION_NONE)
904	EXCEPTION_COMMON(0x2c0)	839	EXCEPTION_COMMON(0x2c0)
905	addi r3,r1,STACK_FRAME_OVERHEAD	840	addi r3,r1,STACK_FRAME_OVERHEAD
906	bl save_nvgprs
907	INTS_RESTORE_HARD
908	bl unknown_exception	841	bl unknown_exception
909	b ret_from_except	842	b interrupt_return
910		843
911	/* Guest Doorbell critical Interrupt */	844	/* Guest Doorbell critical Interrupt */
912	START_EXCEPTION(guest_doorbell_crit);	845	START_EXCEPTION(guest_doorbell_crit);
913	CRIT_EXCEPTION_PROLOG(0x2e0, BOOKE_INTERRUPT_GUEST_DBELL_CRIT,	846	CRIT_EXCEPTION_PROLOG(0x2e0, BOOKE_INTERRUPT_GUEST_DBELL_CRIT,
914	PROLOG_ADDITION_NONE)	847	PROLOG_ADDITION_NONE)
915	EXCEPTION_COMMON_CRIT(0x2e0)	848	EXCEPTION_COMMON_CRIT(0x2e0)
916	bl save_nvgprs
917	bl special_reg_save	849	bl special_reg_save
918	CHECK_NAPPING();	850	CHECK_NAPPING();
919	addi r3,r1,STACK_FRAME_OVERHEAD	851	addi r3,r1,STACK_FRAME_OVERHEAD
920	bl unknown_exception	852	bl unknown_nmi_exception
921	b ret_from_crit_except	853	b ret_from_crit_except
922		854
923	/* Hypervisor call */	855	/* Hypervisor call */
@@ -926,10 +858,8 @@ kernel_dbg_exc: · 2 unmodified lines
926	PROLOG_ADDITION_NONE)	858	PROLOG_ADDITION_NONE)
927	EXCEPTION_COMMON(0x310)	859	EXCEPTION_COMMON(0x310)
928	addi r3,r1,STACK_FRAME_OVERHEAD	860	addi r3,r1,STACK_FRAME_OVERHEAD
929	bl save_nvgprs
930	INTS_RESTORE_HARD
931	bl unknown_exception	861	bl unknown_exception
932	b ret_from_except	862	b interrupt_return
933		863
934	/* Embedded Hypervisor priviledged */	864	/* Embedded Hypervisor priviledged */
935	START_EXCEPTION(ehpriv);	865	START_EXCEPTION(ehpriv);
@@ -937,10 +867,8 @@ kernel_dbg_exc: · 1 unmodified line
937	PROLOG_ADDITION_NONE)	867	PROLOG_ADDITION_NONE)
938	EXCEPTION_COMMON(0x320)	868	EXCEPTION_COMMON(0x320)
939	addi r3,r1,STACK_FRAME_OVERHEAD	869	addi r3,r1,STACK_FRAME_OVERHEAD
940	bl save_nvgprs
941	INTS_RESTORE_HARD
942	bl unknown_exception	870	bl unknown_exception
943	b ret_from_except	871	b interrupt_return
944		872
945	/* LRAT Error interrupt */	873	/* LRAT Error interrupt */
946	START_EXCEPTION(lrat_error);	874	START_EXCEPTION(lrat_error);
@@ -948,10 +876,8 @@ kernel_dbg_exc: · 1 unmodified line
948	PROLOG_ADDITION_NONE)	876	PROLOG_ADDITION_NONE)
949	EXCEPTION_COMMON(0x340)	877	EXCEPTION_COMMON(0x340)
950	addi r3,r1,STACK_FRAME_OVERHEAD	878	addi r3,r1,STACK_FRAME_OVERHEAD
951	bl save_nvgprs
952	INTS_RESTORE_HARD
953	bl unknown_exception	879	bl unknown_exception
954	b ret_from_except	880	b interrupt_return
955		881
956	/*	882	/*
957	* An interrupt came in while soft-disabled; We mark paca->irq_happened	883	* An interrupt came in while soft-disabled; We mark paca->irq_happened
@@ -1011,14 +937,7 @@ storage_fault_common: · 53 unmodified lines
1011	ld r14,PACA_EXGEN+EX_R14(r13)	937	ld r14,PACA_EXGEN+EX_R14(r13)
1012	ld r15,PACA_EXGEN+EX_R15(r13)	938	ld r15,PACA_EXGEN+EX_R15(r13)
1013	bl do_page_fault	939	bl do_page_fault
1014	cmpdi r3,0	940	b interrupt_return
1015	bne- 1f
1016	b ret_from_except_lite
1017	1: bl save_nvgprs
1018	mr r4,r3
1019	addi r3,r1,STACK_FRAME_OVERHEAD
1020	bl __bad_page_fault
1021	b ret_from_except
1022		941
1023	/*	942	/*
1024	* Alignment exception doesn't fit entirely in the 0x100 bytes so it	943	* Alignment exception doesn't fit entirely in the 0x100 bytes so it
@@ -1030,291 +949,9 @@ alignment_more: · 5 unmodified lines
1030	addi r3,r1,STACK_FRAME_OVERHEAD	949	addi r3,r1,STACK_FRAME_OVERHEAD
1031	ld r14,PACA_EXGEN+EX_R14(r13)	950	ld r14,PACA_EXGEN+EX_R14(r13)
1032	ld r15,PACA_EXGEN+EX_R15(r13)	951	ld r15,PACA_EXGEN+EX_R15(r13)
1033	bl save_nvgprs
1034	INTS_RESTORE_HARD
1035	bl alignment_exception	952	bl alignment_exception
1036	b ret_from_except
1037
1038	.align 7
1039	_GLOBAL(ret_from_except)
1040	ld r11,_TRAP(r1)
1041	andi. r0,r11,1
1042	bne ret_from_except_lite
1043	REST_NVGPRS(r1)	953	REST_NVGPRS(r1)
1044		954	b interrupt_return
1045	_GLOBAL(ret_from_except_lite)
1046	/*
1047	* Disable interrupts so that current_thread_info()->flags
1048	* can't change between when we test it and when we return
1049	* from the interrupt.
1050	*/
1051	wrteei 0
1052
1053	ld r9, PACA_THREAD_INFO(r13)
1054	ld r3,_MSR(r1)
1055	ld r10,PACACURRENT(r13)
1056	ld r4,TI_FLAGS(r9)
1057	andi. r3,r3,MSR_PR
1058	beq resume_kernel
1059	lwz r3,(THREAD+THREAD_DBCR0)(r10)
1060
1061	/* Check current_thread_info()->flags */
1062	andi. r0,r4,_TIF_USER_WORK_MASK
1063	bne 1f
1064	/*
1065	* Check to see if the dbcr0 register is set up to debug.
1066	* Use the internal debug mode bit to do this.
1067	*/
1068	andis. r0,r3,DBCR0_IDM@h
1069	beq restore
1070	mfmsr r0
1071	rlwinm r0,r0,0,~MSR_DE /* Clear MSR.DE */
1072	mtmsr r0
1073	mtspr SPRN_DBCR0,r3
1074	li r10, -1
1075	mtspr SPRN_DBSR,r10
1076	b restore
1077	1: andi. r0,r4,_TIF_NEED_RESCHED
1078	beq 2f
1079	bl restore_interrupts
1080	SCHEDULE_USER
1081	b ret_from_except_lite
1082	2:
1083	bl save_nvgprs
1084	/*
1085	* Use a non volatile GPR to save and restore our thread_info flags
1086	* across the call to restore_interrupts.
1087	*/
1088	mr r30,r4
1089	bl restore_interrupts
1090	mr r4,r30
1091	addi r3,r1,STACK_FRAME_OVERHEAD
1092	bl do_notify_resume
1093	b ret_from_except
1094
1095	resume_kernel:
1096	/* check current_thread_info, _TIF_EMULATE_STACK_STORE */
1097	andis. r8,r4,_TIF_EMULATE_STACK_STORE@h
1098	beq+ 1f
1099
1100	addi r8,r1,INT_FRAME_SIZE /* Get the kprobed function entry */
1101
1102	ld r3,GPR1(r1)
1103	subi r3,r3,INT_FRAME_SIZE /* dst: Allocate a trampoline exception frame */
1104	mr r4,r1 /* src: current exception frame */
1105	mr r1,r3 /* Reroute the trampoline frame to r1 */
1106
1107	/* Copy from the original to the trampoline. */
1108	li r5,INT_FRAME_SIZE/8 /* size: INT_FRAME_SIZE */
1109	li r6,0 /* start offset: 0 */
1110	mtctr r5
1111	2: ldx r0,r6,r4
1112	stdx r0,r6,r3
1113	addi r6,r6,8
1114	bdnz 2b
1115
1116	/* Do real store operation to complete stdu */
1117	ld r5,GPR1(r1)
1118	std r8,0(r5)
1119
1120	/* Clear _TIF_EMULATE_STACK_STORE flag */
1121	lis r11,_TIF_EMULATE_STACK_STORE@h
1122	addi r5,r9,TI_FLAGS
1123	0: ldarx r4,0,r5
1124	andc r4,r4,r11
1125	stdcx. r4,0,r5
1126	bne- 0b
1127	1:
1128
1129	#ifdef CONFIG_PREEMPT
1130	/* Check if we need to preempt */
1131	andi. r0,r4,_TIF_NEED_RESCHED
1132	beq+ restore
1133	/* Check that preempt_count() == 0 and interrupts are enabled */
1134	lwz r8,TI_PREEMPT(r9)
1135	cmpwi cr0,r8,0
1136	bne restore
1137	ld r0,SOFTE(r1)
1138	andi. r0,r0,IRQS_DISABLED
1139	bne restore
1140
1141	/*
1142	* Here we are preempting the current task. We want to make
1143	* sure we are soft-disabled first and reconcile irq state.
1144	*/
1145	RECONCILE_IRQ_STATE(r3,r4)
1146	bl preempt_schedule_irq
1147
1148	/*
1149	* arch_local_irq_restore() from preempt_schedule_irq above may
1150	* enable hard interrupt but we really should disable interrupts
1151	* when we return from the interrupt, and so that we don't get
1152	* interrupted after loading SRR0/1.
1153	*/
1154	wrteei 0
1155	#endif /* CONFIG_PREEMPT */
1156
1157	restore:
1158	/*
1159	* This is the main kernel exit path. First we check if we
1160	* are about to re-enable interrupts
1161	*/
1162	ld r5,SOFTE(r1)
1163	lbz r6,PACAIRQSOFTMASK(r13)
1164	andi. r5,r5,IRQS_DISABLED
1165	bne .Lrestore_irq_off
1166
1167	/* We are enabling, were we already enabled ? Yes, just return */
1168	andi. r6,r6,IRQS_DISABLED
1169	beq cr0,fast_exception_return
1170
1171	/*
1172	* We are about to soft-enable interrupts (we are hard disabled
1173	* at this point). We check if there's anything that needs to
1174	* be replayed first.
1175	*/
1176	lbz r0,PACAIRQHAPPENED(r13)
1177	cmpwi cr0,r0,0
1178	bne- .Lrestore_check_irq_replay
1179
1180	/*
1181	* Get here when nothing happened while soft-disabled, just
1182	* soft-enable and move-on. We will hard-enable as a side
1183	* effect of rfi
1184	*/
1185	.Lrestore_no_replay:
1186	TRACE_ENABLE_INTS
1187	li r0,IRQS_ENABLED
1188	stb r0,PACAIRQSOFTMASK(r13);
1189
1190	/* This is the return from load_up_fpu fast path which could do with
1191	* less GPR restores in fact, but for now we have a single return path
1192	*/
1193	fast_exception_return:
1194	wrteei 0
1195	1: mr r0,r13
1196	ld r10,_MSR(r1)
1197	REST_4GPRS(2, r1)
1198	andi. r6,r10,MSR_PR
1199	REST_2GPRS(6, r1)
1200	beq 1f
1201	ACCOUNT_CPU_USER_EXIT(r13, r10, r11)
1202	ld r0,GPR13(r1)
1203
1204	1: stdcx. r0,0,r1 /* to clear the reservation */
1205
1206	ld r8,_CCR(r1)
1207	ld r9,_LINK(r1)
1208	ld r10,_CTR(r1)
1209	ld r11,_XER(r1)
1210	mtcr r8
1211	mtlr r9
1212	mtctr r10
1213	mtxer r11
1214	REST_2GPRS(8, r1)
1215	ld r10,GPR10(r1)
1216	ld r11,GPR11(r1)
1217	ld r12,GPR12(r1)
1218	mtspr SPRN_SPRG_GEN_SCRATCH,r0
1219
1220	std r10,PACA_EXGEN+EX_R10(r13);
1221	std r11,PACA_EXGEN+EX_R11(r13);
1222	ld r10,_NIP(r1)
1223	ld r11,_MSR(r1)
1224	ld r0,GPR0(r1)
1225	ld r1,GPR1(r1)
1226	mtspr SPRN_SRR0,r10
1227	mtspr SPRN_SRR1,r11
1228	ld r10,PACA_EXGEN+EX_R10(r13)
1229	ld r11,PACA_EXGEN+EX_R11(r13)
1230	mfspr r13,SPRN_SPRG_GEN_SCRATCH
1231	rfi
1232
1233	/*
1234	* We are returning to a context with interrupts soft disabled.
1235	*
1236	* However, we may also about to hard enable, so we need to
1237	* make sure that in this case, we also clear PACA_IRQ_HARD_DIS
1238	* or that bit can get out of sync and bad things will happen
1239	*/
1240	.Lrestore_irq_off:
1241	ld r3,_MSR(r1)
1242	lbz r7,PACAIRQHAPPENED(r13)
1243	andi. r0,r3,MSR_EE
1244	beq 1f
1245	rlwinm r7,r7,0,~PACA_IRQ_HARD_DIS
1246	stb r7,PACAIRQHAPPENED(r13)
1247	1:
1248	#if defined(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG) && defined(CONFIG_BUG)
1249	/* The interrupt should not have soft enabled. */
1250	lbz r7,PACAIRQSOFTMASK(r13)
1251	1: tdeqi r7,IRQS_ENABLED
1252	EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,BUGFLAG_WARNING
1253	#endif
1254	b fast_exception_return
1255
1256	/*
1257	* Something did happen, check if a re-emit is needed
1258	* (this also clears paca->irq_happened)
1259	*/
1260	.Lrestore_check_irq_replay:
1261	/* XXX: We could implement a fast path here where we check
1262	* for irq_happened being just 0x01, in which case we can
1263	* clear it and return. That means that we would potentially
1264	* miss a decrementer having wrapped all the way around.
1265	*
1266	* Still, this might be useful for things like hash_page
1267	*/
1268	bl __check_irq_replay
1269	cmpwi cr0,r3,0
1270	beq .Lrestore_no_replay
1271
1272	/*
1273	* We need to re-emit an interrupt. We do so by re-using our
1274	* existing exception frame. We first change the trap value,
1275	* but we need to ensure we preserve the low nibble of it
1276	*/
1277	ld r4,_TRAP(r1)
1278	clrldi r4,r4,60
1279	or r4,r4,r3
1280	std r4,_TRAP(r1)
1281
1282	/*
1283	* PACA_IRQ_HARD_DIS won't always be set here, so set it now
1284	* to reconcile the IRQ state. Tracing is already accounted for.
1285	*/
1286	lbz r4,PACAIRQHAPPENED(r13)
1287	ori r4,r4,PACA_IRQ_HARD_DIS
1288	stb r4,PACAIRQHAPPENED(r13)
1289
1290	/*
1291	* Then find the right handler and call it. Interrupts are
1292	* still soft-disabled and we keep them that way.
1293	*/
1294	cmpwi cr0,r3,0x500
1295	bne 1f
1296	addi r3,r1,STACK_FRAME_OVERHEAD;
1297	bl do_IRQ
1298	b ret_from_except
1299	1: cmpwi cr0,r3,0x900
1300	bne 1f
1301	addi r3,r1,STACK_FRAME_OVERHEAD;
1302	bl timer_interrupt
1303	b ret_from_except
1304	#ifdef CONFIG_PPC_DOORBELL
1305	1:
1306	cmpwi cr0,r3,0x280
1307	bne 1f
1308	addi r3,r1,STACK_FRAME_OVERHEAD;
1309	bl doorbell_exception
1310	#endif /* CONFIG_PPC_DOORBELL */
1311	1: b ret_from_except /* What else to do here ? */
1312
1313	_ASM_NOKPROBE_SYMBOL(ret_from_except);
1314	_ASM_NOKPROBE_SYMBOL(ret_from_except_lite);
1315	_ASM_NOKPROBE_SYMBOL(resume_kernel);
1316	_ASM_NOKPROBE_SYMBOL(restore);
1317	_ASM_NOKPROBE_SYMBOL(fast_exception_return);
1318		955
1319	/*	956	/*
1320	* Trampolines used when spotting a bad kernel stack pointer in	957	* Trampolines used when spotting a bad kernel stack pointer in

arch/powerpc/kernel/exceptions-64s.S+7−53

@@ -692,25 +692,6 @@ END_FTR_SECTION_IFSET(CPU_FTR_CFAR) · 691 unmodified lines
692	ld r1,GPR1(r1)	692	ld r1,GPR1(r1)
693	.endm	693	.endm
694		694
695	/*
696	* When the idle code in power4_idle puts the CPU into NAP mode,
697	* it has to do so in a loop, and relies on the external interrupt
698	* and decrementer interrupt entry code to get it out of the loop.
699	* It sets the _TLF_NAPPING bit in current_thread_info()->local_flags
700	* to signal that it is in the loop and needs help to get out.
701	*/
702	#ifdef CONFIG_PPC_970_NAP
703	#define FINISH_NAP \
704	BEGIN_FTR_SECTION \
705	ld r11, PACA_THREAD_INFO(r13); \
706	ld r9,TI_LOCAL_FLAGS(r11); \
707	andi. r10,r9,_TLF_NAPPING; \
708	bnel power4_fixup_nap; \
709	END_FTR_SECTION_IFSET(CPU_FTR_CAN_NAP)
710	#else
711	#define FINISH_NAP
712	#endif
713
714	/*	695	/*
715	* There are a few constraints to be concerned with.	696	* There are a few constraints to be concerned with.
716	* - Real mode exceptions code/data must be located at their physical location.	697	* - Real mode exceptions code/data must be located at their physical location.
@@ -1248,7 +1229,6 @@ EXC_COMMON_BEGIN(machine_check_common) · 531 unmodified lines
1248	*/	1229	*/
1249	GEN_COMMON machine_check	1230	GEN_COMMON machine_check
1250		1231
1251	FINISH_NAP
1252	/* Enable MSR_RI when finished with PACA_EXMC */	1232	/* Enable MSR_RI when finished with PACA_EXMC */
1253	li r10,MSR_RI	1233	li r10,MSR_RI
1254	mtmsrd r10,1	1234	mtmsrd r10,1
@@ -1571,7 +1551,6 @@ EXC_VIRT_BEGIN(hardware_interrupt, 0x4500, 0x100) · 316 unmodified lines
1571	EXC_VIRT_END(hardware_interrupt, 0x4500, 0x100)	1551	EXC_VIRT_END(hardware_interrupt, 0x4500, 0x100)
1572	EXC_COMMON_BEGIN(hardware_interrupt_common)	1552	EXC_COMMON_BEGIN(hardware_interrupt_common)
1573	GEN_COMMON hardware_interrupt	1553	GEN_COMMON hardware_interrupt
1574	FINISH_NAP
1575	addi r3,r1,STACK_FRAME_OVERHEAD	1554	addi r3,r1,STACK_FRAME_OVERHEAD
1576	bl do_IRQ	1555	bl do_IRQ
1577	b interrupt_return	1556	b interrupt_return
@@ -1801,7 +1780,6 @@ EXC_VIRT_BEGIN(decrementer, 0x4900, 0x80) · 223 unmodified lines
1801	EXC_VIRT_END(decrementer, 0x4900, 0x80)	1780	EXC_VIRT_END(decrementer, 0x4900, 0x80)
1802	EXC_COMMON_BEGIN(decrementer_common)	1781	EXC_COMMON_BEGIN(decrementer_common)
1803	GEN_COMMON decrementer	1782	GEN_COMMON decrementer
1804	FINISH_NAP
1805	addi r3,r1,STACK_FRAME_OVERHEAD	1783	addi r3,r1,STACK_FRAME_OVERHEAD
1806	bl timer_interrupt	1784	bl timer_interrupt
1807	b interrupt_return	1785	b interrupt_return
@@ -1886,7 +1864,6 @@ EXC_VIRT_BEGIN(doorbell_super, 0x4a00, 0x100) · 78 unmodified lines
1886	EXC_VIRT_END(doorbell_super, 0x4a00, 0x100)	1864	EXC_VIRT_END(doorbell_super, 0x4a00, 0x100)
1887	EXC_COMMON_BEGIN(doorbell_super_common)	1865	EXC_COMMON_BEGIN(doorbell_super_common)
1888	GEN_COMMON doorbell_super	1866	GEN_COMMON doorbell_super
1889	FINISH_NAP
1890	addi r3,r1,STACK_FRAME_OVERHEAD	1867	addi r3,r1,STACK_FRAME_OVERHEAD
1891	#ifdef CONFIG_PPC_DOORBELL	1868	#ifdef CONFIG_PPC_DOORBELL
1892	bl doorbell_exception	1869	bl doorbell_exception
@@ -2237,7 +2214,6 @@ EXC_COMMON_BEGIN(hmi_exception_early_common) · 344 unmodified lines
2237		2214
2238	EXC_COMMON_BEGIN(hmi_exception_common)	2215	EXC_COMMON_BEGIN(hmi_exception_common)
2239	GEN_COMMON hmi_exception	2216	GEN_COMMON hmi_exception
2240	FINISH_NAP
2241	addi r3,r1,STACK_FRAME_OVERHEAD	2217	addi r3,r1,STACK_FRAME_OVERHEAD
2242	bl handle_hmi_exception	2218	bl handle_hmi_exception
2243	b interrupt_return	2219	b interrupt_return
@@ -2266,7 +2242,6 @@ EXC_VIRT_BEGIN(h_doorbell, 0x4e80, 0x20) · 22 unmodified lines
2266	EXC_VIRT_END(h_doorbell, 0x4e80, 0x20)	2242	EXC_VIRT_END(h_doorbell, 0x4e80, 0x20)
2267	EXC_COMMON_BEGIN(h_doorbell_common)	2243	EXC_COMMON_BEGIN(h_doorbell_common)
2268	GEN_COMMON h_doorbell	2244	GEN_COMMON h_doorbell
2269	FINISH_NAP
2270	addi r3,r1,STACK_FRAME_OVERHEAD	2245	addi r3,r1,STACK_FRAME_OVERHEAD
2271	#ifdef CONFIG_PPC_DOORBELL	2246	#ifdef CONFIG_PPC_DOORBELL
2272	bl doorbell_exception	2247	bl doorbell_exception
@@ -2299,7 +2274,6 @@ EXC_VIRT_BEGIN(h_virt_irq, 0x4ea0, 0x20) · 26 unmodified lines
2299	EXC_VIRT_END(h_virt_irq, 0x4ea0, 0x20)	2274	EXC_VIRT_END(h_virt_irq, 0x4ea0, 0x20)
2300	EXC_COMMON_BEGIN(h_virt_irq_common)	2275	EXC_COMMON_BEGIN(h_virt_irq_common)
2301	GEN_COMMON h_virt_irq	2276	GEN_COMMON h_virt_irq
2302	FINISH_NAP
2303	addi r3,r1,STACK_FRAME_OVERHEAD	2277	addi r3,r1,STACK_FRAME_OVERHEAD
2304	bl do_IRQ	2278	bl do_IRQ
2305	b interrupt_return	2279	b interrupt_return
@@ -2345,7 +2319,6 @@ EXC_VIRT_BEGIN(performance_monitor, 0x4f00, 0x20) · 39 unmodified lines
2345	EXC_VIRT_END(performance_monitor, 0x4f00, 0x20)	2319	EXC_VIRT_END(performance_monitor, 0x4f00, 0x20)
2346	EXC_COMMON_BEGIN(performance_monitor_common)	2320	EXC_COMMON_BEGIN(performance_monitor_common)
2347	GEN_COMMON performance_monitor	2321	GEN_COMMON performance_monitor
2348	FINISH_NAP
2349	addi r3,r1,STACK_FRAME_OVERHEAD	2322	addi r3,r1,STACK_FRAME_OVERHEAD
2350	bl performance_monitor_exception	2323	bl performance_monitor_exception
2351	b interrupt_return	2324	b interrupt_return
@@ -2530,8 +2503,6 @@ EXC_VIRT_NONE(0x5100, 0x100) · 178 unmodified lines
2530	INT_DEFINE_BEGIN(cbe_system_error)	2503	INT_DEFINE_BEGIN(cbe_system_error)
2531	IVEC=0x1200	2504	IVEC=0x1200
2532	IHSRR=1	2505	IHSRR=1
2533	IKVM_SKIP=1
2534	IKVM_REAL=1
2535	INT_DEFINE_END(cbe_system_error)	2506	INT_DEFINE_END(cbe_system_error)
2536		2507
2537	EXC_REAL_BEGIN(cbe_system_error, 0x1200, 0x100)	2508	EXC_REAL_BEGIN(cbe_system_error, 0x1200, 0x100)
@@ -2551,11 +2522,16 @@ EXC_REAL_NONE(0x1200, 0x100) · 13 unmodified lines
2551	EXC_VIRT_NONE(0x5200, 0x100)	2522	EXC_VIRT_NONE(0x5200, 0x100)
2552	#endif	2523	#endif
2553		2524
2554		2525	/**
		2526	* Interrupt 0x1300 - Instruction Address Breakpoint Interrupt.
		2527	* This has been removed from the ISA before 2.01, which is the earliest
		2528	* 64-bit BookS ISA supported, however the G5 / 970 implements this
		2529	* interrupt with a non-architected feature available through the support
		2530	* processor interface.
		2531	*/
2555	INT_DEFINE_BEGIN(instruction_breakpoint)	2532	INT_DEFINE_BEGIN(instruction_breakpoint)
2556	IVEC=0x1300	2533	IVEC=0x1300
2557	#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE	2534	#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
2558	IKVM_SKIP=1
2559	IKVM_REAL=1	2535	IKVM_REAL=1
2560	#endif	2536	#endif
2561	INT_DEFINE_END(instruction_breakpoint)	2537	INT_DEFINE_END(instruction_breakpoint)
@@ -2701,8 +2677,6 @@ EXC_COMMON_BEGIN(denorm_exception_common) · 139 unmodified lines
2701	INT_DEFINE_BEGIN(cbe_maintenance)	2677	INT_DEFINE_BEGIN(cbe_maintenance)
2702	IVEC=0x1600	2678	IVEC=0x1600
2703	IHSRR=1	2679	IHSRR=1
2704	IKVM_SKIP=1
2705	IKVM_REAL=1
2706	INT_DEFINE_END(cbe_maintenance)	2680	INT_DEFINE_END(cbe_maintenance)
2707		2681
2708	EXC_REAL_BEGIN(cbe_maintenance, 0x1600, 0x100)	2682	EXC_REAL_BEGIN(cbe_maintenance, 0x1600, 0x100)
@@ -2754,8 +2728,6 @@ EXC_COMMON_BEGIN(altivec_assist_common) · 45 unmodified lines
2754	INT_DEFINE_BEGIN(cbe_thermal)	2728	INT_DEFINE_BEGIN(cbe_thermal)
2755	IVEC=0x1800	2729	IVEC=0x1800
2756	IHSRR=1	2730	IHSRR=1
2757	IKVM_SKIP=1
2758	IKVM_REAL=1
2759	INT_DEFINE_END(cbe_thermal)	2731	INT_DEFINE_END(cbe_thermal)
2760		2732
2761	EXC_REAL_BEGIN(cbe_thermal, 0x1800, 0x100)	2733	EXC_REAL_BEGIN(cbe_thermal, 0x1800, 0x100)
@@ -3096,24 +3068,6 @@ USE_FIXED_SECTION(virt_trampolines) · 334 unmodified lines
3096	__end_interrupts:	3068	__end_interrupts:
3097	DEFINE_FIXED_SYMBOL(__end_interrupts)	3069	DEFINE_FIXED_SYMBOL(__end_interrupts)
3098		3070
3099	#ifdef CONFIG_PPC_970_NAP
3100	/*
3101	* Called by exception entry code if _TLF_NAPPING was set, this clears
3102	* the NAPPING flag, and redirects the exception exit to
3103	* power4_fixup_nap_return.
3104	*/
3105	.globl power4_fixup_nap
3106	EXC_COMMON_BEGIN(power4_fixup_nap)
3107	andc r9,r9,r10
3108	std r9,TI_LOCAL_FLAGS(r11)
3109	LOAD_REG_ADDR(r10, power4_idle_nap_return)
3110	std r10,_NIP(r1)
3111	blr
3112
3113	power4_idle_nap_return:
3114	blr
3115	#endif
3116
3117	CLOSE_FIXED_SECTION(real_vectors);	3071	CLOSE_FIXED_SECTION(real_vectors);
3118	CLOSE_FIXED_SECTION(real_trampolines);	3072	CLOSE_FIXED_SECTION(real_trampolines);
3119	CLOSE_FIXED_SECTION(virt_vectors);	3073	CLOSE_FIXED_SECTION(virt_vectors);

arch/powerpc/kernel/fadump.c+9−9

 #include <asm/fadump.h>
 #include <asm/fadump-internal.h>
 #include <asm/setup.h>
+#include <asm/interrupt.h>
 /*
  * The CPU who acquired the lock to trigger the fadump crash should
 static void __init fadump_reserve_crash_area(u64 base);
-struct kobject *fadump_kobj;
 #ifndef CONFIG_PRESERVE_FA_DUMP
+static struct kobject *fadump_kobj;
 static atomic_t cpus_in_fadump;
 static DEFINE_MUTEX(fadump_mutex);
-struct fadump_mrange_info crash_mrange_info = { "crash", NULL, 0, 0, 0, false };
+static struct fadump_mrange_info crash_mrange_info = { "crash", NULL, 0, 0, 0, false };
 #define RESERVED_RNGS_SZ	16384 /* 16K - 128 entries */
 #define RESERVED_RNGS_CNT	(RESERVED_RNGS_SZ / \
 				 sizeof(struct fadump_memory_range))
 static struct fadump_memory_range rngs[RESERVED_RNGS_CNT];
-struct fadump_mrange_info reserved_mrange_info = { "reserved", rngs,
+static struct fadump_mrange_info
-						   RESERVED_RNGS_SZ, 0,
+reserved_mrange_info = { "reserved", rngs, RESERVED_RNGS_SZ, 0, RESERVED_RNGS_CNT, true };
-						   RESERVED_RNGS_CNT, true };
 static void __init early_init_dt_scan_reserved_ranges(unsigned long node);
  * But for some reason even if it fails we still have the memory reservation
  * with us and we can still continue doing fadump.
  */
-int __init fadump_cma_init(void)
+static int __init fadump_cma_init(void)
 {
 	unsigned long long base, size;
 	int rc;
  * that is required for a kernel to boot successfully.
  *
  */
-static inline u64 fadump_calculate_reserve_size(void)
+static __init u64 fadump_calculate_reserve_size(void)
 {
 	u64 base, size, bootmem_min;
 	int ret;
 	 * If we came in via system reset, wait a while for the secondary
 	 * CPUs to enter.
 	 */
-	if (TRAP(&(fdh->regs)) == 0x100) {
+	if (TRAP(&(fdh->regs)) == INTERRUPT_SYSTEM_RESET) {
 		msecs = CRASH_TIMEOUT;
 		while ((atomic_read(&cpus_in_fadump) < ncpus) && (--msecs > 0))
 			mdelay(1);

arch/powerpc/kernel/fpu.S+0−2

 	/* enable use of FP after return */
 #ifdef CONFIG_PPC32
 	mfspr	r5,SPRN_SPRG_THREAD	/* current task's THREAD (phys) */
-#ifdef CONFIG_VMAP_STACK
 	tovirt(r5, r5)
-#endif
 	lwz	r4,THREAD_FPEXC_MODE(r5)
 	ori	r9,r9,MSR_FP		/* enable FP for current */
 	or	r9,r9,r4

arch/powerpc/kernel/head_32.h+79−119

@@ -10,36 +10,39 @@ · 9 unmodified lines
10	* We assume sprg3 has the physical address of the current	10	* We assume sprg3 has the physical address of the current
11	* task's thread_struct.	11	* task's thread_struct.
12	*/	12	*/
13	.macro EXCEPTION_PROLOG handle_dar_dsisr=0	13	.macro EXCEPTION_PROLOG trapno name handle_dar_dsisr=0
14	EXCEPTION_PROLOG_0 handle_dar_dsisr=\handle_dar_dsisr	14	EXCEPTION_PROLOG_0 handle_dar_dsisr=\handle_dar_dsisr
15	EXCEPTION_PROLOG_1	15	EXCEPTION_PROLOG_1
16	EXCEPTION_PROLOG_2 handle_dar_dsisr=\handle_dar_dsisr	16	EXCEPTION_PROLOG_2 \trapno \name handle_dar_dsisr=\handle_dar_dsisr
17	.endm	17	.endm
18		18
19	.macro EXCEPTION_PROLOG_0 handle_dar_dsisr=0	19	.macro EXCEPTION_PROLOG_0 handle_dar_dsisr=0
20	mtspr SPRN_SPRG_SCRATCH0,r10	20	mtspr SPRN_SPRG_SCRATCH0,r10
21	mtspr SPRN_SPRG_SCRATCH1,r11	21	mtspr SPRN_SPRG_SCRATCH1,r11
22	#ifdef CONFIG_VMAP_STACK
23	mfspr r10, SPRN_SPRG_THREAD	22	mfspr r10, SPRN_SPRG_THREAD
24	.if \handle_dar_dsisr	23	.if \handle_dar_dsisr
		24	#ifdef CONFIG_40x
		25	mfspr r11, SPRN_DEAR
		26	#else
25	mfspr r11, SPRN_DAR	27	mfspr r11, SPRN_DAR
		28	#endif
26	stw r11, DAR(r10)	29	stw r11, DAR(r10)
		30	#ifdef CONFIG_40x
		31	mfspr r11, SPRN_ESR
		32	#else
27	mfspr r11, SPRN_DSISR	33	mfspr r11, SPRN_DSISR
		34	#endif
28	stw r11, DSISR(r10)	35	stw r11, DSISR(r10)
29	.endif	36	.endif
30	mfspr r11, SPRN_SRR0	37	mfspr r11, SPRN_SRR0
31	stw r11, SRR0(r10)	38	stw r11, SRR0(r10)
32	#endif
33	mfspr r11, SPRN_SRR1 /* check whether user or kernel */	39	mfspr r11, SPRN_SRR1 /* check whether user or kernel */
34	#ifdef CONFIG_VMAP_STACK
35	stw r11, SRR1(r10)	40	stw r11, SRR1(r10)
36	#endif
37	mfcr r10	41	mfcr r10
38	andi. r11, r11, MSR_PR	42	andi. r11, r11, MSR_PR
39	.endm	43	.endm
40		44
41	.macro EXCEPTION_PROLOG_1 for_rtas=0	45	.macro EXCEPTION_PROLOG_1
42	#ifdef CONFIG_VMAP_STACK
43	mtspr SPRN_SPRG_SCRATCH2,r1	46	mtspr SPRN_SPRG_SCRATCH2,r1
44	subi r1, r1, INT_FRAME_SIZE /* use r1 if kernel */	47	subi r1, r1, INT_FRAME_SIZE /* use r1 if kernel */
45	beq 1f	48	beq 1f
@@ -47,32 +50,33 @@ · 1 unmodified line
47	lwz r1,TASK_STACK-THREAD(r1)	50	lwz r1,TASK_STACK-THREAD(r1)
48	addi r1, r1, THREAD_SIZE - INT_FRAME_SIZE	51	addi r1, r1, THREAD_SIZE - INT_FRAME_SIZE
49	1:	52	1:
		53	#ifdef CONFIG_VMAP_STACK
50	mtcrf 0x3f, r1	54	mtcrf 0x3f, r1
51	bt 32 - THREAD_ALIGN_SHIFT, stack_overflow	55	bt 32 - THREAD_ALIGN_SHIFT, vmap_stack_overflow
52	#else
53	subi r11, r1, INT_FRAME_SIZE /* use r1 if kernel */
54	beq 1f
55	mfspr r11,SPRN_SPRG_THREAD
56	lwz r11,TASK_STACK-THREAD(r11)
57	addi r11, r11, THREAD_SIZE - INT_FRAME_SIZE
58	1: tophys(r11, r11)
59	#endif	56	#endif
60	.endm	57	.endm
61		58
62	.macro EXCEPTION_PROLOG_2 handle_dar_dsisr=0	59	.macro EXCEPTION_PROLOG_2 trapno name handle_dar_dsisr=0
63	#ifdef CONFIG_VMAP_STACK	60	#ifdef CONFIG_PPC_8xx
64	li r11, MSR_KERNEL & ~(MSR_IR \| MSR_RI) /* can take DTLB miss */	61	.if \handle_dar_dsisr
65	mtmsr r11	62	li r11, RPN_PATTERN
66	isync	63	mtspr SPRN_DAR, r11 /* Tag DAR, to be used in DTLB Error */
		64	.endif
		65	#endif
		66	LOAD_REG_IMMEDIATE(r11, MSR_KERNEL & ~MSR_RI) /* re-enable MMU */
		67	mtspr SPRN_SRR1, r11
		68	lis r11, 1f@h
		69	ori r11, r11, 1f@l
		70	mtspr SPRN_SRR0, r11
67	mfspr r11, SPRN_SPRG_SCRATCH2	71	mfspr r11, SPRN_SPRG_SCRATCH2
		72	rfi
		73
		74	.text
		75	\name\()_virt:
		76	1:
68	stw r11,GPR1(r1)	77	stw r11,GPR1(r1)
69	stw r11,0(r1)	78	stw r11,0(r1)
70	mr r11, r1	79	mr r11, r1
71	#else
72	stw r1,GPR1(r11)
73	stw r1,0(r11)
74	tovirt(r1, r11) /* set new kernel sp */
75	#endif
76	stw r10,_CCR(r11) /* save registers */	80	stw r10,_CCR(r11) /* save registers */
77	stw r12,GPR12(r11)	81	stw r12,GPR12(r11)
78	stw r9,GPR9(r11)	82	stw r9,GPR9(r11)
@@ -82,7 +86,6 @@ · 3 unmodified lines
82	stw r12,GPR11(r11)	86	stw r12,GPR11(r11)
83	mflr r10	87	mflr r10
84	stw r10,_LINK(r11)	88	stw r10,_LINK(r11)
85	#ifdef CONFIG_VMAP_STACK
86	mfspr r12, SPRN_SPRG_THREAD	89	mfspr r12, SPRN_SPRG_THREAD
87	tovirt(r12, r12)	90	tovirt(r12, r12)
88	.if \handle_dar_dsisr	91	.if \handle_dar_dsisr
@@ -93,26 +96,48 @@ · 4 unmodified lines
93	.endif	96	.endif
94	lwz r9, SRR1(r12)	97	lwz r9, SRR1(r12)
95	lwz r12, SRR0(r12)	98	lwz r12, SRR0(r12)
96	#else
97	mfspr r12,SPRN_SRR0
98	mfspr r9,SPRN_SRR1
99	#endif
100	#ifdef CONFIG_40x	99	#ifdef CONFIG_40x
101	rlwinm r9,r9,0,14,12 /* clear MSR_WE (necessary?) */	100	rlwinm r9,r9,0,14,12 /* clear MSR_WE (necessary?) */
		101	#elif defined(CONFIG_PPC_8xx)
		102	mtspr SPRN_EID, r2 /* Set MSR_RI */
102	#else	103	#else
103	#ifdef CONFIG_VMAP_STACK	104	li r10, MSR_KERNEL /* can take exceptions */
104	li r10, MSR_KERNEL & ~MSR_IR /* can take exceptions */
105	#else
106	li r10,MSR_KERNEL & ~(MSR_IR\|MSR_DR) /* can take exceptions */
107	#endif
108	mtmsr r10 /* (except for mach check in rtas) */	105	mtmsr r10 /* (except for mach check in rtas) */
109	#endif	106	#endif
110	stw r0,GPR0(r11)	107	COMMON_EXCEPTION_PROLOG_END \trapno
		108	_ASM_NOKPROBE_SYMBOL(\name\()_virt)
		109	.endm
		110
		111	.macro COMMON_EXCEPTION_PROLOG_END trapno
		112	stw r0,GPR0(r1)
111	lis r10,STACK_FRAME_REGS_MARKER@ha /* exception frame marker */	113	lis r10,STACK_FRAME_REGS_MARKER@ha /* exception frame marker */
112	addi r10,r10,STACK_FRAME_REGS_MARKER@l	114	addi r10,r10,STACK_FRAME_REGS_MARKER@l
113	stw r10,8(r11)	115	stw r10,8(r1)
114	SAVE_4GPRS(3, r11)	116	li r10, \trapno
115	SAVE_2GPRS(7, r11)	117	stw r10,_TRAP(r1)
		118	SAVE_4GPRS(3, r1)
		119	SAVE_2GPRS(7, r1)
		120	SAVE_NVGPRS(r1)
		121	stw r2,GPR2(r1)
		122	stw r12,_NIP(r1)
		123	stw r9,_MSR(r1)
		124	mfctr r10
		125	mfspr r2,SPRN_SPRG_THREAD
		126	stw r10,_CTR(r1)
		127	tovirt(r2, r2)
		128	mfspr r10,SPRN_XER
		129	addi r2, r2, -THREAD
		130	stw r10,_XER(r1)
		131	addi r3,r1,STACK_FRAME_OVERHEAD
		132	.endm
		133
		134	.macro prepare_transfer_to_handler
		135	#ifdef CONFIG_PPC_BOOK3S_32
		136	andi. r12,r9,MSR_PR
		137	bne 777f
		138	bl prepare_transfer_to_handler
		139	777:
		140	#endif
116	.endm	141	.endm
117		142
118	.macro SYSCALL_ENTRY trapno	143	.macro SYSCALL_ENTRY trapno
@@ -156,54 +181,6 @@ · 37 unmodified lines
156	b transfer_to_syscall /* jump to handler */	181	b transfer_to_syscall /* jump to handler */
157	.endm	182	.endm
158		183
159	.macro save_dar_dsisr_on_stack reg1, reg2, sp
160	#ifndef CONFIG_VMAP_STACK
161	mfspr \reg1, SPRN_DAR
162	mfspr \reg2, SPRN_DSISR
163	stw \reg1, _DAR(\sp)
164	stw \reg2, _DSISR(\sp)
165	#endif
166	.endm
167
168	.macro get_and_save_dar_dsisr_on_stack reg1, reg2, sp
169	#ifdef CONFIG_VMAP_STACK
170	lwz \reg1, _DAR(\sp)
171	lwz \reg2, _DSISR(\sp)
172	#else
173	save_dar_dsisr_on_stack \reg1, \reg2, \sp
174	#endif
175	.endm
176
177	.macro tovirt_vmstack dst, src
178	#ifdef CONFIG_VMAP_STACK
179	tovirt(\dst, \src)
180	#else
181	.ifnc \dst, \src
182	mr \dst, \src
183	.endif
184	#endif
185	.endm
186
187	.macro tovirt_novmstack dst, src
188	#ifndef CONFIG_VMAP_STACK
189	tovirt(\dst, \src)
190	#else
191	.ifnc \dst, \src
192	mr \dst, \src
193	.endif
194	#endif
195	.endm
196
197	.macro tophys_novmstack dst, src
198	#ifndef CONFIG_VMAP_STACK
199	tophys(\dst, \src)
200	#else
201	.ifnc \dst, \src
202	mr \dst, \src
203	.endif
204	#endif
205	.endm
206
207	/*	184	/*
208	* Note: code which follows this uses cr0.eq (set if from kernel),	185	* Note: code which follows this uses cr0.eq (set if from kernel),
209	* r11, r12 (SRR0), and r9 (SRR1).	186	* r11, r12 (SRR0), and r9 (SRR1).
@@ -217,41 +194,29 @@ · 7 unmodified lines
217	*/	194	*/
218	#ifdef CONFIG_PPC_BOOK3S	195	#ifdef CONFIG_PPC_BOOK3S
219	#define START_EXCEPTION(n, label) \	196	#define START_EXCEPTION(n, label) \
		197	__HEAD; \
220	. = n; \	198	. = n; \
221	DO_KVM n; \	199	DO_KVM n; \
222	label:	200	label:
223		201
224	#else	202	#else
225	#define START_EXCEPTION(n, label) \	203	#define START_EXCEPTION(n, label) \
		204	__HEAD; \
226	. = n; \	205	. = n; \
227	label:	206	label:
228		207
229	#endif	208	#endif
230		209
231	#define EXCEPTION(n, label, hdlr, xfer) \	210	#define EXCEPTION(n, label, hdlr) \
232	START_EXCEPTION(n, label) \	211	START_EXCEPTION(n, label) \
233	EXCEPTION_PROLOG; \	212	EXCEPTION_PROLOG n label; \
234	addi r3,r1,STACK_FRAME_OVERHEAD; \	213	prepare_transfer_to_handler; \
235	xfer(n, hdlr)	214	bl hdlr; \
236		215	b interrupt_return
237	#define EXC_XFER_TEMPLATE(hdlr, trap, msr, tfer, ret) \
238	li r10,trap; \
239	stw r10,_TRAP(r11); \
240	LOAD_REG_IMMEDIATE(r10, msr); \
241	bl tfer; \
242	.long hdlr; \
243	.long ret
244
245	#define EXC_XFER_STD(n, hdlr) \
246	EXC_XFER_TEMPLATE(hdlr, n, MSR_KERNEL, transfer_to_handler_full, \
247	ret_from_except_full)
248
249	#define EXC_XFER_LITE(n, hdlr) \
250	EXC_XFER_TEMPLATE(hdlr, n+1, MSR_KERNEL, transfer_to_handler, \
251	ret_from_except)
252		216
253	.macro vmap_stack_overflow_exception	217	.macro vmap_stack_overflow_exception
254	#ifdef CONFIG_VMAP_STACK	218	__HEAD
		219	vmap_stack_overflow:
255	#ifdef CONFIG_SMP	220	#ifdef CONFIG_SMP
256	mfspr r1, SPRN_SPRG_THREAD	221	mfspr r1, SPRN_SPRG_THREAD
257	lwz r1, TASK_CPU - THREAD(r1)	222	lwz r1, TASK_CPU - THREAD(r1)
@@ -261,16 +226,11 @@ label: · 3 unmodified lines
261	lis r1, emergency_ctx@ha	226	lis r1, emergency_ctx@ha
262	#endif	227	#endif
263	lwz r1, emergency_ctx@l(r1)	228	lwz r1, emergency_ctx@l(r1)
264	cmpwi cr1, r1, 0	229	addi r1, r1, THREAD_SIZE - INT_FRAME_SIZE
265	bne cr1, 1f	230	EXCEPTION_PROLOG_2 0 vmap_stack_overflow
266	lis r1, init_thread_union@ha	231	prepare_transfer_to_handler
267	addi r1, r1, init_thread_union@l	232	bl stack_overflow_exception
268	1: addi r1, r1, THREAD_SIZE - INT_FRAME_SIZE	233	b interrupt_return
269	EXCEPTION_PROLOG_2
270	SAVE_NVGPRS(r11)
271	addi r3, r1, STACK_FRAME_OVERHEAD
272	EXC_XFER_STD(0, stack_overflow_exception)
273	#endif
274	.endm	234	.endm
275		235
276	#endif /* __HEAD_32_H__ */	236	#endif /* __HEAD_32_H__ */

arch/powerpc/kernel/head_40x.S+147−124

@@ -89,7 +89,11 @@ _ENTRY(crit_srr0) · 88 unmodified lines
89	.space 4	89	.space 4
90	_ENTRY(crit_srr1)	90	_ENTRY(crit_srr1)
91	.space 4	91	.space 4
92	_ENTRY(saved_ksp_limit)	92	_ENTRY(crit_r1)
		93	.space 4
		94	_ENTRY(crit_dear)
		95	.space 4
		96	_ENTRY(crit_esr)
93	.space 4	97	.space 4
94		98
95	/*	99	/*
@@ -100,42 +104,62 @@ _ENTRY(saved_ksp_limit) · 4 unmodified lines
100	* Instead we use a couple of words of memory at low physical addresses.	104	* Instead we use a couple of words of memory at low physical addresses.
101	* This is OK since we don't support SMP on these processors.	105	* This is OK since we don't support SMP on these processors.
102	*/	106	*/
103	#define CRITICAL_EXCEPTION_PROLOG \	107	.macro CRITICAL_EXCEPTION_PROLOG trapno name
104	stw r10,crit_r10@l(0); /* save two registers to work with */\	108	stw r10,crit_r10@l(0) /* save two registers to work with */
105	stw r11,crit_r11@l(0); \	109	stw r11,crit_r11@l(0)
106	mfcr r10; /* save CR in r10 for now */\	110	mfspr r10,SPRN_SRR0
107	mfspr r11,SPRN_SRR3; /* check whether user or kernel */\	111	mfspr r11,SPRN_SRR1
108	andi. r11,r11,MSR_PR; \	112	stw r10,crit_srr0@l(0)
109	lis r11,critirq_ctx@ha; \	113	stw r11,crit_srr1@l(0)
110	tophys(r11,r11); \	114	mfspr r10,SPRN_DEAR
111	lwz r11,critirq_ctx@l(r11); \	115	mfspr r11,SPRN_ESR
112	beq 1f; \	116	stw r10,crit_dear@l(0)
113	/* COMING FROM USER MODE */ \	117	stw r11,crit_esr@l(0)
114	mfspr r11,SPRN_SPRG_THREAD; /* if from user, start at top of */\	118	mfcr r10 /* save CR in r10 for now */
115	lwz r11,TASK_STACK-THREAD(r11); /* this thread's kernel stack */\	119	mfspr r11,SPRN_SRR3 /* check whether user or kernel */
116	1: addi r11,r11,THREAD_SIZE-INT_FRAME_SIZE; /* Alloc an excpt frm */\	120	andi. r11,r11,MSR_PR
117	tophys(r11,r11); \	121	lis r11,(critirq_ctx-PAGE_OFFSET)@ha
118	stw r10,_CCR(r11); /* save various registers */\	122	lwz r11,(critirq_ctx-PAGE_OFFSET)@l(r11)
119	stw r12,GPR12(r11); \	123	beq 1f
120	stw r9,GPR9(r11); \	124	/* COMING FROM USER MODE */
121	mflr r10; \	125	mfspr r11,SPRN_SPRG_THREAD /* if from user, start at top of */
122	stw r10,_LINK(r11); \	126	lwz r11,TASK_STACK-THREAD(r11) /* this thread's kernel stack */
123	mfspr r12,SPRN_DEAR; /* save DEAR and ESR in the frame */\	127	1: stw r1,crit_r1@l(0)
124	stw r12,_DEAR(r11); /* since they may have had stuff */\	128	addi r1,r11,THREAD_SIZE-INT_FRAME_SIZE /* Alloc an excpt frm */
125	mfspr r9,SPRN_ESR; /* in them at the point where the */\	129	LOAD_REG_IMMEDIATE(r11, MSR_KERNEL & ~(MSR_ME\|MSR_DE\|MSR_CE)) /* re-enable MMU */
126	stw r9,_ESR(r11); /* exception was taken */\	130	mtspr SPRN_SRR1, r11
127	mfspr r12,SPRN_SRR2; \	131	lis r11, 1f@h
128	stw r1,GPR1(r11); \	132	ori r11, r11, 1f@l
129	mfspr r9,SPRN_SRR3; \	133	mtspr SPRN_SRR0, r11
130	stw r1,0(r11); \	134	rfi
131	tovirt(r1,r11); \	135
132	rlwinm r9,r9,0,14,12; /* clear MSR_WE (necessary?) */\	136	.text
133	stw r0,GPR0(r11); \	137	1:
134	lis r10, STACK_FRAME_REGS_MARKER@ha; /* exception frame marker */\	138	\name\()_virt:
135	addi r10, r10, STACK_FRAME_REGS_MARKER@l; \	139	lwz r11,crit_r1@l(0)
136	stw r10, 8(r11); \	140	stw r11,GPR1(r1)
137	SAVE_4GPRS(3, r11); \	141	stw r11,0(r1)
138	SAVE_2GPRS(7, r11)	142	mr r11,r1
		143	stw r10,_CCR(r11) /* save various registers */
		144	stw r12,GPR12(r11)
		145	stw r9,GPR9(r11)
		146	mflr r10
		147	stw r10,_LINK(r11)
		148	lis r9,PAGE_OFFSET@ha
		149	lwz r10,crit_r10@l(r9)
		150	lwz r12,crit_r11@l(r9)
		151	stw r10,GPR10(r11)
		152	stw r12,GPR11(r11)
		153	lwz r12,crit_dear@l(r9)
		154	lwz r9,crit_esr@l(r9)
		155	stw r12,_DEAR(r11) /* since they may have had stuff */
		156	stw r9,_ESR(r11) /* exception was taken */
		157	mfspr r12,SPRN_SRR2
		158	mfspr r9,SPRN_SRR3
		159	rlwinm r9,r9,0,14,12 /* clear MSR_WE (necessary?) */
		160	COMMON_EXCEPTION_PROLOG_END \trapno + 2
		161	_ASM_NOKPROBE_SYMBOL(\name\()_virt)
		162	.endm
139		163
140	/*	164	/*
141	* State at this point:	165	* State at this point:
@@ -155,10 +179,10 @@ _ENTRY(saved_ksp_limit) · 13 unmodified lines
155	*/	179	*/
156	#define CRITICAL_EXCEPTION(n, label, hdlr) \	180	#define CRITICAL_EXCEPTION(n, label, hdlr) \
157	START_EXCEPTION(n, label); \	181	START_EXCEPTION(n, label); \
158	CRITICAL_EXCEPTION_PROLOG; \	182	CRITICAL_EXCEPTION_PROLOG n label; \
159	addi r3,r1,STACK_FRAME_OVERHEAD; \	183	prepare_transfer_to_handler; \
160	EXC_XFER_TEMPLATE(hdlr, n+2, (MSR_KERNEL & ~(MSR_ME\|MSR_DE\|MSR_CE)), \	184	bl hdlr; \
161	crit_transfer_to_handler, ret_from_crit_exc)	185	b ret_from_crit_exc
162		186
163	/*	187	/*
164	* 0x0100 - Critical Interrupt Exception	188	* 0x0100 - Critical Interrupt Exception
@@ -178,69 +202,67 @@ _ENTRY(saved_ksp_limit) · 13 unmodified lines
178	* if they can't resolve the lightweight TLB fault.	202	* if they can't resolve the lightweight TLB fault.
179	*/	203	*/
180	START_EXCEPTION(0x0300, DataStorage)	204	START_EXCEPTION(0x0300, DataStorage)
181	EXCEPTION_PROLOG	205	EXCEPTION_PROLOG 0x300 DataStorage handle_dar_dsisr=1
182	mfspr r5, SPRN_ESR /* Grab the ESR, save it */	206	prepare_transfer_to_handler
183	stw r5, _ESR(r11)	207	bl do_page_fault
184	mfspr r4, SPRN_DEAR /* Grab the DEAR, save it */	208	b interrupt_return
185	stw r4, _DEAR(r11)
186	EXC_XFER_LITE(0x300, handle_page_fault)
187		209
188	/*	210	/*
189	* 0x0400 - Instruction Storage Exception	211	* 0x0400 - Instruction Storage Exception
190	* This is caused by a fetch from non-execute or guarded pages.	212	* This is caused by a fetch from non-execute or guarded pages.
191	*/	213	*/
192	START_EXCEPTION(0x0400, InstructionAccess)	214	START_EXCEPTION(0x0400, InstructionAccess)
193	EXCEPTION_PROLOG	215	EXCEPTION_PROLOG 0x400 InstructionAccess
194	li r5,0	216	li r5,0
195	stw r5, _ESR(r11) /* Zero ESR */	217	stw r5, _ESR(r11) /* Zero ESR */
196	stw r12, _DEAR(r11) /* SRR0 as DEAR */	218	stw r12, _DEAR(r11) /* SRR0 as DEAR */
197	EXC_XFER_LITE(0x400, handle_page_fault)	219	prepare_transfer_to_handler
		220	bl do_page_fault
		221	b interrupt_return
198		222
199	/* 0x0500 - External Interrupt Exception */	223	/* 0x0500 - External Interrupt Exception */
200	EXCEPTION(0x0500, HardwareInterrupt, do_IRQ, EXC_XFER_LITE)	224	EXCEPTION(0x0500, HardwareInterrupt, do_IRQ)
201		225
202	/* 0x0600 - Alignment Exception */	226	/* 0x0600 - Alignment Exception */
203	START_EXCEPTION(0x0600, Alignment)	227	START_EXCEPTION(0x0600, Alignment)
204	EXCEPTION_PROLOG	228	EXCEPTION_PROLOG 0x600 Alignment handle_dar_dsisr=1
205	mfspr r4,SPRN_DEAR /* Grab the DEAR and save it */	229	prepare_transfer_to_handler
206	stw r4,_DEAR(r11)	230	bl alignment_exception
207	addi r3,r1,STACK_FRAME_OVERHEAD	231	REST_NVGPRS(r1)
208	EXC_XFER_STD(0x600, alignment_exception)	232	b interrupt_return
209		233
210	/* 0x0700 - Program Exception */	234	/* 0x0700 - Program Exception */
211	START_EXCEPTION(0x0700, ProgramCheck)	235	START_EXCEPTION(0x0700, ProgramCheck)
212	EXCEPTION_PROLOG	236	EXCEPTION_PROLOG 0x700 ProgramCheck handle_dar_dsisr=1
213	mfspr r4,SPRN_ESR /* Grab the ESR and save it */	237	prepare_transfer_to_handler
214	stw r4,_ESR(r11)	238	bl program_check_exception
215	addi r3,r1,STACK_FRAME_OVERHEAD	239	REST_NVGPRS(r1)
216	EXC_XFER_STD(0x700, program_check_exception)	240	b interrupt_return
217		241
218	EXCEPTION(0x0800, Trap_08, unknown_exception, EXC_XFER_STD)	242	EXCEPTION(0x0800, Trap_08, unknown_exception)
219	EXCEPTION(0x0900, Trap_09, unknown_exception, EXC_XFER_STD)	243	EXCEPTION(0x0900, Trap_09, unknown_exception)
220	EXCEPTION(0x0A00, Trap_0A, unknown_exception, EXC_XFER_STD)	244	EXCEPTION(0x0A00, Trap_0A, unknown_exception)
221	EXCEPTION(0x0B00, Trap_0B, unknown_exception, EXC_XFER_STD)	245	EXCEPTION(0x0B00, Trap_0B, unknown_exception)
222		246
223	/* 0x0C00 - System Call Exception */	247	/* 0x0C00 - System Call Exception */
224	START_EXCEPTION(0x0C00, SystemCall)	248	START_EXCEPTION(0x0C00, SystemCall)
225	SYSCALL_ENTRY 0xc00	249	SYSCALL_ENTRY 0xc00
226	/* Trap_0D is commented out to get more space for system call exception */	250	/* Trap_0D is commented out to get more space for system call exception */
227		251
228	/* EXCEPTION(0x0D00, Trap_0D, unknown_exception, EXC_XFER_STD) */	252	/* EXCEPTION(0x0D00, Trap_0D, unknown_exception) */
229	EXCEPTION(0x0E00, Trap_0E, unknown_exception, EXC_XFER_STD)	253	EXCEPTION(0x0E00, Trap_0E, unknown_exception)
230	EXCEPTION(0x0F00, Trap_0F, unknown_exception, EXC_XFER_STD)	254	EXCEPTION(0x0F00, Trap_0F, unknown_exception)
231		255
232	/* 0x1000 - Programmable Interval Timer (PIT) Exception */	256	/* 0x1000 - Programmable Interval Timer (PIT) Exception */
233	. = 0x1000	257	START_EXCEPTION(0x1000, DecrementerTrap)
234	b Decrementer	258	b Decrementer
235		259
236	/* 0x1010 - Fixed Interval Timer (FIT) Exception	260	/* 0x1010 - Fixed Interval Timer (FIT) Exception */
237	*/	261	START_EXCEPTION(0x1010, FITExceptionTrap)
238	. = 0x1010
239	b FITException	262	b FITException
240		263
241	/* 0x1020 - Watchdog Timer (WDT) Exception	264	/* 0x1020 - Watchdog Timer (WDT) Exception */
242	*/	265	START_EXCEPTION(0x1020, WDTExceptionTrap)
243	. = 0x1020
244	b WDTException	266	b WDTException
245		267
246	/* 0x1100 - Data TLB Miss Exception	268	/* 0x1100 - Data TLB Miss Exception
@@ -249,13 +271,13 @@ _ENTRY(saved_ksp_limit) · 2 unmodified lines
249	* load TLB entries from the page table if they exist.	271	* load TLB entries from the page table if they exist.
250	*/	272	*/
251	START_EXCEPTION(0x1100, DTLBMiss)	273	START_EXCEPTION(0x1100, DTLBMiss)
252	mtspr SPRN_SPRG_SCRATCH0, r10 /* Save some working registers */	274	mtspr SPRN_SPRG_SCRATCH5, r10 /* Save some working registers */
253	mtspr SPRN_SPRG_SCRATCH1, r11	275	mtspr SPRN_SPRG_SCRATCH6, r11
254	mtspr SPRN_SPRG_SCRATCH3, r12	276	mtspr SPRN_SPRG_SCRATCH3, r12
255	mtspr SPRN_SPRG_SCRATCH4, r9	277	mtspr SPRN_SPRG_SCRATCH4, r9
256	mfcr r12	278	mfcr r12
257	mfspr r9, SPRN_PID	279	mfspr r9, SPRN_PID
258	mtspr SPRN_SPRG_SCRATCH5, r9	280	rlwimi r12, r9, 0, 0xff
259	mfspr r10, SPRN_DEAR /* Get faulting address */	281	mfspr r10, SPRN_DEAR /* Get faulting address */
260		282
261	/* If we are faulting a kernel address, we have to use the	283	/* If we are faulting a kernel address, we have to use the
@@ -316,13 +338,12 @@ _ENTRY(saved_ksp_limit) · 54 unmodified lines
316	/* The bailout. Restore registers to pre-exception conditions	338	/* The bailout. Restore registers to pre-exception conditions
317	* and call the heavyweights to help us out.	339	* and call the heavyweights to help us out.
318	*/	340	*/
319	mfspr r9, SPRN_SPRG_SCRATCH5	341	mtspr SPRN_PID, r12
320	mtspr SPRN_PID, r9	342	mtcrf 0x80, r12
321	mtcr r12
322	mfspr r9, SPRN_SPRG_SCRATCH4	343	mfspr r9, SPRN_SPRG_SCRATCH4
323	mfspr r12, SPRN_SPRG_SCRATCH3	344	mfspr r12, SPRN_SPRG_SCRATCH3
324	mfspr r11, SPRN_SPRG_SCRATCH1	345	mfspr r11, SPRN_SPRG_SCRATCH6
325	mfspr r10, SPRN_SPRG_SCRATCH0	346	mfspr r10, SPRN_SPRG_SCRATCH5
326	b DataStorage	347	b DataStorage
327		348
328	/* 0x1200 - Instruction TLB Miss Exception	349	/* 0x1200 - Instruction TLB Miss Exception
@@ -330,13 +351,13 @@ _ENTRY(saved_ksp_limit) · 1 unmodified line
330	* registers and bailout to a different point.	351	* registers and bailout to a different point.
331	*/	352	*/
332	START_EXCEPTION(0x1200, ITLBMiss)	353	START_EXCEPTION(0x1200, ITLBMiss)
333	mtspr SPRN_SPRG_SCRATCH0, r10 /* Save some working registers */	354	mtspr SPRN_SPRG_SCRATCH5, r10 /* Save some working registers */
334	mtspr SPRN_SPRG_SCRATCH1, r11	355	mtspr SPRN_SPRG_SCRATCH6, r11
335	mtspr SPRN_SPRG_SCRATCH3, r12	356	mtspr SPRN_SPRG_SCRATCH3, r12
336	mtspr SPRN_SPRG_SCRATCH4, r9	357	mtspr SPRN_SPRG_SCRATCH4, r9
337	mfcr r12	358	mfcr r12
338	mfspr r9, SPRN_PID	359	mfspr r9, SPRN_PID
339	mtspr SPRN_SPRG_SCRATCH5, r9	360	rlwimi r12, r9, 0, 0xff
340	mfspr r10, SPRN_SRR0 /* Get faulting address */	361	mfspr r10, SPRN_SRR0 /* Get faulting address */
341		362
342	/* If we are faulting a kernel address, we have to use the	363	/* If we are faulting a kernel address, we have to use the
@@ -397,28 +418,27 @@ _ENTRY(saved_ksp_limit) · 54 unmodified lines
397	/* The bailout. Restore registers to pre-exception conditions	418	/* The bailout. Restore registers to pre-exception conditions
398	* and call the heavyweights to help us out.	419	* and call the heavyweights to help us out.
399	*/	420	*/
400	mfspr r9, SPRN_SPRG_SCRATCH5	421	mtspr SPRN_PID, r12
401	mtspr SPRN_PID, r9	422	mtcrf 0x80, r12
402	mtcr r12
403	mfspr r9, SPRN_SPRG_SCRATCH4	423	mfspr r9, SPRN_SPRG_SCRATCH4
404	mfspr r12, SPRN_SPRG_SCRATCH3	424	mfspr r12, SPRN_SPRG_SCRATCH3
405	mfspr r11, SPRN_SPRG_SCRATCH1	425	mfspr r11, SPRN_SPRG_SCRATCH6
406	mfspr r10, SPRN_SPRG_SCRATCH0	426	mfspr r10, SPRN_SPRG_SCRATCH5
407	b InstructionAccess	427	b InstructionAccess
408		428
409	EXCEPTION(0x1300, Trap_13, unknown_exception, EXC_XFER_STD)	429	EXCEPTION(0x1300, Trap_13, unknown_exception)
410	EXCEPTION(0x1400, Trap_14, unknown_exception, EXC_XFER_STD)	430	EXCEPTION(0x1400, Trap_14, unknown_exception)
411	EXCEPTION(0x1500, Trap_15, unknown_exception, EXC_XFER_STD)	431	EXCEPTION(0x1500, Trap_15, unknown_exception)
412	EXCEPTION(0x1600, Trap_16, unknown_exception, EXC_XFER_STD)	432	EXCEPTION(0x1600, Trap_16, unknown_exception)
413	EXCEPTION(0x1700, Trap_17, unknown_exception, EXC_XFER_STD)	433	EXCEPTION(0x1700, Trap_17, unknown_exception)
414	EXCEPTION(0x1800, Trap_18, unknown_exception, EXC_XFER_STD)	434	EXCEPTION(0x1800, Trap_18, unknown_exception)
415	EXCEPTION(0x1900, Trap_19, unknown_exception, EXC_XFER_STD)	435	EXCEPTION(0x1900, Trap_19, unknown_exception)
416	EXCEPTION(0x1A00, Trap_1A, unknown_exception, EXC_XFER_STD)	436	EXCEPTION(0x1A00, Trap_1A, unknown_exception)
417	EXCEPTION(0x1B00, Trap_1B, unknown_exception, EXC_XFER_STD)	437	EXCEPTION(0x1B00, Trap_1B, unknown_exception)
418	EXCEPTION(0x1C00, Trap_1C, unknown_exception, EXC_XFER_STD)	438	EXCEPTION(0x1C00, Trap_1C, unknown_exception)
419	EXCEPTION(0x1D00, Trap_1D, unknown_exception, EXC_XFER_STD)	439	EXCEPTION(0x1D00, Trap_1D, unknown_exception)
420	EXCEPTION(0x1E00, Trap_1E, unknown_exception, EXC_XFER_STD)	440	EXCEPTION(0x1E00, Trap_1E, unknown_exception)
421	EXCEPTION(0x1F00, Trap_1F, unknown_exception, EXC_XFER_STD)	441	EXCEPTION(0x1F00, Trap_1F, unknown_exception)
422		442
423	/* Check for a single step debug exception while in an exception	443	/* Check for a single step debug exception while in an exception
424	* handler before state has been saved. This is to catch the case	444	* handler before state has been saved. This is to catch the case
@@ -435,7 +455,7 @@ _ENTRY(saved_ksp_limit) · 10 unmodified lines
435	*/	455	*/
436	/* 0x2000 - Debug Exception */	456	/* 0x2000 - Debug Exception */
437	START_EXCEPTION(0x2000, DebugTrap)	457	START_EXCEPTION(0x2000, DebugTrap)
438	CRITICAL_EXCEPTION_PROLOG	458	CRITICAL_EXCEPTION_PROLOG 0x2000 DebugTrap
439		459
440	/*	460	/*
441	* If this is a single step or branch-taken exception in an	461	* If this is a single step or branch-taken exception in an
@@ -477,32 +497,35 @@ _ENTRY(saved_ksp_limit) · 35 unmodified lines
477	/* continue normal handling for a critical exception... */	497	/* continue normal handling for a critical exception... */
478	2: mfspr r4,SPRN_DBSR	498	2: mfspr r4,SPRN_DBSR
479	stw r4,_ESR(r11) /* DebugException takes DBSR in _ESR */	499	stw r4,_ESR(r11) /* DebugException takes DBSR in _ESR */
480	addi r3,r1,STACK_FRAME_OVERHEAD	500	prepare_transfer_to_handler
481	EXC_XFER_TEMPLATE(DebugException, 0x2002, \	501	bl DebugException
482	(MSR_KERNEL & ~(MSR_ME\|MSR_DE\|MSR_CE)), \	502	b ret_from_crit_exc
483	crit_transfer_to_handler, ret_from_crit_exc)
484		503
485	/* Programmable Interval Timer (PIT) Exception. (from 0x1000) */	504	/* Programmable Interval Timer (PIT) Exception. (from 0x1000) */
		505	__HEAD
486	Decrementer:	506	Decrementer:
487	EXCEPTION_PROLOG	507	EXCEPTION_PROLOG 0x1000 Decrementer
488	lis r0,TSR_PIS@h	508	lis r0,TSR_PIS@h
489	mtspr SPRN_TSR,r0 /* Clear the PIT exception */	509	mtspr SPRN_TSR,r0 /* Clear the PIT exception */
490	addi r3,r1,STACK_FRAME_OVERHEAD	510	prepare_transfer_to_handler
491	EXC_XFER_LITE(0x1000, timer_interrupt)	511	bl timer_interrupt
		512	b interrupt_return
492		513
493	/* Fixed Interval Timer (FIT) Exception. (from 0x1010) */	514	/* Fixed Interval Timer (FIT) Exception. (from 0x1010) */
		515	__HEAD
494	FITException:	516	FITException:
495	EXCEPTION_PROLOG	517	EXCEPTION_PROLOG 0x1010 FITException
496	addi r3,r1,STACK_FRAME_OVERHEAD;	518	prepare_transfer_to_handler
497	EXC_XFER_STD(0x1010, unknown_exception)	519	bl unknown_exception
		520	b interrupt_return
498		521
499	/* Watchdog Timer (WDT) Exception. (from 0x1020) */	522	/* Watchdog Timer (WDT) Exception. (from 0x1020) */
		523	__HEAD
500	WDTException:	524	WDTException:
501	CRITICAL_EXCEPTION_PROLOG;	525	CRITICAL_EXCEPTION_PROLOG 0x1020 WDTException
502	addi r3,r1,STACK_FRAME_OVERHEAD;	526	prepare_transfer_to_handler
503	EXC_XFER_TEMPLATE(WatchdogException, 0x1020+2,	527	bl WatchdogException
504	(MSR_KERNEL & ~(MSR_ME\|MSR_DE\|MSR_CE)),	528	b ret_from_crit_exc
505	crit_transfer_to_handler, ret_from_crit_exc)
506		529
507	/* Other PowerPC processors, namely those derived from the 6xx-series	530	/* Other PowerPC processors, namely those derived from the 6xx-series
508	* have vectors from 0x2100 through 0x2F00 defined, but marked as reserved.	531	* have vectors from 0x2100 through 0x2F00 defined, but marked as reserved.
@@ -510,6 +533,7 @@ WDTException: · 1 unmodified line
510	* reserved.	533	* reserved.
511	*/	534	*/
512		535
		536	__HEAD
513	/* Damn, I came up one instruction too many to fit into the	537	/* Damn, I came up one instruction too many to fit into the
514	* exception space :-). Both the instruction and data TLB	538	* exception space :-). Both the instruction and data TLB
515	* miss get to this point to load the TLB.	539	* miss get to this point to load the TLB.
@@ -543,13 +567,12 @@ finish_tlb_load: · 27 unmodified lines
543		567
544	/* Done...restore registers and get out of here.	568	/* Done...restore registers and get out of here.
545	*/	569	*/
546	mfspr r9, SPRN_SPRG_SCRATCH5	570	mtspr SPRN_PID, r12
547	mtspr SPRN_PID, r9	571	mtcrf 0x80, r12
548	mtcr r12
549	mfspr r9, SPRN_SPRG_SCRATCH4	572	mfspr r9, SPRN_SPRG_SCRATCH4
550	mfspr r12, SPRN_SPRG_SCRATCH3	573	mfspr r12, SPRN_SPRG_SCRATCH3
551	mfspr r11, SPRN_SPRG_SCRATCH1	574	mfspr r11, SPRN_SPRG_SCRATCH6
552	mfspr r10, SPRN_SPRG_SCRATCH0	575	mfspr r10, SPRN_SPRG_SCRATCH5
553	rfi /* Should sync shadow TLBs */	576	rfi /* Should sync shadow TLBs */
554	b . /* prevent prefetch past rfi */	577	b . /* prevent prefetch past rfi */
555		578

arch/powerpc/kernel/head_44x.S+4−6

 	INSTRUCTION_STORAGE_EXCEPTION
 	/* External Input Interrupt */
-	EXCEPTION(0x0500, BOOKE_INTERRUPT_EXTERNAL, ExternalInput, \
+	EXCEPTION(0x0500, BOOKE_INTERRUPT_EXTERNAL, ExternalInput, do_IRQ)
-		  do_IRQ, EXC_XFER_LITE)
 	/* Alignment Interrupt */
 	ALIGNMENT_EXCEPTION
 	FP_UNAVAILABLE_EXCEPTION
 #else
 	EXCEPTION(0x2010, BOOKE_INTERRUPT_FP_UNAVAIL, \
-		  FloatingPointUnavailable, unknown_exception, EXC_XFER_STD)
+		  FloatingPointUnavailable, unknown_exception)
 #endif
 	/* System Call Interrupt */
 	START_EXCEPTION(SystemCall)
 	/* Auxiliary Processor Unavailable Interrupt */
 	EXCEPTION(0x2020, BOOKE_INTERRUPT_AP_UNAVAIL, \
-		  AuxillaryProcessorUnavailable, unknown_exception, EXC_XFER_STD)
+		  AuxillaryProcessorUnavailable, unknown_exception)
 	/* Decrementer Interrupt */
 	DECREMENTER_EXCEPTION
 	/* Fixed Internal Timer Interrupt */
 	/* TODO: Add FIT support */
-	EXCEPTION(0x1010, BOOKE_INTERRUPT_FIT, FixedIntervalTimer, \
+	EXCEPTION(0x1010, BOOKE_INTERRUPT_FIT, FixedIntervalTimer, unknown_exception)
-		  unknown_exception, EXC_XFER_STD)
 	/* Watchdog Timer Interrupt */
 	/* TODO: Add watchdog support */

arch/powerpc/kernel/head_8xx.S+71−85

 #include <asm/ptrace.h>
 #include <asm/export.h>
 #include <asm/code-patching-asm.h>
+#include <asm/interrupt.h>
+/*
+ * Value for the bits that have fixed value in RPN entries.
+ * Also used for tagging DAR for DTLBerror.
+ */
+#define RPN_PATTERN	0x00f0
 #include "head_32.h"
 #endif
 .endm
-/*
- * Value for the bits that have fixed value in RPN entries.
- * Also used for tagging DAR for DTLBerror.
- */
-#define RPN_PATTERN	0x00f0
 #define PAGE_SHIFT_512K		19
 #define PAGE_SHIFT_8M		23
 #endif
 /* System reset */
-	EXCEPTION(0x100, Reset, system_reset_exception, EXC_XFER_STD)
+	EXCEPTION(INTERRUPT_SYSTEM_RESET, Reset, system_reset_exception)
 /* Machine check */
-	. = 0x200
+	START_EXCEPTION(INTERRUPT_MACHINE_CHECK, MachineCheck)
-MachineCheck:
+	EXCEPTION_PROLOG INTERRUPT_MACHINE_CHECK MachineCheck handle_dar_dsisr=1
-	EXCEPTION_PROLOG handle_dar_dsisr=1
+	prepare_transfer_to_handler
-	save_dar_dsisr_on_stack r4, r5, r11
+	bl	machine_check_exception
-	li	r6, RPN_PATTERN
+	b	interrupt_return
-	mtspr	SPRN_DAR, r6	/* Tag DAR, to be used in DTLB Error */
-	addi r3,r1,STACK_FRAME_OVERHEAD
-	EXC_XFER_STD(0x200, machine_check_exception)
 /* External interrupt */
-	EXCEPTION(0x500, HardwareInterrupt, do_IRQ, EXC_XFER_LITE)
+	EXCEPTION(INTERRUPT_EXTERNAL, HardwareInterrupt, do_IRQ)
 /* Alignment exception */
-	. = 0x600
+	START_EXCEPTION(INTERRUPT_ALIGNMENT, Alignment)
-Alignment:
+	EXCEPTION_PROLOG INTERRUPT_ALIGNMENT Alignment handle_dar_dsisr=1
-	EXCEPTION_PROLOG handle_dar_dsisr=1
+	prepare_transfer_to_handler
-	save_dar_dsisr_on_stack r4, r5, r11
+	bl	alignment_exception
-	li	r6, RPN_PATTERN
+	REST_NVGPRS(r1)
-	mtspr	SPRN_DAR, r6	/* Tag DAR, to be used in DTLB Error */
+	b	interrupt_return
-	addi	r3,r1,STACK_FRAME_OVERHEAD
-	b	.Lalignment_exception_ool
 /* Program check exception */
-	EXCEPTION(0x700, ProgramCheck, program_check_exception, EXC_XFER_STD)
+	START_EXCEPTION(INTERRUPT_PROGRAM, ProgramCheck)
+	EXCEPTION_PROLOG INTERRUPT_PROGRAM ProgramCheck
+	prepare_transfer_to_handler
+	bl	program_check_exception
+	REST_NVGPRS(r1)
+	b	interrupt_return
 /* Decrementer */
-	EXCEPTION(0x900, Decrementer, timer_interrupt, EXC_XFER_LITE)
+	EXCEPTION(INTERRUPT_DECREMENTER, Decrementer, timer_interrupt)
-	/* With VMAP_STACK there's not enough room for this at 0x600 */
-	. = 0xa00
-.Lalignment_exception_ool:
-	EXC_XFER_STD(0x600, alignment_exception)
 /* System call */
-	. = 0xc00
+	START_EXCEPTION(INTERRUPT_SYSCALL, SystemCall)
-SystemCall:
+	SYSCALL_ENTRY	INTERRUPT_SYSCALL
-	SYSCALL_ENTRY	0xc00
 /* Single step - not used on 601 */
-	EXCEPTION(0xd00, SingleStep, single_step_exception, EXC_XFER_STD)
+	EXCEPTION(INTERRUPT_TRACE, SingleStep, single_step_exception)
 /* On the MPC8xx, this is a software emulation interrupt.  It occurs
  * for all unimplemented and illegal instructions.
  */
-	EXCEPTION(0x1000, SoftEmu, emulation_assist_interrupt, EXC_XFER_STD)
+	START_EXCEPTION(INTERRUPT_SOFT_EMU_8xx, SoftEmu)
+	EXCEPTION_PROLOG INTERRUPT_SOFT_EMU_8xx SoftEmu
+	prepare_transfer_to_handler
+	bl	emulation_assist_interrupt
+	REST_NVGPRS(r1)
+	b	interrupt_return
-	. = 0x1100
 /*
  * For the MPC8xx, this is a software tablewalk to load the instruction
  * TLB.  The task switch loads the M_TWB register with the pointer to the first
 #define INVALIDATE_ADJACENT_PAGES_CPU15(addr, tmp)
 #endif
-InstructionTLBMiss:
+	START_EXCEPTION(INTERRUPT_INST_TLB_MISS_8xx, InstructionTLBMiss)
 	mtspr	SPRN_SPRG_SCRATCH2, r10
 	mtspr	SPRN_M_TW, r11
 	rfi
 #endif
-	. = 0x1200
+	START_EXCEPTION(INTERRUPT_DATA_TLB_MISS_8xx, DataStoreTLBMiss)
-DataStoreTLBMiss:
 	mtspr	SPRN_SPRG_SCRATCH2, r10
 	mtspr	SPRN_M_TW, r11
 	mfcr	r11
  * to many reasons, such as executing guarded memory or illegal instruction
  * addresses.  There is nothing to do but handle a big time error fault.
  */
-	. = 0x1300
+	START_EXCEPTION(INTERRUPT_INST_TLB_ERROR_8xx, InstructionTLBError)
-InstructionTLBError:
+	/* 0x400 is InstructionAccess exception, needed by bad_page_fault() */
-	EXCEPTION_PROLOG
+	EXCEPTION_PROLOG INTERRUPT_INST_STORAGE InstructionTLBError
 	andis.	r5,r9,DSISR_SRR1_MATCH_32S@h /* Filter relevant SRR1 bits */
 	andis.	r10,r9,SRR1_ISI_NOPT@h
 	beq+	.Litlbie
 	tlbie	r12
-	/* 0x400 is InstructionAccess exception, needed by bad_page_fault() */
 .Litlbie:
 	stw	r12, _DAR(r11)
 	stw	r5, _DSISR(r11)
-	EXC_XFER_LITE(0x400, handle_page_fault)
+	prepare_transfer_to_handler
+	bl	do_page_fault
+	b	interrupt_return
 /* This is the data TLB error on the MPC8xx.  This could be due to
  * many reasons, including a dirty update to a pte.  We bail out to
  * a higher level function that can handle it.
  */
-	. = 0x1400
+	START_EXCEPTION(INTERRUPT_DATA_TLB_ERROR_8xx, DataTLBError)
-DataTLBError:
 	EXCEPTION_PROLOG_0 handle_dar_dsisr=1
 	mfspr	r11, SPRN_DAR
 	cmpwi	cr1, r11, RPN_PATTERN
 	beq-	cr1, FixupDAR	/* must be a buggy dcbX, icbi insn. */
 DARFixed:/* Return from dcbx instruction bug workaround */
-#ifdef CONFIG_VMAP_STACK
-	li	r11, RPN_PATTERN
-	mtspr	SPRN_DAR, r11	/* Tag DAR, to be used in DTLB Error */
-#endif
 	EXCEPTION_PROLOG_1
-	EXCEPTION_PROLOG_2 handle_dar_dsisr=1
+	/* 0x300 is DataAccess exception, needed by bad_page_fault() */
-	get_and_save_dar_dsisr_on_stack r4, r5, r11
+	EXCEPTION_PROLOG_2 INTERRUPT_DATA_STORAGE DataTLBError handle_dar_dsisr=1
+	lwz	r4, _DAR(r11)
+	lwz	r5, _DSISR(r11)
 	andis.	r10,r5,DSISR_NOHPTE@h
 	beq+	.Ldtlbie
 	tlbie	r4
 .Ldtlbie:
-#ifndef CONFIG_VMAP_STACK
+	prepare_transfer_to_handler
-	li	r10,RPN_PATTERN
+	bl	do_page_fault
-	mtspr	SPRN_DAR,r10	/* Tag DAR, to be used in DTLB Error */
+	b	interrupt_return
-#endif
-	/* 0x300 is DataAccess exception, needed by bad_page_fault() */
-	EXC_XFER_LITE(0x300, handle_page_fault)
-stack_overflow:
+#ifdef CONFIG_VMAP_STACK
 	vmap_stack_overflow_exception
+#endif
 /* On the MPC8xx, these next four traps are used for development
  * support of breakpoints and such.  Someday I will get around to
  * using them.
  */
-do_databreakpoint:
+	START_EXCEPTION(INTERRUPT_DATA_BREAKPOINT_8xx, DataBreakpoint)
-	EXCEPTION_PROLOG_1
-	EXCEPTION_PROLOG_2 handle_dar_dsisr=1
-	addi	r3,r1,STACK_FRAME_OVERHEAD
-	mfspr	r4,SPRN_BAR
-	stw	r4,_DAR(r11)
-#ifndef CONFIG_VMAP_STACK
-	mfspr	r5,SPRN_DSISR
-	stw	r5,_DSISR(r11)
-#endif
-	EXC_XFER_STD(0x1c00, do_break)
-	. = 0x1c00
-DataBreakpoint:
 	EXCEPTION_PROLOG_0 handle_dar_dsisr=1
 	mfspr	r11, SPRN_SRR0
 	cmplwi	cr1, r11, (.Ldtlbie - PAGE_OFFSET)@l
 	cmplwi	cr7, r11, (.Litlbie - PAGE_OFFSET)@l
 	cror	4*cr1+eq, 4*cr1+eq, 4*cr7+eq
-	bne	cr1, do_databreakpoint
+	bne	cr1, 1f
 	mtcr	r10
 	mfspr	r10, SPRN_SPRG_SCRATCH0
 	mfspr	r11, SPRN_SPRG_SCRATCH1
 	rfi
+1:	EXCEPTION_PROLOG_1
+	EXCEPTION_PROLOG_2 INTERRUPT_DATA_BREAKPOINT_8xx DataBreakpoint handle_dar_dsisr=1
+	mfspr	r4,SPRN_BAR
+	stw	r4,_DAR(r11)
+	prepare_transfer_to_handler
+	bl	do_break
+	REST_NVGPRS(r1)
+	b	interrupt_return
 #ifdef CONFIG_PERF_EVENTS
-	. = 0x1d00
+	START_EXCEPTION(INTERRUPT_INST_BREAKPOINT_8xx, InstructionBreakpoint)
-InstructionBreakpoint:
 	mtspr	SPRN_SPRG_SCRATCH0, r10
 	lwz	r10, (instruction_counter - PAGE_OFFSET)@l(0)
 	addi	r10, r10, -1
 	mfspr	r10, SPRN_SPRG_SCRATCH0
 	rfi
 #else
-	EXCEPTION(0x1d00, Trap_1d, unknown_exception, EXC_XFER_STD)
+	EXCEPTION(INTERRUPT_INST_BREAKPOINT_8xx, Trap_1d, unknown_exception)
 #endif
-	EXCEPTION(0x1e00, Trap_1e, unknown_exception, EXC_XFER_STD)
+	EXCEPTION(0x1e00, Trap_1e, unknown_exception)
-	EXCEPTION(0x1f00, Trap_1f, unknown_exception, EXC_XFER_STD)
+	EXCEPTION(0x1f00, Trap_1f, unknown_exception)
+	__HEAD
 	. = 0x2000
 /* This is the procedure to calculate the data EA for buggy dcbx,dcbi instructions
 152:
 	mfdar	r11
 	mtctr	r11			/* restore ctr reg from DAR */
-#ifdef CONFIG_VMAP_STACK
 	mfspr	r11, SPRN_SPRG_THREAD
 	stw	r10, DAR(r11)
 	mfspr	r10, SPRN_DSISR
 	stw	r10, DSISR(r11)
-#else
-	mtdar	r10			/* save fault EA to DAR */
-#endif
 	mfspr	r10,SPRN_M_TW
 	b	DARFixed		/* Go back to normal TLB handling */
 swapper_pg_dir:
 	.space	PGD_TABLE_SIZE
-/* Room for two PTE table poiners, usually the kernel and current user
+/* Room for two PTE table pointers, usually the kernel and current user
  * pointer to their respective root page table (pgdir).
  */
 	.globl	abatron_pteptrs

arch/powerpc/kernel/head_book3s_32.S+99−143

@@ -31,6 +31,7 @@ · 30 unmodified lines
31	#include <asm/kvm_book3s_asm.h>	31	#include <asm/kvm_book3s_asm.h>
32	#include <asm/export.h>	32	#include <asm/export.h>
33	#include <asm/feature-fixups.h>	33	#include <asm/feature-fixups.h>
		34	#include <asm/interrupt.h>
34		35
35	#include "head_32.h"	36	#include "head_32.h"
36		37
@@ -239,7 +240,7 @@ __secondary_hold_acknowledge: · 202 unmodified lines
239	/* System reset */	240	/* System reset */
240	/* core99 pmac starts the seconary here by changing the vector, and	241	/* core99 pmac starts the seconary here by changing the vector, and
241	putting it back to what it was (unknown_async_exception) when done. */	242	putting it back to what it was (unknown_async_exception) when done. */
242	EXCEPTION(0x100, Reset, unknown_async_exception, EXC_XFER_STD)	243	EXCEPTION(INTERRUPT_SYSTEM_RESET, Reset, unknown_async_exception)
243		244
244	/* Machine check */	245	/* Machine check */
245	/*	246	/*
@@ -255,40 +256,28 @@ __secondary_hold_acknowledge: · 9 unmodified lines
255	* pointer when we take an exception from supervisor mode.)	256	* pointer when we take an exception from supervisor mode.)
256	* -- paulus.	257	* -- paulus.
257	*/	258	*/
258	. = 0x200	259	START_EXCEPTION(INTERRUPT_MACHINE_CHECK, MachineCheck)
259	DO_KVM 0x200
260	MachineCheck:
261	EXCEPTION_PROLOG_0	260	EXCEPTION_PROLOG_0
262	#ifdef CONFIG_PPC_CHRP	261	#ifdef CONFIG_PPC_CHRP
263	#ifdef CONFIG_VMAP_STACK
264	mtspr SPRN_SPRG_SCRATCH2,r1	262	mtspr SPRN_SPRG_SCRATCH2,r1
265	mfspr r1, SPRN_SPRG_THREAD	263	mfspr r1, SPRN_SPRG_THREAD
266	lwz r1, RTAS_SP(r1)	264	lwz r1, RTAS_SP(r1)
267	cmpwi cr1, r1, 0	265	cmpwi cr1, r1, 0
268	bne cr1, 7f	266	bne cr1, 7f
269	mfspr r1, SPRN_SPRG_SCRATCH2	267	mfspr r1, SPRN_SPRG_SCRATCH2
270	#else
271	mfspr r11, SPRN_SPRG_THREAD
272	lwz r11, RTAS_SP(r11)
273	cmpwi cr1, r11, 0
274	bne cr1, 7f
275	#endif
276	#endif /* CONFIG_PPC_CHRP */	268	#endif /* CONFIG_PPC_CHRP */
277	EXCEPTION_PROLOG_1 for_rtas=1	269	EXCEPTION_PROLOG_1
278	7: EXCEPTION_PROLOG_2	270	7: EXCEPTION_PROLOG_2 0x200 MachineCheck
279	addi r3,r1,STACK_FRAME_OVERHEAD
280	#ifdef CONFIG_PPC_CHRP	271	#ifdef CONFIG_PPC_CHRP
281	beq cr1, machine_check_tramp	272	beq cr1, 1f
282	twi 31, 0, 0	273	twi 31, 0, 0
283	#else
284	b machine_check_tramp
285	#endif	274	#endif
		275	1: prepare_transfer_to_handler
		276	bl machine_check_exception
		277	b interrupt_return
286		278
287	/* Data access exception. */	279	/* Data access exception. */
288	. = 0x300	280	START_EXCEPTION(INTERRUPT_DATA_STORAGE, DataAccess)
289	DO_KVM 0x300
290	DataAccess:
291	#ifdef CONFIG_VMAP_STACK
292	#ifdef CONFIG_PPC_BOOK3S_604	281	#ifdef CONFIG_PPC_BOOK3S_604
293	BEGIN_MMU_FTR_SECTION	282	BEGIN_MMU_FTR_SECTION
294	mtspr SPRN_SPRG_SCRATCH2,r10	283	mtspr SPRN_SPRG_SCRATCH2,r10
@@ -309,30 +298,20 @@ ALT_MMU_FTR_SECTION_END_IFSET(MMU_FTR_HPTE_TABLE) · 14 unmodified lines
309	#endif	298	#endif
310	1: EXCEPTION_PROLOG_0 handle_dar_dsisr=1	299	1: EXCEPTION_PROLOG_0 handle_dar_dsisr=1
311	EXCEPTION_PROLOG_1	300	EXCEPTION_PROLOG_1
312	b handle_page_fault_tramp_1	301	EXCEPTION_PROLOG_2 INTERRUPT_DATA_STORAGE DataAccess handle_dar_dsisr=1
313	#else /* CONFIG_VMAP_STACK */	302	prepare_transfer_to_handler
314	EXCEPTION_PROLOG handle_dar_dsisr=1	303	lwz r5, _DSISR(r11)
315	get_and_save_dar_dsisr_on_stack r4, r5, r11	304	andis. r0, r5, DSISR_DABRMATCH@h
316	#ifdef CONFIG_PPC_BOOK3S_604	305	bne- 1f
317	BEGIN_MMU_FTR_SECTION	306	bl do_page_fault
318	andis. r0, r5, (DSISR_BAD_FAULT_32S \| DSISR_DABRMATCH)@h	307	b interrupt_return
319	bne handle_page_fault_tramp_2 /* if not, try to put a PTE */	308	1: bl do_break
320	rlwinm r3, r5, 32 - 15, 21, 21 /* DSISR_STORE -> _PAGE_RW */	309	REST_NVGPRS(r1)
321	bl hash_page	310	b interrupt_return
322	b handle_page_fault_tramp_1	311
323	MMU_FTR_SECTION_ELSE
324	#endif
325	b handle_page_fault_tramp_2
326	#ifdef CONFIG_PPC_BOOK3S_604
327	ALT_MMU_FTR_SECTION_END_IFSET(MMU_FTR_HPTE_TABLE)
328	#endif
329	#endif /* CONFIG_VMAP_STACK */
330		312
331	/* Instruction access exception. */	313	/* Instruction access exception. */
332	. = 0x400	314	START_EXCEPTION(INTERRUPT_INST_STORAGE, InstructionAccess)
333	DO_KVM 0x400
334	InstructionAccess:
335	#ifdef CONFIG_VMAP_STACK
336	mtspr SPRN_SPRG_SCRATCH0,r10	315	mtspr SPRN_SPRG_SCRATCH0,r10
337	mtspr SPRN_SPRG_SCRATCH1,r11	316	mtspr SPRN_SPRG_SCRATCH1,r11
338	mfspr r10, SPRN_SPRG_THREAD	317	mfspr r10, SPRN_SPRG_THREAD
@@ -352,43 +331,35 @@ END_MMU_FTR_SECTION_IFSET(MMU_FTR_HPTE_TABLE) · 13 unmodified lines
352	andi. r11, r11, MSR_PR	331	andi. r11, r11, MSR_PR
353		332
354	EXCEPTION_PROLOG_1	333	EXCEPTION_PROLOG_1
355	EXCEPTION_PROLOG_2	334	EXCEPTION_PROLOG_2 INTERRUPT_INST_STORAGE InstructionAccess
356	#else /* CONFIG_VMAP_STACK */
357	EXCEPTION_PROLOG
358	andis. r0,r9,SRR1_ISI_NOPT@h /* no pte found? */
359	beq 1f /* if so, try to put a PTE */
360	li r3,0 /* into the hash table */
361	mr r4,r12 /* SRR0 is fault address */
362	#ifdef CONFIG_PPC_BOOK3S_604
363	BEGIN_MMU_FTR_SECTION
364	bl hash_page
365	END_MMU_FTR_SECTION_IFSET(MMU_FTR_HPTE_TABLE)
366	#endif
367	#endif /* CONFIG_VMAP_STACK */
368	andis. r5,r9,DSISR_SRR1_MATCH_32S@h /* Filter relevant SRR1 bits */	335	andis. r5,r9,DSISR_SRR1_MATCH_32S@h /* Filter relevant SRR1 bits */
369	stw r5, _DSISR(r11)	336	stw r5, _DSISR(r11)
370	stw r12, _DAR(r11)	337	stw r12, _DAR(r11)
371	EXC_XFER_LITE(0x400, handle_page_fault)	338	prepare_transfer_to_handler
		339	bl do_page_fault
		340	b interrupt_return
372		341
373	/* External interrupt */	342	/* External interrupt */
374	EXCEPTION(0x500, HardwareInterrupt, do_IRQ, EXC_XFER_LITE)	343	EXCEPTION(INTERRUPT_EXTERNAL, HardwareInterrupt, do_IRQ)
375		344
376	/* Alignment exception */	345	/* Alignment exception */
377	. = 0x600	346	START_EXCEPTION(INTERRUPT_ALIGNMENT, Alignment)
378	DO_KVM 0x600	347	EXCEPTION_PROLOG INTERRUPT_ALIGNMENT Alignment handle_dar_dsisr=1
379	Alignment:	348	prepare_transfer_to_handler
380	EXCEPTION_PROLOG handle_dar_dsisr=1	349	bl alignment_exception
381	save_dar_dsisr_on_stack r4, r5, r11	350	REST_NVGPRS(r1)
382	addi r3,r1,STACK_FRAME_OVERHEAD	351	b interrupt_return
383	b alignment_exception_tramp
384		352
385	/* Program check exception */	353	/* Program check exception */
386	EXCEPTION(0x700, ProgramCheck, program_check_exception, EXC_XFER_STD)	354	START_EXCEPTION(INTERRUPT_PROGRAM, ProgramCheck)
		355	EXCEPTION_PROLOG INTERRUPT_PROGRAM ProgramCheck
		356	prepare_transfer_to_handler
		357	bl program_check_exception
		358	REST_NVGPRS(r1)
		359	b interrupt_return
387		360
388	/* Floating-point unavailable */	361	/* Floating-point unavailable */
389	. = 0x800	362	START_EXCEPTION(0x800, FPUnavailable)
390	DO_KVM 0x800
391	FPUnavailable:
392	#ifdef CONFIG_PPC_FPU	363	#ifdef CONFIG_PPC_FPU
393	BEGIN_FTR_SECTION	364	BEGIN_FTR_SECTION
394	/*	365	/*
@@ -397,30 +368,29 @@ BEGIN_FTR_SECTION · 2 unmodified lines
397	*/	368	*/
398	b ProgramCheck	369	b ProgramCheck
399	END_FTR_SECTION_IFSET(CPU_FTR_FPU_UNAVAILABLE)	370	END_FTR_SECTION_IFSET(CPU_FTR_FPU_UNAVAILABLE)
400	EXCEPTION_PROLOG	371	EXCEPTION_PROLOG INTERRUPT_FP_UNAVAIL FPUnavailable
401	beq 1f	372	beq 1f
402	bl load_up_fpu /* if from user, just load it up */	373	bl load_up_fpu /* if from user, just load it up */
403	b fast_exception_return	374	b fast_exception_return
404	1: addi r3,r1,STACK_FRAME_OVERHEAD	375	1: prepare_transfer_to_handler
405	EXC_XFER_LITE(0x800, kernel_fp_unavailable_exception)	376	bl kernel_fp_unavailable_exception
		377	b interrupt_return
406	#else	378	#else
407	b ProgramCheck	379	b ProgramCheck
408	#endif	380	#endif
409		381
410	/* Decrementer */	382	/* Decrementer */
411	EXCEPTION(0x900, Decrementer, timer_interrupt, EXC_XFER_LITE)	383	EXCEPTION(INTERRUPT_DECREMENTER, Decrementer, timer_interrupt)
412		384
413	EXCEPTION(0xa00, Trap_0a, unknown_exception, EXC_XFER_STD)	385	EXCEPTION(0xa00, Trap_0a, unknown_exception)
414	EXCEPTION(0xb00, Trap_0b, unknown_exception, EXC_XFER_STD)	386	EXCEPTION(0xb00, Trap_0b, unknown_exception)
415		387
416	/* System call */	388	/* System call */
417	. = 0xc00	389	START_EXCEPTION(INTERRUPT_SYSCALL, SystemCall)
418	DO_KVM 0xc00	390	SYSCALL_ENTRY INTERRUPT_SYSCALL
419	SystemCall:
420	SYSCALL_ENTRY 0xc00
421		391
422	EXCEPTION(0xd00, SingleStep, single_step_exception, EXC_XFER_STD)	392	EXCEPTION(INTERRUPT_TRACE, SingleStep, single_step_exception)
423	EXCEPTION(0xe00, Trap_0e, unknown_exception, EXC_XFER_STD)	393	EXCEPTION(0xe00, Trap_0e, unknown_exception)
424		394
425	/*	395	/*
426	* The Altivec unavailable trap is at 0x0f20. Foo.	396	* The Altivec unavailable trap is at 0x0f20. Foo.
@@ -430,19 +400,18 @@ SystemCall: · 3 unmodified lines
430	* non-altivec kernel running on a machine with altivec just	400	* non-altivec kernel running on a machine with altivec just
431	* by executing an altivec instruction.	401	* by executing an altivec instruction.
432	*/	402	*/
433	. = 0xf00	403	START_EXCEPTION(INTERRUPT_PERFMON, PerformanceMonitorTrap)
434	DO_KVM 0xf00
435	b PerformanceMonitor	404	b PerformanceMonitor
436		405
437	. = 0xf20	406	START_EXCEPTION(INTERRUPT_ALTIVEC_UNAVAIL, AltiVecUnavailableTrap)
438	DO_KVM 0xf20
439	b AltiVecUnavailable	407	b AltiVecUnavailable
440		408
		409	__HEAD
441	/*	410	/*
442	* Handle TLB miss for instruction on 603/603e.	411	* Handle TLB miss for instruction on 603/603e.
443	* Note: we get an alternate set of r0 - r3 to use automatically.	412	* Note: we get an alternate set of r0 - r3 to use automatically.
444	*/	413	*/
445	. = 0x1000	414	. = INTERRUPT_INST_TLB_MISS_603
446	InstructionTLBMiss:	415	InstructionTLBMiss:
447	/*	416	/*
448	* r0: scratch	417	* r0: scratch
@@ -508,7 +477,7 @@ InstructionAddressInvalid: · 59 unmodified lines
508	/*	477	/*
509	* Handle TLB miss for DATA Load operation on 603/603e	478	* Handle TLB miss for DATA Load operation on 603/603e
510	*/	479	*/
511	. = 0x1100	480	. = INTERRUPT_DATA_LOAD_TLB_MISS_603
512	DataLoadTLBMiss:	481	DataLoadTLBMiss:
513	/*	482	/*
514	* r0: scratch	483	* r0: scratch
@@ -586,7 +555,7 @@ DataAddressInvalid: · 71 unmodified lines
586	/*	555	/*
587	* Handle TLB miss for DATA Store on 603/603e	556	* Handle TLB miss for DATA Store on 603/603e
588	*/	557	*/
589	. = 0x1200	558	. = INTERRUPT_DATA_STORE_TLB_MISS_603
590	DataStoreTLBMiss:	559	DataStoreTLBMiss:
591	/*	560	/*
592	* r0: scratch	561	* r0: scratch
@@ -650,57 +619,39 @@ END_MMU_FTR_SECTION_IFSET(MMU_FTR_NEED_DTLB_SW_LRU) · 57 unmodified lines
650	#define TAUException unknown_async_exception	619	#define TAUException unknown_async_exception
651	#endif	620	#endif
652		621
653	EXCEPTION(0x1300, Trap_13, instruction_breakpoint_exception, EXC_XFER_STD)	622	EXCEPTION(0x1300, Trap_13, instruction_breakpoint_exception)
654	EXCEPTION(0x1400, SMI, SMIException, EXC_XFER_STD)	623	EXCEPTION(0x1400, SMI, SMIException)
655	EXCEPTION(0x1500, Trap_15, unknown_exception, EXC_XFER_STD)	624	EXCEPTION(0x1500, Trap_15, unknown_exception)
656	EXCEPTION(0x1600, Trap_16, altivec_assist_exception, EXC_XFER_STD)	625	EXCEPTION(0x1600, Trap_16, altivec_assist_exception)
657	EXCEPTION(0x1700, Trap_17, TAUException, EXC_XFER_STD)	626	EXCEPTION(0x1700, Trap_17, TAUException)
658	EXCEPTION(0x1800, Trap_18, unknown_exception, EXC_XFER_STD)	627	EXCEPTION(0x1800, Trap_18, unknown_exception)
659	EXCEPTION(0x1900, Trap_19, unknown_exception, EXC_XFER_STD)	628	EXCEPTION(0x1900, Trap_19, unknown_exception)
660	EXCEPTION(0x1a00, Trap_1a, unknown_exception, EXC_XFER_STD)	629	EXCEPTION(0x1a00, Trap_1a, unknown_exception)
661	EXCEPTION(0x1b00, Trap_1b, unknown_exception, EXC_XFER_STD)	630	EXCEPTION(0x1b00, Trap_1b, unknown_exception)
662	EXCEPTION(0x1c00, Trap_1c, unknown_exception, EXC_XFER_STD)	631	EXCEPTION(0x1c00, Trap_1c, unknown_exception)
663	EXCEPTION(0x1d00, Trap_1d, unknown_exception, EXC_XFER_STD)	632	EXCEPTION(0x1d00, Trap_1d, unknown_exception)
664	EXCEPTION(0x1e00, Trap_1e, unknown_exception, EXC_XFER_STD)	633	EXCEPTION(0x1e00, Trap_1e, unknown_exception)
665	EXCEPTION(0x1f00, Trap_1f, unknown_exception, EXC_XFER_STD)	634	EXCEPTION(0x1f00, Trap_1f, unknown_exception)
666	EXCEPTION(0x2000, RunMode, RunModeException, EXC_XFER_STD)	635	EXCEPTION(0x2000, RunMode, RunModeException)
667	EXCEPTION(0x2100, Trap_21, unknown_exception, EXC_XFER_STD)	636	EXCEPTION(0x2100, Trap_21, unknown_exception)
668	EXCEPTION(0x2200, Trap_22, unknown_exception, EXC_XFER_STD)	637	EXCEPTION(0x2200, Trap_22, unknown_exception)
669	EXCEPTION(0x2300, Trap_23, unknown_exception, EXC_XFER_STD)	638	EXCEPTION(0x2300, Trap_23, unknown_exception)
670	EXCEPTION(0x2400, Trap_24, unknown_exception, EXC_XFER_STD)	639	EXCEPTION(0x2400, Trap_24, unknown_exception)
671	EXCEPTION(0x2500, Trap_25, unknown_exception, EXC_XFER_STD)	640	EXCEPTION(0x2500, Trap_25, unknown_exception)
672	EXCEPTION(0x2600, Trap_26, unknown_exception, EXC_XFER_STD)	641	EXCEPTION(0x2600, Trap_26, unknown_exception)
673	EXCEPTION(0x2700, Trap_27, unknown_exception, EXC_XFER_STD)	642	EXCEPTION(0x2700, Trap_27, unknown_exception)
674	EXCEPTION(0x2800, Trap_28, unknown_exception, EXC_XFER_STD)	643	EXCEPTION(0x2800, Trap_28, unknown_exception)
675	EXCEPTION(0x2900, Trap_29, unknown_exception, EXC_XFER_STD)	644	EXCEPTION(0x2900, Trap_29, unknown_exception)
676	EXCEPTION(0x2a00, Trap_2a, unknown_exception, EXC_XFER_STD)	645	EXCEPTION(0x2a00, Trap_2a, unknown_exception)
677	EXCEPTION(0x2b00, Trap_2b, unknown_exception, EXC_XFER_STD)	646	EXCEPTION(0x2b00, Trap_2b, unknown_exception)
678	EXCEPTION(0x2c00, Trap_2c, unknown_exception, EXC_XFER_STD)	647	EXCEPTION(0x2c00, Trap_2c, unknown_exception)
679	EXCEPTION(0x2d00, Trap_2d, unknown_exception, EXC_XFER_STD)	648	EXCEPTION(0x2d00, Trap_2d, unknown_exception)
680	EXCEPTION(0x2e00, Trap_2e, unknown_exception, EXC_XFER_STD)	649	EXCEPTION(0x2e00, Trap_2e, unknown_exception)
681	EXCEPTION(0x2f00, Trap_2f, unknown_exception, EXC_XFER_STD)	650	EXCEPTION(0x2f00, Trap_2f, unknown_exception)
682		651
		652	__HEAD
683	. = 0x3000	653	. = 0x3000
684		654
685	machine_check_tramp:
686	EXC_XFER_STD(0x200, machine_check_exception)
687
688	alignment_exception_tramp:
689	EXC_XFER_STD(0x600, alignment_exception)
690
691	handle_page_fault_tramp_1:
692	#ifdef CONFIG_VMAP_STACK
693	EXCEPTION_PROLOG_2 handle_dar_dsisr=1
694	#endif
695	lwz r5, _DSISR(r11)
696	/* fall through */
697	handle_page_fault_tramp_2:
698	andis. r0, r5, DSISR_DABRMATCH@h
699	bne- 1f
700	EXC_XFER_LITE(0x300, handle_page_fault)
701	1: EXC_XFER_STD(0x300, do_break)
702
703	#ifdef CONFIG_VMAP_STACK
704	#ifdef CONFIG_PPC_BOOK3S_604	655	#ifdef CONFIG_PPC_BOOK3S_604
705	.macro save_regs_thread thread	656	.macro save_regs_thread thread
706	stw r0, THR0(\thread)	657	stw r0, THR0(\thread)
@@ -775,26 +726,31 @@ fast_hash_page_return: · 68 unmodified lines
775	rfi	726	rfi
776	#endif /* CONFIG_PPC_BOOK3S_604 */	727	#endif /* CONFIG_PPC_BOOK3S_604 */
777		728
778	stack_overflow:	729	#ifdef CONFIG_VMAP_STACK
779	vmap_stack_overflow_exception	730	vmap_stack_overflow_exception
780	#endif	731	#endif
781		732
		733	__HEAD
782	AltiVecUnavailable:	734	AltiVecUnavailable:
783	EXCEPTION_PROLOG	735	EXCEPTION_PROLOG 0xf20 AltiVecUnavailable
784	#ifdef CONFIG_ALTIVEC	736	#ifdef CONFIG_ALTIVEC
785	beq 1f	737	beq 1f
786	bl load_up_altivec /* if from user, just load it up */	738	bl load_up_altivec /* if from user, just load it up */
787	b fast_exception_return	739	b fast_exception_return
788	#endif /* CONFIG_ALTIVEC */	740	#endif /* CONFIG_ALTIVEC */
789	1: addi r3,r1,STACK_FRAME_OVERHEAD	741	1: prepare_transfer_to_handler
790	EXC_XFER_LITE(0xf20, altivec_unavailable_exception)	742	bl altivec_unavailable_exception
		743	b interrupt_return
791		744
		745	__HEAD
792	PerformanceMonitor:	746	PerformanceMonitor:
793	EXCEPTION_PROLOG	747	EXCEPTION_PROLOG 0xf00 PerformanceMonitor
794	addi r3,r1,STACK_FRAME_OVERHEAD	748	prepare_transfer_to_handler
795	EXC_XFER_STD(0xf00, performance_monitor_exception)	749	bl performance_monitor_exception
		750	b interrupt_return
796		751
797		752
		753	__HEAD
798	/*	754	/*
799	* This code is jumped to from the startup code to copy	755	* This code is jumped to from the startup code to copy
800	* the kernel image to physical address PHYSICAL_START.	756	* the kernel image to physical address PHYSICAL_START.

arch/powerpc/kernel/head_booke.h+134−69

@@ -44,7 +44,7 @@ END_BTB_FLUSH_SECTION · 43 unmodified lines
44	#endif	44	#endif
45		45
46		46
47	#define NORMAL_EXCEPTION_PROLOG(intno) \	47	#define NORMAL_EXCEPTION_PROLOG(trapno, intno) \
48	mtspr SPRN_SPRG_WSCRATCH0, r10; /* save one register */ \	48	mtspr SPRN_SPRG_WSCRATCH0, r10; /* save one register */ \
49	mfspr r10, SPRN_SPRG_THREAD; \	49	mfspr r10, SPRN_SPRG_THREAD; \
50	stw r11, THREAD_NORMSAVE(0)(r10); \	50	stw r11, THREAD_NORMSAVE(0)(r10); \
@@ -53,6 +53,8 @@ END_BTB_FLUSH_SECTION · 2 unmodified lines
53	mfspr r11, SPRN_SRR1; \	53	mfspr r11, SPRN_SRR1; \
54	DO_KVM BOOKE_INTERRUPT_##intno SPRN_SRR1; \	54	DO_KVM BOOKE_INTERRUPT_##intno SPRN_SRR1; \
55	andi. r11, r11, MSR_PR; /* check whether user or kernel */\	55	andi. r11, r11, MSR_PR; /* check whether user or kernel */\
		56	LOAD_REG_IMMEDIATE(r11, MSR_KERNEL); \
		57	mtmsr r11; \
56	mr r11, r1; \	58	mr r11, r1; \
57	beq 1f; \	59	beq 1f; \
58	BOOKE_CLEAR_BTB(r11) \	60	BOOKE_CLEAR_BTB(r11) \
@@ -76,12 +78,39 @@ END_BTB_FLUSH_SECTION · 17 unmodified lines
76	stw r1, 0(r11); \	78	stw r1, 0(r11); \
77	mr r1, r11; \	79	mr r1, r11; \
78	rlwinm r9,r9,0,14,12; /* clear MSR_WE (necessary?) */\	80	rlwinm r9,r9,0,14,12; /* clear MSR_WE (necessary?) */\
79	stw r0,GPR0(r11); \	81	COMMON_EXCEPTION_PROLOG_END trapno
80	lis r10, STACK_FRAME_REGS_MARKER@ha;/* exception frame marker */ \	82
81	addi r10, r10, STACK_FRAME_REGS_MARKER@l; \	83	.macro COMMON_EXCEPTION_PROLOG_END trapno
82	stw r10, 8(r11); \	84	stw r0,GPR0(r1)
83	SAVE_4GPRS(3, r11); \	85	lis r10, STACK_FRAME_REGS_MARKER@ha /* exception frame marker */
84	SAVE_2GPRS(7, r11)	86	addi r10, r10, STACK_FRAME_REGS_MARKER@l
		87	stw r10, 8(r1)
		88	li r10, \trapno
		89	stw r10,_TRAP(r1)
		90	SAVE_4GPRS(3, r1)
		91	SAVE_2GPRS(7, r1)
		92	SAVE_NVGPRS(r1)
		93	stw r2,GPR2(r1)
		94	stw r12,_NIP(r1)
		95	stw r9,_MSR(r1)
		96	mfctr r10
		97	mfspr r2,SPRN_SPRG_THREAD
		98	stw r10,_CTR(r1)
		99	tovirt(r2, r2)
		100	mfspr r10,SPRN_XER
		101	addi r2, r2, -THREAD
		102	stw r10,_XER(r1)
		103	addi r3,r1,STACK_FRAME_OVERHEAD
		104	.endm
		105
		106	.macro prepare_transfer_to_handler
		107	#ifdef CONFIG_E500
		108	andi. r12,r9,MSR_PR
		109	bne 777f
		110	bl prepare_transfer_to_handler
		111	777:
		112	#endif
		113	.endm
85		114
86	.macro SYSCALL_ENTRY trapno intno srr1	115	.macro SYSCALL_ENTRY trapno intno srr1
87	mfspr r10, SPRN_SPRG_THREAD	116	mfspr r10, SPRN_SPRG_THREAD
@@ -180,7 +209,7 @@ ALT_FTR_SECTION_END_IFSET(CPU_FTR_EMB_HV) · 92 unmodified lines
180	* registers as the normal prolog above. Instead we use a portion of the	209	* registers as the normal prolog above. Instead we use a portion of the
181	* critical/machine check exception stack at low physical addresses.	210	* critical/machine check exception stack at low physical addresses.
182	*/	211	*/
183	#define EXC_LEVEL_EXCEPTION_PROLOG(exc_level, intno, exc_level_srr0, exc_level_srr1) \	212	#define EXC_LEVEL_EXCEPTION_PROLOG(exc_level, trapno, intno, exc_level_srr0, exc_level_srr1) \
184	mtspr SPRN_SPRG_WSCRATCH_##exc_level,r8; \	213	mtspr SPRN_SPRG_WSCRATCH_##exc_level,r8; \
185	BOOKE_LOAD_EXC_LEVEL_STACK(exc_level);/* r8 points to the exc_level stack*/ \	214	BOOKE_LOAD_EXC_LEVEL_STACK(exc_level);/* r8 points to the exc_level stack*/ \
186	stw r9,GPR9(r8); /* save various registers */\	215	stw r9,GPR9(r8); /* save various registers */\
@@ -192,6 +221,8 @@ ALT_FTR_SECTION_END_IFSET(CPU_FTR_EMB_HV) · 5 unmodified lines
192	DO_KVM BOOKE_INTERRUPT_##intno exc_level_srr1; \	221	DO_KVM BOOKE_INTERRUPT_##intno exc_level_srr1; \
193	BOOKE_CLEAR_BTB(r10) \	222	BOOKE_CLEAR_BTB(r10) \
194	andi. r11,r11,MSR_PR; \	223	andi. r11,r11,MSR_PR; \
		224	LOAD_REG_IMMEDIATE(r11, MSR_KERNEL & ~(MSR_ME\|MSR_DE\|MSR_CE)); \
		225	mtmsr r11; \
195	mfspr r11,SPRN_SPRG_THREAD; /* if from user, start at top of */\	226	mfspr r11,SPRN_SPRG_THREAD; /* if from user, start at top of */\
196	lwz r11, TASK_STACK - THREAD(r11); /* this thread's kernel stack */\	227	lwz r11, TASK_STACK - THREAD(r11); /* this thread's kernel stack */\
197	addi r11,r11,EXC_LVL_FRAME_OVERHEAD; /* allocate stack frame */\	228	addi r11,r11,EXC_LVL_FRAME_OVERHEAD; /* allocate stack frame */\
@@ -221,16 +252,44 @@ ALT_FTR_SECTION_END_IFSET(CPU_FTR_EMB_HV) · 23 unmodified lines
221	stw r1,0(r11); \	252	stw r1,0(r11); \
222	mr r1,r11; \	253	mr r1,r11; \
223	rlwinm r9,r9,0,14,12; /* clear MSR_WE (necessary?) */\	254	rlwinm r9,r9,0,14,12; /* clear MSR_WE (necessary?) */\
224	stw r0,GPR0(r11); \	255	COMMON_EXCEPTION_PROLOG_END trapno
225	SAVE_4GPRS(3, r11); \	256
226	SAVE_2GPRS(7, r11)	257	#define SAVE_xSRR(xSRR) \
		258	mfspr r0,SPRN_##xSRR##0; \
		259	stw r0,_##xSRR##0(r1); \
		260	mfspr r0,SPRN_##xSRR##1; \
		261	stw r0,_##xSRR##1(r1)
		262
		263
		264	.macro SAVE_MMU_REGS
		265	#ifdef CONFIG_PPC_BOOK3E_MMU
		266	mfspr r0,SPRN_MAS0
		267	stw r0,MAS0(r1)
		268	mfspr r0,SPRN_MAS1
		269	stw r0,MAS1(r1)
		270	mfspr r0,SPRN_MAS2
		271	stw r0,MAS2(r1)
		272	mfspr r0,SPRN_MAS3
		273	stw r0,MAS3(r1)
		274	mfspr r0,SPRN_MAS6
		275	stw r0,MAS6(r1)
		276	#ifdef CONFIG_PHYS_64BIT
		277	mfspr r0,SPRN_MAS7
		278	stw r0,MAS7(r1)
		279	#endif /* CONFIG_PHYS_64BIT */
		280	#endif /* CONFIG_PPC_BOOK3E_MMU */
		281	#ifdef CONFIG_44x
		282	mfspr r0,SPRN_MMUCR
		283	stw r0,MMUCR(r1)
		284	#endif
		285	.endm
227		286
228	#define CRITICAL_EXCEPTION_PROLOG(intno) \	287	#define CRITICAL_EXCEPTION_PROLOG(trapno, intno) \
229	EXC_LEVEL_EXCEPTION_PROLOG(CRIT, intno, SPRN_CSRR0, SPRN_CSRR1)	288	EXC_LEVEL_EXCEPTION_PROLOG(CRIT, trapno+2, intno, SPRN_CSRR0, SPRN_CSRR1)
230	#define DEBUG_EXCEPTION_PROLOG \	289	#define DEBUG_EXCEPTION_PROLOG(trapno) \
231	EXC_LEVEL_EXCEPTION_PROLOG(DBG, DEBUG, SPRN_DSRR0, SPRN_DSRR1)	290	EXC_LEVEL_EXCEPTION_PROLOG(DBG, trapno+8, DEBUG, SPRN_DSRR0, SPRN_DSRR1)
232	#define MCHECK_EXCEPTION_PROLOG \	291	#define MCHECK_EXCEPTION_PROLOG(trapno) \
233	EXC_LEVEL_EXCEPTION_PROLOG(MC, MACHINE_CHECK, \	292	EXC_LEVEL_EXCEPTION_PROLOG(MC, trapno+4, MACHINE_CHECK, \
234	SPRN_MCSRR0, SPRN_MCSRR1)	293	SPRN_MCSRR0, SPRN_MCSRR1)
235		294
236	/*	295	/*
@@ -257,44 +316,34 @@ ALT_FTR_SECTION_END_IFSET(CPU_FTR_EMB_HV) · 20 unmodified lines
257	.align 5; \	316	.align 5; \
258	label:	317	label:
259		318
260	#define EXCEPTION(n, intno, label, hdlr, xfer) \	319	#define EXCEPTION(n, intno, label, hdlr) \
261	START_EXCEPTION(label); \	320	START_EXCEPTION(label); \
262	NORMAL_EXCEPTION_PROLOG(intno); \	321	NORMAL_EXCEPTION_PROLOG(n, intno); \
263	addi r3,r1,STACK_FRAME_OVERHEAD; \	322	prepare_transfer_to_handler; \
264	xfer(n, hdlr)	323	bl hdlr; \
		324	b interrupt_return
265		325
266	#define CRITICAL_EXCEPTION(n, intno, label, hdlr) \	326	#define CRITICAL_EXCEPTION(n, intno, label, hdlr) \
267	START_EXCEPTION(label); \	327	START_EXCEPTION(label); \
268	CRITICAL_EXCEPTION_PROLOG(intno); \	328	CRITICAL_EXCEPTION_PROLOG(n, intno); \
269	addi r3,r1,STACK_FRAME_OVERHEAD; \	329	SAVE_MMU_REGS; \
270	EXC_XFER_TEMPLATE(hdlr, n+2, (MSR_KERNEL & ~(MSR_ME\|MSR_DE\|MSR_CE)), \	330	SAVE_xSRR(SRR); \
271	crit_transfer_to_handler, ret_from_crit_exc)	331	prepare_transfer_to_handler; \
		332	bl hdlr; \
		333	b ret_from_crit_exc
272		334
273	#define MCHECK_EXCEPTION(n, label, hdlr) \	335	#define MCHECK_EXCEPTION(n, label, hdlr) \
274	START_EXCEPTION(label); \	336	START_EXCEPTION(label); \
275	MCHECK_EXCEPTION_PROLOG; \	337	MCHECK_EXCEPTION_PROLOG(n); \
276	mfspr r5,SPRN_ESR; \	338	mfspr r5,SPRN_ESR; \
277	stw r5,_ESR(r11); \	339	stw r5,_ESR(r11); \
278	addi r3,r1,STACK_FRAME_OVERHEAD; \	340	SAVE_xSRR(DSRR); \
279	EXC_XFER_TEMPLATE(hdlr, n+4, (MSR_KERNEL & ~(MSR_ME\|MSR_DE\|MSR_CE)), \	341	SAVE_xSRR(CSRR); \
280	mcheck_transfer_to_handler, ret_from_mcheck_exc)	342	SAVE_MMU_REGS; \
281		343	SAVE_xSRR(SRR); \
282	#define EXC_XFER_TEMPLATE(hdlr, trap, msr, tfer, ret) \	344	prepare_transfer_to_handler; \
283	li r10,trap; \	345	bl hdlr; \
284	stw r10,_TRAP(r11); \	346	b ret_from_mcheck_exc
285	lis r10,msr@h; \
286	ori r10,r10,msr@l; \
287	bl tfer; \
288	.long hdlr; \
289	.long ret
290
291	#define EXC_XFER_STD(n, hdlr) \
292	EXC_XFER_TEMPLATE(hdlr, n, MSR_KERNEL, transfer_to_handler_full, \
293	ret_from_except_full)
294
295	#define EXC_XFER_LITE(n, hdlr) \
296	EXC_XFER_TEMPLATE(hdlr, n+1, MSR_KERNEL, transfer_to_handler, \
297	ret_from_except)
298		347
299	/* Check for a single step debug exception while in an exception	348	/* Check for a single step debug exception while in an exception
300	* handler before state has been saved. This is to catch the case	349	* handler before state has been saved. This is to catch the case
@@ -311,7 +360,7 @@ label: · 10 unmodified lines
311	*/	360	*/
312	#define DEBUG_DEBUG_EXCEPTION \	361	#define DEBUG_DEBUG_EXCEPTION \
313	START_EXCEPTION(DebugDebug); \	362	START_EXCEPTION(DebugDebug); \
314	DEBUG_EXCEPTION_PROLOG; \	363	DEBUG_EXCEPTION_PROLOG(2000); \
315	\	364	\
316	/* \	365	/* \
317	* If there is a single step or branch-taken exception in an \	366	* If there is a single step or branch-taken exception in an \
@@ -360,12 +409,16 @@ label: · 42 unmodified lines
360	/* continue normal handling for a debug exception... */ \	409	/* continue normal handling for a debug exception... */ \
361	2: mfspr r4,SPRN_DBSR; \	410	2: mfspr r4,SPRN_DBSR; \
362	stw r4,_ESR(r11); /* DebugException takes DBSR in _ESR */\	411	stw r4,_ESR(r11); /* DebugException takes DBSR in _ESR */\
363	addi r3,r1,STACK_FRAME_OVERHEAD; \	412	SAVE_xSRR(CSRR); \
364	EXC_XFER_TEMPLATE(DebugException, 0x2008, (MSR_KERNEL & ~(MSR_ME\|MSR_DE\|MSR_CE)), debug_transfer_to_handler, ret_from_debug_exc)	413	SAVE_MMU_REGS; \
		414	SAVE_xSRR(SRR); \
		415	prepare_transfer_to_handler; \
		416	bl DebugException; \
		417	b ret_from_debug_exc
365		418
366	#define DEBUG_CRIT_EXCEPTION \	419	#define DEBUG_CRIT_EXCEPTION \
367	START_EXCEPTION(DebugCrit); \	420	START_EXCEPTION(DebugCrit); \
368	CRITICAL_EXCEPTION_PROLOG(DEBUG); \	421	CRITICAL_EXCEPTION_PROLOG(2000,DEBUG); \
369	\	422	\
370	/* \	423	/* \
371	* If there is a single step or branch-taken exception in an \	424	* If there is a single step or branch-taken exception in an \
@@ -414,58 +467,71 @@ label: · 42 unmodified lines
414	/* continue normal handling for a critical exception... */ \	467	/* continue normal handling for a critical exception... */ \
415	2: mfspr r4,SPRN_DBSR; \	468	2: mfspr r4,SPRN_DBSR; \
416	stw r4,_ESR(r11); /* DebugException takes DBSR in _ESR */\	469	stw r4,_ESR(r11); /* DebugException takes DBSR in _ESR */\
417	addi r3,r1,STACK_FRAME_OVERHEAD; \	470	SAVE_MMU_REGS; \
418	EXC_XFER_TEMPLATE(DebugException, 0x2002, (MSR_KERNEL & ~(MSR_ME\|MSR_DE\|MSR_CE)), crit_transfer_to_handler, ret_from_crit_exc)	471	SAVE_xSRR(SRR); \
		472	prepare_transfer_to_handler; \
		473	bl DebugException; \
		474	b ret_from_crit_exc
419		475
420	#define DATA_STORAGE_EXCEPTION \	476	#define DATA_STORAGE_EXCEPTION \
421	START_EXCEPTION(DataStorage) \	477	START_EXCEPTION(DataStorage) \
422	NORMAL_EXCEPTION_PROLOG(DATA_STORAGE); \	478	NORMAL_EXCEPTION_PROLOG(0x300, DATA_STORAGE); \
423	mfspr r5,SPRN_ESR; /* Grab the ESR and save it */ \	479	mfspr r5,SPRN_ESR; /* Grab the ESR and save it */ \
424	stw r5,_ESR(r11); \	480	stw r5,_ESR(r11); \
425	mfspr r4,SPRN_DEAR; /* Grab the DEAR */ \	481	mfspr r4,SPRN_DEAR; /* Grab the DEAR */ \
426	stw r4, _DEAR(r11); \	482	stw r4, _DEAR(r11); \
427	EXC_XFER_LITE(0x0300, handle_page_fault)	483	prepare_transfer_to_handler; \
		484	bl do_page_fault; \
		485	b interrupt_return
428		486
429	#define INSTRUCTION_STORAGE_EXCEPTION \	487	#define INSTRUCTION_STORAGE_EXCEPTION \
430	START_EXCEPTION(InstructionStorage) \	488	START_EXCEPTION(InstructionStorage) \
431	NORMAL_EXCEPTION_PROLOG(INST_STORAGE); \	489	NORMAL_EXCEPTION_PROLOG(0x400, INST_STORAGE); \
432	mfspr r5,SPRN_ESR; /* Grab the ESR and save it */ \	490	mfspr r5,SPRN_ESR; /* Grab the ESR and save it */ \
433	stw r5,_ESR(r11); \	491	stw r5,_ESR(r11); \
434	stw r12, _DEAR(r11); /* Pass SRR0 as arg2 */ \	492	stw r12, _DEAR(r11); /* Pass SRR0 as arg2 */ \
435	EXC_XFER_LITE(0x0400, handle_page_fault)	493	prepare_transfer_to_handler; \
		494	bl do_page_fault; \
		495	b interrupt_return
436		496
437	#define ALIGNMENT_EXCEPTION \	497	#define ALIGNMENT_EXCEPTION \
438	START_EXCEPTION(Alignment) \	498	START_EXCEPTION(Alignment) \
439	NORMAL_EXCEPTION_PROLOG(ALIGNMENT); \	499	NORMAL_EXCEPTION_PROLOG(0x600, ALIGNMENT); \
440	mfspr r4,SPRN_DEAR; /* Grab the DEAR and save it */ \	500	mfspr r4,SPRN_DEAR; /* Grab the DEAR and save it */ \
441	stw r4,_DEAR(r11); \	501	stw r4,_DEAR(r11); \
442	addi r3,r1,STACK_FRAME_OVERHEAD; \	502	prepare_transfer_to_handler; \
443	EXC_XFER_STD(0x0600, alignment_exception)	503	bl alignment_exception; \
		504	REST_NVGPRS(r1); \
		505	b interrupt_return
444		506
445	#define PROGRAM_EXCEPTION \	507	#define PROGRAM_EXCEPTION \
446	START_EXCEPTION(Program) \	508	START_EXCEPTION(Program) \
447	NORMAL_EXCEPTION_PROLOG(PROGRAM); \	509	NORMAL_EXCEPTION_PROLOG(0x700, PROGRAM); \
448	mfspr r4,SPRN_ESR; /* Grab the ESR and save it */ \	510	mfspr r4,SPRN_ESR; /* Grab the ESR and save it */ \
449	stw r4,_ESR(r11); \	511	stw r4,_ESR(r11); \
450	addi r3,r1,STACK_FRAME_OVERHEAD; \	512	prepare_transfer_to_handler; \
451	EXC_XFER_STD(0x0700, program_check_exception)	513	bl program_check_exception; \
		514	REST_NVGPRS(r1); \
		515	b interrupt_return
452		516
453	#define DECREMENTER_EXCEPTION \	517	#define DECREMENTER_EXCEPTION \
454	START_EXCEPTION(Decrementer) \	518	START_EXCEPTION(Decrementer) \
455	NORMAL_EXCEPTION_PROLOG(DECREMENTER); \	519	NORMAL_EXCEPTION_PROLOG(0x900, DECREMENTER); \
456	lis r0,TSR_DIS@h; /* Setup the DEC interrupt mask */ \	520	lis r0,TSR_DIS@h; /* Setup the DEC interrupt mask */ \
457	mtspr SPRN_TSR,r0; /* Clear the DEC interrupt */ \	521	mtspr SPRN_TSR,r0; /* Clear the DEC interrupt */ \
458	addi r3,r1,STACK_FRAME_OVERHEAD; \	522	prepare_transfer_to_handler; \
459	EXC_XFER_LITE(0x0900, timer_interrupt)	523	bl timer_interrupt; \
		524	b interrupt_return
460		525
461	#define FP_UNAVAILABLE_EXCEPTION \	526	#define FP_UNAVAILABLE_EXCEPTION \
462	START_EXCEPTION(FloatingPointUnavailable) \	527	START_EXCEPTION(FloatingPointUnavailable) \
463	NORMAL_EXCEPTION_PROLOG(FP_UNAVAIL); \	528	NORMAL_EXCEPTION_PROLOG(0x800, FP_UNAVAIL); \
464	beq 1f; \	529	beq 1f; \
465	bl load_up_fpu; /* if from user, just load it up */ \	530	bl load_up_fpu; /* if from user, just load it up */ \
466	b fast_exception_return; \	531	b fast_exception_return; \
467	1: addi r3,r1,STACK_FRAME_OVERHEAD; \	532	1: prepare_transfer_to_handler; \
468	EXC_XFER_STD(0x800, kernel_fp_unavailable_exception)	533	bl kernel_fp_unavailable_exception; \
		534	b interrupt_return
469		535
470	#else /* __ASSEMBLY__ */	536	#else /* __ASSEMBLY__ */
471	struct exception_regs {	537	struct exception_regs {
@@ -481,7 +547,6 @@ struct exception_regs { · 9 unmodified lines
481	unsigned long csrr1;	547	unsigned long csrr1;
482	unsigned long dsrr0;	548	unsigned long dsrr0;
483	unsigned long dsrr1;	549	unsigned long dsrr1;
484	unsigned long saved_ksp_limit;
485	};	550	};
486		551
487	/* ensure this structure is always sized to a multiple of the stack alignment */	552	/* ensure this structure is always sized to a multiple of the stack alignment */

arch/powerpc/kernel/head_fsl_booke.S+38−32

 1:
 	/*
-	 * We have the runtime (virutal) address of our base.
+	 * We have the runtime (virtual) address of our base.
 	 * We calculate our shift of offset from a 64M page.
 	 * We could map the 64M page we belong to at PAGE_OFFSET and
 	 * get going from there.
 	/* Data Storage Interrupt */
 	START_EXCEPTION(DataStorage)
-	NORMAL_EXCEPTION_PROLOG(DATA_STORAGE)
+	NORMAL_EXCEPTION_PROLOG(0x300, DATA_STORAGE)
 	mfspr	r5,SPRN_ESR		/* Grab the ESR, save it */
 	stw	r5,_ESR(r11)
 	mfspr	r4,SPRN_DEAR		/* Grab the DEAR, save it */
 	stw	r4, _DEAR(r11)
 	andis.	r10,r5,(ESR_ILK|ESR_DLK)@h
 	bne	1f
-	EXC_XFER_LITE(0x0300, handle_page_fault)
+	prepare_transfer_to_handler
+	bl	do_page_fault
+	b	interrupt_return
 1:
-	addi	r3,r1,STACK_FRAME_OVERHEAD
+	prepare_transfer_to_handler
-	EXC_XFER_LITE(0x0300, CacheLockingException)
+	bl	CacheLockingException
+	b	interrupt_return
 	/* Instruction Storage Interrupt */
 	INSTRUCTION_STORAGE_EXCEPTION
 	/* External Input Interrupt */
-	EXCEPTION(0x0500, EXTERNAL, ExternalInput, do_IRQ, EXC_XFER_LITE)
+	EXCEPTION(0x0500, EXTERNAL, ExternalInput, do_IRQ)
 	/* Alignment Interrupt */
 	ALIGNMENT_EXCEPTION
 #ifdef CONFIG_PPC_FPU
 	FP_UNAVAILABLE_EXCEPTION
 #else
-	EXCEPTION(0x0800, FP_UNAVAIL, FloatingPointUnavailable, \
+	EXCEPTION(0x0800, FP_UNAVAIL, FloatingPointUnavailable, unknown_exception)
-		  unknown_exception, EXC_XFER_STD)
 #endif
 	/* System Call Interrupt */
 	SYSCALL_ENTRY   0xc00 BOOKE_INTERRUPT_SYSCALL SPRN_SRR1
 	/* Auxiliary Processor Unavailable Interrupt */
-	EXCEPTION(0x2900, AP_UNAVAIL, AuxillaryProcessorUnavailable, \
+	EXCEPTION(0x2900, AP_UNAVAIL, AuxillaryProcessorUnavailable, unknown_exception)
-		  unknown_exception, EXC_XFER_STD)
 	/* Decrementer Interrupt */
 	DECREMENTER_EXCEPTION
 	/* Fixed Internal Timer Interrupt */
 	/* TODO: Add FIT support */
-	EXCEPTION(0x3100, FIT, FixedIntervalTimer, \
+	EXCEPTION(0x3100, FIT, FixedIntervalTimer, unknown_exception)
-		  unknown_exception, EXC_XFER_STD)
 	/* Watchdog Timer Interrupt */
 #ifdef CONFIG_BOOKE_WDT
 #endif
 #endif
-	bne	2f			/* Bail if permission/valid mismach */
+	bne	2f			/* Bail if permission/valid mismatch */
 	/* Jump to common tlb load */
 	b	finish_tlb_load
 #endif
 #endif
-	bne	2f			/* Bail if permission mismach */
+	bne	2f			/* Bail if permission mismatch */
 	/* Jump to common TLB load point */
 	b	finish_tlb_load
 #ifdef CONFIG_SPE
 	/* SPE Unavailable */
 	START_EXCEPTION(SPEUnavailable)
-	NORMAL_EXCEPTION_PROLOG(SPE_UNAVAIL)
+	NORMAL_EXCEPTION_PROLOG(0x2010, SPE_UNAVAIL)
 	beq	1f
 	bl	load_up_spe
 	b	fast_exception_return
-1:	addi	r3,r1,STACK_FRAME_OVERHEAD
+1:	prepare_transfer_to_handler
-	EXC_XFER_LITE(0x2010, KernelSPE)
+	bl	KernelSPE
+	b	interrupt_return
 #elif defined(CONFIG_SPE_POSSIBLE)
-	EXCEPTION(0x2020, SPE_UNAVAIL, SPEUnavailable, \
+	EXCEPTION(0x2020, SPE_UNAVAIL, SPEUnavailable, unknown_exception)
-		  unknown_exception, EXC_XFER_STD)
 #endif /* CONFIG_SPE_POSSIBLE */
 	/* SPE Floating Point Data */
 #ifdef CONFIG_SPE
-	EXCEPTION(0x2030, SPE_FP_DATA, SPEFloatingPointData,
+	START_EXCEPTION(SPEFloatingPointData)
-		  SPEFloatingPointException, EXC_XFER_STD)
+	NORMAL_EXCEPTION_PROLOG(0x2030, SPE_FP_DATA)
+	prepare_transfer_to_handler
+	bl	SPEFloatingPointException
+	REST_NVGPRS(r1)
+	b	interrupt_return
 	/* SPE Floating Point Round */
-	EXCEPTION(0x2050, SPE_FP_ROUND, SPEFloatingPointRound, \
+	START_EXCEPTION(SPEFloatingPointRound)
-		  SPEFloatingPointRoundException, EXC_XFER_STD)
+	NORMAL_EXCEPTION_PROLOG(0x2050, SPE_FP_ROUND)
+	prepare_transfer_to_handler
+	bl	SPEFloatingPointRoundException
+	REST_NVGPRS(r1)
+	b	interrupt_return
 #elif defined(CONFIG_SPE_POSSIBLE)
-	EXCEPTION(0x2040, SPE_FP_DATA, SPEFloatingPointData,
+	EXCEPTION(0x2040, SPE_FP_DATA, SPEFloatingPointData, unknown_exception)
-		  unknown_exception, EXC_XFER_STD)
+	EXCEPTION(0x2050, SPE_FP_ROUND, SPEFloatingPointRound, unknown_exception)
-	EXCEPTION(0x2050, SPE_FP_ROUND, SPEFloatingPointRound, \
-		  unknown_exception, EXC_XFER_STD)
 #endif /* CONFIG_SPE_POSSIBLE */
 	/* Performance Monitor */
 	EXCEPTION(0x2060, PERFORMANCE_MONITOR, PerformanceMonitor, \
-		  performance_monitor_exception, EXC_XFER_STD)
+		  performance_monitor_exception)
-	EXCEPTION(0x2070, DOORBELL, Doorbell, doorbell_exception, EXC_XFER_STD)
+	EXCEPTION(0x2070, DOORBELL, Doorbell, doorbell_exception)
 	CRITICAL_EXCEPTION(0x2080, DOORBELL_CRITICAL, \
 			   CriticalDoorbell, unknown_exception)
 			   unknown_exception)
 	/* Hypercall */
-	EXCEPTION(0, HV_SYSCALL, Hypercall, unknown_exception, EXC_XFER_STD)
+	EXCEPTION(0, HV_SYSCALL, Hypercall, unknown_exception)
 	/* Embedded Hypervisor Privilege */
-	EXCEPTION(0, HV_PRIV, Ehvpriv, unknown_exception, EXC_XFER_STD)
+	EXCEPTION(0, HV_PRIV, Ehvpriv, unknown_exception)
 interrupt_end:
 	lwz	r5,_NIP(r1)
 	bl	printk
 #endif
-	b	ret_from_except
+	b	interrupt_return
 #ifdef CONFIG_PRINTK
 87:	.string	"SPE used in kernel  (task=%p, pc=%x)  \n"
 #endif

arch/powerpc/kernel/hw_breakpoint_constraints.c+1−1

@@ -141,7 +141,7 @@ void wp_get_instr_detail(struct pt_regs regs, struct ppc_inst instr, · 140 unmodified lines
141	{	141	{
142	struct instruction_op op;	142	struct instruction_op op;
143		143
144	if (__get_user_instr_inatomic(instr, (void __user )regs->nip))	144	if (__get_user_instr(instr, (void __user )regs->nip))
145	return;	145	return;
146		146
147	analyse_instr(&op, regs, *instr);	147	analyse_instr(&op, regs, *instr);

arch/powerpc/kernel/idle_6xx.S+2−12

 /*
  * Return from NAP/DOZE mode, restore some CPU specific registers,
- * we are called with DR/IR still off and r2 containing physical
+ * R11 points to the exception frame. We have to preserve r10.
- * address of current.  R11 points to the exception frame (physical
- * address).  We have to preserve r10.
  */
 _GLOBAL(power_save_ppc32_restore)
 	lwz	r9,_LINK(r11)		/* interrupted in ppc6xx_idle: */
 	mfspr	r9,SPRN_HID0
 	andis.	r9,r9,HID0_NAP@h
 	beq	1f
-#ifdef CONFIG_VMAP_STACK
 	addis	r9, r11, nap_save_msscr0@ha
-#else
-	addis	r9,r11,(nap_save_msscr0-KERNELBASE)@ha
-#endif
 	lwz	r9,nap_save_msscr0@l(r9)
 	mtspr	SPRN_MSSCR0, r9
 	sync
 1:
 END_FTR_SECTION_IFSET(CPU_FTR_NAP_DISABLE_L2_PR)
 BEGIN_FTR_SECTION
-#ifdef CONFIG_VMAP_STACK
 	addis	r9, r11, nap_save_hid1@ha
-#else
-	addis	r9,r11,(nap_save_hid1-KERNELBASE)@ha
-#endif
 	lwz	r9,nap_save_hid1@l(r9)
 	mtspr	SPRN_HID1, r9
 END_FTR_SECTION_IFSET(CPU_FTR_DUAL_PLL_750FX)
-	b	transfer_to_handler_cont
+	blr
 _ASM_NOKPROBE_SYMBOL(power_save_ppc32_restore)
 	.data

arch/powerpc/kernel/idle_book3s.S+4−0

@@ -209,4 +209,8 @@ _GLOBAL(power4_idle_nap) · 208 unmodified lines
209	mtmsrd r7	209	mtmsrd r7
210	isync	210	isync
211	b 1b	211	b 1b
		212
		213	.globl power4_idle_nap_return
		214	power4_idle_nap_return:
		215	blr
212	#endif	216	#endif

arch/powerpc/kernel/idle_e500.S+3−11

@@ -74,20 +74,12 @@ END_FTR_SECTION_IFSET(CPU_FTR_CAN_NAP) · 73 unmodified lines
74		74
75	/*	75	/*
76	* Return from NAP/DOZE mode, restore some CPU specific registers,	76	* Return from NAP/DOZE mode, restore some CPU specific registers,
77	* r2 containing physical address of current.	77	* r2 containing address of current.
78	* r11 points to the exception frame (physical address).	78	* r11 points to the exception frame.
79	* We have to preserve r10.	79	* We have to preserve r10.
80	*/	80	*/
81	_GLOBAL(power_save_ppc32_restore)	81	_GLOBAL(power_save_ppc32_restore)
82	lwz r9,_LINK(r11) /* interrupted in e500_idle */	82	lwz r9,_LINK(r11) /* interrupted in e500_idle */
83	stw r9,_NIP(r11) /* make it do a blr */	83	stw r9,_NIP(r11) /* make it do a blr */
84		84	blr
85	#ifdef CONFIG_SMP
86	lwz r11,TASK_CPU(r2) /* get cpu number * 4 */
87	slwi r11,r11,2
88	#else
89	li r11,0
90	#endif
91
92	b transfer_to_handler_cont
93	_ASM_NOKPROBE_SYMBOL(power_save_ppc32_restore)	85	_ASM_NOKPROBE_SYMBOL(power_save_ppc32_restore)

arch/powerpc/kernel/interrupt.c+27−40

@@ -20,6 +20,10 @@ · 19 unmodified lines
20	#include <asm/time.h>	20	#include <asm/time.h>
21	#include <asm/unistd.h>	21	#include <asm/unistd.h>
22		22
		23	#if defined(CONFIG_PPC_ADV_DEBUG_REGS) && defined(CONFIG_PPC32)
		24	unsigned long global_dbcr0[NR_CPUS];
		25	#endif
		26
23	typedef long (*syscall_fn)(long, long, long, long, long, long);	27	typedef long (*syscall_fn)(long, long, long, long, long, long);
24		28
25	/* Has to run notrace because it is entered not completely "reconciled" */	29	/* Has to run notrace because it is entered not completely "reconciled" */
@@ -29,20 +33,24 @@ notrace long system_call_exception(long r3, long r4, long r5, · 3 unmodified lines
29	{	33	{
30	syscall_fn f;	34	syscall_fn f;
31		35
		36	kuep_lock();
		37	#ifdef CONFIG_PPC32
		38	kuap_save_and_lock(regs);
		39	#endif
		40
32	regs->orig_gpr3 = r3;	41	regs->orig_gpr3 = r3;
33		42
34	if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG))	43	if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG))
35	BUG_ON(irq_soft_mask_return() != IRQS_ALL_DISABLED);	44	BUG_ON(irq_soft_mask_return() != IRQS_ALL_DISABLED);
36		45
		46	trace_hardirqs_off(); /* finish reconciling */
		47
37	CT_WARN_ON(ct_state() == CONTEXT_KERNEL);	48	CT_WARN_ON(ct_state() == CONTEXT_KERNEL);
38	user_exit_irqoff();	49	user_exit_irqoff();
39		50
40	trace_hardirqs_off(); /* finish reconciling */
41
42	if (!IS_ENABLED(CONFIG_BOOKE) && !IS_ENABLED(CONFIG_40x))	51	if (!IS_ENABLED(CONFIG_BOOKE) && !IS_ENABLED(CONFIG_40x))
43	BUG_ON(!(regs->msr & MSR_RI));	52	BUG_ON(!(regs->msr & MSR_RI));
44	BUG_ON(!(regs->msr & MSR_PR));	53	BUG_ON(!(regs->msr & MSR_PR));
45	BUG_ON(!FULL_REGS(regs));
46	BUG_ON(arch_irq_disabled_regs(regs));	54	BUG_ON(arch_irq_disabled_regs(regs));
47		55
48	#ifdef CONFIG_PPC_PKEY	56	#ifdef CONFIG_PPC_PKEY
@@ -69,9 +77,7 @@ notrace long system_call_exception(long r3, long r4, long r5, · 20 unmodified lines
69	isync();	77	isync();
70	} else	78	} else
71	#endif	79	#endif
72	#ifdef CONFIG_PPC64	80	kuap_assert_locked();
73	kuap_check_amr();
74	#endif
75		81
76	booke_restore_dbcr0();	82	booke_restore_dbcr0();
77		83
@@ -247,9 +253,7 @@ notrace unsigned long syscall_exit_prepare(unsigned long r3, · 169 unmodified lines
247		253
248	CT_WARN_ON(ct_state() == CONTEXT_USER);	254	CT_WARN_ON(ct_state() == CONTEXT_USER);
249		255
250	#ifdef CONFIG_PPC64	256	kuap_assert_locked();
251	kuap_check_amr();
252	#endif
253		257
254	regs->result = r3;	258	regs->result = r3;
255		259
@@ -344,16 +348,13 @@ again: · 88 unmodified lines
344		348
345	account_cpu_user_exit();	349	account_cpu_user_exit();
346		350
347	#ifdef CONFIG_PPC_BOOK3S_64 /* BOOK3E and ppc32 not using this */	351	/* Restore user access locks last */
348	/*
349	* We do this at the end so that we do context switch with KERNEL AMR
350	*/
351	kuap_user_restore(regs);	352	kuap_user_restore(regs);
352	#endif	353	kuep_unlock();
		354
353	return ret;	355	return ret;
354	}	356	}
355		357
356	#ifndef CONFIG_PPC_BOOK3E_64 /* BOOK3E not yet using this */
357	notrace unsigned long interrupt_exit_user_prepare(struct pt_regs *regs, unsigned long msr)	358	notrace unsigned long interrupt_exit_user_prepare(struct pt_regs *regs, unsigned long msr)
358	{	359	{
359	unsigned long ti_flags;	360	unsigned long ti_flags;
@@ -363,7 +364,6 @@ notrace unsigned long interrupt_exit_user_prepare(struct pt_regs *regs, unsigned · 3 unmodified lines
363	if (!IS_ENABLED(CONFIG_BOOKE) && !IS_ENABLED(CONFIG_40x))	364	if (!IS_ENABLED(CONFIG_BOOKE) && !IS_ENABLED(CONFIG_40x))
364	BUG_ON(!(regs->msr & MSR_RI));	365	BUG_ON(!(regs->msr & MSR_RI));
365	BUG_ON(!(regs->msr & MSR_PR));	366	BUG_ON(!(regs->msr & MSR_PR));
366	BUG_ON(!FULL_REGS(regs));
367	BUG_ON(arch_irq_disabled_regs(regs));	367	BUG_ON(arch_irq_disabled_regs(regs));
368	CT_WARN_ON(ct_state() == CONTEXT_USER);	368	CT_WARN_ON(ct_state() == CONTEXT_USER);
369		369
@@ -371,9 +371,7 @@ notrace unsigned long interrupt_exit_user_prepare(struct pt_regs *regs, unsigned · 1 unmodified line
371	* We don't need to restore AMR on the way back to userspace for KUAP.	371	* We don't need to restore AMR on the way back to userspace for KUAP.
372	* AMR can only have been unlocked if we interrupted the kernel.	372	* AMR can only have been unlocked if we interrupted the kernel.
373	*/	373	*/
374	#ifdef CONFIG_PPC64	374	kuap_assert_locked();
375	kuap_check_amr();
376	#endif
377		375
378	local_irq_save(flags);	376	local_irq_save(flags);
379		377
@@ -392,7 +390,7 @@ again: · 12 unmodified lines
392	ti_flags = READ_ONCE(current_thread_info()->flags);	390	ti_flags = READ_ONCE(current_thread_info()->flags);
393	}	391	}
394		392
395	if (IS_ENABLED(CONFIG_PPC_BOOK3S) && IS_ENABLED(CONFIG_PPC_FPU)) {	393	if (IS_ENABLED(CONFIG_PPC_BOOK3S_64) && IS_ENABLED(CONFIG_PPC_FPU)) {
396	if (IS_ENABLED(CONFIG_PPC_TRANSACTIONAL_MEM) &&	394	if (IS_ENABLED(CONFIG_PPC_TRANSACTIONAL_MEM) &&
397	unlikely((ti_flags & _TIF_RESTORE_TM))) {	395	unlikely((ti_flags & _TIF_RESTORE_TM))) {
398	restore_tm_state(regs);	396	restore_tm_state(regs);
@@ -427,12 +425,9 @@ again: · 28 unmodified lines
427		425
428	account_cpu_user_exit();	426	account_cpu_user_exit();
429		427
430	/*	428	/* Restore user access locks last */
431	* We do this at the end so that we do context switch with KERNEL AMR
432	*/
433	#ifdef CONFIG_PPC64
434	kuap_user_restore(regs);	429	kuap_user_restore(regs);
435	#endif	430
436	return ret;	431	return ret;
437	}	432	}
438		433
@@ -442,25 +437,20 @@ notrace unsigned long interrupt_exit_kernel_prepare(struct pt_regs *regs, unsign · 3 unmodified lines
442	{	437	{
443	unsigned long flags;	438	unsigned long flags;
444	unsigned long ret = 0;	439	unsigned long ret = 0;
445	#ifdef CONFIG_PPC64	440	unsigned long kuap;
446	unsigned long amr;
447	#endif
448		441
449	if (!IS_ENABLED(CONFIG_BOOKE) && !IS_ENABLED(CONFIG_40x) &&	442	if (!IS_ENABLED(CONFIG_BOOKE) && !IS_ENABLED(CONFIG_40x) &&
450	unlikely(!(regs->msr & MSR_RI)))	443	unlikely(!(regs->msr & MSR_RI)))
451	unrecoverable_exception(regs);	444	unrecoverable_exception(regs);
452	BUG_ON(regs->msr & MSR_PR);	445	BUG_ON(regs->msr & MSR_PR);
453	BUG_ON(!FULL_REGS(regs));
454	/*	446	/*
455	* CT_WARN_ON comes here via program_check_exception,	447	* CT_WARN_ON comes here via program_check_exception,
456	* so avoid recursion.	448	* so avoid recursion.
457	*/	449	*/
458	if (TRAP(regs) != 0x700)	450	if (TRAP(regs) != INTERRUPT_PROGRAM)
459	CT_WARN_ON(ct_state() == CONTEXT_USER);	451	CT_WARN_ON(ct_state() == CONTEXT_USER);
460		452
461	#ifdef CONFIG_PPC64	453	kuap = kuap_get_and_assert_locked();
462	amr = kuap_get_and_check_amr();
463	#endif
464		454
465	if (unlikely(current_thread_info()->flags & _TIF_EMULATE_STACK_STORE)) {	455	if (unlikely(current_thread_info()->flags & _TIF_EMULATE_STACK_STORE)) {
466	clear_bits(_TIF_EMULATE_STACK_STORE, &current_thread_info()->flags);	456	clear_bits(_TIF_EMULATE_STACK_STORE, &current_thread_info()->flags);
@@ -498,14 +488,11 @@ again: · 31 unmodified lines
498	#endif	488	#endif
499		489
500	/*	490	/*
501	* Don't want to mfspr(SPRN_AMR) here, because this comes after mtmsr,	491	* 64s does not want to mfspr(SPRN_AMR) here, because this comes after
502	* which would cause Read-After-Write stalls. Hence, we take the AMR	492	* mtmsr, which would cause Read-After-Write stalls. Hence, take the
503	* value from the check above.	493	* AMR value from the check above.
504	*/	494	*/
505	#ifdef CONFIG_PPC64	495	kuap_kernel_restore(regs, kuap);
506	kuap_kernel_restore(regs, amr);
507	#endif
508		496
509	return ret;	497	return ret;
510	}	498	}
511	#endif

arch/powerpc/kernel/iommu.c+24−22

 	sprintf(name, "%08lx", tbl->it_index);
 	liobn_entry = debugfs_lookup(name, iommu_debugfs_dir);
-	if (liobn_entry)
+	debugfs_remove(liobn_entry);
-		debugfs_remove(liobn_entry);
 }
 #else
 static void iommu_debugfs_add(struct iommu_table *tbl){}
 			pass++;
 			goto again;
+		} else if (pass == tbl->nr_pools + 1) {
+			/* Last resort: try largepool */
+			spin_unlock(&pool->lock);
+			pool = &tbl->large_pool;
+			spin_lock(&pool->lock);
+			pool->hint = pool->start;
+			pass++;
+			goto again;
 		} else {
 			/* Give up */
 			spin_unlock_irqrestore(&(pool->lock), flags);
 {
 	unsigned long sz;
 	static int welcomed = 0;
-	struct page *page;
 	unsigned int i;
 	struct iommu_pool *p;
 	/* number of bytes needed for the bitmap */
 	sz = BITS_TO_LONGS(tbl->it_size) * sizeof(unsigned long);
-	page = alloc_pages_node(nid, GFP_KERNEL, get_order(sz));
+	tbl->it_map = vzalloc_node(sz, nid);
-	if (!page)
+	if (!tbl->it_map) {
-		panic("iommu_init_table: Can't allocate %ld bytes\n", sz);
+		pr_err("%s: Can't allocate %ld bytes\n", __func__, sz);
-	tbl->it_map = page_address(page);
+		return NULL;
-	memset(tbl->it_map, 0, sz);
+	}
 	iommu_table_reserve_pages(tbl, res_start, res_end);
 static void iommu_table_free(struct kref *kref)
 {
-	unsigned long bitmap_sz;
-	unsigned int order;
 	struct iommu_table *tbl;
 	tbl = container_of(kref, struct iommu_table, it_kref);
 	if (!bitmap_empty(tbl->it_map, tbl->it_size))
 		pr_warn("%s: Unexpected TCEs\n", __func__);
-	/* calculate bitmap size in bytes */
-	bitmap_sz = BITS_TO_LONGS(tbl->it_size) * sizeof(unsigned long);
 	/* free bitmap */
-	order = get_order(bitmap_sz);
+	vfree(tbl->it_map);
-	free_pages((unsigned long) tbl->it_map, order);
 	/* free table */
 	kfree(tbl);
 	unsigned int order;
 	unsigned int nio_pages, io_order;
 	struct page *page;
+	size_t size_io = size;
 	size = PAGE_ALIGN(size);
 	order = get_order(size);
 	memset(ret, 0, size);
 	/* Set up tces to cover the allocated range */
-	nio_pages = size >> tbl->it_page_shift;
+	size_io = IOMMU_PAGE_ALIGN(size_io, tbl);
-	io_order = get_iommu_order(size, tbl);
+	nio_pages = size_io >> tbl->it_page_shift;
+	io_order = get_iommu_order(size_io, tbl);
 	mapping = iommu_alloc(dev, tbl, ret, nio_pages, DMA_BIDIRECTIONAL,
 			      mask >> tbl->it_page_shift, io_order, 0);
 	if (mapping == DMA_MAPPING_ERROR) {
 			 void *vaddr, dma_addr_t dma_handle)
 {
 	if (tbl) {
-		unsigned int nio_pages;
+		size_t size_io = IOMMU_PAGE_ALIGN(size, tbl);
+		unsigned int nio_pages = size_io >> tbl->it_page_shift;
-		size = PAGE_ALIGN(size);
-		nio_pages = size >> tbl->it_page_shift;
 		iommu_free(tbl, dma_handle, nio_pages);
 		size = PAGE_ALIGN(size);
 		free_pages((unsigned long)vaddr, get_order(size));
 	spin_lock_irqsave(&tbl->large_pool.lock, flags);
 	for (i = 0; i < tbl->nr_pools; i++)
-		spin_lock(&tbl->pools[i].lock);
+		spin_lock_nest_lock(&tbl->pools[i].lock, &tbl->large_pool.lock);
 	iommu_table_release_pages(tbl);
 	spin_lock_irqsave(&tbl->large_pool.lock, flags);
 	for (i = 0; i < tbl->nr_pools; i++)
-		spin_lock(&tbl->pools[i].lock);
+		spin_lock_nest_lock(&tbl->pools[i].lock, &tbl->large_pool.lock);
 	memset(tbl->it_map, 0, sz);

arch/powerpc/kernel/irq.c+47−85

 	return happened;
 }
-#ifdef CONFIG_PPC_BOOK3E
-/* This is called whenever we are re-enabling interrupts
- * and returns either 0 (nothing to do) or 500/900/280 if
- * there's an EE, DEC or DBELL to generate.
- *
- * This is called in two contexts: From arch_local_irq_restore()
- * before soft-enabling interrupts, and from the exception exit
- * path when returning from an interrupt from a soft-disabled to
- * a soft enabled context. In both case we have interrupts hard
- * disabled.
- *
- * We take care of only clearing the bits we handled in the
- * PACA irq_happened field since we can only re-emit one at a
- * time and we don't want to "lose" one.
- */
-notrace unsigned int __check_irq_replay(void)
-{
-	/*
-	 * We use local_paca rather than get_paca() to avoid all
-	 * the debug_smp_processor_id() business in this low level
-	 * function
-	 */
-	unsigned char happened = local_paca->irq_happened;
-	/*
-	 * We are responding to the next interrupt, so interrupt-off
-	 * latencies should be reset here.
-	 */
-	trace_hardirqs_on();
-	trace_hardirqs_off();
-	if (happened & PACA_IRQ_DEC) {
-		local_paca->irq_happened &= ~PACA_IRQ_DEC;
-		return 0x900;
-	}
-	if (happened & PACA_IRQ_EE) {
-		local_paca->irq_happened &= ~PACA_IRQ_EE;
-		return 0x500;
-	}
-	if (happened & PACA_IRQ_DBELL) {
-		local_paca->irq_happened &= ~PACA_IRQ_DBELL;
-		return 0x280;
-	}
-	if (happened & PACA_IRQ_HARD_DIS)
-		local_paca->irq_happened &= ~PACA_IRQ_HARD_DIS;
-	/* There should be nothing left ! */
-	BUG_ON(local_paca->irq_happened != 0);
-	return 0;
-}
-/*
- * This is specifically called by assembly code to re-enable interrupts
- * if they are currently disabled. This is typically called before
- * schedule() or do_signal() when returning to userspace. We do it
- * in C to avoid the burden of dealing with lockdep etc...
- *
- * NOTE: This is called with interrupts hard disabled but not marked
- * as such in paca->irq_happened, so we need to resync this.
- */
-void notrace restore_interrupts(void)
-{
-	if (irqs_disabled()) {
-		local_paca->irq_happened |= PACA_IRQ_HARD_DIS;
-		local_irq_enable();
-	} else
-		__hard_irq_enable();
-}
-#endif /* CONFIG_PPC_BOOK3E */
 void replay_soft_interrupts(void)
 {
 	struct pt_regs regs;
 	 */
 	if (IS_ENABLED(CONFIG_PPC_BOOK3S) && (local_paca->irq_happened & PACA_IRQ_HMI)) {
 		local_paca->irq_happened &= ~PACA_IRQ_HMI;
-		regs.trap = 0xe60;
+		regs.trap = INTERRUPT_HMI;
 		handle_hmi_exception(&regs);
 		if (!(local_paca->irq_happened & PACA_IRQ_HARD_DIS))
 			hard_irq_disable();
 	if (local_paca->irq_happened & PACA_IRQ_DEC) {
 		local_paca->irq_happened &= ~PACA_IRQ_DEC;
-		regs.trap = 0x900;
+		regs.trap = INTERRUPT_DECREMENTER;
 		timer_interrupt(&regs);
 		if (!(local_paca->irq_happened & PACA_IRQ_HARD_DIS))
 			hard_irq_disable();
 	if (local_paca->irq_happened & PACA_IRQ_EE) {
 		local_paca->irq_happened &= ~PACA_IRQ_EE;
-		regs.trap = 0x500;
+		regs.trap = INTERRUPT_EXTERNAL;
 		do_IRQ(&regs);
 		if (!(local_paca->irq_happened & PACA_IRQ_HARD_DIS))
 			hard_irq_disable();
 	if (IS_ENABLED(CONFIG_PPC_DOORBELL) && (local_paca->irq_happened & PACA_IRQ_DBELL)) {
 		local_paca->irq_happened &= ~PACA_IRQ_DBELL;
-		if (IS_ENABLED(CONFIG_PPC_BOOK3E))
+		regs.trap = INTERRUPT_DOORBELL;
-			regs.trap = 0x280;
-		else
-			regs.trap = 0xa00;
 		doorbell_exception(&regs);
 		if (!(local_paca->irq_happened & PACA_IRQ_HARD_DIS))
 			hard_irq_disable();
 	/* Book3E does not support soft-masking PMI interrupts */
 	if (IS_ENABLED(CONFIG_PPC_BOOK3S) && (local_paca->irq_happened & PACA_IRQ_PMI)) {
 		local_paca->irq_happened &= ~PACA_IRQ_PMI;
-		regs.trap = 0xf00;
+		regs.trap = INTERRUPT_PERFMON;
 		performance_monitor_exception(&regs);
 		if (!(local_paca->irq_happened & PACA_IRQ_HARD_DIS))
 			hard_irq_disable();
 	 * and re-locking AMR but we shouldn't get here in the first place,
 	 * hence the warning.
 	 */
-	kuap_check_amr();
+	kuap_assert_locked();
 	if (kuap_state != AMR_KUAP_BLOCKED)
 		set_kuap(AMR_KUAP_BLOCKED);
 	}
 }
+static __always_inline void call_do_softirq(const void *sp)
+{
+	/* Temporarily switch r1 to sp, call __do_softirq() then restore r1. */
+	asm volatile (
+		 PPC_STLU "	%%r1, %[offset](%[sp])	;"
+		"mr		%%r1, %[sp]		;"
+		"bl		%[callee]		;"
+		 PPC_LL "	%%r1, 0(%%r1)		;"
+		 : // Outputs
+		 : // Inputs
+		   [sp] "b" (sp), [offset] "i" (THREAD_SIZE - STACK_FRAME_OVERHEAD),
+		   [callee] "i" (__do_softirq)
+		 : // Clobbers
+		   "lr", "xer", "ctr", "memory", "cr0", "cr1", "cr5", "cr6",
+		   "cr7", "r0", "r3", "r4", "r5", "r6", "r7", "r8", "r9", "r10",
+		   "r11", "r12"
+	);
+}
+static __always_inline void call_do_irq(struct pt_regs *regs, void *sp)
+{
+	register unsigned long r3 asm("r3") = (unsigned long)regs;
+	/* Temporarily switch r1 to sp, call __do_irq() then restore r1. */
+	asm volatile (
+		 PPC_STLU "	%%r1, %[offset](%[sp])	;"
+		"mr		%%r1, %[sp]		;"
+		"bl		%[callee]		;"
+		 PPC_LL "	%%r1, 0(%%r1)		;"
+		 : // Outputs
+		   "+r" (r3)
+		 : // Inputs
+		   [sp] "b" (sp), [offset] "i" (THREAD_SIZE - STACK_FRAME_OVERHEAD),
+		   [callee] "i" (__do_irq)
+		 : // Clobbers
+		   "lr", "xer", "ctr", "memory", "cr0", "cr1", "cr5", "cr6",
+		   "cr7", "r0", "r4", "r5", "r6", "r7", "r8", "r9", "r10",
+		   "r11", "r12"
+	);
+}
 void __do_irq(struct pt_regs *regs)
 {
 	unsigned int irq;

arch/powerpc/kernel/jump_label.c+2−2

@@ -11,10 +11,10 @@ · 10 unmodified lines
11	void arch_jump_label_transform(struct jump_entry *entry,	11	void arch_jump_label_transform(struct jump_entry *entry,
12	enum jump_label_type type)	12	enum jump_label_type type)
13	{	13	{
14	struct ppc_inst addr = (struct ppc_inst )(unsigned long)entry->code;	14	struct ppc_inst addr = (struct ppc_inst )jump_entry_code(entry);
15		15
16	if (type == JUMP_LABEL_JMP)	16	if (type == JUMP_LABEL_JMP)
17	patch_branch(addr, entry->target, 0);	17	patch_branch(addr, jump_entry_target(entry), 0);
18	else	18	else
19	patch_instruction(addr, ppc_inst(PPC_INST_NOP));	19	patch_instruction(addr, ppc_inst(PPC_INST_NOP));
20	}	20	}

arch/powerpc/kernel/kgdb.c+1−1

@@ -376,7 +376,7 @@ void kgdb_arch_set_pc(struct pt_regs *regs, unsigned long pc) · 375 unmodified lines
376	}	376	}
377		377
378	/*	378	/*
379	* This function does PowerPC specific procesing for interfacing to gdb.	379	* This function does PowerPC specific processing for interfacing to gdb.
380	*/	380	*/
381	int kgdb_arch_handle_exception(int vector, int signo, int err_code,	381	int kgdb_arch_handle_exception(int vector, int signo, int err_code,
382	char remcom_in_buffer, char remcom_out_buffer,	382	char remcom_in_buffer, char remcom_out_buffer,

arch/powerpc/kernel/legacy_serial.c+29−4

 #include <asm/udbg.h>
 #include <asm/pci-bridge.h>
 #include <asm/ppc-pci.h>
+#include <asm/early_ioremap.h>
 #undef DEBUG
 	unsigned int			clock;
 	int				irq_check_parent;
 	phys_addr_t			taddr;
+	void __iomem			*early_addr;
 } legacy_serial_infos[MAX_LEGACY_SERIAL_PORTS];
 static const struct of_device_id legacy_serial_parents[] __initconst = {
 {
 	struct legacy_serial_info *info = &legacy_serial_infos[console];
 	struct plat_serial8250_port *port = &legacy_serial_ports[console];
-	void __iomem *addr;
 	unsigned int stride;
 	stride = 1 << port->regshift;
 	/* Check if a translated MMIO address has been found */
 	if (info->taddr) {
-		addr = ioremap(info->taddr, 0x1000);
+		info->early_addr = early_ioremap(info->taddr, 0x1000);
-		if (addr == NULL)
+		if (info->early_addr == NULL)
 			return;
-		udbg_uart_init_mmio(addr, stride);
+		udbg_uart_init_mmio(info->early_addr, stride);
 	} else {
 		/* Check if it's PIO and we support untranslated PIO */
 		if (port->iotype == UPIO_PORT && isa_io_special)
 	udbg_uart_setup(info->speed, info->clock);
 }
+static int __init ioremap_legacy_serial_console(void)
+{
+	struct legacy_serial_info *info = &legacy_serial_infos[legacy_serial_console];
+	struct plat_serial8250_port *port = &legacy_serial_ports[legacy_serial_console];
+	void __iomem *vaddr;
+	if (legacy_serial_console < 0)
+		return 0;
+	if (!info->early_addr)
+		return 0;
+	vaddr = ioremap(info->taddr, 0x1000);
+	if (WARN_ON(!vaddr))
+		return -ENOMEM;
+	udbg_uart_init_mmio(vaddr, 1 << port->regshift);
+	early_iounmap(info->early_addr, 0x1000);
+	info->early_addr = NULL;
+	return 0;
+}
+early_initcall(ioremap_legacy_serial_console);
 /*
  * This is called very early, as part of setup_system() or eventually
  * setup_arch(), basically before anything else in this file. This function

arch/powerpc/kernel/mce.c+3−2

 	.func = machine_check_ue_irq_work,
 };
-DECLARE_WORK(mce_ue_event_work, machine_process_ue_event);
+static DECLARE_WORK(mce_ue_event_work, machine_process_ue_event);
 static BLOCKING_NOTIFIER_HEAD(mce_notifier_list);
 	 * Populate the mce error_type and type-specific error_type.
 	 */
 	mce_set_error_info(mce, mce_err);
+	if (mce->error_type == MCE_ERROR_TYPE_UE)
+		mce->u.ue_error.ignore_event = mce_err->ignore_event;
 	if (!addr)
 		return;
 		if (phys_addr != ULONG_MAX) {
 			mce->u.ue_error.physical_address_provided = true;
 			mce->u.ue_error.physical_address = phys_addr;
-			mce->u.ue_error.ignore_event = mce_err->ignore_event;
 			machine_check_ue_event(mce);
 		}
 	}

arch/powerpc/kernel/misc_32.S+0−39

@@ -27,45 +27,6 @@ · 26 unmodified lines
27		27
28	.text	28	.text
29		29
30	/*
31	* We store the saved ksp_limit in the unused part
32	* of the STACK_FRAME_OVERHEAD
33	*/
34	_GLOBAL(call_do_softirq)
35	mflr r0
36	stw r0,4(r1)
37	lwz r10,THREAD+KSP_LIMIT(r2)
38	stw r3, THREAD+KSP_LIMIT(r2)
39	stwu r1,THREAD_SIZE-STACK_FRAME_OVERHEAD(r3)
40	mr r1,r3
41	stw r10,8(r1)
42	bl __do_softirq
43	lwz r10,8(r1)
44	lwz r1,0(r1)
45	lwz r0,4(r1)
46	stw r10,THREAD+KSP_LIMIT(r2)
47	mtlr r0
48	blr
49
50	/*
51	* void call_do_irq(struct pt_regs regs, void sp);
52	*/
53	_GLOBAL(call_do_irq)
54	mflr r0
55	stw r0,4(r1)
56	lwz r10,THREAD+KSP_LIMIT(r2)
57	stw r4, THREAD+KSP_LIMIT(r2)
58	stwu r1,THREAD_SIZE-STACK_FRAME_OVERHEAD(r4)
59	mr r1,r4
60	stw r10,8(r1)
61	bl __do_irq
62	lwz r10,8(r1)
63	lwz r1,0(r1)
64	lwz r0,4(r1)
65	stw r10,THREAD+KSP_LIMIT(r2)
66	mtlr r0
67	blr
68
69	/*	30	/*
70	* This returns the high 64 bits of the product of two 64-bit numbers.	31	* This returns the high 64 bits of the product of two 64-bit numbers.
71	*/	32	*/

arch/powerpc/kernel/misc_64.S+0−22

@@ -27,28 +27,6 @@ · 26 unmodified lines
27		27
28	.text	28	.text
29		29
30	_GLOBAL(call_do_softirq)
31	mflr r0
32	std r0,16(r1)
33	stdu r1,THREAD_SIZE-STACK_FRAME_OVERHEAD(r3)
34	mr r1,r3
35	bl __do_softirq
36	ld r1,0(r1)
37	ld r0,16(r1)
38	mtlr r0
39	blr
40
41	_GLOBAL(call_do_irq)
42	mflr r0
43	std r0,16(r1)
44	stdu r1,THREAD_SIZE-STACK_FRAME_OVERHEAD(r4)
45	mr r1,r4
46	bl __do_irq
47	ld r1,0(r1)
48	ld r0,16(r1)
49	mtlr r0
50	blr
51
52	_GLOBAL(__bswapdi2)	30	_GLOBAL(__bswapdi2)
53	EXPORT_SYMBOL(__bswapdi2)	31	EXPORT_SYMBOL(__bswapdi2)
54	srdi r8,r3,32	32	srdi r8,r3,32

arch/powerpc/kernel/module.c+20−3

 #include <asm/firmware.h>
 #include <linux/sort.h>
 #include <asm/setup.h>
+#include <asm/sections.h>
 static LIST_HEAD(module_bug_list);
 }
 #ifdef MODULES_VADDR
+static __always_inline void *
+__module_alloc(unsigned long size, unsigned long start, unsigned long end)
+{
+	return __vmalloc_node_range(size, 1, start, end, GFP_KERNEL,
+				    PAGE_KERNEL_EXEC, VM_FLUSH_RESET_PERMS, NUMA_NO_NODE,
+				    __builtin_return_address(0));
+}
 void *module_alloc(unsigned long size)
 {
+	unsigned long limit = (unsigned long)_etext - SZ_32M;
+	void *ptr = NULL;
 	BUILD_BUG_ON(TASK_SIZE > MODULES_VADDR);
-	return __vmalloc_node_range(size, 1, MODULES_VADDR, MODULES_END, GFP_KERNEL,
+	/* First try within 32M limit from _etext to avoid branch trampolines */
-				    PAGE_KERNEL_EXEC, VM_FLUSH_RESET_PERMS, NUMA_NO_NODE,
+	if (MODULES_VADDR < PAGE_OFFSET && MODULES_END > limit)
-				    __builtin_return_address(0));
+		ptr = __module_alloc(size, limit, MODULES_END);
+	if (!ptr)
+		ptr = __module_alloc(size, MODULES_VADDR, MODULES_END);
+	return ptr;
 }
 #endif

arch/powerpc/kernel/optprobes.c+21−3

 	}
 }
+static void patch_imm32_load_insns(unsigned long val, int reg, kprobe_opcode_t *addr)
+{
+	patch_instruction((struct ppc_inst *)addr,
+			  ppc_inst(PPC_RAW_LIS(reg, IMM_H(val))));
+	addr++;
+	patch_instruction((struct ppc_inst *)addr,
+			  ppc_inst(PPC_RAW_ORI(reg, reg, IMM_L(val))));
+}
 /*
  * Generate instructions to load provided immediate 64-bit value
  * to register 'reg' and patch these instructions at 'addr'.
  */
-static void patch_imm64_load_insns(unsigned long val, int reg, kprobe_opcode_t *addr)
+static void patch_imm64_load_insns(unsigned long long val, int reg, kprobe_opcode_t *addr)
 {
 	/* lis reg,(op)@highest */
 	patch_instruction((struct ppc_inst *)addr,
 				   ___PPC_RS(reg) | (val & 0xffff)));
 }
+static void patch_imm_load_insns(unsigned long val, int reg, kprobe_opcode_t *addr)
+{
+	if (IS_ENABLED(CONFIG_PPC64))
+		patch_imm64_load_insns(val, reg, addr);
+	else
+		patch_imm32_load_insns(val, reg, addr);
+}
 int arch_prepare_optimized_kprobe(struct optimized_kprobe *op, struct kprobe *p)
 {
 	struct ppc_inst branch_op_callback, branch_emulate_step, temp;
 	 * Fixup the template with instructions to:
 	 * 1. load the address of the actual probepoint
 	 */
-	patch_imm64_load_insns((unsigned long)op, 3, buff + TMPL_OP_IDX);
+	patch_imm_load_insns((unsigned long)op, 3, buff + TMPL_OP_IDX);
 	/*
 	 * 2. branch to optimized_callback() and emulate_step()
 	 * 3. load instruction to be emulated into relevant register, and
 	 */
 	temp = ppc_inst_read((struct ppc_inst *)p->ainsn.insn);
-	patch_imm64_load_insns(ppc_inst_as_u64(temp), 4, buff + TMPL_INSN_IDX);
+	patch_imm_load_insns(ppc_inst_as_ulong(temp), 4, buff + TMPL_INSN_IDX);
 	/*
 	 * 4. branch back from trampoline

arch/powerpc/kernel/optprobes_head.S+34−31

@@ -9,6 +9,16 @@ · 8 unmodified lines
9	#include <asm/ptrace.h>	9	#include <asm/ptrace.h>
10	#include <asm/asm-offsets.h>	10	#include <asm/asm-offsets.h>
11		11
		12	#ifdef CONFIG_PPC64
		13	#define SAVE_30GPRS(base) SAVE_10GPRS(2,base); SAVE_10GPRS(12,base); SAVE_10GPRS(22,base)
		14	#define REST_30GPRS(base) REST_10GPRS(2,base); REST_10GPRS(12,base); REST_10GPRS(22,base)
		15	#define TEMPLATE_FOR_IMM_LOAD_INSNS nop; nop; nop; nop; nop
		16	#else
		17	#define SAVE_30GPRS(base) stmw r2, GPR2(base)
		18	#define REST_30GPRS(base) lmw r2, GPR2(base)
		19	#define TEMPLATE_FOR_IMM_LOAD_INSNS nop; nop; nop
		20	#endif
		21
12	#define OPT_SLOT_SIZE 65536	22	#define OPT_SLOT_SIZE 65536
13		23
14	.balign 4	24	.balign 4
@@ -30,39 +40,41 @@ optinsn_slot: · 15 unmodified lines
30	.global optprobe_template_entry	40	.global optprobe_template_entry
31	optprobe_template_entry:	41	optprobe_template_entry:
32	/* Create an in-memory pt_regs */	42	/* Create an in-memory pt_regs */
33	stdu r1,-INT_FRAME_SIZE(r1)	43	PPC_STLU r1,-INT_FRAME_SIZE(r1)
34	SAVE_GPR(0,r1)	44	SAVE_GPR(0,r1)
35	/* Save the previous SP into stack */	45	/* Save the previous SP into stack */
36	addi r0,r1,INT_FRAME_SIZE	46	addi r0,r1,INT_FRAME_SIZE
37	std r0,GPR1(r1)	47	PPC_STL r0,GPR1(r1)
38	SAVE_10GPRS(2,r1)	48	SAVE_30GPRS(r1)
39	SAVE_10GPRS(12,r1)
40	SAVE_10GPRS(22,r1)
41	/* Save SPRS */	49	/* Save SPRS */
42	mfmsr r5	50	mfmsr r5
43	std r5,_MSR(r1)	51	PPC_STL r5,_MSR(r1)
44	li r5,0x700	52	li r5,0x700
45	std r5,_TRAP(r1)	53	PPC_STL r5,_TRAP(r1)
46	li r5,0	54	li r5,0
47	std r5,ORIG_GPR3(r1)	55	PPC_STL r5,ORIG_GPR3(r1)
48	std r5,RESULT(r1)	56	PPC_STL r5,RESULT(r1)
49	mfctr r5	57	mfctr r5
50	std r5,_CTR(r1)	58	PPC_STL r5,_CTR(r1)
51	mflr r5	59	mflr r5
52	std r5,_LINK(r1)	60	PPC_STL r5,_LINK(r1)
53	mfspr r5,SPRN_XER	61	mfspr r5,SPRN_XER
54	std r5,_XER(r1)	62	PPC_STL r5,_XER(r1)
55	mfcr r5	63	mfcr r5
56	std r5,_CCR(r1)	64	PPC_STL r5,_CCR(r1)
		65	#ifdef CONFIG_PPC64
57	lbz r5,PACAIRQSOFTMASK(r13)	66	lbz r5,PACAIRQSOFTMASK(r13)
58	std r5,SOFTE(r1)	67	std r5,SOFTE(r1)
		68	#endif
59		69
60	/*	70	/*
61	* We may get here from a module, so load the kernel TOC in r2.	71	* We may get here from a module, so load the kernel TOC in r2.
62	* The original TOC gets restored when pt_regs is restored	72	* The original TOC gets restored when pt_regs is restored
63	* further below.	73	* further below.
64	*/	74	*/
		75	#ifdef CONFIG_PPC64
65	ld r2,PACATOC(r13)	76	ld r2,PACATOC(r13)
		77	#endif
66		78
67	.global optprobe_template_op_address	79	.global optprobe_template_op_address
68	optprobe_template_op_address:	80	optprobe_template_op_address:
@@ -70,11 +82,8 @@ optprobe_template_op_address: · 1 unmodified line
70	* Parameters to optimized_callback():	82	* Parameters to optimized_callback():
71	* 1. optimized_kprobe structure in r3	83	* 1. optimized_kprobe structure in r3
72	*/	84	*/
73	nop	85	TEMPLATE_FOR_IMM_LOAD_INSNS
74	nop	86
75	nop
76	nop
77	nop
78	/* 2. pt_regs pointer in r4 */	87	/* 2. pt_regs pointer in r4 */
79	addi r4,r1,STACK_FRAME_OVERHEAD	88	addi r4,r1,STACK_FRAME_OVERHEAD
80		89
@@ -92,11 +101,7 @@ optprobe_template_call_handler: · 11 unmodified lines
92	.global optprobe_template_insn	101	.global optprobe_template_insn
93	optprobe_template_insn:	102	optprobe_template_insn:
94	/* 2, Pass instruction to be emulated in r4 */	103	/* 2, Pass instruction to be emulated in r4 */
95	nop	104	TEMPLATE_FOR_IMM_LOAD_INSNS
96	nop
97	nop
98	nop
99	nop
100		105
101	.global optprobe_template_call_emulate	106	.global optprobe_template_call_emulate
102	optprobe_template_call_emulate:	107	optprobe_template_call_emulate:
@@ -107,20 +112,18 @@ optprobe_template_call_emulate: · 4 unmodified lines
107	* All done.	112	* All done.
108	* Now, restore the registers...	113	* Now, restore the registers...
109	*/	114	*/
110	ld r5,_MSR(r1)	115	PPC_LL r5,_MSR(r1)
111	mtmsr r5	116	mtmsr r5
112	ld r5,_CTR(r1)	117	PPC_LL r5,_CTR(r1)
113	mtctr r5	118	mtctr r5
114	ld r5,_LINK(r1)	119	PPC_LL r5,_LINK(r1)
115	mtlr r5	120	mtlr r5
116	ld r5,_XER(r1)	121	PPC_LL r5,_XER(r1)
117	mtxer r5	122	mtxer r5
118	ld r5,_CCR(r1)	123	PPC_LL r5,_CCR(r1)
119	mtcr r5	124	mtcr r5
120	REST_GPR(0,r1)	125	REST_GPR(0,r1)
121	REST_10GPRS(2,r1)	126	REST_30GPRS(r1)
122	REST_10GPRS(12,r1)
123	REST_10GPRS(22,r1)
124	/* Restore the previous SP */	127	/* Restore the previous SP */
125	addi r1,r1,INT_FRAME_SIZE	128	addi r1,r1,INT_FRAME_SIZE
126		129

arch/powerpc/kernel/process.c+8−17

 	regs->msr |= msr_diff;
 }
-#else
+#else /* !CONFIG_PPC_TRANSACTIONAL_MEM */
 #define tm_recheckpoint_new_task(new)
 #define __switch_to_tm(prev, new)
+void tm_reclaim_current(uint8_t cause) {}
 #endif /* CONFIG_PPC_TRANSACTIONAL_MEM */
 static inline void save_sprs(struct thread_struct *t)
 	 */
 	restore_sprs(old_thread, new_thread);
+#ifdef CONFIG_PPC32
+	kuap_assert_locked();
+#endif
 	last = _switch(old_thread, new_thread);
 #ifdef CONFIG_PPC_BOOK3S_64
 #ifdef CONFIG_PPC64
 #define REG		"%016lx"
 #define REGS_PER_LINE	4
-#define LAST_VOLATILE	13
 #else
 #define REG		"%08lx"
 #define REGS_PER_LINE	8
-#define LAST_VOLATILE	12
 #endif
 static void __show_regs(struct pt_regs *regs)
 	trap = TRAP(regs);
 	if (!trap_is_syscall(regs) && cpu_has_feature(CPU_FTR_CFAR))
 		pr_cont("CFAR: "REG" ", regs->orig_gpr3);
-	if (trap == 0x200 || trap == 0x300 || trap == 0x600) {
+	if (trap == INTERRUPT_MACHINE_CHECK ||
+	    trap == INTERRUPT_DATA_STORAGE ||
+	    trap == INTERRUPT_ALIGNMENT) {
 		if (IS_ENABLED(CONFIG_4xx) || IS_ENABLED(CONFIG_BOOKE))
 			pr_cont("DEAR: "REG" ESR: "REG" ", regs->dar, regs->dsisr);
 		else
 		if ((i % REGS_PER_LINE) == 0)
 			pr_cont("\nGPR%02d: ", i);
 		pr_cont(REG " ", regs->gpr[i]);
-		if (i == LAST_VOLATILE && !FULL_REGS(regs))
-			break;
 	}
 	pr_cont("\n");
 	/*
 	} else {
 		/* user thread */
 		struct pt_regs *regs = current_pt_regs();
-		CHECK_FULL_REGS(regs);
 		*childregs = *regs;
 		if (usp)
 			childregs->gpr[1] = usp;
 	kregs = (struct pt_regs *) sp;
 	sp -= STACK_FRAME_OVERHEAD;
 	p->thread.ksp = sp;
-#ifdef CONFIG_PPC32
-	p->thread.ksp_limit = (unsigned long)end_of_stack(p);
-#endif
 #ifdef CONFIG_HAVE_HW_BREAKPOINT
 	for (i = 0; i < nr_wp_slots(); i++)
 		p->thread.ptrace_bps[i] = NULL;
 	regs->ccr = 0;
 	regs->gpr[1] = sp;
-	/*
-	 * We have just cleared all the nonvolatile GPRs, so make
-	 * FULL_REGS(regs) return true.  This is necessary to allow
-	 * ptrace to examine the thread immediately after exec.
-	 */
-	SET_FULL_REGS(regs);
 #ifdef CONFIG_PPC32
 	regs->mq = 0;
 	regs->nip = start;

arch/powerpc/kernel/prom.c+16−5

 #define DBG(fmt...)
 #endif
+int *chip_id_lookup_table;
 #ifdef CONFIG_PPC64
 int __initdata iommu_is_off;
 int __initdata iommu_force_on;
 };
 #if defined(CONFIG_44x) && defined(CONFIG_PPC_FPU)
-static inline void identical_pvr_fixup(unsigned long node)
+static __init void identical_pvr_fixup(unsigned long node)
 {
 	unsigned int pvr;
 	const char *model = of_get_flat_dt_prop(node, "model", NULL);
 int cpu_to_chip_id(int cpu)
 {
 	struct device_node *np;
+	int ret = -1, idx;
+	idx = cpu / threads_per_core;
+	if (chip_id_lookup_table && chip_id_lookup_table[idx] != -1)
+		return chip_id_lookup_table[idx];
 	np = of_get_cpu_node(cpu, NULL);
-	if (!np)
+	if (np) {
-		return -1;
+		ret = of_get_ibm_chip_id(np);
+		of_node_put(np);
+		if (chip_id_lookup_table)
+			chip_id_lookup_table[idx] = ret;
+	}
-	of_node_put(np);
+	return ret;
-	return of_get_ibm_chip_id(np);
 }
 EXPORT_SYMBOL(cpu_to_chip_id);

arch/powerpc/kernel/prom_init.c+1−1

@@ -2983,7 +2983,7 @@ static void __init fixup_device_tree_efika_add_phy(void) · 2982 unmodified lines
2983	" 0x3 encode-int encode+"	2983	" 0x3 encode-int encode+"
2984	" s\" interrupts\" property"	2984	" s\" interrupts\" property"
2985	" finish-device");	2985	" finish-device");
2986	};	2986	}
2987		2987
2988	/* Check for a PHY device node - if missing then create one and	2988	/* Check for a PHY device node - if missing then create one and
2989	* give it's phandle to the ethernet node */	2989	* give it's phandle to the ethernet node */

arch/powerpc/kernel/ptrace/ptrace-view.c+18−33

 	return task->thread.regs->msr | task->thread.fpexc_mode;
 }
-static int set_user_msr(struct task_struct *task, unsigned long msr)
+static __always_inline int set_user_msr(struct task_struct *task, unsigned long msr)
 {
 	task->thread.regs->msr &= ~MSR_DEBUGCHANGE;
 	task->thread.regs->msr |= msr & MSR_DEBUGCHANGE;
  * We prevent mucking around with the reserved area of trap
  * which are used internally by the kernel.
  */
-static int set_user_trap(struct task_struct *task, unsigned long trap)
+static __always_inline int set_user_trap(struct task_struct *task, unsigned long trap)
 {
 	set_trap(task->thread.regs, trap);
 	return 0;
 #ifdef CONFIG_PPC64
 	struct membuf to_softe = membuf_at(&to, offsetof(struct pt_regs, softe));
 #endif
-	int i;
 	if (target->thread.regs == NULL)
 		return -EIO;
-	if (!FULL_REGS(target->thread.regs)) {
-		/* We have a partial register set.  Fill 14-31 with bogus values */
-		for (i = 14; i < 32; i++)
-			target->thread.regs->gpr[i] = NV_REG_POISON;
-	}
 	membuf_write(&to, target->thread.regs, sizeof(struct user_pt_regs));
 	membuf_store(&to_msr, get_user_msr(target));
 	if (target->thread.regs == NULL)
 		return -EIO;
-	CHECK_FULL_REGS(target->thread.regs);
 	ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf,
 				 target->thread.regs,
 				 0, PT_MSR * sizeof(reg));
 	const compat_ulong_t __user *u = ubuf;
 	compat_ulong_t reg;
+	if (!kbuf && !user_read_access_begin(u, count))
+		return -EFAULT;
 	pos /= sizeof(reg);
 	count /= sizeof(reg);
 			regs[pos++] = *k++;
 	else
 		for (; count > 0 && pos < PT_MSR; --count) {
-			if (__get_user(reg, u++))
+			unsafe_get_user(reg, u++, Efault);
-				return -EFAULT;
 			regs[pos++] = reg;
 		}
 	if (count > 0 && pos == PT_MSR) {
 		if (kbuf)
 			reg = *k++;
-		else if (__get_user(reg, u++))
+		else
-			return -EFAULT;
+			unsafe_get_user(reg, u++, Efault);
 		set_user_msr(target, reg);
 		++pos;
 		--count;
 			++k;
 	} else {
 		for (; count > 0 && pos <= PT_MAX_PUT_REG; --count) {
-			if (__get_user(reg, u++))
+			unsafe_get_user(reg, u++, Efault);
-				return -EFAULT;
 			regs[pos++] = reg;
 		}
 		for (; count > 0 && pos < PT_TRAP; --count, ++pos)
-			if (__get_user(reg, u++))
+			unsafe_get_user(reg, u++, Efault);
-				return -EFAULT;
 	}
 	if (count > 0 && pos == PT_TRAP) {
 		if (kbuf)
 			reg = *k++;
-		else if (__get_user(reg, u++))
+		else
-			return -EFAULT;
+			unsafe_get_user(reg, u++, Efault);
 		set_user_trap(target, reg);
 		++pos;
 		--count;
 	}
+	if (!kbuf)
+		user_read_access_end();
 	kbuf = k;
 	ubuf = u;
 	count *= sizeof(reg);
 	return user_regset_copyin_ignore(&pos, &count, &kbuf, &ubuf,
 					 (PT_TRAP + 1) * sizeof(reg), -1);
+Efault:
+	user_read_access_end();
+	return -EFAULT;
 }
 static int gpr32_get(struct task_struct *target,
 		     const struct user_regset *regset,
 		     struct membuf to)
 {
-	int i;
 	if (target->thread.regs == NULL)
 		return -EIO;
-	if (!FULL_REGS(target->thread.regs)) {
-		/*
-		 * We have a partial register set.
-		 * Fill 14-31 with bogus values.
-		 */
-		for (i = 14; i < 32; i++)
-			target->thread.regs->gpr[i] = NV_REG_POISON;
-	}
 	return gpr32_get_common(target, regset, to,
 			&target->thread.regs->gpr[0]);
 }
 	if (target->thread.regs == NULL)
 		return -EIO;
-	CHECK_FULL_REGS(target->thread.regs);
 	return gpr32_set_common(target, regset, pos, count, kbuf, ubuf,
 			&target->thread.regs->gpr[0]);
 }

arch/powerpc/kernel/ptrace/ptrace.c+0−4

 		if ((addr & (sizeof(long) - 1)) || !child->thread.regs)
 			break;
-		CHECK_FULL_REGS(child->thread.regs);
 		if (index < PT_FPR0)
 			ret = ptrace_get_reg(child, (int) index, &tmp);
 		else
 		if ((addr & (sizeof(long) - 1)) || !child->thread.regs)
 			break;
-		CHECK_FULL_REGS(child->thread.regs);
 		if (index < PT_FPR0)
 			ret = ptrace_put_reg(child, index, data);
 		else
 		     offsetof(struct user_pt_regs, nip));
 	BUILD_BUG_ON(offsetof(struct pt_regs, msr) !=
 		     offsetof(struct user_pt_regs, msr));
-	BUILD_BUG_ON(offsetof(struct pt_regs, msr) !=
-		     offsetof(struct user_pt_regs, msr));
 	BUILD_BUG_ON(offsetof(struct pt_regs, orig_gpr3) !=
 		     offsetof(struct user_pt_regs, orig_gpr3));
 	BUILD_BUG_ON(offsetof(struct pt_regs, ctr) !=

arch/powerpc/kernel/ptrace/ptrace32.c+0−4

 		if ((addr & 3) || (index > PT_FPSCR32))
 			break;
-		CHECK_FULL_REGS(child->thread.regs);
 		if (index < PT_FPR0) {
 			ret = ptrace_get_reg(child, index, &tmp);
 			if (ret)
 		if ((addr & 3) || numReg > PT_FPSCR)
 			break;
-		CHECK_FULL_REGS(child->thread.regs);
 		if (numReg >= PT_FPR0) {
 			flush_fp_to_thread(child);
 			/* get 64 bit FPR */
 		if ((addr & 3) || (index > PT_FPSCR32))
 			break;
-		CHECK_FULL_REGS(child->thread.regs);
 		if (index < PT_FPR0) {
 			ret = ptrace_put_reg(child, index, data);
 		} else {
 		 */
 		if ((addr & 3) || (numReg > PT_FPSCR))
 			break;
-		CHECK_FULL_REGS(child->thread.regs);
 		if (numReg < PT_FPR0) {
 			unsigned long freg;
 			ret = ptrace_get_reg(child, numReg, &freg);

arch/powerpc/kernel/rtas-proc.c+11−4

@@ -755,11 +755,18 @@ static int ppc_rtas_tone_volume_show(struct seq_file m, void v) · 754 unmodified lines
755	return 0;	755	return 0;
756	}	756	}
757		757
758	#define RMO_READ_BUF_MAX 30	758	/**
759		759	* ppc_rtas_rmo_buf_show() - Describe RTAS-addressable region for user space.
760	/* RTAS Userspace access */	760	*
		761	* Base + size description of a range of RTAS-addressable memory set
		762	* aside for user space to use as work area(s) for certain RTAS
		763	* functions. User space accesses this region via /dev/mem. Apart from
		764	* security policies, the kernel does not arbitrate or serialize
		765	* access to this region, and user space must ensure that concurrent
		766	* users do not interfere with each other.
		767	*/
761	static int ppc_rtas_rmo_buf_show(struct seq_file m, void v)	768	static int ppc_rtas_rmo_buf_show(struct seq_file m, void v)
762	{	769	{
763	seq_printf(m, "%016lx %x\n", rtas_rmo_buf, RTAS_RMOBUF_MAX);	770	seq_printf(m, "%016lx %x\n", rtas_rmo_buf, RTAS_USER_REGION_SIZE);
764	return 0;	771	return 0;
765	}	772	}

arch/powerpc/kernel/rtas.c+18−16

 		pr_err("ibm,activate-firmware failed (%i)\n", fwrc);
 }
-static int ibm_suspend_me_token = RTAS_UNKNOWN_SERVICE;
 #ifdef CONFIG_PPC_PSERIES
 /**
  * rtas_call_reentrant() - Used for reentrant rtas calls
 static bool in_rmo_buf(u32 base, u32 end)
 {
 	return base >= rtas_rmo_buf &&
-		base < (rtas_rmo_buf + RTAS_RMOBUF_MAX) &&
+		base < (rtas_rmo_buf + RTAS_USER_REGION_SIZE) &&
 		base <= end &&
 		end >= rtas_rmo_buf &&
-		end < (rtas_rmo_buf + RTAS_RMOBUF_MAX);
+		end < (rtas_rmo_buf + RTAS_USER_REGION_SIZE);
 }
 static bool block_rtas_call(int token, int nargs,
 	return true;
 }
+static void __init rtas_syscall_filter_init(void)
+{
+	unsigned int i;
+	for (i = 0; i < ARRAY_SIZE(rtas_filters); i++)
+		rtas_filters[i].token = rtas_token(rtas_filters[i].name);
+}
 #else
 static bool block_rtas_call(int token, int nargs,
 	return false;
 }
+static void __init rtas_syscall_filter_init(void)
+{
+}
 #endif /* CONFIG_PPC_RTAS_FILTER */
 /* We assume to be passed big endian arguments */
 		return -EINVAL;
 	/* Need to handle ibm,suspend_me call specially */
-	if (token == ibm_suspend_me_token) {
+	if (token == rtas_token("ibm,suspend-me")) {
 		/*
 		 * rtas_ibm_suspend_me assumes the streamid handle is in cpu
 	unsigned long rtas_region = RTAS_INSTANTIATE_MAX;
 	u32 base, size, entry;
 	int no_base, no_size, no_entry;
-#ifdef CONFIG_PPC_RTAS_FILTER
-	int i;
-#endif
 	/* Get RTAS dev node and fill up our "rtas" structure with infos
 	 * about it.
 	 * the stop-self token if any
 	 */
 #ifdef CONFIG_PPC64
-	if (firmware_has_feature(FW_FEATURE_LPAR)) {
+	if (firmware_has_feature(FW_FEATURE_LPAR))
 		rtas_region = min(ppc64_rma_size, RTAS_INSTANTIATE_MAX);
-		ibm_suspend_me_token = rtas_token("ibm,suspend-me");
-	}
 #endif
-	rtas_rmo_buf = memblock_phys_alloc_range(RTAS_RMOBUF_MAX, PAGE_SIZE,
+	rtas_rmo_buf = memblock_phys_alloc_range(RTAS_USER_REGION_SIZE, PAGE_SIZE,
 						 0, rtas_region);
 	if (!rtas_rmo_buf)
 		panic("ERROR: RTAS: Failed to allocate %lx bytes below %pa\n",
 	rtas_last_error_token = rtas_token("rtas-last-error");
 #endif
-#ifdef CONFIG_PPC_RTAS_FILTER
+	rtas_syscall_filter_init();
-	for (i = 0; i < ARRAY_SIZE(rtas_filters); i++) {
-		rtas_filters[i].token = rtas_token(rtas_filters[i].name);
-	}
-#endif
 }
 int __init early_init_dt_scan_rtas(unsigned long node,

arch/powerpc/kernel/security.c+262−1

@@ -7,6 +7,7 @@ · 6 unmodified lines
7	#include <linux/cpu.h>	7	#include <linux/cpu.h>
8	#include <linux/kernel.h>	8	#include <linux/kernel.h>
9	#include <linux/device.h>	9	#include <linux/device.h>
		10	#include <linux/memblock.h>
10	#include <linux/nospec.h>	11	#include <linux/nospec.h>
11	#include <linux/prctl.h>	12	#include <linux/prctl.h>
12	#include <linux/seq_buf.h>	13	#include <linux/seq_buf.h>
@@ -18,6 +19,7 @@ · 5 unmodified lines
18	#include <asm/setup.h>	19	#include <asm/setup.h>
19	#include <asm/inst.h>	20	#include <asm/inst.h>
20		21
		22	#include "setup.h"
21		23
22	u64 powerpc_security_features __read_mostly = SEC_FTR_DEFAULT;	24	u64 powerpc_security_features __read_mostly = SEC_FTR_DEFAULT;
23		25
@@ -250,7 +252,7 @@ ssize_t cpu_show_spectre_v2(struct device dev, struct device_attribute attr, c · 226 unmodified lines
250		252
251	static enum stf_barrier_type stf_enabled_flush_types;	253	static enum stf_barrier_type stf_enabled_flush_types;
252	static bool no_stf_barrier;	254	static bool no_stf_barrier;
253	bool stf_barrier;	255	static bool stf_barrier;
254		256
255	static int __init handle_no_stf_barrier(char *p)	257	static int __init handle_no_stf_barrier(char *p)
256	{	258	{
@@ -541,6 +543,178 @@ void setup_count_cache_flush(void) · 284 unmodified lines
541	toggle_branch_cache_flush(enable);	543	toggle_branch_cache_flush(enable);
542	}	544	}
543		545
		546	static enum l1d_flush_type enabled_flush_types;
		547	static void *l1d_flush_fallback_area;
		548	static bool no_rfi_flush;
		549	static bool no_entry_flush;
		550	static bool no_uaccess_flush;
		551	bool rfi_flush;
		552	static bool entry_flush;
		553	static bool uaccess_flush;
		554	DEFINE_STATIC_KEY_FALSE(uaccess_flush_key);
		555	EXPORT_SYMBOL(uaccess_flush_key);
		556
		557	static int __init handle_no_rfi_flush(char *p)
		558	{
		559	pr_info("rfi-flush: disabled on command line.");
		560	no_rfi_flush = true;
		561	return 0;
		562	}
		563	early_param("no_rfi_flush", handle_no_rfi_flush);
		564
		565	static int __init handle_no_entry_flush(char *p)
		566	{
		567	pr_info("entry-flush: disabled on command line.");
		568	no_entry_flush = true;
		569	return 0;
		570	}
		571	early_param("no_entry_flush", handle_no_entry_flush);
		572
		573	static int __init handle_no_uaccess_flush(char *p)
		574	{
		575	pr_info("uaccess-flush: disabled on command line.");
		576	no_uaccess_flush = true;
		577	return 0;
		578	}
		579	early_param("no_uaccess_flush", handle_no_uaccess_flush);
		580
		581	/*
		582	* The RFI flush is not KPTI, but because users will see doco that says to use
		583	* nopti we hijack that option here to also disable the RFI flush.
		584	*/
		585	static int __init handle_no_pti(char *p)
		586	{
		587	pr_info("rfi-flush: disabling due to 'nopti' on command line.\n");
		588	handle_no_rfi_flush(NULL);
		589	return 0;
		590	}
		591	early_param("nopti", handle_no_pti);
		592
		593	static void do_nothing(void *unused)
		594	{
		595	/*
		596	* We don't need to do the flush explicitly, just enter+exit kernel is
		597	* sufficient, the RFI exit handlers will do the right thing.
		598	*/
		599	}
		600
		601	void rfi_flush_enable(bool enable)
		602	{
		603	if (enable) {
		604	do_rfi_flush_fixups(enabled_flush_types);
		605	on_each_cpu(do_nothing, NULL, 1);
		606	} else
		607	do_rfi_flush_fixups(L1D_FLUSH_NONE);
		608
		609	rfi_flush = enable;
		610	}
		611
		612	static void entry_flush_enable(bool enable)
		613	{
		614	if (enable) {
		615	do_entry_flush_fixups(enabled_flush_types);
		616	on_each_cpu(do_nothing, NULL, 1);
		617	} else {
		618	do_entry_flush_fixups(L1D_FLUSH_NONE);
		619	}
		620
		621	entry_flush = enable;
		622	}
		623
		624	static void uaccess_flush_enable(bool enable)
		625	{
		626	if (enable) {
		627	do_uaccess_flush_fixups(enabled_flush_types);
		628	static_branch_enable(&uaccess_flush_key);
		629	on_each_cpu(do_nothing, NULL, 1);
		630	} else {
		631	static_branch_disable(&uaccess_flush_key);
		632	do_uaccess_flush_fixups(L1D_FLUSH_NONE);
		633	}
		634
		635	uaccess_flush = enable;
		636	}
		637
		638	static void __ref init_fallback_flush(void)
		639	{
		640	u64 l1d_size, limit;
		641	int cpu;
		642
		643	/* Only allocate the fallback flush area once (at boot time). */
		644	if (l1d_flush_fallback_area)
		645	return;
		646
		647	l1d_size = ppc64_caches.l1d.size;
		648
		649	/*
		650	* If there is no d-cache-size property in the device tree, l1d_size
		651	* could be zero. That leads to the loop in the asm wrapping around to
		652	* 2^64-1, and then walking off the end of the fallback area and
		653	* eventually causing a page fault which is fatal. Just default to
		654	* something vaguely sane.
		655	*/
		656	if (!l1d_size)
		657	l1d_size = (64 * 1024);
		658
		659	limit = min(ppc64_bolted_size(), ppc64_rma_size);
		660
		661	/*
		662	* Align to L1d size, and size it at 2x L1d size, to catch possible
		663	* hardware prefetch runoff. We don't have a recipe for load patterns to
		664	* reliably avoid the prefetcher.
		665	*/
		666	l1d_flush_fallback_area = memblock_alloc_try_nid(l1d_size * 2,
		667	l1d_size, MEMBLOCK_LOW_LIMIT,
		668	limit, NUMA_NO_NODE);
		669	if (!l1d_flush_fallback_area)
		670	panic("%s: Failed to allocate %llu bytes align=0x%llx max_addr=%pa\n",
		671	__func__, l1d_size * 2, l1d_size, &limit);
		672
		673
		674	for_each_possible_cpu(cpu) {
		675	struct paca_struct *paca = paca_ptrs[cpu];
		676	paca->rfi_flush_fallback_area = l1d_flush_fallback_area;
		677	paca->l1d_flush_size = l1d_size;
		678	}
		679	}
		680
		681	void setup_rfi_flush(enum l1d_flush_type types, bool enable)
		682	{
		683	if (types & L1D_FLUSH_FALLBACK) {
		684	pr_info("rfi-flush: fallback displacement flush available\n");
		685	init_fallback_flush();
		686	}
		687
		688	if (types & L1D_FLUSH_ORI)
		689	pr_info("rfi-flush: ori type flush available\n");
		690
		691	if (types & L1D_FLUSH_MTTRIG)
		692	pr_info("rfi-flush: mttrig type flush available\n");
		693
		694	enabled_flush_types = types;
		695
		696	if (!cpu_mitigations_off() && !no_rfi_flush)
		697	rfi_flush_enable(enable);
		698	}
		699
		700	void setup_entry_flush(bool enable)
		701	{
		702	if (cpu_mitigations_off())
		703	return;
		704
		705	if (!no_entry_flush)
		706	entry_flush_enable(enable);
		707	}
		708
		709	void setup_uaccess_flush(bool enable)
		710	{
		711	if (cpu_mitigations_off())
		712	return;
		713
		714	if (!no_uaccess_flush)
		715	uaccess_flush_enable(enable);
		716	}
		717
544	#ifdef CONFIG_DEBUG_FS	718	#ifdef CONFIG_DEBUG_FS
545	static int count_cache_flush_set(void *data, u64 val)	719	static int count_cache_flush_set(void *data, u64 val)
546	{	720	{
@@ -579,5 +753,92 @@ static __init int count_cache_flush_debugfs_init(void) · 32 unmodified lines
579	return 0;	753	return 0;
580	}	754	}
581	device_initcall(count_cache_flush_debugfs_init);	755	device_initcall(count_cache_flush_debugfs_init);
		756
		757	static int rfi_flush_set(void *data, u64 val)
		758	{
		759	bool enable;
		760
		761	if (val == 1)
		762	enable = true;
		763	else if (val == 0)
		764	enable = false;
		765	else
		766	return -EINVAL;
		767
		768	/* Only do anything if we're changing state */
		769	if (enable != rfi_flush)
		770	rfi_flush_enable(enable);
		771
		772	return 0;
		773	}
		774
		775	static int rfi_flush_get(void data, u64 val)
		776	{
		777	*val = rfi_flush ? 1 : 0;
		778	return 0;
		779	}
		780
		781	DEFINE_SIMPLE_ATTRIBUTE(fops_rfi_flush, rfi_flush_get, rfi_flush_set, "%llu\n");
		782
		783	static int entry_flush_set(void *data, u64 val)
		784	{
		785	bool enable;
		786
		787	if (val == 1)
		788	enable = true;
		789	else if (val == 0)
		790	enable = false;
		791	else
		792	return -EINVAL;
		793
		794	/* Only do anything if we're changing state */
		795	if (enable != entry_flush)
		796	entry_flush_enable(enable);
		797
		798	return 0;
		799	}
		800
		801	static int entry_flush_get(void data, u64 val)
		802	{
		803	*val = entry_flush ? 1 : 0;
		804	return 0;
		805	}
		806
		807	DEFINE_SIMPLE_ATTRIBUTE(fops_entry_flush, entry_flush_get, entry_flush_set, "%llu\n");
		808
		809	static int uaccess_flush_set(void *data, u64 val)
		810	{
		811	bool enable;
		812
		813	if (val == 1)
		814	enable = true;
		815	else if (val == 0)
		816	enable = false;
		817	else
		818	return -EINVAL;
		819
		820	/* Only do anything if we're changing state */
		821	if (enable != uaccess_flush)
		822	uaccess_flush_enable(enable);
		823
		824	return 0;
		825	}
		826
		827	static int uaccess_flush_get(void data, u64 val)
		828	{
		829	*val = uaccess_flush ? 1 : 0;
		830	return 0;
		831	}
		832
		833	DEFINE_SIMPLE_ATTRIBUTE(fops_uaccess_flush, uaccess_flush_get, uaccess_flush_set, "%llu\n");
		834
		835	static __init int rfi_flush_debugfs_init(void)
		836	{
		837	debugfs_create_file("rfi_flush", 0600, powerpc_debugfs_root, NULL, &fops_rfi_flush);
		838	debugfs_create_file("entry_flush", 0600, powerpc_debugfs_root, NULL, &fops_entry_flush);
		839	debugfs_create_file("uaccess_flush", 0600, powerpc_debugfs_root, NULL, &fops_uaccess_flush);
		840	return 0;
		841	}
		842	device_initcall(rfi_flush_debugfs_init);
582	#endif /* CONFIG_DEBUG_FS */	843	#endif /* CONFIG_DEBUG_FS */
583	#endif /* CONFIG_PPC_BOOK3S_64 */	844	#endif /* CONFIG_PPC_BOOK3S_64 */

arch/powerpc/kernel/setup-common.c+1−2

 #include "setup.h"
 #ifdef DEBUG
-#include <asm/udbg.h>
 #define DBG(fmt...) udbg_printf(fmt)
 #else
 #define DBG(fmt...)
 }
 #ifdef CONFIG_SMP
-static void smp_setup_pacas(void)
+static void __init smp_setup_pacas(void)
 {
 	int cpu;

arch/powerpc/kernel/setup_32.c+1−1

@@ -164,7 +164,7 @@ void __init irqstack_early_init(void) · 163 unmodified lines
164	}	164	}
165		165
166	#ifdef CONFIG_VMAP_STACK	166	#ifdef CONFIG_VMAP_STACK
167	void *emergency_ctx[NR_CPUS] __ro_after_init;	167	void *emergency_ctx[NR_CPUS] __ro_after_init = {[0] = &init_stack};
168		168
169	void __init emergency_stack_init(void)	169	void __init emergency_stack_init(void)
170	{	170	{

arch/powerpc/kernel/setup_64.c+16−266

@@ -232,10 +232,23 @@ static void cpu_ready_for_interrupts(void) · 231 unmodified lines
232	* If we are not in hypervisor mode the job is done once for	232	* If we are not in hypervisor mode the job is done once for
233	* the whole partition in configure_exceptions().	233	* the whole partition in configure_exceptions().
234	*/	234	*/
235	if (cpu_has_feature(CPU_FTR_HVMODE) &&	235	if (cpu_has_feature(CPU_FTR_HVMODE)) {
236	cpu_has_feature(CPU_FTR_ARCH_207S)) {
237	unsigned long lpcr = mfspr(SPRN_LPCR);	236	unsigned long lpcr = mfspr(SPRN_LPCR);
238	mtspr(SPRN_LPCR, lpcr \| LPCR_AIL_3);	237	unsigned long new_lpcr = lpcr;
		238
		239	if (cpu_has_feature(CPU_FTR_ARCH_31)) {
		240	/* P10 DD1 does not have HAIL */
		241	if (pvr_version_is(PVR_POWER10) &&
		242	(mfspr(SPRN_PVR) & 0xf00) == 0x100)
		243	new_lpcr \|= LPCR_AIL_3;
		244	else
		245	new_lpcr \|= LPCR_HAIL;
		246	} else if (cpu_has_feature(CPU_FTR_ARCH_207S)) {
		247	new_lpcr \|= LPCR_AIL_3;
		248	}
		249
		250	if (new_lpcr != lpcr)
		251	mtspr(SPRN_LPCR, new_lpcr);
239	}	252	}
240		253
241	/*	254	/*
@@ -941,266 +954,3 @@ static int __init disable_hardlockup_detector(void) · 699 unmodified lines
941	return 0;	954	return 0;
942	}	955	}
943	early_initcall(disable_hardlockup_detector);	956	early_initcall(disable_hardlockup_detector);
944
945	#ifdef CONFIG_PPC_BOOK3S_64
946	static enum l1d_flush_type enabled_flush_types;
947	static void *l1d_flush_fallback_area;
948	static bool no_rfi_flush;
949	static bool no_entry_flush;
950	static bool no_uaccess_flush;
951	bool rfi_flush;
952	bool entry_flush;
953	bool uaccess_flush;
954	DEFINE_STATIC_KEY_FALSE(uaccess_flush_key);
955	EXPORT_SYMBOL(uaccess_flush_key);
956
957	static int __init handle_no_rfi_flush(char *p)
958	{
959	pr_info("rfi-flush: disabled on command line.");
960	no_rfi_flush = true;
961	return 0;
962	}
963	early_param("no_rfi_flush", handle_no_rfi_flush);
964
965	static int __init handle_no_entry_flush(char *p)
966	{
967	pr_info("entry-flush: disabled on command line.");
968	no_entry_flush = true;
969	return 0;
970	}
971	early_param("no_entry_flush", handle_no_entry_flush);
972
973	static int __init handle_no_uaccess_flush(char *p)
974	{
975	pr_info("uaccess-flush: disabled on command line.");
976	no_uaccess_flush = true;
977	return 0;
978	}
979	early_param("no_uaccess_flush", handle_no_uaccess_flush);
980
981	/*
982	* The RFI flush is not KPTI, but because users will see doco that says to use
983	* nopti we hijack that option here to also disable the RFI flush.
984	*/
985	static int __init handle_no_pti(char *p)
986	{
987	pr_info("rfi-flush: disabling due to 'nopti' on command line.\n");
988	handle_no_rfi_flush(NULL);
989	return 0;
990	}
991	early_param("nopti", handle_no_pti);
992
993	static void do_nothing(void *unused)
994	{
995	/*
996	* We don't need to do the flush explicitly, just enter+exit kernel is
997	* sufficient, the RFI exit handlers will do the right thing.
998	*/
999	}
1000
1001	void rfi_flush_enable(bool enable)
1002	{
1003	if (enable) {
1004	do_rfi_flush_fixups(enabled_flush_types);
1005	on_each_cpu(do_nothing, NULL, 1);
1006	} else
1007	do_rfi_flush_fixups(L1D_FLUSH_NONE);
1008
1009	rfi_flush = enable;
1010	}
1011
1012	static void entry_flush_enable(bool enable)
1013	{
1014	if (enable) {
1015	do_entry_flush_fixups(enabled_flush_types);
1016	on_each_cpu(do_nothing, NULL, 1);
1017	} else {
1018	do_entry_flush_fixups(L1D_FLUSH_NONE);
1019	}
1020
1021	entry_flush = enable;
1022	}
1023
1024	static void uaccess_flush_enable(bool enable)
1025	{
1026	if (enable) {
1027	do_uaccess_flush_fixups(enabled_flush_types);
1028	static_branch_enable(&uaccess_flush_key);
1029	on_each_cpu(do_nothing, NULL, 1);
1030	} else {
1031	static_branch_disable(&uaccess_flush_key);
1032	do_uaccess_flush_fixups(L1D_FLUSH_NONE);
1033	}
1034
1035	uaccess_flush = enable;
1036	}
1037
1038	static void __ref init_fallback_flush(void)
1039	{
1040	u64 l1d_size, limit;
1041	int cpu;
1042
1043	/* Only allocate the fallback flush area once (at boot time). */
1044	if (l1d_flush_fallback_area)
1045	return;
1046
1047	l1d_size = ppc64_caches.l1d.size;
1048
1049	/*
1050	* If there is no d-cache-size property in the device tree, l1d_size
1051	* could be zero. That leads to the loop in the asm wrapping around to
1052	* 2^64-1, and then walking off the end of the fallback area and
1053	* eventually causing a page fault which is fatal. Just default to
1054	* something vaguely sane.
1055	*/
1056	if (!l1d_size)
1057	l1d_size = (64 * 1024);
1058
1059	limit = min(ppc64_bolted_size(), ppc64_rma_size);
1060
1061	/*
1062	* Align to L1d size, and size it at 2x L1d size, to catch possible
1063	* hardware prefetch runoff. We don't have a recipe for load patterns to
1064	* reliably avoid the prefetcher.
1065	*/
1066	l1d_flush_fallback_area = memblock_alloc_try_nid(l1d_size * 2,
1067	l1d_size, MEMBLOCK_LOW_LIMIT,
1068	limit, NUMA_NO_NODE);
1069	if (!l1d_flush_fallback_area)
1070	panic("%s: Failed to allocate %llu bytes align=0x%llx max_addr=%pa\n",
1071	__func__, l1d_size * 2, l1d_size, &limit);
1072
1073
1074	for_each_possible_cpu(cpu) {
1075	struct paca_struct *paca = paca_ptrs[cpu];
1076	paca->rfi_flush_fallback_area = l1d_flush_fallback_area;
1077	paca->l1d_flush_size = l1d_size;
1078	}
1079	}
1080
1081	void setup_rfi_flush(enum l1d_flush_type types, bool enable)
1082	{
1083	if (types & L1D_FLUSH_FALLBACK) {
1084	pr_info("rfi-flush: fallback displacement flush available\n");
1085	init_fallback_flush();
1086	}
1087
1088	if (types & L1D_FLUSH_ORI)
1089	pr_info("rfi-flush: ori type flush available\n");
1090
1091	if (types & L1D_FLUSH_MTTRIG)
1092	pr_info("rfi-flush: mttrig type flush available\n");
1093
1094	enabled_flush_types = types;
1095
1096	if (!cpu_mitigations_off() && !no_rfi_flush)
1097	rfi_flush_enable(enable);
1098	}
1099
1100	void setup_entry_flush(bool enable)
1101	{
1102	if (cpu_mitigations_off())
1103	return;
1104
1105	if (!no_entry_flush)
1106	entry_flush_enable(enable);
1107	}
1108
1109	void setup_uaccess_flush(bool enable)
1110	{
1111	if (cpu_mitigations_off())
1112	return;
1113
1114	if (!no_uaccess_flush)
1115	uaccess_flush_enable(enable);
1116	}
1117
1118	#ifdef CONFIG_DEBUG_FS
1119	static int rfi_flush_set(void *data, u64 val)
1120	{
1121	bool enable;
1122
1123	if (val == 1)
1124	enable = true;
1125	else if (val == 0)
1126	enable = false;
1127	else
1128	return -EINVAL;
1129
1130	/* Only do anything if we're changing state */
1131	if (enable != rfi_flush)
1132	rfi_flush_enable(enable);
1133
1134	return 0;
1135	}
1136
1137	static int rfi_flush_get(void data, u64 val)
1138	{
1139	*val = rfi_flush ? 1 : 0;
1140	return 0;
1141	}
1142
1143	DEFINE_SIMPLE_ATTRIBUTE(fops_rfi_flush, rfi_flush_get, rfi_flush_set, "%llu\n");
1144
1145	static int entry_flush_set(void *data, u64 val)
1146	{
1147	bool enable;
1148
1149	if (val == 1)
1150	enable = true;
1151	else if (val == 0)
1152	enable = false;
1153	else
1154	return -EINVAL;
1155
1156	/* Only do anything if we're changing state */
1157	if (enable != entry_flush)
1158	entry_flush_enable(enable);
1159
1160	return 0;
1161	}
1162
1163	static int entry_flush_get(void data, u64 val)
1164	{
1165	*val = entry_flush ? 1 : 0;
1166	return 0;
1167	}
1168
1169	DEFINE_SIMPLE_ATTRIBUTE(fops_entry_flush, entry_flush_get, entry_flush_set, "%llu\n");
1170
1171	static int uaccess_flush_set(void *data, u64 val)
1172	{
1173	bool enable;
1174
1175	if (val == 1)
1176	enable = true;
1177	else if (val == 0)
1178	enable = false;
1179	else
1180	return -EINVAL;
1181
1182	/* Only do anything if we're changing state */
1183	if (enable != uaccess_flush)
1184	uaccess_flush_enable(enable);
1185
1186	return 0;
1187	}
1188
1189	static int uaccess_flush_get(void data, u64 val)
1190	{
1191	*val = uaccess_flush ? 1 : 0;
1192	return 0;
1193	}
1194
1195	DEFINE_SIMPLE_ATTRIBUTE(fops_uaccess_flush, uaccess_flush_get, uaccess_flush_set, "%llu\n");
1196
1197	static __init int rfi_flush_debugfs_init(void)
1198	{
1199	debugfs_create_file("rfi_flush", 0600, powerpc_debugfs_root, NULL, &fops_rfi_flush);
1200	debugfs_create_file("entry_flush", 0600, powerpc_debugfs_root, NULL, &fops_entry_flush);
1201	debugfs_create_file("uaccess_flush", 0600, powerpc_debugfs_root, NULL, &fops_uaccess_flush);
1202	return 0;
1203	}
1204	device_initcall(rfi_flush_debugfs_init);
1205	#endif
1206	#endif /* CONFIG_PPC_BOOK3S_64 */

arch/powerpc/kernel/signal.h+55−0

@@ -19,6 +19,15 @@ extern int handle_signal32(struct ksignal ksig, sigset_t oldset, · 18 unmodified lines
19	extern int handle_rt_signal32(struct ksignal ksig, sigset_t oldset,	19	extern int handle_rt_signal32(struct ksignal ksig, sigset_t oldset,
20	struct task_struct *tsk);	20	struct task_struct *tsk);
21		21
		22	static inline int __get_user_sigset(sigset_t dst, const sigset_t __user src)
		23	{
		24	BUILD_BUG_ON(sizeof(sigset_t) != sizeof(u64));
		25
		26	return __get_user(dst->sig[0], (u64 __user *)&src->sig[0]);
		27	}
		28	#define unsafe_get_user_sigset(dst, src, label) \
		29	unsafe_get_user((dst)->sig[0], (u64 __user *)&(src)->sig[0], label)
		30
22	#ifdef CONFIG_VSX	31	#ifdef CONFIG_VSX
23	extern unsigned long copy_vsx_to_user(void __user *to,	32	extern unsigned long copy_vsx_to_user(void __user *to,
24	struct task_struct *task);	33	struct task_struct *task);
@@ -53,6 +62,26 @@ unsigned long copy_ckfpr_from_user(struct task_struct task, void __user from); · 28 unmodified lines
53	&buf[i], label);\	62	&buf[i], label);\
54	} while (0)	63	} while (0)
55		64
		65	#define unsafe_copy_fpr_from_user(task, from, label) do { \
		66	struct task_struct *__t = task; \
		67	u64 __user buf = (u64 __user )from; \
		68	int i; \
		69	\
		70	for (i = 0; i < ELF_NFPREG - 1; i++) \
		71	unsafe_get_user(__t->thread.TS_FPR(i), &buf[i], label); \
		72	unsafe_get_user(__t->thread.fp_state.fpscr, &buf[i], label); \
		73	} while (0)
		74
		75	#define unsafe_copy_vsx_from_user(task, from, label) do { \
		76	struct task_struct *__t = task; \
		77	u64 __user buf = (u64 __user )from; \
		78	int i; \
		79	\
		80	for (i = 0; i < ELF_NVSRHALFREG ; i++) \
		81	unsafe_get_user(__t->thread.fp_state.fpr[i][TS_VSRLOWOFFSET], \
		82	&buf[i], label); \
		83	} while (0)
		84
56	#ifdef CONFIG_PPC_TRANSACTIONAL_MEM	85	#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
57	#define unsafe_copy_ckfpr_to_user(to, task, label) do { \	86	#define unsafe_copy_ckfpr_to_user(to, task, label) do { \
58	struct task_struct *__t = task; \	87	struct task_struct *__t = task; \
@@ -73,6 +102,26 @@ unsigned long copy_ckfpr_from_user(struct task_struct task, void __user from); · 14 unmodified lines
73	unsafe_put_user(__t->thread.ckfp_state.fpr[i][TS_VSRLOWOFFSET], \	102	unsafe_put_user(__t->thread.ckfp_state.fpr[i][TS_VSRLOWOFFSET], \
74	&buf[i], label);\	103	&buf[i], label);\
75	} while (0)	104	} while (0)
		105
		106	#define unsafe_copy_ckfpr_from_user(task, from, label) do { \
		107	struct task_struct *__t = task; \
		108	u64 __user buf = (u64 __user )from; \
		109	int i; \
		110	\
		111	for (i = 0; i < ELF_NFPREG - 1 ; i++) \
		112	unsafe_get_user(__t->thread.TS_CKFPR(i), &buf[i], label);\
		113	unsafe_get_user(__t->thread.ckfp_state.fpscr, &buf[i], failed); \
		114	} while (0)
		115
		116	#define unsafe_copy_ckvsx_from_user(task, from, label) do { \
		117	struct task_struct *__t = task; \
		118	u64 __user buf = (u64 __user )from; \
		119	int i; \
		120	\
		121	for (i = 0; i < ELF_NVSRHALFREG ; i++) \
		122	unsafe_get_user(__t->thread.ckfp_state.fpr[i][TS_VSRLOWOFFSET], \
		123	&buf[i], label); \
		124	} while (0)
76	#endif	125	#endif
77	#elif defined(CONFIG_PPC_FPU_REGS)	126	#elif defined(CONFIG_PPC_FPU_REGS)
78		127
@@ -80,6 +129,10 @@ unsigned long copy_ckfpr_from_user(struct task_struct task, void __user from); · 1 unmodified line
80	unsafe_copy_to_user(to, (task)->thread.fp_state.fpr, \	129	unsafe_copy_to_user(to, (task)->thread.fp_state.fpr, \
81	ELF_NFPREG * sizeof(double), label)	130	ELF_NFPREG * sizeof(double), label)
82		131
		132	#define unsafe_copy_fpr_from_user(task, from, label) \
		133	unsafe_copy_from_user((task)->thread.fp_state.fpr, from, \
		134	ELF_NFPREG * sizeof(double), label)
		135
83	static inline unsigned long	136	static inline unsigned long
84	copy_fpr_to_user(void __user to, struct task_struct task)	137	copy_fpr_to_user(void __user to, struct task_struct task)
85	{	138	{
@@ -115,6 +168,8 @@ copy_ckfpr_from_user(struct task_struct task, void __user from) · 29 unmodified lines
115	#else	168	#else
116	#define unsafe_copy_fpr_to_user(to, task, label) do { } while (0)	169	#define unsafe_copy_fpr_to_user(to, task, label) do { } while (0)
117		170
		171	#define unsafe_copy_fpr_from_user(task, from, label) do { } while (0)
		172
118	static inline unsigned long	173	static inline unsigned long
119	copy_fpr_to_user(void __user to, struct task_struct task)	174	copy_fpr_to_user(void __user to, struct task_struct task)
120	{	175	{

arch/powerpc/kernel/signal_32.c+143−111

@@ -83,24 +83,17 @@ · 82 unmodified lines
83	* implementation that makes things simple for little endian only)	83	* implementation that makes things simple for little endian only)
84	*/	84	*/
85	#define unsafe_put_sigset_t unsafe_put_compat_sigset	85	#define unsafe_put_sigset_t unsafe_put_compat_sigset
86		86	#define unsafe_get_sigset_t unsafe_get_compat_sigset
87	static inline int get_sigset_t(sigset_t *set,
88	const compat_sigset_t __user *uset)
89	{
90	return get_compat_sigset(set, uset);
91	}
92		87
93	#define to_user_ptr(p) ptr_to_compat(p)	88	#define to_user_ptr(p) ptr_to_compat(p)
94	#define from_user_ptr(p) compat_ptr(p)	89	#define from_user_ptr(p) compat_ptr(p)
95		90
96	static __always_inline int	91	static __always_inline int
97	save_general_regs_unsafe(struct pt_regs regs, struct mcontext __user frame)	92	__unsafe_save_general_regs(struct pt_regs regs, struct mcontext __user frame)
98	{	93	{
99	elf_greg_t64 gregs = (elf_greg_t64 )regs;	94	elf_greg_t64 gregs = (elf_greg_t64 )regs;
100	int val, i;	95	int val, i;
101		96
102	WARN_ON(!FULL_REGS(regs));
103
104	for (i = 0; i <= PT_RESULT; i ++) {	97	for (i = 0; i <= PT_RESULT; i ++) {
105	/* Force usr to alway see softe as 1 (interrupts enabled) */	98	/* Force usr to alway see softe as 1 (interrupts enabled) */
106	if (i == PT_SOFTE)	99	if (i == PT_SOFTE)
@@ -116,8 +109,8 @@ failed: · 9 unmodified lines
116	return 1;	109	return 1;
117	}	110	}
118		111
119	static inline int restore_general_regs(struct pt_regs *regs,	112	static __always_inline int
120	struct mcontext __user *sr)	113	__unsafe_restore_general_regs(struct pt_regs regs, struct mcontext __user sr)
121	{	114	{
122	elf_greg_t64 gregs = (elf_greg_t64 )regs;	115	elf_greg_t64 gregs = (elf_greg_t64 )regs;
123	int i;	116	int i;
@@ -125,10 +118,12 @@ static inline int restore_general_regs(struct pt_regs *regs, · 1 unmodified line
125	for (i = 0; i <= PT_RESULT; i++) {	118	for (i = 0; i <= PT_RESULT; i++) {
126	if ((i == PT_MSR) \|\| (i == PT_SOFTE))	119	if ((i == PT_MSR) \|\| (i == PT_SOFTE))
127	continue;	120	continue;
128	if (__get_user(gregs[i], &sr->mc_gregs[i]))	121	unsafe_get_user(gregs[i], &sr->mc_gregs[i], failed);
129	return -EFAULT;
130	}	122	}
131	return 0;	123	return 0;
		124
		125	failed:
		126	return 1;
132	}	127	}
133		128
134	#else /* CONFIG_PPC64 */	129	#else /* CONFIG_PPC64 */
@@ -142,18 +137,14 @@ static inline int restore_general_regs(struct pt_regs *regs, · 7 unmodified lines
142	unsafe_copy_to_user(__us, __s, sizeof(*__us), label); \	137	unsafe_copy_to_user(__us, __s, sizeof(*__us), label); \
143	} while (0)	138	} while (0)
144		139
145	static inline int get_sigset_t(sigset_t set, const sigset_t __user uset)	140	#define unsafe_get_sigset_t unsafe_get_user_sigset
146	{
147	return copy_from_user(set, uset, sizeof(*uset));
148	}
149		141
150	#define to_user_ptr(p) ((unsigned long)(p))	142	#define to_user_ptr(p) ((unsigned long)(p))
151	#define from_user_ptr(p) ((void __user *)(p))	143	#define from_user_ptr(p) ((void __user *)(p))
152		144
153	static __always_inline int	145	static __always_inline int
154	save_general_regs_unsafe(struct pt_regs regs, struct mcontext __user frame)	146	__unsafe_save_general_regs(struct pt_regs regs, struct mcontext __user frame)
155	{	147	{
156	WARN_ON(!FULL_REGS(regs));
157	unsafe_copy_to_user(&frame->mc_gregs, regs, GP_REGS_SIZE, failed);	148	unsafe_copy_to_user(&frame->mc_gregs, regs, GP_REGS_SIZE, failed);
158	return 0;	149	return 0;
159		150
@@ -161,23 +152,30 @@ failed: · 1 unmodified line
161	return 1;	152	return 1;
162	}	153	}
163		154
164	static inline int restore_general_regs(struct pt_regs *regs,	155	static __always_inline
165	struct mcontext __user *sr)	156	int __unsafe_restore_general_regs(struct pt_regs regs, struct mcontext __user sr)
166	{	157	{
167	/* copy up to but not including MSR */	158	/* copy up to but not including MSR */
168	if (__copy_from_user(regs, &sr->mc_gregs,	159	unsafe_copy_from_user(regs, &sr->mc_gregs, PT_MSR * sizeof(elf_greg_t), failed);
169	PT_MSR * sizeof(elf_greg_t)))	160
170	return -EFAULT;
171	/* copy from orig_r3 (the word after the MSR) up to the end */	161	/* copy from orig_r3 (the word after the MSR) up to the end */
172	if (__copy_from_user(&regs->orig_gpr3, &sr->mc_gregs[PT_ORIG_R3],	162	unsafe_copy_from_user(&regs->orig_gpr3, &sr->mc_gregs[PT_ORIG_R3],
173	GP_REGS_SIZE - PT_ORIG_R3 * sizeof(elf_greg_t)))	163	GP_REGS_SIZE - PT_ORIG_R3 * sizeof(elf_greg_t), failed);
174	return -EFAULT;	164
175	return 0;	165	return 0;
		166
		167	failed:
		168	return 1;
176	}	169	}
177	#endif	170	#endif
178		171
179	#define unsafe_save_general_regs(regs, frame, label) do { \	172	#define unsafe_save_general_regs(regs, frame, label) do { \
180	if (save_general_regs_unsafe(regs, frame)) \	173	if (__unsafe_save_general_regs(regs, frame)) \
		174	goto label; \
		175	} while (0)
		176
		177	#define unsafe_restore_general_regs(regs, frame, label) do { \
		178	if (__unsafe_restore_general_regs(regs, frame)) \
181	goto label; \	179	goto label; \
182	} while (0)	180	} while (0)
183		181
@@ -260,8 +258,8 @@ static void prepare_save_user_regs(int ctx_has_vsx_region) · 76 unmodified lines
260	#endif	258	#endif
261	}	259	}
262		260
263	static int save_user_regs_unsafe(struct pt_regs regs, struct mcontext __user frame,	261	static int __unsafe_save_user_regs(struct pt_regs regs, struct mcontext __user frame,
264	struct mcontext __user *tm_frame, int ctx_has_vsx_region)	262	struct mcontext __user *tm_frame, int ctx_has_vsx_region)
265	{	263	{
266	unsigned long msr = regs->msr;	264	unsigned long msr = regs->msr;
267		265
@@ -338,7 +336,7 @@ failed: · 70 unmodified lines
338	}	336	}
339		337
340	#define unsafe_save_user_regs(regs, frame, tm_frame, has_vsx, label) do { \	338	#define unsafe_save_user_regs(regs, frame, tm_frame, has_vsx, label) do { \
341	if (save_user_regs_unsafe(regs, frame, tm_frame, has_vsx)) \	339	if (__unsafe_save_user_regs(regs, frame, tm_frame, has_vsx)) \
342	goto label; \	340	goto label; \
343	} while (0)	341	} while (0)
344		342
@@ -350,7 +348,7 @@ failed: · 5 unmodified lines
350	* We also save the transactional registers to a second ucontext in the	348	* We also save the transactional registers to a second ucontext in the
351	* frame.	349	* frame.
352	*	350	*
353	* See save_user_regs_unsafe() and signal_64.c:setup_tm_sigcontexts().	351	* See __unsafe_save_user_regs() and signal_64.c:setup_tm_sigcontexts().
354	*/	352	*/
355	static void prepare_save_tm_user_regs(void)	353	static void prepare_save_tm_user_regs(void)
356	{	354	{
@@ -441,7 +439,7 @@ static int save_tm_user_regs_unsafe(struct pt_regs *regs, struct mcontext __user · 84 unmodified lines
441	#endif /* CONFIG_VSX */	439	#endif /* CONFIG_VSX */
442	#ifdef CONFIG_SPE	440	#ifdef CONFIG_SPE
443	/* SPE regs are not checkpointed with TM, so this section is	441	/* SPE regs are not checkpointed with TM, so this section is
444	* simply the same as in save_user_regs_unsafe().	442	* simply the same as in __unsafe_save_user_regs().
445	*/	443	*/
446	if (current->thread.used_spe) {	444	if (current->thread.used_spe) {
447	unsafe_copy_to_user(&frame->mc_vregs, current->thread.evr,	445	unsafe_copy_to_user(&frame->mc_vregs, current->thread.evr,
@@ -485,26 +483,25 @@ static int save_tm_user_regs_unsafe(struct pt_regs *regs, struct mcontext __user · 37 unmodified lines
485	static long restore_user_regs(struct pt_regs *regs,	483	static long restore_user_regs(struct pt_regs *regs,
486	struct mcontext __user *sr, int sig)	484	struct mcontext __user *sr, int sig)
487	{	485	{
488	long err;
489	unsigned int save_r2 = 0;	486	unsigned int save_r2 = 0;
490	unsigned long msr;	487	unsigned long msr;
491	#ifdef CONFIG_VSX	488	#ifdef CONFIG_VSX
492	int i;	489	int i;
493	#endif	490	#endif
494		491
		492	if (!user_read_access_begin(sr, sizeof(*sr)))
		493	return 1;
495	/*	494	/*
496	* restore general registers but not including MSR or SOFTE. Also	495	* restore general registers but not including MSR or SOFTE. Also
497	* take care of keeping r2 (TLS) intact if not a signal	496	* take care of keeping r2 (TLS) intact if not a signal
498	*/	497	*/
499	if (!sig)	498	if (!sig)
500	save_r2 = (unsigned int)regs->gpr[2];	499	save_r2 = (unsigned int)regs->gpr[2];
501	err = restore_general_regs(regs, sr);	500	unsafe_restore_general_regs(regs, sr, failed);
502	set_trap_norestart(regs);	501	set_trap_norestart(regs);
503	err \|= __get_user(msr, &sr->mc_gregs[PT_MSR]);	502	unsafe_get_user(msr, &sr->mc_gregs[PT_MSR], failed);
504	if (!sig)	503	if (!sig)
505	regs->gpr[2] = (unsigned long) save_r2;	504	regs->gpr[2] = (unsigned long) save_r2;
506	if (err)
507	return 1;
508		505
509	/* if doing signal return, restore the previous little-endian mode */	506	/* if doing signal return, restore the previous little-endian mode */
510	if (sig)	507	if (sig)
@@ -518,22 +515,19 @@ static long restore_user_regs(struct pt_regs *regs, · 7 unmodified lines
518	regs->msr &= ~MSR_VEC;	515	regs->msr &= ~MSR_VEC;
519	if (msr & MSR_VEC) {	516	if (msr & MSR_VEC) {
520	/* restore altivec registers from the stack */	517	/* restore altivec registers from the stack */
521	if (__copy_from_user(&current->thread.vr_state, &sr->mc_vregs,	518	unsafe_copy_from_user(&current->thread.vr_state, &sr->mc_vregs,
522	sizeof(sr->mc_vregs)))	519	sizeof(sr->mc_vregs), failed);
523	return 1;
524	current->thread.used_vr = true;	520	current->thread.used_vr = true;
525	} else if (current->thread.used_vr)	521	} else if (current->thread.used_vr)
526	memset(&current->thread.vr_state, 0,	522	memset(&current->thread.vr_state, 0,
527	ELF_NVRREG * sizeof(vector128));	523	ELF_NVRREG * sizeof(vector128));
528		524
529	/* Always get VRSAVE back */	525	/* Always get VRSAVE back */
530	if (__get_user(current->thread.vrsave, (u32 __user *)&sr->mc_vregs[32]))	526	unsafe_get_user(current->thread.vrsave, (u32 __user *)&sr->mc_vregs[32], failed);
531	return 1;
532	if (cpu_has_feature(CPU_FTR_ALTIVEC))	527	if (cpu_has_feature(CPU_FTR_ALTIVEC))
533	mtspr(SPRN_VRSAVE, current->thread.vrsave);	528	mtspr(SPRN_VRSAVE, current->thread.vrsave);
534	#endif /* CONFIG_ALTIVEC */	529	#endif /* CONFIG_ALTIVEC */
535	if (copy_fpr_from_user(current, &sr->mc_fregs))	530	unsafe_copy_fpr_from_user(current, &sr->mc_fregs, failed);
536	return 1;
537		531
538	#ifdef CONFIG_VSX	532	#ifdef CONFIG_VSX
539	/*	533	/*
@@ -546,8 +540,7 @@ static long restore_user_regs(struct pt_regs *regs, · 6 unmodified lines
546	* Restore altivec registers from the stack to a local	540	* Restore altivec registers from the stack to a local
547	* buffer, then write this out to the thread_struct	541	* buffer, then write this out to the thread_struct
548	*/	542	*/
549	if (copy_vsx_from_user(current, &sr->mc_vsregs))	543	unsafe_copy_vsx_from_user(current, &sr->mc_vsregs, failed);
550	return 1;
551	current->thread.used_vsr = true;	544	current->thread.used_vsr = true;
552	} else if (current->thread.used_vsr)	545	} else if (current->thread.used_vsr)
553	for (i = 0; i < 32 ; i++)	546	for (i = 0; i < 32 ; i++)
@@ -565,19 +558,22 @@ static long restore_user_regs(struct pt_regs *regs, · 11 unmodified lines
565	regs->msr &= ~MSR_SPE;	558	regs->msr &= ~MSR_SPE;
566	if (msr & MSR_SPE) {	559	if (msr & MSR_SPE) {
567	/* restore spe registers from the stack */	560	/* restore spe registers from the stack */
568	if (__copy_from_user(current->thread.evr, &sr->mc_vregs,	561	unsafe_copy_from_user(current->thread.evr, &sr->mc_vregs,
569	ELF_NEVRREG * sizeof(u32)))	562	ELF_NEVRREG * sizeof(u32), failed);
570	return 1;
571	current->thread.used_spe = true;	563	current->thread.used_spe = true;
572	} else if (current->thread.used_spe)	564	} else if (current->thread.used_spe)
573	memset(current->thread.evr, 0, ELF_NEVRREG * sizeof(u32));	565	memset(current->thread.evr, 0, ELF_NEVRREG * sizeof(u32));
574		566
575	/* Always get SPEFSCR back */	567	/* Always get SPEFSCR back */
576	if (__get_user(current->thread.spefscr, (u32 __user *)&sr->mc_vregs + ELF_NEVRREG))	568	unsafe_get_user(current->thread.spefscr, (u32 __user *)&sr->mc_vregs + ELF_NEVRREG, failed);
577	return 1;
578	#endif /* CONFIG_SPE */	569	#endif /* CONFIG_SPE */
579		570
		571	user_read_access_end();
580	return 0;	572	return 0;
		573
		574	failed:
		575	user_read_access_end();
		576	return 1;
581	}	577	}
582		578
583	#ifdef CONFIG_PPC_TRANSACTIONAL_MEM	579	#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
@@ -590,7 +586,6 @@ static long restore_tm_user_regs(struct pt_regs *regs, · 6 unmodified lines
590	struct mcontext __user *sr,	586	struct mcontext __user *sr,
591	struct mcontext __user *tm_sr)	587	struct mcontext __user *tm_sr)
592	{	588	{
593	long err;
594	unsigned long msr, msr_hi;	589	unsigned long msr, msr_hi;
595	#ifdef CONFIG_VSX	590	#ifdef CONFIG_VSX
596	int i;	591	int i;
@@ -605,15 +600,13 @@ static long restore_tm_user_regs(struct pt_regs *regs, · 8 unmodified lines
605	* TFHAR is restored from the checkpointed NIP; TEXASR and TFIAR	600	* TFHAR is restored from the checkpointed NIP; TEXASR and TFIAR
606	* were set by the signal delivery.	601	* were set by the signal delivery.
607	*/	602	*/
608	err = restore_general_regs(regs, tm_sr);	603	if (!user_read_access_begin(sr, sizeof(*sr)))
609	err \|= restore_general_regs(&current->thread.ckpt_regs, sr);
610
611	err \|= __get_user(current->thread.tm_tfhar, &sr->mc_gregs[PT_NIP]);
612
613	err \|= __get_user(msr, &sr->mc_gregs[PT_MSR]);
614	if (err)
615	return 1;	604	return 1;
616		605
		606	unsafe_restore_general_regs(&current->thread.ckpt_regs, sr, failed);
		607	unsafe_get_user(current->thread.tm_tfhar, &sr->mc_gregs[PT_NIP], failed);
		608	unsafe_get_user(msr, &sr->mc_gregs[PT_MSR], failed);
		609
617	/* Restore the previous little-endian mode */	610	/* Restore the previous little-endian mode */
618	regs->msr = (regs->msr & ~MSR_LE) \| (msr & MSR_LE);	611	regs->msr = (regs->msr & ~MSR_LE) \| (msr & MSR_LE);
619		612
@@ -621,12 +614,8 @@ static long restore_tm_user_regs(struct pt_regs *regs, · 1 unmodified line
621	regs->msr &= ~MSR_VEC;	614	regs->msr &= ~MSR_VEC;
622	if (msr & MSR_VEC) {	615	if (msr & MSR_VEC) {
623	/* restore altivec registers from the stack */	616	/* restore altivec registers from the stack */
624	if (__copy_from_user(&current->thread.ckvr_state, &sr->mc_vregs,	617	unsafe_copy_from_user(&current->thread.ckvr_state, &sr->mc_vregs,
625	sizeof(sr->mc_vregs)) \|\|	618	sizeof(sr->mc_vregs), failed);
626	__copy_from_user(&current->thread.vr_state,
627	&tm_sr->mc_vregs,
628	sizeof(sr->mc_vregs)))
629	return 1;
630	current->thread.used_vr = true;	619	current->thread.used_vr = true;
631	} else if (current->thread.used_vr) {	620	} else if (current->thread.used_vr) {
632	memset(&current->thread.vr_state, 0,	621	memset(&current->thread.vr_state, 0,
@@ -636,20 +625,15 @@ static long restore_tm_user_regs(struct pt_regs *regs, · 3 unmodified lines
636	}	625	}
637		626
638	/* Always get VRSAVE back */	627	/* Always get VRSAVE back */
639	if (__get_user(current->thread.ckvrsave,	628	unsafe_get_user(current->thread.ckvrsave,
640	(u32 __user *)&sr->mc_vregs[32]) \|\|	629	(u32 __user *)&sr->mc_vregs[32], failed);
641	__get_user(current->thread.vrsave,
642	(u32 __user *)&tm_sr->mc_vregs[32]))
643	return 1;
644	if (cpu_has_feature(CPU_FTR_ALTIVEC))	630	if (cpu_has_feature(CPU_FTR_ALTIVEC))
645	mtspr(SPRN_VRSAVE, current->thread.ckvrsave);	631	mtspr(SPRN_VRSAVE, current->thread.ckvrsave);
646	#endif /* CONFIG_ALTIVEC */	632	#endif /* CONFIG_ALTIVEC */
647		633
648	regs->msr &= ~(MSR_FP \| MSR_FE0 \| MSR_FE1);	634	regs->msr &= ~(MSR_FP \| MSR_FE0 \| MSR_FE1);
649		635
650	if (copy_fpr_from_user(current, &sr->mc_fregs) \|\|	636	unsafe_copy_fpr_from_user(current, &sr->mc_fregs, failed);
651	copy_ckfpr_from_user(current, &tm_sr->mc_fregs))
652	return 1;
653		637
654	#ifdef CONFIG_VSX	638	#ifdef CONFIG_VSX
655	regs->msr &= ~MSR_VSX;	639	regs->msr &= ~MSR_VSX;
@@ -658,9 +642,7 @@ static long restore_tm_user_regs(struct pt_regs *regs, · 2 unmodified lines
658	* Restore altivec registers from the stack to a local	642	* Restore altivec registers from the stack to a local
659	* buffer, then write this out to the thread_struct	643	* buffer, then write this out to the thread_struct
660	*/	644	*/
661	if (copy_vsx_from_user(current, &tm_sr->mc_vsregs) \|\|	645	unsafe_copy_ckvsx_from_user(current, &sr->mc_vsregs, failed);
662	copy_ckvsx_from_user(current, &sr->mc_vsregs))
663	return 1;
664	current->thread.used_vsr = true;	646	current->thread.used_vsr = true;
665	} else if (current->thread.used_vsr)	647	} else if (current->thread.used_vsr)
666	for (i = 0; i < 32 ; i++) {	648	for (i = 0; i < 32 ; i++) {
@@ -675,23 +657,54 @@ static long restore_tm_user_regs(struct pt_regs *regs, · 8 unmodified lines
675	*/	657	*/
676	regs->msr &= ~MSR_SPE;	658	regs->msr &= ~MSR_SPE;
677	if (msr & MSR_SPE) {	659	if (msr & MSR_SPE) {
678	if (__copy_from_user(current->thread.evr, &sr->mc_vregs,	660	unsafe_copy_from_user(current->thread.evr, &sr->mc_vregs,
679	ELF_NEVRREG * sizeof(u32)))	661	ELF_NEVRREG * sizeof(u32), failed);
680	return 1;
681	current->thread.used_spe = true;	662	current->thread.used_spe = true;
682	} else if (current->thread.used_spe)	663	} else if (current->thread.used_spe)
683	memset(current->thread.evr, 0, ELF_NEVRREG * sizeof(u32));	664	memset(current->thread.evr, 0, ELF_NEVRREG * sizeof(u32));
684		665
685	/* Always get SPEFSCR back */	666	/* Always get SPEFSCR back */
686	if (__get_user(current->thread.spefscr, (u32 __user *)&sr->mc_vregs	667	unsafe_get_user(current->thread.spefscr,
687	+ ELF_NEVRREG))	668	(u32 __user *)&sr->mc_vregs + ELF_NEVRREG, failed);
688	return 1;
689	#endif /* CONFIG_SPE */	669	#endif /* CONFIG_SPE */
690		670
691	/* Get the top half of the MSR from the user context */	671	user_read_access_end();
692	if (__get_user(msr_hi, &tm_sr->mc_gregs[PT_MSR]))	672
		673	if (!user_read_access_begin(tm_sr, sizeof(*tm_sr)))
693	return 1;	674	return 1;
		675
		676	unsafe_restore_general_regs(regs, tm_sr, failed);
		677
		678	#ifdef CONFIG_ALTIVEC
		679	/* restore altivec registers from the stack */
		680	if (msr & MSR_VEC)
		681	unsafe_copy_from_user(&current->thread.vr_state, &tm_sr->mc_vregs,
		682	sizeof(sr->mc_vregs), failed);
		683
		684	/* Always get VRSAVE back */
		685	unsafe_get_user(current->thread.vrsave,
		686	(u32 __user *)&tm_sr->mc_vregs[32], failed);
		687	#endif /* CONFIG_ALTIVEC */
		688
		689	unsafe_copy_ckfpr_from_user(current, &tm_sr->mc_fregs, failed);
		690
		691	#ifdef CONFIG_VSX
		692	if (msr & MSR_VSX) {
		693	/*
		694	* Restore altivec registers from the stack to a local
		695	* buffer, then write this out to the thread_struct
		696	*/
		697	unsafe_copy_vsx_from_user(current, &tm_sr->mc_vsregs, failed);
		698	current->thread.used_vsr = true;
		699	}
		700	#endif /* CONFIG_VSX */
		701
		702	/* Get the top half of the MSR from the user context */
		703	unsafe_get_user(msr_hi, &tm_sr->mc_gregs[PT_MSR], failed);
694	msr_hi <<= 32;	704	msr_hi <<= 32;
		705
		706	user_read_access_end();
		707
695	/* If TM bits are set to the reserved value, it's an invalid context */	708	/* If TM bits are set to the reserved value, it's an invalid context */
696	if (MSR_TM_RESV(msr_hi))	709	if (MSR_TM_RESV(msr_hi))
697	return 1;	710	return 1;
@@ -739,6 +752,16 @@ static long restore_tm_user_regs(struct pt_regs *regs, · 41 unmodified lines
739	preempt_enable();	752	preempt_enable();
740		753
741	return 0;	754	return 0;
		755
		756	failed:
		757	user_read_access_end();
		758	return 1;
		759	}
		760	#else
		761	static long restore_tm_user_regs(struct pt_regs regs, struct mcontext __user sr,
		762	struct mcontext __user *tm_sr)
		763	{
		764	return 0;
742	}	765	}
743	#endif	766	#endif
744		767
@@ -944,28 +967,31 @@ static int do_setcontext(struct ucontext __user ucp, struct pt_regs regs, int · 199 unmodified lines
944	sigset_t set;	967	sigset_t set;
945	struct mcontext __user *mcp;	968	struct mcontext __user *mcp;
946		969
947	if (get_sigset_t(&set, &ucp->uc_sigmask))	970	if (!user_read_access_begin(ucp, sizeof(*ucp)))
948	return -EFAULT;	971	return -EFAULT;
		972
		973	unsafe_get_sigset_t(&set, &ucp->uc_sigmask, failed);
949	#ifdef CONFIG_PPC64	974	#ifdef CONFIG_PPC64
950	{	975	{
951	u32 cmcp;	976	u32 cmcp;
952		977
953	if (__get_user(cmcp, &ucp->uc_regs))	978	unsafe_get_user(cmcp, &ucp->uc_regs, failed);
954	return -EFAULT;
955	mcp = (struct mcontext __user *)(u64)cmcp;	979	mcp = (struct mcontext __user *)(u64)cmcp;
956	/* no need to check access_ok(mcp), since mcp < 4GB */
957	}	980	}
958	#else	981	#else
959	if (__get_user(mcp, &ucp->uc_regs))	982	unsafe_get_user(mcp, &ucp->uc_regs, failed);
960	return -EFAULT;
961	if (!access_ok(mcp, sizeof(*mcp)))
962	return -EFAULT;
963	#endif	983	#endif
		984	user_read_access_end();
		985
964	set_current_blocked(&set);	986	set_current_blocked(&set);
965	if (restore_user_regs(regs, mcp, sig))	987	if (restore_user_regs(regs, mcp, sig))
966	return -EFAULT;	988	return -EFAULT;
967		989
968	return 0;	990	return 0;
		991
		992	failed:
		993	user_read_access_end();
		994	return -EFAULT;
969	}	995	}
970		996
971	#ifdef CONFIG_PPC_TRANSACTIONAL_MEM	997	#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
@@ -979,11 +1005,15 @@ static int do_setcontext_tm(struct ucontext __user *ucp, · 7 unmodified lines
979	u32 cmcp;	1005	u32 cmcp;
980	u32 tm_cmcp;	1006	u32 tm_cmcp;
981		1007
982	if (get_sigset_t(&set, &ucp->uc_sigmask))	1008	if (!user_read_access_begin(ucp, sizeof(*ucp)))
983	return -EFAULT;	1009	return -EFAULT;
984		1010
985	if (__get_user(cmcp, &ucp->uc_regs) \|\|	1011	unsafe_get_sigset_t(&set, &ucp->uc_sigmask, failed);
986	__get_user(tm_cmcp, &tm_ucp->uc_regs))	1012	unsafe_get_user(cmcp, &ucp->uc_regs, failed);
		1013
		1014	user_read_access_end();
		1015
		1016	if (__get_user(tm_cmcp, &tm_ucp->uc_regs))
987	return -EFAULT;	1017	return -EFAULT;
988	mcp = (struct mcontext __user *)(u64)cmcp;	1018	mcp = (struct mcontext __user *)(u64)cmcp;
989	tm_mcp = (struct mcontext __user *)(u64)tm_cmcp;	1019	tm_mcp = (struct mcontext __user *)(u64)tm_cmcp;
@@ -994,6 +1024,10 @@ static int do_setcontext_tm(struct ucontext __user *ucp, · 4 unmodified lines
994	return -EFAULT;	1024	return -EFAULT;
995		1025
996	return 0;	1026	return 0;
		1027
		1028	failed:
		1029	user_read_access_end();
		1030	return -EFAULT;
997	}	1031	}
998	#endif	1032	#endif
999		1033
@@ -1311,19 +1345,16 @@ SYSCALL_DEFINE0(sigreturn) · 311 unmodified lines
1311	struct sigcontext __user *sc;	1345	struct sigcontext __user *sc;
1312	struct sigcontext sigctx;	1346	struct sigcontext sigctx;
1313	struct mcontext __user *sr;	1347	struct mcontext __user *sr;
1314	void __user *addr;
1315	sigset_t set;	1348	sigset_t set;
1316	#ifdef CONFIG_PPC_TRANSACTIONAL_MEM	1349	struct mcontext __user *mcp;
1317	struct mcontext __user mcp, tm_mcp;	1350	struct mcontext __user *tm_mcp = NULL;
1318	unsigned long msr_hi;	1351	unsigned long long msr_hi = 0;
1319	#endif
1320		1352
1321	/* Always make any pending restarted system calls return -EINTR */	1353	/* Always make any pending restarted system calls return -EINTR */
1322	current->restart_block.fn = do_no_restart_syscall;	1354	current->restart_block.fn = do_no_restart_syscall;
1323		1355
1324	sf = (struct sigframe __user *)(regs->gpr[1] + __SIGNAL_FRAMESIZE);	1356	sf = (struct sigframe __user *)(regs->gpr[1] + __SIGNAL_FRAMESIZE);
1325	sc = &sf->sctx;	1357	sc = &sf->sctx;
1326	addr = sc;
1327	if (copy_from_user(&sigctx, sc, sizeof(sigctx)))	1358	if (copy_from_user(&sigctx, sc, sizeof(sigctx)))
1328	goto badframe;	1359	goto badframe;
1329		1360
@@ -1339,31 +1370,32 @@ SYSCALL_DEFINE0(sigreturn) · 9 unmodified lines
1339	#endif	1370	#endif
1340	set_current_blocked(&set);	1371	set_current_blocked(&set);
1341		1372
1342	#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
1343	mcp = (struct mcontext __user *)&sf->mctx;	1373	mcp = (struct mcontext __user *)&sf->mctx;
		1374	#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
1344	tm_mcp = (struct mcontext __user *)&sf->mctx_transact;	1375	tm_mcp = (struct mcontext __user *)&sf->mctx_transact;
1345	if (__get_user(msr_hi, &tm_mcp->mc_gregs[PT_MSR]))	1376	if (__get_user(msr_hi, &tm_mcp->mc_gregs[PT_MSR]))
1346	goto badframe;	1377	goto badframe;
		1378	#endif
1347	if (MSR_TM_ACTIVE(msr_hi<<32)) {	1379	if (MSR_TM_ACTIVE(msr_hi<<32)) {
1348	if (!cpu_has_feature(CPU_FTR_TM))	1380	if (!cpu_has_feature(CPU_FTR_TM))
1349	goto badframe;	1381	goto badframe;
1350	if (restore_tm_user_regs(regs, mcp, tm_mcp))	1382	if (restore_tm_user_regs(regs, mcp, tm_mcp))
1351	goto badframe;	1383	goto badframe;
1352	} else	1384	} else {
1353	#endif
1354	{
1355	sr = (struct mcontext __user *)from_user_ptr(sigctx.regs);	1385	sr = (struct mcontext __user *)from_user_ptr(sigctx.regs);
1356	addr = sr;	1386	if (restore_user_regs(regs, sr, 1)) {
1357	if (!access_ok(sr, sizeof(*sr))	1387	signal_fault(current, regs, "sys_sigreturn", sr);
1358	\|\| restore_user_regs(regs, sr, 1))	1388
1359	goto badframe;	1389	force_sig(SIGSEGV);
		1390	return 0;
		1391	}
1360	}	1392	}
1361		1393
1362	set_thread_flag(TIF_RESTOREALL);	1394	set_thread_flag(TIF_RESTOREALL);
1363	return 0;	1395	return 0;
1364		1396
1365	badframe:	1397	badframe:
1366	signal_fault(current, regs, "sys_sigreturn", addr);	1398	signal_fault(current, regs, "sys_sigreturn", sc);
1367		1399
1368	force_sig(SIGSEGV);	1400	force_sig(SIGSEGV);
1369	return 0;	1401	return 0;

arch/powerpc/kernel/signal_64.c+189−130

@@ -79,13 +79,36 @@ static elf_vrreg_t __user sigcontext_vmx_regs(struct sigcontext __user sc) · 78 unmodified lines
79	}	79	}
80	#endif	80	#endif
81		81
		82	static void prepare_setup_sigcontext(struct task_struct *tsk)
		83	{
		84	#ifdef CONFIG_ALTIVEC
		85	/* save altivec registers */
		86	if (tsk->thread.used_vr)
		87	flush_altivec_to_thread(tsk);
		88	if (cpu_has_feature(CPU_FTR_ALTIVEC))
		89	tsk->thread.vrsave = mfspr(SPRN_VRSAVE);
		90	#endif /* CONFIG_ALTIVEC */
		91
		92	flush_fp_to_thread(tsk);
		93
		94	#ifdef CONFIG_VSX
		95	if (tsk->thread.used_vsr)
		96	flush_vsx_to_thread(tsk);
		97	#endif /* CONFIG_VSX */
		98	}
		99
82	/*	100	/*
83	* Set up the sigcontext for the signal frame.	101	* Set up the sigcontext for the signal frame.
84	*/	102	*/
85		103
86	static long setup_sigcontext(struct sigcontext __user *sc,	104	#define unsafe_setup_sigcontext(sc, tsk, signr, set, handler, ctx_has_vsx_region, label)\
87	struct task_struct tsk, int signr, sigset_t set,	105	do { \
88	unsigned long handler, int ctx_has_vsx_region)	106	if (__unsafe_setup_sigcontext(sc, tsk, signr, set, handler, ctx_has_vsx_region))\
		107	goto label; \
		108	} while (0)
		109	static long notrace __unsafe_setup_sigcontext(struct sigcontext __user *sc,
		110	struct task_struct tsk, int signr, sigset_t set,
		111	unsigned long handler, int ctx_has_vsx_region)
89	{	112	{
90	/* When CONFIG_ALTIVEC is set, we _always_ setup v_regs even if the	113	/* When CONFIG_ALTIVEC is set, we _always_ setup v_regs even if the
91	* process never used altivec yet (MSR_VEC is zero in pt_regs of	114	* process never used altivec yet (MSR_VEC is zero in pt_regs of
@@ -97,25 +120,22 @@ static long setup_sigcontext(struct sigcontext __user *sc, · 5 unmodified lines
97	*/	120	*/
98	#ifdef CONFIG_ALTIVEC	121	#ifdef CONFIG_ALTIVEC
99	elf_vrreg_t __user *v_regs = sigcontext_vmx_regs(sc);	122	elf_vrreg_t __user *v_regs = sigcontext_vmx_regs(sc);
100	unsigned long vrsave;
101	#endif	123	#endif
102	struct pt_regs *regs = tsk->thread.regs;	124	struct pt_regs *regs = tsk->thread.regs;
103	unsigned long msr = regs->msr;	125	unsigned long msr = regs->msr;
104	long err = 0;
105	/* Force usr to alway see softe as 1 (interrupts enabled) */	126	/* Force usr to alway see softe as 1 (interrupts enabled) */
106	unsigned long softe = 0x1;	127	unsigned long softe = 0x1;
107		128
108	BUG_ON(tsk != current);	129	BUG_ON(tsk != current);
109		130
110	#ifdef CONFIG_ALTIVEC	131	#ifdef CONFIG_ALTIVEC
111	err \|= __put_user(v_regs, &sc->v_regs);	132	unsafe_put_user(v_regs, &sc->v_regs, efault_out);
112		133
113	/* save altivec registers */	134	/* save altivec registers */
114	if (tsk->thread.used_vr) {	135	if (tsk->thread.used_vr) {
115	flush_altivec_to_thread(tsk);
116	/* Copy 33 vec registers (vr0..31 and vscr) to the stack */	136	/* Copy 33 vec registers (vr0..31 and vscr) to the stack */
117	err \|= __copy_to_user(v_regs, &tsk->thread.vr_state,	137	unsafe_copy_to_user(v_regs, &tsk->thread.vr_state,
118	33 * sizeof(vector128));	138	33 * sizeof(vector128), efault_out);
119	/* set MSR_VEC in the MSR value in the frame to indicate that sc->v_reg)	139	/* set MSR_VEC in the MSR value in the frame to indicate that sc->v_reg)
120	* contains valid data.	140	* contains valid data.
121	*/	141	*/
@@ -124,19 +144,12 @@ static long setup_sigcontext(struct sigcontext __user *sc, · 2 unmodified lines
124	/* We always copy to/from vrsave, it's 0 if we don't have or don't	144	/* We always copy to/from vrsave, it's 0 if we don't have or don't
125	* use altivec.	145	* use altivec.
126	*/	146	*/
127	vrsave = 0;	147	unsafe_put_user(tsk->thread.vrsave, (u32 __user *)&v_regs[33], efault_out);
128	if (cpu_has_feature(CPU_FTR_ALTIVEC)) {
129	vrsave = mfspr(SPRN_VRSAVE);
130	tsk->thread.vrsave = vrsave;
131	}
132
133	err \|= __put_user(vrsave, (u32 __user *)&v_regs[33]);
134	#else /* CONFIG_ALTIVEC */	148	#else /* CONFIG_ALTIVEC */
135	err \|= __put_user(0, &sc->v_regs);	149	unsafe_put_user(0, &sc->v_regs, efault_out);
136	#endif /* CONFIG_ALTIVEC */	150	#endif /* CONFIG_ALTIVEC */
137	flush_fp_to_thread(tsk);
138	/* copy fpr regs and fpscr */	151	/* copy fpr regs and fpscr */
139	err \|= copy_fpr_to_user(&sc->fp_regs, tsk);	152	unsafe_copy_fpr_to_user(&sc->fp_regs, tsk, efault_out);
140		153
141	/*	154	/*
142	* Clear the MSR VSX bit to indicate there is no valid state attached	155	* Clear the MSR VSX bit to indicate there is no valid state attached
@@ -150,26 +163,27 @@ static long setup_sigcontext(struct sigcontext __user *sc, · 7 unmodified lines
150	* VMX data.	163	* VMX data.
151	*/	164	*/
152	if (tsk->thread.used_vsr && ctx_has_vsx_region) {	165	if (tsk->thread.used_vsr && ctx_has_vsx_region) {
153	flush_vsx_to_thread(tsk);
154	v_regs += ELF_NVRREG;	166	v_regs += ELF_NVRREG;
155	err \|= copy_vsx_to_user(v_regs, tsk);	167	unsafe_copy_vsx_to_user(v_regs, tsk, efault_out);
156	/* set MSR_VSX in the MSR value in the frame to	168	/* set MSR_VSX in the MSR value in the frame to
157	* indicate that sc->vs_reg) contains valid data.	169	* indicate that sc->vs_reg) contains valid data.
158	*/	170	*/
159	msr \|= MSR_VSX;	171	msr \|= MSR_VSX;
160	}	172	}
161	#endif /* CONFIG_VSX */	173	#endif /* CONFIG_VSX */
162	err \|= __put_user(&sc->gp_regs, &sc->regs);	174	unsafe_put_user(&sc->gp_regs, &sc->regs, efault_out);
163	WARN_ON(!FULL_REGS(regs));	175	unsafe_copy_to_user(&sc->gp_regs, regs, GP_REGS_SIZE, efault_out);
164	err \|= __copy_to_user(&sc->gp_regs, regs, GP_REGS_SIZE);	176	unsafe_put_user(msr, &sc->gp_regs[PT_MSR], efault_out);
165	err \|= __put_user(msr, &sc->gp_regs[PT_MSR]);	177	unsafe_put_user(softe, &sc->gp_regs[PT_SOFTE], efault_out);
166	err \|= __put_user(softe, &sc->gp_regs[PT_SOFTE]);	178	unsafe_put_user(signr, &sc->signal, efault_out);
167	err \|= __put_user(signr, &sc->signal);	179	unsafe_put_user(handler, &sc->handler, efault_out);
168	err \|= __put_user(handler, &sc->handler);
169	if (set != NULL)	180	if (set != NULL)
170	err \|= __put_user(set->sig[0], &sc->oldmask);	181	unsafe_put_user(set->sig[0], &sc->oldmask, efault_out);
171		182
172	return err;	183	return 0;
		184
		185	efault_out:
		186	return -EFAULT;
173	}	187	}
174		188
175	#ifdef CONFIG_PPC_TRANSACTIONAL_MEM	189	#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
@@ -294,7 +308,6 @@ static long setup_tm_sigcontexts(struct sigcontext __user *sc, · 118 unmodified lines
294		308
295	err \|= __put_user(&sc->gp_regs, &sc->regs);	309	err \|= __put_user(&sc->gp_regs, &sc->regs);
296	err \|= __put_user(&tm_sc->gp_regs, &tm_sc->regs);	310	err \|= __put_user(&tm_sc->gp_regs, &tm_sc->regs);
297	WARN_ON(!FULL_REGS(regs));
298	err \|= __copy_to_user(&tm_sc->gp_regs, regs, GP_REGS_SIZE);	311	err \|= __copy_to_user(&tm_sc->gp_regs, regs, GP_REGS_SIZE);
299	err \|= __copy_to_user(&sc->gp_regs,	312	err \|= __copy_to_user(&sc->gp_regs,
300	&tsk->thread.ckpt_regs, GP_REGS_SIZE);	313	&tsk->thread.ckpt_regs, GP_REGS_SIZE);
@@ -312,14 +325,16 @@ static long setup_tm_sigcontexts(struct sigcontext __user *sc, · 11 unmodified lines
312	/*	325	/*
313	* Restore the sigcontext from the signal frame.	326	* Restore the sigcontext from the signal frame.
314	*/	327	*/
315		328	#define unsafe_restore_sigcontext(tsk, set, sig, sc, label) do { \
316	static long restore_sigcontext(struct task_struct tsk, sigset_t set, int sig,	329	if (__unsafe_restore_sigcontext(tsk, set, sig, sc)) \
317	struct sigcontext __user *sc)	330	goto label; \
		331	} while (0)
		332	static long notrace __unsafe_restore_sigcontext(struct task_struct tsk, sigset_t set,
		333	int sig, struct sigcontext __user *sc)
318	{	334	{
319	#ifdef CONFIG_ALTIVEC	335	#ifdef CONFIG_ALTIVEC
320	elf_vrreg_t __user *v_regs;	336	elf_vrreg_t __user *v_regs;
321	#endif	337	#endif
322	unsigned long err = 0;
323	unsigned long save_r13 = 0;	338	unsigned long save_r13 = 0;
324	unsigned long msr;	339	unsigned long msr;
325	struct pt_regs *regs = tsk->thread.regs;	340	struct pt_regs *regs = tsk->thread.regs;
@@ -334,27 +349,27 @@ static long restore_sigcontext(struct task_struct tsk, sigset_t set, int sig, · 8 unmodified lines
334	save_r13 = regs->gpr[13];	349	save_r13 = regs->gpr[13];
335		350
336	/* copy the GPRs */	351	/* copy the GPRs */
337	err \|= __copy_from_user(regs->gpr, sc->gp_regs, sizeof(regs->gpr));	352	unsafe_copy_from_user(regs->gpr, sc->gp_regs, sizeof(regs->gpr), efault_out);
338	err \|= __get_user(regs->nip, &sc->gp_regs[PT_NIP]);	353	unsafe_get_user(regs->nip, &sc->gp_regs[PT_NIP], efault_out);
339	/* get MSR separately, transfer the LE bit if doing signal return */	354	/* get MSR separately, transfer the LE bit if doing signal return */
340	err \|= __get_user(msr, &sc->gp_regs[PT_MSR]);	355	unsafe_get_user(msr, &sc->gp_regs[PT_MSR], efault_out);
341	if (sig)	356	if (sig)
342	regs->msr = (regs->msr & ~MSR_LE) \| (msr & MSR_LE);	357	regs->msr = (regs->msr & ~MSR_LE) \| (msr & MSR_LE);
343	err \|= __get_user(regs->orig_gpr3, &sc->gp_regs[PT_ORIG_R3]);	358	unsafe_get_user(regs->orig_gpr3, &sc->gp_regs[PT_ORIG_R3], efault_out);
344	err \|= __get_user(regs->ctr, &sc->gp_regs[PT_CTR]);	359	unsafe_get_user(regs->ctr, &sc->gp_regs[PT_CTR], efault_out);
345	err \|= __get_user(regs->link, &sc->gp_regs[PT_LNK]);	360	unsafe_get_user(regs->link, &sc->gp_regs[PT_LNK], efault_out);
346	err \|= __get_user(regs->xer, &sc->gp_regs[PT_XER]);	361	unsafe_get_user(regs->xer, &sc->gp_regs[PT_XER], efault_out);
347	err \|= __get_user(regs->ccr, &sc->gp_regs[PT_CCR]);	362	unsafe_get_user(regs->ccr, &sc->gp_regs[PT_CCR], efault_out);
348	/* Don't allow userspace to set SOFTE */	363	/* Don't allow userspace to set SOFTE */
349	set_trap_norestart(regs);	364	set_trap_norestart(regs);
350	err \|= __get_user(regs->dar, &sc->gp_regs[PT_DAR]);	365	unsafe_get_user(regs->dar, &sc->gp_regs[PT_DAR], efault_out);
351	err \|= __get_user(regs->dsisr, &sc->gp_regs[PT_DSISR]);	366	unsafe_get_user(regs->dsisr, &sc->gp_regs[PT_DSISR], efault_out);
352	err \|= __get_user(regs->result, &sc->gp_regs[PT_RESULT]);	367	unsafe_get_user(regs->result, &sc->gp_regs[PT_RESULT], efault_out);
353		368
354	if (!sig)	369	if (!sig)
355	regs->gpr[13] = save_r13;	370	regs->gpr[13] = save_r13;
356	if (set != NULL)	371	if (set != NULL)
357	err \|= __get_user(set->sig[0], &sc->oldmask);	372	unsafe_get_user(set->sig[0], &sc->oldmask, efault_out);
358		373
359	/*	374	/*
360	* Force reload of FP/VEC.	375	* Force reload of FP/VEC.
@@ -364,29 +379,27 @@ static long restore_sigcontext(struct task_struct tsk, sigset_t set, int sig, · 3 unmodified lines
364	regs->msr &= ~(MSR_FP \| MSR_FE0 \| MSR_FE1 \| MSR_VEC \| MSR_VSX);	379	regs->msr &= ~(MSR_FP \| MSR_FE0 \| MSR_FE1 \| MSR_VEC \| MSR_VSX);
365		380
366	#ifdef CONFIG_ALTIVEC	381	#ifdef CONFIG_ALTIVEC
367	err \|= __get_user(v_regs, &sc->v_regs);	382	unsafe_get_user(v_regs, &sc->v_regs, efault_out);
368	if (err)
369	return err;
370	if (v_regs && !access_ok(v_regs, 34 * sizeof(vector128)))	383	if (v_regs && !access_ok(v_regs, 34 * sizeof(vector128)))
371	return -EFAULT;	384	return -EFAULT;
372	/* Copy 33 vec registers (vr0..31 and vscr) from the stack */	385	/* Copy 33 vec registers (vr0..31 and vscr) from the stack */
373	if (v_regs != NULL && (msr & MSR_VEC) != 0) {	386	if (v_regs != NULL && (msr & MSR_VEC) != 0) {
374	err \|= __copy_from_user(&tsk->thread.vr_state, v_regs,	387	unsafe_copy_from_user(&tsk->thread.vr_state, v_regs,
375	33 * sizeof(vector128));	388	33 * sizeof(vector128), efault_out);
376	tsk->thread.used_vr = true;	389	tsk->thread.used_vr = true;
377	} else if (tsk->thread.used_vr) {	390	} else if (tsk->thread.used_vr) {
378	memset(&tsk->thread.vr_state, 0, 33 * sizeof(vector128));	391	memset(&tsk->thread.vr_state, 0, 33 * sizeof(vector128));
379	}	392	}
380	/* Always get VRSAVE back */	393	/* Always get VRSAVE back */
381	if (v_regs != NULL)	394	if (v_regs != NULL)
382	err \|= __get_user(tsk->thread.vrsave, (u32 __user *)&v_regs[33]);	395	unsafe_get_user(tsk->thread.vrsave, (u32 __user *)&v_regs[33], efault_out);
383	else	396	else
384	tsk->thread.vrsave = 0;	397	tsk->thread.vrsave = 0;
385	if (cpu_has_feature(CPU_FTR_ALTIVEC))	398	if (cpu_has_feature(CPU_FTR_ALTIVEC))
386	mtspr(SPRN_VRSAVE, tsk->thread.vrsave);	399	mtspr(SPRN_VRSAVE, tsk->thread.vrsave);
387	#endif /* CONFIG_ALTIVEC */	400	#endif /* CONFIG_ALTIVEC */
388	/* restore floating point */	401	/* restore floating point */
389	err \|= copy_fpr_from_user(tsk, &sc->fp_regs);	402	unsafe_copy_fpr_from_user(tsk, &sc->fp_regs, efault_out);
390	#ifdef CONFIG_VSX	403	#ifdef CONFIG_VSX
391	/*	404	/*
392	* Get additional VSX data. Update v_regs to point after the	405	* Get additional VSX data. Update v_regs to point after the
@@ -395,14 +408,17 @@ static long restore_sigcontext(struct task_struct tsk, sigset_t set, int sig, · 2 unmodified lines
395	*/	408	*/
396	v_regs += ELF_NVRREG;	409	v_regs += ELF_NVRREG;
397	if ((msr & MSR_VSX) != 0) {	410	if ((msr & MSR_VSX) != 0) {
398	err \|= copy_vsx_from_user(tsk, v_regs);	411	unsafe_copy_vsx_from_user(tsk, v_regs, efault_out);
399	tsk->thread.used_vsr = true;	412	tsk->thread.used_vsr = true;
400	} else {	413	} else {
401	for (i = 0; i < 32 ; i++)	414	for (i = 0; i < 32 ; i++)
402	tsk->thread.fp_state.fpr[i][TS_VSRLOWOFFSET] = 0;	415	tsk->thread.fp_state.fpr[i][TS_VSRLOWOFFSET] = 0;
403	}	416	}
404	#endif	417	#endif
405	return err;	418	return 0;
		419
		420	efault_out:
		421	return -EFAULT;
406	}	422	}
407		423
408	#ifdef CONFIG_PPC_TRANSACTIONAL_MEM	424	#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
@@ -586,6 +602,12 @@ static long restore_tm_sigcontexts(struct task_struct *tsk, · 177 unmodified lines
586		602
587	return err;	603	return err;
588	}	604	}
		605	#else /* !CONFIG_PPC_TRANSACTIONAL_MEM */
		606	static long restore_tm_sigcontexts(struct task_struct tsk, struct sigcontext __user sc,
		607	struct sigcontext __user *tm_sc)
		608	{
		609	return -EINVAL;
		610	}
589	#endif	611	#endif
590		612
591	/*	613	/*
@@ -655,12 +677,16 @@ SYSCALL_DEFINE3(swapcontext, struct ucontext __user *, old_ctx, · 63 unmodified lines
655	ctx_has_vsx_region = 1;	677	ctx_has_vsx_region = 1;
656		678
657	if (old_ctx != NULL) {	679	if (old_ctx != NULL) {
658	if (!access_ok(old_ctx, ctx_size)	680	prepare_setup_sigcontext(current);
659	\|\| setup_sigcontext(&old_ctx->uc_mcontext, current, 0, NULL, 0,	681	if (!user_write_access_begin(old_ctx, ctx_size))
660	ctx_has_vsx_region)
661	\|\| __copy_to_user(&old_ctx->uc_sigmask,
662	&current->blocked, sizeof(sigset_t)))
663	return -EFAULT;	682	return -EFAULT;
		683
		684	unsafe_setup_sigcontext(&old_ctx->uc_mcontext, current, 0, NULL,
		685	0, ctx_has_vsx_region, efault_out);
		686	unsafe_copy_to_user(&old_ctx->uc_sigmask, &current->blocked,
		687	sizeof(sigset_t), efault_out);
		688
		689	user_write_access_end();
664	}	690	}
665	if (new_ctx == NULL)	691	if (new_ctx == NULL)
666	return 0;	692	return 0;
@@ -680,15 +706,25 @@ SYSCALL_DEFINE3(swapcontext, struct ucontext __user *, old_ctx, · 13 unmodified lines
680	* We kill the task with a SIGSEGV in this situation.	706	* We kill the task with a SIGSEGV in this situation.
681	*/	707	*/
682		708
683	if (__copy_from_user(&set, &new_ctx->uc_sigmask, sizeof(set)))	709	if (__get_user_sigset(&set, &new_ctx->uc_sigmask))
684	do_exit(SIGSEGV);	710	do_exit(SIGSEGV);
685	set_current_blocked(&set);	711	set_current_blocked(&set);
686	if (restore_sigcontext(current, NULL, 0, &new_ctx->uc_mcontext))	712
		713	if (!user_read_access_begin(new_ctx, ctx_size))
		714	return -EFAULT;
		715	if (__unsafe_restore_sigcontext(current, NULL, 0, &new_ctx->uc_mcontext)) {
		716	user_read_access_end();
687	do_exit(SIGSEGV);	717	do_exit(SIGSEGV);
		718	}
		719	user_read_access_end();
688		720
689	/* This returns like rt_sigreturn */	721	/* This returns like rt_sigreturn */
690	set_thread_flag(TIF_RESTOREALL);	722	set_thread_flag(TIF_RESTOREALL);
691	return 0;	723	return 0;
		724
		725	efault_out:
		726	user_write_access_end();
		727	return -EFAULT;
692	}	728	}
693		729
694		730
@@ -701,9 +737,7 @@ SYSCALL_DEFINE0(rt_sigreturn) · 6 unmodified lines
701	struct pt_regs *regs = current_pt_regs();	737	struct pt_regs *regs = current_pt_regs();
702	struct ucontext __user uc = (struct ucontext __user )regs->gpr[1];	738	struct ucontext __user uc = (struct ucontext __user )regs->gpr[1];
703	sigset_t set;	739	sigset_t set;
704	#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
705	unsigned long msr;	740	unsigned long msr;
706	#endif
707		741
708	/* Always make any pending restarted system calls return -EINTR */	742	/* Always make any pending restarted system calls return -EINTR */
709	current->restart_block.fn = do_no_restart_syscall;	743	current->restart_block.fn = do_no_restart_syscall;
@@ -711,52 +745,54 @@ SYSCALL_DEFINE0(rt_sigreturn) · 1 unmodified line
711	if (!access_ok(uc, sizeof(*uc)))	745	if (!access_ok(uc, sizeof(*uc)))
712	goto badframe;	746	goto badframe;
713		747
714	if (__copy_from_user(&set, &uc->uc_sigmask, sizeof(set)))	748	if (__get_user_sigset(&set, &uc->uc_sigmask))
715	goto badframe;	749	goto badframe;
716	set_current_blocked(&set);	750	set_current_blocked(&set);
717		751
718	#ifdef CONFIG_PPC_TRANSACTIONAL_MEM	752	if (IS_ENABLED(CONFIG_PPC_TRANSACTIONAL_MEM)) {
719	/*	753	/*
720	* If there is a transactional state then throw it away.	754	* If there is a transactional state then throw it away.
721	* The purpose of a sigreturn is to destroy all traces of the	755	* The purpose of a sigreturn is to destroy all traces of the
722	* signal frame, this includes any transactional state created	756	* signal frame, this includes any transactional state created
723	* within in. We only check for suspended as we can never be	757	* within in. We only check for suspended as we can never be
724	* active in the kernel, we are active, there is nothing better to	758	* active in the kernel, we are active, there is nothing better to
725	* do than go ahead and Bad Thing later.	759	* do than go ahead and Bad Thing later.
726	* The cause is not important as there will never be a	760	* The cause is not important as there will never be a
727	* recheckpoint so it's not user visible.	761	* recheckpoint so it's not user visible.
728	*/	762	*/
729	if (MSR_TM_SUSPENDED(mfmsr()))	763	if (MSR_TM_SUSPENDED(mfmsr()))
730	tm_reclaim_current(0);	764	tm_reclaim_current(0);
731		765
732	/*	766	/*
733	* Disable MSR[TS] bit also, so, if there is an exception in the	767	* Disable MSR[TS] bit also, so, if there is an exception in the
734	* code below (as a page fault in copy_ckvsx_to_user()), it does	768	* code below (as a page fault in copy_ckvsx_to_user()), it does
735	* not recheckpoint this task if there was a context switch inside	769	* not recheckpoint this task if there was a context switch inside
736	* the exception.	770	* the exception.
737	*	771	*
738	* A major page fault can indirectly call schedule(). A reschedule	772	* A major page fault can indirectly call schedule(). A reschedule
739	* process in the middle of an exception can have a side effect	773	* process in the middle of an exception can have a side effect
740	* (Changing the CPU MSR[TS] state), since schedule() is called	774	* (Changing the CPU MSR[TS] state), since schedule() is called
741	* with the CPU MSR[TS] disable and returns with MSR[TS]=Suspended	775	* with the CPU MSR[TS] disable and returns with MSR[TS]=Suspended
742	* (switch_to() calls tm_recheckpoint() for the 'new' process). In	776	* (switch_to() calls tm_recheckpoint() for the 'new' process). In
743	* this case, the process continues to be the same in the CPU, but	777	* this case, the process continues to be the same in the CPU, but
744	* the CPU state just changed.	778	* the CPU state just changed.
745	*	779	*
746	* This can cause a TM Bad Thing, since the MSR in the stack will	780	* This can cause a TM Bad Thing, since the MSR in the stack will
747	* have the MSR[TS]=0, and this is what will be used to RFID.	781	* have the MSR[TS]=0, and this is what will be used to RFID.
748	*	782	*
749	* Clearing MSR[TS] state here will avoid a recheckpoint if there	783	* Clearing MSR[TS] state here will avoid a recheckpoint if there
750	* is any process reschedule in kernel space. The MSR[TS] state	784	* is any process reschedule in kernel space. The MSR[TS] state
751	* does not need to be saved also, since it will be replaced with	785	* does not need to be saved also, since it will be replaced with
752	* the MSR[TS] that came from user context later, at	786	* the MSR[TS] that came from user context later, at
753	* restore_tm_sigcontexts.	787	* restore_tm_sigcontexts.
754	*/	788	*/
755	regs->msr &= ~MSR_TS_MASK;	789	regs->msr &= ~MSR_TS_MASK;
756		790
757	if (__get_user(msr, &uc->uc_mcontext.gp_regs[PT_MSR]))	791	if (__get_user(msr, &uc->uc_mcontext.gp_regs[PT_MSR]))
758	goto badframe;	792	goto badframe;
759	if (MSR_TM_ACTIVE(msr)) {	793	}
		794
		795	if (IS_ENABLED(CONFIG_PPC_TRANSACTIONAL_MEM) && MSR_TM_ACTIVE(msr)) {
760	/* We recheckpoint on return. */	796	/* We recheckpoint on return. */
761	struct ucontext __user *uc_transact;	797	struct ucontext __user *uc_transact;
762		798
@@ -769,9 +805,7 @@ SYSCALL_DEFINE0(rt_sigreturn) · 6 unmodified lines
769	if (restore_tm_sigcontexts(current, &uc->uc_mcontext,	805	if (restore_tm_sigcontexts(current, &uc->uc_mcontext,
770	&uc_transact->uc_mcontext))	806	&uc_transact->uc_mcontext))
771	goto badframe;	807	goto badframe;
772	} else	808	} else {
773	#endif
774	{
775	/*	809	/*
776	* Fall through, for non-TM restore	810	* Fall through, for non-TM restore
777	*	811	*
@@ -785,8 +819,13 @@ SYSCALL_DEFINE0(rt_sigreturn) · 7 unmodified lines
785	* causing a TM bad thing.	819	* causing a TM bad thing.
786	*/	820	*/
787	current->thread.regs->msr &= ~MSR_TS_MASK;	821	current->thread.regs->msr &= ~MSR_TS_MASK;
788	if (restore_sigcontext(current, NULL, 1, &uc->uc_mcontext))	822	if (!user_read_access_begin(&uc->uc_mcontext, sizeof(uc->uc_mcontext)))
789	goto badframe;	823	goto badframe;
		824
		825	unsafe_restore_sigcontext(current, NULL, 1, &uc->uc_mcontext,
		826	badframe_block);
		827
		828	user_read_access_end();
790	}	829	}
791		830
792	if (restore_altstack(&uc->uc_stack))	831	if (restore_altstack(&uc->uc_stack))
@@ -795,6 +834,8 @@ SYSCALL_DEFINE0(rt_sigreturn) · 2 unmodified lines
795	set_thread_flag(TIF_RESTOREALL);	834	set_thread_flag(TIF_RESTOREALL);
796	return 0;	835	return 0;
797		836
		837	badframe_block:
		838	user_read_access_end();
798	badframe:	839	badframe:
799	signal_fault(current, regs, "rt_sigreturn", uc);	840	signal_fault(current, regs, "rt_sigreturn", uc);
800		841
@@ -809,46 +850,57 @@ int handle_rt_signal64(struct ksignal ksig, sigset_t set, · 8 unmodified lines
809	unsigned long newsp = 0;	850	unsigned long newsp = 0;
810	long err = 0;	851	long err = 0;
811	struct pt_regs *regs = tsk->thread.regs;	852	struct pt_regs *regs = tsk->thread.regs;
812	#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
813	/* Save the thread's msr before get_tm_stackpointer() changes it */	853	/* Save the thread's msr before get_tm_stackpointer() changes it */
814	unsigned long msr = regs->msr;	854	unsigned long msr = regs->msr;
815	#endif
816		855
817	frame = get_sigframe(ksig, tsk, sizeof(*frame), 0);	856	frame = get_sigframe(ksig, tsk, sizeof(*frame), 0);
818	if (!access_ok(frame, sizeof(*frame)))
819	goto badframe;
820		857
821	err \|= __put_user(&frame->info, &frame->pinfo);	858	/*
822	err \|= __put_user(&frame->uc, &frame->puc);	859	* This only applies when calling unsafe_setup_sigcontext() and must be
823	err \|= copy_siginfo_to_user(&frame->info, &ksig->info);	860	* called before opening the uaccess window.
824	if (err)	861	*/
		862	if (!MSR_TM_ACTIVE(msr))
		863	prepare_setup_sigcontext(tsk);
		864
		865	if (!user_write_access_begin(frame, sizeof(*frame)))
825	goto badframe;	866	goto badframe;
826		867
		868	unsafe_put_user(&frame->info, &frame->pinfo, badframe_block);
		869	unsafe_put_user(&frame->uc, &frame->puc, badframe_block);
		870
827	/* Create the ucontext. */	871	/* Create the ucontext. */
828	err \|= __put_user(0, &frame->uc.uc_flags);	872	unsafe_put_user(0, &frame->uc.uc_flags, badframe_block);
829	err \|= __save_altstack(&frame->uc.uc_stack, regs->gpr[1]);	873	unsafe_save_altstack(&frame->uc.uc_stack, regs->gpr[1], badframe_block);
830	#ifdef CONFIG_PPC_TRANSACTIONAL_MEM	874
831	if (MSR_TM_ACTIVE(msr)) {	875	if (MSR_TM_ACTIVE(msr)) {
		876	#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
832	/* The ucontext_t passed to userland points to the second	877	/* The ucontext_t passed to userland points to the second
833	* ucontext_t (for transactional state) with its uc_link ptr.	878	* ucontext_t (for transactional state) with its uc_link ptr.
834	*/	879	*/
835	err \|= __put_user(&frame->uc_transact, &frame->uc.uc_link);	880	unsafe_put_user(&frame->uc_transact, &frame->uc.uc_link, badframe_block);
		881
		882	user_write_access_end();
		883
836	err \|= setup_tm_sigcontexts(&frame->uc.uc_mcontext,	884	err \|= setup_tm_sigcontexts(&frame->uc.uc_mcontext,
837	&frame->uc_transact.uc_mcontext,	885	&frame->uc_transact.uc_mcontext,
838	tsk, ksig->sig, NULL,	886	tsk, ksig->sig, NULL,
839	(unsigned long)ksig->ka.sa.sa_handler,	887	(unsigned long)ksig->ka.sa.sa_handler,
840	msr);	888	msr);
841	} else	889
		890	if (!user_write_access_begin(&frame->uc.uc_sigmask,
		891	sizeof(frame->uc.uc_sigmask)))
		892	goto badframe;
		893
842	#endif	894	#endif
843	{	895	} else {
844	err \|= __put_user(0, &frame->uc.uc_link);	896	unsafe_put_user(0, &frame->uc.uc_link, badframe_block);
845	err \|= setup_sigcontext(&frame->uc.uc_mcontext, tsk, ksig->sig,	897	unsafe_setup_sigcontext(&frame->uc.uc_mcontext, tsk, ksig->sig,
846	NULL, (unsigned long)ksig->ka.sa.sa_handler,	898	NULL, (unsigned long)ksig->ka.sa.sa_handler,
847	1);	899	1, badframe_block);
848	}	900	}
849	err \|= __copy_to_user(&frame->uc.uc_sigmask, set, sizeof(*set));	901
850	if (err)	902	unsafe_copy_to_user(&frame->uc.uc_sigmask, set, sizeof(*set), badframe_block);
851	goto badframe;	903	user_write_access_end();
852		904
853	/* Make sure signal handler doesn't get spurious FP exceptions */	905	/* Make sure signal handler doesn't get spurious FP exceptions */
854	tsk->thread.fp_state.fpscr = 0;	906	tsk->thread.fp_state.fpscr = 0;
@@ -863,6 +915,11 @@ int handle_rt_signal64(struct ksignal ksig, sigset_t set, · 8 unmodified lines
863	regs->nip = (unsigned long) &frame->tramp[0];	915	regs->nip = (unsigned long) &frame->tramp[0];
864	}	916	}
865		917
		918
		919	/* Save the siginfo outside of the unsafe block. */
		920	if (copy_siginfo_to_user(&frame->info, &ksig->info))
		921	goto badframe;
		922
866	/* Allocate a dummy caller frame for the signal handler. */	923	/* Allocate a dummy caller frame for the signal handler. */
867	newsp = ((unsigned long)frame) - __SIGNAL_FRAMESIZE;	924	newsp = ((unsigned long)frame) - __SIGNAL_FRAMESIZE;
868	err \|= put_user(regs->gpr[1], (unsigned long __user *)newsp);	925	err \|= put_user(regs->gpr[1], (unsigned long __user *)newsp);
@@ -902,6 +959,8 @@ int handle_rt_signal64(struct ksignal ksig, sigset_t set, · 33 unmodified lines
902		959
903	return 0;	960	return 0;
904		961
		962	badframe_block:
		963	user_write_access_end();
905	badframe:	964	badframe:
906	signal_fault(current, regs, "handle_rt_signal64", frame);	965	signal_fault(current, regs, "handle_rt_signal64", frame);
907		966

arch/powerpc/kernel/smp.c+55−13

 DEFINE_PER_CPU(cpumask_var_t, cpu_smallcore_map);
 DEFINE_PER_CPU(cpumask_var_t, cpu_l2_cache_map);
 DEFINE_PER_CPU(cpumask_var_t, cpu_core_map);
-DEFINE_PER_CPU(cpumask_var_t, cpu_coregroup_map);
+static DEFINE_PER_CPU(cpumask_var_t, cpu_coregroup_map);
 EXPORT_PER_CPU_SYMBOL(cpu_sibling_map);
 EXPORT_PER_CPU_SYMBOL(cpu_l2_cache_map);
  * On big-cores system, thread_group_l1_cache_map for each CPU corresponds to
  * the set its siblings that share the L1-cache.
  */
-DEFINE_PER_CPU(cpumask_var_t, thread_group_l1_cache_map);
+static DEFINE_PER_CPU(cpumask_var_t, thread_group_l1_cache_map);
 /*
  * On some big-cores system, thread_group_l2_cache_map for each CPU
  * corresponds to the set its siblings within the core that share the
  * L2-cache.
  */
-DEFINE_PER_CPU(cpumask_var_t, thread_group_l2_cache_map);
+static DEFINE_PER_CPU(cpumask_var_t, thread_group_l2_cache_map);
 /* SMP operations for this machine */
 struct smp_ops_t *smp_ops;
 				local_memory_node(numa_cpu_lookup_table[cpu]));
 		}
 #endif
-		/*
-		 * cpu_core_map is now more updated and exists only since
-		 * its been exported for long. It only will have a snapshot
-		 * of cpu_cpu_mask.
-		 */
-		cpumask_copy(per_cpu(cpu_core_map, cpu), cpu_cpu_mask(cpu));
 	}
 	/* Init the cpumasks so the boot CPU is related to itself */
 	cpumask_set_cpu(boot_cpuid, cpu_sibling_mask(boot_cpuid));
 	cpumask_set_cpu(boot_cpuid, cpu_l2_cache_mask(boot_cpuid));
+	cpumask_set_cpu(boot_cpuid, cpu_core_mask(boot_cpuid));
 	if (has_coregroup_support())
 		cpumask_set_cpu(boot_cpuid, cpu_coregroup_mask(boot_cpuid));
 				cpu_smallcore_mask(boot_cpuid));
 	}
+	if (cpu_to_chip_id(boot_cpuid) != -1) {
+		int idx = num_possible_cpus() / threads_per_core;
+		/*
+		 * All threads of a core will all belong to the same core,
+		 * chip_id_lookup_table will have one entry per core.
+		 * Assumption: if boot_cpuid doesn't have a chip-id, then no
+		 * other CPUs, will also not have chip-id.
+		 */
+		chip_id_lookup_table = kcalloc(idx, sizeof(int), GFP_KERNEL);
+		if (chip_id_lookup_table)
+			memset(chip_id_lookup_table, -1, sizeof(int) * idx);
+	}
 	if (smp_ops && smp_ops->probe)
 		smp_ops->probe();
 }
 			set_cpus_unrelated(cpu, i, cpu_smallcore_mask);
 	}
+	for_each_cpu(i, cpu_core_mask(cpu))
+		set_cpus_unrelated(cpu, i, cpu_core_mask);
 	if (has_coregroup_support()) {
 		for_each_cpu(i, cpu_coregroup_mask(cpu))
 			set_cpus_unrelated(cpu, i, cpu_coregroup_mask);
 static void add_cpu_to_masks(int cpu)
 {
+	struct cpumask *(*submask_fn)(int) = cpu_sibling_mask;
 	int first_thread = cpu_first_thread_sibling(cpu);
 	cpumask_var_t mask;
+	int chip_id = -1;
+	bool ret;
 	int i;
 	/*
 	add_cpu_to_smallcore_masks(cpu);
 	/* In CPU-hotplug path, hence use GFP_ATOMIC */
-	alloc_cpumask_var_node(&mask, GFP_ATOMIC, cpu_to_node(cpu));
+	ret = alloc_cpumask_var_node(&mask, GFP_ATOMIC, cpu_to_node(cpu));
 	update_mask_by_l2(cpu, &mask);
 	if (has_coregroup_support())
 		update_coregroup_mask(cpu, &mask);
+	if (chip_id_lookup_table && ret)
+		chip_id = cpu_to_chip_id(cpu);
+	if (chip_id == -1) {
+		cpumask_copy(per_cpu(cpu_core_map, cpu), cpu_cpu_mask(cpu));
+		goto out;
+	}
+	if (shared_caches)
+		submask_fn = cpu_l2_cache_mask;
+	/* Update core_mask with all the CPUs that are part of submask */
+	or_cpumasks_related(cpu, cpu, submask_fn, cpu_core_mask);
+	/* Skip all CPUs already part of current CPU core mask */
+	cpumask_andnot(mask, cpu_online_mask, cpu_core_mask(cpu));
+	for_each_cpu(i, mask) {
+		if (chip_id == cpu_to_chip_id(i)) {
+			or_cpumasks_related(cpu, i, submask_fn, cpu_core_mask);
+			cpumask_andnot(mask, mask, submask_fn(i));
+		} else {
+			cpumask_andnot(mask, mask, cpu_core_mask(i));
+		}
+	}
+out:
 	free_cpumask_var(mask);
 }
 	vdso_getcpu_init();
 #endif
+	set_numa_node(numa_cpu_lookup_table[cpu]);
+	set_numa_mem(local_memory_node(numa_cpu_lookup_table[cpu]));
 	/* Update topology CPU masks */
 	add_cpu_to_masks(cpu);
 			shared_caches = true;
 	}
-	set_numa_node(numa_cpu_lookup_table[cpu]);
-	set_numa_mem(local_memory_node(numa_cpu_lookup_table[cpu]));
 	smp_wmb();
 	notify_cpu_starting(cpu);
 	set_cpu_online(cpu, true);

arch/powerpc/kernel/stacktrace.c+25−83

 #include <asm/paca.h>
-/*
+void arch_stack_walk(stack_trace_consume_fn consume_entry, void *cookie,
- * Save stack-backtrace addresses into a stack_trace buffer.
+		     struct task_struct *task, struct pt_regs *regs)
- */
-static void save_context_stack(struct stack_trace *trace, unsigned long sp,
-			struct task_struct *tsk, int savesched)
 {
+	unsigned long sp;
+	if (regs && !consume_entry(cookie, regs->nip))
+		return;
+	if (regs)
+		sp = regs->gpr[1];
+	else if (task == current)
+		sp = current_stack_frame();
+	else
+		sp = task->thread.ksp;
 	for (;;) {
 		unsigned long *stack = (unsigned long *) sp;
 		unsigned long newsp, ip;
-		if (!validate_sp(sp, tsk, STACK_FRAME_OVERHEAD))
+		if (!validate_sp(sp, task, STACK_FRAME_OVERHEAD))
 			return;
 		newsp = stack[0];
 		ip = stack[STACK_FRAME_LR_SAVE];
-		if (savesched || !in_sched_functions(ip)) {
+		if (!consume_entry(cookie, ip))
-			if (!trace->skip)
-				trace->entries[trace->nr_entries++] = ip;
-			else
-				trace->skip--;
-		}
-		if (trace->nr_entries >= trace->max_entries)
 			return;
 		sp = newsp;
 	}
 }
-void save_stack_trace(struct stack_trace *trace)
-{
-	unsigned long sp;
-	sp = current_stack_frame();
-	save_context_stack(trace, sp, current, 1);
-}
-EXPORT_SYMBOL_GPL(save_stack_trace);
-void save_stack_trace_tsk(struct task_struct *tsk, struct stack_trace *trace)
-{
-	unsigned long sp;
-	if (!try_get_task_stack(tsk))
-		return;
-	if (tsk == current)
-		sp = current_stack_frame();
-	else
-		sp = tsk->thread.ksp;
-	save_context_stack(trace, sp, tsk, 0);
-	put_task_stack(tsk);
-}
-EXPORT_SYMBOL_GPL(save_stack_trace_tsk);
-void
-save_stack_trace_regs(struct pt_regs *regs, struct stack_trace *trace)
-{
-	save_context_stack(trace, regs->gpr[1], current, 0);
-}
-EXPORT_SYMBOL_GPL(save_stack_trace_regs);
-#ifdef CONFIG_HAVE_RELIABLE_STACKTRACE
 /*
  * This function returns an error if it detects any unreliable features of the
  * stack.  Otherwise it guarantees that the stack trace is reliable.
  *
  * If the task is not 'current', the caller *must* ensure the task is inactive.
  */
-static int __save_stack_trace_tsk_reliable(struct task_struct *tsk,
+int arch_stack_walk_reliable(stack_trace_consume_fn consume_entry,
-					   struct stack_trace *trace)
+			     void *cookie, struct task_struct *task)
 {
 	unsigned long sp;
 	unsigned long newsp;
-	unsigned long stack_page = (unsigned long)task_stack_page(tsk);
+	unsigned long stack_page = (unsigned long)task_stack_page(task);
 	unsigned long stack_end;
 	int graph_idx = 0;
 	bool firstframe;
 	stack_end = stack_page + THREAD_SIZE;
-	if (!is_idle_task(tsk)) {
+	if (!is_idle_task(task)) {
 		/*
 		 * For user tasks, this is the SP value loaded on
 		 * kernel entry, see "PACAKSAVE(r13)" in _switch() and
 		stack_end -= STACK_FRAME_OVERHEAD;
 	}
-	if (tsk == current)
+	if (task == current)
 		sp = current_stack_frame();
 	else
-		sp = tsk->thread.ksp;
+		sp = task->thread.ksp;
 	if (sp < stack_page + sizeof(struct thread_struct) ||
 	    sp > stack_end - STACK_FRAME_MIN_SIZE) {
 		 * FIXME: IMHO these tests do not belong in
 		 * arch-dependent code, they are generic.
 		 */
-		ip = ftrace_graph_ret_addr(tsk, &graph_idx, ip, stack);
+		ip = ftrace_graph_ret_addr(task, &graph_idx, ip, stack);
 #ifdef CONFIG_KPROBES
 		/*
 		 * Mark stacktraces with kretprobed functions on them
 			return -EINVAL;
 #endif
-		if (trace->nr_entries >= trace->max_entries)
+		if (!consume_entry(cookie, ip))
-			return -E2BIG;
+			return -EINVAL;
-		if (!trace->skip)
-			trace->entries[trace->nr_entries++] = ip;
-		else
-			trace->skip--;
 	}
 	return 0;
 }
-int save_stack_trace_tsk_reliable(struct task_struct *tsk,
-				  struct stack_trace *trace)
-{
-	int ret;
-	/*
-	 * If the task doesn't have a stack (e.g., a zombie), the stack is
-	 * "reliably" empty.
-	 */
-	if (!try_get_task_stack(tsk))
-		return 0;
-	ret = __save_stack_trace_tsk_reliable(tsk, trace);
-	put_task_stack(tsk);
-	return ret;
-}
-#endif /* CONFIG_HAVE_RELIABLE_STACKTRACE */
 #if defined(CONFIG_PPC_BOOK3S_64) && defined(CONFIG_NMI_IPI)
 static void handle_backtrace_ipi(struct pt_regs *regs)
 {

arch/powerpc/kernel/syscalls.c+2−10

@@ -82,16 +82,8 @@ int · 81 unmodified lines
82	ppc_select(int n, fd_set __user inp, fd_set __user outp, fd_set __user exp, struct __kernel_old_timeval __user tvp)	82	ppc_select(int n, fd_set __user inp, fd_set __user outp, fd_set __user exp, struct __kernel_old_timeval __user tvp)
83	{	83	{
84	if ( (unsigned long)n >= 4096 )	84	if ( (unsigned long)n >= 4096 )
85	{	85	return sys_old_select((void __user *)n);
86	unsigned long __user buffer = (unsigned long __user )n;	86
87	if (!access_ok(buffer, 5*sizeof(unsigned long))
88	\|\| __get_user(n, buffer)
89	\|\| __get_user(inp, ((fd_set __user * __user *)(buffer+1)))
90	\|\| __get_user(outp, ((fd_set __user * __user *)(buffer+2)))
91	\|\| __get_user(exp, ((fd_set __user * __user *)(buffer+3)))
92	\|\| __get_user(tvp, ((struct __kernel_old_timeval __user * __user *)(buffer+4))))
93	return -EFAULT;
94	}
95	return sys_select(n, inp, outp, exp, tvp);	87	return sys_select(n, inp, outp, exp, tvp);
96	}	88	}
97	#endif	89	#endif

arch/powerpc/kernel/syscalls/Makefile+9−24

 	  $(shell [ -d '$(kapi)' ] || mkdir -p '$(kapi)')
 syscall := $(src)/syscall.tbl
-syshdr := $(srctree)/$(src)/syscallhdr.sh
+syshdr := $(srctree)/scripts/syscallhdr.sh
-systbl := $(srctree)/$(src)/syscalltbl.sh
+systbl := $(srctree)/scripts/syscalltbl.sh
 quiet_cmd_syshdr = SYSHDR  $@
-      cmd_syshdr = $(CONFIG_SHELL) '$(syshdr)' '$<' '$@'	\
+      cmd_syshdr = $(CONFIG_SHELL) $(syshdr) --emit-nr --abis $(abis) $< $@
-		   '$(syshdr_abis_$(basetarget))'		\
-		   '$(syshdr_pfx_$(basetarget))'		\
-		   '$(syshdr_offset_$(basetarget))'
 quiet_cmd_systbl = SYSTBL  $@
-      cmd_systbl = $(CONFIG_SHELL) '$(systbl)' '$<' '$@'	\
+      cmd_systbl = $(CONFIG_SHELL) $(systbl) --abis $(abis) $< $@
-		   '$(systbl_abis_$(basetarget))'		\
-		   '$(systbl_abi_$(basetarget))'		\
-		   '$(systbl_offset_$(basetarget))'
-syshdr_abis_unistd_32 := common,nospu,32
+$(uapi)/unistd_32.h: abis := common,nospu,32
 $(uapi)/unistd_32.h: $(syscall) $(syshdr) FORCE
 	$(call if_changed,syshdr)
-syshdr_abis_unistd_64 := common,nospu,64
+$(uapi)/unistd_64.h: abis := common,nospu,64
 $(uapi)/unistd_64.h: $(syscall) $(syshdr) FORCE
 	$(call if_changed,syshdr)
-systbl_abis_syscall_table_32 := common,nospu,32
+$(kapi)/syscall_table_32.h: abis := common,nospu,32
-systbl_abi_syscall_table_32 := 32
 $(kapi)/syscall_table_32.h: $(syscall) $(systbl) FORCE
 	$(call if_changed,systbl)
-systbl_abis_syscall_table_64 := common,nospu,64
+$(kapi)/syscall_table_64.h: abis := common,nospu,64
-systbl_abi_syscall_table_64 := 64
 $(kapi)/syscall_table_64.h: $(syscall) $(systbl) FORCE
 	$(call if_changed,systbl)
-systbl_abis_syscall_table_c32 := common,nospu,32
+$(kapi)/syscall_table_spu.h: abis := common,spu
-systbl_abi_syscall_table_c32 := c32
-$(kapi)/syscall_table_c32.h: $(syscall) $(systbl) FORCE
-	$(call if_changed,systbl)
-systbl_abis_syscall_table_spu := common,spu
-systbl_abi_syscall_table_spu := spu
 $(kapi)/syscall_table_spu.h: $(syscall) $(systbl) FORCE
 	$(call if_changed,systbl)
 uapisyshdr-y		+= unistd_32.h unistd_64.h
 kapisyshdr-y		+= syscall_table_32.h		\
 			   syscall_table_64.h		\
-			   syscall_table_c32.h		\
 			   syscall_table_spu.h
 uapisyshdr-y	:= $(addprefix $(uapi)/, $(uapisyshdr-y))

arch/powerpc/kernel/syscalls/syscallhdr.sh+0−36

@@ -1,36 +0,0 @@
1	#!/bin/sh
2	# SPDX-License-Identifier: GPL-2.0
3
4	in="$1"
5	out="$2"
6	my_abis=`echo "($3)" \| tr ',' '\|'`
7	prefix="$4"
8	offset="$5"
9
10	fileguard=_UAPI_ASM_POWERPC_`basename "$out" \| sed \
11	-e 'y/abcdefghijklmnopqrstuvwxyz/ABCDEFGHIJKLMNOPQRSTUVWXYZ/' \
12	-e 's/[^A-Z0-9_]/_/g' -e 's/__/_/g'`
13	grep -E "^[0-9A-Fa-fXx]+[[:space:]]+${my_abis}" "$in" \| sort -n \| (
14	printf "#ifndef %s\n" "${fileguard}"
15	printf "#define %s\n" "${fileguard}"
16	printf "\n"
17
18	nxt=0
19	while read nr abi name entry compat ; do
20	if [ -z "$offset" ]; then
21	printf "#define __NR_%s%s\t%s\n" \
22	"${prefix}" "${name}" "${nr}"
23	else
24	printf "#define __NR_%s%s\t(%s + %s)\n" \
25	"${prefix}" "${name}" "${offset}" "${nr}"
26	fi
27	nxt=$((nr+1))
28	done
29
30	printf "\n"
31	printf "#ifdef __KERNEL__\n"
32	printf "#define __NR_syscalls\t%s\n" "${nxt}"
33	printf "#endif\n"
34	printf "\n"
35	printf "#endif /* %s */\n" "${fileguard}"
36	) > "$out"

arch/powerpc/kernel/syscalls/syscalltbl.sh+0−36

@@ -1,36 +0,0 @@
1	#!/bin/sh
2	# SPDX-License-Identifier: GPL-2.0
3
4	in="$1"
5	out="$2"
6	my_abis=`echo "($3)" \| tr ',' '\|'`
7	my_abi="$4"
8	offset="$5"
9
10	emit() {
11	t_nxt="$1"
12	t_nr="$2"
13	t_entry="$3"
14
15	while [ $t_nxt -lt $t_nr ]; do
16	printf "__SYSCALL(%s,sys_ni_syscall)\n" "${t_nxt}"
17	t_nxt=$((t_nxt+1))
18	done
19	printf "__SYSCALL(%s,%s)\n" "${t_nxt}" "${t_entry}"
20	}
21
22	grep -E "^[0-9A-Fa-fXx]+[[:space:]]+${my_abis}" "$in" \| sort -n \| (
23	nxt=0
24	if [ -z "$offset" ]; then
25	offset=0
26	fi
27
28	while read nr abi name entry compat ; do
29	if [ "$my_abi" = "c32" ] && [ ! -z "$compat" ]; then
30	emit $((nxt+offset)) $((nr+offset)) $compat
31	else
32	emit $((nxt+offset)) $((nr+offset)) $entry
33	fi
34	nxt=$((nr+1))
35	done
36	) > "$out"

arch/powerpc/kernel/systbl.S+4−1

 #define __SYSCALL(nr, entry)	.long entry
 #endif
+#define __SYSCALL_WITH_COMPAT(nr, native, compat)	__SYSCALL(nr, native)
 .globl sys_call_table
 sys_call_table:
 #ifdef CONFIG_PPC64
 #endif
 #ifdef CONFIG_COMPAT
+#undef __SYSCALL_WITH_COMPAT
+#define __SYSCALL_WITH_COMPAT(nr, native, compat)	__SYSCALL(nr, compat)
 .globl compat_sys_call_table
 compat_sys_call_table:
 #define compat_sys_sigsuspend	sys_sigsuspend
-#include <asm/syscall_table_c32.h>
+#include <asm/syscall_table_32.h>
 #endif

arch/powerpc/kernel/trace/ftrace.c+11−11

@@ -68,7 +68,7 @@ ftrace_modify_code(unsigned long ip, struct ppc_inst old, struct ppc_inst new) · 67 unmodified lines
68	*/	68	*/
69		69
70	/* read the text we want to modify */	70	/* read the text we want to modify */
71	if (probe_kernel_read_inst(&replaced, (void *)ip))	71	if (copy_inst_from_kernel_nofault(&replaced, (void *)ip))
72	return -EFAULT;	72	return -EFAULT;
73		73
74	/* Make sure it is what we expect it to be */	74	/* Make sure it is what we expect it to be */
@@ -130,7 +130,7 @@ __ftrace_make_nop(struct module *mod, · 55 unmodified lines
130	struct ppc_inst op, pop;	130	struct ppc_inst op, pop;
131		131
132	/* read where this goes */	132	/* read where this goes */
133	if (probe_kernel_read_inst(&op, (void *)ip)) {	133	if (copy_inst_from_kernel_nofault(&op, (void *)ip)) {
134	pr_err("Fetching opcode failed.\n");	134	pr_err("Fetching opcode failed.\n");
135	return -EFAULT;	135	return -EFAULT;
136	}	136	}
@@ -164,7 +164,7 @@ __ftrace_make_nop(struct module *mod, · 27 unmodified lines
164	/* When using -mkernel_profile there is no load to jump over */	164	/* When using -mkernel_profile there is no load to jump over */
165	pop = ppc_inst(PPC_INST_NOP);	165	pop = ppc_inst(PPC_INST_NOP);
166		166
167	if (probe_kernel_read_inst(&op, (void *)(ip - 4))) {	167	if (copy_inst_from_kernel_nofault(&op, (void *)(ip - 4))) {
168	pr_err("Fetching instruction at %lx failed.\n", ip - 4);	168	pr_err("Fetching instruction at %lx failed.\n", ip - 4);
169	return -EFAULT;	169	return -EFAULT;
170	}	170	}
@@ -197,7 +197,7 @@ __ftrace_make_nop(struct module *mod, · 26 unmodified lines
197	* Check what is in the next instruction. We can see ld r2,40(r1), but	197	* Check what is in the next instruction. We can see ld r2,40(r1), but
198	* on first pass after boot we will see mflr r0.	198	* on first pass after boot we will see mflr r0.
199	*/	199	*/
200	if (probe_kernel_read_inst(&op, (void *)(ip + 4))) {	200	if (copy_inst_from_kernel_nofault(&op, (void *)(ip + 4))) {
201	pr_err("Fetching op failed.\n");	201	pr_err("Fetching op failed.\n");
202	return -EFAULT;	202	return -EFAULT;
203	}	203	}
@@ -349,7 +349,7 @@ static int setup_mcount_compiler_tramp(unsigned long tramp) · 145 unmodified lines
349	return -1;	349	return -1;
350		350
351	/* New trampoline -- read where this goes */	351	/* New trampoline -- read where this goes */
352	if (probe_kernel_read_inst(&op, (void *)tramp)) {	352	if (copy_inst_from_kernel_nofault(&op, (void *)tramp)) {
353	pr_debug("Fetching opcode failed.\n");	353	pr_debug("Fetching opcode failed.\n");
354	return -1;	354	return -1;
355	}	355	}
@@ -399,7 +399,7 @@ static int __ftrace_make_nop_kernel(struct dyn_ftrace *rec, unsigned long addr) · 43 unmodified lines
399	struct ppc_inst op;	399	struct ppc_inst op;
400		400
401	/* Read where this goes */	401	/* Read where this goes */
402	if (probe_kernel_read_inst(&op, (void *)ip)) {	402	if (copy_inst_from_kernel_nofault(&op, (void *)ip)) {
403	pr_err("Fetching opcode failed.\n");	403	pr_err("Fetching opcode failed.\n");
404	return -EFAULT;	404	return -EFAULT;
405	}	405	}
@@ -526,10 +526,10 @@ __ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) · 120 unmodified lines
526	struct module *mod = rec->arch.mod;	526	struct module *mod = rec->arch.mod;
527		527
528	/* read where this goes */	528	/* read where this goes */
529	if (probe_kernel_read_inst(op, ip))	529	if (copy_inst_from_kernel_nofault(op, ip))
530	return -EFAULT;	530	return -EFAULT;
531		531
532	if (probe_kernel_read_inst(op + 1, ip + 4))	532	if (copy_inst_from_kernel_nofault(op + 1, ip + 4))
533	return -EFAULT;	533	return -EFAULT;
534		534
535	if (!expected_nop_sequence(ip, op[0], op[1])) {	535	if (!expected_nop_sequence(ip, op[0], op[1])) {
@@ -592,7 +592,7 @@ __ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) · 56 unmodified lines
592	unsigned long ip = rec->ip;	592	unsigned long ip = rec->ip;
593		593
594	/* read where this goes */	594	/* read where this goes */
595	if (probe_kernel_read_inst(&op, (void *)ip))	595	if (copy_inst_from_kernel_nofault(&op, (void *)ip))
596	return -EFAULT;	596	return -EFAULT;
597		597
598	/* It should be pointing to a nop */	598	/* It should be pointing to a nop */
@@ -648,7 +648,7 @@ static int __ftrace_make_call_kernel(struct dyn_ftrace *rec, unsigned long addr) · 49 unmodified lines
648	}	648	}
649		649
650	/* Make sure we have a nop */	650	/* Make sure we have a nop */
651	if (probe_kernel_read_inst(&op, ip)) {	651	if (copy_inst_from_kernel_nofault(&op, ip)) {
652	pr_err("Unable to read ftrace location %p\n", ip);	652	pr_err("Unable to read ftrace location %p\n", ip);
653	return -EFAULT;	653	return -EFAULT;
654	}	654	}
@@ -726,7 +726,7 @@ __ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr, · 71 unmodified lines
726	}	726	}
727		727
728	/* read where this goes */	728	/* read where this goes */
729	if (probe_kernel_read_inst(&op, (void *)ip)) {	729	if (copy_inst_from_kernel_nofault(&op, (void *)ip)) {
730	pr_err("Fetching opcode failed.\n");	730	pr_err("Fetching opcode failed.\n");
731	return -EFAULT;	731	return -EFAULT;
732	}	732	}

arch/powerpc/kernel/traps.c+21−19

 #ifdef CONFIG_PPC64
 #include <asm/firmware.h>
 #include <asm/processor.h>
-#include <asm/tm.h>
 #endif
 #include <asm/kexec.h>
 #include <asm/ppc-opcode.h>
 	/*
 	 * system_reset_excption handles debugger, crash dump, panic, for 0x100
 	 */
-	if (TRAP(regs) == 0x100)
+	if (TRAP(regs) == INTERRUPT_SYSTEM_RESET)
 		return;
 	crash_fadump(regs, "die oops");
 	/*
 	 * system_reset_excption handles debugger, crash dump, panic, for 0x100
 	 */
-	if (TRAP(regs) != 0x100) {
+	if (TRAP(regs) != INTERRUPT_SYSTEM_RESET) {
 		if (debugger(regs))
 			return;
 	}
 	 * Now test if the interrupt has hit a range that may be using
 	 * HSPRG1 without having RI=0 (i.e., an HSRR interrupt). The
 	 * problem ranges all run un-relocated. Test real and virt modes
-	 * at the same time by droping the high bit of the nip (virt mode
+	 * at the same time by dropping the high bit of the nip (virt mode
 	 * entry points still have the +0x4000 offset).
 	 */
 	nip &= ~0xc000000000000000ULL;
 	unsigned long ea, msr, msr_mask;
 	bool swap;
-	if (__get_user_inatomic(instr, (unsigned int __user *)regs->nip))
+	if (__get_user(instr, (unsigned int __user *)regs->nip))
 		return;
 	/*
 	_exception(SIGTRAP, regs, TRAP_UNK, 0);
 }
+DEFINE_INTERRUPT_HANDLER_NMI(unknown_nmi_exception)
+{
+	printk("Bad trap at PC: %lx, SR: %lx, vector=%lx\n",
+	       regs->nip, regs->msr, regs->trap);
+	_exception(SIGTRAP, regs, TRAP_UNK, 0);
+	return 0;
+}
 DEFINE_INTERRUPT_HANDLER(instruction_breakpoint_exception)
 {
 	if (notify_die(DIE_IABR_MATCH, "iabr_match", regs, 5,
 	if (!user_mode(regs))
 		return -EINVAL;
-	CHECK_FULL_REGS(regs);
 	if (get_user(instword, (u32 __user *)(regs->nip)))
 		return -EFAULT;
 static int emulate_math(struct pt_regs *regs)
 {
 	int ret;
-	extern int do_mathemu(struct pt_regs *regs);
 	ret = do_mathemu(regs);
 	if (ret >= 0)
 		bad_page_fault(regs, sig);
 }
-DEFINE_INTERRUPT_HANDLER(StackOverflow)
-{
-	pr_crit("Kernel stack overflow in process %s[%d], r1=%lx\n",
-		current->comm, task_pid_nr(current), regs->gpr[1]);
-	debugger(regs);
-	show_regs(regs);
-	panic("kernel stack overflow");
-}
 DEFINE_INTERRUPT_HANDLER(stack_overflow_exception)
 {
 	die("Kernel stack overflow", regs, SIGSEGV);
 	u8 status;
 	bool hv;
-	hv = (TRAP(regs) == 0xf80);
+	hv = (TRAP(regs) == INTERRUPT_H_FAC_UNAVAIL);
 	if (hv)
 		value = mfspr(SPRN_HFSCR);
 	else
  * in the MSR is 0.  This indicates that SRR0/1 are live, and that
  * we therefore lost state by taking this exception.
  */
-void unrecoverable_exception(struct pt_regs *regs)
+void __noreturn unrecoverable_exception(struct pt_regs *regs)
 {
 	pr_emerg("Unrecoverable exception %lx at %lx (msr=%lx)\n",
 		 regs->trap, regs->nip, regs->msr);
 	die("Unrecoverable exception", regs, SIGABRT);
+	/* die() should not return */
+	for (;;)
+		;
 }
 #if defined(CONFIG_BOOKE_WDT) || defined(CONFIG_40x)
 	return;
 }
-DEFINE_INTERRUPT_HANDLER(WatchdogException) /* XXX NMI? async? */
+DEFINE_INTERRUPT_HANDLER_NMI(WatchdogException)
 {
 	printk (KERN_EMERG "PowerPC Book-E Watchdog Exception\n");
 	WatchdogHandler(regs);
+	return 0;
 }
 #endif

arch/powerpc/kernel/uprobes.c+7−0

@@ -41,6 +41,13 @@ int arch_uprobe_analyze_insn(struct arch_uprobe *auprobe, · 40 unmodified lines
41	if (addr & 0x03)	41	if (addr & 0x03)
42	return -EINVAL;	42	return -EINVAL;
43		43
		44	if (cpu_has_feature(CPU_FTR_ARCH_31) &&
		45	ppc_inst_prefixed(auprobe->insn) &&
		46	(addr & 0x3f) == 60) {
		47	pr_info_ratelimited("Cannot register a uprobe on 64 byte unaligned prefixed instruction\n");
		48	return -EINVAL;
		49	}
		50
44	return 0;	51	return 0;
45	}	52	}
46		53

arch/powerpc/kernel/vdso.c+123−15

@@ -18,6 +18,7 @@ · 17 unmodified lines
18	#include <linux/security.h>	18	#include <linux/security.h>
19	#include <linux/memblock.h>	19	#include <linux/memblock.h>
20	#include <linux/syscalls.h>	20	#include <linux/syscalls.h>
		21	#include <linux/time_namespace.h>
21	#include <vdso/datapage.h>	22	#include <vdso/datapage.h>
22		23
23	#include <asm/syscall.h>	24	#include <asm/syscall.h>
@@ -50,15 +51,21 @@ static union { · 26 unmodified lines
50	} vdso_data_store __page_aligned_data;	51	} vdso_data_store __page_aligned_data;
51	struct vdso_arch_data *vdso_data = &vdso_data_store.data;	52	struct vdso_arch_data *vdso_data = &vdso_data_store.data;
52		53
		54	enum vvar_pages {
		55	VVAR_DATA_PAGE_OFFSET,
		56	VVAR_TIMENS_PAGE_OFFSET,
		57	VVAR_NR_PAGES,
		58	};
		59
53	static int vdso_mremap(const struct vm_special_mapping sm, struct vm_area_struct new_vma,	60	static int vdso_mremap(const struct vm_special_mapping sm, struct vm_area_struct new_vma,
54	unsigned long text_size)	61	unsigned long text_size)
55	{	62	{
56	unsigned long new_size = new_vma->vm_end - new_vma->vm_start;	63	unsigned long new_size = new_vma->vm_end - new_vma->vm_start;
57		64
58	if (new_size != text_size + PAGE_SIZE)	65	if (new_size != text_size)
59	return -EINVAL;	66	return -EINVAL;
60		67
61	current->mm->context.vdso = (void __user *)new_vma->vm_start + PAGE_SIZE;	68	current->mm->context.vdso = (void __user *)new_vma->vm_start;
62		69
63	return 0;	70	return 0;
64	}	71	}
@@ -73,6 +80,14 @@ static int vdso64_mremap(const struct vm_special_mapping *sm, struct vm_area_str · 8 unmodified lines
73	return vdso_mremap(sm, new_vma, &vdso64_end - &vdso64_start);	80	return vdso_mremap(sm, new_vma, &vdso64_end - &vdso64_start);
74	}	81	}
75		82
		83	static vm_fault_t vvar_fault(const struct vm_special_mapping *sm,
		84	struct vm_area_struct vma, struct vm_fault vmf);
		85
		86	static struct vm_special_mapping vvar_spec __ro_after_init = {
		87	.name = "[vvar]",
		88	.fault = vvar_fault,
		89	};
		90
76	static struct vm_special_mapping vdso32_spec __ro_after_init = {	91	static struct vm_special_mapping vdso32_spec __ro_after_init = {
77	.name = "[vdso]",	92	.name = "[vdso]",
78	.mremap = vdso32_mremap,	93	.mremap = vdso32_mremap,
@@ -83,17 +98,105 @@ static struct vm_special_mapping vdso64_spec __ro_after_init = { · 4 unmodified lines
83	.mremap = vdso64_mremap,	98	.mremap = vdso64_mremap,
84	};	99	};
85		100
		101	#ifdef CONFIG_TIME_NS
		102	struct vdso_data arch_get_vdso_data(void vvar_page)
		103	{
		104	return ((struct vdso_arch_data *)vvar_page)->data;
		105	}
		106
		107	/*
		108	* The vvar mapping contains data for a specific time namespace, so when a task
		109	* changes namespace we must unmap its vvar data for the old namespace.
		110	* Subsequent faults will map in data for the new namespace.
		111	*
		112	* For more details see timens_setup_vdso_data().
		113	*/
		114	int vdso_join_timens(struct task_struct task, struct time_namespace ns)
		115	{
		116	struct mm_struct *mm = task->mm;
		117	struct vm_area_struct *vma;
		118
		119	mmap_read_lock(mm);
		120
		121	for (vma = mm->mmap; vma; vma = vma->vm_next) {
		122	unsigned long size = vma->vm_end - vma->vm_start;
		123
		124	if (vma_is_special_mapping(vma, &vvar_spec))
		125	zap_page_range(vma, vma->vm_start, size);
		126	}
		127
		128	mmap_read_unlock(mm);
		129	return 0;
		130	}
		131
		132	static struct page find_timens_vvar_page(struct vm_area_struct vma)
		133	{
		134	if (likely(vma->vm_mm == current->mm))
		135	return current->nsproxy->time_ns->vvar_page;
		136
		137	/*
		138	* VM_PFNMAP \| VM_IO protect .fault() handler from being called
		139	* through interfaces like /proc/$pid/mem or
		140	* process_vm_{readv,writev}() as long as there's no .access()
		141	* in special_mapping_vmops.
		142	* For more details check_vma_flags() and __access_remote_vm()
		143	*/
		144	WARN(1, "vvar_page accessed remotely");
		145
		146	return NULL;
		147	}
		148	#else
		149	static struct page find_timens_vvar_page(struct vm_area_struct vma)
		150	{
		151	return NULL;
		152	}
		153	#endif
		154
		155	static vm_fault_t vvar_fault(const struct vm_special_mapping *sm,
		156	struct vm_area_struct vma, struct vm_fault vmf)
		157	{
		158	struct page *timens_page = find_timens_vvar_page(vma);
		159	unsigned long pfn;
		160
		161	switch (vmf->pgoff) {
		162	case VVAR_DATA_PAGE_OFFSET:
		163	if (timens_page)
		164	pfn = page_to_pfn(timens_page);
		165	else
		166	pfn = virt_to_pfn(vdso_data);
		167	break;
		168	#ifdef CONFIG_TIME_NS
		169	case VVAR_TIMENS_PAGE_OFFSET:
		170	/*
		171	* If a task belongs to a time namespace then a namespace
		172	* specific VVAR is mapped with the VVAR_DATA_PAGE_OFFSET and
		173	* the real VVAR page is mapped with the VVAR_TIMENS_PAGE_OFFSET
		174	* offset.
		175	* See also the comment near timens_setup_vdso_data().
		176	*/
		177	if (!timens_page)
		178	return VM_FAULT_SIGBUS;
		179	pfn = virt_to_pfn(vdso_data);
		180	break;
		181	#endif /* CONFIG_TIME_NS */
		182	default:
		183	return VM_FAULT_SIGBUS;
		184	}
		185
		186	return vmf_insert_pfn(vma, vmf->address, pfn);
		187	}
		188
86	/*	189	/*
87	* This is called from binfmt_elf, we create the special vma for the	190	* This is called from binfmt_elf, we create the special vma for the
88	* vDSO and insert it into the mm struct tree	191	* vDSO and insert it into the mm struct tree
89	*/	192	*/
90	static int __arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp)	193	static int __arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp)
91	{	194	{
92	struct mm_struct *mm = current->mm;	195	unsigned long vdso_size, vdso_base, mappings_size;
93	struct vm_special_mapping *vdso_spec;	196	struct vm_special_mapping *vdso_spec;
		197	unsigned long vvar_size = VVAR_NR_PAGES * PAGE_SIZE;
		198	struct mm_struct *mm = current->mm;
94	struct vm_area_struct *vma;	199	struct vm_area_struct *vma;
95	unsigned long vdso_size;
96	unsigned long vdso_base;
97		200
98	if (is_32bit_task()) {	201	if (is_32bit_task()) {
99	vdso_spec = &vdso32_spec;	202	vdso_spec = &vdso32_spec;
@@ -110,8 +213,8 @@ static int __arch_setup_additional_pages(struct linux_binprm *bprm, int uses_int · 10 unmodified lines
110	vdso_base = 0;	213	vdso_base = 0;
111	}	214	}
112		215
113	/* Add a page to the vdso size for the data page */	216	mappings_size = vdso_size + vvar_size;
114	vdso_size += PAGE_SIZE;	217	mappings_size += (VDSO_ALIGNMENT - 1) & PAGE_MASK;
115		218
116	/*	219	/*
117	* pick a base address for the vDSO in process space. We try to put it	220	* pick a base address for the vDSO in process space. We try to put it
@@ -119,9 +222,7 @@ static int __arch_setup_additional_pages(struct linux_binprm *bprm, int uses_int · 1 unmodified line
119	* and end up putting it elsewhere.	222	* and end up putting it elsewhere.
120	* Add enough to the size so that the result can be aligned.	223	* Add enough to the size so that the result can be aligned.
121	*/	224	*/
122	vdso_base = get_unmapped_area(NULL, vdso_base,	225	vdso_base = get_unmapped_area(NULL, vdso_base, mappings_size, 0, 0);
123	vdso_size + ((VDSO_ALIGNMENT - 1) & PAGE_MASK),
124	0, 0);
125	if (IS_ERR_VALUE(vdso_base))	226	if (IS_ERR_VALUE(vdso_base))
126	return vdso_base;	227	return vdso_base;
127		228
@@ -133,7 +234,13 @@ static int __arch_setup_additional_pages(struct linux_binprm *bprm, int uses_int · 5 unmodified lines
133	* install_special_mapping or the perf counter mmap tracking code	234	* install_special_mapping or the perf counter mmap tracking code
134	* will fail to recognise it as a vDSO.	235	* will fail to recognise it as a vDSO.
135	*/	236	*/
136	mm->context.vdso = (void __user *)vdso_base + PAGE_SIZE;	237	mm->context.vdso = (void __user *)vdso_base + vvar_size;
		238
		239	vma = _install_special_mapping(mm, vdso_base, vvar_size,
		240	VM_READ \| VM_MAYREAD \| VM_IO \|
		241	VM_DONTDUMP \| VM_PFNMAP, &vvar_spec);
		242	if (IS_ERR(vma))
		243	return PTR_ERR(vma);
137		244
138	/*	245	/*
139	* our vma flags don't have VM_WRITE so by default, the process isn't	246	* our vma flags don't have VM_WRITE so by default, the process isn't
@@ -145,9 +252,12 @@ static int __arch_setup_additional_pages(struct linux_binprm *bprm, int uses_int · 5 unmodified lines
145	* It's fine to use that for setting breakpoints in the vDSO code	252	* It's fine to use that for setting breakpoints in the vDSO code
146	* pages though.	253	* pages though.
147	*/	254	*/
148	vma = _install_special_mapping(mm, vdso_base, vdso_size,	255	vma = _install_special_mapping(mm, vdso_base + vvar_size, vdso_size,
149	VM_READ \| VM_EXEC \| VM_MAYREAD \|	256	VM_READ \| VM_EXEC \| VM_MAYREAD \|
150	VM_MAYWRITE \| VM_MAYEXEC, vdso_spec);	257	VM_MAYWRITE \| VM_MAYEXEC, vdso_spec);
		258	if (IS_ERR(vma))
		259	do_munmap(mm, vdso_base, vvar_size, NULL);
		260
151	return PTR_ERR_OR_ZERO(vma);	261	return PTR_ERR_OR_ZERO(vma);
152	}	262	}
153		263
@@ -249,10 +359,8 @@ static struct page ** __init vdso_setup_pages(void start, void end) · 95 unmodified lines
249	if (!pagelist)	359	if (!pagelist)
250	panic("%s: Cannot allocate page list for VDSO", __func__);	360	panic("%s: Cannot allocate page list for VDSO", __func__);
251		361
252	pagelist[0] = virt_to_page(vdso_data);
253
254	for (i = 0; i < pages; i++)	362	for (i = 0; i < pages; i++)
255	pagelist[i + 1] = virt_to_page(start + i * PAGE_SIZE);	363	pagelist[i] = virt_to_page(start + i * PAGE_SIZE);
256		364
257	return pagelist;	365	return pagelist;
258	}	366	}

arch/powerpc/kernel/vdso32/vdso32.lds.S+1−1

@@ -17,7 +17,7 @@ ENTRY(_start) · 16 unmodified lines
17		17
18	SECTIONS	18	SECTIONS
19	{	19	{
20	PROVIDE(_vdso_datapage = . - PAGE_SIZE);	20	PROVIDE(_vdso_datapage = . - 2 * PAGE_SIZE);
21	. = SIZEOF_HEADERS;	21	. = SIZEOF_HEADERS;
22		22
23	.hash : { *(.hash) } :text	23	.hash : { *(.hash) } :text

arch/powerpc/kernel/vdso64/vdso64.lds.S+1−1

@@ -17,7 +17,7 @@ ENTRY(_start) · 16 unmodified lines
17		17
18	SECTIONS	18	SECTIONS
19	{	19	{
20	PROVIDE(_vdso_datapage = . - PAGE_SIZE);	20	PROVIDE(_vdso_datapage = . - 2 * PAGE_SIZE);
21	. = SIZEOF_HEADERS;	21	. = SIZEOF_HEADERS;
22		22
23	.hash : { *(.hash) } :text	23	.hash : { *(.hash) } :text

arch/powerpc/kernel/vector.S+0−2

 #ifdef CONFIG_PPC32
 	mfspr	r5,SPRN_SPRG_THREAD		/* current task's THREAD (phys) */
 	oris	r9,r9,MSR_VEC@h
-#ifdef CONFIG_VMAP_STACK
 	tovirt(r5, r5)
-#endif
 #else
 	ld	r4,PACACURRENT(r13)
 	addi	r5,r4,THREAD		/* Get THREAD */

arch/powerpc/kexec/crash.c+2−1

 #include <asm/smp.h>
 #include <asm/setjmp.h>
 #include <asm/debug.h>
+#include <asm/interrupt.h>
 /*
  * The primary CPU waits a while for all secondary CPUs to enter. This is to
 	 * If we came in via system reset, wait a while for the secondary
 	 * CPUs to enter.
 	 */
-	if (TRAP(regs) == 0x100)
+	if (TRAP(regs) == INTERRUPT_SYSTEM_RESET)
 		mdelay(PRIMARY_TIMEOUT);
 	crash_kexec_prepare_cpus(crashing_cpu);

arch/powerpc/kvm/book3s_64_mmu_host.c+2−0

  */
 #include <linux/kvm_host.h>
+#include <linux/pkeys.h>
 #include <asm/kvm_ppc.h>
 #include <asm/kvm_book3s.h>
 	else
 		kvmppc_mmu_flush_icache(pfn);
+	rflags |= pte_to_hpte_pkey_bits(0, HPTE_USE_KERNEL_KEY);
 	rflags = (rflags & ~HPTE_R_WIMG) | orig_pte->wimg;
 	/*

arch/powerpc/kvm/book3s_hv.c+65−20

 		vcpu->arch.dawrx1 = value2;
 		return H_SUCCESS;
 	case H_SET_MODE_RESOURCE_ADDR_TRANS_MODE:
-		/* KVM does not support mflags=2 (AIL=2) */
+		/*
+		 * KVM does not support mflags=2 (AIL=2) and AIL=1 is reserved.
+		 * Keep this in synch with kvmppc_filter_guest_lpcr_hv.
+		 */
 		if (mflags != 0 && mflags != 3)
 			return H_UNSUPPORTED_FLAG_START;
 		return H_TOO_HARD;
 	return 0;
 }
+/*
+ * Enforce limits on guest LPCR values based on hardware availability,
+ * guest configuration, and possibly hypervisor support and security
+ * concerns.
+ */
+unsigned long kvmppc_filter_lpcr_hv(struct kvm *kvm, unsigned long lpcr)
+{
+	/* LPCR_TC only applies to HPT guests */
+	if (kvm_is_radix(kvm))
+		lpcr &= ~LPCR_TC;
+	/* On POWER8 and above, userspace can modify AIL */
+	if (!cpu_has_feature(CPU_FTR_ARCH_207S))
+		lpcr &= ~LPCR_AIL;
+	if ((lpcr & LPCR_AIL) != LPCR_AIL_3)
+		lpcr &= ~LPCR_AIL; /* LPCR[AIL]=1/2 is disallowed */
+	/*
+	 * On POWER9, allow userspace to enable large decrementer for the
+	 * guest, whether or not the host has it enabled.
+	 */
+	if (!cpu_has_feature(CPU_FTR_ARCH_300))
+		lpcr &= ~LPCR_LD;
+	return lpcr;
+}
+static void verify_lpcr(struct kvm *kvm, unsigned long lpcr)
+{
+	if (lpcr != kvmppc_filter_lpcr_hv(kvm, lpcr)) {
+		WARN_ONCE(1, "lpcr 0x%lx differs from filtered 0x%lx\n",
+			  lpcr, kvmppc_filter_lpcr_hv(kvm, lpcr));
+	}
+}
 static void kvmppc_set_lpcr(struct kvm_vcpu *vcpu, u64 new_lpcr,
 		bool preserve_top32)
 {
 	u64 mask;
 	spin_lock(&vc->lock);
+	/*
+	 * Userspace can only modify
+	 * DPFD (default prefetch depth), ILE (interrupt little-endian),
+	 * TC (translation control), AIL (alternate interrupt location),
+	 * LD (large decrementer).
+	 * These are subject to restrictions from kvmppc_filter_lcpr_hv().
+	 */
+	mask = LPCR_DPFD | LPCR_ILE | LPCR_TC | LPCR_AIL | LPCR_LD;
+	/* Broken 32-bit version of LPCR must not clear top bits */
+	if (preserve_top32)
+		mask &= 0xFFFFFFFF;
+	new_lpcr = kvmppc_filter_lpcr_hv(kvm,
+			(vc->lpcr & ~mask) | (new_lpcr & mask));
 	/*
 	 * If ILE (interrupt little-endian) has changed, update the
 	 * MSR_LE bit in the intr_msr for each vcpu in this vcore.
 		}
 	}
-	/*
+	vc->lpcr = new_lpcr;
-	 * Userspace can only modify DPFD (default prefetch depth),
-	 * ILE (interrupt little-endian) and TC (translation control).
-	 * On POWER8 and POWER9 userspace can also modify AIL (alt. interrupt loc.).
-	 */
-	mask = LPCR_DPFD | LPCR_ILE | LPCR_TC;
-	if (cpu_has_feature(CPU_FTR_ARCH_207S))
-		mask |= LPCR_AIL;
-	/*
-	 * On POWER9, allow userspace to enable large decrementer for the
-	 * guest, whether or not the host has it enabled.
-	 */
-	if (cpu_has_feature(CPU_FTR_ARCH_300))
-		mask |= LPCR_LD;
-	/* Broken 32-bit version of LPCR must not clear top bits */
-	if (preserve_top32)
-		mask &= 0xFFFFFFFF;
-	vc->lpcr = (vc->lpcr & ~mask) | (new_lpcr & mask);
 	spin_unlock(&vc->lock);
 }
 	vcpu->arch.dec_expires = dec + tb;
 	vcpu->cpu = -1;
 	vcpu->arch.thread_cpu = -1;
+	/* Save guest CTRL register, set runlatch to 1 */
 	vcpu->arch.ctrl = mfspr(SPRN_CTRLF);
+	if (!(vcpu->arch.ctrl & 1))
+		mtspr(SPRN_CTRLT, vcpu->arch.ctrl | 1);
 	vcpu->arch.iamr = mfspr(SPRN_IAMR);
 	vcpu->arch.pspb = mfspr(SPRN_PSPB);
 	mtspr(SPRN_DSCR, host_dscr);
 	mtspr(SPRN_TIDR, host_tidr);
 	mtspr(SPRN_IAMR, host_iamr);
-	mtspr(SPRN_PSPB, 0);
 	if (host_amr != vcpu->arch.amr)
 		mtspr(SPRN_AMR, host_amr);
 		struct kvmppc_vcore *vc = kvm->arch.vcores[i];
 		if (!vc)
 			continue;
 		spin_lock(&vc->lock);
 		vc->lpcr = (vc->lpcr & ~mask) | lpcr;
+		verify_lpcr(kvm, vc->lpcr);
 		spin_unlock(&vc->lock);
 		if (++cores_done >= kvm->arch.online_vcores)
 			break;
 		kvmppc_setup_partition_table(kvm);
 	}
+	verify_lpcr(kvm, lpcr);
 	kvm->arch.lpcr = lpcr;
 	/* Initialization for future HPT resizes */
 	H_READ,
 	H_PROTECT,
 	H_BULK_REMOVE,
+#ifdef CONFIG_SPAPR_TCE_IOMMU
 	H_GET_TCE,
 	H_PUT_TCE,
+#endif
 	H_SET_DABR,
 	H_SET_XDABR,
 	H_CEDE,

arch/powerpc/kvm/book3s_hv_builtin.c+3−0

@@ -662,6 +662,9 @@ static void kvmppc_end_cede(struct kvm_vcpu *vcpu) · 661 unmodified lines
662		662
663	void kvmppc_set_msr_hv(struct kvm_vcpu *vcpu, u64 msr)	663	void kvmppc_set_msr_hv(struct kvm_vcpu *vcpu, u64 msr)
664	{	664	{
		665	/* Guest must always run with ME enabled, HV disabled. */
		666	msr = (msr \| MSR_ME) & ~MSR_HV;
		667
665	/*	668	/*
666	* Check for illegal transactional state bit combination	669	* Check for illegal transactional state bit combination
667	* and if we find it, force the TS field to a safe state.	670	* and if we find it, force the TS field to a safe state.

arch/powerpc/kvm/book3s_hv_nested.c+30−7

@@ -132,8 +132,33 @@ static void save_hv_return_state(struct kvm_vcpu *vcpu, int trap, · 131 unmodified lines
132	}	132	}
133	}	133	}
134		134
		135	/*
		136	* This can result in some L0 HV register state being leaked to an L1
		137	* hypervisor when the hv_guest_state is copied back to the guest after
		138	* being modified here.
		139	*
		140	* There is no known problem with such a leak, and in many cases these
		141	* register settings could be derived by the guest by observing behaviour
		142	* and timing, interrupts, etc., but it is an issue to consider.
		143	*/
135	static void sanitise_hv_regs(struct kvm_vcpu vcpu, struct hv_guest_state hr)	144	static void sanitise_hv_regs(struct kvm_vcpu vcpu, struct hv_guest_state hr)
136	{	145	{
		146	struct kvmppc_vcore *vc = vcpu->arch.vcore;
		147	u64 mask;
		148
		149	/*
		150	* Don't let L1 change LPCR bits for the L2 except these:
		151	*/
		152	mask = LPCR_DPFD \| LPCR_ILE \| LPCR_TC \| LPCR_AIL \| LPCR_LD \|
		153	LPCR_LPES \| LPCR_MER;
		154
		155	/*
		156	* Additional filtering is required depending on hardware
		157	* and configuration.
		158	*/
		159	hr->lpcr = kvmppc_filter_lpcr_hv(vcpu->kvm,
		160	(vc->lpcr & ~mask) \| (hr->lpcr & mask));
		161
137	/*	162	/*
138	* Don't let L1 enable features for L2 which we've disabled for L1,	163	* Don't let L1 enable features for L2 which we've disabled for L1,
139	* but preserve the interrupt cause field.	164	* but preserve the interrupt cause field.
@@ -271,8 +296,6 @@ long kvmhv_enter_nested_guest(struct kvm_vcpu *vcpu) · 131 unmodified lines
271	u64 hv_ptr, regs_ptr;	296	u64 hv_ptr, regs_ptr;
272	u64 hdec_exp;	297	u64 hdec_exp;
273	s64 delta_purr, delta_spurr, delta_ic, delta_vtb;	298	s64 delta_purr, delta_spurr, delta_ic, delta_vtb;
274	u64 mask;
275	unsigned long lpcr;
276		299
277	if (vcpu->kvm->arch.l1_ptcr == 0)	300	if (vcpu->kvm->arch.l1_ptcr == 0)
278	return H_NOT_AVAILABLE;	301	return H_NOT_AVAILABLE;
@@ -320,10 +343,10 @@ long kvmhv_enter_nested_guest(struct kvm_vcpu *vcpu) · 41 unmodified lines
320	vcpu->arch.nested = l2;	343	vcpu->arch.nested = l2;
321	vcpu->arch.nested_vcpu_id = l2_hv.vcpu_token;	344	vcpu->arch.nested_vcpu_id = l2_hv.vcpu_token;
322	vcpu->arch.regs = l2_regs;	345	vcpu->arch.regs = l2_regs;
323	vcpu->arch.shregs.msr = vcpu->arch.regs.msr;	346
324	mask = LPCR_DPFD \| LPCR_ILE \| LPCR_TC \| LPCR_AIL \| LPCR_LD \|	347	/* Guest must always run with ME enabled, HV disabled. */
325	LPCR_LPES \| LPCR_MER;	348	vcpu->arch.shregs.msr = (vcpu->arch.regs.msr \| MSR_ME) & ~MSR_HV;
326	lpcr = (vc->lpcr & ~mask) \| (l2_hv.lpcr & mask);	349
327	sanitise_hv_regs(vcpu, &l2_hv);	350	sanitise_hv_regs(vcpu, &l2_hv);
328	restore_hv_regs(vcpu, &l2_hv);	351	restore_hv_regs(vcpu, &l2_hv);
329		352
@@ -335,7 +358,7 @@ long kvmhv_enter_nested_guest(struct kvm_vcpu *vcpu) · 5 unmodified lines
335	r = RESUME_HOST;	358	r = RESUME_HOST;
336	break;	359	break;
337	}	360	}
338	r = kvmhv_run_single_vcpu(vcpu, hdec_exp, lpcr);	361	r = kvmhv_run_single_vcpu(vcpu, hdec_exp, l2_hv.lpcr);
339	} while (is_kvmppc_resume_guest(r));	362	} while (is_kvmppc_resume_guest(r));
340		363
341	/* save L2 state for return */	364	/* save L2 state for return */

arch/powerpc/kvm/book3s_hv_rm_mmu.c+1−2

@@ -673,8 +673,7 @@ long kvmppc_h_bulk_remove(struct kvm_vcpu *vcpu) · 672 unmodified lines
673	}	673	}
674		674
675	long kvmppc_h_protect(struct kvm_vcpu *vcpu, unsigned long flags,	675	long kvmppc_h_protect(struct kvm_vcpu *vcpu, unsigned long flags,
676	unsigned long pte_index, unsigned long avpn,	676	unsigned long pte_index, unsigned long avpn)
677	unsigned long va)
678	{	677	{
679	struct kvm *kvm = vcpu->kvm;	678	struct kvm *kvm = vcpu->kvm;
680	__be64 *hpte;	679	__be64 *hpte;

arch/powerpc/lib/Makefile+1−1

@@ -16,7 +16,7 @@ CFLAGS_code-patching.o += -DDISABLE_BRANCH_PROFILING · 15 unmodified lines
16	CFLAGS_feature-fixups.o += -DDISABLE_BRANCH_PROFILING	16	CFLAGS_feature-fixups.o += -DDISABLE_BRANCH_PROFILING
17	endif	17	endif
18		18
19	obj-y += alloc.o code-patching.o feature-fixups.o pmem.o inst.o test_code-patching.o	19	obj-y += alloc.o code-patching.o feature-fixups.o pmem.o test_code-patching.o
20		20
21	ifndef CONFIG_KASAN	21	ifndef CONFIG_KASAN
22	obj-y += string.o memcmp_$(BITS).o	22	obj-y += string.o memcmp_$(BITS).o

arch/powerpc/lib/checksum_wrappers.c+4−11

 {
 	__wsum csum;
-	might_sleep();
+	if (unlikely(!user_read_access_begin(src, len)))
-	if (unlikely(!access_ok(src, len)))
 		return 0;
-	allow_read_from_user(src, len);
 	csum = csum_partial_copy_generic((void __force *)src, dst, len);
-	prevent_read_from_user(src, len);
+	user_read_access_end();
 	return csum;
 }
 EXPORT_SYMBOL(csum_and_copy_from_user);
 {
 	__wsum csum;
-	might_sleep();
+	if (unlikely(!user_write_access_begin(dst, len)))
-	if (unlikely(!access_ok(dst, len)))
 		return 0;
-	allow_write_to_user(dst, len);
 	csum = csum_partial_copy_generic(src, (void __force *)dst, len);
-	prevent_write_to_user(dst, len);
+	user_write_access_end();
 	return csum;
 }
 EXPORT_SYMBOL(csum_and_copy_to_user);

arch/powerpc/lib/code-patching.c+9−4

@@ -21,10 +21,15 @@ · 20 unmodified lines
21	static int __patch_instruction(struct ppc_inst *exec_addr, struct ppc_inst instr,	21	static int __patch_instruction(struct ppc_inst *exec_addr, struct ppc_inst instr,
22	struct ppc_inst *patch_addr)	22	struct ppc_inst *patch_addr)
23	{	23	{
24	if (!ppc_inst_prefixed(instr))	24	if (!ppc_inst_prefixed(instr)) {
25	__put_user_asm_goto(ppc_inst_val(instr), patch_addr, failed, "stw");	25	u32 val = ppc_inst_val(instr);
26	else	26
27	__put_user_asm_goto(ppc_inst_as_u64(instr), patch_addr, failed, "std");	27	__put_kernel_nofault(patch_addr, &val, u32, failed);
		28	} else {
		29	u64 val = ppc_inst_as_ulong(instr);
		30
		31	__put_kernel_nofault(patch_addr, &val, u64, failed);
		32	}
28		33
29	asm ("dcbst 0, %0; sync; icbi 0,%1; sync; isync" :: "r" (patch_addr),	34	asm ("dcbst 0, %0; sync; icbi 0,%1; sync; isync" :: "r" (patch_addr),
30	"r" (exec_addr));	35	"r" (exec_addr));

arch/powerpc/lib/inst.c+0−73

@@ -1,73 +0,0 @@
1	// SPDX-License-Identifier: GPL-2.0-or-later
2	/*
3	* Copyright 2020, IBM Corporation.
4	*/
5
6	#include <linux/uaccess.h>
7	#include <asm/disassemble.h>
8	#include <asm/inst.h>
9	#include <asm/ppc-opcode.h>
10
11	#ifdef CONFIG_PPC64
12	int probe_user_read_inst(struct ppc_inst *inst,
13	struct ppc_inst __user *nip)
14	{
15	unsigned int val, suffix;
16	int err;
17
18	err = copy_from_user_nofault(&val, nip, sizeof(val));
19	if (err)
20	return err;
21	if (get_op(val) == OP_PREFIX) {
22	err = copy_from_user_nofault(&suffix, (void __user *)nip + 4, 4);
23	*inst = ppc_inst_prefix(val, suffix);
24	} else {
25	*inst = ppc_inst(val);
26	}
27	return err;
28	}
29
30	int probe_kernel_read_inst(struct ppc_inst *inst,
31	struct ppc_inst *src)
32	{
33	unsigned int val, suffix;
34	int err;
35
36	err = copy_from_kernel_nofault(&val, src, sizeof(val));
37	if (err)
38	return err;
39	if (get_op(val) == OP_PREFIX) {
40	err = copy_from_kernel_nofault(&suffix, (void *)src + 4, 4);
41	*inst = ppc_inst_prefix(val, suffix);
42	} else {
43	*inst = ppc_inst(val);
44	}
45	return err;
46	}
47	#else /* !CONFIG_PPC64 */
48	int probe_user_read_inst(struct ppc_inst *inst,
49	struct ppc_inst __user *nip)
50	{
51	unsigned int val;
52	int err;
53
54	err = copy_from_user_nofault(&val, nip, sizeof(val));
55	if (!err)
56	*inst = ppc_inst(val);
57
58	return err;
59	}
60
61	int probe_kernel_read_inst(struct ppc_inst *inst,
62	struct ppc_inst *src)
63	{
64	unsigned int val;
65	int err;
66
67	err = copy_from_kernel_nofault(&val, src, sizeof(val));
68	if (!err)
69	*inst = ppc_inst(val);
70
71	return err;
72	}
73	#endif /* CONFIG_PPC64 */

arch/powerpc/lib/sstep.c+0−13

@@ -1401,10 +1401,6 @@ int analyse_instr(struct instruction_op op, const struct pt_regs regs, · 1400 unmodified lines
1401	break;	1401	break;
1402	}	1402	}
1403		1403
1404	/* Following cases refer to regs->gpr[], so we need all regs */
1405	if (!FULL_REGS(regs))
1406	return -1;
1407
1408	rd = (word >> 21) & 0x1f;	1404	rd = (word >> 21) & 0x1f;
1409	ra = (word >> 16) & 0x1f;	1405	ra = (word >> 16) & 0x1f;
1410	rb = (word >> 11) & 0x1f;	1406	rb = (word >> 11) & 0x1f;
@@ -3086,15 +3082,6 @@ NOKPROBE_SYMBOL(analyse_instr); · 1675 unmodified lines
3086	*/	3082	*/
3087	static nokprobe_inline int handle_stack_update(unsigned long ea, struct pt_regs *regs)	3083	static nokprobe_inline int handle_stack_update(unsigned long ea, struct pt_regs *regs)
3088	{	3084	{
3089	#ifdef CONFIG_PPC32
3090	/*
3091	* Check if we will touch kernel stack overflow
3092	*/
3093	if (ea - STACK_INT_FRAME_SIZE <= current->thread.ksp_limit) {
3094	printk(KERN_CRIT "Can't kprobe this since kernel stack would overflow.\n");
3095	return -EINVAL;
3096	}
3097	#endif /* CONFIG_PPC32 */
3098	/*	3085	/*
3099	* Check if we already set since that means we'll	3086	* Check if we already set since that means we'll
3100	* lose the previous value.	3087	* lose the previous value.

arch/powerpc/math-emu/math.c+2−2

@@ -225,7 +225,7 @@ record_exception(struct pt_regs *regs, int eflag) · 224 unmodified lines
225	int	225	int
226	do_mathemu(struct pt_regs *regs)	226	do_mathemu(struct pt_regs *regs)
227	{	227	{
228	void op0 = 0, op1 = 0, op2 = 0, op3 = 0;	228	void op0 = NULL, op1 = NULL, op2 = NULL, op3 = NULL;
229	unsigned long pc = regs->nip;	229	unsigned long pc = regs->nip;
230	signed short sdisp;	230	signed short sdisp;
231	u32 insn = 0;	231	u32 insn = 0;
@@ -234,7 +234,7 @@ do_mathemu(struct pt_regs *regs) · 2 unmodified lines
234	int type = 0;	234	int type = 0;
235	int eflag, trap;	235	int eflag, trap;
236		236
237	if (get_user(insn, (u32 *)pc))	237	if (get_user(insn, (u32 __user *)pc))
238	return -EFAULT;	238	return -EFAULT;
239		239
240	switch (insn >> 26) {	240	switch (insn >> 26) {

arch/powerpc/mm/Makefile+2−1

 obj-y				:= fault.o mem.o pgtable.o mmap.o maccess.o \
 				   init_$(BITS).o pgtable_$(BITS).o \
 				   pgtable-frag.o ioremap.o ioremap_$(BITS).o \
-				   init-common.o mmu_context.o drmem.o
+				   init-common.o mmu_context.o drmem.o \
+				   cacheflush.o
 obj-$(CONFIG_PPC_MMU_NOHASH)	+= nohash/
 obj-$(CONFIG_PPC_BOOK3S_32)	+= book3s32/
 obj-$(CONFIG_PPC_BOOK3S_64)	+= book3s64/

arch/powerpc/mm/book3s32/Makefile+1−0

@@ -9,3 +9,4 @@ endif · 8 unmodified lines
9	obj-y += mmu.o mmu_context.o	9	obj-y += mmu.o mmu_context.o
10	obj-$(CONFIG_PPC_BOOK3S_603) += nohash_low.o	10	obj-$(CONFIG_PPC_BOOK3S_603) += nohash_low.o
11	obj-$(CONFIG_PPC_BOOK3S_604) += hash_low.o tlb.o	11	obj-$(CONFIG_PPC_BOOK3S_604) += hash_low.o tlb.o
		12	obj-$(CONFIG_PPC_KUEP) += kuep.o

arch/powerpc/mm/book3s32/hash_low.S+0−14

@@ -140,10 +140,6 @@ _GLOBAL(hash_page) · 139 unmodified lines
140	bne- .Lretry /* retry if someone got there first */	140	bne- .Lretry /* retry if someone got there first */
141		141
142	mfsrin r3,r4 /* get segment reg for segment */	142	mfsrin r3,r4 /* get segment reg for segment */
143	#ifndef CONFIG_VMAP_STACK
144	mfctr r0
145	stw r0,_CTR(r11)
146	#endif
147	bl create_hpte /* add the hash table entry */	143	bl create_hpte /* add the hash table entry */
148		144
149	#ifdef CONFIG_SMP	145	#ifdef CONFIG_SMP
@@ -152,17 +148,7 @@ _GLOBAL(hash_page) · 2 unmodified lines
152	li r0,0	148	li r0,0
153	stw r0, (mmu_hash_lock - PAGE_OFFSET)@l(r8)	149	stw r0, (mmu_hash_lock - PAGE_OFFSET)@l(r8)
154	#endif	150	#endif
155
156	#ifdef CONFIG_VMAP_STACK
157	b fast_hash_page_return	151	b fast_hash_page_return
158	#else
159	/* Return from the exception */
160	lwz r5,_CTR(r11)
161	mtctr r5
162	lwz r0,GPR0(r11)
163	lwz r8,GPR8(r11)
164	b fast_exception_return
165	#endif
166		152
167	#ifdef CONFIG_SMP	153	#ifdef CONFIG_SMP
168	.Lhash_page_out:	154	.Lhash_page_out:

arch/powerpc/mm/book3s32/kuep.c+40−0

@@ -0,0 +1,40 @@
	1	// SPDX-License-Identifier: GPL-2.0-or-later
	2
	3	#include <asm/kup.h>
	4	#include <asm/reg.h>
	5	#include <asm/task_size_32.h>
	6	#include <asm/mmu.h>
	7
	8	#define KUEP_UPDATE_TWO_USER_SEGMENTS(n) do { \
	9	if (TASK_SIZE > ((n) << 28)) \
	10	mtsr(val1, (n) << 28); \
	11	if (TASK_SIZE > (((n) + 1) << 28)) \
	12	mtsr(val2, ((n) + 1) << 28); \
	13	val1 = (val1 + 0x222) & 0xf0ffffff; \
	14	val2 = (val2 + 0x222) & 0xf0ffffff; \
	15	} while (0)
	16
	17	static __always_inline void kuep_update(u32 val)
	18	{
	19	int val1 = val;
	20	int val2 = (val + 0x111) & 0xf0ffffff;
	21
	22	KUEP_UPDATE_TWO_USER_SEGMENTS(0);
	23	KUEP_UPDATE_TWO_USER_SEGMENTS(2);
	24	KUEP_UPDATE_TWO_USER_SEGMENTS(4);
	25	KUEP_UPDATE_TWO_USER_SEGMENTS(6);
	26	KUEP_UPDATE_TWO_USER_SEGMENTS(8);
	27	KUEP_UPDATE_TWO_USER_SEGMENTS(10);
	28	KUEP_UPDATE_TWO_USER_SEGMENTS(12);
	29	KUEP_UPDATE_TWO_USER_SEGMENTS(14);
	30	}
	31
	32	void kuep_lock(void)
	33	{
	34	kuep_update(mfsr(0) \| SR_NX);
	35	}
	36
	37	void kuep_unlock(void)
	38	{
	39	kuep_update(mfsr(0) & ~SR_NX);
	40	}

arch/powerpc/mm/book3s32/mmu.c+1−8

 	unsigned long border = (unsigned long)__init_begin - PAGE_OFFSET;
-	if (debug_pagealloc_enabled() || __map_without_bats) {
+	if (debug_pagealloc_enabled_or_kfence() || __map_without_bats) {
 		pr_debug_once("Read-Write memory mapped without BATs\n");
 		if (base >= border)
 			return base;
 {
 	if (!IS_ENABLED(CONFIG_MODULES))
 		return false;
-#ifdef MODULES_VADDR
 	if (addr < ALIGN_DOWN(MODULES_VADDR, SZ_256M))
 		return false;
 	if (addr > ALIGN(MODULES_END, SZ_256M) - 1)
 		return false;
-#else
-	if (addr < ALIGN_DOWN(VMALLOC_START, SZ_256M))
-		return false;
-	if (addr > ALIGN(VMALLOC_END, SZ_256M) - 1)
-		return false;
-#endif
 	return true;
 }

arch/powerpc/mm/book3s64/hash_pgtable.c+121−9

@@ -8,6 +8,7 @@ · 7 unmodified lines
8	#include <linux/sched.h>	8	#include <linux/sched.h>
9	#include <linux/mm_types.h>	9	#include <linux/mm_types.h>
10	#include <linux/mm.h>	10	#include <linux/mm.h>
		11	#include <linux/stop_machine.h>
11		12
12	#include <asm/sections.h>	13	#include <asm/sections.h>
13	#include <asm/mmu.h>	14	#include <asm/mmu.h>
@@ -400,10 +401,103 @@ EXPORT_SYMBOL_GPL(hash__has_transparent_hugepage); · 386 unmodified lines
400	#endif /* CONFIG_TRANSPARENT_HUGEPAGE */	401	#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
401		402
402	#ifdef CONFIG_STRICT_KERNEL_RWX	403	#ifdef CONFIG_STRICT_KERNEL_RWX
		404
		405	struct change_memory_parms {
		406	unsigned long start, end, newpp;
		407	unsigned int step, nr_cpus, master_cpu;
		408	atomic_t cpu_counter;
		409	};
		410
		411	// We'd rather this was on the stack but it has to be in the RMO
		412	static struct change_memory_parms chmem_parms;
		413
		414	// And therefore we need a lock to protect it from concurrent use
		415	static DEFINE_MUTEX(chmem_lock);
		416
		417	static void change_memory_range(unsigned long start, unsigned long end,
		418	unsigned int step, unsigned long newpp)
		419	{
		420	unsigned long idx;
		421
		422	pr_debug("Changing page protection on range 0x%lx-0x%lx, to 0x%lx, step 0x%x\n",
		423	start, end, newpp, step);
		424
		425	for (idx = start; idx < end; idx += step)
		426	/* Not sure if we can do much with the return value */
		427	mmu_hash_ops.hpte_updateboltedpp(newpp, idx, mmu_linear_psize,
		428	mmu_kernel_ssize);
		429	}
		430
		431	static int notrace chmem_secondary_loop(struct change_memory_parms *parms)
		432	{
		433	unsigned long msr, tmp, flags;
		434	int *p;
		435
		436	p = &parms->cpu_counter.counter;
		437
		438	local_irq_save(flags);
		439	hard_irq_disable();
		440
		441	asm volatile (
		442	// Switch to real mode and leave interrupts off
		443	"mfmsr %[msr] ;"
		444	"li %[tmp], %[MSR_IR_DR] ;"
		445	"andc %[tmp], %[msr], %[tmp] ;"
		446	"mtmsrd %[tmp] ;"
		447
		448	// Tell the master we are in real mode
		449	"1: "
		450	"lwarx %[tmp], 0, %[p] ;"
		451	"addic %[tmp], %[tmp], -1 ;"
		452	"stwcx. %[tmp], 0, %[p] ;"
		453	"bne- 1b ;"
		454
		455	// Spin until the counter goes to zero
		456	"2: ;"
		457	"lwz %[tmp], 0(%[p]) ;"
		458	"cmpwi %[tmp], 0 ;"
		459	"bne- 2b ;"
		460
		461	// Switch back to virtual mode
		462	"mtmsrd %[msr] ;"
		463
		464	: // outputs
		465	[msr] "=&r" (msr), [tmp] "=&b" (tmp), "+m" (*p)
		466	: // inputs
		467	[p] "b" (p), [MSR_IR_DR] "i" (MSR_IR \| MSR_DR)
		468	: // clobbers
		469	"cc", "xer"
		470	);
		471
		472	local_irq_restore(flags);
		473
		474	return 0;
		475	}
		476
		477	static int change_memory_range_fn(void *data)
		478	{
		479	struct change_memory_parms *parms = data;
		480
		481	if (parms->master_cpu != smp_processor_id())
		482	return chmem_secondary_loop(parms);
		483
		484	// Wait for all but one CPU (this one) to call-in
		485	while (atomic_read(&parms->cpu_counter) > 1)
		486	barrier();
		487
		488	change_memory_range(parms->start, parms->end, parms->step, parms->newpp);
		489
		490	mb();
		491
		492	// Signal the other CPUs that we're done
		493	atomic_dec(&parms->cpu_counter);
		494
		495	return 0;
		496	}
		497
403	static bool hash__change_memory_range(unsigned long start, unsigned long end,	498	static bool hash__change_memory_range(unsigned long start, unsigned long end,
404	unsigned long newpp)	499	unsigned long newpp)
405	{	500	{
406	unsigned long idx;
407	unsigned int step, shift;	501	unsigned int step, shift;
408		502
409	shift = mmu_psize_defs[mmu_linear_psize].shift;	503	shift = mmu_psize_defs[mmu_linear_psize].shift;
@@ -415,25 +509,43 @@ static bool hash__change_memory_range(unsigned long start, unsigned long end, · 5 unmodified lines
415	if (start >= end)	509	if (start >= end)
416	return false;	510	return false;
417		511
418	pr_debug("Changing page protection on range 0x%lx-0x%lx, to 0x%lx, step 0x%x\n",	512	if (firmware_has_feature(FW_FEATURE_LPAR)) {
419	start, end, newpp, step);	513	mutex_lock(&chmem_lock);
420		514
421	for (idx = start; idx < end; idx += step)	515	chmem_parms.start = start;
422	/* Not sure if we can do much with the return value */	516	chmem_parms.end = end;
423	mmu_hash_ops.hpte_updateboltedpp(newpp, idx, mmu_linear_psize,	517	chmem_parms.step = step;
424	mmu_kernel_ssize);	518	chmem_parms.newpp = newpp;
		519	chmem_parms.master_cpu = smp_processor_id();
		520
		521	cpus_read_lock();
		522
		523	atomic_set(&chmem_parms.cpu_counter, num_online_cpus());
		524
		525	// Ensure state is consistent before we call the other CPUs
		526	mb();
		527
		528	stop_machine_cpuslocked(change_memory_range_fn, &chmem_parms,
		529	cpu_online_mask);
		530
		531	cpus_read_unlock();
		532	mutex_unlock(&chmem_lock);
		533	} else
		534	change_memory_range(start, end, step, newpp);
425		535
426	return true;	536	return true;
427	}	537	}
428		538
429	void hash__mark_rodata_ro(void)	539	void hash__mark_rodata_ro(void)
430	{	540	{
431	unsigned long start, end;	541	unsigned long start, end, pp;
432		542
433	start = (unsigned long)_stext;	543	start = (unsigned long)_stext;
434	end = (unsigned long)__init_begin;	544	end = (unsigned long)__init_begin;
435		545
436	WARN_ON(!hash__change_memory_range(start, end, PP_RXXX));	546	pp = htab_convert_pte_flags(pgprot_val(PAGE_KERNEL_ROX), HPTE_USE_KERNEL_KEY);
		547
		548	WARN_ON(!hash__change_memory_range(start, end, pp));
437	}	549	}
438		550
439	void hash__mark_initmem_nx(void)	551	void hash__mark_initmem_nx(void)

arch/powerpc/mm/book3s64/hash_utils.c+22−13

 int htab_remove_mapping(unsigned long vstart, unsigned long vend,
 		      int psize, int ssize)
 {
-	unsigned long vaddr;
+	unsigned long vaddr, time_limit;
 	unsigned int step, shift;
 	int rc;
 	int ret = 0;
 	/* Unmap the full range specificied */
 	vaddr = ALIGN_DOWN(vstart, step);
+	time_limit = jiffies + HZ;
 	for (;vaddr < vend; vaddr += step) {
 		rc = mmu_hash_ops.hpte_removebolted(vaddr, psize, ssize);
+		/*
+		 * For large number of mappings introduce a cond_resched()
+		 * to prevent softlockup warnings.
+		 */
+		if (time_after(jiffies, time_limit)) {
+			cond_resched();
+			time_limit = jiffies + HZ;
+		}
 		if (rc == -ENOENT) {
 			ret = -ENOENT;
 			continue;
 	/* page is dirty */
 	if (!test_bit(PG_dcache_clean, &page->flags) && !PageReserved(page)) {
-		if (trap == 0x400) {
+		if (trap == INTERRUPT_INST_STORAGE) {
 			flush_dcache_icache_page(page);
 			set_bit(PG_dcache_clean, &page->flags);
 		} else
 	if (user_mode(regs) || (region_id == USER_REGION_ID))
 		access &= ~_PAGE_PRIVILEGED;
-	if (regs->trap == 0x400)
+	if (TRAP(regs) == INTERRUPT_INST_STORAGE)
 		access |= _PAGE_EXEC;
-	err = hash_page_mm(mm, ea, access, regs->trap, flags);
+	err = hash_page_mm(mm, ea, access, TRAP(regs), flags);
 	if (unlikely(err < 0)) {
 		// failed to instert a hash PTE due to an hypervisor error
 		if (user_mode(regs)) {
 DEFINE_INTERRUPT_HANDLER_RAW(do_hash_fault)
 {
 	unsigned long dsisr = regs->dsisr;
-	long err;
-	if (unlikely(dsisr & (DSISR_BAD_FAULT_64S | DSISR_KEYFAULT)))
+	if (unlikely(dsisr & (DSISR_BAD_FAULT_64S | DSISR_KEYFAULT))) {
-		goto page_fault;
+		hash__do_page_fault(regs);
+		return 0;
+	}
 	/*
 	 * If we are in an "NMI" (e.g., an interrupt when soft-disabled), then
 		return 0;
 	}
-	err = __do_hash_fault(regs);
+	if (__do_hash_fault(regs))
-	if (err) {
+		hash__do_page_fault(regs);
-page_fault:
-		err = hash__do_page_fault(regs);
-	}
-	return err;
+	return 0;
 }
 #ifdef CONFIG_PPC_MM_SLICES

arch/powerpc/mm/book3s64/mmu_context.c+1−1

@@ -119,7 +119,7 @@ static int hash__init_new_context(struct mm_struct *mm) · 118 unmodified lines
119	/* This is fork. Copy hash_context details from current->mm */	119	/* This is fork. Copy hash_context details from current->mm */
120	memcpy(mm->context.hash_context, current->mm->context.hash_context, sizeof(struct hash_mm_context));	120	memcpy(mm->context.hash_context, current->mm->context.hash_context, sizeof(struct hash_mm_context));
121	#ifdef CONFIG_PPC_SUBPAGE_PROT	121	#ifdef CONFIG_PPC_SUBPAGE_PROT
122	/* inherit subpage prot detalis if we have one. */	122	/* inherit subpage prot details if we have one. */
123	if (current->mm->context.hash_context->spt) {	123	if (current->mm->context.hash_context->spt) {
124	mm->context.hash_context->spt = kmalloc(sizeof(struct subpage_prot_table),	124	mm->context.hash_context->spt = kmalloc(sizeof(struct subpage_prot_table),
125	GFP_KERNEL);	125	GFP_KERNEL);

arch/powerpc/mm/book3s64/pkeys.c+5−15

 }
 #endif
-static inline void update_current_thread_amr(u64 value)
-{
-	current->thread.regs->amr = value;
-}
-static inline void update_current_thread_iamr(u64 value)
-{
-	if (!likely(pkey_execute_disable_supported))
-		return;
-	current->thread.regs->iamr = value;
-}
 #ifdef CONFIG_PPC_MEM_KEYS
 void pkey_mm_init(struct mm_struct *mm)
 {
 	u64 new_amr_bits = (((u64)init_bits & 0x3UL) << pkeyshift(pkey));
 	u64 old_amr = current_thread_amr() & ~((u64)(0x3ul) << pkeyshift(pkey));
-	update_current_thread_amr(old_amr | new_amr_bits);
+	current->thread.regs->amr = old_amr | new_amr_bits;
 }
 static inline void init_iamr(int pkey, u8 init_bits)
 	u64 new_iamr_bits = (((u64)init_bits & 0x1UL) << pkeyshift(pkey));
 	u64 old_iamr = current_thread_iamr() & ~((u64)(0x1ul) << pkeyshift(pkey));
-	update_current_thread_iamr(old_iamr | new_iamr_bits);
+	if (!likely(pkey_execute_disable_supported))
+		return;
+	current->thread.regs->iamr = old_iamr | new_iamr_bits;
 }
 /*

arch/powerpc/mm/book3s64/radix_pgtable.c+5−5

@@ -108,7 +108,7 @@ static int early_map_kernel_page(unsigned long ea, unsigned long pa, · 107 unmodified lines
108		108
109	set_the_pte:	109	set_the_pte:
110	set_pte_at(&init_mm, ea, ptep, pfn_pte(pfn, flags));	110	set_pte_at(&init_mm, ea, ptep, pfn_pte(pfn, flags));
111	smp_wmb();	111	asm volatile("ptesync": : :"memory");
112	return 0;	112	return 0;
113	}	113	}
114		114
@@ -168,7 +168,7 @@ static int __map_kernel_page(unsigned long ea, unsigned long pa, · 53 unmodified lines
168		168
169	set_the_pte:	169	set_the_pte:
170	set_pte_at(&init_mm, ea, ptep, pfn_pte(pfn, flags));	170	set_pte_at(&init_mm, ea, ptep, pfn_pte(pfn, flags));
171	smp_wmb();	171	asm volatile("ptesync": : :"memory");
172	return 0;	172	return 0;
173	}	173	}
174		174
@@ -180,8 +180,8 @@ int radix__map_kernel_page(unsigned long ea, unsigned long pa, · 5 unmodified lines
180	}	180	}
181		181
182	#ifdef CONFIG_STRICT_KERNEL_RWX	182	#ifdef CONFIG_STRICT_KERNEL_RWX
183	void radix__change_memory_range(unsigned long start, unsigned long end,	183	static void radix__change_memory_range(unsigned long start, unsigned long end,
184	unsigned long clear)	184	unsigned long clear)
185	{	185	{
186	unsigned long idx;	186	unsigned long idx;
187	pgd_t *pgdp;	187	pgd_t *pgdp;
@@ -1058,7 +1058,7 @@ void radix__ptep_set_access_flags(struct vm_area_struct vma, pte_t ptep, · 870 unmodified lines
1058	* Book3S does not require a TLB flush when relaxing access	1058	* Book3S does not require a TLB flush when relaxing access
1059	* restrictions when the address space is not attached to a	1059	* restrictions when the address space is not attached to a
1060	* NMMU, because the core MMU will reload the pte after taking	1060	* NMMU, because the core MMU will reload the pte after taking
1061	* an access fault, which is defined by the architectue.	1061	* an access fault, which is defined by the architecture.
1062	*/	1062	*/
1063	}	1063	}
1064	/* See ptesync comment in radix__set_pte_at */	1064	/* See ptesync comment in radix__set_pte_at */

arch/powerpc/mm/cacheflush.c+234−0

@@ -0,0 +1,234 @@
	1	// SPDX-License-Identifier: GPL-2.0-or-later
	2
	3	#include <linux/highmem.h>
	4	#include <linux/kprobes.h>
	5
	6	/**
	7	* flush_coherent_icache() - if a CPU has a coherent icache, flush it
	8	* Return true if the cache was flushed, false otherwise
	9	*/
	10	static inline bool flush_coherent_icache(void)
	11	{
	12	/*
	13	* For a snooping icache, we still need a dummy icbi to purge all the
	14	* prefetched instructions from the ifetch buffers. We also need a sync
	15	* before the icbi to order the the actual stores to memory that might
	16	* have modified instructions with the icbi.
	17	*/
	18	if (cpu_has_feature(CPU_FTR_COHERENT_ICACHE)) {
	19	mb(); /* sync */
	20	icbi((void *)PAGE_OFFSET);
	21	mb(); /* sync */
	22	isync();
	23	return true;
	24	}
	25
	26	return false;
	27	}
	28
	29	/**
	30	* invalidate_icache_range() - Flush the icache by issuing icbi across an address range
	31	* @start: the start address
	32	* @stop: the stop address (exclusive)
	33	*/
	34	static void invalidate_icache_range(unsigned long start, unsigned long stop)
	35	{
	36	unsigned long shift = l1_icache_shift();
	37	unsigned long bytes = l1_icache_bytes();
	38	char addr = (char )(start & ~(bytes - 1));
	39	unsigned long size = stop - (unsigned long)addr + (bytes - 1);
	40	unsigned long i;
	41
	42	for (i = 0; i < size >> shift; i++, addr += bytes)
	43	icbi(addr);
	44
	45	mb(); /* sync */
	46	isync();
	47	}
	48
	49	/**
	50	* flush_icache_range: Write any modified data cache blocks out to memory
	51	* and invalidate the corresponding blocks in the instruction cache
	52	*
	53	* Generic code will call this after writing memory, before executing from it.
	54	*
	55	* @start: the start address
	56	* @stop: the stop address (exclusive)
	57	*/
	58	void flush_icache_range(unsigned long start, unsigned long stop)
	59	{
	60	if (flush_coherent_icache())
	61	return;
	62
	63	clean_dcache_range(start, stop);
	64
	65	if (IS_ENABLED(CONFIG_44x)) {
	66	/*
	67	* Flash invalidate on 44x because we are passed kmapped
	68	* addresses and this doesn't work for userspace pages due to
	69	* the virtually tagged icache.
	70	*/
	71	iccci((void *)start);
	72	mb(); /* sync */
	73	isync();
	74	} else
	75	invalidate_icache_range(start, stop);
	76	}
	77	EXPORT_SYMBOL(flush_icache_range);
	78
	79	#ifdef CONFIG_HIGHMEM
	80	/**
	81	* flush_dcache_icache_phys() - Flush a page by it's physical address
	82	* @physaddr: the physical address of the page
	83	*/
	84	static void flush_dcache_icache_phys(unsigned long physaddr)
	85	{
	86	unsigned long bytes = l1_dcache_bytes();
	87	unsigned long nb = PAGE_SIZE / bytes;
	88	unsigned long addr = physaddr & PAGE_MASK;
	89	unsigned long msr, msr0;
	90	unsigned long loop1 = addr, loop2 = addr;
	91
	92	msr0 = mfmsr();
	93	msr = msr0 & ~MSR_DR;
	94	/*
	95	* This must remain as ASM to prevent potential memory accesses
	96	* while the data MMU is disabled
	97	*/
	98	asm volatile(
	99	" mtctr %2;\n"
	100	" mtmsr %3;\n"
	101	" isync;\n"
	102	"0: dcbst 0, %0;\n"
	103	" addi %0, %0, %4;\n"
	104	" bdnz 0b;\n"
	105	" sync;\n"
	106	" mtctr %2;\n"
	107	"1: icbi 0, %1;\n"
	108	" addi %1, %1, %4;\n"
	109	" bdnz 1b;\n"
	110	" sync;\n"
	111	" mtmsr %5;\n"
	112	" isync;\n"
	113	: "+&r" (loop1), "+&r" (loop2)
	114	: "r" (nb), "r" (msr), "i" (bytes), "r" (msr0)
	115	: "ctr", "memory");
	116	}
	117	NOKPROBE_SYMBOL(flush_dcache_icache_phys)
	118	#else
	119	static void flush_dcache_icache_phys(unsigned long physaddr)
	120	{
	121	}
	122	#endif
	123
	124	/**
	125	* __flush_dcache_icache(): Flush a particular page from the data cache to RAM.
	126	* Note: this is necessary because the instruction cache does not
	127	* snoop from the data cache.
	128	*
	129	* @p: the address of the page to flush
	130	*/
	131	static void __flush_dcache_icache(void *p)
	132	{
	133	unsigned long addr = (unsigned long)p & PAGE_MASK;
	134
	135	clean_dcache_range(addr, addr + PAGE_SIZE);
	136
	137	/*
	138	* We don't flush the icache on 44x. Those have a virtual icache and we
	139	* don't have access to the virtual address here (it's not the page
	140	* vaddr but where it's mapped in user space). The flushing of the
	141	* icache on these is handled elsewhere, when a change in the address
	142	* space occurs, before returning to user space.
	143	*/
	144
	145	if (mmu_has_feature(MMU_FTR_TYPE_44x))
	146	return;
	147
	148	invalidate_icache_range(addr, addr + PAGE_SIZE);
	149	}
	150
	151	static void flush_dcache_icache_hugepage(struct page *page)
	152	{
	153	int i;
	154	int nr = compound_nr(page);
	155
	156	if (!PageHighMem(page)) {
	157	for (i = 0; i < nr; i++)
	158	__flush_dcache_icache(lowmem_page_address(page + i));
	159	} else {
	160	for (i = 0; i < nr; i++) {
	161	void *start = kmap_local_page(page + i);
	162
	163	__flush_dcache_icache(start);
	164	kunmap_local(start);
	165	}
	166	}
	167	}
	168
	169	void flush_dcache_icache_page(struct page *page)
	170	{
	171	if (flush_coherent_icache())
	172	return;
	173
	174	if (PageCompound(page))
	175	return flush_dcache_icache_hugepage(page);
	176
	177	if (!PageHighMem(page)) {
	178	__flush_dcache_icache(lowmem_page_address(page));
	179	} else if (IS_ENABLED(CONFIG_BOOKE) \|\| sizeof(phys_addr_t) > sizeof(void *)) {
	180	void *start = kmap_local_page(page);
	181
	182	__flush_dcache_icache(start);
	183	kunmap_local(start);
	184	} else {
	185	flush_dcache_icache_phys(page_to_phys(page));
	186	}
	187	}
	188	EXPORT_SYMBOL(flush_dcache_icache_page);
	189
	190	void clear_user_page(void page, unsigned long vaddr, struct page pg)
	191	{
	192	clear_page(page);
	193
	194	/*
	195	* We shouldn't have to do this, but some versions of glibc
	196	* require it (ld.so assumes zero filled pages are icache clean)
	197	* - Anton
	198	*/
	199	flush_dcache_page(pg);
	200	}
	201	EXPORT_SYMBOL(clear_user_page);
	202
	203	void copy_user_page(void vto, void vfrom, unsigned long vaddr,
	204	struct page *pg)
	205	{
	206	copy_page(vto, vfrom);
	207
	208	/*
	209	* We should be able to use the following optimisation, however
	210	* there are two problems.
	211	* Firstly a bug in some versions of binutils meant PLT sections
	212	* were not marked executable.
	213	* Secondly the first word in the GOT section is blrl, used
	214	* to establish the GOT address. Until recently the GOT was
	215	* not marked executable.
	216	* - Anton
	217	*/
	218	#if 0
	219	if (!vma->vm_file && ((vma->vm_flags & VM_EXEC) == 0))
	220	return;
	221	#endif
	222
	223	flush_dcache_page(pg);
	224	}
	225
	226	void flush_icache_user_page(struct vm_area_struct vma, struct page page,
	227	unsigned long addr, int len)
	228	{
	229	void *maddr;
	230
	231	maddr = kmap_local_page(page) + (addr & ~PAGE_MASK);
	232	flush_icache_range((unsigned long)maddr, (unsigned long)maddr + len);
	233	kunmap_local(maddr);
	234	}

arch/powerpc/mm/fault.c+23−37

 #include <linux/context_tracking.h>
 #include <linux/hugetlb.h>
 #include <linux/uaccess.h>
+#include <linux/kfence.h>
+#include <linux/pkeys.h>
 #include <asm/firmware.h>
 #include <asm/interrupt.h>
 	return __bad_area(regs, address, SEGV_MAPERR);
 }
-#ifdef CONFIG_PPC_MEM_KEYS
 static noinline int bad_access_pkey(struct pt_regs *regs, unsigned long address,
 				    struct vm_area_struct *vma)
 {
 	return 0;
 }
-#endif
 static noinline int bad_access(struct pt_regs *regs, unsigned long address)
 {
 static bool bad_kernel_fault(struct pt_regs *regs, unsigned long error_code,
 			     unsigned long address, bool is_write)
 {
-	int is_exec = TRAP(regs) == 0x400;
+	int is_exec = TRAP(regs) == INTERRUPT_INST_STORAGE;
 	/* NX faults set DSISR_PROTFAULT on the 8xx, DSISR_NOEXEC_OR_G on others */
 	if (is_exec && (error_code & (DSISR_NOEXEC_OR_G | DSISR_KEYFAULT |
 	return false;
 }
-#ifdef CONFIG_PPC_MEM_KEYS
 static bool access_pkey_error(bool is_write, bool is_exec, bool is_pkey,
 			      struct vm_area_struct *vma)
 {
 	return false;
 }
-#endif
 static bool access_error(bool is_write, bool is_exec, struct vm_area_struct *vma)
 {
 	struct vm_area_struct * vma;
 	struct mm_struct *mm = current->mm;
 	unsigned int flags = FAULT_FLAG_DEFAULT;
- 	int is_exec = TRAP(regs) == 0x400;
+	int is_exec = TRAP(regs) == INTERRUPT_INST_STORAGE;
 	int is_user = user_mode(regs);
 	int is_write = page_fault_is_write(error_code);
 	vm_fault_t fault, major = 0;
 	 * take a page fault to a kernel address or a page fault to a user
 	 * address outside of dedicated places
 	 */
-	if (unlikely(!is_user && bad_kernel_fault(regs, error_code, address, is_write)))
+	if (unlikely(!is_user && bad_kernel_fault(regs, error_code, address, is_write))) {
+		if (kfence_handle_page_fault(address, is_write, regs))
+			return 0;
 		return SIGSEGV;
+	}
 	/*
 	 * If we're in an interrupt, have no user context or are running
 			return bad_area(regs, address);
 	}
-#ifdef CONFIG_PPC_MEM_KEYS
 	if (unlikely(access_pkey_error(is_write, is_exec,
 				       (error_code & DSISR_KEYFAULT), vma)))
 		return bad_access_pkey(regs, address, vma);
-#endif /* CONFIG_PPC_MEM_KEYS */
 	if (unlikely(access_error(is_write, is_exec, vma)))
 		return bad_access(regs, address);
 }
 NOKPROBE_SYMBOL(___do_page_fault);
-static long __do_page_fault(struct pt_regs *regs)
+static __always_inline void __do_page_fault(struct pt_regs *regs)
 {
-	const struct exception_table_entry *entry;
 	long err;
 	err = ___do_page_fault(regs, regs->dar, regs->dsisr);
-	if (likely(!err))
+	if (unlikely(err))
-		return err;
+		bad_page_fault(regs, err);
-	entry = search_exception_tables(regs->nip);
-	if (likely(entry)) {
-		instruction_pointer_set(regs, extable_fixup(entry));
-		return 0;
-	} else if (IS_ENABLED(CONFIG_PPC_BOOK3S_64)) {
-		__bad_page_fault(regs, err);
-		return 0;
-	} else {
-		/* 32 and 64e handle the bad page fault in asm */
-		return err;
-	}
 }
-NOKPROBE_SYMBOL(__do_page_fault);
-DEFINE_INTERRUPT_HANDLER_RET(do_page_fault)
+DEFINE_INTERRUPT_HANDLER(do_page_fault)
 {
-	return __do_page_fault(regs);
+	__do_page_fault(regs);
 }
 #ifdef CONFIG_PPC_BOOK3S_64
 /* Same as do_page_fault but interrupt entry has already run in do_hash_fault */
-long hash__do_page_fault(struct pt_regs *regs)
+void hash__do_page_fault(struct pt_regs *regs)
 {
-	return __do_page_fault(regs);
+	__do_page_fault(regs);
 }
 NOKPROBE_SYMBOL(hash__do_page_fault);
 #endif
  * It is called from the DSI and ISI handlers in head.S and from some
  * of the procedures in traps.c.
  */
-void __bad_page_fault(struct pt_regs *regs, int sig)
+static void __bad_page_fault(struct pt_regs *regs, int sig)
 {
 	int is_write = page_fault_is_write(regs->dsisr);
 	/* kernel has accessed a bad area */
 	switch (TRAP(regs)) {
-	case 0x300:
+	case INTERRUPT_DATA_STORAGE:
-	case 0x380:
+	case INTERRUPT_DATA_SEGMENT:
-	case 0xe00:
+	case INTERRUPT_H_DATA_STORAGE:
 		pr_alert("BUG: %s on %s at 0x%08lx\n",
 			 regs->dar < PAGE_SIZE ? "Kernel NULL pointer dereference" :
 			 "Unable to handle kernel data access",
 			 is_write ? "write" : "read", regs->dar);
 		break;
-	case 0x400:
+	case INTERRUPT_INST_STORAGE:
-	case 0x480:
+	case INTERRUPT_INST_SEGMENT:
 		pr_alert("BUG: Unable to handle kernel instruction fetch%s",
 			 regs->nip < PAGE_SIZE ? " (NULL pointer?)\n" : "\n");
 		break;
-	case 0x600:
+	case INTERRUPT_ALIGNMENT:
 		pr_alert("BUG: Unable to handle kernel unaligned access at 0x%08lx\n",
 			 regs->dar);
 		break;

arch/powerpc/mm/init_32.c+3−0

@@ -97,6 +97,9 @@ static void __init MMU_setup(void) · 96 unmodified lines
97	if (IS_ENABLED(CONFIG_PPC_8xx))	97	if (IS_ENABLED(CONFIG_PPC_8xx))
98	return;	98	return;
99		99
		100	if (IS_ENABLED(CONFIG_KFENCE))
		101	__map_without_ltlbs = 1;
		102
100	if (debug_pagealloc_enabled())	103	if (debug_pagealloc_enabled())
101	__map_without_ltlbs = 1;	104	__map_without_ltlbs = 1;
102		105

arch/powerpc/mm/maccess.c+21−0

@@ -3,7 +3,28 @@ · 2 unmodified lines
3	#include <linux/uaccess.h>	3	#include <linux/uaccess.h>
4	#include <linux/kernel.h>	4	#include <linux/kernel.h>
5		5
		6	#include <asm/disassemble.h>
		7	#include <asm/inst.h>
		8	#include <asm/ppc-opcode.h>
		9
6	bool copy_from_kernel_nofault_allowed(const void *unsafe_src, size_t size)	10	bool copy_from_kernel_nofault_allowed(const void *unsafe_src, size_t size)
7	{	11	{
8	return is_kernel_addr((unsigned long)unsafe_src);	12	return is_kernel_addr((unsigned long)unsafe_src);
9	}	13	}
		14
		15	int copy_inst_from_kernel_nofault(struct ppc_inst inst, struct ppc_inst src)
		16	{
		17	unsigned int val, suffix;
		18	int err;
		19
		20	err = copy_from_kernel_nofault(&val, src, sizeof(val));
		21	if (err)
		22	return err;
		23	if (IS_ENABLED(CONFIG_PPC64) && get_op(val) == OP_PREFIX) {
		24	err = copy_from_kernel_nofault(&suffix, (void *)src + 4, 4);
		25	*inst = ppc_inst_prefix(val, suffix);
		26	} else {
		27	*inst = ppc_inst(val);
		28	}
		29	return err;
		30	}

arch/powerpc/mm/mem.c+1−282

@@ -12,49 +12,18 @@ · 11 unmodified lines
12	* Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds	12	* Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
13	*/	13	*/
14		14
15	#include <linux/export.h>
16	#include <linux/sched.h>
17	#include <linux/kernel.h>
18	#include <linux/errno.h>
19	#include <linux/string.h>
20	#include <linux/gfp.h>
21	#include <linux/types.h>
22	#include <linux/mm.h>
23	#include <linux/stddef.h>
24	#include <linux/init.h>
25	#include <linux/memblock.h>	15	#include <linux/memblock.h>
26	#include <linux/highmem.h>	16	#include <linux/highmem.h>
27	#include <linux/initrd.h>
28	#include <linux/pagemap.h>
29	#include <linux/suspend.h>	17	#include <linux/suspend.h>
30	#include <linux/hugetlb.h>
31	#include <linux/slab.h>
32	#include <linux/vmalloc.h>
33	#include <linux/memremap.h>
34	#include <linux/dma-direct.h>	18	#include <linux/dma-direct.h>
35	#include <linux/kprobes.h>
36		19
37	#include <asm/prom.h>
38	#include <asm/io.h>
39	#include <asm/mmu_context.h>
40	#include <asm/mmu.h>
41	#include <asm/smp.h>
42	#include <asm/machdep.h>	20	#include <asm/machdep.h>
43	#include <asm/btext.h>
44	#include <asm/tlb.h>
45	#include <asm/sections.h>
46	#include <asm/sparsemem.h>
47	#include <asm/vdso.h>
48	#include <asm/fixmap.h>
49	#include <asm/swiotlb.h>
50	#include <asm/rtas.h>	21	#include <asm/rtas.h>
51	#include <asm/kasan.h>	22	#include <asm/kasan.h>
52	#include <asm/svm.h>	23	#include <asm/svm.h>
53	#include <asm/mmzone.h>
54		24
55	#include <mm/mmu_decl.h>	25	#include <mm/mmu_decl.h>
56		26
57	static DEFINE_MUTEX(linear_mapping_mutex);
58	unsigned long long memory_limit;	27	unsigned long long memory_limit;
59	bool init_mem_is_free;	28	bool init_mem_is_free;
60		29
@@ -72,6 +41,7 @@ pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn, · 11 unmodified lines
72	EXPORT_SYMBOL(phys_mem_access_prot);	41	EXPORT_SYMBOL(phys_mem_access_prot);
73		42
74	#ifdef CONFIG_MEMORY_HOTPLUG	43	#ifdef CONFIG_MEMORY_HOTPLUG
		44	static DEFINE_MUTEX(linear_mapping_mutex);
75		45
76	#ifdef CONFIG_NUMA	46	#ifdef CONFIG_NUMA
77	int memory_add_physaddr_to_nid(u64 start)	47	int memory_add_physaddr_to_nid(u64 start)
@@ -340,257 +310,6 @@ void free_initmem(void) · 262 unmodified lines
340	free_initmem_default(POISON_FREE_INITMEM);	310	free_initmem_default(POISON_FREE_INITMEM);
341	}	311	}
342		312
343	/**
344	* flush_coherent_icache() - if a CPU has a coherent icache, flush it
345	* @addr: The base address to use (can be any valid address, the whole cache will be flushed)
346	* Return true if the cache was flushed, false otherwise
347	*/
348	static inline bool flush_coherent_icache(unsigned long addr)
349	{
350	/*
351	* For a snooping icache, we still need a dummy icbi to purge all the
352	* prefetched instructions from the ifetch buffers. We also need a sync
353	* before the icbi to order the the actual stores to memory that might
354	* have modified instructions with the icbi.
355	*/
356	if (cpu_has_feature(CPU_FTR_COHERENT_ICACHE)) {
357	mb(); /* sync */
358	allow_read_from_user((const void __user *)addr, L1_CACHE_BYTES);
359	icbi((void *)addr);
360	prevent_read_from_user((const void __user *)addr, L1_CACHE_BYTES);
361	mb(); /* sync */
362	isync();
363	return true;
364	}
365
366	return false;
367	}
368
369	/**
370	* invalidate_icache_range() - Flush the icache by issuing icbi across an address range
371	* @start: the start address
372	* @stop: the stop address (exclusive)
373	*/
374	static void invalidate_icache_range(unsigned long start, unsigned long stop)
375	{
376	unsigned long shift = l1_icache_shift();
377	unsigned long bytes = l1_icache_bytes();
378	char addr = (char )(start & ~(bytes - 1));
379	unsigned long size = stop - (unsigned long)addr + (bytes - 1);
380	unsigned long i;
381
382	for (i = 0; i < size >> shift; i++, addr += bytes)
383	icbi(addr);
384
385	mb(); /* sync */
386	isync();
387	}
388
389	/**
390	* flush_icache_range: Write any modified data cache blocks out to memory
391	* and invalidate the corresponding blocks in the instruction cache
392	*
393	* Generic code will call this after writing memory, before executing from it.
394	*
395	* @start: the start address
396	* @stop: the stop address (exclusive)
397	*/
398	void flush_icache_range(unsigned long start, unsigned long stop)
399	{
400	if (flush_coherent_icache(start))
401	return;
402
403	clean_dcache_range(start, stop);
404
405	if (IS_ENABLED(CONFIG_44x)) {
406	/*
407	* Flash invalidate on 44x because we are passed kmapped
408	* addresses and this doesn't work for userspace pages due to
409	* the virtually tagged icache.
410	*/
411	iccci((void *)start);
412	mb(); /* sync */
413	isync();
414	} else
415	invalidate_icache_range(start, stop);
416	}
417	EXPORT_SYMBOL(flush_icache_range);
418
419	#if !defined(CONFIG_PPC_8xx) && !defined(CONFIG_PPC64)
420	/**
421	* flush_dcache_icache_phys() - Flush a page by it's physical address
422	* @physaddr: the physical address of the page
423	*/
424	static void flush_dcache_icache_phys(unsigned long physaddr)
425	{
426	unsigned long bytes = l1_dcache_bytes();
427	unsigned long nb = PAGE_SIZE / bytes;
428	unsigned long addr = physaddr & PAGE_MASK;
429	unsigned long msr, msr0;
430	unsigned long loop1 = addr, loop2 = addr;
431
432	msr0 = mfmsr();
433	msr = msr0 & ~MSR_DR;
434	/*
435	* This must remain as ASM to prevent potential memory accesses
436	* while the data MMU is disabled
437	*/
438	asm volatile(
439	" mtctr %2;\n"
440	" mtmsr %3;\n"
441	" isync;\n"
442	"0: dcbst 0, %0;\n"
443	" addi %0, %0, %4;\n"
444	" bdnz 0b;\n"
445	" sync;\n"
446	" mtctr %2;\n"
447	"1: icbi 0, %1;\n"
448	" addi %1, %1, %4;\n"
449	" bdnz 1b;\n"
450	" sync;\n"
451	" mtmsr %5;\n"
452	" isync;\n"
453	: "+&r" (loop1), "+&r" (loop2)
454	: "r" (nb), "r" (msr), "i" (bytes), "r" (msr0)
455	: "ctr", "memory");
456	}
457	NOKPROBE_SYMBOL(flush_dcache_icache_phys)
458	#endif // !defined(CONFIG_PPC_8xx) && !defined(CONFIG_PPC64)
459
460	/*
461	* This is called when a page has been modified by the kernel.
462	* It just marks the page as not i-cache clean. We do the i-cache
463	* flush later when the page is given to a user process, if necessary.
464	*/
465	void flush_dcache_page(struct page *page)
466	{
467	if (cpu_has_feature(CPU_FTR_COHERENT_ICACHE))
468	return;
469	/* avoid an atomic op if possible */
470	if (test_bit(PG_dcache_clean, &page->flags))
471	clear_bit(PG_dcache_clean, &page->flags);
472	}
473	EXPORT_SYMBOL(flush_dcache_page);
474
475	static void flush_dcache_icache_hugepage(struct page *page)
476	{
477	int i;
478	void *start;
479
480	BUG_ON(!PageCompound(page));
481
482	for (i = 0; i < compound_nr(page); i++) {
483	if (!PageHighMem(page)) {
484	__flush_dcache_icache(page_address(page+i));
485	} else {
486	start = kmap_atomic(page+i);
487	__flush_dcache_icache(start);
488	kunmap_atomic(start);
489	}
490	}
491	}
492
493	void flush_dcache_icache_page(struct page *page)
494	{
495
496	if (PageCompound(page))
497	return flush_dcache_icache_hugepage(page);
498
499	#if defined(CONFIG_PPC_8xx) \|\| defined(CONFIG_PPC64)
500	/* On 8xx there is no need to kmap since highmem is not supported */
501	__flush_dcache_icache(page_address(page));
502	#else
503	if (IS_ENABLED(CONFIG_BOOKE) \|\| sizeof(phys_addr_t) > sizeof(void *)) {
504	void *start = kmap_atomic(page);
505	__flush_dcache_icache(start);
506	kunmap_atomic(start);
507	} else {
508	unsigned long addr = page_to_pfn(page) << PAGE_SHIFT;
509
510	if (flush_coherent_icache(addr))
511	return;
512	flush_dcache_icache_phys(addr);
513	}
514	#endif
515	}
516	EXPORT_SYMBOL(flush_dcache_icache_page);
517
518	/**
519	* __flush_dcache_icache(): Flush a particular page from the data cache to RAM.
520	* Note: this is necessary because the instruction cache does not
521	* snoop from the data cache.
522	*
523	* @page: the address of the page to flush
524	*/
525	void __flush_dcache_icache(void *p)
526	{
527	unsigned long addr = (unsigned long)p;
528
529	if (flush_coherent_icache(addr))
530	return;
531
532	clean_dcache_range(addr, addr + PAGE_SIZE);
533
534	/*
535	* We don't flush the icache on 44x. Those have a virtual icache and we
536	* don't have access to the virtual address here (it's not the page
537	* vaddr but where it's mapped in user space). The flushing of the
538	* icache on these is handled elsewhere, when a change in the address
539	* space occurs, before returning to user space.
540	*/
541
542	if (mmu_has_feature(MMU_FTR_TYPE_44x))
543	return;
544
545	invalidate_icache_range(addr, addr + PAGE_SIZE);
546	}
547
548	void clear_user_page(void page, unsigned long vaddr, struct page pg)
549	{
550	clear_page(page);
551
552	/*
553	* We shouldn't have to do this, but some versions of glibc
554	* require it (ld.so assumes zero filled pages are icache clean)
555	* - Anton
556	*/
557	flush_dcache_page(pg);
558	}
559	EXPORT_SYMBOL(clear_user_page);
560
561	void copy_user_page(void vto, void vfrom, unsigned long vaddr,
562	struct page *pg)
563	{
564	copy_page(vto, vfrom);
565
566	/*
567	* We should be able to use the following optimisation, however
568	* there are two problems.
569	* Firstly a bug in some versions of binutils meant PLT sections
570	* were not marked executable.
571	* Secondly the first word in the GOT section is blrl, used
572	* to establish the GOT address. Until recently the GOT was
573	* not marked executable.
574	* - Anton
575	*/
576	#if 0
577	if (!vma->vm_file && ((vma->vm_flags & VM_EXEC) == 0))
578	return;
579	#endif
580
581	flush_dcache_page(pg);
582	}
583
584	void flush_icache_user_page(struct vm_area_struct vma, struct page page,
585	unsigned long addr, int len)
586	{
587	unsigned long maddr;
588
589	maddr = (unsigned long) kmap(page) + (addr & ~PAGE_MASK);
590	flush_icache_range(maddr, maddr + len);
591	kunmap(page);
592	}
593
594	/*	313	/*
595	* System memory should not be in /proc/iomem but various tools expect it	314	* System memory should not be in /proc/iomem but various tools expect it
596	* (eg kdump).	315	* (eg kdump).

arch/powerpc/mm/mmu_context.c+13−11

@@ -43,24 +43,26 @@ void switch_mm_irqs_off(struct mm_struct prev, struct mm_struct next, · 42 unmodified lines
43		43
44	/*	44	/*
45	* This full barrier orders the store to the cpumask above vs	45	* This full barrier orders the store to the cpumask above vs
46	* a subsequent operation which allows this CPU to begin loading	46	* a subsequent load which allows this CPU/MMU to begin loading
47	* translations for next.	47	* translations for 'next' from page table PTEs into the TLB.
48	*	48	*
49	* When using the radix MMU that operation is the load of the	49	* When using the radix MMU, that operation is the load of the
50	* MMU context id, which is then moved to SPRN_PID.	50	* MMU context id, which is then moved to SPRN_PID.
51	*	51	*
52	* For the hash MMU it is either the first load from slb_cache	52	* For the hash MMU it is either the first load from slb_cache
53	* in switch_slb(), and/or the store of paca->mm_ctx_id in	53	* in switch_slb() to preload the SLBs, or the load of
54	* copy_mm_to_paca().	54	* get_user_context which loads the context for the VSID hash
		55	* to insert a new SLB, in the SLB fault handler.
55	*	56	*
56	* On the other side, the barrier is in mm/tlb-radix.c for	57	* On the other side, the barrier is in mm/tlb-radix.c for
57	* radix which orders earlier stores to clear the PTEs vs	58	* radix which orders earlier stores to clear the PTEs before
58	* the load of mm_cpumask. And pte_xchg which does the same	59	* the load of mm_cpumask to check which CPU TLBs should be
59	* thing for hash.	60	* flushed. For hash, pte_xchg to clear the PTE includes the
		61	* barrier.
60	*	62	*
61	* This full barrier is needed by membarrier when switching	63	* This full barrier is also needed by membarrier when
62	* between processes after store to rq->curr, before user-space	64	* switching between processes after store to rq->curr, before
63	* memory accesses.	65	* user-space memory accesses.
64	*/	66	*/
65	smp_mb();	67	smp_mb();
66		68

arch/powerpc/mm/mmu_decl.h+5−0

@@ -185,3 +185,8 @@ void ptdump_check_wx(void); · 184 unmodified lines
185	#else	185	#else
186	static inline void ptdump_check_wx(void) { }	186	static inline void ptdump_check_wx(void) { }
187	#endif	187	#endif
		188
		189	static inline bool debug_pagealloc_enabled_or_kfence(void)
		190	{
		191	return IS_ENABLED(CONFIG_KFENCE) \|\| debug_pagealloc_enabled();
		192	}

arch/powerpc/mm/nohash/8xx.c+2−2

@@ -149,7 +149,7 @@ unsigned long __init mmu_mapin_ram(unsigned long base, unsigned long top) · 148 unmodified lines
149	{	149	{
150	unsigned long etext8 = ALIGN(__pa(_etext), SZ_8M);	150	unsigned long etext8 = ALIGN(__pa(_etext), SZ_8M);
151	unsigned long sinittext = __pa(_sinittext);	151	unsigned long sinittext = __pa(_sinittext);
152	bool strict_boundary = strict_kernel_rwx_enabled() \|\| debug_pagealloc_enabled();	152	bool strict_boundary = strict_kernel_rwx_enabled() \|\| debug_pagealloc_enabled_or_kfence();
153	unsigned long boundary = strict_boundary ? sinittext : etext8;	153	unsigned long boundary = strict_boundary ? sinittext : etext8;
154	unsigned long einittext8 = ALIGN(__pa(_einittext), SZ_8M);	154	unsigned long einittext8 = ALIGN(__pa(_einittext), SZ_8M);
155		155
@@ -161,7 +161,7 @@ unsigned long __init mmu_mapin_ram(unsigned long base, unsigned long top) · 5 unmodified lines
161	return 0;	161	return 0;
162		162
163	mmu_mapin_ram_chunk(0, boundary, PAGE_KERNEL_TEXT, true);	163	mmu_mapin_ram_chunk(0, boundary, PAGE_KERNEL_TEXT, true);
164	if (debug_pagealloc_enabled()) {	164	if (debug_pagealloc_enabled_or_kfence()) {
165	top = boundary;	165	top = boundary;
166	} else {	166	} else {
167	mmu_mapin_ram_chunk(boundary, einittext8, PAGE_KERNEL_TEXT, true);	167	mmu_mapin_ram_chunk(boundary, einittext8, PAGE_KERNEL_TEXT, true);

arch/powerpc/net/Makefile+1−5

 #
 # Arch-specific network modules
 #
-ifdef CONFIG_PPC64
+obj-$(CONFIG_BPF_JIT) += bpf_jit_comp.o bpf_jit_comp$(BITS).o
-obj-$(CONFIG_BPF_JIT) += bpf_jit_comp64.o
-else
-obj-$(CONFIG_BPF_JIT) += bpf_jit_asm.o bpf_jit_comp.o
-endif

arch/powerpc/net/bpf_jit.h+64−0

@@ -26,6 +26,9 @@ · 25 unmodified lines
26	/* Long jump; (unconditional 'branch') */	26	/* Long jump; (unconditional 'branch') */
27	#define PPC_JMP(dest) EMIT(PPC_INST_BRANCH \| \	27	#define PPC_JMP(dest) EMIT(PPC_INST_BRANCH \| \
28	(((dest) - (ctx->idx * 4)) & 0x03fffffc))	28	(((dest) - (ctx->idx * 4)) & 0x03fffffc))
		29	/* blr; (unconditional 'branch' with link) to absolute address */
		30	#define PPC_BL_ABS(dest) EMIT(PPC_INST_BL \| \
		31	(((dest) - (unsigned long)(image + ctx->idx)) & 0x03fffffc))
29	/* "cond" here covers BO:BI fields. */	32	/* "cond" here covers BO:BI fields. */
30	#define PPC_BCC_SHORT(cond, dest) EMIT(PPC_INST_BRANCH_COND \| \	33	#define PPC_BCC_SHORT(cond, dest) EMIT(PPC_INST_BRANCH_COND \| \
31	(((cond) & 0x3ff) << 16) \| \	34	(((cond) & 0x3ff) << 16) \| \
@@ -42,6 +45,10 @@ · 10 unmodified lines
42	EMIT(PPC_RAW_ORI(d, d, IMM_L(i))); \	45	EMIT(PPC_RAW_ORI(d, d, IMM_L(i))); \
43	} } while(0)	46	} } while(0)
44		47
		48	#ifdef CONFIG_PPC32
		49	#define PPC_EX32(r, i) EMIT(PPC_RAW_LI((r), (i) < 0 ? -1 : 0))
		50	#endif
		51
45	#define PPC_LI64(d, i) do { \	52	#define PPC_LI64(d, i) do { \
46	if ((long)(i) >= -2147483648 && \	53	if ((long)(i) >= -2147483648 && \
47	(long)(i) < 2147483648) \	54	(long)(i) < 2147483648) \
@@ -108,6 +115,63 @@ static inline bool is_nearbranch(int offset) · 60 unmodified lines
108	#define COND_LT (CR0_LT \| COND_CMP_TRUE)	115	#define COND_LT (CR0_LT \| COND_CMP_TRUE)
109	#define COND_LE (CR0_GT \| COND_CMP_FALSE)	116	#define COND_LE (CR0_GT \| COND_CMP_FALSE)
110		117
		118	#define SEEN_FUNC 0x20000000 /* might call external helpers */
		119	#define SEEN_STACK 0x40000000 /* uses BPF stack */
		120	#define SEEN_TAILCALL 0x80000000 /* uses tail calls */
		121
		122	#define SEEN_VREG_MASK 0x1ff80000 /* Volatile registers r3-r12 */
		123	#define SEEN_NVREG_MASK 0x0003ffff /* Non volatile registers r14-r31 */
		124
		125	#ifdef CONFIG_PPC64
		126	extern const int b2p[MAX_BPF_JIT_REG + 2];
		127	#else
		128	extern const int b2p[MAX_BPF_JIT_REG + 1];
		129	#endif
		130
		131	struct codegen_context {
		132	/*
		133	* This is used to track register usage as well
		134	* as calls to external helpers.
		135	* - register usage is tracked with corresponding
		136	* bits (r3-r31)
		137	* - rest of the bits can be used to track other
		138	* things -- for now, we use bits 0 to 2
		139	* encoded in SEEN_* macros above
		140	*/
		141	unsigned int seen;
		142	unsigned int idx;
		143	unsigned int stack_size;
		144	int b2p[ARRAY_SIZE(b2p)];
		145	};
		146
		147	static inline void bpf_flush_icache(void start, void end)
		148	{
		149	smp_wmb(); /* smp write barrier */
		150	flush_icache_range((unsigned long)start, (unsigned long)end);
		151	}
		152
		153	static inline bool bpf_is_seen_register(struct codegen_context *ctx, int i)
		154	{
		155	return ctx->seen & (1 << (31 - i));
		156	}
		157
		158	static inline void bpf_set_seen_register(struct codegen_context *ctx, int i)
		159	{
		160	ctx->seen \|= 1 << (31 - i);
		161	}
		162
		163	static inline void bpf_clear_seen_register(struct codegen_context *ctx, int i)
		164	{
		165	ctx->seen &= ~(1 << (31 - i));
		166	}
		167
		168	void bpf_jit_emit_func_call_rel(u32 image, struct codegen_context ctx, u64 func);
		169	int bpf_jit_build_body(struct bpf_prog fp, u32 image, struct codegen_context *ctx,
		170	u32 *addrs, bool extra_pass);
		171	void bpf_jit_build_prologue(u32 image, struct codegen_context ctx);
		172	void bpf_jit_build_epilogue(u32 image, struct codegen_context ctx);
		173	void bpf_jit_realloc_regs(struct codegen_context *ctx);
		174
111	#endif	175	#endif
112		176
113	#endif	177	#endif

arch/powerpc/net/bpf_jit32.h+0−139

@@ -1,139 +0,0 @@
1	/* SPDX-License-Identifier: GPL-2.0-only */
2	/*
3	* bpf_jit32.h: BPF JIT compiler for PPC
4	*
5	* Copyright 2011 Matt Evans <matt@ozlabs.org>, IBM Corporation
6	*
7	* Split from bpf_jit.h
8	*/
9	#ifndef _BPF_JIT32_H
10	#define _BPF_JIT32_H
11
12	#include <asm/asm-compat.h>
13	#include "bpf_jit.h"
14
15	#ifdef CONFIG_PPC64
16	#define BPF_PPC_STACK_R3_OFF 48
17	#define BPF_PPC_STACK_LOCALS 32
18	#define BPF_PPC_STACK_BASIC (48+64)
19	#define BPF_PPC_STACK_SAVE (18*8)
20	#define BPF_PPC_STACKFRAME (BPF_PPC_STACK_BASIC+BPF_PPC_STACK_LOCALS+ \
21	BPF_PPC_STACK_SAVE)
22	#define BPF_PPC_SLOWPATH_FRAME (48+64)
23	#else
24	#define BPF_PPC_STACK_R3_OFF 24
25	#define BPF_PPC_STACK_LOCALS 16
26	#define BPF_PPC_STACK_BASIC (24+32)
27	#define BPF_PPC_STACK_SAVE (18*4)
28	#define BPF_PPC_STACKFRAME (BPF_PPC_STACK_BASIC+BPF_PPC_STACK_LOCALS+ \
29	BPF_PPC_STACK_SAVE)
30	#define BPF_PPC_SLOWPATH_FRAME (24+32)
31	#endif
32
33	#define REG_SZ (BITS_PER_LONG/8)
34
35	/*
36	* Generated code register usage:
37	*
38	* As normal PPC C ABI (e.g. r1=sp, r2=TOC), with:
39	*
40	* skb r3 (Entry parameter)
41	* A register r4
42	* X register r5
43	* addr param r6
44	* r7-r10 scratch
45	* skb->data r14
46	* skb headlen r15 (skb->len - skb->data_len)
47	* m[0] r16
48	* m[...] ...
49	* m[15] r31
50	*/
51	#define r_skb 3
52	#define r_ret 3
53	#define r_A 4
54	#define r_X 5
55	#define r_addr 6
56	#define r_scratch1 7
57	#define r_scratch2 8
58	#define r_D 14
59	#define r_HL 15
60	#define r_M 16
61
62	#ifndef __ASSEMBLY__
63
64	/*
65	* Assembly helpers from arch/powerpc/net/bpf_jit.S:
66	*/
67	#define DECLARE_LOAD_FUNC(func) \
68	extern u8 func[], func##_negative_offset[], func##_positive_offset[]
69
70	DECLARE_LOAD_FUNC(sk_load_word);
71	DECLARE_LOAD_FUNC(sk_load_half);
72	DECLARE_LOAD_FUNC(sk_load_byte);
73	DECLARE_LOAD_FUNC(sk_load_byte_msh);
74
75	#define PPC_LBZ_OFFS(r, base, i) do { if ((i) < 32768) EMIT(PPC_RAW_LBZ(r, base, i)); \
76	else { EMIT(PPC_RAW_ADDIS(r, base, IMM_HA(i))); \
77	EMIT(PPC_RAW_LBZ(r, r, IMM_L(i))); } } while(0)
78
79	#define PPC_LD_OFFS(r, base, i) do { if ((i) < 32768) EMIT(PPC_RAW_LD(r, base, i)); \
80	else { EMIT(PPC_RAW_ADDIS(r, base, IMM_HA(i))); \
81	EMIT(PPC_RAW_LD(r, r, IMM_L(i))); } } while(0)
82
83	#define PPC_LWZ_OFFS(r, base, i) do { if ((i) < 32768) EMIT(PPC_RAW_LWZ(r, base, i)); \
84	else { EMIT(PPC_RAW_ADDIS(r, base, IMM_HA(i))); \
85	EMIT(PPC_RAW_LWZ(r, r, IMM_L(i))); } } while(0)
86
87	#define PPC_LHZ_OFFS(r, base, i) do { if ((i) < 32768) EMIT(PPC_RAW_LHZ(r, base, i)); \
88	else { EMIT(PPC_RAW_ADDIS(r, base, IMM_HA(i))); \
89	EMIT(PPC_RAW_LHZ(r, r, IMM_L(i))); } } while(0)
90
91	#ifdef CONFIG_PPC64
92	#define PPC_LL_OFFS(r, base, i) do { PPC_LD_OFFS(r, base, i); } while(0)
93	#else
94	#define PPC_LL_OFFS(r, base, i) do { PPC_LWZ_OFFS(r, base, i); } while(0)
95	#endif
96
97	#ifdef CONFIG_SMP
98	#ifdef CONFIG_PPC64
99	#define PPC_BPF_LOAD_CPU(r) \
100	do { BUILD_BUG_ON(sizeof_field(struct paca_struct, paca_index) != 2); \
101	PPC_LHZ_OFFS(r, 13, offsetof(struct paca_struct, paca_index)); \
102	} while (0)
103	#else
104	#define PPC_BPF_LOAD_CPU(r) \
105	do { BUILD_BUG_ON(sizeof_field(struct task_struct, cpu) != 4); \
106	PPC_LHZ_OFFS(r, 2, offsetof(struct task_struct, cpu)); \
107	} while(0)
108	#endif
109	#else
110	#define PPC_BPF_LOAD_CPU(r) do { EMIT(PPC_RAW_LI(r, 0)); } while(0)
111	#endif
112
113	#define PPC_LHBRX_OFFS(r, base, i) \
114	do { PPC_LI32(r, i); EMIT(PPC_RAW_LHBRX(r, r, base)); } while(0)
115	#ifdef __LITTLE_ENDIAN__
116	#define PPC_NTOHS_OFFS(r, base, i) PPC_LHBRX_OFFS(r, base, i)
117	#else
118	#define PPC_NTOHS_OFFS(r, base, i) PPC_LHZ_OFFS(r, base, i)
119	#endif
120
121	#define PPC_BPF_LL(r, base, i) do { EMIT(PPC_RAW_LWZ(r, base, i)); } while(0)
122	#define PPC_BPF_STL(r, base, i) do { EMIT(PPC_RAW_STW(r, base, i)); } while(0)
123	#define PPC_BPF_STLU(r, base, i) do { EMIT(PPC_RAW_STWU(r, base, i)); } while(0)
124
125	#define SEEN_DATAREF 0x10000 /* might call external helpers */
126	#define SEEN_XREG 0x20000 /* X reg is used */
127	#define SEEN_MEM 0x40000 /* SEEN_MEM+(1<<n) = use mem[n] for temporary
128	* storage */
129	#define SEEN_MEM_MSK 0x0ffff
130
131	struct codegen_context {
132	unsigned int seen;
133	unsigned int idx;
134	int pc_ret0; /* bpf index of first RET #0 instruction (if any) */
135	};
136
137	#endif
138
139	#endif

arch/powerpc/net/bpf_jit64.h+1−20

@@ -39,7 +39,7 @@ · 38 unmodified lines
39	#define TMP_REG_2 (MAX_BPF_JIT_REG + 1)	39	#define TMP_REG_2 (MAX_BPF_JIT_REG + 1)
40		40
41	/* BPF to ppc register mappings */	41	/* BPF to ppc register mappings */
42	static const int b2p[] = {	42	const int b2p[MAX_BPF_JIT_REG + 2] = {
43	/* function return value */	43	/* function return value */
44	[BPF_REG_0] = 8,	44	[BPF_REG_0] = 8,
45	/* function arguments */	45	/* function arguments */
@@ -86,25 +86,6 @@ static const int b2p[] = { · 40 unmodified lines
86	} while(0)	86	} while(0)
87	#define PPC_BPF_STLU(r, base, i) do { EMIT(PPC_RAW_STDU(r, base, i)); } while(0)	87	#define PPC_BPF_STLU(r, base, i) do { EMIT(PPC_RAW_STDU(r, base, i)); } while(0)
88		88
89	#define SEEN_FUNC 0x1000 /* might call external helpers */
90	#define SEEN_STACK 0x2000 /* uses BPF stack */
91	#define SEEN_TAILCALL 0x4000 /* uses tail calls */
92
93	struct codegen_context {
94	/*
95	* This is used to track register usage as well
96	* as calls to external helpers.
97	* - register usage is tracked with corresponding
98	* bits (r3-r10 and r27-r31)
99	* - rest of the bits can be used to track other
100	* things -- for now, we use bits 16 to 23
101	* encoded in SEEN_* macros above
102	*/
103	unsigned int seen;
104	unsigned int idx;
105	unsigned int stack_size;
106	};
107
108	#endif /* !__ASSEMBLY__ */	89	#endif /* !__ASSEMBLY__ */
109		90
110	#endif	91	#endif

arch/powerpc/net/bpf_jit_asm.S+0−226

@@ -1,226 +0,0 @@
1	/* SPDX-License-Identifier: GPL-2.0-only */
2	/* bpf_jit.S: Packet/header access helper functions
3	* for PPC64 BPF compiler.
4	*
5	* Copyright 2011 Matt Evans <matt@ozlabs.org>, IBM Corporation
6	*/
7
8	#include <asm/ppc_asm.h>
9	#include <asm/asm-compat.h>
10	#include "bpf_jit32.h"
11
12	/*
13	* All of these routines are called directly from generated code,
14	* whose register usage is:
15	*
16	* r3 skb
17	* r4,r5 A,X
18	* r6 * address parameter to helper *
19	* r7-r10 scratch
20	* r14 skb->data
21	* r15 skb headlen
22	* r16-31 M[]
23	*/
24
25	/*
26	* To consider: These helpers are so small it could be better to just
27	* generate them inline. Inline code can do the simple headlen check
28	* then branch directly to slow_path_XXX if required. (In fact, could
29	* load a spare GPR with the address of slow_path_generic and pass size
30	* as an argument, making the call site a mtlr, li and bllr.)
31	*/
32	.globl sk_load_word
33	sk_load_word:
34	PPC_LCMPI r_addr, 0
35	blt bpf_slow_path_word_neg
36	.globl sk_load_word_positive_offset
37	sk_load_word_positive_offset:
38	/* Are we accessing past headlen? */
39	subi r_scratch1, r_HL, 4
40	PPC_LCMP r_scratch1, r_addr
41	blt bpf_slow_path_word
42	/* Nope, just hitting the header. cr0 here is eq or gt! */
43	#ifdef __LITTLE_ENDIAN__
44	lwbrx r_A, r_D, r_addr
45	#else
46	lwzx r_A, r_D, r_addr
47	#endif
48	blr /* Return success, cr0 != LT */
49
50	.globl sk_load_half
51	sk_load_half:
52	PPC_LCMPI r_addr, 0
53	blt bpf_slow_path_half_neg
54	.globl sk_load_half_positive_offset
55	sk_load_half_positive_offset:
56	subi r_scratch1, r_HL, 2
57	PPC_LCMP r_scratch1, r_addr
58	blt bpf_slow_path_half
59	#ifdef __LITTLE_ENDIAN__
60	lhbrx r_A, r_D, r_addr
61	#else
62	lhzx r_A, r_D, r_addr
63	#endif
64	blr
65
66	.globl sk_load_byte
67	sk_load_byte:
68	PPC_LCMPI r_addr, 0
69	blt bpf_slow_path_byte_neg
70	.globl sk_load_byte_positive_offset
71	sk_load_byte_positive_offset:
72	PPC_LCMP r_HL, r_addr
73	ble bpf_slow_path_byte
74	lbzx r_A, r_D, r_addr
75	blr
76
77	/*
78	* BPF_LDX \| BPF_B \| BPF_MSH: ldxb 4*([offset]&0xf)
79	* r_addr is the offset value
80	*/
81	.globl sk_load_byte_msh
82	sk_load_byte_msh:
83	PPC_LCMPI r_addr, 0
84	blt bpf_slow_path_byte_msh_neg
85	.globl sk_load_byte_msh_positive_offset
86	sk_load_byte_msh_positive_offset:
87	PPC_LCMP r_HL, r_addr
88	ble bpf_slow_path_byte_msh
89	lbzx r_X, r_D, r_addr
90	rlwinm r_X, r_X, 2, 32-4-2, 31-2
91	blr
92
93	/* Call out to skb_copy_bits:
94	* We'll need to back up our volatile regs first; we have
95	* local variable space at r1+(BPF_PPC_STACK_BASIC).
96	* Allocate a new stack frame here to remain ABI-compliant in
97	* stashing LR.
98	*/
99	#define bpf_slow_path_common(SIZE) \
100	mflr r0; \
101	PPC_STL r0, PPC_LR_STKOFF(r1); \
102	/* R3 goes in parameter space of caller's frame */ \
103	PPC_STL r_skb, (BPF_PPC_STACKFRAME+BPF_PPC_STACK_R3_OFF)(r1); \
104	PPC_STL r_A, (BPF_PPC_STACK_BASIC+(0*REG_SZ))(r1); \
105	PPC_STL r_X, (BPF_PPC_STACK_BASIC+(1*REG_SZ))(r1); \
106	addi r5, r1, BPF_PPC_STACK_BASIC+(2*REG_SZ); \
107	PPC_STLU r1, -BPF_PPC_SLOWPATH_FRAME(r1); \
108	/* R3 = r_skb, as passed */ \
109	mr r4, r_addr; \
110	li r6, SIZE; \
111	bl skb_copy_bits; \
112	nop; \
113	/* R3 = 0 on success */ \
114	addi r1, r1, BPF_PPC_SLOWPATH_FRAME; \
115	PPC_LL r0, PPC_LR_STKOFF(r1); \
116	PPC_LL r_A, (BPF_PPC_STACK_BASIC+(0*REG_SZ))(r1); \
117	PPC_LL r_X, (BPF_PPC_STACK_BASIC+(1*REG_SZ))(r1); \
118	mtlr r0; \
119	PPC_LCMPI r3, 0; \
120	blt bpf_error; /* cr0 = LT */ \
121	PPC_LL r_skb, (BPF_PPC_STACKFRAME+BPF_PPC_STACK_R3_OFF)(r1); \
122	/* Great success! */
123
124	bpf_slow_path_word:
125	bpf_slow_path_common(4)
126	/* Data value is on stack, and cr0 != LT */
127	lwz r_A, BPF_PPC_STACK_BASIC+(2*REG_SZ)(r1)
128	blr
129
130	bpf_slow_path_half:
131	bpf_slow_path_common(2)
132	lhz r_A, BPF_PPC_STACK_BASIC+(2*8)(r1)
133	blr
134
135	bpf_slow_path_byte:
136	bpf_slow_path_common(1)
137	lbz r_A, BPF_PPC_STACK_BASIC+(2*8)(r1)
138	blr
139
140	bpf_slow_path_byte_msh:
141	bpf_slow_path_common(1)
142	lbz r_X, BPF_PPC_STACK_BASIC+(2*8)(r1)
143	rlwinm r_X, r_X, 2, 32-4-2, 31-2
144	blr
145
146	/* Call out to bpf_internal_load_pointer_neg_helper:
147	* We'll need to back up our volatile regs first; we have
148	* local variable space at r1+(BPF_PPC_STACK_BASIC).
149	* Allocate a new stack frame here to remain ABI-compliant in
150	* stashing LR.
151	*/
152	#define sk_negative_common(SIZE) \
153	mflr r0; \
154	PPC_STL r0, PPC_LR_STKOFF(r1); \
155	/* R3 goes in parameter space of caller's frame */ \
156	PPC_STL r_skb, (BPF_PPC_STACKFRAME+BPF_PPC_STACK_R3_OFF)(r1); \
157	PPC_STL r_A, (BPF_PPC_STACK_BASIC+(0*REG_SZ))(r1); \
158	PPC_STL r_X, (BPF_PPC_STACK_BASIC+(1*REG_SZ))(r1); \
159	PPC_STLU r1, -BPF_PPC_SLOWPATH_FRAME(r1); \
160	/* R3 = r_skb, as passed */ \
161	mr r4, r_addr; \
162	li r5, SIZE; \
163	bl bpf_internal_load_pointer_neg_helper; \
164	nop; \
165	/* R3 != 0 on success */ \
166	addi r1, r1, BPF_PPC_SLOWPATH_FRAME; \
167	PPC_LL r0, PPC_LR_STKOFF(r1); \
168	PPC_LL r_A, (BPF_PPC_STACK_BASIC+(0*REG_SZ))(r1); \
169	PPC_LL r_X, (BPF_PPC_STACK_BASIC+(1*REG_SZ))(r1); \
170	mtlr r0; \
171	PPC_LCMPLI r3, 0; \
172	beq bpf_error_slow; /* cr0 = EQ */ \
173	mr r_addr, r3; \
174	PPC_LL r_skb, (BPF_PPC_STACKFRAME+BPF_PPC_STACK_R3_OFF)(r1); \
175	/* Great success! */
176
177	bpf_slow_path_word_neg:
178	lis r_scratch1,-32 /* SKF_LL_OFF */
179	PPC_LCMP r_addr, r_scratch1 /* addr < SKF_* */
180	blt bpf_error /* cr0 = LT */
181	.globl sk_load_word_negative_offset
182	sk_load_word_negative_offset:
183	sk_negative_common(4)
184	lwz r_A, 0(r_addr)
185	blr
186
187	bpf_slow_path_half_neg:
188	lis r_scratch1,-32 /* SKF_LL_OFF */
189	PPC_LCMP r_addr, r_scratch1 /* addr < SKF_* */
190	blt bpf_error /* cr0 = LT */
191	.globl sk_load_half_negative_offset
192	sk_load_half_negative_offset:
193	sk_negative_common(2)
194	lhz r_A, 0(r_addr)
195	blr
196
197	bpf_slow_path_byte_neg:
198	lis r_scratch1,-32 /* SKF_LL_OFF */
199	PPC_LCMP r_addr, r_scratch1 /* addr < SKF_* */
200	blt bpf_error /* cr0 = LT */
201	.globl sk_load_byte_negative_offset
202	sk_load_byte_negative_offset:
203	sk_negative_common(1)
204	lbz r_A, 0(r_addr)
205	blr
206
207	bpf_slow_path_byte_msh_neg:
208	lis r_scratch1,-32 /* SKF_LL_OFF */
209	PPC_LCMP r_addr, r_scratch1 /* addr < SKF_* */
210	blt bpf_error /* cr0 = LT */
211	.globl sk_load_byte_msh_negative_offset
212	sk_load_byte_msh_negative_offset:
213	sk_negative_common(1)
214	lbz r_X, 0(r_addr)
215	rlwinm r_X, r_X, 2, 32-4-2, 31-2
216	blr
217
218	bpf_error_slow:
219	/* fabricate a cr0 = lt */
220	li r_scratch1, -1
221	PPC_LCMPI r_scratch1, 0
222	bpf_error:
223	/* Entered with cr0 = lt */
224	li r3, 0
225	/* Generated code will 'blt epilogue', returning 0. */
226	blr

arch/powerpc/net/bpf_jit_comp.c+185−597

@@ -1,10 +1,11 @@
1	// SPDX-License-Identifier: GPL-2.0-only	1	// SPDX-License-Identifier: GPL-2.0-only
2	/* bpf_jit_comp.c: BPF JIT compiler	2	/*
		3	* eBPF JIT compiler
3	*	4	*
4	* Copyright 2011 Matt Evans <matt@ozlabs.org>, IBM Corporation	5	* Copyright 2016 Naveen N. Rao <naveen.n.rao@linux.vnet.ibm.com>
		6	* IBM Corporation
5	*	7	*
6	* Based on the x86 BPF compiler, by Eric Dumazet (eric.dumazet@gmail.com)	8	* Based on the powerpc classic BPF JIT compiler by Matt Evans
7	* Ported to ppc32 by Denis Kirjanov <kda@linux-powerpc.org>
8	*/	9	*/
9	#include <linux/moduleloader.h>	10	#include <linux/moduleloader.h>
10	#include <asm/cacheflush.h>	11	#include <asm/cacheflush.h>
@@ -12,639 +13,204 @@ · 1 unmodified line
12	#include <linux/netdevice.h>	13	#include <linux/netdevice.h>
13	#include <linux/filter.h>	14	#include <linux/filter.h>
14	#include <linux/if_vlan.h>	15	#include <linux/if_vlan.h>
		16	#include <asm/kprobes.h>
		17	#include <linux/bpf.h>
15		18
16	#include "bpf_jit32.h"	19	#include "bpf_jit.h"
17		20
18	static inline void bpf_flush_icache(void start, void end)	21	static void bpf_jit_fill_ill_insns(void *area, unsigned int size)
19	{	22	{
20	smp_wmb();	23	memset32(area, BREAKPOINT_INSTRUCTION, size / 4);
21	flush_icache_range((unsigned long)start, (unsigned long)end);
22	}	24	}
23		25
24	static void bpf_jit_build_prologue(struct bpf_prog fp, u32 image,	26	/* Fix the branch target addresses for subprog calls */
25	struct codegen_context *ctx)	27	static int bpf_jit_fixup_subprog_calls(struct bpf_prog fp, u32 image,
		28	struct codegen_context ctx, u32 addrs)
26	{	29	{
27	int i;	30	const struct bpf_insn *insn = fp->insnsi;
28	const struct sock_filter *filter = fp->insns;	31	bool func_addr_fixed;
29		32	u64 func_addr;
30	if (ctx->seen & (SEEN_MEM \| SEEN_DATAREF)) {	33	u32 tmp_idx;
31	/* Make stackframe */	34	int i, ret;
32	if (ctx->seen & SEEN_DATAREF) {
33	/* If we call any helpers (for loads), save LR */
34	EMIT(PPC_INST_MFLR \| __PPC_RT(R0));
35	PPC_BPF_STL(0, 1, PPC_LR_STKOFF);
36
37	/* Back up non-volatile regs. */
38	PPC_BPF_STL(r_D, 1, -(REG_SZ*(32-r_D)));
39	PPC_BPF_STL(r_HL, 1, -(REG_SZ*(32-r_HL)));
40	}
41	if (ctx->seen & SEEN_MEM) {
42	/*
43	* Conditionally save regs r15-r31 as some will be used
44	* for M[] data.
45	*/
46	for (i = r_M; i < (r_M+16); i++) {
47	if (ctx->seen & (1 << (i-r_M)))
48	PPC_BPF_STL(i, 1, -(REG_SZ*(32-i)));
49	}
50	}
51	PPC_BPF_STLU(1, 1, -BPF_PPC_STACKFRAME);
52	}
53
54	if (ctx->seen & SEEN_DATAREF) {
55	/*
56	* If this filter needs to access skb data,
57	* prepare r_D and r_HL:
58	* r_HL = skb->len - skb->data_len
59	* r_D = skb->data
60	*/
61	PPC_LWZ_OFFS(r_scratch1, r_skb, offsetof(struct sk_buff,
62	data_len));
63	PPC_LWZ_OFFS(r_HL, r_skb, offsetof(struct sk_buff, len));
64	EMIT(PPC_RAW_SUB(r_HL, r_HL, r_scratch1));
65	PPC_LL_OFFS(r_D, r_skb, offsetof(struct sk_buff, data));
66	}
67		35
68	if (ctx->seen & SEEN_XREG) {	36	for (i = 0; i < fp->len; i++) {
69	/*	37	/*
70	* TODO: Could also detect whether first instr. sets X and	38	* During the extra pass, only the branch target addresses for
71	* avoid this (as below, with A).	39	* the subprog calls need to be fixed. All other instructions
		40	* can left untouched.
		41	*
		42	* The JITed image length does not change because we already
		43	* ensure that the JITed instruction sequence for these calls
		44	* are of fixed length by padding them with NOPs.
72	*/	45	*/
73	EMIT(PPC_RAW_LI(r_X, 0));	46	if (insn[i].code == (BPF_JMP \| BPF_CALL) &&
74	}	47	insn[i].src_reg == BPF_PSEUDO_CALL) {
75		48	ret = bpf_jit_get_func_addr(fp, &insn[i], true,
76	/* make sure we dont leak kernel information to user */	49	&func_addr,
77	if (bpf_needs_clear_a(&filter[0]))	50	&func_addr_fixed);
78	EMIT(PPC_RAW_LI(r_A, 0));	51	if (ret < 0)
79	}	52	return ret;
80
81	static void bpf_jit_build_epilogue(u32 image, struct codegen_context ctx)
82	{
83	int i;
84
85	if (ctx->seen & (SEEN_MEM \| SEEN_DATAREF)) {
86	EMIT(PPC_RAW_ADDI(1, 1, BPF_PPC_STACKFRAME));
87	if (ctx->seen & SEEN_DATAREF) {
88	PPC_BPF_LL(0, 1, PPC_LR_STKOFF);
89	EMIT(PPC_RAW_MTLR(0));
90	PPC_BPF_LL(r_D, 1, -(REG_SZ*(32-r_D)));
91	PPC_BPF_LL(r_HL, 1, -(REG_SZ*(32-r_HL)));
92	}
93	if (ctx->seen & SEEN_MEM) {
94	/* Restore any saved non-vol registers */
95	for (i = r_M; i < (r_M+16); i++) {
96	if (ctx->seen & (1 << (i-r_M)))
97	PPC_BPF_LL(i, 1, -(REG_SZ*(32-i)));
98	}
99	}
100	}
101	/* The RETs have left a return value in R3. */
102
103	EMIT(PPC_RAW_BLR());
104	}
105
106	#define CHOOSE_LOAD_FUNC(K, func) \
107	((int)K < 0 ? ((int)K >= SKF_LL_OFF ? func##_negative_offset : func) : func##_positive_offset)
108
109	/* Assemble the body code between the prologue & epilogue. */
110	static int bpf_jit_build_body(struct bpf_prog fp, u32 image,
111	struct codegen_context *ctx,
112	unsigned int *addrs)
113	{
114	const struct sock_filter *filter = fp->insns;
115	int flen = fp->len;
116	u8 *func;
117	unsigned int true_cond;
118	int i;
119
120	/* Start of epilogue code */
121	unsigned int exit_addr = addrs[flen];
122
123	for (i = 0; i < flen; i++) {
124	unsigned int K = filter[i].k;
125	u16 code = bpf_anc_helper(&filter[i]);
126		53
127	/*
128	* addrs[] maps a BPF bytecode address into a real offset from
129	* the start of the body code.
130	*/
131	addrs[i] = ctx->idx * 4;
132
133	switch (code) {
134	/* ALU ops */
135	case BPF_ALU \| BPF_ADD \| BPF_X: /* A += X; */
136	ctx->seen \|= SEEN_XREG;
137	EMIT(PPC_RAW_ADD(r_A, r_A, r_X));
138	break;
139	case BPF_ALU \| BPF_ADD \| BPF_K: /* A += K; */
140	if (!K)
141	break;
142	EMIT(PPC_RAW_ADDI(r_A, r_A, IMM_L(K)));
143	if (K >= 32768)
144	EMIT(PPC_RAW_ADDIS(r_A, r_A, IMM_HA(K)));
145	break;
146	case BPF_ALU \| BPF_SUB \| BPF_X: /* A -= X; */
147	ctx->seen \|= SEEN_XREG;
148	EMIT(PPC_RAW_SUB(r_A, r_A, r_X));
149	break;
150	case BPF_ALU \| BPF_SUB \| BPF_K: /* A -= K */
151	if (!K)
152	break;
153	EMIT(PPC_RAW_ADDI(r_A, r_A, IMM_L(-K)));
154	if (K >= 32768)
155	EMIT(PPC_RAW_ADDIS(r_A, r_A, IMM_HA(-K)));
156	break;
157	case BPF_ALU \| BPF_MUL \| BPF_X: /* A = X; /
158	ctx->seen \|= SEEN_XREG;
159	EMIT(PPC_RAW_MULW(r_A, r_A, r_X));
160	break;
161	case BPF_ALU \| BPF_MUL \| BPF_K: /* A = K /
162	if (K < 32768)
163	EMIT(PPC_RAW_MULI(r_A, r_A, K));
164	else {
165	PPC_LI32(r_scratch1, K);
166	EMIT(PPC_RAW_MULW(r_A, r_A, r_scratch1));
167	}
168	break;
169	case BPF_ALU \| BPF_MOD \| BPF_X: /* A %= X; */
170	case BPF_ALU \| BPF_DIV \| BPF_X: /* A /= X; */
171	ctx->seen \|= SEEN_XREG;
172	EMIT(PPC_RAW_CMPWI(r_X, 0));
173	if (ctx->pc_ret0 != -1) {
174	PPC_BCC(COND_EQ, addrs[ctx->pc_ret0]);
175	} else {
176	PPC_BCC_SHORT(COND_NE, (ctx->idx*4)+12);
177	EMIT(PPC_RAW_LI(r_ret, 0));
178	PPC_JMP(exit_addr);
179	}
180	if (code == (BPF_ALU \| BPF_MOD \| BPF_X)) {
181	EMIT(PPC_RAW_DIVWU(r_scratch1, r_A, r_X));
182	EMIT(PPC_RAW_MULW(r_scratch1, r_X, r_scratch1));
183	EMIT(PPC_RAW_SUB(r_A, r_A, r_scratch1));
184	} else {
185	EMIT(PPC_RAW_DIVWU(r_A, r_A, r_X));
186	}
187	break;
188	case BPF_ALU \| BPF_MOD \| BPF_K: /* A %= K; */
189	PPC_LI32(r_scratch2, K);
190	EMIT(PPC_RAW_DIVWU(r_scratch1, r_A, r_scratch2));
191	EMIT(PPC_RAW_MULW(r_scratch1, r_scratch2, r_scratch1));
192	EMIT(PPC_RAW_SUB(r_A, r_A, r_scratch1));
193	break;
194	case BPF_ALU \| BPF_DIV \| BPF_K: /* A /= K */
195	if (K == 1)
196	break;
197	PPC_LI32(r_scratch1, K);
198	EMIT(PPC_RAW_DIVWU(r_A, r_A, r_scratch1));
199	break;
200	case BPF_ALU \| BPF_AND \| BPF_X:
201	ctx->seen \|= SEEN_XREG;
202	EMIT(PPC_RAW_AND(r_A, r_A, r_X));
203	break;
204	case BPF_ALU \| BPF_AND \| BPF_K:
205	if (!IMM_H(K))
206	EMIT(PPC_RAW_ANDI(r_A, r_A, K));
207	else {
208	PPC_LI32(r_scratch1, K);
209	EMIT(PPC_RAW_AND(r_A, r_A, r_scratch1));
210	}
211	break;
212	case BPF_ALU \| BPF_OR \| BPF_X:
213	ctx->seen \|= SEEN_XREG;
214	EMIT(PPC_RAW_OR(r_A, r_A, r_X));
215	break;
216	case BPF_ALU \| BPF_OR \| BPF_K:
217	if (IMM_L(K))
218	EMIT(PPC_RAW_ORI(r_A, r_A, IMM_L(K)));
219	if (K >= 65536)
220	EMIT(PPC_RAW_ORIS(r_A, r_A, IMM_H(K)));
221	break;
222	case BPF_ANC \| SKF_AD_ALU_XOR_X:
223	case BPF_ALU \| BPF_XOR \| BPF_X: /* A ^= X */
224	ctx->seen \|= SEEN_XREG;
225	EMIT(PPC_RAW_XOR(r_A, r_A, r_X));
226	break;
227	case BPF_ALU \| BPF_XOR \| BPF_K: /* A ^= K */
228	if (IMM_L(K))
229	EMIT(PPC_RAW_XORI(r_A, r_A, IMM_L(K)));
230	if (K >= 65536)
231	EMIT(PPC_RAW_XORIS(r_A, r_A, IMM_H(K)));
232	break;
233	case BPF_ALU \| BPF_LSH \| BPF_X: /* A <<= X; */
234	ctx->seen \|= SEEN_XREG;
235	EMIT(PPC_RAW_SLW(r_A, r_A, r_X));
236	break;
237	case BPF_ALU \| BPF_LSH \| BPF_K:
238	if (K == 0)
239	break;
240	else
241	EMIT(PPC_RAW_SLWI(r_A, r_A, K));
242	break;
243	case BPF_ALU \| BPF_RSH \| BPF_X: /* A >>= X; */
244	ctx->seen \|= SEEN_XREG;
245	EMIT(PPC_RAW_SRW(r_A, r_A, r_X));
246	break;
247	case BPF_ALU \| BPF_RSH \| BPF_K: /* A >>= K; */
248	if (K == 0)
249	break;
250	else
251	EMIT(PPC_RAW_SRWI(r_A, r_A, K));
252	break;
253	case BPF_ALU \| BPF_NEG:
254	EMIT(PPC_RAW_NEG(r_A, r_A));
255	break;
256	case BPF_RET \| BPF_K:
257	PPC_LI32(r_ret, K);
258	if (!K) {
259	if (ctx->pc_ret0 == -1)
260	ctx->pc_ret0 = i;
261	}
262	/*
263	* If this isn't the very last instruction, branch to
264	* the epilogue if we've stuff to clean up. Otherwise,
265	* if there's nothing to tidy, just return. If we /are/
266	* the last instruction, we're about to fall through to
267	* the epilogue to return.
268	*/
269	if (i != flen - 1) {
270	/*
271	* Note: 'seen' is properly valid only on pass
272	* #2. Both parts of this conditional are the
273	* same instruction size though, meaning the
274	* first pass will still correctly determine the
275	* code size/addresses.
276	*/
277	if (ctx->seen)
278	PPC_JMP(exit_addr);
279	else
280	EMIT(PPC_RAW_BLR());
281	}
282	break;
283	case BPF_RET \| BPF_A:
284	EMIT(PPC_RAW_MR(r_ret, r_A));
285	if (i != flen - 1) {
286	if (ctx->seen)
287	PPC_JMP(exit_addr);
288	else
289	EMIT(PPC_RAW_BLR());
290	}
291	break;
292	case BPF_MISC \| BPF_TAX: /* X = A */
293	EMIT(PPC_RAW_MR(r_X, r_A));
294	break;
295	case BPF_MISC \| BPF_TXA: /* A = X */
296	ctx->seen \|= SEEN_XREG;
297	EMIT(PPC_RAW_MR(r_A, r_X));
298	break;
299
300	/* Constant loads/M[] access */
301	case BPF_LD \| BPF_IMM: /* A = K */
302	PPC_LI32(r_A, K);
303	break;
304	case BPF_LDX \| BPF_IMM: /* X = K */
305	PPC_LI32(r_X, K);
306	break;
307	case BPF_LD \| BPF_MEM: /* A = mem[K] */
308	EMIT(PPC_RAW_MR(r_A, r_M + (K & 0xf)));
309	ctx->seen \|= SEEN_MEM \| (1<<(K & 0xf));
310	break;
311	case BPF_LDX \| BPF_MEM: /* X = mem[K] */
312	EMIT(PPC_RAW_MR(r_X, r_M + (K & 0xf)));
313	ctx->seen \|= SEEN_MEM \| (1<<(K & 0xf));
314	break;
315	case BPF_ST: /* mem[K] = A */
316	EMIT(PPC_RAW_MR(r_M + (K & 0xf), r_A));
317	ctx->seen \|= SEEN_MEM \| (1<<(K & 0xf));
318	break;
319	case BPF_STX: /* mem[K] = X */
320	EMIT(PPC_RAW_MR(r_M + (K & 0xf), r_X));
321	ctx->seen \|= SEEN_XREG \| SEEN_MEM \| (1<<(K & 0xf));
322	break;
323	case BPF_LD \| BPF_W \| BPF_LEN: /* A = skb->len; */
324	BUILD_BUG_ON(sizeof_field(struct sk_buff, len) != 4);
325	PPC_LWZ_OFFS(r_A, r_skb, offsetof(struct sk_buff, len));
326	break;
327	case BPF_LDX \| BPF_W \| BPF_ABS: /* A = ((u32 )(seccomp_data + K)); */
328	PPC_LWZ_OFFS(r_A, r_skb, K);
329	break;
330	case BPF_LDX \| BPF_W \| BPF_LEN: /* X = skb->len; */
331	PPC_LWZ_OFFS(r_X, r_skb, offsetof(struct sk_buff, len));
332	break;
333
334	/* Ancillary info loads */
335	case BPF_ANC \| SKF_AD_PROTOCOL: /* A = ntohs(skb->protocol); */
336	BUILD_BUG_ON(sizeof_field(struct sk_buff,
337	protocol) != 2);
338	PPC_NTOHS_OFFS(r_A, r_skb, offsetof(struct sk_buff,
339	protocol));
340	break;
341	case BPF_ANC \| SKF_AD_IFINDEX:
342	case BPF_ANC \| SKF_AD_HATYPE:
343	BUILD_BUG_ON(sizeof_field(struct net_device,
344	ifindex) != 4);
345	BUILD_BUG_ON(sizeof_field(struct net_device,
346	type) != 2);
347	PPC_LL_OFFS(r_scratch1, r_skb, offsetof(struct sk_buff,
348	dev));
349	EMIT(PPC_RAW_CMPDI(r_scratch1, 0));
350	if (ctx->pc_ret0 != -1) {
351	PPC_BCC(COND_EQ, addrs[ctx->pc_ret0]);
352	} else {
353	/* Exit, returning 0; first pass hits here. */
354	PPC_BCC_SHORT(COND_NE, ctx->idx * 4 + 12);
355	EMIT(PPC_RAW_LI(r_ret, 0));
356	PPC_JMP(exit_addr);
357	}
358	if (code == (BPF_ANC \| SKF_AD_IFINDEX)) {
359	PPC_LWZ_OFFS(r_A, r_scratch1,
360	offsetof(struct net_device, ifindex));
361	} else {
362	PPC_LHZ_OFFS(r_A, r_scratch1,
363	offsetof(struct net_device, type));
364	}
365
366	break;
367	case BPF_ANC \| SKF_AD_MARK:
368	BUILD_BUG_ON(sizeof_field(struct sk_buff, mark) != 4);
369	PPC_LWZ_OFFS(r_A, r_skb, offsetof(struct sk_buff,
370	mark));
371	break;
372	case BPF_ANC \| SKF_AD_RXHASH:
373	BUILD_BUG_ON(sizeof_field(struct sk_buff, hash) != 4);
374	PPC_LWZ_OFFS(r_A, r_skb, offsetof(struct sk_buff,
375	hash));
376	break;
377	case BPF_ANC \| SKF_AD_VLAN_TAG:
378	BUILD_BUG_ON(sizeof_field(struct sk_buff, vlan_tci) != 2);
379
380	PPC_LHZ_OFFS(r_A, r_skb, offsetof(struct sk_buff,
381	vlan_tci));
382	break;
383	case BPF_ANC \| SKF_AD_VLAN_TAG_PRESENT:
384	PPC_LBZ_OFFS(r_A, r_skb, PKT_VLAN_PRESENT_OFFSET());
385	if (PKT_VLAN_PRESENT_BIT)
386	EMIT(PPC_RAW_SRWI(r_A, r_A, PKT_VLAN_PRESENT_BIT));
387	if (PKT_VLAN_PRESENT_BIT < 7)
388	EMIT(PPC_RAW_ANDI(r_A, r_A, 1));
389	break;
390	case BPF_ANC \| SKF_AD_QUEUE:
391	BUILD_BUG_ON(sizeof_field(struct sk_buff,
392	queue_mapping) != 2);
393	PPC_LHZ_OFFS(r_A, r_skb, offsetof(struct sk_buff,
394	queue_mapping));
395	break;
396	case BPF_ANC \| SKF_AD_PKTTYPE:
397	PPC_LBZ_OFFS(r_A, r_skb, PKT_TYPE_OFFSET());
398	EMIT(PPC_RAW_ANDI(r_A, r_A, PKT_TYPE_MAX));
399	EMIT(PPC_RAW_SRWI(r_A, r_A, 5));
400	break;
401	case BPF_ANC \| SKF_AD_CPU:
402	PPC_BPF_LOAD_CPU(r_A);
403	break;
404	/* Absolute loads from packet header/data */
405	case BPF_LD \| BPF_W \| BPF_ABS:
406	func = CHOOSE_LOAD_FUNC(K, sk_load_word);
407	goto common_load;
408	case BPF_LD \| BPF_H \| BPF_ABS:
409	func = CHOOSE_LOAD_FUNC(K, sk_load_half);
410	goto common_load;
411	case BPF_LD \| BPF_B \| BPF_ABS:
412	func = CHOOSE_LOAD_FUNC(K, sk_load_byte);
413	common_load:
414	/* Load from [K]. */
415	ctx->seen \|= SEEN_DATAREF;
416	PPC_FUNC_ADDR(r_scratch1, func);
417	EMIT(PPC_RAW_MTLR(r_scratch1));
418	PPC_LI32(r_addr, K);
419	EMIT(PPC_RAW_BLRL());
420	/*	54	/*
421	* Helper returns 'lt' condition on error, and an	55	* Save ctx->idx as this would currently point to the
422	* appropriate return value in r3	56	* end of the JITed image and set it to the offset of
		57	* the instruction sequence corresponding to the
		58	* subprog call temporarily.
423	*/	59	*/
424	PPC_BCC(COND_LT, exit_addr);	60	tmp_idx = ctx->idx;
425	break;	61	ctx->idx = addrs[i] / 4;
426		62	bpf_jit_emit_func_call_rel(image, ctx, func_addr);
427	/* Indirect loads from packet header/data */	63
428	case BPF_LD \| BPF_W \| BPF_IND:
429	func = sk_load_word;
430	goto common_load_ind;
431	case BPF_LD \| BPF_H \| BPF_IND:
432	func = sk_load_half;
433	goto common_load_ind;
434	case BPF_LD \| BPF_B \| BPF_IND:
435	func = sk_load_byte;
436	common_load_ind:
437	/*	64	/*
438	* Load from [X + K]. Negative offsets are tested for	65	* Restore ctx->idx here. This is safe as the length
439	* in the helper functions.	66	* of the JITed sequence remains unchanged.
440	*/
441	ctx->seen \|= SEEN_DATAREF \| SEEN_XREG;
442	PPC_FUNC_ADDR(r_scratch1, func);
443	EMIT(PPC_RAW_MTLR(r_scratch1));
444	EMIT(PPC_RAW_ADDI(r_addr, r_X, IMM_L(K)));
445	if (K >= 32768)
446	EMIT(PPC_RAW_ADDIS(r_addr, r_addr, IMM_HA(K)));
447	EMIT(PPC_RAW_BLRL());
448	/* If error, cr0.LT set */
449	PPC_BCC(COND_LT, exit_addr);
450	break;
451
452	case BPF_LDX \| BPF_B \| BPF_MSH:
453	func = CHOOSE_LOAD_FUNC(K, sk_load_byte_msh);
454	goto common_load;
455	break;
456
457	/* Jump and branches */
458	case BPF_JMP \| BPF_JA:
459	if (K != 0)
460	PPC_JMP(addrs[i + 1 + K]);
461	break;
462
463	case BPF_JMP \| BPF_JGT \| BPF_K:
464	case BPF_JMP \| BPF_JGT \| BPF_X:
465	true_cond = COND_GT;
466	goto cond_branch;
467	case BPF_JMP \| BPF_JGE \| BPF_K:
468	case BPF_JMP \| BPF_JGE \| BPF_X:
469	true_cond = COND_GE;
470	goto cond_branch;
471	case BPF_JMP \| BPF_JEQ \| BPF_K:
472	case BPF_JMP \| BPF_JEQ \| BPF_X:
473	true_cond = COND_EQ;
474	goto cond_branch;
475	case BPF_JMP \| BPF_JSET \| BPF_K:
476	case BPF_JMP \| BPF_JSET \| BPF_X:
477	true_cond = COND_NE;
478	cond_branch:
479	/* same targets, can avoid doing the test :) */
480	if (filter[i].jt == filter[i].jf) {
481	if (filter[i].jt > 0)
482	PPC_JMP(addrs[i + 1 + filter[i].jt]);
483	break;
484	}
485
486	switch (code) {
487	case BPF_JMP \| BPF_JGT \| BPF_X:
488	case BPF_JMP \| BPF_JGE \| BPF_X:
489	case BPF_JMP \| BPF_JEQ \| BPF_X:
490	ctx->seen \|= SEEN_XREG;
491	EMIT(PPC_RAW_CMPLW(r_A, r_X));
492	break;
493	case BPF_JMP \| BPF_JSET \| BPF_X:
494	ctx->seen \|= SEEN_XREG;
495	EMIT(PPC_RAW_AND_DOT(r_scratch1, r_A, r_X));
496	break;
497	case BPF_JMP \| BPF_JEQ \| BPF_K:
498	case BPF_JMP \| BPF_JGT \| BPF_K:
499	case BPF_JMP \| BPF_JGE \| BPF_K:
500	if (K < 32768)
501	EMIT(PPC_RAW_CMPLWI(r_A, K));
502	else {
503	PPC_LI32(r_scratch1, K);
504	EMIT(PPC_RAW_CMPLW(r_A, r_scratch1));
505	}
506	break;
507	case BPF_JMP \| BPF_JSET \| BPF_K:
508	if (K < 32768)
509	/* PPC_ANDI is /only/ dot-form */
510	EMIT(PPC_RAW_ANDI(r_scratch1, r_A, K));
511	else {
512	PPC_LI32(r_scratch1, K);
513	EMIT(PPC_RAW_AND_DOT(r_scratch1, r_A,
514	r_scratch1));
515	}
516	break;
517	}
518	/* Sometimes branches are constructed "backward", with
519	* the false path being the branch and true path being
520	* a fallthrough to the next instruction.
521	*/	67	*/
522	if (filter[i].jt == 0)	68	ctx->idx = tmp_idx;
523	/* Swap the sense of the branch */
524	PPC_BCC(true_cond ^ COND_CMP_TRUE,
525	addrs[i + 1 + filter[i].jf]);
526	else {
527	PPC_BCC(true_cond, addrs[i + 1 + filter[i].jt]);
528	if (filter[i].jf != 0)
529	PPC_JMP(addrs[i + 1 + filter[i].jf]);
530	}
531	break;
532	default:
533	/* The filter contains something cruel & unusual.
534	* We don't handle it, but also there shouldn't be
535	* anything missing from our list.
536	*/
537	if (printk_ratelimit())
538	pr_err("BPF filter opcode %04x (@%d) unsupported\n",
539	filter[i].code, i);
540	return -ENOTSUPP;
541	}	69	}
542
543	}	70	}
544	/* Set end-of-body-code address for exit. */
545	addrs[i] = ctx->idx * 4;
546		71
547	return 0;	72	return 0;
548	}	73	}
549		74
550	void bpf_jit_compile(struct bpf_prog *fp)	75	struct powerpc64_jit_data {
		76	struct bpf_binary_header *header;
		77	u32 *addrs;
		78	u8 *image;
		79	u32 proglen;
		80	struct codegen_context ctx;
		81	};
		82
		83	bool bpf_jit_needs_zext(void)
		84	{
		85	return true;
		86	}
		87
		88	struct bpf_prog bpf_int_jit_compile(struct bpf_prog fp)
551	{	89	{
552	unsigned int proglen;	90	u32 proglen;
553	unsigned int alloclen;	91	u32 alloclen;
554	u32 *image = NULL;	92	u8 *image = NULL;
555	u32 *code_base;	93	u32 *code_base;
556	unsigned int *addrs;	94	u32 *addrs;
		95	struct powerpc64_jit_data *jit_data;
557	struct codegen_context cgctx;	96	struct codegen_context cgctx;
558	int pass;	97	int pass;
559	int flen = fp->len;	98	int flen;
		99	struct bpf_binary_header *bpf_hdr;
		100	struct bpf_prog *org_fp = fp;
		101	struct bpf_prog *tmp_fp;
		102	bool bpf_blinded = false;
		103	bool extra_pass = false;
		104
		105	if (!fp->jit_requested)
		106	return org_fp;
		107
		108	tmp_fp = bpf_jit_blind_constants(org_fp);
		109	if (IS_ERR(tmp_fp))
		110	return org_fp;
		111
		112	if (tmp_fp != org_fp) {
		113	bpf_blinded = true;
		114	fp = tmp_fp;
		115	}
		116
		117	jit_data = fp->aux->jit_data;
		118	if (!jit_data) {
		119	jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL);
		120	if (!jit_data) {
		121	fp = org_fp;
		122	goto out;
		123	}
		124	fp->aux->jit_data = jit_data;
		125	}
560		126
561	if (!bpf_jit_enable)	127	flen = fp->len;
562	return;	128	addrs = jit_data->addrs;
		129	if (addrs) {
		130	cgctx = jit_data->ctx;
		131	image = jit_data->image;
		132	bpf_hdr = jit_data->header;
		133	proglen = jit_data->proglen;
		134	alloclen = proglen + FUNCTION_DESCR_SIZE;
		135	extra_pass = true;
		136	goto skip_init_ctx;
		137	}
563		138
564	addrs = kcalloc(flen + 1, sizeof(*addrs), GFP_KERNEL);	139	addrs = kcalloc(flen + 1, sizeof(*addrs), GFP_KERNEL);
565	if (addrs == NULL)	140	if (addrs == NULL) {
566	return;	141	fp = org_fp;
		142	goto out_addrs;
		143	}
567		144
568	/*	145	memset(&cgctx, 0, sizeof(struct codegen_context));
569	* There are multiple assembly passes as the generated code will change	146	memcpy(cgctx.b2p, b2p, sizeof(cgctx.b2p));
570	* size as it settles down, figuring out the max branch offsets/exit	147
571	* paths required.	148	/* Make sure that the stack is quadword aligned. */
572	*	149	cgctx.stack_size = round_up(fp->aux->stack_depth, 16);
573	* The range of standard conditional branches is +/- 32Kbytes. Since
574	* BPF_MAXINSNS = 4096, we can only jump from (worst case) start to
575	* finish with 8 bytes/instruction. Not feasible, so long jumps are
576	* used, distinct from short branches.
577	*
578	* Current:
579	*
580	* For now, both branch types assemble to 2 words (short branches padded
581	* with a NOP); this is less efficient, but assembly will always complete
582	* after exactly 3 passes:
583	*
584	* First pass: No code buffer; Program is "faux-generated" -- no code
585	* emitted but maximum size of output determined (and addrs[] filled
586	* in). Also, we note whether we use M[], whether we use skb data, etc.
587	* All generation choices assumed to be 'worst-case', e.g. branches all
588	* far (2 instructions), return path code reduction not available, etc.
589	*
590	* Second pass: Code buffer allocated with size determined previously.
591	* Prologue generated to support features we have seen used. Exit paths
592	* determined and addrs[] is filled in again, as code may be slightly
593	* smaller as a result.
594	*
595	* Third pass: Code generated 'for real', and branch destinations
596	* determined from now-accurate addrs[] map.
597	*
598	* Ideal:
599	*
600	* If we optimise this, near branches will be shorter. On the
601	* first assembly pass, we should err on the side of caution and
602	* generate the biggest code. On subsequent passes, branches will be
603	* generated short or long and code size will reduce. With smaller
604	* code, more branches may fall into the short category, and code will
605	* reduce more.
606	*
607	* Finally, if we see one pass generate code the same size as the
608	* previous pass we have converged and should now generate code for
609	* real. Allocating at the end will also save the memory that would
610	* otherwise be wasted by the (small) current code shrinkage.
611	* Preferably, we should do a small number of passes (e.g. 5) and if we
612	* haven't converged by then, get impatient and force code to generate
613	* as-is, even if the odd branch would be left long. The chances of a
614	* long jump are tiny with all but the most enormous of BPF filter
615	* inputs, so we should usually converge on the third pass.
616	*/
617		150
618	cgctx.idx = 0;
619	cgctx.seen = 0;
620	cgctx.pc_ret0 = -1;
621	/* Scouting faux-generate pass 0 */	151	/* Scouting faux-generate pass 0 */
622	if (bpf_jit_build_body(fp, 0, &cgctx, addrs))	152	if (bpf_jit_build_body(fp, 0, &cgctx, addrs, false)) {
623	/* We hit something illegal or unsupported. */	153	/* We hit something illegal or unsupported. */
624	goto out;	154	fp = org_fp;
		155	goto out_addrs;
		156	}
		157
		158	/*
		159	* If we have seen a tail call, we need a second pass.
		160	* This is because bpf_jit_emit_common_epilogue() is called
		161	* from bpf_jit_emit_tail_call() with a not yet stable ctx->seen.
		162	*/
		163	if (cgctx.seen & SEEN_TAILCALL) {
		164	cgctx.idx = 0;
		165	if (bpf_jit_build_body(fp, 0, &cgctx, addrs, false)) {
		166	fp = org_fp;
		167	goto out_addrs;
		168	}
		169	}
625		170
		171	bpf_jit_realloc_regs(&cgctx);
626	/*	172	/*
627	* Pretend to build prologue, given the features we've seen. This will	173	* Pretend to build prologue, given the features we've seen. This will
628	* update ctgtx.idx as it pretends to output instructions, then we can	174	* update ctgtx.idx as it pretends to output instructions, then we can
629	* calculate total size from idx.	175	* calculate total size from idx.
630	*/	176	*/
631	bpf_jit_build_prologue(fp, 0, &cgctx);	177	bpf_jit_build_prologue(0, &cgctx);
632	bpf_jit_build_epilogue(0, &cgctx);	178	bpf_jit_build_epilogue(0, &cgctx);
633		179
634	proglen = cgctx.idx * 4;	180	proglen = cgctx.idx * 4;
635	alloclen = proglen + FUNCTION_DESCR_SIZE;	181	alloclen = proglen + FUNCTION_DESCR_SIZE;
636	image = module_alloc(alloclen);
637	if (!image)
638	goto out;
639		182
640	code_base = image + (FUNCTION_DESCR_SIZE/4);	183	bpf_hdr = bpf_jit_binary_alloc(alloclen, &image, 4, bpf_jit_fill_ill_insns);
		184	if (!bpf_hdr) {
		185	fp = org_fp;
		186	goto out_addrs;
		187	}
		188
		189	skip_init_ctx:
		190	code_base = (u32 *)(image + FUNCTION_DESCR_SIZE);
		191
		192	if (extra_pass) {
		193	/*
		194	* Do not touch the prologue and epilogue as they will remain
		195	* unchanged. Only fix the branch target address for subprog
		196	* calls in the body.
		197	*
		198	* This does not change the offsets and lengths of the subprog
		199	* call instruction sequences and hence, the size of the JITed
		200	* image as well.
		201	*/
		202	bpf_jit_fixup_subprog_calls(fp, code_base, &cgctx, addrs);
		203
		204	/* There is no need to perform the usual passes. */
		205	goto skip_codegen_passes;
		206	}
641		207
642	/* Code generation passes 1-2 */	208	/* Code generation passes 1-2 */
643	for (pass = 1; pass < 3; pass++) {	209	for (pass = 1; pass < 3; pass++) {
644	/* Now build the prologue, body code & epilogue for real. */	210	/* Now build the prologue, body code & epilogue for real. */
645	cgctx.idx = 0;	211	cgctx.idx = 0;
646	bpf_jit_build_prologue(fp, code_base, &cgctx);	212	bpf_jit_build_prologue(code_base, &cgctx);
647	bpf_jit_build_body(fp, code_base, &cgctx, addrs);	213	bpf_jit_build_body(fp, code_base, &cgctx, addrs, extra_pass);
648	bpf_jit_build_epilogue(code_base, &cgctx);	214	bpf_jit_build_epilogue(code_base, &cgctx);
649		215
650	if (bpf_jit_enable > 1)	216	if (bpf_jit_enable > 1)
@@ -652,15 +218,15 @@ void bpf_jit_compile(struct bpf_prog *fp) · 1 unmodified line
652	proglen - (cgctx.idx * 4), cgctx.seen);	218	proglen - (cgctx.idx * 4), cgctx.seen);
653	}	219	}
654		220
		221	skip_codegen_passes:
655	if (bpf_jit_enable > 1)	222	if (bpf_jit_enable > 1)
656	/* Note that we output the base address of the code_base	223	/*
		224	* Note that we output the base address of the code_base
657	* rather than image, since opcodes are in code_base.	225	* rather than image, since opcodes are in code_base.
658	*/	226	*/
659	bpf_jit_dump(flen, proglen, pass, code_base);	227	bpf_jit_dump(flen, proglen, pass, code_base);
660		228
661	bpf_flush_icache(code_base, code_base + (proglen/4));	229	#ifdef PPC64_ELF_ABI_v1
662
663	#ifdef CONFIG_PPC64
664	/* Function descriptor nastiness: Address + TOC */	230	/* Function descriptor nastiness: Address + TOC */
665	((u64 *)image)[0] = (u64)code_base;	231	((u64 *)image)[0] = (u64)code_base;
666	((u64 *)image)[1] = local_paca->kernel_toc;	232	((u64 *)image)[1] = local_paca->kernel_toc;
@@ -668,16 +234,38 @@ void bpf_jit_compile(struct bpf_prog *fp) · 1 unmodified line
668		234
669	fp->bpf_func = (void *)image;	235	fp->bpf_func = (void *)image;
670	fp->jited = 1;	236	fp->jited = 1;
		237	fp->jited_len = alloclen;
		238
		239	bpf_flush_icache(bpf_hdr, (u8 )bpf_hdr + (bpf_hdr->pages PAGE_SIZE));
		240	if (!fp->is_func \|\| extra_pass) {
		241	bpf_prog_fill_jited_linfo(fp, addrs);
		242	out_addrs:
		243	kfree(addrs);
		244	kfree(jit_data);
		245	fp->aux->jit_data = NULL;
		246	} else {
		247	jit_data->addrs = addrs;
		248	jit_data->ctx = cgctx;
		249	jit_data->proglen = proglen;
		250	jit_data->image = image;
		251	jit_data->header = bpf_hdr;
		252	}
671		253
672	out:	254	out:
673	kfree(addrs);	255	if (bpf_blinded)
674	return;	256	bpf_jit_prog_release_other(fp, fp == org_fp ? tmp_fp : org_fp);
		257
		258	return fp;
675	}	259	}
676		260
		261	/* Overriding bpf_jit_free() as we don't set images read-only. */
677	void bpf_jit_free(struct bpf_prog *fp)	262	void bpf_jit_free(struct bpf_prog *fp)
678	{	263	{
		264	unsigned long addr = (unsigned long)fp->bpf_func & PAGE_MASK;
		265	struct bpf_binary_header bpf_hdr = (void )addr;
		266
679	if (fp->jited)	267	if (fp->jited)
680	module_memfree(fp->bpf_func);	268	bpf_jit_binary_free(bpf_hdr);
681		269
682	bpf_prog_unlock_free(fp);	270	bpf_prog_unlock_free(fp);
683	}	271	}

arch/powerpc/net/bpf_jit_comp32.c+1100−0

@@ -0,0 +1,1100 @@
	1	// SPDX-License-Identifier: GPL-2.0-only
	2	/*
	3	* eBPF JIT compiler for PPC32
	4	*
	5	* Copyright 2020 Christophe Leroy <christophe.leroy@csgroup.eu>
	6	* CS GROUP France
	7	*
	8	* Based on PPC64 eBPF JIT compiler by Naveen N. Rao
	9	*/
	10	#include <linux/moduleloader.h>
	11	#include <asm/cacheflush.h>
	12	#include <asm/asm-compat.h>
	13	#include <linux/netdevice.h>
	14	#include <linux/filter.h>
	15	#include <linux/if_vlan.h>
	16	#include <asm/kprobes.h>
	17	#include <linux/bpf.h>
	18
	19	#include "bpf_jit.h"
	20
	21	/*
	22	* Stack layout:
	23	*
	24	* [ prev sp ] <-------------
	25	* [ nv gpr save area ] 16 * 4 \|
	26	* fp (r31) --> [ ebpf stack space ] upto 512 \|
	27	* [ frame header ] 16 \|
	28	* sp (r1) ---> [ stack pointer ] --------------
	29	*/
	30
	31	/* for gpr non volatile registers r17 to r31 (14) + tail call */
	32	#define BPF_PPC_STACK_SAVE (15 * 4 + 4)
	33	/* stack frame, ensure this is quadword aligned */
	34	#define BPF_PPC_STACKFRAME(ctx) (STACK_FRAME_MIN_SIZE + BPF_PPC_STACK_SAVE + (ctx)->stack_size)
	35
	36	/* BPF register usage */
	37	#define TMP_REG (MAX_BPF_JIT_REG + 0)
	38
	39	/* BPF to ppc register mappings */
	40	const int b2p[MAX_BPF_JIT_REG + 1] = {
	41	/* function return value */
	42	[BPF_REG_0] = 12,
	43	/* function arguments */
	44	[BPF_REG_1] = 4,
	45	[BPF_REG_2] = 6,
	46	[BPF_REG_3] = 8,
	47	[BPF_REG_4] = 10,
	48	[BPF_REG_5] = 22,
	49	/* non volatile registers */
	50	[BPF_REG_6] = 24,
	51	[BPF_REG_7] = 26,
	52	[BPF_REG_8] = 28,
	53	[BPF_REG_9] = 30,
	54	/* frame pointer aka BPF_REG_10 */
	55	[BPF_REG_FP] = 18,
	56	/* eBPF jit internal registers */
	57	[BPF_REG_AX] = 20,
	58	[TMP_REG] = 31, /* 32 bits */
	59	};
	60
	61	static int bpf_to_ppc(struct codegen_context *ctx, int reg)
	62	{
	63	return ctx->b2p[reg];
	64	}
	65
	66	/* PPC NVR range -- update this if we ever use NVRs below r17 */
	67	#define BPF_PPC_NVR_MIN 17
	68	#define BPF_PPC_TC 16
	69
	70	static int bpf_jit_stack_offsetof(struct codegen_context *ctx, int reg)
	71	{
	72	if ((reg >= BPF_PPC_NVR_MIN && reg < 32) \|\| reg == BPF_PPC_TC)
	73	return BPF_PPC_STACKFRAME(ctx) - 4 * (32 - reg);
	74
	75	WARN(true, "BPF JIT is asking about unknown registers, will crash the stack");
	76	/* Use the hole we have left for alignment */
	77	return BPF_PPC_STACKFRAME(ctx) - 4;
	78	}
	79
	80	void bpf_jit_realloc_regs(struct codegen_context *ctx)
	81	{
	82	if (ctx->seen & SEEN_FUNC)
	83	return;
	84
	85	while (ctx->seen & SEEN_NVREG_MASK &&
	86	(ctx->seen & SEEN_VREG_MASK) != SEEN_VREG_MASK) {
	87	int old = 32 - fls(ctx->seen & (SEEN_NVREG_MASK & 0xaaaaaaab));
	88	int new = 32 - fls(~ctx->seen & (SEEN_VREG_MASK & 0xaaaaaaaa));
	89	int i;
	90
	91	for (i = BPF_REG_0; i <= TMP_REG; i++) {
	92	if (ctx->b2p[i] != old)
	93	continue;
	94	ctx->b2p[i] = new;
	95	bpf_set_seen_register(ctx, new);
	96	bpf_clear_seen_register(ctx, old);
	97	if (i != TMP_REG) {
	98	bpf_set_seen_register(ctx, new - 1);
	99	bpf_clear_seen_register(ctx, old - 1);
	100	}
	101	break;
	102	}
	103	}
	104	}
	105
	106	void bpf_jit_build_prologue(u32 image, struct codegen_context ctx)
	107	{
	108	int i;
	109
	110	/* First arg comes in as a 32 bits pointer. */
	111	EMIT(PPC_RAW_MR(bpf_to_ppc(ctx, BPF_REG_1), __REG_R3));
	112	EMIT(PPC_RAW_LI(bpf_to_ppc(ctx, BPF_REG_1) - 1, 0));
	113	EMIT(PPC_RAW_STWU(__REG_R1, __REG_R1, -BPF_PPC_STACKFRAME(ctx)));
	114
	115	/*
	116	* Initialize tail_call_cnt in stack frame if we do tail calls.
	117	* Otherwise, put in NOPs so that it can be skipped when we are
	118	* invoked through a tail call.
	119	*/
	120	if (ctx->seen & SEEN_TAILCALL) {
	121	EMIT(PPC_RAW_STW(bpf_to_ppc(ctx, BPF_REG_1) - 1, __REG_R1, bpf_jit_stack_offsetof(ctx, BPF_PPC_TC)));
	122	} else {
	123	EMIT(PPC_RAW_NOP());
	124	}
	125
	126	#define BPF_TAILCALL_PROLOGUE_SIZE 16
	127
	128	/*
	129	* We need a stack frame, but we don't necessarily need to
	130	* save/restore LR unless we call other functions
	131	*/
	132	if (ctx->seen & SEEN_FUNC)
	133	EMIT(PPC_RAW_MFLR(__REG_R0));
	134
	135	/*
	136	* Back up non-volatile regs -- registers r18-r31
	137	*/
	138	for (i = BPF_PPC_NVR_MIN; i <= 31; i++)
	139	if (bpf_is_seen_register(ctx, i))
	140	EMIT(PPC_RAW_STW(i, __REG_R1, bpf_jit_stack_offsetof(ctx, i)));
	141
	142	/* If needed retrieve arguments 9 and 10, ie 5th 64 bits arg.*/
	143	if (bpf_is_seen_register(ctx, bpf_to_ppc(ctx, BPF_REG_5))) {
	144	EMIT(PPC_RAW_LWZ(bpf_to_ppc(ctx, BPF_REG_5) - 1, __REG_R1, BPF_PPC_STACKFRAME(ctx)) + 8);
	145	EMIT(PPC_RAW_LWZ(bpf_to_ppc(ctx, BPF_REG_5), __REG_R1, BPF_PPC_STACKFRAME(ctx)) + 12);
	146	}
	147
	148	/* Setup frame pointer to point to the bpf stack area */
	149	if (bpf_is_seen_register(ctx, bpf_to_ppc(ctx, BPF_REG_FP))) {
	150	EMIT(PPC_RAW_LI(bpf_to_ppc(ctx, BPF_REG_FP) - 1, 0));
	151	EMIT(PPC_RAW_ADDI(bpf_to_ppc(ctx, BPF_REG_FP), __REG_R1,
	152	STACK_FRAME_MIN_SIZE + ctx->stack_size));
	153	}
	154
	155	if (ctx->seen & SEEN_FUNC)
	156	EMIT(PPC_RAW_STW(__REG_R0, __REG_R1, BPF_PPC_STACKFRAME(ctx) + PPC_LR_STKOFF));
	157	}
	158
	159	static void bpf_jit_emit_common_epilogue(u32 image, struct codegen_context ctx)
	160	{
	161	int i;
	162
	163	/* Restore NVRs */
	164	for (i = BPF_PPC_NVR_MIN; i <= 31; i++)
	165	if (bpf_is_seen_register(ctx, i))
	166	EMIT(PPC_RAW_LWZ(i, __REG_R1, bpf_jit_stack_offsetof(ctx, i)));
	167	}
	168
	169	void bpf_jit_build_epilogue(u32 image, struct codegen_context ctx)
	170	{
	171	EMIT(PPC_RAW_MR(__REG_R3, bpf_to_ppc(ctx, BPF_REG_0)));
	172
	173	bpf_jit_emit_common_epilogue(image, ctx);
	174
	175	/* Tear down our stack frame */
	176
	177	if (ctx->seen & SEEN_FUNC)
	178	EMIT(PPC_RAW_LWZ(__REG_R0, __REG_R1, BPF_PPC_STACKFRAME(ctx) + PPC_LR_STKOFF));
	179
	180	EMIT(PPC_RAW_ADDI(__REG_R1, __REG_R1, BPF_PPC_STACKFRAME(ctx)));
	181
	182	if (ctx->seen & SEEN_FUNC)
	183	EMIT(PPC_RAW_MTLR(__REG_R0));
	184
	185	EMIT(PPC_RAW_BLR());
	186	}
	187
	188	void bpf_jit_emit_func_call_rel(u32 image, struct codegen_context ctx, u64 func)
	189	{
	190	s32 rel = (s32)func - (s32)(image + ctx->idx);
	191
	192	if (image && rel < 0x2000000 && rel >= -0x2000000) {
	193	PPC_BL_ABS(func);
	194	} else {
	195	/* Load function address into r0 */
	196	EMIT(PPC_RAW_LIS(__REG_R0, IMM_H(func)));
	197	EMIT(PPC_RAW_ORI(__REG_R0, __REG_R0, IMM_L(func)));
	198	EMIT(PPC_RAW_MTLR(__REG_R0));
	199	EMIT(PPC_RAW_BLRL());
	200	}
	201	}
	202
	203	static void bpf_jit_emit_tail_call(u32 image, struct codegen_context ctx, u32 out)
	204	{
	205	/*
	206	* By now, the eBPF program has already setup parameters in r3-r6
	207	* r3-r4/BPF_REG_1 - pointer to ctx -- passed as is to the next bpf program
	208	* r5-r6/BPF_REG_2 - pointer to bpf_array
	209	* r7-r8/BPF_REG_3 - index in bpf_array
	210	*/
	211	int b2p_bpf_array = bpf_to_ppc(ctx, BPF_REG_2);
	212	int b2p_index = bpf_to_ppc(ctx, BPF_REG_3);
	213
	214	/*
	215	* if (index >= array->map.max_entries)
	216	* goto out;
	217	*/
	218	EMIT(PPC_RAW_LWZ(__REG_R0, b2p_bpf_array, offsetof(struct bpf_array, map.max_entries)));
	219	EMIT(PPC_RAW_CMPLW(b2p_index, __REG_R0));
	220	EMIT(PPC_RAW_LWZ(__REG_R0, __REG_R1, bpf_jit_stack_offsetof(ctx, BPF_PPC_TC)));
	221	PPC_BCC(COND_GE, out);
	222
	223	/*
	224	* if (tail_call_cnt > MAX_TAIL_CALL_CNT)
	225	* goto out;
	226	*/
	227	EMIT(PPC_RAW_CMPLWI(__REG_R0, MAX_TAIL_CALL_CNT));
	228	/* tail_call_cnt++; */
	229	EMIT(PPC_RAW_ADDIC(__REG_R0, __REG_R0, 1));
	230	PPC_BCC(COND_GT, out);
	231
	232	/* prog = array->ptrs[index]; */
	233	EMIT(PPC_RAW_RLWINM(__REG_R3, b2p_index, 2, 0, 29));
	234	EMIT(PPC_RAW_ADD(__REG_R3, __REG_R3, b2p_bpf_array));
	235	EMIT(PPC_RAW_LWZ(__REG_R3, __REG_R3, offsetof(struct bpf_array, ptrs)));
	236	EMIT(PPC_RAW_STW(__REG_R0, __REG_R1, bpf_jit_stack_offsetof(ctx, BPF_PPC_TC)));
	237
	238	/*
	239	* if (prog == NULL)
	240	* goto out;
	241	*/
	242	EMIT(PPC_RAW_CMPLWI(__REG_R3, 0));
	243	PPC_BCC(COND_EQ, out);
	244
	245	/* goto (prog->bpf_func + prologue_size); /
	246	EMIT(PPC_RAW_LWZ(__REG_R3, __REG_R3, offsetof(struct bpf_prog, bpf_func)));
	247
	248	if (ctx->seen & SEEN_FUNC)
	249	EMIT(PPC_RAW_LWZ(__REG_R0, __REG_R1, BPF_PPC_STACKFRAME(ctx) + PPC_LR_STKOFF));
	250
	251	EMIT(PPC_RAW_ADDIC(__REG_R3, __REG_R3, BPF_TAILCALL_PROLOGUE_SIZE));
	252
	253	if (ctx->seen & SEEN_FUNC)
	254	EMIT(PPC_RAW_MTLR(__REG_R0));
	255
	256	EMIT(PPC_RAW_MTCTR(__REG_R3));
	257
	258	EMIT(PPC_RAW_MR(__REG_R3, bpf_to_ppc(ctx, BPF_REG_1)));
	259
	260	/* tear restore NVRs, ... */
	261	bpf_jit_emit_common_epilogue(image, ctx);
	262
	263	EMIT(PPC_RAW_BCTR());
	264	/* out: */
	265	}
	266
	267	/* Assemble the body code between the prologue & epilogue */
	268	int bpf_jit_build_body(struct bpf_prog fp, u32 image, struct codegen_context *ctx,
	269	u32 *addrs, bool extra_pass)
	270	{
	271	const struct bpf_insn *insn = fp->insnsi;
	272	int flen = fp->len;
	273	int i, ret;
	274
	275	/* Start of epilogue code - will only be valid 2nd pass onwards */
	276	u32 exit_addr = addrs[flen];
	277
	278	for (i = 0; i < flen; i++) {
	279	u32 code = insn[i].code;
	280	u32 dst_reg = bpf_to_ppc(ctx, insn[i].dst_reg);
	281	u32 dst_reg_h = dst_reg - 1;
	282	u32 src_reg = bpf_to_ppc(ctx, insn[i].src_reg);
	283	u32 src_reg_h = src_reg - 1;
	284	u32 tmp_reg = bpf_to_ppc(ctx, TMP_REG);
	285	s16 off = insn[i].off;
	286	s32 imm = insn[i].imm;
	287	bool func_addr_fixed;
	288	u64 func_addr;
	289	u32 true_cond;
	290
	291	/*
	292	* addrs[] maps a BPF bytecode address into a real offset from
	293	* the start of the body code.
	294	*/
	295	addrs[i] = ctx->idx * 4;
	296
	297	/*
	298	* As an optimization, we note down which registers
	299	* are used so that we can only save/restore those in our
	300	* prologue and epilogue. We do this here regardless of whether
	301	* the actual BPF instruction uses src/dst registers or not
	302	* (for instance, BPF_CALL does not use them). The expectation
	303	* is that those instructions will have src_reg/dst_reg set to
	304	* 0. Even otherwise, we just lose some prologue/epilogue
	305	* optimization but everything else should work without
	306	* any issues.
	307	*/
	308	if (dst_reg >= 3 && dst_reg < 32) {
	309	bpf_set_seen_register(ctx, dst_reg);
	310	bpf_set_seen_register(ctx, dst_reg_h);
	311	}
	312
	313	if (src_reg >= 3 && src_reg < 32) {
	314	bpf_set_seen_register(ctx, src_reg);
	315	bpf_set_seen_register(ctx, src_reg_h);
	316	}
	317
	318	switch (code) {
	319	/*
	320	* Arithmetic operations: ADD/SUB/MUL/DIV/MOD/NEG
	321	*/
	322	case BPF_ALU \| BPF_ADD \| BPF_X: /* (u32) dst += (u32) src */
	323	EMIT(PPC_RAW_ADD(dst_reg, dst_reg, src_reg));
	324	break;
	325	case BPF_ALU64 \| BPF_ADD \| BPF_X: /* dst += src */
	326	EMIT(PPC_RAW_ADDC(dst_reg, dst_reg, src_reg));
	327	EMIT(PPC_RAW_ADDE(dst_reg_h, dst_reg_h, src_reg_h));
	328	break;
	329	case BPF_ALU \| BPF_SUB \| BPF_X: /* (u32) dst -= (u32) src */
	330	EMIT(PPC_RAW_SUB(dst_reg, dst_reg, src_reg));
	331	break;
	332	case BPF_ALU64 \| BPF_SUB \| BPF_X: /* dst -= src */
	333	EMIT(PPC_RAW_SUBFC(dst_reg, src_reg, dst_reg));
	334	EMIT(PPC_RAW_SUBFE(dst_reg_h, src_reg_h, dst_reg_h));
	335	break;
	336	case BPF_ALU \| BPF_SUB \| BPF_K: /* (u32) dst -= (u32) imm */
	337	imm = -imm;
	338	fallthrough;
	339	case BPF_ALU \| BPF_ADD \| BPF_K: /* (u32) dst += (u32) imm */
	340	if (IMM_HA(imm) & 0xffff)
	341	EMIT(PPC_RAW_ADDIS(dst_reg, dst_reg, IMM_HA(imm)));
	342	if (IMM_L(imm))
	343	EMIT(PPC_RAW_ADDI(dst_reg, dst_reg, IMM_L(imm)));
	344	break;
	345	case BPF_ALU64 \| BPF_SUB \| BPF_K: /* dst -= imm */
	346	imm = -imm;
	347	fallthrough;
	348	case BPF_ALU64 \| BPF_ADD \| BPF_K: /* dst += imm */
	349	if (!imm)
	350	break;
	351
	352	if (imm >= -32768 && imm < 32768) {
	353	EMIT(PPC_RAW_ADDIC(dst_reg, dst_reg, imm));
	354	} else {
	355	PPC_LI32(__REG_R0, imm);
	356	EMIT(PPC_RAW_ADDC(dst_reg, dst_reg, __REG_R0));
	357	}
	358	if (imm >= 0)
	359	EMIT(PPC_RAW_ADDZE(dst_reg_h, dst_reg_h));
	360	else
	361	EMIT(PPC_RAW_ADDME(dst_reg_h, dst_reg_h));
	362	break;
	363	case BPF_ALU64 \| BPF_MUL \| BPF_X: /* dst = src /
	364	bpf_set_seen_register(ctx, tmp_reg);
	365	EMIT(PPC_RAW_MULW(__REG_R0, dst_reg, src_reg_h));
	366	EMIT(PPC_RAW_MULW(dst_reg_h, dst_reg_h, src_reg));
	367	EMIT(PPC_RAW_MULHWU(tmp_reg, dst_reg, src_reg));
	368	EMIT(PPC_RAW_MULW(dst_reg, dst_reg, src_reg));
	369	EMIT(PPC_RAW_ADD(dst_reg_h, dst_reg_h, __REG_R0));
	370	EMIT(PPC_RAW_ADD(dst_reg_h, dst_reg_h, tmp_reg));
	371	break;
	372	case BPF_ALU \| BPF_MUL \| BPF_X: /* (u32) dst = (u32) src /
	373	EMIT(PPC_RAW_MULW(dst_reg, dst_reg, src_reg));
	374	break;
	375	case BPF_ALU \| BPF_MUL \| BPF_K: /* (u32) dst = (u32) imm /
	376	if (imm >= -32768 && imm < 32768) {
	377	EMIT(PPC_RAW_MULI(dst_reg, dst_reg, imm));
	378	} else {
	379	PPC_LI32(__REG_R0, imm);
	380	EMIT(PPC_RAW_MULW(dst_reg, dst_reg, __REG_R0));
	381	}
	382	break;
	383	case BPF_ALU64 \| BPF_MUL \| BPF_K: /* dst = imm /
	384	if (!imm) {
	385	PPC_LI32(dst_reg, 0);
	386	PPC_LI32(dst_reg_h, 0);
	387	break;
	388	}
	389	if (imm == 1)
	390	break;
	391	if (imm == -1) {
	392	EMIT(PPC_RAW_SUBFIC(dst_reg, dst_reg, 0));
	393	EMIT(PPC_RAW_SUBFZE(dst_reg_h, dst_reg_h));
	394	break;
	395	}
	396	bpf_set_seen_register(ctx, tmp_reg);
	397	PPC_LI32(tmp_reg, imm);
	398	EMIT(PPC_RAW_MULW(dst_reg_h, dst_reg_h, tmp_reg));
	399	if (imm < 0)
	400	EMIT(PPC_RAW_SUB(dst_reg_h, dst_reg_h, dst_reg));
	401	EMIT(PPC_RAW_MULHWU(__REG_R0, dst_reg, tmp_reg));
	402	EMIT(PPC_RAW_MULW(dst_reg, dst_reg, tmp_reg));
	403	EMIT(PPC_RAW_ADD(dst_reg_h, dst_reg_h, __REG_R0));
	404	break;
	405	case BPF_ALU \| BPF_DIV \| BPF_X: /* (u32) dst /= (u32) src */
	406	EMIT(PPC_RAW_DIVWU(dst_reg, dst_reg, src_reg));
	407	break;
	408	case BPF_ALU \| BPF_MOD \| BPF_X: /* (u32) dst %= (u32) src */
	409	EMIT(PPC_RAW_DIVWU(__REG_R0, dst_reg, src_reg));
	410	EMIT(PPC_RAW_MULW(__REG_R0, src_reg, __REG_R0));
	411	EMIT(PPC_RAW_SUB(dst_reg, dst_reg, __REG_R0));
	412	break;
	413	case BPF_ALU64 \| BPF_DIV \| BPF_X: /* dst /= src */
	414	return -EOPNOTSUPP;
	415	case BPF_ALU64 \| BPF_MOD \| BPF_X: /* dst %= src */
	416	return -EOPNOTSUPP;
	417	case BPF_ALU \| BPF_DIV \| BPF_K: /* (u32) dst /= (u32) imm */
	418	if (!imm)
	419	return -EINVAL;
	420	if (imm == 1)
	421	break;
	422
	423	PPC_LI32(__REG_R0, imm);
	424	EMIT(PPC_RAW_DIVWU(dst_reg, dst_reg, __REG_R0));
	425	break;
	426	case BPF_ALU \| BPF_MOD \| BPF_K: /* (u32) dst %= (u32) imm */
	427	if (!imm)
	428	return -EINVAL;
	429
	430	if (!is_power_of_2((u32)imm)) {
	431	bpf_set_seen_register(ctx, tmp_reg);
	432	PPC_LI32(tmp_reg, imm);
	433	EMIT(PPC_RAW_DIVWU(__REG_R0, dst_reg, tmp_reg));
	434	EMIT(PPC_RAW_MULW(__REG_R0, tmp_reg, __REG_R0));
	435	EMIT(PPC_RAW_SUB(dst_reg, dst_reg, __REG_R0));
	436	break;
	437	}
	438	if (imm == 1)
	439	EMIT(PPC_RAW_LI(dst_reg, 0));
	440	else
	441	EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 0, 32 - ilog2((u32)imm), 31));
	442
	443	break;
	444	case BPF_ALU64 \| BPF_MOD \| BPF_K: /* dst %= imm */
	445	if (!imm)
	446	return -EINVAL;
	447	if (imm < 0)
	448	imm = -imm;
	449	if (!is_power_of_2(imm))
	450	return -EOPNOTSUPP;
	451	if (imm == 1)
	452	EMIT(PPC_RAW_LI(dst_reg, 0));
	453	else
	454	EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 0, 32 - ilog2(imm), 31));
	455	EMIT(PPC_RAW_LI(dst_reg_h, 0));
	456	break;
	457	case BPF_ALU64 \| BPF_DIV \| BPF_K: /* dst /= imm */
	458	if (!imm)
	459	return -EINVAL;
	460	if (!is_power_of_2(abs(imm)))
	461	return -EOPNOTSUPP;
	462
	463	if (imm < 0) {
	464	EMIT(PPC_RAW_SUBFIC(dst_reg, dst_reg, 0));
	465	EMIT(PPC_RAW_SUBFZE(dst_reg_h, dst_reg_h));
	466	imm = -imm;
	467	}
	468	if (imm == 1)
	469	break;
	470	imm = ilog2(imm);
	471	EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 32 - imm, imm, 31));
	472	EMIT(PPC_RAW_RLWIMI(dst_reg, dst_reg_h, 32 - imm, 0, imm - 1));
	473	EMIT(PPC_RAW_SRAWI(dst_reg_h, dst_reg_h, imm));
	474	break;
	475	case BPF_ALU \| BPF_NEG: /* (u32) dst = -dst */
	476	EMIT(PPC_RAW_NEG(dst_reg, dst_reg));
	477	break;
	478	case BPF_ALU64 \| BPF_NEG: /* dst = -dst */
	479	EMIT(PPC_RAW_SUBFIC(dst_reg, dst_reg, 0));
	480	EMIT(PPC_RAW_SUBFZE(dst_reg_h, dst_reg_h));
	481	break;
	482
	483	/*
	484	* Logical operations: AND/OR/XOR/[A]LSH/[A]RSH
	485	*/
	486	case BPF_ALU64 \| BPF_AND \| BPF_X: /* dst = dst & src */
	487	EMIT(PPC_RAW_AND(dst_reg, dst_reg, src_reg));
	488	EMIT(PPC_RAW_AND(dst_reg_h, dst_reg_h, src_reg_h));
	489	break;
	490	case BPF_ALU \| BPF_AND \| BPF_X: /* (u32) dst = dst & src */
	491	EMIT(PPC_RAW_AND(dst_reg, dst_reg, src_reg));
	492	break;
	493	case BPF_ALU64 \| BPF_AND \| BPF_K: /* dst = dst & imm */
	494	if (imm >= 0)
	495	EMIT(PPC_RAW_LI(dst_reg_h, 0));
	496	fallthrough;
	497	case BPF_ALU \| BPF_AND \| BPF_K: /* (u32) dst = dst & imm */
	498	if (!IMM_H(imm)) {
	499	EMIT(PPC_RAW_ANDI(dst_reg, dst_reg, IMM_L(imm)));
	500	} else if (!IMM_L(imm)) {
	501	EMIT(PPC_RAW_ANDIS(dst_reg, dst_reg, IMM_H(imm)));
	502	} else if (imm == (((1 << fls(imm)) - 1) ^ ((1 << (ffs(i) - 1)) - 1))) {
	503	EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 0,
	504	32 - fls(imm), 32 - ffs(imm)));
	505	} else {
	506	PPC_LI32(__REG_R0, imm);
	507	EMIT(PPC_RAW_AND(dst_reg, dst_reg, __REG_R0));
	508	}
	509	break;
	510	case BPF_ALU64 \| BPF_OR \| BPF_X: /* dst = dst \| src */
	511	EMIT(PPC_RAW_OR(dst_reg, dst_reg, src_reg));
	512	EMIT(PPC_RAW_OR(dst_reg_h, dst_reg_h, src_reg_h));
	513	break;
	514	case BPF_ALU \| BPF_OR \| BPF_X: /* dst = (u32) dst \| (u32) src */
	515	EMIT(PPC_RAW_OR(dst_reg, dst_reg, src_reg));
	516	break;
	517	case BPF_ALU64 \| BPF_OR \| BPF_K:/* dst = dst \| imm */
	518	/* Sign-extended */
	519	if (imm < 0)
	520	EMIT(PPC_RAW_LI(dst_reg_h, -1));
	521	fallthrough;
	522	case BPF_ALU \| BPF_OR \| BPF_K:/* dst = (u32) dst \| (u32) imm */
	523	if (IMM_L(imm))
	524	EMIT(PPC_RAW_ORI(dst_reg, dst_reg, IMM_L(imm)));
	525	if (IMM_H(imm))
	526	EMIT(PPC_RAW_ORIS(dst_reg, dst_reg, IMM_H(imm)));
	527	break;
	528	case BPF_ALU64 \| BPF_XOR \| BPF_X: /* dst ^= src */
	529	if (dst_reg == src_reg) {
	530	EMIT(PPC_RAW_LI(dst_reg, 0));
	531	EMIT(PPC_RAW_LI(dst_reg_h, 0));
	532	} else {
	533	EMIT(PPC_RAW_XOR(dst_reg, dst_reg, src_reg));
	534	EMIT(PPC_RAW_XOR(dst_reg_h, dst_reg_h, src_reg_h));
	535	}
	536	break;
	537	case BPF_ALU \| BPF_XOR \| BPF_X: /* (u32) dst ^= src */
	538	if (dst_reg == src_reg)
	539	EMIT(PPC_RAW_LI(dst_reg, 0));
	540	else
	541	EMIT(PPC_RAW_XOR(dst_reg, dst_reg, src_reg));
	542	break;
	543	case BPF_ALU64 \| BPF_XOR \| BPF_K: /* dst ^= imm */
	544	if (imm < 0)
	545	EMIT(PPC_RAW_NOR(dst_reg_h, dst_reg_h, dst_reg_h));
	546	fallthrough;
	547	case BPF_ALU \| BPF_XOR \| BPF_K: /* (u32) dst ^= (u32) imm */
	548	if (IMM_L(imm))
	549	EMIT(PPC_RAW_XORI(dst_reg, dst_reg, IMM_L(imm)));
	550	if (IMM_H(imm))
	551	EMIT(PPC_RAW_XORIS(dst_reg, dst_reg, IMM_H(imm)));
	552	break;
	553	case BPF_ALU \| BPF_LSH \| BPF_X: /* (u32) dst <<= (u32) src */
	554	EMIT(PPC_RAW_SLW(dst_reg, dst_reg, src_reg));
	555	break;
	556	case BPF_ALU64 \| BPF_LSH \| BPF_X: /* dst <<= src; */
	557	bpf_set_seen_register(ctx, tmp_reg);
	558	EMIT(PPC_RAW_SUBFIC(__REG_R0, src_reg, 32));
	559	EMIT(PPC_RAW_SLW(dst_reg_h, dst_reg_h, src_reg));
	560	EMIT(PPC_RAW_ADDI(tmp_reg, src_reg, 32));
	561	EMIT(PPC_RAW_SRW(__REG_R0, dst_reg, __REG_R0));
	562	EMIT(PPC_RAW_SLW(tmp_reg, dst_reg, tmp_reg));
	563	EMIT(PPC_RAW_OR(dst_reg_h, dst_reg_h, __REG_R0));
	564	EMIT(PPC_RAW_SLW(dst_reg, dst_reg, src_reg));
	565	EMIT(PPC_RAW_OR(dst_reg_h, dst_reg_h, tmp_reg));
	566	break;
	567	case BPF_ALU \| BPF_LSH \| BPF_K: /* (u32) dst <<= (u32) imm */
	568	if (!imm)
	569	break;
	570	EMIT(PPC_RAW_SLWI(dst_reg, dst_reg, imm));
	571	break;
	572	case BPF_ALU64 \| BPF_LSH \| BPF_K: /* dst <<= imm */
	573	if (imm < 0)
	574	return -EINVAL;
	575	if (!imm)
	576	break;
	577	if (imm < 32) {
	578	EMIT(PPC_RAW_RLWINM(dst_reg_h, dst_reg_h, imm, 0, 31 - imm));
	579	EMIT(PPC_RAW_RLWIMI(dst_reg_h, dst_reg, imm, 32 - imm, 31));
	580	EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, imm, 0, 31 - imm));
	581	break;
	582	}
	583	if (imm < 64)
	584	EMIT(PPC_RAW_RLWINM(dst_reg_h, dst_reg, imm, 0, 31 - imm));
	585	else
	586	EMIT(PPC_RAW_LI(dst_reg_h, 0));
	587	EMIT(PPC_RAW_LI(dst_reg, 0));
	588	break;
	589	case BPF_ALU \| BPF_RSH \| BPF_X: /* (u32) dst >>= (u32) src */
	590	EMIT(PPC_RAW_SRW(dst_reg, dst_reg, src_reg));
	591	break;
	592	case BPF_ALU64 \| BPF_RSH \| BPF_X: /* dst >>= src */
	593	bpf_set_seen_register(ctx, tmp_reg);
	594	EMIT(PPC_RAW_SUBFIC(__REG_R0, src_reg, 32));
	595	EMIT(PPC_RAW_SRW(dst_reg, dst_reg, src_reg));
	596	EMIT(PPC_RAW_ADDI(tmp_reg, src_reg, 32));
	597	EMIT(PPC_RAW_SLW(__REG_R0, dst_reg_h, __REG_R0));
	598	EMIT(PPC_RAW_SRW(tmp_reg, dst_reg_h, tmp_reg));
	599	EMIT(PPC_RAW_OR(dst_reg, dst_reg, __REG_R0));
	600	EMIT(PPC_RAW_SRW(dst_reg_h, dst_reg_h, src_reg));
	601	EMIT(PPC_RAW_OR(dst_reg, dst_reg, tmp_reg));
	602	break;
	603	case BPF_ALU \| BPF_RSH \| BPF_K: /* (u32) dst >>= (u32) imm */
	604	if (!imm)
	605	break;
	606	EMIT(PPC_RAW_SRWI(dst_reg, dst_reg, imm));
	607	break;
	608	case BPF_ALU64 \| BPF_RSH \| BPF_K: /* dst >>= imm */
	609	if (imm < 0)
	610	return -EINVAL;
	611	if (!imm)
	612	break;
	613	if (imm < 32) {
	614	EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 32 - imm, imm, 31));
	615	EMIT(PPC_RAW_RLWIMI(dst_reg, dst_reg_h, 32 - imm, 0, imm - 1));
	616	EMIT(PPC_RAW_RLWINM(dst_reg_h, dst_reg_h, 32 - imm, imm, 31));
	617	break;
	618	}
	619	if (imm < 64)
	620	EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg_h, 64 - imm, imm - 32, 31));
	621	else
	622	EMIT(PPC_RAW_LI(dst_reg, 0));
	623	EMIT(PPC_RAW_LI(dst_reg_h, 0));
	624	break;
	625	case BPF_ALU \| BPF_ARSH \| BPF_X: /* (s32) dst >>= src */
	626	EMIT(PPC_RAW_SRAW(dst_reg_h, dst_reg, src_reg));
	627	break;
	628	case BPF_ALU64 \| BPF_ARSH \| BPF_X: /* (s64) dst >>= src */
	629	bpf_set_seen_register(ctx, tmp_reg);
	630	EMIT(PPC_RAW_SUBFIC(__REG_R0, src_reg, 32));
	631	EMIT(PPC_RAW_SRW(dst_reg, dst_reg, src_reg));
	632	EMIT(PPC_RAW_SLW(__REG_R0, dst_reg_h, __REG_R0));
	633	EMIT(PPC_RAW_ADDI(tmp_reg, src_reg, 32));
	634	EMIT(PPC_RAW_OR(dst_reg, dst_reg, __REG_R0));
	635	EMIT(PPC_RAW_RLWINM(__REG_R0, tmp_reg, 0, 26, 26));
	636	EMIT(PPC_RAW_SRAW(tmp_reg, dst_reg_h, tmp_reg));
	637	EMIT(PPC_RAW_SRAW(dst_reg_h, dst_reg_h, src_reg));
	638	EMIT(PPC_RAW_SLW(tmp_reg, tmp_reg, __REG_R0));
	639	EMIT(PPC_RAW_OR(dst_reg, dst_reg, tmp_reg));
	640	break;
	641	case BPF_ALU \| BPF_ARSH \| BPF_K: /* (s32) dst >>= imm */
	642	if (!imm)
	643	break;
	644	EMIT(PPC_RAW_SRAWI(dst_reg, dst_reg, imm));
	645	break;
	646	case BPF_ALU64 \| BPF_ARSH \| BPF_K: /* (s64) dst >>= imm */
	647	if (imm < 0)
	648	return -EINVAL;
	649	if (!imm)
	650	break;
	651	if (imm < 32) {
	652	EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 32 - imm, imm, 31));
	653	EMIT(PPC_RAW_RLWIMI(dst_reg, dst_reg_h, 32 - imm, 0, imm - 1));
	654	EMIT(PPC_RAW_SRAWI(dst_reg_h, dst_reg_h, imm));
	655	break;
	656	}
	657	if (imm < 64)
	658	EMIT(PPC_RAW_SRAWI(dst_reg, dst_reg_h, imm - 32));
	659	else
	660	EMIT(PPC_RAW_SRAWI(dst_reg, dst_reg_h, 31));
	661	EMIT(PPC_RAW_SRAWI(dst_reg_h, dst_reg_h, 31));
	662	break;
	663
	664	/*
	665	* MOV
	666	*/
	667	case BPF_ALU64 \| BPF_MOV \| BPF_X: /* dst = src */
	668	if (dst_reg == src_reg)
	669	break;
	670	EMIT(PPC_RAW_MR(dst_reg, src_reg));
	671	EMIT(PPC_RAW_MR(dst_reg_h, src_reg_h));
	672	break;
	673	case BPF_ALU \| BPF_MOV \| BPF_X: /* (u32) dst = src */
	674	/* special mov32 for zext */
	675	if (imm == 1)
	676	EMIT(PPC_RAW_LI(dst_reg_h, 0));
	677	else if (dst_reg != src_reg)
	678	EMIT(PPC_RAW_MR(dst_reg, src_reg));
	679	break;
	680	case BPF_ALU64 \| BPF_MOV \| BPF_K: /* dst = (s64) imm */
	681	PPC_LI32(dst_reg, imm);
	682	PPC_EX32(dst_reg_h, imm);
	683	break;
	684	case BPF_ALU \| BPF_MOV \| BPF_K: /* (u32) dst = imm */
	685	PPC_LI32(dst_reg, imm);
	686	break;
	687
	688	/*
	689	* BPF_FROM_BE/LE
	690	*/
	691	case BPF_ALU \| BPF_END \| BPF_FROM_LE:
	692	switch (imm) {
	693	case 16:
	694	/* Copy 16 bits to upper part */
	695	EMIT(PPC_RAW_RLWIMI(dst_reg, dst_reg, 16, 0, 15));
	696	/* Rotate 8 bits right & mask */
	697	EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 24, 16, 31));
	698	break;
	699	case 32:
	700	/*
	701	* Rotate word left by 8 bits:
	702	* 2 bytes are already in their final position
	703	* -- byte 2 and 4 (of bytes 1, 2, 3 and 4)
	704	*/
	705	EMIT(PPC_RAW_RLWINM(__REG_R0, dst_reg, 8, 0, 31));
	706	/* Rotate 24 bits and insert byte 1 */
	707	EMIT(PPC_RAW_RLWIMI(__REG_R0, dst_reg, 24, 0, 7));
	708	/* Rotate 24 bits and insert byte 3 */
	709	EMIT(PPC_RAW_RLWIMI(__REG_R0, dst_reg, 24, 16, 23));
	710	EMIT(PPC_RAW_MR(dst_reg, __REG_R0));
	711	break;
	712	case 64:
	713	bpf_set_seen_register(ctx, tmp_reg);
	714	EMIT(PPC_RAW_RLWINM(tmp_reg, dst_reg, 8, 0, 31));
	715	EMIT(PPC_RAW_RLWINM(__REG_R0, dst_reg_h, 8, 0, 31));
	716	/* Rotate 24 bits and insert byte 1 */
	717	EMIT(PPC_RAW_RLWIMI(tmp_reg, dst_reg, 24, 0, 7));
	718	EMIT(PPC_RAW_RLWIMI(__REG_R0, dst_reg_h, 24, 0, 7));
	719	/* Rotate 24 bits and insert byte 3 */
	720	EMIT(PPC_RAW_RLWIMI(tmp_reg, dst_reg, 24, 16, 23));
	721	EMIT(PPC_RAW_RLWIMI(__REG_R0, dst_reg_h, 24, 16, 23));
	722	EMIT(PPC_RAW_MR(dst_reg, __REG_R0));
	723	EMIT(PPC_RAW_MR(dst_reg_h, tmp_reg));
	724	break;
	725	}
	726	break;
	727	case BPF_ALU \| BPF_END \| BPF_FROM_BE:
	728	switch (imm) {
	729	case 16:
	730	/* zero-extend 16 bits into 32 bits */
	731	EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 0, 16, 31));
	732	break;
	733	case 32:
	734	case 64:
	735	/* nop */
	736	break;
	737	}
	738	break;
	739
	740	/*
	741	* BPF_ST(X)
	742	*/
	743	case BPF_STX \| BPF_MEM \| BPF_B: /* (u8 )(dst + off) = src */
	744	EMIT(PPC_RAW_STB(src_reg, dst_reg, off));
	745	break;
	746	case BPF_ST \| BPF_MEM \| BPF_B: /* (u8 )(dst + off) = imm */
	747	PPC_LI32(__REG_R0, imm);
	748	EMIT(PPC_RAW_STB(__REG_R0, dst_reg, off));
	749	break;
	750	case BPF_STX \| BPF_MEM \| BPF_H: /* (u16 )(dst + off) = src /
	751	EMIT(PPC_RAW_STH(src_reg, dst_reg, off));
	752	break;
	753	case BPF_ST \| BPF_MEM \| BPF_H: /* (u16 )(dst + off) = imm /
	754	PPC_LI32(__REG_R0, imm);
	755	EMIT(PPC_RAW_STH(__REG_R0, dst_reg, off));
	756	break;
	757	case BPF_STX \| BPF_MEM \| BPF_W: /* (u32 )(dst + off) = src */
	758	EMIT(PPC_RAW_STW(src_reg, dst_reg, off));
	759	break;
	760	case BPF_ST \| BPF_MEM \| BPF_W: /* (u32 )(dst + off) = imm */
	761	PPC_LI32(__REG_R0, imm);
	762	EMIT(PPC_RAW_STW(__REG_R0, dst_reg, off));
	763	break;
	764	case BPF_STX \| BPF_MEM \| BPF_DW: /* (u64 )(dst + off) = src /
	765	EMIT(PPC_RAW_STW(src_reg_h, dst_reg, off));
	766	EMIT(PPC_RAW_STW(src_reg, dst_reg, off + 4));
	767	break;
	768	case BPF_ST \| BPF_MEM \| BPF_DW: /* (u64 )(dst + off) = imm */
	769	PPC_LI32(__REG_R0, imm);
	770	EMIT(PPC_RAW_STW(__REG_R0, dst_reg, off + 4));
	771	PPC_EX32(__REG_R0, imm);
	772	EMIT(PPC_RAW_STW(__REG_R0, dst_reg, off));
	773	break;
	774
	775	/*
	776	* BPF_STX XADD (atomic_add)
	777	*/
	778	case BPF_STX \| BPF_XADD \| BPF_W: /* (u32 )(dst + off) += src */
	779	bpf_set_seen_register(ctx, tmp_reg);
	780	/* Get offset into TMP_REG */
	781	EMIT(PPC_RAW_LI(tmp_reg, off));
	782	/* load value from memory into r0 */
	783	EMIT(PPC_RAW_LWARX(__REG_R0, tmp_reg, dst_reg, 0));
	784	/* add value from src_reg into this */
	785	EMIT(PPC_RAW_ADD(__REG_R0, __REG_R0, src_reg));
	786	/* store result back */
	787	EMIT(PPC_RAW_STWCX(__REG_R0, tmp_reg, dst_reg));
	788	/* we're done if this succeeded */
	789	PPC_BCC_SHORT(COND_NE, (ctx->idx - 3) * 4);
	790	break;
	791
	792	case BPF_STX \| BPF_XADD \| BPF_DW: /* (u64 )(dst + off) += src */
	793	return -EOPNOTSUPP;
	794
	795	/*
	796	* BPF_LDX
	797	*/
	798	case BPF_LDX \| BPF_MEM \| BPF_B: /* dst = (u8 )(ul) (src + off) */
	799	EMIT(PPC_RAW_LBZ(dst_reg, src_reg, off));
	800	if (!fp->aux->verifier_zext)
	801	EMIT(PPC_RAW_LI(dst_reg_h, 0));
	802	break;
	803	case BPF_LDX \| BPF_MEM \| BPF_H: /* dst = (u16 )(ul) (src + off) */
	804	EMIT(PPC_RAW_LHZ(dst_reg, src_reg, off));
	805	if (!fp->aux->verifier_zext)
	806	EMIT(PPC_RAW_LI(dst_reg_h, 0));
	807	break;
	808	case BPF_LDX \| BPF_MEM \| BPF_W: /* dst = (u32 )(ul) (src + off) */
	809	EMIT(PPC_RAW_LWZ(dst_reg, src_reg, off));
	810	if (!fp->aux->verifier_zext)
	811	EMIT(PPC_RAW_LI(dst_reg_h, 0));
	812	break;
	813	case BPF_LDX \| BPF_MEM \| BPF_DW: /* dst = (u64 )(ul) (src + off) */
	814	EMIT(PPC_RAW_LWZ(dst_reg_h, src_reg, off));
	815	EMIT(PPC_RAW_LWZ(dst_reg, src_reg, off + 4));
	816	break;
	817
	818	/*
	819	* Doubleword load
	820	* 16 byte instruction that uses two 'struct bpf_insn'
	821	*/
	822	case BPF_LD \| BPF_IMM \| BPF_DW: /* dst = (u64) imm */
	823	PPC_LI32(dst_reg_h, (u32)insn[i + 1].imm);
	824	PPC_LI32(dst_reg, (u32)insn[i].imm);
	825	/* Adjust for two bpf instructions */
	826	addrs[++i] = ctx->idx * 4;
	827	break;
	828
	829	/*
	830	* Return/Exit
	831	*/
	832	case BPF_JMP \| BPF_EXIT:
	833	/*
	834	* If this isn't the very last instruction, branch to
	835	* the epilogue. If we _are_ the last instruction,
	836	* we'll just fall through to the epilogue.
	837	*/
	838	if (i != flen - 1)
	839	PPC_JMP(exit_addr);
	840	/* else fall through to the epilogue */
	841	break;
	842
	843	/*
	844	* Call kernel helper or bpf function
	845	*/
	846	case BPF_JMP \| BPF_CALL:
	847	ctx->seen \|= SEEN_FUNC;
	848
	849	ret = bpf_jit_get_func_addr(fp, &insn[i], extra_pass,
	850	&func_addr, &func_addr_fixed);
	851	if (ret < 0)
	852	return ret;
	853
	854	if (bpf_is_seen_register(ctx, bpf_to_ppc(ctx, BPF_REG_5))) {
	855	EMIT(PPC_RAW_STW(bpf_to_ppc(ctx, BPF_REG_5) - 1, __REG_R1, 8));
	856	EMIT(PPC_RAW_STW(bpf_to_ppc(ctx, BPF_REG_5), __REG_R1, 12));
	857	}
	858
	859	bpf_jit_emit_func_call_rel(image, ctx, func_addr);
	860
	861	EMIT(PPC_RAW_MR(bpf_to_ppc(ctx, BPF_REG_0) - 1, __REG_R3));
	862	EMIT(PPC_RAW_MR(bpf_to_ppc(ctx, BPF_REG_0), __REG_R4));
	863	break;
	864
	865	/*
	866	* Jumps and branches
	867	*/
	868	case BPF_JMP \| BPF_JA:
	869	PPC_JMP(addrs[i + 1 + off]);
	870	break;
	871
	872	case BPF_JMP \| BPF_JGT \| BPF_K:
	873	case BPF_JMP \| BPF_JGT \| BPF_X:
	874	case BPF_JMP \| BPF_JSGT \| BPF_K:
	875	case BPF_JMP \| BPF_JSGT \| BPF_X:
	876	case BPF_JMP32 \| BPF_JGT \| BPF_K:
	877	case BPF_JMP32 \| BPF_JGT \| BPF_X:
	878	case BPF_JMP32 \| BPF_JSGT \| BPF_K:
	879	case BPF_JMP32 \| BPF_JSGT \| BPF_X:
	880	true_cond = COND_GT;
	881	goto cond_branch;
	882	case BPF_JMP \| BPF_JLT \| BPF_K:
	883	case BPF_JMP \| BPF_JLT \| BPF_X:
	884	case BPF_JMP \| BPF_JSLT \| BPF_K:
	885	case BPF_JMP \| BPF_JSLT \| BPF_X:
	886	case BPF_JMP32 \| BPF_JLT \| BPF_K:
	887	case BPF_JMP32 \| BPF_JLT \| BPF_X:
	888	case BPF_JMP32 \| BPF_JSLT \| BPF_K:
	889	case BPF_JMP32 \| BPF_JSLT \| BPF_X:
	890	true_cond = COND_LT;
	891	goto cond_branch;
	892	case BPF_JMP \| BPF_JGE \| BPF_K:
	893	case BPF_JMP \| BPF_JGE \| BPF_X:
	894	case BPF_JMP \| BPF_JSGE \| BPF_K:
	895	case BPF_JMP \| BPF_JSGE \| BPF_X:
	896	case BPF_JMP32 \| BPF_JGE \| BPF_K:
	897	case BPF_JMP32 \| BPF_JGE \| BPF_X:
	898	case BPF_JMP32 \| BPF_JSGE \| BPF_K:
	899	case BPF_JMP32 \| BPF_JSGE \| BPF_X:
	900	true_cond = COND_GE;
	901	goto cond_branch;
	902	case BPF_JMP \| BPF_JLE \| BPF_K:
	903	case BPF_JMP \| BPF_JLE \| BPF_X:
	904	case BPF_JMP \| BPF_JSLE \| BPF_K:
	905	case BPF_JMP \| BPF_JSLE \| BPF_X:
	906	case BPF_JMP32 \| BPF_JLE \| BPF_K:
	907	case BPF_JMP32 \| BPF_JLE \| BPF_X:
	908	case BPF_JMP32 \| BPF_JSLE \| BPF_K:
	909	case BPF_JMP32 \| BPF_JSLE \| BPF_X:
	910	true_cond = COND_LE;
	911	goto cond_branch;
	912	case BPF_JMP \| BPF_JEQ \| BPF_K:
	913	case BPF_JMP \| BPF_JEQ \| BPF_X:
	914	case BPF_JMP32 \| BPF_JEQ \| BPF_K:
	915	case BPF_JMP32 \| BPF_JEQ \| BPF_X:
	916	true_cond = COND_EQ;
	917	goto cond_branch;
	918	case BPF_JMP \| BPF_JNE \| BPF_K:
	919	case BPF_JMP \| BPF_JNE \| BPF_X:
	920	case BPF_JMP32 \| BPF_JNE \| BPF_K:
	921	case BPF_JMP32 \| BPF_JNE \| BPF_X:
	922	true_cond = COND_NE;
	923	goto cond_branch;
	924	case BPF_JMP \| BPF_JSET \| BPF_K:
	925	case BPF_JMP \| BPF_JSET \| BPF_X:
	926	case BPF_JMP32 \| BPF_JSET \| BPF_K:
	927	case BPF_JMP32 \| BPF_JSET \| BPF_X:
	928	true_cond = COND_NE;
	929	/* fallthrough; */
	930
	931	cond_branch:
	932	switch (code) {
	933	case BPF_JMP \| BPF_JGT \| BPF_X:
	934	case BPF_JMP \| BPF_JLT \| BPF_X:
	935	case BPF_JMP \| BPF_JGE \| BPF_X:
	936	case BPF_JMP \| BPF_JLE \| BPF_X:
	937	case BPF_JMP \| BPF_JEQ \| BPF_X:
	938	case BPF_JMP \| BPF_JNE \| BPF_X:
	939	/* unsigned comparison */
	940	EMIT(PPC_RAW_CMPLW(dst_reg_h, src_reg_h));
	941	PPC_BCC_SHORT(COND_NE, (ctx->idx + 2) * 4);
	942	EMIT(PPC_RAW_CMPLW(dst_reg, src_reg));
	943	break;
	944	case BPF_JMP32 \| BPF_JGT \| BPF_X:
	945	case BPF_JMP32 \| BPF_JLT \| BPF_X:
	946	case BPF_JMP32 \| BPF_JGE \| BPF_X:
	947	case BPF_JMP32 \| BPF_JLE \| BPF_X:
	948	case BPF_JMP32 \| BPF_JEQ \| BPF_X:
	949	case BPF_JMP32 \| BPF_JNE \| BPF_X:
	950	/* unsigned comparison */
	951	EMIT(PPC_RAW_CMPLW(dst_reg, src_reg));
	952	break;
	953	case BPF_JMP \| BPF_JSGT \| BPF_X:
	954	case BPF_JMP \| BPF_JSLT \| BPF_X:
	955	case BPF_JMP \| BPF_JSGE \| BPF_X:
	956	case BPF_JMP \| BPF_JSLE \| BPF_X:
	957	/* signed comparison */
	958	EMIT(PPC_RAW_CMPW(dst_reg_h, src_reg_h));
	959	PPC_BCC_SHORT(COND_NE, (ctx->idx + 2) * 4);
	960	EMIT(PPC_RAW_CMPLW(dst_reg, src_reg));
	961	break;
	962	case BPF_JMP32 \| BPF_JSGT \| BPF_X:
	963	case BPF_JMP32 \| BPF_JSLT \| BPF_X:
	964	case BPF_JMP32 \| BPF_JSGE \| BPF_X:
	965	case BPF_JMP32 \| BPF_JSLE \| BPF_X:
	966	/* signed comparison */
	967	EMIT(PPC_RAW_CMPW(dst_reg, src_reg));
	968	break;
	969	case BPF_JMP \| BPF_JSET \| BPF_X:
	970	EMIT(PPC_RAW_AND_DOT(__REG_R0, dst_reg_h, src_reg_h));
	971	PPC_BCC_SHORT(COND_NE, (ctx->idx + 2) * 4);
	972	EMIT(PPC_RAW_AND_DOT(__REG_R0, dst_reg, src_reg));
	973	break;
	974	case BPF_JMP32 \| BPF_JSET \| BPF_X: {
	975	EMIT(PPC_RAW_AND_DOT(__REG_R0, dst_reg, src_reg));
	976	break;
	977	case BPF_JMP \| BPF_JNE \| BPF_K:
	978	case BPF_JMP \| BPF_JEQ \| BPF_K:
	979	case BPF_JMP \| BPF_JGT \| BPF_K:
	980	case BPF_JMP \| BPF_JLT \| BPF_K:
	981	case BPF_JMP \| BPF_JGE \| BPF_K:
	982	case BPF_JMP \| BPF_JLE \| BPF_K:
	983	/*
	984	* Need sign-extended load, so only positive
	985	* values can be used as imm in cmplwi
	986	*/
	987	if (imm >= 0 && imm < 32768) {
	988	EMIT(PPC_RAW_CMPLWI(dst_reg_h, 0));
	989	PPC_BCC_SHORT(COND_NE, (ctx->idx + 2) * 4);
	990	EMIT(PPC_RAW_CMPLWI(dst_reg, imm));
	991	} else {
	992	/* sign-extending load ... but unsigned comparison */
	993	PPC_EX32(__REG_R0, imm);
	994	EMIT(PPC_RAW_CMPLW(dst_reg_h, __REG_R0));
	995	PPC_LI32(__REG_R0, imm);
	996	PPC_BCC_SHORT(COND_NE, (ctx->idx + 2) * 4);
	997	EMIT(PPC_RAW_CMPLW(dst_reg, __REG_R0));
	998	}
	999	break;
	1000	case BPF_JMP32 \| BPF_JNE \| BPF_K:
	1001	case BPF_JMP32 \| BPF_JEQ \| BPF_K:
	1002	case BPF_JMP32 \| BPF_JGT \| BPF_K:
	1003	case BPF_JMP32 \| BPF_JLT \| BPF_K:
	1004	case BPF_JMP32 \| BPF_JGE \| BPF_K:
	1005	case BPF_JMP32 \| BPF_JLE \| BPF_K:
	1006	if (imm >= 0 && imm < 65536) {
	1007	EMIT(PPC_RAW_CMPLWI(dst_reg, imm));
	1008	} else {
	1009	PPC_LI32(__REG_R0, imm);
	1010	EMIT(PPC_RAW_CMPLW(dst_reg, __REG_R0));
	1011	}
	1012	break;
	1013	}
	1014	case BPF_JMP \| BPF_JSGT \| BPF_K:
	1015	case BPF_JMP \| BPF_JSLT \| BPF_K:
	1016	case BPF_JMP \| BPF_JSGE \| BPF_K:
	1017	case BPF_JMP \| BPF_JSLE \| BPF_K:
	1018	if (imm >= 0 && imm < 65536) {
	1019	EMIT(PPC_RAW_CMPWI(dst_reg_h, imm < 0 ? -1 : 0));
	1020	PPC_BCC_SHORT(COND_NE, (ctx->idx + 2) * 4);
	1021	EMIT(PPC_RAW_CMPLWI(dst_reg, imm));
	1022	} else {
	1023	/* sign-extending load */
	1024	EMIT(PPC_RAW_CMPWI(dst_reg_h, imm < 0 ? -1 : 0));
	1025	PPC_LI32(__REG_R0, imm);
	1026	PPC_BCC_SHORT(COND_NE, (ctx->idx + 2) * 4);
	1027	EMIT(PPC_RAW_CMPLW(dst_reg, __REG_R0));
	1028	}
	1029	break;
	1030	case BPF_JMP32 \| BPF_JSGT \| BPF_K:
	1031	case BPF_JMP32 \| BPF_JSLT \| BPF_K:
	1032	case BPF_JMP32 \| BPF_JSGE \| BPF_K:
	1033	case BPF_JMP32 \| BPF_JSLE \| BPF_K:
	1034	/*
	1035	* signed comparison, so any 16-bit value
	1036	* can be used in cmpwi
	1037	*/
	1038	if (imm >= -32768 && imm < 32768) {
	1039	EMIT(PPC_RAW_CMPWI(dst_reg, imm));
	1040	} else {
	1041	/* sign-extending load */
	1042	PPC_LI32(__REG_R0, imm);
	1043	EMIT(PPC_RAW_CMPW(dst_reg, __REG_R0));
	1044	}
	1045	break;
	1046	case BPF_JMP \| BPF_JSET \| BPF_K:
	1047	/* andi does not sign-extend the immediate */
	1048	if (imm >= 0 && imm < 32768) {
	1049	/* PPC_ANDI is _only/always_ dot-form */
	1050	EMIT(PPC_RAW_ANDI(__REG_R0, dst_reg, imm));
	1051	} else {
	1052	PPC_LI32(__REG_R0, imm);
	1053	if (imm < 0) {
	1054	EMIT(PPC_RAW_CMPWI(dst_reg_h, 0));
	1055	PPC_BCC_SHORT(COND_NE, (ctx->idx + 2) * 4);
	1056	}
	1057	EMIT(PPC_RAW_AND_DOT(__REG_R0, dst_reg, __REG_R0));
	1058	}
	1059	break;
	1060	case BPF_JMP32 \| BPF_JSET \| BPF_K:
	1061	/* andi does not sign-extend the immediate */
	1062	if (imm >= -32768 && imm < 32768) {
	1063	/* PPC_ANDI is _only/always_ dot-form */
	1064	EMIT(PPC_RAW_ANDI(__REG_R0, dst_reg, imm));
	1065	} else {
	1066	PPC_LI32(__REG_R0, imm);
	1067	EMIT(PPC_RAW_AND_DOT(__REG_R0, dst_reg, __REG_R0));
	1068	}
	1069	break;
	1070	}
	1071	PPC_BCC(true_cond, addrs[i + 1 + off]);
	1072	break;
	1073
	1074	/*
	1075	* Tail call
	1076	*/
	1077	case BPF_JMP \| BPF_TAIL_CALL:
	1078	ctx->seen \|= SEEN_TAILCALL;
	1079	bpf_jit_emit_tail_call(image, ctx, addrs[i + 1]);
	1080	break;
	1081
	1082	default:
	1083	/*
	1084	* The filter contains something cruel & unusual.
	1085	* We don't handle it, but also there shouldn't be
	1086	* anything missing from our list.
	1087	*/
	1088	pr_err_ratelimited("eBPF filter opcode %04x (@%d) unsupported\n", code, i);
	1089	return -EOPNOTSUPP;
	1090	}
	1091	if (BPF_CLASS(code) == BPF_ALU && !fp->aux->verifier_zext &&
	1092	!insn_is_zext(&insn[i + 1]))
	1093	EMIT(PPC_RAW_LI(dst_reg_h, 0));
	1094	}
	1095
	1096	/* Set end-of-body-code address for exit. */
	1097	addrs[i] = ctx->idx * 4;
	1098
	1099	return 0;
	1100	}

arch/powerpc/net/bpf_jit_comp64.c+15−280

@@ -18,27 +18,6 @@ · 17 unmodified lines
18		18
19	#include "bpf_jit64.h"	19	#include "bpf_jit64.h"
20		20
21	static void bpf_jit_fill_ill_insns(void *area, unsigned int size)
22	{
23	memset32(area, BREAKPOINT_INSTRUCTION, size/4);
24	}
25
26	static inline void bpf_flush_icache(void start, void end)
27	{
28	smp_wmb();
29	flush_icache_range((unsigned long)start, (unsigned long)end);
30	}
31
32	static inline bool bpf_is_seen_register(struct codegen_context *ctx, int i)
33	{
34	return (ctx->seen & (1 << (31 - b2p[i])));
35	}
36
37	static inline void bpf_set_seen_register(struct codegen_context *ctx, int i)
38	{
39	ctx->seen \|= (1 << (31 - b2p[i]));
40	}
41
42	static inline bool bpf_has_stack_frame(struct codegen_context *ctx)	21	static inline bool bpf_has_stack_frame(struct codegen_context *ctx)
43	{	22	{
44	/*	23	/*
@@ -47,7 +26,7 @@ static inline bool bpf_has_stack_frame(struct codegen_context *ctx) · 2 unmodified lines
47	* - the bpf program uses its stack area	26	* - the bpf program uses its stack area
48	* The latter condition is deduced from the usage of BPF_REG_FP	27	* The latter condition is deduced from the usage of BPF_REG_FP
49	*/	28	*/
50	return ctx->seen & SEEN_FUNC \|\| bpf_is_seen_register(ctx, BPF_REG_FP);	29	return ctx->seen & SEEN_FUNC \|\| bpf_is_seen_register(ctx, b2p[BPF_REG_FP]);
51	}	30	}
52		31
53	/*	32	/*
@@ -85,7 +64,11 @@ static int bpf_jit_stack_offsetof(struct codegen_context *ctx, int reg) · 31 unmodified lines
85	BUG();	64	BUG();
86	}	65	}
87		66
88	static void bpf_jit_build_prologue(u32 image, struct codegen_context ctx)	67	void bpf_jit_realloc_regs(struct codegen_context *ctx)
		68	{
		69	}
		70
		71	void bpf_jit_build_prologue(u32 image, struct codegen_context ctx)
89	{	72	{
90	int i;	73	int i;
91		74
@@ -124,11 +107,11 @@ static void bpf_jit_build_prologue(u32 image, struct codegen_context ctx) · 32 unmodified lines
124	* in the protected zone below the previous stack frame	107	* in the protected zone below the previous stack frame
125	*/	108	*/
126	for (i = BPF_REG_6; i <= BPF_REG_10; i++)	109	for (i = BPF_REG_6; i <= BPF_REG_10; i++)
127	if (bpf_is_seen_register(ctx, i))	110	if (bpf_is_seen_register(ctx, b2p[i]))
128	PPC_BPF_STL(b2p[i], 1, bpf_jit_stack_offsetof(ctx, b2p[i]));	111	PPC_BPF_STL(b2p[i], 1, bpf_jit_stack_offsetof(ctx, b2p[i]));
129		112
130	/* Setup frame pointer to point to the bpf stack area */	113	/* Setup frame pointer to point to the bpf stack area */
131	if (bpf_is_seen_register(ctx, BPF_REG_FP))	114	if (bpf_is_seen_register(ctx, b2p[BPF_REG_FP]))
132	EMIT(PPC_RAW_ADDI(b2p[BPF_REG_FP], 1,	115	EMIT(PPC_RAW_ADDI(b2p[BPF_REG_FP], 1,
133	STACK_FRAME_MIN_SIZE + ctx->stack_size));	116	STACK_FRAME_MIN_SIZE + ctx->stack_size));
134	}	117	}
@@ -139,7 +122,7 @@ static void bpf_jit_emit_common_epilogue(u32 image, struct codegen_context ctx · 4 unmodified lines
139		122
140	/* Restore NVRs */	123	/* Restore NVRs */
141	for (i = BPF_REG_6; i <= BPF_REG_10; i++)	124	for (i = BPF_REG_6; i <= BPF_REG_10; i++)
142	if (bpf_is_seen_register(ctx, i))	125	if (bpf_is_seen_register(ctx, b2p[i]))
143	PPC_BPF_LL(b2p[i], 1, bpf_jit_stack_offsetof(ctx, b2p[i]));	126	PPC_BPF_LL(b2p[i], 1, bpf_jit_stack_offsetof(ctx, b2p[i]));
144		127
145	/* Tear down our stack frame */	128	/* Tear down our stack frame */
@@ -152,7 +135,7 @@ static void bpf_jit_emit_common_epilogue(u32 image, struct codegen_context ctx · 6 unmodified lines
152	}	135	}
153	}	136	}
154		137
155	static void bpf_jit_build_epilogue(u32 image, struct codegen_context ctx)	138	void bpf_jit_build_epilogue(u32 image, struct codegen_context ctx)
156	{	139	{
157	bpf_jit_emit_common_epilogue(image, ctx);	140	bpf_jit_emit_common_epilogue(image, ctx);
158		141
@@ -187,8 +170,7 @@ static void bpf_jit_emit_func_call_hlp(u32 image, struct codegen_context ctx, · 28 unmodified lines
187	EMIT(PPC_RAW_BLRL());	170	EMIT(PPC_RAW_BLRL());
188	}	171	}
189		172
190	static void bpf_jit_emit_func_call_rel(u32 image, struct codegen_context ctx,	173	void bpf_jit_emit_func_call_rel(u32 image, struct codegen_context ctx, u64 func)
191	u64 func)
192	{	174	{
193	unsigned int i, ctx_idx = ctx->idx;	175	unsigned int i, ctx_idx = ctx->idx;
194		176
@@ -289,9 +271,8 @@ static void bpf_jit_emit_tail_call(u32 image, struct codegen_context ctx, u32 · 94 unmodified lines
289	}	271	}
290		272
291	/* Assemble the body code between the prologue & epilogue */	273	/* Assemble the body code between the prologue & epilogue */
292	static int bpf_jit_build_body(struct bpf_prog fp, u32 image,	274	int bpf_jit_build_body(struct bpf_prog fp, u32 image, struct codegen_context *ctx,
293	struct codegen_context *ctx,	275	u32 *addrs, bool extra_pass)
294	u32 *addrs, bool extra_pass)
295	{	276	{
296	const struct bpf_insn *insn = fp->insnsi;	277	const struct bpf_insn *insn = fp->insnsi;
297	int flen = fp->len;	278	int flen = fp->len;
@@ -330,9 +311,9 @@ static int bpf_jit_build_body(struct bpf_prog fp, u32 image, · 32 unmodified lines
330	* any issues.	311	* any issues.
331	*/	312	*/
332	if (dst_reg >= BPF_PPC_NVR_MIN && dst_reg < 32)	313	if (dst_reg >= BPF_PPC_NVR_MIN && dst_reg < 32)
333	bpf_set_seen_register(ctx, insn[i].dst_reg);	314	bpf_set_seen_register(ctx, dst_reg);
334	if (src_reg >= BPF_PPC_NVR_MIN && src_reg < 32)	315	if (src_reg >= BPF_PPC_NVR_MIN && src_reg < 32)
335	bpf_set_seen_register(ctx, insn[i].src_reg);	316	bpf_set_seen_register(ctx, src_reg);
336		317
337	switch (code) {	318	switch (code) {
338	/*	319	/*
@@ -1026,249 +1007,3 @@ cond_branch: · 687 unmodified lines
1026		1007
1027	return 0;	1008	return 0;
1028	}	1009	}
1029
1030	/* Fix the branch target addresses for subprog calls */
1031	static int bpf_jit_fixup_subprog_calls(struct bpf_prog fp, u32 image,
1032	struct codegen_context ctx, u32 addrs)
1033	{
1034	const struct bpf_insn *insn = fp->insnsi;
1035	bool func_addr_fixed;
1036	u64 func_addr;
1037	u32 tmp_idx;
1038	int i, ret;
1039
1040	for (i = 0; i < fp->len; i++) {
1041	/*
1042	* During the extra pass, only the branch target addresses for
1043	* the subprog calls need to be fixed. All other instructions
1044	* can left untouched.
1045	*
1046	* The JITed image length does not change because we already
1047	* ensure that the JITed instruction sequence for these calls
1048	* are of fixed length by padding them with NOPs.
1049	*/
1050	if (insn[i].code == (BPF_JMP \| BPF_CALL) &&
1051	insn[i].src_reg == BPF_PSEUDO_CALL) {
1052	ret = bpf_jit_get_func_addr(fp, &insn[i], true,
1053	&func_addr,
1054	&func_addr_fixed);
1055	if (ret < 0)
1056	return ret;
1057
1058	/*
1059	* Save ctx->idx as this would currently point to the
1060	* end of the JITed image and set it to the offset of
1061	* the instruction sequence corresponding to the
1062	* subprog call temporarily.
1063	*/
1064	tmp_idx = ctx->idx;
1065	ctx->idx = addrs[i] / 4;
1066	bpf_jit_emit_func_call_rel(image, ctx, func_addr);
1067
1068	/*
1069	* Restore ctx->idx here. This is safe as the length
1070	* of the JITed sequence remains unchanged.
1071	*/
1072	ctx->idx = tmp_idx;
1073	}
1074	}
1075
1076	return 0;
1077	}
1078
1079	struct powerpc64_jit_data {
1080	struct bpf_binary_header *header;
1081	u32 *addrs;
1082	u8 *image;
1083	u32 proglen;
1084	struct codegen_context ctx;
1085	};
1086
1087	bool bpf_jit_needs_zext(void)
1088	{
1089	return true;
1090	}
1091
1092	struct bpf_prog bpf_int_jit_compile(struct bpf_prog fp)
1093	{
1094	u32 proglen;
1095	u32 alloclen;
1096	u8 *image = NULL;
1097	u32 *code_base;
1098	u32 *addrs;
1099	struct powerpc64_jit_data *jit_data;
1100	struct codegen_context cgctx;
1101	int pass;
1102	int flen;
1103	struct bpf_binary_header *bpf_hdr;
1104	struct bpf_prog *org_fp = fp;
1105	struct bpf_prog *tmp_fp;
1106	bool bpf_blinded = false;
1107	bool extra_pass = false;
1108
1109	if (!fp->jit_requested)
1110	return org_fp;
1111
1112	tmp_fp = bpf_jit_blind_constants(org_fp);
1113	if (IS_ERR(tmp_fp))
1114	return org_fp;
1115
1116	if (tmp_fp != org_fp) {
1117	bpf_blinded = true;
1118	fp = tmp_fp;
1119	}
1120
1121	jit_data = fp->aux->jit_data;
1122	if (!jit_data) {
1123	jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL);
1124	if (!jit_data) {
1125	fp = org_fp;
1126	goto out;
1127	}
1128	fp->aux->jit_data = jit_data;
1129	}
1130
1131	flen = fp->len;
1132	addrs = jit_data->addrs;
1133	if (addrs) {
1134	cgctx = jit_data->ctx;
1135	image = jit_data->image;
1136	bpf_hdr = jit_data->header;
1137	proglen = jit_data->proglen;
1138	alloclen = proglen + FUNCTION_DESCR_SIZE;
1139	extra_pass = true;
1140	goto skip_init_ctx;
1141	}
1142
1143	addrs = kcalloc(flen + 1, sizeof(*addrs), GFP_KERNEL);
1144	if (addrs == NULL) {
1145	fp = org_fp;
1146	goto out_addrs;
1147	}
1148
1149	memset(&cgctx, 0, sizeof(struct codegen_context));
1150
1151	/* Make sure that the stack is quadword aligned. */
1152	cgctx.stack_size = round_up(fp->aux->stack_depth, 16);
1153
1154	/* Scouting faux-generate pass 0 */
1155	if (bpf_jit_build_body(fp, 0, &cgctx, addrs, false)) {
1156	/* We hit something illegal or unsupported. */
1157	fp = org_fp;
1158	goto out_addrs;
1159	}
1160
1161	/*
1162	* If we have seen a tail call, we need a second pass.
1163	* This is because bpf_jit_emit_common_epilogue() is called
1164	* from bpf_jit_emit_tail_call() with a not yet stable ctx->seen.
1165	*/
1166	if (cgctx.seen & SEEN_TAILCALL) {
1167	cgctx.idx = 0;
1168	if (bpf_jit_build_body(fp, 0, &cgctx, addrs, false)) {
1169	fp = org_fp;
1170	goto out_addrs;
1171	}
1172	}
1173
1174	/*
1175	* Pretend to build prologue, given the features we've seen. This will
1176	* update ctgtx.idx as it pretends to output instructions, then we can
1177	* calculate total size from idx.
1178	*/
1179	bpf_jit_build_prologue(0, &cgctx);
1180	bpf_jit_build_epilogue(0, &cgctx);
1181
1182	proglen = cgctx.idx * 4;
1183	alloclen = proglen + FUNCTION_DESCR_SIZE;
1184
1185	bpf_hdr = bpf_jit_binary_alloc(alloclen, &image, 4,
1186	bpf_jit_fill_ill_insns);
1187	if (!bpf_hdr) {
1188	fp = org_fp;
1189	goto out_addrs;
1190	}
1191
1192	skip_init_ctx:
1193	code_base = (u32 *)(image + FUNCTION_DESCR_SIZE);
1194
1195	if (extra_pass) {
1196	/*
1197	* Do not touch the prologue and epilogue as they will remain
1198	* unchanged. Only fix the branch target address for subprog
1199	* calls in the body.
1200	*
1201	* This does not change the offsets and lengths of the subprog
1202	* call instruction sequences and hence, the size of the JITed
1203	* image as well.
1204	*/
1205	bpf_jit_fixup_subprog_calls(fp, code_base, &cgctx, addrs);
1206
1207	/* There is no need to perform the usual passes. */
1208	goto skip_codegen_passes;
1209	}
1210
1211	/* Code generation passes 1-2 */
1212	for (pass = 1; pass < 3; pass++) {
1213	/* Now build the prologue, body code & epilogue for real. */
1214	cgctx.idx = 0;
1215	bpf_jit_build_prologue(code_base, &cgctx);
1216	bpf_jit_build_body(fp, code_base, &cgctx, addrs, extra_pass);
1217	bpf_jit_build_epilogue(code_base, &cgctx);
1218
1219	if (bpf_jit_enable > 1)
1220	pr_info("Pass %d: shrink = %d, seen = 0x%x\n", pass,
1221	proglen - (cgctx.idx * 4), cgctx.seen);
1222	}
1223
1224	skip_codegen_passes:
1225	if (bpf_jit_enable > 1)
1226	/*
1227	* Note that we output the base address of the code_base
1228	* rather than image, since opcodes are in code_base.
1229	*/
1230	bpf_jit_dump(flen, proglen, pass, code_base);
1231
1232	#ifdef PPC64_ELF_ABI_v1
1233	/* Function descriptor nastiness: Address + TOC */
1234	((u64 *)image)[0] = (u64)code_base;
1235	((u64 *)image)[1] = local_paca->kernel_toc;
1236	#endif
1237
1238	fp->bpf_func = (void *)image;
1239	fp->jited = 1;
1240	fp->jited_len = alloclen;
1241
1242	bpf_flush_icache(bpf_hdr, (u8 )bpf_hdr + (bpf_hdr->pages PAGE_SIZE));
1243	if (!fp->is_func \|\| extra_pass) {
1244	bpf_prog_fill_jited_linfo(fp, addrs);
1245	out_addrs:
1246	kfree(addrs);
1247	kfree(jit_data);
1248	fp->aux->jit_data = NULL;
1249	} else {
1250	jit_data->addrs = addrs;
1251	jit_data->ctx = cgctx;
1252	jit_data->proglen = proglen;
1253	jit_data->image = image;
1254	jit_data->header = bpf_hdr;
1255	}
1256
1257	out:
1258	if (bpf_blinded)
1259	bpf_jit_prog_release_other(fp, fp == org_fp ? tmp_fp : org_fp);
1260
1261	return fp;
1262	}
1263
1264	/* Overriding bpf_jit_free() as we don't set images read-only. */
1265	void bpf_jit_free(struct bpf_prog *fp)
1266	{
1267	unsigned long addr = (unsigned long)fp->bpf_func & PAGE_MASK;
1268	struct bpf_binary_header bpf_hdr = (void )addr;
1269
1270	if (fp->jited)
1271	bpf_jit_binary_free(bpf_hdr);
1272
1273	bpf_prog_unlock_free(fp);
1274	}

arch/powerpc/perf/core-book3s.c+16−4

 #include <asm/firmware.h>
 #include <asm/ptrace.h>
 #include <asm/code-patching.h>
+#include <asm/interrupt.h>
 #ifdef CONFIG_PPC64
 #include "internal.h"
 	 * they have not been setup using perf_read_regs() and so regs->result
 	 * is something random.
 	 */
-	return ((TRAP(regs) == 0xf00) && regs->result);
+	return ((TRAP(regs) == INTERRUPT_PERFMON) && regs->result);
 }
 /*
 	 * hypervisor samples as well as samples in the kernel with
 	 * interrupts off hence the userspace check.
 	 */
-	if (TRAP(regs) != 0xf00)
+	if (TRAP(regs) != INTERRUPT_PERFMON)
 		use_siar = 0;
 	else if ((ppmu->flags & PPMU_NO_SIAR))
 		use_siar = 0;
 		return -ENOENT;
 	}
+	/*
+	 * PMU config registers have fields that are
+	 * reserved and some specific values for bit fields are reserved.
+	 * For ex., MMCRA[61:62] is Randome Sampling Mode (SM)
+	 * and value of 0b11 to this field is reserved.
+	 * Check for invalid values in attr.config.
+	 */
+	if (ppmu->check_attr_config &&
+	    ppmu->check_attr_config(event))
+		return -EINVAL;
 	event->hw.config_base = ev;
 	event->hw.idx = 0;
 						ppmu->get_mem_data_src)
 			ppmu->get_mem_data_src(&data.data_src, ppmu->flags, regs);
-		if (event->attr.sample_type & PERF_SAMPLE_WEIGHT &&
+		if (event->attr.sample_type & PERF_SAMPLE_WEIGHT_TYPE &&
 						ppmu->get_mem_weight)
-			ppmu->get_mem_weight(&data.weight.full);
+			ppmu->get_mem_weight(&data.weight.full, event->attr.sample_type);
 		if (perf_event_overflow(event, &data, regs))
 			power_pmu_stop(event, 0);

arch/powerpc/perf/hv-24x7.c+5−5

@@ -226,14 +226,14 @@ static struct attribute_group event_long_desc_group = { · 225 unmodified lines
226		226
227	static struct kmem_cache *hv_page_cache;	227	static struct kmem_cache *hv_page_cache;
228		228
229	DEFINE_PER_CPU(int, hv_24x7_txn_flags);	229	static DEFINE_PER_CPU(int, hv_24x7_txn_flags);
230	DEFINE_PER_CPU(int, hv_24x7_txn_err);	230	static DEFINE_PER_CPU(int, hv_24x7_txn_err);
231		231
232	struct hv_24x7_hw {	232	struct hv_24x7_hw {
233	struct perf_event *events[255];	233	struct perf_event *events[255];
234	};	234	};
235		235
236	DEFINE_PER_CPU(struct hv_24x7_hw, hv_24x7_hw);	236	static DEFINE_PER_CPU(struct hv_24x7_hw, hv_24x7_hw);
237		237
238	/*	238	/*
239	* request_buffer and result_buffer are not required to be 4k aligned,	239	* request_buffer and result_buffer are not required to be 4k aligned,
@@ -241,8 +241,8 @@ DEFINE_PER_CPU(struct hv_24x7_hw, hv_24x7_hw); · 1 unmodified line
241	* the simplest way to ensure that.	241	* the simplest way to ensure that.
242	*/	242	*/
243	#define H24x7_DATA_BUFFER_SIZE 4096	243	#define H24x7_DATA_BUFFER_SIZE 4096
244	DEFINE_PER_CPU(char, hv_24x7_reqb[H24x7_DATA_BUFFER_SIZE]) __aligned(4096);	244	static DEFINE_PER_CPU(char, hv_24x7_reqb[H24x7_DATA_BUFFER_SIZE]) __aligned(4096);
245	DEFINE_PER_CPU(char, hv_24x7_resb[H24x7_DATA_BUFFER_SIZE]) __aligned(4096);	245	static DEFINE_PER_CPU(char, hv_24x7_resb[H24x7_DATA_BUFFER_SIZE]) __aligned(4096);
246		246
247	static unsigned int max_num_requests(int interface_version)	247	static unsigned int max_num_requests(int interface_version)
248	{	248	{

arch/powerpc/perf/isa207-common.c+104−11

@@ -21,7 +21,7 @@ PMU_FORMAT_ATTR(thresh_stop, "config:32-35"); · 20 unmodified lines
21	PMU_FORMAT_ATTR(thresh_start, "config:36-39");	21	PMU_FORMAT_ATTR(thresh_start, "config:36-39");
22	PMU_FORMAT_ATTR(thresh_cmp, "config:40-49");	22	PMU_FORMAT_ATTR(thresh_cmp, "config:40-49");
23		23
24	struct attribute *isa207_pmu_format_attr[] = {	24	static struct attribute *isa207_pmu_format_attr[] = {
25	&format_attr_event.attr,	25	&format_attr_event.attr,
26	&format_attr_pmcxsel.attr,	26	&format_attr_pmcxsel.attr,
27	&format_attr_mark.attr,	27	&format_attr_mark.attr,
@@ -275,17 +275,47 @@ void isa207_get_mem_data_src(union perf_mem_data_src *dsrc, u32 flags, · 247 unmodified lines
275		275
276	sier = mfspr(SPRN_SIER);	276	sier = mfspr(SPRN_SIER);
277	val = (sier & ISA207_SIER_TYPE_MASK) >> ISA207_SIER_TYPE_SHIFT;	277	val = (sier & ISA207_SIER_TYPE_MASK) >> ISA207_SIER_TYPE_SHIFT;
278	if (val == 1 \|\| val == 2) {	278	if (val != 1 && val != 2 && !(val == 7 && cpu_has_feature(CPU_FTR_ARCH_31)))
279	idx = (sier & ISA207_SIER_LDST_MASK) >> ISA207_SIER_LDST_SHIFT;	279	return;
280	sub_idx = (sier & ISA207_SIER_DATA_SRC_MASK) >> ISA207_SIER_DATA_SRC_SHIFT;	280
		281	idx = (sier & ISA207_SIER_LDST_MASK) >> ISA207_SIER_LDST_SHIFT;
		282	sub_idx = (sier & ISA207_SIER_DATA_SRC_MASK) >> ISA207_SIER_DATA_SRC_SHIFT;
		283
		284	dsrc->val = isa207_find_source(idx, sub_idx);
		285	if (val == 7) {
		286	u64 mmcra;
		287	u32 op_type;
281		288
282	dsrc->val = isa207_find_source(idx, sub_idx);	289	/*
		290	* Type 0b111 denotes either larx or stcx instruction. Use the
		291	* MMCRA sampling bits [57:59] along with the type value
		292	* to determine the exact instruction type. If the sampling
		293	* criteria is neither load or store, set the type as default
		294	* to NA.
		295	*/
		296	mmcra = mfspr(SPRN_MMCRA);
		297
		298	op_type = (mmcra >> MMCRA_SAMP_ELIG_SHIFT) & MMCRA_SAMP_ELIG_MASK;
		299	switch (op_type) {
		300	case 5:
		301	dsrc->val \|= P(OP, LOAD);
		302	break;
		303	case 7:
		304	dsrc->val \|= P(OP, STORE);
		305	break;
		306	default:
		307	dsrc->val \|= P(OP, NA);
		308	break;
		309	}
		310	} else {
283	dsrc->val \|= (val == 1) ? P(OP, LOAD) : P(OP, STORE);	311	dsrc->val \|= (val == 1) ? P(OP, LOAD) : P(OP, STORE);
284	}	312	}
285	}	313	}
286		314
287	void isa207_get_mem_weight(u64 *weight)	315	void isa207_get_mem_weight(u64 *weight, u64 type)
288	{	316	{
		317	union perf_sample_weight *weight_fields;
		318	u64 weight_lat;
289	u64 mmcra = mfspr(SPRN_MMCRA);	319	u64 mmcra = mfspr(SPRN_MMCRA);
290	u64 exp = MMCRA_THR_CTR_EXP(mmcra);	320	u64 exp = MMCRA_THR_CTR_EXP(mmcra);
291	u64 mantissa = MMCRA_THR_CTR_MANT(mmcra);	321	u64 mantissa = MMCRA_THR_CTR_MANT(mmcra);
@@ -295,10 +325,31 @@ void isa207_get_mem_weight(u64 *weight) · 3 unmodified lines
295	if (cpu_has_feature(CPU_FTR_ARCH_31))	325	if (cpu_has_feature(CPU_FTR_ARCH_31))
296	mantissa = P10_MMCRA_THR_CTR_MANT(mmcra);	326	mantissa = P10_MMCRA_THR_CTR_MANT(mmcra);
297		327
298	if (val == 0 \|\| val == 7)	328	if (val == 0 \|\| (val == 7 && !cpu_has_feature(CPU_FTR_ARCH_31)))
299	*weight = 0;	329	weight_lat = 0;
300	else	330	else
301	weight = mantissa << (2 exp);	331	weight_lat = mantissa << (2 * exp);
		332
		333	/*
		334	* Use 64 bit weight field (full) if sample type is
		335	* WEIGHT.
		336	*
		337	* if sample type is WEIGHT_STRUCT:
		338	* - store memory latency in the lower 32 bits.
		339	* - For ISA v3.1, use remaining two 16 bit fields of
		340	* perf_sample_weight to store cycle counter values
		341	* from sier2.
		342	*/
		343	weight_fields = (union perf_sample_weight *)weight;
		344	if (type & PERF_SAMPLE_WEIGHT)
		345	weight_fields->full = weight_lat;
		346	else {
		347	weight_fields->var1_dw = (u32)weight_lat;
		348	if (cpu_has_feature(CPU_FTR_ARCH_31)) {
		349	weight_fields->var2_w = P10_SIER2_FINISH_CYC(mfspr(SPRN_SIER2));
		350	weight_fields->var3_w = P10_SIER2_DISPATCH_CYC(mfspr(SPRN_SIER2));
		351	}
		352	}
302	}	353	}
303		354
304	int isa207_get_constraint(u64 event, unsigned long maskp, unsigned long valp, u64 event_config1)	355	int isa207_get_constraint(u64 event, unsigned long maskp, unsigned long valp, u64 event_config1)
@@ -447,8 +498,8 @@ ebb_bhrb: · 142 unmodified lines
447	* EBB events are pinned & exclusive, so this should never actually	498	* EBB events are pinned & exclusive, so this should never actually
448	* hit, but we leave it as a fallback in case.	499	* hit, but we leave it as a fallback in case.
449	*/	500	*/
450	mask \|= CNST_EBB_VAL(ebb);	501	mask \|= CNST_EBB_MASK;
451	value \|= CNST_EBB_MASK;	502	value \|= CNST_EBB_VAL(ebb);
452		503
453	*maskp = mask;	504	*maskp = mask;
454	*valp = value;	505	*valp = value;
@@ -694,3 +745,45 @@ int isa207_get_alternatives(u64 event, u64 alt[], int size, unsigned int flags, · 239 unmodified lines
694		745
695	return num_alt;	746	return num_alt;
696	}	747	}
		748
		749	int isa3XX_check_attr_config(struct perf_event *ev)
		750	{
		751	u64 val, sample_mode;
		752	u64 event = ev->attr.config;
		753
		754	val = (event >> EVENT_SAMPLE_SHIFT) & EVENT_SAMPLE_MASK;
		755	sample_mode = val & 0x3;
		756
		757	/*
		758	* MMCRA[61:62] is Random Sampling Mode (SM).
		759	* value of 0b11 is reserved.
		760	*/
		761	if (sample_mode == 0x3)
		762	return -EINVAL;
		763
		764	/*
		765	* Check for all reserved value
		766	* Source: Performance Monitoring Unit User Guide
		767	*/
		768	switch (val) {
		769	case 0x5:
		770	case 0x9:
		771	case 0xD:
		772	case 0x19:
		773	case 0x1D:
		774	case 0x1A:
		775	case 0x1E:
		776	return -EINVAL;
		777	}
		778
		779	/*
		780	* MMCRA[48:51]/[52:55]) Threshold Start/Stop
		781	* Events Selection.
		782	* 0b11110000/0b00001111 is reserved.
		783	*/
		784	val = (event >> EVENT_THR_CTL_SHIFT) & EVENT_THR_CTL_MASK;
		785	if (((val & 0xF0) == 0xF0) \|\| ((val & 0xF) == 0xF))
		786	return -EINVAL;
		787
		788	return 0;
		789	}

arch/powerpc/perf/isa207-common.h+8−1

 /* Bits in MMCRA for PowerISA v2.07 */
 #define MMCRA_SAMP_MODE_SHIFT		1
 #define MMCRA_SAMP_ELIG_SHIFT		4
+#define MMCRA_SAMP_ELIG_MASK		7
 #define MMCRA_THR_CTL_SHIFT		8
 #define MMCRA_THR_SEL_SHIFT		16
 #define MMCRA_THR_CMP_SHIFT		32
 #define ISA207_SIER_DATA_SRC_SHIFT	53
 #define ISA207_SIER_DATA_SRC_MASK	(0x7ull << ISA207_SIER_DATA_SRC_SHIFT)
+/* Bits in SIER2/SIER3 for Power10 */
+#define P10_SIER2_FINISH_CYC(sier2)	(((sier2) >> (63 - 37)) & 0x7fful)
+#define P10_SIER2_DISPATCH_CYC(sier2)	(((sier2) >> (63 - 13)) & 0x7fful)
 #define P(a, b)				PERF_MEM_S(a, b)
 #define PH(a, b)			(P(LVL, HIT) | P(a, b))
 #define PM(a, b)			(P(LVL, MISS) | P(a, b))
 					const unsigned int ev_alt[][MAX_ALT]);
 void isa207_get_mem_data_src(union perf_mem_data_src *dsrc, u32 flags,
 							struct pt_regs *regs);
-void isa207_get_mem_weight(u64 *weight);
+void isa207_get_mem_weight(u64 *weight, u64 type);
+int isa3XX_check_attr_config(struct perf_event *ev);
 #endif

arch/powerpc/perf/power10-events-list.h+2−2

@@ -75,5 +75,5 @@ EVENT(PM_RUN_INST_CMPL_ALT, 0x00002); · 74 unmodified lines
75	* thresh end (TE)	75	* thresh end (TE)
76	*/	76	*/
77		77
78	EVENT(MEM_LOADS, 0x34340401e0);	78	EVENT(MEM_LOADS, 0x35340401e0);
79	EVENT(MEM_STORES, 0x343c0401e0);	79	EVENT(MEM_STORES, 0x353c0401e0);

arch/powerpc/perf/power10-pmu.c+13−0

@@ -106,6 +106,18 @@ static int power10_get_alternatives(u64 event, unsigned int flags, u64 alt[]) · 105 unmodified lines
106	return num_alt;	106	return num_alt;
107	}	107	}
108		108
		109	static int power10_check_attr_config(struct perf_event *ev)
		110	{
		111	u64 val;
		112	u64 event = ev->attr.config;
		113
		114	val = (event >> EVENT_SAMPLE_SHIFT) & EVENT_SAMPLE_MASK;
		115	if (val == 0x10 \|\| isa3XX_check_attr_config(ev))
		116	return -EINVAL;
		117
		118	return 0;
		119	}
		120
109	GENERIC_EVENT_ATTR(cpu-cycles, PM_RUN_CYC);	121	GENERIC_EVENT_ATTR(cpu-cycles, PM_RUN_CYC);
110	GENERIC_EVENT_ATTR(instructions, PM_RUN_INST_CMPL);	122	GENERIC_EVENT_ATTR(instructions, PM_RUN_INST_CMPL);
111	GENERIC_EVENT_ATTR(branch-instructions, PM_BR_CMPL);	123	GENERIC_EVENT_ATTR(branch-instructions, PM_BR_CMPL);
@@ -559,6 +571,7 @@ static struct power_pmu power10_pmu = { · 447 unmodified lines
559	.attr_groups = power10_pmu_attr_groups,	571	.attr_groups = power10_pmu_attr_groups,
560	.bhrb_nr = 32,	572	.bhrb_nr = 32,
561	.capabilities = PERF_PMU_CAP_EXTENDED_REGS,	573	.capabilities = PERF_PMU_CAP_EXTENDED_REGS,
		574	.check_attr_config = power10_check_attr_config,
562	};	575	};
563		576
564	int init_power10_pmu(void)	577	int init_power10_pmu(void)

arch/powerpc/perf/power9-pmu.c+13−0

@@ -151,6 +151,18 @@ static int power9_get_alternatives(u64 event, unsigned int flags, u64 alt[]) · 150 unmodified lines
151	return num_alt;	151	return num_alt;
152	}	152	}
153		153
		154	static int power9_check_attr_config(struct perf_event *ev)
		155	{
		156	u64 val;
		157	u64 event = ev->attr.config;
		158
		159	val = (event >> EVENT_SAMPLE_SHIFT) & EVENT_SAMPLE_MASK;
		160	if (val == 0xC \|\| isa3XX_check_attr_config(ev))
		161	return -EINVAL;
		162
		163	return 0;
		164	}
		165
154	GENERIC_EVENT_ATTR(cpu-cycles, PM_CYC);	166	GENERIC_EVENT_ATTR(cpu-cycles, PM_CYC);
155	GENERIC_EVENT_ATTR(stalled-cycles-frontend, PM_ICT_NOSLOT_CYC);	167	GENERIC_EVENT_ATTR(stalled-cycles-frontend, PM_ICT_NOSLOT_CYC);
156	GENERIC_EVENT_ATTR(stalled-cycles-backend, PM_CMPLU_STALL);	168	GENERIC_EVENT_ATTR(stalled-cycles-backend, PM_CMPLU_STALL);
@@ -437,6 +449,7 @@ static struct power_pmu power9_pmu = { · 280 unmodified lines
437	.attr_groups = power9_pmu_attr_groups,	449	.attr_groups = power9_pmu_attr_groups,
438	.bhrb_nr = 32,	450	.bhrb_nr = 32,
439	.capabilities = PERF_PMU_CAP_EXTENDED_REGS,	451	.capabilities = PERF_PMU_CAP_EXTENDED_REGS,
		452	.check_attr_config = power9_check_attr_config,
440	};	453	};
441		454
442	int init_power9_pmu(void)	455	int init_power9_pmu(void)

arch/powerpc/platforms/44x/Kconfig+1−1

@@ -5,7 +5,7 @@ config PPC_47x · 4 unmodified lines
5	select MPIC	5	select MPIC
6	help	6	help
7	This option enables support for the 47x family of processors and is	7	This option enables support for the 47x family of processors and is
8	not currently compatible with other 44x or 46x varients	8	not currently compatible with other 44x or 46x variants
9		9
10	config BAMBOO	10	config BAMBOO
11	bool "Bamboo"	11	bool "Bamboo"

arch/powerpc/platforms/52xx/lite5200_sleep.S+1−1

@@ -181,7 +181,7 @@ sram_code: · 180 unmodified lines
181	udelay: /* r11 - tb_ticks_per_usec, r12 - usecs, overwrites r13 */	181	udelay: /* r11 - tb_ticks_per_usec, r12 - usecs, overwrites r13 */
182	mullw r12, r12, r11	182	mullw r12, r12, r11
183	mftb r13 /* start */	183	mftb r13 /* start */
184	addi r12, r13, r12 /* end */	184	add r12, r13, r12 /* end */
185	1:	185	1:
186	mftb r13 /* current */	186	mftb r13 /* current */
187	cmp cr0, r13, r12	187	cmp cr0, r13, r12

arch/powerpc/platforms/Kconfig.cputype+3−2

 	select ARCH_SUPPORTS_NUMA_BALANCING
 	select IRQ_WORK
 	select PPC_MM_SLICES
+	select PPC_HAVE_KUEP
+	select PPC_HAVE_KUAP
 config PPC_BOOK3E_64
 	bool "Embedded processors"
 config ALTIVEC
 	bool "AltiVec Support"
 	depends on PPC_BOOK3S_32 || PPC_BOOK3S_64 || (PPC_E500MC && PPC64)
+	select PPC_FPU
 	help
 	  This option enables kernel support for the Altivec extensions to the
 	  PowerPC processor. The kernel currently supports saving and restoring
 	bool "Radix MMU Support"
 	depends on PPC_BOOK3S_64
 	select ARCH_HAS_GIGANTIC_PAGE
-	select PPC_HAVE_KUEP
-	select PPC_HAVE_KUAP
 	default y
 	help
 	  Enable support for the Power ISA 3.0 Radix style MMU. Currently this

arch/powerpc/platforms/cell/iommu.c+2−1

@@ -486,7 +486,8 @@ cell_iommu_setup_window(struct cbe_iommu iommu, struct device_node np, · 485 unmodified lines
486	window->table.it_size = size >> window->table.it_page_shift;	486	window->table.it_size = size >> window->table.it_page_shift;
487	window->table.it_ops = &cell_iommu_ops;	487	window->table.it_ops = &cell_iommu_ops;
488		488
489	iommu_init_table(&window->table, iommu->nid, 0, 0);	489	if (!iommu_init_table(&window->table, iommu->nid, 0, 0))
		490	panic("Failed to initialize iommu table");
490		491
491	pr_debug("\tioid %d\n", window->ioid);	492	pr_debug("\tioid %d\n", window->ioid);
492	pr_debug("\tblocksize %ld\n", window->table.it_blocksize);	493	pr_debug("\tblocksize %ld\n", window->table.it_blocksize);

arch/powerpc/platforms/cell/spu_callbacks.c+1−1

  */
 static void *spu_syscall_table[] = {
+#define __SYSCALL_WITH_COMPAT(nr, entry, compat) __SYSCALL(nr, entry)
 #define __SYSCALL(nr, entry) [nr] = entry,
 #include <asm/syscall_table_spu.h>
-#undef __SYSCALL
 };
 long spu_sys_callback(struct spu_syscall_block *s)

arch/powerpc/platforms/chrp/pci.c+1−2

@@ -131,8 +131,7 @@ static struct pci_ops rtas_pci_ops = · 130 unmodified lines
131		131
132	volatile struct Hydra __iomem *Hydra = NULL;	132	volatile struct Hydra __iomem *Hydra = NULL;
133		133
134	int __init	134	static int __init hydra_init(void)
135	hydra_init(void)
136	{	135	{
137	struct device_node *np;	136	struct device_node *np;
138	struct resource r;	137	struct resource r;

arch/powerpc/platforms/embedded6xx/Kconfig+0−5

 	bool
 	select PPC_INDIRECT_PCI
-config MV64X60
-	bool
-	select PPC_INDIRECT_PCI
-	select CHECK_CACHE_COHERENCY
 config GAMECUBE_COMMON
 	bool

arch/powerpc/platforms/maple/pci.c+1−1

@@ -34,7 +34,7 @@ static struct pci_controller u3_agp, u3_ht, *u4_pcie; · 33 unmodified lines
34		34
35	static int __init fixup_one_level_bus_range(struct device_node *node, int higher)	35	static int __init fixup_one_level_bus_range(struct device_node *node, int higher)
36	{	36	{
37	for (; node != 0;node = node->sibling) {	37	for (; node; node = node->sibling) {
38	const int *bus_range;	38	const int *bus_range;
39	const unsigned int *class_code;	39	const unsigned int *class_code;
40	int len;	40	int len;

arch/powerpc/platforms/pasemi/iommu.c+3−1

@@ -146,7 +146,9 @@ static void iommu_table_iobmap_setup(void) · 145 unmodified lines
146	*/	146	*/
147	iommu_table_iobmap.it_blocksize = 4;	147	iommu_table_iobmap.it_blocksize = 4;
148	iommu_table_iobmap.it_ops = &iommu_table_iobmap_ops;	148	iommu_table_iobmap.it_ops = &iommu_table_iobmap_ops;
149	iommu_init_table(&iommu_table_iobmap, 0, 0, 0);	149	if (!iommu_init_table(&iommu_table_iobmap, 0, 0, 0))
		150	panic("Failed to initialize iommu table");
		151
150	pr_debug(" <- %s\n", __func__);	152	pr_debug(" <- %s\n", __func__);
151	}	153	}
152		154

arch/powerpc/platforms/powernv/memtrace.c+17−1

 	return simple_read_from_buffer(ubuf, count, ppos, ent->mem, ent->size);
 }
+static int memtrace_mmap(struct file *filp, struct vm_area_struct *vma)
+{
+	struct memtrace_entry *ent = filp->private_data;
+	if (ent->size < vma->vm_end - vma->vm_start)
+		return -EINVAL;
+	if (vma->vm_pgoff << PAGE_SHIFT >= ent->size)
+		return -EINVAL;
+	vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
+	return remap_pfn_range(vma, vma->vm_start, PHYS_PFN(ent->start) + vma->vm_pgoff,
+			       vma->vm_end - vma->vm_start, vma->vm_page_prot);
+}
 static const struct file_operations memtrace_fops = {
 	.llseek = default_llseek,
 	.read	= memtrace_read,
 	.open	= simple_open,
+	.mmap   = memtrace_mmap,
 };
 #define FLUSH_CHUNK_SIZE SZ_1G
 		dir = debugfs_create_dir(ent->name, memtrace_debugfs_dir);
 		ent->dir = dir;
-		debugfs_create_file("trace", 0400, dir, ent, &memtrace_fops);
+		debugfs_create_file_unsafe("trace", 0600, dir, ent, &memtrace_fops);
 		debugfs_create_x64("start", 0400, dir, &ent->start);
 		debugfs_create_x64("size", 0400, dir, &ent->size);
 	}

arch/powerpc/platforms/powernv/opal-core.c+1−1

@@ -71,7 +71,7 @@ static LIST_HEAD(opalcore_list); · 70 unmodified lines
71	static struct opalcore_config *oc_conf;	71	static struct opalcore_config *oc_conf;
72	static const struct opal_mpipl_fadump *opalc_metadata;	72	static const struct opal_mpipl_fadump *opalc_metadata;
73	static const struct opal_mpipl_fadump *opalc_cpu_metadata;	73	static const struct opal_mpipl_fadump *opalc_cpu_metadata;
74	struct kobject *mpipl_kobj;	74	static struct kobject *mpipl_kobj;
75		75
76	/*	76	/*
77	* Set crashing CPU's signal to SIGUSR1. if the kernel is triggered	77	* Set crashing CPU's signal to SIGUSR1. if the kernel is triggered

arch/powerpc/platforms/powernv/opal-prd.c+1−4

 {
 	size_t addr, size;
 	pgprot_t page_prot;
-	int rc;
 	pr_devel("opal_prd_mmap(0x%016lx, 0x%016lx, 0x%lx, 0x%lx)\n",
 			vma->vm_start, vma->vm_end, vma->vm_pgoff,
 	page_prot = phys_mem_access_prot(file, vma->vm_pgoff,
 					 size, vma->vm_page_prot);
-	rc = remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, size,
+	return remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, size,
 				page_prot);
-	return rc;
 }
 static bool opal_msg_queue_empty(void)

arch/powerpc/platforms/powernv/pci-ioda.c+8−7

 	tbl->it_ops = &pnv_ioda1_iommu_ops;
 	pe->table_group.tce32_start = tbl->it_offset << tbl->it_page_shift;
 	pe->table_group.tce32_size = tbl->it_size << tbl->it_page_shift;
-	iommu_init_table(tbl, phb->hose->node, 0, 0);
+	if (!iommu_init_table(tbl, phb->hose->node, 0, 0))
+		panic("Failed to initialize iommu table");
 	pe->dma_setup_done = true;
 	return;
 		res_start = pe->phb->ioda.m32_pci_base >> tbl->it_page_shift;
 		res_end = min(window_size, SZ_4G) >> tbl->it_page_shift;
 	}
-	iommu_init_table(tbl, pe->phb->hose->node, res_start, res_end);
-	rc = pnv_pci_ioda2_set_window(&pe->table_group, 0, tbl);
+	if (iommu_init_table(tbl, pe->phb->hose->node, res_start, res_end))
+		rc = pnv_pci_ioda2_set_window(&pe->table_group, 0, tbl);
+	else
+		rc = -ENOMEM;
 	if (rc) {
-		pe_err(pe, "Failed to configure 32-bit TCE table, err %ld\n",
+		pe_err(pe, "Failed to configure 32-bit TCE table, err %ld\n", rc);
-				rc);
 		iommu_tce_table_put(tbl);
-		return rc;
+		tbl = NULL; /* This clears iommu_table_base below */
 	}
 	if (!pnv_iommu_bypass_disabled)
 		pnv_pci_ioda2_set_bypass(pe, true);

arch/powerpc/platforms/powernv/setup.c+1−1

@@ -157,7 +157,7 @@ static void __init pnv_check_guarded_cores(void) · 156 unmodified lines
157	for_each_node_by_type(dn, "cpu") {	157	for_each_node_by_type(dn, "cpu") {
158	if (of_property_match_string(dn, "status", "bad") >= 0)	158	if (of_property_match_string(dn, "status", "bad") >= 0)
159	bad_count++;	159	bad_count++;
160	};	160	}
161		161
162	if (bad_count) {	162	if (bad_count) {
163	printk(" _ _______________\n");	163	printk(" _ _______________\n");

arch/powerpc/platforms/pseries/dlpar.c+14−0

@@ -329,6 +329,20 @@ int dlpar_release_drc(u32 drc_index) · 328 unmodified lines
329	return 0;	329	return 0;
330	}	330	}
331		331
		332	int dlpar_unisolate_drc(u32 drc_index)
		333	{
		334	int dr_status, rc;
		335
		336	rc = rtas_call(rtas_token("get-sensor-state"), 2, 2, &dr_status,
		337	DR_ENTITY_SENSE, drc_index);
		338	if (rc \|\| dr_status != DR_ENTITY_PRESENT)
		339	return -1;
		340
		341	rtas_set_indicator(ISOLATION_STATE, drc_index, UNISOLATE);
		342
		343	return 0;
		344	}
		345
332	int handle_dlpar_errorlog(struct pseries_hp_errorlog *hp_elog)	346	int handle_dlpar_errorlog(struct pseries_hp_errorlog *hp_elog)
333	{	347	{
334	int rc;	348	int rc;

arch/powerpc/platforms/pseries/hotplug-cpu.c+23−4

 	BUG_ON(rtas_stop_self_token == RTAS_UNKNOWN_SERVICE);
-	printk("cpu %u (hwid %u) Ready to die...\n",
-	       smp_processor_id(), hard_smp_processor_id());
 	rtas_call_unlocked(&args, rtas_stop_self_token, 0, 1, NULL);
 	panic("Alas, I survived.\n");
 			if (!cpu_online(cpu))
 				break;
+			/*
+			 * device_offline() will return -EBUSY (via cpu_down()) if there
+			 * is only one CPU left. Check it here to fail earlier and with a
+			 * more informative error message, while also retaining the
+			 * cpu_add_remove_lock to be sure that no CPUs are being
+			 * online/offlined during this check.
+			 */
+			if (num_online_cpus() == 1) {
+				pr_warn("Unable to remove last online CPU %pOFn\n", dn);
+				rc = -EBUSY;
+				goto out_unlock;
+			}
 			cpu_maps_update_done();
 			rc = device_offline(get_cpu_device(cpu));
 			if (rc)
 				thread);
 		}
 	}
+out_unlock:
 	cpu_maps_update_done();
 out:
 	case PSERIES_HP_ELOG_ACTION_REMOVE:
 		if (hp_elog->id_type == PSERIES_HP_ELOG_ID_DRC_COUNT)
 			rc = dlpar_cpu_remove_by_count(count);
-		else if (hp_elog->id_type == PSERIES_HP_ELOG_ID_DRC_INDEX)
+		else if (hp_elog->id_type == PSERIES_HP_ELOG_ID_DRC_INDEX) {
 			rc = dlpar_cpu_remove_by_index(drc_index);
+			/*
+			 * Setting the isolation state of an UNISOLATED/CONFIGURED
+			 * device to UNISOLATE is a no-op, but the hypervisor can
+			 * use it as a hint that the CPU removal failed.
+			 */
+			if (rc)
+				dlpar_unisolate_drc(drc_index);
+		}
 		else
 			rc = -EINVAL;
 		break;

arch/powerpc/platforms/pseries/hvCall_inst.c+1−1

@@ -26,7 +26,7 @@ struct hcall_stats { · 25 unmodified lines
26	};	26	};
27	#define HCALL_STAT_ARRAY_SIZE ((MAX_HCALL_OPCODE >> 2) + 1)	27	#define HCALL_STAT_ARRAY_SIZE ((MAX_HCALL_OPCODE >> 2) + 1)
28		28
29	DEFINE_PER_CPU(struct hcall_stats[HCALL_STAT_ARRAY_SIZE], hcall_stats);	29	static DEFINE_PER_CPU(struct hcall_stats[HCALL_STAT_ARRAY_SIZE], hcall_stats);
30		30
31	/*	31	/*
32	* Routines for displaying the statistics in debugfs	32	* Routines for displaying the statistics in debugfs

arch/powerpc/platforms/pseries/iommu.c+38−11

@@ -638,7 +638,8 @@ static void pci_dma_bus_setup_pSeries(struct pci_bus *bus) · 637 unmodified lines
638		638
639	iommu_table_setparms(pci->phb, dn, tbl);	639	iommu_table_setparms(pci->phb, dn, tbl);
640	tbl->it_ops = &iommu_table_pseries_ops;	640	tbl->it_ops = &iommu_table_pseries_ops;
641	iommu_init_table(tbl, pci->phb->node, 0, 0);	641	if (!iommu_init_table(tbl, pci->phb->node, 0, 0))
		642	panic("Failed to initialize iommu table");
642		643
643	/* Divide the rest (1.75GB) among the children */	644	/* Divide the rest (1.75GB) among the children */
644	pci->phb->dma_window_size = 0x80000000ul;	645	pci->phb->dma_window_size = 0x80000000ul;
@@ -720,7 +721,8 @@ static void pci_dma_bus_setup_pSeriesLP(struct pci_bus *bus) · 75 unmodified lines
720	iommu_table_setparms_lpar(ppci->phb, pdn, tbl,	721	iommu_table_setparms_lpar(ppci->phb, pdn, tbl,
721	ppci->table_group, dma_window);	722	ppci->table_group, dma_window);
722	tbl->it_ops = &iommu_table_lpar_multi_ops;	723	tbl->it_ops = &iommu_table_lpar_multi_ops;
723	iommu_init_table(tbl, ppci->phb->node, 0, 0);	724	if (!iommu_init_table(tbl, ppci->phb->node, 0, 0))
		725	panic("Failed to initialize iommu table");
724	iommu_register_group(ppci->table_group,	726	iommu_register_group(ppci->table_group,
725	pci_domain_nr(bus), 0);	727	pci_domain_nr(bus), 0);
726	pr_debug(" created table: %p\n", ppci->table_group);	728	pr_debug(" created table: %p\n", ppci->table_group);
@@ -749,7 +751,9 @@ static void pci_dma_dev_setup_pSeries(struct pci_dev *dev) · 22 unmodified lines
749	tbl = PCI_DN(dn)->table_group->tables[0];	751	tbl = PCI_DN(dn)->table_group->tables[0];
750	iommu_table_setparms(phb, dn, tbl);	752	iommu_table_setparms(phb, dn, tbl);
751	tbl->it_ops = &iommu_table_pseries_ops;	753	tbl->it_ops = &iommu_table_pseries_ops;
752	iommu_init_table(tbl, phb->node, 0, 0);	754	if (!iommu_init_table(tbl, phb->node, 0, 0))
		755	panic("Failed to initialize iommu table");
		756
753	set_iommu_table_base(&dev->dev, tbl);	757	set_iommu_table_base(&dev->dev, tbl);
754	return;	758	return;
755	}	759	}
@@ -1099,6 +1103,33 @@ static void reset_dma_window(struct pci_dev dev, struct device_node par_dn) · 343 unmodified lines
1099	ret);	1103	ret);
1100	}	1104	}
1101		1105
		1106	/* Return largest page shift based on "IO Page Sizes" output of ibm,query-pe-dma-window. */
		1107	static int iommu_get_page_shift(u32 query_page_size)
		1108	{
		1109	/* Supported IO page-sizes according to LoPAR */
		1110	const int shift[] = {
		1111	__builtin_ctzll(SZ_4K), __builtin_ctzll(SZ_64K), __builtin_ctzll(SZ_16M),
		1112	__builtin_ctzll(SZ_32M), __builtin_ctzll(SZ_64M), __builtin_ctzll(SZ_128M),
		1113	__builtin_ctzll(SZ_256M), __builtin_ctzll(SZ_16G)
		1114	};
		1115
		1116	int i = ARRAY_SIZE(shift) - 1;
		1117
		1118	/*
		1119	* On LoPAR, ibm,query-pe-dma-window outputs "IO Page Sizes" using a bit field:
		1120	* - bit 31 means 4k pages are supported,
		1121	* - bit 30 means 64k pages are supported, and so on.
		1122	* Larger pagesizes map more memory with the same amount of TCEs, so start probing them.
		1123	*/
		1124	for (; i >= 0 ; i--) {
		1125	if (query_page_size & (1 << i))
		1126	return shift[i];
		1127	}
		1128
		1129	/* No valid page size found. */
		1130	return 0;
		1131	}
		1132
1102	/*	1133	/*
1103	* If the PE supports dynamic dma windows, and there is space for a table	1134	* If the PE supports dynamic dma windows, and there is space for a table
1104	* that can map all pages in a linear offset, then setup such a table,	1135	* that can map all pages in a linear offset, then setup such a table,
@@ -1206,13 +1237,9 @@ static u64 enable_ddw(struct pci_dev dev, struct device_node pdn) · 101 unmodified lines
1206	goto out_failed;	1237	goto out_failed;
1207	}	1238	}
1208	}	1239	}
1209	if (query.page_size & 4) {	1240
1210	page_shift = 24; /* 16MB */	1241	page_shift = iommu_get_page_shift(query.page_size);
1211	} else if (query.page_size & 2) {	1242	if (!page_shift) {
1212	page_shift = 16; /* 64kB */
1213	} else if (query.page_size & 1) {
1214	page_shift = 12; /* 4kB */
1215	} else {
1216	dev_dbg(&dev->dev, "no supported direct page size in mask %x",	1243	dev_dbg(&dev->dev, "no supported direct page size in mask %x",
1217	query.page_size);	1244	query.page_size);
1218	goto out_failed;	1245	goto out_failed;
@@ -1229,7 +1256,7 @@ static u64 enable_ddw(struct pci_dev dev, struct device_node pdn) · 10 unmodified lines
1229	if (pmem_present) {	1256	if (pmem_present) {
1230	if (query.largest_available_block >=	1257	if (query.largest_available_block >=
1231	(1ULL << (MAX_PHYSMEM_BITS - page_shift)))	1258	(1ULL << (MAX_PHYSMEM_BITS - page_shift)))
1232	len = MAX_PHYSMEM_BITS - page_shift;	1259	len = MAX_PHYSMEM_BITS;
1233	else	1260	else
1234	dev_info(&dev->dev, "Skipping ibm,pmemory");	1261	dev_info(&dev->dev, "Skipping ibm,pmemory");
1235	}	1262	}

arch/powerpc/platforms/pseries/lpar.c+4−2

 	slot = pSeries_lpar_hpte_find(vpn, psize, ssize);
 	BUG_ON(slot == -1);
-	flags = newpp & 7;
+	flags = newpp & (HPTE_R_PP | HPTE_R_N);
 	if (mmu_has_feature(MMU_FTR_KERNEL_RO))
 		/* Move pp0 into bit 8 (IBM 55) */
 		flags |= (newpp & HPTE_R_PP0) >> 55;
+	flags |= ((newpp & HPTE_R_KEY_HI) >> 48) | (newpp & HPTE_R_KEY_LO);
 	lpar_rc = plpar_pte_protect(flags, slot, 0);
 	BUG_ON(lpar_rc != H_SUCCESS);
 		}
 		msleep(delay);
 		rc = plpar_resize_hpt_prepare(0, shift);
-	};
+	}
 	switch (rc) {
 	case H_SUCCESS:

arch/powerpc/platforms/pseries/lparcfg.c+2−0

@@ -537,6 +537,8 @@ static int pseries_lparcfg_data(struct seq_file m, void v) · 536 unmodified lines
537	parse_em_data(m);	537	parse_em_data(m);
538	maxmem_data(m);	538	maxmem_data(m);
539		539
		540	seq_printf(m, "security_flavor=%u\n", pseries_security_flavor);
		541
540	return 0;	542	return 0;
541	}	543	}
542		544

arch/powerpc/platforms/pseries/papr_scm.c+48−0

@@ -93,6 +93,7 @@ struct papr_scm_priv { · 92 unmodified lines
93	uint64_t block_size;	93	uint64_t block_size;
94	int metadata_size;	94	int metadata_size;
95	bool is_volatile;	95	bool is_volatile;
		96	bool hcall_flush_required;
96		97
97	uint64_t bound_addr;	98	uint64_t bound_addr;
98		99
@@ -117,6 +118,38 @@ struct papr_scm_priv { · 18 unmodified lines
117	size_t stat_buffer_len;	118	size_t stat_buffer_len;
118	};	119	};
119		120
		121	static int papr_scm_pmem_flush(struct nd_region *nd_region,
		122	struct bio *bio __maybe_unused)
		123	{
		124	struct papr_scm_priv *p = nd_region_provider_data(nd_region);
		125	unsigned long ret_buf[PLPAR_HCALL_BUFSIZE], token = 0;
		126	long rc;
		127
		128	dev_dbg(&p->pdev->dev, "flush drc 0x%x", p->drc_index);
		129
		130	do {
		131	rc = plpar_hcall(H_SCM_FLUSH, ret_buf, p->drc_index, token);
		132	token = ret_buf[0];
		133
		134	/* Check if we are stalled for some time */
		135	if (H_IS_LONG_BUSY(rc)) {
		136	msleep(get_longbusy_msecs(rc));
		137	rc = H_BUSY;
		138	} else if (rc == H_BUSY) {
		139	cond_resched();
		140	}
		141	} while (rc == H_BUSY);
		142
		143	if (rc) {
		144	dev_err(&p->pdev->dev, "flush error: %ld", rc);
		145	rc = -EIO;
		146	} else {
		147	dev_dbg(&p->pdev->dev, "flush drc 0x%x complete", p->drc_index);
		148	}
		149
		150	return rc;
		151	}
		152
120	static LIST_HEAD(papr_nd_regions);	153	static LIST_HEAD(papr_nd_regions);
121	static DEFINE_MUTEX(papr_ndr_lock);	154	static DEFINE_MUTEX(papr_ndr_lock);
122		155
@@ -914,6 +947,15 @@ static int papr_scm_nvdimm_init(struct papr_scm_priv *p) · 791 unmodified lines
914	dimm_flags = 0;	947	dimm_flags = 0;
915	set_bit(NDD_LABELING, &dimm_flags);	948	set_bit(NDD_LABELING, &dimm_flags);
916		949
		950	/*
		951	* Check if the nvdimm is unarmed. No locking needed as we are still
		952	* initializing. Ignore error encountered if any.
		953	*/
		954	__drc_pmem_query_health(p);
		955
		956	if (p->health_bitmap & PAPR_PMEM_UNARMED_MASK)
		957	set_bit(NDD_UNARMED, &dimm_flags);
		958
917	p->nvdimm = nvdimm_create(p->bus, p, papr_nd_attr_groups,	959	p->nvdimm = nvdimm_create(p->bus, p, papr_nd_attr_groups,
918	dimm_flags, PAPR_SCM_DIMM_CMD_MASK, 0, NULL);	960	dimm_flags, PAPR_SCM_DIMM_CMD_MASK, 0, NULL);
919	if (!p->nvdimm) {	961	if (!p->nvdimm) {
@@ -943,6 +985,11 @@ static int papr_scm_nvdimm_init(struct papr_scm_priv *p) · 23 unmodified lines
943	ndr_desc.num_mappings = 1;	985	ndr_desc.num_mappings = 1;
944	ndr_desc.nd_set = &p->nd_set;	986	ndr_desc.nd_set = &p->nd_set;
945		987
		988	if (p->hcall_flush_required) {
		989	set_bit(ND_REGION_ASYNC, &ndr_desc.flags);
		990	ndr_desc.flush = papr_scm_pmem_flush;
		991	}
		992
946	if (p->is_volatile)	993	if (p->is_volatile)
947	p->region = nvdimm_volatile_region_create(p->bus, &ndr_desc);	994	p->region = nvdimm_volatile_region_create(p->bus, &ndr_desc);
948	else {	995	else {
@@ -1088,6 +1135,7 @@ static int papr_scm_probe(struct platform_device *pdev) · 139 unmodified lines
1088	p->block_size = block_size;	1135	p->block_size = block_size;
1089	p->blocks = blocks;	1136	p->blocks = blocks;
1090	p->is_volatile = !of_property_read_bool(dn, "ibm,cache-flush-required");	1137	p->is_volatile = !of_property_read_bool(dn, "ibm,cache-flush-required");
		1138	p->hcall_flush_required = of_property_read_bool(dn, "ibm,hcall-flush-required");
1091		1139
1092	/* We just need to ensure that set cookies are unique across */	1140	/* We just need to ensure that set cookies are unique across */
1093	uuid_parse(uuid_str, (uuid_t *) uuid);	1141	uuid_parse(uuid_str, (uuid_t *) uuid);

arch/powerpc/platforms/pseries/pci_dlpar.c+3−1

 int remove_phb_dynamic(struct pci_controller *phb)
 {
 	struct pci_bus *b = phb->bus;
+	struct pci_host_bridge *host_bridge = to_pci_host_bridge(b->bridge);
 	struct resource *res;
 	int rc, i;
 	/* Remove the PCI bus and unregister the bridge device from sysfs */
 	phb->bus = NULL;
 	pci_remove_bus(b);
-	device_unregister(b->bridge);
+	host_bridge->bus = NULL;
+	device_unregister(&host_bridge->dev);
 	/* Now release the IO resource */
 	if (res->flags & IORESOURCE_IO)

arch/powerpc/platforms/pseries/pmem.c+1−1

@@ -139,7 +139,7 @@ int dlpar_hp_pmem(struct pseries_hp_errorlog *hp_elog) · 138 unmodified lines
139	return rc;	139	return rc;
140	}	140	}
141		141
142	const struct of_device_id drc_pmem_match[] = {	142	static const struct of_device_id drc_pmem_match[] = {
143	{ .type = "ibm,persistent-memory", },	143	{ .type = "ibm,persistent-memory", },
144	{}	144	{}
145	};	145	};

arch/powerpc/platforms/pseries/pseries.h+2−3

 /* Poweron flag used for enabling auto ups restart */
 extern unsigned long rtas_poweron_auto;
-/* Provided by HVC VIO */
-extern void hvc_vio_init_early(void);
 /* Dynamic logical Partitioning/Mobility */
 extern void dlpar_free_cc_nodes(struct device_node *);
 extern void dlpar_free_cc_property(struct property *);
 extern int dlpar_detach_node(struct device_node *);
 extern int dlpar_acquire_drc(u32 drc_index);
 extern int dlpar_release_drc(u32 drc_index);
+extern int dlpar_unisolate_drc(u32 drc_index);
 void queue_hotplug_event(struct pseries_hp_errorlog *hp_errlog);
 int handle_dlpar_errorlog(struct pseries_hp_errorlog *hp_errlog);
 int dlpar_workqueue_init(void);
+extern u32 pseries_security_flavor;
 void pseries_setup_security_mitigations(void);
 void pseries_lpar_read_hblkrm_characteristics(void);

arch/powerpc/platforms/pseries/ras.c+1−1

@@ -699,7 +699,7 @@ static int mce_handle_err_virtmode(struct pt_regs *regs, · 698 unmodified lines
699	mce_err.error_type = MCE_ERROR_TYPE_DCACHE;	699	mce_err.error_type = MCE_ERROR_TYPE_DCACHE;
700	break;	700	break;
701	case MC_ERROR_TYPE_I_CACHE:	701	case MC_ERROR_TYPE_I_CACHE:
702	mce_err.error_type = MCE_ERROR_TYPE_DCACHE;	702	mce_err.error_type = MCE_ERROR_TYPE_ICACHE;
703	break;	703	break;
704	case MC_ERROR_TYPE_UNKNOWN:	704	case MC_ERROR_TYPE_UNKNOWN:
705	default:	705	default:

arch/powerpc/platforms/pseries/rtas-fadump.c+1−1

@@ -247,7 +247,7 @@ static inline int rtas_fadump_gpr_index(u64 id) · 246 unmodified lines
247	return i;	247	return i;
248	}	248	}
249		249
250	void rtas_fadump_set_regval(struct pt_regs *regs, u64 reg_id, u64 reg_val)	250	static void rtas_fadump_set_regval(struct pt_regs *regs, u64 reg_id, u64 reg_val)
251	{	251	{
252	int i;	252	int i;
253		253

arch/powerpc/platforms/pseries/setup.c+8−0

 #include <asm/swiotlb.h>
 #include <asm/svm.h>
 #include <asm/dtl.h>
+#include <asm/hvconsole.h>
 #include "pseries.h"
 #include "../../../../drivers/pci/pci.h"
 int fwnmi_active;  /* TRUE if an FWNMI handler is present */
 int ibm_nmi_interlock_token;
+u32 pseries_security_flavor;
 static void pSeries_show_cpuinfo(struct seq_file *m)
 {
 	/*
 	 * The features below are enabled by default, so we instead look to see
 	 * if firmware has *disabled* them, and clear them if so.
+	 * H_CPU_BEHAV_FAVOUR_SECURITY_H could be set only if
+	 * H_CPU_BEHAV_FAVOUR_SECURITY is.
 	 */
 	if (!(result->behaviour & H_CPU_BEHAV_FAVOUR_SECURITY))
 		security_ftr_clear(SEC_FTR_FAVOUR_SECURITY);
+	else if (result->behaviour & H_CPU_BEHAV_FAVOUR_SECURITY_H)
+		pseries_security_flavor = 1;
+	else
+		pseries_security_flavor = 2;
 	if (!(result->behaviour & H_CPU_BEHAV_L1D_FLUSH_PR))
 		security_ftr_clear(SEC_FTR_L1D_FLUSH_PR);

arch/powerpc/platforms/pseries/vio.c+20−0

 #include <linux/mm.h>
 #include <linux/dma-map-ops.h>
 #include <linux/kobject.h>
+#include <linux/kexec.h>
 #include <asm/iommu.h>
 #include <asm/dma.h>
 	return 0;
 }
+static void vio_bus_shutdown(struct device *dev)
+{
+	struct vio_dev *viodev = to_vio_dev(dev);
+	struct vio_driver *viodrv;
+	if (dev->driver) {
+		viodrv = to_vio_driver(dev->driver);
+		if (viodrv->shutdown)
+			viodrv->shutdown(viodev);
+		else if (kexec_in_progress)
+			vio_bus_remove(dev);
+	}
+}
 /**
  * vio_register_driver: - Register a new vio driver
  * @viodrv:	The vio_driver structure to be registered.
 int __vio_register_driver(struct vio_driver *viodrv, struct module *owner,
 			  const char *mod_name)
 {
+	// vio_bus_type is only initialised for pseries
+	if (!machine_is(pseries))
+		return -ENODEV;
 	pr_debug("%s: driver %s registering\n", __func__, viodrv->name);
 	/* fill in 'struct driver' fields */
 	.match = vio_bus_match,
 	.probe = vio_bus_probe,
 	.remove = vio_bus_remove,
+	.shutdown = vio_bus_shutdown,
 };
 /**

arch/powerpc/purgatory/trampoline_64.S+0−1

@@ -12,7 +12,6 @@ · 11 unmodified lines
12	#include <asm/asm-compat.h>	12	#include <asm/asm-compat.h>
13	#include <asm/crashdump-ppc64.h>	13	#include <asm/crashdump-ppc64.h>
14		14
15	.machine ppc64
16	.balign 256	15	.balign 256
17	.globl purgatory_start	16	.globl purgatory_start
18	purgatory_start:	17	purgatory_start:

arch/powerpc/sysdev/dart_iommu.c+2−1

@@ -344,7 +344,8 @@ static void iommu_table_dart_setup(void) · 343 unmodified lines
344	iommu_table_dart.it_index = 0;	344	iommu_table_dart.it_index = 0;
345	iommu_table_dart.it_blocksize = 1;	345	iommu_table_dart.it_blocksize = 1;
346	iommu_table_dart.it_ops = &iommu_dart_ops;	346	iommu_table_dart.it_ops = &iommu_dart_ops;
347	iommu_init_table(&iommu_table_dart, -1, 0, 0);	347	if (!iommu_init_table(&iommu_table_dart, -1, 0, 0))
		348	panic("Failed to initialize iommu table");
348		349
349	/* Reserve the last page of the DART to avoid possible prefetch	350	/* Reserve the last page of the DART to avoid possible prefetch
350	* past the DART mapped area	351	* past the DART mapped area

arch/powerpc/sysdev/fsl_pci.c+1−1

@@ -455,7 +455,7 @@ static void setup_pci_atmu(struct pci_controller *hose) · 454 unmodified lines
455	}	455	}
456	}	456	}
457		457
458	static void __init setup_pci_cmd(struct pci_controller *hose)	458	static void setup_pci_cmd(struct pci_controller *hose)
459	{	459	{
460	u16 cmd;	460	u16 cmd;
461	int cap_x;	461	int cap_x;

arch/powerpc/sysdev/xive/common.c+138−75

@@ -63,8 +63,19 @@ static const struct xive_ops *xive_ops; · 62 unmodified lines
63	static struct irq_domain *xive_irq_domain;	63	static struct irq_domain *xive_irq_domain;
64		64
65	#ifdef CONFIG_SMP	65	#ifdef CONFIG_SMP
66	/* The IPIs all use the same logical irq number */	66	/* The IPIs use the same logical irq number when on the same chip */
67	static u32 xive_ipi_irq;	67	static struct xive_ipi_desc {
		68	unsigned int irq;
		69	char name[16];
		70	} *xive_ipis;
		71
		72	/*
		73	* Use early_cpu_to_node() for hot-plugged CPUs
		74	*/
		75	static unsigned int xive_ipi_cpu_to_irq(unsigned int cpu)
		76	{
		77	return xive_ipis[early_cpu_to_node(cpu)].irq;
		78	}
68	#endif	79	#endif
69		80
70	/* Xive state for each CPU */	81	/* Xive state for each CPU */
@@ -253,17 +264,20 @@ notrace void xmon_xive_do_dump(int cpu) · 182 unmodified lines
253	xmon_printf("\n");	264	xmon_printf("\n");
254	}	265	}
255		266
		267	static struct irq_data *xive_get_irq_data(u32 hw_irq)
		268	{
		269	unsigned int irq = irq_find_mapping(xive_irq_domain, hw_irq);
		270
		271	return irq ? irq_get_irq_data(irq) : NULL;
		272	}
		273
256	int xmon_xive_get_irq_config(u32 hw_irq, struct irq_data *d)	274	int xmon_xive_get_irq_config(u32 hw_irq, struct irq_data *d)
257	{	275	{
258	struct irq_chip *chip = irq_data_get_irq_chip(d);
259	int rc;	276	int rc;
260	u32 target;	277	u32 target;
261	u8 prio;	278	u8 prio;
262	u32 lirq;	279	u32 lirq;
263		280
264	if (!is_xive_irq(chip))
265	return -EINVAL;
266
267	rc = xive_ops->get_irq_config(hw_irq, &target, &prio, &lirq);	281	rc = xive_ops->get_irq_config(hw_irq, &target, &prio, &lirq);
268	if (rc) {	282	if (rc) {
269	xmon_printf("IRQ 0x%08x : no config rc=%d\n", hw_irq, rc);	283	xmon_printf("IRQ 0x%08x : no config rc=%d\n", hw_irq, rc);
@@ -273,6 +287,9 @@ int xmon_xive_get_irq_config(u32 hw_irq, struct irq_data *d) · 3 unmodified lines
273	xmon_printf("IRQ 0x%08x : target=0x%x prio=%02x lirq=0x%x ",	287	xmon_printf("IRQ 0x%08x : target=0x%x prio=%02x lirq=0x%x ",
274	hw_irq, target, prio, lirq);	288	hw_irq, target, prio, lirq);
275		289
		290	if (!d)
		291	d = xive_get_irq_data(hw_irq);
		292
276	if (d) {	293	if (d) {
277	struct xive_irq_data *xd = irq_data_get_irq_handler_data(d);	294	struct xive_irq_data *xd = irq_data_get_irq_handler_data(d);
278	u64 val = xive_esb_read(xd, XIVE_ESB_GET);	295	u64 val = xive_esb_read(xd, XIVE_ESB_GET);
@@ -289,6 +306,20 @@ int xmon_xive_get_irq_config(u32 hw_irq, struct irq_data *d) · 10 unmodified lines
289	return 0;	306	return 0;
290	}	307	}
291		308
		309	void xmon_xive_get_irq_all(void)
		310	{
		311	unsigned int i;
		312	struct irq_desc *desc;
		313
		314	for_each_irq_desc(i, desc) {
		315	struct irq_data *d = irq_desc_get_irq_data(desc);
		316	unsigned int hwirq = (unsigned int)irqd_to_hwirq(d);
		317
		318	if (d->domain == xive_irq_domain)
		319	xmon_xive_get_irq_config(hwirq, d);
		320	}
		321	}
		322
292	#endif /* CONFIG_XMON */	323	#endif /* CONFIG_XMON */
293		324
294	static unsigned int xive_get_irq(void)	325	static unsigned int xive_get_irq(void)
@@ -1067,28 +1098,94 @@ static struct irq_chip xive_ipi_chip = { · 772 unmodified lines
1067	.irq_unmask = xive_ipi_do_nothing,	1098	.irq_unmask = xive_ipi_do_nothing,
1068	};	1099	};
1069		1100
1070	static void __init xive_request_ipi(void)	1101	/*
		1102	* IPIs are marked per-cpu. We use separate HW interrupts under the
		1103	* hood but associated with the same "linux" interrupt
		1104	*/
		1105	struct xive_ipi_alloc_info {
		1106	irq_hw_number_t hwirq;
		1107	};
		1108
		1109	static int xive_ipi_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
		1110	unsigned int nr_irqs, void *arg)
1071	{	1111	{
1072	unsigned int virq;	1112	struct xive_ipi_alloc_info *info = arg;
		1113	int i;
1073		1114
1074	/*	1115	for (i = 0; i < nr_irqs; i++) {
1075	* Initialization failed, move on, we might manage to	1116	irq_domain_set_info(domain, virq + i, info->hwirq + i, &xive_ipi_chip,
1076	* reach the point where we display our errors before	1117	domain->host_data, handle_percpu_irq,
1077	* the system falls appart	1118	NULL, NULL);
1078	*/	1119	}
1079	if (!xive_irq_domain)	1120	return 0;
1080	return;	1121	}
1081		1122
1082	/* Initialize it */	1123	static const struct irq_domain_ops xive_ipi_irq_domain_ops = {
1083	virq = irq_create_mapping(xive_irq_domain, XIVE_IPI_HW_IRQ);	1124	.alloc = xive_ipi_irq_domain_alloc,
1084	xive_ipi_irq = virq;	1125	};
1085		1126
1086	WARN_ON(request_irq(virq, xive_muxed_ipi_action,	1127	static int __init xive_request_ipi(void)
1087	IRQF_PERCPU \| IRQF_NO_THREAD, "IPI", NULL));	1128	{
		1129	struct fwnode_handle *fwnode;
		1130	struct irq_domain *ipi_domain;
		1131	unsigned int node;
		1132	int ret = -ENOMEM;
		1133
		1134	fwnode = irq_domain_alloc_named_fwnode("XIVE-IPI");
		1135	if (!fwnode)
		1136	goto out;
		1137
		1138	ipi_domain = irq_domain_create_linear(fwnode, nr_node_ids,
		1139	&xive_ipi_irq_domain_ops, NULL);
		1140	if (!ipi_domain)
		1141	goto out_free_fwnode;
		1142
		1143	xive_ipis = kcalloc(nr_node_ids, sizeof(*xive_ipis), GFP_KERNEL \| __GFP_NOFAIL);
		1144	if (!xive_ipis)
		1145	goto out_free_domain;
		1146
		1147	for_each_node(node) {
		1148	struct xive_ipi_desc *xid = &xive_ipis[node];
		1149	struct xive_ipi_alloc_info info = { node };
		1150
		1151	/* Skip nodes without CPUs */
		1152	if (cpumask_empty(cpumask_of_node(node)))
		1153	continue;
		1154
		1155	/*
		1156	* Map one IPI interrupt per node for all cpus of that node.
		1157	* Since the HW interrupt number doesn't have any meaning,
		1158	* simply use the node number.
		1159	*/
		1160	xid->irq = irq_domain_alloc_irqs(ipi_domain, 1, node, &info);
		1161	if (xid->irq < 0) {
		1162	ret = xid->irq;
		1163	goto out_free_xive_ipis;
		1164	}
		1165
		1166	snprintf(xid->name, sizeof(xid->name), "IPI-%d", node);
		1167
		1168	ret = request_irq(xid->irq, xive_muxed_ipi_action,
		1169	IRQF_PERCPU \| IRQF_NO_THREAD, xid->name, NULL);
		1170
		1171	WARN(ret < 0, "Failed to request IPI %d: %d\n", xid->irq, ret);
		1172	}
		1173
		1174	return ret;
		1175
		1176	out_free_xive_ipis:
		1177	kfree(xive_ipis);
		1178	out_free_domain:
		1179	irq_domain_remove(ipi_domain);
		1180	out_free_fwnode:
		1181	irq_domain_free_fwnode(fwnode);
		1182	out:
		1183	return ret;
1088	}	1184	}
1089		1185
1090	static int xive_setup_cpu_ipi(unsigned int cpu)	1186	static int xive_setup_cpu_ipi(unsigned int cpu)
1091	{	1187	{
		1188	unsigned int xive_ipi_irq = xive_ipi_cpu_to_irq(cpu);
1092	struct xive_cpu *xc;	1189	struct xive_cpu *xc;
1093	int rc;	1190	int rc;
1094		1191
@@ -1131,6 +1228,8 @@ static int xive_setup_cpu_ipi(unsigned int cpu) · 36 unmodified lines
1131		1228
1132	static void xive_cleanup_cpu_ipi(unsigned int cpu, struct xive_cpu *xc)	1229	static void xive_cleanup_cpu_ipi(unsigned int cpu, struct xive_cpu *xc)
1133	{	1230	{
		1231	unsigned int xive_ipi_irq = xive_ipi_cpu_to_irq(cpu);
		1232
1134	/* Disable the IPI and free the IRQ data */	1233	/* Disable the IPI and free the IRQ data */
1135		1234
1136	/* Already cleaned up ? */	1235	/* Already cleaned up ? */
@@ -1178,19 +1277,6 @@ static int xive_irq_domain_map(struct irq_domain *h, unsigned int virq, · 41 unmodified lines
1178	*/	1277	*/
1179	irq_clear_status_flags(virq, IRQ_LEVEL);	1278	irq_clear_status_flags(virq, IRQ_LEVEL);
1180		1279
1181	#ifdef CONFIG_SMP
1182	/* IPIs are special and come up with HW number 0 */
1183	if (hw == XIVE_IPI_HW_IRQ) {
1184	/*
1185	* IPIs are marked per-cpu. We use separate HW interrupts under
1186	* the hood but associated with the same "linux" interrupt
1187	*/
1188	irq_set_chip_and_handler(virq, &xive_ipi_chip,
1189	handle_percpu_irq);
1190	return 0;
1191	}
1192	#endif
1193
1194	rc = xive_irq_alloc_data(virq, hw);	1280	rc = xive_irq_alloc_data(virq, hw);
1195	if (rc)	1281	if (rc)
1196	return rc;	1282	return rc;
@@ -1202,15 +1288,7 @@ static int xive_irq_domain_map(struct irq_domain *h, unsigned int virq, · 5 unmodified lines
1202		1288
1203	static void xive_irq_domain_unmap(struct irq_domain *d, unsigned int virq)	1289	static void xive_irq_domain_unmap(struct irq_domain *d, unsigned int virq)
1204	{	1290	{
1205	struct irq_data *data = irq_get_irq_data(virq);	1291	xive_irq_free_data(virq);
1206	unsigned int hw_irq;
1207
1208	/* XXX Assign BAD number */
1209	if (!data)
1210	return;
1211	hw_irq = (unsigned int)irqd_to_hwirq(data);
1212	if (hw_irq != XIVE_IPI_HW_IRQ)
1213	xive_irq_free_data(virq);
1214	}	1292	}
1215		1293
1216	static int xive_irq_domain_xlate(struct irq_domain h, struct device_node ct,	1294	static int xive_irq_domain_xlate(struct irq_domain h, struct device_node ct,
@@ -1335,17 +1413,14 @@ static int xive_prepare_cpu(unsigned int cpu) · 118 unmodified lines
1335		1413
1336	xc = per_cpu(xive_cpu, cpu);	1414	xc = per_cpu(xive_cpu, cpu);
1337	if (!xc) {	1415	if (!xc) {
1338	struct device_node *np;
1339
1340	xc = kzalloc_node(sizeof(struct xive_cpu),	1416	xc = kzalloc_node(sizeof(struct xive_cpu),
1341	GFP_KERNEL, cpu_to_node(cpu));	1417	GFP_KERNEL, cpu_to_node(cpu));
1342	if (!xc)	1418	if (!xc)
1343	return -ENOMEM;	1419	return -ENOMEM;
1344	np = of_get_cpu_node(cpu, NULL);
1345	if (np)
1346	xc->chip_id = of_get_ibm_chip_id(np);
1347	of_node_put(np);
1348	xc->hw_ipi = XIVE_BAD_IRQ;	1420	xc->hw_ipi = XIVE_BAD_IRQ;
		1421	xc->chip_id = XIVE_INVALID_CHIP_ID;
		1422	if (xive_ops->prepare_cpu)
		1423	xive_ops->prepare_cpu(cpu, xc);
1349		1424
1350	per_cpu(xive_cpu, cpu) = xc;	1425	per_cpu(xive_cpu, cpu) = xc;
1351	}	1426	}
@@ -1408,13 +1483,12 @@ static void xive_flush_cpu_queue(unsigned int cpu, struct xive_cpu *xc) · 56 unmodified lines
1408	struct irq_desc *desc = irq_to_desc(irq);	1483	struct irq_desc *desc = irq_to_desc(irq);
1409	struct irq_data *d = irq_desc_get_irq_data(desc);	1484	struct irq_data *d = irq_desc_get_irq_data(desc);
1410	struct xive_irq_data *xd;	1485	struct xive_irq_data *xd;
1411	unsigned int hw_irq = (unsigned int)irqd_to_hwirq(d);
1412		1486
1413	/*	1487	/*
1414	* Ignore anything that isn't a XIVE irq and ignore	1488	* Ignore anything that isn't a XIVE irq and ignore
1415	* IPIs, so can just be dropped.	1489	* IPIs, so can just be dropped.
1416	*/	1490	*/
1417	if (d->domain != xive_irq_domain \|\| hw_irq == XIVE_IPI_HW_IRQ)	1491	if (d->domain != xive_irq_domain)
1418	continue;	1492	continue;
1419		1493
1420	/*	1494	/*
@@ -1592,16 +1666,15 @@ static void xive_debug_show_cpu(struct seq_file *m, int cpu) · 171 unmodified lines
1592	seq_puts(m, "\n");	1666	seq_puts(m, "\n");
1593	}	1667	}
1594		1668
1595	static void xive_debug_show_irq(struct seq_file m, u32 hw_irq, struct irq_data d)	1669	static void xive_debug_show_irq(struct seq_file m, struct irq_data d)
1596	{	1670	{
1597	struct irq_chip *chip = irq_data_get_irq_chip(d);	1671	unsigned int hw_irq = (unsigned int)irqd_to_hwirq(d);
1598	int rc;	1672	int rc;
1599	u32 target;	1673	u32 target;
1600	u8 prio;	1674	u8 prio;
1601	u32 lirq;	1675	u32 lirq;
1602		1676	struct xive_irq_data *xd;
1603	if (!is_xive_irq(chip))	1677	u64 val;
1604	return;
1605		1678
1606	rc = xive_ops->get_irq_config(hw_irq, &target, &prio, &lirq);	1679	rc = xive_ops->get_irq_config(hw_irq, &target, &prio, &lirq);
1607	if (rc) {	1680	if (rc) {
@@ -1612,17 +1685,14 @@ static void xive_debug_show_irq(struct seq_file *m, u32 hw_irq, struct irq_data · 4 unmodified lines
1612	seq_printf(m, "IRQ 0x%08x : target=0x%x prio=%02x lirq=0x%x ",	1685	seq_printf(m, "IRQ 0x%08x : target=0x%x prio=%02x lirq=0x%x ",
1613	hw_irq, target, prio, lirq);	1686	hw_irq, target, prio, lirq);
1614		1687
1615	if (d) {	1688	xd = irq_data_get_irq_handler_data(d);
1616	struct xive_irq_data *xd = irq_data_get_irq_handler_data(d);	1689	val = xive_esb_read(xd, XIVE_ESB_GET);
1617	u64 val = xive_esb_read(xd, XIVE_ESB_GET);	1690	seq_printf(m, "flags=%c%c%c PQ=%c%c",
1618		1691	xd->flags & XIVE_IRQ_FLAG_STORE_EOI ? 'S' : ' ',
1619	seq_printf(m, "flags=%c%c%c PQ=%c%c",	1692	xd->flags & XIVE_IRQ_FLAG_LSI ? 'L' : ' ',
1620	xd->flags & XIVE_IRQ_FLAG_STORE_EOI ? 'S' : ' ',	1693	xd->flags & XIVE_IRQ_FLAG_H_INT_ESB ? 'H' : ' ',
1621	xd->flags & XIVE_IRQ_FLAG_LSI ? 'L' : ' ',	1694	val & XIVE_ESB_VAL_P ? 'P' : '-',
1622	xd->flags & XIVE_IRQ_FLAG_H_INT_ESB ? 'H' : ' ',	1695	val & XIVE_ESB_VAL_Q ? 'Q' : '-');
1623	val & XIVE_ESB_VAL_P ? 'P' : '-',
1624	val & XIVE_ESB_VAL_Q ? 'Q' : '-');
1625	}
1626	seq_puts(m, "\n");	1696	seq_puts(m, "\n");
1627	}	1697	}
1628		1698
@@ -1640,16 +1710,9 @@ static int xive_core_debug_show(struct seq_file m, void private) · 11 unmodified lines
1640		1710
1641	for_each_irq_desc(i, desc) {	1711	for_each_irq_desc(i, desc) {
1642	struct irq_data *d = irq_desc_get_irq_data(desc);	1712	struct irq_data *d = irq_desc_get_irq_data(desc);
1643	unsigned int hw_irq;
1644
1645	if (!d)
1646	continue;
1647
1648	hw_irq = (unsigned int)irqd_to_hwirq(d);
1649		1713
1650	/* IPIs are special (HW number 0) */	1714	if (d->domain == xive_irq_domain)
1651	if (hw_irq != XIVE_IPI_HW_IRQ)	1715	xive_debug_show_irq(m, d);
1652	xive_debug_show_irq(m, hw_irq, d);
1653	}	1716	}
1654	return 0;	1717	return 0;
1655	}	1718	}

arch/powerpc/sysdev/xive/native.c+6−0

 	}
 }
+static void xive_native_prepare_cpu(unsigned int cpu, struct xive_cpu *xc)
+{
+	xc->chip_id = cpu_to_chip_id(cpu);
+}
 static void xive_native_setup_cpu(unsigned int cpu, struct xive_cpu *xc)
 {
 	s64 rc;
 	.match			= xive_native_match,
 	.shutdown		= xive_native_shutdown,
 	.update_pending		= xive_native_update_pending,
+	.prepare_cpu		= xive_native_prepare_cpu,
 	.setup_cpu		= xive_native_setup_cpu,
 	.teardown_cpu		= xive_native_teardown_cpu,
 	.sync_source		= xive_native_sync_source,

arch/powerpc/sysdev/xive/spapr.c+1−1

@@ -549,7 +549,7 @@ static void xive_spapr_cleanup_queue(unsigned int cpu, struct xive_cpu *xc, · 548 unmodified lines
549	static bool xive_spapr_match(struct device_node *node)	549	static bool xive_spapr_match(struct device_node *node)
550	{	550	{
551	/* Ignore cascaded controllers for the moment */	551	/* Ignore cascaded controllers for the moment */
552	return 1;	552	return true;
553	}	553	}
554		554
555	#ifdef CONFIG_SMP	555	#ifdef CONFIG_SMP

arch/powerpc/sysdev/xive/xive-internal.h+1−2

 #ifndef __XIVE_INTERNAL_H
 #define __XIVE_INTERNAL_H
-#define XIVE_IPI_HW_IRQ		0 /* interrupt source # for IPIs */
 /*
  * A "disabled" interrupt should never fire, to catch problems
  * we set its logical number to this
 				  u32 *sw_irq);
 	int	(*setup_queue)(unsigned int cpu, struct xive_cpu *xc, u8 prio);
 	void	(*cleanup_queue)(unsigned int cpu, struct xive_cpu *xc, u8 prio);
+	void	(*prepare_cpu)(unsigned int cpu, struct xive_cpu *xc);
 	void	(*setup_cpu)(unsigned int cpu, struct xive_cpu *xc);
 	void	(*teardown_cpu)(unsigned int cpu, struct xive_cpu *xc);
 	bool	(*match)(struct device_node *np);

arch/powerpc/xmon/xmon.c+24−52

@@ -54,6 +54,7 @@ · 53 unmodified lines
54	#include <asm/code-patching.h>	54	#include <asm/code-patching.h>
55	#include <asm/sections.h>	55	#include <asm/sections.h>
56	#include <asm/inst.h>	56	#include <asm/inst.h>
		57	#include <asm/interrupt.h>
57		58
58	#ifdef CONFIG_PPC64	59	#ifdef CONFIG_PPC64
59	#include <asm/hvcall.h>	60	#include <asm/hvcall.h>
@@ -605,7 +606,7 @@ static int xmon_core(struct pt_regs *regs, int fromipi) · 545 unmodified lines
605	* debugger break (IPI). This is similar to	606	* debugger break (IPI). This is similar to
606	* crash_kexec_secondary().	607	* crash_kexec_secondary().
607	*/	608	*/
608	if (TRAP(regs) != 0x100 \|\| !wait_for_other_cpus(ncpus))	609	if (TRAP(regs) != INTERRUPT_SYSTEM_RESET \|\| !wait_for_other_cpus(ncpus))
609	smp_send_debugger_break();	610	smp_send_debugger_break();
610		611
611	wait_for_other_cpus(ncpus);	612	wait_for_other_cpus(ncpus);
@@ -615,7 +616,7 @@ static int xmon_core(struct pt_regs *regs, int fromipi) · 3 unmodified lines
615		616
616	if (!locked_down) {	617	if (!locked_down) {
617	/* for breakpoint or single step, print curr insn */	618	/* for breakpoint or single step, print curr insn */
618	if (bp \|\| TRAP(regs) == 0xd00)	619	if (bp \|\| TRAP(regs) == INTERRUPT_TRACE)
619	ppc_inst_dump(regs->nip, 1, 0);	620	ppc_inst_dump(regs->nip, 1, 0);
620	printf("enter ? for help\n");	621	printf("enter ? for help\n");
621	}	622	}
@@ -684,7 +685,7 @@ static int xmon_core(struct pt_regs *regs, int fromipi) · 62 unmodified lines
684	disable_surveillance();	685	disable_surveillance();
685	if (!locked_down) {	686	if (!locked_down) {
686	/* for breakpoint or single step, print current insn */	687	/* for breakpoint or single step, print current insn */
687	if (bp \|\| TRAP(regs) == 0xd00)	688	if (bp \|\| TRAP(regs) == INTERRUPT_TRACE)
688	ppc_inst_dump(regs->nip, 1, 0);	689	ppc_inst_dump(regs->nip, 1, 0);
689	printf("enter ? for help\n");	690	printf("enter ? for help\n");
690	}	691	}
@@ -1769,9 +1770,12 @@ static void excprint(struct pt_regs *fp) · 1078 unmodified lines
1769	printf(" sp: %lx\n", fp->gpr[1]);	1770	printf(" sp: %lx\n", fp->gpr[1]);
1770	printf(" msr: %lx\n", fp->msr);	1771	printf(" msr: %lx\n", fp->msr);
1771		1772
1772	if (trap == 0x300 \|\| trap == 0x380 \|\| trap == 0x600 \|\| trap == 0x200) {	1773	if (trap == INTERRUPT_DATA_STORAGE \|\|
		1774	trap == INTERRUPT_DATA_SEGMENT \|\|
		1775	trap == INTERRUPT_ALIGNMENT \|\|
		1776	trap == INTERRUPT_MACHINE_CHECK) {
1773	printf(" dar: %lx\n", fp->dar);	1777	printf(" dar: %lx\n", fp->dar);
1774	if (trap != 0x380)	1778	if (trap != INTERRUPT_DATA_SEGMENT)
1775	printf(" dsisr: %lx\n", fp->dsisr);	1779	printf(" dsisr: %lx\n", fp->dsisr);
1776	}	1780	}
1777		1781
@@ -1785,7 +1789,7 @@ static void excprint(struct pt_regs *fp) · 7 unmodified lines
1785	current->pid, current->comm);	1789	current->pid, current->comm);
1786	}	1790	}
1787		1791
1788	if (trap == 0x700)	1792	if (trap == INTERRUPT_PROGRAM)
1789	print_bug_trap(fp);	1793	print_bug_trap(fp);
1790		1794
1791	printf(linux_banner);	1795	printf(linux_banner);
@@ -1815,25 +1819,16 @@ static void prregs(struct pt_regs *fp) · 23 unmodified lines
1815	}	1819	}
1816		1820
1817	#ifdef CONFIG_PPC64	1821	#ifdef CONFIG_PPC64
1818	if (FULL_REGS(fp)) {	1822	#define R_PER_LINE 2
1819	for (n = 0; n < 16; ++n)
1820	printf("R%.2d = "REG" R%.2d = "REG"\n",
1821	n, fp->gpr[n], n+16, fp->gpr[n+16]);
1822	} else {
1823	for (n = 0; n < 7; ++n)
1824	printf("R%.2d = "REG" R%.2d = "REG"\n",
1825	n, fp->gpr[n], n+7, fp->gpr[n+7]);
1826	}
1827	#else	1823	#else
		1824	#define R_PER_LINE 4
		1825	#endif
		1826
1828	for (n = 0; n < 32; ++n) {	1827	for (n = 0; n < 32; ++n) {
1829	printf("R%.2d = %.8lx%s", n, fp->gpr[n],	1828	printf("R%.2d = "REG"%s", n, fp->gpr[n],
1830	(n & 3) == 3? "\n": " ");	1829	(n % R_PER_LINE) == R_PER_LINE - 1 ? "\n" : " ");
1831	if (n == 12 && !FULL_REGS(fp)) {
1832	printf("\n");
1833	break;
1834	}
1835	}	1830	}
1836	#endif	1831
1837	printf("pc = ");	1832	printf("pc = ");
1838	xmon_print_symbol(fp->nip, " ", "\n");	1833	xmon_print_symbol(fp->nip, " ", "\n");
1839	if (!trap_is_syscall(fp) && cpu_has_feature(CPU_FTR_CFAR)) {	1834	if (!trap_is_syscall(fp) && cpu_has_feature(CPU_FTR_CFAR)) {
@@ -1846,7 +1841,9 @@ static void prregs(struct pt_regs *fp) · 6 unmodified lines
1846	printf("ctr = "REG" xer = "REG" trap = %4lx\n",	1841	printf("ctr = "REG" xer = "REG" trap = %4lx\n",
1847	fp->ctr, fp->xer, fp->trap);	1842	fp->ctr, fp->xer, fp->trap);
1848	trap = TRAP(fp);	1843	trap = TRAP(fp);
1849	if (trap == 0x300 \|\| trap == 0x380 \|\| trap == 0x600)	1844	if (trap == INTERRUPT_DATA_STORAGE \|\|
		1845	trap == INTERRUPT_DATA_SEGMENT \|\|
		1846	trap == INTERRUPT_ALIGNMENT)
1850	printf("dar = "REG" dsisr = %.8lx\n", fp->dar, fp->dsisr);	1847	printf("dar = "REG" dsisr = %.8lx\n", fp->dar, fp->dsisr);
1851	}	1848	}
1852		1849
@@ -2727,30 +2724,6 @@ static void dump_all_xives(void) · 874 unmodified lines
2727	dump_one_xive(cpu);	2724	dump_one_xive(cpu);
2728	}	2725	}
2729		2726
2730	static void dump_one_xive_irq(u32 num, struct irq_data *d)
2731	{
2732	xmon_xive_get_irq_config(num, d);
2733	}
2734
2735	static void dump_all_xive_irq(void)
2736	{
2737	unsigned int i;
2738	struct irq_desc *desc;
2739
2740	for_each_irq_desc(i, desc) {
2741	struct irq_data *d = irq_desc_get_irq_data(desc);
2742	unsigned int hwirq;
2743
2744	if (!d)
2745	continue;
2746
2747	hwirq = (unsigned int)irqd_to_hwirq(d);
2748	/* IPIs are special (HW number 0) */
2749	if (hwirq)
2750	dump_one_xive_irq(hwirq, d);
2751	}
2752	}
2753
2754	static void dump_xives(void)	2727	static void dump_xives(void)
2755	{	2728	{
2756	unsigned long num;	2729	unsigned long num;
@@ -2767,9 +2740,9 @@ static void dump_xives(void) · 10 unmodified lines
2767	return;	2740	return;
2768	} else if (c == 'i') {	2741	} else if (c == 'i') {
2769	if (scanhex(&num))	2742	if (scanhex(&num))
2770	dump_one_xive_irq(num, NULL);	2743	xmon_xive_get_irq_config(num, NULL);
2771	else	2744	else
2772	dump_all_xive_irq();	2745	xmon_xive_get_irq_all();
2773	return;	2746	return;
2774	}	2747	}
2775		2748
@@ -2980,7 +2953,7 @@ generic_inst_dump(unsigned long adr, long count, int praddr, · 204 unmodified lines
2980	if (!ppc_inst_prefixed(inst))	2953	if (!ppc_inst_prefixed(inst))
2981	dump_func(ppc_inst_val(inst), adr);	2954	dump_func(ppc_inst_val(inst), adr);
2982	else	2955	else
2983	dump_func(ppc_inst_as_u64(inst), adr);	2956	dump_func(ppc_inst_as_ulong(inst), adr);
2984	printf("\n");	2957	printf("\n");
2985	}	2958	}
2986	return adr - first_adr;	2959	return adr - first_adr;
@@ -4212,8 +4185,7 @@ static void dump_spu_fields(struct spu *spu) · 1225 unmodified lines
4212	DUMP_FIELD(spu, "0x%p", pdata);	4185	DUMP_FIELD(spu, "0x%p", pdata);
4213	}	4186	}
4214		4187
4215	int	4188	static int spu_inst_dump(unsigned long adr, long count, int praddr)
4216	spu_inst_dump(unsigned long adr, long count, int praddr)
4217	{	4189	{
4218	return generic_inst_dump(adr, count, praddr, print_insn_spu);	4190	return generic_inst_dump(adr, count, praddr, print_insn_spu);
4219	}	4191	}

arch/s390/include/asm/vdso/gettimeofday.h+2−1

@@ -68,7 +68,8 @@ long clock_getres_fallback(clockid_t clkid, struct __kernel_timespec *ts) · 67 unmodified lines
68	}	68	}
69		69
70	#ifdef CONFIG_TIME_NS	70	#ifdef CONFIG_TIME_NS
71	static __always_inline const struct vdso_data *__arch_get_timens_vdso_data(void)	71	static __always_inline
		72	const struct vdso_data __arch_get_timens_vdso_data(const struct vdso_data vd)
72	{	73	{
73	return _timens_data;	74	return _timens_data;
74	}	75	}

arch/x86/include/asm/vdso/gettimeofday.h+2−1

@@ -58,7 +58,8 @@ extern struct ms_hyperv_tsc_page hvclock_page · 57 unmodified lines
58	#endif	58	#endif
59		59
60	#ifdef CONFIG_TIME_NS	60	#ifdef CONFIG_TIME_NS
61	static __always_inline const struct vdso_data *__arch_get_timens_vdso_data(void)	61	static __always_inline
		62	const struct vdso_data __arch_get_timens_vdso_data(const struct vdso_data vd)
62	{	63	{
63	return __timens_vdso_data;	64	return __timens_vdso_data;
64	}	65	}

drivers/i2c/busses/Kconfig+1−1

@@ -776,7 +776,7 @@ config I2C_MT7621 · 775 unmodified lines
776		776
777	config I2C_MV64XXX	777	config I2C_MV64XXX
778	tristate "Marvell mv64xxx I2C Controller"	778	tristate "Marvell mv64xxx I2C Controller"
779	depends on MV64X60 \|\| PLAT_ORION \|\| ARCH_SUNXI \|\| ARCH_MVEBU \|\| COMPILE_TEST	779	depends on PLAT_ORION \|\| ARCH_SUNXI \|\| ARCH_MVEBU \|\| COMPILE_TEST
780	help	780	help
781	If you say yes to this option, support will be included for the	781	If you say yes to this option, support will be included for the
782	built-in I2C interface on the Marvell 64xxx line of host bridges.	782	built-in I2C interface on the Marvell 64xxx line of host bridges.

drivers/macintosh/via-pmu.c+2−2

 static int option_server_mode;
 int pmu_battery_count;
-int pmu_cur_battery;
+static int pmu_cur_battery;
 unsigned int pmu_power_flags = PMU_PWR_AC_PRESENT;
 struct pmu_battery_info pmu_batteries[PMU_MAX_BATTERIES];
 static int query_batt_timer = BATTERY_POLLING_COUNT;
 static struct adb_request batt_req;
 static struct proc_dir_entry *proc_pmu_batt[PMU_MAX_BATTERIES];
-int __fake_sleep;
 int asleep;
 #ifdef CONFIG_ADB
  */
 static u32 save_via[8];
+static int __fake_sleep;
 static void
 save_via_state(void)

drivers/macintosh/windfarm_core.c+1−1

@@ -56,7 +56,7 @@ static BLOCKING_NOTIFIER_HEAD(wf_client_list); · 55 unmodified lines
56	static int wf_client_count;	56	static int wf_client_count;
57	static unsigned int wf_overtemp;	57	static unsigned int wf_overtemp;
58	static unsigned int wf_overtemp_counter;	58	static unsigned int wf_overtemp_counter;
59	struct task_struct *wf_thread;	59	static struct task_struct *wf_thread;
60		60
61	static struct platform_device wf_platform_device = {	61	static struct platform_device wf_platform_device = {
62	.name = "windfarm",	62	.name = "windfarm",

drivers/macintosh/windfarm_pm121.c+1−1

@@ -433,7 +433,7 @@ struct pm121_sys_state { · 432 unmodified lines
433	struct wf_pid_state pid;	433	struct wf_pid_state pid;
434	};	434	};
435		435
436	struct pm121_sys_state *pm121_sys_state[N_LOOPS] = {};	436	static struct pm121_sys_state *pm121_sys_state[N_LOOPS] = {};
437		437
438	/*	438	/*
439	* **** CPU Fans Control Loop ****	439	* **** CPU Fans Control Loop ****

drivers/macintosh/windfarm_smu_controls.c+1−1

@@ -94,7 +94,7 @@ static int smu_set_fan(int pwm, u8 id, u16 value) · 93 unmodified lines
94	return rc;	94	return rc;
95	wait_for_completion(&comp);	95	wait_for_completion(&comp);
96		96
97	/* Handle fallback (see coment above) */	97	/* Handle fallback (see comment above) */
98	if (cmd.status != 0 && smu_supports_new_fans_ops) {	98	if (cmd.status != 0 && smu_supports_new_fans_ops) {
99	printk(KERN_WARNING "windfarm: SMU failed new fan command "	99	printk(KERN_WARNING "windfarm: SMU failed new fan command "
100	"falling back to old method\n");	100	"falling back to old method\n");

include/linux/compat.h+35−0

@@ -467,6 +467,34 @@ put_compat_sigset(compat_sigset_t __user compat, const sigset_t set, · 466 unmodified lines
467	unsafe_put_user(__s->sig[0], &__c->sig[0], label); \	467	unsafe_put_user(__s->sig[0], &__c->sig[0], label); \
468	} \	468	} \
469	} while (0)	469	} while (0)
		470
		471	#define unsafe_get_compat_sigset(set, compat, label) do { \
		472	const compat_sigset_t __user *__c = compat; \
		473	compat_sigset_word hi, lo; \
		474	sigset_t *__s = set; \
		475	\
		476	switch (_NSIG_WORDS) { \
		477	case 4: \
		478	unsafe_get_user(lo, &__c->sig[7], label); \
		479	unsafe_get_user(hi, &__c->sig[6], label); \
		480	__s->sig[3] = hi \| (((long)lo) << 32); \
		481	fallthrough; \
		482	case 3: \
		483	unsafe_get_user(lo, &__c->sig[5], label); \
		484	unsafe_get_user(hi, &__c->sig[4], label); \
		485	__s->sig[2] = hi \| (((long)lo) << 32); \
		486	fallthrough; \
		487	case 2: \
		488	unsafe_get_user(lo, &__c->sig[3], label); \
		489	unsafe_get_user(hi, &__c->sig[2], label); \
		490	__s->sig[1] = hi \| (((long)lo) << 32); \
		491	fallthrough; \
		492	case 1: \
		493	unsafe_get_user(lo, &__c->sig[1], label); \
		494	unsafe_get_user(hi, &__c->sig[0], label); \
		495	__s->sig[0] = hi \| (((long)lo) << 32); \
		496	} \
		497	} while (0)
470	#else	498	#else
471	#define unsafe_put_compat_sigset(compat, set, label) do { \	499	#define unsafe_put_compat_sigset(compat, set, label) do { \
472	compat_sigset_t __user *__c = compat; \	500	compat_sigset_t __user *__c = compat; \
@@ -474,6 +502,13 @@ put_compat_sigset(compat_sigset_t __user compat, const sigset_t set, · 1 unmodified line
474	\	502	\
475	unsafe_copy_to_user(__c, __s, sizeof(*__c), label); \	503	unsafe_copy_to_user(__c, __s, sizeof(*__c), label); \
476	} while (0)	504	} while (0)
		505
		506	#define unsafe_get_compat_sigset(set, compat, label) do { \
		507	const compat_sigset_t __user *__c = compat; \
		508	sigset_t *__s = set; \
		509	\
		510	unsafe_copy_from_user(__s, __c, sizeof(*__c), label); \
		511	} while (0)
477	#endif	512	#endif
478		513
479	extern int compat_ptrace_request(struct task_struct *child,	514	extern int compat_ptrace_request(struct task_struct *child,

include/linux/uaccess.h+1−0

@@ -397,6 +397,7 @@ long strnlen_user_nofault(const void __user *unsafe_addr, long count); · 396 unmodified lines
397	#define unsafe_get_user(x,p,e) unsafe_op_wrap(__get_user(x,p),e)	397	#define unsafe_get_user(x,p,e) unsafe_op_wrap(__get_user(x,p),e)
398	#define unsafe_put_user(x,p,e) unsafe_op_wrap(__put_user(x,p),e)	398	#define unsafe_put_user(x,p,e) unsafe_op_wrap(__put_user(x,p),e)
399	#define unsafe_copy_to_user(d,s,l,e) unsafe_op_wrap(__copy_to_user(d,s,l),e)	399	#define unsafe_copy_to_user(d,s,l,e) unsafe_op_wrap(__copy_to_user(d,s,l),e)
		400	#define unsafe_copy_from_user(d,s,l,e) unsafe_op_wrap(__copy_from_user(d,s,l),e)
400	static inline unsigned long user_access_save(void) { return 0UL; }	401	static inline unsigned long user_access_save(void) { return 0UL; }
401	static inline void user_access_restore(unsigned long flags) { }	402	static inline void user_access_restore(unsigned long flags) { }
402	#endif	403	#endif

lib/vdso/gettimeofday.c+17−14

 #endif
 #ifdef CONFIG_TIME_NS
-static int do_hres_timens(const struct vdso_data *vdns, clockid_t clk,
+static __always_inline int do_hres_timens(const struct vdso_data *vdns, clockid_t clk,
-			  struct __kernel_timespec *ts)
+					  struct __kernel_timespec *ts)
 {
-	const struct vdso_data *vd = __arch_get_timens_vdso_data();
+	const struct vdso_data *vd;
 	const struct timens_offset *offs = &vdns->offset[clk];
 	const struct vdso_timestamp *vdso_ts;
 	u64 cycles, last, ns;
 	u32 seq;
 	s64 sec;
+	vd = vdns - (clk == CLOCK_MONOTONIC_RAW ? CS_RAW : CS_HRES_COARSE);
+	vd = __arch_get_timens_vdso_data(vd);
 	if (clk != CLOCK_MONOTONIC_RAW)
 		vd = &vd[CS_HRES_COARSE];
 	else
 	return 0;
 }
 #else
-static __always_inline const struct vdso_data *__arch_get_timens_vdso_data(void)
+static __always_inline
+const struct vdso_data *__arch_get_timens_vdso_data(const struct vdso_data *vd)
 {
 	return NULL;
 }
-static int do_hres_timens(const struct vdso_data *vdns, clockid_t clk,
+static __always_inline int do_hres_timens(const struct vdso_data *vdns, clockid_t clk,
-			  struct __kernel_timespec *ts)
+					  struct __kernel_timespec *ts)
 {
 	return -EINVAL;
 }
 }
 #ifdef CONFIG_TIME_NS
-static int do_coarse_timens(const struct vdso_data *vdns, clockid_t clk,
+static __always_inline int do_coarse_timens(const struct vdso_data *vdns, clockid_t clk,
-			    struct __kernel_timespec *ts)
+					    struct __kernel_timespec *ts)
 {
-	const struct vdso_data *vd = __arch_get_timens_vdso_data();
+	const struct vdso_data *vd = __arch_get_timens_vdso_data(vdns);
 	const struct vdso_timestamp *vdso_ts = &vd->basetime[clk];
 	const struct timens_offset *offs = &vdns->offset[clk];
 	u64 nsec;
 	return 0;
 }
 #else
-static int do_coarse_timens(const struct vdso_data *vdns, clockid_t clk,
+static __always_inline int do_coarse_timens(const struct vdso_data *vdns, clockid_t clk,
-			    struct __kernel_timespec *ts)
+					    struct __kernel_timespec *ts)
 {
 	return -1;
 }
 	if (unlikely(tz != NULL)) {
 		if (IS_ENABLED(CONFIG_TIME_NS) &&
 		    vd->clock_mode == VDSO_CLOCKMODE_TIMENS)
-			vd = __arch_get_timens_vdso_data();
+			vd = __arch_get_timens_vdso_data(vd);
 		tz->tz_minuteswest = vd[CS_HRES_COARSE].tz_minuteswest;
 		tz->tz_dsttime = vd[CS_HRES_COARSE].tz_dsttime;
 	if (IS_ENABLED(CONFIG_TIME_NS) &&
 	    vd->clock_mode == VDSO_CLOCKMODE_TIMENS)
-		vd = __arch_get_timens_vdso_data();
+		vd = __arch_get_timens_vdso_data(vd);
 	t = READ_ONCE(vd[CS_HRES_COARSE].basetime[CLOCK_REALTIME].sec);
 	if (IS_ENABLED(CONFIG_TIME_NS) &&
 	    vd->clock_mode == VDSO_CLOCKMODE_TIMENS)
-		vd = __arch_get_timens_vdso_data();
+		vd = __arch_get_timens_vdso_data(vd);
 	/*
 	 * Convert the clockid to a bitmask and use it to check which

tools/testing/selftests/powerpc/alignment/alignment_handler.c+1−10

@@ -10,16 +10,7 @@ · 9 unmodified lines
10	*	10	*
11	* We create two sets of source and destination buffers, one in regular memory,	11	* We create two sets of source and destination buffers, one in regular memory,
12	* the other cache-inhibited (by default we use /dev/fb0 for this, but an	12	* the other cache-inhibited (by default we use /dev/fb0 for this, but an
13	* alterative path for cache-inhibited memory may be provided).	13	* alterative path for cache-inhibited memory may be provided, e.g. memtrace).
14	*
15	* One way to get cache-inhibited memory is to use the "mem" kernel parameter
16	* to limit the kernel to less memory than actually exists. Addresses above
17	* the limit may still be accessed but will be treated as cache-inhibited. For
18	* example, if there is actually 4GB of memory and the parameter "mem=3GB" is
19	* used, memory from address 0xC0000000 onwards is treated as cache-inhibited.
20	* To access this region /dev/mem is used. The kernel should be configured
21	* without CONFIG_STRICT_DEVMEM. In this case use:
22	* ./alignment_handler /dev/mem 0xc0000000
23	*	14	*
24	* We initialise the source buffers, then use whichever set of load/store	15	* We initialise the source buffers, then use whichever set of load/store
25	* instructions is under test to copy bytes from the source buffers to the	16	* instructions is under test to copy bytes from the source buffers to the

tools/testing/selftests/powerpc/mm/Makefile+1−0

@@ -5,6 +5,7 @@ noarg: · 4 unmodified lines
5	TEST_GEN_PROGS := hugetlb_vs_thp_test subpage_prot prot_sao segv_errors wild_bctr \	5	TEST_GEN_PROGS := hugetlb_vs_thp_test subpage_prot prot_sao segv_errors wild_bctr \
6	large_vm_fork_separation bad_accesses pkey_exec_prot \	6	large_vm_fork_separation bad_accesses pkey_exec_prot \
7	pkey_siginfo stack_expansion_signal stack_expansion_ldst	7	pkey_siginfo stack_expansion_signal stack_expansion_ldst
		8	TEST_PROGS := stress_code_patching.sh
8		9
9	TEST_GEN_PROGS_EXTENDED := tlbie_test	10	TEST_GEN_PROGS_EXTENDED := tlbie_test
10	TEST_GEN_FILES := tempfile	11	TEST_GEN_FILES := tempfile

tools/testing/selftests/powerpc/mm/stress_code_patching.sh+49−0

@@ -0,0 +1,49 @@
	1	#!/bin/bash
	2	# SPDX-License-Identifier: GPL-2.0-or-later
	3
	4	TIMEOUT=30
	5
	6	DEBUFS_DIR=`cat /proc/mounts \| grep debugfs \| awk '{print $2}'`
	7	if [ ! -e "$DEBUFS_DIR" ]
	8	then
	9	echo "debugfs not found, skipping" 1>&2
	10	exit 4
	11	fi
	12
	13	if [ ! -e "$DEBUFS_DIR/tracing/current_tracer" ]
	14	then
	15	echo "Tracing files not found, skipping" 1>&2
	16	exit 4
	17	fi
	18
	19
	20	echo "Testing for spurious faults when mapping kernel memory..."
	21
	22	if grep -q "FUNCTION TRACING IS CORRUPTED" "$DEBUFS_DIR/tracing/trace"
	23	then
	24	echo "FAILED: Ftrace already dead. Probably due to a spurious fault" 1>&2
	25	exit 1
	26	fi
	27
	28	dmesg -C
	29	START_TIME=`date +%s`
	30	END_TIME=`expr $START_TIME + $TIMEOUT`
	31	while [ `date +%s` -lt $END_TIME ]
	32	do
	33	echo function > $DEBUFS_DIR/tracing/current_tracer
	34	echo nop > $DEBUFS_DIR/tracing/current_tracer
	35	if dmesg \| grep -q 'ftrace bug'
	36	then
	37	break
	38	fi
	39	done
	40
	41	echo nop > $DEBUFS_DIR/tracing/current_tracer
	42	if dmesg \| grep -q 'ftrace bug'
	43	then
	44	echo "FAILED: Mapping kernel memory causes spurious faults" 1>&2
	45	exit 1
	46	else
	47	echo "OK: Mapping kernel memory does not cause spurious faults"
	48	exit 0
	49	fi

tools/testing/selftests/powerpc/nx-gzip/gzfht_test.c+1−1

@@ -324,7 +324,7 @@ int compress_file(int argc, char *argv, void handle) · 323 unmodified lines
324	fprintf(stderr, "error: cannot progress; ");	324	fprintf(stderr, "error: cannot progress; ");
325	fprintf(stderr, "too many faults\n");	325	fprintf(stderr, "too many faults\n");
326	exit(-1);	326	exit(-1);
327	};	327	}
328	}	328	}
329		329
330	fault_tries = NX_MAX_FAULTS; /* Reset for the next chunk */	330	fault_tries = NX_MAX_FAULTS; /* Reset for the next chunk */

tools/testing/selftests/powerpc/ptrace/.gitignore+1−0

@@ -14,3 +14,4 @@ perf-hwbreak · 13 unmodified lines
14	core-pkey	14	core-pkey
15	ptrace-pkey	15	ptrace-pkey
16	ptrace-syscall	16	ptrace-syscall
		17	ptrace-perf-hwbreak

tools/testing/selftests/powerpc/ptrace/Makefile+1−1

@@ -2,7 +2,7 @@ · 1 unmodified line
2	TEST_GEN_PROGS := ptrace-gpr ptrace-tm-gpr ptrace-tm-spd-gpr \	2	TEST_GEN_PROGS := ptrace-gpr ptrace-tm-gpr ptrace-tm-spd-gpr \
3	ptrace-tar ptrace-tm-tar ptrace-tm-spd-tar ptrace-vsx ptrace-tm-vsx \	3	ptrace-tar ptrace-tm-tar ptrace-tm-spd-tar ptrace-vsx ptrace-tm-vsx \
4	ptrace-tm-spd-vsx ptrace-tm-spr ptrace-hwbreak ptrace-pkey core-pkey \	4	ptrace-tm-spd-vsx ptrace-tm-spr ptrace-hwbreak ptrace-pkey core-pkey \
5	perf-hwbreak ptrace-syscall	5	perf-hwbreak ptrace-syscall ptrace-perf-hwbreak
6		6
7	top_srcdir = ../../../../..	7	top_srcdir = ../../../../..
8	include ../../lib.mk	8	include ../../lib.mk

tools/testing/selftests/powerpc/ptrace/perf-hwbreak.c+594−41

@@ -21,8 +21,13 @@ · 20 unmodified lines
21	#include <assert.h>	21	#include <assert.h>
22	#include <stdio.h>	22	#include <stdio.h>
23	#include <stdlib.h>	23	#include <stdlib.h>
		24	#include <signal.h>
24	#include <string.h>	25	#include <string.h>
25	#include <sys/ioctl.h>	26	#include <sys/ioctl.h>
		27	#include <sys/wait.h>
		28	#include <sys/ptrace.h>
		29	#include <sys/sysinfo.h>
		30	#include <asm/ptrace.h>
26	#include <elf.h>	31	#include <elf.h>
27	#include <pthread.h>	32	#include <pthread.h>
28	#include <sys/syscall.h>	33	#include <sys/syscall.h>
@@ -30,32 +35,130 @@ · 1 unmodified line
30	#include <linux/hw_breakpoint.h>	35	#include <linux/hw_breakpoint.h>
31	#include "utils.h"	36	#include "utils.h"
32		37
		38	#ifndef PPC_DEBUG_FEATURE_DATA_BP_ARCH_31
		39	#define PPC_DEBUG_FEATURE_DATA_BP_ARCH_31 0x20
		40	#endif
		41
33	#define MAX_LOOPS 10000	42	#define MAX_LOOPS 10000
34		43
35	#define DAWR_LENGTH_MAX ((0x3f + 1) * 8)	44	#define DAWR_LENGTH_MAX ((0x3f + 1) * 8)
36		45
37	static inline int sys_perf_event_open(struct perf_event_attr *attr, pid_t pid,	46	int nprocs;
38	int cpu, int group_fd,	47
39	unsigned long flags)	48	static volatile int a = 10;
		49	static volatile int b = 10;
		50	static volatile char c[512 + 8] __attribute__((aligned(512)));
		51
		52	static void perf_event_attr_set(struct perf_event_attr *attr,
		53	__u32 type, __u64 addr, __u64 len,
		54	bool exclude_user)
40	{	55	{
41	attr->size = sizeof(*attr);	56	memset(attr, 0, sizeof(struct perf_event_attr));
42	return syscall(__NR_perf_event_open, attr, pid, cpu, group_fd, flags);	57	attr->type = PERF_TYPE_BREAKPOINT;
		58	attr->size = sizeof(struct perf_event_attr);
		59	attr->bp_type = type;
		60	attr->bp_addr = addr;
		61	attr->bp_len = len;
		62	attr->exclude_kernel = 1;
		63	attr->exclude_hv = 1;
		64	attr->exclude_guest = 1;
		65	attr->exclude_user = exclude_user;
		66	attr->disabled = 1;
43	}	67	}
44		68
45	static inline bool breakpoint_test(int len)	69	static int
		70	perf_process_event_open_exclude_user(__u32 type, __u64 addr, __u64 len, bool exclude_user)
		71	{
		72	struct perf_event_attr attr;
		73
		74	perf_event_attr_set(&attr, type, addr, len, exclude_user);
		75	return syscall(__NR_perf_event_open, &attr, getpid(), -1, -1, 0);
		76	}
		77
		78	static int perf_process_event_open(__u32 type, __u64 addr, __u64 len)
		79	{
		80	struct perf_event_attr attr;
		81
		82	perf_event_attr_set(&attr, type, addr, len, 0);
		83	return syscall(__NR_perf_event_open, &attr, getpid(), -1, -1, 0);
		84	}
		85
		86	static int perf_cpu_event_open(long cpu, __u32 type, __u64 addr, __u64 len)
46	{	87	{
47	struct perf_event_attr attr;	88	struct perf_event_attr attr;
		89
		90	perf_event_attr_set(&attr, type, addr, len, 0);
		91	return syscall(__NR_perf_event_open, &attr, -1, cpu, -1, 0);
		92	}
		93
		94	static void close_fds(int *fd, int n)
		95	{
		96	int i;
		97
		98	for (i = 0; i < n; i++)
		99	close(fd[i]);
		100	}
		101
		102	static unsigned long read_fds(int *fd, int n)
		103	{
		104	int i;
		105	unsigned long c = 0;
		106	unsigned long count = 0;
		107	size_t res;
		108
		109	for (i = 0; i < n; i++) {
		110	res = read(fd[i], &c, sizeof(c));
		111	assert(res == sizeof(unsigned long long));
		112	count += c;
		113	}
		114	return count;
		115	}
		116
		117	static void reset_fds(int *fd, int n)
		118	{
		119	int i;
		120
		121	for (i = 0; i < n; i++)
		122	ioctl(fd[i], PERF_EVENT_IOC_RESET);
		123	}
		124
		125	static void enable_fds(int *fd, int n)
		126	{
		127	int i;
		128
		129	for (i = 0; i < n; i++)
		130	ioctl(fd[i], PERF_EVENT_IOC_ENABLE);
		131	}
		132
		133	static void disable_fds(int *fd, int n)
		134	{
		135	int i;
		136
		137	for (i = 0; i < n; i++)
		138	ioctl(fd[i], PERF_EVENT_IOC_DISABLE);
		139	}
		140
		141	static int perf_systemwide_event_open(int *fd, __u32 type, __u64 addr, __u64 len)
		142	{
		143	int i = 0;
		144
		145	/* Assume online processors are 0 to nprocs for simplisity */
		146	for (i = 0; i < nprocs; i++) {
		147	fd[i] = perf_cpu_event_open(i, type, addr, len);
		148	if (fd[i] < 0) {
		149	close_fds(fd, i);
		150	return fd[i];
		151	}
		152	}
		153	return 0;
		154	}
		155
		156	static inline bool breakpoint_test(int len)
		157	{
48	int fd;	158	int fd;
49		159
50	/* setup counters */
51	memset(&attr, 0, sizeof(attr));
52	attr.disabled = 1;
53	attr.type = PERF_TYPE_BREAKPOINT;
54	attr.bp_type = HW_BREAKPOINT_R;
55	/* bp_addr can point anywhere but needs to be aligned */	160	/* bp_addr can point anywhere but needs to be aligned */
56	attr.bp_addr = (__u64)(&attr) & 0xfffffffffffff800;	161	fd = perf_process_event_open(HW_BREAKPOINT_R, (__u64)(&fd) & 0xfffffffffffff800, len);
57	attr.bp_len = len;
58	fd = sys_perf_event_open(&attr, 0, -1, -1, 0);
59	if (fd < 0)	162	if (fd < 0)
60	return false;	163	return false;
61	close(fd);	164	close(fd);
@@ -75,7 +178,6 @@ static inline bool dawr_supported(void) · 13 unmodified lines
75	static int runtestsingle(int readwriteflag, int exclude_user, int arraytest)	178	static int runtestsingle(int readwriteflag, int exclude_user, int arraytest)
76	{	179	{
77	int i,j;	180	int i,j;
78	struct perf_event_attr attr;
79	size_t res;	181	size_t res;
80	unsigned long long breaks, needed;	182	unsigned long long breaks, needed;
81	int readint;	183	int readint;
@@ -85,6 +187,7 @@ static int runtestsingle(int readwriteflag, int exclude_user, int arraytest) · 3 unmodified lines
85	int break_fd;	187	int break_fd;
86	int loop_num = MAX_LOOPS - (rand() % 100); /* provide some variability */	188	int loop_num = MAX_LOOPS - (rand() % 100); /* provide some variability */
87	volatile int *k;	189	volatile int *k;
		190	__u64 len;
88		191
89	/* align to 0x400 boundary as required by DAWR */	192	/* align to 0x400 boundary as required by DAWR */
90	readintalign = (int *)(((unsigned long)readintarraybig + 0x7ff) &	193	readintalign = (int *)(((unsigned long)readintarraybig + 0x7ff) &
@@ -94,19 +197,11 @@ static int runtestsingle(int readwriteflag, int exclude_user, int arraytest) · 3 unmodified lines
94	if (arraytest)	197	if (arraytest)
95	ptr = &readintalign[0];	198	ptr = &readintalign[0];
96		199
97	/* setup counters */	200	len = arraytest ? DAWR_LENGTH_MAX : sizeof(int);
98	memset(&attr, 0, sizeof(attr));	201	break_fd = perf_process_event_open_exclude_user(readwriteflag, (__u64)ptr,
99	attr.disabled = 1;	202	len, exclude_user);
100	attr.type = PERF_TYPE_BREAKPOINT;
101	attr.bp_type = readwriteflag;
102	attr.bp_addr = (__u64)ptr;
103	attr.bp_len = sizeof(int);
104	if (arraytest)
105	attr.bp_len = DAWR_LENGTH_MAX;
106	attr.exclude_user = exclude_user;
107	break_fd = sys_perf_event_open(&attr, 0, -1, -1, 0);
108	if (break_fd < 0) {	203	if (break_fd < 0) {
109	perror("sys_perf_event_open");	204	perror("perf_process_event_open_exclude_user");
110	exit(1);	205	exit(1);
111	}	206	}
112		207
@@ -153,7 +248,6 @@ static int runtest_dar_outside(void) · 40 unmodified lines
153	void *target;	248	void *target;
154	volatile __u16 temp16;	249	volatile __u16 temp16;
155	volatile __u64 temp64;	250	volatile __u64 temp64;
156	struct perf_event_attr attr;
157	int break_fd;	251	int break_fd;
158	unsigned long long breaks;	252	unsigned long long breaks;
159	int fail = 0;	253	int fail = 0;
@@ -165,21 +259,11 @@ static int runtest_dar_outside(void) · 5 unmodified lines
165	exit(EXIT_FAILURE);	259	exit(EXIT_FAILURE);
166	}	260	}
167		261
168	/* setup counters */
169	memset(&attr, 0, sizeof(attr));
170	attr.disabled = 1;
171	attr.type = PERF_TYPE_BREAKPOINT;
172	attr.exclude_kernel = 1;
173	attr.exclude_hv = 1;
174	attr.exclude_guest = 1;
175	attr.bp_type = HW_BREAKPOINT_RW;
176	/* watch middle half of target array */	262	/* watch middle half of target array */
177	attr.bp_addr = (__u64)(target + 2);	263	break_fd = perf_process_event_open(HW_BREAKPOINT_RW, (__u64)(target + 2), 4);
178	attr.bp_len = 4;
179	break_fd = sys_perf_event_open(&attr, 0, -1, -1, 0);
180	if (break_fd < 0) {	264	if (break_fd < 0) {
181	free(target);	265	free(target);
182	perror("sys_perf_event_open");	266	perror("perf_process_event_open");
183	exit(EXIT_FAILURE);	267	exit(EXIT_FAILURE);
184	}	268	}
185		269
@@ -263,11 +347,467 @@ static int runtest_dar_outside(void) · 77 unmodified lines
263	return fail;	347	return fail;
264	}	348	}
265		349
		350	static void multi_dawr_workload(void)
		351	{
		352	a += 10;
		353	b += 10;
		354	c[512 + 1] += 'a';
		355	}
		356
		357	static int test_process_multi_diff_addr(void)
		358	{
		359	unsigned long long breaks1 = 0, breaks2 = 0;
		360	int fd1, fd2;
		361	char *desc = "Process specific, Two events, diff addr";
		362	size_t res;
		363
		364	fd1 = perf_process_event_open(HW_BREAKPOINT_RW, (__u64)&a, (__u64)sizeof(a));
		365	if (fd1 < 0) {
		366	perror("perf_process_event_open");
		367	exit(EXIT_FAILURE);
		368	}
		369
		370	fd2 = perf_process_event_open(HW_BREAKPOINT_RW, (__u64)&b, (__u64)sizeof(b));
		371	if (fd2 < 0) {
		372	close(fd1);
		373	perror("perf_process_event_open");
		374	exit(EXIT_FAILURE);
		375	}
		376
		377	ioctl(fd1, PERF_EVENT_IOC_RESET);
		378	ioctl(fd2, PERF_EVENT_IOC_RESET);
		379	ioctl(fd1, PERF_EVENT_IOC_ENABLE);
		380	ioctl(fd2, PERF_EVENT_IOC_ENABLE);
		381	multi_dawr_workload();
		382	ioctl(fd1, PERF_EVENT_IOC_DISABLE);
		383	ioctl(fd2, PERF_EVENT_IOC_DISABLE);
		384
		385	res = read(fd1, &breaks1, sizeof(breaks1));
		386	assert(res == sizeof(unsigned long long));
		387	res = read(fd2, &breaks2, sizeof(breaks2));
		388	assert(res == sizeof(unsigned long long));
		389
		390	close(fd1);
		391	close(fd2);
		392
		393	if (breaks1 != 2 \|\| breaks2 != 2) {
		394	printf("FAILED: %s: %lld != 2 \|\| %lld != 2\n", desc, breaks1, breaks2);
		395	return 1;
		396	}
		397
		398	printf("TESTED: %s\n", desc);
		399	return 0;
		400	}
		401
		402	static int test_process_multi_same_addr(void)
		403	{
		404	unsigned long long breaks1 = 0, breaks2 = 0;
		405	int fd1, fd2;
		406	char *desc = "Process specific, Two events, same addr";
		407	size_t res;
		408
		409	fd1 = perf_process_event_open(HW_BREAKPOINT_RW, (__u64)&a, (__u64)sizeof(a));
		410	if (fd1 < 0) {
		411	perror("perf_process_event_open");
		412	exit(EXIT_FAILURE);
		413	}
		414
		415	fd2 = perf_process_event_open(HW_BREAKPOINT_RW, (__u64)&a, (__u64)sizeof(a));
		416	if (fd2 < 0) {
		417	close(fd1);
		418	perror("perf_process_event_open");
		419	exit(EXIT_FAILURE);
		420	}
		421
		422	ioctl(fd1, PERF_EVENT_IOC_RESET);
		423	ioctl(fd2, PERF_EVENT_IOC_RESET);
		424	ioctl(fd1, PERF_EVENT_IOC_ENABLE);
		425	ioctl(fd2, PERF_EVENT_IOC_ENABLE);
		426	multi_dawr_workload();
		427	ioctl(fd1, PERF_EVENT_IOC_DISABLE);
		428	ioctl(fd2, PERF_EVENT_IOC_DISABLE);
		429
		430	res = read(fd1, &breaks1, sizeof(breaks1));
		431	assert(res == sizeof(unsigned long long));
		432	res = read(fd2, &breaks2, sizeof(breaks2));
		433	assert(res == sizeof(unsigned long long));
		434
		435	close(fd1);
		436	close(fd2);
		437
		438	if (breaks1 != 2 \|\| breaks2 != 2) {
		439	printf("FAILED: %s: %lld != 2 \|\| %lld != 2\n", desc, breaks1, breaks2);
		440	return 1;
		441	}
		442
		443	printf("TESTED: %s\n", desc);
		444	return 0;
		445	}
		446
		447	static int test_process_multi_diff_addr_ro_wo(void)
		448	{
		449	unsigned long long breaks1 = 0, breaks2 = 0;
		450	int fd1, fd2;
		451	char *desc = "Process specific, Two events, diff addr, one is RO, other is WO";
		452	size_t res;
		453
		454	fd1 = perf_process_event_open(HW_BREAKPOINT_W, (__u64)&a, (__u64)sizeof(a));
		455	if (fd1 < 0) {
		456	perror("perf_process_event_open");
		457	exit(EXIT_FAILURE);
		458	}
		459
		460	fd2 = perf_process_event_open(HW_BREAKPOINT_R, (__u64)&b, (__u64)sizeof(b));
		461	if (fd2 < 0) {
		462	close(fd1);
		463	perror("perf_process_event_open");
		464	exit(EXIT_FAILURE);
		465	}
		466
		467	ioctl(fd1, PERF_EVENT_IOC_RESET);
		468	ioctl(fd2, PERF_EVENT_IOC_RESET);
		469	ioctl(fd1, PERF_EVENT_IOC_ENABLE);
		470	ioctl(fd2, PERF_EVENT_IOC_ENABLE);
		471	multi_dawr_workload();
		472	ioctl(fd1, PERF_EVENT_IOC_DISABLE);
		473	ioctl(fd2, PERF_EVENT_IOC_DISABLE);
		474
		475	res = read(fd1, &breaks1, sizeof(breaks1));
		476	assert(res == sizeof(unsigned long long));
		477	res = read(fd2, &breaks2, sizeof(breaks2));
		478	assert(res == sizeof(unsigned long long));
		479
		480	close(fd1);
		481	close(fd2);
		482
		483	if (breaks1 != 1 \|\| breaks2 != 1) {
		484	printf("FAILED: %s: %lld != 1 \|\| %lld != 1\n", desc, breaks1, breaks2);
		485	return 1;
		486	}
		487
		488	printf("TESTED: %s\n", desc);
		489	return 0;
		490	}
		491
		492	static int test_process_multi_same_addr_ro_wo(void)
		493	{
		494	unsigned long long breaks1 = 0, breaks2 = 0;
		495	int fd1, fd2;
		496	char *desc = "Process specific, Two events, same addr, one is RO, other is WO";
		497	size_t res;
		498
		499	fd1 = perf_process_event_open(HW_BREAKPOINT_R, (__u64)&a, (__u64)sizeof(a));
		500	if (fd1 < 0) {
		501	perror("perf_process_event_open");
		502	exit(EXIT_FAILURE);
		503	}
		504
		505	fd2 = perf_process_event_open(HW_BREAKPOINT_W, (__u64)&a, (__u64)sizeof(a));
		506	if (fd2 < 0) {
		507	close(fd1);
		508	perror("perf_process_event_open");
		509	exit(EXIT_FAILURE);
		510	}
		511
		512	ioctl(fd1, PERF_EVENT_IOC_RESET);
		513	ioctl(fd2, PERF_EVENT_IOC_RESET);
		514	ioctl(fd1, PERF_EVENT_IOC_ENABLE);
		515	ioctl(fd2, PERF_EVENT_IOC_ENABLE);
		516	multi_dawr_workload();
		517	ioctl(fd1, PERF_EVENT_IOC_DISABLE);
		518	ioctl(fd2, PERF_EVENT_IOC_DISABLE);
		519
		520	res = read(fd1, &breaks1, sizeof(breaks1));
		521	assert(res == sizeof(unsigned long long));
		522	res = read(fd2, &breaks2, sizeof(breaks2));
		523	assert(res == sizeof(unsigned long long));
		524
		525	close(fd1);
		526	close(fd2);
		527
		528	if (breaks1 != 1 \|\| breaks2 != 1) {
		529	printf("FAILED: %s: %lld != 1 \|\| %lld != 1\n", desc, breaks1, breaks2);
		530	return 1;
		531	}
		532
		533	printf("TESTED: %s\n", desc);
		534	return 0;
		535	}
		536
		537	static int test_syswide_multi_diff_addr(void)
		538	{
		539	unsigned long long breaks1 = 0, breaks2 = 0;
		540	int fd1 = malloc(nprocs sizeof(int));
		541	int fd2 = malloc(nprocs sizeof(int));
		542	char *desc = "Systemwide, Two events, diff addr";
		543	int ret;
		544
		545	ret = perf_systemwide_event_open(fd1, HW_BREAKPOINT_RW, (__u64)&a, (__u64)sizeof(a));
		546	if (ret) {
		547	perror("perf_systemwide_event_open");
		548	exit(EXIT_FAILURE);
		549	}
		550
		551	ret = perf_systemwide_event_open(fd2, HW_BREAKPOINT_RW, (__u64)&b, (__u64)sizeof(b));
		552	if (ret) {
		553	close_fds(fd1, nprocs);
		554	perror("perf_systemwide_event_open");
		555	exit(EXIT_FAILURE);
		556	}
		557
		558	reset_fds(fd1, nprocs);
		559	reset_fds(fd2, nprocs);
		560	enable_fds(fd1, nprocs);
		561	enable_fds(fd2, nprocs);
		562	multi_dawr_workload();
		563	disable_fds(fd1, nprocs);
		564	disable_fds(fd2, nprocs);
		565
		566	breaks1 = read_fds(fd1, nprocs);
		567	breaks2 = read_fds(fd2, nprocs);
		568
		569	close_fds(fd1, nprocs);
		570	close_fds(fd2, nprocs);
		571
		572	free(fd1);
		573	free(fd2);
		574
		575	if (breaks1 != 2 \|\| breaks2 != 2) {
		576	printf("FAILED: %s: %lld != 2 \|\| %lld != 2\n", desc, breaks1, breaks2);
		577	return 1;
		578	}
		579
		580	printf("TESTED: %s\n", desc);
		581	return 0;
		582	}
		583
		584	static int test_syswide_multi_same_addr(void)
		585	{
		586	unsigned long long breaks1 = 0, breaks2 = 0;
		587	int fd1 = malloc(nprocs sizeof(int));
		588	int fd2 = malloc(nprocs sizeof(int));
		589	char *desc = "Systemwide, Two events, same addr";
		590	int ret;
		591
		592	ret = perf_systemwide_event_open(fd1, HW_BREAKPOINT_RW, (__u64)&a, (__u64)sizeof(a));
		593	if (ret) {
		594	perror("perf_systemwide_event_open");
		595	exit(EXIT_FAILURE);
		596	}
		597
		598	ret = perf_systemwide_event_open(fd2, HW_BREAKPOINT_RW, (__u64)&a, (__u64)sizeof(a));
		599	if (ret) {
		600	close_fds(fd1, nprocs);
		601	perror("perf_systemwide_event_open");
		602	exit(EXIT_FAILURE);
		603	}
		604
		605	reset_fds(fd1, nprocs);
		606	reset_fds(fd2, nprocs);
		607	enable_fds(fd1, nprocs);
		608	enable_fds(fd2, nprocs);
		609	multi_dawr_workload();
		610	disable_fds(fd1, nprocs);
		611	disable_fds(fd2, nprocs);
		612
		613	breaks1 = read_fds(fd1, nprocs);
		614	breaks2 = read_fds(fd2, nprocs);
		615
		616	close_fds(fd1, nprocs);
		617	close_fds(fd2, nprocs);
		618
		619	free(fd1);
		620	free(fd2);
		621
		622	if (breaks1 != 2 \|\| breaks2 != 2) {
		623	printf("FAILED: %s: %lld != 2 \|\| %lld != 2\n", desc, breaks1, breaks2);
		624	return 1;
		625	}
		626
		627	printf("TESTED: %s\n", desc);
		628	return 0;
		629	}
		630
		631	static int test_syswide_multi_diff_addr_ro_wo(void)
		632	{
		633	unsigned long long breaks1 = 0, breaks2 = 0;
		634	int fd1 = malloc(nprocs sizeof(int));
		635	int fd2 = malloc(nprocs sizeof(int));
		636	char *desc = "Systemwide, Two events, diff addr, one is RO, other is WO";
		637	int ret;
		638
		639	ret = perf_systemwide_event_open(fd1, HW_BREAKPOINT_W, (__u64)&a, (__u64)sizeof(a));
		640	if (ret) {
		641	perror("perf_systemwide_event_open");
		642	exit(EXIT_FAILURE);
		643	}
		644
		645	ret = perf_systemwide_event_open(fd2, HW_BREAKPOINT_R, (__u64)&b, (__u64)sizeof(b));
		646	if (ret) {
		647	close_fds(fd1, nprocs);
		648	perror("perf_systemwide_event_open");
		649	exit(EXIT_FAILURE);
		650	}
		651
		652	reset_fds(fd1, nprocs);
		653	reset_fds(fd2, nprocs);
		654	enable_fds(fd1, nprocs);
		655	enable_fds(fd2, nprocs);
		656	multi_dawr_workload();
		657	disable_fds(fd1, nprocs);
		658	disable_fds(fd2, nprocs);
		659
		660	breaks1 = read_fds(fd1, nprocs);
		661	breaks2 = read_fds(fd2, nprocs);
		662
		663	close_fds(fd1, nprocs);
		664	close_fds(fd2, nprocs);
		665
		666	free(fd1);
		667	free(fd2);
		668
		669	if (breaks1 != 1 \|\| breaks2 != 1) {
		670	printf("FAILED: %s: %lld != 1 \|\| %lld != 1\n", desc, breaks1, breaks2);
		671	return 1;
		672	}
		673
		674	printf("TESTED: %s\n", desc);
		675	return 0;
		676	}
		677
		678	static int test_syswide_multi_same_addr_ro_wo(void)
		679	{
		680	unsigned long long breaks1 = 0, breaks2 = 0;
		681	int fd1 = malloc(nprocs sizeof(int));
		682	int fd2 = malloc(nprocs sizeof(int));
		683	char *desc = "Systemwide, Two events, same addr, one is RO, other is WO";
		684	int ret;
		685
		686	ret = perf_systemwide_event_open(fd1, HW_BREAKPOINT_W, (__u64)&a, (__u64)sizeof(a));
		687	if (ret) {
		688	perror("perf_systemwide_event_open");
		689	exit(EXIT_FAILURE);
		690	}
		691
		692	ret = perf_systemwide_event_open(fd2, HW_BREAKPOINT_R, (__u64)&a, (__u64)sizeof(a));
		693	if (ret) {
		694	close_fds(fd1, nprocs);
		695	perror("perf_systemwide_event_open");
		696	exit(EXIT_FAILURE);
		697	}
		698
		699	reset_fds(fd1, nprocs);
		700	reset_fds(fd2, nprocs);
		701	enable_fds(fd1, nprocs);
		702	enable_fds(fd2, nprocs);
		703	multi_dawr_workload();
		704	disable_fds(fd1, nprocs);
		705	disable_fds(fd2, nprocs);
		706
		707	breaks1 = read_fds(fd1, nprocs);
		708	breaks2 = read_fds(fd2, nprocs);
		709
		710	close_fds(fd1, nprocs);
		711	close_fds(fd2, nprocs);
		712
		713	free(fd1);
		714	free(fd2);
		715
		716	if (breaks1 != 1 \|\| breaks2 != 1) {
		717	printf("FAILED: %s: %lld != 1 \|\| %lld != 1\n", desc, breaks1, breaks2);
		718	return 1;
		719	}
		720
		721	printf("TESTED: %s\n", desc);
		722	return 0;
		723	}
		724
		725	static int runtest_multi_dawr(void)
		726	{
		727	int ret = 0;
		728
		729	ret \|= test_process_multi_diff_addr();
		730	ret \|= test_process_multi_same_addr();
		731	ret \|= test_process_multi_diff_addr_ro_wo();
		732	ret \|= test_process_multi_same_addr_ro_wo();
		733	ret \|= test_syswide_multi_diff_addr();
		734	ret \|= test_syswide_multi_same_addr();
		735	ret \|= test_syswide_multi_diff_addr_ro_wo();
		736	ret \|= test_syswide_multi_same_addr_ro_wo();
		737
		738	return ret;
		739	}
		740
		741	static int runtest_unaligned_512bytes(void)
		742	{
		743	unsigned long long breaks = 0;
		744	int fd;
		745	char *desc = "Process specific, 512 bytes, unaligned";
		746	__u64 addr = (__u64)&c + 8;
		747	size_t res;
		748
		749	fd = perf_process_event_open(HW_BREAKPOINT_RW, addr, 512);
		750	if (fd < 0) {
		751	perror("perf_process_event_open");
		752	exit(EXIT_FAILURE);
		753	}
		754
		755	ioctl(fd, PERF_EVENT_IOC_RESET);
		756	ioctl(fd, PERF_EVENT_IOC_ENABLE);
		757	multi_dawr_workload();
		758	ioctl(fd, PERF_EVENT_IOC_DISABLE);
		759
		760	res = read(fd, &breaks, sizeof(breaks));
		761	assert(res == sizeof(unsigned long long));
		762
		763	close(fd);
		764
		765	if (breaks != 2) {
		766	printf("FAILED: %s: %lld != 2\n", desc, breaks);
		767	return 1;
		768	}
		769
		770	printf("TESTED: %s\n", desc);
		771	return 0;
		772	}
		773
		774	/* There is no perf api to find number of available watchpoints. Use ptrace. */
		775	static int get_nr_wps(bool *arch_31)
		776	{
		777	struct ppc_debug_info dbginfo;
		778	int child_pid;
		779
		780	child_pid = fork();
		781	if (!child_pid) {
		782	int ret = ptrace(PTRACE_TRACEME, 0, NULL, 0);
		783	if (ret) {
		784	perror("PTRACE_TRACEME failed\n");
		785	exit(EXIT_FAILURE);
		786	}
		787	kill(getpid(), SIGUSR1);
		788
		789	sleep(1);
		790	exit(EXIT_SUCCESS);
		791	}
		792
		793	wait(NULL);
		794	if (ptrace(PPC_PTRACE_GETHWDBGINFO, child_pid, NULL, &dbginfo)) {
		795	perror("Can't get breakpoint info");
		796	exit(EXIT_FAILURE);
		797	}
		798
		799	*arch_31 = !!(dbginfo.features & PPC_DEBUG_FEATURE_DATA_BP_ARCH_31);
		800	return dbginfo.num_data_bps;
		801	}
		802
266	static int runtest(void)	803	static int runtest(void)
267	{	804	{
268	int rwflag;	805	int rwflag;
269	int exclude_user;	806	int exclude_user;
270	int ret;	807	int ret;
		808	bool dawr = dawr_supported();
		809	bool arch_31 = false;
		810	int nr_wps = get_nr_wps(&arch_31);
271		811
272	/*	812	/*
273	* perf defines rwflag as two bits read and write and at least	813	* perf defines rwflag as two bits read and write and at least
@@ -280,7 +820,7 @@ static int runtest(void) · 6 unmodified lines
280	return ret;	820	return ret;
281		821
282	/* if we have the dawr, we can do an array test */	822	/* if we have the dawr, we can do an array test */
283	if (!dawr_supported())	823	if (!dawr)
284	continue;	824	continue;
285	ret = runtestsingle(rwflag, exclude_user, 1);	825	ret = runtestsingle(rwflag, exclude_user, 1);
286	if (ret)	826	if (ret)
@@ -289,6 +829,19 @@ static int runtest(void) · 2 unmodified lines
289	}	829	}
290		830
291	ret = runtest_dar_outside();	831	ret = runtest_dar_outside();
		832	if (ret)
		833	return ret;
		834
		835	if (dawr && nr_wps > 1) {
		836	nprocs = get_nprocs();
		837	ret = runtest_multi_dawr();
		838	if (ret)
		839	return ret;
		840	}
		841
		842	if (dawr && arch_31)
		843	ret = runtest_unaligned_512bytes();
		844
292	return ret;	845	return ret;
293	}	846	}
294		847

tools/testing/selftests/powerpc/ptrace/ptrace-hwbreak.c+79−0

@@ -194,6 +194,18 @@ static void test_workload(void) · 193 unmodified lines
194	big_var[rand() % DAWR_MAX_LEN] = 'a';	194	big_var[rand() % DAWR_MAX_LEN] = 'a';
195	else	195	else
196	cvar = big_var[rand() % DAWR_MAX_LEN];	196	cvar = big_var[rand() % DAWR_MAX_LEN];
		197
		198	/* PPC_PTRACE_SETHWDEBUG 2, MODE_RANGE, DW ALIGNED, WO test */
		199	gstruct.a[rand() % A_LEN] = 'a';
		200
		201	/* PPC_PTRACE_SETHWDEBUG 2, MODE_RANGE, DW UNALIGNED, RO test */
		202	cvar = gstruct.b[rand() % B_LEN];
		203
		204	/* PPC_PTRACE_SETHWDEBUG 2, MODE_RANGE, DAWR Overlap, WO test */
		205	gstruct.a[rand() % A_LEN] = 'a';
		206
		207	/* PPC_PTRACE_SETHWDEBUG 2, MODE_RANGE, DAWR Overlap, RO test */
		208	cvar = gstruct.a[rand() % A_LEN];
197	}	209	}
198		210
199	static void check_success(pid_t child_pid, const char name, const char type,	211	static void check_success(pid_t child_pid, const char name, const char type,
@@ -417,6 +429,69 @@ static void test_sethwdebug_range_aligned(pid_t child_pid) · 217 unmodified lines
417	ptrace_delhwdebug(child_pid, wh);	429	ptrace_delhwdebug(child_pid, wh);
418	}	430	}
419		431
		432	static void test_multi_sethwdebug_range(pid_t child_pid)
		433	{
		434	struct ppc_hw_breakpoint info1, info2;
		435	unsigned long wp_addr1, wp_addr2;
		436	char *name1 = "PPC_PTRACE_SETHWDEBUG 2, MODE_RANGE, DW ALIGNED";
		437	char *name2 = "PPC_PTRACE_SETHWDEBUG 2, MODE_RANGE, DW UNALIGNED";
		438	int len1, len2;
		439	int wh1, wh2;
		440
		441	wp_addr1 = (unsigned long)&gstruct.a;
		442	wp_addr2 = (unsigned long)&gstruct.b;
		443	len1 = A_LEN;
		444	len2 = B_LEN;
		445	get_ppc_hw_breakpoint(&info1, PPC_BREAKPOINT_TRIGGER_WRITE, wp_addr1, len1);
		446	get_ppc_hw_breakpoint(&info2, PPC_BREAKPOINT_TRIGGER_READ, wp_addr2, len2);
		447
		448	/* PPC_PTRACE_SETHWDEBUG 2, MODE_RANGE, DW ALIGNED, WO test */
		449	wh1 = ptrace_sethwdebug(child_pid, &info1);
		450
		451	/* PPC_PTRACE_SETHWDEBUG 2, MODE_RANGE, DW UNALIGNED, RO test */
		452	wh2 = ptrace_sethwdebug(child_pid, &info2);
		453
		454	ptrace(PTRACE_CONT, child_pid, NULL, 0);
		455	check_success(child_pid, name1, "WO", wp_addr1, len1);
		456
		457	ptrace(PTRACE_CONT, child_pid, NULL, 0);
		458	check_success(child_pid, name2, "RO", wp_addr2, len2);
		459
		460	ptrace_delhwdebug(child_pid, wh1);
		461	ptrace_delhwdebug(child_pid, wh2);
		462	}
		463
		464	static void test_multi_sethwdebug_range_dawr_overlap(pid_t child_pid)
		465	{
		466	struct ppc_hw_breakpoint info1, info2;
		467	unsigned long wp_addr1, wp_addr2;
		468	char *name = "PPC_PTRACE_SETHWDEBUG 2, MODE_RANGE, DAWR Overlap";
		469	int len1, len2;
		470	int wh1, wh2;
		471
		472	wp_addr1 = (unsigned long)&gstruct.a;
		473	wp_addr2 = (unsigned long)&gstruct.a;
		474	len1 = A_LEN;
		475	len2 = A_LEN;
		476	get_ppc_hw_breakpoint(&info1, PPC_BREAKPOINT_TRIGGER_WRITE, wp_addr1, len1);
		477	get_ppc_hw_breakpoint(&info2, PPC_BREAKPOINT_TRIGGER_READ, wp_addr2, len2);
		478
		479	/* PPC_PTRACE_SETHWDEBUG 2, MODE_RANGE, DAWR Overlap, WO test */
		480	wh1 = ptrace_sethwdebug(child_pid, &info1);
		481
		482	/* PPC_PTRACE_SETHWDEBUG 2, MODE_RANGE, DAWR Overlap, RO test */
		483	wh2 = ptrace_sethwdebug(child_pid, &info2);
		484
		485	ptrace(PTRACE_CONT, child_pid, NULL, 0);
		486	check_success(child_pid, name, "WO", wp_addr1, len1);
		487
		488	ptrace(PTRACE_CONT, child_pid, NULL, 0);
		489	check_success(child_pid, name, "RO", wp_addr2, len2);
		490
		491	ptrace_delhwdebug(child_pid, wh1);
		492	ptrace_delhwdebug(child_pid, wh2);
		493	}
		494
420	static void test_sethwdebug_range_unaligned(pid_t child_pid)	495	static void test_sethwdebug_range_unaligned(pid_t child_pid)
421	{	496	{
422	struct ppc_hw_breakpoint info;	497	struct ppc_hw_breakpoint info;
@@ -504,6 +579,10 @@ run_tests(pid_t child_pid, struct ppc_debug_info *dbginfo, bool dawr) · 81 unmodified lines
504	test_sethwdebug_range_unaligned(child_pid);	579	test_sethwdebug_range_unaligned(child_pid);
505	test_sethwdebug_range_unaligned_dar(child_pid);	580	test_sethwdebug_range_unaligned_dar(child_pid);
506	test_sethwdebug_dawr_max_range(child_pid);	581	test_sethwdebug_dawr_max_range(child_pid);
		582	if (dbginfo->num_data_bps > 1) {
		583	test_multi_sethwdebug_range(child_pid);
		584	test_multi_sethwdebug_range_dawr_overlap(child_pid);
		585	}
507	}	586	}
508	}	587	}
509	}	588	}

tools/testing/selftests/powerpc/ptrace/ptrace-perf-hwbreak.c+659−0

@@ -0,0 +1,659 @@
	1	// SPDX-License-Identifier: GPL-2.0+
	2	#include <stdio.h>
	3	#include <string.h>
	4	#include <signal.h>
	5	#include <stdlib.h>
	6	#include <unistd.h>
	7	#include <errno.h>
	8	#include <linux/hw_breakpoint.h>
	9	#include <linux/perf_event.h>
	10	#include <asm/unistd.h>
	11	#include <sys/ptrace.h>
	12	#include <sys/wait.h>
	13	#include "ptrace.h"
	14
	15	char data[16];
	16
	17	/* Overlapping address range */
	18	volatile __u64 ptrace_data1 = (__u64 )&data[0];
	19	volatile __u64 perf_data1 = (__u64 )&data[4];
	20
	21	/* Non-overlapping address range */
	22	volatile __u64 ptrace_data2 = (__u64 )&data[0];
	23	volatile __u64 perf_data2 = (__u64 )&data[8];
	24
	25	static unsigned long pid_max_addr(void)
	26	{
	27	FILE *fp;
	28	char line, c;
	29	char addr[100];
	30	size_t len = 0;
	31
	32	fp = fopen("/proc/kallsyms", "r");
	33	if (!fp) {
	34	printf("Failed to read /proc/kallsyms. Exiting..\n");
	35	exit(EXIT_FAILURE);
	36	}
	37
	38	while (getline(&line, &len, fp) != -1) {
	39	if (!strstr(line, "pid_max") \|\| strstr(line, "pid_max_max") \|\|
	40	strstr(line, "pid_max_min"))
	41	continue;
	42
	43	strncpy(addr, line, len < 100 ? len : 100);
	44	c = strchr(addr, ' ');
	45	*c = '\0';
	46	return strtoul(addr, &c, 16);
	47	}
	48	fclose(fp);
	49	printf("Could not find pix_max. Exiting..\n");
	50	exit(EXIT_FAILURE);
	51	return -1;
	52	}
	53
	54	static void perf_user_event_attr_set(struct perf_event_attr *attr, __u64 addr, __u64 len)
	55	{
	56	memset(attr, 0, sizeof(struct perf_event_attr));
	57	attr->type = PERF_TYPE_BREAKPOINT;
	58	attr->size = sizeof(struct perf_event_attr);
	59	attr->bp_type = HW_BREAKPOINT_R;
	60	attr->bp_addr = addr;
	61	attr->bp_len = len;
	62	attr->exclude_kernel = 1;
	63	attr->exclude_hv = 1;
	64	}
	65
	66	static void perf_kernel_event_attr_set(struct perf_event_attr *attr)
	67	{
	68	memset(attr, 0, sizeof(struct perf_event_attr));
	69	attr->type = PERF_TYPE_BREAKPOINT;
	70	attr->size = sizeof(struct perf_event_attr);
	71	attr->bp_type = HW_BREAKPOINT_R;
	72	attr->bp_addr = pid_max_addr();
	73	attr->bp_len = sizeof(unsigned long);
	74	attr->exclude_user = 1;
	75	attr->exclude_hv = 1;
	76	}
	77
	78	static int perf_cpu_event_open(int cpu, __u64 addr, __u64 len)
	79	{
	80	struct perf_event_attr attr;
	81
	82	perf_user_event_attr_set(&attr, addr, len);
	83	return syscall(__NR_perf_event_open, &attr, -1, cpu, -1, 0);
	84	}
	85
	86	static int perf_thread_event_open(pid_t child_pid, __u64 addr, __u64 len)
	87	{
	88	struct perf_event_attr attr;
	89
	90	perf_user_event_attr_set(&attr, addr, len);
	91	return syscall(__NR_perf_event_open, &attr, child_pid, -1, -1, 0);
	92	}
	93
	94	static int perf_thread_cpu_event_open(pid_t child_pid, int cpu, __u64 addr, __u64 len)
	95	{
	96	struct perf_event_attr attr;
	97
	98	perf_user_event_attr_set(&attr, addr, len);
	99	return syscall(__NR_perf_event_open, &attr, child_pid, cpu, -1, 0);
	100	}
	101
	102	static int perf_thread_kernel_event_open(pid_t child_pid)
	103	{
	104	struct perf_event_attr attr;
	105
	106	perf_kernel_event_attr_set(&attr);
	107	return syscall(__NR_perf_event_open, &attr, child_pid, -1, -1, 0);
	108	}
	109
	110	static int perf_cpu_kernel_event_open(int cpu)
	111	{
	112	struct perf_event_attr attr;
	113
	114	perf_kernel_event_attr_set(&attr);
	115	return syscall(__NR_perf_event_open, &attr, -1, cpu, -1, 0);
	116	}
	117
	118	static int child(void)
	119	{
	120	int ret;
	121
	122	ret = ptrace(PTRACE_TRACEME, 0, NULL, 0);
	123	if (ret) {
	124	printf("Error: PTRACE_TRACEME failed\n");
	125	return 0;
	126	}
	127	kill(getpid(), SIGUSR1); /* --> parent (SIGUSR1) */
	128
	129	return 0;
	130	}
	131
	132	static void ptrace_ppc_hw_breakpoint(struct ppc_hw_breakpoint *info, int type,
	133	__u64 addr, int len)
	134	{
	135	info->version = 1;
	136	info->trigger_type = type;
	137	info->condition_mode = PPC_BREAKPOINT_CONDITION_NONE;
	138	info->addr = addr;
	139	info->addr2 = addr + len;
	140	info->condition_value = 0;
	141	if (!len)
	142	info->addr_mode = PPC_BREAKPOINT_MODE_EXACT;
	143	else
	144	info->addr_mode = PPC_BREAKPOINT_MODE_RANGE_INCLUSIVE;
	145	}
	146
	147	static int ptrace_open(pid_t child_pid, __u64 wp_addr, int len)
	148	{
	149	struct ppc_hw_breakpoint info;
	150
	151	ptrace_ppc_hw_breakpoint(&info, PPC_BREAKPOINT_TRIGGER_RW, wp_addr, len);
	152	return ptrace(PPC_PTRACE_SETHWDEBUG, child_pid, 0, &info);
	153	}
	154
	155	static int test1(pid_t child_pid)
	156	{
	157	int perf_fd;
	158	int ptrace_fd;
	159	int ret = 0;
	160
	161	/* Test:
	162	* if (new per thread event by ptrace)
	163	* if (existing cpu event by perf)
	164	* if (addr range overlaps)
	165	* fail;
	166	*/
	167
	168	perf_fd = perf_cpu_event_open(0, (__u64)perf_data1, sizeof(*perf_data1));
	169	if (perf_fd < 0)
	170	return -1;
	171
	172	ptrace_fd = ptrace_open(child_pid, (__u64)ptrace_data1, sizeof(*ptrace_data1));
	173	if (ptrace_fd > 0 \|\| errno != ENOSPC)
	174	ret = -1;
	175
	176	close(perf_fd);
	177	return ret;
	178	}
	179
	180	static int test2(pid_t child_pid)
	181	{
	182	int perf_fd;
	183	int ptrace_fd;
	184	int ret = 0;
	185
	186	/* Test:
	187	* if (new per thread event by ptrace)
	188	* if (existing cpu event by perf)
	189	* if (addr range does not overlaps)
	190	* allow;
	191	*/
	192
	193	perf_fd = perf_cpu_event_open(0, (__u64)perf_data2, sizeof(*perf_data2));
	194	if (perf_fd < 0)
	195	return -1;
	196
	197	ptrace_fd = ptrace_open(child_pid, (__u64)ptrace_data2, sizeof(*ptrace_data2));
	198	if (ptrace_fd < 0) {
	199	ret = -1;
	200	goto perf_close;
	201	}
	202	ptrace(PPC_PTRACE_DELHWDEBUG, child_pid, 0, ptrace_fd);
	203
	204	perf_close:
	205	close(perf_fd);
	206	return ret;
	207	}
	208
	209	static int test3(pid_t child_pid)
	210	{
	211	int perf_fd;
	212	int ptrace_fd;
	213	int ret = 0;
	214
	215	/* Test:
	216	* if (new per thread event by ptrace)
	217	* if (existing thread event by perf on the same thread)
	218	* if (addr range overlaps)
	219	* fail;
	220	*/
	221	perf_fd = perf_thread_event_open(child_pid, (__u64)perf_data1,
	222	sizeof(*perf_data1));
	223	if (perf_fd < 0)
	224	return -1;
	225
	226	ptrace_fd = ptrace_open(child_pid, (__u64)ptrace_data1, sizeof(*ptrace_data1));
	227	if (ptrace_fd > 0 \|\| errno != ENOSPC)
	228	ret = -1;
	229
	230	close(perf_fd);
	231	return ret;
	232	}
	233
	234	static int test4(pid_t child_pid)
	235	{
	236	int perf_fd;
	237	int ptrace_fd;
	238	int ret = 0;
	239
	240	/* Test:
	241	* if (new per thread event by ptrace)
	242	* if (existing thread event by perf on the same thread)
	243	* if (addr range does not overlaps)
	244	* fail;
	245	*/
	246	perf_fd = perf_thread_event_open(child_pid, (__u64)perf_data2,
	247	sizeof(*perf_data2));
	248	if (perf_fd < 0)
	249	return -1;
	250
	251	ptrace_fd = ptrace_open(child_pid, (__u64)ptrace_data2, sizeof(*ptrace_data2));
	252	if (ptrace_fd < 0) {
	253	ret = -1;
	254	goto perf_close;
	255	}
	256	ptrace(PPC_PTRACE_DELHWDEBUG, child_pid, 0, ptrace_fd);
	257
	258	perf_close:
	259	close(perf_fd);
	260	return ret;
	261	}
	262
	263	static int test5(pid_t child_pid)
	264	{
	265	int perf_fd;
	266	int ptrace_fd;
	267	int cpid;
	268	int ret = 0;
	269
	270	/* Test:
	271	* if (new per thread event by ptrace)
	272	* if (existing thread event by perf on the different thread)
	273	* allow;
	274	*/
	275	cpid = fork();
	276	if (!cpid) {
	277	/* Temporary Child */
	278	pause();
	279	exit(EXIT_SUCCESS);
	280	}
	281
	282	perf_fd = perf_thread_event_open(cpid, (__u64)perf_data1, sizeof(*perf_data1));
	283	if (perf_fd < 0) {
	284	ret = -1;
	285	goto kill_child;
	286	}
	287
	288	ptrace_fd = ptrace_open(child_pid, (__u64)ptrace_data1, sizeof(*ptrace_data1));
	289	if (ptrace_fd < 0) {
	290	ret = -1;
	291	goto perf_close;
	292	}
	293
	294	ptrace(PPC_PTRACE_DELHWDEBUG, child_pid, 0, ptrace_fd);
	295	perf_close:
	296	close(perf_fd);
	297	kill_child:
	298	kill(cpid, SIGINT);
	299	return ret;
	300	}
	301
	302	static int test6(pid_t child_pid)
	303	{
	304	int perf_fd;
	305	int ptrace_fd;
	306	int ret = 0;
	307
	308	/* Test:
	309	* if (new per thread kernel event by perf)
	310	* if (existing thread event by ptrace on the same thread)
	311	* allow;
	312	* -- OR --
	313	* if (new per cpu kernel event by perf)
	314	* if (existing thread event by ptrace)
	315	* allow;
	316	*/
	317	ptrace_fd = ptrace_open(child_pid, (__u64)ptrace_data1, sizeof(*ptrace_data1));
	318	if (ptrace_fd < 0)
	319	return -1;
	320
	321	perf_fd = perf_thread_kernel_event_open(child_pid);
	322	if (perf_fd < 0) {
	323	ret = -1;
	324	goto ptrace_close;
	325	}
	326	close(perf_fd);
	327
	328	perf_fd = perf_cpu_kernel_event_open(0);
	329	if (perf_fd < 0) {
	330	ret = -1;
	331	goto ptrace_close;
	332	}
	333	close(perf_fd);
	334
	335	ptrace_close:
	336	ptrace(PPC_PTRACE_DELHWDEBUG, child_pid, 0, ptrace_fd);
	337	return ret;
	338	}
	339
	340	static int test7(pid_t child_pid)
	341	{
	342	int perf_fd;
	343	int ptrace_fd;
	344	int ret = 0;
	345
	346	/* Test:
	347	* if (new per thread event by perf)
	348	* if (existing thread event by ptrace on the same thread)
	349	* if (addr range overlaps)
	350	* fail;
	351	*/
	352	ptrace_fd = ptrace_open(child_pid, (__u64)ptrace_data1, sizeof(*ptrace_data1));
	353	if (ptrace_fd < 0)
	354	return -1;
	355
	356	perf_fd = perf_thread_event_open(child_pid, (__u64)perf_data1,
	357	sizeof(*perf_data1));
	358	if (perf_fd > 0 \|\| errno != ENOSPC)
	359	ret = -1;
	360
	361	ptrace(PPC_PTRACE_DELHWDEBUG, child_pid, 0, ptrace_fd);
	362	return ret;
	363	}
	364
	365	static int test8(pid_t child_pid)
	366	{
	367	int perf_fd;
	368	int ptrace_fd;
	369	int ret = 0;
	370
	371	/* Test:
	372	* if (new per thread event by perf)
	373	* if (existing thread event by ptrace on the same thread)
	374	* if (addr range does not overlaps)
	375	* allow;
	376	*/
	377	ptrace_fd = ptrace_open(child_pid, (__u64)ptrace_data2, sizeof(*ptrace_data2));
	378	if (ptrace_fd < 0)
	379	return -1;
	380
	381	perf_fd = perf_thread_event_open(child_pid, (__u64)perf_data2,
	382	sizeof(*perf_data2));
	383	if (perf_fd < 0) {
	384	ret = -1;
	385	goto ptrace_close;
	386	}
	387	close(perf_fd);
	388
	389	ptrace_close:
	390	ptrace(PPC_PTRACE_DELHWDEBUG, child_pid, 0, ptrace_fd);
	391	return ret;
	392	}
	393
	394	static int test9(pid_t child_pid)
	395	{
	396	int perf_fd;
	397	int ptrace_fd;
	398	int cpid;
	399	int ret = 0;
	400
	401	/* Test:
	402	* if (new per thread event by perf)
	403	* if (existing thread event by ptrace on the other thread)
	404	* allow;
	405	*/
	406	ptrace_fd = ptrace_open(child_pid, (__u64)ptrace_data1, sizeof(*ptrace_data1));
	407	if (ptrace_fd < 0)
	408	return -1;
	409
	410	cpid = fork();
	411	if (!cpid) {
	412	/* Temporary Child */
	413	pause();
	414	exit(EXIT_SUCCESS);
	415	}
	416
	417	perf_fd = perf_thread_event_open(cpid, (__u64)perf_data1, sizeof(*perf_data1));
	418	if (perf_fd < 0) {
	419	ret = -1;
	420	goto kill_child;
	421	}
	422	close(perf_fd);
	423
	424	kill_child:
	425	kill(cpid, SIGINT);
	426	ptrace(PPC_PTRACE_DELHWDEBUG, child_pid, 0, ptrace_fd);
	427	return ret;
	428	}
	429
	430	static int test10(pid_t child_pid)
	431	{
	432	int perf_fd;
	433	int ptrace_fd;
	434	int ret = 0;
	435
	436	/* Test:
	437	* if (new per cpu event by perf)
	438	* if (existing thread event by ptrace on the same thread)
	439	* if (addr range overlaps)
	440	* fail;
	441	*/
	442	ptrace_fd = ptrace_open(child_pid, (__u64)ptrace_data1, sizeof(*ptrace_data1));
	443	if (ptrace_fd < 0)
	444	return -1;
	445
	446	perf_fd = perf_cpu_event_open(0, (__u64)perf_data1, sizeof(*perf_data1));
	447	if (perf_fd > 0 \|\| errno != ENOSPC)
	448	ret = -1;
	449
	450	ptrace(PPC_PTRACE_DELHWDEBUG, child_pid, 0, ptrace_fd);
	451	return ret;
	452	}
	453
	454	static int test11(pid_t child_pid)
	455	{
	456	int perf_fd;
	457	int ptrace_fd;
	458	int ret = 0;
	459
	460	/* Test:
	461	* if (new per cpu event by perf)
	462	* if (existing thread event by ptrace on the same thread)
	463	* if (addr range does not overlap)
	464	* allow;
	465	*/
	466	ptrace_fd = ptrace_open(child_pid, (__u64)ptrace_data2, sizeof(*ptrace_data2));
	467	if (ptrace_fd < 0)
	468	return -1;
	469
	470	perf_fd = perf_cpu_event_open(0, (__u64)perf_data2, sizeof(*perf_data2));
	471	if (perf_fd < 0) {
	472	ret = -1;
	473	goto ptrace_close;
	474	}
	475	close(perf_fd);
	476
	477	ptrace_close:
	478	ptrace(PPC_PTRACE_DELHWDEBUG, child_pid, 0, ptrace_fd);
	479	return ret;
	480	}
	481
	482	static int test12(pid_t child_pid)
	483	{
	484	int perf_fd;
	485	int ptrace_fd;
	486	int ret = 0;
	487
	488	/* Test:
	489	* if (new per thread and per cpu event by perf)
	490	* if (existing thread event by ptrace on the same thread)
	491	* if (addr range overlaps)
	492	* fail;
	493	*/
	494	ptrace_fd = ptrace_open(child_pid, (__u64)ptrace_data1, sizeof(*ptrace_data1));
	495	if (ptrace_fd < 0)
	496	return -1;
	497
	498	perf_fd = perf_thread_cpu_event_open(child_pid, 0, (__u64)perf_data1, sizeof(*perf_data1));
	499	if (perf_fd > 0 \|\| errno != ENOSPC)
	500	ret = -1;
	501
	502	ptrace(PPC_PTRACE_DELHWDEBUG, child_pid, 0, ptrace_fd);
	503	return ret;
	504	}
	505
	506	static int test13(pid_t child_pid)
	507	{
	508	int perf_fd;
	509	int ptrace_fd;
	510	int ret = 0;
	511
	512	/* Test:
	513	* if (new per thread and per cpu event by perf)
	514	* if (existing thread event by ptrace on the same thread)
	515	* if (addr range does not overlap)
	516	* allow;
	517	*/
	518	ptrace_fd = ptrace_open(child_pid, (__u64)ptrace_data2, sizeof(*ptrace_data2));
	519	if (ptrace_fd < 0)
	520	return -1;
	521
	522	perf_fd = perf_thread_cpu_event_open(child_pid, 0, (__u64)perf_data2, sizeof(*perf_data2));
	523	if (perf_fd < 0) {
	524	ret = -1;
	525	goto ptrace_close;
	526	}
	527	close(perf_fd);
	528
	529	ptrace_close:
	530	ptrace(PPC_PTRACE_DELHWDEBUG, child_pid, 0, ptrace_fd);
	531	return ret;
	532	}
	533
	534	static int test14(pid_t child_pid)
	535	{
	536	int perf_fd;
	537	int ptrace_fd;
	538	int cpid;
	539	int ret = 0;
	540
	541	/* Test:
	542	* if (new per thread and per cpu event by perf)
	543	* if (existing thread event by ptrace on the other thread)
	544	* allow;
	545	*/
	546	ptrace_fd = ptrace_open(child_pid, (__u64)ptrace_data1, sizeof(*ptrace_data1));
	547	if (ptrace_fd < 0)
	548	return -1;
	549
	550	cpid = fork();
	551	if (!cpid) {
	552	/* Temporary Child */
	553	pause();
	554	exit(EXIT_SUCCESS);
	555	}
	556
	557	perf_fd = perf_thread_cpu_event_open(cpid, 0, (__u64)perf_data1,
	558	sizeof(*perf_data1));
	559	if (perf_fd < 0) {
	560	ret = -1;
	561	goto kill_child;
	562	}
	563	close(perf_fd);
	564
	565	kill_child:
	566	kill(cpid, SIGINT);
	567	ptrace(PPC_PTRACE_DELHWDEBUG, child_pid, 0, ptrace_fd);
	568	return ret;
	569	}
	570
	571	static int do_test(const char msg, int (fun)(pid_t arg), pid_t arg)
	572	{
	573	int ret;
	574
	575	ret = fun(arg);
	576	if (ret)
	577	printf("%s: Error\n", msg);
	578	else
	579	printf("%s: Ok\n", msg);
	580	return ret;
	581	}
	582
	583	char *desc[14] = {
	584	"perf cpu event -> ptrace thread event (Overlapping)",
	585	"perf cpu event -> ptrace thread event (Non-overlapping)",
	586	"perf thread event -> ptrace same thread event (Overlapping)",
	587	"perf thread event -> ptrace same thread event (Non-overlapping)",
	588	"perf thread event -> ptrace other thread event",
	589	"ptrace thread event -> perf kernel event",
	590	"ptrace thread event -> perf same thread event (Overlapping)",
	591	"ptrace thread event -> perf same thread event (Non-overlapping)",
	592	"ptrace thread event -> perf other thread event",
	593	"ptrace thread event -> perf cpu event (Overlapping)",
	594	"ptrace thread event -> perf cpu event (Non-overlapping)",
	595	"ptrace thread event -> perf same thread & cpu event (Overlapping)",
	596	"ptrace thread event -> perf same thread & cpu event (Non-overlapping)",
	597	"ptrace thread event -> perf other thread & cpu event",
	598	};
	599
	600	static int test(pid_t child_pid)
	601	{
	602	int ret = TEST_PASS;
	603
	604	ret \|= do_test(desc[0], test1, child_pid);
	605	ret \|= do_test(desc[1], test2, child_pid);
	606	ret \|= do_test(desc[2], test3, child_pid);
	607	ret \|= do_test(desc[3], test4, child_pid);
	608	ret \|= do_test(desc[4], test5, child_pid);
	609	ret \|= do_test(desc[5], test6, child_pid);
	610	ret \|= do_test(desc[6], test7, child_pid);
	611	ret \|= do_test(desc[7], test8, child_pid);
	612	ret \|= do_test(desc[8], test9, child_pid);
	613	ret \|= do_test(desc[9], test10, child_pid);
	614	ret \|= do_test(desc[10], test11, child_pid);
	615	ret \|= do_test(desc[11], test12, child_pid);
	616	ret \|= do_test(desc[12], test13, child_pid);
	617	ret \|= do_test(desc[13], test14, child_pid);
	618
	619	return ret;
	620	}
	621
	622	static void get_dbginfo(pid_t child_pid, struct ppc_debug_info *dbginfo)
	623	{
	624	if (ptrace(PPC_PTRACE_GETHWDBGINFO, child_pid, NULL, dbginfo)) {
	625	perror("Can't get breakpoint info");
	626	exit(-1);
	627	}
	628	}
	629
	630	static int ptrace_perf_hwbreak(void)
	631	{
	632	int ret;
	633	pid_t child_pid;
	634	struct ppc_debug_info dbginfo;
	635
	636	child_pid = fork();
	637	if (!child_pid)
	638	return child();
	639
	640	/* parent */
	641	wait(NULL); /* <-- child (SIGUSR1) */
	642
	643	get_dbginfo(child_pid, &dbginfo);
	644	SKIP_IF(dbginfo.num_data_bps <= 1);
	645
	646	ret = perf_cpu_event_open(0, (__u64)perf_data1, sizeof(*perf_data1));
	647	SKIP_IF(ret < 0);
	648	close(ret);
	649
	650	ret = test(child_pid);
	651
	652	ptrace(PTRACE_CONT, child_pid, NULL, 0);
	653	return ret;
	654	}
	655
	656	int main(int argc, char *argv[])
	657	{
	658	return test_harness(ptrace_perf_hwbreak, "ptrace-perf-hwbreak");
	659	}

tools/testing/selftests/powerpc/security/Makefile+2−1

 # SPDX-License-Identifier: GPL-2.0+
-TEST_GEN_PROGS := rfi_flush entry_flush spectre_v2
+TEST_GEN_PROGS := rfi_flush entry_flush uaccess_flush spectre_v2
 top_srcdir = ../../../../..
 CFLAGS += -I../../../../../usr/include
 $(OUTPUT)/spectre_v2: ../pmu/event.c branch_loops.S
 $(OUTPUT)/rfi_flush: flush_utils.c
 $(OUTPUT)/entry_flush: flush_utils.c
+$(OUTPUT)/uaccess_flush: flush_utils.c

tools/testing/selftests/powerpc/security/entry_flush.c+1−1

@@ -53,7 +53,7 @@ int entry_flush_test(void) · 52 unmodified lines
53		53
54	entry_flush = entry_flush_orig;	54	entry_flush = entry_flush_orig;
55		55
56	fd = perf_event_open_counter(PERF_TYPE_RAW, /* L1d miss */ 0x400f0, -1);	56	fd = perf_event_open_counter(PERF_TYPE_HW_CACHE, PERF_L1D_READ_MISS_CONFIG, -1);
57	FAIL_IF(fd < 0);	57	FAIL_IF(fd < 0);
58		58
59	p = (char *)memalign(zero_size, CACHELINE_SIZE);	59	p = (char *)memalign(zero_size, CACHELINE_SIZE);

tools/testing/selftests/powerpc/security/flush_utils.c+13−0

@@ -13,6 +13,7 @@ · 12 unmodified lines
13	#include <stdlib.h>	13	#include <stdlib.h>
14	#include <string.h>	14	#include <string.h>
15	#include <stdio.h>	15	#include <stdio.h>
		16	#include <sys/utsname.h>
16	#include "utils.h"	17	#include "utils.h"
17	#include "flush_utils.h"	18	#include "flush_utils.h"
18		19
@@ -35,6 +36,18 @@ void syscall_loop(char *p, unsigned long iterations, · 16 unmodified lines
35	}	36	}
36	}	37	}
37		38
		39	void syscall_loop_uaccess(char *p, unsigned long iterations,
		40	unsigned long zero_size)
		41	{
		42	struct utsname utsname;
		43
		44	for (unsigned long i = 0; i < iterations; i++) {
		45	for (unsigned long j = 0; j < zero_size; j += CACHELINE_SIZE)
		46	load(p + j);
		47	uname(&utsname);
		48	}
		49	}
		50
38	static void sigill_handler(int signr, siginfo_t info, void unused)	51	static void sigill_handler(int signr, siginfo_t info, void unused)
39	{	52	{
40	static int warned;	53	static int warned;

tools/testing/selftests/powerpc/security/flush_utils.h+7−0

 #define CACHELINE_SIZE 128
+#define PERF_L1D_READ_MISS_CONFIG	((PERF_COUNT_HW_CACHE_L1D) | 		\
+					(PERF_COUNT_HW_CACHE_OP_READ << 8) |	\
+					(PERF_COUNT_HW_CACHE_RESULT_MISS << 16))
 void syscall_loop(char *p, unsigned long iterations,
 		  unsigned long zero_size);
+void syscall_loop_uaccess(char *p, unsigned long iterations,
+			  unsigned long zero_size);
 void set_dscr(unsigned long val);
 #endif /* _SELFTESTS_POWERPC_SECURITY_FLUSH_UTILS_H */

tools/testing/selftests/powerpc/security/rfi_flush.c+1−1

@@ -54,7 +54,7 @@ int rfi_flush_test(void) · 53 unmodified lines
54		54
55	rfi_flush = rfi_flush_orig;	55	rfi_flush = rfi_flush_orig;
56		56
57	fd = perf_event_open_counter(PERF_TYPE_RAW, /* L1d miss */ 0x400f0, -1);	57	fd = perf_event_open_counter(PERF_TYPE_HW_CACHE, PERF_L1D_READ_MISS_CONFIG, -1);
58	FAIL_IF(fd < 0);	58	FAIL_IF(fd < 0);
59		59
60	p = (char *)memalign(zero_size, CACHELINE_SIZE);	60	p = (char *)memalign(zero_size, CACHELINE_SIZE);

tools/testing/selftests/powerpc/security/uaccess_flush.c+158−0

@@ -0,0 +1,158 @@
	1	// SPDX-License-Identifier: GPL-2.0+
	2
	3	/*
	4	* Copyright 2018 IBM Corporation.
	5	* Copyright 2020 Canonical Ltd.
	6	*/
	7
	8	#define __SANE_USERSPACE_TYPES__
	9
	10	#include <sys/types.h>
	11	#include <stdint.h>
	12	#include <malloc.h>
	13	#include <unistd.h>
	14	#include <signal.h>
	15	#include <stdlib.h>
	16	#include <string.h>
	17	#include <stdio.h>
	18	#include "utils.h"
	19	#include "flush_utils.h"
	20
	21	int uaccess_flush_test(void)
	22	{
	23	char *p;
	24	int repetitions = 10;
	25	int fd, passes = 0, iter, rc = 0;
	26	struct perf_event_read v;
	27	__u64 l1d_misses_total = 0;
	28	unsigned long iterations = 100000, zero_size = 24 * 1024;
	29	unsigned long l1d_misses_expected;
	30	int rfi_flush_orig;
	31	int entry_flush_orig;
	32	int uaccess_flush, uaccess_flush_orig;
	33
	34	SKIP_IF(geteuid() != 0);
	35
	36	// The PMU event we use only works on Power7 or later
	37	SKIP_IF(!have_hwcap(PPC_FEATURE_ARCH_2_06));
	38
	39	if (read_debugfs_file("powerpc/rfi_flush", &rfi_flush_orig) < 0) {
	40	perror("Unable to read powerpc/rfi_flush debugfs file");
	41	SKIP_IF(1);
	42	}
	43
	44	if (read_debugfs_file("powerpc/entry_flush", &entry_flush_orig) < 0) {
	45	perror("Unable to read powerpc/entry_flush debugfs file");
	46	SKIP_IF(1);
	47	}
	48
	49	if (read_debugfs_file("powerpc/uaccess_flush", &uaccess_flush_orig) < 0) {
	50	perror("Unable to read powerpc/entry_flush debugfs file");
	51	SKIP_IF(1);
	52	}
	53
	54	if (rfi_flush_orig != 0) {
	55	if (write_debugfs_file("powerpc/rfi_flush", 0) < 0) {
	56	perror("error writing to powerpc/rfi_flush debugfs file");
	57	FAIL_IF(1);
	58	}
	59	}
	60
	61	if (entry_flush_orig != 0) {
	62	if (write_debugfs_file("powerpc/entry_flush", 0) < 0) {
	63	perror("error writing to powerpc/entry_flush debugfs file");
	64	FAIL_IF(1);
	65	}
	66	}
	67
	68	uaccess_flush = uaccess_flush_orig;
	69
	70	fd = perf_event_open_counter(PERF_TYPE_HW_CACHE, PERF_L1D_READ_MISS_CONFIG, -1);
	71	FAIL_IF(fd < 0);
	72
	73	p = (char *)memalign(zero_size, CACHELINE_SIZE);
	74
	75	FAIL_IF(perf_event_enable(fd));
	76
	77	// disable L1 prefetching
	78	set_dscr(1);
	79
	80	iter = repetitions;
	81
	82	/*
	83	* We expect to see l1d miss for each cacheline access when entry_flush
	84	* is set. Allow a small variation on this.
	85	*/
	86	l1d_misses_expected = iterations * (zero_size / CACHELINE_SIZE - 2);
	87
	88	again:
	89	FAIL_IF(perf_event_reset(fd));
	90
	91	syscall_loop_uaccess(p, iterations, zero_size);
	92
	93	FAIL_IF(read(fd, &v, sizeof(v)) != sizeof(v));
	94
	95	if (uaccess_flush && v.l1d_misses >= l1d_misses_expected)
	96	passes++;
	97	else if (!uaccess_flush && v.l1d_misses < (l1d_misses_expected / 2))
	98	passes++;
	99
	100	l1d_misses_total += v.l1d_misses;
	101
	102	while (--iter)
	103	goto again;
	104
	105	if (passes < repetitions) {
	106	printf("FAIL (L1D misses with uaccess_flush=%d: %llu %c %lu) [%d/%d failures]\n",
	107	uaccess_flush, l1d_misses_total, uaccess_flush ? '<' : '>',
	108	uaccess_flush ? repetitions * l1d_misses_expected :
	109	repetitions * l1d_misses_expected / 2,
	110	repetitions - passes, repetitions);
	111	rc = 1;
	112	} else {
	113	printf("PASS (L1D misses with uaccess_flush=%d: %llu %c %lu) [%d/%d pass]\n",
	114	uaccess_flush, l1d_misses_total, uaccess_flush ? '>' : '<',
	115	uaccess_flush ? repetitions * l1d_misses_expected :
	116	repetitions * l1d_misses_expected / 2,
	117	passes, repetitions);
	118	}
	119
	120	if (uaccess_flush == uaccess_flush_orig) {
	121	uaccess_flush = !uaccess_flush_orig;
	122	if (write_debugfs_file("powerpc/uaccess_flush", uaccess_flush) < 0) {
	123	perror("error writing to powerpc/uaccess_flush debugfs file");
	124	return 1;
	125	}
	126	iter = repetitions;
	127	l1d_misses_total = 0;
	128	passes = 0;
	129	goto again;
	130	}
	131
	132	perf_event_disable(fd);
	133	close(fd);
	134
	135	set_dscr(0);
	136
	137	if (write_debugfs_file("powerpc/rfi_flush", rfi_flush_orig) < 0) {
	138	perror("unable to restore original value of powerpc/rfi_flush debugfs file");
	139	return 1;
	140	}
	141
	142	if (write_debugfs_file("powerpc/entry_flush", entry_flush_orig) < 0) {
	143	perror("unable to restore original value of powerpc/entry_flush debugfs file");
	144	return 1;
	145	}
	146
	147	if (write_debugfs_file("powerpc/uaccess_flush", uaccess_flush_orig) < 0) {
	148	perror("unable to restore original value of powerpc/uaccess_flush debugfs file");
	149	return 1;
	150	}
	151
	152	return rc;
	153	}
	154
	155	int main(int argc, char *argv[])
	156	{
	157	return test_harness(uaccess_flush_test, "uaccess_flush_test");
	158	}

tools/testing/selftests/powerpc/tm/tm-trap.c+2−2

@@ -66,7 +66,7 @@ void trap_signal_handler(int signo, siginfo_t si, void uc) · 65 unmodified lines
66	/* Get thread endianness: extract bit LE from MSR */	66	/* Get thread endianness: extract bit LE from MSR */
67	thread_endianness = MSR_LE & ucp->uc_mcontext.gp_regs[PT_MSR];	67	thread_endianness = MSR_LE & ucp->uc_mcontext.gp_regs[PT_MSR];
68		68
69	/***	69	/*
70	* Little-Endian Machine	70	* Little-Endian Machine
71	*/	71	*/
72		72
@@ -126,7 +126,7 @@ void trap_signal_handler(int signo, siginfo_t si, void uc) · 53 unmodified lines
126	}	126	}
127	}	127	}
128		128
129	/***	129	/*
130	* Big-Endian Machine	130	* Big-Endian Machine
131	*/	131	*/
132		132

tools/testing/selftests/timens/gettime_perf.c+8−0

 	if (!vdso)
 		vdso = dlopen("linux-gate.so.1",
 			      RTLD_LAZY | RTLD_LOCAL | RTLD_NOLOAD);
+	if (!vdso)
+		vdso = dlopen("linux-vdso32.so.1",
+			      RTLD_LAZY | RTLD_LOCAL | RTLD_NOLOAD);
+	if (!vdso)
+		vdso = dlopen("linux-vdso64.so.1",
+			      RTLD_LAZY | RTLD_LOCAL | RTLD_NOLOAD);
 	if (!vdso) {
 		pr_err("[WARN]\tfailed to find vDSO\n");
 		return;
 	}
 	vdso_clock_gettime = (vgettime_t)dlsym(vdso, "__vdso_clock_gettime");
+	if (!vdso_clock_gettime)
+		vdso_clock_gettime = (vgettime_t)dlsym(vdso, "__kernel_clock_gettime");
 	if (!vdso_clock_gettime)
 		pr_err("Warning: failed to find clock_gettime in vDSO\n");

Merge tag 'powerpc-5.13-1' of git://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code