Merge tag 'powerpc-5.13-1' of git://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux

Pull powerpc updates from Michael Ellerman: - Enable KFENCE for 32-bit. - Implement EBPF for 32-bit. - Convert 32-bit to do interrupt entry/exit in C. - Convert 64-bit BookE to do interrupt entry/exit in C. - Changes to our signal handling code to use user_access_begin/end() more extensively. - Add support for time namespaces (CONFIG_TIME_NS) - A series of fixes that allow us to reenable STRICT_KERNEL_RWX. - Other smaller features, fixes & cleanups. Thanks to Alexey Kardashevskiy, Andreas Schwab, Andrew Donnellan, Aneesh Kumar K.V, Athira Rajeev, Bhaskar Chowdhury, Bixuan Cui, Cédric Le Goater, Chen Huang, Chris Packham, Christophe Leroy, Christopher M. Riedl, Colin Ian King, Dan Carpenter, Daniel Axtens, Daniel Henrique Barboza, David Gibson, Davidlohr Bueso, Denis Efremov, dingsenjie, Dmitry Safonov, Dominic DeMarco, Fabiano Rosas, Ganesh Goudar, Geert Uytterhoeven, Geetika Moolchandani, Greg Kurz, Guenter Roeck, Haren Myneni, He Ying, Jiapeng Chong, Jordan Niethe, Laurent Dufour, Lee Jones, Leonardo Bras, Li Huafei, Madhavan Srinivasan, Mahesh Salgaonkar, Masahiro Yamada, Nathan Chancellor, Nathan Lynch, Nicholas Piggin, Oliver O'Halloran, Paul Menzel, Pu Lehui, Randy Dunlap, Ravi Bangoria, Rosen Penev, Russell Currey, Santosh Sivaraj, Sebastian Andrzej Siewior, Segher Boessenkool, Shivaprasad G Bhat, Srikar Dronamraju, Stephen Rothwell, Thadeu Lima de Souza Cascardo, Thomas Gleixner, Tony Ambardar, Tyrel Datwyler, Vaibhav Jain, Vincenzo Frascino, Xiongwei Song, Yang Li, Yu Kuai, and Zhang Yunkai. * tag 'powerpc-5.13-1' of git://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux: (302 commits) powerpc/signal32: Fix erroneous SIGSEGV on RT signal return powerpc: Avoid clang uninitialized warning in __get_user_size_allowed powerpc/papr_scm: Mark nvdimm as unarmed if needed during probe powerpc/kvm: Fix build error when PPC_MEM_KEYS/PPC_PSERIES=n powerpc/kasan: Fix shadow start address with modules powerpc/kernel/iommu: Use largepool as a last resort when !largealloc powerpc/kernel/iommu: Align size for IOMMU_PAGE_SIZE() to save TCEs powerpc/44x: fix spelling mistake in Kconfig "varients" -> "variants" powerpc/iommu: Annotate nested lock for lockdep powerpc/iommu: Do not immediately panic when failed IOMMU table allocation powerpc/iommu: Allocate it_map by vmalloc selftests/powerpc: remove unneeded semicolon powerpc/64s: remove unneeded semicolon powerpc/eeh: remove unneeded semicolon powerpc/selftests: Add selftest to test concurrent perf/ptrace events powerpc/selftests/perf-hwbreak: Add testcases for 2nd DAWR powerpc/selftests/perf-hwbreak: Coalesce event creation code powerpc/selftests/ptrace-hwbreak: Add testcases for 2nd DAWR powerpc/configs: Add IBMVNIC to some 64-bit configs selftests/powerpc: Add uaccess flush test ...

Linus Torvalds committed 5 years ago

Commit c70a4be2 parents 437d1a5 5256426234 files · +6868 −5596Browse files

Unified · Split

234 files

Documentation/admin-guide/sysctl/net.rst +1 −1
Documentation/features/debug/debug-vm-pgtable/arch-support.txt +1 −1
Documentation/powerpc/papr_hcalls.rst +14 −0
Documentation/powerpc/vas-api.rst +2 −2
arch/arm64/include/asm/vdso/compat_gettimeofday.h +2 −1
arch/arm64/include/asm/vdso/gettimeofday.h +1 −1
arch/powerpc/Kconfig +18 −14
arch/powerpc/Kconfig.debug +1 −0
arch/powerpc/Makefile +4 −7
arch/powerpc/configs/ppc64_defconfig +2 −0
arch/powerpc/configs/pseries_defconfig +2 −0
arch/powerpc/include/asm/Kbuild +0 −1
arch/powerpc/include/asm/asm-prototypes.h +0 −2
arch/powerpc/include/asm/barrier.h +0 −16
arch/powerpc/include/asm/book3s/32/kup.h +46 −80
arch/powerpc/include/asm/book3s/32/pgtable.h +0 −2
arch/powerpc/include/asm/book3s/32/tlbflush.h +1 −1
arch/powerpc/include/asm/book3s/64/kup.h +2 −22
arch/powerpc/include/asm/book3s/64/mmu-hash.h +0 −1
arch/powerpc/include/asm/book3s/64/pgtable.h +4 −1
arch/powerpc/include/asm/book3s/64/radix.h +4 −2
arch/powerpc/include/asm/bug.h +1 −4
arch/powerpc/include/asm/cacheflush.h +13 −2
arch/powerpc/include/asm/cpm2.h +1 −1
arch/powerpc/include/asm/fixmap.h +9 −0
arch/powerpc/include/asm/futex.h +5 −7
arch/powerpc/include/asm/hvcall.h +3 −1
arch/powerpc/include/asm/hvconsole.h +3 −0
arch/powerpc/include/asm/hydra.h +0 −2
arch/powerpc/include/asm/inst.h +44 −11
arch/powerpc/include/asm/interrupt.h +131 −42
arch/powerpc/include/asm/irq.h +0 −2
arch/powerpc/include/asm/jump_label.h +6 −15
arch/powerpc/include/asm/kasan.h +1 −1
arch/powerpc/include/asm/kfence.h +33 −0
arch/powerpc/include/asm/kup.h +17 −10
arch/powerpc/include/asm/kvm_book3s.h +2 −0
arch/powerpc/include/asm/kvm_ppc.h +1 −2
arch/powerpc/include/asm/mmu_context.h +1 −1
arch/powerpc/include/asm/nohash/32/kup-8xx.h +32 −24
arch/powerpc/include/asm/nohash/32/mmu-8xx.h +3 −0
arch/powerpc/include/asm/nohash/64/pgtable.h +4 −1
arch/powerpc/include/asm/opal.h +1 −1
arch/powerpc/include/asm/perf_event_server.h +7 −1
arch/powerpc/include/asm/pgtable.h +0 −2
arch/powerpc/include/asm/ppc-opcode.h +13 −0
arch/powerpc/include/asm/ppc_asm.h +0 −30
arch/powerpc/include/asm/processor.h +4 −5
arch/powerpc/include/asm/ptrace.h +14 −31
arch/powerpc/include/asm/qspinlock.h +7 −14
arch/powerpc/include/asm/reg.h +3 −3
arch/powerpc/include/asm/rtas.h +3 −3
arch/powerpc/include/asm/simple_spinlock.h +1 −5
arch/powerpc/include/asm/smp.h +6 −0
arch/powerpc/include/asm/spinlock.h +3 −0
arch/powerpc/include/asm/thread_info.h +6 −1
arch/powerpc/include/asm/topology.h +1 −1
arch/powerpc/include/asm/uaccess.h +171 −218
arch/powerpc/include/asm/unistd.h +1 −0
arch/powerpc/include/asm/vdso/gettimeofday.h +10 −0
arch/powerpc/include/asm/vdso_datapage.h +0 −2
arch/powerpc/include/asm/vio.h +1 −0
arch/powerpc/include/asm/xive.h +1 −0
arch/powerpc/include/uapi/asm/errno.h +1 −0
arch/powerpc/include/uapi/asm/posix_types.h +0 −5
arch/powerpc/kernel/align.c +38 −34
arch/powerpc/kernel/asm-offsets.c +0 −17
arch/powerpc/kernel/eeh.c +7 −8
arch/powerpc/kernel/entry_32.S +137 −659
arch/powerpc/kernel/entry_64.S +15 −25
arch/powerpc/kernel/exceptions-64e.S +31 −394
arch/powerpc/kernel/exceptions-64s.S +7 −53
arch/powerpc/kernel/fadump.c +9 −9
arch/powerpc/kernel/fpu.S +0 −2
arch/powerpc/kernel/head_32.h +79 −119
arch/powerpc/kernel/head_40x.S +147 −124
arch/powerpc/kernel/head_44x.S +4 −6
arch/powerpc/kernel/head_8xx.S +71 −85
arch/powerpc/kernel/head_book3s_32.S +99 −143
arch/powerpc/kernel/head_booke.h +134 −69
arch/powerpc/kernel/head_fsl_booke.S +38 −32
arch/powerpc/kernel/hw_breakpoint_constraints.c +1 −1
arch/powerpc/kernel/idle_6xx.S +2 −12
arch/powerpc/kernel/idle_book3s.S +4 −0
arch/powerpc/kernel/idle_e500.S +3 −11
arch/powerpc/kernel/interrupt.c +27 −40
arch/powerpc/kernel/iommu.c +24 −22
arch/powerpc/kernel/irq.c +47 −85
arch/powerpc/kernel/jump_label.c +2 −2
arch/powerpc/kernel/kgdb.c +1 −1
arch/powerpc/kernel/legacy_serial.c +29 −4
arch/powerpc/kernel/mce.c +3 −2
arch/powerpc/kernel/misc_32.S +0 −39
arch/powerpc/kernel/misc_64.S +0 −22
arch/powerpc/kernel/module.c +20 −3
arch/powerpc/kernel/optprobes.c +21 −3
arch/powerpc/kernel/optprobes_head.S +34 −31
arch/powerpc/kernel/process.c +8 −17
arch/powerpc/kernel/prom.c +16 −5
arch/powerpc/kernel/prom_init.c +1 −1
arch/powerpc/kernel/ptrace/ptrace-view.c +18 −33
arch/powerpc/kernel/ptrace/ptrace.c +0 −4
arch/powerpc/kernel/ptrace/ptrace32.c +0 −4
arch/powerpc/kernel/rtas-proc.c +11 −4
arch/powerpc/kernel/rtas.c +18 −16
arch/powerpc/kernel/security.c +262 −1
arch/powerpc/kernel/setup-common.c +1 −2
arch/powerpc/kernel/setup_32.c +1 −1
arch/powerpc/kernel/setup_64.c +16 −266
arch/powerpc/kernel/signal.h +55 −0
arch/powerpc/kernel/signal_32.c +143 −111
arch/powerpc/kernel/signal_64.c +189 −130
arch/powerpc/kernel/smp.c +55 −13
arch/powerpc/kernel/stacktrace.c +25 −83
arch/powerpc/kernel/syscalls.c +2 −10
arch/powerpc/kernel/syscalls/Makefile +9 −24
arch/powerpc/kernel/syscalls/syscallhdr.sh +0 −36
arch/powerpc/kernel/syscalls/syscalltbl.sh +0 −36
arch/powerpc/kernel/systbl.S +4 −1
arch/powerpc/kernel/trace/ftrace.c +11 −11
arch/powerpc/kernel/traps.c +21 −19
arch/powerpc/kernel/uprobes.c +7 −0
arch/powerpc/kernel/vdso.c +123 −15
arch/powerpc/kernel/vdso32/vdso32.lds.S +1 −1
arch/powerpc/kernel/vdso64/vdso64.lds.S +1 −1
arch/powerpc/kernel/vector.S +0 −2
arch/powerpc/kexec/crash.c +2 −1
arch/powerpc/kvm/book3s_64_mmu_host.c +2 −0
arch/powerpc/kvm/book3s_hv.c +65 −20
arch/powerpc/kvm/book3s_hv_builtin.c +3 −0
arch/powerpc/kvm/book3s_hv_nested.c +30 −7
arch/powerpc/kvm/book3s_hv_rm_mmu.c +1 −2
arch/powerpc/lib/Makefile +1 −1
arch/powerpc/lib/checksum_wrappers.c +4 −11
arch/powerpc/lib/code-patching.c +9 −4
arch/powerpc/lib/inst.c +0 −73
arch/powerpc/lib/sstep.c +0 −13
arch/powerpc/math-emu/math.c +2 −2
arch/powerpc/mm/Makefile +2 −1
arch/powerpc/mm/book3s32/Makefile +1 −0
arch/powerpc/mm/book3s32/hash_low.S +0 −14
arch/powerpc/mm/book3s32/kuep.c +40 −0
arch/powerpc/mm/book3s32/mmu.c +1 −8
arch/powerpc/mm/book3s64/hash_pgtable.c +121 −9
arch/powerpc/mm/book3s64/hash_utils.c +22 −13
arch/powerpc/mm/book3s64/mmu_context.c +1 −1
arch/powerpc/mm/book3s64/pkeys.c +5 −15
arch/powerpc/mm/book3s64/radix_pgtable.c +5 −5
arch/powerpc/mm/cacheflush.c +234 −0
arch/powerpc/mm/fault.c +23 −37
arch/powerpc/mm/init_32.c +3 −0
arch/powerpc/mm/maccess.c +21 −0
arch/powerpc/mm/mem.c +1 −282
arch/powerpc/mm/mmu_context.c +13 −11
arch/powerpc/mm/mmu_decl.h +5 −0
arch/powerpc/mm/nohash/8xx.c +2 −2
arch/powerpc/net/Makefile +1 −5
arch/powerpc/net/bpf_jit.h +64 −0
arch/powerpc/net/bpf_jit32.h +0 −139
arch/powerpc/net/bpf_jit64.h +1 −20
arch/powerpc/net/bpf_jit_asm.S +0 −226
arch/powerpc/net/bpf_jit_comp.c +185 −597
arch/powerpc/net/bpf_jit_comp32.c +1100 −0
arch/powerpc/net/bpf_jit_comp64.c +15 −280
arch/powerpc/perf/core-book3s.c +16 −4
arch/powerpc/perf/hv-24x7.c +5 −5
arch/powerpc/perf/isa207-common.c +104 −11
arch/powerpc/perf/isa207-common.h +8 −1
arch/powerpc/perf/power10-events-list.h +2 −2
arch/powerpc/perf/power10-pmu.c +13 −0
arch/powerpc/perf/power9-pmu.c +13 −0
arch/powerpc/platforms/44x/Kconfig +1 −1
arch/powerpc/platforms/52xx/lite5200_sleep.S +1 −1
arch/powerpc/platforms/Kconfig.cputype +3 −2
arch/powerpc/platforms/cell/iommu.c +2 −1
arch/powerpc/platforms/cell/spu_callbacks.c +1 −1
arch/powerpc/platforms/chrp/pci.c +1 −2
arch/powerpc/platforms/embedded6xx/Kconfig +0 −5
arch/powerpc/platforms/maple/pci.c +1 −1
arch/powerpc/platforms/pasemi/iommu.c +3 −1
arch/powerpc/platforms/powernv/memtrace.c +17 −1
arch/powerpc/platforms/powernv/opal-core.c +1 −1
arch/powerpc/platforms/powernv/opal-prd.c +1 −4
arch/powerpc/platforms/powernv/pci-ioda.c +8 −7
arch/powerpc/platforms/powernv/setup.c +1 −1
arch/powerpc/platforms/pseries/dlpar.c +14 −0
arch/powerpc/platforms/pseries/hotplug-cpu.c +23 −4
arch/powerpc/platforms/pseries/hvCall_inst.c +1 −1
arch/powerpc/platforms/pseries/iommu.c +38 −11
arch/powerpc/platforms/pseries/lpar.c +4 −2
arch/powerpc/platforms/pseries/lparcfg.c +2 −0
arch/powerpc/platforms/pseries/papr_scm.c +48 −0
arch/powerpc/platforms/pseries/pci_dlpar.c +3 −1
arch/powerpc/platforms/pseries/pmem.c +1 −1
arch/powerpc/platforms/pseries/pseries.h +2 −3
arch/powerpc/platforms/pseries/ras.c +1 −1
arch/powerpc/platforms/pseries/rtas-fadump.c +1 −1
arch/powerpc/platforms/pseries/setup.c +8 −0
arch/powerpc/platforms/pseries/vio.c +20 −0
arch/powerpc/purgatory/trampoline_64.S +0 −1
arch/powerpc/sysdev/dart_iommu.c +2 −1
arch/powerpc/sysdev/fsl_pci.c +1 −1
arch/powerpc/sysdev/xive/common.c +138 −75
arch/powerpc/sysdev/xive/native.c +6 −0
arch/powerpc/sysdev/xive/spapr.c +1 −1
arch/powerpc/sysdev/xive/xive-internal.h +1 −2
arch/powerpc/xmon/xmon.c +24 −52
arch/s390/include/asm/vdso/gettimeofday.h +2 −1
arch/x86/include/asm/vdso/gettimeofday.h +2 −1
drivers/i2c/busses/Kconfig +1 −1
drivers/macintosh/via-pmu.c +2 −2
drivers/macintosh/windfarm_core.c +1 −1
drivers/macintosh/windfarm_pm121.c +1 −1
drivers/macintosh/windfarm_smu_controls.c +1 −1
include/linux/compat.h +35 −0
include/linux/uaccess.h +1 −0
lib/vdso/gettimeofday.c +17 −14
tools/testing/selftests/powerpc/alignment/alignment_handler.c +1 −10
tools/testing/selftests/powerpc/mm/Makefile +1 −0
tools/testing/selftests/powerpc/mm/stress_code_patching.sh +49 −0
tools/testing/selftests/powerpc/nx-gzip/gzfht_test.c +1 −1
tools/testing/selftests/powerpc/ptrace/.gitignore +1 −0
tools/testing/selftests/powerpc/ptrace/Makefile +1 −1
tools/testing/selftests/powerpc/ptrace/perf-hwbreak.c +594 −41
tools/testing/selftests/powerpc/ptrace/ptrace-hwbreak.c +79 −0
tools/testing/selftests/powerpc/ptrace/ptrace-perf-hwbreak.c +659 −0
tools/testing/selftests/powerpc/security/Makefile +2 −1
tools/testing/selftests/powerpc/security/entry_flush.c +1 −1
tools/testing/selftests/powerpc/security/flush_utils.c +13 −0
tools/testing/selftests/powerpc/security/flush_utils.h +7 −0
tools/testing/selftests/powerpc/security/rfi_flush.c +1 −1
tools/testing/selftests/powerpc/security/uaccess_flush.c +158 −0
tools/testing/selftests/powerpc/tm/tm-trap.c +2 −2
tools/testing/selftests/timens/gettime_perf.c +8 −0

Documentation/admin-guide/sysctl/net.rst+1−1

@@ -64,6 +64,7 @@ two flavors of JITs, the newer eBPF JIT currently supported on: · 63 unmodified lines
64	64	- arm64
65	65	- arm32
66	66	- ppc64
	67	+ - ppc32
67	68	- sparc64
68	69	- mips64
69	70	- s390x
@@ -73,7 +74,6 @@ two flavors of JITs, the newer eBPF JIT currently supported on: · 3 unmodified lines
73	74	And the older cBPF JIT supported on the following archs:
74	75
75	76	- mips
76		− - ppc
77	77	- sparc
78	78
79	79	eBPF JITs are a superset of cBPF JITs, meaning the kernel will

Documentation/features/debug/debug-vm-pgtable/arch-support.txt+1−1

@@ -21,7 +21,7 @@ · 20 unmodified lines
21	21	\| nios2: \| TODO \|
22	22	\| openrisc: \| TODO \|
23	23	\| parisc: \| TODO \|
24		− \| powerpc: \| TODO \|
	24	+ \| powerpc: \| ok \|
25	25	\| riscv: \| ok \|
26	26	\| s390: \| ok \|
27	27	\| sh: \| TODO \|

Documentation/powerpc/papr_hcalls.rst+14−0

@@ -275,6 +275,20 @@ Health Bitmap Flags: · 274 unmodified lines
275	275	Given a DRC Index collect the performance statistics for NVDIMM and copy them
276	276	to the resultBuffer.
277	277
	278	+H_SCM_FLUSH
	279	+
	280	+\| Input: drcIndex, continue-token
	281	+\| Out: continue-token
	282	+\| Return Value: H_SUCCESS, H_Parameter, H_P2, H_BUSY
	283	+
	284	+Given a DRC Index Flush the data to backend NVDIMM device.
	285	+
	286	+The hcall returns H_BUSY when the flush takes longer time and the hcall needs
	287	+to be issued multiple times in order to be completely serviced. The
	288	+continue-token from the output to be passed in the argument list of
	289	+subsequent hcalls to the hypervisor until the hcall is completely serviced
	290	+at which point H_SUCCESS or other error is returned by the hypervisor.
	291	+
278	292	References
279	293	==========
280	294	.. [1] "Power Architecture Platform Reference"

Documentation/powerpc/vas-api.rst+2−2

@@ -254,7 +254,7 @@ using this window. the signal will be issued to the thread group leader · 253 unmodified lines
254	254	signals.
255	255
256	256	NX-GZIP User's Manual:
257		−https://github.com/libnxz/power-gzip/blob/master/power_nx_gzip_um.pdf
	257	+https://github.com/libnxz/power-gzip/blob/master/doc/power_nx_gzip_um.pdf
258	258
259	259	Simple example
260	260	==============
@@ -301,5 +301,5 @@ Simple example · 40 unmodified lines
301	301	close(fd) or window can be closed upon process exit
302	302	}
303	303
304		− Refer https://github.com/abalib/power-gzip for tests or more
	304	+ Refer https://github.com/libnxz/power-gzip for tests or more
305	305	use cases.

arch/arm64/include/asm/vdso/compat_gettimeofday.h+2−1

@@ -155,7 +155,8 @@ static __always_inline const struct vdso_data *__arch_get_vdso_data(void) · 154 unmodified lines
155	155	}
156	156
157	157	#ifdef CONFIG_TIME_NS
158		−static __always_inline const struct vdso_data *__arch_get_timens_vdso_data(void)
	158	+static __always_inline
	159	+const struct vdso_data __arch_get_timens_vdso_data(const struct vdso_data vd)
159	160	{
160	161	const struct vdso_data *ret;
161	162

arch/arm64/include/asm/vdso/gettimeofday.h+1−1

@@ -96,7 +96,7 @@ const struct vdso_data *__arch_get_vdso_data(void) · 95 unmodified lines
96	96
97	97	#ifdef CONFIG_TIME_NS
98	98	static __always_inline
99		−const struct vdso_data *__arch_get_timens_vdso_data(void)
	99	+const struct vdso_data __arch_get_timens_vdso_data(const struct vdso_data vd)
100	100	{
101	101	return _timens_data;
102	102	}

arch/powerpc/Kconfig+18−14

@@ -119,6 +119,7 @@ config PPC · 118 unmodified lines
119	119	#
120	120	select ARCH_32BIT_OFF_T if PPC32
121	121	select ARCH_HAS_DEBUG_VIRTUAL
	122	+ select ARCH_HAS_DEBUG_VM_PGTABLE
122	123	select ARCH_HAS_DEVMEM_IS_ALLOWED
123	124	select ARCH_HAS_ELF_RANDOMIZE
124	125	select ARCH_HAS_FORTIFY_SOURCE
@@ -135,7 +136,7 @@ config PPC · 10 unmodified lines
135	136	select ARCH_HAS_MEMBARRIER_CALLBACKS
136	137	select ARCH_HAS_MEMBARRIER_SYNC_CORE
137	138	select ARCH_HAS_SCALED_CPUTIME if VIRT_CPU_ACCOUNTING_NATIVE && PPC_BOOK3S_64
138		− select ARCH_HAS_STRICT_KERNEL_RWX if (PPC32 && !HIBERNATION)
	139	+ select ARCH_HAS_STRICT_KERNEL_RWX if ((PPC_BOOK3S_64 \|\| PPC32) && !HIBERNATION)
139	140	select ARCH_HAS_TICK_BROADCAST if GENERIC_CLOCKEVENTS_BROADCAST
140	141	select ARCH_HAS_UACCESS_FLUSHCACHE
141	142	select ARCH_HAS_COPY_MC if PPC64
@@ -145,6 +146,7 @@ config PPC · 3 unmodified lines
145	146	select ARCH_MIGHT_HAVE_PC_PARPORT
146	147	select ARCH_MIGHT_HAVE_PC_SERIO
147	148	select ARCH_OPTIONAL_KERNEL_RWX if ARCH_HAS_STRICT_KERNEL_RWX
	149	+ select ARCH_STACKWALK
148	150	select ARCH_SUPPORTS_ATOMIC_RMW
149	151	select ARCH_SUPPORTS_DEBUG_PAGEALLOC if PPC32 \|\| PPC_BOOK3S_64
150	152	select ARCH_USE_BUILTIN_BSWAP
@@ -171,6 +173,7 @@ config PPC · 20 unmodified lines
171	173	select GENERIC_CPU_AUTOPROBE
172	174	select GENERIC_CPU_VULNERABILITIES if PPC_BARRIER_NOSPEC
173	175	select GENERIC_EARLY_IOREMAP
	176	+ select GENERIC_GETTIMEOFDAY
174	177	select GENERIC_IRQ_SHOW
175	178	select GENERIC_IRQ_SHOW_LEVEL
176	179	select GENERIC_PCI_IOMAP if PCI
@@ -178,13 +181,15 @@ config PPC · 1 unmodified line
178	181	select GENERIC_STRNCPY_FROM_USER
179	182	select GENERIC_STRNLEN_USER
180	183	select GENERIC_TIME_VSYSCALL
181		− select GENERIC_GETTIMEOFDAY
	184	+ select GENERIC_VDSO_TIME_NS
182	185	select HAVE_ARCH_AUDITSYSCALL
183	186	select HAVE_ARCH_HUGE_VMAP if PPC_BOOK3S_64 && PPC_RADIX_MMU
184	187	select HAVE_ARCH_JUMP_LABEL
	188	+ select HAVE_ARCH_JUMP_LABEL_RELATIVE
185	189	select HAVE_ARCH_KASAN if PPC32 && PPC_PAGE_SHIFT <= 14
186	190	select HAVE_ARCH_KASAN_VMALLOC if PPC32 && PPC_PAGE_SHIFT <= 14
187	191	select HAVE_ARCH_KGDB
	192	+ select HAVE_ARCH_KFENCE if PPC32
188	193	select HAVE_ARCH_MMAP_RND_BITS
189	194	select HAVE_ARCH_MMAP_RND_COMPAT_BITS if COMPAT
190	195	select HAVE_ARCH_NVRAM_OPS
@@ -192,7 +197,6 @@ config PPC · 1 unmodified line
192	197	select HAVE_ARCH_TRACEHOOK
193	198	select HAVE_ASM_MODVERSIONS
194	199	select HAVE_C_RECORDMCOUNT
195		− select HAVE_CBPF_JIT if !PPC64
196	200	select HAVE_STACKPROTECTOR if PPC64 && $(cc-option,-mstack-protector-guard=tls -mstack-protector-guard-reg=r13)
197	201	select HAVE_STACKPROTECTOR if PPC32 && $(cc-option,-mstack-protector-guard=tls -mstack-protector-guard-reg=r2)
198	202	select HAVE_CONTEXT_TRACKING if PPC64
@@ -200,7 +204,7 @@ config PPC · 1 unmodified line
200	204	select HAVE_DEBUG_STACKOVERFLOW
201	205	select HAVE_DYNAMIC_FTRACE
202	206	select HAVE_DYNAMIC_FTRACE_WITH_REGS if MPROFILE_KERNEL
203		− select HAVE_EBPF_JIT if PPC64
	207	+ select HAVE_EBPF_JIT
204	208	select HAVE_EFFICIENT_UNALIGNED_ACCESS if !(CPU_LITTLE_ENDIAN && POWER7_CPU)
205	209	select HAVE_FAST_GUP
206	210	select HAVE_FTRACE_MCOUNT_RECORD
@@ -224,8 +228,8 @@ config PPC · 17 unmodified lines
224	228	select HAVE_LIVEPATCH if HAVE_DYNAMIC_FTRACE_WITH_REGS
225	229	select HAVE_MOD_ARCH_SPECIFIC
226	230	select HAVE_NMI if PERF_EVENTS \|\| (PPC64 && PPC_BOOK3S)
227		− select HAVE_HARDLOCKUP_DETECTOR_ARCH if (PPC64 && PPC_BOOK3S)
228		− select HAVE_OPTPROBES if PPC64
	231	+ select HAVE_HARDLOCKUP_DETECTOR_ARCH if PPC64 && PPC_BOOK3S && SMP
	232	+ select HAVE_OPTPROBES
229	233	select HAVE_PERF_EVENTS
230	234	select HAVE_PERF_EVENTS_NMI if PPC64
231	235	select HAVE_HARDLOCKUP_DETECTOR_PERF if PERF_EVENTS && HAVE_PERF_EVENTS_NMI && !HAVE_HARDLOCKUP_DETECTOR_ARCH
@@ -234,7 +238,7 @@ config PPC · 2 unmodified lines
234	238	select MMU_GATHER_RCU_TABLE_FREE
235	239	select MMU_GATHER_PAGE_SIZE
236	240	select HAVE_REGS_AND_STACK_ACCESS_API
237		− select HAVE_RELIABLE_STACKTRACE if PPC_BOOK3S_64 && CPU_LITTLE_ENDIAN
	241	+ select HAVE_RELIABLE_STACKTRACE
238	242	select HAVE_SOFTIRQ_ON_OWN_STACK
239	243	select HAVE_SYSCALL_TRACEPOINTS
240	244	select HAVE_VIRT_CPU_ACCOUNTING
@@ -786,7 +790,7 @@ config THREAD_SHIFT · 545 unmodified lines
786	790	config DATA_SHIFT_BOOL
787	791	bool "Set custom data alignment"
788	792	depends on ADVANCED_OPTIONS
789		− depends on STRICT_KERNEL_RWX \|\| DEBUG_PAGEALLOC
	793	+ depends on STRICT_KERNEL_RWX \|\| DEBUG_PAGEALLOC \|\| KFENCE
790	794	depends on PPC_BOOK3S_32 \|\| (PPC_8xx && !PIN_TLB_DATA && !STRICT_KERNEL_RWX)
791	795	help
792	796	This option allows you to set the kernel data alignment. When
@@ -798,13 +802,13 @@ config DATA_SHIFT_BOOL · 5 unmodified lines
798	802	config DATA_SHIFT
799	803	int "Data shift" if DATA_SHIFT_BOOL
800	804	default 24 if STRICT_KERNEL_RWX && PPC64
801		− range 17 28 if (STRICT_KERNEL_RWX \|\| DEBUG_PAGEALLOC) && PPC_BOOK3S_32
802		− range 19 23 if (STRICT_KERNEL_RWX \|\| DEBUG_PAGEALLOC) && PPC_8xx
	805	+ range 17 28 if (STRICT_KERNEL_RWX \|\| DEBUG_PAGEALLOC \|\| KFENCE) && PPC_BOOK3S_32
	806	+ range 19 23 if (STRICT_KERNEL_RWX \|\| DEBUG_PAGEALLOC \|\| KFENCE) && PPC_8xx
803	807	default 22 if STRICT_KERNEL_RWX && PPC_BOOK3S_32
804		− default 18 if DEBUG_PAGEALLOC && PPC_BOOK3S_32
	808	+ default 18 if (DEBUG_PAGEALLOC \|\| KFENCE) && PPC_BOOK3S_32
805	809	default 23 if STRICT_KERNEL_RWX && PPC_8xx
806		− default 23 if DEBUG_PAGEALLOC && PPC_8xx && PIN_TLB_DATA
807		− default 19 if DEBUG_PAGEALLOC && PPC_8xx
	810	+ default 23 if (DEBUG_PAGEALLOC \|\| KFENCE) && PPC_8xx && PIN_TLB_DATA
	811	+ default 19 if (DEBUG_PAGEALLOC \|\| KFENCE) && PPC_8xx
808	812	default PPC_PAGE_SHIFT
809	813	help
810	814	On Book3S 32 (603+), DBATs are used to map kernel text and rodata RO.
@@ -1217,7 +1221,7 @@ config TASK_SIZE_BOOL · 406 unmodified lines
1217	1221	config TASK_SIZE
1218	1222	hex "Size of user task space" if TASK_SIZE_BOOL
1219	1223	default "0x80000000" if PPC_8xx
1220		− default "0xb0000000" if PPC_BOOK3S_32 && STRICT_KERNEL_RWX
	1224	+ default "0xb0000000" if PPC_BOOK3S_32
1221	1225	default "0xc0000000"
1222	1226	endmenu
1223	1227

arch/powerpc/Kconfig.debug+1−0

@@ -353,6 +353,7 @@ config PPC_EARLY_DEBUG_CPM_ADDR · 352 unmodified lines
353	353	config FAIL_IOMMU
354	354	bool "Fault-injection capability for IOMMU"
355	355	depends on FAULT_INJECTION
	356	+ depends on PCI \|\| IBMVIO
356	357	help
357	358	Provide fault-injection capability for IOMMU. Each device can
358	359	be selectively enabled via the fail_iommu property.

arch/powerpc/Makefile+4−7

@@ -181,12 +181,6 @@ CC_FLAGS_FTRACE := -pg · 180 unmodified lines
181	181	ifdef CONFIG_MPROFILE_KERNEL
182	182	CC_FLAGS_FTRACE += -mprofile-kernel
183	183	endif
184		−# Work around gcc code-gen bugs with -pg / -fno-omit-frame-pointer in gcc <= 4.8
185		−# https://gcc.gnu.org/bugzilla/show_bug.cgi?id=44199
186		−# https://gcc.gnu.org/bugzilla/show_bug.cgi?id=52828
187		−ifndef CONFIG_CC_IS_CLANG
188		−CC_FLAGS_FTRACE += $(call cc-ifversion, -lt, 0409, -mno-sched-epilog)
189		−endif
190	184	endif
191	185
192	186	CFLAGS-$(CONFIG_TARGET_CPU_BOOL) += $(call cc-option,-mcpu=$(CONFIG_TARGET_CPU))
@@ -444,12 +438,15 @@ endif · 251 unmodified lines
444	438	endif
445	439
446	440	ifdef CONFIG_SMP
	441	+ifdef CONFIG_PPC32
447	442	prepare: task_cpu_prepare
448	443
449	444	PHONY += task_cpu_prepare
450	445	task_cpu_prepare: prepare0
451	446	$(eval KBUILD_CFLAGS += -D_TASK_CPU=$(shell awk '{if ($$2 == "TASK_CPU") print $$3;}' include/generated/asm-offsets.h))
452		−endif
	447	+
	448	+endif # CONFIG_PPC32
	449	+endif # CONFIG_SMP
453	450
454	451	PHONY += checkbin
455	452	# Check toolchain versions:

arch/powerpc/configs/ppc64_defconfig+2−0

@@ -50,6 +50,7 @@ CONFIG_PPC_TRANSACTIONAL_MEM=y · 49 unmodified lines
50	50	CONFIG_KEXEC=y
51	51	CONFIG_KEXEC_FILE=y
52	52	CONFIG_CRASH_DUMP=y
	53	+CONFIG_FA_DUMP=y
53	54	CONFIG_IRQ_ALL_CPUS=y
54	55	CONFIG_PPC_64K_PAGES=y
55	56	CONFIG_SCHED_SMT=y
@@ -177,6 +178,7 @@ CONFIG_CHELSIO_T1=m · 121 unmodified lines
177	178	CONFIG_BE2NET=m
178	179	CONFIG_IBMVETH=m
179	180	CONFIG_EHEA=m
	181	+CONFIG_IBMVNIC=m
180	182	CONFIG_E100=y
181	183	CONFIG_E1000=y
182	184	CONFIG_E1000E=y

arch/powerpc/configs/pseries_defconfig+2−0

@@ -41,6 +41,7 @@ CONFIG_DTL=y · 40 unmodified lines
41	41	CONFIG_SCANLOG=m
42	42	CONFIG_PPC_SMLPAR=y
43	43	CONFIG_IBMEBUS=y
	44	+CONFIG_PAPR_SCM=m
44	45	CONFIG_PPC_SVM=y
45	46	# CONFIG_PPC_PMAC is not set
46	47	CONFIG_RTAS_FLASH=m
@@ -159,6 +160,7 @@ CONFIG_BE2NET=m · 112 unmodified lines
159	160	CONFIG_S2IO=m
160	161	CONFIG_IBMVETH=y
161	162	CONFIG_EHEA=y
	163	+CONFIG_IBMVNIC=y
162	164	CONFIG_E100=y
163	165	CONFIG_E1000=y
164	166	CONFIG_E1000E=y

arch/powerpc/include/asm/Kbuild+0−1

@@ -1,7 +1,6 @@
1	1	# SPDX-License-Identifier: GPL-2.0
2	2	generated-y += syscall_table_32.h
3	3	generated-y += syscall_table_64.h
4		−generated-y += syscall_table_c32.h
5	4	generated-y += syscall_table_spu.h
6	5	generic-y += export.h
7	6	generic-y += kvm_types.h

arch/powerpc/include/asm/asm-prototypes.h+0−2

@@ -77,8 +77,6 @@ notrace unsigned long interrupt_exit_kernel_prepare(struct pt_regs *regs, unsign · 76 unmodified lines
77	77	long ppc_fadvise64_64(int fd, int advice, u32 offset_high, u32 offset_low,
78	78	u32 len_high, u32 len_low);
79	79	long sys_switch_endian(void);
80		−notrace unsigned int __check_irq_replay(void);
81		−void notrace restore_interrupts(void);
82	80
83	81	/* prom_init (OpenFirmware) */
84	82	unsigned long __init prom_init(unsigned long r3, unsigned long r4,

arch/powerpc/include/asm/barrier.h+0−16

@@ -80,22 +80,6 @@ do { \ · 79 unmodified lines
80	80	___p1; \
81	81	})
82	82
83		−#ifdef CONFIG_PPC64
84		−#define smp_cond_load_relaxed(ptr, cond_expr) ({ \
85		− typeof(ptr) __PTR = (ptr); \
86		− __unqual_scalar_typeof(*ptr) VAL; \
87		− VAL = READ_ONCE(*__PTR); \
88		− if (unlikely(!(cond_expr))) { \
89		− spin_begin(); \
90		− do { \
91		− VAL = READ_ONCE(*__PTR); \
92		− } while (!(cond_expr)); \
93		− spin_end(); \
94		− } \
95		− (typeof(*ptr))VAL; \
96		−})
97		−#endif
98		−
99	83	#ifdef CONFIG_PPC_BOOK3S_64
100	84	#define NOSPEC_BARRIER_SLOT nop
101	85	#elif defined(CONFIG_PPC_FSL_BOOK3E)

arch/powerpc/include/asm/book3s/32/kup.h+46−80

@@ -5,86 +5,7 @@ · 4 unmodified lines
5	5	#include <asm/bug.h>
6	6	#include <asm/book3s/32/mmu-hash.h>
7	7
8		−#ifdef __ASSEMBLY__
9		−
10		−.macro kuep_update_sr gpr1, gpr2 /* NEVER use r0 as gpr2 due to addis */
11		−101: mtsrin \gpr1, \gpr2
12		− addi \gpr1, \gpr1, 0x111 /* next VSID */
13		− rlwinm \gpr1, \gpr1, 0, 0xf0ffffff /* clear VSID overflow */
14		− addis \gpr2, \gpr2, 0x1000 /* address of next segment */
15		− bdnz 101b
16		− isync
17		−.endm
18		−
19		−.macro kuep_lock gpr1, gpr2
20		−#ifdef CONFIG_PPC_KUEP
21		− li \gpr1, NUM_USER_SEGMENTS
22		− li \gpr2, 0
23		− mtctr \gpr1
24		− mfsrin \gpr1, \gpr2
25		− oris \gpr1, \gpr1, SR_NX@h /* set Nx */
26		− kuep_update_sr \gpr1, \gpr2
27		−#endif
28		−.endm
29		−
30		−.macro kuep_unlock gpr1, gpr2
31		−#ifdef CONFIG_PPC_KUEP
32		− li \gpr1, NUM_USER_SEGMENTS
33		− li \gpr2, 0
34		− mtctr \gpr1
35		− mfsrin \gpr1, \gpr2
36		− rlwinm \gpr1, \gpr1, 0, ~SR_NX /* Clear Nx */
37		− kuep_update_sr \gpr1, \gpr2
38		−#endif
39		−.endm
40		−
41		−#ifdef CONFIG_PPC_KUAP
42		−
43		−.macro kuap_update_sr gpr1, gpr2, gpr3 /* NEVER use r0 as gpr2 due to addis */
44		−101: mtsrin \gpr1, \gpr2
45		− addi \gpr1, \gpr1, 0x111 /* next VSID */
46		− rlwinm \gpr1, \gpr1, 0, 0xf0ffffff /* clear VSID overflow */
47		− addis \gpr2, \gpr2, 0x1000 /* address of next segment */
48		− cmplw \gpr2, \gpr3
49		− blt- 101b
50		− isync
51		−.endm
52		−
53		−.macro kuap_save_and_lock sp, thread, gpr1, gpr2, gpr3
54		− lwz \gpr2, KUAP(\thread)
55		− rlwinm. \gpr3, \gpr2, 28, 0xf0000000
56		− stw \gpr2, STACK_REGS_KUAP(\sp)
57		− beq+ 102f
58		− li \gpr1, 0
59		− stw \gpr1, KUAP(\thread)
60		− mfsrin \gpr1, \gpr2
61		− oris \gpr1, \gpr1, SR_KS@h /* set Ks */
62		− kuap_update_sr \gpr1, \gpr2, \gpr3
63		−102:
64		−.endm
65		−
66		−.macro kuap_restore sp, current, gpr1, gpr2, gpr3
67		− lwz \gpr2, STACK_REGS_KUAP(\sp)
68		− rlwinm. \gpr3, \gpr2, 28, 0xf0000000
69		− stw \gpr2, THREAD + KUAP(\current)
70		− beq+ 102f
71		− mfsrin \gpr1, \gpr2
72		− rlwinm \gpr1, \gpr1, 0, ~SR_KS /* Clear Ks */
73		− kuap_update_sr \gpr1, \gpr2, \gpr3
74		−102:
75		−.endm
76		−
77		−.macro kuap_check current, gpr
78		−#ifdef CONFIG_PPC_KUAP_DEBUG
79		− lwz \gpr, THREAD + KUAP(\current)
80		−999: twnei \gpr, 0
81		− EMIT_BUG_ENTRY 999b, __FILE__, __LINE__, (BUGFLAG_WARNING \| BUGFLAG_ONCE)
82		−#endif
83		−.endm
84		−
85		−#endif /* CONFIG_PPC_KUAP */
86		−
87		−#else /* !__ASSEMBLY__ */
	8	+#ifndef __ASSEMBLY__
88	9
89	10	#ifdef CONFIG_PPC_KUAP
90	11
@@ -103,6 +24,51 @@ static inline void kuap_update_sr(u32 sr, u32 addr, u32 end) · 12 unmodified lines
103	24	isync(); /* Context sync required after mtsr() */
104	25	}
105	26
	27	+static inline void kuap_save_and_lock(struct pt_regs *regs)
	28	+{
	29	+ unsigned long kuap = current->thread.kuap;
	30	+ u32 addr = kuap & 0xf0000000;
	31	+ u32 end = kuap << 28;
	32	+
	33	+ regs->kuap = kuap;
	34	+ if (unlikely(!kuap))
	35	+ return;
	36	+
	37	+ current->thread.kuap = 0;
	38	+ kuap_update_sr(mfsr(addr) \| SR_KS, addr, end); /* Set Ks */
	39	+}
	40	+
	41	+static inline void kuap_user_restore(struct pt_regs *regs)
	42	+{
	43	+}
	44	+
	45	+static inline void kuap_kernel_restore(struct pt_regs *regs, unsigned long kuap)
	46	+{
	47	+ u32 addr = regs->kuap & 0xf0000000;
	48	+ u32 end = regs->kuap << 28;
	49	+
	50	+ current->thread.kuap = regs->kuap;
	51	+
	52	+ if (unlikely(regs->kuap == kuap))
	53	+ return;
	54	+
	55	+ kuap_update_sr(mfsr(addr) & ~SR_KS, addr, end); /* Clear Ks */
	56	+}
	57	+
	58	+static inline unsigned long kuap_get_and_assert_locked(void)
	59	+{
	60	+ unsigned long kuap = current->thread.kuap;
	61	+
	62	+ WARN_ON_ONCE(IS_ENABLED(CONFIG_PPC_KUAP_DEBUG) && kuap != 0);
	63	+
	64	+ return kuap;
	65	+}
	66	+
	67	+static inline void kuap_assert_locked(void)
	68	+{
	69	+ kuap_get_and_assert_locked();
	70	+}
	71	+
106	72	static __always_inline void allow_user_access(void __user to, const void __user from,
107	73	u32 size, unsigned long dir)
108	74	{

arch/powerpc/include/asm/book3s/32/pgtable.h+0−2

@@ -194,10 +194,8 @@ int map_kernel_page(unsigned long va, phys_addr_t pa, pgprot_t prot); · 193 unmodified lines
194	194	#define VMALLOC_END ioremap_bot
195	195	#endif
196	196
197		−#ifdef CONFIG_STRICT_KERNEL_RWX
198	197	#define MODULES_END ALIGN_DOWN(PAGE_OFFSET, SZ_256M)
199	198	#define MODULES_VADDR (MODULES_END - SZ_256M)
200		−#endif
201	199
202	200	#ifndef __ASSEMBLY__
203	201	#include <linux/sched.h>

arch/powerpc/include/asm/book3s/32/tlbflush.h+1−1

@@ -79,4 +79,4 @@ static inline void local_flush_tlb_mm(struct mm_struct *mm) · 78 unmodified lines
79	79	flush_tlb_mm(mm);
80	80	}
81	81
82		−#endif /* _ASM_POWERPC_TLBFLUSH_H */
	82	+#endif /* _ASM_POWERPC_BOOK3S_32_TLBFLUSH_H */

arch/powerpc/include/asm/book3s/64/kup.h+2−22

@@ -287,7 +287,7 @@ static inline void kuap_kernel_restore(struct pt_regs *regs, · 286 unmodified lines
287	287	*/
288	288	}
289	289
290		−static inline unsigned long kuap_get_and_check_amr(void)
	290	+static inline unsigned long kuap_get_and_assert_locked(void)
291	291	{
292	292	if (mmu_has_feature(MMU_FTR_BOOK3S_KUAP)) {
293	293	unsigned long amr = mfspr(SPRN_AMR);
@@ -298,27 +298,7 @@ static inline unsigned long kuap_get_and_check_amr(void) · 4 unmodified lines
298	298	return 0;
299	299	}
300	300
301		−#else /* CONFIG_PPC_PKEY */
302		−
303		−static inline void kuap_user_restore(struct pt_regs *regs)
304		−{
305		−}
306		−
307		−static inline void kuap_kernel_restore(struct pt_regs *regs, unsigned long amr)
308		−{
309		−}
310		−
311		−static inline unsigned long kuap_get_and_check_amr(void)
312		−{
313		− return 0;
314		−}
315		−
316		−#endif /* CONFIG_PPC_PKEY */
317		−
318		−
319		−#ifdef CONFIG_PPC_KUAP
320		−
321		−static inline void kuap_check_amr(void)
	301	+static inline void kuap_assert_locked(void)
322	302	{
323	303	if (IS_ENABLED(CONFIG_PPC_KUAP_DEBUG) && mmu_has_feature(MMU_FTR_BOOK3S_KUAP))
324	304	WARN_ON_ONCE(mfspr(SPRN_AMR) != AMR_KUAP_BLOCKED);

arch/powerpc/include/asm/book3s/64/mmu-hash.h+0−1

@@ -18,7 +18,6 @@ · 17 unmodified lines
18	18	* complete pgtable.h but only a portion of it.
19	19	*/
20	20	#include <asm/book3s/64/pgtable.h>
21		−#include <asm/bug.h>
22	21	#include <asm/task_size_64.h>
23	22	#include <asm/cpu_has_feature.h>
24	23

arch/powerpc/include/asm/book3s/64/pgtable.h+4−1

@@ -7,6 +7,7 @@ · 6 unmodified lines
7	7	#ifndef __ASSEMBLY__
8	8	#include <linux/mmdebug.h>
9	9	#include <linux/bug.h>
	10	+#include <linux/sizes.h>
10	11	#endif
11	12
12	13	/*
@@ -116,6 +117,7 @@ · 103 unmodified lines
116	117	*/
117	118	#define _PAGE_KERNEL_RW (_PAGE_PRIVILEGED \| _PAGE_RW \| _PAGE_DIRTY)
118	119	#define _PAGE_KERNEL_RO (_PAGE_PRIVILEGED \| _PAGE_READ)
	120	+#define _PAGE_KERNEL_ROX (_PAGE_PRIVILEGED \| _PAGE_READ \| _PAGE_EXEC)
119	121	#define _PAGE_KERNEL_RWX (_PAGE_PRIVILEGED \| _PAGE_DIRTY \| \
120	122	_PAGE_RW \| _PAGE_EXEC)
121	123	/*
@@ -323,7 +325,8 @@ extern unsigned long pci_io_base; · 201 unmodified lines
323	325	#define PHB_IO_END (KERN_IO_START + FULL_IO_SIZE)
324	326	#define IOREMAP_BASE (PHB_IO_END)
325	327	#define IOREMAP_START (ioremap_bot)
326		−#define IOREMAP_END (KERN_IO_END)
	328	+#define IOREMAP_END (KERN_IO_END - FIXADDR_SIZE)
	329	+#define FIXADDR_SIZE SZ_32M
327	330
328	331	/* Advertise special mapping type for AGP */
329	332	#define HAVE_PAGE_AGP

arch/powerpc/include/asm/book3s/64/radix.h+4−2

@@ -222,8 +222,10 @@ static inline void radix__set_pte_at(struct mm_struct *mm, unsigned long addr, · 221 unmodified lines
222	222	* from ptesync, it should probably go into update_mmu_cache, rather
223	223	* than set_pte_at (which is used to set ptes unrelated to faults).
224	224	*
225		− * Spurious faults to vmalloc region are not tolerated, so there is
226		− * a ptesync in flush_cache_vmap.
	225	+ * Spurious faults from the kernel memory are not tolerated, so there
	226	+ * is a ptesync in flush_cache_vmap, and __map_kernel_page() follows
	227	+ * the pte update sequence from ISA Book III 6.10 Translation Table
	228	+ * Update Synchronization Requirements.
227	229	*/
228	230	}
229	231

arch/powerpc/include/asm/bug.h+1−4

@@ -111,11 +111,8 @@ · 110 unmodified lines
111	111	#ifndef __ASSEMBLY__
112	112
113	113	struct pt_regs;
114		−long do_page_fault(struct pt_regs *);
115		−long hash__do_page_fault(struct pt_regs *);
	114	+void hash__do_page_fault(struct pt_regs *);
116	115	void bad_page_fault(struct pt_regs *, int);
117		−void __bad_page_fault(struct pt_regs *regs, int sig);
118		−void do_bad_page_fault_segv(struct pt_regs *regs);
119	116	extern void _exception(int, struct pt_regs *, int, unsigned long);
120	117	extern void _exception_pkey(struct pt_regs *, unsigned long, int);
121	118	extern void die(const char , struct pt_regs , long);

arch/powerpc/include/asm/cacheflush.h+13−2

@@ -30,7 +30,19 @@ static inline void flush_cache_vmap(unsigned long start, unsigned long end) · 29 unmodified lines
30	30	#endif /* CONFIG_PPC_BOOK3S_64 */
31	31
32	32	#define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE 1
33		−extern void flush_dcache_page(struct page *page);
	33	+/*
	34	+ * This is called when a page has been modified by the kernel.
	35	+ * It just marks the page as not i-cache clean. We do the i-cache
	36	+ * flush later when the page is given to a user process, if necessary.
	37	+ */
	38	+static inline void flush_dcache_page(struct page *page)
	39	+{
	40	+ if (cpu_has_feature(CPU_FTR_COHERENT_ICACHE))
	41	+ return;
	42	+ /* avoid an atomic op if possible */
	43	+ if (test_bit(PG_dcache_clean, &page->flags))
	44	+ clear_bit(PG_dcache_clean, &page->flags);
	45	+}
34	46
35	47	void flush_icache_range(unsigned long start, unsigned long stop);
36	48	#define flush_icache_range flush_icache_range
@@ -40,7 +52,6 @@ void flush_icache_user_page(struct vm_area_struct vma, struct page page, · 3 unmodified lines
40	52	#define flush_icache_user_page flush_icache_user_page
41	53
42	54	void flush_dcache_icache_page(struct page *page);
43		−void __flush_dcache_icache(void *page);
44	55
45	56	/**
46	57	* flush_dcache_range(): Write any modified data cache blocks out to memory and

arch/powerpc/include/asm/cpm2.h+1−1

@@ -594,7 +594,7 @@ typedef struct fcc_enet { · 593 unmodified lines
594	594	uint fen_p256c; /* Total packets 256 < bytes <= 511 */
595	595	uint fen_p512c; /* Total packets 512 < bytes <= 1023 */
596	596	uint fen_p1024c; /* Total packets 1024 < bytes <= 1518 */
597		− uint fen_cambuf; /* Internal CAM buffer poiner */
	597	+ uint fen_cambuf; /* Internal CAM buffer pointer */
598	598	ushort fen_rfthr; /* Received frames threshold */
599	599	ushort fen_rfcnt; /* Received frames count */
600	600	} fcc_enet_t;

arch/powerpc/include/asm/fixmap.h+9−0

@@ -23,12 +23,17 @@ · 22 unmodified lines
23	23	#include <asm/kmap_size.h>
24	24	#endif
25	25
	26	+#ifdef CONFIG_PPC64
	27	+#define FIXADDR_TOP (IOREMAP_END + FIXADDR_SIZE)
	28	+#else
	29	+#define FIXADDR_SIZE 0
26	30	#ifdef CONFIG_KASAN
27	31	#include <asm/kasan.h>
28	32	#define FIXADDR_TOP (KASAN_SHADOW_START - PAGE_SIZE)
29	33	#else
30	34	#define FIXADDR_TOP ((unsigned long)(-PAGE_SIZE))
31	35	#endif
	36	+#endif
32	37
33	38	/*
34	39	* Here we define all the compile-time 'special' virtual
@@ -50,6 +55,7 @@ · 15 unmodified lines
50	55	*/
51	56	enum fixed_addresses {
52	57	FIX_HOLE,
	58	+#ifdef CONFIG_PPC32
53	59	/* reserve the top 128K for early debugging purposes */
54	60	FIX_EARLY_DEBUG_TOP = FIX_HOLE,
55	61	FIX_EARLY_DEBUG_BASE = FIX_EARLY_DEBUG_TOP+(ALIGN(SZ_128K, PAGE_SIZE)/PAGE_SIZE)-1,
@@ -72,6 +78,7 @@ enum fixed_addresses { · 16 unmodified lines
72	78	FIX_IMMR_SIZE,
73	79	#endif
74	80	/* FIX_PCIE_MCFG, */
	81	+#endif /* CONFIG_PPC32 */
75	82	__end_of_permanent_fixed_addresses,
76	83
77	84	#define NR_FIX_BTMAPS (SZ_256K / PAGE_SIZE)
@@ -98,6 +105,8 @@ enum fixed_addresses { · 20 unmodified lines
98	105	static inline void __set_fixmap(enum fixed_addresses idx,
99	106	phys_addr_t phys, pgprot_t flags)
100	107	{
	108	+ BUILD_BUG_ON(IS_ENABLED(CONFIG_PPC64) && __FIXADDR_SIZE > FIXADDR_SIZE);
	109	+
101	110	if (__builtin_constant_p(idx))
102	111	BUILD_BUG_ON(idx >= __end_of_fixed_addresses);
103	112	else if (WARN_ON(idx >= __end_of_fixed_addresses))

arch/powerpc/include/asm/futex.h+5−7

@@ -33,9 +33,8 @@ static inline int arch_futex_atomic_op_inuser(int op, int oparg, int *oval, · 32 unmodified lines
33	33	{
34	34	int oldval = 0, ret;
35	35
36		− if (!access_ok(uaddr, sizeof(u32)))
	36	+ if (!user_access_begin(uaddr, sizeof(u32)))
37	37	return -EFAULT;
38		− allow_read_write_user(uaddr, uaddr, sizeof(*uaddr));
39	38
40	39	switch (op) {
41	40	case FUTEX_OP_SET:
@@ -56,10 +55,10 @@ static inline int arch_futex_atomic_op_inuser(int op, int oparg, int *oval, · 14 unmodified lines
56	55	default:
57	56	ret = -ENOSYS;
58	57	}
	58	+ user_access_end();
59	59
60	60	*oval = oldval;
61	61
62		− prevent_read_write_user(uaddr, uaddr, sizeof(*uaddr));
63	62	return ret;
64	63	}
65	64
@@ -70,11 +69,9 @@ futex_atomic_cmpxchg_inatomic(u32 uval, u32 __user uaddr, · 4 unmodified lines
70	69	int ret = 0;
71	70	u32 prev;
72	71
73		− if (!access_ok(uaddr, sizeof(u32)))
	72	+ if (!user_access_begin(uaddr, sizeof(u32)))
74	73	return -EFAULT;
75	74
76		− allow_read_write_user(uaddr, uaddr, sizeof(*uaddr));
77		−
78	75	__asm__ __volatile__ (
79	76	PPC_ATOMIC_ENTRY_BARRIER
80	77	"1: lwarx %1,0,%3 # futex_atomic_cmpxchg_inatomic\n\
@@ -93,8 +90,9 @@ futex_atomic_cmpxchg_inatomic(u32 uval, u32 __user uaddr, · 12 unmodified lines
93	90	: "r" (uaddr), "r" (oldval), "r" (newval), "i" (-EFAULT)
94	91	: "cc", "memory");
95	92
	93	+ user_access_end();
	94	+
96	95	*uval = prev;
97		− prevent_read_write_user(uaddr, uaddr, sizeof(*uaddr));
98	96
99	97	return ret;
100	98	}

arch/powerpc/include/asm/hvcall.h+3−1

@@ -315,7 +315,8 @@ · 314 unmodified lines
315	315	#define H_SCM_HEALTH 0x400
316	316	#define H_SCM_PERFORMANCE_STATS 0x418
317	317	#define H_RPT_INVALIDATE 0x448
318		−#define MAX_HCALL_OPCODE H_RPT_INVALIDATE
	318	+#define H_SCM_FLUSH 0x44C
	319	+#define MAX_HCALL_OPCODE H_SCM_FLUSH
319	320
320	321	/* Scope args for H_SCM_UNBIND_ALL */
321	322	#define H_UNBIND_SCOPE_ALL (0x1)
@@ -389,6 +390,7 @@ · 67 unmodified lines
389	390	#define H_CPU_BEHAV_FAVOUR_SECURITY (1ull << 63) // IBM bit 0
390	391	#define H_CPU_BEHAV_L1D_FLUSH_PR (1ull << 62) // IBM bit 1
391	392	#define H_CPU_BEHAV_BNDS_CHK_SPEC_BAR (1ull << 61) // IBM bit 2
	393	+#define H_CPU_BEHAV_FAVOUR_SECURITY_H (1ull << 60) // IBM bit 3
392	394	#define H_CPU_BEHAV_FLUSH_COUNT_CACHE (1ull << 58) // IBM bit 5
393	395	#define H_CPU_BEHAV_FLUSH_LINK_STACK (1ull << 57) // IBM bit 6
394	396

arch/powerpc/include/asm/hvconsole.h+3−0

@@ -24,5 +24,8 @@ · 23 unmodified lines
24	24	extern int hvc_get_chars(uint32_t vtermno, char *buf, int count);
25	25	extern int hvc_put_chars(uint32_t vtermno, const char *buf, int count);
26	26
	27	+/* Provided by HVC VIO */
	28	+void hvc_vio_init_early(void);
	29	+
27	30	#endif /* __KERNEL__ */
28	31	#endif /* _PPC64_HVCONSOLE_H */

arch/powerpc/include/asm/hydra.h+0−2

@@ -94,8 +94,6 @@ extern volatile struct Hydra __iomem *Hydra; · 93 unmodified lines
94	94	#define HYDRA_INT_EXT7 18 /* Power Off Request */
95	95	#define HYDRA_INT_SPARE 19
96	96
97		−extern int hydra_init(void);
98		−
99	97	#endif /* __KERNEL__ */
100	98
101	99	#endif /* _ASMPPC_HYDRA_H */

arch/powerpc/include/asm/inst.h+44−11

@@ -4,6 +4,40 @@ · 3 unmodified lines
4	4
5	5	#include <asm/ppc-opcode.h>
6	6
	7	+#ifdef CONFIG_PPC64
	8	+
	9	+#define ___get_user_instr(gu_op, dest, ptr) \
	10	+({ \
	11	+ long __gui_ret = 0; \
	12	+ unsigned long __gui_ptr = (unsigned long)ptr; \
	13	+ struct ppc_inst __gui_inst; \
	14	+ unsigned int __prefix, __suffix; \
	15	+ __gui_ret = gu_op(__prefix, (unsigned int __user *)__gui_ptr); \
	16	+ if (__gui_ret == 0) { \
	17	+ if ((__prefix >> 26) == OP_PREFIX) { \
	18	+ __gui_ret = gu_op(__suffix, \
	19	+ (unsigned int __user *)__gui_ptr + 1); \
	20	+ __gui_inst = ppc_inst_prefix(__prefix, \
	21	+ __suffix); \
	22	+ } else { \
	23	+ __gui_inst = ppc_inst(__prefix); \
	24	+ } \
	25	+ if (__gui_ret == 0) \
	26	+ (dest) = __gui_inst; \
	27	+ } \
	28	+ __gui_ret; \
	29	+})
	30	+#else /* !CONFIG_PPC64 */
	31	+#define ___get_user_instr(gu_op, dest, ptr) \
	32	+ gu_op((dest).val, (u32 __user *)(ptr))
	33	+#endif /* CONFIG_PPC64 */
	34	+
	35	+#define get_user_instr(x, ptr) \
	36	+ ___get_user_instr(get_user, x, ptr)
	37	+
	38	+#define __get_user_instr(x, ptr) \
	39	+ ___get_user_instr(__get_user, x, ptr)
	40	+
7	41	/*
8	42	* Instruction data type for POWER
9	43	*/
@@ -68,6 +102,8 @@ static inline bool ppc_inst_equal(struct ppc_inst x, struct ppc_inst y) · 58 unmodified lines
68	102
69	103	#define ppc_inst(x) ((struct ppc_inst){ .val = x })
70	104
	105	+#define ppc_inst_prefix(x, y) ppc_inst(x)
	106	+
71	107	static inline bool ppc_inst_prefixed(struct ppc_inst x)
72	108	{
73	109	return false;
@@ -113,13 +149,14 @@ static inline struct ppc_inst ppc_inst_next(void location, struct ppc_inst *va · 39 unmodified lines
113	149	return location + ppc_inst_len(tmp);
114	150	}
115	151
116		−static inline u64 ppc_inst_as_u64(struct ppc_inst x)
	152	+static inline unsigned long ppc_inst_as_ulong(struct ppc_inst x)
117	153	{
118		−#ifdef CONFIG_CPU_LITTLE_ENDIAN
119		− return (u64)ppc_inst_suffix(x) << 32 \| ppc_inst_val(x);
120		−#else
121		− return (u64)ppc_inst_val(x) << 32 \| ppc_inst_suffix(x);
122		−#endif
	154	+ if (IS_ENABLED(CONFIG_PPC32))
	155	+ return ppc_inst_val(x);
	156	+ else if (IS_ENABLED(CONFIG_CPU_LITTLE_ENDIAN))
	157	+ return (u64)ppc_inst_suffix(x) << 32 \| ppc_inst_val(x);
	158	+ else
	159	+ return (u64)ppc_inst_val(x) << 32 \| ppc_inst_suffix(x);
123	160	}
124	161
125	162	#define PPC_INST_STR_LEN sizeof("00000000 00000000")
@@ -141,10 +178,6 @@ static inline char *__ppc_inst_as_str(char str[PPC_INST_STR_LEN], struct ppc_ins · 15 unmodified lines
141	178	__str; \
142	179	})
143	180
144		−int probe_user_read_inst(struct ppc_inst *inst,
145		− struct ppc_inst __user *nip);
146		−
147		−int probe_kernel_read_inst(struct ppc_inst *inst,
148		− struct ppc_inst *src);
	181	+int copy_inst_from_kernel_nofault(struct ppc_inst inst, struct ppc_inst src);
149	182
150	183	#endif /* _ASM_POWERPC_INST_H */

arch/powerpc/include/asm/interrupt.h+131−42

@@ -2,6 +2,70 @@ · 1 unmodified line
2	2	#ifndef _ASM_POWERPC_INTERRUPT_H
3	3	#define _ASM_POWERPC_INTERRUPT_H
4	4
	5	+/* BookE/4xx */
	6	+#define INTERRUPT_CRITICAL_INPUT 0x100
	7	+
	8	+/* BookE */
	9	+#define INTERRUPT_DEBUG 0xd00
	10	+#ifdef CONFIG_BOOKE
	11	+#define INTERRUPT_PERFMON 0x260
	12	+#define INTERRUPT_DOORBELL 0x280
	13	+#endif
	14	+
	15	+/* BookS/4xx/8xx */
	16	+#define INTERRUPT_MACHINE_CHECK 0x200
	17	+
	18	+/* BookS/8xx */
	19	+#define INTERRUPT_SYSTEM_RESET 0x100
	20	+
	21	+/* BookS */
	22	+#define INTERRUPT_DATA_SEGMENT 0x380
	23	+#define INTERRUPT_INST_SEGMENT 0x480
	24	+#define INTERRUPT_TRACE 0xd00
	25	+#define INTERRUPT_H_DATA_STORAGE 0xe00
	26	+#define INTERRUPT_HMI 0xe60
	27	+#define INTERRUPT_H_FAC_UNAVAIL 0xf80
	28	+#ifdef CONFIG_PPC_BOOK3S
	29	+#define INTERRUPT_DOORBELL 0xa00
	30	+#define INTERRUPT_PERFMON 0xf00
	31	+#define INTERRUPT_ALTIVEC_UNAVAIL 0xf20
	32	+#endif
	33	+
	34	+/* BookE/BookS/4xx/8xx */
	35	+#define INTERRUPT_DATA_STORAGE 0x300
	36	+#define INTERRUPT_INST_STORAGE 0x400
	37	+#define INTERRUPT_EXTERNAL 0x500
	38	+#define INTERRUPT_ALIGNMENT 0x600
	39	+#define INTERRUPT_PROGRAM 0x700
	40	+#define INTERRUPT_SYSCALL 0xc00
	41	+#define INTERRUPT_TRACE 0xd00
	42	+
	43	+/* BookE/BookS/44x */
	44	+#define INTERRUPT_FP_UNAVAIL 0x800
	45	+
	46	+/* BookE/BookS/44x/8xx */
	47	+#define INTERRUPT_DECREMENTER 0x900
	48	+
	49	+#ifndef INTERRUPT_PERFMON
	50	+#define INTERRUPT_PERFMON 0x0
	51	+#endif
	52	+
	53	+/* 8xx */
	54	+#define INTERRUPT_SOFT_EMU_8xx 0x1000
	55	+#define INTERRUPT_INST_TLB_MISS_8xx 0x1100
	56	+#define INTERRUPT_DATA_TLB_MISS_8xx 0x1200
	57	+#define INTERRUPT_INST_TLB_ERROR_8xx 0x1300
	58	+#define INTERRUPT_DATA_TLB_ERROR_8xx 0x1400
	59	+#define INTERRUPT_DATA_BREAKPOINT_8xx 0x1c00
	60	+#define INTERRUPT_INST_BREAKPOINT_8xx 0x1d00
	61	+
	62	+/* 603 */
	63	+#define INTERRUPT_INST_TLB_MISS_603 0x1000
	64	+#define INTERRUPT_DATA_LOAD_TLB_MISS_603 0x1100
	65	+#define INTERRUPT_DATA_STORE_TLB_MISS_603 0x1200
	66	+
	67	+#ifndef __ASSEMBLY__
	68	+
5	69	#include <linux/context_tracking.h>
6	70	#include <linux/hardirq.h>
7	71	#include <asm/cputime.h>
@@ -9,10 +73,18 @@ · 1 unmodified line
9	73	#include <asm/kprobes.h>
10	74	#include <asm/runlatch.h>
11	75
12		−struct interrupt_state {
13		−#ifdef CONFIG_PPC_BOOK3E_64
14		− enum ctx_state ctx_state;
	76	+static inline void nap_adjust_return(struct pt_regs *regs)
	77	+{
	78	+#ifdef CONFIG_PPC_970_NAP
	79	+ if (unlikely(test_thread_local_flags(_TLF_NAPPING))) {
	80	+ /* Can avoid a test-and-clear because NMIs do not call this */
	81	+ clear_thread_local_flags(_TLF_NAPPING);
	82	+ regs->nip = (unsigned long)power4_idle_nap_return;
	83	+ }
15	84	#endif
	85	+}
	86	+
	87	+struct interrupt_state {
16	88	};
17	89
18	90	static inline void booke_restore_dbcr0(void)
@@ -29,10 +101,19 @@ static inline void booke_restore_dbcr0(void) · 10 unmodified lines
29	101
30	102	static inline void interrupt_enter_prepare(struct pt_regs regs, struct interrupt_state state)
31	103	{
32		− /*
33		− * Book3E reconciles irq soft mask in asm
34		− */
35		−#ifdef CONFIG_PPC_BOOK3S_64
	104	+#ifdef CONFIG_PPC32
	105	+ if (!arch_irq_disabled_regs(regs))
	106	+ trace_hardirqs_off();
	107	+
	108	+ if (user_mode(regs)) {
	109	+ kuep_lock();
	110	+ account_cpu_user_entry();
	111	+ } else {
	112	+ kuap_save_and_lock(regs);
	113	+ }
	114	+#endif
	115	+
	116	+#ifdef CONFIG_PPC64
36	117	if (irq_soft_mask_set_return(IRQS_ALL_DISABLED) == IRQS_ENABLED)
37	118	trace_hardirqs_off();
38	119	local_paca->irq_happened \|= PACA_IRQ_HARD_DIS;
@@ -48,16 +129,12 @@ static inline void interrupt_enter_prepare(struct pt_regs *regs, struct interrup · 9 unmodified lines
48	129	* CT_WARN_ON comes here via program_check_exception,
49	130	* so avoid recursion.
50	131	*/
51		− if (TRAP(regs) != 0x700)
	132	+ if (TRAP(regs) != INTERRUPT_PROGRAM)
52	133	CT_WARN_ON(ct_state() != CONTEXT_KERNEL);
53	134	}
54	135	#endif
55	136
56		−#ifdef CONFIG_PPC_BOOK3E_64
57		− state->ctx_state = exception_enter();
58		− if (user_mode(regs))
59		− account_cpu_user_entry();
60		−#endif
	137	+ booke_restore_dbcr0();
61	138	}
62	139
63	140	/*
@@ -76,23 +153,8 @@ static inline void interrupt_enter_prepare(struct pt_regs *regs, struct interrup · 12 unmodified lines
76	153	*/
77	154	static inline void interrupt_exit_prepare(struct pt_regs regs, struct interrupt_state state)
78	155	{
79		−#ifdef CONFIG_PPC_BOOK3E_64
80		− exception_exit(state->ctx_state);
81		−#endif
82		−
83		− /*
84		− * Book3S exits to user via interrupt_exit_user_prepare(), which does
85		− * context tracking, which is a cleaner way to handle PREEMPT=y
86		− * and avoid context entry/exit in e.g., preempt_schedule_irq()),
87		− * which is likely to be where the core code wants to end up.
88		− *
89		− * The above comment explains why we can't do the
90		− *
91		− * if (user_mode(regs))
92		− * user_exit_irqoff();
93		− *
94		− * sequence here.
95		− */
	156	+ if (user_mode(regs))
	157	+ kuep_unlock();
96	158	}
97	159
98	160	static inline void interrupt_async_enter_prepare(struct pt_regs regs, struct interrupt_state state)
@@ -109,24 +171,46 @@ static inline void interrupt_async_enter_prepare(struct pt_regs *regs, struct in · 10 unmodified lines
109	171
110	172	static inline void interrupt_async_exit_prepare(struct pt_regs regs, struct interrupt_state state)
111	173	{
	174	+ /*
	175	+ * Adjust at exit so the main handler sees the true NIA. This must
	176	+ * come before irq_exit() because irq_exit can enable interrupts, and
	177	+ * if another interrupt is taken before nap_adjust_return has run
	178	+ * here, then that interrupt would return directly to idle nap return.
	179	+ */
	180	+ nap_adjust_return(regs);
	181	+
112	182	irq_exit();
113	183	interrupt_exit_prepare(regs, state);
114	184	}
115	185
116	186	struct interrupt_nmi_state {
117	187	#ifdef CONFIG_PPC64
118		−#ifdef CONFIG_PPC_BOOK3S_64
119	188	u8 irq_soft_mask;
120	189	u8 irq_happened;
121		−#endif
122	190	u8 ftrace_enabled;
123	191	#endif
124	192	};
125	193
	194	+static inline bool nmi_disables_ftrace(struct pt_regs *regs)
	195	+{
	196	+ /* Allow DEC and PMI to be traced when they are soft-NMI */
	197	+ if (IS_ENABLED(CONFIG_PPC_BOOK3S_64)) {
	198	+ if (TRAP(regs) == INTERRUPT_DECREMENTER)
	199	+ return false;
	200	+ if (TRAP(regs) == INTERRUPT_PERFMON)
	201	+ return false;
	202	+ }
	203	+ if (IS_ENABLED(CONFIG_PPC_BOOK3E)) {
	204	+ if (TRAP(regs) == INTERRUPT_PERFMON)
	205	+ return false;
	206	+ }
	207	+
	208	+ return true;
	209	+}
	210	+
126	211	static inline void interrupt_nmi_enter_prepare(struct pt_regs regs, struct interrupt_nmi_state state)
127	212	{
128	213	#ifdef CONFIG_PPC64
129		−#ifdef CONFIG_PPC_BOOK3S_64
130	214	state->irq_soft_mask = local_paca->irq_soft_mask;
131	215	state->irq_happened = local_paca->irq_happened;
132	216
@@ -139,9 +223,8 @@ static inline void interrupt_nmi_enter_prepare(struct pt_regs *regs, struct inte · 6 unmodified lines
139	223	local_paca->irq_happened \|= PACA_IRQ_HARD_DIS;
140	224
141	225	/* Don't do any per-CPU operations until interrupt state is fixed */
142		−#endif
143		− /* Allow DEC and PMI to be traced when they are soft-NMI */
144		− if (TRAP(regs) != 0x900 && TRAP(regs) != 0xf00 && TRAP(regs) != 0x260) {
	226	+
	227	+ if (nmi_disables_ftrace(regs)) {
145	228	state->ftrace_enabled = this_cpu_get_ftrace_enabled();
146	229	this_cpu_set_ftrace_enabled(0);
147	230	}
@@ -164,17 +247,20 @@ static inline void interrupt_nmi_exit_prepare(struct pt_regs *regs, struct inter · 16 unmodified lines
164	247	radix_enabled() \|\| (mfmsr() & MSR_DR))
165	248	nmi_exit();
166	249
	250	+ /*
	251	+ * nmi does not call nap_adjust_return because nmi should not create
	252	+ * new work to do (must use irq_work for that).
	253	+ */
	254	+
167	255	#ifdef CONFIG_PPC64
168		− if (TRAP(regs) != 0x900 && TRAP(regs) != 0xf00 && TRAP(regs) != 0x260)
	256	+ if (nmi_disables_ftrace(regs))
169	257	this_cpu_set_ftrace_enabled(state->ftrace_enabled);
170	258
171		−#ifdef CONFIG_PPC_BOOK3S_64
172	259	/* Check we didn't change the pending interrupt mask. */
173	260	WARN_ON_ONCE((state->irq_happened \| PACA_IRQ_HARD_DIS) != local_paca->irq_happened);
174	261	local_paca->irq_happened = state->irq_happened;
175	262	local_paca->irq_soft_mask = state->irq_soft_mask;
176	263	#endif
177		−#endif
178	264	}
179	265
180	266	/*
@@ -387,6 +473,7 @@ DECLARE_INTERRUPT_HANDLER(SMIException); · 206 unmodified lines
387	473	DECLARE_INTERRUPT_HANDLER(handle_hmi_exception);
388	474	DECLARE_INTERRUPT_HANDLER(unknown_exception);
389	475	DECLARE_INTERRUPT_HANDLER_ASYNC(unknown_async_exception);
	476	+DECLARE_INTERRUPT_HANDLER_NMI(unknown_nmi_exception);
390	477	DECLARE_INTERRUPT_HANDLER(instruction_breakpoint_exception);
391	478	DECLARE_INTERRUPT_HANDLER(RunModeException);
392	479	DECLARE_INTERRUPT_HANDLER(single_step_exception);
@@ -410,7 +497,7 @@ DECLARE_INTERRUPT_HANDLER(altivec_assist_exception); · 17 unmodified lines
410	497	DECLARE_INTERRUPT_HANDLER(CacheLockingException);
411	498	DECLARE_INTERRUPT_HANDLER(SPEFloatingPointException);
412	499	DECLARE_INTERRUPT_HANDLER(SPEFloatingPointRoundException);
413		−DECLARE_INTERRUPT_HANDLER(WatchdogException);
	500	+DECLARE_INTERRUPT_HANDLER_NMI(WatchdogException);
414	501	DECLARE_INTERRUPT_HANDLER(kernel_bad_stack);
415	502
416	503	/* slb.c */
@@ -421,7 +508,7 @@ DECLARE_INTERRUPT_HANDLER(do_bad_slb_fault); · 4 unmodified lines
421	508	DECLARE_INTERRUPT_HANDLER_RAW(do_hash_fault);
422	509
423	510	/* fault.c */
424		−DECLARE_INTERRUPT_HANDLER_RET(do_page_fault);
	511	+DECLARE_INTERRUPT_HANDLER(do_page_fault);
425	512	DECLARE_INTERRUPT_HANDLER(do_bad_page_fault_segv);
426	513
427	514	/* process.c */
@@ -436,7 +523,7 @@ DECLARE_INTERRUPT_HANDLER_NMI(hmi_exception_realmode); · 8 unmodified lines
436	523
437	524	DECLARE_INTERRUPT_HANDLER_ASYNC(TAUException);
438	525
439		−void unrecoverable_exception(struct pt_regs *regs);
	526	+void __noreturn unrecoverable_exception(struct pt_regs *regs);
440	527
441	528	void replay_system_reset(void);
442	529	void replay_soft_interrupts(void);
@@ -447,4 +534,6 @@ static inline void interrupt_cond_local_irq_enable(struct pt_regs *regs) · 4 unmodified lines
447	534	local_irq_enable();
448	535	}
449	536
	537	+#endif /* __ASSEMBLY__ */
	538	+
450	539	#endif /* _ASM_POWERPC_INTERRUPT_H */

arch/powerpc/include/asm/irq.h+0−2

@@ -53,8 +53,6 @@ extern void *mcheckirq_ctx[NR_CPUS]; · 52 unmodified lines
53	53	extern void *hardirq_ctx[NR_CPUS];
54	54	extern void *softirq_ctx[NR_CPUS];
55	55
56		−void call_do_softirq(void *sp);
57		−void call_do_irq(struct pt_regs regs, void sp);
58	56	extern void do_IRQ(struct pt_regs *regs);
59	57	extern void __init init_IRQ(void);
60	58	extern void __do_irq(struct pt_regs *regs);

arch/powerpc/include/asm/jump_label.h+6−15

@@ -20,7 +20,8 @@ static __always_inline bool arch_static_branch(struct static_key *key, bool bran · 19 unmodified lines
20	20	asm_volatile_goto("1:\n\t"
21	21	"nop # arch_static_branch\n\t"
22	22	".pushsection __jump_table, \"aw\"\n\t"
23		− JUMP_ENTRY_TYPE "1b, %l[l_yes], %c0\n\t"
	23	+ ".long 1b - ., %l[l_yes] - .\n\t"
	24	+ JUMP_ENTRY_TYPE "%c0 - .\n\t"
24	25	".popsection \n\t"
25	26	: : "i" (&((char *)key)[branch]) : : l_yes);
26	27
@@ -34,7 +35,8 @@ static __always_inline bool arch_static_branch_jump(struct static_key *key, bool · 7 unmodified lines
34	35	asm_volatile_goto("1:\n\t"
35	36	"b %l[l_yes] # arch_static_branch_jump\n\t"
36	37	".pushsection __jump_table, \"aw\"\n\t"
37		− JUMP_ENTRY_TYPE "1b, %l[l_yes], %c0\n\t"
	38	+ ".long 1b - ., %l[l_yes] - .\n\t"
	39	+ JUMP_ENTRY_TYPE "%c0 - .\n\t"
38	40	".popsection \n\t"
39	41	: : "i" (&((char *)key)[branch]) : : l_yes);
40	42
@@ -43,23 +45,12 @@ l_yes: · 2 unmodified lines
43	45	return true;
44	46	}
45	47
46		−#ifdef CONFIG_PPC64
47		−typedef u64 jump_label_t;
48		−#else
49		−typedef u32 jump_label_t;
50		−#endif
51		−
52		−struct jump_entry {
53		− jump_label_t code;
54		− jump_label_t target;
55		− jump_label_t key;
56		−};
57		−
58	48	#else
59	49	#define ARCH_STATIC_BRANCH(LABEL, KEY) \
60	50	1098: nop; \
61	51	.pushsection __jump_table, "aw"; \
62		− FTR_ENTRY_LONG 1098b, LABEL, KEY; \
	52	+ .long 1098b - ., LABEL - .; \
	53	+ FTR_ENTRY_LONG KEY; \
63	54	.popsection
64	55	#endif
65	56

arch/powerpc/include/asm/kasan.h+1−1

@@ -19,7 +19,7 @@ · 18 unmodified lines
19	19
20	20	#define KASAN_SHADOW_SCALE_SHIFT 3
21	21
22		−#if defined(CONFIG_PPC_BOOK3S_32) && defined(CONFIG_MODULES) && defined(CONFIG_STRICT_KERNEL_RWX)
	22	+#ifdef CONFIG_MODULES
23	23	#define KASAN_KERN_START ALIGN_DOWN(PAGE_OFFSET - SZ_256M, SZ_256M)
24	24	#else
25	25	#define KASAN_KERN_START PAGE_OFFSET

arch/powerpc/include/asm/kfence.h+33−0

@@ -0,0 +1,33 @@
	1	+/* SPDX-License-Identifier: GPL-2.0 */
	2	+/*
	3	+ * powerpc KFENCE support.
	4	+ *
	5	+ * Copyright (C) 2020 CS GROUP France
	6	+ */
	7	+
	8	+#ifndef __ASM_POWERPC_KFENCE_H
	9	+#define __ASM_POWERPC_KFENCE_H
	10	+
	11	+#include <linux/mm.h>
	12	+#include <asm/pgtable.h>
	13	+
	14	+static inline bool arch_kfence_init_pool(void)
	15	+{
	16	+ return true;
	17	+}
	18	+
	19	+static inline bool kfence_protect_page(unsigned long addr, bool protect)
	20	+{
	21	+ pte_t *kpte = virt_to_kpte(addr);
	22	+
	23	+ if (protect) {
	24	+ pte_update(&init_mm, addr, kpte, _PAGE_PRESENT, 0, 0);
	25	+ flush_tlb_kernel_range(addr, addr + PAGE_SIZE);
	26	+ } else {
	27	+ pte_update(&init_mm, addr, kpte, 0, _PAGE_PRESENT, 0);
	28	+ }
	29	+
	30	+ return true;
	31	+}
	32	+
	33	+#endif /* __ASM_POWERPC_KFENCE_H */

arch/powerpc/include/asm/kup.h+17−10

@@ -28,15 +28,6 @@ · 27 unmodified lines
28	28
29	29	#ifdef __ASSEMBLY__
30	30	#ifndef CONFIG_PPC_KUAP
31		−.macro kuap_save_and_lock sp, thread, gpr1, gpr2, gpr3
32		−.endm
33		−
34		−.macro kuap_restore sp, current, gpr1, gpr2, gpr3
35		−.endm
36		−
37		−.macro kuap_check current, gpr
38		−.endm
39		−
40	31	.macro kuap_check_amr gpr1, gpr2
41	32	.endm
42	33
@@ -55,6 +46,14 @@ void setup_kuep(bool disabled); · 12 unmodified lines
55	46	static inline void setup_kuep(bool disabled) { }
56	47	#endif /* CONFIG_PPC_KUEP */
57	48
	49	+#if defined(CONFIG_PPC_KUEP) && defined(CONFIG_PPC_BOOK3S_32)
	50	+void kuep_lock(void);
	51	+void kuep_unlock(void);
	52	+#else
	53	+static inline void kuep_lock(void) { }
	54	+static inline void kuep_unlock(void) { }
	55	+#endif
	56	+
58	57	#ifdef CONFIG_PPC_KUAP
59	58	void setup_kuap(bool disabled);
60	59	#else
@@ -66,7 +65,15 @@ bad_kuap_fault(struct pt_regs *regs, unsigned long address, bool is_write) · 5 unmodified lines
66	65	return false;
67	66	}
68	67
69		−static inline void kuap_check_amr(void) { }
	68	+static inline void kuap_assert_locked(void) { }
	69	+static inline void kuap_save_and_lock(struct pt_regs *regs) { }
	70	+static inline void kuap_user_restore(struct pt_regs *regs) { }
	71	+static inline void kuap_kernel_restore(struct pt_regs *regs, unsigned long amr) { }
	72	+
	73	+static inline unsigned long kuap_get_and_assert_locked(void)
	74	+{
	75	+ return 0;
	76	+}
70	77
71	78	/*
72	79	* book3s/64/kup-radix.h defines these functions for the !KUAP case to flush

arch/powerpc/include/asm/kvm_book3s.h+2−0

@@ -258,6 +258,8 @@ extern long kvmppc_hv_get_dirty_log_hpt(struct kvm *kvm, · 257 unmodified lines
258	258	extern void kvmppc_harvest_vpa_dirty(struct kvmppc_vpa *vpa,
259	259	struct kvm_memory_slot *memslot,
260	260	unsigned long *map);
	261	+extern unsigned long kvmppc_filter_lpcr_hv(struct kvm *kvm,
	262	+ unsigned long lpcr);
261	263	extern void kvmppc_update_lpcr(struct kvm *kvm, unsigned long lpcr,
262	264	unsigned long mask);
263	265	extern void kvmppc_set_fscr(struct kvm_vcpu *vcpu, u64 fscr);

arch/powerpc/include/asm/kvm_ppc.h+1−2

@@ -767,8 +767,7 @@ long kvmppc_h_remove(struct kvm_vcpu *vcpu, unsigned long flags, · 766 unmodified lines
767	767	unsigned long pte_index, unsigned long avpn);
768	768	long kvmppc_h_bulk_remove(struct kvm_vcpu *vcpu);
769	769	long kvmppc_h_protect(struct kvm_vcpu *vcpu, unsigned long flags,
770		− unsigned long pte_index, unsigned long avpn,
771		− unsigned long va);
	770	+ unsigned long pte_index, unsigned long avpn);
772	771	long kvmppc_h_read(struct kvm_vcpu *vcpu, unsigned long flags,
773	772	unsigned long pte_index);
774	773	long kvmppc_h_clear_ref(struct kvm_vcpu *vcpu, unsigned long flags,

arch/powerpc/include/asm/mmu_context.h+1−1

@@ -263,7 +263,7 @@ extern void arch_exit_mmap(struct mm_struct *mm); · 262 unmodified lines
263	263	static inline void arch_unmap(struct mm_struct *mm,
264	264	unsigned long start, unsigned long end)
265	265	{
266		− unsigned long vdso_base = (unsigned long)mm->context.vdso - PAGE_SIZE;
	266	+ unsigned long vdso_base = (unsigned long)mm->context.vdso;
267	267
268	268	if (start <= vdso_base && vdso_base < end)
269	269	mm->context.vdso = NULL;

arch/powerpc/include/asm/nohash/32/kup-8xx.h+32−24

@@ -7,33 +7,41 @@ · 6 unmodified lines
7	7
8	8	#ifdef CONFIG_PPC_KUAP
9	9
10		−#ifdef __ASSEMBLY__
11		−
12		−.macro kuap_save_and_lock sp, thread, gpr1, gpr2, gpr3
13		− lis \gpr2, MD_APG_KUAP@h /* only APG0 and APG1 are used */
14		− mfspr \gpr1, SPRN_MD_AP
15		− mtspr SPRN_MD_AP, \gpr2
16		− stw \gpr1, STACK_REGS_KUAP(\sp)
17		−.endm
18		−
19		−.macro kuap_restore sp, current, gpr1, gpr2, gpr3
20		− lwz \gpr1, STACK_REGS_KUAP(\sp)
21		− mtspr SPRN_MD_AP, \gpr1
22		−.endm
23		−
24		−.macro kuap_check current, gpr
25		−#ifdef CONFIG_PPC_KUAP_DEBUG
26		− mfspr \gpr, SPRN_MD_AP
27		− rlwinm \gpr, \gpr, 16, 0xffff
28		−999: twnei \gpr, MD_APG_KUAP@h
29		− EMIT_BUG_ENTRY 999b, __FILE__, __LINE__, (BUGFLAG_WARNING \| BUGFLAG_ONCE)
30		−#endif
31		−.endm
32		−
33		−#else /* !__ASSEMBLY__ */
	10	+#ifndef __ASSEMBLY__
34	11
35	12	#include <asm/reg.h>
36	13
	14	+static inline void kuap_save_and_lock(struct pt_regs *regs)
	15	+{
	16	+ regs->kuap = mfspr(SPRN_MD_AP);
	17	+ mtspr(SPRN_MD_AP, MD_APG_KUAP);
	18	+}
	19	+
	20	+static inline void kuap_user_restore(struct pt_regs *regs)
	21	+{
	22	+}
	23	+
	24	+static inline void kuap_kernel_restore(struct pt_regs *regs, unsigned long kuap)
	25	+{
	26	+ mtspr(SPRN_MD_AP, regs->kuap);
	27	+}
	28	+
	29	+static inline unsigned long kuap_get_and_assert_locked(void)
	30	+{
	31	+ unsigned long kuap = mfspr(SPRN_MD_AP);
	32	+
	33	+ if (IS_ENABLED(CONFIG_PPC_KUAP_DEBUG))
	34	+ WARN_ON_ONCE(kuap >> 16 != MD_APG_KUAP >> 16);
	35	+
	36	+ return kuap;
	37	+}
	38	+
	39	+static inline void kuap_assert_locked(void)
	40	+{
	41	+ if (IS_ENABLED(CONFIG_PPC_KUAP_DEBUG))
	42	+ kuap_get_and_assert_locked();
	43	+}
	44	+
37	45	static inline void allow_user_access(void __user to, const void __user from,
38	46	unsigned long size, unsigned long dir)
39	47	{

arch/powerpc/include/asm/nohash/32/mmu-8xx.h+3−0

@@ -172,6 +172,9 @@ · 171 unmodified lines
172	172
173	173	#define mmu_linear_psize MMU_PAGE_8M
174	174
	175	+#define MODULES_VADDR (PAGE_OFFSET - SZ_256M)
	176	+#define MODULES_END PAGE_OFFSET
	177	+
175	178	#ifndef __ASSEMBLY__
176	179
177	180	#include <linux/mmdebug.h>

arch/powerpc/include/asm/nohash/64/pgtable.h+4−1

@@ -6,6 +6,8 @@ · 5 unmodified lines
6	6	* the ppc64 non-hashed page table.
7	7	*/
8	8
	9	+#include <linux/sizes.h>
	10	+
9	11	#include <asm/nohash/64/pgtable-4k.h>
10	12	#include <asm/barrier.h>
11	13	#include <asm/asm-const.h>
@@ -54,7 +56,8 @@ · 42 unmodified lines
54	56	#define PHB_IO_END (KERN_IO_START + FULL_IO_SIZE)
55	57	#define IOREMAP_BASE (PHB_IO_END)
56	58	#define IOREMAP_START (ioremap_bot)
57		−#define IOREMAP_END (KERN_VIRT_START + KERN_VIRT_SIZE)
	59	+#define IOREMAP_END (KERN_VIRT_START + KERN_VIRT_SIZE - FIXADDR_SIZE)
	60	+#define FIXADDR_SIZE SZ_32M
58	61
59	62
60	63	/*

arch/powerpc/include/asm/opal.h+1−1

@@ -307,7 +307,7 @@ int opal_secvar_enqueue_update(const char key, uint64_t key_len, u8 data, · 306 unmodified lines
307	307
308	308	s64 opal_mpipl_update(enum opal_mpipl_ops op, u64 src, u64 dest, u64 size);
309	309	s64 opal_mpipl_register_tag(enum opal_mpipl_tags tag, u64 addr);
310		−s64 opal_mpipl_query_tag(enum opal_mpipl_tags tag, u64 *addr);
	310	+s64 opal_mpipl_query_tag(enum opal_mpipl_tags tag, __be64 *addr);
311	311
312	312	s64 opal_signal_system_reset(s32 cpu);
313	313	s64 opal_quiesce(u64 shutdown_type, s32 cpu);

arch/powerpc/include/asm/perf_event_server.h+7−1

@@ -43,7 +43,7 @@ struct power_pmu { · 42 unmodified lines
43	43	u64 alt[]);
44	44	void (get_mem_data_src)(union perf_mem_data_src dsrc,
45	45	u32 flags, struct pt_regs *regs);
46		− void (get_mem_weight)(u64 weight);
	46	+ void (get_mem_weight)(u64 weight, u64 type);
47	47	unsigned long group_constraint_mask;
48	48	unsigned long group_constraint_val;
49	49	u64 (*bhrb_filter_map)(u64 branch_sample_type);
@@ -67,6 +67,12 @@ struct power_pmu { · 17 unmodified lines
67	67	* the pmu supports extended perf regs capability
68	68	*/
69	69	int capabilities;
	70	+ /*
	71	+ * Function to check event code for values which are
	72	+ * reserved. Function takes struct perf_event as input,
	73	+ * since event code could be spread in attr.config*
	74	+ */
	75	+ int (check_attr_config)(struct perf_event ev);
70	76	};
71	77
72	78	/*

arch/powerpc/include/asm/pgtable.h+0−2

@@ -41,8 +41,6 @@ struct mm_struct; · 40 unmodified lines
41	41
42	42	#ifndef __ASSEMBLY__
43	43
44		−#include <asm/tlbflush.h>
45		−
46	44	/* Keep these as a macros to avoid include dependency mess */
47	45	#define pte_page(x) pfn_to_page(pte_pfn(x))
48	46	#define mk_pte(page, pgprot) pfn_pte(page_to_pfn(page), (pgprot))

arch/powerpc/include/asm/ppc-opcode.h+13−0

@@ -265,6 +265,7 @@ · 264 unmodified lines
265	265	#define PPC_INST_ORI 0x60000000
266	266	#define PPC_INST_ORIS 0x64000000
267	267	#define PPC_INST_BRANCH 0x48000000
	268	+#define PPC_INST_BL 0x48000001
268	269	#define PPC_INST_BRANCH_COND 0x40800000
269	270
270	271	/* Prefixes */
@@ -437,6 +438,9 @@ · 166 unmodified lines
437	438	#define PPC_RAW_STFDX(s, a, b) (0x7c0005ae \| ___PPC_RS(s) \| ___PPC_RA(a) \| ___PPC_RB(b))
438	439	#define PPC_RAW_LVX(t, a, b) (0x7c0000ce \| ___PPC_RT(t) \| ___PPC_RA(a) \| ___PPC_RB(b))
439	440	#define PPC_RAW_STVX(s, a, b) (0x7c0001ce \| ___PPC_RS(s) \| ___PPC_RA(a) \| ___PPC_RB(b))
	441	+#define PPC_RAW_ADDE(t, a, b) (0x7c000114 \| ___PPC_RT(t) \| ___PPC_RA(a) \| ___PPC_RB(b))
	442	+#define PPC_RAW_ADDZE(t, a) (0x7c000194 \| ___PPC_RT(t) \| ___PPC_RA(a))
	443	+#define PPC_RAW_ADDME(t, a) (0x7c0001d4 \| ___PPC_RT(t) \| ___PPC_RA(a))
440	444	#define PPC_RAW_ADD(t, a, b) (PPC_INST_ADD \| ___PPC_RT(t) \| ___PPC_RA(a) \| ___PPC_RB(b))
441	445	#define PPC_RAW_ADD_DOT(t, a, b) (PPC_INST_ADD \| ___PPC_RT(t) \| ___PPC_RA(a) \| ___PPC_RB(b) \| 0x1)
442	446	#define PPC_RAW_ADDC(t, a, b) (0x7c000014 \| ___PPC_RT(t) \| ___PPC_RA(a) \| ___PPC_RB(b))
@@ -445,11 +449,14 @@ · 2 unmodified lines
445	449	#define PPC_RAW_BLR() (PPC_INST_BLR)
446	450	#define PPC_RAW_BLRL() (0x4e800021)
447	451	#define PPC_RAW_MTLR(r) (0x7c0803a6 \| ___PPC_RT(r))
	452	+#define PPC_RAW_MFLR(t) (PPC_INST_MFLR \| ___PPC_RT(t))
448	453	#define PPC_RAW_BCTR() (PPC_INST_BCTR)
449	454	#define PPC_RAW_MTCTR(r) (PPC_INST_MTCTR \| ___PPC_RT(r))
450	455	#define PPC_RAW_ADDI(d, a, i) (PPC_INST_ADDI \| ___PPC_RT(d) \| ___PPC_RA(a) \| IMM_L(i))
451	456	#define PPC_RAW_LI(r, i) PPC_RAW_ADDI(r, 0, i)
452	457	#define PPC_RAW_ADDIS(d, a, i) (PPC_INST_ADDIS \| ___PPC_RT(d) \| ___PPC_RA(a) \| IMM_L(i))
	458	+#define PPC_RAW_ADDIC(d, a, i) (0x30000000 \| ___PPC_RT(d) \| ___PPC_RA(a) \| IMM_L(i))
	459	+#define PPC_RAW_ADDIC_DOT(d, a, i) (0x34000000 \| ___PPC_RT(d) \| ___PPC_RA(a) \| IMM_L(i))
453	460	#define PPC_RAW_LIS(r, i) PPC_RAW_ADDIS(r, 0, i)
454	461	#define PPC_RAW_STDX(r, base, b) (0x7c00012a \| ___PPC_RS(r) \| ___PPC_RA(base) \| ___PPC_RB(b))
455	462	#define PPC_RAW_STDU(r, base, i) (0xf8000001 \| ___PPC_RS(r) \| ___PPC_RA(base) \| ((i) & 0xfffc))
@@ -472,6 +479,10 @@ · 16 unmodified lines
472	479	#define PPC_RAW_CMPLW(a, b) (0x7c000040 \| ___PPC_RA(a) \| ___PPC_RB(b))
473	480	#define PPC_RAW_CMPLD(a, b) (0x7c200040 \| ___PPC_RA(a) \| ___PPC_RB(b))
474	481	#define PPC_RAW_SUB(d, a, b) (0x7c000050 \| ___PPC_RT(d) \| ___PPC_RB(a) \| ___PPC_RA(b))
	482	+#define PPC_RAW_SUBFC(d, a, b) (0x7c000010 \| ___PPC_RT(d) \| ___PPC_RA(a) \| ___PPC_RB(b))
	483	+#define PPC_RAW_SUBFE(d, a, b) (0x7c000110 \| ___PPC_RT(d) \| ___PPC_RA(a) \| ___PPC_RB(b))
	484	+#define PPC_RAW_SUBFIC(d, a, i) (0x20000000 \| ___PPC_RT(d) \| ___PPC_RA(a) \| IMM_L(i))
	485	+#define PPC_RAW_SUBFZE(d, a) (0x7c000190 \| ___PPC_RT(d) \| ___PPC_RA(a))
475	486	#define PPC_RAW_MULD(d, a, b) (0x7c0001d2 \| ___PPC_RT(d) \| ___PPC_RA(a) \| ___PPC_RB(b))
476	487	#define PPC_RAW_MULW(d, a, b) (0x7c0001d6 \| ___PPC_RT(d) \| ___PPC_RA(a) \| ___PPC_RB(b))
477	488	#define PPC_RAW_MULHWU(d, a, b) (0x7c000016 \| ___PPC_RT(d) \| ___PPC_RA(a) \| ___PPC_RB(b))
@@ -484,11 +495,13 @@ · 6 unmodified lines
484	495	#define PPC_RAW_DIVDEU_DOT(t, a, b) (0x7c000312 \| ___PPC_RT(t) \| ___PPC_RA(a) \| ___PPC_RB(b) \| 0x1)
485	496	#define PPC_RAW_AND(d, a, b) (0x7c000038 \| ___PPC_RA(d) \| ___PPC_RS(a) \| ___PPC_RB(b))
486	497	#define PPC_RAW_ANDI(d, a, i) (0x70000000 \| ___PPC_RA(d) \| ___PPC_RS(a) \| IMM_L(i))
	498	+#define PPC_RAW_ANDIS(d, a, i) (0x74000000 \| ___PPC_RA(d) \| ___PPC_RS(a) \| IMM_L(i))
487	499	#define PPC_RAW_AND_DOT(d, a, b) (0x7c000039 \| ___PPC_RA(d) \| ___PPC_RS(a) \| ___PPC_RB(b))
488	500	#define PPC_RAW_OR(d, a, b) (0x7c000378 \| ___PPC_RA(d) \| ___PPC_RS(a) \| ___PPC_RB(b))
489	501	#define PPC_RAW_MR(d, a) PPC_RAW_OR(d, a, a)
490	502	#define PPC_RAW_ORI(d, a, i) (PPC_INST_ORI \| ___PPC_RA(d) \| ___PPC_RS(a) \| IMM_L(i))
491	503	#define PPC_RAW_ORIS(d, a, i) (PPC_INST_ORIS \| ___PPC_RA(d) \| ___PPC_RS(a) \| IMM_L(i))
	504	+#define PPC_RAW_NOR(d, a, b) (0x7c0000f8 \| ___PPC_RA(d) \| ___PPC_RS(a) \| ___PPC_RB(b))
492	505	#define PPC_RAW_XOR(d, a, b) (0x7c000278 \| ___PPC_RA(d) \| ___PPC_RS(a) \| ___PPC_RB(b))
493	506	#define PPC_RAW_XORI(d, a, i) (0x68000000 \| ___PPC_RA(d) \| ___PPC_RS(a) \| IMM_L(i))
494	507	#define PPC_RAW_XORIS(d, a, i) (0x6c000000 \| ___PPC_RA(d) \| ___PPC_RS(a) \| IMM_L(i))

arch/powerpc/include/asm/ppc_asm.h+0−30

@@ -15,36 +15,6 @@ · 14 unmodified lines
15	15
16	16	#define SZL (BITS_PER_LONG/8)
17	17
18		−/*
19		− * Stuff for accurate CPU time accounting.
20		− * These macros handle transitions between user and system state
21		− * in exception entry and exit and accumulate time to the
22		− * user_time and system_time fields in the paca.
23		− */
24		−
25		−#ifndef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
26		−#define ACCOUNT_CPU_USER_ENTRY(ptr, ra, rb)
27		−#define ACCOUNT_CPU_USER_EXIT(ptr, ra, rb)
28		−#else
29		−#define ACCOUNT_CPU_USER_ENTRY(ptr, ra, rb) \
30		− MFTB(ra); /* get timebase */ \
31		− PPC_LL rb, ACCOUNT_STARTTIME_USER(ptr); \
32		− PPC_STL ra, ACCOUNT_STARTTIME(ptr); \
33		− subf rb,rb,ra; /* subtract start value */ \
34		− PPC_LL ra, ACCOUNT_USER_TIME(ptr); \
35		− add ra,ra,rb; /* add on to user time */ \
36		− PPC_STL ra, ACCOUNT_USER_TIME(ptr); \
37		−
38		−#define ACCOUNT_CPU_USER_EXIT(ptr, ra, rb) \
39		− MFTB(ra); /* get timebase */ \
40		− PPC_LL rb, ACCOUNT_STARTTIME(ptr); \
41		− PPC_STL ra, ACCOUNT_STARTTIME_USER(ptr); \
42		− subf rb,rb,ra; /* subtract start value */ \
43		− PPC_LL ra, ACCOUNT_SYSTEM_TIME(ptr); \
44		− add ra,ra,rb; /* add on to system time */ \
45		− PPC_STL ra, ACCOUNT_SYSTEM_TIME(ptr)
46		−#endif /* CONFIG_VIRT_CPU_ACCOUNTING_NATIVE */
47		−
48	18	/*
49	19	* Macros for storing registers into and loading registers from
50	20	* exception frames.

arch/powerpc/include/asm/processor.h+4−5

@@ -144,15 +144,12 @@ struct thread_struct { · 143 unmodified lines
144	144	#endif
145	145	#ifdef CONFIG_PPC32
146	146	void pgdir; / root of page-table tree */
147		− unsigned long ksp_limit; /* if ksp <= ksp_limit stack overflow */
148	147	#ifdef CONFIG_PPC_RTAS
149	148	unsigned long rtas_sp; /* stack pointer for when in RTAS */
150	149	#endif
151		−#endif
152	150	#if defined(CONFIG_PPC_BOOK3S_32) && defined(CONFIG_PPC_KUAP)
153	151	unsigned long kuap; /* opened segments for user access */
154	152	#endif
155		−#ifdef CONFIG_VMAP_STACK
156	153	unsigned long srr0;
157	154	unsigned long srr1;
158	155	unsigned long dar;
@@ -161,7 +158,7 @@ struct thread_struct { · 2 unmodified lines
161	158	unsigned long r0, r3, r4, r5, r6, r8, r9, r11;
162	159	unsigned long lr, ctr;
163	160	#endif
164		−#endif
	161	+#endif /* CONFIG_PPC32 */
165	162	/* Debug Registers */
166	163	struct debug_reg debug;
167	164	#ifdef CONFIG_PPC_FPU_REGS
@@ -282,7 +279,6 @@ struct thread_struct { · 114 unmodified lines
282	279	#ifdef CONFIG_PPC32
283	280	#define INIT_THREAD { \
284	281	.ksp = INIT_SP, \
285		− .ksp_limit = INIT_SP_LIMIT, \
286	282	.pgdir = swapper_pg_dir, \
287	283	.fpexc_mode = MSR_FE0 \| MSR_FE1, \
288	284	SPEFSCR_INIT \
@@ -393,6 +389,7 @@ extern unsigned long isa300_idle_stop_mayloss(unsigned long psscr_val); · 104 unmodified lines
393	389	extern unsigned long isa206_idle_insn_mayloss(unsigned long type);
394	390	#ifdef CONFIG_PPC_970_NAP
395	391	extern void power4_idle_nap(void);
	392	+void power4_idle_nap_return(void);
396	393	#endif
397	394
398	395	extern unsigned long cpuidle_disable;
@@ -417,6 +414,8 @@ extern int fix_alignment(struct pt_regs *); · 18 unmodified lines
417	414	#define NET_IP_ALIGN 0
418	415	#endif
419	416
	417	+int do_mathemu(struct pt_regs *regs);
	418	+
420	419	#endif /* __KERNEL__ */
421	420	#endif /* __ASSEMBLY__ */
422	421	#endif /* _ASM_POWERPC_PROCESSOR_H */

arch/powerpc/include/asm/ptrace.h+14−31

@@ -185,44 +185,27 @@ static inline void regs_set_return_value(struct pt_regs *regs, unsigned long rc) · 184 unmodified lines
185	185	#define current_pt_regs() \
186	186	((struct pt_regs *)((unsigned long)task_stack_page(current) + THREAD_SIZE) - 1)
187	187
	188	+/*
	189	+ * The 4 low bits (0xf) are available as flags to overload the trap word,
	190	+ * because interrupt vectors have minimum alignment of 0x10. TRAP_FLAGS_MASK
	191	+ * must cover the bits used as flags, including bit 0 which is used as the
	192	+ * "norestart" bit.
	193	+ */
188	194	#ifdef __powerpc64__
189		−#ifdef CONFIG_PPC_BOOK3S
190		−#define TRAP_FLAGS_MASK 0x10
191		−#define TRAP(regs) ((regs)->trap & ~TRAP_FLAGS_MASK)
192		−#define FULL_REGS(regs) true
193		−#define SET_FULL_REGS(regs) do { } while (0)
194		−#else
195		−#define TRAP_FLAGS_MASK 0x11
196		−#define TRAP(regs) ((regs)->trap & ~TRAP_FLAGS_MASK)
197		−#define FULL_REGS(regs) (((regs)->trap & 1) == 0)
198		−#define SET_FULL_REGS(regs) ((regs)->trap &= ~1)
199		−#endif
200		−#define CHECK_FULL_REGS(regs) BUG_ON(!FULL_REGS(regs))
201		−#define NV_REG_POISON 0xdeadbeefdeadbeefUL
	195	+#define TRAP_FLAGS_MASK 0x1
202	196	#else
203	197	/*
204		− * We use the least-significant bit of the trap field to indicate
205		− * whether we have saved the full set of registers, or only a
206		− * partial set. A 1 there means the partial set.
207		− * On 4xx we use the next bit to indicate whether the exception
	198	+ * On 4xx we use bit 1 in the trap word to indicate whether the exception
208	199	* is a critical exception (1 means it is).
209	200	*/
210		−#define TRAP_FLAGS_MASK 0x1F
211		−#define TRAP(regs) ((regs)->trap & ~TRAP_FLAGS_MASK)
212		−#define FULL_REGS(regs) (((regs)->trap & 1) == 0)
213		−#define SET_FULL_REGS(regs) ((regs)->trap &= ~1)
	201	+#define TRAP_FLAGS_MASK 0xf
214	202	#define IS_CRITICAL_EXC(regs) (((regs)->trap & 2) != 0)
215	203	#define IS_MCHECK_EXC(regs) (((regs)->trap & 4) != 0)
216	204	#define IS_DEBUG_EXC(regs) (((regs)->trap & 8) != 0)
217		−#define NV_REG_POISON 0xdeadbeef
218		−#define CHECK_FULL_REGS(regs) \
219		−do { \
220		− if ((regs)->trap & 1) \
221		− printk(KERN_CRIT "%s: partial register set\n", __func__); \
222		−} while (0)
223	205	#endif /* __powerpc64__ */
	206	+#define TRAP(regs) ((regs)->trap & ~TRAP_FLAGS_MASK)
224	207
225		−static inline void set_trap(struct pt_regs *regs, unsigned long val)
	208	+static __always_inline void set_trap(struct pt_regs *regs, unsigned long val)
226	209	{
227	210	regs->trap = (regs->trap & TRAP_FLAGS_MASK) \| (val & ~TRAP_FLAGS_MASK);
228	211	}
@@ -244,12 +227,12 @@ static inline bool trap_is_syscall(struct pt_regs *regs) · 15 unmodified lines
244	227
245	228	static inline bool trap_norestart(struct pt_regs *regs)
246	229	{
247		− return regs->trap & 0x10;
	230	+ return regs->trap & 0x1;
248	231	}
249	232
250		−static inline void set_trap_norestart(struct pt_regs *regs)
	233	+static __always_inline void set_trap_norestart(struct pt_regs *regs)
251	234	{
252		− regs->trap \|= 0x10;
	235	+ regs->trap \|= 0x1;
253	236	}
254	237
255	238	#define arch_has_single_step() (1)

arch/powerpc/include/asm/qspinlock.h+7−14

@@ -44,20 +44,6 @@ static __always_inline void queued_spin_lock(struct qspinlock *lock) · 43 unmodified lines
44	44	}
45	45	#define queued_spin_lock queued_spin_lock
46	46
47		−#define smp_mb__after_spinlock() smp_mb()
48		−
49		−static __always_inline int queued_spin_is_locked(struct qspinlock *lock)
50		−{
51		− /*
52		− * This barrier was added to simple spinlocks by commit 51d7d5205d338,
53		− * but it should now be possible to remove it, asm arm64 has done with
54		− * commit c6f5d02b6a0f.
55		− */
56		− smp_mb();
57		− return atomic_read(&lock->val);
58		−}
59		−#define queued_spin_is_locked queued_spin_is_locked
60		−
61	47	#ifdef CONFIG_PARAVIRT_SPINLOCKS
62	48	#define SPIN_THRESHOLD (1<<15) /* not tuned */
63	49
@@ -86,6 +72,13 @@ static inline void pv_spinlocks_init(void) · 22 unmodified lines
86	72
87	73	#endif
88	74
	75	+/*
	76	+ * Queued spinlocks rely heavily on smp_cond_load_relaxed() to busy-wait,
	77	+ * which was found to have performance problems if implemented with
	78	+ * the preferred spin_begin()/spin_end() SMT priority pattern. Use the
	79	+ * generic version instead.
	80	+ */
	81	+
89	82	#include <asm-generic/qspinlock.h>
90	83
91	84	#endif /* _ASM_POWERPC_QSPINLOCK_H */

arch/powerpc/include/asm/reg.h+3−3

@@ -124,7 +124,7 @@ · 123 unmodified lines
124	124	#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
125	125	#define MSR_TM_ACTIVE(x) (((x) & MSR_TS_MASK) != 0) /* Transaction active? */
126	126	#else
127		−#define MSR_TM_ACTIVE(x) 0
	127	+#define MSR_TM_ACTIVE(x) ((void)(x), 0)
128	128	#endif
129	129
130	130	#if defined(CONFIG_PPC_BOOK3S_64)
@@ -441,6 +441,7 @@ · 310 unmodified lines
441	441	#define LPCR_VRMA_LP1 ASM_CONST(0x0000800000000000)
442	442	#define LPCR_RMLS 0x1C000000 /* Implementation dependent RMO limit sel */
443	443	#define LPCR_RMLS_SH 26
	444	+#define LPCR_HAIL ASM_CONST(0x0000000004000000) /* HV AIL (ISAv3.1) */
444	445	#define LPCR_ILE ASM_CONST(0x0000000002000000) /* !HV irqs set MSR:LE */
445	446	#define LPCR_AIL ASM_CONST(0x0000000001800000) /* Alternate interrupt location */
446	447	#define LPCR_AIL_0 ASM_CONST(0x0000000000000000) /* MMU off exception offset 0x0 */
@@ -1393,8 +1394,7 @@ static inline void mtmsr_isync(unsigned long val) · 946 unmodified lines
1393	1394	: "r" ((unsigned long)(v)) \
1394	1395	: "memory")
1395	1396	#endif
1396		−#define wrtspr(rn) asm volatile("mtspr " __stringify(rn) ",0" : \
1397		− : : "memory")
	1397	+#define wrtspr(rn) asm volatile("mtspr " __stringify(rn) ",2" : : : "memory")
1398	1398
1399	1399	static inline void wrtee(unsigned long val)
1400	1400	{

arch/powerpc/include/asm/rtas.h+3−3

@@ -19,8 +19,8 @@ · 18 unmodified lines
19	19	#define RTAS_UNKNOWN_SERVICE (-1)
20	20	#define RTAS_INSTANTIATE_MAX (1ULL<<30) /* Don't instantiate rtas at/above this value */
21	21
22		−/* Buffer size for ppc_rtas system call. */
23		−#define RTAS_RMOBUF_MAX (64 * 1024)
	22	+/* Memory set aside for sys_rtas to use with calls that need a work area. */
	23	+#define RTAS_USER_REGION_SIZE (64 * 1024)
24	24
25	25	/* RTAS return status codes */
26	26	#define RTAS_BUSY -2 /* RTAS Busy */
@@ -357,7 +357,7 @@ extern void rtas_take_timebase(void); · 330 unmodified lines
357	357	static inline int page_is_rtas_user_buf(unsigned long pfn)
358	358	{
359	359	unsigned long paddr = (pfn << PAGE_SHIFT);
360		− if (paddr >= rtas_rmo_buf && paddr < (rtas_rmo_buf + RTAS_RMOBUF_MAX))
	360	+ if (paddr >= rtas_rmo_buf && paddr < (rtas_rmo_buf + RTAS_USER_REGION_SIZE))
361	361	return 1;
362	362	return 0;
363	363	}

arch/powerpc/include/asm/simple_spinlock.h+1−5

@@ -38,8 +38,7 @@ static __always_inline int arch_spin_value_unlocked(arch_spinlock_t lock) · 37 unmodified lines
38	38
39	39	static inline int arch_spin_is_locked(arch_spinlock_t *lock)
40	40	{
41		− smp_mb();
42		− return !arch_spin_value_unlocked(*lock);
	41	+ return !arch_spin_value_unlocked(READ_ONCE(*lock));
43	42	}
44	43
45	44	/*
@@ -282,7 +281,4 @@ static inline void arch_write_unlock(arch_rwlock_t *rw) · 236 unmodified lines
282	281	#define arch_read_relax(lock) rw_yield(lock)
283	282	#define arch_write_relax(lock) rw_yield(lock)
284	283
285		−/* See include/linux/spinlock.h */
286		−#define smp_mb__after_spinlock() smp_mb()
287		−
288	284	#endif /* _ASM_POWERPC_SIMPLE_SPINLOCK_H */

arch/powerpc/include/asm/smp.h+6−0

@@ -31,6 +31,7 @@ extern u32 *cpu_to_phys_id; · 30 unmodified lines
31	31	extern bool coregroup_enabled;
32	32
33	33	extern int cpu_to_chip_id(int cpu);
	34	+extern int *chip_id_lookup_table;
34	35
35	36	#ifdef CONFIG_SMP
36	37
@@ -121,6 +122,11 @@ static inline struct cpumask *cpu_sibling_mask(int cpu) · 84 unmodified lines
121	122	return per_cpu(cpu_sibling_map, cpu);
122	123	}
123	124
	125	+static inline struct cpumask *cpu_core_mask(int cpu)
	126	+{
	127	+ return per_cpu(cpu_core_map, cpu);
	128	+}
	129	+
124	130	static inline struct cpumask *cpu_l2_cache_mask(int cpu)
125	131	{
126	132	return per_cpu(cpu_l2_cache_map, cpu);

arch/powerpc/include/asm/spinlock.h+3−0

@@ -10,6 +10,9 @@ · 9 unmodified lines
10	10	#include <asm/simple_spinlock.h>
11	11	#endif
12	12
	13	+/* See include/linux/spinlock.h */
	14	+#define smp_mb__after_spinlock() smp_mb()
	15	+
13	16	#ifndef CONFIG_PARAVIRT_SPINLOCKS
14	17	static inline void pv_spinlocks_init(void) { }
15	18	#endif

arch/powerpc/include/asm/thread_info.h+6−1

@@ -38,7 +38,6 @@ · 37 unmodified lines
38	38	#ifndef __ASSEMBLY__
39	39	#include <linux/cache.h>
40	40	#include <asm/processor.h>
41		−#include <asm/page.h>
42	41	#include <asm/accounting.h>
43	42
44	43	#define SLB_PRELOAD_NR 16U
@@ -152,6 +151,12 @@ void arch_setup_new_exec(void); · 107 unmodified lines
152	151
153	152	#ifndef __ASSEMBLY__
154	153
	154	+static inline void clear_thread_local_flags(unsigned int flags)
	155	+{
	156	+ struct thread_info *ti = current_thread_info();
	157	+ ti->local_flags &= ~flags;
	158	+}
	159	+
155	160	static inline bool test_thread_local_flags(unsigned int flags)
156	161	{
157	162	struct thread_info *ti = current_thread_info();

arch/powerpc/include/asm/topology.h+1−1

@@ -126,7 +126,7 @@ static inline int cpu_to_coregroup_id(int cpu) · 125 unmodified lines
126	126	#define topology_physical_package_id(cpu) (cpu_to_chip_id(cpu))
127	127
128	128	#define topology_sibling_cpumask(cpu) (per_cpu(cpu_sibling_map, cpu))
129		−#define topology_core_cpumask(cpu) (cpu_cpu_mask(cpu))
	129	+#define topology_core_cpumask(cpu) (per_cpu(cpu_core_map, cpu))
130	130	#define topology_core_id(cpu) (cpu_to_core_id(cpu))
131	131
132	132	#endif

arch/powerpc/include/asm/uaccess.h+171−218

@@ -43,129 +43,39 @@ static inline bool __access_ok(unsigned long addr, unsigned long size) · 42 unmodified lines
43	43	* exception handling means that it's no longer "just"...)
44	44	*
45	45	*/
46		−#define get_user(x, ptr) \
47		− __get_user_check((x), (ptr), sizeof(*(ptr)))
48		−#define put_user(x, ptr) \
49		− __put_user_check((__typeof__((ptr)))(x), (ptr), sizeof((ptr)))
50		−
51		−#define __get_user(x, ptr) \
52		− __get_user_nocheck((x), (ptr), sizeof(*(ptr)), true)
53		−#define __put_user(x, ptr) \
54		− __put_user_nocheck((__typeof__((ptr)))(x), (ptr), sizeof((ptr)))
55		−
56		−#define __get_user_allowed(x, ptr) \
57		− __get_user_nocheck((x), (ptr), sizeof(*(ptr)), false)
58		−
59		−#define __get_user_inatomic(x, ptr) \
60		− __get_user_nosleep((x), (ptr), sizeof(*(ptr)))
61		−#define __put_user_inatomic(x, ptr) \
62		− __put_user_nosleep((__typeof__((ptr)))(x), (ptr), sizeof((ptr)))
63		−
64		−#ifdef CONFIG_PPC64
65		−
66		−#define ___get_user_instr(gu_op, dest, ptr) \
67		−({ \
68		− long __gui_ret = 0; \
69		− unsigned long __gui_ptr = (unsigned long)ptr; \
70		− struct ppc_inst __gui_inst; \
71		− unsigned int __prefix, __suffix; \
72		− __gui_ret = gu_op(__prefix, (unsigned int __user *)__gui_ptr); \
73		− if (__gui_ret == 0) { \
74		− if ((__prefix >> 26) == OP_PREFIX) { \
75		− __gui_ret = gu_op(__suffix, \
76		− (unsigned int __user *)__gui_ptr + 1); \
77		− __gui_inst = ppc_inst_prefix(__prefix, \
78		− __suffix); \
79		− } else { \
80		− __gui_inst = ppc_inst(__prefix); \
81		− } \
82		− if (__gui_ret == 0) \
83		− (dest) = __gui_inst; \
84		− } \
85		− __gui_ret; \
86		−})
87		−
88		−#define get_user_instr(x, ptr) \
89		− ___get_user_instr(get_user, x, ptr)
90		−
91		−#define __get_user_instr(x, ptr) \
92		− ___get_user_instr(__get_user, x, ptr)
93		−
94		−#define __get_user_instr_inatomic(x, ptr) \
95		− ___get_user_instr(__get_user_inatomic, x, ptr)
96		−
97		−#else /* !CONFIG_PPC64 */
98		−#define get_user_instr(x, ptr) \
99		− get_user((x).val, (u32 __user *)(ptr))
100		−
101		−#define __get_user_instr(x, ptr) \
102		− __get_user_nocheck((x).val, (u32 __user *)(ptr), sizeof(u32), true)
103		−
104		−#define __get_user_instr_inatomic(x, ptr) \
105		− __get_user_nosleep((x).val, (u32 __user *)(ptr), sizeof(u32))
106		−
107		−#endif /* CONFIG_PPC64 */
108		−
109		−extern long __put_user_bad(void);
110		−
111		−#define __put_user_size(x, ptr, size, retval) \
112		−do { \
113		− __label__ __pu_failed; \
114		− \
115		− retval = 0; \
116		− allow_write_to_user(ptr, size); \
117		− __put_user_size_goto(x, ptr, size, __pu_failed); \
118		− prevent_write_to_user(ptr, size); \
119		− break; \
120		− \
121		−__pu_failed: \
122		− retval = -EFAULT; \
123		− prevent_write_to_user(ptr, size); \
124		−} while (0)
125		−
126		−#define __put_user_nocheck(x, ptr, size) \
	46	+#define __put_user(x, ptr) \
127	47	({ \
128	48	long __pu_err; \
129	49	__typeof__((ptr)) __user __pu_addr = (ptr); \
130		− __typeof__(*(ptr)) __pu_val = (x); \
131		− __typeof__(size) __pu_size = (size); \
	50	+ __typeof__((ptr)) __pu_val = (__typeof__((ptr)))(x); \
	51	+ __typeof__(sizeof((ptr))) __pu_size = sizeof((ptr)); \
132	52	\
133		− if (!is_kernel_addr((unsigned long)__pu_addr)) \
134		− might_fault(); \
135		− __chk_user_ptr(__pu_addr); \
136		− __put_user_size(__pu_val, __pu_addr, __pu_size, __pu_err); \
	53	+ might_fault(); \
	54	+ do { \
	55	+ __label__ __pu_failed; \
	56	+ \
	57	+ allow_write_to_user(__pu_addr, __pu_size); \
	58	+ __put_user_size_goto(__pu_val, __pu_addr, __pu_size, __pu_failed); \
	59	+ prevent_write_to_user(__pu_addr, __pu_size); \
	60	+ __pu_err = 0; \
	61	+ break; \
	62	+ \
	63	+__pu_failed: \
	64	+ prevent_write_to_user(__pu_addr, __pu_size); \
	65	+ __pu_err = -EFAULT; \
	66	+ } while (0); \
137	67	\
138	68	__pu_err; \
139	69	})
140	70
141		−#define __put_user_check(x, ptr, size) \
	71	+#define put_user(x, ptr) \
142	72	({ \
143		− long __pu_err = -EFAULT; \
144		− __typeof__((ptr)) __user __pu_addr = (ptr); \
145		− __typeof__(*(ptr)) __pu_val = (x); \
146		− __typeof__(size) __pu_size = (size); \
147		− \
148		− might_fault(); \
149		− if (access_ok(__pu_addr, __pu_size)) \
150		− __put_user_size(__pu_val, __pu_addr, __pu_size, __pu_err); \
	73	+ __typeof__((ptr)) __user _pu_addr = (ptr); \
151	74	\
152		− __pu_err; \
	75	+ access_ok(_pu_addr, sizeof(*(ptr))) ? \
	76	+ __put_user(x, _pu_addr) : -EFAULT; \
153	77	})
154	78
155		−#define __put_user_nosleep(x, ptr, size) \
156		−({ \
157		− long __pu_err; \
158		− __typeof__((ptr)) __user __pu_addr = (ptr); \
159		− __typeof__(*(ptr)) __pu_val = (x); \
160		− __typeof__(size) __pu_size = (size); \
161		− \
162		− __chk_user_ptr(__pu_addr); \
163		− __put_user_size(__pu_val, __pu_addr, __pu_size, __pu_err); \
164		− \
165		− __pu_err; \
166		−})
167		−
168		−
169	79	/*
170	80	* We don't tell gcc that we are accessing memory, but this is OK
171	81	* because we do not write to any memory gcc knows about, so there
@@ -198,25 +108,17 @@ __pu_failed: \ · 26 unmodified lines
198	108
199	109	#define __put_user_size_goto(x, ptr, size, label) \
200	110	do { \
	111	+ __typeof__((ptr)) __user __pus_addr = (ptr); \
	112	+ \
201	113	switch (size) { \
202		− case 1: __put_user_asm_goto(x, ptr, label, "stb"); break; \
203		− case 2: __put_user_asm_goto(x, ptr, label, "sth"); break; \
204		− case 4: __put_user_asm_goto(x, ptr, label, "stw"); break; \
205		− case 8: __put_user_asm2_goto(x, ptr, label); break; \
206		− default: __put_user_bad(); \
	114	+ case 1: __put_user_asm_goto(x, __pus_addr, label, "stb"); break; \
	115	+ case 2: __put_user_asm_goto(x, __pus_addr, label, "sth"); break; \
	116	+ case 4: __put_user_asm_goto(x, __pus_addr, label, "stw"); break; \
	117	+ case 8: __put_user_asm2_goto(x, __pus_addr, label); break; \
	118	+ default: BUILD_BUG(); \
207	119	} \
208	120	} while (0)
209	121
210		−#define __unsafe_put_user_goto(x, ptr, size, label) \
211		−do { \
212		− __typeof__((ptr)) __user __pu_addr = (ptr); \
213		− __chk_user_ptr(ptr); \
214		− __put_user_size_goto((x), __pu_addr, (size), label); \
215		−} while (0)
216		−
217		−
218		−extern long __get_user_bad(void);
219		−
220	122	/*
221	123	* This does an atomic 128 byte aligned load from userspace.
222	124	* Upto caller to do enable_kernel_vmx() before calling!
@@ -234,6 +136,59 @@ extern long __get_user_bad(void); · 11 unmodified lines
234	136	: "=r" (err) \
235	137	: "b" (uaddr), "b" (kaddr), "i" (-EFAULT), "0" (err))
236	138
	139	+#ifdef CONFIG_CC_HAS_ASM_GOTO_OUTPUT
	140	+
	141	+#define __get_user_asm_goto(x, addr, label, op) \
	142	+ asm_volatile_goto( \
	143	+ "1: "op"%U1%X1 %0, %1 # get_user\n" \
	144	+ EX_TABLE(1b, %l2) \
	145	+ : "=r" (x) \
	146	+ : "m"UPD_CONSTR (*addr) \
	147	+ : \
	148	+ : label)
	149	+
	150	+#ifdef __powerpc64__
	151	+#define __get_user_asm2_goto(x, addr, label) \
	152	+ __get_user_asm_goto(x, addr, label, "ld")
	153	+#else /* __powerpc64__ */
	154	+#define __get_user_asm2_goto(x, addr, label) \
	155	+ asm_volatile_goto( \
	156	+ "1: lwz%X1 %0, %1\n" \
	157	+ "2: lwz%X1 %L0, %L1\n" \
	158	+ EX_TABLE(1b, %l2) \
	159	+ EX_TABLE(2b, %l2) \
	160	+ : "=r" (x) \
	161	+ : "m" (*addr) \
	162	+ : \
	163	+ : label)
	164	+#endif /* __powerpc64__ */
	165	+
	166	+#define __get_user_size_goto(x, ptr, size, label) \
	167	+do { \
	168	+ BUILD_BUG_ON(size > sizeof(x)); \
	169	+ switch (size) { \
	170	+ case 1: __get_user_asm_goto(x, (u8 __user *)ptr, label, "lbz"); break; \
	171	+ case 2: __get_user_asm_goto(x, (u16 __user *)ptr, label, "lhz"); break; \
	172	+ case 4: __get_user_asm_goto(x, (u32 __user *)ptr, label, "lwz"); break; \
	173	+ case 8: __get_user_asm2_goto(x, (u64 __user *)ptr, label); break; \
	174	+ default: x = 0; BUILD_BUG(); \
	175	+ } \
	176	+} while (0)
	177	+
	178	+#define __get_user_size_allowed(x, ptr, size, retval) \
	179	+do { \
	180	+ __label__ __gus_failed; \
	181	+ \
	182	+ __get_user_size_goto(x, ptr, size, __gus_failed); \
	183	+ retval = 0; \
	184	+ break; \
	185	+__gus_failed: \
	186	+ x = 0; \
	187	+ retval = -EFAULT; \
	188	+} while (0)
	189	+
	190	+#else /* CONFIG_CC_HAS_ASM_GOTO_OUTPUT */
	191	+
237	192	#define __get_user_asm(x, addr, err, op) \
238	193	__asm__ __volatile__( \
239	194	"1: "op"%U2%X2 %1, %2 # get_user\n" \
@@ -271,25 +226,27 @@ extern long __get_user_bad(void); · 31 unmodified lines
271	226	#define __get_user_size_allowed(x, ptr, size, retval) \
272	227	do { \
273	228	retval = 0; \
274		− __chk_user_ptr(ptr); \
275		− if (size > sizeof(x)) \
276		− (x) = __get_user_bad(); \
	229	+ BUILD_BUG_ON(size > sizeof(x)); \
277	230	switch (size) { \
278	231	case 1: __get_user_asm(x, (u8 __user *)ptr, retval, "lbz"); break; \
279	232	case 2: __get_user_asm(x, (u16 __user *)ptr, retval, "lhz"); break; \
280	233	case 4: __get_user_asm(x, (u32 __user *)ptr, retval, "lwz"); break; \
281	234	case 8: __get_user_asm2(x, (u64 __user *)ptr, retval); break; \
282		− default: (x) = __get_user_bad(); \
	235	+ default: x = 0; BUILD_BUG(); \
283	236	} \
284	237	} while (0)
285	238
286		−#define __get_user_size(x, ptr, size, retval) \
	239	+#define __get_user_size_goto(x, ptr, size, label) \
287	240	do { \
288		− allow_read_from_user(ptr, size); \
289		− __get_user_size_allowed(x, ptr, size, retval); \
290		− prevent_read_from_user(ptr, size); \
	241	+ long __gus_retval; \
	242	+ \
	243	+ __get_user_size_allowed(x, ptr, size, __gus_retval); \
	244	+ if (__gus_retval) \
	245	+ goto label; \
291	246	} while (0)
292	247
	248	+#endif /* CONFIG_CC_HAS_ASM_GOTO_OUTPUT */
	249	+
293	250	/*
294	251	* This is a type: either unsigned long, if the argument fits into
295	252	* that type, or otherwise unsigned long long.
@@ -297,86 +254,36 @@ do { \ · 1 unmodified line
297	254	#define __long_type(x) \
298	255	__typeof__(__builtin_choose_expr(sizeof(x) > sizeof(0UL), 0ULL, 0UL))
299	256
300		−#define __get_user_nocheck(x, ptr, size, do_allow) \
	257	+#define __get_user(x, ptr) \
301	258	({ \
302	259	long __gu_err; \
303	260	__long_type(*(ptr)) __gu_val; \
304	261	__typeof__((ptr)) __user __gu_addr = (ptr); \
305		− __typeof__(size) __gu_size = (size); \
	262	+ __typeof__(sizeof((ptr))) __gu_size = sizeof((ptr)); \
306	263	\
307		− __chk_user_ptr(__gu_addr); \
308		− if (do_allow && !is_kernel_addr((unsigned long)__gu_addr)) \
309		− might_fault(); \
310		− if (do_allow) \
311		− __get_user_size(__gu_val, __gu_addr, __gu_size, __gu_err); \
312		− else \
313		− __get_user_size_allowed(__gu_val, __gu_addr, __gu_size, __gu_err); \
	264	+ might_fault(); \
	265	+ allow_read_from_user(__gu_addr, __gu_size); \
	266	+ __get_user_size_allowed(__gu_val, __gu_addr, __gu_size, __gu_err); \
	267	+ prevent_read_from_user(__gu_addr, __gu_size); \
314	268	(x) = (__typeof__(*(ptr)))__gu_val; \
315	269	\
316	270	__gu_err; \
317	271	})
318	272
319		−#define __get_user_check(x, ptr, size) \
	273	+#define get_user(x, ptr) \
320	274	({ \
321		− long __gu_err = -EFAULT; \
322		− __long_type(*(ptr)) __gu_val = 0; \
323		− __typeof__((ptr)) __user __gu_addr = (ptr); \
324		− __typeof__(size) __gu_size = (size); \
325		− \
326		− might_fault(); \
327		− if (access_ok(__gu_addr, __gu_size)) \
328		− __get_user_size(__gu_val, __gu_addr, __gu_size, __gu_err); \
329		− (x) = (__force __typeof__(*(ptr)))__gu_val; \
	275	+ __typeof__((ptr)) __user _gu_addr = (ptr); \
330	276	\
331		− __gu_err; \
	277	+ access_ok(_gu_addr, sizeof(*(ptr))) ? \
	278	+ __get_user(x, _gu_addr) : \
	279	+ ((x) = (__force __typeof__(*(ptr)))0, -EFAULT); \
332	280	})
333	281
334		−#define __get_user_nosleep(x, ptr, size) \
335		−({ \
336		− long __gu_err; \
337		− __long_type(*(ptr)) __gu_val; \
338		− __typeof__((ptr)) __user __gu_addr = (ptr); \
339		− __typeof__(size) __gu_size = (size); \
340		− \
341		− __chk_user_ptr(__gu_addr); \
342		− __get_user_size(__gu_val, __gu_addr, __gu_size, __gu_err); \
343		− (x) = (__force __typeof__(*(ptr)))__gu_val; \
344		− \
345		− __gu_err; \
346		−})
347		−
348		−
349	282	/* more complex routines */
350	283
351	284	extern unsigned long __copy_tofrom_user(void __user *to,
352	285	const void __user *from, unsigned long size);
353	286
354		−#ifdef CONFIG_ARCH_HAS_COPY_MC
355		−unsigned long __must_check
356		−copy_mc_generic(void to, const void from, unsigned long size);
357		−
358		−static inline unsigned long __must_check
359		−copy_mc_to_kernel(void to, const void from, unsigned long size)
360		−{
361		− return copy_mc_generic(to, from, size);
362		−}
363		−#define copy_mc_to_kernel copy_mc_to_kernel
364		−
365		−static inline unsigned long __must_check
366		−copy_mc_to_user(void __user to, const void from, unsigned long n)
367		−{
368		− if (likely(check_copy_size(from, n, true))) {
369		− if (access_ok(to, n)) {
370		− allow_write_to_user(to, n);
371		− n = copy_mc_generic((void *)to, from, n);
372		− prevent_write_to_user(to, n);
373		− }
374		− }
375		−
376		− return n;
377		−}
378		−#endif
379		−
380	287	#ifdef __powerpc64__
381	288	static inline unsigned long
382	289	raw_copy_in_user(void __user to, const void __user from, unsigned long n)
@@ -414,26 +321,51 @@ raw_copy_to_user(void __user to, const void from, unsigned long n) · 31 unmodified lines
414	321
415	322	unsigned long __arch_clear_user(void __user *addr, unsigned long size);
416	323
417		−static inline unsigned long clear_user(void __user *addr, unsigned long size)
	324	+static inline unsigned long __clear_user(void __user *addr, unsigned long size)
418	325	{
419		− unsigned long ret = size;
	326	+ unsigned long ret;
	327	+
420	328	might_fault();
421		− if (likely(access_ok(addr, size))) {
422		− allow_write_to_user(addr, size);
423		− ret = __arch_clear_user(addr, size);
424		− prevent_write_to_user(addr, size);
425		− }
	329	+ allow_write_to_user(addr, size);
	330	+ ret = __arch_clear_user(addr, size);
	331	+ prevent_write_to_user(addr, size);
426	332	return ret;
427	333	}
428	334
429		−static inline unsigned long __clear_user(void __user *addr, unsigned long size)
	335	+static inline unsigned long clear_user(void __user *addr, unsigned long size)
430	336	{
431		− return clear_user(addr, size);
	337	+ return likely(access_ok(addr, size)) ? __clear_user(addr, size) : size;
432	338	}
433	339
434	340	extern long strncpy_from_user(char dst, const char __user src, long count);
435	341	extern __must_check long strnlen_user(const char __user *str, long n);
436	342
	343	+#ifdef CONFIG_ARCH_HAS_COPY_MC
	344	+unsigned long __must_check
	345	+copy_mc_generic(void to, const void from, unsigned long size);
	346	+
	347	+static inline unsigned long __must_check
	348	+copy_mc_to_kernel(void to, const void from, unsigned long size)
	349	+{
	350	+ return copy_mc_generic(to, from, size);
	351	+}
	352	+#define copy_mc_to_kernel copy_mc_to_kernel
	353	+
	354	+static inline unsigned long __must_check
	355	+copy_mc_to_user(void __user to, const void from, unsigned long n)
	356	+{
	357	+ if (likely(check_copy_size(from, n, true))) {
	358	+ if (access_ok(to, n)) {
	359	+ allow_write_to_user(to, n);
	360	+ n = copy_mc_generic((void *)to, from, n);
	361	+ prevent_write_to_user(to, n);
	362	+ }
	363	+ }
	364	+
	365	+ return n;
	366	+}
	367	+#endif
	368	+
437	369	extern long __copy_from_user_flushcache(void dst, const void __user src,
438	370	unsigned size);
439	371	extern void memcpy_page_flushcache(char to, struct page page, size_t offset,
@@ -482,10 +414,37 @@ user_write_access_begin(const void __user *ptr, size_t len) · 42 unmodified lines
482	414	#define user_write_access_begin user_write_access_begin
483	415	#define user_write_access_end prevent_current_write_to_user
484	416
485		−#define unsafe_op_wrap(op, err) do { if (unlikely(op)) goto err; } while (0)
486		−#define unsafe_get_user(x, p, e) unsafe_op_wrap(__get_user_allowed(x, p), e)
	417	+#define unsafe_get_user(x, p, e) do { \
	418	+ __long_type(*(p)) __gu_val; \
	419	+ __typeof__((p)) __user __gu_addr = (p); \
	420	+ \
	421	+ __get_user_size_goto(__gu_val, __gu_addr, sizeof(*(p)), e); \
	422	+ (x) = (__typeof__(*(p)))__gu_val; \
	423	+} while (0)
	424	+
487	425	#define unsafe_put_user(x, p, e) \
488		− __unsafe_put_user_goto((__typeof__((p)))(x), (p), sizeof((p)), e)
	426	+ __put_user_size_goto((__typeof__((p)))(x), (p), sizeof((p)), e)
	427	+
	428	+#define unsafe_copy_from_user(d, s, l, e) \
	429	+do { \
	430	+ u8 _dst = (u8 )(d); \
	431	+ const u8 __user _src = (const u8 __user )(s); \
	432	+ size_t _len = (l); \
	433	+ int _i; \
	434	+ \
	435	+ for (_i = 0; _i < (_len & ~(sizeof(u64) - 1)); _i += sizeof(u64)) \
	436	+ unsafe_get_user((u64 )(_dst + _i), (u64 __user *)(_src + _i), e); \
	437	+ if (_len & 4) { \
	438	+ unsafe_get_user((u32 )(_dst + _i), (u32 __user *)(_src + _i), e); \
	439	+ _i += 4; \
	440	+ } \
	441	+ if (_len & 2) { \
	442	+ unsafe_get_user((u16 )(_dst + _i), (u16 __user *)(_src + _i), e); \
	443	+ _i += 2; \
	444	+ } \
	445	+ if (_len & 1) \
	446	+ unsafe_get_user((u8 )(_dst + _i), (u8 __user *)(_src + _i), e); \
	447	+} while (0)
489	448
490	449	#define unsafe_copy_to_user(d, s, l, e) \
491	450	do { \
@@ -494,9 +453,9 @@ do { \ · 2 unmodified lines
494	453	size_t _len = (l); \
495	454	int _i; \
496	455	\
497		− for (_i = 0; _i < (_len & ~(sizeof(long) - 1)); _i += sizeof(long)) \
498		− unsafe_put_user((long)(_src + _i), (long __user *)(_dst + _i), e); \
499		− if (IS_ENABLED(CONFIG_PPC64) && (_len & 4)) { \
	456	+ for (_i = 0; _i < (_len & ~(sizeof(u64) - 1)); _i += sizeof(u64)) \
	457	+ unsafe_put_user((u64 )(_src + _i), (u64 __user *)(_dst + _i), e); \
	458	+ if (_len & 4) { \
500	459	unsafe_put_user((u32)(_src + _i), (u32 __user *)(_dst + _i), e); \
501	460	_i += 4; \
502	461	} \
@@ -511,14 +470,8 @@ do { \ · 8 unmodified lines
511	470	#define HAVE_GET_KERNEL_NOFAULT
512	471
513	472	#define __get_kernel_nofault(dst, src, type, err_label) \
514		−do { \
515		− int __kr_err; \
516		− \
517		− __get_user_size_allowed(((type )(dst)), (__force type __user *)(src),\
518		− sizeof(type), __kr_err); \
519		− if (unlikely(__kr_err)) \
520		− goto err_label; \
521		−} while (0)
	473	+ __get_user_size_goto(((type )(dst)), \
	474	+ (__force type __user *)(src), sizeof(type), err_label)
522	475
523	476	#define __put_kernel_nofault(dst, src, type, err_label) \
524	477	__put_user_size_goto(((type )(src)), \

arch/powerpc/include/asm/unistd.h+1−0

@@ -40,6 +40,7 @@ · 39 unmodified lines
40	40	#define __ARCH_WANT_SYS_SIGPROCMASK
41	41	#ifdef CONFIG_PPC32
42	42	#define __ARCH_WANT_OLD_STAT
	43	+#define __ARCH_WANT_SYS_OLD_SELECT
43	44	#endif
44	45	#ifdef CONFIG_PPC64
45	46	#define __ARCH_WANT_SYS_TIME

arch/powerpc/include/asm/vdso/gettimeofday.h+10−0

@@ -2,6 +2,8 @@ · 1 unmodified line
2	2	#ifndef _ASM_POWERPC_VDSO_GETTIMEOFDAY_H
3	3	#define _ASM_POWERPC_VDSO_GETTIMEOFDAY_H
4	4
	5	+#include <asm/page.h>
	6	+
5	7	#ifdef __ASSEMBLY__
6	8
7	9	#include <asm/ppc_asm.h>
@@ -154,6 +156,14 @@ static __always_inline u64 __arch_get_hw_counter(s32 clock_mode, · 146 unmodified lines
154	156
155	157	const struct vdso_data *__arch_get_vdso_data(void);
156	158
	159	+#ifdef CONFIG_TIME_NS
	160	+static __always_inline
	161	+const struct vdso_data __arch_get_timens_vdso_data(const struct vdso_data vd)
	162	+{
	163	+ return (void *)vd + PAGE_SIZE;
	164	+}
	165	+#endif
	166	+
157	167	static inline bool vdso_clocksource_ok(const struct vdso_data *vd)
158	168	{
159	169	return true;

arch/powerpc/include/asm/vdso_datapage.h+0−2

@@ -107,9 +107,7 @@ extern struct vdso_arch_data *vdso_data; · 106 unmodified lines
107	107	bcl 20, 31, .+4
108	108	999:
109	109	mflr \ptr
110		−#if CONFIG_PPC_PAGE_SHIFT > 14
111	110	addis \ptr, \ptr, (_vdso_datapage - 999b)@ha
112		−#endif
113	111	addi \ptr, \ptr, (_vdso_datapage - 999b)@l
114	112	.endm
115	113

arch/powerpc/include/asm/vio.h+1−0

@@ -114,6 +114,7 @@ struct vio_driver { · 113 unmodified lines
114	114	const struct vio_device_id *id_table;
115	115	int (probe)(struct vio_dev dev, const struct vio_device_id *id);
116	116	void (remove)(struct vio_dev dev);
	117	+ void (shutdown)(struct vio_dev dev);
117	118	/* A driver must have a get_desired_dma() function to
118	119	* be loaded in a CMO environment if it uses DMA.
119	120	*/

arch/powerpc/include/asm/xive.h+1−0

@@ -102,6 +102,7 @@ void xive_flush_interrupt(void); · 101 unmodified lines
102	102	/* xmon hook */
103	103	void xmon_xive_do_dump(int cpu);
104	104	int xmon_xive_get_irq_config(u32 hw_irq, struct irq_data *d);
	105	+void xmon_xive_get_irq_all(void);
105	106
106	107	/* APIs used by KVM */
107	108	u32 xive_native_default_eq_shift(void);

arch/powerpc/include/uapi/asm/errno.h+1−0

@@ -2,6 +2,7 @@ · 1 unmodified line
2	2	#ifndef _ASM_POWERPC_ERRNO_H
3	3	#define _ASM_POWERPC_ERRNO_H
4	4
	5	+#undef EDEADLOCK
5	6	#include <asm-generic/errno.h>
6	7
7	8	#undef EDEADLOCK

arch/powerpc/include/uapi/asm/posix_types.h+0−5

@@ -12,11 +12,6 @@ · 11 unmodified lines
12	12	typedef unsigned long __kernel_old_dev_t;
13	13	#define __kernel_old_dev_t __kernel_old_dev_t
14	14	#else
15		−typedef unsigned int __kernel_size_t;
16		−typedef int __kernel_ssize_t;
17		−typedef long __kernel_ptrdiff_t;
18		−#define __kernel_size_t __kernel_size_t
19		−
20	15	typedef short __kernel_ipc_pid_t;
21	16	#define __kernel_ipc_pid_t __kernel_ipc_pid_t
22	17	#endif

arch/powerpc/kernel/align.c+38−34

@@ -107,7 +107,6 @@ static struct aligninfo spe_aligninfo[32] = { · 106 unmodified lines
107	107	static int emulate_spe(struct pt_regs *regs, unsigned int reg,
108	108	struct ppc_inst ppc_instr)
109	109	{
110		− int ret;
111	110	union {
112	111	u64 ll;
113	112	u32 w[2];
@@ -127,11 +126,6 @@ static int emulate_spe(struct pt_regs *regs, unsigned int reg, · 13 unmodified lines
127	126	nb = spe_aligninfo[instr].len;
128	127	flags = spe_aligninfo[instr].flags;
129	128
130		− /* Verify the address of the operand */
131		− if (unlikely(user_mode(regs) &&
132		− !access_ok(addr, nb)))
133		− return -EFAULT;
134		−
135	129	/* userland only */
136	130	if (unlikely(!user_mode(regs)))
137	131	return 0;
@@ -169,26 +163,27 @@ static int emulate_spe(struct pt_regs *regs, unsigned int reg, · 31 unmodified lines
169	163	}
170	164	} else {
171	165	temp.ll = data.ll = 0;
172		− ret = 0;
173	166	p = addr;
174	167
	168	+ if (!user_read_access_begin(addr, nb))
	169	+ return -EFAULT;
	170	+
175	171	switch (nb) {
176	172	case 8:
177		− ret \|= __get_user_inatomic(temp.v[0], p++);
178		− ret \|= __get_user_inatomic(temp.v[1], p++);
179		− ret \|= __get_user_inatomic(temp.v[2], p++);
180		− ret \|= __get_user_inatomic(temp.v[3], p++);
	173	+ unsafe_get_user(temp.v[0], p++, Efault_read);
	174	+ unsafe_get_user(temp.v[1], p++, Efault_read);
	175	+ unsafe_get_user(temp.v[2], p++, Efault_read);
	176	+ unsafe_get_user(temp.v[3], p++, Efault_read);
181	177	fallthrough;
182	178	case 4:
183		− ret \|= __get_user_inatomic(temp.v[4], p++);
184		− ret \|= __get_user_inatomic(temp.v[5], p++);
	179	+ unsafe_get_user(temp.v[4], p++, Efault_read);
	180	+ unsafe_get_user(temp.v[5], p++, Efault_read);
185	181	fallthrough;
186	182	case 2:
187		− ret \|= __get_user_inatomic(temp.v[6], p++);
188		− ret \|= __get_user_inatomic(temp.v[7], p++);
189		− if (unlikely(ret))
190		− return -EFAULT;
	183	+ unsafe_get_user(temp.v[6], p++, Efault_read);
	184	+ unsafe_get_user(temp.v[7], p++, Efault_read);
191	185	}
	186	+ user_read_access_end();
192	187
193	188	switch (instr) {
194	189	case EVLDD:
@@ -255,31 +250,41 @@ static int emulate_spe(struct pt_regs *regs, unsigned int reg, · 60 unmodified lines
255	250
256	251	/* Store result to memory or update registers */
257	252	if (flags & ST) {
258		− ret = 0;
259	253	p = addr;
	254	+
	255	+ if (!user_write_access_begin(addr, nb))
	256	+ return -EFAULT;
	257	+
260	258	switch (nb) {
261	259	case 8:
262		− ret \|= __put_user_inatomic(data.v[0], p++);
263		− ret \|= __put_user_inatomic(data.v[1], p++);
264		− ret \|= __put_user_inatomic(data.v[2], p++);
265		− ret \|= __put_user_inatomic(data.v[3], p++);
	260	+ unsafe_put_user(data.v[0], p++, Efault_write);
	261	+ unsafe_put_user(data.v[1], p++, Efault_write);
	262	+ unsafe_put_user(data.v[2], p++, Efault_write);
	263	+ unsafe_put_user(data.v[3], p++, Efault_write);
266	264	fallthrough;
267	265	case 4:
268		− ret \|= __put_user_inatomic(data.v[4], p++);
269		− ret \|= __put_user_inatomic(data.v[5], p++);
	266	+ unsafe_put_user(data.v[4], p++, Efault_write);
	267	+ unsafe_put_user(data.v[5], p++, Efault_write);
270	268	fallthrough;
271	269	case 2:
272		− ret \|= __put_user_inatomic(data.v[6], p++);
273		− ret \|= __put_user_inatomic(data.v[7], p++);
	270	+ unsafe_put_user(data.v[6], p++, Efault_write);
	271	+ unsafe_put_user(data.v[7], p++, Efault_write);
274	272	}
275		− if (unlikely(ret))
276		− return -EFAULT;
	273	+ user_write_access_end();
277	274	} else {
278	275	*evr = data.w[0];
279	276	regs->gpr[reg] = data.w[1];
280	277	}
281	278
282	279	return 1;
	280	+
	281	+Efault_read:
	282	+ user_read_access_end();
	283	+ return -EFAULT;
	284	+
	285	+Efault_write:
	286	+ user_write_access_end();
	287	+ return -EFAULT;
283	288	}
284	289	#endif /* CONFIG_SPE */
285	290
@@ -299,13 +304,12 @@ int fix_alignment(struct pt_regs *regs) · 13 unmodified lines
299	304	struct instruction_op op;
300	305	int r, type;
301	306
302		− /*
303		− * We require a complete register set, if not, then our assembly
304		− * is broken
305		− */
306		− CHECK_FULL_REGS(regs);
	307	+ if (is_kernel_addr(regs->nip))
	308	+ r = copy_inst_from_kernel_nofault(&instr, (void *)regs->nip);
	309	+ else
	310	+ r = __get_user_instr(instr, (void __user *)regs->nip);
307	311
308		− if (unlikely(__get_user_instr(instr, (void __user *)regs->nip)))
	312	+ if (unlikely(r))
309	313	return -EFAULT;
310	314	if ((regs->msr & MSR_LE) != (MSR_KERNEL & MSR_LE)) {
311	315	/* We don't handle PPC little-endian any more... */

arch/powerpc/kernel/asm-offsets.c+0−17

@@ -91,7 +91,6 @@ int main(void) · 90 unmodified lines
91	91	DEFINE(SIGSEGV, SIGSEGV);
92	92	DEFINE(NMI_MASK, NMI_MASK);
93	93	#else
94		− OFFSET(KSP_LIMIT, thread_struct, ksp_limit);
95	94	#ifdef CONFIG_PPC_RTAS
96	95	OFFSET(RTAS_SP, thread_struct, rtas_sp);
97	96	#endif
@@ -132,7 +131,6 @@ int main(void) · 34 unmodified lines
132	131	OFFSET(KSP_VSID, thread_struct, ksp_vsid);
133	132	#else /* CONFIG_PPC64 */
134	133	OFFSET(PGDIR, thread_struct, pgdir);
135		−#ifdef CONFIG_VMAP_STACK
136	134	OFFSET(SRR0, thread_struct, srr0);
137	135	OFFSET(SRR1, thread_struct, srr1);
138	136	OFFSET(DAR, thread_struct, dar);
@@ -149,7 +147,6 @@ int main(void) · 10 unmodified lines
149	147	OFFSET(THLR, thread_struct, lr);
150	148	OFFSET(THCTR, thread_struct, ctr);
151	149	#endif
152		−#endif
153	150	#ifdef CONFIG_SPE
154	151	OFFSET(THREAD_EVR0, thread_struct, evr[0]);
155	152	OFFSET(THREAD_ACC, thread_struct, acc);
@@ -285,21 +282,11 @@ int main(void) · 129 unmodified lines
285	282	OFFSET(PACAHWCPUID, paca_struct, hw_cpu_id);
286	283	OFFSET(PACAKEXECSTATE, paca_struct, kexec_state);
287	284	OFFSET(PACA_DSCR_DEFAULT, paca_struct, dscr_default);
288		− OFFSET(ACCOUNT_STARTTIME, paca_struct, accounting.starttime);
289		− OFFSET(ACCOUNT_STARTTIME_USER, paca_struct, accounting.starttime_user);
290		− OFFSET(ACCOUNT_USER_TIME, paca_struct, accounting.utime);
291		− OFFSET(ACCOUNT_SYSTEM_TIME, paca_struct, accounting.stime);
292	285	#ifdef CONFIG_PPC_BOOK3E
293	286	OFFSET(PACA_TRAP_SAVE, paca_struct, trap_save);
294	287	#endif
295	288	OFFSET(PACA_SPRG_VDSO, paca_struct, sprg_vdso);
296	289	#else /* CONFIG_PPC64 */
297		−#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
298		− OFFSET(ACCOUNT_STARTTIME, thread_info, accounting.starttime);
299		− OFFSET(ACCOUNT_STARTTIME_USER, thread_info, accounting.starttime_user);
300		− OFFSET(ACCOUNT_USER_TIME, thread_info, accounting.utime);
301		− OFFSET(ACCOUNT_SYSTEM_TIME, thread_info, accounting.stime);
302		−#endif
303	290	#endif /* CONFIG_PPC64 */
304	291
305	292	/* RTAS */
@@ -323,9 +310,6 @@ int main(void) · 17 unmodified lines
323	310	STACK_PT_REGS_OFFSET(GPR11, gpr[11]);
324	311	STACK_PT_REGS_OFFSET(GPR12, gpr[12]);
325	312	STACK_PT_REGS_OFFSET(GPR13, gpr[13]);
326		−#ifndef CONFIG_PPC64
327		− STACK_PT_REGS_OFFSET(GPR14, gpr[14]);
328		−#endif /* CONFIG_PPC64 */
329	313	/*
330	314	* Note: these symbols include _ because they overlap with special
331	315	* register names
@@ -381,7 +365,6 @@ int main(void) · 49 unmodified lines
381	365	DEFINE(_CSRR1, STACK_INT_FRAME_SIZE+offsetof(struct exception_regs, csrr1));
382	366	DEFINE(_DSRR0, STACK_INT_FRAME_SIZE+offsetof(struct exception_regs, dsrr0));
383	367	DEFINE(_DSRR1, STACK_INT_FRAME_SIZE+offsetof(struct exception_regs, dsrr1));
384		− DEFINE(SAVED_KSP_LIMIT, STACK_INT_FRAME_SIZE+offsetof(struct exception_regs, saved_ksp_limit));
385	368	#endif
386	369	#endif
387	370

arch/powerpc/kernel/eeh.c+7−8

@@ -362,14 +362,11 @@ static inline unsigned long eeh_token_to_phys(unsigned long token) · 361 unmodified lines
362	362	pa = pte_pfn(*ptep);
363	363
364	364	/* On radix we can do hugepage mappings for io, so handle that */
365		− if (hugepage_shift) {
366		− pa <<= hugepage_shift;
367		− pa \|= token & ((1ul << hugepage_shift) - 1);
368		− } else {
369		− pa <<= PAGE_SHIFT;
370		− pa \|= token & (PAGE_SIZE - 1);
371		− }
	365	+ if (!hugepage_shift)
	366	+ hugepage_shift = PAGE_SHIFT;
372	367
	368	+ pa <<= PAGE_SHIFT;
	369	+ pa \|= token & ((1ul << hugepage_shift) - 1);
373	370	return pa;
374	371	}
375	372
@@ -779,7 +776,7 @@ int pcibios_set_pcie_reset_state(struct pci_dev *dev, enum pcie_reset_state stat · 403 unmodified lines
779	776	default:
780	777	eeh_pe_state_clear(pe, EEH_PE_ISOLATED \| EEH_PE_CFG_BLOCKED, true);
781	778	return -EINVAL;
782		− };
	779	+ }
783	780
784	781	return 0;
785	782	}
@@ -1568,6 +1565,7 @@ int eeh_pe_inject_err(struct eeh_pe *pe, int type, int func, · 782 unmodified lines
1568	1565	}
1569	1566	EXPORT_SYMBOL_GPL(eeh_pe_inject_err);
1570	1567
	1568	+#ifdef CONFIG_PROC_FS
1571	1569	static int proc_eeh_show(struct seq_file m, void v)
1572	1570	{
1573	1571	if (!eeh_enabled()) {
@@ -1594,6 +1592,7 @@ static int proc_eeh_show(struct seq_file m, void v) · 20 unmodified lines
1594	1592
1595	1593	return 0;
1596	1594	}
	1595	+#endif /* CONFIG_PROC_FS */
1597	1596
1598	1597	#ifdef CONFIG_DEBUG_FS
1599	1598

arch/powerpc/kernel/entry_32.S+137−659

@@ -48,195 +48,16 @@ · 47 unmodified lines
48	48	*/
49	49	.align 12
50	50
51		−#ifdef CONFIG_BOOKE
52		− .globl mcheck_transfer_to_handler
53		−mcheck_transfer_to_handler:
54		− mfspr r0,SPRN_DSRR0
55		− stw r0,_DSRR0(r11)
56		− mfspr r0,SPRN_DSRR1
57		− stw r0,_DSRR1(r11)
58		− /* fall through */
59		−_ASM_NOKPROBE_SYMBOL(mcheck_transfer_to_handler)
60		−
61		− .globl debug_transfer_to_handler
62		−debug_transfer_to_handler:
63		− mfspr r0,SPRN_CSRR0
64		− stw r0,_CSRR0(r11)
65		− mfspr r0,SPRN_CSRR1
66		− stw r0,_CSRR1(r11)
67		− /* fall through */
68		−_ASM_NOKPROBE_SYMBOL(debug_transfer_to_handler)
69		−
70		− .globl crit_transfer_to_handler
71		−crit_transfer_to_handler:
72		−#ifdef CONFIG_PPC_BOOK3E_MMU
73		− mfspr r0,SPRN_MAS0
74		− stw r0,MAS0(r11)
75		− mfspr r0,SPRN_MAS1
76		− stw r0,MAS1(r11)
77		− mfspr r0,SPRN_MAS2
78		− stw r0,MAS2(r11)
79		− mfspr r0,SPRN_MAS3
80		− stw r0,MAS3(r11)
81		− mfspr r0,SPRN_MAS6
82		− stw r0,MAS6(r11)
83		−#ifdef CONFIG_PHYS_64BIT
84		− mfspr r0,SPRN_MAS7
85		− stw r0,MAS7(r11)
86		−#endif /* CONFIG_PHYS_64BIT */
87		−#endif /* CONFIG_PPC_BOOK3E_MMU */
88		−#ifdef CONFIG_44x
89		− mfspr r0,SPRN_MMUCR
90		− stw r0,MMUCR(r11)
91		−#endif
92		− mfspr r0,SPRN_SRR0
93		− stw r0,_SRR0(r11)
94		− mfspr r0,SPRN_SRR1
95		− stw r0,_SRR1(r11)
96		−
97		− /* set the stack limit to the current stack */
98		− mfspr r8,SPRN_SPRG_THREAD
99		− lwz r0,KSP_LIMIT(r8)
100		− stw r0,SAVED_KSP_LIMIT(r11)
101		− rlwinm r0,r1,0,0,(31 - THREAD_SHIFT)
102		− stw r0,KSP_LIMIT(r8)
103		− /* fall through */
104		−_ASM_NOKPROBE_SYMBOL(crit_transfer_to_handler)
105		−#endif
106		−
107		−#ifdef CONFIG_40x
108		− .globl crit_transfer_to_handler
109		−crit_transfer_to_handler:
110		− lwz r0,crit_r10@l(0)
111		− stw r0,GPR10(r11)
112		− lwz r0,crit_r11@l(0)
113		− stw r0,GPR11(r11)
114		− mfspr r0,SPRN_SRR0
115		− stw r0,crit_srr0@l(0)
116		− mfspr r0,SPRN_SRR1
117		− stw r0,crit_srr1@l(0)
118		−
119		− /* set the stack limit to the current stack */
120		− mfspr r8,SPRN_SPRG_THREAD
121		− lwz r0,KSP_LIMIT(r8)
122		− stw r0,saved_ksp_limit@l(0)
123		− rlwinm r0,r1,0,0,(31 - THREAD_SHIFT)
124		− stw r0,KSP_LIMIT(r8)
125		− /* fall through */
126		−_ASM_NOKPROBE_SYMBOL(crit_transfer_to_handler)
127		−#endif
128		−
129		−/*
130		− * This code finishes saving the registers to the exception frame
131		− * and jumps to the appropriate handler for the exception, turning
132		− * on address translation.
133		− * Note that we rely on the caller having set cr0.eq iff the exception
134		− * occurred in kernel mode (i.e. MSR:PR = 0).
135		− */
136		− .globl transfer_to_handler_full
137		−transfer_to_handler_full:
138		− SAVE_NVGPRS(r11)
139		−_ASM_NOKPROBE_SYMBOL(transfer_to_handler_full)
140		− /* fall through */
141		−
142		− .globl transfer_to_handler
143		−transfer_to_handler:
144		− stw r2,GPR2(r11)
145		− stw r12,_NIP(r11)
146		− stw r9,_MSR(r11)
147		− andi. r2,r9,MSR_PR
148		− mfctr r12
149		− mfspr r2,SPRN_XER
150		− stw r12,_CTR(r11)
151		− stw r2,_XER(r11)
152		− mfspr r12,SPRN_SPRG_THREAD
153		− tovirt_vmstack r12, r12
154		− beq 2f /* if from user, fix up THREAD.regs */
155		− addi r2, r12, -THREAD
156		− addi r11,r1,STACK_FRAME_OVERHEAD
157		− stw r11,PT_REGS(r12)
158		−#if defined(CONFIG_40x) \|\| defined(CONFIG_BOOKE)
159		− /* Check to see if the dbcr0 register is set up to debug. Use the
160		− internal debug mode bit to do this. */
161		− lwz r12,THREAD_DBCR0(r12)
162		− andis. r12,r12,DBCR0_IDM@h
163		−#endif
164		− ACCOUNT_CPU_USER_ENTRY(r2, r11, r12)
165		−#ifdef CONFIG_PPC_BOOK3S_32
166		− kuep_lock r11, r12
167		−#endif
168		−#if defined(CONFIG_40x) \|\| defined(CONFIG_BOOKE)
169		− beq+ 3f
170		− /* From user and task is ptraced - load up global dbcr0 */
171		− li r12,-1 /* clear all pending debug events */
172		− mtspr SPRN_DBSR,r12
173		− lis r11,global_dbcr0@ha
174		− tophys(r11,r11)
175		− addi r11,r11,global_dbcr0@l
176		−#ifdef CONFIG_SMP
177		− lwz r9,TASK_CPU(r2)
178		− slwi r9,r9,2
179		− add r11,r11,r9
180		−#endif
181		− lwz r12,0(r11)
182		− mtspr SPRN_DBCR0,r12
183		−#endif
184		−
185		− b 3f
186		−
187		−2: /* if from kernel, check interrupted DOZE/NAP mode and
188		− * check for stack overflow
189		− */
190		− kuap_save_and_lock r11, r12, r9, r2, r6
191		− addi r2, r12, -THREAD
192		−#ifndef CONFIG_VMAP_STACK
193		− lwz r9,KSP_LIMIT(r12)
194		− cmplw r1,r9 /* if r1 <= ksp_limit */
195		− ble- stack_ovf /* then the kernel stack overflowed */
196		−#endif
197		−5:
198	51	#if defined(CONFIG_PPC_BOOK3S_32) \|\| defined(CONFIG_E500)
	52	+ .globl prepare_transfer_to_handler
	53	+prepare_transfer_to_handler:
	54	+ /* if from kernel, check interrupted DOZE/NAP mode */
199	55	lwz r12,TI_LOCAL_FLAGS(r2)
200	56	mtcrf 0x01,r12
201	57	bt- 31-TLF_NAPPING,4f
202	58	bt- 31-TLF_SLEEPING,7f
203		−#endif /* CONFIG_PPC_BOOK3S_32 \|\| CONFIG_E500 */
204		− .globl transfer_to_handler_cont
205		−transfer_to_handler_cont:
206		−3:
207		− mflr r9
208		− tovirt_novmstack r2, r2 /* set r2 to current */
209		− tovirt_vmstack r9, r9
210		− lwz r11,0(r9) /* virtual address of handler */
211		− lwz r9,4(r9) /* where to go when done */
212		−#if defined(CONFIG_PPC_8xx) && defined(CONFIG_PERF_EVENTS)
213		− mtspr SPRN_NRI, r0
214		−#endif
215		−#ifdef CONFIG_TRACE_IRQFLAGS
216		− /*
217		− * When tracing IRQ state (lockdep) we enable the MMU before we call
218		− * the IRQ tracing functions as they might access vmalloc space or
219		− * perform IOs for console output.
220		− *
221		− * To speed up the syscall path where interrupts stay on, let's check
222		− * first if we are changing the MSR value at all.
223		− */
224		− tophys_novmstack r12, r1
225		− lwz r12,_MSR(r12)
226		− andi. r12,r12,MSR_EE
227		− bne 1f
228		−
229		− /* MSR isn't changing, just transition directly */
230		−#endif
231		− mtspr SPRN_SRR0,r11
232		− mtspr SPRN_SRR1,r10
233		− mtlr r9
234		− rfi /* jump to handler, enable MMU */
235		−#ifdef CONFIG_40x
236		− b . /* Prevent prefetch past rfi */
237		−#endif
	59	+ blr
238	60
239		−#if defined (CONFIG_PPC_BOOK3S_32) \|\| defined(CONFIG_E500)
240	61	4: rlwinm r12,r12,0,~_TLF_NAPPING
241	62	stw r12,TI_LOCAL_FLAGS(r2)
242	63	b power_save_ppc32_restore
@@ -246,97 +67,18 @@ transfer_to_handler_cont: · 3 unmodified lines
246	67	lwz r9,_MSR(r11) /* if sleeping, clear MSR.EE */
247	68	rlwinm r9,r9,0,~MSR_EE
248	69	lwz r12,_LINK(r11) /* and return to address in LR */
249		− kuap_restore r11, r2, r3, r4, r5
250	70	lwz r2, GPR2(r11)
251	71	b fast_exception_return
252		−#endif
253		−_ASM_NOKPROBE_SYMBOL(transfer_to_handler)
254		−_ASM_NOKPROBE_SYMBOL(transfer_to_handler_cont)
255		−
256		−#ifdef CONFIG_TRACE_IRQFLAGS
257		−1: /* MSR is changing, re-enable MMU so we can notify lockdep. We need to
258		− * keep interrupts disabled at this point otherwise we might risk
259		− * taking an interrupt before we tell lockdep they are enabled.
260		− */
261		− lis r12,reenable_mmu@h
262		− ori r12,r12,reenable_mmu@l
263		− LOAD_REG_IMMEDIATE(r0, MSR_KERNEL)
264		− mtspr SPRN_SRR0,r12
265		− mtspr SPRN_SRR1,r0
266		− rfi
267		−#ifdef CONFIG_40x
268		− b . /* Prevent prefetch past rfi */
269		−#endif
270		−
271		−reenable_mmu:
272		− /*
273		− * We save a bunch of GPRs,
274		− * r3 can be different from GPR3(r1) at this point, r9 and r11
275		− * contains the old MSR and handler address respectively,
276		− * r0, r4-r8, r12, CCR, CTR, XER etc... are left
277		− * clobbered as they aren't useful past this point.
278		− */
279		−
280		− stwu r1,-32(r1)
281		− stw r9,8(r1)
282		− stw r11,12(r1)
283		− stw r3,16(r1)
284		−
285		− /* If we are disabling interrupts (normal case), simply log it with
286		− * lockdep
287		− */
288		−1: bl trace_hardirqs_off
289		− lwz r3,16(r1)
290		− lwz r11,12(r1)
291		− lwz r9,8(r1)
292		− addi r1,r1,32
293		− mtctr r11
294		− mtlr r9
295		− bctr /* jump to handler */
296		−#endif /* CONFIG_TRACE_IRQFLAGS */
297		−
298		−#ifndef CONFIG_VMAP_STACK
299		−/*
300		− * On kernel stack overflow, load up an initial stack pointer
301		− * and call StackOverflow(regs), which should not return.
302		− */
303		−stack_ovf:
304		− /* sometimes we use a statically-allocated stack, which is OK. */
305		− lis r12,_end@h
306		− ori r12,r12,_end@l
307		− cmplw r1,r12
308		− ble 5b /* r1 <= &_end is OK */
309		− SAVE_NVGPRS(r11)
310		− addi r3,r1,STACK_FRAME_OVERHEAD
311		− lis r1,init_thread_union@ha
312		− addi r1,r1,init_thread_union@l
313		− addi r1,r1,THREAD_SIZE-STACK_FRAME_OVERHEAD
314		− lis r9,StackOverflow@ha
315		− addi r9,r9,StackOverflow@l
316		− LOAD_REG_IMMEDIATE(r10,MSR_KERNEL)
317		−#if defined(CONFIG_PPC_8xx) && defined(CONFIG_PERF_EVENTS)
318		− mtspr SPRN_NRI, r0
319		−#endif
320		− mtspr SPRN_SRR0,r9
321		− mtspr SPRN_SRR1,r10
322		− rfi
323		−#ifdef CONFIG_40x
324		− b . /* Prevent prefetch past rfi */
325		−#endif
326		−_ASM_NOKPROBE_SYMBOL(stack_ovf)
327		−#endif
	72	+_ASM_NOKPROBE_SYMBOL(prepare_transfer_to_handler)
	73	+#endif /* CONFIG_PPC_BOOK3S_32 \|\| CONFIG_E500 */
328	74
329	75	.globl transfer_to_syscall
330	76	transfer_to_syscall:
331	77	SAVE_NVGPRS(r1)
332		−#ifdef CONFIG_PPC_BOOK3S_32
333		− kuep_lock r11, r12
334		−#endif
335	78
336	79	/* Calling convention has r9 = orig r0, r10 = regs */
337	80	addi r10,r1,STACK_FRAME_OVERHEAD
338	81	mr r9,r0
339		− stw r10,THREAD+PT_REGS(r2)
340	82	bl system_call_exception
341	83
342	84	ret_from_syscall:
@@ -349,10 +91,6 @@ ret_from_syscall: · 6 unmodified lines
349	91	cmplwi cr0,r5,0
350	92	bne- 2f
351	93	#endif /* CONFIG_PPC_47x */
352		−#ifdef CONFIG_PPC_BOOK3S_32
353		− kuep_unlock r5, r7
354		−#endif
355		− kuap_check r2, r4
356	94	lwz r4,_LINK(r1)
357	95	lwz r5,_CCR(r1)
358	96	mtlr r4
@@ -411,27 +149,6 @@ ret_from_kernel_thread: · 52 unmodified lines
411	149	li r3,0
412	150	b ret_from_syscall
413	151
414		−/*
415		− * Top-level page fault handling.
416		− * This is in assembler because if do_page_fault tells us that
417		− * it is a bad kernel page fault, we want to save the non-volatile
418		− * registers before calling bad_page_fault.
419		− */
420		− .globl handle_page_fault
421		−handle_page_fault:
422		− addi r3,r1,STACK_FRAME_OVERHEAD
423		− bl do_page_fault
424		− cmpwi r3,0
425		− beq+ ret_from_except
426		− SAVE_NVGPRS(r1)
427		− lwz r0,_TRAP(r1)
428		− clrrwi r0,r0,1
429		− stw r0,_TRAP(r1)
430		− mr r4,r3 /* err arg for bad_page_fault */
431		− addi r3,r1,STACK_FRAME_OVERHEAD
432		− bl __bad_page_fault
433		− b ret_from_except_full
434		−
435	152	/*
436	153	* This routine switches between two different tasks. The process
437	154	* state of one is saved on its kernel stack. Then the state
@@ -485,7 +202,6 @@ END_FTR_SECTION_IFSET(CPU_FTR_SPE) · 47 unmodified lines
485	202	stw r10,_CCR(r1)
486	203	stw r1,KSP(r3) /* Set old stack pointer */
487	204
488		− kuap_check r2, r0
489	205	#ifdef CONFIG_SMP
490	206	/* We need a sync somewhere here to make sure that if the
491	207	* previous task gets rescheduled on another CPU, it sees all
@@ -529,12 +245,12 @@ END_FTR_SECTION_IFSET(CPU_FTR_SPE) · 37 unmodified lines
529	245	fast_exception_return:
530	246	#if !(defined(CONFIG_4xx) \|\| defined(CONFIG_BOOKE))
531	247	andi. r10,r9,MSR_RI /* check for recoverable interrupt */
532		− beq 1f /* if not, we've got problems */
	248	+ beq 3f /* if not, we've got problems */
533	249	#endif
534	250
535	251	2: REST_4GPRS(3, r11)
536	252	lwz r10,_CCR(r11)
537		− REST_GPR(1, r11)
	253	+ REST_2GPRS(1, r11)
538	254	mtcr r10
539	255	lwz r10,_LINK(r11)
540	256	mtlr r10
@@ -556,257 +272,147 @@ fast_exception_return: · 15 unmodified lines
556	272	#endif
557	273	_ASM_NOKPROBE_SYMBOL(fast_exception_return)
558	274
559		−#if !(defined(CONFIG_4xx) \|\| defined(CONFIG_BOOKE))
560		−/* check if the exception happened in a restartable section */
561		−1: lis r3,exc_exit_restart_end@ha
562		− addi r3,r3,exc_exit_restart_end@l
563		− cmplw r12,r3
564		− bge 3f
565		− lis r4,exc_exit_restart@ha
566		− addi r4,r4,exc_exit_restart@l
567		− cmplw r12,r4
568		− blt 3f
569		− lis r3,fee_restarts@ha
570		− tophys(r3,r3)
571		− lwz r5,fee_restarts@l(r3)
572		− addi r5,r5,1
573		− stw r5,fee_restarts@l(r3)
574		− mr r12,r4 /* restart at exc_exit_restart */
575		− b 2b
576		−
577		− .section .bss
578		− .align 2
579		−fee_restarts:
580		− .space 4
581		− .previous
582		−
583	275	/* aargh, a nonrecoverable interrupt, panic */
584	276	/* aargh, we don't know which trap this is */
585	277	3:
586	278	li r10,-1
587	279	stw r10,_TRAP(r11)
588		− addi r3,r1,STACK_FRAME_OVERHEAD
589		− lis r10,MSR_KERNEL@h
590		− ori r10,r10,MSR_KERNEL@l
591		− bl transfer_to_handler_full
592		− .long unrecoverable_exception
593		− .long ret_from_except
594		−#endif
595		−
596		− .globl ret_from_except_full
597		−ret_from_except_full:
598		− REST_NVGPRS(r1)
599		− /* fall through */
600		−
601		− .globl ret_from_except
602		−ret_from_except:
603		− /* Hard-disable interrupts so that current_thread_info()->flags
604		− * can't change between when we test it and when we return
605		− * from the interrupt. */
606		− /* Note: We don't bother telling lockdep about it */
607		− LOAD_REG_IMMEDIATE(r10,MSR_KERNEL)
608		− mtmsr r10 /* disable interrupts */
609		−
610		− lwz r3,_MSR(r1) /* Returning to user mode? */
611		− andi. r0,r3,MSR_PR
612		− beq resume_kernel
613		−
614		−user_exc_return: /* r10 contains MSR_KERNEL here */
615		− /* Check current_thread_info()->flags */
616		− lwz r9,TI_FLAGS(r2)
617		− andi. r0,r9,_TIF_USER_WORK_MASK
618		− bne do_work
	280	+ prepare_transfer_to_handler
	281	+ bl unrecoverable_exception
	282	+ trap /* should not get here */
619	283
620		−restore_user:
621		−#if defined(CONFIG_4xx) \|\| defined(CONFIG_BOOKE)
622		− /* Check whether this process has its own DBCR0 value. The internal
623		− debug mode bit tells us that dbcr0 should be loaded. */
624		− lwz r0,THREAD+THREAD_DBCR0(r2)
625		− andis. r10,r0,DBCR0_IDM@h
626		− bnel- load_dbcr0
627		−#endif
628		− ACCOUNT_CPU_USER_EXIT(r2, r10, r11)
629		−#ifdef CONFIG_PPC_BOOK3S_32
630		− kuep_unlock r10, r11
631		−#endif
	284	+ .globl interrupt_return
	285	+interrupt_return:
	286	+ lwz r4,_MSR(r1)
	287	+ addi r3,r1,STACK_FRAME_OVERHEAD
	288	+ andi. r0,r4,MSR_PR
	289	+ beq .Lkernel_interrupt_return
	290	+ bl interrupt_exit_user_prepare
	291	+ cmpwi r3,0
	292	+ bne- .Lrestore_nvgprs
632	293
633		− b restore
	294	+.Lfast_user_interrupt_return:
	295	+ lwz r11,_NIP(r1)
	296	+ lwz r12,_MSR(r1)
	297	+ mtspr SPRN_SRR0,r11
	298	+ mtspr SPRN_SRR1,r12
634	299
635		−/* N.B. the only way to get here is from the beq following ret_from_except. */
636		−resume_kernel:
637		− /* check current_thread_info, _TIF_EMULATE_STACK_STORE */
638		− lwz r8,TI_FLAGS(r2)
639		− andis. r0,r8,_TIF_EMULATE_STACK_STORE@h
640		− beq+ 1f
	300	+BEGIN_FTR_SECTION
	301	+ stwcx. r0,0,r1 /* to clear the reservation */
	302	+FTR_SECTION_ELSE
	303	+ lwarx r0,0,r1
	304	+ALT_FTR_SECTION_END_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
641	305
642		− addi r8,r1,INT_FRAME_SIZE /* Get the kprobed function entry */
	306	+ lwz r3,_CCR(r1)
	307	+ lwz r4,_LINK(r1)
	308	+ lwz r5,_CTR(r1)
	309	+ lwz r6,_XER(r1)
	310	+ li r0,0
643	311
644		− lwz r3,GPR1(r1)
645		− subi r3,r3,INT_FRAME_SIZE /* dst: Allocate a trampoline exception frame */
646		− mr r4,r1 /* src: current exception frame */
647		− mr r1,r3 /* Reroute the trampoline frame to r1 */
	312	+ /*
	313	+ * Leaving a stale exception_marker on the stack can confuse
	314	+ * the reliable stack unwinder later on. Clear it.
	315	+ */
	316	+ stw r0,8(r1)
	317	+ REST_4GPRS(7, r1)
	318	+ REST_2GPRS(11, r1)
648	319
649		− /* Copy from the original to the trampoline. */
650		− li r5,INT_FRAME_SIZE/4 /* size: INT_FRAME_SIZE */
651		− li r6,0 /* start offset: 0 */
	320	+ mtcr r3
	321	+ mtlr r4
652	322	mtctr r5
653		−2: lwzx r0,r6,r4
654		− stwx r0,r6,r3
655		− addi r6,r6,4
656		− bdnz 2b
657		−
658		− /* Do real store operation to complete stwu */
659		− lwz r5,GPR1(r1)
660		− stw r8,0(r5)
	323	+ mtspr SPRN_XER,r6
661	324
662		− /* Clear _TIF_EMULATE_STACK_STORE flag */
663		− lis r11,_TIF_EMULATE_STACK_STORE@h
664		− addi r5,r2,TI_FLAGS
665		−0: lwarx r8,0,r5
666		− andc r8,r8,r11
667		− stwcx. r8,0,r5
668		− bne- 0b
669		−1:
670		−
671		−#ifdef CONFIG_PREEMPTION
672		− /* check current_thread_info->preempt_count */
673		− lwz r0,TI_PREEMPT(r2)
674		− cmpwi 0,r0,0 /* if non-zero, just restore regs and return */
675		− bne restore_kuap
676		− andi. r8,r8,_TIF_NEED_RESCHED
677		− beq+ restore_kuap
678		− lwz r3,_MSR(r1)
679		− andi. r0,r3,MSR_EE /* interrupts off? */
680		− beq restore_kuap /* don't schedule if so */
681		−#ifdef CONFIG_TRACE_IRQFLAGS
682		− /* Lockdep thinks irqs are enabled, we need to call
683		− * preempt_schedule_irq with IRQs off, so we inform lockdep
684		− * now that we -did- turn them off already
685		− */
686		− bl trace_hardirqs_off
687		−#endif
688		− bl preempt_schedule_irq
689		−#ifdef CONFIG_TRACE_IRQFLAGS
690		− /* And now, to properly rebalance the above, we tell lockdep they
691		− * are being turned back on, which will happen when we return
692		− */
693		− bl trace_hardirqs_on
	325	+ REST_4GPRS(2, r1)
	326	+ REST_GPR(6, r1)
	327	+ REST_GPR(0, r1)
	328	+ REST_GPR(1, r1)
	329	+ rfi
	330	+#ifdef CONFIG_40x
	331	+ b . /* Prevent prefetch past rfi */
694	332	#endif
695		−#endif /* CONFIG_PREEMPTION */
696		−restore_kuap:
697		− kuap_restore r1, r2, r9, r10, r0
698		−
699		− /* interrupts are hard-disabled at this point */
700		−restore:
701		−#if defined(CONFIG_44x) && !defined(CONFIG_PPC_47x)
702		− lis r4,icache_44x_need_flush@ha
703		− lwz r5,icache_44x_need_flush@l(r4)
704		− cmplwi cr0,r5,0
705		− beq+ 1f
706		− li r6,0
707		− iccci r0,r0
708		− stw r6,icache_44x_need_flush@l(r4)
709		−1:
710		−#endif /* CONFIG_44x */
711	333
712		− lwz r9,_MSR(r1)
713		−#ifdef CONFIG_TRACE_IRQFLAGS
714		− /* Lockdep doesn't know about the fact that IRQs are temporarily turned
715		− * off in this assembly code while peeking at TI_FLAGS() and such. However
716		− * we need to inform it if the exception turned interrupts off, and we
717		− * are about to trun them back on.
718		− */
719		− andi. r10,r9,MSR_EE
720		− beq 1f
721		− stwu r1,-32(r1)
722		− mflr r0
723		− stw r0,4(r1)
724		− bl trace_hardirqs_on
725		− addi r1, r1, 32
726		− lwz r9,_MSR(r1)
727		−1:
728		−#endif /* CONFIG_TRACE_IRQFLAGS */
	334	+.Lrestore_nvgprs:
	335	+ REST_NVGPRS(r1)
	336	+ b .Lfast_user_interrupt_return
729	337
730		− lwz r0,GPR0(r1)
731		− lwz r2,GPR2(r1)
732		− REST_4GPRS(3, r1)
733		− REST_2GPRS(7, r1)
	338	+.Lkernel_interrupt_return:
	339	+ bl interrupt_exit_kernel_prepare
734	340
735		− lwz r10,_XER(r1)
736		− lwz r11,_CTR(r1)
737		− mtspr SPRN_XER,r10
738		− mtctr r11
	341	+.Lfast_kernel_interrupt_return:
	342	+ cmpwi cr1,r3,0
	343	+ lwz r11,_NIP(r1)
	344	+ lwz r12,_MSR(r1)
	345	+ mtspr SPRN_SRR0,r11
	346	+ mtspr SPRN_SRR1,r12
739	347
740	348	BEGIN_FTR_SECTION
741		− lwarx r11,0,r1
742		−END_FTR_SECTION_IFSET(CPU_FTR_NEED_PAIRED_STWCX)
743		− stwcx. r0,0,r1 /* to clear the reservation */
	349	+ stwcx. r0,0,r1 /* to clear the reservation */
	350	+FTR_SECTION_ELSE
	351	+ lwarx r0,0,r1
	352	+ALT_FTR_SECTION_END_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
744	353
745		−#if !(defined(CONFIG_4xx) \|\| defined(CONFIG_BOOKE))
746		− andi. r10,r9,MSR_RI /* check if this exception occurred */
747		− beql nonrecoverable /* at a bad place (MSR:RI = 0) */
	354	+ lwz r3,_LINK(r1)
	355	+ lwz r4,_CTR(r1)
	356	+ lwz r5,_XER(r1)
	357	+ lwz r6,_CCR(r1)
	358	+ li r0,0
	359	+
	360	+ REST_4GPRS(7, r1)
	361	+ REST_2GPRS(11, r1)
748	362
749		− lwz r10,_CCR(r1)
750		− lwz r11,_LINK(r1)
751		− mtcrf 0xFF,r10
752		− mtlr r11
	363	+ mtlr r3
	364	+ mtctr r4
	365	+ mtspr SPRN_XER,r5
753	366
754		− /* Clear the exception_marker on the stack to avoid confusing stacktrace */
755		− li r10, 0
756		− stw r10, 8(r1)
757	367	/*
758		− * Once we put values in SRR0 and SRR1, we are in a state
759		− * where exceptions are not recoverable, since taking an
760		− * exception will trash SRR0 and SRR1. Therefore we clear the
761		− * MSR:RI bit to indicate this. If we do take an exception,
762		− * we can't return to the point of the exception but we
763		− * can restart the exception exit path at the label
764		− * exc_exit_restart below. -- paulus
	368	+ * Leaving a stale exception_marker on the stack can confuse
	369	+ * the reliable stack unwinder later on. Clear it.
765	370	*/
766		− LOAD_REG_IMMEDIATE(r10,MSR_KERNEL & ~MSR_RI)
767		− mtmsr r10 /* clear the RI bit */
768		− .globl exc_exit_restart
769		−exc_exit_restart:
770		− lwz r12,_NIP(r1)
771		− mtspr SPRN_SRR0,r12
772		− mtspr SPRN_SRR1,r9
773		− REST_4GPRS(9, r1)
774		− lwz r1,GPR1(r1)
775		− .globl exc_exit_restart_end
776		−exc_exit_restart_end:
	371	+ stw r0,8(r1)
	372	+
	373	+ REST_4GPRS(2, r1)
	374	+
	375	+ bne- cr1,1f /* emulate stack store */
	376	+ mtcr r6
	377	+ REST_GPR(6, r1)
	378	+ REST_GPR(0, r1)
	379	+ REST_GPR(1, r1)
777	380	rfi
778		−_ASM_NOKPROBE_SYMBOL(exc_exit_restart)
779		−_ASM_NOKPROBE_SYMBOL(exc_exit_restart_end)
	381	+#ifdef CONFIG_40x
	382	+ b . /* Prevent prefetch past rfi */
	383	+#endif
780	384
781		−#else /* !(CONFIG_4xx \|\| CONFIG_BOOKE) */
782		− /*
783		− * This is a bit different on 4xx/Book-E because it doesn't have
784		− * the RI bit in the MSR.
785		− * The TLB miss handler checks if we have interrupted
786		− * the exception exit path and restarts it if so
787		− * (well maybe one day it will... :).
	385	+1: /*
	386	+ * Emulate stack store with update. New r1 value was already calculated
	387	+ * and updated in our interrupt regs by emulate_loadstore, but we can't
	388	+ * store the previous value of r1 to the stack before re-loading our
	389	+ * registers from it, otherwise they could be clobbered. Use
	390	+ * SPRG Scratch0 as temporary storage to hold the store
	391	+ * data, as interrupts are disabled here so it won't be clobbered.
788	392	*/
789		− lwz r11,_LINK(r1)
790		− mtlr r11
791		− lwz r10,_CCR(r1)
792		− mtcrf 0xff,r10
793		− /* Clear the exception_marker on the stack to avoid confusing stacktrace */
794		− li r10, 0
795		− stw r10, 8(r1)
796		− REST_2GPRS(9, r1)
797		− .globl exc_exit_restart
798		−exc_exit_restart:
799		− lwz r11,_NIP(r1)
800		− lwz r12,_MSR(r1)
801		− mtspr SPRN_SRR0,r11
802		− mtspr SPRN_SRR1,r12
803		− REST_2GPRS(11, r1)
804		− lwz r1,GPR1(r1)
805		− .globl exc_exit_restart_end
806		−exc_exit_restart_end:
	393	+ mtcr r6
	394	+#ifdef CONFIG_BOOKE
	395	+ mtspr SPRN_SPRG_WSCRATCH0, r9
	396	+#else
	397	+ mtspr SPRN_SPRG_SCRATCH0, r9
	398	+#endif
	399	+ addi r9,r1,INT_FRAME_SIZE /* get original r1 */
	400	+ REST_GPR(6, r1)
	401	+ REST_GPR(0, r1)
	402	+ REST_GPR(1, r1)
	403	+ stw r9,0(r1) /* perform store component of stwu */
	404	+#ifdef CONFIG_BOOKE
	405	+ mfspr r9, SPRN_SPRG_RSCRATCH0
	406	+#else
	407	+ mfspr r9, SPRN_SPRG_SCRATCH0
	408	+#endif
807	409	rfi
808		− b . /* prevent prefetch past rfi */
809		−_ASM_NOKPROBE_SYMBOL(exc_exit_restart)
	410	+#ifdef CONFIG_40x
	411	+ b . /* Prevent prefetch past rfi */
	412	+#endif
	413	+_ASM_NOKPROBE_SYMBOL(interrupt_return)
	414	+
	415	+#if defined(CONFIG_4xx) \|\| defined(CONFIG_BOOKE)
810	416
811	417	/*
812	418	* Returning from a critical interrupt in user mode doesn't need
@@ -837,8 +443,7 @@ _ASM_NOKPROBE_SYMBOL(exc_exit_restart) · 24 unmodified lines
837	443	REST_NVGPRS(r1); \
838	444	lwz r3,_MSR(r1); \
839	445	andi. r3,r3,MSR_PR; \
840		− LOAD_REG_IMMEDIATE(r10,MSR_KERNEL); \
841		− bne user_exc_return; \
	446	+ bne interrupt_return; \
842	447	lwz r0,GPR0(r1); \
843	448	lwz r2,GPR2(r1); \
844	449	REST_4GPRS(3, r1); \
@@ -906,11 +511,6 @@ _ASM_NOKPROBE_SYMBOL(exc_exit_restart) · 61 unmodified lines
906	511	#ifdef CONFIG_40x
907	512	.globl ret_from_crit_exc
908	513	ret_from_crit_exc:
909		− mfspr r9,SPRN_SPRG_THREAD
910		− lis r10,saved_ksp_limit@ha;
911		− lwz r10,saved_ksp_limit@l(r10);
912		− tovirt(r9,r9);
913		− stw r10,KSP_LIMIT(r9)
914	514	lis r9,crit_srr0@ha;
915	515	lwz r9,crit_srr0@l(r9);
916	516	lis r10,crit_srr1@ha;
@@ -924,9 +524,6 @@ _ASM_NOKPROBE_SYMBOL(ret_from_crit_exc) · 7 unmodified lines
924	524	#ifdef CONFIG_BOOKE
925	525	.globl ret_from_crit_exc
926	526	ret_from_crit_exc:
927		− mfspr r9,SPRN_SPRG_THREAD
928		− lwz r10,SAVED_KSP_LIMIT(r1)
929		− stw r10,KSP_LIMIT(r9)
930	527	RESTORE_xSRR(SRR0,SRR1);
931	528	RESTORE_MMU_REGS;
932	529	RET_FROM_EXC_LEVEL(SPRN_CSRR0, SPRN_CSRR1, PPC_RFCI)
@@ -934,9 +531,6 @@ _ASM_NOKPROBE_SYMBOL(ret_from_crit_exc) · 1 unmodified line
934	531
935	532	.globl ret_from_debug_exc
936	533	ret_from_debug_exc:
937		− mfspr r9,SPRN_SPRG_THREAD
938		− lwz r10,SAVED_KSP_LIMIT(r1)
939		− stw r10,KSP_LIMIT(r9)
940	534	RESTORE_xSRR(SRR0,SRR1);
941	535	RESTORE_xSRR(CSRR0,CSRR1);
942	536	RESTORE_MMU_REGS;
@@ -945,9 +539,6 @@ _ASM_NOKPROBE_SYMBOL(ret_from_debug_exc) · 2 unmodified lines
945	539
946	540	.globl ret_from_mcheck_exc
947	541	ret_from_mcheck_exc:
948		− mfspr r9,SPRN_SPRG_THREAD
949		− lwz r10,SAVED_KSP_LIMIT(r1)
950		− stw r10,KSP_LIMIT(r9)
951	542	RESTORE_xSRR(SRR0,SRR1);
952	543	RESTORE_xSRR(CSRR0,CSRR1);
953	544	RESTORE_xSRR(DSRR0,DSRR1);
@@ -955,121 +546,8 @@ ret_from_mcheck_exc: · 1 unmodified line
955	546	RET_FROM_EXC_LEVEL(SPRN_MCSRR0, SPRN_MCSRR1, PPC_RFMCI)
956	547	_ASM_NOKPROBE_SYMBOL(ret_from_mcheck_exc)
957	548	#endif /* CONFIG_BOOKE */
958		−
959		−/*
960		− * Load the DBCR0 value for a task that is being ptraced,
961		− * having first saved away the global DBCR0. Note that r0
962		− * has the dbcr0 value to set upon entry to this.
963		− */
964		−load_dbcr0:
965		− mfmsr r10 /* first disable debug exceptions */
966		− rlwinm r10,r10,0,~MSR_DE
967		− mtmsr r10
968		− isync
969		− mfspr r10,SPRN_DBCR0
970		− lis r11,global_dbcr0@ha
971		− addi r11,r11,global_dbcr0@l
972		−#ifdef CONFIG_SMP
973		− lwz r9,TASK_CPU(r2)
974		− slwi r9,r9,2
975		− add r11,r11,r9
976		−#endif
977		− stw r10,0(r11)
978		− mtspr SPRN_DBCR0,r0
979		− li r11,-1
980		− mtspr SPRN_DBSR,r11 /* clear all pending debug events */
981		− blr
982		−
983		− .section .bss
984		− .align 4
985		− .global global_dbcr0
986		−global_dbcr0:
987		− .space 4*NR_CPUS
988		− .previous
989	549	#endif /* !(CONFIG_4xx \|\| CONFIG_BOOKE) */
990	550
991		−do_work: /* r10 contains MSR_KERNEL here */
992		− andi. r0,r9,_TIF_NEED_RESCHED
993		− beq do_user_signal
994		−
995		−do_resched: /* r10 contains MSR_KERNEL here */
996		−#ifdef CONFIG_TRACE_IRQFLAGS
997		− bl trace_hardirqs_on
998		− mfmsr r10
999		−#endif
1000		− ori r10,r10,MSR_EE
1001		− mtmsr r10 /* hard-enable interrupts */
1002		− bl schedule
1003		−recheck:
1004		− /* Note: And we don't tell it we are disabling them again
1005		− * neither. Those disable/enable cycles used to peek at
1006		− * TI_FLAGS aren't advertised.
1007		− */
1008		− LOAD_REG_IMMEDIATE(r10,MSR_KERNEL)
1009		− mtmsr r10 /* disable interrupts */
1010		− lwz r9,TI_FLAGS(r2)
1011		− andi. r0,r9,_TIF_NEED_RESCHED
1012		− bne- do_resched
1013		− andi. r0,r9,_TIF_USER_WORK_MASK
1014		− beq restore_user
1015		−do_user_signal: /* r10 contains MSR_KERNEL here */
1016		− ori r10,r10,MSR_EE
1017		− mtmsr r10 /* hard-enable interrupts */
1018		− /* save r13-r31 in the exception frame, if not already done */
1019		− lwz r3,_TRAP(r1)
1020		− andi. r0,r3,1
1021		− beq 2f
1022		− SAVE_NVGPRS(r1)
1023		− rlwinm r3,r3,0,0,30
1024		− stw r3,_TRAP(r1)
1025		−2: addi r3,r1,STACK_FRAME_OVERHEAD
1026		− mr r4,r9
1027		− bl do_notify_resume
1028		− REST_NVGPRS(r1)
1029		− b recheck
1030		−
1031		−/*
1032		− * We come here when we are at the end of handling an exception
1033		− * that occurred at a place where taking an exception will lose
1034		− * state information, such as the contents of SRR0 and SRR1.
1035		− */
1036		−nonrecoverable:
1037		− lis r10,exc_exit_restart_end@ha
1038		− addi r10,r10,exc_exit_restart_end@l
1039		− cmplw r12,r10
1040		− bge 3f
1041		− lis r11,exc_exit_restart@ha
1042		− addi r11,r11,exc_exit_restart@l
1043		− cmplw r12,r11
1044		− blt 3f
1045		− lis r10,ee_restarts@ha
1046		− lwz r12,ee_restarts@l(r10)
1047		− addi r12,r12,1
1048		− stw r12,ee_restarts@l(r10)
1049		− mr r12,r11 /* restart at exc_exit_restart */
1050		− blr
1051		−3: /* OK, we can't recover, kill this process */
1052		− lwz r3,_TRAP(r1)
1053		− andi. r0,r3,1
1054		− beq 5f
1055		− SAVE_NVGPRS(r1)
1056		− rlwinm r3,r3,0,0,30
1057		− stw r3,_TRAP(r1)
1058		−5: mfspr r2,SPRN_SPRG_THREAD
1059		− addi r2,r2,-THREAD
1060		− tovirt(r2,r2) /* set back r2 to current */
1061		−4: addi r3,r1,STACK_FRAME_OVERHEAD
1062		− bl unrecoverable_exception
1063		− /* shouldn't return */
1064		− b 4b
1065		−_ASM_NOKPROBE_SYMBOL(nonrecoverable)
1066		−
1067		− .section .bss
1068		− .align 2
1069		−ee_restarts:
1070		− .space 4
1071		− .previous
1072		−
1073	551	/*
1074	552	* PROM code for specific machines follows. Put it
1075	553	* here so it's easy to add arch-specific sections later.
@@ -1088,7 +566,6 @@ _GLOBAL(enter_rtas) · 12 unmodified lines
1088	566	lis r6,1f@ha /* physical return address for rtas */
1089	567	addi r6,r6,1f@l
1090	568	tophys(r6,r6)
1091		− tophys_novmstack r7, r1
1092	569	lwz r8,RTASENTRY(r4)
1093	570	lwz r4,RTASBASE(r4)
1094	571	mfmsr r9
@@ -1097,24 +574,25 @@ _GLOBAL(enter_rtas) · 2 unmodified lines
1097	574	mtmsr r0 /* disable interrupts so SRR0/1 don't get trashed */
1098	575	li r9,MSR_KERNEL & ~(MSR_IR\|MSR_DR)
1099	576	mtlr r6
1100		− stw r7, THREAD + RTAS_SP(r2)
	577	+ stw r1, THREAD + RTAS_SP(r2)
1101	578	mtspr SPRN_SRR0,r8
1102	579	mtspr SPRN_SRR1,r9
1103	580	rfi
1104		−1: tophys_novmstack r9, r1
1105		−#ifdef CONFIG_VMAP_STACK
1106		− li r0, MSR_KERNEL & ~MSR_IR /* can take DTLB miss */
1107		− mtmsr r0
1108		− isync
1109		−#endif
1110		− lwz r8,INT_FRAME_SIZE+4(r9) /* get return address */
1111		− lwz r9,8(r9) /* original msr value */
1112		− addi r1,r1,INT_FRAME_SIZE
1113		− li r0,0
1114		− tophys_novmstack r7, r2
1115		− stw r0, THREAD + RTAS_SP(r7)
	581	+1:
	582	+ lis r8, 1f@h
	583	+ ori r8, r8, 1f@l
	584	+ LOAD_REG_IMMEDIATE(r9,MSR_KERNEL)
1116	585	mtspr SPRN_SRR0,r8
1117	586	mtspr SPRN_SRR1,r9
1118		− rfi /* return to caller */
	587	+ rfi /* Reactivate MMU translation */
	588	+1:
	589	+ lwz r8,INT_FRAME_SIZE+4(r1) /* get return address */
	590	+ lwz r9,8(r1) /* original msr value */
	591	+ addi r1,r1,INT_FRAME_SIZE
	592	+ li r0,0
	593	+ stw r0, THREAD + RTAS_SP(r2)
	594	+ mtlr r8
	595	+ mtmsr r9
	596	+ blr /* return to caller */
1119	597	_ASM_NOKPROBE_SYMBOL(enter_rtas)
1120	598	#endif /* CONFIG_PPC_RTAS */

arch/powerpc/kernel/entry_64.S+15−25

@@ -117,13 +117,12 @@ BEGIN_FTR_SECTION · 116 unmodified lines
117	117	END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
118	118
119	119	/*
120		− * RECONCILE_IRQ_STATE without calling trace_hardirqs_off(), which
121		− * would clobber syscall parameters. Also we always enter with IRQs
122		− * enabled and nothing pending. system_call_exception() will call
123		− * trace_hardirqs_off().
124		− *
125		− * scv enters with MSR[EE]=1, so don't set PACA_IRQ_HARD_DIS. The
126		− * entry vector already sets PACAIRQSOFTMASK to IRQS_ALL_DISABLED.
	120	+ * scv enters with MSR[EE]=1 and is immediately considered soft-masked.
	121	+ * The entry vector already sets PACAIRQSOFTMASK to IRQS_ALL_DISABLED,
	122	+ * and interrupts may be masked and pending already.
	123	+ * system_call_exception() will call trace_hardirqs_off() which means
	124	+ * interrupts could already have been blocked before trace_hardirqs_off,
	125	+ * but this is the best we can do.
127	126	*/
128	127
129	128	/* Calling convention has r9 = orig r0, r10 = regs */
@@ -288,9 +287,8 @@ END_BTB_FLUSH_SECTION · 158 unmodified lines
288	287	std r11,-16(r10) /* "regshere" marker */
289	288
290	289	/*
291		− * RECONCILE_IRQ_STATE without calling trace_hardirqs_off(), which
292		− * would clobber syscall parameters. Also we always enter with IRQs
293		− * enabled and nothing pending. system_call_exception() will call
	290	+ * We always enter kernel from userspace with irq soft-mask enabled and
	291	+ * nothing pending. system_call_exception() will call
294	292	* trace_hardirqs_off().
295	293	*/
296	294	li r11,IRQS_ALL_DISABLED
@@ -417,19 +415,6 @@ _GLOBAL(ret_from_kernel_thread) · 120 unmodified lines
417	415	li r3,0
418	416	b .Lsyscall_exit
419	417
420		−#ifdef CONFIG_PPC_BOOK3E
421		−/* Save non-volatile GPRs, if not already saved. */
422		−_GLOBAL(save_nvgprs)
423		− ld r11,_TRAP(r1)
424		− andi. r0,r11,1
425		− beqlr-
426		− SAVE_NVGPRS(r1)
427		− clrrdi r0,r11,1
428		− std r0,_TRAP(r1)
429		− blr
430		−_ASM_NOKPROBE_SYMBOL(save_nvgprs);
431		−#endif
432		−
433	418	#ifdef CONFIG_PPC_BOOK3S_64
434	419
435	420	#define FLUSH_COUNT_CACHE \
@@ -645,7 +630,6 @@ END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S) · 209 unmodified lines
645	630	addi r1,r1,SWITCH_FRAME_SIZE
646	631	blr
647	632
648		−#ifdef CONFIG_PPC_BOOK3S
649	633	/*
650	634	* If MSR EE/RI was never enabled, IRQs not reconciled, NVGPRs not
651	635	* touched, no exit work created, then this can be used.
@@ -657,6 +641,7 @@ _ASM_NOKPROBE_SYMBOL(fast_interrupt_return) · 5 unmodified lines
657	641	kuap_check_amr r3, r4
658	642	ld r5,_MSR(r1)
659	643	andi. r0,r5,MSR_PR
	644	+#ifdef CONFIG_PPC_BOOK3S
660	645	bne .Lfast_user_interrupt_return_amr
661	646	kuap_kernel_restore r3, r4
662	647	andi. r0,r5,MSR_RI
@@ -665,6 +650,10 @@ _ASM_NOKPROBE_SYMBOL(fast_interrupt_return) · 2 unmodified lines
665	650	addi r3,r1,STACK_FRAME_OVERHEAD
666	651	bl unrecoverable_exception
667	652	b . /* should not get here */
	653	+#else
	654	+ bne .Lfast_user_interrupt_return
	655	+ b .Lfast_kernel_interrupt_return
	656	+#endif
668	657
669	658	.balign IFETCH_ALIGN_BYTES
670	659	.globl interrupt_return
@@ -678,8 +667,10 @@ _ASM_NOKPROBE_SYMBOL(interrupt_return) · 7 unmodified lines
678	667	cmpdi r3,0
679	668	bne- .Lrestore_nvgprs
680	669
	670	+#ifdef CONFIG_PPC_BOOK3S
681	671	.Lfast_user_interrupt_return_amr:
682	672	kuap_user_restore r3, r4
	673	+#endif
683	674	.Lfast_user_interrupt_return:
684	675	ld r11,_NIP(r1)
685	676	ld r12,_MSR(r1)
@@ -788,7 +779,6 @@ ALT_FTR_SECTION_END_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS) · 102 unmodified lines
788	779
789	780	RFI_TO_KERNEL
790	781	b . /* prevent speculative execution */
791		−#endif /* CONFIG_PPC_BOOK3S */
792	782
793	783	#ifdef CONFIG_PPC_RTAS
794	784	/*

arch/powerpc/kernel/exceptions-64e.S+31−394

@@ -63,9 +63,6 @@ · 62 unmodified lines
63	63	ld reg, (SPECIAL_EXC_##name * 8 + SPECIAL_EXC_FRAME_OFFS)(r1)
64	64
65	65	special_reg_save:
66		− lbz r9,PACAIRQHAPPENED(r13)
67		− RECONCILE_IRQ_STATE(r3,r4)
68		−
69	66	/*
70	67	* We only need (or have stack space) to save this stuff if
71	68	* we interrupted the kernel.
@@ -119,15 +116,11 @@ BEGIN_FTR_SECTION · 47 unmodified lines
119	116	mtspr SPRN_MAS5,r10
120	117	mtspr SPRN_MAS8,r10
121	118	END_FTR_SECTION_IFSET(CPU_FTR_EMB_HV)
122		− SPECIAL_EXC_STORE(r9,IRQHAPPENED)
123		−
124	119	mfspr r10,SPRN_DEAR
125	120	SPECIAL_EXC_STORE(r10,DEAR)
126	121	mfspr r10,SPRN_ESR
127	122	SPECIAL_EXC_STORE(r10,ESR)
128	123
129		− lbz r10,PACAIRQSOFTMASK(r13)
130		− SPECIAL_EXC_STORE(r10,SOFTE)
131	124	ld r10,_NIP(r1)
132	125	SPECIAL_EXC_STORE(r10,CSRR0)
133	126	ld r10,_MSR(r1)
@@ -139,7 +132,8 @@ ret_from_level_except: · 5 unmodified lines
139	132	ld r3,_MSR(r1)
140	133	andi. r3,r3,MSR_PR
141	134	beq 1f
142		− b ret_from_except
	135	+ REST_NVGPRS(r1)
	136	+ b interrupt_return
143	137	1:
144	138
145	139	LOAD_REG_ADDR(r11,extlb_level_exc)
@@ -193,27 +187,6 @@ BEGIN_FTR_SECTION · 47 unmodified lines
193	187	mtspr SPRN_MAS8,r10
194	188	END_FTR_SECTION_IFSET(CPU_FTR_EMB_HV)
195	189
196		− lbz r6,PACAIRQSOFTMASK(r13)
197		− ld r5,SOFTE(r1)
198		−
199		− /* Interrupts had better not already be enabled... */
200		− tweqi r6,IRQS_ENABLED
201		−
202		− andi. r6,r5,IRQS_DISABLED
203		− bne 1f
204		−
205		− TRACE_ENABLE_INTS
206		− stb r5,PACAIRQSOFTMASK(r13)
207		−1:
208		− /*
209		− * Restore PACAIRQHAPPENED rather than setting it based on
210		− * the return MSR[EE], since we could have interrupted
211		− * __check_irq_replay() or other inconsistent transitory
212		− * states that must remain that way.
213		− */
214		− SPECIAL_EXC_LOAD(r10,IRQHAPPENED)
215		− stb r10,PACAIRQHAPPENED(r13)
216		−
217	190	SPECIAL_EXC_LOAD(r10,DEAR)
218	191	mtspr SPRN_DEAR,r10
219	192	SPECIAL_EXC_LOAD(r10,ESR)
@@ -417,14 +390,15 @@ exc_##n##_common: \ · 197 unmodified lines
417	390	std r6,_LINK(r1); \
418	391	std r7,_CTR(r1); \
419	392	std r8,_XER(r1); \
420		− li r3,(n)+1; /* indicate partial regs in trap */ \
	393	+ li r3,(n); /* regs.trap vector */ \
421	394	std r9,0(r1); /* store stack frame back link */ \
422	395	std r10,_CCR(r1); /* store orig CR in stackframe */ \
423	396	std r9,GPR1(r1); /* store stack frame back link */ \
424	397	std r11,SOFTE(r1); /* and save it to stackframe */ \
425	398	std r12,STACK_FRAME_OVERHEAD-16(r1); /* mark the frame */ \
426	399	std r3,_TRAP(r1); /* set trap number */ \
427		− std r0,RESULT(r1); /* clear regs->result */
	400	+ std r0,RESULT(r1); /* clear regs->result */ \
	401	+ SAVE_NVGPRS(r1);
428	402
429	403	#define EXCEPTION_COMMON(n) \
430	404	EXCEPTION_COMMON_LVL(n, SPRN_SPRG_GEN_SCRATCH, PACA_EXGEN)
@@ -435,28 +409,6 @@ exc_##n##_common: \ · 4 unmodified lines
435	409	#define EXCEPTION_COMMON_DBG(n) \
436	410	EXCEPTION_COMMON_LVL(n, SPRN_SPRG_DBG_SCRATCH, PACA_EXDBG)
437	411
438		−/*
439		− * This is meant for exceptions that don't immediately hard-enable. We
440		− * set a bit in paca->irq_happened to ensure that a subsequent call to
441		− * arch_local_irq_restore() will properly hard-enable and avoid the
442		− * fast-path, and then reconcile irq state.
443		− */
444		−#define INTS_DISABLE RECONCILE_IRQ_STATE(r3,r4)
445		−
446		−/*
447		− * This is called by exceptions that don't use INTS_DISABLE (that did not
448		− * touch irq indicators in the PACA). This will restore MSR:EE to it's
449		− * previous value
450		− *
451		− * XXX In the long run, we may want to open-code it in order to separate the
452		− * load from the wrtee, thus limiting the latency caused by the dependency
453		− * but at this point, I'll favor code clarity until we have a near to final
454		− * implementation
455		− */
456		−#define INTS_RESTORE_HARD \
457		− ld r11,_MSR(r1); \
458		− wrtee r11;
459		−
460	412	/* XXX FIXME: Restore r14/r15 when necessary */
461	413	#define BAD_STACK_TRAMPOLINE(n) \
462	414	exc_##n##_bad_stack: \
@@ -505,12 +457,11 @@ exc_##n##_bad_stack: \ · 42 unmodified lines
505	457	START_EXCEPTION(label); \
506	458	NORMAL_EXCEPTION_PROLOG(trapnum, intnum, PROLOG_ADDITION_MASKABLE)\
507	459	EXCEPTION_COMMON(trapnum) \
508		− INTS_DISABLE; \
509	460	ack(r8); \
510	461	CHECK_NAPPING(); \
511	462	addi r3,r1,STACK_FRAME_OVERHEAD; \
512	463	bl hdlr; \
513		− b ret_from_except_lite;
	464	+ b interrupt_return
514	465
515	466	/* This value is used to mark exception frames on the stack. */
516	467	.section ".toc","aw"
@@ -561,11 +512,10 @@ __end_interrupts: · 44 unmodified lines
561	512	CRIT_EXCEPTION_PROLOG(0x100, BOOKE_INTERRUPT_CRITICAL,
562	513	PROLOG_ADDITION_NONE)
563	514	EXCEPTION_COMMON_CRIT(0x100)
564		− bl save_nvgprs
565	515	bl special_reg_save
566	516	CHECK_NAPPING();
567	517	addi r3,r1,STACK_FRAME_OVERHEAD
568		− bl unknown_exception
	518	+ bl unknown_nmi_exception
569	519	b ret_from_crit_except
570	520
571	521	/* Machine Check Interrupt */
@@ -573,7 +523,6 @@ __end_interrupts: · 1 unmodified line
573	523	MC_EXCEPTION_PROLOG(0x000, BOOKE_INTERRUPT_MACHINE_CHECK,
574	524	PROLOG_ADDITION_NONE)
575	525	EXCEPTION_COMMON_MC(0x000)
576		− bl save_nvgprs
577	526	bl special_reg_save
578	527	CHECK_NAPPING();
579	528	addi r3,r1,STACK_FRAME_OVERHEAD
@@ -587,7 +536,6 @@ __end_interrupts: · 7 unmodified lines
587	536	mfspr r14,SPRN_DEAR
588	537	mfspr r15,SPRN_ESR
589	538	EXCEPTION_COMMON(0x300)
590		− INTS_DISABLE
591	539	b storage_fault_common
592	540
593	541	/* Instruction Storage Interrupt */
@@ -597,7 +545,6 @@ __end_interrupts: · 3 unmodified lines
597	545	li r15,0
598	546	mr r14,r10
599	547	EXCEPTION_COMMON(0x400)
600		− INTS_DISABLE
601	548	b storage_fault_common
602	549
603	550	/* External Input Interrupt */
@@ -619,13 +566,12 @@ __end_interrupts: · 15 unmodified lines
619	566	PROLOG_ADDITION_1REG)
620	567	mfspr r14,SPRN_ESR
621	568	EXCEPTION_COMMON(0x700)
622		− INTS_DISABLE
623	569	std r14,_DSISR(r1)
624	570	addi r3,r1,STACK_FRAME_OVERHEAD
625	571	ld r14,PACA_EXGEN+EX_R14(r13)
626		− bl save_nvgprs
627	572	bl program_check_exception
628		− b ret_from_except
	573	+ REST_NVGPRS(r1)
	574	+ b interrupt_return
629	575
630	576	/* Floating Point Unavailable Interrupt */
631	577	START_EXCEPTION(fp_unavailable);
@@ -637,12 +583,10 @@ __end_interrupts: · 5 unmodified lines
637	583	andi. r0,r12,MSR_PR;
638	584	beq- 1f
639	585	bl load_up_fpu
640		− b fast_exception_return
641		−1: INTS_DISABLE
642		− bl save_nvgprs
643		− addi r3,r1,STACK_FRAME_OVERHEAD
	586	+ b fast_interrupt_return
	587	+1: addi r3,r1,STACK_FRAME_OVERHEAD
644	588	bl kernel_fp_unavailable_exception
645		− b ret_from_except
	589	+ b interrupt_return
646	590
647	591	/* Altivec Unavailable Interrupt */
648	592	START_EXCEPTION(altivec_unavailable);
@@ -656,15 +600,13 @@ BEGIN_FTR_SECTION · 7 unmodified lines
656	600	andi. r0,r12,MSR_PR;
657	601	beq- 1f
658	602	bl load_up_altivec
659		− b fast_exception_return
	603	+ b fast_interrupt_return
660	604	1:
661	605	END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
662	606	#endif
663		− INTS_DISABLE
664		− bl save_nvgprs
665	607	addi r3,r1,STACK_FRAME_OVERHEAD
666	608	bl altivec_unavailable_exception
667		− b ret_from_except
	609	+ b interrupt_return
668	610
669	611	/* AltiVec Assist */
670	612	START_EXCEPTION(altivec_assist);
@@ -672,17 +614,16 @@ END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) · 1 unmodified line
672	614	BOOKE_INTERRUPT_ALTIVEC_ASSIST,
673	615	PROLOG_ADDITION_NONE)
674	616	EXCEPTION_COMMON(0x220)
675		− INTS_DISABLE
676		− bl save_nvgprs
677	617	addi r3,r1,STACK_FRAME_OVERHEAD
678	618	#ifdef CONFIG_ALTIVEC
679	619	BEGIN_FTR_SECTION
680	620	bl altivec_assist_exception
681	621	END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
	622	+ REST_NVGPRS(r1)
682	623	#else
683	624	bl unknown_exception
684	625	#endif
685		− b ret_from_except
	626	+ b interrupt_return
686	627
687	628
688	629	/* Decrementer Interrupt */
@@ -698,14 +639,13 @@ END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) · 9 unmodified lines
698	639	CRIT_EXCEPTION_PROLOG(0x9f0, BOOKE_INTERRUPT_WATCHDOG,
699	640	PROLOG_ADDITION_NONE)
700	641	EXCEPTION_COMMON_CRIT(0x9f0)
701		− bl save_nvgprs
702	642	bl special_reg_save
703	643	CHECK_NAPPING();
704	644	addi r3,r1,STACK_FRAME_OVERHEAD
705	645	#ifdef CONFIG_BOOKE_WDT
706	646	bl WatchdogException
707	647	#else
708		− bl unknown_exception
	648	+ bl unknown_nmi_exception
709	649	#endif
710	650	b ret_from_crit_except
711	651
@@ -722,11 +662,9 @@ END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) · 10 unmodified lines
722	662	NORMAL_EXCEPTION_PROLOG(0xf20, BOOKE_INTERRUPT_AP_UNAVAIL,
723	663	PROLOG_ADDITION_NONE)
724	664	EXCEPTION_COMMON(0xf20)
725		− INTS_DISABLE
726		− bl save_nvgprs
727	665	addi r3,r1,STACK_FRAME_OVERHEAD
728	666	bl unknown_exception
729		− b ret_from_except
	667	+ b interrupt_return
730	668
731	669	/* Debug exception as a critical interrupt*/
732	670	START_EXCEPTION(debug_crit);
@@ -792,9 +730,9 @@ END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) · 59 unmodified lines
792	730	addi r3,r1,STACK_FRAME_OVERHEAD
793	731	ld r14,PACA_EXCRIT+EX_R14(r13)
794	732	ld r15,PACA_EXCRIT+EX_R15(r13)
795		− bl save_nvgprs
796	733	bl DebugException
797		− b ret_from_except
	734	+ REST_NVGPRS(r1)
	735	+ b interrupt_return
798	736
799	737	kernel_dbg_exc:
800	738	b . /* NYI */
@@ -859,24 +797,22 @@ kernel_dbg_exc: · 58 unmodified lines
859	797	*/
860	798	mfspr r14,SPRN_DBSR
861	799	EXCEPTION_COMMON_DBG(0xd08)
862		− INTS_DISABLE
863	800	std r14,_DSISR(r1)
864	801	addi r3,r1,STACK_FRAME_OVERHEAD
865	802	ld r14,PACA_EXDBG+EX_R14(r13)
866	803	ld r15,PACA_EXDBG+EX_R15(r13)
867		− bl save_nvgprs
868	804	bl DebugException
869		− b ret_from_except
	805	+ REST_NVGPRS(r1)
	806	+ b interrupt_return
870	807
871	808	START_EXCEPTION(perfmon);
872	809	NORMAL_EXCEPTION_PROLOG(0x260, BOOKE_INTERRUPT_PERFORMANCE_MONITOR,
873	810	PROLOG_ADDITION_NONE)
874	811	EXCEPTION_COMMON(0x260)
875		− INTS_DISABLE
876	812	CHECK_NAPPING()
877	813	addi r3,r1,STACK_FRAME_OVERHEAD
878	814	bl performance_monitor_exception
879		− b ret_from_except_lite
	815	+ b interrupt_return
880	816
881	817	/* Doorbell interrupt */
882	818	MASKABLE_EXCEPTION(0x280, BOOKE_INTERRUPT_DOORBELL,
@@ -887,11 +823,10 @@ kernel_dbg_exc: · 4 unmodified lines
887	823	CRIT_EXCEPTION_PROLOG(0x2a0, BOOKE_INTERRUPT_DOORBELL_CRITICAL,
888	824	PROLOG_ADDITION_NONE)
889	825	EXCEPTION_COMMON_CRIT(0x2a0)
890		− bl save_nvgprs
891	826	bl special_reg_save
892	827	CHECK_NAPPING();
893	828	addi r3,r1,STACK_FRAME_OVERHEAD
894		− bl unknown_exception
	829	+ bl unknown_nmi_exception
895	830	b ret_from_crit_except
896	831
897	832	/*
@@ -903,21 +838,18 @@ kernel_dbg_exc: · 5 unmodified lines
903	838	PROLOG_ADDITION_NONE)
904	839	EXCEPTION_COMMON(0x2c0)
905	840	addi r3,r1,STACK_FRAME_OVERHEAD
906		− bl save_nvgprs
907		− INTS_RESTORE_HARD
908	841	bl unknown_exception
909		− b ret_from_except
	842	+ b interrupt_return
910	843
911	844	/* Guest Doorbell critical Interrupt */
912	845	START_EXCEPTION(guest_doorbell_crit);
913	846	CRIT_EXCEPTION_PROLOG(0x2e0, BOOKE_INTERRUPT_GUEST_DBELL_CRIT,
914	847	PROLOG_ADDITION_NONE)
915	848	EXCEPTION_COMMON_CRIT(0x2e0)
916		− bl save_nvgprs
917	849	bl special_reg_save
918	850	CHECK_NAPPING();
919	851	addi r3,r1,STACK_FRAME_OVERHEAD
920		− bl unknown_exception
	852	+ bl unknown_nmi_exception
921	853	b ret_from_crit_except
922	854
923	855	/* Hypervisor call */
@@ -926,10 +858,8 @@ kernel_dbg_exc: · 2 unmodified lines
926	858	PROLOG_ADDITION_NONE)
927	859	EXCEPTION_COMMON(0x310)
928	860	addi r3,r1,STACK_FRAME_OVERHEAD
929		− bl save_nvgprs
930		− INTS_RESTORE_HARD
931	861	bl unknown_exception
932		− b ret_from_except
	862	+ b interrupt_return
933	863
934	864	/* Embedded Hypervisor priviledged */
935	865	START_EXCEPTION(ehpriv);
@@ -937,10 +867,8 @@ kernel_dbg_exc: · 1 unmodified line
937	867	PROLOG_ADDITION_NONE)
938	868	EXCEPTION_COMMON(0x320)
939	869	addi r3,r1,STACK_FRAME_OVERHEAD
940		− bl save_nvgprs
941		− INTS_RESTORE_HARD
942	870	bl unknown_exception
943		− b ret_from_except
	871	+ b interrupt_return
944	872
945	873	/* LRAT Error interrupt */
946	874	START_EXCEPTION(lrat_error);
@@ -948,10 +876,8 @@ kernel_dbg_exc: · 1 unmodified line
948	876	PROLOG_ADDITION_NONE)
949	877	EXCEPTION_COMMON(0x340)
950	878	addi r3,r1,STACK_FRAME_OVERHEAD
951		− bl save_nvgprs
952		− INTS_RESTORE_HARD
953	879	bl unknown_exception
954		− b ret_from_except
	880	+ b interrupt_return
955	881
956	882	/*
957	883	* An interrupt came in while soft-disabled; We mark paca->irq_happened
@@ -1011,14 +937,7 @@ storage_fault_common: · 53 unmodified lines
1011	937	ld r14,PACA_EXGEN+EX_R14(r13)
1012	938	ld r15,PACA_EXGEN+EX_R15(r13)
1013	939	bl do_page_fault
1014		− cmpdi r3,0
1015		− bne- 1f
1016		− b ret_from_except_lite
1017		−1: bl save_nvgprs
1018		− mr r4,r3
1019		− addi r3,r1,STACK_FRAME_OVERHEAD
1020		− bl __bad_page_fault
1021		− b ret_from_except
	940	+ b interrupt_return
1022	941
1023	942	/*
1024	943	* Alignment exception doesn't fit entirely in the 0x100 bytes so it
@@ -1030,291 +949,9 @@ alignment_more: · 5 unmodified lines
1030	949	addi r3,r1,STACK_FRAME_OVERHEAD
1031	950	ld r14,PACA_EXGEN+EX_R14(r13)
1032	951	ld r15,PACA_EXGEN+EX_R15(r13)
1033		− bl save_nvgprs
1034		− INTS_RESTORE_HARD
1035	952	bl alignment_exception
1036		− b ret_from_except
1037		−
1038		− .align 7
1039		−_GLOBAL(ret_from_except)
1040		− ld r11,_TRAP(r1)
1041		− andi. r0,r11,1
1042		− bne ret_from_except_lite
1043	953	REST_NVGPRS(r1)
1044		−
1045		−_GLOBAL(ret_from_except_lite)
1046		− /*
1047		− * Disable interrupts so that current_thread_info()->flags
1048		− * can't change between when we test it and when we return
1049		− * from the interrupt.
1050		− */
1051		− wrteei 0
1052		−
1053		− ld r9, PACA_THREAD_INFO(r13)
1054		− ld r3,_MSR(r1)
1055		− ld r10,PACACURRENT(r13)
1056		− ld r4,TI_FLAGS(r9)
1057		− andi. r3,r3,MSR_PR
1058		− beq resume_kernel
1059		− lwz r3,(THREAD+THREAD_DBCR0)(r10)
1060		−
1061		− /* Check current_thread_info()->flags */
1062		− andi. r0,r4,_TIF_USER_WORK_MASK
1063		− bne 1f
1064		− /*
1065		− * Check to see if the dbcr0 register is set up to debug.
1066		− * Use the internal debug mode bit to do this.
1067		− */
1068		− andis. r0,r3,DBCR0_IDM@h
1069		− beq restore
1070		− mfmsr r0
1071		− rlwinm r0,r0,0,~MSR_DE /* Clear MSR.DE */
1072		− mtmsr r0
1073		− mtspr SPRN_DBCR0,r3
1074		− li r10, -1
1075		− mtspr SPRN_DBSR,r10
1076		− b restore
1077		−1: andi. r0,r4,_TIF_NEED_RESCHED
1078		− beq 2f
1079		− bl restore_interrupts
1080		− SCHEDULE_USER
1081		− b ret_from_except_lite
1082		−2:
1083		− bl save_nvgprs
1084		− /*
1085		− * Use a non volatile GPR to save and restore our thread_info flags
1086		− * across the call to restore_interrupts.
1087		− */
1088		− mr r30,r4
1089		− bl restore_interrupts
1090		− mr r4,r30
1091		− addi r3,r1,STACK_FRAME_OVERHEAD
1092		− bl do_notify_resume
1093		− b ret_from_except
1094		−
1095		−resume_kernel:
1096		− /* check current_thread_info, _TIF_EMULATE_STACK_STORE */
1097		− andis. r8,r4,_TIF_EMULATE_STACK_STORE@h
1098		− beq+ 1f
1099		−
1100		− addi r8,r1,INT_FRAME_SIZE /* Get the kprobed function entry */
1101		−
1102		− ld r3,GPR1(r1)
1103		− subi r3,r3,INT_FRAME_SIZE /* dst: Allocate a trampoline exception frame */
1104		− mr r4,r1 /* src: current exception frame */
1105		− mr r1,r3 /* Reroute the trampoline frame to r1 */
1106		−
1107		− /* Copy from the original to the trampoline. */
1108		− li r5,INT_FRAME_SIZE/8 /* size: INT_FRAME_SIZE */
1109		− li r6,0 /* start offset: 0 */
1110		− mtctr r5
1111		−2: ldx r0,r6,r4
1112		− stdx r0,r6,r3
1113		− addi r6,r6,8
1114		− bdnz 2b
1115		−
1116		− /* Do real store operation to complete stdu */
1117		− ld r5,GPR1(r1)
1118		− std r8,0(r5)
1119		−
1120		− /* Clear _TIF_EMULATE_STACK_STORE flag */
1121		− lis r11,_TIF_EMULATE_STACK_STORE@h
1122		− addi r5,r9,TI_FLAGS
1123		−0: ldarx r4,0,r5
1124		− andc r4,r4,r11
1125		− stdcx. r4,0,r5
1126		− bne- 0b
1127		−1:
1128		−
1129		−#ifdef CONFIG_PREEMPT
1130		− /* Check if we need to preempt */
1131		− andi. r0,r4,_TIF_NEED_RESCHED
1132		− beq+ restore
1133		− /* Check that preempt_count() == 0 and interrupts are enabled */
1134		− lwz r8,TI_PREEMPT(r9)
1135		− cmpwi cr0,r8,0
1136		− bne restore
1137		− ld r0,SOFTE(r1)
1138		− andi. r0,r0,IRQS_DISABLED
1139		− bne restore
1140		−
1141		− /*
1142		− * Here we are preempting the current task. We want to make
1143		− * sure we are soft-disabled first and reconcile irq state.
1144		− */
1145		− RECONCILE_IRQ_STATE(r3,r4)
1146		− bl preempt_schedule_irq
1147		−
1148		− /*
1149		− * arch_local_irq_restore() from preempt_schedule_irq above may
1150		− * enable hard interrupt but we really should disable interrupts
1151		− * when we return from the interrupt, and so that we don't get
1152		− * interrupted after loading SRR0/1.
1153		− */
1154		− wrteei 0
1155		−#endif /* CONFIG_PREEMPT */
1156		−
1157		−restore:
1158		− /*
1159		− * This is the main kernel exit path. First we check if we
1160		− * are about to re-enable interrupts
1161		− */
1162		− ld r5,SOFTE(r1)
1163		− lbz r6,PACAIRQSOFTMASK(r13)
1164		− andi. r5,r5,IRQS_DISABLED
1165		− bne .Lrestore_irq_off
1166		−
1167		− /* We are enabling, were we already enabled ? Yes, just return */
1168		− andi. r6,r6,IRQS_DISABLED
1169		− beq cr0,fast_exception_return
1170		−
1171		− /*
1172		− * We are about to soft-enable interrupts (we are hard disabled
1173		− * at this point). We check if there's anything that needs to
1174		− * be replayed first.
1175		− */
1176		− lbz r0,PACAIRQHAPPENED(r13)
1177		− cmpwi cr0,r0,0
1178		− bne- .Lrestore_check_irq_replay
1179		−
1180		− /*
1181		− * Get here when nothing happened while soft-disabled, just
1182		− * soft-enable and move-on. We will hard-enable as a side
1183		− * effect of rfi
1184		− */
1185		−.Lrestore_no_replay:
1186		− TRACE_ENABLE_INTS
1187		− li r0,IRQS_ENABLED
1188		− stb r0,PACAIRQSOFTMASK(r13);
1189		−
1190		−/* This is the return from load_up_fpu fast path which could do with
1191		− * less GPR restores in fact, but for now we have a single return path
1192		− */
1193		−fast_exception_return:
1194		− wrteei 0
1195		−1: mr r0,r13
1196		− ld r10,_MSR(r1)
1197		− REST_4GPRS(2, r1)
1198		− andi. r6,r10,MSR_PR
1199		− REST_2GPRS(6, r1)
1200		− beq 1f
1201		− ACCOUNT_CPU_USER_EXIT(r13, r10, r11)
1202		− ld r0,GPR13(r1)
1203		−
1204		−1: stdcx. r0,0,r1 /* to clear the reservation */
1205		−
1206		− ld r8,_CCR(r1)
1207		− ld r9,_LINK(r1)
1208		− ld r10,_CTR(r1)
1209		− ld r11,_XER(r1)
1210		− mtcr r8
1211		− mtlr r9
1212		− mtctr r10
1213		− mtxer r11
1214		− REST_2GPRS(8, r1)
1215		− ld r10,GPR10(r1)
1216		− ld r11,GPR11(r1)
1217		− ld r12,GPR12(r1)
1218		− mtspr SPRN_SPRG_GEN_SCRATCH,r0
1219		−
1220		− std r10,PACA_EXGEN+EX_R10(r13);
1221		− std r11,PACA_EXGEN+EX_R11(r13);
1222		− ld r10,_NIP(r1)
1223		− ld r11,_MSR(r1)
1224		− ld r0,GPR0(r1)
1225		− ld r1,GPR1(r1)
1226		− mtspr SPRN_SRR0,r10
1227		− mtspr SPRN_SRR1,r11
1228		− ld r10,PACA_EXGEN+EX_R10(r13)
1229		− ld r11,PACA_EXGEN+EX_R11(r13)
1230		− mfspr r13,SPRN_SPRG_GEN_SCRATCH
1231		− rfi
1232		−
1233		− /*
1234		− * We are returning to a context with interrupts soft disabled.
1235		− *
1236		− * However, we may also about to hard enable, so we need to
1237		− * make sure that in this case, we also clear PACA_IRQ_HARD_DIS
1238		− * or that bit can get out of sync and bad things will happen
1239		− */
1240		−.Lrestore_irq_off:
1241		− ld r3,_MSR(r1)
1242		− lbz r7,PACAIRQHAPPENED(r13)
1243		− andi. r0,r3,MSR_EE
1244		− beq 1f
1245		− rlwinm r7,r7,0,~PACA_IRQ_HARD_DIS
1246		− stb r7,PACAIRQHAPPENED(r13)
1247		−1:
1248		−#if defined(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG) && defined(CONFIG_BUG)
1249		− /* The interrupt should not have soft enabled. */
1250		− lbz r7,PACAIRQSOFTMASK(r13)
1251		−1: tdeqi r7,IRQS_ENABLED
1252		− EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,BUGFLAG_WARNING
1253		−#endif
1254		− b fast_exception_return
1255		−
1256		− /*
1257		− * Something did happen, check if a re-emit is needed
1258		− * (this also clears paca->irq_happened)
1259		− */
1260		−.Lrestore_check_irq_replay:
1261		− /* XXX: We could implement a fast path here where we check
1262		− * for irq_happened being just 0x01, in which case we can
1263		− * clear it and return. That means that we would potentially
1264		− * miss a decrementer having wrapped all the way around.
1265		− *
1266		− * Still, this might be useful for things like hash_page
1267		− */
1268		− bl __check_irq_replay
1269		− cmpwi cr0,r3,0
1270		− beq .Lrestore_no_replay
1271		−
1272		− /*
1273		− * We need to re-emit an interrupt. We do so by re-using our
1274		− * existing exception frame. We first change the trap value,
1275		− * but we need to ensure we preserve the low nibble of it
1276		− */
1277		− ld r4,_TRAP(r1)
1278		− clrldi r4,r4,60
1279		− or r4,r4,r3
1280		− std r4,_TRAP(r1)
1281		−
1282		− /*
1283		− * PACA_IRQ_HARD_DIS won't always be set here, so set it now
1284		− * to reconcile the IRQ state. Tracing is already accounted for.
1285		− */
1286		− lbz r4,PACAIRQHAPPENED(r13)
1287		− ori r4,r4,PACA_IRQ_HARD_DIS
1288		− stb r4,PACAIRQHAPPENED(r13)
1289		−
1290		− /*
1291		− * Then find the right handler and call it. Interrupts are
1292		− * still soft-disabled and we keep them that way.
1293		− */
1294		− cmpwi cr0,r3,0x500
1295		− bne 1f
1296		− addi r3,r1,STACK_FRAME_OVERHEAD;
1297		− bl do_IRQ
1298		− b ret_from_except
1299		−1: cmpwi cr0,r3,0x900
1300		− bne 1f
1301		− addi r3,r1,STACK_FRAME_OVERHEAD;
1302		− bl timer_interrupt
1303		− b ret_from_except
1304		−#ifdef CONFIG_PPC_DOORBELL
1305		−1:
1306		− cmpwi cr0,r3,0x280
1307		− bne 1f
1308		− addi r3,r1,STACK_FRAME_OVERHEAD;
1309		− bl doorbell_exception
1310		−#endif /* CONFIG_PPC_DOORBELL */
1311		−1: b ret_from_except /* What else to do here ? */
1312		−
1313		−_ASM_NOKPROBE_SYMBOL(ret_from_except);
1314		−_ASM_NOKPROBE_SYMBOL(ret_from_except_lite);
1315		−_ASM_NOKPROBE_SYMBOL(resume_kernel);
1316		−_ASM_NOKPROBE_SYMBOL(restore);
1317		−_ASM_NOKPROBE_SYMBOL(fast_exception_return);
	954	+ b interrupt_return
1318	955
1319	956	/*
1320	957	* Trampolines used when spotting a bad kernel stack pointer in

arch/powerpc/kernel/exceptions-64s.S+7−53

@@ -692,25 +692,6 @@ END_FTR_SECTION_IFSET(CPU_FTR_CFAR) · 691 unmodified lines
692	692	ld r1,GPR1(r1)
693	693	.endm
694	694
695		−/*
696		− * When the idle code in power4_idle puts the CPU into NAP mode,
697		− * it has to do so in a loop, and relies on the external interrupt
698		− * and decrementer interrupt entry code to get it out of the loop.
699		− * It sets the _TLF_NAPPING bit in current_thread_info()->local_flags
700		− * to signal that it is in the loop and needs help to get out.
701		− */
702		−#ifdef CONFIG_PPC_970_NAP
703		−#define FINISH_NAP \
704		−BEGIN_FTR_SECTION \
705		− ld r11, PACA_THREAD_INFO(r13); \
706		− ld r9,TI_LOCAL_FLAGS(r11); \
707		− andi. r10,r9,_TLF_NAPPING; \
708		− bnel power4_fixup_nap; \
709		−END_FTR_SECTION_IFSET(CPU_FTR_CAN_NAP)
710		−#else
711		−#define FINISH_NAP
712		−#endif
713		−
714	695	/*
715	696	* There are a few constraints to be concerned with.
716	697	* - Real mode exceptions code/data must be located at their physical location.
@@ -1248,7 +1229,6 @@ EXC_COMMON_BEGIN(machine_check_common) · 531 unmodified lines
1248	1229	*/
1249	1230	GEN_COMMON machine_check
1250	1231
1251		− FINISH_NAP
1252	1232	/* Enable MSR_RI when finished with PACA_EXMC */
1253	1233	li r10,MSR_RI
1254	1234	mtmsrd r10,1
@@ -1571,7 +1551,6 @@ EXC_VIRT_BEGIN(hardware_interrupt, 0x4500, 0x100) · 316 unmodified lines
1571	1551	EXC_VIRT_END(hardware_interrupt, 0x4500, 0x100)
1572	1552	EXC_COMMON_BEGIN(hardware_interrupt_common)
1573	1553	GEN_COMMON hardware_interrupt
1574		− FINISH_NAP
1575	1554	addi r3,r1,STACK_FRAME_OVERHEAD
1576	1555	bl do_IRQ
1577	1556	b interrupt_return
@@ -1801,7 +1780,6 @@ EXC_VIRT_BEGIN(decrementer, 0x4900, 0x80) · 223 unmodified lines
1801	1780	EXC_VIRT_END(decrementer, 0x4900, 0x80)
1802	1781	EXC_COMMON_BEGIN(decrementer_common)
1803	1782	GEN_COMMON decrementer
1804		− FINISH_NAP
1805	1783	addi r3,r1,STACK_FRAME_OVERHEAD
1806	1784	bl timer_interrupt
1807	1785	b interrupt_return
@@ -1886,7 +1864,6 @@ EXC_VIRT_BEGIN(doorbell_super, 0x4a00, 0x100) · 78 unmodified lines
1886	1864	EXC_VIRT_END(doorbell_super, 0x4a00, 0x100)
1887	1865	EXC_COMMON_BEGIN(doorbell_super_common)
1888	1866	GEN_COMMON doorbell_super
1889		− FINISH_NAP
1890	1867	addi r3,r1,STACK_FRAME_OVERHEAD
1891	1868	#ifdef CONFIG_PPC_DOORBELL
1892	1869	bl doorbell_exception
@@ -2237,7 +2214,6 @@ EXC_COMMON_BEGIN(hmi_exception_early_common) · 344 unmodified lines
2237	2214
2238	2215	EXC_COMMON_BEGIN(hmi_exception_common)
2239	2216	GEN_COMMON hmi_exception
2240		− FINISH_NAP
2241	2217	addi r3,r1,STACK_FRAME_OVERHEAD
2242	2218	bl handle_hmi_exception
2243	2219	b interrupt_return
@@ -2266,7 +2242,6 @@ EXC_VIRT_BEGIN(h_doorbell, 0x4e80, 0x20) · 22 unmodified lines
2266	2242	EXC_VIRT_END(h_doorbell, 0x4e80, 0x20)
2267	2243	EXC_COMMON_BEGIN(h_doorbell_common)
2268	2244	GEN_COMMON h_doorbell
2269		− FINISH_NAP
2270	2245	addi r3,r1,STACK_FRAME_OVERHEAD
2271	2246	#ifdef CONFIG_PPC_DOORBELL
2272	2247	bl doorbell_exception
@@ -2299,7 +2274,6 @@ EXC_VIRT_BEGIN(h_virt_irq, 0x4ea0, 0x20) · 26 unmodified lines
2299	2274	EXC_VIRT_END(h_virt_irq, 0x4ea0, 0x20)
2300	2275	EXC_COMMON_BEGIN(h_virt_irq_common)
2301	2276	GEN_COMMON h_virt_irq
2302		− FINISH_NAP
2303	2277	addi r3,r1,STACK_FRAME_OVERHEAD
2304	2278	bl do_IRQ
2305	2279	b interrupt_return
@@ -2345,7 +2319,6 @@ EXC_VIRT_BEGIN(performance_monitor, 0x4f00, 0x20) · 39 unmodified lines
2345	2319	EXC_VIRT_END(performance_monitor, 0x4f00, 0x20)
2346	2320	EXC_COMMON_BEGIN(performance_monitor_common)
2347	2321	GEN_COMMON performance_monitor
2348		− FINISH_NAP
2349	2322	addi r3,r1,STACK_FRAME_OVERHEAD
2350	2323	bl performance_monitor_exception
2351	2324	b interrupt_return
@@ -2530,8 +2503,6 @@ EXC_VIRT_NONE(0x5100, 0x100) · 178 unmodified lines
2530	2503	INT_DEFINE_BEGIN(cbe_system_error)
2531	2504	IVEC=0x1200
2532	2505	IHSRR=1
2533		− IKVM_SKIP=1
2534		− IKVM_REAL=1
2535	2506	INT_DEFINE_END(cbe_system_error)
2536	2507
2537	2508	EXC_REAL_BEGIN(cbe_system_error, 0x1200, 0x100)
@@ -2551,11 +2522,16 @@ EXC_REAL_NONE(0x1200, 0x100) · 13 unmodified lines
2551	2522	EXC_VIRT_NONE(0x5200, 0x100)
2552	2523	#endif
2553	2524
2554		−
	2525	+/**
	2526	+ * Interrupt 0x1300 - Instruction Address Breakpoint Interrupt.
	2527	+ * This has been removed from the ISA before 2.01, which is the earliest
	2528	+ * 64-bit BookS ISA supported, however the G5 / 970 implements this
	2529	+ * interrupt with a non-architected feature available through the support
	2530	+ * processor interface.
	2531	+ */
2555	2532	INT_DEFINE_BEGIN(instruction_breakpoint)
2556	2533	IVEC=0x1300
2557	2534	#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
2558		− IKVM_SKIP=1
2559	2535	IKVM_REAL=1
2560	2536	#endif
2561	2537	INT_DEFINE_END(instruction_breakpoint)
@@ -2701,8 +2677,6 @@ EXC_COMMON_BEGIN(denorm_exception_common) · 139 unmodified lines
2701	2677	INT_DEFINE_BEGIN(cbe_maintenance)
2702	2678	IVEC=0x1600
2703	2679	IHSRR=1
2704		− IKVM_SKIP=1
2705		− IKVM_REAL=1
2706	2680	INT_DEFINE_END(cbe_maintenance)
2707	2681
2708	2682	EXC_REAL_BEGIN(cbe_maintenance, 0x1600, 0x100)
@@ -2754,8 +2728,6 @@ EXC_COMMON_BEGIN(altivec_assist_common) · 45 unmodified lines
2754	2728	INT_DEFINE_BEGIN(cbe_thermal)
2755	2729	IVEC=0x1800
2756	2730	IHSRR=1
2757		− IKVM_SKIP=1
2758		− IKVM_REAL=1
2759	2731	INT_DEFINE_END(cbe_thermal)
2760	2732
2761	2733	EXC_REAL_BEGIN(cbe_thermal, 0x1800, 0x100)
@@ -3096,24 +3068,6 @@ USE_FIXED_SECTION(virt_trampolines) · 334 unmodified lines
3096	3068	__end_interrupts:
3097	3069	DEFINE_FIXED_SYMBOL(__end_interrupts)
3098	3070
3099		−#ifdef CONFIG_PPC_970_NAP
3100		− /*
3101		− * Called by exception entry code if _TLF_NAPPING was set, this clears
3102		− * the NAPPING flag, and redirects the exception exit to
3103		− * power4_fixup_nap_return.
3104		− */
3105		− .globl power4_fixup_nap
3106		−EXC_COMMON_BEGIN(power4_fixup_nap)
3107		− andc r9,r9,r10
3108		− std r9,TI_LOCAL_FLAGS(r11)
3109		− LOAD_REG_ADDR(r10, power4_idle_nap_return)
3110		− std r10,_NIP(r1)
3111		− blr
3112		−
3113		−power4_idle_nap_return:
3114		− blr
3115		−#endif
3116		−
3117	3071	CLOSE_FIXED_SECTION(real_vectors);
3118	3072	CLOSE_FIXED_SECTION(real_trampolines);
3119	3073	CLOSE_FIXED_SECTION(virt_vectors);

arch/powerpc/kernel/fadump.c+9−9

@@ -31,6 +31,7 @@ · 30 unmodified lines
31	31	#include <asm/fadump.h>
32	32	#include <asm/fadump-internal.h>
33	33	#include <asm/setup.h>
	34	+#include <asm/interrupt.h>
34	35
35	36	/*
36	37	* The CPU who acquired the lock to trigger the fadump crash should
@@ -44,22 +45,21 @@ static struct fw_dump fw_dump; · 7 unmodified lines
44	45
45	46	static void __init fadump_reserve_crash_area(u64 base);
46	47
47		−struct kobject *fadump_kobj;
48		−
49	48	#ifndef CONFIG_PRESERVE_FA_DUMP
50	49
	50	+static struct kobject *fadump_kobj;
	51	+
51	52	static atomic_t cpus_in_fadump;
52	53	static DEFINE_MUTEX(fadump_mutex);
53	54
54		−struct fadump_mrange_info crash_mrange_info = { "crash", NULL, 0, 0, 0, false };
	55	+static struct fadump_mrange_info crash_mrange_info = { "crash", NULL, 0, 0, 0, false };
55	56
56	57	#define RESERVED_RNGS_SZ 16384 /* 16K - 128 entries */
57	58	#define RESERVED_RNGS_CNT (RESERVED_RNGS_SZ / \
58	59	sizeof(struct fadump_memory_range))
59	60	static struct fadump_memory_range rngs[RESERVED_RNGS_CNT];
60		−struct fadump_mrange_info reserved_mrange_info = { "reserved", rngs,
61		− RESERVED_RNGS_SZ, 0,
62		− RESERVED_RNGS_CNT, true };
	61	+static struct fadump_mrange_info
	62	+reserved_mrange_info = { "reserved", rngs, RESERVED_RNGS_SZ, 0, RESERVED_RNGS_CNT, true };
63	63
64	64	static void __init early_init_dt_scan_reserved_ranges(unsigned long node);
65	65
@@ -79,7 +79,7 @@ static struct cma *fadump_cma; · 13 unmodified lines
79	79	* But for some reason even if it fails we still have the memory reservation
80	80	* with us and we can still continue doing fadump.
81	81	*/
82		−int __init fadump_cma_init(void)
	82	+static int __init fadump_cma_init(void)
83	83	{
84	84	unsigned long long base, size;
85	85	int rc;
@@ -292,7 +292,7 @@ static void fadump_show_config(void) · 206 unmodified lines
292	292	* that is required for a kernel to boot successfully.
293	293	*
294	294	*/
295		−static inline u64 fadump_calculate_reserve_size(void)
	295	+static __init u64 fadump_calculate_reserve_size(void)
296	296	{
297	297	u64 base, size, bootmem_min;
298	298	int ret;
@@ -728,7 +728,7 @@ void crash_fadump(struct pt_regs regs, const char str) · 429 unmodified lines
728	728	* If we came in via system reset, wait a while for the secondary
729	729	* CPUs to enter.
730	730	*/
731		− if (TRAP(&(fdh->regs)) == 0x100) {
	731	+ if (TRAP(&(fdh->regs)) == INTERRUPT_SYSTEM_RESET) {
732	732	msecs = CRASH_TIMEOUT;
733	733	while ((atomic_read(&cpus_in_fadump) < ncpus) && (--msecs > 0))
734	734	mdelay(1);

arch/powerpc/kernel/fpu.S+0−2

@@ -92,9 +92,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_VSX) · 91 unmodified lines
92	92	/* enable use of FP after return */
93	93	#ifdef CONFIG_PPC32
94	94	mfspr r5,SPRN_SPRG_THREAD /* current task's THREAD (phys) */
95		−#ifdef CONFIG_VMAP_STACK
96	95	tovirt(r5, r5)
97		−#endif
98	96	lwz r4,THREAD_FPEXC_MODE(r5)
99	97	ori r9,r9,MSR_FP /* enable FP for current */
100	98	or r9,r9,r4

arch/powerpc/kernel/head_32.h+79−119

@@ -10,36 +10,39 @@ · 9 unmodified lines
10	10	* We assume sprg3 has the physical address of the current
11	11	* task's thread_struct.
12	12	*/
13		−.macro EXCEPTION_PROLOG handle_dar_dsisr=0
	13	+.macro EXCEPTION_PROLOG trapno name handle_dar_dsisr=0
14	14	EXCEPTION_PROLOG_0 handle_dar_dsisr=\handle_dar_dsisr
15	15	EXCEPTION_PROLOG_1
16		− EXCEPTION_PROLOG_2 handle_dar_dsisr=\handle_dar_dsisr
	16	+ EXCEPTION_PROLOG_2 \trapno \name handle_dar_dsisr=\handle_dar_dsisr
17	17	.endm
18	18
19	19	.macro EXCEPTION_PROLOG_0 handle_dar_dsisr=0
20	20	mtspr SPRN_SPRG_SCRATCH0,r10
21	21	mtspr SPRN_SPRG_SCRATCH1,r11
22		−#ifdef CONFIG_VMAP_STACK
23	22	mfspr r10, SPRN_SPRG_THREAD
24	23	.if \handle_dar_dsisr
	24	+#ifdef CONFIG_40x
	25	+ mfspr r11, SPRN_DEAR
	26	+#else
25	27	mfspr r11, SPRN_DAR
	28	+#endif
26	29	stw r11, DAR(r10)
	30	+#ifdef CONFIG_40x
	31	+ mfspr r11, SPRN_ESR
	32	+#else
27	33	mfspr r11, SPRN_DSISR
	34	+#endif
28	35	stw r11, DSISR(r10)
29	36	.endif
30	37	mfspr r11, SPRN_SRR0
31	38	stw r11, SRR0(r10)
32		−#endif
33	39	mfspr r11, SPRN_SRR1 /* check whether user or kernel */
34		−#ifdef CONFIG_VMAP_STACK
35	40	stw r11, SRR1(r10)
36		−#endif
37	41	mfcr r10
38	42	andi. r11, r11, MSR_PR
39	43	.endm
40	44
41		−.macro EXCEPTION_PROLOG_1 for_rtas=0
42		−#ifdef CONFIG_VMAP_STACK
	45	+.macro EXCEPTION_PROLOG_1
43	46	mtspr SPRN_SPRG_SCRATCH2,r1
44	47	subi r1, r1, INT_FRAME_SIZE /* use r1 if kernel */
45	48	beq 1f
@@ -47,32 +50,33 @@ · 1 unmodified line
47	50	lwz r1,TASK_STACK-THREAD(r1)
48	51	addi r1, r1, THREAD_SIZE - INT_FRAME_SIZE
49	52	1:
	53	+#ifdef CONFIG_VMAP_STACK
50	54	mtcrf 0x3f, r1
51		− bt 32 - THREAD_ALIGN_SHIFT, stack_overflow
52		−#else
53		− subi r11, r1, INT_FRAME_SIZE /* use r1 if kernel */
54		− beq 1f
55		− mfspr r11,SPRN_SPRG_THREAD
56		− lwz r11,TASK_STACK-THREAD(r11)
57		− addi r11, r11, THREAD_SIZE - INT_FRAME_SIZE
58		−1: tophys(r11, r11)
	55	+ bt 32 - THREAD_ALIGN_SHIFT, vmap_stack_overflow
59	56	#endif
60	57	.endm
61	58
62		−.macro EXCEPTION_PROLOG_2 handle_dar_dsisr=0
63		−#ifdef CONFIG_VMAP_STACK
64		− li r11, MSR_KERNEL & ~(MSR_IR \| MSR_RI) /* can take DTLB miss */
65		− mtmsr r11
66		− isync
	59	+.macro EXCEPTION_PROLOG_2 trapno name handle_dar_dsisr=0
	60	+#ifdef CONFIG_PPC_8xx
	61	+ .if \handle_dar_dsisr
	62	+ li r11, RPN_PATTERN
	63	+ mtspr SPRN_DAR, r11 /* Tag DAR, to be used in DTLB Error */
	64	+ .endif
	65	+#endif
	66	+ LOAD_REG_IMMEDIATE(r11, MSR_KERNEL & ~MSR_RI) /* re-enable MMU */
	67	+ mtspr SPRN_SRR1, r11
	68	+ lis r11, 1f@h
	69	+ ori r11, r11, 1f@l
	70	+ mtspr SPRN_SRR0, r11
67	71	mfspr r11, SPRN_SPRG_SCRATCH2
	72	+ rfi
	73	+
	74	+ .text
	75	+\name\()_virt:
	76	+1:
68	77	stw r11,GPR1(r1)
69	78	stw r11,0(r1)
70	79	mr r11, r1
71		−#else
72		− stw r1,GPR1(r11)
73		− stw r1,0(r11)
74		− tovirt(r1, r11) /* set new kernel sp */
75		−#endif
76	80	stw r10,_CCR(r11) /* save registers */
77	81	stw r12,GPR12(r11)
78	82	stw r9,GPR9(r11)
@@ -82,7 +86,6 @@ · 3 unmodified lines
82	86	stw r12,GPR11(r11)
83	87	mflr r10
84	88	stw r10,_LINK(r11)
85		−#ifdef CONFIG_VMAP_STACK
86	89	mfspr r12, SPRN_SPRG_THREAD
87	90	tovirt(r12, r12)
88	91	.if \handle_dar_dsisr
@@ -93,26 +96,48 @@ · 4 unmodified lines
93	96	.endif
94	97	lwz r9, SRR1(r12)
95	98	lwz r12, SRR0(r12)
96		−#else
97		− mfspr r12,SPRN_SRR0
98		− mfspr r9,SPRN_SRR1
99		−#endif
100	99	#ifdef CONFIG_40x
101	100	rlwinm r9,r9,0,14,12 /* clear MSR_WE (necessary?) */
	101	+#elif defined(CONFIG_PPC_8xx)
	102	+ mtspr SPRN_EID, r2 /* Set MSR_RI */
102	103	#else
103		−#ifdef CONFIG_VMAP_STACK
104		− li r10, MSR_KERNEL & ~MSR_IR /* can take exceptions */
105		−#else
106		− li r10,MSR_KERNEL & ~(MSR_IR\|MSR_DR) /* can take exceptions */
107		−#endif
	104	+ li r10, MSR_KERNEL /* can take exceptions */
108	105	mtmsr r10 /* (except for mach check in rtas) */
109	106	#endif
110		− stw r0,GPR0(r11)
	107	+ COMMON_EXCEPTION_PROLOG_END \trapno
	108	+_ASM_NOKPROBE_SYMBOL(\name\()_virt)
	109	+.endm
	110	+
	111	+.macro COMMON_EXCEPTION_PROLOG_END trapno
	112	+ stw r0,GPR0(r1)
111	113	lis r10,STACK_FRAME_REGS_MARKER@ha /* exception frame marker */
112	114	addi r10,r10,STACK_FRAME_REGS_MARKER@l
113		− stw r10,8(r11)
114		− SAVE_4GPRS(3, r11)
115		− SAVE_2GPRS(7, r11)
	115	+ stw r10,8(r1)
	116	+ li r10, \trapno
	117	+ stw r10,_TRAP(r1)
	118	+ SAVE_4GPRS(3, r1)
	119	+ SAVE_2GPRS(7, r1)
	120	+ SAVE_NVGPRS(r1)
	121	+ stw r2,GPR2(r1)
	122	+ stw r12,_NIP(r1)
	123	+ stw r9,_MSR(r1)
	124	+ mfctr r10
	125	+ mfspr r2,SPRN_SPRG_THREAD
	126	+ stw r10,_CTR(r1)
	127	+ tovirt(r2, r2)
	128	+ mfspr r10,SPRN_XER
	129	+ addi r2, r2, -THREAD
	130	+ stw r10,_XER(r1)
	131	+ addi r3,r1,STACK_FRAME_OVERHEAD
	132	+.endm
	133	+
	134	+.macro prepare_transfer_to_handler
	135	+#ifdef CONFIG_PPC_BOOK3S_32
	136	+ andi. r12,r9,MSR_PR
	137	+ bne 777f
	138	+ bl prepare_transfer_to_handler
	139	+777:
	140	+#endif
116	141	.endm
117	142
118	143	.macro SYSCALL_ENTRY trapno
@@ -156,54 +181,6 @@ · 37 unmodified lines
156	181	b transfer_to_syscall /* jump to handler */
157	182	.endm
158	183
159		−.macro save_dar_dsisr_on_stack reg1, reg2, sp
160		−#ifndef CONFIG_VMAP_STACK
161		− mfspr \reg1, SPRN_DAR
162		− mfspr \reg2, SPRN_DSISR
163		− stw \reg1, _DAR(\sp)
164		− stw \reg2, _DSISR(\sp)
165		−#endif
166		−.endm
167		−
168		−.macro get_and_save_dar_dsisr_on_stack reg1, reg2, sp
169		−#ifdef CONFIG_VMAP_STACK
170		− lwz \reg1, _DAR(\sp)
171		− lwz \reg2, _DSISR(\sp)
172		−#else
173		− save_dar_dsisr_on_stack \reg1, \reg2, \sp
174		−#endif
175		−.endm
176		−
177		−.macro tovirt_vmstack dst, src
178		−#ifdef CONFIG_VMAP_STACK
179		− tovirt(\dst, \src)
180		−#else
181		− .ifnc \dst, \src
182		− mr \dst, \src
183		− .endif
184		−#endif
185		−.endm
186		−
187		−.macro tovirt_novmstack dst, src
188		−#ifndef CONFIG_VMAP_STACK
189		− tovirt(\dst, \src)
190		−#else
191		− .ifnc \dst, \src
192		− mr \dst, \src
193		− .endif
194		−#endif
195		−.endm
196		−
197		−.macro tophys_novmstack dst, src
198		−#ifndef CONFIG_VMAP_STACK
199		− tophys(\dst, \src)
200		−#else
201		− .ifnc \dst, \src
202		− mr \dst, \src
203		− .endif
204		−#endif
205		−.endm
206		−
207	184	/*
208	185	* Note: code which follows this uses cr0.eq (set if from kernel),
209	186	* r11, r12 (SRR0), and r9 (SRR1).
@@ -217,41 +194,29 @@ · 7 unmodified lines
217	194	*/
218	195	#ifdef CONFIG_PPC_BOOK3S
219	196	#define START_EXCEPTION(n, label) \
	197	+ __HEAD; \
220	198	. = n; \
221	199	DO_KVM n; \
222	200	label:
223	201
224	202	#else
225	203	#define START_EXCEPTION(n, label) \
	204	+ __HEAD; \
226	205	. = n; \
227	206	label:
228	207
229	208	#endif
230	209
231		−#define EXCEPTION(n, label, hdlr, xfer) \
	210	+#define EXCEPTION(n, label, hdlr) \
232	211	START_EXCEPTION(n, label) \
233		− EXCEPTION_PROLOG; \
234		− addi r3,r1,STACK_FRAME_OVERHEAD; \
235		− xfer(n, hdlr)
236		−
237		−#define EXC_XFER_TEMPLATE(hdlr, trap, msr, tfer, ret) \
238		− li r10,trap; \
239		− stw r10,_TRAP(r11); \
240		− LOAD_REG_IMMEDIATE(r10, msr); \
241		− bl tfer; \
242		− .long hdlr; \
243		− .long ret
244		−
245		−#define EXC_XFER_STD(n, hdlr) \
246		− EXC_XFER_TEMPLATE(hdlr, n, MSR_KERNEL, transfer_to_handler_full, \
247		− ret_from_except_full)
248		−
249		−#define EXC_XFER_LITE(n, hdlr) \
250		− EXC_XFER_TEMPLATE(hdlr, n+1, MSR_KERNEL, transfer_to_handler, \
251		− ret_from_except)
	212	+ EXCEPTION_PROLOG n label; \
	213	+ prepare_transfer_to_handler; \
	214	+ bl hdlr; \
	215	+ b interrupt_return
252	216
253	217	.macro vmap_stack_overflow_exception
254		−#ifdef CONFIG_VMAP_STACK
	218	+ __HEAD
	219	+vmap_stack_overflow:
255	220	#ifdef CONFIG_SMP
256	221	mfspr r1, SPRN_SPRG_THREAD
257	222	lwz r1, TASK_CPU - THREAD(r1)
@@ -261,16 +226,11 @@ label: · 3 unmodified lines
261	226	lis r1, emergency_ctx@ha
262	227	#endif
263	228	lwz r1, emergency_ctx@l(r1)
264		− cmpwi cr1, r1, 0
265		− bne cr1, 1f
266		− lis r1, init_thread_union@ha
267		− addi r1, r1, init_thread_union@l
268		−1: addi r1, r1, THREAD_SIZE - INT_FRAME_SIZE
269		− EXCEPTION_PROLOG_2
270		− SAVE_NVGPRS(r11)
271		− addi r3, r1, STACK_FRAME_OVERHEAD
272		− EXC_XFER_STD(0, stack_overflow_exception)
273		−#endif
	229	+ addi r1, r1, THREAD_SIZE - INT_FRAME_SIZE
	230	+ EXCEPTION_PROLOG_2 0 vmap_stack_overflow
	231	+ prepare_transfer_to_handler
	232	+ bl stack_overflow_exception
	233	+ b interrupt_return
274	234	.endm
275	235
276	236	#endif /* __HEAD_32_H__ */

arch/powerpc/kernel/head_40x.S+147−124

@@ -89,7 +89,11 @@ _ENTRY(crit_srr0) · 88 unmodified lines
89	89	.space 4
90	90	_ENTRY(crit_srr1)
91	91	.space 4
92		−_ENTRY(saved_ksp_limit)
	92	+_ENTRY(crit_r1)
	93	+ .space 4
	94	+_ENTRY(crit_dear)
	95	+ .space 4
	96	+_ENTRY(crit_esr)
93	97	.space 4
94	98
95	99	/*
@@ -100,42 +104,62 @@ _ENTRY(saved_ksp_limit) · 4 unmodified lines
100	104	* Instead we use a couple of words of memory at low physical addresses.
101	105	* This is OK since we don't support SMP on these processors.
102	106	*/
103		−#define CRITICAL_EXCEPTION_PROLOG \
104		− stw r10,crit_r10@l(0); /* save two registers to work with */\
105		− stw r11,crit_r11@l(0); \
106		− mfcr r10; /* save CR in r10 for now */\
107		− mfspr r11,SPRN_SRR3; /* check whether user or kernel */\
108		− andi. r11,r11,MSR_PR; \
109		− lis r11,critirq_ctx@ha; \
110		− tophys(r11,r11); \
111		− lwz r11,critirq_ctx@l(r11); \
112		− beq 1f; \
113		− /* COMING FROM USER MODE */ \
114		− mfspr r11,SPRN_SPRG_THREAD; /* if from user, start at top of */\
115		− lwz r11,TASK_STACK-THREAD(r11); /* this thread's kernel stack */\
116		−1: addi r11,r11,THREAD_SIZE-INT_FRAME_SIZE; /* Alloc an excpt frm */\
117		− tophys(r11,r11); \
118		− stw r10,_CCR(r11); /* save various registers */\
119		− stw r12,GPR12(r11); \
120		− stw r9,GPR9(r11); \
121		− mflr r10; \
122		− stw r10,_LINK(r11); \
123		− mfspr r12,SPRN_DEAR; /* save DEAR and ESR in the frame */\
124		− stw r12,_DEAR(r11); /* since they may have had stuff */\
125		− mfspr r9,SPRN_ESR; /* in them at the point where the */\
126		− stw r9,_ESR(r11); /* exception was taken */\
127		− mfspr r12,SPRN_SRR2; \
128		− stw r1,GPR1(r11); \
129		− mfspr r9,SPRN_SRR3; \
130		− stw r1,0(r11); \
131		− tovirt(r1,r11); \
132		− rlwinm r9,r9,0,14,12; /* clear MSR_WE (necessary?) */\
133		− stw r0,GPR0(r11); \
134		− lis r10, STACK_FRAME_REGS_MARKER@ha; /* exception frame marker */\
135		− addi r10, r10, STACK_FRAME_REGS_MARKER@l; \
136		− stw r10, 8(r11); \
137		− SAVE_4GPRS(3, r11); \
138		− SAVE_2GPRS(7, r11)
	107	+.macro CRITICAL_EXCEPTION_PROLOG trapno name
	108	+ stw r10,crit_r10@l(0) /* save two registers to work with */
	109	+ stw r11,crit_r11@l(0)
	110	+ mfspr r10,SPRN_SRR0
	111	+ mfspr r11,SPRN_SRR1
	112	+ stw r10,crit_srr0@l(0)
	113	+ stw r11,crit_srr1@l(0)
	114	+ mfspr r10,SPRN_DEAR
	115	+ mfspr r11,SPRN_ESR
	116	+ stw r10,crit_dear@l(0)
	117	+ stw r11,crit_esr@l(0)
	118	+ mfcr r10 /* save CR in r10 for now */
	119	+ mfspr r11,SPRN_SRR3 /* check whether user or kernel */
	120	+ andi. r11,r11,MSR_PR
	121	+ lis r11,(critirq_ctx-PAGE_OFFSET)@ha
	122	+ lwz r11,(critirq_ctx-PAGE_OFFSET)@l(r11)
	123	+ beq 1f
	124	+ /* COMING FROM USER MODE */
	125	+ mfspr r11,SPRN_SPRG_THREAD /* if from user, start at top of */
	126	+ lwz r11,TASK_STACK-THREAD(r11) /* this thread's kernel stack */
	127	+1: stw r1,crit_r1@l(0)
	128	+ addi r1,r11,THREAD_SIZE-INT_FRAME_SIZE /* Alloc an excpt frm */
	129	+ LOAD_REG_IMMEDIATE(r11, MSR_KERNEL & ~(MSR_ME\|MSR_DE\|MSR_CE)) /* re-enable MMU */
	130	+ mtspr SPRN_SRR1, r11
	131	+ lis r11, 1f@h
	132	+ ori r11, r11, 1f@l
	133	+ mtspr SPRN_SRR0, r11
	134	+ rfi
	135	+
	136	+ .text
	137	+1:
	138	+\name\()_virt:
	139	+ lwz r11,crit_r1@l(0)
	140	+ stw r11,GPR1(r1)
	141	+ stw r11,0(r1)
	142	+ mr r11,r1
	143	+ stw r10,_CCR(r11) /* save various registers */
	144	+ stw r12,GPR12(r11)
	145	+ stw r9,GPR9(r11)
	146	+ mflr r10
	147	+ stw r10,_LINK(r11)
	148	+ lis r9,PAGE_OFFSET@ha
	149	+ lwz r10,crit_r10@l(r9)
	150	+ lwz r12,crit_r11@l(r9)
	151	+ stw r10,GPR10(r11)
	152	+ stw r12,GPR11(r11)
	153	+ lwz r12,crit_dear@l(r9)
	154	+ lwz r9,crit_esr@l(r9)
	155	+ stw r12,_DEAR(r11) /* since they may have had stuff */
	156	+ stw r9,_ESR(r11) /* exception was taken */
	157	+ mfspr r12,SPRN_SRR2
	158	+ mfspr r9,SPRN_SRR3
	159	+ rlwinm r9,r9,0,14,12 /* clear MSR_WE (necessary?) */
	160	+ COMMON_EXCEPTION_PROLOG_END \trapno + 2
	161	+_ASM_NOKPROBE_SYMBOL(\name\()_virt)
	162	+.endm
139	163
140	164	/*
141	165	* State at this point:
@@ -155,10 +179,10 @@ _ENTRY(saved_ksp_limit) · 13 unmodified lines
155	179	*/
156	180	#define CRITICAL_EXCEPTION(n, label, hdlr) \
157	181	START_EXCEPTION(n, label); \
158		− CRITICAL_EXCEPTION_PROLOG; \
159		− addi r3,r1,STACK_FRAME_OVERHEAD; \
160		− EXC_XFER_TEMPLATE(hdlr, n+2, (MSR_KERNEL & ~(MSR_ME\|MSR_DE\|MSR_CE)), \
161		− crit_transfer_to_handler, ret_from_crit_exc)
	182	+ CRITICAL_EXCEPTION_PROLOG n label; \
	183	+ prepare_transfer_to_handler; \
	184	+ bl hdlr; \
	185	+ b ret_from_crit_exc
162	186
163	187	/*
164	188	* 0x0100 - Critical Interrupt Exception
@@ -178,69 +202,67 @@ _ENTRY(saved_ksp_limit) · 13 unmodified lines
178	202	* if they can't resolve the lightweight TLB fault.
179	203	*/
180	204	START_EXCEPTION(0x0300, DataStorage)
181		− EXCEPTION_PROLOG
182		− mfspr r5, SPRN_ESR /* Grab the ESR, save it */
183		− stw r5, _ESR(r11)
184		− mfspr r4, SPRN_DEAR /* Grab the DEAR, save it */
185		− stw r4, _DEAR(r11)
186		− EXC_XFER_LITE(0x300, handle_page_fault)
	205	+ EXCEPTION_PROLOG 0x300 DataStorage handle_dar_dsisr=1
	206	+ prepare_transfer_to_handler
	207	+ bl do_page_fault
	208	+ b interrupt_return
187	209
188	210	/*
189	211	* 0x0400 - Instruction Storage Exception
190	212	* This is caused by a fetch from non-execute or guarded pages.
191	213	*/
192	214	START_EXCEPTION(0x0400, InstructionAccess)
193		− EXCEPTION_PROLOG
	215	+ EXCEPTION_PROLOG 0x400 InstructionAccess
194	216	li r5,0
195	217	stw r5, _ESR(r11) /* Zero ESR */
196	218	stw r12, _DEAR(r11) /* SRR0 as DEAR */
197		− EXC_XFER_LITE(0x400, handle_page_fault)
	219	+ prepare_transfer_to_handler
	220	+ bl do_page_fault
	221	+ b interrupt_return
198	222
199	223	/* 0x0500 - External Interrupt Exception */
200		− EXCEPTION(0x0500, HardwareInterrupt, do_IRQ, EXC_XFER_LITE)
	224	+ EXCEPTION(0x0500, HardwareInterrupt, do_IRQ)
201	225
202	226	/* 0x0600 - Alignment Exception */
203	227	START_EXCEPTION(0x0600, Alignment)
204		− EXCEPTION_PROLOG
205		− mfspr r4,SPRN_DEAR /* Grab the DEAR and save it */
206		− stw r4,_DEAR(r11)
207		− addi r3,r1,STACK_FRAME_OVERHEAD
208		− EXC_XFER_STD(0x600, alignment_exception)
	228	+ EXCEPTION_PROLOG 0x600 Alignment handle_dar_dsisr=1
	229	+ prepare_transfer_to_handler
	230	+ bl alignment_exception
	231	+ REST_NVGPRS(r1)
	232	+ b interrupt_return
209	233
210	234	/* 0x0700 - Program Exception */
211	235	START_EXCEPTION(0x0700, ProgramCheck)
212		− EXCEPTION_PROLOG
213		− mfspr r4,SPRN_ESR /* Grab the ESR and save it */
214		− stw r4,_ESR(r11)
215		− addi r3,r1,STACK_FRAME_OVERHEAD
216		− EXC_XFER_STD(0x700, program_check_exception)
	236	+ EXCEPTION_PROLOG 0x700 ProgramCheck handle_dar_dsisr=1
	237	+ prepare_transfer_to_handler
	238	+ bl program_check_exception
	239	+ REST_NVGPRS(r1)
	240	+ b interrupt_return
217	241
218		− EXCEPTION(0x0800, Trap_08, unknown_exception, EXC_XFER_STD)
219		− EXCEPTION(0x0900, Trap_09, unknown_exception, EXC_XFER_STD)
220		− EXCEPTION(0x0A00, Trap_0A, unknown_exception, EXC_XFER_STD)
221		− EXCEPTION(0x0B00, Trap_0B, unknown_exception, EXC_XFER_STD)
	242	+ EXCEPTION(0x0800, Trap_08, unknown_exception)
	243	+ EXCEPTION(0x0900, Trap_09, unknown_exception)
	244	+ EXCEPTION(0x0A00, Trap_0A, unknown_exception)
	245	+ EXCEPTION(0x0B00, Trap_0B, unknown_exception)
222	246
223	247	/* 0x0C00 - System Call Exception */
224	248	START_EXCEPTION(0x0C00, SystemCall)
225	249	SYSCALL_ENTRY 0xc00
226	250	/* Trap_0D is commented out to get more space for system call exception */
227	251
228		−/* EXCEPTION(0x0D00, Trap_0D, unknown_exception, EXC_XFER_STD) */
229		− EXCEPTION(0x0E00, Trap_0E, unknown_exception, EXC_XFER_STD)
230		− EXCEPTION(0x0F00, Trap_0F, unknown_exception, EXC_XFER_STD)
	252	+/* EXCEPTION(0x0D00, Trap_0D, unknown_exception) */
	253	+ EXCEPTION(0x0E00, Trap_0E, unknown_exception)
	254	+ EXCEPTION(0x0F00, Trap_0F, unknown_exception)
231	255
232	256	/* 0x1000 - Programmable Interval Timer (PIT) Exception */
233		− . = 0x1000
	257	+ START_EXCEPTION(0x1000, DecrementerTrap)
234	258	b Decrementer
235	259
236		−/* 0x1010 - Fixed Interval Timer (FIT) Exception
237		−*/
238		− . = 0x1010
	260	+/* 0x1010 - Fixed Interval Timer (FIT) Exception */
	261	+ START_EXCEPTION(0x1010, FITExceptionTrap)
239	262	b FITException
240	263
241		−/* 0x1020 - Watchdog Timer (WDT) Exception
242		−*/
243		− . = 0x1020
	264	+/* 0x1020 - Watchdog Timer (WDT) Exception */
	265	+ START_EXCEPTION(0x1020, WDTExceptionTrap)
244	266	b WDTException
245	267
246	268	/* 0x1100 - Data TLB Miss Exception
@@ -249,13 +271,13 @@ _ENTRY(saved_ksp_limit) · 2 unmodified lines
249	271	* load TLB entries from the page table if they exist.
250	272	*/
251	273	START_EXCEPTION(0x1100, DTLBMiss)
252		− mtspr SPRN_SPRG_SCRATCH0, r10 /* Save some working registers */
253		− mtspr SPRN_SPRG_SCRATCH1, r11
	274	+ mtspr SPRN_SPRG_SCRATCH5, r10 /* Save some working registers */
	275	+ mtspr SPRN_SPRG_SCRATCH6, r11
254	276	mtspr SPRN_SPRG_SCRATCH3, r12
255	277	mtspr SPRN_SPRG_SCRATCH4, r9
256	278	mfcr r12
257	279	mfspr r9, SPRN_PID
258		− mtspr SPRN_SPRG_SCRATCH5, r9
	280	+ rlwimi r12, r9, 0, 0xff
259	281	mfspr r10, SPRN_DEAR /* Get faulting address */
260	282
261	283	/* If we are faulting a kernel address, we have to use the
@@ -316,13 +338,12 @@ _ENTRY(saved_ksp_limit) · 54 unmodified lines
316	338	/* The bailout. Restore registers to pre-exception conditions
317	339	* and call the heavyweights to help us out.
318	340	*/
319		− mfspr r9, SPRN_SPRG_SCRATCH5
320		− mtspr SPRN_PID, r9
321		− mtcr r12
	341	+ mtspr SPRN_PID, r12
	342	+ mtcrf 0x80, r12
322	343	mfspr r9, SPRN_SPRG_SCRATCH4
323	344	mfspr r12, SPRN_SPRG_SCRATCH3
324		− mfspr r11, SPRN_SPRG_SCRATCH1
325		− mfspr r10, SPRN_SPRG_SCRATCH0
	345	+ mfspr r11, SPRN_SPRG_SCRATCH6
	346	+ mfspr r10, SPRN_SPRG_SCRATCH5
326	347	b DataStorage
327	348
328	349	/* 0x1200 - Instruction TLB Miss Exception
@@ -330,13 +351,13 @@ _ENTRY(saved_ksp_limit) · 1 unmodified line
330	351	* registers and bailout to a different point.
331	352	*/
332	353	START_EXCEPTION(0x1200, ITLBMiss)
333		− mtspr SPRN_SPRG_SCRATCH0, r10 /* Save some working registers */
334		− mtspr SPRN_SPRG_SCRATCH1, r11
	354	+ mtspr SPRN_SPRG_SCRATCH5, r10 /* Save some working registers */
	355	+ mtspr SPRN_SPRG_SCRATCH6, r11
335	356	mtspr SPRN_SPRG_SCRATCH3, r12
336	357	mtspr SPRN_SPRG_SCRATCH4, r9
337	358	mfcr r12
338	359	mfspr r9, SPRN_PID
339		− mtspr SPRN_SPRG_SCRATCH5, r9
	360	+ rlwimi r12, r9, 0, 0xff
340	361	mfspr r10, SPRN_SRR0 /* Get faulting address */
341	362
342	363	/* If we are faulting a kernel address, we have to use the
@@ -397,28 +418,27 @@ _ENTRY(saved_ksp_limit) · 54 unmodified lines
397	418	/* The bailout. Restore registers to pre-exception conditions
398	419	* and call the heavyweights to help us out.
399	420	*/
400		− mfspr r9, SPRN_SPRG_SCRATCH5
401		− mtspr SPRN_PID, r9
402		− mtcr r12
	421	+ mtspr SPRN_PID, r12
	422	+ mtcrf 0x80, r12
403	423	mfspr r9, SPRN_SPRG_SCRATCH4
404	424	mfspr r12, SPRN_SPRG_SCRATCH3
405		− mfspr r11, SPRN_SPRG_SCRATCH1
406		− mfspr r10, SPRN_SPRG_SCRATCH0
	425	+ mfspr r11, SPRN_SPRG_SCRATCH6
	426	+ mfspr r10, SPRN_SPRG_SCRATCH5
407	427	b InstructionAccess
408	428
409		− EXCEPTION(0x1300, Trap_13, unknown_exception, EXC_XFER_STD)
410		− EXCEPTION(0x1400, Trap_14, unknown_exception, EXC_XFER_STD)
411		− EXCEPTION(0x1500, Trap_15, unknown_exception, EXC_XFER_STD)
412		− EXCEPTION(0x1600, Trap_16, unknown_exception, EXC_XFER_STD)
413		− EXCEPTION(0x1700, Trap_17, unknown_exception, EXC_XFER_STD)
414		− EXCEPTION(0x1800, Trap_18, unknown_exception, EXC_XFER_STD)
415		− EXCEPTION(0x1900, Trap_19, unknown_exception, EXC_XFER_STD)
416		− EXCEPTION(0x1A00, Trap_1A, unknown_exception, EXC_XFER_STD)
417		− EXCEPTION(0x1B00, Trap_1B, unknown_exception, EXC_XFER_STD)
418		− EXCEPTION(0x1C00, Trap_1C, unknown_exception, EXC_XFER_STD)
419		− EXCEPTION(0x1D00, Trap_1D, unknown_exception, EXC_XFER_STD)
420		− EXCEPTION(0x1E00, Trap_1E, unknown_exception, EXC_XFER_STD)
421		− EXCEPTION(0x1F00, Trap_1F, unknown_exception, EXC_XFER_STD)
	429	+ EXCEPTION(0x1300, Trap_13, unknown_exception)
	430	+ EXCEPTION(0x1400, Trap_14, unknown_exception)
	431	+ EXCEPTION(0x1500, Trap_15, unknown_exception)
	432	+ EXCEPTION(0x1600, Trap_16, unknown_exception)
	433	+ EXCEPTION(0x1700, Trap_17, unknown_exception)
	434	+ EXCEPTION(0x1800, Trap_18, unknown_exception)
	435	+ EXCEPTION(0x1900, Trap_19, unknown_exception)
	436	+ EXCEPTION(0x1A00, Trap_1A, unknown_exception)
	437	+ EXCEPTION(0x1B00, Trap_1B, unknown_exception)
	438	+ EXCEPTION(0x1C00, Trap_1C, unknown_exception)
	439	+ EXCEPTION(0x1D00, Trap_1D, unknown_exception)
	440	+ EXCEPTION(0x1E00, Trap_1E, unknown_exception)
	441	+ EXCEPTION(0x1F00, Trap_1F, unknown_exception)
422	442
423	443	/* Check for a single step debug exception while in an exception
424	444	* handler before state has been saved. This is to catch the case
@@ -435,7 +455,7 @@ _ENTRY(saved_ksp_limit) · 10 unmodified lines
435	455	*/
436	456	/* 0x2000 - Debug Exception */
437	457	START_EXCEPTION(0x2000, DebugTrap)
438		− CRITICAL_EXCEPTION_PROLOG
	458	+ CRITICAL_EXCEPTION_PROLOG 0x2000 DebugTrap
439	459
440	460	/*
441	461	* If this is a single step or branch-taken exception in an
@@ -477,32 +497,35 @@ _ENTRY(saved_ksp_limit) · 35 unmodified lines
477	497	/* continue normal handling for a critical exception... */
478	498	2: mfspr r4,SPRN_DBSR
479	499	stw r4,_ESR(r11) /* DebugException takes DBSR in _ESR */
480		− addi r3,r1,STACK_FRAME_OVERHEAD
481		− EXC_XFER_TEMPLATE(DebugException, 0x2002, \
482		− (MSR_KERNEL & ~(MSR_ME\|MSR_DE\|MSR_CE)), \
483		− crit_transfer_to_handler, ret_from_crit_exc)
	500	+ prepare_transfer_to_handler
	501	+ bl DebugException
	502	+ b ret_from_crit_exc
484	503
485	504	/* Programmable Interval Timer (PIT) Exception. (from 0x1000) */
	505	+ __HEAD
486	506	Decrementer:
487		− EXCEPTION_PROLOG
	507	+ EXCEPTION_PROLOG 0x1000 Decrementer
488	508	lis r0,TSR_PIS@h
489	509	mtspr SPRN_TSR,r0 /* Clear the PIT exception */
490		− addi r3,r1,STACK_FRAME_OVERHEAD
491		− EXC_XFER_LITE(0x1000, timer_interrupt)
	510	+ prepare_transfer_to_handler
	511	+ bl timer_interrupt
	512	+ b interrupt_return
492	513
493	514	/* Fixed Interval Timer (FIT) Exception. (from 0x1010) */
	515	+ __HEAD
494	516	FITException:
495		− EXCEPTION_PROLOG
496		− addi r3,r1,STACK_FRAME_OVERHEAD;
497		− EXC_XFER_STD(0x1010, unknown_exception)
	517	+ EXCEPTION_PROLOG 0x1010 FITException
	518	+ prepare_transfer_to_handler
	519	+ bl unknown_exception
	520	+ b interrupt_return
498	521
499	522	/* Watchdog Timer (WDT) Exception. (from 0x1020) */
	523	+ __HEAD
500	524	WDTException:
501		− CRITICAL_EXCEPTION_PROLOG;
502		− addi r3,r1,STACK_FRAME_OVERHEAD;
503		− EXC_XFER_TEMPLATE(WatchdogException, 0x1020+2,
504		− (MSR_KERNEL & ~(MSR_ME\|MSR_DE\|MSR_CE)),
505		− crit_transfer_to_handler, ret_from_crit_exc)
	525	+ CRITICAL_EXCEPTION_PROLOG 0x1020 WDTException
	526	+ prepare_transfer_to_handler
	527	+ bl WatchdogException
	528	+ b ret_from_crit_exc
506	529
507	530	/* Other PowerPC processors, namely those derived from the 6xx-series
508	531	* have vectors from 0x2100 through 0x2F00 defined, but marked as reserved.
@@ -510,6 +533,7 @@ WDTException: · 1 unmodified line
510	533	* reserved.
511	534	*/
512	535
	536	+ __HEAD
513	537	/* Damn, I came up one instruction too many to fit into the
514	538	* exception space :-). Both the instruction and data TLB
515	539	* miss get to this point to load the TLB.
@@ -543,13 +567,12 @@ finish_tlb_load: · 27 unmodified lines
543	567
544	568	/* Done...restore registers and get out of here.
545	569	*/
546		− mfspr r9, SPRN_SPRG_SCRATCH5
547		− mtspr SPRN_PID, r9
548		− mtcr r12
	570	+ mtspr SPRN_PID, r12
	571	+ mtcrf 0x80, r12
549	572	mfspr r9, SPRN_SPRG_SCRATCH4
550	573	mfspr r12, SPRN_SPRG_SCRATCH3
551		− mfspr r11, SPRN_SPRG_SCRATCH1
552		− mfspr r10, SPRN_SPRG_SCRATCH0
	574	+ mfspr r11, SPRN_SPRG_SCRATCH6
	575	+ mfspr r10, SPRN_SPRG_SCRATCH5
553	576	rfi /* Should sync shadow TLBs */
554	577	b . /* prevent prefetch past rfi */
555	578

arch/powerpc/kernel/head_44x.S+4−6

@@ -263,8 +263,7 @@ interrupt_base: · 262 unmodified lines
263	263	INSTRUCTION_STORAGE_EXCEPTION
264	264
265	265	/* External Input Interrupt */
266		− EXCEPTION(0x0500, BOOKE_INTERRUPT_EXTERNAL, ExternalInput, \
267		− do_IRQ, EXC_XFER_LITE)
	266	+ EXCEPTION(0x0500, BOOKE_INTERRUPT_EXTERNAL, ExternalInput, do_IRQ)
268	267
269	268	/* Alignment Interrupt */
270	269	ALIGNMENT_EXCEPTION
@@ -277,7 +276,7 @@ interrupt_base: · 6 unmodified lines
277	276	FP_UNAVAILABLE_EXCEPTION
278	277	#else
279	278	EXCEPTION(0x2010, BOOKE_INTERRUPT_FP_UNAVAIL, \
280		− FloatingPointUnavailable, unknown_exception, EXC_XFER_STD)
	279	+ FloatingPointUnavailable, unknown_exception)
281	280	#endif
282	281	/* System Call Interrupt */
283	282	START_EXCEPTION(SystemCall)
@@ -285,15 +284,14 @@ interrupt_base: · 1 unmodified line
285	284
286	285	/* Auxiliary Processor Unavailable Interrupt */
287	286	EXCEPTION(0x2020, BOOKE_INTERRUPT_AP_UNAVAIL, \
288		− AuxillaryProcessorUnavailable, unknown_exception, EXC_XFER_STD)
	287	+ AuxillaryProcessorUnavailable, unknown_exception)
289	288
290	289	/* Decrementer Interrupt */
291	290	DECREMENTER_EXCEPTION
292	291
293	292	/* Fixed Internal Timer Interrupt */
294	293	/* TODO: Add FIT support */
295		− EXCEPTION(0x1010, BOOKE_INTERRUPT_FIT, FixedIntervalTimer, \
296		− unknown_exception, EXC_XFER_STD)
	294	+ EXCEPTION(0x1010, BOOKE_INTERRUPT_FIT, FixedIntervalTimer, unknown_exception)
297	295
298	296	/* Watchdog Timer Interrupt */
299	297	/* TODO: Add watchdog support */

arch/powerpc/kernel/head_8xx.S+71−85

@@ -29,6 +29,13 @@ · 28 unmodified lines
29	29	#include <asm/ptrace.h>
30	30	#include <asm/export.h>
31	31	#include <asm/code-patching-asm.h>
	32	+#include <asm/interrupt.h>
	33	+
	34	+/*
	35	+ * Value for the bits that have fixed value in RPN entries.
	36	+ * Also used for tagging DAR for DTLBerror.
	37	+ */
	38	+#define RPN_PATTERN 0x00f0
32	39
33	40	#include "head_32.h"
34	41
@@ -42,12 +49,6 @@ · 7 unmodified lines
42	49	#endif
43	50	.endm
44	51
45		−/*
46		− * Value for the bits that have fixed value in RPN entries.
47		− * Also used for tagging DAR for DTLBerror.
48		− */
49		−#define RPN_PATTERN 0x00f0
50		−
51	52	#define PAGE_SHIFT_512K 19
52	53	#define PAGE_SHIFT_8M 23
53	54
@@ -118,56 +119,54 @@ instruction_counter: · 64 unmodified lines
118	119	#endif
119	120
120	121	/* System reset */
121		− EXCEPTION(0x100, Reset, system_reset_exception, EXC_XFER_STD)
	122	+ EXCEPTION(INTERRUPT_SYSTEM_RESET, Reset, system_reset_exception)
122	123
123	124	/* Machine check */
124		− . = 0x200
125		−MachineCheck:
126		− EXCEPTION_PROLOG handle_dar_dsisr=1
127		− save_dar_dsisr_on_stack r4, r5, r11
128		− li r6, RPN_PATTERN
129		− mtspr SPRN_DAR, r6 /* Tag DAR, to be used in DTLB Error */
130		− addi r3,r1,STACK_FRAME_OVERHEAD
131		− EXC_XFER_STD(0x200, machine_check_exception)
	125	+ START_EXCEPTION(INTERRUPT_MACHINE_CHECK, MachineCheck)
	126	+ EXCEPTION_PROLOG INTERRUPT_MACHINE_CHECK MachineCheck handle_dar_dsisr=1
	127	+ prepare_transfer_to_handler
	128	+ bl machine_check_exception
	129	+ b interrupt_return
132	130
133	131	/* External interrupt */
134		− EXCEPTION(0x500, HardwareInterrupt, do_IRQ, EXC_XFER_LITE)
	132	+ EXCEPTION(INTERRUPT_EXTERNAL, HardwareInterrupt, do_IRQ)
135	133
136	134	/* Alignment exception */
137		− . = 0x600
138		−Alignment:
139		− EXCEPTION_PROLOG handle_dar_dsisr=1
140		− save_dar_dsisr_on_stack r4, r5, r11
141		− li r6, RPN_PATTERN
142		− mtspr SPRN_DAR, r6 /* Tag DAR, to be used in DTLB Error */
143		− addi r3,r1,STACK_FRAME_OVERHEAD
144		− b .Lalignment_exception_ool
	135	+ START_EXCEPTION(INTERRUPT_ALIGNMENT, Alignment)
	136	+ EXCEPTION_PROLOG INTERRUPT_ALIGNMENT Alignment handle_dar_dsisr=1
	137	+ prepare_transfer_to_handler
	138	+ bl alignment_exception
	139	+ REST_NVGPRS(r1)
	140	+ b interrupt_return
145	141
146	142	/* Program check exception */
147		− EXCEPTION(0x700, ProgramCheck, program_check_exception, EXC_XFER_STD)
	143	+ START_EXCEPTION(INTERRUPT_PROGRAM, ProgramCheck)
	144	+ EXCEPTION_PROLOG INTERRUPT_PROGRAM ProgramCheck
	145	+ prepare_transfer_to_handler
	146	+ bl program_check_exception
	147	+ REST_NVGPRS(r1)
	148	+ b interrupt_return
148	149
149	150	/* Decrementer */
150		− EXCEPTION(0x900, Decrementer, timer_interrupt, EXC_XFER_LITE)
151		−
152		− /* With VMAP_STACK there's not enough room for this at 0x600 */
153		− . = 0xa00
154		−.Lalignment_exception_ool:
155		− EXC_XFER_STD(0x600, alignment_exception)
	151	+ EXCEPTION(INTERRUPT_DECREMENTER, Decrementer, timer_interrupt)
156	152
157	153	/* System call */
158		− . = 0xc00
159		−SystemCall:
160		− SYSCALL_ENTRY 0xc00
	154	+ START_EXCEPTION(INTERRUPT_SYSCALL, SystemCall)
	155	+ SYSCALL_ENTRY INTERRUPT_SYSCALL
161	156
162	157	/* Single step - not used on 601 */
163		− EXCEPTION(0xd00, SingleStep, single_step_exception, EXC_XFER_STD)
	158	+ EXCEPTION(INTERRUPT_TRACE, SingleStep, single_step_exception)
164	159
165	160	/* On the MPC8xx, this is a software emulation interrupt. It occurs
166	161	* for all unimplemented and illegal instructions.
167	162	*/
168		− EXCEPTION(0x1000, SoftEmu, emulation_assist_interrupt, EXC_XFER_STD)
	163	+ START_EXCEPTION(INTERRUPT_SOFT_EMU_8xx, SoftEmu)
	164	+ EXCEPTION_PROLOG INTERRUPT_SOFT_EMU_8xx SoftEmu
	165	+ prepare_transfer_to_handler
	166	+ bl emulation_assist_interrupt
	167	+ REST_NVGPRS(r1)
	168	+ b interrupt_return
169	169
170		− . = 0x1100
171	170	/*
172	171	* For the MPC8xx, this is a software tablewalk to load the instruction
173	172	* TLB. The task switch loads the M_TWB register with the pointer to the first
@@ -189,7 +188,7 @@ SystemCall: · 15 unmodified lines
189	188	#define INVALIDATE_ADJACENT_PAGES_CPU15(addr, tmp)
190	189	#endif
191	190
192		−InstructionTLBMiss:
	191	+ START_EXCEPTION(INTERRUPT_INST_TLB_MISS_8xx, InstructionTLBMiss)
193	192	mtspr SPRN_SPRG_SCRATCH2, r10
194	193	mtspr SPRN_M_TW, r11
195	194
@@ -245,8 +244,7 @@ InstructionTLBMiss: · 49 unmodified lines
245	244	rfi
246	245	#endif
247	246
248		− . = 0x1200
249		−DataStoreTLBMiss:
	247	+ START_EXCEPTION(INTERRUPT_DATA_TLB_MISS_8xx, DataStoreTLBMiss)
250	248	mtspr SPRN_SPRG_SCRATCH2, r10
251	249	mtspr SPRN_M_TW, r11
252	250	mfcr r11
@@ -309,83 +307,74 @@ DataStoreTLBMiss: · 56 unmodified lines
309	307	* to many reasons, such as executing guarded memory or illegal instruction
310	308	* addresses. There is nothing to do but handle a big time error fault.
311	309	*/
312		− . = 0x1300
313		−InstructionTLBError:
314		− EXCEPTION_PROLOG
	310	+ START_EXCEPTION(INTERRUPT_INST_TLB_ERROR_8xx, InstructionTLBError)
	311	+ /* 0x400 is InstructionAccess exception, needed by bad_page_fault() */
	312	+ EXCEPTION_PROLOG INTERRUPT_INST_STORAGE InstructionTLBError
315	313	andis. r5,r9,DSISR_SRR1_MATCH_32S@h /* Filter relevant SRR1 bits */
316	314	andis. r10,r9,SRR1_ISI_NOPT@h
317	315	beq+ .Litlbie
318	316	tlbie r12
319		− /* 0x400 is InstructionAccess exception, needed by bad_page_fault() */
320	317	.Litlbie:
321	318	stw r12, _DAR(r11)
322	319	stw r5, _DSISR(r11)
323		− EXC_XFER_LITE(0x400, handle_page_fault)
	320	+ prepare_transfer_to_handler
	321	+ bl do_page_fault
	322	+ b interrupt_return
324	323
325	324	/* This is the data TLB error on the MPC8xx. This could be due to
326	325	* many reasons, including a dirty update to a pte. We bail out to
327	326	* a higher level function that can handle it.
328	327	*/
329		− . = 0x1400
330		−DataTLBError:
	328	+ START_EXCEPTION(INTERRUPT_DATA_TLB_ERROR_8xx, DataTLBError)
331	329	EXCEPTION_PROLOG_0 handle_dar_dsisr=1
332	330	mfspr r11, SPRN_DAR
333	331	cmpwi cr1, r11, RPN_PATTERN
334	332	beq- cr1, FixupDAR /* must be a buggy dcbX, icbi insn. */
335	333	DARFixed:/* Return from dcbx instruction bug workaround */
336		−#ifdef CONFIG_VMAP_STACK
337		− li r11, RPN_PATTERN
338		− mtspr SPRN_DAR, r11 /* Tag DAR, to be used in DTLB Error */
339		−#endif
340	334	EXCEPTION_PROLOG_1
341		− EXCEPTION_PROLOG_2 handle_dar_dsisr=1
342		− get_and_save_dar_dsisr_on_stack r4, r5, r11
	335	+ /* 0x300 is DataAccess exception, needed by bad_page_fault() */
	336	+ EXCEPTION_PROLOG_2 INTERRUPT_DATA_STORAGE DataTLBError handle_dar_dsisr=1
	337	+ lwz r4, _DAR(r11)
	338	+ lwz r5, _DSISR(r11)
343	339	andis. r10,r5,DSISR_NOHPTE@h
344	340	beq+ .Ldtlbie
345	341	tlbie r4
346	342	.Ldtlbie:
347		−#ifndef CONFIG_VMAP_STACK
348		− li r10,RPN_PATTERN
349		− mtspr SPRN_DAR,r10 /* Tag DAR, to be used in DTLB Error */
350		−#endif
351		− /* 0x300 is DataAccess exception, needed by bad_page_fault() */
352		− EXC_XFER_LITE(0x300, handle_page_fault)
	343	+ prepare_transfer_to_handler
	344	+ bl do_page_fault
	345	+ b interrupt_return
353	346
354		−stack_overflow:
	347	+#ifdef CONFIG_VMAP_STACK
355	348	vmap_stack_overflow_exception
	349	+#endif
356	350
357	351	/* On the MPC8xx, these next four traps are used for development
358	352	* support of breakpoints and such. Someday I will get around to
359	353	* using them.
360	354	*/
361		−do_databreakpoint:
362		− EXCEPTION_PROLOG_1
363		− EXCEPTION_PROLOG_2 handle_dar_dsisr=1
364		− addi r3,r1,STACK_FRAME_OVERHEAD
365		− mfspr r4,SPRN_BAR
366		− stw r4,_DAR(r11)
367		−#ifndef CONFIG_VMAP_STACK
368		− mfspr r5,SPRN_DSISR
369		− stw r5,_DSISR(r11)
370		−#endif
371		− EXC_XFER_STD(0x1c00, do_break)
372		−
373		− . = 0x1c00
374		−DataBreakpoint:
	355	+ START_EXCEPTION(INTERRUPT_DATA_BREAKPOINT_8xx, DataBreakpoint)
375	356	EXCEPTION_PROLOG_0 handle_dar_dsisr=1
376	357	mfspr r11, SPRN_SRR0
377	358	cmplwi cr1, r11, (.Ldtlbie - PAGE_OFFSET)@l
378	359	cmplwi cr7, r11, (.Litlbie - PAGE_OFFSET)@l
379	360	cror 4cr1+eq, 4cr1+eq, 4*cr7+eq
380		− bne cr1, do_databreakpoint
	361	+ bne cr1, 1f
381	362	mtcr r10
382	363	mfspr r10, SPRN_SPRG_SCRATCH0
383	364	mfspr r11, SPRN_SPRG_SCRATCH1
384	365	rfi
385	366
	367	+1: EXCEPTION_PROLOG_1
	368	+ EXCEPTION_PROLOG_2 INTERRUPT_DATA_BREAKPOINT_8xx DataBreakpoint handle_dar_dsisr=1
	369	+ mfspr r4,SPRN_BAR
	370	+ stw r4,_DAR(r11)
	371	+ prepare_transfer_to_handler
	372	+ bl do_break
	373	+ REST_NVGPRS(r1)
	374	+ b interrupt_return
	375	+
386	376	#ifdef CONFIG_PERF_EVENTS
387		− . = 0x1d00
388		−InstructionBreakpoint:
	377	+ START_EXCEPTION(INTERRUPT_INST_BREAKPOINT_8xx, InstructionBreakpoint)
389	378	mtspr SPRN_SPRG_SCRATCH0, r10
390	379	lwz r10, (instruction_counter - PAGE_OFFSET)@l(0)
391	380	addi r10, r10, -1
@@ -396,11 +385,12 @@ InstructionBreakpoint: · 4 unmodified lines
396	385	mfspr r10, SPRN_SPRG_SCRATCH0
397	386	rfi
398	387	#else
399		− EXCEPTION(0x1d00, Trap_1d, unknown_exception, EXC_XFER_STD)
	388	+ EXCEPTION(INTERRUPT_INST_BREAKPOINT_8xx, Trap_1d, unknown_exception)
400	389	#endif
401		− EXCEPTION(0x1e00, Trap_1e, unknown_exception, EXC_XFER_STD)
402		− EXCEPTION(0x1f00, Trap_1f, unknown_exception, EXC_XFER_STD)
	390	+ EXCEPTION(0x1e00, Trap_1e, unknown_exception)
	391	+ EXCEPTION(0x1f00, Trap_1f, unknown_exception)
403	392
	393	+ __HEAD
404	394	. = 0x2000
405	395
406	396	/* This is the procedure to calculate the data EA for buggy dcbx,dcbi instructions
@@ -510,14 +500,10 @@ FixupDAR:/* Entry point for dcbx workaround. */ · 103 unmodified lines
510	500	152:
511	501	mfdar r11
512	502	mtctr r11 /* restore ctr reg from DAR */
513		−#ifdef CONFIG_VMAP_STACK
514	503	mfspr r11, SPRN_SPRG_THREAD
515	504	stw r10, DAR(r11)
516	505	mfspr r10, SPRN_DSISR
517	506	stw r10, DSISR(r11)
518		−#else
519		− mtdar r10 /* save fault EA to DAR */
520		−#endif
521	507	mfspr r10,SPRN_M_TW
522	508	b DARFixed /* Go back to normal TLB handling */
523	509
@@ -819,7 +805,7 @@ EXPORT_SYMBOL(empty_zero_page) · 295 unmodified lines
819	805	swapper_pg_dir:
820	806	.space PGD_TABLE_SIZE
821	807
822		−/* Room for two PTE table poiners, usually the kernel and current user
	808	+/* Room for two PTE table pointers, usually the kernel and current user
823	809	* pointer to their respective root page table (pgdir).
824	810	*/
825	811	.globl abatron_pteptrs

arch/powerpc/kernel/head_book3s_32.S+99−143

@@ -31,6 +31,7 @@ · 30 unmodified lines
31	31	#include <asm/kvm_book3s_asm.h>
32	32	#include <asm/export.h>
33	33	#include <asm/feature-fixups.h>
	34	+#include <asm/interrupt.h>
34	35
35	36	#include "head_32.h"
36	37
@@ -239,7 +240,7 @@ __secondary_hold_acknowledge: · 202 unmodified lines
239	240	/* System reset */
240	241	/* core99 pmac starts the seconary here by changing the vector, and
241	242	putting it back to what it was (unknown_async_exception) when done. */
242		− EXCEPTION(0x100, Reset, unknown_async_exception, EXC_XFER_STD)
	243	+ EXCEPTION(INTERRUPT_SYSTEM_RESET, Reset, unknown_async_exception)
243	244
244	245	/* Machine check */
245	246	/*
@@ -255,40 +256,28 @@ __secondary_hold_acknowledge: · 9 unmodified lines
255	256	* pointer when we take an exception from supervisor mode.)
256	257	* -- paulus.
257	258	*/
258		− . = 0x200
259		− DO_KVM 0x200
260		−MachineCheck:
	259	+ START_EXCEPTION(INTERRUPT_MACHINE_CHECK, MachineCheck)
261	260	EXCEPTION_PROLOG_0
262	261	#ifdef CONFIG_PPC_CHRP
263		−#ifdef CONFIG_VMAP_STACK
264	262	mtspr SPRN_SPRG_SCRATCH2,r1
265	263	mfspr r1, SPRN_SPRG_THREAD
266	264	lwz r1, RTAS_SP(r1)
267	265	cmpwi cr1, r1, 0
268	266	bne cr1, 7f
269	267	mfspr r1, SPRN_SPRG_SCRATCH2
270		−#else
271		− mfspr r11, SPRN_SPRG_THREAD
272		− lwz r11, RTAS_SP(r11)
273		− cmpwi cr1, r11, 0
274		− bne cr1, 7f
275		−#endif
276	268	#endif /* CONFIG_PPC_CHRP */
277		− EXCEPTION_PROLOG_1 for_rtas=1
278		−7: EXCEPTION_PROLOG_2
279		− addi r3,r1,STACK_FRAME_OVERHEAD
	269	+ EXCEPTION_PROLOG_1
	270	+7: EXCEPTION_PROLOG_2 0x200 MachineCheck
280	271	#ifdef CONFIG_PPC_CHRP
281		− beq cr1, machine_check_tramp
	272	+ beq cr1, 1f
282	273	twi 31, 0, 0
283		−#else
284		− b machine_check_tramp
285	274	#endif
	275	+1: prepare_transfer_to_handler
	276	+ bl machine_check_exception
	277	+ b interrupt_return
286	278
287	279	/* Data access exception. */
288		− . = 0x300
289		− DO_KVM 0x300
290		−DataAccess:
291		−#ifdef CONFIG_VMAP_STACK
	280	+ START_EXCEPTION(INTERRUPT_DATA_STORAGE, DataAccess)
292	281	#ifdef CONFIG_PPC_BOOK3S_604
293	282	BEGIN_MMU_FTR_SECTION
294	283	mtspr SPRN_SPRG_SCRATCH2,r10
@@ -309,30 +298,20 @@ ALT_MMU_FTR_SECTION_END_IFSET(MMU_FTR_HPTE_TABLE) · 14 unmodified lines
309	298	#endif
310	299	1: EXCEPTION_PROLOG_0 handle_dar_dsisr=1
311	300	EXCEPTION_PROLOG_1
312		− b handle_page_fault_tramp_1
313		−#else /* CONFIG_VMAP_STACK */
314		− EXCEPTION_PROLOG handle_dar_dsisr=1
315		− get_and_save_dar_dsisr_on_stack r4, r5, r11
316		−#ifdef CONFIG_PPC_BOOK3S_604
317		−BEGIN_MMU_FTR_SECTION
318		− andis. r0, r5, (DSISR_BAD_FAULT_32S \| DSISR_DABRMATCH)@h
319		− bne handle_page_fault_tramp_2 /* if not, try to put a PTE */
320		− rlwinm r3, r5, 32 - 15, 21, 21 /* DSISR_STORE -> _PAGE_RW */
321		− bl hash_page
322		− b handle_page_fault_tramp_1
323		−MMU_FTR_SECTION_ELSE
324		−#endif
325		− b handle_page_fault_tramp_2
326		−#ifdef CONFIG_PPC_BOOK3S_604
327		−ALT_MMU_FTR_SECTION_END_IFSET(MMU_FTR_HPTE_TABLE)
328		−#endif
329		−#endif /* CONFIG_VMAP_STACK */
	301	+ EXCEPTION_PROLOG_2 INTERRUPT_DATA_STORAGE DataAccess handle_dar_dsisr=1
	302	+ prepare_transfer_to_handler
	303	+ lwz r5, _DSISR(r11)
	304	+ andis. r0, r5, DSISR_DABRMATCH@h
	305	+ bne- 1f
	306	+ bl do_page_fault
	307	+ b interrupt_return
	308	+1: bl do_break
	309	+ REST_NVGPRS(r1)
	310	+ b interrupt_return
	311	+
330	312
331	313	/* Instruction access exception. */
332		− . = 0x400
333		− DO_KVM 0x400
334		−InstructionAccess:
335		−#ifdef CONFIG_VMAP_STACK
	314	+ START_EXCEPTION(INTERRUPT_INST_STORAGE, InstructionAccess)
336	315	mtspr SPRN_SPRG_SCRATCH0,r10
337	316	mtspr SPRN_SPRG_SCRATCH1,r11
338	317	mfspr r10, SPRN_SPRG_THREAD
@@ -352,43 +331,35 @@ END_MMU_FTR_SECTION_IFSET(MMU_FTR_HPTE_TABLE) · 13 unmodified lines
352	331	andi. r11, r11, MSR_PR
353	332
354	333	EXCEPTION_PROLOG_1
355		− EXCEPTION_PROLOG_2
356		−#else /* CONFIG_VMAP_STACK */
357		− EXCEPTION_PROLOG
358		− andis. r0,r9,SRR1_ISI_NOPT@h /* no pte found? */
359		− beq 1f /* if so, try to put a PTE */
360		− li r3,0 /* into the hash table */
361		− mr r4,r12 /* SRR0 is fault address */
362		−#ifdef CONFIG_PPC_BOOK3S_604
363		−BEGIN_MMU_FTR_SECTION
364		− bl hash_page
365		−END_MMU_FTR_SECTION_IFSET(MMU_FTR_HPTE_TABLE)
366		−#endif
367		−#endif /* CONFIG_VMAP_STACK */
	334	+ EXCEPTION_PROLOG_2 INTERRUPT_INST_STORAGE InstructionAccess
368	335	andis. r5,r9,DSISR_SRR1_MATCH_32S@h /* Filter relevant SRR1 bits */
369	336	stw r5, _DSISR(r11)
370	337	stw r12, _DAR(r11)
371		− EXC_XFER_LITE(0x400, handle_page_fault)
	338	+ prepare_transfer_to_handler
	339	+ bl do_page_fault
	340	+ b interrupt_return
372	341
373	342	/* External interrupt */
374		− EXCEPTION(0x500, HardwareInterrupt, do_IRQ, EXC_XFER_LITE)
	343	+ EXCEPTION(INTERRUPT_EXTERNAL, HardwareInterrupt, do_IRQ)
375	344
376	345	/* Alignment exception */
377		− . = 0x600
378		− DO_KVM 0x600
379		−Alignment:
380		− EXCEPTION_PROLOG handle_dar_dsisr=1
381		− save_dar_dsisr_on_stack r4, r5, r11
382		− addi r3,r1,STACK_FRAME_OVERHEAD
383		− b alignment_exception_tramp
	346	+ START_EXCEPTION(INTERRUPT_ALIGNMENT, Alignment)
	347	+ EXCEPTION_PROLOG INTERRUPT_ALIGNMENT Alignment handle_dar_dsisr=1
	348	+ prepare_transfer_to_handler
	349	+ bl alignment_exception
	350	+ REST_NVGPRS(r1)
	351	+ b interrupt_return
384	352
385	353	/* Program check exception */
386		− EXCEPTION(0x700, ProgramCheck, program_check_exception, EXC_XFER_STD)
	354	+ START_EXCEPTION(INTERRUPT_PROGRAM, ProgramCheck)
	355	+ EXCEPTION_PROLOG INTERRUPT_PROGRAM ProgramCheck
	356	+ prepare_transfer_to_handler
	357	+ bl program_check_exception
	358	+ REST_NVGPRS(r1)
	359	+ b interrupt_return
387	360
388	361	/* Floating-point unavailable */
389		− . = 0x800
390		− DO_KVM 0x800
391		−FPUnavailable:
	362	+ START_EXCEPTION(0x800, FPUnavailable)
392	363	#ifdef CONFIG_PPC_FPU
393	364	BEGIN_FTR_SECTION
394	365	/*
@@ -397,30 +368,29 @@ BEGIN_FTR_SECTION · 2 unmodified lines
397	368	*/
398	369	b ProgramCheck
399	370	END_FTR_SECTION_IFSET(CPU_FTR_FPU_UNAVAILABLE)
400		− EXCEPTION_PROLOG
	371	+ EXCEPTION_PROLOG INTERRUPT_FP_UNAVAIL FPUnavailable
401	372	beq 1f
402	373	bl load_up_fpu /* if from user, just load it up */
403	374	b fast_exception_return
404		−1: addi r3,r1,STACK_FRAME_OVERHEAD
405		− EXC_XFER_LITE(0x800, kernel_fp_unavailable_exception)
	375	+1: prepare_transfer_to_handler
	376	+ bl kernel_fp_unavailable_exception
	377	+ b interrupt_return
406	378	#else
407	379	b ProgramCheck
408	380	#endif
409	381
410	382	/* Decrementer */
411		− EXCEPTION(0x900, Decrementer, timer_interrupt, EXC_XFER_LITE)
	383	+ EXCEPTION(INTERRUPT_DECREMENTER, Decrementer, timer_interrupt)
412	384
413		− EXCEPTION(0xa00, Trap_0a, unknown_exception, EXC_XFER_STD)
414		− EXCEPTION(0xb00, Trap_0b, unknown_exception, EXC_XFER_STD)
	385	+ EXCEPTION(0xa00, Trap_0a, unknown_exception)
	386	+ EXCEPTION(0xb00, Trap_0b, unknown_exception)
415	387
416	388	/* System call */
417		− . = 0xc00
418		− DO_KVM 0xc00
419		−SystemCall:
420		− SYSCALL_ENTRY 0xc00
	389	+ START_EXCEPTION(INTERRUPT_SYSCALL, SystemCall)
	390	+ SYSCALL_ENTRY INTERRUPT_SYSCALL
421	391
422		− EXCEPTION(0xd00, SingleStep, single_step_exception, EXC_XFER_STD)
423		− EXCEPTION(0xe00, Trap_0e, unknown_exception, EXC_XFER_STD)
	392	+ EXCEPTION(INTERRUPT_TRACE, SingleStep, single_step_exception)
	393	+ EXCEPTION(0xe00, Trap_0e, unknown_exception)
424	394
425	395	/*
426	396	* The Altivec unavailable trap is at 0x0f20. Foo.
@@ -430,19 +400,18 @@ SystemCall: · 3 unmodified lines
430	400	* non-altivec kernel running on a machine with altivec just
431	401	* by executing an altivec instruction.
432	402	*/
433		− . = 0xf00
434		− DO_KVM 0xf00
	403	+ START_EXCEPTION(INTERRUPT_PERFMON, PerformanceMonitorTrap)
435	404	b PerformanceMonitor
436	405
437		− . = 0xf20
438		− DO_KVM 0xf20
	406	+ START_EXCEPTION(INTERRUPT_ALTIVEC_UNAVAIL, AltiVecUnavailableTrap)
439	407	b AltiVecUnavailable
440	408
	409	+ __HEAD
441	410	/*
442	411	* Handle TLB miss for instruction on 603/603e.
443	412	* Note: we get an alternate set of r0 - r3 to use automatically.
444	413	*/
445		− . = 0x1000
	414	+ . = INTERRUPT_INST_TLB_MISS_603
446	415	InstructionTLBMiss:
447	416	/*
448	417	* r0: scratch
@@ -508,7 +477,7 @@ InstructionAddressInvalid: · 59 unmodified lines
508	477	/*
509	478	* Handle TLB miss for DATA Load operation on 603/603e
510	479	*/
511		− . = 0x1100
	480	+ . = INTERRUPT_DATA_LOAD_TLB_MISS_603
512	481	DataLoadTLBMiss:
513	482	/*
514	483	* r0: scratch
@@ -586,7 +555,7 @@ DataAddressInvalid: · 71 unmodified lines
586	555	/*
587	556	* Handle TLB miss for DATA Store on 603/603e
588	557	*/
589		− . = 0x1200
	558	+ . = INTERRUPT_DATA_STORE_TLB_MISS_603
590	559	DataStoreTLBMiss:
591	560	/*
592	561	* r0: scratch
@@ -650,57 +619,39 @@ END_MMU_FTR_SECTION_IFSET(MMU_FTR_NEED_DTLB_SW_LRU) · 57 unmodified lines
650	619	#define TAUException unknown_async_exception
651	620	#endif
652	621
653		− EXCEPTION(0x1300, Trap_13, instruction_breakpoint_exception, EXC_XFER_STD)
654		− EXCEPTION(0x1400, SMI, SMIException, EXC_XFER_STD)
655		− EXCEPTION(0x1500, Trap_15, unknown_exception, EXC_XFER_STD)
656		− EXCEPTION(0x1600, Trap_16, altivec_assist_exception, EXC_XFER_STD)
657		− EXCEPTION(0x1700, Trap_17, TAUException, EXC_XFER_STD)
658		− EXCEPTION(0x1800, Trap_18, unknown_exception, EXC_XFER_STD)
659		− EXCEPTION(0x1900, Trap_19, unknown_exception, EXC_XFER_STD)
660		− EXCEPTION(0x1a00, Trap_1a, unknown_exception, EXC_XFER_STD)
661		− EXCEPTION(0x1b00, Trap_1b, unknown_exception, EXC_XFER_STD)
662		− EXCEPTION(0x1c00, Trap_1c, unknown_exception, EXC_XFER_STD)
663		− EXCEPTION(0x1d00, Trap_1d, unknown_exception, EXC_XFER_STD)
664		− EXCEPTION(0x1e00, Trap_1e, unknown_exception, EXC_XFER_STD)
665		− EXCEPTION(0x1f00, Trap_1f, unknown_exception, EXC_XFER_STD)
666		− EXCEPTION(0x2000, RunMode, RunModeException, EXC_XFER_STD)
667		− EXCEPTION(0x2100, Trap_21, unknown_exception, EXC_XFER_STD)
668		− EXCEPTION(0x2200, Trap_22, unknown_exception, EXC_XFER_STD)
669		− EXCEPTION(0x2300, Trap_23, unknown_exception, EXC_XFER_STD)
670		− EXCEPTION(0x2400, Trap_24, unknown_exception, EXC_XFER_STD)
671		− EXCEPTION(0x2500, Trap_25, unknown_exception, EXC_XFER_STD)
672		− EXCEPTION(0x2600, Trap_26, unknown_exception, EXC_XFER_STD)
673		− EXCEPTION(0x2700, Trap_27, unknown_exception, EXC_XFER_STD)
674		− EXCEPTION(0x2800, Trap_28, unknown_exception, EXC_XFER_STD)
675		− EXCEPTION(0x2900, Trap_29, unknown_exception, EXC_XFER_STD)
676		− EXCEPTION(0x2a00, Trap_2a, unknown_exception, EXC_XFER_STD)
677		− EXCEPTION(0x2b00, Trap_2b, unknown_exception, EXC_XFER_STD)
678		− EXCEPTION(0x2c00, Trap_2c, unknown_exception, EXC_XFER_STD)
679		− EXCEPTION(0x2d00, Trap_2d, unknown_exception, EXC_XFER_STD)
680		− EXCEPTION(0x2e00, Trap_2e, unknown_exception, EXC_XFER_STD)
681		− EXCEPTION(0x2f00, Trap_2f, unknown_exception, EXC_XFER_STD)
	622	+ EXCEPTION(0x1300, Trap_13, instruction_breakpoint_exception)
	623	+ EXCEPTION(0x1400, SMI, SMIException)
	624	+ EXCEPTION(0x1500, Trap_15, unknown_exception)
	625	+ EXCEPTION(0x1600, Trap_16, altivec_assist_exception)
	626	+ EXCEPTION(0x1700, Trap_17, TAUException)
	627	+ EXCEPTION(0x1800, Trap_18, unknown_exception)
	628	+ EXCEPTION(0x1900, Trap_19, unknown_exception)
	629	+ EXCEPTION(0x1a00, Trap_1a, unknown_exception)
	630	+ EXCEPTION(0x1b00, Trap_1b, unknown_exception)
	631	+ EXCEPTION(0x1c00, Trap_1c, unknown_exception)
	632	+ EXCEPTION(0x1d00, Trap_1d, unknown_exception)
	633	+ EXCEPTION(0x1e00, Trap_1e, unknown_exception)
	634	+ EXCEPTION(0x1f00, Trap_1f, unknown_exception)
	635	+ EXCEPTION(0x2000, RunMode, RunModeException)
	636	+ EXCEPTION(0x2100, Trap_21, unknown_exception)
	637	+ EXCEPTION(0x2200, Trap_22, unknown_exception)
	638	+ EXCEPTION(0x2300, Trap_23, unknown_exception)
	639	+ EXCEPTION(0x2400, Trap_24, unknown_exception)
	640	+ EXCEPTION(0x2500, Trap_25, unknown_exception)
	641	+ EXCEPTION(0x2600, Trap_26, unknown_exception)
	642	+ EXCEPTION(0x2700, Trap_27, unknown_exception)
	643	+ EXCEPTION(0x2800, Trap_28, unknown_exception)
	644	+ EXCEPTION(0x2900, Trap_29, unknown_exception)
	645	+ EXCEPTION(0x2a00, Trap_2a, unknown_exception)
	646	+ EXCEPTION(0x2b00, Trap_2b, unknown_exception)
	647	+ EXCEPTION(0x2c00, Trap_2c, unknown_exception)
	648	+ EXCEPTION(0x2d00, Trap_2d, unknown_exception)
	649	+ EXCEPTION(0x2e00, Trap_2e, unknown_exception)
	650	+ EXCEPTION(0x2f00, Trap_2f, unknown_exception)
682	651
	652	+ __HEAD
683	653	. = 0x3000
684	654
685		−machine_check_tramp:
686		− EXC_XFER_STD(0x200, machine_check_exception)
687		−
688		−alignment_exception_tramp:
689		− EXC_XFER_STD(0x600, alignment_exception)
690		−
691		−handle_page_fault_tramp_1:
692		−#ifdef CONFIG_VMAP_STACK
693		− EXCEPTION_PROLOG_2 handle_dar_dsisr=1
694		−#endif
695		− lwz r5, _DSISR(r11)
696		− /* fall through */
697		−handle_page_fault_tramp_2:
698		− andis. r0, r5, DSISR_DABRMATCH@h
699		− bne- 1f
700		− EXC_XFER_LITE(0x300, handle_page_fault)
701		−1: EXC_XFER_STD(0x300, do_break)
702		−
703		−#ifdef CONFIG_VMAP_STACK
704	655	#ifdef CONFIG_PPC_BOOK3S_604
705	656	.macro save_regs_thread thread
706	657	stw r0, THR0(\thread)
@@ -775,26 +726,31 @@ fast_hash_page_return: · 68 unmodified lines
775	726	rfi
776	727	#endif /* CONFIG_PPC_BOOK3S_604 */
777	728
778		−stack_overflow:
	729	+#ifdef CONFIG_VMAP_STACK
779	730	vmap_stack_overflow_exception
780	731	#endif
781	732
	733	+ __HEAD
782	734	AltiVecUnavailable:
783		− EXCEPTION_PROLOG
	735	+ EXCEPTION_PROLOG 0xf20 AltiVecUnavailable
784	736	#ifdef CONFIG_ALTIVEC
785	737	beq 1f
786	738	bl load_up_altivec /* if from user, just load it up */
787	739	b fast_exception_return
788	740	#endif /* CONFIG_ALTIVEC */
789		−1: addi r3,r1,STACK_FRAME_OVERHEAD
790		− EXC_XFER_LITE(0xf20, altivec_unavailable_exception)
	741	+1: prepare_transfer_to_handler
	742	+ bl altivec_unavailable_exception
	743	+ b interrupt_return
791	744
	745	+ __HEAD
792	746	PerformanceMonitor:
793		− EXCEPTION_PROLOG
794		− addi r3,r1,STACK_FRAME_OVERHEAD
795		− EXC_XFER_STD(0xf00, performance_monitor_exception)
	747	+ EXCEPTION_PROLOG 0xf00 PerformanceMonitor
	748	+ prepare_transfer_to_handler
	749	+ bl performance_monitor_exception
	750	+ b interrupt_return
796	751
797	752
	753	+ __HEAD
798	754	/*
799	755	* This code is jumped to from the startup code to copy
800	756	* the kernel image to physical address PHYSICAL_START.

arch/powerpc/kernel/head_booke.h+134−69

@@ -44,7 +44,7 @@ END_BTB_FLUSH_SECTION · 43 unmodified lines
44	44	#endif
45	45
46	46
47		−#define NORMAL_EXCEPTION_PROLOG(intno) \
	47	+#define NORMAL_EXCEPTION_PROLOG(trapno, intno) \
48	48	mtspr SPRN_SPRG_WSCRATCH0, r10; /* save one register */ \
49	49	mfspr r10, SPRN_SPRG_THREAD; \
50	50	stw r11, THREAD_NORMSAVE(0)(r10); \
@@ -53,6 +53,8 @@ END_BTB_FLUSH_SECTION · 2 unmodified lines
53	53	mfspr r11, SPRN_SRR1; \
54	54	DO_KVM BOOKE_INTERRUPT_##intno SPRN_SRR1; \
55	55	andi. r11, r11, MSR_PR; /* check whether user or kernel */\
	56	+ LOAD_REG_IMMEDIATE(r11, MSR_KERNEL); \
	57	+ mtmsr r11; \
56	58	mr r11, r1; \
57	59	beq 1f; \
58	60	BOOKE_CLEAR_BTB(r11) \
@@ -76,12 +78,39 @@ END_BTB_FLUSH_SECTION · 17 unmodified lines
76	78	stw r1, 0(r11); \
77	79	mr r1, r11; \
78	80	rlwinm r9,r9,0,14,12; /* clear MSR_WE (necessary?) */\
79		− stw r0,GPR0(r11); \
80		− lis r10, STACK_FRAME_REGS_MARKER@ha;/* exception frame marker */ \
81		− addi r10, r10, STACK_FRAME_REGS_MARKER@l; \
82		− stw r10, 8(r11); \
83		− SAVE_4GPRS(3, r11); \
84		− SAVE_2GPRS(7, r11)
	81	+ COMMON_EXCEPTION_PROLOG_END trapno
	82	+
	83	+.macro COMMON_EXCEPTION_PROLOG_END trapno
	84	+ stw r0,GPR0(r1)
	85	+ lis r10, STACK_FRAME_REGS_MARKER@ha /* exception frame marker */
	86	+ addi r10, r10, STACK_FRAME_REGS_MARKER@l
	87	+ stw r10, 8(r1)
	88	+ li r10, \trapno
	89	+ stw r10,_TRAP(r1)
	90	+ SAVE_4GPRS(3, r1)
	91	+ SAVE_2GPRS(7, r1)
	92	+ SAVE_NVGPRS(r1)
	93	+ stw r2,GPR2(r1)
	94	+ stw r12,_NIP(r1)
	95	+ stw r9,_MSR(r1)
	96	+ mfctr r10
	97	+ mfspr r2,SPRN_SPRG_THREAD
	98	+ stw r10,_CTR(r1)
	99	+ tovirt(r2, r2)
	100	+ mfspr r10,SPRN_XER
	101	+ addi r2, r2, -THREAD
	102	+ stw r10,_XER(r1)
	103	+ addi r3,r1,STACK_FRAME_OVERHEAD
	104	+.endm
	105	+
	106	+.macro prepare_transfer_to_handler
	107	+#ifdef CONFIG_E500
	108	+ andi. r12,r9,MSR_PR
	109	+ bne 777f
	110	+ bl prepare_transfer_to_handler
	111	+777:
	112	+#endif
	113	+.endm
85	114
86	115	.macro SYSCALL_ENTRY trapno intno srr1
87	116	mfspr r10, SPRN_SPRG_THREAD
@@ -180,7 +209,7 @@ ALT_FTR_SECTION_END_IFSET(CPU_FTR_EMB_HV) · 92 unmodified lines
180	209	* registers as the normal prolog above. Instead we use a portion of the
181	210	* critical/machine check exception stack at low physical addresses.
182	211	*/
183		−#define EXC_LEVEL_EXCEPTION_PROLOG(exc_level, intno, exc_level_srr0, exc_level_srr1) \
	212	+#define EXC_LEVEL_EXCEPTION_PROLOG(exc_level, trapno, intno, exc_level_srr0, exc_level_srr1) \
184	213	mtspr SPRN_SPRG_WSCRATCH_##exc_level,r8; \
185	214	BOOKE_LOAD_EXC_LEVEL_STACK(exc_level);/* r8 points to the exc_level stack*/ \
186	215	stw r9,GPR9(r8); /* save various registers */\
@@ -192,6 +221,8 @@ ALT_FTR_SECTION_END_IFSET(CPU_FTR_EMB_HV) · 5 unmodified lines
192	221	DO_KVM BOOKE_INTERRUPT_##intno exc_level_srr1; \
193	222	BOOKE_CLEAR_BTB(r10) \
194	223	andi. r11,r11,MSR_PR; \
	224	+ LOAD_REG_IMMEDIATE(r11, MSR_KERNEL & ~(MSR_ME\|MSR_DE\|MSR_CE)); \
	225	+ mtmsr r11; \
195	226	mfspr r11,SPRN_SPRG_THREAD; /* if from user, start at top of */\
196	227	lwz r11, TASK_STACK - THREAD(r11); /* this thread's kernel stack */\
197	228	addi r11,r11,EXC_LVL_FRAME_OVERHEAD; /* allocate stack frame */\
@@ -221,16 +252,44 @@ ALT_FTR_SECTION_END_IFSET(CPU_FTR_EMB_HV) · 23 unmodified lines
221	252	stw r1,0(r11); \
222	253	mr r1,r11; \
223	254	rlwinm r9,r9,0,14,12; /* clear MSR_WE (necessary?) */\
224		− stw r0,GPR0(r11); \
225		− SAVE_4GPRS(3, r11); \
226		− SAVE_2GPRS(7, r11)
	255	+ COMMON_EXCEPTION_PROLOG_END trapno
	256	+
	257	+#define SAVE_xSRR(xSRR) \
	258	+ mfspr r0,SPRN_##xSRR##0; \
	259	+ stw r0,_##xSRR##0(r1); \
	260	+ mfspr r0,SPRN_##xSRR##1; \
	261	+ stw r0,_##xSRR##1(r1)
	262	+
	263	+
	264	+.macro SAVE_MMU_REGS
	265	+#ifdef CONFIG_PPC_BOOK3E_MMU
	266	+ mfspr r0,SPRN_MAS0
	267	+ stw r0,MAS0(r1)
	268	+ mfspr r0,SPRN_MAS1
	269	+ stw r0,MAS1(r1)
	270	+ mfspr r0,SPRN_MAS2
	271	+ stw r0,MAS2(r1)
	272	+ mfspr r0,SPRN_MAS3
	273	+ stw r0,MAS3(r1)
	274	+ mfspr r0,SPRN_MAS6
	275	+ stw r0,MAS6(r1)
	276	+#ifdef CONFIG_PHYS_64BIT
	277	+ mfspr r0,SPRN_MAS7
	278	+ stw r0,MAS7(r1)
	279	+#endif /* CONFIG_PHYS_64BIT */
	280	+#endif /* CONFIG_PPC_BOOK3E_MMU */
	281	+#ifdef CONFIG_44x
	282	+ mfspr r0,SPRN_MMUCR
	283	+ stw r0,MMUCR(r1)
	284	+#endif
	285	+.endm
227	286
228		−#define CRITICAL_EXCEPTION_PROLOG(intno) \
229		− EXC_LEVEL_EXCEPTION_PROLOG(CRIT, intno, SPRN_CSRR0, SPRN_CSRR1)
230		−#define DEBUG_EXCEPTION_PROLOG \
231		− EXC_LEVEL_EXCEPTION_PROLOG(DBG, DEBUG, SPRN_DSRR0, SPRN_DSRR1)
232		−#define MCHECK_EXCEPTION_PROLOG \
233		− EXC_LEVEL_EXCEPTION_PROLOG(MC, MACHINE_CHECK, \
	287	+#define CRITICAL_EXCEPTION_PROLOG(trapno, intno) \
	288	+ EXC_LEVEL_EXCEPTION_PROLOG(CRIT, trapno+2, intno, SPRN_CSRR0, SPRN_CSRR1)
	289	+#define DEBUG_EXCEPTION_PROLOG(trapno) \
	290	+ EXC_LEVEL_EXCEPTION_PROLOG(DBG, trapno+8, DEBUG, SPRN_DSRR0, SPRN_DSRR1)
	291	+#define MCHECK_EXCEPTION_PROLOG(trapno) \
	292	+ EXC_LEVEL_EXCEPTION_PROLOG(MC, trapno+4, MACHINE_CHECK, \
234	293	SPRN_MCSRR0, SPRN_MCSRR1)
235	294
236	295	/*
@@ -257,44 +316,34 @@ ALT_FTR_SECTION_END_IFSET(CPU_FTR_EMB_HV) · 20 unmodified lines
257	316	.align 5; \
258	317	label:
259	318
260		−#define EXCEPTION(n, intno, label, hdlr, xfer) \
	319	+#define EXCEPTION(n, intno, label, hdlr) \
261	320	START_EXCEPTION(label); \
262		− NORMAL_EXCEPTION_PROLOG(intno); \
263		− addi r3,r1,STACK_FRAME_OVERHEAD; \
264		− xfer(n, hdlr)
	321	+ NORMAL_EXCEPTION_PROLOG(n, intno); \
	322	+ prepare_transfer_to_handler; \
	323	+ bl hdlr; \
	324	+ b interrupt_return
265	325
266	326	#define CRITICAL_EXCEPTION(n, intno, label, hdlr) \
267	327	START_EXCEPTION(label); \
268		− CRITICAL_EXCEPTION_PROLOG(intno); \
269		− addi r3,r1,STACK_FRAME_OVERHEAD; \
270		− EXC_XFER_TEMPLATE(hdlr, n+2, (MSR_KERNEL & ~(MSR_ME\|MSR_DE\|MSR_CE)), \
271		− crit_transfer_to_handler, ret_from_crit_exc)
	328	+ CRITICAL_EXCEPTION_PROLOG(n, intno); \
	329	+ SAVE_MMU_REGS; \
	330	+ SAVE_xSRR(SRR); \
	331	+ prepare_transfer_to_handler; \
	332	+ bl hdlr; \
	333	+ b ret_from_crit_exc
272	334
273	335	#define MCHECK_EXCEPTION(n, label, hdlr) \
274	336	START_EXCEPTION(label); \
275		− MCHECK_EXCEPTION_PROLOG; \
	337	+ MCHECK_EXCEPTION_PROLOG(n); \
276	338	mfspr r5,SPRN_ESR; \
277	339	stw r5,_ESR(r11); \
278		− addi r3,r1,STACK_FRAME_OVERHEAD; \
279		− EXC_XFER_TEMPLATE(hdlr, n+4, (MSR_KERNEL & ~(MSR_ME\|MSR_DE\|MSR_CE)), \
280		− mcheck_transfer_to_handler, ret_from_mcheck_exc)
281		−
282		−#define EXC_XFER_TEMPLATE(hdlr, trap, msr, tfer, ret) \
283		− li r10,trap; \
284		− stw r10,_TRAP(r11); \
285		− lis r10,msr@h; \
286		− ori r10,r10,msr@l; \
287		− bl tfer; \
288		− .long hdlr; \
289		− .long ret
290		−
291		−#define EXC_XFER_STD(n, hdlr) \
292		− EXC_XFER_TEMPLATE(hdlr, n, MSR_KERNEL, transfer_to_handler_full, \
293		− ret_from_except_full)
294		−
295		−#define EXC_XFER_LITE(n, hdlr) \
296		− EXC_XFER_TEMPLATE(hdlr, n+1, MSR_KERNEL, transfer_to_handler, \
297		− ret_from_except)
	340	+ SAVE_xSRR(DSRR); \
	341	+ SAVE_xSRR(CSRR); \
	342	+ SAVE_MMU_REGS; \
	343	+ SAVE_xSRR(SRR); \
	344	+ prepare_transfer_to_handler; \
	345	+ bl hdlr; \
	346	+ b ret_from_mcheck_exc
298	347
299	348	/* Check for a single step debug exception while in an exception
300	349	* handler before state has been saved. This is to catch the case
@@ -311,7 +360,7 @@ label: · 10 unmodified lines
311	360	*/
312	361	#define DEBUG_DEBUG_EXCEPTION \
313	362	START_EXCEPTION(DebugDebug); \
314		− DEBUG_EXCEPTION_PROLOG; \
	363	+ DEBUG_EXCEPTION_PROLOG(2000); \
315	364	\
316	365	/* \
317	366	* If there is a single step or branch-taken exception in an \
@@ -360,12 +409,16 @@ label: · 42 unmodified lines
360	409	/* continue normal handling for a debug exception... */ \
361	410	2: mfspr r4,SPRN_DBSR; \
362	411	stw r4,_ESR(r11); /* DebugException takes DBSR in _ESR */\
363		− addi r3,r1,STACK_FRAME_OVERHEAD; \
364		− EXC_XFER_TEMPLATE(DebugException, 0x2008, (MSR_KERNEL & ~(MSR_ME\|MSR_DE\|MSR_CE)), debug_transfer_to_handler, ret_from_debug_exc)
	412	+ SAVE_xSRR(CSRR); \
	413	+ SAVE_MMU_REGS; \
	414	+ SAVE_xSRR(SRR); \
	415	+ prepare_transfer_to_handler; \
	416	+ bl DebugException; \
	417	+ b ret_from_debug_exc
365	418
366	419	#define DEBUG_CRIT_EXCEPTION \
367	420	START_EXCEPTION(DebugCrit); \
368		− CRITICAL_EXCEPTION_PROLOG(DEBUG); \
	421	+ CRITICAL_EXCEPTION_PROLOG(2000,DEBUG); \
369	422	\
370	423	/* \
371	424	* If there is a single step or branch-taken exception in an \
@@ -414,58 +467,71 @@ label: · 42 unmodified lines
414	467	/* continue normal handling for a critical exception... */ \
415	468	2: mfspr r4,SPRN_DBSR; \
416	469	stw r4,_ESR(r11); /* DebugException takes DBSR in _ESR */\
417		− addi r3,r1,STACK_FRAME_OVERHEAD; \
418		− EXC_XFER_TEMPLATE(DebugException, 0x2002, (MSR_KERNEL & ~(MSR_ME\|MSR_DE\|MSR_CE)), crit_transfer_to_handler, ret_from_crit_exc)
	470	+ SAVE_MMU_REGS; \
	471	+ SAVE_xSRR(SRR); \
	472	+ prepare_transfer_to_handler; \
	473	+ bl DebugException; \
	474	+ b ret_from_crit_exc
419	475
420	476	#define DATA_STORAGE_EXCEPTION \
421	477	START_EXCEPTION(DataStorage) \
422		− NORMAL_EXCEPTION_PROLOG(DATA_STORAGE); \
	478	+ NORMAL_EXCEPTION_PROLOG(0x300, DATA_STORAGE); \
423	479	mfspr r5,SPRN_ESR; /* Grab the ESR and save it */ \
424	480	stw r5,_ESR(r11); \
425	481	mfspr r4,SPRN_DEAR; /* Grab the DEAR */ \
426	482	stw r4, _DEAR(r11); \
427		− EXC_XFER_LITE(0x0300, handle_page_fault)
	483	+ prepare_transfer_to_handler; \
	484	+ bl do_page_fault; \
	485	+ b interrupt_return
428	486
429	487	#define INSTRUCTION_STORAGE_EXCEPTION \
430	488	START_EXCEPTION(InstructionStorage) \
431		− NORMAL_EXCEPTION_PROLOG(INST_STORAGE); \
	489	+ NORMAL_EXCEPTION_PROLOG(0x400, INST_STORAGE); \
432	490	mfspr r5,SPRN_ESR; /* Grab the ESR and save it */ \
433	491	stw r5,_ESR(r11); \
434	492	stw r12, _DEAR(r11); /* Pass SRR0 as arg2 */ \
435		− EXC_XFER_LITE(0x0400, handle_page_fault)
	493	+ prepare_transfer_to_handler; \
	494	+ bl do_page_fault; \
	495	+ b interrupt_return
436	496
437	497	#define ALIGNMENT_EXCEPTION \
438	498	START_EXCEPTION(Alignment) \
439		− NORMAL_EXCEPTION_PROLOG(ALIGNMENT); \
	499	+ NORMAL_EXCEPTION_PROLOG(0x600, ALIGNMENT); \
440	500	mfspr r4,SPRN_DEAR; /* Grab the DEAR and save it */ \
441	501	stw r4,_DEAR(r11); \
442		− addi r3,r1,STACK_FRAME_OVERHEAD; \
443		− EXC_XFER_STD(0x0600, alignment_exception)
	502	+ prepare_transfer_to_handler; \
	503	+ bl alignment_exception; \
	504	+ REST_NVGPRS(r1); \
	505	+ b interrupt_return
444	506
445	507	#define PROGRAM_EXCEPTION \
446	508	START_EXCEPTION(Program) \
447		− NORMAL_EXCEPTION_PROLOG(PROGRAM); \
	509	+ NORMAL_EXCEPTION_PROLOG(0x700, PROGRAM); \
448	510	mfspr r4,SPRN_ESR; /* Grab the ESR and save it */ \
449	511	stw r4,_ESR(r11); \
450		− addi r3,r1,STACK_FRAME_OVERHEAD; \
451		− EXC_XFER_STD(0x0700, program_check_exception)
	512	+ prepare_transfer_to_handler; \
	513	+ bl program_check_exception; \
	514	+ REST_NVGPRS(r1); \
	515	+ b interrupt_return
452	516
453	517	#define DECREMENTER_EXCEPTION \
454	518	START_EXCEPTION(Decrementer) \
455		− NORMAL_EXCEPTION_PROLOG(DECREMENTER); \
	519	+ NORMAL_EXCEPTION_PROLOG(0x900, DECREMENTER); \
456	520	lis r0,TSR_DIS@h; /* Setup the DEC interrupt mask */ \
457	521	mtspr SPRN_TSR,r0; /* Clear the DEC interrupt */ \
458		− addi r3,r1,STACK_FRAME_OVERHEAD; \
459		− EXC_XFER_LITE(0x0900, timer_interrupt)
	522	+ prepare_transfer_to_handler; \
	523	+ bl timer_interrupt; \
	524	+ b interrupt_return
460	525
461	526	#define FP_UNAVAILABLE_EXCEPTION \
462	527	START_EXCEPTION(FloatingPointUnavailable) \
463		− NORMAL_EXCEPTION_PROLOG(FP_UNAVAIL); \
	528	+ NORMAL_EXCEPTION_PROLOG(0x800, FP_UNAVAIL); \
464	529	beq 1f; \
465	530	bl load_up_fpu; /* if from user, just load it up */ \
466	531	b fast_exception_return; \
467		−1: addi r3,r1,STACK_FRAME_OVERHEAD; \
468		− EXC_XFER_STD(0x800, kernel_fp_unavailable_exception)
	532	+1: prepare_transfer_to_handler; \
	533	+ bl kernel_fp_unavailable_exception; \
	534	+ b interrupt_return
469	535
470	536	#else /* __ASSEMBLY__ */
471	537	struct exception_regs {
@@ -481,7 +547,6 @@ struct exception_regs { · 9 unmodified lines
481	547	unsigned long csrr1;
482	548	unsigned long dsrr0;
483	549	unsigned long dsrr1;
484		− unsigned long saved_ksp_limit;
485	550	};
486	551
487	552	/* ensure this structure is always sized to a multiple of the stack alignment */

arch/powerpc/kernel/head_fsl_booke.S+38−32

@@ -113,7 +113,7 @@ _ENTRY(_start); · 112 unmodified lines
113	113
114	114	1:
115	115	/*
116		− * We have the runtime (virutal) address of our base.
	116	+ * We have the runtime (virtual) address of our base.
117	117	* We calculate our shift of offset from a 64M page.
118	118	* We could map the 64M page we belong to at PAGE_OFFSET and
119	119	* get going from there.
@@ -363,23 +363,26 @@ interrupt_base: · 243 unmodified lines
363	363
364	364	/* Data Storage Interrupt */
365	365	START_EXCEPTION(DataStorage)
366		− NORMAL_EXCEPTION_PROLOG(DATA_STORAGE)
	366	+ NORMAL_EXCEPTION_PROLOG(0x300, DATA_STORAGE)
367	367	mfspr r5,SPRN_ESR /* Grab the ESR, save it */
368	368	stw r5,_ESR(r11)
369	369	mfspr r4,SPRN_DEAR /* Grab the DEAR, save it */
370	370	stw r4, _DEAR(r11)
371	371	andis. r10,r5,(ESR_ILK\|ESR_DLK)@h
372	372	bne 1f
373		− EXC_XFER_LITE(0x0300, handle_page_fault)
	373	+ prepare_transfer_to_handler
	374	+ bl do_page_fault
	375	+ b interrupt_return
374	376	1:
375		− addi r3,r1,STACK_FRAME_OVERHEAD
376		− EXC_XFER_LITE(0x0300, CacheLockingException)
	377	+ prepare_transfer_to_handler
	378	+ bl CacheLockingException
	379	+ b interrupt_return
377	380
378	381	/* Instruction Storage Interrupt */
379	382	INSTRUCTION_STORAGE_EXCEPTION
380	383
381	384	/* External Input Interrupt */
382		− EXCEPTION(0x0500, EXTERNAL, ExternalInput, do_IRQ, EXC_XFER_LITE)
	385	+ EXCEPTION(0x0500, EXTERNAL, ExternalInput, do_IRQ)
383	386
384	387	/* Alignment Interrupt */
385	388	ALIGNMENT_EXCEPTION
@@ -391,8 +394,7 @@ interrupt_base: · 5 unmodified lines
391	394	#ifdef CONFIG_PPC_FPU
392	395	FP_UNAVAILABLE_EXCEPTION
393	396	#else
394		− EXCEPTION(0x0800, FP_UNAVAIL, FloatingPointUnavailable, \
395		− unknown_exception, EXC_XFER_STD)
	397	+ EXCEPTION(0x0800, FP_UNAVAIL, FloatingPointUnavailable, unknown_exception)
396	398	#endif
397	399
398	400	/* System Call Interrupt */
@@ -400,16 +402,14 @@ interrupt_base: · 1 unmodified line
400	402	SYSCALL_ENTRY 0xc00 BOOKE_INTERRUPT_SYSCALL SPRN_SRR1
401	403
402	404	/* Auxiliary Processor Unavailable Interrupt */
403		− EXCEPTION(0x2900, AP_UNAVAIL, AuxillaryProcessorUnavailable, \
404		− unknown_exception, EXC_XFER_STD)
	405	+ EXCEPTION(0x2900, AP_UNAVAIL, AuxillaryProcessorUnavailable, unknown_exception)
405	406
406	407	/* Decrementer Interrupt */
407	408	DECREMENTER_EXCEPTION
408	409
409	410	/* Fixed Internal Timer Interrupt */
410	411	/* TODO: Add FIT support */
411		− EXCEPTION(0x3100, FIT, FixedIntervalTimer, \
412		− unknown_exception, EXC_XFER_STD)
	412	+ EXCEPTION(0x3100, FIT, FixedIntervalTimer, unknown_exception)
413	413
414	414	/* Watchdog Timer Interrupt */
415	415	#ifdef CONFIG_BOOKE_WDT
@@ -497,7 +497,7 @@ END_BTB_FLUSH_SECTION · 81 unmodified lines
497	497	#endif
498	498	#endif
499	499
500		− bne 2f /* Bail if permission/valid mismach */
	500	+ bne 2f /* Bail if permission/valid mismatch */
501	501
502	502	/* Jump to common tlb load */
503	503	b finish_tlb_load
@@ -592,7 +592,7 @@ END_BTB_FLUSH_SECTION · 88 unmodified lines
592	592	#endif
593	593	#endif
594	594
595		− bne 2f /* Bail if permission mismach */
	595	+ bne 2f /* Bail if permission mismatch */
596	596
597	597	/* Jump to common TLB load point */
598	598	b finish_tlb_load
@@ -614,38 +614,44 @@ END_BTB_FLUSH_SECTION · 15 unmodified lines
614	614	#ifdef CONFIG_SPE
615	615	/* SPE Unavailable */
616	616	START_EXCEPTION(SPEUnavailable)
617		− NORMAL_EXCEPTION_PROLOG(SPE_UNAVAIL)
	617	+ NORMAL_EXCEPTION_PROLOG(0x2010, SPE_UNAVAIL)
618	618	beq 1f
619	619	bl load_up_spe
620	620	b fast_exception_return
621		−1: addi r3,r1,STACK_FRAME_OVERHEAD
622		− EXC_XFER_LITE(0x2010, KernelSPE)
	621	+1: prepare_transfer_to_handler
	622	+ bl KernelSPE
	623	+ b interrupt_return
623	624	#elif defined(CONFIG_SPE_POSSIBLE)
624		− EXCEPTION(0x2020, SPE_UNAVAIL, SPEUnavailable, \
625		− unknown_exception, EXC_XFER_STD)
	625	+ EXCEPTION(0x2020, SPE_UNAVAIL, SPEUnavailable, unknown_exception)
626	626	#endif /* CONFIG_SPE_POSSIBLE */
627	627
628	628	/* SPE Floating Point Data */
629	629	#ifdef CONFIG_SPE
630		− EXCEPTION(0x2030, SPE_FP_DATA, SPEFloatingPointData,
631		− SPEFloatingPointException, EXC_XFER_STD)
	630	+ START_EXCEPTION(SPEFloatingPointData)
	631	+ NORMAL_EXCEPTION_PROLOG(0x2030, SPE_FP_DATA)
	632	+ prepare_transfer_to_handler
	633	+ bl SPEFloatingPointException
	634	+ REST_NVGPRS(r1)
	635	+ b interrupt_return
632	636
633	637	/* SPE Floating Point Round */
634		− EXCEPTION(0x2050, SPE_FP_ROUND, SPEFloatingPointRound, \
635		− SPEFloatingPointRoundException, EXC_XFER_STD)
	638	+ START_EXCEPTION(SPEFloatingPointRound)
	639	+ NORMAL_EXCEPTION_PROLOG(0x2050, SPE_FP_ROUND)
	640	+ prepare_transfer_to_handler
	641	+ bl SPEFloatingPointRoundException
	642	+ REST_NVGPRS(r1)
	643	+ b interrupt_return
636	644	#elif defined(CONFIG_SPE_POSSIBLE)
637		− EXCEPTION(0x2040, SPE_FP_DATA, SPEFloatingPointData,
638		− unknown_exception, EXC_XFER_STD)
639		− EXCEPTION(0x2050, SPE_FP_ROUND, SPEFloatingPointRound, \
640		− unknown_exception, EXC_XFER_STD)
	645	+ EXCEPTION(0x2040, SPE_FP_DATA, SPEFloatingPointData, unknown_exception)
	646	+ EXCEPTION(0x2050, SPE_FP_ROUND, SPEFloatingPointRound, unknown_exception)
641	647	#endif /* CONFIG_SPE_POSSIBLE */
642	648
643	649
644	650	/* Performance Monitor */
645	651	EXCEPTION(0x2060, PERFORMANCE_MONITOR, PerformanceMonitor, \
646		− performance_monitor_exception, EXC_XFER_STD)
	652	+ performance_monitor_exception)
647	653
648		− EXCEPTION(0x2070, DOORBELL, Doorbell, doorbell_exception, EXC_XFER_STD)
	654	+ EXCEPTION(0x2070, DOORBELL, Doorbell, doorbell_exception)
649	655
650	656	CRITICAL_EXCEPTION(0x2080, DOORBELL_CRITICAL, \
651	657	CriticalDoorbell, unknown_exception)
@@ -660,10 +666,10 @@ END_BTB_FLUSH_SECTION · 8 unmodified lines
660	666	unknown_exception)
661	667
662	668	/* Hypercall */
663		− EXCEPTION(0, HV_SYSCALL, Hypercall, unknown_exception, EXC_XFER_STD)
	669	+ EXCEPTION(0, HV_SYSCALL, Hypercall, unknown_exception)
664	670
665	671	/* Embedded Hypervisor Privilege */
666		− EXCEPTION(0, HV_PRIV, Ehvpriv, unknown_exception, EXC_XFER_STD)
	672	+ EXCEPTION(0, HV_PRIV, Ehvpriv, unknown_exception)
667	673
668	674	interrupt_end:
669	675
@@ -854,7 +860,7 @@ KernelSPE: · 184 unmodified lines
854	860	lwz r5,_NIP(r1)
855	861	bl printk
856	862	#endif
857		− b ret_from_except
	863	+ b interrupt_return
858	864	#ifdef CONFIG_PRINTK
859	865	87: .string "SPE used in kernel (task=%p, pc=%x) \n"
860	866	#endif

arch/powerpc/kernel/hw_breakpoint_constraints.c+1−1

@@ -141,7 +141,7 @@ void wp_get_instr_detail(struct pt_regs regs, struct ppc_inst instr, · 140 unmodified lines
141	141	{
142	142	struct instruction_op op;
143	143
144		− if (__get_user_instr_inatomic(instr, (void __user )regs->nip))
	144	+ if (__get_user_instr(instr, (void __user )regs->nip))
145	145	return;
146	146
147	147	analyse_instr(&op, regs, *instr);

arch/powerpc/kernel/idle_6xx.S+2−12

@@ -145,9 +145,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) · 144 unmodified lines
145	145
146	146	/*
147	147	* Return from NAP/DOZE mode, restore some CPU specific registers,
148		− * we are called with DR/IR still off and r2 containing physical
149		− * address of current. R11 points to the exception frame (physical
150		− * address). We have to preserve r10.
	148	+ * R11 points to the exception frame. We have to preserve r10.
151	149	*/
152	150	_GLOBAL(power_save_ppc32_restore)
153	151	lwz r9,_LINK(r11) /* interrupted in ppc6xx_idle: */
@@ -166,11 +164,7 @@ BEGIN_FTR_SECTION · 12 unmodified lines
166	164	mfspr r9,SPRN_HID0
167	165	andis. r9,r9,HID0_NAP@h
168	166	beq 1f
169		−#ifdef CONFIG_VMAP_STACK
170	167	addis r9, r11, nap_save_msscr0@ha
171		−#else
172		− addis r9,r11,(nap_save_msscr0-KERNELBASE)@ha
173		−#endif
174	168	lwz r9,nap_save_msscr0@l(r9)
175	169	mtspr SPRN_MSSCR0, r9
176	170	sync
@@ -178,15 +172,11 @@ BEGIN_FTR_SECTION · 1 unmodified line
178	172	1:
179	173	END_FTR_SECTION_IFSET(CPU_FTR_NAP_DISABLE_L2_PR)
180	174	BEGIN_FTR_SECTION
181		−#ifdef CONFIG_VMAP_STACK
182	175	addis r9, r11, nap_save_hid1@ha
183		−#else
184		− addis r9,r11,(nap_save_hid1-KERNELBASE)@ha
185		−#endif
186	176	lwz r9,nap_save_hid1@l(r9)
187	177	mtspr SPRN_HID1, r9
188	178	END_FTR_SECTION_IFSET(CPU_FTR_DUAL_PLL_750FX)
189		− b transfer_to_handler_cont
	179	+ blr
190	180	_ASM_NOKPROBE_SYMBOL(power_save_ppc32_restore)
191	181
192	182	.data

arch/powerpc/kernel/idle_book3s.S+4−0

@@ -209,4 +209,8 @@ _GLOBAL(power4_idle_nap) · 208 unmodified lines
209	209	mtmsrd r7
210	210	isync
211	211	b 1b
	212	+
	213	+ .globl power4_idle_nap_return
	214	+power4_idle_nap_return:
	215	+ blr
212	216	#endif

arch/powerpc/kernel/idle_e500.S+3−11

@@ -74,20 +74,12 @@ END_FTR_SECTION_IFSET(CPU_FTR_CAN_NAP) · 73 unmodified lines
74	74
75	75	/*
76	76	* Return from NAP/DOZE mode, restore some CPU specific registers,
77		− * r2 containing physical address of current.
78		− * r11 points to the exception frame (physical address).
	77	+ * r2 containing address of current.
	78	+ * r11 points to the exception frame.
79	79	* We have to preserve r10.
80	80	*/
81	81	_GLOBAL(power_save_ppc32_restore)
82	82	lwz r9,_LINK(r11) /* interrupted in e500_idle */
83	83	stw r9,_NIP(r11) /* make it do a blr */
84		−
85		−#ifdef CONFIG_SMP
86		− lwz r11,TASK_CPU(r2) /* get cpu number * 4 */
87		− slwi r11,r11,2
88		−#else
89		− li r11,0
90		−#endif
91		−
92		− b transfer_to_handler_cont
	84	+ blr
93	85	_ASM_NOKPROBE_SYMBOL(power_save_ppc32_restore)

arch/powerpc/kernel/interrupt.c+27−40

@@ -20,6 +20,10 @@ · 19 unmodified lines
20	20	#include <asm/time.h>
21	21	#include <asm/unistd.h>
22	22
	23	+#if defined(CONFIG_PPC_ADV_DEBUG_REGS) && defined(CONFIG_PPC32)
	24	+unsigned long global_dbcr0[NR_CPUS];
	25	+#endif
	26	+
23	27	typedef long (*syscall_fn)(long, long, long, long, long, long);
24	28
25	29	/* Has to run notrace because it is entered not completely "reconciled" */
@@ -29,20 +33,24 @@ notrace long system_call_exception(long r3, long r4, long r5, · 3 unmodified lines
29	33	{
30	34	syscall_fn f;
31	35
	36	+ kuep_lock();
	37	+#ifdef CONFIG_PPC32
	38	+ kuap_save_and_lock(regs);
	39	+#endif
	40	+
32	41	regs->orig_gpr3 = r3;
33	42
34	43	if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG))
35	44	BUG_ON(irq_soft_mask_return() != IRQS_ALL_DISABLED);
36	45
	46	+ trace_hardirqs_off(); /* finish reconciling */
	47	+
37	48	CT_WARN_ON(ct_state() == CONTEXT_KERNEL);
38	49	user_exit_irqoff();
39	50
40		− trace_hardirqs_off(); /* finish reconciling */
41		−
42	51	if (!IS_ENABLED(CONFIG_BOOKE) && !IS_ENABLED(CONFIG_40x))
43	52	BUG_ON(!(regs->msr & MSR_RI));
44	53	BUG_ON(!(regs->msr & MSR_PR));
45		− BUG_ON(!FULL_REGS(regs));
46	54	BUG_ON(arch_irq_disabled_regs(regs));
47	55
48	56	#ifdef CONFIG_PPC_PKEY
@@ -69,9 +77,7 @@ notrace long system_call_exception(long r3, long r4, long r5, · 20 unmodified lines
69	77	isync();
70	78	} else
71	79	#endif
72		−#ifdef CONFIG_PPC64
73		− kuap_check_amr();
74		−#endif
	80	+ kuap_assert_locked();
75	81
76	82	booke_restore_dbcr0();
77	83
@@ -247,9 +253,7 @@ notrace unsigned long syscall_exit_prepare(unsigned long r3, · 169 unmodified lines
247	253
248	254	CT_WARN_ON(ct_state() == CONTEXT_USER);
249	255
250		−#ifdef CONFIG_PPC64
251		− kuap_check_amr();
252		−#endif
	256	+ kuap_assert_locked();
253	257
254	258	regs->result = r3;
255	259
@@ -344,16 +348,13 @@ again: · 88 unmodified lines
344	348
345	349	account_cpu_user_exit();
346	350
347		−#ifdef CONFIG_PPC_BOOK3S_64 /* BOOK3E and ppc32 not using this */
348		− /*
349		− * We do this at the end so that we do context switch with KERNEL AMR
350		− */
	351	+ /* Restore user access locks last */
351	352	kuap_user_restore(regs);
352		−#endif
	353	+ kuep_unlock();
	354	+
353	355	return ret;
354	356	}
355	357
356		−#ifndef CONFIG_PPC_BOOK3E_64 /* BOOK3E not yet using this */
357	358	notrace unsigned long interrupt_exit_user_prepare(struct pt_regs *regs, unsigned long msr)
358	359	{
359	360	unsigned long ti_flags;
@@ -363,7 +364,6 @@ notrace unsigned long interrupt_exit_user_prepare(struct pt_regs *regs, unsigned · 3 unmodified lines
363	364	if (!IS_ENABLED(CONFIG_BOOKE) && !IS_ENABLED(CONFIG_40x))
364	365	BUG_ON(!(regs->msr & MSR_RI));
365	366	BUG_ON(!(regs->msr & MSR_PR));
366		− BUG_ON(!FULL_REGS(regs));
367	367	BUG_ON(arch_irq_disabled_regs(regs));
368	368	CT_WARN_ON(ct_state() == CONTEXT_USER);
369	369
@@ -371,9 +371,7 @@ notrace unsigned long interrupt_exit_user_prepare(struct pt_regs *regs, unsigned · 1 unmodified line
371	371	* We don't need to restore AMR on the way back to userspace for KUAP.
372	372	* AMR can only have been unlocked if we interrupted the kernel.
373	373	*/
374		−#ifdef CONFIG_PPC64
375		− kuap_check_amr();
376		−#endif
	374	+ kuap_assert_locked();
377	375
378	376	local_irq_save(flags);
379	377
@@ -392,7 +390,7 @@ again: · 12 unmodified lines
392	390	ti_flags = READ_ONCE(current_thread_info()->flags);
393	391	}
394	392
395		− if (IS_ENABLED(CONFIG_PPC_BOOK3S) && IS_ENABLED(CONFIG_PPC_FPU)) {
	393	+ if (IS_ENABLED(CONFIG_PPC_BOOK3S_64) && IS_ENABLED(CONFIG_PPC_FPU)) {
396	394	if (IS_ENABLED(CONFIG_PPC_TRANSACTIONAL_MEM) &&
397	395	unlikely((ti_flags & _TIF_RESTORE_TM))) {
398	396	restore_tm_state(regs);
@@ -427,12 +425,9 @@ again: · 28 unmodified lines
427	425
428	426	account_cpu_user_exit();
429	427
430		− /*
431		− * We do this at the end so that we do context switch with KERNEL AMR
432		− */
433		−#ifdef CONFIG_PPC64
	428	+ /* Restore user access locks last */
434	429	kuap_user_restore(regs);
435		−#endif
	430	+
436	431	return ret;
437	432	}
438	433
@@ -442,25 +437,20 @@ notrace unsigned long interrupt_exit_kernel_prepare(struct pt_regs *regs, unsign · 3 unmodified lines
442	437	{
443	438	unsigned long flags;
444	439	unsigned long ret = 0;
445		−#ifdef CONFIG_PPC64
446		− unsigned long amr;
447		−#endif
	440	+ unsigned long kuap;
448	441
449	442	if (!IS_ENABLED(CONFIG_BOOKE) && !IS_ENABLED(CONFIG_40x) &&
450	443	unlikely(!(regs->msr & MSR_RI)))
451	444	unrecoverable_exception(regs);
452	445	BUG_ON(regs->msr & MSR_PR);
453		− BUG_ON(!FULL_REGS(regs));
454	446	/*
455	447	* CT_WARN_ON comes here via program_check_exception,
456	448	* so avoid recursion.
457	449	*/
458		− if (TRAP(regs) != 0x700)
	450	+ if (TRAP(regs) != INTERRUPT_PROGRAM)
459	451	CT_WARN_ON(ct_state() == CONTEXT_USER);
460	452
461		−#ifdef CONFIG_PPC64
462		− amr = kuap_get_and_check_amr();
463		−#endif
	453	+ kuap = kuap_get_and_assert_locked();
464	454
465	455	if (unlikely(current_thread_info()->flags & _TIF_EMULATE_STACK_STORE)) {
466	456	clear_bits(_TIF_EMULATE_STACK_STORE, &current_thread_info()->flags);
@@ -498,14 +488,11 @@ again: · 31 unmodified lines
498	488	#endif
499	489
500	490	/*
501		− * Don't want to mfspr(SPRN_AMR) here, because this comes after mtmsr,
502		− * which would cause Read-After-Write stalls. Hence, we take the AMR
503		− * value from the check above.
	491	+ * 64s does not want to mfspr(SPRN_AMR) here, because this comes after
	492	+ * mtmsr, which would cause Read-After-Write stalls. Hence, take the
	493	+ * AMR value from the check above.
504	494	*/
505		−#ifdef CONFIG_PPC64
506		− kuap_kernel_restore(regs, amr);
507		−#endif
	495	+ kuap_kernel_restore(regs, kuap);
508	496
509	497	return ret;
510	498	}
511		−#endif

arch/powerpc/kernel/iommu.c+24−22

@@ -72,8 +72,7 @@ static void iommu_debugfs_del(struct iommu_table *tbl) · 71 unmodified lines
72	72
73	73	sprintf(name, "%08lx", tbl->it_index);
74	74	liobn_entry = debugfs_lookup(name, iommu_debugfs_dir);
75		− if (liobn_entry)
76		− debugfs_remove(liobn_entry);
	75	+ debugfs_remove(liobn_entry);
77	76	}
78	77	#else
79	78	static void iommu_debugfs_add(struct iommu_table *tbl){}
@@ -297,6 +296,15 @@ again: · 217 unmodified lines
297	296	pass++;
298	297	goto again;
299	298
	299	+ } else if (pass == tbl->nr_pools + 1) {
	300	+ /* Last resort: try largepool */
	301	+ spin_unlock(&pool->lock);
	302	+ pool = &tbl->large_pool;
	303	+ spin_lock(&pool->lock);
	304	+ pool->hint = pool->start;
	305	+ pass++;
	306	+ goto again;
	307	+
300	308	} else {
301	309	/* Give up */
302	310	spin_unlock_irqrestore(&(pool->lock), flags);
@@ -719,7 +727,6 @@ struct iommu_table iommu_init_table(struct iommu_table tbl, int nid, · 416 unmodified lines
719	727	{
720	728	unsigned long sz;
721	729	static int welcomed = 0;
722		− struct page *page;
723	730	unsigned int i;
724	731	struct iommu_pool *p;
725	732
@@ -728,11 +735,11 @@ struct iommu_table iommu_init_table(struct iommu_table tbl, int nid, · 2 unmodified lines
728	735	/* number of bytes needed for the bitmap */
729	736	sz = BITS_TO_LONGS(tbl->it_size) * sizeof(unsigned long);
730	737
731		− page = alloc_pages_node(nid, GFP_KERNEL, get_order(sz));
732		− if (!page)
733		− panic("iommu_init_table: Can't allocate %ld bytes\n", sz);
734		− tbl->it_map = page_address(page);
735		− memset(tbl->it_map, 0, sz);
	738	+ tbl->it_map = vzalloc_node(sz, nid);
	739	+ if (!tbl->it_map) {
	740	+ pr_err("%s: Can't allocate %ld bytes\n", __func__, sz);
	741	+ return NULL;
	742	+ }
736	743
737	744	iommu_table_reserve_pages(tbl, res_start, res_end);
738	745
@@ -774,8 +781,6 @@ struct iommu_table iommu_init_table(struct iommu_table tbl, int nid, · 35 unmodified lines
774	781
775	782	static void iommu_table_free(struct kref *kref)
776	783	{
777		− unsigned long bitmap_sz;
778		− unsigned int order;
779	784	struct iommu_table *tbl;
780	785
781	786	tbl = container_of(kref, struct iommu_table, it_kref);
@@ -796,12 +801,8 @@ static void iommu_table_free(struct kref *kref) · 14 unmodified lines
796	801	if (!bitmap_empty(tbl->it_map, tbl->it_size))
797	802	pr_warn("%s: Unexpected TCEs\n", __func__);
798	803
799		− /* calculate bitmap size in bytes */
800		− bitmap_sz = BITS_TO_LONGS(tbl->it_size) * sizeof(unsigned long);
801		−
802	804	/* free bitmap */
803		− order = get_order(bitmap_sz);
804		− free_pages((unsigned long) tbl->it_map, order);
	805	+ vfree(tbl->it_map);
805	806
806	807	/* free table */
807	808	kfree(tbl);
@@ -897,6 +898,7 @@ void iommu_alloc_coherent(struct device dev, struct iommu_table *tbl, · 89 unmodified lines
897	898	unsigned int order;
898	899	unsigned int nio_pages, io_order;
899	900	struct page *page;
	901	+ size_t size_io = size;
900	902
901	903	size = PAGE_ALIGN(size);
902	904	order = get_order(size);
@@ -923,8 +925,9 @@ void iommu_alloc_coherent(struct device dev, struct iommu_table *tbl, · 20 unmodified lines
923	925	memset(ret, 0, size);
924	926
925	927	/* Set up tces to cover the allocated range */
926		− nio_pages = size >> tbl->it_page_shift;
927		− io_order = get_iommu_order(size, tbl);
	928	+ size_io = IOMMU_PAGE_ALIGN(size_io, tbl);
	929	+ nio_pages = size_io >> tbl->it_page_shift;
	930	+ io_order = get_iommu_order(size_io, tbl);
928	931	mapping = iommu_alloc(dev, tbl, ret, nio_pages, DMA_BIDIRECTIONAL,
929	932	mask >> tbl->it_page_shift, io_order, 0);
930	933	if (mapping == DMA_MAPPING_ERROR) {
@@ -939,10 +942,9 @@ void iommu_free_coherent(struct iommu_table *tbl, size_t size, · 8 unmodified lines
939	942	void *vaddr, dma_addr_t dma_handle)
940	943	{
941	944	if (tbl) {
942		− unsigned int nio_pages;
	945	+ size_t size_io = IOMMU_PAGE_ALIGN(size, tbl);
	946	+ unsigned int nio_pages = size_io >> tbl->it_page_shift;
943	947
944		− size = PAGE_ALIGN(size);
945		− nio_pages = size >> tbl->it_page_shift;
946	948	iommu_free(tbl, dma_handle, nio_pages);
947	949	size = PAGE_ALIGN(size);
948	950	free_pages((unsigned long)vaddr, get_order(size));
@@ -1096,7 +1098,7 @@ int iommu_take_ownership(struct iommu_table *tbl) · 147 unmodified lines
1096	1098
1097	1099	spin_lock_irqsave(&tbl->large_pool.lock, flags);
1098	1100	for (i = 0; i < tbl->nr_pools; i++)
1099		− spin_lock(&tbl->pools[i].lock);
	1101	+ spin_lock_nest_lock(&tbl->pools[i].lock, &tbl->large_pool.lock);
1100	1102
1101	1103	iommu_table_release_pages(tbl);
1102	1104
@@ -1124,7 +1126,7 @@ void iommu_release_ownership(struct iommu_table *tbl) · 21 unmodified lines
1124	1126
1125	1127	spin_lock_irqsave(&tbl->large_pool.lock, flags);
1126	1128	for (i = 0; i < tbl->nr_pools; i++)
1127		− spin_lock(&tbl->pools[i].lock);
	1129	+ spin_lock_nest_lock(&tbl->pools[i].lock, &tbl->large_pool.lock);
1128	1130
1129	1131	memset(tbl->it_map, 0, sz);
1130	1132

arch/powerpc/kernel/irq.c+47−85

@@ -104,82 +104,6 @@ static inline notrace unsigned long get_irq_happened(void) · 103 unmodified lines
104	104	return happened;
105	105	}
106	106
107		−#ifdef CONFIG_PPC_BOOK3E
108		−
109		−/* This is called whenever we are re-enabling interrupts
110		− * and returns either 0 (nothing to do) or 500/900/280 if
111		− * there's an EE, DEC or DBELL to generate.
112		− *
113		− * This is called in two contexts: From arch_local_irq_restore()
114		− * before soft-enabling interrupts, and from the exception exit
115		− * path when returning from an interrupt from a soft-disabled to
116		− * a soft enabled context. In both case we have interrupts hard
117		− * disabled.
118		− *
119		− * We take care of only clearing the bits we handled in the
120		− * PACA irq_happened field since we can only re-emit one at a
121		− * time and we don't want to "lose" one.
122		− */
123		−notrace unsigned int __check_irq_replay(void)
124		−{
125		− /*
126		− * We use local_paca rather than get_paca() to avoid all
127		− * the debug_smp_processor_id() business in this low level
128		− * function
129		− */
130		− unsigned char happened = local_paca->irq_happened;
131		−
132		− /*
133		− * We are responding to the next interrupt, so interrupt-off
134		− * latencies should be reset here.
135		− */
136		− trace_hardirqs_on();
137		− trace_hardirqs_off();
138		−
139		− if (happened & PACA_IRQ_DEC) {
140		− local_paca->irq_happened &= ~PACA_IRQ_DEC;
141		− return 0x900;
142		− }
143		−
144		− if (happened & PACA_IRQ_EE) {
145		− local_paca->irq_happened &= ~PACA_IRQ_EE;
146		− return 0x500;
147		− }
148		−
149		− if (happened & PACA_IRQ_DBELL) {
150		− local_paca->irq_happened &= ~PACA_IRQ_DBELL;
151		− return 0x280;
152		− }
153		−
154		− if (happened & PACA_IRQ_HARD_DIS)
155		− local_paca->irq_happened &= ~PACA_IRQ_HARD_DIS;
156		−
157		− /* There should be nothing left ! */
158		− BUG_ON(local_paca->irq_happened != 0);
159		−
160		− return 0;
161		−}
162		−
163		−/*
164		− * This is specifically called by assembly code to re-enable interrupts
165		− * if they are currently disabled. This is typically called before
166		− * schedule() or do_signal() when returning to userspace. We do it
167		− * in C to avoid the burden of dealing with lockdep etc...
168		− *
169		− * NOTE: This is called with interrupts hard disabled but not marked
170		− * as such in paca->irq_happened, so we need to resync this.
171		− */
172		−void notrace restore_interrupts(void)
173		−{
174		− if (irqs_disabled()) {
175		− local_paca->irq_happened \|= PACA_IRQ_HARD_DIS;
176		− local_irq_enable();
177		− } else
178		− __hard_irq_enable();
179		−}
180		−
181		−#endif /* CONFIG_PPC_BOOK3E */
182		−
183	107	void replay_soft_interrupts(void)
184	108	{
185	109	struct pt_regs regs;
@@ -218,7 +142,7 @@ again: · 32 unmodified lines
218	142	*/
219	143	if (IS_ENABLED(CONFIG_PPC_BOOK3S) && (local_paca->irq_happened & PACA_IRQ_HMI)) {
220	144	local_paca->irq_happened &= ~PACA_IRQ_HMI;
221		− regs.trap = 0xe60;
	145	+ regs.trap = INTERRUPT_HMI;
222	146	handle_hmi_exception(&regs);
223	147	if (!(local_paca->irq_happened & PACA_IRQ_HARD_DIS))
224	148	hard_irq_disable();
@@ -226,7 +150,7 @@ again: · 1 unmodified line
226	150
227	151	if (local_paca->irq_happened & PACA_IRQ_DEC) {
228	152	local_paca->irq_happened &= ~PACA_IRQ_DEC;
229		− regs.trap = 0x900;
	153	+ regs.trap = INTERRUPT_DECREMENTER;
230	154	timer_interrupt(&regs);
231	155	if (!(local_paca->irq_happened & PACA_IRQ_HARD_DIS))
232	156	hard_irq_disable();
@@ -234,7 +158,7 @@ again: · 1 unmodified line
234	158
235	159	if (local_paca->irq_happened & PACA_IRQ_EE) {
236	160	local_paca->irq_happened &= ~PACA_IRQ_EE;
237		− regs.trap = 0x500;
	161	+ regs.trap = INTERRUPT_EXTERNAL;
238	162	do_IRQ(&regs);
239	163	if (!(local_paca->irq_happened & PACA_IRQ_HARD_DIS))
240	164	hard_irq_disable();
@@ -242,10 +166,7 @@ again: · 1 unmodified line
242	166
243	167	if (IS_ENABLED(CONFIG_PPC_DOORBELL) && (local_paca->irq_happened & PACA_IRQ_DBELL)) {
244	168	local_paca->irq_happened &= ~PACA_IRQ_DBELL;
245		− if (IS_ENABLED(CONFIG_PPC_BOOK3E))
246		− regs.trap = 0x280;
247		− else
248		− regs.trap = 0xa00;
	169	+ regs.trap = INTERRUPT_DOORBELL;
249	170	doorbell_exception(&regs);
250	171	if (!(local_paca->irq_happened & PACA_IRQ_HARD_DIS))
251	172	hard_irq_disable();
@@ -254,7 +175,7 @@ again: · 2 unmodified lines
254	175	/* Book3E does not support soft-masking PMI interrupts */
255	176	if (IS_ENABLED(CONFIG_PPC_BOOK3S) && (local_paca->irq_happened & PACA_IRQ_PMI)) {
256	177	local_paca->irq_happened &= ~PACA_IRQ_PMI;
257		− regs.trap = 0xf00;
	178	+ regs.trap = INTERRUPT_PERFMON;
258	179	performance_monitor_exception(&regs);
259	180	if (!(local_paca->irq_happened & PACA_IRQ_HARD_DIS))
260	181	hard_irq_disable();
@@ -282,7 +203,7 @@ static inline void replay_soft_interrupts_irqrestore(void) · 21 unmodified lines
282	203	* and re-locking AMR but we shouldn't get here in the first place,
283	204	* hence the warning.
284	205	*/
285		− kuap_check_amr();
	206	+ kuap_assert_locked();
286	207
287	208	if (kuap_state != AMR_KUAP_BLOCKED)
288	209	set_kuap(AMR_KUAP_BLOCKED);
@@ -667,6 +588,47 @@ static inline void check_stack_overflow(void) · 378 unmodified lines
667	588	}
668	589	}
669	590
	591	+static __always_inline void call_do_softirq(const void *sp)
	592	+{
	593	+ /* Temporarily switch r1 to sp, call __do_softirq() then restore r1. */
	594	+ asm volatile (
	595	+ PPC_STLU " %%r1, %[offset](%[sp]) ;"
	596	+ "mr %%r1, %[sp] ;"
	597	+ "bl %[callee] ;"
	598	+ PPC_LL " %%r1, 0(%%r1) ;"
	599	+ : // Outputs
	600	+ : // Inputs
	601	+ [sp] "b" (sp), [offset] "i" (THREAD_SIZE - STACK_FRAME_OVERHEAD),
	602	+ [callee] "i" (__do_softirq)
	603	+ : // Clobbers
	604	+ "lr", "xer", "ctr", "memory", "cr0", "cr1", "cr5", "cr6",
	605	+ "cr7", "r0", "r3", "r4", "r5", "r6", "r7", "r8", "r9", "r10",
	606	+ "r11", "r12"
	607	+ );
	608	+}
	609	+
	610	+static __always_inline void call_do_irq(struct pt_regs regs, void sp)
	611	+{
	612	+ register unsigned long r3 asm("r3") = (unsigned long)regs;
	613	+
	614	+ /* Temporarily switch r1 to sp, call __do_irq() then restore r1. */
	615	+ asm volatile (
	616	+ PPC_STLU " %%r1, %[offset](%[sp]) ;"
	617	+ "mr %%r1, %[sp] ;"
	618	+ "bl %[callee] ;"
	619	+ PPC_LL " %%r1, 0(%%r1) ;"
	620	+ : // Outputs
	621	+ "+r" (r3)
	622	+ : // Inputs
	623	+ [sp] "b" (sp), [offset] "i" (THREAD_SIZE - STACK_FRAME_OVERHEAD),
	624	+ [callee] "i" (__do_irq)
	625	+ : // Clobbers
	626	+ "lr", "xer", "ctr", "memory", "cr0", "cr1", "cr5", "cr6",
	627	+ "cr7", "r0", "r4", "r5", "r6", "r7", "r8", "r9", "r10",
	628	+ "r11", "r12"
	629	+ );
	630	+}
	631	+
670	632	void __do_irq(struct pt_regs *regs)
671	633	{
672	634	unsigned int irq;

arch/powerpc/kernel/jump_label.c+2−2

@@ -11,10 +11,10 @@ · 10 unmodified lines
11	11	void arch_jump_label_transform(struct jump_entry *entry,
12	12	enum jump_label_type type)
13	13	{
14		− struct ppc_inst addr = (struct ppc_inst )(unsigned long)entry->code;
	14	+ struct ppc_inst addr = (struct ppc_inst )jump_entry_code(entry);
15	15
16	16	if (type == JUMP_LABEL_JMP)
17		− patch_branch(addr, entry->target, 0);
	17	+ patch_branch(addr, jump_entry_target(entry), 0);
18	18	else
19	19	patch_instruction(addr, ppc_inst(PPC_INST_NOP));
20	20	}

arch/powerpc/kernel/kgdb.c+1−1

@@ -376,7 +376,7 @@ void kgdb_arch_set_pc(struct pt_regs *regs, unsigned long pc) · 375 unmodified lines
376	376	}
377	377
378	378	/*
379		− * This function does PowerPC specific procesing for interfacing to gdb.
	379	+ * This function does PowerPC specific processing for interfacing to gdb.
380	380	*/
381	381	int kgdb_arch_handle_exception(int vector, int signo, int err_code,
382	382	char remcom_in_buffer, char remcom_out_buffer,

arch/powerpc/kernel/legacy_serial.c+29−4

@@ -15,6 +15,7 @@ · 14 unmodified lines
15	15	#include <asm/udbg.h>
16	16	#include <asm/pci-bridge.h>
17	17	#include <asm/ppc-pci.h>
	18	+#include <asm/early_ioremap.h>
18	19
19	20	#undef DEBUG
20	21
@@ -34,6 +35,7 @@ static struct legacy_serial_info { · 13 unmodified lines
34	35	unsigned int clock;
35	36	int irq_check_parent;
36	37	phys_addr_t taddr;
	38	+ void __iomem *early_addr;
37	39	} legacy_serial_infos[MAX_LEGACY_SERIAL_PORTS];
38	40
39	41	static const struct of_device_id legacy_serial_parents[] __initconst = {
@@ -325,17 +327,16 @@ static void __init setup_legacy_serial_console(int console) · 285 unmodified lines
325	327	{
326	328	struct legacy_serial_info *info = &legacy_serial_infos[console];
327	329	struct plat_serial8250_port *port = &legacy_serial_ports[console];
328		− void __iomem *addr;
329	330	unsigned int stride;
330	331
331	332	stride = 1 << port->regshift;
332	333
333	334	/* Check if a translated MMIO address has been found */
334	335	if (info->taddr) {
335		− addr = ioremap(info->taddr, 0x1000);
336		− if (addr == NULL)
	336	+ info->early_addr = early_ioremap(info->taddr, 0x1000);
	337	+ if (info->early_addr == NULL)
337	338	return;
338		− udbg_uart_init_mmio(addr, stride);
	339	+ udbg_uart_init_mmio(info->early_addr, stride);
339	340	} else {
340	341	/* Check if it's PIO and we support untranslated PIO */
341	342	if (port->iotype == UPIO_PORT && isa_io_special)
@@ -353,6 +354,30 @@ static void __init setup_legacy_serial_console(int console) · 11 unmodified lines
353	354	udbg_uart_setup(info->speed, info->clock);
354	355	}
355	356
	357	+static int __init ioremap_legacy_serial_console(void)
	358	+{
	359	+ struct legacy_serial_info *info = &legacy_serial_infos[legacy_serial_console];
	360	+ struct plat_serial8250_port *port = &legacy_serial_ports[legacy_serial_console];
	361	+ void __iomem *vaddr;
	362	+
	363	+ if (legacy_serial_console < 0)
	364	+ return 0;
	365	+
	366	+ if (!info->early_addr)
	367	+ return 0;
	368	+
	369	+ vaddr = ioremap(info->taddr, 0x1000);
	370	+ if (WARN_ON(!vaddr))
	371	+ return -ENOMEM;
	372	+
	373	+ udbg_uart_init_mmio(vaddr, 1 << port->regshift);
	374	+ early_iounmap(info->early_addr, 0x1000);
	375	+ info->early_addr = NULL;
	376	+
	377	+ return 0;
	378	+}
	379	+early_initcall(ioremap_legacy_serial_console);
	380	+
356	381	/*
357	382	* This is called very early, as part of setup_system() or eventually
358	383	* setup_arch(), basically before anything else in this file. This function

arch/powerpc/kernel/mce.c+3−2

@@ -40,7 +40,7 @@ static struct irq_work mce_ue_event_irq_work = { · 39 unmodified lines
40	40	.func = machine_check_ue_irq_work,
41	41	};
42	42
43		−DECLARE_WORK(mce_ue_event_work, machine_process_ue_event);
	43	+static DECLARE_WORK(mce_ue_event_work, machine_process_ue_event);
44	44
45	45	static BLOCKING_NOTIFIER_HEAD(mce_notifier_list);
46	46
@@ -131,6 +131,8 @@ void save_mce_event(struct pt_regs *regs, long handled, · 84 unmodified lines
131	131	* Populate the mce error_type and type-specific error_type.
132	132	*/
133	133	mce_set_error_info(mce, mce_err);
	134	+ if (mce->error_type == MCE_ERROR_TYPE_UE)
	135	+ mce->u.ue_error.ignore_event = mce_err->ignore_event;
134	136
135	137	if (!addr)
136	138	return;
@@ -159,7 +161,6 @@ void save_mce_event(struct pt_regs *regs, long handled, · 22 unmodified lines
159	161	if (phys_addr != ULONG_MAX) {
160	162	mce->u.ue_error.physical_address_provided = true;
161	163	mce->u.ue_error.physical_address = phys_addr;
162		− mce->u.ue_error.ignore_event = mce_err->ignore_event;
163	164	machine_check_ue_event(mce);
164	165	}
165	166	}

arch/powerpc/kernel/misc_32.S+0−39

@@ -27,45 +27,6 @@ · 26 unmodified lines
27	27
28	28	.text
29	29
30		−/*
31		− * We store the saved ksp_limit in the unused part
32		− * of the STACK_FRAME_OVERHEAD
33		− */
34		−_GLOBAL(call_do_softirq)
35		− mflr r0
36		− stw r0,4(r1)
37		− lwz r10,THREAD+KSP_LIMIT(r2)
38		− stw r3, THREAD+KSP_LIMIT(r2)
39		− stwu r1,THREAD_SIZE-STACK_FRAME_OVERHEAD(r3)
40		− mr r1,r3
41		− stw r10,8(r1)
42		− bl __do_softirq
43		− lwz r10,8(r1)
44		− lwz r1,0(r1)
45		− lwz r0,4(r1)
46		− stw r10,THREAD+KSP_LIMIT(r2)
47		− mtlr r0
48		− blr
49		−
50		−/*
51		− * void call_do_irq(struct pt_regs regs, void sp);
52		− */
53		−_GLOBAL(call_do_irq)
54		− mflr r0
55		− stw r0,4(r1)
56		− lwz r10,THREAD+KSP_LIMIT(r2)
57		− stw r4, THREAD+KSP_LIMIT(r2)
58		− stwu r1,THREAD_SIZE-STACK_FRAME_OVERHEAD(r4)
59		− mr r1,r4
60		− stw r10,8(r1)
61		− bl __do_irq
62		− lwz r10,8(r1)
63		− lwz r1,0(r1)
64		− lwz r0,4(r1)
65		− stw r10,THREAD+KSP_LIMIT(r2)
66		− mtlr r0
67		− blr
68		−
69	30	/*
70	31	* This returns the high 64 bits of the product of two 64-bit numbers.
71	32	*/

arch/powerpc/kernel/misc_64.S+0−22

@@ -27,28 +27,6 @@ · 26 unmodified lines
27	27
28	28	.text
29	29
30		−_GLOBAL(call_do_softirq)
31		− mflr r0
32		− std r0,16(r1)
33		− stdu r1,THREAD_SIZE-STACK_FRAME_OVERHEAD(r3)
34		− mr r1,r3
35		− bl __do_softirq
36		− ld r1,0(r1)
37		− ld r0,16(r1)
38		− mtlr r0
39		− blr
40		−
41		−_GLOBAL(call_do_irq)
42		− mflr r0
43		− std r0,16(r1)
44		− stdu r1,THREAD_SIZE-STACK_FRAME_OVERHEAD(r4)
45		− mr r1,r4
46		− bl __do_irq
47		− ld r1,0(r1)
48		− ld r0,16(r1)
49		− mtlr r0
50		− blr
51		−
52	30	_GLOBAL(__bswapdi2)
53	31	EXPORT_SYMBOL(__bswapdi2)
54	32	srdi r8,r3,32

arch/powerpc/kernel/module.c+20−3

@@ -14,6 +14,7 @@ · 13 unmodified lines
14	14	#include <asm/firmware.h>
15	15	#include <linux/sort.h>
16	16	#include <asm/setup.h>
	17	+#include <asm/sections.h>
17	18
18	19	static LIST_HEAD(module_bug_list);
19	20
@@ -88,12 +89,28 @@ int module_finalize(const Elf_Ehdr *hdr, · 68 unmodified lines
88	89	}
89	90
90	91	#ifdef MODULES_VADDR
	92	+static __always_inline void *
	93	+__module_alloc(unsigned long size, unsigned long start, unsigned long end)
	94	+{
	95	+ return __vmalloc_node_range(size, 1, start, end, GFP_KERNEL,
	96	+ PAGE_KERNEL_EXEC, VM_FLUSH_RESET_PERMS, NUMA_NO_NODE,
	97	+ __builtin_return_address(0));
	98	+}
	99	+
91	100	void *module_alloc(unsigned long size)
92	101	{
	102	+ unsigned long limit = (unsigned long)_etext - SZ_32M;
	103	+ void *ptr = NULL;
	104	+
93	105	BUILD_BUG_ON(TASK_SIZE > MODULES_VADDR);
94	106
95		− return __vmalloc_node_range(size, 1, MODULES_VADDR, MODULES_END, GFP_KERNEL,
96		− PAGE_KERNEL_EXEC, VM_FLUSH_RESET_PERMS, NUMA_NO_NODE,
97		− __builtin_return_address(0));
	107	+ /* First try within 32M limit from _etext to avoid branch trampolines */
	108	+ if (MODULES_VADDR < PAGE_OFFSET && MODULES_END > limit)
	109	+ ptr = __module_alloc(size, limit, MODULES_END);
	110	+
	111	+ if (!ptr)
	112	+ ptr = __module_alloc(size, MODULES_VADDR, MODULES_END);
	113	+
	114	+ return ptr;
98	115	}
99	116	#endif

arch/powerpc/kernel/optprobes.c+21−3

@@ -141,11 +141,21 @@ void arch_remove_optimized_kprobe(struct optimized_kprobe *op) · 140 unmodified lines
141	141	}
142	142	}
143	143
	144	+static void patch_imm32_load_insns(unsigned long val, int reg, kprobe_opcode_t *addr)
	145	+{
	146	+ patch_instruction((struct ppc_inst *)addr,
	147	+ ppc_inst(PPC_RAW_LIS(reg, IMM_H(val))));
	148	+ addr++;
	149	+
	150	+ patch_instruction((struct ppc_inst *)addr,
	151	+ ppc_inst(PPC_RAW_ORI(reg, reg, IMM_L(val))));
	152	+}
	153	+
144	154	/*
145	155	* Generate instructions to load provided immediate 64-bit value
146	156	* to register 'reg' and patch these instructions at 'addr'.
147	157	*/
148		−static void patch_imm64_load_insns(unsigned long val, int reg, kprobe_opcode_t *addr)
	158	+static void patch_imm64_load_insns(unsigned long long val, int reg, kprobe_opcode_t *addr)
149	159	{
150	160	/* lis reg,(op)@highest */
151	161	patch_instruction((struct ppc_inst *)addr,
@@ -177,6 +187,14 @@ static void patch_imm64_load_insns(unsigned long val, int reg, kprobe_opcode_t * · 25 unmodified lines
177	187	___PPC_RS(reg) \| (val & 0xffff)));
178	188	}
179	189
	190	+static void patch_imm_load_insns(unsigned long val, int reg, kprobe_opcode_t *addr)
	191	+{
	192	+ if (IS_ENABLED(CONFIG_PPC64))
	193	+ patch_imm64_load_insns(val, reg, addr);
	194	+ else
	195	+ patch_imm32_load_insns(val, reg, addr);
	196	+}
	197	+
180	198	int arch_prepare_optimized_kprobe(struct optimized_kprobe op, struct kprobe p)
181	199	{
182	200	struct ppc_inst branch_op_callback, branch_emulate_step, temp;
@@ -230,7 +248,7 @@ int arch_prepare_optimized_kprobe(struct optimized_kprobe op, struct kprobe p) · 47 unmodified lines
230	248	* Fixup the template with instructions to:
231	249	* 1. load the address of the actual probepoint
232	250	*/
233		− patch_imm64_load_insns((unsigned long)op, 3, buff + TMPL_OP_IDX);
	251	+ patch_imm_load_insns((unsigned long)op, 3, buff + TMPL_OP_IDX);
234	252
235	253	/*
236	254	* 2. branch to optimized_callback() and emulate_step()
@@ -264,7 +282,7 @@ int arch_prepare_optimized_kprobe(struct optimized_kprobe op, struct kprobe p) · 27 unmodified lines
264	282	* 3. load instruction to be emulated into relevant register, and
265	283	*/
266	284	temp = ppc_inst_read((struct ppc_inst *)p->ainsn.insn);
267		− patch_imm64_load_insns(ppc_inst_as_u64(temp), 4, buff + TMPL_INSN_IDX);
	285	+ patch_imm_load_insns(ppc_inst_as_ulong(temp), 4, buff + TMPL_INSN_IDX);
268	286
269	287	/*
270	288	* 4. branch back from trampoline

arch/powerpc/kernel/optprobes_head.S+34−31

@@ -9,6 +9,16 @@ · 8 unmodified lines
9	9	#include <asm/ptrace.h>
10	10	#include <asm/asm-offsets.h>
11	11
	12	+#ifdef CONFIG_PPC64
	13	+#define SAVE_30GPRS(base) SAVE_10GPRS(2,base); SAVE_10GPRS(12,base); SAVE_10GPRS(22,base)
	14	+#define REST_30GPRS(base) REST_10GPRS(2,base); REST_10GPRS(12,base); REST_10GPRS(22,base)
	15	+#define TEMPLATE_FOR_IMM_LOAD_INSNS nop; nop; nop; nop; nop
	16	+#else
	17	+#define SAVE_30GPRS(base) stmw r2, GPR2(base)
	18	+#define REST_30GPRS(base) lmw r2, GPR2(base)
	19	+#define TEMPLATE_FOR_IMM_LOAD_INSNS nop; nop; nop
	20	+#endif
	21	+
12	22	#define OPT_SLOT_SIZE 65536
13	23
14	24	.balign 4
@@ -30,39 +40,41 @@ optinsn_slot: · 15 unmodified lines
30	40	.global optprobe_template_entry
31	41	optprobe_template_entry:
32	42	/* Create an in-memory pt_regs */
33		− stdu r1,-INT_FRAME_SIZE(r1)
	43	+ PPC_STLU r1,-INT_FRAME_SIZE(r1)
34	44	SAVE_GPR(0,r1)
35	45	/* Save the previous SP into stack */
36	46	addi r0,r1,INT_FRAME_SIZE
37		− std r0,GPR1(r1)
38		− SAVE_10GPRS(2,r1)
39		− SAVE_10GPRS(12,r1)
40		− SAVE_10GPRS(22,r1)
	47	+ PPC_STL r0,GPR1(r1)
	48	+ SAVE_30GPRS(r1)
41	49	/* Save SPRS */
42	50	mfmsr r5
43		− std r5,_MSR(r1)
	51	+ PPC_STL r5,_MSR(r1)
44	52	li r5,0x700
45		− std r5,_TRAP(r1)
	53	+ PPC_STL r5,_TRAP(r1)
46	54	li r5,0
47		− std r5,ORIG_GPR3(r1)
48		− std r5,RESULT(r1)
	55	+ PPC_STL r5,ORIG_GPR3(r1)
	56	+ PPC_STL r5,RESULT(r1)
49	57	mfctr r5
50		− std r5,_CTR(r1)
	58	+ PPC_STL r5,_CTR(r1)
51	59	mflr r5
52		− std r5,_LINK(r1)
	60	+ PPC_STL r5,_LINK(r1)
53	61	mfspr r5,SPRN_XER
54		− std r5,_XER(r1)
	62	+ PPC_STL r5,_XER(r1)
55	63	mfcr r5
56		− std r5,_CCR(r1)
	64	+ PPC_STL r5,_CCR(r1)
	65	+#ifdef CONFIG_PPC64
57	66	lbz r5,PACAIRQSOFTMASK(r13)
58	67	std r5,SOFTE(r1)
	68	+#endif
59	69
60	70	/*
61	71	* We may get here from a module, so load the kernel TOC in r2.
62	72	* The original TOC gets restored when pt_regs is restored
63	73	* further below.
64	74	*/
	75	+#ifdef CONFIG_PPC64
65	76	ld r2,PACATOC(r13)
	77	+#endif
66	78
67	79	.global optprobe_template_op_address
68	80	optprobe_template_op_address:
@@ -70,11 +82,8 @@ optprobe_template_op_address: · 1 unmodified line
70	82	* Parameters to optimized_callback():
71	83	* 1. optimized_kprobe structure in r3
72	84	*/
73		− nop
74		− nop
75		− nop
76		− nop
77		− nop
	85	+ TEMPLATE_FOR_IMM_LOAD_INSNS
	86	+
78	87	/* 2. pt_regs pointer in r4 */
79	88	addi r4,r1,STACK_FRAME_OVERHEAD
80	89
@@ -92,11 +101,7 @@ optprobe_template_call_handler: · 11 unmodified lines
92	101	.global optprobe_template_insn
93	102	optprobe_template_insn:
94	103	/* 2, Pass instruction to be emulated in r4 */
95		− nop
96		− nop
97		− nop
98		− nop
99		− nop
	104	+ TEMPLATE_FOR_IMM_LOAD_INSNS
100	105
101	106	.global optprobe_template_call_emulate
102	107	optprobe_template_call_emulate:
@@ -107,20 +112,18 @@ optprobe_template_call_emulate: · 4 unmodified lines
107	112	* All done.
108	113	* Now, restore the registers...
109	114	*/
110		− ld r5,_MSR(r1)
	115	+ PPC_LL r5,_MSR(r1)
111	116	mtmsr r5
112		− ld r5,_CTR(r1)
	117	+ PPC_LL r5,_CTR(r1)
113	118	mtctr r5
114		− ld r5,_LINK(r1)
	119	+ PPC_LL r5,_LINK(r1)
115	120	mtlr r5
116		− ld r5,_XER(r1)
	121	+ PPC_LL r5,_XER(r1)
117	122	mtxer r5
118		− ld r5,_CCR(r1)
	123	+ PPC_LL r5,_CCR(r1)
119	124	mtcr r5
120	125	REST_GPR(0,r1)
121		− REST_10GPRS(2,r1)
122		− REST_10GPRS(12,r1)
123		− REST_10GPRS(22,r1)
	126	+ REST_30GPRS(r1)
124	127	/* Restore the previous SP */
125	128	addi r1,r1,INT_FRAME_SIZE
126	129

arch/powerpc/kernel/process.c+8−17

@@ -1117,9 +1117,10 @@ void restore_tm_state(struct pt_regs *regs) · 1116 unmodified lines
1117	1117	regs->msr \|= msr_diff;
1118	1118	}
1119	1119
1120		−#else
	1120	+#else /* !CONFIG_PPC_TRANSACTIONAL_MEM */
1121	1121	#define tm_recheckpoint_new_task(new)
1122	1122	#define __switch_to_tm(prev, new)
	1123	+void tm_reclaim_current(uint8_t cause) {}
1123	1124	#endif /* CONFIG_PPC_TRANSACTIONAL_MEM */
1124	1125
1125	1126	static inline void save_sprs(struct thread_struct *t)
@@ -1255,6 +1256,9 @@ struct task_struct __switch_to(struct task_struct prev, · 129 unmodified lines
1255	1256	*/
1256	1257	restore_sprs(old_thread, new_thread);
1257	1258
	1259	+#ifdef CONFIG_PPC32
	1260	+ kuap_assert_locked();
	1261	+#endif
1258	1262	last = _switch(old_thread, new_thread);
1259	1263
1260	1264	#ifdef CONFIG_PPC_BOOK3S_64
@@ -1444,11 +1448,9 @@ static void print_msr_bits(unsigned long val) · 183 unmodified lines
1444	1448	#ifdef CONFIG_PPC64
1445	1449	#define REG "%016lx"
1446	1450	#define REGS_PER_LINE 4
1447		−#define LAST_VOLATILE 13
1448	1451	#else
1449	1452	#define REG "%08lx"
1450	1453	#define REGS_PER_LINE 8
1451		−#define LAST_VOLATILE 12
1452	1454	#endif
1453	1455
1454	1456	static void __show_regs(struct pt_regs *regs)
@@ -1465,7 +1467,9 @@ static void __show_regs(struct pt_regs *regs) · 10 unmodified lines
1465	1467	trap = TRAP(regs);
1466	1468	if (!trap_is_syscall(regs) && cpu_has_feature(CPU_FTR_CFAR))
1467	1469	pr_cont("CFAR: "REG" ", regs->orig_gpr3);
1468		− if (trap == 0x200 \|\| trap == 0x300 \|\| trap == 0x600) {
	1470	+ if (trap == INTERRUPT_MACHINE_CHECK \|\|
	1471	+ trap == INTERRUPT_DATA_STORAGE \|\|
	1472	+ trap == INTERRUPT_ALIGNMENT) {
1469	1473	if (IS_ENABLED(CONFIG_4xx) \|\| IS_ENABLED(CONFIG_BOOKE))
1470	1474	pr_cont("DEAR: "REG" ESR: "REG" ", regs->dar, regs->dsisr);
1471	1475	else
@@ -1484,8 +1488,6 @@ static void __show_regs(struct pt_regs *regs) · 12 unmodified lines
1484	1488	if ((i % REGS_PER_LINE) == 0)
1485	1489	pr_cont("\nGPR%02d: ", i);
1486	1490	pr_cont(REG " ", regs->gpr[i]);
1487		− if (i == LAST_VOLATILE && !FULL_REGS(regs))
1488		− break;
1489	1491	}
1490	1492	pr_cont("\n");
1491	1493	/*
@@ -1688,7 +1690,6 @@ int copy_thread(unsigned long clone_flags, unsigned long usp, · 196 unmodified lines
1688	1690	} else {
1689	1691	/* user thread */
1690	1692	struct pt_regs *regs = current_pt_regs();
1691		− CHECK_FULL_REGS(regs);
1692	1693	childregs = regs;
1693	1694	if (usp)
1694	1695	childregs->gpr[1] = usp;
@@ -1724,9 +1725,6 @@ int copy_thread(unsigned long clone_flags, unsigned long usp, · 29 unmodified lines
1724	1725	kregs = (struct pt_regs *) sp;
1725	1726	sp -= STACK_FRAME_OVERHEAD;
1726	1727	p->thread.ksp = sp;
1727		−#ifdef CONFIG_PPC32
1728		− p->thread.ksp_limit = (unsigned long)end_of_stack(p);
1729		−#endif
1730	1728	#ifdef CONFIG_HAVE_HW_BREAKPOINT
1731	1729	for (i = 0; i < nr_wp_slots(); i++)
1732	1730	p->thread.ptrace_bps[i] = NULL;
@@ -1796,13 +1794,6 @@ void start_thread(struct pt_regs *regs, unsigned long start, unsigned long sp) · 63 unmodified lines
1796	1794	regs->ccr = 0;
1797	1795	regs->gpr[1] = sp;
1798	1796
1799		− /*
1800		− * We have just cleared all the nonvolatile GPRs, so make
1801		− * FULL_REGS(regs) return true. This is necessary to allow
1802		− * ptrace to examine the thread immediately after exec.
1803		− */
1804		− SET_FULL_REGS(regs);
1805		−
1806	1797	#ifdef CONFIG_PPC32
1807	1798	regs->mq = 0;
1808	1799	regs->nip = start;

arch/powerpc/kernel/prom.c+16−5

@@ -65,6 +65,8 @@ · 64 unmodified lines
65	65	#define DBG(fmt...)
66	66	#endif
67	67
	68	+int *chip_id_lookup_table;
	69	+
68	70	#ifdef CONFIG_PPC64
69	71	int __initdata iommu_is_off;
70	72	int __initdata iommu_force_on;
@@ -267,7 +269,7 @@ static struct feature_property { · 196 unmodified lines
267	269	};
268	270
269	271	#if defined(CONFIG_44x) && defined(CONFIG_PPC_FPU)
270		−static inline void identical_pvr_fixup(unsigned long node)
	272	+static __init void identical_pvr_fixup(unsigned long node)
271	273	{
272	274	unsigned int pvr;
273	275	const char *model = of_get_flat_dt_prop(node, "model", NULL);
@@ -914,13 +916,22 @@ EXPORT_SYMBOL(of_get_ibm_chip_id); · 640 unmodified lines
914	916	int cpu_to_chip_id(int cpu)
915	917	{
916	918	struct device_node *np;
	919	+ int ret = -1, idx;
	920	+
	921	+ idx = cpu / threads_per_core;
	922	+ if (chip_id_lookup_table && chip_id_lookup_table[idx] != -1)
	923	+ return chip_id_lookup_table[idx];
917	924
918	925	np = of_get_cpu_node(cpu, NULL);
919		− if (!np)
920		− return -1;
	926	+ if (np) {
	927	+ ret = of_get_ibm_chip_id(np);
	928	+ of_node_put(np);
	929	+
	930	+ if (chip_id_lookup_table)
	931	+ chip_id_lookup_table[idx] = ret;
	932	+ }
921	933
922		− of_node_put(np);
923		− return of_get_ibm_chip_id(np);
	934	+ return ret;
924	935	}
925	936	EXPORT_SYMBOL(cpu_to_chip_id);
926	937

arch/powerpc/kernel/prom_init.c+1−1

@@ -2983,7 +2983,7 @@ static void __init fixup_device_tree_efika_add_phy(void) · 2982 unmodified lines
2983	2983	" 0x3 encode-int encode+"
2984	2984	" s\" interrupts\" property"
2985	2985	" finish-device");
2986		− };
	2986	+ }
2987	2987
2988	2988	/* Check for a PHY device node - if missing then create one and
2989	2989	* give it's phandle to the ethernet node */

arch/powerpc/kernel/ptrace/ptrace-view.c+18−33

@@ -111,7 +111,7 @@ static unsigned long get_user_msr(struct task_struct *task) · 110 unmodified lines
111	111	return task->thread.regs->msr \| task->thread.fpexc_mode;
112	112	}
113	113
114		−static int set_user_msr(struct task_struct *task, unsigned long msr)
	114	+static __always_inline int set_user_msr(struct task_struct *task, unsigned long msr)
115	115	{
116	116	task->thread.regs->msr &= ~MSR_DEBUGCHANGE;
117	117	task->thread.regs->msr \|= msr & MSR_DEBUGCHANGE;
@@ -147,7 +147,7 @@ static int set_user_dscr(struct task_struct *task, unsigned long dscr) · 29 unmodified lines
147	147	* We prevent mucking around with the reserved area of trap
148	148	* which are used internally by the kernel.
149	149	*/
150		−static int set_user_trap(struct task_struct *task, unsigned long trap)
	150	+static __always_inline int set_user_trap(struct task_struct *task, unsigned long trap)
151	151	{
152	152	set_trap(task->thread.regs, trap);
153	153	return 0;
@@ -221,17 +221,9 @@ static int gpr_get(struct task_struct target, const struct user_regset regset, · 67 unmodified lines
221	221	#ifdef CONFIG_PPC64
222	222	struct membuf to_softe = membuf_at(&to, offsetof(struct pt_regs, softe));
223	223	#endif
224		− int i;
225		−
226	224	if (target->thread.regs == NULL)
227	225	return -EIO;
228	226
229		− if (!FULL_REGS(target->thread.regs)) {
230		− /* We have a partial register set. Fill 14-31 with bogus values */
231		− for (i = 14; i < 32; i++)
232		− target->thread.regs->gpr[i] = NV_REG_POISON;
233		− }
234		−
235	227	membuf_write(&to, target->thread.regs, sizeof(struct user_pt_regs));
236	228
237	229	membuf_store(&to_msr, get_user_msr(target));
@@ -252,8 +244,6 @@ static int gpr_set(struct task_struct target, const struct user_regset regset, · 14 unmodified lines
252	244	if (target->thread.regs == NULL)
253	245	return -EIO;
254	246
255		− CHECK_FULL_REGS(target->thread.regs);
256		−
257	247	ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf,
258	248	target->thread.regs,
259	249	0, PT_MSR * sizeof(reg));
@@ -659,6 +649,9 @@ int gpr32_set_common(struct task_struct *target, · 399 unmodified lines
659	649	const compat_ulong_t __user *u = ubuf;
660	650	compat_ulong_t reg;
661	651
	652	+ if (!kbuf && !user_read_access_begin(u, count))
	653	+ return -EFAULT;
	654	+
662	655	pos /= sizeof(reg);
663	656	count /= sizeof(reg);
664	657
@@ -667,8 +660,7 @@ int gpr32_set_common(struct task_struct *target, · 2 unmodified lines
667	660	regs[pos++] = *k++;
668	661	else
669	662	for (; count > 0 && pos < PT_MSR; --count) {
670		− if (__get_user(reg, u++))
671		− return -EFAULT;
	663	+ unsafe_get_user(reg, u++, Efault);
672	664	regs[pos++] = reg;
673	665	}
674	666
@@ -676,8 +668,8 @@ int gpr32_set_common(struct task_struct *target, · 1 unmodified line
676	668	if (count > 0 && pos == PT_MSR) {
677	669	if (kbuf)
678	670	reg = *k++;
679		− else if (__get_user(reg, u++))
680		− return -EFAULT;
	671	+ else
	672	+ unsafe_get_user(reg, u++, Efault);
681	673	set_user_msr(target, reg);
682	674	++pos;
683	675	--count;
@@ -690,24 +682,24 @@ int gpr32_set_common(struct task_struct *target, · 6 unmodified lines
690	682	++k;
691	683	} else {
692	684	for (; count > 0 && pos <= PT_MAX_PUT_REG; --count) {
693		− if (__get_user(reg, u++))
694		− return -EFAULT;
	685	+ unsafe_get_user(reg, u++, Efault);
695	686	regs[pos++] = reg;
696	687	}
697	688	for (; count > 0 && pos < PT_TRAP; --count, ++pos)
698		− if (__get_user(reg, u++))
699		− return -EFAULT;
	689	+ unsafe_get_user(reg, u++, Efault);
700	690	}
701	691
702	692	if (count > 0 && pos == PT_TRAP) {
703	693	if (kbuf)
704	694	reg = *k++;
705		− else if (__get_user(reg, u++))
706		− return -EFAULT;
	695	+ else
	696	+ unsafe_get_user(reg, u++, Efault);
707	697	set_user_trap(target, reg);
708	698	++pos;
709	699	--count;
710	700	}
	701	+ if (!kbuf)
	702	+ user_read_access_end();
711	703
712	704	kbuf = k;
713	705	ubuf = u;
@@ -715,25 +707,19 @@ int gpr32_set_common(struct task_struct *target, · 1 unmodified line
715	707	count *= sizeof(reg);
716	708	return user_regset_copyin_ignore(&pos, &count, &kbuf, &ubuf,
717	709	(PT_TRAP + 1) * sizeof(reg), -1);
	710	+
	711	+Efault:
	712	+ user_read_access_end();
	713	+ return -EFAULT;
718	714	}
719	715
720	716	static int gpr32_get(struct task_struct *target,
721	717	const struct user_regset *regset,
722	718	struct membuf to)
723	719	{
724		− int i;
725		−
726	720	if (target->thread.regs == NULL)
727	721	return -EIO;
728	722
729		− if (!FULL_REGS(target->thread.regs)) {
730		− /*
731		− * We have a partial register set.
732		− * Fill 14-31 with bogus values.
733		− */
734		− for (i = 14; i < 32; i++)
735		− target->thread.regs->gpr[i] = NV_REG_POISON;
736		− }
737	723	return gpr32_get_common(target, regset, to,
738	724	&target->thread.regs->gpr[0]);
739	725	}
@@ -746,7 +732,6 @@ static int gpr32_set(struct task_struct *target, · 6 unmodified lines
746	732	if (target->thread.regs == NULL)
747	733	return -EIO;
748	734
749		− CHECK_FULL_REGS(target->thread.regs);
750	735	return gpr32_set_common(target, regset, pos, count, kbuf, ubuf,
751	736	&target->thread.regs->gpr[0]);
752	737	}

arch/powerpc/kernel/ptrace/ptrace.c+0−4

@@ -59,7 +59,6 @@ long arch_ptrace(struct task_struct *child, long request, · 58 unmodified lines
59	59	if ((addr & (sizeof(long) - 1)) \|\| !child->thread.regs)
60	60	break;
61	61
62		− CHECK_FULL_REGS(child->thread.regs);
63	62	if (index < PT_FPR0)
64	63	ret = ptrace_get_reg(child, (int) index, &tmp);
65	64	else
@@ -81,7 +80,6 @@ long arch_ptrace(struct task_struct *child, long request, · 15 unmodified lines
81	80	if ((addr & (sizeof(long) - 1)) \|\| !child->thread.regs)
82	81	break;
83	82
84		− CHECK_FULL_REGS(child->thread.regs);
85	83	if (index < PT_FPR0)
86	84	ret = ptrace_put_reg(child, index, data);
87	85	else
@@ -354,8 +352,6 @@ void __init pt_regs_check(void) · 266 unmodified lines
354	352	offsetof(struct user_pt_regs, nip));
355	353	BUILD_BUG_ON(offsetof(struct pt_regs, msr) !=
356	354	offsetof(struct user_pt_regs, msr));
357		− BUILD_BUG_ON(offsetof(struct pt_regs, msr) !=
358		− offsetof(struct user_pt_regs, msr));
359	355	BUILD_BUG_ON(offsetof(struct pt_regs, orig_gpr3) !=
360	356	offsetof(struct user_pt_regs, orig_gpr3));
361	357	BUILD_BUG_ON(offsetof(struct pt_regs, ctr) !=

arch/powerpc/kernel/ptrace/ptrace32.c+0−4

@@ -83,7 +83,6 @@ long compat_arch_ptrace(struct task_struct *child, compat_long_t request, · 82 unmodified lines
83	83	if ((addr & 3) \|\| (index > PT_FPSCR32))
84	84	break;
85	85
86		− CHECK_FULL_REGS(child->thread.regs);
87	86	if (index < PT_FPR0) {
88	87	ret = ptrace_get_reg(child, index, &tmp);
89	88	if (ret)
@@ -133,7 +132,6 @@ long compat_arch_ptrace(struct task_struct *child, compat_long_t request, · 43 unmodified lines
133	132	if ((addr & 3) \|\| numReg > PT_FPSCR)
134	133	break;
135	134
136		− CHECK_FULL_REGS(child->thread.regs);
137	135	if (numReg >= PT_FPR0) {
138	136	flush_fp_to_thread(child);
139	137	/* get 64 bit FPR */
@@ -187,7 +185,6 @@ long compat_arch_ptrace(struct task_struct *child, compat_long_t request, · 47 unmodified lines
187	185	if ((addr & 3) \|\| (index > PT_FPSCR32))
188	186	break;
189	187
190		− CHECK_FULL_REGS(child->thread.regs);
191	188	if (index < PT_FPR0) {
192	189	ret = ptrace_put_reg(child, index, data);
193	190	} else {
@@ -226,7 +223,6 @@ long compat_arch_ptrace(struct task_struct *child, compat_long_t request, · 32 unmodified lines
226	223	*/
227	224	if ((addr & 3) \|\| (numReg > PT_FPSCR))
228	225	break;
229		− CHECK_FULL_REGS(child->thread.regs);
230	226	if (numReg < PT_FPR0) {
231	227	unsigned long freg;
232	228	ret = ptrace_get_reg(child, numReg, &freg);

arch/powerpc/kernel/rtas-proc.c+11−4

@@ -755,11 +755,18 @@ static int ppc_rtas_tone_volume_show(struct seq_file m, void v) · 754 unmodified lines
755	755	return 0;
756	756	}
757	757
758		−#define RMO_READ_BUF_MAX 30
759		−
760		−/* RTAS Userspace access */
	758	+/**
	759	+ * ppc_rtas_rmo_buf_show() - Describe RTAS-addressable region for user space.
	760	+ *
	761	+ * Base + size description of a range of RTAS-addressable memory set
	762	+ * aside for user space to use as work area(s) for certain RTAS
	763	+ * functions. User space accesses this region via /dev/mem. Apart from
	764	+ * security policies, the kernel does not arbitrate or serialize
	765	+ * access to this region, and user space must ensure that concurrent
	766	+ * users do not interfere with each other.
	767	+ */
761	768	static int ppc_rtas_rmo_buf_show(struct seq_file m, void v)
762	769	{
763		− seq_printf(m, "%016lx %x\n", rtas_rmo_buf, RTAS_RMOBUF_MAX);
	770	+ seq_printf(m, "%016lx %x\n", rtas_rmo_buf, RTAS_USER_REGION_SIZE);
764	771	return 0;
765	772	}

arch/powerpc/kernel/rtas.c+18−16

@@ -828,7 +828,6 @@ void rtas_activate_firmware(void) · 827 unmodified lines
828	828	pr_err("ibm,activate-firmware failed (%i)\n", fwrc);
829	829	}
830	830
831		−static int ibm_suspend_me_token = RTAS_UNKNOWN_SERVICE;
832	831	#ifdef CONFIG_PPC_PSERIES
833	832	/**
834	833	* rtas_call_reentrant() - Used for reentrant rtas calls
@@ -988,10 +987,10 @@ static struct rtas_filter rtas_filters[] __ro_after_init = { · 153 unmodified lines
988	987	static bool in_rmo_buf(u32 base, u32 end)
989	988	{
990	989	return base >= rtas_rmo_buf &&
991		− base < (rtas_rmo_buf + RTAS_RMOBUF_MAX) &&
	990	+ base < (rtas_rmo_buf + RTAS_USER_REGION_SIZE) &&
992	991	base <= end &&
993	992	end >= rtas_rmo_buf &&
994		− end < (rtas_rmo_buf + RTAS_RMOBUF_MAX);
	993	+ end < (rtas_rmo_buf + RTAS_USER_REGION_SIZE);
995	994	}
996	995
997	996	static bool block_rtas_call(int token, int nargs,
@@ -1052,6 +1051,14 @@ err: · 54 unmodified lines
1052	1051	return true;
1053	1052	}
1054	1053
	1054	+static void __init rtas_syscall_filter_init(void)
	1055	+{
	1056	+ unsigned int i;
	1057	+
	1058	+ for (i = 0; i < ARRAY_SIZE(rtas_filters); i++)
	1059	+ rtas_filters[i].token = rtas_token(rtas_filters[i].name);
	1060	+}
	1061	+
1055	1062	#else
1056	1063
1057	1064	static bool block_rtas_call(int token, int nargs,
@@ -1060,6 +1067,10 @@ static bool block_rtas_call(int token, int nargs, · 2 unmodified lines
1060	1067	return false;
1061	1068	}
1062	1069
	1070	+static void __init rtas_syscall_filter_init(void)
	1071	+{
	1072	+}
	1073	+
1063	1074	#endif /* CONFIG_PPC_RTAS_FILTER */
1064	1075
1065	1076	/* We assume to be passed big endian arguments */
@@ -1103,7 +1114,7 @@ SYSCALL_DEFINE1(rtas, struct rtas_args __user *, uargs) · 37 unmodified lines
1103	1114	return -EINVAL;
1104	1115
1105	1116	/* Need to handle ibm,suspend_me call specially */
1106		− if (token == ibm_suspend_me_token) {
	1117	+ if (token == rtas_token("ibm,suspend-me")) {
1107	1118
1108	1119	/*
1109	1120	* rtas_ibm_suspend_me assumes the streamid handle is in cpu
@@ -1163,9 +1174,6 @@ void __init rtas_initialize(void) · 53 unmodified lines
1163	1174	unsigned long rtas_region = RTAS_INSTANTIATE_MAX;
1164	1175	u32 base, size, entry;
1165	1176	int no_base, no_size, no_entry;
1166		−#ifdef CONFIG_PPC_RTAS_FILTER
1167		− int i;
1168		−#endif
1169	1177
1170	1178	/* Get RTAS dev node and fill up our "rtas" structure with infos
1171	1179	* about it.
@@ -1191,12 +1199,10 @@ void __init rtas_initialize(void) · 19 unmodified lines
1191	1199	* the stop-self token if any
1192	1200	*/
1193	1201	#ifdef CONFIG_PPC64
1194		− if (firmware_has_feature(FW_FEATURE_LPAR)) {
	1202	+ if (firmware_has_feature(FW_FEATURE_LPAR))
1195	1203	rtas_region = min(ppc64_rma_size, RTAS_INSTANTIATE_MAX);
1196		− ibm_suspend_me_token = rtas_token("ibm,suspend-me");
1197		− }
1198	1204	#endif
1199		− rtas_rmo_buf = memblock_phys_alloc_range(RTAS_RMOBUF_MAX, PAGE_SIZE,
	1205	+ rtas_rmo_buf = memblock_phys_alloc_range(RTAS_USER_REGION_SIZE, PAGE_SIZE,
1200	1206	0, rtas_region);
1201	1207	if (!rtas_rmo_buf)
1202	1208	panic("ERROR: RTAS: Failed to allocate %lx bytes below %pa\n",
@@ -1206,11 +1212,7 @@ void __init rtas_initialize(void) · 3 unmodified lines
1206	1212	rtas_last_error_token = rtas_token("rtas-last-error");
1207	1213	#endif
1208	1214
1209		−#ifdef CONFIG_PPC_RTAS_FILTER
1210		− for (i = 0; i < ARRAY_SIZE(rtas_filters); i++) {
1211		− rtas_filters[i].token = rtas_token(rtas_filters[i].name);
1212		− }
1213		−#endif
	1215	+ rtas_syscall_filter_init();
1214	1216	}
1215	1217
1216	1218	int __init early_init_dt_scan_rtas(unsigned long node,

arch/powerpc/kernel/security.c+262−1

@@ -7,6 +7,7 @@ · 6 unmodified lines
7	7	#include <linux/cpu.h>
8	8	#include <linux/kernel.h>
9	9	#include <linux/device.h>
	10	+#include <linux/memblock.h>
10	11	#include <linux/nospec.h>
11	12	#include <linux/prctl.h>
12	13	#include <linux/seq_buf.h>
@@ -18,6 +19,7 @@ · 5 unmodified lines
18	19	#include <asm/setup.h>
19	20	#include <asm/inst.h>
20	21
	22	+#include "setup.h"
21	23
22	24	u64 powerpc_security_features __read_mostly = SEC_FTR_DEFAULT;
23	25
@@ -250,7 +252,7 @@ ssize_t cpu_show_spectre_v2(struct device dev, struct device_attribute attr, c · 226 unmodified lines
250	252
251	253	static enum stf_barrier_type stf_enabled_flush_types;
252	254	static bool no_stf_barrier;
253		−bool stf_barrier;
	255	+static bool stf_barrier;
254	256
255	257	static int __init handle_no_stf_barrier(char *p)
256	258	{
@@ -541,6 +543,178 @@ void setup_count_cache_flush(void) · 284 unmodified lines
541	543	toggle_branch_cache_flush(enable);
542	544	}
543	545
	546	+static enum l1d_flush_type enabled_flush_types;
	547	+static void *l1d_flush_fallback_area;
	548	+static bool no_rfi_flush;
	549	+static bool no_entry_flush;
	550	+static bool no_uaccess_flush;
	551	+bool rfi_flush;
	552	+static bool entry_flush;
	553	+static bool uaccess_flush;
	554	+DEFINE_STATIC_KEY_FALSE(uaccess_flush_key);
	555	+EXPORT_SYMBOL(uaccess_flush_key);
	556	+
	557	+static int __init handle_no_rfi_flush(char *p)
	558	+{
	559	+ pr_info("rfi-flush: disabled on command line.");
	560	+ no_rfi_flush = true;
	561	+ return 0;
	562	+}
	563	+early_param("no_rfi_flush", handle_no_rfi_flush);
	564	+
	565	+static int __init handle_no_entry_flush(char *p)
	566	+{
	567	+ pr_info("entry-flush: disabled on command line.");
	568	+ no_entry_flush = true;
	569	+ return 0;
	570	+}
	571	+early_param("no_entry_flush", handle_no_entry_flush);
	572	+
	573	+static int __init handle_no_uaccess_flush(char *p)
	574	+{
	575	+ pr_info("uaccess-flush: disabled on command line.");
	576	+ no_uaccess_flush = true;
	577	+ return 0;
	578	+}
	579	+early_param("no_uaccess_flush", handle_no_uaccess_flush);
	580	+
	581	+/*
	582	+ * The RFI flush is not KPTI, but because users will see doco that says to use
	583	+ * nopti we hijack that option here to also disable the RFI flush.
	584	+ */
	585	+static int __init handle_no_pti(char *p)
	586	+{
	587	+ pr_info("rfi-flush: disabling due to 'nopti' on command line.\n");
	588	+ handle_no_rfi_flush(NULL);
	589	+ return 0;
	590	+}
	591	+early_param("nopti", handle_no_pti);
	592	+
	593	+static void do_nothing(void *unused)
	594	+{
	595	+ /*
	596	+ * We don't need to do the flush explicitly, just enter+exit kernel is
	597	+ * sufficient, the RFI exit handlers will do the right thing.
	598	+ */
	599	+}
	600	+
	601	+void rfi_flush_enable(bool enable)
	602	+{
	603	+ if (enable) {
	604	+ do_rfi_flush_fixups(enabled_flush_types);
	605	+ on_each_cpu(do_nothing, NULL, 1);
	606	+ } else
	607	+ do_rfi_flush_fixups(L1D_FLUSH_NONE);
	608	+
	609	+ rfi_flush = enable;
	610	+}
	611	+
	612	+static void entry_flush_enable(bool enable)
	613	+{
	614	+ if (enable) {
	615	+ do_entry_flush_fixups(enabled_flush_types);
	616	+ on_each_cpu(do_nothing, NULL, 1);
	617	+ } else {
	618	+ do_entry_flush_fixups(L1D_FLUSH_NONE);
	619	+ }
	620	+
	621	+ entry_flush = enable;
	622	+}
	623	+
	624	+static void uaccess_flush_enable(bool enable)
	625	+{
	626	+ if (enable) {
	627	+ do_uaccess_flush_fixups(enabled_flush_types);
	628	+ static_branch_enable(&uaccess_flush_key);
	629	+ on_each_cpu(do_nothing, NULL, 1);
	630	+ } else {
	631	+ static_branch_disable(&uaccess_flush_key);
	632	+ do_uaccess_flush_fixups(L1D_FLUSH_NONE);
	633	+ }
	634	+
	635	+ uaccess_flush = enable;
	636	+}
	637	+
	638	+static void __ref init_fallback_flush(void)
	639	+{
	640	+ u64 l1d_size, limit;
	641	+ int cpu;
	642	+
	643	+ /* Only allocate the fallback flush area once (at boot time). */
	644	+ if (l1d_flush_fallback_area)
	645	+ return;
	646	+
	647	+ l1d_size = ppc64_caches.l1d.size;
	648	+
	649	+ /*
	650	+ * If there is no d-cache-size property in the device tree, l1d_size
	651	+ * could be zero. That leads to the loop in the asm wrapping around to
	652	+ * 2^64-1, and then walking off the end of the fallback area and
	653	+ * eventually causing a page fault which is fatal. Just default to
	654	+ * something vaguely sane.
	655	+ */
	656	+ if (!l1d_size)
	657	+ l1d_size = (64 * 1024);
	658	+
	659	+ limit = min(ppc64_bolted_size(), ppc64_rma_size);
	660	+
	661	+ /*
	662	+ * Align to L1d size, and size it at 2x L1d size, to catch possible
	663	+ * hardware prefetch runoff. We don't have a recipe for load patterns to
	664	+ * reliably avoid the prefetcher.
	665	+ */
	666	+ l1d_flush_fallback_area = memblock_alloc_try_nid(l1d_size * 2,
	667	+ l1d_size, MEMBLOCK_LOW_LIMIT,
	668	+ limit, NUMA_NO_NODE);
	669	+ if (!l1d_flush_fallback_area)
	670	+ panic("%s: Failed to allocate %llu bytes align=0x%llx max_addr=%pa\n",
	671	+ __func__, l1d_size * 2, l1d_size, &limit);
	672	+
	673	+
	674	+ for_each_possible_cpu(cpu) {
	675	+ struct paca_struct *paca = paca_ptrs[cpu];
	676	+ paca->rfi_flush_fallback_area = l1d_flush_fallback_area;
	677	+ paca->l1d_flush_size = l1d_size;
	678	+ }
	679	+}
	680	+
	681	+void setup_rfi_flush(enum l1d_flush_type types, bool enable)
	682	+{
	683	+ if (types & L1D_FLUSH_FALLBACK) {
	684	+ pr_info("rfi-flush: fallback displacement flush available\n");
	685	+ init_fallback_flush();
	686	+ }
	687	+
	688	+ if (types & L1D_FLUSH_ORI)
	689	+ pr_info("rfi-flush: ori type flush available\n");
	690	+
	691	+ if (types & L1D_FLUSH_MTTRIG)
	692	+ pr_info("rfi-flush: mttrig type flush available\n");
	693	+
	694	+ enabled_flush_types = types;
	695	+
	696	+ if (!cpu_mitigations_off() && !no_rfi_flush)
	697	+ rfi_flush_enable(enable);
	698	+}
	699	+
	700	+void setup_entry_flush(bool enable)
	701	+{
	702	+ if (cpu_mitigations_off())
	703	+ return;
	704	+
	705	+ if (!no_entry_flush)
	706	+ entry_flush_enable(enable);
	707	+}
	708	+
	709	+void setup_uaccess_flush(bool enable)
	710	+{
	711	+ if (cpu_mitigations_off())
	712	+ return;
	713	+
	714	+ if (!no_uaccess_flush)
	715	+ uaccess_flush_enable(enable);
	716	+}
	717	+
544	718	#ifdef CONFIG_DEBUG_FS
545	719	static int count_cache_flush_set(void *data, u64 val)
546	720	{
@@ -579,5 +753,92 @@ static __init int count_cache_flush_debugfs_init(void) · 32 unmodified lines
579	753	return 0;
580	754	}
581	755	device_initcall(count_cache_flush_debugfs_init);
	756	+
	757	+static int rfi_flush_set(void *data, u64 val)
	758	+{
	759	+ bool enable;
	760	+
	761	+ if (val == 1)
	762	+ enable = true;
	763	+ else if (val == 0)
	764	+ enable = false;
	765	+ else
	766	+ return -EINVAL;
	767	+
	768	+ /* Only do anything if we're changing state */
	769	+ if (enable != rfi_flush)
	770	+ rfi_flush_enable(enable);
	771	+
	772	+ return 0;
	773	+}
	774	+
	775	+static int rfi_flush_get(void data, u64 val)
	776	+{
	777	+ *val = rfi_flush ? 1 : 0;
	778	+ return 0;
	779	+}
	780	+
	781	+DEFINE_SIMPLE_ATTRIBUTE(fops_rfi_flush, rfi_flush_get, rfi_flush_set, "%llu\n");
	782	+
	783	+static int entry_flush_set(void *data, u64 val)
	784	+{
	785	+ bool enable;
	786	+
	787	+ if (val == 1)
	788	+ enable = true;
	789	+ else if (val == 0)
	790	+ enable = false;
	791	+ else
	792	+ return -EINVAL;
	793	+
	794	+ /* Only do anything if we're changing state */
	795	+ if (enable != entry_flush)
	796	+ entry_flush_enable(enable);
	797	+
	798	+ return 0;
	799	+}
	800	+
	801	+static int entry_flush_get(void data, u64 val)
	802	+{
	803	+ *val = entry_flush ? 1 : 0;
	804	+ return 0;
	805	+}
	806	+
	807	+DEFINE_SIMPLE_ATTRIBUTE(fops_entry_flush, entry_flush_get, entry_flush_set, "%llu\n");
	808	+
	809	+static int uaccess_flush_set(void *data, u64 val)
	810	+{
	811	+ bool enable;
	812	+
	813	+ if (val == 1)
	814	+ enable = true;
	815	+ else if (val == 0)
	816	+ enable = false;
	817	+ else
	818	+ return -EINVAL;
	819	+
	820	+ /* Only do anything if we're changing state */
	821	+ if (enable != uaccess_flush)
	822	+ uaccess_flush_enable(enable);
	823	+
	824	+ return 0;
	825	+}
	826	+
	827	+static int uaccess_flush_get(void data, u64 val)
	828	+{
	829	+ *val = uaccess_flush ? 1 : 0;
	830	+ return 0;
	831	+}
	832	+
	833	+DEFINE_SIMPLE_ATTRIBUTE(fops_uaccess_flush, uaccess_flush_get, uaccess_flush_set, "%llu\n");
	834	+
	835	+static __init int rfi_flush_debugfs_init(void)
	836	+{
	837	+ debugfs_create_file("rfi_flush", 0600, powerpc_debugfs_root, NULL, &fops_rfi_flush);
	838	+ debugfs_create_file("entry_flush", 0600, powerpc_debugfs_root, NULL, &fops_entry_flush);
	839	+ debugfs_create_file("uaccess_flush", 0600, powerpc_debugfs_root, NULL, &fops_uaccess_flush);
	840	+ return 0;
	841	+}
	842	+device_initcall(rfi_flush_debugfs_init);
582	843	#endif /* CONFIG_DEBUG_FS */
583	844	#endif /* CONFIG_PPC_BOOK3S_64 */

arch/powerpc/kernel/setup-common.c+1−2

@@ -69,7 +69,6 @@ · 68 unmodified lines
69	69	#include "setup.h"
70	70
71	71	#ifdef DEBUG
72		−#include <asm/udbg.h>
73	72	#define DBG(fmt...) udbg_printf(fmt)
74	73	#else
75	74	#define DBG(fmt...)
@@ -829,7 +828,7 @@ static __init void print_system_info(void) · 753 unmodified lines
829	828	}
830	829
831	830	#ifdef CONFIG_SMP
832		−static void smp_setup_pacas(void)
	831	+static void __init smp_setup_pacas(void)
833	832	{
834	833	int cpu;
835	834

arch/powerpc/kernel/setup_32.c+1−1

@@ -164,7 +164,7 @@ void __init irqstack_early_init(void) · 163 unmodified lines
164	164	}
165	165
166	166	#ifdef CONFIG_VMAP_STACK
167		−void *emergency_ctx[NR_CPUS] __ro_after_init;
	167	+void *emergency_ctx[NR_CPUS] __ro_after_init = {[0] = &init_stack};
168	168
169	169	void __init emergency_stack_init(void)
170	170	{

arch/powerpc/kernel/setup_64.c+16−266

@@ -232,10 +232,23 @@ static void cpu_ready_for_interrupts(void) · 231 unmodified lines
232	232	* If we are not in hypervisor mode the job is done once for
233	233	* the whole partition in configure_exceptions().
234	234	*/
235		− if (cpu_has_feature(CPU_FTR_HVMODE) &&
236		− cpu_has_feature(CPU_FTR_ARCH_207S)) {
	235	+ if (cpu_has_feature(CPU_FTR_HVMODE)) {
237	236	unsigned long lpcr = mfspr(SPRN_LPCR);
238		− mtspr(SPRN_LPCR, lpcr \| LPCR_AIL_3);
	237	+ unsigned long new_lpcr = lpcr;
	238	+
	239	+ if (cpu_has_feature(CPU_FTR_ARCH_31)) {
	240	+ /* P10 DD1 does not have HAIL */
	241	+ if (pvr_version_is(PVR_POWER10) &&
	242	+ (mfspr(SPRN_PVR) & 0xf00) == 0x100)
	243	+ new_lpcr \|= LPCR_AIL_3;
	244	+ else
	245	+ new_lpcr \|= LPCR_HAIL;
	246	+ } else if (cpu_has_feature(CPU_FTR_ARCH_207S)) {
	247	+ new_lpcr \|= LPCR_AIL_3;
	248	+ }
	249	+
	250	+ if (new_lpcr != lpcr)
	251	+ mtspr(SPRN_LPCR, new_lpcr);
239	252	}
240	253
241	254	/*
@@ -941,266 +954,3 @@ static int __init disable_hardlockup_detector(void) · 699 unmodified lines
941	954	return 0;
942	955	}
943	956	early_initcall(disable_hardlockup_detector);
944		−
945		−#ifdef CONFIG_PPC_BOOK3S_64
946		−static enum l1d_flush_type enabled_flush_types;
947		−static void *l1d_flush_fallback_area;
948		−static bool no_rfi_flush;
949		−static bool no_entry_flush;
950		−static bool no_uaccess_flush;
951		−bool rfi_flush;
952		−bool entry_flush;
953		−bool uaccess_flush;
954		−DEFINE_STATIC_KEY_FALSE(uaccess_flush_key);
955		−EXPORT_SYMBOL(uaccess_flush_key);
956		−
957		−static int __init handle_no_rfi_flush(char *p)
958		−{
959		− pr_info("rfi-flush: disabled on command line.");
960		− no_rfi_flush = true;
961		− return 0;
962		−}
963		−early_param("no_rfi_flush", handle_no_rfi_flush);
964		−
965		−static int __init handle_no_entry_flush(char *p)
966		−{
967		− pr_info("entry-flush: disabled on command line.");
968		− no_entry_flush = true;
969		− return 0;
970		−}
971		−early_param("no_entry_flush", handle_no_entry_flush);
972		−
973		−static int __init handle_no_uaccess_flush(char *p)
974		−{
975		− pr_info("uaccess-flush: disabled on command line.");
976		− no_uaccess_flush = true;
977		− return 0;
978		−}
979		−early_param("no_uaccess_flush", handle_no_uaccess_flush);
980		−
981		−/*
982		− * The RFI flush is not KPTI, but because users will see doco that says to use
983		− * nopti we hijack that option here to also disable the RFI flush.
984		− */
985		−static int __init handle_no_pti(char *p)
986		−{
987		− pr_info("rfi-flush: disabling due to 'nopti' on command line.\n");
988		− handle_no_rfi_flush(NULL);
989		− return 0;
990		−}
991		−early_param("nopti", handle_no_pti);
992		−
993		−static void do_nothing(void *unused)
994		−{
995		− /*
996		− * We don't need to do the flush explicitly, just enter+exit kernel is
997		− * sufficient, the RFI exit handlers will do the right thing.
998		− */
999		−}
1000		−
1001		−void rfi_flush_enable(bool enable)
1002		−{
1003		− if (enable) {
1004		− do_rfi_flush_fixups(enabled_flush_types);
1005		− on_each_cpu(do_nothing, NULL, 1);
1006		− } else
1007		− do_rfi_flush_fixups(L1D_FLUSH_NONE);
1008		−
1009		− rfi_flush = enable;
1010		−}
1011		−
1012		−static void entry_flush_enable(bool enable)
1013		−{
1014		− if (enable) {
1015		− do_entry_flush_fixups(enabled_flush_types);
1016		− on_each_cpu(do_nothing, NULL, 1);
1017		− } else {
1018		− do_entry_flush_fixups(L1D_FLUSH_NONE);
1019		− }
1020		−
1021		− entry_flush = enable;
1022		−}
1023		−
1024		−static void uaccess_flush_enable(bool enable)
1025		−{
1026		− if (enable) {
1027		− do_uaccess_flush_fixups(enabled_flush_types);
1028		− static_branch_enable(&uaccess_flush_key);
1029		− on_each_cpu(do_nothing, NULL, 1);
1030		− } else {
1031		− static_branch_disable(&uaccess_flush_key);
1032		− do_uaccess_flush_fixups(L1D_FLUSH_NONE);
1033		− }
1034		−
1035		− uaccess_flush = enable;
1036		−}
1037		−
1038		−static void __ref init_fallback_flush(void)
1039		−{
1040		− u64 l1d_size, limit;
1041		− int cpu;
1042		−
1043		− /* Only allocate the fallback flush area once (at boot time). */
1044		− if (l1d_flush_fallback_area)
1045		− return;
1046		−
1047		− l1d_size = ppc64_caches.l1d.size;
1048		−
1049		− /*
1050		− * If there is no d-cache-size property in the device tree, l1d_size
1051		− * could be zero. That leads to the loop in the asm wrapping around to
1052		− * 2^64-1, and then walking off the end of the fallback area and
1053		− * eventually causing a page fault which is fatal. Just default to
1054		− * something vaguely sane.
1055		− */
1056		− if (!l1d_size)
1057		− l1d_size = (64 * 1024);
1058		−
1059		− limit = min(ppc64_bolted_size(), ppc64_rma_size);
1060		−
1061		− /*
1062		− * Align to L1d size, and size it at 2x L1d size, to catch possible
1063		− * hardware prefetch runoff. We don't have a recipe for load patterns to
1064		− * reliably avoid the prefetcher.
1065		− */
1066		− l1d_flush_fallback_area = memblock_alloc_try_nid(l1d_size * 2,
1067		− l1d_size, MEMBLOCK_LOW_LIMIT,
1068		− limit, NUMA_NO_NODE);
1069		− if (!l1d_flush_fallback_area)
1070		− panic("%s: Failed to allocate %llu bytes align=0x%llx max_addr=%pa\n",
1071		− __func__, l1d_size * 2, l1d_size, &limit);
1072		−
1073		−
1074		− for_each_possible_cpu(cpu) {
1075		− struct paca_struct *paca = paca_ptrs[cpu];
1076		− paca->rfi_flush_fallback_area = l1d_flush_fallback_area;
1077		− paca->l1d_flush_size = l1d_size;
1078		− }
1079		−}
1080		−
1081		−void setup_rfi_flush(enum l1d_flush_type types, bool enable)
1082		−{
1083		− if (types & L1D_FLUSH_FALLBACK) {
1084		− pr_info("rfi-flush: fallback displacement flush available\n");
1085		− init_fallback_flush();
1086		− }
1087		−
1088		− if (types & L1D_FLUSH_ORI)
1089		− pr_info("rfi-flush: ori type flush available\n");
1090		−
1091		− if (types & L1D_FLUSH_MTTRIG)
1092		− pr_info("rfi-flush: mttrig type flush available\n");
1093		−
1094		− enabled_flush_types = types;
1095		−
1096		− if (!cpu_mitigations_off() && !no_rfi_flush)
1097		− rfi_flush_enable(enable);
1098		−}
1099		−
1100		−void setup_entry_flush(bool enable)
1101		−{
1102		− if (cpu_mitigations_off())
1103		− return;
1104		−
1105		− if (!no_entry_flush)
1106		− entry_flush_enable(enable);
1107		−}
1108		−
1109		−void setup_uaccess_flush(bool enable)
1110		−{
1111		− if (cpu_mitigations_off())
1112		− return;
1113		−
1114		− if (!no_uaccess_flush)
1115		− uaccess_flush_enable(enable);
1116		−}
1117		−
1118		−#ifdef CONFIG_DEBUG_FS
1119		−static int rfi_flush_set(void *data, u64 val)
1120		−{
1121		− bool enable;
1122		−
1123		− if (val == 1)
1124		− enable = true;
1125		− else if (val == 0)
1126		− enable = false;
1127		− else
1128		− return -EINVAL;
1129		−
1130		− /* Only do anything if we're changing state */
1131		− if (enable != rfi_flush)
1132		− rfi_flush_enable(enable);
1133		−
1134		− return 0;
1135		−}
1136		−
1137		−static int rfi_flush_get(void data, u64 val)
1138		−{
1139		− *val = rfi_flush ? 1 : 0;
1140		− return 0;
1141		−}
1142		−
1143		−DEFINE_SIMPLE_ATTRIBUTE(fops_rfi_flush, rfi_flush_get, rfi_flush_set, "%llu\n");
1144		−
1145		−static int entry_flush_set(void *data, u64 val)
1146		−{
1147		− bool enable;
1148		−
1149		− if (val == 1)
1150		− enable = true;
1151		− else if (val == 0)
1152		− enable = false;
1153		− else
1154		− return -EINVAL;
1155		−
1156		− /* Only do anything if we're changing state */
1157		− if (enable != entry_flush)
1158		− entry_flush_enable(enable);
1159		−
1160		− return 0;
1161		−}
1162		−
1163		−static int entry_flush_get(void data, u64 val)
1164		−{
1165		− *val = entry_flush ? 1 : 0;
1166		− return 0;
1167		−}
1168		−
1169		−DEFINE_SIMPLE_ATTRIBUTE(fops_entry_flush, entry_flush_get, entry_flush_set, "%llu\n");
1170		−
1171		−static int uaccess_flush_set(void *data, u64 val)
1172		−{
1173		− bool enable;
1174		−
1175		− if (val == 1)
1176		− enable = true;
1177		− else if (val == 0)
1178		− enable = false;
1179		− else
1180		− return -EINVAL;
1181		−
1182		− /* Only do anything if we're changing state */
1183		− if (enable != uaccess_flush)
1184		− uaccess_flush_enable(enable);
1185		−
1186		− return 0;
1187		−}
1188		−
1189		−static int uaccess_flush_get(void data, u64 val)
1190		−{
1191		− *val = uaccess_flush ? 1 : 0;
1192		− return 0;
1193		−}
1194		−
1195		−DEFINE_SIMPLE_ATTRIBUTE(fops_uaccess_flush, uaccess_flush_get, uaccess_flush_set, "%llu\n");
1196		−
1197		−static __init int rfi_flush_debugfs_init(void)
1198		−{
1199		− debugfs_create_file("rfi_flush", 0600, powerpc_debugfs_root, NULL, &fops_rfi_flush);
1200		− debugfs_create_file("entry_flush", 0600, powerpc_debugfs_root, NULL, &fops_entry_flush);
1201		− debugfs_create_file("uaccess_flush", 0600, powerpc_debugfs_root, NULL, &fops_uaccess_flush);
1202		− return 0;
1203		−}
1204		−device_initcall(rfi_flush_debugfs_init);
1205		−#endif
1206		−#endif /* CONFIG_PPC_BOOK3S_64 */

arch/powerpc/kernel/signal.h+55−0

@@ -19,6 +19,15 @@ extern int handle_signal32(struct ksignal ksig, sigset_t oldset, · 18 unmodified lines
19	19	extern int handle_rt_signal32(struct ksignal ksig, sigset_t oldset,
20	20	struct task_struct *tsk);
21	21
	22	+static inline int __get_user_sigset(sigset_t dst, const sigset_t __user src)
	23	+{
	24	+ BUILD_BUG_ON(sizeof(sigset_t) != sizeof(u64));
	25	+
	26	+ return __get_user(dst->sig[0], (u64 __user *)&src->sig[0]);
	27	+}
	28	+#define unsafe_get_user_sigset(dst, src, label) \
	29	+ unsafe_get_user((dst)->sig[0], (u64 __user *)&(src)->sig[0], label)
	30	+
22	31	#ifdef CONFIG_VSX
23	32	extern unsigned long copy_vsx_to_user(void __user *to,
24	33	struct task_struct *task);
@@ -53,6 +62,26 @@ unsigned long copy_ckfpr_from_user(struct task_struct task, void __user from); · 28 unmodified lines
53	62	&buf[i], label);\
54	63	} while (0)
55	64
	65	+#define unsafe_copy_fpr_from_user(task, from, label) do { \
	66	+ struct task_struct *__t = task; \
	67	+ u64 __user buf = (u64 __user )from; \
	68	+ int i; \
	69	+ \
	70	+ for (i = 0; i < ELF_NFPREG - 1; i++) \
	71	+ unsafe_get_user(__t->thread.TS_FPR(i), &buf[i], label); \
	72	+ unsafe_get_user(__t->thread.fp_state.fpscr, &buf[i], label); \
	73	+} while (0)
	74	+
	75	+#define unsafe_copy_vsx_from_user(task, from, label) do { \
	76	+ struct task_struct *__t = task; \
	77	+ u64 __user buf = (u64 __user )from; \
	78	+ int i; \
	79	+ \
	80	+ for (i = 0; i < ELF_NVSRHALFREG ; i++) \
	81	+ unsafe_get_user(__t->thread.fp_state.fpr[i][TS_VSRLOWOFFSET], \
	82	+ &buf[i], label); \
	83	+} while (0)
	84	+
56	85	#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
57	86	#define unsafe_copy_ckfpr_to_user(to, task, label) do { \
58	87	struct task_struct *__t = task; \
@@ -73,6 +102,26 @@ unsigned long copy_ckfpr_from_user(struct task_struct task, void __user from); · 14 unmodified lines
73	102	unsafe_put_user(__t->thread.ckfp_state.fpr[i][TS_VSRLOWOFFSET], \
74	103	&buf[i], label);\
75	104	} while (0)
	105	+
	106	+#define unsafe_copy_ckfpr_from_user(task, from, label) do { \
	107	+ struct task_struct *__t = task; \
	108	+ u64 __user buf = (u64 __user )from; \
	109	+ int i; \
	110	+ \
	111	+ for (i = 0; i < ELF_NFPREG - 1 ; i++) \
	112	+ unsafe_get_user(__t->thread.TS_CKFPR(i), &buf[i], label);\
	113	+ unsafe_get_user(__t->thread.ckfp_state.fpscr, &buf[i], failed); \
	114	+} while (0)
	115	+
	116	+#define unsafe_copy_ckvsx_from_user(task, from, label) do { \
	117	+ struct task_struct *__t = task; \
	118	+ u64 __user buf = (u64 __user )from; \
	119	+ int i; \
	120	+ \
	121	+ for (i = 0; i < ELF_NVSRHALFREG ; i++) \
	122	+ unsafe_get_user(__t->thread.ckfp_state.fpr[i][TS_VSRLOWOFFSET], \
	123	+ &buf[i], label); \
	124	+} while (0)
76	125	#endif
77	126	#elif defined(CONFIG_PPC_FPU_REGS)
78	127
@@ -80,6 +129,10 @@ unsigned long copy_ckfpr_from_user(struct task_struct task, void __user from); · 1 unmodified line
80	129	unsafe_copy_to_user(to, (task)->thread.fp_state.fpr, \
81	130	ELF_NFPREG * sizeof(double), label)
82	131
	132	+#define unsafe_copy_fpr_from_user(task, from, label) \
	133	+ unsafe_copy_from_user((task)->thread.fp_state.fpr, from, \
	134	+ ELF_NFPREG * sizeof(double), label)
	135	+
83	136	static inline unsigned long
84	137	copy_fpr_to_user(void __user to, struct task_struct task)
85	138	{
@@ -115,6 +168,8 @@ copy_ckfpr_from_user(struct task_struct task, void __user from) · 29 unmodified lines
115	168	#else
116	169	#define unsafe_copy_fpr_to_user(to, task, label) do { } while (0)
117	170
	171	+#define unsafe_copy_fpr_from_user(task, from, label) do { } while (0)
	172	+
118	173	static inline unsigned long
119	174	copy_fpr_to_user(void __user to, struct task_struct task)
120	175	{

arch/powerpc/kernel/signal_32.c+143−111

@@ -83,24 +83,17 @@ · 82 unmodified lines
83	83	* implementation that makes things simple for little endian only)
84	84	*/
85	85	#define unsafe_put_sigset_t unsafe_put_compat_sigset
86		−
87		−static inline int get_sigset_t(sigset_t *set,
88		− const compat_sigset_t __user *uset)
89		−{
90		− return get_compat_sigset(set, uset);
91		−}
	86	+#define unsafe_get_sigset_t unsafe_get_compat_sigset
92	87
93	88	#define to_user_ptr(p) ptr_to_compat(p)
94	89	#define from_user_ptr(p) compat_ptr(p)
95	90
96	91	static __always_inline int
97		−save_general_regs_unsafe(struct pt_regs regs, struct mcontext __user frame)
	92	+__unsafe_save_general_regs(struct pt_regs regs, struct mcontext __user frame)
98	93	{
99	94	elf_greg_t64 gregs = (elf_greg_t64 )regs;
100	95	int val, i;
101	96
102		− WARN_ON(!FULL_REGS(regs));
103		−
104	97	for (i = 0; i <= PT_RESULT; i ++) {
105	98	/* Force usr to alway see softe as 1 (interrupts enabled) */
106	99	if (i == PT_SOFTE)
@@ -116,8 +109,8 @@ failed: · 9 unmodified lines
116	109	return 1;
117	110	}
118	111
119		−static inline int restore_general_regs(struct pt_regs *regs,
120		− struct mcontext __user *sr)
	112	+static __always_inline int
	113	+__unsafe_restore_general_regs(struct pt_regs regs, struct mcontext __user sr)
121	114	{
122	115	elf_greg_t64 gregs = (elf_greg_t64 )regs;
123	116	int i;
@@ -125,10 +118,12 @@ static inline int restore_general_regs(struct pt_regs *regs, · 1 unmodified line
125	118	for (i = 0; i <= PT_RESULT; i++) {
126	119	if ((i == PT_MSR) \|\| (i == PT_SOFTE))
127	120	continue;
128		− if (__get_user(gregs[i], &sr->mc_gregs[i]))
129		− return -EFAULT;
	121	+ unsafe_get_user(gregs[i], &sr->mc_gregs[i], failed);
130	122	}
131	123	return 0;
	124	+
	125	+failed:
	126	+ return 1;
132	127	}
133	128
134	129	#else /* CONFIG_PPC64 */
@@ -142,18 +137,14 @@ static inline int restore_general_regs(struct pt_regs *regs, · 7 unmodified lines
142	137	unsafe_copy_to_user(__us, __s, sizeof(*__us), label); \
143	138	} while (0)
144	139
145		−static inline int get_sigset_t(sigset_t set, const sigset_t __user uset)
146		−{
147		− return copy_from_user(set, uset, sizeof(*uset));
148		−}
	140	+#define unsafe_get_sigset_t unsafe_get_user_sigset
149	141
150	142	#define to_user_ptr(p) ((unsigned long)(p))
151	143	#define from_user_ptr(p) ((void __user *)(p))
152	144
153	145	static __always_inline int
154		−save_general_regs_unsafe(struct pt_regs regs, struct mcontext __user frame)
	146	+__unsafe_save_general_regs(struct pt_regs regs, struct mcontext __user frame)
155	147	{
156		− WARN_ON(!FULL_REGS(regs));
157	148	unsafe_copy_to_user(&frame->mc_gregs, regs, GP_REGS_SIZE, failed);
158	149	return 0;
159	150
@@ -161,23 +152,30 @@ failed: · 1 unmodified line
161	152	return 1;
162	153	}
163	154
164		−static inline int restore_general_regs(struct pt_regs *regs,
165		− struct mcontext __user *sr)
	155	+static __always_inline
	156	+int __unsafe_restore_general_regs(struct pt_regs regs, struct mcontext __user sr)
166	157	{
167	158	/* copy up to but not including MSR */
168		− if (__copy_from_user(regs, &sr->mc_gregs,
169		− PT_MSR * sizeof(elf_greg_t)))
170		− return -EFAULT;
	159	+ unsafe_copy_from_user(regs, &sr->mc_gregs, PT_MSR * sizeof(elf_greg_t), failed);
	160	+
171	161	/* copy from orig_r3 (the word after the MSR) up to the end */
172		− if (__copy_from_user(&regs->orig_gpr3, &sr->mc_gregs[PT_ORIG_R3],
173		− GP_REGS_SIZE - PT_ORIG_R3 * sizeof(elf_greg_t)))
174		− return -EFAULT;
	162	+ unsafe_copy_from_user(&regs->orig_gpr3, &sr->mc_gregs[PT_ORIG_R3],
	163	+ GP_REGS_SIZE - PT_ORIG_R3 * sizeof(elf_greg_t), failed);
	164	+
175	165	return 0;
	166	+
	167	+failed:
	168	+ return 1;
176	169	}
177	170	#endif
178	171
179	172	#define unsafe_save_general_regs(regs, frame, label) do { \
180		− if (save_general_regs_unsafe(regs, frame)) \
	173	+ if (__unsafe_save_general_regs(regs, frame)) \
	174	+ goto label; \
	175	+} while (0)
	176	+
	177	+#define unsafe_restore_general_regs(regs, frame, label) do { \
	178	+ if (__unsafe_restore_general_regs(regs, frame)) \
181	179	goto label; \
182	180	} while (0)
183	181
@@ -260,8 +258,8 @@ static void prepare_save_user_regs(int ctx_has_vsx_region) · 76 unmodified lines
260	258	#endif
261	259	}
262	260
263		−static int save_user_regs_unsafe(struct pt_regs regs, struct mcontext __user frame,
264		− struct mcontext __user *tm_frame, int ctx_has_vsx_region)
	261	+static int __unsafe_save_user_regs(struct pt_regs regs, struct mcontext __user frame,
	262	+ struct mcontext __user *tm_frame, int ctx_has_vsx_region)
265	263	{
266	264	unsigned long msr = regs->msr;
267	265
@@ -338,7 +336,7 @@ failed: · 70 unmodified lines
338	336	}
339	337
340	338	#define unsafe_save_user_regs(regs, frame, tm_frame, has_vsx, label) do { \
341		− if (save_user_regs_unsafe(regs, frame, tm_frame, has_vsx)) \
	339	+ if (__unsafe_save_user_regs(regs, frame, tm_frame, has_vsx)) \
342	340	goto label; \
343	341	} while (0)
344	342
@@ -350,7 +348,7 @@ failed: · 5 unmodified lines
350	348	* We also save the transactional registers to a second ucontext in the
351	349	* frame.
352	350	*
353		− * See save_user_regs_unsafe() and signal_64.c:setup_tm_sigcontexts().
	351	+ * See __unsafe_save_user_regs() and signal_64.c:setup_tm_sigcontexts().
354	352	*/
355	353	static void prepare_save_tm_user_regs(void)
356	354	{
@@ -441,7 +439,7 @@ static int save_tm_user_regs_unsafe(struct pt_regs *regs, struct mcontext __user · 84 unmodified lines
441	439	#endif /* CONFIG_VSX */
442	440	#ifdef CONFIG_SPE
443	441	/* SPE regs are not checkpointed with TM, so this section is
444		− * simply the same as in save_user_regs_unsafe().
	442	+ * simply the same as in __unsafe_save_user_regs().
445	443	*/
446	444	if (current->thread.used_spe) {
447	445	unsafe_copy_to_user(&frame->mc_vregs, current->thread.evr,
@@ -485,26 +483,25 @@ static int save_tm_user_regs_unsafe(struct pt_regs *regs, struct mcontext __user · 37 unmodified lines
485	483	static long restore_user_regs(struct pt_regs *regs,
486	484	struct mcontext __user *sr, int sig)
487	485	{
488		− long err;
489	486	unsigned int save_r2 = 0;
490	487	unsigned long msr;
491	488	#ifdef CONFIG_VSX
492	489	int i;
493	490	#endif
494	491
	492	+ if (!user_read_access_begin(sr, sizeof(*sr)))
	493	+ return 1;
495	494	/*
496	495	* restore general registers but not including MSR or SOFTE. Also
497	496	* take care of keeping r2 (TLS) intact if not a signal
498	497	*/
499	498	if (!sig)
500	499	save_r2 = (unsigned int)regs->gpr[2];
501		− err = restore_general_regs(regs, sr);
	500	+ unsafe_restore_general_regs(regs, sr, failed);
502	501	set_trap_norestart(regs);
503		− err \|= __get_user(msr, &sr->mc_gregs[PT_MSR]);
	502	+ unsafe_get_user(msr, &sr->mc_gregs[PT_MSR], failed);
504	503	if (!sig)
505	504	regs->gpr[2] = (unsigned long) save_r2;
506		− if (err)
507		− return 1;
508	505
509	506	/* if doing signal return, restore the previous little-endian mode */
510	507	if (sig)
@@ -518,22 +515,19 @@ static long restore_user_regs(struct pt_regs *regs, · 7 unmodified lines
518	515	regs->msr &= ~MSR_VEC;
519	516	if (msr & MSR_VEC) {
520	517	/* restore altivec registers from the stack */
521		− if (__copy_from_user(&current->thread.vr_state, &sr->mc_vregs,
522		− sizeof(sr->mc_vregs)))
523		− return 1;
	518	+ unsafe_copy_from_user(&current->thread.vr_state, &sr->mc_vregs,
	519	+ sizeof(sr->mc_vregs), failed);
524	520	current->thread.used_vr = true;
525	521	} else if (current->thread.used_vr)
526	522	memset(&current->thread.vr_state, 0,
527	523	ELF_NVRREG * sizeof(vector128));
528	524
529	525	/* Always get VRSAVE back */
530		− if (__get_user(current->thread.vrsave, (u32 __user *)&sr->mc_vregs[32]))
531		− return 1;
	526	+ unsafe_get_user(current->thread.vrsave, (u32 __user *)&sr->mc_vregs[32], failed);
532	527	if (cpu_has_feature(CPU_FTR_ALTIVEC))
533	528	mtspr(SPRN_VRSAVE, current->thread.vrsave);
534	529	#endif /* CONFIG_ALTIVEC */
535		− if (copy_fpr_from_user(current, &sr->mc_fregs))
536		− return 1;
	530	+ unsafe_copy_fpr_from_user(current, &sr->mc_fregs, failed);
537	531
538	532	#ifdef CONFIG_VSX
539	533	/*
@@ -546,8 +540,7 @@ static long restore_user_regs(struct pt_regs *regs, · 6 unmodified lines
546	540	* Restore altivec registers from the stack to a local
547	541	* buffer, then write this out to the thread_struct
548	542	*/
549		− if (copy_vsx_from_user(current, &sr->mc_vsregs))
550		− return 1;
	543	+ unsafe_copy_vsx_from_user(current, &sr->mc_vsregs, failed);
551	544	current->thread.used_vsr = true;
552	545	} else if (current->thread.used_vsr)
553	546	for (i = 0; i < 32 ; i++)
@@ -565,19 +558,22 @@ static long restore_user_regs(struct pt_regs *regs, · 11 unmodified lines
565	558	regs->msr &= ~MSR_SPE;
566	559	if (msr & MSR_SPE) {
567	560	/* restore spe registers from the stack */
568		− if (__copy_from_user(current->thread.evr, &sr->mc_vregs,
569		− ELF_NEVRREG * sizeof(u32)))
570		− return 1;
	561	+ unsafe_copy_from_user(current->thread.evr, &sr->mc_vregs,
	562	+ ELF_NEVRREG * sizeof(u32), failed);
571	563	current->thread.used_spe = true;
572	564	} else if (current->thread.used_spe)
573	565	memset(current->thread.evr, 0, ELF_NEVRREG * sizeof(u32));
574	566
575	567	/* Always get SPEFSCR back */
576		− if (__get_user(current->thread.spefscr, (u32 __user *)&sr->mc_vregs + ELF_NEVRREG))
577		− return 1;
	568	+ unsafe_get_user(current->thread.spefscr, (u32 __user *)&sr->mc_vregs + ELF_NEVRREG, failed);
578	569	#endif /* CONFIG_SPE */
579	570
	571	+ user_read_access_end();
580	572	return 0;
	573	+
	574	+failed:
	575	+ user_read_access_end();
	576	+ return 1;
581	577	}
582	578
583	579	#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
@@ -590,7 +586,6 @@ static long restore_tm_user_regs(struct pt_regs *regs, · 6 unmodified lines
590	586	struct mcontext __user *sr,
591	587	struct mcontext __user *tm_sr)
592	588	{
593		− long err;
594	589	unsigned long msr, msr_hi;
595	590	#ifdef CONFIG_VSX
596	591	int i;
@@ -605,15 +600,13 @@ static long restore_tm_user_regs(struct pt_regs *regs, · 8 unmodified lines
605	600	* TFHAR is restored from the checkpointed NIP; TEXASR and TFIAR
606	601	* were set by the signal delivery.
607	602	*/
608		− err = restore_general_regs(regs, tm_sr);
609		− err \|= restore_general_regs(&current->thread.ckpt_regs, sr);
610		−
611		− err \|= __get_user(current->thread.tm_tfhar, &sr->mc_gregs[PT_NIP]);
612		−
613		− err \|= __get_user(msr, &sr->mc_gregs[PT_MSR]);
614		− if (err)
	603	+ if (!user_read_access_begin(sr, sizeof(*sr)))
615	604	return 1;
616	605
	606	+ unsafe_restore_general_regs(&current->thread.ckpt_regs, sr, failed);
	607	+ unsafe_get_user(current->thread.tm_tfhar, &sr->mc_gregs[PT_NIP], failed);
	608	+ unsafe_get_user(msr, &sr->mc_gregs[PT_MSR], failed);
	609	+
617	610	/* Restore the previous little-endian mode */
618	611	regs->msr = (regs->msr & ~MSR_LE) \| (msr & MSR_LE);
619	612
@@ -621,12 +614,8 @@ static long restore_tm_user_regs(struct pt_regs *regs, · 1 unmodified line
621	614	regs->msr &= ~MSR_VEC;
622	615	if (msr & MSR_VEC) {
623	616	/* restore altivec registers from the stack */
624		− if (__copy_from_user(&current->thread.ckvr_state, &sr->mc_vregs,
625		− sizeof(sr->mc_vregs)) \|\|
626		− __copy_from_user(&current->thread.vr_state,
627		− &tm_sr->mc_vregs,
628		− sizeof(sr->mc_vregs)))
629		− return 1;
	617	+ unsafe_copy_from_user(&current->thread.ckvr_state, &sr->mc_vregs,
	618	+ sizeof(sr->mc_vregs), failed);
630	619	current->thread.used_vr = true;
631	620	} else if (current->thread.used_vr) {
632	621	memset(&current->thread.vr_state, 0,
@@ -636,20 +625,15 @@ static long restore_tm_user_regs(struct pt_regs *regs, · 3 unmodified lines
636	625	}
637	626
638	627	/* Always get VRSAVE back */
639		− if (__get_user(current->thread.ckvrsave,
640		− (u32 __user *)&sr->mc_vregs[32]) \|\|
641		− __get_user(current->thread.vrsave,
642		− (u32 __user *)&tm_sr->mc_vregs[32]))
643		− return 1;
	628	+ unsafe_get_user(current->thread.ckvrsave,
	629	+ (u32 __user *)&sr->mc_vregs[32], failed);
644	630	if (cpu_has_feature(CPU_FTR_ALTIVEC))
645	631	mtspr(SPRN_VRSAVE, current->thread.ckvrsave);
646	632	#endif /* CONFIG_ALTIVEC */
647	633
648	634	regs->msr &= ~(MSR_FP \| MSR_FE0 \| MSR_FE1);
649	635
650		− if (copy_fpr_from_user(current, &sr->mc_fregs) \|\|
651		− copy_ckfpr_from_user(current, &tm_sr->mc_fregs))
652		− return 1;
	636	+ unsafe_copy_fpr_from_user(current, &sr->mc_fregs, failed);
653	637
654	638	#ifdef CONFIG_VSX
655	639	regs->msr &= ~MSR_VSX;
@@ -658,9 +642,7 @@ static long restore_tm_user_regs(struct pt_regs *regs, · 2 unmodified lines
658	642	* Restore altivec registers from the stack to a local
659	643	* buffer, then write this out to the thread_struct
660	644	*/
661		− if (copy_vsx_from_user(current, &tm_sr->mc_vsregs) \|\|
662		− copy_ckvsx_from_user(current, &sr->mc_vsregs))
663		− return 1;
	645	+ unsafe_copy_ckvsx_from_user(current, &sr->mc_vsregs, failed);
664	646	current->thread.used_vsr = true;
665	647	} else if (current->thread.used_vsr)
666	648	for (i = 0; i < 32 ; i++) {
@@ -675,23 +657,54 @@ static long restore_tm_user_regs(struct pt_regs *regs, · 8 unmodified lines
675	657	*/
676	658	regs->msr &= ~MSR_SPE;
677	659	if (msr & MSR_SPE) {
678		− if (__copy_from_user(current->thread.evr, &sr->mc_vregs,
679		− ELF_NEVRREG * sizeof(u32)))
680		− return 1;
	660	+ unsafe_copy_from_user(current->thread.evr, &sr->mc_vregs,
	661	+ ELF_NEVRREG * sizeof(u32), failed);
681	662	current->thread.used_spe = true;
682	663	} else if (current->thread.used_spe)
683	664	memset(current->thread.evr, 0, ELF_NEVRREG * sizeof(u32));
684	665
685	666	/* Always get SPEFSCR back */
686		− if (__get_user(current->thread.spefscr, (u32 __user *)&sr->mc_vregs
687		− + ELF_NEVRREG))
688		− return 1;
	667	+ unsafe_get_user(current->thread.spefscr,
	668	+ (u32 __user *)&sr->mc_vregs + ELF_NEVRREG, failed);
689	669	#endif /* CONFIG_SPE */
690	670
691		− /* Get the top half of the MSR from the user context */
692		− if (__get_user(msr_hi, &tm_sr->mc_gregs[PT_MSR]))
	671	+ user_read_access_end();
	672	+
	673	+ if (!user_read_access_begin(tm_sr, sizeof(*tm_sr)))
693	674	return 1;
	675	+
	676	+ unsafe_restore_general_regs(regs, tm_sr, failed);
	677	+
	678	+#ifdef CONFIG_ALTIVEC
	679	+ /* restore altivec registers from the stack */
	680	+ if (msr & MSR_VEC)
	681	+ unsafe_copy_from_user(&current->thread.vr_state, &tm_sr->mc_vregs,
	682	+ sizeof(sr->mc_vregs), failed);
	683	+
	684	+ /* Always get VRSAVE back */
	685	+ unsafe_get_user(current->thread.vrsave,
	686	+ (u32 __user *)&tm_sr->mc_vregs[32], failed);
	687	+#endif /* CONFIG_ALTIVEC */
	688	+
	689	+ unsafe_copy_ckfpr_from_user(current, &tm_sr->mc_fregs, failed);
	690	+
	691	+#ifdef CONFIG_VSX
	692	+ if (msr & MSR_VSX) {
	693	+ /*
	694	+ * Restore altivec registers from the stack to a local
	695	+ * buffer, then write this out to the thread_struct
	696	+ */
	697	+ unsafe_copy_vsx_from_user(current, &tm_sr->mc_vsregs, failed);
	698	+ current->thread.used_vsr = true;
	699	+ }
	700	+#endif /* CONFIG_VSX */
	701	+
	702	+ /* Get the top half of the MSR from the user context */
	703	+ unsafe_get_user(msr_hi, &tm_sr->mc_gregs[PT_MSR], failed);
694	704	msr_hi <<= 32;
	705	+
	706	+ user_read_access_end();
	707	+
695	708	/* If TM bits are set to the reserved value, it's an invalid context */
696	709	if (MSR_TM_RESV(msr_hi))
697	710	return 1;
@@ -739,6 +752,16 @@ static long restore_tm_user_regs(struct pt_regs *regs, · 41 unmodified lines
739	752	preempt_enable();
740	753
741	754	return 0;
	755	+
	756	+failed:
	757	+ user_read_access_end();
	758	+ return 1;
	759	+}
	760	+#else
	761	+static long restore_tm_user_regs(struct pt_regs regs, struct mcontext __user sr,
	762	+ struct mcontext __user *tm_sr)
	763	+{
	764	+ return 0;
742	765	}
743	766	#endif
744	767
@@ -944,28 +967,31 @@ static int do_setcontext(struct ucontext __user ucp, struct pt_regs regs, int · 199 unmodified lines
944	967	sigset_t set;
945	968	struct mcontext __user *mcp;
946	969
947		− if (get_sigset_t(&set, &ucp->uc_sigmask))
	970	+ if (!user_read_access_begin(ucp, sizeof(*ucp)))
948	971	return -EFAULT;
	972	+
	973	+ unsafe_get_sigset_t(&set, &ucp->uc_sigmask, failed);
949	974	#ifdef CONFIG_PPC64
950	975	{
951	976	u32 cmcp;
952	977
953		− if (__get_user(cmcp, &ucp->uc_regs))
954		− return -EFAULT;
	978	+ unsafe_get_user(cmcp, &ucp->uc_regs, failed);
955	979	mcp = (struct mcontext __user *)(u64)cmcp;
956		− /* no need to check access_ok(mcp), since mcp < 4GB */
957	980	}
958	981	#else
959		− if (__get_user(mcp, &ucp->uc_regs))
960		− return -EFAULT;
961		− if (!access_ok(mcp, sizeof(*mcp)))
962		− return -EFAULT;
	982	+ unsafe_get_user(mcp, &ucp->uc_regs, failed);
963	983	#endif
	984	+ user_read_access_end();
	985	+
964	986	set_current_blocked(&set);
965	987	if (restore_user_regs(regs, mcp, sig))
966	988	return -EFAULT;
967	989
968	990	return 0;
	991	+
	992	+failed:
	993	+ user_read_access_end();
	994	+ return -EFAULT;
969	995	}
970	996
971	997	#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
@@ -979,11 +1005,15 @@ static int do_setcontext_tm(struct ucontext __user *ucp, · 7 unmodified lines
979	1005	u32 cmcp;
980	1006	u32 tm_cmcp;
981	1007
982		− if (get_sigset_t(&set, &ucp->uc_sigmask))
	1008	+ if (!user_read_access_begin(ucp, sizeof(*ucp)))
983	1009	return -EFAULT;
984	1010
985		− if (__get_user(cmcp, &ucp->uc_regs) \|\|
986		− __get_user(tm_cmcp, &tm_ucp->uc_regs))
	1011	+ unsafe_get_sigset_t(&set, &ucp->uc_sigmask, failed);
	1012	+ unsafe_get_user(cmcp, &ucp->uc_regs, failed);
	1013	+
	1014	+ user_read_access_end();
	1015	+
	1016	+ if (__get_user(tm_cmcp, &tm_ucp->uc_regs))
987	1017	return -EFAULT;
988	1018	mcp = (struct mcontext __user *)(u64)cmcp;
989	1019	tm_mcp = (struct mcontext __user *)(u64)tm_cmcp;
@@ -994,6 +1024,10 @@ static int do_setcontext_tm(struct ucontext __user *ucp, · 4 unmodified lines
994	1024	return -EFAULT;
995	1025
996	1026	return 0;
	1027	+
	1028	+failed:
	1029	+ user_read_access_end();
	1030	+ return -EFAULT;
997	1031	}
998	1032	#endif
999	1033
@@ -1311,19 +1345,16 @@ SYSCALL_DEFINE0(sigreturn) · 311 unmodified lines
1311	1345	struct sigcontext __user *sc;
1312	1346	struct sigcontext sigctx;
1313	1347	struct mcontext __user *sr;
1314		− void __user *addr;
1315	1348	sigset_t set;
1316		−#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
1317		− struct mcontext __user mcp, tm_mcp;
1318		− unsigned long msr_hi;
1319		−#endif
	1349	+ struct mcontext __user *mcp;
	1350	+ struct mcontext __user *tm_mcp = NULL;
	1351	+ unsigned long long msr_hi = 0;
1320	1352
1321	1353	/* Always make any pending restarted system calls return -EINTR */
1322	1354	current->restart_block.fn = do_no_restart_syscall;
1323	1355
1324	1356	sf = (struct sigframe __user *)(regs->gpr[1] + __SIGNAL_FRAMESIZE);
1325	1357	sc = &sf->sctx;
1326		− addr = sc;
1327	1358	if (copy_from_user(&sigctx, sc, sizeof(sigctx)))
1328	1359	goto badframe;
1329	1360
@@ -1339,31 +1370,32 @@ SYSCALL_DEFINE0(sigreturn) · 9 unmodified lines
1339	1370	#endif
1340	1371	set_current_blocked(&set);
1341	1372
1342		−#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
1343	1373	mcp = (struct mcontext __user *)&sf->mctx;
	1374	+#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
1344	1375	tm_mcp = (struct mcontext __user *)&sf->mctx_transact;
1345	1376	if (__get_user(msr_hi, &tm_mcp->mc_gregs[PT_MSR]))
1346	1377	goto badframe;
	1378	+#endif
1347	1379	if (MSR_TM_ACTIVE(msr_hi<<32)) {
1348	1380	if (!cpu_has_feature(CPU_FTR_TM))
1349	1381	goto badframe;
1350	1382	if (restore_tm_user_regs(regs, mcp, tm_mcp))
1351	1383	goto badframe;
1352		− } else
1353		−#endif
1354		− {
	1384	+ } else {
1355	1385	sr = (struct mcontext __user *)from_user_ptr(sigctx.regs);
1356		− addr = sr;
1357		− if (!access_ok(sr, sizeof(*sr))
1358		− \|\| restore_user_regs(regs, sr, 1))
1359		− goto badframe;
	1386	+ if (restore_user_regs(regs, sr, 1)) {
	1387	+ signal_fault(current, regs, "sys_sigreturn", sr);
	1388	+
	1389	+ force_sig(SIGSEGV);
	1390	+ return 0;
	1391	+ }
1360	1392	}
1361	1393
1362	1394	set_thread_flag(TIF_RESTOREALL);
1363	1395	return 0;
1364	1396
1365	1397	badframe:
1366		− signal_fault(current, regs, "sys_sigreturn", addr);
	1398	+ signal_fault(current, regs, "sys_sigreturn", sc);
1367	1399
1368	1400	force_sig(SIGSEGV);
1369	1401	return 0;

arch/powerpc/kernel/signal_64.c+189−130

@@ -79,13 +79,36 @@ static elf_vrreg_t __user sigcontext_vmx_regs(struct sigcontext __user sc) · 78 unmodified lines
79	79	}
80	80	#endif
81	81
	82	+static void prepare_setup_sigcontext(struct task_struct *tsk)
	83	+{
	84	+#ifdef CONFIG_ALTIVEC
	85	+ /* save altivec registers */
	86	+ if (tsk->thread.used_vr)
	87	+ flush_altivec_to_thread(tsk);
	88	+ if (cpu_has_feature(CPU_FTR_ALTIVEC))
	89	+ tsk->thread.vrsave = mfspr(SPRN_VRSAVE);
	90	+#endif /* CONFIG_ALTIVEC */
	91	+
	92	+ flush_fp_to_thread(tsk);
	93	+
	94	+#ifdef CONFIG_VSX
	95	+ if (tsk->thread.used_vsr)
	96	+ flush_vsx_to_thread(tsk);
	97	+#endif /* CONFIG_VSX */
	98	+}
	99	+
82	100	/*
83	101	* Set up the sigcontext for the signal frame.
84	102	*/
85	103
86		−static long setup_sigcontext(struct sigcontext __user *sc,
87		− struct task_struct tsk, int signr, sigset_t set,
88		− unsigned long handler, int ctx_has_vsx_region)
	104	+#define unsafe_setup_sigcontext(sc, tsk, signr, set, handler, ctx_has_vsx_region, label)\
	105	+do { \
	106	+ if (__unsafe_setup_sigcontext(sc, tsk, signr, set, handler, ctx_has_vsx_region))\
	107	+ goto label; \
	108	+} while (0)
	109	+static long notrace __unsafe_setup_sigcontext(struct sigcontext __user *sc,
	110	+ struct task_struct tsk, int signr, sigset_t set,
	111	+ unsigned long handler, int ctx_has_vsx_region)
89	112	{
90	113	/* When CONFIG_ALTIVEC is set, we _always_ setup v_regs even if the
91	114	* process never used altivec yet (MSR_VEC is zero in pt_regs of
@@ -97,25 +120,22 @@ static long setup_sigcontext(struct sigcontext __user *sc, · 5 unmodified lines
97	120	*/
98	121	#ifdef CONFIG_ALTIVEC
99	122	elf_vrreg_t __user *v_regs = sigcontext_vmx_regs(sc);
100		− unsigned long vrsave;
101	123	#endif
102	124	struct pt_regs *regs = tsk->thread.regs;
103	125	unsigned long msr = regs->msr;
104		− long err = 0;
105	126	/* Force usr to alway see softe as 1 (interrupts enabled) */
106	127	unsigned long softe = 0x1;
107	128
108	129	BUG_ON(tsk != current);
109	130
110	131	#ifdef CONFIG_ALTIVEC
111		− err \|= __put_user(v_regs, &sc->v_regs);
	132	+ unsafe_put_user(v_regs, &sc->v_regs, efault_out);
112	133
113	134	/* save altivec registers */
114	135	if (tsk->thread.used_vr) {
115		− flush_altivec_to_thread(tsk);
116	136	/* Copy 33 vec registers (vr0..31 and vscr) to the stack */
117		− err \|= __copy_to_user(v_regs, &tsk->thread.vr_state,
118		− 33 * sizeof(vector128));
	137	+ unsafe_copy_to_user(v_regs, &tsk->thread.vr_state,
	138	+ 33 * sizeof(vector128), efault_out);
119	139	/* set MSR_VEC in the MSR value in the frame to indicate that sc->v_reg)
120	140	* contains valid data.
121	141	*/
@@ -124,19 +144,12 @@ static long setup_sigcontext(struct sigcontext __user *sc, · 2 unmodified lines
124	144	/* We always copy to/from vrsave, it's 0 if we don't have or don't
125	145	* use altivec.
126	146	*/
127		− vrsave = 0;
128		− if (cpu_has_feature(CPU_FTR_ALTIVEC)) {
129		− vrsave = mfspr(SPRN_VRSAVE);
130		− tsk->thread.vrsave = vrsave;
131		− }
132		−
133		− err \|= __put_user(vrsave, (u32 __user *)&v_regs[33]);
	147	+ unsafe_put_user(tsk->thread.vrsave, (u32 __user *)&v_regs[33], efault_out);
134	148	#else /* CONFIG_ALTIVEC */
135		− err \|= __put_user(0, &sc->v_regs);
	149	+ unsafe_put_user(0, &sc->v_regs, efault_out);
136	150	#endif /* CONFIG_ALTIVEC */
137		− flush_fp_to_thread(tsk);
138	151	/* copy fpr regs and fpscr */
139		− err \|= copy_fpr_to_user(&sc->fp_regs, tsk);
	152	+ unsafe_copy_fpr_to_user(&sc->fp_regs, tsk, efault_out);
140	153
141	154	/*
142	155	* Clear the MSR VSX bit to indicate there is no valid state attached
@@ -150,26 +163,27 @@ static long setup_sigcontext(struct sigcontext __user *sc, · 7 unmodified lines
150	163	* VMX data.
151	164	*/
152	165	if (tsk->thread.used_vsr && ctx_has_vsx_region) {
153		− flush_vsx_to_thread(tsk);
154	166	v_regs += ELF_NVRREG;
155		− err \|= copy_vsx_to_user(v_regs, tsk);
	167	+ unsafe_copy_vsx_to_user(v_regs, tsk, efault_out);
156	168	/* set MSR_VSX in the MSR value in the frame to
157	169	* indicate that sc->vs_reg) contains valid data.
158	170	*/
159	171	msr \|= MSR_VSX;
160	172	}
161	173	#endif /* CONFIG_VSX */
162		− err \|= __put_user(&sc->gp_regs, &sc->regs);
163		− WARN_ON(!FULL_REGS(regs));
164		− err \|= __copy_to_user(&sc->gp_regs, regs, GP_REGS_SIZE);
165		− err \|= __put_user(msr, &sc->gp_regs[PT_MSR]);
166		− err \|= __put_user(softe, &sc->gp_regs[PT_SOFTE]);
167		− err \|= __put_user(signr, &sc->signal);
168		− err \|= __put_user(handler, &sc->handler);
	174	+ unsafe_put_user(&sc->gp_regs, &sc->regs, efault_out);
	175	+ unsafe_copy_to_user(&sc->gp_regs, regs, GP_REGS_SIZE, efault_out);
	176	+ unsafe_put_user(msr, &sc->gp_regs[PT_MSR], efault_out);
	177	+ unsafe_put_user(softe, &sc->gp_regs[PT_SOFTE], efault_out);
	178	+ unsafe_put_user(signr, &sc->signal, efault_out);
	179	+ unsafe_put_user(handler, &sc->handler, efault_out);
169	180	if (set != NULL)
170		− err \|= __put_user(set->sig[0], &sc->oldmask);
	181	+ unsafe_put_user(set->sig[0], &sc->oldmask, efault_out);
171	182
172		− return err;
	183	+ return 0;
	184	+
	185	+efault_out:
	186	+ return -EFAULT;
173	187	}
174	188
175	189	#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
@@ -294,7 +308,6 @@ static long setup_tm_sigcontexts(struct sigcontext __user *sc, · 118 unmodified lines
294	308
295	309	err \|= __put_user(&sc->gp_regs, &sc->regs);
296	310	err \|= __put_user(&tm_sc->gp_regs, &tm_sc->regs);
297		− WARN_ON(!FULL_REGS(regs));
298	311	err \|= __copy_to_user(&tm_sc->gp_regs, regs, GP_REGS_SIZE);
299	312	err \|= __copy_to_user(&sc->gp_regs,
300	313	&tsk->thread.ckpt_regs, GP_REGS_SIZE);
@@ -312,14 +325,16 @@ static long setup_tm_sigcontexts(struct sigcontext __user *sc, · 11 unmodified lines
312	325	/*
313	326	* Restore the sigcontext from the signal frame.
314	327	*/
315		−
316		−static long restore_sigcontext(struct task_struct tsk, sigset_t set, int sig,
317		− struct sigcontext __user *sc)
	328	+#define unsafe_restore_sigcontext(tsk, set, sig, sc, label) do { \
	329	+ if (__unsafe_restore_sigcontext(tsk, set, sig, sc)) \
	330	+ goto label; \
	331	+} while (0)
	332	+static long notrace __unsafe_restore_sigcontext(struct task_struct tsk, sigset_t set,
	333	+ int sig, struct sigcontext __user *sc)
318	334	{
319	335	#ifdef CONFIG_ALTIVEC
320	336	elf_vrreg_t __user *v_regs;
321	337	#endif
322		− unsigned long err = 0;
323	338	unsigned long save_r13 = 0;
324	339	unsigned long msr;
325	340	struct pt_regs *regs = tsk->thread.regs;
@@ -334,27 +349,27 @@ static long restore_sigcontext(struct task_struct tsk, sigset_t set, int sig, · 8 unmodified lines
334	349	save_r13 = regs->gpr[13];
335	350
336	351	/* copy the GPRs */
337		− err \|= __copy_from_user(regs->gpr, sc->gp_regs, sizeof(regs->gpr));
338		− err \|= __get_user(regs->nip, &sc->gp_regs[PT_NIP]);
	352	+ unsafe_copy_from_user(regs->gpr, sc->gp_regs, sizeof(regs->gpr), efault_out);
	353	+ unsafe_get_user(regs->nip, &sc->gp_regs[PT_NIP], efault_out);
339	354	/* get MSR separately, transfer the LE bit if doing signal return */
340		− err \|= __get_user(msr, &sc->gp_regs[PT_MSR]);
	355	+ unsafe_get_user(msr, &sc->gp_regs[PT_MSR], efault_out);
341	356	if (sig)
342	357	regs->msr = (regs->msr & ~MSR_LE) \| (msr & MSR_LE);
343		− err \|= __get_user(regs->orig_gpr3, &sc->gp_regs[PT_ORIG_R3]);
344		− err \|= __get_user(regs->ctr, &sc->gp_regs[PT_CTR]);
345		− err \|= __get_user(regs->link, &sc->gp_regs[PT_LNK]);
346		− err \|= __get_user(regs->xer, &sc->gp_regs[PT_XER]);
347		− err \|= __get_user(regs->ccr, &sc->gp_regs[PT_CCR]);
	358	+ unsafe_get_user(regs->orig_gpr3, &sc->gp_regs[PT_ORIG_R3], efault_out);
	359	+ unsafe_get_user(regs->ctr, &sc->gp_regs[PT_CTR], efault_out);
	360	+ unsafe_get_user(regs->link, &sc->gp_regs[PT_LNK], efault_out);
	361	+ unsafe_get_user(regs->xer, &sc->gp_regs[PT_XER], efault_out);
	362	+ unsafe_get_user(regs->ccr, &sc->gp_regs[PT_CCR], efault_out);
348	363	/* Don't allow userspace to set SOFTE */
349	364	set_trap_norestart(regs);
350		− err \|= __get_user(regs->dar, &sc->gp_regs[PT_DAR]);
351		− err \|= __get_user(regs->dsisr, &sc->gp_regs[PT_DSISR]);
352		− err \|= __get_user(regs->result, &sc->gp_regs[PT_RESULT]);
	365	+ unsafe_get_user(regs->dar, &sc->gp_regs[PT_DAR], efault_out);
	366	+ unsafe_get_user(regs->dsisr, &sc->gp_regs[PT_DSISR], efault_out);
	367	+ unsafe_get_user(regs->result, &sc->gp_regs[PT_RESULT], efault_out);
353	368
354	369	if (!sig)
355	370	regs->gpr[13] = save_r13;
356	371	if (set != NULL)
357		− err \|= __get_user(set->sig[0], &sc->oldmask);
	372	+ unsafe_get_user(set->sig[0], &sc->oldmask, efault_out);
358	373
359	374	/*
360	375	* Force reload of FP/VEC.
@@ -364,29 +379,27 @@ static long restore_sigcontext(struct task_struct tsk, sigset_t set, int sig, · 3 unmodified lines
364	379	regs->msr &= ~(MSR_FP \| MSR_FE0 \| MSR_FE1 \| MSR_VEC \| MSR_VSX);
365	380
366	381	#ifdef CONFIG_ALTIVEC
367		− err \|= __get_user(v_regs, &sc->v_regs);
368		− if (err)
369		− return err;
	382	+ unsafe_get_user(v_regs, &sc->v_regs, efault_out);
370	383	if (v_regs && !access_ok(v_regs, 34 * sizeof(vector128)))
371	384	return -EFAULT;
372	385	/* Copy 33 vec registers (vr0..31 and vscr) from the stack */
373	386	if (v_regs != NULL && (msr & MSR_VEC) != 0) {
374		− err \|= __copy_from_user(&tsk->thread.vr_state, v_regs,
375		− 33 * sizeof(vector128));
	387	+ unsafe_copy_from_user(&tsk->thread.vr_state, v_regs,
	388	+ 33 * sizeof(vector128), efault_out);
376	389	tsk->thread.used_vr = true;
377	390	} else if (tsk->thread.used_vr) {
378	391	memset(&tsk->thread.vr_state, 0, 33 * sizeof(vector128));
379	392	}
380	393	/* Always get VRSAVE back */
381	394	if (v_regs != NULL)
382		− err \|= __get_user(tsk->thread.vrsave, (u32 __user *)&v_regs[33]);
	395	+ unsafe_get_user(tsk->thread.vrsave, (u32 __user *)&v_regs[33], efault_out);
383	396	else
384	397	tsk->thread.vrsave = 0;
385	398	if (cpu_has_feature(CPU_FTR_ALTIVEC))
386	399	mtspr(SPRN_VRSAVE, tsk->thread.vrsave);
387	400	#endif /* CONFIG_ALTIVEC */
388	401	/* restore floating point */
389		− err \|= copy_fpr_from_user(tsk, &sc->fp_regs);
	402	+ unsafe_copy_fpr_from_user(tsk, &sc->fp_regs, efault_out);
390	403	#ifdef CONFIG_VSX
391	404	/*
392	405	* Get additional VSX data. Update v_regs to point after the
@@ -395,14 +408,17 @@ static long restore_sigcontext(struct task_struct tsk, sigset_t set, int sig, · 2 unmodified lines
395	408	*/
396	409	v_regs += ELF_NVRREG;
397	410	if ((msr & MSR_VSX) != 0) {
398		− err \|= copy_vsx_from_user(tsk, v_regs);
	411	+ unsafe_copy_vsx_from_user(tsk, v_regs, efault_out);
399	412	tsk->thread.used_vsr = true;
400	413	} else {
401	414	for (i = 0; i < 32 ; i++)
402	415	tsk->thread.fp_state.fpr[i][TS_VSRLOWOFFSET] = 0;
403	416	}
404	417	#endif
405		− return err;
	418	+ return 0;
	419	+
	420	+efault_out:
	421	+ return -EFAULT;
406	422	}
407	423
408	424	#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
@@ -586,6 +602,12 @@ static long restore_tm_sigcontexts(struct task_struct *tsk, · 177 unmodified lines
586	602
587	603	return err;
588	604	}
	605	+#else /* !CONFIG_PPC_TRANSACTIONAL_MEM */
	606	+static long restore_tm_sigcontexts(struct task_struct tsk, struct sigcontext __user sc,
	607	+ struct sigcontext __user *tm_sc)
	608	+{
	609	+ return -EINVAL;
	610	+}
589	611	#endif
590	612
591	613	/*
@@ -655,12 +677,16 @@ SYSCALL_DEFINE3(swapcontext, struct ucontext __user *, old_ctx, · 63 unmodified lines
655	677	ctx_has_vsx_region = 1;
656	678
657	679	if (old_ctx != NULL) {
658		− if (!access_ok(old_ctx, ctx_size)
659		− \|\| setup_sigcontext(&old_ctx->uc_mcontext, current, 0, NULL, 0,
660		− ctx_has_vsx_region)
661		− \|\| __copy_to_user(&old_ctx->uc_sigmask,
662		− &current->blocked, sizeof(sigset_t)))
	680	+ prepare_setup_sigcontext(current);
	681	+ if (!user_write_access_begin(old_ctx, ctx_size))
663	682	return -EFAULT;
	683	+
	684	+ unsafe_setup_sigcontext(&old_ctx->uc_mcontext, current, 0, NULL,
	685	+ 0, ctx_has_vsx_region, efault_out);
	686	+ unsafe_copy_to_user(&old_ctx->uc_sigmask, &current->blocked,
	687	+ sizeof(sigset_t), efault_out);
	688	+
	689	+ user_write_access_end();
664	690	}
665	691	if (new_ctx == NULL)
666	692	return 0;
@@ -680,15 +706,25 @@ SYSCALL_DEFINE3(swapcontext, struct ucontext __user *, old_ctx, · 13 unmodified lines
680	706	* We kill the task with a SIGSEGV in this situation.
681	707	*/
682	708
683		− if (__copy_from_user(&set, &new_ctx->uc_sigmask, sizeof(set)))
	709	+ if (__get_user_sigset(&set, &new_ctx->uc_sigmask))
684	710	do_exit(SIGSEGV);
685	711	set_current_blocked(&set);
686		− if (restore_sigcontext(current, NULL, 0, &new_ctx->uc_mcontext))
	712	+
	713	+ if (!user_read_access_begin(new_ctx, ctx_size))
	714	+ return -EFAULT;
	715	+ if (__unsafe_restore_sigcontext(current, NULL, 0, &new_ctx->uc_mcontext)) {
	716	+ user_read_access_end();
687	717	do_exit(SIGSEGV);
	718	+ }
	719	+ user_read_access_end();
688	720
689	721	/* This returns like rt_sigreturn */
690	722	set_thread_flag(TIF_RESTOREALL);
691	723	return 0;
	724	+
	725	+efault_out:
	726	+ user_write_access_end();
	727	+ return -EFAULT;
692	728	}
693	729
694	730
@@ -701,9 +737,7 @@ SYSCALL_DEFINE0(rt_sigreturn) · 6 unmodified lines
701	737	struct pt_regs *regs = current_pt_regs();
702	738	struct ucontext __user uc = (struct ucontext __user )regs->gpr[1];
703	739	sigset_t set;
704		−#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
705	740	unsigned long msr;
706		−#endif
707	741
708	742	/* Always make any pending restarted system calls return -EINTR */
709	743	current->restart_block.fn = do_no_restart_syscall;
@@ -711,52 +745,54 @@ SYSCALL_DEFINE0(rt_sigreturn) · 1 unmodified line
711	745	if (!access_ok(uc, sizeof(*uc)))
712	746	goto badframe;
713	747
714		− if (__copy_from_user(&set, &uc->uc_sigmask, sizeof(set)))
	748	+ if (__get_user_sigset(&set, &uc->uc_sigmask))
715	749	goto badframe;
716	750	set_current_blocked(&set);
717	751
718		−#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
719		− /*
720		− * If there is a transactional state then throw it away.
721		− * The purpose of a sigreturn is to destroy all traces of the
722		− * signal frame, this includes any transactional state created
723		− * within in. We only check for suspended as we can never be
724		− * active in the kernel, we are active, there is nothing better to
725		− * do than go ahead and Bad Thing later.
726		− * The cause is not important as there will never be a
727		− * recheckpoint so it's not user visible.
728		− */
729		− if (MSR_TM_SUSPENDED(mfmsr()))
730		− tm_reclaim_current(0);
	752	+ if (IS_ENABLED(CONFIG_PPC_TRANSACTIONAL_MEM)) {
	753	+ /*
	754	+ * If there is a transactional state then throw it away.
	755	+ * The purpose of a sigreturn is to destroy all traces of the
	756	+ * signal frame, this includes any transactional state created
	757	+ * within in. We only check for suspended as we can never be
	758	+ * active in the kernel, we are active, there is nothing better to
	759	+ * do than go ahead and Bad Thing later.
	760	+ * The cause is not important as there will never be a
	761	+ * recheckpoint so it's not user visible.
	762	+ */
	763	+ if (MSR_TM_SUSPENDED(mfmsr()))
	764	+ tm_reclaim_current(0);
731	765
732		− /*
733		− * Disable MSR[TS] bit also, so, if there is an exception in the
734		− * code below (as a page fault in copy_ckvsx_to_user()), it does
735		− * not recheckpoint this task if there was a context switch inside
736		− * the exception.
737		− *
738		− * A major page fault can indirectly call schedule(). A reschedule
739		− * process in the middle of an exception can have a side effect
740		− * (Changing the CPU MSR[TS] state), since schedule() is called
741		− * with the CPU MSR[TS] disable and returns with MSR[TS]=Suspended
742		− * (switch_to() calls tm_recheckpoint() for the 'new' process). In
743		− * this case, the process continues to be the same in the CPU, but
744		− * the CPU state just changed.
745		− *
746		− * This can cause a TM Bad Thing, since the MSR in the stack will
747		− * have the MSR[TS]=0, and this is what will be used to RFID.
748		− *
749		− * Clearing MSR[TS] state here will avoid a recheckpoint if there
750		− * is any process reschedule in kernel space. The MSR[TS] state
751		− * does not need to be saved also, since it will be replaced with
752		− * the MSR[TS] that came from user context later, at
753		− * restore_tm_sigcontexts.
754		− */
755		− regs->msr &= ~MSR_TS_MASK;
	766	+ /*
	767	+ * Disable MSR[TS] bit also, so, if there is an exception in the
	768	+ * code below (as a page fault in copy_ckvsx_to_user()), it does
	769	+ * not recheckpoint this task if there was a context switch inside
	770	+ * the exception.
	771	+ *
	772	+ * A major page fault can indirectly call schedule(). A reschedule
	773	+ * process in the middle of an exception can have a side effect
	774	+ * (Changing the CPU MSR[TS] state), since schedule() is called
	775	+ * with the CPU MSR[TS] disable and returns with MSR[TS]=Suspended
	776	+ * (switch_to() calls tm_recheckpoint() for the 'new' process). In
	777	+ * this case, the process continues to be the same in the CPU, but
	778	+ * the CPU state just changed.
	779	+ *
	780	+ * This can cause a TM Bad Thing, since the MSR in the stack will
	781	+ * have the MSR[TS]=0, and this is what will be used to RFID.
	782	+ *
	783	+ * Clearing MSR[TS] state here will avoid a recheckpoint if there
	784	+ * is any process reschedule in kernel space. The MSR[TS] state
	785	+ * does not need to be saved also, since it will be replaced with
	786	+ * the MSR[TS] that came from user context later, at
	787	+ * restore_tm_sigcontexts.
	788	+ */
	789	+ regs->msr &= ~MSR_TS_MASK;
756	790
757		− if (__get_user(msr, &uc->uc_mcontext.gp_regs[PT_MSR]))
758		− goto badframe;
759		− if (MSR_TM_ACTIVE(msr)) {
	791	+ if (__get_user(msr, &uc->uc_mcontext.gp_regs[PT_MSR]))
	792	+ goto badframe;
	793	+ }
	794	+
	795	+ if (IS_ENABLED(CONFIG_PPC_TRANSACTIONAL_MEM) && MSR_TM_ACTIVE(msr)) {
760	796	/* We recheckpoint on return. */
761	797	struct ucontext __user *uc_transact;
762	798
@@ -769,9 +805,7 @@ SYSCALL_DEFINE0(rt_sigreturn) · 6 unmodified lines
769	805	if (restore_tm_sigcontexts(current, &uc->uc_mcontext,
770	806	&uc_transact->uc_mcontext))
771	807	goto badframe;
772		− } else
773		−#endif
774		− {
	808	+ } else {
775	809	/*
776	810	* Fall through, for non-TM restore
777	811	*
@@ -785,8 +819,13 @@ SYSCALL_DEFINE0(rt_sigreturn) · 7 unmodified lines
785	819	* causing a TM bad thing.
786	820	*/
787	821	current->thread.regs->msr &= ~MSR_TS_MASK;
788		− if (restore_sigcontext(current, NULL, 1, &uc->uc_mcontext))
	822	+ if (!user_read_access_begin(&uc->uc_mcontext, sizeof(uc->uc_mcontext)))
789	823	goto badframe;
	824	+
	825	+ unsafe_restore_sigcontext(current, NULL, 1, &uc->uc_mcontext,
	826	+ badframe_block);
	827	+
	828	+ user_read_access_end();
790	829	}
791	830
792	831	if (restore_altstack(&uc->uc_stack))
@@ -795,6 +834,8 @@ SYSCALL_DEFINE0(rt_sigreturn) · 2 unmodified lines
795	834	set_thread_flag(TIF_RESTOREALL);
796	835	return 0;
797	836
	837	+badframe_block:
	838	+ user_read_access_end();
798	839	badframe:
799	840	signal_fault(current, regs, "rt_sigreturn", uc);
800	841
@@ -809,46 +850,57 @@ int handle_rt_signal64(struct ksignal ksig, sigset_t set, · 8 unmodified lines
809	850	unsigned long newsp = 0;
810	851	long err = 0;
811	852	struct pt_regs *regs = tsk->thread.regs;
812		−#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
813	853	/* Save the thread's msr before get_tm_stackpointer() changes it */
814	854	unsigned long msr = regs->msr;
815		−#endif
816	855
817	856	frame = get_sigframe(ksig, tsk, sizeof(*frame), 0);
818		− if (!access_ok(frame, sizeof(*frame)))
819		− goto badframe;
820	857
821		− err \|= __put_user(&frame->info, &frame->pinfo);
822		− err \|= __put_user(&frame->uc, &frame->puc);
823		− err \|= copy_siginfo_to_user(&frame->info, &ksig->info);
824		− if (err)
	858	+ /*
	859	+ * This only applies when calling unsafe_setup_sigcontext() and must be
	860	+ * called before opening the uaccess window.
	861	+ */
	862	+ if (!MSR_TM_ACTIVE(msr))
	863	+ prepare_setup_sigcontext(tsk);
	864	+
	865	+ if (!user_write_access_begin(frame, sizeof(*frame)))
825	866	goto badframe;
826	867
	868	+ unsafe_put_user(&frame->info, &frame->pinfo, badframe_block);
	869	+ unsafe_put_user(&frame->uc, &frame->puc, badframe_block);
	870	+
827	871	/* Create the ucontext. */
828		− err \|= __put_user(0, &frame->uc.uc_flags);
829		− err \|= __save_altstack(&frame->uc.uc_stack, regs->gpr[1]);
830		−#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
	872	+ unsafe_put_user(0, &frame->uc.uc_flags, badframe_block);
	873	+ unsafe_save_altstack(&frame->uc.uc_stack, regs->gpr[1], badframe_block);
	874	+
831	875	if (MSR_TM_ACTIVE(msr)) {
	876	+#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
832	877	/* The ucontext_t passed to userland points to the second
833	878	* ucontext_t (for transactional state) with its uc_link ptr.
834	879	*/
835		− err \|= __put_user(&frame->uc_transact, &frame->uc.uc_link);
	880	+ unsafe_put_user(&frame->uc_transact, &frame->uc.uc_link, badframe_block);
	881	+
	882	+ user_write_access_end();
	883	+
836	884	err \|= setup_tm_sigcontexts(&frame->uc.uc_mcontext,
837	885	&frame->uc_transact.uc_mcontext,
838	886	tsk, ksig->sig, NULL,
839	887	(unsigned long)ksig->ka.sa.sa_handler,
840	888	msr);
841		− } else
	889	+
	890	+ if (!user_write_access_begin(&frame->uc.uc_sigmask,
	891	+ sizeof(frame->uc.uc_sigmask)))
	892	+ goto badframe;
	893	+
842	894	#endif
843		− {
844		− err \|= __put_user(0, &frame->uc.uc_link);
845		− err \|= setup_sigcontext(&frame->uc.uc_mcontext, tsk, ksig->sig,
	895	+ } else {
	896	+ unsafe_put_user(0, &frame->uc.uc_link, badframe_block);
	897	+ unsafe_setup_sigcontext(&frame->uc.uc_mcontext, tsk, ksig->sig,
846	898	NULL, (unsigned long)ksig->ka.sa.sa_handler,
847		− 1);
	899	+ 1, badframe_block);
848	900	}
849		− err \|= __copy_to_user(&frame->uc.uc_sigmask, set, sizeof(*set));
850		− if (err)
851		− goto badframe;
	901	+
	902	+ unsafe_copy_to_user(&frame->uc.uc_sigmask, set, sizeof(*set), badframe_block);
	903	+ user_write_access_end();
852	904
853	905	/* Make sure signal handler doesn't get spurious FP exceptions */
854	906	tsk->thread.fp_state.fpscr = 0;
@@ -863,6 +915,11 @@ int handle_rt_signal64(struct ksignal ksig, sigset_t set, · 8 unmodified lines
863	915	regs->nip = (unsigned long) &frame->tramp[0];
864	916	}
865	917
	918	+
	919	+ /* Save the siginfo outside of the unsafe block. */
	920	+ if (copy_siginfo_to_user(&frame->info, &ksig->info))
	921	+ goto badframe;
	922	+
866	923	/* Allocate a dummy caller frame for the signal handler. */
867	924	newsp = ((unsigned long)frame) - __SIGNAL_FRAMESIZE;
868	925	err \|= put_user(regs->gpr[1], (unsigned long __user *)newsp);
@@ -902,6 +959,8 @@ int handle_rt_signal64(struct ksignal ksig, sigset_t set, · 33 unmodified lines
902	959
903	960	return 0;
904	961
	962	+badframe_block:
	963	+ user_write_access_end();
905	964	badframe:
906	965	signal_fault(current, regs, "handle_rt_signal64", frame);
907	966

arch/powerpc/kernel/smp.c+55−13

@@ -83,7 +83,7 @@ DEFINE_PER_CPU(cpumask_var_t, cpu_sibling_map); · 82 unmodified lines
83	83	DEFINE_PER_CPU(cpumask_var_t, cpu_smallcore_map);
84	84	DEFINE_PER_CPU(cpumask_var_t, cpu_l2_cache_map);
85	85	DEFINE_PER_CPU(cpumask_var_t, cpu_core_map);
86		−DEFINE_PER_CPU(cpumask_var_t, cpu_coregroup_map);
	86	+static DEFINE_PER_CPU(cpumask_var_t, cpu_coregroup_map);
87	87
88	88	EXPORT_PER_CPU_SYMBOL(cpu_sibling_map);
89	89	EXPORT_PER_CPU_SYMBOL(cpu_l2_cache_map);
@@ -122,14 +122,14 @@ static struct thread_groups_list tgl[NR_CPUS] __initdata; · 32 unmodified lines
122	122	* On big-cores system, thread_group_l1_cache_map for each CPU corresponds to
123	123	* the set its siblings that share the L1-cache.
124	124	*/
125		−DEFINE_PER_CPU(cpumask_var_t, thread_group_l1_cache_map);
	125	+static DEFINE_PER_CPU(cpumask_var_t, thread_group_l1_cache_map);
126	126
127	127	/*
128	128	* On some big-cores system, thread_group_l2_cache_map for each CPU
129	129	* corresponds to the set its siblings within the core that share the
130	130	* L2-cache.
131	131	*/
132		−DEFINE_PER_CPU(cpumask_var_t, thread_group_l2_cache_map);
	132	+static DEFINE_PER_CPU(cpumask_var_t, thread_group_l2_cache_map);
133	133
134	134	/* SMP operations for this machine */
135	135	struct smp_ops_t *smp_ops;
@@ -1057,17 +1057,12 @@ void __init smp_prepare_cpus(unsigned int max_cpus) · 921 unmodified lines
1057	1057	local_memory_node(numa_cpu_lookup_table[cpu]));
1058	1058	}
1059	1059	#endif
1060		− /*
1061		− * cpu_core_map is now more updated and exists only since
1062		− * its been exported for long. It only will have a snapshot
1063		− * of cpu_cpu_mask.
1064		− */
1065		− cpumask_copy(per_cpu(cpu_core_map, cpu), cpu_cpu_mask(cpu));
1066	1060	}
1067	1061
1068	1062	/* Init the cpumasks so the boot CPU is related to itself */
1069	1063	cpumask_set_cpu(boot_cpuid, cpu_sibling_mask(boot_cpuid));
1070	1064	cpumask_set_cpu(boot_cpuid, cpu_l2_cache_mask(boot_cpuid));
	1065	+ cpumask_set_cpu(boot_cpuid, cpu_core_mask(boot_cpuid));
1071	1066
1072	1067	if (has_coregroup_support())
1073	1068	cpumask_set_cpu(boot_cpuid, cpu_coregroup_mask(boot_cpuid));
@@ -1078,6 +1073,20 @@ void __init smp_prepare_cpus(unsigned int max_cpus) · 4 unmodified lines
1078	1073	cpu_smallcore_mask(boot_cpuid));
1079	1074	}
1080	1075
	1076	+ if (cpu_to_chip_id(boot_cpuid) != -1) {
	1077	+ int idx = num_possible_cpus() / threads_per_core;
	1078	+
	1079	+ /*
	1080	+ * All threads of a core will all belong to the same core,
	1081	+ * chip_id_lookup_table will have one entry per core.
	1082	+ * Assumption: if boot_cpuid doesn't have a chip-id, then no
	1083	+ * other CPUs, will also not have chip-id.
	1084	+ */
	1085	+ chip_id_lookup_table = kcalloc(idx, sizeof(int), GFP_KERNEL);
	1086	+ if (chip_id_lookup_table)
	1087	+ memset(chip_id_lookup_table, -1, sizeof(int) * idx);
	1088	+ }
	1089	+
1081	1090	if (smp_ops && smp_ops->probe)
1082	1091	smp_ops->probe();
1083	1092	}
@@ -1408,6 +1417,9 @@ static void remove_cpu_from_masks(int cpu) · 324 unmodified lines
1408	1417	set_cpus_unrelated(cpu, i, cpu_smallcore_mask);
1409	1418	}
1410	1419
	1420	+ for_each_cpu(i, cpu_core_mask(cpu))
	1421	+ set_cpus_unrelated(cpu, i, cpu_core_mask);
	1422	+
1411	1423	if (has_coregroup_support()) {
1412	1424	for_each_cpu(i, cpu_coregroup_mask(cpu))
1413	1425	set_cpus_unrelated(cpu, i, cpu_coregroup_mask);
@@ -1468,8 +1480,11 @@ static void update_coregroup_mask(int cpu, cpumask_var_t *mask) · 54 unmodified lines
1468	1480
1469	1481	static void add_cpu_to_masks(int cpu)
1470	1482	{
	1483	+ struct cpumask (submask_fn)(int) = cpu_sibling_mask;
1471	1484	int first_thread = cpu_first_thread_sibling(cpu);
1472	1485	cpumask_var_t mask;
	1486	+ int chip_id = -1;
	1487	+ bool ret;
1473	1488	int i;
1474	1489
1475	1490	/*
@@ -1485,12 +1500,39 @@ static void add_cpu_to_masks(int cpu) · 9 unmodified lines
1485	1500	add_cpu_to_smallcore_masks(cpu);
1486	1501
1487	1502	/* In CPU-hotplug path, hence use GFP_ATOMIC */
1488		− alloc_cpumask_var_node(&mask, GFP_ATOMIC, cpu_to_node(cpu));
	1503	+ ret = alloc_cpumask_var_node(&mask, GFP_ATOMIC, cpu_to_node(cpu));
1489	1504	update_mask_by_l2(cpu, &mask);
1490	1505
1491	1506	if (has_coregroup_support())
1492	1507	update_coregroup_mask(cpu, &mask);
1493	1508
	1509	+ if (chip_id_lookup_table && ret)
	1510	+ chip_id = cpu_to_chip_id(cpu);
	1511	+
	1512	+ if (chip_id == -1) {
	1513	+ cpumask_copy(per_cpu(cpu_core_map, cpu), cpu_cpu_mask(cpu));
	1514	+ goto out;
	1515	+ }
	1516	+
	1517	+ if (shared_caches)
	1518	+ submask_fn = cpu_l2_cache_mask;
	1519	+
	1520	+ /* Update core_mask with all the CPUs that are part of submask */
	1521	+ or_cpumasks_related(cpu, cpu, submask_fn, cpu_core_mask);
	1522	+
	1523	+ /* Skip all CPUs already part of current CPU core mask */
	1524	+ cpumask_andnot(mask, cpu_online_mask, cpu_core_mask(cpu));
	1525	+
	1526	+ for_each_cpu(i, mask) {
	1527	+ if (chip_id == cpu_to_chip_id(i)) {
	1528	+ or_cpumasks_related(cpu, i, submask_fn, cpu_core_mask);
	1529	+ cpumask_andnot(mask, mask, submask_fn(i));
	1530	+ } else {
	1531	+ cpumask_andnot(mask, mask, cpu_core_mask(i));
	1532	+ }
	1533	+ }
	1534	+
	1535	+out:
1494	1536	free_cpumask_var(mask);
1495	1537	}
1496	1538
@@ -1521,6 +1563,9 @@ void start_secondary(void *unused) · 24 unmodified lines
1521	1563
1522	1564	vdso_getcpu_init();
1523	1565	#endif
	1566	+ set_numa_node(numa_cpu_lookup_table[cpu]);
	1567	+ set_numa_mem(local_memory_node(numa_cpu_lookup_table[cpu]));
	1568	+
1524	1569	/* Update topology CPU masks */
1525	1570	add_cpu_to_masks(cpu);
1526	1571
@@ -1539,9 +1584,6 @@ void start_secondary(void *unused) · 12 unmodified lines
1539	1584	shared_caches = true;
1540	1585	}
1541	1586
1542		− set_numa_node(numa_cpu_lookup_table[cpu]);
1543		− set_numa_mem(local_memory_node(numa_cpu_lookup_table[cpu]));
1544		−
1545	1587	smp_wmb();
1546	1588	notify_cpu_starting(cpu);
1547	1589	set_cpu_online(cpu, true);

arch/powerpc/kernel/stacktrace.c+25−83

@@ -23,90 +23,56 @@ · 22 unmodified lines
23	23
24	24	#include <asm/paca.h>
25	25
26		−/*
27		− * Save stack-backtrace addresses into a stack_trace buffer.
28		− */
29		−static void save_context_stack(struct stack_trace *trace, unsigned long sp,
30		− struct task_struct *tsk, int savesched)
	26	+void arch_stack_walk(stack_trace_consume_fn consume_entry, void *cookie,
	27	+ struct task_struct task, struct pt_regs regs)
31	28	{
	29	+ unsigned long sp;
	30	+
	31	+ if (regs && !consume_entry(cookie, regs->nip))
	32	+ return;
	33	+
	34	+ if (regs)
	35	+ sp = regs->gpr[1];
	36	+ else if (task == current)
	37	+ sp = current_stack_frame();
	38	+ else
	39	+ sp = task->thread.ksp;
	40	+
32	41	for (;;) {
33	42	unsigned long stack = (unsigned long ) sp;
34	43	unsigned long newsp, ip;
35	44
36		− if (!validate_sp(sp, tsk, STACK_FRAME_OVERHEAD))
	45	+ if (!validate_sp(sp, task, STACK_FRAME_OVERHEAD))
37	46	return;
38	47
39	48	newsp = stack[0];
40	49	ip = stack[STACK_FRAME_LR_SAVE];
41	50
42		− if (savesched \|\| !in_sched_functions(ip)) {
43		− if (!trace->skip)
44		− trace->entries[trace->nr_entries++] = ip;
45		− else
46		− trace->skip--;
47		− }
48		−
49		− if (trace->nr_entries >= trace->max_entries)
	51	+ if (!consume_entry(cookie, ip))
50	52	return;
51	53
52	54	sp = newsp;
53	55	}
54	56	}
55	57
56		−void save_stack_trace(struct stack_trace *trace)
57		−{
58		− unsigned long sp;
59		−
60		− sp = current_stack_frame();
61		−
62		− save_context_stack(trace, sp, current, 1);
63		−}
64		−EXPORT_SYMBOL_GPL(save_stack_trace);
65		−
66		−void save_stack_trace_tsk(struct task_struct tsk, struct stack_trace trace)
67		−{
68		− unsigned long sp;
69		−
70		− if (!try_get_task_stack(tsk))
71		− return;
72		−
73		− if (tsk == current)
74		− sp = current_stack_frame();
75		− else
76		− sp = tsk->thread.ksp;
77		−
78		− save_context_stack(trace, sp, tsk, 0);
79		−
80		− put_task_stack(tsk);
81		−}
82		−EXPORT_SYMBOL_GPL(save_stack_trace_tsk);
83		−
84		−void
85		−save_stack_trace_regs(struct pt_regs regs, struct stack_trace trace)
86		−{
87		− save_context_stack(trace, regs->gpr[1], current, 0);
88		−}
89		−EXPORT_SYMBOL_GPL(save_stack_trace_regs);
90		−
91		−#ifdef CONFIG_HAVE_RELIABLE_STACKTRACE
92	58	/*
93	59	* This function returns an error if it detects any unreliable features of the
94	60	* stack. Otherwise it guarantees that the stack trace is reliable.
95	61	*
96	62	* If the task is not 'current', the caller must ensure the task is inactive.
97	63	*/
98		−static int __save_stack_trace_tsk_reliable(struct task_struct *tsk,
99		− struct stack_trace *trace)
	64	+int arch_stack_walk_reliable(stack_trace_consume_fn consume_entry,
	65	+ void cookie, struct task_struct task)
100	66	{
101	67	unsigned long sp;
102	68	unsigned long newsp;
103		− unsigned long stack_page = (unsigned long)task_stack_page(tsk);
	69	+ unsigned long stack_page = (unsigned long)task_stack_page(task);
104	70	unsigned long stack_end;
105	71	int graph_idx = 0;
106	72	bool firstframe;
107	73
108	74	stack_end = stack_page + THREAD_SIZE;
109		− if (!is_idle_task(tsk)) {
	75	+ if (!is_idle_task(task)) {
110	76	/*
111	77	* For user tasks, this is the SP value loaded on
112	78	* kernel entry, see "PACAKSAVE(r13)" in _switch() and
@@ -130,10 +96,10 @@ static int __save_stack_trace_tsk_reliable(struct task_struct *tsk, · 17 unmodified lines
130	96	stack_end -= STACK_FRAME_OVERHEAD;
131	97	}
132	98
133		− if (tsk == current)
	99	+ if (task == current)
134	100	sp = current_stack_frame();
135	101	else
136		− sp = tsk->thread.ksp;
	102	+ sp = task->thread.ksp;
137	103
138	104	if (sp < stack_page + sizeof(struct thread_struct) \|\|
139	105	sp > stack_end - STACK_FRAME_MIN_SIZE) {
@@ -182,7 +148,7 @@ static int __save_stack_trace_tsk_reliable(struct task_struct *tsk, · 42 unmodified lines
182	148	* FIXME: IMHO these tests do not belong in
183	149	* arch-dependent code, they are generic.
184	150	*/
185		− ip = ftrace_graph_ret_addr(tsk, &graph_idx, ip, stack);
	151	+ ip = ftrace_graph_ret_addr(task, &graph_idx, ip, stack);
186	152	#ifdef CONFIG_KPROBES
187	153	/*
188	154	* Mark stacktraces with kretprobed functions on them
@@ -192,36 +158,12 @@ static int __save_stack_trace_tsk_reliable(struct task_struct *tsk, · 3 unmodified lines
192	158	return -EINVAL;
193	159	#endif
194	160
195		− if (trace->nr_entries >= trace->max_entries)
196		− return -E2BIG;
197		− if (!trace->skip)
198		− trace->entries[trace->nr_entries++] = ip;
199		− else
200		− trace->skip--;
	161	+ if (!consume_entry(cookie, ip))
	162	+ return -EINVAL;
201	163	}
202	164	return 0;
203	165	}
204	166
205		−int save_stack_trace_tsk_reliable(struct task_struct *tsk,
206		− struct stack_trace *trace)
207		−{
208		− int ret;
209		−
210		− /*
211		− * If the task doesn't have a stack (e.g., a zombie), the stack is
212		− * "reliably" empty.
213		− */
214		− if (!try_get_task_stack(tsk))
215		− return 0;
216		−
217		− ret = __save_stack_trace_tsk_reliable(tsk, trace);
218		−
219		− put_task_stack(tsk);
220		−
221		− return ret;
222		−}
223		−#endif /* CONFIG_HAVE_RELIABLE_STACKTRACE */
224		−
225	167	#if defined(CONFIG_PPC_BOOK3S_64) && defined(CONFIG_NMI_IPI)
226	168	static void handle_backtrace_ipi(struct pt_regs *regs)
227	169	{

arch/powerpc/kernel/syscalls.c+2−10

@@ -82,16 +82,8 @@ int · 81 unmodified lines
82	82	ppc_select(int n, fd_set __user inp, fd_set __user outp, fd_set __user exp, struct __kernel_old_timeval __user tvp)
83	83	{
84	84	if ( (unsigned long)n >= 4096 )
85		− {
86		− unsigned long __user buffer = (unsigned long __user )n;
87		− if (!access_ok(buffer, 5*sizeof(unsigned long))
88		− \|\| __get_user(n, buffer)
89		− \|\| __get_user(inp, ((fd_set __user * __user *)(buffer+1)))
90		− \|\| __get_user(outp, ((fd_set __user * __user *)(buffer+2)))
91		− \|\| __get_user(exp, ((fd_set __user * __user *)(buffer+3)))
92		− \|\| __get_user(tvp, ((struct __kernel_old_timeval __user * __user *)(buffer+4))))
93		− return -EFAULT;
94		− }
	85	+ return sys_old_select((void __user *)n);
	86	+
95	87	return sys_select(n, inp, outp, exp, tvp);
96	88	}
97	89	#endif

arch/powerpc/kernel/syscalls/Makefile+9−24

@@ -6,53 +6,38 @@ _dummy := $(shell [ -d '$(uapi)' ] \|\| mkdir -p '$(uapi)') \ · 5 unmodified lines
6	6	$(shell [ -d '$(kapi)' ] \|\| mkdir -p '$(kapi)')
7	7
8	8	syscall := $(src)/syscall.tbl
9		−syshdr := $(srctree)/$(src)/syscallhdr.sh
10		−systbl := $(srctree)/$(src)/syscalltbl.sh
	9	+syshdr := $(srctree)/scripts/syscallhdr.sh
	10	+systbl := $(srctree)/scripts/syscalltbl.sh
11	11
12	12	quiet_cmd_syshdr = SYSHDR $@
13		− cmd_syshdr = $(CONFIG_SHELL) '$(syshdr)' '$<' '$@' \
14		− '$(syshdr_abis_$(basetarget))' \
15		− '$(syshdr_pfx_$(basetarget))' \
16		− '$(syshdr_offset_$(basetarget))'
	13	+ cmd_syshdr = $(CONFIG_SHELL) $(syshdr) --emit-nr --abis $(abis) $< $@
17	14
18	15	quiet_cmd_systbl = SYSTBL $@
19		− cmd_systbl = $(CONFIG_SHELL) '$(systbl)' '$<' '$@' \
20		− '$(systbl_abis_$(basetarget))' \
21		− '$(systbl_abi_$(basetarget))' \
22		− '$(systbl_offset_$(basetarget))'
	16	+ cmd_systbl = $(CONFIG_SHELL) $(systbl) --abis $(abis) $< $@
23	17
24		−syshdr_abis_unistd_32 := common,nospu,32
	18	+$(uapi)/unistd_32.h: abis := common,nospu,32
25	19	$(uapi)/unistd_32.h: $(syscall) $(syshdr) FORCE
26	20	$(call if_changed,syshdr)
27	21
28		−syshdr_abis_unistd_64 := common,nospu,64
	22	+$(uapi)/unistd_64.h: abis := common,nospu,64
29	23	$(uapi)/unistd_64.h: $(syscall) $(syshdr) FORCE
30	24	$(call if_changed,syshdr)
31	25
32		−systbl_abis_syscall_table_32 := common,nospu,32
33		−systbl_abi_syscall_table_32 := 32
	26	+$(kapi)/syscall_table_32.h: abis := common,nospu,32
34	27	$(kapi)/syscall_table_32.h: $(syscall) $(systbl) FORCE
35	28	$(call if_changed,systbl)
36	29
37		−systbl_abis_syscall_table_64 := common,nospu,64
38		−systbl_abi_syscall_table_64 := 64
	30	+$(kapi)/syscall_table_64.h: abis := common,nospu,64
39	31	$(kapi)/syscall_table_64.h: $(syscall) $(systbl) FORCE
40	32	$(call if_changed,systbl)
41	33
42		−systbl_abis_syscall_table_c32 := common,nospu,32
43		−systbl_abi_syscall_table_c32 := c32
44		−$(kapi)/syscall_table_c32.h: $(syscall) $(systbl) FORCE
45		− $(call if_changed,systbl)
46		−
47		−systbl_abis_syscall_table_spu := common,spu
48		−systbl_abi_syscall_table_spu := spu
	34	+$(kapi)/syscall_table_spu.h: abis := common,spu
49	35	$(kapi)/syscall_table_spu.h: $(syscall) $(systbl) FORCE
50	36	$(call if_changed,systbl)
51	37
52	38	uapisyshdr-y += unistd_32.h unistd_64.h
53	39	kapisyshdr-y += syscall_table_32.h \
54	40	syscall_table_64.h \
55		− syscall_table_c32.h \
56	41	syscall_table_spu.h
57	42
58	43	uapisyshdr-y := $(addprefix $(uapi)/, $(uapisyshdr-y))

arch/powerpc/kernel/syscalls/syscallhdr.sh+0−36

@@ -1,36 +0,0 @@
1		−#!/bin/sh
2		−# SPDX-License-Identifier: GPL-2.0
3		−
4		−in="$1"
5		−out="$2"
6		−my_abis=`echo "($3)" \| tr ',' '\|'`
7		−prefix="$4"
8		−offset="$5"
9		−
10		−fileguard=_UAPI_ASM_POWERPC_`basename "$out" \| sed \
11		− -e 'y/abcdefghijklmnopqrstuvwxyz/ABCDEFGHIJKLMNOPQRSTUVWXYZ/' \
12		− -e 's/[^A-Z0-9_]/_/g' -e 's/__/_/g'`
13		−grep -E "^[0-9A-Fa-fXx]+[[:space:]]+${my_abis}" "$in" \| sort -n \| (
14		− printf "#ifndef %s\n" "${fileguard}"
15		− printf "#define %s\n" "${fileguard}"
16		− printf "\n"
17		−
18		− nxt=0
19		− while read nr abi name entry compat ; do
20		− if [ -z "$offset" ]; then
21		− printf "#define __NR_%s%s\t%s\n" \
22		− "${prefix}" "${name}" "${nr}"
23		− else
24		− printf "#define __NR_%s%s\t(%s + %s)\n" \
25		− "${prefix}" "${name}" "${offset}" "${nr}"
26		− fi
27		− nxt=$((nr+1))
28		− done
29		−
30		− printf "\n"
31		− printf "#ifdef __KERNEL__\n"
32		− printf "#define __NR_syscalls\t%s\n" "${nxt}"
33		− printf "#endif\n"
34		− printf "\n"
35		− printf "#endif /* %s */\n" "${fileguard}"
36		−) > "$out"

arch/powerpc/kernel/syscalls/syscalltbl.sh+0−36

@@ -1,36 +0,0 @@
1		−#!/bin/sh
2		−# SPDX-License-Identifier: GPL-2.0
3		−
4		−in="$1"
5		−out="$2"
6		−my_abis=`echo "($3)" \| tr ',' '\|'`
7		−my_abi="$4"
8		−offset="$5"
9		−
10		−emit() {
11		− t_nxt="$1"
12		− t_nr="$2"
13		− t_entry="$3"
14		−
15		− while [ $t_nxt -lt $t_nr ]; do
16		− printf "__SYSCALL(%s,sys_ni_syscall)\n" "${t_nxt}"
17		− t_nxt=$((t_nxt+1))
18		− done
19		− printf "__SYSCALL(%s,%s)\n" "${t_nxt}" "${t_entry}"
20		−}
21		−
22		−grep -E "^[0-9A-Fa-fXx]+[[:space:]]+${my_abis}" "$in" \| sort -n \| (
23		− nxt=0
24		− if [ -z "$offset" ]; then
25		− offset=0
26		− fi
27		−
28		− while read nr abi name entry compat ; do
29		− if [ "$my_abi" = "c32" ] && [ ! -z "$compat" ]; then
30		− emit $((nxt+offset)) $((nr+offset)) $compat
31		− else
32		− emit $((nxt+offset)) $((nr+offset)) $entry
33		− fi
34		− nxt=$((nr+1))
35		− done
36		−) > "$out"

arch/powerpc/kernel/systbl.S+4−1

@@ -21,6 +21,7 @@ · 20 unmodified lines
21	21	#define __SYSCALL(nr, entry) .long entry
22	22	#endif
23	23
	24	+#define __SYSCALL_WITH_COMPAT(nr, native, compat) __SYSCALL(nr, native)
24	25	.globl sys_call_table
25	26	sys_call_table:
26	27	#ifdef CONFIG_PPC64
@@ -30,8 +31,10 @@ sys_call_table: · 3 unmodified lines
30	31	#endif
31	32
32	33	#ifdef CONFIG_COMPAT
	34	+#undef __SYSCALL_WITH_COMPAT
	35	+#define __SYSCALL_WITH_COMPAT(nr, native, compat) __SYSCALL(nr, compat)
33	36	.globl compat_sys_call_table
34	37	compat_sys_call_table:
35	38	#define compat_sys_sigsuspend sys_sigsuspend
36		−#include <asm/syscall_table_c32.h>
	39	+#include <asm/syscall_table_32.h>
37	40	#endif

arch/powerpc/kernel/trace/ftrace.c+11−11

@@ -68,7 +68,7 @@ ftrace_modify_code(unsigned long ip, struct ppc_inst old, struct ppc_inst new) · 67 unmodified lines
68	68	*/
69	69
70	70	/* read the text we want to modify */
71		− if (probe_kernel_read_inst(&replaced, (void *)ip))
	71	+ if (copy_inst_from_kernel_nofault(&replaced, (void *)ip))
72	72	return -EFAULT;
73	73
74	74	/* Make sure it is what we expect it to be */
@@ -130,7 +130,7 @@ __ftrace_make_nop(struct module *mod, · 55 unmodified lines
130	130	struct ppc_inst op, pop;
131	131
132	132	/* read where this goes */
133		− if (probe_kernel_read_inst(&op, (void *)ip)) {
	133	+ if (copy_inst_from_kernel_nofault(&op, (void *)ip)) {
134	134	pr_err("Fetching opcode failed.\n");
135	135	return -EFAULT;
136	136	}
@@ -164,7 +164,7 @@ __ftrace_make_nop(struct module *mod, · 27 unmodified lines
164	164	/* When using -mkernel_profile there is no load to jump over */
165	165	pop = ppc_inst(PPC_INST_NOP);
166	166
167		− if (probe_kernel_read_inst(&op, (void *)(ip - 4))) {
	167	+ if (copy_inst_from_kernel_nofault(&op, (void *)(ip - 4))) {
168	168	pr_err("Fetching instruction at %lx failed.\n", ip - 4);
169	169	return -EFAULT;
170	170	}
@@ -197,7 +197,7 @@ __ftrace_make_nop(struct module *mod, · 26 unmodified lines
197	197	* Check what is in the next instruction. We can see ld r2,40(r1), but
198	198	* on first pass after boot we will see mflr r0.
199	199	*/
200		− if (probe_kernel_read_inst(&op, (void *)(ip + 4))) {
	200	+ if (copy_inst_from_kernel_nofault(&op, (void *)(ip + 4))) {
201	201	pr_err("Fetching op failed.\n");
202	202	return -EFAULT;
203	203	}
@@ -349,7 +349,7 @@ static int setup_mcount_compiler_tramp(unsigned long tramp) · 145 unmodified lines
349	349	return -1;
350	350
351	351	/* New trampoline -- read where this goes */
352		− if (probe_kernel_read_inst(&op, (void *)tramp)) {
	352	+ if (copy_inst_from_kernel_nofault(&op, (void *)tramp)) {
353	353	pr_debug("Fetching opcode failed.\n");
354	354	return -1;
355	355	}
@@ -399,7 +399,7 @@ static int __ftrace_make_nop_kernel(struct dyn_ftrace *rec, unsigned long addr) · 43 unmodified lines
399	399	struct ppc_inst op;
400	400
401	401	/* Read where this goes */
402		− if (probe_kernel_read_inst(&op, (void *)ip)) {
	402	+ if (copy_inst_from_kernel_nofault(&op, (void *)ip)) {
403	403	pr_err("Fetching opcode failed.\n");
404	404	return -EFAULT;
405	405	}
@@ -526,10 +526,10 @@ __ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) · 120 unmodified lines
526	526	struct module *mod = rec->arch.mod;
527	527
528	528	/* read where this goes */
529		− if (probe_kernel_read_inst(op, ip))
	529	+ if (copy_inst_from_kernel_nofault(op, ip))
530	530	return -EFAULT;
531	531
532		− if (probe_kernel_read_inst(op + 1, ip + 4))
	532	+ if (copy_inst_from_kernel_nofault(op + 1, ip + 4))
533	533	return -EFAULT;
534	534
535	535	if (!expected_nop_sequence(ip, op[0], op[1])) {
@@ -592,7 +592,7 @@ __ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) · 56 unmodified lines
592	592	unsigned long ip = rec->ip;
593	593
594	594	/* read where this goes */
595		− if (probe_kernel_read_inst(&op, (void *)ip))
	595	+ if (copy_inst_from_kernel_nofault(&op, (void *)ip))
596	596	return -EFAULT;
597	597
598	598	/* It should be pointing to a nop */
@@ -648,7 +648,7 @@ static int __ftrace_make_call_kernel(struct dyn_ftrace *rec, unsigned long addr) · 49 unmodified lines
648	648	}
649	649
650	650	/* Make sure we have a nop */
651		− if (probe_kernel_read_inst(&op, ip)) {
	651	+ if (copy_inst_from_kernel_nofault(&op, ip)) {
652	652	pr_err("Unable to read ftrace location %p\n", ip);
653	653	return -EFAULT;
654	654	}
@@ -726,7 +726,7 @@ __ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr, · 71 unmodified lines
726	726	}
727	727
728	728	/* read where this goes */
729		− if (probe_kernel_read_inst(&op, (void *)ip)) {
	729	+ if (copy_inst_from_kernel_nofault(&op, (void *)ip)) {
730	730	pr_err("Fetching opcode failed.\n");
731	731	return -EFAULT;
732	732	}

arch/powerpc/kernel/traps.c+21−19

@@ -53,7 +53,6 @@ · 52 unmodified lines
53	53	#ifdef CONFIG_PPC64
54	54	#include <asm/firmware.h>
55	55	#include <asm/processor.h>
56		−#include <asm/tm.h>
57	56	#endif
58	57	#include <asm/kexec.h>
59	58	#include <asm/ppc-opcode.h>
@@ -222,7 +221,7 @@ static void oops_end(unsigned long flags, struct pt_regs *regs, · 162 unmodified lines
222	221	/*
223	222	* system_reset_excption handles debugger, crash dump, panic, for 0x100
224	223	*/
225		− if (TRAP(regs) == 0x100)
	224	+ if (TRAP(regs) == INTERRUPT_SYSTEM_RESET)
226	225	return;
227	226
228	227	crash_fadump(regs, "die oops");
@@ -290,7 +289,7 @@ void die(const char str, struct pt_regs regs, long err) · 61 unmodified lines
290	289	/*
291	290	* system_reset_excption handles debugger, crash dump, panic, for 0x100
292	291	*/
293		− if (TRAP(regs) != 0x100) {
	292	+ if (TRAP(regs) != INTERRUPT_SYSTEM_RESET) {
294	293	if (debugger(regs))
295	294	return;
296	295	}
@@ -405,7 +404,7 @@ void hv_nmi_check_nonrecoverable(struct pt_regs *regs) · 108 unmodified lines
405	404	* Now test if the interrupt has hit a range that may be using
406	405	* HSPRG1 without having RI=0 (i.e., an HSRR interrupt). The
407	406	* problem ranges all run un-relocated. Test real and virt modes
408		− * at the same time by droping the high bit of the nip (virt mode
	407	+ * at the same time by dropping the high bit of the nip (virt mode
409	408	* entry points still have the +0x4000 offset).
410	409	*/
411	410	nip &= ~0xc000000000000000ULL;
@@ -864,7 +863,7 @@ static void p9_hmi_special_emu(struct pt_regs *regs) · 452 unmodified lines
864	863	unsigned long ea, msr, msr_mask;
865	864	bool swap;
866	865
867		− if (__get_user_inatomic(instr, (unsigned int __user *)regs->nip))
	866	+ if (__get_user(instr, (unsigned int __user *)regs->nip))
868	867	return;
869	868
870	869	/*
@@ -1079,6 +1078,16 @@ DEFINE_INTERRUPT_HANDLER_ASYNC(unknown_async_exception) · 208 unmodified lines
1079	1078	_exception(SIGTRAP, regs, TRAP_UNK, 0);
1080	1079	}
1081	1080
	1081	+DEFINE_INTERRUPT_HANDLER_NMI(unknown_nmi_exception)
	1082	+{
	1083	+ printk("Bad trap at PC: %lx, SR: %lx, vector=%lx\n",
	1084	+ regs->nip, regs->msr, regs->trap);
	1085	+
	1086	+ _exception(SIGTRAP, regs, TRAP_UNK, 0);
	1087	+
	1088	+ return 0;
	1089	+}
	1090	+
1082	1091	DEFINE_INTERRUPT_HANDLER(instruction_breakpoint_exception)
1083	1092	{
1084	1093	if (notify_die(DIE_IABR_MATCH, "iabr_match", regs, 5,
@@ -1309,7 +1318,6 @@ static int emulate_instruction(struct pt_regs *regs) · 224 unmodified lines
1309	1318
1310	1319	if (!user_mode(regs))
1311	1320	return -EINVAL;
1312		− CHECK_FULL_REGS(regs);
1313	1321
1314	1322	if (get_user(instword, (u32 __user *)(regs->nip)))
1315	1323	return -EFAULT;
@@ -1406,7 +1414,6 @@ int is_valid_bugaddr(unsigned long addr) · 90 unmodified lines
1406	1414	static int emulate_math(struct pt_regs *regs)
1407	1415	{
1408	1416	int ret;
1409		− extern int do_mathemu(struct pt_regs *regs);
1410	1417
1411	1418	ret = do_mathemu(regs);
1412	1419	if (ret >= 0)
@@ -1606,15 +1613,6 @@ bad: · 193 unmodified lines
1606	1613	bad_page_fault(regs, sig);
1607	1614	}
1608	1615
1609		−DEFINE_INTERRUPT_HANDLER(StackOverflow)
1610		−{
1611		− pr_crit("Kernel stack overflow in process %s[%d], r1=%lx\n",
1612		− current->comm, task_pid_nr(current), regs->gpr[1]);
1613		− debugger(regs);
1614		− show_regs(regs);
1615		− panic("kernel stack overflow");
1616		−}
1617		−
1618	1616	DEFINE_INTERRUPT_HANDLER(stack_overflow_exception)
1619	1617	{
1620	1618	die("Kernel stack overflow", regs, SIGSEGV);
@@ -1693,7 +1691,7 @@ DEFINE_INTERRUPT_HANDLER(facility_unavailable_exception) · 72 unmodified lines
1693	1691	u8 status;
1694	1692	bool hv;
1695	1693
1696		− hv = (TRAP(regs) == 0xf80);
	1694	+ hv = (TRAP(regs) == INTERRUPT_H_FAC_UNAVAIL);
1697	1695	if (hv)
1698	1696	value = mfspr(SPRN_HFSCR);
1699	1697	else
@@ -2170,11 +2168,14 @@ DEFINE_INTERRUPT_HANDLER(SPEFloatingPointRoundException) · 470 unmodified lines
2170	2168	* in the MSR is 0. This indicates that SRR0/1 are live, and that
2171	2169	* we therefore lost state by taking this exception.
2172	2170	*/
2173		−void unrecoverable_exception(struct pt_regs *regs)
	2171	+void __noreturn unrecoverable_exception(struct pt_regs *regs)
2174	2172	{
2175	2173	pr_emerg("Unrecoverable exception %lx at %lx (msr=%lx)\n",
2176	2174	regs->trap, regs->nip, regs->msr);
2177	2175	die("Unrecoverable exception", regs, SIGABRT);
	2176	+ /* die() should not return */
	2177	+ for (;;)
	2178	+ ;
2178	2179	}
2179	2180
2180	2181	#if defined(CONFIG_BOOKE_WDT) \|\| defined(CONFIG_40x)
@@ -2189,10 +2190,11 @@ void __attribute__ ((weak)) WatchdogHandler(struct pt_regs *regs) · 8 unmodified lines
2189	2190	return;
2190	2191	}
2191	2192
2192		−DEFINE_INTERRUPT_HANDLER(WatchdogException) /* XXX NMI? async? */
	2193	+DEFINE_INTERRUPT_HANDLER_NMI(WatchdogException)
2193	2194	{
2194	2195	printk (KERN_EMERG "PowerPC Book-E Watchdog Exception\n");
2195	2196	WatchdogHandler(regs);
	2197	+ return 0;
2196	2198	}
2197	2199	#endif
2198	2200

arch/powerpc/kernel/uprobes.c+7−0

@@ -41,6 +41,13 @@ int arch_uprobe_analyze_insn(struct arch_uprobe *auprobe, · 40 unmodified lines
41	41	if (addr & 0x03)
42	42	return -EINVAL;
43	43
	44	+ if (cpu_has_feature(CPU_FTR_ARCH_31) &&
	45	+ ppc_inst_prefixed(auprobe->insn) &&
	46	+ (addr & 0x3f) == 60) {
	47	+ pr_info_ratelimited("Cannot register a uprobe on 64 byte unaligned prefixed instruction\n");
	48	+ return -EINVAL;
	49	+ }
	50	+
44	51	return 0;
45	52	}
46	53

arch/powerpc/kernel/vdso.c+123−15

@@ -18,6 +18,7 @@ · 17 unmodified lines
18	18	#include <linux/security.h>
19	19	#include <linux/memblock.h>
20	20	#include <linux/syscalls.h>
	21	+#include <linux/time_namespace.h>
21	22	#include <vdso/datapage.h>
22	23
23	24	#include <asm/syscall.h>
@@ -50,15 +51,21 @@ static union { · 26 unmodified lines
50	51	} vdso_data_store __page_aligned_data;
51	52	struct vdso_arch_data *vdso_data = &vdso_data_store.data;
52	53
	54	+enum vvar_pages {
	55	+ VVAR_DATA_PAGE_OFFSET,
	56	+ VVAR_TIMENS_PAGE_OFFSET,
	57	+ VVAR_NR_PAGES,
	58	+};
	59	+
53	60	static int vdso_mremap(const struct vm_special_mapping sm, struct vm_area_struct new_vma,
54	61	unsigned long text_size)
55	62	{
56	63	unsigned long new_size = new_vma->vm_end - new_vma->vm_start;
57	64
58		− if (new_size != text_size + PAGE_SIZE)
	65	+ if (new_size != text_size)
59	66	return -EINVAL;
60	67
61		− current->mm->context.vdso = (void __user *)new_vma->vm_start + PAGE_SIZE;
	68	+ current->mm->context.vdso = (void __user *)new_vma->vm_start;
62	69
63	70	return 0;
64	71	}
@@ -73,6 +80,14 @@ static int vdso64_mremap(const struct vm_special_mapping *sm, struct vm_area_str · 8 unmodified lines
73	80	return vdso_mremap(sm, new_vma, &vdso64_end - &vdso64_start);
74	81	}
75	82
	83	+static vm_fault_t vvar_fault(const struct vm_special_mapping *sm,
	84	+ struct vm_area_struct vma, struct vm_fault vmf);
	85	+
	86	+static struct vm_special_mapping vvar_spec __ro_after_init = {
	87	+ .name = "[vvar]",
	88	+ .fault = vvar_fault,
	89	+};
	90	+
76	91	static struct vm_special_mapping vdso32_spec __ro_after_init = {
77	92	.name = "[vdso]",
78	93	.mremap = vdso32_mremap,
@@ -83,17 +98,105 @@ static struct vm_special_mapping vdso64_spec __ro_after_init = { · 4 unmodified lines
83	98	.mremap = vdso64_mremap,
84	99	};
85	100
	101	+#ifdef CONFIG_TIME_NS
	102	+struct vdso_data arch_get_vdso_data(void vvar_page)
	103	+{
	104	+ return ((struct vdso_arch_data *)vvar_page)->data;
	105	+}
	106	+
	107	+/*
	108	+ * The vvar mapping contains data for a specific time namespace, so when a task
	109	+ * changes namespace we must unmap its vvar data for the old namespace.
	110	+ * Subsequent faults will map in data for the new namespace.
	111	+ *
	112	+ * For more details see timens_setup_vdso_data().
	113	+ */
	114	+int vdso_join_timens(struct task_struct task, struct time_namespace ns)
	115	+{
	116	+ struct mm_struct *mm = task->mm;
	117	+ struct vm_area_struct *vma;
	118	+
	119	+ mmap_read_lock(mm);
	120	+
	121	+ for (vma = mm->mmap; vma; vma = vma->vm_next) {
	122	+ unsigned long size = vma->vm_end - vma->vm_start;
	123	+
	124	+ if (vma_is_special_mapping(vma, &vvar_spec))
	125	+ zap_page_range(vma, vma->vm_start, size);
	126	+ }
	127	+
	128	+ mmap_read_unlock(mm);
	129	+ return 0;
	130	+}
	131	+
	132	+static struct page find_timens_vvar_page(struct vm_area_struct vma)
	133	+{
	134	+ if (likely(vma->vm_mm == current->mm))
	135	+ return current->nsproxy->time_ns->vvar_page;
	136	+
	137	+ /*
	138	+ * VM_PFNMAP \| VM_IO protect .fault() handler from being called
	139	+ * through interfaces like /proc/$pid/mem or
	140	+ * process_vm_{readv,writev}() as long as there's no .access()
	141	+ * in special_mapping_vmops.
	142	+ * For more details check_vma_flags() and __access_remote_vm()
	143	+ */
	144	+ WARN(1, "vvar_page accessed remotely");
	145	+
	146	+ return NULL;
	147	+}
	148	+#else
	149	+static struct page find_timens_vvar_page(struct vm_area_struct vma)
	150	+{
	151	+ return NULL;
	152	+}
	153	+#endif
	154	+
	155	+static vm_fault_t vvar_fault(const struct vm_special_mapping *sm,
	156	+ struct vm_area_struct vma, struct vm_fault vmf)
	157	+{
	158	+ struct page *timens_page = find_timens_vvar_page(vma);
	159	+ unsigned long pfn;
	160	+
	161	+ switch (vmf->pgoff) {
	162	+ case VVAR_DATA_PAGE_OFFSET:
	163	+ if (timens_page)
	164	+ pfn = page_to_pfn(timens_page);
	165	+ else
	166	+ pfn = virt_to_pfn(vdso_data);
	167	+ break;
	168	+#ifdef CONFIG_TIME_NS
	169	+ case VVAR_TIMENS_PAGE_OFFSET:
	170	+ /*
	171	+ * If a task belongs to a time namespace then a namespace
	172	+ * specific VVAR is mapped with the VVAR_DATA_PAGE_OFFSET and
	173	+ * the real VVAR page is mapped with the VVAR_TIMENS_PAGE_OFFSET
	174	+ * offset.
	175	+ * See also the comment near timens_setup_vdso_data().
	176	+ */
	177	+ if (!timens_page)
	178	+ return VM_FAULT_SIGBUS;
	179	+ pfn = virt_to_pfn(vdso_data);
	180	+ break;
	181	+#endif /* CONFIG_TIME_NS */
	182	+ default:
	183	+ return VM_FAULT_SIGBUS;
	184	+ }
	185	+
	186	+ return vmf_insert_pfn(vma, vmf->address, pfn);
	187	+}
	188	+
86	189	/*
87	190	* This is called from binfmt_elf, we create the special vma for the
88	191	* vDSO and insert it into the mm struct tree
89	192	*/
90	193	static int __arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp)
91	194	{
92		− struct mm_struct *mm = current->mm;
	195	+ unsigned long vdso_size, vdso_base, mappings_size;
93	196	struct vm_special_mapping *vdso_spec;
	197	+ unsigned long vvar_size = VVAR_NR_PAGES * PAGE_SIZE;
	198	+ struct mm_struct *mm = current->mm;
94	199	struct vm_area_struct *vma;
95		− unsigned long vdso_size;
96		− unsigned long vdso_base;
97	200
98	201	if (is_32bit_task()) {
99	202	vdso_spec = &vdso32_spec;
@@ -110,8 +213,8 @@ static int __arch_setup_additional_pages(struct linux_binprm *bprm, int uses_int · 10 unmodified lines
110	213	vdso_base = 0;
111	214	}
112	215
113		− /* Add a page to the vdso size for the data page */
114		− vdso_size += PAGE_SIZE;
	216	+ mappings_size = vdso_size + vvar_size;
	217	+ mappings_size += (VDSO_ALIGNMENT - 1) & PAGE_MASK;
115	218
116	219	/*
117	220	* pick a base address for the vDSO in process space. We try to put it
@@ -119,9 +222,7 @@ static int __arch_setup_additional_pages(struct linux_binprm *bprm, int uses_int · 1 unmodified line
119	222	* and end up putting it elsewhere.
120	223	* Add enough to the size so that the result can be aligned.
121	224	*/
122		− vdso_base = get_unmapped_area(NULL, vdso_base,
123		− vdso_size + ((VDSO_ALIGNMENT - 1) & PAGE_MASK),
124		− 0, 0);
	225	+ vdso_base = get_unmapped_area(NULL, vdso_base, mappings_size, 0, 0);
125	226	if (IS_ERR_VALUE(vdso_base))
126	227	return vdso_base;
127	228
@@ -133,7 +234,13 @@ static int __arch_setup_additional_pages(struct linux_binprm *bprm, int uses_int · 5 unmodified lines
133	234	* install_special_mapping or the perf counter mmap tracking code
134	235	* will fail to recognise it as a vDSO.
135	236	*/
136		− mm->context.vdso = (void __user *)vdso_base + PAGE_SIZE;
	237	+ mm->context.vdso = (void __user *)vdso_base + vvar_size;
	238	+
	239	+ vma = _install_special_mapping(mm, vdso_base, vvar_size,
	240	+ VM_READ \| VM_MAYREAD \| VM_IO \|
	241	+ VM_DONTDUMP \| VM_PFNMAP, &vvar_spec);
	242	+ if (IS_ERR(vma))
	243	+ return PTR_ERR(vma);
137	244
138	245	/*
139	246	* our vma flags don't have VM_WRITE so by default, the process isn't
@@ -145,9 +252,12 @@ static int __arch_setup_additional_pages(struct linux_binprm *bprm, int uses_int · 5 unmodified lines
145	252	* It's fine to use that for setting breakpoints in the vDSO code
146	253	* pages though.
147	254	*/
148		− vma = _install_special_mapping(mm, vdso_base, vdso_size,
	255	+ vma = _install_special_mapping(mm, vdso_base + vvar_size, vdso_size,
149	256	VM_READ \| VM_EXEC \| VM_MAYREAD \|
150	257	VM_MAYWRITE \| VM_MAYEXEC, vdso_spec);
	258	+ if (IS_ERR(vma))
	259	+ do_munmap(mm, vdso_base, vvar_size, NULL);
	260	+
151	261	return PTR_ERR_OR_ZERO(vma);
152	262	}
153	263
@@ -249,10 +359,8 @@ static struct page ** __init vdso_setup_pages(void start, void end) · 95 unmodified lines
249	359	if (!pagelist)
250	360	panic("%s: Cannot allocate page list for VDSO", __func__);
251	361
252		− pagelist[0] = virt_to_page(vdso_data);
253		−
254	362	for (i = 0; i < pages; i++)
255		− pagelist[i + 1] = virt_to_page(start + i * PAGE_SIZE);
	363	+ pagelist[i] = virt_to_page(start + i * PAGE_SIZE);
256	364
257	365	return pagelist;
258	366	}

arch/powerpc/kernel/vdso32/vdso32.lds.S+1−1

@@ -17,7 +17,7 @@ ENTRY(_start) · 16 unmodified lines
17	17
18	18	SECTIONS
19	19	{
20		− PROVIDE(_vdso_datapage = . - PAGE_SIZE);
	20	+ PROVIDE(_vdso_datapage = . - 2 * PAGE_SIZE);
21	21	. = SIZEOF_HEADERS;
22	22
23	23	.hash : { *(.hash) } :text

arch/powerpc/kernel/vdso64/vdso64.lds.S+1−1

@@ -17,7 +17,7 @@ ENTRY(_start) · 16 unmodified lines
17	17
18	18	SECTIONS
19	19	{
20		− PROVIDE(_vdso_datapage = . - PAGE_SIZE);
	20	+ PROVIDE(_vdso_datapage = . - 2 * PAGE_SIZE);
21	21	. = SIZEOF_HEADERS;
22	22
23	23	.hash : { *(.hash) } :text

arch/powerpc/kernel/vector.S+0−2

@@ -67,9 +67,7 @@ _GLOBAL(load_up_altivec) · 66 unmodified lines
67	67	#ifdef CONFIG_PPC32
68	68	mfspr r5,SPRN_SPRG_THREAD /* current task's THREAD (phys) */
69	69	oris r9,r9,MSR_VEC@h
70		−#ifdef CONFIG_VMAP_STACK
71	70	tovirt(r5, r5)
72		−#endif
73	71	#else
74	72	ld r4,PACACURRENT(r13)
75	73	addi r5,r4,THREAD /* Get THREAD */

arch/powerpc/kexec/crash.c+2−1

@@ -24,6 +24,7 @@ · 23 unmodified lines
24	24	#include <asm/smp.h>
25	25	#include <asm/setjmp.h>
26	26	#include <asm/debug.h>
	27	+#include <asm/interrupt.h>
27	28
28	29	/*
29	30	* The primary CPU waits a while for all secondary CPUs to enter. This is to
@@ -336,7 +337,7 @@ void default_machine_crash_shutdown(struct pt_regs *regs) · 306 unmodified lines
336	337	* If we came in via system reset, wait a while for the secondary
337	338	* CPUs to enter.
338	339	*/
339		− if (TRAP(regs) == 0x100)
	340	+ if (TRAP(regs) == INTERRUPT_SYSTEM_RESET)
340	341	mdelay(PRIMARY_TIMEOUT);
341	342
342	343	crash_kexec_prepare_cpus(crashing_cpu);

arch/powerpc/kvm/book3s_64_mmu_host.c+2−0

@@ -8,6 +8,7 @@ · 7 unmodified lines
8	8	*/
9	9
10	10	#include <linux/kvm_host.h>
	11	+#include <linux/pkeys.h>
11	12
12	13	#include <asm/kvm_ppc.h>
13	14	#include <asm/kvm_book3s.h>
@@ -133,6 +134,7 @@ int kvmppc_mmu_map_page(struct kvm_vcpu vcpu, struct kvmppc_pte orig_pte, · 119 unmodified lines
133	134	else
134	135	kvmppc_mmu_flush_icache(pfn);
135	136
	137	+ rflags \|= pte_to_hpte_pkey_bits(0, HPTE_USE_KERNEL_KEY);
136	138	rflags = (rflags & ~HPTE_R_WIMG) \| orig_pte->wimg;
137	139
138	140	/*

arch/powerpc/kvm/book3s_hv.c+65−20

@@ -803,7 +803,10 @@ static int kvmppc_h_set_mode(struct kvm_vcpu *vcpu, unsigned long mflags, · 802 unmodified lines
803	803	vcpu->arch.dawrx1 = value2;
804	804	return H_SUCCESS;
805	805	case H_SET_MODE_RESOURCE_ADDR_TRANS_MODE:
806		− /* KVM does not support mflags=2 (AIL=2) */
	806	+ /*
	807	+ * KVM does not support mflags=2 (AIL=2) and AIL=1 is reserved.
	808	+ * Keep this in synch with kvmppc_filter_guest_lpcr_hv.
	809	+ */
807	810	if (mflags != 0 && mflags != 3)
808	811	return H_UNSUPPORTED_FLAG_START;
809	812	return H_TOO_HARD;
@@ -1635,6 +1638,41 @@ static int kvm_arch_vcpu_ioctl_set_sregs_hv(struct kvm_vcpu *vcpu, · 825 unmodified lines
1635	1638	return 0;
1636	1639	}
1637	1640
	1641	+/*
	1642	+ * Enforce limits on guest LPCR values based on hardware availability,
	1643	+ * guest configuration, and possibly hypervisor support and security
	1644	+ * concerns.
	1645	+ */
	1646	+unsigned long kvmppc_filter_lpcr_hv(struct kvm *kvm, unsigned long lpcr)
	1647	+{
	1648	+ /* LPCR_TC only applies to HPT guests */
	1649	+ if (kvm_is_radix(kvm))
	1650	+ lpcr &= ~LPCR_TC;
	1651	+
	1652	+ /* On POWER8 and above, userspace can modify AIL */
	1653	+ if (!cpu_has_feature(CPU_FTR_ARCH_207S))
	1654	+ lpcr &= ~LPCR_AIL;
	1655	+ if ((lpcr & LPCR_AIL) != LPCR_AIL_3)
	1656	+ lpcr &= ~LPCR_AIL; /* LPCR[AIL]=1/2 is disallowed */
	1657	+
	1658	+ /*
	1659	+ * On POWER9, allow userspace to enable large decrementer for the
	1660	+ * guest, whether or not the host has it enabled.
	1661	+ */
	1662	+ if (!cpu_has_feature(CPU_FTR_ARCH_300))
	1663	+ lpcr &= ~LPCR_LD;
	1664	+
	1665	+ return lpcr;
	1666	+}
	1667	+
	1668	+static void verify_lpcr(struct kvm *kvm, unsigned long lpcr)
	1669	+{
	1670	+ if (lpcr != kvmppc_filter_lpcr_hv(kvm, lpcr)) {
	1671	+ WARN_ONCE(1, "lpcr 0x%lx differs from filtered 0x%lx\n",
	1672	+ lpcr, kvmppc_filter_lpcr_hv(kvm, lpcr));
	1673	+ }
	1674	+}
	1675	+
1638	1676	static void kvmppc_set_lpcr(struct kvm_vcpu *vcpu, u64 new_lpcr,
1639	1677	bool preserve_top32)
1640	1678	{
@@ -1643,6 +1681,23 @@ static void kvmppc_set_lpcr(struct kvm_vcpu *vcpu, u64 new_lpcr, · 2 unmodified lines
1643	1681	u64 mask;
1644	1682
1645	1683	spin_lock(&vc->lock);
	1684	+
	1685	+ /*
	1686	+ * Userspace can only modify
	1687	+ * DPFD (default prefetch depth), ILE (interrupt little-endian),
	1688	+ * TC (translation control), AIL (alternate interrupt location),
	1689	+ * LD (large decrementer).
	1690	+ * These are subject to restrictions from kvmppc_filter_lcpr_hv().
	1691	+ */
	1692	+ mask = LPCR_DPFD \| LPCR_ILE \| LPCR_TC \| LPCR_AIL \| LPCR_LD;
	1693	+
	1694	+ /* Broken 32-bit version of LPCR must not clear top bits */
	1695	+ if (preserve_top32)
	1696	+ mask &= 0xFFFFFFFF;
	1697	+
	1698	+ new_lpcr = kvmppc_filter_lpcr_hv(kvm,
	1699	+ (vc->lpcr & ~mask) \| (new_lpcr & mask));
	1700	+
1646	1701	/*
1647	1702	* If ILE (interrupt little-endian) has changed, update the
1648	1703	* MSR_LE bit in the intr_msr for each vcpu in this vcore.
@@ -1661,25 +1716,8 @@ static void kvmppc_set_lpcr(struct kvm_vcpu *vcpu, u64 new_lpcr, · 12 unmodified lines
1661	1716	}
1662	1717	}
1663	1718
1664		− /*
1665		− * Userspace can only modify DPFD (default prefetch depth),
1666		− * ILE (interrupt little-endian) and TC (translation control).
1667		− * On POWER8 and POWER9 userspace can also modify AIL (alt. interrupt loc.).
1668		− */
1669		− mask = LPCR_DPFD \| LPCR_ILE \| LPCR_TC;
1670		− if (cpu_has_feature(CPU_FTR_ARCH_207S))
1671		− mask \|= LPCR_AIL;
1672		− /*
1673		− * On POWER9, allow userspace to enable large decrementer for the
1674		− * guest, whether or not the host has it enabled.
1675		− */
1676		− if (cpu_has_feature(CPU_FTR_ARCH_300))
1677		− mask \|= LPCR_LD;
	1719	+ vc->lpcr = new_lpcr;
1678	1720
1679		− /* Broken 32-bit version of LPCR must not clear top bits */
1680		− if (preserve_top32)
1681		− mask &= 0xFFFFFFFF;
1682		− vc->lpcr = (vc->lpcr & ~mask) \| (new_lpcr & mask);
1683	1721	spin_unlock(&vc->lock);
1684	1722	}
1685	1723
@@ -3728,7 +3766,10 @@ static int kvmhv_p9_guest_entry(struct kvm_vcpu *vcpu, u64 time_limit, · 2042 unmodified lines
3728	3766	vcpu->arch.dec_expires = dec + tb;
3729	3767	vcpu->cpu = -1;
3730	3768	vcpu->arch.thread_cpu = -1;
	3769	+ /* Save guest CTRL register, set runlatch to 1 */
3731	3770	vcpu->arch.ctrl = mfspr(SPRN_CTRLF);
	3771	+ if (!(vcpu->arch.ctrl & 1))
	3772	+ mtspr(SPRN_CTRLT, vcpu->arch.ctrl \| 1);
3732	3773
3733	3774	vcpu->arch.iamr = mfspr(SPRN_IAMR);
3734	3775	vcpu->arch.pspb = mfspr(SPRN_PSPB);
@@ -3749,7 +3790,6 @@ static int kvmhv_p9_guest_entry(struct kvm_vcpu *vcpu, u64 time_limit, · 14 unmodified lines
3749	3790	mtspr(SPRN_DSCR, host_dscr);
3750	3791	mtspr(SPRN_TIDR, host_tidr);
3751	3792	mtspr(SPRN_IAMR, host_iamr);
3752		− mtspr(SPRN_PSPB, 0);
3753	3793
3754	3794	if (host_amr != vcpu->arch.amr)
3755	3795	mtspr(SPRN_AMR, host_amr);
@@ -4641,8 +4681,10 @@ void kvmppc_update_lpcr(struct kvm *kvm, unsigned long lpcr, unsigned long mask) · 885 unmodified lines
4641	4681	struct kvmppc_vcore *vc = kvm->arch.vcores[i];
4642	4682	if (!vc)
4643	4683	continue;
	4684	+
4644	4685	spin_lock(&vc->lock);
4645	4686	vc->lpcr = (vc->lpcr & ~mask) \| lpcr;
	4687	+ verify_lpcr(kvm, vc->lpcr);
4646	4688	spin_unlock(&vc->lock);
4647	4689	if (++cores_done >= kvm->arch.online_vcores)
4648	4690	break;
@@ -4970,6 +5012,7 @@ static int kvmppc_core_init_vm_hv(struct kvm *kvm) · 321 unmodified lines
4970	5012	kvmppc_setup_partition_table(kvm);
4971	5013	}
4972	5014
	5015	+ verify_lpcr(kvm, lpcr);
4973	5016	kvm->arch.lpcr = lpcr;
4974	5017
4975	5018	/* Initialization for future HPT resizes */
@@ -5369,8 +5412,10 @@ static unsigned int default_hcall_list[] = { · 393 unmodified lines
5369	5412	H_READ,
5370	5413	H_PROTECT,
5371	5414	H_BULK_REMOVE,
	5415	+#ifdef CONFIG_SPAPR_TCE_IOMMU
5372	5416	H_GET_TCE,
5373	5417	H_PUT_TCE,
	5418	+#endif
5374	5419	H_SET_DABR,
5375	5420	H_SET_XDABR,
5376	5421	H_CEDE,

arch/powerpc/kvm/book3s_hv_builtin.c+3−0

@@ -662,6 +662,9 @@ static void kvmppc_end_cede(struct kvm_vcpu *vcpu) · 661 unmodified lines
662	662
663	663	void kvmppc_set_msr_hv(struct kvm_vcpu *vcpu, u64 msr)
664	664	{
	665	+ /* Guest must always run with ME enabled, HV disabled. */
	666	+ msr = (msr \| MSR_ME) & ~MSR_HV;
	667	+
665	668	/*
666	669	* Check for illegal transactional state bit combination
667	670	* and if we find it, force the TS field to a safe state.

arch/powerpc/kvm/book3s_hv_nested.c+30−7

@@ -132,8 +132,33 @@ static void save_hv_return_state(struct kvm_vcpu *vcpu, int trap, · 131 unmodified lines
132	132	}
133	133	}
134	134
	135	+/*
	136	+ * This can result in some L0 HV register state being leaked to an L1
	137	+ * hypervisor when the hv_guest_state is copied back to the guest after
	138	+ * being modified here.
	139	+ *
	140	+ * There is no known problem with such a leak, and in many cases these
	141	+ * register settings could be derived by the guest by observing behaviour
	142	+ * and timing, interrupts, etc., but it is an issue to consider.
	143	+ */
135	144	static void sanitise_hv_regs(struct kvm_vcpu vcpu, struct hv_guest_state hr)
136	145	{
	146	+ struct kvmppc_vcore *vc = vcpu->arch.vcore;
	147	+ u64 mask;
	148	+
	149	+ /*
	150	+ * Don't let L1 change LPCR bits for the L2 except these:
	151	+ */
	152	+ mask = LPCR_DPFD \| LPCR_ILE \| LPCR_TC \| LPCR_AIL \| LPCR_LD \|
	153	+ LPCR_LPES \| LPCR_MER;
	154	+
	155	+ /*
	156	+ * Additional filtering is required depending on hardware
	157	+ * and configuration.
	158	+ */
	159	+ hr->lpcr = kvmppc_filter_lpcr_hv(vcpu->kvm,
	160	+ (vc->lpcr & ~mask) \| (hr->lpcr & mask));
	161	+
137	162	/*
138	163	* Don't let L1 enable features for L2 which we've disabled for L1,
139	164	* but preserve the interrupt cause field.
@@ -271,8 +296,6 @@ long kvmhv_enter_nested_guest(struct kvm_vcpu *vcpu) · 131 unmodified lines
271	296	u64 hv_ptr, regs_ptr;
272	297	u64 hdec_exp;
273	298	s64 delta_purr, delta_spurr, delta_ic, delta_vtb;
274		− u64 mask;
275		− unsigned long lpcr;
276	299
277	300	if (vcpu->kvm->arch.l1_ptcr == 0)
278	301	return H_NOT_AVAILABLE;
@@ -320,10 +343,10 @@ long kvmhv_enter_nested_guest(struct kvm_vcpu *vcpu) · 41 unmodified lines
320	343	vcpu->arch.nested = l2;
321	344	vcpu->arch.nested_vcpu_id = l2_hv.vcpu_token;
322	345	vcpu->arch.regs = l2_regs;
323		− vcpu->arch.shregs.msr = vcpu->arch.regs.msr;
324		− mask = LPCR_DPFD \| LPCR_ILE \| LPCR_TC \| LPCR_AIL \| LPCR_LD \|
325		− LPCR_LPES \| LPCR_MER;
326		− lpcr = (vc->lpcr & ~mask) \| (l2_hv.lpcr & mask);
	346	+
	347	+ /* Guest must always run with ME enabled, HV disabled. */
	348	+ vcpu->arch.shregs.msr = (vcpu->arch.regs.msr \| MSR_ME) & ~MSR_HV;
	349	+
327	350	sanitise_hv_regs(vcpu, &l2_hv);
328	351	restore_hv_regs(vcpu, &l2_hv);
329	352
@@ -335,7 +358,7 @@ long kvmhv_enter_nested_guest(struct kvm_vcpu *vcpu) · 5 unmodified lines
335	358	r = RESUME_HOST;
336	359	break;
337	360	}
338		− r = kvmhv_run_single_vcpu(vcpu, hdec_exp, lpcr);
	361	+ r = kvmhv_run_single_vcpu(vcpu, hdec_exp, l2_hv.lpcr);
339	362	} while (is_kvmppc_resume_guest(r));
340	363
341	364	/* save L2 state for return */

arch/powerpc/kvm/book3s_hv_rm_mmu.c+1−2

@@ -673,8 +673,7 @@ long kvmppc_h_bulk_remove(struct kvm_vcpu *vcpu) · 672 unmodified lines
673	673	}
674	674
675	675	long kvmppc_h_protect(struct kvm_vcpu *vcpu, unsigned long flags,
676		− unsigned long pte_index, unsigned long avpn,
677		− unsigned long va)
	676	+ unsigned long pte_index, unsigned long avpn)
678	677	{
679	678	struct kvm *kvm = vcpu->kvm;
680	679	__be64 *hpte;

arch/powerpc/lib/Makefile+1−1

@@ -16,7 +16,7 @@ CFLAGS_code-patching.o += -DDISABLE_BRANCH_PROFILING · 15 unmodified lines
16	16	CFLAGS_feature-fixups.o += -DDISABLE_BRANCH_PROFILING
17	17	endif
18	18
19		−obj-y += alloc.o code-patching.o feature-fixups.o pmem.o inst.o test_code-patching.o
	19	+obj-y += alloc.o code-patching.o feature-fixups.o pmem.o test_code-patching.o
20	20
21	21	ifndef CONFIG_KASAN
22	22	obj-y += string.o memcmp_$(BITS).o

arch/powerpc/lib/checksum_wrappers.c+4−11

@@ -16,16 +16,12 @@ __wsum csum_and_copy_from_user(const void __user src, void dst, · 15 unmodified lines
16	16	{
17	17	__wsum csum;
18	18
19		− might_sleep();
20		−
21		− if (unlikely(!access_ok(src, len)))
	19	+ if (unlikely(!user_read_access_begin(src, len)))
22	20	return 0;
23	21
24		− allow_read_from_user(src, len);
25		−
26	22	csum = csum_partial_copy_generic((void __force *)src, dst, len);
27	23
28		− prevent_read_from_user(src, len);
	24	+ user_read_access_end();
29	25	return csum;
30	26	}
31	27	EXPORT_SYMBOL(csum_and_copy_from_user);
@@ -34,15 +30,12 @@ __wsum csum_and_copy_to_user(const void src, void __user dst, int len) · 2 unmodified lines
34	30	{
35	31	__wsum csum;
36	32
37		− might_sleep();
38		− if (unlikely(!access_ok(dst, len)))
	33	+ if (unlikely(!user_write_access_begin(dst, len)))
39	34	return 0;
40	35
41		− allow_write_to_user(dst, len);
42		−
43	36	csum = csum_partial_copy_generic(src, (void __force *)dst, len);
44	37
45		− prevent_write_to_user(dst, len);
	38	+ user_write_access_end();
46	39	return csum;
47	40	}
48	41	EXPORT_SYMBOL(csum_and_copy_to_user);

arch/powerpc/lib/code-patching.c+9−4

@@ -21,10 +21,15 @@ · 20 unmodified lines
21	21	static int __patch_instruction(struct ppc_inst *exec_addr, struct ppc_inst instr,
22	22	struct ppc_inst *patch_addr)
23	23	{
24		− if (!ppc_inst_prefixed(instr))
25		− __put_user_asm_goto(ppc_inst_val(instr), patch_addr, failed, "stw");
26		− else
27		− __put_user_asm_goto(ppc_inst_as_u64(instr), patch_addr, failed, "std");
	24	+ if (!ppc_inst_prefixed(instr)) {
	25	+ u32 val = ppc_inst_val(instr);
	26	+
	27	+ __put_kernel_nofault(patch_addr, &val, u32, failed);
	28	+ } else {
	29	+ u64 val = ppc_inst_as_ulong(instr);
	30	+
	31	+ __put_kernel_nofault(patch_addr, &val, u64, failed);
	32	+ }
28	33
29	34	asm ("dcbst 0, %0; sync; icbi 0,%1; sync; isync" :: "r" (patch_addr),
30	35	"r" (exec_addr));

arch/powerpc/lib/inst.c+0−73

@@ -1,73 +0,0 @@
1		−// SPDX-License-Identifier: GPL-2.0-or-later
2		−/*
3		− * Copyright 2020, IBM Corporation.
4		− */
5		−
6		−#include <linux/uaccess.h>
7		−#include <asm/disassemble.h>
8		−#include <asm/inst.h>
9		−#include <asm/ppc-opcode.h>
10		−
11		−#ifdef CONFIG_PPC64
12		−int probe_user_read_inst(struct ppc_inst *inst,
13		− struct ppc_inst __user *nip)
14		−{
15		− unsigned int val, suffix;
16		− int err;
17		−
18		− err = copy_from_user_nofault(&val, nip, sizeof(val));
19		− if (err)
20		− return err;
21		− if (get_op(val) == OP_PREFIX) {
22		− err = copy_from_user_nofault(&suffix, (void __user *)nip + 4, 4);
23		− *inst = ppc_inst_prefix(val, suffix);
24		− } else {
25		− *inst = ppc_inst(val);
26		− }
27		− return err;
28		−}
29		−
30		−int probe_kernel_read_inst(struct ppc_inst *inst,
31		− struct ppc_inst *src)
32		−{
33		− unsigned int val, suffix;
34		− int err;
35		−
36		− err = copy_from_kernel_nofault(&val, src, sizeof(val));
37		− if (err)
38		− return err;
39		− if (get_op(val) == OP_PREFIX) {
40		− err = copy_from_kernel_nofault(&suffix, (void *)src + 4, 4);
41		− *inst = ppc_inst_prefix(val, suffix);
42		− } else {
43		− *inst = ppc_inst(val);
44		− }
45		− return err;
46		−}
47		−#else /* !CONFIG_PPC64 */
48		−int probe_user_read_inst(struct ppc_inst *inst,
49		− struct ppc_inst __user *nip)
50		−{
51		− unsigned int val;
52		− int err;
53		−
54		− err = copy_from_user_nofault(&val, nip, sizeof(val));
55		− if (!err)
56		− *inst = ppc_inst(val);
57		−
58		− return err;
59		−}
60		−
61		−int probe_kernel_read_inst(struct ppc_inst *inst,
62		− struct ppc_inst *src)
63		−{
64		− unsigned int val;
65		− int err;
66		−
67		− err = copy_from_kernel_nofault(&val, src, sizeof(val));
68		− if (!err)
69		− *inst = ppc_inst(val);
70		−
71		− return err;
72		−}
73		−#endif /* CONFIG_PPC64 */

arch/powerpc/lib/sstep.c+0−13

@@ -1401,10 +1401,6 @@ int analyse_instr(struct instruction_op op, const struct pt_regs regs, · 1400 unmodified lines
1401	1401	break;
1402	1402	}
1403	1403
1404		− /* Following cases refer to regs->gpr[], so we need all regs */
1405		− if (!FULL_REGS(regs))
1406		− return -1;
1407		−
1408	1404	rd = (word >> 21) & 0x1f;
1409	1405	ra = (word >> 16) & 0x1f;
1410	1406	rb = (word >> 11) & 0x1f;
@@ -3086,15 +3082,6 @@ NOKPROBE_SYMBOL(analyse_instr); · 1675 unmodified lines
3086	3082	*/
3087	3083	static nokprobe_inline int handle_stack_update(unsigned long ea, struct pt_regs *regs)
3088	3084	{
3089		−#ifdef CONFIG_PPC32
3090		− /*
3091		− * Check if we will touch kernel stack overflow
3092		− */
3093		− if (ea - STACK_INT_FRAME_SIZE <= current->thread.ksp_limit) {
3094		− printk(KERN_CRIT "Can't kprobe this since kernel stack would overflow.\n");
3095		− return -EINVAL;
3096		− }
3097		−#endif /* CONFIG_PPC32 */
3098	3085	/*
3099	3086	* Check if we already set since that means we'll
3100	3087	* lose the previous value.

arch/powerpc/math-emu/math.c+2−2

@@ -225,7 +225,7 @@ record_exception(struct pt_regs *regs, int eflag) · 224 unmodified lines
225	225	int
226	226	do_mathemu(struct pt_regs *regs)
227	227	{
228		− void op0 = 0, op1 = 0, op2 = 0, op3 = 0;
	228	+ void op0 = NULL, op1 = NULL, op2 = NULL, op3 = NULL;
229	229	unsigned long pc = regs->nip;
230	230	signed short sdisp;
231	231	u32 insn = 0;
@@ -234,7 +234,7 @@ do_mathemu(struct pt_regs *regs) · 2 unmodified lines
234	234	int type = 0;
235	235	int eflag, trap;
236	236
237		− if (get_user(insn, (u32 *)pc))
	237	+ if (get_user(insn, (u32 __user *)pc))
238	238	return -EFAULT;
239	239
240	240	switch (insn >> 26) {

arch/powerpc/mm/Makefile+2−1

@@ -8,7 +8,8 @@ ccflags-$(CONFIG_PPC64) := $(NO_MINIMAL_TOC) · 7 unmodified lines
8	8	obj-y := fault.o mem.o pgtable.o mmap.o maccess.o \
9	9	init_$(BITS).o pgtable_$(BITS).o \
10	10	pgtable-frag.o ioremap.o ioremap_$(BITS).o \
11		− init-common.o mmu_context.o drmem.o
	11	+ init-common.o mmu_context.o drmem.o \
	12	+ cacheflush.o
12	13	obj-$(CONFIG_PPC_MMU_NOHASH) += nohash/
13	14	obj-$(CONFIG_PPC_BOOK3S_32) += book3s32/
14	15	obj-$(CONFIG_PPC_BOOK3S_64) += book3s64/

arch/powerpc/mm/book3s32/Makefile+1−0

@@ -9,3 +9,4 @@ endif · 8 unmodified lines
9	9	obj-y += mmu.o mmu_context.o
10	10	obj-$(CONFIG_PPC_BOOK3S_603) += nohash_low.o
11	11	obj-$(CONFIG_PPC_BOOK3S_604) += hash_low.o tlb.o
	12	+obj-$(CONFIG_PPC_KUEP) += kuep.o

arch/powerpc/mm/book3s32/hash_low.S+0−14

@@ -140,10 +140,6 @@ _GLOBAL(hash_page) · 139 unmodified lines
140	140	bne- .Lretry /* retry if someone got there first */
141	141
142	142	mfsrin r3,r4 /* get segment reg for segment */
143		−#ifndef CONFIG_VMAP_STACK
144		− mfctr r0
145		− stw r0,_CTR(r11)
146		−#endif
147	143	bl create_hpte /* add the hash table entry */
148	144
149	145	#ifdef CONFIG_SMP
@@ -152,17 +148,7 @@ _GLOBAL(hash_page) · 2 unmodified lines
152	148	li r0,0
153	149	stw r0, (mmu_hash_lock - PAGE_OFFSET)@l(r8)
154	150	#endif
155		−
156		−#ifdef CONFIG_VMAP_STACK
157	151	b fast_hash_page_return
158		−#else
159		− /* Return from the exception */
160		− lwz r5,_CTR(r11)
161		− mtctr r5
162		− lwz r0,GPR0(r11)
163		− lwz r8,GPR8(r11)
164		− b fast_exception_return
165		−#endif
166	152
167	153	#ifdef CONFIG_SMP
168	154	.Lhash_page_out:

arch/powerpc/mm/book3s32/kuep.c+40−0

@@ -0,0 +1,40 @@
	1	+// SPDX-License-Identifier: GPL-2.0-or-later
	2	+
	3	+#include <asm/kup.h>
	4	+#include <asm/reg.h>
	5	+#include <asm/task_size_32.h>
	6	+#include <asm/mmu.h>
	7	+
	8	+#define KUEP_UPDATE_TWO_USER_SEGMENTS(n) do { \
	9	+ if (TASK_SIZE > ((n) << 28)) \
	10	+ mtsr(val1, (n) << 28); \
	11	+ if (TASK_SIZE > (((n) + 1) << 28)) \
	12	+ mtsr(val2, ((n) + 1) << 28); \
	13	+ val1 = (val1 + 0x222) & 0xf0ffffff; \
	14	+ val2 = (val2 + 0x222) & 0xf0ffffff; \
	15	+} while (0)
	16	+
	17	+static __always_inline void kuep_update(u32 val)
	18	+{
	19	+ int val1 = val;
	20	+ int val2 = (val + 0x111) & 0xf0ffffff;
	21	+
	22	+ KUEP_UPDATE_TWO_USER_SEGMENTS(0);
	23	+ KUEP_UPDATE_TWO_USER_SEGMENTS(2);
	24	+ KUEP_UPDATE_TWO_USER_SEGMENTS(4);
	25	+ KUEP_UPDATE_TWO_USER_SEGMENTS(6);
	26	+ KUEP_UPDATE_TWO_USER_SEGMENTS(8);
	27	+ KUEP_UPDATE_TWO_USER_SEGMENTS(10);
	28	+ KUEP_UPDATE_TWO_USER_SEGMENTS(12);
	29	+ KUEP_UPDATE_TWO_USER_SEGMENTS(14);
	30	+}
	31	+
	32	+void kuep_lock(void)
	33	+{
	34	+ kuep_update(mfsr(0) \| SR_NX);
	35	+}
	36	+
	37	+void kuep_unlock(void)
	38	+{
	39	+ kuep_update(mfsr(0) & ~SR_NX);
	40	+}

arch/powerpc/mm/book3s32/mmu.c+1−8

@@ -162,7 +162,7 @@ unsigned long __init mmu_mapin_ram(unsigned long base, unsigned long top) · 161 unmodified lines
162	162	unsigned long border = (unsigned long)__init_begin - PAGE_OFFSET;
163	163
164	164
165		− if (debug_pagealloc_enabled() \|\| __map_without_bats) {
	165	+ if (debug_pagealloc_enabled_or_kfence() \|\| __map_without_bats) {
166	166	pr_debug_once("Read-Write memory mapped without BATs\n");
167	167	if (base >= border)
168	168	return base;
@@ -184,17 +184,10 @@ static bool is_module_segment(unsigned long addr) · 15 unmodified lines
184	184	{
185	185	if (!IS_ENABLED(CONFIG_MODULES))
186	186	return false;
187		−#ifdef MODULES_VADDR
188	187	if (addr < ALIGN_DOWN(MODULES_VADDR, SZ_256M))
189	188	return false;
190	189	if (addr > ALIGN(MODULES_END, SZ_256M) - 1)
191	190	return false;
192		−#else
193		− if (addr < ALIGN_DOWN(VMALLOC_START, SZ_256M))
194		− return false;
195		− if (addr > ALIGN(VMALLOC_END, SZ_256M) - 1)
196		− return false;
197		−#endif
198	191	return true;
199	192	}
200	193

arch/powerpc/mm/book3s64/hash_pgtable.c+121−9

@@ -8,6 +8,7 @@ · 7 unmodified lines
8	8	#include <linux/sched.h>
9	9	#include <linux/mm_types.h>
10	10	#include <linux/mm.h>
	11	+#include <linux/stop_machine.h>
11	12
12	13	#include <asm/sections.h>
13	14	#include <asm/mmu.h>
@@ -400,10 +401,103 @@ EXPORT_SYMBOL_GPL(hash__has_transparent_hugepage); · 386 unmodified lines
400	401	#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
401	402
402	403	#ifdef CONFIG_STRICT_KERNEL_RWX
	404	+
	405	+struct change_memory_parms {
	406	+ unsigned long start, end, newpp;
	407	+ unsigned int step, nr_cpus, master_cpu;
	408	+ atomic_t cpu_counter;
	409	+};
	410	+
	411	+// We'd rather this was on the stack but it has to be in the RMO
	412	+static struct change_memory_parms chmem_parms;
	413	+
	414	+// And therefore we need a lock to protect it from concurrent use
	415	+static DEFINE_MUTEX(chmem_lock);
	416	+
	417	+static void change_memory_range(unsigned long start, unsigned long end,
	418	+ unsigned int step, unsigned long newpp)
	419	+{
	420	+ unsigned long idx;
	421	+
	422	+ pr_debug("Changing page protection on range 0x%lx-0x%lx, to 0x%lx, step 0x%x\n",
	423	+ start, end, newpp, step);
	424	+
	425	+ for (idx = start; idx < end; idx += step)
	426	+ /* Not sure if we can do much with the return value */
	427	+ mmu_hash_ops.hpte_updateboltedpp(newpp, idx, mmu_linear_psize,
	428	+ mmu_kernel_ssize);
	429	+}
	430	+
	431	+static int notrace chmem_secondary_loop(struct change_memory_parms *parms)
	432	+{
	433	+ unsigned long msr, tmp, flags;
	434	+ int *p;
	435	+
	436	+ p = &parms->cpu_counter.counter;
	437	+
	438	+ local_irq_save(flags);
	439	+ hard_irq_disable();
	440	+
	441	+ asm volatile (
	442	+ // Switch to real mode and leave interrupts off
	443	+ "mfmsr %[msr] ;"
	444	+ "li %[tmp], %[MSR_IR_DR] ;"
	445	+ "andc %[tmp], %[msr], %[tmp] ;"
	446	+ "mtmsrd %[tmp] ;"
	447	+
	448	+ // Tell the master we are in real mode
	449	+ "1: "
	450	+ "lwarx %[tmp], 0, %[p] ;"
	451	+ "addic %[tmp], %[tmp], -1 ;"
	452	+ "stwcx. %[tmp], 0, %[p] ;"
	453	+ "bne- 1b ;"
	454	+
	455	+ // Spin until the counter goes to zero
	456	+ "2: ;"
	457	+ "lwz %[tmp], 0(%[p]) ;"
	458	+ "cmpwi %[tmp], 0 ;"
	459	+ "bne- 2b ;"
	460	+
	461	+ // Switch back to virtual mode
	462	+ "mtmsrd %[msr] ;"
	463	+
	464	+ : // outputs
	465	+ [msr] "=&r" (msr), [tmp] "=&b" (tmp), "+m" (*p)
	466	+ : // inputs
	467	+ [p] "b" (p), [MSR_IR_DR] "i" (MSR_IR \| MSR_DR)
	468	+ : // clobbers
	469	+ "cc", "xer"
	470	+ );
	471	+
	472	+ local_irq_restore(flags);
	473	+
	474	+ return 0;
	475	+}
	476	+
	477	+static int change_memory_range_fn(void *data)
	478	+{
	479	+ struct change_memory_parms *parms = data;
	480	+
	481	+ if (parms->master_cpu != smp_processor_id())
	482	+ return chmem_secondary_loop(parms);
	483	+
	484	+ // Wait for all but one CPU (this one) to call-in
	485	+ while (atomic_read(&parms->cpu_counter) > 1)
	486	+ barrier();
	487	+
	488	+ change_memory_range(parms->start, parms->end, parms->step, parms->newpp);
	489	+
	490	+ mb();
	491	+
	492	+ // Signal the other CPUs that we're done
	493	+ atomic_dec(&parms->cpu_counter);
	494	+
	495	+ return 0;
	496	+}
	497	+
403	498	static bool hash__change_memory_range(unsigned long start, unsigned long end,
404	499	unsigned long newpp)
405	500	{
406		− unsigned long idx;
407	501	unsigned int step, shift;
408	502
409	503	shift = mmu_psize_defs[mmu_linear_psize].shift;
@@ -415,25 +509,43 @@ static bool hash__change_memory_range(unsigned long start, unsigned long end, · 5 unmodified lines
415	509	if (start >= end)
416	510	return false;
417	511
418		− pr_debug("Changing page protection on range 0x%lx-0x%lx, to 0x%lx, step 0x%x\n",
419		− start, end, newpp, step);
	512	+ if (firmware_has_feature(FW_FEATURE_LPAR)) {
	513	+ mutex_lock(&chmem_lock);
420	514
421		− for (idx = start; idx < end; idx += step)
422		− /* Not sure if we can do much with the return value */
423		− mmu_hash_ops.hpte_updateboltedpp(newpp, idx, mmu_linear_psize,
424		− mmu_kernel_ssize);
	515	+ chmem_parms.start = start;
	516	+ chmem_parms.end = end;
	517	+ chmem_parms.step = step;
	518	+ chmem_parms.newpp = newpp;
	519	+ chmem_parms.master_cpu = smp_processor_id();
	520	+
	521	+ cpus_read_lock();
	522	+
	523	+ atomic_set(&chmem_parms.cpu_counter, num_online_cpus());
	524	+
	525	+ // Ensure state is consistent before we call the other CPUs
	526	+ mb();
	527	+
	528	+ stop_machine_cpuslocked(change_memory_range_fn, &chmem_parms,
	529	+ cpu_online_mask);
	530	+
	531	+ cpus_read_unlock();
	532	+ mutex_unlock(&chmem_lock);
	533	+ } else
	534	+ change_memory_range(start, end, step, newpp);
425	535
426	536	return true;
427	537	}
428	538
429	539	void hash__mark_rodata_ro(void)
430	540	{
431		− unsigned long start, end;
	541	+ unsigned long start, end, pp;
432	542
433	543	start = (unsigned long)_stext;
434	544	end = (unsigned long)__init_begin;
435	545
436		− WARN_ON(!hash__change_memory_range(start, end, PP_RXXX));
	546	+ pp = htab_convert_pte_flags(pgprot_val(PAGE_KERNEL_ROX), HPTE_USE_KERNEL_KEY);
	547	+
	548	+ WARN_ON(!hash__change_memory_range(start, end, pp));
437	549	}
438	550
439	551	void hash__mark_initmem_nx(void)

arch/powerpc/mm/book3s64/hash_utils.c+22−13

@@ -338,7 +338,7 @@ repeat: · 337 unmodified lines
338	338	int htab_remove_mapping(unsigned long vstart, unsigned long vend,
339	339	int psize, int ssize)
340	340	{
341		− unsigned long vaddr;
	341	+ unsigned long vaddr, time_limit;
342	342	unsigned int step, shift;
343	343	int rc;
344	344	int ret = 0;
@@ -351,8 +351,19 @@ int htab_remove_mapping(unsigned long vstart, unsigned long vend, · 6 unmodified lines
351	351
352	352	/* Unmap the full range specificied */
353	353	vaddr = ALIGN_DOWN(vstart, step);
	354	+ time_limit = jiffies + HZ;
	355	+
354	356	for (;vaddr < vend; vaddr += step) {
355	357	rc = mmu_hash_ops.hpte_removebolted(vaddr, psize, ssize);
	358	+
	359	+ /*
	360	+ * For large number of mappings introduce a cond_resched()
	361	+ * to prevent softlockup warnings.
	362	+ */
	363	+ if (time_after(jiffies, time_limit)) {
	364	+ cond_resched();
	365	+ time_limit = jiffies + HZ;
	366	+ }
356	367	if (rc == -ENOENT) {
357	368	ret = -ENOENT;
358	369	continue;
@@ -1145,7 +1156,7 @@ unsigned int hash_page_do_lazy_icache(unsigned int pp, pte_t pte, int trap) · 786 unmodified lines
1145	1156
1146	1157	/* page is dirty */
1147	1158	if (!test_bit(PG_dcache_clean, &page->flags) && !PageReserved(page)) {
1148		− if (trap == 0x400) {
	1159	+ if (trap == INTERRUPT_INST_STORAGE) {
1149	1160	flush_dcache_icache_page(page);
1150	1161	set_bit(PG_dcache_clean, &page->flags);
1151	1162	} else
@@ -1545,10 +1556,10 @@ DEFINE_INTERRUPT_HANDLER_RET(__do_hash_fault) · 393 unmodified lines
1545	1556	if (user_mode(regs) \|\| (region_id == USER_REGION_ID))
1546	1557	access &= ~_PAGE_PRIVILEGED;
1547	1558
1548		− if (regs->trap == 0x400)
	1559	+ if (TRAP(regs) == INTERRUPT_INST_STORAGE)
1549	1560	access \|= _PAGE_EXEC;
1550	1561
1551		− err = hash_page_mm(mm, ea, access, regs->trap, flags);
	1562	+ err = hash_page_mm(mm, ea, access, TRAP(regs), flags);
1552	1563	if (unlikely(err < 0)) {
1553	1564	// failed to instert a hash PTE due to an hypervisor error
1554	1565	if (user_mode(regs)) {
@@ -1572,10 +1583,11 @@ DEFINE_INTERRUPT_HANDLER_RET(__do_hash_fault) · 17 unmodified lines
1572	1583	DEFINE_INTERRUPT_HANDLER_RAW(do_hash_fault)
1573	1584	{
1574	1585	unsigned long dsisr = regs->dsisr;
1575		− long err;
1576	1586
1577		− if (unlikely(dsisr & (DSISR_BAD_FAULT_64S \| DSISR_KEYFAULT)))
1578		− goto page_fault;
	1587	+ if (unlikely(dsisr & (DSISR_BAD_FAULT_64S \| DSISR_KEYFAULT))) {
	1588	+ hash__do_page_fault(regs);
	1589	+ return 0;
	1590	+ }
1579	1591
1580	1592	/*
1581	1593	* If we are in an "NMI" (e.g., an interrupt when soft-disabled), then
@@ -1595,13 +1607,10 @@ DEFINE_INTERRUPT_HANDLER_RAW(do_hash_fault) · 13 unmodified lines
1595	1607	return 0;
1596	1608	}
1597	1609
1598		− err = __do_hash_fault(regs);
1599		− if (err) {
1600		−page_fault:
1601		− err = hash__do_page_fault(regs);
1602		− }
	1610	+ if (__do_hash_fault(regs))
	1611	+ hash__do_page_fault(regs);
1603	1612
1604		− return err;
	1613	+ return 0;
1605	1614	}
1606	1615
1607	1616	#ifdef CONFIG_PPC_MM_SLICES

arch/powerpc/mm/book3s64/mmu_context.c+1−1

@@ -119,7 +119,7 @@ static int hash__init_new_context(struct mm_struct *mm) · 118 unmodified lines
119	119	/* This is fork. Copy hash_context details from current->mm */
120	120	memcpy(mm->context.hash_context, current->mm->context.hash_context, sizeof(struct hash_mm_context));
121	121	#ifdef CONFIG_PPC_SUBPAGE_PROT
122		− /* inherit subpage prot detalis if we have one. */
	122	+ /* inherit subpage prot details if we have one. */
123	123	if (current->mm->context.hash_context->spt) {
124	124	mm->context.hash_context->spt = kmalloc(sizeof(struct subpage_prot_table),
125	125	GFP_KERNEL);

arch/powerpc/mm/book3s64/pkeys.c+5−15

@@ -301,19 +301,6 @@ void setup_kuap(bool disabled) · 300 unmodified lines
301	301	}
302	302	#endif
303	303
304		−static inline void update_current_thread_amr(u64 value)
305		−{
306		− current->thread.regs->amr = value;
307		−}
308		−
309		−static inline void update_current_thread_iamr(u64 value)
310		−{
311		− if (!likely(pkey_execute_disable_supported))
312		− return;
313		−
314		− current->thread.regs->iamr = value;
315		−}
316		−
317	304	#ifdef CONFIG_PPC_MEM_KEYS
318	305	void pkey_mm_init(struct mm_struct *mm)
319	306	{
@@ -328,7 +315,7 @@ static inline void init_amr(int pkey, u8 init_bits) · 8 unmodified lines
328	315	u64 new_amr_bits = (((u64)init_bits & 0x3UL) << pkeyshift(pkey));
329	316	u64 old_amr = current_thread_amr() & ~((u64)(0x3ul) << pkeyshift(pkey));
330	317
331		− update_current_thread_amr(old_amr \| new_amr_bits);
	318	+ current->thread.regs->amr = old_amr \| new_amr_bits;
332	319	}
333	320
334	321	static inline void init_iamr(int pkey, u8 init_bits)
@@ -336,7 +323,10 @@ static inline void init_iamr(int pkey, u8 init_bits) · 1 unmodified line
336	323	u64 new_iamr_bits = (((u64)init_bits & 0x1UL) << pkeyshift(pkey));
337	324	u64 old_iamr = current_thread_iamr() & ~((u64)(0x1ul) << pkeyshift(pkey));
338	325
339		− update_current_thread_iamr(old_iamr \| new_iamr_bits);
	326	+ if (!likely(pkey_execute_disable_supported))
	327	+ return;
	328	+
	329	+ current->thread.regs->iamr = old_iamr \| new_iamr_bits;
340	330	}
341	331
342	332	/*

arch/powerpc/mm/book3s64/radix_pgtable.c+5−5

@@ -108,7 +108,7 @@ static int early_map_kernel_page(unsigned long ea, unsigned long pa, · 107 unmodified lines
108	108
109	109	set_the_pte:
110	110	set_pte_at(&init_mm, ea, ptep, pfn_pte(pfn, flags));
111		− smp_wmb();
	111	+ asm volatile("ptesync": : :"memory");
112	112	return 0;
113	113	}
114	114
@@ -168,7 +168,7 @@ static int __map_kernel_page(unsigned long ea, unsigned long pa, · 53 unmodified lines
168	168
169	169	set_the_pte:
170	170	set_pte_at(&init_mm, ea, ptep, pfn_pte(pfn, flags));
171		− smp_wmb();
	171	+ asm volatile("ptesync": : :"memory");
172	172	return 0;
173	173	}
174	174
@@ -180,8 +180,8 @@ int radix__map_kernel_page(unsigned long ea, unsigned long pa, · 5 unmodified lines
180	180	}
181	181
182	182	#ifdef CONFIG_STRICT_KERNEL_RWX
183		−void radix__change_memory_range(unsigned long start, unsigned long end,
184		− unsigned long clear)
	183	+static void radix__change_memory_range(unsigned long start, unsigned long end,
	184	+ unsigned long clear)
185	185	{
186	186	unsigned long idx;
187	187	pgd_t *pgdp;
@@ -1058,7 +1058,7 @@ void radix__ptep_set_access_flags(struct vm_area_struct vma, pte_t ptep, · 870 unmodified lines
1058	1058	* Book3S does not require a TLB flush when relaxing access
1059	1059	* restrictions when the address space is not attached to a
1060	1060	* NMMU, because the core MMU will reload the pte after taking
1061		− * an access fault, which is defined by the architectue.
	1061	+ * an access fault, which is defined by the architecture.
1062	1062	*/
1063	1063	}
1064	1064	/* See ptesync comment in radix__set_pte_at */

arch/powerpc/mm/cacheflush.c+234−0

@@ -0,0 +1,234 @@
	1	+// SPDX-License-Identifier: GPL-2.0-or-later
	2	+
	3	+#include <linux/highmem.h>
	4	+#include <linux/kprobes.h>
	5	+
	6	+/**
	7	+ * flush_coherent_icache() - if a CPU has a coherent icache, flush it
	8	+ * Return true if the cache was flushed, false otherwise
	9	+ */
	10	+static inline bool flush_coherent_icache(void)
	11	+{
	12	+ /*
	13	+ * For a snooping icache, we still need a dummy icbi to purge all the
	14	+ * prefetched instructions from the ifetch buffers. We also need a sync
	15	+ * before the icbi to order the the actual stores to memory that might
	16	+ * have modified instructions with the icbi.
	17	+ */
	18	+ if (cpu_has_feature(CPU_FTR_COHERENT_ICACHE)) {
	19	+ mb(); /* sync */
	20	+ icbi((void *)PAGE_OFFSET);
	21	+ mb(); /* sync */
	22	+ isync();
	23	+ return true;
	24	+ }
	25	+
	26	+ return false;
	27	+}
	28	+
	29	+/**
	30	+ * invalidate_icache_range() - Flush the icache by issuing icbi across an address range
	31	+ * @start: the start address
	32	+ * @stop: the stop address (exclusive)
	33	+ */
	34	+static void invalidate_icache_range(unsigned long start, unsigned long stop)
	35	+{
	36	+ unsigned long shift = l1_icache_shift();
	37	+ unsigned long bytes = l1_icache_bytes();
	38	+ char addr = (char )(start & ~(bytes - 1));
	39	+ unsigned long size = stop - (unsigned long)addr + (bytes - 1);
	40	+ unsigned long i;
	41	+
	42	+ for (i = 0; i < size >> shift; i++, addr += bytes)
	43	+ icbi(addr);
	44	+
	45	+ mb(); /* sync */
	46	+ isync();
	47	+}
	48	+
	49	+/**
	50	+ * flush_icache_range: Write any modified data cache blocks out to memory
	51	+ * and invalidate the corresponding blocks in the instruction cache
	52	+ *
	53	+ * Generic code will call this after writing memory, before executing from it.
	54	+ *
	55	+ * @start: the start address
	56	+ * @stop: the stop address (exclusive)
	57	+ */
	58	+void flush_icache_range(unsigned long start, unsigned long stop)
	59	+{
	60	+ if (flush_coherent_icache())
	61	+ return;
	62	+
	63	+ clean_dcache_range(start, stop);
	64	+
	65	+ if (IS_ENABLED(CONFIG_44x)) {
	66	+ /*
	67	+ * Flash invalidate on 44x because we are passed kmapped
	68	+ * addresses and this doesn't work for userspace pages due to
	69	+ * the virtually tagged icache.
	70	+ */
	71	+ iccci((void *)start);
	72	+ mb(); /* sync */
	73	+ isync();
	74	+ } else
	75	+ invalidate_icache_range(start, stop);
	76	+}
	77	+EXPORT_SYMBOL(flush_icache_range);
	78	+
	79	+#ifdef CONFIG_HIGHMEM
	80	+/**
	81	+ * flush_dcache_icache_phys() - Flush a page by it's physical address
	82	+ * @physaddr: the physical address of the page
	83	+ */
	84	+static void flush_dcache_icache_phys(unsigned long physaddr)
	85	+{
	86	+ unsigned long bytes = l1_dcache_bytes();
	87	+ unsigned long nb = PAGE_SIZE / bytes;
	88	+ unsigned long addr = physaddr & PAGE_MASK;
	89	+ unsigned long msr, msr0;
	90	+ unsigned long loop1 = addr, loop2 = addr;
	91	+
	92	+ msr0 = mfmsr();
	93	+ msr = msr0 & ~MSR_DR;
	94	+ /*
	95	+ * This must remain as ASM to prevent potential memory accesses
	96	+ * while the data MMU is disabled
	97	+ */
	98	+ asm volatile(
	99	+ " mtctr %2;\n"
	100	+ " mtmsr %3;\n"
	101	+ " isync;\n"
	102	+ "0: dcbst 0, %0;\n"
	103	+ " addi %0, %0, %4;\n"
	104	+ " bdnz 0b;\n"
	105	+ " sync;\n"
	106	+ " mtctr %2;\n"
	107	+ "1: icbi 0, %1;\n"
	108	+ " addi %1, %1, %4;\n"
	109	+ " bdnz 1b;\n"
	110	+ " sync;\n"
	111	+ " mtmsr %5;\n"
	112	+ " isync;\n"
	113	+ : "+&r" (loop1), "+&r" (loop2)
	114	+ : "r" (nb), "r" (msr), "i" (bytes), "r" (msr0)
	115	+ : "ctr", "memory");
	116	+}
	117	+NOKPROBE_SYMBOL(flush_dcache_icache_phys)
	118	+#else
	119	+static void flush_dcache_icache_phys(unsigned long physaddr)
	120	+{
	121	+}
	122	+#endif
	123	+
	124	+/**
	125	+ * __flush_dcache_icache(): Flush a particular page from the data cache to RAM.
	126	+ * Note: this is necessary because the instruction cache does not
	127	+ * snoop from the data cache.
	128	+ *
	129	+ * @p: the address of the page to flush
	130	+ */
	131	+static void __flush_dcache_icache(void *p)
	132	+{
	133	+ unsigned long addr = (unsigned long)p & PAGE_MASK;
	134	+
	135	+ clean_dcache_range(addr, addr + PAGE_SIZE);
	136	+
	137	+ /*
	138	+ * We don't flush the icache on 44x. Those have a virtual icache and we
	139	+ * don't have access to the virtual address here (it's not the page
	140	+ * vaddr but where it's mapped in user space). The flushing of the
	141	+ * icache on these is handled elsewhere, when a change in the address
	142	+ * space occurs, before returning to user space.
	143	+ */
	144	+
	145	+ if (mmu_has_feature(MMU_FTR_TYPE_44x))
	146	+ return;
	147	+
	148	+ invalidate_icache_range(addr, addr + PAGE_SIZE);
	149	+}
	150	+
	151	+static void flush_dcache_icache_hugepage(struct page *page)
	152	+{
	153	+ int i;
	154	+ int nr = compound_nr(page);
	155	+
	156	+ if (!PageHighMem(page)) {
	157	+ for (i = 0; i < nr; i++)
	158	+ __flush_dcache_icache(lowmem_page_address(page + i));
	159	+ } else {
	160	+ for (i = 0; i < nr; i++) {
	161	+ void *start = kmap_local_page(page + i);
	162	+
	163	+ __flush_dcache_icache(start);
	164	+ kunmap_local(start);
	165	+ }
	166	+ }
	167	+}
	168	+
	169	+void flush_dcache_icache_page(struct page *page)
	170	+{
	171	+ if (flush_coherent_icache())
	172	+ return;
	173	+
	174	+ if (PageCompound(page))
	175	+ return flush_dcache_icache_hugepage(page);
	176	+
	177	+ if (!PageHighMem(page)) {
	178	+ __flush_dcache_icache(lowmem_page_address(page));
	179	+ } else if (IS_ENABLED(CONFIG_BOOKE) \|\| sizeof(phys_addr_t) > sizeof(void *)) {
	180	+ void *start = kmap_local_page(page);
	181	+
	182	+ __flush_dcache_icache(start);
	183	+ kunmap_local(start);
	184	+ } else {
	185	+ flush_dcache_icache_phys(page_to_phys(page));
	186	+ }
	187	+}
	188	+EXPORT_SYMBOL(flush_dcache_icache_page);
	189	+
	190	+void clear_user_page(void page, unsigned long vaddr, struct page pg)
	191	+{
	192	+ clear_page(page);
	193	+
	194	+ /*
	195	+ * We shouldn't have to do this, but some versions of glibc
	196	+ * require it (ld.so assumes zero filled pages are icache clean)
	197	+ * - Anton
	198	+ */
	199	+ flush_dcache_page(pg);
	200	+}
	201	+EXPORT_SYMBOL(clear_user_page);
	202	+
	203	+void copy_user_page(void vto, void vfrom, unsigned long vaddr,
	204	+ struct page *pg)
	205	+{
	206	+ copy_page(vto, vfrom);
	207	+
	208	+ /*
	209	+ * We should be able to use the following optimisation, however
	210	+ * there are two problems.
	211	+ * Firstly a bug in some versions of binutils meant PLT sections
	212	+ * were not marked executable.
	213	+ * Secondly the first word in the GOT section is blrl, used
	214	+ * to establish the GOT address. Until recently the GOT was
	215	+ * not marked executable.
	216	+ * - Anton
	217	+ */
	218	+#if 0
	219	+ if (!vma->vm_file && ((vma->vm_flags & VM_EXEC) == 0))
	220	+ return;
	221	+#endif
	222	+
	223	+ flush_dcache_page(pg);
	224	+}
	225	+
	226	+void flush_icache_user_page(struct vm_area_struct vma, struct page page,
	227	+ unsigned long addr, int len)
	228	+{
	229	+ void *maddr;
	230	+
	231	+ maddr = kmap_local_page(page) + (addr & ~PAGE_MASK);
	232	+ flush_icache_range((unsigned long)maddr, (unsigned long)maddr + len);
	233	+ kunmap_local(maddr);
	234	+}

arch/powerpc/mm/fault.c+23−37

@@ -32,6 +32,8 @@ · 31 unmodified lines
32	32	#include <linux/context_tracking.h>
33	33	#include <linux/hugetlb.h>
34	34	#include <linux/uaccess.h>
	35	+#include <linux/kfence.h>
	36	+#include <linux/pkeys.h>
35	37
36	38	#include <asm/firmware.h>
37	39	#include <asm/interrupt.h>
@@ -87,7 +89,6 @@ static noinline int bad_area(struct pt_regs *regs, unsigned long address) · 49 unmodified lines
87	89	return __bad_area(regs, address, SEGV_MAPERR);
88	90	}
89	91
90		−#ifdef CONFIG_PPC_MEM_KEYS
91	92	static noinline int bad_access_pkey(struct pt_regs *regs, unsigned long address,
92	93	struct vm_area_struct *vma)
93	94	{
@@ -127,7 +128,6 @@ static noinline int bad_access_pkey(struct pt_regs *regs, unsigned long address, · 33 unmodified lines
127	128
128	129	return 0;
129	130	}
130		−#endif
131	131
132	132	static noinline int bad_access(struct pt_regs *regs, unsigned long address)
133	133	{
@@ -197,7 +197,7 @@ static int mm_fault_error(struct pt_regs *regs, unsigned long addr, · 63 unmodified lines
197	197	static bool bad_kernel_fault(struct pt_regs *regs, unsigned long error_code,
198	198	unsigned long address, bool is_write)
199	199	{
200		− int is_exec = TRAP(regs) == 0x400;
	200	+ int is_exec = TRAP(regs) == INTERRUPT_INST_STORAGE;
201	201
202	202	/* NX faults set DSISR_PROTFAULT on the 8xx, DSISR_NOEXEC_OR_G on others */
203	203	if (is_exec && (error_code & (DSISR_NOEXEC_OR_G \| DSISR_KEYFAULT \|
@@ -234,7 +234,6 @@ static bool bad_kernel_fault(struct pt_regs *regs, unsigned long error_code, · 30 unmodified lines
234	234	return false;
235	235	}
236	236
237		−#ifdef CONFIG_PPC_MEM_KEYS
238	237	static bool access_pkey_error(bool is_write, bool is_exec, bool is_pkey,
239	238	struct vm_area_struct *vma)
240	239	{
@@ -248,7 +247,6 @@ static bool access_pkey_error(bool is_write, bool is_exec, bool is_pkey, · 7 unmodified lines
248	247
249	248	return false;
250	249	}
251		−#endif
252	250
253	251	static bool access_error(bool is_write, bool is_exec, struct vm_area_struct *vma)
254	252	{
@@ -393,7 +391,7 @@ static int ___do_page_fault(struct pt_regs *regs, unsigned long address, · 138 unmodified lines
393	391	struct vm_area_struct * vma;
394	392	struct mm_struct *mm = current->mm;
395	393	unsigned int flags = FAULT_FLAG_DEFAULT;
396		− int is_exec = TRAP(regs) == 0x400;
	394	+ int is_exec = TRAP(regs) == INTERRUPT_INST_STORAGE;
397	395	int is_user = user_mode(regs);
398	396	int is_write = page_fault_is_write(error_code);
399	397	vm_fault_t fault, major = 0;
@@ -418,8 +416,12 @@ static int ___do_page_fault(struct pt_regs *regs, unsigned long address, · 18 unmodified lines
418	416	* take a page fault to a kernel address or a page fault to a user
419	417	* address outside of dedicated places
420	418	*/
421		− if (unlikely(!is_user && bad_kernel_fault(regs, error_code, address, is_write)))
	419	+ if (unlikely(!is_user && bad_kernel_fault(regs, error_code, address, is_write))) {
	420	+ if (kfence_handle_page_fault(address, is_write, regs))
	421	+ return 0;
	422	+
422	423	return SIGSEGV;
	424	+ }
423	425
424	426	/*
425	427	* If we're in an interrupt, have no user context or are running
@@ -492,11 +494,9 @@ retry: · 66 unmodified lines
492	494	return bad_area(regs, address);
493	495	}
494	496
495		−#ifdef CONFIG_PPC_MEM_KEYS
496	497	if (unlikely(access_pkey_error(is_write, is_exec,
497	498	(error_code & DSISR_KEYFAULT), vma)))
498	499	return bad_access_pkey(regs, address, vma);
499		−#endif /* CONFIG_PPC_MEM_KEYS */
500	500
501	501	if (unlikely(access_error(is_write, is_exec, vma)))
502	502	return bad_access(regs, address);
@@ -539,39 +539,25 @@ retry: · 36 unmodified lines
539	539	}
540	540	NOKPROBE_SYMBOL(___do_page_fault);
541	541
542		−static long __do_page_fault(struct pt_regs *regs)
	542	+static __always_inline void __do_page_fault(struct pt_regs *regs)
543	543	{
544		− const struct exception_table_entry *entry;
545	544	long err;
546	545
547	546	err = ___do_page_fault(regs, regs->dar, regs->dsisr);
548		− if (likely(!err))
549		− return err;
550		−
551		− entry = search_exception_tables(regs->nip);
552		− if (likely(entry)) {
553		− instruction_pointer_set(regs, extable_fixup(entry));
554		− return 0;
555		− } else if (IS_ENABLED(CONFIG_PPC_BOOK3S_64)) {
556		− __bad_page_fault(regs, err);
557		− return 0;
558		− } else {
559		− /* 32 and 64e handle the bad page fault in asm */
560		− return err;
561		− }
	547	+ if (unlikely(err))
	548	+ bad_page_fault(regs, err);
562	549	}
563		−NOKPROBE_SYMBOL(__do_page_fault);
564	550
565		−DEFINE_INTERRUPT_HANDLER_RET(do_page_fault)
	551	+DEFINE_INTERRUPT_HANDLER(do_page_fault)
566	552	{
567		− return __do_page_fault(regs);
	553	+ __do_page_fault(regs);
568	554	}
569	555
570	556	#ifdef CONFIG_PPC_BOOK3S_64
571	557	/* Same as do_page_fault but interrupt entry has already run in do_hash_fault */
572		−long hash__do_page_fault(struct pt_regs *regs)
	558	+void hash__do_page_fault(struct pt_regs *regs)
573	559	{
574		− return __do_page_fault(regs);
	560	+ __do_page_fault(regs);
575	561	}
576	562	NOKPROBE_SYMBOL(hash__do_page_fault);
577	563	#endif
@@ -581,27 +567,27 @@ NOKPROBE_SYMBOL(hash__do_page_fault); · 3 unmodified lines
581	567	* It is called from the DSI and ISI handlers in head.S and from some
582	568	* of the procedures in traps.c.
583	569	*/
584		−void __bad_page_fault(struct pt_regs *regs, int sig)
	570	+static void __bad_page_fault(struct pt_regs *regs, int sig)
585	571	{
586	572	int is_write = page_fault_is_write(regs->dsisr);
587	573
588	574	/* kernel has accessed a bad area */
589	575
590	576	switch (TRAP(regs)) {
591		− case 0x300:
592		− case 0x380:
593		− case 0xe00:
	577	+ case INTERRUPT_DATA_STORAGE:
	578	+ case INTERRUPT_DATA_SEGMENT:
	579	+ case INTERRUPT_H_DATA_STORAGE:
594	580	pr_alert("BUG: %s on %s at 0x%08lx\n",
595	581	regs->dar < PAGE_SIZE ? "Kernel NULL pointer dereference" :
596	582	"Unable to handle kernel data access",
597	583	is_write ? "write" : "read", regs->dar);
598	584	break;
599		− case 0x400:
600		− case 0x480:
	585	+ case INTERRUPT_INST_STORAGE:
	586	+ case INTERRUPT_INST_SEGMENT:
601	587	pr_alert("BUG: Unable to handle kernel instruction fetch%s",
602	588	regs->nip < PAGE_SIZE ? " (NULL pointer?)\n" : "\n");
603	589	break;
604		− case 0x600:
	590	+ case INTERRUPT_ALIGNMENT:
605	591	pr_alert("BUG: Unable to handle kernel unaligned access at 0x%08lx\n",
606	592	regs->dar);
607	593	break;

arch/powerpc/mm/init_32.c+3−0

@@ -97,6 +97,9 @@ static void __init MMU_setup(void) · 96 unmodified lines
97	97	if (IS_ENABLED(CONFIG_PPC_8xx))
98	98	return;
99	99
	100	+ if (IS_ENABLED(CONFIG_KFENCE))
	101	+ __map_without_ltlbs = 1;
	102	+
100	103	if (debug_pagealloc_enabled())
101	104	__map_without_ltlbs = 1;
102	105

arch/powerpc/mm/maccess.c+21−0

@@ -3,7 +3,28 @@ · 2 unmodified lines
3	3	#include <linux/uaccess.h>
4	4	#include <linux/kernel.h>
5	5
	6	+#include <asm/disassemble.h>
	7	+#include <asm/inst.h>
	8	+#include <asm/ppc-opcode.h>
	9	+
6	10	bool copy_from_kernel_nofault_allowed(const void *unsafe_src, size_t size)
7	11	{
8	12	return is_kernel_addr((unsigned long)unsafe_src);
9	13	}
	14	+
	15	+int copy_inst_from_kernel_nofault(struct ppc_inst inst, struct ppc_inst src)
	16	+{
	17	+ unsigned int val, suffix;
	18	+ int err;
	19	+
	20	+ err = copy_from_kernel_nofault(&val, src, sizeof(val));
	21	+ if (err)
	22	+ return err;
	23	+ if (IS_ENABLED(CONFIG_PPC64) && get_op(val) == OP_PREFIX) {
	24	+ err = copy_from_kernel_nofault(&suffix, (void *)src + 4, 4);
	25	+ *inst = ppc_inst_prefix(val, suffix);
	26	+ } else {
	27	+ *inst = ppc_inst(val);
	28	+ }
	29	+ return err;
	30	+}

arch/powerpc/mm/mem.c+1−282

@@ -12,49 +12,18 @@ · 11 unmodified lines
12	12	* Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
13	13	*/
14	14
15		−#include <linux/export.h>
16		−#include <linux/sched.h>
17		−#include <linux/kernel.h>
18		−#include <linux/errno.h>
19		−#include <linux/string.h>
20		−#include <linux/gfp.h>
21		−#include <linux/types.h>
22		−#include <linux/mm.h>
23		−#include <linux/stddef.h>
24		−#include <linux/init.h>
25	15	#include <linux/memblock.h>
26	16	#include <linux/highmem.h>
27		−#include <linux/initrd.h>
28		−#include <linux/pagemap.h>
29	17	#include <linux/suspend.h>
30		−#include <linux/hugetlb.h>
31		−#include <linux/slab.h>
32		−#include <linux/vmalloc.h>
33		−#include <linux/memremap.h>
34	18	#include <linux/dma-direct.h>
35		−#include <linux/kprobes.h>
36	19
37		−#include <asm/prom.h>
38		−#include <asm/io.h>
39		−#include <asm/mmu_context.h>
40		−#include <asm/mmu.h>
41		−#include <asm/smp.h>
42	20	#include <asm/machdep.h>
43		−#include <asm/btext.h>
44		−#include <asm/tlb.h>
45		−#include <asm/sections.h>
46		−#include <asm/sparsemem.h>
47		−#include <asm/vdso.h>
48		−#include <asm/fixmap.h>
49		−#include <asm/swiotlb.h>
50	21	#include <asm/rtas.h>
51	22	#include <asm/kasan.h>
52	23	#include <asm/svm.h>
53		−#include <asm/mmzone.h>
54	24
55	25	#include <mm/mmu_decl.h>
56	26
57		−static DEFINE_MUTEX(linear_mapping_mutex);
58	27	unsigned long long memory_limit;
59	28	bool init_mem_is_free;
60	29
@@ -72,6 +41,7 @@ pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn, · 11 unmodified lines
72	41	EXPORT_SYMBOL(phys_mem_access_prot);
73	42
74	43	#ifdef CONFIG_MEMORY_HOTPLUG
	44	+static DEFINE_MUTEX(linear_mapping_mutex);
75	45
76	46	#ifdef CONFIG_NUMA
77	47	int memory_add_physaddr_to_nid(u64 start)
@@ -340,257 +310,6 @@ void free_initmem(void) · 262 unmodified lines
340	310	free_initmem_default(POISON_FREE_INITMEM);
341	311	}
342	312
343		−/**
344		− * flush_coherent_icache() - if a CPU has a coherent icache, flush it
345		− * @addr: The base address to use (can be any valid address, the whole cache will be flushed)
346		− * Return true if the cache was flushed, false otherwise
347		− */
348		−static inline bool flush_coherent_icache(unsigned long addr)
349		−{
350		− /*
351		− * For a snooping icache, we still need a dummy icbi to purge all the
352		− * prefetched instructions from the ifetch buffers. We also need a sync
353		− * before the icbi to order the the actual stores to memory that might
354		− * have modified instructions with the icbi.
355		− */
356		− if (cpu_has_feature(CPU_FTR_COHERENT_ICACHE)) {
357		− mb(); /* sync */
358		− allow_read_from_user((const void __user *)addr, L1_CACHE_BYTES);
359		− icbi((void *)addr);
360		− prevent_read_from_user((const void __user *)addr, L1_CACHE_BYTES);
361		− mb(); /* sync */
362		− isync();
363		− return true;
364		− }
365		−
366		− return false;
367		−}
368		−
369		−/**
370		− * invalidate_icache_range() - Flush the icache by issuing icbi across an address range
371		− * @start: the start address
372		− * @stop: the stop address (exclusive)
373		− */
374		−static void invalidate_icache_range(unsigned long start, unsigned long stop)
375		−{
376		− unsigned long shift = l1_icache_shift();
377		− unsigned long bytes = l1_icache_bytes();
378		− char addr = (char )(start & ~(bytes - 1));
379		− unsigned long size = stop - (unsigned long)addr + (bytes - 1);
380		− unsigned long i;
381		−
382		− for (i = 0; i < size >> shift; i++, addr += bytes)
383		− icbi(addr);
384		−
385		− mb(); /* sync */
386		− isync();
387		−}
388		−
389		−/**
390		− * flush_icache_range: Write any modified data cache blocks out to memory
391		− * and invalidate the corresponding blocks in the instruction cache
392		− *
393		− * Generic code will call this after writing memory, before executing from it.
394		− *
395		− * @start: the start address
396		− * @stop: the stop address (exclusive)
397		− */
398		−void flush_icache_range(unsigned long start, unsigned long stop)
399		−{
400		− if (flush_coherent_icache(start))
401		− return;
402		−
403		− clean_dcache_range(start, stop);
404		−
405		− if (IS_ENABLED(CONFIG_44x)) {
406		− /*
407		− * Flash invalidate on 44x because we are passed kmapped
408		− * addresses and this doesn't work for userspace pages due to
409		− * the virtually tagged icache.
410		− */
411		− iccci((void *)start);
412		− mb(); /* sync */
413		− isync();
414		− } else
415		− invalidate_icache_range(start, stop);
416		−}
417		−EXPORT_SYMBOL(flush_icache_range);
418		−
419		−#if !defined(CONFIG_PPC_8xx) && !defined(CONFIG_PPC64)
420		−/**
421		− * flush_dcache_icache_phys() - Flush a page by it's physical address
422		− * @physaddr: the physical address of the page
423		− */
424		−static void flush_dcache_icache_phys(unsigned long physaddr)
425		−{
426		− unsigned long bytes = l1_dcache_bytes();
427		− unsigned long nb = PAGE_SIZE / bytes;
428		− unsigned long addr = physaddr & PAGE_MASK;
429		− unsigned long msr, msr0;
430		− unsigned long loop1 = addr, loop2 = addr;
431		−
432		− msr0 = mfmsr();
433		− msr = msr0 & ~MSR_DR;
434		− /*
435		− * This must remain as ASM to prevent potential memory accesses
436		− * while the data MMU is disabled
437		− */
438		− asm volatile(
439		− " mtctr %2;\n"
440		− " mtmsr %3;\n"
441		− " isync;\n"
442		− "0: dcbst 0, %0;\n"
443		− " addi %0, %0, %4;\n"
444		− " bdnz 0b;\n"
445		− " sync;\n"
446		− " mtctr %2;\n"
447		− "1: icbi 0, %1;\n"
448		− " addi %1, %1, %4;\n"
449		− " bdnz 1b;\n"
450		− " sync;\n"
451		− " mtmsr %5;\n"
452		− " isync;\n"
453		− : "+&r" (loop1), "+&r" (loop2)
454		− : "r" (nb), "r" (msr), "i" (bytes), "r" (msr0)
455		− : "ctr", "memory");
456		−}
457		−NOKPROBE_SYMBOL(flush_dcache_icache_phys)
458		−#endif // !defined(CONFIG_PPC_8xx) && !defined(CONFIG_PPC64)
459		−
460		−/*
461		− * This is called when a page has been modified by the kernel.
462		− * It just marks the page as not i-cache clean. We do the i-cache
463		− * flush later when the page is given to a user process, if necessary.
464		− */
465		−void flush_dcache_page(struct page *page)
466		−{
467		− if (cpu_has_feature(CPU_FTR_COHERENT_ICACHE))
468		− return;
469		− /* avoid an atomic op if possible */
470		− if (test_bit(PG_dcache_clean, &page->flags))
471		− clear_bit(PG_dcache_clean, &page->flags);
472		−}
473		−EXPORT_SYMBOL(flush_dcache_page);
474		−
475		−static void flush_dcache_icache_hugepage(struct page *page)
476		−{
477		− int i;
478		− void *start;
479		−
480		− BUG_ON(!PageCompound(page));
481		−
482		− for (i = 0; i < compound_nr(page); i++) {
483		− if (!PageHighMem(page)) {
484		− __flush_dcache_icache(page_address(page+i));
485		− } else {
486		− start = kmap_atomic(page+i);
487		− __flush_dcache_icache(start);
488		− kunmap_atomic(start);
489		− }
490		− }
491		−}
492		−
493		−void flush_dcache_icache_page(struct page *page)
494		−{
495		−
496		− if (PageCompound(page))
497		− return flush_dcache_icache_hugepage(page);
498		−
499		−#if defined(CONFIG_PPC_8xx) \|\| defined(CONFIG_PPC64)
500		− /* On 8xx there is no need to kmap since highmem is not supported */
501		− __flush_dcache_icache(page_address(page));
502		−#else
503		− if (IS_ENABLED(CONFIG_BOOKE) \|\| sizeof(phys_addr_t) > sizeof(void *)) {
504		− void *start = kmap_atomic(page);
505		− __flush_dcache_icache(start);
506		− kunmap_atomic(start);
507		− } else {
508		− unsigned long addr = page_to_pfn(page) << PAGE_SHIFT;
509		−
510		− if (flush_coherent_icache(addr))
511		− return;
512		− flush_dcache_icache_phys(addr);
513		− }
514		−#endif
515		−}
516		−EXPORT_SYMBOL(flush_dcache_icache_page);
517		−
518		−/**
519		− * __flush_dcache_icache(): Flush a particular page from the data cache to RAM.
520		− * Note: this is necessary because the instruction cache does not
521		− * snoop from the data cache.
522		− *
523		− * @page: the address of the page to flush
524		− */
525		−void __flush_dcache_icache(void *p)
526		−{
527		− unsigned long addr = (unsigned long)p;
528		−
529		− if (flush_coherent_icache(addr))
530		− return;
531		−
532		− clean_dcache_range(addr, addr + PAGE_SIZE);
533		−
534		− /*
535		− * We don't flush the icache on 44x. Those have a virtual icache and we
536		− * don't have access to the virtual address here (it's not the page
537		− * vaddr but where it's mapped in user space). The flushing of the
538		− * icache on these is handled elsewhere, when a change in the address
539		− * space occurs, before returning to user space.
540		− */
541		−
542		− if (mmu_has_feature(MMU_FTR_TYPE_44x))
543		− return;
544		−
545		− invalidate_icache_range(addr, addr + PAGE_SIZE);
546		−}
547		−
548		−void clear_user_page(void page, unsigned long vaddr, struct page pg)
549		−{
550		− clear_page(page);
551		−
552		− /*
553		− * We shouldn't have to do this, but some versions of glibc
554		− * require it (ld.so assumes zero filled pages are icache clean)
555		− * - Anton
556		− */
557		− flush_dcache_page(pg);
558		−}
559		−EXPORT_SYMBOL(clear_user_page);
560		−
561		−void copy_user_page(void vto, void vfrom, unsigned long vaddr,
562		− struct page *pg)
563		−{
564		− copy_page(vto, vfrom);
565		−
566		− /*
567		− * We should be able to use the following optimisation, however
568		− * there are two problems.
569		− * Firstly a bug in some versions of binutils meant PLT sections
570		− * were not marked executable.
571		− * Secondly the first word in the GOT section is blrl, used
572		− * to establish the GOT address. Until recently the GOT was
573		− * not marked executable.
574		− * - Anton
575		− */
576		−#if 0
577		− if (!vma->vm_file && ((vma->vm_flags & VM_EXEC) == 0))
578		− return;
579		−#endif
580		−
581		− flush_dcache_page(pg);
582		−}
583		−
584		−void flush_icache_user_page(struct vm_area_struct vma, struct page page,
585		− unsigned long addr, int len)
586		−{
587		− unsigned long maddr;
588		−
589		− maddr = (unsigned long) kmap(page) + (addr & ~PAGE_MASK);
590		− flush_icache_range(maddr, maddr + len);
591		− kunmap(page);
592		−}
593		−
594	313	/*
595	314	* System memory should not be in /proc/iomem but various tools expect it
596	315	* (eg kdump).

arch/powerpc/mm/mmu_context.c+13−11

@@ -43,24 +43,26 @@ void switch_mm_irqs_off(struct mm_struct prev, struct mm_struct next, · 42 unmodified lines
43	43
44	44	/*
45	45	* This full barrier orders the store to the cpumask above vs
46		− * a subsequent operation which allows this CPU to begin loading
47		− * translations for next.
	46	+ * a subsequent load which allows this CPU/MMU to begin loading
	47	+ * translations for 'next' from page table PTEs into the TLB.
48	48	*
49		− * When using the radix MMU that operation is the load of the
	49	+ * When using the radix MMU, that operation is the load of the
50	50	* MMU context id, which is then moved to SPRN_PID.
51	51	*
52	52	* For the hash MMU it is either the first load from slb_cache
53		− * in switch_slb(), and/or the store of paca->mm_ctx_id in
54		− * copy_mm_to_paca().
	53	+ * in switch_slb() to preload the SLBs, or the load of
	54	+ * get_user_context which loads the context for the VSID hash
	55	+ * to insert a new SLB, in the SLB fault handler.
55	56	*
56	57	* On the other side, the barrier is in mm/tlb-radix.c for
57		− * radix which orders earlier stores to clear the PTEs vs
58		− * the load of mm_cpumask. And pte_xchg which does the same
59		− * thing for hash.
	58	+ * radix which orders earlier stores to clear the PTEs before
	59	+ * the load of mm_cpumask to check which CPU TLBs should be
	60	+ * flushed. For hash, pte_xchg to clear the PTE includes the
	61	+ * barrier.
60	62	*
61		− * This full barrier is needed by membarrier when switching
62		− * between processes after store to rq->curr, before user-space
63		− * memory accesses.
	63	+ * This full barrier is also needed by membarrier when
	64	+ * switching between processes after store to rq->curr, before
	65	+ * user-space memory accesses.
64	66	*/
65	67	smp_mb();
66	68

arch/powerpc/mm/mmu_decl.h+5−0

@@ -185,3 +185,8 @@ void ptdump_check_wx(void); · 184 unmodified lines
185	185	#else
186	186	static inline void ptdump_check_wx(void) { }
187	187	#endif
	188	+
	189	+static inline bool debug_pagealloc_enabled_or_kfence(void)
	190	+{
	191	+ return IS_ENABLED(CONFIG_KFENCE) \|\| debug_pagealloc_enabled();
	192	+}

arch/powerpc/mm/nohash/8xx.c+2−2

@@ -149,7 +149,7 @@ unsigned long __init mmu_mapin_ram(unsigned long base, unsigned long top) · 148 unmodified lines
149	149	{
150	150	unsigned long etext8 = ALIGN(__pa(_etext), SZ_8M);
151	151	unsigned long sinittext = __pa(_sinittext);
152		− bool strict_boundary = strict_kernel_rwx_enabled() \|\| debug_pagealloc_enabled();
	152	+ bool strict_boundary = strict_kernel_rwx_enabled() \|\| debug_pagealloc_enabled_or_kfence();
153	153	unsigned long boundary = strict_boundary ? sinittext : etext8;
154	154	unsigned long einittext8 = ALIGN(__pa(_einittext), SZ_8M);
155	155
@@ -161,7 +161,7 @@ unsigned long __init mmu_mapin_ram(unsigned long base, unsigned long top) · 5 unmodified lines
161	161	return 0;
162	162
163	163	mmu_mapin_ram_chunk(0, boundary, PAGE_KERNEL_TEXT, true);
164		− if (debug_pagealloc_enabled()) {
	164	+ if (debug_pagealloc_enabled_or_kfence()) {
165	165	top = boundary;
166	166	} else {
167	167	mmu_mapin_ram_chunk(boundary, einittext8, PAGE_KERNEL_TEXT, true);

arch/powerpc/net/Makefile+1−5

@@ -2,8 +2,4 @@ · 1 unmodified line
2	2	#
3	3	# Arch-specific network modules
4	4	#
5		−ifdef CONFIG_PPC64
6		−obj-$(CONFIG_BPF_JIT) += bpf_jit_comp64.o
7		−else
8		−obj-$(CONFIG_BPF_JIT) += bpf_jit_asm.o bpf_jit_comp.o
9		−endif
	5	+obj-$(CONFIG_BPF_JIT) += bpf_jit_comp.o bpf_jit_comp$(BITS).o

arch/powerpc/net/bpf_jit.h+64−0

@@ -26,6 +26,9 @@ · 25 unmodified lines
26	26	/* Long jump; (unconditional 'branch') */
27	27	#define PPC_JMP(dest) EMIT(PPC_INST_BRANCH \| \
28	28	(((dest) - (ctx->idx * 4)) & 0x03fffffc))
	29	+/* blr; (unconditional 'branch' with link) to absolute address */
	30	+#define PPC_BL_ABS(dest) EMIT(PPC_INST_BL \| \
	31	+ (((dest) - (unsigned long)(image + ctx->idx)) & 0x03fffffc))
29	32	/* "cond" here covers BO:BI fields. */
30	33	#define PPC_BCC_SHORT(cond, dest) EMIT(PPC_INST_BRANCH_COND \| \
31	34	(((cond) & 0x3ff) << 16) \| \
@@ -42,6 +45,10 @@ · 10 unmodified lines
42	45	EMIT(PPC_RAW_ORI(d, d, IMM_L(i))); \
43	46	} } while(0)
44	47
	48	+#ifdef CONFIG_PPC32
	49	+#define PPC_EX32(r, i) EMIT(PPC_RAW_LI((r), (i) < 0 ? -1 : 0))
	50	+#endif
	51	+
45	52	#define PPC_LI64(d, i) do { \
46	53	if ((long)(i) >= -2147483648 && \
47	54	(long)(i) < 2147483648) \
@@ -108,6 +115,63 @@ static inline bool is_nearbranch(int offset) · 60 unmodified lines
108	115	#define COND_LT (CR0_LT \| COND_CMP_TRUE)
109	116	#define COND_LE (CR0_GT \| COND_CMP_FALSE)
110	117
	118	+#define SEEN_FUNC 0x20000000 /* might call external helpers */
	119	+#define SEEN_STACK 0x40000000 /* uses BPF stack */
	120	+#define SEEN_TAILCALL 0x80000000 /* uses tail calls */
	121	+
	122	+#define SEEN_VREG_MASK 0x1ff80000 /* Volatile registers r3-r12 */
	123	+#define SEEN_NVREG_MASK 0x0003ffff /* Non volatile registers r14-r31 */
	124	+
	125	+#ifdef CONFIG_PPC64
	126	+extern const int b2p[MAX_BPF_JIT_REG + 2];
	127	+#else
	128	+extern const int b2p[MAX_BPF_JIT_REG + 1];
	129	+#endif
	130	+
	131	+struct codegen_context {
	132	+ /*
	133	+ * This is used to track register usage as well
	134	+ * as calls to external helpers.
	135	+ * - register usage is tracked with corresponding
	136	+ * bits (r3-r31)
	137	+ * - rest of the bits can be used to track other
	138	+ * things -- for now, we use bits 0 to 2
	139	+ * encoded in SEEN_* macros above
	140	+ */
	141	+ unsigned int seen;
	142	+ unsigned int idx;
	143	+ unsigned int stack_size;
	144	+ int b2p[ARRAY_SIZE(b2p)];
	145	+};
	146	+
	147	+static inline void bpf_flush_icache(void start, void end)
	148	+{
	149	+ smp_wmb(); /* smp write barrier */
	150	+ flush_icache_range((unsigned long)start, (unsigned long)end);
	151	+}
	152	+
	153	+static inline bool bpf_is_seen_register(struct codegen_context *ctx, int i)
	154	+{
	155	+ return ctx->seen & (1 << (31 - i));
	156	+}
	157	+
	158	+static inline void bpf_set_seen_register(struct codegen_context *ctx, int i)
	159	+{
	160	+ ctx->seen \|= 1 << (31 - i);
	161	+}
	162	+
	163	+static inline void bpf_clear_seen_register(struct codegen_context *ctx, int i)
	164	+{
	165	+ ctx->seen &= ~(1 << (31 - i));
	166	+}
	167	+
	168	+void bpf_jit_emit_func_call_rel(u32 image, struct codegen_context ctx, u64 func);
	169	+int bpf_jit_build_body(struct bpf_prog fp, u32 image, struct codegen_context *ctx,
	170	+ u32 *addrs, bool extra_pass);
	171	+void bpf_jit_build_prologue(u32 image, struct codegen_context ctx);
	172	+void bpf_jit_build_epilogue(u32 image, struct codegen_context ctx);
	173	+void bpf_jit_realloc_regs(struct codegen_context *ctx);
	174	+
111	175	#endif
112	176
113	177	#endif

arch/powerpc/net/bpf_jit32.h+0−139

@@ -1,139 +0,0 @@
1		−/* SPDX-License-Identifier: GPL-2.0-only */
2		−/*
3		− * bpf_jit32.h: BPF JIT compiler for PPC
4		− *
5		− * Copyright 2011 Matt Evans <matt@ozlabs.org>, IBM Corporation
6		− *
7		− * Split from bpf_jit.h
8		− */
9		−#ifndef _BPF_JIT32_H
10		−#define _BPF_JIT32_H
11		−
12		−#include <asm/asm-compat.h>
13		−#include "bpf_jit.h"
14		−
15		−#ifdef CONFIG_PPC64
16		−#define BPF_PPC_STACK_R3_OFF 48
17		−#define BPF_PPC_STACK_LOCALS 32
18		−#define BPF_PPC_STACK_BASIC (48+64)
19		−#define BPF_PPC_STACK_SAVE (18*8)
20		−#define BPF_PPC_STACKFRAME (BPF_PPC_STACK_BASIC+BPF_PPC_STACK_LOCALS+ \
21		− BPF_PPC_STACK_SAVE)
22		−#define BPF_PPC_SLOWPATH_FRAME (48+64)
23		−#else
24		−#define BPF_PPC_STACK_R3_OFF 24
25		−#define BPF_PPC_STACK_LOCALS 16
26		−#define BPF_PPC_STACK_BASIC (24+32)
27		−#define BPF_PPC_STACK_SAVE (18*4)
28		−#define BPF_PPC_STACKFRAME (BPF_PPC_STACK_BASIC+BPF_PPC_STACK_LOCALS+ \
29		− BPF_PPC_STACK_SAVE)
30		−#define BPF_PPC_SLOWPATH_FRAME (24+32)
31		−#endif
32		−
33		−#define REG_SZ (BITS_PER_LONG/8)
34		−
35		−/*
36		− * Generated code register usage:
37		− *
38		− * As normal PPC C ABI (e.g. r1=sp, r2=TOC), with:
39		− *
40		− * skb r3 (Entry parameter)
41		− * A register r4
42		− * X register r5
43		− * addr param r6
44		− * r7-r10 scratch
45		− * skb->data r14
46		− * skb headlen r15 (skb->len - skb->data_len)
47		− * m[0] r16
48		− * m[...] ...
49		− * m[15] r31
50		− */
51		−#define r_skb 3
52		−#define r_ret 3
53		−#define r_A 4
54		−#define r_X 5
55		−#define r_addr 6
56		−#define r_scratch1 7
57		−#define r_scratch2 8
58		−#define r_D 14
59		−#define r_HL 15
60		−#define r_M 16
61		−
62		−#ifndef __ASSEMBLY__
63		−
64		−/*
65		− * Assembly helpers from arch/powerpc/net/bpf_jit.S:
66		− */
67		−#define DECLARE_LOAD_FUNC(func) \
68		− extern u8 func[], func##_negative_offset[], func##_positive_offset[]
69		−
70		−DECLARE_LOAD_FUNC(sk_load_word);
71		−DECLARE_LOAD_FUNC(sk_load_half);
72		−DECLARE_LOAD_FUNC(sk_load_byte);
73		−DECLARE_LOAD_FUNC(sk_load_byte_msh);
74		−
75		−#define PPC_LBZ_OFFS(r, base, i) do { if ((i) < 32768) EMIT(PPC_RAW_LBZ(r, base, i)); \
76		− else { EMIT(PPC_RAW_ADDIS(r, base, IMM_HA(i))); \
77		− EMIT(PPC_RAW_LBZ(r, r, IMM_L(i))); } } while(0)
78		−
79		−#define PPC_LD_OFFS(r, base, i) do { if ((i) < 32768) EMIT(PPC_RAW_LD(r, base, i)); \
80		− else { EMIT(PPC_RAW_ADDIS(r, base, IMM_HA(i))); \
81		− EMIT(PPC_RAW_LD(r, r, IMM_L(i))); } } while(0)
82		−
83		−#define PPC_LWZ_OFFS(r, base, i) do { if ((i) < 32768) EMIT(PPC_RAW_LWZ(r, base, i)); \
84		− else { EMIT(PPC_RAW_ADDIS(r, base, IMM_HA(i))); \
85		− EMIT(PPC_RAW_LWZ(r, r, IMM_L(i))); } } while(0)
86		−
87		−#define PPC_LHZ_OFFS(r, base, i) do { if ((i) < 32768) EMIT(PPC_RAW_LHZ(r, base, i)); \
88		− else { EMIT(PPC_RAW_ADDIS(r, base, IMM_HA(i))); \
89		− EMIT(PPC_RAW_LHZ(r, r, IMM_L(i))); } } while(0)
90		−
91		−#ifdef CONFIG_PPC64
92		−#define PPC_LL_OFFS(r, base, i) do { PPC_LD_OFFS(r, base, i); } while(0)
93		−#else
94		−#define PPC_LL_OFFS(r, base, i) do { PPC_LWZ_OFFS(r, base, i); } while(0)
95		−#endif
96		−
97		−#ifdef CONFIG_SMP
98		−#ifdef CONFIG_PPC64
99		−#define PPC_BPF_LOAD_CPU(r) \
100		− do { BUILD_BUG_ON(sizeof_field(struct paca_struct, paca_index) != 2); \
101		− PPC_LHZ_OFFS(r, 13, offsetof(struct paca_struct, paca_index)); \
102		− } while (0)
103		−#else
104		−#define PPC_BPF_LOAD_CPU(r) \
105		− do { BUILD_BUG_ON(sizeof_field(struct task_struct, cpu) != 4); \
106		− PPC_LHZ_OFFS(r, 2, offsetof(struct task_struct, cpu)); \
107		− } while(0)
108		−#endif
109		−#else
110		−#define PPC_BPF_LOAD_CPU(r) do { EMIT(PPC_RAW_LI(r, 0)); } while(0)
111		−#endif
112		−
113		−#define PPC_LHBRX_OFFS(r, base, i) \
114		− do { PPC_LI32(r, i); EMIT(PPC_RAW_LHBRX(r, r, base)); } while(0)
115		−#ifdef __LITTLE_ENDIAN__
116		−#define PPC_NTOHS_OFFS(r, base, i) PPC_LHBRX_OFFS(r, base, i)
117		−#else
118		−#define PPC_NTOHS_OFFS(r, base, i) PPC_LHZ_OFFS(r, base, i)
119		−#endif
120		−
121		−#define PPC_BPF_LL(r, base, i) do { EMIT(PPC_RAW_LWZ(r, base, i)); } while(0)
122		−#define PPC_BPF_STL(r, base, i) do { EMIT(PPC_RAW_STW(r, base, i)); } while(0)
123		−#define PPC_BPF_STLU(r, base, i) do { EMIT(PPC_RAW_STWU(r, base, i)); } while(0)
124		−
125		−#define SEEN_DATAREF 0x10000 /* might call external helpers */
126		−#define SEEN_XREG 0x20000 /* X reg is used */
127		−#define SEEN_MEM 0x40000 /* SEEN_MEM+(1<<n) = use mem[n] for temporary
128		− * storage */
129		−#define SEEN_MEM_MSK 0x0ffff
130		−
131		−struct codegen_context {
132		− unsigned int seen;
133		− unsigned int idx;
134		− int pc_ret0; /* bpf index of first RET #0 instruction (if any) */
135		−};
136		−
137		−#endif
138		−
139		−#endif

arch/powerpc/net/bpf_jit64.h+1−20

@@ -39,7 +39,7 @@ · 38 unmodified lines
39	39	#define TMP_REG_2 (MAX_BPF_JIT_REG + 1)
40	40
41	41	/* BPF to ppc register mappings */
42		−static const int b2p[] = {
	42	+const int b2p[MAX_BPF_JIT_REG + 2] = {
43	43	/* function return value */
44	44	[BPF_REG_0] = 8,
45	45	/* function arguments */
@@ -86,25 +86,6 @@ static const int b2p[] = { · 40 unmodified lines
86	86	} while(0)
87	87	#define PPC_BPF_STLU(r, base, i) do { EMIT(PPC_RAW_STDU(r, base, i)); } while(0)
88	88
89		−#define SEEN_FUNC 0x1000 /* might call external helpers */
90		−#define SEEN_STACK 0x2000 /* uses BPF stack */
91		−#define SEEN_TAILCALL 0x4000 /* uses tail calls */
92		−
93		−struct codegen_context {
94		− /*
95		− * This is used to track register usage as well
96		− * as calls to external helpers.
97		− * - register usage is tracked with corresponding
98		− * bits (r3-r10 and r27-r31)
99		− * - rest of the bits can be used to track other
100		− * things -- for now, we use bits 16 to 23
101		− * encoded in SEEN_* macros above
102		− */
103		− unsigned int seen;
104		− unsigned int idx;
105		− unsigned int stack_size;
106		−};
107		−
108	89	#endif /* !__ASSEMBLY__ */
109	90
110	91	#endif

arch/powerpc/net/bpf_jit_asm.S+0−226

@@ -1,226 +0,0 @@
1		−/* SPDX-License-Identifier: GPL-2.0-only */
2		−/* bpf_jit.S: Packet/header access helper functions
3		− * for PPC64 BPF compiler.
4		− *
5		− * Copyright 2011 Matt Evans <matt@ozlabs.org>, IBM Corporation
6		− */
7		−
8		−#include <asm/ppc_asm.h>
9		−#include <asm/asm-compat.h>
10		−#include "bpf_jit32.h"
11		−
12		−/*
13		− * All of these routines are called directly from generated code,
14		− * whose register usage is:
15		− *
16		− * r3 skb
17		− * r4,r5 A,X
18		− * r6 * address parameter to helper *
19		− * r7-r10 scratch
20		− * r14 skb->data
21		− * r15 skb headlen
22		− * r16-31 M[]
23		− */
24		−
25		−/*
26		− * To consider: These helpers are so small it could be better to just
27		− * generate them inline. Inline code can do the simple headlen check
28		− * then branch directly to slow_path_XXX if required. (In fact, could
29		− * load a spare GPR with the address of slow_path_generic and pass size
30		− * as an argument, making the call site a mtlr, li and bllr.)
31		− */
32		− .globl sk_load_word
33		−sk_load_word:
34		− PPC_LCMPI r_addr, 0
35		− blt bpf_slow_path_word_neg
36		− .globl sk_load_word_positive_offset
37		−sk_load_word_positive_offset:
38		− /* Are we accessing past headlen? */
39		− subi r_scratch1, r_HL, 4
40		− PPC_LCMP r_scratch1, r_addr
41		− blt bpf_slow_path_word
42		− /* Nope, just hitting the header. cr0 here is eq or gt! */
43		−#ifdef __LITTLE_ENDIAN__
44		− lwbrx r_A, r_D, r_addr
45		−#else
46		− lwzx r_A, r_D, r_addr
47		−#endif
48		− blr /* Return success, cr0 != LT */
49		−
50		− .globl sk_load_half
51		−sk_load_half:
52		− PPC_LCMPI r_addr, 0
53		− blt bpf_slow_path_half_neg
54		− .globl sk_load_half_positive_offset
55		−sk_load_half_positive_offset:
56		− subi r_scratch1, r_HL, 2
57		− PPC_LCMP r_scratch1, r_addr
58		− blt bpf_slow_path_half
59		−#ifdef __LITTLE_ENDIAN__
60		− lhbrx r_A, r_D, r_addr
61		−#else
62		− lhzx r_A, r_D, r_addr
63		−#endif
64		− blr
65		−
66		− .globl sk_load_byte
67		−sk_load_byte:
68		− PPC_LCMPI r_addr, 0
69		− blt bpf_slow_path_byte_neg
70		− .globl sk_load_byte_positive_offset
71		−sk_load_byte_positive_offset:
72		− PPC_LCMP r_HL, r_addr
73		− ble bpf_slow_path_byte
74		− lbzx r_A, r_D, r_addr
75		− blr
76		−
77		−/*
78		− * BPF_LDX \| BPF_B \| BPF_MSH: ldxb 4*([offset]&0xf)
79		− * r_addr is the offset value
80		− */
81		− .globl sk_load_byte_msh
82		−sk_load_byte_msh:
83		− PPC_LCMPI r_addr, 0
84		− blt bpf_slow_path_byte_msh_neg
85		− .globl sk_load_byte_msh_positive_offset
86		−sk_load_byte_msh_positive_offset:
87		− PPC_LCMP r_HL, r_addr
88		− ble bpf_slow_path_byte_msh
89		− lbzx r_X, r_D, r_addr
90		− rlwinm r_X, r_X, 2, 32-4-2, 31-2
91		− blr
92		−
93		−/* Call out to skb_copy_bits:
94		− * We'll need to back up our volatile regs first; we have
95		− * local variable space at r1+(BPF_PPC_STACK_BASIC).
96		− * Allocate a new stack frame here to remain ABI-compliant in
97		− * stashing LR.
98		− */
99		−#define bpf_slow_path_common(SIZE) \
100		− mflr r0; \
101		− PPC_STL r0, PPC_LR_STKOFF(r1); \
102		− /* R3 goes in parameter space of caller's frame */ \
103		− PPC_STL r_skb, (BPF_PPC_STACKFRAME+BPF_PPC_STACK_R3_OFF)(r1); \
104		− PPC_STL r_A, (BPF_PPC_STACK_BASIC+(0*REG_SZ))(r1); \
105		− PPC_STL r_X, (BPF_PPC_STACK_BASIC+(1*REG_SZ))(r1); \
106		− addi r5, r1, BPF_PPC_STACK_BASIC+(2*REG_SZ); \
107		− PPC_STLU r1, -BPF_PPC_SLOWPATH_FRAME(r1); \
108		− /* R3 = r_skb, as passed */ \
109		− mr r4, r_addr; \
110		− li r6, SIZE; \
111		− bl skb_copy_bits; \
112		− nop; \
113		− /* R3 = 0 on success */ \
114		− addi r1, r1, BPF_PPC_SLOWPATH_FRAME; \
115		− PPC_LL r0, PPC_LR_STKOFF(r1); \
116		− PPC_LL r_A, (BPF_PPC_STACK_BASIC+(0*REG_SZ))(r1); \
117		− PPC_LL r_X, (BPF_PPC_STACK_BASIC+(1*REG_SZ))(r1); \
118		− mtlr r0; \
119		− PPC_LCMPI r3, 0; \
120		− blt bpf_error; /* cr0 = LT */ \
121		− PPC_LL r_skb, (BPF_PPC_STACKFRAME+BPF_PPC_STACK_R3_OFF)(r1); \
122		− /* Great success! */
123		−
124		−bpf_slow_path_word:
125		− bpf_slow_path_common(4)
126		− /* Data value is on stack, and cr0 != LT */
127		− lwz r_A, BPF_PPC_STACK_BASIC+(2*REG_SZ)(r1)
128		− blr
129		−
130		−bpf_slow_path_half:
131		− bpf_slow_path_common(2)
132		− lhz r_A, BPF_PPC_STACK_BASIC+(2*8)(r1)
133		− blr
134		−
135		−bpf_slow_path_byte:
136		− bpf_slow_path_common(1)
137		− lbz r_A, BPF_PPC_STACK_BASIC+(2*8)(r1)
138		− blr
139		−
140		−bpf_slow_path_byte_msh:
141		− bpf_slow_path_common(1)
142		− lbz r_X, BPF_PPC_STACK_BASIC+(2*8)(r1)
143		− rlwinm r_X, r_X, 2, 32-4-2, 31-2
144		− blr
145		−
146		−/* Call out to bpf_internal_load_pointer_neg_helper:
147		− * We'll need to back up our volatile regs first; we have
148		− * local variable space at r1+(BPF_PPC_STACK_BASIC).
149		− * Allocate a new stack frame here to remain ABI-compliant in
150		− * stashing LR.
151		− */
152		−#define sk_negative_common(SIZE) \
153		− mflr r0; \
154		− PPC_STL r0, PPC_LR_STKOFF(r1); \
155		− /* R3 goes in parameter space of caller's frame */ \
156		− PPC_STL r_skb, (BPF_PPC_STACKFRAME+BPF_PPC_STACK_R3_OFF)(r1); \
157		− PPC_STL r_A, (BPF_PPC_STACK_BASIC+(0*REG_SZ))(r1); \
158		− PPC_STL r_X, (BPF_PPC_STACK_BASIC+(1*REG_SZ))(r1); \
159		− PPC_STLU r1, -BPF_PPC_SLOWPATH_FRAME(r1); \
160		− /* R3 = r_skb, as passed */ \
161		− mr r4, r_addr; \
162		− li r5, SIZE; \
163		− bl bpf_internal_load_pointer_neg_helper; \
164		− nop; \
165		− /* R3 != 0 on success */ \
166		− addi r1, r1, BPF_PPC_SLOWPATH_FRAME; \
167		− PPC_LL r0, PPC_LR_STKOFF(r1); \
168		− PPC_LL r_A, (BPF_PPC_STACK_BASIC+(0*REG_SZ))(r1); \
169		− PPC_LL r_X, (BPF_PPC_STACK_BASIC+(1*REG_SZ))(r1); \
170		− mtlr r0; \
171		− PPC_LCMPLI r3, 0; \
172		− beq bpf_error_slow; /* cr0 = EQ */ \
173		− mr r_addr, r3; \
174		− PPC_LL r_skb, (BPF_PPC_STACKFRAME+BPF_PPC_STACK_R3_OFF)(r1); \
175		− /* Great success! */
176		−
177		−bpf_slow_path_word_neg:
178		− lis r_scratch1,-32 /* SKF_LL_OFF */
179		− PPC_LCMP r_addr, r_scratch1 /* addr < SKF_* */
180		− blt bpf_error /* cr0 = LT */
181		− .globl sk_load_word_negative_offset
182		−sk_load_word_negative_offset:
183		− sk_negative_common(4)
184		− lwz r_A, 0(r_addr)
185		− blr
186		−
187		−bpf_slow_path_half_neg:
188		− lis r_scratch1,-32 /* SKF_LL_OFF */
189		− PPC_LCMP r_addr, r_scratch1 /* addr < SKF_* */
190		− blt bpf_error /* cr0 = LT */
191		− .globl sk_load_half_negative_offset
192		−sk_load_half_negative_offset:
193		− sk_negative_common(2)
194		− lhz r_A, 0(r_addr)
195		− blr
196		−
197		−bpf_slow_path_byte_neg:
198		− lis r_scratch1,-32 /* SKF_LL_OFF */
199		− PPC_LCMP r_addr, r_scratch1 /* addr < SKF_* */
200		− blt bpf_error /* cr0 = LT */
201		− .globl sk_load_byte_negative_offset
202		−sk_load_byte_negative_offset:
203		− sk_negative_common(1)
204		− lbz r_A, 0(r_addr)
205		− blr
206		−
207		−bpf_slow_path_byte_msh_neg:
208		− lis r_scratch1,-32 /* SKF_LL_OFF */
209		− PPC_LCMP r_addr, r_scratch1 /* addr < SKF_* */
210		− blt bpf_error /* cr0 = LT */
211		− .globl sk_load_byte_msh_negative_offset
212		−sk_load_byte_msh_negative_offset:
213		− sk_negative_common(1)
214		− lbz r_X, 0(r_addr)
215		− rlwinm r_X, r_X, 2, 32-4-2, 31-2
216		− blr
217		−
218		−bpf_error_slow:
219		− /* fabricate a cr0 = lt */
220		− li r_scratch1, -1
221		− PPC_LCMPI r_scratch1, 0
222		−bpf_error:
223		− /* Entered with cr0 = lt */
224		− li r3, 0
225		− /* Generated code will 'blt epilogue', returning 0. */
226		− blr

arch/powerpc/net/bpf_jit_comp.c+185−597

@@ -1,10 +1,11 @@
1	1	// SPDX-License-Identifier: GPL-2.0-only
2		−/* bpf_jit_comp.c: BPF JIT compiler
	2	+/*
	3	+ * eBPF JIT compiler
3	4	*
4		− * Copyright 2011 Matt Evans <matt@ozlabs.org>, IBM Corporation
	5	+ * Copyright 2016 Naveen N. Rao <naveen.n.rao@linux.vnet.ibm.com>
	6	+ * IBM Corporation
5	7	*
6		− * Based on the x86 BPF compiler, by Eric Dumazet (eric.dumazet@gmail.com)
7		− * Ported to ppc32 by Denis Kirjanov <kda@linux-powerpc.org>
	8	+ * Based on the powerpc classic BPF JIT compiler by Matt Evans
8	9	*/
9	10	#include <linux/moduleloader.h>
10	11	#include <asm/cacheflush.h>
@@ -12,639 +13,204 @@ · 1 unmodified line
12	13	#include <linux/netdevice.h>
13	14	#include <linux/filter.h>
14	15	#include <linux/if_vlan.h>
	16	+#include <asm/kprobes.h>
	17	+#include <linux/bpf.h>
15	18
16		−#include "bpf_jit32.h"
	19	+#include "bpf_jit.h"
17	20
18		−static inline void bpf_flush_icache(void start, void end)
	21	+static void bpf_jit_fill_ill_insns(void *area, unsigned int size)
19	22	{
20		− smp_wmb();
21		− flush_icache_range((unsigned long)start, (unsigned long)end);
	23	+ memset32(area, BREAKPOINT_INSTRUCTION, size / 4);
22	24	}
23	25
24		−static void bpf_jit_build_prologue(struct bpf_prog fp, u32 image,
25		− struct codegen_context *ctx)
	26	+/* Fix the branch target addresses for subprog calls */
	27	+static int bpf_jit_fixup_subprog_calls(struct bpf_prog fp, u32 image,
	28	+ struct codegen_context ctx, u32 addrs)
26	29	{
27		− int i;
28		− const struct sock_filter *filter = fp->insns;
29		−
30		− if (ctx->seen & (SEEN_MEM \| SEEN_DATAREF)) {
31		− /* Make stackframe */
32		− if (ctx->seen & SEEN_DATAREF) {
33		− /* If we call any helpers (for loads), save LR */
34		− EMIT(PPC_INST_MFLR \| __PPC_RT(R0));
35		− PPC_BPF_STL(0, 1, PPC_LR_STKOFF);
36		−
37		− /* Back up non-volatile regs. */
38		− PPC_BPF_STL(r_D, 1, -(REG_SZ*(32-r_D)));
39		− PPC_BPF_STL(r_HL, 1, -(REG_SZ*(32-r_HL)));
40		− }
41		− if (ctx->seen & SEEN_MEM) {
42		− /*
43		− * Conditionally save regs r15-r31 as some will be used
44		− * for M[] data.
45		− */
46		− for (i = r_M; i < (r_M+16); i++) {
47		− if (ctx->seen & (1 << (i-r_M)))
48		− PPC_BPF_STL(i, 1, -(REG_SZ*(32-i)));
49		− }
50		− }
51		− PPC_BPF_STLU(1, 1, -BPF_PPC_STACKFRAME);
52		− }
53		−
54		− if (ctx->seen & SEEN_DATAREF) {
55		− /*
56		− * If this filter needs to access skb data,
57		− * prepare r_D and r_HL:
58		− * r_HL = skb->len - skb->data_len
59		− * r_D = skb->data
60		− */
61		− PPC_LWZ_OFFS(r_scratch1, r_skb, offsetof(struct sk_buff,
62		− data_len));
63		− PPC_LWZ_OFFS(r_HL, r_skb, offsetof(struct sk_buff, len));
64		− EMIT(PPC_RAW_SUB(r_HL, r_HL, r_scratch1));
65		− PPC_LL_OFFS(r_D, r_skb, offsetof(struct sk_buff, data));
66		− }
	30	+ const struct bpf_insn *insn = fp->insnsi;
	31	+ bool func_addr_fixed;
	32	+ u64 func_addr;
	33	+ u32 tmp_idx;
	34	+ int i, ret;
67	35
68		− if (ctx->seen & SEEN_XREG) {
	36	+ for (i = 0; i < fp->len; i++) {
69	37	/*
70		− * TODO: Could also detect whether first instr. sets X and
71		− * avoid this (as below, with A).
	38	+ * During the extra pass, only the branch target addresses for
	39	+ * the subprog calls need to be fixed. All other instructions
	40	+ * can left untouched.
	41	+ *
	42	+ * The JITed image length does not change because we already
	43	+ * ensure that the JITed instruction sequence for these calls
	44	+ * are of fixed length by padding them with NOPs.
72	45	*/
73		− EMIT(PPC_RAW_LI(r_X, 0));
74		− }
75		−
76		− /* make sure we dont leak kernel information to user */
77		− if (bpf_needs_clear_a(&filter[0]))
78		− EMIT(PPC_RAW_LI(r_A, 0));
79		−}
80		−
81		−static void bpf_jit_build_epilogue(u32 image, struct codegen_context ctx)
82		−{
83		− int i;
84		−
85		− if (ctx->seen & (SEEN_MEM \| SEEN_DATAREF)) {
86		− EMIT(PPC_RAW_ADDI(1, 1, BPF_PPC_STACKFRAME));
87		− if (ctx->seen & SEEN_DATAREF) {
88		− PPC_BPF_LL(0, 1, PPC_LR_STKOFF);
89		− EMIT(PPC_RAW_MTLR(0));
90		− PPC_BPF_LL(r_D, 1, -(REG_SZ*(32-r_D)));
91		− PPC_BPF_LL(r_HL, 1, -(REG_SZ*(32-r_HL)));
92		− }
93		− if (ctx->seen & SEEN_MEM) {
94		− /* Restore any saved non-vol registers */
95		− for (i = r_M; i < (r_M+16); i++) {
96		− if (ctx->seen & (1 << (i-r_M)))
97		− PPC_BPF_LL(i, 1, -(REG_SZ*(32-i)));
98		− }
99		− }
100		− }
101		− /* The RETs have left a return value in R3. */
102		−
103		− EMIT(PPC_RAW_BLR());
104		−}
105		−
106		−#define CHOOSE_LOAD_FUNC(K, func) \
107		− ((int)K < 0 ? ((int)K >= SKF_LL_OFF ? func##_negative_offset : func) : func##_positive_offset)
108		−
109		−/* Assemble the body code between the prologue & epilogue. */
110		−static int bpf_jit_build_body(struct bpf_prog fp, u32 image,
111		− struct codegen_context *ctx,
112		− unsigned int *addrs)
113		−{
114		− const struct sock_filter *filter = fp->insns;
115		− int flen = fp->len;
116		− u8 *func;
117		− unsigned int true_cond;
118		− int i;
119		−
120		− /* Start of epilogue code */
121		− unsigned int exit_addr = addrs[flen];
122		−
123		− for (i = 0; i < flen; i++) {
124		− unsigned int K = filter[i].k;
125		− u16 code = bpf_anc_helper(&filter[i]);
	46	+ if (insn[i].code == (BPF_JMP \| BPF_CALL) &&
	47	+ insn[i].src_reg == BPF_PSEUDO_CALL) {
	48	+ ret = bpf_jit_get_func_addr(fp, &insn[i], true,
	49	+ &func_addr,
	50	+ &func_addr_fixed);
	51	+ if (ret < 0)
	52	+ return ret;
126	53
127		− /*
128		− * addrs[] maps a BPF bytecode address into a real offset from
129		− * the start of the body code.
130		− */
131		− addrs[i] = ctx->idx * 4;
132		−
133		− switch (code) {
134		− /* ALU ops */
135		− case BPF_ALU \| BPF_ADD \| BPF_X: /* A += X; */
136		− ctx->seen \|= SEEN_XREG;
137		− EMIT(PPC_RAW_ADD(r_A, r_A, r_X));
138		− break;
139		− case BPF_ALU \| BPF_ADD \| BPF_K: /* A += K; */
140		− if (!K)
141		− break;
142		− EMIT(PPC_RAW_ADDI(r_A, r_A, IMM_L(K)));
143		− if (K >= 32768)
144		− EMIT(PPC_RAW_ADDIS(r_A, r_A, IMM_HA(K)));
145		− break;
146		− case BPF_ALU \| BPF_SUB \| BPF_X: /* A -= X; */
147		− ctx->seen \|= SEEN_XREG;
148		− EMIT(PPC_RAW_SUB(r_A, r_A, r_X));
149		− break;
150		− case BPF_ALU \| BPF_SUB \| BPF_K: /* A -= K */
151		− if (!K)
152		− break;
153		− EMIT(PPC_RAW_ADDI(r_A, r_A, IMM_L(-K)));
154		− if (K >= 32768)
155		− EMIT(PPC_RAW_ADDIS(r_A, r_A, IMM_HA(-K)));
156		− break;
157		− case BPF_ALU \| BPF_MUL \| BPF_X: /* A = X; /
158		− ctx->seen \|= SEEN_XREG;
159		− EMIT(PPC_RAW_MULW(r_A, r_A, r_X));
160		− break;
161		− case BPF_ALU \| BPF_MUL \| BPF_K: /* A = K /
162		− if (K < 32768)
163		− EMIT(PPC_RAW_MULI(r_A, r_A, K));
164		− else {
165		− PPC_LI32(r_scratch1, K);
166		− EMIT(PPC_RAW_MULW(r_A, r_A, r_scratch1));
167		− }
168		− break;
169		− case BPF_ALU \| BPF_MOD \| BPF_X: /* A %= X; */
170		− case BPF_ALU \| BPF_DIV \| BPF_X: /* A /= X; */
171		− ctx->seen \|= SEEN_XREG;
172		− EMIT(PPC_RAW_CMPWI(r_X, 0));
173		− if (ctx->pc_ret0 != -1) {
174		− PPC_BCC(COND_EQ, addrs[ctx->pc_ret0]);
175		− } else {
176		− PPC_BCC_SHORT(COND_NE, (ctx->idx*4)+12);
177		− EMIT(PPC_RAW_LI(r_ret, 0));
178		− PPC_JMP(exit_addr);
179		− }
180		− if (code == (BPF_ALU \| BPF_MOD \| BPF_X)) {
181		− EMIT(PPC_RAW_DIVWU(r_scratch1, r_A, r_X));
182		− EMIT(PPC_RAW_MULW(r_scratch1, r_X, r_scratch1));
183		− EMIT(PPC_RAW_SUB(r_A, r_A, r_scratch1));
184		− } else {
185		− EMIT(PPC_RAW_DIVWU(r_A, r_A, r_X));
186		− }
187		− break;
188		− case BPF_ALU \| BPF_MOD \| BPF_K: /* A %= K; */
189		− PPC_LI32(r_scratch2, K);
190		− EMIT(PPC_RAW_DIVWU(r_scratch1, r_A, r_scratch2));
191		− EMIT(PPC_RAW_MULW(r_scratch1, r_scratch2, r_scratch1));
192		− EMIT(PPC_RAW_SUB(r_A, r_A, r_scratch1));
193		− break;
194		− case BPF_ALU \| BPF_DIV \| BPF_K: /* A /= K */
195		− if (K == 1)
196		− break;
197		− PPC_LI32(r_scratch1, K);
198		− EMIT(PPC_RAW_DIVWU(r_A, r_A, r_scratch1));
199		− break;
200		− case BPF_ALU \| BPF_AND \| BPF_X:
201		− ctx->seen \|= SEEN_XREG;
202		− EMIT(PPC_RAW_AND(r_A, r_A, r_X));
203		− break;
204		− case BPF_ALU \| BPF_AND \| BPF_K:
205		− if (!IMM_H(K))
206		− EMIT(PPC_RAW_ANDI(r_A, r_A, K));
207		− else {
208		− PPC_LI32(r_scratch1, K);
209		− EMIT(PPC_RAW_AND(r_A, r_A, r_scratch1));
210		− }
211		− break;
212		− case BPF_ALU \| BPF_OR \| BPF_X:
213		− ctx->seen \|= SEEN_XREG;
214		− EMIT(PPC_RAW_OR(r_A, r_A, r_X));
215		− break;
216		− case BPF_ALU \| BPF_OR \| BPF_K:
217		− if (IMM_L(K))
218		− EMIT(PPC_RAW_ORI(r_A, r_A, IMM_L(K)));
219		− if (K >= 65536)
220		− EMIT(PPC_RAW_ORIS(r_A, r_A, IMM_H(K)));
221		− break;
222		− case BPF_ANC \| SKF_AD_ALU_XOR_X:
223		− case BPF_ALU \| BPF_XOR \| BPF_X: /* A ^= X */
224		− ctx->seen \|= SEEN_XREG;
225		− EMIT(PPC_RAW_XOR(r_A, r_A, r_X));
226		− break;
227		− case BPF_ALU \| BPF_XOR \| BPF_K: /* A ^= K */
228		− if (IMM_L(K))
229		− EMIT(PPC_RAW_XORI(r_A, r_A, IMM_L(K)));
230		− if (K >= 65536)
231		− EMIT(PPC_RAW_XORIS(r_A, r_A, IMM_H(K)));
232		− break;
233		− case BPF_ALU \| BPF_LSH \| BPF_X: /* A <<= X; */
234		− ctx->seen \|= SEEN_XREG;
235		− EMIT(PPC_RAW_SLW(r_A, r_A, r_X));
236		− break;
237		− case BPF_ALU \| BPF_LSH \| BPF_K:
238		− if (K == 0)
239		− break;
240		− else
241		− EMIT(PPC_RAW_SLWI(r_A, r_A, K));
242		− break;
243		− case BPF_ALU \| BPF_RSH \| BPF_X: /* A >>= X; */
244		− ctx->seen \|= SEEN_XREG;
245		− EMIT(PPC_RAW_SRW(r_A, r_A, r_X));
246		− break;
247		− case BPF_ALU \| BPF_RSH \| BPF_K: /* A >>= K; */
248		− if (K == 0)
249		− break;
250		− else
251		− EMIT(PPC_RAW_SRWI(r_A, r_A, K));
252		− break;
253		− case BPF_ALU \| BPF_NEG:
254		− EMIT(PPC_RAW_NEG(r_A, r_A));
255		− break;
256		− case BPF_RET \| BPF_K:
257		− PPC_LI32(r_ret, K);
258		− if (!K) {
259		− if (ctx->pc_ret0 == -1)
260		− ctx->pc_ret0 = i;
261		− }
262		− /*
263		− * If this isn't the very last instruction, branch to
264		− * the epilogue if we've stuff to clean up. Otherwise,
265		− * if there's nothing to tidy, just return. If we /are/
266		− * the last instruction, we're about to fall through to
267		− * the epilogue to return.
268		− */
269		− if (i != flen - 1) {
270		− /*
271		− * Note: 'seen' is properly valid only on pass
272		− * #2. Both parts of this conditional are the
273		− * same instruction size though, meaning the
274		− * first pass will still correctly determine the
275		− * code size/addresses.
276		− */
277		− if (ctx->seen)
278		− PPC_JMP(exit_addr);
279		− else
280		− EMIT(PPC_RAW_BLR());
281		− }
282		− break;
283		− case BPF_RET \| BPF_A:
284		− EMIT(PPC_RAW_MR(r_ret, r_A));
285		− if (i != flen - 1) {
286		− if (ctx->seen)
287		− PPC_JMP(exit_addr);
288		− else
289		− EMIT(PPC_RAW_BLR());
290		− }
291		− break;
292		− case BPF_MISC \| BPF_TAX: /* X = A */
293		− EMIT(PPC_RAW_MR(r_X, r_A));
294		− break;
295		− case BPF_MISC \| BPF_TXA: /* A = X */
296		− ctx->seen \|= SEEN_XREG;
297		− EMIT(PPC_RAW_MR(r_A, r_X));
298		− break;
299		−
300		− /* Constant loads/M[] access */
301		− case BPF_LD \| BPF_IMM: /* A = K */
302		− PPC_LI32(r_A, K);
303		− break;
304		− case BPF_LDX \| BPF_IMM: /* X = K */
305		− PPC_LI32(r_X, K);
306		− break;
307		− case BPF_LD \| BPF_MEM: /* A = mem[K] */
308		− EMIT(PPC_RAW_MR(r_A, r_M + (K & 0xf)));
309		− ctx->seen \|= SEEN_MEM \| (1<<(K & 0xf));
310		− break;
311		− case BPF_LDX \| BPF_MEM: /* X = mem[K] */
312		− EMIT(PPC_RAW_MR(r_X, r_M + (K & 0xf)));
313		− ctx->seen \|= SEEN_MEM \| (1<<(K & 0xf));
314		− break;
315		− case BPF_ST: /* mem[K] = A */
316		− EMIT(PPC_RAW_MR(r_M + (K & 0xf), r_A));
317		− ctx->seen \|= SEEN_MEM \| (1<<(K & 0xf));
318		− break;
319		− case BPF_STX: /* mem[K] = X */
320		− EMIT(PPC_RAW_MR(r_M + (K & 0xf), r_X));
321		− ctx->seen \|= SEEN_XREG \| SEEN_MEM \| (1<<(K & 0xf));
322		− break;
323		− case BPF_LD \| BPF_W \| BPF_LEN: /* A = skb->len; */
324		− BUILD_BUG_ON(sizeof_field(struct sk_buff, len) != 4);
325		− PPC_LWZ_OFFS(r_A, r_skb, offsetof(struct sk_buff, len));
326		− break;
327		− case BPF_LDX \| BPF_W \| BPF_ABS: /* A = ((u32 )(seccomp_data + K)); */
328		− PPC_LWZ_OFFS(r_A, r_skb, K);
329		− break;
330		− case BPF_LDX \| BPF_W \| BPF_LEN: /* X = skb->len; */
331		− PPC_LWZ_OFFS(r_X, r_skb, offsetof(struct sk_buff, len));
332		− break;
333		−
334		− /* Ancillary info loads */
335		− case BPF_ANC \| SKF_AD_PROTOCOL: /* A = ntohs(skb->protocol); */
336		− BUILD_BUG_ON(sizeof_field(struct sk_buff,
337		− protocol) != 2);
338		− PPC_NTOHS_OFFS(r_A, r_skb, offsetof(struct sk_buff,
339		− protocol));
340		− break;
341		− case BPF_ANC \| SKF_AD_IFINDEX:
342		− case BPF_ANC \| SKF_AD_HATYPE:
343		− BUILD_BUG_ON(sizeof_field(struct net_device,
344		− ifindex) != 4);
345		− BUILD_BUG_ON(sizeof_field(struct net_device,
346		− type) != 2);
347		− PPC_LL_OFFS(r_scratch1, r_skb, offsetof(struct sk_buff,
348		− dev));
349		− EMIT(PPC_RAW_CMPDI(r_scratch1, 0));
350		− if (ctx->pc_ret0 != -1) {
351		− PPC_BCC(COND_EQ, addrs[ctx->pc_ret0]);
352		− } else {
353		− /* Exit, returning 0; first pass hits here. */
354		− PPC_BCC_SHORT(COND_NE, ctx->idx * 4 + 12);
355		− EMIT(PPC_RAW_LI(r_ret, 0));
356		− PPC_JMP(exit_addr);
357		− }
358		− if (code == (BPF_ANC \| SKF_AD_IFINDEX)) {
359		− PPC_LWZ_OFFS(r_A, r_scratch1,
360		− offsetof(struct net_device, ifindex));
361		− } else {
362		− PPC_LHZ_OFFS(r_A, r_scratch1,
363		− offsetof(struct net_device, type));
364		− }
365		−
366		− break;
367		− case BPF_ANC \| SKF_AD_MARK:
368		− BUILD_BUG_ON(sizeof_field(struct sk_buff, mark) != 4);
369		− PPC_LWZ_OFFS(r_A, r_skb, offsetof(struct sk_buff,
370		− mark));
371		− break;
372		− case BPF_ANC \| SKF_AD_RXHASH:
373		− BUILD_BUG_ON(sizeof_field(struct sk_buff, hash) != 4);
374		− PPC_LWZ_OFFS(r_A, r_skb, offsetof(struct sk_buff,
375		− hash));
376		− break;
377		− case BPF_ANC \| SKF_AD_VLAN_TAG:
378		− BUILD_BUG_ON(sizeof_field(struct sk_buff, vlan_tci) != 2);
379		−
380		− PPC_LHZ_OFFS(r_A, r_skb, offsetof(struct sk_buff,
381		− vlan_tci));
382		− break;
383		− case BPF_ANC \| SKF_AD_VLAN_TAG_PRESENT:
384		− PPC_LBZ_OFFS(r_A, r_skb, PKT_VLAN_PRESENT_OFFSET());
385		− if (PKT_VLAN_PRESENT_BIT)
386		− EMIT(PPC_RAW_SRWI(r_A, r_A, PKT_VLAN_PRESENT_BIT));
387		− if (PKT_VLAN_PRESENT_BIT < 7)
388		− EMIT(PPC_RAW_ANDI(r_A, r_A, 1));
389		− break;
390		− case BPF_ANC \| SKF_AD_QUEUE:
391		− BUILD_BUG_ON(sizeof_field(struct sk_buff,
392		− queue_mapping) != 2);
393		− PPC_LHZ_OFFS(r_A, r_skb, offsetof(struct sk_buff,
394		− queue_mapping));
395		− break;
396		− case BPF_ANC \| SKF_AD_PKTTYPE:
397		− PPC_LBZ_OFFS(r_A, r_skb, PKT_TYPE_OFFSET());
398		− EMIT(PPC_RAW_ANDI(r_A, r_A, PKT_TYPE_MAX));
399		− EMIT(PPC_RAW_SRWI(r_A, r_A, 5));
400		− break;
401		− case BPF_ANC \| SKF_AD_CPU:
402		− PPC_BPF_LOAD_CPU(r_A);
403		− break;
404		− /* Absolute loads from packet header/data */
405		− case BPF_LD \| BPF_W \| BPF_ABS:
406		− func = CHOOSE_LOAD_FUNC(K, sk_load_word);
407		− goto common_load;
408		− case BPF_LD \| BPF_H \| BPF_ABS:
409		− func = CHOOSE_LOAD_FUNC(K, sk_load_half);
410		− goto common_load;
411		− case BPF_LD \| BPF_B \| BPF_ABS:
412		− func = CHOOSE_LOAD_FUNC(K, sk_load_byte);
413		− common_load:
414		− /* Load from [K]. */
415		− ctx->seen \|= SEEN_DATAREF;
416		− PPC_FUNC_ADDR(r_scratch1, func);
417		− EMIT(PPC_RAW_MTLR(r_scratch1));
418		− PPC_LI32(r_addr, K);
419		− EMIT(PPC_RAW_BLRL());
420	54	/*
421		− * Helper returns 'lt' condition on error, and an
422		− * appropriate return value in r3
	55	+ * Save ctx->idx as this would currently point to the
	56	+ * end of the JITed image and set it to the offset of
	57	+ * the instruction sequence corresponding to the
	58	+ * subprog call temporarily.
423	59	*/
424		− PPC_BCC(COND_LT, exit_addr);
425		− break;
426		−
427		− /* Indirect loads from packet header/data */
428		− case BPF_LD \| BPF_W \| BPF_IND:
429		− func = sk_load_word;
430		− goto common_load_ind;
431		− case BPF_LD \| BPF_H \| BPF_IND:
432		− func = sk_load_half;
433		− goto common_load_ind;
434		− case BPF_LD \| BPF_B \| BPF_IND:
435		− func = sk_load_byte;
436		− common_load_ind:
	60	+ tmp_idx = ctx->idx;
	61	+ ctx->idx = addrs[i] / 4;
	62	+ bpf_jit_emit_func_call_rel(image, ctx, func_addr);
	63	+
437	64	/*
438		− * Load from [X + K]. Negative offsets are tested for
439		− * in the helper functions.
440		− */
441		− ctx->seen \|= SEEN_DATAREF \| SEEN_XREG;
442		− PPC_FUNC_ADDR(r_scratch1, func);
443		− EMIT(PPC_RAW_MTLR(r_scratch1));
444		− EMIT(PPC_RAW_ADDI(r_addr, r_X, IMM_L(K)));
445		− if (K >= 32768)
446		− EMIT(PPC_RAW_ADDIS(r_addr, r_addr, IMM_HA(K)));
447		− EMIT(PPC_RAW_BLRL());
448		− /* If error, cr0.LT set */
449		− PPC_BCC(COND_LT, exit_addr);
450		− break;
451		−
452		− case BPF_LDX \| BPF_B \| BPF_MSH:
453		− func = CHOOSE_LOAD_FUNC(K, sk_load_byte_msh);
454		− goto common_load;
455		− break;
456		−
457		− /* Jump and branches */
458		− case BPF_JMP \| BPF_JA:
459		− if (K != 0)
460		− PPC_JMP(addrs[i + 1 + K]);
461		− break;
462		−
463		− case BPF_JMP \| BPF_JGT \| BPF_K:
464		− case BPF_JMP \| BPF_JGT \| BPF_X:
465		− true_cond = COND_GT;
466		− goto cond_branch;
467		− case BPF_JMP \| BPF_JGE \| BPF_K:
468		− case BPF_JMP \| BPF_JGE \| BPF_X:
469		− true_cond = COND_GE;
470		− goto cond_branch;
471		− case BPF_JMP \| BPF_JEQ \| BPF_K:
472		− case BPF_JMP \| BPF_JEQ \| BPF_X:
473		− true_cond = COND_EQ;
474		− goto cond_branch;
475		− case BPF_JMP \| BPF_JSET \| BPF_K:
476		− case BPF_JMP \| BPF_JSET \| BPF_X:
477		− true_cond = COND_NE;
478		− cond_branch:
479		− /* same targets, can avoid doing the test :) */
480		− if (filter[i].jt == filter[i].jf) {
481		− if (filter[i].jt > 0)
482		− PPC_JMP(addrs[i + 1 + filter[i].jt]);
483		− break;
484		− }
485		−
486		− switch (code) {
487		− case BPF_JMP \| BPF_JGT \| BPF_X:
488		− case BPF_JMP \| BPF_JGE \| BPF_X:
489		− case BPF_JMP \| BPF_JEQ \| BPF_X:
490		− ctx->seen \|= SEEN_XREG;
491		− EMIT(PPC_RAW_CMPLW(r_A, r_X));
492		− break;
493		− case BPF_JMP \| BPF_JSET \| BPF_X:
494		− ctx->seen \|= SEEN_XREG;
495		− EMIT(PPC_RAW_AND_DOT(r_scratch1, r_A, r_X));
496		− break;
497		− case BPF_JMP \| BPF_JEQ \| BPF_K:
498		− case BPF_JMP \| BPF_JGT \| BPF_K:
499		− case BPF_JMP \| BPF_JGE \| BPF_K:
500		− if (K < 32768)
501		− EMIT(PPC_RAW_CMPLWI(r_A, K));
502		− else {
503		− PPC_LI32(r_scratch1, K);
504		− EMIT(PPC_RAW_CMPLW(r_A, r_scratch1));
505		− }
506		− break;
507		− case BPF_JMP \| BPF_JSET \| BPF_K:
508		− if (K < 32768)
509		− /* PPC_ANDI is /only/ dot-form */
510		− EMIT(PPC_RAW_ANDI(r_scratch1, r_A, K));
511		− else {
512		− PPC_LI32(r_scratch1, K);
513		− EMIT(PPC_RAW_AND_DOT(r_scratch1, r_A,
514		− r_scratch1));
515		− }
516		− break;
517		− }
518		− /* Sometimes branches are constructed "backward", with
519		− * the false path being the branch and true path being
520		− * a fallthrough to the next instruction.
	65	+ * Restore ctx->idx here. This is safe as the length
	66	+ * of the JITed sequence remains unchanged.
521	67	*/
522		− if (filter[i].jt == 0)
523		− /* Swap the sense of the branch */
524		− PPC_BCC(true_cond ^ COND_CMP_TRUE,
525		− addrs[i + 1 + filter[i].jf]);
526		− else {
527		− PPC_BCC(true_cond, addrs[i + 1 + filter[i].jt]);
528		− if (filter[i].jf != 0)
529		− PPC_JMP(addrs[i + 1 + filter[i].jf]);
530		− }
531		− break;
532		− default:
533		− /* The filter contains something cruel & unusual.
534		− * We don't handle it, but also there shouldn't be
535		− * anything missing from our list.
536		− */
537		− if (printk_ratelimit())
538		− pr_err("BPF filter opcode %04x (@%d) unsupported\n",
539		− filter[i].code, i);
540		− return -ENOTSUPP;
	68	+ ctx->idx = tmp_idx;
541	69	}
542		−
543	70	}
544		− /* Set end-of-body-code address for exit. */
545		− addrs[i] = ctx->idx * 4;
546	71
547	72	return 0;
548	73	}
549	74
550		−void bpf_jit_compile(struct bpf_prog *fp)
	75	+struct powerpc64_jit_data {
	76	+ struct bpf_binary_header *header;
	77	+ u32 *addrs;
	78	+ u8 *image;
	79	+ u32 proglen;
	80	+ struct codegen_context ctx;
	81	+};
	82	+
	83	+bool bpf_jit_needs_zext(void)
	84	+{
	85	+ return true;
	86	+}
	87	+
	88	+struct bpf_prog bpf_int_jit_compile(struct bpf_prog fp)
551	89	{
552		− unsigned int proglen;
553		− unsigned int alloclen;
554		− u32 *image = NULL;
	90	+ u32 proglen;
	91	+ u32 alloclen;
	92	+ u8 *image = NULL;
555	93	u32 *code_base;
556		− unsigned int *addrs;
	94	+ u32 *addrs;
	95	+ struct powerpc64_jit_data *jit_data;
557	96	struct codegen_context cgctx;
558	97	int pass;
559		− int flen = fp->len;
	98	+ int flen;
	99	+ struct bpf_binary_header *bpf_hdr;
	100	+ struct bpf_prog *org_fp = fp;
	101	+ struct bpf_prog *tmp_fp;
	102	+ bool bpf_blinded = false;
	103	+ bool extra_pass = false;
	104	+
	105	+ if (!fp->jit_requested)
	106	+ return org_fp;
	107	+
	108	+ tmp_fp = bpf_jit_blind_constants(org_fp);
	109	+ if (IS_ERR(tmp_fp))
	110	+ return org_fp;
	111	+
	112	+ if (tmp_fp != org_fp) {
	113	+ bpf_blinded = true;
	114	+ fp = tmp_fp;
	115	+ }
	116	+
	117	+ jit_data = fp->aux->jit_data;
	118	+ if (!jit_data) {
	119	+ jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL);
	120	+ if (!jit_data) {
	121	+ fp = org_fp;
	122	+ goto out;
	123	+ }
	124	+ fp->aux->jit_data = jit_data;
	125	+ }
560	126
561		− if (!bpf_jit_enable)
562		− return;
	127	+ flen = fp->len;
	128	+ addrs = jit_data->addrs;
	129	+ if (addrs) {
	130	+ cgctx = jit_data->ctx;
	131	+ image = jit_data->image;
	132	+ bpf_hdr = jit_data->header;
	133	+ proglen = jit_data->proglen;
	134	+ alloclen = proglen + FUNCTION_DESCR_SIZE;
	135	+ extra_pass = true;
	136	+ goto skip_init_ctx;
	137	+ }
563	138
564	139	addrs = kcalloc(flen + 1, sizeof(*addrs), GFP_KERNEL);
565		− if (addrs == NULL)
566		− return;
	140	+ if (addrs == NULL) {
	141	+ fp = org_fp;
	142	+ goto out_addrs;
	143	+ }
567	144
568		− /*
569		− * There are multiple assembly passes as the generated code will change
570		− * size as it settles down, figuring out the max branch offsets/exit
571		− * paths required.
572		− *
573		− * The range of standard conditional branches is +/- 32Kbytes. Since
574		− * BPF_MAXINSNS = 4096, we can only jump from (worst case) start to
575		− * finish with 8 bytes/instruction. Not feasible, so long jumps are
576		− * used, distinct from short branches.
577		− *
578		− * Current:
579		− *
580		− * For now, both branch types assemble to 2 words (short branches padded
581		− * with a NOP); this is less efficient, but assembly will always complete
582		− * after exactly 3 passes:
583		− *
584		− * First pass: No code buffer; Program is "faux-generated" -- no code
585		− * emitted but maximum size of output determined (and addrs[] filled
586		− * in). Also, we note whether we use M[], whether we use skb data, etc.
587		− * All generation choices assumed to be 'worst-case', e.g. branches all
588		− * far (2 instructions), return path code reduction not available, etc.
589		− *
590		− * Second pass: Code buffer allocated with size determined previously.
591		− * Prologue generated to support features we have seen used. Exit paths
592		− * determined and addrs[] is filled in again, as code may be slightly
593		− * smaller as a result.
594		− *
595		− * Third pass: Code generated 'for real', and branch destinations
596		− * determined from now-accurate addrs[] map.
597		− *
598		− * Ideal:
599		− *
600		− * If we optimise this, near branches will be shorter. On the
601		− * first assembly pass, we should err on the side of caution and
602		− * generate the biggest code. On subsequent passes, branches will be
603		− * generated short or long and code size will reduce. With smaller
604		− * code, more branches may fall into the short category, and code will
605		− * reduce more.
606		− *
607		− * Finally, if we see one pass generate code the same size as the
608		− * previous pass we have converged and should now generate code for
609		− * real. Allocating at the end will also save the memory that would
610		− * otherwise be wasted by the (small) current code shrinkage.
611		− * Preferably, we should do a small number of passes (e.g. 5) and if we
612		− * haven't converged by then, get impatient and force code to generate
613		− * as-is, even if the odd branch would be left long. The chances of a
614		− * long jump are tiny with all but the most enormous of BPF filter
615		− * inputs, so we should usually converge on the third pass.
616		− */
	145	+ memset(&cgctx, 0, sizeof(struct codegen_context));
	146	+ memcpy(cgctx.b2p, b2p, sizeof(cgctx.b2p));
	147	+
	148	+ /* Make sure that the stack is quadword aligned. */
	149	+ cgctx.stack_size = round_up(fp->aux->stack_depth, 16);
617	150
618		− cgctx.idx = 0;
619		− cgctx.seen = 0;
620		− cgctx.pc_ret0 = -1;
621	151	/* Scouting faux-generate pass 0 */
622		− if (bpf_jit_build_body(fp, 0, &cgctx, addrs))
	152	+ if (bpf_jit_build_body(fp, 0, &cgctx, addrs, false)) {
623	153	/* We hit something illegal or unsupported. */
624		− goto out;
	154	+ fp = org_fp;
	155	+ goto out_addrs;
	156	+ }
	157	+
	158	+ /*
	159	+ * If we have seen a tail call, we need a second pass.
	160	+ * This is because bpf_jit_emit_common_epilogue() is called
	161	+ * from bpf_jit_emit_tail_call() with a not yet stable ctx->seen.
	162	+ */
	163	+ if (cgctx.seen & SEEN_TAILCALL) {
	164	+ cgctx.idx = 0;
	165	+ if (bpf_jit_build_body(fp, 0, &cgctx, addrs, false)) {
	166	+ fp = org_fp;
	167	+ goto out_addrs;
	168	+ }
	169	+ }
625	170
	171	+ bpf_jit_realloc_regs(&cgctx);
626	172	/*
627	173	* Pretend to build prologue, given the features we've seen. This will
628	174	* update ctgtx.idx as it pretends to output instructions, then we can
629	175	* calculate total size from idx.
630	176	*/
631		− bpf_jit_build_prologue(fp, 0, &cgctx);
	177	+ bpf_jit_build_prologue(0, &cgctx);
632	178	bpf_jit_build_epilogue(0, &cgctx);
633	179
634	180	proglen = cgctx.idx * 4;
635	181	alloclen = proglen + FUNCTION_DESCR_SIZE;
636		− image = module_alloc(alloclen);
637		− if (!image)
638		− goto out;
639	182
640		− code_base = image + (FUNCTION_DESCR_SIZE/4);
	183	+ bpf_hdr = bpf_jit_binary_alloc(alloclen, &image, 4, bpf_jit_fill_ill_insns);
	184	+ if (!bpf_hdr) {
	185	+ fp = org_fp;
	186	+ goto out_addrs;
	187	+ }
	188	+
	189	+skip_init_ctx:
	190	+ code_base = (u32 *)(image + FUNCTION_DESCR_SIZE);
	191	+
	192	+ if (extra_pass) {
	193	+ /*
	194	+ * Do not touch the prologue and epilogue as they will remain
	195	+ * unchanged. Only fix the branch target address for subprog
	196	+ * calls in the body.
	197	+ *
	198	+ * This does not change the offsets and lengths of the subprog
	199	+ * call instruction sequences and hence, the size of the JITed
	200	+ * image as well.
	201	+ */
	202	+ bpf_jit_fixup_subprog_calls(fp, code_base, &cgctx, addrs);
	203	+
	204	+ /* There is no need to perform the usual passes. */
	205	+ goto skip_codegen_passes;
	206	+ }
641	207
642	208	/* Code generation passes 1-2 */
643	209	for (pass = 1; pass < 3; pass++) {
644	210	/* Now build the prologue, body code & epilogue for real. */
645	211	cgctx.idx = 0;
646		− bpf_jit_build_prologue(fp, code_base, &cgctx);
647		− bpf_jit_build_body(fp, code_base, &cgctx, addrs);
	212	+ bpf_jit_build_prologue(code_base, &cgctx);
	213	+ bpf_jit_build_body(fp, code_base, &cgctx, addrs, extra_pass);
648	214	bpf_jit_build_epilogue(code_base, &cgctx);
649	215
650	216	if (bpf_jit_enable > 1)
@@ -652,15 +218,15 @@ void bpf_jit_compile(struct bpf_prog *fp) · 1 unmodified line
652	218	proglen - (cgctx.idx * 4), cgctx.seen);
653	219	}
654	220
	221	+skip_codegen_passes:
655	222	if (bpf_jit_enable > 1)
656		− /* Note that we output the base address of the code_base
	223	+ /*
	224	+ * Note that we output the base address of the code_base
657	225	* rather than image, since opcodes are in code_base.
658	226	*/
659	227	bpf_jit_dump(flen, proglen, pass, code_base);
660	228
661		− bpf_flush_icache(code_base, code_base + (proglen/4));
662		−
663		−#ifdef CONFIG_PPC64
	229	+#ifdef PPC64_ELF_ABI_v1
664	230	/* Function descriptor nastiness: Address + TOC */
665	231	((u64 *)image)[0] = (u64)code_base;
666	232	((u64 *)image)[1] = local_paca->kernel_toc;
@@ -668,16 +234,38 @@ void bpf_jit_compile(struct bpf_prog *fp) · 1 unmodified line
668	234
669	235	fp->bpf_func = (void *)image;
670	236	fp->jited = 1;
	237	+ fp->jited_len = alloclen;
	238	+
	239	+ bpf_flush_icache(bpf_hdr, (u8 )bpf_hdr + (bpf_hdr->pages PAGE_SIZE));
	240	+ if (!fp->is_func \|\| extra_pass) {
	241	+ bpf_prog_fill_jited_linfo(fp, addrs);
	242	+out_addrs:
	243	+ kfree(addrs);
	244	+ kfree(jit_data);
	245	+ fp->aux->jit_data = NULL;
	246	+ } else {
	247	+ jit_data->addrs = addrs;
	248	+ jit_data->ctx = cgctx;
	249	+ jit_data->proglen = proglen;
	250	+ jit_data->image = image;
	251	+ jit_data->header = bpf_hdr;
	252	+ }
671	253
672	254	out:
673		− kfree(addrs);
674		− return;
	255	+ if (bpf_blinded)
	256	+ bpf_jit_prog_release_other(fp, fp == org_fp ? tmp_fp : org_fp);
	257	+
	258	+ return fp;
675	259	}
676	260
	261	+/* Overriding bpf_jit_free() as we don't set images read-only. */
677	262	void bpf_jit_free(struct bpf_prog *fp)
678	263	{
	264	+ unsigned long addr = (unsigned long)fp->bpf_func & PAGE_MASK;
	265	+ struct bpf_binary_header bpf_hdr = (void )addr;
	266	+
679	267	if (fp->jited)
680		− module_memfree(fp->bpf_func);
	268	+ bpf_jit_binary_free(bpf_hdr);
681	269
682	270	bpf_prog_unlock_free(fp);
683	271	}

arch/powerpc/net/bpf_jit_comp32.c+1100−0

@@ -0,0 +1,1100 @@
	1	+// SPDX-License-Identifier: GPL-2.0-only
	2	+/*
	3	+ * eBPF JIT compiler for PPC32
	4	+ *
	5	+ * Copyright 2020 Christophe Leroy <christophe.leroy@csgroup.eu>
	6	+ * CS GROUP France
	7	+ *
	8	+ * Based on PPC64 eBPF JIT compiler by Naveen N. Rao
	9	+ */
	10	+#include <linux/moduleloader.h>
	11	+#include <asm/cacheflush.h>
	12	+#include <asm/asm-compat.h>
	13	+#include <linux/netdevice.h>
	14	+#include <linux/filter.h>
	15	+#include <linux/if_vlan.h>
	16	+#include <asm/kprobes.h>
	17	+#include <linux/bpf.h>
	18	+
	19	+#include "bpf_jit.h"
	20	+
	21	+/*
	22	+ * Stack layout:
	23	+ *
	24	+ * [ prev sp ] <-------------
	25	+ * [ nv gpr save area ] 16 * 4 \|
	26	+ * fp (r31) --> [ ebpf stack space ] upto 512 \|
	27	+ * [ frame header ] 16 \|
	28	+ * sp (r1) ---> [ stack pointer ] --------------
	29	+ */
	30	+
	31	+/* for gpr non volatile registers r17 to r31 (14) + tail call */
	32	+#define BPF_PPC_STACK_SAVE (15 * 4 + 4)
	33	+/* stack frame, ensure this is quadword aligned */
	34	+#define BPF_PPC_STACKFRAME(ctx) (STACK_FRAME_MIN_SIZE + BPF_PPC_STACK_SAVE + (ctx)->stack_size)
	35	+
	36	+/* BPF register usage */
	37	+#define TMP_REG (MAX_BPF_JIT_REG + 0)
	38	+
	39	+/* BPF to ppc register mappings */
	40	+const int b2p[MAX_BPF_JIT_REG + 1] = {
	41	+ /* function return value */
	42	+ [BPF_REG_0] = 12,
	43	+ /* function arguments */
	44	+ [BPF_REG_1] = 4,
	45	+ [BPF_REG_2] = 6,
	46	+ [BPF_REG_3] = 8,
	47	+ [BPF_REG_4] = 10,
	48	+ [BPF_REG_5] = 22,
	49	+ /* non volatile registers */
	50	+ [BPF_REG_6] = 24,
	51	+ [BPF_REG_7] = 26,
	52	+ [BPF_REG_8] = 28,
	53	+ [BPF_REG_9] = 30,
	54	+ /* frame pointer aka BPF_REG_10 */
	55	+ [BPF_REG_FP] = 18,
	56	+ /* eBPF jit internal registers */
	57	+ [BPF_REG_AX] = 20,
	58	+ [TMP_REG] = 31, /* 32 bits */
	59	+};
	60	+
	61	+static int bpf_to_ppc(struct codegen_context *ctx, int reg)
	62	+{
	63	+ return ctx->b2p[reg];
	64	+}
	65	+
	66	+/* PPC NVR range -- update this if we ever use NVRs below r17 */
	67	+#define BPF_PPC_NVR_MIN 17
	68	+#define BPF_PPC_TC 16
	69	+
	70	+static int bpf_jit_stack_offsetof(struct codegen_context *ctx, int reg)
	71	+{
	72	+ if ((reg >= BPF_PPC_NVR_MIN && reg < 32) \|\| reg == BPF_PPC_TC)
	73	+ return BPF_PPC_STACKFRAME(ctx) - 4 * (32 - reg);
	74	+
	75	+ WARN(true, "BPF JIT is asking about unknown registers, will crash the stack");
	76	+ /* Use the hole we have left for alignment */
	77	+ return BPF_PPC_STACKFRAME(ctx) - 4;
	78	+}
	79	+
	80	+void bpf_jit_realloc_regs(struct codegen_context *ctx)
	81	+{
	82	+ if (ctx->seen & SEEN_FUNC)
	83	+ return;
	84	+
	85	+ while (ctx->seen & SEEN_NVREG_MASK &&
	86	+ (ctx->seen & SEEN_VREG_MASK) != SEEN_VREG_MASK) {
	87	+ int old = 32 - fls(ctx->seen & (SEEN_NVREG_MASK & 0xaaaaaaab));
	88	+ int new = 32 - fls(~ctx->seen & (SEEN_VREG_MASK & 0xaaaaaaaa));
	89	+ int i;
	90	+
	91	+ for (i = BPF_REG_0; i <= TMP_REG; i++) {
	92	+ if (ctx->b2p[i] != old)
	93	+ continue;
	94	+ ctx->b2p[i] = new;
	95	+ bpf_set_seen_register(ctx, new);
	96	+ bpf_clear_seen_register(ctx, old);
	97	+ if (i != TMP_REG) {
	98	+ bpf_set_seen_register(ctx, new - 1);
	99	+ bpf_clear_seen_register(ctx, old - 1);
	100	+ }
	101	+ break;
	102	+ }
	103	+ }
	104	+}
	105	+
	106	+void bpf_jit_build_prologue(u32 image, struct codegen_context ctx)
	107	+{
	108	+ int i;
	109	+
	110	+ /* First arg comes in as a 32 bits pointer. */
	111	+ EMIT(PPC_RAW_MR(bpf_to_ppc(ctx, BPF_REG_1), __REG_R3));
	112	+ EMIT(PPC_RAW_LI(bpf_to_ppc(ctx, BPF_REG_1) - 1, 0));
	113	+ EMIT(PPC_RAW_STWU(__REG_R1, __REG_R1, -BPF_PPC_STACKFRAME(ctx)));
	114	+
	115	+ /*
	116	+ * Initialize tail_call_cnt in stack frame if we do tail calls.
	117	+ * Otherwise, put in NOPs so that it can be skipped when we are
	118	+ * invoked through a tail call.
	119	+ */
	120	+ if (ctx->seen & SEEN_TAILCALL) {
	121	+ EMIT(PPC_RAW_STW(bpf_to_ppc(ctx, BPF_REG_1) - 1, __REG_R1, bpf_jit_stack_offsetof(ctx, BPF_PPC_TC)));
	122	+ } else {
	123	+ EMIT(PPC_RAW_NOP());
	124	+ }
	125	+
	126	+#define BPF_TAILCALL_PROLOGUE_SIZE 16
	127	+
	128	+ /*
	129	+ * We need a stack frame, but we don't necessarily need to
	130	+ * save/restore LR unless we call other functions
	131	+ */
	132	+ if (ctx->seen & SEEN_FUNC)
	133	+ EMIT(PPC_RAW_MFLR(__REG_R0));
	134	+
	135	+ /*
	136	+ * Back up non-volatile regs -- registers r18-r31
	137	+ */
	138	+ for (i = BPF_PPC_NVR_MIN; i <= 31; i++)
	139	+ if (bpf_is_seen_register(ctx, i))
	140	+ EMIT(PPC_RAW_STW(i, __REG_R1, bpf_jit_stack_offsetof(ctx, i)));
	141	+
	142	+ /* If needed retrieve arguments 9 and 10, ie 5th 64 bits arg.*/
	143	+ if (bpf_is_seen_register(ctx, bpf_to_ppc(ctx, BPF_REG_5))) {
	144	+ EMIT(PPC_RAW_LWZ(bpf_to_ppc(ctx, BPF_REG_5) - 1, __REG_R1, BPF_PPC_STACKFRAME(ctx)) + 8);
	145	+ EMIT(PPC_RAW_LWZ(bpf_to_ppc(ctx, BPF_REG_5), __REG_R1, BPF_PPC_STACKFRAME(ctx)) + 12);
	146	+ }
	147	+
	148	+ /* Setup frame pointer to point to the bpf stack area */
	149	+ if (bpf_is_seen_register(ctx, bpf_to_ppc(ctx, BPF_REG_FP))) {
	150	+ EMIT(PPC_RAW_LI(bpf_to_ppc(ctx, BPF_REG_FP) - 1, 0));
	151	+ EMIT(PPC_RAW_ADDI(bpf_to_ppc(ctx, BPF_REG_FP), __REG_R1,
	152	+ STACK_FRAME_MIN_SIZE + ctx->stack_size));
	153	+ }
	154	+
	155	+ if (ctx->seen & SEEN_FUNC)
	156	+ EMIT(PPC_RAW_STW(__REG_R0, __REG_R1, BPF_PPC_STACKFRAME(ctx) + PPC_LR_STKOFF));
	157	+}
	158	+
	159	+static void bpf_jit_emit_common_epilogue(u32 image, struct codegen_context ctx)
	160	+{
	161	+ int i;
	162	+
	163	+ /* Restore NVRs */
	164	+ for (i = BPF_PPC_NVR_MIN; i <= 31; i++)
	165	+ if (bpf_is_seen_register(ctx, i))
	166	+ EMIT(PPC_RAW_LWZ(i, __REG_R1, bpf_jit_stack_offsetof(ctx, i)));
	167	+}
	168	+
	169	+void bpf_jit_build_epilogue(u32 image, struct codegen_context ctx)
	170	+{
	171	+ EMIT(PPC_RAW_MR(__REG_R3, bpf_to_ppc(ctx, BPF_REG_0)));
	172	+
	173	+ bpf_jit_emit_common_epilogue(image, ctx);
	174	+
	175	+ /* Tear down our stack frame */
	176	+
	177	+ if (ctx->seen & SEEN_FUNC)
	178	+ EMIT(PPC_RAW_LWZ(__REG_R0, __REG_R1, BPF_PPC_STACKFRAME(ctx) + PPC_LR_STKOFF));
	179	+
	180	+ EMIT(PPC_RAW_ADDI(__REG_R1, __REG_R1, BPF_PPC_STACKFRAME(ctx)));
	181	+
	182	+ if (ctx->seen & SEEN_FUNC)
	183	+ EMIT(PPC_RAW_MTLR(__REG_R0));
	184	+
	185	+ EMIT(PPC_RAW_BLR());
	186	+}
	187	+
	188	+void bpf_jit_emit_func_call_rel(u32 image, struct codegen_context ctx, u64 func)
	189	+{
	190	+ s32 rel = (s32)func - (s32)(image + ctx->idx);
	191	+
	192	+ if (image && rel < 0x2000000 && rel >= -0x2000000) {
	193	+ PPC_BL_ABS(func);
	194	+ } else {
	195	+ /* Load function address into r0 */
	196	+ EMIT(PPC_RAW_LIS(__REG_R0, IMM_H(func)));
	197	+ EMIT(PPC_RAW_ORI(__REG_R0, __REG_R0, IMM_L(func)));
	198	+ EMIT(PPC_RAW_MTLR(__REG_R0));
	199	+ EMIT(PPC_RAW_BLRL());
	200	+ }
	201	+}
	202	+
	203	+static void bpf_jit_emit_tail_call(u32 image, struct codegen_context ctx, u32 out)
	204	+{
	205	+ /*
	206	+ * By now, the eBPF program has already setup parameters in r3-r6
	207	+ * r3-r4/BPF_REG_1 - pointer to ctx -- passed as is to the next bpf program
	208	+ * r5-r6/BPF_REG_2 - pointer to bpf_array
	209	+ * r7-r8/BPF_REG_3 - index in bpf_array
	210	+ */
	211	+ int b2p_bpf_array = bpf_to_ppc(ctx, BPF_REG_2);
	212	+ int b2p_index = bpf_to_ppc(ctx, BPF_REG_3);
	213	+
	214	+ /*
	215	+ * if (index >= array->map.max_entries)
	216	+ * goto out;
	217	+ */
	218	+ EMIT(PPC_RAW_LWZ(__REG_R0, b2p_bpf_array, offsetof(struct bpf_array, map.max_entries)));
	219	+ EMIT(PPC_RAW_CMPLW(b2p_index, __REG_R0));
	220	+ EMIT(PPC_RAW_LWZ(__REG_R0, __REG_R1, bpf_jit_stack_offsetof(ctx, BPF_PPC_TC)));
	221	+ PPC_BCC(COND_GE, out);
	222	+
	223	+ /*
	224	+ * if (tail_call_cnt > MAX_TAIL_CALL_CNT)
	225	+ * goto out;
	226	+ */
	227	+ EMIT(PPC_RAW_CMPLWI(__REG_R0, MAX_TAIL_CALL_CNT));
	228	+ /* tail_call_cnt++; */
	229	+ EMIT(PPC_RAW_ADDIC(__REG_R0, __REG_R0, 1));
	230	+ PPC_BCC(COND_GT, out);
	231	+
	232	+ /* prog = array->ptrs[index]; */
	233	+ EMIT(PPC_RAW_RLWINM(__REG_R3, b2p_index, 2, 0, 29));
	234	+ EMIT(PPC_RAW_ADD(__REG_R3, __REG_R3, b2p_bpf_array));
	235	+ EMIT(PPC_RAW_LWZ(__REG_R3, __REG_R3, offsetof(struct bpf_array, ptrs)));
	236	+ EMIT(PPC_RAW_STW(__REG_R0, __REG_R1, bpf_jit_stack_offsetof(ctx, BPF_PPC_TC)));
	237	+
	238	+ /*
	239	+ * if (prog == NULL)
	240	+ * goto out;
	241	+ */
	242	+ EMIT(PPC_RAW_CMPLWI(__REG_R3, 0));
	243	+ PPC_BCC(COND_EQ, out);
	244	+
	245	+ /* goto (prog->bpf_func + prologue_size); /
	246	+ EMIT(PPC_RAW_LWZ(__REG_R3, __REG_R3, offsetof(struct bpf_prog, bpf_func)));
	247	+
	248	+ if (ctx->seen & SEEN_FUNC)
	249	+ EMIT(PPC_RAW_LWZ(__REG_R0, __REG_R1, BPF_PPC_STACKFRAME(ctx) + PPC_LR_STKOFF));
	250	+
	251	+ EMIT(PPC_RAW_ADDIC(__REG_R3, __REG_R3, BPF_TAILCALL_PROLOGUE_SIZE));
	252	+
	253	+ if (ctx->seen & SEEN_FUNC)
	254	+ EMIT(PPC_RAW_MTLR(__REG_R0));
	255	+
	256	+ EMIT(PPC_RAW_MTCTR(__REG_R3));
	257	+
	258	+ EMIT(PPC_RAW_MR(__REG_R3, bpf_to_ppc(ctx, BPF_REG_1)));
	259	+
	260	+ /* tear restore NVRs, ... */
	261	+ bpf_jit_emit_common_epilogue(image, ctx);
	262	+
	263	+ EMIT(PPC_RAW_BCTR());
	264	+ /* out: */
	265	+}
	266	+
	267	+/* Assemble the body code between the prologue & epilogue */
	268	+int bpf_jit_build_body(struct bpf_prog fp, u32 image, struct codegen_context *ctx,
	269	+ u32 *addrs, bool extra_pass)
	270	+{
	271	+ const struct bpf_insn *insn = fp->insnsi;
	272	+ int flen = fp->len;
	273	+ int i, ret;
	274	+
	275	+ /* Start of epilogue code - will only be valid 2nd pass onwards */
	276	+ u32 exit_addr = addrs[flen];
	277	+
	278	+ for (i = 0; i < flen; i++) {
	279	+ u32 code = insn[i].code;
	280	+ u32 dst_reg = bpf_to_ppc(ctx, insn[i].dst_reg);
	281	+ u32 dst_reg_h = dst_reg - 1;
	282	+ u32 src_reg = bpf_to_ppc(ctx, insn[i].src_reg);
	283	+ u32 src_reg_h = src_reg - 1;
	284	+ u32 tmp_reg = bpf_to_ppc(ctx, TMP_REG);
	285	+ s16 off = insn[i].off;
	286	+ s32 imm = insn[i].imm;
	287	+ bool func_addr_fixed;
	288	+ u64 func_addr;
	289	+ u32 true_cond;
	290	+
	291	+ /*
	292	+ * addrs[] maps a BPF bytecode address into a real offset from
	293	+ * the start of the body code.
	294	+ */
	295	+ addrs[i] = ctx->idx * 4;
	296	+
	297	+ /*
	298	+ * As an optimization, we note down which registers
	299	+ * are used so that we can only save/restore those in our
	300	+ * prologue and epilogue. We do this here regardless of whether
	301	+ * the actual BPF instruction uses src/dst registers or not
	302	+ * (for instance, BPF_CALL does not use them). The expectation
	303	+ * is that those instructions will have src_reg/dst_reg set to
	304	+ * 0. Even otherwise, we just lose some prologue/epilogue
	305	+ * optimization but everything else should work without
	306	+ * any issues.
	307	+ */
	308	+ if (dst_reg >= 3 && dst_reg < 32) {
	309	+ bpf_set_seen_register(ctx, dst_reg);
	310	+ bpf_set_seen_register(ctx, dst_reg_h);
	311	+ }
	312	+
	313	+ if (src_reg >= 3 && src_reg < 32) {
	314	+ bpf_set_seen_register(ctx, src_reg);
	315	+ bpf_set_seen_register(ctx, src_reg_h);
	316	+ }
	317	+
	318	+ switch (code) {
	319	+ /*
	320	+ * Arithmetic operations: ADD/SUB/MUL/DIV/MOD/NEG
	321	+ */
	322	+ case BPF_ALU \| BPF_ADD \| BPF_X: /* (u32) dst += (u32) src */
	323	+ EMIT(PPC_RAW_ADD(dst_reg, dst_reg, src_reg));
	324	+ break;
	325	+ case BPF_ALU64 \| BPF_ADD \| BPF_X: /* dst += src */
	326	+ EMIT(PPC_RAW_ADDC(dst_reg, dst_reg, src_reg));
	327	+ EMIT(PPC_RAW_ADDE(dst_reg_h, dst_reg_h, src_reg_h));
	328	+ break;
	329	+ case BPF_ALU \| BPF_SUB \| BPF_X: /* (u32) dst -= (u32) src */
	330	+ EMIT(PPC_RAW_SUB(dst_reg, dst_reg, src_reg));
	331	+ break;
	332	+ case BPF_ALU64 \| BPF_SUB \| BPF_X: /* dst -= src */
	333	+ EMIT(PPC_RAW_SUBFC(dst_reg, src_reg, dst_reg));
	334	+ EMIT(PPC_RAW_SUBFE(dst_reg_h, src_reg_h, dst_reg_h));
	335	+ break;
	336	+ case BPF_ALU \| BPF_SUB \| BPF_K: /* (u32) dst -= (u32) imm */
	337	+ imm = -imm;
	338	+ fallthrough;
	339	+ case BPF_ALU \| BPF_ADD \| BPF_K: /* (u32) dst += (u32) imm */
	340	+ if (IMM_HA(imm) & 0xffff)
	341	+ EMIT(PPC_RAW_ADDIS(dst_reg, dst_reg, IMM_HA(imm)));
	342	+ if (IMM_L(imm))
	343	+ EMIT(PPC_RAW_ADDI(dst_reg, dst_reg, IMM_L(imm)));
	344	+ break;
	345	+ case BPF_ALU64 \| BPF_SUB \| BPF_K: /* dst -= imm */
	346	+ imm = -imm;
	347	+ fallthrough;
	348	+ case BPF_ALU64 \| BPF_ADD \| BPF_K: /* dst += imm */
	349	+ if (!imm)
	350	+ break;
	351	+
	352	+ if (imm >= -32768 && imm < 32768) {
	353	+ EMIT(PPC_RAW_ADDIC(dst_reg, dst_reg, imm));
	354	+ } else {
	355	+ PPC_LI32(__REG_R0, imm);
	356	+ EMIT(PPC_RAW_ADDC(dst_reg, dst_reg, __REG_R0));
	357	+ }
	358	+ if (imm >= 0)
	359	+ EMIT(PPC_RAW_ADDZE(dst_reg_h, dst_reg_h));
	360	+ else
	361	+ EMIT(PPC_RAW_ADDME(dst_reg_h, dst_reg_h));
	362	+ break;
	363	+ case BPF_ALU64 \| BPF_MUL \| BPF_X: /* dst = src /
	364	+ bpf_set_seen_register(ctx, tmp_reg);
	365	+ EMIT(PPC_RAW_MULW(__REG_R0, dst_reg, src_reg_h));
	366	+ EMIT(PPC_RAW_MULW(dst_reg_h, dst_reg_h, src_reg));
	367	+ EMIT(PPC_RAW_MULHWU(tmp_reg, dst_reg, src_reg));
	368	+ EMIT(PPC_RAW_MULW(dst_reg, dst_reg, src_reg));
	369	+ EMIT(PPC_RAW_ADD(dst_reg_h, dst_reg_h, __REG_R0));
	370	+ EMIT(PPC_RAW_ADD(dst_reg_h, dst_reg_h, tmp_reg));
	371	+ break;
	372	+ case BPF_ALU \| BPF_MUL \| BPF_X: /* (u32) dst = (u32) src /
	373	+ EMIT(PPC_RAW_MULW(dst_reg, dst_reg, src_reg));
	374	+ break;
	375	+ case BPF_ALU \| BPF_MUL \| BPF_K: /* (u32) dst = (u32) imm /
	376	+ if (imm >= -32768 && imm < 32768) {
	377	+ EMIT(PPC_RAW_MULI(dst_reg, dst_reg, imm));
	378	+ } else {
	379	+ PPC_LI32(__REG_R0, imm);
	380	+ EMIT(PPC_RAW_MULW(dst_reg, dst_reg, __REG_R0));
	381	+ }
	382	+ break;
	383	+ case BPF_ALU64 \| BPF_MUL \| BPF_K: /* dst = imm /
	384	+ if (!imm) {
	385	+ PPC_LI32(dst_reg, 0);
	386	+ PPC_LI32(dst_reg_h, 0);
	387	+ break;
	388	+ }
	389	+ if (imm == 1)
	390	+ break;
	391	+ if (imm == -1) {
	392	+ EMIT(PPC_RAW_SUBFIC(dst_reg, dst_reg, 0));
	393	+ EMIT(PPC_RAW_SUBFZE(dst_reg_h, dst_reg_h));
	394	+ break;
	395	+ }
	396	+ bpf_set_seen_register(ctx, tmp_reg);
	397	+ PPC_LI32(tmp_reg, imm);
	398	+ EMIT(PPC_RAW_MULW(dst_reg_h, dst_reg_h, tmp_reg));
	399	+ if (imm < 0)
	400	+ EMIT(PPC_RAW_SUB(dst_reg_h, dst_reg_h, dst_reg));
	401	+ EMIT(PPC_RAW_MULHWU(__REG_R0, dst_reg, tmp_reg));
	402	+ EMIT(PPC_RAW_MULW(dst_reg, dst_reg, tmp_reg));
	403	+ EMIT(PPC_RAW_ADD(dst_reg_h, dst_reg_h, __REG_R0));
	404	+ break;
	405	+ case BPF_ALU \| BPF_DIV \| BPF_X: /* (u32) dst /= (u32) src */
	406	+ EMIT(PPC_RAW_DIVWU(dst_reg, dst_reg, src_reg));
	407	+ break;
	408	+ case BPF_ALU \| BPF_MOD \| BPF_X: /* (u32) dst %= (u32) src */
	409	+ EMIT(PPC_RAW_DIVWU(__REG_R0, dst_reg, src_reg));
	410	+ EMIT(PPC_RAW_MULW(__REG_R0, src_reg, __REG_R0));
	411	+ EMIT(PPC_RAW_SUB(dst_reg, dst_reg, __REG_R0));
	412	+ break;
	413	+ case BPF_ALU64 \| BPF_DIV \| BPF_X: /* dst /= src */
	414	+ return -EOPNOTSUPP;
	415	+ case BPF_ALU64 \| BPF_MOD \| BPF_X: /* dst %= src */
	416	+ return -EOPNOTSUPP;
	417	+ case BPF_ALU \| BPF_DIV \| BPF_K: /* (u32) dst /= (u32) imm */
	418	+ if (!imm)
	419	+ return -EINVAL;
	420	+ if (imm == 1)
	421	+ break;
	422	+
	423	+ PPC_LI32(__REG_R0, imm);
	424	+ EMIT(PPC_RAW_DIVWU(dst_reg, dst_reg, __REG_R0));
	425	+ break;
	426	+ case BPF_ALU \| BPF_MOD \| BPF_K: /* (u32) dst %= (u32) imm */
	427	+ if (!imm)
	428	+ return -EINVAL;
	429	+
	430	+ if (!is_power_of_2((u32)imm)) {
	431	+ bpf_set_seen_register(ctx, tmp_reg);
	432	+ PPC_LI32(tmp_reg, imm);
	433	+ EMIT(PPC_RAW_DIVWU(__REG_R0, dst_reg, tmp_reg));
	434	+ EMIT(PPC_RAW_MULW(__REG_R0, tmp_reg, __REG_R0));
	435	+ EMIT(PPC_RAW_SUB(dst_reg, dst_reg, __REG_R0));
	436	+ break;
	437	+ }
	438	+ if (imm == 1)
	439	+ EMIT(PPC_RAW_LI(dst_reg, 0));
	440	+ else
	441	+ EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 0, 32 - ilog2((u32)imm), 31));
	442	+
	443	+ break;
	444	+ case BPF_ALU64 \| BPF_MOD \| BPF_K: /* dst %= imm */
	445	+ if (!imm)
	446	+ return -EINVAL;
	447	+ if (imm < 0)
	448	+ imm = -imm;
	449	+ if (!is_power_of_2(imm))
	450	+ return -EOPNOTSUPP;
	451	+ if (imm == 1)
	452	+ EMIT(PPC_RAW_LI(dst_reg, 0));
	453	+ else
	454	+ EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 0, 32 - ilog2(imm), 31));
	455	+ EMIT(PPC_RAW_LI(dst_reg_h, 0));
	456	+ break;
	457	+ case BPF_ALU64 \| BPF_DIV \| BPF_K: /* dst /= imm */
	458	+ if (!imm)
	459	+ return -EINVAL;
	460	+ if (!is_power_of_2(abs(imm)))
	461	+ return -EOPNOTSUPP;
	462	+
	463	+ if (imm < 0) {
	464	+ EMIT(PPC_RAW_SUBFIC(dst_reg, dst_reg, 0));
	465	+ EMIT(PPC_RAW_SUBFZE(dst_reg_h, dst_reg_h));
	466	+ imm = -imm;
	467	+ }
	468	+ if (imm == 1)
	469	+ break;
	470	+ imm = ilog2(imm);
	471	+ EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 32 - imm, imm, 31));
	472	+ EMIT(PPC_RAW_RLWIMI(dst_reg, dst_reg_h, 32 - imm, 0, imm - 1));
	473	+ EMIT(PPC_RAW_SRAWI(dst_reg_h, dst_reg_h, imm));
	474	+ break;
	475	+ case BPF_ALU \| BPF_NEG: /* (u32) dst = -dst */
	476	+ EMIT(PPC_RAW_NEG(dst_reg, dst_reg));
	477	+ break;
	478	+ case BPF_ALU64 \| BPF_NEG: /* dst = -dst */
	479	+ EMIT(PPC_RAW_SUBFIC(dst_reg, dst_reg, 0));
	480	+ EMIT(PPC_RAW_SUBFZE(dst_reg_h, dst_reg_h));
	481	+ break;
	482	+
	483	+ /*
	484	+ * Logical operations: AND/OR/XOR/[A]LSH/[A]RSH
	485	+ */
	486	+ case BPF_ALU64 \| BPF_AND \| BPF_X: /* dst = dst & src */
	487	+ EMIT(PPC_RAW_AND(dst_reg, dst_reg, src_reg));
	488	+ EMIT(PPC_RAW_AND(dst_reg_h, dst_reg_h, src_reg_h));
	489	+ break;
	490	+ case BPF_ALU \| BPF_AND \| BPF_X: /* (u32) dst = dst & src */
	491	+ EMIT(PPC_RAW_AND(dst_reg, dst_reg, src_reg));
	492	+ break;
	493	+ case BPF_ALU64 \| BPF_AND \| BPF_K: /* dst = dst & imm */
	494	+ if (imm >= 0)
	495	+ EMIT(PPC_RAW_LI(dst_reg_h, 0));
	496	+ fallthrough;
	497	+ case BPF_ALU \| BPF_AND \| BPF_K: /* (u32) dst = dst & imm */
	498	+ if (!IMM_H(imm)) {
	499	+ EMIT(PPC_RAW_ANDI(dst_reg, dst_reg, IMM_L(imm)));
	500	+ } else if (!IMM_L(imm)) {
	501	+ EMIT(PPC_RAW_ANDIS(dst_reg, dst_reg, IMM_H(imm)));
	502	+ } else if (imm == (((1 << fls(imm)) - 1) ^ ((1 << (ffs(i) - 1)) - 1))) {
	503	+ EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 0,
	504	+ 32 - fls(imm), 32 - ffs(imm)));
	505	+ } else {
	506	+ PPC_LI32(__REG_R0, imm);
	507	+ EMIT(PPC_RAW_AND(dst_reg, dst_reg, __REG_R0));
	508	+ }
	509	+ break;
	510	+ case BPF_ALU64 \| BPF_OR \| BPF_X: /* dst = dst \| src */
	511	+ EMIT(PPC_RAW_OR(dst_reg, dst_reg, src_reg));
	512	+ EMIT(PPC_RAW_OR(dst_reg_h, dst_reg_h, src_reg_h));
	513	+ break;
	514	+ case BPF_ALU \| BPF_OR \| BPF_X: /* dst = (u32) dst \| (u32) src */
	515	+ EMIT(PPC_RAW_OR(dst_reg, dst_reg, src_reg));
	516	+ break;
	517	+ case BPF_ALU64 \| BPF_OR \| BPF_K:/* dst = dst \| imm */
	518	+ /* Sign-extended */
	519	+ if (imm < 0)
	520	+ EMIT(PPC_RAW_LI(dst_reg_h, -1));
	521	+ fallthrough;
	522	+ case BPF_ALU \| BPF_OR \| BPF_K:/* dst = (u32) dst \| (u32) imm */
	523	+ if (IMM_L(imm))
	524	+ EMIT(PPC_RAW_ORI(dst_reg, dst_reg, IMM_L(imm)));
	525	+ if (IMM_H(imm))
	526	+ EMIT(PPC_RAW_ORIS(dst_reg, dst_reg, IMM_H(imm)));
	527	+ break;
	528	+ case BPF_ALU64 \| BPF_XOR \| BPF_X: /* dst ^= src */
	529	+ if (dst_reg == src_reg) {
	530	+ EMIT(PPC_RAW_LI(dst_reg, 0));
	531	+ EMIT(PPC_RAW_LI(dst_reg_h, 0));
	532	+ } else {
	533	+ EMIT(PPC_RAW_XOR(dst_reg, dst_reg, src_reg));
	534	+ EMIT(PPC_RAW_XOR(dst_reg_h, dst_reg_h, src_reg_h));
	535	+ }
	536	+ break;
	537	+ case BPF_ALU \| BPF_XOR \| BPF_X: /* (u32) dst ^= src */
	538	+ if (dst_reg == src_reg)
	539	+ EMIT(PPC_RAW_LI(dst_reg, 0));
	540	+ else
	541	+ EMIT(PPC_RAW_XOR(dst_reg, dst_reg, src_reg));
	542	+ break;
	543	+ case BPF_ALU64 \| BPF_XOR \| BPF_K: /* dst ^= imm */
	544	+ if (imm < 0)
	545	+ EMIT(PPC_RAW_NOR(dst_reg_h, dst_reg_h, dst_reg_h));
	546	+ fallthrough;
	547	+ case BPF_ALU \| BPF_XOR \| BPF_K: /* (u32) dst ^= (u32) imm */
	548	+ if (IMM_L(imm))
	549	+ EMIT(PPC_RAW_XORI(dst_reg, dst_reg, IMM_L(imm)));
	550	+ if (IMM_H(imm))
	551	+ EMIT(PPC_RAW_XORIS(dst_reg, dst_reg, IMM_H(imm)));
	552	+ break;
	553	+ case BPF_ALU \| BPF_LSH \| BPF_X: /* (u32) dst <<= (u32) src */
	554	+ EMIT(PPC_RAW_SLW(dst_reg, dst_reg, src_reg));
	555	+ break;
	556	+ case BPF_ALU64 \| BPF_LSH \| BPF_X: /* dst <<= src; */
	557	+ bpf_set_seen_register(ctx, tmp_reg);
	558	+ EMIT(PPC_RAW_SUBFIC(__REG_R0, src_reg, 32));
	559	+ EMIT(PPC_RAW_SLW(dst_reg_h, dst_reg_h, src_reg));
	560	+ EMIT(PPC_RAW_ADDI(tmp_reg, src_reg, 32));
	561	+ EMIT(PPC_RAW_SRW(__REG_R0, dst_reg, __REG_R0));
	562	+ EMIT(PPC_RAW_SLW(tmp_reg, dst_reg, tmp_reg));
	563	+ EMIT(PPC_RAW_OR(dst_reg_h, dst_reg_h, __REG_R0));
	564	+ EMIT(PPC_RAW_SLW(dst_reg, dst_reg, src_reg));
	565	+ EMIT(PPC_RAW_OR(dst_reg_h, dst_reg_h, tmp_reg));
	566	+ break;
	567	+ case BPF_ALU \| BPF_LSH \| BPF_K: /* (u32) dst <<= (u32) imm */
	568	+ if (!imm)
	569	+ break;
	570	+ EMIT(PPC_RAW_SLWI(dst_reg, dst_reg, imm));
	571	+ break;
	572	+ case BPF_ALU64 \| BPF_LSH \| BPF_K: /* dst <<= imm */
	573	+ if (imm < 0)
	574	+ return -EINVAL;
	575	+ if (!imm)
	576	+ break;
	577	+ if (imm < 32) {
	578	+ EMIT(PPC_RAW_RLWINM(dst_reg_h, dst_reg_h, imm, 0, 31 - imm));
	579	+ EMIT(PPC_RAW_RLWIMI(dst_reg_h, dst_reg, imm, 32 - imm, 31));
	580	+ EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, imm, 0, 31 - imm));
	581	+ break;
	582	+ }
	583	+ if (imm < 64)
	584	+ EMIT(PPC_RAW_RLWINM(dst_reg_h, dst_reg, imm, 0, 31 - imm));
	585	+ else
	586	+ EMIT(PPC_RAW_LI(dst_reg_h, 0));
	587	+ EMIT(PPC_RAW_LI(dst_reg, 0));
	588	+ break;
	589	+ case BPF_ALU \| BPF_RSH \| BPF_X: /* (u32) dst >>= (u32) src */
	590	+ EMIT(PPC_RAW_SRW(dst_reg, dst_reg, src_reg));
	591	+ break;
	592	+ case BPF_ALU64 \| BPF_RSH \| BPF_X: /* dst >>= src */
	593	+ bpf_set_seen_register(ctx, tmp_reg);
	594	+ EMIT(PPC_RAW_SUBFIC(__REG_R0, src_reg, 32));
	595	+ EMIT(PPC_RAW_SRW(dst_reg, dst_reg, src_reg));
	596	+ EMIT(PPC_RAW_ADDI(tmp_reg, src_reg, 32));
	597	+ EMIT(PPC_RAW_SLW(__REG_R0, dst_reg_h, __REG_R0));
	598	+ EMIT(PPC_RAW_SRW(tmp_reg, dst_reg_h, tmp_reg));
	599	+ EMIT(PPC_RAW_OR(dst_reg, dst_reg, __REG_R0));
	600	+ EMIT(PPC_RAW_SRW(dst_reg_h, dst_reg_h, src_reg));
	601	+ EMIT(PPC_RAW_OR(dst_reg, dst_reg, tmp_reg));
	602	+ break;
	603	+ case BPF_ALU \| BPF_RSH \| BPF_K: /* (u32) dst >>= (u32) imm */
	604	+ if (!imm)
	605	+ break;
	606	+ EMIT(PPC_RAW_SRWI(dst_reg, dst_reg, imm));
	607	+ break;
	608	+ case BPF_ALU64 \| BPF_RSH \| BPF_K: /* dst >>= imm */
	609	+ if (imm < 0)
	610	+ return -EINVAL;
	611	+ if (!imm)
	612	+ break;
	613	+ if (imm < 32) {
	614	+ EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 32 - imm, imm, 31));
	615	+ EMIT(PPC_RAW_RLWIMI(dst_reg, dst_reg_h, 32 - imm, 0, imm - 1));
	616	+ EMIT(PPC_RAW_RLWINM(dst_reg_h, dst_reg_h, 32 - imm, imm, 31));
	617	+ break;
	618	+ }
	619	+ if (imm < 64)
	620	+ EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg_h, 64 - imm, imm - 32, 31));
	621	+ else
	622	+ EMIT(PPC_RAW_LI(dst_reg, 0));
	623	+ EMIT(PPC_RAW_LI(dst_reg_h, 0));
	624	+ break;
	625	+ case BPF_ALU \| BPF_ARSH \| BPF_X: /* (s32) dst >>= src */
	626	+ EMIT(PPC_RAW_SRAW(dst_reg_h, dst_reg, src_reg));
	627	+ break;
	628	+ case BPF_ALU64 \| BPF_ARSH \| BPF_X: /* (s64) dst >>= src */
	629	+ bpf_set_seen_register(ctx, tmp_reg);
	630	+ EMIT(PPC_RAW_SUBFIC(__REG_R0, src_reg, 32));
	631	+ EMIT(PPC_RAW_SRW(dst_reg, dst_reg, src_reg));
	632	+ EMIT(PPC_RAW_SLW(__REG_R0, dst_reg_h, __REG_R0));
	633	+ EMIT(PPC_RAW_ADDI(tmp_reg, src_reg, 32));
	634	+ EMIT(PPC_RAW_OR(dst_reg, dst_reg, __REG_R0));
	635	+ EMIT(PPC_RAW_RLWINM(__REG_R0, tmp_reg, 0, 26, 26));
	636	+ EMIT(PPC_RAW_SRAW(tmp_reg, dst_reg_h, tmp_reg));
	637	+ EMIT(PPC_RAW_SRAW(dst_reg_h, dst_reg_h, src_reg));
	638	+ EMIT(PPC_RAW_SLW(tmp_reg, tmp_reg, __REG_R0));
	639	+ EMIT(PPC_RAW_OR(dst_reg, dst_reg, tmp_reg));
	640	+ break;
	641	+ case BPF_ALU \| BPF_ARSH \| BPF_K: /* (s32) dst >>= imm */
	642	+ if (!imm)
	643	+ break;
	644	+ EMIT(PPC_RAW_SRAWI(dst_reg, dst_reg, imm));
	645	+ break;
	646	+ case BPF_ALU64 \| BPF_ARSH \| BPF_K: /* (s64) dst >>= imm */
	647	+ if (imm < 0)
	648	+ return -EINVAL;
	649	+ if (!imm)
	650	+ break;
	651	+ if (imm < 32) {
	652	+ EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 32 - imm, imm, 31));
	653	+ EMIT(PPC_RAW_RLWIMI(dst_reg, dst_reg_h, 32 - imm, 0, imm - 1));
	654	+ EMIT(PPC_RAW_SRAWI(dst_reg_h, dst_reg_h, imm));
	655	+ break;
	656	+ }
	657	+ if (imm < 64)
	658	+ EMIT(PPC_RAW_SRAWI(dst_reg, dst_reg_h, imm - 32));
	659	+ else
	660	+ EMIT(PPC_RAW_SRAWI(dst_reg, dst_reg_h, 31));
	661	+ EMIT(PPC_RAW_SRAWI(dst_reg_h, dst_reg_h, 31));
	662	+ break;
	663	+
	664	+ /*
	665	+ * MOV
	666	+ */
	667	+ case BPF_ALU64 \| BPF_MOV \| BPF_X: /* dst = src */
	668	+ if (dst_reg == src_reg)
	669	+ break;
	670	+ EMIT(PPC_RAW_MR(dst_reg, src_reg));
	671	+ EMIT(PPC_RAW_MR(dst_reg_h, src_reg_h));
	672	+ break;
	673	+ case BPF_ALU \| BPF_MOV \| BPF_X: /* (u32) dst = src */
	674	+ /* special mov32 for zext */
	675	+ if (imm == 1)
	676	+ EMIT(PPC_RAW_LI(dst_reg_h, 0));
	677	+ else if (dst_reg != src_reg)
	678	+ EMIT(PPC_RAW_MR(dst_reg, src_reg));
	679	+ break;
	680	+ case BPF_ALU64 \| BPF_MOV \| BPF_K: /* dst = (s64) imm */
	681	+ PPC_LI32(dst_reg, imm);
	682	+ PPC_EX32(dst_reg_h, imm);
	683	+ break;
	684	+ case BPF_ALU \| BPF_MOV \| BPF_K: /* (u32) dst = imm */
	685	+ PPC_LI32(dst_reg, imm);
	686	+ break;
	687	+
	688	+ /*
	689	+ * BPF_FROM_BE/LE
	690	+ */
	691	+ case BPF_ALU \| BPF_END \| BPF_FROM_LE:
	692	+ switch (imm) {
	693	+ case 16:
	694	+ /* Copy 16 bits to upper part */
	695	+ EMIT(PPC_RAW_RLWIMI(dst_reg, dst_reg, 16, 0, 15));
	696	+ /* Rotate 8 bits right & mask */
	697	+ EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 24, 16, 31));
	698	+ break;
	699	+ case 32:
	700	+ /*
	701	+ * Rotate word left by 8 bits:
	702	+ * 2 bytes are already in their final position
	703	+ * -- byte 2 and 4 (of bytes 1, 2, 3 and 4)
	704	+ */
	705	+ EMIT(PPC_RAW_RLWINM(__REG_R0, dst_reg, 8, 0, 31));
	706	+ /* Rotate 24 bits and insert byte 1 */
	707	+ EMIT(PPC_RAW_RLWIMI(__REG_R0, dst_reg, 24, 0, 7));
	708	+ /* Rotate 24 bits and insert byte 3 */
	709	+ EMIT(PPC_RAW_RLWIMI(__REG_R0, dst_reg, 24, 16, 23));
	710	+ EMIT(PPC_RAW_MR(dst_reg, __REG_R0));
	711	+ break;
	712	+ case 64:
	713	+ bpf_set_seen_register(ctx, tmp_reg);
	714	+ EMIT(PPC_RAW_RLWINM(tmp_reg, dst_reg, 8, 0, 31));
	715	+ EMIT(PPC_RAW_RLWINM(__REG_R0, dst_reg_h, 8, 0, 31));
	716	+ /* Rotate 24 bits and insert byte 1 */
	717	+ EMIT(PPC_RAW_RLWIMI(tmp_reg, dst_reg, 24, 0, 7));
	718	+ EMIT(PPC_RAW_RLWIMI(__REG_R0, dst_reg_h, 24, 0, 7));
	719	+ /* Rotate 24 bits and insert byte 3 */
	720	+ EMIT(PPC_RAW_RLWIMI(tmp_reg, dst_reg, 24, 16, 23));
	721	+ EMIT(PPC_RAW_RLWIMI(__REG_R0, dst_reg_h, 24, 16, 23));
	722	+ EMIT(PPC_RAW_MR(dst_reg, __REG_R0));
	723	+ EMIT(PPC_RAW_MR(dst_reg_h, tmp_reg));
	724	+ break;
	725	+ }
	726	+ break;
	727	+ case BPF_ALU \| BPF_END \| BPF_FROM_BE:
	728	+ switch (imm) {
	729	+ case 16:
	730	+ /* zero-extend 16 bits into 32 bits */
	731	+ EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 0, 16, 31));
	732	+ break;
	733	+ case 32:
	734	+ case 64:
	735	+ /* nop */
	736	+ break;
	737	+ }
	738	+ break;
	739	+
	740	+ /*
	741	+ * BPF_ST(X)
	742	+ */
	743	+ case BPF_STX \| BPF_MEM \| BPF_B: /* (u8 )(dst + off) = src */
	744	+ EMIT(PPC_RAW_STB(src_reg, dst_reg, off));
	745	+ break;
	746	+ case BPF_ST \| BPF_MEM \| BPF_B: /* (u8 )(dst + off) = imm */
	747	+ PPC_LI32(__REG_R0, imm);
	748	+ EMIT(PPC_RAW_STB(__REG_R0, dst_reg, off));
	749	+ break;
	750	+ case BPF_STX \| BPF_MEM \| BPF_H: /* (u16 )(dst + off) = src /
	751	+ EMIT(PPC_RAW_STH(src_reg, dst_reg, off));
	752	+ break;
	753	+ case BPF_ST \| BPF_MEM \| BPF_H: /* (u16 )(dst + off) = imm /
	754	+ PPC_LI32(__REG_R0, imm);
	755	+ EMIT(PPC_RAW_STH(__REG_R0, dst_reg, off));
	756	+ break;
	757	+ case BPF_STX \| BPF_MEM \| BPF_W: /* (u32 )(dst + off) = src */
	758	+ EMIT(PPC_RAW_STW(src_reg, dst_reg, off));
	759	+ break;
	760	+ case BPF_ST \| BPF_MEM \| BPF_W: /* (u32 )(dst + off) = imm */
	761	+ PPC_LI32(__REG_R0, imm);
	762	+ EMIT(PPC_RAW_STW(__REG_R0, dst_reg, off));
	763	+ break;
	764	+ case BPF_STX \| BPF_MEM \| BPF_DW: /* (u64 )(dst + off) = src /
	765	+ EMIT(PPC_RAW_STW(src_reg_h, dst_reg, off));
	766	+ EMIT(PPC_RAW_STW(src_reg, dst_reg, off + 4));
	767	+ break;
	768	+ case BPF_ST \| BPF_MEM \| BPF_DW: /* (u64 )(dst + off) = imm */
	769	+ PPC_LI32(__REG_R0, imm);
	770	+ EMIT(PPC_RAW_STW(__REG_R0, dst_reg, off + 4));
	771	+ PPC_EX32(__REG_R0, imm);
	772	+ EMIT(PPC_RAW_STW(__REG_R0, dst_reg, off));
	773	+ break;
	774	+
	775	+ /*
	776	+ * BPF_STX XADD (atomic_add)
	777	+ */
	778	+ case BPF_STX \| BPF_XADD \| BPF_W: /* (u32 )(dst + off) += src */
	779	+ bpf_set_seen_register(ctx, tmp_reg);
	780	+ /* Get offset into TMP_REG */
	781	+ EMIT(PPC_RAW_LI(tmp_reg, off));
	782	+ /* load value from memory into r0 */
	783	+ EMIT(PPC_RAW_LWARX(__REG_R0, tmp_reg, dst_reg, 0));
	784	+ /* add value from src_reg into this */
	785	+ EMIT(PPC_RAW_ADD(__REG_R0, __REG_R0, src_reg));
	786	+ /* store result back */
	787	+ EMIT(PPC_RAW_STWCX(__REG_R0, tmp_reg, dst_reg));
	788	+ /* we're done if this succeeded */
	789	+ PPC_BCC_SHORT(COND_NE, (ctx->idx - 3) * 4);
	790	+ break;
	791	+
	792	+ case BPF_STX \| BPF_XADD \| BPF_DW: /* (u64 )(dst + off) += src */
	793	+ return -EOPNOTSUPP;
	794	+
	795	+ /*
	796	+ * BPF_LDX
	797	+ */
	798	+ case BPF_LDX \| BPF_MEM \| BPF_B: /* dst = (u8 )(ul) (src + off) */
	799	+ EMIT(PPC_RAW_LBZ(dst_reg, src_reg, off));
	800	+ if (!fp->aux->verifier_zext)
	801	+ EMIT(PPC_RAW_LI(dst_reg_h, 0));
	802	+ break;
	803	+ case BPF_LDX \| BPF_MEM \| BPF_H: /* dst = (u16 )(ul) (src + off) */
	804	+ EMIT(PPC_RAW_LHZ(dst_reg, src_reg, off));
	805	+ if (!fp->aux->verifier_zext)
	806	+ EMIT(PPC_RAW_LI(dst_reg_h, 0));
	807	+ break;
	808	+ case BPF_LDX \| BPF_MEM \| BPF_W: /* dst = (u32 )(ul) (src + off) */
	809	+ EMIT(PPC_RAW_LWZ(dst_reg, src_reg, off));
	810	+ if (!fp->aux->verifier_zext)
	811	+ EMIT(PPC_RAW_LI(dst_reg_h, 0));
	812	+ break;
	813	+ case BPF_LDX \| BPF_MEM \| BPF_DW: /* dst = (u64 )(ul) (src + off) */
	814	+ EMIT(PPC_RAW_LWZ(dst_reg_h, src_reg, off));
	815	+ EMIT(PPC_RAW_LWZ(dst_reg, src_reg, off + 4));
	816	+ break;
	817	+
	818	+ /*
	819	+ * Doubleword load
	820	+ * 16 byte instruction that uses two 'struct bpf_insn'
	821	+ */
	822	+ case BPF_LD \| BPF_IMM \| BPF_DW: /* dst = (u64) imm */
	823	+ PPC_LI32(dst_reg_h, (u32)insn[i + 1].imm);
	824	+ PPC_LI32(dst_reg, (u32)insn[i].imm);
	825	+ /* Adjust for two bpf instructions */
	826	+ addrs[++i] = ctx->idx * 4;
	827	+ break;
	828	+
	829	+ /*
	830	+ * Return/Exit
	831	+ */
	832	+ case BPF_JMP \| BPF_EXIT:
	833	+ /*
	834	+ * If this isn't the very last instruction, branch to
	835	+ * the epilogue. If we _are_ the last instruction,
	836	+ * we'll just fall through to the epilogue.
	837	+ */
	838	+ if (i != flen - 1)
	839	+ PPC_JMP(exit_addr);
	840	+ /* else fall through to the epilogue */
	841	+ break;
	842	+
	843	+ /*
	844	+ * Call kernel helper or bpf function
	845	+ */
	846	+ case BPF_JMP \| BPF_CALL:
	847	+ ctx->seen \|= SEEN_FUNC;
	848	+
	849	+ ret = bpf_jit_get_func_addr(fp, &insn[i], extra_pass,
	850	+ &func_addr, &func_addr_fixed);
	851	+ if (ret < 0)
	852	+ return ret;
	853	+
	854	+ if (bpf_is_seen_register(ctx, bpf_to_ppc(ctx, BPF_REG_5))) {
	855	+ EMIT(PPC_RAW_STW(bpf_to_ppc(ctx, BPF_REG_5) - 1, __REG_R1, 8));
	856	+ EMIT(PPC_RAW_STW(bpf_to_ppc(ctx, BPF_REG_5), __REG_R1, 12));
	857	+ }
	858	+
	859	+ bpf_jit_emit_func_call_rel(image, ctx, func_addr);
	860	+
	861	+ EMIT(PPC_RAW_MR(bpf_to_ppc(ctx, BPF_REG_0) - 1, __REG_R3));
	862	+ EMIT(PPC_RAW_MR(bpf_to_ppc(ctx, BPF_REG_0), __REG_R4));
	863	+ break;
	864	+
	865	+ /*
	866	+ * Jumps and branches
	867	+ */
	868	+ case BPF_JMP \| BPF_JA:
	869	+ PPC_JMP(addrs[i + 1 + off]);
	870	+ break;
	871	+
	872	+ case BPF_JMP \| BPF_JGT \| BPF_K:
	873	+ case BPF_JMP \| BPF_JGT \| BPF_X:
	874	+ case BPF_JMP \| BPF_JSGT \| BPF_K:
	875	+ case BPF_JMP \| BPF_JSGT \| BPF_X:
	876	+ case BPF_JMP32 \| BPF_JGT \| BPF_K:
	877	+ case BPF_JMP32 \| BPF_JGT \| BPF_X:
	878	+ case BPF_JMP32 \| BPF_JSGT \| BPF_K:
	879	+ case BPF_JMP32 \| BPF_JSGT \| BPF_X:
	880	+ true_cond = COND_GT;
	881	+ goto cond_branch;
	882	+ case BPF_JMP \| BPF_JLT \| BPF_K:
	883	+ case BPF_JMP \| BPF_JLT \| BPF_X:
	884	+ case BPF_JMP \| BPF_JSLT \| BPF_K:
	885	+ case BPF_JMP \| BPF_JSLT \| BPF_X:
	886	+ case BPF_JMP32 \| BPF_JLT \| BPF_K:
	887	+ case BPF_JMP32 \| BPF_JLT \| BPF_X:
	888	+ case BPF_JMP32 \| BPF_JSLT \| BPF_K:
	889	+ case BPF_JMP32 \| BPF_JSLT \| BPF_X:
	890	+ true_cond = COND_LT;
	891	+ goto cond_branch;
	892	+ case BPF_JMP \| BPF_JGE \| BPF_K:
	893	+ case BPF_JMP \| BPF_JGE \| BPF_X:
	894	+ case BPF_JMP \| BPF_JSGE \| BPF_K:
	895	+ case BPF_JMP \| BPF_JSGE \| BPF_X:
	896	+ case BPF_JMP32 \| BPF_JGE \| BPF_K:
	897	+ case BPF_JMP32 \| BPF_JGE \| BPF_X:
	898	+ case BPF_JMP32 \| BPF_JSGE \| BPF_K:
	899	+ case BPF_JMP32 \| BPF_JSGE \| BPF_X:
	900	+ true_cond = COND_GE;
	901	+ goto cond_branch;
	902	+ case BPF_JMP \| BPF_JLE \| BPF_K:
	903	+ case BPF_JMP \| BPF_JLE \| BPF_X:
	904	+ case BPF_JMP \| BPF_JSLE \| BPF_K:
	905	+ case BPF_JMP \| BPF_JSLE \| BPF_X:
	906	+ case BPF_JMP32 \| BPF_JLE \| BPF_K:
	907	+ case BPF_JMP32 \| BPF_JLE \| BPF_X:
	908	+ case BPF_JMP32 \| BPF_JSLE \| BPF_K:
	909	+ case BPF_JMP32 \| BPF_JSLE \| BPF_X:
	910	+ true_cond = COND_LE;
	911	+ goto cond_branch;
	912	+ case BPF_JMP \| BPF_JEQ \| BPF_K:
	913	+ case BPF_JMP \| BPF_JEQ \| BPF_X:
	914	+ case BPF_JMP32 \| BPF_JEQ \| BPF_K:
	915	+ case BPF_JMP32 \| BPF_JEQ \| BPF_X:
	916	+ true_cond = COND_EQ;
	917	+ goto cond_branch;
	918	+ case BPF_JMP \| BPF_JNE \| BPF_K:
	919	+ case BPF_JMP \| BPF_JNE \| BPF_X:
	920	+ case BPF_JMP32 \| BPF_JNE \| BPF_K:
	921	+ case BPF_JMP32 \| BPF_JNE \| BPF_X:
	922	+ true_cond = COND_NE;
	923	+ goto cond_branch;
	924	+ case BPF_JMP \| BPF_JSET \| BPF_K:
	925	+ case BPF_JMP \| BPF_JSET \| BPF_X:
	926	+ case BPF_JMP32 \| BPF_JSET \| BPF_K:
	927	+ case BPF_JMP32 \| BPF_JSET \| BPF_X:
	928	+ true_cond = COND_NE;
	929	+ /* fallthrough; */
	930	+
	931	+cond_branch:
	932	+ switch (code) {
	933	+ case BPF_JMP \| BPF_JGT \| BPF_X:
	934	+ case BPF_JMP \| BPF_JLT \| BPF_X:
	935	+ case BPF_JMP \| BPF_JGE \| BPF_X:
	936	+ case BPF_JMP \| BPF_JLE \| BPF_X:
	937	+ case BPF_JMP \| BPF_JEQ \| BPF_X:
	938	+ case BPF_JMP \| BPF_JNE \| BPF_X:
	939	+ /* unsigned comparison */
	940	+ EMIT(PPC_RAW_CMPLW(dst_reg_h, src_reg_h));
	941	+ PPC_BCC_SHORT(COND_NE, (ctx->idx + 2) * 4);
	942	+ EMIT(PPC_RAW_CMPLW(dst_reg, src_reg));
	943	+ break;
	944	+ case BPF_JMP32 \| BPF_JGT \| BPF_X:
	945	+ case BPF_JMP32 \| BPF_JLT \| BPF_X:
	946	+ case BPF_JMP32 \| BPF_JGE \| BPF_X:
	947	+ case BPF_JMP32 \| BPF_JLE \| BPF_X:
	948	+ case BPF_JMP32 \| BPF_JEQ \| BPF_X:
	949	+ case BPF_JMP32 \| BPF_JNE \| BPF_X:
	950	+ /* unsigned comparison */
	951	+ EMIT(PPC_RAW_CMPLW(dst_reg, src_reg));
	952	+ break;
	953	+ case BPF_JMP \| BPF_JSGT \| BPF_X:
	954	+ case BPF_JMP \| BPF_JSLT \| BPF_X:
	955	+ case BPF_JMP \| BPF_JSGE \| BPF_X:
	956	+ case BPF_JMP \| BPF_JSLE \| BPF_X:
	957	+ /* signed comparison */
	958	+ EMIT(PPC_RAW_CMPW(dst_reg_h, src_reg_h));
	959	+ PPC_BCC_SHORT(COND_NE, (ctx->idx + 2) * 4);
	960	+ EMIT(PPC_RAW_CMPLW(dst_reg, src_reg));
	961	+ break;
	962	+ case BPF_JMP32 \| BPF_JSGT \| BPF_X:
	963	+ case BPF_JMP32 \| BPF_JSLT \| BPF_X:
	964	+ case BPF_JMP32 \| BPF_JSGE \| BPF_X:
	965	+ case BPF_JMP32 \| BPF_JSLE \| BPF_X:
	966	+ /* signed comparison */
	967	+ EMIT(PPC_RAW_CMPW(dst_reg, src_reg));
	968	+ break;
	969	+ case BPF_JMP \| BPF_JSET \| BPF_X:
	970	+ EMIT(PPC_RAW_AND_DOT(__REG_R0, dst_reg_h, src_reg_h));
	971	+ PPC_BCC_SHORT(COND_NE, (ctx->idx + 2) * 4);
	972	+ EMIT(PPC_RAW_AND_DOT(__REG_R0, dst_reg, src_reg));
	973	+ break;
	974	+ case BPF_JMP32 \| BPF_JSET \| BPF_X: {
	975	+ EMIT(PPC_RAW_AND_DOT(__REG_R0, dst_reg, src_reg));
	976	+ break;
	977	+ case BPF_JMP \| BPF_JNE \| BPF_K:
	978	+ case BPF_JMP \| BPF_JEQ \| BPF_K:
	979	+ case BPF_JMP \| BPF_JGT \| BPF_K:
	980	+ case BPF_JMP \| BPF_JLT \| BPF_K:
	981	+ case BPF_JMP \| BPF_JGE \| BPF_K:
	982	+ case BPF_JMP \| BPF_JLE \| BPF_K:
	983	+ /*
	984	+ * Need sign-extended load, so only positive
	985	+ * values can be used as imm in cmplwi
	986	+ */
	987	+ if (imm >= 0 && imm < 32768) {
	988	+ EMIT(PPC_RAW_CMPLWI(dst_reg_h, 0));
	989	+ PPC_BCC_SHORT(COND_NE, (ctx->idx + 2) * 4);
	990	+ EMIT(PPC_RAW_CMPLWI(dst_reg, imm));
	991	+ } else {
	992	+ /* sign-extending load ... but unsigned comparison */
	993	+ PPC_EX32(__REG_R0, imm);
	994	+ EMIT(PPC_RAW_CMPLW(dst_reg_h, __REG_R0));
	995	+ PPC_LI32(__REG_R0, imm);
	996	+ PPC_BCC_SHORT(COND_NE, (ctx->idx + 2) * 4);
	997	+ EMIT(PPC_RAW_CMPLW(dst_reg, __REG_R0));
	998	+ }
	999	+ break;
	1000	+ case BPF_JMP32 \| BPF_JNE \| BPF_K:
	1001	+ case BPF_JMP32 \| BPF_JEQ \| BPF_K:
	1002	+ case BPF_JMP32 \| BPF_JGT \| BPF_K:
	1003	+ case BPF_JMP32 \| BPF_JLT \| BPF_K:
	1004	+ case BPF_JMP32 \| BPF_JGE \| BPF_K:
	1005	+ case BPF_JMP32 \| BPF_JLE \| BPF_K:
	1006	+ if (imm >= 0 && imm < 65536) {
	1007	+ EMIT(PPC_RAW_CMPLWI(dst_reg, imm));
	1008	+ } else {
	1009	+ PPC_LI32(__REG_R0, imm);
	1010	+ EMIT(PPC_RAW_CMPLW(dst_reg, __REG_R0));
	1011	+ }
	1012	+ break;
	1013	+ }
	1014	+ case BPF_JMP \| BPF_JSGT \| BPF_K:
	1015	+ case BPF_JMP \| BPF_JSLT \| BPF_K:
	1016	+ case BPF_JMP \| BPF_JSGE \| BPF_K:
	1017	+ case BPF_JMP \| BPF_JSLE \| BPF_K:
	1018	+ if (imm >= 0 && imm < 65536) {
	1019	+ EMIT(PPC_RAW_CMPWI(dst_reg_h, imm < 0 ? -1 : 0));
	1020	+ PPC_BCC_SHORT(COND_NE, (ctx->idx + 2) * 4);
	1021	+ EMIT(PPC_RAW_CMPLWI(dst_reg, imm));
	1022	+ } else {
	1023	+ /* sign-extending load */
	1024	+ EMIT(PPC_RAW_CMPWI(dst_reg_h, imm < 0 ? -1 : 0));
	1025	+ PPC_LI32(__REG_R0, imm);
	1026	+ PPC_BCC_SHORT(COND_NE, (ctx->idx + 2) * 4);
	1027	+ EMIT(PPC_RAW_CMPLW(dst_reg, __REG_R0));
	1028	+ }
	1029	+ break;
	1030	+ case BPF_JMP32 \| BPF_JSGT \| BPF_K:
	1031	+ case BPF_JMP32 \| BPF_JSLT \| BPF_K:
	1032	+ case BPF_JMP32 \| BPF_JSGE \| BPF_K:
	1033	+ case BPF_JMP32 \| BPF_JSLE \| BPF_K:
	1034	+ /*
	1035	+ * signed comparison, so any 16-bit value
	1036	+ * can be used in cmpwi
	1037	+ */
	1038	+ if (imm >= -32768 && imm < 32768) {
	1039	+ EMIT(PPC_RAW_CMPWI(dst_reg, imm));
	1040	+ } else {
	1041	+ /* sign-extending load */
	1042	+ PPC_LI32(__REG_R0, imm);
	1043	+ EMIT(PPC_RAW_CMPW(dst_reg, __REG_R0));
	1044	+ }
	1045	+ break;
	1046	+ case BPF_JMP \| BPF_JSET \| BPF_K:
	1047	+ /* andi does not sign-extend the immediate */
	1048	+ if (imm >= 0 && imm < 32768) {
	1049	+ /* PPC_ANDI is _only/always_ dot-form */
	1050	+ EMIT(PPC_RAW_ANDI(__REG_R0, dst_reg, imm));
	1051	+ } else {
	1052	+ PPC_LI32(__REG_R0, imm);
	1053	+ if (imm < 0) {
	1054	+ EMIT(PPC_RAW_CMPWI(dst_reg_h, 0));
	1055	+ PPC_BCC_SHORT(COND_NE, (ctx->idx + 2) * 4);
	1056	+ }
	1057	+ EMIT(PPC_RAW_AND_DOT(__REG_R0, dst_reg, __REG_R0));
	1058	+ }
	1059	+ break;
	1060	+ case BPF_JMP32 \| BPF_JSET \| BPF_K:
	1061	+ /* andi does not sign-extend the immediate */
	1062	+ if (imm >= -32768 && imm < 32768) {
	1063	+ /* PPC_ANDI is _only/always_ dot-form */
	1064	+ EMIT(PPC_RAW_ANDI(__REG_R0, dst_reg, imm));
	1065	+ } else {
	1066	+ PPC_LI32(__REG_R0, imm);
	1067	+ EMIT(PPC_RAW_AND_DOT(__REG_R0, dst_reg, __REG_R0));
	1068	+ }
	1069	+ break;
	1070	+ }
	1071	+ PPC_BCC(true_cond, addrs[i + 1 + off]);
	1072	+ break;
	1073	+
	1074	+ /*
	1075	+ * Tail call
	1076	+ */
	1077	+ case BPF_JMP \| BPF_TAIL_CALL:
	1078	+ ctx->seen \|= SEEN_TAILCALL;
	1079	+ bpf_jit_emit_tail_call(image, ctx, addrs[i + 1]);
	1080	+ break;
	1081	+
	1082	+ default:
	1083	+ /*
	1084	+ * The filter contains something cruel & unusual.
	1085	+ * We don't handle it, but also there shouldn't be
	1086	+ * anything missing from our list.
	1087	+ */
	1088	+ pr_err_ratelimited("eBPF filter opcode %04x (@%d) unsupported\n", code, i);
	1089	+ return -EOPNOTSUPP;
	1090	+ }
	1091	+ if (BPF_CLASS(code) == BPF_ALU && !fp->aux->verifier_zext &&
	1092	+ !insn_is_zext(&insn[i + 1]))
	1093	+ EMIT(PPC_RAW_LI(dst_reg_h, 0));
	1094	+ }
	1095	+
	1096	+ /* Set end-of-body-code address for exit. */
	1097	+ addrs[i] = ctx->idx * 4;
	1098	+
	1099	+ return 0;
	1100	+}

arch/powerpc/net/bpf_jit_comp64.c+15−280

@@ -18,27 +18,6 @@ · 17 unmodified lines
18	18
19	19	#include "bpf_jit64.h"
20	20
21		−static void bpf_jit_fill_ill_insns(void *area, unsigned int size)
22		−{
23		− memset32(area, BREAKPOINT_INSTRUCTION, size/4);
24		−}
25		−
26		−static inline void bpf_flush_icache(void start, void end)
27		−{
28		− smp_wmb();
29		− flush_icache_range((unsigned long)start, (unsigned long)end);
30		−}
31		−
32		−static inline bool bpf_is_seen_register(struct codegen_context *ctx, int i)
33		−{
34		− return (ctx->seen & (1 << (31 - b2p[i])));
35		−}
36		−
37		−static inline void bpf_set_seen_register(struct codegen_context *ctx, int i)
38		−{
39		− ctx->seen \|= (1 << (31 - b2p[i]));
40		−}
41		−
42	21	static inline bool bpf_has_stack_frame(struct codegen_context *ctx)
43	22	{
44	23	/*
@@ -47,7 +26,7 @@ static inline bool bpf_has_stack_frame(struct codegen_context *ctx) · 2 unmodified lines
47	26	* - the bpf program uses its stack area
48	27	* The latter condition is deduced from the usage of BPF_REG_FP
49	28	*/
50		− return ctx->seen & SEEN_FUNC \|\| bpf_is_seen_register(ctx, BPF_REG_FP);
	29	+ return ctx->seen & SEEN_FUNC \|\| bpf_is_seen_register(ctx, b2p[BPF_REG_FP]);
51	30	}
52	31
53	32	/*
@@ -85,7 +64,11 @@ static int bpf_jit_stack_offsetof(struct codegen_context *ctx, int reg) · 31 unmodified lines
85	64	BUG();
86	65	}
87	66
88		−static void bpf_jit_build_prologue(u32 image, struct codegen_context ctx)
	67	+void bpf_jit_realloc_regs(struct codegen_context *ctx)
	68	+{
	69	+}
	70	+
	71	+void bpf_jit_build_prologue(u32 image, struct codegen_context ctx)
89	72	{
90	73	int i;
91	74
@@ -124,11 +107,11 @@ static void bpf_jit_build_prologue(u32 image, struct codegen_context ctx) · 32 unmodified lines
124	107	* in the protected zone below the previous stack frame
125	108	*/
126	109	for (i = BPF_REG_6; i <= BPF_REG_10; i++)
127		− if (bpf_is_seen_register(ctx, i))
	110	+ if (bpf_is_seen_register(ctx, b2p[i]))
128	111	PPC_BPF_STL(b2p[i], 1, bpf_jit_stack_offsetof(ctx, b2p[i]));
129	112
130	113	/* Setup frame pointer to point to the bpf stack area */
131		− if (bpf_is_seen_register(ctx, BPF_REG_FP))
	114	+ if (bpf_is_seen_register(ctx, b2p[BPF_REG_FP]))
132	115	EMIT(PPC_RAW_ADDI(b2p[BPF_REG_FP], 1,
133	116	STACK_FRAME_MIN_SIZE + ctx->stack_size));
134	117	}
@@ -139,7 +122,7 @@ static void bpf_jit_emit_common_epilogue(u32 image, struct codegen_context ctx · 4 unmodified lines
139	122
140	123	/* Restore NVRs */
141	124	for (i = BPF_REG_6; i <= BPF_REG_10; i++)
142		− if (bpf_is_seen_register(ctx, i))
	125	+ if (bpf_is_seen_register(ctx, b2p[i]))
143	126	PPC_BPF_LL(b2p[i], 1, bpf_jit_stack_offsetof(ctx, b2p[i]));
144	127
145	128	/* Tear down our stack frame */
@@ -152,7 +135,7 @@ static void bpf_jit_emit_common_epilogue(u32 image, struct codegen_context ctx · 6 unmodified lines
152	135	}
153	136	}
154	137
155		−static void bpf_jit_build_epilogue(u32 image, struct codegen_context ctx)
	138	+void bpf_jit_build_epilogue(u32 image, struct codegen_context ctx)
156	139	{
157	140	bpf_jit_emit_common_epilogue(image, ctx);
158	141
@@ -187,8 +170,7 @@ static void bpf_jit_emit_func_call_hlp(u32 image, struct codegen_context ctx, · 28 unmodified lines
187	170	EMIT(PPC_RAW_BLRL());
188	171	}
189	172
190		−static void bpf_jit_emit_func_call_rel(u32 image, struct codegen_context ctx,
191		− u64 func)
	173	+void bpf_jit_emit_func_call_rel(u32 image, struct codegen_context ctx, u64 func)
192	174	{
193	175	unsigned int i, ctx_idx = ctx->idx;
194	176
@@ -289,9 +271,8 @@ static void bpf_jit_emit_tail_call(u32 image, struct codegen_context ctx, u32 · 94 unmodified lines
289	271	}
290	272
291	273	/* Assemble the body code between the prologue & epilogue */
292		−static int bpf_jit_build_body(struct bpf_prog fp, u32 image,
293		− struct codegen_context *ctx,
294		− u32 *addrs, bool extra_pass)
	274	+int bpf_jit_build_body(struct bpf_prog fp, u32 image, struct codegen_context *ctx,
	275	+ u32 *addrs, bool extra_pass)
295	276	{
296	277	const struct bpf_insn *insn = fp->insnsi;
297	278	int flen = fp->len;
@@ -330,9 +311,9 @@ static int bpf_jit_build_body(struct bpf_prog fp, u32 image, · 32 unmodified lines
330	311	* any issues.
331	312	*/
332	313	if (dst_reg >= BPF_PPC_NVR_MIN && dst_reg < 32)
333		− bpf_set_seen_register(ctx, insn[i].dst_reg);
	314	+ bpf_set_seen_register(ctx, dst_reg);
334	315	if (src_reg >= BPF_PPC_NVR_MIN && src_reg < 32)
335		− bpf_set_seen_register(ctx, insn[i].src_reg);
	316	+ bpf_set_seen_register(ctx, src_reg);
336	317
337	318	switch (code) {
338	319	/*
@@ -1026,249 +1007,3 @@ cond_branch: · 687 unmodified lines
1026	1007
1027	1008	return 0;
1028	1009	}
1029		−
1030		−/* Fix the branch target addresses for subprog calls */
1031		−static int bpf_jit_fixup_subprog_calls(struct bpf_prog fp, u32 image,
1032		− struct codegen_context ctx, u32 addrs)
1033		−{
1034		− const struct bpf_insn *insn = fp->insnsi;
1035		− bool func_addr_fixed;
1036		− u64 func_addr;
1037		− u32 tmp_idx;
1038		− int i, ret;
1039		−
1040		− for (i = 0; i < fp->len; i++) {
1041		− /*
1042		− * During the extra pass, only the branch target addresses for
1043		− * the subprog calls need to be fixed. All other instructions
1044		− * can left untouched.
1045		− *
1046		− * The JITed image length does not change because we already
1047		− * ensure that the JITed instruction sequence for these calls
1048		− * are of fixed length by padding them with NOPs.
1049		− */
1050		− if (insn[i].code == (BPF_JMP \| BPF_CALL) &&
1051		− insn[i].src_reg == BPF_PSEUDO_CALL) {
1052		− ret = bpf_jit_get_func_addr(fp, &insn[i], true,
1053		− &func_addr,
1054		− &func_addr_fixed);
1055		− if (ret < 0)
1056		− return ret;
1057		−
1058		− /*
1059		− * Save ctx->idx as this would currently point to the
1060		− * end of the JITed image and set it to the offset of
1061		− * the instruction sequence corresponding to the
1062		− * subprog call temporarily.
1063		− */
1064		− tmp_idx = ctx->idx;
1065		− ctx->idx = addrs[i] / 4;
1066		− bpf_jit_emit_func_call_rel(image, ctx, func_addr);
1067		−
1068		− /*
1069		− * Restore ctx->idx here. This is safe as the length
1070		− * of the JITed sequence remains unchanged.
1071		− */
1072		− ctx->idx = tmp_idx;
1073		− }
1074		− }
1075		−
1076		− return 0;
1077		−}
1078		−
1079		−struct powerpc64_jit_data {
1080		− struct bpf_binary_header *header;
1081		− u32 *addrs;
1082		− u8 *image;
1083		− u32 proglen;
1084		− struct codegen_context ctx;
1085		−};
1086		−
1087		−bool bpf_jit_needs_zext(void)
1088		−{
1089		− return true;
1090		−}
1091		−
1092		−struct bpf_prog bpf_int_jit_compile(struct bpf_prog fp)
1093		−{
1094		− u32 proglen;
1095		− u32 alloclen;
1096		− u8 *image = NULL;
1097		− u32 *code_base;
1098		− u32 *addrs;
1099		− struct powerpc64_jit_data *jit_data;
1100		− struct codegen_context cgctx;
1101		− int pass;
1102		− int flen;
1103		− struct bpf_binary_header *bpf_hdr;
1104		− struct bpf_prog *org_fp = fp;
1105		− struct bpf_prog *tmp_fp;
1106		− bool bpf_blinded = false;
1107		− bool extra_pass = false;
1108		−
1109		− if (!fp->jit_requested)
1110		− return org_fp;
1111		−
1112		− tmp_fp = bpf_jit_blind_constants(org_fp);
1113		− if (IS_ERR(tmp_fp))
1114		− return org_fp;
1115		−
1116		− if (tmp_fp != org_fp) {
1117		− bpf_blinded = true;
1118		− fp = tmp_fp;
1119		− }
1120		−
1121		− jit_data = fp->aux->jit_data;
1122		− if (!jit_data) {
1123		− jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL);
1124		− if (!jit_data) {
1125		− fp = org_fp;
1126		− goto out;
1127		− }
1128		− fp->aux->jit_data = jit_data;
1129		− }
1130		−
1131		− flen = fp->len;
1132		− addrs = jit_data->addrs;
1133		− if (addrs) {
1134		− cgctx = jit_data->ctx;
1135		− image = jit_data->image;
1136		− bpf_hdr = jit_data->header;
1137		− proglen = jit_data->proglen;
1138		− alloclen = proglen + FUNCTION_DESCR_SIZE;
1139		− extra_pass = true;
1140		− goto skip_init_ctx;
1141		− }
1142		−
1143		− addrs = kcalloc(flen + 1, sizeof(*addrs), GFP_KERNEL);
1144		− if (addrs == NULL) {
1145		− fp = org_fp;
1146		− goto out_addrs;
1147		− }
1148		−
1149		− memset(&cgctx, 0, sizeof(struct codegen_context));
1150		−
1151		− /* Make sure that the stack is quadword aligned. */
1152		− cgctx.stack_size = round_up(fp->aux->stack_depth, 16);
1153		−
1154		− /* Scouting faux-generate pass 0 */
1155		− if (bpf_jit_build_body(fp, 0, &cgctx, addrs, false)) {
1156		− /* We hit something illegal or unsupported. */
1157		− fp = org_fp;
1158		− goto out_addrs;
1159		− }
1160		−
1161		− /*
1162		− * If we have seen a tail call, we need a second pass.
1163		− * This is because bpf_jit_emit_common_epilogue() is called
1164		− * from bpf_jit_emit_tail_call() with a not yet stable ctx->seen.
1165		− */
1166		− if (cgctx.seen & SEEN_TAILCALL) {
1167		− cgctx.idx = 0;
1168		− if (bpf_jit_build_body(fp, 0, &cgctx, addrs, false)) {
1169		− fp = org_fp;
1170		− goto out_addrs;
1171		− }
1172		− }
1173		−
1174		− /*
1175		− * Pretend to build prologue, given the features we've seen. This will
1176		− * update ctgtx.idx as it pretends to output instructions, then we can
1177		− * calculate total size from idx.
1178		− */
1179		− bpf_jit_build_prologue(0, &cgctx);
1180		− bpf_jit_build_epilogue(0, &cgctx);
1181		−
1182		− proglen = cgctx.idx * 4;
1183		− alloclen = proglen + FUNCTION_DESCR_SIZE;
1184		−
1185		− bpf_hdr = bpf_jit_binary_alloc(alloclen, &image, 4,
1186		− bpf_jit_fill_ill_insns);
1187		− if (!bpf_hdr) {
1188		− fp = org_fp;
1189		− goto out_addrs;
1190		− }
1191		−
1192		−skip_init_ctx:
1193		− code_base = (u32 *)(image + FUNCTION_DESCR_SIZE);
1194		−
1195		− if (extra_pass) {
1196		− /*
1197		− * Do not touch the prologue and epilogue as they will remain
1198		− * unchanged. Only fix the branch target address for subprog
1199		− * calls in the body.
1200		− *
1201		− * This does not change the offsets and lengths of the subprog
1202		− * call instruction sequences and hence, the size of the JITed
1203		− * image as well.
1204		− */
1205		− bpf_jit_fixup_subprog_calls(fp, code_base, &cgctx, addrs);
1206		−
1207		− /* There is no need to perform the usual passes. */
1208		− goto skip_codegen_passes;
1209		− }
1210		−
1211		− /* Code generation passes 1-2 */
1212		− for (pass = 1; pass < 3; pass++) {
1213		− /* Now build the prologue, body code & epilogue for real. */
1214		− cgctx.idx = 0;
1215		− bpf_jit_build_prologue(code_base, &cgctx);
1216		− bpf_jit_build_body(fp, code_base, &cgctx, addrs, extra_pass);
1217		− bpf_jit_build_epilogue(code_base, &cgctx);
1218		−
1219		− if (bpf_jit_enable > 1)
1220		− pr_info("Pass %d: shrink = %d, seen = 0x%x\n", pass,
1221		− proglen - (cgctx.idx * 4), cgctx.seen);
1222		− }
1223		−
1224		−skip_codegen_passes:
1225		− if (bpf_jit_enable > 1)
1226		− /*
1227		− * Note that we output the base address of the code_base
1228		− * rather than image, since opcodes are in code_base.
1229		− */
1230		− bpf_jit_dump(flen, proglen, pass, code_base);
1231		−
1232		−#ifdef PPC64_ELF_ABI_v1
1233		− /* Function descriptor nastiness: Address + TOC */
1234		− ((u64 *)image)[0] = (u64)code_base;
1235		− ((u64 *)image)[1] = local_paca->kernel_toc;
1236		−#endif
1237		−
1238		− fp->bpf_func = (void *)image;
1239		− fp->jited = 1;
1240		− fp->jited_len = alloclen;
1241		−
1242		− bpf_flush_icache(bpf_hdr, (u8 )bpf_hdr + (bpf_hdr->pages PAGE_SIZE));
1243		− if (!fp->is_func \|\| extra_pass) {
1244		− bpf_prog_fill_jited_linfo(fp, addrs);
1245		−out_addrs:
1246		− kfree(addrs);
1247		− kfree(jit_data);
1248		− fp->aux->jit_data = NULL;
1249		− } else {
1250		− jit_data->addrs = addrs;
1251		− jit_data->ctx = cgctx;
1252		− jit_data->proglen = proglen;
1253		− jit_data->image = image;
1254		− jit_data->header = bpf_hdr;
1255		− }
1256		−
1257		−out:
1258		− if (bpf_blinded)
1259		− bpf_jit_prog_release_other(fp, fp == org_fp ? tmp_fp : org_fp);
1260		−
1261		− return fp;
1262		−}
1263		−
1264		−/* Overriding bpf_jit_free() as we don't set images read-only. */
1265		−void bpf_jit_free(struct bpf_prog *fp)
1266		−{
1267		− unsigned long addr = (unsigned long)fp->bpf_func & PAGE_MASK;
1268		− struct bpf_binary_header bpf_hdr = (void )addr;
1269		−
1270		− if (fp->jited)
1271		− bpf_jit_binary_free(bpf_hdr);
1272		−
1273		− bpf_prog_unlock_free(fp);
1274		−}

arch/powerpc/perf/core-book3s.c+16−4

@@ -17,6 +17,7 @@ · 16 unmodified lines
17	17	#include <asm/firmware.h>
18	18	#include <asm/ptrace.h>
19	19	#include <asm/code-patching.h>
	20	+#include <asm/interrupt.h>
20	21
21	22	#ifdef CONFIG_PPC64
22	23	#include "internal.h"
@@ -168,7 +169,7 @@ static bool regs_use_siar(struct pt_regs *regs) · 145 unmodified lines
168	169	* they have not been setup using perf_read_regs() and so regs->result
169	170	* is something random.
170	171	*/
171		− return ((TRAP(regs) == 0xf00) && regs->result);
	172	+ return ((TRAP(regs) == INTERRUPT_PERFMON) && regs->result);
172	173	}
173	174
174	175	/*
@@ -347,7 +348,7 @@ static inline void perf_read_regs(struct pt_regs *regs) · 172 unmodified lines
347	348	* hypervisor samples as well as samples in the kernel with
348	349	* interrupts off hence the userspace check.
349	350	*/
350		− if (TRAP(regs) != 0xf00)
	351	+ if (TRAP(regs) != INTERRUPT_PERFMON)
351	352	use_siar = 0;
352	353	else if ((ppmu->flags & PPMU_NO_SIAR))
353	354	use_siar = 0;
@@ -1963,6 +1964,17 @@ static int power_pmu_event_init(struct perf_event *event) · 1609 unmodified lines
1963	1964	return -ENOENT;
1964	1965	}
1965	1966
	1967	+ /*
	1968	+ * PMU config registers have fields that are
	1969	+ * reserved and some specific values for bit fields are reserved.
	1970	+ * For ex., MMCRA[61:62] is Randome Sampling Mode (SM)
	1971	+ * and value of 0b11 to this field is reserved.
	1972	+ * Check for invalid values in attr.config.
	1973	+ */
	1974	+ if (ppmu->check_attr_config &&
	1975	+ ppmu->check_attr_config(event))
	1976	+ return -EINVAL;
	1977	+
1966	1978	event->hw.config_base = ev;
1967	1979	event->hw.idx = 0;
1968	1980
@@ -2206,9 +2218,9 @@ static void record_and_restart(struct perf_event *event, unsigned long val, · 237 unmodified lines
2206	2218	ppmu->get_mem_data_src)
2207	2219	ppmu->get_mem_data_src(&data.data_src, ppmu->flags, regs);
2208	2220
2209		− if (event->attr.sample_type & PERF_SAMPLE_WEIGHT &&
	2221	+ if (event->attr.sample_type & PERF_SAMPLE_WEIGHT_TYPE &&
2210	2222	ppmu->get_mem_weight)
2211		− ppmu->get_mem_weight(&data.weight.full);
	2223	+ ppmu->get_mem_weight(&data.weight.full, event->attr.sample_type);
2212	2224
2213	2225	if (perf_event_overflow(event, &data, regs))
2214	2226	power_pmu_stop(event, 0);

arch/powerpc/perf/hv-24x7.c+5−5

@@ -226,14 +226,14 @@ static struct attribute_group event_long_desc_group = { · 225 unmodified lines
226	226
227	227	static struct kmem_cache *hv_page_cache;
228	228
229		−DEFINE_PER_CPU(int, hv_24x7_txn_flags);
230		−DEFINE_PER_CPU(int, hv_24x7_txn_err);
	229	+static DEFINE_PER_CPU(int, hv_24x7_txn_flags);
	230	+static DEFINE_PER_CPU(int, hv_24x7_txn_err);
231	231
232	232	struct hv_24x7_hw {
233	233	struct perf_event *events[255];
234	234	};
235	235
236		−DEFINE_PER_CPU(struct hv_24x7_hw, hv_24x7_hw);
	236	+static DEFINE_PER_CPU(struct hv_24x7_hw, hv_24x7_hw);
237	237
238	238	/*
239	239	* request_buffer and result_buffer are not required to be 4k aligned,
@@ -241,8 +241,8 @@ DEFINE_PER_CPU(struct hv_24x7_hw, hv_24x7_hw); · 1 unmodified line
241	241	* the simplest way to ensure that.
242	242	*/
243	243	#define H24x7_DATA_BUFFER_SIZE 4096
244		−DEFINE_PER_CPU(char, hv_24x7_reqb[H24x7_DATA_BUFFER_SIZE]) __aligned(4096);
245		−DEFINE_PER_CPU(char, hv_24x7_resb[H24x7_DATA_BUFFER_SIZE]) __aligned(4096);
	244	+static DEFINE_PER_CPU(char, hv_24x7_reqb[H24x7_DATA_BUFFER_SIZE]) __aligned(4096);
	245	+static DEFINE_PER_CPU(char, hv_24x7_resb[H24x7_DATA_BUFFER_SIZE]) __aligned(4096);
246	246
247	247	static unsigned int max_num_requests(int interface_version)
248	248	{

arch/powerpc/perf/isa207-common.c+104−11

@@ -21,7 +21,7 @@ PMU_FORMAT_ATTR(thresh_stop, "config:32-35"); · 20 unmodified lines
21	21	PMU_FORMAT_ATTR(thresh_start, "config:36-39");
22	22	PMU_FORMAT_ATTR(thresh_cmp, "config:40-49");
23	23
24		−struct attribute *isa207_pmu_format_attr[] = {
	24	+static struct attribute *isa207_pmu_format_attr[] = {
25	25	&format_attr_event.attr,
26	26	&format_attr_pmcxsel.attr,
27	27	&format_attr_mark.attr,
@@ -275,17 +275,47 @@ void isa207_get_mem_data_src(union perf_mem_data_src *dsrc, u32 flags, · 247 unmodified lines
275	275
276	276	sier = mfspr(SPRN_SIER);
277	277	val = (sier & ISA207_SIER_TYPE_MASK) >> ISA207_SIER_TYPE_SHIFT;
278		− if (val == 1 \|\| val == 2) {
279		− idx = (sier & ISA207_SIER_LDST_MASK) >> ISA207_SIER_LDST_SHIFT;
280		− sub_idx = (sier & ISA207_SIER_DATA_SRC_MASK) >> ISA207_SIER_DATA_SRC_SHIFT;
	278	+ if (val != 1 && val != 2 && !(val == 7 && cpu_has_feature(CPU_FTR_ARCH_31)))
	279	+ return;
	280	+
	281	+ idx = (sier & ISA207_SIER_LDST_MASK) >> ISA207_SIER_LDST_SHIFT;
	282	+ sub_idx = (sier & ISA207_SIER_DATA_SRC_MASK) >> ISA207_SIER_DATA_SRC_SHIFT;
	283	+
	284	+ dsrc->val = isa207_find_source(idx, sub_idx);
	285	+ if (val == 7) {
	286	+ u64 mmcra;
	287	+ u32 op_type;
281	288
282		− dsrc->val = isa207_find_source(idx, sub_idx);
	289	+ /*
	290	+ * Type 0b111 denotes either larx or stcx instruction. Use the
	291	+ * MMCRA sampling bits [57:59] along with the type value
	292	+ * to determine the exact instruction type. If the sampling
	293	+ * criteria is neither load or store, set the type as default
	294	+ * to NA.
	295	+ */
	296	+ mmcra = mfspr(SPRN_MMCRA);
	297	+
	298	+ op_type = (mmcra >> MMCRA_SAMP_ELIG_SHIFT) & MMCRA_SAMP_ELIG_MASK;
	299	+ switch (op_type) {
	300	+ case 5:
	301	+ dsrc->val \|= P(OP, LOAD);
	302	+ break;
	303	+ case 7:
	304	+ dsrc->val \|= P(OP, STORE);
	305	+ break;
	306	+ default:
	307	+ dsrc->val \|= P(OP, NA);
	308	+ break;
	309	+ }
	310	+ } else {
283	311	dsrc->val \|= (val == 1) ? P(OP, LOAD) : P(OP, STORE);
284	312	}
285	313	}
286	314
287		−void isa207_get_mem_weight(u64 *weight)
	315	+void isa207_get_mem_weight(u64 *weight, u64 type)
288	316	{
	317	+ union perf_sample_weight *weight_fields;
	318	+ u64 weight_lat;
289	319	u64 mmcra = mfspr(SPRN_MMCRA);
290	320	u64 exp = MMCRA_THR_CTR_EXP(mmcra);
291	321	u64 mantissa = MMCRA_THR_CTR_MANT(mmcra);
@@ -295,10 +325,31 @@ void isa207_get_mem_weight(u64 *weight) · 3 unmodified lines
295	325	if (cpu_has_feature(CPU_FTR_ARCH_31))
296	326	mantissa = P10_MMCRA_THR_CTR_MANT(mmcra);
297	327
298		− if (val == 0 \|\| val == 7)
299		− *weight = 0;
	328	+ if (val == 0 \|\| (val == 7 && !cpu_has_feature(CPU_FTR_ARCH_31)))
	329	+ weight_lat = 0;
300	330	else
301		− weight = mantissa << (2 exp);
	331	+ weight_lat = mantissa << (2 * exp);
	332	+
	333	+ /*
	334	+ * Use 64 bit weight field (full) if sample type is
	335	+ * WEIGHT.
	336	+ *
	337	+ * if sample type is WEIGHT_STRUCT:
	338	+ * - store memory latency in the lower 32 bits.
	339	+ * - For ISA v3.1, use remaining two 16 bit fields of
	340	+ * perf_sample_weight to store cycle counter values
	341	+ * from sier2.
	342	+ */
	343	+ weight_fields = (union perf_sample_weight *)weight;
	344	+ if (type & PERF_SAMPLE_WEIGHT)
	345	+ weight_fields->full = weight_lat;
	346	+ else {
	347	+ weight_fields->var1_dw = (u32)weight_lat;
	348	+ if (cpu_has_feature(CPU_FTR_ARCH_31)) {
	349	+ weight_fields->var2_w = P10_SIER2_FINISH_CYC(mfspr(SPRN_SIER2));
	350	+ weight_fields->var3_w = P10_SIER2_DISPATCH_CYC(mfspr(SPRN_SIER2));
	351	+ }
	352	+ }
302	353	}
303	354
304	355	int isa207_get_constraint(u64 event, unsigned long maskp, unsigned long valp, u64 event_config1)
@@ -447,8 +498,8 @@ ebb_bhrb: · 142 unmodified lines
447	498	* EBB events are pinned & exclusive, so this should never actually
448	499	* hit, but we leave it as a fallback in case.
449	500	*/
450		− mask \|= CNST_EBB_VAL(ebb);
451		− value \|= CNST_EBB_MASK;
	501	+ mask \|= CNST_EBB_MASK;
	502	+ value \|= CNST_EBB_VAL(ebb);
452	503
453	504	*maskp = mask;
454	505	*valp = value;
@@ -694,3 +745,45 @@ int isa207_get_alternatives(u64 event, u64 alt[], int size, unsigned int flags, · 239 unmodified lines
694	745
695	746	return num_alt;
696	747	}
	748	+
	749	+int isa3XX_check_attr_config(struct perf_event *ev)
	750	+{
	751	+ u64 val, sample_mode;
	752	+ u64 event = ev->attr.config;
	753	+
	754	+ val = (event >> EVENT_SAMPLE_SHIFT) & EVENT_SAMPLE_MASK;
	755	+ sample_mode = val & 0x3;
	756	+
	757	+ /*
	758	+ * MMCRA[61:62] is Random Sampling Mode (SM).
	759	+ * value of 0b11 is reserved.
	760	+ */
	761	+ if (sample_mode == 0x3)
	762	+ return -EINVAL;
	763	+
	764	+ /*
	765	+ * Check for all reserved value
	766	+ * Source: Performance Monitoring Unit User Guide
	767	+ */
	768	+ switch (val) {
	769	+ case 0x5:
	770	+ case 0x9:
	771	+ case 0xD:
	772	+ case 0x19:
	773	+ case 0x1D:
	774	+ case 0x1A:
	775	+ case 0x1E:
	776	+ return -EINVAL;
	777	+ }
	778	+
	779	+ /*
	780	+ * MMCRA[48:51]/[52:55]) Threshold Start/Stop
	781	+ * Events Selection.
	782	+ * 0b11110000/0b00001111 is reserved.
	783	+ */
	784	+ val = (event >> EVENT_THR_CTL_SHIFT) & EVENT_THR_CTL_MASK;
	785	+ if (((val & 0xF0) == 0xF0) \|\| ((val & 0xF) == 0xF))
	786	+ return -EINVAL;
	787	+
	788	+ return 0;
	789	+}

arch/powerpc/perf/isa207-common.h+8−1

@@ -220,6 +220,7 @@ · 219 unmodified lines
220	220	/* Bits in MMCRA for PowerISA v2.07 */
221	221	#define MMCRA_SAMP_MODE_SHIFT 1
222	222	#define MMCRA_SAMP_ELIG_SHIFT 4
	223	+#define MMCRA_SAMP_ELIG_MASK 7
223	224	#define MMCRA_THR_CTL_SHIFT 8
224	225	#define MMCRA_THR_SEL_SHIFT 16
225	226	#define MMCRA_THR_CMP_SHIFT 32
@@ -265,6 +266,10 @@ · 39 unmodified lines
265	266	#define ISA207_SIER_DATA_SRC_SHIFT 53
266	267	#define ISA207_SIER_DATA_SRC_MASK (0x7ull << ISA207_SIER_DATA_SRC_SHIFT)
267	268
	269	+/* Bits in SIER2/SIER3 for Power10 */
	270	+#define P10_SIER2_FINISH_CYC(sier2) (((sier2) >> (63 - 37)) & 0x7fful)
	271	+#define P10_SIER2_DISPATCH_CYC(sier2) (((sier2) >> (63 - 13)) & 0x7fful)
	272	+
268	273	#define P(a, b) PERF_MEM_S(a, b)
269	274	#define PH(a, b) (P(LVL, HIT) \| P(a, b))
270	275	#define PM(a, b) (P(LVL, MISS) \| P(a, b))
@@ -278,6 +283,8 @@ int isa207_get_alternatives(u64 event, u64 alt[], int size, unsigned int flags, · 7 unmodified lines
278	283	const unsigned int ev_alt[][MAX_ALT]);
279	284	void isa207_get_mem_data_src(union perf_mem_data_src *dsrc, u32 flags,
280	285	struct pt_regs *regs);
281		−void isa207_get_mem_weight(u64 *weight);
	286	+void isa207_get_mem_weight(u64 *weight, u64 type);
	287	+
	288	+int isa3XX_check_attr_config(struct perf_event *ev);
282	289
283	290	#endif

arch/powerpc/perf/power10-events-list.h+2−2

@@ -75,5 +75,5 @@ EVENT(PM_RUN_INST_CMPL_ALT, 0x00002); · 74 unmodified lines
75	75	* thresh end (TE)
76	76	*/
77	77
78		−EVENT(MEM_LOADS, 0x34340401e0);
79		−EVENT(MEM_STORES, 0x343c0401e0);
	78	+EVENT(MEM_LOADS, 0x35340401e0);
	79	+EVENT(MEM_STORES, 0x353c0401e0);

arch/powerpc/perf/power10-pmu.c+13−0

@@ -106,6 +106,18 @@ static int power10_get_alternatives(u64 event, unsigned int flags, u64 alt[]) · 105 unmodified lines
106	106	return num_alt;
107	107	}
108	108
	109	+static int power10_check_attr_config(struct perf_event *ev)
	110	+{
	111	+ u64 val;
	112	+ u64 event = ev->attr.config;
	113	+
	114	+ val = (event >> EVENT_SAMPLE_SHIFT) & EVENT_SAMPLE_MASK;
	115	+ if (val == 0x10 \|\| isa3XX_check_attr_config(ev))
	116	+ return -EINVAL;
	117	+
	118	+ return 0;
	119	+}
	120	+
109	121	GENERIC_EVENT_ATTR(cpu-cycles, PM_RUN_CYC);
110	122	GENERIC_EVENT_ATTR(instructions, PM_RUN_INST_CMPL);
111	123	GENERIC_EVENT_ATTR(branch-instructions, PM_BR_CMPL);
@@ -559,6 +571,7 @@ static struct power_pmu power10_pmu = { · 447 unmodified lines
559	571	.attr_groups = power10_pmu_attr_groups,
560	572	.bhrb_nr = 32,
561	573	.capabilities = PERF_PMU_CAP_EXTENDED_REGS,
	574	+ .check_attr_config = power10_check_attr_config,
562	575	};
563	576
564	577	int init_power10_pmu(void)

arch/powerpc/perf/power9-pmu.c+13−0

@@ -151,6 +151,18 @@ static int power9_get_alternatives(u64 event, unsigned int flags, u64 alt[]) · 150 unmodified lines
151	151	return num_alt;
152	152	}
153	153
	154	+static int power9_check_attr_config(struct perf_event *ev)
	155	+{
	156	+ u64 val;
	157	+ u64 event = ev->attr.config;
	158	+
	159	+ val = (event >> EVENT_SAMPLE_SHIFT) & EVENT_SAMPLE_MASK;
	160	+ if (val == 0xC \|\| isa3XX_check_attr_config(ev))
	161	+ return -EINVAL;
	162	+
	163	+ return 0;
	164	+}
	165	+
154	166	GENERIC_EVENT_ATTR(cpu-cycles, PM_CYC);
155	167	GENERIC_EVENT_ATTR(stalled-cycles-frontend, PM_ICT_NOSLOT_CYC);
156	168	GENERIC_EVENT_ATTR(stalled-cycles-backend, PM_CMPLU_STALL);
@@ -437,6 +449,7 @@ static struct power_pmu power9_pmu = { · 280 unmodified lines
437	449	.attr_groups = power9_pmu_attr_groups,
438	450	.bhrb_nr = 32,
439	451	.capabilities = PERF_PMU_CAP_EXTENDED_REGS,
	452	+ .check_attr_config = power9_check_attr_config,
440	453	};
441	454
442	455	int init_power9_pmu(void)

arch/powerpc/platforms/44x/Kconfig+1−1

@@ -5,7 +5,7 @@ config PPC_47x · 4 unmodified lines
5	5	select MPIC
6	6	help
7	7	This option enables support for the 47x family of processors and is
8		− not currently compatible with other 44x or 46x varients
	8	+ not currently compatible with other 44x or 46x variants
9	9
10	10	config BAMBOO
11	11	bool "Bamboo"

arch/powerpc/platforms/52xx/lite5200_sleep.S+1−1

@@ -181,7 +181,7 @@ sram_code: · 180 unmodified lines
181	181	udelay: /* r11 - tb_ticks_per_usec, r12 - usecs, overwrites r13 */
182	182	mullw r12, r12, r11
183	183	mftb r13 /* start */
184		− addi r12, r13, r12 /* end */
	184	+ add r12, r13, r12 /* end */
185	185	1:
186	186	mftb r13 /* current */
187	187	cmp cr0, r13, r12

arch/powerpc/platforms/Kconfig.cputype+3−2

@@ -101,6 +101,8 @@ config PPC_BOOK3S_64 · 100 unmodified lines
101	101	select ARCH_SUPPORTS_NUMA_BALANCING
102	102	select IRQ_WORK
103	103	select PPC_MM_SLICES
	104	+ select PPC_HAVE_KUEP
	105	+ select PPC_HAVE_KUAP
104	106
105	107	config PPC_BOOK3E_64
106	108	bool "Embedded processors"
@@ -306,6 +308,7 @@ config PHYS_64BIT · 199 unmodified lines
306	308	config ALTIVEC
307	309	bool "AltiVec Support"
308	310	depends on PPC_BOOK3S_32 \|\| PPC_BOOK3S_64 \|\| (PPC_E500MC && PPC64)
	311	+ select PPC_FPU
309	312	help
310	313	This option enables kernel support for the Altivec extensions to the
311	314	PowerPC processor. The kernel currently supports saving and restoring
@@ -363,8 +366,6 @@ config PPC_RADIX_MMU · 51 unmodified lines
363	366	bool "Radix MMU Support"
364	367	depends on PPC_BOOK3S_64
365	368	select ARCH_HAS_GIGANTIC_PAGE
366		− select PPC_HAVE_KUEP
367		− select PPC_HAVE_KUAP
368	369	default y
369	370	help
370	371	Enable support for the Power ISA 3.0 Radix style MMU. Currently this

arch/powerpc/platforms/cell/iommu.c+2−1

@@ -486,7 +486,8 @@ cell_iommu_setup_window(struct cbe_iommu iommu, struct device_node np, · 485 unmodified lines
486	486	window->table.it_size = size >> window->table.it_page_shift;
487	487	window->table.it_ops = &cell_iommu_ops;
488	488
489		− iommu_init_table(&window->table, iommu->nid, 0, 0);
	489	+ if (!iommu_init_table(&window->table, iommu->nid, 0, 0))
	490	+ panic("Failed to initialize iommu table");
490	491
491	492	pr_debug("\tioid %d\n", window->ioid);
492	493	pr_debug("\tblocksize %ld\n", window->table.it_blocksize);

arch/powerpc/platforms/cell/spu_callbacks.c+1−1

@@ -35,9 +35,9 @@ · 34 unmodified lines
35	35	*/
36	36
37	37	static void *spu_syscall_table[] = {
	38	+#define __SYSCALL_WITH_COMPAT(nr, entry, compat) __SYSCALL(nr, entry)
38	39	#define __SYSCALL(nr, entry) [nr] = entry,
39	40	#include <asm/syscall_table_spu.h>
40		−#undef __SYSCALL
41	41	};
42	42
43	43	long spu_sys_callback(struct spu_syscall_block *s)

arch/powerpc/platforms/chrp/pci.c+1−2

@@ -131,8 +131,7 @@ static struct pci_ops rtas_pci_ops = · 130 unmodified lines
131	131
132	132	volatile struct Hydra __iomem *Hydra = NULL;
133	133
134		−int __init
135		−hydra_init(void)
	134	+static int __init hydra_init(void)
136	135	{
137	136	struct device_node *np;
138	137	struct resource r;

arch/powerpc/platforms/embedded6xx/Kconfig+0−5

@@ -71,11 +71,6 @@ config MPC10X_BRIDGE · 70 unmodified lines
71	71	bool
72	72	select PPC_INDIRECT_PCI
73	73
74		−config MV64X60
75		− bool
76		− select PPC_INDIRECT_PCI
77		− select CHECK_CACHE_COHERENCY
78		−
79	74	config GAMECUBE_COMMON
80	75	bool
81	76

arch/powerpc/platforms/maple/pci.c+1−1

@@ -34,7 +34,7 @@ static struct pci_controller u3_agp, u3_ht, *u4_pcie; · 33 unmodified lines
34	34
35	35	static int __init fixup_one_level_bus_range(struct device_node *node, int higher)
36	36	{
37		− for (; node != 0;node = node->sibling) {
	37	+ for (; node; node = node->sibling) {
38	38	const int *bus_range;
39	39	const unsigned int *class_code;
40	40	int len;

arch/powerpc/platforms/pasemi/iommu.c+3−1

@@ -146,7 +146,9 @@ static void iommu_table_iobmap_setup(void) · 145 unmodified lines
146	146	*/
147	147	iommu_table_iobmap.it_blocksize = 4;
148	148	iommu_table_iobmap.it_ops = &iommu_table_iobmap_ops;
149		− iommu_init_table(&iommu_table_iobmap, 0, 0, 0);
	149	+ if (!iommu_init_table(&iommu_table_iobmap, 0, 0, 0))
	150	+ panic("Failed to initialize iommu table");
	151	+
150	152	pr_debug(" <- %s\n", __func__);
151	153	}
152	154

arch/powerpc/platforms/powernv/memtrace.c+17−1

@@ -46,10 +46,26 @@ static ssize_t memtrace_read(struct file filp, char __user ubuf, · 45 unmodified lines
46	46	return simple_read_from_buffer(ubuf, count, ppos, ent->mem, ent->size);
47	47	}
48	48
	49	+static int memtrace_mmap(struct file filp, struct vm_area_struct vma)
	50	+{
	51	+ struct memtrace_entry *ent = filp->private_data;
	52	+
	53	+ if (ent->size < vma->vm_end - vma->vm_start)
	54	+ return -EINVAL;
	55	+
	56	+ if (vma->vm_pgoff << PAGE_SHIFT >= ent->size)
	57	+ return -EINVAL;
	58	+
	59	+ vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
	60	+ return remap_pfn_range(vma, vma->vm_start, PHYS_PFN(ent->start) + vma->vm_pgoff,
	61	+ vma->vm_end - vma->vm_start, vma->vm_page_prot);
	62	+}
	63	+
49	64	static const struct file_operations memtrace_fops = {
50	65	.llseek = default_llseek,
51	66	.read = memtrace_read,
52	67	.open = simple_open,
	68	+ .mmap = memtrace_mmap,
53	69	};
54	70
55	71	#define FLUSH_CHUNK_SIZE SZ_1G
@@ -187,7 +203,7 @@ static int memtrace_init_debugfs(void) · 131 unmodified lines
187	203	dir = debugfs_create_dir(ent->name, memtrace_debugfs_dir);
188	204
189	205	ent->dir = dir;
190		− debugfs_create_file("trace", 0400, dir, ent, &memtrace_fops);
	206	+ debugfs_create_file_unsafe("trace", 0600, dir, ent, &memtrace_fops);
191	207	debugfs_create_x64("start", 0400, dir, &ent->start);
192	208	debugfs_create_x64("size", 0400, dir, &ent->size);
193	209	}

arch/powerpc/platforms/powernv/opal-core.c+1−1

@@ -71,7 +71,7 @@ static LIST_HEAD(opalcore_list); · 70 unmodified lines
71	71	static struct opalcore_config *oc_conf;
72	72	static const struct opal_mpipl_fadump *opalc_metadata;
73	73	static const struct opal_mpipl_fadump *opalc_cpu_metadata;
74		−struct kobject *mpipl_kobj;
	74	+static struct kobject *mpipl_kobj;
75	75
76	76	/*
77	77	* Set crashing CPU's signal to SIGUSR1. if the kernel is triggered

arch/powerpc/platforms/powernv/opal-prd.c+1−4

@@ -105,7 +105,6 @@ static int opal_prd_mmap(struct file file, struct vm_area_struct vma) · 104 unmodified lines
105	105	{
106	106	size_t addr, size;
107	107	pgprot_t page_prot;
108		− int rc;
109	108
110	109	pr_devel("opal_prd_mmap(0x%016lx, 0x%016lx, 0x%lx, 0x%lx)\n",
111	110	vma->vm_start, vma->vm_end, vma->vm_pgoff,
@@ -121,10 +120,8 @@ static int opal_prd_mmap(struct file file, struct vm_area_struct vma) · 9 unmodified lines
121	120	page_prot = phys_mem_access_prot(file, vma->vm_pgoff,
122	121	size, vma->vm_page_prot);
123	122
124		− rc = remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, size,
	123	+ return remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, size,
125	124	page_prot);
126		−
127		− return rc;
128	125	}
129	126
130	127	static bool opal_msg_queue_empty(void)

arch/powerpc/platforms/powernv/pci-ioda.c+8−7

@@ -1762,7 +1762,8 @@ found: · 1761 unmodified lines
1762	1762	tbl->it_ops = &pnv_ioda1_iommu_ops;
1763	1763	pe->table_group.tce32_start = tbl->it_offset << tbl->it_page_shift;
1764	1764	pe->table_group.tce32_size = tbl->it_size << tbl->it_page_shift;
1765		− iommu_init_table(tbl, phb->hose->node, 0, 0);
	1765	+ if (!iommu_init_table(tbl, phb->hose->node, 0, 0))
	1766	+ panic("Failed to initialize iommu table");
1766	1767
1767	1768	pe->dma_setup_done = true;
1768	1769	return;
@@ -1930,16 +1931,16 @@ static long pnv_pci_ioda2_setup_default_config(struct pnv_ioda_pe *pe) · 161 unmodified lines
1930	1931	res_start = pe->phb->ioda.m32_pci_base >> tbl->it_page_shift;
1931	1932	res_end = min(window_size, SZ_4G) >> tbl->it_page_shift;
1932	1933	}
1933		− iommu_init_table(tbl, pe->phb->hose->node, res_start, res_end);
1934	1934
1935		− rc = pnv_pci_ioda2_set_window(&pe->table_group, 0, tbl);
	1935	+ if (iommu_init_table(tbl, pe->phb->hose->node, res_start, res_end))
	1936	+ rc = pnv_pci_ioda2_set_window(&pe->table_group, 0, tbl);
	1937	+ else
	1938	+ rc = -ENOMEM;
1936	1939	if (rc) {
1937		− pe_err(pe, "Failed to configure 32-bit TCE table, err %ld\n",
1938		− rc);
	1940	+ pe_err(pe, "Failed to configure 32-bit TCE table, err %ld\n", rc);
1939	1941	iommu_tce_table_put(tbl);
1940		− return rc;
	1942	+ tbl = NULL; /* This clears iommu_table_base below */
1941	1943	}
1942		−
1943	1944	if (!pnv_iommu_bypass_disabled)
1944	1945	pnv_pci_ioda2_set_bypass(pe, true);
1945	1946

arch/powerpc/platforms/powernv/setup.c+1−1

@@ -157,7 +157,7 @@ static void __init pnv_check_guarded_cores(void) · 156 unmodified lines
157	157	for_each_node_by_type(dn, "cpu") {
158	158	if (of_property_match_string(dn, "status", "bad") >= 0)
159	159	bad_count++;
160		− };
	160	+ }
161	161
162	162	if (bad_count) {
163	163	printk(" _ _______________\n");

arch/powerpc/platforms/pseries/dlpar.c+14−0

@@ -329,6 +329,20 @@ int dlpar_release_drc(u32 drc_index) · 328 unmodified lines
329	329	return 0;
330	330	}
331	331
	332	+int dlpar_unisolate_drc(u32 drc_index)
	333	+{
	334	+ int dr_status, rc;
	335	+
	336	+ rc = rtas_call(rtas_token("get-sensor-state"), 2, 2, &dr_status,
	337	+ DR_ENTITY_SENSE, drc_index);
	338	+ if (rc \|\| dr_status != DR_ENTITY_PRESENT)
	339	+ return -1;
	340	+
	341	+ rtas_set_indicator(ISOLATION_STATE, drc_index, UNISOLATE);
	342	+
	343	+ return 0;
	344	+}
	345	+
332	346	int handle_dlpar_errorlog(struct pseries_hp_errorlog *hp_elog)
333	347	{
334	348	int rc;

arch/powerpc/platforms/pseries/hotplug-cpu.c+23−4

@@ -47,9 +47,6 @@ static void rtas_stop_self(void) · 46 unmodified lines
47	47
48	48	BUG_ON(rtas_stop_self_token == RTAS_UNKNOWN_SERVICE);
49	49
50		− printk("cpu %u (hwid %u) Ready to die...\n",
51		− smp_processor_id(), hard_smp_processor_id());
52		−
53	50	rtas_call_unlocked(&args, rtas_stop_self_token, 0, 1, NULL);
54	51
55	52	panic("Alas, I survived.\n");
@@ -271,6 +268,19 @@ static int dlpar_offline_cpu(struct device_node *dn) · 215 unmodified lines
271	268	if (!cpu_online(cpu))
272	269	break;
273	270
	271	+ /*
	272	+ * device_offline() will return -EBUSY (via cpu_down()) if there
	273	+ * is only one CPU left. Check it here to fail earlier and with a
	274	+ * more informative error message, while also retaining the
	275	+ * cpu_add_remove_lock to be sure that no CPUs are being
	276	+ * online/offlined during this check.
	277	+ */
	278	+ if (num_online_cpus() == 1) {
	279	+ pr_warn("Unable to remove last online CPU %pOFn\n", dn);
	280	+ rc = -EBUSY;
	281	+ goto out_unlock;
	282	+ }
	283	+
274	284	cpu_maps_update_done();
275	285	rc = device_offline(get_cpu_device(cpu));
276	286	if (rc)
@@ -283,6 +293,7 @@ static int dlpar_offline_cpu(struct device_node *dn) · 6 unmodified lines
283	293	thread);
284	294	}
285	295	}
	296	+out_unlock:
286	297	cpu_maps_update_done();
287	298
288	299	out:
@@ -802,8 +813,16 @@ int dlpar_cpu(struct pseries_hp_errorlog *hp_elog) · 513 unmodified lines
802	813	case PSERIES_HP_ELOG_ACTION_REMOVE:
803	814	if (hp_elog->id_type == PSERIES_HP_ELOG_ID_DRC_COUNT)
804	815	rc = dlpar_cpu_remove_by_count(count);
805		− else if (hp_elog->id_type == PSERIES_HP_ELOG_ID_DRC_INDEX)
	816	+ else if (hp_elog->id_type == PSERIES_HP_ELOG_ID_DRC_INDEX) {
806	817	rc = dlpar_cpu_remove_by_index(drc_index);
	818	+ /*
	819	+ * Setting the isolation state of an UNISOLATED/CONFIGURED
	820	+ * device to UNISOLATE is a no-op, but the hypervisor can
	821	+ * use it as a hint that the CPU removal failed.
	822	+ */
	823	+ if (rc)
	824	+ dlpar_unisolate_drc(drc_index);
	825	+ }
807	826	else
808	827	rc = -EINVAL;
809	828	break;

arch/powerpc/platforms/pseries/hvCall_inst.c+1−1

@@ -26,7 +26,7 @@ struct hcall_stats { · 25 unmodified lines
26	26	};
27	27	#define HCALL_STAT_ARRAY_SIZE ((MAX_HCALL_OPCODE >> 2) + 1)
28	28
29		−DEFINE_PER_CPU(struct hcall_stats[HCALL_STAT_ARRAY_SIZE], hcall_stats);
	29	+static DEFINE_PER_CPU(struct hcall_stats[HCALL_STAT_ARRAY_SIZE], hcall_stats);
30	30
31	31	/*
32	32	* Routines for displaying the statistics in debugfs

arch/powerpc/platforms/pseries/iommu.c+38−11

@@ -638,7 +638,8 @@ static void pci_dma_bus_setup_pSeries(struct pci_bus *bus) · 637 unmodified lines
638	638
639	639	iommu_table_setparms(pci->phb, dn, tbl);
640	640	tbl->it_ops = &iommu_table_pseries_ops;
641		− iommu_init_table(tbl, pci->phb->node, 0, 0);
	641	+ if (!iommu_init_table(tbl, pci->phb->node, 0, 0))
	642	+ panic("Failed to initialize iommu table");
642	643
643	644	/* Divide the rest (1.75GB) among the children */
644	645	pci->phb->dma_window_size = 0x80000000ul;
@@ -720,7 +721,8 @@ static void pci_dma_bus_setup_pSeriesLP(struct pci_bus *bus) · 75 unmodified lines
720	721	iommu_table_setparms_lpar(ppci->phb, pdn, tbl,
721	722	ppci->table_group, dma_window);
722	723	tbl->it_ops = &iommu_table_lpar_multi_ops;
723		− iommu_init_table(tbl, ppci->phb->node, 0, 0);
	724	+ if (!iommu_init_table(tbl, ppci->phb->node, 0, 0))
	725	+ panic("Failed to initialize iommu table");
724	726	iommu_register_group(ppci->table_group,
725	727	pci_domain_nr(bus), 0);
726	728	pr_debug(" created table: %p\n", ppci->table_group);
@@ -749,7 +751,9 @@ static void pci_dma_dev_setup_pSeries(struct pci_dev *dev) · 22 unmodified lines
749	751	tbl = PCI_DN(dn)->table_group->tables[0];
750	752	iommu_table_setparms(phb, dn, tbl);
751	753	tbl->it_ops = &iommu_table_pseries_ops;
752		− iommu_init_table(tbl, phb->node, 0, 0);
	754	+ if (!iommu_init_table(tbl, phb->node, 0, 0))
	755	+ panic("Failed to initialize iommu table");
	756	+
753	757	set_iommu_table_base(&dev->dev, tbl);
754	758	return;
755	759	}
@@ -1099,6 +1103,33 @@ static void reset_dma_window(struct pci_dev dev, struct device_node par_dn) · 343 unmodified lines
1099	1103	ret);
1100	1104	}
1101	1105
	1106	+/* Return largest page shift based on "IO Page Sizes" output of ibm,query-pe-dma-window. */
	1107	+static int iommu_get_page_shift(u32 query_page_size)
	1108	+{
	1109	+ /* Supported IO page-sizes according to LoPAR */
	1110	+ const int shift[] = {
	1111	+ __builtin_ctzll(SZ_4K), __builtin_ctzll(SZ_64K), __builtin_ctzll(SZ_16M),
	1112	+ __builtin_ctzll(SZ_32M), __builtin_ctzll(SZ_64M), __builtin_ctzll(SZ_128M),
	1113	+ __builtin_ctzll(SZ_256M), __builtin_ctzll(SZ_16G)
	1114	+ };
	1115	+
	1116	+ int i = ARRAY_SIZE(shift) - 1;
	1117	+
	1118	+ /*
	1119	+ * On LoPAR, ibm,query-pe-dma-window outputs "IO Page Sizes" using a bit field:
	1120	+ * - bit 31 means 4k pages are supported,
	1121	+ * - bit 30 means 64k pages are supported, and so on.
	1122	+ * Larger pagesizes map more memory with the same amount of TCEs, so start probing them.
	1123	+ */
	1124	+ for (; i >= 0 ; i--) {
	1125	+ if (query_page_size & (1 << i))
	1126	+ return shift[i];
	1127	+ }
	1128	+
	1129	+ /* No valid page size found. */
	1130	+ return 0;
	1131	+}
	1132	+
1102	1133	/*
1103	1134	* If the PE supports dynamic dma windows, and there is space for a table
1104	1135	* that can map all pages in a linear offset, then setup such a table,
@@ -1206,13 +1237,9 @@ static u64 enable_ddw(struct pci_dev dev, struct device_node pdn) · 101 unmodified lines
1206	1237	goto out_failed;
1207	1238	}
1208	1239	}
1209		− if (query.page_size & 4) {
1210		− page_shift = 24; /* 16MB */
1211		− } else if (query.page_size & 2) {
1212		− page_shift = 16; /* 64kB */
1213		− } else if (query.page_size & 1) {
1214		− page_shift = 12; /* 4kB */
1215		− } else {
	1240	+
	1241	+ page_shift = iommu_get_page_shift(query.page_size);
	1242	+ if (!page_shift) {
1216	1243	dev_dbg(&dev->dev, "no supported direct page size in mask %x",
1217	1244	query.page_size);
1218	1245	goto out_failed;
@@ -1229,7 +1256,7 @@ static u64 enable_ddw(struct pci_dev dev, struct device_node pdn) · 10 unmodified lines
1229	1256	if (pmem_present) {
1230	1257	if (query.largest_available_block >=
1231	1258	(1ULL << (MAX_PHYSMEM_BITS - page_shift)))
1232		− len = MAX_PHYSMEM_BITS - page_shift;
	1259	+ len = MAX_PHYSMEM_BITS;
1233	1260	else
1234	1261	dev_info(&dev->dev, "Skipping ibm,pmemory");
1235	1262	}

arch/powerpc/platforms/pseries/lpar.c+4−2

@@ -977,11 +977,13 @@ static void pSeries_lpar_hpte_updateboltedpp(unsigned long newpp, · 976 unmodified lines
977	977	slot = pSeries_lpar_hpte_find(vpn, psize, ssize);
978	978	BUG_ON(slot == -1);
979	979
980		− flags = newpp & 7;
	980	+ flags = newpp & (HPTE_R_PP \| HPTE_R_N);
981	981	if (mmu_has_feature(MMU_FTR_KERNEL_RO))
982	982	/* Move pp0 into bit 8 (IBM 55) */
983	983	flags \|= (newpp & HPTE_R_PP0) >> 55;
984	984
	985	+ flags \|= ((newpp & HPTE_R_KEY_HI) >> 48) \| (newpp & HPTE_R_KEY_LO);
	986	+
985	987	lpar_rc = plpar_pte_protect(flags, slot, 0);
986	988
987	989	BUG_ON(lpar_rc != H_SUCCESS);
@@ -1630,7 +1632,7 @@ static int pseries_lpar_resize_hpt(unsigned long shift) · 642 unmodified lines
1630	1632	}
1631	1633	msleep(delay);
1632	1634	rc = plpar_resize_hpt_prepare(0, shift);
1633		− };
	1635	+ }
1634	1636
1635	1637	switch (rc) {
1636	1638	case H_SUCCESS:

arch/powerpc/platforms/pseries/lparcfg.c+2−0

@@ -537,6 +537,8 @@ static int pseries_lparcfg_data(struct seq_file m, void v) · 536 unmodified lines
537	537	parse_em_data(m);
538	538	maxmem_data(m);
539	539
	540	+ seq_printf(m, "security_flavor=%u\n", pseries_security_flavor);
	541	+
540	542	return 0;
541	543	}
542	544

arch/powerpc/platforms/pseries/papr_scm.c+48−0

@@ -93,6 +93,7 @@ struct papr_scm_priv { · 92 unmodified lines
93	93	uint64_t block_size;
94	94	int metadata_size;
95	95	bool is_volatile;
	96	+ bool hcall_flush_required;
96	97
97	98	uint64_t bound_addr;
98	99
@@ -117,6 +118,38 @@ struct papr_scm_priv { · 18 unmodified lines
117	118	size_t stat_buffer_len;
118	119	};
119	120
	121	+static int papr_scm_pmem_flush(struct nd_region *nd_region,
	122	+ struct bio *bio __maybe_unused)
	123	+{
	124	+ struct papr_scm_priv *p = nd_region_provider_data(nd_region);
	125	+ unsigned long ret_buf[PLPAR_HCALL_BUFSIZE], token = 0;
	126	+ long rc;
	127	+
	128	+ dev_dbg(&p->pdev->dev, "flush drc 0x%x", p->drc_index);
	129	+
	130	+ do {
	131	+ rc = plpar_hcall(H_SCM_FLUSH, ret_buf, p->drc_index, token);
	132	+ token = ret_buf[0];
	133	+
	134	+ /* Check if we are stalled for some time */
	135	+ if (H_IS_LONG_BUSY(rc)) {
	136	+ msleep(get_longbusy_msecs(rc));
	137	+ rc = H_BUSY;
	138	+ } else if (rc == H_BUSY) {
	139	+ cond_resched();
	140	+ }
	141	+ } while (rc == H_BUSY);
	142	+
	143	+ if (rc) {
	144	+ dev_err(&p->pdev->dev, "flush error: %ld", rc);
	145	+ rc = -EIO;
	146	+ } else {
	147	+ dev_dbg(&p->pdev->dev, "flush drc 0x%x complete", p->drc_index);
	148	+ }
	149	+
	150	+ return rc;
	151	+}
	152	+
120	153	static LIST_HEAD(papr_nd_regions);
121	154	static DEFINE_MUTEX(papr_ndr_lock);
122	155
@@ -914,6 +947,15 @@ static int papr_scm_nvdimm_init(struct papr_scm_priv *p) · 791 unmodified lines
914	947	dimm_flags = 0;
915	948	set_bit(NDD_LABELING, &dimm_flags);
916	949
	950	+ /*
	951	+ * Check if the nvdimm is unarmed. No locking needed as we are still
	952	+ * initializing. Ignore error encountered if any.
	953	+ */
	954	+ __drc_pmem_query_health(p);
	955	+
	956	+ if (p->health_bitmap & PAPR_PMEM_UNARMED_MASK)
	957	+ set_bit(NDD_UNARMED, &dimm_flags);
	958	+
917	959	p->nvdimm = nvdimm_create(p->bus, p, papr_nd_attr_groups,
918	960	dimm_flags, PAPR_SCM_DIMM_CMD_MASK, 0, NULL);
919	961	if (!p->nvdimm) {
@@ -943,6 +985,11 @@ static int papr_scm_nvdimm_init(struct papr_scm_priv *p) · 23 unmodified lines
943	985	ndr_desc.num_mappings = 1;
944	986	ndr_desc.nd_set = &p->nd_set;
945	987
	988	+ if (p->hcall_flush_required) {
	989	+ set_bit(ND_REGION_ASYNC, &ndr_desc.flags);
	990	+ ndr_desc.flush = papr_scm_pmem_flush;
	991	+ }
	992	+
946	993	if (p->is_volatile)
947	994	p->region = nvdimm_volatile_region_create(p->bus, &ndr_desc);
948	995	else {
@@ -1088,6 +1135,7 @@ static int papr_scm_probe(struct platform_device *pdev) · 139 unmodified lines
1088	1135	p->block_size = block_size;
1089	1136	p->blocks = blocks;
1090	1137	p->is_volatile = !of_property_read_bool(dn, "ibm,cache-flush-required");
	1138	+ p->hcall_flush_required = of_property_read_bool(dn, "ibm,hcall-flush-required");
1091	1139
1092	1140	/* We just need to ensure that set cookies are unique across */
1093	1141	uuid_parse(uuid_str, (uuid_t *) uuid);

arch/powerpc/platforms/pseries/pci_dlpar.c+3−1

@@ -50,6 +50,7 @@ EXPORT_SYMBOL_GPL(init_phb_dynamic); · 49 unmodified lines
50	50	int remove_phb_dynamic(struct pci_controller *phb)
51	51	{
52	52	struct pci_bus *b = phb->bus;
	53	+ struct pci_host_bridge *host_bridge = to_pci_host_bridge(b->bridge);
53	54	struct resource *res;
54	55	int rc, i;
55	56
@@ -76,7 +77,8 @@ int remove_phb_dynamic(struct pci_controller *phb) · 20 unmodified lines
76	77	/* Remove the PCI bus and unregister the bridge device from sysfs */
77	78	phb->bus = NULL;
78	79	pci_remove_bus(b);
79		− device_unregister(b->bridge);
	80	+ host_bridge->bus = NULL;
	81	+ device_unregister(&host_bridge->dev);
80	82
81	83	/* Now release the IO resource */
82	84	if (res->flags & IORESOURCE_IO)

arch/powerpc/platforms/pseries/pmem.c+1−1

@@ -139,7 +139,7 @@ int dlpar_hp_pmem(struct pseries_hp_errorlog *hp_elog) · 138 unmodified lines
139	139	return rc;
140	140	}
141	141
142		−const struct of_device_id drc_pmem_match[] = {
	142	+static const struct of_device_id drc_pmem_match[] = {
143	143	{ .type = "ibm,persistent-memory", },
144	144	{}
145	145	};

arch/powerpc/platforms/pseries/pseries.h+2−3

@@ -43,9 +43,6 @@ extern void pSeries_final_fixup(void); · 42 unmodified lines
43	43	/* Poweron flag used for enabling auto ups restart */
44	44	extern unsigned long rtas_poweron_auto;
45	45
46		−/* Provided by HVC VIO */
47		−extern void hvc_vio_init_early(void);
48		−
49	46	/* Dynamic logical Partitioning/Mobility */
50	47	extern void dlpar_free_cc_nodes(struct device_node *);
51	48	extern void dlpar_free_cc_property(struct property *);
@@ -55,6 +52,7 @@ extern int dlpar_attach_node(struct device_node , struct device_node ); · 3 unmodified lines
55	52	extern int dlpar_detach_node(struct device_node *);
56	53	extern int dlpar_acquire_drc(u32 drc_index);
57	54	extern int dlpar_release_drc(u32 drc_index);
	55	+extern int dlpar_unisolate_drc(u32 drc_index);
58	56
59	57	void queue_hotplug_event(struct pseries_hp_errorlog *hp_errlog);
60	58	int handle_dlpar_errorlog(struct pseries_hp_errorlog *hp_errlog);
@@ -111,6 +109,7 @@ static inline unsigned long cmo_get_page_size(void) · 50 unmodified lines
111	109
112	110	int dlpar_workqueue_init(void);
113	111
	112	+extern u32 pseries_security_flavor;
114	113	void pseries_setup_security_mitigations(void);
115	114	void pseries_lpar_read_hblkrm_characteristics(void);
116	115

arch/powerpc/platforms/pseries/ras.c+1−1

@@ -699,7 +699,7 @@ static int mce_handle_err_virtmode(struct pt_regs *regs, · 698 unmodified lines
699	699	mce_err.error_type = MCE_ERROR_TYPE_DCACHE;
700	700	break;
701	701	case MC_ERROR_TYPE_I_CACHE:
702		− mce_err.error_type = MCE_ERROR_TYPE_DCACHE;
	702	+ mce_err.error_type = MCE_ERROR_TYPE_ICACHE;
703	703	break;
704	704	case MC_ERROR_TYPE_UNKNOWN:
705	705	default:

arch/powerpc/platforms/pseries/rtas-fadump.c+1−1

@@ -247,7 +247,7 @@ static inline int rtas_fadump_gpr_index(u64 id) · 246 unmodified lines
247	247	return i;
248	248	}
249	249
250		−void rtas_fadump_set_regval(struct pt_regs *regs, u64 reg_id, u64 reg_val)
	250	+static void rtas_fadump_set_regval(struct pt_regs *regs, u64 reg_id, u64 reg_val)
251	251	{
252	252	int i;
253	253

arch/powerpc/platforms/pseries/setup.c+8−0

@@ -71,6 +71,7 @@ · 70 unmodified lines
71	71	#include <asm/swiotlb.h>
72	72	#include <asm/svm.h>
73	73	#include <asm/dtl.h>
	74	+#include <asm/hvconsole.h>
74	75
75	76	#include "pseries.h"
76	77	#include "../../../../drivers/pci/pci.h"
@@ -85,6 +86,7 @@ EXPORT_SYMBOL(CMO_PageSize); · 8 unmodified lines
85	86
86	87	int fwnmi_active; /* TRUE if an FWNMI handler is present */
87	88	int ibm_nmi_interlock_token;
	89	+u32 pseries_security_flavor;
88	90
89	91	static void pSeries_show_cpuinfo(struct seq_file *m)
90	92	{
@@ -534,9 +536,15 @@ static void init_cpu_char_feature_flags(struct h_cpu_char_result *result) · 443 unmodified lines
534	536	/*
535	537	* The features below are enabled by default, so we instead look to see
536	538	* if firmware has disabled them, and clear them if so.
	539	+ * H_CPU_BEHAV_FAVOUR_SECURITY_H could be set only if
	540	+ * H_CPU_BEHAV_FAVOUR_SECURITY is.
537	541	*/
538	542	if (!(result->behaviour & H_CPU_BEHAV_FAVOUR_SECURITY))
539	543	security_ftr_clear(SEC_FTR_FAVOUR_SECURITY);
	544	+ else if (result->behaviour & H_CPU_BEHAV_FAVOUR_SECURITY_H)
	545	+ pseries_security_flavor = 1;
	546	+ else
	547	+ pseries_security_flavor = 2;
540	548
541	549	if (!(result->behaviour & H_CPU_BEHAV_L1D_FLUSH_PR))
542	550	security_ftr_clear(SEC_FTR_L1D_FLUSH_PR);

arch/powerpc/platforms/pseries/vio.c+20−0

@@ -22,6 +22,7 @@ · 21 unmodified lines
22	22	#include <linux/mm.h>
23	23	#include <linux/dma-map-ops.h>
24	24	#include <linux/kobject.h>
	25	+#include <linux/kexec.h>
25	26
26	27	#include <asm/iommu.h>
27	28	#include <asm/dma.h>
@@ -1278,6 +1279,20 @@ static int vio_bus_remove(struct device *dev) · 1250 unmodified lines
1278	1279	return 0;
1279	1280	}
1280	1281
	1282	+static void vio_bus_shutdown(struct device *dev)
	1283	+{
	1284	+ struct vio_dev *viodev = to_vio_dev(dev);
	1285	+ struct vio_driver *viodrv;
	1286	+
	1287	+ if (dev->driver) {
	1288	+ viodrv = to_vio_driver(dev->driver);
	1289	+ if (viodrv->shutdown)
	1290	+ viodrv->shutdown(viodev);
	1291	+ else if (kexec_in_progress)
	1292	+ vio_bus_remove(dev);
	1293	+ }
	1294	+}
	1295	+
1281	1296	/**
1282	1297	* vio_register_driver: - Register a new vio driver
1283	1298	* @viodrv: The vio_driver structure to be registered.
@@ -1285,6 +1300,10 @@ static int vio_bus_remove(struct device *dev) · 1 unmodified line
1285	1300	int __vio_register_driver(struct vio_driver viodrv, struct module owner,
1286	1301	const char *mod_name)
1287	1302	{
	1303	+ // vio_bus_type is only initialised for pseries
	1304	+ if (!machine_is(pseries))
	1305	+ return -ENODEV;
	1306	+
1288	1307	pr_debug("%s: driver %s registering\n", __func__, viodrv->name);
1289	1308
1290	1309	/* fill in 'struct driver' fields */
@@ -1613,6 +1632,7 @@ struct bus_type vio_bus_type = { · 322 unmodified lines
1613	1632	.match = vio_bus_match,
1614	1633	.probe = vio_bus_probe,
1615	1634	.remove = vio_bus_remove,
	1635	+ .shutdown = vio_bus_shutdown,
1616	1636	};
1617	1637
1618	1638	/**

arch/powerpc/purgatory/trampoline_64.S+0−1

@@ -12,7 +12,6 @@ · 11 unmodified lines
12	12	#include <asm/asm-compat.h>
13	13	#include <asm/crashdump-ppc64.h>
14	14
15		− .machine ppc64
16	15	.balign 256
17	16	.globl purgatory_start
18	17	purgatory_start:

arch/powerpc/sysdev/dart_iommu.c+2−1

@@ -344,7 +344,8 @@ static void iommu_table_dart_setup(void) · 343 unmodified lines
344	344	iommu_table_dart.it_index = 0;
345	345	iommu_table_dart.it_blocksize = 1;
346	346	iommu_table_dart.it_ops = &iommu_dart_ops;
347		− iommu_init_table(&iommu_table_dart, -1, 0, 0);
	347	+ if (!iommu_init_table(&iommu_table_dart, -1, 0, 0))
	348	+ panic("Failed to initialize iommu table");
348	349
349	350	/* Reserve the last page of the DART to avoid possible prefetch
350	351	* past the DART mapped area

arch/powerpc/sysdev/fsl_pci.c+1−1

@@ -455,7 +455,7 @@ static void setup_pci_atmu(struct pci_controller *hose) · 454 unmodified lines
455	455	}
456	456	}
457	457
458		−static void __init setup_pci_cmd(struct pci_controller *hose)
	458	+static void setup_pci_cmd(struct pci_controller *hose)
459	459	{
460	460	u16 cmd;
461	461	int cap_x;

arch/powerpc/sysdev/xive/common.c+138−75

@@ -63,8 +63,19 @@ static const struct xive_ops *xive_ops; · 62 unmodified lines
63	63	static struct irq_domain *xive_irq_domain;
64	64
65	65	#ifdef CONFIG_SMP
66		−/* The IPIs all use the same logical irq number */
67		−static u32 xive_ipi_irq;
	66	+/* The IPIs use the same logical irq number when on the same chip */
	67	+static struct xive_ipi_desc {
	68	+ unsigned int irq;
	69	+ char name[16];
	70	+} *xive_ipis;
	71	+
	72	+/*
	73	+ * Use early_cpu_to_node() for hot-plugged CPUs
	74	+ */
	75	+static unsigned int xive_ipi_cpu_to_irq(unsigned int cpu)
	76	+{
	77	+ return xive_ipis[early_cpu_to_node(cpu)].irq;
	78	+}
68	79	#endif
69	80
70	81	/* Xive state for each CPU */
@@ -253,17 +264,20 @@ notrace void xmon_xive_do_dump(int cpu) · 182 unmodified lines
253	264	xmon_printf("\n");
254	265	}
255	266
	267	+static struct irq_data *xive_get_irq_data(u32 hw_irq)
	268	+{
	269	+ unsigned int irq = irq_find_mapping(xive_irq_domain, hw_irq);
	270	+
	271	+ return irq ? irq_get_irq_data(irq) : NULL;
	272	+}
	273	+
256	274	int xmon_xive_get_irq_config(u32 hw_irq, struct irq_data *d)
257	275	{
258		− struct irq_chip *chip = irq_data_get_irq_chip(d);
259	276	int rc;
260	277	u32 target;
261	278	u8 prio;
262	279	u32 lirq;
263	280
264		− if (!is_xive_irq(chip))
265		− return -EINVAL;
266		−
267	281	rc = xive_ops->get_irq_config(hw_irq, &target, &prio, &lirq);
268	282	if (rc) {
269	283	xmon_printf("IRQ 0x%08x : no config rc=%d\n", hw_irq, rc);
@@ -273,6 +287,9 @@ int xmon_xive_get_irq_config(u32 hw_irq, struct irq_data *d) · 3 unmodified lines
273	287	xmon_printf("IRQ 0x%08x : target=0x%x prio=%02x lirq=0x%x ",
274	288	hw_irq, target, prio, lirq);
275	289
	290	+ if (!d)
	291	+ d = xive_get_irq_data(hw_irq);
	292	+
276	293	if (d) {
277	294	struct xive_irq_data *xd = irq_data_get_irq_handler_data(d);
278	295	u64 val = xive_esb_read(xd, XIVE_ESB_GET);
@@ -289,6 +306,20 @@ int xmon_xive_get_irq_config(u32 hw_irq, struct irq_data *d) · 10 unmodified lines
289	306	return 0;
290	307	}
291	308
	309	+void xmon_xive_get_irq_all(void)
	310	+{
	311	+ unsigned int i;
	312	+ struct irq_desc *desc;
	313	+
	314	+ for_each_irq_desc(i, desc) {
	315	+ struct irq_data *d = irq_desc_get_irq_data(desc);
	316	+ unsigned int hwirq = (unsigned int)irqd_to_hwirq(d);
	317	+
	318	+ if (d->domain == xive_irq_domain)
	319	+ xmon_xive_get_irq_config(hwirq, d);
	320	+ }
	321	+}
	322	+
292	323	#endif /* CONFIG_XMON */
293	324
294	325	static unsigned int xive_get_irq(void)
@@ -1067,28 +1098,94 @@ static struct irq_chip xive_ipi_chip = { · 772 unmodified lines
1067	1098	.irq_unmask = xive_ipi_do_nothing,
1068	1099	};
1069	1100
1070		−static void __init xive_request_ipi(void)
	1101	+/*
	1102	+ * IPIs are marked per-cpu. We use separate HW interrupts under the
	1103	+ * hood but associated with the same "linux" interrupt
	1104	+ */
	1105	+struct xive_ipi_alloc_info {
	1106	+ irq_hw_number_t hwirq;
	1107	+};
	1108	+
	1109	+static int xive_ipi_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
	1110	+ unsigned int nr_irqs, void *arg)
1071	1111	{
1072		− unsigned int virq;
	1112	+ struct xive_ipi_alloc_info *info = arg;
	1113	+ int i;
1073	1114
1074		− /*
1075		− * Initialization failed, move on, we might manage to
1076		− * reach the point where we display our errors before
1077		− * the system falls appart
1078		− */
1079		− if (!xive_irq_domain)
1080		− return;
	1115	+ for (i = 0; i < nr_irqs; i++) {
	1116	+ irq_domain_set_info(domain, virq + i, info->hwirq + i, &xive_ipi_chip,
	1117	+ domain->host_data, handle_percpu_irq,
	1118	+ NULL, NULL);
	1119	+ }
	1120	+ return 0;
	1121	+}
1081	1122
1082		− /* Initialize it */
1083		− virq = irq_create_mapping(xive_irq_domain, XIVE_IPI_HW_IRQ);
1084		− xive_ipi_irq = virq;
	1123	+static const struct irq_domain_ops xive_ipi_irq_domain_ops = {
	1124	+ .alloc = xive_ipi_irq_domain_alloc,
	1125	+};
1085	1126
1086		− WARN_ON(request_irq(virq, xive_muxed_ipi_action,
1087		− IRQF_PERCPU \| IRQF_NO_THREAD, "IPI", NULL));
	1127	+static int __init xive_request_ipi(void)
	1128	+{
	1129	+ struct fwnode_handle *fwnode;
	1130	+ struct irq_domain *ipi_domain;
	1131	+ unsigned int node;
	1132	+ int ret = -ENOMEM;
	1133	+
	1134	+ fwnode = irq_domain_alloc_named_fwnode("XIVE-IPI");
	1135	+ if (!fwnode)
	1136	+ goto out;
	1137	+
	1138	+ ipi_domain = irq_domain_create_linear(fwnode, nr_node_ids,
	1139	+ &xive_ipi_irq_domain_ops, NULL);
	1140	+ if (!ipi_domain)
	1141	+ goto out_free_fwnode;
	1142	+
	1143	+ xive_ipis = kcalloc(nr_node_ids, sizeof(*xive_ipis), GFP_KERNEL \| __GFP_NOFAIL);
	1144	+ if (!xive_ipis)
	1145	+ goto out_free_domain;
	1146	+
	1147	+ for_each_node(node) {
	1148	+ struct xive_ipi_desc *xid = &xive_ipis[node];
	1149	+ struct xive_ipi_alloc_info info = { node };
	1150	+
	1151	+ /* Skip nodes without CPUs */
	1152	+ if (cpumask_empty(cpumask_of_node(node)))
	1153	+ continue;
	1154	+
	1155	+ /*
	1156	+ * Map one IPI interrupt per node for all cpus of that node.
	1157	+ * Since the HW interrupt number doesn't have any meaning,
	1158	+ * simply use the node number.
	1159	+ */
	1160	+ xid->irq = irq_domain_alloc_irqs(ipi_domain, 1, node, &info);
	1161	+ if (xid->irq < 0) {
	1162	+ ret = xid->irq;
	1163	+ goto out_free_xive_ipis;
	1164	+ }
	1165	+
	1166	+ snprintf(xid->name, sizeof(xid->name), "IPI-%d", node);
	1167	+
	1168	+ ret = request_irq(xid->irq, xive_muxed_ipi_action,
	1169	+ IRQF_PERCPU \| IRQF_NO_THREAD, xid->name, NULL);
	1170	+
	1171	+ WARN(ret < 0, "Failed to request IPI %d: %d\n", xid->irq, ret);
	1172	+ }
	1173	+
	1174	+ return ret;
	1175	+
	1176	+out_free_xive_ipis:
	1177	+ kfree(xive_ipis);
	1178	+out_free_domain:
	1179	+ irq_domain_remove(ipi_domain);
	1180	+out_free_fwnode:
	1181	+ irq_domain_free_fwnode(fwnode);
	1182	+out:
	1183	+ return ret;
1088	1184	}
1089	1185
1090	1186	static int xive_setup_cpu_ipi(unsigned int cpu)
1091	1187	{
	1188	+ unsigned int xive_ipi_irq = xive_ipi_cpu_to_irq(cpu);
1092	1189	struct xive_cpu *xc;
1093	1190	int rc;
1094	1191
@@ -1131,6 +1228,8 @@ static int xive_setup_cpu_ipi(unsigned int cpu) · 36 unmodified lines
1131	1228
1132	1229	static void xive_cleanup_cpu_ipi(unsigned int cpu, struct xive_cpu *xc)
1133	1230	{
	1231	+ unsigned int xive_ipi_irq = xive_ipi_cpu_to_irq(cpu);
	1232	+
1134	1233	/* Disable the IPI and free the IRQ data */
1135	1234
1136	1235	/* Already cleaned up ? */
@@ -1178,19 +1277,6 @@ static int xive_irq_domain_map(struct irq_domain *h, unsigned int virq, · 41 unmodified lines
1178	1277	*/
1179	1278	irq_clear_status_flags(virq, IRQ_LEVEL);
1180	1279
1181		−#ifdef CONFIG_SMP
1182		− /* IPIs are special and come up with HW number 0 */
1183		− if (hw == XIVE_IPI_HW_IRQ) {
1184		− /*
1185		− * IPIs are marked per-cpu. We use separate HW interrupts under
1186		− * the hood but associated with the same "linux" interrupt
1187		− */
1188		− irq_set_chip_and_handler(virq, &xive_ipi_chip,
1189		− handle_percpu_irq);
1190		− return 0;
1191		− }
1192		−#endif
1193		−
1194	1280	rc = xive_irq_alloc_data(virq, hw);
1195	1281	if (rc)
1196	1282	return rc;
@@ -1202,15 +1288,7 @@ static int xive_irq_domain_map(struct irq_domain *h, unsigned int virq, · 5 unmodified lines
1202	1288
1203	1289	static void xive_irq_domain_unmap(struct irq_domain *d, unsigned int virq)
1204	1290	{
1205		− struct irq_data *data = irq_get_irq_data(virq);
1206		− unsigned int hw_irq;
1207		−
1208		− /* XXX Assign BAD number */
1209		− if (!data)
1210		− return;
1211		− hw_irq = (unsigned int)irqd_to_hwirq(data);
1212		− if (hw_irq != XIVE_IPI_HW_IRQ)
1213		− xive_irq_free_data(virq);
	1291	+ xive_irq_free_data(virq);
1214	1292	}
1215	1293
1216	1294	static int xive_irq_domain_xlate(struct irq_domain h, struct device_node ct,
@@ -1335,17 +1413,14 @@ static int xive_prepare_cpu(unsigned int cpu) · 118 unmodified lines
1335	1413
1336	1414	xc = per_cpu(xive_cpu, cpu);
1337	1415	if (!xc) {
1338		− struct device_node *np;
1339		−
1340	1416	xc = kzalloc_node(sizeof(struct xive_cpu),
1341	1417	GFP_KERNEL, cpu_to_node(cpu));
1342	1418	if (!xc)
1343	1419	return -ENOMEM;
1344		− np = of_get_cpu_node(cpu, NULL);
1345		− if (np)
1346		− xc->chip_id = of_get_ibm_chip_id(np);
1347		− of_node_put(np);
1348	1420	xc->hw_ipi = XIVE_BAD_IRQ;
	1421	+ xc->chip_id = XIVE_INVALID_CHIP_ID;
	1422	+ if (xive_ops->prepare_cpu)
	1423	+ xive_ops->prepare_cpu(cpu, xc);
1349	1424
1350	1425	per_cpu(xive_cpu, cpu) = xc;
1351	1426	}
@@ -1408,13 +1483,12 @@ static void xive_flush_cpu_queue(unsigned int cpu, struct xive_cpu *xc) · 56 unmodified lines
1408	1483	struct irq_desc *desc = irq_to_desc(irq);
1409	1484	struct irq_data *d = irq_desc_get_irq_data(desc);
1410	1485	struct xive_irq_data *xd;
1411		− unsigned int hw_irq = (unsigned int)irqd_to_hwirq(d);
1412	1486
1413	1487	/*
1414	1488	* Ignore anything that isn't a XIVE irq and ignore
1415	1489	* IPIs, so can just be dropped.
1416	1490	*/
1417		− if (d->domain != xive_irq_domain \|\| hw_irq == XIVE_IPI_HW_IRQ)
	1491	+ if (d->domain != xive_irq_domain)
1418	1492	continue;
1419	1493
1420	1494	/*
@@ -1592,16 +1666,15 @@ static void xive_debug_show_cpu(struct seq_file *m, int cpu) · 171 unmodified lines
1592	1666	seq_puts(m, "\n");
1593	1667	}
1594	1668
1595		−static void xive_debug_show_irq(struct seq_file m, u32 hw_irq, struct irq_data d)
	1669	+static void xive_debug_show_irq(struct seq_file m, struct irq_data d)
1596	1670	{
1597		− struct irq_chip *chip = irq_data_get_irq_chip(d);
	1671	+ unsigned int hw_irq = (unsigned int)irqd_to_hwirq(d);
1598	1672	int rc;
1599	1673	u32 target;
1600	1674	u8 prio;
1601	1675	u32 lirq;
1602		−
1603		− if (!is_xive_irq(chip))
1604		− return;
	1676	+ struct xive_irq_data *xd;
	1677	+ u64 val;
1605	1678
1606	1679	rc = xive_ops->get_irq_config(hw_irq, &target, &prio, &lirq);
1607	1680	if (rc) {
@@ -1612,17 +1685,14 @@ static void xive_debug_show_irq(struct seq_file *m, u32 hw_irq, struct irq_data · 4 unmodified lines
1612	1685	seq_printf(m, "IRQ 0x%08x : target=0x%x prio=%02x lirq=0x%x ",
1613	1686	hw_irq, target, prio, lirq);
1614	1687
1615		− if (d) {
1616		− struct xive_irq_data *xd = irq_data_get_irq_handler_data(d);
1617		− u64 val = xive_esb_read(xd, XIVE_ESB_GET);
1618		−
1619		− seq_printf(m, "flags=%c%c%c PQ=%c%c",
1620		− xd->flags & XIVE_IRQ_FLAG_STORE_EOI ? 'S' : ' ',
1621		− xd->flags & XIVE_IRQ_FLAG_LSI ? 'L' : ' ',
1622		− xd->flags & XIVE_IRQ_FLAG_H_INT_ESB ? 'H' : ' ',
1623		− val & XIVE_ESB_VAL_P ? 'P' : '-',
1624		− val & XIVE_ESB_VAL_Q ? 'Q' : '-');
1625		− }
	1688	+ xd = irq_data_get_irq_handler_data(d);
	1689	+ val = xive_esb_read(xd, XIVE_ESB_GET);
	1690	+ seq_printf(m, "flags=%c%c%c PQ=%c%c",
	1691	+ xd->flags & XIVE_IRQ_FLAG_STORE_EOI ? 'S' : ' ',
	1692	+ xd->flags & XIVE_IRQ_FLAG_LSI ? 'L' : ' ',
	1693	+ xd->flags & XIVE_IRQ_FLAG_H_INT_ESB ? 'H' : ' ',
	1694	+ val & XIVE_ESB_VAL_P ? 'P' : '-',
	1695	+ val & XIVE_ESB_VAL_Q ? 'Q' : '-');
1626	1696	seq_puts(m, "\n");
1627	1697	}
1628	1698
@@ -1640,16 +1710,9 @@ static int xive_core_debug_show(struct seq_file m, void private) · 11 unmodified lines
1640	1710
1641	1711	for_each_irq_desc(i, desc) {
1642	1712	struct irq_data *d = irq_desc_get_irq_data(desc);
1643		− unsigned int hw_irq;
1644		−
1645		− if (!d)
1646		− continue;
1647		−
1648		− hw_irq = (unsigned int)irqd_to_hwirq(d);
1649	1713
1650		− /* IPIs are special (HW number 0) */
1651		− if (hw_irq != XIVE_IPI_HW_IRQ)
1652		− xive_debug_show_irq(m, hw_irq, d);
	1714	+ if (d->domain == xive_irq_domain)
	1715	+ xive_debug_show_irq(m, d);
1653	1716	}
1654	1717	return 0;
1655	1718	}

arch/powerpc/sysdev/xive/native.c+6−0

@@ -380,6 +380,11 @@ static void xive_native_update_pending(struct xive_cpu *xc) · 379 unmodified lines
380	380	}
381	381	}
382	382
	383	+static void xive_native_prepare_cpu(unsigned int cpu, struct xive_cpu *xc)
	384	+{
	385	+ xc->chip_id = cpu_to_chip_id(cpu);
	386	+}
	387	+
383	388	static void xive_native_setup_cpu(unsigned int cpu, struct xive_cpu *xc)
384	389	{
385	390	s64 rc;
@@ -462,6 +467,7 @@ static const struct xive_ops xive_native_ops = { · 76 unmodified lines
462	467	.match = xive_native_match,
463	468	.shutdown = xive_native_shutdown,
464	469	.update_pending = xive_native_update_pending,
	470	+ .prepare_cpu = xive_native_prepare_cpu,
465	471	.setup_cpu = xive_native_setup_cpu,
466	472	.teardown_cpu = xive_native_teardown_cpu,
467	473	.sync_source = xive_native_sync_source,

arch/powerpc/sysdev/xive/spapr.c+1−1

@@ -549,7 +549,7 @@ static void xive_spapr_cleanup_queue(unsigned int cpu, struct xive_cpu *xc, · 548 unmodified lines
549	549	static bool xive_spapr_match(struct device_node *node)
550	550	{
551	551	/* Ignore cascaded controllers for the moment */
552		− return 1;
	552	+ return true;
553	553	}
554	554
555	555	#ifdef CONFIG_SMP

arch/powerpc/sysdev/xive/xive-internal.h+1−2

@@ -5,8 +5,6 @@ · 4 unmodified lines
5	5	#ifndef __XIVE_INTERNAL_H
6	6	#define __XIVE_INTERNAL_H
7	7
8		−#define XIVE_IPI_HW_IRQ 0 /* interrupt source # for IPIs */
9		−
10	8	/*
11	9	* A "disabled" interrupt should never fire, to catch problems
12	10	* we set its logical number to this
@@ -46,6 +44,7 @@ struct xive_ops { · 33 unmodified lines
46	44	u32 *sw_irq);
47	45	int (setup_queue)(unsigned int cpu, struct xive_cpu xc, u8 prio);
48	46	void (cleanup_queue)(unsigned int cpu, struct xive_cpu xc, u8 prio);
	47	+ void (prepare_cpu)(unsigned int cpu, struct xive_cpu xc);
49	48	void (setup_cpu)(unsigned int cpu, struct xive_cpu xc);
50	49	void (teardown_cpu)(unsigned int cpu, struct xive_cpu xc);
51	50	bool (match)(struct device_node np);

arch/powerpc/xmon/xmon.c+24−52

@@ -54,6 +54,7 @@ · 53 unmodified lines
54	54	#include <asm/code-patching.h>
55	55	#include <asm/sections.h>
56	56	#include <asm/inst.h>
	57	+#include <asm/interrupt.h>
57	58
58	59	#ifdef CONFIG_PPC64
59	60	#include <asm/hvcall.h>
@@ -605,7 +606,7 @@ static int xmon_core(struct pt_regs *regs, int fromipi) · 545 unmodified lines
605	606	* debugger break (IPI). This is similar to
606	607	* crash_kexec_secondary().
607	608	*/
608		− if (TRAP(regs) != 0x100 \|\| !wait_for_other_cpus(ncpus))
	609	+ if (TRAP(regs) != INTERRUPT_SYSTEM_RESET \|\| !wait_for_other_cpus(ncpus))
609	610	smp_send_debugger_break();
610	611
611	612	wait_for_other_cpus(ncpus);
@@ -615,7 +616,7 @@ static int xmon_core(struct pt_regs *regs, int fromipi) · 3 unmodified lines
615	616
616	617	if (!locked_down) {
617	618	/* for breakpoint or single step, print curr insn */
618		− if (bp \|\| TRAP(regs) == 0xd00)
	619	+ if (bp \|\| TRAP(regs) == INTERRUPT_TRACE)
619	620	ppc_inst_dump(regs->nip, 1, 0);
620	621	printf("enter ? for help\n");
621	622	}
@@ -684,7 +685,7 @@ static int xmon_core(struct pt_regs *regs, int fromipi) · 62 unmodified lines
684	685	disable_surveillance();
685	686	if (!locked_down) {
686	687	/* for breakpoint or single step, print current insn */
687		− if (bp \|\| TRAP(regs) == 0xd00)
	688	+ if (bp \|\| TRAP(regs) == INTERRUPT_TRACE)
688	689	ppc_inst_dump(regs->nip, 1, 0);
689	690	printf("enter ? for help\n");
690	691	}
@@ -1769,9 +1770,12 @@ static void excprint(struct pt_regs *fp) · 1078 unmodified lines
1769	1770	printf(" sp: %lx\n", fp->gpr[1]);
1770	1771	printf(" msr: %lx\n", fp->msr);
1771	1772
1772		− if (trap == 0x300 \|\| trap == 0x380 \|\| trap == 0x600 \|\| trap == 0x200) {
	1773	+ if (trap == INTERRUPT_DATA_STORAGE \|\|
	1774	+ trap == INTERRUPT_DATA_SEGMENT \|\|
	1775	+ trap == INTERRUPT_ALIGNMENT \|\|
	1776	+ trap == INTERRUPT_MACHINE_CHECK) {
1773	1777	printf(" dar: %lx\n", fp->dar);
1774		− if (trap != 0x380)
	1778	+ if (trap != INTERRUPT_DATA_SEGMENT)
1775	1779	printf(" dsisr: %lx\n", fp->dsisr);
1776	1780	}
1777	1781
@@ -1785,7 +1789,7 @@ static void excprint(struct pt_regs *fp) · 7 unmodified lines
1785	1789	current->pid, current->comm);
1786	1790	}
1787	1791
1788		− if (trap == 0x700)
	1792	+ if (trap == INTERRUPT_PROGRAM)
1789	1793	print_bug_trap(fp);
1790	1794
1791	1795	printf(linux_banner);
@@ -1815,25 +1819,16 @@ static void prregs(struct pt_regs *fp) · 23 unmodified lines
1815	1819	}
1816	1820
1817	1821	#ifdef CONFIG_PPC64
1818		− if (FULL_REGS(fp)) {
1819		− for (n = 0; n < 16; ++n)
1820		− printf("R%.2d = "REG" R%.2d = "REG"\n",
1821		− n, fp->gpr[n], n+16, fp->gpr[n+16]);
1822		− } else {
1823		− for (n = 0; n < 7; ++n)
1824		− printf("R%.2d = "REG" R%.2d = "REG"\n",
1825		− n, fp->gpr[n], n+7, fp->gpr[n+7]);
1826		− }
	1822	+#define R_PER_LINE 2
1827	1823	#else
	1824	+#define R_PER_LINE 4
	1825	+#endif
	1826	+
1828	1827	for (n = 0; n < 32; ++n) {
1829		− printf("R%.2d = %.8lx%s", n, fp->gpr[n],
1830		− (n & 3) == 3? "\n": " ");
1831		− if (n == 12 && !FULL_REGS(fp)) {
1832		− printf("\n");
1833		− break;
1834		− }
	1828	+ printf("R%.2d = "REG"%s", n, fp->gpr[n],
	1829	+ (n % R_PER_LINE) == R_PER_LINE - 1 ? "\n" : " ");
1835	1830	}
1836		−#endif
	1831	+
1837	1832	printf("pc = ");
1838	1833	xmon_print_symbol(fp->nip, " ", "\n");
1839	1834	if (!trap_is_syscall(fp) && cpu_has_feature(CPU_FTR_CFAR)) {
@@ -1846,7 +1841,9 @@ static void prregs(struct pt_regs *fp) · 6 unmodified lines
1846	1841	printf("ctr = "REG" xer = "REG" trap = %4lx\n",
1847	1842	fp->ctr, fp->xer, fp->trap);
1848	1843	trap = TRAP(fp);
1849		− if (trap == 0x300 \|\| trap == 0x380 \|\| trap == 0x600)
	1844	+ if (trap == INTERRUPT_DATA_STORAGE \|\|
	1845	+ trap == INTERRUPT_DATA_SEGMENT \|\|
	1846	+ trap == INTERRUPT_ALIGNMENT)
1850	1847	printf("dar = "REG" dsisr = %.8lx\n", fp->dar, fp->dsisr);
1851	1848	}
1852	1849
@@ -2727,30 +2724,6 @@ static void dump_all_xives(void) · 874 unmodified lines
2727	2724	dump_one_xive(cpu);
2728	2725	}
2729	2726
2730		−static void dump_one_xive_irq(u32 num, struct irq_data *d)
2731		−{
2732		− xmon_xive_get_irq_config(num, d);
2733		−}
2734		−
2735		−static void dump_all_xive_irq(void)
2736		−{
2737		− unsigned int i;
2738		− struct irq_desc *desc;
2739		−
2740		− for_each_irq_desc(i, desc) {
2741		− struct irq_data *d = irq_desc_get_irq_data(desc);
2742		− unsigned int hwirq;
2743		−
2744		− if (!d)
2745		− continue;
2746		−
2747		− hwirq = (unsigned int)irqd_to_hwirq(d);
2748		− /* IPIs are special (HW number 0) */
2749		− if (hwirq)
2750		− dump_one_xive_irq(hwirq, d);
2751		− }
2752		−}
2753		−
2754	2727	static void dump_xives(void)
2755	2728	{
2756	2729	unsigned long num;
@@ -2767,9 +2740,9 @@ static void dump_xives(void) · 10 unmodified lines
2767	2740	return;
2768	2741	} else if (c == 'i') {
2769	2742	if (scanhex(&num))
2770		− dump_one_xive_irq(num, NULL);
	2743	+ xmon_xive_get_irq_config(num, NULL);
2771	2744	else
2772		− dump_all_xive_irq();
	2745	+ xmon_xive_get_irq_all();
2773	2746	return;
2774	2747	}
2775	2748
@@ -2980,7 +2953,7 @@ generic_inst_dump(unsigned long adr, long count, int praddr, · 204 unmodified lines
2980	2953	if (!ppc_inst_prefixed(inst))
2981	2954	dump_func(ppc_inst_val(inst), adr);
2982	2955	else
2983		− dump_func(ppc_inst_as_u64(inst), adr);
	2956	+ dump_func(ppc_inst_as_ulong(inst), adr);
2984	2957	printf("\n");
2985	2958	}
2986	2959	return adr - first_adr;
@@ -4212,8 +4185,7 @@ static void dump_spu_fields(struct spu *spu) · 1225 unmodified lines
4212	4185	DUMP_FIELD(spu, "0x%p", pdata);
4213	4186	}
4214	4187
4215		−int
4216		−spu_inst_dump(unsigned long adr, long count, int praddr)
	4188	+static int spu_inst_dump(unsigned long adr, long count, int praddr)
4217	4189	{
4218	4190	return generic_inst_dump(adr, count, praddr, print_insn_spu);
4219	4191	}

arch/s390/include/asm/vdso/gettimeofday.h+2−1

@@ -68,7 +68,8 @@ long clock_getres_fallback(clockid_t clkid, struct __kernel_timespec *ts) · 67 unmodified lines
68	68	}
69	69
70	70	#ifdef CONFIG_TIME_NS
71		−static __always_inline const struct vdso_data *__arch_get_timens_vdso_data(void)
	71	+static __always_inline
	72	+const struct vdso_data __arch_get_timens_vdso_data(const struct vdso_data vd)
72	73	{
73	74	return _timens_data;
74	75	}

arch/x86/include/asm/vdso/gettimeofday.h+2−1

@@ -58,7 +58,8 @@ extern struct ms_hyperv_tsc_page hvclock_page · 57 unmodified lines
58	58	#endif
59	59
60	60	#ifdef CONFIG_TIME_NS
61		−static __always_inline const struct vdso_data *__arch_get_timens_vdso_data(void)
	61	+static __always_inline
	62	+const struct vdso_data __arch_get_timens_vdso_data(const struct vdso_data vd)
62	63	{
63	64	return __timens_vdso_data;
64	65	}

drivers/i2c/busses/Kconfig+1−1

@@ -776,7 +776,7 @@ config I2C_MT7621 · 775 unmodified lines
776	776
777	777	config I2C_MV64XXX
778	778	tristate "Marvell mv64xxx I2C Controller"
779		− depends on MV64X60 \|\| PLAT_ORION \|\| ARCH_SUNXI \|\| ARCH_MVEBU \|\| COMPILE_TEST
	779	+ depends on PLAT_ORION \|\| ARCH_SUNXI \|\| ARCH_MVEBU \|\| COMPILE_TEST
780	780	help
781	781	If you say yes to this option, support will be included for the
782	782	built-in I2C interface on the Marvell 64xxx line of host bridges.

drivers/macintosh/via-pmu.c+2−2

@@ -180,14 +180,13 @@ static struct proc_dir_entry *proc_pmu_options; · 179 unmodified lines
180	180	static int option_server_mode;
181	181
182	182	int pmu_battery_count;
183		−int pmu_cur_battery;
	183	+static int pmu_cur_battery;
184	184	unsigned int pmu_power_flags = PMU_PWR_AC_PRESENT;
185	185	struct pmu_battery_info pmu_batteries[PMU_MAX_BATTERIES];
186	186	static int query_batt_timer = BATTERY_POLLING_COUNT;
187	187	static struct adb_request batt_req;
188	188	static struct proc_dir_entry *proc_pmu_batt[PMU_MAX_BATTERIES];
189	189
190		−int __fake_sleep;
191	190	int asleep;
192	191
193	192	#ifdef CONFIG_ADB
@@ -1833,6 +1832,7 @@ pmu_present(void) · 1639 unmodified lines
1833	1832	*/
1834	1833
1835	1834	static u32 save_via[8];
	1835	+static int __fake_sleep;
1836	1836
1837	1837	static void
1838	1838	save_via_state(void)

drivers/macintosh/windfarm_core.c+1−1

@@ -56,7 +56,7 @@ static BLOCKING_NOTIFIER_HEAD(wf_client_list); · 55 unmodified lines
56	56	static int wf_client_count;
57	57	static unsigned int wf_overtemp;
58	58	static unsigned int wf_overtemp_counter;
59		−struct task_struct *wf_thread;
	59	+static struct task_struct *wf_thread;
60	60
61	61	static struct platform_device wf_platform_device = {
62	62	.name = "windfarm",

drivers/macintosh/windfarm_pm121.c+1−1

@@ -433,7 +433,7 @@ struct pm121_sys_state { · 432 unmodified lines
433	433	struct wf_pid_state pid;
434	434	};
435	435
436		−struct pm121_sys_state *pm121_sys_state[N_LOOPS] = {};
	436	+static struct pm121_sys_state *pm121_sys_state[N_LOOPS] = {};
437	437
438	438	/*
439	439	* **** CPU Fans Control Loop ****

drivers/macintosh/windfarm_smu_controls.c+1−1

@@ -94,7 +94,7 @@ static int smu_set_fan(int pwm, u8 id, u16 value) · 93 unmodified lines
94	94	return rc;
95	95	wait_for_completion(&comp);
96	96
97		− /* Handle fallback (see coment above) */
	97	+ /* Handle fallback (see comment above) */
98	98	if (cmd.status != 0 && smu_supports_new_fans_ops) {
99	99	printk(KERN_WARNING "windfarm: SMU failed new fan command "
100	100	"falling back to old method\n");

include/linux/compat.h+35−0

@@ -467,6 +467,34 @@ put_compat_sigset(compat_sigset_t __user compat, const sigset_t set, · 466 unmodified lines
467	467	unsafe_put_user(__s->sig[0], &__c->sig[0], label); \
468	468	} \
469	469	} while (0)
	470	+
	471	+#define unsafe_get_compat_sigset(set, compat, label) do { \
	472	+ const compat_sigset_t __user *__c = compat; \
	473	+ compat_sigset_word hi, lo; \
	474	+ sigset_t *__s = set; \
	475	+ \
	476	+ switch (_NSIG_WORDS) { \
	477	+ case 4: \
	478	+ unsafe_get_user(lo, &__c->sig[7], label); \
	479	+ unsafe_get_user(hi, &__c->sig[6], label); \
	480	+ __s->sig[3] = hi \| (((long)lo) << 32); \
	481	+ fallthrough; \
	482	+ case 3: \
	483	+ unsafe_get_user(lo, &__c->sig[5], label); \
	484	+ unsafe_get_user(hi, &__c->sig[4], label); \
	485	+ __s->sig[2] = hi \| (((long)lo) << 32); \
	486	+ fallthrough; \
	487	+ case 2: \
	488	+ unsafe_get_user(lo, &__c->sig[3], label); \
	489	+ unsafe_get_user(hi, &__c->sig[2], label); \
	490	+ __s->sig[1] = hi \| (((long)lo) << 32); \
	491	+ fallthrough; \
	492	+ case 1: \
	493	+ unsafe_get_user(lo, &__c->sig[1], label); \
	494	+ unsafe_get_user(hi, &__c->sig[0], label); \
	495	+ __s->sig[0] = hi \| (((long)lo) << 32); \
	496	+ } \
	497	+} while (0)
470	498	#else
471	499	#define unsafe_put_compat_sigset(compat, set, label) do { \
472	500	compat_sigset_t __user *__c = compat; \
@@ -474,6 +502,13 @@ put_compat_sigset(compat_sigset_t __user compat, const sigset_t set, · 1 unmodified line
474	502	\
475	503	unsafe_copy_to_user(__c, __s, sizeof(*__c), label); \
476	504	} while (0)
	505	+
	506	+#define unsafe_get_compat_sigset(set, compat, label) do { \
	507	+ const compat_sigset_t __user *__c = compat; \
	508	+ sigset_t *__s = set; \
	509	+ \
	510	+ unsafe_copy_from_user(__s, __c, sizeof(*__c), label); \
	511	+} while (0)
477	512	#endif
478	513
479	514	extern int compat_ptrace_request(struct task_struct *child,

include/linux/uaccess.h+1−0

@@ -397,6 +397,7 @@ long strnlen_user_nofault(const void __user *unsafe_addr, long count); · 396 unmodified lines
397	397	#define unsafe_get_user(x,p,e) unsafe_op_wrap(__get_user(x,p),e)
398	398	#define unsafe_put_user(x,p,e) unsafe_op_wrap(__put_user(x,p),e)
399	399	#define unsafe_copy_to_user(d,s,l,e) unsafe_op_wrap(__copy_to_user(d,s,l),e)
	400	+#define unsafe_copy_from_user(d,s,l,e) unsafe_op_wrap(__copy_from_user(d,s,l),e)
400	401	static inline unsigned long user_access_save(void) { return 0UL; }
401	402	static inline void user_access_restore(unsigned long flags) { }
402	403	#endif

lib/vdso/gettimeofday.c+17−14

@@ -46,16 +46,18 @@ static inline bool vdso_cycles_ok(u64 cycles) · 45 unmodified lines
46	46	#endif
47	47
48	48	#ifdef CONFIG_TIME_NS
49		−static int do_hres_timens(const struct vdso_data *vdns, clockid_t clk,
50		− struct __kernel_timespec *ts)
	49	+static __always_inline int do_hres_timens(const struct vdso_data *vdns, clockid_t clk,
	50	+ struct __kernel_timespec *ts)
51	51	{
52		− const struct vdso_data *vd = __arch_get_timens_vdso_data();
	52	+ const struct vdso_data *vd;
53	53	const struct timens_offset *offs = &vdns->offset[clk];
54	54	const struct vdso_timestamp *vdso_ts;
55	55	u64 cycles, last, ns;
56	56	u32 seq;
57	57	s64 sec;
58	58
	59	+ vd = vdns - (clk == CLOCK_MONOTONIC_RAW ? CS_RAW : CS_HRES_COARSE);
	60	+ vd = __arch_get_timens_vdso_data(vd);
59	61	if (clk != CLOCK_MONOTONIC_RAW)
60	62	vd = &vd[CS_HRES_COARSE];
61	63	else
@@ -92,13 +94,14 @@ static int do_hres_timens(const struct vdso_data *vdns, clockid_t clk, · 30 unmodified lines
92	94	return 0;
93	95	}
94	96	#else
95		−static __always_inline const struct vdso_data *__arch_get_timens_vdso_data(void)
	97	+static __always_inline
	98	+const struct vdso_data __arch_get_timens_vdso_data(const struct vdso_data vd)
96	99	{
97	100	return NULL;
98	101	}
99	102
100		−static int do_hres_timens(const struct vdso_data *vdns, clockid_t clk,
101		− struct __kernel_timespec *ts)
	103	+static __always_inline int do_hres_timens(const struct vdso_data *vdns, clockid_t clk,
	104	+ struct __kernel_timespec *ts)
102	105	{
103	106	return -EINVAL;
104	107	}
@@ -159,10 +162,10 @@ static __always_inline int do_hres(const struct vdso_data *vd, clockid_t clk, · 54 unmodified lines
159	162	}
160	163
161	164	#ifdef CONFIG_TIME_NS
162		−static int do_coarse_timens(const struct vdso_data *vdns, clockid_t clk,
163		− struct __kernel_timespec *ts)
	165	+static __always_inline int do_coarse_timens(const struct vdso_data *vdns, clockid_t clk,
	166	+ struct __kernel_timespec *ts)
164	167	{
165		− const struct vdso_data *vd = __arch_get_timens_vdso_data();
	168	+ const struct vdso_data *vd = __arch_get_timens_vdso_data(vdns);
166	169	const struct vdso_timestamp *vdso_ts = &vd->basetime[clk];
167	170	const struct timens_offset *offs = &vdns->offset[clk];
168	171	u64 nsec;
@@ -188,8 +191,8 @@ static int do_coarse_timens(const struct vdso_data *vdns, clockid_t clk, · 19 unmodified lines
188	191	return 0;
189	192	}
190	193	#else
191		−static int do_coarse_timens(const struct vdso_data *vdns, clockid_t clk,
192		− struct __kernel_timespec *ts)
	194	+static __always_inline int do_coarse_timens(const struct vdso_data *vdns, clockid_t clk,
	195	+ struct __kernel_timespec *ts)
193	196	{
194	197	return -1;
195	198	}
@@ -310,7 +313,7 @@ __cvdso_gettimeofday_data(const struct vdso_data *vd, · 114 unmodified lines
310	313	if (unlikely(tz != NULL)) {
311	314	if (IS_ENABLED(CONFIG_TIME_NS) &&
312	315	vd->clock_mode == VDSO_CLOCKMODE_TIMENS)
313		− vd = __arch_get_timens_vdso_data();
	316	+ vd = __arch_get_timens_vdso_data(vd);
314	317
315	318	tz->tz_minuteswest = vd[CS_HRES_COARSE].tz_minuteswest;
316	319	tz->tz_dsttime = vd[CS_HRES_COARSE].tz_dsttime;
@@ -333,7 +336,7 @@ __cvdso_time_data(const struct vdso_data vd, __kernel_old_time_t time) · 16 unmodified lines
333	336
334	337	if (IS_ENABLED(CONFIG_TIME_NS) &&
335	338	vd->clock_mode == VDSO_CLOCKMODE_TIMENS)
336		− vd = __arch_get_timens_vdso_data();
	339	+ vd = __arch_get_timens_vdso_data(vd);
337	340
338	341	t = READ_ONCE(vd[CS_HRES_COARSE].basetime[CLOCK_REALTIME].sec);
339	342
@@ -363,7 +366,7 @@ int __cvdso_clock_getres_common(const struct vdso_data *vd, clockid_t clock, · 23 unmodified lines
363	366
364	367	if (IS_ENABLED(CONFIG_TIME_NS) &&
365	368	vd->clock_mode == VDSO_CLOCKMODE_TIMENS)
366		− vd = __arch_get_timens_vdso_data();
	369	+ vd = __arch_get_timens_vdso_data(vd);
367	370
368	371	/*
369	372	* Convert the clockid to a bitmask and use it to check which

tools/testing/selftests/powerpc/alignment/alignment_handler.c+1−10

@@ -10,16 +10,7 @@ · 9 unmodified lines
10	10	*
11	11	* We create two sets of source and destination buffers, one in regular memory,
12	12	* the other cache-inhibited (by default we use /dev/fb0 for this, but an
13		− * alterative path for cache-inhibited memory may be provided).
14		− *
15		− * One way to get cache-inhibited memory is to use the "mem" kernel parameter
16		− * to limit the kernel to less memory than actually exists. Addresses above
17		− * the limit may still be accessed but will be treated as cache-inhibited. For
18		− * example, if there is actually 4GB of memory and the parameter "mem=3GB" is
19		− * used, memory from address 0xC0000000 onwards is treated as cache-inhibited.
20		− * To access this region /dev/mem is used. The kernel should be configured
21		− * without CONFIG_STRICT_DEVMEM. In this case use:
22		− * ./alignment_handler /dev/mem 0xc0000000
	13	+ * alterative path for cache-inhibited memory may be provided, e.g. memtrace).
23	14	*
24	15	* We initialise the source buffers, then use whichever set of load/store
25	16	* instructions is under test to copy bytes from the source buffers to the

tools/testing/selftests/powerpc/mm/Makefile+1−0

@@ -5,6 +5,7 @@ noarg: · 4 unmodified lines
5	5	TEST_GEN_PROGS := hugetlb_vs_thp_test subpage_prot prot_sao segv_errors wild_bctr \
6	6	large_vm_fork_separation bad_accesses pkey_exec_prot \
7	7	pkey_siginfo stack_expansion_signal stack_expansion_ldst
	8	+TEST_PROGS := stress_code_patching.sh
8	9
9	10	TEST_GEN_PROGS_EXTENDED := tlbie_test
10	11	TEST_GEN_FILES := tempfile

tools/testing/selftests/powerpc/mm/stress_code_patching.sh+49−0

@@ -0,0 +1,49 @@
	1	+#!/bin/bash
	2	+# SPDX-License-Identifier: GPL-2.0-or-later
	3	+
	4	+TIMEOUT=30
	5	+
	6	+DEBUFS_DIR=`cat /proc/mounts \| grep debugfs \| awk '{print $2}'`
	7	+if [ ! -e "$DEBUFS_DIR" ]
	8	+then
	9	+ echo "debugfs not found, skipping" 1>&2
	10	+ exit 4
	11	+fi
	12	+
	13	+if [ ! -e "$DEBUFS_DIR/tracing/current_tracer" ]
	14	+then
	15	+ echo "Tracing files not found, skipping" 1>&2
	16	+ exit 4
	17	+fi
	18	+
	19	+
	20	+echo "Testing for spurious faults when mapping kernel memory..."
	21	+
	22	+if grep -q "FUNCTION TRACING IS CORRUPTED" "$DEBUFS_DIR/tracing/trace"
	23	+then
	24	+ echo "FAILED: Ftrace already dead. Probably due to a spurious fault" 1>&2
	25	+ exit 1
	26	+fi
	27	+
	28	+dmesg -C
	29	+START_TIME=`date +%s`
	30	+END_TIME=`expr $START_TIME + $TIMEOUT`
	31	+while [ `date +%s` -lt $END_TIME ]
	32	+do
	33	+ echo function > $DEBUFS_DIR/tracing/current_tracer
	34	+ echo nop > $DEBUFS_DIR/tracing/current_tracer
	35	+ if dmesg \| grep -q 'ftrace bug'
	36	+ then
	37	+ break
	38	+ fi
	39	+done
	40	+
	41	+echo nop > $DEBUFS_DIR/tracing/current_tracer
	42	+if dmesg \| grep -q 'ftrace bug'
	43	+then
	44	+ echo "FAILED: Mapping kernel memory causes spurious faults" 1>&2
	45	+ exit 1
	46	+else
	47	+ echo "OK: Mapping kernel memory does not cause spurious faults"
	48	+ exit 0
	49	+fi

tools/testing/selftests/powerpc/nx-gzip/gzfht_test.c+1−1

@@ -324,7 +324,7 @@ int compress_file(int argc, char *argv, void handle) · 323 unmodified lines
324	324	fprintf(stderr, "error: cannot progress; ");
325	325	fprintf(stderr, "too many faults\n");
326	326	exit(-1);
327		− };
	327	+ }
328	328	}
329	329
330	330	fault_tries = NX_MAX_FAULTS; /* Reset for the next chunk */

tools/testing/selftests/powerpc/ptrace/.gitignore+1−0

@@ -14,3 +14,4 @@ perf-hwbreak · 13 unmodified lines
14	14	core-pkey
15	15	ptrace-pkey
16	16	ptrace-syscall
	17	+ptrace-perf-hwbreak

tools/testing/selftests/powerpc/ptrace/Makefile+1−1

@@ -2,7 +2,7 @@ · 1 unmodified line
2	2	TEST_GEN_PROGS := ptrace-gpr ptrace-tm-gpr ptrace-tm-spd-gpr \
3	3	ptrace-tar ptrace-tm-tar ptrace-tm-spd-tar ptrace-vsx ptrace-tm-vsx \
4	4	ptrace-tm-spd-vsx ptrace-tm-spr ptrace-hwbreak ptrace-pkey core-pkey \
5		− perf-hwbreak ptrace-syscall
	5	+ perf-hwbreak ptrace-syscall ptrace-perf-hwbreak
6	6
7	7	top_srcdir = ../../../../..
8	8	include ../../lib.mk

tools/testing/selftests/powerpc/ptrace/perf-hwbreak.c+594−41

@@ -21,8 +21,13 @@ · 20 unmodified lines
21	21	#include <assert.h>
22	22	#include <stdio.h>
23	23	#include <stdlib.h>
	24	+#include <signal.h>
24	25	#include <string.h>
25	26	#include <sys/ioctl.h>
	27	+#include <sys/wait.h>
	28	+#include <sys/ptrace.h>
	29	+#include <sys/sysinfo.h>
	30	+#include <asm/ptrace.h>
26	31	#include <elf.h>
27	32	#include <pthread.h>
28	33	#include <sys/syscall.h>
@@ -30,32 +35,130 @@ · 1 unmodified line
30	35	#include <linux/hw_breakpoint.h>
31	36	#include "utils.h"
32	37
	38	+#ifndef PPC_DEBUG_FEATURE_DATA_BP_ARCH_31
	39	+#define PPC_DEBUG_FEATURE_DATA_BP_ARCH_31 0x20
	40	+#endif
	41	+
33	42	#define MAX_LOOPS 10000
34	43
35	44	#define DAWR_LENGTH_MAX ((0x3f + 1) * 8)
36	45
37		−static inline int sys_perf_event_open(struct perf_event_attr *attr, pid_t pid,
38		− int cpu, int group_fd,
39		− unsigned long flags)
	46	+int nprocs;
	47	+
	48	+static volatile int a = 10;
	49	+static volatile int b = 10;
	50	+static volatile char c[512 + 8] __attribute__((aligned(512)));
	51	+
	52	+static void perf_event_attr_set(struct perf_event_attr *attr,
	53	+ __u32 type, __u64 addr, __u64 len,
	54	+ bool exclude_user)
40	55	{
41		− attr->size = sizeof(*attr);
42		− return syscall(__NR_perf_event_open, attr, pid, cpu, group_fd, flags);
	56	+ memset(attr, 0, sizeof(struct perf_event_attr));
	57	+ attr->type = PERF_TYPE_BREAKPOINT;
	58	+ attr->size = sizeof(struct perf_event_attr);
	59	+ attr->bp_type = type;
	60	+ attr->bp_addr = addr;
	61	+ attr->bp_len = len;
	62	+ attr->exclude_kernel = 1;
	63	+ attr->exclude_hv = 1;
	64	+ attr->exclude_guest = 1;
	65	+ attr->exclude_user = exclude_user;
	66	+ attr->disabled = 1;
43	67	}
44	68
45		−static inline bool breakpoint_test(int len)
	69	+static int
	70	+perf_process_event_open_exclude_user(__u32 type, __u64 addr, __u64 len, bool exclude_user)
	71	+{
	72	+ struct perf_event_attr attr;
	73	+
	74	+ perf_event_attr_set(&attr, type, addr, len, exclude_user);
	75	+ return syscall(__NR_perf_event_open, &attr, getpid(), -1, -1, 0);
	76	+}
	77	+
	78	+static int perf_process_event_open(__u32 type, __u64 addr, __u64 len)
	79	+{
	80	+ struct perf_event_attr attr;
	81	+
	82	+ perf_event_attr_set(&attr, type, addr, len, 0);
	83	+ return syscall(__NR_perf_event_open, &attr, getpid(), -1, -1, 0);
	84	+}
	85	+
	86	+static int perf_cpu_event_open(long cpu, __u32 type, __u64 addr, __u64 len)
46	87	{
47	88	struct perf_event_attr attr;
	89	+
	90	+ perf_event_attr_set(&attr, type, addr, len, 0);
	91	+ return syscall(__NR_perf_event_open, &attr, -1, cpu, -1, 0);
	92	+}
	93	+
	94	+static void close_fds(int *fd, int n)
	95	+{
	96	+ int i;
	97	+
	98	+ for (i = 0; i < n; i++)
	99	+ close(fd[i]);
	100	+}
	101	+
	102	+static unsigned long read_fds(int *fd, int n)
	103	+{
	104	+ int i;
	105	+ unsigned long c = 0;
	106	+ unsigned long count = 0;
	107	+ size_t res;
	108	+
	109	+ for (i = 0; i < n; i++) {
	110	+ res = read(fd[i], &c, sizeof(c));
	111	+ assert(res == sizeof(unsigned long long));
	112	+ count += c;
	113	+ }
	114	+ return count;
	115	+}
	116	+
	117	+static void reset_fds(int *fd, int n)
	118	+{
	119	+ int i;
	120	+
	121	+ for (i = 0; i < n; i++)
	122	+ ioctl(fd[i], PERF_EVENT_IOC_RESET);
	123	+}
	124	+
	125	+static void enable_fds(int *fd, int n)
	126	+{
	127	+ int i;
	128	+
	129	+ for (i = 0; i < n; i++)
	130	+ ioctl(fd[i], PERF_EVENT_IOC_ENABLE);
	131	+}
	132	+
	133	+static void disable_fds(int *fd, int n)
	134	+{
	135	+ int i;
	136	+
	137	+ for (i = 0; i < n; i++)
	138	+ ioctl(fd[i], PERF_EVENT_IOC_DISABLE);
	139	+}
	140	+
	141	+static int perf_systemwide_event_open(int *fd, __u32 type, __u64 addr, __u64 len)
	142	+{
	143	+ int i = 0;
	144	+
	145	+ /* Assume online processors are 0 to nprocs for simplisity */
	146	+ for (i = 0; i < nprocs; i++) {
	147	+ fd[i] = perf_cpu_event_open(i, type, addr, len);
	148	+ if (fd[i] < 0) {
	149	+ close_fds(fd, i);
	150	+ return fd[i];
	151	+ }
	152	+ }
	153	+ return 0;
	154	+}
	155	+
	156	+static inline bool breakpoint_test(int len)
	157	+{
48	158	int fd;
49	159
50		− /* setup counters */
51		− memset(&attr, 0, sizeof(attr));
52		− attr.disabled = 1;
53		− attr.type = PERF_TYPE_BREAKPOINT;
54		− attr.bp_type = HW_BREAKPOINT_R;
55	160	/* bp_addr can point anywhere but needs to be aligned */
56		− attr.bp_addr = (__u64)(&attr) & 0xfffffffffffff800;
57		− attr.bp_len = len;
58		− fd = sys_perf_event_open(&attr, 0, -1, -1, 0);
	161	+ fd = perf_process_event_open(HW_BREAKPOINT_R, (__u64)(&fd) & 0xfffffffffffff800, len);
59	162	if (fd < 0)
60	163	return false;
61	164	close(fd);
@@ -75,7 +178,6 @@ static inline bool dawr_supported(void) · 13 unmodified lines
75	178	static int runtestsingle(int readwriteflag, int exclude_user, int arraytest)
76	179	{
77	180	int i,j;
78		− struct perf_event_attr attr;
79	181	size_t res;
80	182	unsigned long long breaks, needed;
81	183	int readint;
@@ -85,6 +187,7 @@ static int runtestsingle(int readwriteflag, int exclude_user, int arraytest) · 3 unmodified lines
85	187	int break_fd;
86	188	int loop_num = MAX_LOOPS - (rand() % 100); /* provide some variability */
87	189	volatile int *k;
	190	+ __u64 len;
88	191
89	192	/* align to 0x400 boundary as required by DAWR */
90	193	readintalign = (int *)(((unsigned long)readintarraybig + 0x7ff) &
@@ -94,19 +197,11 @@ static int runtestsingle(int readwriteflag, int exclude_user, int arraytest) · 3 unmodified lines
94	197	if (arraytest)
95	198	ptr = &readintalign[0];
96	199
97		− /* setup counters */
98		− memset(&attr, 0, sizeof(attr));
99		− attr.disabled = 1;
100		− attr.type = PERF_TYPE_BREAKPOINT;
101		− attr.bp_type = readwriteflag;
102		− attr.bp_addr = (__u64)ptr;
103		− attr.bp_len = sizeof(int);
104		− if (arraytest)
105		− attr.bp_len = DAWR_LENGTH_MAX;
106		− attr.exclude_user = exclude_user;
107		− break_fd = sys_perf_event_open(&attr, 0, -1, -1, 0);
	200	+ len = arraytest ? DAWR_LENGTH_MAX : sizeof(int);
	201	+ break_fd = perf_process_event_open_exclude_user(readwriteflag, (__u64)ptr,
	202	+ len, exclude_user);
108	203	if (break_fd < 0) {
109		− perror("sys_perf_event_open");
	204	+ perror("perf_process_event_open_exclude_user");
110	205	exit(1);
111	206	}
112	207
@@ -153,7 +248,6 @@ static int runtest_dar_outside(void) · 40 unmodified lines
153	248	void *target;
154	249	volatile __u16 temp16;
155	250	volatile __u64 temp64;
156		− struct perf_event_attr attr;
157	251	int break_fd;
158	252	unsigned long long breaks;
159	253	int fail = 0;
@@ -165,21 +259,11 @@ static int runtest_dar_outside(void) · 5 unmodified lines
165	259	exit(EXIT_FAILURE);
166	260	}
167	261
168		− /* setup counters */
169		− memset(&attr, 0, sizeof(attr));
170		− attr.disabled = 1;
171		− attr.type = PERF_TYPE_BREAKPOINT;
172		− attr.exclude_kernel = 1;
173		− attr.exclude_hv = 1;
174		− attr.exclude_guest = 1;
175		− attr.bp_type = HW_BREAKPOINT_RW;
176	262	/* watch middle half of target array */
177		− attr.bp_addr = (__u64)(target + 2);
178		− attr.bp_len = 4;
179		− break_fd = sys_perf_event_open(&attr, 0, -1, -1, 0);
	263	+ break_fd = perf_process_event_open(HW_BREAKPOINT_RW, (__u64)(target + 2), 4);
180	264	if (break_fd < 0) {
181	265	free(target);
182		− perror("sys_perf_event_open");
	266	+ perror("perf_process_event_open");
183	267	exit(EXIT_FAILURE);
184	268	}
185	269
@@ -263,11 +347,467 @@ static int runtest_dar_outside(void) · 77 unmodified lines
263	347	return fail;
264	348	}
265	349
	350	+static void multi_dawr_workload(void)
	351	+{
	352	+ a += 10;
	353	+ b += 10;
	354	+ c[512 + 1] += 'a';
	355	+}
	356	+
	357	+static int test_process_multi_diff_addr(void)
	358	+{
	359	+ unsigned long long breaks1 = 0, breaks2 = 0;
	360	+ int fd1, fd2;
	361	+ char *desc = "Process specific, Two events, diff addr";
	362	+ size_t res;
	363	+
	364	+ fd1 = perf_process_event_open(HW_BREAKPOINT_RW, (__u64)&a, (__u64)sizeof(a));
	365	+ if (fd1 < 0) {
	366	+ perror("perf_process_event_open");
	367	+ exit(EXIT_FAILURE);
	368	+ }
	369	+
	370	+ fd2 = perf_process_event_open(HW_BREAKPOINT_RW, (__u64)&b, (__u64)sizeof(b));
	371	+ if (fd2 < 0) {
	372	+ close(fd1);
	373	+ perror("perf_process_event_open");
	374	+ exit(EXIT_FAILURE);
	375	+ }
	376	+
	377	+ ioctl(fd1, PERF_EVENT_IOC_RESET);
	378	+ ioctl(fd2, PERF_EVENT_IOC_RESET);
	379	+ ioctl(fd1, PERF_EVENT_IOC_ENABLE);
	380	+ ioctl(fd2, PERF_EVENT_IOC_ENABLE);
	381	+ multi_dawr_workload();
	382	+ ioctl(fd1, PERF_EVENT_IOC_DISABLE);
	383	+ ioctl(fd2, PERF_EVENT_IOC_DISABLE);
	384	+
	385	+ res = read(fd1, &breaks1, sizeof(breaks1));
	386	+ assert(res == sizeof(unsigned long long));
	387	+ res = read(fd2, &breaks2, sizeof(breaks2));
	388	+ assert(res == sizeof(unsigned long long));
	389	+
	390	+ close(fd1);
	391	+ close(fd2);
	392	+
	393	+ if (breaks1 != 2 \|\| breaks2 != 2) {
	394	+ printf("FAILED: %s: %lld != 2 \|\| %lld != 2\n", desc, breaks1, breaks2);
	395	+ return 1;
	396	+ }
	397	+
	398	+ printf("TESTED: %s\n", desc);
	399	+ return 0;
	400	+}
	401	+
	402	+static int test_process_multi_same_addr(void)
	403	+{
	404	+ unsigned long long breaks1 = 0, breaks2 = 0;
	405	+ int fd1, fd2;
	406	+ char *desc = "Process specific, Two events, same addr";
	407	+ size_t res;
	408	+
	409	+ fd1 = perf_process_event_open(HW_BREAKPOINT_RW, (__u64)&a, (__u64)sizeof(a));
	410	+ if (fd1 < 0) {
	411	+ perror("perf_process_event_open");
	412	+ exit(EXIT_FAILURE);
	413	+ }
	414	+
	415	+ fd2 = perf_process_event_open(HW_BREAKPOINT_RW, (__u64)&a, (__u64)sizeof(a));
	416	+ if (fd2 < 0) {
	417	+ close(fd1);
	418	+ perror("perf_process_event_open");
	419	+ exit(EXIT_FAILURE);
	420	+ }
	421	+
	422	+ ioctl(fd1, PERF_EVENT_IOC_RESET);
	423	+ ioctl(fd2, PERF_EVENT_IOC_RESET);
	424	+ ioctl(fd1, PERF_EVENT_IOC_ENABLE);
	425	+ ioctl(fd2, PERF_EVENT_IOC_ENABLE);
	426	+ multi_dawr_workload();
	427	+ ioctl(fd1, PERF_EVENT_IOC_DISABLE);
	428	+ ioctl(fd2, PERF_EVENT_IOC_DISABLE);
	429	+
	430	+ res = read(fd1, &breaks1, sizeof(breaks1));
	431	+ assert(res == sizeof(unsigned long long));
	432	+ res = read(fd2, &breaks2, sizeof(breaks2));
	433	+ assert(res == sizeof(unsigned long long));
	434	+
	435	+ close(fd1);
	436	+ close(fd2);
	437	+
	438	+ if (breaks1 != 2 \|\| breaks2 != 2) {
	439	+ printf("FAILED: %s: %lld != 2 \|\| %lld != 2\n", desc, breaks1, breaks2);
	440	+ return 1;
	441	+ }
	442	+
	443	+ printf("TESTED: %s\n", desc);
	444	+ return 0;
	445	+}
	446	+
	447	+static int test_process_multi_diff_addr_ro_wo(void)
	448	+{
	449	+ unsigned long long breaks1 = 0, breaks2 = 0;
	450	+ int fd1, fd2;
	451	+ char *desc = "Process specific, Two events, diff addr, one is RO, other is WO";
	452	+ size_t res;
	453	+
	454	+ fd1 = perf_process_event_open(HW_BREAKPOINT_W, (__u64)&a, (__u64)sizeof(a));
	455	+ if (fd1 < 0) {
	456	+ perror("perf_process_event_open");
	457	+ exit(EXIT_FAILURE);
	458	+ }
	459	+
	460	+ fd2 = perf_process_event_open(HW_BREAKPOINT_R, (__u64)&b, (__u64)sizeof(b));
	461	+ if (fd2 < 0) {
	462	+ close(fd1);
	463	+ perror("perf_process_event_open");
	464	+ exit(EXIT_FAILURE);
	465	+ }
	466	+
	467	+ ioctl(fd1, PERF_EVENT_IOC_RESET);
	468	+ ioctl(fd2, PERF_EVENT_IOC_RESET);
	469	+ ioctl(fd1, PERF_EVENT_IOC_ENABLE);
	470	+ ioctl(fd2, PERF_EVENT_IOC_ENABLE);
	471	+ multi_dawr_workload();
	472	+ ioctl(fd1, PERF_EVENT_IOC_DISABLE);
	473	+ ioctl(fd2, PERF_EVENT_IOC_DISABLE);
	474	+
	475	+ res = read(fd1, &breaks1, sizeof(breaks1));
	476	+ assert(res == sizeof(unsigned long long));
	477	+ res = read(fd2, &breaks2, sizeof(breaks2));
	478	+ assert(res == sizeof(unsigned long long));
	479	+
	480	+ close(fd1);
	481	+ close(fd2);
	482	+
	483	+ if (breaks1 != 1 \|\| breaks2 != 1) {
	484	+ printf("FAILED: %s: %lld != 1 \|\| %lld != 1\n", desc, breaks1, breaks2);
	485	+ return 1;
	486	+ }
	487	+
	488	+ printf("TESTED: %s\n", desc);
	489	+ return 0;
	490	+}
	491	+
	492	+static int test_process_multi_same_addr_ro_wo(void)
	493	+{
	494	+ unsigned long long breaks1 = 0, breaks2 = 0;
	495	+ int fd1, fd2;
	496	+ char *desc = "Process specific, Two events, same addr, one is RO, other is WO";
	497	+ size_t res;
	498	+
	499	+ fd1 = perf_process_event_open(HW_BREAKPOINT_R, (__u64)&a, (__u64)sizeof(a));
	500	+ if (fd1 < 0) {
	501	+ perror("perf_process_event_open");
	502	+ exit(EXIT_FAILURE);
	503	+ }
	504	+
	505	+ fd2 = perf_process_event_open(HW_BREAKPOINT_W, (__u64)&a, (__u64)sizeof(a));
	506	+ if (fd2 < 0) {
	507	+ close(fd1);
	508	+ perror("perf_process_event_open");
	509	+ exit(EXIT_FAILURE);
	510	+ }
	511	+
	512	+ ioctl(fd1, PERF_EVENT_IOC_RESET);
	513	+ ioctl(fd2, PERF_EVENT_IOC_RESET);
	514	+ ioctl(fd1, PERF_EVENT_IOC_ENABLE);
	515	+ ioctl(fd2, PERF_EVENT_IOC_ENABLE);
	516	+ multi_dawr_workload();
	517	+ ioctl(fd1, PERF_EVENT_IOC_DISABLE);
	518	+ ioctl(fd2, PERF_EVENT_IOC_DISABLE);
	519	+
	520	+ res = read(fd1, &breaks1, sizeof(breaks1));
	521	+ assert(res == sizeof(unsigned long long));
	522	+ res = read(fd2, &breaks2, sizeof(breaks2));
	523	+ assert(res == sizeof(unsigned long long));
	524	+
	525	+ close(fd1);
	526	+ close(fd2);
	527	+
	528	+ if (breaks1 != 1 \|\| breaks2 != 1) {
	529	+ printf("FAILED: %s: %lld != 1 \|\| %lld != 1\n", desc, breaks1, breaks2);
	530	+ return 1;
	531	+ }
	532	+
	533	+ printf("TESTED: %s\n", desc);
	534	+ return 0;
	535	+}
	536	+
	537	+static int test_syswide_multi_diff_addr(void)
	538	+{
	539	+ unsigned long long breaks1 = 0, breaks2 = 0;
	540	+ int fd1 = malloc(nprocs sizeof(int));
	541	+ int fd2 = malloc(nprocs sizeof(int));
	542	+ char *desc = "Systemwide, Two events, diff addr";
	543	+ int ret;
	544	+
	545	+ ret = perf_systemwide_event_open(fd1, HW_BREAKPOINT_RW, (__u64)&a, (__u64)sizeof(a));
	546	+ if (ret) {
	547	+ perror("perf_systemwide_event_open");
	548	+ exit(EXIT_FAILURE);
	549	+ }
	550	+
	551	+ ret = perf_systemwide_event_open(fd2, HW_BREAKPOINT_RW, (__u64)&b, (__u64)sizeof(b));
	552	+ if (ret) {
	553	+ close_fds(fd1, nprocs);
	554	+ perror("perf_systemwide_event_open");
	555	+ exit(EXIT_FAILURE);
	556	+ }
	557	+
	558	+ reset_fds(fd1, nprocs);
	559	+ reset_fds(fd2, nprocs);
	560	+ enable_fds(fd1, nprocs);
	561	+ enable_fds(fd2, nprocs);
	562	+ multi_dawr_workload();
	563	+ disable_fds(fd1, nprocs);
	564	+ disable_fds(fd2, nprocs);
	565	+
	566	+ breaks1 = read_fds(fd1, nprocs);
	567	+ breaks2 = read_fds(fd2, nprocs);
	568	+
	569	+ close_fds(fd1, nprocs);
	570	+ close_fds(fd2, nprocs);
	571	+
	572	+ free(fd1);
	573	+ free(fd2);
	574	+
	575	+ if (breaks1 != 2 \|\| breaks2 != 2) {
	576	+ printf("FAILED: %s: %lld != 2 \|\| %lld != 2\n", desc, breaks1, breaks2);
	577	+ return 1;
	578	+ }
	579	+
	580	+ printf("TESTED: %s\n", desc);
	581	+ return 0;
	582	+}
	583	+
	584	+static int test_syswide_multi_same_addr(void)
	585	+{
	586	+ unsigned long long breaks1 = 0, breaks2 = 0;
	587	+ int fd1 = malloc(nprocs sizeof(int));
	588	+ int fd2 = malloc(nprocs sizeof(int));
	589	+ char *desc = "Systemwide, Two events, same addr";
	590	+ int ret;
	591	+
	592	+ ret = perf_systemwide_event_open(fd1, HW_BREAKPOINT_RW, (__u64)&a, (__u64)sizeof(a));
	593	+ if (ret) {
	594	+ perror("perf_systemwide_event_open");
	595	+ exit(EXIT_FAILURE);
	596	+ }
	597	+
	598	+ ret = perf_systemwide_event_open(fd2, HW_BREAKPOINT_RW, (__u64)&a, (__u64)sizeof(a));
	599	+ if (ret) {
	600	+ close_fds(fd1, nprocs);
	601	+ perror("perf_systemwide_event_open");
	602	+ exit(EXIT_FAILURE);
	603	+ }
	604	+
	605	+ reset_fds(fd1, nprocs);
	606	+ reset_fds(fd2, nprocs);
	607	+ enable_fds(fd1, nprocs);
	608	+ enable_fds(fd2, nprocs);
	609	+ multi_dawr_workload();
	610	+ disable_fds(fd1, nprocs);
	611	+ disable_fds(fd2, nprocs);
	612	+
	613	+ breaks1 = read_fds(fd1, nprocs);
	614	+ breaks2 = read_fds(fd2, nprocs);
	615	+
	616	+ close_fds(fd1, nprocs);
	617	+ close_fds(fd2, nprocs);
	618	+
	619	+ free(fd1);
	620	+ free(fd2);
	621	+
	622	+ if (breaks1 != 2 \|\| breaks2 != 2) {
	623	+ printf("FAILED: %s: %lld != 2 \|\| %lld != 2\n", desc, breaks1, breaks2);
	624	+ return 1;
	625	+ }
	626	+
	627	+ printf("TESTED: %s\n", desc);
	628	+ return 0;
	629	+}
	630	+
	631	+static int test_syswide_multi_diff_addr_ro_wo(void)
	632	+{
	633	+ unsigned long long breaks1 = 0, breaks2 = 0;
	634	+ int fd1 = malloc(nprocs sizeof(int));
	635	+ int fd2 = malloc(nprocs sizeof(int));
	636	+ char *desc = "Systemwide, Two events, diff addr, one is RO, other is WO";
	637	+ int ret;
	638	+
	639	+ ret = perf_systemwide_event_open(fd1, HW_BREAKPOINT_W, (__u64)&a, (__u64)sizeof(a));
	640	+ if (ret) {
	641	+ perror("perf_systemwide_event_open");
	642	+ exit(EXIT_FAILURE);
	643	+ }
	644	+
	645	+ ret = perf_systemwide_event_open(fd2, HW_BREAKPOINT_R, (__u64)&b, (__u64)sizeof(b));
	646	+ if (ret) {
	647	+ close_fds(fd1, nprocs);
	648	+ perror("perf_systemwide_event_open");
	649	+ exit(EXIT_FAILURE);
	650	+ }
	651	+
	652	+ reset_fds(fd1, nprocs);
	653	+ reset_fds(fd2, nprocs);
	654	+ enable_fds(fd1, nprocs);
	655	+ enable_fds(fd2, nprocs);
	656	+ multi_dawr_workload();
	657	+ disable_fds(fd1, nprocs);
	658	+ disable_fds(fd2, nprocs);
	659	+
	660	+ breaks1 = read_fds(fd1, nprocs);
	661	+ breaks2 = read_fds(fd2, nprocs);
	662	+
	663	+ close_fds(fd1, nprocs);
	664	+ close_fds(fd2, nprocs);
	665	+
	666	+ free(fd1);
	667	+ free(fd2);
	668	+
	669	+ if (breaks1 != 1 \|\| breaks2 != 1) {
	670	+ printf("FAILED: %s: %lld != 1 \|\| %lld != 1\n", desc, breaks1, breaks2);
	671	+ return 1;
	672	+ }
	673	+
	674	+ printf("TESTED: %s\n", desc);
	675	+ return 0;
	676	+}
	677	+
	678	+static int test_syswide_multi_same_addr_ro_wo(void)
	679	+{
	680	+ unsigned long long breaks1 = 0, breaks2 = 0;
	681	+ int fd1 = malloc(nprocs sizeof(int));
	682	+ int fd2 = malloc(nprocs sizeof(int));
	683	+ char *desc = "Systemwide, Two events, same addr, one is RO, other is WO";
	684	+ int ret;
	685	+
	686	+ ret = perf_systemwide_event_open(fd1, HW_BREAKPOINT_W, (__u64)&a, (__u64)sizeof(a));
	687	+ if (ret) {
	688	+ perror("perf_systemwide_event_open");
	689	+ exit(EXIT_FAILURE);
	690	+ }
	691	+
	692	+ ret = perf_systemwide_event_open(fd2, HW_BREAKPOINT_R, (__u64)&a, (__u64)sizeof(a));
	693	+ if (ret) {
	694	+ close_fds(fd1, nprocs);
	695	+ perror("perf_systemwide_event_open");
	696	+ exit(EXIT_FAILURE);
	697	+ }
	698	+
	699	+ reset_fds(fd1, nprocs);
	700	+ reset_fds(fd2, nprocs);
	701	+ enable_fds(fd1, nprocs);
	702	+ enable_fds(fd2, nprocs);
	703	+ multi_dawr_workload();
	704	+ disable_fds(fd1, nprocs);
	705	+ disable_fds(fd2, nprocs);
	706	+
	707	+ breaks1 = read_fds(fd1, nprocs);
	708	+ breaks2 = read_fds(fd2, nprocs);
	709	+
	710	+ close_fds(fd1, nprocs);
	711	+ close_fds(fd2, nprocs);
	712	+
	713	+ free(fd1);
	714	+ free(fd2);
	715	+
	716	+ if (breaks1 != 1 \|\| breaks2 != 1) {
	717	+ printf("FAILED: %s: %lld != 1 \|\| %lld != 1\n", desc, breaks1, breaks2);
	718	+ return 1;
	719	+ }
	720	+
	721	+ printf("TESTED: %s\n", desc);
	722	+ return 0;
	723	+}
	724	+
	725	+static int runtest_multi_dawr(void)
	726	+{
	727	+ int ret = 0;
	728	+
	729	+ ret \|= test_process_multi_diff_addr();
	730	+ ret \|= test_process_multi_same_addr();
	731	+ ret \|= test_process_multi_diff_addr_ro_wo();
	732	+ ret \|= test_process_multi_same_addr_ro_wo();
	733	+ ret \|= test_syswide_multi_diff_addr();
	734	+ ret \|= test_syswide_multi_same_addr();
	735	+ ret \|= test_syswide_multi_diff_addr_ro_wo();
	736	+ ret \|= test_syswide_multi_same_addr_ro_wo();
	737	+
	738	+ return ret;
	739	+}
	740	+
	741	+static int runtest_unaligned_512bytes(void)
	742	+{
	743	+ unsigned long long breaks = 0;
	744	+ int fd;
	745	+ char *desc = "Process specific, 512 bytes, unaligned";
	746	+ __u64 addr = (__u64)&c + 8;
	747	+ size_t res;
	748	+
	749	+ fd = perf_process_event_open(HW_BREAKPOINT_RW, addr, 512);
	750	+ if (fd < 0) {
	751	+ perror("perf_process_event_open");
	752	+ exit(EXIT_FAILURE);
	753	+ }
	754	+
	755	+ ioctl(fd, PERF_EVENT_IOC_RESET);
	756	+ ioctl(fd, PERF_EVENT_IOC_ENABLE);
	757	+ multi_dawr_workload();
	758	+ ioctl(fd, PERF_EVENT_IOC_DISABLE);
	759	+
	760	+ res = read(fd, &breaks, sizeof(breaks));
	761	+ assert(res == sizeof(unsigned long long));
	762	+
	763	+ close(fd);
	764	+
	765	+ if (breaks != 2) {
	766	+ printf("FAILED: %s: %lld != 2\n", desc, breaks);
	767	+ return 1;
	768	+ }
	769	+
	770	+ printf("TESTED: %s\n", desc);
	771	+ return 0;
	772	+}
	773	+
	774	+/* There is no perf api to find number of available watchpoints. Use ptrace. */
	775	+static int get_nr_wps(bool *arch_31)
	776	+{
	777	+ struct ppc_debug_info dbginfo;
	778	+ int child_pid;
	779	+
	780	+ child_pid = fork();
	781	+ if (!child_pid) {
	782	+ int ret = ptrace(PTRACE_TRACEME, 0, NULL, 0);
	783	+ if (ret) {
	784	+ perror("PTRACE_TRACEME failed\n");
	785	+ exit(EXIT_FAILURE);
	786	+ }
	787	+ kill(getpid(), SIGUSR1);
	788	+
	789	+ sleep(1);
	790	+ exit(EXIT_SUCCESS);
	791	+ }
	792	+
	793	+ wait(NULL);
	794	+ if (ptrace(PPC_PTRACE_GETHWDBGINFO, child_pid, NULL, &dbginfo)) {
	795	+ perror("Can't get breakpoint info");
	796	+ exit(EXIT_FAILURE);
	797	+ }
	798	+
	799	+ *arch_31 = !!(dbginfo.features & PPC_DEBUG_FEATURE_DATA_BP_ARCH_31);
	800	+ return dbginfo.num_data_bps;
	801	+}
	802	+
266	803	static int runtest(void)
267	804	{
268	805	int rwflag;
269	806	int exclude_user;
270	807	int ret;
	808	+ bool dawr = dawr_supported();
	809	+ bool arch_31 = false;
	810	+ int nr_wps = get_nr_wps(&arch_31);
271	811
272	812	/*
273	813	* perf defines rwflag as two bits read and write and at least
@@ -280,7 +820,7 @@ static int runtest(void) · 6 unmodified lines
280	820	return ret;
281	821
282	822	/* if we have the dawr, we can do an array test */
283		− if (!dawr_supported())
	823	+ if (!dawr)
284	824	continue;
285	825	ret = runtestsingle(rwflag, exclude_user, 1);
286	826	if (ret)
@@ -289,6 +829,19 @@ static int runtest(void) · 2 unmodified lines
289	829	}
290	830
291	831	ret = runtest_dar_outside();
	832	+ if (ret)
	833	+ return ret;
	834	+
	835	+ if (dawr && nr_wps > 1) {
	836	+ nprocs = get_nprocs();
	837	+ ret = runtest_multi_dawr();
	838	+ if (ret)
	839	+ return ret;
	840	+ }
	841	+
	842	+ if (dawr && arch_31)
	843	+ ret = runtest_unaligned_512bytes();
	844	+
292	845	return ret;
293	846	}
294	847

tools/testing/selftests/powerpc/ptrace/ptrace-hwbreak.c+79−0

@@ -194,6 +194,18 @@ static void test_workload(void) · 193 unmodified lines
194	194	big_var[rand() % DAWR_MAX_LEN] = 'a';
195	195	else
196	196	cvar = big_var[rand() % DAWR_MAX_LEN];
	197	+
	198	+ /* PPC_PTRACE_SETHWDEBUG 2, MODE_RANGE, DW ALIGNED, WO test */
	199	+ gstruct.a[rand() % A_LEN] = 'a';
	200	+
	201	+ /* PPC_PTRACE_SETHWDEBUG 2, MODE_RANGE, DW UNALIGNED, RO test */
	202	+ cvar = gstruct.b[rand() % B_LEN];
	203	+
	204	+ /* PPC_PTRACE_SETHWDEBUG 2, MODE_RANGE, DAWR Overlap, WO test */
	205	+ gstruct.a[rand() % A_LEN] = 'a';
	206	+
	207	+ /* PPC_PTRACE_SETHWDEBUG 2, MODE_RANGE, DAWR Overlap, RO test */
	208	+ cvar = gstruct.a[rand() % A_LEN];
197	209	}
198	210
199	211	static void check_success(pid_t child_pid, const char name, const char type,
@@ -417,6 +429,69 @@ static void test_sethwdebug_range_aligned(pid_t child_pid) · 217 unmodified lines
417	429	ptrace_delhwdebug(child_pid, wh);
418	430	}
419	431
	432	+static void test_multi_sethwdebug_range(pid_t child_pid)
	433	+{
	434	+ struct ppc_hw_breakpoint info1, info2;
	435	+ unsigned long wp_addr1, wp_addr2;
	436	+ char *name1 = "PPC_PTRACE_SETHWDEBUG 2, MODE_RANGE, DW ALIGNED";
	437	+ char *name2 = "PPC_PTRACE_SETHWDEBUG 2, MODE_RANGE, DW UNALIGNED";
	438	+ int len1, len2;
	439	+ int wh1, wh2;
	440	+
	441	+ wp_addr1 = (unsigned long)&gstruct.a;
	442	+ wp_addr2 = (unsigned long)&gstruct.b;
	443	+ len1 = A_LEN;
	444	+ len2 = B_LEN;
	445	+ get_ppc_hw_breakpoint(&info1, PPC_BREAKPOINT_TRIGGER_WRITE, wp_addr1, len1);
	446	+ get_ppc_hw_breakpoint(&info2, PPC_BREAKPOINT_TRIGGER_READ, wp_addr2, len2);
	447	+
	448	+ /* PPC_PTRACE_SETHWDEBUG 2, MODE_RANGE, DW ALIGNED, WO test */
	449	+ wh1 = ptrace_sethwdebug(child_pid, &info1);
	450	+
	451	+ /* PPC_PTRACE_SETHWDEBUG 2, MODE_RANGE, DW UNALIGNED, RO test */
	452	+ wh2 = ptrace_sethwdebug(child_pid, &info2);
	453	+
	454	+ ptrace(PTRACE_CONT, child_pid, NULL, 0);
	455	+ check_success(child_pid, name1, "WO", wp_addr1, len1);
	456	+
	457	+ ptrace(PTRACE_CONT, child_pid, NULL, 0);
	458	+ check_success(child_pid, name2, "RO", wp_addr2, len2);
	459	+
	460	+ ptrace_delhwdebug(child_pid, wh1);
	461	+ ptrace_delhwdebug(child_pid, wh2);
	462	+}
	463	+
	464	+static void test_multi_sethwdebug_range_dawr_overlap(pid_t child_pid)
	465	+{
	466	+ struct ppc_hw_breakpoint info1, info2;
	467	+ unsigned long wp_addr1, wp_addr2;
	468	+ char *name = "PPC_PTRACE_SETHWDEBUG 2, MODE_RANGE, DAWR Overlap";
	469	+ int len1, len2;
	470	+ int wh1, wh2;
	471	+
	472	+ wp_addr1 = (unsigned long)&gstruct.a;
	473	+ wp_addr2 = (unsigned long)&gstruct.a;
	474	+ len1 = A_LEN;
	475	+ len2 = A_LEN;
	476	+ get_ppc_hw_breakpoint(&info1, PPC_BREAKPOINT_TRIGGER_WRITE, wp_addr1, len1);
	477	+ get_ppc_hw_breakpoint(&info2, PPC_BREAKPOINT_TRIGGER_READ, wp_addr2, len2);
	478	+
	479	+ /* PPC_PTRACE_SETHWDEBUG 2, MODE_RANGE, DAWR Overlap, WO test */
	480	+ wh1 = ptrace_sethwdebug(child_pid, &info1);
	481	+
	482	+ /* PPC_PTRACE_SETHWDEBUG 2, MODE_RANGE, DAWR Overlap, RO test */
	483	+ wh2 = ptrace_sethwdebug(child_pid, &info2);
	484	+
	485	+ ptrace(PTRACE_CONT, child_pid, NULL, 0);
	486	+ check_success(child_pid, name, "WO", wp_addr1, len1);
	487	+
	488	+ ptrace(PTRACE_CONT, child_pid, NULL, 0);
	489	+ check_success(child_pid, name, "RO", wp_addr2, len2);
	490	+
	491	+ ptrace_delhwdebug(child_pid, wh1);
	492	+ ptrace_delhwdebug(child_pid, wh2);
	493	+}
	494	+
420	495	static void test_sethwdebug_range_unaligned(pid_t child_pid)
421	496	{
422	497	struct ppc_hw_breakpoint info;
@@ -504,6 +579,10 @@ run_tests(pid_t child_pid, struct ppc_debug_info *dbginfo, bool dawr) · 81 unmodified lines
504	579	test_sethwdebug_range_unaligned(child_pid);
505	580	test_sethwdebug_range_unaligned_dar(child_pid);
506	581	test_sethwdebug_dawr_max_range(child_pid);
	582	+ if (dbginfo->num_data_bps > 1) {
	583	+ test_multi_sethwdebug_range(child_pid);
	584	+ test_multi_sethwdebug_range_dawr_overlap(child_pid);
	585	+ }
507	586	}
508	587	}
509	588	}

tools/testing/selftests/powerpc/ptrace/ptrace-perf-hwbreak.c+659−0

@@ -0,0 +1,659 @@
	1	+// SPDX-License-Identifier: GPL-2.0+
	2	+#include <stdio.h>
	3	+#include <string.h>
	4	+#include <signal.h>
	5	+#include <stdlib.h>
	6	+#include <unistd.h>
	7	+#include <errno.h>
	8	+#include <linux/hw_breakpoint.h>
	9	+#include <linux/perf_event.h>
	10	+#include <asm/unistd.h>
	11	+#include <sys/ptrace.h>
	12	+#include <sys/wait.h>
	13	+#include "ptrace.h"
	14	+
	15	+char data[16];
	16	+
	17	+/* Overlapping address range */
	18	+volatile __u64 ptrace_data1 = (__u64 )&data[0];
	19	+volatile __u64 perf_data1 = (__u64 )&data[4];
	20	+
	21	+/* Non-overlapping address range */
	22	+volatile __u64 ptrace_data2 = (__u64 )&data[0];
	23	+volatile __u64 perf_data2 = (__u64 )&data[8];
	24	+
	25	+static unsigned long pid_max_addr(void)
	26	+{
	27	+ FILE *fp;
	28	+ char line, c;
	29	+ char addr[100];
	30	+ size_t len = 0;
	31	+
	32	+ fp = fopen("/proc/kallsyms", "r");
	33	+ if (!fp) {
	34	+ printf("Failed to read /proc/kallsyms. Exiting..\n");
	35	+ exit(EXIT_FAILURE);
	36	+ }
	37	+
	38	+ while (getline(&line, &len, fp) != -1) {
	39	+ if (!strstr(line, "pid_max") \|\| strstr(line, "pid_max_max") \|\|
	40	+ strstr(line, "pid_max_min"))
	41	+ continue;
	42	+
	43	+ strncpy(addr, line, len < 100 ? len : 100);
	44	+ c = strchr(addr, ' ');
	45	+ *c = '\0';
	46	+ return strtoul(addr, &c, 16);
	47	+ }
	48	+ fclose(fp);
	49	+ printf("Could not find pix_max. Exiting..\n");
	50	+ exit(EXIT_FAILURE);
	51	+ return -1;
	52	+}
	53	+
	54	+static void perf_user_event_attr_set(struct perf_event_attr *attr, __u64 addr, __u64 len)
	55	+{
	56	+ memset(attr, 0, sizeof(struct perf_event_attr));
	57	+ attr->type = PERF_TYPE_BREAKPOINT;
	58	+ attr->size = sizeof(struct perf_event_attr);
	59	+ attr->bp_type = HW_BREAKPOINT_R;
	60	+ attr->bp_addr = addr;
	61	+ attr->bp_len = len;
	62	+ attr->exclude_kernel = 1;
	63	+ attr->exclude_hv = 1;
	64	+}
	65	+
	66	+static void perf_kernel_event_attr_set(struct perf_event_attr *attr)
	67	+{
	68	+ memset(attr, 0, sizeof(struct perf_event_attr));
	69	+ attr->type = PERF_TYPE_BREAKPOINT;
	70	+ attr->size = sizeof(struct perf_event_attr);
	71	+ attr->bp_type = HW_BREAKPOINT_R;
	72	+ attr->bp_addr = pid_max_addr();
	73	+ attr->bp_len = sizeof(unsigned long);
	74	+ attr->exclude_user = 1;
	75	+ attr->exclude_hv = 1;
	76	+}
	77	+
	78	+static int perf_cpu_event_open(int cpu, __u64 addr, __u64 len)
	79	+{
	80	+ struct perf_event_attr attr;
	81	+
	82	+ perf_user_event_attr_set(&attr, addr, len);
	83	+ return syscall(__NR_perf_event_open, &attr, -1, cpu, -1, 0);
	84	+}
	85	+
	86	+static int perf_thread_event_open(pid_t child_pid, __u64 addr, __u64 len)
	87	+{
	88	+ struct perf_event_attr attr;
	89	+
	90	+ perf_user_event_attr_set(&attr, addr, len);
	91	+ return syscall(__NR_perf_event_open, &attr, child_pid, -1, -1, 0);
	92	+}
	93	+
	94	+static int perf_thread_cpu_event_open(pid_t child_pid, int cpu, __u64 addr, __u64 len)
	95	+{
	96	+ struct perf_event_attr attr;
	97	+
	98	+ perf_user_event_attr_set(&attr, addr, len);
	99	+ return syscall(__NR_perf_event_open, &attr, child_pid, cpu, -1, 0);
	100	+}
	101	+
	102	+static int perf_thread_kernel_event_open(pid_t child_pid)
	103	+{
	104	+ struct perf_event_attr attr;
	105	+
	106	+ perf_kernel_event_attr_set(&attr);
	107	+ return syscall(__NR_perf_event_open, &attr, child_pid, -1, -1, 0);
	108	+}
	109	+
	110	+static int perf_cpu_kernel_event_open(int cpu)
	111	+{
	112	+ struct perf_event_attr attr;
	113	+
	114	+ perf_kernel_event_attr_set(&attr);
	115	+ return syscall(__NR_perf_event_open, &attr, -1, cpu, -1, 0);
	116	+}
	117	+
	118	+static int child(void)
	119	+{
	120	+ int ret;
	121	+
	122	+ ret = ptrace(PTRACE_TRACEME, 0, NULL, 0);
	123	+ if (ret) {
	124	+ printf("Error: PTRACE_TRACEME failed\n");
	125	+ return 0;
	126	+ }
	127	+ kill(getpid(), SIGUSR1); /* --> parent (SIGUSR1) */
	128	+
	129	+ return 0;
	130	+}
	131	+
	132	+static void ptrace_ppc_hw_breakpoint(struct ppc_hw_breakpoint *info, int type,
	133	+ __u64 addr, int len)
	134	+{
	135	+ info->version = 1;
	136	+ info->trigger_type = type;
	137	+ info->condition_mode = PPC_BREAKPOINT_CONDITION_NONE;
	138	+ info->addr = addr;
	139	+ info->addr2 = addr + len;
	140	+ info->condition_value = 0;
	141	+ if (!len)
	142	+ info->addr_mode = PPC_BREAKPOINT_MODE_EXACT;
	143	+ else
	144	+ info->addr_mode = PPC_BREAKPOINT_MODE_RANGE_INCLUSIVE;
	145	+}
	146	+
	147	+static int ptrace_open(pid_t child_pid, __u64 wp_addr, int len)
	148	+{
	149	+ struct ppc_hw_breakpoint info;
	150	+
	151	+ ptrace_ppc_hw_breakpoint(&info, PPC_BREAKPOINT_TRIGGER_RW, wp_addr, len);
	152	+ return ptrace(PPC_PTRACE_SETHWDEBUG, child_pid, 0, &info);
	153	+}
	154	+
	155	+static int test1(pid_t child_pid)
	156	+{
	157	+ int perf_fd;
	158	+ int ptrace_fd;
	159	+ int ret = 0;
	160	+
	161	+ /* Test:
	162	+ * if (new per thread event by ptrace)
	163	+ * if (existing cpu event by perf)
	164	+ * if (addr range overlaps)
	165	+ * fail;
	166	+ */
	167	+
	168	+ perf_fd = perf_cpu_event_open(0, (__u64)perf_data1, sizeof(*perf_data1));
	169	+ if (perf_fd < 0)
	170	+ return -1;
	171	+
	172	+ ptrace_fd = ptrace_open(child_pid, (__u64)ptrace_data1, sizeof(*ptrace_data1));
	173	+ if (ptrace_fd > 0 \|\| errno != ENOSPC)
	174	+ ret = -1;
	175	+
	176	+ close(perf_fd);
	177	+ return ret;
	178	+}
	179	+
	180	+static int test2(pid_t child_pid)
	181	+{
	182	+ int perf_fd;
	183	+ int ptrace_fd;
	184	+ int ret = 0;
	185	+
	186	+ /* Test:
	187	+ * if (new per thread event by ptrace)
	188	+ * if (existing cpu event by perf)
	189	+ * if (addr range does not overlaps)
	190	+ * allow;
	191	+ */
	192	+
	193	+ perf_fd = perf_cpu_event_open(0, (__u64)perf_data2, sizeof(*perf_data2));
	194	+ if (perf_fd < 0)
	195	+ return -1;
	196	+
	197	+ ptrace_fd = ptrace_open(child_pid, (__u64)ptrace_data2, sizeof(*ptrace_data2));
	198	+ if (ptrace_fd < 0) {
	199	+ ret = -1;
	200	+ goto perf_close;
	201	+ }
	202	+ ptrace(PPC_PTRACE_DELHWDEBUG, child_pid, 0, ptrace_fd);
	203	+
	204	+perf_close:
	205	+ close(perf_fd);
	206	+ return ret;
	207	+}
	208	+
	209	+static int test3(pid_t child_pid)
	210	+{
	211	+ int perf_fd;
	212	+ int ptrace_fd;
	213	+ int ret = 0;
	214	+
	215	+ /* Test:
	216	+ * if (new per thread event by ptrace)
	217	+ * if (existing thread event by perf on the same thread)
	218	+ * if (addr range overlaps)
	219	+ * fail;
	220	+ */
	221	+ perf_fd = perf_thread_event_open(child_pid, (__u64)perf_data1,
	222	+ sizeof(*perf_data1));
	223	+ if (perf_fd < 0)
	224	+ return -1;
	225	+
	226	+ ptrace_fd = ptrace_open(child_pid, (__u64)ptrace_data1, sizeof(*ptrace_data1));
	227	+ if (ptrace_fd > 0 \|\| errno != ENOSPC)
	228	+ ret = -1;
	229	+
	230	+ close(perf_fd);
	231	+ return ret;
	232	+}
	233	+
	234	+static int test4(pid_t child_pid)
	235	+{
	236	+ int perf_fd;
	237	+ int ptrace_fd;
	238	+ int ret = 0;
	239	+
	240	+ /* Test:
	241	+ * if (new per thread event by ptrace)
	242	+ * if (existing thread event by perf on the same thread)
	243	+ * if (addr range does not overlaps)
	244	+ * fail;
	245	+ */
	246	+ perf_fd = perf_thread_event_open(child_pid, (__u64)perf_data2,
	247	+ sizeof(*perf_data2));
	248	+ if (perf_fd < 0)
	249	+ return -1;
	250	+
	251	+ ptrace_fd = ptrace_open(child_pid, (__u64)ptrace_data2, sizeof(*ptrace_data2));
	252	+ if (ptrace_fd < 0) {
	253	+ ret = -1;
	254	+ goto perf_close;
	255	+ }
	256	+ ptrace(PPC_PTRACE_DELHWDEBUG, child_pid, 0, ptrace_fd);
	257	+
	258	+perf_close:
	259	+ close(perf_fd);
	260	+ return ret;
	261	+}
	262	+
	263	+static int test5(pid_t child_pid)
	264	+{
	265	+ int perf_fd;
	266	+ int ptrace_fd;
	267	+ int cpid;
	268	+ int ret = 0;
	269	+
	270	+ /* Test:
	271	+ * if (new per thread event by ptrace)
	272	+ * if (existing thread event by perf on the different thread)
	273	+ * allow;
	274	+ */
	275	+ cpid = fork();
	276	+ if (!cpid) {
	277	+ /* Temporary Child */
	278	+ pause();
	279	+ exit(EXIT_SUCCESS);
	280	+ }
	281	+
	282	+ perf_fd = perf_thread_event_open(cpid, (__u64)perf_data1, sizeof(*perf_data1));
	283	+ if (perf_fd < 0) {
	284	+ ret = -1;
	285	+ goto kill_child;
	286	+ }
	287	+
	288	+ ptrace_fd = ptrace_open(child_pid, (__u64)ptrace_data1, sizeof(*ptrace_data1));
	289	+ if (ptrace_fd < 0) {
	290	+ ret = -1;
	291	+ goto perf_close;
	292	+ }
	293	+
	294	+ ptrace(PPC_PTRACE_DELHWDEBUG, child_pid, 0, ptrace_fd);
	295	+perf_close:
	296	+ close(perf_fd);
	297	+kill_child:
	298	+ kill(cpid, SIGINT);
	299	+ return ret;
	300	+}
	301	+
	302	+static int test6(pid_t child_pid)
	303	+{
	304	+ int perf_fd;
	305	+ int ptrace_fd;
	306	+ int ret = 0;
	307	+
	308	+ /* Test:
	309	+ * if (new per thread kernel event by perf)
	310	+ * if (existing thread event by ptrace on the same thread)
	311	+ * allow;
	312	+ * -- OR --
	313	+ * if (new per cpu kernel event by perf)
	314	+ * if (existing thread event by ptrace)
	315	+ * allow;
	316	+ */
	317	+ ptrace_fd = ptrace_open(child_pid, (__u64)ptrace_data1, sizeof(*ptrace_data1));
	318	+ if (ptrace_fd < 0)
	319	+ return -1;
	320	+
	321	+ perf_fd = perf_thread_kernel_event_open(child_pid);
	322	+ if (perf_fd < 0) {
	323	+ ret = -1;
	324	+ goto ptrace_close;
	325	+ }
	326	+ close(perf_fd);
	327	+
	328	+ perf_fd = perf_cpu_kernel_event_open(0);
	329	+ if (perf_fd < 0) {
	330	+ ret = -1;
	331	+ goto ptrace_close;
	332	+ }
	333	+ close(perf_fd);
	334	+
	335	+ptrace_close:
	336	+ ptrace(PPC_PTRACE_DELHWDEBUG, child_pid, 0, ptrace_fd);
	337	+ return ret;
	338	+}
	339	+
	340	+static int test7(pid_t child_pid)
	341	+{
	342	+ int perf_fd;
	343	+ int ptrace_fd;
	344	+ int ret = 0;
	345	+
	346	+ /* Test:
	347	+ * if (new per thread event by perf)
	348	+ * if (existing thread event by ptrace on the same thread)
	349	+ * if (addr range overlaps)
	350	+ * fail;
	351	+ */
	352	+ ptrace_fd = ptrace_open(child_pid, (__u64)ptrace_data1, sizeof(*ptrace_data1));
	353	+ if (ptrace_fd < 0)
	354	+ return -1;
	355	+
	356	+ perf_fd = perf_thread_event_open(child_pid, (__u64)perf_data1,
	357	+ sizeof(*perf_data1));
	358	+ if (perf_fd > 0 \|\| errno != ENOSPC)
	359	+ ret = -1;
	360	+
	361	+ ptrace(PPC_PTRACE_DELHWDEBUG, child_pid, 0, ptrace_fd);
	362	+ return ret;
	363	+}
	364	+
	365	+static int test8(pid_t child_pid)
	366	+{
	367	+ int perf_fd;
	368	+ int ptrace_fd;
	369	+ int ret = 0;
	370	+
	371	+ /* Test:
	372	+ * if (new per thread event by perf)
	373	+ * if (existing thread event by ptrace on the same thread)
	374	+ * if (addr range does not overlaps)
	375	+ * allow;
	376	+ */
	377	+ ptrace_fd = ptrace_open(child_pid, (__u64)ptrace_data2, sizeof(*ptrace_data2));
	378	+ if (ptrace_fd < 0)
	379	+ return -1;
	380	+
	381	+ perf_fd = perf_thread_event_open(child_pid, (__u64)perf_data2,
	382	+ sizeof(*perf_data2));
	383	+ if (perf_fd < 0) {
	384	+ ret = -1;
	385	+ goto ptrace_close;
	386	+ }
	387	+ close(perf_fd);
	388	+
	389	+ptrace_close:
	390	+ ptrace(PPC_PTRACE_DELHWDEBUG, child_pid, 0, ptrace_fd);
	391	+ return ret;
	392	+}
	393	+
	394	+static int test9(pid_t child_pid)
	395	+{
	396	+ int perf_fd;
	397	+ int ptrace_fd;
	398	+ int cpid;
	399	+ int ret = 0;
	400	+
	401	+ /* Test:
	402	+ * if (new per thread event by perf)
	403	+ * if (existing thread event by ptrace on the other thread)
	404	+ * allow;
	405	+ */
	406	+ ptrace_fd = ptrace_open(child_pid, (__u64)ptrace_data1, sizeof(*ptrace_data1));
	407	+ if (ptrace_fd < 0)
	408	+ return -1;
	409	+
	410	+ cpid = fork();
	411	+ if (!cpid) {
	412	+ /* Temporary Child */
	413	+ pause();
	414	+ exit(EXIT_SUCCESS);
	415	+ }
	416	+
	417	+ perf_fd = perf_thread_event_open(cpid, (__u64)perf_data1, sizeof(*perf_data1));
	418	+ if (perf_fd < 0) {
	419	+ ret = -1;
	420	+ goto kill_child;
	421	+ }
	422	+ close(perf_fd);
	423	+
	424	+kill_child:
	425	+ kill(cpid, SIGINT);
	426	+ ptrace(PPC_PTRACE_DELHWDEBUG, child_pid, 0, ptrace_fd);
	427	+ return ret;
	428	+}
	429	+
	430	+static int test10(pid_t child_pid)
	431	+{
	432	+ int perf_fd;
	433	+ int ptrace_fd;
	434	+ int ret = 0;
	435	+
	436	+ /* Test:
	437	+ * if (new per cpu event by perf)
	438	+ * if (existing thread event by ptrace on the same thread)
	439	+ * if (addr range overlaps)
	440	+ * fail;
	441	+ */
	442	+ ptrace_fd = ptrace_open(child_pid, (__u64)ptrace_data1, sizeof(*ptrace_data1));
	443	+ if (ptrace_fd < 0)
	444	+ return -1;
	445	+
	446	+ perf_fd = perf_cpu_event_open(0, (__u64)perf_data1, sizeof(*perf_data1));
	447	+ if (perf_fd > 0 \|\| errno != ENOSPC)
	448	+ ret = -1;
	449	+
	450	+ ptrace(PPC_PTRACE_DELHWDEBUG, child_pid, 0, ptrace_fd);
	451	+ return ret;
	452	+}
	453	+
	454	+static int test11(pid_t child_pid)
	455	+{
	456	+ int perf_fd;
	457	+ int ptrace_fd;
	458	+ int ret = 0;
	459	+
	460	+ /* Test:
	461	+ * if (new per cpu event by perf)
	462	+ * if (existing thread event by ptrace on the same thread)
	463	+ * if (addr range does not overlap)
	464	+ * allow;
	465	+ */
	466	+ ptrace_fd = ptrace_open(child_pid, (__u64)ptrace_data2, sizeof(*ptrace_data2));
	467	+ if (ptrace_fd < 0)
	468	+ return -1;
	469	+
	470	+ perf_fd = perf_cpu_event_open(0, (__u64)perf_data2, sizeof(*perf_data2));
	471	+ if (perf_fd < 0) {
	472	+ ret = -1;
	473	+ goto ptrace_close;
	474	+ }
	475	+ close(perf_fd);
	476	+
	477	+ptrace_close:
	478	+ ptrace(PPC_PTRACE_DELHWDEBUG, child_pid, 0, ptrace_fd);
	479	+ return ret;
	480	+}
	481	+
	482	+static int test12(pid_t child_pid)
	483	+{
	484	+ int perf_fd;
	485	+ int ptrace_fd;
	486	+ int ret = 0;
	487	+
	488	+ /* Test:
	489	+ * if (new per thread and per cpu event by perf)
	490	+ * if (existing thread event by ptrace on the same thread)
	491	+ * if (addr range overlaps)
	492	+ * fail;
	493	+ */
	494	+ ptrace_fd = ptrace_open(child_pid, (__u64)ptrace_data1, sizeof(*ptrace_data1));
	495	+ if (ptrace_fd < 0)
	496	+ return -1;
	497	+
	498	+ perf_fd = perf_thread_cpu_event_open(child_pid, 0, (__u64)perf_data1, sizeof(*perf_data1));
	499	+ if (perf_fd > 0 \|\| errno != ENOSPC)
	500	+ ret = -1;
	501	+
	502	+ ptrace(PPC_PTRACE_DELHWDEBUG, child_pid, 0, ptrace_fd);
	503	+ return ret;
	504	+}
	505	+
	506	+static int test13(pid_t child_pid)
	507	+{
	508	+ int perf_fd;
	509	+ int ptrace_fd;
	510	+ int ret = 0;
	511	+
	512	+ /* Test:
	513	+ * if (new per thread and per cpu event by perf)
	514	+ * if (existing thread event by ptrace on the same thread)
	515	+ * if (addr range does not overlap)
	516	+ * allow;
	517	+ */
	518	+ ptrace_fd = ptrace_open(child_pid, (__u64)ptrace_data2, sizeof(*ptrace_data2));
	519	+ if (ptrace_fd < 0)
	520	+ return -1;
	521	+
	522	+ perf_fd = perf_thread_cpu_event_open(child_pid, 0, (__u64)perf_data2, sizeof(*perf_data2));
	523	+ if (perf_fd < 0) {
	524	+ ret = -1;
	525	+ goto ptrace_close;
	526	+ }
	527	+ close(perf_fd);
	528	+
	529	+ptrace_close:
	530	+ ptrace(PPC_PTRACE_DELHWDEBUG, child_pid, 0, ptrace_fd);
	531	+ return ret;
	532	+}
	533	+
	534	+static int test14(pid_t child_pid)
	535	+{
	536	+ int perf_fd;
	537	+ int ptrace_fd;
	538	+ int cpid;
	539	+ int ret = 0;
	540	+
	541	+ /* Test:
	542	+ * if (new per thread and per cpu event by perf)
	543	+ * if (existing thread event by ptrace on the other thread)
	544	+ * allow;
	545	+ */
	546	+ ptrace_fd = ptrace_open(child_pid, (__u64)ptrace_data1, sizeof(*ptrace_data1));
	547	+ if (ptrace_fd < 0)
	548	+ return -1;
	549	+
	550	+ cpid = fork();
	551	+ if (!cpid) {
	552	+ /* Temporary Child */
	553	+ pause();
	554	+ exit(EXIT_SUCCESS);
	555	+ }
	556	+
	557	+ perf_fd = perf_thread_cpu_event_open(cpid, 0, (__u64)perf_data1,
	558	+ sizeof(*perf_data1));
	559	+ if (perf_fd < 0) {
	560	+ ret = -1;
	561	+ goto kill_child;
	562	+ }
	563	+ close(perf_fd);
	564	+
	565	+kill_child:
	566	+ kill(cpid, SIGINT);
	567	+ ptrace(PPC_PTRACE_DELHWDEBUG, child_pid, 0, ptrace_fd);
	568	+ return ret;
	569	+}
	570	+
	571	+static int do_test(const char msg, int (fun)(pid_t arg), pid_t arg)
	572	+{
	573	+ int ret;
	574	+
	575	+ ret = fun(arg);
	576	+ if (ret)
	577	+ printf("%s: Error\n", msg);
	578	+ else
	579	+ printf("%s: Ok\n", msg);
	580	+ return ret;
	581	+}
	582	+
	583	+char *desc[14] = {
	584	+ "perf cpu event -> ptrace thread event (Overlapping)",
	585	+ "perf cpu event -> ptrace thread event (Non-overlapping)",
	586	+ "perf thread event -> ptrace same thread event (Overlapping)",
	587	+ "perf thread event -> ptrace same thread event (Non-overlapping)",
	588	+ "perf thread event -> ptrace other thread event",
	589	+ "ptrace thread event -> perf kernel event",
	590	+ "ptrace thread event -> perf same thread event (Overlapping)",
	591	+ "ptrace thread event -> perf same thread event (Non-overlapping)",
	592	+ "ptrace thread event -> perf other thread event",
	593	+ "ptrace thread event -> perf cpu event (Overlapping)",
	594	+ "ptrace thread event -> perf cpu event (Non-overlapping)",
	595	+ "ptrace thread event -> perf same thread & cpu event (Overlapping)",
	596	+ "ptrace thread event -> perf same thread & cpu event (Non-overlapping)",
	597	+ "ptrace thread event -> perf other thread & cpu event",
	598	+};
	599	+
	600	+static int test(pid_t child_pid)
	601	+{
	602	+ int ret = TEST_PASS;
	603	+
	604	+ ret \|= do_test(desc[0], test1, child_pid);
	605	+ ret \|= do_test(desc[1], test2, child_pid);
	606	+ ret \|= do_test(desc[2], test3, child_pid);
	607	+ ret \|= do_test(desc[3], test4, child_pid);
	608	+ ret \|= do_test(desc[4], test5, child_pid);
	609	+ ret \|= do_test(desc[5], test6, child_pid);
	610	+ ret \|= do_test(desc[6], test7, child_pid);
	611	+ ret \|= do_test(desc[7], test8, child_pid);
	612	+ ret \|= do_test(desc[8], test9, child_pid);
	613	+ ret \|= do_test(desc[9], test10, child_pid);
	614	+ ret \|= do_test(desc[10], test11, child_pid);
	615	+ ret \|= do_test(desc[11], test12, child_pid);
	616	+ ret \|= do_test(desc[12], test13, child_pid);
	617	+ ret \|= do_test(desc[13], test14, child_pid);
	618	+
	619	+ return ret;
	620	+}
	621	+
	622	+static void get_dbginfo(pid_t child_pid, struct ppc_debug_info *dbginfo)
	623	+{
	624	+ if (ptrace(PPC_PTRACE_GETHWDBGINFO, child_pid, NULL, dbginfo)) {
	625	+ perror("Can't get breakpoint info");
	626	+ exit(-1);
	627	+ }
	628	+}
	629	+
	630	+static int ptrace_perf_hwbreak(void)
	631	+{
	632	+ int ret;
	633	+ pid_t child_pid;
	634	+ struct ppc_debug_info dbginfo;
	635	+
	636	+ child_pid = fork();
	637	+ if (!child_pid)
	638	+ return child();
	639	+
	640	+ /* parent */
	641	+ wait(NULL); /* <-- child (SIGUSR1) */
	642	+
	643	+ get_dbginfo(child_pid, &dbginfo);
	644	+ SKIP_IF(dbginfo.num_data_bps <= 1);
	645	+
	646	+ ret = perf_cpu_event_open(0, (__u64)perf_data1, sizeof(*perf_data1));
	647	+ SKIP_IF(ret < 0);
	648	+ close(ret);
	649	+
	650	+ ret = test(child_pid);
	651	+
	652	+ ptrace(PTRACE_CONT, child_pid, NULL, 0);
	653	+ return ret;
	654	+}
	655	+
	656	+int main(int argc, char *argv[])
	657	+{
	658	+ return test_harness(ptrace_perf_hwbreak, "ptrace-perf-hwbreak");
	659	+}

tools/testing/selftests/powerpc/security/Makefile+2−1

@@ -1,6 +1,6 @@
1	1	# SPDX-License-Identifier: GPL-2.0+
2	2
3		−TEST_GEN_PROGS := rfi_flush entry_flush spectre_v2
	3	+TEST_GEN_PROGS := rfi_flush entry_flush uaccess_flush spectre_v2
4	4	top_srcdir = ../../../../..
5	5
6	6	CFLAGS += -I../../../../../usr/include
@@ -13,3 +13,4 @@ $(OUTPUT)/spectre_v2: CFLAGS += -m64 · 6 unmodified lines
13	13	$(OUTPUT)/spectre_v2: ../pmu/event.c branch_loops.S
14	14	$(OUTPUT)/rfi_flush: flush_utils.c
15	15	$(OUTPUT)/entry_flush: flush_utils.c
	16	+$(OUTPUT)/uaccess_flush: flush_utils.c

tools/testing/selftests/powerpc/security/entry_flush.c+1−1

@@ -53,7 +53,7 @@ int entry_flush_test(void) · 52 unmodified lines
53	53
54	54	entry_flush = entry_flush_orig;
55	55
56		− fd = perf_event_open_counter(PERF_TYPE_RAW, /* L1d miss */ 0x400f0, -1);
	56	+ fd = perf_event_open_counter(PERF_TYPE_HW_CACHE, PERF_L1D_READ_MISS_CONFIG, -1);
57	57	FAIL_IF(fd < 0);
58	58
59	59	p = (char *)memalign(zero_size, CACHELINE_SIZE);

tools/testing/selftests/powerpc/security/flush_utils.c+13−0

@@ -13,6 +13,7 @@ · 12 unmodified lines
13	13	#include <stdlib.h>
14	14	#include <string.h>
15	15	#include <stdio.h>
	16	+#include <sys/utsname.h>
16	17	#include "utils.h"
17	18	#include "flush_utils.h"
18	19
@@ -35,6 +36,18 @@ void syscall_loop(char *p, unsigned long iterations, · 16 unmodified lines
35	36	}
36	37	}
37	38
	39	+void syscall_loop_uaccess(char *p, unsigned long iterations,
	40	+ unsigned long zero_size)
	41	+{
	42	+ struct utsname utsname;
	43	+
	44	+ for (unsigned long i = 0; i < iterations; i++) {
	45	+ for (unsigned long j = 0; j < zero_size; j += CACHELINE_SIZE)
	46	+ load(p + j);
	47	+ uname(&utsname);
	48	+ }
	49	+}
	50	+
38	51	static void sigill_handler(int signr, siginfo_t info, void unused)
39	52	{
40	53	static int warned;

tools/testing/selftests/powerpc/security/flush_utils.h+7−0

@@ -9,9 +9,16 @@ · 8 unmodified lines
9	9
10	10	#define CACHELINE_SIZE 128
11	11
	12	+#define PERF_L1D_READ_MISS_CONFIG ((PERF_COUNT_HW_CACHE_L1D) \| \
	13	+ (PERF_COUNT_HW_CACHE_OP_READ << 8) \| \
	14	+ (PERF_COUNT_HW_CACHE_RESULT_MISS << 16))
	15	+
12	16	void syscall_loop(char *p, unsigned long iterations,
13	17	unsigned long zero_size);
14	18
	19	+void syscall_loop_uaccess(char *p, unsigned long iterations,
	20	+ unsigned long zero_size);
	21	+
15	22	void set_dscr(unsigned long val);
16	23
17	24	#endif /* _SELFTESTS_POWERPC_SECURITY_FLUSH_UTILS_H */

tools/testing/selftests/powerpc/security/rfi_flush.c+1−1

@@ -54,7 +54,7 @@ int rfi_flush_test(void) · 53 unmodified lines
54	54
55	55	rfi_flush = rfi_flush_orig;
56	56
57		− fd = perf_event_open_counter(PERF_TYPE_RAW, /* L1d miss */ 0x400f0, -1);
	57	+ fd = perf_event_open_counter(PERF_TYPE_HW_CACHE, PERF_L1D_READ_MISS_CONFIG, -1);
58	58	FAIL_IF(fd < 0);
59	59
60	60	p = (char *)memalign(zero_size, CACHELINE_SIZE);

tools/testing/selftests/powerpc/security/uaccess_flush.c+158−0

@@ -0,0 +1,158 @@
	1	+// SPDX-License-Identifier: GPL-2.0+
	2	+
	3	+/*
	4	+ * Copyright 2018 IBM Corporation.
	5	+ * Copyright 2020 Canonical Ltd.
	6	+ */
	7	+
	8	+#define __SANE_USERSPACE_TYPES__
	9	+
	10	+#include <sys/types.h>
	11	+#include <stdint.h>
	12	+#include <malloc.h>
	13	+#include <unistd.h>
	14	+#include <signal.h>
	15	+#include <stdlib.h>
	16	+#include <string.h>
	17	+#include <stdio.h>
	18	+#include "utils.h"
	19	+#include "flush_utils.h"
	20	+
	21	+int uaccess_flush_test(void)
	22	+{
	23	+ char *p;
	24	+ int repetitions = 10;
	25	+ int fd, passes = 0, iter, rc = 0;
	26	+ struct perf_event_read v;
	27	+ __u64 l1d_misses_total = 0;
	28	+ unsigned long iterations = 100000, zero_size = 24 * 1024;
	29	+ unsigned long l1d_misses_expected;
	30	+ int rfi_flush_orig;
	31	+ int entry_flush_orig;
	32	+ int uaccess_flush, uaccess_flush_orig;
	33	+
	34	+ SKIP_IF(geteuid() != 0);
	35	+
	36	+ // The PMU event we use only works on Power7 or later
	37	+ SKIP_IF(!have_hwcap(PPC_FEATURE_ARCH_2_06));
	38	+
	39	+ if (read_debugfs_file("powerpc/rfi_flush", &rfi_flush_orig) < 0) {
	40	+ perror("Unable to read powerpc/rfi_flush debugfs file");
	41	+ SKIP_IF(1);
	42	+ }
	43	+
	44	+ if (read_debugfs_file("powerpc/entry_flush", &entry_flush_orig) < 0) {
	45	+ perror("Unable to read powerpc/entry_flush debugfs file");
	46	+ SKIP_IF(1);
	47	+ }
	48	+
	49	+ if (read_debugfs_file("powerpc/uaccess_flush", &uaccess_flush_orig) < 0) {
	50	+ perror("Unable to read powerpc/entry_flush debugfs file");
	51	+ SKIP_IF(1);
	52	+ }
	53	+
	54	+ if (rfi_flush_orig != 0) {
	55	+ if (write_debugfs_file("powerpc/rfi_flush", 0) < 0) {
	56	+ perror("error writing to powerpc/rfi_flush debugfs file");
	57	+ FAIL_IF(1);
	58	+ }
	59	+ }
	60	+
	61	+ if (entry_flush_orig != 0) {
	62	+ if (write_debugfs_file("powerpc/entry_flush", 0) < 0) {
	63	+ perror("error writing to powerpc/entry_flush debugfs file");
	64	+ FAIL_IF(1);
	65	+ }
	66	+ }
	67	+
	68	+ uaccess_flush = uaccess_flush_orig;
	69	+
	70	+ fd = perf_event_open_counter(PERF_TYPE_HW_CACHE, PERF_L1D_READ_MISS_CONFIG, -1);
	71	+ FAIL_IF(fd < 0);
	72	+
	73	+ p = (char *)memalign(zero_size, CACHELINE_SIZE);
	74	+
	75	+ FAIL_IF(perf_event_enable(fd));
	76	+
	77	+ // disable L1 prefetching
	78	+ set_dscr(1);
	79	+
	80	+ iter = repetitions;
	81	+
	82	+ /*
	83	+ * We expect to see l1d miss for each cacheline access when entry_flush
	84	+ * is set. Allow a small variation on this.
	85	+ */
	86	+ l1d_misses_expected = iterations * (zero_size / CACHELINE_SIZE - 2);
	87	+
	88	+again:
	89	+ FAIL_IF(perf_event_reset(fd));
	90	+
	91	+ syscall_loop_uaccess(p, iterations, zero_size);
	92	+
	93	+ FAIL_IF(read(fd, &v, sizeof(v)) != sizeof(v));
	94	+
	95	+ if (uaccess_flush && v.l1d_misses >= l1d_misses_expected)
	96	+ passes++;
	97	+ else if (!uaccess_flush && v.l1d_misses < (l1d_misses_expected / 2))
	98	+ passes++;
	99	+
	100	+ l1d_misses_total += v.l1d_misses;
	101	+
	102	+ while (--iter)
	103	+ goto again;
	104	+
	105	+ if (passes < repetitions) {
	106	+ printf("FAIL (L1D misses with uaccess_flush=%d: %llu %c %lu) [%d/%d failures]\n",
	107	+ uaccess_flush, l1d_misses_total, uaccess_flush ? '<' : '>',
	108	+ uaccess_flush ? repetitions * l1d_misses_expected :
	109	+ repetitions * l1d_misses_expected / 2,
	110	+ repetitions - passes, repetitions);
	111	+ rc = 1;
	112	+ } else {
	113	+ printf("PASS (L1D misses with uaccess_flush=%d: %llu %c %lu) [%d/%d pass]\n",
	114	+ uaccess_flush, l1d_misses_total, uaccess_flush ? '>' : '<',
	115	+ uaccess_flush ? repetitions * l1d_misses_expected :
	116	+ repetitions * l1d_misses_expected / 2,
	117	+ passes, repetitions);
	118	+ }
	119	+
	120	+ if (uaccess_flush == uaccess_flush_orig) {
	121	+ uaccess_flush = !uaccess_flush_orig;
	122	+ if (write_debugfs_file("powerpc/uaccess_flush", uaccess_flush) < 0) {
	123	+ perror("error writing to powerpc/uaccess_flush debugfs file");
	124	+ return 1;
	125	+ }
	126	+ iter = repetitions;
	127	+ l1d_misses_total = 0;
	128	+ passes = 0;
	129	+ goto again;
	130	+ }
	131	+
	132	+ perf_event_disable(fd);
	133	+ close(fd);
	134	+
	135	+ set_dscr(0);
	136	+
	137	+ if (write_debugfs_file("powerpc/rfi_flush", rfi_flush_orig) < 0) {
	138	+ perror("unable to restore original value of powerpc/rfi_flush debugfs file");
	139	+ return 1;
	140	+ }
	141	+
	142	+ if (write_debugfs_file("powerpc/entry_flush", entry_flush_orig) < 0) {
	143	+ perror("unable to restore original value of powerpc/entry_flush debugfs file");
	144	+ return 1;
	145	+ }
	146	+
	147	+ if (write_debugfs_file("powerpc/uaccess_flush", uaccess_flush_orig) < 0) {
	148	+ perror("unable to restore original value of powerpc/uaccess_flush debugfs file");
	149	+ return 1;
	150	+ }
	151	+
	152	+ return rc;
	153	+}
	154	+
	155	+int main(int argc, char *argv[])
	156	+{
	157	+ return test_harness(uaccess_flush_test, "uaccess_flush_test");
	158	+}

tools/testing/selftests/powerpc/tm/tm-trap.c+2−2

@@ -66,7 +66,7 @@ void trap_signal_handler(int signo, siginfo_t si, void uc) · 65 unmodified lines
66	66	/* Get thread endianness: extract bit LE from MSR */
67	67	thread_endianness = MSR_LE & ucp->uc_mcontext.gp_regs[PT_MSR];
68	68
69		− /***
	69	+ /*
70	70	* Little-Endian Machine
71	71	*/
72	72
@@ -126,7 +126,7 @@ void trap_signal_handler(int signo, siginfo_t si, void uc) · 53 unmodified lines
126	126	}
127	127	}
128	128
129		− /***
	129	+ /*
130	130	* Big-Endian Machine
131	131	*/
132	132

tools/testing/selftests/timens/gettime_perf.c+8−0

@@ -25,12 +25,20 @@ static void fill_function_pointers(void) · 24 unmodified lines
25	25	if (!vdso)
26	26	vdso = dlopen("linux-gate.so.1",
27	27	RTLD_LAZY \| RTLD_LOCAL \| RTLD_NOLOAD);
	28	+ if (!vdso)
	29	+ vdso = dlopen("linux-vdso32.so.1",
	30	+ RTLD_LAZY \| RTLD_LOCAL \| RTLD_NOLOAD);
	31	+ if (!vdso)
	32	+ vdso = dlopen("linux-vdso64.so.1",
	33	+ RTLD_LAZY \| RTLD_LOCAL \| RTLD_NOLOAD);
28	34	if (!vdso) {
29	35	pr_err("[WARN]\tfailed to find vDSO\n");
30	36	return;
31	37	}
32	38
33	39	vdso_clock_gettime = (vgettime_t)dlsym(vdso, "__vdso_clock_gettime");
	40	+ if (!vdso_clock_gettime)
	41	+ vdso_clock_gettime = (vgettime_t)dlsym(vdso, "__kernel_clock_gettime");
34	42	if (!vdso_clock_gettime)
35	43	pr_err("Warning: failed to find clock_gettime in vDSO\n");
36	44

Merge tag 'powerpc-5.13-1' of git://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code