Merge tag 'arm64-upstream' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux

Pull arm64 updates from Will Deacon: "The highlights this time are support for dynamically enabling and disabling Clang's Shadow Call Stack at boot and a long-awaited optimisation to the way in which we handle the SVE register state on system call entry to avoid taking unnecessary traps from userspace. Summary: ACPI: - Enable FPDT support for boot-time profiling - Fix CPU PMU probing to work better with PREEMPT_RT - Update SMMUv3 MSI DeviceID parsing to latest IORT spec - APMT support for probing Arm CoreSight PMU devices CPU features: - Advertise new SVE instructions (v2.1) - Advertise range prefetch instruction - Advertise CSSC ("Common Short Sequence Compression") scalar instructions, adding things like min, max, abs, popcount - Enable DIT (Data Independent Timing) when running in the kernel - More conversion of system register fields over to the generated header CPU misfeatures: - Workaround for Cortex-A715 erratum #2645198 Dynamic SCS: - Support for dynamic shadow call stacks to allow switching at runtime between Clang's SCS implementation and the CPU's pointer authentication feature when it is supported (complete with scary DWARF parser!) Tracing and debug: - Remove static ftrace in favour of, err, dynamic ftrace! - Seperate 'struct ftrace_regs' from 'struct pt_regs' in core ftrace and existing arch code - Introduce and implement FTRACE_WITH_ARGS on arm64 to replace the old FTRACE_WITH_REGS - Extend 'crashkernel=' parameter with default value and fallback to placement above 4G physical if initial (low) allocation fails SVE: - Optimisation to avoid disabling SVE unconditionally on syscall entry and just zeroing the non-shared state on return instead Exceptions: - Rework of undefined instruction handling to avoid serialisation on global lock (this includes emulation of user accesses to the ID registers) Perf and PMU: - Support for TLP filters in Hisilicon's PCIe PMU device - Support for the DDR PMU present in Amlogic Meson G12 SoCs - Support for the terribly-named "CoreSight PMU" architecture from Arm (and Nvidia's implementation of said architecture) Misc: - Tighten up our boot protocol for systems with memory above 52 bits physical - Const-ify static keys to satisty jump label asm constraints - Trivial FFA driver cleanups in preparation for v1.1 support - Export the kernel_neon_* APIs as GPL symbols - Harden our instruction generation routines against instrumentation - A bunch of robustness improvements to our arch-specific selftests - Minor cleanups and fixes all over (kbuild, kprobes, kfence, PMU, ...)" * tag 'arm64-upstream' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux: (151 commits) arm64: kprobes: Return DBG_HOOK_ERROR if kprobes can not handle a BRK arm64: kprobes: Let arch do_page_fault() fix up page fault in user handler arm64: Prohibit instrumentation on arch_stack_walk() arm64:uprobe fix the uprobe SWBP_INSN in big-endian arm64: alternatives: add __init/__initconst to some functions/variables arm_pmu: Drop redundant armpmu->map_event() in armpmu_event_init() kselftest/arm64: Allow epoll_wait() to return more than one result kselftest/arm64: Don't drain output while spawning children kselftest/arm64: Hold fp-stress children until they're all spawned arm64/sysreg: Remove duplicate definitions from asm/sysreg.h arm64/sysreg: Convert ID_DFR1_EL1 to automatic generation arm64/sysreg: Convert ID_DFR0_EL1 to automatic generation arm64/sysreg: Convert ID_AFR0_EL1 to automatic generation arm64/sysreg: Convert ID_MMFR5_EL1 to automatic generation arm64/sysreg: Convert MVFR2_EL1 to automatic generation arm64/sysreg: Convert MVFR1_EL1 to automatic generation arm64/sysreg: Convert MVFR0_EL1 to automatic generation arm64/sysreg: Convert ID_PFR2_EL1 to automatic generation arm64/sysreg: Convert ID_PFR1_EL1 to automatic generation arm64/sysreg: Convert ID_PFR0_EL1 to automatic generation ...

Linus Torvalds committed 3 years ago

Commit 06cff4a2 parents 164f590 5f4c374138 files · +6535 −1583Browse files

Unified · Split

138 files

Documentation/admin-guide/kernel-parameters.txt +6 −9
Documentation/admin-guide/perf/hisi-pcie-pmu.rst +68 −44
Documentation/admin-guide/perf/index.rst +2 −0
Documentation/admin-guide/perf/meson-ddr-pmu.rst +70 −0
Documentation/admin-guide/perf/nvidia-pmu.rst +299 −0
Documentation/arm64/acpi_object_usage.rst +1 −1
Documentation/arm64/booting.rst +4 −3
Documentation/arm64/elf_hwcaps.rst +9 −0
Documentation/arm64/silicon-errata.rst +2 −0
Documentation/arm64/sve.rst +1 −0
Documentation/devicetree/bindings/perf/amlogic,g12-ddr-pmu.yaml +54 −0
MAINTAINERS +11 −1
Makefile +2 −0
arch/Kconfig +7 −0
arch/arm64/Kconfig +40 −9
arch/arm64/Makefile +14 −3
arch/arm64/include/asm/alternative-macros.h +2 −2
arch/arm64/include/asm/assembler.h +3 −30
arch/arm64/include/asm/cpufeature.h +2 −1
arch/arm64/include/asm/cputype.h +2 −0
arch/arm64/include/asm/exception.h +4 −3
arch/arm64/include/asm/fpsimd.h +13 −4
arch/arm64/include/asm/ftrace.h +66 −6
arch/arm64/include/asm/hugetlb.h +9 −0
arch/arm64/include/asm/hwcap.h +3 −0
arch/arm64/include/asm/insn.h +109 −47
arch/arm64/include/asm/jump_label.h +4 −4
arch/arm64/include/asm/kernel-pgtable.h +3 −8
arch/arm64/include/asm/kvm_host.h +11 −1
arch/arm64/include/asm/lse.h +0 −1
arch/arm64/include/asm/mmu_context.h +10 −0
arch/arm64/include/asm/module.lds.h +8 −0
arch/arm64/include/asm/pgtable-hwdef.h +1 −0
arch/arm64/include/asm/pgtable.h +11 −3
arch/arm64/include/asm/processor.h +9 −15
arch/arm64/include/asm/scs.h +49 −0
arch/arm64/include/asm/spectre.h +2 −0
arch/arm64/include/asm/stacktrace.h +2 −0
arch/arm64/include/asm/sysreg.h +8 −142
arch/arm64/include/asm/traps.h +9 −10
arch/arm64/include/asm/uprobes.h +1 −1
arch/arm64/include/uapi/asm/hwcap.h +3 −0
arch/arm64/include/uapi/asm/sigcontext.h +4 −0
arch/arm64/kernel/Makefile +2 −0
arch/arm64/kernel/alternative.c +3 −3
arch/arm64/kernel/armv8_deprecated.c +276 −291
arch/arm64/kernel/asm-offsets.c +13 −0
arch/arm64/kernel/cpu_errata.c +7 −0
arch/arm64/kernel/cpufeature.c +131 −122
arch/arm64/kernel/cpuinfo.c +3 −0
arch/arm64/kernel/entry-common.c +12 −12
arch/arm64/kernel/entry-ftrace.S +46 −110
arch/arm64/kernel/entry.S +3 −0
arch/arm64/kernel/fpsimd.c +117 −52
arch/arm64/kernel/ftrace.c +48 −39
arch/arm64/kernel/head.S +3 −0
arch/arm64/kernel/irq.c +6 −5
arch/arm64/kernel/module.c +8 −3
arch/arm64/kernel/paravirt.c +0 −4
arch/arm64/kernel/patch-scs.c +257 −0
arch/arm64/kernel/perf_event.c +2 −1
arch/arm64/kernel/pi/Makefile +1 −0
arch/arm64/kernel/probes/decode-insn.c +1 −1
arch/arm64/kernel/probes/kprobes.c +36 −50
arch/arm64/kernel/process.c +2 −0
arch/arm64/kernel/proton-pack.c +7 −19
arch/arm64/kernel/ptrace.c +3 −2
arch/arm64/kernel/sdei.c +1 −1
arch/arm64/kernel/setup.c +4 −0
arch/arm64/kernel/signal.c +6 −1
arch/arm64/kernel/stacktrace.c +5 −5
arch/arm64/kernel/suspend.c +2 −0
arch/arm64/kernel/syscall.c +5 −14
arch/arm64/kernel/traps.c +36 −57
arch/arm64/kernel/vmlinux.lds.S +13 −0
arch/arm64/kvm/fpsimd.c +18 −8
arch/arm64/kvm/hyp/nvhe/Makefile +1 −0
arch/arm64/kvm/sys_regs.c +2 −2
arch/arm64/lib/insn.c +0 −165
arch/arm64/lib/mte.S +1 −1
arch/arm64/mm/fault.c +7 −1
arch/arm64/mm/hugetlbpage.c +21 −0
arch/arm64/mm/init.c +22 −3
arch/arm64/mm/mmu.c +22 −1
arch/arm64/mm/proc.S +0 −4
arch/arm64/tools/cpucaps +2 −0
arch/arm64/tools/gen-sysreg.awk +1 −1
arch/arm64/tools/sysreg +765 −1
arch/powerpc/include/asm/ftrace.h +22 −2
arch/s390/include/asm/ftrace.h +26 −3
arch/x86/include/asm/ftrace.h +35 −14
drivers/acpi/Kconfig +1 −1
drivers/acpi/arm64/Kconfig +3 −0
drivers/acpi/arm64/Makefile +1 −0
drivers/acpi/arm64/apmt.c +178 −0
drivers/acpi/arm64/iort.c +12 −4
drivers/acpi/bus.c +2 −0
drivers/firmware/arm_ffa/driver.c +10 −91
drivers/firmware/efi/libstub/Makefile +1 −0
drivers/perf/Kconfig +4 −0
drivers/perf/Makefile +2 −0
drivers/perf/amlogic/Kconfig +10 −0
drivers/perf/amlogic/Makefile +5 −0
drivers/perf/amlogic/meson_ddr_pmu_core.c +561 −0
drivers/perf/amlogic/meson_g12_ddr_pmu.c +394 −0
drivers/perf/arm_cspmu/Kconfig +13 −0
drivers/perf/arm_cspmu/Makefile +6 −0
drivers/perf/arm_cspmu/arm_cspmu.c +1303 −0
drivers/perf/arm_cspmu/arm_cspmu.h +151 −0
drivers/perf/arm_cspmu/nvidia_cspmu.c +400 −0
drivers/perf/arm_cspmu/nvidia_cspmu.h +17 −0
drivers/perf/arm_dmc620_pmu.c +7 −1
drivers/perf/arm_dsu_pmu.c +5 −1
drivers/perf/arm_pmu.c +3 −17
drivers/perf/arm_pmu_acpi.c +67 −47
drivers/perf/arm_smmuv3_pmu.c +7 −1
drivers/perf/hisilicon/hisi_pcie_pmu.c +17 −5
drivers/perf/marvell_cn10k_tad_pmu.c +5 −1
include/asm-generic/vmlinux.lds.h +7 −2
include/linux/acpi_apmt.h +19 −0
include/linux/arm_ffa.h +84 −1
include/linux/ftrace.h +37 −10
include/linux/perf/arm_pmu.h +0 −1
include/linux/scs.h +18 −0
include/soc/amlogic/meson_ddr_pmu.h +66 −0
kernel/livepatch/patch.c +1 −1
kernel/scs.c +12 −2
kernel/trace/Kconfig +3 −3
kernel/trace/ftrace.c +1 −2
scripts/head-object-list.txt +0 −1
scripts/module.lds.S +6 −0
tools/testing/selftests/arm64/abi/hwcap.c +32 −0
tools/testing/selftests/arm64/abi/syscall-abi-asm.S +2 −2
tools/testing/selftests/arm64/fp/fp-stress.c +93 −27
tools/testing/selftests/arm64/mte/check_buffer_fill.c +6 −6
tools/testing/selftests/arm64/mte/check_mmap_options.c +4 −5
tools/testing/selftests/arm64/signal/testcases/TODO +0 −1
tools/testing/selftests/arm64/signal/testcases/testcases.c +17 −4

Documentation/admin-guide/kernel-parameters.txt+6−9

@@ -831,7 +831,7 @@ · 830 unmodified lines
831	831	memory region [offset, offset + size] for that kernel
832	832	image. If '@offset' is omitted, then a suitable offset
833	833	is selected automatically.
834		− [KNL, X86-64] Select a region under 4G first, and
	834	+ [KNL, X86-64, ARM64] Select a region under 4G first, and
835	835	fall back to reserve region above 4G when '@offset'
836	836	hasn't been specified.
837	837	See Documentation/admin-guide/kdump/kdump.rst for further details.
@@ -851,26 +851,23 @@ · 13 unmodified lines
851	851	available.
852	852	It will be ignored if crashkernel=X is specified.
853	853	crashkernel=size[KMG],low
854		− [KNL, X86-64] range under 4G. When crashkernel=X,high
	854	+ [KNL, X86-64, ARM64] range under 4G. When crashkernel=X,high
855	855	is passed, kernel could allocate physical memory region
856	856	above 4G, that cause second kernel crash on system
857	857	that require some amount of low memory, e.g. swiotlb
858	858	requires at least 64M+32K low memory, also enough extra
859	859	low memory is needed to make sure DMA buffers for 32-bit
860	860	devices won't run out. Kernel would try to allocate
861		− at least 256M below 4G automatically.
	861	+ default size of memory below 4G automatically. The default
	862	+ size is platform dependent.
	863	+ --> x86: max(swiotlb_size_or_default() + 8MiB, 256MiB)
	864	+ --> arm64: 128MiB
862	865	This one lets the user specify own low range under 4G
863	866	for second kernel instead.
864	867	0: to disable low allocation.
865	868	It will be ignored when crashkernel=X,high is not used
866	869	or memory reserved is below 4G.
867	870
868		− [KNL, ARM64] range in low memory.
869		− This one lets the user specify a low range in the
870		− DMA zone for the crash dump kernel.
871		− It will be ignored when crashkernel=X,high is not used
872		− or memory reserved is located in the DMA zones.
873		−
874	871	cryptomgr.notests
875	872	[KNL] Disable crypto self-tests
876	873

Documentation/admin-guide/perf/hisi-pcie-pmu.rst+68−44

@@ -15,10 +15,10 @@ HiSilicon PCIe PMU driver · 14 unmodified lines
15	15	The PCIe PMU driver registers a perf PMU with the name of its sicl-id and PCIe
16	16	Core id.::
17	17
18		− /sys/bus/event_source/hisi_pcie<sicl>_<core>
	18	+ /sys/bus/event_source/hisi_pcie<sicl>_core<core>
19	19
20	20	PMU driver provides description of available events and filter options in sysfs,
21		−see /sys/bus/event_source/devices/hisi_pcie<sicl>_<core>.
	21	+see /sys/bus/event_source/devices/hisi_pcie<sicl>_core<core>.
22	22
23	23	The "format" directory describes all formats of the config (events) and config1
24	24	(filter options) fields of the perf_event_attr structure. The "events" directory
@@ -33,13 +33,13 @@ monitored by PMU. · 8 unmodified lines
33	33	Example usage of perf::
34	34
35	35	$# perf list
36		− hisi_pcie0_0/rx_mwr_latency/ [kernel PMU event]
37		− hisi_pcie0_0/rx_mwr_cnt/ [kernel PMU event]
	36	+ hisi_pcie0_core0/rx_mwr_latency/ [kernel PMU event]
	37	+ hisi_pcie0_core0/rx_mwr_cnt/ [kernel PMU event]
38	38	------------------------------------------
39	39
40		− $# perf stat -e hisi_pcie0_0/rx_mwr_latency/
41		− $# perf stat -e hisi_pcie0_0/rx_mwr_cnt/
42		− $# perf stat -g -e hisi_pcie0_0/rx_mwr_latency/ -e hisi_pcie0_0/rx_mwr_cnt/
	40	+ $# perf stat -e hisi_pcie0_core0/rx_mwr_latency/
	41	+ $# perf stat -e hisi_pcie0_core0/rx_mwr_cnt/
	42	+ $# perf stat -g -e hisi_pcie0_core0/rx_mwr_latency/ -e hisi_pcie0_core0/rx_mwr_cnt/
43	43
44	44	The current driver does not support sampling. So "perf record" is unsupported.
45	45	Also attach to a task is unsupported for PCIe PMU.
@@ -48,59 +48,83 @@ Filter options · 2 unmodified lines
48	48	--------------
49	49
50	50	1. Target filter
51		−PMU could only monitor the performance of traffic downstream target Root Ports
52		−or downstream target Endpoint. PCIe PMU driver support "port" and "bdf"
53		−interfaces for users, and these two interfaces aren't supported at the same
54		−time.
55	51
56		−-port
57		−"port" filter can be used in all PCIe PMU events, target Root Port can be
58		−selected by configuring the 16-bits-bitmap "port". Multi ports can be selected
59		−for AP-layer-events, and only one port can be selected for TL/DL-layer-events.
	52	+ PMU could only monitor the performance of traffic downstream target Root
	53	+ Ports or downstream target Endpoint. PCIe PMU driver support "port" and
	54	+ "bdf" interfaces for users, and these two interfaces aren't supported at the
	55	+ same time.
60	56
61		−For example, if target Root Port is 0000:00:00.0 (x8 lanes), bit0 of bitmap
62		−should be set, port=0x1; if target Root Port is 0000:00:04.0 (x4 lanes),
63		−bit8 is set, port=0x100; if these two Root Ports are both monitored, port=0x101.
	57	+ - port
64	58
65		−Example usage of perf::
	59	+ "port" filter can be used in all PCIe PMU events, target Root Port can be
	60	+ selected by configuring the 16-bits-bitmap "port". Multi ports can be
	61	+ selected for AP-layer-events, and only one port can be selected for
	62	+ TL/DL-layer-events.
66	63
67		− $# perf stat -e hisi_pcie0_0/rx_mwr_latency,port=0x1/ sleep 5
	64	+ For example, if target Root Port is 0000:00:00.0 (x8 lanes), bit0 of
	65	+ bitmap should be set, port=0x1; if target Root Port is 0000:00:04.0 (x4
	66	+ lanes), bit8 is set, port=0x100; if these two Root Ports are both
	67	+ monitored, port=0x101.
68	68
69		−-bdf
	69	+ Example usage of perf::
70	70
71		−"bdf" filter can only be used in bandwidth events, target Endpoint is selected
72		−by configuring BDF to "bdf". Counter only counts the bandwidth of message
73		−requested by target Endpoint.
	71	+ $# perf stat -e hisi_pcie0_core0/rx_mwr_latency,port=0x1/ sleep 5
74	72
75		−For example, "bdf=0x3900" means BDF of target Endpoint is 0000:39:00.0.
	73	+ - bdf
76	74
77		−Example usage of perf::
	75	+ "bdf" filter can only be used in bandwidth events, target Endpoint is
	76	+ selected by configuring BDF to "bdf". Counter only counts the bandwidth of
	77	+ message requested by target Endpoint.
	78	+
	79	+ For example, "bdf=0x3900" means BDF of target Endpoint is 0000:39:00.0.
	80	+
	81	+ Example usage of perf::
78	82
79		− $# perf stat -e hisi_pcie0_0/rx_mrd_flux,bdf=0x3900/ sleep 5
	83	+ $# perf stat -e hisi_pcie0_core0/rx_mrd_flux,bdf=0x3900/ sleep 5
80	84
81	85	2. Trigger filter
82		−Event statistics start when the first time TLP length is greater/smaller
83		−than trigger condition. You can set the trigger condition by writing "trig_len",
84		−and set the trigger mode by writing "trig_mode". This filter can only be used
85		−in bandwidth events.
86	86
87		−For example, "trig_len=4" means trigger condition is 2^4 DW, "trig_mode=0"
88		−means statistics start when TLP length > trigger condition, "trig_mode=1"
89		−means start when TLP length < condition.
	87	+ Event statistics start when the first time TLP length is greater/smaller
	88	+ than trigger condition. You can set the trigger condition by writing
	89	+ "trig_len", and set the trigger mode by writing "trig_mode". This filter can
	90	+ only be used in bandwidth events.
90	91
91		−Example usage of perf::
	92	+ For example, "trig_len=4" means trigger condition is 2^4 DW, "trig_mode=0"
	93	+ means statistics start when TLP length > trigger condition, "trig_mode=1"
	94	+ means start when TLP length < condition.
	95	+
	96	+ Example usage of perf::
92	97
93		− $# perf stat -e hisi_pcie0_0/rx_mrd_flux,trig_len=0x4,trig_mode=1/ sleep 5
	98	+ $# perf stat -e hisi_pcie0_core0/rx_mrd_flux,trig_len=0x4,trig_mode=1/ sleep 5
94	99
95	100	3. Threshold filter
96		−Counter counts when TLP length within the specified range. You can set the
97		−threshold by writing "thr_len", and set the threshold mode by writing
98		−"thr_mode". This filter can only be used in bandwidth events.
99	101
100		−For example, "thr_len=4" means threshold is 2^4 DW, "thr_mode=0" means
101		−counter counts when TLP length >= threshold, and "thr_mode=1" means counts
102		−when TLP length < threshold.
	102	+ Counter counts when TLP length within the specified range. You can set the
	103	+ threshold by writing "thr_len", and set the threshold mode by writing
	104	+ "thr_mode". This filter can only be used in bandwidth events.
103	105
104		−Example usage of perf::
	106	+ For example, "thr_len=4" means threshold is 2^4 DW, "thr_mode=0" means
	107	+ counter counts when TLP length >= threshold, and "thr_mode=1" means counts
	108	+ when TLP length < threshold.
	109	+
	110	+ Example usage of perf::
	111	+
	112	+ $# perf stat -e hisi_pcie0_core0/rx_mrd_flux,thr_len=0x4,thr_mode=1/ sleep 5
	113	+
	114	+4. TLP Length filter
	115	+
	116	+ When counting bandwidth, the data can be composed of certain parts of TLP
	117	+ packets. You can specify it through "len_mode":
	118	+
	119	+ - 2'b00: Reserved (Do not use this since the behaviour is undefined)
	120	+ - 2'b01: Bandwidth of TLP payloads
	121	+ - 2'b10: Bandwidth of TLP headers
	122	+ - 2'b11: Bandwidth of both TLP payloads and headers
	123	+
	124	+ For example, "len_mode=2" means only counting the bandwidth of TLP headers
	125	+ and "len_mode=3" means the final bandwidth data is composed of both TLP
	126	+ headers and payloads. Default value if not specified is 2'b11.
	127	+
	128	+ Example usage of perf::
105	129
106		− $# perf stat -e hisi_pcie0_0/rx_mrd_flux,thr_len=0x4,thr_mode=1/ sleep 5
	130	+ $# perf stat -e hisi_pcie0_core0/rx_mrd_flux,len_mode=0x1/ sleep 5

Documentation/admin-guide/perf/index.rst+2−0

@@ -19,3 +19,5 @@ Performance monitor support · 18 unmodified lines
19	19	arm_dsu_pmu
20	20	thunderx2-pmu
21	21	alibaba_pmu
	22	+ nvidia-pmu
	23	+ meson-ddr-pmu

Documentation/admin-guide/perf/meson-ddr-pmu.rst+70−0

@@ -0,0 +1,70 @@
	1	+.. SPDX-License-Identifier: GPL-2.0
	2	+
	3	+===========================================================
	4	+Amlogic SoC DDR Bandwidth Performance Monitoring Unit (PMU)
	5	+===========================================================
	6	+
	7	+The Amlogic Meson G12 SoC contains a bandwidth monitor inside DRAM controller.
	8	+The monitor includes 4 channels. Each channel can count the request accessing
	9	+DRAM. The channel can count up to 3 AXI port simultaneously. It can be helpful
	10	+to show if the performance bottleneck is on DDR bandwidth.
	11	+
	12	+Currently, this driver supports the following 5 perf events:
	13	+
	14	++ meson_ddr_bw/total_rw_bytes/
	15	++ meson_ddr_bw/chan_1_rw_bytes/
	16	++ meson_ddr_bw/chan_2_rw_bytes/
	17	++ meson_ddr_bw/chan_3_rw_bytes/
	18	++ meson_ddr_bw/chan_4_rw_bytes/
	19	+
	20	+meson_ddr_bw/chan_{1,2,3,4}_rw_bytes/ events are channel-specific events.
	21	+Each channel support filtering, which can let the channel to monitor
	22	+individual IP module in SoC.
	23	+
	24	+Below are DDR access request event filter keywords:
	25	+
	26	++ arm - from CPU
	27	++ vpu_read1 - from OSD + VPP read
	28	++ gpu - from 3D GPU
	29	++ pcie - from PCIe controller
	30	++ hdcp - from HDCP controller
	31	++ hevc_front - from HEVC codec front end
	32	++ usb3_0 - from USB3.0 controller
	33	++ hevc_back - from HEVC codec back end
	34	++ h265enc - from HEVC encoder
	35	++ vpu_read2 - from DI read
	36	++ vpu_write1 - from VDIN write
	37	++ vpu_write2 - from di write
	38	++ vdec - from legacy codec video decoder
	39	++ hcodec - from H264 encoder
	40	++ ge2d - from ge2d
	41	++ spicc1 - from SPI controller 1
	42	++ usb0 - from USB2.0 controller 0
	43	++ dma - from system DMA controller 1
	44	++ arb0 - from arb0
	45	++ sd_emmc_b - from SD eMMC b controller
	46	++ usb1 - from USB2.0 controller 1
	47	++ audio - from Audio module
	48	++ sd_emmc_c - from SD eMMC c controller
	49	++ spicc2 - from SPI controller 2
	50	++ ethernet - from Ethernet controller
	51	+
	52	+
	53	+Examples:
	54	+
	55	+ + Show the total DDR bandwidth per seconds:
	56	+
	57	+ .. code-block:: bash
	58	+
	59	+ perf stat -a -e meson_ddr_bw/total_rw_bytes/ -I 1000 sleep 10
	60	+
	61	+
	62	+ + Show individual DDR bandwidth from CPU and GPU respectively, as well as
	63	+ sum of them:
	64	+
	65	+ .. code-block:: bash
	66	+
	67	+ perf stat -a -e meson_ddr_bw/chan_1_rw_bytes,arm=1/ -I 1000 sleep 10
	68	+ perf stat -a -e meson_ddr_bw/chan_2_rw_bytes,gpu=1/ -I 1000 sleep 10
	69	+ perf stat -a -e meson_ddr_bw/chan_3_rw_bytes,arm=1,gpu=1/ -I 1000 sleep 10
	70	+

Documentation/admin-guide/perf/nvidia-pmu.rst+299−0

@@ -0,0 +1,299 @@
	1	+=========================================================
	2	+NVIDIA Tegra SoC Uncore Performance Monitoring Unit (PMU)
	3	+=========================================================
	4	+
	5	+The NVIDIA Tegra SoC includes various system PMUs to measure key performance
	6	+metrics like memory bandwidth, latency, and utilization:
	7	+
	8	+* Scalable Coherency Fabric (SCF)
	9	+* NVLink-C2C0
	10	+* NVLink-C2C1
	11	+* CNVLink
	12	+* PCIE
	13	+
	14	+PMU Driver
	15	+----------
	16	+
	17	+The PMUs in this document are based on ARM CoreSight PMU Architecture as
	18	+described in document: ARM IHI 0091. Since this is a standard architecture, the
	19	+PMUs are managed by a common driver "arm-cs-arch-pmu". This driver describes
	20	+the available events and configuration of each PMU in sysfs. Please see the
	21	+sections below to get the sysfs path of each PMU. Like other uncore PMU drivers,
	22	+the driver provides "cpumask" sysfs attribute to show the CPU id used to handle
	23	+the PMU event. There is also "associated_cpus" sysfs attribute, which contains a
	24	+list of CPUs associated with the PMU instance.
	25	+
	26	+.. _SCF_PMU_Section:
	27	+
	28	+SCF PMU
	29	+-------
	30	+
	31	+The SCF PMU monitors system level cache events, CPU traffic, and
	32	+strongly-ordered (SO) PCIE write traffic to local/remote memory. Please see
	33	+:ref:`NVIDIA_Uncore_PMU_Traffic_Coverage_Section` for more info about the PMU
	34	+traffic coverage.
	35	+
	36	+The events and configuration options of this PMU device are described in sysfs,
	37	+see /sys/bus/event_sources/devices/nvidia_scf_pmu_<socket-id>.
	38	+
	39	+Example usage:
	40	+
	41	+* Count event id 0x0 in socket 0::
	42	+
	43	+ perf stat -a -e nvidia_scf_pmu_0/event=0x0/
	44	+
	45	+* Count event id 0x0 in socket 1::
	46	+
	47	+ perf stat -a -e nvidia_scf_pmu_1/event=0x0/
	48	+
	49	+NVLink-C2C0 PMU
	50	+--------------------
	51	+
	52	+The NVLink-C2C0 PMU monitors incoming traffic from a GPU/CPU connected with
	53	+NVLink-C2C (Chip-2-Chip) interconnect. The type of traffic captured by this PMU
	54	+varies dependent on the chip configuration:
	55	+
	56	+* NVIDIA Grace Hopper Superchip: Hopper GPU is connected with Grace SoC.
	57	+
	58	+ In this config, the PMU captures GPU ATS translated or EGM traffic from the GPU.
	59	+
	60	+* NVIDIA Grace CPU Superchip: two Grace CPU SoCs are connected.
	61	+
	62	+ In this config, the PMU captures read and relaxed ordered (RO) writes from
	63	+ PCIE device of the remote SoC.
	64	+
	65	+Please see :ref:`NVIDIA_Uncore_PMU_Traffic_Coverage_Section` for more info about
	66	+the PMU traffic coverage.
	67	+
	68	+The events and configuration options of this PMU device are described in sysfs,
	69	+see /sys/bus/event_sources/devices/nvidia_nvlink_c2c0_pmu_<socket-id>.
	70	+
	71	+Example usage:
	72	+
	73	+* Count event id 0x0 from the GPU/CPU connected with socket 0::
	74	+
	75	+ perf stat -a -e nvidia_nvlink_c2c0_pmu_0/event=0x0/
	76	+
	77	+* Count event id 0x0 from the GPU/CPU connected with socket 1::
	78	+
	79	+ perf stat -a -e nvidia_nvlink_c2c0_pmu_1/event=0x0/
	80	+
	81	+* Count event id 0x0 from the GPU/CPU connected with socket 2::
	82	+
	83	+ perf stat -a -e nvidia_nvlink_c2c0_pmu_2/event=0x0/
	84	+
	85	+* Count event id 0x0 from the GPU/CPU connected with socket 3::
	86	+
	87	+ perf stat -a -e nvidia_nvlink_c2c0_pmu_3/event=0x0/
	88	+
	89	+NVLink-C2C1 PMU
	90	+-------------------
	91	+
	92	+The NVLink-C2C1 PMU monitors incoming traffic from a GPU connected with
	93	+NVLink-C2C (Chip-2-Chip) interconnect. This PMU captures untranslated GPU
	94	+traffic, in contrast with NvLink-C2C0 PMU that captures ATS translated traffic.
	95	+Please see :ref:`NVIDIA_Uncore_PMU_Traffic_Coverage_Section` for more info about
	96	+the PMU traffic coverage.
	97	+
	98	+The events and configuration options of this PMU device are described in sysfs,
	99	+see /sys/bus/event_sources/devices/nvidia_nvlink_c2c1_pmu_<socket-id>.
	100	+
	101	+Example usage:
	102	+
	103	+* Count event id 0x0 from the GPU connected with socket 0::
	104	+
	105	+ perf stat -a -e nvidia_nvlink_c2c1_pmu_0/event=0x0/
	106	+
	107	+* Count event id 0x0 from the GPU connected with socket 1::
	108	+
	109	+ perf stat -a -e nvidia_nvlink_c2c1_pmu_1/event=0x0/
	110	+
	111	+* Count event id 0x0 from the GPU connected with socket 2::
	112	+
	113	+ perf stat -a -e nvidia_nvlink_c2c1_pmu_2/event=0x0/
	114	+
	115	+* Count event id 0x0 from the GPU connected with socket 3::
	116	+
	117	+ perf stat -a -e nvidia_nvlink_c2c1_pmu_3/event=0x0/
	118	+
	119	+CNVLink PMU
	120	+---------------
	121	+
	122	+The CNVLink PMU monitors traffic from GPU and PCIE device on remote sockets
	123	+to local memory. For PCIE traffic, this PMU captures read and relaxed ordered
	124	+(RO) write traffic. Please see :ref:`NVIDIA_Uncore_PMU_Traffic_Coverage_Section`
	125	+for more info about the PMU traffic coverage.
	126	+
	127	+The events and configuration options of this PMU device are described in sysfs,
	128	+see /sys/bus/event_sources/devices/nvidia_cnvlink_pmu_<socket-id>.
	129	+
	130	+Each SoC socket can be connected to one or more sockets via CNVLink. The user can
	131	+use "rem_socket" bitmap parameter to select the remote socket(s) to monitor.
	132	+Each bit represents the socket number, e.g. "rem_socket=0xE" corresponds to
	133	+socket 1 to 3.
	134	+/sys/bus/event_sources/devices/nvidia_cnvlink_pmu_<socket-id>/format/rem_socket
	135	+shows the valid bits that can be set in the "rem_socket" parameter.
	136	+
	137	+The PMU can not distinguish the remote traffic initiator, therefore it does not
	138	+provide filter to select the traffic source to monitor. It reports combined
	139	+traffic from remote GPU and PCIE devices.
	140	+
	141	+Example usage:
	142	+
	143	+* Count event id 0x0 for the traffic from remote socket 1, 2, and 3 to socket 0::
	144	+
	145	+ perf stat -a -e nvidia_cnvlink_pmu_0/event=0x0,rem_socket=0xE/
	146	+
	147	+* Count event id 0x0 for the traffic from remote socket 0, 2, and 3 to socket 1::
	148	+
	149	+ perf stat -a -e nvidia_cnvlink_pmu_1/event=0x0,rem_socket=0xD/
	150	+
	151	+* Count event id 0x0 for the traffic from remote socket 0, 1, and 3 to socket 2::
	152	+
	153	+ perf stat -a -e nvidia_cnvlink_pmu_2/event=0x0,rem_socket=0xB/
	154	+
	155	+* Count event id 0x0 for the traffic from remote socket 0, 1, and 2 to socket 3::
	156	+
	157	+ perf stat -a -e nvidia_cnvlink_pmu_3/event=0x0,rem_socket=0x7/
	158	+
	159	+
	160	+PCIE PMU
	161	+------------
	162	+
	163	+The PCIE PMU monitors all read/write traffic from PCIE root ports to
	164	+local/remote memory. Please see :ref:`NVIDIA_Uncore_PMU_Traffic_Coverage_Section`
	165	+for more info about the PMU traffic coverage.
	166	+
	167	+The events and configuration options of this PMU device are described in sysfs,
	168	+see /sys/bus/event_sources/devices/nvidia_pcie_pmu_<socket-id>.
	169	+
	170	+Each SoC socket can support multiple root ports. The user can use
	171	+"root_port" bitmap parameter to select the port(s) to monitor, i.e.
	172	+"root_port=0xF" corresponds to root port 0 to 3.
	173	+/sys/bus/event_sources/devices/nvidia_pcie_pmu_<socket-id>/format/root_port
	174	+shows the valid bits that can be set in the "root_port" parameter.
	175	+
	176	+Example usage:
	177	+
	178	+* Count event id 0x0 from root port 0 and 1 of socket 0::
	179	+
	180	+ perf stat -a -e nvidia_pcie_pmu_0/event=0x0,root_port=0x3/
	181	+
	182	+* Count event id 0x0 from root port 0 and 1 of socket 1::
	183	+
	184	+ perf stat -a -e nvidia_pcie_pmu_1/event=0x0,root_port=0x3/
	185	+
	186	+.. _NVIDIA_Uncore_PMU_Traffic_Coverage_Section:
	187	+
	188	+Traffic Coverage
	189	+----------------
	190	+
	191	+The PMU traffic coverage may vary dependent on the chip configuration:
	192	+
	193	+* NVIDIA Grace Hopper Superchip: Hopper GPU is connected with Grace SoC.
	194	+
	195	+ Example configuration with two Grace SoCs::
	196	+
	197	+ ******************************* *******************************
	198	+ * SOCKET-A * * SOCKET-B *
	199	+ * * * *
	200	+ * :::::::: * * :::::::: *
	201	+ * : PCIE : * * : PCIE : *
	202	+ * :::::::: * * :::::::: *
	203	+ * \| * * \| *
	204	+ * \| * * \| *
	205	+ * ::::::: ::::::::: * * ::::::::: ::::::: *
	206	+ * : : : : * * : : : : *
	207	+ * : GPU :<--NVLink-->: Grace :<---CNVLink--->: Grace :<--NVLink-->: GPU : *
	208	+ * : : C2C : SoC : * * : SoC : C2C : : *
	209	+ * ::::::: ::::::::: * * ::::::::: ::::::: *
	210	+ * \| \| * * \| \| *
	211	+ * \| \| * * \| \| *
	212	+ * &&&&&&&& &&&&&&&& * * &&&&&&&& &&&&&&&& *
	213	+ * & GMEM & & CMEM & * * & CMEM & & GMEM & *
	214	+ * &&&&&&&& &&&&&&&& * * &&&&&&&& &&&&&&&& *
	215	+ * * * *
	216	+ ******************************* *******************************
	217	+
	218	+ GMEM = GPU Memory (e.g. HBM)
	219	+ CMEM = CPU Memory (e.g. LPDDR5X)
	220	+
	221	+ \|
	222	+ \| Following table contains traffic coverage of Grace SoC PMU in socket-A:
	223	+
	224	+ ::
	225	+
	226	+ +--------------+-------+-----------+-----------+-----+----------+----------+
	227	+ \| \| Source \|
	228	+ + +-------+-----------+-----------+-----+----------+----------+
	229	+ \| Destination \| \|GPU ATS \|GPU Not-ATS\| \| Socket-B \| Socket-B \|
	230	+ \| \|PCI R/W\|Translated,\|Translated \| CPU \| CPU/PCIE1\| GPU/PCIE2\|
	231	+ \| \| \|EGM \| \| \| \| \|
	232	+ +==============+=======+===========+===========+=====+==========+==========+
	233	+ \| Local \| PCIE \|NVLink-C2C0\|NVLink-C2C1\| SCF \| SCF PMU \| CNVLink \|
	234	+ \| SYSRAM/CMEM \| PMU \|PMU \|PMU \| PMU \| \| PMU \|
	235	+ +--------------+-------+-----------+-----------+-----+----------+----------+
	236	+ \| Local GMEM \| PCIE \| N/A \|NVLink-C2C1\| SCF \| SCF PMU \| CNVLink \|
	237	+ \| \| PMU \| \|PMU \| PMU \| \| PMU \|
	238	+ +--------------+-------+-----------+-----------+-----+----------+----------+
	239	+ \| Remote \| PCIE \|NVLink-C2C0\|NVLink-C2C1\| SCF \| \| \|
	240	+ \| SYSRAM/CMEM \| PMU \|PMU \|PMU \| PMU \| N/A \| N/A \|
	241	+ \| over CNVLink \| \| \| \| \| \| \|
	242	+ +--------------+-------+-----------+-----------+-----+----------+----------+
	243	+ \| Remote GMEM \| PCIE \|NVLink-C2C0\|NVLink-C2C1\| SCF \| \| \|
	244	+ \| over CNVLink \| PMU \|PMU \|PMU \| PMU \| N/A \| N/A \|
	245	+ +--------------+-------+-----------+-----------+-----+----------+----------+
	246	+
	247	+ PCIE1 traffic represents strongly ordered (SO) writes.
	248	+ PCIE2 traffic represents reads and relaxed ordered (RO) writes.
	249	+
	250	+* NVIDIA Grace CPU Superchip: two Grace CPU SoCs are connected.
	251	+
	252	+ Example configuration with two Grace SoCs::
	253	+
	254	+ ***************** *****************
	255	+ * SOCKET-A * * SOCKET-B *
	256	+ * * * *
	257	+ * :::::::: * * :::::::: *
	258	+ * : PCIE : * * : PCIE : *
	259	+ * :::::::: * * :::::::: *
	260	+ * \| * * \| *
	261	+ * \| * * \| *
	262	+ * ::::::::: * * ::::::::: *
	263	+ * : : * * : : *
	264	+ * : Grace :<--------NVLink------->: Grace : *
	265	+ * : SoC : * C2C * : SoC : *
	266	+ * ::::::::: * * ::::::::: *
	267	+ * \| * * \| *
	268	+ * \| * * \| *
	269	+ * &&&&&&&& * * &&&&&&&& *
	270	+ * & CMEM & * * & CMEM & *
	271	+ * &&&&&&&& * * &&&&&&&& *
	272	+ * * * *
	273	+ ***************** *****************
	274	+
	275	+ GMEM = GPU Memory (e.g. HBM)
	276	+ CMEM = CPU Memory (e.g. LPDDR5X)
	277	+
	278	+ \|
	279	+ \| Following table contains traffic coverage of Grace SoC PMU in socket-A:
	280	+
	281	+ ::
	282	+
	283	+ +-----------------+-----------+---------+----------+-------------+
	284	+ \| \| Source \|
	285	+ + +-----------+---------+----------+-------------+
	286	+ \| Destination \| \| \| Socket-B \| Socket-B \|
	287	+ \| \| PCI R/W \| CPU \| CPU/PCIE1\| PCIE2 \|
	288	+ \| \| \| \| \| \|
	289	+ +=================+===========+=========+==========+=============+
	290	+ \| Local \| PCIE PMU \| SCF PMU \| SCF PMU \| NVLink-C2C0 \|
	291	+ \| SYSRAM/CMEM \| \| \| \| PMU \|
	292	+ +-----------------+-----------+---------+----------+-------------+
	293	+ \| Remote \| \| \| \| \|
	294	+ \| SYSRAM/CMEM \| PCIE PMU \| SCF PMU \| N/A \| N/A \|
	295	+ \| over NVLink-C2C \| \| \| \| \|
	296	+ +-----------------+-----------+---------+----------+-------------+
	297	+
	298	+ PCIE1 traffic represents strongly ordered (SO) writes.
	299	+ PCIE2 traffic represents reads and relaxed ordered (RO) writes.

Documentation/arm64/acpi_object_usage.rst+1−1

@@ -163,7 +163,7 @@ FPDT Section 5.2.23 (signature == "FPDT") · 162 unmodified lines
163	163
164	164	Firmware Performance Data Table
165	165
166		− Optional, not currently supported.
	166	+ Optional, useful for boot performance profiling.
167	167
168	168	GTDT Section 5.2.24 (signature == "GTDT")
169	169

Documentation/arm64/booting.rst+4−3

@@ -121,8 +121,9 @@ Header notes: · 120 unmodified lines
121	121	to the base of DRAM, since memory below it is not
122	122	accessible via the linear mapping
123	123	1
124		− 2MB aligned base may be anywhere in physical
125		− memory
	124	+ 2MB aligned base such that all image_size bytes
	125	+ counted from the start of the image are within
	126	+ the 48-bit addressable range of physical memory
126	127	Bits 4-63 Reserved.
127	128	============= ===============================================================
128	129
@@ -348,7 +349,7 @@ Before jumping into the kernel, the following conditions must be met: · 219 unmodified lines
348	349
349	350	- HWFGWTR_EL2.nSMPRI_EL1 (bit 54) must be initialised to 0b01.
350	351
351		− For CPUs with the Scalable Matrix Extension FA64 feature (FEAT_SME_FA64)
	352	+ For CPUs with the Scalable Matrix Extension FA64 feature (FEAT_SME_FA64):
352	353
353	354	- If EL3 is present:
354	355

Documentation/arm64/elf_hwcaps.rst+9−0

@@ -275,6 +275,15 @@ HWCAP2_EBF16 · 274 unmodified lines
275	275	HWCAP2_SVE_EBF16
276	276	Functionality implied by ID_AA64ZFR0_EL1.BF16 == 0b0010.
277	277
	278	+HWCAP2_CSSC
	279	+ Functionality implied by ID_AA64ISAR2_EL1.CSSC == 0b0001.
	280	+
	281	+HWCAP2_RPRFM
	282	+ Functionality implied by ID_AA64ISAR2_EL1.RPRFM == 0b0001.
	283	+
	284	+HWCAP2_SVE2P1
	285	+ Functionality implied by ID_AA64ZFR0_EL1.SVEver == 0b0010.
	286	+
278	287	4. Unused AT_HWCAP bits
279	288	-----------------------
280	289

Documentation/arm64/silicon-errata.rst+2−0

@@ -120,6 +120,8 @@ stable kernels. · 119 unmodified lines
120	120	+----------------+-----------------+-----------------+-----------------------------+
121	121	\| ARM \| Cortex-A710 \| #2224489 \| ARM64_ERRATUM_2224489 \|
122	122	+----------------+-----------------+-----------------+-----------------------------+
	123	+\| ARM \| Cortex-A715 \| #2645198 \| ARM64_ERRATUM_2645198 \|
	124	++----------------+-----------------+-----------------+-----------------------------+
123	125	\| ARM \| Cortex-X2 \| #2119858 \| ARM64_ERRATUM_2119858 \|
124	126	+----------------+-----------------+-----------------+-----------------------------+
125	127	\| ARM \| Cortex-X2 \| #2224489 \| ARM64_ERRATUM_2224489 \|

Documentation/arm64/sve.rst+1−0

@@ -52,6 +52,7 @@ model features for SVE is included in Appendix A. · 51 unmodified lines
52	52	HWCAP2_SVEBITPERM
53	53	HWCAP2_SVESHA3
54	54	HWCAP2_SVESM4
	55	+ HWCAP2_SVE2P1
55	56
56	57	This list may be extended over time as the SVE architecture evolves.
57	58

Documentation/devicetree/bindings/perf/amlogic,g12-ddr-pmu.yaml+54−0

@@ -0,0 +1,54 @@
	1	+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
	2	+%YAML 1.2
	3	+---
	4	+$id: http://devicetree.org/schemas/perf/amlogic,g12-ddr-pmu.yaml#
	5	+$schema: http://devicetree.org/meta-schemas/core.yaml#
	6	+
	7	+title: Amlogic G12 DDR performance monitor
	8	+
	9	+maintainers:
	10	+ - Jiucheng Xu <jiucheng.xu@amlogic.com>
	11	+
	12	+description: \|
	13	+ Amlogic G12 series SoC integrate DDR bandwidth monitor.
	14	+ A timer is inside and can generate interrupt when timeout.
	15	+ The bandwidth is counted in the timer ISR. Different platform
	16	+ has different subset of event format attribute.
	17	+
	18	+properties:
	19	+ compatible:
	20	+ enum:
	21	+ - amlogic,g12a-ddr-pmu
	22	+ - amlogic,g12b-ddr-pmu
	23	+ - amlogic,sm1-ddr-pmu
	24	+
	25	+ reg:
	26	+ items:
	27	+ - description: DMC bandwidth register space.
	28	+ - description: DMC PLL register space.
	29	+
	30	+ interrupts:
	31	+ items:
	32	+ - description: The IRQ of the inside timer timeout.
	33	+
	34	+required:
	35	+ - compatible
	36	+ - reg
	37	+ - interrupts
	38	+
	39	+additionalProperties: false
	40	+
	41	+examples:
	42	+ - \|
	43	+ #include <dt-bindings/interrupt-controller/arm-gic.h>
	44	+ pmu {
	45	+ #address-cells=<2>;
	46	+ #size-cells=<2>;
	47	+
	48	+ pmu@ff638000 {
	49	+ compatible = "amlogic,g12a-ddr-pmu";
	50	+ reg = <0x0 0xff638000 0x0 0x100>,
	51	+ <0x0 0xff638c00 0x0 0x100>;
	52	+ interrupts = <GIC_SPI 52 IRQ_TYPE_EDGE_RISING>;
	53	+ };
	54	+ };

MAINTAINERS+11−1

@@ -1093,6 +1093,16 @@ S: Maintained · 1092 unmodified lines
1093	1093	F: Documentation/hid/amd-sfh*
1094	1094	F: drivers/hid/amd-sfh-hid/
1095	1095
	1096	+AMLOGIC DDR PMU DRIVER
	1097	+M: Jiucheng Xu <jiucheng.xu@amlogic.com>
	1098	+L: linux-amlogic@lists.infradead.org
	1099	+S: Supported
	1100	+W: http://www.amlogic.com
	1101	+F: Documentation/admin-guide/perf/meson-ddr-pmu.rst
	1102	+F: Documentation/devicetree/bindings/perf/amlogic,g12-ddr-pmu.yaml
	1103	+F: drivers/perf/amlogic/
	1104	+F: include/soc/amlogic/
	1105	+
1096	1106	AMPHION VPU CODEC V4L2 DRIVER
1097	1107	M: Ming Qian <ming.qian@nxp.com>
1098	1108	M: Shijie Qin <shijie.qin@nxp.com>
@@ -9244,7 +9254,7 @@ F: drivers/misc/hisi_hikey_usb.c · 8145 unmodified lines
9244	9254
9245	9255	HISILICON PMU DRIVER
9246	9256	M: Shaokun Zhang <zhangshaokun@hisilicon.com>
9247		−M: Qi Liu <liuqi115@huawei.com>
	9257	+M: Jonathan Cameron <jonathan.cameron@huawei.com>
9248	9258	S: Supported
9249	9259	W: http://www.hisilicon.com
9250	9260	F: Documentation/admin-guide/perf/hisi-pcie-pmu.rst

Makefile+2−0

@@ -966,8 +966,10 @@ LDFLAGS_vmlinux += --gc-sections · 965 unmodified lines
966	966	endif
967	967
968	968	ifdef CONFIG_SHADOW_CALL_STACK
	969	+ifndef CONFIG_DYNAMIC_SCS
969	970	CC_FLAGS_SCS := -fsanitize=shadow-call-stack
970	971	KBUILD_CFLAGS += $(CC_FLAGS_SCS)
	972	+endif
971	973	export CC_FLAGS_SCS
972	974	endif
973	975

arch/Kconfig+7−0

@@ -654,6 +654,13 @@ config SHADOW_CALL_STACK · 653 unmodified lines
654	654	reading and writing arbitrary memory may be able to locate them
655	655	and hijack control flow by modifying the stacks.
656	656
	657	+config DYNAMIC_SCS
	658	+ bool
	659	+ help
	660	+ Set by the arch code if it relies on code patching to insert the
	661	+ shadow call stack push and pop instructions rather than on the
	662	+ compiler.
	663	+
657	664	config LTO
658	665	bool
659	666	help

arch/arm64/Kconfig+40−9

@@ -1,6 +1,7 @@
1	1	# SPDX-License-Identifier: GPL-2.0-only
2	2	config ARM64
3	3	def_bool y
	4	+ select ACPI_APMT if ACPI
4	5	select ACPI_CCA_REQUIRED if ACPI
5	6	select ACPI_GENERIC_GSI if ACPI
6	7	select ACPI_GTDT if ACPI
@@ -118,6 +119,7 @@ config ARM64 · 111 unmodified lines
118	119	select CPU_PM if (SUSPEND \|\| CPU_IDLE)
119	120	select CRC32
120	121	select DCACHE_WORD_ACCESS
	122	+ select DYNAMIC_FTRACE if FUNCTION_TRACER
121	123	select DMA_DIRECT_REMAP
122	124	select EDAC_SUPPORT
123	125	select FRAME_POINTER
@@ -182,8 +184,10 @@ config ARM64 · 58 unmodified lines
182	184	select HAVE_DEBUG_KMEMLEAK
183	185	select HAVE_DMA_CONTIGUOUS
184	186	select HAVE_DYNAMIC_FTRACE
	187	+ select HAVE_DYNAMIC_FTRACE_WITH_ARGS \
	188	+ if $(cc-option,-fpatchable-function-entry=2)
185	189	select FTRACE_MCOUNT_USE_PATCHABLE_FUNCTION_ENTRY \
186		− if DYNAMIC_FTRACE_WITH_REGS
	190	+ if DYNAMIC_FTRACE_WITH_ARGS
187	191	select HAVE_EFFICIENT_UNALIGNED_ACCESS
188	192	select HAVE_FAST_GUP
189	193	select HAVE_FTRACE_MCOUNT_RECORD
@@ -234,16 +238,16 @@ config ARM64 · 44 unmodified lines
234	238	help
235	239	ARM 64-bit (AArch64) Linux support.
236	240
237		−config CLANG_SUPPORTS_DYNAMIC_FTRACE_WITH_REGS
	241	+config CLANG_SUPPORTS_DYNAMIC_FTRACE_WITH_ARGS
238	242	def_bool CC_IS_CLANG
239	243	# https://github.com/ClangBuiltLinux/linux/issues/1507
240	244	depends on AS_IS_GNU \|\| (AS_IS_LLVM && (LD_IS_LLD \|\| LD_VERSION >= 23600))
241		− select HAVE_DYNAMIC_FTRACE_WITH_REGS
	245	+ select HAVE_DYNAMIC_FTRACE_WITH_ARGS
242	246
243		−config GCC_SUPPORTS_DYNAMIC_FTRACE_WITH_REGS
	247	+config GCC_SUPPORTS_DYNAMIC_FTRACE_WITH_ARGS
244	248	def_bool CC_IS_GCC
245	249	depends on $(cc-option,-fpatchable-function-entry=2)
246		− select HAVE_DYNAMIC_FTRACE_WITH_REGS
	250	+ select HAVE_DYNAMIC_FTRACE_WITH_ARGS
247	251
248	252	config 64BIT
249	253	def_bool y
@@ -371,6 +375,9 @@ config KASAN_SHADOW_OFFSET · 121 unmodified lines
371	375	default 0xeffffff800000000 if ARM64_VA_BITS_36 && KASAN_SW_TAGS
372	376	default 0xffffffffffffffff
373	377
	378	+config UNWIND_TABLES
	379	+ bool
	380	+
374	381	source "arch/arm64/Kconfig.platforms"
375	382
376	383	menu "Kernel Features"
@@ -965,6 +972,22 @@ config ARM64_ERRATUM_2457168 · 588 unmodified lines
965	972
966	973	If unsure, say Y.
967	974
	975	+config ARM64_ERRATUM_2645198
	976	+ bool "Cortex-A715: 2645198: Workaround possible [ESR\|FAR]_ELx corruption"
	977	+ default y
	978	+ help
	979	+ This option adds the workaround for ARM Cortex-A715 erratum 2645198.
	980	+
	981	+ If a Cortex-A715 cpu sees a page mapping permissions change from executable
	982	+ to non-executable, it may corrupt the ESR_ELx and FAR_ELx registers on the
	983	+ next instruction abort caused by permission fault.
	984	+
	985	+ Only user-space does executable to non-executable permission transition via
	986	+ mprotect() system call. Workaround the problem by doing a break-before-make
	987	+ TLB invalidation, for all changes to executable user space mappings.
	988	+
	989	+ If unsure, say Y.
	990	+
968	991	config CAVIUM_ERRATUM_22375
969	992	bool "Cavium erratum 22375, 24313"
970	993	default y
@@ -1715,7 +1738,6 @@ config ARM64_LSE_ATOMICS · 744 unmodified lines
1715	1738
1716	1739	config ARM64_USE_LSE_ATOMICS
1717	1740	bool "Atomic instructions"
1718		− depends on JUMP_LABEL
1719	1741	default y
1720	1742	help
1721	1743	As part of the Large System Extensions, ARMv8.1 introduces new
@@ -1817,7 +1839,7 @@ config ARM64_PTR_AUTH_KERNEL · 95 unmodified lines
1817	1839	# which is only understood by binutils starting with version 2.33.1.
1818	1840	depends on LD_IS_LLD \|\| LD_VERSION >= 23301 \|\| (CC_IS_GCC && GCC_VERSION < 90100)
1819	1841	depends on !CC_IS_CLANG \|\| AS_HAS_CFI_NEGATE_RA_STATE
1820		− depends on (!FUNCTION_GRAPH_TRACER \|\| DYNAMIC_FTRACE_WITH_REGS)
	1842	+ depends on (!FUNCTION_GRAPH_TRACER \|\| DYNAMIC_FTRACE_WITH_ARGS)
1821	1843	help
1822	1844	If the compiler supports the -mbranch-protection or
1823	1845	-msign-return-address flag (e.g. GCC 7 or later), then this option
@@ -1827,7 +1849,7 @@ config ARM64_PTR_AUTH_KERNEL · 3 unmodified lines
1827	1849	disabled with minimal loss of protection.
1828	1850
1829	1851	This feature works with FUNCTION_GRAPH_TRACER option only if
1830		− DYNAMIC_FTRACE_WITH_REGS is enabled.
	1852	+ DYNAMIC_FTRACE_WITH_ARGS is enabled.
1831	1853
1832	1854	config CC_HAS_BRANCH_PROT_PAC_RET
1833	1855	# GCC 9 or later, clang 8 or later
@@ -1925,7 +1947,7 @@ config ARM64_BTI_KERNEL · 91 unmodified lines
1925	1947	depends on !CC_IS_GCC
1926	1948	# https://github.com/llvm/llvm-project/commit/a88c722e687e6780dcd6a58718350dc76fcc4cc9
1927	1949	depends on !CC_IS_CLANG \|\| CLANG_VERSION >= 120000
1928		− depends on (!FUNCTION_GRAPH_TRACER \|\| DYNAMIC_FTRACE_WITH_REGS)
	1950	+ depends on (!FUNCTION_GRAPH_TRACER \|\| DYNAMIC_FTRACE_WITH_ARGS)
1929	1951	help
1930	1952	Build the kernel with Branch Target Identification annotations
1931	1953	and enable enforcement of this for kernel code. When this option
@@ -2158,6 +2180,15 @@ config ARCH_NR_GPIO · 226 unmodified lines
2158	2180
2159	2181	If unsure, leave the default value.
2160	2182
	2183	+config UNWIND_PATCH_PAC_INTO_SCS
	2184	+ bool "Enable shadow call stack dynamically using code patching"
	2185	+ # needs Clang with https://reviews.llvm.org/D111780 incorporated
	2186	+ depends on CC_IS_CLANG && CLANG_VERSION >= 150000
	2187	+ depends on ARM64_PTR_AUTH_KERNEL && CC_HAS_BRANCH_PROT_PAC_RET
	2188	+ depends on SHADOW_CALL_STACK
	2189	+ select UNWIND_TABLES
	2190	+ select DYNAMIC_SCS
	2191	+
2161	2192	endmenu # "Kernel Features"
2162	2193
2163	2194	menu "Boot options"

arch/arm64/Makefile+14−3

@@ -45,8 +45,13 @@ KBUILD_CFLAGS += $(call cc-option,-mabi=lp64) · 44 unmodified lines
45	45	KBUILD_AFLAGS += $(call cc-option,-mabi=lp64)
46	46
47	47	# Avoid generating .eh_frame* sections.
	48	+ifneq ($(CONFIG_UNWIND_TABLES),y)
48	49	KBUILD_CFLAGS += -fno-asynchronous-unwind-tables -fno-unwind-tables
49	50	KBUILD_AFLAGS += -fno-asynchronous-unwind-tables -fno-unwind-tables
	51	+else
	52	+KBUILD_CFLAGS += -fasynchronous-unwind-tables
	53	+KBUILD_AFLAGS += -fasynchronous-unwind-tables
	54	+endif
50	55
51	56	ifeq ($(CONFIG_STACKPROTECTOR_PER_TASK),y)
52	57	prepare: stack_protector_prepare
@@ -72,10 +77,16 @@ branch-prot-flags-$(CONFIG_CC_HAS_SIGN_RETURN_ADDRESS) := -msign-return-address= · 19 unmodified lines
72	77	# We enable additional protection for leaf functions as there is some
73	78	# narrow potential for ROP protection benefits and no substantial
74	79	# performance impact has been observed.
	80	+PACRET-y := pac-ret+leaf
	81	+
	82	+# Using a shadow call stack in leaf functions is too costly, so avoid PAC there
	83	+# as well when we may be patching PAC into SCS
	84	+PACRET-$(CONFIG_UNWIND_PATCH_PAC_INTO_SCS) := pac-ret
	85	+
75	86	ifeq ($(CONFIG_ARM64_BTI_KERNEL),y)
76		−branch-prot-flags-$(CONFIG_CC_HAS_BRANCH_PROT_PAC_RET_BTI) := -mbranch-protection=pac-ret+leaf+bti
	87	+branch-prot-flags-$(CONFIG_CC_HAS_BRANCH_PROT_PAC_RET_BTI) := -mbranch-protection=$(PACRET-y)+bti
77	88	else
78		−branch-prot-flags-$(CONFIG_CC_HAS_BRANCH_PROT_PAC_RET) := -mbranch-protection=pac-ret+leaf
	89	+branch-prot-flags-$(CONFIG_CC_HAS_BRANCH_PROT_PAC_RET) := -mbranch-protection=$(PACRET-y)
79	90	endif
80	91	# -march=armv8.3-a enables the non-nops instructions for PAC, to avoid the
81	92	# compiler to generate them and consequently to break the single image contract
@@ -128,7 +139,7 @@ endif · 46 unmodified lines
128	139
129	140	CHECKFLAGS += -D__aarch64__
130	141
131		−ifeq ($(CONFIG_DYNAMIC_FTRACE_WITH_REGS),y)
	142	+ifeq ($(CONFIG_DYNAMIC_FTRACE_WITH_ARGS),y)
132	143	KBUILD_CPPFLAGS += -DCC_USING_PATCHABLE_FUNCTION_ENTRY
133	144	CC_FLAGS_FTRACE := -fpatchable-function-entry=2
134	145	endif

arch/arm64/include/asm/alternative-macros.h+2−2

@@ -224,7 +224,7 @@ alternative_endif · 223 unmodified lines
224	224	#include <linux/types.h>
225	225
226	226	static __always_inline bool
227		−alternative_has_feature_likely(unsigned long feature)
	227	+alternative_has_feature_likely(const unsigned long feature)
228	228	{
229	229	compiletime_assert(feature < ARM64_NCAPS,
230	230	"feature must be < ARM64_NCAPS");
@@ -242,7 +242,7 @@ l_no: · 11 unmodified lines
242	242	}
243	243
244	244	static __always_inline bool
245		−alternative_has_feature_unlikely(unsigned long feature)
	245	+alternative_has_feature_unlikely(const unsigned long feature)
246	246	{
247	247	compiletime_assert(feature < ARM64_NCAPS,
248	248	"feature must be < ARM64_NCAPS");

arch/arm64/include/asm/assembler.h+3−30

@@ -34,11 +34,6 @@ · 33 unmodified lines
34	34	wx\n .req w\n
35	35	.endr
36	36
37		− .macro save_and_disable_daif, flags
38		− mrs \flags, daif
39		− msr daifset, #0xf
40		− .endm
41		−
42	37	.macro disable_daif
43	38	msr daifset, #0xf
44	39	.endm
@@ -47,15 +42,6 @@ · 2 unmodified lines
47	42	msr daifclr, #0xf
48	43	.endm
49	44
50		− .macro restore_daif, flags:req
51		− msr daif, \flags
52		− .endm
53		−
54		− /* IRQ/FIQ are the lowest priority flags, unconditionally unmask the rest. */
55		− .macro enable_da
56		− msr daifclr, #(8 \| 4)
57		− .endm
58		−
59	45	/*
60	46	* Save/restore interrupts.
61	47	*/
@@ -619,17 +605,6 @@ alternative_endif · 557 unmodified lines
619	605	#endif
620	606	.endm
621	607
622		−/*
623		− * Perform the reverse of offset_ttbr1.
624		− * bic is used as it can cover the immediate value and, in future, won't need
625		− * to be nop'ed out when dealing with 52-bit kernel VAs.
626		− */
627		− .macro restore_ttbr1, ttbr
628		−#ifdef CONFIG_ARM64_VA_BITS_52
629		− bic \ttbr, \ttbr, #TTBR1_BADDR_4852_OFFSET
630		−#endif
631		− .endm
632		−
633	608	/*
634	609	* Arrange a physical address in a TTBR register, taking care of 52-bit
635	610	* addresses.
@@ -660,12 +635,10 @@ alternative_endif · 24 unmodified lines
660	635	.endm
661	636
662	637	.macro pte_to_phys, phys, pte
663		−#ifdef CONFIG_ARM64_PA_BITS_52
664		− ubfiz \phys, \pte, #(48 - 16 - 12), #16
665		− bfxil \phys, \pte, #16, #32
666		− lsl \phys, \phys, #16
667		−#else
668	638	and \phys, \pte, #PTE_ADDR_MASK
	639	+#ifdef CONFIG_ARM64_PA_BITS_52
	640	+ orr \phys, \phys, \phys, lsl #PTE_ADDR_HIGH_SHIFT
	641	+ and \phys, \phys, GENMASK_ULL(PHYS_MASK_SHIFT - 1, PAGE_SHIFT)
669	642	#endif
670	643	.endm
671	644

arch/arm64/include/asm/cpufeature.h+2−1

@@ -832,7 +832,8 @@ static inline bool system_supports_tlb_range(void) · 831 unmodified lines
832	832	cpus_have_const_cap(ARM64_HAS_TLB_RANGE);
833	833	}
834	834
835		−extern int do_emulate_mrs(struct pt_regs *regs, u32 sys_reg, u32 rt);
	835	+int do_emulate_mrs(struct pt_regs *regs, u32 sys_reg, u32 rt);
	836	+bool try_emulate_mrs(struct pt_regs *regs, u32 isn);
836	837
837	838	static inline u32 id_aa64mmfr0_parange_to_phys_shift(int parange)
838	839	{

arch/arm64/include/asm/cputype.h+2−0

@@ -80,6 +80,7 @@ · 79 unmodified lines
80	80	#define ARM_CPU_PART_CORTEX_X1 0xD44
81	81	#define ARM_CPU_PART_CORTEX_A510 0xD46
82	82	#define ARM_CPU_PART_CORTEX_A710 0xD47
	83	+#define ARM_CPU_PART_CORTEX_A715 0xD4D
83	84	#define ARM_CPU_PART_CORTEX_X2 0xD48
84	85	#define ARM_CPU_PART_NEOVERSE_N2 0xD49
85	86	#define ARM_CPU_PART_CORTEX_A78C 0xD4B
@@ -142,6 +143,7 @@ · 56 unmodified lines
142	143	#define MIDR_CORTEX_X1 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_X1)
143	144	#define MIDR_CORTEX_A510 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A510)
144	145	#define MIDR_CORTEX_A710 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A710)
	146	+#define MIDR_CORTEX_A715 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A715)
145	147	#define MIDR_CORTEX_X2 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_X2)
146	148	#define MIDR_NEOVERSE_N2 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_NEOVERSE_N2)
147	149	#define MIDR_CORTEX_A78C MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A78C)

arch/arm64/include/asm/exception.h+4−3

@@ -58,7 +58,8 @@ asmlinkage void call_on_irq_stack(struct pt_regs *regs, · 57 unmodified lines
58	58	asmlinkage void asm_exit_to_user_mode(struct pt_regs *regs);
59	59
60	60	void do_mem_abort(unsigned long far, unsigned long esr, struct pt_regs *regs);
61		−void do_undefinstr(struct pt_regs *regs, unsigned long esr);
	61	+void do_el0_undef(struct pt_regs *regs, unsigned long esr);
	62	+void do_el1_undef(struct pt_regs *regs, unsigned long esr);
62	63	void do_el0_bti(struct pt_regs *regs);
63	64	void do_el1_bti(struct pt_regs *regs, unsigned long esr);
64	65	void do_debug_exception(unsigned long addr_if_watchpoint, unsigned long esr,
@@ -67,10 +68,10 @@ void do_fpsimd_acc(unsigned long esr, struct pt_regs *regs); · 2 unmodified lines
67	68	void do_sve_acc(unsigned long esr, struct pt_regs *regs);
68	69	void do_sme_acc(unsigned long esr, struct pt_regs *regs);
69	70	void do_fpsimd_exc(unsigned long esr, struct pt_regs *regs);
70		−void do_sysinstr(unsigned long esr, struct pt_regs *regs);
	71	+void do_el0_sys(unsigned long esr, struct pt_regs *regs);
71	72	void do_sp_pc_abort(unsigned long addr, unsigned long esr, struct pt_regs *regs);
72	73	void bad_el0_sync(struct pt_regs *regs, int reason, unsigned long esr);
73		−void do_cp15instr(unsigned long esr, struct pt_regs *regs);
	74	+void do_el0_cp15(unsigned long esr, struct pt_regs *regs);
74	75	int do_compat_alignment_fixup(unsigned long addr, struct pt_regs *regs);
75	76	void do_el0_svc(struct pt_regs *regs);
76	77	void do_el0_svc_compat(struct pt_regs *regs);

arch/arm64/include/asm/fpsimd.h+13−4

@@ -56,11 +56,20 @@ extern void fpsimd_signal_preserve_current_state(void); · 55 unmodified lines
56	56	extern void fpsimd_preserve_current_state(void);
57	57	extern void fpsimd_restore_current_state(void);
58	58	extern void fpsimd_update_current_state(struct user_fpsimd_state const *state);
	59	+extern void fpsimd_kvm_prepare(void);
	60	+
	61	+struct cpu_fp_state {
	62	+ struct user_fpsimd_state *st;
	63	+ void *sve_state;
	64	+ void *za_state;
	65	+ u64 *svcr;
	66	+ unsigned int sve_vl;
	67	+ unsigned int sme_vl;
	68	+ enum fp_type *fp_type;
	69	+ enum fp_type to_save;
	70	+};
59	71
60		−extern void fpsimd_bind_state_to_cpu(struct user_fpsimd_state *state,
61		− void *sve_state, unsigned int sve_vl,
62		− void *za_state, unsigned int sme_vl,
63		− u64 *svcr);
	72	+extern void fpsimd_bind_state_to_cpu(struct cpu_fp_state *fp_state);
64	73
65	74	extern void fpsimd_flush_task_state(struct task_struct *target);
66	75	extern void fpsimd_save_and_flush_cpu_state(void);

arch/arm64/include/asm/ftrace.h+66−6

@@ -23,7 +23,7 @@ · 22 unmodified lines
23	23	*/
24	24	#define HAVE_FUNCTION_GRAPH_RET_ADDR_PTR
25	25
26		−#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
	26	+#ifdef CONFIG_DYNAMIC_FTRACE_WITH_ARGS
27	27	#define ARCH_SUPPORTS_FTRACE_OPS 1
28	28	#else
29	29	#define MCOUNT_ADDR ((unsigned long)_mcount)
@@ -33,8 +33,7 @@ · 3 unmodified lines
33	33	#define MCOUNT_INSN_SIZE AARCH64_INSN_SIZE
34	34
35	35	#define FTRACE_PLT_IDX 0
36		−#define FTRACE_REGS_PLT_IDX 1
37		−#define NR_FTRACE_PLTS 2
	36	+#define NR_FTRACE_PLTS 1
38	37
39	38	/*
40	39	* Currently, gcc tends to save the link register after the local variables
@@ -69,7 +68,7 @@ static inline unsigned long ftrace_call_adjust(unsigned long addr) · 28 unmodified lines
69	68	* Adjust addr to point at the BL in the callsite.
70	69	* See ftrace_init_nop() for the callsite sequence.
71	70	*/
72		− if (IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_REGS))
	71	+ if (IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_ARGS))
73	72	return addr + AARCH64_INSN_SIZE;
74	73	/*
75	74	* addr is the address of the mcount call instruction.
@@ -78,10 +77,71 @@ static inline unsigned long ftrace_call_adjust(unsigned long addr) · 2 unmodified lines
78	77	return addr;
79	78	}
80	79
81		−#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
	80	+#ifdef CONFIG_DYNAMIC_FTRACE_WITH_ARGS
82	81	struct dyn_ftrace;
83	82	struct ftrace_ops;
84		−struct ftrace_regs;
	83	+
	84	+#define arch_ftrace_get_regs(regs) NULL
	85	+
	86	+struct ftrace_regs {
	87	+ /* x0 - x8 */
	88	+ unsigned long regs[9];
	89	+ unsigned long __unused;
	90	+
	91	+ unsigned long fp;
	92	+ unsigned long lr;
	93	+
	94	+ unsigned long sp;
	95	+ unsigned long pc;
	96	+};
	97	+
	98	+static __always_inline unsigned long
	99	+ftrace_regs_get_instruction_pointer(const struct ftrace_regs *fregs)
	100	+{
	101	+ return fregs->pc;
	102	+}
	103	+
	104	+static __always_inline void
	105	+ftrace_regs_set_instruction_pointer(struct ftrace_regs *fregs,
	106	+ unsigned long pc)
	107	+{
	108	+ fregs->pc = pc;
	109	+}
	110	+
	111	+static __always_inline unsigned long
	112	+ftrace_regs_get_stack_pointer(const struct ftrace_regs *fregs)
	113	+{
	114	+ return fregs->sp;
	115	+}
	116	+
	117	+static __always_inline unsigned long
	118	+ftrace_regs_get_argument(struct ftrace_regs *fregs, unsigned int n)
	119	+{
	120	+ if (n < 8)
	121	+ return fregs->regs[n];
	122	+ return 0;
	123	+}
	124	+
	125	+static __always_inline unsigned long
	126	+ftrace_regs_get_return_value(const struct ftrace_regs *fregs)
	127	+{
	128	+ return fregs->regs[0];
	129	+}
	130	+
	131	+static __always_inline void
	132	+ftrace_regs_set_return_value(struct ftrace_regs *fregs,
	133	+ unsigned long ret)
	134	+{
	135	+ fregs->regs[0] = ret;
	136	+}
	137	+
	138	+static __always_inline void
	139	+ftrace_override_function_with_return(struct ftrace_regs *fregs)
	140	+{
	141	+ fregs->pc = fregs->lr;
	142	+}
	143	+
	144	+int ftrace_regs_query_register_offset(const char *name);
85	145
86	146	int ftrace_init_nop(struct module mod, struct dyn_ftrace rec);
87	147	#define ftrace_init_nop ftrace_init_nop

arch/arm64/include/asm/hugetlb.h+9−0

@@ -49,6 +49,15 @@ extern pte_t huge_ptep_get(pte_t *ptep); · 48 unmodified lines
49	49
50	50	void __init arm64_hugetlb_cma_reserve(void);
51	51
	52	+#define huge_ptep_modify_prot_start huge_ptep_modify_prot_start
	53	+extern pte_t huge_ptep_modify_prot_start(struct vm_area_struct *vma,
	54	+ unsigned long addr, pte_t *ptep);
	55	+
	56	+#define huge_ptep_modify_prot_commit huge_ptep_modify_prot_commit
	57	+extern void huge_ptep_modify_prot_commit(struct vm_area_struct *vma,
	58	+ unsigned long addr, pte_t *ptep,
	59	+ pte_t old_pte, pte_t new_pte);
	60	+
52	61	#include <asm-generic/hugetlb.h>
53	62
54	63	#endif /* __ASM_HUGETLB_H */

arch/arm64/include/asm/hwcap.h+3−0

@@ -120,6 +120,9 @@ · 119 unmodified lines
120	120	#define KERNEL_HWCAP_WFXT __khwcap2_feature(WFXT)
121	121	#define KERNEL_HWCAP_EBF16 __khwcap2_feature(EBF16)
122	122	#define KERNEL_HWCAP_SVE_EBF16 __khwcap2_feature(SVE_EBF16)
	123	+#define KERNEL_HWCAP_CSSC __khwcap2_feature(CSSC)
	124	+#define KERNEL_HWCAP_RPRFM __khwcap2_feature(RPRFM)
	125	+#define KERNEL_HWCAP_SVE2P1 __khwcap2_feature(SVE2P1)
123	126
124	127	/*
125	128	* This yields a mask that user programs can use to figure out what

arch/arm64/include/asm/insn.h+109−47

@@ -13,31 +13,6 @@ · 12 unmodified lines
13	13	#include <asm/insn-def.h>
14	14
15	15	#ifndef __ASSEMBLY__
16		−/*
17		− * ARM Architecture Reference Manual for ARMv8 Profile-A, Issue A.a
18		− * Section C3.1 "A64 instruction index by encoding":
19		− * AArch64 main encoding table
20		− * Bit position
21		− * 28 27 26 25 Encoding Group
22		− * 0 0 - - Unallocated
23		− * 1 0 0 - Data processing, immediate
24		− * 1 0 1 - Branch, exception generation and system instructions
25		− * - 1 - 0 Loads and stores
26		− * - 1 0 1 Data processing - register
27		− * 0 1 1 1 Data processing - SIMD and floating point
28		− * 1 1 1 1 Data processing - SIMD and floating point
29		− * "-" means "don't care"
30		− */
31		−enum aarch64_insn_encoding_class {
32		− AARCH64_INSN_CLS_UNKNOWN, /* UNALLOCATED */
33		− AARCH64_INSN_CLS_SVE, /* SVE instructions */
34		− AARCH64_INSN_CLS_DP_IMM, /* Data processing - immediate */
35		− AARCH64_INSN_CLS_DP_REG, /* Data processing - register */
36		− AARCH64_INSN_CLS_DP_FPSIMD, /* Data processing - SIMD and FP */
37		− AARCH64_INSN_CLS_LDST, /* Loads and stores */
38		− AARCH64_INSN_CLS_BR_SYS, /* Branch, exception generation and
39		− * system instructions */
40		−};
41	16
42	17	enum aarch64_insn_hint_cr_op {
43	18	AARCH64_INSN_HINT_NOP = 0x0 << 5,
@@ -326,6 +301,23 @@ static __always_inline u32 aarch64_insn_get_##abbr##_value(void) \ · 282 unmodified lines
326	301	return (val); \
327	302	}
328	303
	304	+/*
	305	+ * ARM Architecture Reference Manual for ARMv8 Profile-A, Issue A.a
	306	+ * Section C3.1 "A64 instruction index by encoding":
	307	+ * AArch64 main encoding table
	308	+ * Bit position
	309	+ * 28 27 26 25 Encoding Group
	310	+ * 0 0 - - Unallocated
	311	+ * 1 0 0 - Data processing, immediate
	312	+ * 1 0 1 - Branch, exception generation and system instructions
	313	+ * - 1 - 0 Loads and stores
	314	+ * - 1 0 1 Data processing - register
	315	+ * 0 1 1 1 Data processing - SIMD and floating point
	316	+ * 1 1 1 1 Data processing - SIMD and floating point
	317	+ * "-" means "don't care"
	318	+ */
	319	+__AARCH64_INSN_FUNCS(class_branch_sys, 0x1c000000, 0x14000000)
	320	+
329	321	__AARCH64_INSN_FUNCS(adr, 0x9F000000, 0x10000000)
330	322	__AARCH64_INSN_FUNCS(adrp, 0x9F000000, 0x90000000)
331	323	__AARCH64_INSN_FUNCS(prfm, 0x3FC00000, 0x39800000)
@@ -431,58 +423,122 @@ __AARCH64_INSN_FUNCS(pssbb, 0xFFFFFFFF, 0xD503349F) · 99 unmodified lines
431	423
432	424	#undef __AARCH64_INSN_FUNCS
433	425
434		−bool aarch64_insn_is_steppable_hint(u32 insn);
435		−bool aarch64_insn_is_branch_imm(u32 insn);
	426	+static __always_inline bool aarch64_insn_is_steppable_hint(u32 insn)
	427	+{
	428	+ if (!aarch64_insn_is_hint(insn))
	429	+ return false;
	430	+
	431	+ switch (insn & 0xFE0) {
	432	+ case AARCH64_INSN_HINT_XPACLRI:
	433	+ case AARCH64_INSN_HINT_PACIA_1716:
	434	+ case AARCH64_INSN_HINT_PACIB_1716:
	435	+ case AARCH64_INSN_HINT_PACIAZ:
	436	+ case AARCH64_INSN_HINT_PACIASP:
	437	+ case AARCH64_INSN_HINT_PACIBZ:
	438	+ case AARCH64_INSN_HINT_PACIBSP:
	439	+ case AARCH64_INSN_HINT_BTI:
	440	+ case AARCH64_INSN_HINT_BTIC:
	441	+ case AARCH64_INSN_HINT_BTIJ:
	442	+ case AARCH64_INSN_HINT_BTIJC:
	443	+ case AARCH64_INSN_HINT_NOP:
	444	+ return true;
	445	+ default:
	446	+ return false;
	447	+ }
	448	+}
	449	+
	450	+static __always_inline bool aarch64_insn_is_branch(u32 insn)
	451	+{
	452	+ /* b, bl, cb, tb, ret, b.cond, br, blr* */
	453	+
	454	+ return aarch64_insn_is_b(insn) \|\|
	455	+ aarch64_insn_is_bl(insn) \|\|
	456	+ aarch64_insn_is_cbz(insn) \|\|
	457	+ aarch64_insn_is_cbnz(insn) \|\|
	458	+ aarch64_insn_is_tbz(insn) \|\|
	459	+ aarch64_insn_is_tbnz(insn) \|\|
	460	+ aarch64_insn_is_ret(insn) \|\|
	461	+ aarch64_insn_is_ret_auth(insn) \|\|
	462	+ aarch64_insn_is_br(insn) \|\|
	463	+ aarch64_insn_is_br_auth(insn) \|\|
	464	+ aarch64_insn_is_blr(insn) \|\|
	465	+ aarch64_insn_is_blr_auth(insn) \|\|
	466	+ aarch64_insn_is_bcond(insn);
	467	+}
436	468
437		−static inline bool aarch64_insn_is_adr_adrp(u32 insn)
	469	+static __always_inline bool aarch64_insn_is_branch_imm(u32 insn)
438	470	{
439		− return aarch64_insn_is_adr(insn) \|\| aarch64_insn_is_adrp(insn);
	471	+ return aarch64_insn_is_b(insn) \|\|
	472	+ aarch64_insn_is_bl(insn) \|\|
	473	+ aarch64_insn_is_tbz(insn) \|\|
	474	+ aarch64_insn_is_tbnz(insn) \|\|
	475	+ aarch64_insn_is_cbz(insn) \|\|
	476	+ aarch64_insn_is_cbnz(insn) \|\|
	477	+ aarch64_insn_is_bcond(insn);
440	478	}
441	479
442		−static inline bool aarch64_insn_is_dsb(u32 insn)
	480	+static __always_inline bool aarch64_insn_is_adr_adrp(u32 insn)
443	481	{
444		− return aarch64_insn_is_dsb_base(insn) \|\| aarch64_insn_is_dsb_nxs(insn);
	482	+ return aarch64_insn_is_adr(insn) \|\|
	483	+ aarch64_insn_is_adrp(insn);
445	484	}
446	485
447		−static inline bool aarch64_insn_is_barrier(u32 insn)
	486	+static __always_inline bool aarch64_insn_is_dsb(u32 insn)
448	487	{
449		− return aarch64_insn_is_dmb(insn) \|\| aarch64_insn_is_dsb(insn) \|\|
450		− aarch64_insn_is_isb(insn) \|\| aarch64_insn_is_sb(insn) \|\|
451		− aarch64_insn_is_clrex(insn) \|\| aarch64_insn_is_ssbb(insn) \|\|
	488	+ return aarch64_insn_is_dsb_base(insn) \|\|
	489	+ aarch64_insn_is_dsb_nxs(insn);
	490	+}
	491	+
	492	+static __always_inline bool aarch64_insn_is_barrier(u32 insn)
	493	+{
	494	+ return aarch64_insn_is_dmb(insn) \|\|
	495	+ aarch64_insn_is_dsb(insn) \|\|
	496	+ aarch64_insn_is_isb(insn) \|\|
	497	+ aarch64_insn_is_sb(insn) \|\|
	498	+ aarch64_insn_is_clrex(insn) \|\|
	499	+ aarch64_insn_is_ssbb(insn) \|\|
452	500	aarch64_insn_is_pssbb(insn);
453	501	}
454	502
455		−static inline bool aarch64_insn_is_store_single(u32 insn)
	503	+static __always_inline bool aarch64_insn_is_store_single(u32 insn)
456	504	{
457	505	return aarch64_insn_is_store_imm(insn) \|\|
458	506	aarch64_insn_is_store_pre(insn) \|\|
459	507	aarch64_insn_is_store_post(insn);
460	508	}
461	509
462		−static inline bool aarch64_insn_is_store_pair(u32 insn)
	510	+static __always_inline bool aarch64_insn_is_store_pair(u32 insn)
463	511	{
464	512	return aarch64_insn_is_stp(insn) \|\|
465	513	aarch64_insn_is_stp_pre(insn) \|\|
466	514	aarch64_insn_is_stp_post(insn);
467	515	}
468	516
469		−static inline bool aarch64_insn_is_load_single(u32 insn)
	517	+static __always_inline bool aarch64_insn_is_load_single(u32 insn)
470	518	{
471	519	return aarch64_insn_is_load_imm(insn) \|\|
472	520	aarch64_insn_is_load_pre(insn) \|\|
473	521	aarch64_insn_is_load_post(insn);
474	522	}
475	523
476		−static inline bool aarch64_insn_is_load_pair(u32 insn)
	524	+static __always_inline bool aarch64_insn_is_load_pair(u32 insn)
477	525	{
478	526	return aarch64_insn_is_ldp(insn) \|\|
479	527	aarch64_insn_is_ldp_pre(insn) \|\|
480	528	aarch64_insn_is_ldp_post(insn);
481	529	}
482	530
	531	+static __always_inline bool aarch64_insn_uses_literal(u32 insn)
	532	+{
	533	+ /* ldr/ldrsw (literal), prfm */
	534	+
	535	+ return aarch64_insn_is_ldr_lit(insn) \|\|
	536	+ aarch64_insn_is_ldrsw_lit(insn) \|\|
	537	+ aarch64_insn_is_adr_adrp(insn) \|\|
	538	+ aarch64_insn_is_prfm_lit(insn);
	539	+}
	540	+
483	541	enum aarch64_insn_encoding_class aarch64_get_insn_class(u32 insn);
484		−bool aarch64_insn_uses_literal(u32 insn);
485		−bool aarch64_insn_is_branch(u32 insn);
486	542	u64 aarch64_insn_decode_immediate(enum aarch64_insn_imm_type type, u32 insn);
487	543	u32 aarch64_insn_encode_immediate(enum aarch64_insn_imm_type type,
488	544	u32 insn, u64 imm);
@@ -496,8 +552,18 @@ u32 aarch64_insn_gen_comp_branch_imm(unsigned long pc, unsigned long addr, · 7 unmodified lines
496	552	enum aarch64_insn_branch_type type);
497	553	u32 aarch64_insn_gen_cond_branch_imm(unsigned long pc, unsigned long addr,
498	554	enum aarch64_insn_condition cond);
499		−u32 aarch64_insn_gen_hint(enum aarch64_insn_hint_cr_op op);
500		−u32 aarch64_insn_gen_nop(void);
	555	+
	556	+static __always_inline u32
	557	+aarch64_insn_gen_hint(enum aarch64_insn_hint_cr_op op)
	558	+{
	559	+ return aarch64_insn_get_hint_value() \| op;
	560	+}
	561	+
	562	+static __always_inline u32 aarch64_insn_gen_nop(void)
	563	+{
	564	+ return aarch64_insn_gen_hint(AARCH64_INSN_HINT_NOP);
	565	+}
	566	+
501	567	u32 aarch64_insn_gen_branch_reg(enum aarch64_insn_register reg,
502	568	enum aarch64_insn_branch_type type);
503	569	u32 aarch64_insn_gen_load_store_reg(enum aarch64_insn_register reg,
@@ -580,10 +646,6 @@ u32 aarch64_insn_gen_extr(enum aarch64_insn_variant variant, · 76 unmodified lines
580	646	enum aarch64_insn_register Rn,
581	647	enum aarch64_insn_register Rd,
582	648	u8 lsb);
583		−u32 aarch64_insn_gen_prefetch(enum aarch64_insn_register base,
584		− enum aarch64_insn_prfm_type type,
585		− enum aarch64_insn_prfm_target target,
586		− enum aarch64_insn_prfm_policy policy);
587	649	#ifdef CONFIG_ARM64_LSE_ATOMICS
588	650	u32 aarch64_insn_gen_atomic_ld_op(enum aarch64_insn_register result,
589	651	enum aarch64_insn_register address,

arch/arm64/include/asm/jump_label.h+4−4

@@ -15,8 +15,8 @@ · 14 unmodified lines
15	15
16	16	#define JUMP_LABEL_NOP_SIZE AARCH64_INSN_SIZE
17	17
18		−static __always_inline bool arch_static_branch(struct static_key *key,
19		− bool branch)
	18	+static __always_inline bool arch_static_branch(struct static_key * const key,
	19	+ const bool branch)
20	20	{
21	21	asm_volatile_goto(
22	22	"1: nop \n\t"
@@ -32,8 +32,8 @@ l_yes: · 9 unmodified lines
32	32	return true;
33	33	}
34	34
35		−static __always_inline bool arch_static_branch_jump(struct static_key *key,
36		− bool branch)
	35	+static __always_inline bool arch_static_branch_jump(struct static_key * const key,
	36	+ const bool branch)
37	37	{
38	38	asm_volatile_goto(
39	39	"1: b %l[l_yes] \n\t"

arch/arm64/include/asm/kernel-pgtable.h+3−8

@@ -18,11 +18,6 @@ · 17 unmodified lines
18	18	* with 4K (section size = 2M) but not with 16K (section size = 32M) or
19	19	* 64K (section size = 512M).
20	20	*/
21		−#ifdef CONFIG_ARM64_4K_PAGES
22		−#define ARM64_KERNEL_USES_PMD_MAPS 1
23		−#else
24		−#define ARM64_KERNEL_USES_PMD_MAPS 0
25		−#endif
26	21
27	22	/*
28	23	* The idmap and swapper page tables need some space reserved in the kernel
@@ -34,7 +29,7 @@ · 5 unmodified lines
34	29	* VA range, so pages required to map highest possible PA are reserved in all
35	30	* cases.
36	31	*/
37		−#if ARM64_KERNEL_USES_PMD_MAPS
	32	+#ifdef CONFIG_ARM64_4K_PAGES
38	33	#define SWAPPER_PGTABLE_LEVELS (CONFIG_PGTABLE_LEVELS - 1)
39	34	#else
40	35	#define SWAPPER_PGTABLE_LEVELS (CONFIG_PGTABLE_LEVELS)
@@ -96,7 +91,7 @@ · 55 unmodified lines
96	91	#define INIT_IDMAP_DIR_PAGES EARLY_PAGES(KIMAGE_VADDR, _end + MAX_FDT_SIZE + SWAPPER_BLOCK_SIZE, 1)
97	92
98	93	/* Initial memory map size */
99		−#if ARM64_KERNEL_USES_PMD_MAPS
	94	+#ifdef CONFIG_ARM64_4K_PAGES
100	95	#define SWAPPER_BLOCK_SHIFT PMD_SHIFT
101	96	#define SWAPPER_BLOCK_SIZE PMD_SIZE
102	97	#define SWAPPER_TABLE_SHIFT PUD_SHIFT
@@ -112,7 +107,7 @@ · 9 unmodified lines
112	107	#define SWAPPER_PTE_FLAGS (PTE_TYPE_PAGE \| PTE_AF \| PTE_SHARED)
113	108	#define SWAPPER_PMD_FLAGS (PMD_TYPE_SECT \| PMD_SECT_AF \| PMD_SECT_S)
114	109
115		−#if ARM64_KERNEL_USES_PMD_MAPS
	110	+#ifdef CONFIG_ARM64_4K_PAGES
116	111	#define SWAPPER_RW_MMUFLAGS (PMD_ATTRINDX(MT_NORMAL) \| SWAPPER_PMD_FLAGS)
117	112	#define SWAPPER_RX_MMUFLAGS (SWAPPER_RW_MMUFLAGS \| PMD_SECT_RDONLY)
118	113	#else

arch/arm64/include/asm/kvm_host.h+11−1

@@ -306,8 +306,18 @@ struct vcpu_reset_state { · 305 unmodified lines
306	306	struct kvm_vcpu_arch {
307	307	struct kvm_cpu_context ctxt;
308	308
309		− /* Guest floating point state */
	309	+ /*
	310	+ * Guest floating point state
	311	+ *
	312	+ * The architecture has two main floating point extensions,
	313	+ * the original FPSIMD and SVE. These have overlapping
	314	+ * register views, with the FPSIMD V registers occupying the
	315	+ * low 128 bits of the SVE Z registers. When the core
	316	+ * floating point code saves the register state of a task it
	317	+ * records which view it saved in fp_type.
	318	+ */
310	319	void *sve_state;
	320	+ enum fp_type fp_type;
311	321	unsigned int sve_max_vl;
312	322	u64 svcr;
313	323

arch/arm64/include/asm/lse.h+0−1

@@ -10,7 +10,6 @@ · 9 unmodified lines
10	10
11	11	#include <linux/compiler_types.h>
12	12	#include <linux/export.h>
13		−#include <linux/jump_label.h>
14	13	#include <linux/stringify.h>
15	14	#include <asm/alternative.h>
16	15	#include <asm/alternative-macros.h>

arch/arm64/include/asm/mmu_context.h+10−0

@@ -18,6 +18,7 @@ · 17 unmodified lines
18	18
19	19	#include <asm/cacheflush.h>
20	20	#include <asm/cpufeature.h>
	21	+#include <asm/daifflags.h>
21	22	#include <asm/proc-fns.h>
22	23	#include <asm-generic/mm_hooks.h>
23	24	#include <asm/cputype.h>
@@ -152,6 +153,7 @@ static inline void cpu_replace_ttbr1(pgd_t pgdp, pgd_t idmap) · 128 unmodified lines
152	153	typedef void (ttbr_replace_func)(phys_addr_t);
153	154	extern ttbr_replace_func idmap_cpu_replace_ttbr1;
154	155	ttbr_replace_func *replace_phys;
	156	+ unsigned long daif;
155	157
156	158	/* phys_to_ttbr() zeros lower 2 bits of ttbr with 52-bit PA */
157	159	phys_addr_t ttbr1 = phys_to_ttbr(virt_to_phys(pgdp));
@@ -171,7 +173,15 @@ static inline void cpu_replace_ttbr1(pgd_t pgdp, pgd_t idmap) · 13 unmodified lines
171	173	replace_phys = (void *)__pa_symbol(idmap_cpu_replace_ttbr1);
172	174
173	175	__cpu_install_idmap(idmap);
	176	+
	177	+ /*
	178	+ * We really don't want to take any exceptions while TTBR1 is
	179	+ * in the process of being replaced so mask everything.
	180	+ */
	181	+ daif = local_daif_save();
174	182	replace_phys(ttbr1);
	183	+ local_daif_restore(daif);
	184	+
175	185	cpu_uninstall_idmap();
176	186	}
177	187

arch/arm64/include/asm/module.lds.h+8−0

@@ -17,4 +17,12 @@ SECTIONS { · 16 unmodified lines
17	17	*/
18	18	.text.hot : { *(.text.hot) }
19	19	#endif
	20	+
	21	+#ifdef CONFIG_UNWIND_TABLES
	22	+ /*
	23	+ * Currently, we only use unwind info at module load time, so we can
	24	+ * put it into the .init allocation.
	25	+ */
	26	+ .init.eh_frame : { *(.eh_frame) }
	27	+#endif
20	28	}

arch/arm64/include/asm/pgtable-hwdef.h+1−0

@@ -159,6 +159,7 @@ · 158 unmodified lines
159	159	#ifdef CONFIG_ARM64_PA_BITS_52
160	160	#define PTE_ADDR_HIGH (_AT(pteval_t, 0xf) << 12)
161	161	#define PTE_ADDR_MASK (PTE_ADDR_LOW \| PTE_ADDR_HIGH)
	162	+#define PTE_ADDR_HIGH_SHIFT 36
162	163	#else
163	164	#define PTE_ADDR_MASK PTE_ADDR_LOW
164	165	#endif

arch/arm64/include/asm/pgtable.h+11−3

@@ -77,11 +77,11 @@ extern unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)]; · 76 unmodified lines
77	77	static inline phys_addr_t __pte_to_phys(pte_t pte)
78	78	{
79	79	return (pte_val(pte) & PTE_ADDR_LOW) \|
80		− ((pte_val(pte) & PTE_ADDR_HIGH) << 36);
	80	+ ((pte_val(pte) & PTE_ADDR_HIGH) << PTE_ADDR_HIGH_SHIFT);
81	81	}
82	82	static inline pteval_t __phys_to_pte_val(phys_addr_t phys)
83	83	{
84		− return (phys \| (phys >> 36)) & PTE_ADDR_MASK;
	84	+ return (phys \| (phys >> PTE_ADDR_HIGH_SHIFT)) & PTE_ADDR_MASK;
85	85	}
86	86	#else
87	87	#define __pte_to_phys(pte) (pte_val(pte) & PTE_ADDR_MASK)
@@ -609,7 +609,6 @@ extern pgd_t init_pg_dir[PTRS_PER_PGD]; · 521 unmodified lines
609	609	extern pgd_t init_pg_end[];
610	610	extern pgd_t swapper_pg_dir[PTRS_PER_PGD];
611	611	extern pgd_t idmap_pg_dir[PTRS_PER_PGD];
612		−extern pgd_t idmap_pg_end[];
613	612	extern pgd_t tramp_pg_dir[PTRS_PER_PGD];
614	613	extern pgd_t reserved_pg_dir[PTRS_PER_PGD];
615	614
@@ -1096,6 +1095,15 @@ static inline bool pud_sect_supported(void) · 480 unmodified lines
1096	1095	}
1097	1096
1098	1097
	1098	+#define __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION
	1099	+#define ptep_modify_prot_start ptep_modify_prot_start
	1100	+extern pte_t ptep_modify_prot_start(struct vm_area_struct *vma,
	1101	+ unsigned long addr, pte_t *ptep);
	1102	+
	1103	+#define ptep_modify_prot_commit ptep_modify_prot_commit
	1104	+extern void ptep_modify_prot_commit(struct vm_area_struct *vma,
	1105	+ unsigned long addr, pte_t *ptep,
	1106	+ pte_t old_pte, pte_t new_pte);
1099	1107	#endif /* !__ASSEMBLY__ */
1100	1108
1101	1109	#endif /* __ASM_PGTABLE_H */

arch/arm64/include/asm/processor.h+9−15

@@ -122,6 +122,12 @@ enum vec_type { · 121 unmodified lines
122	122	ARM64_VEC_MAX,
123	123	};
124	124
	125	+enum fp_type {
	126	+ FP_STATE_CURRENT, /* Save based on current task state. */
	127	+ FP_STATE_FPSIMD,
	128	+ FP_STATE_SVE,
	129	+};
	130	+
125	131	struct cpu_context {
126	132	unsigned long x19;
127	133	unsigned long x20;
@@ -152,6 +158,7 @@ struct thread_struct { · 24 unmodified lines
152	158	struct user_fpsimd_state fpsimd_state;
153	159	} uw;
154	160
	161	+ enum fp_type fp_type; /* registers FPSIMD or SVE? */
155	162	unsigned int fpsimd_cpu;
156	163	void sve_state; / SVE registers, if any */
157	164	void za_state; / ZA register, if any */
@@ -308,13 +315,13 @@ static inline void compat_start_thread(struct pt_regs *regs, unsigned long pc, · 150 unmodified lines
308	315	}
309	316	#endif
310	317
311		−static inline bool is_ttbr0_addr(unsigned long addr)
	318	+static __always_inline bool is_ttbr0_addr(unsigned long addr)
312	319	{
313	320	/* entry assembly clears tags for TTBR0 addrs */
314	321	return addr < TASK_SIZE;
315	322	}
316	323
317		−static inline bool is_ttbr1_addr(unsigned long addr)
	324	+static __always_inline bool is_ttbr1_addr(unsigned long addr)
318	325	{
319	326	/* TTBR1 addresses may have a tag if KASAN_SW_TAGS is in use */
320	327	return arch_kasan_reset_tag(addr) >= PAGE_OFFSET;
@@ -396,18 +403,5 @@ long get_tagged_addr_ctrl(struct task_struct *task); · 75 unmodified lines
396	403	#define GET_TAGGED_ADDR_CTRL() get_tagged_addr_ctrl(current)
397	404	#endif
398	405
399		−/*
400		− * For CONFIG_GCC_PLUGIN_STACKLEAK
401		− *
402		− * These need to be macros because otherwise we get stuck in a nightmare
403		− * of header definitions for the use of task_stack_page.
404		− */
405		−
406		−/*
407		− * The top of the current task's task stack
408		− */
409		−#define current_top_of_stack() ((unsigned long)current->stack + THREAD_SIZE)
410		−#define on_thread_stack() (on_task_stack(current, current_stack_pointer, 1))
411		−
412	406	#endif /* __ASSEMBLY__ */
413	407	#endif /* __ASM_PROCESSOR_H */

arch/arm64/include/asm/scs.h+49−0

@@ -5,6 +5,7 @@ · 4 unmodified lines
5	5	#ifdef __ASSEMBLY__
6	6
7	7	#include <asm/asm-offsets.h>
	8	+#include <asm/sysreg.h>
8	9
9	10	#ifdef CONFIG_SHADOW_CALL_STACK
10	11	scs_sp .req x18
@@ -24,6 +25,54 @@ · 13 unmodified lines
24	25	.endm
25	26	#endif /* CONFIG_SHADOW_CALL_STACK */
26	27
	28	+
	29	+#else
	30	+
	31	+#include <linux/scs.h>
	32	+#include <asm/cpufeature.h>
	33	+
	34	+#ifdef CONFIG_UNWIND_PATCH_PAC_INTO_SCS
	35	+static inline bool should_patch_pac_into_scs(void)
	36	+{
	37	+ u64 reg;
	38	+
	39	+ /*
	40	+ * We only enable the shadow call stack dynamically if we are running
	41	+ * on a system that does not implement PAC or BTI. PAC and SCS provide
	42	+ * roughly the same level of protection, and BTI relies on the PACIASP
	43	+ * instructions serving as landing pads, preventing us from patching
	44	+ * those instructions into something else.
	45	+ */
	46	+ reg = read_sysreg_s(SYS_ID_AA64ISAR1_EL1);
	47	+ if (SYS_FIELD_GET(ID_AA64ISAR1_EL1, APA, reg) \|
	48	+ SYS_FIELD_GET(ID_AA64ISAR1_EL1, API, reg))
	49	+ return false;
	50	+
	51	+ reg = read_sysreg_s(SYS_ID_AA64ISAR2_EL1);
	52	+ if (SYS_FIELD_GET(ID_AA64ISAR2_EL1, APA3, reg))
	53	+ return false;
	54	+
	55	+ if (IS_ENABLED(CONFIG_ARM64_BTI_KERNEL)) {
	56	+ reg = read_sysreg_s(SYS_ID_AA64PFR1_EL1);
	57	+ if (reg & (0xf << ID_AA64PFR1_EL1_BT_SHIFT))
	58	+ return false;
	59	+ }
	60	+ return true;
	61	+}
	62	+
	63	+static inline void dynamic_scs_init(void)
	64	+{
	65	+ if (should_patch_pac_into_scs()) {
	66	+ pr_info("Enabling dynamic shadow call stack\n");
	67	+ static_branch_enable(&dynamic_scs_enabled);
	68	+ }
	69	+}
	70	+#else
	71	+static inline void dynamic_scs_init(void) {}
	72	+#endif
	73	+
	74	+int scs_patch(const u8 eh_frame[], int size);
	75	+
27	76	#endif /* __ASSEMBLY __ */
28	77
29	78	#endif /* _ASM_SCS_H */

arch/arm64/include/asm/spectre.h+2−0

@@ -26,6 +26,7 @@ enum mitigation_state { · 25 unmodified lines
26	26	SPECTRE_VULNERABLE,
27	27	};
28	28
	29	+struct pt_regs;
29	30	struct task_struct;
30	31
31	32	/*
@@ -98,5 +99,6 @@ enum mitigation_state arm64_get_spectre_bhb_state(void); · 66 unmodified lines
98	99	bool is_spectre_bhb_affected(const struct arm64_cpu_capabilities *entry, int scope);
99	100	u8 spectre_bhb_loop_affected(int scope);
100	101	void spectre_bhb_enable_mitigation(const struct arm64_cpu_capabilities *__unused);
	102	+bool try_emulate_el1_ssbs(struct pt_regs *regs, u32 instr);
101	103	#endif /* __ASSEMBLY__ */
102	104	#endif /* __ASM_SPECTRE_H */

arch/arm64/include/asm/stacktrace.h+2−0

@@ -57,6 +57,8 @@ static inline bool on_task_stack(const struct task_struct *tsk, · 56 unmodified lines
57	57	return stackinfo_on_stack(&info, sp, size);
58	58	}
59	59
	60	+#define on_thread_stack() (on_task_stack(current, current_stack_pointer, 1))
	61	+
60	62	#ifdef CONFIG_VMAP_STACK
61	63	DECLARE_PER_CPU(unsigned long [OVERFLOW_STACK_SIZE/sizeof(long)], overflow_stack);
62	64

arch/arm64/include/asm/sysreg.h+8−142

@@ -90,20 +90,24 @@ · 89 unmodified lines
90	90	*/
91	91	#define pstate_field(op1, op2) ((op1) << Op1_shift \| (op2) << Op2_shift)
92	92	#define PSTATE_Imm_shift CRm_shift
	93	+#define SET_PSTATE(x, r) __emit_inst(0xd500401f \| PSTATE_ ## r \| ((!!x) << PSTATE_Imm_shift))
93	94
94	95	#define PSTATE_PAN pstate_field(0, 4)
95	96	#define PSTATE_UAO pstate_field(0, 3)
96	97	#define PSTATE_SSBS pstate_field(3, 1)
	98	+#define PSTATE_DIT pstate_field(3, 2)
97	99	#define PSTATE_TCO pstate_field(3, 4)
98	100
99		−#define SET_PSTATE_PAN(x) __emit_inst(0xd500401f \| PSTATE_PAN \| ((!!x) << PSTATE_Imm_shift))
100		−#define SET_PSTATE_UAO(x) __emit_inst(0xd500401f \| PSTATE_UAO \| ((!!x) << PSTATE_Imm_shift))
101		−#define SET_PSTATE_SSBS(x) __emit_inst(0xd500401f \| PSTATE_SSBS \| ((!!x) << PSTATE_Imm_shift))
102		−#define SET_PSTATE_TCO(x) __emit_inst(0xd500401f \| PSTATE_TCO \| ((!!x) << PSTATE_Imm_shift))
	101	+#define SET_PSTATE_PAN(x) SET_PSTATE((x), PAN)
	102	+#define SET_PSTATE_UAO(x) SET_PSTATE((x), UAO)
	103	+#define SET_PSTATE_SSBS(x) SET_PSTATE((x), SSBS)
	104	+#define SET_PSTATE_DIT(x) SET_PSTATE((x), DIT)
	105	+#define SET_PSTATE_TCO(x) SET_PSTATE((x), TCO)
103	106
104	107	#define set_pstate_pan(x) asm volatile(SET_PSTATE_PAN(x))
105	108	#define set_pstate_uao(x) asm volatile(SET_PSTATE_UAO(x))
106	109	#define set_pstate_ssbs(x) asm volatile(SET_PSTATE_SSBS(x))
	110	+#define set_pstate_dit(x) asm volatile(SET_PSTATE_DIT(x))
107	111
108	112	#define __SYS_BARRIER_INSN(CRm, op2, Rt) \
109	113	__emit_inst(0xd5000000 \| sys_insn(0, 3, 3, (CRm), (op2)) \| ((Rt) & 0x1f))
@@ -165,31 +169,6 @@ · 55 unmodified lines
165	169	#define SYS_MPIDR_EL1 sys_reg(3, 0, 0, 0, 5)
166	170	#define SYS_REVIDR_EL1 sys_reg(3, 0, 0, 0, 6)
167	171
168		−#define SYS_ID_PFR0_EL1 sys_reg(3, 0, 0, 1, 0)
169		−#define SYS_ID_PFR1_EL1 sys_reg(3, 0, 0, 1, 1)
170		−#define SYS_ID_PFR2_EL1 sys_reg(3, 0, 0, 3, 4)
171		−#define SYS_ID_DFR0_EL1 sys_reg(3, 0, 0, 1, 2)
172		−#define SYS_ID_DFR1_EL1 sys_reg(3, 0, 0, 3, 5)
173		−#define SYS_ID_AFR0_EL1 sys_reg(3, 0, 0, 1, 3)
174		−#define SYS_ID_MMFR0_EL1 sys_reg(3, 0, 0, 1, 4)
175		−#define SYS_ID_MMFR1_EL1 sys_reg(3, 0, 0, 1, 5)
176		−#define SYS_ID_MMFR2_EL1 sys_reg(3, 0, 0, 1, 6)
177		−#define SYS_ID_MMFR3_EL1 sys_reg(3, 0, 0, 1, 7)
178		−#define SYS_ID_MMFR4_EL1 sys_reg(3, 0, 0, 2, 6)
179		−#define SYS_ID_MMFR5_EL1 sys_reg(3, 0, 0, 3, 6)
180		−
181		−#define SYS_ID_ISAR0_EL1 sys_reg(3, 0, 0, 2, 0)
182		−#define SYS_ID_ISAR1_EL1 sys_reg(3, 0, 0, 2, 1)
183		−#define SYS_ID_ISAR2_EL1 sys_reg(3, 0, 0, 2, 2)
184		−#define SYS_ID_ISAR3_EL1 sys_reg(3, 0, 0, 2, 3)
185		−#define SYS_ID_ISAR4_EL1 sys_reg(3, 0, 0, 2, 4)
186		−#define SYS_ID_ISAR5_EL1 sys_reg(3, 0, 0, 2, 5)
187		−#define SYS_ID_ISAR6_EL1 sys_reg(3, 0, 0, 2, 7)
188		−
189		−#define SYS_MVFR0_EL1 sys_reg(3, 0, 0, 3, 0)
190		−#define SYS_MVFR1_EL1 sys_reg(3, 0, 0, 3, 1)
191		−#define SYS_MVFR2_EL1 sys_reg(3, 0, 0, 3, 2)
192		−
193	172	#define SYS_ACTLR_EL1 sys_reg(3, 0, 1, 0, 1)
194	173	#define SYS_RGSR_EL1 sys_reg(3, 0, 1, 0, 5)
195	174	#define SYS_GCR_EL1 sys_reg(3, 0, 1, 0, 6)
@@ -692,112 +671,6 @@ · 496 unmodified lines
692	671	#define ID_AA64MMFR0_EL1_PARANGE_MAX ID_AA64MMFR0_EL1_PARANGE_48
693	672	#endif
694	673
695		−#define ID_DFR0_PERFMON_SHIFT 24
696		−
697		−#define ID_DFR0_PERFMON_8_0 0x3
698		−#define ID_DFR0_PERFMON_8_1 0x4
699		−#define ID_DFR0_PERFMON_8_4 0x5
700		−#define ID_DFR0_PERFMON_8_5 0x6
701		−
702		−#define ID_ISAR4_SWP_FRAC_SHIFT 28
703		−#define ID_ISAR4_PSR_M_SHIFT 24
704		−#define ID_ISAR4_SYNCH_PRIM_FRAC_SHIFT 20
705		−#define ID_ISAR4_BARRIER_SHIFT 16
706		−#define ID_ISAR4_SMC_SHIFT 12
707		−#define ID_ISAR4_WRITEBACK_SHIFT 8
708		−#define ID_ISAR4_WITHSHIFTS_SHIFT 4
709		−#define ID_ISAR4_UNPRIV_SHIFT 0
710		−
711		−#define ID_DFR1_MTPMU_SHIFT 0
712		−
713		−#define ID_ISAR0_DIVIDE_SHIFT 24
714		−#define ID_ISAR0_DEBUG_SHIFT 20
715		−#define ID_ISAR0_COPROC_SHIFT 16
716		−#define ID_ISAR0_CMPBRANCH_SHIFT 12
717		−#define ID_ISAR0_BITFIELD_SHIFT 8
718		−#define ID_ISAR0_BITCOUNT_SHIFT 4
719		−#define ID_ISAR0_SWAP_SHIFT 0
720		−
721		−#define ID_ISAR5_RDM_SHIFT 24
722		−#define ID_ISAR5_CRC32_SHIFT 16
723		−#define ID_ISAR5_SHA2_SHIFT 12
724		−#define ID_ISAR5_SHA1_SHIFT 8
725		−#define ID_ISAR5_AES_SHIFT 4
726		−#define ID_ISAR5_SEVL_SHIFT 0
727		−
728		−#define ID_ISAR6_I8MM_SHIFT 24
729		−#define ID_ISAR6_BF16_SHIFT 20
730		−#define ID_ISAR6_SPECRES_SHIFT 16
731		−#define ID_ISAR6_SB_SHIFT 12
732		−#define ID_ISAR6_FHM_SHIFT 8
733		−#define ID_ISAR6_DP_SHIFT 4
734		−#define ID_ISAR6_JSCVT_SHIFT 0
735		−
736		−#define ID_MMFR0_INNERSHR_SHIFT 28
737		−#define ID_MMFR0_FCSE_SHIFT 24
738		−#define ID_MMFR0_AUXREG_SHIFT 20
739		−#define ID_MMFR0_TCM_SHIFT 16
740		−#define ID_MMFR0_SHARELVL_SHIFT 12
741		−#define ID_MMFR0_OUTERSHR_SHIFT 8
742		−#define ID_MMFR0_PMSA_SHIFT 4
743		−#define ID_MMFR0_VMSA_SHIFT 0
744		−
745		−#define ID_MMFR4_EVT_SHIFT 28
746		−#define ID_MMFR4_CCIDX_SHIFT 24
747		−#define ID_MMFR4_LSM_SHIFT 20
748		−#define ID_MMFR4_HPDS_SHIFT 16
749		−#define ID_MMFR4_CNP_SHIFT 12
750		−#define ID_MMFR4_XNX_SHIFT 8
751		−#define ID_MMFR4_AC2_SHIFT 4
752		−#define ID_MMFR4_SPECSEI_SHIFT 0
753		−
754		−#define ID_MMFR5_ETS_SHIFT 0
755		−
756		−#define ID_PFR0_DIT_SHIFT 24
757		−#define ID_PFR0_CSV2_SHIFT 16
758		−#define ID_PFR0_STATE3_SHIFT 12
759		−#define ID_PFR0_STATE2_SHIFT 8
760		−#define ID_PFR0_STATE1_SHIFT 4
761		−#define ID_PFR0_STATE0_SHIFT 0
762		−
763		−#define ID_DFR0_PERFMON_SHIFT 24
764		−#define ID_DFR0_MPROFDBG_SHIFT 20
765		−#define ID_DFR0_MMAPTRC_SHIFT 16
766		−#define ID_DFR0_COPTRC_SHIFT 12
767		−#define ID_DFR0_MMAPDBG_SHIFT 8
768		−#define ID_DFR0_COPSDBG_SHIFT 4
769		−#define ID_DFR0_COPDBG_SHIFT 0
770		−
771		−#define ID_PFR2_SSBS_SHIFT 4
772		−#define ID_PFR2_CSV3_SHIFT 0
773		−
774		−#define MVFR0_FPROUND_SHIFT 28
775		−#define MVFR0_FPSHVEC_SHIFT 24
776		−#define MVFR0_FPSQRT_SHIFT 20
777		−#define MVFR0_FPDIVIDE_SHIFT 16
778		−#define MVFR0_FPTRAP_SHIFT 12
779		−#define MVFR0_FPDP_SHIFT 8
780		−#define MVFR0_FPSP_SHIFT 4
781		−#define MVFR0_SIMD_SHIFT 0
782		−
783		−#define MVFR1_SIMDFMAC_SHIFT 28
784		−#define MVFR1_FPHP_SHIFT 24
785		−#define MVFR1_SIMDHP_SHIFT 20
786		−#define MVFR1_SIMDSP_SHIFT 16
787		−#define MVFR1_SIMDINT_SHIFT 12
788		−#define MVFR1_SIMDLS_SHIFT 8
789		−#define MVFR1_FPDNAN_SHIFT 4
790		−#define MVFR1_FPFTZ_SHIFT 0
791		−
792		−#define ID_PFR1_GIC_SHIFT 28
793		−#define ID_PFR1_VIRT_FRAC_SHIFT 24
794		−#define ID_PFR1_SEC_FRAC_SHIFT 20
795		−#define ID_PFR1_GENTIMER_SHIFT 16
796		−#define ID_PFR1_VIRTUALIZATION_SHIFT 12
797		−#define ID_PFR1_MPROGMOD_SHIFT 8
798		−#define ID_PFR1_SECURITY_SHIFT 4
799		−#define ID_PFR1_PROGMOD_SHIFT 0
800		−
801	674	#if defined(CONFIG_ARM64_4K_PAGES)
802	675	#define ID_AA64MMFR0_EL1_TGRAN_SHIFT ID_AA64MMFR0_EL1_TGRAN4_SHIFT
803	676	#define ID_AA64MMFR0_EL1_TGRAN_SUPPORTED_MIN ID_AA64MMFR0_EL1_TGRAN4_SUPPORTED_MIN
@@ -815,9 +688,6 @@ · 11 unmodified lines
815	688	#define ID_AA64MMFR0_EL1_TGRAN_2_SHIFT ID_AA64MMFR0_EL1_TGRAN64_2_SHIFT
816	689	#endif
817	690
818		−#define MVFR2_FPMISC_SHIFT 4
819		−#define MVFR2_SIMDMISC_SHIFT 0
820		−
821	691	#define CPACR_EL1_FPEN_EL1EN (BIT(20)) /* enable EL1 access */
822	692	#define CPACR_EL1_FPEN_EL0EN (BIT(21)) /* enable EL0 access, if EL1EN set */
823	693
@@ -851,10 +721,6 @@ · 27 unmodified lines
851	721	#define SYS_RGSR_EL1_SEED_SHIFT 8
852	722	#define SYS_RGSR_EL1_SEED_MASK 0xffffUL
853	723
854		−/* GMID_EL1 field definitions */
855		−#define GMID_EL1_BS_SHIFT 0
856		−#define GMID_EL1_BS_SIZE 4
857		−
858	724	/* TFSR{,E0}_EL1 bit definitions */
859	725	#define SYS_TFSR_EL1_TF0_SHIFT 0
860	726	#define SYS_TFSR_EL1_TF1_SHIFT 1

arch/arm64/include/asm/traps.h+9−10

@@ -13,17 +13,16 @@ · 12 unmodified lines
13	13
14	14	struct pt_regs;
15	15
16		−struct undef_hook {
17		− struct list_head node;
18		− u32 instr_mask;
19		− u32 instr_val;
20		− u64 pstate_mask;
21		− u64 pstate_val;
22		− int (fn)(struct pt_regs regs, u32 instr);
23		−};
	16	+#ifdef CONFIG_ARMV8_DEPRECATED
	17	+bool try_emulate_armv8_deprecated(struct pt_regs *regs, u32 insn);
	18	+#else
	19	+static inline bool
	20	+try_emulate_armv8_deprecated(struct pt_regs *regs, u32 insn)
	21	+{
	22	+ return false;
	23	+}
	24	+#endif /* CONFIG_ARMV8_DEPRECATED */
24	25
25		−void register_undef_hook(struct undef_hook *hook);
26		−void unregister_undef_hook(struct undef_hook *hook);
27	26	void force_signal_inject(int signal, int code, unsigned long address, unsigned long err);
28	27	void arm64_notify_segfault(unsigned long addr);
29	28	void arm64_force_sig_fault(int signo, int code, unsigned long far, const char *str);

arch/arm64/include/asm/uprobes.h+1−1

@@ -12,7 +12,7 @@ · 11 unmodified lines
12	12
13	13	#define MAX_UINSN_BYTES AARCH64_INSN_SIZE
14	14
15		−#define UPROBE_SWBP_INSN BRK64_OPCODE_UPROBES
	15	+#define UPROBE_SWBP_INSN cpu_to_le32(BRK64_OPCODE_UPROBES)
16	16	#define UPROBE_SWBP_INSN_SIZE AARCH64_INSN_SIZE
17	17	#define UPROBE_XOL_SLOT_BYTES MAX_UINSN_BYTES
18	18

arch/arm64/include/uapi/asm/hwcap.h+3−0

@@ -93,5 +93,8 @@ · 92 unmodified lines
93	93	#define HWCAP2_WFXT (1UL << 31)
94	94	#define HWCAP2_EBF16 (1UL << 32)
95	95	#define HWCAP2_SVE_EBF16 (1UL << 33)
	96	+#define HWCAP2_CSSC (1UL << 34)
	97	+#define HWCAP2_RPRFM (1UL << 35)
	98	+#define HWCAP2_SVE2P1 (1UL << 36)
96	99
97	100	#endif /* _UAPI__ASM_HWCAP_H */

arch/arm64/include/uapi/asm/sigcontext.h+4−0

@@ -62,6 +62,10 @@ struct sigcontext { · 61 unmodified lines
62	62	* context. Such structures must be placed after the rt_sigframe on the stack
63	63	* and be 16-byte aligned. The last structure must be a dummy one with the
64	64	* magic and size set to 0.
	65	+ *
	66	+ * Note that the values allocated for use as magic should be chosen to
	67	+ * be meaningful in ASCII to aid manual parsing, ZA doesn't follow this
	68	+ * convention due to oversight but it should be observed for future additions.
65	69	*/
66	70	struct _aarch64_ctx {
67	71	__u32 magic;

arch/arm64/kernel/Makefile+2−0

@@ -80,6 +80,8 @@ obj-$(CONFIG_ARM64_PTR_AUTH) += pointer_auth.o · 79 unmodified lines
80	80	obj-$(CONFIG_ARM64_MTE) += mte.o
81	81	obj-y += vdso-wrap.o
82	82	obj-$(CONFIG_COMPAT_VDSO) += vdso32-wrap.o
	83	+obj-$(CONFIG_UNWIND_PATCH_PAC_INTO_SCS) += patch-scs.o
	84	+CFLAGS_patch-scs.o += -mbranch-protection=none
83	85
84	86	# Force dependency (vdso*-wrap.S includes vdso.so through incbin)
85	87	$(obj)/vdso-wrap.o: $(obj)/vdso/vdso.so

arch/arm64/kernel/alternative.c+3−3

@@ -196,7 +196,7 @@ static void __apply_alternatives(const struct alt_region *region, · 195 unmodified lines
196	196	}
197	197	}
198	198
199		−void apply_alternatives_vdso(void)
	199	+static void __init apply_alternatives_vdso(void)
200	200	{
201	201	struct alt_region region;
202	202	const struct elf64_hdr *hdr;
@@ -220,7 +220,7 @@ void apply_alternatives_vdso(void) · 17 unmodified lines
220	220	__apply_alternatives(&region, false, &all_capabilities[0]);
221	221	}
222	222
223		−static const struct alt_region kernel_alternatives = {
	223	+static const struct alt_region kernel_alternatives __initconst = {
224	224	.begin = (struct alt_instr *)__alt_instructions,
225	225	.end = (struct alt_instr *)__alt_instructions_end,
226	226	};
@@ -229,7 +229,7 @@ static const struct alt_region kernel_alternatives = { · 2 unmodified lines
229	229	* We might be patching the stop_machine state machine, so implement a
230	230	* really simple polling protocol here.
231	231	*/
232		−static int __apply_alternatives_multi_stop(void *unused)
	232	+static int __init __apply_alternatives_multi_stop(void *unused)
233	233	{
234	234	/* We always have a CPU 0 at this point (__init) */
235	235	if (smp_processor_id()) {

arch/arm64/kernel/armv8_deprecated.c+276−291

@@ -17,7 +17,6 @@ · 16 unmodified lines
17	17	#include <asm/sysreg.h>
18	18	#include <asm/system_misc.h>
19	19	#include <asm/traps.h>
20		−#include <asm/kprobes.h>
21	20
22	21	#define CREATE_TRACE_POINTS
23	22	#include "trace-events-emulation.h"
@@ -39,226 +38,46 @@ enum insn_emulation_mode { · 15 unmodified lines
39	38	enum legacy_insn_status {
40	39	INSN_DEPRECATED,
41	40	INSN_OBSOLETE,
42		−};
43		−
44		−struct insn_emulation_ops {
45		− const char *name;
46		− enum legacy_insn_status status;
47		− struct undef_hook *hooks;
48		− int (*set_hw_mode)(bool enable);
	41	+ INSN_UNAVAILABLE,
49	42	};
50	43
51	44	struct insn_emulation {
52		− struct list_head node;
53		− struct insn_emulation_ops *ops;
	45	+ const char *name;
	46	+ enum legacy_insn_status status;
	47	+ bool (try_emulate)(struct pt_regs regs,
	48	+ u32 insn);
	49	+ int (*set_hw_mode)(bool enable);
	50	+
54	51	int current_mode;
55	52	int min;
56	53	int max;
57		−};
58		−
59		−static LIST_HEAD(insn_emulation);
60		−static int nr_insn_emulated __initdata;
61		−static DEFINE_RAW_SPINLOCK(insn_emulation_lock);
62		−static DEFINE_MUTEX(insn_emulation_mutex);
63		−
64		−static void register_emulation_hooks(struct insn_emulation_ops *ops)
65		−{
66		− struct undef_hook *hook;
67		−
68		− BUG_ON(!ops->hooks);
69		−
70		− for (hook = ops->hooks; hook->instr_mask; hook++)
71		− register_undef_hook(hook);
72		−
73		− pr_notice("Registered %s emulation handler\n", ops->name);
74		−}
75		−
76		−static void remove_emulation_hooks(struct insn_emulation_ops *ops)
77		−{
78		− struct undef_hook *hook;
79		−
80		− BUG_ON(!ops->hooks);
81		−
82		− for (hook = ops->hooks; hook->instr_mask; hook++)
83		− unregister_undef_hook(hook);
84		−
85		− pr_notice("Removed %s emulation handler\n", ops->name);
86		−}
87		−
88		−static void enable_insn_hw_mode(void *data)
89		−{
90		− struct insn_emulation insn = (struct insn_emulation )data;
91		− if (insn->ops->set_hw_mode)
92		− insn->ops->set_hw_mode(true);
93		−}
94		−
95		−static void disable_insn_hw_mode(void *data)
96		−{
97		− struct insn_emulation insn = (struct insn_emulation )data;
98		− if (insn->ops->set_hw_mode)
99		− insn->ops->set_hw_mode(false);
100		−}
101		−
102		−/* Run set_hw_mode(mode) on all active CPUs */
103		−static int run_all_cpu_set_hw_mode(struct insn_emulation *insn, bool enable)
104		−{
105		− if (!insn->ops->set_hw_mode)
106		− return -EINVAL;
107		− if (enable)
108		− on_each_cpu(enable_insn_hw_mode, (void *)insn, true);
109		− else
110		− on_each_cpu(disable_insn_hw_mode, (void *)insn, true);
111		− return 0;
112		−}
113		−
114		−/*
115		− * Run set_hw_mode for all insns on a starting CPU.
116		− * Returns:
117		− * 0 - If all the hooks ran successfully.
118		− * -EINVAL - At least one hook is not supported by the CPU.
119		− */
120		−static int run_all_insn_set_hw_mode(unsigned int cpu)
121		−{
122		− int rc = 0;
123		− unsigned long flags;
124		− struct insn_emulation *insn;
125		−
126		− raw_spin_lock_irqsave(&insn_emulation_lock, flags);
127		− list_for_each_entry(insn, &insn_emulation, node) {
128		− bool enable = (insn->current_mode == INSN_HW);
129		− if (insn->ops->set_hw_mode && insn->ops->set_hw_mode(enable)) {
130		− pr_warn("CPU[%u] cannot support the emulation of %s",
131		− cpu, insn->ops->name);
132		− rc = -EINVAL;
133		− }
134		− }
135		− raw_spin_unlock_irqrestore(&insn_emulation_lock, flags);
136		− return rc;
137		−}
138		−
139		−static int update_insn_emulation_mode(struct insn_emulation *insn,
140		− enum insn_emulation_mode prev)
141		−{
142		− int ret = 0;
143		−
144		− switch (prev) {
145		− case INSN_UNDEF: /* Nothing to be done */
146		− break;
147		− case INSN_EMULATE:
148		− remove_emulation_hooks(insn->ops);
149		− break;
150		− case INSN_HW:
151		− if (!run_all_cpu_set_hw_mode(insn, false))
152		− pr_notice("Disabled %s support\n", insn->ops->name);
153		− break;
154		− }
155		−
156		− switch (insn->current_mode) {
157		− case INSN_UNDEF:
158		− break;
159		− case INSN_EMULATE:
160		− register_emulation_hooks(insn->ops);
161		− break;
162		− case INSN_HW:
163		− ret = run_all_cpu_set_hw_mode(insn, true);
164		− if (!ret)
165		− pr_notice("Enabled %s support\n", insn->ops->name);
166		− break;
167		− }
168	54
169		− return ret;
170		−}
171		−
172		−static void __init register_insn_emulation(struct insn_emulation_ops *ops)
173		−{
174		− unsigned long flags;
175		− struct insn_emulation *insn;
176		−
177		− insn = kzalloc(sizeof(*insn), GFP_KERNEL);
178		− if (!insn)
179		− return;
180		−
181		− insn->ops = ops;
182		− insn->min = INSN_UNDEF;
183		−
184		− switch (ops->status) {
185		− case INSN_DEPRECATED:
186		− insn->current_mode = INSN_EMULATE;
187		− /* Disable the HW mode if it was turned on at early boot time */
188		− run_all_cpu_set_hw_mode(insn, false);
189		− insn->max = INSN_HW;
190		− break;
191		− case INSN_OBSOLETE:
192		− insn->current_mode = INSN_UNDEF;
193		− insn->max = INSN_EMULATE;
194		− break;
195		− }
196		−
197		− raw_spin_lock_irqsave(&insn_emulation_lock, flags);
198		− list_add(&insn->node, &insn_emulation);
199		− nr_insn_emulated++;
200		− raw_spin_unlock_irqrestore(&insn_emulation_lock, flags);
201		−
202		− /* Register any handlers if required */
203		− update_insn_emulation_mode(insn, INSN_UNDEF);
204		−}
205		−
206		−static int emulation_proc_handler(struct ctl_table *table, int write,
207		− void buffer, size_t lenp,
208		− loff_t *ppos)
209		−{
210		− int ret = 0;
211		− struct insn_emulation *insn = container_of(table->data, struct insn_emulation, current_mode);
212		− enum insn_emulation_mode prev_mode = insn->current_mode;
213		−
214		− mutex_lock(&insn_emulation_mutex);
215		− ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
	55	+ /*
	56	+ * sysctl for this emulation + a sentinal entry.
	57	+ */
	58	+ struct ctl_table sysctl[2];
	59	+};
216	60
217		− if (ret \|\| !write \|\| prev_mode == insn->current_mode)
218		− goto ret;
	61	+#define ARM_OPCODE_CONDTEST_FAIL 0
	62	+#define ARM_OPCODE_CONDTEST_PASS 1
	63	+#define ARM_OPCODE_CONDTEST_UNCOND 2
219	64
220		− ret = update_insn_emulation_mode(insn, prev_mode);
221		− if (ret) {
222		− /* Mode change failed, revert to previous mode. */
223		− insn->current_mode = prev_mode;
224		− update_insn_emulation_mode(insn, INSN_UNDEF);
225		− }
226		−ret:
227		− mutex_unlock(&insn_emulation_mutex);
228		− return ret;
229		−}
	65	+#define ARM_OPCODE_CONDITION_UNCOND 0xf
230	66
231		−static void __init register_insn_emulation_sysctl(void)
	67	+static unsigned int __maybe_unused aarch32_check_condition(u32 opcode, u32 psr)
232	68	{
233		− unsigned long flags;
234		− int i = 0;
235		− struct insn_emulation *insn;
236		− struct ctl_table insns_sysctl, sysctl;
237		−
238		− insns_sysctl = kcalloc(nr_insn_emulated + 1, sizeof(*sysctl),
239		− GFP_KERNEL);
240		− if (!insns_sysctl)
241		− return;
242		−
243		− raw_spin_lock_irqsave(&insn_emulation_lock, flags);
244		− list_for_each_entry(insn, &insn_emulation, node) {
245		− sysctl = &insns_sysctl[i];
246		−
247		− sysctl->mode = 0644;
248		− sysctl->maxlen = sizeof(int);
	69	+ u32 cc_bits = opcode >> 28;
249	70
250		− sysctl->procname = insn->ops->name;
251		− sysctl->data = &insn->current_mode;
252		− sysctl->extra1 = &insn->min;
253		− sysctl->extra2 = &insn->max;
254		− sysctl->proc_handler = emulation_proc_handler;
255		− i++;
	71	+ if (cc_bits != ARM_OPCODE_CONDITION_UNCOND) {
	72	+ if ((*aarch32_opcode_cond_checks[cc_bits])(psr))
	73	+ return ARM_OPCODE_CONDTEST_PASS;
	74	+ else
	75	+ return ARM_OPCODE_CONDTEST_FAIL;
256	76	}
257		− raw_spin_unlock_irqrestore(&insn_emulation_lock, flags);
258		−
259		− register_sysctl("abi", insns_sysctl);
	77	+ return ARM_OPCODE_CONDTEST_UNCOND;
260	78	}
261	79
	80	+#ifdef CONFIG_SWP_EMULATION
262	81	/*
263	82	* Implement emulation of the SWP/SWPB instructions using load-exclusive and
264	83	* store-exclusive.
@@ -339,25 +158,6 @@ static int emulate_swpX(unsigned int address, unsigned int *data, · 74 unmodified lines
339	158	return res;
340	159	}
341	160
342		−#define ARM_OPCODE_CONDTEST_FAIL 0
343		−#define ARM_OPCODE_CONDTEST_PASS 1
344		−#define ARM_OPCODE_CONDTEST_UNCOND 2
345		−
346		−#define ARM_OPCODE_CONDITION_UNCOND 0xf
347		−
348		−static unsigned int __kprobes aarch32_check_condition(u32 opcode, u32 psr)
349		−{
350		− u32 cc_bits = opcode >> 28;
351		−
352		− if (cc_bits != ARM_OPCODE_CONDITION_UNCOND) {
353		− if ((*aarch32_opcode_cond_checks[cc_bits])(psr))
354		− return ARM_OPCODE_CONDTEST_PASS;
355		− else
356		− return ARM_OPCODE_CONDTEST_FAIL;
357		− }
358		− return ARM_OPCODE_CONDTEST_UNCOND;
359		−}
360		−
361	161	/*
362	162	* swp_handler logs the id of calling process, dissects the instruction, sanity
363	163	* checks the memory location, calls emulate_swpX for the actual operation and
@@ -430,28 +230,27 @@ fault: · 66 unmodified lines
430	230	return 0;
431	231	}
432	232
433		−/*
434		− * Only emulate SWP/SWPB executed in ARM state/User mode.
435		− * The kernel must be SWP free and SWP{B} does not exist in Thumb.
436		− */
437		−static struct undef_hook swp_hooks[] = {
438		− {
439		− .instr_mask = 0x0fb00ff0,
440		− .instr_val = 0x01000090,
441		− .pstate_mask = PSR_AA32_MODE_MASK,
442		− .pstate_val = PSR_AA32_MODE_USR,
443		− .fn = swp_handler
444		− },
445		− { }
446		−};
	233	+static bool try_emulate_swp(struct pt_regs *regs, u32 insn)
	234	+{
	235	+ /* SWP{B} only exists in ARM state and does not exist in Thumb */
	236	+ if (!compat_user_mode(regs) \|\| compat_thumb_mode(regs))
	237	+ return false;
447	238
448		−static struct insn_emulation_ops swp_ops = {
	239	+ if ((insn & 0x0fb00ff0) != 0x01000090)
	240	+ return false;
	241	+
	242	+ return swp_handler(regs, insn) == 0;
	243	+}
	244	+
	245	+static struct insn_emulation insn_swp = {
449	246	.name = "swp",
450	247	.status = INSN_OBSOLETE,
451		− .hooks = swp_hooks,
	248	+ .try_emulate = try_emulate_swp,
452	249	.set_hw_mode = NULL,
453	250	};
	251	+#endif /* CONFIG_SWP_EMULATION */
454	252
	253	+#ifdef CONFIG_CP15_BARRIER_EMULATION
455	254	static int cp15barrier_handler(struct pt_regs *regs, u32 instr)
456	255	{
457	256	perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, regs->pc);
@@ -514,31 +313,29 @@ static int cp15_barrier_set_hw_mode(bool enable) · 56 unmodified lines
514	313	return 0;
515	314	}
516	315
517		−static struct undef_hook cp15_barrier_hooks[] = {
518		− {
519		− .instr_mask = 0x0fff0fdf,
520		− .instr_val = 0x0e070f9a,
521		− .pstate_mask = PSR_AA32_MODE_MASK,
522		− .pstate_val = PSR_AA32_MODE_USR,
523		− .fn = cp15barrier_handler,
524		− },
525		− {
526		− .instr_mask = 0x0fff0fff,
527		− .instr_val = 0x0e070f95,
528		− .pstate_mask = PSR_AA32_MODE_MASK,
529		− .pstate_val = PSR_AA32_MODE_USR,
530		− .fn = cp15barrier_handler,
531		− },
532		− { }
533		−};
	316	+static bool try_emulate_cp15_barrier(struct pt_regs *regs, u32 insn)
	317	+{
	318	+ if (!compat_user_mode(regs) \|\| compat_thumb_mode(regs))
	319	+ return false;
	320	+
	321	+ if ((insn & 0x0fff0fdf) == 0x0e070f9a)
	322	+ return cp15barrier_handler(regs, insn) == 0;
	323	+
	324	+ if ((insn & 0x0fff0fff) == 0x0e070f95)
	325	+ return cp15barrier_handler(regs, insn) == 0;
534	326
535		−static struct insn_emulation_ops cp15_barrier_ops = {
	327	+ return false;
	328	+}
	329	+
	330	+static struct insn_emulation insn_cp15_barrier = {
536	331	.name = "cp15_barrier",
537	332	.status = INSN_DEPRECATED,
538		− .hooks = cp15_barrier_hooks,
	333	+ .try_emulate = try_emulate_cp15_barrier,
539	334	.set_hw_mode = cp15_barrier_set_hw_mode,
540	335	};
	336	+#endif /* CONFIG_CP15_BARRIER_EMULATION */
541	337
	338	+#ifdef CONFIG_SETEND_EMULATION
542	339	static int setend_set_hw_mode(bool enable)
543	340	{
544	341	if (!cpu_supports_mixed_endian_el0())
@@ -586,31 +383,218 @@ static int t16_setend_handler(struct pt_regs *regs, u32 instr) · 41 unmodified lines
586	383	return rc;
587	384	}
588	385
589		−static struct undef_hook setend_hooks[] = {
590		− {
591		− .instr_mask = 0xfffffdff,
592		− .instr_val = 0xf1010000,
593		− .pstate_mask = PSR_AA32_MODE_MASK,
594		− .pstate_val = PSR_AA32_MODE_USR,
595		− .fn = a32_setend_handler,
596		− },
597		− {
598		− /* Thumb mode */
599		− .instr_mask = 0xfffffff7,
600		− .instr_val = 0x0000b650,
601		− .pstate_mask = (PSR_AA32_T_BIT \| PSR_AA32_MODE_MASK),
602		− .pstate_val = (PSR_AA32_T_BIT \| PSR_AA32_MODE_USR),
603		− .fn = t16_setend_handler,
604		− },
605		− {}
606		−};
	386	+static bool try_emulate_setend(struct pt_regs *regs, u32 insn)
	387	+{
	388	+ if (compat_thumb_mode(regs) &&
	389	+ (insn & 0xfffffff7) == 0x0000b650)
	390	+ return t16_setend_handler(regs, insn) == 0;
	391	+
	392	+ if (compat_user_mode(regs) &&
	393	+ (insn & 0xfffffdff) == 0xf1010000)
	394	+ return a32_setend_handler(regs, insn) == 0;
607	395
608		−static struct insn_emulation_ops setend_ops = {
	396	+ return false;
	397	+}
	398	+
	399	+static struct insn_emulation insn_setend = {
609	400	.name = "setend",
610	401	.status = INSN_DEPRECATED,
611		− .hooks = setend_hooks,
	402	+ .try_emulate = try_emulate_setend,
612	403	.set_hw_mode = setend_set_hw_mode,
613	404	};
	405	+#endif /* CONFIG_SETEND_EMULATION */
	406	+
	407	+static struct insn_emulation *insn_emulations[] = {
	408	+#ifdef CONFIG_SWP_EMULATION
	409	+ &insn_swp,
	410	+#endif
	411	+#ifdef CONFIG_CP15_BARRIER_EMULATION
	412	+ &insn_cp15_barrier,
	413	+#endif
	414	+#ifdef CONFIG_SETEND_EMULATION
	415	+ &insn_setend,
	416	+#endif
	417	+};
	418	+
	419	+static DEFINE_MUTEX(insn_emulation_mutex);
	420	+
	421	+static void enable_insn_hw_mode(void *data)
	422	+{
	423	+ struct insn_emulation insn = (struct insn_emulation )data;
	424	+ if (insn->set_hw_mode)
	425	+ insn->set_hw_mode(true);
	426	+}
	427	+
	428	+static void disable_insn_hw_mode(void *data)
	429	+{
	430	+ struct insn_emulation insn = (struct insn_emulation )data;
	431	+ if (insn->set_hw_mode)
	432	+ insn->set_hw_mode(false);
	433	+}
	434	+
	435	+/* Run set_hw_mode(mode) on all active CPUs */
	436	+static int run_all_cpu_set_hw_mode(struct insn_emulation *insn, bool enable)
	437	+{
	438	+ if (!insn->set_hw_mode)
	439	+ return -EINVAL;
	440	+ if (enable)
	441	+ on_each_cpu(enable_insn_hw_mode, (void *)insn, true);
	442	+ else
	443	+ on_each_cpu(disable_insn_hw_mode, (void *)insn, true);
	444	+ return 0;
	445	+}
	446	+
	447	+/*
	448	+ * Run set_hw_mode for all insns on a starting CPU.
	449	+ * Returns:
	450	+ * 0 - If all the hooks ran successfully.
	451	+ * -EINVAL - At least one hook is not supported by the CPU.
	452	+ */
	453	+static int run_all_insn_set_hw_mode(unsigned int cpu)
	454	+{
	455	+ int rc = 0;
	456	+ unsigned long flags;
	457	+
	458	+ /*
	459	+ * Disable IRQs to serialize against an IPI from
	460	+ * run_all_cpu_set_hw_mode(), ensuring the HW is programmed to the most
	461	+ * recent enablement state if the two race with one another.
	462	+ */
	463	+ local_irq_save(flags);
	464	+ for (int i = 0; i < ARRAY_SIZE(insn_emulations); i++) {
	465	+ struct insn_emulation *insn = insn_emulations[i];
	466	+ bool enable = READ_ONCE(insn->current_mode) == INSN_HW;
	467	+ if (insn->set_hw_mode && insn->set_hw_mode(enable)) {
	468	+ pr_warn("CPU[%u] cannot support the emulation of %s",
	469	+ cpu, insn->name);
	470	+ rc = -EINVAL;
	471	+ }
	472	+ }
	473	+ local_irq_restore(flags);
	474	+
	475	+ return rc;
	476	+}
	477	+
	478	+static int update_insn_emulation_mode(struct insn_emulation *insn,
	479	+ enum insn_emulation_mode prev)
	480	+{
	481	+ int ret = 0;
	482	+
	483	+ switch (prev) {
	484	+ case INSN_UNDEF: /* Nothing to be done */
	485	+ break;
	486	+ case INSN_EMULATE:
	487	+ break;
	488	+ case INSN_HW:
	489	+ if (!run_all_cpu_set_hw_mode(insn, false))
	490	+ pr_notice("Disabled %s support\n", insn->name);
	491	+ break;
	492	+ }
	493	+
	494	+ switch (insn->current_mode) {
	495	+ case INSN_UNDEF:
	496	+ break;
	497	+ case INSN_EMULATE:
	498	+ break;
	499	+ case INSN_HW:
	500	+ ret = run_all_cpu_set_hw_mode(insn, true);
	501	+ if (!ret)
	502	+ pr_notice("Enabled %s support\n", insn->name);
	503	+ break;
	504	+ }
	505	+
	506	+ return ret;
	507	+}
	508	+
	509	+static int emulation_proc_handler(struct ctl_table *table, int write,
	510	+ void buffer, size_t lenp,
	511	+ loff_t *ppos)
	512	+{
	513	+ int ret = 0;
	514	+ struct insn_emulation *insn = container_of(table->data, struct insn_emulation, current_mode);
	515	+ enum insn_emulation_mode prev_mode = insn->current_mode;
	516	+
	517	+ mutex_lock(&insn_emulation_mutex);
	518	+ ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
	519	+
	520	+ if (ret \|\| !write \|\| prev_mode == insn->current_mode)
	521	+ goto ret;
	522	+
	523	+ ret = update_insn_emulation_mode(insn, prev_mode);
	524	+ if (ret) {
	525	+ /* Mode change failed, revert to previous mode. */
	526	+ WRITE_ONCE(insn->current_mode, prev_mode);
	527	+ update_insn_emulation_mode(insn, INSN_UNDEF);
	528	+ }
	529	+ret:
	530	+ mutex_unlock(&insn_emulation_mutex);
	531	+ return ret;
	532	+}
	533	+
	534	+static void __init register_insn_emulation(struct insn_emulation *insn)
	535	+{
	536	+ struct ctl_table *sysctl;
	537	+
	538	+ insn->min = INSN_UNDEF;
	539	+
	540	+ switch (insn->status) {
	541	+ case INSN_DEPRECATED:
	542	+ insn->current_mode = INSN_EMULATE;
	543	+ /* Disable the HW mode if it was turned on at early boot time */
	544	+ run_all_cpu_set_hw_mode(insn, false);
	545	+ insn->max = INSN_HW;
	546	+ break;
	547	+ case INSN_OBSOLETE:
	548	+ insn->current_mode = INSN_UNDEF;
	549	+ insn->max = INSN_EMULATE;
	550	+ break;
	551	+ case INSN_UNAVAILABLE:
	552	+ insn->current_mode = INSN_UNDEF;
	553	+ insn->max = INSN_UNDEF;
	554	+ break;
	555	+ }
	556	+
	557	+ /* Program the HW if required */
	558	+ update_insn_emulation_mode(insn, INSN_UNDEF);
	559	+
	560	+ if (insn->status != INSN_UNAVAILABLE) {
	561	+ sysctl = &insn->sysctl[0];
	562	+
	563	+ sysctl->mode = 0644;
	564	+ sysctl->maxlen = sizeof(int);
	565	+
	566	+ sysctl->procname = insn->name;
	567	+ sysctl->data = &insn->current_mode;
	568	+ sysctl->extra1 = &insn->min;
	569	+ sysctl->extra2 = &insn->max;
	570	+ sysctl->proc_handler = emulation_proc_handler;
	571	+
	572	+ register_sysctl("abi", sysctl);
	573	+ }
	574	+}
	575	+
	576	+bool try_emulate_armv8_deprecated(struct pt_regs *regs, u32 insn)
	577	+{
	578	+ for (int i = 0; i < ARRAY_SIZE(insn_emulations); i++) {
	579	+ struct insn_emulation *ie = insn_emulations[i];
	580	+
	581	+ if (ie->status == INSN_UNAVAILABLE)
	582	+ continue;
	583	+
	584	+ /*
	585	+ * A trap may race with the mode being changed
	586	+ * INSN_EMULATE<->INSN_HW. Try to emulate the instruction to
	587	+ * avoid a spurious UNDEF.
	588	+ */
	589	+ if (READ_ONCE(ie->current_mode) == INSN_UNDEF)
	590	+ continue;
	591	+
	592	+ if (ie->try_emulate(regs, insn))
	593	+ return true;
	594	+ }
	595	+
	596	+ return false;
	597	+}
614	598
615	599	/*
616	600	* Invoked as core_initcall, which guarantees that the instruction
@@ -618,24 +602,25 @@ static struct insn_emulation_ops setend_ops = { · 1 unmodified line
618	602	*/
619	603	static int __init armv8_deprecated_init(void)
620	604	{
621		− if (IS_ENABLED(CONFIG_SWP_EMULATION))
622		− register_insn_emulation(&swp_ops);
	605	+#ifdef CONFIG_SETEND_EMULATION
	606	+ if (!system_supports_mixed_endian_el0()) {
	607	+ insn_setend.status = INSN_UNAVAILABLE;
	608	+ pr_info("setend instruction emulation is not supported on this system\n");
	609	+ }
623	610
624		− if (IS_ENABLED(CONFIG_CP15_BARRIER_EMULATION))
625		− register_insn_emulation(&cp15_barrier_ops);
	611	+#endif
	612	+ for (int i = 0; i < ARRAY_SIZE(insn_emulations); i++) {
	613	+ struct insn_emulation *ie = insn_emulations[i];
626	614
627		− if (IS_ENABLED(CONFIG_SETEND_EMULATION)) {
628		− if (system_supports_mixed_endian_el0())
629		− register_insn_emulation(&setend_ops);
630		− else
631		− pr_info("setend instruction emulation is not supported on this system\n");
	615	+ if (ie->status == INSN_UNAVAILABLE)
	616	+ continue;
	617	+
	618	+ register_insn_emulation(ie);
632	619	}
633	620
634	621	cpuhp_setup_state_nocalls(CPUHP_AP_ARM64_ISNDEP_STARTING,
635	622	"arm64/isndep:starting",
636	623	run_all_insn_set_hw_mode, NULL);
637		− register_insn_emulation_sysctl();
638		−
639	624	return 0;
640	625	}
641	626

arch/arm64/kernel/asm-offsets.c+13−0

@@ -82,6 +82,19 @@ int main(void) · 81 unmodified lines
82	82	DEFINE(S_STACKFRAME, offsetof(struct pt_regs, stackframe));
83	83	DEFINE(PT_REGS_SIZE, sizeof(struct pt_regs));
84	84	BLANK();
	85	+#ifdef CONFIG_DYNAMIC_FTRACE_WITH_ARGS
	86	+ DEFINE(FREGS_X0, offsetof(struct ftrace_regs, regs[0]));
	87	+ DEFINE(FREGS_X2, offsetof(struct ftrace_regs, regs[2]));
	88	+ DEFINE(FREGS_X4, offsetof(struct ftrace_regs, regs[4]));
	89	+ DEFINE(FREGS_X6, offsetof(struct ftrace_regs, regs[6]));
	90	+ DEFINE(FREGS_X8, offsetof(struct ftrace_regs, regs[8]));
	91	+ DEFINE(FREGS_FP, offsetof(struct ftrace_regs, fp));
	92	+ DEFINE(FREGS_LR, offsetof(struct ftrace_regs, lr));
	93	+ DEFINE(FREGS_SP, offsetof(struct ftrace_regs, sp));
	94	+ DEFINE(FREGS_PC, offsetof(struct ftrace_regs, pc));
	95	+ DEFINE(FREGS_SIZE, sizeof(struct ftrace_regs));
	96	+ BLANK();
	97	+#endif
85	98	#ifdef CONFIG_COMPAT
86	99	DEFINE(COMPAT_SIGFRAME_REGS_OFFSET, offsetof(struct compat_sigframe, uc.uc_mcontext.arm_r0));
87	100	DEFINE(COMPAT_RT_SIGFRAME_REGS_OFFSET, offsetof(struct compat_rt_sigframe, sig.uc.uc_mcontext.arm_r0));

arch/arm64/kernel/cpu_errata.c+7−0

@@ -661,6 +661,13 @@ const struct arm64_cpu_capabilities arm64_errata[] = { · 660 unmodified lines
661	661	CAP_MIDR_RANGE_LIST(trbe_write_out_of_range_cpus),
662	662	},
663	663	#endif
	664	+#ifdef CONFIG_ARM64_ERRATUM_2645198
	665	+ {
	666	+ .desc = "ARM erratum 2645198",
	667	+ .capability = ARM64_WORKAROUND_2645198,
	668	+ ERRATA_MIDR_ALL_VERSIONS(MIDR_CORTEX_A715)
	669	+ },
	670	+#endif
664	671	#ifdef CONFIG_ARM64_ERRATUM_2077057
665	672	{
666	673	.desc = "ARM erratum 2077057",

arch/arm64/kernel/cpufeature.c+131−122

@@ -212,6 +212,8 @@ static const struct arm64_ftr_bits ftr_id_aa64isar1[] = { · 211 unmodified lines
212	212	};
213	213
214	214	static const struct arm64_ftr_bits ftr_id_aa64isar2[] = {
	215	+ ARM64_FTR_BITS(FTR_VISIBLE, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64ISAR2_EL1_CSSC_SHIFT, 4, 0),
	216	+ ARM64_FTR_BITS(FTR_VISIBLE, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64ISAR2_EL1_RPRFM_SHIFT, 4, 0),
215	217	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_HIGHER_SAFE, ID_AA64ISAR2_EL1_BC_SHIFT, 4, 0),
216	218	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_PTR_AUTH),
217	219	FTR_STRICT, FTR_EXACT, ID_AA64ISAR2_EL1_APA3_SHIFT, 4, 0),
@@ -402,14 +404,14 @@ struct arm64_ftr_reg arm64_ftr_reg_ctrel0 = { · 184 unmodified lines
402	404	};
403	405
404	406	static const struct arm64_ftr_bits ftr_id_mmfr0[] = {
405		− S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR0_INNERSHR_SHIFT, 4, 0xf),
406		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR0_FCSE_SHIFT, 4, 0),
407		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_MMFR0_AUXREG_SHIFT, 4, 0),
408		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR0_TCM_SHIFT, 4, 0),
409		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR0_SHARELVL_SHIFT, 4, 0),
410		− S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR0_OUTERSHR_SHIFT, 4, 0xf),
411		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR0_PMSA_SHIFT, 4, 0),
412		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR0_VMSA_SHIFT, 4, 0),
	407	+ S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR0_EL1_InnerShr_SHIFT, 4, 0xf),
	408	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR0_EL1_FCSE_SHIFT, 4, 0),
	409	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_MMFR0_EL1_AuxReg_SHIFT, 4, 0),
	410	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR0_EL1_TCM_SHIFT, 4, 0),
	411	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR0_EL1_ShareLvl_SHIFT, 4, 0),
	412	+ S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR0_EL1_OuterShr_SHIFT, 4, 0xf),
	413	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR0_EL1_PMSA_SHIFT, 4, 0),
	414	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR0_EL1_VMSA_SHIFT, 4, 0),
413	415	ARM64_FTR_END,
414	416	};
415	417
@@ -429,32 +431,32 @@ static const struct arm64_ftr_bits ftr_id_aa64dfr0[] = { · 13 unmodified lines
429	431	};
430	432
431	433	static const struct arm64_ftr_bits ftr_mvfr0[] = {
432		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_FPROUND_SHIFT, 4, 0),
433		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_FPSHVEC_SHIFT, 4, 0),
434		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_FPSQRT_SHIFT, 4, 0),
435		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_FPDIVIDE_SHIFT, 4, 0),
436		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_FPTRAP_SHIFT, 4, 0),
437		− ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_FPDP_SHIFT, 4, 0),
438		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_FPSP_SHIFT, 4, 0),
439		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_SIMD_SHIFT, 4, 0),
	434	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_EL1_FPRound_SHIFT, 4, 0),
	435	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_EL1_FPShVec_SHIFT, 4, 0),
	436	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_EL1_FPSqrt_SHIFT, 4, 0),
	437	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_EL1_FPDivide_SHIFT, 4, 0),
	438	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_EL1_FPTrap_SHIFT, 4, 0),
	439	+ ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_EL1_FPDP_SHIFT, 4, 0),
	440	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_EL1_FPSP_SHIFT, 4, 0),
	441	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_EL1_SIMDReg_SHIFT, 4, 0),
440	442	ARM64_FTR_END,
441	443	};
442	444
443	445	static const struct arm64_ftr_bits ftr_mvfr1[] = {
444		− ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_SIMDFMAC_SHIFT, 4, 0),
445		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_FPHP_SHIFT, 4, 0),
446		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_SIMDHP_SHIFT, 4, 0),
447		− ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_SIMDSP_SHIFT, 4, 0),
448		− ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_SIMDINT_SHIFT, 4, 0),
449		− ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_SIMDLS_SHIFT, 4, 0),
450		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_FPDNAN_SHIFT, 4, 0),
451		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_FPFTZ_SHIFT, 4, 0),
	446	+ ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_EL1_SIMDFMAC_SHIFT, 4, 0),
	447	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_EL1_FPHP_SHIFT, 4, 0),
	448	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_EL1_SIMDHP_SHIFT, 4, 0),
	449	+ ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_EL1_SIMDSP_SHIFT, 4, 0),
	450	+ ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_EL1_SIMDInt_SHIFT, 4, 0),
	451	+ ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_EL1_SIMDLS_SHIFT, 4, 0),
	452	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_EL1_FPDNaN_SHIFT, 4, 0),
	453	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_EL1_FPFtZ_SHIFT, 4, 0),
452	454	ARM64_FTR_END,
453	455	};
454	456
455	457	static const struct arm64_ftr_bits ftr_mvfr2[] = {
456		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR2_FPMISC_SHIFT, 4, 0),
457		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR2_SIMDMISC_SHIFT, 4, 0),
	458	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR2_EL1_FPMisc_SHIFT, 4, 0),
	459	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR2_EL1_SIMDMisc_SHIFT, 4, 0),
458	460	ARM64_FTR_END,
459	461	};
460	462
@@ -470,34 +472,34 @@ static const struct arm64_ftr_bits ftr_gmid[] = { · 9 unmodified lines
470	472	};
471	473
472	474	static const struct arm64_ftr_bits ftr_id_isar0[] = {
473		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_DIVIDE_SHIFT, 4, 0),
474		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_DEBUG_SHIFT, 4, 0),
475		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_COPROC_SHIFT, 4, 0),
476		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_CMPBRANCH_SHIFT, 4, 0),
477		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_BITFIELD_SHIFT, 4, 0),
478		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_BITCOUNT_SHIFT, 4, 0),
479		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_SWAP_SHIFT, 4, 0),
	475	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_EL1_Divide_SHIFT, 4, 0),
	476	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_EL1_Debug_SHIFT, 4, 0),
	477	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_EL1_Coproc_SHIFT, 4, 0),
	478	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_EL1_CmpBranch_SHIFT, 4, 0),
	479	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_EL1_BitField_SHIFT, 4, 0),
	480	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_EL1_BitCount_SHIFT, 4, 0),
	481	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_EL1_Swap_SHIFT, 4, 0),
480	482	ARM64_FTR_END,
481	483	};
482	484
483	485	static const struct arm64_ftr_bits ftr_id_isar5[] = {
484		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_RDM_SHIFT, 4, 0),
485		− ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_CRC32_SHIFT, 4, 0),
486		− ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_SHA2_SHIFT, 4, 0),
487		− ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_SHA1_SHIFT, 4, 0),
488		− ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_AES_SHIFT, 4, 0),
489		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_SEVL_SHIFT, 4, 0),
	486	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_EL1_RDM_SHIFT, 4, 0),
	487	+ ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_EL1_CRC32_SHIFT, 4, 0),
	488	+ ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_EL1_SHA2_SHIFT, 4, 0),
	489	+ ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_EL1_SHA1_SHIFT, 4, 0),
	490	+ ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_EL1_AES_SHIFT, 4, 0),
	491	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_EL1_SEVL_SHIFT, 4, 0),
490	492	ARM64_FTR_END,
491	493	};
492	494
493	495	static const struct arm64_ftr_bits ftr_id_mmfr4[] = {
494		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR4_EVT_SHIFT, 4, 0),
495		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR4_CCIDX_SHIFT, 4, 0),
496		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR4_LSM_SHIFT, 4, 0),
497		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR4_HPDS_SHIFT, 4, 0),
498		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR4_CNP_SHIFT, 4, 0),
499		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR4_XNX_SHIFT, 4, 0),
500		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR4_AC2_SHIFT, 4, 0),
	496	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR4_EL1_EVT_SHIFT, 4, 0),
	497	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR4_EL1_CCIDX_SHIFT, 4, 0),
	498	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR4_EL1_LSM_SHIFT, 4, 0),
	499	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR4_EL1_HPDS_SHIFT, 4, 0),
	500	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR4_EL1_CnP_SHIFT, 4, 0),
	501	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR4_EL1_XNX_SHIFT, 4, 0),
	502	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR4_EL1_AC2_SHIFT, 4, 0),
501	503
502	504	/*
503	505	* SpecSEI = 1 indicates that the PE might generate an SError on an
@@ -505,80 +507,80 @@ static const struct arm64_ftr_bits ftr_id_mmfr4[] = { · 1 unmodified line
505	507	* SError might be generated than it will not be. Hence it has been
506	508	* classified as FTR_HIGHER_SAFE.
507	509	*/
508		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_HIGHER_SAFE, ID_MMFR4_SPECSEI_SHIFT, 4, 0),
	510	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_HIGHER_SAFE, ID_MMFR4_EL1_SpecSEI_SHIFT, 4, 0),
509	511	ARM64_FTR_END,
510	512	};
511	513
512	514	static const struct arm64_ftr_bits ftr_id_isar4[] = {
513		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_SWP_FRAC_SHIFT, 4, 0),
514		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_PSR_M_SHIFT, 4, 0),
515		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_SYNCH_PRIM_FRAC_SHIFT, 4, 0),
516		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_BARRIER_SHIFT, 4, 0),
517		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_SMC_SHIFT, 4, 0),
518		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_WRITEBACK_SHIFT, 4, 0),
519		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_WITHSHIFTS_SHIFT, 4, 0),
520		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_UNPRIV_SHIFT, 4, 0),
	515	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_EL1_SWP_frac_SHIFT, 4, 0),
	516	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_EL1_PSR_M_SHIFT, 4, 0),
	517	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_EL1_SynchPrim_frac_SHIFT, 4, 0),
	518	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_EL1_Barrier_SHIFT, 4, 0),
	519	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_EL1_SMC_SHIFT, 4, 0),
	520	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_EL1_Writeback_SHIFT, 4, 0),
	521	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_EL1_WithShifts_SHIFT, 4, 0),
	522	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_EL1_Unpriv_SHIFT, 4, 0),
521	523	ARM64_FTR_END,
522	524	};
523	525
524	526	static const struct arm64_ftr_bits ftr_id_mmfr5[] = {
525		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR5_ETS_SHIFT, 4, 0),
	527	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR5_EL1_ETS_SHIFT, 4, 0),
526	528	ARM64_FTR_END,
527	529	};
528	530
529	531	static const struct arm64_ftr_bits ftr_id_isar6[] = {
530		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR6_I8MM_SHIFT, 4, 0),
531		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR6_BF16_SHIFT, 4, 0),
532		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR6_SPECRES_SHIFT, 4, 0),
533		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR6_SB_SHIFT, 4, 0),
534		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR6_FHM_SHIFT, 4, 0),
535		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR6_DP_SHIFT, 4, 0),
536		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR6_JSCVT_SHIFT, 4, 0),
	532	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR6_EL1_I8MM_SHIFT, 4, 0),
	533	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR6_EL1_BF16_SHIFT, 4, 0),
	534	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR6_EL1_SPECRES_SHIFT, 4, 0),
	535	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR6_EL1_SB_SHIFT, 4, 0),
	536	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR6_EL1_FHM_SHIFT, 4, 0),
	537	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR6_EL1_DP_SHIFT, 4, 0),
	538	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR6_EL1_JSCVT_SHIFT, 4, 0),
537	539	ARM64_FTR_END,
538	540	};
539	541
540	542	static const struct arm64_ftr_bits ftr_id_pfr0[] = {
541		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR0_DIT_SHIFT, 4, 0),
542		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_PFR0_CSV2_SHIFT, 4, 0),
543		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR0_STATE3_SHIFT, 4, 0),
544		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR0_STATE2_SHIFT, 4, 0),
545		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR0_STATE1_SHIFT, 4, 0),
546		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR0_STATE0_SHIFT, 4, 0),
	543	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR0_EL1_DIT_SHIFT, 4, 0),
	544	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_PFR0_EL1_CSV2_SHIFT, 4, 0),
	545	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR0_EL1_State3_SHIFT, 4, 0),
	546	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR0_EL1_State2_SHIFT, 4, 0),
	547	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR0_EL1_State1_SHIFT, 4, 0),
	548	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR0_EL1_State0_SHIFT, 4, 0),
547	549	ARM64_FTR_END,
548	550	};
549	551
550	552	static const struct arm64_ftr_bits ftr_id_pfr1[] = {
551		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_GIC_SHIFT, 4, 0),
552		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_VIRT_FRAC_SHIFT, 4, 0),
553		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_SEC_FRAC_SHIFT, 4, 0),
554		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_GENTIMER_SHIFT, 4, 0),
555		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_VIRTUALIZATION_SHIFT, 4, 0),
556		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_MPROGMOD_SHIFT, 4, 0),
557		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_SECURITY_SHIFT, 4, 0),
558		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_PROGMOD_SHIFT, 4, 0),
	553	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_EL1_GIC_SHIFT, 4, 0),
	554	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_EL1_Virt_frac_SHIFT, 4, 0),
	555	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_EL1_Sec_frac_SHIFT, 4, 0),
	556	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_EL1_GenTimer_SHIFT, 4, 0),
	557	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_EL1_Virtualization_SHIFT, 4, 0),
	558	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_EL1_MProgMod_SHIFT, 4, 0),
	559	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_EL1_Security_SHIFT, 4, 0),
	560	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_EL1_ProgMod_SHIFT, 4, 0),
559	561	ARM64_FTR_END,
560	562	};
561	563
562	564	static const struct arm64_ftr_bits ftr_id_pfr2[] = {
563		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_PFR2_SSBS_SHIFT, 4, 0),
564		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_PFR2_CSV3_SHIFT, 4, 0),
	565	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_PFR2_EL1_SSBS_SHIFT, 4, 0),
	566	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_PFR2_EL1_CSV3_SHIFT, 4, 0),
565	567	ARM64_FTR_END,
566	568	};
567	569
568	570	static const struct arm64_ftr_bits ftr_id_dfr0[] = {
569	571	/* [31:28] TraceFilt */
570		− S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_EXACT, ID_DFR0_PERFMON_SHIFT, 4, 0),
571		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR0_MPROFDBG_SHIFT, 4, 0),
572		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR0_MMAPTRC_SHIFT, 4, 0),
573		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR0_COPTRC_SHIFT, 4, 0),
574		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR0_MMAPDBG_SHIFT, 4, 0),
575		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR0_COPSDBG_SHIFT, 4, 0),
576		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR0_COPDBG_SHIFT, 4, 0),
	572	+ S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_EXACT, ID_DFR0_EL1_PerfMon_SHIFT, 4, 0),
	573	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR0_EL1_MProfDbg_SHIFT, 4, 0),
	574	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR0_EL1_MMapTrc_SHIFT, 4, 0),
	575	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR0_EL1_CopTrc_SHIFT, 4, 0),
	576	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR0_EL1_MMapDbg_SHIFT, 4, 0),
	577	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR0_EL1_CopSDbg_SHIFT, 4, 0),
	578	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR0_EL1_CopDbg_SHIFT, 4, 0),
577	579	ARM64_FTR_END,
578	580	};
579	581
580	582	static const struct arm64_ftr_bits ftr_id_dfr1[] = {
581		− S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR1_MTPMU_SHIFT, 4, 0),
	583	+ S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR1_EL1_MTPMU_SHIFT, 4, 0),
582	584	ARM64_FTR_END,
583	585	};
584	586
@@ -1119,12 +1121,12 @@ static int update_32bit_cpu_features(int cpu, struct cpuinfo_32bit *info, · 534 unmodified lines
1119	1121	* EL1-dependent register fields to avoid spurious sanity check fails.
1120	1122	*/
1121	1123	if (!id_aa64pfr0_32bit_el1(pfr0)) {
1122		− relax_cpu_ftr_reg(SYS_ID_ISAR4_EL1, ID_ISAR4_SMC_SHIFT);
1123		− relax_cpu_ftr_reg(SYS_ID_PFR1_EL1, ID_PFR1_VIRT_FRAC_SHIFT);
1124		− relax_cpu_ftr_reg(SYS_ID_PFR1_EL1, ID_PFR1_SEC_FRAC_SHIFT);
1125		− relax_cpu_ftr_reg(SYS_ID_PFR1_EL1, ID_PFR1_VIRTUALIZATION_SHIFT);
1126		− relax_cpu_ftr_reg(SYS_ID_PFR1_EL1, ID_PFR1_SECURITY_SHIFT);
1127		− relax_cpu_ftr_reg(SYS_ID_PFR1_EL1, ID_PFR1_PROGMOD_SHIFT);
	1124	+ relax_cpu_ftr_reg(SYS_ID_ISAR4_EL1, ID_ISAR4_EL1_SMC_SHIFT);
	1125	+ relax_cpu_ftr_reg(SYS_ID_PFR1_EL1, ID_PFR1_EL1_Virt_frac_SHIFT);
	1126	+ relax_cpu_ftr_reg(SYS_ID_PFR1_EL1, ID_PFR1_EL1_Sec_frac_SHIFT);
	1127	+ relax_cpu_ftr_reg(SYS_ID_PFR1_EL1, ID_PFR1_EL1_Virtualization_SHIFT);
	1128	+ relax_cpu_ftr_reg(SYS_ID_PFR1_EL1, ID_PFR1_EL1_Security_SHIFT);
	1129	+ relax_cpu_ftr_reg(SYS_ID_PFR1_EL1, ID_PFR1_EL1_ProgMod_SHIFT);
1128	1130	}
1129	1131
1130	1132	taint \|= check_update_ftr_reg(SYS_ID_DFR0_EL1, cpu,
@@ -2101,6 +2103,11 @@ static void cpu_trap_el0_impdef(const struct arm64_cpu_capabilities *__unused) · 970 unmodified lines
2101	2103	sysreg_clear_set(sctlr_el1, 0, SCTLR_EL1_TIDCP);
2102	2104	}
2103	2105
	2106	+static void cpu_enable_dit(const struct arm64_cpu_capabilities *__unused)
	2107	+{
	2108	+ set_pstate_dit(1);
	2109	+}
	2110	+
2104	2111	/* Internal helper functions to match cpu capability type */
2105	2112	static bool
2106	2113	cpucap_late_cpu_optional(const struct arm64_cpu_capabilities *cap)
@@ -2664,6 +2671,18 @@ static const struct arm64_cpu_capabilities arm64_features[] = { · 557 unmodified lines
2664	2671	.matches = has_cpuid_feature,
2665	2672	.cpu_enable = cpu_trap_el0_impdef,
2666	2673	},
	2674	+ {
	2675	+ .desc = "Data independent timing control (DIT)",
	2676	+ .capability = ARM64_HAS_DIT,
	2677	+ .type = ARM64_CPUCAP_SYSTEM_FEATURE,
	2678	+ .sys_reg = SYS_ID_AA64PFR0_EL1,
	2679	+ .sign = FTR_UNSIGNED,
	2680	+ .field_pos = ID_AA64PFR0_EL1_DIT_SHIFT,
	2681	+ .field_width = 4,
	2682	+ .min_field_value = ID_AA64PFR0_EL1_DIT_IMP,
	2683	+ .matches = has_cpuid_feature,
	2684	+ .cpu_enable = cpu_enable_dit,
	2685	+ },
2667	2686	{},
2668	2687	};
2669	2688
@@ -2772,6 +2791,7 @@ static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = { · 102 unmodified lines
2772	2791	HWCAP_CAP(SYS_ID_AA64MMFR2_EL1, ID_AA64MMFR2_EL1_AT_SHIFT, 4, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_USCAT),
2773	2792	#ifdef CONFIG_ARM64_SVE
2774	2793	HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_EL1_SVE_SHIFT, 4, FTR_UNSIGNED, ID_AA64PFR0_EL1_SVE_IMP, CAP_HWCAP, KERNEL_HWCAP_SVE),
	2794	+ HWCAP_CAP(SYS_ID_AA64ZFR0_EL1, ID_AA64ZFR0_EL1_SVEver_SHIFT, 4, FTR_UNSIGNED, ID_AA64ZFR0_EL1_SVEver_SVE2p1, CAP_HWCAP, KERNEL_HWCAP_SVE2P1),
2775	2795	HWCAP_CAP(SYS_ID_AA64ZFR0_EL1, ID_AA64ZFR0_EL1_SVEver_SHIFT, 4, FTR_UNSIGNED, ID_AA64ZFR0_EL1_SVEver_SVE2, CAP_HWCAP, KERNEL_HWCAP_SVE2),
2776	2796	HWCAP_CAP(SYS_ID_AA64ZFR0_EL1, ID_AA64ZFR0_EL1_AES_SHIFT, 4, FTR_UNSIGNED, ID_AA64ZFR0_EL1_AES_IMP, CAP_HWCAP, KERNEL_HWCAP_SVEAES),
2777	2797	HWCAP_CAP(SYS_ID_AA64ZFR0_EL1, ID_AA64ZFR0_EL1_AES_SHIFT, 4, FTR_UNSIGNED, ID_AA64ZFR0_EL1_AES_PMULL128, CAP_HWCAP, KERNEL_HWCAP_SVEPMULL),
@@ -2798,6 +2818,8 @@ static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = { · 20 unmodified lines
2798	2818	#endif /* CONFIG_ARM64_MTE */
2799	2819	HWCAP_CAP(SYS_ID_AA64MMFR0_EL1, ID_AA64MMFR0_EL1_ECV_SHIFT, 4, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_ECV),
2800	2820	HWCAP_CAP(SYS_ID_AA64MMFR1_EL1, ID_AA64MMFR1_EL1_AFP_SHIFT, 4, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_AFP),
	2821	+ HWCAP_CAP(SYS_ID_AA64ISAR2_EL1, ID_AA64ISAR2_EL1_CSSC_SHIFT, 4, FTR_UNSIGNED, ID_AA64ISAR2_EL1_CSSC_IMP, CAP_HWCAP, KERNEL_HWCAP_CSSC),
	2822	+ HWCAP_CAP(SYS_ID_AA64ISAR2_EL1, ID_AA64ISAR2_EL1_RPRFM_SHIFT, 4, FTR_UNSIGNED, ID_AA64ISAR2_EL1_RPRFM_IMP, CAP_HWCAP, KERNEL_HWCAP_RPRFM),
2801	2823	HWCAP_CAP(SYS_ID_AA64ISAR2_EL1, ID_AA64ISAR2_EL1_RPRES_SHIFT, 4, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_RPRES),
2802	2824	HWCAP_CAP(SYS_ID_AA64ISAR2_EL1, ID_AA64ISAR2_EL1_WFxT_SHIFT, 4, FTR_UNSIGNED, ID_AA64ISAR2_EL1_WFxT_IMP, CAP_HWCAP, KERNEL_HWCAP_WFXT),
2803	2825	#ifdef CONFIG_ARM64_SME
@@ -2829,24 +2851,24 @@ static bool compat_has_neon(const struct arm64_cpu_capabilities *cap, int scope) · 25 unmodified lines
2829	2851	else
2830	2852	mvfr1 = read_sysreg_s(SYS_MVFR1_EL1);
2831	2853
2832		− return cpuid_feature_extract_unsigned_field(mvfr1, MVFR1_SIMDSP_SHIFT) &&
2833		− cpuid_feature_extract_unsigned_field(mvfr1, MVFR1_SIMDINT_SHIFT) &&
2834		− cpuid_feature_extract_unsigned_field(mvfr1, MVFR1_SIMDLS_SHIFT);
	2854	+ return cpuid_feature_extract_unsigned_field(mvfr1, MVFR1_EL1_SIMDSP_SHIFT) &&
	2855	+ cpuid_feature_extract_unsigned_field(mvfr1, MVFR1_EL1_SIMDInt_SHIFT) &&
	2856	+ cpuid_feature_extract_unsigned_field(mvfr1, MVFR1_EL1_SIMDLS_SHIFT);
2835	2857	}
2836	2858	#endif
2837	2859
2838	2860	static const struct arm64_cpu_capabilities compat_elf_hwcaps[] = {
2839	2861	#ifdef CONFIG_COMPAT
2840	2862	HWCAP_CAP_MATCH(compat_has_neon, CAP_COMPAT_HWCAP, COMPAT_HWCAP_NEON),
2841		− HWCAP_CAP(SYS_MVFR1_EL1, MVFR1_SIMDFMAC_SHIFT, 4, FTR_UNSIGNED, 1, CAP_COMPAT_HWCAP, COMPAT_HWCAP_VFPv4),
	2863	+ HWCAP_CAP(SYS_MVFR1_EL1, MVFR1_EL1_SIMDFMAC_SHIFT, 4, FTR_UNSIGNED, 1, CAP_COMPAT_HWCAP, COMPAT_HWCAP_VFPv4),
2842	2864	/* Arm v8 mandates MVFR0.FPDP == {0, 2}. So, piggy back on this for the presence of VFP support */
2843		− HWCAP_CAP(SYS_MVFR0_EL1, MVFR0_FPDP_SHIFT, 4, FTR_UNSIGNED, 2, CAP_COMPAT_HWCAP, COMPAT_HWCAP_VFP),
2844		− HWCAP_CAP(SYS_MVFR0_EL1, MVFR0_FPDP_SHIFT, 4, FTR_UNSIGNED, 2, CAP_COMPAT_HWCAP, COMPAT_HWCAP_VFPv3),
2845		− HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_AES_SHIFT, 4, FTR_UNSIGNED, 2, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_PMULL),
2846		− HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_AES_SHIFT, 4, FTR_UNSIGNED, 1, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_AES),
2847		− HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_SHA1_SHIFT, 4, FTR_UNSIGNED, 1, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_SHA1),
2848		− HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_SHA2_SHIFT, 4, FTR_UNSIGNED, 1, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_SHA2),
2849		− HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_CRC32_SHIFT, 4, FTR_UNSIGNED, 1, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_CRC32),
	2865	+ HWCAP_CAP(SYS_MVFR0_EL1, MVFR0_EL1_FPDP_SHIFT, 4, FTR_UNSIGNED, 2, CAP_COMPAT_HWCAP, COMPAT_HWCAP_VFP),
	2866	+ HWCAP_CAP(SYS_MVFR0_EL1, MVFR0_EL1_FPDP_SHIFT, 4, FTR_UNSIGNED, 2, CAP_COMPAT_HWCAP, COMPAT_HWCAP_VFPv3),
	2867	+ HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_EL1_AES_SHIFT, 4, FTR_UNSIGNED, 2, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_PMULL),
	2868	+ HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_EL1_AES_SHIFT, 4, FTR_UNSIGNED, 1, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_AES),
	2869	+ HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_EL1_SHA1_SHIFT, 4, FTR_UNSIGNED, 1, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_SHA1),
	2870	+ HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_EL1_SHA2_SHIFT, 4, FTR_UNSIGNED, 1, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_SHA2),
	2871	+ HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_EL1_CRC32_SHIFT, 4, FTR_UNSIGNED, 1, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_CRC32),
2850	2872	#endif
2851	2873	{},
2852	2874	};
@@ -3435,35 +3457,22 @@ int do_emulate_mrs(struct pt_regs *regs, u32 sys_reg, u32 rt) · 582 unmodified lines
3435	3457	return rc;
3436	3458	}
3437	3459
3438		−static int emulate_mrs(struct pt_regs *regs, u32 insn)
	3460	+bool try_emulate_mrs(struct pt_regs *regs, u32 insn)
3439	3461	{
3440	3462	u32 sys_reg, rt;
3441	3463
	3464	+ if (compat_user_mode(regs) \|\| !aarch64_insn_is_mrs(insn))
	3465	+ return false;
	3466	+
3442	3467	/*
3443	3468	* sys_reg values are defined as used in mrs/msr instruction.
3444	3469	* shift the imm value to get the encoding.
3445	3470	*/
3446	3471	sys_reg = (u32)aarch64_insn_decode_immediate(AARCH64_INSN_IMM_16, insn) << 5;
3447	3472	rt = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RT, insn);
3448		− return do_emulate_mrs(regs, sys_reg, rt);
	3473	+ return do_emulate_mrs(regs, sys_reg, rt) == 0;
3449	3474	}
3450	3475
3451		−static struct undef_hook mrs_hook = {
3452		− .instr_mask = 0xffff0000,
3453		− .instr_val = 0xd5380000,
3454		− .pstate_mask = PSR_AA32_MODE_MASK,
3455		− .pstate_val = PSR_MODE_EL0t,
3456		− .fn = emulate_mrs,
3457		−};
3458		−
3459		−static int __init enable_mrs_emulation(void)
3460		−{
3461		− register_undef_hook(&mrs_hook);
3462		− return 0;
3463		−}
3464		−
3465		−core_initcall(enable_mrs_emulation);
3466		−
3467	3476	enum mitigation_state arm64_get_meltdown_state(void)
3468	3477	{
3469	3478	if (__meltdown_safe)

arch/arm64/kernel/cpuinfo.c+3−0

@@ -116,6 +116,9 @@ static const char *const hwcap_str[] = { · 115 unmodified lines
116	116	[KERNEL_HWCAP_WFXT] = "wfxt",
117	117	[KERNEL_HWCAP_EBF16] = "ebf16",
118	118	[KERNEL_HWCAP_SVE_EBF16] = "sveebf16",
	119	+ [KERNEL_HWCAP_CSSC] = "cssc",
	120	+ [KERNEL_HWCAP_RPRFM] = "rprfm",
	121	+ [KERNEL_HWCAP_SVE2P1] = "sve2p1",
119	122	};
120	123
121	124	#ifdef CONFIG_COMPAT

arch/arm64/kernel/entry-common.c+12−12

@@ -30,7 +30,7 @@ · 29 unmodified lines
30	30	/*
31	31	* Handle IRQ/context state management when entering from kernel mode.
32	32	* Before this function is called it is not safe to call regular kernel code,
33		− * intrumentable code, or any code which may trigger an exception.
	33	+ * instrumentable code, or any code which may trigger an exception.
34	34	*
35	35	* This is intended to match the logic in irqentry_enter(), handling the kernel
36	36	* mode transitions only.
@@ -63,7 +63,7 @@ static void noinstr enter_from_kernel_mode(struct pt_regs *regs) · 26 unmodified lines
63	63	/*
64	64	* Handle IRQ/context state management when exiting to kernel mode.
65	65	* After this function returns it is not safe to call regular kernel code,
66		− * intrumentable code, or any code which may trigger an exception.
	66	+ * instrumentable code, or any code which may trigger an exception.
67	67	*
68	68	* This is intended to match the logic in irqentry_exit(), handling the kernel
69	69	* mode transitions only, and with preemption handled elsewhere.
@@ -97,7 +97,7 @@ static void noinstr exit_to_kernel_mode(struct pt_regs *regs) · 27 unmodified lines
97	97	/*
98	98	* Handle IRQ/context state management when entering from user mode.
99	99	* Before this function is called it is not safe to call regular kernel code,
100		− * intrumentable code, or any code which may trigger an exception.
	100	+ * instrumentable code, or any code which may trigger an exception.
101	101	*/
102	102	static __always_inline void __enter_from_user_mode(void)
103	103	{
@@ -116,7 +116,7 @@ static __always_inline void enter_from_user_mode(struct pt_regs *regs) · 12 unmodified lines
116	116	/*
117	117	* Handle IRQ/context state management when exiting to user mode.
118	118	* After this function returns it is not safe to call regular kernel code,
119		− * intrumentable code, or any code which may trigger an exception.
	119	+ * instrumentable code, or any code which may trigger an exception.
120	120	*/
121	121	static __always_inline void __exit_to_user_mode(void)
122	122	{
@@ -152,7 +152,7 @@ asmlinkage void noinstr asm_exit_to_user_mode(struct pt_regs *regs) · 29 unmodified lines
152	152	/*
153	153	* Handle IRQ/context state management when entering an NMI from user/kernel
154	154	* mode. Before this function is called it is not safe to call regular kernel
155		− * code, intrumentable code, or any code which may trigger an exception.
	155	+ * code, instrumentable code, or any code which may trigger an exception.
156	156	*/
157	157	static void noinstr arm64_enter_nmi(struct pt_regs *regs)
158	158	{
@@ -170,7 +170,7 @@ static void noinstr arm64_enter_nmi(struct pt_regs *regs) · 11 unmodified lines
170	170	/*
171	171	* Handle IRQ/context state management when exiting an NMI from user/kernel
172	172	* mode. After this function returns it is not safe to call regular kernel
173		− * code, intrumentable code, or any code which may trigger an exception.
	173	+ * code, instrumentable code, or any code which may trigger an exception.
174	174	*/
175	175	static void noinstr arm64_exit_nmi(struct pt_regs *regs)
176	176	{
@@ -192,7 +192,7 @@ static void noinstr arm64_exit_nmi(struct pt_regs *regs) · 15 unmodified lines
192	192	/*
193	193	* Handle IRQ/context state management when entering a debug exception from
194	194	* kernel mode. Before this function is called it is not safe to call regular
195		− * kernel code, intrumentable code, or any code which may trigger an exception.
	195	+ * kernel code, instrumentable code, or any code which may trigger an exception.
196	196	*/
197	197	static void noinstr arm64_enter_el1_dbg(struct pt_regs *regs)
198	198	{
@@ -207,7 +207,7 @@ static void noinstr arm64_enter_el1_dbg(struct pt_regs *regs) · 8 unmodified lines
207	207	/*
208	208	* Handle IRQ/context state management when exiting a debug exception from
209	209	* kernel mode. After this function returns it is not safe to call regular
210		− * kernel code, intrumentable code, or any code which may trigger an exception.
	210	+ * kernel code, instrumentable code, or any code which may trigger an exception.
211	211	*/
212	212	static void noinstr arm64_exit_el1_dbg(struct pt_regs *regs)
213	213	{
@@ -384,7 +384,7 @@ static void noinstr el1_undef(struct pt_regs *regs, unsigned long esr) · 170 unmodified lines
384	384	{
385	385	enter_from_kernel_mode(regs);
386	386	local_daif_inherit(regs);
387		− do_undefinstr(regs, esr);
	387	+ do_el1_undef(regs, esr);
388	388	local_daif_mask();
389	389	exit_to_kernel_mode(regs);
390	390	}
@@ -570,7 +570,7 @@ static void noinstr el0_sys(struct pt_regs *regs, unsigned long esr) · 179 unmodified lines
570	570	{
571	571	enter_from_user_mode(regs);
572	572	local_daif_restore(DAIF_PROCCTX);
573		− do_sysinstr(esr, regs);
	573	+ do_el0_sys(esr, regs);
574	574	exit_to_user_mode(regs);
575	575	}
576	576
@@ -599,7 +599,7 @@ static void noinstr el0_undef(struct pt_regs *regs, unsigned long esr) · 22 unmodified lines
599	599	{
600	600	enter_from_user_mode(regs);
601	601	local_daif_restore(DAIF_PROCCTX);
602		− do_undefinstr(regs, esr);
	602	+ do_el0_undef(regs, esr);
603	603	exit_to_user_mode(regs);
604	604	}
605	605
@@ -762,7 +762,7 @@ static void noinstr el0_cp15(struct pt_regs *regs, unsigned long esr) · 156 unmodified lines
762	762	{
763	763	enter_from_user_mode(regs);
764	764	local_daif_restore(DAIF_PROCCTX);
765		− do_cp15instr(esr, regs);
	765	+ do_el0_cp15(esr, regs);
766	766	exit_to_user_mode(regs);
767	767	}
768	768

arch/arm64/kernel/entry-ftrace.S+46−110

@@ -13,83 +13,58 @@ · 12 unmodified lines
13	13	#include <asm/ftrace.h>
14	14	#include <asm/insn.h>
15	15
16		−#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
	16	+#ifdef CONFIG_DYNAMIC_FTRACE_WITH_ARGS
17	17	/*
18	18	* Due to -fpatchable-function-entry=2, the compiler has placed two NOPs before
19	19	* the regular function prologue. For an enabled callsite, ftrace_init_nop() and
20	20	* ftrace_make_call() have patched those NOPs to:
21	21	*
22	22	* MOV X9, LR
23		− * BL <entry>
24		− *
25		− * ... where <entry> is either ftrace_caller or ftrace_regs_caller.
	23	+ * BL ftrace_caller
26	24	*
27	25	* Each instrumented function follows the AAPCS, so here x0-x8 and x18-x30 are
28	26	* live (x18 holds the Shadow Call Stack pointer), and x9-x17 are safe to
29	27	* clobber.
30	28	*
31		− * We save the callsite's context into a pt_regs before invoking any ftrace
32		− * callbacks. So that we can get a sensible backtrace, we create a stack record
33		− * for the callsite and the ftrace entry assembly. This is not sufficient for
34		− * reliable stacktrace: until we create the callsite stack record, its caller
35		− * is missing from the LR and existing chain of frame records.
	29	+ * We save the callsite's context into a struct ftrace_regs before invoking any
	30	+ * ftrace callbacks. So that we can get a sensible backtrace, we create frame
	31	+ * records for the callsite and the ftrace entry assembly. This is not
	32	+ * sufficient for reliable stacktrace: until we create the callsite stack
	33	+ * record, its caller is missing from the LR and existing chain of frame
	34	+ * records.
36	35	*/
37		− .macro ftrace_regs_entry, allregs=0
38		− /* Make room for pt_regs, plus a callee frame */
39		− sub sp, sp, #(PT_REGS_SIZE + 16)
40		−
41		− /* Save function arguments (and x9 for simplicity) */
42		− stp x0, x1, [sp, #S_X0]
43		− stp x2, x3, [sp, #S_X2]
44		− stp x4, x5, [sp, #S_X4]
45		− stp x6, x7, [sp, #S_X6]
46		− stp x8, x9, [sp, #S_X8]
47		−
48		− /* Optionally save the callee-saved registers, always save the FP */
49		− .if \allregs == 1
50		− stp x10, x11, [sp, #S_X10]
51		− stp x12, x13, [sp, #S_X12]
52		− stp x14, x15, [sp, #S_X14]
53		− stp x16, x17, [sp, #S_X16]
54		− stp x18, x19, [sp, #S_X18]
55		− stp x20, x21, [sp, #S_X20]
56		− stp x22, x23, [sp, #S_X22]
57		− stp x24, x25, [sp, #S_X24]
58		− stp x26, x27, [sp, #S_X26]
59		− stp x28, x29, [sp, #S_X28]
60		− .else
61		− str x29, [sp, #S_FP]
62		− .endif
63		−
64		− /* Save the callsite's SP and LR */
65		− add x10, sp, #(PT_REGS_SIZE + 16)
66		− stp x9, x10, [sp, #S_LR]
	36	+SYM_CODE_START(ftrace_caller)
	37	+ bti c
67	38
68		− /* Save the PC after the ftrace callsite */
69		− str x30, [sp, #S_PC]
	39	+ /* Save original SP */
	40	+ mov x10, sp
70	41
71		− /* Create a frame record for the callsite above pt_regs */
72		− stp x29, x9, [sp, #PT_REGS_SIZE]
73		− add x29, sp, #PT_REGS_SIZE
	42	+ /* Make room for ftrace regs, plus two frame records */
	43	+ sub sp, sp, #(FREGS_SIZE + 32)
74	44
75		− /* Create our frame record within pt_regs. */
76		− stp x29, x30, [sp, #S_STACKFRAME]
77		− add x29, sp, #S_STACKFRAME
78		− .endm
	45	+ /* Save function arguments */
	46	+ stp x0, x1, [sp, #FREGS_X0]
	47	+ stp x2, x3, [sp, #FREGS_X2]
	48	+ stp x4, x5, [sp, #FREGS_X4]
	49	+ stp x6, x7, [sp, #FREGS_X6]
	50	+ str x8, [sp, #FREGS_X8]
79	51
80		−SYM_CODE_START(ftrace_regs_caller)
81		− bti c
82		− ftrace_regs_entry 1
83		− b ftrace_common
84		−SYM_CODE_END(ftrace_regs_caller)
	52	+ /* Save the callsite's FP, LR, SP */
	53	+ str x29, [sp, #FREGS_FP]
	54	+ str x9, [sp, #FREGS_LR]
	55	+ str x10, [sp, #FREGS_SP]
85	56
86		−SYM_CODE_START(ftrace_caller)
87		− bti c
88		− ftrace_regs_entry 0
89		− b ftrace_common
90		−SYM_CODE_END(ftrace_caller)
	57	+ /* Save the PC after the ftrace callsite */
	58	+ str x30, [sp, #FREGS_PC]
	59	+
	60	+ /* Create a frame record for the callsite above the ftrace regs */
	61	+ stp x29, x9, [sp, #FREGS_SIZE + 16]
	62	+ add x29, sp, #FREGS_SIZE + 16
	63	+
	64	+ /* Create our frame record above the ftrace regs */
	65	+ stp x29, x30, [sp, #FREGS_SIZE]
	66	+ add x29, sp, #FREGS_SIZE
91	67
92		−SYM_CODE_START(ftrace_common)
93	68	sub x0, x30, #AARCH64_INSN_SIZE // ip (callsite's BL insn)
94	69	mov x1, x9 // parent_ip (callsite's LR)
95	70	ldr_l x2, function_trace_op // op
@@ -104,24 +79,24 @@ SYM_INNER_LABEL(ftrace_call, SYM_L_GLOBAL) · 8 unmodified lines
104	79	* to restore x0-x8, x29, and x30.
105	80	*/
106	81	/* Restore function arguments */
107		− ldp x0, x1, [sp]
108		− ldp x2, x3, [sp, #S_X2]
109		− ldp x4, x5, [sp, #S_X4]
110		− ldp x6, x7, [sp, #S_X6]
111		− ldr x8, [sp, #S_X8]
	82	+ ldp x0, x1, [sp, #FREGS_X0]
	83	+ ldp x2, x3, [sp, #FREGS_X2]
	84	+ ldp x4, x5, [sp, #FREGS_X4]
	85	+ ldp x6, x7, [sp, #FREGS_X6]
	86	+ ldr x8, [sp, #FREGS_X8]
112	87
113	88	/* Restore the callsite's FP, LR, PC */
114		− ldr x29, [sp, #S_FP]
115		− ldr x30, [sp, #S_LR]
116		− ldr x9, [sp, #S_PC]
	89	+ ldr x29, [sp, #FREGS_FP]
	90	+ ldr x30, [sp, #FREGS_LR]
	91	+ ldr x9, [sp, #FREGS_PC]
117	92
118	93	/* Restore the callsite's SP */
119		− add sp, sp, #PT_REGS_SIZE + 16
	94	+ add sp, sp, #FREGS_SIZE + 32
120	95
121	96	ret x9
122		−SYM_CODE_END(ftrace_common)
	97	+SYM_CODE_END(ftrace_caller)
123	98
124		−#else /* CONFIG_DYNAMIC_FTRACE_WITH_REGS */
	99	+#else /* CONFIG_DYNAMIC_FTRACE_WITH_ARGS */
125	100
126	101	/*
127	102	* Gcc with -pg will put the following code in the beginning of each function:
@@ -195,44 +170,6 @@ SYM_CODE_END(ftrace_common) · 67 unmodified lines
195	170	add \reg, \reg, #8
196	171	.endm
197	172
198		−#ifndef CONFIG_DYNAMIC_FTRACE
199		−/*
200		− * void _mcount(unsigned long return_address)
201		− * @return_address: return address to instrumented function
202		− *
203		− * This function makes calls, if enabled, to:
204		− * - tracer function to probe instrumented function's entry,
205		− * - ftrace_graph_caller to set up an exit hook
206		− */
207		−SYM_FUNC_START(_mcount)
208		− mcount_enter
209		−
210		− ldr_l x2, ftrace_trace_function
211		− adr x0, ftrace_stub
212		− cmp x0, x2 // if (ftrace_trace_function
213		− b.eq skip_ftrace_call // != ftrace_stub) {
214		−
215		− mcount_get_pc x0 // function's pc
216		− mcount_get_lr x1 // function's lr (= parent's pc)
217		− blr x2 // (*ftrace_trace_function)(pc, lr);
218		−
219		−skip_ftrace_call: // }
220		−#ifdef CONFIG_FUNCTION_GRAPH_TRACER
221		− ldr_l x2, ftrace_graph_return
222		− cmp x0, x2 // if ((ftrace_graph_return
223		− b.ne ftrace_graph_caller // != ftrace_stub)
224		−
225		− ldr_l x2, ftrace_graph_entry // \|\| (ftrace_graph_entry
226		− adr_l x0, ftrace_graph_entry_stub // != ftrace_graph_entry_stub))
227		− cmp x0, x2
228		− b.ne ftrace_graph_caller // ftrace_graph_caller();
229		−#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
230		− mcount_exit
231		−SYM_FUNC_END(_mcount)
232		−EXPORT_SYMBOL(_mcount)
233		−NOKPROBE(_mcount)
234		−
235		−#else /* CONFIG_DYNAMIC_FTRACE */
236	173	/*
237	174	* _mcount() is used to build the kernel with -pg option, but all the branch
238	175	* instructions to _mcount() are replaced to NOP initially at kernel start up,
@@ -272,7 +209,6 @@ SYM_INNER_LABEL(ftrace_graph_call, SYM_L_GLOBAL) // ftrace_graph_caller(); · 33 unmodified lines
272	209
273	210	mcount_exit
274	211	SYM_FUNC_END(ftrace_caller)
275		−#endif /* CONFIG_DYNAMIC_FTRACE */
276	212
277	213	#ifdef CONFIG_FUNCTION_GRAPH_TRACER
278	214	/*
@@ -293,7 +229,7 @@ SYM_FUNC_START(ftrace_graph_caller) · 14 unmodified lines
293	229	mcount_exit
294	230	SYM_FUNC_END(ftrace_graph_caller)
295	231	#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
296		−#endif /* CONFIG_DYNAMIC_FTRACE_WITH_REGS */
	232	+#endif /* CONFIG_DYNAMIC_FTRACE_WITH_ARGS */
297	233
298	234	SYM_TYPED_FUNC_START(ftrace_stub)
299	235	ret

arch/arm64/kernel/entry.S+3−0

@@ -197,6 +197,9 @@ alternative_cb_end · 196 unmodified lines
197	197	.endm
198	198
199	199	.macro kernel_entry, el, regsize = 64
	200	+ .if \el == 0
	201	+ alternative_insn nop, SET_PSTATE_DIT(1), ARM64_HAS_DIT
	202	+ .endif
200	203	.if \regsize == 32
201	204	mov w0, w0 // zero upper 32 bits of x0
202	205	.endif

arch/arm64/kernel/fpsimd.c+117−52

@@ -118,16 +118,8 @@ · 117 unmodified lines
118	118	* returned from the 2nd syscall yet, TIF_FOREIGN_FPSTATE is still set so
119	119	* whatever is in the FPSIMD registers is not saved to memory, but discarded.
120	120	*/
121		−struct fpsimd_last_state_struct {
122		− struct user_fpsimd_state *st;
123		− void *sve_state;
124		− void *za_state;
125		− u64 *svcr;
126		− unsigned int sve_vl;
127		− unsigned int sme_vl;
128		−};
129	121
130		−static DEFINE_PER_CPU(struct fpsimd_last_state_struct, fpsimd_last_state);
	122	+static DEFINE_PER_CPU(struct cpu_fp_state, fpsimd_last_state);
131	123
132	124	__ro_after_init struct vl_info vl_info[ARM64_VEC_MAX] = {
133	125	#ifdef CONFIG_ARM64_SVE
@@ -330,15 +322,6 @@ void task_set_vl_onexec(struct task_struct *task, enum vec_type type, · 196 unmodified lines
330	322	* The task can execute SVE instructions while in userspace without
331	323	* trapping to the kernel.
332	324	*
333		− * When stored, Z0-Z31 (incorporating Vn in bits[127:0] or the
334		− * corresponding Zn), P0-P15 and FFR are encoded in
335		− * task->thread.sve_state, formatted appropriately for vector
336		− * length task->thread.sve_vl or, if SVCR.SM is set,
337		− * task->thread.sme_vl.
338		− *
339		− * task->thread.sve_state must point to a valid buffer at least
340		− * sve_state_size(task) bytes in size.
341		− *
342	325	* During any syscall, the kernel may optionally clear TIF_SVE and
343	326	* discard the vector state except for the FPSIMD subset.
344	327	*
@@ -348,7 +331,15 @@ void task_set_vl_onexec(struct task_struct *task, enum vec_type type, · 3 unmodified lines
348	331	* do_sve_acc() to be called, which does some preparation and then
349	332	* sets TIF_SVE.
350	333	*
351		− * When stored, FPSIMD registers V0-V31 are encoded in
	334	+ * During any syscall, the kernel may optionally clear TIF_SVE and
	335	+ * discard the vector state except for the FPSIMD subset.
	336	+ *
	337	+ * The data will be stored in one of two formats:
	338	+ *
	339	+ * * FPSIMD only - FP_STATE_FPSIMD:
	340	+ *
	341	+ * When the FPSIMD only state stored task->thread.fp_type is set to
	342	+ * FP_STATE_FPSIMD, the FPSIMD registers V0-V31 are encoded in
352	343	* task->thread.uw.fpsimd_state; bits [max : 128] for each of Z0-Z31 are
353	344	* logically zero but not stored anywhere; P0-P15 and FFR are not
354	345	* stored and have unspecified values from userspace's point of
@@ -356,7 +347,23 @@ void task_set_vl_onexec(struct task_struct *task, enum vec_type type, · 1 unmodified line
356	347	* but userspace is discouraged from relying on this.
357	348	*
358	349	* task->thread.sve_state does not need to be non-NULL, valid or any
359		− * particular size: it must not be dereferenced.
	350	+ * particular size: it must not be dereferenced and any data stored
	351	+ * there should be considered stale and not referenced.
	352	+ *
	353	+ * * SVE state - FP_STATE_SVE:
	354	+ *
	355	+ * When the full SVE state is stored task->thread.fp_type is set to
	356	+ * FP_STATE_SVE and Z0-Z31 (incorporating Vn in bits[127:0] or the
	357	+ * corresponding Zn), P0-P15 and FFR are encoded in in
	358	+ * task->thread.sve_state, formatted appropriately for vector
	359	+ * length task->thread.sve_vl or, if SVCR.SM is set,
	360	+ * task->thread.sme_vl. The storage for the vector registers in
	361	+ * task->thread.uw.fpsimd_state should be ignored.
	362	+ *
	363	+ * task->thread.sve_state must point to a valid buffer at least
	364	+ * sve_state_size(task) bytes in size. The data stored in
	365	+ * task->thread.uw.fpsimd_state.vregs should be considered stale
	366	+ * and not referenced.
360	367	*
361	368	* * FPSR and FPCR are always stored in task->thread.uw.fpsimd_state
362	369	* irrespective of whether TIF_SVE is clear or set, since these are
@@ -378,11 +385,37 @@ static void task_fpsimd_load(void) · 15 unmodified lines
378	385	WARN_ON(!system_supports_fpsimd());
379	386	WARN_ON(!have_cpu_fpsimd_context());
380	387
381		− /* Check if we should restore SVE first */
382		− if (IS_ENABLED(CONFIG_ARM64_SVE) && test_thread_flag(TIF_SVE)) {
383		− sve_set_vq(sve_vq_from_vl(task_get_sve_vl(current)) - 1);
384		− restore_sve_regs = true;
385		− restore_ffr = true;
	388	+ if (system_supports_sve()) {
	389	+ switch (current->thread.fp_type) {
	390	+ case FP_STATE_FPSIMD:
	391	+ /* Stop tracking SVE for this task until next use. */
	392	+ if (test_and_clear_thread_flag(TIF_SVE))
	393	+ sve_user_disable();
	394	+ break;
	395	+ case FP_STATE_SVE:
	396	+ if (!thread_sm_enabled(&current->thread) &&
	397	+ !WARN_ON_ONCE(!test_and_set_thread_flag(TIF_SVE)))
	398	+ sve_user_enable();
	399	+
	400	+ if (test_thread_flag(TIF_SVE))
	401	+ sve_set_vq(sve_vq_from_vl(task_get_sve_vl(current)) - 1);
	402	+
	403	+ restore_sve_regs = true;
	404	+ restore_ffr = true;
	405	+ break;
	406	+ default:
	407	+ /*
	408	+ * This indicates either a bug in
	409	+ * fpsimd_save() or memory corruption, we
	410	+ * should always record an explicit format
	411	+ * when we save. We always at least have the
	412	+ * memory allocated for FPSMID registers so
	413	+ * try that and hope for the best.
	414	+ */
	415	+ WARN_ON_ONCE(1);
	416	+ clear_thread_flag(TIF_SVE);
	417	+ break;
	418	+ }
386	419	}
387	420
388	421	/* Restore SME, override SVE register configuration if needed */
@@ -398,18 +431,19 @@ static void task_fpsimd_load(void) · 9 unmodified lines
398	431	if (thread_za_enabled(&current->thread))
399	432	za_load_state(current->thread.za_state);
400	433
401		− if (thread_sm_enabled(&current->thread)) {
402		− restore_sve_regs = true;
	434	+ if (thread_sm_enabled(&current->thread))
403	435	restore_ffr = system_supports_fa64();
404		− }
405	436	}
406	437
407		− if (restore_sve_regs)
	438	+ if (restore_sve_regs) {
	439	+ WARN_ON_ONCE(current->thread.fp_type != FP_STATE_SVE);
408	440	sve_load_state(sve_pffr(&current->thread),
409	441	&current->thread.uw.fpsimd_state.fpsr,
410	442	restore_ffr);
411		− else
	443	+ } else {
	444	+ WARN_ON_ONCE(current->thread.fp_type != FP_STATE_FPSIMD);
412	445	fpsimd_load_state(&current->thread.uw.fpsimd_state);
	446	+ }
413	447	}
414	448
415	449	/*
@@ -419,12 +453,12 @@ static void task_fpsimd_load(void) · 3 unmodified lines
419	453	* last, if KVM is involved this may be the guest VM context rather
420	454	* than the host thread for the VM pointed to by current. This means
421	455	* that we must always reference the state storage via last rather
422		− * than via current, other than the TIF_ flags which KVM will
423		− * carefully maintain for us.
	456	+ * than via current, if we are saving KVM state then it will have
	457	+ * ensured that the type of registers to save is set in last->to_save.
424	458	*/
425	459	static void fpsimd_save(void)
426	460	{
427		− struct fpsimd_last_state_struct const *last =
	461	+ struct cpu_fp_state const *last =
428	462	this_cpu_ptr(&fpsimd_last_state);
429	463	/* set by fpsimd_bind_task_to_cpu() or fpsimd_bind_state_to_cpu() */
430	464	bool save_sve_regs = false;
@@ -437,7 +471,14 @@ static void fpsimd_save(void) · 6 unmodified lines
437	471	if (test_thread_flag(TIF_FOREIGN_FPSTATE))
438	472	return;
439	473
440		− if (test_thread_flag(TIF_SVE)) {
	474	+ /*
	475	+ * If a task is in a syscall the ABI allows us to only
	476	+ * preserve the state shared with FPSIMD so don't bother
	477	+ * saving the full SVE state in that case.
	478	+ */
	479	+ if ((last->to_save == FP_STATE_CURRENT && test_thread_flag(TIF_SVE) &&
	480	+ !in_syscall(current_pt_regs())) \|\|
	481	+ last->to_save == FP_STATE_SVE) {
441	482	save_sve_regs = true;
442	483	save_ffr = true;
443	484	vl = last->sve_vl;
@@ -474,8 +515,10 @@ static void fpsimd_save(void) · 30 unmodified lines
474	515	sve_save_state((char *)last->sve_state +
475	516	sve_ffr_offset(vl),
476	517	&last->st->fpsr, save_ffr);
	518	+ *last->fp_type = FP_STATE_SVE;
477	519	} else {
478	520	fpsimd_save_state(last->st);
	521	+ *last->fp_type = FP_STATE_FPSIMD;
479	522	}
480	523	}
481	524
@@ -768,8 +811,7 @@ void fpsimd_sync_to_sve(struct task_struct *task) · 286 unmodified lines
768	811	*/
769	812	void sve_sync_to_fpsimd(struct task_struct *task)
770	813	{
771		− if (test_tsk_thread_flag(task, TIF_SVE) \|\|
772		− thread_sm_enabled(&task->thread))
	814	+ if (task->thread.fp_type == FP_STATE_SVE)
773	815	sve_to_fpsimd(task);
774	816	}
775	817
@@ -848,8 +890,10 @@ int vec_set_vector_length(struct task_struct *task, enum vec_type type, · 72 unmodified lines
848	890
849	891	fpsimd_flush_task_state(task);
850	892	if (test_and_clear_tsk_thread_flag(task, TIF_SVE) \|\|
851		− thread_sm_enabled(&task->thread))
	893	+ thread_sm_enabled(&task->thread)) {
852	894	sve_to_fpsimd(task);
	895	+ task->thread.fp_type = FP_STATE_FPSIMD;
	896	+ }
853	897
854	898	if (system_supports_sme() && type == ARM64_VEC_SME) {
855	899	task->thread.svcr &= ~(SVCR_SM_MASK \|
@@ -1368,6 +1412,7 @@ static void sve_init_regs(void) · 512 unmodified lines
1368	1412	fpsimd_bind_task_to_cpu();
1369	1413	} else {
1370	1414	fpsimd_to_sve(current);
	1415	+ current->thread.fp_type = FP_STATE_SVE;
1371	1416	}
1372	1417	}
1373	1418
@@ -1596,6 +1641,8 @@ void fpsimd_flush_thread(void) · 222 unmodified lines
1596	1641	current->thread.svcr = 0;
1597	1642	}
1598	1643
	1644	+ current->thread.fp_type = FP_STATE_FPSIMD;
	1645	+
1599	1646	put_cpu_fpsimd_context();
1600	1647	kfree(sve_state);
1601	1648	kfree(za_state);
@@ -1627,6 +1674,31 @@ void fpsimd_signal_preserve_current_state(void) · 25 unmodified lines
1627	1674	sve_to_fpsimd(current);
1628	1675	}
1629	1676
	1677	+/*
	1678	+ * Called by KVM when entering the guest.
	1679	+ */
	1680	+void fpsimd_kvm_prepare(void)
	1681	+{
	1682	+ if (!system_supports_sve())
	1683	+ return;
	1684	+
	1685	+ /*
	1686	+ * KVM does not save host SVE state since we can only enter
	1687	+ * the guest from a syscall so the ABI means that only the
	1688	+ * non-saved SVE state needs to be saved. If we have left
	1689	+ * SVE enabled for performance reasons then update the task
	1690	+ * state to be FPSIMD only.
	1691	+ */
	1692	+ get_cpu_fpsimd_context();
	1693	+
	1694	+ if (test_and_clear_thread_flag(TIF_SVE)) {
	1695	+ sve_to_fpsimd(current);
	1696	+ current->thread.fp_type = FP_STATE_FPSIMD;
	1697	+ }
	1698	+
	1699	+ put_cpu_fpsimd_context();
	1700	+}
	1701	+
1630	1702	/*
1631	1703	* Associate current's FPSIMD context with this cpu
1632	1704	* The caller must have ownership of the cpu FPSIMD context before calling
@@ -1634,8 +1706,7 @@ void fpsimd_signal_preserve_current_state(void) · 1 unmodified line
1634	1706	*/
1635	1707	static void fpsimd_bind_task_to_cpu(void)
1636	1708	{
1637		− struct fpsimd_last_state_struct *last =
1638		− this_cpu_ptr(&fpsimd_last_state);
	1709	+ struct cpu_fp_state *last = this_cpu_ptr(&fpsimd_last_state);
1639	1710
1640	1711	WARN_ON(!system_supports_fpsimd());
1641	1712	last->st = &current->thread.uw.fpsimd_state;
@@ -1644,6 +1715,8 @@ static void fpsimd_bind_task_to_cpu(void) · 2 unmodified lines
1644	1715	last->sve_vl = task_get_sve_vl(current);
1645	1716	last->sme_vl = task_get_sme_vl(current);
1646	1717	last->svcr = &current->thread.svcr;
	1718	+ last->fp_type = &current->thread.fp_type;
	1719	+ last->to_save = FP_STATE_CURRENT;
1647	1720	current->thread.fpsimd_cpu = smp_processor_id();
1648	1721
1649	1722	/*
@@ -1665,22 +1738,14 @@ static void fpsimd_bind_task_to_cpu(void) · 15 unmodified lines
1665	1738	}
1666	1739	}
1667	1740
1668		−void fpsimd_bind_state_to_cpu(struct user_fpsimd_state st, void sve_state,
1669		− unsigned int sve_vl, void *za_state,
1670		− unsigned int sme_vl, u64 *svcr)
	1741	+void fpsimd_bind_state_to_cpu(struct cpu_fp_state *state)
1671	1742	{
1672		− struct fpsimd_last_state_struct *last =
1673		− this_cpu_ptr(&fpsimd_last_state);
	1743	+ struct cpu_fp_state *last = this_cpu_ptr(&fpsimd_last_state);
1674	1744
1675	1745	WARN_ON(!system_supports_fpsimd());
1676	1746	WARN_ON(!in_softirq() && !irqs_disabled());
1677	1747
1678		− last->st = st;
1679		− last->svcr = svcr;
1680		− last->sve_state = sve_state;
1681		− last->za_state = za_state;
1682		− last->sve_vl = sve_vl;
1683		− last->sme_vl = sme_vl;
	1748	+ last = state;
1684	1749	}
1685	1750
1686	1751	/*
@@ -1838,7 +1903,7 @@ void kernel_neon_begin(void) · 151 unmodified lines
1838	1903	/* Invalidate any task state remaining in the fpsimd regs: */
1839	1904	fpsimd_flush_cpu_state();
1840	1905	}
1841		−EXPORT_SYMBOL(kernel_neon_begin);
	1906	+EXPORT_SYMBOL_GPL(kernel_neon_begin);
1842	1907
1843	1908	/*
1844	1909	* kernel_neon_end(): give the CPU FPSIMD registers back to the current task
@@ -1856,7 +1921,7 @@ void kernel_neon_end(void) · 11 unmodified lines
1856	1921
1857	1922	put_cpu_fpsimd_context();
1858	1923	}
1859		−EXPORT_SYMBOL(kernel_neon_end);
	1924	+EXPORT_SYMBOL_GPL(kernel_neon_end);
1860	1925
1861	1926	#ifdef CONFIG_EFI
1862	1927

arch/arm64/kernel/ftrace.c+48−39

@@ -17,7 +17,49 @@ · 16 unmodified lines
17	17	#include <asm/insn.h>
18	18	#include <asm/patching.h>
19	19
20		−#ifdef CONFIG_DYNAMIC_FTRACE
	20	+#ifdef CONFIG_DYNAMIC_FTRACE_WITH_ARGS
	21	+struct fregs_offset {
	22	+ const char *name;
	23	+ int offset;
	24	+};
	25	+
	26	+#define FREGS_OFFSET(n, field) \
	27	+{ \
	28	+ .name = n, \
	29	+ .offset = offsetof(struct ftrace_regs, field), \
	30	+}
	31	+
	32	+static const struct fregs_offset fregs_offsets[] = {
	33	+ FREGS_OFFSET("x0", regs[0]),
	34	+ FREGS_OFFSET("x1", regs[1]),
	35	+ FREGS_OFFSET("x2", regs[2]),
	36	+ FREGS_OFFSET("x3", regs[3]),
	37	+ FREGS_OFFSET("x4", regs[4]),
	38	+ FREGS_OFFSET("x5", regs[5]),
	39	+ FREGS_OFFSET("x6", regs[6]),
	40	+ FREGS_OFFSET("x7", regs[7]),
	41	+ FREGS_OFFSET("x8", regs[8]),
	42	+
	43	+ FREGS_OFFSET("x29", fp),
	44	+ FREGS_OFFSET("x30", lr),
	45	+ FREGS_OFFSET("lr", lr),
	46	+
	47	+ FREGS_OFFSET("sp", sp),
	48	+ FREGS_OFFSET("pc", pc),
	49	+};
	50	+
	51	+int ftrace_regs_query_register_offset(const char *name)
	52	+{
	53	+ for (int i = 0; i < ARRAY_SIZE(fregs_offsets); i++) {
	54	+ const struct fregs_offset *roff = &fregs_offsets[i];
	55	+ if (!strcmp(roff->name, name))
	56	+ return roff->offset;
	57	+ }
	58	+
	59	+ return -EINVAL;
	60	+}
	61	+#endif
	62	+
21	63	/*
22	64	* Replace a single instruction, which may be a branch or NOP.
23	65	* If @validate == true, a replaced instruction is checked against 'old'.
@@ -70,9 +112,6 @@ static struct plt_entry get_ftrace_plt(struct module mod, unsigned long addr) · 46 unmodified lines
70	112
71	113	if (addr == FTRACE_ADDR)
72	114	return &plt[FTRACE_PLT_IDX];
73		− if (addr == FTRACE_REGS_ADDR &&
74		− IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_REGS))
75		− return &plt[FTRACE_REGS_PLT_IDX];
76	115	#endif
77	116	return NULL;
78	117	}
@@ -154,25 +193,7 @@ int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) · 75 unmodified lines
154	193	return ftrace_modify_code(pc, old, new, true);
155	194	}
156	195
157		−#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
158		−int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr,
159		− unsigned long addr)
160		−{
161		− unsigned long pc = rec->ip;
162		− u32 old, new;
163		−
164		− if (!ftrace_find_callable_addr(rec, NULL, &old_addr))
165		− return -EINVAL;
166		− if (!ftrace_find_callable_addr(rec, NULL, &addr))
167		− return -EINVAL;
168		−
169		− old = aarch64_insn_gen_branch_imm(pc, old_addr,
170		− AARCH64_INSN_BRANCH_LINK);
171		− new = aarch64_insn_gen_branch_imm(pc, addr, AARCH64_INSN_BRANCH_LINK);
172		−
173		− return ftrace_modify_code(pc, old, new, true);
174		−}
175		−
	196	+#ifdef CONFIG_DYNAMIC_FTRACE_WITH_ARGS
176	197	/*
177	198	* The compiler has inserted two NOPs before the regular function prologue.
178	199	* All instrumented functions follow the AAPCS, so x0-x8 and x19-x30 are live,
@@ -228,7 +249,7 @@ int ftrace_make_nop(struct module mod, struct dyn_ftrace rec, · 49 unmodified lines
228	249	*
229	250	* Note: 'mod' is only set at module load time.
230	251	*/
231		− if (!IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_REGS) &&
	252	+ if (!IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_ARGS) &&
232	253	IS_ENABLED(CONFIG_ARM64_MODULE_PLTS) && mod) {
233	254	return aarch64_insn_patch_text_nosync((void *)pc, new);
234	255	}
@@ -246,7 +267,6 @@ void arch_ftrace_update_code(int command) · 11 unmodified lines
246	267	command \|= FTRACE_MAY_SLEEP;
247	268	ftrace_modify_all_code(command);
248	269	}
249		−#endif /* CONFIG_DYNAMIC_FTRACE */
250	270
251	271	#ifdef CONFIG_FUNCTION_GRAPH_TRACER
252	272	/*
@@ -277,21 +297,11 @@ void prepare_ftrace_return(unsigned long self_addr, unsigned long *parent, · 24 unmodified lines
277	297	}
278	298	}
279	299
280		−#ifdef CONFIG_DYNAMIC_FTRACE
281		−
282		−#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
	300	+#ifdef CONFIG_DYNAMIC_FTRACE_WITH_ARGS
283	301	void ftrace_graph_func(unsigned long ip, unsigned long parent_ip,
284	302	struct ftrace_ops op, struct ftrace_regs fregs)
285	303	{
286		− /*
287		− * When DYNAMIC_FTRACE_WITH_REGS is selected, `fregs` can never be NULL
288		− * and arch_ftrace_get_regs(fregs) will always give a non-NULL pt_regs
289		− * in which we can safely modify the LR.
290		− */
291		− struct pt_regs *regs = arch_ftrace_get_regs(fregs);
292		− unsigned long parent = (unsigned long )&procedure_link_pointer(regs);
293		−
294		− prepare_ftrace_return(ip, parent, frame_pointer(regs));
	304	+ prepare_ftrace_return(ip, &fregs->lr, fregs->fp);
295	305	}
296	306	#else
297	307	/*
@@ -323,6 +333,5 @@ int ftrace_disable_ftrace_graph_caller(void) · 25 unmodified lines
323	333	{
324	334	return ftrace_modify_graph_caller(false);
325	335	}
326		−#endif /* CONFIG_DYNAMIC_FTRACE_WITH_REGS */
327		−#endif /* CONFIG_DYNAMIC_FTRACE */
	336	+#endif /* CONFIG_DYNAMIC_FTRACE_WITH_ARGS */
328	337	#endif /* CONFIG_FUNCTION_GRAPH_TRACER */

arch/arm64/kernel/head.S+3−0

@@ -462,6 +462,9 @@ SYM_FUNC_START_LOCAL(__primary_switched) · 461 unmodified lines
462	462	bl early_fdt_map // Try mapping the FDT early
463	463	mov x0, x20 // pass the full boot status
464	464	bl init_feature_override // Parse cpu feature overrides
	465	+#ifdef CONFIG_UNWIND_PATCH_PAC_INTO_SCS
	466	+ bl scs_patch_vmlinux
	467	+#endif
465	468	mov x0, x20
466	469	bl finalise_el2 // Prefer VHE if possible
467	470	ldp x29, x30, [sp], #16

arch/arm64/kernel/irq.c+6−5

@@ -10,20 +10,21 @@ · 9 unmodified lines
10	10	* Copyright (C) 2012 ARM Ltd.
11	11	*/
12	12
13		−#include <linux/irq.h>
14		−#include <linux/memory.h>
15		−#include <linux/smp.h>
16	13	#include <linux/hardirq.h>
17	14	#include <linux/init.h>
	15	+#include <linux/irq.h>
18	16	#include <linux/irqchip.h>
19	17	#include <linux/kprobes.h>
	18	+#include <linux/memory.h>
20	19	#include <linux/scs.h>
21	20	#include <linux/seq_file.h>
	21	+#include <linux/smp.h>
22	22	#include <linux/vmalloc.h>
23	23	#include <asm/daifflags.h>
24	24	#include <asm/exception.h>
25		−#include <asm/vmap_stack.h>
26	25	#include <asm/softirq_stack.h>
	26	+#include <asm/stacktrace.h>
	27	+#include <asm/vmap_stack.h>
27	28
28	29	/* Only access this in an NMI enter/exit */
29	30	DEFINE_PER_CPU(struct nmi_ctx, nmi_contexts);
@@ -41,7 +42,7 @@ static void init_irq_scs(void) · 11 unmodified lines
41	42	{
42	43	int cpu;
43	44
44		− if (!IS_ENABLED(CONFIG_SHADOW_CALL_STACK))
	45	+ if (!scs_is_enabled())
45	46	return;
46	47
47	48	for_each_possible_cpu(cpu)

arch/arm64/kernel/module.c+8−3

@@ -15,9 +15,11 @@ · 14 unmodified lines
15	15	#include <linux/kernel.h>
16	16	#include <linux/mm.h>
17	17	#include <linux/moduleloader.h>
	18	+#include <linux/scs.h>
18	19	#include <linux/vmalloc.h>
19	20	#include <asm/alternative.h>
20	21	#include <asm/insn.h>
	22	+#include <asm/scs.h>
21	23	#include <asm/sections.h>
22	24
23	25	void *module_alloc(unsigned long size)
@@ -497,9 +499,6 @@ static int module_init_ftrace_plt(const Elf_Ehdr *hdr, · 473 unmodified lines
497	499
498	500	__init_plt(&plts[FTRACE_PLT_IDX], FTRACE_ADDR);
499	501
500		− if (IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_REGS))
501		− __init_plt(&plts[FTRACE_REGS_PLT_IDX], FTRACE_REGS_ADDR);
502		−
503	502	mod->arch.ftrace_trampolines = plts;
504	503	#endif
505	504	return 0;
@@ -514,5 +513,11 @@ int module_finalize(const Elf_Ehdr *hdr, · 8 unmodified lines
514	513	if (s)
515	514	apply_alternatives_module((void *)s->sh_addr, s->sh_size);
516	515
	516	+ if (scs_is_dynamic()) {
	517	+ s = find_section(hdr, sechdrs, ".init.eh_frame");
	518	+ if (s)
	519	+ scs_patch((void *)s->sh_addr, s->sh_size);
	520	+ }
	521	+
517	522	return module_init_ftrace_plt(hdr, sechdrs, me);
518	523	}

arch/arm64/kernel/paravirt.c+0−4

@@ -141,10 +141,6 @@ static bool __init has_pv_steal_clock(void) · 140 unmodified lines
141	141	{
142	142	struct arm_smccc_res res;
143	143
144		− /* To detect the presence of PV time support we require SMCCC 1.1+ */
145		− if (arm_smccc_1_1_get_conduit() == SMCCC_CONDUIT_NONE)
146		− return false;
147		−
148	144	arm_smccc_1_1_invoke(ARM_SMCCC_ARCH_FEATURES_FUNC_ID,
149	145	ARM_SMCCC_HV_PV_TIME_FEATURES, &res);
150	146

arch/arm64/kernel/patch-scs.c+257−0

@@ -0,0 +1,257 @@
	1	+// SPDX-License-Identifier: GPL-2.0-only
	2	+/*
	3	+ * Copyright (C) 2022 - Google LLC
	4	+ * Author: Ard Biesheuvel <ardb@google.com>
	5	+ */
	6	+
	7	+#include <linux/bug.h>
	8	+#include <linux/errno.h>
	9	+#include <linux/init.h>
	10	+#include <linux/linkage.h>
	11	+#include <linux/printk.h>
	12	+#include <linux/types.h>
	13	+
	14	+#include <asm/cacheflush.h>
	15	+#include <asm/scs.h>
	16	+
	17	+//
	18	+// This minimal DWARF CFI parser is partially based on the code in
	19	+// arch/arc/kernel/unwind.c, and on the document below:
	20	+// https://refspecs.linuxbase.org/LSB_4.0.0/LSB-Core-generic/LSB-Core-generic/ehframechpt.html
	21	+//
	22	+
	23	+#define DW_CFA_nop 0x00
	24	+#define DW_CFA_set_loc 0x01
	25	+#define DW_CFA_advance_loc1 0x02
	26	+#define DW_CFA_advance_loc2 0x03
	27	+#define DW_CFA_advance_loc4 0x04
	28	+#define DW_CFA_offset_extended 0x05
	29	+#define DW_CFA_restore_extended 0x06
	30	+#define DW_CFA_undefined 0x07
	31	+#define DW_CFA_same_value 0x08
	32	+#define DW_CFA_register 0x09
	33	+#define DW_CFA_remember_state 0x0a
	34	+#define DW_CFA_restore_state 0x0b
	35	+#define DW_CFA_def_cfa 0x0c
	36	+#define DW_CFA_def_cfa_register 0x0d
	37	+#define DW_CFA_def_cfa_offset 0x0e
	38	+#define DW_CFA_def_cfa_expression 0x0f
	39	+#define DW_CFA_expression 0x10
	40	+#define DW_CFA_offset_extended_sf 0x11
	41	+#define DW_CFA_def_cfa_sf 0x12
	42	+#define DW_CFA_def_cfa_offset_sf 0x13
	43	+#define DW_CFA_val_offset 0x14
	44	+#define DW_CFA_val_offset_sf 0x15
	45	+#define DW_CFA_val_expression 0x16
	46	+#define DW_CFA_lo_user 0x1c
	47	+#define DW_CFA_negate_ra_state 0x2d
	48	+#define DW_CFA_GNU_args_size 0x2e
	49	+#define DW_CFA_GNU_negative_offset_extended 0x2f
	50	+#define DW_CFA_hi_user 0x3f
	51	+
	52	+extern const u8 __eh_frame_start[], __eh_frame_end[];
	53	+
	54	+enum {
	55	+ PACIASP = 0xd503233f,
	56	+ AUTIASP = 0xd50323bf,
	57	+ SCS_PUSH = 0xf800865e,
	58	+ SCS_POP = 0xf85f8e5e,
	59	+};
	60	+
	61	+static void __always_inline scs_patch_loc(u64 loc)
	62	+{
	63	+ u32 insn = le32_to_cpup((void *)loc);
	64	+
	65	+ switch (insn) {
	66	+ case PACIASP:
	67	+ (u32 )loc = cpu_to_le32(SCS_PUSH);
	68	+ break;
	69	+ case AUTIASP:
	70	+ (u32 )loc = cpu_to_le32(SCS_POP);
	71	+ break;
	72	+ default:
	73	+ /*
	74	+ * While the DW_CFA_negate_ra_state directive is guaranteed to
	75	+ * appear right after a PACIASP/AUTIASP instruction, it may
	76	+ * also appear after a DW_CFA_restore_state directive that
	77	+ * restores a state that is only partially accurate, and is
	78	+ * followed by DW_CFA_negate_ra_state directive to toggle the
	79	+ * PAC bit again. So we permit other instructions here, and ignore
	80	+ * them.
	81	+ */
	82	+ return;
	83	+ }
	84	+ dcache_clean_pou(loc, loc + sizeof(u32));
	85	+}
	86	+
	87	+/*
	88	+ * Skip one uleb128/sleb128 encoded quantity from the opcode stream. All bytes
	89	+ * except the last one have bit #7 set.
	90	+ */
	91	+static int __always_inline skip_xleb128(const u8 **opcode, int size)
	92	+{
	93	+ u8 c;
	94	+
	95	+ do {
	96	+ c = (opcode)++;
	97	+ size--;
	98	+ } while (c & BIT(7));
	99	+
	100	+ return size;
	101	+}
	102	+
	103	+struct eh_frame {
	104	+ /*
	105	+ * The size of this frame if 0 < size < U32_MAX, 0 terminates the list.
	106	+ */
	107	+ u32 size;
	108	+
	109	+ /*
	110	+ * The first frame is a Common Information Entry (CIE) frame, followed
	111	+ * by one or more Frame Description Entry (FDE) frames. In the former
	112	+ * case, this field is 0, otherwise it is the negated offset relative
	113	+ * to the associated CIE frame.
	114	+ */
	115	+ u32 cie_id_or_pointer;
	116	+
	117	+ union {
	118	+ struct { // CIE
	119	+ u8 version;
	120	+ u8 augmentation_string[];
	121	+ };
	122	+
	123	+ struct { // FDE
	124	+ s32 initial_loc;
	125	+ s32 range;
	126	+ u8 opcodes[];
	127	+ };
	128	+ };
	129	+};
	130	+
	131	+static int noinstr scs_handle_fde_frame(const struct eh_frame *frame,
	132	+ bool fde_has_augmentation_data,
	133	+ int code_alignment_factor)
	134	+{
	135	+ int size = frame->size - offsetof(struct eh_frame, opcodes) + 4;
	136	+ u64 loc = (u64)offset_to_ptr(&frame->initial_loc);
	137	+ const u8 *opcode = frame->opcodes;
	138	+
	139	+ if (fde_has_augmentation_data) {
	140	+ int l;
	141	+
	142	+ // assume single byte uleb128_t
	143	+ if (WARN_ON(*opcode & BIT(7)))
	144	+ return -ENOEXEC;
	145	+
	146	+ l = *opcode++;
	147	+ opcode += l;
	148	+ size -= l + 1;
	149	+ }
	150	+
	151	+ /*
	152	+ * Starting from 'loc', apply the CFA opcodes that advance the location
	153	+ * pointer, and identify the locations of the PAC instructions.
	154	+ */
	155	+ while (size-- > 0) {
	156	+ switch (*opcode++) {
	157	+ case DW_CFA_nop:
	158	+ case DW_CFA_remember_state:
	159	+ case DW_CFA_restore_state:
	160	+ break;
	161	+
	162	+ case DW_CFA_advance_loc1:
	163	+ loc += opcode++ code_alignment_factor;
	164	+ size--;
	165	+ break;
	166	+
	167	+ case DW_CFA_advance_loc2:
	168	+ loc += opcode++ code_alignment_factor;
	169	+ loc += (opcode++ << 8) code_alignment_factor;
	170	+ size -= 2;
	171	+ break;
	172	+
	173	+ case DW_CFA_def_cfa:
	174	+ case DW_CFA_offset_extended:
	175	+ size = skip_xleb128(&opcode, size);
	176	+ fallthrough;
	177	+ case DW_CFA_def_cfa_offset:
	178	+ case DW_CFA_def_cfa_offset_sf:
	179	+ case DW_CFA_def_cfa_register:
	180	+ case DW_CFA_same_value:
	181	+ case DW_CFA_restore_extended:
	182	+ case 0x80 ... 0xbf:
	183	+ size = skip_xleb128(&opcode, size);
	184	+ break;
	185	+
	186	+ case DW_CFA_negate_ra_state:
	187	+ scs_patch_loc(loc - 4);
	188	+ break;
	189	+
	190	+ case 0x40 ... 0x7f:
	191	+ // advance loc
	192	+ loc += (opcode[-1] & 0x3f) * code_alignment_factor;
	193	+ break;
	194	+
	195	+ case 0xc0 ... 0xff:
	196	+ break;
	197	+
	198	+ default:
	199	+ pr_err("unhandled opcode: %02x in FDE frame %lx\n", opcode[-1], (uintptr_t)frame);
	200	+ return -ENOEXEC;
	201	+ }
	202	+ }
	203	+ return 0;
	204	+}
	205	+
	206	+int noinstr scs_patch(const u8 eh_frame[], int size)
	207	+{
	208	+ const u8 *p = eh_frame;
	209	+
	210	+ while (size > 4) {
	211	+ const struct eh_frame frame = (const void )p;
	212	+ bool fde_has_augmentation_data = true;
	213	+ int code_alignment_factor = 1;
	214	+ int ret;
	215	+
	216	+ if (frame->size == 0 \|\|
	217	+ frame->size == U32_MAX \|\|
	218	+ frame->size > size)
	219	+ break;
	220	+
	221	+ if (frame->cie_id_or_pointer == 0) {
	222	+ const u8 *p = frame->augmentation_string;
	223	+
	224	+ /* a 'z' in the augmentation string must come first */
	225	+ fde_has_augmentation_data = *p == 'z';
	226	+
	227	+ /*
	228	+ * The code alignment factor is a uleb128 encoded field
	229	+ * but given that the only sensible values are 1 or 4,
	230	+ * there is no point in decoding the whole thing.
	231	+ */
	232	+ p += strlen(p) + 1;
	233	+ if (!WARN_ON(*p & BIT(7)))
	234	+ code_alignment_factor = *p;
	235	+ } else {
	236	+ ret = scs_handle_fde_frame(frame,
	237	+ fde_has_augmentation_data,
	238	+ code_alignment_factor);
	239	+ if (ret)
	240	+ return ret;
	241	+ }
	242	+
	243	+ p += sizeof(frame->size) + frame->size;
	244	+ size -= sizeof(frame->size) + frame->size;
	245	+ }
	246	+ return 0;
	247	+}
	248	+
	249	+asmlinkage void __init scs_patch_vmlinux(void)
	250	+{
	251	+ if (!should_patch_pac_into_scs())
	252	+ return;
	253	+
	254	+ WARN_ON(scs_patch(__eh_frame_start, __eh_frame_end - __eh_frame_start));
	255	+ icache_inval_all_pou();
	256	+ isb();
	257	+}

arch/arm64/kernel/perf_event.c+2−1

@@ -1146,7 +1146,8 @@ static void __armv8pmu_probe_pmu(void *info) · 1145 unmodified lines
1146	1146	dfr0 = read_sysreg(id_aa64dfr0_el1);
1147	1147	pmuver = cpuid_feature_extract_unsigned_field(dfr0,
1148	1148	ID_AA64DFR0_EL1_PMUVer_SHIFT);
1149		− if (pmuver == ID_AA64DFR0_EL1_PMUVer_IMP_DEF \|\| pmuver == 0)
	1149	+ if (pmuver == ID_AA64DFR0_EL1_PMUVer_IMP_DEF \|\|
	1150	+ pmuver == ID_AA64DFR0_EL1_PMUVer_NI)
1150	1151	return;
1151	1152
1152	1153	cpu_pmu->pmuver = pmuver;

arch/arm64/kernel/pi/Makefile+1−0

@@ -7,6 +7,7 @@ KBUILD_CFLAGS := $(subst $(CC_FLAGS_FTRACE),,$(KBUILD_CFLAGS)) -fpie \ · 6 unmodified lines
7	7	-I$(srctree)/scripts/dtc/libfdt -fno-stack-protector \
8	8	-include $(srctree)/include/linux/hidden.h \
9	9	-D__DISABLE_EXPORTS -ffreestanding -D__NO_FORTIFY \
	10	+ -fno-asynchronous-unwind-tables -fno-unwind-tables \
10	11	$(call cc-option,-fno-addrsig)
11	12
12	13	# remove SCS flags from all objects in this directory

arch/arm64/kernel/probes/decode-insn.c+1−1

@@ -24,7 +24,7 @@ static bool __kprobes aarch64_insn_is_steppable(u32 insn) · 23 unmodified lines
24	24	* currently safe. Lastly, MSR instructions can do any number of nasty
25	25	* things we can't handle during single-stepping.
26	26	*/
27		− if (aarch64_get_insn_class(insn) == AARCH64_INSN_CLS_BR_SYS) {
	27	+ if (aarch64_insn_is_class_branch_sys(insn)) {
28	28	if (aarch64_insn_is_branch(insn) \|\|
29	29	aarch64_insn_is_msr_imm(insn) \|\|
30	30	aarch64_insn_is_msr_reg(insn) \|\|

arch/arm64/kernel/probes/kprobes.c+36−50

@@ -294,19 +294,12 @@ int __kprobes kprobe_fault_handler(struct pt_regs *regs, unsigned int fsr) · 293 unmodified lines
294	294	}
295	295
296	296	break;
297		− case KPROBE_HIT_ACTIVE:
298		− case KPROBE_HIT_SSDONE:
299		− /*
300		− * In case the user-specified fault handler returned
301		− * zero, try to fix up.
302		− */
303		− if (fixup_exception(regs))
304		− return 1;
305	297	}
306	298	return 0;
307	299	}
308	300
309		−static void __kprobes kprobe_handler(struct pt_regs *regs)
	301	+static int __kprobes
	302	+kprobe_breakpoint_handler(struct pt_regs *regs, unsigned long esr)
310	303	{
311	304	struct kprobe p, cur_kprobe;
312	305	struct kprobe_ctlblk *kcb;
@@ -316,39 +309,44 @@ static void __kprobes kprobe_handler(struct pt_regs *regs) · 3 unmodified lines
316	309	cur_kprobe = kprobe_running();
317	310
318	311	p = get_kprobe((kprobe_opcode_t *) addr);
	312	+ if (WARN_ON_ONCE(!p)) {
	313	+ /*
	314	+ * Something went wrong. This BRK used an immediate reserved
	315	+ * for kprobes, but we couldn't find any corresponding probe.
	316	+ */
	317	+ return DBG_HOOK_ERROR;
	318	+ }
319	319
320		− if (p) {
321		− if (cur_kprobe) {
322		− if (reenter_kprobe(p, regs, kcb))
323		− return;
324		− } else {
325		− /* Probe hit */
326		− set_current_kprobe(p);
327		− kcb->kprobe_status = KPROBE_HIT_ACTIVE;
328		−
329		− /*
330		− * If we have no pre-handler or it returned 0, we
331		− * continue with normal processing. If we have a
332		− * pre-handler and it returned non-zero, it will
333		− * modify the execution path and no need to single
334		− * stepping. Let's just reset current kprobe and exit.
335		− */
336		− if (!p->pre_handler \|\| !p->pre_handler(p, regs)) {
337		− setup_singlestep(p, regs, kcb, 0);
338		− } else
339		− reset_current_kprobe();
340		− }
	320	+ if (cur_kprobe) {
	321	+ /* Hit a kprobe inside another kprobe */
	322	+ if (!reenter_kprobe(p, regs, kcb))
	323	+ return DBG_HOOK_ERROR;
	324	+ } else {
	325	+ /* Probe hit */
	326	+ set_current_kprobe(p);
	327	+ kcb->kprobe_status = KPROBE_HIT_ACTIVE;
	328	+
	329	+ /*
	330	+ * If we have no pre-handler or it returned 0, we
	331	+ * continue with normal processing. If we have a
	332	+ * pre-handler and it returned non-zero, it will
	333	+ * modify the execution path and not need to single-step
	334	+ * Let's just reset current kprobe and exit.
	335	+ */
	336	+ if (!p->pre_handler \|\| !p->pre_handler(p, regs))
	337	+ setup_singlestep(p, regs, kcb, 0);
	338	+ else
	339	+ reset_current_kprobe();
341	340	}
342		− /*
343		− * The breakpoint instruction was removed right
344		− * after we hit it. Another cpu has removed
345		− * either a probepoint or a debugger breakpoint
346		− * at this address. In either case, no further
347		− * handling of this interrupt is appropriate.
348		− * Return back to original instruction, and continue.
349		− */
	341	+
	342	+ return DBG_HOOK_HANDLED;
350	343	}
351	344
	345	+static struct break_hook kprobes_break_hook = {
	346	+ .imm = KPROBES_BRK_IMM,
	347	+ .fn = kprobe_breakpoint_handler,
	348	+};
	349	+
352	350	static int __kprobes
353	351	kprobe_breakpoint_ss_handler(struct pt_regs *regs, unsigned long esr)
354	352	{
@@ -373,18 +371,6 @@ static struct break_hook kprobes_break_ss_hook = { · 18 unmodified lines
373	371	.fn = kprobe_breakpoint_ss_handler,
374	372	};
375	373
376		−static int __kprobes
377		−kprobe_breakpoint_handler(struct pt_regs *regs, unsigned long esr)
378		−{
379		− kprobe_handler(regs);
380		− return DBG_HOOK_HANDLED;
381		−}
382		−
383		−static struct break_hook kprobes_break_hook = {
384		− .imm = KPROBES_BRK_IMM,
385		− .fn = kprobe_breakpoint_handler,
386		−};
387		−
388	374	/*
389	375	* Provide a blacklist of symbols identifying ranges which cannot be kprobed.
390	376	* This blacklist is exposed to userspace via debugfs (kprobes/blacklist).

arch/arm64/kernel/process.c+2−0

@@ -331,6 +331,8 @@ int arch_dup_task_struct(struct task_struct dst, struct task_struct src) · 330 unmodified lines
331	331	clear_tsk_thread_flag(dst, TIF_SME);
332	332	}
333	333
	334	+ dst->thread.fp_type = FP_STATE_FPSIMD;
	335	+
334	336	/* clear any pending asynchronous tag fault raised by the parent */
335	337	clear_tsk_thread_flag(dst, TIF_MTE_ASYNC_FAULT);
336	338

arch/arm64/kernel/proton-pack.c+7−19

@@ -521,10 +521,13 @@ bool has_spectre_v4(const struct arm64_cpu_capabilities *cap, int scope) · 520 unmodified lines
521	521	return state != SPECTRE_UNAFFECTED;
522	522	}
523	523
524		−static int ssbs_emulation_handler(struct pt_regs *regs, u32 instr)
	524	+bool try_emulate_el1_ssbs(struct pt_regs *regs, u32 instr)
525	525	{
526		− if (user_mode(regs))
527		− return 1;
	526	+ const u32 instr_mask = ~(1U << PSTATE_Imm_shift);
	527	+ const u32 instr_val = 0xd500401f \| PSTATE_SSBS;
	528	+
	529	+ if ((instr & instr_mask) != instr_val)
	530	+ return false;
528	531
529	532	if (instr & BIT(PSTATE_Imm_shift))
530	533	regs->pstate \|= PSR_SSBS_BIT;
@@ -532,19 +535,11 @@ static int ssbs_emulation_handler(struct pt_regs *regs, u32 instr) · 1 unmodified line
532	535	regs->pstate &= ~PSR_SSBS_BIT;
533	536
534	537	arm64_skip_faulting_instruction(regs, 4);
535		− return 0;
	538	+ return true;
536	539	}
537	540
538		−static struct undef_hook ssbs_emulation_hook = {
539		− .instr_mask = ~(1U << PSTATE_Imm_shift),
540		− .instr_val = 0xd500401f \| PSTATE_SSBS,
541		− .fn = ssbs_emulation_handler,
542		−};
543		−
544	541	static enum mitigation_state spectre_v4_enable_hw_mitigation(void)
545	542	{
546		− static bool undef_hook_registered = false;
547		− static DEFINE_RAW_SPINLOCK(hook_lock);
548	543	enum mitigation_state state;
549	544
550	545	/*
@@ -555,13 +550,6 @@ static enum mitigation_state spectre_v4_enable_hw_mitigation(void) · 4 unmodified lines
555	550	if (state != SPECTRE_MITIGATED \|\| !this_cpu_has_cap(ARM64_SSBS))
556	551	return state;
557	552
558		− raw_spin_lock(&hook_lock);
559		− if (!undef_hook_registered) {
560		− register_undef_hook(&ssbs_emulation_hook);
561		− undef_hook_registered = true;
562		− }
563		− raw_spin_unlock(&hook_lock);
564		−
565	553	if (spectre_v4_mitigations_off()) {
566	554	sysreg_clear_set(sctlr_el1, 0, SCTLR_ELx_DSSBS);
567	555	set_pstate_ssbs(1);

arch/arm64/kernel/ptrace.c+3−2

@@ -907,8 +907,7 @@ static int sve_set_common(struct task_struct *target, · 906 unmodified lines
907	907	ret = __fpr_set(target, regset, pos, count, kbuf, ubuf,
908	908	SVE_PT_FPSIMD_OFFSET);
909	909	clear_tsk_thread_flag(target, TIF_SVE);
910		− if (type == ARM64_VEC_SME)
911		− fpsimd_force_sync_to_sve(target);
	910	+ target->thread.fp_type = FP_STATE_FPSIMD;
912	911	goto out;
913	912	}
914	913
@@ -931,6 +930,7 @@ static int sve_set_common(struct task_struct *target, · 16 unmodified lines
931	930	if (!target->thread.sve_state) {
932	931	ret = -ENOMEM;
933	932	clear_tsk_thread_flag(target, TIF_SVE);
	933	+ target->thread.fp_type = FP_STATE_FPSIMD;
934	934	goto out;
935	935	}
936	936
@@ -942,6 +942,7 @@ static int sve_set_common(struct task_struct *target, · 5 unmodified lines
942	942	*/
943	943	fpsimd_sync_to_sve(target);
944	944	set_tsk_thread_flag(target, TIF_SVE);
	945	+ target->thread.fp_type = FP_STATE_SVE;
945	946
946	947	BUILD_BUG_ON(SVE_PT_SVE_OFFSET != sizeof(header));
947	948	start = SVE_PT_SVE_OFFSET;

arch/arm64/kernel/sdei.c+1−1

@@ -144,7 +144,7 @@ static int init_sdei_scs(void) · 143 unmodified lines
144	144	int cpu;
145	145	int err = 0;
146	146
147		− if (!IS_ENABLED(CONFIG_SHADOW_CALL_STACK))
	147	+ if (!scs_is_enabled())
148	148	return 0;
149	149
150	150	for_each_possible_cpu(cpu) {

arch/arm64/kernel/setup.c+4−0

@@ -30,6 +30,7 @@ · 29 unmodified lines
30	30	#include <linux/efi.h>
31	31	#include <linux/psci.h>
32	32	#include <linux/sched/task.h>
	33	+#include <linux/scs.h>
33	34	#include <linux/mm.h>
34	35
35	36	#include <asm/acpi.h>
@@ -42,6 +43,7 @@ · 6 unmodified lines
42	43	#include <asm/cpu_ops.h>
43	44	#include <asm/kasan.h>
44	45	#include <asm/numa.h>
	46	+#include <asm/scs.h>
45	47	#include <asm/sections.h>
46	48	#include <asm/setup.h>
47	49	#include <asm/smp_plat.h>
@@ -312,6 +314,8 @@ void __init __no_sanitize_address setup_arch(char **cmdline_p) · 264 unmodified lines
312	314	jump_label_init();
313	315	parse_early_param();
314	316
	317	+ dynamic_scs_init();
	318	+
315	319	/*
316	320	* Unmask asynchronous aborts and fiq after bringing up possible
317	321	* earlycon. (Report possible System Errors once we can report this

arch/arm64/kernel/signal.c+6−1

@@ -207,6 +207,7 @@ static int restore_fpsimd_context(struct fpsimd_context __user *ctx) · 206 unmodified lines
207	207	__get_user_error(fpsimd.fpcr, &ctx->fpcr, err);
208	208
209	209	clear_thread_flag(TIF_SVE);
	210	+ current->thread.fp_type = FP_STATE_FPSIMD;
210	211
211	212	/* load the hardware registers from the fpsimd_state structure */
212	213	if (!err)
@@ -292,6 +293,7 @@ static int restore_sve_fpsimd_context(struct user_ctxs *user) · 79 unmodified lines
292	293	if (sve.head.size <= sizeof(*user->sve)) {
293	294	clear_thread_flag(TIF_SVE);
294	295	current->thread.svcr &= ~SVCR_SM_MASK;
	296	+ current->thread.fp_type = FP_STATE_FPSIMD;
295	297	goto fpsimd_only;
296	298	}
297	299
@@ -327,6 +329,7 @@ static int restore_sve_fpsimd_context(struct user_ctxs *user) · 29 unmodified lines
327	329	current->thread.svcr \|= SVCR_SM_MASK;
328	330	else
329	331	set_thread_flag(TIF_SVE);
	332	+ current->thread.fp_type = FP_STATE_SVE;
330	333
331	334	fpsimd_only:
332	335	/* copy the FP and status/control registers */
@@ -932,9 +935,11 @@ static void setup_return(struct pt_regs regs, struct k_sigaction ka, · 599 unmodified lines
932	935	* FPSIMD register state - flush the saved FPSIMD
933	936	* register state in case it gets loaded.
934	937	*/
935		− if (current->thread.svcr & SVCR_SM_MASK)
	938	+ if (current->thread.svcr & SVCR_SM_MASK) {
936	939	memset(&current->thread.uw.fpsimd_state, 0,
937	940	sizeof(current->thread.uw.fpsimd_state));
	941	+ current->thread.fp_type = FP_STATE_FPSIMD;
	942	+ }
938	943
939	944	current->thread.svcr &= ~(SVCR_ZA_MASK \|
940	945	SVCR_SM_MASK);

arch/arm64/kernel/stacktrace.c+5−5

@@ -23,8 +23,8 @@ · 22 unmodified lines
23	23	*
24	24	* The regs must be on a stack currently owned by the calling task.
25	25	*/
26		−static inline void unwind_init_from_regs(struct unwind_state *state,
27		− struct pt_regs *regs)
	26	+static __always_inline void unwind_init_from_regs(struct unwind_state *state,
	27	+ struct pt_regs *regs)
28	28	{
29	29	unwind_init_common(state, current);
30	30
@@ -58,8 +58,8 @@ static __always_inline void unwind_init_from_caller(struct unwind_state *state) · 27 unmodified lines
58	58	* duration of the unwind, or the unwind will be bogus. It is never valid to
59	59	* call this for the current task.
60	60	*/
61		−static inline void unwind_init_from_task(struct unwind_state *state,
62		− struct task_struct *task)
	61	+static __always_inline void unwind_init_from_task(struct unwind_state *state,
	62	+ struct task_struct *task)
63	63	{
64	64	unwind_init_common(state, task);
65	65
@@ -186,7 +186,7 @@ void show_stack(struct task_struct tsk, unsigned long sp, const char *loglvl) · 120 unmodified lines
186	186	: stackinfo_get_unknown(); \
187	187	})
188	188
189		−noinline notrace void arch_stack_walk(stack_trace_consume_fn consume_entry,
	189	+noinline noinstr void arch_stack_walk(stack_trace_consume_fn consume_entry,
190	190	void cookie, struct task_struct task,
191	191	struct pt_regs *regs)
192	192	{

arch/arm64/kernel/suspend.c+2−0

@@ -60,6 +60,8 @@ void notrace __cpu_suspend_exit(void) · 59 unmodified lines
60	60	* PSTATE was not saved over suspend/resume, re-enable any detected
61	61	* features that might not have been set correctly.
62	62	*/
	63	+ if (cpus_have_const_cap(ARM64_HAS_DIT))
	64	+ set_pstate_dit(1);
63	65	__uaccess_enable_hw_pan();
64	66
65	67	/*

arch/arm64/kernel/syscall.c+5−14

@@ -183,21 +183,12 @@ static inline void fp_user_discard(void) · 182 unmodified lines
183	183	if (!system_supports_sve())
184	184	return;
185	185
186		− /*
187		− * If SME is not active then disable SVE, the registers will
188		− * be cleared when userspace next attempts to access them and
189		− * we do not need to track the SVE register state until then.
190		− */
191		− clear_thread_flag(TIF_SVE);
	186	+ if (test_thread_flag(TIF_SVE)) {
	187	+ unsigned int sve_vq_minus_one;
192	188
193		− /*
194		− * task_fpsimd_load() won't be called to update CPACR_EL1 in
195		− * ret_to_user unless TIF_FOREIGN_FPSTATE is still set, which only
196		− * happens if a context switch or kernel_neon_begin() or context
197		− * modification (sigreturn, ptrace) intervenes.
198		− * So, ensure that CPACR_EL1 is already correct for the fast-path case.
199		− */
200		− sve_user_disable();
	189	+ sve_vq_minus_one = sve_vq_from_vl(task_get_sve_vl(current)) - 1;
	190	+ sve_flush_live(true, sve_vq_minus_one);
	191	+ }
201	192	}
202	193
203	194	void do_el0_svc(struct pt_regs *regs)

arch/arm64/kernel/traps.c+36−57

@@ -373,51 +373,22 @@ void arm64_skip_faulting_instruction(struct pt_regs *regs, unsigned long size) · 372 unmodified lines
373	373	regs->pstate &= ~PSR_BTYPE_MASK;
374	374	}
375	375
376		−static LIST_HEAD(undef_hook);
377		−static DEFINE_RAW_SPINLOCK(undef_lock);
378		−
379		−void register_undef_hook(struct undef_hook *hook)
	376	+static int user_insn_read(struct pt_regs regs, u32 insnp)
380	377	{
381		− unsigned long flags;
382		−
383		− raw_spin_lock_irqsave(&undef_lock, flags);
384		− list_add(&hook->node, &undef_hook);
385		− raw_spin_unlock_irqrestore(&undef_lock, flags);
386		−}
387		−
388		−void unregister_undef_hook(struct undef_hook *hook)
389		−{
390		− unsigned long flags;
391		−
392		− raw_spin_lock_irqsave(&undef_lock, flags);
393		− list_del(&hook->node);
394		− raw_spin_unlock_irqrestore(&undef_lock, flags);
395		−}
396		−
397		−static int call_undef_hook(struct pt_regs *regs)
398		−{
399		− struct undef_hook *hook;
400		− unsigned long flags;
401	378	u32 instr;
402		− int (fn)(struct pt_regs regs, u32 instr) = NULL;
403	379	unsigned long pc = instruction_pointer(regs);
404	380
405		− if (!user_mode(regs)) {
406		− __le32 instr_le;
407		− if (get_kernel_nofault(instr_le, (__le32 *)pc))
408		− goto exit;
409		− instr = le32_to_cpu(instr_le);
410		− } else if (compat_thumb_mode(regs)) {
	381	+ if (compat_thumb_mode(regs)) {
411	382	/* 16-bit Thumb instruction */
412	383	__le16 instr_le;
413	384	if (get_user(instr_le, (__le16 __user *)pc))
414		− goto exit;
	385	+ return -EFAULT;
415	386	instr = le16_to_cpu(instr_le);
416	387	if (aarch32_insn_is_wide(instr)) {
417	388	u32 instr2;
418	389
419	390	if (get_user(instr_le, (__le16 __user *)(pc + 2)))
420		− goto exit;
	391	+ return -EFAULT;
421	392	instr2 = le16_to_cpu(instr_le);
422	393	instr = (instr << 16) \| instr2;
423	394	}
@@ -425,19 +396,12 @@ static int call_undef_hook(struct pt_regs *regs) · 1 unmodified line
425	396	/* 32-bit ARM instruction */
426	397	__le32 instr_le;
427	398	if (get_user(instr_le, (__le32 __user *)pc))
428		− goto exit;
	399	+ return -EFAULT;
429	400	instr = le32_to_cpu(instr_le);
430	401	}
431	402
432		− raw_spin_lock_irqsave(&undef_lock, flags);
433		− list_for_each_entry(hook, &undef_hook, node)
434		− if ((instr & hook->instr_mask) == hook->instr_val &&
435		− (regs->pstate & hook->pstate_mask) == hook->pstate_val)
436		− fn = hook->fn;
437		−
438		− raw_spin_unlock_irqrestore(&undef_lock, flags);
439		−exit:
440		− return fn ? fn(regs, instr) : 1;
	403	+ *insnp = instr;
	404	+ return 0;
441	405	}
442	406
443	407	void force_signal_inject(int signal, int code, unsigned long address, unsigned long err)
@@ -486,21 +450,40 @@ void arm64_notify_segfault(unsigned long addr) · 42 unmodified lines
486	450	force_signal_inject(SIGSEGV, code, addr, 0);
487	451	}
488	452
489		−void do_undefinstr(struct pt_regs *regs, unsigned long esr)
	453	+void do_el0_undef(struct pt_regs *regs, unsigned long esr)
490	454	{
	455	+ u32 insn;
	456	+
491	457	/* check for AArch32 breakpoint instructions */
492	458	if (!aarch32_break_handler(regs))
493	459	return;
494	460
495		− if (call_undef_hook(regs) == 0)
	461	+ if (user_insn_read(regs, &insn))
	462	+ goto out_err;
	463	+
	464	+ if (try_emulate_mrs(regs, insn))
496	465	return;
497	466
498		− if (!user_mode(regs))
499		− die("Oops - Undefined instruction", regs, esr);
	467	+ if (try_emulate_armv8_deprecated(regs, insn))
	468	+ return;
500	469
	470	+out_err:
501	471	force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc, 0);
502	472	}
503		−NOKPROBE_SYMBOL(do_undefinstr);
	473	+
	474	+void do_el1_undef(struct pt_regs *regs, unsigned long esr)
	475	+{
	476	+ u32 insn;
	477	+
	478	+ if (aarch64_insn_read((void *)regs->pc, &insn))
	479	+ goto out_err;
	480	+
	481	+ if (try_emulate_el1_ssbs(regs, insn))
	482	+ return;
	483	+
	484	+out_err:
	485	+ die("Oops - Undefined instruction", regs, esr);
	486	+}
504	487
505	488	void do_el0_bti(struct pt_regs *regs)
506	489	{
@@ -511,7 +494,6 @@ void do_el1_bti(struct pt_regs *regs, unsigned long esr) · 4 unmodified lines
511	494	{
512	495	die("Oops - BTI", regs, esr);
513	496	}
514		−NOKPROBE_SYMBOL(do_el1_bti);
515	497
516	498	void do_el0_fpac(struct pt_regs *regs, unsigned long esr)
517	499	{
@@ -526,7 +508,6 @@ void do_el1_fpac(struct pt_regs *regs, unsigned long esr) · 8 unmodified lines
526	508	*/
527	509	die("Oops - FPAC", regs, esr);
528	510	}
529		−NOKPROBE_SYMBOL(do_el1_fpac)
530	511
531	512	#define __user_cache_maint(insn, address, res) \
532	513	if (address >= TASK_SIZE_MAX) { \
@@ -748,7 +729,7 @@ static const struct sys64_hook cp15_64_hooks[] = { · 215 unmodified lines
748	729	{},
749	730	};
750	731
751		−void do_cp15instr(unsigned long esr, struct pt_regs *regs)
	732	+void do_el0_cp15(unsigned long esr, struct pt_regs *regs)
752	733	{
753	734	const struct sys64_hook hook, hook_base;
754	735
@@ -769,7 +750,7 @@ void do_cp15instr(unsigned long esr, struct pt_regs *regs) · 14 unmodified lines
769	750	hook_base = cp15_64_hooks;
770	751	break;
771	752	default:
772		− do_undefinstr(regs, esr);
	753	+ do_el0_undef(regs, esr);
773	754	return;
774	755	}
775	756
@@ -784,12 +765,11 @@ void do_cp15instr(unsigned long esr, struct pt_regs *regs) · 8 unmodified lines
784	765	* EL0. Fall back to our usual undefined instruction handler
785	766	* so that we handle these consistently.
786	767	*/
787		− do_undefinstr(regs, esr);
	768	+ do_el0_undef(regs, esr);
788	769	}
789		−NOKPROBE_SYMBOL(do_cp15instr);
790	770	#endif
791	771
792		−void do_sysinstr(unsigned long esr, struct pt_regs *regs)
	772	+void do_el0_sys(unsigned long esr, struct pt_regs *regs)
793	773	{
794	774	const struct sys64_hook *hook;
795	775
@@ -804,9 +784,8 @@ void do_sysinstr(unsigned long esr, struct pt_regs *regs) · 8 unmodified lines
804	784	* back to our usual undefined instruction handler so that we handle
805	785	* these consistently.
806	786	*/
807		− do_undefinstr(regs, esr);
	787	+ do_el0_undef(regs, esr);
808	788	}
809		−NOKPROBE_SYMBOL(do_sysinstr);
810	789
811	790	static const char *esr_class_str[] = {
812	791	[0 ... ESR_ELx_EC_MAX] = "UNRECOGNIZED EC",

arch/arm64/kernel/vmlinux.lds.S+13−0

@@ -121,6 +121,17 @@ jiffies = jiffies_64; · 120 unmodified lines
121	121	#define TRAMP_TEXT
122	122	#endif
123	123
	124	+#ifdef CONFIG_UNWIND_TABLES
	125	+#define UNWIND_DATA_SECTIONS \
	126	+ .eh_frame : { \
	127	+ __eh_frame_start = .; \
	128	+ *(.eh_frame) \
	129	+ __eh_frame_end = .; \
	130	+ }
	131	+#else
	132	+#define UNWIND_DATA_SECTIONS
	133	+#endif
	134	+
124	135	/*
125	136	* The size of the PE/COFF section that covers the kernel image, which
126	137	* runs from _stext to _edata, must be a round multiple of the PE/COFF
@@ -231,6 +242,8 @@ SECTIONS · 104 unmodified lines
231	242	__alt_instructions_end = .;
232	243	}
233	244
	245	+ UNWIND_DATA_SECTIONS
	246	+
234	247	. = ALIGN(SEGMENT_ALIGN);
235	248	__inittext_end = .;
236	249	__initdata_begin = .;

arch/arm64/kvm/fpsimd.c+18−8

@@ -75,11 +75,12 @@ int kvm_arch_vcpu_run_map_fp(struct kvm_vcpu *vcpu) · 74 unmodified lines
75	75	void kvm_arch_vcpu_load_fp(struct kvm_vcpu *vcpu)
76	76	{
77	77	BUG_ON(!current->mm);
78		− BUG_ON(test_thread_flag(TIF_SVE));
79	78
80	79	if (!system_supports_fpsimd())
81	80	return;
82	81
	82	+ fpsimd_kvm_prepare();
	83	+
83	84	vcpu->arch.fp_state = FP_STATE_HOST_OWNED;
84	85
85	86	vcpu_clear_flag(vcpu, HOST_SVE_ENABLED);
@@ -129,20 +130,31 @@ void kvm_arch_vcpu_ctxflush_fp(struct kvm_vcpu *vcpu) · 43 unmodified lines
129	130	*/
130	131	void kvm_arch_vcpu_ctxsync_fp(struct kvm_vcpu *vcpu)
131	132	{
	133	+ struct cpu_fp_state fp_state;
	134	+
132	135	WARN_ON_ONCE(!irqs_disabled());
133	136
134	137	if (vcpu->arch.fp_state == FP_STATE_GUEST_OWNED) {
	138	+
135	139	/*
136	140	* Currently we do not support SME guests so SVCR is
137	141	* always 0 and we just need a variable to point to.
138	142	*/
139		− fpsimd_bind_state_to_cpu(&vcpu->arch.ctxt.fp_regs,
140		− vcpu->arch.sve_state,
141		− vcpu->arch.sve_max_vl,
142		− NULL, 0, &vcpu->arch.svcr);
	143	+ fp_state.st = &vcpu->arch.ctxt.fp_regs;
	144	+ fp_state.sve_state = vcpu->arch.sve_state;
	145	+ fp_state.sve_vl = vcpu->arch.sve_max_vl;
	146	+ fp_state.za_state = NULL;
	147	+ fp_state.svcr = &vcpu->arch.svcr;
	148	+ fp_state.fp_type = &vcpu->arch.fp_type;
	149	+
	150	+ if (vcpu_has_sve(vcpu))
	151	+ fp_state.to_save = FP_STATE_SVE;
	152	+ else
	153	+ fp_state.to_save = FP_STATE_FPSIMD;
	154	+
	155	+ fpsimd_bind_state_to_cpu(&fp_state);
143	156
144	157	clear_thread_flag(TIF_FOREIGN_FPSTATE);
145		− update_thread_flag(TIF_SVE, vcpu_has_sve(vcpu));
146	158	}
147	159	}
148	160
@@ -199,7 +211,5 @@ void kvm_arch_vcpu_put_fp(struct kvm_vcpu *vcpu) · 50 unmodified lines
199	211	sysreg_clear_set(CPACR_EL1, CPACR_EL1_ZEN_EL0EN, 0);
200	212	}
201	213
202		− update_thread_flag(TIF_SVE, 0);
203		−
204	214	local_irq_restore(flags);
205	215	}

arch/arm64/kvm/hyp/nvhe/Makefile+1−0

@@ -96,6 +96,7 @@ KBUILD_CFLAGS := $(filter-out $(CC_FLAGS_FTRACE) $(CC_FLAGS_SCS) $(CC_FLAGS_CFI) · 95 unmodified lines
96	96	# when profile optimization is applied. gen-hyprel does not support SHT_REL and
97	97	# causes a build failure. Remove profile optimization flags.
98	98	KBUILD_CFLAGS := $(filter-out -fprofile-sample-use=% -fprofile-use=%, $(KBUILD_CFLAGS))
	99	+KBUILD_CFLAGS += -fno-asynchronous-unwind-tables -fno-unwind-tables
99	100
100	101	# KVM nVHE code is run at a different exception code with a different map, so
101	102	# compiler instrumentation that inserts callbacks or checks into the code may

arch/arm64/kvm/sys_regs.c+2−2

@@ -1121,8 +1121,8 @@ static u64 read_id_reg(const struct kvm_vcpu vcpu, struct sys_reg_desc const r · 1120 unmodified lines
1121	1121	case SYS_ID_DFR0_EL1:
1122	1122	/* Limit guests to PMUv3 for ARMv8.4 */
1123	1123	val = cpuid_feature_cap_perfmon_field(val,
1124		− ID_DFR0_PERFMON_SHIFT,
1125		− kvm_vcpu_has_pmu(vcpu) ? ID_DFR0_PERFMON_8_4 : 0);
	1124	+ ID_DFR0_EL1_PerfMon_SHIFT,
	1125	+ kvm_vcpu_has_pmu(vcpu) ? ID_DFR0_EL1_PerfMon_PMUv3p4 : 0);
1126	1126	break;
1127	1127	}
1128	1128

arch/arm64/lib/insn.c+0−165

@@ -20,91 +20,6 @@ · 19 unmodified lines
20	20	#define AARCH64_INSN_N_BIT BIT(22)
21	21	#define AARCH64_INSN_LSL_12 BIT(22)
22	22
23		−static const int aarch64_insn_encoding_class[] = {
24		− AARCH64_INSN_CLS_UNKNOWN,
25		− AARCH64_INSN_CLS_UNKNOWN,
26		− AARCH64_INSN_CLS_SVE,
27		− AARCH64_INSN_CLS_UNKNOWN,
28		− AARCH64_INSN_CLS_LDST,
29		− AARCH64_INSN_CLS_DP_REG,
30		− AARCH64_INSN_CLS_LDST,
31		− AARCH64_INSN_CLS_DP_FPSIMD,
32		− AARCH64_INSN_CLS_DP_IMM,
33		− AARCH64_INSN_CLS_DP_IMM,
34		− AARCH64_INSN_CLS_BR_SYS,
35		− AARCH64_INSN_CLS_BR_SYS,
36		− AARCH64_INSN_CLS_LDST,
37		− AARCH64_INSN_CLS_DP_REG,
38		− AARCH64_INSN_CLS_LDST,
39		− AARCH64_INSN_CLS_DP_FPSIMD,
40		−};
41		−
42		−enum aarch64_insn_encoding_class __kprobes aarch64_get_insn_class(u32 insn)
43		−{
44		− return aarch64_insn_encoding_class[(insn >> 25) & 0xf];
45		−}
46		−
47		−bool __kprobes aarch64_insn_is_steppable_hint(u32 insn)
48		−{
49		− if (!aarch64_insn_is_hint(insn))
50		− return false;
51		−
52		− switch (insn & 0xFE0) {
53		− case AARCH64_INSN_HINT_XPACLRI:
54		− case AARCH64_INSN_HINT_PACIA_1716:
55		− case AARCH64_INSN_HINT_PACIB_1716:
56		− case AARCH64_INSN_HINT_PACIAZ:
57		− case AARCH64_INSN_HINT_PACIASP:
58		− case AARCH64_INSN_HINT_PACIBZ:
59		− case AARCH64_INSN_HINT_PACIBSP:
60		− case AARCH64_INSN_HINT_BTI:
61		− case AARCH64_INSN_HINT_BTIC:
62		− case AARCH64_INSN_HINT_BTIJ:
63		− case AARCH64_INSN_HINT_BTIJC:
64		− case AARCH64_INSN_HINT_NOP:
65		− return true;
66		− default:
67		− return false;
68		− }
69		−}
70		−
71		−bool aarch64_insn_is_branch_imm(u32 insn)
72		−{
73		− return (aarch64_insn_is_b(insn) \|\| aarch64_insn_is_bl(insn) \|\|
74		− aarch64_insn_is_tbz(insn) \|\| aarch64_insn_is_tbnz(insn) \|\|
75		− aarch64_insn_is_cbz(insn) \|\| aarch64_insn_is_cbnz(insn) \|\|
76		− aarch64_insn_is_bcond(insn));
77		−}
78		−
79		−bool __kprobes aarch64_insn_uses_literal(u32 insn)
80		−{
81		− /* ldr/ldrsw (literal), prfm */
82		−
83		− return aarch64_insn_is_ldr_lit(insn) \|\|
84		− aarch64_insn_is_ldrsw_lit(insn) \|\|
85		− aarch64_insn_is_adr_adrp(insn) \|\|
86		− aarch64_insn_is_prfm_lit(insn);
87		−}
88		−
89		−bool __kprobes aarch64_insn_is_branch(u32 insn)
90		−{
91		− /* b, bl, cb, tb, ret, b.cond, br, blr* */
92		−
93		− return aarch64_insn_is_b(insn) \|\|
94		− aarch64_insn_is_bl(insn) \|\|
95		− aarch64_insn_is_cbz(insn) \|\|
96		− aarch64_insn_is_cbnz(insn) \|\|
97		− aarch64_insn_is_tbz(insn) \|\|
98		− aarch64_insn_is_tbnz(insn) \|\|
99		− aarch64_insn_is_ret(insn) \|\|
100		− aarch64_insn_is_ret_auth(insn) \|\|
101		− aarch64_insn_is_br(insn) \|\|
102		− aarch64_insn_is_br_auth(insn) \|\|
103		− aarch64_insn_is_blr(insn) \|\|
104		− aarch64_insn_is_blr_auth(insn) \|\|
105		− aarch64_insn_is_bcond(insn);
106		−}
107		−
108	23	static int __kprobes aarch64_get_imm_shift_mask(enum aarch64_insn_imm_type type,
109	24	u32 maskp, int shiftp)
110	25	{
@@ -435,16 +350,6 @@ u32 aarch64_insn_gen_cond_branch_imm(unsigned long pc, unsigned long addr, · 324 unmodified lines
435	350	offset >> 2);
436	351	}
437	352
438		−u32 __kprobes aarch64_insn_gen_hint(enum aarch64_insn_hint_cr_op op)
439		−{
440		− return aarch64_insn_get_hint_value() \| op;
441		−}
442		−
443		−u32 __kprobes aarch64_insn_gen_nop(void)
444		−{
445		− return aarch64_insn_gen_hint(AARCH64_INSN_HINT_NOP);
446		−}
447		−
448	353	u32 aarch64_insn_gen_branch_reg(enum aarch64_insn_register reg,
449	354	enum aarch64_insn_branch_type type)
450	355	{
@@ -816,76 +721,6 @@ u32 aarch64_insn_gen_cas(enum aarch64_insn_register result, · 365 unmodified lines
816	721	}
817	722	#endif
818	723
819		−static u32 aarch64_insn_encode_prfm_imm(enum aarch64_insn_prfm_type type,
820		− enum aarch64_insn_prfm_target target,
821		− enum aarch64_insn_prfm_policy policy,
822		− u32 insn)
823		−{
824		− u32 imm_type = 0, imm_target = 0, imm_policy = 0;
825		−
826		− switch (type) {
827		− case AARCH64_INSN_PRFM_TYPE_PLD:
828		− break;
829		− case AARCH64_INSN_PRFM_TYPE_PLI:
830		− imm_type = BIT(0);
831		− break;
832		− case AARCH64_INSN_PRFM_TYPE_PST:
833		− imm_type = BIT(1);
834		− break;
835		− default:
836		− pr_err("%s: unknown prfm type encoding %d\n", __func__, type);
837		− return AARCH64_BREAK_FAULT;
838		− }
839		−
840		− switch (target) {
841		− case AARCH64_INSN_PRFM_TARGET_L1:
842		− break;
843		− case AARCH64_INSN_PRFM_TARGET_L2:
844		− imm_target = BIT(0);
845		− break;
846		− case AARCH64_INSN_PRFM_TARGET_L3:
847		− imm_target = BIT(1);
848		− break;
849		− default:
850		− pr_err("%s: unknown prfm target encoding %d\n", __func__, target);
851		− return AARCH64_BREAK_FAULT;
852		− }
853		−
854		− switch (policy) {
855		− case AARCH64_INSN_PRFM_POLICY_KEEP:
856		− break;
857		− case AARCH64_INSN_PRFM_POLICY_STRM:
858		− imm_policy = BIT(0);
859		− break;
860		− default:
861		− pr_err("%s: unknown prfm policy encoding %d\n", __func__, policy);
862		− return AARCH64_BREAK_FAULT;
863		− }
864		−
865		− /* In this case, imm5 is encoded into Rt field. */
866		− insn &= ~GENMASK(4, 0);
867		− insn \|= imm_policy \| (imm_target << 1) \| (imm_type << 3);
868		−
869		− return insn;
870		−}
871		−
872		−u32 aarch64_insn_gen_prefetch(enum aarch64_insn_register base,
873		− enum aarch64_insn_prfm_type type,
874		− enum aarch64_insn_prfm_target target,
875		− enum aarch64_insn_prfm_policy policy)
876		−{
877		− u32 insn = aarch64_insn_get_prfm_value();
878		−
879		− insn = aarch64_insn_encode_ldst_size(AARCH64_INSN_SIZE_64, insn);
880		−
881		− insn = aarch64_insn_encode_prfm_imm(type, target, policy, insn);
882		−
883		− insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn,
884		− base);
885		−
886		− return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_12, insn, 0);
887		−}
888		−
889	724	u32 aarch64_insn_gen_add_sub_imm(enum aarch64_insn_register dst,
890	725	enum aarch64_insn_register src,
891	726	int imm, enum aarch64_insn_variant variant,

arch/arm64/lib/mte.S+1−1

@@ -18,7 +18,7 @@ · 17 unmodified lines
18	18	*/
19	19	.macro multitag_transfer_size, reg, tmp
20	20	mrs_s \reg, SYS_GMID_EL1
21		− ubfx \reg, \reg, #GMID_EL1_BS_SHIFT, #GMID_EL1_BS_SIZE
	21	+ ubfx \reg, \reg, #GMID_EL1_BS_SHIFT, #GMID_EL1_BS_WIDTH
22	22	mov \tmp, #4
23	23	lsl \reg, \tmp, \reg
24	24	.endm

arch/arm64/mm/fault.c+7−1

@@ -353,6 +353,11 @@ static bool is_el1_mte_sync_tag_check_fault(unsigned long esr) · 352 unmodified lines
353	353	return false;
354	354	}
355	355
	356	+static bool is_translation_fault(unsigned long esr)
	357	+{
	358	+ return (esr & ESR_ELx_FSC_TYPE) == ESR_ELx_FSC_FAULT;
	359	+}
	360	+
356	361	static void __do_kernel_fault(unsigned long addr, unsigned long esr,
357	362	struct pt_regs *regs)
358	363	{
@@ -385,7 +390,8 @@ static void __do_kernel_fault(unsigned long addr, unsigned long esr, · 26 unmodified lines
385	390	} else if (addr < PAGE_SIZE) {
386	391	msg = "NULL pointer dereference";
387	392	} else {
388		− if (kfence_handle_page_fault(addr, esr & ESR_ELx_WNR, regs))
	393	+ if (is_translation_fault(esr) &&
	394	+ kfence_handle_page_fault(addr, esr & ESR_ELx_WNR, regs))
389	395	return;
390	396
391	397	msg = "paging request";

arch/arm64/mm/hugetlbpage.c+21−0

@@ -559,3 +559,24 @@ bool __init arch_hugetlb_valid_size(unsigned long size) · 558 unmodified lines
559	559	{
560	560	return __hugetlb_valid_size(size);
561	561	}
	562	+
	563	+pte_t huge_ptep_modify_prot_start(struct vm_area_struct vma, unsigned long addr, pte_t ptep)
	564	+{
	565	+ if (IS_ENABLED(CONFIG_ARM64_WORKAROUND_2645198) &&
	566	+ cpus_have_const_cap(ARM64_WORKAROUND_2645198)) {
	567	+ /*
	568	+ * Break-before-make (BBM) is required for all user space mappings
	569	+ * when the permission changes from executable to non-executable
	570	+ * in cases where cpu is affected with errata #2645198.
	571	+ */
	572	+ if (pte_user_exec(READ_ONCE(*ptep)))
	573	+ return huge_ptep_clear_flush(vma, addr, ptep);
	574	+ }
	575	+ return huge_ptep_get_and_clear(vma->vm_mm, addr, ptep);
	576	+}
	577	+
	578	+void huge_ptep_modify_prot_commit(struct vm_area_struct vma, unsigned long addr, pte_t ptep,
	579	+ pte_t old_pte, pte_t pte)
	580	+{
	581	+ set_huge_pte_at(vma->vm_mm, addr, ptep, pte);
	582	+}

arch/arm64/mm/init.c+22−3

@@ -96,6 +96,8 @@ phys_addr_t __ro_after_init arm64_dma_phys_limit = PHYS_MASK + 1; · 95 unmodified lines
96	96	#define CRASH_ADDR_LOW_MAX arm64_dma_phys_limit
97	97	#define CRASH_ADDR_HIGH_MAX (PHYS_MASK + 1)
98	98
	99	+#define DEFAULT_CRASH_KERNEL_LOW_SIZE (128UL << 20)
	100	+
99	101	static int __init reserve_crashkernel_low(unsigned long long low_size)
100	102	{
101	103	unsigned long long low_base;
@@ -130,6 +132,7 @@ static void __init reserve_crashkernel(void) · 28 unmodified lines
130	132	unsigned long long crash_max = CRASH_ADDR_LOW_MAX;
131	133	char *cmdline = boot_command_line;
132	134	int ret;
	135	+ bool fixed_base = false;
133	136
134	137	if (!IS_ENABLED(CONFIG_KEXEC_CORE))
135	138	return;
@@ -147,7 +150,9 @@ static void __init reserve_crashkernel(void) · 11 unmodified lines
147	150	* is not allowed.
148	151	*/
149	152	ret = parse_crashkernel_low(cmdline, 0, &crash_low_size, &crash_base);
150		− if (ret && (ret != -ENOENT))
	153	+ if (ret == -ENOENT)
	154	+ crash_low_size = DEFAULT_CRASH_KERNEL_LOW_SIZE;
	155	+ else if (ret)
151	156	return;
152	157
153	158	crash_max = CRASH_ADDR_HIGH_MAX;
@@ -159,18 +164,32 @@ static void __init reserve_crashkernel(void) · 5 unmodified lines
159	164	crash_size = PAGE_ALIGN(crash_size);
160	165
161	166	/* User specifies base address explicitly. */
162		− if (crash_base)
	167	+ if (crash_base) {
	168	+ fixed_base = true;
163	169	crash_max = crash_base + crash_size;
	170	+ }
164	171
	172	+retry:
165	173	crash_base = memblock_phys_alloc_range(crash_size, CRASH_ALIGN,
166	174	crash_base, crash_max);
167	175	if (!crash_base) {
	176	+ /*
	177	+ * If the first attempt was for low memory, fall back to
	178	+ * high memory, the minimum required low memory will be
	179	+ * reserved later.
	180	+ */
	181	+ if (!fixed_base && (crash_max == CRASH_ADDR_LOW_MAX)) {
	182	+ crash_max = CRASH_ADDR_HIGH_MAX;
	183	+ crash_low_size = DEFAULT_CRASH_KERNEL_LOW_SIZE;
	184	+ goto retry;
	185	+ }
	186	+
168	187	pr_warn("cannot allocate crashkernel (size:0x%llx)\n",
169	188	crash_size);
170	189	return;
171	190	}
172	191
173		− if ((crash_base >= CRASH_ADDR_LOW_MAX) &&
	192	+ if ((crash_base > CRASH_ADDR_LOW_MAX - crash_low_size) &&
174	193	crash_low_size && reserve_crashkernel_low(crash_low_size)) {
175	194	memblock_phys_free(crash_base, crash_size);
176	195	return;

arch/arm64/mm/mmu.c+22−1

@@ -1196,7 +1196,7 @@ int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node, · 1195 unmodified lines
1196	1196
1197	1197	WARN_ON((start < VMEMMAP_START) \|\| (end > VMEMMAP_END));
1198	1198
1199		− if (!ARM64_KERNEL_USES_PMD_MAPS)
	1199	+ if (!IS_ENABLED(CONFIG_ARM64_4K_PAGES))
1200	1200	return vmemmap_populate_basepages(start, end, node, altmap);
1201	1201
1202	1202	do {
@@ -1702,3 +1702,24 @@ static int __init prevent_bootmem_remove_init(void) · 499 unmodified lines
1702	1702	}
1703	1703	early_initcall(prevent_bootmem_remove_init);
1704	1704	#endif
	1705	+
	1706	+pte_t ptep_modify_prot_start(struct vm_area_struct vma, unsigned long addr, pte_t ptep)
	1707	+{
	1708	+ if (IS_ENABLED(CONFIG_ARM64_WORKAROUND_2645198) &&
	1709	+ cpus_have_const_cap(ARM64_WORKAROUND_2645198)) {
	1710	+ /*
	1711	+ * Break-before-make (BBM) is required for all user space mappings
	1712	+ * when the permission changes from executable to non-executable
	1713	+ * in cases where cpu is affected with errata #2645198.
	1714	+ */
	1715	+ if (pte_user_exec(READ_ONCE(*ptep)))
	1716	+ return ptep_clear_flush(vma, addr, ptep);
	1717	+ }
	1718	+ return ptep_get_and_clear(vma->vm_mm, addr, ptep);
	1719	+}
	1720	+
	1721	+void ptep_modify_prot_commit(struct vm_area_struct vma, unsigned long addr, pte_t ptep,
	1722	+ pte_t old_pte, pte_t pte)
	1723	+{
	1724	+ set_pte_at(vma->vm_mm, addr, ptep, pte);
	1725	+}

arch/arm64/mm/proc.S+0−4

@@ -189,16 +189,12 @@ SYM_FUNC_END(cpu_do_resume) · 188 unmodified lines
189	189	* called by anything else. It can only be executed from a TTBR0 mapping.
190	190	*/
191	191	SYM_TYPED_FUNC_START(idmap_cpu_replace_ttbr1)
192		− save_and_disable_daif flags=x2
193		−
194	192	__idmap_cpu_set_reserved_ttbr1 x1, x3
195	193
196	194	offset_ttbr1 x0, x3
197	195	msr ttbr1_el1, x0
198	196	isb
199	197
200		− restore_daif x2
201		−
202	198	ret
203	199	SYM_FUNC_END(idmap_cpu_replace_ttbr1)
204	200	.popsection

arch/arm64/tools/cpucaps+2−0

@@ -20,6 +20,7 @@ HAS_CNP · 19 unmodified lines
20	20	HAS_CRC32
21	21	HAS_DCPODP
22	22	HAS_DCPOP
	23	+HAS_DIT
23	24	HAS_E0PD
24	25	HAS_ECV
25	26	HAS_EPAN
@@ -70,6 +71,7 @@ WORKAROUND_2038923 · 44 unmodified lines
70	71	WORKAROUND_2064142
71	72	WORKAROUND_2077057
72	73	WORKAROUND_2457168
	74	+WORKAROUND_2645198
73	75	WORKAROUND_2658417
74	76	WORKAROUND_TRBE_OVERWRITE_FILL_MODE
75	77	WORKAROUND_TSB_FLUSH_FAILURE

arch/arm64/tools/gen-sysreg.awk+1−1

@@ -33,7 +33,7 @@ function expect_fields(nf) { · 32 unmodified lines
33	33	# Print a CPP macro definition, padded with spaces so that the macro bodies
34	34	# line up in a column
35	35	function define(name, val) {
36		− printf "%-48s%s\n", "#define " name, val
	36	+ printf "%-56s%s\n", "#define " name, val
37	37	}
38	38
39	39	# Print standard BITMASK/SHIFT/WIDTH CPP definitions for a field

arch/arm64/tools/sysreg+765−1

@@ -46,6 +46,760 @@ · 45 unmodified lines
46	46	# feature that introduces them (eg, FEAT_LS64_ACCDATA introduces enumeration
47	47	# item ACCDATA) though it may be more taseful to do something else.
48	48
	49	+Sysreg ID_PFR0_EL1 3 0 0 1 0
	50	+Res0 63:32
	51	+Enum 31:28 RAS
	52	+ 0b0000 NI
	53	+ 0b0001 RAS
	54	+ 0b0010 RASv1p1
	55	+EndEnum
	56	+Enum 27:24 DIT
	57	+ 0b0000 NI
	58	+ 0b0001 IMP
	59	+EndEnum
	60	+Enum 23:20 AMU
	61	+ 0b0000 NI
	62	+ 0b0001 AMUv1
	63	+ 0b0010 AMUv1p1
	64	+EndEnum
	65	+Enum 19:16 CSV2
	66	+ 0b0000 UNDISCLOSED
	67	+ 0b0001 IMP
	68	+ 0b0010 CSV2p1
	69	+EndEnum
	70	+Enum 15:12 State3
	71	+ 0b0000 NI
	72	+ 0b0001 IMP
	73	+EndEnum
	74	+Enum 11:8 State2
	75	+ 0b0000 NI
	76	+ 0b0001 NO_CV
	77	+ 0b0010 CV
	78	+EndEnum
	79	+Enum 7:4 State1
	80	+ 0b0000 NI
	81	+ 0b0001 THUMB
	82	+ 0b0010 THUMB2
	83	+EndEnum
	84	+Enum 3:0 State0
	85	+ 0b0000 NI
	86	+ 0b0001 IMP
	87	+EndEnum
	88	+EndSysreg
	89	+
	90	+Sysreg ID_PFR1_EL1 3 0 0 1 1
	91	+Res0 63:32
	92	+Enum 31:28 GIC
	93	+ 0b0000 NI
	94	+ 0b0001 GICv3
	95	+ 0b0010 GICv4p1
	96	+EndEnum
	97	+Enum 27:24 Virt_frac
	98	+ 0b0000 NI
	99	+ 0b0001 IMP
	100	+EndEnum
	101	+Enum 23:20 Sec_frac
	102	+ 0b0000 NI
	103	+ 0b0001 WALK_DISABLE
	104	+ 0b0010 SECURE_MEMORY
	105	+EndEnum
	106	+Enum 19:16 GenTimer
	107	+ 0b0000 NI
	108	+ 0b0001 IMP
	109	+ 0b0010 ECV
	110	+EndEnum
	111	+Enum 15:12 Virtualization
	112	+ 0b0000 NI
	113	+ 0b0001 IMP
	114	+EndEnum
	115	+Enum 11:8 MProgMod
	116	+ 0b0000 NI
	117	+ 0b0001 IMP
	118	+EndEnum
	119	+Enum 7:4 Security
	120	+ 0b0000 NI
	121	+ 0b0001 EL3
	122	+ 0b0001 NSACR_RFR
	123	+EndEnum
	124	+Enum 3:0 ProgMod
	125	+ 0b0000 NI
	126	+ 0b0001 IMP
	127	+EndEnum
	128	+EndSysreg
	129	+
	130	+Sysreg ID_DFR0_EL1 3 0 0 1 2
	131	+Res0 63:32
	132	+Enum 31:28 TraceFilt
	133	+ 0b0000 NI
	134	+ 0b0001 IMP
	135	+EndEnum
	136	+Enum 27:24 PerfMon
	137	+ 0b0000 NI
	138	+ 0b0001 PMUv1
	139	+ 0b0010 PMUv2
	140	+ 0b0011 PMUv3
	141	+ 0b0100 PMUv3p1
	142	+ 0b0101 PMUv3p4
	143	+ 0b0110 PMUv3p5
	144	+ 0b0111 PMUv3p7
	145	+ 0b1000 PMUv3p8
	146	+ 0b1111 IMPDEF
	147	+EndEnum
	148	+Enum 23:20 MProfDbg
	149	+ 0b0000 NI
	150	+ 0b0001 IMP
	151	+EndEnum
	152	+Enum 19:16 MMapTrc
	153	+ 0b0000 NI
	154	+ 0b0001 IMP
	155	+EndEnum
	156	+Enum 15:12 CopTrc
	157	+ 0b0000 NI
	158	+ 0b0001 IMP
	159	+EndEnum
	160	+Enum 11:8 MMapDbg
	161	+ 0b0000 NI
	162	+ 0b0100 Armv7
	163	+ 0b0101 Armv7p1
	164	+EndEnum
	165	+Field 7:4 CopSDbg
	166	+Enum 3:0 CopDbg
	167	+ 0b0000 NI
	168	+ 0b0010 Armv6
	169	+ 0b0011 Armv6p1
	170	+ 0b0100 Armv7
	171	+ 0b0101 Armv7p1
	172	+ 0b0110 Armv8
	173	+ 0b0111 VHE
	174	+ 0b1000 Debugv8p2
	175	+ 0b1001 Debugv8p4
	176	+ 0b1010 Debugv8p8
	177	+EndEnum
	178	+EndSysreg
	179	+
	180	+Sysreg ID_AFR0_EL1 3 0 0 1 3
	181	+Res0 63:16
	182	+Field 15:12 IMPDEF3
	183	+Field 11:8 IMPDEF2
	184	+Field 7:4 IMPDEF1
	185	+Field 3:0 IMPDEF0
	186	+EndSysreg
	187	+
	188	+Sysreg ID_MMFR0_EL1 3 0 0 1 4
	189	+Res0 63:32
	190	+Enum 31:28 InnerShr
	191	+ 0b0000 NC
	192	+ 0b0001 HW
	193	+ 0b1111 IGNORED
	194	+EndEnum
	195	+Enum 27:24 FCSE
	196	+ 0b0000 NI
	197	+ 0b0001 IMP
	198	+EndEnum
	199	+Enum 23:20 AuxReg
	200	+ 0b0000 NI
	201	+ 0b0001 ACTLR
	202	+ 0b0010 AIFSR
	203	+EndEnum
	204	+Enum 19:16 TCM
	205	+ 0b0000 NI
	206	+ 0b0001 IMPDEF
	207	+ 0b0010 TCM
	208	+ 0b0011 TCM_DMA
	209	+EndEnum
	210	+Enum 15:12 ShareLvl
	211	+ 0b0000 ONE
	212	+ 0b0001 TWO
	213	+EndEnum
	214	+Enum 11:8 OuterShr
	215	+ 0b0000 NC
	216	+ 0b0001 HW
	217	+ 0b1111 IGNORED
	218	+EndEnum
	219	+Enum 7:4 PMSA
	220	+ 0b0000 NI
	221	+ 0b0001 IMPDEF
	222	+ 0b0010 PMSAv6
	223	+ 0b0011 PMSAv7
	224	+EndEnum
	225	+Enum 3:0 VMSA
	226	+ 0b0000 NI
	227	+ 0b0001 IMPDEF
	228	+ 0b0010 VMSAv6
	229	+ 0b0011 VMSAv7
	230	+ 0b0100 VMSAv7_PXN
	231	+ 0b0101 VMSAv7_LONG
	232	+EndEnum
	233	+EndSysreg
	234	+
	235	+Sysreg ID_MMFR1_EL1 3 0 0 1 5
	236	+Res0 63:32
	237	+Enum 31:28 BPred
	238	+ 0b0000 NI
	239	+ 0b0001 BP_SW_MANGED
	240	+ 0b0010 BP_ASID_AWARE
	241	+ 0b0011 BP_NOSNOOP
	242	+ 0b0100 BP_INVISIBLE
	243	+EndEnum
	244	+Enum 27:24 L1TstCln
	245	+ 0b0000 NI
	246	+ 0b0001 NOINVALIDATE
	247	+ 0b0010 INVALIDATE
	248	+EndEnum
	249	+Enum 23:20 L1Uni
	250	+ 0b0000 NI
	251	+ 0b0001 INVALIDATE
	252	+ 0b0010 CLEAN_AND_INVALIDATE
	253	+EndEnum
	254	+Enum 19:16 L1Hvd
	255	+ 0b0000 NI
	256	+ 0b0001 INVALIDATE_ISIDE_ONLY
	257	+ 0b0010 INVALIDATE
	258	+ 0b0011 CLEAN_AND_INVALIDATE
	259	+EndEnum
	260	+Enum 15:12 L1UniSW
	261	+ 0b0000 NI
	262	+ 0b0001 CLEAN
	263	+ 0b0010 CLEAN_AND_INVALIDATE
	264	+ 0b0011 INVALIDATE
	265	+EndEnum
	266	+Enum 11:8 L1HvdSW
	267	+ 0b0000 NI
	268	+ 0b0001 CLEAN_AND_INVALIDATE
	269	+ 0b0010 INVALIDATE_DSIDE_ONLY
	270	+ 0b0011 INVALIDATE
	271	+EndEnum
	272	+Enum 7:4 L1UniVA
	273	+ 0b0000 NI
	274	+ 0b0001 CLEAN_AND_INVALIDATE
	275	+ 0b0010 INVALIDATE_BP
	276	+EndEnum
	277	+Enum 3:0 L1HvdVA
	278	+ 0b0000 NI
	279	+ 0b0001 CLEAN_AND_INVALIDATE
	280	+ 0b0010 INVALIDATE_BP
	281	+EndEnum
	282	+EndSysreg
	283	+
	284	+Sysreg ID_MMFR2_EL1 3 0 0 1 6
	285	+Res0 63:32
	286	+Enum 31:28 HWAccFlg
	287	+ 0b0000 NI
	288	+ 0b0001 IMP
	289	+EndEnum
	290	+Enum 27:24 WFIStall
	291	+ 0b0000 NI
	292	+ 0b0001 IMP
	293	+EndEnum
	294	+Enum 23:20 MemBarr
	295	+ 0b0000 NI
	296	+ 0b0001 DSB_ONLY
	297	+ 0b0010 IMP
	298	+EndEnum
	299	+Enum 19:16 UniTLB
	300	+ 0b0000 NI
	301	+ 0b0001 BY_VA
	302	+ 0b0010 BY_MATCH_ASID
	303	+ 0b0011 BY_ALL_ASID
	304	+ 0b0100 OTHER_TLBS
	305	+ 0b0101 BROADCAST
	306	+ 0b0110 BY_IPA
	307	+EndEnum
	308	+Enum 15:12 HvdTLB
	309	+ 0b0000 NI
	310	+EndEnum
	311	+Enum 11:8 L1HvdRng
	312	+ 0b0000 NI
	313	+ 0b0001 IMP
	314	+EndEnum
	315	+Enum 7:4 L1HvdBG
	316	+ 0b0000 NI
	317	+ 0b0001 IMP
	318	+EndEnum
	319	+Enum 3:0 L1HvdFG
	320	+ 0b0000 NI
	321	+ 0b0001 IMP
	322	+EndEnum
	323	+EndSysreg
	324	+
	325	+Sysreg ID_MMFR3_EL1 3 0 0 1 7
	326	+Res0 63:32
	327	+Enum 31:28 Supersec
	328	+ 0b0000 IMP
	329	+ 0b1111 NI
	330	+EndEnum
	331	+Enum 27:24 CMemSz
	332	+ 0b0000 4GB
	333	+ 0b0001 64GB
	334	+ 0b0010 1TB
	335	+EndEnum
	336	+Enum 23:20 CohWalk
	337	+ 0b0000 NI
	338	+ 0b0001 IMP
	339	+EndEnum
	340	+Enum 19:16 PAN
	341	+ 0b0000 NI
	342	+ 0b0001 PAN
	343	+ 0b0010 PAN2
	344	+EndEnum
	345	+Enum 15:12 MaintBcst
	346	+ 0b0000 NI
	347	+ 0b0001 NO_TLB
	348	+ 0b0010 ALL
	349	+EndEnum
	350	+Enum 11:8 BPMaint
	351	+ 0b0000 NI
	352	+ 0b0001 ALL
	353	+ 0b0010 BY_VA
	354	+EndEnum
	355	+Enum 7:4 CMaintSW
	356	+ 0b0000 NI
	357	+ 0b0001 IMP
	358	+EndEnum
	359	+Enum 3:0 CMaintVA
	360	+ 0b0000 NI
	361	+ 0b0001 IMP
	362	+EndEnum
	363	+EndSysreg
	364	+
	365	+Sysreg ID_ISAR0_EL1 3 0 0 2 0
	366	+Res0 63:28
	367	+Enum 27:24 Divide
	368	+ 0b0000 NI
	369	+ 0b0001 xDIV_T32
	370	+ 0b0010 xDIV_A32
	371	+EndEnum
	372	+Enum 23:20 Debug
	373	+ 0b0000 NI
	374	+ 0b0001 IMP
	375	+EndEnum
	376	+Enum 19:16 Coproc
	377	+ 0b0000 NI
	378	+ 0b0001 MRC
	379	+ 0b0010 MRC2
	380	+ 0b0011 MRRC
	381	+ 0b0100 MRRC2
	382	+EndEnum
	383	+Enum 15:12 CmpBranch
	384	+ 0b0000 NI
	385	+ 0b0001 IMP
	386	+EndEnum
	387	+Enum 11:8 BitField
	388	+ 0b0000 NI
	389	+ 0b0001 IMP
	390	+EndEnum
	391	+Enum 7:4 BitCount
	392	+ 0b0000 NI
	393	+ 0b0001 IMP
	394	+EndEnum
	395	+Enum 3:0 Swap
	396	+ 0b0000 NI
	397	+ 0b0001 IMP
	398	+EndEnum
	399	+EndSysreg
	400	+
	401	+Sysreg ID_ISAR1_EL1 3 0 0 2 1
	402	+Res0 63:32
	403	+Enum 31:28 Jazelle
	404	+ 0b0000 NI
	405	+ 0b0001 IMP
	406	+EndEnum
	407	+Enum 27:24 Interwork
	408	+ 0b0000 NI
	409	+ 0b0001 BX
	410	+ 0b0010 BLX
	411	+ 0b0011 A32_BX
	412	+EndEnum
	413	+Enum 23:20 Immediate
	414	+ 0b0000 NI
	415	+ 0b0001 IMP
	416	+EndEnum
	417	+Enum 19:16 IfThen
	418	+ 0b0000 NI
	419	+ 0b0001 IMP
	420	+EndEnum
	421	+Enum 15:12 Extend
	422	+ 0b0000 NI
	423	+ 0b0001 SXTB
	424	+ 0b0010 SXTB16
	425	+EndEnum
	426	+Enum 11:8 Except_AR
	427	+ 0b0000 NI
	428	+ 0b0001 IMP
	429	+EndEnum
	430	+Enum 7:4 Except
	431	+ 0b0000 NI
	432	+ 0b0001 IMP
	433	+EndEnum
	434	+Enum 3:0 Endian
	435	+ 0b0000 NI
	436	+ 0b0001 IMP
	437	+EndEnum
	438	+EndSysreg
	439	+
	440	+Sysreg ID_ISAR2_EL1 3 0 0 2 2
	441	+Res0 63:32
	442	+Enum 31:28 Reversal
	443	+ 0b0000 NI
	444	+ 0b0001 REV
	445	+ 0b0010 RBIT
	446	+EndEnum
	447	+Enum 27:24 PSR_AR
	448	+ 0b0000 NI
	449	+ 0b0001 IMP
	450	+EndEnum
	451	+Enum 23:20 MultU
	452	+ 0b0000 NI
	453	+ 0b0001 UMULL
	454	+ 0b0010 UMAAL
	455	+EndEnum
	456	+Enum 19:16 MultS
	457	+ 0b0000 NI
	458	+ 0b0001 SMULL
	459	+ 0b0010 SMLABB
	460	+ 0b0011 SMLAD
	461	+EndEnum
	462	+Enum 15:12 Mult
	463	+ 0b0000 NI
	464	+ 0b0001 MLA
	465	+ 0b0010 MLS
	466	+EndEnum
	467	+Enum 11:8 MultiAccessInt
	468	+ 0b0000 NI
	469	+ 0b0001 RESTARTABLE
	470	+ 0b0010 CONTINUABLE
	471	+EndEnum
	472	+Enum 7:4 MemHint
	473	+ 0b0000 NI
	474	+ 0b0001 PLD
	475	+ 0b0010 PLD2
	476	+ 0b0011 PLI
	477	+ 0b0100 PLDW
	478	+EndEnum
	479	+Enum 3:0 LoadStore
	480	+ 0b0000 NI
	481	+ 0b0001 DOUBLE
	482	+ 0b0010 ACQUIRE
	483	+EndEnum
	484	+EndSysreg
	485	+
	486	+Sysreg ID_ISAR3_EL1 3 0 0 2 3
	487	+Res0 63:32
	488	+Enum 31:28 T32EE
	489	+ 0b0000 NI
	490	+ 0b0001 IMP
	491	+EndEnum
	492	+Enum 27:24 TrueNOP
	493	+ 0b0000 NI
	494	+ 0b0001 IMP
	495	+EndEnum
	496	+Enum 23:20 T32Copy
	497	+ 0b0000 NI
	498	+ 0b0001 IMP
	499	+EndEnum
	500	+Enum 19:16 TabBranch
	501	+ 0b0000 NI
	502	+ 0b0001 IMP
	503	+EndEnum
	504	+Enum 15:12 SynchPrim
	505	+ 0b0000 NI
	506	+ 0b0001 EXCLUSIVE
	507	+ 0b0010 DOUBLE
	508	+EndEnum
	509	+Enum 11:8 SVC
	510	+ 0b0000 NI
	511	+ 0b0001 IMP
	512	+EndEnum
	513	+Enum 7:4 SIMD
	514	+ 0b0000 NI
	515	+ 0b0001 SSAT
	516	+ 0b0011 PKHBT
	517	+EndEnum
	518	+Enum 3:0 Saturate
	519	+ 0b0000 NI
	520	+ 0b0001 IMP
	521	+EndEnum
	522	+EndSysreg
	523	+
	524	+Sysreg ID_ISAR4_EL1 3 0 0 2 4
	525	+Res0 63:32
	526	+Enum 31:28 SWP_frac
	527	+ 0b0000 NI
	528	+ 0b0001 IMP
	529	+EndEnum
	530	+Enum 27:24 PSR_M
	531	+ 0b0000 NI
	532	+ 0b0001 IMP
	533	+EndEnum
	534	+Enum 23:20 SynchPrim_frac
	535	+ 0b0000 NI
	536	+ 0b0011 IMP
	537	+EndEnum
	538	+Enum 19:16 Barrier
	539	+ 0b0000 NI
	540	+ 0b0001 IMP
	541	+EndEnum
	542	+Enum 15:12 SMC
	543	+ 0b0000 NI
	544	+ 0b0001 IMP
	545	+EndEnum
	546	+Enum 11:8 Writeback
	547	+ 0b0000 NI
	548	+ 0b0001 IMP
	549	+EndEnum
	550	+Enum 7:4 WithShifts
	551	+ 0b0000 NI
	552	+ 0b0001 LSL3
	553	+ 0b0011 LS
	554	+ 0b0100 REG
	555	+EndEnum
	556	+Enum 3:0 Unpriv
	557	+ 0b0000 NI
	558	+ 0b0001 REG_BYTE
	559	+ 0b0010 SIGNED_HALFWORD
	560	+EndEnum
	561	+EndSysreg
	562	+
	563	+Sysreg ID_ISAR5_EL1 3 0 0 2 5
	564	+Res0 63:32
	565	+Enum 31:28 VCMA
	566	+ 0b0000 NI
	567	+ 0b0001 IMP
	568	+EndEnum
	569	+Enum 27:24 RDM
	570	+ 0b0000 NI
	571	+ 0b0001 IMP
	572	+EndEnum
	573	+Res0 23:20
	574	+Enum 19:16 CRC32
	575	+ 0b0000 NI
	576	+ 0b0001 IMP
	577	+EndEnum
	578	+Enum 15:12 SHA2
	579	+ 0b0000 NI
	580	+ 0b0001 IMP
	581	+EndEnum
	582	+Enum 11:8 SHA1
	583	+ 0b0000 NI
	584	+ 0b0001 IMP
	585	+EndEnum
	586	+Enum 7:4 AES
	587	+ 0b0000 NI
	588	+ 0b0001 IMP
	589	+ 0b0010 VMULL
	590	+EndEnum
	591	+Enum 3:0 SEVL
	592	+ 0b0000 NI
	593	+ 0b0001 IMP
	594	+EndEnum
	595	+EndSysreg
	596	+
	597	+Sysreg ID_ISAR6_EL1 3 0 0 2 7
	598	+Res0 63:28
	599	+Enum 27:24 I8MM
	600	+ 0b0000 NI
	601	+ 0b0001 IMP
	602	+EndEnum
	603	+Enum 23:20 BF16
	604	+ 0b0000 NI
	605	+ 0b0001 IMP
	606	+EndEnum
	607	+Enum 19:16 SPECRES
	608	+ 0b0000 NI
	609	+ 0b0001 IMP
	610	+EndEnum
	611	+Enum 15:12 SB
	612	+ 0b0000 NI
	613	+ 0b0001 IMP
	614	+EndEnum
	615	+Enum 11:8 FHM
	616	+ 0b0000 NI
	617	+ 0b0001 IMP
	618	+EndEnum
	619	+Enum 7:4 DP
	620	+ 0b0000 NI
	621	+ 0b0001 IMP
	622	+EndEnum
	623	+Enum 3:0 JSCVT
	624	+ 0b0000 NI
	625	+ 0b0001 IMP
	626	+EndEnum
	627	+EndSysreg
	628	+
	629	+Sysreg ID_MMFR4_EL1 3 0 0 2 6
	630	+Res0 63:32
	631	+Enum 31:28 EVT
	632	+ 0b0000 NI
	633	+ 0b0001 NO_TLBIS
	634	+ 0b0010 TLBIS
	635	+EndEnum
	636	+Enum 27:24 CCIDX
	637	+ 0b0000 NI
	638	+ 0b0001 IMP
	639	+EndEnum
	640	+Enum 23:20 LSM
	641	+ 0b0000 NI
	642	+ 0b0001 IMP
	643	+EndEnum
	644	+Enum 19:16 HPDS
	645	+ 0b0000 NI
	646	+ 0b0001 AA32HPD
	647	+ 0b0010 HPDS2
	648	+EndEnum
	649	+Enum 15:12 CnP
	650	+ 0b0000 NI
	651	+ 0b0001 IMP
	652	+EndEnum
	653	+Enum 11:8 XNX
	654	+ 0b0000 NI
	655	+ 0b0001 IMP
	656	+EndEnum
	657	+Enum 7:4 AC2
	658	+ 0b0000 NI
	659	+ 0b0001 IMP
	660	+EndEnum
	661	+Enum 3:0 SpecSEI
	662	+ 0b0000 NI
	663	+ 0b0001 IMP
	664	+EndEnum
	665	+EndSysreg
	666	+
	667	+Sysreg MVFR0_EL1 3 0 0 3 0
	668	+Res0 63:32
	669	+Enum 31:28 FPRound
	670	+ 0b0000 NI
	671	+ 0b0001 IMP
	672	+EndEnum
	673	+Enum 27:24 FPShVec
	674	+ 0b0000 NI
	675	+ 0b0001 IMP
	676	+EndEnum
	677	+Enum 23:20 FPSqrt
	678	+ 0b0000 NI
	679	+ 0b0001 IMP
	680	+EndEnum
	681	+Enum 19:16 FPDivide
	682	+ 0b0000 NI
	683	+ 0b0001 IMP
	684	+EndEnum
	685	+Enum 15:12 FPTrap
	686	+ 0b0000 NI
	687	+ 0b0001 IMP
	688	+EndEnum
	689	+Enum 11:8 FPDP
	690	+ 0b0000 NI
	691	+ 0b0001 VFPv2
	692	+ 0b0001 VFPv3
	693	+EndEnum
	694	+Enum 7:4 FPSP
	695	+ 0b0000 NI
	696	+ 0b0001 VFPv2
	697	+ 0b0001 VFPv3
	698	+EndEnum
	699	+Enum 3:0 SIMDReg
	700	+ 0b0000 NI
	701	+ 0b0001 IMP_16x64
	702	+ 0b0001 IMP_32x64
	703	+EndEnum
	704	+EndSysreg
	705	+
	706	+Sysreg MVFR1_EL1 3 0 0 3 1
	707	+Res0 63:32
	708	+Enum 31:28 SIMDFMAC
	709	+ 0b0000 NI
	710	+ 0b0001 IMP
	711	+EndEnum
	712	+Enum 27:24 FPHP
	713	+ 0b0000 NI
	714	+ 0b0001 FPHP
	715	+ 0b0010 FPHP_CONV
	716	+ 0b0011 FP16
	717	+EndEnum
	718	+Enum 23:20 SIMDHP
	719	+ 0b0000 NI
	720	+ 0b0001 SIMDHP
	721	+ 0b0001 SIMDHP_FLOAT
	722	+EndEnum
	723	+Enum 19:16 SIMDSP
	724	+ 0b0000 NI
	725	+ 0b0001 IMP
	726	+EndEnum
	727	+Enum 15:12 SIMDInt
	728	+ 0b0000 NI
	729	+ 0b0001 IMP
	730	+EndEnum
	731	+Enum 11:8 SIMDLS
	732	+ 0b0000 NI
	733	+ 0b0001 IMP
	734	+EndEnum
	735	+Enum 7:4 FPDNaN
	736	+ 0b0000 NI
	737	+ 0b0001 IMP
	738	+EndEnum
	739	+Enum 3:0 FPFtZ
	740	+ 0b0000 NI
	741	+ 0b0001 IMP
	742	+EndEnum
	743	+EndSysreg
	744	+
	745	+Sysreg MVFR2_EL1 3 0 0 3 2
	746	+Res0 63:8
	747	+Enum 7:4 FPMisc
	748	+ 0b0000 NI
	749	+ 0b0001 FP
	750	+ 0b0010 FP_DIRECTED_ROUNDING
	751	+ 0b0011 FP_ROUNDING
	752	+ 0b0100 FP_MAX_MIN
	753	+EndEnum
	754	+Enum 3:0 SIMDMisc
	755	+ 0b0000 NI
	756	+ 0b0001 SIMD_DIRECTED_ROUNDING
	757	+ 0b0010 SIMD_ROUNDING
	758	+ 0b0011 SIMD_MAX_MIN
	759	+EndEnum
	760	+EndSysreg
	761	+
	762	+Sysreg ID_PFR2_EL1 3 0 0 3 4
	763	+Res0 63:12
	764	+Enum 11:8 RAS_frac
	765	+ 0b0000 NI
	766	+ 0b0001 RASv1p1
	767	+EndEnum
	768	+Enum 7:4 SSBS
	769	+ 0b0000 NI
	770	+ 0b0001 IMP
	771	+EndEnum
	772	+Enum 3:0 CSV3
	773	+ 0b0000 NI
	774	+ 0b0001 IMP
	775	+EndEnum
	776	+EndSysreg
	777	+
	778	+Sysreg ID_DFR1_EL1 3 0 0 3 5
	779	+Res0 63:8
	780	+Enum 7:4 HPMN0
	781	+ 0b0000 NI
	782	+ 0b0001 IMP
	783	+EndEnum
	784	+Enum 3:0 MTPMU
	785	+ 0b0000 IMPDEF
	786	+ 0b0001 IMP
	787	+ 0b1111 NI
	788	+EndEnum
	789	+EndSysreg
	790	+
	791	+Sysreg ID_MMFR5_EL1 3 0 0 3 6
	792	+Res0 63:8
	793	+Enum 7:4 nTLBPA
	794	+ 0b0000 NI
	795	+ 0b0001 IMP
	796	+EndEnum
	797	+Enum 3:0 ETS
	798	+ 0b0000 NI
	799	+ 0b0001 IMP
	800	+EndEnum
	801	+EndSysreg
	802	+
49	803	Sysreg ID_AA64PFR0_EL1 3 0 0 4 0
50	804	Enum 63:60 CSV3
51	805	0b0000 NI
@@ -210,6 +964,7 @@ EndEnum · 158 unmodified lines
210	964	Enum 3:0 SVEver
211	965	0b0000 IMP
212	966	0b0001 SVE2
	967	+ 0b0010 SVE2p1
213	968	EndEnum
214	969	EndSysreg
215	970
@@ -484,7 +1239,16 @@ EndEnum · 268 unmodified lines
484	1239	EndSysreg
485	1240
486	1241	Sysreg ID_AA64ISAR2_EL1 3 0 0 6 2
487		−Res0 63:28
	1242	+Res0 63:56
	1243	+Enum 55:52 CSSC
	1244	+ 0b0000 NI
	1245	+ 0b0001 IMP
	1246	+EndEnum
	1247	+Enum 51:48 RPRFM
	1248	+ 0b0000 NI
	1249	+ 0b0001 IMP
	1250	+EndEnum
	1251	+Res0 47:28
488	1252	Enum 27:24 PAC_frac
489	1253	0b0000 NI
490	1254	0b0001 IMP

arch/powerpc/include/asm/ftrace.h+22−2

@@ -37,12 +37,32 @@ static __always_inline struct pt_regs arch_ftrace_get_regs(struct ftrace_regs · 36 unmodified lines
37	37	return fregs->regs.msr ? &fregs->regs : NULL;
38	38	}
39	39
40		−static __always_inline void ftrace_instruction_pointer_set(struct ftrace_regs *fregs,
41		− unsigned long ip)
	40	+static __always_inline void
	41	+ftrace_regs_set_instruction_pointer(struct ftrace_regs *fregs,
	42	+ unsigned long ip)
42	43	{
43	44	regs_set_return_ip(&fregs->regs, ip);
44	45	}
45	46
	47	+static __always_inline unsigned long
	48	+ftrace_regs_get_instruction_pointer(struct ftrace_regs *fregs)
	49	+{
	50	+ return instruction_pointer(&fregs->regs);
	51	+}
	52	+
	53	+#define ftrace_regs_get_argument(fregs, n) \
	54	+ regs_get_kernel_argument(&(fregs)->regs, n)
	55	+#define ftrace_regs_get_stack_pointer(fregs) \
	56	+ kernel_stack_pointer(&(fregs)->regs)
	57	+#define ftrace_regs_return_value(fregs) \
	58	+ regs_return_value(&(fregs)->regs)
	59	+#define ftrace_regs_set_return_value(fregs, ret) \
	60	+ regs_set_return_value(&(fregs)->regs, ret)
	61	+#define ftrace_override_function_with_return(fregs) \
	62	+ override_function_with_return(&(fregs)->regs)
	63	+#define ftrace_regs_query_register_offset(name) \
	64	+ regs_query_register_offset(name)
	65	+
46	66	struct ftrace_ops;
47	67
48	68	#define ftrace_graph_func ftrace_graph_func

arch/s390/include/asm/ftrace.h+26−3

@@ -54,12 +54,33 @@ static __always_inline struct pt_regs arch_ftrace_get_regs(struct ftrace_regs · 53 unmodified lines
54	54	return NULL;
55	55	}
56	56
57		−static __always_inline void ftrace_instruction_pointer_set(struct ftrace_regs *fregs,
58		− unsigned long ip)
	57	+static __always_inline unsigned long
	58	+ftrace_regs_get_instruction_pointer(const struct ftrace_regs *fregs)
	59	+{
	60	+ return fregs->regs.psw.addr;
	61	+}
	62	+
	63	+static __always_inline void
	64	+ftrace_regs_set_instruction_pointer(struct ftrace_regs *fregs,
	65	+ unsigned long ip)
59	66	{
60	67	fregs->regs.psw.addr = ip;
61	68	}
62	69
	70	+#define ftrace_regs_get_argument(fregs, n) \
	71	+ regs_get_kernel_argument(&(fregs)->regs, n)
	72	+#define ftrace_regs_get_stack_pointer(fregs) \
	73	+ kernel_stack_pointer(&(fregs)->regs)
	74	+#define ftrace_regs_return_value(fregs) \
	75	+ regs_return_value(&(fregs)->regs)
	76	+#define ftrace_regs_set_return_value(fregs, ret) \
	77	+ regs_set_return_value(&(fregs)->regs, ret)
	78	+#define ftrace_override_function_with_return(fregs) \
	79	+ override_function_with_return(&(fregs)->regs)
	80	+#define ftrace_regs_query_register_offset(name) \
	81	+ regs_query_register_offset(name)
	82	+
	83	+#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
63	84	/*
64	85	* When an ftrace registered caller is tracing a function that is
65	86	* also set by a register_ftrace_direct() call, it needs to be
@@ -67,10 +88,12 @@ static __always_inline void ftrace_instruction_pointer_set(struct ftrace_regs *f · 1 unmodified line
67	88	* place the direct caller in the ORIG_GPR2 part of pt_regs. This
68	89	* tells the ftrace_caller that there's a direct caller.
69	90	*/
70		−static inline void arch_ftrace_set_direct_caller(struct pt_regs *regs, unsigned long addr)
	91	+static inline void arch_ftrace_set_direct_caller(struct ftrace_regs *fregs, unsigned long addr)
71	92	{
	93	+ struct pt_regs *regs = &fregs->regs;
72	94	regs->orig_gpr2 = addr;
73	95	}
	96	+#endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */
74	97
75	98	/*
76	99	* Even though the system call numbers are identical for s390/s390x a

arch/x86/include/asm/ftrace.h+35−14

@@ -34,19 +34,6 @@ static inline unsigned long ftrace_call_adjust(unsigned long addr) · 33 unmodified lines
34	34	return addr;
35	35	}
36	36
37		−/*
38		− * When a ftrace registered caller is tracing a function that is
39		− * also set by a register_ftrace_direct() call, it needs to be
40		− * differentiated in the ftrace_caller trampoline. To do this, we
41		− * place the direct caller in the ORIG_AX part of pt_regs. This
42		− * tells the ftrace_caller that there's a direct caller.
43		− */
44		−static inline void arch_ftrace_set_direct_caller(struct pt_regs *regs, unsigned long addr)
45		−{
46		− /* Emulate a call */
47		− regs->orig_ax = addr;
48		−}
49		−
50	37	#ifdef CONFIG_HAVE_DYNAMIC_FTRACE_WITH_ARGS
51	38	struct ftrace_regs {
52	39	struct pt_regs regs;
@@ -61,9 +48,25 @@ arch_ftrace_get_regs(struct ftrace_regs *fregs) · 8 unmodified lines
61	48	return &fregs->regs;
62	49	}
63	50
64		−#define ftrace_instruction_pointer_set(fregs, _ip) \
	51	+#define ftrace_regs_set_instruction_pointer(fregs, _ip) \
65	52	do { (fregs)->regs.ip = (_ip); } while (0)
66	53
	54	+#define ftrace_regs_get_instruction_pointer(fregs) \
	55	+ ((fregs)->regs.ip)
	56	+
	57	+#define ftrace_regs_get_argument(fregs, n) \
	58	+ regs_get_kernel_argument(&(fregs)->regs, n)
	59	+#define ftrace_regs_get_stack_pointer(fregs) \
	60	+ kernel_stack_pointer(&(fregs)->regs)
	61	+#define ftrace_regs_return_value(fregs) \
	62	+ regs_return_value(&(fregs)->regs)
	63	+#define ftrace_regs_set_return_value(fregs, ret) \
	64	+ regs_set_return_value(&(fregs)->regs, ret)
	65	+#define ftrace_override_function_with_return(fregs) \
	66	+ override_function_with_return(&(fregs)->regs)
	67	+#define ftrace_regs_query_register_offset(name) \
	68	+ regs_query_register_offset(name)
	69	+
67	70	struct ftrace_ops;
68	71	#define ftrace_graph_func ftrace_graph_func
69	72	void ftrace_graph_func(unsigned long ip, unsigned long parent_ip,
@@ -72,6 +75,24 @@ void ftrace_graph_func(unsigned long ip, unsigned long parent_ip, · 2 unmodified lines
72	75	#define FTRACE_GRAPH_TRAMP_ADDR FTRACE_GRAPH_ADDR
73	76	#endif
74	77
	78	+#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
	79	+/*
	80	+ * When a ftrace registered caller is tracing a function that is
	81	+ * also set by a register_ftrace_direct() call, it needs to be
	82	+ * differentiated in the ftrace_caller trampoline. To do this, we
	83	+ * place the direct caller in the ORIG_AX part of pt_regs. This
	84	+ * tells the ftrace_caller that there's a direct caller.
	85	+ */
	86	+static inline void
	87	+__arch_ftrace_set_direct_caller(struct pt_regs *regs, unsigned long addr)
	88	+{
	89	+ /* Emulate a call */
	90	+ regs->orig_ax = addr;
	91	+}
	92	+#define arch_ftrace_set_direct_caller(fregs, addr) \
	93	+ __arch_ftrace_set_direct_caller(&(fregs)->regs, addr)
	94	+#endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */
	95	+
75	96	#ifdef CONFIG_DYNAMIC_FTRACE
76	97
77	98	struct dyn_arch_ftrace {

drivers/acpi/Kconfig+1−1

@@ -90,7 +90,7 @@ config ACPI_SPCR_TABLE · 89 unmodified lines
90	90
91	91	config ACPI_FPDT
92	92	bool "ACPI Firmware Performance Data Table (FPDT) support"
93		− depends on X86_64
	93	+ depends on X86_64 \|\| ARM64
94	94	help
95	95	Enable support for the Firmware Performance Data Table (FPDT).
96	96	This table provides information on the timing of the system

drivers/acpi/arm64/Kconfig+3−0

@@ -18,3 +18,6 @@ config ACPI_AGDI · 17 unmodified lines
18	18	reset command.
19	19
20	20	If set, the kernel parses AGDI table and listens for the command.
	21	+
	22	+config ACPI_APMT
	23	+ bool

drivers/acpi/arm64/Makefile+1−0

@@ -2,4 +2,5 @@ · 1 unmodified line
2	2	obj-$(CONFIG_ACPI_AGDI) += agdi.o
3	3	obj-$(CONFIG_ACPI_IORT) += iort.o
4	4	obj-$(CONFIG_ACPI_GTDT) += gtdt.o
	5	+obj-$(CONFIG_ACPI_APMT) += apmt.o
5	6	obj-y += dma.o

drivers/acpi/arm64/apmt.c+178−0

@@ -0,0 +1,178 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+/*
	3	+ * ARM APMT table support.
	4	+ * Design document number: ARM DEN0117.
	5	+ *
	6	+ * Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES.
	7	+ *
	8	+ */
	9	+
	10	+#define pr_fmt(fmt) "ACPI: APMT: " fmt
	11	+
	12	+#include <linux/acpi.h>
	13	+#include <linux/acpi_apmt.h>
	14	+#include <linux/init.h>
	15	+#include <linux/kernel.h>
	16	+#include <linux/platform_device.h>
	17	+
	18	+#define DEV_NAME "arm-cs-arch-pmu"
	19	+
	20	+/* There can be up to 3 resources: page 0 and 1 address, and interrupt. */
	21	+#define DEV_MAX_RESOURCE_COUNT 3
	22	+
	23	+/* Root pointer to the mapped APMT table */
	24	+static struct acpi_table_header *apmt_table;
	25	+
	26	+static int __init apmt_init_resources(struct resource *res,
	27	+ struct acpi_apmt_node *node)
	28	+{
	29	+ int irq, trigger;
	30	+ int num_res = 0;
	31	+
	32	+ res[num_res].start = node->base_address0;
	33	+ res[num_res].end = node->base_address0 + SZ_4K - 1;
	34	+ res[num_res].flags = IORESOURCE_MEM;
	35	+
	36	+ num_res++;
	37	+
	38	+ res[num_res].start = node->base_address1;
	39	+ res[num_res].end = node->base_address1 + SZ_4K - 1;
	40	+ res[num_res].flags = IORESOURCE_MEM;
	41	+
	42	+ num_res++;
	43	+
	44	+ if (node->ovflw_irq != 0) {
	45	+ trigger = (node->ovflw_irq_flags & ACPI_APMT_OVFLW_IRQ_FLAGS_MODE);
	46	+ trigger = (trigger == ACPI_APMT_OVFLW_IRQ_FLAGS_MODE_LEVEL) ?
	47	+ ACPI_LEVEL_SENSITIVE : ACPI_EDGE_SENSITIVE;
	48	+ irq = acpi_register_gsi(NULL, node->ovflw_irq, trigger,
	49	+ ACPI_ACTIVE_HIGH);
	50	+
	51	+ if (irq <= 0) {
	52	+ pr_warn("APMT could not register gsi hwirq %d\n", irq);
	53	+ return num_res;
	54	+ }
	55	+
	56	+ res[num_res].start = irq;
	57	+ res[num_res].end = irq;
	58	+ res[num_res].flags = IORESOURCE_IRQ;
	59	+
	60	+ num_res++;
	61	+ }
	62	+
	63	+ return num_res;
	64	+}
	65	+
	66	+/**
	67	+ * apmt_add_platform_device() - Allocate a platform device for APMT node
	68	+ * @node: Pointer to device ACPI APMT node
	69	+ * @fwnode: fwnode associated with the APMT node
	70	+ *
	71	+ * Returns: 0 on success, <0 failure
	72	+ */
	73	+static int __init apmt_add_platform_device(struct acpi_apmt_node *node,
	74	+ struct fwnode_handle *fwnode)
	75	+{
	76	+ struct platform_device *pdev;
	77	+ int ret, count;
	78	+ struct resource res[DEV_MAX_RESOURCE_COUNT];
	79	+
	80	+ pdev = platform_device_alloc(DEV_NAME, PLATFORM_DEVID_AUTO);
	81	+ if (!pdev)
	82	+ return -ENOMEM;
	83	+
	84	+ memset(res, 0, sizeof(res));
	85	+
	86	+ count = apmt_init_resources(res, node);
	87	+
	88	+ ret = platform_device_add_resources(pdev, res, count);
	89	+ if (ret)
	90	+ goto dev_put;
	91	+
	92	+ /*
	93	+ * Add a copy of APMT node pointer to platform_data to be used to
	94	+ * retrieve APMT data information.
	95	+ */
	96	+ ret = platform_device_add_data(pdev, &node, sizeof(node));
	97	+ if (ret)
	98	+ goto dev_put;
	99	+
	100	+ pdev->dev.fwnode = fwnode;
	101	+
	102	+ ret = platform_device_add(pdev);
	103	+
	104	+ if (ret)
	105	+ goto dev_put;
	106	+
	107	+ return 0;
	108	+
	109	+dev_put:
	110	+ platform_device_put(pdev);
	111	+
	112	+ return ret;
	113	+}
	114	+
	115	+static int __init apmt_init_platform_devices(void)
	116	+{
	117	+ struct acpi_apmt_node *apmt_node;
	118	+ struct acpi_table_apmt *apmt;
	119	+ struct fwnode_handle *fwnode;
	120	+ u64 offset, end;
	121	+ int ret;
	122	+
	123	+ /*
	124	+ * apmt_table and apmt both point to the start of APMT table, but
	125	+ * have different struct types
	126	+ */
	127	+ apmt = (struct acpi_table_apmt *)apmt_table;
	128	+ offset = sizeof(*apmt);
	129	+ end = apmt->header.length;
	130	+
	131	+ while (offset < end) {
	132	+ apmt_node = ACPI_ADD_PTR(struct acpi_apmt_node, apmt,
	133	+ offset);
	134	+
	135	+ fwnode = acpi_alloc_fwnode_static();
	136	+ if (!fwnode)
	137	+ return -ENOMEM;
	138	+
	139	+ ret = apmt_add_platform_device(apmt_node, fwnode);
	140	+ if (ret) {
	141	+ acpi_free_fwnode_static(fwnode);
	142	+ return ret;
	143	+ }
	144	+
	145	+ offset += apmt_node->length;
	146	+ }
	147	+
	148	+ return 0;
	149	+}
	150	+
	151	+void __init acpi_apmt_init(void)
	152	+{
	153	+ acpi_status status;
	154	+ int ret;
	155	+
	156	+ /**
	157	+ * APMT table nodes will be used at runtime after the apmt init,
	158	+ * so we don't need to call acpi_put_table() to release
	159	+ * the APMT table mapping.
	160	+ */
	161	+ status = acpi_get_table(ACPI_SIG_APMT, 0, &apmt_table);
	162	+
	163	+ if (ACPI_FAILURE(status)) {
	164	+ if (status != AE_NOT_FOUND) {
	165	+ const char *msg = acpi_format_exception(status);
	166	+
	167	+ pr_err("Failed to get APMT table, %s\n", msg);
	168	+ }
	169	+
	170	+ return;
	171	+ }
	172	+
	173	+ ret = apmt_init_platform_devices();
	174	+ if (ret) {
	175	+ pr_err("Failed to initialize APMT platform devices, ret: %d\n", ret);
	176	+ acpi_put_table(apmt_table);
	177	+ }
	178	+}

drivers/acpi/arm64/iort.c+12−4

@@ -402,6 +402,10 @@ static struct acpi_iort_node iort_node_get_id(struct acpi_iort_node node, · 401 unmodified lines
402	402	return NULL;
403	403	}
404	404
	405	+#ifndef ACPI_IORT_SMMU_V3_DEVICEID_VALID
	406	+#define ACPI_IORT_SMMU_V3_DEVICEID_VALID (1 << 4)
	407	+#endif
	408	+
405	409	static int iort_get_id_mapping_index(struct acpi_iort_node *node)
406	410	{
407	411	struct acpi_iort_smmu_v3 *smmu;
@@ -418,12 +422,16 @@ static int iort_get_id_mapping_index(struct acpi_iort_node *node) · 10 unmodified lines
418	422
419	423	smmu = (struct acpi_iort_smmu_v3 *)node->node_data;
420	424	/*
421		− * ID mapping index is only ignored if all interrupts are
422		− * GSIV based
	425	+ * Until IORT E.e (node rev. 5), the ID mapping index was
	426	+ * defined to be valid unless all interrupts are GSIV-based.
423	427	*/
424		− if (smmu->event_gsiv && smmu->pri_gsiv && smmu->gerr_gsiv
425		− && smmu->sync_gsiv)
	428	+ if (node->revision < 5) {
	429	+ if (smmu->event_gsiv && smmu->pri_gsiv &&
	430	+ smmu->gerr_gsiv && smmu->sync_gsiv)
	431	+ return -EINVAL;
	432	+ } else if (!(smmu->flags & ACPI_IORT_SMMU_V3_DEVICEID_VALID)) {
426	433	return -EINVAL;
	434	+ }
427	435
428	436	if (smmu->id_mapping_index >= node->mapping_count) {
429	437	pr_err(FW_BUG "[node %p type %d] ID mapping index overflows valid mappings\n",

drivers/acpi/bus.c+2−0

@@ -27,6 +27,7 @@ · 26 unmodified lines
27	27	#include <linux/dmi.h>
28	28	#endif
29	29	#include <linux/acpi_agdi.h>
	30	+#include <linux/acpi_apmt.h>
30	31	#include <linux/acpi_iort.h>
31	32	#include <linux/acpi_viot.h>
32	33	#include <linux/pci.h>
@@ -1423,6 +1424,7 @@ static int __init acpi_init(void) · 1390 unmodified lines
1423	1424	acpi_setup_sb_notify_handler();
1424	1425	acpi_viot_init();
1425	1426	acpi_agdi_init();
	1427	+ acpi_apmt_init();
1426	1428	return 0;
1427	1429	}
1428	1430

drivers/firmware/arm_ffa/driver.c+10−91

@@ -36,81 +36,6 @@ · 35 unmodified lines
36	36	#include "common.h"
37	37
38	38	#define FFA_DRIVER_VERSION FFA_VERSION_1_0
39		−
40		−#define FFA_SMC(calling_convention, func_num) \
41		− ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, (calling_convention), \
42		− ARM_SMCCC_OWNER_STANDARD, (func_num))
43		−
44		−#define FFA_SMC_32(func_num) FFA_SMC(ARM_SMCCC_SMC_32, (func_num))
45		−#define FFA_SMC_64(func_num) FFA_SMC(ARM_SMCCC_SMC_64, (func_num))
46		−
47		−#define FFA_ERROR FFA_SMC_32(0x60)
48		−#define FFA_SUCCESS FFA_SMC_32(0x61)
49		−#define FFA_INTERRUPT FFA_SMC_32(0x62)
50		−#define FFA_VERSION FFA_SMC_32(0x63)
51		−#define FFA_FEATURES FFA_SMC_32(0x64)
52		−#define FFA_RX_RELEASE FFA_SMC_32(0x65)
53		−#define FFA_RXTX_MAP FFA_SMC_32(0x66)
54		−#define FFA_FN64_RXTX_MAP FFA_SMC_64(0x66)
55		−#define FFA_RXTX_UNMAP FFA_SMC_32(0x67)
56		−#define FFA_PARTITION_INFO_GET FFA_SMC_32(0x68)
57		−#define FFA_ID_GET FFA_SMC_32(0x69)
58		−#define FFA_MSG_POLL FFA_SMC_32(0x6A)
59		−#define FFA_MSG_WAIT FFA_SMC_32(0x6B)
60		−#define FFA_YIELD FFA_SMC_32(0x6C)
61		−#define FFA_RUN FFA_SMC_32(0x6D)
62		−#define FFA_MSG_SEND FFA_SMC_32(0x6E)
63		−#define FFA_MSG_SEND_DIRECT_REQ FFA_SMC_32(0x6F)
64		−#define FFA_FN64_MSG_SEND_DIRECT_REQ FFA_SMC_64(0x6F)
65		−#define FFA_MSG_SEND_DIRECT_RESP FFA_SMC_32(0x70)
66		−#define FFA_FN64_MSG_SEND_DIRECT_RESP FFA_SMC_64(0x70)
67		−#define FFA_MEM_DONATE FFA_SMC_32(0x71)
68		−#define FFA_FN64_MEM_DONATE FFA_SMC_64(0x71)
69		−#define FFA_MEM_LEND FFA_SMC_32(0x72)
70		−#define FFA_FN64_MEM_LEND FFA_SMC_64(0x72)
71		−#define FFA_MEM_SHARE FFA_SMC_32(0x73)
72		−#define FFA_FN64_MEM_SHARE FFA_SMC_64(0x73)
73		−#define FFA_MEM_RETRIEVE_REQ FFA_SMC_32(0x74)
74		−#define FFA_FN64_MEM_RETRIEVE_REQ FFA_SMC_64(0x74)
75		−#define FFA_MEM_RETRIEVE_RESP FFA_SMC_32(0x75)
76		−#define FFA_MEM_RELINQUISH FFA_SMC_32(0x76)
77		−#define FFA_MEM_RECLAIM FFA_SMC_32(0x77)
78		−#define FFA_MEM_OP_PAUSE FFA_SMC_32(0x78)
79		−#define FFA_MEM_OP_RESUME FFA_SMC_32(0x79)
80		−#define FFA_MEM_FRAG_RX FFA_SMC_32(0x7A)
81		−#define FFA_MEM_FRAG_TX FFA_SMC_32(0x7B)
82		−#define FFA_NORMAL_WORLD_RESUME FFA_SMC_32(0x7C)
83		−
84		−/*
85		− * For some calls it is necessary to use SMC64 to pass or return 64-bit values.
86		− * For such calls FFA_FN_NATIVE(name) will choose the appropriate
87		− * (native-width) function ID.
88		− */
89		−#ifdef CONFIG_64BIT
90		−#define FFA_FN_NATIVE(name) FFA_FN64_##name
91		−#else
92		−#define FFA_FN_NATIVE(name) FFA_##name
93		−#endif
94		−
95		−/* FFA error codes. */
96		−#define FFA_RET_SUCCESS (0)
97		−#define FFA_RET_NOT_SUPPORTED (-1)
98		−#define FFA_RET_INVALID_PARAMETERS (-2)
99		−#define FFA_RET_NO_MEMORY (-3)
100		−#define FFA_RET_BUSY (-4)
101		−#define FFA_RET_INTERRUPTED (-5)
102		−#define FFA_RET_DENIED (-6)
103		−#define FFA_RET_RETRY (-7)
104		−#define FFA_RET_ABORTED (-8)
105		−
106		−#define MAJOR_VERSION_MASK GENMASK(30, 16)
107		−#define MINOR_VERSION_MASK GENMASK(15, 0)
108		−#define MAJOR_VERSION(x) ((u16)(FIELD_GET(MAJOR_VERSION_MASK, (x))))
109		−#define MINOR_VERSION(x) ((u16)(FIELD_GET(MINOR_VERSION_MASK, (x))))
110		−#define PACK_VERSION_INFO(major, minor) \
111		− (FIELD_PREP(MAJOR_VERSION_MASK, (major)) \| \
112		− FIELD_PREP(MINOR_VERSION_MASK, (minor)))
113		−#define FFA_VERSION_1_0 PACK_VERSION_INFO(1, 0)
114	39	#define FFA_MIN_VERSION FFA_VERSION_1_0
115	40
116	41	#define SENDER_ID_MASK GENMASK(31, 16)
@@ -120,12 +45,6 @@ · 3 unmodified lines
120	45	#define PACK_TARGET_INFO(s, r) \
121	46	(FIELD_PREP(SENDER_ID_MASK, (s)) \| FIELD_PREP(RECEIVER_ID_MASK, (r)))
122	47
123		−/*
124		− * FF-A specification mentions explicitly about '4K pages'. This should
125		− * not be confused with the kernel PAGE_SIZE, which is the translation
126		− * granule kernel is configured and may be one among 4K, 16K and 64K.
127		− */
128		−#define FFA_PAGE_SIZE SZ_4K
129	48	/*
130	49	* Keeping RX TX buffer size as 4K for now
131	50	* 64K may be preferred to keep it min a page in 64K PAGE_SIZE config
@@ -178,9 +97,9 @@ static struct ffa_drv_info *drv_info; · 46 unmodified lines
178	97	*/
179	98	static u32 ffa_compatible_version_find(u32 version)
180	99	{
181		− u16 major = MAJOR_VERSION(version), minor = MINOR_VERSION(version);
182		− u16 drv_major = MAJOR_VERSION(FFA_DRIVER_VERSION);
183		− u16 drv_minor = MINOR_VERSION(FFA_DRIVER_VERSION);
	100	+ u16 major = FFA_MAJOR_VERSION(version), minor = FFA_MINOR_VERSION(version);
	101	+ u16 drv_major = FFA_MAJOR_VERSION(FFA_DRIVER_VERSION);
	102	+ u16 drv_minor = FFA_MINOR_VERSION(FFA_DRIVER_VERSION);
184	103
185	104	if ((major < drv_major) \|\| (major == drv_major && minor <= drv_minor))
186	105	return version;
@@ -204,16 +123,16 @@ static int ffa_version_check(u32 *version) · 17 unmodified lines
204	123
205	124	if (ver.a0 < FFA_MIN_VERSION) {
206	125	pr_err("Incompatible v%d.%d! Earliest supported v%d.%d\n",
207		− MAJOR_VERSION(ver.a0), MINOR_VERSION(ver.a0),
208		− MAJOR_VERSION(FFA_MIN_VERSION),
209		− MINOR_VERSION(FFA_MIN_VERSION));
	126	+ FFA_MAJOR_VERSION(ver.a0), FFA_MINOR_VERSION(ver.a0),
	127	+ FFA_MAJOR_VERSION(FFA_MIN_VERSION),
	128	+ FFA_MINOR_VERSION(FFA_MIN_VERSION));
210	129	return -EINVAL;
211	130	}
212	131
213		− pr_info("Driver version %d.%d\n", MAJOR_VERSION(FFA_DRIVER_VERSION),
214		− MINOR_VERSION(FFA_DRIVER_VERSION));
215		− pr_info("Firmware version %d.%d found\n", MAJOR_VERSION(ver.a0),
216		− MINOR_VERSION(ver.a0));
	132	+ pr_info("Driver version %d.%d\n", FFA_MAJOR_VERSION(FFA_DRIVER_VERSION),
	133	+ FFA_MINOR_VERSION(FFA_DRIVER_VERSION));
	134	+ pr_info("Firmware version %d.%d found\n", FFA_MAJOR_VERSION(ver.a0),
	135	+ FFA_MINOR_VERSION(ver.a0));
217	136	*version = ffa_compatible_version_find(ver.a0);
218	137
219	138	return 0;

drivers/firmware/efi/libstub/Makefile+1−0

@@ -20,6 +20,7 @@ cflags-$(CONFIG_X86) += -m$(BITS) -D__KERNEL__ \ · 19 unmodified lines
20	20	# disable the stackleak plugin
21	21	cflags-$(CONFIG_ARM64) := $(subst $(CC_FLAGS_FTRACE),,$(KBUILD_CFLAGS)) \
22	22	-fpie $(DISABLE_STACKLEAK_PLUGIN) \
	23	+ -fno-unwind-tables -fno-asynchronous-unwind-tables \
23	24	$(call cc-option,-mbranch-protection=none)
24	25	cflags-$(CONFIG_ARM) := $(subst $(CC_FLAGS_FTRACE),,$(KBUILD_CFLAGS)) \
25	26	-fno-builtin -fpic \

drivers/perf/Kconfig+4−0

@@ -199,4 +199,8 @@ config MARVELL_CN10K_DDR_PMU · 198 unmodified lines
199	199	Enable perf support for Marvell DDR Performance monitoring
200	200	event on CN10K platform.
201	201
	202	+source "drivers/perf/arm_cspmu/Kconfig"
	203	+
	204	+source "drivers/perf/amlogic/Kconfig"
	205	+
202	206	endmenu

drivers/perf/Makefile+2−0

@@ -21,3 +21,5 @@ obj-$(CONFIG_MARVELL_CN10K_TAD_PMU) += marvell_cn10k_tad_pmu.o · 20 unmodified lines
21	21	obj-$(CONFIG_MARVELL_CN10K_DDR_PMU) += marvell_cn10k_ddr_pmu.o
22	22	obj-$(CONFIG_APPLE_M1_CPU_PMU) += apple_m1_cpu_pmu.o
23	23	obj-$(CONFIG_ALIBABA_UNCORE_DRW_PMU) += alibaba_uncore_drw_pmu.o
	24	+obj-$(CONFIG_ARM_CORESIGHT_PMU_ARCH_SYSTEM_PMU) += arm_cspmu/
	25	+obj-$(CONFIG_MESON_DDR_PMU) += amlogic/

drivers/perf/amlogic/Kconfig+10−0

@@ -0,0 +1,10 @@
	1	+# SPDX-License-Identifier: GPL-2.0-only
	2	+config MESON_DDR_PMU
	3	+ tristate "Amlogic DDR Bandwidth Performance Monitor"
	4	+ depends on ARCH_MESON \|\| COMPILE_TEST
	5	+ help
	6	+ Provides support for the DDR performance monitor
	7	+ in Amlogic SoCs, which can give information about
	8	+ memory throughput and other related events. It
	9	+ supports multiple channels to monitor the memory
	10	+ bandwidth simultaneously.

drivers/perf/amlogic/Makefile+5−0

@@ -0,0 +1,5 @@
	1	+# SPDX-License-Identifier: GPL-2.0-only
	2	+
	3	+obj-$(CONFIG_MESON_DDR_PMU) += meson_ddr_pmu_g12.o
	4	+
	5	+meson_ddr_pmu_g12-y := meson_ddr_pmu_core.o meson_g12_ddr_pmu.o

drivers/perf/amlogic/meson_ddr_pmu_core.c+561−0

@@ -0,0 +1,561 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+/*
	3	+ * Copyright (c) 2022 Amlogic, Inc. All rights reserved.
	4	+ */
	5	+
	6	+#include <linux/bitfield.h>
	7	+#include <linux/init.h>
	8	+#include <linux/irqreturn.h>
	9	+#include <linux/kernel.h>
	10	+#include <linux/module.h>
	11	+#include <linux/of.h>
	12	+#include <linux/of_device.h>
	13	+#include <linux/of_irq.h>
	14	+#include <linux/perf_event.h>
	15	+#include <linux/platform_device.h>
	16	+#include <linux/printk.h>
	17	+#include <linux/sysfs.h>
	18	+#include <linux/types.h>
	19	+
	20	+#include <soc/amlogic/meson_ddr_pmu.h>
	21	+
	22	+struct ddr_pmu {
	23	+ struct pmu pmu;
	24	+ struct dmc_info info;
	25	+ struct dmc_counter counters; /* save counters from hw */
	26	+ bool pmu_enabled;
	27	+ struct device *dev;
	28	+ char *name;
	29	+ struct hlist_node node;
	30	+ enum cpuhp_state cpuhp_state;
	31	+ int cpu; /* for cpu hotplug */
	32	+};
	33	+
	34	+#define DDR_PERF_DEV_NAME "meson_ddr_bw"
	35	+#define MAX_AXI_PORTS_OF_CHANNEL 4 /* A DMC channel can monitor max 4 axi ports */
	36	+
	37	+#define to_ddr_pmu(p) container_of(p, struct ddr_pmu, pmu)
	38	+#define dmc_info_to_pmu(p) container_of(p, struct ddr_pmu, info)
	39	+
	40	+static void dmc_pmu_enable(struct ddr_pmu *pmu)
	41	+{
	42	+ if (!pmu->pmu_enabled)
	43	+ pmu->info.hw_info->enable(&pmu->info);
	44	+
	45	+ pmu->pmu_enabled = true;
	46	+}
	47	+
	48	+static void dmc_pmu_disable(struct ddr_pmu *pmu)
	49	+{
	50	+ if (pmu->pmu_enabled)
	51	+ pmu->info.hw_info->disable(&pmu->info);
	52	+
	53	+ pmu->pmu_enabled = false;
	54	+}
	55	+
	56	+static void meson_ddr_set_axi_filter(struct perf_event *event, u8 axi_id)
	57	+{
	58	+ struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);
	59	+ int chann;
	60	+
	61	+ if (event->attr.config > ALL_CHAN_COUNTER_ID &&
	62	+ event->attr.config < COUNTER_MAX_ID) {
	63	+ chann = event->attr.config - CHAN1_COUNTER_ID;
	64	+
	65	+ pmu->info.hw_info->set_axi_filter(&pmu->info, axi_id, chann);
	66	+ }
	67	+}
	68	+
	69	+static void ddr_cnt_addition(struct dmc_counter *sum,
	70	+ struct dmc_counter *add1,
	71	+ struct dmc_counter *add2,
	72	+ int chann_nr)
	73	+{
	74	+ int i;
	75	+ u64 cnt1, cnt2;
	76	+
	77	+ sum->all_cnt = add1->all_cnt + add2->all_cnt;
	78	+ sum->all_req = add1->all_req + add2->all_req;
	79	+ for (i = 0; i < chann_nr; i++) {
	80	+ cnt1 = add1->channel_cnt[i];
	81	+ cnt2 = add2->channel_cnt[i];
	82	+
	83	+ sum->channel_cnt[i] = cnt1 + cnt2;
	84	+ }
	85	+}
	86	+
	87	+static void meson_ddr_perf_event_update(struct perf_event *event)
	88	+{
	89	+ struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);
	90	+ u64 new_raw_count = 0;
	91	+ struct dmc_counter dc = {0}, sum_dc = {0};
	92	+ int idx;
	93	+ int chann_nr = pmu->info.hw_info->chann_nr;
	94	+
	95	+ /* get the remain counters in register. */
	96	+ pmu->info.hw_info->get_counters(&pmu->info, &dc);
	97	+
	98	+ ddr_cnt_addition(&sum_dc, &pmu->counters, &dc, chann_nr);
	99	+
	100	+ switch (event->attr.config) {
	101	+ case ALL_CHAN_COUNTER_ID:
	102	+ new_raw_count = sum_dc.all_cnt;
	103	+ break;
	104	+ case CHAN1_COUNTER_ID:
	105	+ case CHAN2_COUNTER_ID:
	106	+ case CHAN3_COUNTER_ID:
	107	+ case CHAN4_COUNTER_ID:
	108	+ case CHAN5_COUNTER_ID:
	109	+ case CHAN6_COUNTER_ID:
	110	+ case CHAN7_COUNTER_ID:
	111	+ case CHAN8_COUNTER_ID:
	112	+ idx = event->attr.config - CHAN1_COUNTER_ID;
	113	+ new_raw_count = sum_dc.channel_cnt[idx];
	114	+ break;
	115	+ }
	116	+
	117	+ local64_set(&event->count, new_raw_count);
	118	+}
	119	+
	120	+static int meson_ddr_perf_event_init(struct perf_event *event)
	121	+{
	122	+ struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);
	123	+ u64 config1 = event->attr.config1;
	124	+ u64 config2 = event->attr.config2;
	125	+
	126	+ if (event->attr.type != event->pmu->type)
	127	+ return -ENOENT;
	128	+
	129	+ if (is_sampling_event(event) \|\| event->attach_state & PERF_ATTACH_TASK)
	130	+ return -EOPNOTSUPP;
	131	+
	132	+ if (event->cpu < 0)
	133	+ return -EOPNOTSUPP;
	134	+
	135	+ /* check if the number of parameters is too much */
	136	+ if (event->attr.config != ALL_CHAN_COUNTER_ID &&
	137	+ hweight64(config1) + hweight64(config2) > MAX_AXI_PORTS_OF_CHANNEL)
	138	+ return -EOPNOTSUPP;
	139	+
	140	+ event->cpu = pmu->cpu;
	141	+
	142	+ return 0;
	143	+}
	144	+
	145	+static void meson_ddr_perf_event_start(struct perf_event *event, int flags)
	146	+{
	147	+ struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);
	148	+
	149	+ memset(&pmu->counters, 0, sizeof(pmu->counters));
	150	+ dmc_pmu_enable(pmu);
	151	+}
	152	+
	153	+static int meson_ddr_perf_event_add(struct perf_event *event, int flags)
	154	+{
	155	+ u64 config1 = event->attr.config1;
	156	+ u64 config2 = event->attr.config2;
	157	+ int i;
	158	+
	159	+ for_each_set_bit(i, (const unsigned long *)&config1, sizeof(config1))
	160	+ meson_ddr_set_axi_filter(event, i);
	161	+
	162	+ for_each_set_bit(i, (const unsigned long *)&config2, sizeof(config2))
	163	+ meson_ddr_set_axi_filter(event, i + 64);
	164	+
	165	+ if (flags & PERF_EF_START)
	166	+ meson_ddr_perf_event_start(event, flags);
	167	+
	168	+ return 0;
	169	+}
	170	+
	171	+static void meson_ddr_perf_event_stop(struct perf_event *event, int flags)
	172	+{
	173	+ struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);
	174	+
	175	+ if (flags & PERF_EF_UPDATE)
	176	+ meson_ddr_perf_event_update(event);
	177	+
	178	+ dmc_pmu_disable(pmu);
	179	+}
	180	+
	181	+static void meson_ddr_perf_event_del(struct perf_event *event, int flags)
	182	+{
	183	+ meson_ddr_perf_event_stop(event, PERF_EF_UPDATE);
	184	+}
	185	+
	186	+static ssize_t meson_ddr_perf_cpumask_show(struct device *dev,
	187	+ struct device_attribute *attr,
	188	+ char *buf)
	189	+{
	190	+ struct ddr_pmu *pmu = dev_get_drvdata(dev);
	191	+
	192	+ return cpumap_print_to_pagebuf(true, buf, cpumask_of(pmu->cpu));
	193	+}
	194	+
	195	+static struct device_attribute meson_ddr_perf_cpumask_attr =
	196	+__ATTR(cpumask, 0444, meson_ddr_perf_cpumask_show, NULL);
	197	+
	198	+static struct attribute *meson_ddr_perf_cpumask_attrs[] = {
	199	+ &meson_ddr_perf_cpumask_attr.attr,
	200	+ NULL,
	201	+};
	202	+
	203	+static const struct attribute_group ddr_perf_cpumask_attr_group = {
	204	+ .attrs = meson_ddr_perf_cpumask_attrs,
	205	+};
	206	+
	207	+static ssize_t
	208	+pmu_event_show(struct device dev, struct device_attribute attr,
	209	+ char *page)
	210	+{
	211	+ struct perf_pmu_events_attr *pmu_attr;
	212	+
	213	+ pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr);
	214	+ return sysfs_emit(page, "event=0x%02llx\n", pmu_attr->id);
	215	+}
	216	+
	217	+static ssize_t
	218	+event_show_unit(struct device dev, struct device_attribute attr,
	219	+ char *page)
	220	+{
	221	+ return sysfs_emit(page, "MB\n");
	222	+}
	223	+
	224	+static ssize_t
	225	+event_show_scale(struct device dev, struct device_attribute attr,
	226	+ char *page)
	227	+{
	228	+ /* one count = 16byte = 1.52587890625e-05 MB */
	229	+ return sysfs_emit(page, "1.52587890625e-05\n");
	230	+}
	231	+
	232	+#define AML_DDR_PMU_EVENT_ATTR(_name, _id) \
	233	+{ \
	234	+ .attr = __ATTR(_name, 0444, pmu_event_show, NULL), \
	235	+ .id = _id, \
	236	+}
	237	+
	238	+#define AML_DDR_PMU_EVENT_UNIT_ATTR(_name) \
	239	+ __ATTR(_name.unit, 0444, event_show_unit, NULL)
	240	+
	241	+#define AML_DDR_PMU_EVENT_SCALE_ATTR(_name) \
	242	+ __ATTR(_name.scale, 0444, event_show_scale, NULL)
	243	+
	244	+static struct device_attribute event_unit_attrs[] = {
	245	+ AML_DDR_PMU_EVENT_UNIT_ATTR(total_rw_bytes),
	246	+ AML_DDR_PMU_EVENT_UNIT_ATTR(chan_1_rw_bytes),
	247	+ AML_DDR_PMU_EVENT_UNIT_ATTR(chan_2_rw_bytes),
	248	+ AML_DDR_PMU_EVENT_UNIT_ATTR(chan_3_rw_bytes),
	249	+ AML_DDR_PMU_EVENT_UNIT_ATTR(chan_4_rw_bytes),
	250	+ AML_DDR_PMU_EVENT_UNIT_ATTR(chan_5_rw_bytes),
	251	+ AML_DDR_PMU_EVENT_UNIT_ATTR(chan_6_rw_bytes),
	252	+ AML_DDR_PMU_EVENT_UNIT_ATTR(chan_7_rw_bytes),
	253	+ AML_DDR_PMU_EVENT_UNIT_ATTR(chan_8_rw_bytes),
	254	+};
	255	+
	256	+static struct device_attribute event_scale_attrs[] = {
	257	+ AML_DDR_PMU_EVENT_SCALE_ATTR(total_rw_bytes),
	258	+ AML_DDR_PMU_EVENT_SCALE_ATTR(chan_1_rw_bytes),
	259	+ AML_DDR_PMU_EVENT_SCALE_ATTR(chan_2_rw_bytes),
	260	+ AML_DDR_PMU_EVENT_SCALE_ATTR(chan_3_rw_bytes),
	261	+ AML_DDR_PMU_EVENT_SCALE_ATTR(chan_4_rw_bytes),
	262	+ AML_DDR_PMU_EVENT_SCALE_ATTR(chan_5_rw_bytes),
	263	+ AML_DDR_PMU_EVENT_SCALE_ATTR(chan_6_rw_bytes),
	264	+ AML_DDR_PMU_EVENT_SCALE_ATTR(chan_7_rw_bytes),
	265	+ AML_DDR_PMU_EVENT_SCALE_ATTR(chan_8_rw_bytes),
	266	+};
	267	+
	268	+static struct perf_pmu_events_attr event_attrs[] = {
	269	+ AML_DDR_PMU_EVENT_ATTR(total_rw_bytes, ALL_CHAN_COUNTER_ID),
	270	+ AML_DDR_PMU_EVENT_ATTR(chan_1_rw_bytes, CHAN1_COUNTER_ID),
	271	+ AML_DDR_PMU_EVENT_ATTR(chan_2_rw_bytes, CHAN2_COUNTER_ID),
	272	+ AML_DDR_PMU_EVENT_ATTR(chan_3_rw_bytes, CHAN3_COUNTER_ID),
	273	+ AML_DDR_PMU_EVENT_ATTR(chan_4_rw_bytes, CHAN4_COUNTER_ID),
	274	+ AML_DDR_PMU_EVENT_ATTR(chan_5_rw_bytes, CHAN5_COUNTER_ID),
	275	+ AML_DDR_PMU_EVENT_ATTR(chan_6_rw_bytes, CHAN6_COUNTER_ID),
	276	+ AML_DDR_PMU_EVENT_ATTR(chan_7_rw_bytes, CHAN7_COUNTER_ID),
	277	+ AML_DDR_PMU_EVENT_ATTR(chan_8_rw_bytes, CHAN8_COUNTER_ID),
	278	+};
	279	+
	280	+/* three attrs are combined an event */
	281	+static struct attribute ddr_perf_events_attrs[COUNTER_MAX_ID 3];
	282	+
	283	+static struct attribute_group ddr_perf_events_attr_group = {
	284	+ .name = "events",
	285	+ .attrs = ddr_perf_events_attrs,
	286	+};
	287	+
	288	+static umode_t meson_ddr_perf_format_attr_visible(struct kobject *kobj,
	289	+ struct attribute *attr,
	290	+ int n)
	291	+{
	292	+ struct pmu *pmu = dev_get_drvdata(kobj_to_dev(kobj));
	293	+ struct ddr_pmu *ddr_pmu = to_ddr_pmu(pmu);
	294	+ const u64 *capability = ddr_pmu->info.hw_info->capability;
	295	+ struct device_attribute *dev_attr;
	296	+ int id;
	297	+ char value[20]; // config1:xxx, 20 is enough
	298	+
	299	+ dev_attr = container_of(attr, struct device_attribute, attr);
	300	+ dev_attr->show(NULL, NULL, value);
	301	+
	302	+ if (sscanf(value, "config1:%d", &id) == 1)
	303	+ return capability[0] & (1ULL << id) ? attr->mode : 0;
	304	+
	305	+ if (sscanf(value, "config2:%d", &id) == 1)
	306	+ return capability[1] & (1ULL << id) ? attr->mode : 0;
	307	+
	308	+ return attr->mode;
	309	+}
	310	+
	311	+static struct attribute_group ddr_perf_format_attr_group = {
	312	+ .name = "format",
	313	+ .is_visible = meson_ddr_perf_format_attr_visible,
	314	+};
	315	+
	316	+static ssize_t meson_ddr_perf_identifier_show(struct device *dev,
	317	+ struct device_attribute *attr,
	318	+ char *page)
	319	+{
	320	+ struct ddr_pmu *pmu = dev_get_drvdata(dev);
	321	+
	322	+ return sysfs_emit(page, "%s\n", pmu->name);
	323	+}
	324	+
	325	+static struct device_attribute meson_ddr_perf_identifier_attr =
	326	+__ATTR(identifier, 0444, meson_ddr_perf_identifier_show, NULL);
	327	+
	328	+static struct attribute *meson_ddr_perf_identifier_attrs[] = {
	329	+ &meson_ddr_perf_identifier_attr.attr,
	330	+ NULL,
	331	+};
	332	+
	333	+static const struct attribute_group ddr_perf_identifier_attr_group = {
	334	+ .attrs = meson_ddr_perf_identifier_attrs,
	335	+};
	336	+
	337	+static const struct attribute_group *attr_groups[] = {
	338	+ &ddr_perf_events_attr_group,
	339	+ &ddr_perf_format_attr_group,
	340	+ &ddr_perf_cpumask_attr_group,
	341	+ &ddr_perf_identifier_attr_group,
	342	+ NULL,
	343	+};
	344	+
	345	+static irqreturn_t dmc_irq_handler(int irq, void *dev_id)
	346	+{
	347	+ struct dmc_info *info = dev_id;
	348	+ struct ddr_pmu *pmu;
	349	+ struct dmc_counter counters, *sum_cnter;
	350	+ int i;
	351	+
	352	+ pmu = dmc_info_to_pmu(info);
	353	+
	354	+ if (info->hw_info->irq_handler(info, &counters) != 0)
	355	+ goto out;
	356	+
	357	+ sum_cnter = &pmu->counters;
	358	+ sum_cnter->all_cnt += counters.all_cnt;
	359	+ sum_cnter->all_req += counters.all_req;
	360	+
	361	+ for (i = 0; i < pmu->info.hw_info->chann_nr; i++)
	362	+ sum_cnter->channel_cnt[i] += counters.channel_cnt[i];
	363	+
	364	+ if (pmu->pmu_enabled)
	365	+ /*
	366	+ * the timer interrupt only supprt
	367	+ * one shot mode, we have to re-enable
	368	+ * it in ISR to support continue mode.
	369	+ */
	370	+ info->hw_info->enable(info);
	371	+
	372	+ dev_dbg(pmu->dev, "counts: %llu %llu %llu, %llu, %llu, %llu\t\t"
	373	+ "sum: %llu %llu %llu, %llu, %llu, %llu\n",
	374	+ counters.all_req,
	375	+ counters.all_cnt,
	376	+ counters.channel_cnt[0],
	377	+ counters.channel_cnt[1],
	378	+ counters.channel_cnt[2],
	379	+ counters.channel_cnt[3],
	380	+
	381	+ pmu->counters.all_req,
	382	+ pmu->counters.all_cnt,
	383	+ pmu->counters.channel_cnt[0],
	384	+ pmu->counters.channel_cnt[1],
	385	+ pmu->counters.channel_cnt[2],
	386	+ pmu->counters.channel_cnt[3]);
	387	+out:
	388	+ return IRQ_HANDLED;
	389	+}
	390	+
	391	+static int ddr_perf_offline_cpu(unsigned int cpu, struct hlist_node *node)
	392	+{
	393	+ struct ddr_pmu *pmu = hlist_entry_safe(node, struct ddr_pmu, node);
	394	+ int target;
	395	+
	396	+ if (cpu != pmu->cpu)
	397	+ return 0;
	398	+
	399	+ target = cpumask_any_but(cpu_online_mask, cpu);
	400	+ if (target >= nr_cpu_ids)
	401	+ return 0;
	402	+
	403	+ perf_pmu_migrate_context(&pmu->pmu, cpu, target);
	404	+ pmu->cpu = target;
	405	+
	406	+ WARN_ON(irq_set_affinity(pmu->info.irq_num, cpumask_of(pmu->cpu)));
	407	+
	408	+ return 0;
	409	+}
	410	+
	411	+static void fill_event_attr(struct ddr_pmu *pmu)
	412	+{
	413	+ int i, j, k;
	414	+ struct attribute **dst = ddr_perf_events_attrs;
	415	+
	416	+ j = 0;
	417	+ k = 0;
	418	+
	419	+ /* fill ALL_CHAN_COUNTER_ID event */
	420	+ dst[j++] = &event_attrs[k].attr.attr;
	421	+ dst[j++] = &event_unit_attrs[k].attr;
	422	+ dst[j++] = &event_scale_attrs[k].attr;
	423	+
	424	+ k++;
	425	+
	426	+ /* fill each channel event */
	427	+ for (i = 0; i < pmu->info.hw_info->chann_nr; i++, k++) {
	428	+ dst[j++] = &event_attrs[k].attr.attr;
	429	+ dst[j++] = &event_unit_attrs[k].attr;
	430	+ dst[j++] = &event_scale_attrs[k].attr;
	431	+ }
	432	+
	433	+ dst[j] = NULL; /* mark end */
	434	+}
	435	+
	436	+static void fmt_attr_fill(struct attribute **fmt_attr)
	437	+{
	438	+ ddr_perf_format_attr_group.attrs = fmt_attr;
	439	+}
	440	+
	441	+static int ddr_pmu_parse_dt(struct platform_device *pdev,
	442	+ struct dmc_info *info)
	443	+{
	444	+ void __iomem *base;
	445	+ int i, ret;
	446	+
	447	+ info->hw_info = of_device_get_match_data(&pdev->dev);
	448	+
	449	+ for (i = 0; i < info->hw_info->dmc_nr; i++) {
	450	+ /* resource 0 for ddr register base */
	451	+ base = devm_platform_ioremap_resource(pdev, i);
	452	+ if (IS_ERR(base))
	453	+ return PTR_ERR(base);
	454	+
	455	+ info->ddr_reg[i] = base;
	456	+ }
	457	+
	458	+ /* resource i for pll register base */
	459	+ base = devm_platform_ioremap_resource(pdev, i);
	460	+ if (IS_ERR(base))
	461	+ return PTR_ERR(base);
	462	+
	463	+ info->pll_reg = base;
	464	+
	465	+ ret = platform_get_irq(pdev, 0);
	466	+ if (ret < 0)
	467	+ return ret;
	468	+
	469	+ info->irq_num = ret;
	470	+
	471	+ ret = devm_request_irq(&pdev->dev, info->irq_num, dmc_irq_handler,
	472	+ IRQF_NOBALANCING, dev_name(&pdev->dev),
	473	+ (void *)info);
	474	+ if (ret < 0)
	475	+ return ret;
	476	+
	477	+ return 0;
	478	+}
	479	+
	480	+int meson_ddr_pmu_create(struct platform_device *pdev)
	481	+{
	482	+ int ret;
	483	+ char *name;
	484	+ struct ddr_pmu *pmu;
	485	+
	486	+ pmu = devm_kzalloc(&pdev->dev, sizeof(struct ddr_pmu), GFP_KERNEL);
	487	+ if (!pmu)
	488	+ return -ENOMEM;
	489	+
	490	+ *pmu = (struct ddr_pmu) {
	491	+ .pmu = {
	492	+ .module = THIS_MODULE,
	493	+ .capabilities = PERF_PMU_CAP_NO_EXCLUDE,
	494	+ .task_ctx_nr = perf_invalid_context,
	495	+ .attr_groups = attr_groups,
	496	+ .event_init = meson_ddr_perf_event_init,
	497	+ .add = meson_ddr_perf_event_add,
	498	+ .del = meson_ddr_perf_event_del,
	499	+ .start = meson_ddr_perf_event_start,
	500	+ .stop = meson_ddr_perf_event_stop,
	501	+ .read = meson_ddr_perf_event_update,
	502	+ },
	503	+ };
	504	+
	505	+ ret = ddr_pmu_parse_dt(pdev, &pmu->info);
	506	+ if (ret < 0)
	507	+ return ret;
	508	+
	509	+ fmt_attr_fill(pmu->info.hw_info->fmt_attr);
	510	+
	511	+ pmu->cpu = smp_processor_id();
	512	+
	513	+ name = devm_kasprintf(&pdev->dev, GFP_KERNEL, DDR_PERF_DEV_NAME);
	514	+ if (!name)
	515	+ return -ENOMEM;
	516	+
	517	+ ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, name, NULL,
	518	+ ddr_perf_offline_cpu);
	519	+ if (ret < 0)
	520	+ return ret;
	521	+
	522	+ pmu->cpuhp_state = ret;
	523	+
	524	+ /* Register the pmu instance for cpu hotplug */
	525	+ ret = cpuhp_state_add_instance_nocalls(pmu->cpuhp_state, &pmu->node);
	526	+ if (ret)
	527	+ goto cpuhp_instance_err;
	528	+
	529	+ fill_event_attr(pmu);
	530	+
	531	+ ret = perf_pmu_register(&pmu->pmu, name, -1);
	532	+ if (ret)
	533	+ goto pmu_register_err;
	534	+
	535	+ pmu->name = name;
	536	+ pmu->dev = &pdev->dev;
	537	+ pmu->pmu_enabled = false;
	538	+
	539	+ platform_set_drvdata(pdev, pmu);
	540	+
	541	+ return 0;
	542	+
	543	+pmu_register_err:
	544	+ cpuhp_state_remove_instance_nocalls(pmu->cpuhp_state, &pmu->node);
	545	+
	546	+cpuhp_instance_err:
	547	+ cpuhp_remove_state(pmu->cpuhp_state);
	548	+
	549	+ return ret;
	550	+}
	551	+
	552	+int meson_ddr_pmu_remove(struct platform_device *pdev)
	553	+{
	554	+ struct ddr_pmu *pmu = platform_get_drvdata(pdev);
	555	+
	556	+ perf_pmu_unregister(&pmu->pmu);
	557	+ cpuhp_state_remove_instance_nocalls(pmu->cpuhp_state, &pmu->node);
	558	+ cpuhp_remove_state(pmu->cpuhp_state);
	559	+
	560	+ return 0;
	561	+}

drivers/perf/amlogic/meson_g12_ddr_pmu.c+394−0

@@ -0,0 +1,394 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+/*
	3	+ * Copyright (c) 2022 Amlogic, Inc. All rights reserved.
	4	+ */
	5	+
	6	+#include <linux/err.h>
	7	+#include <linux/io.h>
	8	+#include <linux/kernel.h>
	9	+#include <linux/module.h>
	10	+#include <linux/of.h>
	11	+#include <linux/perf_event.h>
	12	+#include <linux/platform_device.h>
	13	+#include <linux/printk.h>
	14	+#include <linux/types.h>
	15	+
	16	+#include <soc/amlogic/meson_ddr_pmu.h>
	17	+
	18	+#define PORT_MAJOR 32
	19	+#define DEFAULT_XTAL_FREQ 24000000UL
	20	+
	21	+#define DMC_QOS_IRQ BIT(30)
	22	+
	23	+/* DMC bandwidth monitor register address offset */
	24	+#define DMC_MON_G12_CTRL0 (0x20 << 2)
	25	+#define DMC_MON_G12_CTRL1 (0x21 << 2)
	26	+#define DMC_MON_G12_CTRL2 (0x22 << 2)
	27	+#define DMC_MON_G12_CTRL3 (0x23 << 2)
	28	+#define DMC_MON_G12_CTRL4 (0x24 << 2)
	29	+#define DMC_MON_G12_CTRL5 (0x25 << 2)
	30	+#define DMC_MON_G12_CTRL6 (0x26 << 2)
	31	+#define DMC_MON_G12_CTRL7 (0x27 << 2)
	32	+#define DMC_MON_G12_CTRL8 (0x28 << 2)
	33	+
	34	+#define DMC_MON_G12_ALL_REQ_CNT (0x29 << 2)
	35	+#define DMC_MON_G12_ALL_GRANT_CNT (0x2a << 2)
	36	+#define DMC_MON_G12_ONE_GRANT_CNT (0x2b << 2)
	37	+#define DMC_MON_G12_SEC_GRANT_CNT (0x2c << 2)
	38	+#define DMC_MON_G12_THD_GRANT_CNT (0x2d << 2)
	39	+#define DMC_MON_G12_FOR_GRANT_CNT (0x2e << 2)
	40	+#define DMC_MON_G12_TIMER (0x2f << 2)
	41	+
	42	+/* Each bit represent a axi line */
	43	+PMU_FORMAT_ATTR(event, "config:0-7");
	44	+PMU_FORMAT_ATTR(arm, "config1:0");
	45	+PMU_FORMAT_ATTR(gpu, "config1:1");
	46	+PMU_FORMAT_ATTR(pcie, "config1:2");
	47	+PMU_FORMAT_ATTR(hdcp, "config1:3");
	48	+PMU_FORMAT_ATTR(hevc_front, "config1:4");
	49	+PMU_FORMAT_ATTR(usb3_0, "config1:6");
	50	+PMU_FORMAT_ATTR(device, "config1:7");
	51	+PMU_FORMAT_ATTR(hevc_back, "config1:8");
	52	+PMU_FORMAT_ATTR(h265enc, "config1:9");
	53	+PMU_FORMAT_ATTR(vpu_read1, "config1:16");
	54	+PMU_FORMAT_ATTR(vpu_read2, "config1:17");
	55	+PMU_FORMAT_ATTR(vpu_read3, "config1:18");
	56	+PMU_FORMAT_ATTR(vpu_write1, "config1:19");
	57	+PMU_FORMAT_ATTR(vpu_write2, "config1:20");
	58	+PMU_FORMAT_ATTR(vdec, "config1:21");
	59	+PMU_FORMAT_ATTR(hcodec, "config1:22");
	60	+PMU_FORMAT_ATTR(ge2d, "config1:23");
	61	+
	62	+PMU_FORMAT_ATTR(spicc1, "config1:32");
	63	+PMU_FORMAT_ATTR(usb0, "config1:33");
	64	+PMU_FORMAT_ATTR(dma, "config1:34");
	65	+PMU_FORMAT_ATTR(arb0, "config1:35");
	66	+PMU_FORMAT_ATTR(sd_emmc_b, "config1:36");
	67	+PMU_FORMAT_ATTR(usb1, "config1:37");
	68	+PMU_FORMAT_ATTR(audio, "config1:38");
	69	+PMU_FORMAT_ATTR(aififo, "config1:39");
	70	+PMU_FORMAT_ATTR(parser, "config1:41");
	71	+PMU_FORMAT_ATTR(ao_cpu, "config1:42");
	72	+PMU_FORMAT_ATTR(sd_emmc_c, "config1:43");
	73	+PMU_FORMAT_ATTR(spicc2, "config1:44");
	74	+PMU_FORMAT_ATTR(ethernet, "config1:45");
	75	+PMU_FORMAT_ATTR(sana, "config1:46");
	76	+
	77	+/* for sm1 and g12b */
	78	+PMU_FORMAT_ATTR(nna, "config1:10");
	79	+
	80	+/* for g12b only */
	81	+PMU_FORMAT_ATTR(gdc, "config1:11");
	82	+PMU_FORMAT_ATTR(mipi_isp, "config1:12");
	83	+PMU_FORMAT_ATTR(arm1, "config1:13");
	84	+PMU_FORMAT_ATTR(sd_emmc_a, "config1:40");
	85	+
	86	+static struct attribute *g12_pmu_format_attrs[] = {
	87	+ &format_attr_event.attr,
	88	+ &format_attr_arm.attr,
	89	+ &format_attr_gpu.attr,
	90	+ &format_attr_nna.attr,
	91	+ &format_attr_gdc.attr,
	92	+ &format_attr_arm1.attr,
	93	+ &format_attr_mipi_isp.attr,
	94	+ &format_attr_sd_emmc_a.attr,
	95	+ &format_attr_pcie.attr,
	96	+ &format_attr_hdcp.attr,
	97	+ &format_attr_hevc_front.attr,
	98	+ &format_attr_usb3_0.attr,
	99	+ &format_attr_device.attr,
	100	+ &format_attr_hevc_back.attr,
	101	+ &format_attr_h265enc.attr,
	102	+ &format_attr_vpu_read1.attr,
	103	+ &format_attr_vpu_read2.attr,
	104	+ &format_attr_vpu_read3.attr,
	105	+ &format_attr_vpu_write1.attr,
	106	+ &format_attr_vpu_write2.attr,
	107	+ &format_attr_vdec.attr,
	108	+ &format_attr_hcodec.attr,
	109	+ &format_attr_ge2d.attr,
	110	+ &format_attr_spicc1.attr,
	111	+ &format_attr_usb0.attr,
	112	+ &format_attr_dma.attr,
	113	+ &format_attr_arb0.attr,
	114	+ &format_attr_sd_emmc_b.attr,
	115	+ &format_attr_usb1.attr,
	116	+ &format_attr_audio.attr,
	117	+ &format_attr_aififo.attr,
	118	+ &format_attr_parser.attr,
	119	+ &format_attr_ao_cpu.attr,
	120	+ &format_attr_sd_emmc_c.attr,
	121	+ &format_attr_spicc2.attr,
	122	+ &format_attr_ethernet.attr,
	123	+ &format_attr_sana.attr,
	124	+ NULL,
	125	+};
	126	+
	127	+/* calculate ddr clock */
	128	+static unsigned long dmc_g12_get_freq_quick(struct dmc_info *info)
	129	+{
	130	+ unsigned int val;
	131	+ unsigned int n, m, od1;
	132	+ unsigned int od_div = 0xfff;
	133	+ unsigned long freq = 0;
	134	+
	135	+ val = readl(info->pll_reg);
	136	+ val = val & 0xfffff;
	137	+ switch ((val >> 16) & 7) {
	138	+ case 0:
	139	+ od_div = 2;
	140	+ break;
	141	+
	142	+ case 1:
	143	+ od_div = 3;
	144	+ break;
	145	+
	146	+ case 2:
	147	+ od_div = 4;
	148	+ break;
	149	+
	150	+ case 3:
	151	+ od_div = 6;
	152	+ break;
	153	+
	154	+ case 4:
	155	+ od_div = 8;
	156	+ break;
	157	+
	158	+ default:
	159	+ break;
	160	+ }
	161	+
	162	+ m = val & 0x1ff;
	163	+ n = ((val >> 10) & 0x1f);
	164	+ od1 = (((val >> 19) & 0x1)) == 1 ? 2 : 1;
	165	+ freq = DEFAULT_XTAL_FREQ / 1000; /* avoid overflow */
	166	+ if (n)
	167	+ freq = ((((freq * m) / n) >> od1) / od_div) * 1000;
	168	+
	169	+ return freq;
	170	+}
	171	+
	172	+#ifdef DEBUG
	173	+static void g12_dump_reg(struct dmc_info *db)
	174	+{
	175	+ int s = 0, i;
	176	+ unsigned int r;
	177	+
	178	+ for (i = 0; i < 9; i++) {
	179	+ r = readl(db->ddr_reg[0] + (DMC_MON_G12_CTRL0 + (i << 2)));
	180	+ pr_notice("DMC_MON_CTRL%d: %08x\n", i, r);
	181	+ }
	182	+ r = readl(db->ddr_reg[0] + DMC_MON_G12_ALL_REQ_CNT);
	183	+ pr_notice("DMC_MON_ALL_REQ_CNT: %08x\n", r);
	184	+ r = readl(db->ddr_reg[0] + DMC_MON_G12_ALL_GRANT_CNT);
	185	+ pr_notice("DMC_MON_ALL_GRANT_CNT:%08x\n", r);
	186	+ r = readl(db->ddr_reg[0] + DMC_MON_G12_ONE_GRANT_CNT);
	187	+ pr_notice("DMC_MON_ONE_GRANT_CNT:%08x\n", r);
	188	+ r = readl(db->ddr_reg[0] + DMC_MON_G12_SEC_GRANT_CNT);
	189	+ pr_notice("DMC_MON_SEC_GRANT_CNT:%08x\n", r);
	190	+ r = readl(db->ddr_reg[0] + DMC_MON_G12_THD_GRANT_CNT);
	191	+ pr_notice("DMC_MON_THD_GRANT_CNT:%08x\n", r);
	192	+ r = readl(db->ddr_reg[0] + DMC_MON_G12_FOR_GRANT_CNT);
	193	+ pr_notice("DMC_MON_FOR_GRANT_CNT:%08x\n", r);
	194	+ r = readl(db->ddr_reg[0] + DMC_MON_G12_TIMER);
	195	+ pr_notice("DMC_MON_TIMER: %08x\n", r);
	196	+}
	197	+#endif
	198	+
	199	+static void dmc_g12_counter_enable(struct dmc_info *info)
	200	+{
	201	+ unsigned int val;
	202	+ unsigned long clock_count = dmc_g12_get_freq_quick(info) / 10; /* 100ms */
	203	+
	204	+ writel(clock_count, info->ddr_reg[0] + DMC_MON_G12_TIMER);
	205	+
	206	+ val = readl(info->ddr_reg[0] + DMC_MON_G12_CTRL0);
	207	+
	208	+ /* enable all channel */
	209	+ val = BIT(31) \| /* enable bit */
	210	+ BIT(20) \| /* use timer */
	211	+ 0x0f; /* 4 channels */
	212	+
	213	+ writel(val, info->ddr_reg[0] + DMC_MON_G12_CTRL0);
	214	+
	215	+#ifdef DEBUG
	216	+ g12_dump_reg(info);
	217	+#endif
	218	+}
	219	+
	220	+static void dmc_g12_config_fiter(struct dmc_info *info,
	221	+ int port, int channel)
	222	+{
	223	+ u32 val;
	224	+ u32 rp[MAX_CHANNEL_NUM] = {DMC_MON_G12_CTRL1, DMC_MON_G12_CTRL3,
	225	+ DMC_MON_G12_CTRL5, DMC_MON_G12_CTRL7};
	226	+ u32 rs[MAX_CHANNEL_NUM] = {DMC_MON_G12_CTRL2, DMC_MON_G12_CTRL4,
	227	+ DMC_MON_G12_CTRL6, DMC_MON_G12_CTRL8};
	228	+ int subport = -1;
	229	+
	230	+ /* clear all port mask */
	231	+ if (port < 0) {
	232	+ writel(0, info->ddr_reg[0] + rp[channel]);
	233	+ writel(0, info->ddr_reg[0] + rs[channel]);
	234	+ return;
	235	+ }
	236	+
	237	+ if (port >= PORT_MAJOR)
	238	+ subport = port - PORT_MAJOR;
	239	+
	240	+ if (subport < 0) {
	241	+ val = readl(info->ddr_reg[0] + rp[channel]);
	242	+ val \|= (1 << port);
	243	+ writel(val, info->ddr_reg[0] + rp[channel]);
	244	+ val = 0xffff;
	245	+ writel(val, info->ddr_reg[0] + rs[channel]);
	246	+ } else {
	247	+ val = BIT(23); /* select device */
	248	+ writel(val, info->ddr_reg[0] + rp[channel]);
	249	+ val = readl(info->ddr_reg[0] + rs[channel]);
	250	+ val \|= (1 << subport);
	251	+ writel(val, info->ddr_reg[0] + rs[channel]);
	252	+ }
	253	+}
	254	+
	255	+static void dmc_g12_set_axi_filter(struct dmc_info *info, int axi_id, int channel)
	256	+{
	257	+ if (channel > info->hw_info->chann_nr)
	258	+ return;
	259	+
	260	+ dmc_g12_config_fiter(info, axi_id, channel);
	261	+}
	262	+
	263	+static void dmc_g12_counter_disable(struct dmc_info *info)
	264	+{
	265	+ int i;
	266	+
	267	+ /* clear timer */
	268	+ writel(0, info->ddr_reg[0] + DMC_MON_G12_CTRL0);
	269	+ writel(0, info->ddr_reg[0] + DMC_MON_G12_TIMER);
	270	+
	271	+ writel(0, info->ddr_reg[0] + DMC_MON_G12_ALL_REQ_CNT);
	272	+ writel(0, info->ddr_reg[0] + DMC_MON_G12_ALL_GRANT_CNT);
	273	+ writel(0, info->ddr_reg[0] + DMC_MON_G12_ONE_GRANT_CNT);
	274	+ writel(0, info->ddr_reg[0] + DMC_MON_G12_SEC_GRANT_CNT);
	275	+ writel(0, info->ddr_reg[0] + DMC_MON_G12_THD_GRANT_CNT);
	276	+ writel(0, info->ddr_reg[0] + DMC_MON_G12_FOR_GRANT_CNT);
	277	+
	278	+ /* clear port channel mapping */
	279	+ for (i = 0; i < info->hw_info->chann_nr; i++)
	280	+ dmc_g12_config_fiter(info, -1, i);
	281	+}
	282	+
	283	+static void dmc_g12_get_counters(struct dmc_info *info,
	284	+ struct dmc_counter *counter)
	285	+{
	286	+ int i;
	287	+ unsigned int reg;
	288	+
	289	+ counter->all_cnt = readl(info->ddr_reg[0] + DMC_MON_G12_ALL_GRANT_CNT);
	290	+ counter->all_req = readl(info->ddr_reg[0] + DMC_MON_G12_ALL_REQ_CNT);
	291	+
	292	+ for (i = 0; i < info->hw_info->chann_nr; i++) {
	293	+ reg = DMC_MON_G12_ONE_GRANT_CNT + (i << 2);
	294	+ counter->channel_cnt[i] = readl(info->ddr_reg[0] + reg);
	295	+ }
	296	+}
	297	+
	298	+static int dmc_g12_irq_handler(struct dmc_info *info,
	299	+ struct dmc_counter *counter)
	300	+{
	301	+ unsigned int val;
	302	+ int ret = -EINVAL;
	303	+
	304	+ val = readl(info->ddr_reg[0] + DMC_MON_G12_CTRL0);
	305	+ if (val & DMC_QOS_IRQ) {
	306	+ dmc_g12_get_counters(info, counter);
	307	+ /* clear irq flags */
	308	+ writel(val, info->ddr_reg[0] + DMC_MON_G12_CTRL0);
	309	+ ret = 0;
	310	+ }
	311	+ return ret;
	312	+}
	313	+
	314	+static const struct dmc_hw_info g12a_dmc_info = {
	315	+ .enable = dmc_g12_counter_enable,
	316	+ .disable = dmc_g12_counter_disable,
	317	+ .irq_handler = dmc_g12_irq_handler,
	318	+ .get_counters = dmc_g12_get_counters,
	319	+ .set_axi_filter = dmc_g12_set_axi_filter,
	320	+
	321	+ .dmc_nr = 1,
	322	+ .chann_nr = 4,
	323	+ .capability = {0X7EFF00FF03DF, 0},
	324	+ .fmt_attr = g12_pmu_format_attrs,
	325	+};
	326	+
	327	+static const struct dmc_hw_info g12b_dmc_info = {
	328	+ .enable = dmc_g12_counter_enable,
	329	+ .disable = dmc_g12_counter_disable,
	330	+ .irq_handler = dmc_g12_irq_handler,
	331	+ .get_counters = dmc_g12_get_counters,
	332	+ .set_axi_filter = dmc_g12_set_axi_filter,
	333	+
	334	+ .dmc_nr = 1,
	335	+ .chann_nr = 4,
	336	+ .capability = {0X7FFF00FF3FDF, 0},
	337	+ .fmt_attr = g12_pmu_format_attrs,
	338	+};
	339	+
	340	+static const struct dmc_hw_info sm1_dmc_info = {
	341	+ .enable = dmc_g12_counter_enable,
	342	+ .disable = dmc_g12_counter_disable,
	343	+ .irq_handler = dmc_g12_irq_handler,
	344	+ .get_counters = dmc_g12_get_counters,
	345	+ .set_axi_filter = dmc_g12_set_axi_filter,
	346	+
	347	+ .dmc_nr = 1,
	348	+ .chann_nr = 4,
	349	+ .capability = {0X7EFF00FF07DF, 0},
	350	+ .fmt_attr = g12_pmu_format_attrs,
	351	+};
	352	+
	353	+static int g12_ddr_pmu_probe(struct platform_device *pdev)
	354	+{
	355	+ return meson_ddr_pmu_create(pdev);
	356	+}
	357	+
	358	+static int g12_ddr_pmu_remove(struct platform_device *pdev)
	359	+{
	360	+ meson_ddr_pmu_remove(pdev);
	361	+
	362	+ return 0;
	363	+}
	364	+
	365	+static const struct of_device_id meson_ddr_pmu_dt_match[] = {
	366	+ {
	367	+ .compatible = "amlogic,g12a-ddr-pmu",
	368	+ .data = &g12a_dmc_info,
	369	+ },
	370	+ {
	371	+ .compatible = "amlogic,g12b-ddr-pmu",
	372	+ .data = &g12b_dmc_info,
	373	+ },
	374	+ {
	375	+ .compatible = "amlogic,sm1-ddr-pmu",
	376	+ .data = &sm1_dmc_info,
	377	+ },
	378	+ {}
	379	+};
	380	+
	381	+static struct platform_driver g12_ddr_pmu_driver = {
	382	+ .probe = g12_ddr_pmu_probe,
	383	+ .remove = g12_ddr_pmu_remove,
	384	+
	385	+ .driver = {
	386	+ .name = "meson-g12-ddr-pmu",
	387	+ .of_match_table = meson_ddr_pmu_dt_match,
	388	+ },
	389	+};
	390	+
	391	+module_platform_driver(g12_ddr_pmu_driver);
	392	+MODULE_AUTHOR("Jiucheng Xu");
	393	+MODULE_LICENSE("GPL");
	394	+MODULE_DESCRIPTION("Amlogic G12 series SoC DDR PMU");

drivers/perf/arm_cspmu/Kconfig+13−0

@@ -0,0 +1,13 @@
	1	+# SPDX-License-Identifier: GPL-2.0
	2	+#
	3	+# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
	4	+
	5	+config ARM_CORESIGHT_PMU_ARCH_SYSTEM_PMU
	6	+ tristate "ARM Coresight Architecture PMU"
	7	+ depends on ARM64 && ACPI
	8	+ depends on ACPI_APMT \|\| COMPILE_TEST
	9	+ help
	10	+ Provides support for performance monitoring unit (PMU) devices
	11	+ based on ARM CoreSight PMU architecture. Note that this PMU
	12	+ architecture does not have relationship with the ARM CoreSight
	13	+ Self-Hosted Tracing.

drivers/perf/arm_cspmu/Makefile+6−0

@@ -0,0 +1,6 @@
	1	+# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
	2	+#
	3	+# SPDX-License-Identifier: GPL-2.0
	4	+
	5	+obj-$(CONFIG_ARM_CORESIGHT_PMU_ARCH_SYSTEM_PMU) += arm_cspmu_module.o
	6	+arm_cspmu_module-y := arm_cspmu.o nvidia_cspmu.o

drivers/perf/arm_cspmu/arm_cspmu.c+1303−0

@@ -0,0 +1,1303 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+/*
	3	+ * ARM CoreSight Architecture PMU driver.
	4	+ *
	5	+ * This driver adds support for uncore PMU based on ARM CoreSight Performance
	6	+ * Monitoring Unit Architecture. The PMU is accessible via MMIO registers and
	7	+ * like other uncore PMUs, it does not support process specific events and
	8	+ * cannot be used in sampling mode.
	9	+ *
	10	+ * This code is based on other uncore PMUs like ARM DSU PMU. It provides a
	11	+ * generic implementation to operate the PMU according to CoreSight PMU
	12	+ * architecture and ACPI ARM PMU table (APMT) documents below:
	13	+ * - ARM CoreSight PMU architecture document number: ARM IHI 0091 A.a-00bet0.
	14	+ * - APMT document number: ARM DEN0117.
	15	+ *
	16	+ * The user should refer to the vendor technical documentation to get details
	17	+ * about the supported events.
	18	+ *
	19	+ * Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
	20	+ *
	21	+ */
	22	+
	23	+#include <linux/acpi.h>
	24	+#include <linux/cacheinfo.h>
	25	+#include <linux/ctype.h>
	26	+#include <linux/interrupt.h>
	27	+#include <linux/io-64-nonatomic-lo-hi.h>
	28	+#include <linux/module.h>
	29	+#include <linux/perf_event.h>
	30	+#include <linux/platform_device.h>
	31	+#include <acpi/processor.h>
	32	+
	33	+#include "arm_cspmu.h"
	34	+#include "nvidia_cspmu.h"
	35	+
	36	+#define PMUNAME "arm_cspmu"
	37	+#define DRVNAME "arm-cs-arch-pmu"
	38	+
	39	+#define ARM_CSPMU_CPUMASK_ATTR(_name, _config) \
	40	+ ARM_CSPMU_EXT_ATTR(_name, arm_cspmu_cpumask_show, \
	41	+ (unsigned long)_config)
	42	+
	43	+/*
	44	+ * CoreSight PMU Arch register offsets.
	45	+ */
	46	+#define PMEVCNTR_LO 0x0
	47	+#define PMEVCNTR_HI 0x4
	48	+#define PMEVTYPER 0x400
	49	+#define PMCCFILTR 0x47C
	50	+#define PMEVFILTR 0xA00
	51	+#define PMCNTENSET 0xC00
	52	+#define PMCNTENCLR 0xC20
	53	+#define PMINTENSET 0xC40
	54	+#define PMINTENCLR 0xC60
	55	+#define PMOVSCLR 0xC80
	56	+#define PMOVSSET 0xCC0
	57	+#define PMCFGR 0xE00
	58	+#define PMCR 0xE04
	59	+#define PMIIDR 0xE08
	60	+
	61	+/* PMCFGR register field */
	62	+#define PMCFGR_NCG GENMASK(31, 28)
	63	+#define PMCFGR_HDBG BIT(24)
	64	+#define PMCFGR_TRO BIT(23)
	65	+#define PMCFGR_SS BIT(22)
	66	+#define PMCFGR_FZO BIT(21)
	67	+#define PMCFGR_MSI BIT(20)
	68	+#define PMCFGR_UEN BIT(19)
	69	+#define PMCFGR_NA BIT(17)
	70	+#define PMCFGR_EX BIT(16)
	71	+#define PMCFGR_CCD BIT(15)
	72	+#define PMCFGR_CC BIT(14)
	73	+#define PMCFGR_SIZE GENMASK(13, 8)
	74	+#define PMCFGR_N GENMASK(7, 0)
	75	+
	76	+/* PMCR register field */
	77	+#define PMCR_TRO BIT(11)
	78	+#define PMCR_HDBG BIT(10)
	79	+#define PMCR_FZO BIT(9)
	80	+#define PMCR_NA BIT(8)
	81	+#define PMCR_DP BIT(5)
	82	+#define PMCR_X BIT(4)
	83	+#define PMCR_D BIT(3)
	84	+#define PMCR_C BIT(2)
	85	+#define PMCR_P BIT(1)
	86	+#define PMCR_E BIT(0)
	87	+
	88	+/* Each SET/CLR register supports up to 32 counters. */
	89	+#define ARM_CSPMU_SET_CLR_COUNTER_SHIFT 5
	90	+#define ARM_CSPMU_SET_CLR_COUNTER_NUM \
	91	+ (1 << ARM_CSPMU_SET_CLR_COUNTER_SHIFT)
	92	+
	93	+/* Convert counter idx into SET/CLR register number. */
	94	+#define COUNTER_TO_SET_CLR_ID(idx) \
	95	+ (idx >> ARM_CSPMU_SET_CLR_COUNTER_SHIFT)
	96	+
	97	+/* Convert counter idx into SET/CLR register bit. */
	98	+#define COUNTER_TO_SET_CLR_BIT(idx) \
	99	+ (idx & (ARM_CSPMU_SET_CLR_COUNTER_NUM - 1))
	100	+
	101	+#define ARM_CSPMU_ACTIVE_CPU_MASK 0x0
	102	+#define ARM_CSPMU_ASSOCIATED_CPU_MASK 0x1
	103	+
	104	+/* Check if field f in flags is set with value v */
	105	+#define CHECK_APMT_FLAG(flags, f, v) \
	106	+ ((flags & (ACPI_APMT_FLAGS_ ## f)) == (ACPI_APMT_FLAGS_ ## f ## _ ## v))
	107	+
	108	+/* Check and use default if implementer doesn't provide attribute callback */
	109	+#define CHECK_DEFAULT_IMPL_OPS(ops, callback) \
	110	+ do { \
	111	+ if (!ops->callback) \
	112	+ ops->callback = arm_cspmu_ ## callback; \
	113	+ } while (0)
	114	+
	115	+/*
	116	+ * Maximum poll count for reading counter value using high-low-high sequence.
	117	+ */
	118	+#define HILOHI_MAX_POLL 1000
	119	+
	120	+/* JEDEC-assigned JEP106 identification code */
	121	+#define ARM_CSPMU_IMPL_ID_NVIDIA 0x36B
	122	+
	123	+static unsigned long arm_cspmu_cpuhp_state;
	124	+
	125	+/*
	126	+ * In CoreSight PMU architecture, all of the MMIO registers are 32-bit except
	127	+ * counter register. The counter register can be implemented as 32-bit or 64-bit
	128	+ * register depending on the value of PMCFGR.SIZE field. For 64-bit access,
	129	+ * single-copy 64-bit atomic support is implementation defined. APMT node flag
	130	+ * is used to identify if the PMU supports 64-bit single copy atomic. If 64-bit
	131	+ * single copy atomic is not supported, the driver treats the register as a pair
	132	+ * of 32-bit register.
	133	+ */
	134	+
	135	+/*
	136	+ * Read 64-bit register as a pair of 32-bit registers using hi-lo-hi sequence.
	137	+ */
	138	+static u64 read_reg64_hilohi(const void __iomem *addr, u32 max_poll_count)
	139	+{
	140	+ u32 val_lo, val_hi;
	141	+ u64 val;
	142	+
	143	+ /* Use high-low-high sequence to avoid tearing */
	144	+ do {
	145	+ if (max_poll_count-- == 0) {
	146	+ pr_err("ARM CSPMU: timeout hi-low-high sequence\n");
	147	+ return 0;
	148	+ }
	149	+
	150	+ val_hi = readl(addr + 4);
	151	+ val_lo = readl(addr);
	152	+ } while (val_hi != readl(addr + 4));
	153	+
	154	+ val = (((u64)val_hi << 32) \| val_lo);
	155	+
	156	+ return val;
	157	+}
	158	+
	159	+/* Check if PMU supports 64-bit single copy atomic. */
	160	+static inline bool supports_64bit_atomics(const struct arm_cspmu *cspmu)
	161	+{
	162	+ return CHECK_APMT_FLAG(cspmu->apmt_node->flags, ATOMIC, SUPP);
	163	+}
	164	+
	165	+/* Check if cycle counter is supported. */
	166	+static inline bool supports_cycle_counter(const struct arm_cspmu *cspmu)
	167	+{
	168	+ return (cspmu->pmcfgr & PMCFGR_CC);
	169	+}
	170	+
	171	+/* Get counter size, which is (PMCFGR_SIZE + 1). */
	172	+static inline u32 counter_size(const struct arm_cspmu *cspmu)
	173	+{
	174	+ return FIELD_GET(PMCFGR_SIZE, cspmu->pmcfgr) + 1;
	175	+}
	176	+
	177	+/* Get counter mask. */
	178	+static inline u64 counter_mask(const struct arm_cspmu *cspmu)
	179	+{
	180	+ return GENMASK_ULL(counter_size(cspmu) - 1, 0);
	181	+}
	182	+
	183	+/* Check if counter is implemented as 64-bit register. */
	184	+static inline bool use_64b_counter_reg(const struct arm_cspmu *cspmu)
	185	+{
	186	+ return (counter_size(cspmu) > 32);
	187	+}
	188	+
	189	+ssize_t arm_cspmu_sysfs_event_show(struct device *dev,
	190	+ struct device_attribute attr, char buf)
	191	+{
	192	+ struct dev_ext_attribute *eattr =
	193	+ container_of(attr, struct dev_ext_attribute, attr);
	194	+ return sysfs_emit(buf, "event=0x%llx\n",
	195	+ (unsigned long long)eattr->var);
	196	+}
	197	+EXPORT_SYMBOL_GPL(arm_cspmu_sysfs_event_show);
	198	+
	199	+/* Default event list. */
	200	+static struct attribute *arm_cspmu_event_attrs[] = {
	201	+ ARM_CSPMU_EVENT_ATTR(cycles, ARM_CSPMU_EVT_CYCLES_DEFAULT),
	202	+ NULL,
	203	+};
	204	+
	205	+static struct attribute **
	206	+arm_cspmu_get_event_attrs(const struct arm_cspmu *cspmu)
	207	+{
	208	+ struct attribute **attrs;
	209	+
	210	+ attrs = devm_kmemdup(cspmu->dev, arm_cspmu_event_attrs,
	211	+ sizeof(arm_cspmu_event_attrs), GFP_KERNEL);
	212	+
	213	+ return attrs;
	214	+}
	215	+
	216	+static umode_t
	217	+arm_cspmu_event_attr_is_visible(struct kobject *kobj,
	218	+ struct attribute *attr, int unused)
	219	+{
	220	+ struct device *dev = kobj_to_dev(kobj);
	221	+ struct arm_cspmu *cspmu = to_arm_cspmu(dev_get_drvdata(dev));
	222	+ struct perf_pmu_events_attr *eattr;
	223	+
	224	+ eattr = container_of(attr, typeof(*eattr), attr.attr);
	225	+
	226	+ /* Hide cycle event if not supported */
	227	+ if (!supports_cycle_counter(cspmu) &&
	228	+ eattr->id == ARM_CSPMU_EVT_CYCLES_DEFAULT)
	229	+ return 0;
	230	+
	231	+ return attr->mode;
	232	+}
	233	+
	234	+ssize_t arm_cspmu_sysfs_format_show(struct device *dev,
	235	+ struct device_attribute *attr,
	236	+ char *buf)
	237	+{
	238	+ struct dev_ext_attribute *eattr =
	239	+ container_of(attr, struct dev_ext_attribute, attr);
	240	+ return sysfs_emit(buf, "%s\n", (char *)eattr->var);
	241	+}
	242	+EXPORT_SYMBOL_GPL(arm_cspmu_sysfs_format_show);
	243	+
	244	+static struct attribute *arm_cspmu_format_attrs[] = {
	245	+ ARM_CSPMU_FORMAT_EVENT_ATTR,
	246	+ ARM_CSPMU_FORMAT_FILTER_ATTR,
	247	+ NULL,
	248	+};
	249	+
	250	+static struct attribute **
	251	+arm_cspmu_get_format_attrs(const struct arm_cspmu *cspmu)
	252	+{
	253	+ struct attribute **attrs;
	254	+
	255	+ attrs = devm_kmemdup(cspmu->dev, arm_cspmu_format_attrs,
	256	+ sizeof(arm_cspmu_format_attrs), GFP_KERNEL);
	257	+
	258	+ return attrs;
	259	+}
	260	+
	261	+static u32 arm_cspmu_event_type(const struct perf_event *event)
	262	+{
	263	+ return event->attr.config & ARM_CSPMU_EVENT_MASK;
	264	+}
	265	+
	266	+static bool arm_cspmu_is_cycle_counter_event(const struct perf_event *event)
	267	+{
	268	+ return (event->attr.config == ARM_CSPMU_EVT_CYCLES_DEFAULT);
	269	+}
	270	+
	271	+static u32 arm_cspmu_event_filter(const struct perf_event *event)
	272	+{
	273	+ return event->attr.config1 & ARM_CSPMU_FILTER_MASK;
	274	+}
	275	+
	276	+static ssize_t arm_cspmu_identifier_show(struct device *dev,
	277	+ struct device_attribute *attr,
	278	+ char *page)
	279	+{
	280	+ struct arm_cspmu *cspmu = to_arm_cspmu(dev_get_drvdata(dev));
	281	+
	282	+ return sysfs_emit(page, "%s\n", cspmu->identifier);
	283	+}
	284	+
	285	+static struct device_attribute arm_cspmu_identifier_attr =
	286	+ __ATTR(identifier, 0444, arm_cspmu_identifier_show, NULL);
	287	+
	288	+static struct attribute *arm_cspmu_identifier_attrs[] = {
	289	+ &arm_cspmu_identifier_attr.attr,
	290	+ NULL,
	291	+};
	292	+
	293	+static struct attribute_group arm_cspmu_identifier_attr_group = {
	294	+ .attrs = arm_cspmu_identifier_attrs,
	295	+};
	296	+
	297	+static const char arm_cspmu_get_identifier(const struct arm_cspmu cspmu)
	298	+{
	299	+ const char *identifier =
	300	+ devm_kasprintf(cspmu->dev, GFP_KERNEL, "%x",
	301	+ cspmu->impl.pmiidr);
	302	+ return identifier;
	303	+}
	304	+
	305	+static const char *arm_cspmu_type_str[ACPI_APMT_NODE_TYPE_COUNT] = {
	306	+ "mc",
	307	+ "smmu",
	308	+ "pcie",
	309	+ "acpi",
	310	+ "cache",
	311	+};
	312	+
	313	+static const char arm_cspmu_get_name(const struct arm_cspmu cspmu)
	314	+{
	315	+ struct device *dev;
	316	+ struct acpi_apmt_node *apmt_node;
	317	+ u8 pmu_type;
	318	+ char *name;
	319	+ char acpi_hid_string[ACPI_ID_LEN] = { 0 };
	320	+ static atomic_t pmu_idx[ACPI_APMT_NODE_TYPE_COUNT] = { 0 };
	321	+
	322	+ dev = cspmu->dev;
	323	+ apmt_node = cspmu->apmt_node;
	324	+ pmu_type = apmt_node->type;
	325	+
	326	+ if (pmu_type >= ACPI_APMT_NODE_TYPE_COUNT) {
	327	+ dev_err(dev, "unsupported PMU type-%u\n", pmu_type);
	328	+ return NULL;
	329	+ }
	330	+
	331	+ if (pmu_type == ACPI_APMT_NODE_TYPE_ACPI) {
	332	+ memcpy(acpi_hid_string,
	333	+ &apmt_node->inst_primary,
	334	+ sizeof(apmt_node->inst_primary));
	335	+ name = devm_kasprintf(dev, GFP_KERNEL, "%s_%s_%s_%u", PMUNAME,
	336	+ arm_cspmu_type_str[pmu_type],
	337	+ acpi_hid_string,
	338	+ apmt_node->inst_secondary);
	339	+ } else {
	340	+ name = devm_kasprintf(dev, GFP_KERNEL, "%s_%s_%d", PMUNAME,
	341	+ arm_cspmu_type_str[pmu_type],
	342	+ atomic_fetch_inc(&pmu_idx[pmu_type]));
	343	+ }
	344	+
	345	+ return name;
	346	+}
	347	+
	348	+static ssize_t arm_cspmu_cpumask_show(struct device *dev,
	349	+ struct device_attribute *attr,
	350	+ char *buf)
	351	+{
	352	+ struct pmu *pmu = dev_get_drvdata(dev);
	353	+ struct arm_cspmu *cspmu = to_arm_cspmu(pmu);
	354	+ struct dev_ext_attribute *eattr =
	355	+ container_of(attr, struct dev_ext_attribute, attr);
	356	+ unsigned long mask_id = (unsigned long)eattr->var;
	357	+ const cpumask_t *cpumask;
	358	+
	359	+ switch (mask_id) {
	360	+ case ARM_CSPMU_ACTIVE_CPU_MASK:
	361	+ cpumask = &cspmu->active_cpu;
	362	+ break;
	363	+ case ARM_CSPMU_ASSOCIATED_CPU_MASK:
	364	+ cpumask = &cspmu->associated_cpus;
	365	+ break;
	366	+ default:
	367	+ return 0;
	368	+ }
	369	+ return cpumap_print_to_pagebuf(true, buf, cpumask);
	370	+}
	371	+
	372	+static struct attribute *arm_cspmu_cpumask_attrs[] = {
	373	+ ARM_CSPMU_CPUMASK_ATTR(cpumask, ARM_CSPMU_ACTIVE_CPU_MASK),
	374	+ ARM_CSPMU_CPUMASK_ATTR(associated_cpus, ARM_CSPMU_ASSOCIATED_CPU_MASK),
	375	+ NULL,
	376	+};
	377	+
	378	+static struct attribute_group arm_cspmu_cpumask_attr_group = {
	379	+ .attrs = arm_cspmu_cpumask_attrs,
	380	+};
	381	+
	382	+struct impl_match {
	383	+ u32 pmiidr;
	384	+ u32 mask;
	385	+ int (impl_init_ops)(struct arm_cspmu cspmu);
	386	+};
	387	+
	388	+static const struct impl_match impl_match[] = {
	389	+ {
	390	+ .pmiidr = ARM_CSPMU_IMPL_ID_NVIDIA,
	391	+ .mask = ARM_CSPMU_PMIIDR_IMPLEMENTER,
	392	+ .impl_init_ops = nv_cspmu_init_ops
	393	+ },
	394	+ {}
	395	+};
	396	+
	397	+static int arm_cspmu_init_impl_ops(struct arm_cspmu *cspmu)
	398	+{
	399	+ int ret;
	400	+ struct acpi_apmt_node *apmt_node = cspmu->apmt_node;
	401	+ struct arm_cspmu_impl_ops *impl_ops = &cspmu->impl.ops;
	402	+ const struct impl_match *match = impl_match;
	403	+
	404	+ /*
	405	+ * Get PMU implementer and product id from APMT node.
	406	+ * If APMT node doesn't have implementer/product id, try get it
	407	+ * from PMIIDR.
	408	+ */
	409	+ cspmu->impl.pmiidr =
	410	+ (apmt_node->impl_id) ? apmt_node->impl_id :
	411	+ readl(cspmu->base0 + PMIIDR);
	412	+
	413	+ /* Find implementer specific attribute ops. */
	414	+ for (; match->pmiidr; match++) {
	415	+ const u32 mask = match->mask;
	416	+
	417	+ if ((match->pmiidr & mask) == (cspmu->impl.pmiidr & mask)) {
	418	+ ret = match->impl_init_ops(cspmu);
	419	+ if (ret)
	420	+ return ret;
	421	+
	422	+ break;
	423	+ }
	424	+ }
	425	+
	426	+ /* Use default callbacks if implementer doesn't provide one. */
	427	+ CHECK_DEFAULT_IMPL_OPS(impl_ops, get_event_attrs);
	428	+ CHECK_DEFAULT_IMPL_OPS(impl_ops, get_format_attrs);
	429	+ CHECK_DEFAULT_IMPL_OPS(impl_ops, get_identifier);
	430	+ CHECK_DEFAULT_IMPL_OPS(impl_ops, get_name);
	431	+ CHECK_DEFAULT_IMPL_OPS(impl_ops, is_cycle_counter_event);
	432	+ CHECK_DEFAULT_IMPL_OPS(impl_ops, event_type);
	433	+ CHECK_DEFAULT_IMPL_OPS(impl_ops, event_filter);
	434	+ CHECK_DEFAULT_IMPL_OPS(impl_ops, event_attr_is_visible);
	435	+
	436	+ return 0;
	437	+}
	438	+
	439	+static struct attribute_group *
	440	+arm_cspmu_alloc_event_attr_group(struct arm_cspmu *cspmu)
	441	+{
	442	+ struct attribute_group *event_group;
	443	+ struct device *dev = cspmu->dev;
	444	+ const struct arm_cspmu_impl_ops *impl_ops = &cspmu->impl.ops;
	445	+
	446	+ event_group =
	447	+ devm_kzalloc(dev, sizeof(struct attribute_group), GFP_KERNEL);
	448	+ if (!event_group)
	449	+ return NULL;
	450	+
	451	+ event_group->name = "events";
	452	+ event_group->is_visible = impl_ops->event_attr_is_visible;
	453	+ event_group->attrs = impl_ops->get_event_attrs(cspmu);
	454	+
	455	+ if (!event_group->attrs)
	456	+ return NULL;
	457	+
	458	+ return event_group;
	459	+}
	460	+
	461	+static struct attribute_group *
	462	+arm_cspmu_alloc_format_attr_group(struct arm_cspmu *cspmu)
	463	+{
	464	+ struct attribute_group *format_group;
	465	+ struct device *dev = cspmu->dev;
	466	+
	467	+ format_group =
	468	+ devm_kzalloc(dev, sizeof(struct attribute_group), GFP_KERNEL);
	469	+ if (!format_group)
	470	+ return NULL;
	471	+
	472	+ format_group->name = "format";
	473	+ format_group->attrs = cspmu->impl.ops.get_format_attrs(cspmu);
	474	+
	475	+ if (!format_group->attrs)
	476	+ return NULL;
	477	+
	478	+ return format_group;
	479	+}
	480	+
	481	+static struct attribute_group **
	482	+arm_cspmu_alloc_attr_group(struct arm_cspmu *cspmu)
	483	+{
	484	+ struct attribute_group **attr_groups = NULL;
	485	+ struct device *dev = cspmu->dev;
	486	+ const struct arm_cspmu_impl_ops *impl_ops = &cspmu->impl.ops;
	487	+ int ret;
	488	+
	489	+ ret = arm_cspmu_init_impl_ops(cspmu);
	490	+ if (ret)
	491	+ return NULL;
	492	+
	493	+ cspmu->identifier = impl_ops->get_identifier(cspmu);
	494	+ cspmu->name = impl_ops->get_name(cspmu);
	495	+
	496	+ if (!cspmu->identifier \|\| !cspmu->name)
	497	+ return NULL;
	498	+
	499	+ attr_groups = devm_kcalloc(dev, 5, sizeof(struct attribute_group *),
	500	+ GFP_KERNEL);
	501	+ if (!attr_groups)
	502	+ return NULL;
	503	+
	504	+ attr_groups[0] = arm_cspmu_alloc_event_attr_group(cspmu);
	505	+ attr_groups[1] = arm_cspmu_alloc_format_attr_group(cspmu);
	506	+ attr_groups[2] = &arm_cspmu_identifier_attr_group;
	507	+ attr_groups[3] = &arm_cspmu_cpumask_attr_group;
	508	+
	509	+ if (!attr_groups[0] \|\| !attr_groups[1])
	510	+ return NULL;
	511	+
	512	+ return attr_groups;
	513	+}
	514	+
	515	+static inline void arm_cspmu_reset_counters(struct arm_cspmu *cspmu)
	516	+{
	517	+ u32 pmcr = 0;
	518	+
	519	+ pmcr \|= PMCR_P;
	520	+ pmcr \|= PMCR_C;
	521	+ writel(pmcr, cspmu->base0 + PMCR);
	522	+}
	523	+
	524	+static inline void arm_cspmu_start_counters(struct arm_cspmu *cspmu)
	525	+{
	526	+ writel(PMCR_E, cspmu->base0 + PMCR);
	527	+}
	528	+
	529	+static inline void arm_cspmu_stop_counters(struct arm_cspmu *cspmu)
	530	+{
	531	+ writel(0, cspmu->base0 + PMCR);
	532	+}
	533	+
	534	+static void arm_cspmu_enable(struct pmu *pmu)
	535	+{
	536	+ bool disabled;
	537	+ struct arm_cspmu *cspmu = to_arm_cspmu(pmu);
	538	+
	539	+ disabled = bitmap_empty(cspmu->hw_events.used_ctrs,
	540	+ cspmu->num_logical_ctrs);
	541	+
	542	+ if (disabled)
	543	+ return;
	544	+
	545	+ arm_cspmu_start_counters(cspmu);
	546	+}
	547	+
	548	+static void arm_cspmu_disable(struct pmu *pmu)
	549	+{
	550	+ struct arm_cspmu *cspmu = to_arm_cspmu(pmu);
	551	+
	552	+ arm_cspmu_stop_counters(cspmu);
	553	+}
	554	+
	555	+static int arm_cspmu_get_event_idx(struct arm_cspmu_hw_events *hw_events,
	556	+ struct perf_event *event)
	557	+{
	558	+ int idx;
	559	+ struct arm_cspmu *cspmu = to_arm_cspmu(event->pmu);
	560	+
	561	+ if (supports_cycle_counter(cspmu)) {
	562	+ if (cspmu->impl.ops.is_cycle_counter_event(event)) {
	563	+ /* Search for available cycle counter. */
	564	+ if (test_and_set_bit(cspmu->cycle_counter_logical_idx,
	565	+ hw_events->used_ctrs))
	566	+ return -EAGAIN;
	567	+
	568	+ return cspmu->cycle_counter_logical_idx;
	569	+ }
	570	+
	571	+ /*
	572	+ * Search a regular counter from the used counter bitmap.
	573	+ * The cycle counter divides the bitmap into two parts. Search
	574	+ * the first then second half to exclude the cycle counter bit.
	575	+ */
	576	+ idx = find_first_zero_bit(hw_events->used_ctrs,
	577	+ cspmu->cycle_counter_logical_idx);
	578	+ if (idx >= cspmu->cycle_counter_logical_idx) {
	579	+ idx = find_next_zero_bit(
	580	+ hw_events->used_ctrs,
	581	+ cspmu->num_logical_ctrs,
	582	+ cspmu->cycle_counter_logical_idx + 1);
	583	+ }
	584	+ } else {
	585	+ idx = find_first_zero_bit(hw_events->used_ctrs,
	586	+ cspmu->num_logical_ctrs);
	587	+ }
	588	+
	589	+ if (idx >= cspmu->num_logical_ctrs)
	590	+ return -EAGAIN;
	591	+
	592	+ set_bit(idx, hw_events->used_ctrs);
	593	+
	594	+ return idx;
	595	+}
	596	+
	597	+static bool arm_cspmu_validate_event(struct pmu *pmu,
	598	+ struct arm_cspmu_hw_events *hw_events,
	599	+ struct perf_event *event)
	600	+{
	601	+ if (is_software_event(event))
	602	+ return true;
	603	+
	604	+ /* Reject groups spanning multiple HW PMUs. */
	605	+ if (event->pmu != pmu)
	606	+ return false;
	607	+
	608	+ return (arm_cspmu_get_event_idx(hw_events, event) >= 0);
	609	+}
	610	+
	611	+/*
	612	+ * Make sure the group of events can be scheduled at once
	613	+ * on the PMU.
	614	+ */
	615	+static bool arm_cspmu_validate_group(struct perf_event *event)
	616	+{
	617	+ struct perf_event sibling, leader = event->group_leader;
	618	+ struct arm_cspmu_hw_events fake_hw_events;
	619	+
	620	+ if (event->group_leader == event)
	621	+ return true;
	622	+
	623	+ memset(&fake_hw_events, 0, sizeof(fake_hw_events));
	624	+
	625	+ if (!arm_cspmu_validate_event(event->pmu, &fake_hw_events, leader))
	626	+ return false;
	627	+
	628	+ for_each_sibling_event(sibling, leader) {
	629	+ if (!arm_cspmu_validate_event(event->pmu, &fake_hw_events,
	630	+ sibling))
	631	+ return false;
	632	+ }
	633	+
	634	+ return arm_cspmu_validate_event(event->pmu, &fake_hw_events, event);
	635	+}
	636	+
	637	+static int arm_cspmu_event_init(struct perf_event *event)
	638	+{
	639	+ struct arm_cspmu *cspmu;
	640	+ struct hw_perf_event *hwc = &event->hw;
	641	+
	642	+ cspmu = to_arm_cspmu(event->pmu);
	643	+
	644	+ /*
	645	+ * Following other "uncore" PMUs, we do not support sampling mode or
	646	+ * attach to a task (per-process mode).
	647	+ */
	648	+ if (is_sampling_event(event)) {
	649	+ dev_dbg(cspmu->pmu.dev,
	650	+ "Can't support sampling events\n");
	651	+ return -EOPNOTSUPP;
	652	+ }
	653	+
	654	+ if (event->cpu < 0 \|\| event->attach_state & PERF_ATTACH_TASK) {
	655	+ dev_dbg(cspmu->pmu.dev,
	656	+ "Can't support per-task counters\n");
	657	+ return -EINVAL;
	658	+ }
	659	+
	660	+ /*
	661	+ * Make sure the CPU assignment is on one of the CPUs associated with
	662	+ * this PMU.
	663	+ */
	664	+ if (!cpumask_test_cpu(event->cpu, &cspmu->associated_cpus)) {
	665	+ dev_dbg(cspmu->pmu.dev,
	666	+ "Requested cpu is not associated with the PMU\n");
	667	+ return -EINVAL;
	668	+ }
	669	+
	670	+ /* Enforce the current active CPU to handle the events in this PMU. */
	671	+ event->cpu = cpumask_first(&cspmu->active_cpu);
	672	+ if (event->cpu >= nr_cpu_ids)
	673	+ return -EINVAL;
	674	+
	675	+ if (!arm_cspmu_validate_group(event))
	676	+ return -EINVAL;
	677	+
	678	+ /*
	679	+ * The logical counter id is tracked with hw_perf_event.extra_reg.idx.
	680	+ * The physical counter id is tracked with hw_perf_event.idx.
	681	+ * We don't assign an index until we actually place the event onto
	682	+ * hardware. Use -1 to signify that we haven't decided where to put it
	683	+ * yet.
	684	+ */
	685	+ hwc->idx = -1;
	686	+ hwc->extra_reg.idx = -1;
	687	+ hwc->config = cspmu->impl.ops.event_type(event);
	688	+
	689	+ return 0;
	690	+}
	691	+
	692	+static inline u32 counter_offset(u32 reg_sz, u32 ctr_idx)
	693	+{
	694	+ return (PMEVCNTR_LO + (reg_sz * ctr_idx));
	695	+}
	696	+
	697	+static void arm_cspmu_write_counter(struct perf_event *event, u64 val)
	698	+{
	699	+ u32 offset;
	700	+ struct arm_cspmu *cspmu = to_arm_cspmu(event->pmu);
	701	+
	702	+ if (use_64b_counter_reg(cspmu)) {
	703	+ offset = counter_offset(sizeof(u64), event->hw.idx);
	704	+
	705	+ writeq(val, cspmu->base1 + offset);
	706	+ } else {
	707	+ offset = counter_offset(sizeof(u32), event->hw.idx);
	708	+
	709	+ writel(lower_32_bits(val), cspmu->base1 + offset);
	710	+ }
	711	+}
	712	+
	713	+static u64 arm_cspmu_read_counter(struct perf_event *event)
	714	+{
	715	+ u32 offset;
	716	+ const void __iomem *counter_addr;
	717	+ struct arm_cspmu *cspmu = to_arm_cspmu(event->pmu);
	718	+
	719	+ if (use_64b_counter_reg(cspmu)) {
	720	+ offset = counter_offset(sizeof(u64), event->hw.idx);
	721	+ counter_addr = cspmu->base1 + offset;
	722	+
	723	+ return supports_64bit_atomics(cspmu) ?
	724	+ readq(counter_addr) :
	725	+ read_reg64_hilohi(counter_addr, HILOHI_MAX_POLL);
	726	+ }
	727	+
	728	+ offset = counter_offset(sizeof(u32), event->hw.idx);
	729	+ return readl(cspmu->base1 + offset);
	730	+}
	731	+
	732	+/*
	733	+ * arm_cspmu_set_event_period: Set the period for the counter.
	734	+ *
	735	+ * To handle cases of extreme interrupt latency, we program
	736	+ * the counter with half of the max count for the counters.
	737	+ */
	738	+static void arm_cspmu_set_event_period(struct perf_event *event)
	739	+{
	740	+ struct arm_cspmu *cspmu = to_arm_cspmu(event->pmu);
	741	+ u64 val = counter_mask(cspmu) >> 1ULL;
	742	+
	743	+ local64_set(&event->hw.prev_count, val);
	744	+ arm_cspmu_write_counter(event, val);
	745	+}
	746	+
	747	+static void arm_cspmu_enable_counter(struct arm_cspmu *cspmu, int idx)
	748	+{
	749	+ u32 reg_id, reg_bit, inten_off, cnten_off;
	750	+
	751	+ reg_id = COUNTER_TO_SET_CLR_ID(idx);
	752	+ reg_bit = COUNTER_TO_SET_CLR_BIT(idx);
	753	+
	754	+ inten_off = PMINTENSET + (4 * reg_id);
	755	+ cnten_off = PMCNTENSET + (4 * reg_id);
	756	+
	757	+ writel(BIT(reg_bit), cspmu->base0 + inten_off);
	758	+ writel(BIT(reg_bit), cspmu->base0 + cnten_off);
	759	+}
	760	+
	761	+static void arm_cspmu_disable_counter(struct arm_cspmu *cspmu, int idx)
	762	+{
	763	+ u32 reg_id, reg_bit, inten_off, cnten_off;
	764	+
	765	+ reg_id = COUNTER_TO_SET_CLR_ID(idx);
	766	+ reg_bit = COUNTER_TO_SET_CLR_BIT(idx);
	767	+
	768	+ inten_off = PMINTENCLR + (4 * reg_id);
	769	+ cnten_off = PMCNTENCLR + (4 * reg_id);
	770	+
	771	+ writel(BIT(reg_bit), cspmu->base0 + cnten_off);
	772	+ writel(BIT(reg_bit), cspmu->base0 + inten_off);
	773	+}
	774	+
	775	+static void arm_cspmu_event_update(struct perf_event *event)
	776	+{
	777	+ struct arm_cspmu *cspmu = to_arm_cspmu(event->pmu);
	778	+ struct hw_perf_event *hwc = &event->hw;
	779	+ u64 delta, prev, now;
	780	+
	781	+ do {
	782	+ prev = local64_read(&hwc->prev_count);
	783	+ now = arm_cspmu_read_counter(event);
	784	+ } while (local64_cmpxchg(&hwc->prev_count, prev, now) != prev);
	785	+
	786	+ delta = (now - prev) & counter_mask(cspmu);
	787	+ local64_add(delta, &event->count);
	788	+}
	789	+
	790	+static inline void arm_cspmu_set_event(struct arm_cspmu *cspmu,
	791	+ struct hw_perf_event *hwc)
	792	+{
	793	+ u32 offset = PMEVTYPER + (4 * hwc->idx);
	794	+
	795	+ writel(hwc->config, cspmu->base0 + offset);
	796	+}
	797	+
	798	+static inline void arm_cspmu_set_ev_filter(struct arm_cspmu *cspmu,
	799	+ struct hw_perf_event *hwc,
	800	+ u32 filter)
	801	+{
	802	+ u32 offset = PMEVFILTR + (4 * hwc->idx);
	803	+
	804	+ writel(filter, cspmu->base0 + offset);
	805	+}
	806	+
	807	+static inline void arm_cspmu_set_cc_filter(struct arm_cspmu *cspmu, u32 filter)
	808	+{
	809	+ u32 offset = PMCCFILTR;
	810	+
	811	+ writel(filter, cspmu->base0 + offset);
	812	+}
	813	+
	814	+static void arm_cspmu_start(struct perf_event *event, int pmu_flags)
	815	+{
	816	+ struct arm_cspmu *cspmu = to_arm_cspmu(event->pmu);
	817	+ struct hw_perf_event *hwc = &event->hw;
	818	+ u32 filter;
	819	+
	820	+ /* We always reprogram the counter */
	821	+ if (pmu_flags & PERF_EF_RELOAD)
	822	+ WARN_ON(!(hwc->state & PERF_HES_UPTODATE));
	823	+
	824	+ arm_cspmu_set_event_period(event);
	825	+
	826	+ filter = cspmu->impl.ops.event_filter(event);
	827	+
	828	+ if (event->hw.extra_reg.idx == cspmu->cycle_counter_logical_idx) {
	829	+ arm_cspmu_set_cc_filter(cspmu, filter);
	830	+ } else {
	831	+ arm_cspmu_set_event(cspmu, hwc);
	832	+ arm_cspmu_set_ev_filter(cspmu, hwc, filter);
	833	+ }
	834	+
	835	+ hwc->state = 0;
	836	+
	837	+ arm_cspmu_enable_counter(cspmu, hwc->idx);
	838	+}
	839	+
	840	+static void arm_cspmu_stop(struct perf_event *event, int pmu_flags)
	841	+{
	842	+ struct arm_cspmu *cspmu = to_arm_cspmu(event->pmu);
	843	+ struct hw_perf_event *hwc = &event->hw;
	844	+
	845	+ if (hwc->state & PERF_HES_STOPPED)
	846	+ return;
	847	+
	848	+ arm_cspmu_disable_counter(cspmu, hwc->idx);
	849	+ arm_cspmu_event_update(event);
	850	+
	851	+ hwc->state \|= PERF_HES_STOPPED \| PERF_HES_UPTODATE;
	852	+}
	853	+
	854	+static inline u32 to_phys_idx(struct arm_cspmu *cspmu, u32 idx)
	855	+{
	856	+ return (idx == cspmu->cycle_counter_logical_idx) ?
	857	+ ARM_CSPMU_CYCLE_CNTR_IDX : idx;
	858	+}
	859	+
	860	+static int arm_cspmu_add(struct perf_event *event, int flags)
	861	+{
	862	+ struct arm_cspmu *cspmu = to_arm_cspmu(event->pmu);
	863	+ struct arm_cspmu_hw_events *hw_events = &cspmu->hw_events;
	864	+ struct hw_perf_event *hwc = &event->hw;
	865	+ int idx;
	866	+
	867	+ if (WARN_ON_ONCE(!cpumask_test_cpu(smp_processor_id(),
	868	+ &cspmu->associated_cpus)))
	869	+ return -ENOENT;
	870	+
	871	+ idx = arm_cspmu_get_event_idx(hw_events, event);
	872	+ if (idx < 0)
	873	+ return idx;
	874	+
	875	+ hw_events->events[idx] = event;
	876	+ hwc->idx = to_phys_idx(cspmu, idx);
	877	+ hwc->extra_reg.idx = idx;
	878	+ hwc->state = PERF_HES_STOPPED \| PERF_HES_UPTODATE;
	879	+
	880	+ if (flags & PERF_EF_START)
	881	+ arm_cspmu_start(event, PERF_EF_RELOAD);
	882	+
	883	+ /* Propagate changes to the userspace mapping. */
	884	+ perf_event_update_userpage(event);
	885	+
	886	+ return 0;
	887	+}
	888	+
	889	+static void arm_cspmu_del(struct perf_event *event, int flags)
	890	+{
	891	+ struct arm_cspmu *cspmu = to_arm_cspmu(event->pmu);
	892	+ struct arm_cspmu_hw_events *hw_events = &cspmu->hw_events;
	893	+ struct hw_perf_event *hwc = &event->hw;
	894	+ int idx = hwc->extra_reg.idx;
	895	+
	896	+ arm_cspmu_stop(event, PERF_EF_UPDATE);
	897	+
	898	+ hw_events->events[idx] = NULL;
	899	+
	900	+ clear_bit(idx, hw_events->used_ctrs);
	901	+
	902	+ perf_event_update_userpage(event);
	903	+}
	904	+
	905	+static void arm_cspmu_read(struct perf_event *event)
	906	+{
	907	+ arm_cspmu_event_update(event);
	908	+}
	909	+
	910	+static struct arm_cspmu arm_cspmu_alloc(struct platform_device pdev)
	911	+{
	912	+ struct acpi_apmt_node *apmt_node;
	913	+ struct arm_cspmu *cspmu;
	914	+ struct device *dev;
	915	+
	916	+ dev = &pdev->dev;
	917	+ apmt_node = (struct acpi_apmt_node *)dev_get_platdata(dev);
	918	+ if (!apmt_node) {
	919	+ dev_err(dev, "failed to get APMT node\n");
	920	+ return NULL;
	921	+ }
	922	+
	923	+ cspmu = devm_kzalloc(dev, sizeof(*cspmu), GFP_KERNEL);
	924	+ if (!cspmu)
	925	+ return NULL;
	926	+
	927	+ cspmu->dev = dev;
	928	+ cspmu->apmt_node = apmt_node;
	929	+
	930	+ platform_set_drvdata(pdev, cspmu);
	931	+
	932	+ return cspmu;
	933	+}
	934	+
	935	+static int arm_cspmu_init_mmio(struct arm_cspmu *cspmu)
	936	+{
	937	+ struct device *dev;
	938	+ struct platform_device *pdev;
	939	+ struct acpi_apmt_node *apmt_node;
	940	+
	941	+ dev = cspmu->dev;
	942	+ pdev = to_platform_device(dev);
	943	+ apmt_node = cspmu->apmt_node;
	944	+
	945	+ /* Base address for page 0. */
	946	+ cspmu->base0 = devm_platform_ioremap_resource(pdev, 0);
	947	+ if (IS_ERR(cspmu->base0)) {
	948	+ dev_err(dev, "ioremap failed for page-0 resource\n");
	949	+ return PTR_ERR(cspmu->base0);
	950	+ }
	951	+
	952	+ /* Base address for page 1 if supported. Otherwise point to page 0. */
	953	+ cspmu->base1 = cspmu->base0;
	954	+ if (CHECK_APMT_FLAG(apmt_node->flags, DUAL_PAGE, SUPP)) {
	955	+ cspmu->base1 = devm_platform_ioremap_resource(pdev, 1);
	956	+ if (IS_ERR(cspmu->base1)) {
	957	+ dev_err(dev, "ioremap failed for page-1 resource\n");
	958	+ return PTR_ERR(cspmu->base1);
	959	+ }
	960	+ }
	961	+
	962	+ cspmu->pmcfgr = readl(cspmu->base0 + PMCFGR);
	963	+
	964	+ cspmu->num_logical_ctrs = FIELD_GET(PMCFGR_N, cspmu->pmcfgr) + 1;
	965	+
	966	+ cspmu->cycle_counter_logical_idx = ARM_CSPMU_MAX_HW_CNTRS;
	967	+
	968	+ if (supports_cycle_counter(cspmu)) {
	969	+ /*
	970	+ * The last logical counter is mapped to cycle counter if
	971	+ * there is a gap between regular and cycle counter. Otherwise,
	972	+ * logical and physical have 1-to-1 mapping.
	973	+ */
	974	+ cspmu->cycle_counter_logical_idx =
	975	+ (cspmu->num_logical_ctrs <= ARM_CSPMU_CYCLE_CNTR_IDX) ?
	976	+ cspmu->num_logical_ctrs - 1 :
	977	+ ARM_CSPMU_CYCLE_CNTR_IDX;
	978	+ }
	979	+
	980	+ cspmu->num_set_clr_reg =
	981	+ DIV_ROUND_UP(cspmu->num_logical_ctrs,
	982	+ ARM_CSPMU_SET_CLR_COUNTER_NUM);
	983	+
	984	+ cspmu->hw_events.events =
	985	+ devm_kcalloc(dev, cspmu->num_logical_ctrs,
	986	+ sizeof(*cspmu->hw_events.events), GFP_KERNEL);
	987	+
	988	+ if (!cspmu->hw_events.events)
	989	+ return -ENOMEM;
	990	+
	991	+ return 0;
	992	+}
	993	+
	994	+static inline int arm_cspmu_get_reset_overflow(struct arm_cspmu *cspmu,
	995	+ u32 *pmovs)
	996	+{
	997	+ int i;
	998	+ u32 pmovclr_offset = PMOVSCLR;
	999	+ u32 has_overflowed = 0;
	1000	+
	1001	+ for (i = 0; i < cspmu->num_set_clr_reg; ++i) {
	1002	+ pmovs[i] = readl(cspmu->base1 + pmovclr_offset);
	1003	+ has_overflowed \|= pmovs[i];
	1004	+ writel(pmovs[i], cspmu->base1 + pmovclr_offset);
	1005	+ pmovclr_offset += sizeof(u32);
	1006	+ }
	1007	+
	1008	+ return has_overflowed != 0;
	1009	+}
	1010	+
	1011	+static irqreturn_t arm_cspmu_handle_irq(int irq_num, void *dev)
	1012	+{
	1013	+ int idx, has_overflowed;
	1014	+ struct perf_event *event;
	1015	+ struct arm_cspmu *cspmu = dev;
	1016	+ DECLARE_BITMAP(pmovs, ARM_CSPMU_MAX_HW_CNTRS);
	1017	+ bool handled = false;
	1018	+
	1019	+ arm_cspmu_stop_counters(cspmu);
	1020	+
	1021	+ has_overflowed = arm_cspmu_get_reset_overflow(cspmu, (u32 *)pmovs);
	1022	+ if (!has_overflowed)
	1023	+ goto done;
	1024	+
	1025	+ for_each_set_bit(idx, cspmu->hw_events.used_ctrs,
	1026	+ cspmu->num_logical_ctrs) {
	1027	+ event = cspmu->hw_events.events[idx];
	1028	+
	1029	+ if (!event)
	1030	+ continue;
	1031	+
	1032	+ if (!test_bit(event->hw.idx, pmovs))
	1033	+ continue;
	1034	+
	1035	+ arm_cspmu_event_update(event);
	1036	+ arm_cspmu_set_event_period(event);
	1037	+
	1038	+ handled = true;
	1039	+ }
	1040	+
	1041	+done:
	1042	+ arm_cspmu_start_counters(cspmu);
	1043	+ return IRQ_RETVAL(handled);
	1044	+}
	1045	+
	1046	+static int arm_cspmu_request_irq(struct arm_cspmu *cspmu)
	1047	+{
	1048	+ int irq, ret;
	1049	+ struct device *dev;
	1050	+ struct platform_device *pdev;
	1051	+ struct acpi_apmt_node *apmt_node;
	1052	+
	1053	+ dev = cspmu->dev;
	1054	+ pdev = to_platform_device(dev);
	1055	+ apmt_node = cspmu->apmt_node;
	1056	+
	1057	+ /* Skip IRQ request if the PMU does not support overflow interrupt. */
	1058	+ if (apmt_node->ovflw_irq == 0)
	1059	+ return 0;
	1060	+
	1061	+ irq = platform_get_irq(pdev, 0);
	1062	+ if (irq < 0)
	1063	+ return irq;
	1064	+
	1065	+ ret = devm_request_irq(dev, irq, arm_cspmu_handle_irq,
	1066	+ IRQF_NOBALANCING \| IRQF_NO_THREAD, dev_name(dev),
	1067	+ cspmu);
	1068	+ if (ret) {
	1069	+ dev_err(dev, "Could not request IRQ %d\n", irq);
	1070	+ return ret;
	1071	+ }
	1072	+
	1073	+ cspmu->irq = irq;
	1074	+
	1075	+ return 0;
	1076	+}
	1077	+
	1078	+static inline int arm_cspmu_find_cpu_container(int cpu, u32 container_uid)
	1079	+{
	1080	+ u32 acpi_uid;
	1081	+ struct device *cpu_dev = get_cpu_device(cpu);
	1082	+ struct acpi_device *acpi_dev = ACPI_COMPANION(cpu_dev);
	1083	+
	1084	+ if (!cpu_dev)
	1085	+ return -ENODEV;
	1086	+
	1087	+ while (acpi_dev) {
	1088	+ if (!strcmp(acpi_device_hid(acpi_dev),
	1089	+ ACPI_PROCESSOR_CONTAINER_HID) &&
	1090	+ !kstrtouint(acpi_device_uid(acpi_dev), 0, &acpi_uid) &&
	1091	+ acpi_uid == container_uid)
	1092	+ return 0;
	1093	+
	1094	+ acpi_dev = acpi_dev_parent(acpi_dev);
	1095	+ }
	1096	+
	1097	+ return -ENODEV;
	1098	+}
	1099	+
	1100	+static int arm_cspmu_get_cpus(struct arm_cspmu *cspmu)
	1101	+{
	1102	+ struct device *dev;
	1103	+ struct acpi_apmt_node *apmt_node;
	1104	+ int affinity_flag;
	1105	+ int cpu;
	1106	+
	1107	+ dev = cspmu->pmu.dev;
	1108	+ apmt_node = cspmu->apmt_node;
	1109	+ affinity_flag = apmt_node->flags & ACPI_APMT_FLAGS_AFFINITY;
	1110	+
	1111	+ if (affinity_flag == ACPI_APMT_FLAGS_AFFINITY_PROC) {
	1112	+ for_each_possible_cpu(cpu) {
	1113	+ if (apmt_node->proc_affinity ==
	1114	+ get_acpi_id_for_cpu(cpu)) {
	1115	+ cpumask_set_cpu(cpu, &cspmu->associated_cpus);
	1116	+ break;
	1117	+ }
	1118	+ }
	1119	+ } else {
	1120	+ for_each_possible_cpu(cpu) {
	1121	+ if (arm_cspmu_find_cpu_container(
	1122	+ cpu, apmt_node->proc_affinity))
	1123	+ continue;
	1124	+
	1125	+ cpumask_set_cpu(cpu, &cspmu->associated_cpus);
	1126	+ }
	1127	+ }
	1128	+
	1129	+ if (cpumask_empty(&cspmu->associated_cpus)) {
	1130	+ dev_dbg(dev, "No cpu associated with the PMU\n");
	1131	+ return -ENODEV;
	1132	+ }
	1133	+
	1134	+ return 0;
	1135	+}
	1136	+
	1137	+static int arm_cspmu_register_pmu(struct arm_cspmu *cspmu)
	1138	+{
	1139	+ int ret, capabilities;
	1140	+ struct attribute_group **attr_groups;
	1141	+
	1142	+ attr_groups = arm_cspmu_alloc_attr_group(cspmu);
	1143	+ if (!attr_groups)
	1144	+ return -ENOMEM;
	1145	+
	1146	+ ret = cpuhp_state_add_instance(arm_cspmu_cpuhp_state,
	1147	+ &cspmu->cpuhp_node);
	1148	+ if (ret)
	1149	+ return ret;
	1150	+
	1151	+ capabilities = PERF_PMU_CAP_NO_EXCLUDE;
	1152	+ if (cspmu->irq == 0)
	1153	+ capabilities \|= PERF_PMU_CAP_NO_INTERRUPT;
	1154	+
	1155	+ cspmu->pmu = (struct pmu){
	1156	+ .task_ctx_nr = perf_invalid_context,
	1157	+ .module = THIS_MODULE,
	1158	+ .pmu_enable = arm_cspmu_enable,
	1159	+ .pmu_disable = arm_cspmu_disable,
	1160	+ .event_init = arm_cspmu_event_init,
	1161	+ .add = arm_cspmu_add,
	1162	+ .del = arm_cspmu_del,
	1163	+ .start = arm_cspmu_start,
	1164	+ .stop = arm_cspmu_stop,
	1165	+ .read = arm_cspmu_read,
	1166	+ .attr_groups = (const struct attribute_group **)attr_groups,
	1167	+ .capabilities = capabilities,
	1168	+ };
	1169	+
	1170	+ /* Hardware counter init */
	1171	+ arm_cspmu_stop_counters(cspmu);
	1172	+ arm_cspmu_reset_counters(cspmu);
	1173	+
	1174	+ ret = perf_pmu_register(&cspmu->pmu, cspmu->name, -1);
	1175	+ if (ret) {
	1176	+ cpuhp_state_remove_instance(arm_cspmu_cpuhp_state,
	1177	+ &cspmu->cpuhp_node);
	1178	+ }
	1179	+
	1180	+ return ret;
	1181	+}
	1182	+
	1183	+static int arm_cspmu_device_probe(struct platform_device *pdev)
	1184	+{
	1185	+ int ret;
	1186	+ struct arm_cspmu *cspmu;
	1187	+
	1188	+ cspmu = arm_cspmu_alloc(pdev);
	1189	+ if (!cspmu)
	1190	+ return -ENOMEM;
	1191	+
	1192	+ ret = arm_cspmu_init_mmio(cspmu);
	1193	+ if (ret)
	1194	+ return ret;
	1195	+
	1196	+ ret = arm_cspmu_request_irq(cspmu);
	1197	+ if (ret)
	1198	+ return ret;
	1199	+
	1200	+ ret = arm_cspmu_get_cpus(cspmu);
	1201	+ if (ret)
	1202	+ return ret;
	1203	+
	1204	+ ret = arm_cspmu_register_pmu(cspmu);
	1205	+ if (ret)
	1206	+ return ret;
	1207	+
	1208	+ return 0;
	1209	+}
	1210	+
	1211	+static int arm_cspmu_device_remove(struct platform_device *pdev)
	1212	+{
	1213	+ struct arm_cspmu *cspmu = platform_get_drvdata(pdev);
	1214	+
	1215	+ perf_pmu_unregister(&cspmu->pmu);
	1216	+ cpuhp_state_remove_instance(arm_cspmu_cpuhp_state, &cspmu->cpuhp_node);
	1217	+
	1218	+ return 0;
	1219	+}
	1220	+
	1221	+static struct platform_driver arm_cspmu_driver = {
	1222	+ .driver = {
	1223	+ .name = DRVNAME,
	1224	+ .suppress_bind_attrs = true,
	1225	+ },
	1226	+ .probe = arm_cspmu_device_probe,
	1227	+ .remove = arm_cspmu_device_remove,
	1228	+};
	1229	+
	1230	+static void arm_cspmu_set_active_cpu(int cpu, struct arm_cspmu *cspmu)
	1231	+{
	1232	+ cpumask_set_cpu(cpu, &cspmu->active_cpu);
	1233	+ WARN_ON(irq_set_affinity(cspmu->irq, &cspmu->active_cpu));
	1234	+}
	1235	+
	1236	+static int arm_cspmu_cpu_online(unsigned int cpu, struct hlist_node *node)
	1237	+{
	1238	+ struct arm_cspmu *cspmu =
	1239	+ hlist_entry_safe(node, struct arm_cspmu, cpuhp_node);
	1240	+
	1241	+ if (!cpumask_test_cpu(cpu, &cspmu->associated_cpus))
	1242	+ return 0;
	1243	+
	1244	+ /* If the PMU is already managed, there is nothing to do */
	1245	+ if (!cpumask_empty(&cspmu->active_cpu))
	1246	+ return 0;
	1247	+
	1248	+ /* Use this CPU for event counting */
	1249	+ arm_cspmu_set_active_cpu(cpu, cspmu);
	1250	+
	1251	+ return 0;
	1252	+}
	1253	+
	1254	+static int arm_cspmu_cpu_teardown(unsigned int cpu, struct hlist_node *node)
	1255	+{
	1256	+ int dst;
	1257	+ struct cpumask online_supported;
	1258	+
	1259	+ struct arm_cspmu *cspmu =
	1260	+ hlist_entry_safe(node, struct arm_cspmu, cpuhp_node);
	1261	+
	1262	+ /* Nothing to do if this CPU doesn't own the PMU */
	1263	+ if (!cpumask_test_and_clear_cpu(cpu, &cspmu->active_cpu))
	1264	+ return 0;
	1265	+
	1266	+ /* Choose a new CPU to migrate ownership of the PMU to */
	1267	+ cpumask_and(&online_supported, &cspmu->associated_cpus,
	1268	+ cpu_online_mask);
	1269	+ dst = cpumask_any_but(&online_supported, cpu);
	1270	+ if (dst >= nr_cpu_ids)
	1271	+ return 0;
	1272	+
	1273	+ /* Use this CPU for event counting */
	1274	+ perf_pmu_migrate_context(&cspmu->pmu, cpu, dst);
	1275	+ arm_cspmu_set_active_cpu(dst, cspmu);
	1276	+
	1277	+ return 0;
	1278	+}
	1279	+
	1280	+static int __init arm_cspmu_init(void)
	1281	+{
	1282	+ int ret;
	1283	+
	1284	+ ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN,
	1285	+ "perf/arm/cspmu:online",
	1286	+ arm_cspmu_cpu_online,
	1287	+ arm_cspmu_cpu_teardown);
	1288	+ if (ret < 0)
	1289	+ return ret;
	1290	+ arm_cspmu_cpuhp_state = ret;
	1291	+ return platform_driver_register(&arm_cspmu_driver);
	1292	+}
	1293	+
	1294	+static void __exit arm_cspmu_exit(void)
	1295	+{
	1296	+ platform_driver_unregister(&arm_cspmu_driver);
	1297	+ cpuhp_remove_multi_state(arm_cspmu_cpuhp_state);
	1298	+}
	1299	+
	1300	+module_init(arm_cspmu_init);
	1301	+module_exit(arm_cspmu_exit);
	1302	+
	1303	+MODULE_LICENSE("GPL v2");

drivers/perf/arm_cspmu/arm_cspmu.h+151−0

@@ -0,0 +1,151 @@
	1	+/* SPDX-License-Identifier: GPL-2.0
	2	+ *
	3	+ * ARM CoreSight Architecture PMU driver.
	4	+ * Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
	5	+ *
	6	+ */
	7	+
	8	+#ifndef __ARM_CSPMU_H__
	9	+#define __ARM_CSPMU_H__
	10	+
	11	+#include <linux/acpi.h>
	12	+#include <linux/bitfield.h>
	13	+#include <linux/cpumask.h>
	14	+#include <linux/device.h>
	15	+#include <linux/kernel.h>
	16	+#include <linux/module.h>
	17	+#include <linux/perf_event.h>
	18	+#include <linux/platform_device.h>
	19	+#include <linux/types.h>
	20	+
	21	+#define to_arm_cspmu(p) (container_of(p, struct arm_cspmu, pmu))
	22	+
	23	+#define ARM_CSPMU_EXT_ATTR(_name, _func, _config) \
	24	+ (&((struct dev_ext_attribute[]){ \
	25	+ { \
	26	+ .attr = __ATTR(_name, 0444, _func, NULL), \
	27	+ .var = (void *)_config \
	28	+ } \
	29	+ })[0].attr.attr)
	30	+
	31	+#define ARM_CSPMU_FORMAT_ATTR(_name, _config) \
	32	+ ARM_CSPMU_EXT_ATTR(_name, arm_cspmu_sysfs_format_show, (char *)_config)
	33	+
	34	+#define ARM_CSPMU_EVENT_ATTR(_name, _config) \
	35	+ PMU_EVENT_ATTR_ID(_name, arm_cspmu_sysfs_event_show, _config)
	36	+
	37	+
	38	+/* Default event id mask */
	39	+#define ARM_CSPMU_EVENT_MASK GENMASK_ULL(63, 0)
	40	+
	41	+/* Default filter value mask */
	42	+#define ARM_CSPMU_FILTER_MASK GENMASK_ULL(63, 0)
	43	+
	44	+/* Default event format */
	45	+#define ARM_CSPMU_FORMAT_EVENT_ATTR \
	46	+ ARM_CSPMU_FORMAT_ATTR(event, "config:0-32")
	47	+
	48	+/* Default filter format */
	49	+#define ARM_CSPMU_FORMAT_FILTER_ATTR \
	50	+ ARM_CSPMU_FORMAT_ATTR(filter, "config1:0-31")
	51	+
	52	+/*
	53	+ * This is the default event number for cycle count, if supported, since the
	54	+ * ARM Coresight PMU specification does not define a standard event code
	55	+ * for cycle count.
	56	+ */
	57	+#define ARM_CSPMU_EVT_CYCLES_DEFAULT (0x1ULL << 32)
	58	+
	59	+/*
	60	+ * The ARM Coresight PMU supports up to 256 event counters.
	61	+ * If the counters are larger-than 32-bits, then the PMU includes at
	62	+ * most 128 counters.
	63	+ */
	64	+#define ARM_CSPMU_MAX_HW_CNTRS 256
	65	+
	66	+/* The cycle counter, if implemented, is located at counter[31]. */
	67	+#define ARM_CSPMU_CYCLE_CNTR_IDX 31
	68	+
	69	+/* PMIIDR register field */
	70	+#define ARM_CSPMU_PMIIDR_IMPLEMENTER GENMASK(11, 0)
	71	+#define ARM_CSPMU_PMIIDR_PRODUCTID GENMASK(31, 20)
	72	+
	73	+struct arm_cspmu;
	74	+
	75	+/* This tracks the events assigned to each counter in the PMU. */
	76	+struct arm_cspmu_hw_events {
	77	+ /* The events that are active on the PMU for a given logical index. */
	78	+ struct perf_event **events;
	79	+
	80	+ /*
	81	+ * Each bit indicates a logical counter is being used (or not) for an
	82	+ * event. If cycle counter is supported and there is a gap between
	83	+ * regular and cycle counter, the last logical counter is mapped to
	84	+ * cycle counter. Otherwise, logical and physical have 1-to-1 mapping.
	85	+ */
	86	+ DECLARE_BITMAP(used_ctrs, ARM_CSPMU_MAX_HW_CNTRS);
	87	+};
	88	+
	89	+/* Contains ops to query vendor/implementer specific attribute. */
	90	+struct arm_cspmu_impl_ops {
	91	+ /* Get event attributes */
	92	+ struct attribute *(get_event_attrs)(const struct arm_cspmu *cspmu);
	93	+ /* Get format attributes */
	94	+ struct attribute *(get_format_attrs)(const struct arm_cspmu *cspmu);
	95	+ /* Get string identifier */
	96	+ const char (get_identifier)(const struct arm_cspmu *cspmu);
	97	+ /* Get PMU name to register to core perf */
	98	+ const char (get_name)(const struct arm_cspmu *cspmu);
	99	+ /* Check if the event corresponds to cycle count event */
	100	+ bool (is_cycle_counter_event)(const struct perf_event event);
	101	+ /* Decode event type/id from configs */
	102	+ u32 (event_type)(const struct perf_event event);
	103	+ /* Decode filter value from configs */
	104	+ u32 (event_filter)(const struct perf_event event);
	105	+ /* Hide/show unsupported events */
	106	+ umode_t (event_attr_is_visible)(struct kobject kobj,
	107	+ struct attribute *attr, int unused);
	108	+};
	109	+
	110	+/* Vendor/implementer descriptor. */
	111	+struct arm_cspmu_impl {
	112	+ u32 pmiidr;
	113	+ struct arm_cspmu_impl_ops ops;
	114	+ void *ctx;
	115	+};
	116	+
	117	+/* Coresight PMU descriptor. */
	118	+struct arm_cspmu {
	119	+ struct pmu pmu;
	120	+ struct device *dev;
	121	+ struct acpi_apmt_node *apmt_node;
	122	+ const char *name;
	123	+ const char *identifier;
	124	+ void __iomem *base0;
	125	+ void __iomem *base1;
	126	+ int irq;
	127	+ cpumask_t associated_cpus;
	128	+ cpumask_t active_cpu;
	129	+ struct hlist_node cpuhp_node;
	130	+
	131	+ u32 pmcfgr;
	132	+ u32 num_logical_ctrs;
	133	+ u32 num_set_clr_reg;
	134	+ int cycle_counter_logical_idx;
	135	+
	136	+ struct arm_cspmu_hw_events hw_events;
	137	+
	138	+ struct arm_cspmu_impl impl;
	139	+};
	140	+
	141	+/* Default function to show event attribute in sysfs. */
	142	+ssize_t arm_cspmu_sysfs_event_show(struct device *dev,
	143	+ struct device_attribute *attr,
	144	+ char *buf);
	145	+
	146	+/* Default function to show format attribute in sysfs. */
	147	+ssize_t arm_cspmu_sysfs_format_show(struct device *dev,
	148	+ struct device_attribute *attr,
	149	+ char *buf);
	150	+
	151	+#endif /* __ARM_CSPMU_H__ */

drivers/perf/arm_cspmu/nvidia_cspmu.c+400−0

@@ -0,0 +1,400 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+/*
	3	+ * Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
	4	+ *
	5	+ */
	6	+
	7	+/* Support for NVIDIA specific attributes. */
	8	+
	9	+#include <linux/topology.h>
	10	+
	11	+#include "nvidia_cspmu.h"
	12	+
	13	+#define NV_PCIE_PORT_COUNT 10ULL
	14	+#define NV_PCIE_FILTER_ID_MASK GENMASK_ULL(NV_PCIE_PORT_COUNT - 1, 0)
	15	+
	16	+#define NV_NVL_C2C_PORT_COUNT 2ULL
	17	+#define NV_NVL_C2C_FILTER_ID_MASK GENMASK_ULL(NV_NVL_C2C_PORT_COUNT - 1, 0)
	18	+
	19	+#define NV_CNVL_PORT_COUNT 4ULL
	20	+#define NV_CNVL_FILTER_ID_MASK GENMASK_ULL(NV_CNVL_PORT_COUNT - 1, 0)
	21	+
	22	+#define NV_GENERIC_FILTER_ID_MASK GENMASK_ULL(31, 0)
	23	+
	24	+#define NV_PRODID_MASK GENMASK(31, 0)
	25	+
	26	+#define NV_FORMAT_NAME_GENERIC 0
	27	+
	28	+#define to_nv_cspmu_ctx(cspmu) ((struct nv_cspmu_ctx *)(cspmu->impl.ctx))
	29	+
	30	+#define NV_CSPMU_EVENT_ATTR_4_INNER(_pref, _num, _suff, _config) \
	31	+ ARM_CSPMU_EVENT_ATTR(_pref##_num##_suff, _config)
	32	+
	33	+#define NV_CSPMU_EVENT_ATTR_4(_pref, _suff, _config) \
	34	+ NV_CSPMU_EVENT_ATTR_4_INNER(_pref, _0_, _suff, _config), \
	35	+ NV_CSPMU_EVENT_ATTR_4_INNER(_pref, _1_, _suff, _config + 1), \
	36	+ NV_CSPMU_EVENT_ATTR_4_INNER(_pref, _2_, _suff, _config + 2), \
	37	+ NV_CSPMU_EVENT_ATTR_4_INNER(_pref, _3_, _suff, _config + 3)
	38	+
	39	+struct nv_cspmu_ctx {
	40	+ const char *name;
	41	+ u32 filter_mask;
	42	+ u32 filter_default_val;
	43	+ struct attribute **event_attr;
	44	+ struct attribute **format_attr;
	45	+};
	46	+
	47	+static struct attribute *scf_pmu_event_attrs[] = {
	48	+ ARM_CSPMU_EVENT_ATTR(bus_cycles, 0x1d),
	49	+
	50	+ ARM_CSPMU_EVENT_ATTR(scf_cache_allocate, 0xF0),
	51	+ ARM_CSPMU_EVENT_ATTR(scf_cache_refill, 0xF1),
	52	+ ARM_CSPMU_EVENT_ATTR(scf_cache, 0xF2),
	53	+ ARM_CSPMU_EVENT_ATTR(scf_cache_wb, 0xF3),
	54	+
	55	+ NV_CSPMU_EVENT_ATTR_4(socket, rd_data, 0x101),
	56	+ NV_CSPMU_EVENT_ATTR_4(socket, dl_rsp, 0x105),
	57	+ NV_CSPMU_EVENT_ATTR_4(socket, wb_data, 0x109),
	58	+ NV_CSPMU_EVENT_ATTR_4(socket, ev_rsp, 0x10d),
	59	+ NV_CSPMU_EVENT_ATTR_4(socket, prb_data, 0x111),
	60	+
	61	+ NV_CSPMU_EVENT_ATTR_4(socket, rd_outstanding, 0x115),
	62	+ NV_CSPMU_EVENT_ATTR_4(socket, dl_outstanding, 0x119),
	63	+ NV_CSPMU_EVENT_ATTR_4(socket, wb_outstanding, 0x11d),
	64	+ NV_CSPMU_EVENT_ATTR_4(socket, wr_outstanding, 0x121),
	65	+ NV_CSPMU_EVENT_ATTR_4(socket, ev_outstanding, 0x125),
	66	+ NV_CSPMU_EVENT_ATTR_4(socket, prb_outstanding, 0x129),
	67	+
	68	+ NV_CSPMU_EVENT_ATTR_4(socket, rd_access, 0x12d),
	69	+ NV_CSPMU_EVENT_ATTR_4(socket, dl_access, 0x131),
	70	+ NV_CSPMU_EVENT_ATTR_4(socket, wb_access, 0x135),
	71	+ NV_CSPMU_EVENT_ATTR_4(socket, wr_access, 0x139),
	72	+ NV_CSPMU_EVENT_ATTR_4(socket, ev_access, 0x13d),
	73	+ NV_CSPMU_EVENT_ATTR_4(socket, prb_access, 0x141),
	74	+
	75	+ NV_CSPMU_EVENT_ATTR_4(ocu, gmem_rd_data, 0x145),
	76	+ NV_CSPMU_EVENT_ATTR_4(ocu, gmem_rd_access, 0x149),
	77	+ NV_CSPMU_EVENT_ATTR_4(ocu, gmem_wb_access, 0x14d),
	78	+ NV_CSPMU_EVENT_ATTR_4(ocu, gmem_rd_outstanding, 0x151),
	79	+ NV_CSPMU_EVENT_ATTR_4(ocu, gmem_wr_outstanding, 0x155),
	80	+
	81	+ NV_CSPMU_EVENT_ATTR_4(ocu, rem_rd_data, 0x159),
	82	+ NV_CSPMU_EVENT_ATTR_4(ocu, rem_rd_access, 0x15d),
	83	+ NV_CSPMU_EVENT_ATTR_4(ocu, rem_wb_access, 0x161),
	84	+ NV_CSPMU_EVENT_ATTR_4(ocu, rem_rd_outstanding, 0x165),
	85	+ NV_CSPMU_EVENT_ATTR_4(ocu, rem_wr_outstanding, 0x169),
	86	+
	87	+ ARM_CSPMU_EVENT_ATTR(gmem_rd_data, 0x16d),
	88	+ ARM_CSPMU_EVENT_ATTR(gmem_rd_access, 0x16e),
	89	+ ARM_CSPMU_EVENT_ATTR(gmem_rd_outstanding, 0x16f),
	90	+ ARM_CSPMU_EVENT_ATTR(gmem_dl_rsp, 0x170),
	91	+ ARM_CSPMU_EVENT_ATTR(gmem_dl_access, 0x171),
	92	+ ARM_CSPMU_EVENT_ATTR(gmem_dl_outstanding, 0x172),
	93	+ ARM_CSPMU_EVENT_ATTR(gmem_wb_data, 0x173),
	94	+ ARM_CSPMU_EVENT_ATTR(gmem_wb_access, 0x174),
	95	+ ARM_CSPMU_EVENT_ATTR(gmem_wb_outstanding, 0x175),
	96	+ ARM_CSPMU_EVENT_ATTR(gmem_ev_rsp, 0x176),
	97	+ ARM_CSPMU_EVENT_ATTR(gmem_ev_access, 0x177),
	98	+ ARM_CSPMU_EVENT_ATTR(gmem_ev_outstanding, 0x178),
	99	+ ARM_CSPMU_EVENT_ATTR(gmem_wr_data, 0x179),
	100	+ ARM_CSPMU_EVENT_ATTR(gmem_wr_outstanding, 0x17a),
	101	+ ARM_CSPMU_EVENT_ATTR(gmem_wr_access, 0x17b),
	102	+
	103	+ NV_CSPMU_EVENT_ATTR_4(socket, wr_data, 0x17c),
	104	+
	105	+ NV_CSPMU_EVENT_ATTR_4(ocu, gmem_wr_data, 0x180),
	106	+ NV_CSPMU_EVENT_ATTR_4(ocu, gmem_wb_data, 0x184),
	107	+ NV_CSPMU_EVENT_ATTR_4(ocu, gmem_wr_access, 0x188),
	108	+ NV_CSPMU_EVENT_ATTR_4(ocu, gmem_wb_outstanding, 0x18c),
	109	+
	110	+ NV_CSPMU_EVENT_ATTR_4(ocu, rem_wr_data, 0x190),
	111	+ NV_CSPMU_EVENT_ATTR_4(ocu, rem_wb_data, 0x194),
	112	+ NV_CSPMU_EVENT_ATTR_4(ocu, rem_wr_access, 0x198),
	113	+ NV_CSPMU_EVENT_ATTR_4(ocu, rem_wb_outstanding, 0x19c),
	114	+
	115	+ ARM_CSPMU_EVENT_ATTR(gmem_wr_total_bytes, 0x1a0),
	116	+ ARM_CSPMU_EVENT_ATTR(remote_socket_wr_total_bytes, 0x1a1),
	117	+ ARM_CSPMU_EVENT_ATTR(remote_socket_rd_data, 0x1a2),
	118	+ ARM_CSPMU_EVENT_ATTR(remote_socket_rd_outstanding, 0x1a3),
	119	+ ARM_CSPMU_EVENT_ATTR(remote_socket_rd_access, 0x1a4),
	120	+
	121	+ ARM_CSPMU_EVENT_ATTR(cmem_rd_data, 0x1a5),
	122	+ ARM_CSPMU_EVENT_ATTR(cmem_rd_access, 0x1a6),
	123	+ ARM_CSPMU_EVENT_ATTR(cmem_rd_outstanding, 0x1a7),
	124	+ ARM_CSPMU_EVENT_ATTR(cmem_dl_rsp, 0x1a8),
	125	+ ARM_CSPMU_EVENT_ATTR(cmem_dl_access, 0x1a9),
	126	+ ARM_CSPMU_EVENT_ATTR(cmem_dl_outstanding, 0x1aa),
	127	+ ARM_CSPMU_EVENT_ATTR(cmem_wb_data, 0x1ab),
	128	+ ARM_CSPMU_EVENT_ATTR(cmem_wb_access, 0x1ac),
	129	+ ARM_CSPMU_EVENT_ATTR(cmem_wb_outstanding, 0x1ad),
	130	+ ARM_CSPMU_EVENT_ATTR(cmem_ev_rsp, 0x1ae),
	131	+ ARM_CSPMU_EVENT_ATTR(cmem_ev_access, 0x1af),
	132	+ ARM_CSPMU_EVENT_ATTR(cmem_ev_outstanding, 0x1b0),
	133	+ ARM_CSPMU_EVENT_ATTR(cmem_wr_data, 0x1b1),
	134	+ ARM_CSPMU_EVENT_ATTR(cmem_wr_outstanding, 0x1b2),
	135	+
	136	+ NV_CSPMU_EVENT_ATTR_4(ocu, cmem_rd_data, 0x1b3),
	137	+ NV_CSPMU_EVENT_ATTR_4(ocu, cmem_rd_access, 0x1b7),
	138	+ NV_CSPMU_EVENT_ATTR_4(ocu, cmem_wb_access, 0x1bb),
	139	+ NV_CSPMU_EVENT_ATTR_4(ocu, cmem_rd_outstanding, 0x1bf),
	140	+ NV_CSPMU_EVENT_ATTR_4(ocu, cmem_wr_outstanding, 0x1c3),
	141	+
	142	+ ARM_CSPMU_EVENT_ATTR(ocu_prb_access, 0x1c7),
	143	+ ARM_CSPMU_EVENT_ATTR(ocu_prb_data, 0x1c8),
	144	+ ARM_CSPMU_EVENT_ATTR(ocu_prb_outstanding, 0x1c9),
	145	+
	146	+ ARM_CSPMU_EVENT_ATTR(cmem_wr_access, 0x1ca),
	147	+
	148	+ NV_CSPMU_EVENT_ATTR_4(ocu, cmem_wr_access, 0x1cb),
	149	+ NV_CSPMU_EVENT_ATTR_4(ocu, cmem_wb_data, 0x1cf),
	150	+ NV_CSPMU_EVENT_ATTR_4(ocu, cmem_wr_data, 0x1d3),
	151	+ NV_CSPMU_EVENT_ATTR_4(ocu, cmem_wb_outstanding, 0x1d7),
	152	+
	153	+ ARM_CSPMU_EVENT_ATTR(cmem_wr_total_bytes, 0x1db),
	154	+
	155	+ ARM_CSPMU_EVENT_ATTR(cycles, ARM_CSPMU_EVT_CYCLES_DEFAULT),
	156	+ NULL,
	157	+};
	158	+
	159	+static struct attribute *mcf_pmu_event_attrs[] = {
	160	+ ARM_CSPMU_EVENT_ATTR(rd_bytes_loc, 0x0),
	161	+ ARM_CSPMU_EVENT_ATTR(rd_bytes_rem, 0x1),
	162	+ ARM_CSPMU_EVENT_ATTR(wr_bytes_loc, 0x2),
	163	+ ARM_CSPMU_EVENT_ATTR(wr_bytes_rem, 0x3),
	164	+ ARM_CSPMU_EVENT_ATTR(total_bytes_loc, 0x4),
	165	+ ARM_CSPMU_EVENT_ATTR(total_bytes_rem, 0x5),
	166	+ ARM_CSPMU_EVENT_ATTR(rd_req_loc, 0x6),
	167	+ ARM_CSPMU_EVENT_ATTR(rd_req_rem, 0x7),
	168	+ ARM_CSPMU_EVENT_ATTR(wr_req_loc, 0x8),
	169	+ ARM_CSPMU_EVENT_ATTR(wr_req_rem, 0x9),
	170	+ ARM_CSPMU_EVENT_ATTR(total_req_loc, 0xa),
	171	+ ARM_CSPMU_EVENT_ATTR(total_req_rem, 0xb),
	172	+ ARM_CSPMU_EVENT_ATTR(rd_cum_outs_loc, 0xc),
	173	+ ARM_CSPMU_EVENT_ATTR(rd_cum_outs_rem, 0xd),
	174	+ ARM_CSPMU_EVENT_ATTR(cycles, ARM_CSPMU_EVT_CYCLES_DEFAULT),
	175	+ NULL,
	176	+};
	177	+
	178	+static struct attribute *generic_pmu_event_attrs[] = {
	179	+ ARM_CSPMU_EVENT_ATTR(cycles, ARM_CSPMU_EVT_CYCLES_DEFAULT),
	180	+ NULL,
	181	+};
	182	+
	183	+static struct attribute *scf_pmu_format_attrs[] = {
	184	+ ARM_CSPMU_FORMAT_EVENT_ATTR,
	185	+ NULL,
	186	+};
	187	+
	188	+static struct attribute *pcie_pmu_format_attrs[] = {
	189	+ ARM_CSPMU_FORMAT_EVENT_ATTR,
	190	+ ARM_CSPMU_FORMAT_ATTR(root_port, "config1:0-9"),
	191	+ NULL,
	192	+};
	193	+
	194	+static struct attribute *nvlink_c2c_pmu_format_attrs[] = {
	195	+ ARM_CSPMU_FORMAT_EVENT_ATTR,
	196	+ NULL,
	197	+};
	198	+
	199	+static struct attribute *cnvlink_pmu_format_attrs[] = {
	200	+ ARM_CSPMU_FORMAT_EVENT_ATTR,
	201	+ ARM_CSPMU_FORMAT_ATTR(rem_socket, "config1:0-3"),
	202	+ NULL,
	203	+};
	204	+
	205	+static struct attribute *generic_pmu_format_attrs[] = {
	206	+ ARM_CSPMU_FORMAT_EVENT_ATTR,
	207	+ ARM_CSPMU_FORMAT_FILTER_ATTR,
	208	+ NULL,
	209	+};
	210	+
	211	+static struct attribute **
	212	+nv_cspmu_get_event_attrs(const struct arm_cspmu *cspmu)
	213	+{
	214	+ const struct nv_cspmu_ctx *ctx = to_nv_cspmu_ctx(cspmu);
	215	+
	216	+ return ctx->event_attr;
	217	+}
	218	+
	219	+static struct attribute **
	220	+nv_cspmu_get_format_attrs(const struct arm_cspmu *cspmu)
	221	+{
	222	+ const struct nv_cspmu_ctx *ctx = to_nv_cspmu_ctx(cspmu);
	223	+
	224	+ return ctx->format_attr;
	225	+}
	226	+
	227	+static const char *
	228	+nv_cspmu_get_name(const struct arm_cspmu *cspmu)
	229	+{
	230	+ const struct nv_cspmu_ctx *ctx = to_nv_cspmu_ctx(cspmu);
	231	+
	232	+ return ctx->name;
	233	+}
	234	+
	235	+static u32 nv_cspmu_event_filter(const struct perf_event *event)
	236	+{
	237	+ const struct nv_cspmu_ctx *ctx =
	238	+ to_nv_cspmu_ctx(to_arm_cspmu(event->pmu));
	239	+
	240	+ if (ctx->filter_mask == 0)
	241	+ return ctx->filter_default_val;
	242	+
	243	+ return event->attr.config1 & ctx->filter_mask;
	244	+}
	245	+
	246	+enum nv_cspmu_name_fmt {
	247	+ NAME_FMT_GENERIC,
	248	+ NAME_FMT_SOCKET
	249	+};
	250	+
	251	+struct nv_cspmu_match {
	252	+ u32 prodid;
	253	+ u32 prodid_mask;
	254	+ u64 filter_mask;
	255	+ u32 filter_default_val;
	256	+ const char *name_pattern;
	257	+ enum nv_cspmu_name_fmt name_fmt;
	258	+ struct attribute **event_attr;
	259	+ struct attribute **format_attr;
	260	+};
	261	+
	262	+static const struct nv_cspmu_match nv_cspmu_match[] = {
	263	+ {
	264	+ .prodid = 0x103,
	265	+ .prodid_mask = NV_PRODID_MASK,
	266	+ .filter_mask = NV_PCIE_FILTER_ID_MASK,
	267	+ .filter_default_val = NV_PCIE_FILTER_ID_MASK,
	268	+ .name_pattern = "nvidia_pcie_pmu_%u",
	269	+ .name_fmt = NAME_FMT_SOCKET,
	270	+ .event_attr = mcf_pmu_event_attrs,
	271	+ .format_attr = pcie_pmu_format_attrs
	272	+ },
	273	+ {
	274	+ .prodid = 0x104,
	275	+ .prodid_mask = NV_PRODID_MASK,
	276	+ .filter_mask = 0x0,
	277	+ .filter_default_val = NV_NVL_C2C_FILTER_ID_MASK,
	278	+ .name_pattern = "nvidia_nvlink_c2c1_pmu_%u",
	279	+ .name_fmt = NAME_FMT_SOCKET,
	280	+ .event_attr = mcf_pmu_event_attrs,
	281	+ .format_attr = nvlink_c2c_pmu_format_attrs
	282	+ },
	283	+ {
	284	+ .prodid = 0x105,
	285	+ .prodid_mask = NV_PRODID_MASK,
	286	+ .filter_mask = 0x0,
	287	+ .filter_default_val = NV_NVL_C2C_FILTER_ID_MASK,
	288	+ .name_pattern = "nvidia_nvlink_c2c0_pmu_%u",
	289	+ .name_fmt = NAME_FMT_SOCKET,
	290	+ .event_attr = mcf_pmu_event_attrs,
	291	+ .format_attr = nvlink_c2c_pmu_format_attrs
	292	+ },
	293	+ {
	294	+ .prodid = 0x106,
	295	+ .prodid_mask = NV_PRODID_MASK,
	296	+ .filter_mask = NV_CNVL_FILTER_ID_MASK,
	297	+ .filter_default_val = NV_CNVL_FILTER_ID_MASK,
	298	+ .name_pattern = "nvidia_cnvlink_pmu_%u",
	299	+ .name_fmt = NAME_FMT_SOCKET,
	300	+ .event_attr = mcf_pmu_event_attrs,
	301	+ .format_attr = cnvlink_pmu_format_attrs
	302	+ },
	303	+ {
	304	+ .prodid = 0x2CF,
	305	+ .prodid_mask = NV_PRODID_MASK,
	306	+ .filter_mask = 0x0,
	307	+ .filter_default_val = 0x0,
	308	+ .name_pattern = "nvidia_scf_pmu_%u",
	309	+ .name_fmt = NAME_FMT_SOCKET,
	310	+ .event_attr = scf_pmu_event_attrs,
	311	+ .format_attr = scf_pmu_format_attrs
	312	+ },
	313	+ {
	314	+ .prodid = 0,
	315	+ .prodid_mask = 0,
	316	+ .filter_mask = NV_GENERIC_FILTER_ID_MASK,
	317	+ .filter_default_val = NV_GENERIC_FILTER_ID_MASK,
	318	+ .name_pattern = "nvidia_uncore_pmu_%u",
	319	+ .name_fmt = NAME_FMT_GENERIC,
	320	+ .event_attr = generic_pmu_event_attrs,
	321	+ .format_attr = generic_pmu_format_attrs
	322	+ },
	323	+};
	324	+
	325	+static char nv_cspmu_format_name(const struct arm_cspmu cspmu,
	326	+ const struct nv_cspmu_match *match)
	327	+{
	328	+ char *name;
	329	+ struct device *dev = cspmu->dev;
	330	+
	331	+ static atomic_t pmu_generic_idx = {0};
	332	+
	333	+ switch (match->name_fmt) {
	334	+ case NAME_FMT_SOCKET: {
	335	+ const int cpu = cpumask_first(&cspmu->associated_cpus);
	336	+ const int socket = cpu_to_node(cpu);
	337	+
	338	+ name = devm_kasprintf(dev, GFP_KERNEL, match->name_pattern,
	339	+ socket);
	340	+ break;
	341	+ }
	342	+ case NAME_FMT_GENERIC:
	343	+ name = devm_kasprintf(dev, GFP_KERNEL, match->name_pattern,
	344	+ atomic_fetch_inc(&pmu_generic_idx));
	345	+ break;
	346	+ default:
	347	+ name = NULL;
	348	+ break;
	349	+ }
	350	+
	351	+ return name;
	352	+}
	353	+
	354	+int nv_cspmu_init_ops(struct arm_cspmu *cspmu)
	355	+{
	356	+ u32 prodid;
	357	+ struct nv_cspmu_ctx *ctx;
	358	+ struct device *dev = cspmu->dev;
	359	+ struct arm_cspmu_impl_ops *impl_ops = &cspmu->impl.ops;
	360	+ const struct nv_cspmu_match *match = nv_cspmu_match;
	361	+
	362	+ ctx = devm_kzalloc(dev, sizeof(struct nv_cspmu_ctx), GFP_KERNEL);
	363	+ if (!ctx)
	364	+ return -ENOMEM;
	365	+
	366	+ prodid = FIELD_GET(ARM_CSPMU_PMIIDR_PRODUCTID, cspmu->impl.pmiidr);
	367	+
	368	+ /* Find matching PMU. */
	369	+ for (; match->prodid; match++) {
	370	+ const u32 prodid_mask = match->prodid_mask;
	371	+
	372	+ if ((match->prodid & prodid_mask) == (prodid & prodid_mask))
	373	+ break;
	374	+ }
	375	+
	376	+ ctx->name = nv_cspmu_format_name(cspmu, match);
	377	+ ctx->filter_mask = match->filter_mask;
	378	+ ctx->filter_default_val = match->filter_default_val;
	379	+ ctx->event_attr = match->event_attr;
	380	+ ctx->format_attr = match->format_attr;
	381	+
	382	+ cspmu->impl.ctx = ctx;
	383	+
	384	+ /* NVIDIA specific callbacks. */
	385	+ impl_ops->event_filter = nv_cspmu_event_filter;
	386	+ impl_ops->get_event_attrs = nv_cspmu_get_event_attrs;
	387	+ impl_ops->get_format_attrs = nv_cspmu_get_format_attrs;
	388	+ impl_ops->get_name = nv_cspmu_get_name;
	389	+
	390	+ /* Set others to NULL to use default callback. */
	391	+ impl_ops->event_type = NULL;
	392	+ impl_ops->event_attr_is_visible = NULL;
	393	+ impl_ops->get_identifier = NULL;
	394	+ impl_ops->is_cycle_counter_event = NULL;
	395	+
	396	+ return 0;
	397	+}
	398	+EXPORT_SYMBOL_GPL(nv_cspmu_init_ops);
	399	+
	400	+MODULE_LICENSE("GPL v2");

drivers/perf/arm_cspmu/nvidia_cspmu.h+17−0

@@ -0,0 +1,17 @@
	1	+/* SPDX-License-Identifier: GPL-2.0
	2	+ *
	3	+ * Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
	4	+ *
	5	+ */
	6	+
	7	+/* Support for NVIDIA specific attributes. */
	8	+
	9	+#ifndef __NVIDIA_CSPMU_H__
	10	+#define __NVIDIA_CSPMU_H__
	11	+
	12	+#include "arm_cspmu.h"
	13	+
	14	+/* Allocate NVIDIA descriptor. */
	15	+int nv_cspmu_init_ops(struct arm_cspmu *cspmu);
	16	+
	17	+#endif /* __NVIDIA_CSPMU_H__ */

drivers/perf/arm_dmc620_pmu.c+7−1

@@ -725,6 +725,8 @@ static struct platform_driver dmc620_pmu_driver = { · 724 unmodified lines
725	725
726	726	static int __init dmc620_pmu_init(void)
727	727	{
	728	+ int ret;
	729	+
728	730	cpuhp_state_num = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN,
729	731	DMC620_DRVNAME,
730	732	NULL,
@@ -732,7 +734,11 @@ static int __init dmc620_pmu_init(void) · 1 unmodified line
732	734	if (cpuhp_state_num < 0)
733	735	return cpuhp_state_num;
734	736
735		− return platform_driver_register(&dmc620_pmu_driver);
	737	+ ret = platform_driver_register(&dmc620_pmu_driver);
	738	+ if (ret)
	739	+ cpuhp_remove_multi_state(cpuhp_state_num);
	740	+
	741	+ return ret;
736	742	}
737	743
738	744	static void __exit dmc620_pmu_exit(void)

drivers/perf/arm_dsu_pmu.c+5−1

@@ -858,7 +858,11 @@ static int __init dsu_pmu_init(void) · 857 unmodified lines
858	858	if (ret < 0)
859	859	return ret;
860	860	dsu_pmu_cpuhp_state = ret;
861		− return platform_driver_register(&dsu_pmu_driver);
	861	+ ret = platform_driver_register(&dsu_pmu_driver);
	862	+ if (ret)
	863	+ cpuhp_remove_multi_state(dsu_pmu_cpuhp_state);
	864	+
	865	+ return ret;
862	866	}
863	867
864	868	static void __exit dsu_pmu_exit(void)

drivers/perf/arm_pmu.c+3−17

@@ -514,9 +514,6 @@ static int armpmu_event_init(struct perf_event *event) · 513 unmodified lines
514	514	if (has_branch_stack(event))
515	515	return -EOPNOTSUPP;
516	516
517		− if (armpmu->map_event(event) == -ENOENT)
518		− return -ENOENT;
519		−
520	517	return __hw_perf_event_init(event);
521	518	}
522	519
@@ -861,16 +858,16 @@ static void cpu_pmu_destroy(struct arm_pmu *cpu_pmu) · 338 unmodified lines
861	858	&cpu_pmu->node);
862	859	}
863	860
864		−static struct arm_pmu *__armpmu_alloc(gfp_t flags)
	861	+struct arm_pmu *armpmu_alloc(void)
865	862	{
866	863	struct arm_pmu *pmu;
867	864	int cpu;
868	865
869		− pmu = kzalloc(sizeof(*pmu), flags);
	866	+ pmu = kzalloc(sizeof(*pmu), GFP_KERNEL);
870	867	if (!pmu)
871	868	goto out;
872	869
873		− pmu->hw_events = alloc_percpu_gfp(struct pmu_hw_events, flags);
	870	+ pmu->hw_events = alloc_percpu_gfp(struct pmu_hw_events, GFP_KERNEL);
874	871	if (!pmu->hw_events) {
875	872	pr_info("failed to allocate per-cpu PMU data.\n");
876	873	goto out_free_pmu;
@@ -916,17 +913,6 @@ out: · 39 unmodified lines
916	913	return NULL;
917	914	}
918	915
919		−struct arm_pmu *armpmu_alloc(void)
920		−{
921		− return __armpmu_alloc(GFP_KERNEL);
922		−}
923		−
924		−struct arm_pmu *armpmu_alloc_atomic(void)
925		−{
926		− return __armpmu_alloc(GFP_ATOMIC);
927		−}
928		−
929		−
930	916	void armpmu_free(struct arm_pmu *pmu)
931	917	{
932	918	free_percpu(pmu->hw_events);

drivers/perf/arm_pmu_acpi.c+67−47

@@ -13,6 +13,7 @@ · 12 unmodified lines
13	13	#include <linux/percpu.h>
14	14	#include <linux/perf/arm_pmu.h>
15	15
	16	+#include <asm/cpu.h>
16	17	#include <asm/cputype.h>
17	18
18	19	static DEFINE_PER_CPU(struct arm_pmu *, probed_pmus);
@@ -187,7 +188,7 @@ out_err: · 168 unmodified lines
187	188	return err;
188	189	}
189	190
190		−static struct arm_pmu *arm_pmu_acpi_find_alloc_pmu(void)
	191	+static struct arm_pmu *arm_pmu_acpi_find_pmu(void)
191	192	{
192	193	unsigned long cpuid = read_cpuid_id();
193	194	struct arm_pmu *pmu;
@@ -201,16 +202,7 @@ static struct arm_pmu *arm_pmu_acpi_find_alloc_pmu(void) · 7 unmodified lines
201	202	return pmu;
202	203	}
203	204
204		− pmu = armpmu_alloc_atomic();
205		− if (!pmu) {
206		− pr_warn("Unable to allocate PMU for CPU%d\n",
207		− smp_processor_id());
208		− return NULL;
209		− }
210		−
211		− pmu->acpi_cpuid = cpuid;
212		−
213		− return pmu;
	205	+ return NULL;
214	206	}
215	207
216	208	/*
@@ -242,6 +234,22 @@ static bool pmu_irq_matches(struct arm_pmu *pmu, int irq) · 25 unmodified lines
242	234	return true;
243	235	}
244	236
	237	+static void arm_pmu_acpi_associate_pmu_cpu(struct arm_pmu *pmu,
	238	+ unsigned int cpu)
	239	+{
	240	+ int irq = per_cpu(pmu_irqs, cpu);
	241	+
	242	+ per_cpu(probed_pmus, cpu) = pmu;
	243	+
	244	+ if (pmu_irq_matches(pmu, irq)) {
	245	+ struct pmu_hw_events __percpu *hw_events;
	246	+ hw_events = pmu->hw_events;
	247	+ per_cpu(hw_events->irq, cpu) = irq;
	248	+ }
	249	+
	250	+ cpumask_set_cpu(cpu, &pmu->supported_cpus);
	251	+}
	252	+
245	253	/*
246	254	* This must run before the common arm_pmu hotplug logic, so that we can
247	255	* associate a CPU and its interrupt before the common code tries to manage the
@@ -254,42 +262,50 @@ static bool pmu_irq_matches(struct arm_pmu *pmu, int irq) · 6 unmodified lines
254	262	static int arm_pmu_acpi_cpu_starting(unsigned int cpu)
255	263	{
256	264	struct arm_pmu *pmu;
257		− struct pmu_hw_events __percpu *hw_events;
258		− int irq;
259	265
260	266	/* If we've already probed this CPU, we have nothing to do */
261	267	if (per_cpu(probed_pmus, cpu))
262	268	return 0;
263	269
264		− irq = per_cpu(pmu_irqs, cpu);
265		−
266		− pmu = arm_pmu_acpi_find_alloc_pmu();
267		− if (!pmu)
268		− return -ENOMEM;
	270	+ pmu = arm_pmu_acpi_find_pmu();
	271	+ if (!pmu) {
	272	+ pr_warn_ratelimited("Unable to associate CPU%d with a PMU\n",
	273	+ cpu);
	274	+ return 0;
	275	+ }
269	276
270		− per_cpu(probed_pmus, cpu) = pmu;
	277	+ arm_pmu_acpi_associate_pmu_cpu(pmu, cpu);
	278	+ return 0;
	279	+}
271	280
272		− if (pmu_irq_matches(pmu, irq)) {
273		− hw_events = pmu->hw_events;
274		− per_cpu(hw_events->irq, cpu) = irq;
275		− }
	281	+static void arm_pmu_acpi_probe_matching_cpus(struct arm_pmu *pmu,
	282	+ unsigned long cpuid)
	283	+{
	284	+ int cpu;
276	285
277		− cpumask_set_cpu(cpu, &pmu->supported_cpus);
	286	+ for_each_online_cpu(cpu) {
	287	+ unsigned long cpu_cpuid = per_cpu(cpu_data, cpu).reg_midr;
278	288
279		− /*
280		− * Ideally, we'd probe the PMU here when we find the first matching
281		− * CPU. We can't do that for several reasons; see the comment in
282		− * arm_pmu_acpi_init().
283		− *
284		− * So for the time being, we're done.
285		− */
286		− return 0;
	289	+ if (cpu_cpuid == cpuid)
	290	+ arm_pmu_acpi_associate_pmu_cpu(pmu, cpu);
	291	+ }
287	292	}
288	293
289	294	int arm_pmu_acpi_probe(armpmu_init_fn init_fn)
290	295	{
291	296	int pmu_idx = 0;
292		− int cpu, ret;
	297	+ unsigned int cpu;
	298	+ int ret;
	299	+
	300	+ ret = arm_pmu_acpi_parse_irqs();
	301	+ if (ret)
	302	+ return ret;
	303	+
	304	+ ret = cpuhp_setup_state_nocalls(CPUHP_AP_PERF_ARM_ACPI_STARTING,
	305	+ "perf/arm/pmu_acpi:starting",
	306	+ arm_pmu_acpi_cpu_starting, NULL);
	307	+ if (ret)
	308	+ return ret;
293	309
294	310	/*
295	311	* Initialise and register the set of PMUs which we know about right
@@ -304,13 +320,27 @@ int arm_pmu_acpi_probe(armpmu_init_fn init_fn) · 8 unmodified lines
304	320	* For the moment, as with the platform/DT case, we need at least one
305	321	* of a PMU's CPUs to be online at probe time.
306	322	*/
307		− for_each_possible_cpu(cpu) {
	323	+ for_each_online_cpu(cpu) {
308	324	struct arm_pmu *pmu = per_cpu(probed_pmus, cpu);
	325	+ unsigned long cpuid;
309	326	char *base_name;
310	327
311		− if (!pmu \|\| pmu->name)
	328	+ /* If we've already probed this CPU, we have nothing to do */
	329	+ if (pmu)
312	330	continue;
313	331
	332	+ pmu = armpmu_alloc();
	333	+ if (!pmu) {
	334	+ pr_warn("Unable to allocate PMU for CPU%d\n",
	335	+ cpu);
	336	+ return -ENOMEM;
	337	+ }
	338	+
	339	+ cpuid = per_cpu(cpu_data, cpu).reg_midr;
	340	+ pmu->acpi_cpuid = cpuid;
	341	+
	342	+ arm_pmu_acpi_probe_matching_cpus(pmu, cpuid);
	343	+
314	344	ret = init_fn(pmu);
315	345	if (ret == -ENODEV) {
316	346	/* PMU not handled by this driver, or not present */
@@ -335,26 +365,16 @@ int arm_pmu_acpi_probe(armpmu_init_fn init_fn) · 18 unmodified lines
335	365	}
336	366	}
337	367
338		− return 0;
	368	+ return ret;
339	369	}
340	370
341	371	static int arm_pmu_acpi_init(void)
342	372	{
343		− int ret;
344		−
345	373	if (acpi_disabled)
346	374	return 0;
347	375
348	376	arm_spe_acpi_register_device();
349	377
350		− ret = arm_pmu_acpi_parse_irqs();
351		− if (ret)
352		− return ret;
353		−
354		− ret = cpuhp_setup_state(CPUHP_AP_PERF_ARM_ACPI_STARTING,
355		− "perf/arm/pmu_acpi:starting",
356		− arm_pmu_acpi_cpu_starting, NULL);
357		−
358		− return ret;
	378	+ return 0;
359	379	}
360	380	subsys_initcall(arm_pmu_acpi_init)

drivers/perf/arm_smmuv3_pmu.c+7−1

@@ -959,6 +959,8 @@ static struct platform_driver smmu_pmu_driver = { · 958 unmodified lines
959	959
960	960	static int __init arm_smmu_pmu_init(void)
961	961	{
	962	+ int ret;
	963	+
962	964	cpuhp_state_num = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN,
963	965	"perf/arm/pmcg:online",
964	966	NULL,
@@ -966,7 +968,11 @@ static int __init arm_smmu_pmu_init(void) · 1 unmodified line
966	968	if (cpuhp_state_num < 0)
967	969	return cpuhp_state_num;
968	970
969		− return platform_driver_register(&smmu_pmu_driver);
	971	+ ret = platform_driver_register(&smmu_pmu_driver);
	972	+ if (ret)
	973	+ cpuhp_remove_multi_state(cpuhp_state_num);
	974	+
	975	+ return ret;
970	976	}
971	977	module_init(arm_smmu_pmu_init);
972	978

drivers/perf/hisilicon/hisi_pcie_pmu.c+17−5

@@ -47,10 +47,14 @@ · 46 unmodified lines
47	47	#define HISI_PCIE_EVENT_M GENMASK_ULL(15, 0)
48	48	#define HISI_PCIE_THR_MODE_M GENMASK_ULL(27, 27)
49	49	#define HISI_PCIE_THR_M GENMASK_ULL(31, 28)
	50	+#define HISI_PCIE_LEN_M GENMASK_ULL(35, 34)
50	51	#define HISI_PCIE_TARGET_M GENMASK_ULL(52, 36)
51	52	#define HISI_PCIE_TRIG_MODE_M GENMASK_ULL(53, 53)
52	53	#define HISI_PCIE_TRIG_M GENMASK_ULL(59, 56)
53	54
	55	+/* Default config of TLP length mode, will count both TLP headers and payloads */
	56	+#define HISI_PCIE_LEN_M_DEFAULT 3ULL
	57	+
54	58	#define HISI_PCIE_MAX_COUNTERS 8
55	59	#define HISI_PCIE_REG_STEP 8
56	60	#define HISI_PCIE_THR_MAX_VAL 10
@@ -91,6 +95,7 @@ HISI_PCIE_PMU_FILTER_ATTR(thr_len, config1, 3, 0); · 34 unmodified lines
91	95	HISI_PCIE_PMU_FILTER_ATTR(thr_mode, config1, 4, 4);
92	96	HISI_PCIE_PMU_FILTER_ATTR(trig_len, config1, 8, 5);
93	97	HISI_PCIE_PMU_FILTER_ATTR(trig_mode, config1, 9, 9);
	98	+HISI_PCIE_PMU_FILTER_ATTR(len_mode, config1, 11, 10);
94	99	HISI_PCIE_PMU_FILTER_ATTR(port, config2, 15, 0);
95	100	HISI_PCIE_PMU_FILTER_ATTR(bdf, config2, 31, 16);
96	101
@@ -215,8 +220,8 @@ static void hisi_pcie_pmu_config_filter(struct perf_event *event) · 118 unmodified lines
215	220	{
216	221	struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(event->pmu);
217	222	struct hw_perf_event *hwc = &event->hw;
	223	+ u64 port, trig_len, thr_len, len_mode;
218	224	u64 reg = HISI_PCIE_INIT_SET;
219		− u64 port, trig_len, thr_len;
220	225
221	226	/* Config HISI_PCIE_EVENT_CTRL according to event. */
222	227	reg \|= FIELD_PREP(HISI_PCIE_EVENT_M, hisi_pcie_get_real_event(event));
@@ -245,6 +250,12 @@ static void hisi_pcie_pmu_config_filter(struct perf_event *event) · 22 unmodified lines
245	250	reg \|= HISI_PCIE_THR_EN;
246	251	}
247	252
	253	+ len_mode = hisi_pcie_get_len_mode(event);
	254	+ if (len_mode)
	255	+ reg \|= FIELD_PREP(HISI_PCIE_LEN_M, len_mode);
	256	+ else
	257	+ reg \|= FIELD_PREP(HISI_PCIE_LEN_M, HISI_PCIE_LEN_M_DEFAULT);
	258	+
248	259	hisi_pcie_pmu_writeq(pcie_pmu, HISI_PCIE_EVENT_CTRL, hwc->idx, reg);
249	260	}
250	261
@@ -693,10 +704,10 @@ static struct attribute *hisi_pcie_pmu_events_attr[] = { · 442 unmodified lines
693	704	HISI_PCIE_PMU_EVENT_ATTR(rx_mrd_cnt, 0x10210),
694	705	HISI_PCIE_PMU_EVENT_ATTR(tx_mrd_latency, 0x0011),
695	706	HISI_PCIE_PMU_EVENT_ATTR(tx_mrd_cnt, 0x10011),
696		− HISI_PCIE_PMU_EVENT_ATTR(rx_mrd_flux, 0x1005),
697		− HISI_PCIE_PMU_EVENT_ATTR(rx_mrd_time, 0x11005),
698		− HISI_PCIE_PMU_EVENT_ATTR(tx_mrd_flux, 0x2004),
699		− HISI_PCIE_PMU_EVENT_ATTR(tx_mrd_time, 0x12004),
	707	+ HISI_PCIE_PMU_EVENT_ATTR(rx_mrd_flux, 0x0804),
	708	+ HISI_PCIE_PMU_EVENT_ATTR(rx_mrd_time, 0x10804),
	709	+ HISI_PCIE_PMU_EVENT_ATTR(tx_mrd_flux, 0x0405),
	710	+ HISI_PCIE_PMU_EVENT_ATTR(tx_mrd_time, 0x10405),
700	711	NULL
701	712	};
702	713
@@ -711,6 +722,7 @@ static struct attribute *hisi_pcie_pmu_format_attr[] = { · 8 unmodified lines
711	722	HISI_PCIE_PMU_FORMAT_ATTR(thr_mode, "config1:4"),
712	723	HISI_PCIE_PMU_FORMAT_ATTR(trig_len, "config1:5-8"),
713	724	HISI_PCIE_PMU_FORMAT_ATTR(trig_mode, "config1:9"),
	725	+ HISI_PCIE_PMU_FORMAT_ATTR(len_mode, "config1:10-11"),
714	726	HISI_PCIE_PMU_FORMAT_ATTR(port, "config2:0-15"),
715	727	HISI_PCIE_PMU_FORMAT_ATTR(bdf, "config2:16-31"),
716	728	NULL

drivers/perf/marvell_cn10k_tad_pmu.c+5−1

@@ -408,7 +408,11 @@ static int __init tad_pmu_init(void) · 407 unmodified lines
408	408	if (ret < 0)
409	409	return ret;
410	410	tad_pmu_cpuhp_state = ret;
411		− return platform_driver_register(&tad_pmu_driver);
	411	+ ret = platform_driver_register(&tad_pmu_driver);
	412	+ if (ret)
	413	+ cpuhp_remove_multi_state(tad_pmu_cpuhp_state);
	414	+
	415	+ return ret;
412	416	}
413	417
414	418	static void __exit tad_pmu_exit(void)

include/asm-generic/vmlinux.lds.h+7−2

@@ -1027,14 +1027,19 @@ · 1026 unmodified lines
1027	1027	* keep any .init_array.* sections.
1028	1028	* https://bugs.llvm.org/show_bug.cgi?id=46478
1029	1029	*/
	1030	+#ifdef CONFIG_UNWIND_TABLES
	1031	+#define DISCARD_EH_FRAME
	1032	+#else
	1033	+#define DISCARD_EH_FRAME *(.eh_frame)
	1034	+#endif
1030	1035	#if defined(CONFIG_GCOV_KERNEL) \|\| defined(CONFIG_KASAN_GENERIC) \|\| defined(CONFIG_KCSAN)
1031	1036	# ifdef CONFIG_CONSTRUCTORS
1032	1037	# define SANITIZER_DISCARDS \
1033		− *(.eh_frame)
	1038	+ DISCARD_EH_FRAME
1034	1039	# else
1035	1040	# define SANITIZER_DISCARDS \
1036	1041	(.init_array) (.init_array.*) \
1037		− *(.eh_frame)
	1042	+ DISCARD_EH_FRAME
1038	1043	# endif
1039	1044	#else
1040	1045	# define SANITIZER_DISCARDS

include/linux/acpi_apmt.h+19−0

@@ -0,0 +1,19 @@
	1	+/* SPDX-License-Identifier: GPL-2.0
	2	+ *
	3	+ * ARM CoreSight PMU driver.
	4	+ * Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES.
	5	+ *
	6	+ */
	7	+
	8	+#ifndef __ACPI_APMT_H__
	9	+#define __ACPI_APMT_H__
	10	+
	11	+#include <linux/acpi.h>
	12	+
	13	+#ifdef CONFIG_ACPI_APMT
	14	+void acpi_apmt_init(void);
	15	+#else
	16	+static inline void acpi_apmt_init(void) { }
	17	+#endif /* CONFIG_ACPI_APMT */
	18	+
	19	+#endif /* __ACPI_APMT_H__ */

include/linux/arm_ffa.h+84−1

@@ -11,6 +11,89 @@ · 10 unmodified lines
11	11	#include <linux/types.h>
12	12	#include <linux/uuid.h>
13	13
	14	+#define FFA_SMC(calling_convention, func_num) \
	15	+ ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, (calling_convention), \
	16	+ ARM_SMCCC_OWNER_STANDARD, (func_num))
	17	+
	18	+#define FFA_SMC_32(func_num) FFA_SMC(ARM_SMCCC_SMC_32, (func_num))
	19	+#define FFA_SMC_64(func_num) FFA_SMC(ARM_SMCCC_SMC_64, (func_num))
	20	+
	21	+#define FFA_ERROR FFA_SMC_32(0x60)
	22	+#define FFA_SUCCESS FFA_SMC_32(0x61)
	23	+#define FFA_INTERRUPT FFA_SMC_32(0x62)
	24	+#define FFA_VERSION FFA_SMC_32(0x63)
	25	+#define FFA_FEATURES FFA_SMC_32(0x64)
	26	+#define FFA_RX_RELEASE FFA_SMC_32(0x65)
	27	+#define FFA_RXTX_MAP FFA_SMC_32(0x66)
	28	+#define FFA_FN64_RXTX_MAP FFA_SMC_64(0x66)
	29	+#define FFA_RXTX_UNMAP FFA_SMC_32(0x67)
	30	+#define FFA_PARTITION_INFO_GET FFA_SMC_32(0x68)
	31	+#define FFA_ID_GET FFA_SMC_32(0x69)
	32	+#define FFA_MSG_POLL FFA_SMC_32(0x6A)
	33	+#define FFA_MSG_WAIT FFA_SMC_32(0x6B)
	34	+#define FFA_YIELD FFA_SMC_32(0x6C)
	35	+#define FFA_RUN FFA_SMC_32(0x6D)
	36	+#define FFA_MSG_SEND FFA_SMC_32(0x6E)
	37	+#define FFA_MSG_SEND_DIRECT_REQ FFA_SMC_32(0x6F)
	38	+#define FFA_FN64_MSG_SEND_DIRECT_REQ FFA_SMC_64(0x6F)
	39	+#define FFA_MSG_SEND_DIRECT_RESP FFA_SMC_32(0x70)
	40	+#define FFA_FN64_MSG_SEND_DIRECT_RESP FFA_SMC_64(0x70)
	41	+#define FFA_MEM_DONATE FFA_SMC_32(0x71)
	42	+#define FFA_FN64_MEM_DONATE FFA_SMC_64(0x71)
	43	+#define FFA_MEM_LEND FFA_SMC_32(0x72)
	44	+#define FFA_FN64_MEM_LEND FFA_SMC_64(0x72)
	45	+#define FFA_MEM_SHARE FFA_SMC_32(0x73)
	46	+#define FFA_FN64_MEM_SHARE FFA_SMC_64(0x73)
	47	+#define FFA_MEM_RETRIEVE_REQ FFA_SMC_32(0x74)
	48	+#define FFA_FN64_MEM_RETRIEVE_REQ FFA_SMC_64(0x74)
	49	+#define FFA_MEM_RETRIEVE_RESP FFA_SMC_32(0x75)
	50	+#define FFA_MEM_RELINQUISH FFA_SMC_32(0x76)
	51	+#define FFA_MEM_RECLAIM FFA_SMC_32(0x77)
	52	+#define FFA_MEM_OP_PAUSE FFA_SMC_32(0x78)
	53	+#define FFA_MEM_OP_RESUME FFA_SMC_32(0x79)
	54	+#define FFA_MEM_FRAG_RX FFA_SMC_32(0x7A)
	55	+#define FFA_MEM_FRAG_TX FFA_SMC_32(0x7B)
	56	+#define FFA_NORMAL_WORLD_RESUME FFA_SMC_32(0x7C)
	57	+
	58	+/*
	59	+ * For some calls it is necessary to use SMC64 to pass or return 64-bit values.
	60	+ * For such calls FFA_FN_NATIVE(name) will choose the appropriate
	61	+ * (native-width) function ID.
	62	+ */
	63	+#ifdef CONFIG_64BIT
	64	+#define FFA_FN_NATIVE(name) FFA_FN64_##name
	65	+#else
	66	+#define FFA_FN_NATIVE(name) FFA_##name
	67	+#endif
	68	+
	69	+/* FFA error codes. */
	70	+#define FFA_RET_SUCCESS (0)
	71	+#define FFA_RET_NOT_SUPPORTED (-1)
	72	+#define FFA_RET_INVALID_PARAMETERS (-2)
	73	+#define FFA_RET_NO_MEMORY (-3)
	74	+#define FFA_RET_BUSY (-4)
	75	+#define FFA_RET_INTERRUPTED (-5)
	76	+#define FFA_RET_DENIED (-6)
	77	+#define FFA_RET_RETRY (-7)
	78	+#define FFA_RET_ABORTED (-8)
	79	+
	80	+/* FFA version encoding */
	81	+#define FFA_MAJOR_VERSION_MASK GENMASK(30, 16)
	82	+#define FFA_MINOR_VERSION_MASK GENMASK(15, 0)
	83	+#define FFA_MAJOR_VERSION(x) ((u16)(FIELD_GET(FFA_MAJOR_VERSION_MASK, (x))))
	84	+#define FFA_MINOR_VERSION(x) ((u16)(FIELD_GET(FFA_MINOR_VERSION_MASK, (x))))
	85	+#define FFA_PACK_VERSION_INFO(major, minor) \
	86	+ (FIELD_PREP(FFA_MAJOR_VERSION_MASK, (major)) \| \
	87	+ FIELD_PREP(FFA_MINOR_VERSION_MASK, (minor)))
	88	+#define FFA_VERSION_1_0 FFA_PACK_VERSION_INFO(1, 0)
	89	+
	90	+/**
	91	+ * FF-A specification mentions explicitly about '4K pages'. This should
	92	+ * not be confused with the kernel PAGE_SIZE, which is the translation
	93	+ * granule kernel is configured and may be one among 4K, 16K and 64K.
	94	+ */
	95	+#define FFA_PAGE_SIZE SZ_4K
	96	+
14	97	/* FFA Bus/Device/Driver related */
15	98	struct ffa_device {
16	99	int vm_id;
@@ -161,11 +244,11 @@ struct ffa_mem_region_attributes { · 144 unmodified lines
161	244	*/
162	245	#define FFA_MEM_RETRIEVE_SELF_BORROWER BIT(0)
163	246	u8 flag;
164		− u32 composite_off;
165	247	/*
166	248	* Offset in bytes from the start of the outer `ffa_memory_region` to
167	249	* an `struct ffa_mem_region_addr_range`.
168	250	*/
	251	+ u32 composite_off;
169	252	u64 reserved;
170	253	};
171	254

include/linux/ftrace.h+37−10

@@ -37,9 +37,10 @@ extern void ftrace_boot_snapshot(void); · 36 unmodified lines
37	37	static inline void ftrace_boot_snapshot(void) { }
38	38	#endif
39	39
40		−#ifdef CONFIG_FUNCTION_TRACER
41	40	struct ftrace_ops;
42	41	struct ftrace_regs;
	42	+
	43	+#ifdef CONFIG_FUNCTION_TRACER
43	44	/*
44	45	* If the arch's mcount caller does not support all of ftrace's
45	46	* features, then it must call an indirect function that
@@ -110,12 +111,11 @@ struct ftrace_regs { · 64 unmodified lines
110	111	#define arch_ftrace_get_regs(fregs) (&(fregs)->regs)
111	112
112	113	/*
113		− * ftrace_instruction_pointer_set() is to be defined by the architecture
114		− * if to allow setting of the instruction pointer from the ftrace_regs
115		− * when HAVE_DYNAMIC_FTRACE_WITH_ARGS is set and it supports
116		− * live kernel patching.
	114	+ * ftrace_regs_set_instruction_pointer() is to be defined by the architecture
	115	+ * if to allow setting of the instruction pointer from the ftrace_regs when
	116	+ * HAVE_DYNAMIC_FTRACE_WITH_ARGS is set and it supports live kernel patching.
117	117	*/
118		−#define ftrace_instruction_pointer_set(fregs, ip) do { } while (0)
	118	+#define ftrace_regs_set_instruction_pointer(fregs, ip) do { } while (0)
119	119	#endif /* CONFIG_HAVE_DYNAMIC_FTRACE_WITH_ARGS */
120	120
121	121	static __always_inline struct pt_regs ftrace_get_regs(struct ftrace_regs fregs)
@@ -126,6 +126,35 @@ static __always_inline struct pt_regs ftrace_get_regs(struct ftrace_regs fregs · 4 unmodified lines
126	126	return arch_ftrace_get_regs(fregs);
127	127	}
128	128
	129	+/*
	130	+ * When true, the ftrace_regs_{get,set}_*() functions may be used on fregs.
	131	+ * Note: this can be true even when ftrace_get_regs() cannot provide a pt_regs.
	132	+ */
	133	+static __always_inline bool ftrace_regs_has_args(struct ftrace_regs *fregs)
	134	+{
	135	+ if (IS_ENABLED(CONFIG_HAVE_DYNAMIC_FTRACE_WITH_ARGS))
	136	+ return true;
	137	+
	138	+ return ftrace_get_regs(fregs) != NULL;
	139	+}
	140	+
	141	+#ifndef CONFIG_HAVE_DYNAMIC_FTRACE_WITH_ARGS
	142	+#define ftrace_regs_get_instruction_pointer(fregs) \
	143	+ instruction_pointer(ftrace_get_regs(fregs))
	144	+#define ftrace_regs_get_argument(fregs, n) \
	145	+ regs_get_kernel_argument(ftrace_get_regs(fregs), n)
	146	+#define ftrace_regs_get_stack_pointer(fregs) \
	147	+ kernel_stack_pointer(ftrace_get_regs(fregs))
	148	+#define ftrace_regs_return_value(fregs) \
	149	+ regs_return_value(ftrace_get_regs(fregs))
	150	+#define ftrace_regs_set_return_value(fregs, ret) \
	151	+ regs_set_return_value(ftrace_get_regs(fregs), ret)
	152	+#define ftrace_override_function_with_return(fregs) \
	153	+ override_function_with_return(ftrace_get_regs(fregs))
	154	+#define ftrace_regs_query_register_offset(name) \
	155	+ regs_query_register_offset(name)
	156	+#endif
	157	+
129	158	typedef void (*ftrace_func_t)(unsigned long ip, unsigned long parent_ip,
130	159	struct ftrace_ops op, struct ftrace_regs fregs);
131	160
@@ -427,9 +456,7 @@ static inline int modify_ftrace_direct_multi_nolock(struct ftrace_ops *ops, unsi · 295 unmodified lines
427	456	{
428	457	return -ENODEV;
429	458	}
430		−#endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */
431	459
432		−#ifndef CONFIG_HAVE_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
433	460	/*
434	461	* This must be implemented by the architecture.
435	462	* It is the way the ftrace direct_ops helper, when called
@@ -443,9 +470,9 @@ static inline int modify_ftrace_direct_multi_nolock(struct ftrace_ops *ops, unsi · 7 unmodified lines
443	470	* the return from the trampoline jump to the direct caller
444	471	* instead of going back to the function it just traced.
445	472	*/
446		−static inline void arch_ftrace_set_direct_caller(struct pt_regs *regs,
	473	+static inline void arch_ftrace_set_direct_caller(struct ftrace_regs *fregs,
447	474	unsigned long addr) { }
448		−#endif /* CONFIG_HAVE_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */
	475	+#endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */
449	476
450	477	#ifdef CONFIG_STACK_TRACER
451	478

include/linux/perf/arm_pmu.h+0−1

@@ -174,7 +174,6 @@ void kvm_host_pmu_init(struct arm_pmu *pmu); · 173 unmodified lines
174	174
175	175	/* Internal functions only for core arm_pmu code */
176	176	struct arm_pmu *armpmu_alloc(void);
177		−struct arm_pmu *armpmu_alloc_atomic(void);
178	177	void armpmu_free(struct arm_pmu *pmu);
179	178	int armpmu_register(struct arm_pmu *pmu);
180	179	int armpmu_request_irq(int irq, int cpu);

include/linux/scs.h+18−0

@@ -53,6 +53,22 @@ static inline bool task_scs_end_corrupted(struct task_struct *tsk) · 52 unmodified lines
53	53	return sz >= SCS_SIZE - 1 \|\| READ_ONCE_NOCHECK(*magic) != SCS_END_MAGIC;
54	54	}
55	55
	56	+DECLARE_STATIC_KEY_FALSE(dynamic_scs_enabled);
	57	+
	58	+static inline bool scs_is_dynamic(void)
	59	+{
	60	+ if (!IS_ENABLED(CONFIG_DYNAMIC_SCS))
	61	+ return false;
	62	+ return static_branch_likely(&dynamic_scs_enabled);
	63	+}
	64	+
	65	+static inline bool scs_is_enabled(void)
	66	+{
	67	+ if (!IS_ENABLED(CONFIG_DYNAMIC_SCS))
	68	+ return true;
	69	+ return scs_is_dynamic();
	70	+}
	71	+
56	72	#else /* CONFIG_SHADOW_CALL_STACK */
57	73
58	74	static inline void *scs_alloc(int node) { return NULL; }
@@ -62,6 +78,8 @@ static inline void scs_task_reset(struct task_struct *tsk) {} · 3 unmodified lines
62	78	static inline int scs_prepare(struct task_struct *tsk, int node) { return 0; }
63	79	static inline void scs_release(struct task_struct *tsk) {}
64	80	static inline bool task_scs_end_corrupted(struct task_struct *tsk) { return false; }
	81	+static inline bool scs_is_enabled(void) { return false; }
	82	+static inline bool scs_is_dynamic(void) { return false; }
65	83
66	84	#endif /* CONFIG_SHADOW_CALL_STACK */
67	85

include/soc/amlogic/meson_ddr_pmu.h+66−0

@@ -0,0 +1,66 @@
	1	+/* SPDX-License-Identifier: GPL-2.0 */
	2	+/*
	3	+ * Copyright (c) 2022 Amlogic, Inc. All rights reserved.
	4	+ */
	5	+
	6	+#ifndef __MESON_DDR_PMU_H__
	7	+#define __MESON_DDR_PMU_H__
	8	+
	9	+#define MAX_CHANNEL_NUM 8
	10	+
	11	+enum {
	12	+ ALL_CHAN_COUNTER_ID,
	13	+ CHAN1_COUNTER_ID,
	14	+ CHAN2_COUNTER_ID,
	15	+ CHAN3_COUNTER_ID,
	16	+ CHAN4_COUNTER_ID,
	17	+ CHAN5_COUNTER_ID,
	18	+ CHAN6_COUNTER_ID,
	19	+ CHAN7_COUNTER_ID,
	20	+ CHAN8_COUNTER_ID,
	21	+ COUNTER_MAX_ID,
	22	+};
	23	+
	24	+struct dmc_info;
	25	+
	26	+struct dmc_counter {
	27	+ u64 all_cnt; /* The count of all requests come in/out ddr controller */
	28	+ union {
	29	+ u64 all_req;
	30	+ struct {
	31	+ u64 all_idle_cnt;
	32	+ u64 all_16bit_cnt;
	33	+ };
	34	+ };
	35	+ u64 channel_cnt[MAX_CHANNEL_NUM]; /* To save a DMC bandwidth-monitor channel counter */
	36	+};
	37	+
	38	+struct dmc_hw_info {
	39	+ void (enable)(struct dmc_info info);
	40	+ void (disable)(struct dmc_info info);
	41	+ /* Bind an axi line to a bandwidth-monitor channel */
	42	+ void (set_axi_filter)(struct dmc_info info, int axi_id, int chann);
	43	+ int (irq_handler)(struct dmc_info info,
	44	+ struct dmc_counter *counter);
	45	+ void (get_counters)(struct dmc_info info,
	46	+ struct dmc_counter *counter);
	47	+
	48	+ int dmc_nr; /* The number of dmc controller */
	49	+ int chann_nr; /* The number of dmc bandwidth monitor channels */
	50	+ struct attribute **fmt_attr;
	51	+ const u64 capability[2];
	52	+};
	53	+
	54	+struct dmc_info {
	55	+ const struct dmc_hw_info *hw_info;
	56	+
	57	+ void __iomem *ddr_reg[4];
	58	+ unsigned long timer_value; /* Timer value in TIMER register */
	59	+ void __iomem *pll_reg;
	60	+ int irq_num; /* irq vector number */
	61	+};
	62	+
	63	+int meson_ddr_pmu_create(struct platform_device *pdev);
	64	+int meson_ddr_pmu_remove(struct platform_device *pdev);
	65	+
	66	+#endif /* __MESON_DDR_PMU_H__ */

kernel/livepatch/patch.c+1−1

@@ -118,7 +118,7 @@ static void notrace klp_ftrace_handler(unsigned long ip, · 117 unmodified lines
118	118	if (func->nop)
119	119	goto unlock;
120	120
121		− ftrace_instruction_pointer_set(fregs, (unsigned long)func->new_func);
	121	+ ftrace_regs_set_instruction_pointer(fregs, (unsigned long)func->new_func);
122	122
123	123	unlock:
124	124	ftrace_test_recursion_unlock(bit);

kernel/scs.c+12−2

@@ -12,6 +12,10 @@ · 11 unmodified lines
12	12	#include <linux/vmalloc.h>
13	13	#include <linux/vmstat.h>
14	14
	15	+#ifdef CONFIG_DYNAMIC_SCS
	16	+DEFINE_STATIC_KEY_FALSE(dynamic_scs_enabled);
	17	+#endif
	18	+
15	19	static void __scs_account(void *s, int account)
16	20	{
17	21	struct page *scs_page = vmalloc_to_page(s);
@@ -101,14 +105,20 @@ static int scs_cleanup(unsigned int cpu) · 83 unmodified lines
101	105
102	106	void __init scs_init(void)
103	107	{
	108	+ if (!scs_is_enabled())
	109	+ return;
104	110	cpuhp_setup_state(CPUHP_BP_PREPARE_DYN, "scs:scs_cache", NULL,
105	111	scs_cleanup);
106	112	}
107	113
108	114	int scs_prepare(struct task_struct *tsk, int node)
109	115	{
110		− void *s = scs_alloc(node);
	116	+ void *s;
111	117
	118	+ if (!scs_is_enabled())
	119	+ return 0;
	120	+
	121	+ s = scs_alloc(node);
112	122	if (!s)
113	123	return -ENOMEM;
114	124
@@ -148,7 +158,7 @@ void scs_release(struct task_struct *tsk) · 33 unmodified lines
148	158	{
149	159	void *s = task_scs(tsk);
150	160
151		− if (!s)
	161	+ if (!scs_is_enabled() \|\| !s)
152	162	return;
153	163
154	164	WARN(task_scs_end_corrupted(tsk),

kernel/trace/Kconfig+3−3

@@ -46,10 +46,10 @@ config HAVE_DYNAMIC_FTRACE_WITH_ARGS · 45 unmodified lines
46	46	bool
47	47	help
48	48	If this is set, then arguments and stack can be found from
49		− the pt_regs passed into the function callback regs parameter
	49	+ the ftrace_regs passed into the function callback regs parameter
50	50	by default, even without setting the REGS flag in the ftrace_ops.
51		− This allows for use of regs_get_kernel_argument() and
52		− kernel_stack_pointer().
	51	+ This allows for use of ftrace_regs_get_argument() and
	52	+ ftrace_regs_get_stack_pointer().
53	53
54	54	config HAVE_DYNAMIC_FTRACE_NO_PATCHABLE
55	55	bool

kernel/trace/ftrace.c+1−2

@@ -2488,14 +2488,13 @@ ftrace_add_rec_direct(unsigned long ip, unsigned long addr, · 2487 unmodified lines
2488	2488	static void call_direct_funcs(unsigned long ip, unsigned long pip,
2489	2489	struct ftrace_ops ops, struct ftrace_regs fregs)
2490	2490	{
2491		− struct pt_regs *regs = ftrace_get_regs(fregs);
2492	2491	unsigned long addr;
2493	2492
2494	2493	addr = ftrace_find_rec_direct(ip);
2495	2494	if (!addr)
2496	2495	return;
2497	2496
2498		− arch_ftrace_set_direct_caller(regs, addr);
	2497	+ arch_ftrace_set_direct_caller(fregs, addr);
2499	2498	}
2500	2499
2501	2500	struct ftrace_ops direct_ops = {

scripts/head-object-list.txt+0−1

@@ -15,7 +15,6 @@ arch/alpha/kernel/head.o · 14 unmodified lines
15	15	arch/arc/kernel/head.o
16	16	arch/arm/kernel/head-nommu.o
17	17	arch/arm/kernel/head.o
18		−arch/arm64/kernel/head.o
19	18	arch/csky/kernel/head.o
20	19	arch/hexagon/kernel/head.o
21	20	arch/ia64/kernel/head.o

scripts/module.lds.S+6−0

@@ -3,6 +3,12 @@ · 2 unmodified lines
3	3	* Archs are free to supply their own linker scripts. ld will
4	4	* combine them automatically.
5	5	*/
	6	+#ifdef CONFIG_UNWIND_TABLES
	7	+#define DISCARD_EH_FRAME
	8	+#else
	9	+#define DISCARD_EH_FRAME *(.eh_frame)
	10	+#endif
	11	+
6	12	SECTIONS {
7	13	/DISCARD/ : {
8	14	*(.discard)

tools/testing/selftests/arm64/abi/hwcap.c+32−0

@@ -33,6 +33,12 @@ · 32 unmodified lines
33	33	*/
34	34	typedef void (*sigill_fn)(void);
35	35
	36	+static void cssc_sigill(void)
	37	+{
	38	+ /* CNT x0, x0 */
	39	+ asm volatile(".inst 0xdac01c00" : : : "x0");
	40	+}
	41	+
36	42	static void rng_sigill(void)
37	43	{
38	44	asm volatile("mrs x0, S3_3_C2_C4_0" : : : "x0");
@@ -56,6 +62,12 @@ static void sve2_sigill(void) · 17 unmodified lines
56	62	asm volatile(".inst 0x4408A000" : : : "z0");
57	63	}
58	64
	65	+static void sve2p1_sigill(void)
	66	+{
	67	+ /* BFADD Z0.H, Z0.H, Z0.H */
	68	+ asm volatile(".inst 0x65000000" : : : "z0");
	69	+}
	70	+
59	71	static void sveaes_sigill(void)
60	72	{
61	73	/* AESD z0.b, z0.b, z0.b */
@@ -118,6 +130,13 @@ static const struct hwcap_data { · 56 unmodified lines
118	130	sigill_fn sigill_fn;
119	131	bool sigill_reliable;
120	132	} hwcaps[] = {
	133	+ {
	134	+ .name = "CSSC",
	135	+ .at_hwcap = AT_HWCAP2,
	136	+ .hwcap_bit = HWCAP2_CSSC,
	137	+ .cpuinfo = "cssc",
	138	+ .sigill_fn = cssc_sigill,
	139	+ },
121	140	{
122	141	.name = "RNG",
123	142	.at_hwcap = AT_HWCAP2,
@@ -125,6 +144,12 @@ static const struct hwcap_data { · 1 unmodified line
125	144	.cpuinfo = "rng",
126	145	.sigill_fn = rng_sigill,
127	146	},
	147	+ {
	148	+ .name = "RPRFM",
	149	+ .at_hwcap = AT_HWCAP2,
	150	+ .hwcap_bit = HWCAP2_RPRFM,
	151	+ .cpuinfo = "rprfm",
	152	+ },
128	153	{
129	154	.name = "SME",
130	155	.at_hwcap = AT_HWCAP2,
@@ -148,6 +173,13 @@ static const struct hwcap_data { · 17 unmodified lines
148	173	.cpuinfo = "sve2",
149	174	.sigill_fn = sve2_sigill,
150	175	},
	176	+ {
	177	+ .name = "SVE 2.1",
	178	+ .at_hwcap = AT_HWCAP2,
	179	+ .hwcap_bit = HWCAP2_SVE2P1,
	180	+ .cpuinfo = "sve2p1",
	181	+ .sigill_fn = sve2p1_sigill,
	182	+ },
151	183	{
152	184	.name = "SVE AES",
153	185	.at_hwcap = AT_HWCAP2,

tools/testing/selftests/arm64/abi/syscall-abi-asm.S+2−2

@@ -153,7 +153,7 @@ do_syscall: · 152 unmodified lines
153	153	// Only set a non-zero FFR, test patterns must be zero since the
154	154	// syscall should clear it - this lets us handle FA64.
155	155	ldr x2, =ffr_in
156		− ldr p0, [x2, #0]
	156	+ ldr p0, [x2]
157	157	ldr x2, [x2, #0]
158	158	cbz x2, 2f
159	159	wrffr p0.b
@@ -298,7 +298,7 @@ do_syscall: · 138 unmodified lines
298	298	cbz x2, 1f
299	299	ldr x2, =ffr_out
300	300	rdffr p0.b
301		− str p0, [x2, #0]
	301	+ str p0, [x2]
302	302	1:
303	303
304	304	// Restore callee saved registers x19-x30

tools/testing/selftests/arm64/fp/fp-stress.c+93−27

@@ -39,10 +39,12 @@ struct child_data { · 38 unmodified lines
39	39
40	40	static int epoll_fd;
41	41	static struct child_data *children;
	42	+static struct epoll_event *evs;
	43	+static int tests;
42	44	static int num_children;
43	45	static bool terminate;
44	46
45		−static void drain_output(bool flush);
	47	+static int startup_pipe[2];
46	48
47	49	static int num_processors(void)
48	50	{
@@ -81,13 +83,37 @@ static void child_start(struct child_data child, const char program) · 32 unmodified lines
81	83	exit(EXIT_FAILURE);
82	84	}
83	85
	86	+ /*
	87	+ * Duplicate the read side of the startup pipe to
	88	+ * FD 3 so we can close everything else.
	89	+ */
	90	+ ret = dup2(startup_pipe[0], 3);
	91	+ if (ret == -1) {
	92	+ fprintf(stderr, "dup2() %d\n", errno);
	93	+ exit(EXIT_FAILURE);
	94	+ }
	95	+
84	96	/*
85	97	* Very dumb mechanism to clean open FDs other than
86	98	* stdio. We don't want O_CLOEXEC for the pipes...
87	99	*/
88		− for (i = 3; i < 8192; i++)
	100	+ for (i = 4; i < 8192; i++)
89	101	close(i);
90	102
	103	+ /*
	104	+ * Read from the startup pipe, there should be no data
	105	+ * and we should block until it is closed. We just
	106	+ * carry on on error since this isn't super critical.
	107	+ */
	108	+ ret = read(3, &i, sizeof(i));
	109	+ if (ret < 0)
	110	+ fprintf(stderr, "read(startp pipe) failed: %s (%d)\n",
	111	+ strerror(errno), errno);
	112	+ if (ret > 0)
	113	+ fprintf(stderr, "%d bytes of data on startup pipe\n",
	114	+ ret);
	115	+ close(3);
	116	+
91	117	ret = execl(program, program, NULL);
92	118	fprintf(stderr, "execl(%s) failed: %d (%s)\n",
93	119	program, errno, strerror(errno));
@@ -112,12 +138,6 @@ static void child_start(struct child_data child, const char program) · 18 unmodified lines
112	138	ksft_exit_fail_msg("%s EPOLL_CTL_ADD failed: %s (%d)\n",
113	139	child->name, strerror(errno), errno);
114	140	}
115		−
116		− /*
117		− * Keep output flowing during child startup so logs
118		− * are more timely, can help debugging.
119		− */
120		− drain_output(false);
121	141	}
122	142	}
123	143
@@ -290,12 +310,12 @@ static void start_fpsimd(struct child_data *child, int cpu, int copy) · 166 unmodified lines
290	310	{
291	311	int ret;
292	312
293		− child_start(child, "./fpsimd-test");
294		−
295	313	ret = asprintf(&child->name, "FPSIMD-%d-%d", cpu, copy);
296	314	if (ret == -1)
297	315	ksft_exit_fail_msg("asprintf() failed\n");
298	316
	317	+ child_start(child, "./fpsimd-test");
	318	+
299	319	ksft_print_msg("Started %s\n", child->name);
300	320	}
301	321
@@ -307,12 +327,12 @@ static void start_sve(struct child_data *child, int vl, int cpu) · 5 unmodified lines
307	327	if (ret < 0)
308	328	ksft_exit_fail_msg("Failed to set SVE VL %d\n", vl);
309	329
310		− child_start(child, "./sve-test");
311		−
312	330	ret = asprintf(&child->name, "SVE-VL-%d-%d", vl, cpu);
313	331	if (ret == -1)
314	332	ksft_exit_fail_msg("asprintf() failed\n");
315	333
	334	+ child_start(child, "./sve-test");
	335	+
316	336	ksft_print_msg("Started %s\n", child->name);
317	337	}
318	338
@@ -320,16 +340,16 @@ static void start_ssve(struct child_data *child, int vl, int cpu) · 1 unmodified line
320	340	{
321	341	int ret;
322	342
	343	+ ret = asprintf(&child->name, "SSVE-VL-%d-%d", vl, cpu);
	344	+ if (ret == -1)
	345	+ ksft_exit_fail_msg("asprintf() failed\n");
	346	+
323	347	ret = prctl(PR_SME_SET_VL, vl \| PR_SME_VL_INHERIT);
324	348	if (ret < 0)
325	349	ksft_exit_fail_msg("Failed to set SME VL %d\n", ret);
326	350
327	351	child_start(child, "./ssve-test");
328	352
329		− ret = asprintf(&child->name, "SSVE-VL-%d-%d", vl, cpu);
330		− if (ret == -1)
331		− ksft_exit_fail_msg("asprintf() failed\n");
332		−
333	353	ksft_print_msg("Started %s\n", child->name);
334	354	}
335	355
@@ -341,12 +361,12 @@ static void start_za(struct child_data *child, int vl, int cpu) · 5 unmodified lines
341	361	if (ret < 0)
342	362	ksft_exit_fail_msg("Failed to set SME VL %d\n", ret);
343	363
344		− child_start(child, "./za-test");
345		−
346	364	ret = asprintf(&child->name, "ZA-VL-%d-%d", vl, cpu);
347	365	if (ret == -1)
348	366	ksft_exit_fail_msg("asprintf() failed\n");
349	367
	368	+ child_start(child, "./za-test");
	369	+
350	370	ksft_print_msg("Started %s\n", child->name);
351	371	}
352	372
@@ -375,11 +395,11 @@ static void probe_vls(int vls[], int *vl_count, int set_vl) · 22 unmodified lines
375	395	/* Handle any pending output without blocking */
376	396	static void drain_output(bool flush)
377	397	{
378		− struct epoll_event ev;
379	398	int ret = 1;
	399	+ int i;
380	400
381	401	while (ret > 0) {
382		− ret = epoll_wait(epoll_fd, &ev, 1, 0);
	402	+ ret = epoll_wait(epoll_fd, evs, tests, 0);
383	403	if (ret < 0) {
384	404	if (errno == EINTR)
385	405	continue;
@@ -387,8 +407,8 @@ static void drain_output(bool flush) · 1 unmodified line
387	407	strerror(errno), errno);
388	408	}
389	409
390		− if (ret == 1)
391		− child_output(ev.data.ptr, ev.events, flush);
	410	+ for (i = 0; i < ret; i++)
	411	+ child_output(evs[i].data.ptr, evs[i].events, flush);
392	412	}
393	413	}
394	414
@@ -401,10 +421,11 @@ int main(int argc, char **argv) · 6 unmodified lines
401	421	{
402	422	int ret;
403	423	int timeout = 10;
404		− int cpus, tests, i, j, c;
	424	+ int cpus, i, j, c;
405	425	int sve_vl_count, sme_vl_count, fpsimd_per_cpu;
	426	+ bool all_children_started = false;
	427	+ int seen_children;
406	428	int sve_vls[MAX_VLS], sme_vls[MAX_VLS];
407		− struct epoll_event ev;
408	429	struct sigaction sa;
409	430
410	431	while ((c = getopt_long(argc, argv, "t:", options, NULL)) != -1) {
@@ -465,6 +486,12 @@ int main(int argc, char **argv) · 54 unmodified lines
465	486	strerror(errno), ret);
466	487	epoll_fd = ret;
467	488
	489	+ /* Create a pipe which children will block on before execing */
	490	+ ret = pipe(startup_pipe);
	491	+ if (ret != 0)
	492	+ ksft_exit_fail_msg("Failed to create startup pipe: %s (%d)\n",
	493	+ strerror(errno), errno);
	494	+
468	495	/* Get signal handers ready before we start any children */
469	496	memset(&sa, 0, sizeof(sa));
470	497	sa.sa_sigaction = handle_exit_signal;
@@ -484,6 +511,11 @@ int main(int argc, char **argv) · 13 unmodified lines
484	511	ksft_print_msg("Failed to install SIGCHLD handler: %s (%d)\n",
485	512	strerror(errno), errno);
486	513
	514	+ evs = calloc(tests, sizeof(*evs));
	515	+ if (!evs)
	516	+ ksft_exit_fail_msg("Failed to allocated %d epoll events\n",
	517	+ tests);
	518	+
487	519	for (i = 0; i < cpus; i++) {
488	520	for (j = 0; j < fpsimd_per_cpu; j++)
489	521	start_fpsimd(&children[num_children++], i, j);
@@ -497,6 +529,13 @@ int main(int argc, char **argv) · 7 unmodified lines
497	529	}
498	530	}
499	531
	532	+ /*
	533	+ * All children started, close the startup pipe and let them
	534	+ * run.
	535	+ */
	536	+ close(startup_pipe[0]);
	537	+ close(startup_pipe[1]);
	538	+
500	539	for (;;) {
501	540	/* Did we get a signal asking us to exit? */
502	541	if (terminate)
@@ -510,7 +549,7 @@ int main(int argc, char **argv) · 7 unmodified lines
510	549	* useful in emulation where we will both be slow and
511	550	* likely to have a large set of VLs.
512	551	*/
513		− ret = epoll_wait(epoll_fd, &ev, 1, 1000);
	552	+ ret = epoll_wait(epoll_fd, evs, tests, 1000);
514	553	if (ret < 0) {
515	554	if (errno == EINTR)
516	555	continue;
@@ -519,13 +558,40 @@ int main(int argc, char **argv) · 2 unmodified lines
519	558	}
520	559
521	560	/* Output? */
522		− if (ret == 1) {
523		− child_output(ev.data.ptr, ev.events, false);
	561	+ if (ret > 0) {
	562	+ for (i = 0; i < ret; i++) {
	563	+ child_output(evs[i].data.ptr, evs[i].events,
	564	+ false);
	565	+ }
524	566	continue;
525	567	}
526	568
527	569	/* Otherwise epoll_wait() timed out */
528	570
	571	+ /*
	572	+ * If the child processes have not produced output they
	573	+ * aren't actually running the tests yet .
	574	+ */
	575	+ if (!all_children_started) {
	576	+ seen_children = 0;
	577	+
	578	+ for (i = 0; i < num_children; i++)
	579	+ if (children[i].output_seen \|\|
	580	+ children[i].exited)
	581	+ seen_children++;
	582	+
	583	+ if (seen_children != num_children) {
	584	+ ksft_print_msg("Waiting for %d children\n",
	585	+ num_children - seen_children);
	586	+ continue;
	587	+ }
	588	+
	589	+ all_children_started = true;
	590	+ }
	591	+
	592	+ ksft_print_msg("Sending signals, timeout remaining: %d\n",
	593	+ timeout);
	594	+
529	595	for (i = 0; i < num_children; i++)
530	596	child_tickle(&children[i]);
531	597

tools/testing/selftests/arm64/mte/check_buffer_fill.c+6−6

@@ -32,7 +32,7 @@ static int check_buffer_by_byte(int mem_type, int mode) · 31 unmodified lines
32	32	bool err;
33	33
34	34	mte_switch_mode(mode, MTE_ALLOW_NON_ZERO_TAG);
35		− item = sizeof(sizes)/sizeof(int);
	35	+ item = ARRAY_SIZE(sizes);
36	36
37	37	for (i = 0; i < item; i++) {
38	38	ptr = (char *)mte_allocate_memory(sizes[i], mem_type, 0, true);
@@ -69,7 +69,7 @@ static int check_buffer_underflow_by_byte(int mem_type, int mode, · 30 unmodified lines
69	69	char *und_ptr = NULL;
70	70
71	71	mte_switch_mode(mode, MTE_ALLOW_NON_ZERO_TAG);
72		− item = sizeof(sizes)/sizeof(int);
	72	+ item = ARRAY_SIZE(sizes);
73	73	for (i = 0; i < item; i++) {
74	74	ptr = (char *)mte_allocate_memory_tag_range(sizes[i], mem_type, 0,
75	75	underflow_range, 0);
@@ -165,7 +165,7 @@ static int check_buffer_overflow_by_byte(int mem_type, int mode, · 89 unmodified lines
165	165	char *over_ptr = NULL;
166	166
167	167	mte_switch_mode(mode, MTE_ALLOW_NON_ZERO_TAG);
168		− item = sizeof(sizes)/sizeof(int);
	168	+ item = ARRAY_SIZE(sizes);
169	169	for (i = 0; i < item; i++) {
170	170	ptr = (char *)mte_allocate_memory_tag_range(sizes[i], mem_type, 0,
171	171	0, overflow_range);
@@ -338,7 +338,7 @@ static int check_buffer_by_block(int mem_type, int mode) · 166 unmodified lines
338	338	int i, item, result = KSFT_PASS;
339	339
340	340	mte_switch_mode(mode, MTE_ALLOW_NON_ZERO_TAG);
341		− item = sizeof(sizes)/sizeof(int);
	341	+ item = ARRAY_SIZE(sizes);
342	342	cur_mte_cxt.fault_valid = false;
343	343	for (i = 0; i < item; i++) {
344	344	result = check_buffer_by_block_iterate(mem_type, mode, sizes[i]);
@@ -366,7 +366,7 @@ static int check_memory_initial_tags(int mem_type, int mode, int mapping) · 21 unmodified lines
366	366	{
367	367	char *ptr;
368	368	int run, fd;
369		− int total = sizeof(sizes)/sizeof(int);
	369	+ int total = ARRAY_SIZE(sizes);
370	370
371	371	mte_switch_mode(mode, MTE_ALLOW_NON_ZERO_TAG);
372	372	for (run = 0; run < total; run++) {
@@ -404,7 +404,7 @@ int main(int argc, char *argv[]) · 31 unmodified lines
404	404	{
405	405	int err;
406	406	size_t page_size = getpagesize();
407		− int item = sizeof(sizes)/sizeof(int);
	407	+ int item = ARRAY_SIZE(sizes);
408	408
409	409	sizes[item - 3] = page_size - 1;
410	410	sizes[item - 2] = page_size;

tools/testing/selftests/arm64/mte/check_mmap_options.c+4−5

@@ -61,9 +61,8 @@ static int check_anonymous_memory_mapping(int mem_type, int mode, int mapping, i · 60 unmodified lines
61	61	{
62	62	char ptr, map_ptr;
63	63	int run, result, map_size;
64		− int item = sizeof(sizes)/sizeof(int);
	64	+ int item = ARRAY_SIZE(sizes);
65	65
66		− item = sizeof(sizes)/sizeof(int);
67	66	mte_switch_mode(mode, MTE_ALLOW_NON_ZERO_TAG);
68	67	for (run = 0; run < item; run++) {
69	68	map_size = sizes[run] + OVERFLOW + UNDERFLOW;
@@ -93,7 +92,7 @@ static int check_file_memory_mapping(int mem_type, int mode, int mapping, int ta · 23 unmodified lines
93	92	{
94	93	char ptr, map_ptr;
95	94	int run, fd, map_size;
96		− int total = sizeof(sizes)/sizeof(int);
	95	+ int total = ARRAY_SIZE(sizes);
97	96	int result = KSFT_PASS;
98	97
99	98	mte_switch_mode(mode, MTE_ALLOW_NON_ZERO_TAG);
@@ -132,7 +131,7 @@ static int check_clear_prot_mte_flag(int mem_type, int mode, int mapping) · 32 unmodified lines
132	131	{
133	132	char ptr, map_ptr;
134	133	int run, prot_flag, result, fd, map_size;
135		− int total = sizeof(sizes)/sizeof(int);
	134	+ int total = ARRAY_SIZE(sizes);
136	135
137	136	prot_flag = PROT_READ \| PROT_WRITE;
138	137	mte_switch_mode(mode, MTE_ALLOW_NON_ZERO_TAG);
@@ -187,7 +186,7 @@ static int check_clear_prot_mte_flag(int mem_type, int mode, int mapping) · 48 unmodified lines
187	186	int main(int argc, char *argv[])
188	187	{
189	188	int err;
190		− int item = sizeof(sizes)/sizeof(int);
	189	+ int item = ARRAY_SIZE(sizes);
191	190
192	191	err = mte_default_setup();
193	192	if (err)

tools/testing/selftests/arm64/signal/testcases/TODO+0−1

@@ -1,2 +1 @@
1	1	- Validate that register contents are saved and restored as expected.
2		−- Support and validate extra_context.

tools/testing/selftests/arm64/signal/testcases/testcases.c+17−4

@@ -1,5 +1,9 @@
1	1	// SPDX-License-Identifier: GPL-2.0
2	2	/* Copyright (C) 2019 ARM Limited */
	3	+
	4	+#include <ctype.h>
	5	+#include <string.h>
	6	+
3	7	#include "testcases.h"
4	8
5	9	struct _aarch64_ctx get_header(struct _aarch64_ctx head, uint32_t magic,
@@ -109,7 +113,7 @@ bool validate_reserved(ucontext_t uc, size_t resv_sz, char *err) · 103 unmodified lines
109	113	bool terminated = false;
110	114	size_t offs = 0;
111	115	int flags = 0;
112		− int new_flags;
	116	+ int new_flags, i;
113	117	struct extra_context *extra = NULL;
114	118	struct sve_context *sve = NULL;
115	119	struct za_context *za = NULL;
@@ -117,6 +121,7 @@ bool validate_reserved(ucontext_t uc, size_t resv_sz, char *err) · 1 unmodified line
117	121	(struct _aarch64_ctx *)uc->uc_mcontext.__reserved;
118	122	void *extra_data = NULL;
119	123	size_t extra_sz = 0;
	124	+ char magic[4];
120	125
121	126	if (!err)
122	127	return false;
@@ -194,11 +199,19 @@ bool validate_reserved(ucontext_t uc, size_t resv_sz, char *err) · 71 unmodified lines
194	199	/*
195	200	* A still unknown Magic: potentially freshly added
196	201	* to the Kernel code and still unknown to the
197		− * tests.
	202	+ * tests. Magic numbers are supposed to be allocated
	203	+ * as somewhat meaningful ASCII strings so try to
	204	+ * print as such as well as the raw number.
198	205	*/
	206	+ memcpy(magic, &head->magic, sizeof(magic));
	207	+ for (i = 0; i < sizeof(magic); i++)
	208	+ if (!isalnum(magic[i]))
	209	+ magic[i] = '?';
	210	+
199	211	fprintf(stdout,
200		− "SKIP Unknown MAGIC: 0x%X - Is KSFT arm64/signal up to date ?\n",
201		− head->magic);
	212	+ "SKIP Unknown MAGIC: 0x%X (%c%c%c%c) - Is KSFT arm64/signal up to date ?\n",
	213	+ head->magic,
	214	+ magic[3], magic[2], magic[1], magic[0]);
202	215	break;
203	216	}
204	217