Merge tag 'arm64-upstream' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux

Pull arm64 updates from Will Deacon: "The big highlight is support for the Scalable Vector Extension (SVE) which required extensive ABI work to ensure we don't break existing applications by blowing away their signal stack with the rather large new vector context (<= 2 kbit per vector register). There's further work to be done optimising things like exception return, but the ABI is solid now. Much of the line count comes from some new PMU drivers we have, but they're pretty self-contained and I suspect we'll have more of them in future. Plenty of acronym soup here: - initial support for the Scalable Vector Extension (SVE) - improved handling for SError interrupts (required to handle RAS events) - enable GCC support for 128-bit integer types - remove kernel text addresses from backtraces and register dumps - use of WFE to implement long delay()s - ACPI IORT updates from Lorenzo Pieralisi - perf PMU driver for the Statistical Profiling Extension (SPE) - perf PMU driver for Hisilicon's system PMUs - misc cleanups and non-critical fixes" * tag 'arm64-upstream' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux: (97 commits) arm64: Make ARMV8_DEPRECATED depend on SYSCTL arm64: Implement __lshrti3 library function arm64: support __int128 on gcc 5+ arm64/sve: Add documentation arm64/sve: Detect SVE and activate runtime support arm64/sve: KVM: Hide SVE from CPU features exposed to guests arm64/sve: KVM: Treat guest SVE use as undefined instruction execution arm64/sve: KVM: Prevent guests from using SVE arm64/sve: Add sysctl to set the default vector length for new processes arm64/sve: Add prctl controls for userspace vector length management arm64/sve: ptrace and ELF coredump support arm64/sve: Preserve SVE registers around EFI runtime service calls arm64/sve: Preserve SVE registers around kernel-mode NEON use arm64/sve: Probe SVE capabilities and usable vector lengths arm64: cpufeature: Move sys_caps_initialised declarations arm64/sve: Backend logic for setting the vector length arm64/sve: Signal handling support arm64/sve: Support vector length resetting for new processes arm64/sve: Core task context handling arm64/sve: Low-level CPU setup ...

Linus Torvalds committed 9 years ago

Commit c9b012e2 parents b293fca 6cfa7cc97 files · +7399 −601Browse files

Unified · Split

97 files

Documentation/arm64/cpu-feature-registers.txt +16 −2
Documentation/arm64/elf_hwcaps.txt +160 −0
Documentation/arm64/memory.txt +5 −5
Documentation/arm64/sve.txt +508 −0
Documentation/devicetree/bindings/arm/spe-pmu.txt +20 −0
Documentation/perf/hisi-pmu.txt +53 −0
MAINTAINERS +7 −0
arch/arm/include/asm/arch_timer.h +1 −0
arch/arm/include/asm/kvm_host.h +3 −0
arch/arm64/Kconfig +16 −2
arch/arm64/Makefile +8 −2
arch/arm64/include/asm/arch_timer.h +1 −0
arch/arm64/include/asm/asm-bug.h +4 −4
arch/arm64/include/asm/assembler.h +32 −19
arch/arm64/include/asm/barrier.h +2 −0
arch/arm64/include/asm/cpu.h +4 −0
arch/arm64/include/asm/cpucaps.h +2 −1
arch/arm64/include/asm/cpufeature.h +42 −0
arch/arm64/include/asm/daifflags.h +72 −0
arch/arm64/include/asm/elf.h +2 −2
arch/arm64/include/asm/esr.h +2 −1
arch/arm64/include/asm/fpsimd.h +70 −1
arch/arm64/include/asm/fpsimdmacros.h +148 −0
arch/arm64/include/asm/irqflags.h +13 −27
arch/arm64/include/asm/kvm_arm.h +4 −1
arch/arm64/include/asm/kvm_host.h +11 −0
arch/arm64/include/asm/memory.h +0 −15
arch/arm64/include/asm/pgtable.h +14 −0
arch/arm64/include/asm/processor.h +28 −0
arch/arm64/include/asm/sysreg.h +121 −0
arch/arm64/include/asm/thread_info.h +5 −0
arch/arm64/include/asm/traps.h +8 −0
arch/arm64/include/uapi/asm/hwcap.h +6 −0
arch/arm64/include/uapi/asm/ptrace.h +138 −1
arch/arm64/include/uapi/asm/sigcontext.h +119 −1
arch/arm64/kernel/Makefile +0 −2
arch/arm64/kernel/armv8_deprecated.c +7 −16
arch/arm64/kernel/cpufeature.c +141 −63
arch/arm64/kernel/cpuinfo.c +12 −0
arch/arm64/kernel/debug-monitors.c +3 −2
arch/arm64/kernel/entry-fpsimd.S +17 −0
arch/arm64/kernel/entry-ftrace.S +3 −9
arch/arm64/kernel/entry.S +88 −40
arch/arm64/kernel/fpsimd.c +881 −27
arch/arm64/kernel/head.S +24 −6
arch/arm64/kernel/hibernate.c +3 −2
arch/arm64/kernel/io.c +5 −7
arch/arm64/kernel/machine_kexec.c +2 −2
arch/arm64/kernel/process.c +60 −4
arch/arm64/kernel/ptrace.c +272 −8
arch/arm64/kernel/setup.c +7 −8
arch/arm64/kernel/signal.c +169 −10
arch/arm64/kernel/signal32.c +1 −1
arch/arm64/kernel/smp.c +8 −10
arch/arm64/kernel/suspend.c +4 −4
arch/arm64/kernel/traps.c +38 −71
arch/arm64/kernel/vdso/gettimeofday.S +1 −1
arch/arm64/kvm/handle_exit.c +8 −0
arch/arm64/kvm/hyp/debug-sr.c +7 −17
arch/arm64/kvm/hyp/switch.c +9 −3
arch/arm64/kvm/sys_regs.c +247 −45
arch/arm64/lib/Makefile +1 −1
arch/arm64/lib/delay.c +19 −4
arch/arm64/lib/tishift.S +80 −0
arch/arm64/mm/dma-mapping.c +2 −3
arch/arm64/mm/fault.c +13 −59
arch/arm64/mm/proc.S +4 −5
drivers/acpi/arm64/gtdt.c +1 −1
drivers/acpi/arm64/iort.c +200 −58
drivers/bus/arm-ccn.c +1 −0
drivers/clocksource/arm_arch_timer.c +22 −3
drivers/perf/Kconfig +15 −0
drivers/perf/Makefile +2 −0
drivers/perf/arm_pmu.c +5 −5
drivers/perf/arm_pmu_acpi.c +0 −3
drivers/perf/arm_pmu_platform.c +2 −2
drivers/perf/arm_spe_pmu.c +1249 −0
drivers/perf/hisilicon/Makefile +1 −0
drivers/perf/hisilicon/hisi_uncore_ddrc_pmu.c +463 −0
drivers/perf/hisilicon/hisi_uncore_hha_pmu.c +473 −0
drivers/perf/hisilicon/hisi_uncore_l3c_pmu.c +463 −0
drivers/perf/hisilicon/hisi_uncore_pmu.c +447 −0
drivers/perf/hisilicon/hisi_uncore_pmu.h +102 −0
drivers/perf/qcom_l2_pmu.c +54 −0
fs/binfmt_elf.c +5 −5
include/clocksource/arm_arch_timer.h +9 −1
include/linux/acpi_iort.h +2 −2
include/linux/cpuhotplug.h +3 −0
include/linux/irqdesc.h +8 −0
include/linux/regset.h +60 −7
include/uapi/linux/elf.h +1 −0
include/uapi/linux/perf_event.h +1 −0
include/uapi/linux/prctl.h +9 −0
kernel/events/ring_buffer.c +4 −0
kernel/irq/irqdesc.c +1 −0
kernel/sys.c +12 −0
virt/kvm/arm/arm.c +3 −0

Documentation/arm64/cpu-feature-registers.txt+16−2

@@ -110,10 +110,20 @@ infrastructure: · 109 unmodified lines
110	110	x--------------------------------------------------x
111	111	\| Name \| bits \| visible \|
112	112	\|--------------------------------------------------\|
113		− \| RES0 \| [63-32] \| n \|
	113	+ \| RES0 \| [63-48] \| n \|
	114	+ \|--------------------------------------------------\|
	115	+ \| DP \| [47-44] \| y \|
	116	+ \|--------------------------------------------------\|
	117	+ \| SM4 \| [43-40] \| y \|
	118	+ \|--------------------------------------------------\|
	119	+ \| SM3 \| [39-36] \| y \|
	120	+ \|--------------------------------------------------\|
	121	+ \| SHA3 \| [35-32] \| y \|
114	122	\|--------------------------------------------------\|
115	123	\| RDM \| [31-28] \| y \|
116	124	\|--------------------------------------------------\|
	125	+ \| RES0 \| [27-24] \| n \|
	126	+ \|--------------------------------------------------\|
117	127	\| ATOMICS \| [23-20] \| y \|
118	128	\|--------------------------------------------------\|
119	129	\| CRC32 \| [19-16] \| y \|
@@ -132,7 +142,11 @@ infrastructure: · 12 unmodified lines
132	142	x--------------------------------------------------x
133	143	\| Name \| bits \| visible \|
134	144	\|--------------------------------------------------\|
135		− \| RES0 \| [63-28] \| n \|
	145	+ \| RES0 \| [63-36] \| n \|
	146	+ \|--------------------------------------------------\|
	147	+ \| SVE \| [35-32] \| y \|
	148	+ \|--------------------------------------------------\|
	149	+ \| RES0 \| [31-28] \| n \|
136	150	\|--------------------------------------------------\|
137	151	\| GIC \| [27-24] \| n \|
138	152	\|--------------------------------------------------\|

Documentation/arm64/elf_hwcaps.txt+160−0

@@ -0,0 +1,160 @@
	1	+ARM64 ELF hwcaps
	2	+================
	3	+
	4	+This document describes the usage and semantics of the arm64 ELF hwcaps.
	5	+
	6	+
	7	+1. Introduction
	8	+---------------
	9	+
	10	+Some hardware or software features are only available on some CPU
	11	+implementations, and/or with certain kernel configurations, but have no
	12	+architected discovery mechanism available to userspace code at EL0. The
	13	+kernel exposes the presence of these features to userspace through a set
	14	+of flags called hwcaps, exposed in the auxilliary vector.
	15	+
	16	+Userspace software can test for features by acquiring the AT_HWCAP entry
	17	+of the auxilliary vector, and testing whether the relevant flags are
	18	+set, e.g.
	19	+
	20	+bool floating_point_is_present(void)
	21	+{
	22	+ unsigned long hwcaps = getauxval(AT_HWCAP);
	23	+ if (hwcaps & HWCAP_FP)
	24	+ return true;
	25	+
	26	+ return false;
	27	+}
	28	+
	29	+Where software relies on a feature described by a hwcap, it should check
	30	+the relevant hwcap flag to verify that the feature is present before
	31	+attempting to make use of the feature.
	32	+
	33	+Features cannot be probed reliably through other means. When a feature
	34	+is not available, attempting to use it may result in unpredictable
	35	+behaviour, and is not guaranteed to result in any reliable indication
	36	+that the feature is unavailable, such as a SIGILL.
	37	+
	38	+
	39	+2. Interpretation of hwcaps
	40	+---------------------------
	41	+
	42	+The majority of hwcaps are intended to indicate the presence of features
	43	+which are described by architected ID registers inaccessible to
	44	+userspace code at EL0. These hwcaps are defined in terms of ID register
	45	+fields, and should be interpreted with reference to the definition of
	46	+these fields in the ARM Architecture Reference Manual (ARM ARM).
	47	+
	48	+Such hwcaps are described below in the form:
	49	+
	50	+ Functionality implied by idreg.field == val.
	51	+
	52	+Such hwcaps indicate the availability of functionality that the ARM ARM
	53	+defines as being present when idreg.field has value val, but do not
	54	+indicate that idreg.field is precisely equal to val, nor do they
	55	+indicate the absence of functionality implied by other values of
	56	+idreg.field.
	57	+
	58	+Other hwcaps may indicate the presence of features which cannot be
	59	+described by ID registers alone. These may be described without
	60	+reference to ID registers, and may refer to other documentation.
	61	+
	62	+
	63	+3. The hwcaps exposed in AT_HWCAP
	64	+---------------------------------
	65	+
	66	+HWCAP_FP
	67	+
	68	+ Functionality implied by ID_AA64PFR0_EL1.FP == 0b0000.
	69	+
	70	+HWCAP_ASIMD
	71	+
	72	+ Functionality implied by ID_AA64PFR0_EL1.AdvSIMD == 0b0000.
	73	+
	74	+HWCAP_EVTSTRM
	75	+
	76	+ The generic timer is configured to generate events at a frequency of
	77	+ approximately 100KHz.
	78	+
	79	+HWCAP_AES
	80	+
	81	+ Functionality implied by ID_AA64ISAR1_EL1.AES == 0b0001.
	82	+
	83	+HWCAP_PMULL
	84	+
	85	+ Functionality implied by ID_AA64ISAR1_EL1.AES == 0b0010.
	86	+
	87	+HWCAP_SHA1
	88	+
	89	+ Functionality implied by ID_AA64ISAR0_EL1.SHA1 == 0b0001.
	90	+
	91	+HWCAP_SHA2
	92	+
	93	+ Functionality implied by ID_AA64ISAR0_EL1.SHA2 == 0b0001.
	94	+
	95	+HWCAP_CRC32
	96	+
	97	+ Functionality implied by ID_AA64ISAR0_EL1.CRC32 == 0b0001.
	98	+
	99	+HWCAP_ATOMICS
	100	+
	101	+ Functionality implied by ID_AA64ISAR0_EL1.Atomic == 0b0010.
	102	+
	103	+HWCAP_FPHP
	104	+
	105	+ Functionality implied by ID_AA64PFR0_EL1.FP == 0b0001.
	106	+
	107	+HWCAP_ASIMDHP
	108	+
	109	+ Functionality implied by ID_AA64PFR0_EL1.AdvSIMD == 0b0001.
	110	+
	111	+HWCAP_CPUID
	112	+
	113	+ EL0 access to certain ID registers is available, to the extent
	114	+ described by Documentation/arm64/cpu-feature-registers.txt.
	115	+
	116	+ These ID registers may imply the availability of features.
	117	+
	118	+HWCAP_ASIMDRDM
	119	+
	120	+ Functionality implied by ID_AA64ISAR0_EL1.RDM == 0b0001.
	121	+
	122	+HWCAP_JSCVT
	123	+
	124	+ Functionality implied by ID_AA64ISAR1_EL1.JSCVT == 0b0001.
	125	+
	126	+HWCAP_FCMA
	127	+
	128	+ Functionality implied by ID_AA64ISAR1_EL1.FCMA == 0b0001.
	129	+
	130	+HWCAP_LRCPC
	131	+
	132	+ Functionality implied by ID_AA64ISAR1_EL1.LRCPC == 0b0001.
	133	+
	134	+HWCAP_DCPOP
	135	+
	136	+ Functionality implied by ID_AA64ISAR1_EL1.DPB == 0b0001.
	137	+
	138	+HWCAP_SHA3
	139	+
	140	+ Functionality implied by ID_AA64ISAR0_EL1.SHA3 == 0b0001.
	141	+
	142	+HWCAP_SM3
	143	+
	144	+ Functionality implied by ID_AA64ISAR0_EL1.SM3 == 0b0001.
	145	+
	146	+HWCAP_SM4
	147	+
	148	+ Functionality implied by ID_AA64ISAR0_EL1.SM4 == 0b0001.
	149	+
	150	+HWCAP_ASIMDDP
	151	+
	152	+ Functionality implied by ID_AA64ISAR0_EL1.DP == 0b0001.
	153	+
	154	+HWCAP_SHA512
	155	+
	156	+ Functionality implied by ID_AA64ISAR0_EL1.SHA2 == 0b0002.
	157	+
	158	+HWCAP_SVE
	159	+
	160	+ Functionality implied by ID_AA64PFR0_EL1.SVE == 0b0001.

Documentation/arm64/memory.txt+5−5

@@ -86,9 +86,9 @@ Translation table lookup with 64KB pages: · 85 unmodified lines
86	86	+-------------------------------------------------> [63] TTBR0/1
87	87
88	88
89		−When using KVM, the hypervisor maps kernel pages in EL2, at a fixed
90		−offset from the kernel VA (top 24bits of the kernel VA set to zero):
	89	+When using KVM without the Virtualization Host Extensions, the hypervisor
	90	+maps kernel pages in EL2 at a fixed offset from the kernel VA. See the
	91	+kern_hyp_va macro for more details.
91	92
92		−Start End Size Use
93		−-----------------------------------------------------------------------
94		−0000004000000000 0000007fffffffff 256GB kernel objects mapped in HYP
	93	+When using KVM with the Virtualization Host Extensions, no additional
	94	+mappings are created, since the host kernel runs directly in EL2.

Documentation/arm64/sve.txt+508−0

@@ -0,0 +1,508 @@
	1	+ Scalable Vector Extension support for AArch64 Linux
	2	+ ===================================================
	3	+
	4	+Author: Dave Martin <Dave.Martin@arm.com>
	5	+Date: 4 August 2017
	6	+
	7	+This document outlines briefly the interface provided to userspace by Linux in
	8	+order to support use of the ARM Scalable Vector Extension (SVE).
	9	+
	10	+This is an outline of the most important features and issues only and not
	11	+intended to be exhaustive.
	12	+
	13	+This document does not aim to describe the SVE architecture or programmer's
	14	+model. To aid understanding, a minimal description of relevant programmer's
	15	+model features for SVE is included in Appendix A.
	16	+
	17	+
	18	+1. General
	19	+-----------
	20	+
	21	+* SVE registers Z0..Z31, P0..P15 and FFR and the current vector length VL, are
	22	+ tracked per-thread.
	23	+
	24	+* The presence of SVE is reported to userspace via HWCAP_SVE in the aux vector
	25	+ AT_HWCAP entry. Presence of this flag implies the presence of the SVE
	26	+ instructions and registers, and the Linux-specific system interfaces
	27	+ described in this document. SVE is reported in /proc/cpuinfo as "sve".
	28	+
	29	+* Support for the execution of SVE instructions in userspace can also be
	30	+ detected by reading the CPU ID register ID_AA64PFR0_EL1 using an MRS
	31	+ instruction, and checking that the value of the SVE field is nonzero. [3]
	32	+
	33	+ It does not guarantee the presence of the system interfaces described in the
	34	+ following sections: software that needs to verify that those interfaces are
	35	+ present must check for HWCAP_SVE instead.
	36	+
	37	+* Debuggers should restrict themselves to interacting with the target via the
	38	+ NT_ARM_SVE regset. The recommended way of detecting support for this regset
	39	+ is to connect to a target process first and then attempt a
	40	+ ptrace(PTRACE_GETREGSET, pid, NT_ARM_SVE, &iov).
	41	+
	42	+
	43	+2. Vector length terminology
	44	+-----------------------------
	45	+
	46	+The size of an SVE vector (Z) register is referred to as the "vector length".
	47	+
	48	+To avoid confusion about the units used to express vector length, the kernel
	49	+adopts the following conventions:
	50	+
	51	+* Vector length (VL) = size of a Z-register in bytes
	52	+
	53	+* Vector quadwords (VQ) = size of a Z-register in units of 128 bits
	54	+
	55	+(So, VL = 16 * VQ.)
	56	+
	57	+The VQ convention is used where the underlying granularity is important, such
	58	+as in data structure definitions. In most other situations, the VL convention
	59	+is used. This is consistent with the meaning of the "VL" pseudo-register in
	60	+the SVE instruction set architecture.
	61	+
	62	+
	63	+3. System call behaviour
	64	+-------------------------
	65	+
	66	+* On syscall, V0..V31 are preserved (as without SVE). Thus, bits [127:0] of
	67	+ Z0..Z31 are preserved. All other bits of Z0..Z31, and all of P0..P15 and FFR
	68	+ become unspecified on return from a syscall.
	69	+
	70	+* The SVE registers are not used to pass arguments to or receive results from
	71	+ any syscall.
	72	+
	73	+* In practice the affected registers/bits will be preserved or will be replaced
	74	+ with zeros on return from a syscall, but userspace should not make
	75	+ assumptions about this. The kernel behaviour may vary on a case-by-case
	76	+ basis.
	77	+
	78	+* All other SVE state of a thread, including the currently configured vector
	79	+ length, the state of the PR_SVE_VL_INHERIT flag, and the deferred vector
	80	+ length (if any), is preserved across all syscalls, subject to the specific
	81	+ exceptions for execve() described in section 6.
	82	+
	83	+ In particular, on return from a fork() or clone(), the parent and new child
	84	+ process or thread share identical SVE configuration, matching that of the
	85	+ parent before the call.
	86	+
	87	+
	88	+4. Signal handling
	89	+-------------------
	90	+
	91	+* A new signal frame record sve_context encodes the SVE registers on signal
	92	+ delivery. [1]
	93	+
	94	+* This record is supplementary to fpsimd_context. The FPSR and FPCR registers
	95	+ are only present in fpsimd_context. For convenience, the content of V0..V31
	96	+ is duplicated between sve_context and fpsimd_context.
	97	+
	98	+* The signal frame record for SVE always contains basic metadata, in particular
	99	+ the thread's vector length (in sve_context.vl).
	100	+
	101	+* The SVE registers may or may not be included in the record, depending on
	102	+ whether the registers are live for the thread. The registers are present if
	103	+ and only if:
	104	+ sve_context.head.size >= SVE_SIG_CONTEXT_SIZE(sve_vq_from_vl(sve_context.vl)).
	105	+
	106	+* If the registers are present, the remainder of the record has a vl-dependent
	107	+ size and layout. Macros SVE_SIG_* are defined [1] to facilitate access to
	108	+ the members.
	109	+
	110	+* If the SVE context is too big to fit in sigcontext.__reserved[], then extra
	111	+ space is allocated on the stack, an extra_context record is written in
	112	+ __reserved[] referencing this space. sve_context is then written in the
	113	+ extra space. Refer to [1] for further details about this mechanism.
	114	+
	115	+
	116	+5. Signal return
	117	+-----------------
	118	+
	119	+When returning from a signal handler:
	120	+
	121	+* If there is no sve_context record in the signal frame, or if the record is
	122	+ present but contains no register data as desribed in the previous section,
	123	+ then the SVE registers/bits become non-live and take unspecified values.
	124	+
	125	+* If sve_context is present in the signal frame and contains full register
	126	+ data, the SVE registers become live and are populated with the specified
	127	+ data. However, for backward compatibility reasons, bits [127:0] of Z0..Z31
	128	+ are always restored from the corresponding members of fpsimd_context.vregs[]
	129	+ and not from sve_context. The remaining bits are restored from sve_context.
	130	+
	131	+* Inclusion of fpsimd_context in the signal frame remains mandatory,
	132	+ irrespective of whether sve_context is present or not.
	133	+
	134	+* The vector length cannot be changed via signal return. If sve_context.vl in
	135	+ the signal frame does not match the current vector length, the signal return
	136	+ attempt is treated as illegal, resulting in a forced SIGSEGV.
	137	+
	138	+
	139	+6. prctl extensions
	140	+--------------------
	141	+
	142	+Some new prctl() calls are added to allow programs to manage the SVE vector
	143	+length:
	144	+
	145	+prctl(PR_SVE_SET_VL, unsigned long arg)
	146	+
	147	+ Sets the vector length of the calling thread and related flags, where
	148	+ arg == vl \| flags. Other threads of the calling process are unaffected.
	149	+
	150	+ vl is the desired vector length, where sve_vl_valid(vl) must be true.
	151	+
	152	+ flags:
	153	+
	154	+ PR_SVE_SET_VL_INHERIT
	155	+
	156	+ Inherit the current vector length across execve(). Otherwise, the
	157	+ vector length is reset to the system default at execve(). (See
	158	+ Section 9.)
	159	+
	160	+ PR_SVE_SET_VL_ONEXEC
	161	+
	162	+ Defer the requested vector length change until the next execve()
	163	+ performed by this thread.
	164	+
	165	+ The effect is equivalent to implicit exceution of the following
	166	+ call immediately after the next execve() (if any) by the thread:
	167	+
	168	+ prctl(PR_SVE_SET_VL, arg & ~PR_SVE_SET_VL_ONEXEC)
	169	+
	170	+ This allows launching of a new program with a different vector
	171	+ length, while avoiding runtime side effects in the caller.
	172	+
	173	+
	174	+ Without PR_SVE_SET_VL_ONEXEC, the requested change takes effect
	175	+ immediately.
	176	+
	177	+
	178	+ Return value: a nonnegative on success, or a negative value on error:
	179	+ EINVAL: SVE not supported, invalid vector length requested, or
	180	+ invalid flags.
	181	+
	182	+
	183	+ On success:
	184	+
	185	+ * Either the calling thread's vector length or the deferred vector length
	186	+ to be applied at the next execve() by the thread (dependent on whether
	187	+ PR_SVE_SET_VL_ONEXEC is present in arg), is set to the largest value
	188	+ supported by the system that is less than or equal to vl. If vl ==
	189	+ SVE_VL_MAX, the value set will be the largest value supported by the
	190	+ system.
	191	+
	192	+ * Any previously outstanding deferred vector length change in the calling
	193	+ thread is cancelled.
	194	+
	195	+ * The returned value describes the resulting configuration, encoded as for
	196	+ PR_SVE_GET_VL. The vector length reported in this value is the new
	197	+ current vector length for this thread if PR_SVE_SET_VL_ONEXEC was not
	198	+ present in arg; otherwise, the reported vector length is the deferred
	199	+ vector length that will be applied at the next execve() by the calling
	200	+ thread.
	201	+
	202	+ * Changing the vector length causes all of P0..P15, FFR and all bits of
	203	+ Z0..V31 except for Z0 bits [127:0] .. Z31 bits [127:0] to become
	204	+ unspecified. Calling PR_SVE_SET_VL with vl equal to the thread's current
	205	+ vector length, or calling PR_SVE_SET_VL with the PR_SVE_SET_VL_ONEXEC
	206	+ flag, does not constitute a change to the vector length for this purpose.
	207	+
	208	+
	209	+prctl(PR_SVE_GET_VL)
	210	+
	211	+ Gets the vector length of the calling thread.
	212	+
	213	+ The following flag may be OR-ed into the result:
	214	+
	215	+ PR_SVE_SET_VL_INHERIT
	216	+
	217	+ Vector length will be inherited across execve().
	218	+
	219	+ There is no way to determine whether there is an outstanding deferred
	220	+ vector length change (which would only normally be the case between a
	221	+ fork() or vfork() and the corresponding execve() in typical use).
	222	+
	223	+ To extract the vector length from the result, and it with
	224	+ PR_SVE_VL_LEN_MASK.
	225	+
	226	+ Return value: a nonnegative value on success, or a negative value on error:
	227	+ EINVAL: SVE not supported.
	228	+
	229	+
	230	+7. ptrace extensions
	231	+---------------------
	232	+
	233	+* A new regset NT_ARM_SVE is defined for use with PTRACE_GETREGSET and
	234	+ PTRACE_SETREGSET.
	235	+
	236	+ Refer to [2] for definitions.
	237	+
	238	+The regset data starts with struct user_sve_header, containing:
	239	+
	240	+ size
	241	+
	242	+ Size of the complete regset, in bytes.
	243	+ This depends on vl and possibly on other things in the future.
	244	+
	245	+ If a call to PTRACE_GETREGSET requests less data than the value of
	246	+ size, the caller can allocate a larger buffer and retry in order to
	247	+ read the complete regset.
	248	+
	249	+ max_size
	250	+
	251	+ Maximum size in bytes that the regset can grow to for the target
	252	+ thread. The regset won't grow bigger than this even if the target
	253	+ thread changes its vector length etc.
	254	+
	255	+ vl
	256	+
	257	+ Target thread's current vector length, in bytes.
	258	+
	259	+ max_vl
	260	+
	261	+ Maximum possible vector length for the target thread.
	262	+
	263	+ flags
	264	+
	265	+ either
	266	+
	267	+ SVE_PT_REGS_FPSIMD
	268	+
	269	+ SVE registers are not live (GETREGSET) or are to be made
	270	+ non-live (SETREGSET).
	271	+
	272	+ The payload is of type struct user_fpsimd_state, with the same
	273	+ meaning as for NT_PRFPREG, starting at offset
	274	+ SVE_PT_FPSIMD_OFFSET from the start of user_sve_header.
	275	+
	276	+ Extra data might be appended in the future: the size of the
	277	+ payload should be obtained using SVE_PT_FPSIMD_SIZE(vq, flags).
	278	+
	279	+ vq should be obtained using sve_vq_from_vl(vl).
	280	+
	281	+ or
	282	+
	283	+ SVE_PT_REGS_SVE
	284	+
	285	+ SVE registers are live (GETREGSET) or are to be made live
	286	+ (SETREGSET).
	287	+
	288	+ The payload contains the SVE register data, starting at offset
	289	+ SVE_PT_SVE_OFFSET from the start of user_sve_header, and with
	290	+ size SVE_PT_SVE_SIZE(vq, flags);
	291	+
	292	+ ... OR-ed with zero or more of the following flags, which have the same
	293	+ meaning and behaviour as the corresponding PR_SET_VL_* flags:
	294	+
	295	+ SVE_PT_VL_INHERIT
	296	+
	297	+ SVE_PT_VL_ONEXEC (SETREGSET only).
	298	+
	299	+* The effects of changing the vector length and/or flags are equivalent to
	300	+ those documented for PR_SVE_SET_VL.
	301	+
	302	+ The caller must make a further GETREGSET call if it needs to know what VL is
	303	+ actually set by SETREGSET, unless is it known in advance that the requested
	304	+ VL is supported.
	305	+
	306	+* In the SVE_PT_REGS_SVE case, the size and layout of the payload depends on
	307	+ the header fields. The SVE_PT_SVE_*() macros are provided to facilitate
	308	+ access to the members.
	309	+
	310	+* In either case, for SETREGSET it is permissible to omit the payload, in which
	311	+ case only the vector length and flags are changed (along with any
	312	+ consequences of those changes).
	313	+
	314	+* For SETREGSET, if an SVE_PT_REGS_SVE payload is present and the
	315	+ requested VL is not supported, the effect will be the same as if the
	316	+ payload were omitted, except that an EIO error is reported. No
	317	+ attempt is made to translate the payload data to the correct layout
	318	+ for the vector length actually set. The thread's FPSIMD state is
	319	+ preserved, but the remaining bits of the SVE registers become
	320	+ unspecified. It is up to the caller to translate the payload layout
	321	+ for the actual VL and retry.
	322	+
	323	+* The effect of writing a partial, incomplete payload is unspecified.
	324	+
	325	+
	326	+8. ELF coredump extensions
	327	+---------------------------
	328	+
	329	+* A NT_ARM_SVE note will be added to each coredump for each thread of the
	330	+ dumped process. The contents will be equivalent to the data that would have
	331	+ been read if a PTRACE_GETREGSET of NT_ARM_SVE were executed for each thread
	332	+ when the coredump was generated.
	333	+
	334	+
	335	+9. System runtime configuration
	336	+--------------------------------
	337	+
	338	+* To mitigate the ABI impact of expansion of the signal frame, a policy
	339	+ mechanism is provided for administrators, distro maintainers and developers
	340	+ to set the default vector length for userspace processes:
	341	+
	342	+/proc/sys/abi/sve_default_vector_length
	343	+
	344	+ Writing the text representation of an integer to this file sets the system
	345	+ default vector length to the specified value, unless the value is greater
	346	+ than the maximum vector length supported by the system in which case the
	347	+ default vector length is set to that maximum.
	348	+
	349	+ The result can be determined by reopening the file and reading its
	350	+ contents.
	351	+
	352	+ At boot, the default vector length is initially set to 64 or the maximum
	353	+ supported vector length, whichever is smaller. This determines the initial
	354	+ vector length of the init process (PID 1).
	355	+
	356	+ Reading this file returns the current system default vector length.
	357	+
	358	+* At every execve() call, the new vector length of the new process is set to
	359	+ the system default vector length, unless
	360	+
	361	+ * PR_SVE_SET_VL_INHERIT (or equivalently SVE_PT_VL_INHERIT) is set for the
	362	+ calling thread, or
	363	+
	364	+ * a deferred vector length change is pending, established via the
	365	+ PR_SVE_SET_VL_ONEXEC flag (or SVE_PT_VL_ONEXEC).
	366	+
	367	+* Modifying the system default vector length does not affect the vector length
	368	+ of any existing process or thread that does not make an execve() call.
	369	+
	370	+
	371	+Appendix A. SVE programmer's model (informative)
	372	+=================================================
	373	+
	374	+This section provides a minimal description of the additions made by SVE to the
	375	+ARMv8-A programmer's model that are relevant to this document.
	376	+
	377	+Note: This section is for information only and not intended to be complete or
	378	+to replace any architectural specification.
	379	+
	380	+A.1. Registers
	381	+---------------
	382	+
	383	+In A64 state, SVE adds the following:
	384	+
	385	+* 32 8VL-bit vector registers Z0..Z31
	386	+ For each Zn, Zn bits [127:0] alias the ARMv8-A vector register Vn.
	387	+
	388	+ A register write using a Vn register name zeros all bits of the corresponding
	389	+ Zn except for bits [127:0].
	390	+
	391	+* 16 VL-bit predicate registers P0..P15
	392	+
	393	+* 1 VL-bit special-purpose predicate register FFR (the "first-fault register")
	394	+
	395	+* a VL "pseudo-register" that determines the size of each vector register
	396	+
	397	+ The SVE instruction set architecture provides no way to write VL directly.
	398	+ Instead, it can be modified only by EL1 and above, by writing appropriate
	399	+ system registers.
	400	+
	401	+* The value of VL can be configured at runtime by EL1 and above:
	402	+ 16 <= VL <= VLmax, where VL must be a multiple of 16.
	403	+
	404	+* The maximum vector length is determined by the hardware:
	405	+ 16 <= VLmax <= 256.
	406	+
	407	+ (The SVE architecture specifies 256, but permits future architecture
	408	+ revisions to raise this limit.)
	409	+
	410	+* FPSR and FPCR are retained from ARMv8-A, and interact with SVE floating-point
	411	+ operations in a similar way to the way in which they interact with ARMv8
	412	+ floating-point operations.
	413	+
	414	+ 8VL-1 128 0 bit index
	415	+ +---- //// -----------------+
	416	+ Z0 \| : V0 \|
	417	+ : :
	418	+ Z7 \| : V7 \|
	419	+ Z8 \| : * V8 \|
	420	+ : : :
	421	+ Z15 \| : *V15 \|
	422	+ Z16 \| : V16 \|
	423	+ : :
	424	+ Z31 \| : V31 \|
	425	+ +---- //// -----------------+
	426	+ 31 0
	427	+ VL-1 0 +-------+
	428	+ +---- //// --+ FPSR \| \|
	429	+ P0 \| \| +-------+
	430	+ : \| \| *FPCR \| \|
	431	+ P15 \| \| +-------+
	432	+ +---- //// --+
	433	+ FFR \| \| +-----+
	434	+ +---- //// --+ VL \| \|
	435	+ +-----+
	436	+
	437	+(*) callee-save:
	438	+ This only applies to bits [63:0] of Z-/V-registers.
	439	+ FPCR contains callee-save and caller-save bits. See [4] for details.
	440	+
	441	+
	442	+A.2. Procedure call standard
	443	+-----------------------------
	444	+
	445	+The ARMv8-A base procedure call standard is extended as follows with respect to
	446	+the additional SVE register state:
	447	+
	448	+* All SVE register bits that are not shared with FP/SIMD are caller-save.
	449	+
	450	+* Z8 bits [63:0] .. Z15 bits [63:0] are callee-save.
	451	+
	452	+ This follows from the way these bits are mapped to V8..V15, which are caller-
	453	+ save in the base procedure call standard.
	454	+
	455	+
	456	+Appendix B. ARMv8-A FP/SIMD programmer's model
	457	+===============================================
	458	+
	459	+Note: This section is for information only and not intended to be complete or
	460	+to replace any architectural specification.
	461	+
	462	+Refer to [4] for for more information.
	463	+
	464	+ARMv8-A defines the following floating-point / SIMD register state:
	465	+
	466	+* 32 128-bit vector registers V0..V31
	467	+* 2 32-bit status/control registers FPSR, FPCR
	468	+
	469	+ 127 0 bit index
	470	+ +---------------+
	471	+ V0 \| \|
	472	+ : : :
	473	+ V7 \| \|
	474	+ * V8 \| \|
	475	+ : : : :
	476	+ *V15 \| \|
	477	+ V16 \| \|
	478	+ : : :
	479	+ V31 \| \|
	480	+ +---------------+
	481	+
	482	+ 31 0
	483	+ +-------+
	484	+ FPSR \| \|
	485	+ +-------+
	486	+ *FPCR \| \|
	487	+ +-------+
	488	+
	489	+(*) callee-save:
	490	+ This only applies to bits [63:0] of V-registers.
	491	+ FPCR contains a mixture of callee-save and caller-save bits.
	492	+
	493	+
	494	+References
	495	+==========
	496	+
	497	+[1] arch/arm64/include/uapi/asm/sigcontext.h
	498	+ AArch64 Linux signal ABI definitions
	499	+
	500	+[2] arch/arm64/include/uapi/asm/ptrace.h
	501	+ AArch64 Linux ptrace ABI definitions
	502	+
	503	+[3] linux/Documentation/arm64/cpu-feature-registers.txt
	504	+
	505	+[4] ARM IHI0055C
	506	+ http://infocenter.arm.com/help/topic/com.arm.doc.ihi0055c/IHI0055C_beta_aapcs64.pdf
	507	+ http://infocenter.arm.com/help/topic/com.arm.doc.subset.swdev.abi/index.html
	508	+ Procedure Call Standard for the ARM 64-bit Architecture (AArch64)

Documentation/devicetree/bindings/arm/spe-pmu.txt+20−0

@@ -0,0 +1,20 @@
	1	+* ARMv8.2 Statistical Profiling Extension (SPE) Performance Monitor Units (PMU)
	2	+
	3	+ARMv8.2 introduces the optional Statistical Profiling Extension for collecting
	4	+performance sample data using an in-memory trace buffer.
	5	+
	6	+** SPE Required properties:
	7	+
	8	+- compatible : should be one of:
	9	+ "arm,statistical-profiling-extension-v1"
	10	+
	11	+- interrupts : Exactly 1 PPI must be listed. For heterogeneous systems where
	12	+ SPE is only supported on a subset of the CPUs, please consult
	13	+ the arm,gic-v3 binding for details on describing a PPI partition.
	14	+
	15	+** Example:
	16	+
	17	+spe-pmu {
	18	+ compatible = "arm,statistical-profiling-extension-v1";
	19	+ interrupts = <GIC_PPI 05 IRQ_TYPE_LEVEL_HIGH &part1>;
	20	+};

Documentation/perf/hisi-pmu.txt+53−0

@@ -0,0 +1,53 @@
	1	+HiSilicon SoC uncore Performance Monitoring Unit (PMU)
	2	+======================================================
	3	+The HiSilicon SoC chip includes various independent system device PMUs
	4	+such as L3 cache (L3C), Hydra Home Agent (HHA) and DDRC. These PMUs are
	5	+independent and have hardware logic to gather statistics and performance
	6	+information.
	7	+
	8	+The HiSilicon SoC encapsulates multiple CPU and IO dies. Each CPU cluster
	9	+(CCL) is made up of 4 cpu cores sharing one L3 cache; each CPU die is
	10	+called Super CPU cluster (SCCL) and is made up of 6 CCLs. Each SCCL has
	11	+two HHAs (0 - 1) and four DDRCs (0 - 3), respectively.
	12	+
	13	+HiSilicon SoC uncore PMU driver
	14	+---------------------------------------
	15	+Each device PMU has separate registers for event counting, control and
	16	+interrupt, and the PMU driver shall register perf PMU drivers like L3C,
	17	+HHA and DDRC etc. The available events and configuration options shall
	18	+be described in the sysfs, see :
	19	+/sys/devices/hisi_sccl{X}_<l3c{Y}/hha{Y}/ddrc{Y}>/, or
	20	+/sys/bus/event_source/devices/hisi_sccl{X}_<l3c{Y}/hha{Y}/ddrc{Y}>.
	21	+The "perf list" command shall list the available events from sysfs.
	22	+
	23	+Each L3C, HHA and DDRC is registered as a separate PMU with perf. The PMU
	24	+name will appear in event listing as hisi_sccl<sccl-id>_module<index-id>.
	25	+where "sccl-id" is the identifier of the SCCL and "index-id" is the index of
	26	+module.
	27	+e.g. hisi_sccl3_l3c0/rd_hit_cpipe is READ_HIT_CPIPE event of L3C index #0 in
	28	+SCCL ID #3.
	29	+e.g. hisi_sccl1_hha0/rx_operations is RX_OPERATIONS event of HHA index #0 in
	30	+SCCL ID #1.
	31	+
	32	+The driver also provides a "cpumask" sysfs attribute, which shows the CPU core
	33	+ID used to count the uncore PMU event.
	34	+
	35	+Example usage of perf:
	36	+$# perf list
	37	+hisi_sccl3_l3c0/rd_hit_cpipe/ [kernel PMU event]
	38	+------------------------------------------
	39	+hisi_sccl3_l3c0/wr_hit_cpipe/ [kernel PMU event]
	40	+------------------------------------------
	41	+hisi_sccl1_l3c0/rd_hit_cpipe/ [kernel PMU event]
	42	+------------------------------------------
	43	+hisi_sccl1_l3c0/wr_hit_cpipe/ [kernel PMU event]
	44	+------------------------------------------
	45	+
	46	+$# perf stat -a -e hisi_sccl3_l3c0/rd_hit_cpipe/ sleep 5
	47	+$# perf stat -a -e hisi_sccl3_l3c0/config=0x02/ sleep 5
	48	+
	49	+The current driver does not support sampling. So "perf record" is unsupported.
	50	+Also attach to a task is unsupported as the events are all uncore.
	51	+
	52	+Note: Please contact the maintainer for a complete list of events supported for
	53	+the PMU devices in the SoC and its information if needed.

MAINTAINERS+7−0

@@ -6259,6 +6259,13 @@ S: Maintained · 6258 unmodified lines
6259	6259	F: drivers/net/ethernet/hisilicon/
6260	6260	F: Documentation/devicetree/bindings/net/hisilicon*.txt
6261	6261
	6262	+HISILICON PMU DRIVER
	6263	+M: Shaokun Zhang <zhangshaokun@hisilicon.com>
	6264	+W: http://www.hisilicon.com
	6265	+S: Supported
	6266	+F: drivers/perf/hisilicon
	6267	+F: Documentation/perf/hisi-pmu.txt
	6268	+
6262	6269	HISILICON ROCE DRIVER
6263	6270	M: Lijun Ou <oulijun@huawei.com>
6264	6271	M: Wei Hu(Xavier) <xavier.huwei@huawei.com>

arch/arm/include/asm/arch_timer.h+1−0

@@ -107,6 +107,7 @@ static inline u32 arch_timer_get_cntkctl(void) · 106 unmodified lines
107	107	static inline void arch_timer_set_cntkctl(u32 cntkctl)
108	108	{
109	109	asm volatile("mcr p15, 0, %0, c14, c1, 0" : : "r" (cntkctl));
	110	+ isb();
110	111	}
111	112
112	113	#endif

arch/arm/include/asm/kvm_host.h+3−0

@@ -293,4 +293,7 @@ int kvm_arm_vcpu_arch_get_attr(struct kvm_vcpu *vcpu, · 292 unmodified lines
293	293	int kvm_arm_vcpu_arch_has_attr(struct kvm_vcpu *vcpu,
294	294	struct kvm_device_attr *attr);
295	295
	296	+/* All host FP/SIMD state is restored on guest exit, so nothing to save: */
	297	+static inline void kvm_fpsimd_flush_cpu_state(void) {}
	298	+
296	299	#endif /* __ARM_KVM_HOST_H__ */

arch/arm64/Kconfig+16−2

@@ -21,7 +21,7 @@ config ARM64 · 20 unmodified lines
21	21	select ARCH_HAS_STRICT_KERNEL_RWX
22	22	select ARCH_HAS_STRICT_MODULE_RWX
23	23	select ARCH_HAS_TICK_BROADCAST if GENERIC_CLOCKEVENTS_BROADCAST
24		− select ARCH_HAVE_NMI_SAFE_CMPXCHG if ACPI_APEI_SEA
	24	+ select ARCH_HAVE_NMI_SAFE_CMPXCHG
25	25	select ARCH_INLINE_READ_LOCK if !PREEMPT
26	26	select ARCH_INLINE_READ_LOCK_BH if !PREEMPT
27	27	select ARCH_INLINE_READ_LOCK_IRQ if !PREEMPT
@@ -115,7 +115,7 @@ config ARM64 · 87 unmodified lines
115	115	select HAVE_IRQ_TIME_ACCOUNTING
116	116	select HAVE_MEMBLOCK
117	117	select HAVE_MEMBLOCK_NODE_MAP if NUMA
118		− select HAVE_NMI if ACPI_APEI_SEA
	118	+ select HAVE_NMI
119	119	select HAVE_PATA_PLATFORM
120	120	select HAVE_PERF_EVENTS
121	121	select HAVE_PERF_REGS
@@ -136,6 +136,7 @@ config ARM64 · 14 unmodified lines
136	136	select PCI_ECAM if ACPI
137	137	select POWER_RESET
138	138	select POWER_SUPPLY
	139	+ select REFCOUNT_FULL
139	140	select SPARSE_IRQ
140	141	select SYSCTL_EXCEPTION_TRACE
141	142	select THREAD_INFO_IN_TASK
@@ -842,6 +843,7 @@ config FORCE_MAX_ZONEORDER · 700 unmodified lines
842	843	menuconfig ARMV8_DEPRECATED
843	844	bool "Emulate deprecated/obsolete ARMv8 instructions"
844	845	depends on COMPAT
	846	+ depends on SYSCTL
845	847	help
846	848	Legacy software support may require certain instructions
847	849	that have been deprecated or obsoleted in the architecture.
@@ -1011,6 +1013,17 @@ config ARM64_PMEM · 163 unmodified lines
1011	1013
1012	1014	endmenu
1013	1015
	1016	+config ARM64_SVE
	1017	+ bool "ARM Scalable Vector Extension support"
	1018	+ default y
	1019	+ help
	1020	+ The Scalable Vector Extension (SVE) is an extension to the AArch64
	1021	+ execution state which complements and extends the SIMD functionality
	1022	+ of the base architecture to support much larger vectors and to enable
	1023	+ additional vectorisation opportunities.
	1024	+
	1025	+ To enable use of this extension on CPUs that implement it, say Y.
	1026	+
1014	1027	config ARM64_MODULE_CMODEL_LARGE
1015	1028	bool
1016	1029
@@ -1099,6 +1112,7 @@ config EFI_STUB · 82 unmodified lines
1099	1112	config EFI
1100	1113	bool "UEFI runtime support"
1101	1114	depends on OF && !CPU_BIG_ENDIAN
	1115	+ depends on KERNEL_MODE_NEON
1102	1116	select LIBFDT
1103	1117	select UCS2_STRING
1104	1118	select EFI_PARAMS_FROM_FDT

arch/arm64/Makefile+8−2

@@ -14,8 +14,12 @@ LDFLAGS_vmlinux :=-p --no-undefined -X · 13 unmodified lines
14	14	CPPFLAGS_vmlinux.lds = -DTEXT_OFFSET=$(TEXT_OFFSET)
15	15	GZFLAGS :=-9
16	16
17		−ifneq ($(CONFIG_RELOCATABLE),)
18		−LDFLAGS_vmlinux += -pie -shared -Bsymbolic
	17	+ifeq ($(CONFIG_RELOCATABLE), y)
	18	+# Pass --no-apply-dynamic-relocs to restore pre-binutils-2.27 behaviour
	19	+# for relative relocs, since this leads to better Image compression
	20	+# with the relocation offsets always being zero.
	21	+LDFLAGS_vmlinux += -pie -shared -Bsymbolic \
	22	+ $(call ld-option, --no-apply-dynamic-relocs)
19	23	endif
20	24
21	25	ifeq ($(CONFIG_ARM64_ERRATUM_843419),y)
@@ -53,6 +57,8 @@ KBUILD_AFLAGS += $(lseinstr) $(brokengasinst) · 31 unmodified lines
53	57	KBUILD_CFLAGS += $(call cc-option,-mabi=lp64)
54	58	KBUILD_AFLAGS += $(call cc-option,-mabi=lp64)
55	59
	60	+KBUILD_CFLAGS += $(call cc-ifversion, -ge, 0500, -DCONFIG_ARCH_SUPPORTS_INT128)
	61	+
56	62	ifeq ($(CONFIG_CPU_BIG_ENDIAN), y)
57	63	KBUILD_CPPFLAGS += -mbig-endian
58	64	CHECKFLAGS += -D__AARCH64EB__

arch/arm64/include/asm/arch_timer.h+1−0

@@ -144,6 +144,7 @@ static inline u32 arch_timer_get_cntkctl(void) · 143 unmodified lines
144	144	static inline void arch_timer_set_cntkctl(u32 cntkctl)
145	145	{
146	146	write_sysreg(cntkctl, cntkctl_el1);
	147	+ isb();
147	148	}
148	149
149	150	static inline u64 arch_counter_get_cntpct(void)

arch/arm64/include/asm/asm-bug.h+4−4

@@ -22,10 +22,10 @@ · 21 unmodified lines
22	22	#define _BUGVERBOSE_LOCATION(file, line) __BUGVERBOSE_LOCATION(file, line)
23	23	#define __BUGVERBOSE_LOCATION(file, line) \
24	24	.pushsection .rodata.str,"aMS",@progbits,1; \
25		− 2: .string file; \
	25	+ 14472: .string file; \
26	26	.popsection; \
27	27	\
28		− .long 2b - 0b; \
	28	+ .long 14472b - 14470b; \
29	29	.short line;
30	30	#else
31	31	#define _BUGVERBOSE_LOCATION(file, line)
@@ -36,11 +36,11 @@ · 4 unmodified lines
36	36	#define __BUG_ENTRY(flags) \
37	37	.pushsection __bug_table,"aw"; \
38	38	.align 2; \
39		− 0: .long 1f - 0b; \
	39	+ 14470: .long 14471f - 14470b; \
40	40	_BUGVERBOSE_LOCATION(__FILE__, __LINE__) \
41	41	.short flags; \
42	42	.popsection; \
43		− 1:
	43	+ 14471:
44	44	#else
45	45	#define __BUG_ENTRY(flags)
46	46	#endif

arch/arm64/include/asm/assembler.h+32−19

@@ -25,12 +25,41 @@ · 24 unmodified lines
25	25
26	26	#include <asm/asm-offsets.h>
27	27	#include <asm/cpufeature.h>
	28	+#include <asm/debug-monitors.h>
28	29	#include <asm/mmu_context.h>
29	30	#include <asm/page.h>
30	31	#include <asm/pgtable-hwdef.h>
31	32	#include <asm/ptrace.h>
32	33	#include <asm/thread_info.h>
33	34
	35	+ .macro save_and_disable_daif, flags
	36	+ mrs \flags, daif
	37	+ msr daifset, #0xf
	38	+ .endm
	39	+
	40	+ .macro disable_daif
	41	+ msr daifset, #0xf
	42	+ .endm
	43	+
	44	+ .macro enable_daif
	45	+ msr daifclr, #0xf
	46	+ .endm
	47	+
	48	+ .macro restore_daif, flags:req
	49	+ msr daif, \flags
	50	+ .endm
	51	+
	52	+ /* Only on aarch64 pstate, PSR_D_BIT is different for aarch32 */
	53	+ .macro inherit_daif, pstate:req, tmp:req
	54	+ and \tmp, \pstate, #(PSR_D_BIT \| PSR_A_BIT \| PSR_I_BIT \| PSR_F_BIT)
	55	+ msr daif, \tmp
	56	+ .endm
	57	+
	58	+ /* IRQ is the lowest priority flag, unconditionally unmask the rest. */
	59	+ .macro enable_da_f
	60	+ msr daifclr, #(8 \| 4 \| 1)
	61	+ .endm
	62	+
34	63	/*
35	64	* Enable and disable interrupts.
36	65	*/
@@ -51,13 +80,6 @@ · 14 unmodified lines
51	80	msr daif, \flags
52	81	.endm
53	82
54		−/*
55		− * Enable and disable debug exceptions.
56		− */
57		− .macro disable_dbg
58		− msr daifset, #8
59		− .endm
60		−
61	83	.macro enable_dbg
62	84	msr daifclr, #8
63	85	.endm
@@ -65,30 +87,21 @@ · 1 unmodified line
65	87	.macro disable_step_tsk, flgs, tmp
66	88	tbz \flgs, #TIF_SINGLESTEP, 9990f
67	89	mrs \tmp, mdscr_el1
68		− bic \tmp, \tmp, #1
	90	+ bic \tmp, \tmp, #DBG_MDSCR_SS
69	91	msr mdscr_el1, \tmp
70	92	isb // Synchronise with enable_dbg
71	93	9990:
72	94	.endm
73	95
	96	+ /* call with daif masked */
74	97	.macro enable_step_tsk, flgs, tmp
75	98	tbz \flgs, #TIF_SINGLESTEP, 9990f
76		− disable_dbg
77	99	mrs \tmp, mdscr_el1
78		− orr \tmp, \tmp, #1
	100	+ orr \tmp, \tmp, #DBG_MDSCR_SS
79	101	msr mdscr_el1, \tmp
80	102	9990:
81	103	.endm
82	104
83		−/*
84		− * Enable both debug exceptions and interrupts. This is likely to be
85		− * faster than two daifclr operations, since writes to this register
86		− * are self-synchronising.
87		− */
88		− .macro enable_dbg_and_irq
89		− msr daifclr, #(8 \| 2)
90		− .endm
91		−
92	105	/*
93	106	* SMP data memory barrier
94	107	*/

arch/arm64/include/asm/barrier.h+2−0

@@ -31,6 +31,8 @@ · 30 unmodified lines
31	31	#define dmb(opt) asm volatile("dmb " #opt : : : "memory")
32	32	#define dsb(opt) asm volatile("dsb " #opt : : : "memory")
33	33
	34	+#define psb_csync() asm volatile("hint #17" : : : "memory")
	35	+
34	36	#define mb() dsb(sy)
35	37	#define rmb() dsb(ld)
36	38	#define wmb() dsb(st)

arch/arm64/include/asm/cpu.h+4−0

@@ -41,6 +41,7 @@ struct cpuinfo_arm64 { · 40 unmodified lines
41	41	u64 reg_id_aa64mmfr2;
42	42	u64 reg_id_aa64pfr0;
43	43	u64 reg_id_aa64pfr1;
	44	+ u64 reg_id_aa64zfr0;
44	45
45	46	u32 reg_id_dfr0;
46	47	u32 reg_id_isar0;
@@ -59,6 +60,9 @@ struct cpuinfo_arm64 { · 12 unmodified lines
59	60	u32 reg_mvfr0;
60	61	u32 reg_mvfr1;
61	62	u32 reg_mvfr2;
	63	+
	64	+ /* pseudo-ZCR for recording maximum ZCR_EL1 LEN value: */
	65	+ u64 reg_zcr;
62	66	};
63	67
64	68	DECLARE_PER_CPU(struct cpuinfo_arm64, cpu_data);

arch/arm64/include/asm/cpucaps.h+2−1

@@ -40,7 +40,8 @@ · 39 unmodified lines
40	40	#define ARM64_WORKAROUND_858921 19
41	41	#define ARM64_WORKAROUND_CAVIUM_30115 20
42	42	#define ARM64_HAS_DCPOP 21
	43	+#define ARM64_SVE 22
43	44
44		−#define ARM64_NCAPS 22
	45	+#define ARM64_NCAPS 23
45	46
46	47	#endif /* __ASM_CPUCAPS_H */

arch/arm64/include/asm/cpufeature.h+42−0

@@ -10,7 +10,9 @@ · 9 unmodified lines
10	10	#define __ASM_CPUFEATURE_H
11	11
12	12	#include <asm/cpucaps.h>
	13	+#include <asm/fpsimd.h>
13	14	#include <asm/hwcap.h>
	15	+#include <asm/sigcontext.h>
14	16	#include <asm/sysreg.h>
15	17
16	18	/*
@@ -223,6 +225,13 @@ static inline bool id_aa64pfr0_32bit_el0(u64 pfr0) · 206 unmodified lines
223	225	return val == ID_AA64PFR0_EL0_32BIT_64BIT;
224	226	}
225	227
	228	+static inline bool id_aa64pfr0_sve(u64 pfr0)
	229	+{
	230	+ u32 val = cpuid_feature_extract_unsigned_field(pfr0, ID_AA64PFR0_SVE_SHIFT);
	231	+
	232	+ return val > 0;
	233	+}
	234	+
226	235	void __init setup_cpu_features(void);
227	236
228	237	void update_cpu_capabilities(const struct arm64_cpu_capabilities *caps,
@@ -262,6 +271,39 @@ static inline bool system_uses_ttbr0_pan(void) · 33 unmodified lines
262	271	!cpus_have_const_cap(ARM64_HAS_PAN);
263	272	}
264	273
	274	+static inline bool system_supports_sve(void)
	275	+{
	276	+ return IS_ENABLED(CONFIG_ARM64_SVE) &&
	277	+ cpus_have_const_cap(ARM64_SVE);
	278	+}
	279	+
	280	+/*
	281	+ * Read the pseudo-ZCR used by cpufeatures to identify the supported SVE
	282	+ * vector length.
	283	+ *
	284	+ * Use only if SVE is present.
	285	+ * This function clobbers the SVE vector length.
	286	+ */
	287	+static inline u64 read_zcr_features(void)
	288	+{
	289	+ u64 zcr;
	290	+ unsigned int vq_max;
	291	+
	292	+ /*
	293	+ * Set the maximum possible VL, and write zeroes to all other
	294	+ * bits to see if they stick.
	295	+ */
	296	+ sve_kernel_enable(NULL);
	297	+ write_sysreg_s(ZCR_ELx_LEN_MASK, SYS_ZCR_EL1);
	298	+
	299	+ zcr = read_sysreg_s(SYS_ZCR_EL1);
	300	+ zcr &= ~(u64)ZCR_ELx_LEN_MASK; /* find sticky 1s outside LEN field */
	301	+ vq_max = sve_vq_from_vl(sve_get_vl());
	302	+ zcr \|= vq_max - 1; /* set LEN field to maximum effective value */
	303	+
	304	+ return zcr;
	305	+}
	306	+
265	307	#endif /* __ASSEMBLY__ */
266	308
267	309	#endif

arch/arm64/include/asm/daifflags.h+72−0

@@ -0,0 +1,72 @@
	1	+/*
	2	+ * Copyright (C) 2017 ARM Ltd.
	3	+ *
	4	+ * This program is free software; you can redistribute it and/or modify
	5	+ * it under the terms of the GNU General Public License version 2 as
	6	+ * published by the Free Software Foundation.
	7	+ *
	8	+ * This program is distributed in the hope that it will be useful,
	9	+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
	10	+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	11	+ * GNU General Public License for more details.
	12	+ *
	13	+ * You should have received a copy of the GNU General Public License
	14	+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
	15	+ */
	16	+#ifndef __ASM_DAIFFLAGS_H
	17	+#define __ASM_DAIFFLAGS_H
	18	+
	19	+#include <linux/irqflags.h>
	20	+
	21	+#define DAIF_PROCCTX 0
	22	+#define DAIF_PROCCTX_NOIRQ PSR_I_BIT
	23	+
	24	+/* mask/save/unmask/restore all exceptions, including interrupts. */
	25	+static inline void local_daif_mask(void)
	26	+{
	27	+ asm volatile(
	28	+ "msr daifset, #0xf // local_daif_mask\n"
	29	+ :
	30	+ :
	31	+ : "memory");
	32	+ trace_hardirqs_off();
	33	+}
	34	+
	35	+static inline unsigned long local_daif_save(void)
	36	+{
	37	+ unsigned long flags;
	38	+
	39	+ asm volatile(
	40	+ "mrs %0, daif // local_daif_save\n"
	41	+ : "=r" (flags)
	42	+ :
	43	+ : "memory");
	44	+ local_daif_mask();
	45	+
	46	+ return flags;
	47	+}
	48	+
	49	+static inline void local_daif_unmask(void)
	50	+{
	51	+ trace_hardirqs_on();
	52	+ asm volatile(
	53	+ "msr daifclr, #0xf // local_daif_unmask"
	54	+ :
	55	+ :
	56	+ : "memory");
	57	+}
	58	+
	59	+static inline void local_daif_restore(unsigned long flags)
	60	+{
	61	+ if (!arch_irqs_disabled_flags(flags))
	62	+ trace_hardirqs_on();
	63	+ asm volatile(
	64	+ "msr daif, %0 // local_daif_restore"
	65	+ :
	66	+ : "r" (flags)
	67	+ : "memory");
	68	+ if (arch_irqs_disabled_flags(flags))
	69	+ trace_hardirqs_off();
	70	+}
	71	+
	72	+#endif

arch/arm64/include/asm/elf.h+2−2

@@ -188,8 +188,8 @@ typedef compat_elf_greg_t compat_elf_gregset_t[COMPAT_ELF_NGREG]; · 187 unmodified lines
188	188
189	189	#define compat_start_thread compat_start_thread
190	190	/*
191		− * Unlike the native SET_PERSONALITY macro, the compat version inherits
192		− * READ_IMPLIES_EXEC across a fork() since this is the behaviour on
	191	+ * Unlike the native SET_PERSONALITY macro, the compat version maintains
	192	+ * READ_IMPLIES_EXEC across an execve() since this is the behaviour on
193	193	* arch/arm/.
194	194	*/
195	195	#define COMPAT_SET_PERSONALITY(ex) \

arch/arm64/include/asm/esr.h+2−1

@@ -43,7 +43,8 @@ · 42 unmodified lines
43	43	#define ESR_ELx_EC_HVC64 (0x16)
44	44	#define ESR_ELx_EC_SMC64 (0x17)
45	45	#define ESR_ELx_EC_SYS64 (0x18)
46		−/* Unallocated EC: 0x19 - 0x1E */
	46	+#define ESR_ELx_EC_SVE (0x19)
	47	+/* Unallocated EC: 0x1A - 0x1E */
47	48	#define ESR_ELx_EC_IMP_DEF (0x1f)
48	49	#define ESR_ELx_EC_IABT_LOW (0x20)
49	50	#define ESR_ELx_EC_IABT_CUR (0x21)

arch/arm64/include/asm/fpsimd.h+70−1

@@ -17,9 +17,13 @@ · 16 unmodified lines
17	17	#define __ASM_FP_H
18	18
19	19	#include <asm/ptrace.h>
	20	+#include <asm/errno.h>
20	21
21	22	#ifndef __ASSEMBLY__
22	23
	24	+#include <linux/cache.h>
	25	+#include <linux/stddef.h>
	26	+
23	27	/*
24	28	* FP/SIMD storage area has:
25	29	* - FPSR and FPCR
@@ -35,13 +39,16 @@ struct fpsimd_state { · 9 unmodified lines
35	39	__uint128_t vregs[32];
36	40	u32 fpsr;
37	41	u32 fpcr;
	42	+ /*
	43	+ * For ptrace compatibility, pad to next 128-bit
	44	+ * boundary here if extending this struct.
	45	+ */
38	46	};
39	47	};
40	48	/* the id of the last cpu to have restored this state */
41	49	unsigned int cpu;
42	50	};
43	51
44		−
45	52	#if defined(__KERNEL__) && defined(CONFIG_COMPAT)
46	53	/* Masks for extracting the FPSR and FPCR from the FPSCR */
47	54	#define VFP_FPSCR_STAT_MASK 0xf800009f
@@ -61,11 +68,73 @@ extern void fpsimd_load_state(struct fpsimd_state *state); · 13 unmodified lines
61	68	extern void fpsimd_thread_switch(struct task_struct *next);
62	69	extern void fpsimd_flush_thread(void);
63	70
	71	+extern void fpsimd_signal_preserve_current_state(void);
64	72	extern void fpsimd_preserve_current_state(void);
65	73	extern void fpsimd_restore_current_state(void);
66	74	extern void fpsimd_update_current_state(struct fpsimd_state *state);
67	75
68	76	extern void fpsimd_flush_task_state(struct task_struct *target);
	77	+extern void sve_flush_cpu_state(void);
	78	+
	79	+/* Maximum VL that SVE VL-agnostic software can transparently support */
	80	+#define SVE_VL_ARCH_MAX 0x100
	81	+
	82	+extern void sve_save_state(void state, u32 pfpsr);
	83	+extern void sve_load_state(void const state, u32 const pfpsr,
	84	+ unsigned long vq_minus_1);
	85	+extern unsigned int sve_get_vl(void);
	86	+extern int sve_kernel_enable(void *);
	87	+
	88	+extern int __ro_after_init sve_max_vl;
	89	+
	90	+#ifdef CONFIG_ARM64_SVE
	91	+
	92	+extern size_t sve_state_size(struct task_struct const *task);
	93	+
	94	+extern void sve_alloc(struct task_struct *task);
	95	+extern void fpsimd_release_task(struct task_struct *task);
	96	+extern void fpsimd_sync_to_sve(struct task_struct *task);
	97	+extern void sve_sync_to_fpsimd(struct task_struct *task);
	98	+extern void sve_sync_from_fpsimd_zeropad(struct task_struct *task);
	99	+
	100	+extern int sve_set_vector_length(struct task_struct *task,
	101	+ unsigned long vl, unsigned long flags);
	102	+
	103	+extern int sve_set_current_vl(unsigned long arg);
	104	+extern int sve_get_current_vl(void);
	105	+
	106	+/*
	107	+ * Probing and setup functions.
	108	+ * Calls to these functions must be serialised with one another.
	109	+ */
	110	+extern void __init sve_init_vq_map(void);
	111	+extern void sve_update_vq_map(void);
	112	+extern int sve_verify_vq_map(void);
	113	+extern void __init sve_setup(void);
	114	+
	115	+#else /* ! CONFIG_ARM64_SVE */
	116	+
	117	+static inline void sve_alloc(struct task_struct *task) { }
	118	+static inline void fpsimd_release_task(struct task_struct *task) { }
	119	+static inline void sve_sync_to_fpsimd(struct task_struct *task) { }
	120	+static inline void sve_sync_from_fpsimd_zeropad(struct task_struct *task) { }
	121	+
	122	+static inline int sve_set_current_vl(unsigned long arg)
	123	+{
	124	+ return -EINVAL;
	125	+}
	126	+
	127	+static inline int sve_get_current_vl(void)
	128	+{
	129	+ return -EINVAL;
	130	+}
	131	+
	132	+static inline void sve_init_vq_map(void) { }
	133	+static inline void sve_update_vq_map(void) { }
	134	+static inline int sve_verify_vq_map(void) { return 0; }
	135	+static inline void sve_setup(void) { }
	136	+
	137	+#endif /* ! CONFIG_ARM64_SVE */
69	138
70	139	/* For use by EFI runtime services calls only */
71	140	extern void __efi_fpsimd_begin(void);

arch/arm64/include/asm/fpsimdmacros.h+148−0

@@ -75,3 +75,151 @@ · 74 unmodified lines
75	75	ldr w\tmpnr, [\state, #16 * 2 + 4]
76	76	fpsimd_restore_fpcr x\tmpnr, \state
77	77	.endm
	78	+
	79	+/* Sanity-check macros to help avoid encoding garbage instructions */
	80	+
	81	+.macro _check_general_reg nr
	82	+ .if (\nr) < 0 \|\| (\nr) > 30
	83	+ .error "Bad register number \nr."
	84	+ .endif
	85	+.endm
	86	+
	87	+.macro _sve_check_zreg znr
	88	+ .if (\znr) < 0 \|\| (\znr) > 31
	89	+ .error "Bad Scalable Vector Extension vector register number \znr."
	90	+ .endif
	91	+.endm
	92	+
	93	+.macro _sve_check_preg pnr
	94	+ .if (\pnr) < 0 \|\| (\pnr) > 15
	95	+ .error "Bad Scalable Vector Extension predicate register number \pnr."
	96	+ .endif
	97	+.endm
	98	+
	99	+.macro _check_num n, min, max
	100	+ .if (\n) < (\min) \|\| (\n) > (\max)
	101	+ .error "Number \n out of range [\min,\max]"
	102	+ .endif
	103	+.endm
	104	+
	105	+/* SVE instruction encodings for non-SVE-capable assemblers */
	106	+
	107	+/* STR (vector): STR Z\nz, [X\nxbase, #\offset, MUL VL] */
	108	+.macro _sve_str_v nz, nxbase, offset=0
	109	+ _sve_check_zreg \nz
	110	+ _check_general_reg \nxbase
	111	+ _check_num (\offset), -0x100, 0xff
	112	+ .inst 0xe5804000 \
	113	+ \| (\nz) \
	114	+ \| ((\nxbase) << 5) \
	115	+ \| (((\offset) & 7) << 10) \
	116	+ \| (((\offset) & 0x1f8) << 13)
	117	+.endm
	118	+
	119	+/* LDR (vector): LDR Z\nz, [X\nxbase, #\offset, MUL VL] */
	120	+.macro _sve_ldr_v nz, nxbase, offset=0
	121	+ _sve_check_zreg \nz
	122	+ _check_general_reg \nxbase
	123	+ _check_num (\offset), -0x100, 0xff
	124	+ .inst 0x85804000 \
	125	+ \| (\nz) \
	126	+ \| ((\nxbase) << 5) \
	127	+ \| (((\offset) & 7) << 10) \
	128	+ \| (((\offset) & 0x1f8) << 13)
	129	+.endm
	130	+
	131	+/* STR (predicate): STR P\np, [X\nxbase, #\offset, MUL VL] */
	132	+.macro _sve_str_p np, nxbase, offset=0
	133	+ _sve_check_preg \np
	134	+ _check_general_reg \nxbase
	135	+ _check_num (\offset), -0x100, 0xff
	136	+ .inst 0xe5800000 \
	137	+ \| (\np) \
	138	+ \| ((\nxbase) << 5) \
	139	+ \| (((\offset) & 7) << 10) \
	140	+ \| (((\offset) & 0x1f8) << 13)
	141	+.endm
	142	+
	143	+/* LDR (predicate): LDR P\np, [X\nxbase, #\offset, MUL VL] */
	144	+.macro _sve_ldr_p np, nxbase, offset=0
	145	+ _sve_check_preg \np
	146	+ _check_general_reg \nxbase
	147	+ _check_num (\offset), -0x100, 0xff
	148	+ .inst 0x85800000 \
	149	+ \| (\np) \
	150	+ \| ((\nxbase) << 5) \
	151	+ \| (((\offset) & 7) << 10) \
	152	+ \| (((\offset) & 0x1f8) << 13)
	153	+.endm
	154	+
	155	+/* RDVL X\nx, #\imm */
	156	+.macro _sve_rdvl nx, imm
	157	+ _check_general_reg \nx
	158	+ _check_num (\imm), -0x20, 0x1f
	159	+ .inst 0x04bf5000 \
	160	+ \| (\nx) \
	161	+ \| (((\imm) & 0x3f) << 5)
	162	+.endm
	163	+
	164	+/* RDFFR (unpredicated): RDFFR P\np.B */
	165	+.macro _sve_rdffr np
	166	+ _sve_check_preg \np
	167	+ .inst 0x2519f000 \
	168	+ \| (\np)
	169	+.endm
	170	+
	171	+/* WRFFR P\np.B */
	172	+.macro _sve_wrffr np
	173	+ _sve_check_preg \np
	174	+ .inst 0x25289000 \
	175	+ \| ((\np) << 5)
	176	+.endm
	177	+
	178	+.macro __for from:req, to:req
	179	+ .if (\from) == (\to)
	180	+ _for__body \from
	181	+ .else
	182	+ __for \from, (\from) + ((\to) - (\from)) / 2
	183	+ __for (\from) + ((\to) - (\from)) / 2 + 1, \to
	184	+ .endif
	185	+.endm
	186	+
	187	+.macro _for var:req, from:req, to:req, insn:vararg
	188	+ .macro _for__body \var:req
	189	+ \insn
	190	+ .endm
	191	+
	192	+ __for \from, \to
	193	+
	194	+ .purgem _for__body
	195	+.endm
	196	+
	197	+.macro sve_save nxbase, xpfpsr, nxtmp
	198	+ _for n, 0, 31, _sve_str_v \n, \nxbase, \n - 34
	199	+ _for n, 0, 15, _sve_str_p \n, \nxbase, \n - 16
	200	+ _sve_rdffr 0
	201	+ _sve_str_p 0, \nxbase
	202	+ _sve_ldr_p 0, \nxbase, -16
	203	+
	204	+ mrs x\nxtmp, fpsr
	205	+ str w\nxtmp, [\xpfpsr]
	206	+ mrs x\nxtmp, fpcr
	207	+ str w\nxtmp, [\xpfpsr, #4]
	208	+.endm
	209	+
	210	+.macro sve_load nxbase, xpfpsr, xvqminus1, nxtmp
	211	+ mrs_s x\nxtmp, SYS_ZCR_EL1
	212	+ bic x\nxtmp, x\nxtmp, ZCR_ELx_LEN_MASK
	213	+ orr x\nxtmp, x\nxtmp, \xvqminus1
	214	+ msr_s SYS_ZCR_EL1, x\nxtmp // self-synchronising
	215	+
	216	+ _for n, 0, 31, _sve_ldr_v \n, \nxbase, \n - 34
	217	+ _sve_ldr_p 0, \nxbase
	218	+ _sve_wrffr 0
	219	+ _for n, 0, 15, _sve_ldr_p \n, \nxbase, \n - 16
	220	+
	221	+ ldr w\nxtmp, [\xpfpsr]
	222	+ msr fpsr, x\nxtmp
	223	+ ldr w\nxtmp, [\xpfpsr, #4]
	224	+ msr fpcr, x\nxtmp
	225	+.endm

arch/arm64/include/asm/irqflags.h+13−27

@@ -20,6 +20,19 @@ · 19 unmodified lines
20	20
21	21	#include <asm/ptrace.h>
22	22
	23	+/*
	24	+ * Aarch64 has flags for masking: Debug, Asynchronous (serror), Interrupts and
	25	+ * FIQ exceptions, in the 'daif' register. We mask and unmask them in 'dai'
	26	+ * order:
	27	+ * Masking debug exceptions causes all other exceptions to be masked too/
	28	+ * Masking SError masks irq, but not debug exceptions. Masking irqs has no
	29	+ * side effects for other flags. Keeping to this order makes it easier for
	30	+ * entry.S to know which exceptions should be unmasked.
	31	+ *
	32	+ * FIQ is never expected, but we mask it when we disable debug exceptions, and
	33	+ * unmask it at all other times.
	34	+ */
	35	+
23	36	/*
24	37	* CPU interrupt mask handling.
25	38	*/
@@ -53,12 +66,6 @@ static inline void arch_local_irq_disable(void) · 27 unmodified lines
53	66	: "memory");
54	67	}
55	68
56		−#define local_fiq_enable() asm("msr daifclr, #1" : : : "memory")
57		−#define local_fiq_disable() asm("msr daifset, #1" : : : "memory")
58		−
59		−#define local_async_enable() asm("msr daifclr, #4" : : : "memory")
60		−#define local_async_disable() asm("msr daifset, #4" : : : "memory")
61		−
62	69	/*
63	70	* Save the current interrupt enable state.
64	71	*/
@@ -89,26 +96,5 @@ static inline int arch_irqs_disabled_flags(unsigned long flags) · 24 unmodified lines
89	96	{
90	97	return flags & PSR_I_BIT;
91	98	}
92		−
93		−/*
94		− * save and restore debug state
95		− */
96		−#define local_dbg_save(flags) \
97		− do { \
98		− typecheck(unsigned long, flags); \
99		− asm volatile( \
100		− "mrs %0, daif // local_dbg_save\n" \
101		− "msr daifset, #8" \
102		− : "=r" (flags) : : "memory"); \
103		− } while (0)
104		−
105		−#define local_dbg_restore(flags) \
106		− do { \
107		− typecheck(unsigned long, flags); \
108		− asm volatile( \
109		− "msr daif, %0 // local_dbg_restore\n" \
110		− : : "r" (flags) : "memory"); \
111		− } while (0)
112		−
113	99	#endif
114	100	#endif

arch/arm64/include/asm/kvm_arm.h+4−1

@@ -185,7 +185,9 @@ · 184 unmodified lines
185	185	#define CPTR_EL2_TCPAC (1 << 31)
186	186	#define CPTR_EL2_TTA (1 << 20)
187	187	#define CPTR_EL2_TFP (1 << CPTR_EL2_TFP_SHIFT)
188		−#define CPTR_EL2_DEFAULT 0x000033ff
	188	+#define CPTR_EL2_TZ (1 << 8)
	189	+#define CPTR_EL2_RES1 0x000032ff /* known RES1 bits in CPTR_EL2 */
	190	+#define CPTR_EL2_DEFAULT CPTR_EL2_RES1
189	191
190	192	/* Hyp Debug Configuration Register bits */
191	193	#define MDCR_EL2_TPMS (1 << 14)
@@ -236,5 +238,6 @@ · 44 unmodified lines
236	238
237	239	#define CPACR_EL1_FPEN (3 << 20)
238	240	#define CPACR_EL1_TTA (1 << 28)
	241	+#define CPACR_EL1_DEFAULT (CPACR_EL1_FPEN \| CPACR_EL1_ZEN_EL1EN)
239	242
240	243	#endif /* __ARM64_KVM_ARM_H__ */

arch/arm64/include/asm/kvm_host.h+11−0

@@ -25,6 +25,7 @@ · 24 unmodified lines
25	25	#include <linux/types.h>
26	26	#include <linux/kvm_types.h>
27	27	#include <asm/cpufeature.h>
	28	+#include <asm/fpsimd.h>
28	29	#include <asm/kvm.h>
29	30	#include <asm/kvm_asm.h>
30	31	#include <asm/kvm_mmio.h>
@@ -384,4 +385,14 @@ static inline void __cpu_init_stage2(void) · 353 unmodified lines
384	385	"PARange is %d bits, unsupported configuration!", parange);
385	386	}
386	387
	388	+/*
	389	+ * All host FP/SIMD state is restored on guest exit, so nothing needs
	390	+ * doing here except in the SVE case:
	391	+*/
	392	+static inline void kvm_fpsimd_flush_cpu_state(void)
	393	+{
	394	+ if (system_supports_sve())
	395	+ sve_flush_cpu_state();
	396	+}
	397	+
387	398	#endif /* __ARM64_KVM_HOST_H__ */

arch/arm64/include/asm/memory.h+0−15

@@ -61,8 +61,6 @@ · 60 unmodified lines
61	61	* KIMAGE_VADDR - the virtual address of the start of the kernel image
62	62	* VA_BITS - the maximum number of bits for virtual addresses.
63	63	* VA_START - the first kernel virtual address.
64		− * TASK_SIZE - the maximum size of a user space task.
65		− * TASK_UNMAPPED_BASE - the lower boundary of the mmap VM area.
66	64	*/
67	65	#define VA_BITS (CONFIG_ARM64_VA_BITS)
68	66	#define VA_START (UL(0xffffffffffffffff) - \
@@ -77,19 +75,6 @@ · 8 unmodified lines
77	75	#define PCI_IO_END (VMEMMAP_START - SZ_2M)
78	76	#define PCI_IO_START (PCI_IO_END - PCI_IO_SIZE)
79	77	#define FIXADDR_TOP (PCI_IO_START - SZ_2M)
80		−#define TASK_SIZE_64 (UL(1) << VA_BITS)
81		−
82		−#ifdef CONFIG_COMPAT
83		−#define TASK_SIZE_32 UL(0x100000000)
84		−#define TASK_SIZE (test_thread_flag(TIF_32BIT) ? \
85		− TASK_SIZE_32 : TASK_SIZE_64)
86		−#define TASK_SIZE_OF(tsk) (test_tsk_thread_flag(tsk, TIF_32BIT) ? \
87		− TASK_SIZE_32 : TASK_SIZE_64)
88		−#else
89		−#define TASK_SIZE TASK_SIZE_64
90		−#endif /* CONFIG_COMPAT */
91		−
92		−#define TASK_UNMAPPED_BASE (PAGE_ALIGN(TASK_SIZE / 4))
93	78
94	79	#define KERNEL_START _text
95	80	#define KERNEL_END _end

arch/arm64/include/asm/pgtable.h+14−0

@@ -98,6 +98,8 @@ extern unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)]; · 97 unmodified lines
98	98	((pte_val(pte) & (PTE_VALID \| PTE_USER \| PTE_UXN)) == (PTE_VALID \| PTE_UXN))
99	99	#define pte_valid_young(pte) \
100	100	((pte_val(pte) & (PTE_VALID \| PTE_AF)) == (PTE_VALID \| PTE_AF))
	101	+#define pte_valid_user(pte) \
	102	+ ((pte_val(pte) & (PTE_VALID \| PTE_USER)) == (PTE_VALID \| PTE_USER))
101	103
102	104	/*
103	105	* Could the pte be present in the TLB? We must check mm_tlb_flush_pending
@@ -107,6 +109,18 @@ extern unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)]; · 3 unmodified lines
107	109	#define pte_accessible(mm, pte) \
108	110	(mm_tlb_flush_pending(mm) ? pte_present(pte) : pte_valid_young(pte))
109	111
	112	+/*
	113	+ * p??_access_permitted() is true for valid user mappings (subject to the
	114	+ * write permission check) other than user execute-only which do not have the
	115	+ * PTE_USER bit set. PROT_NONE mappings do not have the PTE_VALID bit set.
	116	+ */
	117	+#define pte_access_permitted(pte, write) \
	118	+ (pte_valid_user(pte) && (!(write) \|\| pte_write(pte)))
	119	+#define pmd_access_permitted(pmd, write) \
	120	+ (pte_access_permitted(pmd_pte(pmd), (write)))
	121	+#define pud_access_permitted(pud, write) \
	122	+ (pte_access_permitted(pud_pte(pud), (write)))
	123	+
110	124	static inline pte_t clear_pte_bit(pte_t pte, pgprot_t prot)
111	125	{
112	126	pte_val(pte) &= ~pgprot_val(prot);

arch/arm64/include/asm/processor.h+28−0

@@ -19,6 +19,10 @@ · 18 unmodified lines
19	19	#ifndef __ASM_PROCESSOR_H
20	20	#define __ASM_PROCESSOR_H
21	21
	22	+#define TASK_SIZE_64 (UL(1) << VA_BITS)
	23	+
	24	+#ifndef __ASSEMBLY__
	25	+
22	26	/*
23	27	* Default implementation of macro that returns current
24	28	* instruction pointer ("program counter").
@@ -37,6 +41,22 @@ · 12 unmodified lines
37	41	#include <asm/ptrace.h>
38	42	#include <asm/types.h>
39	43
	44	+/*
	45	+ * TASK_SIZE - the maximum size of a user space task.
	46	+ * TASK_UNMAPPED_BASE - the lower boundary of the mmap VM area.
	47	+ */
	48	+#ifdef CONFIG_COMPAT
	49	+#define TASK_SIZE_32 UL(0x100000000)
	50	+#define TASK_SIZE (test_thread_flag(TIF_32BIT) ? \
	51	+ TASK_SIZE_32 : TASK_SIZE_64)
	52	+#define TASK_SIZE_OF(tsk) (test_tsk_thread_flag(tsk, TIF_32BIT) ? \
	53	+ TASK_SIZE_32 : TASK_SIZE_64)
	54	+#else
	55	+#define TASK_SIZE TASK_SIZE_64
	56	+#endif /* CONFIG_COMPAT */
	57	+
	58	+#define TASK_UNMAPPED_BASE (PAGE_ALIGN(TASK_SIZE / 4))
	59	+
40	60	#define STACK_TOP_MAX TASK_SIZE_64
41	61	#ifdef CONFIG_COMPAT
42	62	#define AARCH32_VECTORS_BASE 0xffff0000
@@ -85,6 +105,9 @@ struct thread_struct { · 42 unmodified lines
85	105	unsigned long tp2_value;
86	106	#endif
87	107	struct fpsimd_state fpsimd_state;
	108	+ void sve_state; / SVE registers, if any */
	109	+ unsigned int sve_vl; /* SVE vector length */
	110	+ unsigned int sve_vl_onexec; /* SVE vl after next exec */
88	111	unsigned long fault_address; /* fault info */
89	112	unsigned long fault_code; /* ESR_EL1 value */
90	113	struct debug_info debug; /* debugging */
@@ -194,4 +217,9 @@ static inline void spin_lock_prefetch(const void *ptr) · 103 unmodified lines
194	217	int cpu_enable_pan(void *__unused);
195	218	int cpu_enable_cache_maint_trap(void *__unused);
196	219
	220	+/* Userspace interface for PR_SVE_{SET,GET}_VL prctl()s: */
	221	+#define SVE_SET_VL(arg) sve_set_current_vl(arg)
	222	+#define SVE_GET_VL() sve_get_current_vl()
	223	+
	224	+#endif /* __ASSEMBLY__ */
197	225	#endif /* __ASM_PROCESSOR_H */

arch/arm64/include/asm/sysreg.h+121−0

@@ -145,10 +145,14 @@ · 144 unmodified lines
145	145
146	146	#define SYS_ID_AA64PFR0_EL1 sys_reg(3, 0, 0, 4, 0)
147	147	#define SYS_ID_AA64PFR1_EL1 sys_reg(3, 0, 0, 4, 1)
	148	+#define SYS_ID_AA64ZFR0_EL1 sys_reg(3, 0, 0, 4, 4)
148	149
149	150	#define SYS_ID_AA64DFR0_EL1 sys_reg(3, 0, 0, 5, 0)
150	151	#define SYS_ID_AA64DFR1_EL1 sys_reg(3, 0, 0, 5, 1)
151	152
	153	+#define SYS_ID_AA64AFR0_EL1 sys_reg(3, 0, 0, 5, 4)
	154	+#define SYS_ID_AA64AFR1_EL1 sys_reg(3, 0, 0, 5, 5)
	155	+
152	156	#define SYS_ID_AA64ISAR0_EL1 sys_reg(3, 0, 0, 6, 0)
153	157	#define SYS_ID_AA64ISAR1_EL1 sys_reg(3, 0, 0, 6, 1)
154	158
@@ -160,6 +164,8 @@ · 5 unmodified lines
160	164	#define SYS_ACTLR_EL1 sys_reg(3, 0, 1, 0, 1)
161	165	#define SYS_CPACR_EL1 sys_reg(3, 0, 1, 0, 2)
162	166
	167	+#define SYS_ZCR_EL1 sys_reg(3, 0, 1, 2, 0)
	168	+
163	169	#define SYS_TTBR0_EL1 sys_reg(3, 0, 2, 0, 0)
164	170	#define SYS_TTBR1_EL1 sys_reg(3, 0, 2, 0, 1)
165	171	#define SYS_TCR_EL1 sys_reg(3, 0, 2, 0, 2)
@@ -172,6 +178,99 @@ · 6 unmodified lines
172	178	#define SYS_FAR_EL1 sys_reg(3, 0, 6, 0, 0)
173	179	#define SYS_PAR_EL1 sys_reg(3, 0, 7, 4, 0)
174	180
	181	+/* Statistical Profiling Extension */
	182	+/* ID registers */
	183	+#define SYS_PMSIDR_EL1 sys_reg(3, 0, 9, 9, 7)
	184	+#define SYS_PMSIDR_EL1_FE_SHIFT 0
	185	+#define SYS_PMSIDR_EL1_FT_SHIFT 1
	186	+#define SYS_PMSIDR_EL1_FL_SHIFT 2
	187	+#define SYS_PMSIDR_EL1_ARCHINST_SHIFT 3
	188	+#define SYS_PMSIDR_EL1_LDS_SHIFT 4
	189	+#define SYS_PMSIDR_EL1_ERND_SHIFT 5
	190	+#define SYS_PMSIDR_EL1_INTERVAL_SHIFT 8
	191	+#define SYS_PMSIDR_EL1_INTERVAL_MASK 0xfUL
	192	+#define SYS_PMSIDR_EL1_MAXSIZE_SHIFT 12
	193	+#define SYS_PMSIDR_EL1_MAXSIZE_MASK 0xfUL
	194	+#define SYS_PMSIDR_EL1_COUNTSIZE_SHIFT 16
	195	+#define SYS_PMSIDR_EL1_COUNTSIZE_MASK 0xfUL
	196	+
	197	+#define SYS_PMBIDR_EL1 sys_reg(3, 0, 9, 10, 7)
	198	+#define SYS_PMBIDR_EL1_ALIGN_SHIFT 0
	199	+#define SYS_PMBIDR_EL1_ALIGN_MASK 0xfU
	200	+#define SYS_PMBIDR_EL1_P_SHIFT 4
	201	+#define SYS_PMBIDR_EL1_F_SHIFT 5
	202	+
	203	+/* Sampling controls */
	204	+#define SYS_PMSCR_EL1 sys_reg(3, 0, 9, 9, 0)
	205	+#define SYS_PMSCR_EL1_E0SPE_SHIFT 0
	206	+#define SYS_PMSCR_EL1_E1SPE_SHIFT 1
	207	+#define SYS_PMSCR_EL1_CX_SHIFT 3
	208	+#define SYS_PMSCR_EL1_PA_SHIFT 4
	209	+#define SYS_PMSCR_EL1_TS_SHIFT 5
	210	+#define SYS_PMSCR_EL1_PCT_SHIFT 6
	211	+
	212	+#define SYS_PMSCR_EL2 sys_reg(3, 4, 9, 9, 0)
	213	+#define SYS_PMSCR_EL2_E0HSPE_SHIFT 0
	214	+#define SYS_PMSCR_EL2_E2SPE_SHIFT 1
	215	+#define SYS_PMSCR_EL2_CX_SHIFT 3
	216	+#define SYS_PMSCR_EL2_PA_SHIFT 4
	217	+#define SYS_PMSCR_EL2_TS_SHIFT 5
	218	+#define SYS_PMSCR_EL2_PCT_SHIFT 6
	219	+
	220	+#define SYS_PMSICR_EL1 sys_reg(3, 0, 9, 9, 2)
	221	+
	222	+#define SYS_PMSIRR_EL1 sys_reg(3, 0, 9, 9, 3)
	223	+#define SYS_PMSIRR_EL1_RND_SHIFT 0
	224	+#define SYS_PMSIRR_EL1_INTERVAL_SHIFT 8
	225	+#define SYS_PMSIRR_EL1_INTERVAL_MASK 0xffffffUL
	226	+
	227	+/* Filtering controls */
	228	+#define SYS_PMSFCR_EL1 sys_reg(3, 0, 9, 9, 4)
	229	+#define SYS_PMSFCR_EL1_FE_SHIFT 0
	230	+#define SYS_PMSFCR_EL1_FT_SHIFT 1
	231	+#define SYS_PMSFCR_EL1_FL_SHIFT 2
	232	+#define SYS_PMSFCR_EL1_B_SHIFT 16
	233	+#define SYS_PMSFCR_EL1_LD_SHIFT 17
	234	+#define SYS_PMSFCR_EL1_ST_SHIFT 18
	235	+
	236	+#define SYS_PMSEVFR_EL1 sys_reg(3, 0, 9, 9, 5)
	237	+#define SYS_PMSEVFR_EL1_RES0 0x0000ffff00ff0f55UL
	238	+
	239	+#define SYS_PMSLATFR_EL1 sys_reg(3, 0, 9, 9, 6)
	240	+#define SYS_PMSLATFR_EL1_MINLAT_SHIFT 0
	241	+
	242	+/* Buffer controls */
	243	+#define SYS_PMBLIMITR_EL1 sys_reg(3, 0, 9, 10, 0)
	244	+#define SYS_PMBLIMITR_EL1_E_SHIFT 0
	245	+#define SYS_PMBLIMITR_EL1_FM_SHIFT 1
	246	+#define SYS_PMBLIMITR_EL1_FM_MASK 0x3UL
	247	+#define SYS_PMBLIMITR_EL1_FM_STOP_IRQ (0 << SYS_PMBLIMITR_EL1_FM_SHIFT)
	248	+
	249	+#define SYS_PMBPTR_EL1 sys_reg(3, 0, 9, 10, 1)
	250	+
	251	+/* Buffer error reporting */
	252	+#define SYS_PMBSR_EL1 sys_reg(3, 0, 9, 10, 3)
	253	+#define SYS_PMBSR_EL1_COLL_SHIFT 16
	254	+#define SYS_PMBSR_EL1_S_SHIFT 17
	255	+#define SYS_PMBSR_EL1_EA_SHIFT 18
	256	+#define SYS_PMBSR_EL1_DL_SHIFT 19
	257	+#define SYS_PMBSR_EL1_EC_SHIFT 26
	258	+#define SYS_PMBSR_EL1_EC_MASK 0x3fUL
	259	+
	260	+#define SYS_PMBSR_EL1_EC_BUF (0x0UL << SYS_PMBSR_EL1_EC_SHIFT)
	261	+#define SYS_PMBSR_EL1_EC_FAULT_S1 (0x24UL << SYS_PMBSR_EL1_EC_SHIFT)
	262	+#define SYS_PMBSR_EL1_EC_FAULT_S2 (0x25UL << SYS_PMBSR_EL1_EC_SHIFT)
	263	+
	264	+#define SYS_PMBSR_EL1_FAULT_FSC_SHIFT 0
	265	+#define SYS_PMBSR_EL1_FAULT_FSC_MASK 0x3fUL
	266	+
	267	+#define SYS_PMBSR_EL1_BUF_BSC_SHIFT 0
	268	+#define SYS_PMBSR_EL1_BUF_BSC_MASK 0x3fUL
	269	+
	270	+#define SYS_PMBSR_EL1_BUF_BSC_FULL (0x1UL << SYS_PMBSR_EL1_BUF_BSC_SHIFT)
	271	+
	272	+/* End of Statistical Profiling Extension */
	273	+
175	274	#define SYS_PMINTENSET_EL1 sys_reg(3, 0, 9, 14, 1)
176	275	#define SYS_PMINTENCLR_EL1 sys_reg(3, 0, 9, 14, 2)
177	276
@@ -250,6 +349,8 @@ · 72 unmodified lines
250	349
251	350	#define SYS_PMCCFILTR_EL0 sys_reg (3, 3, 14, 15, 7)
252	351
	352	+#define SYS_ZCR_EL2 sys_reg(3, 4, 1, 2, 0)
	353	+
253	354	#define SYS_DACR32_EL2 sys_reg(3, 4, 3, 0, 0)
254	355	#define SYS_IFSR32_EL2 sys_reg(3, 4, 5, 0, 1)
255	356	#define SYS_FPEXC32_EL2 sys_reg(3, 4, 5, 3, 0)
@@ -318,6 +419,10 @@ · 62 unmodified lines
318	419	#define SCTLR_EL1_CP15BEN (1 << 5)
319	420
320	421	/* id_aa64isar0 */
	422	+#define ID_AA64ISAR0_DP_SHIFT 44
	423	+#define ID_AA64ISAR0_SM4_SHIFT 40
	424	+#define ID_AA64ISAR0_SM3_SHIFT 36
	425	+#define ID_AA64ISAR0_SHA3_SHIFT 32
321	426	#define ID_AA64ISAR0_RDM_SHIFT 28
322	427	#define ID_AA64ISAR0_ATOMICS_SHIFT 20
323	428	#define ID_AA64ISAR0_CRC32_SHIFT 16
@@ -332,6 +437,7 @@ · 8 unmodified lines
332	437	#define ID_AA64ISAR1_DPB_SHIFT 0
333	438
334	439	/* id_aa64pfr0 */
	440	+#define ID_AA64PFR0_SVE_SHIFT 32
335	441	#define ID_AA64PFR0_GIC_SHIFT 24
336	442	#define ID_AA64PFR0_ASIMD_SHIFT 20
337	443	#define ID_AA64PFR0_FP_SHIFT 16
@@ -340,6 +446,7 @@ · 2 unmodified lines
340	446	#define ID_AA64PFR0_EL1_SHIFT 4
341	447	#define ID_AA64PFR0_EL0_SHIFT 0
342	448
	449	+#define ID_AA64PFR0_SVE 0x1
343	450	#define ID_AA64PFR0_FP_NI 0xf
344	451	#define ID_AA64PFR0_FP_SUPPORTED 0x0
345	452	#define ID_AA64PFR0_ASIMD_NI 0xf
@@ -441,6 +548,20 @@ · 95 unmodified lines
441	548	#endif
442	549
443	550
	551	+/*
	552	+ * The ZCR_ELx_LEN_* definitions intentionally include bits [8:4] which
	553	+ * are reserved by the SVE architecture for future expansion of the LEN
	554	+ * field, with compatible semantics.
	555	+ */
	556	+#define ZCR_ELx_LEN_SHIFT 0
	557	+#define ZCR_ELx_LEN_SIZE 9
	558	+#define ZCR_ELx_LEN_MASK 0x1ff
	559	+
	560	+#define CPACR_EL1_ZEN_EL1EN (1 << 16) /* enable EL1 access */
	561	+#define CPACR_EL1_ZEN_EL0EN (1 << 17) /* enable EL0 access, if EL1EN set */
	562	+#define CPACR_EL1_ZEN (CPACR_EL1_ZEN_EL1EN \| CPACR_EL1_ZEN_EL0EN)
	563	+
	564	+
444	565	/* Safe value for MPIDR_EL1: Bit31:RES1, Bit30:U:0, Bit24:MT:0 */
445	566	#define SYS_MPIDR_SAFE_VAL (1UL << 31)
446	567

arch/arm64/include/asm/thread_info.h+5−0

@@ -63,6 +63,8 @@ struct thread_info { · 62 unmodified lines
63	63	void arch_setup_new_exec(void);
64	64	#define arch_setup_new_exec arch_setup_new_exec
65	65
	66	+void arch_release_task_struct(struct task_struct *tsk);
	67	+
66	68	#endif
67	69
68	70	/*
@@ -92,6 +94,8 @@ void arch_setup_new_exec(void); · 23 unmodified lines
92	94	#define TIF_RESTORE_SIGMASK 20
93	95	#define TIF_SINGLESTEP 21
94	96	#define TIF_32BIT 22 /* 32bit process */
	97	+#define TIF_SVE 23 /* Scalable Vector Extension in use */
	98	+#define TIF_SVE_VL_INHERIT 24 /* Inherit sve_vl_onexec across exec */
95	99
96	100	#define _TIF_SIGPENDING (1 << TIF_SIGPENDING)
97	101	#define _TIF_NEED_RESCHED (1 << TIF_NEED_RESCHED)
@@ -105,6 +109,7 @@ void arch_setup_new_exec(void); · 7 unmodified lines
105	109	#define _TIF_UPROBE (1 << TIF_UPROBE)
106	110	#define _TIF_FSCHECK (1 << TIF_FSCHECK)
107	111	#define _TIF_32BIT (1 << TIF_32BIT)
	112	+#define _TIF_SVE (1 << TIF_SVE)
108	113
109	114	#define _TIF_WORK_MASK (_TIF_NEED_RESCHED \| _TIF_SIGPENDING \| \
110	115	_TIF_NOTIFY_RESUME \| _TIF_FOREIGN_FPSTATE \| \

arch/arm64/include/asm/traps.h+8−0

@@ -34,9 +34,17 @@ struct undef_hook { · 33 unmodified lines
34	34
35	35	void register_undef_hook(struct undef_hook *hook);
36	36	void unregister_undef_hook(struct undef_hook *hook);
	37	+void force_signal_inject(int signal, int code, struct pt_regs *regs,
	38	+ unsigned long address);
37	39
38	40	void arm64_notify_segfault(struct pt_regs *regs, unsigned long addr);
39	41
	42	+/*
	43	+ * Move regs->pc to next instruction and do necessary setup before it
	44	+ * is executed.
	45	+ */
	46	+void arm64_skip_faulting_instruction(struct pt_regs *regs, unsigned long size);
	47	+
40	48	static inline int __in_irqentry_text(unsigned long ptr)
41	49	{
42	50	return ptr >= (unsigned long)&__irqentry_text_start &&

arch/arm64/include/uapi/asm/hwcap.h+6−0

@@ -37,5 +37,11 @@ · 36 unmodified lines
37	37	#define HWCAP_FCMA (1 << 14)
38	38	#define HWCAP_LRCPC (1 << 15)
39	39	#define HWCAP_DCPOP (1 << 16)
	40	+#define HWCAP_SHA3 (1 << 17)
	41	+#define HWCAP_SM3 (1 << 18)
	42	+#define HWCAP_SM4 (1 << 19)
	43	+#define HWCAP_ASIMDDP (1 << 20)
	44	+#define HWCAP_SHA512 (1 << 21)
	45	+#define HWCAP_SVE (1 << 22)
40	46
41	47	#endif /* _UAPI__ASM_HWCAP_H */

arch/arm64/include/uapi/asm/ptrace.h+138−1

@@ -23,6 +23,7 @@ · 22 unmodified lines
23	23	#include <linux/types.h>
24	24
25	25	#include <asm/hwcap.h>
	26	+#include <asm/sigcontext.h>
26	27
27	28
28	29	/*
@@ -47,7 +48,6 @@ · 18 unmodified lines
47	48	#define PSR_D_BIT 0x00000200
48	49	#define PSR_PAN_BIT 0x00400000
49	50	#define PSR_UAO_BIT 0x00800000
50		−#define PSR_Q_BIT 0x08000000
51	51	#define PSR_V_BIT 0x10000000
52	52	#define PSR_C_BIT 0x20000000
53	53	#define PSR_Z_BIT 0x40000000
@@ -64,6 +64,8 @@ · 10 unmodified lines
64	64
65	65	#ifndef __ASSEMBLY__
66	66
	67	+#include <linux/prctl.h>
	68	+
67	69	/*
68	70	* User structures for general purpose, floating point and debug registers.
69	71	*/
@@ -91,6 +93,141 @@ struct user_hwdebug_state { · 21 unmodified lines
91	93	} dbg_regs[16];
92	94	};
93	95
	96	+/* SVE/FP/SIMD state (NT_ARM_SVE) */
	97	+
	98	+struct user_sve_header {
	99	+ __u32 size; /* total meaningful regset content in bytes */
	100	+ __u32 max_size; /* maxmium possible size for this thread */
	101	+ __u16 vl; /* current vector length */
	102	+ __u16 max_vl; /* maximum possible vector length */
	103	+ __u16 flags;
	104	+ __u16 __reserved;
	105	+};
	106	+
	107	+/* Definitions for user_sve_header.flags: */
	108	+#define SVE_PT_REGS_MASK (1 << 0)
	109	+
	110	+#define SVE_PT_REGS_FPSIMD 0
	111	+#define SVE_PT_REGS_SVE SVE_PT_REGS_MASK
	112	+
	113	+/*
	114	+ * Common SVE_PT_* flags:
	115	+ * These must be kept in sync with prctl interface in <linux/ptrace.h>
	116	+ */
	117	+#define SVE_PT_VL_INHERIT (PR_SVE_VL_INHERIT >> 16)
	118	+#define SVE_PT_VL_ONEXEC (PR_SVE_SET_VL_ONEXEC >> 16)
	119	+
	120	+
	121	+/*
	122	+ * The remainder of the SVE state follows struct user_sve_header. The
	123	+ * total size of the SVE state (including header) depends on the
	124	+ * metadata in the header: SVE_PT_SIZE(vq, flags) gives the total size
	125	+ * of the state in bytes, including the header.
	126	+ *
	127	+ * Refer to <asm/sigcontext.h> for details of how to pass the correct
	128	+ * "vq" argument to these macros.
	129	+ */
	130	+
	131	+/* Offset from the start of struct user_sve_header to the register data */
	132	+#define SVE_PT_REGS_OFFSET \
	133	+ ((sizeof(struct sve_context) + (SVE_VQ_BYTES - 1)) \
	134	+ / SVE_VQ_BYTES * SVE_VQ_BYTES)
	135	+
	136	+/*
	137	+ * The register data content and layout depends on the value of the
	138	+ * flags field.
	139	+ */
	140	+
	141	+/*
	142	+ * (flags & SVE_PT_REGS_MASK) == SVE_PT_REGS_FPSIMD case:
	143	+ *
	144	+ * The payload starts at offset SVE_PT_FPSIMD_OFFSET, and is of type
	145	+ * struct user_fpsimd_state. Additional data might be appended in the
	146	+ * future: use SVE_PT_FPSIMD_SIZE(vq, flags) to compute the total size.
	147	+ * SVE_PT_FPSIMD_SIZE(vq, flags) will never be less than
	148	+ * sizeof(struct user_fpsimd_state).
	149	+ */
	150	+
	151	+#define SVE_PT_FPSIMD_OFFSET SVE_PT_REGS_OFFSET
	152	+
	153	+#define SVE_PT_FPSIMD_SIZE(vq, flags) (sizeof(struct user_fpsimd_state))
	154	+
	155	+/*
	156	+ * (flags & SVE_PT_REGS_MASK) == SVE_PT_REGS_SVE case:
	157	+ *
	158	+ * The payload starts at offset SVE_PT_SVE_OFFSET, and is of size
	159	+ * SVE_PT_SVE_SIZE(vq, flags).
	160	+ *
	161	+ * Additional macros describe the contents and layout of the payload.
	162	+ * For each, SVE_PT_SVE_x_OFFSET(args) is the start offset relative to
	163	+ * the start of struct user_sve_header, and SVE_PT_SVE_x_SIZE(args) is
	164	+ * the size in bytes:
	165	+ *
	166	+ * x type description
	167	+ * - ---- -----------
	168	+ * ZREGS \
	169	+ * ZREG \|
	170	+ * PREGS \| refer to <asm/sigcontext.h>
	171	+ * PREG \|
	172	+ * FFR /
	173	+ *
	174	+ * FPSR uint32_t FPSR
	175	+ * FPCR uint32_t FPCR
	176	+ *
	177	+ * Additional data might be appended in the future.
	178	+ */
	179	+
	180	+#define SVE_PT_SVE_ZREG_SIZE(vq) SVE_SIG_ZREG_SIZE(vq)
	181	+#define SVE_PT_SVE_PREG_SIZE(vq) SVE_SIG_PREG_SIZE(vq)
	182	+#define SVE_PT_SVE_FFR_SIZE(vq) SVE_SIG_FFR_SIZE(vq)
	183	+#define SVE_PT_SVE_FPSR_SIZE sizeof(__u32)
	184	+#define SVE_PT_SVE_FPCR_SIZE sizeof(__u32)
	185	+
	186	+#define __SVE_SIG_TO_PT(offset) \
	187	+ ((offset) - SVE_SIG_REGS_OFFSET + SVE_PT_REGS_OFFSET)
	188	+
	189	+#define SVE_PT_SVE_OFFSET SVE_PT_REGS_OFFSET
	190	+
	191	+#define SVE_PT_SVE_ZREGS_OFFSET \
	192	+ __SVE_SIG_TO_PT(SVE_SIG_ZREGS_OFFSET)
	193	+#define SVE_PT_SVE_ZREG_OFFSET(vq, n) \
	194	+ __SVE_SIG_TO_PT(SVE_SIG_ZREG_OFFSET(vq, n))
	195	+#define SVE_PT_SVE_ZREGS_SIZE(vq) \
	196	+ (SVE_PT_SVE_ZREG_OFFSET(vq, SVE_NUM_ZREGS) - SVE_PT_SVE_ZREGS_OFFSET)
	197	+
	198	+#define SVE_PT_SVE_PREGS_OFFSET(vq) \
	199	+ __SVE_SIG_TO_PT(SVE_SIG_PREGS_OFFSET(vq))
	200	+#define SVE_PT_SVE_PREG_OFFSET(vq, n) \
	201	+ __SVE_SIG_TO_PT(SVE_SIG_PREG_OFFSET(vq, n))
	202	+#define SVE_PT_SVE_PREGS_SIZE(vq) \
	203	+ (SVE_PT_SVE_PREG_OFFSET(vq, SVE_NUM_PREGS) - \
	204	+ SVE_PT_SVE_PREGS_OFFSET(vq))
	205	+
	206	+#define SVE_PT_SVE_FFR_OFFSET(vq) \
	207	+ __SVE_SIG_TO_PT(SVE_SIG_FFR_OFFSET(vq))
	208	+
	209	+#define SVE_PT_SVE_FPSR_OFFSET(vq) \
	210	+ ((SVE_PT_SVE_FFR_OFFSET(vq) + SVE_PT_SVE_FFR_SIZE(vq) + \
	211	+ (SVE_VQ_BYTES - 1)) \
	212	+ / SVE_VQ_BYTES * SVE_VQ_BYTES)
	213	+#define SVE_PT_SVE_FPCR_OFFSET(vq) \
	214	+ (SVE_PT_SVE_FPSR_OFFSET(vq) + SVE_PT_SVE_FPSR_SIZE)
	215	+
	216	+/*
	217	+ * Any future extension appended after FPCR must be aligned to the next
	218	+ * 128-bit boundary.
	219	+ */
	220	+
	221	+#define SVE_PT_SVE_SIZE(vq, flags) \
	222	+ ((SVE_PT_SVE_FPCR_OFFSET(vq) + SVE_PT_SVE_FPCR_SIZE \
	223	+ - SVE_PT_SVE_OFFSET + (SVE_VQ_BYTES - 1)) \
	224	+ / SVE_VQ_BYTES * SVE_VQ_BYTES)
	225	+
	226	+#define SVE_PT_SIZE(vq, flags) \
	227	+ (((flags) & SVE_PT_REGS_MASK) == SVE_PT_REGS_SVE ? \
	228	+ SVE_PT_SVE_OFFSET + SVE_PT_SVE_SIZE(vq, flags) \
	229	+ : SVE_PT_FPSIMD_OFFSET + SVE_PT_FPSIMD_SIZE(vq, flags))
	230	+
94	231	#endif /* __ASSEMBLY__ */
95	232
96	233	#endif /* _UAPI__ASM_PTRACE_H */

arch/arm64/include/uapi/asm/sigcontext.h+119−1

@@ -17,6 +17,8 @@ · 16 unmodified lines
17	17	#ifndef _UAPI__ASM_SIGCONTEXT_H
18	18	#define _UAPI__ASM_SIGCONTEXT_H
19	19
	20	+#ifndef __ASSEMBLY__
	21	+
20	22	#include <linux/types.h>
21	23
22	24	/*
@@ -42,10 +44,11 @@ struct sigcontext { · 19 unmodified lines
42	44	*
43	45	* 0x210 fpsimd_context
44	46	* 0x10 esr_context
	47	+ * 0x8a0 sve_context (vl <= 64) (optional)
45	48	* 0x20 extra_context (optional)
46	49	* 0x10 terminator (null _aarch64_ctx)
47	50	*
48		− * 0xdb0 (reserved for future allocation)
	51	+ * 0x510 (reserved for future allocation)
49	52	*
50	53	* New records that can exceed this space need to be opt-in for userspace, so
51	54	* that an expanded signal frame is not generated unexpectedly. The mechanism
@@ -117,4 +120,119 @@ struct extra_context { · 65 unmodified lines
117	120	__u32 __reserved[3];
118	121	};
119	122
	123	+#define SVE_MAGIC 0x53564501
	124	+
	125	+struct sve_context {
	126	+ struct _aarch64_ctx head;
	127	+ __u16 vl;
	128	+ __u16 __reserved[3];
	129	+};
	130	+
	131	+#endif /* !__ASSEMBLY__ */
	132	+
	133	+/*
	134	+ * The SVE architecture leaves space for future expansion of the
	135	+ * vector length beyond its initial architectural limit of 2048 bits
	136	+ * (16 quadwords).
	137	+ *
	138	+ * See linux/Documentation/arm64/sve.txt for a description of the VL/VQ
	139	+ * terminology.
	140	+ */
	141	+#define SVE_VQ_BYTES 16 /* number of bytes per quadword */
	142	+
	143	+#define SVE_VQ_MIN 1
	144	+#define SVE_VQ_MAX 512
	145	+
	146	+#define SVE_VL_MIN (SVE_VQ_MIN * SVE_VQ_BYTES)
	147	+#define SVE_VL_MAX (SVE_VQ_MAX * SVE_VQ_BYTES)
	148	+
	149	+#define SVE_NUM_ZREGS 32
	150	+#define SVE_NUM_PREGS 16
	151	+
	152	+#define sve_vl_valid(vl) \
	153	+ ((vl) % SVE_VQ_BYTES == 0 && (vl) >= SVE_VL_MIN && (vl) <= SVE_VL_MAX)
	154	+#define sve_vq_from_vl(vl) ((vl) / SVE_VQ_BYTES)
	155	+#define sve_vl_from_vq(vq) ((vq) * SVE_VQ_BYTES)
	156	+
	157	+/*
	158	+ * If the SVE registers are currently live for the thread at signal delivery,
	159	+ * sve_context.head.size >=
	160	+ * SVE_SIG_CONTEXT_SIZE(sve_vq_from_vl(sve_context.vl))
	161	+ * and the register data may be accessed using the SVE_SIG_*() macros.
	162	+ *
	163	+ * If sve_context.head.size <
	164	+ * SVE_SIG_CONTEXT_SIZE(sve_vq_from_vl(sve_context.vl)),
	165	+ * the SVE registers were not live for the thread and no register data
	166	+ * is included: in this case, the SVE_SIG_*() macros should not be
	167	+ * used except for this check.
	168	+ *
	169	+ * The same convention applies when returning from a signal: a caller
	170	+ * will need to remove or resize the sve_context block if it wants to
	171	+ * make the SVE registers live when they were previously non-live or
	172	+ * vice-versa. This may require the the caller to allocate fresh
	173	+ * memory and/or move other context blocks in the signal frame.
	174	+ *
	175	+ * Changing the vector length during signal return is not permitted:
	176	+ * sve_context.vl must equal the thread's current vector length when
	177	+ * doing a sigreturn.
	178	+ *
	179	+ *
	180	+ * Note: for all these macros, the "vq" argument denotes the SVE
	181	+ * vector length in quadwords (i.e., units of 128 bits).
	182	+ *
	183	+ * The correct way to obtain vq is to use sve_vq_from_vl(vl). The
	184	+ * result is valid if and only if sve_vl_valid(vl) is true. This is
	185	+ * guaranteed for a struct sve_context written by the kernel.
	186	+ *
	187	+ *
	188	+ * Additional macros describe the contents and layout of the payload.
	189	+ * For each, SVE_SIG_x_OFFSET(args) is the start offset relative to
	190	+ * the start of struct sve_context, and SVE_SIG_x_SIZE(args) is the
	191	+ * size in bytes:
	192	+ *
	193	+ * x type description
	194	+ * - ---- -----------
	195	+ * REGS the entire SVE context
	196	+ *
	197	+ * ZREGS __uint128_t[SVE_NUM_ZREGS][vq] all Z-registers
	198	+ * ZREG __uint128_t[vq] individual Z-register Zn
	199	+ *
	200	+ * PREGS uint16_t[SVE_NUM_PREGS][vq] all P-registers
	201	+ * PREG uint16_t[vq] individual P-register Pn
	202	+ *
	203	+ * FFR uint16_t[vq] first-fault status register
	204	+ *
	205	+ * Additional data might be appended in the future.
	206	+ */
	207	+
	208	+#define SVE_SIG_ZREG_SIZE(vq) ((__u32)(vq) * SVE_VQ_BYTES)
	209	+#define SVE_SIG_PREG_SIZE(vq) ((__u32)(vq) * (SVE_VQ_BYTES / 8))
	210	+#define SVE_SIG_FFR_SIZE(vq) SVE_SIG_PREG_SIZE(vq)
	211	+
	212	+#define SVE_SIG_REGS_OFFSET \
	213	+ ((sizeof(struct sve_context) + (SVE_VQ_BYTES - 1)) \
	214	+ / SVE_VQ_BYTES * SVE_VQ_BYTES)
	215	+
	216	+#define SVE_SIG_ZREGS_OFFSET SVE_SIG_REGS_OFFSET
	217	+#define SVE_SIG_ZREG_OFFSET(vq, n) \
	218	+ (SVE_SIG_ZREGS_OFFSET + SVE_SIG_ZREG_SIZE(vq) * (n))
	219	+#define SVE_SIG_ZREGS_SIZE(vq) \
	220	+ (SVE_SIG_ZREG_OFFSET(vq, SVE_NUM_ZREGS) - SVE_SIG_ZREGS_OFFSET)
	221	+
	222	+#define SVE_SIG_PREGS_OFFSET(vq) \
	223	+ (SVE_SIG_ZREGS_OFFSET + SVE_SIG_ZREGS_SIZE(vq))
	224	+#define SVE_SIG_PREG_OFFSET(vq, n) \
	225	+ (SVE_SIG_PREGS_OFFSET(vq) + SVE_SIG_PREG_SIZE(vq) * (n))
	226	+#define SVE_SIG_PREGS_SIZE(vq) \
	227	+ (SVE_SIG_PREG_OFFSET(vq, SVE_NUM_PREGS) - SVE_SIG_PREGS_OFFSET(vq))
	228	+
	229	+#define SVE_SIG_FFR_OFFSET(vq) \
	230	+ (SVE_SIG_PREGS_OFFSET(vq) + SVE_SIG_PREGS_SIZE(vq))
	231	+
	232	+#define SVE_SIG_REGS_SIZE(vq) \
	233	+ (SVE_SIG_FFR_OFFSET(vq) + SVE_SIG_FFR_SIZE(vq) - SVE_SIG_REGS_OFFSET)
	234	+
	235	+#define SVE_SIG_CONTEXT_SIZE(vq) (SVE_SIG_REGS_OFFSET + SVE_SIG_REGS_SIZE(vq))
	236	+
	237	+
120	238	#endif /* _UAPI__ASM_SIGCONTEXT_H */

arch/arm64/kernel/Makefile+0−2

@@ -11,8 +11,6 @@ CFLAGS_REMOVE_ftrace.o = -pg · 10 unmodified lines
11	11	CFLAGS_REMOVE_insn.o = -pg
12	12	CFLAGS_REMOVE_return_address.o = -pg
13	13
14		−CFLAGS_setup.o = -DUTS_MACHINE='"$(UTS_MACHINE)"'
15		−
16	14	# Object file lists.
17	15	arm64-obj-y := debug-monitors.o entry.o irq.o fpsimd.o \
18	16	entry-fpsimd.o process.o ptrace.o setup.o signal.o \

arch/arm64/kernel/armv8_deprecated.c+7−16

@@ -228,15 +228,7 @@ ret: · 227 unmodified lines
228	228	return ret;
229	229	}
230	230
231		−static struct ctl_table ctl_abi[] = {
232		− {
233		− .procname = "abi",
234		− .mode = 0555,
235		− },
236		− { }
237		−};
238		−
239		−static void __init register_insn_emulation_sysctl(struct ctl_table *table)
	231	+static void __init register_insn_emulation_sysctl(void)
240	232	{
241	233	unsigned long flags;
242	234	int i = 0;
@@ -262,8 +254,7 @@ static void __init register_insn_emulation_sysctl(struct ctl_table *table) · 19 unmodified lines
262	254	}
263	255	raw_spin_unlock_irqrestore(&insn_emulation_lock, flags);
264	256
265		− table->child = insns_sysctl;
266		− register_sysctl_table(table);
	257	+ register_sysctl("abi", insns_sysctl);
267	258	}
268	259
269	260	/*
@@ -431,7 +422,7 @@ ret: · 161 unmodified lines
431	422	pr_warn_ratelimited("\"%s\" (%ld) uses obsolete SWP{B} instruction at 0x%llx\n",
432	423	current->comm, (unsigned long)current->pid, regs->pc);
433	424
434		− regs->pc += 4;
	425	+ arm64_skip_faulting_instruction(regs, 4);
435	426	return 0;
436	427
437	428	fault:
@@ -512,7 +503,7 @@ ret: · 74 unmodified lines
512	503	pr_warn_ratelimited("\"%s\" (%ld) uses deprecated CP15 Barrier instruction at 0x%llx\n",
513	504	current->comm, (unsigned long)current->pid, regs->pc);
514	505
515		− regs->pc += 4;
	506	+ arm64_skip_faulting_instruction(regs, 4);
516	507	return 0;
517	508	}
518	509
@@ -586,14 +577,14 @@ static int compat_setend_handler(struct pt_regs *regs, u32 big_endian) · 67 unmodified lines
586	577	static int a32_setend_handler(struct pt_regs *regs, u32 instr)
587	578	{
588	579	int rc = compat_setend_handler(regs, (instr >> 9) & 1);
589		− regs->pc += 4;
	580	+ arm64_skip_faulting_instruction(regs, 4);
590	581	return rc;
591	582	}
592	583
593	584	static int t16_setend_handler(struct pt_regs *regs, u32 instr)
594	585	{
595	586	int rc = compat_setend_handler(regs, (instr >> 3) & 1);
596		− regs->pc += 2;
	587	+ arm64_skip_faulting_instruction(regs, 2);
597	588	return rc;
598	589	}
599	590
@@ -644,7 +635,7 @@ static int __init armv8_deprecated_init(void) · 44 unmodified lines
644	635	cpuhp_setup_state_nocalls(CPUHP_AP_ARM64_ISNDEP_STARTING,
645	636	"arm64/isndep:starting",
646	637	run_all_insn_set_hw_mode, NULL);
647		− register_insn_emulation_sysctl(ctl_abi);
	638	+ register_insn_emulation_sysctl();
648	639
649	640	return 0;
650	641	}

arch/arm64/kernel/cpufeature.c+141−63

@@ -27,6 +27,7 @@ · 26 unmodified lines
27	27	#include <asm/cpu.h>
28	28	#include <asm/cpufeature.h>
29	29	#include <asm/cpu_ops.h>
	30	+#include <asm/fpsimd.h>
30	31	#include <asm/mmu_context.h>
31	32	#include <asm/processor.h>
32	33	#include <asm/sysreg.h>
@@ -51,6 +52,21 @@ unsigned int compat_elf_hwcap2 __read_mostly; · 18 unmodified lines
51	52	DECLARE_BITMAP(cpu_hwcaps, ARM64_NCAPS);
52	53	EXPORT_SYMBOL(cpu_hwcaps);
53	54
	55	+/*
	56	+ * Flag to indicate if we have computed the system wide
	57	+ * capabilities based on the boot time active CPUs. This
	58	+ * will be used to determine if a new booting CPU should
	59	+ * go through the verification process to make sure that it
	60	+ * supports the system capabilities, without using a hotplug
	61	+ * notifier.
	62	+ */
	63	+static bool sys_caps_initialised;
	64	+
	65	+static inline void set_sys_caps_initialised(void)
	66	+{
	67	+ sys_caps_initialised = true;
	68	+}
	69	+
54	70	static int dump_cpu_hwcaps(struct notifier_block self, unsigned long v, void p)
55	71	{
56	72	/* file-wide pr_fmt adds "CPU features: " prefix */
@@ -107,7 +123,11 @@ cpufeature_pan_not_uao(const struct arm64_cpu_capabilities *entry, int __unused) · 50 unmodified lines
107	123	* sync with the documentation of the CPU feature register ABI.
108	124	*/
109	125	static const struct arm64_ftr_bits ftr_id_aa64isar0[] = {
110		− ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64ISAR0_RDM_SHIFT, 4, 0),
	126	+ ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_DP_SHIFT, 4, 0),
	127	+ ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_SM4_SHIFT, 4, 0),
	128	+ ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_SM3_SHIFT, 4, 0),
	129	+ ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_SHA3_SHIFT, 4, 0),
	130	+ ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_RDM_SHIFT, 4, 0),
111	131	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_ATOMICS_SHIFT, 4, 0),
112	132	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_CRC32_SHIFT, 4, 0),
113	133	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_SHA2_SHIFT, 4, 0),
@@ -117,34 +137,35 @@ static const struct arm64_ftr_bits ftr_id_aa64isar0[] = { · 3 unmodified lines
117	137	};
118	138
119	139	static const struct arm64_ftr_bits ftr_id_aa64isar1[] = {
120		− ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64ISAR1_LRCPC_SHIFT, 4, 0),
121		− ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64ISAR1_FCMA_SHIFT, 4, 0),
122		− ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64ISAR1_JSCVT_SHIFT, 4, 0),
123		− ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64ISAR1_DPB_SHIFT, 4, 0),
	140	+ ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_LRCPC_SHIFT, 4, 0),
	141	+ ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_FCMA_SHIFT, 4, 0),
	142	+ ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_JSCVT_SHIFT, 4, 0),
	143	+ ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_DPB_SHIFT, 4, 0),
124	144	ARM64_FTR_END,
125	145	};
126	146
127	147	static const struct arm64_ftr_bits ftr_id_aa64pfr0[] = {
128		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_AA64PFR0_GIC_SHIFT, 4, 0),
	148	+ ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_SVE_SHIFT, 4, 0),
	149	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_GIC_SHIFT, 4, 0),
129	150	S_ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_ASIMD_SHIFT, 4, ID_AA64PFR0_ASIMD_NI),
130	151	S_ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_FP_SHIFT, 4, ID_AA64PFR0_FP_NI),
131	152	/* Linux doesn't care about the EL3 */
132		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_EXACT, ID_AA64PFR0_EL3_SHIFT, 4, 0),
133		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_AA64PFR0_EL2_SHIFT, 4, 0),
134		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_AA64PFR0_EL1_SHIFT, 4, ID_AA64PFR0_EL1_64BIT_ONLY),
135		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_AA64PFR0_EL0_SHIFT, 4, ID_AA64PFR0_EL0_64BIT_ONLY),
	153	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL3_SHIFT, 4, 0),
	154	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL2_SHIFT, 4, 0),
	155	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL1_SHIFT, 4, ID_AA64PFR0_EL1_64BIT_ONLY),
	156	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL0_SHIFT, 4, ID_AA64PFR0_EL0_64BIT_ONLY),
136	157	ARM64_FTR_END,
137	158	};
138	159
139	160	static const struct arm64_ftr_bits ftr_id_aa64mmfr0[] = {
140		− S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_AA64MMFR0_TGRAN4_SHIFT, 4, ID_AA64MMFR0_TGRAN4_NI),
141		− S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_AA64MMFR0_TGRAN64_SHIFT, 4, ID_AA64MMFR0_TGRAN64_NI),
142		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_AA64MMFR0_TGRAN16_SHIFT, 4, ID_AA64MMFR0_TGRAN16_NI),
143		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_AA64MMFR0_BIGENDEL0_SHIFT, 4, 0),
	161	+ S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_TGRAN4_SHIFT, 4, ID_AA64MMFR0_TGRAN4_NI),
	162	+ S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_TGRAN64_SHIFT, 4, ID_AA64MMFR0_TGRAN64_NI),
	163	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_TGRAN16_SHIFT, 4, ID_AA64MMFR0_TGRAN16_NI),
	164	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_BIGENDEL0_SHIFT, 4, 0),
144	165	/* Linux shouldn't care about secure memory */
145		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_EXACT, ID_AA64MMFR0_SNSMEM_SHIFT, 4, 0),
146		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_AA64MMFR0_BIGENDEL_SHIFT, 4, 0),
147		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_AA64MMFR0_ASID_SHIFT, 4, 0),
	166	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_SNSMEM_SHIFT, 4, 0),
	167	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_BIGENDEL_SHIFT, 4, 0),
	168	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_ASID_SHIFT, 4, 0),
148	169	/*
149	170	* Differing PARange is fine as long as all peripherals and memory are mapped
150	171	* within the minimum PARange of all CPUs
@@ -155,20 +176,20 @@ static const struct arm64_ftr_bits ftr_id_aa64mmfr0[] = { · 4 unmodified lines
155	176
156	177	static const struct arm64_ftr_bits ftr_id_aa64mmfr1[] = {
157	178	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_PAN_SHIFT, 4, 0),
158		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_AA64MMFR1_LOR_SHIFT, 4, 0),
159		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_AA64MMFR1_HPD_SHIFT, 4, 0),
160		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_AA64MMFR1_VHE_SHIFT, 4, 0),
161		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_AA64MMFR1_VMIDBITS_SHIFT, 4, 0),
162		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_AA64MMFR1_HADBS_SHIFT, 4, 0),
	179	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_LOR_SHIFT, 4, 0),
	180	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_HPD_SHIFT, 4, 0),
	181	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_VHE_SHIFT, 4, 0),
	182	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_VMIDBITS_SHIFT, 4, 0),
	183	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_HADBS_SHIFT, 4, 0),
163	184	ARM64_FTR_END,
164	185	};
165	186
166	187	static const struct arm64_ftr_bits ftr_id_aa64mmfr2[] = {
167		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_AA64MMFR2_LVA_SHIFT, 4, 0),
168		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_AA64MMFR2_IESB_SHIFT, 4, 0),
169		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_AA64MMFR2_LSM_SHIFT, 4, 0),
170		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_AA64MMFR2_UAO_SHIFT, 4, 0),
171		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_AA64MMFR2_CNP_SHIFT, 4, 0),
	188	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_LVA_SHIFT, 4, 0),
	189	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_IESB_SHIFT, 4, 0),
	190	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_LSM_SHIFT, 4, 0),
	191	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_UAO_SHIFT, 4, 0),
	192	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_CNP_SHIFT, 4, 0),
172	193	ARM64_FTR_END,
173	194	};
174	195
@@ -193,14 +214,14 @@ struct arm64_ftr_reg arm64_ftr_reg_ctrel0 = { · 18 unmodified lines
193	214	};
194	215
195	216	static const struct arm64_ftr_bits ftr_id_mmfr0[] = {
196		− S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, 28, 4, 0xf), /* InnerShr */
197		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, 24, 4, 0), /* FCSE */
	217	+ S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 28, 4, 0xf), /* InnerShr */
	218	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 24, 4, 0), /* FCSE */
198	219	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, 20, 4, 0), /* AuxReg */
199		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, 16, 4, 0), /* TCM */
200		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, 12, 4, 0), /* ShareLvl */
201		− S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, 8, 4, 0xf), /* OuterShr */
202		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, 4, 4, 0), /* PMSA */
203		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, 0, 4, 0), /* VMSA */
	220	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 16, 4, 0), /* TCM */
	221	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 12, 4, 0), /* ShareLvl */
	222	+ S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 8, 4, 0xf), /* OuterShr */
	223	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 4, 4, 0), /* PMSA */
	224	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 0, 4, 0), /* VMSA */
204	225	ARM64_FTR_END,
205	226	};
206	227
@@ -221,8 +242,8 @@ static const struct arm64_ftr_bits ftr_id_aa64dfr0[] = { · 14 unmodified lines
221	242	};
222	243
223	244	static const struct arm64_ftr_bits ftr_mvfr2[] = {
224		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, 4, 4, 0), /* FPMisc */
225		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, 0, 4, 0), /* SIMDMisc */
	245	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 4, 4, 0), /* FPMisc */
	246	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 0, 4, 0), /* SIMDMisc */
226	247	ARM64_FTR_END,
227	248	};
228	249
@@ -234,25 +255,25 @@ static const struct arm64_ftr_bits ftr_dczid[] = { · 5 unmodified lines
234	255
235	256
236	257	static const struct arm64_ftr_bits ftr_id_isar5[] = {
237		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_ISAR5_RDM_SHIFT, 4, 0),
238		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_ISAR5_CRC32_SHIFT, 4, 0),
239		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_ISAR5_SHA2_SHIFT, 4, 0),
240		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_ISAR5_SHA1_SHIFT, 4, 0),
241		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_ISAR5_AES_SHIFT, 4, 0),
242		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_ISAR5_SEVL_SHIFT, 4, 0),
	258	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_RDM_SHIFT, 4, 0),
	259	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_CRC32_SHIFT, 4, 0),
	260	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_SHA2_SHIFT, 4, 0),
	261	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_SHA1_SHIFT, 4, 0),
	262	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_AES_SHIFT, 4, 0),
	263	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_SEVL_SHIFT, 4, 0),
243	264	ARM64_FTR_END,
244	265	};
245	266
246	267	static const struct arm64_ftr_bits ftr_id_mmfr4[] = {
247		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, 4, 4, 0), /* ac2 */
	268	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 4, 4, 0), /* ac2 */
248	269	ARM64_FTR_END,
249	270	};
250	271
251	272	static const struct arm64_ftr_bits ftr_id_pfr0[] = {
252		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, 12, 4, 0), /* State3 */
253		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, 8, 4, 0), /* State2 */
254		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, 4, 4, 0), /* State1 */
255		− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, 0, 4, 0), /* State0 */
	273	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 12, 4, 0), /* State3 */
	274	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 8, 4, 0), /* State2 */
	275	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 4, 4, 0), /* State1 */
	276	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 0, 4, 0), /* State0 */
256	277	ARM64_FTR_END,
257	278	};
258	279
@@ -268,6 +289,12 @@ static const struct arm64_ftr_bits ftr_id_dfr0[] = { · 9 unmodified lines
268	289	ARM64_FTR_END,
269	290	};
270	291
	292	+static const struct arm64_ftr_bits ftr_zcr[] = {
	293	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE,
	294	+ ZCR_ELx_LEN_SHIFT, ZCR_ELx_LEN_SIZE, 0), /* LEN */
	295	+ ARM64_FTR_END,
	296	+};
	297	+
271	298	/*
272	299	* Common ftr bits for a 32bit register with all hidden, strict
273	300	* attributes, with 4bit feature fields and a default safe value of
@@ -334,6 +361,7 @@ static const struct __ftr_reg_entry { · 60 unmodified lines
334	361	/* Op1 = 0, CRn = 0, CRm = 4 */
335	362	ARM64_FTR_REG(SYS_ID_AA64PFR0_EL1, ftr_id_aa64pfr0),
336	363	ARM64_FTR_REG(SYS_ID_AA64PFR1_EL1, ftr_raz),
	364	+ ARM64_FTR_REG(SYS_ID_AA64ZFR0_EL1, ftr_raz),
337	365
338	366	/* Op1 = 0, CRn = 0, CRm = 5 */
339	367	ARM64_FTR_REG(SYS_ID_AA64DFR0_EL1, ftr_id_aa64dfr0),
@@ -348,6 +376,9 @@ static const struct __ftr_reg_entry { · 8 unmodified lines
348	376	ARM64_FTR_REG(SYS_ID_AA64MMFR1_EL1, ftr_id_aa64mmfr1),
349	377	ARM64_FTR_REG(SYS_ID_AA64MMFR2_EL1, ftr_id_aa64mmfr2),
350	378
	379	+ /* Op1 = 0, CRn = 1, CRm = 2 */
	380	+ ARM64_FTR_REG(SYS_ZCR_EL1, ftr_zcr),
	381	+
351	382	/* Op1 = 3, CRn = 0, CRm = 0 */
352	383	{ SYS_CTR_EL0, &arm64_ftr_reg_ctrel0 },
353	384	ARM64_FTR_REG(SYS_DCZID_EL0, ftr_dczid),
@@ -485,6 +516,7 @@ void __init init_cpu_features(struct cpuinfo_arm64 *info) · 131 unmodified lines
485	516	init_cpu_ftr_reg(SYS_ID_AA64MMFR2_EL1, info->reg_id_aa64mmfr2);
486	517	init_cpu_ftr_reg(SYS_ID_AA64PFR0_EL1, info->reg_id_aa64pfr0);
487	518	init_cpu_ftr_reg(SYS_ID_AA64PFR1_EL1, info->reg_id_aa64pfr1);
	519	+ init_cpu_ftr_reg(SYS_ID_AA64ZFR0_EL1, info->reg_id_aa64zfr0);
488	520
489	521	if (id_aa64pfr0_32bit_el0(info->reg_id_aa64pfr0)) {
490	522	init_cpu_ftr_reg(SYS_ID_DFR0_EL1, info->reg_id_dfr0);
@@ -505,6 +537,10 @@ void __init init_cpu_features(struct cpuinfo_arm64 *info) · 14 unmodified lines
505	537	init_cpu_ftr_reg(SYS_MVFR2_EL1, info->reg_mvfr2);
506	538	}
507	539
	540	+ if (id_aa64pfr0_sve(info->reg_id_aa64pfr0)) {
	541	+ init_cpu_ftr_reg(SYS_ZCR_EL1, info->reg_zcr);
	542	+ sve_init_vq_map();
	543	+ }
508	544	}
509	545
510	546	static void update_cpu_ftr_reg(struct arm64_ftr_reg *reg, u64 new)
@@ -608,6 +644,9 @@ void update_cpu_features(int cpu, · 97 unmodified lines
608	644	taint \|= check_update_ftr_reg(SYS_ID_AA64PFR1_EL1, cpu,
609	645	info->reg_id_aa64pfr1, boot->reg_id_aa64pfr1);
610	646
	647	+ taint \|= check_update_ftr_reg(SYS_ID_AA64ZFR0_EL1, cpu,
	648	+ info->reg_id_aa64zfr0, boot->reg_id_aa64zfr0);
	649	+
611	650	/*
612	651	* If we have AArch32, we care about 32-bit features for compat.
613	652	* If the system doesn't support AArch32, don't update them.
@@ -655,6 +694,16 @@ void update_cpu_features(int cpu, · 41 unmodified lines
655	694	info->reg_mvfr2, boot->reg_mvfr2);
656	695	}
657	696
	697	+ if (id_aa64pfr0_sve(info->reg_id_aa64pfr0)) {
	698	+ taint \|= check_update_ftr_reg(SYS_ZCR_EL1, cpu,
	699	+ info->reg_zcr, boot->reg_zcr);
	700	+
	701	+ /* Probe vector lengths, unless we already gave up on SVE */
	702	+ if (id_aa64pfr0_sve(read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1)) &&
	703	+ !sys_caps_initialised)
	704	+ sve_update_vq_map();
	705	+ }
	706	+
658	707	/*
659	708	* Mismatched CPU features are a recipe for disaster. Don't even
660	709	* pretend to support them.
@@ -900,6 +949,19 @@ static const struct arm64_cpu_capabilities arm64_features[] = { · 239 unmodified lines
900	949	.min_field_value = 1,
901	950	},
902	951	#endif
	952	+#ifdef CONFIG_ARM64_SVE
	953	+ {
	954	+ .desc = "Scalable Vector Extension",
	955	+ .capability = ARM64_SVE,
	956	+ .def_scope = SCOPE_SYSTEM,
	957	+ .sys_reg = SYS_ID_AA64PFR0_EL1,
	958	+ .sign = FTR_UNSIGNED,
	959	+ .field_pos = ID_AA64PFR0_SVE_SHIFT,
	960	+ .min_field_value = ID_AA64PFR0_SVE,
	961	+ .matches = has_cpuid_feature,
	962	+ .enable = sve_kernel_enable,
	963	+ },
	964	+#endif /* CONFIG_ARM64_SVE */
903	965	{},
904	966	};
905	967
@@ -921,9 +983,14 @@ static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = { · 15 unmodified lines
921	983	HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_AES_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_AES),
922	984	HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_SHA1_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_SHA1),
923	985	HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_SHA2_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_SHA2),
	986	+ HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_SHA2_SHIFT, FTR_UNSIGNED, 2, CAP_HWCAP, HWCAP_SHA512),
924	987	HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_CRC32_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_CRC32),
925	988	HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_ATOMICS_SHIFT, FTR_UNSIGNED, 2, CAP_HWCAP, HWCAP_ATOMICS),
926	989	HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_RDM_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_ASIMDRDM),
	990	+ HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_SHA3_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_SHA3),
	991	+ HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_SM3_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_SM3),
	992	+ HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_SM4_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_SM4),
	993	+ HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_DP_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_ASIMDDP),
927	994	HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_FP_SHIFT, FTR_SIGNED, 0, CAP_HWCAP, HWCAP_FP),
928	995	HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_FP_SHIFT, FTR_SIGNED, 1, CAP_HWCAP, HWCAP_FPHP),
929	996	HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_ASIMD_SHIFT, FTR_SIGNED, 0, CAP_HWCAP, HWCAP_ASIMD),
@@ -932,6 +999,9 @@ static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = { · 2 unmodified lines
932	999	HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_JSCVT_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_JSCVT),
933	1000	HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_FCMA_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_FCMA),
934	1001	HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_LRCPC_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_LRCPC),
	1002	+#ifdef CONFIG_ARM64_SVE
	1003	+ HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_SVE_SHIFT, FTR_UNSIGNED, ID_AA64PFR0_SVE, CAP_HWCAP, HWCAP_SVE),
	1004	+#endif
935	1005	{},
936	1006	};
937	1007
@@ -1040,21 +1110,6 @@ void __init enable_cpu_capabilities(const struct arm64_cpu_capabilities *caps) · 102 unmodified lines
1040	1110	}
1041	1111	}
1042	1112
1043		−/*
1044		− * Flag to indicate if we have computed the system wide
1045		− * capabilities based on the boot time active CPUs. This
1046		− * will be used to determine if a new booting CPU should
1047		− * go through the verification process to make sure that it
1048		− * supports the system capabilities, without using a hotplug
1049		− * notifier.
1050		− */
1051		−static bool sys_caps_initialised;
1052		−
1053		−static inline void set_sys_caps_initialised(void)
1054		−{
1055		− sys_caps_initialised = true;
1056		−}
1057		−
1058	1113	/*
1059	1114	* Check for CPU features that are used in early boot
1060	1115	* based on the Boot CPU value.
@@ -1097,6 +1152,23 @@ verify_local_cpu_features(const struct arm64_cpu_capabilities *caps) · 36 unmodified lines
1097	1152	}
1098	1153	}
1099	1154
	1155	+static void verify_sve_features(void)
	1156	+{
	1157	+ u64 safe_zcr = read_sanitised_ftr_reg(SYS_ZCR_EL1);
	1158	+ u64 zcr = read_zcr_features();
	1159	+
	1160	+ unsigned int safe_len = safe_zcr & ZCR_ELx_LEN_MASK;
	1161	+ unsigned int len = zcr & ZCR_ELx_LEN_MASK;
	1162	+
	1163	+ if (len < safe_len \|\| sve_verify_vq_map()) {
	1164	+ pr_crit("CPU%d: SVE: required vector length(s) missing\n",
	1165	+ smp_processor_id());
	1166	+ cpu_die_early();
	1167	+ }
	1168	+
	1169	+ /* Add checks on other ZCR bits here if necessary */
	1170	+}
	1171	+
1100	1172	/*
1101	1173	* Run through the enabled system capabilities and enable() it on this CPU.
1102	1174	* The capabilities were decided based on the available CPUs at the boot time.
@@ -1110,8 +1182,12 @@ static void verify_local_cpu_capabilities(void) · 7 unmodified lines
1110	1182	verify_local_cpu_errata_workarounds();
1111	1183	verify_local_cpu_features(arm64_features);
1112	1184	verify_local_elf_hwcaps(arm64_elf_hwcaps);
	1185	+
1113	1186	if (system_supports_32bit_el0())
1114	1187	verify_local_elf_hwcaps(compat_elf_hwcaps);
	1188	+
	1189	+ if (system_supports_sve())
	1190	+ verify_sve_features();
1115	1191	}
1116	1192
1117	1193	void check_local_cpu_capabilities(void)
@@ -1189,6 +1265,8 @@ void __init setup_cpu_features(void) · 71 unmodified lines
1189	1265	if (system_supports_32bit_el0())
1190	1266	setup_elf_hwcaps(compat_elf_hwcaps);
1191	1267
	1268	+ sve_setup();
	1269	+
1192	1270	/* Advertise that we have computed the system capabilities */
1193	1271	set_sys_caps_initialised();
1194	1272
@@ -1287,7 +1365,7 @@ static int emulate_mrs(struct pt_regs *regs, u32 insn) · 92 unmodified lines
1287	1365	if (!rc) {
1288	1366	dst = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RT, insn);
1289	1367	pt_regs_write_reg(regs, dst, val);
1290		− regs->pc += 4;
	1368	+ arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE);
1291	1369	}
1292	1370
1293	1371	return rc;

arch/arm64/kernel/cpuinfo.c+12−0

@@ -19,6 +19,7 @@ · 18 unmodified lines
19	19	#include <asm/cpu.h>
20	20	#include <asm/cputype.h>
21	21	#include <asm/cpufeature.h>
	22	+#include <asm/fpsimd.h>
22	23
23	24	#include <linux/bitops.h>
24	25	#include <linux/bug.h>
@@ -69,6 +70,12 @@ static const char *const hwcap_str[] = { · 44 unmodified lines
69	70	"fcma",
70	71	"lrcpc",
71	72	"dcpop",
	73	+ "sha3",
	74	+ "sm3",
	75	+ "sm4",
	76	+ "asimddp",
	77	+ "sha512",
	78	+ "sve",
72	79	NULL
73	80	};
74	81
@@ -326,6 +333,7 @@ static void __cpuinfo_store_cpu(struct cpuinfo_arm64 *info) · 251 unmodified lines
326	333	info->reg_id_aa64mmfr2 = read_cpuid(ID_AA64MMFR2_EL1);
327	334	info->reg_id_aa64pfr0 = read_cpuid(ID_AA64PFR0_EL1);
328	335	info->reg_id_aa64pfr1 = read_cpuid(ID_AA64PFR1_EL1);
	336	+ info->reg_id_aa64zfr0 = read_cpuid(ID_AA64ZFR0_EL1);
329	337
330	338	/* Update the 32bit ID registers only if AArch32 is implemented */
331	339	if (id_aa64pfr0_32bit_el0(info->reg_id_aa64pfr0)) {
@@ -348,6 +356,10 @@ static void __cpuinfo_store_cpu(struct cpuinfo_arm64 *info) · 16 unmodified lines
348	356	info->reg_mvfr2 = read_cpuid(MVFR2_EL1);
349	357	}
350	358
	359	+ if (IS_ENABLED(CONFIG_ARM64_SVE) &&
	360	+ id_aa64pfr0_sve(info->reg_id_aa64pfr0))
	361	+ info->reg_zcr = read_zcr_features();
	362	+
351	363	cpuinfo_detect_icache_policy(info);
352	364	}
353	365

arch/arm64/kernel/debug-monitors.c+3−2

@@ -30,6 +30,7 @@ · 29 unmodified lines
30	30
31	31	#include <asm/cpufeature.h>
32	32	#include <asm/cputype.h>
	33	+#include <asm/daifflags.h>
33	34	#include <asm/debug-monitors.h>
34	35	#include <asm/system_misc.h>
35	36
@@ -46,9 +47,9 @@ u8 debug_monitors_arch(void) · 10 unmodified lines
46	47	static void mdscr_write(u32 mdscr)
47	48	{
48	49	unsigned long flags;
49		− local_dbg_save(flags);
	50	+ flags = local_daif_save();
50	51	write_sysreg(mdscr, mdscr_el1);
51		− local_dbg_restore(flags);
	52	+ local_daif_restore(flags);
52	53	}
53	54	NOKPROBE_SYMBOL(mdscr_write);
54	55

arch/arm64/kernel/entry-fpsimd.S+17−0

@@ -41,3 +41,20 @@ ENTRY(fpsimd_load_state) · 40 unmodified lines
41	41	fpsimd_restore x0, 8
42	42	ret
43	43	ENDPROC(fpsimd_load_state)
	44	+
	45	+#ifdef CONFIG_ARM64_SVE
	46	+ENTRY(sve_save_state)
	47	+ sve_save 0, x1, 2
	48	+ ret
	49	+ENDPROC(sve_save_state)
	50	+
	51	+ENTRY(sve_load_state)
	52	+ sve_load 0, x1, x2, 3
	53	+ ret
	54	+ENDPROC(sve_load_state)
	55	+
	56	+ENTRY(sve_get_vl)
	57	+ _sve_rdvl 0, 1
	58	+ ret
	59	+ENDPROC(sve_get_vl)
	60	+#endif /* CONFIG_ARM64_SVE */

arch/arm64/kernel/entry-ftrace.S+3−9

@@ -108,13 +108,8 @@ ENTRY(_mcount) · 107 unmodified lines
108	108	mcount_get_lr x1 // function's lr (= parent's pc)
109	109	blr x2 // (*ftrace_trace_function)(pc, lr);
110	110
111		−#ifndef CONFIG_FUNCTION_GRAPH_TRACER
112		−skip_ftrace_call: // return;
113		− mcount_exit // }
114		−#else
115		− mcount_exit // return;
116		− // }
117		−skip_ftrace_call:
	111	+skip_ftrace_call: // }
	112	+#ifdef CONFIG_FUNCTION_GRAPH_TRACER
118	113	ldr_l x2, ftrace_graph_return
119	114	cmp x0, x2 // if ((ftrace_graph_return
120	115	b.ne ftrace_graph_caller // != ftrace_stub)
@@ -123,9 +118,8 @@ skip_ftrace_call: · 2 unmodified lines
123	118	adr_l x0, ftrace_graph_entry_stub // != ftrace_graph_entry_stub))
124	119	cmp x0, x2
125	120	b.ne ftrace_graph_caller // ftrace_graph_caller();
126		−
127		− mcount_exit
128	121	#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
	122	+ mcount_exit
129	123	ENDPROC(_mcount)
130	124
131	125	#else /* CONFIG_DYNAMIC_FTRACE */

arch/arm64/kernel/entry.S+88−40

@@ -28,7 +28,7 @@ · 27 unmodified lines
28	28	#include <asm/errno.h>
29	29	#include <asm/esr.h>
30	30	#include <asm/irq.h>
31		−#include <asm/memory.h>
	31	+#include <asm/processor.h>
32	32	#include <asm/ptrace.h>
33	33	#include <asm/thread_info.h>
34	34	#include <asm/asm-uaccess.h>
@@ -221,6 +221,8 @@ alternative_else_nop_endif · 186 unmodified lines
221	221
222	222	.macro kernel_exit, el
223	223	.if \el != 0
	224	+ disable_daif
	225	+
224	226	/* Restore the task's original addr_limit. */
225	227	ldr x20, [sp, #S_ORIG_ADDR_LIMIT]
226	228	str x20, [tsk, #TSK_TI_ADDR_LIMIT]
@@ -373,18 +375,18 @@ ENTRY(vectors) · 146 unmodified lines
373	375	kernel_ventry el1_sync // Synchronous EL1h
374	376	kernel_ventry el1_irq // IRQ EL1h
375	377	kernel_ventry el1_fiq_invalid // FIQ EL1h
376		− kernel_ventry el1_error_invalid // Error EL1h
	378	+ kernel_ventry el1_error // Error EL1h
377	379
378	380	kernel_ventry el0_sync // Synchronous 64-bit EL0
379	381	kernel_ventry el0_irq // IRQ 64-bit EL0
380	382	kernel_ventry el0_fiq_invalid // FIQ 64-bit EL0
381		− kernel_ventry el0_error_invalid // Error 64-bit EL0
	383	+ kernel_ventry el0_error // Error 64-bit EL0
382	384
383	385	#ifdef CONFIG_COMPAT
384	386	kernel_ventry el0_sync_compat // Synchronous 32-bit EL0
385	387	kernel_ventry el0_irq_compat // IRQ 32-bit EL0
386	388	kernel_ventry el0_fiq_invalid_compat // FIQ 32-bit EL0
387		− kernel_ventry el0_error_invalid_compat // Error 32-bit EL0
	389	+ kernel_ventry el0_error_compat // Error 32-bit EL0
388	390	#else
389	391	kernel_ventry el0_sync_invalid // Synchronous 32-bit EL0
390	392	kernel_ventry el0_irq_invalid // IRQ 32-bit EL0
@@ -453,10 +455,6 @@ ENDPROC(el0_error_invalid) · 62 unmodified lines
453	455	el0_fiq_invalid_compat:
454	456	inv_entry 0, BAD_FIQ, 32
455	457	ENDPROC(el0_fiq_invalid_compat)
456		−
457		−el0_error_invalid_compat:
458		− inv_entry 0, BAD_ERROR, 32
459		−ENDPROC(el0_error_invalid_compat)
460	458	#endif
461	459
462	460	el1_sync_invalid:
@@ -508,24 +506,18 @@ el1_da: · 45 unmodified lines
508	506	* Data abort handling
509	507	*/
510	508	mrs x3, far_el1
511		− enable_dbg
512		− // re-enable interrupts if they were enabled in the aborted context
513		− tbnz x23, #7, 1f // PSR_I_BIT
514		− enable_irq
515		−1:
	509	+ inherit_daif pstate=x23, tmp=x2
516	510	clear_address_tag x0, x3
517	511	mov x2, sp // struct pt_regs
518	512	bl do_mem_abort
519	513
520		− // disable interrupts before pulling preserved data off the stack
521		− disable_irq
522	514	kernel_exit 1
523	515	el1_sp_pc:
524	516	/*
525	517	* Stack or PC alignment exception handling
526	518	*/
527	519	mrs x0, far_el1
528		− enable_dbg
	520	+ inherit_daif pstate=x23, tmp=x2
529	521	mov x2, sp
530	522	bl do_sp_pc_abort
531	523	ASM_BUG()
@@ -533,7 +525,7 @@ el1_undef: · 1 unmodified line
533	525	/*
534	526	* Undefined instruction
535	527	*/
536		− enable_dbg
	528	+ inherit_daif pstate=x23, tmp=x2
537	529	mov x0, sp
538	530	bl do_undefinstr
539	531	ASM_BUG()
@@ -550,7 +542,7 @@ el1_dbg: · 10 unmodified lines
550	542	kernel_exit 1
551	543	el1_inv:
552	544	// TODO: add support for undefined instructions in kernel mode
553		− enable_dbg
	545	+ inherit_daif pstate=x23, tmp=x2
554	546	mov x0, sp
555	547	mov x2, x1
556	548	mov x1, #BAD_SYNC
@@ -561,7 +553,7 @@ ENDPROC(el1_sync) · 4 unmodified lines
561	553	.align 6
562	554	el1_irq:
563	555	kernel_entry 1
564		− enable_dbg
	556	+ enable_da_f
565	557	#ifdef CONFIG_TRACE_IRQFLAGS
566	558	bl trace_hardirqs_off
567	559	#endif
@@ -607,6 +599,8 @@ el0_sync: · 39 unmodified lines
607	599	b.eq el0_ia
608	600	cmp x24, #ESR_ELx_EC_FP_ASIMD // FP/ASIMD access
609	601	b.eq el0_fpsimd_acc
	602	+ cmp x24, #ESR_ELx_EC_SVE // SVE access
	603	+ b.eq el0_sve_acc
610	604	cmp x24, #ESR_ELx_EC_FP_EXC64 // FP/ASIMD exception
611	605	b.eq el0_fpsimd_exc
612	606	cmp x24, #ESR_ELx_EC_SYS64 // configurable trap
@@ -658,6 +652,7 @@ el0_svc_compat: · 45 unmodified lines
658	652	/*
659	653	* AArch32 syscall handling
660	654	*/
	655	+ ldr x16, [tsk, #TSK_TI_FLAGS] // load thread flags
661	656	adrp stbl, compat_sys_call_table // load compat syscall table pointer
662	657	mov wscno, w7 // syscall number in w7 (r7)
663	658	mov wsc_nr, #__NR_compat_syscalls
@@ -667,6 +662,10 @@ el0_svc_compat: · 3 unmodified lines
667	662	el0_irq_compat:
668	663	kernel_entry 0, 32
669	664	b el0_irq_naked
	665	+
	666	+el0_error_compat:
	667	+ kernel_entry 0, 32
	668	+ b el0_error_naked
670	669	#endif
671	670
672	671	el0_da:
@@ -674,8 +673,7 @@ el0_da: · 1 unmodified line
674	673	* Data abort handling
675	674	*/
676	675	mrs x26, far_el1
677		− // enable interrupts before calling the main handler
678		− enable_dbg_and_irq
	676	+ enable_daif
679	677	ct_user_exit
680	678	clear_address_tag x0, x26
681	679	mov x1, x25
@@ -687,8 +685,7 @@ el0_ia: · 5 unmodified lines
687	685	* Instruction abort handling
688	686	*/
689	687	mrs x26, far_el1
690		− // enable interrupts before calling the main handler
691		− enable_dbg_and_irq
	688	+ enable_daif
692	689	ct_user_exit
693	690	mov x0, x26
694	691	mov x1, x25
@@ -699,17 +696,27 @@ el0_fpsimd_acc: · 4 unmodified lines
699	696	/*
700	697	* Floating Point or Advanced SIMD access
701	698	*/
702		− enable_dbg
	699	+ enable_daif
703	700	ct_user_exit
704	701	mov x0, x25
705	702	mov x1, sp
706	703	bl do_fpsimd_acc
707	704	b ret_to_user
	705	+el0_sve_acc:
	706	+ /*
	707	+ * Scalable Vector Extension access
	708	+ */
	709	+ enable_daif
	710	+ ct_user_exit
	711	+ mov x0, x25
	712	+ mov x1, sp
	713	+ bl do_sve_acc
	714	+ b ret_to_user
708	715	el0_fpsimd_exc:
709	716	/*
710		− * Floating Point or Advanced SIMD exception
	717	+ * Floating Point, Advanced SIMD or SVE exception
711	718	*/
712		− enable_dbg
	719	+ enable_daif
713	720	ct_user_exit
714	721	mov x0, x25
715	722	mov x1, sp
@@ -720,8 +727,7 @@ el0_sp_pc: · 4 unmodified lines
720	727	* Stack or PC alignment exception handling
721	728	*/
722	729	mrs x26, far_el1
723		− // enable interrupts before calling the main handler
724		− enable_dbg_and_irq
	730	+ enable_daif
725	731	ct_user_exit
726	732	mov x0, x26
727	733	mov x1, x25
@@ -732,8 +738,7 @@ el0_undef: · 4 unmodified lines
732	738	/*
733	739	* Undefined instruction
734	740	*/
735		− // enable interrupts before calling the main handler
736		− enable_dbg_and_irq
	741	+ enable_daif
737	742	ct_user_exit
738	743	mov x0, sp
739	744	bl do_undefinstr
@@ -742,7 +747,7 @@ el0_sys: · 2 unmodified lines
742	747	/*
743	748	* System instructions, for trapped cache maintenance instructions
744	749	*/
745		− enable_dbg_and_irq
	750	+ enable_daif
746	751	ct_user_exit
747	752	mov x0, x25
748	753	mov x1, sp
@@ -757,11 +762,11 @@ el0_dbg: · 8 unmodified lines
757	762	mov x1, x25
758	763	mov x2, sp
759	764	bl do_debug_exception
760		− enable_dbg
	765	+ enable_daif
761	766	ct_user_exit
762	767	b ret_to_user
763	768	el0_inv:
764		− enable_dbg
	769	+ enable_daif
765	770	ct_user_exit
766	771	mov x0, sp
767	772	mov x1, #BAD_SYNC
@@ -774,7 +779,7 @@ ENDPROC(el0_sync) · 6 unmodified lines
774	779	el0_irq:
775	780	kernel_entry 0
776	781	el0_irq_naked:
777		− enable_dbg
	782	+ enable_da_f
778	783	#ifdef CONFIG_TRACE_IRQFLAGS
779	784	bl trace_hardirqs_off
780	785	#endif
@@ -788,12 +793,34 @@ el0_irq_naked: · 7 unmodified lines
788	793	b ret_to_user
789	794	ENDPROC(el0_irq)
790	795
	796	+el1_error:
	797	+ kernel_entry 1
	798	+ mrs x1, esr_el1
	799	+ enable_dbg
	800	+ mov x0, sp
	801	+ bl do_serror
	802	+ kernel_exit 1
	803	+ENDPROC(el1_error)
	804	+
	805	+el0_error:
	806	+ kernel_entry 0
	807	+el0_error_naked:
	808	+ mrs x1, esr_el1
	809	+ enable_dbg
	810	+ mov x0, sp
	811	+ bl do_serror
	812	+ enable_daif
	813	+ ct_user_exit
	814	+ b ret_to_user
	815	+ENDPROC(el0_error)
	816	+
	817	+
791	818	/*
792	819	* This is the fast syscall return path. We do as little as possible here,
793	820	* and this includes saving x0 back into the kernel stack.
794	821	*/
795	822	ret_fast_syscall:
796		− disable_irq // disable interrupts
	823	+ disable_daif
797	824	str x0, [sp, #S_X0] // returned x0
798	825	ldr x1, [tsk, #TSK_TI_FLAGS] // re-check for syscall tracing
799	826	and x2, x1, #_TIF_SYSCALL_WORK
@@ -803,7 +830,7 @@ ret_fast_syscall: · 3 unmodified lines
803	830	enable_step_tsk x1, x2
804	831	kernel_exit 0
805	832	ret_fast_syscall_trace:
806		− enable_irq // enable interrupts
	833	+ enable_daif
807	834	b __sys_trace_return_skipped // we already saved x0
808	835
809	836	/*
@@ -821,7 +848,7 @@ work_pending: · 11 unmodified lines
821	848	* "slow" syscall return path.
822	849	*/
823	850	ret_to_user:
824		− disable_irq // disable interrupts
	851	+ disable_daif
825	852	ldr x1, [tsk, #TSK_TI_FLAGS]
826	853	and x2, x1, #_TIF_WORK_MASK
827	854	cbnz x2, work_pending
@@ -835,16 +862,37 @@ ENDPROC(ret_to_user) · 7 unmodified lines
835	862	*/
836	863	.align 6
837	864	el0_svc:
	865	+ ldr x16, [tsk, #TSK_TI_FLAGS] // load thread flags
838	866	adrp stbl, sys_call_table // load syscall table pointer
839	867	mov wscno, w8 // syscall number in w8
840	868	mov wsc_nr, #__NR_syscalls
	869	+
	870	+#ifdef CONFIG_ARM64_SVE
	871	+alternative_if_not ARM64_SVE
	872	+ b el0_svc_naked
	873	+alternative_else_nop_endif
	874	+ tbz x16, #TIF_SVE, el0_svc_naked // Skip unless TIF_SVE set:
	875	+ bic x16, x16, #_TIF_SVE // discard SVE state
	876	+ str x16, [tsk, #TSK_TI_FLAGS]
	877	+
	878	+ /*
	879	+ * task_fpsimd_load() won't be called to update CPACR_EL1 in
	880	+ * ret_to_user unless TIF_FOREIGN_FPSTATE is still set, which only
	881	+ * happens if a context switch or kernel_neon_begin() or context
	882	+ * modification (sigreturn, ptrace) intervenes.
	883	+ * So, ensure that CPACR_EL1 is already correct for the fast-path case:
	884	+ */
	885	+ mrs x9, cpacr_el1
	886	+ bic x9, x9, #CPACR_EL1_ZEN_EL0EN // disable SVE for el0
	887	+ msr cpacr_el1, x9 // synchronised by eret to el0
	888	+#endif
	889	+
841	890	el0_svc_naked: // compat entry point
842	891	stp x0, xscno, [sp, #S_ORIG_X0] // save the original x0 and syscall number
843		− enable_dbg_and_irq
	892	+ enable_daif
844	893	ct_user_exit 1
845	894
846		− ldr x16, [tsk, #TSK_TI_FLAGS] // check for syscall hooks
847		− tst x16, #_TIF_SYSCALL_WORK
	895	+ tst x16, #_TIF_SYSCALL_WORK // check for syscall hooks
848	896	b.ne __sys_trace
849	897	cmp wscno, wsc_nr // check upper syscall limit
850	898	b.hs ni_sys

arch/arm64/kernel/fpsimd.c+881−27

@@ -17,19 +17,34 @@ · 16 unmodified lines
17	17	* along with this program. If not, see <http://www.gnu.org/licenses/>.
18	18	*/
19	19
	20	+#include <linux/bitmap.h>
20	21	#include <linux/bottom_half.h>
	22	+#include <linux/bug.h>
	23	+#include <linux/cache.h>
	24	+#include <linux/compat.h>
21	25	#include <linux/cpu.h>
22	26	#include <linux/cpu_pm.h>
23	27	#include <linux/kernel.h>
	28	+#include <linux/linkage.h>
	29	+#include <linux/irqflags.h>
24	30	#include <linux/init.h>
25	31	#include <linux/percpu.h>
	32	+#include <linux/prctl.h>
26	33	#include <linux/preempt.h>
	34	+#include <linux/prctl.h>
	35	+#include <linux/ptrace.h>
27	36	#include <linux/sched/signal.h>
	37	+#include <linux/sched/task_stack.h>
28	38	#include <linux/signal.h>
	39	+#include <linux/slab.h>
	40	+#include <linux/sysctl.h>
29	41
30	42	#include <asm/fpsimd.h>
31	43	#include <asm/cputype.h>
32	44	#include <asm/simd.h>
	45	+#include <asm/sigcontext.h>
	46	+#include <asm/sysreg.h>
	47	+#include <asm/traps.h>
33	48
34	49	#define FPEXC_IOF (1 << 0)
35	50	#define FPEXC_DZF (1 << 1)
@@ -39,6 +54,8 @@ · 3 unmodified lines
39	54	#define FPEXC_IDF (1 << 7)
40	55
41	56	/*
	57	+ * (Note: in this discussion, statements about FPSIMD apply equally to SVE.)
	58	+ *
42	59	* In order to reduce the number of times the FPSIMD state is needlessly saved
43	60	* and restored, we need to keep track of two things:
44	61	* (a) for each task, we need to remember which CPU was the last one to have
@@ -99,10 +116,741 @@ · 54 unmodified lines
99	116	*/
100	117	static DEFINE_PER_CPU(struct fpsimd_state *, fpsimd_last_state);
101	118
	119	+/* Default VL for tasks that don't set it explicitly: */
	120	+static int sve_default_vl = -1;
	121	+
	122	+#ifdef CONFIG_ARM64_SVE
	123	+
	124	+/* Maximum supported vector length across all CPUs (initially poisoned) */
	125	+int __ro_after_init sve_max_vl = -1;
	126	+/* Set of available vector lengths, as vq_to_bit(vq): */
	127	+static __ro_after_init DECLARE_BITMAP(sve_vq_map, SVE_VQ_MAX);
	128	+static void __percpu *efi_sve_state;
	129	+
	130	+#else /* ! CONFIG_ARM64_SVE */
	131	+
	132	+/* Dummy declaration for code that will be optimised out: */
	133	+extern __ro_after_init DECLARE_BITMAP(sve_vq_map, SVE_VQ_MAX);
	134	+extern void __percpu *efi_sve_state;
	135	+
	136	+#endif /* ! CONFIG_ARM64_SVE */
	137	+
	138	+/*
	139	+ * Call __sve_free() directly only if you know task can't be scheduled
	140	+ * or preempted.
	141	+ */
	142	+static void __sve_free(struct task_struct *task)
	143	+{
	144	+ kfree(task->thread.sve_state);
	145	+ task->thread.sve_state = NULL;
	146	+}
	147	+
	148	+static void sve_free(struct task_struct *task)
	149	+{
	150	+ WARN_ON(test_tsk_thread_flag(task, TIF_SVE));
	151	+
	152	+ __sve_free(task);
	153	+}
	154	+
	155	+
	156	+/* Offset of FFR in the SVE register dump */
	157	+static size_t sve_ffr_offset(int vl)
	158	+{
	159	+ return SVE_SIG_FFR_OFFSET(sve_vq_from_vl(vl)) - SVE_SIG_REGS_OFFSET;
	160	+}
	161	+
	162	+static void sve_pffr(struct task_struct task)
	163	+{
	164	+ return (char *)task->thread.sve_state +
	165	+ sve_ffr_offset(task->thread.sve_vl);
	166	+}
	167	+
	168	+static void change_cpacr(u64 val, u64 mask)
	169	+{
	170	+ u64 cpacr = read_sysreg(CPACR_EL1);
	171	+ u64 new = (cpacr & ~mask) \| val;
	172	+
	173	+ if (new != cpacr)
	174	+ write_sysreg(new, CPACR_EL1);
	175	+}
	176	+
	177	+static void sve_user_disable(void)
	178	+{
	179	+ change_cpacr(0, CPACR_EL1_ZEN_EL0EN);
	180	+}
	181	+
	182	+static void sve_user_enable(void)
	183	+{
	184	+ change_cpacr(CPACR_EL1_ZEN_EL0EN, CPACR_EL1_ZEN_EL0EN);
	185	+}
	186	+
	187	+/*
	188	+ * TIF_SVE controls whether a task can use SVE without trapping while
	189	+ * in userspace, and also the way a task's FPSIMD/SVE state is stored
	190	+ * in thread_struct.
	191	+ *
	192	+ * The kernel uses this flag to track whether a user task is actively
	193	+ * using SVE, and therefore whether full SVE register state needs to
	194	+ * be tracked. If not, the cheaper FPSIMD context handling code can
	195	+ * be used instead of the more costly SVE equivalents.
	196	+ *
	197	+ * * TIF_SVE set:
	198	+ *
	199	+ * The task can execute SVE instructions while in userspace without
	200	+ * trapping to the kernel.
	201	+ *
	202	+ * When stored, Z0-Z31 (incorporating Vn in bits[127:0] or the
	203	+ * corresponding Zn), P0-P15 and FFR are encoded in in
	204	+ * task->thread.sve_state, formatted appropriately for vector
	205	+ * length task->thread.sve_vl.
	206	+ *
	207	+ * task->thread.sve_state must point to a valid buffer at least
	208	+ * sve_state_size(task) bytes in size.
	209	+ *
	210	+ * During any syscall, the kernel may optionally clear TIF_SVE and
	211	+ * discard the vector state except for the FPSIMD subset.
	212	+ *
	213	+ * * TIF_SVE clear:
	214	+ *
	215	+ * An attempt by the user task to execute an SVE instruction causes
	216	+ * do_sve_acc() to be called, which does some preparation and then
	217	+ * sets TIF_SVE.
	218	+ *
	219	+ * When stored, FPSIMD registers V0-V31 are encoded in
	220	+ * task->fpsimd_state; bits [max : 128] for each of Z0-Z31 are
	221	+ * logically zero but not stored anywhere; P0-P15 and FFR are not
	222	+ * stored and have unspecified values from userspace's point of
	223	+ * view. For hygiene purposes, the kernel zeroes them on next use,
	224	+ * but userspace is discouraged from relying on this.
	225	+ *
	226	+ * task->thread.sve_state does not need to be non-NULL, valid or any
	227	+ * particular size: it must not be dereferenced.
	228	+ *
	229	+ * * FPSR and FPCR are always stored in task->fpsimd_state irrespctive of
	230	+ * whether TIF_SVE is clear or set, since these are not vector length
	231	+ * dependent.
	232	+ */
	233	+
	234	+/*
	235	+ * Update current's FPSIMD/SVE registers from thread_struct.
	236	+ *
	237	+ * This function should be called only when the FPSIMD/SVE state in
	238	+ * thread_struct is known to be up to date, when preparing to enter
	239	+ * userspace.
	240	+ *
	241	+ * Softirqs (and preemption) must be disabled.
	242	+ */
	243	+static void task_fpsimd_load(void)
	244	+{
	245	+ WARN_ON(!in_softirq() && !irqs_disabled());
	246	+
	247	+ if (system_supports_sve() && test_thread_flag(TIF_SVE))
	248	+ sve_load_state(sve_pffr(current),
	249	+ &current->thread.fpsimd_state.fpsr,
	250	+ sve_vq_from_vl(current->thread.sve_vl) - 1);
	251	+ else
	252	+ fpsimd_load_state(&current->thread.fpsimd_state);
	253	+
	254	+ if (system_supports_sve()) {
	255	+ /* Toggle SVE trapping for userspace if needed */
	256	+ if (test_thread_flag(TIF_SVE))
	257	+ sve_user_enable();
	258	+ else
	259	+ sve_user_disable();
	260	+
	261	+ /* Serialised by exception return to user */
	262	+ }
	263	+}
	264	+
	265	+/*
	266	+ * Ensure current's FPSIMD/SVE storage in thread_struct is up to date
	267	+ * with respect to the CPU registers.
	268	+ *
	269	+ * Softirqs (and preemption) must be disabled.
	270	+ */
	271	+static void task_fpsimd_save(void)
	272	+{
	273	+ WARN_ON(!in_softirq() && !irqs_disabled());
	274	+
	275	+ if (!test_thread_flag(TIF_FOREIGN_FPSTATE)) {
	276	+ if (system_supports_sve() && test_thread_flag(TIF_SVE)) {
	277	+ if (WARN_ON(sve_get_vl() != current->thread.sve_vl)) {
	278	+ /*
	279	+ * Can't save the user regs, so current would
	280	+ * re-enter user with corrupt state.
	281	+ * There's no way to recover, so kill it:
	282	+ */
	283	+ force_signal_inject(
	284	+ SIGKILL, 0, current_pt_regs(), 0);
	285	+ return;
	286	+ }
	287	+
	288	+ sve_save_state(sve_pffr(current),
	289	+ &current->thread.fpsimd_state.fpsr);
	290	+ } else
	291	+ fpsimd_save_state(&current->thread.fpsimd_state);
	292	+ }
	293	+}
	294	+
	295	+/*
	296	+ * Helpers to translate bit indices in sve_vq_map to VQ values (and
	297	+ * vice versa). This allows find_next_bit() to be used to find the
	298	+ * _maximum_ VQ not exceeding a certain value.
	299	+ */
	300	+
	301	+static unsigned int vq_to_bit(unsigned int vq)
	302	+{
	303	+ return SVE_VQ_MAX - vq;
	304	+}
	305	+
	306	+static unsigned int bit_to_vq(unsigned int bit)
	307	+{
	308	+ if (WARN_ON(bit >= SVE_VQ_MAX))
	309	+ bit = SVE_VQ_MAX - 1;
	310	+
	311	+ return SVE_VQ_MAX - bit;
	312	+}
	313	+
	314	+/*
	315	+ * All vector length selection from userspace comes through here.
	316	+ * We're on a slow path, so some sanity-checks are included.
	317	+ * If things go wrong there's a bug somewhere, but try to fall back to a
	318	+ * safe choice.
	319	+ */
	320	+static unsigned int find_supported_vector_length(unsigned int vl)
	321	+{
	322	+ int bit;
	323	+ int max_vl = sve_max_vl;
	324	+
	325	+ if (WARN_ON(!sve_vl_valid(vl)))
	326	+ vl = SVE_VL_MIN;
	327	+
	328	+ if (WARN_ON(!sve_vl_valid(max_vl)))
	329	+ max_vl = SVE_VL_MIN;
	330	+
	331	+ if (vl > max_vl)
	332	+ vl = max_vl;
	333	+
	334	+ bit = find_next_bit(sve_vq_map, SVE_VQ_MAX,
	335	+ vq_to_bit(sve_vq_from_vl(vl)));
	336	+ return sve_vl_from_vq(bit_to_vq(bit));
	337	+}
	338	+
	339	+#ifdef CONFIG_SYSCTL
	340	+
	341	+static int sve_proc_do_default_vl(struct ctl_table *table, int write,
	342	+ void __user buffer, size_t lenp,
	343	+ loff_t *ppos)
	344	+{
	345	+ int ret;
	346	+ int vl = sve_default_vl;
	347	+ struct ctl_table tmp_table = {
	348	+ .data = &vl,
	349	+ .maxlen = sizeof(vl),
	350	+ };
	351	+
	352	+ ret = proc_dointvec(&tmp_table, write, buffer, lenp, ppos);
	353	+ if (ret \|\| !write)
	354	+ return ret;
	355	+
	356	+ /* Writing -1 has the special meaning "set to max": */
	357	+ if (vl == -1) {
	358	+ /* Fail safe if sve_max_vl wasn't initialised */
	359	+ if (WARN_ON(!sve_vl_valid(sve_max_vl)))
	360	+ vl = SVE_VL_MIN;
	361	+ else
	362	+ vl = sve_max_vl;
	363	+
	364	+ goto chosen;
	365	+ }
	366	+
	367	+ if (!sve_vl_valid(vl))
	368	+ return -EINVAL;
	369	+
	370	+ vl = find_supported_vector_length(vl);
	371	+chosen:
	372	+ sve_default_vl = vl;
	373	+ return 0;
	374	+}
	375	+
	376	+static struct ctl_table sve_default_vl_table[] = {
	377	+ {
	378	+ .procname = "sve_default_vector_length",
	379	+ .mode = 0644,
	380	+ .proc_handler = sve_proc_do_default_vl,
	381	+ },
	382	+ { }
	383	+};
	384	+
	385	+static int __init sve_sysctl_init(void)
	386	+{
	387	+ if (system_supports_sve())
	388	+ if (!register_sysctl("abi", sve_default_vl_table))
	389	+ return -EINVAL;
	390	+
	391	+ return 0;
	392	+}
	393	+
	394	+#else /* ! CONFIG_SYSCTL */
	395	+static int __init sve_sysctl_init(void) { return 0; }
	396	+#endif /* ! CONFIG_SYSCTL */
	397	+
	398	+#define ZREG(sve_state, vq, n) ((char *)(sve_state) + \
	399	+ (SVE_SIG_ZREG_OFFSET(vq, n) - SVE_SIG_REGS_OFFSET))
	400	+
	401	+/*
	402	+ * Transfer the FPSIMD state in task->thread.fpsimd_state to
	403	+ * task->thread.sve_state.
	404	+ *
	405	+ * Task can be a non-runnable task, or current. In the latter case,
	406	+ * softirqs (and preemption) must be disabled.
	407	+ * task->thread.sve_state must point to at least sve_state_size(task)
	408	+ * bytes of allocated kernel memory.
	409	+ * task->thread.fpsimd_state must be up to date before calling this function.
	410	+ */
	411	+static void fpsimd_to_sve(struct task_struct *task)
	412	+{
	413	+ unsigned int vq;
	414	+ void *sst = task->thread.sve_state;
	415	+ struct fpsimd_state const *fst = &task->thread.fpsimd_state;
	416	+ unsigned int i;
	417	+
	418	+ if (!system_supports_sve())
	419	+ return;
	420	+
	421	+ vq = sve_vq_from_vl(task->thread.sve_vl);
	422	+ for (i = 0; i < 32; ++i)
	423	+ memcpy(ZREG(sst, vq, i), &fst->vregs[i],
	424	+ sizeof(fst->vregs[i]));
	425	+}
	426	+
	427	+/*
	428	+ * Transfer the SVE state in task->thread.sve_state to
	429	+ * task->thread.fpsimd_state.
	430	+ *
	431	+ * Task can be a non-runnable task, or current. In the latter case,
	432	+ * softirqs (and preemption) must be disabled.
	433	+ * task->thread.sve_state must point to at least sve_state_size(task)
	434	+ * bytes of allocated kernel memory.
	435	+ * task->thread.sve_state must be up to date before calling this function.
	436	+ */
	437	+static void sve_to_fpsimd(struct task_struct *task)
	438	+{
	439	+ unsigned int vq;
	440	+ void const *sst = task->thread.sve_state;
	441	+ struct fpsimd_state *fst = &task->thread.fpsimd_state;
	442	+ unsigned int i;
	443	+
	444	+ if (!system_supports_sve())
	445	+ return;
	446	+
	447	+ vq = sve_vq_from_vl(task->thread.sve_vl);
	448	+ for (i = 0; i < 32; ++i)
	449	+ memcpy(&fst->vregs[i], ZREG(sst, vq, i),
	450	+ sizeof(fst->vregs[i]));
	451	+}
	452	+
	453	+#ifdef CONFIG_ARM64_SVE
	454	+
	455	+/*
	456	+ * Return how many bytes of memory are required to store the full SVE
	457	+ * state for task, given task's currently configured vector length.
	458	+ */
	459	+size_t sve_state_size(struct task_struct const *task)
	460	+{
	461	+ return SVE_SIG_REGS_SIZE(sve_vq_from_vl(task->thread.sve_vl));
	462	+}
	463	+
	464	+/*
	465	+ * Ensure that task->thread.sve_state is allocated and sufficiently large.
	466	+ *
	467	+ * This function should be used only in preparation for replacing
	468	+ * task->thread.sve_state with new data. The memory is always zeroed
	469	+ * here to prevent stale data from showing through: this is done in
	470	+ * the interest of testability and predictability: except in the
	471	+ * do_sve_acc() case, there is no ABI requirement to hide stale data
	472	+ * written previously be task.
	473	+ */
	474	+void sve_alloc(struct task_struct *task)
	475	+{
	476	+ if (task->thread.sve_state) {
	477	+ memset(task->thread.sve_state, 0, sve_state_size(current));
	478	+ return;
	479	+ }
	480	+
	481	+ /* This is a small allocation (maximum ~8KB) and Should Not Fail. */
	482	+ task->thread.sve_state =
	483	+ kzalloc(sve_state_size(task), GFP_KERNEL);
	484	+
	485	+ /*
	486	+ * If future SVE revisions can have larger vectors though,
	487	+ * this may cease to be true:
	488	+ */
	489	+ BUG_ON(!task->thread.sve_state);
	490	+}
	491	+
	492	+
	493	+/*
	494	+ * Ensure that task->thread.sve_state is up to date with respect to
	495	+ * the user task, irrespective of when SVE is in use or not.
	496	+ *
	497	+ * This should only be called by ptrace. task must be non-runnable.
	498	+ * task->thread.sve_state must point to at least sve_state_size(task)
	499	+ * bytes of allocated kernel memory.
	500	+ */
	501	+void fpsimd_sync_to_sve(struct task_struct *task)
	502	+{
	503	+ if (!test_tsk_thread_flag(task, TIF_SVE))
	504	+ fpsimd_to_sve(task);
	505	+}
	506	+
	507	+/*
	508	+ * Ensure that task->thread.fpsimd_state is up to date with respect to
	509	+ * the user task, irrespective of whether SVE is in use or not.
	510	+ *
	511	+ * This should only be called by ptrace. task must be non-runnable.
	512	+ * task->thread.sve_state must point to at least sve_state_size(task)
	513	+ * bytes of allocated kernel memory.
	514	+ */
	515	+void sve_sync_to_fpsimd(struct task_struct *task)
	516	+{
	517	+ if (test_tsk_thread_flag(task, TIF_SVE))
	518	+ sve_to_fpsimd(task);
	519	+}
	520	+
	521	+/*
	522	+ * Ensure that task->thread.sve_state is up to date with respect to
	523	+ * the task->thread.fpsimd_state.
	524	+ *
	525	+ * This should only be called by ptrace to merge new FPSIMD register
	526	+ * values into a task for which SVE is currently active.
	527	+ * task must be non-runnable.
	528	+ * task->thread.sve_state must point to at least sve_state_size(task)
	529	+ * bytes of allocated kernel memory.
	530	+ * task->thread.fpsimd_state must already have been initialised with
	531	+ * the new FPSIMD register values to be merged in.
	532	+ */
	533	+void sve_sync_from_fpsimd_zeropad(struct task_struct *task)
	534	+{
	535	+ unsigned int vq;
	536	+ void *sst = task->thread.sve_state;
	537	+ struct fpsimd_state const *fst = &task->thread.fpsimd_state;
	538	+ unsigned int i;
	539	+
	540	+ if (!test_tsk_thread_flag(task, TIF_SVE))
	541	+ return;
	542	+
	543	+ vq = sve_vq_from_vl(task->thread.sve_vl);
	544	+
	545	+ memset(sst, 0, SVE_SIG_REGS_SIZE(vq));
	546	+
	547	+ for (i = 0; i < 32; ++i)
	548	+ memcpy(ZREG(sst, vq, i), &fst->vregs[i],
	549	+ sizeof(fst->vregs[i]));
	550	+}
	551	+
	552	+int sve_set_vector_length(struct task_struct *task,
	553	+ unsigned long vl, unsigned long flags)
	554	+{
	555	+ if (flags & ~(unsigned long)(PR_SVE_VL_INHERIT \|
	556	+ PR_SVE_SET_VL_ONEXEC))
	557	+ return -EINVAL;
	558	+
	559	+ if (!sve_vl_valid(vl))
	560	+ return -EINVAL;
	561	+
	562	+ /*
	563	+ * Clamp to the maximum vector length that VL-agnostic SVE code can
	564	+ * work with. A flag may be assigned in the future to allow setting
	565	+ * of larger vector lengths without confusing older software.
	566	+ */
	567	+ if (vl > SVE_VL_ARCH_MAX)
	568	+ vl = SVE_VL_ARCH_MAX;
	569	+
	570	+ vl = find_supported_vector_length(vl);
	571	+
	572	+ if (flags & (PR_SVE_VL_INHERIT \|
	573	+ PR_SVE_SET_VL_ONEXEC))
	574	+ task->thread.sve_vl_onexec = vl;
	575	+ else
	576	+ /* Reset VL to system default on next exec: */
	577	+ task->thread.sve_vl_onexec = 0;
	578	+
	579	+ /* Only actually set the VL if not deferred: */
	580	+ if (flags & PR_SVE_SET_VL_ONEXEC)
	581	+ goto out;
	582	+
	583	+ if (vl == task->thread.sve_vl)
	584	+ goto out;
	585	+
	586	+ /*
	587	+ * To ensure the FPSIMD bits of the SVE vector registers are preserved,
	588	+ * write any live register state back to task_struct, and convert to a
	589	+ * non-SVE thread.
	590	+ */
	591	+ if (task == current) {
	592	+ local_bh_disable();
	593	+
	594	+ task_fpsimd_save();
	595	+ set_thread_flag(TIF_FOREIGN_FPSTATE);
	596	+ }
	597	+
	598	+ fpsimd_flush_task_state(task);
	599	+ if (test_and_clear_tsk_thread_flag(task, TIF_SVE))
	600	+ sve_to_fpsimd(task);
	601	+
	602	+ if (task == current)
	603	+ local_bh_enable();
	604	+
	605	+ /*
	606	+ * Force reallocation of task SVE state to the correct size
	607	+ * on next use:
	608	+ */
	609	+ sve_free(task);
	610	+
	611	+ task->thread.sve_vl = vl;
	612	+
	613	+out:
	614	+ if (flags & PR_SVE_VL_INHERIT)
	615	+ set_tsk_thread_flag(task, TIF_SVE_VL_INHERIT);
	616	+ else
	617	+ clear_tsk_thread_flag(task, TIF_SVE_VL_INHERIT);
	618	+
	619	+ return 0;
	620	+}
	621	+
	622	+/*
	623	+ * Encode the current vector length and flags for return.
	624	+ * This is only required for prctl(): ptrace has separate fields
	625	+ *
	626	+ * flags are as for sve_set_vector_length().
	627	+ */
	628	+static int sve_prctl_status(unsigned long flags)
	629	+{
	630	+ int ret;
	631	+
	632	+ if (flags & PR_SVE_SET_VL_ONEXEC)
	633	+ ret = current->thread.sve_vl_onexec;
	634	+ else
	635	+ ret = current->thread.sve_vl;
	636	+
	637	+ if (test_thread_flag(TIF_SVE_VL_INHERIT))
	638	+ ret \|= PR_SVE_VL_INHERIT;
	639	+
	640	+ return ret;
	641	+}
	642	+
	643	+/* PR_SVE_SET_VL */
	644	+int sve_set_current_vl(unsigned long arg)
	645	+{
	646	+ unsigned long vl, flags;
	647	+ int ret;
	648	+
	649	+ vl = arg & PR_SVE_VL_LEN_MASK;
	650	+ flags = arg & ~vl;
	651	+
	652	+ if (!system_supports_sve())
	653	+ return -EINVAL;
	654	+
	655	+ ret = sve_set_vector_length(current, vl, flags);
	656	+ if (ret)
	657	+ return ret;
	658	+
	659	+ return sve_prctl_status(flags);
	660	+}
	661	+
	662	+/* PR_SVE_GET_VL */
	663	+int sve_get_current_vl(void)
	664	+{
	665	+ if (!system_supports_sve())
	666	+ return -EINVAL;
	667	+
	668	+ return sve_prctl_status(0);
	669	+}
	670	+
	671	+/*
	672	+ * Bitmap for temporary storage of the per-CPU set of supported vector lengths
	673	+ * during secondary boot.
	674	+ */
	675	+static DECLARE_BITMAP(sve_secondary_vq_map, SVE_VQ_MAX);
	676	+
	677	+static void sve_probe_vqs(DECLARE_BITMAP(map, SVE_VQ_MAX))
	678	+{
	679	+ unsigned int vq, vl;
	680	+ unsigned long zcr;
	681	+
	682	+ bitmap_zero(map, SVE_VQ_MAX);
	683	+
	684	+ zcr = ZCR_ELx_LEN_MASK;
	685	+ zcr = read_sysreg_s(SYS_ZCR_EL1) & ~zcr;
	686	+
	687	+ for (vq = SVE_VQ_MAX; vq >= SVE_VQ_MIN; --vq) {
	688	+ write_sysreg_s(zcr \| (vq - 1), SYS_ZCR_EL1); /* self-syncing */
	689	+ vl = sve_get_vl();
	690	+ vq = sve_vq_from_vl(vl); /* skip intervening lengths */
	691	+ set_bit(vq_to_bit(vq), map);
	692	+ }
	693	+}
	694	+
	695	+void __init sve_init_vq_map(void)
	696	+{
	697	+ sve_probe_vqs(sve_vq_map);
	698	+}
	699	+
	700	+/*
	701	+ * If we haven't committed to the set of supported VQs yet, filter out
	702	+ * those not supported by the current CPU.
	703	+ */
	704	+void sve_update_vq_map(void)
	705	+{
	706	+ sve_probe_vqs(sve_secondary_vq_map);
	707	+ bitmap_and(sve_vq_map, sve_vq_map, sve_secondary_vq_map, SVE_VQ_MAX);
	708	+}
	709	+
	710	+/* Check whether the current CPU supports all VQs in the committed set */
	711	+int sve_verify_vq_map(void)
	712	+{
	713	+ int ret = 0;
	714	+
	715	+ sve_probe_vqs(sve_secondary_vq_map);
	716	+ bitmap_andnot(sve_secondary_vq_map, sve_vq_map, sve_secondary_vq_map,
	717	+ SVE_VQ_MAX);
	718	+ if (!bitmap_empty(sve_secondary_vq_map, SVE_VQ_MAX)) {
	719	+ pr_warn("SVE: cpu%d: Required vector length(s) missing\n",
	720	+ smp_processor_id());
	721	+ ret = -EINVAL;
	722	+ }
	723	+
	724	+ return ret;
	725	+}
	726	+
	727	+static void __init sve_efi_setup(void)
	728	+{
	729	+ if (!IS_ENABLED(CONFIG_EFI))
	730	+ return;
	731	+
	732	+ /*
	733	+ * alloc_percpu() warns and prints a backtrace if this goes wrong.
	734	+ * This is evidence of a crippled system and we are returning void,
	735	+ * so no attempt is made to handle this situation here.
	736	+ */
	737	+ if (!sve_vl_valid(sve_max_vl))
	738	+ goto fail;
	739	+
	740	+ efi_sve_state = __alloc_percpu(
	741	+ SVE_SIG_REGS_SIZE(sve_vq_from_vl(sve_max_vl)), SVE_VQ_BYTES);
	742	+ if (!efi_sve_state)
	743	+ goto fail;
	744	+
	745	+ return;
	746	+
	747	+fail:
	748	+ panic("Cannot allocate percpu memory for EFI SVE save/restore");
	749	+}
	750	+
	751	+/*
	752	+ * Enable SVE for EL1.
	753	+ * Intended for use by the cpufeatures code during CPU boot.
	754	+ */
	755	+int sve_kernel_enable(void *__always_unused p)
	756	+{
	757	+ write_sysreg(read_sysreg(CPACR_EL1) \| CPACR_EL1_ZEN_EL1EN, CPACR_EL1);
	758	+ isb();
	759	+
	760	+ return 0;
	761	+}
	762	+
	763	+void __init sve_setup(void)
	764	+{
	765	+ u64 zcr;
	766	+
	767	+ if (!system_supports_sve())
	768	+ return;
	769	+
	770	+ /*
	771	+ * The SVE architecture mandates support for 128-bit vectors,
	772	+ * so sve_vq_map must have at least SVE_VQ_MIN set.
	773	+ * If something went wrong, at least try to patch it up:
	774	+ */
	775	+ if (WARN_ON(!test_bit(vq_to_bit(SVE_VQ_MIN), sve_vq_map)))
	776	+ set_bit(vq_to_bit(SVE_VQ_MIN), sve_vq_map);
	777	+
	778	+ zcr = read_sanitised_ftr_reg(SYS_ZCR_EL1);
	779	+ sve_max_vl = sve_vl_from_vq((zcr & ZCR_ELx_LEN_MASK) + 1);
	780	+
	781	+ /*
	782	+ * Sanity-check that the max VL we determined through CPU features
	783	+ * corresponds properly to sve_vq_map. If not, do our best:
	784	+ */
	785	+ if (WARN_ON(sve_max_vl != find_supported_vector_length(sve_max_vl)))
	786	+ sve_max_vl = find_supported_vector_length(sve_max_vl);
	787	+
	788	+ /*
	789	+ * For the default VL, pick the maximum supported value <= 64.
	790	+ * VL == 64 is guaranteed not to grow the signal frame.
	791	+ */
	792	+ sve_default_vl = find_supported_vector_length(64);
	793	+
	794	+ pr_info("SVE: maximum available vector length %u bytes per vector\n",
	795	+ sve_max_vl);
	796	+ pr_info("SVE: default vector length %u bytes per vector\n",
	797	+ sve_default_vl);
	798	+
	799	+ sve_efi_setup();
	800	+}
	801	+
	802	+/*
	803	+ * Called from the put_task_struct() path, which cannot get here
	804	+ * unless dead_task is really dead and not schedulable.
	805	+ */
	806	+void fpsimd_release_task(struct task_struct *dead_task)
	807	+{
	808	+ __sve_free(dead_task);
	809	+}
	810	+
	811	+#endif /* CONFIG_ARM64_SVE */
	812	+
	813	+/*
	814	+ * Trapped SVE access
	815	+ *
	816	+ * Storage is allocated for the full SVE state, the current FPSIMD
	817	+ * register contents are migrated across, and TIF_SVE is set so that
	818	+ * the SVE access trap will be disabled the next time this task
	819	+ * reaches ret_to_user.
	820	+ *
	821	+ * TIF_SVE should be clear on entry: otherwise, task_fpsimd_load()
	822	+ * would have disabled the SVE access trap for userspace during
	823	+ * ret_to_user, making an SVE access trap impossible in that case.
	824	+ */
	825	+asmlinkage void do_sve_acc(unsigned int esr, struct pt_regs *regs)
	826	+{
	827	+ /* Even if we chose not to use SVE, the hardware could still trap: */
	828	+ if (unlikely(!system_supports_sve()) \|\| WARN_ON(is_compat_task())) {
	829	+ force_signal_inject(SIGILL, ILL_ILLOPC, regs, 0);
	830	+ return;
	831	+ }
	832	+
	833	+ sve_alloc(current);
	834	+
	835	+ local_bh_disable();
	836	+
	837	+ task_fpsimd_save();
	838	+ fpsimd_to_sve(current);
	839	+
	840	+ /* Force ret_to_user to reload the registers: */
	841	+ fpsimd_flush_task_state(current);
	842	+ set_thread_flag(TIF_FOREIGN_FPSTATE);
	843	+
	844	+ if (test_and_set_thread_flag(TIF_SVE))
	845	+ WARN_ON(1); /* SVE access shouldn't have trapped */
	846	+
	847	+ local_bh_enable();
	848	+}
	849	+
102	850	/*
103	851	* Trapped FP/ASIMD access.
104	852	*/
105		−void do_fpsimd_acc(unsigned int esr, struct pt_regs *regs)
	853	+asmlinkage void do_fpsimd_acc(unsigned int esr, struct pt_regs *regs)
106	854	{
107	855	/* TODO: implement lazy context saving/restoring */
108	856	WARN_ON(1);
@@ -111,7 +859,7 @@ void do_fpsimd_acc(unsigned int esr, struct pt_regs *regs) · 2 unmodified lines
111	859	/*
112	860	* Raise a SIGFPE for the current process.
113	861	*/
114		−void do_fpsimd_exc(unsigned int esr, struct pt_regs *regs)
	862	+asmlinkage void do_fpsimd_exc(unsigned int esr, struct pt_regs *regs)
115	863	{
116	864	siginfo_t info;
117	865	unsigned int si_code = 0;
@@ -144,8 +892,8 @@ void fpsimd_thread_switch(struct task_struct *next) · 26 unmodified lines
144	892	* the registers is in fact the most recent userland FPSIMD state of
145	893	* 'current'.
146	894	*/
147		− if (current->mm && !test_thread_flag(TIF_FOREIGN_FPSTATE))
148		− fpsimd_save_state(&current->thread.fpsimd_state);
	895	+ if (current->mm)
	896	+ task_fpsimd_save();
149	897
150	898	if (next->mm) {
151	899	/*
@@ -159,16 +907,16 @@ void fpsimd_thread_switch(struct task_struct *next) · 7 unmodified lines
159	907
160	908	if (__this_cpu_read(fpsimd_last_state) == st
161	909	&& st->cpu == smp_processor_id())
162		− clear_ti_thread_flag(task_thread_info(next),
163		− TIF_FOREIGN_FPSTATE);
	910	+ clear_tsk_thread_flag(next, TIF_FOREIGN_FPSTATE);
164	911	else
165		− set_ti_thread_flag(task_thread_info(next),
166		− TIF_FOREIGN_FPSTATE);
	912	+ set_tsk_thread_flag(next, TIF_FOREIGN_FPSTATE);
167	913	}
168	914	}
169	915
170	916	void fpsimd_flush_thread(void)
171	917	{
	918	+ int vl, supported_vl;
	919	+
172	920	if (!system_supports_fpsimd())
173	921	return;
174	922
@@ -176,6 +924,42 @@ void fpsimd_flush_thread(void) · 1 unmodified line
176	924
177	925	memset(&current->thread.fpsimd_state, 0, sizeof(struct fpsimd_state));
178	926	fpsimd_flush_task_state(current);
	927	+
	928	+ if (system_supports_sve()) {
	929	+ clear_thread_flag(TIF_SVE);
	930	+ sve_free(current);
	931	+
	932	+ /*
	933	+ * Reset the task vector length as required.
	934	+ * This is where we ensure that all user tasks have a valid
	935	+ * vector length configured: no kernel task can become a user
	936	+ * task without an exec and hence a call to this function.
	937	+ * By the time the first call to this function is made, all
	938	+ * early hardware probing is complete, so sve_default_vl
	939	+ * should be valid.
	940	+ * If a bug causes this to go wrong, we make some noise and
	941	+ * try to fudge thread.sve_vl to a safe value here.
	942	+ */
	943	+ vl = current->thread.sve_vl_onexec ?
	944	+ current->thread.sve_vl_onexec : sve_default_vl;
	945	+
	946	+ if (WARN_ON(!sve_vl_valid(vl)))
	947	+ vl = SVE_VL_MIN;
	948	+
	949	+ supported_vl = find_supported_vector_length(vl);
	950	+ if (WARN_ON(supported_vl != vl))
	951	+ vl = supported_vl;
	952	+
	953	+ current->thread.sve_vl = vl;
	954	+
	955	+ /*
	956	+ * If the task is not set to inherit, ensure that the vector
	957	+ * length will be reset by a subsequent exec:
	958	+ */
	959	+ if (!test_thread_flag(TIF_SVE_VL_INHERIT))
	960	+ current->thread.sve_vl_onexec = 0;
	961	+ }
	962	+
179	963	set_thread_flag(TIF_FOREIGN_FPSTATE);
180	964
181	965	local_bh_enable();
@@ -191,13 +975,22 @@ void fpsimd_preserve_current_state(void) · 9 unmodified lines
191	975	return;
192	976
193	977	local_bh_disable();
194		−
195		− if (!test_thread_flag(TIF_FOREIGN_FPSTATE))
196		− fpsimd_save_state(&current->thread.fpsimd_state);
197		−
	978	+ task_fpsimd_save();
198	979	local_bh_enable();
199	980	}
200	981
	982	+/*
	983	+ * Like fpsimd_preserve_current_state(), but ensure that
	984	+ * current->thread.fpsimd_state is updated so that it can be copied to
	985	+ * the signal frame.
	986	+ */
	987	+void fpsimd_signal_preserve_current_state(void)
	988	+{
	989	+ fpsimd_preserve_current_state();
	990	+ if (system_supports_sve() && test_thread_flag(TIF_SVE))
	991	+ sve_to_fpsimd(current);
	992	+}
	993	+
201	994	/*
202	995	* Load the userland FPSIMD state of 'current' from memory, but only if the
203	996	* FPSIMD state already held in the registers is /not/ the most recent FPSIMD
@@ -213,7 +1006,7 @@ void fpsimd_restore_current_state(void) · 9 unmodified lines
213	1006	if (test_and_clear_thread_flag(TIF_FOREIGN_FPSTATE)) {
214	1007	struct fpsimd_state *st = &current->thread.fpsimd_state;
215	1008
216		− fpsimd_load_state(st);
	1009	+ task_fpsimd_load();
217	1010	__this_cpu_write(fpsimd_last_state, st);
218	1011	st->cpu = smp_processor_id();
219	1012	}
@@ -233,7 +1026,12 @@ void fpsimd_update_current_state(struct fpsimd_state *state) · 13 unmodified lines
233	1026
234	1027	local_bh_disable();
235	1028
236		− fpsimd_load_state(state);
	1029	+ if (system_supports_sve() && test_thread_flag(TIF_SVE)) {
	1030	+ current->thread.fpsimd_state = *state;
	1031	+ fpsimd_to_sve(current);
	1032	+ }
	1033	+ task_fpsimd_load();
	1034	+
237	1035	if (test_and_clear_thread_flag(TIF_FOREIGN_FPSTATE)) {
238	1036	struct fpsimd_state *st = &current->thread.fpsimd_state;
239	1037
@@ -252,6 +1050,33 @@ void fpsimd_flush_task_state(struct task_struct *t) · 12 unmodified lines
252	1050	t->thread.fpsimd_state.cpu = NR_CPUS;
253	1051	}
254	1052
	1053	+static inline void fpsimd_flush_cpu_state(void)
	1054	+{
	1055	+ __this_cpu_write(fpsimd_last_state, NULL);
	1056	+}
	1057	+
	1058	+/*
	1059	+ * Invalidate any task SVE state currently held in this CPU's regs.
	1060	+ *
	1061	+ * This is used to prevent the kernel from trying to reuse SVE register data
	1062	+ * that is detroyed by KVM guest enter/exit. This function should go away when
	1063	+ * KVM SVE support is implemented. Don't use it for anything else.
	1064	+ */
	1065	+#ifdef CONFIG_ARM64_SVE
	1066	+void sve_flush_cpu_state(void)
	1067	+{
	1068	+ struct fpsimd_state *const fpstate = __this_cpu_read(fpsimd_last_state);
	1069	+ struct task_struct *tsk;
	1070	+
	1071	+ if (!fpstate)
	1072	+ return;
	1073	+
	1074	+ tsk = container_of(fpstate, struct task_struct, thread.fpsimd_state);
	1075	+ if (test_tsk_thread_flag(tsk, TIF_SVE))
	1076	+ fpsimd_flush_cpu_state();
	1077	+}
	1078	+#endif /* CONFIG_ARM64_SVE */
	1079	+
255	1080	#ifdef CONFIG_KERNEL_MODE_NEON
256	1081
257	1082	DEFINE_PER_CPU(bool, kernel_neon_busy);
@@ -286,11 +1111,13 @@ void kernel_neon_begin(void) · 28 unmodified lines
286	1111	__this_cpu_write(kernel_neon_busy, true);
287	1112
288	1113	/* Save unsaved task fpsimd state, if any: */
289		− if (current->mm && !test_and_set_thread_flag(TIF_FOREIGN_FPSTATE))
290		− fpsimd_save_state(&current->thread.fpsimd_state);
	1114	+ if (current->mm) {
	1115	+ task_fpsimd_save();
	1116	+ set_thread_flag(TIF_FOREIGN_FPSTATE);
	1117	+ }
291	1118
292	1119	/* Invalidate any task state remaining in the fpsimd regs: */
293		− __this_cpu_write(fpsimd_last_state, NULL);
	1120	+ fpsimd_flush_cpu_state();
294	1121
295	1122	preempt_disable();
296	1123
@@ -325,6 +1152,7 @@ EXPORT_SYMBOL(kernel_neon_end); · 28 unmodified lines
325	1152
326	1153	static DEFINE_PER_CPU(struct fpsimd_state, efi_fpsimd_state);
327	1154	static DEFINE_PER_CPU(bool, efi_fpsimd_state_used);
	1155	+static DEFINE_PER_CPU(bool, efi_sve_state_used);
328	1156
329	1157	/*
330	1158	* EFI runtime services support functions
@@ -350,10 +1178,24 @@ void __efi_fpsimd_begin(void) · 19 unmodified lines
350	1178
351	1179	WARN_ON(preemptible());
352	1180
353		− if (may_use_simd())
	1181	+ if (may_use_simd()) {
354	1182	kernel_neon_begin();
355		− else {
356		− fpsimd_save_state(this_cpu_ptr(&efi_fpsimd_state));
	1183	+ } else {
	1184	+ /*
	1185	+ * If !efi_sve_state, SVE can't be in use yet and doesn't need
	1186	+ * preserving:
	1187	+ */
	1188	+ if (system_supports_sve() && likely(efi_sve_state)) {
	1189	+ char *sve_state = this_cpu_ptr(efi_sve_state);
	1190	+
	1191	+ __this_cpu_write(efi_sve_state_used, true);
	1192	+
	1193	+ sve_save_state(sve_state + sve_ffr_offset(sve_max_vl),
	1194	+ &this_cpu_ptr(&efi_fpsimd_state)->fpsr);
	1195	+ } else {
	1196	+ fpsimd_save_state(this_cpu_ptr(&efi_fpsimd_state));
	1197	+ }
	1198	+
357	1199	__this_cpu_write(efi_fpsimd_state_used, true);
358	1200	}
359	1201	}
@@ -366,10 +1208,22 @@ void __efi_fpsimd_end(void) · 6 unmodified lines
366	1208	if (!system_supports_fpsimd())
367	1209	return;
368	1210
369		− if (__this_cpu_xchg(efi_fpsimd_state_used, false))
370		− fpsimd_load_state(this_cpu_ptr(&efi_fpsimd_state));
371		− else
	1211	+ if (!__this_cpu_xchg(efi_fpsimd_state_used, false)) {
372	1212	kernel_neon_end();
	1213	+ } else {
	1214	+ if (system_supports_sve() &&
	1215	+ likely(__this_cpu_read(efi_sve_state_used))) {
	1216	+ char const *sve_state = this_cpu_ptr(efi_sve_state);
	1217	+
	1218	+ sve_load_state(sve_state + sve_ffr_offset(sve_max_vl),
	1219	+ &this_cpu_ptr(&efi_fpsimd_state)->fpsr,
	1220	+ sve_vq_from_vl(sve_get_vl()) - 1);
	1221	+
	1222	+ __this_cpu_write(efi_sve_state_used, false);
	1223	+ } else {
	1224	+ fpsimd_load_state(this_cpu_ptr(&efi_fpsimd_state));
	1225	+ }
	1226	+ }
373	1227	}
374	1228
375	1229	#endif /* CONFIG_EFI */
@@ -382,9 +1236,9 @@ static int fpsimd_cpu_pm_notifier(struct notifier_block *self, · 6 unmodified lines
382	1236	{
383	1237	switch (cmd) {
384	1238	case CPU_PM_ENTER:
385		− if (current->mm && !test_thread_flag(TIF_FOREIGN_FPSTATE))
386		− fpsimd_save_state(&current->thread.fpsimd_state);
387		− this_cpu_write(fpsimd_last_state, NULL);
	1239	+ if (current->mm)
	1240	+ task_fpsimd_save();
	1241	+ fpsimd_flush_cpu_state();
388	1242	break;
389	1243	case CPU_PM_EXIT:
390	1244	if (current->mm)
@@ -442,6 +1296,6 @@ static int __init fpsimd_init(void) · 51 unmodified lines
442	1296	if (!(elf_hwcap & HWCAP_ASIMD))
443	1297	pr_notice("Advanced SIMD is not implemented\n");
444	1298
445		− return 0;
	1299	+ return sve_sysctl_init();
446	1300	}
447	1301	core_initcall(fpsimd_init);

arch/arm64/kernel/head.S+24−6

@@ -480,14 +480,21 @@ set_hcr: · 479 unmodified lines
480	480
481	481	/* Statistical profiling */
482	482	ubfx x0, x1, #32, #4 // Check ID_AA64DFR0_EL1 PMSVer
483		− cbz x0, 6f // Skip if SPE not present
484		− cbnz x2, 5f // VHE?
	483	+ cbz x0, 7f // Skip if SPE not present
	484	+ cbnz x2, 6f // VHE?
	485	+ mrs_s x4, SYS_PMBIDR_EL1 // If SPE available at EL2,
	486	+ and x4, x4, #(1 << SYS_PMBIDR_EL1_P_SHIFT)
	487	+ cbnz x4, 5f // then permit sampling of physical
	488	+ mov x4, #(1 << SYS_PMSCR_EL2_PCT_SHIFT \| \
	489	+ 1 << SYS_PMSCR_EL2_PA_SHIFT)
	490	+ msr_s SYS_PMSCR_EL2, x4 // addresses and physical counter
	491	+5:
485	492	mov x1, #(MDCR_EL2_E2PB_MASK << MDCR_EL2_E2PB_SHIFT)
486	493	orr x3, x3, x1 // If we don't have VHE, then
487		− b 6f // use EL1&0 translation.
488		−5: // For VHE, use EL2 translation
	494	+ b 7f // use EL1&0 translation.
	495	+6: // For VHE, use EL2 translation
489	496	orr x3, x3, #MDCR_EL2_TPMS // and disable access from EL1
490		−6:
	497	+7:
491	498	msr mdcr_el2, x3 // Configure debug traps
492	499
493	500	/* Stage-2 translation */
@@ -517,8 +524,19 @@ CPU_LE( movk x0, #0x30d0, lsl #16 ) // Clear EE and E0E on LE systems · 23 unmodified lines
517	524	mov x0, #0x33ff
518	525	msr cptr_el2, x0 // Disable copro. traps to EL2
519	526
	527	+ /* SVE register access */
	528	+ mrs x1, id_aa64pfr0_el1
	529	+ ubfx x1, x1, #ID_AA64PFR0_SVE_SHIFT, #4
	530	+ cbz x1, 7f
	531	+
	532	+ bic x0, x0, #CPTR_EL2_TZ // Also disable SVE traps
	533	+ msr cptr_el2, x0 // Disable copro. traps to EL2
	534	+ isb
	535	+ mov x1, #ZCR_ELx_LEN_MASK // SVE: Enable full vector
	536	+ msr_s SYS_ZCR_EL2, x1 // length for EL1.
	537	+
520	538	/* Hypervisor stub */
521		− adr_l x0, __hyp_stub_vectors
	539	+7: adr_l x0, __hyp_stub_vectors
522	540	msr vbar_el2, x0
523	541
524	542	/* spsr */

arch/arm64/kernel/hibernate.c+3−2

@@ -27,6 +27,7 @@ · 26 unmodified lines
27	27	#include <asm/barrier.h>
28	28	#include <asm/cacheflush.h>
29	29	#include <asm/cputype.h>
	30	+#include <asm/daifflags.h>
30	31	#include <asm/irqflags.h>
31	32	#include <asm/kexec.h>
32	33	#include <asm/memory.h>
@@ -285,7 +286,7 @@ int swsusp_arch_suspend(void) · 252 unmodified lines
285	286	return -EBUSY;
286	287	}
287	288
288		− local_dbg_save(flags);
	289	+ flags = local_daif_save();
289	290
290	291	if (__cpu_suspend_enter(&state)) {
291	292	/* make the crash dump kernel image visible/saveable */
@@ -315,7 +316,7 @@ int swsusp_arch_suspend(void) · 23 unmodified lines
315	316	__cpu_suspend_exit();
316	317	}
317	318
318		− local_dbg_restore(flags);
	319	+ local_daif_restore(flags);
319	320
320	321	return ret;
321	322	}

arch/arm64/kernel/io.c+5−7

@@ -25,8 +25,7 @@ · 24 unmodified lines
25	25	*/
26	26	void __memcpy_fromio(void to, const volatile void __iomem from, size_t count)
27	27	{
28		− while (count && (!IS_ALIGNED((unsigned long)from, 8) \|\|
29		− !IS_ALIGNED((unsigned long)to, 8))) {
	28	+ while (count && !IS_ALIGNED((unsigned long)from, 8)) {
30	29	(u8 )to = __raw_readb(from);
31	30	from++;
32	31	to++;
@@ -54,23 +53,22 @@ EXPORT_SYMBOL(__memcpy_fromio); · 21 unmodified lines
54	53	*/
55	54	void __memcpy_toio(volatile void __iomem to, const void from, size_t count)
56	55	{
57		− while (count && (!IS_ALIGNED((unsigned long)to, 8) \|\|
58		− !IS_ALIGNED((unsigned long)from, 8))) {
59		− __raw_writeb((volatile u8 )from, to);
	56	+ while (count && !IS_ALIGNED((unsigned long)to, 8)) {
	57	+ __raw_writeb((u8 )from, to);
60	58	from++;
61	59	to++;
62	60	count--;
63	61	}
64	62
65	63	while (count >= 8) {
66		− __raw_writeq((volatile u64 )from, to);
	64	+ __raw_writeq((u64 )from, to);
67	65	from += 8;
68	66	to += 8;
69	67	count -= 8;
70	68	}
71	69
72	70	while (count) {
73		− __raw_writeb((volatile u8 )from, to);
	71	+ __raw_writeb((u8 )from, to);
74	72	from++;
75	73	to++;
76	74	count--;

arch/arm64/kernel/machine_kexec.c+2−2

@@ -18,6 +18,7 @@ · 17 unmodified lines
18	18
19	19	#include <asm/cacheflush.h>
20	20	#include <asm/cpu_ops.h>
	21	+#include <asm/daifflags.h>
21	22	#include <asm/memory.h>
22	23	#include <asm/mmu.h>
23	24	#include <asm/mmu_context.h>
@@ -195,8 +196,7 @@ void machine_kexec(struct kimage *kimage) · 171 unmodified lines
195	196
196	197	pr_info("Bye!\n");
197	198
198		− /* Disable all DAIF exceptions. */
199		− asm volatile ("msr daifset, #0xf" : : : "memory");
	199	+ local_daif_mask();
200	200
201	201	/*
202	202	* cpu_soft_restart will shutdown the MMU, disable data caches, then

arch/arm64/kernel/process.c+60−4

@@ -49,6 +49,7 @@ · 48 unmodified lines
49	49	#include <linux/notifier.h>
50	50	#include <trace/events/power.h>
51	51	#include <linux/percpu.h>
	52	+#include <linux/thread_info.h>
52	53
53	54	#include <asm/alternative.h>
54	55	#include <asm/compat.h>
@@ -170,6 +171,39 @@ void machine_restart(char *cmd) · 115 unmodified lines
170	171	while (1);
171	172	}
172	173
	174	+static void print_pstate(struct pt_regs *regs)
	175	+{
	176	+ u64 pstate = regs->pstate;
	177	+
	178	+ if (compat_user_mode(regs)) {
	179	+ printk("pstate: %08llx (%c%c%c%c %c %s %s %c%c%c)\n",
	180	+ pstate,
	181	+ pstate & COMPAT_PSR_N_BIT ? 'N' : 'n',
	182	+ pstate & COMPAT_PSR_Z_BIT ? 'Z' : 'z',
	183	+ pstate & COMPAT_PSR_C_BIT ? 'C' : 'c',
	184	+ pstate & COMPAT_PSR_V_BIT ? 'V' : 'v',
	185	+ pstate & COMPAT_PSR_Q_BIT ? 'Q' : 'q',
	186	+ pstate & COMPAT_PSR_T_BIT ? "T32" : "A32",
	187	+ pstate & COMPAT_PSR_E_BIT ? "BE" : "LE",
	188	+ pstate & COMPAT_PSR_A_BIT ? 'A' : 'a',
	189	+ pstate & COMPAT_PSR_I_BIT ? 'I' : 'i',
	190	+ pstate & COMPAT_PSR_F_BIT ? 'F' : 'f');
	191	+ } else {
	192	+ printk("pstate: %08llx (%c%c%c%c %c%c%c%c %cPAN %cUAO)\n",
	193	+ pstate,
	194	+ pstate & PSR_N_BIT ? 'N' : 'n',
	195	+ pstate & PSR_Z_BIT ? 'Z' : 'z',
	196	+ pstate & PSR_C_BIT ? 'C' : 'c',
	197	+ pstate & PSR_V_BIT ? 'V' : 'v',
	198	+ pstate & PSR_D_BIT ? 'D' : 'd',
	199	+ pstate & PSR_A_BIT ? 'A' : 'a',
	200	+ pstate & PSR_I_BIT ? 'I' : 'i',
	201	+ pstate & PSR_F_BIT ? 'F' : 'f',
	202	+ pstate & PSR_PAN_BIT ? '+' : '-',
	203	+ pstate & PSR_UAO_BIT ? '+' : '-');
	204	+ }
	205	+}
	206	+
173	207	void __show_regs(struct pt_regs *regs)
174	208	{
175	209	int i, top_reg;
@@ -186,10 +220,9 @@ void __show_regs(struct pt_regs *regs) · 10 unmodified lines
186	220	}
187	221
188	222	show_regs_print_info(KERN_DEFAULT);
189		− print_symbol("PC is at %s\n", instruction_pointer(regs));
190		− print_symbol("LR is at %s\n", lr);
191		− printk("pc : [<%016llx>] lr : [<%016llx>] pstate: %08llx\n",
192		− regs->pc, lr, regs->pstate);
	223	+ print_pstate(regs);
	224	+ print_symbol("pc : %s\n", regs->pc);
	225	+ print_symbol("lr : %s\n", lr);
193	226	printk("sp : %016llx\n", sp);
194	227
195	228	i = top_reg;
@@ -241,11 +274,27 @@ void release_thread(struct task_struct *dead_task) · 45 unmodified lines
241	274	{
242	275	}
243	276
	277	+void arch_release_task_struct(struct task_struct *tsk)
	278	+{
	279	+ fpsimd_release_task(tsk);
	280	+}
	281	+
	282	+/*
	283	+ * src and dst may temporarily have aliased sve_state after task_struct
	284	+ * is copied. We cannot fix this properly here, because src may have
	285	+ * live SVE state and dst's thread_info may not exist yet, so tweaking
	286	+ * either src's or dst's TIF_SVE is not safe.
	287	+ *
	288	+ * The unaliasing is done in copy_thread() instead. This works because
	289	+ * dst is not schedulable or traceable until both of these functions
	290	+ * have been called.
	291	+ */
244	292	int arch_dup_task_struct(struct task_struct dst, struct task_struct src)
245	293	{
246	294	if (current->mm)
247	295	fpsimd_preserve_current_state();
248	296	dst = src;
	297	+
249	298	return 0;
250	299	}
251	300
@@ -258,6 +307,13 @@ int copy_thread(unsigned long clone_flags, unsigned long stack_start, · 6 unmodified lines
258	307
259	308	memset(&p->thread.cpu_context, 0, sizeof(struct cpu_context));
260	309
	310	+ /*
	311	+ * Unalias p->thread.sve_state (if any) from the parent task
	312	+ * and disable discard SVE state for p:
	313	+ */
	314	+ clear_tsk_thread_flag(p, TIF_SVE);
	315	+ p->thread.sve_state = NULL;
	316	+
261	317	if (likely(!(p->flags & PF_KTHREAD))) {
262	318	childregs = current_pt_regs();
263	319	childregs->regs[0] = 0;

arch/arm64/kernel/ptrace.c+272−8

@@ -32,6 +32,7 @@ · 31 unmodified lines
32	32	#include <linux/security.h>
33	33	#include <linux/init.h>
34	34	#include <linux/signal.h>
	35	+#include <linux/string.h>
35	36	#include <linux/uaccess.h>
36	37	#include <linux/perf_event.h>
37	38	#include <linux/hw_breakpoint.h>
@@ -40,6 +41,7 @@ · 2 unmodified lines
40	41	#include <linux/elf.h>
41	42
42	43	#include <asm/compat.h>
	44	+#include <asm/cpufeature.h>
43	45	#include <asm/debug-monitors.h>
44	46	#include <asm/pgtable.h>
45	47	#include <asm/stacktrace.h>
@@ -618,17 +620,56 @@ static int gpr_set(struct task_struct target, const struct user_regset regset, · 572 unmodified lines
618	620	/*
619	621	* TODO: update fp accessors for lazy context switching (sync/flush hwstate)
620	622	*/
621		−static int fpr_get(struct task_struct target, const struct user_regset regset,
622		− unsigned int pos, unsigned int count,
623		− void kbuf, void __user ubuf)
	623	+static int __fpr_get(struct task_struct *target,
	624	+ const struct user_regset *regset,
	625	+ unsigned int pos, unsigned int count,
	626	+ void kbuf, void __user ubuf, unsigned int start_pos)
624	627	{
625	628	struct user_fpsimd_state *uregs;
	629	+
	630	+ sve_sync_to_fpsimd(target);
	631	+
626	632	uregs = &target->thread.fpsimd_state.user_fpsimd;
627	633
	634	+ return user_regset_copyout(&pos, &count, &kbuf, &ubuf, uregs,
	635	+ start_pos, start_pos + sizeof(*uregs));
	636	+}
	637	+
	638	+static int fpr_get(struct task_struct target, const struct user_regset regset,
	639	+ unsigned int pos, unsigned int count,
	640	+ void kbuf, void __user ubuf)
	641	+{
628	642	if (target == current)
629	643	fpsimd_preserve_current_state();
630	644
631		− return user_regset_copyout(&pos, &count, &kbuf, &ubuf, uregs, 0, -1);
	645	+ return __fpr_get(target, regset, pos, count, kbuf, ubuf, 0);
	646	+}
	647	+
	648	+static int __fpr_set(struct task_struct *target,
	649	+ const struct user_regset *regset,
	650	+ unsigned int pos, unsigned int count,
	651	+ const void kbuf, const void __user ubuf,
	652	+ unsigned int start_pos)
	653	+{
	654	+ int ret;
	655	+ struct user_fpsimd_state newstate;
	656	+
	657	+ /*
	658	+ * Ensure target->thread.fpsimd_state is up to date, so that a
	659	+ * short copyin can't resurrect stale data.
	660	+ */
	661	+ sve_sync_to_fpsimd(target);
	662	+
	663	+ newstate = target->thread.fpsimd_state.user_fpsimd;
	664	+
	665	+ ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &newstate,
	666	+ start_pos, start_pos + sizeof(newstate));
	667	+ if (ret)
	668	+ return ret;
	669	+
	670	+ target->thread.fpsimd_state.user_fpsimd = newstate;
	671	+
	672	+ return ret;
632	673	}
633	674
634	675	static int fpr_set(struct task_struct target, const struct user_regset regset,
@@ -636,15 +677,14 @@ static int fpr_set(struct task_struct target, const struct user_regset regset, · 1 unmodified line
636	677	const void kbuf, const void __user ubuf)
637	678	{
638	679	int ret;
639		− struct user_fpsimd_state newstate =
640		− target->thread.fpsimd_state.user_fpsimd;
641	680
642		− ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &newstate, 0, -1);
	681	+ ret = __fpr_set(target, regset, pos, count, kbuf, ubuf, 0);
643	682	if (ret)
644	683	return ret;
645	684
646		− target->thread.fpsimd_state.user_fpsimd = newstate;
	685	+ sve_sync_from_fpsimd_zeropad(target);
647	686	fpsimd_flush_task_state(target);
	687	+
648	688	return ret;
649	689	}
650	690
@@ -702,6 +742,215 @@ static int system_call_set(struct task_struct *target, · 51 unmodified lines
702	742	return ret;
703	743	}
704	744
	745	+#ifdef CONFIG_ARM64_SVE
	746	+
	747	+static void sve_init_header_from_task(struct user_sve_header *header,
	748	+ struct task_struct *target)
	749	+{
	750	+ unsigned int vq;
	751	+
	752	+ memset(header, 0, sizeof(*header));
	753	+
	754	+ header->flags = test_tsk_thread_flag(target, TIF_SVE) ?
	755	+ SVE_PT_REGS_SVE : SVE_PT_REGS_FPSIMD;
	756	+ if (test_tsk_thread_flag(target, TIF_SVE_VL_INHERIT))
	757	+ header->flags \|= SVE_PT_VL_INHERIT;
	758	+
	759	+ header->vl = target->thread.sve_vl;
	760	+ vq = sve_vq_from_vl(header->vl);
	761	+
	762	+ header->max_vl = sve_max_vl;
	763	+ if (WARN_ON(!sve_vl_valid(sve_max_vl)))
	764	+ header->max_vl = header->vl;
	765	+
	766	+ header->size = SVE_PT_SIZE(vq, header->flags);
	767	+ header->max_size = SVE_PT_SIZE(sve_vq_from_vl(header->max_vl),
	768	+ SVE_PT_REGS_SVE);
	769	+}
	770	+
	771	+static unsigned int sve_size_from_header(struct user_sve_header const *header)
	772	+{
	773	+ return ALIGN(header->size, SVE_VQ_BYTES);
	774	+}
	775	+
	776	+static unsigned int sve_get_size(struct task_struct *target,
	777	+ const struct user_regset *regset)
	778	+{
	779	+ struct user_sve_header header;
	780	+
	781	+ if (!system_supports_sve())
	782	+ return 0;
	783	+
	784	+ sve_init_header_from_task(&header, target);
	785	+ return sve_size_from_header(&header);
	786	+}
	787	+
	788	+static int sve_get(struct task_struct *target,
	789	+ const struct user_regset *regset,
	790	+ unsigned int pos, unsigned int count,
	791	+ void kbuf, void __user ubuf)
	792	+{
	793	+ int ret;
	794	+ struct user_sve_header header;
	795	+ unsigned int vq;
	796	+ unsigned long start, end;
	797	+
	798	+ if (!system_supports_sve())
	799	+ return -EINVAL;
	800	+
	801	+ /* Header */
	802	+ sve_init_header_from_task(&header, target);
	803	+ vq = sve_vq_from_vl(header.vl);
	804	+
	805	+ ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf, &header,
	806	+ 0, sizeof(header));
	807	+ if (ret)
	808	+ return ret;
	809	+
	810	+ if (target == current)
	811	+ fpsimd_preserve_current_state();
	812	+
	813	+ /* Registers: FPSIMD-only case */
	814	+
	815	+ BUILD_BUG_ON(SVE_PT_FPSIMD_OFFSET != sizeof(header));
	816	+ if ((header.flags & SVE_PT_REGS_MASK) == SVE_PT_REGS_FPSIMD)
	817	+ return __fpr_get(target, regset, pos, count, kbuf, ubuf,
	818	+ SVE_PT_FPSIMD_OFFSET);
	819	+
	820	+ /* Otherwise: full SVE case */
	821	+
	822	+ BUILD_BUG_ON(SVE_PT_SVE_OFFSET != sizeof(header));
	823	+ start = SVE_PT_SVE_OFFSET;
	824	+ end = SVE_PT_SVE_FFR_OFFSET(vq) + SVE_PT_SVE_FFR_SIZE(vq);
	825	+ ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf,
	826	+ target->thread.sve_state,
	827	+ start, end);
	828	+ if (ret)
	829	+ return ret;
	830	+
	831	+ start = end;
	832	+ end = SVE_PT_SVE_FPSR_OFFSET(vq);
	833	+ ret = user_regset_copyout_zero(&pos, &count, &kbuf, &ubuf,
	834	+ start, end);
	835	+ if (ret)
	836	+ return ret;
	837	+
	838	+ /*
	839	+ * Copy fpsr, and fpcr which must follow contiguously in
	840	+ * struct fpsimd_state:
	841	+ */
	842	+ start = end;
	843	+ end = SVE_PT_SVE_FPCR_OFFSET(vq) + SVE_PT_SVE_FPCR_SIZE;
	844	+ ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf,
	845	+ &target->thread.fpsimd_state.fpsr,
	846	+ start, end);
	847	+ if (ret)
	848	+ return ret;
	849	+
	850	+ start = end;
	851	+ end = sve_size_from_header(&header);
	852	+ return user_regset_copyout_zero(&pos, &count, &kbuf, &ubuf,
	853	+ start, end);
	854	+}
	855	+
	856	+static int sve_set(struct task_struct *target,
	857	+ const struct user_regset *regset,
	858	+ unsigned int pos, unsigned int count,
	859	+ const void kbuf, const void __user ubuf)
	860	+{
	861	+ int ret;
	862	+ struct user_sve_header header;
	863	+ unsigned int vq;
	864	+ unsigned long start, end;
	865	+
	866	+ if (!system_supports_sve())
	867	+ return -EINVAL;
	868	+
	869	+ /* Header */
	870	+ if (count < sizeof(header))
	871	+ return -EINVAL;
	872	+ ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &header,
	873	+ 0, sizeof(header));
	874	+ if (ret)
	875	+ goto out;
	876	+
	877	+ /*
	878	+ * Apart from PT_SVE_REGS_MASK, all PT_SVE_* flags are consumed by
	879	+ * sve_set_vector_length(), which will also validate them for us:
	880	+ */
	881	+ ret = sve_set_vector_length(target, header.vl,
	882	+ ((unsigned long)header.flags & ~SVE_PT_REGS_MASK) << 16);
	883	+ if (ret)
	884	+ goto out;
	885	+
	886	+ /* Actual VL set may be less than the user asked for: */
	887	+ vq = sve_vq_from_vl(target->thread.sve_vl);
	888	+
	889	+ /* Registers: FPSIMD-only case */
	890	+
	891	+ BUILD_BUG_ON(SVE_PT_FPSIMD_OFFSET != sizeof(header));
	892	+ if ((header.flags & SVE_PT_REGS_MASK) == SVE_PT_REGS_FPSIMD) {
	893	+ ret = __fpr_set(target, regset, pos, count, kbuf, ubuf,
	894	+ SVE_PT_FPSIMD_OFFSET);
	895	+ clear_tsk_thread_flag(target, TIF_SVE);
	896	+ goto out;
	897	+ }
	898	+
	899	+ /* Otherwise: full SVE case */
	900	+
	901	+ /*
	902	+ * If setting a different VL from the requested VL and there is
	903	+ * register data, the data layout will be wrong: don't even
	904	+ * try to set the registers in this case.
	905	+ */
	906	+ if (count && vq != sve_vq_from_vl(header.vl)) {
	907	+ ret = -EIO;
	908	+ goto out;
	909	+ }
	910	+
	911	+ sve_alloc(target);
	912	+
	913	+ /*
	914	+ * Ensure target->thread.sve_state is up to date with target's
	915	+ * FPSIMD regs, so that a short copyin leaves trailing registers
	916	+ * unmodified.
	917	+ */
	918	+ fpsimd_sync_to_sve(target);
	919	+ set_tsk_thread_flag(target, TIF_SVE);
	920	+
	921	+ BUILD_BUG_ON(SVE_PT_SVE_OFFSET != sizeof(header));
	922	+ start = SVE_PT_SVE_OFFSET;
	923	+ end = SVE_PT_SVE_FFR_OFFSET(vq) + SVE_PT_SVE_FFR_SIZE(vq);
	924	+ ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf,
	925	+ target->thread.sve_state,
	926	+ start, end);
	927	+ if (ret)
	928	+ goto out;
	929	+
	930	+ start = end;
	931	+ end = SVE_PT_SVE_FPSR_OFFSET(vq);
	932	+ ret = user_regset_copyin_ignore(&pos, &count, &kbuf, &ubuf,
	933	+ start, end);
	934	+ if (ret)
	935	+ goto out;
	936	+
	937	+ /*
	938	+ * Copy fpsr, and fpcr which must follow contiguously in
	939	+ * struct fpsimd_state:
	940	+ */
	941	+ start = end;
	942	+ end = SVE_PT_SVE_FPCR_OFFSET(vq) + SVE_PT_SVE_FPCR_SIZE;
	943	+ ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf,
	944	+ &target->thread.fpsimd_state.fpsr,
	945	+ start, end);
	946	+
	947	+out:
	948	+ fpsimd_flush_task_state(target);
	949	+ return ret;
	950	+}
	951	+
	952	+#endif /* CONFIG_ARM64_SVE */
	953	+
705	954	enum aarch64_regset {
706	955	REGSET_GPR,
707	956	REGSET_FPR,
@@ -711,6 +960,9 @@ enum aarch64_regset { · 3 unmodified lines
711	960	REGSET_HW_WATCH,
712	961	#endif
713	962	REGSET_SYSTEM_CALL,
	963	+#ifdef CONFIG_ARM64_SVE
	964	+ REGSET_SVE,
	965	+#endif
714	966	};
715	967
716	968	static const struct user_regset aarch64_regsets[] = {
@@ -768,6 +1020,18 @@ static const struct user_regset aarch64_regsets[] = { · 51 unmodified lines
768	1020	.get = system_call_get,
769	1021	.set = system_call_set,
770	1022	},
	1023	+#ifdef CONFIG_ARM64_SVE
	1024	+ [REGSET_SVE] = { /* Scalable Vector Extension */
	1025	+ .core_note_type = NT_ARM_SVE,
	1026	+ .n = DIV_ROUND_UP(SVE_PT_SIZE(SVE_VQ_MAX, SVE_PT_REGS_SVE),
	1027	+ SVE_VQ_BYTES),
	1028	+ .size = SVE_VQ_BYTES,
	1029	+ .align = SVE_VQ_BYTES,
	1030	+ .get = sve_get,
	1031	+ .set = sve_set,
	1032	+ .get_size = sve_get_size,
	1033	+ },
	1034	+#endif
771	1035	};
772	1036
773	1037	static const struct user_regset_view user_aarch64_view = {

arch/arm64/kernel/setup.c+7−8

@@ -23,7 +23,6 @@ · 22 unmodified lines
23	23	#include <linux/stddef.h>
24	24	#include <linux/ioport.h>
25	25	#include <linux/delay.h>
26		−#include <linux/utsname.h>
27	26	#include <linux/initrd.h>
28	27	#include <linux/console.h>
29	28	#include <linux/cache.h>
@@ -48,6 +47,7 @@ · 18 unmodified lines
48	47	#include <asm/fixmap.h>
49	48	#include <asm/cpu.h>
50	49	#include <asm/cputype.h>
	50	+#include <asm/daifflags.h>
51	51	#include <asm/elf.h>
52	52	#include <asm/cpufeature.h>
53	53	#include <asm/cpu_ops.h>
@@ -103,7 +103,8 @@ void __init smp_setup_processor_id(void) · 49 unmodified lines
103	103	* access percpu variable inside lock_release
104	104	*/
105	105	set_my_cpu_offset(0);
106		− pr_info("Booting Linux on physical CPU 0x%lx\n", (unsigned long)mpidr);
	106	+ pr_info("Booting Linux on physical CPU 0x%010lx [0x%08x]\n",
	107	+ (unsigned long)mpidr, read_cpuid_id());
107	108	}
108	109
109	110	bool arch_match_cpu_phys_id(int cpu, u64 phys_id)
@@ -244,9 +245,6 @@ u64 __cpu_logical_map[NR_CPUS] = { [0 ... NR_CPUS-1] = INVALID_HWID }; · 134 unmodified lines
244	245
245	246	void __init setup_arch(char **cmdline_p)
246	247	{
247		− pr_info("Boot CPU: AArch64 Processor [%08x]\n", read_cpuid_id());
248		−
249		− sprintf(init_utsname()->machine, UTS_MACHINE);
250	248	init_mm.start_code = (unsigned long) _text;
251	249	init_mm.end_code = (unsigned long) _etext;
252	250	init_mm.end_data = (unsigned long) _edata;
@@ -262,10 +260,11 @@ void __init setup_arch(char **cmdline_p) · 9 unmodified lines
262	260	parse_early_param();
263	261
264	262	/*
265		− * Unmask asynchronous aborts after bringing up possible earlycon.
266		− * (Report possible System Errors once we can report this occurred)
	263	+ * Unmask asynchronous aborts and fiq after bringing up possible
	264	+ * earlycon. (Report possible System Errors once we can report this
	265	+ * occurred).
267	266	*/
268		− local_async_enable();
	267	+ local_daif_restore(DAIF_PROCCTX_NOIRQ);
269	268
270	269	/*
271	270	* TTBR0 is only used for the identity mapping at this stage. Make it

arch/arm64/kernel/signal.c+169−10

@@ -31,6 +31,7 @@ · 30 unmodified lines
31	31	#include <linux/ratelimit.h>
32	32	#include <linux/syscalls.h>
33	33
	34	+#include <asm/daifflags.h>
34	35	#include <asm/debug-monitors.h>
35	36	#include <asm/elf.h>
36	37	#include <asm/cacheflush.h>
@@ -63,6 +64,7 @@ struct rt_sigframe_user_layout { · 26 unmodified lines
63	64
64	65	unsigned long fpsimd_offset;
65	66	unsigned long esr_offset;
	67	+ unsigned long sve_offset;
66	68	unsigned long extra_offset;
67	69	unsigned long end_offset;
68	70	};
@@ -179,9 +181,6 @@ static int preserve_fpsimd_context(struct fpsimd_context __user *ctx) · 110 unmodified lines
179	181	struct fpsimd_state *fpsimd = &current->thread.fpsimd_state;
180	182	int err;
181	183
182		− /* dump the hardware registers to the fpsimd_state structure */
183		− fpsimd_preserve_current_state();
184		−
185	184	/* copy the FP and status/control registers */
186	185	err = __copy_to_user(ctx->vregs, fpsimd->vregs, sizeof(fpsimd->vregs));
187	186	__put_user_error(fpsimd->fpsr, &ctx->fpsr, err);
@@ -214,6 +213,8 @@ static int restore_fpsimd_context(struct fpsimd_context __user *ctx) · 26 unmodified lines
214	213	__get_user_error(fpsimd.fpsr, &ctx->fpsr, err);
215	214	__get_user_error(fpsimd.fpcr, &ctx->fpcr, err);
216	215
	216	+ clear_thread_flag(TIF_SVE);
	217	+
217	218	/* load the hardware registers from the fpsimd_state structure */
218	219	if (!err)
219	220	fpsimd_update_current_state(&fpsimd);
@@ -221,10 +222,118 @@ static int restore_fpsimd_context(struct fpsimd_context __user *ctx) · 1 unmodified line
221	222	return err ? -EFAULT : 0;
222	223	}
223	224
	225	+
224	226	struct user_ctxs {
225	227	struct fpsimd_context __user *fpsimd;
	228	+ struct sve_context __user *sve;
226	229	};
227	230
	231	+#ifdef CONFIG_ARM64_SVE
	232	+
	233	+static int preserve_sve_context(struct sve_context __user *ctx)
	234	+{
	235	+ int err = 0;
	236	+ u16 reserved[ARRAY_SIZE(ctx->__reserved)];
	237	+ unsigned int vl = current->thread.sve_vl;
	238	+ unsigned int vq = 0;
	239	+
	240	+ if (test_thread_flag(TIF_SVE))
	241	+ vq = sve_vq_from_vl(vl);
	242	+
	243	+ memset(reserved, 0, sizeof(reserved));
	244	+
	245	+ __put_user_error(SVE_MAGIC, &ctx->head.magic, err);
	246	+ __put_user_error(round_up(SVE_SIG_CONTEXT_SIZE(vq), 16),
	247	+ &ctx->head.size, err);
	248	+ __put_user_error(vl, &ctx->vl, err);
	249	+ BUILD_BUG_ON(sizeof(ctx->__reserved) != sizeof(reserved));
	250	+ err \|= __copy_to_user(&ctx->__reserved, reserved, sizeof(reserved));
	251	+
	252	+ if (vq) {
	253	+ /*
	254	+ * This assumes that the SVE state has already been saved to
	255	+ * the task struct by calling preserve_fpsimd_context().
	256	+ */
	257	+ err \|= __copy_to_user((char __user *)ctx + SVE_SIG_REGS_OFFSET,
	258	+ current->thread.sve_state,
	259	+ SVE_SIG_REGS_SIZE(vq));
	260	+ }
	261	+
	262	+ return err ? -EFAULT : 0;
	263	+}
	264	+
	265	+static int restore_sve_fpsimd_context(struct user_ctxs *user)
	266	+{
	267	+ int err;
	268	+ unsigned int vq;
	269	+ struct fpsimd_state fpsimd;
	270	+ struct sve_context sve;
	271	+
	272	+ if (__copy_from_user(&sve, user->sve, sizeof(sve)))
	273	+ return -EFAULT;
	274	+
	275	+ if (sve.vl != current->thread.sve_vl)
	276	+ return -EINVAL;
	277	+
	278	+ if (sve.head.size <= sizeof(*user->sve)) {
	279	+ clear_thread_flag(TIF_SVE);
	280	+ goto fpsimd_only;
	281	+ }
	282	+
	283	+ vq = sve_vq_from_vl(sve.vl);
	284	+
	285	+ if (sve.head.size < SVE_SIG_CONTEXT_SIZE(vq))
	286	+ return -EINVAL;
	287	+
	288	+ /*
	289	+ * Careful: we are about __copy_from_user() directly into
	290	+ * thread.sve_state with preemption enabled, so protection is
	291	+ * needed to prevent a racing context switch from writing stale
	292	+ * registers back over the new data.
	293	+ */
	294	+
	295	+ fpsimd_flush_task_state(current);
	296	+ barrier();
	297	+ /* From now, fpsimd_thread_switch() won't clear TIF_FOREIGN_FPSTATE */
	298	+
	299	+ set_thread_flag(TIF_FOREIGN_FPSTATE);
	300	+ barrier();
	301	+ /* From now, fpsimd_thread_switch() won't touch thread.sve_state */
	302	+
	303	+ sve_alloc(current);
	304	+ err = __copy_from_user(current->thread.sve_state,
	305	+ (char __user const *)user->sve +
	306	+ SVE_SIG_REGS_OFFSET,
	307	+ SVE_SIG_REGS_SIZE(vq));
	308	+ if (err)
	309	+ return -EFAULT;
	310	+
	311	+ set_thread_flag(TIF_SVE);
	312	+
	313	+fpsimd_only:
	314	+ /* copy the FP and status/control registers */
	315	+ /* restore_sigframe() already checked that user->fpsimd != NULL. */
	316	+ err = __copy_from_user(fpsimd.vregs, user->fpsimd->vregs,
	317	+ sizeof(fpsimd.vregs));
	318	+ __get_user_error(fpsimd.fpsr, &user->fpsimd->fpsr, err);
	319	+ __get_user_error(fpsimd.fpcr, &user->fpsimd->fpcr, err);
	320	+
	321	+ /* load the hardware registers from the fpsimd_state structure */
	322	+ if (!err)
	323	+ fpsimd_update_current_state(&fpsimd);
	324	+
	325	+ return err ? -EFAULT : 0;
	326	+}
	327	+
	328	+#else /* ! CONFIG_ARM64_SVE */
	329	+
	330	+/* Turn any non-optimised out attempts to use these into a link error: */
	331	+extern int preserve_sve_context(void __user *ctx);
	332	+extern int restore_sve_fpsimd_context(struct user_ctxs *user);
	333	+
	334	+#endif /* ! CONFIG_ARM64_SVE */
	335	+
	336	+
228	337	static int parse_user_sigframe(struct user_ctxs *user,
229	338	struct rt_sigframe __user *sf)
230	339	{
@@ -237,6 +346,7 @@ static int parse_user_sigframe(struct user_ctxs *user, · 6 unmodified lines
237	346	char const __user const sfp = (char const __user )sf;
238	347
239	348	user->fpsimd = NULL;
	349	+ user->sve = NULL;
240	350
241	351	if (!IS_ALIGNED((unsigned long)base, 16))
242	352	goto invalid;
@@ -287,6 +397,19 @@ static int parse_user_sigframe(struct user_ctxs *user, · 44 unmodified lines
287	397	/* ignore */
288	398	break;
289	399
	400	+ case SVE_MAGIC:
	401	+ if (!system_supports_sve())
	402	+ goto invalid;
	403	+
	404	+ if (user->sve)
	405	+ goto invalid;
	406	+
	407	+ if (size < sizeof(*user->sve))
	408	+ goto invalid;
	409	+
	410	+ user->sve = (struct sve_context __user *)head;
	411	+ break;
	412	+
290	413	case EXTRA_MAGIC:
291	414	if (have_extra_context)
292	415	goto invalid;
@@ -343,6 +466,10 @@ static int parse_user_sigframe(struct user_ctxs *user, · 50 unmodified lines
343	466	*/
344	467	offset = 0;
345	468	limit = extra_size;
	469	+
	470	+ if (!access_ok(VERIFY_READ, base, limit))
	471	+ goto invalid;
	472	+
346	473	continue;
347	474
348	475	default:
@@ -359,9 +486,6 @@ static int parse_user_sigframe(struct user_ctxs *user, · 10 unmodified lines
359	486	}
360	487
361	488	done:
362		− if (!user->fpsimd)
363		− goto invalid;
364		−
365	489	return 0;
366	490
367	491	invalid:
@@ -395,8 +519,19 @@ static int restore_sigframe(struct pt_regs *regs, · 27 unmodified lines
395	519	if (err == 0)
396	520	err = parse_user_sigframe(&user, sf);
397	521
398		− if (err == 0)
399		− err = restore_fpsimd_context(user.fpsimd);
	522	+ if (err == 0) {
	523	+ if (!user.fpsimd)
	524	+ return -EINVAL;
	525	+
	526	+ if (user.sve) {
	527	+ if (!system_supports_sve())
	528	+ return -EINVAL;
	529	+
	530	+ err = restore_sve_fpsimd_context(&user);
	531	+ } else {
	532	+ err = restore_fpsimd_context(user.fpsimd);
	533	+ }
	534	+ }
400	535
401	536	return err;
402	537	}
@@ -455,6 +590,18 @@ static int setup_sigframe_layout(struct rt_sigframe_user_layout *user) · 52 unmodified lines
455	590	return err;
456	591	}
457	592
	593	+ if (system_supports_sve()) {
	594	+ unsigned int vq = 0;
	595	+
	596	+ if (test_thread_flag(TIF_SVE))
	597	+ vq = sve_vq_from_vl(current->thread.sve_vl);
	598	+
	599	+ err = sigframe_alloc(user, &user->sve_offset,
	600	+ SVE_SIG_CONTEXT_SIZE(vq));
	601	+ if (err)
	602	+ return err;
	603	+ }
	604	+
458	605	return sigframe_alloc_end(user);
459	606	}
460	607
@@ -496,6 +643,13 @@ static int setup_sigframe(struct rt_sigframe_user_layout *user, · 35 unmodified lines
496	643	__put_user_error(current->thread.fault_code, &esr_ctx->esr, err);
497	644	}
498	645
	646	+ /* Scalable Vector Extension state, if present */
	647	+ if (system_supports_sve() && err == 0 && user->sve_offset) {
	648	+ struct sve_context __user *sve_ctx =
	649	+ apply_user_offset(user, user->sve_offset);
	650	+ err \|= preserve_sve_context(sve_ctx);
	651	+ }
	652	+
499	653	if (err == 0 && user->extra_offset) {
500	654	char __user sfp = (char __user )user->sigframe;
501	655	char __user *userp =
@@ -595,6 +749,8 @@ static int setup_rt_frame(int usig, struct ksignal ksig, sigset_t set, · 93 unmodified lines
595	749	struct rt_sigframe __user *frame;
596	750	int err = 0;
597	751
	752	+ fpsimd_signal_preserve_current_state();
	753	+
598	754	if (get_sigframe(&user, ksig, regs))
599	755	return 1;
600	756
@@ -756,9 +912,12 @@ asmlinkage void do_notify_resume(struct pt_regs *regs, · 155 unmodified lines
756	912	addr_limit_user_check();
757	913
758	914	if (thread_flags & _TIF_NEED_RESCHED) {
	915	+ /* Unmask Debug and SError for the next task */
	916	+ local_daif_restore(DAIF_PROCCTX_NOIRQ);
	917	+
759	918	schedule();
760	919	} else {
761		− local_irq_enable();
	920	+ local_daif_restore(DAIF_PROCCTX);
762	921
763	922	if (thread_flags & _TIF_UPROBE)
764	923	uprobe_notify_resume(regs);
@@ -775,7 +934,7 @@ asmlinkage void do_notify_resume(struct pt_regs *regs, · 10 unmodified lines
775	934	fpsimd_restore_current_state();
776	935	}
777	936
778		− local_irq_disable();
	937	+ local_daif_mask();
779	938	thread_flags = READ_ONCE(current_thread_info()->flags);
780	939	} while (thread_flags & _TIF_WORK_MASK);
781	940	}

arch/arm64/kernel/signal32.c+1−1

@@ -239,7 +239,7 @@ static int compat_preserve_vfp_context(struct compat_vfp_sigframe __user *frame) · 238 unmodified lines
239	239	* Note that this also saves V16-31, which aren't visible
240	240	* in AArch32.
241	241	*/
242		− fpsimd_preserve_current_state();
	242	+ fpsimd_signal_preserve_current_state();
243	243
244	244	/* Place structure header on the stack */
245	245	__put_user_error(magic, &frame->magic, err);

arch/arm64/kernel/smp.c+8−10

@@ -47,6 +47,7 @@ · 46 unmodified lines
47	47	#include <asm/cpu.h>
48	48	#include <asm/cputype.h>
49	49	#include <asm/cpu_ops.h>
	50	+#include <asm/daifflags.h>
50	51	#include <asm/mmu_context.h>
51	52	#include <asm/numa.h>
52	53	#include <asm/pgtable.h>
@@ -216,6 +217,7 @@ int __cpu_up(unsigned int cpu, struct task_struct *idle) · 163 unmodified lines
216	217	*/
217	218	asmlinkage void secondary_start_kernel(void)
218	219	{
	220	+ u64 mpidr = read_cpuid_mpidr() & MPIDR_HWID_BITMASK;
219	221	struct mm_struct *mm = &init_mm;
220	222	unsigned int cpu;
221	223
@@ -265,14 +267,14 @@ asmlinkage void secondary_start_kernel(void) · 43 unmodified lines
265	267	* the CPU migration code to notice that the CPU is online
266	268	* before we continue.
267	269	*/
268		− pr_info("CPU%u: Booted secondary processor [%08x]\n",
269		− cpu, read_cpuid_id());
	270	+ pr_info("CPU%u: Booted secondary processor 0x%010lx [0x%08x]\n",
	271	+ cpu, (unsigned long)mpidr,
	272	+ read_cpuid_id());
270	273	update_cpu_boot_status(CPU_BOOT_SUCCESS);
271	274	set_cpu_online(cpu, true);
272	275	complete(&cpu_running);
273	276
274		− local_irq_enable();
275		− local_async_enable();
	277	+ local_daif_restore(DAIF_PROCCTX);
276	278
277	279	/*
278	280	* OK, it's off to the idle thread for us
@@ -368,10 +370,6 @@ void __cpu_die(unsigned int cpu) · 89 unmodified lines
368	370	/*
369	371	* Called from the idle thread for the CPU which has been shutdown.
370	372	*
371		− * Note that we disable IRQs here, but do not re-enable them
372		− * before returning to the caller. This is also the behaviour
373		− * of the other hotplug-cpu capable cores, so presumably coming
374		− * out of idle fixes this.
375	373	*/
376	374	void cpu_die(void)
377	375	{
@@ -379,7 +377,7 @@ void cpu_die(void) · 1 unmodified line
379	377
380	378	idle_task_exit();
381	379
382		− local_irq_disable();
	380	+ local_daif_mask();
383	381
384	382	/* Tell __cpu_die() that this CPU is now safe to dispose of */
385	383	(void)cpu_report_death();
@@ -837,7 +835,7 @@ static void ipi_cpu_stop(unsigned int cpu) · 451 unmodified lines
837	835	{
838	836	set_cpu_online(cpu, false);
839	837
840		− local_irq_disable();
	838	+ local_daif_mask();
841	839
842	840	while (1)
843	841	cpu_relax();

arch/arm64/kernel/suspend.c+4−4

@@ -5,6 +5,7 @@ · 4 unmodified lines
5	5	#include <asm/alternative.h>
6	6	#include <asm/cacheflush.h>
7	7	#include <asm/cpufeature.h>
	8	+#include <asm/daifflags.h>
8	9	#include <asm/debug-monitors.h>
9	10	#include <asm/exec.h>
10	11	#include <asm/pgtable.h>
@@ -12,7 +13,6 @@ · 1 unmodified line
12	13	#include <asm/mmu_context.h>
13	14	#include <asm/smp_plat.h>
14	15	#include <asm/suspend.h>
15		−#include <asm/tlbflush.h>
16	16
17	17	/*
18	18	* This is allocated by cpu_suspend_init(), and used to store a pointer to
@@ -58,7 +58,7 @@ void notrace __cpu_suspend_exit(void) · 39 unmodified lines
58	58	/*
59	59	* Restore HW breakpoint registers to sane values
60	60	* before debug exceptions are possibly reenabled
61		− * through local_dbg_restore.
	61	+ * by cpu_suspend()s local_daif_restore() call.
62	62	*/
63	63	if (hw_breakpoint_restore)
64	64	hw_breakpoint_restore(cpu);
@@ -82,7 +82,7 @@ int cpu_suspend(unsigned long arg, int (*fn)(unsigned long)) · 17 unmodified lines
82	82	* updates to mdscr register (saved and restored along with
83	83	* general purpose registers) from kernel debuggers.
84	84	*/
85		− local_dbg_save(flags);
	85	+ flags = local_daif_save();
86	86
87	87	/*
88	88	* Function graph tracer state gets incosistent when the kernel
@@ -115,7 +115,7 @@ int cpu_suspend(unsigned long arg, int (*fn)(unsigned long)) · 26 unmodified lines
115	115	* restored, so from this point onwards, debugging is fully
116	116	* renabled if it was enabled when core started shutdown.
117	117	*/
118		− local_dbg_restore(flags);
	118	+ local_daif_restore(flags);
119	119
120	120	return ret;
121	121	}

arch/arm64/kernel/traps.c+38−71

@@ -38,6 +38,7 @@ · 37 unmodified lines
38	38
39	39	#include <asm/atomic.h>
40	40	#include <asm/bug.h>
	41	+#include <asm/daifflags.h>
41	42	#include <asm/debug-monitors.h>
42	43	#include <asm/esr.h>
43	44	#include <asm/insn.h>
@@ -58,55 +59,9 @@ static const char *handler[]= { · 14 unmodified lines
58	59
59	60	int show_unhandled_signals = 1;
60	61
61		−/*
62		− * Dump out the contents of some kernel memory nicely...
63		− */
64		−static void dump_mem(const char lvl, const char str, unsigned long bottom,
65		− unsigned long top)
66		−{
67		− unsigned long first;
68		− mm_segment_t fs;
69		− int i;
70		−
71		− /*
72		− * We need to switch to kernel mode so that we can use __get_user
73		− * to safely read from kernel space.
74		− */
75		− fs = get_fs();
76		− set_fs(KERNEL_DS);
77		−
78		− printk("%s%s(0x%016lx to 0x%016lx)\n", lvl, str, bottom, top);
79		−
80		− for (first = bottom & ~31; first < top; first += 32) {
81		− unsigned long p;
82		− char str[sizeof(" 12345678") * 8 + 1];
83		−
84		− memset(str, ' ', sizeof(str));
85		− str[sizeof(str) - 1] = '\0';
86		−
87		− for (p = first, i = 0; i < (32 / 8)
88		− && p < top; i++, p += 8) {
89		− if (p >= bottom && p < top) {
90		− unsigned long val;
91		−
92		− if (__get_user(val, (unsigned long *)p) == 0)
93		− sprintf(str + i * 17, " %016lx", val);
94		− else
95		− sprintf(str + i * 17, " ????????????????");
96		− }
97		− }
98		− printk("%s%04lx:%s\n", lvl, first & 0xffff, str);
99		− }
100		−
101		− set_fs(fs);
102		−}
103		−
104	62	static void dump_backtrace_entry(unsigned long where)
105	63	{
106		− /*
107		− * Note that 'where' can have a physical address, but it's not handled.
108		− */
109		− print_ip_sym(where);
	64	+ printk(" %pS\n", (void *)where);
110	65	}
111	66
112	67	static void __dump_instr(const char lvl, struct pt_regs regs)
@@ -171,10 +126,7 @@ void dump_backtrace(struct pt_regs regs, struct task_struct tsk) · 58 unmodified lines
171	126
172	127	skip = !!regs;
173	128	printk("Call trace:\n");
174		− while (1) {
175		− unsigned long stack;
176		− int ret;
177		−
	129	+ do {
178	130	/* skip until specified stack frame */
179	131	if (!skip) {
180	132	dump_backtrace_entry(frame.pc);
@@ -189,17 +141,7 @@ void dump_backtrace(struct pt_regs regs, struct task_struct tsk) · 8 unmodified lines
189	141	*/
190	142	dump_backtrace_entry(regs->pc);
191	143	}
192		− ret = unwind_frame(tsk, &frame);
193		− if (ret < 0)
194		− break;
195		− if (in_entry_text(frame.pc)) {
196		− stack = frame.fp - offsetof(struct pt_regs, stackframe);
197		−
198		− if (on_accessible_stack(tsk, stack))
199		− dump_mem("", "Exception stack", stack,
200		− stack + sizeof(struct pt_regs));
201		− }
202		− }
	144	+ } while (!unwind_frame(tsk, &frame));
203	145
204	146	put_task_stack(tsk);
205	147	}
@@ -293,6 +235,17 @@ void arm64_notify_die(const char str, struct pt_regs regs, · 87 unmodified lines
293	235	}
294	236	}
295	237
	238	+void arm64_skip_faulting_instruction(struct pt_regs *regs, unsigned long size)
	239	+{
	240	+ regs->pc += size;
	241	+
	242	+ /*
	243	+ * If we were single stepping, we want to get the step exception after
	244	+ * we return from the trap.
	245	+ */
	246	+ user_fastforward_single_step(current);
	247	+}
	248	+
296	249	static LIST_HEAD(undef_hook);
297	250	static DEFINE_RAW_SPINLOCK(undef_lock);
298	251
@@ -358,8 +311,8 @@ exit: · 59 unmodified lines
358	311	return fn ? fn(regs, instr) : 1;
359	312	}
360	313
361		−static void force_signal_inject(int signal, int code, struct pt_regs *regs,
362		− unsigned long address)
	314	+void force_signal_inject(int signal, int code, struct pt_regs *regs,
	315	+ unsigned long address)
363	316	{
364	317	siginfo_t info;
365	318	void __user pc = (void __user )instruction_pointer(regs);
@@ -373,7 +326,7 @@ static void force_signal_inject(int signal, int code, struct pt_regs *regs, · 7 unmodified lines
373	326	desc = "illegal memory access";
374	327	break;
375	328	default:
376		− desc = "bad mode";
	329	+ desc = "unknown or unrecoverable error";
377	330	break;
378	331	}
379	332
@@ -480,7 +433,7 @@ static void user_cache_maint_handler(unsigned int esr, struct pt_regs *regs) · 100 unmodified lines
480	433	if (ret)
481	434	arm64_notify_segfault(regs, address);
482	435	else
483		− regs->pc += 4;
	436	+ arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE);
484	437	}
485	438
486	439	static void ctr_read_handler(unsigned int esr, struct pt_regs *regs)
@@ -490,7 +443,7 @@ static void ctr_read_handler(unsigned int esr, struct pt_regs *regs) · 3 unmodified lines
490	443
491	444	pt_regs_write_reg(regs, rt, val);
492	445
493		− regs->pc += 4;
	446	+ arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE);
494	447	}
495	448
496	449	static void cntvct_read_handler(unsigned int esr, struct pt_regs *regs)
@@ -498,7 +451,7 @@ static void cntvct_read_handler(unsigned int esr, struct pt_regs *regs) · 1 unmodified line
498	451	int rt = (esr & ESR_ELx_SYS64_ISS_RT_MASK) >> ESR_ELx_SYS64_ISS_RT_SHIFT;
499	452
500	453	pt_regs_write_reg(regs, rt, arch_counter_get_cntvct());
501		− regs->pc += 4;
	454	+ arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE);
502	455	}
503	456
504	457	static void cntfrq_read_handler(unsigned int esr, struct pt_regs *regs)
@@ -506,7 +459,7 @@ static void cntfrq_read_handler(unsigned int esr, struct pt_regs *regs) · 1 unmodified line
506	459	int rt = (esr & ESR_ELx_SYS64_ISS_RT_MASK) >> ESR_ELx_SYS64_ISS_RT_SHIFT;
507	460
508	461	pt_regs_write_reg(regs, rt, arch_timer_get_rate());
509		− regs->pc += 4;
	462	+ arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE);
510	463	}
511	464
512	465	struct sys64_hook {
@@ -603,6 +556,7 @@ static const char *esr_class_str[] = { · 90 unmodified lines
603	556	[ESR_ELx_EC_HVC64] = "HVC (AArch64)",
604	557	[ESR_ELx_EC_SMC64] = "SMC (AArch64)",
605	558	[ESR_ELx_EC_SYS64] = "MSR/MRS (AArch64)",
	559	+ [ESR_ELx_EC_SVE] = "SVE",
606	560	[ESR_ELx_EC_IMP_DEF] = "EL3 IMP DEF",
607	561	[ESR_ELx_EC_IABT_LOW] = "IABT (lower EL)",
608	562	[ESR_ELx_EC_IABT_CUR] = "IABT (current EL)",
@@ -642,7 +596,7 @@ asmlinkage void bad_mode(struct pt_regs *regs, int reason, unsigned int esr) · 33 unmodified lines
642	596	esr_get_class_string(esr));
643	597
644	598	die("Oops - bad mode", regs, 0);
645		− local_irq_disable();
	599	+ local_daif_mask();
646	600	panic("bad mode");
647	601	}
648	602
@@ -708,6 +662,19 @@ asmlinkage void handle_bad_stack(struct pt_regs *regs) · 59 unmodified lines
708	662	}
709	663	#endif
710	664
	665	+asmlinkage void do_serror(struct pt_regs *regs, unsigned int esr)
	666	+{
	667	+ nmi_enter();
	668	+
	669	+ console_verbose();
	670	+
	671	+ pr_crit("SError Interrupt on CPU%d, code 0x%08x -- %s\n",
	672	+ smp_processor_id(), esr, esr_get_class_string(esr));
	673	+ __show_regs(regs);
	674	+
	675	+ panic("Asynchronous SError Interrupt");
	676	+}
	677	+
711	678	void __pte_error(const char *file, int line, unsigned long val)
712	679	{
713	680	pr_err("%s:%d: bad pte %016lx.\n", file, line, val);
@@ -761,7 +728,7 @@ static int bug_handler(struct pt_regs *regs, unsigned int esr) · 47 unmodified lines
761	728	}
762	729
763	730	/* If thread survives, skip over the BUG instruction and continue: */
764		− regs->pc += AARCH64_INSN_SIZE; /* skip BRK and resume */
	731	+ arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE);
765	732	return DBG_HOOK_HANDLED;
766	733	}
767	734

arch/arm64/kernel/vdso/gettimeofday.S+1−1

@@ -309,7 +309,7 @@ ENTRY(__kernel_clock_getres) · 308 unmodified lines
309	309	b.ne 4f
310	310	ldr x2, 6f
311	311	2:
312		− cbz w1, 3f
	312	+ cbz x1, 3f
313	313	stp xzr, x2, [x1]
314	314
315	315	3: /* res == NULL. */

arch/arm64/kvm/handle_exit.c+8−0

@@ -147,6 +147,13 @@ static int kvm_handle_unknown_ec(struct kvm_vcpu vcpu, struct kvm_run run) · 146 unmodified lines
147	147	return 1;
148	148	}
149	149
	150	+static int handle_sve(struct kvm_vcpu vcpu, struct kvm_run run)
	151	+{
	152	+ /* Until SVE is supported for guests: */
	153	+ kvm_inject_undefined(vcpu);
	154	+ return 1;
	155	+}
	156	+
150	157	static exit_handle_fn arm_exit_handlers[] = {
151	158	[0 ... ESR_ELx_EC_MAX] = kvm_handle_unknown_ec,
152	159	[ESR_ELx_EC_WFx] = kvm_handle_wfx,
@@ -160,6 +167,7 @@ static exit_handle_fn arm_exit_handlers[] = { · 7 unmodified lines
160	167	[ESR_ELx_EC_HVC64] = handle_hvc,
161	168	[ESR_ELx_EC_SMC64] = handle_smc,
162	169	[ESR_ELx_EC_SYS64] = kvm_handle_sys_reg,
	170	+ [ESR_ELx_EC_SVE] = handle_sve,
163	171	[ESR_ELx_EC_IABT_LOW] = kvm_handle_guest_abort,
164	172	[ESR_ELx_EC_DABT_LOW] = kvm_handle_guest_abort,
165	173	[ESR_ELx_EC_SOFTSTP_LOW]= kvm_handle_guest_debug,

arch/arm64/kvm/hyp/debug-sr.c+7−17

@@ -65,16 +65,6 @@ · 64 unmodified lines
65	65	default: write_debug(ptr[0], reg, 0); \
66	66	}
67	67
68		−#define PMSCR_EL1 sys_reg(3, 0, 9, 9, 0)
69		−
70		−#define PMBLIMITR_EL1 sys_reg(3, 0, 9, 10, 0)
71		−#define PMBLIMITR_EL1_E BIT(0)
72		−
73		−#define PMBIDR_EL1 sys_reg(3, 0, 9, 10, 7)
74		−#define PMBIDR_EL1_P BIT(4)
75		−
76		−#define psb_csync() asm volatile("hint #17")
77		−
78	68	static void __hyp_text __debug_save_spe_vhe(u64 *pmscr_el1)
79	69	{
80	70	/* The vcpu can run. but it can't hide. */
@@ -90,18 +80,18 @@ static void __hyp_text __debug_save_spe_nvhe(u64 *pmscr_el1) · 9 unmodified lines
90	80	return;
91	81
92	82	/* Yes; is it owned by EL3? */
93		− reg = read_sysreg_s(PMBIDR_EL1);
94		− if (reg & PMBIDR_EL1_P)
	83	+ reg = read_sysreg_s(SYS_PMBIDR_EL1);
	84	+ if (reg & BIT(SYS_PMBIDR_EL1_P_SHIFT))
95	85	return;
96	86
97	87	/* No; is the host actually using the thing? */
98		− reg = read_sysreg_s(PMBLIMITR_EL1);
99		− if (!(reg & PMBLIMITR_EL1_E))
	88	+ reg = read_sysreg_s(SYS_PMBLIMITR_EL1);
	89	+ if (!(reg & BIT(SYS_PMBLIMITR_EL1_E_SHIFT)))
100	90	return;
101	91
102	92	/* Yes; save the control register and disable data generation */
103		− *pmscr_el1 = read_sysreg_s(PMSCR_EL1);
104		− write_sysreg_s(0, PMSCR_EL1);
	93	+ *pmscr_el1 = read_sysreg_s(SYS_PMSCR_EL1);
	94	+ write_sysreg_s(0, SYS_PMSCR_EL1);
105	95	isb();
106	96
107	97	/* Now drain all buffered data to memory */
@@ -122,7 +112,7 @@ static void __hyp_text __debug_restore_spe(u64 pmscr_el1) · 14 unmodified lines
122	112	isb();
123	113
124	114	/* Re-enable data generation */
125		− write_sysreg_s(pmscr_el1, PMSCR_EL1);
	115	+ write_sysreg_s(pmscr_el1, SYS_PMSCR_EL1);
126	116	}
127	117
128	118	void __hyp_text __debug_save_state(struct kvm_vcpu *vcpu,

arch/arm64/kvm/hyp/switch.c+9−3

@@ -48,7 +48,7 @@ static void __hyp_text __activate_traps_vhe(void) · 47 unmodified lines
48	48
49	49	val = read_sysreg(cpacr_el1);
50	50	val \|= CPACR_EL1_TTA;
51		− val &= ~CPACR_EL1_FPEN;
	51	+ val &= ~(CPACR_EL1_FPEN \| CPACR_EL1_ZEN);
52	52	write_sysreg(val, cpacr_el1);
53	53
54	54	write_sysreg(__kvm_hyp_vector, vbar_el1);
@@ -59,7 +59,7 @@ static void __hyp_text __activate_traps_nvhe(void) · 4 unmodified lines
59	59	u64 val;
60	60
61	61	val = CPTR_EL2_DEFAULT;
62		− val \|= CPTR_EL2_TTA \| CPTR_EL2_TFP;
	62	+ val \|= CPTR_EL2_TTA \| CPTR_EL2_TFP \| CPTR_EL2_TZ;
63	63	write_sysreg(val, cptr_el2);
64	64	}
65	65
@@ -81,11 +81,17 @@ static void __hyp_text __activate_traps(struct kvm_vcpu *vcpu) · 15 unmodified lines
81	81	* it will cause an exception.
82	82	*/
83	83	val = vcpu->arch.hcr_el2;
	84	+
84	85	if (!(val & HCR_RW) && system_supports_fpsimd()) {
85	86	write_sysreg(1 << 30, fpexc32_el2);
86	87	isb();
87	88	}
	89	+
	90	+ if (val & HCR_RW) /* for AArch64 only: */
	91	+ val \|= HCR_TID3; /* TID3: trap feature register accesses */
	92	+
88	93	write_sysreg(val, hcr_el2);
	94	+
89	95	/* Trap on AArch32 cp15 c15 accesses (EL1 or EL0) */
90	96	write_sysreg(1 << 15, hstr_el2);
91	97	/*
@@ -111,7 +117,7 @@ static void __hyp_text __deactivate_traps_vhe(void) · 19 unmodified lines
111	117
112	118	write_sysreg(mdcr_el2, mdcr_el2);
113	119	write_sysreg(HCR_HOST_VHE_FLAGS, hcr_el2);
114		− write_sysreg(CPACR_EL1_FPEN, cpacr_el1);
	120	+ write_sysreg(CPACR_EL1_DEFAULT, cpacr_el1);
115	121	write_sysreg(vectors, vbar_el1);
116	122	}
117	123

arch/arm64/kvm/sys_regs.c+247−45

@@ -23,6 +23,7 @@ · 22 unmodified lines
23	23	#include <linux/bsearch.h>
24	24	#include <linux/kvm_host.h>
25	25	#include <linux/mm.h>
	26	+#include <linux/printk.h>
26	27	#include <linux/uaccess.h>
27	28
28	29	#include <asm/cacheflush.h>
@@ -892,6 +893,146 @@ static bool access_cntp_cval(struct kvm_vcpu *vcpu, · 863 unmodified lines
892	893	return true;
893	894	}
894	895
	896	+/* Read a sanitised cpufeature ID register by sys_reg_desc */
	897	+static u64 read_id_reg(struct sys_reg_desc const *r, bool raz)
	898	+{
	899	+ u32 id = sys_reg((u32)r->Op0, (u32)r->Op1,
	900	+ (u32)r->CRn, (u32)r->CRm, (u32)r->Op2);
	901	+ u64 val = raz ? 0 : read_sanitised_ftr_reg(id);
	902	+
	903	+ if (id == SYS_ID_AA64PFR0_EL1) {
	904	+ if (val & (0xfUL << ID_AA64PFR0_SVE_SHIFT))
	905	+ pr_err_once("kvm [%i]: SVE unsupported for guests, suppressing\n",
	906	+ task_pid_nr(current));
	907	+
	908	+ val &= ~(0xfUL << ID_AA64PFR0_SVE_SHIFT);
	909	+ }
	910	+
	911	+ return val;
	912	+}
	913	+
	914	+/* cpufeature ID register access trap handlers */
	915	+
	916	+static bool __access_id_reg(struct kvm_vcpu *vcpu,
	917	+ struct sys_reg_params *p,
	918	+ const struct sys_reg_desc *r,
	919	+ bool raz)
	920	+{
	921	+ if (p->is_write)
	922	+ return write_to_read_only(vcpu, p, r);
	923	+
	924	+ p->regval = read_id_reg(r, raz);
	925	+ return true;
	926	+}
	927	+
	928	+static bool access_id_reg(struct kvm_vcpu *vcpu,
	929	+ struct sys_reg_params *p,
	930	+ const struct sys_reg_desc *r)
	931	+{
	932	+ return __access_id_reg(vcpu, p, r, false);
	933	+}
	934	+
	935	+static bool access_raz_id_reg(struct kvm_vcpu *vcpu,
	936	+ struct sys_reg_params *p,
	937	+ const struct sys_reg_desc *r)
	938	+{
	939	+ return __access_id_reg(vcpu, p, r, true);
	940	+}
	941	+
	942	+static int reg_from_user(u64 val, const void __user uaddr, u64 id);
	943	+static int reg_to_user(void __user uaddr, const u64 val, u64 id);
	944	+static u64 sys_reg_to_index(const struct sys_reg_desc *reg);
	945	+
	946	+/*
	947	+ * cpufeature ID register user accessors
	948	+ *
	949	+ * For now, these registers are immutable for userspace, so no values
	950	+ * are stored, and for set_id_reg() we don't allow the effective value
	951	+ * to be changed.
	952	+ */
	953	+static int __get_id_reg(const struct sys_reg_desc rd, void __user uaddr,
	954	+ bool raz)
	955	+{
	956	+ const u64 id = sys_reg_to_index(rd);
	957	+ const u64 val = read_id_reg(rd, raz);
	958	+
	959	+ return reg_to_user(uaddr, &val, id);
	960	+}
	961	+
	962	+static int __set_id_reg(const struct sys_reg_desc rd, void __user uaddr,
	963	+ bool raz)
	964	+{
	965	+ const u64 id = sys_reg_to_index(rd);
	966	+ int err;
	967	+ u64 val;
	968	+
	969	+ err = reg_from_user(&val, uaddr, id);
	970	+ if (err)
	971	+ return err;
	972	+
	973	+ /* This is what we mean by invariant: you can't change it. */
	974	+ if (val != read_id_reg(rd, raz))
	975	+ return -EINVAL;
	976	+
	977	+ return 0;
	978	+}
	979	+
	980	+static int get_id_reg(struct kvm_vcpu vcpu, const struct sys_reg_desc rd,
	981	+ const struct kvm_one_reg reg, void __user uaddr)
	982	+{
	983	+ return __get_id_reg(rd, uaddr, false);
	984	+}
	985	+
	986	+static int set_id_reg(struct kvm_vcpu vcpu, const struct sys_reg_desc rd,
	987	+ const struct kvm_one_reg reg, void __user uaddr)
	988	+{
	989	+ return __set_id_reg(rd, uaddr, false);
	990	+}
	991	+
	992	+static int get_raz_id_reg(struct kvm_vcpu vcpu, const struct sys_reg_desc rd,
	993	+ const struct kvm_one_reg reg, void __user uaddr)
	994	+{
	995	+ return __get_id_reg(rd, uaddr, true);
	996	+}
	997	+
	998	+static int set_raz_id_reg(struct kvm_vcpu vcpu, const struct sys_reg_desc rd,
	999	+ const struct kvm_one_reg reg, void __user uaddr)
	1000	+{
	1001	+ return __set_id_reg(rd, uaddr, true);
	1002	+}
	1003	+
	1004	+/* sys_reg_desc initialiser for known cpufeature ID registers */
	1005	+#define ID_SANITISED(name) { \
	1006	+ SYS_DESC(SYS_##name), \
	1007	+ .access = access_id_reg, \
	1008	+ .get_user = get_id_reg, \
	1009	+ .set_user = set_id_reg, \
	1010	+}
	1011	+
	1012	+/*
	1013	+ * sys_reg_desc initialiser for architecturally unallocated cpufeature ID
	1014	+ * register with encoding Op0=3, Op1=0, CRn=0, CRm=crm, Op2=op2
	1015	+ * (1 <= crm < 8, 0 <= Op2 < 8).
	1016	+ */
	1017	+#define ID_UNALLOCATED(crm, op2) { \
	1018	+ Op0(3), Op1(0), CRn(0), CRm(crm), Op2(op2), \
	1019	+ .access = access_raz_id_reg, \
	1020	+ .get_user = get_raz_id_reg, \
	1021	+ .set_user = set_raz_id_reg, \
	1022	+}
	1023	+
	1024	+/*
	1025	+ * sys_reg_desc initialiser for known ID registers that we hide from guests.
	1026	+ * For now, these are exposed just like unallocated ID regs: they appear
	1027	+ * RAZ for the guest.
	1028	+ */
	1029	+#define ID_HIDDEN(name) { \
	1030	+ SYS_DESC(SYS_##name), \
	1031	+ .access = access_raz_id_reg, \
	1032	+ .get_user = get_raz_id_reg, \
	1033	+ .set_user = set_raz_id_reg, \
	1034	+}
	1035	+
895	1036	/*
896	1037	* Architected system registers.
897	1038	* Important: Must be sorted ascending by Op0, Op1, CRn, CRm, Op2
@@ -944,6 +1085,84 @@ static const struct sys_reg_desc sys_reg_descs[] = { · 46 unmodified lines
944	1085	{ SYS_DESC(SYS_DBGVCR32_EL2), NULL, reset_val, DBGVCR32_EL2, 0 },
945	1086
946	1087	{ SYS_DESC(SYS_MPIDR_EL1), NULL, reset_mpidr, MPIDR_EL1 },
	1088	+
	1089	+ /*
	1090	+ * ID regs: all ID_SANITISED() entries here must have corresponding
	1091	+ * entries in arm64_ftr_regs[].
	1092	+ */
	1093	+
	1094	+ /* AArch64 mappings of the AArch32 ID registers */
	1095	+ /* CRm=1 */
	1096	+ ID_SANITISED(ID_PFR0_EL1),
	1097	+ ID_SANITISED(ID_PFR1_EL1),
	1098	+ ID_SANITISED(ID_DFR0_EL1),
	1099	+ ID_HIDDEN(ID_AFR0_EL1),
	1100	+ ID_SANITISED(ID_MMFR0_EL1),
	1101	+ ID_SANITISED(ID_MMFR1_EL1),
	1102	+ ID_SANITISED(ID_MMFR2_EL1),
	1103	+ ID_SANITISED(ID_MMFR3_EL1),
	1104	+
	1105	+ /* CRm=2 */
	1106	+ ID_SANITISED(ID_ISAR0_EL1),
	1107	+ ID_SANITISED(ID_ISAR1_EL1),
	1108	+ ID_SANITISED(ID_ISAR2_EL1),
	1109	+ ID_SANITISED(ID_ISAR3_EL1),
	1110	+ ID_SANITISED(ID_ISAR4_EL1),
	1111	+ ID_SANITISED(ID_ISAR5_EL1),
	1112	+ ID_SANITISED(ID_MMFR4_EL1),
	1113	+ ID_UNALLOCATED(2,7),
	1114	+
	1115	+ /* CRm=3 */
	1116	+ ID_SANITISED(MVFR0_EL1),
	1117	+ ID_SANITISED(MVFR1_EL1),
	1118	+ ID_SANITISED(MVFR2_EL1),
	1119	+ ID_UNALLOCATED(3,3),
	1120	+ ID_UNALLOCATED(3,4),
	1121	+ ID_UNALLOCATED(3,5),
	1122	+ ID_UNALLOCATED(3,6),
	1123	+ ID_UNALLOCATED(3,7),
	1124	+
	1125	+ /* AArch64 ID registers */
	1126	+ /* CRm=4 */
	1127	+ ID_SANITISED(ID_AA64PFR0_EL1),
	1128	+ ID_SANITISED(ID_AA64PFR1_EL1),
	1129	+ ID_UNALLOCATED(4,2),
	1130	+ ID_UNALLOCATED(4,3),
	1131	+ ID_UNALLOCATED(4,4),
	1132	+ ID_UNALLOCATED(4,5),
	1133	+ ID_UNALLOCATED(4,6),
	1134	+ ID_UNALLOCATED(4,7),
	1135	+
	1136	+ /* CRm=5 */
	1137	+ ID_SANITISED(ID_AA64DFR0_EL1),
	1138	+ ID_SANITISED(ID_AA64DFR1_EL1),
	1139	+ ID_UNALLOCATED(5,2),
	1140	+ ID_UNALLOCATED(5,3),
	1141	+ ID_HIDDEN(ID_AA64AFR0_EL1),
	1142	+ ID_HIDDEN(ID_AA64AFR1_EL1),
	1143	+ ID_UNALLOCATED(5,6),
	1144	+ ID_UNALLOCATED(5,7),
	1145	+
	1146	+ /* CRm=6 */
	1147	+ ID_SANITISED(ID_AA64ISAR0_EL1),
	1148	+ ID_SANITISED(ID_AA64ISAR1_EL1),
	1149	+ ID_UNALLOCATED(6,2),
	1150	+ ID_UNALLOCATED(6,3),
	1151	+ ID_UNALLOCATED(6,4),
	1152	+ ID_UNALLOCATED(6,5),
	1153	+ ID_UNALLOCATED(6,6),
	1154	+ ID_UNALLOCATED(6,7),
	1155	+
	1156	+ /* CRm=7 */
	1157	+ ID_SANITISED(ID_AA64MMFR0_EL1),
	1158	+ ID_SANITISED(ID_AA64MMFR1_EL1),
	1159	+ ID_SANITISED(ID_AA64MMFR2_EL1),
	1160	+ ID_UNALLOCATED(7,3),
	1161	+ ID_UNALLOCATED(7,4),
	1162	+ ID_UNALLOCATED(7,5),
	1163	+ ID_UNALLOCATED(7,6),
	1164	+ ID_UNALLOCATED(7,7),
	1165	+
947	1166	{ SYS_DESC(SYS_SCTLR_EL1), access_vm_reg, reset_val, SCTLR_EL1, 0x00C50078 },
948	1167	{ SYS_DESC(SYS_CPACR_EL1), NULL, reset_val, CPACR_EL1, 0 },
949	1168	{ SYS_DESC(SYS_TTBR0_EL1), access_vm_reg, reset_unknown, TTBR0_EL1 },
@@ -1790,8 +2009,8 @@ static const struct sys_reg_desc index_to_sys_reg_desc(struct kvm_vcpu vcpu, · 840 unmodified lines
1790	2009	if (!r)
1791	2010	r = find_reg(&params, sys_reg_descs, ARRAY_SIZE(sys_reg_descs));
1792	2011
1793		− /* Not saved in the sys_reg array? */
1794		− if (r && !r->reg)
	2012	+ /* Not saved in the sys_reg array and not otherwise accessible? */
	2013	+ if (r && !(r->reg \|\| r->get_user))
1795	2014	r = NULL;
1796	2015
1797	2016	return r;
@@ -1815,20 +2034,6 @@ static const struct sys_reg_desc index_to_sys_reg_desc(struct kvm_vcpu vcpu, · 17 unmodified lines
1815	2034	FUNCTION_INVARIANT(midr_el1)
1816	2035	FUNCTION_INVARIANT(ctr_el0)
1817	2036	FUNCTION_INVARIANT(revidr_el1)
1818		−FUNCTION_INVARIANT(id_pfr0_el1)
1819		−FUNCTION_INVARIANT(id_pfr1_el1)
1820		−FUNCTION_INVARIANT(id_dfr0_el1)
1821		−FUNCTION_INVARIANT(id_afr0_el1)
1822		−FUNCTION_INVARIANT(id_mmfr0_el1)
1823		−FUNCTION_INVARIANT(id_mmfr1_el1)
1824		−FUNCTION_INVARIANT(id_mmfr2_el1)
1825		−FUNCTION_INVARIANT(id_mmfr3_el1)
1826		−FUNCTION_INVARIANT(id_isar0_el1)
1827		−FUNCTION_INVARIANT(id_isar1_el1)
1828		−FUNCTION_INVARIANT(id_isar2_el1)
1829		−FUNCTION_INVARIANT(id_isar3_el1)
1830		−FUNCTION_INVARIANT(id_isar4_el1)
1831		−FUNCTION_INVARIANT(id_isar5_el1)
1832	2037	FUNCTION_INVARIANT(clidr_el1)
1833	2038	FUNCTION_INVARIANT(aidr_el1)
1834	2039
@@ -1836,20 +2041,6 @@ FUNCTION_INVARIANT(aidr_el1) · 1 unmodified line
1836	2041	static struct sys_reg_desc invariant_sys_regs[] = {
1837	2042	{ SYS_DESC(SYS_MIDR_EL1), NULL, get_midr_el1 },
1838	2043	{ SYS_DESC(SYS_REVIDR_EL1), NULL, get_revidr_el1 },
1839		− { SYS_DESC(SYS_ID_PFR0_EL1), NULL, get_id_pfr0_el1 },
1840		− { SYS_DESC(SYS_ID_PFR1_EL1), NULL, get_id_pfr1_el1 },
1841		− { SYS_DESC(SYS_ID_DFR0_EL1), NULL, get_id_dfr0_el1 },
1842		− { SYS_DESC(SYS_ID_AFR0_EL1), NULL, get_id_afr0_el1 },
1843		− { SYS_DESC(SYS_ID_MMFR0_EL1), NULL, get_id_mmfr0_el1 },
1844		− { SYS_DESC(SYS_ID_MMFR1_EL1), NULL, get_id_mmfr1_el1 },
1845		− { SYS_DESC(SYS_ID_MMFR2_EL1), NULL, get_id_mmfr2_el1 },
1846		− { SYS_DESC(SYS_ID_MMFR3_EL1), NULL, get_id_mmfr3_el1 },
1847		− { SYS_DESC(SYS_ID_ISAR0_EL1), NULL, get_id_isar0_el1 },
1848		− { SYS_DESC(SYS_ID_ISAR1_EL1), NULL, get_id_isar1_el1 },
1849		− { SYS_DESC(SYS_ID_ISAR2_EL1), NULL, get_id_isar2_el1 },
1850		− { SYS_DESC(SYS_ID_ISAR3_EL1), NULL, get_id_isar3_el1 },
1851		− { SYS_DESC(SYS_ID_ISAR4_EL1), NULL, get_id_isar4_el1 },
1852		− { SYS_DESC(SYS_ID_ISAR5_EL1), NULL, get_id_isar5_el1 },
1853	2044	{ SYS_DESC(SYS_CLIDR_EL1), NULL, get_clidr_el1 },
1854	2045	{ SYS_DESC(SYS_AIDR_EL1), NULL, get_aidr_el1 },
1855	2046	{ SYS_DESC(SYS_CTR_EL0), NULL, get_ctr_el0 },
@@ -2079,12 +2270,31 @@ static bool copy_reg_to_user(const struct sys_reg_desc reg, u64 __user *uind) · 223 unmodified lines
2079	2270	return true;
2080	2271	}
2081	2272
	2273	+static int walk_one_sys_reg(const struct sys_reg_desc *rd,
	2274	+ u64 __user **uind,
	2275	+ unsigned int *total)
	2276	+{
	2277	+ /*
	2278	+ * Ignore registers we trap but don't save,
	2279	+ * and for which no custom user accessor is provided.
	2280	+ */
	2281	+ if (!(rd->reg \|\| rd->get_user))
	2282	+ return 0;
	2283	+
	2284	+ if (!copy_reg_to_user(rd, uind))
	2285	+ return -EFAULT;
	2286	+
	2287	+ (*total)++;
	2288	+ return 0;
	2289	+}
	2290	+
2082	2291	/* Assumed ordered tables, see kvm_sys_reg_table_init. */
2083	2292	static int walk_sys_regs(struct kvm_vcpu vcpu, u64 __user uind)
2084	2293	{
2085	2294	const struct sys_reg_desc i1, i2, end1, end2;
2086	2295	unsigned int total = 0;
2087	2296	size_t num;
	2297	+ int err;
2088	2298
2089	2299	/* We check for duplicates here, to allow arch-specific overrides. */
2090	2300	i1 = get_target_table(vcpu->arch.target, true, &num);
@@ -2098,21 +2308,13 @@ static int walk_sys_regs(struct kvm_vcpu vcpu, u64 __user uind) · 7 unmodified lines
2098	2308	while (i1 \|\| i2) {
2099	2309	int cmp = cmp_sys_reg(i1, i2);
2100	2310	/* target-specific overrides generic entry. */
2101		− if (cmp <= 0) {
2102		− /* Ignore registers we trap but don't save. */
2103		− if (i1->reg) {
2104		− if (!copy_reg_to_user(i1, &uind))
2105		− return -EFAULT;
2106		− total++;
2107		− }
2108		− } else {
2109		− /* Ignore registers we trap but don't save. */
2110		− if (i2->reg) {
2111		− if (!copy_reg_to_user(i2, &uind))
2112		− return -EFAULT;
2113		− total++;
2114		− }
2115		− }
	2311	+ if (cmp <= 0)
	2312	+ err = walk_one_sys_reg(i1, &uind, &total);
	2313	+ else
	2314	+ err = walk_one_sys_reg(i2, &uind, &total);
	2315	+
	2316	+ if (err)
	2317	+ return err;
2116	2318
2117	2319	if (cmp <= 0 && ++i1 == end1)
2118	2320	i1 = NULL;

arch/arm64/lib/Makefile+1−1

@@ -3,7 +3,7 @@ lib-y := bitops.o clear_user.o delay.o copy_from_user.o \ · 2 unmodified lines
3	3	copy_to_user.o copy_in_user.o copy_page.o \
4	4	clear_page.o memchr.o memcpy.o memmove.o memset.o \
5	5	memcmp.o strcmp.o strncmp.o strlen.o strnlen.o \
6		− strchr.o strrchr.o
	6	+ strchr.o strrchr.o tishift.o
7	7
8	8	# Tell the compiler to treat all general purpose registers (with the
9	9	# exception of the IP registers, which are already handled by the caller

arch/arm64/lib/delay.c+19−4

@@ -24,10 +24,28 @@ · 23 unmodified lines
24	24	#include <linux/module.h>
25	25	#include <linux/timex.h>
26	26
	27	+#include <clocksource/arm_arch_timer.h>
	28	+
	29	+#define USECS_TO_CYCLES(time_usecs) \
	30	+ xloops_to_cycles((time_usecs) * 0x10C7UL)
	31	+
	32	+static inline unsigned long xloops_to_cycles(unsigned long xloops)
	33	+{
	34	+ return (xloops * loops_per_jiffy * HZ) >> 32;
	35	+}
	36	+
27	37	void __delay(unsigned long cycles)
28	38	{
29	39	cycles_t start = get_cycles();
30	40
	41	+ if (arch_timer_evtstrm_available()) {
	42	+ const cycles_t timer_evt_period =
	43	+ USECS_TO_CYCLES(ARCH_TIMER_EVT_STREAM_PERIOD_US);
	44	+
	45	+ while ((get_cycles() - start + timer_evt_period) < cycles)
	46	+ wfe();
	47	+ }
	48	+
31	49	while ((get_cycles() - start) < cycles)
32	50	cpu_relax();
33	51	}
@@ -35,10 +53,7 @@ EXPORT_SYMBOL(__delay); · 1 unmodified line
35	53
36	54	inline void __const_udelay(unsigned long xloops)
37	55	{
38		− unsigned long loops;
39		−
40		− loops = xloops * loops_per_jiffy * HZ;
41		− __delay(loops >> 32);
	56	+ __delay(xloops_to_cycles(xloops));
42	57	}
43	58	EXPORT_SYMBOL(__const_udelay);
44	59

arch/arm64/lib/tishift.S+80−0

@@ -0,0 +1,80 @@
	1	+/*
	2	+ * Copyright (C) 2017 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
	3	+ *
	4	+ * This program is free software; you can redistribute it and/or modify
	5	+ * it under the terms of the GNU General Public License version 2 as
	6	+ * published by the Free Software Foundation.
	7	+ *
	8	+ * This program is distributed in the hope that it will be useful,
	9	+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
	10	+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	11	+ * GNU General Public License for more details.
	12	+ *
	13	+ * You should have received a copy of the GNU General Public License
	14	+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
	15	+ */
	16	+
	17	+#include <linux/linkage.h>
	18	+
	19	+ENTRY(__ashlti3)
	20	+ cbz x2, 1f
	21	+ mov x3, #64
	22	+ sub x3, x3, x2
	23	+ cmp x3, #0
	24	+ b.le 2f
	25	+ lsl x1, x1, x2
	26	+ lsr x3, x0, x3
	27	+ lsl x2, x0, x2
	28	+ orr x1, x1, x3
	29	+ mov x0, x2
	30	+1:
	31	+ ret
	32	+2:
	33	+ neg w1, w3
	34	+ mov x2, #0
	35	+ lsl x1, x0, x1
	36	+ mov x0, x2
	37	+ ret
	38	+ENDPROC(__ashlti3)
	39	+
	40	+ENTRY(__ashrti3)
	41	+ cbz x2, 3f
	42	+ mov x3, #64
	43	+ sub x3, x3, x2
	44	+ cmp x3, #0
	45	+ b.le 4f
	46	+ lsr x0, x0, x2
	47	+ lsl x3, x1, x3
	48	+ asr x2, x1, x2
	49	+ orr x0, x0, x3
	50	+ mov x1, x2
	51	+3:
	52	+ ret
	53	+4:
	54	+ neg w0, w3
	55	+ asr x2, x1, #63
	56	+ asr x0, x1, x0
	57	+ mov x1, x2
	58	+ ret
	59	+ENDPROC(__ashrti3)
	60	+
	61	+ENTRY(__lshrti3)
	62	+ cbz x2, 1f
	63	+ mov x3, #64
	64	+ sub x3, x3, x2
	65	+ cmp x3, #0
	66	+ b.le 2f
	67	+ lsr x0, x0, x2
	68	+ lsl x3, x1, x3
	69	+ lsr x2, x1, x2
	70	+ orr x0, x0, x3
	71	+ mov x1, x2
	72	+1:
	73	+ ret
	74	+2:
	75	+ neg w0, w3
	76	+ mov x2, #0
	77	+ lsr x0, x1, x0
	78	+ mov x1, x2
	79	+ ret
	80	+ENDPROC(__lshrti3)

arch/arm64/mm/dma-mapping.c+2−3

@@ -166,7 +166,7 @@ static void __dma_alloc(struct device dev, size_t size, · 165 unmodified lines
166	166	/* create a coherent mapping */
167	167	page = virt_to_page(ptr);
168	168	coherent_ptr = dma_common_contiguous_remap(page, size, VM_USERMAP,
169		− prot, NULL);
	169	+ prot, __builtin_return_address(0));
170	170	if (!coherent_ptr)
171	171	goto no_map;
172	172
@@ -303,8 +303,7 @@ static int __swiotlb_mmap_pfn(struct vm_area_struct *vma, · 130 unmodified lines
303	303	unsigned long pfn, size_t size)
304	304	{
305	305	int ret = -ENXIO;
306		− unsigned long nr_vma_pages = (vma->vm_end - vma->vm_start) >>
307		− PAGE_SHIFT;
	306	+ unsigned long nr_vma_pages = vma_pages(vma);
308	307	unsigned long nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT;
309	308	unsigned long off = vma->vm_pgoff;
310	309

arch/arm64/mm/fault.c+13−59

@@ -105,13 +105,11 @@ static void data_abort_decode(unsigned int esr) · 104 unmodified lines
105	105	(esr & ESR_ELx_WNR) >> ESR_ELx_WNR_SHIFT);
106	106	}
107	107
108		−/*
109		− * Decode mem abort information
110		− */
111	108	static void mem_abort_decode(unsigned int esr)
112	109	{
113	110	pr_alert("Mem abort info:\n");
114	111
	112	+ pr_alert(" ESR = 0x%08x\n", esr);
115	113	pr_alert(" Exception class = %s, IL = %u bits\n",
116	114	esr_get_class_string(esr),
117	115	(esr & ESR_ELx_IL) ? 32 : 16);
@@ -249,9 +247,6 @@ static inline bool is_permission_fault(unsigned int esr, struct pt_regs *regs, · 131 unmodified lines
249	247	return false;
250	248	}
251	249
252		−/*
253		− * The kernel tried to access some page that wasn't present.
254		− */
255	250	static void __do_kernel_fault(unsigned long addr, unsigned int esr,
256	251	struct pt_regs *regs)
257	252	{
@@ -264,9 +259,6 @@ static void __do_kernel_fault(unsigned long addr, unsigned int esr, · 6 unmodified lines
264	259	if (!is_el1_instruction_abort(esr) && fixup_exception(regs))
265	260	return;
266	261
267		− /*
268		− * No handler, we'll have to terminate things with extreme prejudice.
269		− */
270	262	bust_spinlocks(1);
271	263
272	264	if (is_permission_fault(esr, regs, addr)) {
@@ -291,10 +283,6 @@ static void __do_kernel_fault(unsigned long addr, unsigned int esr, · 18 unmodified lines
291	283	do_exit(SIGKILL);
292	284	}
293	285
294		−/*
295		− * Something tried to access memory that isn't in our memory map. User mode
296		− * accesses just cause a SIGSEGV
297		− */
298	286	static void __do_user_fault(struct task_struct *tsk, unsigned long addr,
299	287	unsigned int esr, unsigned int sig, int code,
300	288	struct pt_regs *regs, int fault)
@@ -559,23 +547,6 @@ no_context: · 258 unmodified lines
559	547	return 0;
560	548	}
561	549
562		−/*
563		− * First Level Translation Fault Handler
564		− *
565		− * We enter here because the first level page table doesn't contain a valid
566		− * entry for the address.
567		− *
568		− * If the address is in kernel space (>= TASK_SIZE), then we are probably
569		− * faulting in the vmalloc() area.
570		− *
571		− * If the init_task's first level page tables contains the relevant entry, we
572		− * copy the it to this task. If not, we send the process a signal, fixup the
573		− * exception, or oops the kernel.
574		− *
575		− * NOTE! We MUST NOT take any locks for this case. We may be in an interrupt
576		− * or a critical region, and should only copy the information from the master
577		− * page table, nothing more.
578		− */
579	550	static int __kprobes do_translation_fault(unsigned long addr,
580	551	unsigned int esr,
581	552	struct pt_regs *regs)
@@ -594,18 +565,11 @@ static int do_alignment_fault(unsigned long addr, unsigned int esr, · 12 unmodified lines
594	565	return 0;
595	566	}
596	567
597		−/*
598		− * This abort handler always returns "fault".
599		− */
600	568	static int do_bad(unsigned long addr, unsigned int esr, struct pt_regs *regs)
601	569	{
602		− return 1;
	570	+ return 1; /* "fault" */
603	571	}
604	572
605		−/*
606		− * This abort handler deals with Synchronous External Abort.
607		− * It calls notifiers, and then returns "fault".
608		− */
609	573	static int do_sea(unsigned long addr, unsigned int esr, struct pt_regs *regs)
610	574	{
611	575	struct siginfo info;
@@ -668,14 +632,14 @@ static const struct fault_info fault_info[] = { · 56 unmodified lines
668	632	{ do_sea, SIGBUS, 0, "level 1 (translation table walk)" },
669	633	{ do_sea, SIGBUS, 0, "level 2 (translation table walk)" },
670	634	{ do_sea, SIGBUS, 0, "level 3 (translation table walk)" },
671		− { do_sea, SIGBUS, 0, "synchronous parity or ECC error" },
	635	+ { do_sea, SIGBUS, 0, "synchronous parity or ECC error" }, // Reserved when RAS is implemented
672	636	{ do_bad, SIGBUS, 0, "unknown 25" },
673	637	{ do_bad, SIGBUS, 0, "unknown 26" },
674	638	{ do_bad, SIGBUS, 0, "unknown 27" },
675		− { do_sea, SIGBUS, 0, "level 0 synchronous parity error (translation table walk)" },
676		− { do_sea, SIGBUS, 0, "level 1 synchronous parity error (translation table walk)" },
677		− { do_sea, SIGBUS, 0, "level 2 synchronous parity error (translation table walk)" },
678		− { do_sea, SIGBUS, 0, "level 3 synchronous parity error (translation table walk)" },
	639	+ { do_sea, SIGBUS, 0, "level 0 synchronous parity error (translation table walk)" }, // Reserved when RAS is implemented
	640	+ { do_sea, SIGBUS, 0, "level 1 synchronous parity error (translation table walk)" }, // Reserved when RAS is implemented
	641	+ { do_sea, SIGBUS, 0, "level 2 synchronous parity error (translation table walk)" }, // Reserved when RAS is implemented
	642	+ { do_sea, SIGBUS, 0, "level 3 synchronous parity error (translation table walk)" }, // Reserved when RAS is implemented
679	643	{ do_bad, SIGBUS, 0, "unknown 32" },
680	644	{ do_alignment_fault, SIGBUS, BUS_ADRALN, "alignment fault" },
681	645	{ do_bad, SIGBUS, 0, "unknown 34" },
@@ -693,7 +657,7 @@ static const struct fault_info fault_info[] = { · 11 unmodified lines
693	657	{ do_bad, SIGBUS, 0, "unknown 46" },
694	658	{ do_bad, SIGBUS, 0, "unknown 47" },
695	659	{ do_bad, SIGBUS, 0, "TLB conflict abort" },
696		− { do_bad, SIGBUS, 0, "unknown 49" },
	660	+ { do_bad, SIGBUS, 0, "Unsupported atomic hardware update fault" },
697	661	{ do_bad, SIGBUS, 0, "unknown 50" },
698	662	{ do_bad, SIGBUS, 0, "unknown 51" },
699	663	{ do_bad, SIGBUS, 0, "implementation fault (lockdown abort)" },
@@ -710,13 +674,6 @@ static const struct fault_info fault_info[] = { · 10 unmodified lines
710	674	{ do_bad, SIGBUS, 0, "unknown 63" },
711	675	};
712	676
713		−/*
714		− * Handle Synchronous External Aborts that occur in a guest kernel.
715		− *
716		− * The return value will be zero if the SEA was successfully handled
717		− * and non-zero if there was an error processing the error or there was
718		− * no error to process.
719		− */
720	677	int handle_guest_sea(phys_addr_t addr, unsigned int esr)
721	678	{
722	679	int ret = -ENOENT;
@@ -727,9 +684,6 @@ int handle_guest_sea(phys_addr_t addr, unsigned int esr) · 4 unmodified lines
727	684	return ret;
728	685	}
729	686
730		−/*
731		− * Dispatch a data abort to the relevant handler.
732		− */
733	687	asmlinkage void __exception do_mem_abort(unsigned long addr, unsigned int esr,
734	688	struct pt_regs *regs)
735	689	{
@@ -739,11 +693,14 @@ asmlinkage void __exception do_mem_abort(unsigned long addr, unsigned int esr, · 3 unmodified lines
739	693	if (!inf->fn(addr, esr, regs))
740	694	return;
741	695
742		− pr_alert("Unhandled fault: %s (0x%08x) at 0x%016lx\n",
743		− inf->name, esr, addr);
	696	+ pr_alert("Unhandled fault: %s at 0x%016lx\n",
	697	+ inf->name, addr);
744	698
745	699	mem_abort_decode(esr);
746	700
	701	+ if (!user_mode(regs))
	702	+ show_pte(addr);
	703	+
747	704	info.si_signo = inf->sig;
748	705	info.si_errno = 0;
749	706	info.si_code = inf->code;
@@ -751,9 +708,6 @@ asmlinkage void __exception do_mem_abort(unsigned long addr, unsigned int esr, · 1 unmodified line
751	708	arm64_notify_die("", regs, &info, esr);
752	709	}
753	710
754		−/*
755		− * Handle stack alignment exceptions.
756		− */
757	711	asmlinkage void __exception do_sp_pc_abort(unsigned long addr,
758	712	unsigned int esr,
759	713	struct pt_regs *regs)

arch/arm64/mm/proc.S+4−5

@@ -109,10 +109,10 @@ ENTRY(cpu_do_resume) · 108 unmodified lines
109	109	/*
110	110	* __cpu_setup() cleared MDSCR_EL1.MDE and friends, before unmasking
111	111	* debug exceptions. By restoring MDSCR_EL1 here, we may take a debug
112		− * exception. Mask them until local_dbg_restore() in cpu_suspend()
	112	+ * exception. Mask them until local_daif_restore() in cpu_suspend()
113	113	* resets them.
114	114	*/
115		− disable_dbg
	115	+ disable_daif
116	116	msr mdscr_el1, x10
117	117
118	118	msr sctlr_el1, x12
@@ -155,8 +155,7 @@ ENDPROC(cpu_do_switch_mm) · 36 unmodified lines
155	155	* called by anything else. It can only be executed from a TTBR0 mapping.
156	156	*/
157	157	ENTRY(idmap_cpu_replace_ttbr1)
158		− mrs x2, daif
159		− msr daifset, #0xf
	158	+ save_and_disable_daif flags=x2
160	159
161	160	adrp x1, empty_zero_page
162	161	msr ttbr1_el1, x1
@@ -169,7 +168,7 @@ ENTRY(idmap_cpu_replace_ttbr1) · 6 unmodified lines
169	168	msr ttbr1_el1, x0
170	169	isb
171	170
172		− msr daif, x2
	171	+ restore_daif x2
173	172
174	173	ret
175	174	ENDPROC(idmap_cpu_replace_ttbr1)

drivers/acpi/arm64/gtdt.c+1−1

@@ -199,7 +199,7 @@ static int __init gtdt_parse_timer_block(struct acpi_gtdt_timer_block *block, · 198 unmodified lines
199	199	struct acpi_gtdt_timer_entry *gtdt_frame;
200	200
201	201	if (!block->timer_count) {
202		− pr_err(FW_BUG "GT block present, but frame count is zero.");
	202	+ pr_err(FW_BUG "GT block present, but frame count is zero.\n");
203	203	return -ENODEV;
204	204	}
205	205

drivers/acpi/arm64/iort.c+200−58

@@ -88,8 +88,8 @@ static inline int iort_set_fwnode(struct acpi_iort_node *iort_node, · 87 unmodified lines
88	88	*
89	89	* Returns: fwnode_handle pointer on success, NULL on failure
90	90	*/
91		−static inline
92		−struct fwnode_handle iort_get_fwnode(struct acpi_iort_node node)
	91	+static inline struct fwnode_handle *iort_get_fwnode(
	92	+ struct acpi_iort_node *node)
93	93	{
94	94	struct iort_fwnode *curr;
95	95	struct fwnode_handle *fwnode = NULL;
@@ -126,6 +126,31 @@ static inline void iort_delete_fwnode(struct acpi_iort_node *node) · 30 unmodified lines
126	126	spin_unlock(&iort_fwnode_lock);
127	127	}
128	128
	129	+/**
	130	+ * iort_get_iort_node() - Retrieve iort_node associated with an fwnode
	131	+ *
	132	+ * @fwnode: fwnode associated with device to be looked-up
	133	+ *
	134	+ * Returns: iort_node pointer on success, NULL on failure
	135	+ */
	136	+static inline struct acpi_iort_node *iort_get_iort_node(
	137	+ struct fwnode_handle *fwnode)
	138	+{
	139	+ struct iort_fwnode *curr;
	140	+ struct acpi_iort_node *iort_node = NULL;
	141	+
	142	+ spin_lock(&iort_fwnode_lock);
	143	+ list_for_each_entry(curr, &iort_fwnode_list, list) {
	144	+ if (curr->fwnode == fwnode) {
	145	+ iort_node = curr->iort_node;
	146	+ break;
	147	+ }
	148	+ }
	149	+ spin_unlock(&iort_fwnode_lock);
	150	+
	151	+ return iort_node;
	152	+}
	153	+
129	154	typedef acpi_status (*iort_find_node_callback)
130	155	(struct acpi_iort_node node, void context);
131	156
@@ -306,9 +331,8 @@ static int iort_id_map(struct acpi_iort_id_mapping *map, u8 type, u32 rid_in, · 174 unmodified lines
306	331	return 0;
307	332	}
308	333
309		−static
310		−struct acpi_iort_node iort_node_get_id(struct acpi_iort_node node,
311		− u32 *id_out, int index)
	334	+static struct acpi_iort_node iort_node_get_id(struct acpi_iort_node node,
	335	+ u32 *id_out, int index)
312	336	{
313	337	struct acpi_iort_node *parent;
314	338	struct acpi_iort_id_mapping *map;
@@ -332,7 +356,8 @@ struct acpi_iort_node iort_node_get_id(struct acpi_iort_node node, · 17 unmodified lines
332	356
333	357	if (map->flags & ACPI_IORT_ID_SINGLE_MAPPING) {
334	358	if (node->type == ACPI_IORT_NODE_NAMED_COMPONENT \|\|
335		− node->type == ACPI_IORT_NODE_PCI_ROOT_COMPLEX) {
	359	+ node->type == ACPI_IORT_NODE_PCI_ROOT_COMPLEX \|\|
	360	+ node->type == ACPI_IORT_NODE_SMMU_V3) {
336	361	*id_out = map->output_base;
337	362	return parent;
338	363	}
@@ -341,6 +366,47 @@ struct acpi_iort_node iort_node_get_id(struct acpi_iort_node node, · 2 unmodified lines
341	366	return NULL;
342	367	}
343	368
	369	+#if (ACPI_CA_VERSION > 0x20170929)
	370	+static int iort_get_id_mapping_index(struct acpi_iort_node *node)
	371	+{
	372	+ struct acpi_iort_smmu_v3 *smmu;
	373	+
	374	+ switch (node->type) {
	375	+ case ACPI_IORT_NODE_SMMU_V3:
	376	+ /*
	377	+ * SMMUv3 dev ID mapping index was introduced in revision 1
	378	+ * table, not available in revision 0
	379	+ */
	380	+ if (node->revision < 1)
	381	+ return -EINVAL;
	382	+
	383	+ smmu = (struct acpi_iort_smmu_v3 *)node->node_data;
	384	+ /*
	385	+ * ID mapping index is only ignored if all interrupts are
	386	+ * GSIV based
	387	+ */
	388	+ if (smmu->event_gsiv && smmu->pri_gsiv && smmu->gerr_gsiv
	389	+ && smmu->sync_gsiv)
	390	+ return -EINVAL;
	391	+
	392	+ if (smmu->id_mapping_index >= node->mapping_count) {
	393	+ pr_err(FW_BUG "[node %p type %d] ID mapping index overflows valid mappings\n",
	394	+ node, node->type);
	395	+ return -EINVAL;
	396	+ }
	397	+
	398	+ return smmu->id_mapping_index;
	399	+ default:
	400	+ return -EINVAL;
	401	+ }
	402	+}
	403	+#else
	404	+static inline int iort_get_id_mapping_index(struct acpi_iort_node *node)
	405	+{
	406	+ return -EINVAL;
	407	+}
	408	+#endif
	409	+
344	410	static struct acpi_iort_node iort_node_map_id(struct acpi_iort_node node,
345	411	u32 id_in, u32 *id_out,
346	412	u8 type_mask)
@@ -350,7 +416,7 @@ static struct acpi_iort_node iort_node_map_id(struct acpi_iort_node node, · 3 unmodified lines
350	416	/* Parse the ID mapping tree to find specified node type */
351	417	while (node) {
352	418	struct acpi_iort_id_mapping *map;
353		− int i;
	419	+ int i, index;
354	420
355	421	if (IORT_TYPE_MASK(node->type) & type_mask) {
356	422	if (id_out)
@@ -371,8 +437,19 @@ static struct acpi_iort_node iort_node_map_id(struct acpi_iort_node node, · 14 unmodified lines
371	437	goto fail_map;
372	438	}
373	439
	440	+ /*
	441	+ * Get the special ID mapping index (if any) and skip its
	442	+ * associated ID map to prevent erroneous multi-stage
	443	+ * IORT ID translations.
	444	+ */
	445	+ index = iort_get_id_mapping_index(node);
	446	+
374	447	/* Do the ID translation */
375	448	for (i = 0; i < node->mapping_count; i++, map++) {
	449	+ /* if it is special mapping index, skip it */
	450	+ if (i == index)
	451	+ continue;
	452	+
376	453	if (!iort_id_map(map, node->type, id, &id))
377	454	break;
378	455	}
@@ -392,10 +469,9 @@ fail_map: · 13 unmodified lines
392	469	return NULL;
393	470	}
394	471
395		−static
396		−struct acpi_iort_node iort_node_map_platform_id(struct acpi_iort_node node,
397		− u32 *id_out, u8 type_mask,
398		− int index)
	472	+static struct acpi_iort_node *iort_node_map_platform_id(
	473	+ struct acpi_iort_node node, u32 id_out, u8 type_mask,
	474	+ int index)
399	475	{
400	476	struct acpi_iort_node *parent;
401	477	u32 id;
@@ -424,9 +500,25 @@ static struct acpi_iort_node iort_find_dev_node(struct device dev) · 22 unmodified lines
424	500	{
425	501	struct pci_bus *pbus;
426	502
427		− if (!dev_is_pci(dev))
	503	+ if (!dev_is_pci(dev)) {
	504	+ struct acpi_iort_node *node;
	505	+ /*
	506	+ * scan iort_fwnode_list to see if it's an iort platform
	507	+ * device (such as SMMU, PMCG),its iort node already cached
	508	+ * and associated with fwnode when iort platform devices
	509	+ * were initialized.
	510	+ */
	511	+ node = iort_get_iort_node(dev->fwnode);
	512	+ if (node)
	513	+ return node;
	514	+
	515	+ /*
	516	+ * if not, then it should be a platform device defined in
	517	+ * DSDT/SSDT (with Named Component node in IORT)
	518	+ */
428	519	return iort_scan_node(ACPI_IORT_NODE_NAMED_COMPONENT,
429	520	iort_match_node_callback, dev);
	521	+ }
430	522
431	523	/* Find a PCI root bus */
432	524	pbus = to_pci_dev(dev)->bus;
@@ -466,16 +558,24 @@ u32 iort_msi_map_rid(struct device *dev, u32 req_id) · 33 unmodified lines
466	558	*/
467	559	int iort_pmsi_get_dev_id(struct device dev, u32 dev_id)
468	560	{
469		− int i;
	561	+ int i, index;
470	562	struct acpi_iort_node *node;
471	563
472	564	node = iort_find_dev_node(dev);
473	565	if (!node)
474	566	return -ENODEV;
475	567
476		− for (i = 0; i < node->mapping_count; i++) {
477		− if (iort_node_map_platform_id(node, dev_id, IORT_MSI_TYPE, i))
	568	+ index = iort_get_id_mapping_index(node);
	569	+ /* if there is a valid index, go get the dev_id directly */
	570	+ if (index >= 0) {
	571	+ if (iort_node_get_id(node, dev_id, index))
478	572	return 0;
	573	+ } else {
	574	+ for (i = 0; i < node->mapping_count; i++) {
	575	+ if (iort_node_map_platform_id(node, dev_id,
	576	+ IORT_MSI_TYPE, i))
	577	+ return 0;
	578	+ }
479	579	}
480	580
481	581	return -ENODEV;
@@ -538,6 +638,49 @@ struct irq_domain iort_get_device_domain(struct device dev, u32 req_id) · 56 unmodified lines
538	638	return irq_find_matching_fwnode(handle, DOMAIN_BUS_PCI_MSI);
539	639	}
540	640
	641	+static void iort_set_device_domain(struct device *dev,
	642	+ struct acpi_iort_node *node)
	643	+{
	644	+ struct acpi_iort_its_group *its;
	645	+ struct acpi_iort_node *msi_parent;
	646	+ struct acpi_iort_id_mapping *map;
	647	+ struct fwnode_handle *iort_fwnode;
	648	+ struct irq_domain *domain;
	649	+ int index;
	650	+
	651	+ index = iort_get_id_mapping_index(node);
	652	+ if (index < 0)
	653	+ return;
	654	+
	655	+ map = ACPI_ADD_PTR(struct acpi_iort_id_mapping, node,
	656	+ node->mapping_offset + index * sizeof(*map));
	657	+
	658	+ /* Firmware bug! */
	659	+ if (!map->output_reference \|\|
	660	+ !(map->flags & ACPI_IORT_ID_SINGLE_MAPPING)) {
	661	+ pr_err(FW_BUG "[node %p type %d] Invalid MSI mapping\n",
	662	+ node, node->type);
	663	+ return;
	664	+ }
	665	+
	666	+ msi_parent = ACPI_ADD_PTR(struct acpi_iort_node, iort_table,
	667	+ map->output_reference);
	668	+
	669	+ if (!msi_parent \|\| msi_parent->type != ACPI_IORT_NODE_ITS_GROUP)
	670	+ return;
	671	+
	672	+ /* Move to ITS specific data */
	673	+ its = (struct acpi_iort_its_group *)msi_parent->node_data;
	674	+
	675	+ iort_fwnode = iort_find_domain_token(its->identifiers[0]);
	676	+ if (!iort_fwnode)
	677	+ return;
	678	+
	679	+ domain = irq_find_matching_fwnode(iort_fwnode, DOMAIN_BUS_PLATFORM_MSI);
	680	+ if (domain)
	681	+ dev_set_msi_domain(dev, domain);
	682	+}
	683	+
541	684	/**
542	685	* iort_get_platform_device_domain() - Find MSI domain related to a
543	686	* platform device
@@ -623,14 +766,14 @@ static inline bool iort_iommu_driver_enabled(u8 type) · 79 unmodified lines
623	766	}
624	767
625	768	#ifdef CONFIG_IOMMU_API
626		−static inline
627		−const struct iommu_ops iort_fwspec_iommu_ops(struct iommu_fwspec fwspec)
	769	+static inline const struct iommu_ops *iort_fwspec_iommu_ops(
	770	+ struct iommu_fwspec *fwspec)
628	771	{
629	772	return (fwspec && fwspec->ops) ? fwspec->ops : NULL;
630	773	}
631	774
632		−static inline
633		−int iort_add_device_replay(const struct iommu_ops ops, struct device dev)
	775	+static inline int iort_add_device_replay(const struct iommu_ops *ops,
	776	+ struct device *dev)
634	777	{
635	778	int err = 0;
636	779
@@ -640,11 +783,11 @@ int iort_add_device_replay(const struct iommu_ops ops, struct device dev) · 3 unmodified lines
640	783	return err;
641	784	}
642	785	#else
643		−static inline
644		−const struct iommu_ops iort_fwspec_iommu_ops(struct iommu_fwspec fwspec)
	786	+static inline const struct iommu_ops *iort_fwspec_iommu_ops(
	787	+ struct iommu_fwspec *fwspec)
645	788	{ return NULL; }
646		−static inline
647		−int iort_add_device_replay(const struct iommu_ops ops, struct device dev)
	789	+static inline int iort_add_device_replay(const struct iommu_ops *ops,
	790	+ struct device *dev)
648	791	{ return 0; }
649	792	#endif
650	793
@@ -968,7 +1111,7 @@ static bool __init arm_smmu_v3_is_coherent(struct acpi_iort_node *node) · 317 unmodified lines
968	1111	return smmu->flags & ACPI_IORT_SMMU_V3_COHACC_OVERRIDE;
969	1112	}
970	1113
971		−#if defined(CONFIG_ACPI_NUMA) && defined(ACPI_IORT_SMMU_V3_PXM_VALID)
	1114	+#if defined(CONFIG_ACPI_NUMA)
972	1115	/*
973	1116	* set numa proximity domain for smmuv3 device
974	1117	*/
@@ -1051,34 +1194,34 @@ static bool __init arm_smmu_is_coherent(struct acpi_iort_node *node) · 76 unmodified lines
1051	1194	return smmu->flags & ACPI_IORT_SMMU_COHERENT_WALK;
1052	1195	}
1053	1196
1054		−struct iort_iommu_config {
	1197	+struct iort_dev_config {
1055	1198	const char *name;
1056		− int (iommu_init)(struct acpi_iort_node node);
1057		− bool (iommu_is_coherent)(struct acpi_iort_node node);
1058		− int (iommu_count_resources)(struct acpi_iort_node node);
1059		− void (iommu_init_resources)(struct resource res,
	1199	+ int (dev_init)(struct acpi_iort_node node);
	1200	+ bool (dev_is_coherent)(struct acpi_iort_node node);
	1201	+ int (dev_count_resources)(struct acpi_iort_node node);
	1202	+ void (dev_init_resources)(struct resource res,
1060	1203	struct acpi_iort_node *node);
1061		− void (iommu_set_proximity)(struct device dev,
	1204	+ void (dev_set_proximity)(struct device dev,
1062	1205	struct acpi_iort_node *node);
1063	1206	};
1064	1207
1065		−static const struct iort_iommu_config iort_arm_smmu_v3_cfg __initconst = {
	1208	+static const struct iort_dev_config iort_arm_smmu_v3_cfg __initconst = {
1066	1209	.name = "arm-smmu-v3",
1067		− .iommu_is_coherent = arm_smmu_v3_is_coherent,
1068		− .iommu_count_resources = arm_smmu_v3_count_resources,
1069		− .iommu_init_resources = arm_smmu_v3_init_resources,
1070		− .iommu_set_proximity = arm_smmu_v3_set_proximity,
	1210	+ .dev_is_coherent = arm_smmu_v3_is_coherent,
	1211	+ .dev_count_resources = arm_smmu_v3_count_resources,
	1212	+ .dev_init_resources = arm_smmu_v3_init_resources,
	1213	+ .dev_set_proximity = arm_smmu_v3_set_proximity,
1071	1214	};
1072	1215
1073		−static const struct iort_iommu_config iort_arm_smmu_cfg __initconst = {
	1216	+static const struct iort_dev_config iort_arm_smmu_cfg __initconst = {
1074	1217	.name = "arm-smmu",
1075		− .iommu_is_coherent = arm_smmu_is_coherent,
1076		− .iommu_count_resources = arm_smmu_count_resources,
1077		− .iommu_init_resources = arm_smmu_init_resources
	1218	+ .dev_is_coherent = arm_smmu_is_coherent,
	1219	+ .dev_count_resources = arm_smmu_count_resources,
	1220	+ .dev_init_resources = arm_smmu_init_resources
1078	1221	};
1079	1222
1080		−static __init
1081		−const struct iort_iommu_config iort_get_iommu_cfg(struct acpi_iort_node node)
	1223	+static __init const struct iort_dev_config *iort_get_dev_cfg(
	1224	+ struct acpi_iort_node *node)
1082	1225	{
1083	1226	switch (node->type) {
1084	1227	case ACPI_IORT_NODE_SMMU_V3:
@@ -1091,31 +1234,28 @@ const struct iort_iommu_config iort_get_iommu_cfg(struct acpi_iort_node node) · 6 unmodified lines
1091	1234	}
1092	1235
1093	1236	/**
1094		− * iort_add_smmu_platform_device() - Allocate a platform device for SMMU
1095		− * @node: Pointer to SMMU ACPI IORT node
	1237	+ * iort_add_platform_device() - Allocate a platform device for IORT node
	1238	+ * @node: Pointer to device ACPI IORT node
1096	1239	*
1097	1240	* Returns: 0 on success, <0 failure
1098	1241	*/
1099		−static int __init iort_add_smmu_platform_device(struct acpi_iort_node *node)
	1242	+static int __init iort_add_platform_device(struct acpi_iort_node *node,
	1243	+ const struct iort_dev_config *ops)
1100	1244	{
1101	1245	struct fwnode_handle *fwnode;
1102	1246	struct platform_device *pdev;
1103	1247	struct resource *r;
1104	1248	enum dev_dma_attr attr;
1105	1249	int ret, count;
1106		− const struct iort_iommu_config *ops = iort_get_iommu_cfg(node);
1107		−
1108		− if (!ops)
1109		− return -ENODEV;
1110	1250
1111	1251	pdev = platform_device_alloc(ops->name, PLATFORM_DEVID_AUTO);
1112	1252	if (!pdev)
1113	1253	return -ENOMEM;
1114	1254
1115		− if (ops->iommu_set_proximity)
1116		− ops->iommu_set_proximity(&pdev->dev, node);
	1255	+ if (ops->dev_set_proximity)
	1256	+ ops->dev_set_proximity(&pdev->dev, node);
1117	1257
1118		− count = ops->iommu_count_resources(node);
	1258	+ count = ops->dev_count_resources(node);
1119	1259
1120	1260	r = kcalloc(count, sizeof(*r), GFP_KERNEL);
1121	1261	if (!r) {
@@ -1123,7 +1263,7 @@ static int __init iort_add_smmu_platform_device(struct acpi_iort_node *node) · 1 unmodified line
1123	1263	goto dev_put;
1124	1264	}
1125	1265
1126		− ops->iommu_init_resources(r, node);
	1266	+ ops->dev_init_resources(r, node);
1127	1267
1128	1268	ret = platform_device_add_resources(pdev, r, count);
1129	1269	/*
@@ -1158,12 +1298,14 @@ static int __init iort_add_smmu_platform_device(struct acpi_iort_node *node) · 28 unmodified lines
1158	1298
1159	1299	pdev->dev.fwnode = fwnode;
1160	1300
1161		− attr = ops->iommu_is_coherent(node) ?
1162		− DEV_DMA_COHERENT : DEV_DMA_NON_COHERENT;
	1301	+ attr = ops->dev_is_coherent && ops->dev_is_coherent(node) ?
	1302	+ DEV_DMA_COHERENT : DEV_DMA_NON_COHERENT;
1163	1303
1164	1304	/* Configure DMA for the page table walker */
1165	1305	acpi_dma_configure(&pdev->dev, attr);
1166	1306
	1307	+ iort_set_device_domain(&pdev->dev, node);
	1308	+
1167	1309	ret = platform_device_add(pdev);
1168	1310	if (ret)
1169	1311	goto dma_deconfigure;
@@ -1216,6 +1358,7 @@ static void __init iort_init_platform_devices(void) · 46 unmodified lines
1216	1358	struct fwnode_handle *fwnode;
1217	1359	int i, ret;
1218	1360	bool acs_enabled = false;
	1361	+ const struct iort_dev_config *ops;
1219	1362
1220	1363	/*
1221	1364	* iort_table and iort both point to the start of IORT table, but
@@ -1238,16 +1381,15 @@ static void __init iort_init_platform_devices(void) · 16 unmodified lines
1238	1381	if (!acs_enabled)
1239	1382	acs_enabled = iort_enable_acs(iort_node);
1240	1383
1241		− if ((iort_node->type == ACPI_IORT_NODE_SMMU) \|\|
1242		− (iort_node->type == ACPI_IORT_NODE_SMMU_V3)) {
1243		−
	1384	+ ops = iort_get_dev_cfg(iort_node);
	1385	+ if (ops) {
1244	1386	fwnode = acpi_alloc_fwnode_static();
1245	1387	if (!fwnode)
1246	1388	return;
1247	1389
1248	1390	iort_set_fwnode(iort_node, fwnode);
1249	1391
1250		− ret = iort_add_smmu_platform_device(iort_node);
	1392	+ ret = iort_add_platform_device(iort_node, ops);
1251	1393	if (ret) {
1252	1394	iort_delete_fwnode(iort_node);
1253	1395	acpi_free_fwnode_static(fwnode);

drivers/bus/arm-ccn.c+1−0

@@ -1276,6 +1276,7 @@ static int arm_ccn_pmu_init(struct arm_ccn *ccn) · 1275 unmodified lines
1276	1276
1277	1277	/* Perf driver registration */
1278	1278	ccn->dt.pmu = (struct pmu) {
	1279	+ .module = THIS_MODULE,
1279	1280	.attr_groups = arm_ccn_pmu_attr_groups,
1280	1281	.task_ctx_nr = perf_invalid_context,
1281	1282	.event_init = arm_ccn_pmu_event_init,

drivers/clocksource/arm_arch_timer.c+22−3

@@ -77,6 +77,7 @@ static bool arch_timer_mem_use_virtual; · 76 unmodified lines
77	77	static bool arch_counter_suspend_stop;
78	78	static bool vdso_default = true;
79	79
	80	+static cpumask_t evtstrm_available = CPU_MASK_NONE;
80	81	static bool evtstrm_enable = IS_ENABLED(CONFIG_ARM_ARCH_TIMER_EVTSTREAM);
81	82
82	83	static int __init early_evtstrm_cfg(char *buf)
@@ -739,6 +740,7 @@ static void arch_timer_evtstrm_enable(int divider) · 656 unmodified lines
739	740	#ifdef CONFIG_COMPAT
740	741	compat_elf_hwcap \|= COMPAT_HWCAP_EVTSTRM;
741	742	#endif
	743	+ cpumask_set_cpu(smp_processor_id(), &evtstrm_available);
742	744	}
743	745
744	746	static void arch_timer_configure_evtstream(void)
@@ -863,6 +865,16 @@ u32 arch_timer_get_rate(void) · 118 unmodified lines
863	865	return arch_timer_rate;
864	866	}
865	867
	868	+bool arch_timer_evtstrm_available(void)
	869	+{
	870	+ /*
	871	+ * We might get called from a preemptible context. This is fine
	872	+ * because availability of the event stream should be always the same
	873	+ * for a preemptible context and context where we might resume a task.
	874	+ */
	875	+ return cpumask_test_cpu(raw_smp_processor_id(), &evtstrm_available);
	876	+}
	877	+
866	878	static u64 arch_counter_get_cntvct_mem(void)
867	879	{
868	880	u32 vct_lo, vct_hi, tmp_hi;
@@ -928,6 +940,8 @@ static int arch_timer_dying_cpu(unsigned int cpu) · 59 unmodified lines
928	940	{
929	941	struct clock_event_device *clk = this_cpu_ptr(arch_timer_evt);
930	942
	943	+ cpumask_clear_cpu(smp_processor_id(), &evtstrm_available);
	944	+
931	945	arch_timer_stop(clk);
932	946	return 0;
933	947	}
@@ -937,10 +951,16 @@ static DEFINE_PER_CPU(unsigned long, saved_cntkctl); · 3 unmodified lines
937	951	static int arch_timer_cpu_pm_notify(struct notifier_block *self,
938	952	unsigned long action, void *hcpu)
939	953	{
940		− if (action == CPU_PM_ENTER)
	954	+ if (action == CPU_PM_ENTER) {
941	955	__this_cpu_write(saved_cntkctl, arch_timer_get_cntkctl());
942		− else if (action == CPU_PM_ENTER_FAILED \|\| action == CPU_PM_EXIT)
	956	+
	957	+ cpumask_clear_cpu(smp_processor_id(), &evtstrm_available);
	958	+ } else if (action == CPU_PM_ENTER_FAILED \|\| action == CPU_PM_EXIT) {
943	959	arch_timer_set_cntkctl(__this_cpu_read(saved_cntkctl));
	960	+
	961	+ if (elf_hwcap & HWCAP_EVTSTRM)
	962	+ cpumask_set_cpu(smp_processor_id(), &evtstrm_available);
	963	+ }
944	964	return NOTIFY_OK;
945	965	}
946	966
@@ -1016,7 +1036,6 @@ static int __init arch_timer_register(void) · 69 unmodified lines
1016	1036	if (err)
1017	1037	goto out_unreg_notify;
1018	1038
1019		−
1020	1039	/* Register and immediately configure the timer on the boot CPU */
1021	1040	err = cpuhp_setup_state(CPUHP_AP_ARM_ARCH_TIMER_STARTING,
1022	1041	"clockevents/arm/arch_timer:starting",

drivers/perf/Kconfig+15−0

@@ -17,6 +17,13 @@ config ARM_PMU_ACPI · 16 unmodified lines
17	17	depends on ARM_PMU && ACPI
18	18	def_bool y
19	19
	20	+config HISI_PMU
	21	+ bool "HiSilicon SoC PMU"
	22	+ depends on ARM64 && ACPI
	23	+ help
	24	+ Support for HiSilicon SoC uncore performance monitoring
	25	+ unit (PMU), such as: L3C, HHA and DDRC.
	26	+
20	27	config QCOM_L2_PMU
21	28	bool "Qualcomm Technologies L2-cache PMU"
22	29	depends on ARCH_QCOM && ARM64 && ACPI
@@ -43,4 +50,12 @@ config XGENE_PMU · 20 unmodified lines
43	50	help
44	51	Say y if you want to use APM X-Gene SoC performance monitors.
45	52
	53	+config ARM_SPE_PMU
	54	+ tristate "Enable support for the ARMv8.2 Statistical Profiling Extension"
	55	+ depends on PERF_EVENTS && ARM64
	56	+ help
	57	+ Enable perf support for the ARMv8.2 Statistical Profiling
	58	+ Extension, which provides periodic sampling of operations in
	59	+ the CPU pipeline and reports this via the perf AUX interface.
	60	+
46	61	endmenu

drivers/perf/Makefile+2−0

@@ -1,6 +1,8 @@
1	1	# SPDX-License-Identifier: GPL-2.0
2	2	obj-$(CONFIG_ARM_PMU) += arm_pmu.o arm_pmu_platform.o
3	3	obj-$(CONFIG_ARM_PMU_ACPI) += arm_pmu_acpi.o
	4	+obj-$(CONFIG_HISI_PMU) += hisilicon/
4	5	obj-$(CONFIG_QCOM_L2_PMU) += qcom_l2_pmu.o
5	6	obj-$(CONFIG_QCOM_L3_PMU) += qcom_l3_pmu.o
6	7	obj-$(CONFIG_XGENE_PMU) += xgene_pmu.o
	8	+obj-$(CONFIG_ARM_SPE_PMU) += arm_spe_pmu.o

drivers/perf/arm_pmu.c+5−5

@@ -539,7 +539,7 @@ void armpmu_free_irq(struct arm_pmu *armpmu, int cpu) · 538 unmodified lines
539	539	if (!cpumask_test_and_clear_cpu(cpu, &armpmu->active_irqs))
540	540	return;
541	541
542		− if (irq_is_percpu(irq)) {
	542	+ if (irq_is_percpu_devid(irq)) {
543	543	free_percpu_irq(irq, &hw_events->percpu_pmu);
544	544	cpumask_clear(&armpmu->active_irqs);
545	545	return;
@@ -565,10 +565,10 @@ int armpmu_request_irq(struct arm_pmu *armpmu, int cpu) · 19 unmodified lines
565	565	if (!irq)
566	566	return 0;
567	567
568		− if (irq_is_percpu(irq) && cpumask_empty(&armpmu->active_irqs)) {
	568	+ if (irq_is_percpu_devid(irq) && cpumask_empty(&armpmu->active_irqs)) {
569	569	err = request_percpu_irq(irq, handler, "arm-pmu",
570	570	&hw_events->percpu_pmu);
571		− } else if (irq_is_percpu(irq)) {
	571	+ } else if (irq_is_percpu_devid(irq)) {
572	572	int other_cpu = cpumask_first(&armpmu->active_irqs);
573	573	int other_irq = per_cpu(hw_events->irq, other_cpu);
574	574
@@ -649,7 +649,7 @@ static int arm_perf_starting_cpu(unsigned int cpu, struct hlist_node *node) · 74 unmodified lines
649	649
650	650	irq = armpmu_get_cpu_irq(pmu, cpu);
651	651	if (irq) {
652		− if (irq_is_percpu(irq)) {
	652	+ if (irq_is_percpu_devid(irq)) {
653	653	enable_percpu_irq(irq, IRQ_TYPE_NONE);
654	654	return 0;
655	655	}
@@ -667,7 +667,7 @@ static int arm_perf_teardown_cpu(unsigned int cpu, struct hlist_node *node) · 11 unmodified lines
667	667	return 0;
668	668
669	669	irq = armpmu_get_cpu_irq(pmu, cpu);
670		− if (irq && irq_is_percpu(irq))
	670	+ if (irq && irq_is_percpu_devid(irq))
671	671	disable_percpu_irq(irq);
672	672
673	673	return 0;

drivers/perf/arm_pmu_acpi.c+0−3

@@ -193,9 +193,6 @@ int arm_pmu_acpi_probe(armpmu_init_fn init_fn) · 192 unmodified lines
193	193	int pmu_idx = 0;
194	194	int cpu, ret;
195	195
196		− if (acpi_disabled)
197		− return 0;
198		−
199	196	/*
200	197	* Initialise and register the set of PMUs which we know about right
201	198	* now. Ideally we'd do this in arm_pmu_acpi_cpu_starting() so that we

drivers/perf/arm_pmu_platform.c+2−2

@@ -127,7 +127,7 @@ static int pmu_parse_irqs(struct arm_pmu *pmu) · 126 unmodified lines
127	127
128	128	if (num_irqs == 1) {
129	129	int irq = platform_get_irq(pdev, 0);
130		− if (irq && irq_is_percpu(irq))
	130	+ if (irq && irq_is_percpu_devid(irq))
131	131	return pmu_parse_percpu_irq(pmu, irq);
132	132	}
133	133
@@ -150,7 +150,7 @@ static int pmu_parse_irqs(struct arm_pmu *pmu) · 16 unmodified lines
150	150	if (WARN_ON(irq <= 0))
151	151	continue;
152	152
153		− if (irq_is_percpu(irq)) {
	153	+ if (irq_is_percpu_devid(irq)) {
154	154	pr_warn("multiple PPIs or mismatched SPI/PPI detected\n");
155	155	return -EINVAL;
156	156	}

drivers/perf/arm_spe_pmu.c+1249−0

@@ -0,0 +1,1249 @@
	1	+/*
	2	+ * Perf support for the Statistical Profiling Extension, introduced as
	3	+ * part of ARMv8.2.
	4	+ *
	5	+ * This program is free software; you can redistribute it and/or modify
	6	+ * it under the terms of the GNU General Public License version 2 as
	7	+ * published by the Free Software Foundation.
	8	+ *
	9	+ * This program is distributed in the hope that it will be useful,
	10	+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
	11	+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	12	+ * GNU General Public License for more details.
	13	+ *
	14	+ * You should have received a copy of the GNU General Public License
	15	+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
	16	+ *
	17	+ * Copyright (C) 2016 ARM Limited
	18	+ *
	19	+ * Author: Will Deacon <will.deacon@arm.com>
	20	+ */
	21	+
	22	+#define PMUNAME "arm_spe"
	23	+#define DRVNAME PMUNAME "_pmu"
	24	+#define pr_fmt(fmt) DRVNAME ": " fmt
	25	+
	26	+#include <linux/cpuhotplug.h>
	27	+#include <linux/interrupt.h>
	28	+#include <linux/irq.h>
	29	+#include <linux/module.h>
	30	+#include <linux/of_address.h>
	31	+#include <linux/of_device.h>
	32	+#include <linux/perf_event.h>
	33	+#include <linux/platform_device.h>
	34	+#include <linux/slab.h>
	35	+
	36	+#include <asm/sysreg.h>
	37	+
	38	+#define ARM_SPE_BUF_PAD_BYTE 0
	39	+
	40	+struct arm_spe_pmu_buf {
	41	+ int nr_pages;
	42	+ bool snapshot;
	43	+ void *base;
	44	+};
	45	+
	46	+struct arm_spe_pmu {
	47	+ struct pmu pmu;
	48	+ struct platform_device *pdev;
	49	+ cpumask_t supported_cpus;
	50	+ struct hlist_node hotplug_node;
	51	+
	52	+ int irq; /* PPI */
	53	+
	54	+ u16 min_period;
	55	+ u16 counter_sz;
	56	+
	57	+#define SPE_PMU_FEAT_FILT_EVT (1UL << 0)
	58	+#define SPE_PMU_FEAT_FILT_TYP (1UL << 1)
	59	+#define SPE_PMU_FEAT_FILT_LAT (1UL << 2)
	60	+#define SPE_PMU_FEAT_ARCH_INST (1UL << 3)
	61	+#define SPE_PMU_FEAT_LDS (1UL << 4)
	62	+#define SPE_PMU_FEAT_ERND (1UL << 5)
	63	+#define SPE_PMU_FEAT_DEV_PROBED (1UL << 63)
	64	+ u64 features;
	65	+
	66	+ u16 max_record_sz;
	67	+ u16 align;
	68	+ struct perf_output_handle __percpu *handle;
	69	+};
	70	+
	71	+#define to_spe_pmu(p) (container_of(p, struct arm_spe_pmu, pmu))
	72	+
	73	+/* Convert a free-running index from perf into an SPE buffer offset */
	74	+#define PERF_IDX2OFF(idx, buf) ((idx) % ((buf)->nr_pages << PAGE_SHIFT))
	75	+
	76	+/* Keep track of our dynamic hotplug state */
	77	+static enum cpuhp_state arm_spe_pmu_online;
	78	+
	79	+enum arm_spe_pmu_buf_fault_action {
	80	+ SPE_PMU_BUF_FAULT_ACT_SPURIOUS,
	81	+ SPE_PMU_BUF_FAULT_ACT_FATAL,
	82	+ SPE_PMU_BUF_FAULT_ACT_OK,
	83	+};
	84	+
	85	+/* This sysfs gunk was really good fun to write. */
	86	+enum arm_spe_pmu_capabilities {
	87	+ SPE_PMU_CAP_ARCH_INST = 0,
	88	+ SPE_PMU_CAP_ERND,
	89	+ SPE_PMU_CAP_FEAT_MAX,
	90	+ SPE_PMU_CAP_CNT_SZ = SPE_PMU_CAP_FEAT_MAX,
	91	+ SPE_PMU_CAP_MIN_IVAL,
	92	+};
	93	+
	94	+static int arm_spe_pmu_feat_caps[SPE_PMU_CAP_FEAT_MAX] = {
	95	+ [SPE_PMU_CAP_ARCH_INST] = SPE_PMU_FEAT_ARCH_INST,
	96	+ [SPE_PMU_CAP_ERND] = SPE_PMU_FEAT_ERND,
	97	+};
	98	+
	99	+static u32 arm_spe_pmu_cap_get(struct arm_spe_pmu *spe_pmu, int cap)
	100	+{
	101	+ if (cap < SPE_PMU_CAP_FEAT_MAX)
	102	+ return !!(spe_pmu->features & arm_spe_pmu_feat_caps[cap]);
	103	+
	104	+ switch (cap) {
	105	+ case SPE_PMU_CAP_CNT_SZ:
	106	+ return spe_pmu->counter_sz;
	107	+ case SPE_PMU_CAP_MIN_IVAL:
	108	+ return spe_pmu->min_period;
	109	+ default:
	110	+ WARN(1, "unknown cap %d\n", cap);
	111	+ }
	112	+
	113	+ return 0;
	114	+}
	115	+
	116	+static ssize_t arm_spe_pmu_cap_show(struct device *dev,
	117	+ struct device_attribute *attr,
	118	+ char *buf)
	119	+{
	120	+ struct platform_device *pdev = to_platform_device(dev);
	121	+ struct arm_spe_pmu *spe_pmu = platform_get_drvdata(pdev);
	122	+ struct dev_ext_attribute *ea =
	123	+ container_of(attr, struct dev_ext_attribute, attr);
	124	+ int cap = (long)ea->var;
	125	+
	126	+ return snprintf(buf, PAGE_SIZE, "%u\n",
	127	+ arm_spe_pmu_cap_get(spe_pmu, cap));
	128	+}
	129	+
	130	+#define SPE_EXT_ATTR_ENTRY(_name, _func, _var) \
	131	+ &((struct dev_ext_attribute[]) { \
	132	+ { __ATTR(_name, S_IRUGO, _func, NULL), (void *)_var } \
	133	+ })[0].attr.attr
	134	+
	135	+#define SPE_CAP_EXT_ATTR_ENTRY(_name, _var) \
	136	+ SPE_EXT_ATTR_ENTRY(_name, arm_spe_pmu_cap_show, _var)
	137	+
	138	+static struct attribute *arm_spe_pmu_cap_attr[] = {
	139	+ SPE_CAP_EXT_ATTR_ENTRY(arch_inst, SPE_PMU_CAP_ARCH_INST),
	140	+ SPE_CAP_EXT_ATTR_ENTRY(ernd, SPE_PMU_CAP_ERND),
	141	+ SPE_CAP_EXT_ATTR_ENTRY(count_size, SPE_PMU_CAP_CNT_SZ),
	142	+ SPE_CAP_EXT_ATTR_ENTRY(min_interval, SPE_PMU_CAP_MIN_IVAL),
	143	+ NULL,
	144	+};
	145	+
	146	+static struct attribute_group arm_spe_pmu_cap_group = {
	147	+ .name = "caps",
	148	+ .attrs = arm_spe_pmu_cap_attr,
	149	+};
	150	+
	151	+/* User ABI */
	152	+#define ATTR_CFG_FLD_ts_enable_CFG config /* PMSCR_EL1.TS */
	153	+#define ATTR_CFG_FLD_ts_enable_LO 0
	154	+#define ATTR_CFG_FLD_ts_enable_HI 0
	155	+#define ATTR_CFG_FLD_pa_enable_CFG config /* PMSCR_EL1.PA */
	156	+#define ATTR_CFG_FLD_pa_enable_LO 1
	157	+#define ATTR_CFG_FLD_pa_enable_HI 1
	158	+#define ATTR_CFG_FLD_pct_enable_CFG config /* PMSCR_EL1.PCT */
	159	+#define ATTR_CFG_FLD_pct_enable_LO 2
	160	+#define ATTR_CFG_FLD_pct_enable_HI 2
	161	+#define ATTR_CFG_FLD_jitter_CFG config /* PMSIRR_EL1.RND */
	162	+#define ATTR_CFG_FLD_jitter_LO 16
	163	+#define ATTR_CFG_FLD_jitter_HI 16
	164	+#define ATTR_CFG_FLD_branch_filter_CFG config /* PMSFCR_EL1.B */
	165	+#define ATTR_CFG_FLD_branch_filter_LO 32
	166	+#define ATTR_CFG_FLD_branch_filter_HI 32
	167	+#define ATTR_CFG_FLD_load_filter_CFG config /* PMSFCR_EL1.LD */
	168	+#define ATTR_CFG_FLD_load_filter_LO 33
	169	+#define ATTR_CFG_FLD_load_filter_HI 33
	170	+#define ATTR_CFG_FLD_store_filter_CFG config /* PMSFCR_EL1.ST */
	171	+#define ATTR_CFG_FLD_store_filter_LO 34
	172	+#define ATTR_CFG_FLD_store_filter_HI 34
	173	+
	174	+#define ATTR_CFG_FLD_event_filter_CFG config1 /* PMSEVFR_EL1 */
	175	+#define ATTR_CFG_FLD_event_filter_LO 0
	176	+#define ATTR_CFG_FLD_event_filter_HI 63
	177	+
	178	+#define ATTR_CFG_FLD_min_latency_CFG config2 /* PMSLATFR_EL1.MINLAT */
	179	+#define ATTR_CFG_FLD_min_latency_LO 0
	180	+#define ATTR_CFG_FLD_min_latency_HI 11
	181	+
	182	+/* Why does everything I do descend into this? */
	183	+#define __GEN_PMU_FORMAT_ATTR(cfg, lo, hi) \
	184	+ (lo) == (hi) ? #cfg ":" #lo "\n" : #cfg ":" #lo "-" #hi
	185	+
	186	+#define _GEN_PMU_FORMAT_ATTR(cfg, lo, hi) \
	187	+ __GEN_PMU_FORMAT_ATTR(cfg, lo, hi)
	188	+
	189	+#define GEN_PMU_FORMAT_ATTR(name) \
	190	+ PMU_FORMAT_ATTR(name, \
	191	+ _GEN_PMU_FORMAT_ATTR(ATTR_CFG_FLD_##name##_CFG, \
	192	+ ATTR_CFG_FLD_##name##_LO, \
	193	+ ATTR_CFG_FLD_##name##_HI))
	194	+
	195	+#define _ATTR_CFG_GET_FLD(attr, cfg, lo, hi) \
	196	+ ((((attr)->cfg) >> lo) & GENMASK(hi - lo, 0))
	197	+
	198	+#define ATTR_CFG_GET_FLD(attr, name) \
	199	+ _ATTR_CFG_GET_FLD(attr, \
	200	+ ATTR_CFG_FLD_##name##_CFG, \
	201	+ ATTR_CFG_FLD_##name##_LO, \
	202	+ ATTR_CFG_FLD_##name##_HI)
	203	+
	204	+GEN_PMU_FORMAT_ATTR(ts_enable);
	205	+GEN_PMU_FORMAT_ATTR(pa_enable);
	206	+GEN_PMU_FORMAT_ATTR(pct_enable);
	207	+GEN_PMU_FORMAT_ATTR(jitter);
	208	+GEN_PMU_FORMAT_ATTR(branch_filter);
	209	+GEN_PMU_FORMAT_ATTR(load_filter);
	210	+GEN_PMU_FORMAT_ATTR(store_filter);
	211	+GEN_PMU_FORMAT_ATTR(event_filter);
	212	+GEN_PMU_FORMAT_ATTR(min_latency);
	213	+
	214	+static struct attribute *arm_spe_pmu_formats_attr[] = {
	215	+ &format_attr_ts_enable.attr,
	216	+ &format_attr_pa_enable.attr,
	217	+ &format_attr_pct_enable.attr,
	218	+ &format_attr_jitter.attr,
	219	+ &format_attr_branch_filter.attr,
	220	+ &format_attr_load_filter.attr,
	221	+ &format_attr_store_filter.attr,
	222	+ &format_attr_event_filter.attr,
	223	+ &format_attr_min_latency.attr,
	224	+ NULL,
	225	+};
	226	+
	227	+static struct attribute_group arm_spe_pmu_format_group = {
	228	+ .name = "format",
	229	+ .attrs = arm_spe_pmu_formats_attr,
	230	+};
	231	+
	232	+static ssize_t arm_spe_pmu_get_attr_cpumask(struct device *dev,
	233	+ struct device_attribute *attr,
	234	+ char *buf)
	235	+{
	236	+ struct platform_device *pdev = to_platform_device(dev);
	237	+ struct arm_spe_pmu *spe_pmu = platform_get_drvdata(pdev);
	238	+
	239	+ return cpumap_print_to_pagebuf(true, buf, &spe_pmu->supported_cpus);
	240	+}
	241	+static DEVICE_ATTR(cpumask, S_IRUGO, arm_spe_pmu_get_attr_cpumask, NULL);
	242	+
	243	+static struct attribute *arm_spe_pmu_attrs[] = {
	244	+ &dev_attr_cpumask.attr,
	245	+ NULL,
	246	+};
	247	+
	248	+static struct attribute_group arm_spe_pmu_group = {
	249	+ .attrs = arm_spe_pmu_attrs,
	250	+};
	251	+
	252	+static const struct attribute_group *arm_spe_pmu_attr_groups[] = {
	253	+ &arm_spe_pmu_group,
	254	+ &arm_spe_pmu_cap_group,
	255	+ &arm_spe_pmu_format_group,
	256	+ NULL,
	257	+};
	258	+
	259	+/* Convert between user ABI and register values */
	260	+static u64 arm_spe_event_to_pmscr(struct perf_event *event)
	261	+{
	262	+ struct perf_event_attr *attr = &event->attr;
	263	+ u64 reg = 0;
	264	+
	265	+ reg \|= ATTR_CFG_GET_FLD(attr, ts_enable) << SYS_PMSCR_EL1_TS_SHIFT;
	266	+ reg \|= ATTR_CFG_GET_FLD(attr, pa_enable) << SYS_PMSCR_EL1_PA_SHIFT;
	267	+ reg \|= ATTR_CFG_GET_FLD(attr, pct_enable) << SYS_PMSCR_EL1_PCT_SHIFT;
	268	+
	269	+ if (!attr->exclude_user)
	270	+ reg \|= BIT(SYS_PMSCR_EL1_E0SPE_SHIFT);
	271	+
	272	+ if (!attr->exclude_kernel)
	273	+ reg \|= BIT(SYS_PMSCR_EL1_E1SPE_SHIFT);
	274	+
	275	+ if (IS_ENABLED(CONFIG_PID_IN_CONTEXTIDR) && capable(CAP_SYS_ADMIN))
	276	+ reg \|= BIT(SYS_PMSCR_EL1_CX_SHIFT);
	277	+
	278	+ return reg;
	279	+}
	280	+
	281	+static void arm_spe_event_sanitise_period(struct perf_event *event)
	282	+{
	283	+ struct arm_spe_pmu *spe_pmu = to_spe_pmu(event->pmu);
	284	+ u64 period = event->hw.sample_period;
	285	+ u64 max_period = SYS_PMSIRR_EL1_INTERVAL_MASK
	286	+ << SYS_PMSIRR_EL1_INTERVAL_SHIFT;
	287	+
	288	+ if (period < spe_pmu->min_period)
	289	+ period = spe_pmu->min_period;
	290	+ else if (period > max_period)
	291	+ period = max_period;
	292	+ else
	293	+ period &= max_period;
	294	+
	295	+ event->hw.sample_period = period;
	296	+}
	297	+
	298	+static u64 arm_spe_event_to_pmsirr(struct perf_event *event)
	299	+{
	300	+ struct perf_event_attr *attr = &event->attr;
	301	+ u64 reg = 0;
	302	+
	303	+ arm_spe_event_sanitise_period(event);
	304	+
	305	+ reg \|= ATTR_CFG_GET_FLD(attr, jitter) << SYS_PMSIRR_EL1_RND_SHIFT;
	306	+ reg \|= event->hw.sample_period;
	307	+
	308	+ return reg;
	309	+}
	310	+
	311	+static u64 arm_spe_event_to_pmsfcr(struct perf_event *event)
	312	+{
	313	+ struct perf_event_attr *attr = &event->attr;
	314	+ u64 reg = 0;
	315	+
	316	+ reg \|= ATTR_CFG_GET_FLD(attr, load_filter) << SYS_PMSFCR_EL1_LD_SHIFT;
	317	+ reg \|= ATTR_CFG_GET_FLD(attr, store_filter) << SYS_PMSFCR_EL1_ST_SHIFT;
	318	+ reg \|= ATTR_CFG_GET_FLD(attr, branch_filter) << SYS_PMSFCR_EL1_B_SHIFT;
	319	+
	320	+ if (reg)
	321	+ reg \|= BIT(SYS_PMSFCR_EL1_FT_SHIFT);
	322	+
	323	+ if (ATTR_CFG_GET_FLD(attr, event_filter))
	324	+ reg \|= BIT(SYS_PMSFCR_EL1_FE_SHIFT);
	325	+
	326	+ if (ATTR_CFG_GET_FLD(attr, min_latency))
	327	+ reg \|= BIT(SYS_PMSFCR_EL1_FL_SHIFT);
	328	+
	329	+ return reg;
	330	+}
	331	+
	332	+static u64 arm_spe_event_to_pmsevfr(struct perf_event *event)
	333	+{
	334	+ struct perf_event_attr *attr = &event->attr;
	335	+ return ATTR_CFG_GET_FLD(attr, event_filter);
	336	+}
	337	+
	338	+static u64 arm_spe_event_to_pmslatfr(struct perf_event *event)
	339	+{
	340	+ struct perf_event_attr *attr = &event->attr;
	341	+ return ATTR_CFG_GET_FLD(attr, min_latency)
	342	+ << SYS_PMSLATFR_EL1_MINLAT_SHIFT;
	343	+}
	344	+
	345	+static void arm_spe_pmu_pad_buf(struct perf_output_handle *handle, int len)
	346	+{
	347	+ struct arm_spe_pmu_buf *buf = perf_get_aux(handle);
	348	+ u64 head = PERF_IDX2OFF(handle->head, buf);
	349	+
	350	+ memset(buf->base + head, ARM_SPE_BUF_PAD_BYTE, len);
	351	+ if (!buf->snapshot)
	352	+ perf_aux_output_skip(handle, len);
	353	+}
	354	+
	355	+static u64 arm_spe_pmu_next_snapshot_off(struct perf_output_handle *handle)
	356	+{
	357	+ struct arm_spe_pmu_buf *buf = perf_get_aux(handle);
	358	+ struct arm_spe_pmu *spe_pmu = to_spe_pmu(handle->event->pmu);
	359	+ u64 head = PERF_IDX2OFF(handle->head, buf);
	360	+ u64 limit = buf->nr_pages * PAGE_SIZE;
	361	+
	362	+ /*
	363	+ * The trace format isn't parseable in reverse, so clamp
	364	+ * the limit to half of the buffer size in snapshot mode
	365	+ * so that the worst case is half a buffer of records, as
	366	+ * opposed to a single record.
	367	+ */
	368	+ if (head < limit >> 1)
	369	+ limit >>= 1;
	370	+
	371	+ /*
	372	+ * If we're within max_record_sz of the limit, we must
	373	+ * pad, move the head index and recompute the limit.
	374	+ */
	375	+ if (limit - head < spe_pmu->max_record_sz) {
	376	+ arm_spe_pmu_pad_buf(handle, limit - head);
	377	+ handle->head = PERF_IDX2OFF(limit, buf);
	378	+ limit = ((buf->nr_pages * PAGE_SIZE) >> 1) + handle->head;
	379	+ }
	380	+
	381	+ return limit;
	382	+}
	383	+
	384	+static u64 __arm_spe_pmu_next_off(struct perf_output_handle *handle)
	385	+{
	386	+ struct arm_spe_pmu *spe_pmu = to_spe_pmu(handle->event->pmu);
	387	+ struct arm_spe_pmu_buf *buf = perf_get_aux(handle);
	388	+ const u64 bufsize = buf->nr_pages * PAGE_SIZE;
	389	+ u64 limit = bufsize;
	390	+ u64 head, tail, wakeup;
	391	+
	392	+ /*
	393	+ * The head can be misaligned for two reasons:
	394	+ *
	395	+ * 1. The hardware left PMBPTR pointing to the first byte after
	396	+ * a record when generating a buffer management event.
	397	+ *
	398	+ * 2. We used perf_aux_output_skip to consume handle->size bytes
	399	+ * and CIRC_SPACE was used to compute the size, which always
	400	+ * leaves one entry free.
	401	+ *
	402	+ * Deal with this by padding to the next alignment boundary and
	403	+ * moving the head index. If we run out of buffer space, we'll
	404	+ * reduce handle->size to zero and end up reporting truncation.
	405	+ */
	406	+ head = PERF_IDX2OFF(handle->head, buf);
	407	+ if (!IS_ALIGNED(head, spe_pmu->align)) {
	408	+ unsigned long delta = roundup(head, spe_pmu->align) - head;
	409	+
	410	+ delta = min(delta, handle->size);
	411	+ arm_spe_pmu_pad_buf(handle, delta);
	412	+ head = PERF_IDX2OFF(handle->head, buf);
	413	+ }
	414	+
	415	+ /* If we've run out of free space, then nothing more to do */
	416	+ if (!handle->size)
	417	+ goto no_space;
	418	+
	419	+ /* Compute the tail and wakeup indices now that we've aligned head */
	420	+ tail = PERF_IDX2OFF(handle->head + handle->size, buf);
	421	+ wakeup = PERF_IDX2OFF(handle->wakeup, buf);
	422	+
	423	+ /*
	424	+ * Avoid clobbering unconsumed data. We know we have space, so
	425	+ * if we see head == tail we know that the buffer is empty. If
	426	+ * head > tail, then there's nothing to clobber prior to
	427	+ * wrapping.
	428	+ */
	429	+ if (head < tail)
	430	+ limit = round_down(tail, PAGE_SIZE);
	431	+
	432	+ /*
	433	+ * Wakeup may be arbitrarily far into the future. If it's not in
	434	+ * the current generation, either we'll wrap before hitting it,
	435	+ * or it's in the past and has been handled already.
	436	+ *
	437	+ * If there's a wakeup before we wrap, arrange to be woken up by
	438	+ * the page boundary following it. Keep the tail boundary if
	439	+ * that's lower.
	440	+ */
	441	+ if (handle->wakeup < (handle->head + handle->size) && head <= wakeup)
	442	+ limit = min(limit, round_up(wakeup, PAGE_SIZE));
	443	+
	444	+ if (limit > head)
	445	+ return limit;
	446	+
	447	+ arm_spe_pmu_pad_buf(handle, handle->size);
	448	+no_space:
	449	+ perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED);
	450	+ perf_aux_output_end(handle, 0);
	451	+ return 0;
	452	+}
	453	+
	454	+static u64 arm_spe_pmu_next_off(struct perf_output_handle *handle)
	455	+{
	456	+ struct arm_spe_pmu_buf *buf = perf_get_aux(handle);
	457	+ struct arm_spe_pmu *spe_pmu = to_spe_pmu(handle->event->pmu);
	458	+ u64 limit = __arm_spe_pmu_next_off(handle);
	459	+ u64 head = PERF_IDX2OFF(handle->head, buf);
	460	+
	461	+ /*
	462	+ * If the head has come too close to the end of the buffer,
	463	+ * then pad to the end and recompute the limit.
	464	+ */
	465	+ if (limit && (limit - head < spe_pmu->max_record_sz)) {
	466	+ arm_spe_pmu_pad_buf(handle, limit - head);
	467	+ limit = __arm_spe_pmu_next_off(handle);
	468	+ }
	469	+
	470	+ return limit;
	471	+}
	472	+
	473	+static void arm_spe_perf_aux_output_begin(struct perf_output_handle *handle,
	474	+ struct perf_event *event)
	475	+{
	476	+ u64 base, limit;
	477	+ struct arm_spe_pmu_buf *buf;
	478	+
	479	+ /* Start a new aux session */
	480	+ buf = perf_aux_output_begin(handle, event);
	481	+ if (!buf) {
	482	+ event->hw.state \|= PERF_HES_STOPPED;
	483	+ /*
	484	+ * We still need to clear the limit pointer, since the
	485	+ * profiler might only be disabled by virtue of a fault.
	486	+ */
	487	+ limit = 0;
	488	+ goto out_write_limit;
	489	+ }
	490	+
	491	+ limit = buf->snapshot ? arm_spe_pmu_next_snapshot_off(handle)
	492	+ : arm_spe_pmu_next_off(handle);
	493	+ if (limit)
	494	+ limit \|= BIT(SYS_PMBLIMITR_EL1_E_SHIFT);
	495	+
	496	+ limit += (u64)buf->base;
	497	+ base = (u64)buf->base + PERF_IDX2OFF(handle->head, buf);
	498	+ write_sysreg_s(base, SYS_PMBPTR_EL1);
	499	+
	500	+out_write_limit:
	501	+ write_sysreg_s(limit, SYS_PMBLIMITR_EL1);
	502	+}
	503	+
	504	+static void arm_spe_perf_aux_output_end(struct perf_output_handle *handle)
	505	+{
	506	+ struct arm_spe_pmu_buf *buf = perf_get_aux(handle);
	507	+ u64 offset, size;
	508	+
	509	+ offset = read_sysreg_s(SYS_PMBPTR_EL1) - (u64)buf->base;
	510	+ size = offset - PERF_IDX2OFF(handle->head, buf);
	511	+
	512	+ if (buf->snapshot)
	513	+ handle->head = offset;
	514	+
	515	+ perf_aux_output_end(handle, size);
	516	+}
	517	+
	518	+static void arm_spe_pmu_disable_and_drain_local(void)
	519	+{
	520	+ /* Disable profiling at EL0 and EL1 */
	521	+ write_sysreg_s(0, SYS_PMSCR_EL1);
	522	+ isb();
	523	+
	524	+ /* Drain any buffered data */
	525	+ psb_csync();
	526	+ dsb(nsh);
	527	+
	528	+ /* Disable the profiling buffer */
	529	+ write_sysreg_s(0, SYS_PMBLIMITR_EL1);
	530	+ isb();
	531	+}
	532	+
	533	+/* IRQ handling */
	534	+static enum arm_spe_pmu_buf_fault_action
	535	+arm_spe_pmu_buf_get_fault_act(struct perf_output_handle *handle)
	536	+{
	537	+ const char *err_str;
	538	+ u64 pmbsr;
	539	+ enum arm_spe_pmu_buf_fault_action ret;
	540	+
	541	+ /*
	542	+ * Ensure new profiling data is visible to the CPU and any external
	543	+ * aborts have been resolved.
	544	+ */
	545	+ psb_csync();
	546	+ dsb(nsh);
	547	+
	548	+ /* Ensure hardware updates to PMBPTR_EL1 are visible */
	549	+ isb();
	550	+
	551	+ /* Service required? */
	552	+ pmbsr = read_sysreg_s(SYS_PMBSR_EL1);
	553	+ if (!(pmbsr & BIT(SYS_PMBSR_EL1_S_SHIFT)))
	554	+ return SPE_PMU_BUF_FAULT_ACT_SPURIOUS;
	555	+
	556	+ /*
	557	+ * If we've lost data, disable profiling and also set the PARTIAL
	558	+ * flag to indicate that the last record is corrupted.
	559	+ */
	560	+ if (pmbsr & BIT(SYS_PMBSR_EL1_DL_SHIFT))
	561	+ perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED \|
	562	+ PERF_AUX_FLAG_PARTIAL);
	563	+
	564	+ /* Report collisions to userspace so that it can up the period */
	565	+ if (pmbsr & BIT(SYS_PMBSR_EL1_COLL_SHIFT))
	566	+ perf_aux_output_flag(handle, PERF_AUX_FLAG_COLLISION);
	567	+
	568	+ /* We only expect buffer management events */
	569	+ switch (pmbsr & (SYS_PMBSR_EL1_EC_MASK << SYS_PMBSR_EL1_EC_SHIFT)) {
	570	+ case SYS_PMBSR_EL1_EC_BUF:
	571	+ /* Handled below */
	572	+ break;
	573	+ case SYS_PMBSR_EL1_EC_FAULT_S1:
	574	+ case SYS_PMBSR_EL1_EC_FAULT_S2:
	575	+ err_str = "Unexpected buffer fault";
	576	+ goto out_err;
	577	+ default:
	578	+ err_str = "Unknown error code";
	579	+ goto out_err;
	580	+ }
	581	+
	582	+ /* Buffer management event */
	583	+ switch (pmbsr &
	584	+ (SYS_PMBSR_EL1_BUF_BSC_MASK << SYS_PMBSR_EL1_BUF_BSC_SHIFT)) {
	585	+ case SYS_PMBSR_EL1_BUF_BSC_FULL:
	586	+ ret = SPE_PMU_BUF_FAULT_ACT_OK;
	587	+ goto out_stop;
	588	+ default:
	589	+ err_str = "Unknown buffer status code";
	590	+ }
	591	+
	592	+out_err:
	593	+ pr_err_ratelimited("%s on CPU %d [PMBSR=0x%016llx, PMBPTR=0x%016llx, PMBLIMITR=0x%016llx]\n",
	594	+ err_str, smp_processor_id(), pmbsr,
	595	+ read_sysreg_s(SYS_PMBPTR_EL1),
	596	+ read_sysreg_s(SYS_PMBLIMITR_EL1));
	597	+ ret = SPE_PMU_BUF_FAULT_ACT_FATAL;
	598	+
	599	+out_stop:
	600	+ arm_spe_perf_aux_output_end(handle);
	601	+ return ret;
	602	+}
	603	+
	604	+static irqreturn_t arm_spe_pmu_irq_handler(int irq, void *dev)
	605	+{
	606	+ struct perf_output_handle *handle = dev;
	607	+ struct perf_event *event = handle->event;
	608	+ enum arm_spe_pmu_buf_fault_action act;
	609	+
	610	+ if (!perf_get_aux(handle))
	611	+ return IRQ_NONE;
	612	+
	613	+ act = arm_spe_pmu_buf_get_fault_act(handle);
	614	+ if (act == SPE_PMU_BUF_FAULT_ACT_SPURIOUS)
	615	+ return IRQ_NONE;
	616	+
	617	+ /*
	618	+ * Ensure perf callbacks have completed, which may disable the
	619	+ * profiling buffer in response to a TRUNCATION flag.
	620	+ */
	621	+ irq_work_run();
	622	+
	623	+ switch (act) {
	624	+ case SPE_PMU_BUF_FAULT_ACT_FATAL:
	625	+ /*
	626	+ * If a fatal exception occurred then leaving the profiling
	627	+ * buffer enabled is a recipe waiting to happen. Since
	628	+ * fatal faults don't always imply truncation, make sure
	629	+ * that the profiling buffer is disabled explicitly before
	630	+ * clearing the syndrome register.
	631	+ */
	632	+ arm_spe_pmu_disable_and_drain_local();
	633	+ break;
	634	+ case SPE_PMU_BUF_FAULT_ACT_OK:
	635	+ /*
	636	+ * We handled the fault (the buffer was full), so resume
	637	+ * profiling as long as we didn't detect truncation.
	638	+ * PMBPTR might be misaligned, but we'll burn that bridge
	639	+ * when we get to it.
	640	+ */
	641	+ if (!(handle->aux_flags & PERF_AUX_FLAG_TRUNCATED)) {
	642	+ arm_spe_perf_aux_output_begin(handle, event);
	643	+ isb();
	644	+ }
	645	+ break;
	646	+ case SPE_PMU_BUF_FAULT_ACT_SPURIOUS:
	647	+ /* We've seen you before, but GCC has the memory of a sieve. */
	648	+ break;
	649	+ }
	650	+
	651	+ /* The buffer pointers are now sane, so resume profiling. */
	652	+ write_sysreg_s(0, SYS_PMBSR_EL1);
	653	+ return IRQ_HANDLED;
	654	+}
	655	+
	656	+/* Perf callbacks */
	657	+static int arm_spe_pmu_event_init(struct perf_event *event)
	658	+{
	659	+ u64 reg;
	660	+ struct perf_event_attr *attr = &event->attr;
	661	+ struct arm_spe_pmu *spe_pmu = to_spe_pmu(event->pmu);
	662	+
	663	+ /* This is, of course, deeply driver-specific */
	664	+ if (attr->type != event->pmu->type)
	665	+ return -ENOENT;
	666	+
	667	+ if (event->cpu >= 0 &&
	668	+ !cpumask_test_cpu(event->cpu, &spe_pmu->supported_cpus))
	669	+ return -ENOENT;
	670	+
	671	+ if (arm_spe_event_to_pmsevfr(event) & SYS_PMSEVFR_EL1_RES0)
	672	+ return -EOPNOTSUPP;
	673	+
	674	+ if (attr->exclude_idle)
	675	+ return -EOPNOTSUPP;
	676	+
	677	+ /*
	678	+ * Feedback-directed frequency throttling doesn't work when we
	679	+ * have a buffer of samples. We'd need to manually count the
	680	+ * samples in the buffer when it fills up and adjust the event
	681	+ * count to reflect that. Instead, just force the user to specify
	682	+ * a sample period.
	683	+ */
	684	+ if (attr->freq)
	685	+ return -EINVAL;
	686	+
	687	+ reg = arm_spe_event_to_pmsfcr(event);
	688	+ if ((reg & BIT(SYS_PMSFCR_EL1_FE_SHIFT)) &&
	689	+ !(spe_pmu->features & SPE_PMU_FEAT_FILT_EVT))
	690	+ return -EOPNOTSUPP;
	691	+
	692	+ if ((reg & BIT(SYS_PMSFCR_EL1_FT_SHIFT)) &&
	693	+ !(spe_pmu->features & SPE_PMU_FEAT_FILT_TYP))
	694	+ return -EOPNOTSUPP;
	695	+
	696	+ if ((reg & BIT(SYS_PMSFCR_EL1_FL_SHIFT)) &&
	697	+ !(spe_pmu->features & SPE_PMU_FEAT_FILT_LAT))
	698	+ return -EOPNOTSUPP;
	699	+
	700	+ reg = arm_spe_event_to_pmscr(event);
	701	+ if (!capable(CAP_SYS_ADMIN) &&
	702	+ (reg & (BIT(SYS_PMSCR_EL1_PA_SHIFT) \|
	703	+ BIT(SYS_PMSCR_EL1_CX_SHIFT) \|
	704	+ BIT(SYS_PMSCR_EL1_PCT_SHIFT))))
	705	+ return -EACCES;
	706	+
	707	+ return 0;
	708	+}
	709	+
	710	+static void arm_spe_pmu_start(struct perf_event *event, int flags)
	711	+{
	712	+ u64 reg;
	713	+ struct arm_spe_pmu *spe_pmu = to_spe_pmu(event->pmu);
	714	+ struct hw_perf_event *hwc = &event->hw;
	715	+ struct perf_output_handle *handle = this_cpu_ptr(spe_pmu->handle);
	716	+
	717	+ hwc->state = 0;
	718	+ arm_spe_perf_aux_output_begin(handle, event);
	719	+ if (hwc->state)
	720	+ return;
	721	+
	722	+ reg = arm_spe_event_to_pmsfcr(event);
	723	+ write_sysreg_s(reg, SYS_PMSFCR_EL1);
	724	+
	725	+ reg = arm_spe_event_to_pmsevfr(event);
	726	+ write_sysreg_s(reg, SYS_PMSEVFR_EL1);
	727	+
	728	+ reg = arm_spe_event_to_pmslatfr(event);
	729	+ write_sysreg_s(reg, SYS_PMSLATFR_EL1);
	730	+
	731	+ if (flags & PERF_EF_RELOAD) {
	732	+ reg = arm_spe_event_to_pmsirr(event);
	733	+ write_sysreg_s(reg, SYS_PMSIRR_EL1);
	734	+ isb();
	735	+ reg = local64_read(&hwc->period_left);
	736	+ write_sysreg_s(reg, SYS_PMSICR_EL1);
	737	+ }
	738	+
	739	+ reg = arm_spe_event_to_pmscr(event);
	740	+ isb();
	741	+ write_sysreg_s(reg, SYS_PMSCR_EL1);
	742	+}
	743	+
	744	+static void arm_spe_pmu_stop(struct perf_event *event, int flags)
	745	+{
	746	+ struct arm_spe_pmu *spe_pmu = to_spe_pmu(event->pmu);
	747	+ struct hw_perf_event *hwc = &event->hw;
	748	+ struct perf_output_handle *handle = this_cpu_ptr(spe_pmu->handle);
	749	+
	750	+ /* If we're already stopped, then nothing to do */
	751	+ if (hwc->state & PERF_HES_STOPPED)
	752	+ return;
	753	+
	754	+ /* Stop all trace generation */
	755	+ arm_spe_pmu_disable_and_drain_local();
	756	+
	757	+ if (flags & PERF_EF_UPDATE) {
	758	+ /*
	759	+ * If there's a fault pending then ensure we contain it
	760	+ * to this buffer, since we might be on the context-switch
	761	+ * path.
	762	+ */
	763	+ if (perf_get_aux(handle)) {
	764	+ enum arm_spe_pmu_buf_fault_action act;
	765	+
	766	+ act = arm_spe_pmu_buf_get_fault_act(handle);
	767	+ if (act == SPE_PMU_BUF_FAULT_ACT_SPURIOUS)
	768	+ arm_spe_perf_aux_output_end(handle);
	769	+ else
	770	+ write_sysreg_s(0, SYS_PMBSR_EL1);
	771	+ }
	772	+
	773	+ /*
	774	+ * This may also contain ECOUNT, but nobody else should
	775	+ * be looking at period_left, since we forbid frequency
	776	+ * based sampling.
	777	+ */
	778	+ local64_set(&hwc->period_left, read_sysreg_s(SYS_PMSICR_EL1));
	779	+ hwc->state \|= PERF_HES_UPTODATE;
	780	+ }
	781	+
	782	+ hwc->state \|= PERF_HES_STOPPED;
	783	+}
	784	+
	785	+static int arm_spe_pmu_add(struct perf_event *event, int flags)
	786	+{
	787	+ int ret = 0;
	788	+ struct arm_spe_pmu *spe_pmu = to_spe_pmu(event->pmu);
	789	+ struct hw_perf_event *hwc = &event->hw;
	790	+ int cpu = event->cpu == -1 ? smp_processor_id() : event->cpu;
	791	+
	792	+ if (!cpumask_test_cpu(cpu, &spe_pmu->supported_cpus))
	793	+ return -ENOENT;
	794	+
	795	+ hwc->state = PERF_HES_UPTODATE \| PERF_HES_STOPPED;
	796	+
	797	+ if (flags & PERF_EF_START) {
	798	+ arm_spe_pmu_start(event, PERF_EF_RELOAD);
	799	+ if (hwc->state & PERF_HES_STOPPED)
	800	+ ret = -EINVAL;
	801	+ }
	802	+
	803	+ return ret;
	804	+}
	805	+
	806	+static void arm_spe_pmu_del(struct perf_event *event, int flags)
	807	+{
	808	+ arm_spe_pmu_stop(event, PERF_EF_UPDATE);
	809	+}
	810	+
	811	+static void arm_spe_pmu_read(struct perf_event *event)
	812	+{
	813	+}
	814	+
	815	+static void arm_spe_pmu_setup_aux(int cpu, void *pages, int nr_pages,
	816	+ bool snapshot)
	817	+{
	818	+ int i;
	819	+ struct page **pglist;
	820	+ struct arm_spe_pmu_buf *buf;
	821	+
	822	+ /* We need at least two pages for this to work. */
	823	+ if (nr_pages < 2)
	824	+ return NULL;
	825	+
	826	+ /*
	827	+ * We require an even number of pages for snapshot mode, so that
	828	+ * we can effectively treat the buffer as consisting of two equal
	829	+ * parts and give userspace a fighting chance of getting some
	830	+ * useful data out of it.
	831	+ */
	832	+ if (!nr_pages \|\| (snapshot && (nr_pages & 1)))
	833	+ return NULL;
	834	+
	835	+ if (cpu == -1)
	836	+ cpu = raw_smp_processor_id();
	837	+
	838	+ buf = kzalloc_node(sizeof(*buf), GFP_KERNEL, cpu_to_node(cpu));
	839	+ if (!buf)
	840	+ return NULL;
	841	+
	842	+ pglist = kcalloc(nr_pages, sizeof(*pglist), GFP_KERNEL);
	843	+ if (!pglist)
	844	+ goto out_free_buf;
	845	+
	846	+ for (i = 0; i < nr_pages; ++i) {
	847	+ struct page *page = virt_to_page(pages[i]);
	848	+
	849	+ if (PagePrivate(page)) {
	850	+ pr_warn("unexpected high-order page for auxbuf!");
	851	+ goto out_free_pglist;
	852	+ }
	853	+
	854	+ pglist[i] = virt_to_page(pages[i]);
	855	+ }
	856	+
	857	+ buf->base = vmap(pglist, nr_pages, VM_MAP, PAGE_KERNEL);
	858	+ if (!buf->base)
	859	+ goto out_free_pglist;
	860	+
	861	+ buf->nr_pages = nr_pages;
	862	+ buf->snapshot = snapshot;
	863	+
	864	+ kfree(pglist);
	865	+ return buf;
	866	+
	867	+out_free_pglist:
	868	+ kfree(pglist);
	869	+out_free_buf:
	870	+ kfree(buf);
	871	+ return NULL;
	872	+}
	873	+
	874	+static void arm_spe_pmu_free_aux(void *aux)
	875	+{
	876	+ struct arm_spe_pmu_buf *buf = aux;
	877	+
	878	+ vunmap(buf->base);
	879	+ kfree(buf);
	880	+}
	881	+
	882	+/* Initialisation and teardown functions */
	883	+static int arm_spe_pmu_perf_init(struct arm_spe_pmu *spe_pmu)
	884	+{
	885	+ static atomic_t pmu_idx = ATOMIC_INIT(-1);
	886	+
	887	+ int idx;
	888	+ char *name;
	889	+ struct device *dev = &spe_pmu->pdev->dev;
	890	+
	891	+ spe_pmu->pmu = (struct pmu) {
	892	+ .module = THIS_MODULE,
	893	+ .capabilities = PERF_PMU_CAP_EXCLUSIVE \| PERF_PMU_CAP_ITRACE,
	894	+ .attr_groups = arm_spe_pmu_attr_groups,
	895	+ /*
	896	+ * We hitch a ride on the software context here, so that
	897	+ * we can support per-task profiling (which is not possible
	898	+ * with the invalid context as it doesn't get sched callbacks).
	899	+ * This requires that userspace either uses a dummy event for
	900	+ * perf_event_open, since the aux buffer is not setup until
	901	+ * a subsequent mmap, or creates the profiling event in a
	902	+ * disabled state and explicitly PERF_EVENT_IOC_ENABLEs it
	903	+ * once the buffer has been created.
	904	+ */
	905	+ .task_ctx_nr = perf_sw_context,
	906	+ .event_init = arm_spe_pmu_event_init,
	907	+ .add = arm_spe_pmu_add,
	908	+ .del = arm_spe_pmu_del,
	909	+ .start = arm_spe_pmu_start,
	910	+ .stop = arm_spe_pmu_stop,
	911	+ .read = arm_spe_pmu_read,
	912	+ .setup_aux = arm_spe_pmu_setup_aux,
	913	+ .free_aux = arm_spe_pmu_free_aux,
	914	+ };
	915	+
	916	+ idx = atomic_inc_return(&pmu_idx);
	917	+ name = devm_kasprintf(dev, GFP_KERNEL, "%s_%d", PMUNAME, idx);
	918	+ return perf_pmu_register(&spe_pmu->pmu, name, -1);
	919	+}
	920	+
	921	+static void arm_spe_pmu_perf_destroy(struct arm_spe_pmu *spe_pmu)
	922	+{
	923	+ perf_pmu_unregister(&spe_pmu->pmu);
	924	+}
	925	+
	926	+static void __arm_spe_pmu_dev_probe(void *info)
	927	+{
	928	+ int fld;
	929	+ u64 reg;
	930	+ struct arm_spe_pmu *spe_pmu = info;
	931	+ struct device *dev = &spe_pmu->pdev->dev;
	932	+
	933	+ fld = cpuid_feature_extract_unsigned_field(read_cpuid(ID_AA64DFR0_EL1),
	934	+ ID_AA64DFR0_PMSVER_SHIFT);
	935	+ if (!fld) {
	936	+ dev_err(dev,
	937	+ "unsupported ID_AA64DFR0_EL1.PMSVer [%d] on CPU %d\n",
	938	+ fld, smp_processor_id());
	939	+ return;
	940	+ }
	941	+
	942	+ /* Read PMBIDR first to determine whether or not we have access */
	943	+ reg = read_sysreg_s(SYS_PMBIDR_EL1);
	944	+ if (reg & BIT(SYS_PMBIDR_EL1_P_SHIFT)) {
	945	+ dev_err(dev,
	946	+ "profiling buffer owned by higher exception level\n");
	947	+ return;
	948	+ }
	949	+
	950	+ /* Minimum alignment. If it's out-of-range, then fail the probe */
	951	+ fld = reg >> SYS_PMBIDR_EL1_ALIGN_SHIFT & SYS_PMBIDR_EL1_ALIGN_MASK;
	952	+ spe_pmu->align = 1 << fld;
	953	+ if (spe_pmu->align > SZ_2K) {
	954	+ dev_err(dev, "unsupported PMBIDR.Align [%d] on CPU %d\n",
	955	+ fld, smp_processor_id());
	956	+ return;
	957	+ }
	958	+
	959	+ /* It's now safe to read PMSIDR and figure out what we've got */
	960	+ reg = read_sysreg_s(SYS_PMSIDR_EL1);
	961	+ if (reg & BIT(SYS_PMSIDR_EL1_FE_SHIFT))
	962	+ spe_pmu->features \|= SPE_PMU_FEAT_FILT_EVT;
	963	+
	964	+ if (reg & BIT(SYS_PMSIDR_EL1_FT_SHIFT))
	965	+ spe_pmu->features \|= SPE_PMU_FEAT_FILT_TYP;
	966	+
	967	+ if (reg & BIT(SYS_PMSIDR_EL1_FL_SHIFT))
	968	+ spe_pmu->features \|= SPE_PMU_FEAT_FILT_LAT;
	969	+
	970	+ if (reg & BIT(SYS_PMSIDR_EL1_ARCHINST_SHIFT))
	971	+ spe_pmu->features \|= SPE_PMU_FEAT_ARCH_INST;
	972	+
	973	+ if (reg & BIT(SYS_PMSIDR_EL1_LDS_SHIFT))
	974	+ spe_pmu->features \|= SPE_PMU_FEAT_LDS;
	975	+
	976	+ if (reg & BIT(SYS_PMSIDR_EL1_ERND_SHIFT))
	977	+ spe_pmu->features \|= SPE_PMU_FEAT_ERND;
	978	+
	979	+ /* This field has a spaced out encoding, so just use a look-up */
	980	+ fld = reg >> SYS_PMSIDR_EL1_INTERVAL_SHIFT & SYS_PMSIDR_EL1_INTERVAL_MASK;
	981	+ switch (fld) {
	982	+ case 0:
	983	+ spe_pmu->min_period = 256;
	984	+ break;
	985	+ case 2:
	986	+ spe_pmu->min_period = 512;
	987	+ break;
	988	+ case 3:
	989	+ spe_pmu->min_period = 768;
	990	+ break;
	991	+ case 4:
	992	+ spe_pmu->min_period = 1024;
	993	+ break;
	994	+ case 5:
	995	+ spe_pmu->min_period = 1536;
	996	+ break;
	997	+ case 6:
	998	+ spe_pmu->min_period = 2048;
	999	+ break;
	1000	+ case 7:
	1001	+ spe_pmu->min_period = 3072;
	1002	+ break;
	1003	+ default:
	1004	+ dev_warn(dev, "unknown PMSIDR_EL1.Interval [%d]; assuming 8\n",
	1005	+ fld);
	1006	+ /* Fallthrough */
	1007	+ case 8:
	1008	+ spe_pmu->min_period = 4096;
	1009	+ }
	1010	+
	1011	+ /* Maximum record size. If it's out-of-range, then fail the probe */
	1012	+ fld = reg >> SYS_PMSIDR_EL1_MAXSIZE_SHIFT & SYS_PMSIDR_EL1_MAXSIZE_MASK;
	1013	+ spe_pmu->max_record_sz = 1 << fld;
	1014	+ if (spe_pmu->max_record_sz > SZ_2K \|\| spe_pmu->max_record_sz < 16) {
	1015	+ dev_err(dev, "unsupported PMSIDR_EL1.MaxSize [%d] on CPU %d\n",
	1016	+ fld, smp_processor_id());
	1017	+ return;
	1018	+ }
	1019	+
	1020	+ fld = reg >> SYS_PMSIDR_EL1_COUNTSIZE_SHIFT & SYS_PMSIDR_EL1_COUNTSIZE_MASK;
	1021	+ switch (fld) {
	1022	+ default:
	1023	+ dev_warn(dev, "unknown PMSIDR_EL1.CountSize [%d]; assuming 2\n",
	1024	+ fld);
	1025	+ /* Fallthrough */
	1026	+ case 2:
	1027	+ spe_pmu->counter_sz = 12;
	1028	+ }
	1029	+
	1030	+ dev_info(dev,
	1031	+ "probed for CPUs %*pbl [max_record_sz %u, align %u, features 0x%llx]\n",
	1032	+ cpumask_pr_args(&spe_pmu->supported_cpus),
	1033	+ spe_pmu->max_record_sz, spe_pmu->align, spe_pmu->features);
	1034	+
	1035	+ spe_pmu->features \|= SPE_PMU_FEAT_DEV_PROBED;
	1036	+ return;
	1037	+}
	1038	+
	1039	+static void __arm_spe_pmu_reset_local(void)
	1040	+{
	1041	+ /*
	1042	+ * This is probably overkill, as we have no idea where we're
	1043	+ * draining any buffered data to...
	1044	+ */
	1045	+ arm_spe_pmu_disable_and_drain_local();
	1046	+
	1047	+ /* Reset the buffer base pointer */
	1048	+ write_sysreg_s(0, SYS_PMBPTR_EL1);
	1049	+ isb();
	1050	+
	1051	+ /* Clear any pending management interrupts */
	1052	+ write_sysreg_s(0, SYS_PMBSR_EL1);
	1053	+ isb();
	1054	+}
	1055	+
	1056	+static void __arm_spe_pmu_setup_one(void *info)
	1057	+{
	1058	+ struct arm_spe_pmu *spe_pmu = info;
	1059	+
	1060	+ __arm_spe_pmu_reset_local();
	1061	+ enable_percpu_irq(spe_pmu->irq, IRQ_TYPE_NONE);
	1062	+}
	1063	+
	1064	+static void __arm_spe_pmu_stop_one(void *info)
	1065	+{
	1066	+ struct arm_spe_pmu *spe_pmu = info;
	1067	+
	1068	+ disable_percpu_irq(spe_pmu->irq);
	1069	+ __arm_spe_pmu_reset_local();
	1070	+}
	1071	+
	1072	+static int arm_spe_pmu_cpu_startup(unsigned int cpu, struct hlist_node *node)
	1073	+{
	1074	+ struct arm_spe_pmu *spe_pmu;
	1075	+
	1076	+ spe_pmu = hlist_entry_safe(node, struct arm_spe_pmu, hotplug_node);
	1077	+ if (!cpumask_test_cpu(cpu, &spe_pmu->supported_cpus))
	1078	+ return 0;
	1079	+
	1080	+ __arm_spe_pmu_setup_one(spe_pmu);
	1081	+ return 0;
	1082	+}
	1083	+
	1084	+static int arm_spe_pmu_cpu_teardown(unsigned int cpu, struct hlist_node *node)
	1085	+{
	1086	+ struct arm_spe_pmu *spe_pmu;
	1087	+
	1088	+ spe_pmu = hlist_entry_safe(node, struct arm_spe_pmu, hotplug_node);
	1089	+ if (!cpumask_test_cpu(cpu, &spe_pmu->supported_cpus))
	1090	+ return 0;
	1091	+
	1092	+ __arm_spe_pmu_stop_one(spe_pmu);
	1093	+ return 0;
	1094	+}
	1095	+
	1096	+static int arm_spe_pmu_dev_init(struct arm_spe_pmu *spe_pmu)
	1097	+{
	1098	+ int ret;
	1099	+ cpumask_t *mask = &spe_pmu->supported_cpus;
	1100	+
	1101	+ /* Make sure we probe the hardware on a relevant CPU */
	1102	+ ret = smp_call_function_any(mask, __arm_spe_pmu_dev_probe, spe_pmu, 1);
	1103	+ if (ret \|\| !(spe_pmu->features & SPE_PMU_FEAT_DEV_PROBED))
	1104	+ return -ENXIO;
	1105	+
	1106	+ /* Request our PPIs (note that the IRQ is still disabled) */
	1107	+ ret = request_percpu_irq(spe_pmu->irq, arm_spe_pmu_irq_handler, DRVNAME,
	1108	+ spe_pmu->handle);
	1109	+ if (ret)
	1110	+ return ret;
	1111	+
	1112	+ /*
	1113	+ * Register our hotplug notifier now so we don't miss any events.
	1114	+ * This will enable the IRQ for any supported CPUs that are already
	1115	+ * up.
	1116	+ */
	1117	+ ret = cpuhp_state_add_instance(arm_spe_pmu_online,
	1118	+ &spe_pmu->hotplug_node);
	1119	+ if (ret)
	1120	+ free_percpu_irq(spe_pmu->irq, spe_pmu->handle);
	1121	+
	1122	+ return ret;
	1123	+}
	1124	+
	1125	+static void arm_spe_pmu_dev_teardown(struct arm_spe_pmu *spe_pmu)
	1126	+{
	1127	+ cpuhp_state_remove_instance(arm_spe_pmu_online, &spe_pmu->hotplug_node);
	1128	+ free_percpu_irq(spe_pmu->irq, spe_pmu->handle);
	1129	+}
	1130	+
	1131	+/* Driver and device probing */
	1132	+static int arm_spe_pmu_irq_probe(struct arm_spe_pmu *spe_pmu)
	1133	+{
	1134	+ struct platform_device *pdev = spe_pmu->pdev;
	1135	+ int irq = platform_get_irq(pdev, 0);
	1136	+
	1137	+ if (irq < 0) {
	1138	+ dev_err(&pdev->dev, "failed to get IRQ (%d)\n", irq);
	1139	+ return -ENXIO;
	1140	+ }
	1141	+
	1142	+ if (!irq_is_percpu(irq)) {
	1143	+ dev_err(&pdev->dev, "expected PPI but got SPI (%d)\n", irq);
	1144	+ return -EINVAL;
	1145	+ }
	1146	+
	1147	+ if (irq_get_percpu_devid_partition(irq, &spe_pmu->supported_cpus)) {
	1148	+ dev_err(&pdev->dev, "failed to get PPI partition (%d)\n", irq);
	1149	+ return -EINVAL;
	1150	+ }
	1151	+
	1152	+ spe_pmu->irq = irq;
	1153	+ return 0;
	1154	+}
	1155	+
	1156	+static const struct of_device_id arm_spe_pmu_of_match[] = {
	1157	+ { .compatible = "arm,statistical-profiling-extension-v1", .data = (void *)1 },
	1158	+ { /* Sentinel */ },
	1159	+};
	1160	+
	1161	+static int arm_spe_pmu_device_dt_probe(struct platform_device *pdev)
	1162	+{
	1163	+ int ret;
	1164	+ struct arm_spe_pmu *spe_pmu;
	1165	+ struct device *dev = &pdev->dev;
	1166	+
	1167	+ spe_pmu = devm_kzalloc(dev, sizeof(*spe_pmu), GFP_KERNEL);
	1168	+ if (!spe_pmu) {
	1169	+ dev_err(dev, "failed to allocate spe_pmu\n");
	1170	+ return -ENOMEM;
	1171	+ }
	1172	+
	1173	+ spe_pmu->handle = alloc_percpu(typeof(*spe_pmu->handle));
	1174	+ if (!spe_pmu->handle)
	1175	+ return -ENOMEM;
	1176	+
	1177	+ spe_pmu->pdev = pdev;
	1178	+ platform_set_drvdata(pdev, spe_pmu);
	1179	+
	1180	+ ret = arm_spe_pmu_irq_probe(spe_pmu);
	1181	+ if (ret)
	1182	+ goto out_free_handle;
	1183	+
	1184	+ ret = arm_spe_pmu_dev_init(spe_pmu);
	1185	+ if (ret)
	1186	+ goto out_free_handle;
	1187	+
	1188	+ ret = arm_spe_pmu_perf_init(spe_pmu);
	1189	+ if (ret)
	1190	+ goto out_teardown_dev;
	1191	+
	1192	+ return 0;
	1193	+
	1194	+out_teardown_dev:
	1195	+ arm_spe_pmu_dev_teardown(spe_pmu);
	1196	+out_free_handle:
	1197	+ free_percpu(spe_pmu->handle);
	1198	+ return ret;
	1199	+}
	1200	+
	1201	+static int arm_spe_pmu_device_remove(struct platform_device *pdev)
	1202	+{
	1203	+ struct arm_spe_pmu *spe_pmu = platform_get_drvdata(pdev);
	1204	+
	1205	+ arm_spe_pmu_perf_destroy(spe_pmu);
	1206	+ arm_spe_pmu_dev_teardown(spe_pmu);
	1207	+ free_percpu(spe_pmu->handle);
	1208	+ return 0;
	1209	+}
	1210	+
	1211	+static struct platform_driver arm_spe_pmu_driver = {
	1212	+ .driver = {
	1213	+ .name = DRVNAME,
	1214	+ .of_match_table = of_match_ptr(arm_spe_pmu_of_match),
	1215	+ },
	1216	+ .probe = arm_spe_pmu_device_dt_probe,
	1217	+ .remove = arm_spe_pmu_device_remove,
	1218	+};
	1219	+
	1220	+static int __init arm_spe_pmu_init(void)
	1221	+{
	1222	+ int ret;
	1223	+
	1224	+ ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, DRVNAME,
	1225	+ arm_spe_pmu_cpu_startup,
	1226	+ arm_spe_pmu_cpu_teardown);
	1227	+ if (ret < 0)
	1228	+ return ret;
	1229	+ arm_spe_pmu_online = ret;
	1230	+
	1231	+ ret = platform_driver_register(&arm_spe_pmu_driver);
	1232	+ if (ret)
	1233	+ cpuhp_remove_multi_state(arm_spe_pmu_online);
	1234	+
	1235	+ return ret;
	1236	+}
	1237	+
	1238	+static void __exit arm_spe_pmu_exit(void)
	1239	+{
	1240	+ platform_driver_unregister(&arm_spe_pmu_driver);
	1241	+ cpuhp_remove_multi_state(arm_spe_pmu_online);
	1242	+}
	1243	+
	1244	+module_init(arm_spe_pmu_init);
	1245	+module_exit(arm_spe_pmu_exit);
	1246	+
	1247	+MODULE_DESCRIPTION("Perf driver for the ARMv8.2 Statistical Profiling Extension");
	1248	+MODULE_AUTHOR("Will Deacon <will.deacon@arm.com>");
	1249	+MODULE_LICENSE("GPL v2");

drivers/perf/hisilicon/Makefile+1−0

@@ -0,0 +1 @@
	1	+obj-$(CONFIG_HISI_PMU) += hisi_uncore_pmu.o hisi_uncore_l3c_pmu.o hisi_uncore_hha_pmu.o hisi_uncore_ddrc_pmu.o

drivers/perf/hisilicon/hisi_uncore_ddrc_pmu.c+463−0

@@ -0,0 +1,463 @@
	1	+/*
	2	+ * HiSilicon SoC DDRC uncore Hardware event counters support
	3	+ *
	4	+ * Copyright (C) 2017 Hisilicon Limited
	5	+ * Author: Shaokun Zhang <zhangshaokun@hisilicon.com>
	6	+ * Anurup M <anurup.m@huawei.com>
	7	+ *
	8	+ * This code is based on the uncore PMUs like arm-cci and arm-ccn.
	9	+ *
	10	+ * This program is free software; you can redistribute it and/or modify
	11	+ * it under the terms of the GNU General Public License version 2 as
	12	+ * published by the Free Software Foundation.
	13	+ */
	14	+#include <linux/acpi.h>
	15	+#include <linux/bug.h>
	16	+#include <linux/cpuhotplug.h>
	17	+#include <linux/interrupt.h>
	18	+#include <linux/irq.h>
	19	+#include <linux/list.h>
	20	+#include <linux/platform_device.h>
	21	+#include <linux/smp.h>
	22	+
	23	+#include "hisi_uncore_pmu.h"
	24	+
	25	+/* DDRC register definition */
	26	+#define DDRC_PERF_CTRL 0x010
	27	+#define DDRC_FLUX_WR 0x380
	28	+#define DDRC_FLUX_RD 0x384
	29	+#define DDRC_FLUX_WCMD 0x388
	30	+#define DDRC_FLUX_RCMD 0x38c
	31	+#define DDRC_PRE_CMD 0x3c0
	32	+#define DDRC_ACT_CMD 0x3c4
	33	+#define DDRC_BNK_CHG 0x3c8
	34	+#define DDRC_RNK_CHG 0x3cc
	35	+#define DDRC_EVENT_CTRL 0x6C0
	36	+#define DDRC_INT_MASK 0x6c8
	37	+#define DDRC_INT_STATUS 0x6cc
	38	+#define DDRC_INT_CLEAR 0x6d0
	39	+
	40	+/* DDRC has 8-counters */
	41	+#define DDRC_NR_COUNTERS 0x8
	42	+#define DDRC_PERF_CTRL_EN 0x2
	43	+
	44	+/*
	45	+ * For DDRC PMU, there are eight-events and every event has been mapped
	46	+ * to fixed-purpose counters which register offset is not consistent.
	47	+ * Therefore there is no write event type and we assume that event
	48	+ * code (0 to 7) is equal to counter index in PMU driver.
	49	+ */
	50	+#define GET_DDRC_EVENTID(hwc) (hwc->config_base & 0x7)
	51	+
	52	+static const u32 ddrc_reg_off[] = {
	53	+ DDRC_FLUX_WR, DDRC_FLUX_RD, DDRC_FLUX_WCMD, DDRC_FLUX_RCMD,
	54	+ DDRC_PRE_CMD, DDRC_ACT_CMD, DDRC_BNK_CHG, DDRC_RNK_CHG
	55	+};
	56	+
	57	+/*
	58	+ * Select the counter register offset using the counter index.
	59	+ * In DDRC there are no programmable counter, the count
	60	+ * is readed form the statistics counter register itself.
	61	+ */
	62	+static u32 hisi_ddrc_pmu_get_counter_offset(int cntr_idx)
	63	+{
	64	+ return ddrc_reg_off[cntr_idx];
	65	+}
	66	+
	67	+static u64 hisi_ddrc_pmu_read_counter(struct hisi_pmu *ddrc_pmu,
	68	+ struct hw_perf_event *hwc)
	69	+{
	70	+ /* Use event code as counter index */
	71	+ u32 idx = GET_DDRC_EVENTID(hwc);
	72	+
	73	+ if (!hisi_uncore_pmu_counter_valid(ddrc_pmu, idx)) {
	74	+ dev_err(ddrc_pmu->dev, "Unsupported event index:%d!\n", idx);
	75	+ return 0;
	76	+ }
	77	+
	78	+ return readl(ddrc_pmu->base + hisi_ddrc_pmu_get_counter_offset(idx));
	79	+}
	80	+
	81	+static void hisi_ddrc_pmu_write_counter(struct hisi_pmu *ddrc_pmu,
	82	+ struct hw_perf_event *hwc, u64 val)
	83	+{
	84	+ u32 idx = GET_DDRC_EVENTID(hwc);
	85	+
	86	+ if (!hisi_uncore_pmu_counter_valid(ddrc_pmu, idx)) {
	87	+ dev_err(ddrc_pmu->dev, "Unsupported event index:%d!\n", idx);
	88	+ return;
	89	+ }
	90	+
	91	+ writel((u32)val,
	92	+ ddrc_pmu->base + hisi_ddrc_pmu_get_counter_offset(idx));
	93	+}
	94	+
	95	+/*
	96	+ * For DDRC PMU, event has been mapped to fixed-purpose counter by hardware,
	97	+ * so there is no need to write event type.
	98	+ */
	99	+static void hisi_ddrc_pmu_write_evtype(struct hisi_pmu *hha_pmu, int idx,
	100	+ u32 type)
	101	+{
	102	+}
	103	+
	104	+static void hisi_ddrc_pmu_start_counters(struct hisi_pmu *ddrc_pmu)
	105	+{
	106	+ u32 val;
	107	+
	108	+ /* Set perf_enable in DDRC_PERF_CTRL to start event counting */
	109	+ val = readl(ddrc_pmu->base + DDRC_PERF_CTRL);
	110	+ val \|= DDRC_PERF_CTRL_EN;
	111	+ writel(val, ddrc_pmu->base + DDRC_PERF_CTRL);
	112	+}
	113	+
	114	+static void hisi_ddrc_pmu_stop_counters(struct hisi_pmu *ddrc_pmu)
	115	+{
	116	+ u32 val;
	117	+
	118	+ /* Clear perf_enable in DDRC_PERF_CTRL to stop event counting */
	119	+ val = readl(ddrc_pmu->base + DDRC_PERF_CTRL);
	120	+ val &= ~DDRC_PERF_CTRL_EN;
	121	+ writel(val, ddrc_pmu->base + DDRC_PERF_CTRL);
	122	+}
	123	+
	124	+static void hisi_ddrc_pmu_enable_counter(struct hisi_pmu *ddrc_pmu,
	125	+ struct hw_perf_event *hwc)
	126	+{
	127	+ u32 val;
	128	+
	129	+ /* Set counter index(event code) in DDRC_EVENT_CTRL register */
	130	+ val = readl(ddrc_pmu->base + DDRC_EVENT_CTRL);
	131	+ val \|= (1 << GET_DDRC_EVENTID(hwc));
	132	+ writel(val, ddrc_pmu->base + DDRC_EVENT_CTRL);
	133	+}
	134	+
	135	+static void hisi_ddrc_pmu_disable_counter(struct hisi_pmu *ddrc_pmu,
	136	+ struct hw_perf_event *hwc)
	137	+{
	138	+ u32 val;
	139	+
	140	+ /* Clear counter index(event code) in DDRC_EVENT_CTRL register */
	141	+ val = readl(ddrc_pmu->base + DDRC_EVENT_CTRL);
	142	+ val &= ~(1 << GET_DDRC_EVENTID(hwc));
	143	+ writel(val, ddrc_pmu->base + DDRC_EVENT_CTRL);
	144	+}
	145	+
	146	+static int hisi_ddrc_pmu_get_event_idx(struct perf_event *event)
	147	+{
	148	+ struct hisi_pmu *ddrc_pmu = to_hisi_pmu(event->pmu);
	149	+ unsigned long *used_mask = ddrc_pmu->pmu_events.used_mask;
	150	+ struct hw_perf_event *hwc = &event->hw;
	151	+ /* For DDRC PMU, we use event code as counter index */
	152	+ int idx = GET_DDRC_EVENTID(hwc);
	153	+
	154	+ if (test_bit(idx, used_mask))
	155	+ return -EAGAIN;
	156	+
	157	+ set_bit(idx, used_mask);
	158	+
	159	+ return idx;
	160	+}
	161	+
	162	+static void hisi_ddrc_pmu_enable_counter_int(struct hisi_pmu *ddrc_pmu,
	163	+ struct hw_perf_event *hwc)
	164	+{
	165	+ u32 val;
	166	+
	167	+ /* Write 0 to enable interrupt */
	168	+ val = readl(ddrc_pmu->base + DDRC_INT_MASK);
	169	+ val &= ~(1 << GET_DDRC_EVENTID(hwc));
	170	+ writel(val, ddrc_pmu->base + DDRC_INT_MASK);
	171	+}
	172	+
	173	+static void hisi_ddrc_pmu_disable_counter_int(struct hisi_pmu *ddrc_pmu,
	174	+ struct hw_perf_event *hwc)
	175	+{
	176	+ u32 val;
	177	+
	178	+ /* Write 1 to mask interrupt */
	179	+ val = readl(ddrc_pmu->base + DDRC_INT_MASK);
	180	+ val \|= (1 << GET_DDRC_EVENTID(hwc));
	181	+ writel(val, ddrc_pmu->base + DDRC_INT_MASK);
	182	+}
	183	+
	184	+static irqreturn_t hisi_ddrc_pmu_isr(int irq, void *dev_id)
	185	+{
	186	+ struct hisi_pmu *ddrc_pmu = dev_id;
	187	+ struct perf_event *event;
	188	+ unsigned long overflown;
	189	+ int idx;
	190	+
	191	+ /* Read the DDRC_INT_STATUS register */
	192	+ overflown = readl(ddrc_pmu->base + DDRC_INT_STATUS);
	193	+ if (!overflown)
	194	+ return IRQ_NONE;
	195	+
	196	+ /*
	197	+ * Find the counter index which overflowed if the bit was set
	198	+ * and handle it
	199	+ */
	200	+ for_each_set_bit(idx, &overflown, DDRC_NR_COUNTERS) {
	201	+ /* Write 1 to clear the IRQ status flag */
	202	+ writel((1 << idx), ddrc_pmu->base + DDRC_INT_CLEAR);
	203	+
	204	+ /* Get the corresponding event struct */
	205	+ event = ddrc_pmu->pmu_events.hw_events[idx];
	206	+ if (!event)
	207	+ continue;
	208	+
	209	+ hisi_uncore_pmu_event_update(event);
	210	+ hisi_uncore_pmu_set_event_period(event);
	211	+ }
	212	+
	213	+ return IRQ_HANDLED;
	214	+}
	215	+
	216	+static int hisi_ddrc_pmu_init_irq(struct hisi_pmu *ddrc_pmu,
	217	+ struct platform_device *pdev)
	218	+{
	219	+ int irq, ret;
	220	+
	221	+ /* Read and init IRQ */
	222	+ irq = platform_get_irq(pdev, 0);
	223	+ if (irq < 0) {
	224	+ dev_err(&pdev->dev, "DDRC PMU get irq fail; irq:%d\n", irq);
	225	+ return irq;
	226	+ }
	227	+
	228	+ ret = devm_request_irq(&pdev->dev, irq, hisi_ddrc_pmu_isr,
	229	+ IRQF_NOBALANCING \| IRQF_NO_THREAD,
	230	+ dev_name(&pdev->dev), ddrc_pmu);
	231	+ if (ret < 0) {
	232	+ dev_err(&pdev->dev,
	233	+ "Fail to request IRQ:%d ret:%d\n", irq, ret);
	234	+ return ret;
	235	+ }
	236	+
	237	+ ddrc_pmu->irq = irq;
	238	+
	239	+ return 0;
	240	+}
	241	+
	242	+static const struct acpi_device_id hisi_ddrc_pmu_acpi_match[] = {
	243	+ { "HISI0233", },
	244	+ {},
	245	+};
	246	+MODULE_DEVICE_TABLE(acpi, hisi_ddrc_pmu_acpi_match);
	247	+
	248	+static int hisi_ddrc_pmu_init_data(struct platform_device *pdev,
	249	+ struct hisi_pmu *ddrc_pmu)
	250	+{
	251	+ struct resource *res;
	252	+
	253	+ /*
	254	+ * Use the SCCL_ID and DDRC channel ID to identify the
	255	+ * DDRC PMU, while SCCL_ID is in MPIDR[aff2].
	256	+ */
	257	+ if (device_property_read_u32(&pdev->dev, "hisilicon,ch-id",
	258	+ &ddrc_pmu->index_id)) {
	259	+ dev_err(&pdev->dev, "Can not read ddrc channel-id!\n");
	260	+ return -EINVAL;
	261	+ }
	262	+
	263	+ if (device_property_read_u32(&pdev->dev, "hisilicon,scl-id",
	264	+ &ddrc_pmu->sccl_id)) {
	265	+ dev_err(&pdev->dev, "Can not read ddrc sccl-id!\n");
	266	+ return -EINVAL;
	267	+ }
	268	+ /* DDRC PMUs only share the same SCCL */
	269	+ ddrc_pmu->ccl_id = -1;
	270	+
	271	+ res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
	272	+ ddrc_pmu->base = devm_ioremap_resource(&pdev->dev, res);
	273	+ if (IS_ERR(ddrc_pmu->base)) {
	274	+ dev_err(&pdev->dev, "ioremap failed for ddrc_pmu resource\n");
	275	+ return PTR_ERR(ddrc_pmu->base);
	276	+ }
	277	+
	278	+ return 0;
	279	+}
	280	+
	281	+static struct attribute *hisi_ddrc_pmu_format_attr[] = {
	282	+ HISI_PMU_FORMAT_ATTR(event, "config:0-4"),
	283	+ NULL,
	284	+};
	285	+
	286	+static const struct attribute_group hisi_ddrc_pmu_format_group = {
	287	+ .name = "format",
	288	+ .attrs = hisi_ddrc_pmu_format_attr,
	289	+};
	290	+
	291	+static struct attribute *hisi_ddrc_pmu_events_attr[] = {
	292	+ HISI_PMU_EVENT_ATTR(flux_wr, 0x00),
	293	+ HISI_PMU_EVENT_ATTR(flux_rd, 0x01),
	294	+ HISI_PMU_EVENT_ATTR(flux_wcmd, 0x02),
	295	+ HISI_PMU_EVENT_ATTR(flux_rcmd, 0x03),
	296	+ HISI_PMU_EVENT_ATTR(pre_cmd, 0x04),
	297	+ HISI_PMU_EVENT_ATTR(act_cmd, 0x05),
	298	+ HISI_PMU_EVENT_ATTR(rnk_chg, 0x06),
	299	+ HISI_PMU_EVENT_ATTR(rw_chg, 0x07),
	300	+ NULL,
	301	+};
	302	+
	303	+static const struct attribute_group hisi_ddrc_pmu_events_group = {
	304	+ .name = "events",
	305	+ .attrs = hisi_ddrc_pmu_events_attr,
	306	+};
	307	+
	308	+static DEVICE_ATTR(cpumask, 0444, hisi_cpumask_sysfs_show, NULL);
	309	+
	310	+static struct attribute *hisi_ddrc_pmu_cpumask_attrs[] = {
	311	+ &dev_attr_cpumask.attr,
	312	+ NULL,
	313	+};
	314	+
	315	+static const struct attribute_group hisi_ddrc_pmu_cpumask_attr_group = {
	316	+ .attrs = hisi_ddrc_pmu_cpumask_attrs,
	317	+};
	318	+
	319	+static const struct attribute_group *hisi_ddrc_pmu_attr_groups[] = {
	320	+ &hisi_ddrc_pmu_format_group,
	321	+ &hisi_ddrc_pmu_events_group,
	322	+ &hisi_ddrc_pmu_cpumask_attr_group,
	323	+ NULL,
	324	+};
	325	+
	326	+static const struct hisi_uncore_ops hisi_uncore_ddrc_ops = {
	327	+ .write_evtype = hisi_ddrc_pmu_write_evtype,
	328	+ .get_event_idx = hisi_ddrc_pmu_get_event_idx,
	329	+ .start_counters = hisi_ddrc_pmu_start_counters,
	330	+ .stop_counters = hisi_ddrc_pmu_stop_counters,
	331	+ .enable_counter = hisi_ddrc_pmu_enable_counter,
	332	+ .disable_counter = hisi_ddrc_pmu_disable_counter,
	333	+ .enable_counter_int = hisi_ddrc_pmu_enable_counter_int,
	334	+ .disable_counter_int = hisi_ddrc_pmu_disable_counter_int,
	335	+ .write_counter = hisi_ddrc_pmu_write_counter,
	336	+ .read_counter = hisi_ddrc_pmu_read_counter,
	337	+};
	338	+
	339	+static int hisi_ddrc_pmu_dev_probe(struct platform_device *pdev,
	340	+ struct hisi_pmu *ddrc_pmu)
	341	+{
	342	+ int ret;
	343	+
	344	+ ret = hisi_ddrc_pmu_init_data(pdev, ddrc_pmu);
	345	+ if (ret)
	346	+ return ret;
	347	+
	348	+ ret = hisi_ddrc_pmu_init_irq(ddrc_pmu, pdev);
	349	+ if (ret)
	350	+ return ret;
	351	+
	352	+ ddrc_pmu->num_counters = DDRC_NR_COUNTERS;
	353	+ ddrc_pmu->counter_bits = 32;
	354	+ ddrc_pmu->ops = &hisi_uncore_ddrc_ops;
	355	+ ddrc_pmu->dev = &pdev->dev;
	356	+ ddrc_pmu->on_cpu = -1;
	357	+ ddrc_pmu->check_event = 7;
	358	+
	359	+ return 0;
	360	+}
	361	+
	362	+static int hisi_ddrc_pmu_probe(struct platform_device *pdev)
	363	+{
	364	+ struct hisi_pmu *ddrc_pmu;
	365	+ char *name;
	366	+ int ret;
	367	+
	368	+ ddrc_pmu = devm_kzalloc(&pdev->dev, sizeof(*ddrc_pmu), GFP_KERNEL);
	369	+ if (!ddrc_pmu)
	370	+ return -ENOMEM;
	371	+
	372	+ platform_set_drvdata(pdev, ddrc_pmu);
	373	+
	374	+ ret = hisi_ddrc_pmu_dev_probe(pdev, ddrc_pmu);
	375	+ if (ret)
	376	+ return ret;
	377	+
	378	+ ret = cpuhp_state_add_instance(CPUHP_AP_PERF_ARM_HISI_DDRC_ONLINE,
	379	+ &ddrc_pmu->node);
	380	+ if (ret) {
	381	+ dev_err(&pdev->dev, "Error %d registering hotplug;\n", ret);
	382	+ return ret;
	383	+ }
	384	+
	385	+ name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "hisi_sccl%u_ddrc%u",
	386	+ ddrc_pmu->sccl_id, ddrc_pmu->index_id);
	387	+ ddrc_pmu->pmu = (struct pmu) {
	388	+ .name = name,
	389	+ .task_ctx_nr = perf_invalid_context,
	390	+ .event_init = hisi_uncore_pmu_event_init,
	391	+ .pmu_enable = hisi_uncore_pmu_enable,
	392	+ .pmu_disable = hisi_uncore_pmu_disable,
	393	+ .add = hisi_uncore_pmu_add,
	394	+ .del = hisi_uncore_pmu_del,
	395	+ .start = hisi_uncore_pmu_start,
	396	+ .stop = hisi_uncore_pmu_stop,
	397	+ .read = hisi_uncore_pmu_read,
	398	+ .attr_groups = hisi_ddrc_pmu_attr_groups,
	399	+ };
	400	+
	401	+ ret = perf_pmu_register(&ddrc_pmu->pmu, name, -1);
	402	+ if (ret) {
	403	+ dev_err(ddrc_pmu->dev, "DDRC PMU register failed!\n");
	404	+ cpuhp_state_remove_instance(CPUHP_AP_PERF_ARM_HISI_DDRC_ONLINE,
	405	+ &ddrc_pmu->node);
	406	+ }
	407	+
	408	+ return ret;
	409	+}
	410	+
	411	+static int hisi_ddrc_pmu_remove(struct platform_device *pdev)
	412	+{
	413	+ struct hisi_pmu *ddrc_pmu = platform_get_drvdata(pdev);
	414	+
	415	+ perf_pmu_unregister(&ddrc_pmu->pmu);
	416	+ cpuhp_state_remove_instance(CPUHP_AP_PERF_ARM_HISI_DDRC_ONLINE,
	417	+ &ddrc_pmu->node);
	418	+
	419	+ return 0;
	420	+}
	421	+
	422	+static struct platform_driver hisi_ddrc_pmu_driver = {
	423	+ .driver = {
	424	+ .name = "hisi_ddrc_pmu",
	425	+ .acpi_match_table = ACPI_PTR(hisi_ddrc_pmu_acpi_match),
	426	+ },
	427	+ .probe = hisi_ddrc_pmu_probe,
	428	+ .remove = hisi_ddrc_pmu_remove,
	429	+};
	430	+
	431	+static int __init hisi_ddrc_pmu_module_init(void)
	432	+{
	433	+ int ret;
	434	+
	435	+ ret = cpuhp_setup_state_multi(CPUHP_AP_PERF_ARM_HISI_DDRC_ONLINE,
	436	+ "AP_PERF_ARM_HISI_DDRC_ONLINE",
	437	+ hisi_uncore_pmu_online_cpu,
	438	+ hisi_uncore_pmu_offline_cpu);
	439	+ if (ret) {
	440	+ pr_err("DDRC PMU: setup hotplug, ret = %d\n", ret);
	441	+ return ret;
	442	+ }
	443	+
	444	+ ret = platform_driver_register(&hisi_ddrc_pmu_driver);
	445	+ if (ret)
	446	+ cpuhp_remove_multi_state(CPUHP_AP_PERF_ARM_HISI_DDRC_ONLINE);
	447	+
	448	+ return ret;
	449	+}
	450	+module_init(hisi_ddrc_pmu_module_init);
	451	+
	452	+static void __exit hisi_ddrc_pmu_module_exit(void)
	453	+{
	454	+ platform_driver_unregister(&hisi_ddrc_pmu_driver);
	455	+ cpuhp_remove_multi_state(CPUHP_AP_PERF_ARM_HISI_DDRC_ONLINE);
	456	+
	457	+}
	458	+module_exit(hisi_ddrc_pmu_module_exit);
	459	+
	460	+MODULE_DESCRIPTION("HiSilicon SoC DDRC uncore PMU driver");
	461	+MODULE_LICENSE("GPL v2");
	462	+MODULE_AUTHOR("Shaokun Zhang <zhangshaokun@hisilicon.com>");
	463	+MODULE_AUTHOR("Anurup M <anurup.m@huawei.com>");

drivers/perf/hisilicon/hisi_uncore_hha_pmu.c+473−0

@@ -0,0 +1,473 @@
	1	+/*
	2	+ * HiSilicon SoC HHA uncore Hardware event counters support
	3	+ *
	4	+ * Copyright (C) 2017 Hisilicon Limited
	5	+ * Author: Shaokun Zhang <zhangshaokun@hisilicon.com>
	6	+ * Anurup M <anurup.m@huawei.com>
	7	+ *
	8	+ * This code is based on the uncore PMUs like arm-cci and arm-ccn.
	9	+ *
	10	+ * This program is free software; you can redistribute it and/or modify
	11	+ * it under the terms of the GNU General Public License version 2 as
	12	+ * published by the Free Software Foundation.
	13	+ */
	14	+#include <linux/acpi.h>
	15	+#include <linux/bug.h>
	16	+#include <linux/cpuhotplug.h>
	17	+#include <linux/interrupt.h>
	18	+#include <linux/irq.h>
	19	+#include <linux/list.h>
	20	+#include <linux/platform_device.h>
	21	+#include <linux/smp.h>
	22	+
	23	+#include "hisi_uncore_pmu.h"
	24	+
	25	+/* HHA register definition */
	26	+#define HHA_INT_MASK 0x0804
	27	+#define HHA_INT_STATUS 0x0808
	28	+#define HHA_INT_CLEAR 0x080C
	29	+#define HHA_PERF_CTRL 0x1E00
	30	+#define HHA_EVENT_CTRL 0x1E04
	31	+#define HHA_EVENT_TYPE0 0x1E80
	32	+/*
	33	+ * Each counter is 48-bits and [48:63] are reserved
	34	+ * which are Read-As-Zero and Writes-Ignored.
	35	+ */
	36	+#define HHA_CNT0_LOWER 0x1F00
	37	+
	38	+/* HHA has 16-counters */
	39	+#define HHA_NR_COUNTERS 0x10
	40	+
	41	+#define HHA_PERF_CTRL_EN 0x1
	42	+#define HHA_EVTYPE_NONE 0xff
	43	+
	44	+/*
	45	+ * Select the counter register offset using the counter index
	46	+ * each counter is 48-bits.
	47	+ */
	48	+static u32 hisi_hha_pmu_get_counter_offset(int cntr_idx)
	49	+{
	50	+ return (HHA_CNT0_LOWER + (cntr_idx * 8));
	51	+}
	52	+
	53	+static u64 hisi_hha_pmu_read_counter(struct hisi_pmu *hha_pmu,
	54	+ struct hw_perf_event *hwc)
	55	+{
	56	+ u32 idx = hwc->idx;
	57	+
	58	+ if (!hisi_uncore_pmu_counter_valid(hha_pmu, idx)) {
	59	+ dev_err(hha_pmu->dev, "Unsupported event index:%d!\n", idx);
	60	+ return 0;
	61	+ }
	62	+
	63	+ /* Read 64 bits and like L3C, top 16 bits are RAZ */
	64	+ return readq(hha_pmu->base + hisi_hha_pmu_get_counter_offset(idx));
	65	+}
	66	+
	67	+static void hisi_hha_pmu_write_counter(struct hisi_pmu *hha_pmu,
	68	+ struct hw_perf_event *hwc, u64 val)
	69	+{
	70	+ u32 idx = hwc->idx;
	71	+
	72	+ if (!hisi_uncore_pmu_counter_valid(hha_pmu, idx)) {
	73	+ dev_err(hha_pmu->dev, "Unsupported event index:%d!\n", idx);
	74	+ return;
	75	+ }
	76	+
	77	+ /* Write 64 bits and like L3C, top 16 bits are WI */
	78	+ writeq(val, hha_pmu->base + hisi_hha_pmu_get_counter_offset(idx));
	79	+}
	80	+
	81	+static void hisi_hha_pmu_write_evtype(struct hisi_pmu *hha_pmu, int idx,
	82	+ u32 type)
	83	+{
	84	+ u32 reg, reg_idx, shift, val;
	85	+
	86	+ /*
	87	+ * Select the appropriate event select register(HHA_EVENT_TYPEx).
	88	+ * There are 4 event select registers for the 16 hardware counters.
	89	+ * Event code is 8-bits and for the first 4 hardware counters,
	90	+ * HHA_EVENT_TYPE0 is chosen. For the next 4 hardware counters,
	91	+ * HHA_EVENT_TYPE1 is chosen and so on.
	92	+ */
	93	+ reg = HHA_EVENT_TYPE0 + 4 * (idx / 4);
	94	+ reg_idx = idx % 4;
	95	+ shift = 8 * reg_idx;
	96	+
	97	+ /* Write event code to HHA_EVENT_TYPEx register */
	98	+ val = readl(hha_pmu->base + reg);
	99	+ val &= ~(HHA_EVTYPE_NONE << shift);
	100	+ val \|= (type << shift);
	101	+ writel(val, hha_pmu->base + reg);
	102	+}
	103	+
	104	+static void hisi_hha_pmu_start_counters(struct hisi_pmu *hha_pmu)
	105	+{
	106	+ u32 val;
	107	+
	108	+ /*
	109	+ * Set perf_enable bit in HHA_PERF_CTRL to start event
	110	+ * counting for all enabled counters.
	111	+ */
	112	+ val = readl(hha_pmu->base + HHA_PERF_CTRL);
	113	+ val \|= HHA_PERF_CTRL_EN;
	114	+ writel(val, hha_pmu->base + HHA_PERF_CTRL);
	115	+}
	116	+
	117	+static void hisi_hha_pmu_stop_counters(struct hisi_pmu *hha_pmu)
	118	+{
	119	+ u32 val;
	120	+
	121	+ /*
	122	+ * Clear perf_enable bit in HHA_PERF_CTRL to stop event
	123	+ * counting for all enabled counters.
	124	+ */
	125	+ val = readl(hha_pmu->base + HHA_PERF_CTRL);
	126	+ val &= ~(HHA_PERF_CTRL_EN);
	127	+ writel(val, hha_pmu->base + HHA_PERF_CTRL);
	128	+}
	129	+
	130	+static void hisi_hha_pmu_enable_counter(struct hisi_pmu *hha_pmu,
	131	+ struct hw_perf_event *hwc)
	132	+{
	133	+ u32 val;
	134	+
	135	+ /* Enable counter index in HHA_EVENT_CTRL register */
	136	+ val = readl(hha_pmu->base + HHA_EVENT_CTRL);
	137	+ val \|= (1 << hwc->idx);
	138	+ writel(val, hha_pmu->base + HHA_EVENT_CTRL);
	139	+}
	140	+
	141	+static void hisi_hha_pmu_disable_counter(struct hisi_pmu *hha_pmu,
	142	+ struct hw_perf_event *hwc)
	143	+{
	144	+ u32 val;
	145	+
	146	+ /* Clear counter index in HHA_EVENT_CTRL register */
	147	+ val = readl(hha_pmu->base + HHA_EVENT_CTRL);
	148	+ val &= ~(1 << hwc->idx);
	149	+ writel(val, hha_pmu->base + HHA_EVENT_CTRL);
	150	+}
	151	+
	152	+static void hisi_hha_pmu_enable_counter_int(struct hisi_pmu *hha_pmu,
	153	+ struct hw_perf_event *hwc)
	154	+{
	155	+ u32 val;
	156	+
	157	+ /* Write 0 to enable interrupt */
	158	+ val = readl(hha_pmu->base + HHA_INT_MASK);
	159	+ val &= ~(1 << hwc->idx);
	160	+ writel(val, hha_pmu->base + HHA_INT_MASK);
	161	+}
	162	+
	163	+static void hisi_hha_pmu_disable_counter_int(struct hisi_pmu *hha_pmu,
	164	+ struct hw_perf_event *hwc)
	165	+{
	166	+ u32 val;
	167	+
	168	+ /* Write 1 to mask interrupt */
	169	+ val = readl(hha_pmu->base + HHA_INT_MASK);
	170	+ val \|= (1 << hwc->idx);
	171	+ writel(val, hha_pmu->base + HHA_INT_MASK);
	172	+}
	173	+
	174	+static irqreturn_t hisi_hha_pmu_isr(int irq, void *dev_id)
	175	+{
	176	+ struct hisi_pmu *hha_pmu = dev_id;
	177	+ struct perf_event *event;
	178	+ unsigned long overflown;
	179	+ int idx;
	180	+
	181	+ /* Read HHA_INT_STATUS register */
	182	+ overflown = readl(hha_pmu->base + HHA_INT_STATUS);
	183	+ if (!overflown)
	184	+ return IRQ_NONE;
	185	+
	186	+ /*
	187	+ * Find the counter index which overflowed if the bit was set
	188	+ * and handle it
	189	+ */
	190	+ for_each_set_bit(idx, &overflown, HHA_NR_COUNTERS) {
	191	+ /* Write 1 to clear the IRQ status flag */
	192	+ writel((1 << idx), hha_pmu->base + HHA_INT_CLEAR);
	193	+
	194	+ /* Get the corresponding event struct */
	195	+ event = hha_pmu->pmu_events.hw_events[idx];
	196	+ if (!event)
	197	+ continue;
	198	+
	199	+ hisi_uncore_pmu_event_update(event);
	200	+ hisi_uncore_pmu_set_event_period(event);
	201	+ }
	202	+
	203	+ return IRQ_HANDLED;
	204	+}
	205	+
	206	+static int hisi_hha_pmu_init_irq(struct hisi_pmu *hha_pmu,
	207	+ struct platform_device *pdev)
	208	+{
	209	+ int irq, ret;
	210	+
	211	+ /* Read and init IRQ */
	212	+ irq = platform_get_irq(pdev, 0);
	213	+ if (irq < 0) {
	214	+ dev_err(&pdev->dev, "HHA PMU get irq fail; irq:%d\n", irq);
	215	+ return irq;
	216	+ }
	217	+
	218	+ ret = devm_request_irq(&pdev->dev, irq, hisi_hha_pmu_isr,
	219	+ IRQF_NOBALANCING \| IRQF_NO_THREAD,
	220	+ dev_name(&pdev->dev), hha_pmu);
	221	+ if (ret < 0) {
	222	+ dev_err(&pdev->dev,
	223	+ "Fail to request IRQ:%d ret:%d\n", irq, ret);
	224	+ return ret;
	225	+ }
	226	+
	227	+ hha_pmu->irq = irq;
	228	+
	229	+ return 0;
	230	+}
	231	+
	232	+static const struct acpi_device_id hisi_hha_pmu_acpi_match[] = {
	233	+ { "HISI0243", },
	234	+ {},
	235	+};
	236	+MODULE_DEVICE_TABLE(acpi, hisi_hha_pmu_acpi_match);
	237	+
	238	+static int hisi_hha_pmu_init_data(struct platform_device *pdev,
	239	+ struct hisi_pmu *hha_pmu)
	240	+{
	241	+ unsigned long long id;
	242	+ struct resource *res;
	243	+ acpi_status status;
	244	+
	245	+ status = acpi_evaluate_integer(ACPI_HANDLE(&pdev->dev),
	246	+ "_UID", NULL, &id);
	247	+ if (ACPI_FAILURE(status))
	248	+ return -EINVAL;
	249	+
	250	+ hha_pmu->index_id = id;
	251	+
	252	+ /*
	253	+ * Use SCCL_ID and UID to identify the HHA PMU, while
	254	+ * SCCL_ID is in MPIDR[aff2].
	255	+ */
	256	+ if (device_property_read_u32(&pdev->dev, "hisilicon,scl-id",
	257	+ &hha_pmu->sccl_id)) {
	258	+ dev_err(&pdev->dev, "Can not read hha sccl-id!\n");
	259	+ return -EINVAL;
	260	+ }
	261	+ /* HHA PMUs only share the same SCCL */
	262	+ hha_pmu->ccl_id = -1;
	263	+
	264	+ res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
	265	+ hha_pmu->base = devm_ioremap_resource(&pdev->dev, res);
	266	+ if (IS_ERR(hha_pmu->base)) {
	267	+ dev_err(&pdev->dev, "ioremap failed for hha_pmu resource\n");
	268	+ return PTR_ERR(hha_pmu->base);
	269	+ }
	270	+
	271	+ return 0;
	272	+}
	273	+
	274	+static struct attribute *hisi_hha_pmu_format_attr[] = {
	275	+ HISI_PMU_FORMAT_ATTR(event, "config:0-7"),
	276	+ NULL,
	277	+};
	278	+
	279	+static const struct attribute_group hisi_hha_pmu_format_group = {
	280	+ .name = "format",
	281	+ .attrs = hisi_hha_pmu_format_attr,
	282	+};
	283	+
	284	+static struct attribute *hisi_hha_pmu_events_attr[] = {
	285	+ HISI_PMU_EVENT_ATTR(rx_ops_num, 0x00),
	286	+ HISI_PMU_EVENT_ATTR(rx_outer, 0x01),
	287	+ HISI_PMU_EVENT_ATTR(rx_sccl, 0x02),
	288	+ HISI_PMU_EVENT_ATTR(rx_ccix, 0x03),
	289	+ HISI_PMU_EVENT_ATTR(rx_wbi, 0x04),
	290	+ HISI_PMU_EVENT_ATTR(rx_wbip, 0x05),
	291	+ HISI_PMU_EVENT_ATTR(rx_wtistash, 0x11),
	292	+ HISI_PMU_EVENT_ATTR(rd_ddr_64b, 0x1c),
	293	+ HISI_PMU_EVENT_ATTR(wr_dr_64b, 0x1d),
	294	+ HISI_PMU_EVENT_ATTR(rd_ddr_128b, 0x1e),
	295	+ HISI_PMU_EVENT_ATTR(wr_ddr_128b, 0x1f),
	296	+ HISI_PMU_EVENT_ATTR(spill_num, 0x20),
	297	+ HISI_PMU_EVENT_ATTR(spill_success, 0x21),
	298	+ HISI_PMU_EVENT_ATTR(bi_num, 0x23),
	299	+ HISI_PMU_EVENT_ATTR(mediated_num, 0x32),
	300	+ HISI_PMU_EVENT_ATTR(tx_snp_num, 0x33),
	301	+ HISI_PMU_EVENT_ATTR(tx_snp_outer, 0x34),
	302	+ HISI_PMU_EVENT_ATTR(tx_snp_ccix, 0x35),
	303	+ HISI_PMU_EVENT_ATTR(rx_snprspdata, 0x38),
	304	+ HISI_PMU_EVENT_ATTR(rx_snprsp_outer, 0x3c),
	305	+ HISI_PMU_EVENT_ATTR(sdir-lookup, 0x40),
	306	+ HISI_PMU_EVENT_ATTR(edir-lookup, 0x41),
	307	+ HISI_PMU_EVENT_ATTR(sdir-hit, 0x42),
	308	+ HISI_PMU_EVENT_ATTR(edir-hit, 0x43),
	309	+ HISI_PMU_EVENT_ATTR(sdir-home-migrate, 0x4c),
	310	+ HISI_PMU_EVENT_ATTR(edir-home-migrate, 0x4d),
	311	+ NULL,
	312	+};
	313	+
	314	+static const struct attribute_group hisi_hha_pmu_events_group = {
	315	+ .name = "events",
	316	+ .attrs = hisi_hha_pmu_events_attr,
	317	+};
	318	+
	319	+static DEVICE_ATTR(cpumask, 0444, hisi_cpumask_sysfs_show, NULL);
	320	+
	321	+static struct attribute *hisi_hha_pmu_cpumask_attrs[] = {
	322	+ &dev_attr_cpumask.attr,
	323	+ NULL,
	324	+};
	325	+
	326	+static const struct attribute_group hisi_hha_pmu_cpumask_attr_group = {
	327	+ .attrs = hisi_hha_pmu_cpumask_attrs,
	328	+};
	329	+
	330	+static const struct attribute_group *hisi_hha_pmu_attr_groups[] = {
	331	+ &hisi_hha_pmu_format_group,
	332	+ &hisi_hha_pmu_events_group,
	333	+ &hisi_hha_pmu_cpumask_attr_group,
	334	+ NULL,
	335	+};
	336	+
	337	+static const struct hisi_uncore_ops hisi_uncore_hha_ops = {
	338	+ .write_evtype = hisi_hha_pmu_write_evtype,
	339	+ .get_event_idx = hisi_uncore_pmu_get_event_idx,
	340	+ .start_counters = hisi_hha_pmu_start_counters,
	341	+ .stop_counters = hisi_hha_pmu_stop_counters,
	342	+ .enable_counter = hisi_hha_pmu_enable_counter,
	343	+ .disable_counter = hisi_hha_pmu_disable_counter,
	344	+ .enable_counter_int = hisi_hha_pmu_enable_counter_int,
	345	+ .disable_counter_int = hisi_hha_pmu_disable_counter_int,
	346	+ .write_counter = hisi_hha_pmu_write_counter,
	347	+ .read_counter = hisi_hha_pmu_read_counter,
	348	+};
	349	+
	350	+static int hisi_hha_pmu_dev_probe(struct platform_device *pdev,
	351	+ struct hisi_pmu *hha_pmu)
	352	+{
	353	+ int ret;
	354	+
	355	+ ret = hisi_hha_pmu_init_data(pdev, hha_pmu);
	356	+ if (ret)
	357	+ return ret;
	358	+
	359	+ ret = hisi_hha_pmu_init_irq(hha_pmu, pdev);
	360	+ if (ret)
	361	+ return ret;
	362	+
	363	+ hha_pmu->num_counters = HHA_NR_COUNTERS;
	364	+ hha_pmu->counter_bits = 48;
	365	+ hha_pmu->ops = &hisi_uncore_hha_ops;
	366	+ hha_pmu->dev = &pdev->dev;
	367	+ hha_pmu->on_cpu = -1;
	368	+ hha_pmu->check_event = 0x65;
	369	+
	370	+ return 0;
	371	+}
	372	+
	373	+static int hisi_hha_pmu_probe(struct platform_device *pdev)
	374	+{
	375	+ struct hisi_pmu *hha_pmu;
	376	+ char *name;
	377	+ int ret;
	378	+
	379	+ hha_pmu = devm_kzalloc(&pdev->dev, sizeof(*hha_pmu), GFP_KERNEL);
	380	+ if (!hha_pmu)
	381	+ return -ENOMEM;
	382	+
	383	+ platform_set_drvdata(pdev, hha_pmu);
	384	+
	385	+ ret = hisi_hha_pmu_dev_probe(pdev, hha_pmu);
	386	+ if (ret)
	387	+ return ret;
	388	+
	389	+ ret = cpuhp_state_add_instance(CPUHP_AP_PERF_ARM_HISI_HHA_ONLINE,
	390	+ &hha_pmu->node);
	391	+ if (ret) {
	392	+ dev_err(&pdev->dev, "Error %d registering hotplug\n", ret);
	393	+ return ret;
	394	+ }
	395	+
	396	+ name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "hisi_sccl%u_hha%u",
	397	+ hha_pmu->sccl_id, hha_pmu->index_id);
	398	+ hha_pmu->pmu = (struct pmu) {
	399	+ .name = name,
	400	+ .task_ctx_nr = perf_invalid_context,
	401	+ .event_init = hisi_uncore_pmu_event_init,
	402	+ .pmu_enable = hisi_uncore_pmu_enable,
	403	+ .pmu_disable = hisi_uncore_pmu_disable,
	404	+ .add = hisi_uncore_pmu_add,
	405	+ .del = hisi_uncore_pmu_del,
	406	+ .start = hisi_uncore_pmu_start,
	407	+ .stop = hisi_uncore_pmu_stop,
	408	+ .read = hisi_uncore_pmu_read,
	409	+ .attr_groups = hisi_hha_pmu_attr_groups,
	410	+ };
	411	+
	412	+ ret = perf_pmu_register(&hha_pmu->pmu, name, -1);
	413	+ if (ret) {
	414	+ dev_err(hha_pmu->dev, "HHA PMU register failed!\n");
	415	+ cpuhp_state_remove_instance(CPUHP_AP_PERF_ARM_HISI_HHA_ONLINE,
	416	+ &hha_pmu->node);
	417	+ }
	418	+
	419	+ return ret;
	420	+}
	421	+
	422	+static int hisi_hha_pmu_remove(struct platform_device *pdev)
	423	+{
	424	+ struct hisi_pmu *hha_pmu = platform_get_drvdata(pdev);
	425	+
	426	+ perf_pmu_unregister(&hha_pmu->pmu);
	427	+ cpuhp_state_remove_instance(CPUHP_AP_PERF_ARM_HISI_HHA_ONLINE,
	428	+ &hha_pmu->node);
	429	+
	430	+ return 0;
	431	+}
	432	+
	433	+static struct platform_driver hisi_hha_pmu_driver = {
	434	+ .driver = {
	435	+ .name = "hisi_hha_pmu",
	436	+ .acpi_match_table = ACPI_PTR(hisi_hha_pmu_acpi_match),
	437	+ },
	438	+ .probe = hisi_hha_pmu_probe,
	439	+ .remove = hisi_hha_pmu_remove,
	440	+};
	441	+
	442	+static int __init hisi_hha_pmu_module_init(void)
	443	+{
	444	+ int ret;
	445	+
	446	+ ret = cpuhp_setup_state_multi(CPUHP_AP_PERF_ARM_HISI_HHA_ONLINE,
	447	+ "AP_PERF_ARM_HISI_HHA_ONLINE",
	448	+ hisi_uncore_pmu_online_cpu,
	449	+ hisi_uncore_pmu_offline_cpu);
	450	+ if (ret) {
	451	+ pr_err("HHA PMU: Error setup hotplug, ret = %d;\n", ret);
	452	+ return ret;
	453	+ }
	454	+
	455	+ ret = platform_driver_register(&hisi_hha_pmu_driver);
	456	+ if (ret)
	457	+ cpuhp_remove_multi_state(CPUHP_AP_PERF_ARM_HISI_HHA_ONLINE);
	458	+
	459	+ return ret;
	460	+}
	461	+module_init(hisi_hha_pmu_module_init);
	462	+
	463	+static void __exit hisi_hha_pmu_module_exit(void)
	464	+{
	465	+ platform_driver_unregister(&hisi_hha_pmu_driver);
	466	+ cpuhp_remove_multi_state(CPUHP_AP_PERF_ARM_HISI_HHA_ONLINE);
	467	+}
	468	+module_exit(hisi_hha_pmu_module_exit);
	469	+
	470	+MODULE_DESCRIPTION("HiSilicon SoC HHA uncore PMU driver");
	471	+MODULE_LICENSE("GPL v2");
	472	+MODULE_AUTHOR("Shaokun Zhang <zhangshaokun@hisilicon.com>");
	473	+MODULE_AUTHOR("Anurup M <anurup.m@huawei.com>");

drivers/perf/hisilicon/hisi_uncore_l3c_pmu.c+463−0

@@ -0,0 +1,463 @@
	1	+/*
	2	+ * HiSilicon SoC L3C uncore Hardware event counters support
	3	+ *
	4	+ * Copyright (C) 2017 Hisilicon Limited
	5	+ * Author: Anurup M <anurup.m@huawei.com>
	6	+ * Shaokun Zhang <zhangshaokun@hisilicon.com>
	7	+ *
	8	+ * This code is based on the uncore PMUs like arm-cci and arm-ccn.
	9	+ *
	10	+ * This program is free software; you can redistribute it and/or modify
	11	+ * it under the terms of the GNU General Public License version 2 as
	12	+ * published by the Free Software Foundation.
	13	+ */
	14	+#include <linux/acpi.h>
	15	+#include <linux/bug.h>
	16	+#include <linux/cpuhotplug.h>
	17	+#include <linux/interrupt.h>
	18	+#include <linux/irq.h>
	19	+#include <linux/list.h>
	20	+#include <linux/platform_device.h>
	21	+#include <linux/smp.h>
	22	+
	23	+#include "hisi_uncore_pmu.h"
	24	+
	25	+/* L3C register definition */
	26	+#define L3C_PERF_CTRL 0x0408
	27	+#define L3C_INT_MASK 0x0800
	28	+#define L3C_INT_STATUS 0x0808
	29	+#define L3C_INT_CLEAR 0x080c
	30	+#define L3C_EVENT_CTRL 0x1c00
	31	+#define L3C_EVENT_TYPE0 0x1d00
	32	+/*
	33	+ * Each counter is 48-bits and [48:63] are reserved
	34	+ * which are Read-As-Zero and Writes-Ignored.
	35	+ */
	36	+#define L3C_CNTR0_LOWER 0x1e00
	37	+
	38	+/* L3C has 8-counters */
	39	+#define L3C_NR_COUNTERS 0x8
	40	+
	41	+#define L3C_PERF_CTRL_EN 0x20000
	42	+#define L3C_EVTYPE_NONE 0xff
	43	+
	44	+/*
	45	+ * Select the counter register offset using the counter index
	46	+ */
	47	+static u32 hisi_l3c_pmu_get_counter_offset(int cntr_idx)
	48	+{
	49	+ return (L3C_CNTR0_LOWER + (cntr_idx * 8));
	50	+}
	51	+
	52	+static u64 hisi_l3c_pmu_read_counter(struct hisi_pmu *l3c_pmu,
	53	+ struct hw_perf_event *hwc)
	54	+{
	55	+ u32 idx = hwc->idx;
	56	+
	57	+ if (!hisi_uncore_pmu_counter_valid(l3c_pmu, idx)) {
	58	+ dev_err(l3c_pmu->dev, "Unsupported event index:%d!\n", idx);
	59	+ return 0;
	60	+ }
	61	+
	62	+ /* Read 64-bits and the upper 16 bits are RAZ */
	63	+ return readq(l3c_pmu->base + hisi_l3c_pmu_get_counter_offset(idx));
	64	+}
	65	+
	66	+static void hisi_l3c_pmu_write_counter(struct hisi_pmu *l3c_pmu,
	67	+ struct hw_perf_event *hwc, u64 val)
	68	+{
	69	+ u32 idx = hwc->idx;
	70	+
	71	+ if (!hisi_uncore_pmu_counter_valid(l3c_pmu, idx)) {
	72	+ dev_err(l3c_pmu->dev, "Unsupported event index:%d!\n", idx);
	73	+ return;
	74	+ }
	75	+
	76	+ /* Write 64-bits and the upper 16 bits are WI */
	77	+ writeq(val, l3c_pmu->base + hisi_l3c_pmu_get_counter_offset(idx));
	78	+}
	79	+
	80	+static void hisi_l3c_pmu_write_evtype(struct hisi_pmu *l3c_pmu, int idx,
	81	+ u32 type)
	82	+{
	83	+ u32 reg, reg_idx, shift, val;
	84	+
	85	+ /*
	86	+ * Select the appropriate event select register(L3C_EVENT_TYPE0/1).
	87	+ * There are 2 event select registers for the 8 hardware counters.
	88	+ * Event code is 8-bits and for the former 4 hardware counters,
	89	+ * L3C_EVENT_TYPE0 is chosen. For the latter 4 hardware counters,
	90	+ * L3C_EVENT_TYPE1 is chosen.
	91	+ */
	92	+ reg = L3C_EVENT_TYPE0 + (idx / 4) * 4;
	93	+ reg_idx = idx % 4;
	94	+ shift = 8 * reg_idx;
	95	+
	96	+ /* Write event code to L3C_EVENT_TYPEx Register */
	97	+ val = readl(l3c_pmu->base + reg);
	98	+ val &= ~(L3C_EVTYPE_NONE << shift);
	99	+ val \|= (type << shift);
	100	+ writel(val, l3c_pmu->base + reg);
	101	+}
	102	+
	103	+static void hisi_l3c_pmu_start_counters(struct hisi_pmu *l3c_pmu)
	104	+{
	105	+ u32 val;
	106	+
	107	+ /*
	108	+ * Set perf_enable bit in L3C_PERF_CTRL register to start counting
	109	+ * for all enabled counters.
	110	+ */
	111	+ val = readl(l3c_pmu->base + L3C_PERF_CTRL);
	112	+ val \|= L3C_PERF_CTRL_EN;
	113	+ writel(val, l3c_pmu->base + L3C_PERF_CTRL);
	114	+}
	115	+
	116	+static void hisi_l3c_pmu_stop_counters(struct hisi_pmu *l3c_pmu)
	117	+{
	118	+ u32 val;
	119	+
	120	+ /*
	121	+ * Clear perf_enable bit in L3C_PERF_CTRL register to stop counting
	122	+ * for all enabled counters.
	123	+ */
	124	+ val = readl(l3c_pmu->base + L3C_PERF_CTRL);
	125	+ val &= ~(L3C_PERF_CTRL_EN);
	126	+ writel(val, l3c_pmu->base + L3C_PERF_CTRL);
	127	+}
	128	+
	129	+static void hisi_l3c_pmu_enable_counter(struct hisi_pmu *l3c_pmu,
	130	+ struct hw_perf_event *hwc)
	131	+{
	132	+ u32 val;
	133	+
	134	+ /* Enable counter index in L3C_EVENT_CTRL register */
	135	+ val = readl(l3c_pmu->base + L3C_EVENT_CTRL);
	136	+ val \|= (1 << hwc->idx);
	137	+ writel(val, l3c_pmu->base + L3C_EVENT_CTRL);
	138	+}
	139	+
	140	+static void hisi_l3c_pmu_disable_counter(struct hisi_pmu *l3c_pmu,
	141	+ struct hw_perf_event *hwc)
	142	+{
	143	+ u32 val;
	144	+
	145	+ /* Clear counter index in L3C_EVENT_CTRL register */
	146	+ val = readl(l3c_pmu->base + L3C_EVENT_CTRL);
	147	+ val &= ~(1 << hwc->idx);
	148	+ writel(val, l3c_pmu->base + L3C_EVENT_CTRL);
	149	+}
	150	+
	151	+static void hisi_l3c_pmu_enable_counter_int(struct hisi_pmu *l3c_pmu,
	152	+ struct hw_perf_event *hwc)
	153	+{
	154	+ u32 val;
	155	+
	156	+ val = readl(l3c_pmu->base + L3C_INT_MASK);
	157	+ /* Write 0 to enable interrupt */
	158	+ val &= ~(1 << hwc->idx);
	159	+ writel(val, l3c_pmu->base + L3C_INT_MASK);
	160	+}
	161	+
	162	+static void hisi_l3c_pmu_disable_counter_int(struct hisi_pmu *l3c_pmu,
	163	+ struct hw_perf_event *hwc)
	164	+{
	165	+ u32 val;
	166	+
	167	+ val = readl(l3c_pmu->base + L3C_INT_MASK);
	168	+ /* Write 1 to mask interrupt */
	169	+ val \|= (1 << hwc->idx);
	170	+ writel(val, l3c_pmu->base + L3C_INT_MASK);
	171	+}
	172	+
	173	+static irqreturn_t hisi_l3c_pmu_isr(int irq, void *dev_id)
	174	+{
	175	+ struct hisi_pmu *l3c_pmu = dev_id;
	176	+ struct perf_event *event;
	177	+ unsigned long overflown;
	178	+ int idx;
	179	+
	180	+ /* Read L3C_INT_STATUS register */
	181	+ overflown = readl(l3c_pmu->base + L3C_INT_STATUS);
	182	+ if (!overflown)
	183	+ return IRQ_NONE;
	184	+
	185	+ /*
	186	+ * Find the counter index which overflowed if the bit was set
	187	+ * and handle it.
	188	+ */
	189	+ for_each_set_bit(idx, &overflown, L3C_NR_COUNTERS) {
	190	+ /* Write 1 to clear the IRQ status flag */
	191	+ writel((1 << idx), l3c_pmu->base + L3C_INT_CLEAR);
	192	+
	193	+ /* Get the corresponding event struct */
	194	+ event = l3c_pmu->pmu_events.hw_events[idx];
	195	+ if (!event)
	196	+ continue;
	197	+
	198	+ hisi_uncore_pmu_event_update(event);
	199	+ hisi_uncore_pmu_set_event_period(event);
	200	+ }
	201	+
	202	+ return IRQ_HANDLED;
	203	+}
	204	+
	205	+static int hisi_l3c_pmu_init_irq(struct hisi_pmu *l3c_pmu,
	206	+ struct platform_device *pdev)
	207	+{
	208	+ int irq, ret;
	209	+
	210	+ /* Read and init IRQ */
	211	+ irq = platform_get_irq(pdev, 0);
	212	+ if (irq < 0) {
	213	+ dev_err(&pdev->dev, "L3C PMU get irq fail; irq:%d\n", irq);
	214	+ return irq;
	215	+ }
	216	+
	217	+ ret = devm_request_irq(&pdev->dev, irq, hisi_l3c_pmu_isr,
	218	+ IRQF_NOBALANCING \| IRQF_NO_THREAD,
	219	+ dev_name(&pdev->dev), l3c_pmu);
	220	+ if (ret < 0) {
	221	+ dev_err(&pdev->dev,
	222	+ "Fail to request IRQ:%d ret:%d\n", irq, ret);
	223	+ return ret;
	224	+ }
	225	+
	226	+ l3c_pmu->irq = irq;
	227	+
	228	+ return 0;
	229	+}
	230	+
	231	+static const struct acpi_device_id hisi_l3c_pmu_acpi_match[] = {
	232	+ { "HISI0213", },
	233	+ {},
	234	+};
	235	+MODULE_DEVICE_TABLE(acpi, hisi_l3c_pmu_acpi_match);
	236	+
	237	+static int hisi_l3c_pmu_init_data(struct platform_device *pdev,
	238	+ struct hisi_pmu *l3c_pmu)
	239	+{
	240	+ unsigned long long id;
	241	+ struct resource *res;
	242	+ acpi_status status;
	243	+
	244	+ status = acpi_evaluate_integer(ACPI_HANDLE(&pdev->dev),
	245	+ "_UID", NULL, &id);
	246	+ if (ACPI_FAILURE(status))
	247	+ return -EINVAL;
	248	+
	249	+ l3c_pmu->index_id = id;
	250	+
	251	+ /*
	252	+ * Use the SCCL_ID and CCL_ID to identify the L3C PMU, while
	253	+ * SCCL_ID is in MPIDR[aff2] and CCL_ID is in MPIDR[aff1].
	254	+ */
	255	+ if (device_property_read_u32(&pdev->dev, "hisilicon,scl-id",
	256	+ &l3c_pmu->sccl_id)) {
	257	+ dev_err(&pdev->dev, "Can not read l3c sccl-id!\n");
	258	+ return -EINVAL;
	259	+ }
	260	+
	261	+ if (device_property_read_u32(&pdev->dev, "hisilicon,ccl-id",
	262	+ &l3c_pmu->ccl_id)) {
	263	+ dev_err(&pdev->dev, "Can not read l3c ccl-id!\n");
	264	+ return -EINVAL;
	265	+ }
	266	+
	267	+ res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
	268	+ l3c_pmu->base = devm_ioremap_resource(&pdev->dev, res);
	269	+ if (IS_ERR(l3c_pmu->base)) {
	270	+ dev_err(&pdev->dev, "ioremap failed for l3c_pmu resource\n");
	271	+ return PTR_ERR(l3c_pmu->base);
	272	+ }
	273	+
	274	+ return 0;
	275	+}
	276	+
	277	+static struct attribute *hisi_l3c_pmu_format_attr[] = {
	278	+ HISI_PMU_FORMAT_ATTR(event, "config:0-7"),
	279	+ NULL,
	280	+};
	281	+
	282	+static const struct attribute_group hisi_l3c_pmu_format_group = {
	283	+ .name = "format",
	284	+ .attrs = hisi_l3c_pmu_format_attr,
	285	+};
	286	+
	287	+static struct attribute *hisi_l3c_pmu_events_attr[] = {
	288	+ HISI_PMU_EVENT_ATTR(rd_cpipe, 0x00),
	289	+ HISI_PMU_EVENT_ATTR(wr_cpipe, 0x01),
	290	+ HISI_PMU_EVENT_ATTR(rd_hit_cpipe, 0x02),
	291	+ HISI_PMU_EVENT_ATTR(wr_hit_cpipe, 0x03),
	292	+ HISI_PMU_EVENT_ATTR(victim_num, 0x04),
	293	+ HISI_PMU_EVENT_ATTR(rd_spipe, 0x20),
	294	+ HISI_PMU_EVENT_ATTR(wr_spipe, 0x21),
	295	+ HISI_PMU_EVENT_ATTR(rd_hit_spipe, 0x22),
	296	+ HISI_PMU_EVENT_ATTR(wr_hit_spipe, 0x23),
	297	+ HISI_PMU_EVENT_ATTR(back_invalid, 0x29),
	298	+ HISI_PMU_EVENT_ATTR(retry_cpu, 0x40),
	299	+ HISI_PMU_EVENT_ATTR(retry_ring, 0x41),
	300	+ HISI_PMU_EVENT_ATTR(prefetch_drop, 0x42),
	301	+ NULL,
	302	+};
	303	+
	304	+static const struct attribute_group hisi_l3c_pmu_events_group = {
	305	+ .name = "events",
	306	+ .attrs = hisi_l3c_pmu_events_attr,
	307	+};
	308	+
	309	+static DEVICE_ATTR(cpumask, 0444, hisi_cpumask_sysfs_show, NULL);
	310	+
	311	+static struct attribute *hisi_l3c_pmu_cpumask_attrs[] = {
	312	+ &dev_attr_cpumask.attr,
	313	+ NULL,
	314	+};
	315	+
	316	+static const struct attribute_group hisi_l3c_pmu_cpumask_attr_group = {
	317	+ .attrs = hisi_l3c_pmu_cpumask_attrs,
	318	+};
	319	+
	320	+static const struct attribute_group *hisi_l3c_pmu_attr_groups[] = {
	321	+ &hisi_l3c_pmu_format_group,
	322	+ &hisi_l3c_pmu_events_group,
	323	+ &hisi_l3c_pmu_cpumask_attr_group,
	324	+ NULL,
	325	+};
	326	+
	327	+static const struct hisi_uncore_ops hisi_uncore_l3c_ops = {
	328	+ .write_evtype = hisi_l3c_pmu_write_evtype,
	329	+ .get_event_idx = hisi_uncore_pmu_get_event_idx,
	330	+ .start_counters = hisi_l3c_pmu_start_counters,
	331	+ .stop_counters = hisi_l3c_pmu_stop_counters,
	332	+ .enable_counter = hisi_l3c_pmu_enable_counter,
	333	+ .disable_counter = hisi_l3c_pmu_disable_counter,
	334	+ .enable_counter_int = hisi_l3c_pmu_enable_counter_int,
	335	+ .disable_counter_int = hisi_l3c_pmu_disable_counter_int,
	336	+ .write_counter = hisi_l3c_pmu_write_counter,
	337	+ .read_counter = hisi_l3c_pmu_read_counter,
	338	+};
	339	+
	340	+static int hisi_l3c_pmu_dev_probe(struct platform_device *pdev,
	341	+ struct hisi_pmu *l3c_pmu)
	342	+{
	343	+ int ret;
	344	+
	345	+ ret = hisi_l3c_pmu_init_data(pdev, l3c_pmu);
	346	+ if (ret)
	347	+ return ret;
	348	+
	349	+ ret = hisi_l3c_pmu_init_irq(l3c_pmu, pdev);
	350	+ if (ret)
	351	+ return ret;
	352	+
	353	+ l3c_pmu->num_counters = L3C_NR_COUNTERS;
	354	+ l3c_pmu->counter_bits = 48;
	355	+ l3c_pmu->ops = &hisi_uncore_l3c_ops;
	356	+ l3c_pmu->dev = &pdev->dev;
	357	+ l3c_pmu->on_cpu = -1;
	358	+ l3c_pmu->check_event = 0x59;
	359	+
	360	+ return 0;
	361	+}
	362	+
	363	+static int hisi_l3c_pmu_probe(struct platform_device *pdev)
	364	+{
	365	+ struct hisi_pmu *l3c_pmu;
	366	+ char *name;
	367	+ int ret;
	368	+
	369	+ l3c_pmu = devm_kzalloc(&pdev->dev, sizeof(*l3c_pmu), GFP_KERNEL);
	370	+ if (!l3c_pmu)
	371	+ return -ENOMEM;
	372	+
	373	+ platform_set_drvdata(pdev, l3c_pmu);
	374	+
	375	+ ret = hisi_l3c_pmu_dev_probe(pdev, l3c_pmu);
	376	+ if (ret)
	377	+ return ret;
	378	+
	379	+ ret = cpuhp_state_add_instance(CPUHP_AP_PERF_ARM_HISI_L3_ONLINE,
	380	+ &l3c_pmu->node);
	381	+ if (ret) {
	382	+ dev_err(&pdev->dev, "Error %d registering hotplug\n", ret);
	383	+ return ret;
	384	+ }
	385	+
	386	+ name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "hisi_sccl%u_l3c%u",
	387	+ l3c_pmu->sccl_id, l3c_pmu->index_id);
	388	+ l3c_pmu->pmu = (struct pmu) {
	389	+ .name = name,
	390	+ .task_ctx_nr = perf_invalid_context,
	391	+ .event_init = hisi_uncore_pmu_event_init,
	392	+ .pmu_enable = hisi_uncore_pmu_enable,
	393	+ .pmu_disable = hisi_uncore_pmu_disable,
	394	+ .add = hisi_uncore_pmu_add,
	395	+ .del = hisi_uncore_pmu_del,
	396	+ .start = hisi_uncore_pmu_start,
	397	+ .stop = hisi_uncore_pmu_stop,
	398	+ .read = hisi_uncore_pmu_read,
	399	+ .attr_groups = hisi_l3c_pmu_attr_groups,
	400	+ };
	401	+
	402	+ ret = perf_pmu_register(&l3c_pmu->pmu, name, -1);
	403	+ if (ret) {
	404	+ dev_err(l3c_pmu->dev, "L3C PMU register failed!\n");
	405	+ cpuhp_state_remove_instance(CPUHP_AP_PERF_ARM_HISI_L3_ONLINE,
	406	+ &l3c_pmu->node);
	407	+ }
	408	+
	409	+ return ret;
	410	+}
	411	+
	412	+static int hisi_l3c_pmu_remove(struct platform_device *pdev)
	413	+{
	414	+ struct hisi_pmu *l3c_pmu = platform_get_drvdata(pdev);
	415	+
	416	+ perf_pmu_unregister(&l3c_pmu->pmu);
	417	+ cpuhp_state_remove_instance(CPUHP_AP_PERF_ARM_HISI_L3_ONLINE,
	418	+ &l3c_pmu->node);
	419	+
	420	+ return 0;
	421	+}
	422	+
	423	+static struct platform_driver hisi_l3c_pmu_driver = {
	424	+ .driver = {
	425	+ .name = "hisi_l3c_pmu",
	426	+ .acpi_match_table = ACPI_PTR(hisi_l3c_pmu_acpi_match),
	427	+ },
	428	+ .probe = hisi_l3c_pmu_probe,
	429	+ .remove = hisi_l3c_pmu_remove,
	430	+};
	431	+
	432	+static int __init hisi_l3c_pmu_module_init(void)
	433	+{
	434	+ int ret;
	435	+
	436	+ ret = cpuhp_setup_state_multi(CPUHP_AP_PERF_ARM_HISI_L3_ONLINE,
	437	+ "AP_PERF_ARM_HISI_L3_ONLINE",
	438	+ hisi_uncore_pmu_online_cpu,
	439	+ hisi_uncore_pmu_offline_cpu);
	440	+ if (ret) {
	441	+ pr_err("L3C PMU: Error setup hotplug, ret = %d\n", ret);
	442	+ return ret;
	443	+ }
	444	+
	445	+ ret = platform_driver_register(&hisi_l3c_pmu_driver);
	446	+ if (ret)
	447	+ cpuhp_remove_multi_state(CPUHP_AP_PERF_ARM_HISI_L3_ONLINE);
	448	+
	449	+ return ret;
	450	+}
	451	+module_init(hisi_l3c_pmu_module_init);
	452	+
	453	+static void __exit hisi_l3c_pmu_module_exit(void)
	454	+{
	455	+ platform_driver_unregister(&hisi_l3c_pmu_driver);
	456	+ cpuhp_remove_multi_state(CPUHP_AP_PERF_ARM_HISI_L3_ONLINE);
	457	+}
	458	+module_exit(hisi_l3c_pmu_module_exit);
	459	+
	460	+MODULE_DESCRIPTION("HiSilicon SoC L3C uncore PMU driver");
	461	+MODULE_LICENSE("GPL v2");
	462	+MODULE_AUTHOR("Anurup M <anurup.m@huawei.com>");
	463	+MODULE_AUTHOR("Shaokun Zhang <zhangshaokun@hisilicon.com>");

drivers/perf/hisilicon/hisi_uncore_pmu.c+447−0

@@ -0,0 +1,447 @@
	1	+/*
	2	+ * HiSilicon SoC Hardware event counters support
	3	+ *
	4	+ * Copyright (C) 2017 Hisilicon Limited
	5	+ * Author: Anurup M <anurup.m@huawei.com>
	6	+ * Shaokun Zhang <zhangshaokun@hisilicon.com>
	7	+ *
	8	+ * This code is based on the uncore PMUs like arm-cci and arm-ccn.
	9	+ *
	10	+ * This program is free software; you can redistribute it and/or modify
	11	+ * it under the terms of the GNU General Public License version 2 as
	12	+ * published by the Free Software Foundation.
	13	+ */
	14	+#include <linux/bitmap.h>
	15	+#include <linux/bitops.h>
	16	+#include <linux/bug.h>
	17	+#include <linux/err.h>
	18	+#include <linux/errno.h>
	19	+#include <linux/interrupt.h>
	20	+
	21	+#include <asm/local64.h>
	22	+
	23	+#include "hisi_uncore_pmu.h"
	24	+
	25	+#define HISI_GET_EVENTID(ev) (ev->hw.config_base & 0xff)
	26	+#define HISI_MAX_PERIOD(nr) (BIT_ULL(nr) - 1)
	27	+
	28	+/*
	29	+ * PMU format attributes
	30	+ */
	31	+ssize_t hisi_format_sysfs_show(struct device *dev,
	32	+ struct device_attribute attr, char buf)
	33	+{
	34	+ struct dev_ext_attribute *eattr;
	35	+
	36	+ eattr = container_of(attr, struct dev_ext_attribute, attr);
	37	+
	38	+ return sprintf(buf, "%s\n", (char *)eattr->var);
	39	+}
	40	+
	41	+/*
	42	+ * PMU event attributes
	43	+ */
	44	+ssize_t hisi_event_sysfs_show(struct device *dev,
	45	+ struct device_attribute attr, char page)
	46	+{
	47	+ struct dev_ext_attribute *eattr;
	48	+
	49	+ eattr = container_of(attr, struct dev_ext_attribute, attr);
	50	+
	51	+ return sprintf(page, "config=0x%lx\n", (unsigned long)eattr->var);
	52	+}
	53	+
	54	+/*
	55	+ * sysfs cpumask attributes. For uncore PMU, we only have a single CPU to show
	56	+ */
	57	+ssize_t hisi_cpumask_sysfs_show(struct device *dev,
	58	+ struct device_attribute attr, char buf)
	59	+{
	60	+ struct hisi_pmu *hisi_pmu = to_hisi_pmu(dev_get_drvdata(dev));
	61	+
	62	+ return sprintf(buf, "%d\n", hisi_pmu->on_cpu);
	63	+}
	64	+
	65	+static bool hisi_validate_event_group(struct perf_event *event)
	66	+{
	67	+ struct perf_event sibling, leader = event->group_leader;
	68	+ struct hisi_pmu *hisi_pmu = to_hisi_pmu(event->pmu);
	69	+ /* Include count for the event */
	70	+ int counters = 1;
	71	+
	72	+ if (!is_software_event(leader)) {
	73	+ /*
	74	+ * We must NOT create groups containing mixed PMUs, although
	75	+ * software events are acceptable
	76	+ */
	77	+ if (leader->pmu != event->pmu)
	78	+ return false;
	79	+
	80	+ /* Increment counter for the leader */
	81	+ if (leader != event)
	82	+ counters++;
	83	+ }
	84	+
	85	+ list_for_each_entry(sibling, &event->group_leader->sibling_list,
	86	+ group_entry) {
	87	+ if (is_software_event(sibling))
	88	+ continue;
	89	+ if (sibling->pmu != event->pmu)
	90	+ return false;
	91	+ /* Increment counter for each sibling */
	92	+ counters++;
	93	+ }
	94	+
	95	+ /* The group can not count events more than the counters in the HW */
	96	+ return counters <= hisi_pmu->num_counters;
	97	+}
	98	+
	99	+int hisi_uncore_pmu_counter_valid(struct hisi_pmu *hisi_pmu, int idx)
	100	+{
	101	+ return idx >= 0 && idx < hisi_pmu->num_counters;
	102	+}
	103	+
	104	+int hisi_uncore_pmu_get_event_idx(struct perf_event *event)
	105	+{
	106	+ struct hisi_pmu *hisi_pmu = to_hisi_pmu(event->pmu);
	107	+ unsigned long *used_mask = hisi_pmu->pmu_events.used_mask;
	108	+ u32 num_counters = hisi_pmu->num_counters;
	109	+ int idx;
	110	+
	111	+ idx = find_first_zero_bit(used_mask, num_counters);
	112	+ if (idx == num_counters)
	113	+ return -EAGAIN;
	114	+
	115	+ set_bit(idx, used_mask);
	116	+
	117	+ return idx;
	118	+}
	119	+
	120	+static void hisi_uncore_pmu_clear_event_idx(struct hisi_pmu *hisi_pmu, int idx)
	121	+{
	122	+ if (!hisi_uncore_pmu_counter_valid(hisi_pmu, idx)) {
	123	+ dev_err(hisi_pmu->dev, "Unsupported event index:%d!\n", idx);
	124	+ return;
	125	+ }
	126	+
	127	+ clear_bit(idx, hisi_pmu->pmu_events.used_mask);
	128	+}
	129	+
	130	+int hisi_uncore_pmu_event_init(struct perf_event *event)
	131	+{
	132	+ struct hw_perf_event *hwc = &event->hw;
	133	+ struct hisi_pmu *hisi_pmu;
	134	+
	135	+ if (event->attr.type != event->pmu->type)
	136	+ return -ENOENT;
	137	+
	138	+ /*
	139	+ * We do not support sampling as the counters are all
	140	+ * shared by all CPU cores in a CPU die(SCCL). Also we
	141	+ * do not support attach to a task(per-process mode)
	142	+ */
	143	+ if (is_sampling_event(event) \|\| event->attach_state & PERF_ATTACH_TASK)
	144	+ return -EOPNOTSUPP;
	145	+
	146	+ /* counters do not have these bits */
	147	+ if (event->attr.exclude_user \|\|
	148	+ event->attr.exclude_kernel \|\|
	149	+ event->attr.exclude_host \|\|
	150	+ event->attr.exclude_guest \|\|
	151	+ event->attr.exclude_hv \|\|
	152	+ event->attr.exclude_idle)
	153	+ return -EINVAL;
	154	+
	155	+ /*
	156	+ * The uncore counters not specific to any CPU, so cannot
	157	+ * support per-task
	158	+ */
	159	+ if (event->cpu < 0)
	160	+ return -EINVAL;
	161	+
	162	+ /*
	163	+ * Validate if the events in group does not exceed the
	164	+ * available counters in hardware.
	165	+ */
	166	+ if (!hisi_validate_event_group(event))
	167	+ return -EINVAL;
	168	+
	169	+ hisi_pmu = to_hisi_pmu(event->pmu);
	170	+ if (event->attr.config > hisi_pmu->check_event)
	171	+ return -EINVAL;
	172	+
	173	+ if (hisi_pmu->on_cpu == -1)
	174	+ return -EINVAL;
	175	+ /*
	176	+ * We don't assign an index until we actually place the event onto
	177	+ * hardware. Use -1 to signify that we haven't decided where to put it
	178	+ * yet.
	179	+ */
	180	+ hwc->idx = -1;
	181	+ hwc->config_base = event->attr.config;
	182	+
	183	+ /* Enforce to use the same CPU for all events in this PMU */
	184	+ event->cpu = hisi_pmu->on_cpu;
	185	+
	186	+ return 0;
	187	+}
	188	+
	189	+/*
	190	+ * Set the counter to count the event that we're interested in,
	191	+ * and enable interrupt and counter.
	192	+ */
	193	+static void hisi_uncore_pmu_enable_event(struct perf_event *event)
	194	+{
	195	+ struct hisi_pmu *hisi_pmu = to_hisi_pmu(event->pmu);
	196	+ struct hw_perf_event *hwc = &event->hw;
	197	+
	198	+ hisi_pmu->ops->write_evtype(hisi_pmu, hwc->idx,
	199	+ HISI_GET_EVENTID(event));
	200	+
	201	+ hisi_pmu->ops->enable_counter_int(hisi_pmu, hwc);
	202	+ hisi_pmu->ops->enable_counter(hisi_pmu, hwc);
	203	+}
	204	+
	205	+/*
	206	+ * Disable counter and interrupt.
	207	+ */
	208	+static void hisi_uncore_pmu_disable_event(struct perf_event *event)
	209	+{
	210	+ struct hisi_pmu *hisi_pmu = to_hisi_pmu(event->pmu);
	211	+ struct hw_perf_event *hwc = &event->hw;
	212	+
	213	+ hisi_pmu->ops->disable_counter(hisi_pmu, hwc);
	214	+ hisi_pmu->ops->disable_counter_int(hisi_pmu, hwc);
	215	+}
	216	+
	217	+void hisi_uncore_pmu_set_event_period(struct perf_event *event)
	218	+{
	219	+ struct hisi_pmu *hisi_pmu = to_hisi_pmu(event->pmu);
	220	+ struct hw_perf_event *hwc = &event->hw;
	221	+
	222	+ /*
	223	+ * The HiSilicon PMU counters support 32 bits or 48 bits, depending on
	224	+ * the PMU. We reduce it to 2^(counter_bits - 1) to account for the
	225	+ * extreme interrupt latency. So we could hopefully handle the overflow
	226	+ * interrupt before another 2^(counter_bits - 1) events occur and the
	227	+ * counter overtakes its previous value.
	228	+ */
	229	+ u64 val = BIT_ULL(hisi_pmu->counter_bits - 1);
	230	+
	231	+ local64_set(&hwc->prev_count, val);
	232	+ /* Write start value to the hardware event counter */
	233	+ hisi_pmu->ops->write_counter(hisi_pmu, hwc, val);
	234	+}
	235	+
	236	+void hisi_uncore_pmu_event_update(struct perf_event *event)
	237	+{
	238	+ struct hisi_pmu *hisi_pmu = to_hisi_pmu(event->pmu);
	239	+ struct hw_perf_event *hwc = &event->hw;
	240	+ u64 delta, prev_raw_count, new_raw_count;
	241	+
	242	+ do {
	243	+ /* Read the count from the counter register */
	244	+ new_raw_count = hisi_pmu->ops->read_counter(hisi_pmu, hwc);
	245	+ prev_raw_count = local64_read(&hwc->prev_count);
	246	+ } while (local64_cmpxchg(&hwc->prev_count, prev_raw_count,
	247	+ new_raw_count) != prev_raw_count);
	248	+ /*
	249	+ * compute the delta
	250	+ */
	251	+ delta = (new_raw_count - prev_raw_count) &
	252	+ HISI_MAX_PERIOD(hisi_pmu->counter_bits);
	253	+ local64_add(delta, &event->count);
	254	+}
	255	+
	256	+void hisi_uncore_pmu_start(struct perf_event *event, int flags)
	257	+{
	258	+ struct hisi_pmu *hisi_pmu = to_hisi_pmu(event->pmu);
	259	+ struct hw_perf_event *hwc = &event->hw;
	260	+
	261	+ if (WARN_ON_ONCE(!(hwc->state & PERF_HES_STOPPED)))
	262	+ return;
	263	+
	264	+ WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE));
	265	+ hwc->state = 0;
	266	+ hisi_uncore_pmu_set_event_period(event);
	267	+
	268	+ if (flags & PERF_EF_RELOAD) {
	269	+ u64 prev_raw_count = local64_read(&hwc->prev_count);
	270	+
	271	+ hisi_pmu->ops->write_counter(hisi_pmu, hwc, prev_raw_count);
	272	+ }
	273	+
	274	+ hisi_uncore_pmu_enable_event(event);
	275	+ perf_event_update_userpage(event);
	276	+}
	277	+
	278	+void hisi_uncore_pmu_stop(struct perf_event *event, int flags)
	279	+{
	280	+ struct hw_perf_event *hwc = &event->hw;
	281	+
	282	+ hisi_uncore_pmu_disable_event(event);
	283	+ WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED);
	284	+ hwc->state \|= PERF_HES_STOPPED;
	285	+
	286	+ if (hwc->state & PERF_HES_UPTODATE)
	287	+ return;
	288	+
	289	+ /* Read hardware counter and update the perf counter statistics */
	290	+ hisi_uncore_pmu_event_update(event);
	291	+ hwc->state \|= PERF_HES_UPTODATE;
	292	+}
	293	+
	294	+int hisi_uncore_pmu_add(struct perf_event *event, int flags)
	295	+{
	296	+ struct hisi_pmu *hisi_pmu = to_hisi_pmu(event->pmu);
	297	+ struct hw_perf_event *hwc = &event->hw;
	298	+ int idx;
	299	+
	300	+ hwc->state = PERF_HES_STOPPED \| PERF_HES_UPTODATE;
	301	+
	302	+ /* Get an available counter index for counting */
	303	+ idx = hisi_pmu->ops->get_event_idx(event);
	304	+ if (idx < 0)
	305	+ return idx;
	306	+
	307	+ event->hw.idx = idx;
	308	+ hisi_pmu->pmu_events.hw_events[idx] = event;
	309	+
	310	+ if (flags & PERF_EF_START)
	311	+ hisi_uncore_pmu_start(event, PERF_EF_RELOAD);
	312	+
	313	+ return 0;
	314	+}
	315	+
	316	+void hisi_uncore_pmu_del(struct perf_event *event, int flags)
	317	+{
	318	+ struct hisi_pmu *hisi_pmu = to_hisi_pmu(event->pmu);
	319	+ struct hw_perf_event *hwc = &event->hw;
	320	+
	321	+ hisi_uncore_pmu_stop(event, PERF_EF_UPDATE);
	322	+ hisi_uncore_pmu_clear_event_idx(hisi_pmu, hwc->idx);
	323	+ perf_event_update_userpage(event);
	324	+ hisi_pmu->pmu_events.hw_events[hwc->idx] = NULL;
	325	+}
	326	+
	327	+void hisi_uncore_pmu_read(struct perf_event *event)
	328	+{
	329	+ /* Read hardware counter and update the perf counter statistics */
	330	+ hisi_uncore_pmu_event_update(event);
	331	+}
	332	+
	333	+void hisi_uncore_pmu_enable(struct pmu *pmu)
	334	+{
	335	+ struct hisi_pmu *hisi_pmu = to_hisi_pmu(pmu);
	336	+ int enabled = bitmap_weight(hisi_pmu->pmu_events.used_mask,
	337	+ hisi_pmu->num_counters);
	338	+
	339	+ if (!enabled)
	340	+ return;
	341	+
	342	+ hisi_pmu->ops->start_counters(hisi_pmu);
	343	+}
	344	+
	345	+void hisi_uncore_pmu_disable(struct pmu *pmu)
	346	+{
	347	+ struct hisi_pmu *hisi_pmu = to_hisi_pmu(pmu);
	348	+
	349	+ hisi_pmu->ops->stop_counters(hisi_pmu);
	350	+}
	351	+
	352	+/*
	353	+ * Read Super CPU cluster and CPU cluster ID from MPIDR_EL1.
	354	+ * If multi-threading is supported, SCCL_ID is in MPIDR[aff3] and CCL_ID
	355	+ * is in MPIDR[aff2]; if not, SCCL_ID is in MPIDR[aff2] and CCL_ID is
	356	+ * in MPIDR[aff1]. If this changes in future, this shall be updated.
	357	+ */
	358	+static void hisi_read_sccl_and_ccl_id(int sccl_id, int ccl_id)
	359	+{
	360	+ u64 mpidr = read_cpuid_mpidr();
	361	+
	362	+ if (mpidr & MPIDR_MT_BITMASK) {
	363	+ if (sccl_id)
	364	+ *sccl_id = MPIDR_AFFINITY_LEVEL(mpidr, 3);
	365	+ if (ccl_id)
	366	+ *ccl_id = MPIDR_AFFINITY_LEVEL(mpidr, 2);
	367	+ } else {
	368	+ if (sccl_id)
	369	+ *sccl_id = MPIDR_AFFINITY_LEVEL(mpidr, 2);
	370	+ if (ccl_id)
	371	+ *ccl_id = MPIDR_AFFINITY_LEVEL(mpidr, 1);
	372	+ }
	373	+}
	374	+
	375	+/*
	376	+ * Check whether the CPU is associated with this uncore PMU
	377	+ */
	378	+static bool hisi_pmu_cpu_is_associated_pmu(struct hisi_pmu *hisi_pmu)
	379	+{
	380	+ int sccl_id, ccl_id;
	381	+
	382	+ if (hisi_pmu->ccl_id == -1) {
	383	+ /* If CCL_ID is -1, the PMU only shares the same SCCL */
	384	+ hisi_read_sccl_and_ccl_id(&sccl_id, NULL);
	385	+
	386	+ return sccl_id == hisi_pmu->sccl_id;
	387	+ }
	388	+
	389	+ hisi_read_sccl_and_ccl_id(&sccl_id, &ccl_id);
	390	+
	391	+ return sccl_id == hisi_pmu->sccl_id && ccl_id == hisi_pmu->ccl_id;
	392	+}
	393	+
	394	+int hisi_uncore_pmu_online_cpu(unsigned int cpu, struct hlist_node *node)
	395	+{
	396	+ struct hisi_pmu *hisi_pmu = hlist_entry_safe(node, struct hisi_pmu,
	397	+ node);
	398	+
	399	+ if (!hisi_pmu_cpu_is_associated_pmu(hisi_pmu))
	400	+ return 0;
	401	+
	402	+ cpumask_set_cpu(cpu, &hisi_pmu->associated_cpus);
	403	+
	404	+ /* If another CPU is already managing this PMU, simply return. */
	405	+ if (hisi_pmu->on_cpu != -1)
	406	+ return 0;
	407	+
	408	+ /* Use this CPU in cpumask for event counting */
	409	+ hisi_pmu->on_cpu = cpu;
	410	+
	411	+ /* Overflow interrupt also should use the same CPU */
	412	+ WARN_ON(irq_set_affinity(hisi_pmu->irq, cpumask_of(cpu)));
	413	+
	414	+ return 0;
	415	+}
	416	+
	417	+int hisi_uncore_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node)
	418	+{
	419	+ struct hisi_pmu *hisi_pmu = hlist_entry_safe(node, struct hisi_pmu,
	420	+ node);
	421	+ cpumask_t pmu_online_cpus;
	422	+ unsigned int target;
	423	+
	424	+ if (!cpumask_test_and_clear_cpu(cpu, &hisi_pmu->associated_cpus))
	425	+ return 0;
	426	+
	427	+ /* Nothing to do if this CPU doesn't own the PMU */
	428	+ if (hisi_pmu->on_cpu != cpu)
	429	+ return 0;
	430	+
	431	+ /* Give up ownership of the PMU */
	432	+ hisi_pmu->on_cpu = -1;
	433	+
	434	+ /* Choose a new CPU to migrate ownership of the PMU to */
	435	+ cpumask_and(&pmu_online_cpus, &hisi_pmu->associated_cpus,
	436	+ cpu_online_mask);
	437	+ target = cpumask_any_but(&pmu_online_cpus, cpu);
	438	+ if (target >= nr_cpu_ids)
	439	+ return 0;
	440	+
	441	+ perf_pmu_migrate_context(&hisi_pmu->pmu, cpu, target);
	442	+ /* Use this CPU for event counting */
	443	+ hisi_pmu->on_cpu = target;
	444	+ WARN_ON(irq_set_affinity(hisi_pmu->irq, cpumask_of(target)));
	445	+
	446	+ return 0;
	447	+}

drivers/perf/hisilicon/hisi_uncore_pmu.h+102−0

@@ -0,0 +1,102 @@
	1	+/*
	2	+ * HiSilicon SoC Hardware event counters support
	3	+ *
	4	+ * Copyright (C) 2017 Hisilicon Limited
	5	+ * Author: Anurup M <anurup.m@huawei.com>
	6	+ * Shaokun Zhang <zhangshaokun@hisilicon.com>
	7	+ *
	8	+ * This code is based on the uncore PMUs like arm-cci and arm-ccn.
	9	+ *
	10	+ * This program is free software; you can redistribute it and/or modify
	11	+ * it under the terms of the GNU General Public License version 2 as
	12	+ * published by the Free Software Foundation.
	13	+ */
	14	+#ifndef __HISI_UNCORE_PMU_H__
	15	+#define __HISI_UNCORE_PMU_H__
	16	+
	17	+#include <linux/cpumask.h>
	18	+#include <linux/device.h>
	19	+#include <linux/kernel.h>
	20	+#include <linux/perf_event.h>
	21	+#include <linux/types.h>
	22	+
	23	+#undef pr_fmt
	24	+#define pr_fmt(fmt) "hisi_pmu: " fmt
	25	+
	26	+#define HISI_MAX_COUNTERS 0x10
	27	+#define to_hisi_pmu(p) (container_of(p, struct hisi_pmu, pmu))
	28	+
	29	+#define HISI_PMU_ATTR(_name, _func, _config) \
	30	+ (&((struct dev_ext_attribute[]) { \
	31	+ { __ATTR(_name, 0444, _func, NULL), (void *)_config } \
	32	+ })[0].attr.attr)
	33	+
	34	+#define HISI_PMU_FORMAT_ATTR(_name, _config) \
	35	+ HISI_PMU_ATTR(_name, hisi_format_sysfs_show, (void *)_config)
	36	+#define HISI_PMU_EVENT_ATTR(_name, _config) \
	37	+ HISI_PMU_ATTR(_name, hisi_event_sysfs_show, (unsigned long)_config)
	38	+
	39	+struct hisi_pmu;
	40	+
	41	+struct hisi_uncore_ops {
	42	+ void (write_evtype)(struct hisi_pmu , int, u32);
	43	+ int (get_event_idx)(struct perf_event );
	44	+ u64 (read_counter)(struct hisi_pmu , struct hw_perf_event *);
	45	+ void (write_counter)(struct hisi_pmu , struct hw_perf_event *, u64);
	46	+ void (enable_counter)(struct hisi_pmu , struct hw_perf_event *);
	47	+ void (disable_counter)(struct hisi_pmu , struct hw_perf_event *);
	48	+ void (enable_counter_int)(struct hisi_pmu , struct hw_perf_event *);
	49	+ void (disable_counter_int)(struct hisi_pmu , struct hw_perf_event *);
	50	+ void (start_counters)(struct hisi_pmu );
	51	+ void (stop_counters)(struct hisi_pmu );
	52	+};
	53	+
	54	+struct hisi_pmu_hwevents {
	55	+ struct perf_event *hw_events[HISI_MAX_COUNTERS];
	56	+ DECLARE_BITMAP(used_mask, HISI_MAX_COUNTERS);
	57	+};
	58	+
	59	+/* Generic pmu struct for different pmu types */
	60	+struct hisi_pmu {
	61	+ struct pmu pmu;
	62	+ const struct hisi_uncore_ops *ops;
	63	+ struct hisi_pmu_hwevents pmu_events;
	64	+ /* associated_cpus: All CPUs associated with the PMU */
	65	+ cpumask_t associated_cpus;
	66	+ /* CPU used for counting */
	67	+ int on_cpu;
	68	+ int irq;
	69	+ struct device *dev;
	70	+ struct hlist_node node;
	71	+ int sccl_id;
	72	+ int ccl_id;
	73	+ void __iomem *base;
	74	+ /* the ID of the PMU modules */
	75	+ u32 index_id;
	76	+ int num_counters;
	77	+ int counter_bits;
	78	+ /* check event code range */
	79	+ int check_event;
	80	+};
	81	+
	82	+int hisi_uncore_pmu_counter_valid(struct hisi_pmu *hisi_pmu, int idx);
	83	+int hisi_uncore_pmu_get_event_idx(struct perf_event *event);
	84	+void hisi_uncore_pmu_read(struct perf_event *event);
	85	+int hisi_uncore_pmu_add(struct perf_event *event, int flags);
	86	+void hisi_uncore_pmu_del(struct perf_event *event, int flags);
	87	+void hisi_uncore_pmu_start(struct perf_event *event, int flags);
	88	+void hisi_uncore_pmu_stop(struct perf_event *event, int flags);
	89	+void hisi_uncore_pmu_set_event_period(struct perf_event *event);
	90	+void hisi_uncore_pmu_event_update(struct perf_event *event);
	91	+int hisi_uncore_pmu_event_init(struct perf_event *event);
	92	+void hisi_uncore_pmu_enable(struct pmu *pmu);
	93	+void hisi_uncore_pmu_disable(struct pmu *pmu);
	94	+ssize_t hisi_event_sysfs_show(struct device *dev,
	95	+ struct device_attribute attr, char buf);
	96	+ssize_t hisi_format_sysfs_show(struct device *dev,
	97	+ struct device_attribute attr, char buf);
	98	+ssize_t hisi_cpumask_sysfs_show(struct device *dev,
	99	+ struct device_attribute attr, char buf);
	100	+int hisi_uncore_pmu_online_cpu(unsigned int cpu, struct hlist_node *node);
	101	+int hisi_uncore_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node);
	102	+#endif /* __HISI_UNCORE_PMU_H__ */

drivers/perf/qcom_l2_pmu.c+54−0

@@ -92,6 +92,21 @@ · 91 unmodified lines
92	92
93	93	#define reg_idx(reg, i) (((i) * IA_L2_REG_OFFSET) + reg##_BASE)
94	94
	95	+/*
	96	+ * Events
	97	+ */
	98	+#define L2_EVENT_CYCLES 0xfe
	99	+#define L2_EVENT_DCACHE_OPS 0x400
	100	+#define L2_EVENT_ICACHE_OPS 0x401
	101	+#define L2_EVENT_TLBI 0x402
	102	+#define L2_EVENT_BARRIERS 0x403
	103	+#define L2_EVENT_TOTAL_READS 0x405
	104	+#define L2_EVENT_TOTAL_WRITES 0x406
	105	+#define L2_EVENT_TOTAL_REQUESTS 0x407
	106	+#define L2_EVENT_LDREX 0x420
	107	+#define L2_EVENT_STREX 0x421
	108	+#define L2_EVENT_CLREX 0x422
	109	+
95	110	static DEFINE_RAW_SPINLOCK(l2_access_lock);
96	111
97	112	/**
@@ -700,9 +715,12 @@ static struct attribute_group l2_cache_pmu_cpumask_group = { · 602 unmodified lines
700	715	/* CCG format for perf RAW codes. */
701	716	PMU_FORMAT_ATTR(l2_code, "config:4-11");
702	717	PMU_FORMAT_ATTR(l2_group, "config:0-3");
	718	+PMU_FORMAT_ATTR(event, "config:0-11");
	719	+
703	720	static struct attribute *l2_cache_pmu_formats[] = {
704	721	&format_attr_l2_code.attr,
705	722	&format_attr_l2_group.attr,
	723	+ &format_attr_event.attr,
706	724	NULL,
707	725	};
708	726
@@ -711,9 +729,45 @@ static struct attribute_group l2_cache_pmu_format_group = { · 2 unmodified lines
711	729	.attrs = l2_cache_pmu_formats,
712	730	};
713	731
	732	+static ssize_t l2cache_pmu_event_show(struct device *dev,
	733	+ struct device_attribute attr, char page)
	734	+{
	735	+ struct perf_pmu_events_attr *pmu_attr;
	736	+
	737	+ pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr);
	738	+ return sprintf(page, "event=0x%02llx\n", pmu_attr->id);
	739	+}
	740	+
	741	+#define L2CACHE_EVENT_ATTR(_name, _id) \
	742	+ (&((struct perf_pmu_events_attr[]) { \
	743	+ { .attr = __ATTR(_name, 0444, l2cache_pmu_event_show, NULL), \
	744	+ .id = _id, } \
	745	+ })[0].attr.attr)
	746	+
	747	+static struct attribute *l2_cache_pmu_events[] = {
	748	+ L2CACHE_EVENT_ATTR(cycles, L2_EVENT_CYCLES),
	749	+ L2CACHE_EVENT_ATTR(dcache-ops, L2_EVENT_DCACHE_OPS),
	750	+ L2CACHE_EVENT_ATTR(icache-ops, L2_EVENT_ICACHE_OPS),
	751	+ L2CACHE_EVENT_ATTR(tlbi, L2_EVENT_TLBI),
	752	+ L2CACHE_EVENT_ATTR(barriers, L2_EVENT_BARRIERS),
	753	+ L2CACHE_EVENT_ATTR(total-reads, L2_EVENT_TOTAL_READS),
	754	+ L2CACHE_EVENT_ATTR(total-writes, L2_EVENT_TOTAL_WRITES),
	755	+ L2CACHE_EVENT_ATTR(total-requests, L2_EVENT_TOTAL_REQUESTS),
	756	+ L2CACHE_EVENT_ATTR(ldrex, L2_EVENT_LDREX),
	757	+ L2CACHE_EVENT_ATTR(strex, L2_EVENT_STREX),
	758	+ L2CACHE_EVENT_ATTR(clrex, L2_EVENT_CLREX),
	759	+ NULL
	760	+};
	761	+
	762	+static struct attribute_group l2_cache_pmu_events_group = {
	763	+ .name = "events",
	764	+ .attrs = l2_cache_pmu_events,
	765	+};
	766	+
714	767	static const struct attribute_group *l2_cache_pmu_attr_grps[] = {
715	768	&l2_cache_pmu_format_group,
716	769	&l2_cache_pmu_cpumask_group,
	770	+ &l2_cache_pmu_events_group,
717	771	NULL,
718	772	};
719	773

fs/binfmt_elf.c+5−5

@@ -1699,7 +1699,7 @@ static int fill_thread_core_info(struct elf_thread_core_info *t, · 1698 unmodified lines
1699	1699	long signr, size_t *total)
1700	1700	{
1701	1701	unsigned int i;
1702		− unsigned int regset_size = view->regsets[0].n * view->regsets[0].size;
	1702	+ unsigned int regset0_size = regset_size(t->task, &view->regsets[0]);
1703	1703
1704	1704	/*
1705	1705	* NT_PRSTATUS is the one special case, because the regset data
@@ -1708,11 +1708,11 @@ static int fill_thread_core_info(struct elf_thread_core_info *t, · 2 unmodified lines
1708	1708	* We assume that regset 0 is NT_PRSTATUS.
1709	1709	*/
1710	1710	fill_prstatus(&t->prstatus, t->task, signr);
1711		− (void) view->regsets[0].get(t->task, &view->regsets[0], 0, regset_size,
	1711	+ (void) view->regsets[0].get(t->task, &view->regsets[0], 0, regset0_size,
1712	1712	&t->prstatus.pr_reg, NULL);
1713	1713
1714	1714	fill_note(&t->notes[0], "CORE", NT_PRSTATUS,
1715		− PRSTATUS_SIZE(t->prstatus, regset_size), &t->prstatus);
	1715	+ PRSTATUS_SIZE(t->prstatus, regset0_size), &t->prstatus);
1716	1716	*total += notesize(&t->notes[0]);
1717	1717
1718	1718	do_thread_regset_writeback(t->task, &view->regsets[0]);
@@ -1728,7 +1728,7 @@ static int fill_thread_core_info(struct elf_thread_core_info *t, · 9 unmodified lines
1728	1728	if (regset->core_note_type && regset->get &&
1729	1729	(!regset->active \|\| regset->active(t->task, regset))) {
1730	1730	int ret;
1731		− size_t size = regset->n * regset->size;
	1731	+ size_t size = regset_size(t->task, regset);
1732	1732	void *data = kmalloc(size, GFP_KERNEL);
1733	1733	if (unlikely(!data))
1734	1734	return 0;
@@ -1743,7 +1743,7 @@ static int fill_thread_core_info(struct elf_thread_core_info *t, · 8 unmodified lines
1743	1743	size, data);
1744	1744	else {
1745	1745	SET_PR_FPVALID(&t->prstatus,
1746		− 1, regset_size);
	1746	+ 1, regset0_size);
1747	1747	fill_note(&t->notes[i], "CORE",
1748	1748	NT_PRFPREG, size, data);
1749	1749	}

include/clocksource/arm_arch_timer.h+9−1

@@ -67,7 +67,9 @@ enum arch_timer_spi_nr { · 66 unmodified lines
67	67	#define ARCH_TIMER_USR_VT_ACCESS_EN (1 << 8) /* virtual timer registers */
68	68	#define ARCH_TIMER_USR_PT_ACCESS_EN (1 << 9) /* physical timer registers */
69	69
70		−#define ARCH_TIMER_EVT_STREAM_FREQ 10000 /* 100us */
	70	+#define ARCH_TIMER_EVT_STREAM_PERIOD_US 100
	71	+#define ARCH_TIMER_EVT_STREAM_FREQ \
	72	+ (USEC_PER_SEC / ARCH_TIMER_EVT_STREAM_PERIOD_US)
71	73
72	74	struct arch_timer_kvm_info {
73	75	struct timecounter timecounter;
@@ -93,6 +95,7 @@ struct arch_timer_mem { · 19 unmodified lines
93	95	extern u32 arch_timer_get_rate(void);
94	96	extern u64 (*arch_timer_read_counter)(void);
95	97	extern struct arch_timer_kvm_info *arch_timer_get_kvm_info(void);
	98	+extern bool arch_timer_evtstrm_available(void);
96	99
97	100	#else
98	101
@@ -106,6 +109,11 @@ static inline u64 arch_timer_read_counter(void) · 7 unmodified lines
106	109	return 0;
107	110	}
108	111
	112	+static inline bool arch_timer_evtstrm_available(void)
	113	+{
	114	+ return false;
	115	+}
	116	+
109	117	#endif
110	118
111	119	#endif

include/linux/acpi_iort.h+2−2

@@ -49,8 +49,8 @@ static inline void acpi_configure_pmsi_domain(struct device *dev) { } · 48 unmodified lines
49	49	/* IOMMU interface */
50	50	static inline void iort_dma_setup(struct device dev, u64 dma_addr,
51	51	u64 *size) { }
52		−static inline
53		−const struct iommu_ops iort_iommu_configure(struct device dev)
	52	+static inline const struct iommu_ops *iort_iommu_configure(
	53	+ struct device *dev)
54	54	{ return NULL; }
55	55	#endif
56	56

include/linux/cpuhotplug.h+3−0

@@ -155,6 +155,9 @@ enum cpuhp_state { · 154 unmodified lines
155	155	CPUHP_AP_PERF_S390_SF_ONLINE,
156	156	CPUHP_AP_PERF_ARM_CCI_ONLINE,
157	157	CPUHP_AP_PERF_ARM_CCN_ONLINE,
	158	+ CPUHP_AP_PERF_ARM_HISI_DDRC_ONLINE,
	159	+ CPUHP_AP_PERF_ARM_HISI_HHA_ONLINE,
	160	+ CPUHP_AP_PERF_ARM_HISI_L3_ONLINE,
158	161	CPUHP_AP_PERF_ARM_L2X0_ONLINE,
159	162	CPUHP_AP_PERF_ARM_QCOM_L2_ONLINE,
160	163	CPUHP_AP_PERF_ARM_QCOM_L3_ONLINE,

include/linux/irqdesc.h+8−0

@@ -246,6 +246,14 @@ static inline int irq_is_percpu(unsigned int irq) · 245 unmodified lines
246	246	return desc->status_use_accessors & IRQ_PER_CPU;
247	247	}
248	248
	249	+static inline int irq_is_percpu_devid(unsigned int irq)
	250	+{
	251	+ struct irq_desc *desc;
	252	+
	253	+ desc = irq_to_desc(irq);
	254	+ return desc->status_use_accessors & IRQ_PER_CPU_DEVID;
	255	+}
	256	+
249	257	static inline void
250	258	irq_set_lockdep_class(unsigned int irq, struct lock_class_key *class)
251	259	{

include/linux/regset.h+60−7

@@ -106,6 +106,28 @@ typedef int user_regset_writeback_fn(struct task_struct *target, · 105 unmodified lines
106	106	const struct user_regset *regset,
107	107	int immediate);
108	108
	109	+/**
	110	+ * user_regset_get_size_fn - type of @get_size function in &struct user_regset
	111	+ * @target: thread being examined
	112	+ * @regset: regset being examined
	113	+ *
	114	+ * This call is optional; usually the pointer is %NULL.
	115	+ *
	116	+ * When provided, this function must return the current size of regset
	117	+ * data, as observed by the @get function in &struct user_regset. The
	118	+ * value returned must be a multiple of @size. The returned size is
	119	+ * required to be valid only until the next time (if any) @regset is
	120	+ * modified for @target.
	121	+ *
	122	+ * This function is intended for dynamically sized regsets. A regset
	123	+ * that is statically sized does not need to implement it.
	124	+ *
	125	+ * This function should not be called directly: instead, callers should
	126	+ * call regset_size() to determine the current size of a regset.
	127	+ */
	128	+typedef unsigned int user_regset_get_size_fn(struct task_struct *target,
	129	+ const struct user_regset *regset);
	130	+
109	131	/**
110	132	* struct user_regset - accessible thread CPU state
111	133	* @n: Number of slots (registers).
@@ -117,19 +139,33 @@ typedef int user_regset_writeback_fn(struct task_struct *target, · 5 unmodified lines
117	139	* @set: Function to store values.
118	140	* @active: Function to report if regset is active, or %NULL.
119	141	* @writeback: Function to write data back to user memory, or %NULL.
	142	+ * @get_size: Function to return the regset's size, or %NULL.
120	143	*
121	144	* This data structure describes a machine resource we call a register set.
122	145	* This is part of the state of an individual thread, not necessarily
123	146	* actual CPU registers per se. A register set consists of a number of
124	147	* similar slots, given by @n. Each slot is @size bytes, and aligned to
125		− * @align bytes (which is at least @size).
	148	+ * @align bytes (which is at least @size). For dynamically-sized
	149	+ * regsets, @n must contain the maximum possible number of slots for the
	150	+ * regset, and @get_size must point to a function that returns the
	151	+ * current regset size.
126	152	*
127		− * These functions must be called only on the current thread or on a
128		− * thread that is in %TASK_STOPPED or %TASK_TRACED state, that we are
129		− * guaranteed will not be woken up and return to user mode, and that we
130		− * have called wait_task_inactive() on. (The target thread always might
131		− * wake up for SIGKILL while these functions are working, in which case
132		− * that thread's user_regset state might be scrambled.)
	153	+ * Callers that need to know only the current size of the regset and do
	154	+ * not care about its internal structure should call regset_size()
	155	+ * instead of inspecting @n or calling @get_size.
	156	+ *
	157	+ * For backward compatibility, the @get and @set methods must pad to, or
	158	+ * accept, @n * @size bytes, even if the current regset size is smaller.
	159	+ * The precise semantics of these operations depend on the regset being
	160	+ * accessed.
	161	+ *
	162	+ * The functions to which &struct user_regset members point must be
	163	+ * called only on the current thread or on a thread that is in
	164	+ * %TASK_STOPPED or %TASK_TRACED state, that we are guaranteed will not
	165	+ * be woken up and return to user mode, and that we have called
	166	+ * wait_task_inactive() on. (The target thread always might wake up for
	167	+ * SIGKILL while these functions are working, in which case that
	168	+ * thread's user_regset state might be scrambled.)
133	169	*
134	170	* The @pos argument must be aligned according to @align; the @count
135	171	* argument must be a multiple of @size. These functions are not
@@ -156,6 +192,7 @@ struct user_regset { · 20 unmodified lines
156	192	user_regset_set_fn *set;
157	193	user_regset_active_fn *active;
158	194	user_regset_writeback_fn *writeback;
	195	+ user_regset_get_size_fn *get_size;
159	196	unsigned int n;
160	197	unsigned int size;
161	198	unsigned int align;
@@ -371,5 +408,21 @@ static inline int copy_regset_from_user(struct task_struct *target, · 209 unmodified lines
371	408	return regset->set(target, regset, offset, size, NULL, data);
372	409	}
373	410
	411	+/**
	412	+ * regset_size - determine the current size of a regset
	413	+ * @target: thread to be examined
	414	+ * @regset: regset to be examined
	415	+ *
	416	+ * Note that the returned size is valid only until the next time
	417	+ * (if any) @regset is modified for @target.
	418	+ */
	419	+static inline unsigned int regset_size(struct task_struct *target,
	420	+ const struct user_regset *regset)
	421	+{
	422	+ if (!regset->get_size)
	423	+ return regset->n * regset->size;
	424	+ else
	425	+ return regset->get_size(target, regset);
	426	+}
374	427
375	428	#endif /* <linux/regset.h> */

include/uapi/linux/elf.h+1−0

@@ -418,6 +418,7 @@ typedef struct elf64_shdr { · 417 unmodified lines
418	418	#define NT_ARM_HW_BREAK 0x402 /* ARM hardware breakpoint registers */
419	419	#define NT_ARM_HW_WATCH 0x403 /* ARM hardware watchpoint registers */
420	420	#define NT_ARM_SYSTEM_CALL 0x404 /* ARM system call number */
	421	+#define NT_ARM_SVE 0x405 /* ARM Scalable Vector Extension registers */
421	422	#define NT_METAG_CBUF 0x500 /* Metag catch buffer registers */
422	423	#define NT_METAG_RPIPE 0x501 /* Metag read pipeline state */
423	424	#define NT_METAG_TLS 0x502 /* Metag TLS pointer */

include/uapi/linux/perf_event.h+1−0

@@ -942,6 +942,7 @@ enum perf_callchain_context { · 941 unmodified lines
942	942	#define PERF_AUX_FLAG_TRUNCATED 0x01 /* record was truncated to fit */
943	943	#define PERF_AUX_FLAG_OVERWRITE 0x02 /* snapshot from overwrite mode */
944	944	#define PERF_AUX_FLAG_PARTIAL 0x04 /* record contains gaps */
	945	+#define PERF_AUX_FLAG_COLLISION 0x08 /* sample collided with another */
945	946
946	947	#define PERF_FLAG_FD_NO_GROUP (1UL << 0)
947	948	#define PERF_FLAG_FD_OUTPUT (1UL << 1)

include/uapi/linux/prctl.h+9−0

@@ -198,4 +198,13 @@ struct prctl_mm_map { · 197 unmodified lines
198	198	# define PR_CAP_AMBIENT_LOWER 3
199	199	# define PR_CAP_AMBIENT_CLEAR_ALL 4
200	200
	201	+/* arm64 Scalable Vector Extension controls */
	202	+/* Flag values must be kept in sync with ptrace NT_ARM_SVE interface */
	203	+#define PR_SVE_SET_VL 50 /* set task vector length */
	204	+# define PR_SVE_SET_VL_ONEXEC (1 << 18) /* defer effect until exec */
	205	+#define PR_SVE_GET_VL 51 /* get task vector length */
	206	+/* Bits common to PR_SVE_SET_VL and PR_SVE_GET_VL */
	207	+# define PR_SVE_VL_LEN_MASK 0xffff
	208	+# define PR_SVE_VL_INHERIT (1 << 17) /* inherit across exec */
	209	+
201	210	#endif /* _LINUX_PRCTL_H */

kernel/events/ring_buffer.c+4−0

@@ -411,6 +411,7 @@ err: · 410 unmodified lines
411	411
412	412	return NULL;
413	413	}
	414	+EXPORT_SYMBOL_GPL(perf_aux_output_begin);
414	415
415	416	static bool __always_inline rb_need_aux_wakeup(struct ring_buffer *rb)
416	417	{
@@ -480,6 +481,7 @@ void perf_aux_output_end(struct perf_output_handle *handle, unsigned long size) · 63 unmodified lines
480	481	rb_free_aux(rb);
481	482	ring_buffer_put(rb);
482	483	}
	484	+EXPORT_SYMBOL_GPL(perf_aux_output_end);
483	485
484	486	/*
485	487	* Skip over a given number of bytes in the AUX buffer, due to, for example,
@@ -505,6 +507,7 @@ int perf_aux_output_skip(struct perf_output_handle *handle, unsigned long size) · 19 unmodified lines
505	507
506	508	return 0;
507	509	}
	510	+EXPORT_SYMBOL_GPL(perf_aux_output_skip);
508	511
509	512	void perf_get_aux(struct perf_output_handle handle)
510	513	{
@@ -514,6 +517,7 @@ void perf_get_aux(struct perf_output_handle handle) · 3 unmodified lines
514	517
515	518	return handle->rb->aux_priv;
516	519	}
	520	+EXPORT_SYMBOL_GPL(perf_get_aux);
517	521
518	522	#define PERF_AUX_GFP (GFP_KERNEL \| __GFP_ZERO \| __GFP_NOWARN \| __GFP_NORETRY)
519	523

kernel/irq/irqdesc.c+1−0

@@ -862,6 +862,7 @@ int irq_get_percpu_devid_partition(unsigned int irq, struct cpumask *affinity) · 861 unmodified lines
862	862
863	863	return 0;
864	864	}
	865	+EXPORT_SYMBOL_GPL(irq_get_percpu_devid_partition);
865	866
866	867	void kstat_incr_irq_this_cpu(unsigned int irq)
867	868	{

kernel/sys.c+12−0

@@ -111,6 +111,12 @@ · 110 unmodified lines
111	111	#ifndef SET_FP_MODE
112	112	# define SET_FP_MODE(a,b) (-EINVAL)
113	113	#endif
	114	+#ifndef SVE_SET_VL
	115	+# define SVE_SET_VL(a) (-EINVAL)
	116	+#endif
	117	+#ifndef SVE_GET_VL
	118	+# define SVE_GET_VL() (-EINVAL)
	119	+#endif
114	120
115	121	/*
116	122	* this is where the system-wide overflow UID and GID are defined, for
@@ -2386,6 +2392,12 @@ SYSCALL_DEFINE5(prctl, int, option, unsigned long, arg2, unsigned long, arg3, · 2269 unmodified lines
2386	2392	case PR_GET_FP_MODE:
2387	2393	error = GET_FP_MODE(me);
2388	2394	break;
	2395	+ case PR_SVE_SET_VL:
	2396	+ error = SVE_SET_VL(arg2);
	2397	+ break;
	2398	+ case PR_SVE_GET_VL:
	2399	+ error = SVE_GET_VL();
	2400	+ break;
2389	2401	default:
2390	2402	error = -EINVAL;
2391	2403	break;

virt/kvm/arm/arm.c+3−0

@@ -652,6 +652,9 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu vcpu, struct kvm_run run) · 651 unmodified lines
652	652	*/
653	653	preempt_disable();
654	654
	655	+ /* Flush FP/SIMD state that can't survive guest entry/exit */
	656	+ kvm_fpsimd_flush_cpu_state();
	657	+
655	658	kvm_pmu_flush_hwstate(vcpu);
656	659
657	660	kvm_timer_flush_hwstate(vcpu);

Merge tag 'arm64-upstream' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code