lael-labs/linux

Documentation/arm64/cpu-feature-registers.txt+16−2

      x--------------------------------------------------x
      | Name                         |  bits   | visible |
      |--------------------------------------------------|
-     | RES0                         | [63-32] |    n    |
+     | RES0                         | [63-48] |    n    |
+     |--------------------------------------------------|
+     | DP                           | [47-44] |    y    |
+     |--------------------------------------------------|
+     | SM4                          | [43-40] |    y    |
+     |--------------------------------------------------|
+     | SM3                          | [39-36] |    y    |
+     |--------------------------------------------------|
+     | SHA3                         | [35-32] |    y    |
      |--------------------------------------------------|
      | RDM                          | [31-28] |    y    |
      |--------------------------------------------------|
+     | RES0                         | [27-24] |    n    |
+     |--------------------------------------------------|
      | ATOMICS                      | [23-20] |    y    |
      |--------------------------------------------------|
      | CRC32                        | [19-16] |    y    |
      x--------------------------------------------------x
      | Name                         |  bits   | visible |
      |--------------------------------------------------|
-     | RES0                         | [63-28] |    n    |
+     | RES0                         | [63-36] |    n    |
+     |--------------------------------------------------|
+     | SVE                          | [35-32] |    y    |
+     |--------------------------------------------------|
+     | RES0                         | [31-28] |    n    |
      |--------------------------------------------------|
      | GIC                          | [27-24] |    n    |
      |--------------------------------------------------|

Documentation/arm64/elf_hwcaps.txt+160−0

@@ -0,0 +1,160 @@
	1	ARM64 ELF hwcaps
	2	================
	3
	4	This document describes the usage and semantics of the arm64 ELF hwcaps.
	5
	6
	7	1. Introduction
	8	---------------
	9
	10	Some hardware or software features are only available on some CPU
	11	implementations, and/or with certain kernel configurations, but have no
	12	architected discovery mechanism available to userspace code at EL0. The
	13	kernel exposes the presence of these features to userspace through a set
	14	of flags called hwcaps, exposed in the auxilliary vector.
	15
	16	Userspace software can test for features by acquiring the AT_HWCAP entry
	17	of the auxilliary vector, and testing whether the relevant flags are
	18	set, e.g.
	19
	20	bool floating_point_is_present(void)
	21	{
	22	unsigned long hwcaps = getauxval(AT_HWCAP);
	23	if (hwcaps & HWCAP_FP)
	24	return true;
	25
	26	return false;
	27	}
	28
	29	Where software relies on a feature described by a hwcap, it should check
	30	the relevant hwcap flag to verify that the feature is present before
	31	attempting to make use of the feature.
	32
	33	Features cannot be probed reliably through other means. When a feature
	34	is not available, attempting to use it may result in unpredictable
	35	behaviour, and is not guaranteed to result in any reliable indication
	36	that the feature is unavailable, such as a SIGILL.
	37
	38
	39	2. Interpretation of hwcaps
	40	---------------------------
	41
	42	The majority of hwcaps are intended to indicate the presence of features
	43	which are described by architected ID registers inaccessible to
	44	userspace code at EL0. These hwcaps are defined in terms of ID register
	45	fields, and should be interpreted with reference to the definition of
	46	these fields in the ARM Architecture Reference Manual (ARM ARM).
	47
	48	Such hwcaps are described below in the form:
	49
	50	Functionality implied by idreg.field == val.
	51
	52	Such hwcaps indicate the availability of functionality that the ARM ARM
	53	defines as being present when idreg.field has value val, but do not
	54	indicate that idreg.field is precisely equal to val, nor do they
	55	indicate the absence of functionality implied by other values of
	56	idreg.field.
	57
	58	Other hwcaps may indicate the presence of features which cannot be
	59	described by ID registers alone. These may be described without
	60	reference to ID registers, and may refer to other documentation.
	61
	62
	63	3. The hwcaps exposed in AT_HWCAP
	64	---------------------------------
	65
	66	HWCAP_FP
	67
	68	Functionality implied by ID_AA64PFR0_EL1.FP == 0b0000.
	69
	70	HWCAP_ASIMD
	71
	72	Functionality implied by ID_AA64PFR0_EL1.AdvSIMD == 0b0000.
	73
	74	HWCAP_EVTSTRM
	75
	76	The generic timer is configured to generate events at a frequency of
	77	approximately 100KHz.
	78
	79	HWCAP_AES
	80
	81	Functionality implied by ID_AA64ISAR1_EL1.AES == 0b0001.
	82
	83	HWCAP_PMULL
	84
	85	Functionality implied by ID_AA64ISAR1_EL1.AES == 0b0010.
	86
	87	HWCAP_SHA1
	88
	89	Functionality implied by ID_AA64ISAR0_EL1.SHA1 == 0b0001.
	90
	91	HWCAP_SHA2
	92
	93	Functionality implied by ID_AA64ISAR0_EL1.SHA2 == 0b0001.
	94
	95	HWCAP_CRC32
	96
	97	Functionality implied by ID_AA64ISAR0_EL1.CRC32 == 0b0001.
	98
	99	HWCAP_ATOMICS
	100
	101	Functionality implied by ID_AA64ISAR0_EL1.Atomic == 0b0010.
	102
	103	HWCAP_FPHP
	104
	105	Functionality implied by ID_AA64PFR0_EL1.FP == 0b0001.
	106
	107	HWCAP_ASIMDHP
	108
	109	Functionality implied by ID_AA64PFR0_EL1.AdvSIMD == 0b0001.
	110
	111	HWCAP_CPUID
	112
	113	EL0 access to certain ID registers is available, to the extent
	114	described by Documentation/arm64/cpu-feature-registers.txt.
	115
	116	These ID registers may imply the availability of features.
	117
	118	HWCAP_ASIMDRDM
	119
	120	Functionality implied by ID_AA64ISAR0_EL1.RDM == 0b0001.
	121
	122	HWCAP_JSCVT
	123
	124	Functionality implied by ID_AA64ISAR1_EL1.JSCVT == 0b0001.
	125
	126	HWCAP_FCMA
	127
	128	Functionality implied by ID_AA64ISAR1_EL1.FCMA == 0b0001.
	129
	130	HWCAP_LRCPC
	131
	132	Functionality implied by ID_AA64ISAR1_EL1.LRCPC == 0b0001.
	133
	134	HWCAP_DCPOP
	135
	136	Functionality implied by ID_AA64ISAR1_EL1.DPB == 0b0001.
	137
	138	HWCAP_SHA3
	139
	140	Functionality implied by ID_AA64ISAR0_EL1.SHA3 == 0b0001.
	141
	142	HWCAP_SM3
	143
	144	Functionality implied by ID_AA64ISAR0_EL1.SM3 == 0b0001.
	145
	146	HWCAP_SM4
	147
	148	Functionality implied by ID_AA64ISAR0_EL1.SM4 == 0b0001.
	149
	150	HWCAP_ASIMDDP
	151
	152	Functionality implied by ID_AA64ISAR0_EL1.DP == 0b0001.
	153
	154	HWCAP_SHA512
	155
	156	Functionality implied by ID_AA64ISAR0_EL1.SHA2 == 0b0002.
	157
	158	HWCAP_SVE
	159
	160	Functionality implied by ID_AA64PFR0_EL1.SVE == 0b0001.

Documentation/arm64/memory.txt+5−5

@@ -86,9 +86,9 @@ Translation table lookup with 64KB pages: · 85 unmodified lines
86	+-------------------------------------------------> [63] TTBR0/1	86	+-------------------------------------------------> [63] TTBR0/1
87		87
88		88
89	When using KVM, the hypervisor maps kernel pages in EL2, at a fixed	89	When using KVM without the Virtualization Host Extensions, the hypervisor
90	offset from the kernel VA (top 24bits of the kernel VA set to zero):	90	maps kernel pages in EL2 at a fixed offset from the kernel VA. See the
		91	kern_hyp_va macro for more details.
91		92
92	Start End Size Use	93	When using KVM with the Virtualization Host Extensions, no additional
93	-----------------------------------------------------------------------	94	mappings are created, since the host kernel runs directly in EL2.
94	0000004000000000 0000007fffffffff 256GB kernel objects mapped in HYP

Documentation/arm64/sve.txt+508−0

@@ -0,0 +1,508 @@
	1	Scalable Vector Extension support for AArch64 Linux
	2	===================================================
	3
	4	Author: Dave Martin <Dave.Martin@arm.com>
	5	Date: 4 August 2017
	6
	7	This document outlines briefly the interface provided to userspace by Linux in
	8	order to support use of the ARM Scalable Vector Extension (SVE).
	9
	10	This is an outline of the most important features and issues only and not
	11	intended to be exhaustive.
	12
	13	This document does not aim to describe the SVE architecture or programmer's
	14	model. To aid understanding, a minimal description of relevant programmer's
	15	model features for SVE is included in Appendix A.
	16
	17
	18	1. General
	19	-----------
	20
	21	* SVE registers Z0..Z31, P0..P15 and FFR and the current vector length VL, are
	22	tracked per-thread.
	23
	24	* The presence of SVE is reported to userspace via HWCAP_SVE in the aux vector
	25	AT_HWCAP entry. Presence of this flag implies the presence of the SVE
	26	instructions and registers, and the Linux-specific system interfaces
	27	described in this document. SVE is reported in /proc/cpuinfo as "sve".
	28
	29	* Support for the execution of SVE instructions in userspace can also be
	30	detected by reading the CPU ID register ID_AA64PFR0_EL1 using an MRS
	31	instruction, and checking that the value of the SVE field is nonzero. [3]
	32
	33	It does not guarantee the presence of the system interfaces described in the
	34	following sections: software that needs to verify that those interfaces are
	35	present must check for HWCAP_SVE instead.
	36
	37	* Debuggers should restrict themselves to interacting with the target via the
	38	NT_ARM_SVE regset. The recommended way of detecting support for this regset
	39	is to connect to a target process first and then attempt a
	40	ptrace(PTRACE_GETREGSET, pid, NT_ARM_SVE, &iov).
	41
	42
	43	2. Vector length terminology
	44	-----------------------------
	45
	46	The size of an SVE vector (Z) register is referred to as the "vector length".
	47
	48	To avoid confusion about the units used to express vector length, the kernel
	49	adopts the following conventions:
	50
	51	* Vector length (VL) = size of a Z-register in bytes
	52
	53	* Vector quadwords (VQ) = size of a Z-register in units of 128 bits
	54
	55	(So, VL = 16 * VQ.)
	56
	57	The VQ convention is used where the underlying granularity is important, such
	58	as in data structure definitions. In most other situations, the VL convention
	59	is used. This is consistent with the meaning of the "VL" pseudo-register in
	60	the SVE instruction set architecture.
	61
	62
	63	3. System call behaviour
	64	-------------------------
	65
	66	* On syscall, V0..V31 are preserved (as without SVE). Thus, bits [127:0] of
	67	Z0..Z31 are preserved. All other bits of Z0..Z31, and all of P0..P15 and FFR
	68	become unspecified on return from a syscall.
	69
	70	* The SVE registers are not used to pass arguments to or receive results from
	71	any syscall.
	72
	73	* In practice the affected registers/bits will be preserved or will be replaced
	74	with zeros on return from a syscall, but userspace should not make
	75	assumptions about this. The kernel behaviour may vary on a case-by-case
	76	basis.
	77
	78	* All other SVE state of a thread, including the currently configured vector
	79	length, the state of the PR_SVE_VL_INHERIT flag, and the deferred vector
	80	length (if any), is preserved across all syscalls, subject to the specific
	81	exceptions for execve() described in section 6.
	82
	83	In particular, on return from a fork() or clone(), the parent and new child
	84	process or thread share identical SVE configuration, matching that of the
	85	parent before the call.
	86
	87
	88	4. Signal handling
	89	-------------------
	90
	91	* A new signal frame record sve_context encodes the SVE registers on signal
	92	delivery. [1]
	93
	94	* This record is supplementary to fpsimd_context. The FPSR and FPCR registers
	95	are only present in fpsimd_context. For convenience, the content of V0..V31
	96	is duplicated between sve_context and fpsimd_context.
	97
	98	* The signal frame record for SVE always contains basic metadata, in particular
	99	the thread's vector length (in sve_context.vl).
	100
	101	* The SVE registers may or may not be included in the record, depending on
	102	whether the registers are live for the thread. The registers are present if
	103	and only if:
	104	sve_context.head.size >= SVE_SIG_CONTEXT_SIZE(sve_vq_from_vl(sve_context.vl)).
	105
	106	* If the registers are present, the remainder of the record has a vl-dependent
	107	size and layout. Macros SVE_SIG_* are defined [1] to facilitate access to
	108	the members.
	109
	110	* If the SVE context is too big to fit in sigcontext.__reserved[], then extra
	111	space is allocated on the stack, an extra_context record is written in
	112	__reserved[] referencing this space. sve_context is then written in the
	113	extra space. Refer to [1] for further details about this mechanism.
	114
	115
	116	5. Signal return
	117	-----------------
	118
	119	When returning from a signal handler:
	120
	121	* If there is no sve_context record in the signal frame, or if the record is
	122	present but contains no register data as desribed in the previous section,
	123	then the SVE registers/bits become non-live and take unspecified values.
	124
	125	* If sve_context is present in the signal frame and contains full register
	126	data, the SVE registers become live and are populated with the specified
	127	data. However, for backward compatibility reasons, bits [127:0] of Z0..Z31
	128	are always restored from the corresponding members of fpsimd_context.vregs[]
	129	and not from sve_context. The remaining bits are restored from sve_context.
	130
	131	* Inclusion of fpsimd_context in the signal frame remains mandatory,
	132	irrespective of whether sve_context is present or not.
	133
	134	* The vector length cannot be changed via signal return. If sve_context.vl in
	135	the signal frame does not match the current vector length, the signal return
	136	attempt is treated as illegal, resulting in a forced SIGSEGV.
	137
	138
	139	6. prctl extensions
	140	--------------------
	141
	142	Some new prctl() calls are added to allow programs to manage the SVE vector
	143	length:
	144
	145	prctl(PR_SVE_SET_VL, unsigned long arg)
	146
	147	Sets the vector length of the calling thread and related flags, where
	148	arg == vl \| flags. Other threads of the calling process are unaffected.
	149
	150	vl is the desired vector length, where sve_vl_valid(vl) must be true.
	151
	152	flags:
	153
	154	PR_SVE_SET_VL_INHERIT
	155
	156	Inherit the current vector length across execve(). Otherwise, the
	157	vector length is reset to the system default at execve(). (See
	158	Section 9.)
	159
	160	PR_SVE_SET_VL_ONEXEC
	161
	162	Defer the requested vector length change until the next execve()
	163	performed by this thread.
	164
	165	The effect is equivalent to implicit exceution of the following
	166	call immediately after the next execve() (if any) by the thread:
	167
	168	prctl(PR_SVE_SET_VL, arg & ~PR_SVE_SET_VL_ONEXEC)
	169
	170	This allows launching of a new program with a different vector
	171	length, while avoiding runtime side effects in the caller.
	172
	173
	174	Without PR_SVE_SET_VL_ONEXEC, the requested change takes effect
	175	immediately.
	176
	177
	178	Return value: a nonnegative on success, or a negative value on error:
	179	EINVAL: SVE not supported, invalid vector length requested, or
	180	invalid flags.
	181
	182
	183	On success:
	184
	185	* Either the calling thread's vector length or the deferred vector length
	186	to be applied at the next execve() by the thread (dependent on whether
	187	PR_SVE_SET_VL_ONEXEC is present in arg), is set to the largest value
	188	supported by the system that is less than or equal to vl. If vl ==
	189	SVE_VL_MAX, the value set will be the largest value supported by the
	190	system.
	191
	192	* Any previously outstanding deferred vector length change in the calling
	193	thread is cancelled.
	194
	195	* The returned value describes the resulting configuration, encoded as for
	196	PR_SVE_GET_VL. The vector length reported in this value is the new
	197	current vector length for this thread if PR_SVE_SET_VL_ONEXEC was not
	198	present in arg; otherwise, the reported vector length is the deferred
	199	vector length that will be applied at the next execve() by the calling
	200	thread.
	201
	202	* Changing the vector length causes all of P0..P15, FFR and all bits of
	203	Z0..V31 except for Z0 bits [127:0] .. Z31 bits [127:0] to become
	204	unspecified. Calling PR_SVE_SET_VL with vl equal to the thread's current
	205	vector length, or calling PR_SVE_SET_VL with the PR_SVE_SET_VL_ONEXEC
	206	flag, does not constitute a change to the vector length for this purpose.
	207
	208
	209	prctl(PR_SVE_GET_VL)
	210
	211	Gets the vector length of the calling thread.
	212
	213	The following flag may be OR-ed into the result:
	214
	215	PR_SVE_SET_VL_INHERIT
	216
	217	Vector length will be inherited across execve().
	218
	219	There is no way to determine whether there is an outstanding deferred
	220	vector length change (which would only normally be the case between a
	221	fork() or vfork() and the corresponding execve() in typical use).
	222
	223	To extract the vector length from the result, and it with
	224	PR_SVE_VL_LEN_MASK.
	225
	226	Return value: a nonnegative value on success, or a negative value on error:
	227	EINVAL: SVE not supported.
	228
	229
	230	7. ptrace extensions
	231	---------------------
	232
	233	* A new regset NT_ARM_SVE is defined for use with PTRACE_GETREGSET and
	234	PTRACE_SETREGSET.
	235
	236	Refer to [2] for definitions.
	237
	238	The regset data starts with struct user_sve_header, containing:
	239
	240	size
	241
	242	Size of the complete regset, in bytes.
	243	This depends on vl and possibly on other things in the future.
	244
	245	If a call to PTRACE_GETREGSET requests less data than the value of
	246	size, the caller can allocate a larger buffer and retry in order to
	247	read the complete regset.
	248
	249	max_size
	250
	251	Maximum size in bytes that the regset can grow to for the target
	252	thread. The regset won't grow bigger than this even if the target
	253	thread changes its vector length etc.
	254
	255	vl
	256
	257	Target thread's current vector length, in bytes.
	258
	259	max_vl
	260
	261	Maximum possible vector length for the target thread.
	262
	263	flags
	264
	265	either
	266
	267	SVE_PT_REGS_FPSIMD
	268
	269	SVE registers are not live (GETREGSET) or are to be made
	270	non-live (SETREGSET).
	271
	272	The payload is of type struct user_fpsimd_state, with the same
	273	meaning as for NT_PRFPREG, starting at offset
	274	SVE_PT_FPSIMD_OFFSET from the start of user_sve_header.
	275
	276	Extra data might be appended in the future: the size of the
	277	payload should be obtained using SVE_PT_FPSIMD_SIZE(vq, flags).
	278
	279	vq should be obtained using sve_vq_from_vl(vl).
	280
	281	or
	282
	283	SVE_PT_REGS_SVE
	284
	285	SVE registers are live (GETREGSET) or are to be made live
	286	(SETREGSET).
	287
	288	The payload contains the SVE register data, starting at offset
	289	SVE_PT_SVE_OFFSET from the start of user_sve_header, and with
	290	size SVE_PT_SVE_SIZE(vq, flags);
	291
	292	... OR-ed with zero or more of the following flags, which have the same
	293	meaning and behaviour as the corresponding PR_SET_VL_* flags:
	294
	295	SVE_PT_VL_INHERIT
	296
	297	SVE_PT_VL_ONEXEC (SETREGSET only).
	298
	299	* The effects of changing the vector length and/or flags are equivalent to
	300	those documented for PR_SVE_SET_VL.
	301
	302	The caller must make a further GETREGSET call if it needs to know what VL is
	303	actually set by SETREGSET, unless is it known in advance that the requested
	304	VL is supported.
	305
	306	* In the SVE_PT_REGS_SVE case, the size and layout of the payload depends on
	307	the header fields. The SVE_PT_SVE_*() macros are provided to facilitate
	308	access to the members.
	309
	310	* In either case, for SETREGSET it is permissible to omit the payload, in which
	311	case only the vector length and flags are changed (along with any
	312	consequences of those changes).
	313
	314	* For SETREGSET, if an SVE_PT_REGS_SVE payload is present and the
	315	requested VL is not supported, the effect will be the same as if the
	316	payload were omitted, except that an EIO error is reported. No
	317	attempt is made to translate the payload data to the correct layout
	318	for the vector length actually set. The thread's FPSIMD state is
	319	preserved, but the remaining bits of the SVE registers become
	320	unspecified. It is up to the caller to translate the payload layout
	321	for the actual VL and retry.
	322
	323	* The effect of writing a partial, incomplete payload is unspecified.
	324
	325
	326	8. ELF coredump extensions
	327	---------------------------
	328
	329	* A NT_ARM_SVE note will be added to each coredump for each thread of the
	330	dumped process. The contents will be equivalent to the data that would have
	331	been read if a PTRACE_GETREGSET of NT_ARM_SVE were executed for each thread
	332	when the coredump was generated.
	333
	334
	335	9. System runtime configuration
	336	--------------------------------
	337
	338	* To mitigate the ABI impact of expansion of the signal frame, a policy
	339	mechanism is provided for administrators, distro maintainers and developers
	340	to set the default vector length for userspace processes:
	341
	342	/proc/sys/abi/sve_default_vector_length
	343
	344	Writing the text representation of an integer to this file sets the system
	345	default vector length to the specified value, unless the value is greater
	346	than the maximum vector length supported by the system in which case the
	347	default vector length is set to that maximum.
	348
	349	The result can be determined by reopening the file and reading its
	350	contents.
	351
	352	At boot, the default vector length is initially set to 64 or the maximum
	353	supported vector length, whichever is smaller. This determines the initial
	354	vector length of the init process (PID 1).
	355
	356	Reading this file returns the current system default vector length.
	357
	358	* At every execve() call, the new vector length of the new process is set to
	359	the system default vector length, unless
	360
	361	* PR_SVE_SET_VL_INHERIT (or equivalently SVE_PT_VL_INHERIT) is set for the
	362	calling thread, or
	363
	364	* a deferred vector length change is pending, established via the
	365	PR_SVE_SET_VL_ONEXEC flag (or SVE_PT_VL_ONEXEC).
	366
	367	* Modifying the system default vector length does not affect the vector length
	368	of any existing process or thread that does not make an execve() call.
	369
	370
	371	Appendix A. SVE programmer's model (informative)
	372	=================================================
	373
	374	This section provides a minimal description of the additions made by SVE to the
	375	ARMv8-A programmer's model that are relevant to this document.
	376
	377	Note: This section is for information only and not intended to be complete or
	378	to replace any architectural specification.
	379
	380	A.1. Registers
	381	---------------
	382
	383	In A64 state, SVE adds the following:
	384
	385	* 32 8VL-bit vector registers Z0..Z31
	386	For each Zn, Zn bits [127:0] alias the ARMv8-A vector register Vn.
	387
	388	A register write using a Vn register name zeros all bits of the corresponding
	389	Zn except for bits [127:0].
	390
	391	* 16 VL-bit predicate registers P0..P15
	392
	393	* 1 VL-bit special-purpose predicate register FFR (the "first-fault register")
	394
	395	* a VL "pseudo-register" that determines the size of each vector register
	396
	397	The SVE instruction set architecture provides no way to write VL directly.
	398	Instead, it can be modified only by EL1 and above, by writing appropriate
	399	system registers.
	400
	401	* The value of VL can be configured at runtime by EL1 and above:
	402	16 <= VL <= VLmax, where VL must be a multiple of 16.
	403
	404	* The maximum vector length is determined by the hardware:
	405	16 <= VLmax <= 256.
	406
	407	(The SVE architecture specifies 256, but permits future architecture
	408	revisions to raise this limit.)
	409
	410	* FPSR and FPCR are retained from ARMv8-A, and interact with SVE floating-point
	411	operations in a similar way to the way in which they interact with ARMv8
	412	floating-point operations.
	413
	414	8VL-1 128 0 bit index
	415	+---- //// -----------------+
	416	Z0 \| : V0 \|
	417	: :
	418	Z7 \| : V7 \|
	419	Z8 \| : * V8 \|
	420	: : :
	421	Z15 \| : *V15 \|
	422	Z16 \| : V16 \|
	423	: :
	424	Z31 \| : V31 \|
	425	+---- //// -----------------+
	426	31 0
	427	VL-1 0 +-------+
	428	+---- //// --+ FPSR \| \|
	429	P0 \| \| +-------+
	430	: \| \| *FPCR \| \|
	431	P15 \| \| +-------+
	432	+---- //// --+
	433	FFR \| \| +-----+
	434	+---- //// --+ VL \| \|
	435	+-----+
	436
	437	(*) callee-save:
	438	This only applies to bits [63:0] of Z-/V-registers.
	439	FPCR contains callee-save and caller-save bits. See [4] for details.
	440
	441
	442	A.2. Procedure call standard
	443	-----------------------------
	444
	445	The ARMv8-A base procedure call standard is extended as follows with respect to
	446	the additional SVE register state:
	447
	448	* All SVE register bits that are not shared with FP/SIMD are caller-save.
	449
	450	* Z8 bits [63:0] .. Z15 bits [63:0] are callee-save.
	451
	452	This follows from the way these bits are mapped to V8..V15, which are caller-
	453	save in the base procedure call standard.
	454
	455
	456	Appendix B. ARMv8-A FP/SIMD programmer's model
	457	===============================================
	458
	459	Note: This section is for information only and not intended to be complete or
	460	to replace any architectural specification.
	461
	462	Refer to [4] for for more information.
	463
	464	ARMv8-A defines the following floating-point / SIMD register state:
	465
	466	* 32 128-bit vector registers V0..V31
	467	* 2 32-bit status/control registers FPSR, FPCR
	468
	469	127 0 bit index
	470	+---------------+
	471	V0 \| \|
	472	: : :
	473	V7 \| \|
	474	* V8 \| \|
	475	: : : :
	476	*V15 \| \|
	477	V16 \| \|
	478	: : :
	479	V31 \| \|
	480	+---------------+
	481
	482	31 0
	483	+-------+
	484	FPSR \| \|
	485	+-------+
	486	*FPCR \| \|
	487	+-------+
	488
	489	(*) callee-save:
	490	This only applies to bits [63:0] of V-registers.
	491	FPCR contains a mixture of callee-save and caller-save bits.
	492
	493
	494	References
	495	==========
	496
	497	[1] arch/arm64/include/uapi/asm/sigcontext.h
	498	AArch64 Linux signal ABI definitions
	499
	500	[2] arch/arm64/include/uapi/asm/ptrace.h
	501	AArch64 Linux ptrace ABI definitions
	502
	503	[3] linux/Documentation/arm64/cpu-feature-registers.txt
	504
	505	[4] ARM IHI0055C
	506	http://infocenter.arm.com/help/topic/com.arm.doc.ihi0055c/IHI0055C_beta_aapcs64.pdf
	507	http://infocenter.arm.com/help/topic/com.arm.doc.subset.swdev.abi/index.html
	508	Procedure Call Standard for the ARM 64-bit Architecture (AArch64)

Documentation/devicetree/bindings/arm/spe-pmu.txt+20−0

@@ -0,0 +1,20 @@
	1	* ARMv8.2 Statistical Profiling Extension (SPE) Performance Monitor Units (PMU)
	2
	3	ARMv8.2 introduces the optional Statistical Profiling Extension for collecting
	4	performance sample data using an in-memory trace buffer.
	5
	6	** SPE Required properties:
	7
	8	- compatible : should be one of:
	9	"arm,statistical-profiling-extension-v1"
	10
	11	- interrupts : Exactly 1 PPI must be listed. For heterogeneous systems where
	12	SPE is only supported on a subset of the CPUs, please consult
	13	the arm,gic-v3 binding for details on describing a PPI partition.
	14
	15	** Example:
	16
	17	spe-pmu {
	18	compatible = "arm,statistical-profiling-extension-v1";
	19	interrupts = <GIC_PPI 05 IRQ_TYPE_LEVEL_HIGH &part1>;
	20	};

Documentation/perf/hisi-pmu.txt+53−0

@@ -0,0 +1,53 @@
	1	HiSilicon SoC uncore Performance Monitoring Unit (PMU)
	2	======================================================
	3	The HiSilicon SoC chip includes various independent system device PMUs
	4	such as L3 cache (L3C), Hydra Home Agent (HHA) and DDRC. These PMUs are
	5	independent and have hardware logic to gather statistics and performance
	6	information.
	7
	8	The HiSilicon SoC encapsulates multiple CPU and IO dies. Each CPU cluster
	9	(CCL) is made up of 4 cpu cores sharing one L3 cache; each CPU die is
	10	called Super CPU cluster (SCCL) and is made up of 6 CCLs. Each SCCL has
	11	two HHAs (0 - 1) and four DDRCs (0 - 3), respectively.
	12
	13	HiSilicon SoC uncore PMU driver
	14	---------------------------------------
	15	Each device PMU has separate registers for event counting, control and
	16	interrupt, and the PMU driver shall register perf PMU drivers like L3C,
	17	HHA and DDRC etc. The available events and configuration options shall
	18	be described in the sysfs, see :
	19	/sys/devices/hisi_sccl{X}_<l3c{Y}/hha{Y}/ddrc{Y}>/, or
	20	/sys/bus/event_source/devices/hisi_sccl{X}_<l3c{Y}/hha{Y}/ddrc{Y}>.
	21	The "perf list" command shall list the available events from sysfs.
	22
	23	Each L3C, HHA and DDRC is registered as a separate PMU with perf. The PMU
	24	name will appear in event listing as hisi_sccl<sccl-id>_module<index-id>.
	25	where "sccl-id" is the identifier of the SCCL and "index-id" is the index of
	26	module.
	27	e.g. hisi_sccl3_l3c0/rd_hit_cpipe is READ_HIT_CPIPE event of L3C index #0 in
	28	SCCL ID #3.
	29	e.g. hisi_sccl1_hha0/rx_operations is RX_OPERATIONS event of HHA index #0 in
	30	SCCL ID #1.
	31
	32	The driver also provides a "cpumask" sysfs attribute, which shows the CPU core
	33	ID used to count the uncore PMU event.
	34
	35	Example usage of perf:
	36	$# perf list
	37	hisi_sccl3_l3c0/rd_hit_cpipe/ [kernel PMU event]
	38	------------------------------------------
	39	hisi_sccl3_l3c0/wr_hit_cpipe/ [kernel PMU event]
	40	------------------------------------------
	41	hisi_sccl1_l3c0/rd_hit_cpipe/ [kernel PMU event]
	42	------------------------------------------
	43	hisi_sccl1_l3c0/wr_hit_cpipe/ [kernel PMU event]
	44	------------------------------------------
	45
	46	$# perf stat -a -e hisi_sccl3_l3c0/rd_hit_cpipe/ sleep 5
	47	$# perf stat -a -e hisi_sccl3_l3c0/config=0x02/ sleep 5
	48
	49	The current driver does not support sampling. So "perf record" is unsupported.
	50	Also attach to a task is unsupported as the events are all uncore.
	51
	52	Note: Please contact the maintainer for a complete list of events supported for
	53	the PMU devices in the SoC and its information if needed.

MAINTAINERS+7−0

@@ -6259,6 +6259,13 @@ S: Maintained · 6258 unmodified lines
6259	F: drivers/net/ethernet/hisilicon/	6259	F: drivers/net/ethernet/hisilicon/
6260	F: Documentation/devicetree/bindings/net/hisilicon*.txt	6260	F: Documentation/devicetree/bindings/net/hisilicon*.txt
6261		6261
		6262	HISILICON PMU DRIVER
		6263	M: Shaokun Zhang <zhangshaokun@hisilicon.com>
		6264	W: http://www.hisilicon.com
		6265	S: Supported
		6266	F: drivers/perf/hisilicon
		6267	F: Documentation/perf/hisi-pmu.txt
		6268
6262	HISILICON ROCE DRIVER	6269	HISILICON ROCE DRIVER
6263	M: Lijun Ou <oulijun@huawei.com>	6270	M: Lijun Ou <oulijun@huawei.com>
6264	M: Wei Hu(Xavier) <xavier.huwei@huawei.com>	6271	M: Wei Hu(Xavier) <xavier.huwei@huawei.com>

arch/arm/include/asm/arch_timer.h+1−0

@@ -107,6 +107,7 @@ static inline u32 arch_timer_get_cntkctl(void) · 106 unmodified lines
107	static inline void arch_timer_set_cntkctl(u32 cntkctl)	107	static inline void arch_timer_set_cntkctl(u32 cntkctl)
108	{	108	{
109	asm volatile("mcr p15, 0, %0, c14, c1, 0" : : "r" (cntkctl));	109	asm volatile("mcr p15, 0, %0, c14, c1, 0" : : "r" (cntkctl));
		110	isb();
110	}	111	}
111		112
112	#endif	113	#endif

arch/arm/include/asm/kvm_host.h+3−0

@@ -293,4 +293,7 @@ int kvm_arm_vcpu_arch_get_attr(struct kvm_vcpu *vcpu, · 292 unmodified lines
293	int kvm_arm_vcpu_arch_has_attr(struct kvm_vcpu *vcpu,	293	int kvm_arm_vcpu_arch_has_attr(struct kvm_vcpu *vcpu,
294	struct kvm_device_attr *attr);	294	struct kvm_device_attr *attr);
295		295
		296	/* All host FP/SIMD state is restored on guest exit, so nothing to save: */
		297	static inline void kvm_fpsimd_flush_cpu_state(void) {}
		298
296	#endif /* __ARM_KVM_HOST_H__ */	299	#endif /* __ARM_KVM_HOST_H__ */

arch/arm64/Kconfig+16−2

 	select ARCH_HAS_STRICT_KERNEL_RWX
 	select ARCH_HAS_STRICT_MODULE_RWX
 	select ARCH_HAS_TICK_BROADCAST if GENERIC_CLOCKEVENTS_BROADCAST
-	select ARCH_HAVE_NMI_SAFE_CMPXCHG if ACPI_APEI_SEA
+	select ARCH_HAVE_NMI_SAFE_CMPXCHG
 	select ARCH_INLINE_READ_LOCK if !PREEMPT
 	select ARCH_INLINE_READ_LOCK_BH if !PREEMPT
 	select ARCH_INLINE_READ_LOCK_IRQ if !PREEMPT
 	select HAVE_IRQ_TIME_ACCOUNTING
 	select HAVE_MEMBLOCK
 	select HAVE_MEMBLOCK_NODE_MAP if NUMA
-	select HAVE_NMI if ACPI_APEI_SEA
+	select HAVE_NMI
 	select HAVE_PATA_PLATFORM
 	select HAVE_PERF_EVENTS
 	select HAVE_PERF_REGS
 	select PCI_ECAM if ACPI
 	select POWER_RESET
 	select POWER_SUPPLY
+	select REFCOUNT_FULL
 	select SPARSE_IRQ
 	select SYSCTL_EXCEPTION_TRACE
 	select THREAD_INFO_IN_TASK
 menuconfig ARMV8_DEPRECATED
 	bool "Emulate deprecated/obsolete ARMv8 instructions"
 	depends on COMPAT
+	depends on SYSCTL
 	help
 	  Legacy software support may require certain instructions
 	  that have been deprecated or obsoleted in the architecture.
 endmenu
+config ARM64_SVE
+	bool "ARM Scalable Vector Extension support"
+	default y
+	help
+	  The Scalable Vector Extension (SVE) is an extension to the AArch64
+	  execution state which complements and extends the SIMD functionality
+	  of the base architecture to support much larger vectors and to enable
+	  additional vectorisation opportunities.
+	  To enable use of this extension on CPUs that implement it, say Y.
 config ARM64_MODULE_CMODEL_LARGE
 	bool
 config EFI
 	bool "UEFI runtime support"
 	depends on OF && !CPU_BIG_ENDIAN
+	depends on KERNEL_MODE_NEON
 	select LIBFDT
 	select UCS2_STRING
 	select EFI_PARAMS_FROM_FDT

arch/arm64/Makefile+8−2

 CPPFLAGS_vmlinux.lds = -DTEXT_OFFSET=$(TEXT_OFFSET)
 GZFLAGS		:=-9
-ifneq ($(CONFIG_RELOCATABLE),)
+ifeq ($(CONFIG_RELOCATABLE), y)
-LDFLAGS_vmlinux		+= -pie -shared -Bsymbolic
+# Pass --no-apply-dynamic-relocs to restore pre-binutils-2.27 behaviour
+# for relative relocs, since this leads to better Image compression
+# with the relocation offsets always being zero.
+LDFLAGS_vmlinux		+= -pie -shared -Bsymbolic \
+			$(call ld-option, --no-apply-dynamic-relocs)
 endif
 ifeq ($(CONFIG_ARM64_ERRATUM_843419),y)
 KBUILD_CFLAGS	+= $(call cc-option,-mabi=lp64)
 KBUILD_AFLAGS	+= $(call cc-option,-mabi=lp64)
+KBUILD_CFLAGS	+= $(call cc-ifversion, -ge, 0500, -DCONFIG_ARCH_SUPPORTS_INT128)
 ifeq ($(CONFIG_CPU_BIG_ENDIAN), y)
 KBUILD_CPPFLAGS	+= -mbig-endian
 CHECKFLAGS	+= -D__AARCH64EB__

arch/arm64/include/asm/arch_timer.h+1−0

@@ -144,6 +144,7 @@ static inline u32 arch_timer_get_cntkctl(void) · 143 unmodified lines
144	static inline void arch_timer_set_cntkctl(u32 cntkctl)	144	static inline void arch_timer_set_cntkctl(u32 cntkctl)
145	{	145	{
146	write_sysreg(cntkctl, cntkctl_el1);	146	write_sysreg(cntkctl, cntkctl_el1);
		147	isb();
147	}	148	}
148		149
149	static inline u64 arch_counter_get_cntpct(void)	150	static inline u64 arch_counter_get_cntpct(void)

arch/arm64/include/asm/asm-bug.h+4−4

@@ -22,10 +22,10 @@ · 21 unmodified lines
22	#define _BUGVERBOSE_LOCATION(file, line) __BUGVERBOSE_LOCATION(file, line)	22	#define _BUGVERBOSE_LOCATION(file, line) __BUGVERBOSE_LOCATION(file, line)
23	#define __BUGVERBOSE_LOCATION(file, line) \	23	#define __BUGVERBOSE_LOCATION(file, line) \
24	.pushsection .rodata.str,"aMS",@progbits,1; \	24	.pushsection .rodata.str,"aMS",@progbits,1; \
25	2: .string file; \	25	14472: .string file; \
26	.popsection; \	26	.popsection; \
27	\	27	\
28	.long 2b - 0b; \	28	.long 14472b - 14470b; \
29	.short line;	29	.short line;
30	#else	30	#else
31	#define _BUGVERBOSE_LOCATION(file, line)	31	#define _BUGVERBOSE_LOCATION(file, line)
@@ -36,11 +36,11 @@ · 4 unmodified lines
36	#define __BUG_ENTRY(flags) \	36	#define __BUG_ENTRY(flags) \
37	.pushsection __bug_table,"aw"; \	37	.pushsection __bug_table,"aw"; \
38	.align 2; \	38	.align 2; \
39	0: .long 1f - 0b; \	39	14470: .long 14471f - 14470b; \
40	_BUGVERBOSE_LOCATION(__FILE__, __LINE__) \	40	_BUGVERBOSE_LOCATION(__FILE__, __LINE__) \
41	.short flags; \	41	.short flags; \
42	.popsection; \	42	.popsection; \
43	1:	43	14471:
44	#else	44	#else
45	#define __BUG_ENTRY(flags)	45	#define __BUG_ENTRY(flags)
46	#endif	46	#endif

arch/arm64/include/asm/assembler.h+32−19

 #include <asm/asm-offsets.h>
 #include <asm/cpufeature.h>
+#include <asm/debug-monitors.h>
 #include <asm/mmu_context.h>
 #include <asm/page.h>
 #include <asm/pgtable-hwdef.h>
 #include <asm/ptrace.h>
 #include <asm/thread_info.h>
+	.macro save_and_disable_daif, flags
+	mrs	\flags, daif
+	msr	daifset, #0xf
+	.endm
+	.macro disable_daif
+	msr	daifset, #0xf
+	.endm
+	.macro enable_daif
+	msr	daifclr, #0xf
+	.endm
+	.macro	restore_daif, flags:req
+	msr	daif, \flags
+	.endm
+	/* Only on aarch64 pstate, PSR_D_BIT is different for aarch32 */
+	.macro	inherit_daif, pstate:req, tmp:req
+	and	\tmp, \pstate, #(PSR_D_BIT | PSR_A_BIT | PSR_I_BIT | PSR_F_BIT)
+	msr	daif, \tmp
+	.endm
+	/* IRQ is the lowest priority flag, unconditionally unmask the rest. */
+	.macro enable_da_f
+	msr	daifclr, #(8 | 4 | 1)
+	.endm
 /*
  * Enable and disable interrupts.
  */
 	msr	daif, \flags
 	.endm
-/*
- * Enable and disable debug exceptions.
- */
-	.macro	disable_dbg
-	msr	daifset, #8
-	.endm
 	.macro	enable_dbg
 	msr	daifclr, #8
 	.endm
 	.macro	disable_step_tsk, flgs, tmp
 	tbz	\flgs, #TIF_SINGLESTEP, 9990f
 	mrs	\tmp, mdscr_el1
-	bic	\tmp, \tmp, #1
+	bic	\tmp, \tmp, #DBG_MDSCR_SS
 	msr	mdscr_el1, \tmp
 	isb	// Synchronise with enable_dbg
 9990:
 	.endm
+	/* call with daif masked */
 	.macro	enable_step_tsk, flgs, tmp
 	tbz	\flgs, #TIF_SINGLESTEP, 9990f
-	disable_dbg
 	mrs	\tmp, mdscr_el1
-	orr	\tmp, \tmp, #1
+	orr	\tmp, \tmp, #DBG_MDSCR_SS
 	msr	mdscr_el1, \tmp
 9990:
 	.endm
-/*
- * Enable both debug exceptions and interrupts. This is likely to be
- * faster than two daifclr operations, since writes to this register
- * are self-synchronising.
- */
-	.macro	enable_dbg_and_irq
-	msr	daifclr, #(8 | 2)
-	.endm
 /*
  * SMP data memory barrier
  */

arch/arm64/include/asm/barrier.h+2−0

@@ -31,6 +31,8 @@ · 30 unmodified lines
31	#define dmb(opt) asm volatile("dmb " #opt : : : "memory")	31	#define dmb(opt) asm volatile("dmb " #opt : : : "memory")
32	#define dsb(opt) asm volatile("dsb " #opt : : : "memory")	32	#define dsb(opt) asm volatile("dsb " #opt : : : "memory")
33		33
		34	#define psb_csync() asm volatile("hint #17" : : : "memory")
		35
34	#define mb() dsb(sy)	36	#define mb() dsb(sy)
35	#define rmb() dsb(ld)	37	#define rmb() dsb(ld)
36	#define wmb() dsb(st)	38	#define wmb() dsb(st)

arch/arm64/include/asm/cpu.h+4−0

 	u64		reg_id_aa64mmfr2;
 	u64		reg_id_aa64pfr0;
 	u64		reg_id_aa64pfr1;
+	u64		reg_id_aa64zfr0;
 	u32		reg_id_dfr0;
 	u32		reg_id_isar0;
 	u32		reg_mvfr0;
 	u32		reg_mvfr1;
 	u32		reg_mvfr2;
+	/* pseudo-ZCR for recording maximum ZCR_EL1 LEN value: */
+	u64		reg_zcr;
 };
 DECLARE_PER_CPU(struct cpuinfo_arm64, cpu_data);

arch/arm64/include/asm/cpucaps.h+2−1

@@ -40,7 +40,8 @@ · 39 unmodified lines
40	#define ARM64_WORKAROUND_858921 19	40	#define ARM64_WORKAROUND_858921 19
41	#define ARM64_WORKAROUND_CAVIUM_30115 20	41	#define ARM64_WORKAROUND_CAVIUM_30115 20
42	#define ARM64_HAS_DCPOP 21	42	#define ARM64_HAS_DCPOP 21
		43	#define ARM64_SVE 22
43		44
44	#define ARM64_NCAPS 22	45	#define ARM64_NCAPS 23
45		46
46	#endif /* __ASM_CPUCAPS_H */	47	#endif /* __ASM_CPUCAPS_H */

arch/arm64/include/asm/cpufeature.h+42−0

@@ -10,7 +10,9 @@ · 9 unmodified lines
10	#define __ASM_CPUFEATURE_H	10	#define __ASM_CPUFEATURE_H
11		11
12	#include <asm/cpucaps.h>	12	#include <asm/cpucaps.h>
		13	#include <asm/fpsimd.h>
13	#include <asm/hwcap.h>	14	#include <asm/hwcap.h>
		15	#include <asm/sigcontext.h>
14	#include <asm/sysreg.h>	16	#include <asm/sysreg.h>
15		17
16	/*	18	/*
@@ -223,6 +225,13 @@ static inline bool id_aa64pfr0_32bit_el0(u64 pfr0) · 206 unmodified lines
223	return val == ID_AA64PFR0_EL0_32BIT_64BIT;	225	return val == ID_AA64PFR0_EL0_32BIT_64BIT;
224	}	226	}
225		227
		228	static inline bool id_aa64pfr0_sve(u64 pfr0)
		229	{
		230	u32 val = cpuid_feature_extract_unsigned_field(pfr0, ID_AA64PFR0_SVE_SHIFT);
		231
		232	return val > 0;
		233	}
		234
226	void __init setup_cpu_features(void);	235	void __init setup_cpu_features(void);
227		236
228	void update_cpu_capabilities(const struct arm64_cpu_capabilities *caps,	237	void update_cpu_capabilities(const struct arm64_cpu_capabilities *caps,
@@ -262,6 +271,39 @@ static inline bool system_uses_ttbr0_pan(void) · 33 unmodified lines
262	!cpus_have_const_cap(ARM64_HAS_PAN);	271	!cpus_have_const_cap(ARM64_HAS_PAN);
263	}	272	}
264		273
		274	static inline bool system_supports_sve(void)
		275	{
		276	return IS_ENABLED(CONFIG_ARM64_SVE) &&
		277	cpus_have_const_cap(ARM64_SVE);
		278	}
		279
		280	/*
		281	* Read the pseudo-ZCR used by cpufeatures to identify the supported SVE
		282	* vector length.
		283	*
		284	* Use only if SVE is present.
		285	* This function clobbers the SVE vector length.
		286	*/
		287	static inline u64 read_zcr_features(void)
		288	{
		289	u64 zcr;
		290	unsigned int vq_max;
		291
		292	/*
		293	* Set the maximum possible VL, and write zeroes to all other
		294	* bits to see if they stick.
		295	*/
		296	sve_kernel_enable(NULL);
		297	write_sysreg_s(ZCR_ELx_LEN_MASK, SYS_ZCR_EL1);
		298
		299	zcr = read_sysreg_s(SYS_ZCR_EL1);
		300	zcr &= ~(u64)ZCR_ELx_LEN_MASK; /* find sticky 1s outside LEN field */
		301	vq_max = sve_vq_from_vl(sve_get_vl());
		302	zcr \|= vq_max - 1; /* set LEN field to maximum effective value */
		303
		304	return zcr;
		305	}
		306
265	#endif /* __ASSEMBLY__ */	307	#endif /* __ASSEMBLY__ */
266		308
267	#endif	309	#endif

arch/arm64/include/asm/daifflags.h+72−0

@@ -0,0 +1,72 @@
	1	/*
	2	* Copyright (C) 2017 ARM Ltd.
	3	*
	4	* This program is free software; you can redistribute it and/or modify
	5	* it under the terms of the GNU General Public License version 2 as
	6	* published by the Free Software Foundation.
	7	*
	8	* This program is distributed in the hope that it will be useful,
	9	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	10	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	11	* GNU General Public License for more details.
	12	*
	13	* You should have received a copy of the GNU General Public License
	14	* along with this program. If not, see <http://www.gnu.org/licenses/>.
	15	*/
	16	#ifndef __ASM_DAIFFLAGS_H
	17	#define __ASM_DAIFFLAGS_H
	18
	19	#include <linux/irqflags.h>
	20
	21	#define DAIF_PROCCTX 0
	22	#define DAIF_PROCCTX_NOIRQ PSR_I_BIT
	23
	24	/* mask/save/unmask/restore all exceptions, including interrupts. */
	25	static inline void local_daif_mask(void)
	26	{
	27	asm volatile(
	28	"msr daifset, #0xf // local_daif_mask\n"
	29	:
	30	:
	31	: "memory");
	32	trace_hardirqs_off();
	33	}
	34
	35	static inline unsigned long local_daif_save(void)
	36	{
	37	unsigned long flags;
	38
	39	asm volatile(
	40	"mrs %0, daif // local_daif_save\n"
	41	: "=r" (flags)
	42	:
	43	: "memory");
	44	local_daif_mask();
	45
	46	return flags;
	47	}
	48
	49	static inline void local_daif_unmask(void)
	50	{
	51	trace_hardirqs_on();
	52	asm volatile(
	53	"msr daifclr, #0xf // local_daif_unmask"
	54	:
	55	:
	56	: "memory");
	57	}
	58
	59	static inline void local_daif_restore(unsigned long flags)
	60	{
	61	if (!arch_irqs_disabled_flags(flags))
	62	trace_hardirqs_on();
	63	asm volatile(
	64	"msr daif, %0 // local_daif_restore"
	65	:
	66	: "r" (flags)
	67	: "memory");
	68	if (arch_irqs_disabled_flags(flags))
	69	trace_hardirqs_off();
	70	}
	71
	72	#endif

arch/arm64/include/asm/elf.h+2−2

@@ -188,8 +188,8 @@ typedef compat_elf_greg_t compat_elf_gregset_t[COMPAT_ELF_NGREG]; · 187 unmodified lines
188		188
189	#define compat_start_thread compat_start_thread	189	#define compat_start_thread compat_start_thread
190	/*	190	/*
191	* Unlike the native SET_PERSONALITY macro, the compat version inherits	191	* Unlike the native SET_PERSONALITY macro, the compat version maintains
192	* READ_IMPLIES_EXEC across a fork() since this is the behaviour on	192	* READ_IMPLIES_EXEC across an execve() since this is the behaviour on
193	* arch/arm/.	193	* arch/arm/.
194	*/	194	*/
195	#define COMPAT_SET_PERSONALITY(ex) \	195	#define COMPAT_SET_PERSONALITY(ex) \

arch/arm64/include/asm/esr.h+2−1

 #define ESR_ELx_EC_HVC64	(0x16)
 #define ESR_ELx_EC_SMC64	(0x17)
 #define ESR_ELx_EC_SYS64	(0x18)
-/* Unallocated EC: 0x19 - 0x1E */
+#define ESR_ELx_EC_SVE		(0x19)
+/* Unallocated EC: 0x1A - 0x1E */
 #define ESR_ELx_EC_IMP_DEF	(0x1f)
 #define ESR_ELx_EC_IABT_LOW	(0x20)
 #define ESR_ELx_EC_IABT_CUR	(0x21)

arch/arm64/include/asm/fpsimd.h+70−1

@@ -17,9 +17,13 @@ · 16 unmodified lines
17	#define __ASM_FP_H	17	#define __ASM_FP_H
18		18
19	#include <asm/ptrace.h>	19	#include <asm/ptrace.h>
		20	#include <asm/errno.h>
20		21
21	#ifndef __ASSEMBLY__	22	#ifndef __ASSEMBLY__
22		23
		24	#include <linux/cache.h>
		25	#include <linux/stddef.h>
		26
23	/*	27	/*
24	* FP/SIMD storage area has:	28	* FP/SIMD storage area has:
25	* - FPSR and FPCR	29	* - FPSR and FPCR
@@ -35,13 +39,16 @@ struct fpsimd_state { · 9 unmodified lines
35	__uint128_t vregs[32];	39	__uint128_t vregs[32];
36	u32 fpsr;	40	u32 fpsr;
37	u32 fpcr;	41	u32 fpcr;
		42	/*
		43	* For ptrace compatibility, pad to next 128-bit
		44	* boundary here if extending this struct.
		45	*/
38	};	46	};
39	};	47	};
40	/* the id of the last cpu to have restored this state */	48	/* the id of the last cpu to have restored this state */
41	unsigned int cpu;	49	unsigned int cpu;
42	};	50	};
43		51
44
45	#if defined(__KERNEL__) && defined(CONFIG_COMPAT)	52	#if defined(__KERNEL__) && defined(CONFIG_COMPAT)
46	/* Masks for extracting the FPSR and FPCR from the FPSCR */	53	/* Masks for extracting the FPSR and FPCR from the FPSCR */
47	#define VFP_FPSCR_STAT_MASK 0xf800009f	54	#define VFP_FPSCR_STAT_MASK 0xf800009f
@@ -61,11 +68,73 @@ extern void fpsimd_load_state(struct fpsimd_state *state); · 13 unmodified lines
61	extern void fpsimd_thread_switch(struct task_struct *next);	68	extern void fpsimd_thread_switch(struct task_struct *next);
62	extern void fpsimd_flush_thread(void);	69	extern void fpsimd_flush_thread(void);
63		70
		71	extern void fpsimd_signal_preserve_current_state(void);
64	extern void fpsimd_preserve_current_state(void);	72	extern void fpsimd_preserve_current_state(void);
65	extern void fpsimd_restore_current_state(void);	73	extern void fpsimd_restore_current_state(void);
66	extern void fpsimd_update_current_state(struct fpsimd_state *state);	74	extern void fpsimd_update_current_state(struct fpsimd_state *state);
67		75
68	extern void fpsimd_flush_task_state(struct task_struct *target);	76	extern void fpsimd_flush_task_state(struct task_struct *target);
		77	extern void sve_flush_cpu_state(void);
		78
		79	/* Maximum VL that SVE VL-agnostic software can transparently support */
		80	#define SVE_VL_ARCH_MAX 0x100
		81
		82	extern void sve_save_state(void state, u32 pfpsr);
		83	extern void sve_load_state(void const state, u32 const pfpsr,
		84	unsigned long vq_minus_1);
		85	extern unsigned int sve_get_vl(void);
		86	extern int sve_kernel_enable(void *);
		87
		88	extern int __ro_after_init sve_max_vl;
		89
		90	#ifdef CONFIG_ARM64_SVE
		91
		92	extern size_t sve_state_size(struct task_struct const *task);
		93
		94	extern void sve_alloc(struct task_struct *task);
		95	extern void fpsimd_release_task(struct task_struct *task);
		96	extern void fpsimd_sync_to_sve(struct task_struct *task);
		97	extern void sve_sync_to_fpsimd(struct task_struct *task);
		98	extern void sve_sync_from_fpsimd_zeropad(struct task_struct *task);
		99
		100	extern int sve_set_vector_length(struct task_struct *task,
		101	unsigned long vl, unsigned long flags);
		102
		103	extern int sve_set_current_vl(unsigned long arg);
		104	extern int sve_get_current_vl(void);
		105
		106	/*
		107	* Probing and setup functions.
		108	* Calls to these functions must be serialised with one another.
		109	*/
		110	extern void __init sve_init_vq_map(void);
		111	extern void sve_update_vq_map(void);
		112	extern int sve_verify_vq_map(void);
		113	extern void __init sve_setup(void);
		114
		115	#else /* ! CONFIG_ARM64_SVE */
		116
		117	static inline void sve_alloc(struct task_struct *task) { }
		118	static inline void fpsimd_release_task(struct task_struct *task) { }
		119	static inline void sve_sync_to_fpsimd(struct task_struct *task) { }
		120	static inline void sve_sync_from_fpsimd_zeropad(struct task_struct *task) { }
		121
		122	static inline int sve_set_current_vl(unsigned long arg)
		123	{
		124	return -EINVAL;
		125	}
		126
		127	static inline int sve_get_current_vl(void)
		128	{
		129	return -EINVAL;
		130	}
		131
		132	static inline void sve_init_vq_map(void) { }
		133	static inline void sve_update_vq_map(void) { }
		134	static inline int sve_verify_vq_map(void) { return 0; }
		135	static inline void sve_setup(void) { }
		136
		137	#endif /* ! CONFIG_ARM64_SVE */
69		138
70	/* For use by EFI runtime services calls only */	139	/* For use by EFI runtime services calls only */
71	extern void __efi_fpsimd_begin(void);	140	extern void __efi_fpsimd_begin(void);

arch/arm64/include/asm/fpsimdmacros.h+148−0

@@ -75,3 +75,151 @@ · 74 unmodified lines
75	ldr w\tmpnr, [\state, #16 * 2 + 4]	75	ldr w\tmpnr, [\state, #16 * 2 + 4]
76	fpsimd_restore_fpcr x\tmpnr, \state	76	fpsimd_restore_fpcr x\tmpnr, \state
77	.endm	77	.endm
		78
		79	/* Sanity-check macros to help avoid encoding garbage instructions */
		80
		81	.macro _check_general_reg nr
		82	.if (\nr) < 0 \|\| (\nr) > 30
		83	.error "Bad register number \nr."
		84	.endif
		85	.endm
		86
		87	.macro _sve_check_zreg znr
		88	.if (\znr) < 0 \|\| (\znr) > 31
		89	.error "Bad Scalable Vector Extension vector register number \znr."
		90	.endif
		91	.endm
		92
		93	.macro _sve_check_preg pnr
		94	.if (\pnr) < 0 \|\| (\pnr) > 15
		95	.error "Bad Scalable Vector Extension predicate register number \pnr."
		96	.endif
		97	.endm
		98
		99	.macro _check_num n, min, max
		100	.if (\n) < (\min) \|\| (\n) > (\max)
		101	.error "Number \n out of range [\min,\max]"
		102	.endif
		103	.endm
		104
		105	/* SVE instruction encodings for non-SVE-capable assemblers */
		106
		107	/* STR (vector): STR Z\nz, [X\nxbase, #\offset, MUL VL] */
		108	.macro _sve_str_v nz, nxbase, offset=0
		109	_sve_check_zreg \nz
		110	_check_general_reg \nxbase
		111	_check_num (\offset), -0x100, 0xff
		112	.inst 0xe5804000 \
		113	\| (\nz) \
		114	\| ((\nxbase) << 5) \
		115	\| (((\offset) & 7) << 10) \
		116	\| (((\offset) & 0x1f8) << 13)
		117	.endm
		118
		119	/* LDR (vector): LDR Z\nz, [X\nxbase, #\offset, MUL VL] */
		120	.macro _sve_ldr_v nz, nxbase, offset=0
		121	_sve_check_zreg \nz
		122	_check_general_reg \nxbase
		123	_check_num (\offset), -0x100, 0xff
		124	.inst 0x85804000 \
		125	\| (\nz) \
		126	\| ((\nxbase) << 5) \
		127	\| (((\offset) & 7) << 10) \
		128	\| (((\offset) & 0x1f8) << 13)
		129	.endm
		130
		131	/* STR (predicate): STR P\np, [X\nxbase, #\offset, MUL VL] */
		132	.macro _sve_str_p np, nxbase, offset=0
		133	_sve_check_preg \np
		134	_check_general_reg \nxbase
		135	_check_num (\offset), -0x100, 0xff
		136	.inst 0xe5800000 \
		137	\| (\np) \
		138	\| ((\nxbase) << 5) \
		139	\| (((\offset) & 7) << 10) \
		140	\| (((\offset) & 0x1f8) << 13)
		141	.endm
		142
		143	/* LDR (predicate): LDR P\np, [X\nxbase, #\offset, MUL VL] */
		144	.macro _sve_ldr_p np, nxbase, offset=0
		145	_sve_check_preg \np
		146	_check_general_reg \nxbase
		147	_check_num (\offset), -0x100, 0xff
		148	.inst 0x85800000 \
		149	\| (\np) \
		150	\| ((\nxbase) << 5) \
		151	\| (((\offset) & 7) << 10) \
		152	\| (((\offset) & 0x1f8) << 13)
		153	.endm
		154
		155	/* RDVL X\nx, #\imm */
		156	.macro _sve_rdvl nx, imm
		157	_check_general_reg \nx
		158	_check_num (\imm), -0x20, 0x1f
		159	.inst 0x04bf5000 \
		160	\| (\nx) \
		161	\| (((\imm) & 0x3f) << 5)
		162	.endm
		163
		164	/* RDFFR (unpredicated): RDFFR P\np.B */
		165	.macro _sve_rdffr np
		166	_sve_check_preg \np
		167	.inst 0x2519f000 \
		168	\| (\np)
		169	.endm
		170
		171	/* WRFFR P\np.B */
		172	.macro _sve_wrffr np
		173	_sve_check_preg \np
		174	.inst 0x25289000 \
		175	\| ((\np) << 5)
		176	.endm
		177
		178	.macro __for from:req, to:req
		179	.if (\from) == (\to)
		180	_for__body \from
		181	.else
		182	__for \from, (\from) + ((\to) - (\from)) / 2
		183	__for (\from) + ((\to) - (\from)) / 2 + 1, \to
		184	.endif
		185	.endm
		186
		187	.macro _for var:req, from:req, to:req, insn:vararg
		188	.macro _for__body \var:req
		189	\insn
		190	.endm
		191
		192	__for \from, \to
		193
		194	.purgem _for__body
		195	.endm
		196
		197	.macro sve_save nxbase, xpfpsr, nxtmp
		198	_for n, 0, 31, _sve_str_v \n, \nxbase, \n - 34
		199	_for n, 0, 15, _sve_str_p \n, \nxbase, \n - 16
		200	_sve_rdffr 0
		201	_sve_str_p 0, \nxbase
		202	_sve_ldr_p 0, \nxbase, -16
		203
		204	mrs x\nxtmp, fpsr
		205	str w\nxtmp, [\xpfpsr]
		206	mrs x\nxtmp, fpcr
		207	str w\nxtmp, [\xpfpsr, #4]
		208	.endm
		209
		210	.macro sve_load nxbase, xpfpsr, xvqminus1, nxtmp
		211	mrs_s x\nxtmp, SYS_ZCR_EL1
		212	bic x\nxtmp, x\nxtmp, ZCR_ELx_LEN_MASK
		213	orr x\nxtmp, x\nxtmp, \xvqminus1
		214	msr_s SYS_ZCR_EL1, x\nxtmp // self-synchronising
		215
		216	_for n, 0, 31, _sve_ldr_v \n, \nxbase, \n - 34
		217	_sve_ldr_p 0, \nxbase
		218	_sve_wrffr 0
		219	_for n, 0, 15, _sve_ldr_p \n, \nxbase, \n - 16
		220
		221	ldr w\nxtmp, [\xpfpsr]
		222	msr fpsr, x\nxtmp
		223	ldr w\nxtmp, [\xpfpsr, #4]
		224	msr fpcr, x\nxtmp
		225	.endm

arch/arm64/include/asm/irqflags.h+13−27

 #include <asm/ptrace.h>
+/*
+ * Aarch64 has flags for masking: Debug, Asynchronous (serror), Interrupts and
+ * FIQ exceptions, in the 'daif' register. We mask and unmask them in 'dai'
+ * order:
+ * Masking debug exceptions causes all other exceptions to be masked too/
+ * Masking SError masks irq, but not debug exceptions. Masking irqs has no
+ * side effects for other flags. Keeping to this order makes it easier for
+ * entry.S to know which exceptions should be unmasked.
+ *
+ * FIQ is never expected, but we mask it when we disable debug exceptions, and
+ * unmask it at all other times.
+ */
 /*
  * CPU interrupt mask handling.
  */
 		: "memory");
 }
-#define local_fiq_enable()	asm("msr	daifclr, #1" : : : "memory")
-#define local_fiq_disable()	asm("msr	daifset, #1" : : : "memory")
-#define local_async_enable()	asm("msr	daifclr, #4" : : : "memory")
-#define local_async_disable()	asm("msr	daifset, #4" : : : "memory")
 /*
  * Save the current interrupt enable state.
  */
 {
 	return flags & PSR_I_BIT;
 }
-/*
- * save and restore debug state
- */
-#define local_dbg_save(flags)						\
-	do {								\
-		typecheck(unsigned long, flags);			\
-		asm volatile(						\
-		"mrs    %0, daif		// local_dbg_save\n"	\
-		"msr    daifset, #8"					\
-		: "=r" (flags) : : "memory");				\
-	} while (0)
-#define local_dbg_restore(flags)					\
-	do {								\
-		typecheck(unsigned long, flags);			\
-		asm volatile(						\
-		"msr    daif, %0		// local_dbg_restore\n"	\
-		: : "r" (flags) : "memory");				\
-	} while (0)
 #endif
 #endif

arch/arm64/include/asm/kvm_arm.h+4−1

 #define CPTR_EL2_TCPAC	(1 << 31)
 #define CPTR_EL2_TTA	(1 << 20)
 #define CPTR_EL2_TFP	(1 << CPTR_EL2_TFP_SHIFT)
-#define CPTR_EL2_DEFAULT	0x000033ff
+#define CPTR_EL2_TZ	(1 << 8)
+#define CPTR_EL2_RES1	0x000032ff /* known RES1 bits in CPTR_EL2 */
+#define CPTR_EL2_DEFAULT	CPTR_EL2_RES1
 /* Hyp Debug Configuration Register bits */
 #define MDCR_EL2_TPMS		(1 << 14)
 #define CPACR_EL1_FPEN		(3 << 20)
 #define CPACR_EL1_TTA		(1 << 28)
+#define CPACR_EL1_DEFAULT	(CPACR_EL1_FPEN | CPACR_EL1_ZEN_EL1EN)
 #endif /* __ARM64_KVM_ARM_H__ */

arch/arm64/include/asm/kvm_host.h+11−0

@@ -25,6 +25,7 @@ · 24 unmodified lines
25	#include <linux/types.h>	25	#include <linux/types.h>
26	#include <linux/kvm_types.h>	26	#include <linux/kvm_types.h>
27	#include <asm/cpufeature.h>	27	#include <asm/cpufeature.h>
		28	#include <asm/fpsimd.h>
28	#include <asm/kvm.h>	29	#include <asm/kvm.h>
29	#include <asm/kvm_asm.h>	30	#include <asm/kvm_asm.h>
30	#include <asm/kvm_mmio.h>	31	#include <asm/kvm_mmio.h>
@@ -384,4 +385,14 @@ static inline void __cpu_init_stage2(void) · 353 unmodified lines
384	"PARange is %d bits, unsupported configuration!", parange);	385	"PARange is %d bits, unsupported configuration!", parange);
385	}	386	}
386		387
		388	/*
		389	* All host FP/SIMD state is restored on guest exit, so nothing needs
		390	* doing here except in the SVE case:
		391	*/
		392	static inline void kvm_fpsimd_flush_cpu_state(void)
		393	{
		394	if (system_supports_sve())
		395	sve_flush_cpu_state();
		396	}
		397
387	#endif /* __ARM64_KVM_HOST_H__ */	398	#endif /* __ARM64_KVM_HOST_H__ */

arch/arm64/include/asm/memory.h+0−15

@@ -61,8 +61,6 @@ · 60 unmodified lines
61	* KIMAGE_VADDR - the virtual address of the start of the kernel image	61	* KIMAGE_VADDR - the virtual address of the start of the kernel image
62	* VA_BITS - the maximum number of bits for virtual addresses.	62	* VA_BITS - the maximum number of bits for virtual addresses.
63	* VA_START - the first kernel virtual address.	63	* VA_START - the first kernel virtual address.
64	* TASK_SIZE - the maximum size of a user space task.
65	* TASK_UNMAPPED_BASE - the lower boundary of the mmap VM area.
66	*/	64	*/
67	#define VA_BITS (CONFIG_ARM64_VA_BITS)	65	#define VA_BITS (CONFIG_ARM64_VA_BITS)
68	#define VA_START (UL(0xffffffffffffffff) - \	66	#define VA_START (UL(0xffffffffffffffff) - \
@@ -77,19 +75,6 @@ · 8 unmodified lines
77	#define PCI_IO_END (VMEMMAP_START - SZ_2M)	75	#define PCI_IO_END (VMEMMAP_START - SZ_2M)
78	#define PCI_IO_START (PCI_IO_END - PCI_IO_SIZE)	76	#define PCI_IO_START (PCI_IO_END - PCI_IO_SIZE)
79	#define FIXADDR_TOP (PCI_IO_START - SZ_2M)	77	#define FIXADDR_TOP (PCI_IO_START - SZ_2M)
80	#define TASK_SIZE_64 (UL(1) << VA_BITS)
81
82	#ifdef CONFIG_COMPAT
83	#define TASK_SIZE_32 UL(0x100000000)
84	#define TASK_SIZE (test_thread_flag(TIF_32BIT) ? \
85	TASK_SIZE_32 : TASK_SIZE_64)
86	#define TASK_SIZE_OF(tsk) (test_tsk_thread_flag(tsk, TIF_32BIT) ? \
87	TASK_SIZE_32 : TASK_SIZE_64)
88	#else
89	#define TASK_SIZE TASK_SIZE_64
90	#endif /* CONFIG_COMPAT */
91
92	#define TASK_UNMAPPED_BASE (PAGE_ALIGN(TASK_SIZE / 4))
93		78
94	#define KERNEL_START _text	79	#define KERNEL_START _text
95	#define KERNEL_END _end	80	#define KERNEL_END _end

arch/arm64/include/asm/pgtable.h+14−0

@@ -98,6 +98,8 @@ extern unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)]; · 97 unmodified lines
98	((pte_val(pte) & (PTE_VALID \| PTE_USER \| PTE_UXN)) == (PTE_VALID \| PTE_UXN))	98	((pte_val(pte) & (PTE_VALID \| PTE_USER \| PTE_UXN)) == (PTE_VALID \| PTE_UXN))
99	#define pte_valid_young(pte) \	99	#define pte_valid_young(pte) \
100	((pte_val(pte) & (PTE_VALID \| PTE_AF)) == (PTE_VALID \| PTE_AF))	100	((pte_val(pte) & (PTE_VALID \| PTE_AF)) == (PTE_VALID \| PTE_AF))
		101	#define pte_valid_user(pte) \
		102	((pte_val(pte) & (PTE_VALID \| PTE_USER)) == (PTE_VALID \| PTE_USER))
101		103
102	/*	104	/*
103	* Could the pte be present in the TLB? We must check mm_tlb_flush_pending	105	* Could the pte be present in the TLB? We must check mm_tlb_flush_pending
@@ -107,6 +109,18 @@ extern unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)]; · 3 unmodified lines
107	#define pte_accessible(mm, pte) \	109	#define pte_accessible(mm, pte) \
108	(mm_tlb_flush_pending(mm) ? pte_present(pte) : pte_valid_young(pte))	110	(mm_tlb_flush_pending(mm) ? pte_present(pte) : pte_valid_young(pte))
109		111
		112	/*
		113	* p??_access_permitted() is true for valid user mappings (subject to the
		114	* write permission check) other than user execute-only which do not have the
		115	* PTE_USER bit set. PROT_NONE mappings do not have the PTE_VALID bit set.
		116	*/
		117	#define pte_access_permitted(pte, write) \
		118	(pte_valid_user(pte) && (!(write) \|\| pte_write(pte)))
		119	#define pmd_access_permitted(pmd, write) \
		120	(pte_access_permitted(pmd_pte(pmd), (write)))
		121	#define pud_access_permitted(pud, write) \
		122	(pte_access_permitted(pud_pte(pud), (write)))
		123
110	static inline pte_t clear_pte_bit(pte_t pte, pgprot_t prot)	124	static inline pte_t clear_pte_bit(pte_t pte, pgprot_t prot)
111	{	125	{
112	pte_val(pte) &= ~pgprot_val(prot);	126	pte_val(pte) &= ~pgprot_val(prot);

arch/arm64/include/asm/processor.h+28−0

@@ -19,6 +19,10 @@ · 18 unmodified lines
19	#ifndef __ASM_PROCESSOR_H	19	#ifndef __ASM_PROCESSOR_H
20	#define __ASM_PROCESSOR_H	20	#define __ASM_PROCESSOR_H
21		21
		22	#define TASK_SIZE_64 (UL(1) << VA_BITS)
		23
		24	#ifndef __ASSEMBLY__
		25
22	/*	26	/*
23	* Default implementation of macro that returns current	27	* Default implementation of macro that returns current
24	* instruction pointer ("program counter").	28	* instruction pointer ("program counter").
@@ -37,6 +41,22 @@ · 12 unmodified lines
37	#include <asm/ptrace.h>	41	#include <asm/ptrace.h>
38	#include <asm/types.h>	42	#include <asm/types.h>
39		43
		44	/*
		45	* TASK_SIZE - the maximum size of a user space task.
		46	* TASK_UNMAPPED_BASE - the lower boundary of the mmap VM area.
		47	*/
		48	#ifdef CONFIG_COMPAT
		49	#define TASK_SIZE_32 UL(0x100000000)
		50	#define TASK_SIZE (test_thread_flag(TIF_32BIT) ? \
		51	TASK_SIZE_32 : TASK_SIZE_64)
		52	#define TASK_SIZE_OF(tsk) (test_tsk_thread_flag(tsk, TIF_32BIT) ? \
		53	TASK_SIZE_32 : TASK_SIZE_64)
		54	#else
		55	#define TASK_SIZE TASK_SIZE_64
		56	#endif /* CONFIG_COMPAT */
		57
		58	#define TASK_UNMAPPED_BASE (PAGE_ALIGN(TASK_SIZE / 4))
		59
40	#define STACK_TOP_MAX TASK_SIZE_64	60	#define STACK_TOP_MAX TASK_SIZE_64
41	#ifdef CONFIG_COMPAT	61	#ifdef CONFIG_COMPAT
42	#define AARCH32_VECTORS_BASE 0xffff0000	62	#define AARCH32_VECTORS_BASE 0xffff0000
@@ -85,6 +105,9 @@ struct thread_struct { · 42 unmodified lines
85	unsigned long tp2_value;	105	unsigned long tp2_value;
86	#endif	106	#endif
87	struct fpsimd_state fpsimd_state;	107	struct fpsimd_state fpsimd_state;
		108	void sve_state; / SVE registers, if any */
		109	unsigned int sve_vl; /* SVE vector length */
		110	unsigned int sve_vl_onexec; /* SVE vl after next exec */
88	unsigned long fault_address; /* fault info */	111	unsigned long fault_address; /* fault info */
89	unsigned long fault_code; /* ESR_EL1 value */	112	unsigned long fault_code; /* ESR_EL1 value */
90	struct debug_info debug; /* debugging */	113	struct debug_info debug; /* debugging */
@@ -194,4 +217,9 @@ static inline void spin_lock_prefetch(const void *ptr) · 103 unmodified lines
194	int cpu_enable_pan(void *__unused);	217	int cpu_enable_pan(void *__unused);
195	int cpu_enable_cache_maint_trap(void *__unused);	218	int cpu_enable_cache_maint_trap(void *__unused);
196		219
		220	/* Userspace interface for PR_SVE_{SET,GET}_VL prctl()s: */
		221	#define SVE_SET_VL(arg) sve_set_current_vl(arg)
		222	#define SVE_GET_VL() sve_get_current_vl()
		223
		224	#endif /* __ASSEMBLY__ */
197	#endif /* __ASM_PROCESSOR_H */	225	#endif /* __ASM_PROCESSOR_H */

arch/arm64/include/asm/sysreg.h+121−0

@@ -145,10 +145,14 @@ · 144 unmodified lines
145		145
146	#define SYS_ID_AA64PFR0_EL1 sys_reg(3, 0, 0, 4, 0)	146	#define SYS_ID_AA64PFR0_EL1 sys_reg(3, 0, 0, 4, 0)
147	#define SYS_ID_AA64PFR1_EL1 sys_reg(3, 0, 0, 4, 1)	147	#define SYS_ID_AA64PFR1_EL1 sys_reg(3, 0, 0, 4, 1)
		148	#define SYS_ID_AA64ZFR0_EL1 sys_reg(3, 0, 0, 4, 4)
148		149
149	#define SYS_ID_AA64DFR0_EL1 sys_reg(3, 0, 0, 5, 0)	150	#define SYS_ID_AA64DFR0_EL1 sys_reg(3, 0, 0, 5, 0)
150	#define SYS_ID_AA64DFR1_EL1 sys_reg(3, 0, 0, 5, 1)	151	#define SYS_ID_AA64DFR1_EL1 sys_reg(3, 0, 0, 5, 1)
151		152
		153	#define SYS_ID_AA64AFR0_EL1 sys_reg(3, 0, 0, 5, 4)
		154	#define SYS_ID_AA64AFR1_EL1 sys_reg(3, 0, 0, 5, 5)
		155
152	#define SYS_ID_AA64ISAR0_EL1 sys_reg(3, 0, 0, 6, 0)	156	#define SYS_ID_AA64ISAR0_EL1 sys_reg(3, 0, 0, 6, 0)
153	#define SYS_ID_AA64ISAR1_EL1 sys_reg(3, 0, 0, 6, 1)	157	#define SYS_ID_AA64ISAR1_EL1 sys_reg(3, 0, 0, 6, 1)
154		158
@@ -160,6 +164,8 @@ · 5 unmodified lines
160	#define SYS_ACTLR_EL1 sys_reg(3, 0, 1, 0, 1)	164	#define SYS_ACTLR_EL1 sys_reg(3, 0, 1, 0, 1)
161	#define SYS_CPACR_EL1 sys_reg(3, 0, 1, 0, 2)	165	#define SYS_CPACR_EL1 sys_reg(3, 0, 1, 0, 2)
162		166
		167	#define SYS_ZCR_EL1 sys_reg(3, 0, 1, 2, 0)
		168
163	#define SYS_TTBR0_EL1 sys_reg(3, 0, 2, 0, 0)	169	#define SYS_TTBR0_EL1 sys_reg(3, 0, 2, 0, 0)
164	#define SYS_TTBR1_EL1 sys_reg(3, 0, 2, 0, 1)	170	#define SYS_TTBR1_EL1 sys_reg(3, 0, 2, 0, 1)
165	#define SYS_TCR_EL1 sys_reg(3, 0, 2, 0, 2)	171	#define SYS_TCR_EL1 sys_reg(3, 0, 2, 0, 2)
@@ -172,6 +178,99 @@ · 6 unmodified lines
172	#define SYS_FAR_EL1 sys_reg(3, 0, 6, 0, 0)	178	#define SYS_FAR_EL1 sys_reg(3, 0, 6, 0, 0)
173	#define SYS_PAR_EL1 sys_reg(3, 0, 7, 4, 0)	179	#define SYS_PAR_EL1 sys_reg(3, 0, 7, 4, 0)
174		180
		181	/* Statistical Profiling Extension */
		182	/* ID registers */
		183	#define SYS_PMSIDR_EL1 sys_reg(3, 0, 9, 9, 7)
		184	#define SYS_PMSIDR_EL1_FE_SHIFT 0
		185	#define SYS_PMSIDR_EL1_FT_SHIFT 1
		186	#define SYS_PMSIDR_EL1_FL_SHIFT 2
		187	#define SYS_PMSIDR_EL1_ARCHINST_SHIFT 3
		188	#define SYS_PMSIDR_EL1_LDS_SHIFT 4
		189	#define SYS_PMSIDR_EL1_ERND_SHIFT 5
		190	#define SYS_PMSIDR_EL1_INTERVAL_SHIFT 8
		191	#define SYS_PMSIDR_EL1_INTERVAL_MASK 0xfUL
		192	#define SYS_PMSIDR_EL1_MAXSIZE_SHIFT 12
		193	#define SYS_PMSIDR_EL1_MAXSIZE_MASK 0xfUL
		194	#define SYS_PMSIDR_EL1_COUNTSIZE_SHIFT 16
		195	#define SYS_PMSIDR_EL1_COUNTSIZE_MASK 0xfUL
		196
		197	#define SYS_PMBIDR_EL1 sys_reg(3, 0, 9, 10, 7)
		198	#define SYS_PMBIDR_EL1_ALIGN_SHIFT 0
		199	#define SYS_PMBIDR_EL1_ALIGN_MASK 0xfU
		200	#define SYS_PMBIDR_EL1_P_SHIFT 4
		201	#define SYS_PMBIDR_EL1_F_SHIFT 5
		202
		203	/* Sampling controls */
		204	#define SYS_PMSCR_EL1 sys_reg(3, 0, 9, 9, 0)
		205	#define SYS_PMSCR_EL1_E0SPE_SHIFT 0
		206	#define SYS_PMSCR_EL1_E1SPE_SHIFT 1
		207	#define SYS_PMSCR_EL1_CX_SHIFT 3
		208	#define SYS_PMSCR_EL1_PA_SHIFT 4
		209	#define SYS_PMSCR_EL1_TS_SHIFT 5
		210	#define SYS_PMSCR_EL1_PCT_SHIFT 6
		211
		212	#define SYS_PMSCR_EL2 sys_reg(3, 4, 9, 9, 0)
		213	#define SYS_PMSCR_EL2_E0HSPE_SHIFT 0
		214	#define SYS_PMSCR_EL2_E2SPE_SHIFT 1
		215	#define SYS_PMSCR_EL2_CX_SHIFT 3
		216	#define SYS_PMSCR_EL2_PA_SHIFT 4
		217	#define SYS_PMSCR_EL2_TS_SHIFT 5
		218	#define SYS_PMSCR_EL2_PCT_SHIFT 6
		219
		220	#define SYS_PMSICR_EL1 sys_reg(3, 0, 9, 9, 2)
		221
		222	#define SYS_PMSIRR_EL1 sys_reg(3, 0, 9, 9, 3)
		223	#define SYS_PMSIRR_EL1_RND_SHIFT 0
		224	#define SYS_PMSIRR_EL1_INTERVAL_SHIFT 8
		225	#define SYS_PMSIRR_EL1_INTERVAL_MASK 0xffffffUL
		226
		227	/* Filtering controls */
		228	#define SYS_PMSFCR_EL1 sys_reg(3, 0, 9, 9, 4)
		229	#define SYS_PMSFCR_EL1_FE_SHIFT 0
		230	#define SYS_PMSFCR_EL1_FT_SHIFT 1
		231	#define SYS_PMSFCR_EL1_FL_SHIFT 2
		232	#define SYS_PMSFCR_EL1_B_SHIFT 16
		233	#define SYS_PMSFCR_EL1_LD_SHIFT 17
		234	#define SYS_PMSFCR_EL1_ST_SHIFT 18
		235
		236	#define SYS_PMSEVFR_EL1 sys_reg(3, 0, 9, 9, 5)
		237	#define SYS_PMSEVFR_EL1_RES0 0x0000ffff00ff0f55UL
		238
		239	#define SYS_PMSLATFR_EL1 sys_reg(3, 0, 9, 9, 6)
		240	#define SYS_PMSLATFR_EL1_MINLAT_SHIFT 0
		241
		242	/* Buffer controls */
		243	#define SYS_PMBLIMITR_EL1 sys_reg(3, 0, 9, 10, 0)
		244	#define SYS_PMBLIMITR_EL1_E_SHIFT 0
		245	#define SYS_PMBLIMITR_EL1_FM_SHIFT 1
		246	#define SYS_PMBLIMITR_EL1_FM_MASK 0x3UL
		247	#define SYS_PMBLIMITR_EL1_FM_STOP_IRQ (0 << SYS_PMBLIMITR_EL1_FM_SHIFT)
		248
		249	#define SYS_PMBPTR_EL1 sys_reg(3, 0, 9, 10, 1)
		250
		251	/* Buffer error reporting */
		252	#define SYS_PMBSR_EL1 sys_reg(3, 0, 9, 10, 3)
		253	#define SYS_PMBSR_EL1_COLL_SHIFT 16
		254	#define SYS_PMBSR_EL1_S_SHIFT 17
		255	#define SYS_PMBSR_EL1_EA_SHIFT 18
		256	#define SYS_PMBSR_EL1_DL_SHIFT 19
		257	#define SYS_PMBSR_EL1_EC_SHIFT 26
		258	#define SYS_PMBSR_EL1_EC_MASK 0x3fUL
		259
		260	#define SYS_PMBSR_EL1_EC_BUF (0x0UL << SYS_PMBSR_EL1_EC_SHIFT)
		261	#define SYS_PMBSR_EL1_EC_FAULT_S1 (0x24UL << SYS_PMBSR_EL1_EC_SHIFT)
		262	#define SYS_PMBSR_EL1_EC_FAULT_S2 (0x25UL << SYS_PMBSR_EL1_EC_SHIFT)
		263
		264	#define SYS_PMBSR_EL1_FAULT_FSC_SHIFT 0
		265	#define SYS_PMBSR_EL1_FAULT_FSC_MASK 0x3fUL
		266
		267	#define SYS_PMBSR_EL1_BUF_BSC_SHIFT 0
		268	#define SYS_PMBSR_EL1_BUF_BSC_MASK 0x3fUL
		269
		270	#define SYS_PMBSR_EL1_BUF_BSC_FULL (0x1UL << SYS_PMBSR_EL1_BUF_BSC_SHIFT)
		271
		272	/* End of Statistical Profiling Extension */
		273
175	#define SYS_PMINTENSET_EL1 sys_reg(3, 0, 9, 14, 1)	274	#define SYS_PMINTENSET_EL1 sys_reg(3, 0, 9, 14, 1)
176	#define SYS_PMINTENCLR_EL1 sys_reg(3, 0, 9, 14, 2)	275	#define SYS_PMINTENCLR_EL1 sys_reg(3, 0, 9, 14, 2)
177		276
@@ -250,6 +349,8 @@ · 72 unmodified lines
250		349
251	#define SYS_PMCCFILTR_EL0 sys_reg (3, 3, 14, 15, 7)	350	#define SYS_PMCCFILTR_EL0 sys_reg (3, 3, 14, 15, 7)
252		351
		352	#define SYS_ZCR_EL2 sys_reg(3, 4, 1, 2, 0)
		353
253	#define SYS_DACR32_EL2 sys_reg(3, 4, 3, 0, 0)	354	#define SYS_DACR32_EL2 sys_reg(3, 4, 3, 0, 0)
254	#define SYS_IFSR32_EL2 sys_reg(3, 4, 5, 0, 1)	355	#define SYS_IFSR32_EL2 sys_reg(3, 4, 5, 0, 1)
255	#define SYS_FPEXC32_EL2 sys_reg(3, 4, 5, 3, 0)	356	#define SYS_FPEXC32_EL2 sys_reg(3, 4, 5, 3, 0)
@@ -318,6 +419,10 @@ · 62 unmodified lines
318	#define SCTLR_EL1_CP15BEN (1 << 5)	419	#define SCTLR_EL1_CP15BEN (1 << 5)
319		420
320	/* id_aa64isar0 */	421	/* id_aa64isar0 */
		422	#define ID_AA64ISAR0_DP_SHIFT 44
		423	#define ID_AA64ISAR0_SM4_SHIFT 40
		424	#define ID_AA64ISAR0_SM3_SHIFT 36
		425	#define ID_AA64ISAR0_SHA3_SHIFT 32
321	#define ID_AA64ISAR0_RDM_SHIFT 28	426	#define ID_AA64ISAR0_RDM_SHIFT 28
322	#define ID_AA64ISAR0_ATOMICS_SHIFT 20	427	#define ID_AA64ISAR0_ATOMICS_SHIFT 20
323	#define ID_AA64ISAR0_CRC32_SHIFT 16	428	#define ID_AA64ISAR0_CRC32_SHIFT 16
@@ -332,6 +437,7 @@ · 8 unmodified lines
332	#define ID_AA64ISAR1_DPB_SHIFT 0	437	#define ID_AA64ISAR1_DPB_SHIFT 0
333		438
334	/* id_aa64pfr0 */	439	/* id_aa64pfr0 */
		440	#define ID_AA64PFR0_SVE_SHIFT 32
335	#define ID_AA64PFR0_GIC_SHIFT 24	441	#define ID_AA64PFR0_GIC_SHIFT 24
336	#define ID_AA64PFR0_ASIMD_SHIFT 20	442	#define ID_AA64PFR0_ASIMD_SHIFT 20
337	#define ID_AA64PFR0_FP_SHIFT 16	443	#define ID_AA64PFR0_FP_SHIFT 16
@@ -340,6 +446,7 @@ · 2 unmodified lines
340	#define ID_AA64PFR0_EL1_SHIFT 4	446	#define ID_AA64PFR0_EL1_SHIFT 4
341	#define ID_AA64PFR0_EL0_SHIFT 0	447	#define ID_AA64PFR0_EL0_SHIFT 0
342		448
		449	#define ID_AA64PFR0_SVE 0x1
343	#define ID_AA64PFR0_FP_NI 0xf	450	#define ID_AA64PFR0_FP_NI 0xf
344	#define ID_AA64PFR0_FP_SUPPORTED 0x0	451	#define ID_AA64PFR0_FP_SUPPORTED 0x0
345	#define ID_AA64PFR0_ASIMD_NI 0xf	452	#define ID_AA64PFR0_ASIMD_NI 0xf
@@ -441,6 +548,20 @@ · 95 unmodified lines
441	#endif	548	#endif
442		549
443		550
		551	/*
		552	* The ZCR_ELx_LEN_* definitions intentionally include bits [8:4] which
		553	* are reserved by the SVE architecture for future expansion of the LEN
		554	* field, with compatible semantics.
		555	*/
		556	#define ZCR_ELx_LEN_SHIFT 0
		557	#define ZCR_ELx_LEN_SIZE 9
		558	#define ZCR_ELx_LEN_MASK 0x1ff
		559
		560	#define CPACR_EL1_ZEN_EL1EN (1 << 16) /* enable EL1 access */
		561	#define CPACR_EL1_ZEN_EL0EN (1 << 17) /* enable EL0 access, if EL1EN set */
		562	#define CPACR_EL1_ZEN (CPACR_EL1_ZEN_EL1EN \| CPACR_EL1_ZEN_EL0EN)
		563
		564
444	/* Safe value for MPIDR_EL1: Bit31:RES1, Bit30:U:0, Bit24:MT:0 */	565	/* Safe value for MPIDR_EL1: Bit31:RES1, Bit30:U:0, Bit24:MT:0 */
445	#define SYS_MPIDR_SAFE_VAL (1UL << 31)	566	#define SYS_MPIDR_SAFE_VAL (1UL << 31)
446		567

arch/arm64/include/asm/thread_info.h+5−0

 void arch_setup_new_exec(void);
 #define arch_setup_new_exec     arch_setup_new_exec
+void arch_release_task_struct(struct task_struct *tsk);
 #endif
 /*
 #define TIF_RESTORE_SIGMASK	20
 #define TIF_SINGLESTEP		21
 #define TIF_32BIT		22	/* 32bit process */
+#define TIF_SVE			23	/* Scalable Vector Extension in use */
+#define TIF_SVE_VL_INHERIT	24	/* Inherit sve_vl_onexec across exec */
 #define _TIF_SIGPENDING		(1 << TIF_SIGPENDING)
 #define _TIF_NEED_RESCHED	(1 << TIF_NEED_RESCHED)
 #define _TIF_UPROBE		(1 << TIF_UPROBE)
 #define _TIF_FSCHECK		(1 << TIF_FSCHECK)
 #define _TIF_32BIT		(1 << TIF_32BIT)
+#define _TIF_SVE		(1 << TIF_SVE)
 #define _TIF_WORK_MASK		(_TIF_NEED_RESCHED | _TIF_SIGPENDING | \
 				 _TIF_NOTIFY_RESUME | _TIF_FOREIGN_FPSTATE | \

arch/arm64/include/asm/traps.h+8−0

 void register_undef_hook(struct undef_hook *hook);
 void unregister_undef_hook(struct undef_hook *hook);
+void force_signal_inject(int signal, int code, struct pt_regs *regs,
+			 unsigned long address);
 void arm64_notify_segfault(struct pt_regs *regs, unsigned long addr);
+/*
+ * Move regs->pc to next instruction and do necessary setup before it
+ * is executed.
+ */
+void arm64_skip_faulting_instruction(struct pt_regs *regs, unsigned long size);
 static inline int __in_irqentry_text(unsigned long ptr)
 {
 	return ptr >= (unsigned long)&__irqentry_text_start &&

arch/arm64/include/uapi/asm/hwcap.h+6−0

@@ -37,5 +37,11 @@ · 36 unmodified lines
37	#define HWCAP_FCMA (1 << 14)	37	#define HWCAP_FCMA (1 << 14)
38	#define HWCAP_LRCPC (1 << 15)	38	#define HWCAP_LRCPC (1 << 15)
39	#define HWCAP_DCPOP (1 << 16)	39	#define HWCAP_DCPOP (1 << 16)
		40	#define HWCAP_SHA3 (1 << 17)
		41	#define HWCAP_SM3 (1 << 18)
		42	#define HWCAP_SM4 (1 << 19)
		43	#define HWCAP_ASIMDDP (1 << 20)
		44	#define HWCAP_SHA512 (1 << 21)
		45	#define HWCAP_SVE (1 << 22)
40		46
41	#endif /* _UAPI__ASM_HWCAP_H */	47	#endif /* _UAPI__ASM_HWCAP_H */

arch/arm64/include/uapi/asm/ptrace.h+138−1

@@ -23,6 +23,7 @@ · 22 unmodified lines
23	#include <linux/types.h>	23	#include <linux/types.h>
24		24
25	#include <asm/hwcap.h>	25	#include <asm/hwcap.h>
		26	#include <asm/sigcontext.h>
26		27
27		28
28	/*	29	/*
@@ -47,7 +48,6 @@ · 18 unmodified lines
47	#define PSR_D_BIT 0x00000200	48	#define PSR_D_BIT 0x00000200
48	#define PSR_PAN_BIT 0x00400000	49	#define PSR_PAN_BIT 0x00400000
49	#define PSR_UAO_BIT 0x00800000	50	#define PSR_UAO_BIT 0x00800000
50	#define PSR_Q_BIT 0x08000000
51	#define PSR_V_BIT 0x10000000	51	#define PSR_V_BIT 0x10000000
52	#define PSR_C_BIT 0x20000000	52	#define PSR_C_BIT 0x20000000
53	#define PSR_Z_BIT 0x40000000	53	#define PSR_Z_BIT 0x40000000
@@ -64,6 +64,8 @@ · 10 unmodified lines
64		64
65	#ifndef __ASSEMBLY__	65	#ifndef __ASSEMBLY__
66		66
		67	#include <linux/prctl.h>
		68
67	/*	69	/*
68	* User structures for general purpose, floating point and debug registers.	70	* User structures for general purpose, floating point and debug registers.
69	*/	71	*/
@@ -91,6 +93,141 @@ struct user_hwdebug_state { · 21 unmodified lines
91	} dbg_regs[16];	93	} dbg_regs[16];
92	};	94	};
93		95
		96	/* SVE/FP/SIMD state (NT_ARM_SVE) */
		97
		98	struct user_sve_header {
		99	__u32 size; /* total meaningful regset content in bytes */
		100	__u32 max_size; /* maxmium possible size for this thread */
		101	__u16 vl; /* current vector length */
		102	__u16 max_vl; /* maximum possible vector length */
		103	__u16 flags;
		104	__u16 __reserved;
		105	};
		106
		107	/* Definitions for user_sve_header.flags: */
		108	#define SVE_PT_REGS_MASK (1 << 0)
		109
		110	#define SVE_PT_REGS_FPSIMD 0
		111	#define SVE_PT_REGS_SVE SVE_PT_REGS_MASK
		112
		113	/*
		114	* Common SVE_PT_* flags:
		115	* These must be kept in sync with prctl interface in <linux/ptrace.h>
		116	*/
		117	#define SVE_PT_VL_INHERIT (PR_SVE_VL_INHERIT >> 16)
		118	#define SVE_PT_VL_ONEXEC (PR_SVE_SET_VL_ONEXEC >> 16)
		119
		120
		121	/*
		122	* The remainder of the SVE state follows struct user_sve_header. The
		123	* total size of the SVE state (including header) depends on the
		124	* metadata in the header: SVE_PT_SIZE(vq, flags) gives the total size
		125	* of the state in bytes, including the header.
		126	*
		127	* Refer to <asm/sigcontext.h> for details of how to pass the correct
		128	* "vq" argument to these macros.
		129	*/
		130
		131	/* Offset from the start of struct user_sve_header to the register data */
		132	#define SVE_PT_REGS_OFFSET \
		133	((sizeof(struct sve_context) + (SVE_VQ_BYTES - 1)) \
		134	/ SVE_VQ_BYTES * SVE_VQ_BYTES)
		135
		136	/*
		137	* The register data content and layout depends on the value of the
		138	* flags field.
		139	*/
		140
		141	/*
		142	* (flags & SVE_PT_REGS_MASK) == SVE_PT_REGS_FPSIMD case:
		143	*
		144	* The payload starts at offset SVE_PT_FPSIMD_OFFSET, and is of type
		145	* struct user_fpsimd_state. Additional data might be appended in the
		146	* future: use SVE_PT_FPSIMD_SIZE(vq, flags) to compute the total size.
		147	* SVE_PT_FPSIMD_SIZE(vq, flags) will never be less than
		148	* sizeof(struct user_fpsimd_state).
		149	*/
		150
		151	#define SVE_PT_FPSIMD_OFFSET SVE_PT_REGS_OFFSET
		152
		153	#define SVE_PT_FPSIMD_SIZE(vq, flags) (sizeof(struct user_fpsimd_state))
		154
		155	/*
		156	* (flags & SVE_PT_REGS_MASK) == SVE_PT_REGS_SVE case:
		157	*
		158	* The payload starts at offset SVE_PT_SVE_OFFSET, and is of size
		159	* SVE_PT_SVE_SIZE(vq, flags).
		160	*
		161	* Additional macros describe the contents and layout of the payload.
		162	* For each, SVE_PT_SVE_x_OFFSET(args) is the start offset relative to
		163	* the start of struct user_sve_header, and SVE_PT_SVE_x_SIZE(args) is
		164	* the size in bytes:
		165	*
		166	* x type description
		167	* - ---- -----------
		168	* ZREGS \
		169	* ZREG \|
		170	* PREGS \| refer to <asm/sigcontext.h>
		171	* PREG \|
		172	* FFR /
		173	*
		174	* FPSR uint32_t FPSR
		175	* FPCR uint32_t FPCR
		176	*
		177	* Additional data might be appended in the future.
		178	*/
		179
		180	#define SVE_PT_SVE_ZREG_SIZE(vq) SVE_SIG_ZREG_SIZE(vq)
		181	#define SVE_PT_SVE_PREG_SIZE(vq) SVE_SIG_PREG_SIZE(vq)
		182	#define SVE_PT_SVE_FFR_SIZE(vq) SVE_SIG_FFR_SIZE(vq)
		183	#define SVE_PT_SVE_FPSR_SIZE sizeof(__u32)
		184	#define SVE_PT_SVE_FPCR_SIZE sizeof(__u32)
		185
		186	#define __SVE_SIG_TO_PT(offset) \
		187	((offset) - SVE_SIG_REGS_OFFSET + SVE_PT_REGS_OFFSET)
		188
		189	#define SVE_PT_SVE_OFFSET SVE_PT_REGS_OFFSET
		190
		191	#define SVE_PT_SVE_ZREGS_OFFSET \
		192	__SVE_SIG_TO_PT(SVE_SIG_ZREGS_OFFSET)
		193	#define SVE_PT_SVE_ZREG_OFFSET(vq, n) \
		194	__SVE_SIG_TO_PT(SVE_SIG_ZREG_OFFSET(vq, n))
		195	#define SVE_PT_SVE_ZREGS_SIZE(vq) \
		196	(SVE_PT_SVE_ZREG_OFFSET(vq, SVE_NUM_ZREGS) - SVE_PT_SVE_ZREGS_OFFSET)
		197
		198	#define SVE_PT_SVE_PREGS_OFFSET(vq) \
		199	__SVE_SIG_TO_PT(SVE_SIG_PREGS_OFFSET(vq))
		200	#define SVE_PT_SVE_PREG_OFFSET(vq, n) \
		201	__SVE_SIG_TO_PT(SVE_SIG_PREG_OFFSET(vq, n))
		202	#define SVE_PT_SVE_PREGS_SIZE(vq) \
		203	(SVE_PT_SVE_PREG_OFFSET(vq, SVE_NUM_PREGS) - \
		204	SVE_PT_SVE_PREGS_OFFSET(vq))
		205
		206	#define SVE_PT_SVE_FFR_OFFSET(vq) \
		207	__SVE_SIG_TO_PT(SVE_SIG_FFR_OFFSET(vq))
		208
		209	#define SVE_PT_SVE_FPSR_OFFSET(vq) \
		210	((SVE_PT_SVE_FFR_OFFSET(vq) + SVE_PT_SVE_FFR_SIZE(vq) + \
		211	(SVE_VQ_BYTES - 1)) \
		212	/ SVE_VQ_BYTES * SVE_VQ_BYTES)
		213	#define SVE_PT_SVE_FPCR_OFFSET(vq) \
		214	(SVE_PT_SVE_FPSR_OFFSET(vq) + SVE_PT_SVE_FPSR_SIZE)
		215
		216	/*
		217	* Any future extension appended after FPCR must be aligned to the next
		218	* 128-bit boundary.
		219	*/
		220
		221	#define SVE_PT_SVE_SIZE(vq, flags) \
		222	((SVE_PT_SVE_FPCR_OFFSET(vq) + SVE_PT_SVE_FPCR_SIZE \
		223	- SVE_PT_SVE_OFFSET + (SVE_VQ_BYTES - 1)) \
		224	/ SVE_VQ_BYTES * SVE_VQ_BYTES)
		225
		226	#define SVE_PT_SIZE(vq, flags) \
		227	(((flags) & SVE_PT_REGS_MASK) == SVE_PT_REGS_SVE ? \
		228	SVE_PT_SVE_OFFSET + SVE_PT_SVE_SIZE(vq, flags) \
		229	: SVE_PT_FPSIMD_OFFSET + SVE_PT_FPSIMD_SIZE(vq, flags))
		230
94	#endif /* __ASSEMBLY__ */	231	#endif /* __ASSEMBLY__ */
95		232
96	#endif /* _UAPI__ASM_PTRACE_H */	233	#endif /* _UAPI__ASM_PTRACE_H */

arch/arm64/include/uapi/asm/sigcontext.h+119−1

@@ -17,6 +17,8 @@ · 16 unmodified lines
17	#ifndef _UAPI__ASM_SIGCONTEXT_H	17	#ifndef _UAPI__ASM_SIGCONTEXT_H
18	#define _UAPI__ASM_SIGCONTEXT_H	18	#define _UAPI__ASM_SIGCONTEXT_H
19		19
		20	#ifndef __ASSEMBLY__
		21
20	#include <linux/types.h>	22	#include <linux/types.h>
21		23
22	/*	24	/*
@@ -42,10 +44,11 @@ struct sigcontext { · 19 unmodified lines
42	*	44	*
43	* 0x210 fpsimd_context	45	* 0x210 fpsimd_context
44	* 0x10 esr_context	46	* 0x10 esr_context
		47	* 0x8a0 sve_context (vl <= 64) (optional)
45	* 0x20 extra_context (optional)	48	* 0x20 extra_context (optional)
46	* 0x10 terminator (null _aarch64_ctx)	49	* 0x10 terminator (null _aarch64_ctx)
47	*	50	*
48	* 0xdb0 (reserved for future allocation)	51	* 0x510 (reserved for future allocation)
49	*	52	*
50	* New records that can exceed this space need to be opt-in for userspace, so	53	* New records that can exceed this space need to be opt-in for userspace, so
51	* that an expanded signal frame is not generated unexpectedly. The mechanism	54	* that an expanded signal frame is not generated unexpectedly. The mechanism
@@ -117,4 +120,119 @@ struct extra_context { · 65 unmodified lines
117	__u32 __reserved[3];	120	__u32 __reserved[3];
118	};	121	};
119		122
		123	#define SVE_MAGIC 0x53564501
		124
		125	struct sve_context {
		126	struct _aarch64_ctx head;
		127	__u16 vl;
		128	__u16 __reserved[3];
		129	};
		130
		131	#endif /* !__ASSEMBLY__ */
		132
		133	/*
		134	* The SVE architecture leaves space for future expansion of the
		135	* vector length beyond its initial architectural limit of 2048 bits
		136	* (16 quadwords).
		137	*
		138	* See linux/Documentation/arm64/sve.txt for a description of the VL/VQ
		139	* terminology.
		140	*/
		141	#define SVE_VQ_BYTES 16 /* number of bytes per quadword */
		142
		143	#define SVE_VQ_MIN 1
		144	#define SVE_VQ_MAX 512
		145
		146	#define SVE_VL_MIN (SVE_VQ_MIN * SVE_VQ_BYTES)
		147	#define SVE_VL_MAX (SVE_VQ_MAX * SVE_VQ_BYTES)
		148
		149	#define SVE_NUM_ZREGS 32
		150	#define SVE_NUM_PREGS 16
		151
		152	#define sve_vl_valid(vl) \
		153	((vl) % SVE_VQ_BYTES == 0 && (vl) >= SVE_VL_MIN && (vl) <= SVE_VL_MAX)
		154	#define sve_vq_from_vl(vl) ((vl) / SVE_VQ_BYTES)
		155	#define sve_vl_from_vq(vq) ((vq) * SVE_VQ_BYTES)
		156
		157	/*
		158	* If the SVE registers are currently live for the thread at signal delivery,
		159	* sve_context.head.size >=
		160	* SVE_SIG_CONTEXT_SIZE(sve_vq_from_vl(sve_context.vl))
		161	* and the register data may be accessed using the SVE_SIG_*() macros.
		162	*
		163	* If sve_context.head.size <
		164	* SVE_SIG_CONTEXT_SIZE(sve_vq_from_vl(sve_context.vl)),
		165	* the SVE registers were not live for the thread and no register data
		166	* is included: in this case, the SVE_SIG_*() macros should not be
		167	* used except for this check.
		168	*
		169	* The same convention applies when returning from a signal: a caller
		170	* will need to remove or resize the sve_context block if it wants to
		171	* make the SVE registers live when they were previously non-live or
		172	* vice-versa. This may require the the caller to allocate fresh
		173	* memory and/or move other context blocks in the signal frame.
		174	*
		175	* Changing the vector length during signal return is not permitted:
		176	* sve_context.vl must equal the thread's current vector length when
		177	* doing a sigreturn.
		178	*
		179	*
		180	* Note: for all these macros, the "vq" argument denotes the SVE
		181	* vector length in quadwords (i.e., units of 128 bits).
		182	*
		183	* The correct way to obtain vq is to use sve_vq_from_vl(vl). The
		184	* result is valid if and only if sve_vl_valid(vl) is true. This is
		185	* guaranteed for a struct sve_context written by the kernel.
		186	*
		187	*
		188	* Additional macros describe the contents and layout of the payload.
		189	* For each, SVE_SIG_x_OFFSET(args) is the start offset relative to
		190	* the start of struct sve_context, and SVE_SIG_x_SIZE(args) is the
		191	* size in bytes:
		192	*
		193	* x type description
		194	* - ---- -----------
		195	* REGS the entire SVE context
		196	*
		197	* ZREGS __uint128_t[SVE_NUM_ZREGS][vq] all Z-registers
		198	* ZREG __uint128_t[vq] individual Z-register Zn
		199	*
		200	* PREGS uint16_t[SVE_NUM_PREGS][vq] all P-registers
		201	* PREG uint16_t[vq] individual P-register Pn
		202	*
		203	* FFR uint16_t[vq] first-fault status register
		204	*
		205	* Additional data might be appended in the future.
		206	*/
		207
		208	#define SVE_SIG_ZREG_SIZE(vq) ((__u32)(vq) * SVE_VQ_BYTES)
		209	#define SVE_SIG_PREG_SIZE(vq) ((__u32)(vq) * (SVE_VQ_BYTES / 8))
		210	#define SVE_SIG_FFR_SIZE(vq) SVE_SIG_PREG_SIZE(vq)
		211
		212	#define SVE_SIG_REGS_OFFSET \
		213	((sizeof(struct sve_context) + (SVE_VQ_BYTES - 1)) \
		214	/ SVE_VQ_BYTES * SVE_VQ_BYTES)
		215
		216	#define SVE_SIG_ZREGS_OFFSET SVE_SIG_REGS_OFFSET
		217	#define SVE_SIG_ZREG_OFFSET(vq, n) \
		218	(SVE_SIG_ZREGS_OFFSET + SVE_SIG_ZREG_SIZE(vq) * (n))
		219	#define SVE_SIG_ZREGS_SIZE(vq) \
		220	(SVE_SIG_ZREG_OFFSET(vq, SVE_NUM_ZREGS) - SVE_SIG_ZREGS_OFFSET)
		221
		222	#define SVE_SIG_PREGS_OFFSET(vq) \
		223	(SVE_SIG_ZREGS_OFFSET + SVE_SIG_ZREGS_SIZE(vq))
		224	#define SVE_SIG_PREG_OFFSET(vq, n) \
		225	(SVE_SIG_PREGS_OFFSET(vq) + SVE_SIG_PREG_SIZE(vq) * (n))
		226	#define SVE_SIG_PREGS_SIZE(vq) \
		227	(SVE_SIG_PREG_OFFSET(vq, SVE_NUM_PREGS) - SVE_SIG_PREGS_OFFSET(vq))
		228
		229	#define SVE_SIG_FFR_OFFSET(vq) \
		230	(SVE_SIG_PREGS_OFFSET(vq) + SVE_SIG_PREGS_SIZE(vq))
		231
		232	#define SVE_SIG_REGS_SIZE(vq) \
		233	(SVE_SIG_FFR_OFFSET(vq) + SVE_SIG_FFR_SIZE(vq) - SVE_SIG_REGS_OFFSET)
		234
		235	#define SVE_SIG_CONTEXT_SIZE(vq) (SVE_SIG_REGS_OFFSET + SVE_SIG_REGS_SIZE(vq))
		236
		237
120	#endif /* _UAPI__ASM_SIGCONTEXT_H */	238	#endif /* _UAPI__ASM_SIGCONTEXT_H */

arch/arm64/kernel/Makefile+0−2

@@ -11,8 +11,6 @@ CFLAGS_REMOVE_ftrace.o = -pg · 10 unmodified lines
11	CFLAGS_REMOVE_insn.o = -pg	11	CFLAGS_REMOVE_insn.o = -pg
12	CFLAGS_REMOVE_return_address.o = -pg	12	CFLAGS_REMOVE_return_address.o = -pg
13		13
14	CFLAGS_setup.o = -DUTS_MACHINE='"$(UTS_MACHINE)"'
15
16	# Object file lists.	14	# Object file lists.
17	arm64-obj-y := debug-monitors.o entry.o irq.o fpsimd.o \	15	arm64-obj-y := debug-monitors.o entry.o irq.o fpsimd.o \
18	entry-fpsimd.o process.o ptrace.o setup.o signal.o \	16	entry-fpsimd.o process.o ptrace.o setup.o signal.o \

arch/arm64/kernel/armv8_deprecated.c+7−16

 	return ret;
 }
-static struct ctl_table ctl_abi[] = {
+static void __init register_insn_emulation_sysctl(void)
-	{
-		.procname = "abi",
-		.mode = 0555,
-	},
-	{ }
-};
-static void __init register_insn_emulation_sysctl(struct ctl_table *table)
 {
 	unsigned long flags;
 	int i = 0;
 	}
 	raw_spin_unlock_irqrestore(&insn_emulation_lock, flags);
-	table->child = insns_sysctl;
+	register_sysctl("abi", insns_sysctl);
-	register_sysctl_table(table);
 }
 /*
 	pr_warn_ratelimited("\"%s\" (%ld) uses obsolete SWP{B} instruction at 0x%llx\n",
 			current->comm, (unsigned long)current->pid, regs->pc);
-	regs->pc += 4;
+	arm64_skip_faulting_instruction(regs, 4);
 	return 0;
 fault:
 	pr_warn_ratelimited("\"%s\" (%ld) uses deprecated CP15 Barrier instruction at 0x%llx\n",
 			current->comm, (unsigned long)current->pid, regs->pc);
-	regs->pc += 4;
+	arm64_skip_faulting_instruction(regs, 4);
 	return 0;
 }
 static int a32_setend_handler(struct pt_regs *regs, u32 instr)
 {
 	int rc = compat_setend_handler(regs, (instr >> 9) & 1);
-	regs->pc += 4;
+	arm64_skip_faulting_instruction(regs, 4);
 	return rc;
 }
 static int t16_setend_handler(struct pt_regs *regs, u32 instr)
 {
 	int rc = compat_setend_handler(regs, (instr >> 3) & 1);
-	regs->pc += 2;
+	arm64_skip_faulting_instruction(regs, 2);
 	return rc;
 }
 	cpuhp_setup_state_nocalls(CPUHP_AP_ARM64_ISNDEP_STARTING,
 				  "arm64/isndep:starting",
 				  run_all_insn_set_hw_mode, NULL);
-	register_insn_emulation_sysctl(ctl_abi);
+	register_insn_emulation_sysctl();
 	return 0;
 }

arch/arm64/kernel/cpufeature.c+141−63

 #include <asm/cpu.h>
 #include <asm/cpufeature.h>
 #include <asm/cpu_ops.h>
+#include <asm/fpsimd.h>
 #include <asm/mmu_context.h>
 #include <asm/processor.h>
 #include <asm/sysreg.h>
 DECLARE_BITMAP(cpu_hwcaps, ARM64_NCAPS);
 EXPORT_SYMBOL(cpu_hwcaps);
+/*
+ * Flag to indicate if we have computed the system wide
+ * capabilities based on the boot time active CPUs. This
+ * will be used to determine if a new booting CPU should
+ * go through the verification process to make sure that it
+ * supports the system capabilities, without using a hotplug
+ * notifier.
+ */
+static bool sys_caps_initialised;
+static inline void set_sys_caps_initialised(void)
+{
+	sys_caps_initialised = true;
+}
 static int dump_cpu_hwcaps(struct notifier_block *self, unsigned long v, void *p)
 {
 	/* file-wide pr_fmt adds "CPU features: " prefix */
  * sync with the documentation of the CPU feature register ABI.
  */
 static const struct arm64_ftr_bits ftr_id_aa64isar0[] = {
-	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64ISAR0_RDM_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_DP_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_SM4_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_SM3_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_SHA3_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_RDM_SHIFT, 4, 0),
 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_ATOMICS_SHIFT, 4, 0),
 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_CRC32_SHIFT, 4, 0),
 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_SHA2_SHIFT, 4, 0),
 };
 static const struct arm64_ftr_bits ftr_id_aa64isar1[] = {
-	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64ISAR1_LRCPC_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_LRCPC_SHIFT, 4, 0),
-	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64ISAR1_FCMA_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_FCMA_SHIFT, 4, 0),
-	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64ISAR1_JSCVT_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_JSCVT_SHIFT, 4, 0),
-	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64ISAR1_DPB_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_DPB_SHIFT, 4, 0),
 	ARM64_FTR_END,
 };
 static const struct arm64_ftr_bits ftr_id_aa64pfr0[] = {
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_AA64PFR0_GIC_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_SVE_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_GIC_SHIFT, 4, 0),
 	S_ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_ASIMD_SHIFT, 4, ID_AA64PFR0_ASIMD_NI),
 	S_ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_FP_SHIFT, 4, ID_AA64PFR0_FP_NI),
 	/* Linux doesn't care about the EL3 */
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_EXACT, ID_AA64PFR0_EL3_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL3_SHIFT, 4, 0),
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_AA64PFR0_EL2_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL2_SHIFT, 4, 0),
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_AA64PFR0_EL1_SHIFT, 4, ID_AA64PFR0_EL1_64BIT_ONLY),
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL1_SHIFT, 4, ID_AA64PFR0_EL1_64BIT_ONLY),
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_AA64PFR0_EL0_SHIFT, 4, ID_AA64PFR0_EL0_64BIT_ONLY),
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL0_SHIFT, 4, ID_AA64PFR0_EL0_64BIT_ONLY),
 	ARM64_FTR_END,
 };
 static const struct arm64_ftr_bits ftr_id_aa64mmfr0[] = {
-	S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_AA64MMFR0_TGRAN4_SHIFT, 4, ID_AA64MMFR0_TGRAN4_NI),
+	S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_TGRAN4_SHIFT, 4, ID_AA64MMFR0_TGRAN4_NI),
-	S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_AA64MMFR0_TGRAN64_SHIFT, 4, ID_AA64MMFR0_TGRAN64_NI),
+	S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_TGRAN64_SHIFT, 4, ID_AA64MMFR0_TGRAN64_NI),
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_AA64MMFR0_TGRAN16_SHIFT, 4, ID_AA64MMFR0_TGRAN16_NI),
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_TGRAN16_SHIFT, 4, ID_AA64MMFR0_TGRAN16_NI),
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_AA64MMFR0_BIGENDEL0_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_BIGENDEL0_SHIFT, 4, 0),
 	/* Linux shouldn't care about secure memory */
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_EXACT, ID_AA64MMFR0_SNSMEM_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_SNSMEM_SHIFT, 4, 0),
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_AA64MMFR0_BIGENDEL_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_BIGENDEL_SHIFT, 4, 0),
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_AA64MMFR0_ASID_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_ASID_SHIFT, 4, 0),
 	/*
 	 * Differing PARange is fine as long as all peripherals and memory are mapped
 	 * within the minimum PARange of all CPUs
 static const struct arm64_ftr_bits ftr_id_aa64mmfr1[] = {
 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_PAN_SHIFT, 4, 0),
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_AA64MMFR1_LOR_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_LOR_SHIFT, 4, 0),
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_AA64MMFR1_HPD_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_HPD_SHIFT, 4, 0),
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_AA64MMFR1_VHE_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_VHE_SHIFT, 4, 0),
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_AA64MMFR1_VMIDBITS_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_VMIDBITS_SHIFT, 4, 0),
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_AA64MMFR1_HADBS_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_HADBS_SHIFT, 4, 0),
 	ARM64_FTR_END,
 };
 static const struct arm64_ftr_bits ftr_id_aa64mmfr2[] = {
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_AA64MMFR2_LVA_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_LVA_SHIFT, 4, 0),
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_AA64MMFR2_IESB_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_IESB_SHIFT, 4, 0),
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_AA64MMFR2_LSM_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_LSM_SHIFT, 4, 0),
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_AA64MMFR2_UAO_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_UAO_SHIFT, 4, 0),
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_AA64MMFR2_CNP_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_CNP_SHIFT, 4, 0),
 	ARM64_FTR_END,
 };
 };
 static const struct arm64_ftr_bits ftr_id_mmfr0[] = {
-	S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, 28, 4, 0xf),	/* InnerShr */
+	S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 28, 4, 0xf),	/* InnerShr */
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, 24, 4, 0),	/* FCSE */
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 24, 4, 0),	/* FCSE */
 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, 20, 4, 0),	/* AuxReg */
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, 16, 4, 0),	/* TCM */
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 16, 4, 0),	/* TCM */
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, 12, 4, 0),	/* ShareLvl */
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 12, 4, 0),	/* ShareLvl */
-	S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, 8, 4, 0xf),	/* OuterShr */
+	S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 8, 4, 0xf),	/* OuterShr */
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, 4, 4, 0),	/* PMSA */
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 4, 4, 0),	/* PMSA */
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, 0, 4, 0),	/* VMSA */
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 0, 4, 0),	/* VMSA */
 	ARM64_FTR_END,
 };
 };
 static const struct arm64_ftr_bits ftr_mvfr2[] = {
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, 4, 4, 0),		/* FPMisc */
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 4, 4, 0),		/* FPMisc */
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, 0, 4, 0),		/* SIMDMisc */
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 0, 4, 0),		/* SIMDMisc */
 	ARM64_FTR_END,
 };
 static const struct arm64_ftr_bits ftr_id_isar5[] = {
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_ISAR5_RDM_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_RDM_SHIFT, 4, 0),
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_ISAR5_CRC32_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_CRC32_SHIFT, 4, 0),
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_ISAR5_SHA2_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_SHA2_SHIFT, 4, 0),
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_ISAR5_SHA1_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_SHA1_SHIFT, 4, 0),
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_ISAR5_AES_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_AES_SHIFT, 4, 0),
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_ISAR5_SEVL_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_SEVL_SHIFT, 4, 0),
 	ARM64_FTR_END,
 };
 static const struct arm64_ftr_bits ftr_id_mmfr4[] = {
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, 4, 4, 0),		/* ac2 */
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 4, 4, 0),	/* ac2 */
 	ARM64_FTR_END,
 };
 static const struct arm64_ftr_bits ftr_id_pfr0[] = {
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, 12, 4, 0),	/* State3 */
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 12, 4, 0),		/* State3 */
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, 8, 4, 0),		/* State2 */
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 8, 4, 0),		/* State2 */
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, 4, 4, 0),		/* State1 */
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 4, 4, 0),		/* State1 */
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, 0, 4, 0),		/* State0 */
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 0, 4, 0),		/* State0 */
 	ARM64_FTR_END,
 };
 	ARM64_FTR_END,
 };
+static const struct arm64_ftr_bits ftr_zcr[] = {
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE,
+		ZCR_ELx_LEN_SHIFT, ZCR_ELx_LEN_SIZE, 0),	/* LEN */
+	ARM64_FTR_END,
+};
 /*
  * Common ftr bits for a 32bit register with all hidden, strict
  * attributes, with 4bit feature fields and a default safe value of
 	/* Op1 = 0, CRn = 0, CRm = 4 */
 	ARM64_FTR_REG(SYS_ID_AA64PFR0_EL1, ftr_id_aa64pfr0),
 	ARM64_FTR_REG(SYS_ID_AA64PFR1_EL1, ftr_raz),
+	ARM64_FTR_REG(SYS_ID_AA64ZFR0_EL1, ftr_raz),
 	/* Op1 = 0, CRn = 0, CRm = 5 */
 	ARM64_FTR_REG(SYS_ID_AA64DFR0_EL1, ftr_id_aa64dfr0),
 	ARM64_FTR_REG(SYS_ID_AA64MMFR1_EL1, ftr_id_aa64mmfr1),
 	ARM64_FTR_REG(SYS_ID_AA64MMFR2_EL1, ftr_id_aa64mmfr2),
+	/* Op1 = 0, CRn = 1, CRm = 2 */
+	ARM64_FTR_REG(SYS_ZCR_EL1, ftr_zcr),
 	/* Op1 = 3, CRn = 0, CRm = 0 */
 	{ SYS_CTR_EL0, &arm64_ftr_reg_ctrel0 },
 	ARM64_FTR_REG(SYS_DCZID_EL0, ftr_dczid),
 	init_cpu_ftr_reg(SYS_ID_AA64MMFR2_EL1, info->reg_id_aa64mmfr2);
 	init_cpu_ftr_reg(SYS_ID_AA64PFR0_EL1, info->reg_id_aa64pfr0);
 	init_cpu_ftr_reg(SYS_ID_AA64PFR1_EL1, info->reg_id_aa64pfr1);
+	init_cpu_ftr_reg(SYS_ID_AA64ZFR0_EL1, info->reg_id_aa64zfr0);
 	if (id_aa64pfr0_32bit_el0(info->reg_id_aa64pfr0)) {
 		init_cpu_ftr_reg(SYS_ID_DFR0_EL1, info->reg_id_dfr0);
 		init_cpu_ftr_reg(SYS_MVFR2_EL1, info->reg_mvfr2);
 	}
+	if (id_aa64pfr0_sve(info->reg_id_aa64pfr0)) {
+		init_cpu_ftr_reg(SYS_ZCR_EL1, info->reg_zcr);
+		sve_init_vq_map();
+	}
 }
 static void update_cpu_ftr_reg(struct arm64_ftr_reg *reg, u64 new)
 	taint |= check_update_ftr_reg(SYS_ID_AA64PFR1_EL1, cpu,
 				      info->reg_id_aa64pfr1, boot->reg_id_aa64pfr1);
+	taint |= check_update_ftr_reg(SYS_ID_AA64ZFR0_EL1, cpu,
+				      info->reg_id_aa64zfr0, boot->reg_id_aa64zfr0);
 	/*
 	 * If we have AArch32, we care about 32-bit features for compat.
 	 * If the system doesn't support AArch32, don't update them.
 					info->reg_mvfr2, boot->reg_mvfr2);
 	}
+	if (id_aa64pfr0_sve(info->reg_id_aa64pfr0)) {
+		taint |= check_update_ftr_reg(SYS_ZCR_EL1, cpu,
+					info->reg_zcr, boot->reg_zcr);
+		/* Probe vector lengths, unless we already gave up on SVE */
+		if (id_aa64pfr0_sve(read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1)) &&
+		    !sys_caps_initialised)
+			sve_update_vq_map();
+	}
 	/*
 	 * Mismatched CPU features are a recipe for disaster. Don't even
 	 * pretend to support them.
 		.min_field_value = 1,
 	},
 #endif
+#ifdef CONFIG_ARM64_SVE
+	{
+		.desc = "Scalable Vector Extension",
+		.capability = ARM64_SVE,
+		.def_scope = SCOPE_SYSTEM,
+		.sys_reg = SYS_ID_AA64PFR0_EL1,
+		.sign = FTR_UNSIGNED,
+		.field_pos = ID_AA64PFR0_SVE_SHIFT,
+		.min_field_value = ID_AA64PFR0_SVE,
+		.matches = has_cpuid_feature,
+		.enable = sve_kernel_enable,
+	},
+#endif /* CONFIG_ARM64_SVE */
 	{},
 };
 	HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_AES_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_AES),
 	HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_SHA1_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_SHA1),
 	HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_SHA2_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_SHA2),
+	HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_SHA2_SHIFT, FTR_UNSIGNED, 2, CAP_HWCAP, HWCAP_SHA512),
 	HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_CRC32_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_CRC32),
 	HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_ATOMICS_SHIFT, FTR_UNSIGNED, 2, CAP_HWCAP, HWCAP_ATOMICS),
 	HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_RDM_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_ASIMDRDM),
+	HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_SHA3_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_SHA3),
+	HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_SM3_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_SM3),
+	HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_SM4_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_SM4),
+	HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_DP_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_ASIMDDP),
 	HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_FP_SHIFT, FTR_SIGNED, 0, CAP_HWCAP, HWCAP_FP),
 	HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_FP_SHIFT, FTR_SIGNED, 1, CAP_HWCAP, HWCAP_FPHP),
 	HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_ASIMD_SHIFT, FTR_SIGNED, 0, CAP_HWCAP, HWCAP_ASIMD),
 	HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_JSCVT_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_JSCVT),
 	HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_FCMA_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_FCMA),
 	HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_LRCPC_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_LRCPC),
+#ifdef CONFIG_ARM64_SVE
+	HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_SVE_SHIFT, FTR_UNSIGNED, ID_AA64PFR0_SVE, CAP_HWCAP, HWCAP_SVE),
+#endif
 	{},
 };
 	}
 }
-/*
- * Flag to indicate if we have computed the system wide
- * capabilities based on the boot time active CPUs. This
- * will be used to determine if a new booting CPU should
- * go through the verification process to make sure that it
- * supports the system capabilities, without using a hotplug
- * notifier.
- */
-static bool sys_caps_initialised;
-static inline void set_sys_caps_initialised(void)
-{
-	sys_caps_initialised = true;
-}
 /*
  * Check for CPU features that are used in early boot
  * based on the Boot CPU value.
 	}
 }
+static void verify_sve_features(void)
+{
+	u64 safe_zcr = read_sanitised_ftr_reg(SYS_ZCR_EL1);
+	u64 zcr = read_zcr_features();
+	unsigned int safe_len = safe_zcr & ZCR_ELx_LEN_MASK;
+	unsigned int len = zcr & ZCR_ELx_LEN_MASK;
+	if (len < safe_len || sve_verify_vq_map()) {
+		pr_crit("CPU%d: SVE: required vector length(s) missing\n",
+			smp_processor_id());
+		cpu_die_early();
+	}
+	/* Add checks on other ZCR bits here if necessary */
+}
 /*
  * Run through the enabled system capabilities and enable() it on this CPU.
  * The capabilities were decided based on the available CPUs at the boot time.
 	verify_local_cpu_errata_workarounds();
 	verify_local_cpu_features(arm64_features);
 	verify_local_elf_hwcaps(arm64_elf_hwcaps);
 	if (system_supports_32bit_el0())
 		verify_local_elf_hwcaps(compat_elf_hwcaps);
+	if (system_supports_sve())
+		verify_sve_features();
 }
 void check_local_cpu_capabilities(void)
 	if (system_supports_32bit_el0())
 		setup_elf_hwcaps(compat_elf_hwcaps);
+	sve_setup();
 	/* Advertise that we have computed the system capabilities */
 	set_sys_caps_initialised();
 	if (!rc) {
 		dst = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RT, insn);
 		pt_regs_write_reg(regs, dst, val);
-		regs->pc += 4;
+		arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE);
 	}
 	return rc;

arch/arm64/kernel/cpuinfo.c+12−0

 #include <asm/cpu.h>
 #include <asm/cputype.h>
 #include <asm/cpufeature.h>
+#include <asm/fpsimd.h>
 #include <linux/bitops.h>
 #include <linux/bug.h>
 	"fcma",
 	"lrcpc",
 	"dcpop",
+	"sha3",
+	"sm3",
+	"sm4",
+	"asimddp",
+	"sha512",
+	"sve",
 	NULL
 };
 	info->reg_id_aa64mmfr2 = read_cpuid(ID_AA64MMFR2_EL1);
 	info->reg_id_aa64pfr0 = read_cpuid(ID_AA64PFR0_EL1);
 	info->reg_id_aa64pfr1 = read_cpuid(ID_AA64PFR1_EL1);
+	info->reg_id_aa64zfr0 = read_cpuid(ID_AA64ZFR0_EL1);
 	/* Update the 32bit ID registers only if AArch32 is implemented */
 	if (id_aa64pfr0_32bit_el0(info->reg_id_aa64pfr0)) {
 		info->reg_mvfr2 = read_cpuid(MVFR2_EL1);
 	}
+	if (IS_ENABLED(CONFIG_ARM64_SVE) &&
+	    id_aa64pfr0_sve(info->reg_id_aa64pfr0))
+		info->reg_zcr = read_zcr_features();
 	cpuinfo_detect_icache_policy(info);
 }

arch/arm64/kernel/debug-monitors.c+3−2

 #include <asm/cpufeature.h>
 #include <asm/cputype.h>
+#include <asm/daifflags.h>
 #include <asm/debug-monitors.h>
 #include <asm/system_misc.h>
 static void mdscr_write(u32 mdscr)
 {
 	unsigned long flags;
-	local_dbg_save(flags);
+	flags = local_daif_save();
 	write_sysreg(mdscr, mdscr_el1);
-	local_dbg_restore(flags);
+	local_daif_restore(flags);
 }
 NOKPROBE_SYMBOL(mdscr_write);

arch/arm64/kernel/entry-fpsimd.S+17−0

@@ -41,3 +41,20 @@ ENTRY(fpsimd_load_state) · 40 unmodified lines
41	fpsimd_restore x0, 8	41	fpsimd_restore x0, 8
42	ret	42	ret
43	ENDPROC(fpsimd_load_state)	43	ENDPROC(fpsimd_load_state)
		44
		45	#ifdef CONFIG_ARM64_SVE
		46	ENTRY(sve_save_state)
		47	sve_save 0, x1, 2
		48	ret
		49	ENDPROC(sve_save_state)
		50
		51	ENTRY(sve_load_state)
		52	sve_load 0, x1, x2, 3
		53	ret
		54	ENDPROC(sve_load_state)
		55
		56	ENTRY(sve_get_vl)
		57	_sve_rdvl 0, 1
		58	ret
		59	ENDPROC(sve_get_vl)
		60	#endif /* CONFIG_ARM64_SVE */

arch/arm64/kernel/entry-ftrace.S+3−9

 	mcount_get_lr	x1		//       function's lr (= parent's pc)
 	blr	x2			//   (*ftrace_trace_function)(pc, lr);
-#ifndef CONFIG_FUNCTION_GRAPH_TRACER
+skip_ftrace_call:			// }
-skip_ftrace_call:			//   return;
+#ifdef CONFIG_FUNCTION_GRAPH_TRACER
-	mcount_exit			// }
-#else
-	mcount_exit			//   return;
-					// }
-skip_ftrace_call:
 	ldr_l	x2, ftrace_graph_return
 	cmp	x0, x2			//   if ((ftrace_graph_return
 	b.ne	ftrace_graph_caller	//        != ftrace_stub)
 	adr_l	x0, ftrace_graph_entry_stub //     != ftrace_graph_entry_stub))
 	cmp	x0, x2
 	b.ne	ftrace_graph_caller	//     ftrace_graph_caller();
-	mcount_exit
 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
+	mcount_exit
 ENDPROC(_mcount)
 #else /* CONFIG_DYNAMIC_FTRACE */

arch/arm64/kernel/entry.S+88−40

 #include <asm/errno.h>
 #include <asm/esr.h>
 #include <asm/irq.h>
-#include <asm/memory.h>
+#include <asm/processor.h>
 #include <asm/ptrace.h>
 #include <asm/thread_info.h>
 #include <asm/asm-uaccess.h>
 	.macro	kernel_exit, el
 	.if	\el != 0
+	disable_daif
 	/* Restore the task's original addr_limit. */
 	ldr	x20, [sp, #S_ORIG_ADDR_LIMIT]
 	str	x20, [tsk, #TSK_TI_ADDR_LIMIT]
 	kernel_ventry	el1_sync			// Synchronous EL1h
 	kernel_ventry	el1_irq				// IRQ EL1h
 	kernel_ventry	el1_fiq_invalid			// FIQ EL1h
-	kernel_ventry	el1_error_invalid		// Error EL1h
+	kernel_ventry	el1_error			// Error EL1h
 	kernel_ventry	el0_sync			// Synchronous 64-bit EL0
 	kernel_ventry	el0_irq				// IRQ 64-bit EL0
 	kernel_ventry	el0_fiq_invalid			// FIQ 64-bit EL0
-	kernel_ventry	el0_error_invalid		// Error 64-bit EL0
+	kernel_ventry	el0_error			// Error 64-bit EL0
 #ifdef CONFIG_COMPAT
 	kernel_ventry	el0_sync_compat			// Synchronous 32-bit EL0
 	kernel_ventry	el0_irq_compat			// IRQ 32-bit EL0
 	kernel_ventry	el0_fiq_invalid_compat		// FIQ 32-bit EL0
-	kernel_ventry	el0_error_invalid_compat	// Error 32-bit EL0
+	kernel_ventry	el0_error_compat		// Error 32-bit EL0
 #else
 	kernel_ventry	el0_sync_invalid		// Synchronous 32-bit EL0
 	kernel_ventry	el0_irq_invalid			// IRQ 32-bit EL0
 el0_fiq_invalid_compat:
 	inv_entry 0, BAD_FIQ, 32
 ENDPROC(el0_fiq_invalid_compat)
-el0_error_invalid_compat:
-	inv_entry 0, BAD_ERROR, 32
-ENDPROC(el0_error_invalid_compat)
 #endif
 el1_sync_invalid:
 	 * Data abort handling
 	 */
 	mrs	x3, far_el1
-	enable_dbg
+	inherit_daif	pstate=x23, tmp=x2
-	// re-enable interrupts if they were enabled in the aborted context
-	tbnz	x23, #7, 1f			// PSR_I_BIT
-	enable_irq
-1:
 	clear_address_tag x0, x3
 	mov	x2, sp				// struct pt_regs
 	bl	do_mem_abort
-	// disable interrupts before pulling preserved data off the stack
-	disable_irq
 	kernel_exit 1
 el1_sp_pc:
 	/*
 	 * Stack or PC alignment exception handling
 	 */
 	mrs	x0, far_el1
-	enable_dbg
+	inherit_daif	pstate=x23, tmp=x2
 	mov	x2, sp
 	bl	do_sp_pc_abort
 	ASM_BUG()
 	/*
 	 * Undefined instruction
 	 */
-	enable_dbg
+	inherit_daif	pstate=x23, tmp=x2
 	mov	x0, sp
 	bl	do_undefinstr
 	ASM_BUG()
 	kernel_exit 1
 el1_inv:
 	// TODO: add support for undefined instructions in kernel mode
-	enable_dbg
+	inherit_daif	pstate=x23, tmp=x2
 	mov	x0, sp
 	mov	x2, x1
 	mov	x1, #BAD_SYNC
 	.align	6
 el1_irq:
 	kernel_entry 1
-	enable_dbg
+	enable_da_f
 #ifdef CONFIG_TRACE_IRQFLAGS
 	bl	trace_hardirqs_off
 #endif
 	b.eq	el0_ia
 	cmp	x24, #ESR_ELx_EC_FP_ASIMD	// FP/ASIMD access
 	b.eq	el0_fpsimd_acc
+	cmp	x24, #ESR_ELx_EC_SVE		// SVE access
+	b.eq	el0_sve_acc
 	cmp	x24, #ESR_ELx_EC_FP_EXC64	// FP/ASIMD exception
 	b.eq	el0_fpsimd_exc
 	cmp	x24, #ESR_ELx_EC_SYS64		// configurable trap
 	/*
 	 * AArch32 syscall handling
 	 */
+	ldr	x16, [tsk, #TSK_TI_FLAGS]	// load thread flags
 	adrp	stbl, compat_sys_call_table	// load compat syscall table pointer
 	mov	wscno, w7			// syscall number in w7 (r7)
 	mov     wsc_nr, #__NR_compat_syscalls
 el0_irq_compat:
 	kernel_entry 0, 32
 	b	el0_irq_naked
+el0_error_compat:
+	kernel_entry 0, 32
+	b	el0_error_naked
 #endif
 el0_da:
 	 * Data abort handling
 	 */
 	mrs	x26, far_el1
-	// enable interrupts before calling the main handler
+	enable_daif
-	enable_dbg_and_irq
 	ct_user_exit
 	clear_address_tag x0, x26
 	mov	x1, x25
 	 * Instruction abort handling
 	 */
 	mrs	x26, far_el1
-	// enable interrupts before calling the main handler
+	enable_daif
-	enable_dbg_and_irq
 	ct_user_exit
 	mov	x0, x26
 	mov	x1, x25
 	/*
 	 * Floating Point or Advanced SIMD access
 	 */
-	enable_dbg
+	enable_daif
 	ct_user_exit
 	mov	x0, x25
 	mov	x1, sp
 	bl	do_fpsimd_acc
 	b	ret_to_user
+el0_sve_acc:
+	/*
+	 * Scalable Vector Extension access
+	 */
+	enable_daif
+	ct_user_exit
+	mov	x0, x25
+	mov	x1, sp
+	bl	do_sve_acc
+	b	ret_to_user
 el0_fpsimd_exc:
 	/*
-	 * Floating Point or Advanced SIMD exception
+	 * Floating Point, Advanced SIMD or SVE exception
 	 */
-	enable_dbg
+	enable_daif
 	ct_user_exit
 	mov	x0, x25
 	mov	x1, sp
 	 * Stack or PC alignment exception handling
 	 */
 	mrs	x26, far_el1
-	// enable interrupts before calling the main handler
+	enable_daif
-	enable_dbg_and_irq
 	ct_user_exit
 	mov	x0, x26
 	mov	x1, x25
 	/*
 	 * Undefined instruction
 	 */
-	// enable interrupts before calling the main handler
+	enable_daif
-	enable_dbg_and_irq
 	ct_user_exit
 	mov	x0, sp
 	bl	do_undefinstr
 	/*
 	 * System instructions, for trapped cache maintenance instructions
 	 */
-	enable_dbg_and_irq
+	enable_daif
 	ct_user_exit
 	mov	x0, x25
 	mov	x1, sp
 	mov	x1, x25
 	mov	x2, sp
 	bl	do_debug_exception
-	enable_dbg
+	enable_daif
 	ct_user_exit
 	b	ret_to_user
 el0_inv:
-	enable_dbg
+	enable_daif
 	ct_user_exit
 	mov	x0, sp
 	mov	x1, #BAD_SYNC
 el0_irq:
 	kernel_entry 0
 el0_irq_naked:
-	enable_dbg
+	enable_da_f
 #ifdef CONFIG_TRACE_IRQFLAGS
 	bl	trace_hardirqs_off
 #endif
 	b	ret_to_user
 ENDPROC(el0_irq)
+el1_error:
+	kernel_entry 1
+	mrs	x1, esr_el1
+	enable_dbg
+	mov	x0, sp
+	bl	do_serror
+	kernel_exit 1
+ENDPROC(el1_error)
+el0_error:
+	kernel_entry 0
+el0_error_naked:
+	mrs	x1, esr_el1
+	enable_dbg
+	mov	x0, sp
+	bl	do_serror
+	enable_daif
+	ct_user_exit
+	b	ret_to_user
+ENDPROC(el0_error)
 /*
  * This is the fast syscall return path.  We do as little as possible here,
  * and this includes saving x0 back into the kernel stack.
  */
 ret_fast_syscall:
-	disable_irq				// disable interrupts
+	disable_daif
 	str	x0, [sp, #S_X0]			// returned x0
 	ldr	x1, [tsk, #TSK_TI_FLAGS]	// re-check for syscall tracing
 	and	x2, x1, #_TIF_SYSCALL_WORK
 	enable_step_tsk x1, x2
 	kernel_exit 0
 ret_fast_syscall_trace:
-	enable_irq				// enable interrupts
+	enable_daif
 	b	__sys_trace_return_skipped	// we already saved x0
 /*
  * "slow" syscall return path.
  */
 ret_to_user:
-	disable_irq				// disable interrupts
+	disable_daif
 	ldr	x1, [tsk, #TSK_TI_FLAGS]
 	and	x2, x1, #_TIF_WORK_MASK
 	cbnz	x2, work_pending
  */
 	.align	6
 el0_svc:
+	ldr	x16, [tsk, #TSK_TI_FLAGS]	// load thread flags
 	adrp	stbl, sys_call_table		// load syscall table pointer
 	mov	wscno, w8			// syscall number in w8
 	mov	wsc_nr, #__NR_syscalls
+#ifdef CONFIG_ARM64_SVE
+alternative_if_not ARM64_SVE
+	b	el0_svc_naked
+alternative_else_nop_endif
+	tbz	x16, #TIF_SVE, el0_svc_naked	// Skip unless TIF_SVE set:
+	bic	x16, x16, #_TIF_SVE		// discard SVE state
+	str	x16, [tsk, #TSK_TI_FLAGS]
+	/*
+	 * task_fpsimd_load() won't be called to update CPACR_EL1 in
+	 * ret_to_user unless TIF_FOREIGN_FPSTATE is still set, which only
+	 * happens if a context switch or kernel_neon_begin() or context
+	 * modification (sigreturn, ptrace) intervenes.
+	 * So, ensure that CPACR_EL1 is already correct for the fast-path case:
+	 */
+	mrs	x9, cpacr_el1
+	bic	x9, x9, #CPACR_EL1_ZEN_EL0EN	// disable SVE for el0
+	msr	cpacr_el1, x9			// synchronised by eret to el0
+#endif
 el0_svc_naked:					// compat entry point
 	stp	x0, xscno, [sp, #S_ORIG_X0]	// save the original x0 and syscall number
-	enable_dbg_and_irq
+	enable_daif
 	ct_user_exit 1
-	ldr	x16, [tsk, #TSK_TI_FLAGS]	// check for syscall hooks
+	tst	x16, #_TIF_SYSCALL_WORK		// check for syscall hooks
-	tst	x16, #_TIF_SYSCALL_WORK
 	b.ne	__sys_trace
 	cmp     wscno, wsc_nr			// check upper syscall limit
 	b.hs	ni_sys

arch/arm64/kernel/fpsimd.c+881−27

@@ -17,19 +17,34 @@ · 16 unmodified lines
17	* along with this program. If not, see <http://www.gnu.org/licenses/>.	17	* along with this program. If not, see <http://www.gnu.org/licenses/>.
18	*/	18	*/
19		19
		20	#include <linux/bitmap.h>
20	#include <linux/bottom_half.h>	21	#include <linux/bottom_half.h>
		22	#include <linux/bug.h>
		23	#include <linux/cache.h>
		24	#include <linux/compat.h>
21	#include <linux/cpu.h>	25	#include <linux/cpu.h>
22	#include <linux/cpu_pm.h>	26	#include <linux/cpu_pm.h>
23	#include <linux/kernel.h>	27	#include <linux/kernel.h>
		28	#include <linux/linkage.h>
		29	#include <linux/irqflags.h>
24	#include <linux/init.h>	30	#include <linux/init.h>
25	#include <linux/percpu.h>	31	#include <linux/percpu.h>
		32	#include <linux/prctl.h>
26	#include <linux/preempt.h>	33	#include <linux/preempt.h>
		34	#include <linux/prctl.h>
		35	#include <linux/ptrace.h>
27	#include <linux/sched/signal.h>	36	#include <linux/sched/signal.h>
		37	#include <linux/sched/task_stack.h>
28	#include <linux/signal.h>	38	#include <linux/signal.h>
		39	#include <linux/slab.h>
		40	#include <linux/sysctl.h>
29		41
30	#include <asm/fpsimd.h>	42	#include <asm/fpsimd.h>
31	#include <asm/cputype.h>	43	#include <asm/cputype.h>
32	#include <asm/simd.h>	44	#include <asm/simd.h>
		45	#include <asm/sigcontext.h>
		46	#include <asm/sysreg.h>
		47	#include <asm/traps.h>
33		48
34	#define FPEXC_IOF (1 << 0)	49	#define FPEXC_IOF (1 << 0)
35	#define FPEXC_DZF (1 << 1)	50	#define FPEXC_DZF (1 << 1)
@@ -39,6 +54,8 @@ · 3 unmodified lines
39	#define FPEXC_IDF (1 << 7)	54	#define FPEXC_IDF (1 << 7)
40		55
41	/*	56	/*
		57	* (Note: in this discussion, statements about FPSIMD apply equally to SVE.)
		58	*
42	* In order to reduce the number of times the FPSIMD state is needlessly saved	59	* In order to reduce the number of times the FPSIMD state is needlessly saved
43	* and restored, we need to keep track of two things:	60	* and restored, we need to keep track of two things:
44	* (a) for each task, we need to remember which CPU was the last one to have	61	* (a) for each task, we need to remember which CPU was the last one to have
@@ -99,10 +116,741 @@ · 54 unmodified lines
99	*/	116	*/
100	static DEFINE_PER_CPU(struct fpsimd_state *, fpsimd_last_state);	117	static DEFINE_PER_CPU(struct fpsimd_state *, fpsimd_last_state);
101		118
		119	/* Default VL for tasks that don't set it explicitly: */
		120	static int sve_default_vl = -1;
		121
		122	#ifdef CONFIG_ARM64_SVE
		123
		124	/* Maximum supported vector length across all CPUs (initially poisoned) */
		125	int __ro_after_init sve_max_vl = -1;
		126	/* Set of available vector lengths, as vq_to_bit(vq): */
		127	static __ro_after_init DECLARE_BITMAP(sve_vq_map, SVE_VQ_MAX);
		128	static void __percpu *efi_sve_state;
		129
		130	#else /* ! CONFIG_ARM64_SVE */
		131
		132	/* Dummy declaration for code that will be optimised out: */
		133	extern __ro_after_init DECLARE_BITMAP(sve_vq_map, SVE_VQ_MAX);
		134	extern void __percpu *efi_sve_state;
		135
		136	#endif /* ! CONFIG_ARM64_SVE */
		137
		138	/*
		139	* Call __sve_free() directly only if you know task can't be scheduled
		140	* or preempted.
		141	*/
		142	static void __sve_free(struct task_struct *task)
		143	{
		144	kfree(task->thread.sve_state);
		145	task->thread.sve_state = NULL;
		146	}
		147
		148	static void sve_free(struct task_struct *task)
		149	{
		150	WARN_ON(test_tsk_thread_flag(task, TIF_SVE));
		151
		152	__sve_free(task);
		153	}
		154
		155
		156	/* Offset of FFR in the SVE register dump */
		157	static size_t sve_ffr_offset(int vl)
		158	{
		159	return SVE_SIG_FFR_OFFSET(sve_vq_from_vl(vl)) - SVE_SIG_REGS_OFFSET;
		160	}
		161
		162	static void sve_pffr(struct task_struct task)
		163	{
		164	return (char *)task->thread.sve_state +
		165	sve_ffr_offset(task->thread.sve_vl);
		166	}
		167
		168	static void change_cpacr(u64 val, u64 mask)
		169	{
		170	u64 cpacr = read_sysreg(CPACR_EL1);
		171	u64 new = (cpacr & ~mask) \| val;
		172
		173	if (new != cpacr)
		174	write_sysreg(new, CPACR_EL1);
		175	}
		176
		177	static void sve_user_disable(void)
		178	{
		179	change_cpacr(0, CPACR_EL1_ZEN_EL0EN);
		180	}
		181
		182	static void sve_user_enable(void)
		183	{
		184	change_cpacr(CPACR_EL1_ZEN_EL0EN, CPACR_EL1_ZEN_EL0EN);
		185	}
		186
		187	/*
		188	* TIF_SVE controls whether a task can use SVE without trapping while
		189	* in userspace, and also the way a task's FPSIMD/SVE state is stored
		190	* in thread_struct.
		191	*
		192	* The kernel uses this flag to track whether a user task is actively
		193	* using SVE, and therefore whether full SVE register state needs to
		194	* be tracked. If not, the cheaper FPSIMD context handling code can
		195	* be used instead of the more costly SVE equivalents.
		196	*
		197	* * TIF_SVE set:
		198	*
		199	* The task can execute SVE instructions while in userspace without
		200	* trapping to the kernel.
		201	*
		202	* When stored, Z0-Z31 (incorporating Vn in bits[127:0] or the
		203	* corresponding Zn), P0-P15 and FFR are encoded in in
		204	* task->thread.sve_state, formatted appropriately for vector
		205	* length task->thread.sve_vl.
		206	*
		207	* task->thread.sve_state must point to a valid buffer at least
		208	* sve_state_size(task) bytes in size.
		209	*
		210	* During any syscall, the kernel may optionally clear TIF_SVE and
		211	* discard the vector state except for the FPSIMD subset.
		212	*
		213	* * TIF_SVE clear:
		214	*
		215	* An attempt by the user task to execute an SVE instruction causes
		216	* do_sve_acc() to be called, which does some preparation and then
		217	* sets TIF_SVE.
		218	*
		219	* When stored, FPSIMD registers V0-V31 are encoded in
		220	* task->fpsimd_state; bits [max : 128] for each of Z0-Z31 are
		221	* logically zero but not stored anywhere; P0-P15 and FFR are not
		222	* stored and have unspecified values from userspace's point of
		223	* view. For hygiene purposes, the kernel zeroes them on next use,
		224	* but userspace is discouraged from relying on this.
		225	*
		226	* task->thread.sve_state does not need to be non-NULL, valid or any
		227	* particular size: it must not be dereferenced.
		228	*
		229	* * FPSR and FPCR are always stored in task->fpsimd_state irrespctive of
		230	* whether TIF_SVE is clear or set, since these are not vector length
		231	* dependent.
		232	*/
		233
		234	/*
		235	* Update current's FPSIMD/SVE registers from thread_struct.
		236	*
		237	* This function should be called only when the FPSIMD/SVE state in
		238	* thread_struct is known to be up to date, when preparing to enter
		239	* userspace.
		240	*
		241	* Softirqs (and preemption) must be disabled.
		242	*/
		243	static void task_fpsimd_load(void)
		244	{
		245	WARN_ON(!in_softirq() && !irqs_disabled());
		246
		247	if (system_supports_sve() && test_thread_flag(TIF_SVE))
		248	sve_load_state(sve_pffr(current),
		249	&current->thread.fpsimd_state.fpsr,
		250	sve_vq_from_vl(current->thread.sve_vl) - 1);
		251	else
		252	fpsimd_load_state(&current->thread.fpsimd_state);
		253
		254	if (system_supports_sve()) {
		255	/* Toggle SVE trapping for userspace if needed */
		256	if (test_thread_flag(TIF_SVE))
		257	sve_user_enable();
		258	else
		259	sve_user_disable();
		260
		261	/* Serialised by exception return to user */
		262	}
		263	}
		264
		265	/*
		266	* Ensure current's FPSIMD/SVE storage in thread_struct is up to date
		267	* with respect to the CPU registers.
		268	*
		269	* Softirqs (and preemption) must be disabled.
		270	*/
		271	static void task_fpsimd_save(void)
		272	{
		273	WARN_ON(!in_softirq() && !irqs_disabled());
		274
		275	if (!test_thread_flag(TIF_FOREIGN_FPSTATE)) {
		276	if (system_supports_sve() && test_thread_flag(TIF_SVE)) {
		277	if (WARN_ON(sve_get_vl() != current->thread.sve_vl)) {
		278	/*
		279	* Can't save the user regs, so current would
		280	* re-enter user with corrupt state.
		281	* There's no way to recover, so kill it:
		282	*/
		283	force_signal_inject(
		284	SIGKILL, 0, current_pt_regs(), 0);
		285	return;
		286	}
		287
		288	sve_save_state(sve_pffr(current),
		289	&current->thread.fpsimd_state.fpsr);
		290	} else
		291	fpsimd_save_state(&current->thread.fpsimd_state);
		292	}
		293	}
		294
		295	/*
		296	* Helpers to translate bit indices in sve_vq_map to VQ values (and
		297	* vice versa). This allows find_next_bit() to be used to find the
		298	* _maximum_ VQ not exceeding a certain value.
		299	*/
		300
		301	static unsigned int vq_to_bit(unsigned int vq)
		302	{
		303	return SVE_VQ_MAX - vq;
		304	}
		305
		306	static unsigned int bit_to_vq(unsigned int bit)
		307	{
		308	if (WARN_ON(bit >= SVE_VQ_MAX))
		309	bit = SVE_VQ_MAX - 1;
		310
		311	return SVE_VQ_MAX - bit;
		312	}
		313
		314	/*
		315	* All vector length selection from userspace comes through here.
		316	* We're on a slow path, so some sanity-checks are included.
		317	* If things go wrong there's a bug somewhere, but try to fall back to a
		318	* safe choice.
		319	*/
		320	static unsigned int find_supported_vector_length(unsigned int vl)
		321	{
		322	int bit;
		323	int max_vl = sve_max_vl;
		324
		325	if (WARN_ON(!sve_vl_valid(vl)))
		326	vl = SVE_VL_MIN;
		327
		328	if (WARN_ON(!sve_vl_valid(max_vl)))
		329	max_vl = SVE_VL_MIN;
		330
		331	if (vl > max_vl)
		332	vl = max_vl;
		333
		334	bit = find_next_bit(sve_vq_map, SVE_VQ_MAX,
		335	vq_to_bit(sve_vq_from_vl(vl)));
		336	return sve_vl_from_vq(bit_to_vq(bit));
		337	}
		338
		339	#ifdef CONFIG_SYSCTL
		340
		341	static int sve_proc_do_default_vl(struct ctl_table *table, int write,
		342	void __user buffer, size_t lenp,
		343	loff_t *ppos)
		344	{
		345	int ret;
		346	int vl = sve_default_vl;
		347	struct ctl_table tmp_table = {
		348	.data = &vl,
		349	.maxlen = sizeof(vl),
		350	};
		351
		352	ret = proc_dointvec(&tmp_table, write, buffer, lenp, ppos);
		353	if (ret \|\| !write)
		354	return ret;
		355
		356	/* Writing -1 has the special meaning "set to max": */
		357	if (vl == -1) {
		358	/* Fail safe if sve_max_vl wasn't initialised */
		359	if (WARN_ON(!sve_vl_valid(sve_max_vl)))
		360	vl = SVE_VL_MIN;
		361	else
		362	vl = sve_max_vl;
		363
		364	goto chosen;
		365	}
		366
		367	if (!sve_vl_valid(vl))
		368	return -EINVAL;
		369
		370	vl = find_supported_vector_length(vl);
		371	chosen:
		372	sve_default_vl = vl;
		373	return 0;
		374	}
		375
		376	static struct ctl_table sve_default_vl_table[] = {
		377	{
		378	.procname = "sve_default_vector_length",
		379	.mode = 0644,
		380	.proc_handler = sve_proc_do_default_vl,
		381	},
		382	{ }
		383	};
		384
		385	static int __init sve_sysctl_init(void)
		386	{
		387	if (system_supports_sve())
		388	if (!register_sysctl("abi", sve_default_vl_table))
		389	return -EINVAL;
		390
		391	return 0;
		392	}
		393
		394	#else /* ! CONFIG_SYSCTL */
		395	static int __init sve_sysctl_init(void) { return 0; }
		396	#endif /* ! CONFIG_SYSCTL */
		397
		398	#define ZREG(sve_state, vq, n) ((char *)(sve_state) + \
		399	(SVE_SIG_ZREG_OFFSET(vq, n) - SVE_SIG_REGS_OFFSET))
		400
		401	/*
		402	* Transfer the FPSIMD state in task->thread.fpsimd_state to
		403	* task->thread.sve_state.
		404	*
		405	* Task can be a non-runnable task, or current. In the latter case,
		406	* softirqs (and preemption) must be disabled.
		407	* task->thread.sve_state must point to at least sve_state_size(task)
		408	* bytes of allocated kernel memory.
		409	* task->thread.fpsimd_state must be up to date before calling this function.
		410	*/
		411	static void fpsimd_to_sve(struct task_struct *task)
		412	{
		413	unsigned int vq;
		414	void *sst = task->thread.sve_state;
		415	struct fpsimd_state const *fst = &task->thread.fpsimd_state;
		416	unsigned int i;
		417
		418	if (!system_supports_sve())
		419	return;
		420
		421	vq = sve_vq_from_vl(task->thread.sve_vl);
		422	for (i = 0; i < 32; ++i)
		423	memcpy(ZREG(sst, vq, i), &fst->vregs[i],
		424	sizeof(fst->vregs[i]));
		425	}
		426
		427	/*
		428	* Transfer the SVE state in task->thread.sve_state to
		429	* task->thread.fpsimd_state.
		430	*
		431	* Task can be a non-runnable task, or current. In the latter case,
		432	* softirqs (and preemption) must be disabled.
		433	* task->thread.sve_state must point to at least sve_state_size(task)
		434	* bytes of allocated kernel memory.
		435	* task->thread.sve_state must be up to date before calling this function.
		436	*/
		437	static void sve_to_fpsimd(struct task_struct *task)
		438	{
		439	unsigned int vq;
		440	void const *sst = task->thread.sve_state;
		441	struct fpsimd_state *fst = &task->thread.fpsimd_state;
		442	unsigned int i;
		443
		444	if (!system_supports_sve())
		445	return;
		446
		447	vq = sve_vq_from_vl(task->thread.sve_vl);
		448	for (i = 0; i < 32; ++i)
		449	memcpy(&fst->vregs[i], ZREG(sst, vq, i),
		450	sizeof(fst->vregs[i]));
		451	}
		452
		453	#ifdef CONFIG_ARM64_SVE
		454
		455	/*
		456	* Return how many bytes of memory are required to store the full SVE
		457	* state for task, given task's currently configured vector length.
		458	*/
		459	size_t sve_state_size(struct task_struct const *task)
		460	{
		461	return SVE_SIG_REGS_SIZE(sve_vq_from_vl(task->thread.sve_vl));
		462	}
		463
		464	/*
		465	* Ensure that task->thread.sve_state is allocated and sufficiently large.
		466	*
		467	* This function should be used only in preparation for replacing
		468	* task->thread.sve_state with new data. The memory is always zeroed
		469	* here to prevent stale data from showing through: this is done in
		470	* the interest of testability and predictability: except in the
		471	* do_sve_acc() case, there is no ABI requirement to hide stale data
		472	* written previously be task.
		473	*/
		474	void sve_alloc(struct task_struct *task)
		475	{
		476	if (task->thread.sve_state) {
		477	memset(task->thread.sve_state, 0, sve_state_size(current));
		478	return;
		479	}
		480
		481	/* This is a small allocation (maximum ~8KB) and Should Not Fail. */
		482	task->thread.sve_state =
		483	kzalloc(sve_state_size(task), GFP_KERNEL);
		484
		485	/*
		486	* If future SVE revisions can have larger vectors though,
		487	* this may cease to be true:
		488	*/
		489	BUG_ON(!task->thread.sve_state);
		490	}
		491
		492
		493	/*
		494	* Ensure that task->thread.sve_state is up to date with respect to
		495	* the user task, irrespective of when SVE is in use or not.
		496	*
		497	* This should only be called by ptrace. task must be non-runnable.
		498	* task->thread.sve_state must point to at least sve_state_size(task)
		499	* bytes of allocated kernel memory.
		500	*/
		501	void fpsimd_sync_to_sve(struct task_struct *task)
		502	{
		503	if (!test_tsk_thread_flag(task, TIF_SVE))
		504	fpsimd_to_sve(task);
		505	}
		506
		507	/*
		508	* Ensure that task->thread.fpsimd_state is up to date with respect to
		509	* the user task, irrespective of whether SVE is in use or not.
		510	*
		511	* This should only be called by ptrace. task must be non-runnable.
		512	* task->thread.sve_state must point to at least sve_state_size(task)
		513	* bytes of allocated kernel memory.
		514	*/
		515	void sve_sync_to_fpsimd(struct task_struct *task)
		516	{
		517	if (test_tsk_thread_flag(task, TIF_SVE))
		518	sve_to_fpsimd(task);
		519	}
		520
		521	/*
		522	* Ensure that task->thread.sve_state is up to date with respect to
		523	* the task->thread.fpsimd_state.
		524	*
		525	* This should only be called by ptrace to merge new FPSIMD register
		526	* values into a task for which SVE is currently active.
		527	* task must be non-runnable.
		528	* task->thread.sve_state must point to at least sve_state_size(task)
		529	* bytes of allocated kernel memory.
		530	* task->thread.fpsimd_state must already have been initialised with
		531	* the new FPSIMD register values to be merged in.
		532	*/
		533	void sve_sync_from_fpsimd_zeropad(struct task_struct *task)
		534	{
		535	unsigned int vq;
		536	void *sst = task->thread.sve_state;
		537	struct fpsimd_state const *fst = &task->thread.fpsimd_state;
		538	unsigned int i;
		539
		540	if (!test_tsk_thread_flag(task, TIF_SVE))
		541	return;
		542
		543	vq = sve_vq_from_vl(task->thread.sve_vl);
		544
		545	memset(sst, 0, SVE_SIG_REGS_SIZE(vq));
		546
		547	for (i = 0; i < 32; ++i)
		548	memcpy(ZREG(sst, vq, i), &fst->vregs[i],
		549	sizeof(fst->vregs[i]));
		550	}
		551
		552	int sve_set_vector_length(struct task_struct *task,
		553	unsigned long vl, unsigned long flags)
		554	{
		555	if (flags & ~(unsigned long)(PR_SVE_VL_INHERIT \|
		556	PR_SVE_SET_VL_ONEXEC))
		557	return -EINVAL;
		558
		559	if (!sve_vl_valid(vl))
		560	return -EINVAL;
		561
		562	/*
		563	* Clamp to the maximum vector length that VL-agnostic SVE code can
		564	* work with. A flag may be assigned in the future to allow setting
		565	* of larger vector lengths without confusing older software.
		566	*/
		567	if (vl > SVE_VL_ARCH_MAX)
		568	vl = SVE_VL_ARCH_MAX;
		569
		570	vl = find_supported_vector_length(vl);
		571
		572	if (flags & (PR_SVE_VL_INHERIT \|
		573	PR_SVE_SET_VL_ONEXEC))
		574	task->thread.sve_vl_onexec = vl;
		575	else
		576	/* Reset VL to system default on next exec: */
		577	task->thread.sve_vl_onexec = 0;
		578
		579	/* Only actually set the VL if not deferred: */
		580	if (flags & PR_SVE_SET_VL_ONEXEC)
		581	goto out;
		582
		583	if (vl == task->thread.sve_vl)
		584	goto out;
		585
		586	/*
		587	* To ensure the FPSIMD bits of the SVE vector registers are preserved,
		588	* write any live register state back to task_struct, and convert to a
		589	* non-SVE thread.
		590	*/
		591	if (task == current) {
		592	local_bh_disable();
		593
		594	task_fpsimd_save();
		595	set_thread_flag(TIF_FOREIGN_FPSTATE);
		596	}
		597
		598	fpsimd_flush_task_state(task);
		599	if (test_and_clear_tsk_thread_flag(task, TIF_SVE))
		600	sve_to_fpsimd(task);
		601
		602	if (task == current)
		603	local_bh_enable();
		604
		605	/*
		606	* Force reallocation of task SVE state to the correct size
		607	* on next use:
		608	*/
		609	sve_free(task);
		610
		611	task->thread.sve_vl = vl;
		612
		613	out:
		614	if (flags & PR_SVE_VL_INHERIT)
		615	set_tsk_thread_flag(task, TIF_SVE_VL_INHERIT);
		616	else
		617	clear_tsk_thread_flag(task, TIF_SVE_VL_INHERIT);
		618
		619	return 0;
		620	}
		621
		622	/*
		623	* Encode the current vector length and flags for return.
		624	* This is only required for prctl(): ptrace has separate fields
		625	*
		626	* flags are as for sve_set_vector_length().
		627	*/
		628	static int sve_prctl_status(unsigned long flags)
		629	{
		630	int ret;
		631
		632	if (flags & PR_SVE_SET_VL_ONEXEC)
		633	ret = current->thread.sve_vl_onexec;
		634	else
		635	ret = current->thread.sve_vl;
		636
		637	if (test_thread_flag(TIF_SVE_VL_INHERIT))
		638	ret \|= PR_SVE_VL_INHERIT;
		639
		640	return ret;
		641	}
		642
		643	/* PR_SVE_SET_VL */
		644	int sve_set_current_vl(unsigned long arg)
		645	{
		646	unsigned long vl, flags;
		647	int ret;
		648
		649	vl = arg & PR_SVE_VL_LEN_MASK;
		650	flags = arg & ~vl;
		651
		652	if (!system_supports_sve())
		653	return -EINVAL;
		654
		655	ret = sve_set_vector_length(current, vl, flags);
		656	if (ret)
		657	return ret;
		658
		659	return sve_prctl_status(flags);
		660	}
		661
		662	/* PR_SVE_GET_VL */
		663	int sve_get_current_vl(void)
		664	{
		665	if (!system_supports_sve())
		666	return -EINVAL;
		667
		668	return sve_prctl_status(0);
		669	}
		670
		671	/*
		672	* Bitmap for temporary storage of the per-CPU set of supported vector lengths
		673	* during secondary boot.
		674	*/
		675	static DECLARE_BITMAP(sve_secondary_vq_map, SVE_VQ_MAX);
		676
		677	static void sve_probe_vqs(DECLARE_BITMAP(map, SVE_VQ_MAX))
		678	{
		679	unsigned int vq, vl;
		680	unsigned long zcr;
		681
		682	bitmap_zero(map, SVE_VQ_MAX);
		683
		684	zcr = ZCR_ELx_LEN_MASK;
		685	zcr = read_sysreg_s(SYS_ZCR_EL1) & ~zcr;
		686
		687	for (vq = SVE_VQ_MAX; vq >= SVE_VQ_MIN; --vq) {
		688	write_sysreg_s(zcr \| (vq - 1), SYS_ZCR_EL1); /* self-syncing */
		689	vl = sve_get_vl();
		690	vq = sve_vq_from_vl(vl); /* skip intervening lengths */
		691	set_bit(vq_to_bit(vq), map);
		692	}
		693	}
		694
		695	void __init sve_init_vq_map(void)
		696	{
		697	sve_probe_vqs(sve_vq_map);
		698	}
		699
		700	/*
		701	* If we haven't committed to the set of supported VQs yet, filter out
		702	* those not supported by the current CPU.
		703	*/
		704	void sve_update_vq_map(void)
		705	{
		706	sve_probe_vqs(sve_secondary_vq_map);
		707	bitmap_and(sve_vq_map, sve_vq_map, sve_secondary_vq_map, SVE_VQ_MAX);
		708	}
		709
		710	/* Check whether the current CPU supports all VQs in the committed set */
		711	int sve_verify_vq_map(void)
		712	{
		713	int ret = 0;
		714
		715	sve_probe_vqs(sve_secondary_vq_map);
		716	bitmap_andnot(sve_secondary_vq_map, sve_vq_map, sve_secondary_vq_map,
		717	SVE_VQ_MAX);
		718	if (!bitmap_empty(sve_secondary_vq_map, SVE_VQ_MAX)) {
		719	pr_warn("SVE: cpu%d: Required vector length(s) missing\n",
		720	smp_processor_id());
		721	ret = -EINVAL;
		722	}
		723
		724	return ret;
		725	}
		726
		727	static void __init sve_efi_setup(void)
		728	{
		729	if (!IS_ENABLED(CONFIG_EFI))
		730	return;
		731
		732	/*
		733	* alloc_percpu() warns and prints a backtrace if this goes wrong.
		734	* This is evidence of a crippled system and we are returning void,
		735	* so no attempt is made to handle this situation here.
		736	*/
		737	if (!sve_vl_valid(sve_max_vl))
		738	goto fail;
		739
		740	efi_sve_state = __alloc_percpu(
		741	SVE_SIG_REGS_SIZE(sve_vq_from_vl(sve_max_vl)), SVE_VQ_BYTES);
		742	if (!efi_sve_state)
		743	goto fail;
		744
		745	return;
		746
		747	fail:
		748	panic("Cannot allocate percpu memory for EFI SVE save/restore");
		749	}
		750
		751	/*
		752	* Enable SVE for EL1.
		753	* Intended for use by the cpufeatures code during CPU boot.
		754	*/
		755	int sve_kernel_enable(void *__always_unused p)
		756	{
		757	write_sysreg(read_sysreg(CPACR_EL1) \| CPACR_EL1_ZEN_EL1EN, CPACR_EL1);
		758	isb();
		759
		760	return 0;
		761	}
		762
		763	void __init sve_setup(void)
		764	{
		765	u64 zcr;
		766
		767	if (!system_supports_sve())
		768	return;
		769
		770	/*
		771	* The SVE architecture mandates support for 128-bit vectors,
		772	* so sve_vq_map must have at least SVE_VQ_MIN set.
		773	* If something went wrong, at least try to patch it up:
		774	*/
		775	if (WARN_ON(!test_bit(vq_to_bit(SVE_VQ_MIN), sve_vq_map)))
		776	set_bit(vq_to_bit(SVE_VQ_MIN), sve_vq_map);
		777
		778	zcr = read_sanitised_ftr_reg(SYS_ZCR_EL1);
		779	sve_max_vl = sve_vl_from_vq((zcr & ZCR_ELx_LEN_MASK) + 1);
		780
		781	/*
		782	* Sanity-check that the max VL we determined through CPU features
		783	* corresponds properly to sve_vq_map. If not, do our best:
		784	*/
		785	if (WARN_ON(sve_max_vl != find_supported_vector_length(sve_max_vl)))
		786	sve_max_vl = find_supported_vector_length(sve_max_vl);
		787
		788	/*
		789	* For the default VL, pick the maximum supported value <= 64.
		790	* VL == 64 is guaranteed not to grow the signal frame.
		791	*/
		792	sve_default_vl = find_supported_vector_length(64);
		793
		794	pr_info("SVE: maximum available vector length %u bytes per vector\n",
		795	sve_max_vl);
		796	pr_info("SVE: default vector length %u bytes per vector\n",
		797	sve_default_vl);
		798
		799	sve_efi_setup();
		800	}
		801
		802	/*
		803	* Called from the put_task_struct() path, which cannot get here
		804	* unless dead_task is really dead and not schedulable.
		805	*/
		806	void fpsimd_release_task(struct task_struct *dead_task)
		807	{
		808	__sve_free(dead_task);
		809	}
		810
		811	#endif /* CONFIG_ARM64_SVE */
		812
		813	/*
		814	* Trapped SVE access
		815	*
		816	* Storage is allocated for the full SVE state, the current FPSIMD
		817	* register contents are migrated across, and TIF_SVE is set so that
		818	* the SVE access trap will be disabled the next time this task
		819	* reaches ret_to_user.
		820	*
		821	* TIF_SVE should be clear on entry: otherwise, task_fpsimd_load()
		822	* would have disabled the SVE access trap for userspace during
		823	* ret_to_user, making an SVE access trap impossible in that case.
		824	*/
		825	asmlinkage void do_sve_acc(unsigned int esr, struct pt_regs *regs)
		826	{
		827	/* Even if we chose not to use SVE, the hardware could still trap: */
		828	if (unlikely(!system_supports_sve()) \|\| WARN_ON(is_compat_task())) {
		829	force_signal_inject(SIGILL, ILL_ILLOPC, regs, 0);
		830	return;
		831	}
		832
		833	sve_alloc(current);
		834
		835	local_bh_disable();
		836
		837	task_fpsimd_save();
		838	fpsimd_to_sve(current);
		839
		840	/* Force ret_to_user to reload the registers: */
		841	fpsimd_flush_task_state(current);
		842	set_thread_flag(TIF_FOREIGN_FPSTATE);
		843
		844	if (test_and_set_thread_flag(TIF_SVE))
		845	WARN_ON(1); /* SVE access shouldn't have trapped */
		846
		847	local_bh_enable();
		848	}
		849
102	/*	850	/*
103	* Trapped FP/ASIMD access.	851	* Trapped FP/ASIMD access.
104	*/	852	*/
105	void do_fpsimd_acc(unsigned int esr, struct pt_regs *regs)	853	asmlinkage void do_fpsimd_acc(unsigned int esr, struct pt_regs *regs)
106	{	854	{
107	/* TODO: implement lazy context saving/restoring */	855	/* TODO: implement lazy context saving/restoring */
108	WARN_ON(1);	856	WARN_ON(1);
@@ -111,7 +859,7 @@ void do_fpsimd_acc(unsigned int esr, struct pt_regs *regs) · 2 unmodified lines
111	/*	859	/*
112	* Raise a SIGFPE for the current process.	860	* Raise a SIGFPE for the current process.
113	*/	861	*/
114	void do_fpsimd_exc(unsigned int esr, struct pt_regs *regs)	862	asmlinkage void do_fpsimd_exc(unsigned int esr, struct pt_regs *regs)
115	{	863	{
116	siginfo_t info;	864	siginfo_t info;
117	unsigned int si_code = 0;	865	unsigned int si_code = 0;
@@ -144,8 +892,8 @@ void fpsimd_thread_switch(struct task_struct *next) · 26 unmodified lines
144	* the registers is in fact the most recent userland FPSIMD state of	892	* the registers is in fact the most recent userland FPSIMD state of
145	* 'current'.	893	* 'current'.
146	*/	894	*/
147	if (current->mm && !test_thread_flag(TIF_FOREIGN_FPSTATE))	895	if (current->mm)
148	fpsimd_save_state(&current->thread.fpsimd_state);	896	task_fpsimd_save();
149		897
150	if (next->mm) {	898	if (next->mm) {
151	/*	899	/*
@@ -159,16 +907,16 @@ void fpsimd_thread_switch(struct task_struct *next) · 7 unmodified lines
159		907
160	if (__this_cpu_read(fpsimd_last_state) == st	908	if (__this_cpu_read(fpsimd_last_state) == st
161	&& st->cpu == smp_processor_id())	909	&& st->cpu == smp_processor_id())
162	clear_ti_thread_flag(task_thread_info(next),	910	clear_tsk_thread_flag(next, TIF_FOREIGN_FPSTATE);
163	TIF_FOREIGN_FPSTATE);
164	else	911	else
165	set_ti_thread_flag(task_thread_info(next),	912	set_tsk_thread_flag(next, TIF_FOREIGN_FPSTATE);
166	TIF_FOREIGN_FPSTATE);
167	}	913	}
168	}	914	}
169		915
170	void fpsimd_flush_thread(void)	916	void fpsimd_flush_thread(void)
171	{	917	{
		918	int vl, supported_vl;
		919
172	if (!system_supports_fpsimd())	920	if (!system_supports_fpsimd())
173	return;	921	return;
174		922
@@ -176,6 +924,42 @@ void fpsimd_flush_thread(void) · 1 unmodified line
176		924
177	memset(&current->thread.fpsimd_state, 0, sizeof(struct fpsimd_state));	925	memset(&current->thread.fpsimd_state, 0, sizeof(struct fpsimd_state));
178	fpsimd_flush_task_state(current);	926	fpsimd_flush_task_state(current);
		927
		928	if (system_supports_sve()) {
		929	clear_thread_flag(TIF_SVE);
		930	sve_free(current);
		931
		932	/*
		933	* Reset the task vector length as required.
		934	* This is where we ensure that all user tasks have a valid
		935	* vector length configured: no kernel task can become a user
		936	* task without an exec and hence a call to this function.
		937	* By the time the first call to this function is made, all
		938	* early hardware probing is complete, so sve_default_vl
		939	* should be valid.
		940	* If a bug causes this to go wrong, we make some noise and
		941	* try to fudge thread.sve_vl to a safe value here.
		942	*/
		943	vl = current->thread.sve_vl_onexec ?
		944	current->thread.sve_vl_onexec : sve_default_vl;
		945
		946	if (WARN_ON(!sve_vl_valid(vl)))
		947	vl = SVE_VL_MIN;
		948
		949	supported_vl = find_supported_vector_length(vl);
		950	if (WARN_ON(supported_vl != vl))
		951	vl = supported_vl;
		952
		953	current->thread.sve_vl = vl;
		954
		955	/*
		956	* If the task is not set to inherit, ensure that the vector
		957	* length will be reset by a subsequent exec:
		958	*/
		959	if (!test_thread_flag(TIF_SVE_VL_INHERIT))
		960	current->thread.sve_vl_onexec = 0;
		961	}
		962
179	set_thread_flag(TIF_FOREIGN_FPSTATE);	963	set_thread_flag(TIF_FOREIGN_FPSTATE);
180		964
181	local_bh_enable();	965	local_bh_enable();
@@ -191,13 +975,22 @@ void fpsimd_preserve_current_state(void) · 9 unmodified lines
191	return;	975	return;
192		976
193	local_bh_disable();	977	local_bh_disable();
194		978	task_fpsimd_save();
195	if (!test_thread_flag(TIF_FOREIGN_FPSTATE))
196	fpsimd_save_state(&current->thread.fpsimd_state);
197
198	local_bh_enable();	979	local_bh_enable();
199	}	980	}
200		981
		982	/*
		983	* Like fpsimd_preserve_current_state(), but ensure that
		984	* current->thread.fpsimd_state is updated so that it can be copied to
		985	* the signal frame.
		986	*/
		987	void fpsimd_signal_preserve_current_state(void)
		988	{
		989	fpsimd_preserve_current_state();
		990	if (system_supports_sve() && test_thread_flag(TIF_SVE))
		991	sve_to_fpsimd(current);
		992	}
		993
201	/*	994	/*
202	* Load the userland FPSIMD state of 'current' from memory, but only if the	995	* Load the userland FPSIMD state of 'current' from memory, but only if the
203	* FPSIMD state already held in the registers is /not/ the most recent FPSIMD	996	* FPSIMD state already held in the registers is /not/ the most recent FPSIMD
@@ -213,7 +1006,7 @@ void fpsimd_restore_current_state(void) · 9 unmodified lines
213	if (test_and_clear_thread_flag(TIF_FOREIGN_FPSTATE)) {	1006	if (test_and_clear_thread_flag(TIF_FOREIGN_FPSTATE)) {
214	struct fpsimd_state *st = &current->thread.fpsimd_state;	1007	struct fpsimd_state *st = &current->thread.fpsimd_state;
215		1008
216	fpsimd_load_state(st);	1009	task_fpsimd_load();
217	__this_cpu_write(fpsimd_last_state, st);	1010	__this_cpu_write(fpsimd_last_state, st);
218	st->cpu = smp_processor_id();	1011	st->cpu = smp_processor_id();
219	}	1012	}
@@ -233,7 +1026,12 @@ void fpsimd_update_current_state(struct fpsimd_state *state) · 13 unmodified lines
233		1026
234	local_bh_disable();	1027	local_bh_disable();
235		1028
236	fpsimd_load_state(state);	1029	if (system_supports_sve() && test_thread_flag(TIF_SVE)) {
		1030	current->thread.fpsimd_state = *state;
		1031	fpsimd_to_sve(current);
		1032	}
		1033	task_fpsimd_load();
		1034
237	if (test_and_clear_thread_flag(TIF_FOREIGN_FPSTATE)) {	1035	if (test_and_clear_thread_flag(TIF_FOREIGN_FPSTATE)) {
238	struct fpsimd_state *st = &current->thread.fpsimd_state;	1036	struct fpsimd_state *st = &current->thread.fpsimd_state;
239		1037
@@ -252,6 +1050,33 @@ void fpsimd_flush_task_state(struct task_struct *t) · 12 unmodified lines
252	t->thread.fpsimd_state.cpu = NR_CPUS;	1050	t->thread.fpsimd_state.cpu = NR_CPUS;
253	}	1051	}
254		1052
		1053	static inline void fpsimd_flush_cpu_state(void)
		1054	{
		1055	__this_cpu_write(fpsimd_last_state, NULL);
		1056	}
		1057
		1058	/*
		1059	* Invalidate any task SVE state currently held in this CPU's regs.
		1060	*
		1061	* This is used to prevent the kernel from trying to reuse SVE register data
		1062	* that is detroyed by KVM guest enter/exit. This function should go away when
		1063	* KVM SVE support is implemented. Don't use it for anything else.
		1064	*/
		1065	#ifdef CONFIG_ARM64_SVE
		1066	void sve_flush_cpu_state(void)
		1067	{
		1068	struct fpsimd_state *const fpstate = __this_cpu_read(fpsimd_last_state);
		1069	struct task_struct *tsk;
		1070
		1071	if (!fpstate)
		1072	return;
		1073
		1074	tsk = container_of(fpstate, struct task_struct, thread.fpsimd_state);
		1075	if (test_tsk_thread_flag(tsk, TIF_SVE))
		1076	fpsimd_flush_cpu_state();
		1077	}
		1078	#endif /* CONFIG_ARM64_SVE */
		1079
255	#ifdef CONFIG_KERNEL_MODE_NEON	1080	#ifdef CONFIG_KERNEL_MODE_NEON
256		1081
257	DEFINE_PER_CPU(bool, kernel_neon_busy);	1082	DEFINE_PER_CPU(bool, kernel_neon_busy);
@@ -286,11 +1111,13 @@ void kernel_neon_begin(void) · 28 unmodified lines
286	__this_cpu_write(kernel_neon_busy, true);	1111	__this_cpu_write(kernel_neon_busy, true);
287		1112
288	/* Save unsaved task fpsimd state, if any: */	1113	/* Save unsaved task fpsimd state, if any: */
289	if (current->mm && !test_and_set_thread_flag(TIF_FOREIGN_FPSTATE))	1114	if (current->mm) {
290	fpsimd_save_state(&current->thread.fpsimd_state);	1115	task_fpsimd_save();
		1116	set_thread_flag(TIF_FOREIGN_FPSTATE);
		1117	}
291		1118
292	/* Invalidate any task state remaining in the fpsimd regs: */	1119	/* Invalidate any task state remaining in the fpsimd regs: */
293	__this_cpu_write(fpsimd_last_state, NULL);	1120	fpsimd_flush_cpu_state();
294		1121
295	preempt_disable();	1122	preempt_disable();
296		1123
@@ -325,6 +1152,7 @@ EXPORT_SYMBOL(kernel_neon_end); · 28 unmodified lines
325		1152
326	static DEFINE_PER_CPU(struct fpsimd_state, efi_fpsimd_state);	1153	static DEFINE_PER_CPU(struct fpsimd_state, efi_fpsimd_state);
327	static DEFINE_PER_CPU(bool, efi_fpsimd_state_used);	1154	static DEFINE_PER_CPU(bool, efi_fpsimd_state_used);
		1155	static DEFINE_PER_CPU(bool, efi_sve_state_used);
328		1156
329	/*	1157	/*
330	* EFI runtime services support functions	1158	* EFI runtime services support functions
@@ -350,10 +1178,24 @@ void __efi_fpsimd_begin(void) · 19 unmodified lines
350		1178
351	WARN_ON(preemptible());	1179	WARN_ON(preemptible());
352		1180
353	if (may_use_simd())	1181	if (may_use_simd()) {
354	kernel_neon_begin();	1182	kernel_neon_begin();
355	else {	1183	} else {
356	fpsimd_save_state(this_cpu_ptr(&efi_fpsimd_state));	1184	/*
		1185	* If !efi_sve_state, SVE can't be in use yet and doesn't need
		1186	* preserving:
		1187	*/
		1188	if (system_supports_sve() && likely(efi_sve_state)) {
		1189	char *sve_state = this_cpu_ptr(efi_sve_state);
		1190
		1191	__this_cpu_write(efi_sve_state_used, true);
		1192
		1193	sve_save_state(sve_state + sve_ffr_offset(sve_max_vl),
		1194	&this_cpu_ptr(&efi_fpsimd_state)->fpsr);
		1195	} else {
		1196	fpsimd_save_state(this_cpu_ptr(&efi_fpsimd_state));
		1197	}
		1198
357	__this_cpu_write(efi_fpsimd_state_used, true);	1199	__this_cpu_write(efi_fpsimd_state_used, true);
358	}	1200	}
359	}	1201	}
@@ -366,10 +1208,22 @@ void __efi_fpsimd_end(void) · 6 unmodified lines
366	if (!system_supports_fpsimd())	1208	if (!system_supports_fpsimd())
367	return;	1209	return;
368		1210
369	if (__this_cpu_xchg(efi_fpsimd_state_used, false))	1211	if (!__this_cpu_xchg(efi_fpsimd_state_used, false)) {
370	fpsimd_load_state(this_cpu_ptr(&efi_fpsimd_state));
371	else
372	kernel_neon_end();	1212	kernel_neon_end();
		1213	} else {
		1214	if (system_supports_sve() &&
		1215	likely(__this_cpu_read(efi_sve_state_used))) {
		1216	char const *sve_state = this_cpu_ptr(efi_sve_state);
		1217
		1218	sve_load_state(sve_state + sve_ffr_offset(sve_max_vl),
		1219	&this_cpu_ptr(&efi_fpsimd_state)->fpsr,
		1220	sve_vq_from_vl(sve_get_vl()) - 1);
		1221
		1222	__this_cpu_write(efi_sve_state_used, false);
		1223	} else {
		1224	fpsimd_load_state(this_cpu_ptr(&efi_fpsimd_state));
		1225	}
		1226	}
373	}	1227	}
374		1228
375	#endif /* CONFIG_EFI */	1229	#endif /* CONFIG_EFI */
@@ -382,9 +1236,9 @@ static int fpsimd_cpu_pm_notifier(struct notifier_block *self, · 6 unmodified lines
382	{	1236	{
383	switch (cmd) {	1237	switch (cmd) {
384	case CPU_PM_ENTER:	1238	case CPU_PM_ENTER:
385	if (current->mm && !test_thread_flag(TIF_FOREIGN_FPSTATE))	1239	if (current->mm)
386	fpsimd_save_state(&current->thread.fpsimd_state);	1240	task_fpsimd_save();
387	this_cpu_write(fpsimd_last_state, NULL);	1241	fpsimd_flush_cpu_state();
388	break;	1242	break;
389	case CPU_PM_EXIT:	1243	case CPU_PM_EXIT:
390	if (current->mm)	1244	if (current->mm)
@@ -442,6 +1296,6 @@ static int __init fpsimd_init(void) · 51 unmodified lines
442	if (!(elf_hwcap & HWCAP_ASIMD))	1296	if (!(elf_hwcap & HWCAP_ASIMD))
443	pr_notice("Advanced SIMD is not implemented\n");	1297	pr_notice("Advanced SIMD is not implemented\n");
444		1298
445	return 0;	1299	return sve_sysctl_init();
446	}	1300	}
447	core_initcall(fpsimd_init);	1301	core_initcall(fpsimd_init);

arch/arm64/kernel/head.S+24−6

 	/* Statistical profiling */
 	ubfx	x0, x1, #32, #4			// Check ID_AA64DFR0_EL1 PMSVer
-	cbz	x0, 6f				// Skip if SPE not present
+	cbz	x0, 7f				// Skip if SPE not present
-	cbnz	x2, 5f				// VHE?
+	cbnz	x2, 6f				// VHE?
+	mrs_s	x4, SYS_PMBIDR_EL1		// If SPE available at EL2,
+	and	x4, x4, #(1 << SYS_PMBIDR_EL1_P_SHIFT)
+	cbnz	x4, 5f				// then permit sampling of physical
+	mov	x4, #(1 << SYS_PMSCR_EL2_PCT_SHIFT | \
+		      1 << SYS_PMSCR_EL2_PA_SHIFT)
+	msr_s	SYS_PMSCR_EL2, x4		// addresses and physical counter
+5:
 	mov	x1, #(MDCR_EL2_E2PB_MASK << MDCR_EL2_E2PB_SHIFT)
 	orr	x3, x3, x1			// If we don't have VHE, then
-	b	6f				// use EL1&0 translation.
+	b	7f				// use EL1&0 translation.
-5:						// For VHE, use EL2 translation
+6:						// For VHE, use EL2 translation
 	orr	x3, x3, #MDCR_EL2_TPMS		// and disable access from EL1
-6:
+7:
 	msr	mdcr_el2, x3			// Configure debug traps
 	/* Stage-2 translation */
 	mov	x0, #0x33ff
 	msr	cptr_el2, x0			// Disable copro. traps to EL2
+	/* SVE register access */
+	mrs	x1, id_aa64pfr0_el1
+	ubfx	x1, x1, #ID_AA64PFR0_SVE_SHIFT, #4
+	cbz	x1, 7f
+	bic	x0, x0, #CPTR_EL2_TZ		// Also disable SVE traps
+	msr	cptr_el2, x0			// Disable copro. traps to EL2
+	isb
+	mov	x1, #ZCR_ELx_LEN_MASK		// SVE: Enable full vector
+	msr_s	SYS_ZCR_EL2, x1			// length for EL1.
 	/* Hypervisor stub */
-	adr_l	x0, __hyp_stub_vectors
+7:	adr_l	x0, __hyp_stub_vectors
 	msr	vbar_el2, x0
 	/* spsr */

arch/arm64/kernel/hibernate.c+3−2

 #include <asm/barrier.h>
 #include <asm/cacheflush.h>
 #include <asm/cputype.h>
+#include <asm/daifflags.h>
 #include <asm/irqflags.h>
 #include <asm/kexec.h>
 #include <asm/memory.h>
 		return -EBUSY;
 	}
-	local_dbg_save(flags);
+	flags = local_daif_save();
 	if (__cpu_suspend_enter(&state)) {
 		/* make the crash dump kernel image visible/saveable */
 		__cpu_suspend_exit();
 	}
-	local_dbg_restore(flags);
+	local_daif_restore(flags);
 	return ret;
 }

arch/arm64/kernel/io.c+5−7

  */
 void __memcpy_fromio(void *to, const volatile void __iomem *from, size_t count)
 {
-	while (count && (!IS_ALIGNED((unsigned long)from, 8) ||
+	while (count && !IS_ALIGNED((unsigned long)from, 8)) {
-			 !IS_ALIGNED((unsigned long)to, 8))) {
 		*(u8 *)to = __raw_readb(from);
 		from++;
 		to++;
  */
 void __memcpy_toio(volatile void __iomem *to, const void *from, size_t count)
 {
-	while (count && (!IS_ALIGNED((unsigned long)to, 8) ||
+	while (count && !IS_ALIGNED((unsigned long)to, 8)) {
-			 !IS_ALIGNED((unsigned long)from, 8))) {
+		__raw_writeb(*(u8 *)from, to);
-		__raw_writeb(*(volatile u8 *)from, to);
 		from++;
 		to++;
 		count--;
 	}
 	while (count >= 8) {
-		__raw_writeq(*(volatile u64 *)from, to);
+		__raw_writeq(*(u64 *)from, to);
 		from += 8;
 		to += 8;
 		count -= 8;
 	}
 	while (count) {
-		__raw_writeb(*(volatile u8 *)from, to);
+		__raw_writeb(*(u8 *)from, to);
 		from++;
 		to++;
 		count--;

arch/arm64/kernel/machine_kexec.c+2−2

 #include <asm/cacheflush.h>
 #include <asm/cpu_ops.h>
+#include <asm/daifflags.h>
 #include <asm/memory.h>
 #include <asm/mmu.h>
 #include <asm/mmu_context.h>
 	pr_info("Bye!\n");
-	/* Disable all DAIF exceptions. */
+	local_daif_mask();
-	asm volatile ("msr daifset, #0xf" : : : "memory");
 	/*
 	 * cpu_soft_restart will shutdown the MMU, disable data caches, then

arch/arm64/kernel/process.c+60−4

@@ -49,6 +49,7 @@ · 48 unmodified lines
49	#include <linux/notifier.h>	49	#include <linux/notifier.h>
50	#include <trace/events/power.h>	50	#include <trace/events/power.h>
51	#include <linux/percpu.h>	51	#include <linux/percpu.h>
		52	#include <linux/thread_info.h>
52		53
53	#include <asm/alternative.h>	54	#include <asm/alternative.h>
54	#include <asm/compat.h>	55	#include <asm/compat.h>
@@ -170,6 +171,39 @@ void machine_restart(char *cmd) · 115 unmodified lines
170	while (1);	171	while (1);
171	}	172	}
172		173
		174	static void print_pstate(struct pt_regs *regs)
		175	{
		176	u64 pstate = regs->pstate;
		177
		178	if (compat_user_mode(regs)) {
		179	printk("pstate: %08llx (%c%c%c%c %c %s %s %c%c%c)\n",
		180	pstate,
		181	pstate & COMPAT_PSR_N_BIT ? 'N' : 'n',
		182	pstate & COMPAT_PSR_Z_BIT ? 'Z' : 'z',
		183	pstate & COMPAT_PSR_C_BIT ? 'C' : 'c',
		184	pstate & COMPAT_PSR_V_BIT ? 'V' : 'v',
		185	pstate & COMPAT_PSR_Q_BIT ? 'Q' : 'q',
		186	pstate & COMPAT_PSR_T_BIT ? "T32" : "A32",
		187	pstate & COMPAT_PSR_E_BIT ? "BE" : "LE",
		188	pstate & COMPAT_PSR_A_BIT ? 'A' : 'a',
		189	pstate & COMPAT_PSR_I_BIT ? 'I' : 'i',
		190	pstate & COMPAT_PSR_F_BIT ? 'F' : 'f');
		191	} else {
		192	printk("pstate: %08llx (%c%c%c%c %c%c%c%c %cPAN %cUAO)\n",
		193	pstate,
		194	pstate & PSR_N_BIT ? 'N' : 'n',
		195	pstate & PSR_Z_BIT ? 'Z' : 'z',
		196	pstate & PSR_C_BIT ? 'C' : 'c',
		197	pstate & PSR_V_BIT ? 'V' : 'v',
		198	pstate & PSR_D_BIT ? 'D' : 'd',
		199	pstate & PSR_A_BIT ? 'A' : 'a',
		200	pstate & PSR_I_BIT ? 'I' : 'i',
		201	pstate & PSR_F_BIT ? 'F' : 'f',
		202	pstate & PSR_PAN_BIT ? '+' : '-',
		203	pstate & PSR_UAO_BIT ? '+' : '-');
		204	}
		205	}
		206
173	void __show_regs(struct pt_regs *regs)	207	void __show_regs(struct pt_regs *regs)
174	{	208	{
175	int i, top_reg;	209	int i, top_reg;
@@ -186,10 +220,9 @@ void __show_regs(struct pt_regs *regs) · 10 unmodified lines
186	}	220	}
187		221
188	show_regs_print_info(KERN_DEFAULT);	222	show_regs_print_info(KERN_DEFAULT);
189	print_symbol("PC is at %s\n", instruction_pointer(regs));	223	print_pstate(regs);
190	print_symbol("LR is at %s\n", lr);	224	print_symbol("pc : %s\n", regs->pc);
191	printk("pc : [<%016llx>] lr : [<%016llx>] pstate: %08llx\n",	225	print_symbol("lr : %s\n", lr);
192	regs->pc, lr, regs->pstate);
193	printk("sp : %016llx\n", sp);	226	printk("sp : %016llx\n", sp);
194		227
195	i = top_reg;	228	i = top_reg;
@@ -241,11 +274,27 @@ void release_thread(struct task_struct *dead_task) · 45 unmodified lines
241	{	274	{
242	}	275	}
243		276
		277	void arch_release_task_struct(struct task_struct *tsk)
		278	{
		279	fpsimd_release_task(tsk);
		280	}
		281
		282	/*
		283	* src and dst may temporarily have aliased sve_state after task_struct
		284	* is copied. We cannot fix this properly here, because src may have
		285	* live SVE state and dst's thread_info may not exist yet, so tweaking
		286	* either src's or dst's TIF_SVE is not safe.
		287	*
		288	* The unaliasing is done in copy_thread() instead. This works because
		289	* dst is not schedulable or traceable until both of these functions
		290	* have been called.
		291	*/
244	int arch_dup_task_struct(struct task_struct dst, struct task_struct src)	292	int arch_dup_task_struct(struct task_struct dst, struct task_struct src)
245	{	293	{
246	if (current->mm)	294	if (current->mm)
247	fpsimd_preserve_current_state();	295	fpsimd_preserve_current_state();
248	dst = src;	296	dst = src;
		297
249	return 0;	298	return 0;
250	}	299	}
251		300
@@ -258,6 +307,13 @@ int copy_thread(unsigned long clone_flags, unsigned long stack_start, · 6 unmodified lines
258		307
259	memset(&p->thread.cpu_context, 0, sizeof(struct cpu_context));	308	memset(&p->thread.cpu_context, 0, sizeof(struct cpu_context));
260		309
		310	/*
		311	* Unalias p->thread.sve_state (if any) from the parent task
		312	* and disable discard SVE state for p:
		313	*/
		314	clear_tsk_thread_flag(p, TIF_SVE);
		315	p->thread.sve_state = NULL;
		316
261	if (likely(!(p->flags & PF_KTHREAD))) {	317	if (likely(!(p->flags & PF_KTHREAD))) {
262	childregs = current_pt_regs();	318	childregs = current_pt_regs();
263	childregs->regs[0] = 0;	319	childregs->regs[0] = 0;

arch/arm64/kernel/ptrace.c+272−8

@@ -32,6 +32,7 @@ · 31 unmodified lines
32	#include <linux/security.h>	32	#include <linux/security.h>
33	#include <linux/init.h>	33	#include <linux/init.h>
34	#include <linux/signal.h>	34	#include <linux/signal.h>
		35	#include <linux/string.h>
35	#include <linux/uaccess.h>	36	#include <linux/uaccess.h>
36	#include <linux/perf_event.h>	37	#include <linux/perf_event.h>
37	#include <linux/hw_breakpoint.h>	38	#include <linux/hw_breakpoint.h>
@@ -40,6 +41,7 @@ · 2 unmodified lines
40	#include <linux/elf.h>	41	#include <linux/elf.h>
41		42
42	#include <asm/compat.h>	43	#include <asm/compat.h>
		44	#include <asm/cpufeature.h>
43	#include <asm/debug-monitors.h>	45	#include <asm/debug-monitors.h>
44	#include <asm/pgtable.h>	46	#include <asm/pgtable.h>
45	#include <asm/stacktrace.h>	47	#include <asm/stacktrace.h>
@@ -618,17 +620,56 @@ static int gpr_set(struct task_struct target, const struct user_regset regset, · 572 unmodified lines
618	/*	620	/*
619	* TODO: update fp accessors for lazy context switching (sync/flush hwstate)	621	* TODO: update fp accessors for lazy context switching (sync/flush hwstate)
620	*/	622	*/
621	static int fpr_get(struct task_struct target, const struct user_regset regset,	623	static int __fpr_get(struct task_struct *target,
622	unsigned int pos, unsigned int count,	624	const struct user_regset *regset,
623	void kbuf, void __user ubuf)	625	unsigned int pos, unsigned int count,
		626	void kbuf, void __user ubuf, unsigned int start_pos)
624	{	627	{
625	struct user_fpsimd_state *uregs;	628	struct user_fpsimd_state *uregs;
		629
		630	sve_sync_to_fpsimd(target);
		631
626	uregs = &target->thread.fpsimd_state.user_fpsimd;	632	uregs = &target->thread.fpsimd_state.user_fpsimd;
627		633
		634	return user_regset_copyout(&pos, &count, &kbuf, &ubuf, uregs,
		635	start_pos, start_pos + sizeof(*uregs));
		636	}
		637
		638	static int fpr_get(struct task_struct target, const struct user_regset regset,
		639	unsigned int pos, unsigned int count,
		640	void kbuf, void __user ubuf)
		641	{
628	if (target == current)	642	if (target == current)
629	fpsimd_preserve_current_state();	643	fpsimd_preserve_current_state();
630		644
631	return user_regset_copyout(&pos, &count, &kbuf, &ubuf, uregs, 0, -1);	645	return __fpr_get(target, regset, pos, count, kbuf, ubuf, 0);
		646	}
		647
		648	static int __fpr_set(struct task_struct *target,
		649	const struct user_regset *regset,
		650	unsigned int pos, unsigned int count,
		651	const void kbuf, const void __user ubuf,
		652	unsigned int start_pos)
		653	{
		654	int ret;
		655	struct user_fpsimd_state newstate;
		656
		657	/*
		658	* Ensure target->thread.fpsimd_state is up to date, so that a
		659	* short copyin can't resurrect stale data.
		660	*/
		661	sve_sync_to_fpsimd(target);
		662
		663	newstate = target->thread.fpsimd_state.user_fpsimd;
		664
		665	ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &newstate,
		666	start_pos, start_pos + sizeof(newstate));
		667	if (ret)
		668	return ret;
		669
		670	target->thread.fpsimd_state.user_fpsimd = newstate;
		671
		672	return ret;
632	}	673	}
633		674
634	static int fpr_set(struct task_struct target, const struct user_regset regset,	675	static int fpr_set(struct task_struct target, const struct user_regset regset,
@@ -636,15 +677,14 @@ static int fpr_set(struct task_struct target, const struct user_regset regset, · 1 unmodified line
636	const void kbuf, const void __user ubuf)	677	const void kbuf, const void __user ubuf)
637	{	678	{
638	int ret;	679	int ret;
639	struct user_fpsimd_state newstate =
640	target->thread.fpsimd_state.user_fpsimd;
641		680
642	ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &newstate, 0, -1);	681	ret = __fpr_set(target, regset, pos, count, kbuf, ubuf, 0);
643	if (ret)	682	if (ret)
644	return ret;	683	return ret;
645		684
646	target->thread.fpsimd_state.user_fpsimd = newstate;	685	sve_sync_from_fpsimd_zeropad(target);
647	fpsimd_flush_task_state(target);	686	fpsimd_flush_task_state(target);
		687
648	return ret;	688	return ret;
649	}	689	}
650		690
@@ -702,6 +742,215 @@ static int system_call_set(struct task_struct *target, · 51 unmodified lines
702	return ret;	742	return ret;
703	}	743	}
704		744
		745	#ifdef CONFIG_ARM64_SVE
		746
		747	static void sve_init_header_from_task(struct user_sve_header *header,
		748	struct task_struct *target)
		749	{
		750	unsigned int vq;
		751
		752	memset(header, 0, sizeof(*header));
		753
		754	header->flags = test_tsk_thread_flag(target, TIF_SVE) ?
		755	SVE_PT_REGS_SVE : SVE_PT_REGS_FPSIMD;
		756	if (test_tsk_thread_flag(target, TIF_SVE_VL_INHERIT))
		757	header->flags \|= SVE_PT_VL_INHERIT;
		758
		759	header->vl = target->thread.sve_vl;
		760	vq = sve_vq_from_vl(header->vl);
		761
		762	header->max_vl = sve_max_vl;
		763	if (WARN_ON(!sve_vl_valid(sve_max_vl)))
		764	header->max_vl = header->vl;
		765
		766	header->size = SVE_PT_SIZE(vq, header->flags);
		767	header->max_size = SVE_PT_SIZE(sve_vq_from_vl(header->max_vl),
		768	SVE_PT_REGS_SVE);
		769	}
		770
		771	static unsigned int sve_size_from_header(struct user_sve_header const *header)
		772	{
		773	return ALIGN(header->size, SVE_VQ_BYTES);
		774	}
		775
		776	static unsigned int sve_get_size(struct task_struct *target,
		777	const struct user_regset *regset)
		778	{
		779	struct user_sve_header header;
		780
		781	if (!system_supports_sve())
		782	return 0;
		783
		784	sve_init_header_from_task(&header, target);
		785	return sve_size_from_header(&header);
		786	}
		787
		788	static int sve_get(struct task_struct *target,
		789	const struct user_regset *regset,
		790	unsigned int pos, unsigned int count,
		791	void kbuf, void __user ubuf)
		792	{
		793	int ret;
		794	struct user_sve_header header;
		795	unsigned int vq;
		796	unsigned long start, end;
		797
		798	if (!system_supports_sve())
		799	return -EINVAL;
		800
		801	/* Header */
		802	sve_init_header_from_task(&header, target);
		803	vq = sve_vq_from_vl(header.vl);
		804
		805	ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf, &header,
		806	0, sizeof(header));
		807	if (ret)
		808	return ret;
		809
		810	if (target == current)
		811	fpsimd_preserve_current_state();
		812
		813	/* Registers: FPSIMD-only case */
		814
		815	BUILD_BUG_ON(SVE_PT_FPSIMD_OFFSET != sizeof(header));
		816	if ((header.flags & SVE_PT_REGS_MASK) == SVE_PT_REGS_FPSIMD)
		817	return __fpr_get(target, regset, pos, count, kbuf, ubuf,
		818	SVE_PT_FPSIMD_OFFSET);
		819
		820	/* Otherwise: full SVE case */
		821
		822	BUILD_BUG_ON(SVE_PT_SVE_OFFSET != sizeof(header));
		823	start = SVE_PT_SVE_OFFSET;
		824	end = SVE_PT_SVE_FFR_OFFSET(vq) + SVE_PT_SVE_FFR_SIZE(vq);
		825	ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf,
		826	target->thread.sve_state,
		827	start, end);
		828	if (ret)
		829	return ret;
		830
		831	start = end;
		832	end = SVE_PT_SVE_FPSR_OFFSET(vq);
		833	ret = user_regset_copyout_zero(&pos, &count, &kbuf, &ubuf,
		834	start, end);
		835	if (ret)
		836	return ret;
		837
		838	/*
		839	* Copy fpsr, and fpcr which must follow contiguously in
		840	* struct fpsimd_state:
		841	*/
		842	start = end;
		843	end = SVE_PT_SVE_FPCR_OFFSET(vq) + SVE_PT_SVE_FPCR_SIZE;
		844	ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf,
		845	&target->thread.fpsimd_state.fpsr,
		846	start, end);
		847	if (ret)
		848	return ret;
		849
		850	start = end;
		851	end = sve_size_from_header(&header);
		852	return user_regset_copyout_zero(&pos, &count, &kbuf, &ubuf,
		853	start, end);
		854	}
		855
		856	static int sve_set(struct task_struct *target,
		857	const struct user_regset *regset,
		858	unsigned int pos, unsigned int count,
		859	const void kbuf, const void __user ubuf)
		860	{
		861	int ret;
		862	struct user_sve_header header;
		863	unsigned int vq;
		864	unsigned long start, end;
		865
		866	if (!system_supports_sve())
		867	return -EINVAL;
		868
		869	/* Header */
		870	if (count < sizeof(header))
		871	return -EINVAL;
		872	ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &header,
		873	0, sizeof(header));
		874	if (ret)
		875	goto out;
		876
		877	/*
		878	* Apart from PT_SVE_REGS_MASK, all PT_SVE_* flags are consumed by
		879	* sve_set_vector_length(), which will also validate them for us:
		880	*/
		881	ret = sve_set_vector_length(target, header.vl,
		882	((unsigned long)header.flags & ~SVE_PT_REGS_MASK) << 16);
		883	if (ret)
		884	goto out;
		885
		886	/* Actual VL set may be less than the user asked for: */
		887	vq = sve_vq_from_vl(target->thread.sve_vl);
		888
		889	/* Registers: FPSIMD-only case */
		890
		891	BUILD_BUG_ON(SVE_PT_FPSIMD_OFFSET != sizeof(header));
		892	if ((header.flags & SVE_PT_REGS_MASK) == SVE_PT_REGS_FPSIMD) {
		893	ret = __fpr_set(target, regset, pos, count, kbuf, ubuf,
		894	SVE_PT_FPSIMD_OFFSET);
		895	clear_tsk_thread_flag(target, TIF_SVE);
		896	goto out;
		897	}
		898
		899	/* Otherwise: full SVE case */
		900
		901	/*
		902	* If setting a different VL from the requested VL and there is
		903	* register data, the data layout will be wrong: don't even
		904	* try to set the registers in this case.
		905	*/
		906	if (count && vq != sve_vq_from_vl(header.vl)) {
		907	ret = -EIO;
		908	goto out;
		909	}
		910
		911	sve_alloc(target);
		912
		913	/*
		914	* Ensure target->thread.sve_state is up to date with target's
		915	* FPSIMD regs, so that a short copyin leaves trailing registers
		916	* unmodified.
		917	*/
		918	fpsimd_sync_to_sve(target);
		919	set_tsk_thread_flag(target, TIF_SVE);
		920
		921	BUILD_BUG_ON(SVE_PT_SVE_OFFSET != sizeof(header));
		922	start = SVE_PT_SVE_OFFSET;
		923	end = SVE_PT_SVE_FFR_OFFSET(vq) + SVE_PT_SVE_FFR_SIZE(vq);
		924	ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf,
		925	target->thread.sve_state,
		926	start, end);
		927	if (ret)
		928	goto out;
		929
		930	start = end;
		931	end = SVE_PT_SVE_FPSR_OFFSET(vq);
		932	ret = user_regset_copyin_ignore(&pos, &count, &kbuf, &ubuf,
		933	start, end);
		934	if (ret)
		935	goto out;
		936
		937	/*
		938	* Copy fpsr, and fpcr which must follow contiguously in
		939	* struct fpsimd_state:
		940	*/
		941	start = end;
		942	end = SVE_PT_SVE_FPCR_OFFSET(vq) + SVE_PT_SVE_FPCR_SIZE;
		943	ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf,
		944	&target->thread.fpsimd_state.fpsr,
		945	start, end);
		946
		947	out:
		948	fpsimd_flush_task_state(target);
		949	return ret;
		950	}
		951
		952	#endif /* CONFIG_ARM64_SVE */
		953
705	enum aarch64_regset {	954	enum aarch64_regset {
706	REGSET_GPR,	955	REGSET_GPR,
707	REGSET_FPR,	956	REGSET_FPR,
@@ -711,6 +960,9 @@ enum aarch64_regset { · 3 unmodified lines
711	REGSET_HW_WATCH,	960	REGSET_HW_WATCH,
712	#endif	961	#endif
713	REGSET_SYSTEM_CALL,	962	REGSET_SYSTEM_CALL,
		963	#ifdef CONFIG_ARM64_SVE
		964	REGSET_SVE,
		965	#endif
714	};	966	};
715		967
716	static const struct user_regset aarch64_regsets[] = {	968	static const struct user_regset aarch64_regsets[] = {
@@ -768,6 +1020,18 @@ static const struct user_regset aarch64_regsets[] = { · 51 unmodified lines
768	.get = system_call_get,	1020	.get = system_call_get,
769	.set = system_call_set,	1021	.set = system_call_set,
770	},	1022	},
		1023	#ifdef CONFIG_ARM64_SVE
		1024	[REGSET_SVE] = { /* Scalable Vector Extension */
		1025	.core_note_type = NT_ARM_SVE,
		1026	.n = DIV_ROUND_UP(SVE_PT_SIZE(SVE_VQ_MAX, SVE_PT_REGS_SVE),
		1027	SVE_VQ_BYTES),
		1028	.size = SVE_VQ_BYTES,
		1029	.align = SVE_VQ_BYTES,
		1030	.get = sve_get,
		1031	.set = sve_set,
		1032	.get_size = sve_get_size,
		1033	},
		1034	#endif
771	};	1035	};
772		1036
773	static const struct user_regset_view user_aarch64_view = {	1037	static const struct user_regset_view user_aarch64_view = {

arch/arm64/kernel/setup.c+7−8

 #include <linux/stddef.h>
 #include <linux/ioport.h>
 #include <linux/delay.h>
-#include <linux/utsname.h>
 #include <linux/initrd.h>
 #include <linux/console.h>
 #include <linux/cache.h>
 #include <asm/fixmap.h>
 #include <asm/cpu.h>
 #include <asm/cputype.h>
+#include <asm/daifflags.h>
 #include <asm/elf.h>
 #include <asm/cpufeature.h>
 #include <asm/cpu_ops.h>
 	 * access percpu variable inside lock_release
 	 */
 	set_my_cpu_offset(0);
-	pr_info("Booting Linux on physical CPU 0x%lx\n", (unsigned long)mpidr);
+	pr_info("Booting Linux on physical CPU 0x%010lx [0x%08x]\n",
+		(unsigned long)mpidr, read_cpuid_id());
 }
 bool arch_match_cpu_phys_id(int cpu, u64 phys_id)
 void __init setup_arch(char **cmdline_p)
 {
-	pr_info("Boot CPU: AArch64 Processor [%08x]\n", read_cpuid_id());
-	sprintf(init_utsname()->machine, UTS_MACHINE);
 	init_mm.start_code = (unsigned long) _text;
 	init_mm.end_code   = (unsigned long) _etext;
 	init_mm.end_data   = (unsigned long) _edata;
 	parse_early_param();
 	/*
-	 *  Unmask asynchronous aborts after bringing up possible earlycon.
+	 * Unmask asynchronous aborts and fiq after bringing up possible
-	 * (Report possible System Errors once we can report this occurred)
+	 * earlycon. (Report possible System Errors once we can report this
+	 * occurred).
 	 */
-	local_async_enable();
+	local_daif_restore(DAIF_PROCCTX_NOIRQ);
 	/*
 	 * TTBR0 is only used for the identity mapping at this stage. Make it

arch/arm64/kernel/signal.c+169−10

@@ -31,6 +31,7 @@ · 30 unmodified lines
31	#include <linux/ratelimit.h>	31	#include <linux/ratelimit.h>
32	#include <linux/syscalls.h>	32	#include <linux/syscalls.h>
33		33
		34	#include <asm/daifflags.h>
34	#include <asm/debug-monitors.h>	35	#include <asm/debug-monitors.h>
35	#include <asm/elf.h>	36	#include <asm/elf.h>
36	#include <asm/cacheflush.h>	37	#include <asm/cacheflush.h>
@@ -63,6 +64,7 @@ struct rt_sigframe_user_layout { · 26 unmodified lines
63		64
64	unsigned long fpsimd_offset;	65	unsigned long fpsimd_offset;
65	unsigned long esr_offset;	66	unsigned long esr_offset;
		67	unsigned long sve_offset;
66	unsigned long extra_offset;	68	unsigned long extra_offset;
67	unsigned long end_offset;	69	unsigned long end_offset;
68	};	70	};
@@ -179,9 +181,6 @@ static int preserve_fpsimd_context(struct fpsimd_context __user *ctx) · 110 unmodified lines
179	struct fpsimd_state *fpsimd = &current->thread.fpsimd_state;	181	struct fpsimd_state *fpsimd = &current->thread.fpsimd_state;
180	int err;	182	int err;
181		183
182	/* dump the hardware registers to the fpsimd_state structure */
183	fpsimd_preserve_current_state();
184
185	/* copy the FP and status/control registers */	184	/* copy the FP and status/control registers */
186	err = __copy_to_user(ctx->vregs, fpsimd->vregs, sizeof(fpsimd->vregs));	185	err = __copy_to_user(ctx->vregs, fpsimd->vregs, sizeof(fpsimd->vregs));
187	__put_user_error(fpsimd->fpsr, &ctx->fpsr, err);	186	__put_user_error(fpsimd->fpsr, &ctx->fpsr, err);
@@ -214,6 +213,8 @@ static int restore_fpsimd_context(struct fpsimd_context __user *ctx) · 26 unmodified lines
214	__get_user_error(fpsimd.fpsr, &ctx->fpsr, err);	213	__get_user_error(fpsimd.fpsr, &ctx->fpsr, err);
215	__get_user_error(fpsimd.fpcr, &ctx->fpcr, err);	214	__get_user_error(fpsimd.fpcr, &ctx->fpcr, err);
216		215
		216	clear_thread_flag(TIF_SVE);
		217
217	/* load the hardware registers from the fpsimd_state structure */	218	/* load the hardware registers from the fpsimd_state structure */
218	if (!err)	219	if (!err)
219	fpsimd_update_current_state(&fpsimd);	220	fpsimd_update_current_state(&fpsimd);
@@ -221,10 +222,118 @@ static int restore_fpsimd_context(struct fpsimd_context __user *ctx) · 1 unmodified line
221	return err ? -EFAULT : 0;	222	return err ? -EFAULT : 0;
222	}	223	}
223		224
		225
224	struct user_ctxs {	226	struct user_ctxs {
225	struct fpsimd_context __user *fpsimd;	227	struct fpsimd_context __user *fpsimd;
		228	struct sve_context __user *sve;
226	};	229	};
227		230
		231	#ifdef CONFIG_ARM64_SVE
		232
		233	static int preserve_sve_context(struct sve_context __user *ctx)
		234	{
		235	int err = 0;
		236	u16 reserved[ARRAY_SIZE(ctx->__reserved)];
		237	unsigned int vl = current->thread.sve_vl;
		238	unsigned int vq = 0;
		239
		240	if (test_thread_flag(TIF_SVE))
		241	vq = sve_vq_from_vl(vl);
		242
		243	memset(reserved, 0, sizeof(reserved));
		244
		245	__put_user_error(SVE_MAGIC, &ctx->head.magic, err);
		246	__put_user_error(round_up(SVE_SIG_CONTEXT_SIZE(vq), 16),
		247	&ctx->head.size, err);
		248	__put_user_error(vl, &ctx->vl, err);
		249	BUILD_BUG_ON(sizeof(ctx->__reserved) != sizeof(reserved));
		250	err \|= __copy_to_user(&ctx->__reserved, reserved, sizeof(reserved));
		251
		252	if (vq) {
		253	/*
		254	* This assumes that the SVE state has already been saved to
		255	* the task struct by calling preserve_fpsimd_context().
		256	*/
		257	err \|= __copy_to_user((char __user *)ctx + SVE_SIG_REGS_OFFSET,
		258	current->thread.sve_state,
		259	SVE_SIG_REGS_SIZE(vq));
		260	}
		261
		262	return err ? -EFAULT : 0;
		263	}
		264
		265	static int restore_sve_fpsimd_context(struct user_ctxs *user)
		266	{
		267	int err;
		268	unsigned int vq;
		269	struct fpsimd_state fpsimd;
		270	struct sve_context sve;
		271
		272	if (__copy_from_user(&sve, user->sve, sizeof(sve)))
		273	return -EFAULT;
		274
		275	if (sve.vl != current->thread.sve_vl)
		276	return -EINVAL;
		277
		278	if (sve.head.size <= sizeof(*user->sve)) {
		279	clear_thread_flag(TIF_SVE);
		280	goto fpsimd_only;
		281	}
		282
		283	vq = sve_vq_from_vl(sve.vl);
		284
		285	if (sve.head.size < SVE_SIG_CONTEXT_SIZE(vq))
		286	return -EINVAL;
		287
		288	/*
		289	* Careful: we are about __copy_from_user() directly into
		290	* thread.sve_state with preemption enabled, so protection is
		291	* needed to prevent a racing context switch from writing stale
		292	* registers back over the new data.
		293	*/
		294
		295	fpsimd_flush_task_state(current);
		296	barrier();
		297	/* From now, fpsimd_thread_switch() won't clear TIF_FOREIGN_FPSTATE */
		298
		299	set_thread_flag(TIF_FOREIGN_FPSTATE);
		300	barrier();
		301	/* From now, fpsimd_thread_switch() won't touch thread.sve_state */
		302
		303	sve_alloc(current);
		304	err = __copy_from_user(current->thread.sve_state,
		305	(char __user const *)user->sve +
		306	SVE_SIG_REGS_OFFSET,
		307	SVE_SIG_REGS_SIZE(vq));
		308	if (err)
		309	return -EFAULT;
		310
		311	set_thread_flag(TIF_SVE);
		312
		313	fpsimd_only:
		314	/* copy the FP and status/control registers */
		315	/* restore_sigframe() already checked that user->fpsimd != NULL. */
		316	err = __copy_from_user(fpsimd.vregs, user->fpsimd->vregs,
		317	sizeof(fpsimd.vregs));
		318	__get_user_error(fpsimd.fpsr, &user->fpsimd->fpsr, err);
		319	__get_user_error(fpsimd.fpcr, &user->fpsimd->fpcr, err);
		320
		321	/* load the hardware registers from the fpsimd_state structure */
		322	if (!err)
		323	fpsimd_update_current_state(&fpsimd);
		324
		325	return err ? -EFAULT : 0;
		326	}
		327
		328	#else /* ! CONFIG_ARM64_SVE */
		329
		330	/* Turn any non-optimised out attempts to use these into a link error: */
		331	extern int preserve_sve_context(void __user *ctx);
		332	extern int restore_sve_fpsimd_context(struct user_ctxs *user);
		333
		334	#endif /* ! CONFIG_ARM64_SVE */
		335
		336
228	static int parse_user_sigframe(struct user_ctxs *user,	337	static int parse_user_sigframe(struct user_ctxs *user,
229	struct rt_sigframe __user *sf)	338	struct rt_sigframe __user *sf)
230	{	339	{
@@ -237,6 +346,7 @@ static int parse_user_sigframe(struct user_ctxs *user, · 6 unmodified lines
237	char const __user const sfp = (char const __user )sf;	346	char const __user const sfp = (char const __user )sf;
238		347
239	user->fpsimd = NULL;	348	user->fpsimd = NULL;
		349	user->sve = NULL;
240		350
241	if (!IS_ALIGNED((unsigned long)base, 16))	351	if (!IS_ALIGNED((unsigned long)base, 16))
242	goto invalid;	352	goto invalid;
@@ -287,6 +397,19 @@ static int parse_user_sigframe(struct user_ctxs *user, · 44 unmodified lines
287	/* ignore */	397	/* ignore */
288	break;	398	break;
289		399
		400	case SVE_MAGIC:
		401	if (!system_supports_sve())
		402	goto invalid;
		403
		404	if (user->sve)
		405	goto invalid;
		406
		407	if (size < sizeof(*user->sve))
		408	goto invalid;
		409
		410	user->sve = (struct sve_context __user *)head;
		411	break;
		412
290	case EXTRA_MAGIC:	413	case EXTRA_MAGIC:
291	if (have_extra_context)	414	if (have_extra_context)
292	goto invalid;	415	goto invalid;
@@ -343,6 +466,10 @@ static int parse_user_sigframe(struct user_ctxs *user, · 50 unmodified lines
343	*/	466	*/
344	offset = 0;	467	offset = 0;
345	limit = extra_size;	468	limit = extra_size;
		469
		470	if (!access_ok(VERIFY_READ, base, limit))
		471	goto invalid;
		472
346	continue;	473	continue;
347		474
348	default:	475	default:
@@ -359,9 +486,6 @@ static int parse_user_sigframe(struct user_ctxs *user, · 10 unmodified lines
359	}	486	}
360		487
361	done:	488	done:
362	if (!user->fpsimd)
363	goto invalid;
364
365	return 0;	489	return 0;
366		490
367	invalid:	491	invalid:
@@ -395,8 +519,19 @@ static int restore_sigframe(struct pt_regs *regs, · 27 unmodified lines
395	if (err == 0)	519	if (err == 0)
396	err = parse_user_sigframe(&user, sf);	520	err = parse_user_sigframe(&user, sf);
397		521
398	if (err == 0)	522	if (err == 0) {
399	err = restore_fpsimd_context(user.fpsimd);	523	if (!user.fpsimd)
		524	return -EINVAL;
		525
		526	if (user.sve) {
		527	if (!system_supports_sve())
		528	return -EINVAL;
		529
		530	err = restore_sve_fpsimd_context(&user);
		531	} else {
		532	err = restore_fpsimd_context(user.fpsimd);
		533	}
		534	}
400		535
401	return err;	536	return err;
402	}	537	}
@@ -455,6 +590,18 @@ static int setup_sigframe_layout(struct rt_sigframe_user_layout *user) · 52 unmodified lines
455	return err;	590	return err;
456	}	591	}
457		592
		593	if (system_supports_sve()) {
		594	unsigned int vq = 0;
		595
		596	if (test_thread_flag(TIF_SVE))
		597	vq = sve_vq_from_vl(current->thread.sve_vl);
		598
		599	err = sigframe_alloc(user, &user->sve_offset,
		600	SVE_SIG_CONTEXT_SIZE(vq));
		601	if (err)
		602	return err;
		603	}
		604
458	return sigframe_alloc_end(user);	605	return sigframe_alloc_end(user);
459	}	606	}
460		607
@@ -496,6 +643,13 @@ static int setup_sigframe(struct rt_sigframe_user_layout *user, · 35 unmodified lines
496	__put_user_error(current->thread.fault_code, &esr_ctx->esr, err);	643	__put_user_error(current->thread.fault_code, &esr_ctx->esr, err);
497	}	644	}
498		645
		646	/* Scalable Vector Extension state, if present */
		647	if (system_supports_sve() && err == 0 && user->sve_offset) {
		648	struct sve_context __user *sve_ctx =
		649	apply_user_offset(user, user->sve_offset);
		650	err \|= preserve_sve_context(sve_ctx);
		651	}
		652
499	if (err == 0 && user->extra_offset) {	653	if (err == 0 && user->extra_offset) {
500	char __user sfp = (char __user )user->sigframe;	654	char __user sfp = (char __user )user->sigframe;
501	char __user *userp =	655	char __user *userp =
@@ -595,6 +749,8 @@ static int setup_rt_frame(int usig, struct ksignal ksig, sigset_t set, · 93 unmodified lines
595	struct rt_sigframe __user *frame;	749	struct rt_sigframe __user *frame;
596	int err = 0;	750	int err = 0;
597		751
		752	fpsimd_signal_preserve_current_state();
		753
598	if (get_sigframe(&user, ksig, regs))	754	if (get_sigframe(&user, ksig, regs))
599	return 1;	755	return 1;
600		756
@@ -756,9 +912,12 @@ asmlinkage void do_notify_resume(struct pt_regs *regs, · 155 unmodified lines
756	addr_limit_user_check();	912	addr_limit_user_check();
757		913
758	if (thread_flags & _TIF_NEED_RESCHED) {	914	if (thread_flags & _TIF_NEED_RESCHED) {
		915	/* Unmask Debug and SError for the next task */
		916	local_daif_restore(DAIF_PROCCTX_NOIRQ);
		917
759	schedule();	918	schedule();
760	} else {	919	} else {
761	local_irq_enable();	920	local_daif_restore(DAIF_PROCCTX);
762		921
763	if (thread_flags & _TIF_UPROBE)	922	if (thread_flags & _TIF_UPROBE)
764	uprobe_notify_resume(regs);	923	uprobe_notify_resume(regs);
@@ -775,7 +934,7 @@ asmlinkage void do_notify_resume(struct pt_regs *regs, · 10 unmodified lines
775	fpsimd_restore_current_state();	934	fpsimd_restore_current_state();
776	}	935	}
777		936
778	local_irq_disable();	937	local_daif_mask();
779	thread_flags = READ_ONCE(current_thread_info()->flags);	938	thread_flags = READ_ONCE(current_thread_info()->flags);
780	} while (thread_flags & _TIF_WORK_MASK);	939	} while (thread_flags & _TIF_WORK_MASK);
781	}	940	}

arch/arm64/kernel/signal32.c+1−1

@@ -239,7 +239,7 @@ static int compat_preserve_vfp_context(struct compat_vfp_sigframe __user *frame) · 238 unmodified lines
239	* Note that this also saves V16-31, which aren't visible	239	* Note that this also saves V16-31, which aren't visible
240	* in AArch32.	240	* in AArch32.
241	*/	241	*/
242	fpsimd_preserve_current_state();	242	fpsimd_signal_preserve_current_state();
243		243
244	/* Place structure header on the stack */	244	/* Place structure header on the stack */
245	__put_user_error(magic, &frame->magic, err);	245	__put_user_error(magic, &frame->magic, err);

arch/arm64/kernel/smp.c+8−10

 #include <asm/cpu.h>
 #include <asm/cputype.h>
 #include <asm/cpu_ops.h>
+#include <asm/daifflags.h>
 #include <asm/mmu_context.h>
 #include <asm/numa.h>
 #include <asm/pgtable.h>
  */
 asmlinkage void secondary_start_kernel(void)
 {
+	u64 mpidr = read_cpuid_mpidr() & MPIDR_HWID_BITMASK;
 	struct mm_struct *mm = &init_mm;
 	unsigned int cpu;
 	 * the CPU migration code to notice that the CPU is online
 	 * before we continue.
 	 */
-	pr_info("CPU%u: Booted secondary processor [%08x]\n",
+	pr_info("CPU%u: Booted secondary processor 0x%010lx [0x%08x]\n",
-					 cpu, read_cpuid_id());
+					 cpu, (unsigned long)mpidr,
+					 read_cpuid_id());
 	update_cpu_boot_status(CPU_BOOT_SUCCESS);
 	set_cpu_online(cpu, true);
 	complete(&cpu_running);
-	local_irq_enable();
+	local_daif_restore(DAIF_PROCCTX);
-	local_async_enable();
 	/*
 	 * OK, it's off to the idle thread for us
 /*
  * Called from the idle thread for the CPU which has been shutdown.
  *
- * Note that we disable IRQs here, but do not re-enable them
- * before returning to the caller. This is also the behaviour
- * of the other hotplug-cpu capable cores, so presumably coming
- * out of idle fixes this.
  */
 void cpu_die(void)
 {
 	idle_task_exit();
-	local_irq_disable();
+	local_daif_mask();
 	/* Tell __cpu_die() that this CPU is now safe to dispose of */
 	(void)cpu_report_death();
 {
 	set_cpu_online(cpu, false);
-	local_irq_disable();
+	local_daif_mask();
 	while (1)
 		cpu_relax();

arch/arm64/kernel/suspend.c+4−4

 #include <asm/alternative.h>
 #include <asm/cacheflush.h>
 #include <asm/cpufeature.h>
+#include <asm/daifflags.h>
 #include <asm/debug-monitors.h>
 #include <asm/exec.h>
 #include <asm/pgtable.h>
 #include <asm/mmu_context.h>
 #include <asm/smp_plat.h>
 #include <asm/suspend.h>
-#include <asm/tlbflush.h>
 /*
  * This is allocated by cpu_suspend_init(), and used to store a pointer to
 	/*
 	 * Restore HW breakpoint registers to sane values
 	 * before debug exceptions are possibly reenabled
-	 * through local_dbg_restore.
+	 * by cpu_suspend()s local_daif_restore() call.
 	 */
 	if (hw_breakpoint_restore)
 		hw_breakpoint_restore(cpu);
 	 * updates to mdscr register (saved and restored along with
 	 * general purpose registers) from kernel debuggers.
 	 */
-	local_dbg_save(flags);
+	flags = local_daif_save();
 	/*
 	 * Function graph tracer state gets incosistent when the kernel
 	 * restored, so from this point onwards, debugging is fully
 	 * renabled if it was enabled when core started shutdown.
 	 */
-	local_dbg_restore(flags);
+	local_daif_restore(flags);
 	return ret;
 }

arch/arm64/kernel/traps.c+38−71

 #include <asm/atomic.h>
 #include <asm/bug.h>
+#include <asm/daifflags.h>
 #include <asm/debug-monitors.h>
 #include <asm/esr.h>
 #include <asm/insn.h>
 int show_unhandled_signals = 1;
-/*
- * Dump out the contents of some kernel memory nicely...
- */
-static void dump_mem(const char *lvl, const char *str, unsigned long bottom,
-		     unsigned long top)
-{
-	unsigned long first;
-	mm_segment_t fs;
-	int i;
-	/*
-	 * We need to switch to kernel mode so that we can use __get_user
-	 * to safely read from kernel space.
-	 */
-	fs = get_fs();
-	set_fs(KERNEL_DS);
-	printk("%s%s(0x%016lx to 0x%016lx)\n", lvl, str, bottom, top);
-	for (first = bottom & ~31; first < top; first += 32) {
-		unsigned long p;
-		char str[sizeof(" 12345678") * 8 + 1];
-		memset(str, ' ', sizeof(str));
-		str[sizeof(str) - 1] = '\0';
-		for (p = first, i = 0; i < (32 / 8)
-					&& p < top; i++, p += 8) {
-			if (p >= bottom && p < top) {
-				unsigned long val;
-				if (__get_user(val, (unsigned long *)p) == 0)
-					sprintf(str + i * 17, " %016lx", val);
-				else
-					sprintf(str + i * 17, " ????????????????");
-			}
-		}
-		printk("%s%04lx:%s\n", lvl, first & 0xffff, str);
-	}
-	set_fs(fs);
-}
 static void dump_backtrace_entry(unsigned long where)
 {
-	/*
+	printk(" %pS\n", (void *)where);
-	 * Note that 'where' can have a physical address, but it's not handled.
-	 */
-	print_ip_sym(where);
 }
 static void __dump_instr(const char *lvl, struct pt_regs *regs)
 	skip = !!regs;
 	printk("Call trace:\n");
-	while (1) {
+	do {
-		unsigned long stack;
-		int ret;
 		/* skip until specified stack frame */
 		if (!skip) {
 			dump_backtrace_entry(frame.pc);
 			 */
 			dump_backtrace_entry(regs->pc);
 		}
-		ret = unwind_frame(tsk, &frame);
+	} while (!unwind_frame(tsk, &frame));
-		if (ret < 0)
-			break;
-		if (in_entry_text(frame.pc)) {
-			stack = frame.fp - offsetof(struct pt_regs, stackframe);
-			if (on_accessible_stack(tsk, stack))
-				dump_mem("", "Exception stack", stack,
-					 stack + sizeof(struct pt_regs));
-		}
-	}
 	put_task_stack(tsk);
 }
 	}
 }
+void arm64_skip_faulting_instruction(struct pt_regs *regs, unsigned long size)
+{
+	regs->pc += size;
+	/*
+	 * If we were single stepping, we want to get the step exception after
+	 * we return from the trap.
+	 */
+	user_fastforward_single_step(current);
+}
 static LIST_HEAD(undef_hook);
 static DEFINE_RAW_SPINLOCK(undef_lock);
 	return fn ? fn(regs, instr) : 1;
 }
-static void force_signal_inject(int signal, int code, struct pt_regs *regs,
+void force_signal_inject(int signal, int code, struct pt_regs *regs,
-				unsigned long address)
+			 unsigned long address)
 {
 	siginfo_t info;
 	void __user *pc = (void __user *)instruction_pointer(regs);
 		desc = "illegal memory access";
 		break;
 	default:
-		desc = "bad mode";
+		desc = "unknown or unrecoverable error";
 		break;
 	}
 	if (ret)
 		arm64_notify_segfault(regs, address);
 	else
-		regs->pc += 4;
+		arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE);
 }
 static void ctr_read_handler(unsigned int esr, struct pt_regs *regs)
 	pt_regs_write_reg(regs, rt, val);
-	regs->pc += 4;
+	arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE);
 }
 static void cntvct_read_handler(unsigned int esr, struct pt_regs *regs)
 	int rt = (esr & ESR_ELx_SYS64_ISS_RT_MASK) >> ESR_ELx_SYS64_ISS_RT_SHIFT;
 	pt_regs_write_reg(regs, rt, arch_counter_get_cntvct());
-	regs->pc += 4;
+	arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE);
 }
 static void cntfrq_read_handler(unsigned int esr, struct pt_regs *regs)
 	int rt = (esr & ESR_ELx_SYS64_ISS_RT_MASK) >> ESR_ELx_SYS64_ISS_RT_SHIFT;
 	pt_regs_write_reg(regs, rt, arch_timer_get_rate());
-	regs->pc += 4;
+	arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE);
 }
 struct sys64_hook {
 	[ESR_ELx_EC_HVC64]		= "HVC (AArch64)",
 	[ESR_ELx_EC_SMC64]		= "SMC (AArch64)",
 	[ESR_ELx_EC_SYS64]		= "MSR/MRS (AArch64)",
+	[ESR_ELx_EC_SVE]		= "SVE",
 	[ESR_ELx_EC_IMP_DEF]		= "EL3 IMP DEF",
 	[ESR_ELx_EC_IABT_LOW]		= "IABT (lower EL)",
 	[ESR_ELx_EC_IABT_CUR]		= "IABT (current EL)",
 		esr_get_class_string(esr));
 	die("Oops - bad mode", regs, 0);
-	local_irq_disable();
+	local_daif_mask();
 	panic("bad mode");
 }
 }
 #endif
+asmlinkage void do_serror(struct pt_regs *regs, unsigned int esr)
+{
+	nmi_enter();
+	console_verbose();
+	pr_crit("SError Interrupt on CPU%d, code 0x%08x -- %s\n",
+		smp_processor_id(), esr, esr_get_class_string(esr));
+	__show_regs(regs);
+	panic("Asynchronous SError Interrupt");
+}
 void __pte_error(const char *file, int line, unsigned long val)
 {
 	pr_err("%s:%d: bad pte %016lx.\n", file, line, val);
 	}
 	/* If thread survives, skip over the BUG instruction and continue: */
-	regs->pc += AARCH64_INSN_SIZE;	/* skip BRK and resume */
+	arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE);
 	return DBG_HOOK_HANDLED;
 }

arch/arm64/kernel/vdso/gettimeofday.S+1−1

@@ -309,7 +309,7 @@ ENTRY(__kernel_clock_getres) · 308 unmodified lines
309	b.ne 4f	309	b.ne 4f
310	ldr x2, 6f	310	ldr x2, 6f
311	2:	311	2:
312	cbz w1, 3f	312	cbz x1, 3f
313	stp xzr, x2, [x1]	313	stp xzr, x2, [x1]
314		314
315	3: /* res == NULL. */	315	3: /* res == NULL. */

arch/arm64/kvm/handle_exit.c+8−0

 	return 1;
 }
+static int handle_sve(struct kvm_vcpu *vcpu, struct kvm_run *run)
+{
+	/* Until SVE is supported for guests: */
+	kvm_inject_undefined(vcpu);
+	return 1;
+}
 static exit_handle_fn arm_exit_handlers[] = {
 	[0 ... ESR_ELx_EC_MAX]	= kvm_handle_unknown_ec,
 	[ESR_ELx_EC_WFx]	= kvm_handle_wfx,
 	[ESR_ELx_EC_HVC64]	= handle_hvc,
 	[ESR_ELx_EC_SMC64]	= handle_smc,
 	[ESR_ELx_EC_SYS64]	= kvm_handle_sys_reg,
+	[ESR_ELx_EC_SVE]	= handle_sve,
 	[ESR_ELx_EC_IABT_LOW]	= kvm_handle_guest_abort,
 	[ESR_ELx_EC_DABT_LOW]	= kvm_handle_guest_abort,
 	[ESR_ELx_EC_SOFTSTP_LOW]= kvm_handle_guest_debug,

arch/arm64/kvm/hyp/debug-sr.c+7−17

 	default:	write_debug(ptr[0], reg, 0);			\
 	}
-#define PMSCR_EL1		sys_reg(3, 0, 9, 9, 0)
-#define PMBLIMITR_EL1		sys_reg(3, 0, 9, 10, 0)
-#define PMBLIMITR_EL1_E		BIT(0)
-#define PMBIDR_EL1		sys_reg(3, 0, 9, 10, 7)
-#define PMBIDR_EL1_P		BIT(4)
-#define psb_csync()		asm volatile("hint #17")
 static void __hyp_text __debug_save_spe_vhe(u64 *pmscr_el1)
 {
 	/* The vcpu can run. but it can't hide. */
 		return;
 	/* Yes; is it owned by EL3? */
-	reg = read_sysreg_s(PMBIDR_EL1);
+	reg = read_sysreg_s(SYS_PMBIDR_EL1);
-	if (reg & PMBIDR_EL1_P)
+	if (reg & BIT(SYS_PMBIDR_EL1_P_SHIFT))
 		return;
 	/* No; is the host actually using the thing? */
-	reg = read_sysreg_s(PMBLIMITR_EL1);
+	reg = read_sysreg_s(SYS_PMBLIMITR_EL1);
-	if (!(reg & PMBLIMITR_EL1_E))
+	if (!(reg & BIT(SYS_PMBLIMITR_EL1_E_SHIFT)))
 		return;
 	/* Yes; save the control register and disable data generation */
-	*pmscr_el1 = read_sysreg_s(PMSCR_EL1);
+	*pmscr_el1 = read_sysreg_s(SYS_PMSCR_EL1);
-	write_sysreg_s(0, PMSCR_EL1);
+	write_sysreg_s(0, SYS_PMSCR_EL1);
 	isb();
 	/* Now drain all buffered data to memory */
 	isb();
 	/* Re-enable data generation */
-	write_sysreg_s(pmscr_el1, PMSCR_EL1);
+	write_sysreg_s(pmscr_el1, SYS_PMSCR_EL1);
 }
 void __hyp_text __debug_save_state(struct kvm_vcpu *vcpu,

arch/arm64/kvm/hyp/switch.c+9−3

 	val = read_sysreg(cpacr_el1);
 	val |= CPACR_EL1_TTA;
-	val &= ~CPACR_EL1_FPEN;
+	val &= ~(CPACR_EL1_FPEN | CPACR_EL1_ZEN);
 	write_sysreg(val, cpacr_el1);
 	write_sysreg(__kvm_hyp_vector, vbar_el1);
 	u64 val;
 	val = CPTR_EL2_DEFAULT;
-	val |= CPTR_EL2_TTA | CPTR_EL2_TFP;
+	val |= CPTR_EL2_TTA | CPTR_EL2_TFP | CPTR_EL2_TZ;
 	write_sysreg(val, cptr_el2);
 }
 	 * it will cause an exception.
 	 */
 	val = vcpu->arch.hcr_el2;
 	if (!(val & HCR_RW) && system_supports_fpsimd()) {
 		write_sysreg(1 << 30, fpexc32_el2);
 		isb();
 	}
+	if (val & HCR_RW) /* for AArch64 only: */
+		val |= HCR_TID3; /* TID3: trap feature register accesses */
 	write_sysreg(val, hcr_el2);
 	/* Trap on AArch32 cp15 c15 accesses (EL1 or EL0) */
 	write_sysreg(1 << 15, hstr_el2);
 	/*
 	write_sysreg(mdcr_el2, mdcr_el2);
 	write_sysreg(HCR_HOST_VHE_FLAGS, hcr_el2);
-	write_sysreg(CPACR_EL1_FPEN, cpacr_el1);
+	write_sysreg(CPACR_EL1_DEFAULT, cpacr_el1);
 	write_sysreg(vectors, vbar_el1);
 }

arch/arm64/kvm/sys_regs.c+247−45

@@ -23,6 +23,7 @@ · 22 unmodified lines
23	#include <linux/bsearch.h>	23	#include <linux/bsearch.h>
24	#include <linux/kvm_host.h>	24	#include <linux/kvm_host.h>
25	#include <linux/mm.h>	25	#include <linux/mm.h>
		26	#include <linux/printk.h>
26	#include <linux/uaccess.h>	27	#include <linux/uaccess.h>
27		28
28	#include <asm/cacheflush.h>	29	#include <asm/cacheflush.h>
@@ -892,6 +893,146 @@ static bool access_cntp_cval(struct kvm_vcpu *vcpu, · 863 unmodified lines
892	return true;	893	return true;
893	}	894	}
894		895
		896	/* Read a sanitised cpufeature ID register by sys_reg_desc */
		897	static u64 read_id_reg(struct sys_reg_desc const *r, bool raz)
		898	{
		899	u32 id = sys_reg((u32)r->Op0, (u32)r->Op1,
		900	(u32)r->CRn, (u32)r->CRm, (u32)r->Op2);
		901	u64 val = raz ? 0 : read_sanitised_ftr_reg(id);
		902
		903	if (id == SYS_ID_AA64PFR0_EL1) {
		904	if (val & (0xfUL << ID_AA64PFR0_SVE_SHIFT))
		905	pr_err_once("kvm [%i]: SVE unsupported for guests, suppressing\n",
		906	task_pid_nr(current));
		907
		908	val &= ~(0xfUL << ID_AA64PFR0_SVE_SHIFT);
		909	}
		910
		911	return val;
		912	}
		913
		914	/* cpufeature ID register access trap handlers */
		915
		916	static bool __access_id_reg(struct kvm_vcpu *vcpu,
		917	struct sys_reg_params *p,
		918	const struct sys_reg_desc *r,
		919	bool raz)
		920	{
		921	if (p->is_write)
		922	return write_to_read_only(vcpu, p, r);
		923
		924	p->regval = read_id_reg(r, raz);
		925	return true;
		926	}
		927
		928	static bool access_id_reg(struct kvm_vcpu *vcpu,
		929	struct sys_reg_params *p,
		930	const struct sys_reg_desc *r)
		931	{
		932	return __access_id_reg(vcpu, p, r, false);
		933	}
		934
		935	static bool access_raz_id_reg(struct kvm_vcpu *vcpu,
		936	struct sys_reg_params *p,
		937	const struct sys_reg_desc *r)
		938	{
		939	return __access_id_reg(vcpu, p, r, true);
		940	}
		941
		942	static int reg_from_user(u64 val, const void __user uaddr, u64 id);
		943	static int reg_to_user(void __user uaddr, const u64 val, u64 id);
		944	static u64 sys_reg_to_index(const struct sys_reg_desc *reg);
		945
		946	/*
		947	* cpufeature ID register user accessors
		948	*
		949	* For now, these registers are immutable for userspace, so no values
		950	* are stored, and for set_id_reg() we don't allow the effective value
		951	* to be changed.
		952	*/
		953	static int __get_id_reg(const struct sys_reg_desc rd, void __user uaddr,
		954	bool raz)
		955	{
		956	const u64 id = sys_reg_to_index(rd);
		957	const u64 val = read_id_reg(rd, raz);
		958
		959	return reg_to_user(uaddr, &val, id);
		960	}
		961
		962	static int __set_id_reg(const struct sys_reg_desc rd, void __user uaddr,
		963	bool raz)
		964	{
		965	const u64 id = sys_reg_to_index(rd);
		966	int err;
		967	u64 val;
		968
		969	err = reg_from_user(&val, uaddr, id);
		970	if (err)
		971	return err;
		972
		973	/* This is what we mean by invariant: you can't change it. */
		974	if (val != read_id_reg(rd, raz))
		975	return -EINVAL;
		976
		977	return 0;
		978	}
		979
		980	static int get_id_reg(struct kvm_vcpu vcpu, const struct sys_reg_desc rd,
		981	const struct kvm_one_reg reg, void __user uaddr)
		982	{
		983	return __get_id_reg(rd, uaddr, false);
		984	}
		985
		986	static int set_id_reg(struct kvm_vcpu vcpu, const struct sys_reg_desc rd,
		987	const struct kvm_one_reg reg, void __user uaddr)
		988	{
		989	return __set_id_reg(rd, uaddr, false);
		990	}
		991
		992	static int get_raz_id_reg(struct kvm_vcpu vcpu, const struct sys_reg_desc rd,
		993	const struct kvm_one_reg reg, void __user uaddr)
		994	{
		995	return __get_id_reg(rd, uaddr, true);
		996	}
		997
		998	static int set_raz_id_reg(struct kvm_vcpu vcpu, const struct sys_reg_desc rd,
		999	const struct kvm_one_reg reg, void __user uaddr)
		1000	{
		1001	return __set_id_reg(rd, uaddr, true);
		1002	}
		1003
		1004	/* sys_reg_desc initialiser for known cpufeature ID registers */
		1005	#define ID_SANITISED(name) { \
		1006	SYS_DESC(SYS_##name), \
		1007	.access = access_id_reg, \
		1008	.get_user = get_id_reg, \
		1009	.set_user = set_id_reg, \
		1010	}
		1011
		1012	/*
		1013	* sys_reg_desc initialiser for architecturally unallocated cpufeature ID
		1014	* register with encoding Op0=3, Op1=0, CRn=0, CRm=crm, Op2=op2
		1015	* (1 <= crm < 8, 0 <= Op2 < 8).
		1016	*/
		1017	#define ID_UNALLOCATED(crm, op2) { \
		1018	Op0(3), Op1(0), CRn(0), CRm(crm), Op2(op2), \
		1019	.access = access_raz_id_reg, \
		1020	.get_user = get_raz_id_reg, \
		1021	.set_user = set_raz_id_reg, \
		1022	}
		1023
		1024	/*
		1025	* sys_reg_desc initialiser for known ID registers that we hide from guests.
		1026	* For now, these are exposed just like unallocated ID regs: they appear
		1027	* RAZ for the guest.
		1028	*/
		1029	#define ID_HIDDEN(name) { \
		1030	SYS_DESC(SYS_##name), \
		1031	.access = access_raz_id_reg, \
		1032	.get_user = get_raz_id_reg, \
		1033	.set_user = set_raz_id_reg, \
		1034	}
		1035
895	/*	1036	/*
896	* Architected system registers.	1037	* Architected system registers.
897	* Important: Must be sorted ascending by Op0, Op1, CRn, CRm, Op2	1038	* Important: Must be sorted ascending by Op0, Op1, CRn, CRm, Op2
@@ -944,6 +1085,84 @@ static const struct sys_reg_desc sys_reg_descs[] = { · 46 unmodified lines
944	{ SYS_DESC(SYS_DBGVCR32_EL2), NULL, reset_val, DBGVCR32_EL2, 0 },	1085	{ SYS_DESC(SYS_DBGVCR32_EL2), NULL, reset_val, DBGVCR32_EL2, 0 },
945		1086
946	{ SYS_DESC(SYS_MPIDR_EL1), NULL, reset_mpidr, MPIDR_EL1 },	1087	{ SYS_DESC(SYS_MPIDR_EL1), NULL, reset_mpidr, MPIDR_EL1 },
		1088
		1089	/*
		1090	* ID regs: all ID_SANITISED() entries here must have corresponding
		1091	* entries in arm64_ftr_regs[].
		1092	*/
		1093
		1094	/* AArch64 mappings of the AArch32 ID registers */
		1095	/* CRm=1 */
		1096	ID_SANITISED(ID_PFR0_EL1),
		1097	ID_SANITISED(ID_PFR1_EL1),
		1098	ID_SANITISED(ID_DFR0_EL1),
		1099	ID_HIDDEN(ID_AFR0_EL1),
		1100	ID_SANITISED(ID_MMFR0_EL1),
		1101	ID_SANITISED(ID_MMFR1_EL1),
		1102	ID_SANITISED(ID_MMFR2_EL1),
		1103	ID_SANITISED(ID_MMFR3_EL1),
		1104
		1105	/* CRm=2 */
		1106	ID_SANITISED(ID_ISAR0_EL1),
		1107	ID_SANITISED(ID_ISAR1_EL1),
		1108	ID_SANITISED(ID_ISAR2_EL1),
		1109	ID_SANITISED(ID_ISAR3_EL1),
		1110	ID_SANITISED(ID_ISAR4_EL1),
		1111	ID_SANITISED(ID_ISAR5_EL1),
		1112	ID_SANITISED(ID_MMFR4_EL1),
		1113	ID_UNALLOCATED(2,7),
		1114
		1115	/* CRm=3 */
		1116	ID_SANITISED(MVFR0_EL1),
		1117	ID_SANITISED(MVFR1_EL1),
		1118	ID_SANITISED(MVFR2_EL1),
		1119	ID_UNALLOCATED(3,3),
		1120	ID_UNALLOCATED(3,4),
		1121	ID_UNALLOCATED(3,5),
		1122	ID_UNALLOCATED(3,6),
		1123	ID_UNALLOCATED(3,7),
		1124
		1125	/* AArch64 ID registers */
		1126	/* CRm=4 */
		1127	ID_SANITISED(ID_AA64PFR0_EL1),
		1128	ID_SANITISED(ID_AA64PFR1_EL1),
		1129	ID_UNALLOCATED(4,2),
		1130	ID_UNALLOCATED(4,3),
		1131	ID_UNALLOCATED(4,4),
		1132	ID_UNALLOCATED(4,5),
		1133	ID_UNALLOCATED(4,6),
		1134	ID_UNALLOCATED(4,7),
		1135
		1136	/* CRm=5 */
		1137	ID_SANITISED(ID_AA64DFR0_EL1),
		1138	ID_SANITISED(ID_AA64DFR1_EL1),
		1139	ID_UNALLOCATED(5,2),
		1140	ID_UNALLOCATED(5,3),
		1141	ID_HIDDEN(ID_AA64AFR0_EL1),
		1142	ID_HIDDEN(ID_AA64AFR1_EL1),
		1143	ID_UNALLOCATED(5,6),
		1144	ID_UNALLOCATED(5,7),
		1145
		1146	/* CRm=6 */
		1147	ID_SANITISED(ID_AA64ISAR0_EL1),
		1148	ID_SANITISED(ID_AA64ISAR1_EL1),
		1149	ID_UNALLOCATED(6,2),
		1150	ID_UNALLOCATED(6,3),
		1151	ID_UNALLOCATED(6,4),
		1152	ID_UNALLOCATED(6,5),
		1153	ID_UNALLOCATED(6,6),
		1154	ID_UNALLOCATED(6,7),
		1155
		1156	/* CRm=7 */
		1157	ID_SANITISED(ID_AA64MMFR0_EL1),
		1158	ID_SANITISED(ID_AA64MMFR1_EL1),
		1159	ID_SANITISED(ID_AA64MMFR2_EL1),
		1160	ID_UNALLOCATED(7,3),
		1161	ID_UNALLOCATED(7,4),
		1162	ID_UNALLOCATED(7,5),
		1163	ID_UNALLOCATED(7,6),
		1164	ID_UNALLOCATED(7,7),
		1165
947	{ SYS_DESC(SYS_SCTLR_EL1), access_vm_reg, reset_val, SCTLR_EL1, 0x00C50078 },	1166	{ SYS_DESC(SYS_SCTLR_EL1), access_vm_reg, reset_val, SCTLR_EL1, 0x00C50078 },
948	{ SYS_DESC(SYS_CPACR_EL1), NULL, reset_val, CPACR_EL1, 0 },	1167	{ SYS_DESC(SYS_CPACR_EL1), NULL, reset_val, CPACR_EL1, 0 },
949	{ SYS_DESC(SYS_TTBR0_EL1), access_vm_reg, reset_unknown, TTBR0_EL1 },	1168	{ SYS_DESC(SYS_TTBR0_EL1), access_vm_reg, reset_unknown, TTBR0_EL1 },
@@ -1790,8 +2009,8 @@ static const struct sys_reg_desc index_to_sys_reg_desc(struct kvm_vcpu vcpu, · 840 unmodified lines
1790	if (!r)	2009	if (!r)
1791	r = find_reg(&params, sys_reg_descs, ARRAY_SIZE(sys_reg_descs));	2010	r = find_reg(&params, sys_reg_descs, ARRAY_SIZE(sys_reg_descs));
1792		2011
1793	/* Not saved in the sys_reg array? */	2012	/* Not saved in the sys_reg array and not otherwise accessible? */
1794	if (r && !r->reg)	2013	if (r && !(r->reg \|\| r->get_user))
1795	r = NULL;	2014	r = NULL;
1796		2015
1797	return r;	2016	return r;
@@ -1815,20 +2034,6 @@ static const struct sys_reg_desc index_to_sys_reg_desc(struct kvm_vcpu vcpu, · 17 unmodified lines
1815	FUNCTION_INVARIANT(midr_el1)	2034	FUNCTION_INVARIANT(midr_el1)
1816	FUNCTION_INVARIANT(ctr_el0)	2035	FUNCTION_INVARIANT(ctr_el0)
1817	FUNCTION_INVARIANT(revidr_el1)	2036	FUNCTION_INVARIANT(revidr_el1)
1818	FUNCTION_INVARIANT(id_pfr0_el1)
1819	FUNCTION_INVARIANT(id_pfr1_el1)
1820	FUNCTION_INVARIANT(id_dfr0_el1)
1821	FUNCTION_INVARIANT(id_afr0_el1)
1822	FUNCTION_INVARIANT(id_mmfr0_el1)
1823	FUNCTION_INVARIANT(id_mmfr1_el1)
1824	FUNCTION_INVARIANT(id_mmfr2_el1)
1825	FUNCTION_INVARIANT(id_mmfr3_el1)
1826	FUNCTION_INVARIANT(id_isar0_el1)
1827	FUNCTION_INVARIANT(id_isar1_el1)
1828	FUNCTION_INVARIANT(id_isar2_el1)
1829	FUNCTION_INVARIANT(id_isar3_el1)
1830	FUNCTION_INVARIANT(id_isar4_el1)
1831	FUNCTION_INVARIANT(id_isar5_el1)
1832	FUNCTION_INVARIANT(clidr_el1)	2037	FUNCTION_INVARIANT(clidr_el1)
1833	FUNCTION_INVARIANT(aidr_el1)	2038	FUNCTION_INVARIANT(aidr_el1)
1834		2039
@@ -1836,20 +2041,6 @@ FUNCTION_INVARIANT(aidr_el1) · 1 unmodified line
1836	static struct sys_reg_desc invariant_sys_regs[] = {	2041	static struct sys_reg_desc invariant_sys_regs[] = {
1837	{ SYS_DESC(SYS_MIDR_EL1), NULL, get_midr_el1 },	2042	{ SYS_DESC(SYS_MIDR_EL1), NULL, get_midr_el1 },
1838	{ SYS_DESC(SYS_REVIDR_EL1), NULL, get_revidr_el1 },	2043	{ SYS_DESC(SYS_REVIDR_EL1), NULL, get_revidr_el1 },
1839	{ SYS_DESC(SYS_ID_PFR0_EL1), NULL, get_id_pfr0_el1 },
1840	{ SYS_DESC(SYS_ID_PFR1_EL1), NULL, get_id_pfr1_el1 },
1841	{ SYS_DESC(SYS_ID_DFR0_EL1), NULL, get_id_dfr0_el1 },
1842	{ SYS_DESC(SYS_ID_AFR0_EL1), NULL, get_id_afr0_el1 },
1843	{ SYS_DESC(SYS_ID_MMFR0_EL1), NULL, get_id_mmfr0_el1 },
1844	{ SYS_DESC(SYS_ID_MMFR1_EL1), NULL, get_id_mmfr1_el1 },
1845	{ SYS_DESC(SYS_ID_MMFR2_EL1), NULL, get_id_mmfr2_el1 },
1846	{ SYS_DESC(SYS_ID_MMFR3_EL1), NULL, get_id_mmfr3_el1 },
1847	{ SYS_DESC(SYS_ID_ISAR0_EL1), NULL, get_id_isar0_el1 },
1848	{ SYS_DESC(SYS_ID_ISAR1_EL1), NULL, get_id_isar1_el1 },
1849	{ SYS_DESC(SYS_ID_ISAR2_EL1), NULL, get_id_isar2_el1 },
1850	{ SYS_DESC(SYS_ID_ISAR3_EL1), NULL, get_id_isar3_el1 },
1851	{ SYS_DESC(SYS_ID_ISAR4_EL1), NULL, get_id_isar4_el1 },
1852	{ SYS_DESC(SYS_ID_ISAR5_EL1), NULL, get_id_isar5_el1 },
1853	{ SYS_DESC(SYS_CLIDR_EL1), NULL, get_clidr_el1 },	2044	{ SYS_DESC(SYS_CLIDR_EL1), NULL, get_clidr_el1 },
1854	{ SYS_DESC(SYS_AIDR_EL1), NULL, get_aidr_el1 },	2045	{ SYS_DESC(SYS_AIDR_EL1), NULL, get_aidr_el1 },
1855	{ SYS_DESC(SYS_CTR_EL0), NULL, get_ctr_el0 },	2046	{ SYS_DESC(SYS_CTR_EL0), NULL, get_ctr_el0 },
@@ -2079,12 +2270,31 @@ static bool copy_reg_to_user(const struct sys_reg_desc reg, u64 __user *uind) · 223 unmodified lines
2079	return true;	2270	return true;
2080	}	2271	}
2081		2272
		2273	static int walk_one_sys_reg(const struct sys_reg_desc *rd,
		2274	u64 __user **uind,
		2275	unsigned int *total)
		2276	{
		2277	/*
		2278	* Ignore registers we trap but don't save,
		2279	* and for which no custom user accessor is provided.
		2280	*/
		2281	if (!(rd->reg \|\| rd->get_user))
		2282	return 0;
		2283
		2284	if (!copy_reg_to_user(rd, uind))
		2285	return -EFAULT;
		2286
		2287	(*total)++;
		2288	return 0;
		2289	}
		2290
2082	/* Assumed ordered tables, see kvm_sys_reg_table_init. */	2291	/* Assumed ordered tables, see kvm_sys_reg_table_init. */
2083	static int walk_sys_regs(struct kvm_vcpu vcpu, u64 __user uind)	2292	static int walk_sys_regs(struct kvm_vcpu vcpu, u64 __user uind)
2084	{	2293	{
2085	const struct sys_reg_desc i1, i2, end1, end2;	2294	const struct sys_reg_desc i1, i2, end1, end2;
2086	unsigned int total = 0;	2295	unsigned int total = 0;
2087	size_t num;	2296	size_t num;
		2297	int err;
2088		2298
2089	/* We check for duplicates here, to allow arch-specific overrides. */	2299	/* We check for duplicates here, to allow arch-specific overrides. */
2090	i1 = get_target_table(vcpu->arch.target, true, &num);	2300	i1 = get_target_table(vcpu->arch.target, true, &num);
@@ -2098,21 +2308,13 @@ static int walk_sys_regs(struct kvm_vcpu vcpu, u64 __user uind) · 7 unmodified lines
2098	while (i1 \|\| i2) {	2308	while (i1 \|\| i2) {
2099	int cmp = cmp_sys_reg(i1, i2);	2309	int cmp = cmp_sys_reg(i1, i2);
2100	/* target-specific overrides generic entry. */	2310	/* target-specific overrides generic entry. */
2101	if (cmp <= 0) {	2311	if (cmp <= 0)
2102	/* Ignore registers we trap but don't save. */	2312	err = walk_one_sys_reg(i1, &uind, &total);
2103	if (i1->reg) {	2313	else
2104	if (!copy_reg_to_user(i1, &uind))	2314	err = walk_one_sys_reg(i2, &uind, &total);
2105	return -EFAULT;	2315
2106	total++;	2316	if (err)
2107	}	2317	return err;
2108	} else {
2109	/* Ignore registers we trap but don't save. */
2110	if (i2->reg) {
2111	if (!copy_reg_to_user(i2, &uind))
2112	return -EFAULT;
2113	total++;
2114	}
2115	}
2116		2318
2117	if (cmp <= 0 && ++i1 == end1)	2319	if (cmp <= 0 && ++i1 == end1)
2118	i1 = NULL;	2320	i1 = NULL;

arch/arm64/lib/Makefile+1−1

@@ -3,7 +3,7 @@ lib-y := bitops.o clear_user.o delay.o copy_from_user.o \ · 2 unmodified lines
3	copy_to_user.o copy_in_user.o copy_page.o \	3	copy_to_user.o copy_in_user.o copy_page.o \
4	clear_page.o memchr.o memcpy.o memmove.o memset.o \	4	clear_page.o memchr.o memcpy.o memmove.o memset.o \
5	memcmp.o strcmp.o strncmp.o strlen.o strnlen.o \	5	memcmp.o strcmp.o strncmp.o strlen.o strnlen.o \
6	strchr.o strrchr.o	6	strchr.o strrchr.o tishift.o
7		7
8	# Tell the compiler to treat all general purpose registers (with the	8	# Tell the compiler to treat all general purpose registers (with the
9	# exception of the IP registers, which are already handled by the caller	9	# exception of the IP registers, which are already handled by the caller

arch/arm64/lib/delay.c+19−4

 #include <linux/module.h>
 #include <linux/timex.h>
+#include <clocksource/arm_arch_timer.h>
+#define USECS_TO_CYCLES(time_usecs)			\
+	xloops_to_cycles((time_usecs) * 0x10C7UL)
+static inline unsigned long xloops_to_cycles(unsigned long xloops)
+{
+	return (xloops * loops_per_jiffy * HZ) >> 32;
+}
 void __delay(unsigned long cycles)
 {
 	cycles_t start = get_cycles();
+	if (arch_timer_evtstrm_available()) {
+		const cycles_t timer_evt_period =
+			USECS_TO_CYCLES(ARCH_TIMER_EVT_STREAM_PERIOD_US);
+		while ((get_cycles() - start + timer_evt_period) < cycles)
+			wfe();
+	}
 	while ((get_cycles() - start) < cycles)
 		cpu_relax();
 }
 inline void __const_udelay(unsigned long xloops)
 {
-	unsigned long loops;
+	__delay(xloops_to_cycles(xloops));
-	loops = xloops * loops_per_jiffy * HZ;
-	__delay(loops >> 32);
 }
 EXPORT_SYMBOL(__const_udelay);

arch/arm64/lib/tishift.S+80−0

@@ -0,0 +1,80 @@
	1	/*
	2	* Copyright (C) 2017 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
	3	*
	4	* This program is free software; you can redistribute it and/or modify
	5	* it under the terms of the GNU General Public License version 2 as
	6	* published by the Free Software Foundation.
	7	*
	8	* This program is distributed in the hope that it will be useful,
	9	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	10	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	11	* GNU General Public License for more details.
	12	*
	13	* You should have received a copy of the GNU General Public License
	14	* along with this program. If not, see <http://www.gnu.org/licenses/>.
	15	*/
	16
	17	#include <linux/linkage.h>
	18
	19	ENTRY(__ashlti3)
	20	cbz x2, 1f
	21	mov x3, #64
	22	sub x3, x3, x2
	23	cmp x3, #0
	24	b.le 2f
	25	lsl x1, x1, x2
	26	lsr x3, x0, x3
	27	lsl x2, x0, x2
	28	orr x1, x1, x3
	29	mov x0, x2
	30	1:
	31	ret
	32	2:
	33	neg w1, w3
	34	mov x2, #0
	35	lsl x1, x0, x1
	36	mov x0, x2
	37	ret
	38	ENDPROC(__ashlti3)
	39
	40	ENTRY(__ashrti3)
	41	cbz x2, 3f
	42	mov x3, #64
	43	sub x3, x3, x2
	44	cmp x3, #0
	45	b.le 4f
	46	lsr x0, x0, x2
	47	lsl x3, x1, x3
	48	asr x2, x1, x2
	49	orr x0, x0, x3
	50	mov x1, x2
	51	3:
	52	ret
	53	4:
	54	neg w0, w3
	55	asr x2, x1, #63
	56	asr x0, x1, x0
	57	mov x1, x2
	58	ret
	59	ENDPROC(__ashrti3)
	60
	61	ENTRY(__lshrti3)
	62	cbz x2, 1f
	63	mov x3, #64
	64	sub x3, x3, x2
	65	cmp x3, #0
	66	b.le 2f
	67	lsr x0, x0, x2
	68	lsl x3, x1, x3
	69	lsr x2, x1, x2
	70	orr x0, x0, x3
	71	mov x1, x2
	72	1:
	73	ret
	74	2:
	75	neg w0, w3
	76	mov x2, #0
	77	lsr x0, x1, x0
	78	mov x1, x2
	79	ret
	80	ENDPROC(__lshrti3)

arch/arm64/mm/dma-mapping.c+2−3

 	/* create a coherent mapping */
 	page = virt_to_page(ptr);
 	coherent_ptr = dma_common_contiguous_remap(page, size, VM_USERMAP,
-						   prot, NULL);
+						   prot, __builtin_return_address(0));
 	if (!coherent_ptr)
 		goto no_map;
 			      unsigned long pfn, size_t size)
 {
 	int ret = -ENXIO;
-	unsigned long nr_vma_pages = (vma->vm_end - vma->vm_start) >>
+	unsigned long nr_vma_pages = vma_pages(vma);
-					PAGE_SHIFT;
 	unsigned long nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT;
 	unsigned long off = vma->vm_pgoff;

arch/arm64/mm/fault.c+13−59

 		 (esr & ESR_ELx_WNR) >> ESR_ELx_WNR_SHIFT);
 }
-/*
- * Decode mem abort information
- */
 static void mem_abort_decode(unsigned int esr)
 {
 	pr_alert("Mem abort info:\n");
+	pr_alert("  ESR = 0x%08x\n", esr);
 	pr_alert("  Exception class = %s, IL = %u bits\n",
 		 esr_get_class_string(esr),
 		 (esr & ESR_ELx_IL) ? 32 : 16);
 	return false;
 }
-/*
- * The kernel tried to access some page that wasn't present.
- */
 static void __do_kernel_fault(unsigned long addr, unsigned int esr,
 			      struct pt_regs *regs)
 {
 	if (!is_el1_instruction_abort(esr) && fixup_exception(regs))
 		return;
-	/*
-	 * No handler, we'll have to terminate things with extreme prejudice.
-	 */
 	bust_spinlocks(1);
 	if (is_permission_fault(esr, regs, addr)) {
 	do_exit(SIGKILL);
 }
-/*
- * Something tried to access memory that isn't in our memory map. User mode
- * accesses just cause a SIGSEGV
- */
 static void __do_user_fault(struct task_struct *tsk, unsigned long addr,
 			    unsigned int esr, unsigned int sig, int code,
 			    struct pt_regs *regs, int fault)
 	return 0;
 }
-/*
- * First Level Translation Fault Handler
- *
- * We enter here because the first level page table doesn't contain a valid
- * entry for the address.
- *
- * If the address is in kernel space (>= TASK_SIZE), then we are probably
- * faulting in the vmalloc() area.
- *
- * If the init_task's first level page tables contains the relevant entry, we
- * copy the it to this task.  If not, we send the process a signal, fixup the
- * exception, or oops the kernel.
- *
- * NOTE! We MUST NOT take any locks for this case. We may be in an interrupt
- * or a critical region, and should only copy the information from the master
- * page table, nothing more.
- */
 static int __kprobes do_translation_fault(unsigned long addr,
 					  unsigned int esr,
 					  struct pt_regs *regs)
 	return 0;
 }
-/*
- * This abort handler always returns "fault".
- */
 static int do_bad(unsigned long addr, unsigned int esr, struct pt_regs *regs)
 {
-	return 1;
+	return 1; /* "fault" */
 }
-/*
- * This abort handler deals with Synchronous External Abort.
- * It calls notifiers, and then returns "fault".
- */
 static int do_sea(unsigned long addr, unsigned int esr, struct pt_regs *regs)
 {
 	struct siginfo info;
 	{ do_sea,		SIGBUS,  0,		"level 1 (translation table walk)"	},
 	{ do_sea,		SIGBUS,  0,		"level 2 (translation table walk)"	},
 	{ do_sea,		SIGBUS,  0,		"level 3 (translation table walk)"	},
-	{ do_sea,		SIGBUS,  0,		"synchronous parity or ECC error" },
+	{ do_sea,		SIGBUS,  0,		"synchronous parity or ECC error" },	// Reserved when RAS is implemented
 	{ do_bad,		SIGBUS,  0,		"unknown 25"			},
 	{ do_bad,		SIGBUS,  0,		"unknown 26"			},
 	{ do_bad,		SIGBUS,  0,		"unknown 27"			},
-	{ do_sea,		SIGBUS,  0,		"level 0 synchronous parity error (translation table walk)"	},
+	{ do_sea,		SIGBUS,  0,		"level 0 synchronous parity error (translation table walk)"	},	// Reserved when RAS is implemented
-	{ do_sea,		SIGBUS,  0,		"level 1 synchronous parity error (translation table walk)"	},
+	{ do_sea,		SIGBUS,  0,		"level 1 synchronous parity error (translation table walk)"	},	// Reserved when RAS is implemented
-	{ do_sea,		SIGBUS,  0,		"level 2 synchronous parity error (translation table walk)"	},
+	{ do_sea,		SIGBUS,  0,		"level 2 synchronous parity error (translation table walk)"	},	// Reserved when RAS is implemented
-	{ do_sea,		SIGBUS,  0,		"level 3 synchronous parity error (translation table walk)"	},
+	{ do_sea,		SIGBUS,  0,		"level 3 synchronous parity error (translation table walk)"	},	// Reserved when RAS is implemented
 	{ do_bad,		SIGBUS,  0,		"unknown 32"			},
 	{ do_alignment_fault,	SIGBUS,  BUS_ADRALN,	"alignment fault"		},
 	{ do_bad,		SIGBUS,  0,		"unknown 34"			},
 	{ do_bad,		SIGBUS,  0,		"unknown 46"			},
 	{ do_bad,		SIGBUS,  0,		"unknown 47"			},
 	{ do_bad,		SIGBUS,  0,		"TLB conflict abort"		},
-	{ do_bad,		SIGBUS,  0,		"unknown 49"			},
+	{ do_bad,		SIGBUS,  0,		"Unsupported atomic hardware update fault"	},
 	{ do_bad,		SIGBUS,  0,		"unknown 50"			},
 	{ do_bad,		SIGBUS,  0,		"unknown 51"			},
 	{ do_bad,		SIGBUS,  0,		"implementation fault (lockdown abort)" },
 	{ do_bad,		SIGBUS,  0,		"unknown 63"			},
 };
-/*
- * Handle Synchronous External Aborts that occur in a guest kernel.
- *
- * The return value will be zero if the SEA was successfully handled
- * and non-zero if there was an error processing the error or there was
- * no error to process.
- */
 int handle_guest_sea(phys_addr_t addr, unsigned int esr)
 {
 	int ret = -ENOENT;
 	return ret;
 }
-/*
- * Dispatch a data abort to the relevant handler.
- */
 asmlinkage void __exception do_mem_abort(unsigned long addr, unsigned int esr,
 					 struct pt_regs *regs)
 {
 	if (!inf->fn(addr, esr, regs))
 		return;
-	pr_alert("Unhandled fault: %s (0x%08x) at 0x%016lx\n",
+	pr_alert("Unhandled fault: %s at 0x%016lx\n",
-		 inf->name, esr, addr);
+		 inf->name, addr);
 	mem_abort_decode(esr);
+	if (!user_mode(regs))
+		show_pte(addr);
 	info.si_signo = inf->sig;
 	info.si_errno = 0;
 	info.si_code  = inf->code;
 	arm64_notify_die("", regs, &info, esr);
 }
-/*
- * Handle stack alignment exceptions.
- */
 asmlinkage void __exception do_sp_pc_abort(unsigned long addr,
 					   unsigned int esr,
 					   struct pt_regs *regs)

arch/arm64/mm/proc.S+4−5

 	/*
 	 * __cpu_setup() cleared MDSCR_EL1.MDE and friends, before unmasking
 	 * debug exceptions. By restoring MDSCR_EL1 here, we may take a debug
-	 * exception. Mask them until local_dbg_restore() in cpu_suspend()
+	 * exception. Mask them until local_daif_restore() in cpu_suspend()
 	 * resets them.
 	 */
-	disable_dbg
+	disable_daif
 	msr	mdscr_el1, x10
 	msr	sctlr_el1, x12
  * called by anything else. It can only be executed from a TTBR0 mapping.
  */
 ENTRY(idmap_cpu_replace_ttbr1)
-	mrs	x2, daif
+	save_and_disable_daif flags=x2
-	msr	daifset, #0xf
 	adrp	x1, empty_zero_page
 	msr	ttbr1_el1, x1
 	msr	ttbr1_el1, x0
 	isb
-	msr	daif, x2
+	restore_daif x2
 	ret
 ENDPROC(idmap_cpu_replace_ttbr1)

drivers/acpi/arm64/gtdt.c+1−1

@@ -199,7 +199,7 @@ static int __init gtdt_parse_timer_block(struct acpi_gtdt_timer_block *block, · 198 unmodified lines
199	struct acpi_gtdt_timer_entry *gtdt_frame;	199	struct acpi_gtdt_timer_entry *gtdt_frame;
200		200
201	if (!block->timer_count) {	201	if (!block->timer_count) {
202	pr_err(FW_BUG "GT block present, but frame count is zero.");	202	pr_err(FW_BUG "GT block present, but frame count is zero.\n");
203	return -ENODEV;	203	return -ENODEV;
204	}	204	}
205		205

drivers/acpi/arm64/iort.c+200−58

  *
  * Returns: fwnode_handle pointer on success, NULL on failure
  */
-static inline
+static inline struct fwnode_handle *iort_get_fwnode(
-struct fwnode_handle *iort_get_fwnode(struct acpi_iort_node *node)
+			struct acpi_iort_node *node)
 {
 	struct iort_fwnode *curr;
 	struct fwnode_handle *fwnode = NULL;
 	spin_unlock(&iort_fwnode_lock);
 }
+/**
+ * iort_get_iort_node() - Retrieve iort_node associated with an fwnode
+ *
+ * @fwnode: fwnode associated with device to be looked-up
+ *
+ * Returns: iort_node pointer on success, NULL on failure
+ */
+static inline struct acpi_iort_node *iort_get_iort_node(
+			struct fwnode_handle *fwnode)
+{
+	struct iort_fwnode *curr;
+	struct acpi_iort_node *iort_node = NULL;
+	spin_lock(&iort_fwnode_lock);
+	list_for_each_entry(curr, &iort_fwnode_list, list) {
+		if (curr->fwnode == fwnode) {
+			iort_node = curr->iort_node;
+			break;
+		}
+	}
+	spin_unlock(&iort_fwnode_lock);
+	return iort_node;
+}
 typedef acpi_status (*iort_find_node_callback)
 	(struct acpi_iort_node *node, void *context);
 	return 0;
 }
-static
+static struct acpi_iort_node *iort_node_get_id(struct acpi_iort_node *node,
-struct acpi_iort_node *iort_node_get_id(struct acpi_iort_node *node,
+					       u32 *id_out, int index)
-					u32 *id_out, int index)
 {
 	struct acpi_iort_node *parent;
 	struct acpi_iort_id_mapping *map;
 	if (map->flags & ACPI_IORT_ID_SINGLE_MAPPING) {
 		if (node->type == ACPI_IORT_NODE_NAMED_COMPONENT ||
-		    node->type == ACPI_IORT_NODE_PCI_ROOT_COMPLEX) {
+		    node->type == ACPI_IORT_NODE_PCI_ROOT_COMPLEX ||
+		    node->type == ACPI_IORT_NODE_SMMU_V3) {
 			*id_out = map->output_base;
 			return parent;
 		}
 	return NULL;
 }
+#if (ACPI_CA_VERSION > 0x20170929)
+static int iort_get_id_mapping_index(struct acpi_iort_node *node)
+{
+	struct acpi_iort_smmu_v3 *smmu;
+	switch (node->type) {
+	case ACPI_IORT_NODE_SMMU_V3:
+		/*
+		 * SMMUv3 dev ID mapping index was introduced in revision 1
+		 * table, not available in revision 0
+		 */
+		if (node->revision < 1)
+			return -EINVAL;
+		smmu = (struct acpi_iort_smmu_v3 *)node->node_data;
+		/*
+		 * ID mapping index is only ignored if all interrupts are
+		 * GSIV based
+		 */
+		if (smmu->event_gsiv && smmu->pri_gsiv && smmu->gerr_gsiv
+		    && smmu->sync_gsiv)
+			return -EINVAL;
+		if (smmu->id_mapping_index >= node->mapping_count) {
+			pr_err(FW_BUG "[node %p type %d] ID mapping index overflows valid mappings\n",
+			       node, node->type);
+			return -EINVAL;
+		}
+		return smmu->id_mapping_index;
+	default:
+		return -EINVAL;
+	}
+}
+#else
+static inline int iort_get_id_mapping_index(struct acpi_iort_node *node)
+{
+	return -EINVAL;
+}
+#endif
 static struct acpi_iort_node *iort_node_map_id(struct acpi_iort_node *node,
 					       u32 id_in, u32 *id_out,
 					       u8 type_mask)
 	/* Parse the ID mapping tree to find specified node type */
 	while (node) {
 		struct acpi_iort_id_mapping *map;
-		int i;
+		int i, index;
 		if (IORT_TYPE_MASK(node->type) & type_mask) {
 			if (id_out)
 			goto fail_map;
 		}
+		/*
+		 * Get the special ID mapping index (if any) and skip its
+		 * associated ID map to prevent erroneous multi-stage
+		 * IORT ID translations.
+		 */
+		index = iort_get_id_mapping_index(node);
 		/* Do the ID translation */
 		for (i = 0; i < node->mapping_count; i++, map++) {
+			/* if it is special mapping index, skip it */
+			if (i == index)
+				continue;
 			if (!iort_id_map(map, node->type, id, &id))
 				break;
 		}
 	return NULL;
 }
-static
+static struct acpi_iort_node *iort_node_map_platform_id(
-struct acpi_iort_node *iort_node_map_platform_id(struct acpi_iort_node *node,
+		struct acpi_iort_node *node, u32 *id_out, u8 type_mask,
-						 u32 *id_out, u8 type_mask,
+		int index)
-						 int index)
 {
 	struct acpi_iort_node *parent;
 	u32 id;
 {
 	struct pci_bus *pbus;
-	if (!dev_is_pci(dev))
+	if (!dev_is_pci(dev)) {
+		struct acpi_iort_node *node;
+		/*
+		 * scan iort_fwnode_list to see if it's an iort platform
+		 * device (such as SMMU, PMCG),its iort node already cached
+		 * and associated with fwnode when iort platform devices
+		 * were initialized.
+		 */
+		node = iort_get_iort_node(dev->fwnode);
+		if (node)
+			return node;
+		/*
+		 * if not, then it should be a platform device defined in
+		 * DSDT/SSDT (with Named Component node in IORT)
+		 */
 		return iort_scan_node(ACPI_IORT_NODE_NAMED_COMPONENT,
 				      iort_match_node_callback, dev);
+	}
 	/* Find a PCI root bus */
 	pbus = to_pci_dev(dev)->bus;
  */
 int iort_pmsi_get_dev_id(struct device *dev, u32 *dev_id)
 {
-	int i;
+	int i, index;
 	struct acpi_iort_node *node;
 	node = iort_find_dev_node(dev);
 	if (!node)
 		return -ENODEV;
-	for (i = 0; i < node->mapping_count; i++) {
+	index = iort_get_id_mapping_index(node);
-		if (iort_node_map_platform_id(node, dev_id, IORT_MSI_TYPE, i))
+	/* if there is a valid index, go get the dev_id directly */
+	if (index >= 0) {
+		if (iort_node_get_id(node, dev_id, index))
 			return 0;
+	} else {
+		for (i = 0; i < node->mapping_count; i++) {
+			if (iort_node_map_platform_id(node, dev_id,
+						      IORT_MSI_TYPE, i))
+				return 0;
+		}
 	}
 	return -ENODEV;
 	return irq_find_matching_fwnode(handle, DOMAIN_BUS_PCI_MSI);
 }
+static void iort_set_device_domain(struct device *dev,
+				   struct acpi_iort_node *node)
+{
+	struct acpi_iort_its_group *its;
+	struct acpi_iort_node *msi_parent;
+	struct acpi_iort_id_mapping *map;
+	struct fwnode_handle *iort_fwnode;
+	struct irq_domain *domain;
+	int index;
+	index = iort_get_id_mapping_index(node);
+	if (index < 0)
+		return;
+	map = ACPI_ADD_PTR(struct acpi_iort_id_mapping, node,
+			   node->mapping_offset + index * sizeof(*map));
+	/* Firmware bug! */
+	if (!map->output_reference ||
+	    !(map->flags & ACPI_IORT_ID_SINGLE_MAPPING)) {
+		pr_err(FW_BUG "[node %p type %d] Invalid MSI mapping\n",
+		       node, node->type);
+		return;
+	}
+	msi_parent = ACPI_ADD_PTR(struct acpi_iort_node, iort_table,
+				  map->output_reference);
+	if (!msi_parent || msi_parent->type != ACPI_IORT_NODE_ITS_GROUP)
+		return;
+	/* Move to ITS specific data */
+	its = (struct acpi_iort_its_group *)msi_parent->node_data;
+	iort_fwnode = iort_find_domain_token(its->identifiers[0]);
+	if (!iort_fwnode)
+		return;
+	domain = irq_find_matching_fwnode(iort_fwnode, DOMAIN_BUS_PLATFORM_MSI);
+	if (domain)
+		dev_set_msi_domain(dev, domain);
+}
 /**
  * iort_get_platform_device_domain() - Find MSI domain related to a
  * platform device
 }
 #ifdef CONFIG_IOMMU_API
-static inline
+static inline const struct iommu_ops *iort_fwspec_iommu_ops(
-const struct iommu_ops *iort_fwspec_iommu_ops(struct iommu_fwspec *fwspec)
+				struct iommu_fwspec *fwspec)
 {
 	return (fwspec && fwspec->ops) ? fwspec->ops : NULL;
 }
-static inline
+static inline int iort_add_device_replay(const struct iommu_ops *ops,
-int iort_add_device_replay(const struct iommu_ops *ops, struct device *dev)
+					 struct device *dev)
 {
 	int err = 0;
 	return err;
 }
 #else
-static inline
+static inline const struct iommu_ops *iort_fwspec_iommu_ops(
-const struct iommu_ops *iort_fwspec_iommu_ops(struct iommu_fwspec *fwspec)
+				struct iommu_fwspec *fwspec)
 { return NULL; }
-static inline
+static inline int iort_add_device_replay(const struct iommu_ops *ops,
-int iort_add_device_replay(const struct iommu_ops *ops, struct device *dev)
+					 struct device *dev)
 { return 0; }
 #endif
 	return smmu->flags & ACPI_IORT_SMMU_V3_COHACC_OVERRIDE;
 }
-#if defined(CONFIG_ACPI_NUMA) && defined(ACPI_IORT_SMMU_V3_PXM_VALID)
+#if defined(CONFIG_ACPI_NUMA)
 /*
  * set numa proximity domain for smmuv3 device
  */
 	return smmu->flags & ACPI_IORT_SMMU_COHERENT_WALK;
 }
-struct iort_iommu_config {
+struct iort_dev_config {
 	const char *name;
-	int (*iommu_init)(struct acpi_iort_node *node);
+	int (*dev_init)(struct acpi_iort_node *node);
-	bool (*iommu_is_coherent)(struct acpi_iort_node *node);
+	bool (*dev_is_coherent)(struct acpi_iort_node *node);
-	int (*iommu_count_resources)(struct acpi_iort_node *node);
+	int (*dev_count_resources)(struct acpi_iort_node *node);
-	void (*iommu_init_resources)(struct resource *res,
+	void (*dev_init_resources)(struct resource *res,
 				     struct acpi_iort_node *node);
-	void (*iommu_set_proximity)(struct device *dev,
+	void (*dev_set_proximity)(struct device *dev,
 				    struct acpi_iort_node *node);
 };
-static const struct iort_iommu_config iort_arm_smmu_v3_cfg __initconst = {
+static const struct iort_dev_config iort_arm_smmu_v3_cfg __initconst = {
 	.name = "arm-smmu-v3",
-	.iommu_is_coherent = arm_smmu_v3_is_coherent,
+	.dev_is_coherent = arm_smmu_v3_is_coherent,
-	.iommu_count_resources = arm_smmu_v3_count_resources,
+	.dev_count_resources = arm_smmu_v3_count_resources,
-	.iommu_init_resources = arm_smmu_v3_init_resources,
+	.dev_init_resources = arm_smmu_v3_init_resources,
-	.iommu_set_proximity = arm_smmu_v3_set_proximity,
+	.dev_set_proximity = arm_smmu_v3_set_proximity,
 };
-static const struct iort_iommu_config iort_arm_smmu_cfg __initconst = {
+static const struct iort_dev_config iort_arm_smmu_cfg __initconst = {
 	.name = "arm-smmu",
-	.iommu_is_coherent = arm_smmu_is_coherent,
+	.dev_is_coherent = arm_smmu_is_coherent,
-	.iommu_count_resources = arm_smmu_count_resources,
+	.dev_count_resources = arm_smmu_count_resources,
-	.iommu_init_resources = arm_smmu_init_resources
+	.dev_init_resources = arm_smmu_init_resources
 };
-static __init
+static __init const struct iort_dev_config *iort_get_dev_cfg(
-const struct iort_iommu_config *iort_get_iommu_cfg(struct acpi_iort_node *node)
+			struct acpi_iort_node *node)
 {
 	switch (node->type) {
 	case ACPI_IORT_NODE_SMMU_V3:
 }
 /**
- * iort_add_smmu_platform_device() - Allocate a platform device for SMMU
+ * iort_add_platform_device() - Allocate a platform device for IORT node
- * @node: Pointer to SMMU ACPI IORT node
+ * @node: Pointer to device ACPI IORT node
  *
  * Returns: 0 on success, <0 failure
  */
-static int __init iort_add_smmu_platform_device(struct acpi_iort_node *node)
+static int __init iort_add_platform_device(struct acpi_iort_node *node,
+					   const struct iort_dev_config *ops)
 {
 	struct fwnode_handle *fwnode;
 	struct platform_device *pdev;
 	struct resource *r;
 	enum dev_dma_attr attr;
 	int ret, count;
-	const struct iort_iommu_config *ops = iort_get_iommu_cfg(node);
-	if (!ops)
-		return -ENODEV;
 	pdev = platform_device_alloc(ops->name, PLATFORM_DEVID_AUTO);
 	if (!pdev)
 		return -ENOMEM;
-	if (ops->iommu_set_proximity)
+	if (ops->dev_set_proximity)
-		ops->iommu_set_proximity(&pdev->dev, node);
+		ops->dev_set_proximity(&pdev->dev, node);
-	count = ops->iommu_count_resources(node);
+	count = ops->dev_count_resources(node);
 	r = kcalloc(count, sizeof(*r), GFP_KERNEL);
 	if (!r) {
 		goto dev_put;
 	}
-	ops->iommu_init_resources(r, node);
+	ops->dev_init_resources(r, node);
 	ret = platform_device_add_resources(pdev, r, count);
 	/*
 	pdev->dev.fwnode = fwnode;
-	attr = ops->iommu_is_coherent(node) ?
+	attr = ops->dev_is_coherent && ops->dev_is_coherent(node) ?
-			     DEV_DMA_COHERENT : DEV_DMA_NON_COHERENT;
+			DEV_DMA_COHERENT : DEV_DMA_NON_COHERENT;
 	/* Configure DMA for the page table walker */
 	acpi_dma_configure(&pdev->dev, attr);
+	iort_set_device_domain(&pdev->dev, node);
 	ret = platform_device_add(pdev);
 	if (ret)
 		goto dma_deconfigure;
 	struct fwnode_handle *fwnode;
 	int i, ret;
 	bool acs_enabled = false;
+	const struct iort_dev_config *ops;
 	/*
 	 * iort_table and iort both point to the start of IORT table, but
 		if (!acs_enabled)
 			acs_enabled = iort_enable_acs(iort_node);
-		if ((iort_node->type == ACPI_IORT_NODE_SMMU) ||
+		ops = iort_get_dev_cfg(iort_node);
-			(iort_node->type == ACPI_IORT_NODE_SMMU_V3)) {
+		if (ops) {
 			fwnode = acpi_alloc_fwnode_static();
 			if (!fwnode)
 				return;
 			iort_set_fwnode(iort_node, fwnode);
-			ret = iort_add_smmu_platform_device(iort_node);
+			ret = iort_add_platform_device(iort_node, ops);
 			if (ret) {
 				iort_delete_fwnode(iort_node);
 				acpi_free_fwnode_static(fwnode);

drivers/bus/arm-ccn.c+1−0

@@ -1276,6 +1276,7 @@ static int arm_ccn_pmu_init(struct arm_ccn *ccn) · 1275 unmodified lines
1276		1276
1277	/* Perf driver registration */	1277	/* Perf driver registration */
1278	ccn->dt.pmu = (struct pmu) {	1278	ccn->dt.pmu = (struct pmu) {
		1279	.module = THIS_MODULE,
1279	.attr_groups = arm_ccn_pmu_attr_groups,	1280	.attr_groups = arm_ccn_pmu_attr_groups,
1280	.task_ctx_nr = perf_invalid_context,	1281	.task_ctx_nr = perf_invalid_context,
1281	.event_init = arm_ccn_pmu_event_init,	1282	.event_init = arm_ccn_pmu_event_init,

drivers/clocksource/arm_arch_timer.c+22−3

@@ -77,6 +77,7 @@ static bool arch_timer_mem_use_virtual; · 76 unmodified lines
77	static bool arch_counter_suspend_stop;	77	static bool arch_counter_suspend_stop;
78	static bool vdso_default = true;	78	static bool vdso_default = true;
79		79
		80	static cpumask_t evtstrm_available = CPU_MASK_NONE;
80	static bool evtstrm_enable = IS_ENABLED(CONFIG_ARM_ARCH_TIMER_EVTSTREAM);	81	static bool evtstrm_enable = IS_ENABLED(CONFIG_ARM_ARCH_TIMER_EVTSTREAM);
81		82
82	static int __init early_evtstrm_cfg(char *buf)	83	static int __init early_evtstrm_cfg(char *buf)
@@ -739,6 +740,7 @@ static void arch_timer_evtstrm_enable(int divider) · 656 unmodified lines
739	#ifdef CONFIG_COMPAT	740	#ifdef CONFIG_COMPAT
740	compat_elf_hwcap \|= COMPAT_HWCAP_EVTSTRM;	741	compat_elf_hwcap \|= COMPAT_HWCAP_EVTSTRM;
741	#endif	742	#endif
		743	cpumask_set_cpu(smp_processor_id(), &evtstrm_available);
742	}	744	}
743		745
744	static void arch_timer_configure_evtstream(void)	746	static void arch_timer_configure_evtstream(void)
@@ -863,6 +865,16 @@ u32 arch_timer_get_rate(void) · 118 unmodified lines
863	return arch_timer_rate;	865	return arch_timer_rate;
864	}	866	}
865		867
		868	bool arch_timer_evtstrm_available(void)
		869	{
		870	/*
		871	* We might get called from a preemptible context. This is fine
		872	* because availability of the event stream should be always the same
		873	* for a preemptible context and context where we might resume a task.
		874	*/
		875	return cpumask_test_cpu(raw_smp_processor_id(), &evtstrm_available);
		876	}
		877
866	static u64 arch_counter_get_cntvct_mem(void)	878	static u64 arch_counter_get_cntvct_mem(void)
867	{	879	{
868	u32 vct_lo, vct_hi, tmp_hi;	880	u32 vct_lo, vct_hi, tmp_hi;
@@ -928,6 +940,8 @@ static int arch_timer_dying_cpu(unsigned int cpu) · 59 unmodified lines
928	{	940	{
929	struct clock_event_device *clk = this_cpu_ptr(arch_timer_evt);	941	struct clock_event_device *clk = this_cpu_ptr(arch_timer_evt);
930		942
		943	cpumask_clear_cpu(smp_processor_id(), &evtstrm_available);
		944
931	arch_timer_stop(clk);	945	arch_timer_stop(clk);
932	return 0;	946	return 0;
933	}	947	}
@@ -937,10 +951,16 @@ static DEFINE_PER_CPU(unsigned long, saved_cntkctl); · 3 unmodified lines
937	static int arch_timer_cpu_pm_notify(struct notifier_block *self,	951	static int arch_timer_cpu_pm_notify(struct notifier_block *self,
938	unsigned long action, void *hcpu)	952	unsigned long action, void *hcpu)
939	{	953	{
940	if (action == CPU_PM_ENTER)	954	if (action == CPU_PM_ENTER) {
941	__this_cpu_write(saved_cntkctl, arch_timer_get_cntkctl());	955	__this_cpu_write(saved_cntkctl, arch_timer_get_cntkctl());
942	else if (action == CPU_PM_ENTER_FAILED \|\| action == CPU_PM_EXIT)	956
		957	cpumask_clear_cpu(smp_processor_id(), &evtstrm_available);
		958	} else if (action == CPU_PM_ENTER_FAILED \|\| action == CPU_PM_EXIT) {
943	arch_timer_set_cntkctl(__this_cpu_read(saved_cntkctl));	959	arch_timer_set_cntkctl(__this_cpu_read(saved_cntkctl));
		960
		961	if (elf_hwcap & HWCAP_EVTSTRM)
		962	cpumask_set_cpu(smp_processor_id(), &evtstrm_available);
		963	}
944	return NOTIFY_OK;	964	return NOTIFY_OK;
945	}	965	}
946		966
@@ -1016,7 +1036,6 @@ static int __init arch_timer_register(void) · 69 unmodified lines
1016	if (err)	1036	if (err)
1017	goto out_unreg_notify;	1037	goto out_unreg_notify;
1018		1038
1019
1020	/* Register and immediately configure the timer on the boot CPU */	1039	/* Register and immediately configure the timer on the boot CPU */
1021	err = cpuhp_setup_state(CPUHP_AP_ARM_ARCH_TIMER_STARTING,	1040	err = cpuhp_setup_state(CPUHP_AP_ARM_ARCH_TIMER_STARTING,
1022	"clockevents/arm/arch_timer:starting",	1041	"clockevents/arm/arch_timer:starting",

drivers/perf/Kconfig+15−0

@@ -17,6 +17,13 @@ config ARM_PMU_ACPI · 16 unmodified lines
17	depends on ARM_PMU && ACPI	17	depends on ARM_PMU && ACPI
18	def_bool y	18	def_bool y
19		19
		20	config HISI_PMU
		21	bool "HiSilicon SoC PMU"
		22	depends on ARM64 && ACPI
		23	help
		24	Support for HiSilicon SoC uncore performance monitoring
		25	unit (PMU), such as: L3C, HHA and DDRC.
		26
20	config QCOM_L2_PMU	27	config QCOM_L2_PMU
21	bool "Qualcomm Technologies L2-cache PMU"	28	bool "Qualcomm Technologies L2-cache PMU"
22	depends on ARCH_QCOM && ARM64 && ACPI	29	depends on ARCH_QCOM && ARM64 && ACPI
@@ -43,4 +50,12 @@ config XGENE_PMU · 20 unmodified lines
43	help	50	help
44	Say y if you want to use APM X-Gene SoC performance monitors.	51	Say y if you want to use APM X-Gene SoC performance monitors.
45		52
		53	config ARM_SPE_PMU
		54	tristate "Enable support for the ARMv8.2 Statistical Profiling Extension"
		55	depends on PERF_EVENTS && ARM64
		56	help
		57	Enable perf support for the ARMv8.2 Statistical Profiling
		58	Extension, which provides periodic sampling of operations in
		59	the CPU pipeline and reports this via the perf AUX interface.
		60
46	endmenu	61	endmenu

drivers/perf/Makefile+2−0

 # SPDX-License-Identifier: GPL-2.0
 obj-$(CONFIG_ARM_PMU) += arm_pmu.o arm_pmu_platform.o
 obj-$(CONFIG_ARM_PMU_ACPI) += arm_pmu_acpi.o
+obj-$(CONFIG_HISI_PMU) += hisilicon/
 obj-$(CONFIG_QCOM_L2_PMU)	+= qcom_l2_pmu.o
 obj-$(CONFIG_QCOM_L3_PMU) += qcom_l3_pmu.o
 obj-$(CONFIG_XGENE_PMU) += xgene_pmu.o
+obj-$(CONFIG_ARM_SPE_PMU) += arm_spe_pmu.o

drivers/perf/arm_pmu.c+5−5

@@ -539,7 +539,7 @@ void armpmu_free_irq(struct arm_pmu *armpmu, int cpu) · 538 unmodified lines
539	if (!cpumask_test_and_clear_cpu(cpu, &armpmu->active_irqs))	539	if (!cpumask_test_and_clear_cpu(cpu, &armpmu->active_irqs))
540	return;	540	return;
541		541
542	if (irq_is_percpu(irq)) {	542	if (irq_is_percpu_devid(irq)) {
543	free_percpu_irq(irq, &hw_events->percpu_pmu);	543	free_percpu_irq(irq, &hw_events->percpu_pmu);
544	cpumask_clear(&armpmu->active_irqs);	544	cpumask_clear(&armpmu->active_irqs);
545	return;	545	return;
@@ -565,10 +565,10 @@ int armpmu_request_irq(struct arm_pmu *armpmu, int cpu) · 19 unmodified lines
565	if (!irq)	565	if (!irq)
566	return 0;	566	return 0;
567		567
568	if (irq_is_percpu(irq) && cpumask_empty(&armpmu->active_irqs)) {	568	if (irq_is_percpu_devid(irq) && cpumask_empty(&armpmu->active_irqs)) {
569	err = request_percpu_irq(irq, handler, "arm-pmu",	569	err = request_percpu_irq(irq, handler, "arm-pmu",
570	&hw_events->percpu_pmu);	570	&hw_events->percpu_pmu);
571	} else if (irq_is_percpu(irq)) {	571	} else if (irq_is_percpu_devid(irq)) {
572	int other_cpu = cpumask_first(&armpmu->active_irqs);	572	int other_cpu = cpumask_first(&armpmu->active_irqs);
573	int other_irq = per_cpu(hw_events->irq, other_cpu);	573	int other_irq = per_cpu(hw_events->irq, other_cpu);
574		574
@@ -649,7 +649,7 @@ static int arm_perf_starting_cpu(unsigned int cpu, struct hlist_node *node) · 74 unmodified lines
649		649
650	irq = armpmu_get_cpu_irq(pmu, cpu);	650	irq = armpmu_get_cpu_irq(pmu, cpu);
651	if (irq) {	651	if (irq) {
652	if (irq_is_percpu(irq)) {	652	if (irq_is_percpu_devid(irq)) {
653	enable_percpu_irq(irq, IRQ_TYPE_NONE);	653	enable_percpu_irq(irq, IRQ_TYPE_NONE);
654	return 0;	654	return 0;
655	}	655	}
@@ -667,7 +667,7 @@ static int arm_perf_teardown_cpu(unsigned int cpu, struct hlist_node *node) · 11 unmodified lines
667	return 0;	667	return 0;
668		668
669	irq = armpmu_get_cpu_irq(pmu, cpu);	669	irq = armpmu_get_cpu_irq(pmu, cpu);
670	if (irq && irq_is_percpu(irq))	670	if (irq && irq_is_percpu_devid(irq))
671	disable_percpu_irq(irq);	671	disable_percpu_irq(irq);
672		672
673	return 0;	673	return 0;

drivers/perf/arm_pmu_acpi.c+0−3

@@ -193,9 +193,6 @@ int arm_pmu_acpi_probe(armpmu_init_fn init_fn) · 192 unmodified lines
193	int pmu_idx = 0;	193	int pmu_idx = 0;
194	int cpu, ret;	194	int cpu, ret;
195		195
196	if (acpi_disabled)
197	return 0;
198
199	/*	196	/*
200	* Initialise and register the set of PMUs which we know about right	197	* Initialise and register the set of PMUs which we know about right
201	* now. Ideally we'd do this in arm_pmu_acpi_cpu_starting() so that we	198	* now. Ideally we'd do this in arm_pmu_acpi_cpu_starting() so that we

drivers/perf/arm_pmu_platform.c+2−2

@@ -127,7 +127,7 @@ static int pmu_parse_irqs(struct arm_pmu *pmu) · 126 unmodified lines
127		127
128	if (num_irqs == 1) {	128	if (num_irqs == 1) {
129	int irq = platform_get_irq(pdev, 0);	129	int irq = platform_get_irq(pdev, 0);
130	if (irq && irq_is_percpu(irq))	130	if (irq && irq_is_percpu_devid(irq))
131	return pmu_parse_percpu_irq(pmu, irq);	131	return pmu_parse_percpu_irq(pmu, irq);
132	}	132	}
133		133
@@ -150,7 +150,7 @@ static int pmu_parse_irqs(struct arm_pmu *pmu) · 16 unmodified lines
150	if (WARN_ON(irq <= 0))	150	if (WARN_ON(irq <= 0))
151	continue;	151	continue;
152		152
153	if (irq_is_percpu(irq)) {	153	if (irq_is_percpu_devid(irq)) {
154	pr_warn("multiple PPIs or mismatched SPI/PPI detected\n");	154	pr_warn("multiple PPIs or mismatched SPI/PPI detected\n");
155	return -EINVAL;	155	return -EINVAL;
156	}	156	}

drivers/perf/arm_spe_pmu.c+1249−0

@@ -0,0 +1,1249 @@
	1	/*
	2	* Perf support for the Statistical Profiling Extension, introduced as
	3	* part of ARMv8.2.
	4	*
	5	* This program is free software; you can redistribute it and/or modify
	6	* it under the terms of the GNU General Public License version 2 as
	7	* published by the Free Software Foundation.
	8	*
	9	* This program is distributed in the hope that it will be useful,
	10	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	11	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	12	* GNU General Public License for more details.
	13	*
	14	* You should have received a copy of the GNU General Public License
	15	* along with this program. If not, see <http://www.gnu.org/licenses/>.
	16	*
	17	* Copyright (C) 2016 ARM Limited
	18	*
	19	* Author: Will Deacon <will.deacon@arm.com>
	20	*/
	21
	22	#define PMUNAME "arm_spe"
	23	#define DRVNAME PMUNAME "_pmu"
	24	#define pr_fmt(fmt) DRVNAME ": " fmt
	25
	26	#include <linux/cpuhotplug.h>
	27	#include <linux/interrupt.h>
	28	#include <linux/irq.h>
	29	#include <linux/module.h>
	30	#include <linux/of_address.h>
	31	#include <linux/of_device.h>
	32	#include <linux/perf_event.h>
	33	#include <linux/platform_device.h>
	34	#include <linux/slab.h>
	35
	36	#include <asm/sysreg.h>
	37
	38	#define ARM_SPE_BUF_PAD_BYTE 0
	39
	40	struct arm_spe_pmu_buf {
	41	int nr_pages;
	42	bool snapshot;
	43	void *base;
	44	};
	45
	46	struct arm_spe_pmu {
	47	struct pmu pmu;
	48	struct platform_device *pdev;
	49	cpumask_t supported_cpus;
	50	struct hlist_node hotplug_node;
	51
	52	int irq; /* PPI */
	53
	54	u16 min_period;
	55	u16 counter_sz;
	56
	57	#define SPE_PMU_FEAT_FILT_EVT (1UL << 0)
	58	#define SPE_PMU_FEAT_FILT_TYP (1UL << 1)
	59	#define SPE_PMU_FEAT_FILT_LAT (1UL << 2)
	60	#define SPE_PMU_FEAT_ARCH_INST (1UL << 3)
	61	#define SPE_PMU_FEAT_LDS (1UL << 4)
	62	#define SPE_PMU_FEAT_ERND (1UL << 5)
	63	#define SPE_PMU_FEAT_DEV_PROBED (1UL << 63)
	64	u64 features;
	65
	66	u16 max_record_sz;
	67	u16 align;
	68	struct perf_output_handle __percpu *handle;
	69	};
	70
	71	#define to_spe_pmu(p) (container_of(p, struct arm_spe_pmu, pmu))
	72
	73	/* Convert a free-running index from perf into an SPE buffer offset */
	74	#define PERF_IDX2OFF(idx, buf) ((idx) % ((buf)->nr_pages << PAGE_SHIFT))
	75
	76	/* Keep track of our dynamic hotplug state */
	77	static enum cpuhp_state arm_spe_pmu_online;
	78
	79	enum arm_spe_pmu_buf_fault_action {
	80	SPE_PMU_BUF_FAULT_ACT_SPURIOUS,
	81	SPE_PMU_BUF_FAULT_ACT_FATAL,
	82	SPE_PMU_BUF_FAULT_ACT_OK,
	83	};
	84
	85	/* This sysfs gunk was really good fun to write. */
	86	enum arm_spe_pmu_capabilities {
	87	SPE_PMU_CAP_ARCH_INST = 0,
	88	SPE_PMU_CAP_ERND,
	89	SPE_PMU_CAP_FEAT_MAX,
	90	SPE_PMU_CAP_CNT_SZ = SPE_PMU_CAP_FEAT_MAX,
	91	SPE_PMU_CAP_MIN_IVAL,
	92	};
	93
	94	static int arm_spe_pmu_feat_caps[SPE_PMU_CAP_FEAT_MAX] = {
	95	[SPE_PMU_CAP_ARCH_INST] = SPE_PMU_FEAT_ARCH_INST,
	96	[SPE_PMU_CAP_ERND] = SPE_PMU_FEAT_ERND,
	97	};
	98
	99	static u32 arm_spe_pmu_cap_get(struct arm_spe_pmu *spe_pmu, int cap)
	100	{
	101	if (cap < SPE_PMU_CAP_FEAT_MAX)
	102	return !!(spe_pmu->features & arm_spe_pmu_feat_caps[cap]);
	103
	104	switch (cap) {
	105	case SPE_PMU_CAP_CNT_SZ:
	106	return spe_pmu->counter_sz;
	107	case SPE_PMU_CAP_MIN_IVAL:
	108	return spe_pmu->min_period;
	109	default:
	110	WARN(1, "unknown cap %d\n", cap);
	111	}
	112
	113	return 0;
	114	}
	115
	116	static ssize_t arm_spe_pmu_cap_show(struct device *dev,
	117	struct device_attribute *attr,
	118	char *buf)
	119	{
	120	struct platform_device *pdev = to_platform_device(dev);
	121	struct arm_spe_pmu *spe_pmu = platform_get_drvdata(pdev);
	122	struct dev_ext_attribute *ea =
	123	container_of(attr, struct dev_ext_attribute, attr);
	124	int cap = (long)ea->var;
	125
	126	return snprintf(buf, PAGE_SIZE, "%u\n",
	127	arm_spe_pmu_cap_get(spe_pmu, cap));
	128	}
	129
	130	#define SPE_EXT_ATTR_ENTRY(_name, _func, _var) \
	131	&((struct dev_ext_attribute[]) { \
	132	{ __ATTR(_name, S_IRUGO, _func, NULL), (void *)_var } \
	133	})[0].attr.attr
	134
	135	#define SPE_CAP_EXT_ATTR_ENTRY(_name, _var) \
	136	SPE_EXT_ATTR_ENTRY(_name, arm_spe_pmu_cap_show, _var)
	137
	138	static struct attribute *arm_spe_pmu_cap_attr[] = {
	139	SPE_CAP_EXT_ATTR_ENTRY(arch_inst, SPE_PMU_CAP_ARCH_INST),
	140	SPE_CAP_EXT_ATTR_ENTRY(ernd, SPE_PMU_CAP_ERND),
	141	SPE_CAP_EXT_ATTR_ENTRY(count_size, SPE_PMU_CAP_CNT_SZ),
	142	SPE_CAP_EXT_ATTR_ENTRY(min_interval, SPE_PMU_CAP_MIN_IVAL),
	143	NULL,
	144	};
	145
	146	static struct attribute_group arm_spe_pmu_cap_group = {
	147	.name = "caps",
	148	.attrs = arm_spe_pmu_cap_attr,
	149	};
	150
	151	/* User ABI */
	152	#define ATTR_CFG_FLD_ts_enable_CFG config /* PMSCR_EL1.TS */
	153	#define ATTR_CFG_FLD_ts_enable_LO 0
	154	#define ATTR_CFG_FLD_ts_enable_HI 0
	155	#define ATTR_CFG_FLD_pa_enable_CFG config /* PMSCR_EL1.PA */
	156	#define ATTR_CFG_FLD_pa_enable_LO 1
	157	#define ATTR_CFG_FLD_pa_enable_HI 1
	158	#define ATTR_CFG_FLD_pct_enable_CFG config /* PMSCR_EL1.PCT */
	159	#define ATTR_CFG_FLD_pct_enable_LO 2
	160	#define ATTR_CFG_FLD_pct_enable_HI 2
	161	#define ATTR_CFG_FLD_jitter_CFG config /* PMSIRR_EL1.RND */
	162	#define ATTR_CFG_FLD_jitter_LO 16
	163	#define ATTR_CFG_FLD_jitter_HI 16
	164	#define ATTR_CFG_FLD_branch_filter_CFG config /* PMSFCR_EL1.B */
	165	#define ATTR_CFG_FLD_branch_filter_LO 32
	166	#define ATTR_CFG_FLD_branch_filter_HI 32
	167	#define ATTR_CFG_FLD_load_filter_CFG config /* PMSFCR_EL1.LD */
	168	#define ATTR_CFG_FLD_load_filter_LO 33
	169	#define ATTR_CFG_FLD_load_filter_HI 33
	170	#define ATTR_CFG_FLD_store_filter_CFG config /* PMSFCR_EL1.ST */
	171	#define ATTR_CFG_FLD_store_filter_LO 34
	172	#define ATTR_CFG_FLD_store_filter_HI 34
	173
	174	#define ATTR_CFG_FLD_event_filter_CFG config1 /* PMSEVFR_EL1 */
	175	#define ATTR_CFG_FLD_event_filter_LO 0
	176	#define ATTR_CFG_FLD_event_filter_HI 63
	177
	178	#define ATTR_CFG_FLD_min_latency_CFG config2 /* PMSLATFR_EL1.MINLAT */
	179	#define ATTR_CFG_FLD_min_latency_LO 0
	180	#define ATTR_CFG_FLD_min_latency_HI 11
	181
	182	/* Why does everything I do descend into this? */
	183	#define __GEN_PMU_FORMAT_ATTR(cfg, lo, hi) \
	184	(lo) == (hi) ? #cfg ":" #lo "\n" : #cfg ":" #lo "-" #hi
	185
	186	#define _GEN_PMU_FORMAT_ATTR(cfg, lo, hi) \
	187	__GEN_PMU_FORMAT_ATTR(cfg, lo, hi)
	188
	189	#define GEN_PMU_FORMAT_ATTR(name) \
	190	PMU_FORMAT_ATTR(name, \
	191	_GEN_PMU_FORMAT_ATTR(ATTR_CFG_FLD_##name##_CFG, \
	192	ATTR_CFG_FLD_##name##_LO, \
	193	ATTR_CFG_FLD_##name##_HI))
	194
	195	#define _ATTR_CFG_GET_FLD(attr, cfg, lo, hi) \
	196	((((attr)->cfg) >> lo) & GENMASK(hi - lo, 0))
	197
	198	#define ATTR_CFG_GET_FLD(attr, name) \
	199	_ATTR_CFG_GET_FLD(attr, \
	200	ATTR_CFG_FLD_##name##_CFG, \
	201	ATTR_CFG_FLD_##name##_LO, \
	202	ATTR_CFG_FLD_##name##_HI)
	203
	204	GEN_PMU_FORMAT_ATTR(ts_enable);
	205	GEN_PMU_FORMAT_ATTR(pa_enable);
	206	GEN_PMU_FORMAT_ATTR(pct_enable);
	207	GEN_PMU_FORMAT_ATTR(jitter);
	208	GEN_PMU_FORMAT_ATTR(branch_filter);
	209	GEN_PMU_FORMAT_ATTR(load_filter);
	210	GEN_PMU_FORMAT_ATTR(store_filter);
	211	GEN_PMU_FORMAT_ATTR(event_filter);
	212	GEN_PMU_FORMAT_ATTR(min_latency);
	213
	214	static struct attribute *arm_spe_pmu_formats_attr[] = {
	215	&format_attr_ts_enable.attr,
	216	&format_attr_pa_enable.attr,
	217	&format_attr_pct_enable.attr,
	218	&format_attr_jitter.attr,
	219	&format_attr_branch_filter.attr,
	220	&format_attr_load_filter.attr,
	221	&format_attr_store_filter.attr,
	222	&format_attr_event_filter.attr,
	223	&format_attr_min_latency.attr,
	224	NULL,
	225	};
	226
	227	static struct attribute_group arm_spe_pmu_format_group = {
	228	.name = "format",
	229	.attrs = arm_spe_pmu_formats_attr,
	230	};
	231
	232	static ssize_t arm_spe_pmu_get_attr_cpumask(struct device *dev,
	233	struct device_attribute *attr,
	234	char *buf)
	235	{
	236	struct platform_device *pdev = to_platform_device(dev);
	237	struct arm_spe_pmu *spe_pmu = platform_get_drvdata(pdev);
	238
	239	return cpumap_print_to_pagebuf(true, buf, &spe_pmu->supported_cpus);
	240	}
	241	static DEVICE_ATTR(cpumask, S_IRUGO, arm_spe_pmu_get_attr_cpumask, NULL);
	242
	243	static struct attribute *arm_spe_pmu_attrs[] = {
	244	&dev_attr_cpumask.attr,
	245	NULL,
	246	};
	247
	248	static struct attribute_group arm_spe_pmu_group = {
	249	.attrs = arm_spe_pmu_attrs,
	250	};
	251
	252	static const struct attribute_group *arm_spe_pmu_attr_groups[] = {
	253	&arm_spe_pmu_group,
	254	&arm_spe_pmu_cap_group,
	255	&arm_spe_pmu_format_group,
	256	NULL,
	257	};
	258
	259	/* Convert between user ABI and register values */
	260	static u64 arm_spe_event_to_pmscr(struct perf_event *event)
	261	{
	262	struct perf_event_attr *attr = &event->attr;
	263	u64 reg = 0;
	264
	265	reg \|= ATTR_CFG_GET_FLD(attr, ts_enable) << SYS_PMSCR_EL1_TS_SHIFT;
	266	reg \|= ATTR_CFG_GET_FLD(attr, pa_enable) << SYS_PMSCR_EL1_PA_SHIFT;
	267	reg \|= ATTR_CFG_GET_FLD(attr, pct_enable) << SYS_PMSCR_EL1_PCT_SHIFT;
	268
	269	if (!attr->exclude_user)
	270	reg \|= BIT(SYS_PMSCR_EL1_E0SPE_SHIFT);
	271
	272	if (!attr->exclude_kernel)
	273	reg \|= BIT(SYS_PMSCR_EL1_E1SPE_SHIFT);
	274
	275	if (IS_ENABLED(CONFIG_PID_IN_CONTEXTIDR) && capable(CAP_SYS_ADMIN))
	276	reg \|= BIT(SYS_PMSCR_EL1_CX_SHIFT);
	277
	278	return reg;
	279	}
	280
	281	static void arm_spe_event_sanitise_period(struct perf_event *event)
	282	{
	283	struct arm_spe_pmu *spe_pmu = to_spe_pmu(event->pmu);
	284	u64 period = event->hw.sample_period;
	285	u64 max_period = SYS_PMSIRR_EL1_INTERVAL_MASK
	286	<< SYS_PMSIRR_EL1_INTERVAL_SHIFT;
	287
	288	if (period < spe_pmu->min_period)
	289	period = spe_pmu->min_period;
	290	else if (period > max_period)
	291	period = max_period;
	292	else
	293	period &= max_period;
	294
	295	event->hw.sample_period = period;
	296	}
	297
	298	static u64 arm_spe_event_to_pmsirr(struct perf_event *event)
	299	{
	300	struct perf_event_attr *attr = &event->attr;
	301	u64 reg = 0;
	302
	303	arm_spe_event_sanitise_period(event);
	304
	305	reg \|= ATTR_CFG_GET_FLD(attr, jitter) << SYS_PMSIRR_EL1_RND_SHIFT;
	306	reg \|= event->hw.sample_period;
	307
	308	return reg;
	309	}
	310
	311	static u64 arm_spe_event_to_pmsfcr(struct perf_event *event)
	312	{
	313	struct perf_event_attr *attr = &event->attr;
	314	u64 reg = 0;
	315
	316	reg \|= ATTR_CFG_GET_FLD(attr, load_filter) << SYS_PMSFCR_EL1_LD_SHIFT;
	317	reg \|= ATTR_CFG_GET_FLD(attr, store_filter) << SYS_PMSFCR_EL1_ST_SHIFT;
	318	reg \|= ATTR_CFG_GET_FLD(attr, branch_filter) << SYS_PMSFCR_EL1_B_SHIFT;
	319
	320	if (reg)
	321	reg \|= BIT(SYS_PMSFCR_EL1_FT_SHIFT);
	322
	323	if (ATTR_CFG_GET_FLD(attr, event_filter))
	324	reg \|= BIT(SYS_PMSFCR_EL1_FE_SHIFT);
	325
	326	if (ATTR_CFG_GET_FLD(attr, min_latency))
	327	reg \|= BIT(SYS_PMSFCR_EL1_FL_SHIFT);
	328
	329	return reg;
	330	}
	331
	332	static u64 arm_spe_event_to_pmsevfr(struct perf_event *event)
	333	{
	334	struct perf_event_attr *attr = &event->attr;
	335	return ATTR_CFG_GET_FLD(attr, event_filter);
	336	}
	337
	338	static u64 arm_spe_event_to_pmslatfr(struct perf_event *event)
	339	{
	340	struct perf_event_attr *attr = &event->attr;
	341	return ATTR_CFG_GET_FLD(attr, min_latency)
	342	<< SYS_PMSLATFR_EL1_MINLAT_SHIFT;
	343	}
	344
	345	static void arm_spe_pmu_pad_buf(struct perf_output_handle *handle, int len)
	346	{
	347	struct arm_spe_pmu_buf *buf = perf_get_aux(handle);
	348	u64 head = PERF_IDX2OFF(handle->head, buf);
	349
	350	memset(buf->base + head, ARM_SPE_BUF_PAD_BYTE, len);
	351	if (!buf->snapshot)
	352	perf_aux_output_skip(handle, len);
	353	}
	354
	355	static u64 arm_spe_pmu_next_snapshot_off(struct perf_output_handle *handle)
	356	{
	357	struct arm_spe_pmu_buf *buf = perf_get_aux(handle);
	358	struct arm_spe_pmu *spe_pmu = to_spe_pmu(handle->event->pmu);
	359	u64 head = PERF_IDX2OFF(handle->head, buf);
	360	u64 limit = buf->nr_pages * PAGE_SIZE;
	361
	362	/*
	363	* The trace format isn't parseable in reverse, so clamp
	364	* the limit to half of the buffer size in snapshot mode
	365	* so that the worst case is half a buffer of records, as
	366	* opposed to a single record.
	367	*/
	368	if (head < limit >> 1)
	369	limit >>= 1;
	370
	371	/*
	372	* If we're within max_record_sz of the limit, we must
	373	* pad, move the head index and recompute the limit.
	374	*/
	375	if (limit - head < spe_pmu->max_record_sz) {
	376	arm_spe_pmu_pad_buf(handle, limit - head);
	377	handle->head = PERF_IDX2OFF(limit, buf);
	378	limit = ((buf->nr_pages * PAGE_SIZE) >> 1) + handle->head;
	379	}
	380
	381	return limit;
	382	}
	383
	384	static u64 __arm_spe_pmu_next_off(struct perf_output_handle *handle)
	385	{
	386	struct arm_spe_pmu *spe_pmu = to_spe_pmu(handle->event->pmu);
	387	struct arm_spe_pmu_buf *buf = perf_get_aux(handle);
	388	const u64 bufsize = buf->nr_pages * PAGE_SIZE;
	389	u64 limit = bufsize;
	390	u64 head, tail, wakeup;
	391
	392	/*
	393	* The head can be misaligned for two reasons:
	394	*
	395	* 1. The hardware left PMBPTR pointing to the first byte after
	396	* a record when generating a buffer management event.
	397	*
	398	* 2. We used perf_aux_output_skip to consume handle->size bytes
	399	* and CIRC_SPACE was used to compute the size, which always
	400	* leaves one entry free.
	401	*
	402	* Deal with this by padding to the next alignment boundary and
	403	* moving the head index. If we run out of buffer space, we'll
	404	* reduce handle->size to zero and end up reporting truncation.
	405	*/
	406	head = PERF_IDX2OFF(handle->head, buf);
	407	if (!IS_ALIGNED(head, spe_pmu->align)) {
	408	unsigned long delta = roundup(head, spe_pmu->align) - head;
	409
	410	delta = min(delta, handle->size);
	411	arm_spe_pmu_pad_buf(handle, delta);
	412	head = PERF_IDX2OFF(handle->head, buf);
	413	}
	414
	415	/* If we've run out of free space, then nothing more to do */
	416	if (!handle->size)
	417	goto no_space;
	418
	419	/* Compute the tail and wakeup indices now that we've aligned head */
	420	tail = PERF_IDX2OFF(handle->head + handle->size, buf);
	421	wakeup = PERF_IDX2OFF(handle->wakeup, buf);
	422
	423	/*
	424	* Avoid clobbering unconsumed data. We know we have space, so
	425	* if we see head == tail we know that the buffer is empty. If
	426	* head > tail, then there's nothing to clobber prior to
	427	* wrapping.
	428	*/
	429	if (head < tail)
	430	limit = round_down(tail, PAGE_SIZE);
	431
	432	/*
	433	* Wakeup may be arbitrarily far into the future. If it's not in
	434	* the current generation, either we'll wrap before hitting it,
	435	* or it's in the past and has been handled already.
	436	*
	437	* If there's a wakeup before we wrap, arrange to be woken up by
	438	* the page boundary following it. Keep the tail boundary if
	439	* that's lower.
	440	*/
	441	if (handle->wakeup < (handle->head + handle->size) && head <= wakeup)
	442	limit = min(limit, round_up(wakeup, PAGE_SIZE));
	443
	444	if (limit > head)
	445	return limit;
	446
	447	arm_spe_pmu_pad_buf(handle, handle->size);
	448	no_space:
	449	perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED);
	450	perf_aux_output_end(handle, 0);
	451	return 0;
	452	}
	453
	454	static u64 arm_spe_pmu_next_off(struct perf_output_handle *handle)
	455	{
	456	struct arm_spe_pmu_buf *buf = perf_get_aux(handle);
	457	struct arm_spe_pmu *spe_pmu = to_spe_pmu(handle->event->pmu);
	458	u64 limit = __arm_spe_pmu_next_off(handle);
	459	u64 head = PERF_IDX2OFF(handle->head, buf);
	460
	461	/*
	462	* If the head has come too close to the end of the buffer,
	463	* then pad to the end and recompute the limit.
	464	*/
	465	if (limit && (limit - head < spe_pmu->max_record_sz)) {
	466	arm_spe_pmu_pad_buf(handle, limit - head);
	467	limit = __arm_spe_pmu_next_off(handle);
	468	}
	469
	470	return limit;
	471	}
	472
	473	static void arm_spe_perf_aux_output_begin(struct perf_output_handle *handle,
	474	struct perf_event *event)
	475	{
	476	u64 base, limit;
	477	struct arm_spe_pmu_buf *buf;
	478
	479	/* Start a new aux session */
	480	buf = perf_aux_output_begin(handle, event);
	481	if (!buf) {
	482	event->hw.state \|= PERF_HES_STOPPED;
	483	/*
	484	* We still need to clear the limit pointer, since the
	485	* profiler might only be disabled by virtue of a fault.
	486	*/
	487	limit = 0;
	488	goto out_write_limit;
	489	}
	490
	491	limit = buf->snapshot ? arm_spe_pmu_next_snapshot_off(handle)
	492	: arm_spe_pmu_next_off(handle);
	493	if (limit)
	494	limit \|= BIT(SYS_PMBLIMITR_EL1_E_SHIFT);
	495
	496	limit += (u64)buf->base;
	497	base = (u64)buf->base + PERF_IDX2OFF(handle->head, buf);
	498	write_sysreg_s(base, SYS_PMBPTR_EL1);
	499
	500	out_write_limit:
	501	write_sysreg_s(limit, SYS_PMBLIMITR_EL1);
	502	}
	503
	504	static void arm_spe_perf_aux_output_end(struct perf_output_handle *handle)
	505	{
	506	struct arm_spe_pmu_buf *buf = perf_get_aux(handle);
	507	u64 offset, size;
	508
	509	offset = read_sysreg_s(SYS_PMBPTR_EL1) - (u64)buf->base;
	510	size = offset - PERF_IDX2OFF(handle->head, buf);
	511
	512	if (buf->snapshot)
	513	handle->head = offset;
	514
	515	perf_aux_output_end(handle, size);
	516	}
	517
	518	static void arm_spe_pmu_disable_and_drain_local(void)
	519	{
	520	/* Disable profiling at EL0 and EL1 */
	521	write_sysreg_s(0, SYS_PMSCR_EL1);
	522	isb();
	523
	524	/* Drain any buffered data */
	525	psb_csync();
	526	dsb(nsh);
	527
	528	/* Disable the profiling buffer */
	529	write_sysreg_s(0, SYS_PMBLIMITR_EL1);
	530	isb();
	531	}
	532
	533	/* IRQ handling */
	534	static enum arm_spe_pmu_buf_fault_action
	535	arm_spe_pmu_buf_get_fault_act(struct perf_output_handle *handle)
	536	{
	537	const char *err_str;
	538	u64 pmbsr;
	539	enum arm_spe_pmu_buf_fault_action ret;
	540
	541	/*
	542	* Ensure new profiling data is visible to the CPU and any external
	543	* aborts have been resolved.
	544	*/
	545	psb_csync();
	546	dsb(nsh);
	547
	548	/* Ensure hardware updates to PMBPTR_EL1 are visible */
	549	isb();
	550
	551	/* Service required? */
	552	pmbsr = read_sysreg_s(SYS_PMBSR_EL1);
	553	if (!(pmbsr & BIT(SYS_PMBSR_EL1_S_SHIFT)))
	554	return SPE_PMU_BUF_FAULT_ACT_SPURIOUS;
	555
	556	/*
	557	* If we've lost data, disable profiling and also set the PARTIAL
	558	* flag to indicate that the last record is corrupted.
	559	*/
	560	if (pmbsr & BIT(SYS_PMBSR_EL1_DL_SHIFT))
	561	perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED \|
	562	PERF_AUX_FLAG_PARTIAL);
	563
	564	/* Report collisions to userspace so that it can up the period */
	565	if (pmbsr & BIT(SYS_PMBSR_EL1_COLL_SHIFT))
	566	perf_aux_output_flag(handle, PERF_AUX_FLAG_COLLISION);
	567
	568	/* We only expect buffer management events */
	569	switch (pmbsr & (SYS_PMBSR_EL1_EC_MASK << SYS_PMBSR_EL1_EC_SHIFT)) {
	570	case SYS_PMBSR_EL1_EC_BUF:
	571	/* Handled below */
	572	break;
	573	case SYS_PMBSR_EL1_EC_FAULT_S1:
	574	case SYS_PMBSR_EL1_EC_FAULT_S2:
	575	err_str = "Unexpected buffer fault";
	576	goto out_err;
	577	default:
	578	err_str = "Unknown error code";
	579	goto out_err;
	580	}
	581
	582	/* Buffer management event */
	583	switch (pmbsr &
	584	(SYS_PMBSR_EL1_BUF_BSC_MASK << SYS_PMBSR_EL1_BUF_BSC_SHIFT)) {
	585	case SYS_PMBSR_EL1_BUF_BSC_FULL:
	586	ret = SPE_PMU_BUF_FAULT_ACT_OK;
	587	goto out_stop;
	588	default:
	589	err_str = "Unknown buffer status code";
	590	}
	591
	592	out_err:
	593	pr_err_ratelimited("%s on CPU %d [PMBSR=0x%016llx, PMBPTR=0x%016llx, PMBLIMITR=0x%016llx]\n",
	594	err_str, smp_processor_id(), pmbsr,
	595	read_sysreg_s(SYS_PMBPTR_EL1),
	596	read_sysreg_s(SYS_PMBLIMITR_EL1));
	597	ret = SPE_PMU_BUF_FAULT_ACT_FATAL;
	598
	599	out_stop:
	600	arm_spe_perf_aux_output_end(handle);
	601	return ret;
	602	}
	603
	604	static irqreturn_t arm_spe_pmu_irq_handler(int irq, void *dev)
	605	{
	606	struct perf_output_handle *handle = dev;
	607	struct perf_event *event = handle->event;
	608	enum arm_spe_pmu_buf_fault_action act;
	609
	610	if (!perf_get_aux(handle))
	611	return IRQ_NONE;
	612
	613	act = arm_spe_pmu_buf_get_fault_act(handle);
	614	if (act == SPE_PMU_BUF_FAULT_ACT_SPURIOUS)
	615	return IRQ_NONE;
	616
	617	/*
	618	* Ensure perf callbacks have completed, which may disable the
	619	* profiling buffer in response to a TRUNCATION flag.
	620	*/
	621	irq_work_run();
	622
	623	switch (act) {
	624	case SPE_PMU_BUF_FAULT_ACT_FATAL:
	625	/*
	626	* If a fatal exception occurred then leaving the profiling
	627	* buffer enabled is a recipe waiting to happen. Since
	628	* fatal faults don't always imply truncation, make sure
	629	* that the profiling buffer is disabled explicitly before
	630	* clearing the syndrome register.
	631	*/
	632	arm_spe_pmu_disable_and_drain_local();
	633	break;
	634	case SPE_PMU_BUF_FAULT_ACT_OK:
	635	/*
	636	* We handled the fault (the buffer was full), so resume
	637	* profiling as long as we didn't detect truncation.
	638	* PMBPTR might be misaligned, but we'll burn that bridge
	639	* when we get to it.
	640	*/
	641	if (!(handle->aux_flags & PERF_AUX_FLAG_TRUNCATED)) {
	642	arm_spe_perf_aux_output_begin(handle, event);
	643	isb();
	644	}
	645	break;
	646	case SPE_PMU_BUF_FAULT_ACT_SPURIOUS:
	647	/* We've seen you before, but GCC has the memory of a sieve. */
	648	break;
	649	}
	650
	651	/* The buffer pointers are now sane, so resume profiling. */
	652	write_sysreg_s(0, SYS_PMBSR_EL1);
	653	return IRQ_HANDLED;
	654	}
	655
	656	/* Perf callbacks */
	657	static int arm_spe_pmu_event_init(struct perf_event *event)
	658	{
	659	u64 reg;
	660	struct perf_event_attr *attr = &event->attr;
	661	struct arm_spe_pmu *spe_pmu = to_spe_pmu(event->pmu);
	662
	663	/* This is, of course, deeply driver-specific */
	664	if (attr->type != event->pmu->type)
	665	return -ENOENT;
	666
	667	if (event->cpu >= 0 &&
	668	!cpumask_test_cpu(event->cpu, &spe_pmu->supported_cpus))
	669	return -ENOENT;
	670
	671	if (arm_spe_event_to_pmsevfr(event) & SYS_PMSEVFR_EL1_RES0)
	672	return -EOPNOTSUPP;
	673
	674	if (attr->exclude_idle)
	675	return -EOPNOTSUPP;
	676
	677	/*
	678	* Feedback-directed frequency throttling doesn't work when we
	679	* have a buffer of samples. We'd need to manually count the
	680	* samples in the buffer when it fills up and adjust the event
	681	* count to reflect that. Instead, just force the user to specify
	682	* a sample period.
	683	*/
	684	if (attr->freq)
	685	return -EINVAL;
	686
	687	reg = arm_spe_event_to_pmsfcr(event);
	688	if ((reg & BIT(SYS_PMSFCR_EL1_FE_SHIFT)) &&
	689	!(spe_pmu->features & SPE_PMU_FEAT_FILT_EVT))
	690	return -EOPNOTSUPP;
	691
	692	if ((reg & BIT(SYS_PMSFCR_EL1_FT_SHIFT)) &&
	693	!(spe_pmu->features & SPE_PMU_FEAT_FILT_TYP))
	694	return -EOPNOTSUPP;
	695
	696	if ((reg & BIT(SYS_PMSFCR_EL1_FL_SHIFT)) &&
	697	!(spe_pmu->features & SPE_PMU_FEAT_FILT_LAT))
	698	return -EOPNOTSUPP;
	699
	700	reg = arm_spe_event_to_pmscr(event);
	701	if (!capable(CAP_SYS_ADMIN) &&
	702	(reg & (BIT(SYS_PMSCR_EL1_PA_SHIFT) \|
	703	BIT(SYS_PMSCR_EL1_CX_SHIFT) \|
	704	BIT(SYS_PMSCR_EL1_PCT_SHIFT))))
	705	return -EACCES;
	706
	707	return 0;
	708	}
	709
	710	static void arm_spe_pmu_start(struct perf_event *event, int flags)
	711	{
	712	u64 reg;
	713	struct arm_spe_pmu *spe_pmu = to_spe_pmu(event->pmu);
	714	struct hw_perf_event *hwc = &event->hw;
	715	struct perf_output_handle *handle = this_cpu_ptr(spe_pmu->handle);
	716
	717	hwc->state = 0;
	718	arm_spe_perf_aux_output_begin(handle, event);
	719	if (hwc->state)
	720	return;
	721
	722	reg = arm_spe_event_to_pmsfcr(event);
	723	write_sysreg_s(reg, SYS_PMSFCR_EL1);
	724
	725	reg = arm_spe_event_to_pmsevfr(event);
	726	write_sysreg_s(reg, SYS_PMSEVFR_EL1);
	727
	728	reg = arm_spe_event_to_pmslatfr(event);
	729	write_sysreg_s(reg, SYS_PMSLATFR_EL1);
	730
	731	if (flags & PERF_EF_RELOAD) {
	732	reg = arm_spe_event_to_pmsirr(event);
	733	write_sysreg_s(reg, SYS_PMSIRR_EL1);
	734	isb();
	735	reg = local64_read(&hwc->period_left);
	736	write_sysreg_s(reg, SYS_PMSICR_EL1);
	737	}
	738
	739	reg = arm_spe_event_to_pmscr(event);
	740	isb();
	741	write_sysreg_s(reg, SYS_PMSCR_EL1);
	742	}
	743
	744	static void arm_spe_pmu_stop(struct perf_event *event, int flags)
	745	{
	746	struct arm_spe_pmu *spe_pmu = to_spe_pmu(event->pmu);
	747	struct hw_perf_event *hwc = &event->hw;
	748	struct perf_output_handle *handle = this_cpu_ptr(spe_pmu->handle);
	749
	750	/* If we're already stopped, then nothing to do */
	751	if (hwc->state & PERF_HES_STOPPED)
	752	return;
	753
	754	/* Stop all trace generation */
	755	arm_spe_pmu_disable_and_drain_local();
	756
	757	if (flags & PERF_EF_UPDATE) {
	758	/*
	759	* If there's a fault pending then ensure we contain it
	760	* to this buffer, since we might be on the context-switch
	761	* path.
	762	*/
	763	if (perf_get_aux(handle)) {
	764	enum arm_spe_pmu_buf_fault_action act;
	765
	766	act = arm_spe_pmu_buf_get_fault_act(handle);
	767	if (act == SPE_PMU_BUF_FAULT_ACT_SPURIOUS)
	768	arm_spe_perf_aux_output_end(handle);
	769	else
	770	write_sysreg_s(0, SYS_PMBSR_EL1);
	771	}
	772
	773	/*
	774	* This may also contain ECOUNT, but nobody else should
	775	* be looking at period_left, since we forbid frequency
	776	* based sampling.
	777	*/
	778	local64_set(&hwc->period_left, read_sysreg_s(SYS_PMSICR_EL1));
	779	hwc->state \|= PERF_HES_UPTODATE;
	780	}
	781
	782	hwc->state \|= PERF_HES_STOPPED;
	783	}
	784
	785	static int arm_spe_pmu_add(struct perf_event *event, int flags)
	786	{
	787	int ret = 0;
	788	struct arm_spe_pmu *spe_pmu = to_spe_pmu(event->pmu);
	789	struct hw_perf_event *hwc = &event->hw;
	790	int cpu = event->cpu == -1 ? smp_processor_id() : event->cpu;
	791
	792	if (!cpumask_test_cpu(cpu, &spe_pmu->supported_cpus))
	793	return -ENOENT;
	794
	795	hwc->state = PERF_HES_UPTODATE \| PERF_HES_STOPPED;
	796
	797	if (flags & PERF_EF_START) {
	798	arm_spe_pmu_start(event, PERF_EF_RELOAD);
	799	if (hwc->state & PERF_HES_STOPPED)
	800	ret = -EINVAL;
	801	}
	802
	803	return ret;
	804	}
	805
	806	static void arm_spe_pmu_del(struct perf_event *event, int flags)
	807	{
	808	arm_spe_pmu_stop(event, PERF_EF_UPDATE);
	809	}
	810
	811	static void arm_spe_pmu_read(struct perf_event *event)
	812	{
	813	}
	814
	815	static void arm_spe_pmu_setup_aux(int cpu, void *pages, int nr_pages,
	816	bool snapshot)
	817	{
	818	int i;
	819	struct page **pglist;
	820	struct arm_spe_pmu_buf *buf;
	821
	822	/* We need at least two pages for this to work. */
	823	if (nr_pages < 2)
	824	return NULL;
	825
	826	/*
	827	* We require an even number of pages for snapshot mode, so that
	828	* we can effectively treat the buffer as consisting of two equal
	829	* parts and give userspace a fighting chance of getting some
	830	* useful data out of it.
	831	*/
	832	if (!nr_pages \|\| (snapshot && (nr_pages & 1)))
	833	return NULL;
	834
	835	if (cpu == -1)
	836	cpu = raw_smp_processor_id();
	837
	838	buf = kzalloc_node(sizeof(*buf), GFP_KERNEL, cpu_to_node(cpu));
	839	if (!buf)
	840	return NULL;
	841
	842	pglist = kcalloc(nr_pages, sizeof(*pglist), GFP_KERNEL);
	843	if (!pglist)
	844	goto out_free_buf;
	845
	846	for (i = 0; i < nr_pages; ++i) {
	847	struct page *page = virt_to_page(pages[i]);
	848
	849	if (PagePrivate(page)) {
	850	pr_warn("unexpected high-order page for auxbuf!");
	851	goto out_free_pglist;
	852	}
	853
	854	pglist[i] = virt_to_page(pages[i]);
	855	}
	856
	857	buf->base = vmap(pglist, nr_pages, VM_MAP, PAGE_KERNEL);
	858	if (!buf->base)
	859	goto out_free_pglist;
	860
	861	buf->nr_pages = nr_pages;
	862	buf->snapshot = snapshot;
	863
	864	kfree(pglist);
	865	return buf;
	866
	867	out_free_pglist:
	868	kfree(pglist);
	869	out_free_buf:
	870	kfree(buf);
	871	return NULL;
	872	}
	873
	874	static void arm_spe_pmu_free_aux(void *aux)
	875	{
	876	struct arm_spe_pmu_buf *buf = aux;
	877
	878	vunmap(buf->base);
	879	kfree(buf);
	880	}
	881
	882	/* Initialisation and teardown functions */
	883	static int arm_spe_pmu_perf_init(struct arm_spe_pmu *spe_pmu)
	884	{
	885	static atomic_t pmu_idx = ATOMIC_INIT(-1);
	886
	887	int idx;
	888	char *name;
	889	struct device *dev = &spe_pmu->pdev->dev;
	890
	891	spe_pmu->pmu = (struct pmu) {
	892	.module = THIS_MODULE,
	893	.capabilities = PERF_PMU_CAP_EXCLUSIVE \| PERF_PMU_CAP_ITRACE,
	894	.attr_groups = arm_spe_pmu_attr_groups,
	895	/*
	896	* We hitch a ride on the software context here, so that
	897	* we can support per-task profiling (which is not possible
	898	* with the invalid context as it doesn't get sched callbacks).
	899	* This requires that userspace either uses a dummy event for
	900	* perf_event_open, since the aux buffer is not setup until
	901	* a subsequent mmap, or creates the profiling event in a
	902	* disabled state and explicitly PERF_EVENT_IOC_ENABLEs it
	903	* once the buffer has been created.
	904	*/
	905	.task_ctx_nr = perf_sw_context,
	906	.event_init = arm_spe_pmu_event_init,
	907	.add = arm_spe_pmu_add,
	908	.del = arm_spe_pmu_del,
	909	.start = arm_spe_pmu_start,
	910	.stop = arm_spe_pmu_stop,
	911	.read = arm_spe_pmu_read,
	912	.setup_aux = arm_spe_pmu_setup_aux,
	913	.free_aux = arm_spe_pmu_free_aux,
	914	};
	915
	916	idx = atomic_inc_return(&pmu_idx);
	917	name = devm_kasprintf(dev, GFP_KERNEL, "%s_%d", PMUNAME, idx);
	918	return perf_pmu_register(&spe_pmu->pmu, name, -1);
	919	}
	920
	921	static void arm_spe_pmu_perf_destroy(struct arm_spe_pmu *spe_pmu)
	922	{
	923	perf_pmu_unregister(&spe_pmu->pmu);
	924	}
	925
	926	static void __arm_spe_pmu_dev_probe(void *info)
	927	{
	928	int fld;
	929	u64 reg;
	930	struct arm_spe_pmu *spe_pmu = info;
	931	struct device *dev = &spe_pmu->pdev->dev;
	932
	933	fld = cpuid_feature_extract_unsigned_field(read_cpuid(ID_AA64DFR0_EL1),
	934	ID_AA64DFR0_PMSVER_SHIFT);
	935	if (!fld) {
	936	dev_err(dev,
	937	"unsupported ID_AA64DFR0_EL1.PMSVer [%d] on CPU %d\n",
	938	fld, smp_processor_id());
	939	return;
	940	}
	941
	942	/* Read PMBIDR first to determine whether or not we have access */
	943	reg = read_sysreg_s(SYS_PMBIDR_EL1);
	944	if (reg & BIT(SYS_PMBIDR_EL1_P_SHIFT)) {
	945	dev_err(dev,
	946	"profiling buffer owned by higher exception level\n");
	947	return;
	948	}
	949
	950	/* Minimum alignment. If it's out-of-range, then fail the probe */
	951	fld = reg >> SYS_PMBIDR_EL1_ALIGN_SHIFT & SYS_PMBIDR_EL1_ALIGN_MASK;
	952	spe_pmu->align = 1 << fld;
	953	if (spe_pmu->align > SZ_2K) {
	954	dev_err(dev, "unsupported PMBIDR.Align [%d] on CPU %d\n",
	955	fld, smp_processor_id());
	956	return;
	957	}
	958
	959	/* It's now safe to read PMSIDR and figure out what we've got */
	960	reg = read_sysreg_s(SYS_PMSIDR_EL1);
	961	if (reg & BIT(SYS_PMSIDR_EL1_FE_SHIFT))
	962	spe_pmu->features \|= SPE_PMU_FEAT_FILT_EVT;
	963
	964	if (reg & BIT(SYS_PMSIDR_EL1_FT_SHIFT))
	965	spe_pmu->features \|= SPE_PMU_FEAT_FILT_TYP;
	966
	967	if (reg & BIT(SYS_PMSIDR_EL1_FL_SHIFT))
	968	spe_pmu->features \|= SPE_PMU_FEAT_FILT_LAT;
	969
	970	if (reg & BIT(SYS_PMSIDR_EL1_ARCHINST_SHIFT))
	971	spe_pmu->features \|= SPE_PMU_FEAT_ARCH_INST;
	972
	973	if (reg & BIT(SYS_PMSIDR_EL1_LDS_SHIFT))
	974	spe_pmu->features \|= SPE_PMU_FEAT_LDS;
	975
	976	if (reg & BIT(SYS_PMSIDR_EL1_ERND_SHIFT))
	977	spe_pmu->features \|= SPE_PMU_FEAT_ERND;
	978
	979	/* This field has a spaced out encoding, so just use a look-up */
	980	fld = reg >> SYS_PMSIDR_EL1_INTERVAL_SHIFT & SYS_PMSIDR_EL1_INTERVAL_MASK;
	981	switch (fld) {
	982	case 0:
	983	spe_pmu->min_period = 256;
	984	break;
	985	case 2:
	986	spe_pmu->min_period = 512;
	987	break;
	988	case 3:
	989	spe_pmu->min_period = 768;
	990	break;
	991	case 4:
	992	spe_pmu->min_period = 1024;
	993	break;
	994	case 5:
	995	spe_pmu->min_period = 1536;
	996	break;
	997	case 6:
	998	spe_pmu->min_period = 2048;
	999	break;
	1000	case 7:
	1001	spe_pmu->min_period = 3072;
	1002	break;
	1003	default:
	1004	dev_warn(dev, "unknown PMSIDR_EL1.Interval [%d]; assuming 8\n",
	1005	fld);
	1006	/* Fallthrough */
	1007	case 8:
	1008	spe_pmu->min_period = 4096;
	1009	}
	1010
	1011	/* Maximum record size. If it's out-of-range, then fail the probe */
	1012	fld = reg >> SYS_PMSIDR_EL1_MAXSIZE_SHIFT & SYS_PMSIDR_EL1_MAXSIZE_MASK;
	1013	spe_pmu->max_record_sz = 1 << fld;
	1014	if (spe_pmu->max_record_sz > SZ_2K \|\| spe_pmu->max_record_sz < 16) {
	1015	dev_err(dev, "unsupported PMSIDR_EL1.MaxSize [%d] on CPU %d\n",
	1016	fld, smp_processor_id());
	1017	return;
	1018	}
	1019
	1020	fld = reg >> SYS_PMSIDR_EL1_COUNTSIZE_SHIFT & SYS_PMSIDR_EL1_COUNTSIZE_MASK;
	1021	switch (fld) {
	1022	default:
	1023	dev_warn(dev, "unknown PMSIDR_EL1.CountSize [%d]; assuming 2\n",
	1024	fld);
	1025	/* Fallthrough */
	1026	case 2:
	1027	spe_pmu->counter_sz = 12;
	1028	}
	1029
	1030	dev_info(dev,
	1031	"probed for CPUs %*pbl [max_record_sz %u, align %u, features 0x%llx]\n",
	1032	cpumask_pr_args(&spe_pmu->supported_cpus),
	1033	spe_pmu->max_record_sz, spe_pmu->align, spe_pmu->features);
	1034
	1035	spe_pmu->features \|= SPE_PMU_FEAT_DEV_PROBED;
	1036	return;
	1037	}
	1038
	1039	static void __arm_spe_pmu_reset_local(void)
	1040	{
	1041	/*
	1042	* This is probably overkill, as we have no idea where we're
	1043	* draining any buffered data to...
	1044	*/
	1045	arm_spe_pmu_disable_and_drain_local();
	1046
	1047	/* Reset the buffer base pointer */
	1048	write_sysreg_s(0, SYS_PMBPTR_EL1);
	1049	isb();
	1050
	1051	/* Clear any pending management interrupts */
	1052	write_sysreg_s(0, SYS_PMBSR_EL1);
	1053	isb();
	1054	}
	1055
	1056	static void __arm_spe_pmu_setup_one(void *info)
	1057	{
	1058	struct arm_spe_pmu *spe_pmu = info;
	1059
	1060	__arm_spe_pmu_reset_local();
	1061	enable_percpu_irq(spe_pmu->irq, IRQ_TYPE_NONE);
	1062	}
	1063
	1064	static void __arm_spe_pmu_stop_one(void *info)
	1065	{
	1066	struct arm_spe_pmu *spe_pmu = info;
	1067
	1068	disable_percpu_irq(spe_pmu->irq);
	1069	__arm_spe_pmu_reset_local();
	1070	}
	1071
	1072	static int arm_spe_pmu_cpu_startup(unsigned int cpu, struct hlist_node *node)
	1073	{
	1074	struct arm_spe_pmu *spe_pmu;
	1075
	1076	spe_pmu = hlist_entry_safe(node, struct arm_spe_pmu, hotplug_node);
	1077	if (!cpumask_test_cpu(cpu, &spe_pmu->supported_cpus))
	1078	return 0;
	1079
	1080	__arm_spe_pmu_setup_one(spe_pmu);
	1081	return 0;
	1082	}
	1083
	1084	static int arm_spe_pmu_cpu_teardown(unsigned int cpu, struct hlist_node *node)
	1085	{
	1086	struct arm_spe_pmu *spe_pmu;
	1087
	1088	spe_pmu = hlist_entry_safe(node, struct arm_spe_pmu, hotplug_node);
	1089	if (!cpumask_test_cpu(cpu, &spe_pmu->supported_cpus))
	1090	return 0;
	1091
	1092	__arm_spe_pmu_stop_one(spe_pmu);
	1093	return 0;
	1094	}
	1095
	1096	static int arm_spe_pmu_dev_init(struct arm_spe_pmu *spe_pmu)
	1097	{
	1098	int ret;
	1099	cpumask_t *mask = &spe_pmu->supported_cpus;
	1100
	1101	/* Make sure we probe the hardware on a relevant CPU */
	1102	ret = smp_call_function_any(mask, __arm_spe_pmu_dev_probe, spe_pmu, 1);
	1103	if (ret \|\| !(spe_pmu->features & SPE_PMU_FEAT_DEV_PROBED))
	1104	return -ENXIO;
	1105
	1106	/* Request our PPIs (note that the IRQ is still disabled) */
	1107	ret = request_percpu_irq(spe_pmu->irq, arm_spe_pmu_irq_handler, DRVNAME,
	1108	spe_pmu->handle);
	1109	if (ret)
	1110	return ret;
	1111
	1112	/*
	1113	* Register our hotplug notifier now so we don't miss any events.
	1114	* This will enable the IRQ for any supported CPUs that are already
	1115	* up.
	1116	*/
	1117	ret = cpuhp_state_add_instance(arm_spe_pmu_online,
	1118	&spe_pmu->hotplug_node);
	1119	if (ret)
	1120	free_percpu_irq(spe_pmu->irq, spe_pmu->handle);
	1121
	1122	return ret;
	1123	}
	1124
	1125	static void arm_spe_pmu_dev_teardown(struct arm_spe_pmu *spe_pmu)
	1126	{
	1127	cpuhp_state_remove_instance(arm_spe_pmu_online, &spe_pmu->hotplug_node);
	1128	free_percpu_irq(spe_pmu->irq, spe_pmu->handle);
	1129	}
	1130
	1131	/* Driver and device probing */
	1132	static int arm_spe_pmu_irq_probe(struct arm_spe_pmu *spe_pmu)
	1133	{
	1134	struct platform_device *pdev = spe_pmu->pdev;
	1135	int irq = platform_get_irq(pdev, 0);
	1136
	1137	if (irq < 0) {
	1138	dev_err(&pdev->dev, "failed to get IRQ (%d)\n", irq);
	1139	return -ENXIO;
	1140	}
	1141
	1142	if (!irq_is_percpu(irq)) {
	1143	dev_err(&pdev->dev, "expected PPI but got SPI (%d)\n", irq);
	1144	return -EINVAL;
	1145	}
	1146
	1147	if (irq_get_percpu_devid_partition(irq, &spe_pmu->supported_cpus)) {
	1148	dev_err(&pdev->dev, "failed to get PPI partition (%d)\n", irq);
	1149	return -EINVAL;
	1150	}
	1151
	1152	spe_pmu->irq = irq;
	1153	return 0;
	1154	}
	1155
	1156	static const struct of_device_id arm_spe_pmu_of_match[] = {
	1157	{ .compatible = "arm,statistical-profiling-extension-v1", .data = (void *)1 },
	1158	{ /* Sentinel */ },
	1159	};
	1160
	1161	static int arm_spe_pmu_device_dt_probe(struct platform_device *pdev)
	1162	{
	1163	int ret;
	1164	struct arm_spe_pmu *spe_pmu;
	1165	struct device *dev = &pdev->dev;
	1166
	1167	spe_pmu = devm_kzalloc(dev, sizeof(*spe_pmu), GFP_KERNEL);
	1168	if (!spe_pmu) {
	1169	dev_err(dev, "failed to allocate spe_pmu\n");
	1170	return -ENOMEM;
	1171	}
	1172
	1173	spe_pmu->handle = alloc_percpu(typeof(*spe_pmu->handle));
	1174	if (!spe_pmu->handle)
	1175	return -ENOMEM;
	1176
	1177	spe_pmu->pdev = pdev;
	1178	platform_set_drvdata(pdev, spe_pmu);
	1179
	1180	ret = arm_spe_pmu_irq_probe(spe_pmu);
	1181	if (ret)
	1182	goto out_free_handle;
	1183
	1184	ret = arm_spe_pmu_dev_init(spe_pmu);
	1185	if (ret)
	1186	goto out_free_handle;
	1187
	1188	ret = arm_spe_pmu_perf_init(spe_pmu);
	1189	if (ret)
	1190	goto out_teardown_dev;
	1191
	1192	return 0;
	1193
	1194	out_teardown_dev:
	1195	arm_spe_pmu_dev_teardown(spe_pmu);
	1196	out_free_handle:
	1197	free_percpu(spe_pmu->handle);
	1198	return ret;
	1199	}
	1200
	1201	static int arm_spe_pmu_device_remove(struct platform_device *pdev)
	1202	{
	1203	struct arm_spe_pmu *spe_pmu = platform_get_drvdata(pdev);
	1204
	1205	arm_spe_pmu_perf_destroy(spe_pmu);
	1206	arm_spe_pmu_dev_teardown(spe_pmu);
	1207	free_percpu(spe_pmu->handle);
	1208	return 0;
	1209	}
	1210
	1211	static struct platform_driver arm_spe_pmu_driver = {
	1212	.driver = {
	1213	.name = DRVNAME,
	1214	.of_match_table = of_match_ptr(arm_spe_pmu_of_match),
	1215	},
	1216	.probe = arm_spe_pmu_device_dt_probe,
	1217	.remove = arm_spe_pmu_device_remove,
	1218	};
	1219
	1220	static int __init arm_spe_pmu_init(void)
	1221	{
	1222	int ret;
	1223
	1224	ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, DRVNAME,
	1225	arm_spe_pmu_cpu_startup,
	1226	arm_spe_pmu_cpu_teardown);
	1227	if (ret < 0)
	1228	return ret;
	1229	arm_spe_pmu_online = ret;
	1230
	1231	ret = platform_driver_register(&arm_spe_pmu_driver);
	1232	if (ret)
	1233	cpuhp_remove_multi_state(arm_spe_pmu_online);
	1234
	1235	return ret;
	1236	}
	1237
	1238	static void __exit arm_spe_pmu_exit(void)
	1239	{
	1240	platform_driver_unregister(&arm_spe_pmu_driver);
	1241	cpuhp_remove_multi_state(arm_spe_pmu_online);
	1242	}
	1243
	1244	module_init(arm_spe_pmu_init);
	1245	module_exit(arm_spe_pmu_exit);
	1246
	1247	MODULE_DESCRIPTION("Perf driver for the ARMv8.2 Statistical Profiling Extension");
	1248	MODULE_AUTHOR("Will Deacon <will.deacon@arm.com>");
	1249	MODULE_LICENSE("GPL v2");

drivers/perf/hisilicon/Makefile+1−0

@@ -0,0 +1 @@
		1	obj-$(CONFIG_HISI_PMU) += hisi_uncore_pmu.o hisi_uncore_l3c_pmu.o hisi_uncore_hha_pmu.o hisi_uncore_ddrc_pmu.o

drivers/perf/hisilicon/hisi_uncore_ddrc_pmu.c+463−0

@@ -0,0 +1,463 @@
	1	/*
	2	* HiSilicon SoC DDRC uncore Hardware event counters support
	3	*
	4	* Copyright (C) 2017 Hisilicon Limited
	5	* Author: Shaokun Zhang <zhangshaokun@hisilicon.com>
	6	* Anurup M <anurup.m@huawei.com>
	7	*
	8	* This code is based on the uncore PMUs like arm-cci and arm-ccn.
	9	*
	10	* This program is free software; you can redistribute it and/or modify
	11	* it under the terms of the GNU General Public License version 2 as
	12	* published by the Free Software Foundation.
	13	*/
	14	#include <linux/acpi.h>
	15	#include <linux/bug.h>
	16	#include <linux/cpuhotplug.h>
	17	#include <linux/interrupt.h>
	18	#include <linux/irq.h>
	19	#include <linux/list.h>
	20	#include <linux/platform_device.h>
	21	#include <linux/smp.h>
	22
	23	#include "hisi_uncore_pmu.h"
	24
	25	/* DDRC register definition */
	26	#define DDRC_PERF_CTRL 0x010
	27	#define DDRC_FLUX_WR 0x380
	28	#define DDRC_FLUX_RD 0x384
	29	#define DDRC_FLUX_WCMD 0x388
	30	#define DDRC_FLUX_RCMD 0x38c
	31	#define DDRC_PRE_CMD 0x3c0
	32	#define DDRC_ACT_CMD 0x3c4
	33	#define DDRC_BNK_CHG 0x3c8
	34	#define DDRC_RNK_CHG 0x3cc
	35	#define DDRC_EVENT_CTRL 0x6C0
	36	#define DDRC_INT_MASK 0x6c8
	37	#define DDRC_INT_STATUS 0x6cc
	38	#define DDRC_INT_CLEAR 0x6d0
	39
	40	/* DDRC has 8-counters */
	41	#define DDRC_NR_COUNTERS 0x8
	42	#define DDRC_PERF_CTRL_EN 0x2
	43
	44	/*
	45	* For DDRC PMU, there are eight-events and every event has been mapped
	46	* to fixed-purpose counters which register offset is not consistent.
	47	* Therefore there is no write event type and we assume that event
	48	* code (0 to 7) is equal to counter index in PMU driver.
	49	*/
	50	#define GET_DDRC_EVENTID(hwc) (hwc->config_base & 0x7)
	51
	52	static const u32 ddrc_reg_off[] = {
	53	DDRC_FLUX_WR, DDRC_FLUX_RD, DDRC_FLUX_WCMD, DDRC_FLUX_RCMD,
	54	DDRC_PRE_CMD, DDRC_ACT_CMD, DDRC_BNK_CHG, DDRC_RNK_CHG
	55	};
	56
	57	/*
	58	* Select the counter register offset using the counter index.
	59	* In DDRC there are no programmable counter, the count
	60	* is readed form the statistics counter register itself.
	61	*/
	62	static u32 hisi_ddrc_pmu_get_counter_offset(int cntr_idx)
	63	{
	64	return ddrc_reg_off[cntr_idx];
	65	}
	66
	67	static u64 hisi_ddrc_pmu_read_counter(struct hisi_pmu *ddrc_pmu,
	68	struct hw_perf_event *hwc)
	69	{
	70	/* Use event code as counter index */
	71	u32 idx = GET_DDRC_EVENTID(hwc);
	72
	73	if (!hisi_uncore_pmu_counter_valid(ddrc_pmu, idx)) {
	74	dev_err(ddrc_pmu->dev, "Unsupported event index:%d!\n", idx);
	75	return 0;
	76	}
	77
	78	return readl(ddrc_pmu->base + hisi_ddrc_pmu_get_counter_offset(idx));
	79	}
	80
	81	static void hisi_ddrc_pmu_write_counter(struct hisi_pmu *ddrc_pmu,
	82	struct hw_perf_event *hwc, u64 val)
	83	{
	84	u32 idx = GET_DDRC_EVENTID(hwc);
	85
	86	if (!hisi_uncore_pmu_counter_valid(ddrc_pmu, idx)) {
	87	dev_err(ddrc_pmu->dev, "Unsupported event index:%d!\n", idx);
	88	return;
	89	}
	90
	91	writel((u32)val,
	92	ddrc_pmu->base + hisi_ddrc_pmu_get_counter_offset(idx));
	93	}
	94
	95	/*
	96	* For DDRC PMU, event has been mapped to fixed-purpose counter by hardware,
	97	* so there is no need to write event type.
	98	*/
	99	static void hisi_ddrc_pmu_write_evtype(struct hisi_pmu *hha_pmu, int idx,
	100	u32 type)
	101	{
	102	}
	103
	104	static void hisi_ddrc_pmu_start_counters(struct hisi_pmu *ddrc_pmu)
	105	{
	106	u32 val;
	107
	108	/* Set perf_enable in DDRC_PERF_CTRL to start event counting */
	109	val = readl(ddrc_pmu->base + DDRC_PERF_CTRL);
	110	val \|= DDRC_PERF_CTRL_EN;
	111	writel(val, ddrc_pmu->base + DDRC_PERF_CTRL);
	112	}
	113
	114	static void hisi_ddrc_pmu_stop_counters(struct hisi_pmu *ddrc_pmu)
	115	{
	116	u32 val;
	117
	118	/* Clear perf_enable in DDRC_PERF_CTRL to stop event counting */
	119	val = readl(ddrc_pmu->base + DDRC_PERF_CTRL);
	120	val &= ~DDRC_PERF_CTRL_EN;
	121	writel(val, ddrc_pmu->base + DDRC_PERF_CTRL);
	122	}
	123
	124	static void hisi_ddrc_pmu_enable_counter(struct hisi_pmu *ddrc_pmu,
	125	struct hw_perf_event *hwc)
	126	{
	127	u32 val;
	128
	129	/* Set counter index(event code) in DDRC_EVENT_CTRL register */
	130	val = readl(ddrc_pmu->base + DDRC_EVENT_CTRL);
	131	val \|= (1 << GET_DDRC_EVENTID(hwc));
	132	writel(val, ddrc_pmu->base + DDRC_EVENT_CTRL);
	133	}
	134
	135	static void hisi_ddrc_pmu_disable_counter(struct hisi_pmu *ddrc_pmu,
	136	struct hw_perf_event *hwc)
	137	{
	138	u32 val;
	139
	140	/* Clear counter index(event code) in DDRC_EVENT_CTRL register */
	141	val = readl(ddrc_pmu->base + DDRC_EVENT_CTRL);
	142	val &= ~(1 << GET_DDRC_EVENTID(hwc));
	143	writel(val, ddrc_pmu->base + DDRC_EVENT_CTRL);
	144	}
	145
	146	static int hisi_ddrc_pmu_get_event_idx(struct perf_event *event)
	147	{
	148	struct hisi_pmu *ddrc_pmu = to_hisi_pmu(event->pmu);
	149	unsigned long *used_mask = ddrc_pmu->pmu_events.used_mask;
	150	struct hw_perf_event *hwc = &event->hw;
	151	/* For DDRC PMU, we use event code as counter index */
	152	int idx = GET_DDRC_EVENTID(hwc);
	153
	154	if (test_bit(idx, used_mask))
	155	return -EAGAIN;
	156
	157	set_bit(idx, used_mask);
	158
	159	return idx;
	160	}
	161
	162	static void hisi_ddrc_pmu_enable_counter_int(struct hisi_pmu *ddrc_pmu,
	163	struct hw_perf_event *hwc)
	164	{
	165	u32 val;
	166
	167	/* Write 0 to enable interrupt */
	168	val = readl(ddrc_pmu->base + DDRC_INT_MASK);
	169	val &= ~(1 << GET_DDRC_EVENTID(hwc));
	170	writel(val, ddrc_pmu->base + DDRC_INT_MASK);
	171	}
	172
	173	static void hisi_ddrc_pmu_disable_counter_int(struct hisi_pmu *ddrc_pmu,
	174	struct hw_perf_event *hwc)
	175	{
	176	u32 val;
	177
	178	/* Write 1 to mask interrupt */
	179	val = readl(ddrc_pmu->base + DDRC_INT_MASK);
	180	val \|= (1 << GET_DDRC_EVENTID(hwc));
	181	writel(val, ddrc_pmu->base + DDRC_INT_MASK);
	182	}
	183
	184	static irqreturn_t hisi_ddrc_pmu_isr(int irq, void *dev_id)
	185	{
	186	struct hisi_pmu *ddrc_pmu = dev_id;
	187	struct perf_event *event;
	188	unsigned long overflown;
	189	int idx;
	190
	191	/* Read the DDRC_INT_STATUS register */
	192	overflown = readl(ddrc_pmu->base + DDRC_INT_STATUS);
	193	if (!overflown)
	194	return IRQ_NONE;
	195
	196	/*
	197	* Find the counter index which overflowed if the bit was set
	198	* and handle it
	199	*/
	200	for_each_set_bit(idx, &overflown, DDRC_NR_COUNTERS) {
	201	/* Write 1 to clear the IRQ status flag */
	202	writel((1 << idx), ddrc_pmu->base + DDRC_INT_CLEAR);
	203
	204	/* Get the corresponding event struct */
	205	event = ddrc_pmu->pmu_events.hw_events[idx];
	206	if (!event)
	207	continue;
	208
	209	hisi_uncore_pmu_event_update(event);
	210	hisi_uncore_pmu_set_event_period(event);
	211	}
	212
	213	return IRQ_HANDLED;
	214	}
	215
	216	static int hisi_ddrc_pmu_init_irq(struct hisi_pmu *ddrc_pmu,
	217	struct platform_device *pdev)
	218	{
	219	int irq, ret;
	220
	221	/* Read and init IRQ */
	222	irq = platform_get_irq(pdev, 0);
	223	if (irq < 0) {
	224	dev_err(&pdev->dev, "DDRC PMU get irq fail; irq:%d\n", irq);
	225	return irq;
	226	}
	227
	228	ret = devm_request_irq(&pdev->dev, irq, hisi_ddrc_pmu_isr,
	229	IRQF_NOBALANCING \| IRQF_NO_THREAD,
	230	dev_name(&pdev->dev), ddrc_pmu);
	231	if (ret < 0) {
	232	dev_err(&pdev->dev,
	233	"Fail to request IRQ:%d ret:%d\n", irq, ret);
	234	return ret;
	235	}
	236
	237	ddrc_pmu->irq = irq;
	238
	239	return 0;
	240	}
	241
	242	static const struct acpi_device_id hisi_ddrc_pmu_acpi_match[] = {
	243	{ "HISI0233", },
	244	{},
	245	};
	246	MODULE_DEVICE_TABLE(acpi, hisi_ddrc_pmu_acpi_match);
	247
	248	static int hisi_ddrc_pmu_init_data(struct platform_device *pdev,
	249	struct hisi_pmu *ddrc_pmu)
	250	{
	251	struct resource *res;
	252
	253	/*
	254	* Use the SCCL_ID and DDRC channel ID to identify the
	255	* DDRC PMU, while SCCL_ID is in MPIDR[aff2].
	256	*/
	257	if (device_property_read_u32(&pdev->dev, "hisilicon,ch-id",
	258	&ddrc_pmu->index_id)) {
	259	dev_err(&pdev->dev, "Can not read ddrc channel-id!\n");
	260	return -EINVAL;
	261	}
	262
	263	if (device_property_read_u32(&pdev->dev, "hisilicon,scl-id",
	264	&ddrc_pmu->sccl_id)) {
	265	dev_err(&pdev->dev, "Can not read ddrc sccl-id!\n");
	266	return -EINVAL;
	267	}
	268	/* DDRC PMUs only share the same SCCL */
	269	ddrc_pmu->ccl_id = -1;
	270
	271	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
	272	ddrc_pmu->base = devm_ioremap_resource(&pdev->dev, res);
	273	if (IS_ERR(ddrc_pmu->base)) {
	274	dev_err(&pdev->dev, "ioremap failed for ddrc_pmu resource\n");
	275	return PTR_ERR(ddrc_pmu->base);
	276	}
	277
	278	return 0;
	279	}
	280
	281	static struct attribute *hisi_ddrc_pmu_format_attr[] = {
	282	HISI_PMU_FORMAT_ATTR(event, "config:0-4"),
	283	NULL,
	284	};
	285
	286	static const struct attribute_group hisi_ddrc_pmu_format_group = {
	287	.name = "format",
	288	.attrs = hisi_ddrc_pmu_format_attr,
	289	};
	290
	291	static struct attribute *hisi_ddrc_pmu_events_attr[] = {
	292	HISI_PMU_EVENT_ATTR(flux_wr, 0x00),
	293	HISI_PMU_EVENT_ATTR(flux_rd, 0x01),
	294	HISI_PMU_EVENT_ATTR(flux_wcmd, 0x02),
	295	HISI_PMU_EVENT_ATTR(flux_rcmd, 0x03),
	296	HISI_PMU_EVENT_ATTR(pre_cmd, 0x04),
	297	HISI_PMU_EVENT_ATTR(act_cmd, 0x05),
	298	HISI_PMU_EVENT_ATTR(rnk_chg, 0x06),
	299	HISI_PMU_EVENT_ATTR(rw_chg, 0x07),
	300	NULL,
	301	};
	302
	303	static const struct attribute_group hisi_ddrc_pmu_events_group = {
	304	.name = "events",
	305	.attrs = hisi_ddrc_pmu_events_attr,
	306	};
	307
	308	static DEVICE_ATTR(cpumask, 0444, hisi_cpumask_sysfs_show, NULL);
	309
	310	static struct attribute *hisi_ddrc_pmu_cpumask_attrs[] = {
	311	&dev_attr_cpumask.attr,
	312	NULL,
	313	};
	314
	315	static const struct attribute_group hisi_ddrc_pmu_cpumask_attr_group = {
	316	.attrs = hisi_ddrc_pmu_cpumask_attrs,
	317	};
	318
	319	static const struct attribute_group *hisi_ddrc_pmu_attr_groups[] = {
	320	&hisi_ddrc_pmu_format_group,
	321	&hisi_ddrc_pmu_events_group,
	322	&hisi_ddrc_pmu_cpumask_attr_group,
	323	NULL,
	324	};
	325
	326	static const struct hisi_uncore_ops hisi_uncore_ddrc_ops = {
	327	.write_evtype = hisi_ddrc_pmu_write_evtype,
	328	.get_event_idx = hisi_ddrc_pmu_get_event_idx,
	329	.start_counters = hisi_ddrc_pmu_start_counters,
	330	.stop_counters = hisi_ddrc_pmu_stop_counters,
	331	.enable_counter = hisi_ddrc_pmu_enable_counter,
	332	.disable_counter = hisi_ddrc_pmu_disable_counter,
	333	.enable_counter_int = hisi_ddrc_pmu_enable_counter_int,
	334	.disable_counter_int = hisi_ddrc_pmu_disable_counter_int,
	335	.write_counter = hisi_ddrc_pmu_write_counter,
	336	.read_counter = hisi_ddrc_pmu_read_counter,
	337	};
	338
	339	static int hisi_ddrc_pmu_dev_probe(struct platform_device *pdev,
	340	struct hisi_pmu *ddrc_pmu)
	341	{
	342	int ret;
	343
	344	ret = hisi_ddrc_pmu_init_data(pdev, ddrc_pmu);
	345	if (ret)
	346	return ret;
	347
	348	ret = hisi_ddrc_pmu_init_irq(ddrc_pmu, pdev);
	349	if (ret)
	350	return ret;
	351
	352	ddrc_pmu->num_counters = DDRC_NR_COUNTERS;
	353	ddrc_pmu->counter_bits = 32;
	354	ddrc_pmu->ops = &hisi_uncore_ddrc_ops;
	355	ddrc_pmu->dev = &pdev->dev;
	356	ddrc_pmu->on_cpu = -1;
	357	ddrc_pmu->check_event = 7;
	358
	359	return 0;
	360	}
	361
	362	static int hisi_ddrc_pmu_probe(struct platform_device *pdev)
	363	{
	364	struct hisi_pmu *ddrc_pmu;
	365	char *name;
	366	int ret;
	367
	368	ddrc_pmu = devm_kzalloc(&pdev->dev, sizeof(*ddrc_pmu), GFP_KERNEL);
	369	if (!ddrc_pmu)
	370	return -ENOMEM;
	371
	372	platform_set_drvdata(pdev, ddrc_pmu);
	373
	374	ret = hisi_ddrc_pmu_dev_probe(pdev, ddrc_pmu);
	375	if (ret)
	376	return ret;
	377
	378	ret = cpuhp_state_add_instance(CPUHP_AP_PERF_ARM_HISI_DDRC_ONLINE,
	379	&ddrc_pmu->node);
	380	if (ret) {
	381	dev_err(&pdev->dev, "Error %d registering hotplug;\n", ret);
	382	return ret;
	383	}
	384
	385	name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "hisi_sccl%u_ddrc%u",
	386	ddrc_pmu->sccl_id, ddrc_pmu->index_id);
	387	ddrc_pmu->pmu = (struct pmu) {
	388	.name = name,
	389	.task_ctx_nr = perf_invalid_context,
	390	.event_init = hisi_uncore_pmu_event_init,
	391	.pmu_enable = hisi_uncore_pmu_enable,
	392	.pmu_disable = hisi_uncore_pmu_disable,
	393	.add = hisi_uncore_pmu_add,
	394	.del = hisi_uncore_pmu_del,
	395	.start = hisi_uncore_pmu_start,
	396	.stop = hisi_uncore_pmu_stop,
	397	.read = hisi_uncore_pmu_read,
	398	.attr_groups = hisi_ddrc_pmu_attr_groups,
	399	};
	400
	401	ret = perf_pmu_register(&ddrc_pmu->pmu, name, -1);
	402	if (ret) {
	403	dev_err(ddrc_pmu->dev, "DDRC PMU register failed!\n");
	404	cpuhp_state_remove_instance(CPUHP_AP_PERF_ARM_HISI_DDRC_ONLINE,
	405	&ddrc_pmu->node);
	406	}
	407
	408	return ret;
	409	}
	410
	411	static int hisi_ddrc_pmu_remove(struct platform_device *pdev)
	412	{
	413	struct hisi_pmu *ddrc_pmu = platform_get_drvdata(pdev);
	414
	415	perf_pmu_unregister(&ddrc_pmu->pmu);
	416	cpuhp_state_remove_instance(CPUHP_AP_PERF_ARM_HISI_DDRC_ONLINE,
	417	&ddrc_pmu->node);
	418
	419	return 0;
	420	}
	421
	422	static struct platform_driver hisi_ddrc_pmu_driver = {
	423	.driver = {
	424	.name = "hisi_ddrc_pmu",
	425	.acpi_match_table = ACPI_PTR(hisi_ddrc_pmu_acpi_match),
	426	},
	427	.probe = hisi_ddrc_pmu_probe,
	428	.remove = hisi_ddrc_pmu_remove,
	429	};
	430
	431	static int __init hisi_ddrc_pmu_module_init(void)
	432	{
	433	int ret;
	434
	435	ret = cpuhp_setup_state_multi(CPUHP_AP_PERF_ARM_HISI_DDRC_ONLINE,
	436	"AP_PERF_ARM_HISI_DDRC_ONLINE",
	437	hisi_uncore_pmu_online_cpu,
	438	hisi_uncore_pmu_offline_cpu);
	439	if (ret) {
	440	pr_err("DDRC PMU: setup hotplug, ret = %d\n", ret);
	441	return ret;
	442	}
	443
	444	ret = platform_driver_register(&hisi_ddrc_pmu_driver);
	445	if (ret)
	446	cpuhp_remove_multi_state(CPUHP_AP_PERF_ARM_HISI_DDRC_ONLINE);
	447
	448	return ret;
	449	}
	450	module_init(hisi_ddrc_pmu_module_init);
	451
	452	static void __exit hisi_ddrc_pmu_module_exit(void)
	453	{
	454	platform_driver_unregister(&hisi_ddrc_pmu_driver);
	455	cpuhp_remove_multi_state(CPUHP_AP_PERF_ARM_HISI_DDRC_ONLINE);
	456
	457	}
	458	module_exit(hisi_ddrc_pmu_module_exit);
	459
	460	MODULE_DESCRIPTION("HiSilicon SoC DDRC uncore PMU driver");
	461	MODULE_LICENSE("GPL v2");
	462	MODULE_AUTHOR("Shaokun Zhang <zhangshaokun@hisilicon.com>");
	463	MODULE_AUTHOR("Anurup M <anurup.m@huawei.com>");

drivers/perf/hisilicon/hisi_uncore_hha_pmu.c+473−0

@@ -0,0 +1,473 @@
	1	/*
	2	* HiSilicon SoC HHA uncore Hardware event counters support
	3	*
	4	* Copyright (C) 2017 Hisilicon Limited
	5	* Author: Shaokun Zhang <zhangshaokun@hisilicon.com>
	6	* Anurup M <anurup.m@huawei.com>
	7	*
	8	* This code is based on the uncore PMUs like arm-cci and arm-ccn.
	9	*
	10	* This program is free software; you can redistribute it and/or modify
	11	* it under the terms of the GNU General Public License version 2 as
	12	* published by the Free Software Foundation.
	13	*/
	14	#include <linux/acpi.h>
	15	#include <linux/bug.h>
	16	#include <linux/cpuhotplug.h>
	17	#include <linux/interrupt.h>
	18	#include <linux/irq.h>
	19	#include <linux/list.h>
	20	#include <linux/platform_device.h>
	21	#include <linux/smp.h>
	22
	23	#include "hisi_uncore_pmu.h"
	24
	25	/* HHA register definition */
	26	#define HHA_INT_MASK 0x0804
	27	#define HHA_INT_STATUS 0x0808
	28	#define HHA_INT_CLEAR 0x080C
	29	#define HHA_PERF_CTRL 0x1E00
	30	#define HHA_EVENT_CTRL 0x1E04
	31	#define HHA_EVENT_TYPE0 0x1E80
	32	/*
	33	* Each counter is 48-bits and [48:63] are reserved
	34	* which are Read-As-Zero and Writes-Ignored.
	35	*/
	36	#define HHA_CNT0_LOWER 0x1F00
	37
	38	/* HHA has 16-counters */
	39	#define HHA_NR_COUNTERS 0x10
	40
	41	#define HHA_PERF_CTRL_EN 0x1
	42	#define HHA_EVTYPE_NONE 0xff
	43
	44	/*
	45	* Select the counter register offset using the counter index
	46	* each counter is 48-bits.
	47	*/
	48	static u32 hisi_hha_pmu_get_counter_offset(int cntr_idx)
	49	{
	50	return (HHA_CNT0_LOWER + (cntr_idx * 8));
	51	}
	52
	53	static u64 hisi_hha_pmu_read_counter(struct hisi_pmu *hha_pmu,
	54	struct hw_perf_event *hwc)
	55	{
	56	u32 idx = hwc->idx;
	57
	58	if (!hisi_uncore_pmu_counter_valid(hha_pmu, idx)) {
	59	dev_err(hha_pmu->dev, "Unsupported event index:%d!\n", idx);
	60	return 0;
	61	}
	62
	63	/* Read 64 bits and like L3C, top 16 bits are RAZ */
	64	return readq(hha_pmu->base + hisi_hha_pmu_get_counter_offset(idx));
	65	}
	66
	67	static void hisi_hha_pmu_write_counter(struct hisi_pmu *hha_pmu,
	68	struct hw_perf_event *hwc, u64 val)
	69	{
	70	u32 idx = hwc->idx;
	71
	72	if (!hisi_uncore_pmu_counter_valid(hha_pmu, idx)) {
	73	dev_err(hha_pmu->dev, "Unsupported event index:%d!\n", idx);
	74	return;
	75	}
	76
	77	/* Write 64 bits and like L3C, top 16 bits are WI */
	78	writeq(val, hha_pmu->base + hisi_hha_pmu_get_counter_offset(idx));
	79	}
	80
	81	static void hisi_hha_pmu_write_evtype(struct hisi_pmu *hha_pmu, int idx,
	82	u32 type)
	83	{
	84	u32 reg, reg_idx, shift, val;
	85
	86	/*
	87	* Select the appropriate event select register(HHA_EVENT_TYPEx).
	88	* There are 4 event select registers for the 16 hardware counters.
	89	* Event code is 8-bits and for the first 4 hardware counters,
	90	* HHA_EVENT_TYPE0 is chosen. For the next 4 hardware counters,
	91	* HHA_EVENT_TYPE1 is chosen and so on.
	92	*/
	93	reg = HHA_EVENT_TYPE0 + 4 * (idx / 4);
	94	reg_idx = idx % 4;
	95	shift = 8 * reg_idx;
	96
	97	/* Write event code to HHA_EVENT_TYPEx register */
	98	val = readl(hha_pmu->base + reg);
	99	val &= ~(HHA_EVTYPE_NONE << shift);
	100	val \|= (type << shift);
	101	writel(val, hha_pmu->base + reg);
	102	}
	103
	104	static void hisi_hha_pmu_start_counters(struct hisi_pmu *hha_pmu)
	105	{
	106	u32 val;
	107
	108	/*
	109	* Set perf_enable bit in HHA_PERF_CTRL to start event
	110	* counting for all enabled counters.
	111	*/
	112	val = readl(hha_pmu->base + HHA_PERF_CTRL);
	113	val \|= HHA_PERF_CTRL_EN;
	114	writel(val, hha_pmu->base + HHA_PERF_CTRL);
	115	}
	116
	117	static void hisi_hha_pmu_stop_counters(struct hisi_pmu *hha_pmu)
	118	{
	119	u32 val;
	120
	121	/*
	122	* Clear perf_enable bit in HHA_PERF_CTRL to stop event
	123	* counting for all enabled counters.
	124	*/
	125	val = readl(hha_pmu->base + HHA_PERF_CTRL);
	126	val &= ~(HHA_PERF_CTRL_EN);
	127	writel(val, hha_pmu->base + HHA_PERF_CTRL);
	128	}
	129
	130	static void hisi_hha_pmu_enable_counter(struct hisi_pmu *hha_pmu,
	131	struct hw_perf_event *hwc)
	132	{
	133	u32 val;
	134
	135	/* Enable counter index in HHA_EVENT_CTRL register */
	136	val = readl(hha_pmu->base + HHA_EVENT_CTRL);
	137	val \|= (1 << hwc->idx);
	138	writel(val, hha_pmu->base + HHA_EVENT_CTRL);
	139	}
	140
	141	static void hisi_hha_pmu_disable_counter(struct hisi_pmu *hha_pmu,
	142	struct hw_perf_event *hwc)
	143	{
	144	u32 val;
	145
	146	/* Clear counter index in HHA_EVENT_CTRL register */
	147	val = readl(hha_pmu->base + HHA_EVENT_CTRL);
	148	val &= ~(1 << hwc->idx);
	149	writel(val, hha_pmu->base + HHA_EVENT_CTRL);
	150	}
	151
	152	static void hisi_hha_pmu_enable_counter_int(struct hisi_pmu *hha_pmu,
	153	struct hw_perf_event *hwc)
	154	{
	155	u32 val;
	156
	157	/* Write 0 to enable interrupt */
	158	val = readl(hha_pmu->base + HHA_INT_MASK);
	159	val &= ~(1 << hwc->idx);
	160	writel(val, hha_pmu->base + HHA_INT_MASK);
	161	}
	162
	163	static void hisi_hha_pmu_disable_counter_int(struct hisi_pmu *hha_pmu,
	164	struct hw_perf_event *hwc)
	165	{
	166	u32 val;
	167
	168	/* Write 1 to mask interrupt */
	169	val = readl(hha_pmu->base + HHA_INT_MASK);
	170	val \|= (1 << hwc->idx);
	171	writel(val, hha_pmu->base + HHA_INT_MASK);
	172	}
	173
	174	static irqreturn_t hisi_hha_pmu_isr(int irq, void *dev_id)
	175	{
	176	struct hisi_pmu *hha_pmu = dev_id;
	177	struct perf_event *event;
	178	unsigned long overflown;
	179	int idx;
	180
	181	/* Read HHA_INT_STATUS register */
	182	overflown = readl(hha_pmu->base + HHA_INT_STATUS);
	183	if (!overflown)
	184	return IRQ_NONE;
	185
	186	/*
	187	* Find the counter index which overflowed if the bit was set
	188	* and handle it
	189	*/
	190	for_each_set_bit(idx, &overflown, HHA_NR_COUNTERS) {
	191	/* Write 1 to clear the IRQ status flag */
	192	writel((1 << idx), hha_pmu->base + HHA_INT_CLEAR);
	193
	194	/* Get the corresponding event struct */
	195	event = hha_pmu->pmu_events.hw_events[idx];
	196	if (!event)
	197	continue;
	198
	199	hisi_uncore_pmu_event_update(event);
	200	hisi_uncore_pmu_set_event_period(event);
	201	}
	202
	203	return IRQ_HANDLED;
	204	}
	205
	206	static int hisi_hha_pmu_init_irq(struct hisi_pmu *hha_pmu,
	207	struct platform_device *pdev)
	208	{
	209	int irq, ret;
	210
	211	/* Read and init IRQ */
	212	irq = platform_get_irq(pdev, 0);
	213	if (irq < 0) {
	214	dev_err(&pdev->dev, "HHA PMU get irq fail; irq:%d\n", irq);
	215	return irq;
	216	}
	217
	218	ret = devm_request_irq(&pdev->dev, irq, hisi_hha_pmu_isr,
	219	IRQF_NOBALANCING \| IRQF_NO_THREAD,
	220	dev_name(&pdev->dev), hha_pmu);
	221	if (ret < 0) {
	222	dev_err(&pdev->dev,
	223	"Fail to request IRQ:%d ret:%d\n", irq, ret);
	224	return ret;
	225	}
	226
	227	hha_pmu->irq = irq;
	228
	229	return 0;
	230	}
	231
	232	static const struct acpi_device_id hisi_hha_pmu_acpi_match[] = {
	233	{ "HISI0243", },
	234	{},
	235	};
	236	MODULE_DEVICE_TABLE(acpi, hisi_hha_pmu_acpi_match);
	237
	238	static int hisi_hha_pmu_init_data(struct platform_device *pdev,
	239	struct hisi_pmu *hha_pmu)
	240	{
	241	unsigned long long id;
	242	struct resource *res;
	243	acpi_status status;
	244
	245	status = acpi_evaluate_integer(ACPI_HANDLE(&pdev->dev),
	246	"_UID", NULL, &id);
	247	if (ACPI_FAILURE(status))
	248	return -EINVAL;
	249
	250	hha_pmu->index_id = id;
	251
	252	/*
	253	* Use SCCL_ID and UID to identify the HHA PMU, while
	254	* SCCL_ID is in MPIDR[aff2].
	255	*/
	256	if (device_property_read_u32(&pdev->dev, "hisilicon,scl-id",
	257	&hha_pmu->sccl_id)) {
	258	dev_err(&pdev->dev, "Can not read hha sccl-id!\n");
	259	return -EINVAL;
	260	}
	261	/* HHA PMUs only share the same SCCL */
	262	hha_pmu->ccl_id = -1;
	263
	264	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
	265	hha_pmu->base = devm_ioremap_resource(&pdev->dev, res);
	266	if (IS_ERR(hha_pmu->base)) {
	267	dev_err(&pdev->dev, "ioremap failed for hha_pmu resource\n");
	268	return PTR_ERR(hha_pmu->base);
	269	}
	270
	271	return 0;
	272	}
	273
	274	static struct attribute *hisi_hha_pmu_format_attr[] = {
	275	HISI_PMU_FORMAT_ATTR(event, "config:0-7"),
	276	NULL,
	277	};
	278
	279	static const struct attribute_group hisi_hha_pmu_format_group = {
	280	.name = "format",
	281	.attrs = hisi_hha_pmu_format_attr,
	282	};
	283
	284	static struct attribute *hisi_hha_pmu_events_attr[] = {
	285	HISI_PMU_EVENT_ATTR(rx_ops_num, 0x00),
	286	HISI_PMU_EVENT_ATTR(rx_outer, 0x01),
	287	HISI_PMU_EVENT_ATTR(rx_sccl, 0x02),
	288	HISI_PMU_EVENT_ATTR(rx_ccix, 0x03),
	289	HISI_PMU_EVENT_ATTR(rx_wbi, 0x04),
	290	HISI_PMU_EVENT_ATTR(rx_wbip, 0x05),
	291	HISI_PMU_EVENT_ATTR(rx_wtistash, 0x11),
	292	HISI_PMU_EVENT_ATTR(rd_ddr_64b, 0x1c),
	293	HISI_PMU_EVENT_ATTR(wr_dr_64b, 0x1d),
	294	HISI_PMU_EVENT_ATTR(rd_ddr_128b, 0x1e),
	295	HISI_PMU_EVENT_ATTR(wr_ddr_128b, 0x1f),
	296	HISI_PMU_EVENT_ATTR(spill_num, 0x20),
	297	HISI_PMU_EVENT_ATTR(spill_success, 0x21),
	298	HISI_PMU_EVENT_ATTR(bi_num, 0x23),
	299	HISI_PMU_EVENT_ATTR(mediated_num, 0x32),
	300	HISI_PMU_EVENT_ATTR(tx_snp_num, 0x33),
	301	HISI_PMU_EVENT_ATTR(tx_snp_outer, 0x34),
	302	HISI_PMU_EVENT_ATTR(tx_snp_ccix, 0x35),
	303	HISI_PMU_EVENT_ATTR(rx_snprspdata, 0x38),
	304	HISI_PMU_EVENT_ATTR(rx_snprsp_outer, 0x3c),
	305	HISI_PMU_EVENT_ATTR(sdir-lookup, 0x40),
	306	HISI_PMU_EVENT_ATTR(edir-lookup, 0x41),
	307	HISI_PMU_EVENT_ATTR(sdir-hit, 0x42),
	308	HISI_PMU_EVENT_ATTR(edir-hit, 0x43),
	309	HISI_PMU_EVENT_ATTR(sdir-home-migrate, 0x4c),
	310	HISI_PMU_EVENT_ATTR(edir-home-migrate, 0x4d),
	311	NULL,
	312	};
	313
	314	static const struct attribute_group hisi_hha_pmu_events_group = {
	315	.name = "events",
	316	.attrs = hisi_hha_pmu_events_attr,
	317	};
	318
	319	static DEVICE_ATTR(cpumask, 0444, hisi_cpumask_sysfs_show, NULL);
	320
	321	static struct attribute *hisi_hha_pmu_cpumask_attrs[] = {
	322	&dev_attr_cpumask.attr,
	323	NULL,
	324	};
	325
	326	static const struct attribute_group hisi_hha_pmu_cpumask_attr_group = {
	327	.attrs = hisi_hha_pmu_cpumask_attrs,
	328	};
	329
	330	static const struct attribute_group *hisi_hha_pmu_attr_groups[] = {
	331	&hisi_hha_pmu_format_group,
	332	&hisi_hha_pmu_events_group,
	333	&hisi_hha_pmu_cpumask_attr_group,
	334	NULL,
	335	};
	336
	337	static const struct hisi_uncore_ops hisi_uncore_hha_ops = {
	338	.write_evtype = hisi_hha_pmu_write_evtype,
	339	.get_event_idx = hisi_uncore_pmu_get_event_idx,
	340	.start_counters = hisi_hha_pmu_start_counters,
	341	.stop_counters = hisi_hha_pmu_stop_counters,
	342	.enable_counter = hisi_hha_pmu_enable_counter,
	343	.disable_counter = hisi_hha_pmu_disable_counter,
	344	.enable_counter_int = hisi_hha_pmu_enable_counter_int,
	345	.disable_counter_int = hisi_hha_pmu_disable_counter_int,
	346	.write_counter = hisi_hha_pmu_write_counter,
	347	.read_counter = hisi_hha_pmu_read_counter,
	348	};
	349
	350	static int hisi_hha_pmu_dev_probe(struct platform_device *pdev,
	351	struct hisi_pmu *hha_pmu)
	352	{
	353	int ret;
	354
	355	ret = hisi_hha_pmu_init_data(pdev, hha_pmu);
	356	if (ret)
	357	return ret;
	358
	359	ret = hisi_hha_pmu_init_irq(hha_pmu, pdev);
	360	if (ret)
	361	return ret;
	362
	363	hha_pmu->num_counters = HHA_NR_COUNTERS;
	364	hha_pmu->counter_bits = 48;
	365	hha_pmu->ops = &hisi_uncore_hha_ops;
	366	hha_pmu->dev = &pdev->dev;
	367	hha_pmu->on_cpu = -1;
	368	hha_pmu->check_event = 0x65;
	369
	370	return 0;
	371	}
	372
	373	static int hisi_hha_pmu_probe(struct platform_device *pdev)
	374	{
	375	struct hisi_pmu *hha_pmu;
	376	char *name;
	377	int ret;
	378
	379	hha_pmu = devm_kzalloc(&pdev->dev, sizeof(*hha_pmu), GFP_KERNEL);
	380	if (!hha_pmu)
	381	return -ENOMEM;
	382
	383	platform_set_drvdata(pdev, hha_pmu);
	384
	385	ret = hisi_hha_pmu_dev_probe(pdev, hha_pmu);
	386	if (ret)
	387	return ret;
	388
	389	ret = cpuhp_state_add_instance(CPUHP_AP_PERF_ARM_HISI_HHA_ONLINE,
	390	&hha_pmu->node);
	391	if (ret) {
	392	dev_err(&pdev->dev, "Error %d registering hotplug\n", ret);
	393	return ret;
	394	}
	395
	396	name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "hisi_sccl%u_hha%u",
	397	hha_pmu->sccl_id, hha_pmu->index_id);
	398	hha_pmu->pmu = (struct pmu) {
	399	.name = name,
	400	.task_ctx_nr = perf_invalid_context,
	401	.event_init = hisi_uncore_pmu_event_init,
	402	.pmu_enable = hisi_uncore_pmu_enable,
	403	.pmu_disable = hisi_uncore_pmu_disable,
	404	.add = hisi_uncore_pmu_add,
	405	.del = hisi_uncore_pmu_del,
	406	.start = hisi_uncore_pmu_start,
	407	.stop = hisi_uncore_pmu_stop,
	408	.read = hisi_uncore_pmu_read,
	409	.attr_groups = hisi_hha_pmu_attr_groups,
	410	};
	411
	412	ret = perf_pmu_register(&hha_pmu->pmu, name, -1);
	413	if (ret) {
	414	dev_err(hha_pmu->dev, "HHA PMU register failed!\n");
	415	cpuhp_state_remove_instance(CPUHP_AP_PERF_ARM_HISI_HHA_ONLINE,
	416	&hha_pmu->node);
	417	}
	418
	419	return ret;
	420	}
	421
	422	static int hisi_hha_pmu_remove(struct platform_device *pdev)
	423	{
	424	struct hisi_pmu *hha_pmu = platform_get_drvdata(pdev);
	425
	426	perf_pmu_unregister(&hha_pmu->pmu);
	427	cpuhp_state_remove_instance(CPUHP_AP_PERF_ARM_HISI_HHA_ONLINE,
	428	&hha_pmu->node);
	429
	430	return 0;
	431	}
	432
	433	static struct platform_driver hisi_hha_pmu_driver = {
	434	.driver = {
	435	.name = "hisi_hha_pmu",
	436	.acpi_match_table = ACPI_PTR(hisi_hha_pmu_acpi_match),
	437	},
	438	.probe = hisi_hha_pmu_probe,
	439	.remove = hisi_hha_pmu_remove,
	440	};
	441
	442	static int __init hisi_hha_pmu_module_init(void)
	443	{
	444	int ret;
	445
	446	ret = cpuhp_setup_state_multi(CPUHP_AP_PERF_ARM_HISI_HHA_ONLINE,
	447	"AP_PERF_ARM_HISI_HHA_ONLINE",
	448	hisi_uncore_pmu_online_cpu,
	449	hisi_uncore_pmu_offline_cpu);
	450	if (ret) {
	451	pr_err("HHA PMU: Error setup hotplug, ret = %d;\n", ret);
	452	return ret;
	453	}
	454
	455	ret = platform_driver_register(&hisi_hha_pmu_driver);
	456	if (ret)
	457	cpuhp_remove_multi_state(CPUHP_AP_PERF_ARM_HISI_HHA_ONLINE);
	458
	459	return ret;
	460	}
	461	module_init(hisi_hha_pmu_module_init);
	462
	463	static void __exit hisi_hha_pmu_module_exit(void)
	464	{
	465	platform_driver_unregister(&hisi_hha_pmu_driver);
	466	cpuhp_remove_multi_state(CPUHP_AP_PERF_ARM_HISI_HHA_ONLINE);
	467	}
	468	module_exit(hisi_hha_pmu_module_exit);
	469
	470	MODULE_DESCRIPTION("HiSilicon SoC HHA uncore PMU driver");
	471	MODULE_LICENSE("GPL v2");
	472	MODULE_AUTHOR("Shaokun Zhang <zhangshaokun@hisilicon.com>");
	473	MODULE_AUTHOR("Anurup M <anurup.m@huawei.com>");

drivers/perf/hisilicon/hisi_uncore_l3c_pmu.c+463−0

@@ -0,0 +1,463 @@
	1	/*
	2	* HiSilicon SoC L3C uncore Hardware event counters support
	3	*
	4	* Copyright (C) 2017 Hisilicon Limited
	5	* Author: Anurup M <anurup.m@huawei.com>
	6	* Shaokun Zhang <zhangshaokun@hisilicon.com>
	7	*
	8	* This code is based on the uncore PMUs like arm-cci and arm-ccn.
	9	*
	10	* This program is free software; you can redistribute it and/or modify
	11	* it under the terms of the GNU General Public License version 2 as
	12	* published by the Free Software Foundation.
	13	*/
	14	#include <linux/acpi.h>
	15	#include <linux/bug.h>
	16	#include <linux/cpuhotplug.h>
	17	#include <linux/interrupt.h>
	18	#include <linux/irq.h>
	19	#include <linux/list.h>
	20	#include <linux/platform_device.h>
	21	#include <linux/smp.h>
	22
	23	#include "hisi_uncore_pmu.h"
	24
	25	/* L3C register definition */
	26	#define L3C_PERF_CTRL 0x0408
	27	#define L3C_INT_MASK 0x0800
	28	#define L3C_INT_STATUS 0x0808
	29	#define L3C_INT_CLEAR 0x080c
	30	#define L3C_EVENT_CTRL 0x1c00
	31	#define L3C_EVENT_TYPE0 0x1d00
	32	/*
	33	* Each counter is 48-bits and [48:63] are reserved
	34	* which are Read-As-Zero and Writes-Ignored.
	35	*/
	36	#define L3C_CNTR0_LOWER 0x1e00
	37
	38	/* L3C has 8-counters */
	39	#define L3C_NR_COUNTERS 0x8
	40
	41	#define L3C_PERF_CTRL_EN 0x20000
	42	#define L3C_EVTYPE_NONE 0xff
	43
	44	/*
	45	* Select the counter register offset using the counter index
	46	*/
	47	static u32 hisi_l3c_pmu_get_counter_offset(int cntr_idx)
	48	{
	49	return (L3C_CNTR0_LOWER + (cntr_idx * 8));
	50	}
	51
	52	static u64 hisi_l3c_pmu_read_counter(struct hisi_pmu *l3c_pmu,
	53	struct hw_perf_event *hwc)
	54	{
	55	u32 idx = hwc->idx;
	56
	57	if (!hisi_uncore_pmu_counter_valid(l3c_pmu, idx)) {
	58	dev_err(l3c_pmu->dev, "Unsupported event index:%d!\n", idx);
	59	return 0;
	60	}
	61
	62	/* Read 64-bits and the upper 16 bits are RAZ */
	63	return readq(l3c_pmu->base + hisi_l3c_pmu_get_counter_offset(idx));
	64	}
	65
	66	static void hisi_l3c_pmu_write_counter(struct hisi_pmu *l3c_pmu,
	67	struct hw_perf_event *hwc, u64 val)
	68	{
	69	u32 idx = hwc->idx;
	70
	71	if (!hisi_uncore_pmu_counter_valid(l3c_pmu, idx)) {
	72	dev_err(l3c_pmu->dev, "Unsupported event index:%d!\n", idx);
	73	return;
	74	}
	75
	76	/* Write 64-bits and the upper 16 bits are WI */
	77	writeq(val, l3c_pmu->base + hisi_l3c_pmu_get_counter_offset(idx));
	78	}
	79
	80	static void hisi_l3c_pmu_write_evtype(struct hisi_pmu *l3c_pmu, int idx,
	81	u32 type)
	82	{
	83	u32 reg, reg_idx, shift, val;
	84
	85	/*
	86	* Select the appropriate event select register(L3C_EVENT_TYPE0/1).
	87	* There are 2 event select registers for the 8 hardware counters.
	88	* Event code is 8-bits and for the former 4 hardware counters,
	89	* L3C_EVENT_TYPE0 is chosen. For the latter 4 hardware counters,
	90	* L3C_EVENT_TYPE1 is chosen.
	91	*/
	92	reg = L3C_EVENT_TYPE0 + (idx / 4) * 4;
	93	reg_idx = idx % 4;
	94	shift = 8 * reg_idx;
	95
	96	/* Write event code to L3C_EVENT_TYPEx Register */
	97	val = readl(l3c_pmu->base + reg);
	98	val &= ~(L3C_EVTYPE_NONE << shift);
	99	val \|= (type << shift);
	100	writel(val, l3c_pmu->base + reg);
	101	}
	102
	103	static void hisi_l3c_pmu_start_counters(struct hisi_pmu *l3c_pmu)
	104	{
	105	u32 val;
	106
	107	/*
	108	* Set perf_enable bit in L3C_PERF_CTRL register to start counting
	109	* for all enabled counters.
	110	*/
	111	val = readl(l3c_pmu->base + L3C_PERF_CTRL);
	112	val \|= L3C_PERF_CTRL_EN;
	113	writel(val, l3c_pmu->base + L3C_PERF_CTRL);
	114	}
	115
	116	static void hisi_l3c_pmu_stop_counters(struct hisi_pmu *l3c_pmu)
	117	{
	118	u32 val;
	119
	120	/*
	121	* Clear perf_enable bit in L3C_PERF_CTRL register to stop counting
	122	* for all enabled counters.
	123	*/
	124	val = readl(l3c_pmu->base + L3C_PERF_CTRL);
	125	val &= ~(L3C_PERF_CTRL_EN);
	126	writel(val, l3c_pmu->base + L3C_PERF_CTRL);
	127	}
	128
	129	static void hisi_l3c_pmu_enable_counter(struct hisi_pmu *l3c_pmu,
	130	struct hw_perf_event *hwc)
	131	{
	132	u32 val;
	133
	134	/* Enable counter index in L3C_EVENT_CTRL register */
	135	val = readl(l3c_pmu->base + L3C_EVENT_CTRL);
	136	val \|= (1 << hwc->idx);
	137	writel(val, l3c_pmu->base + L3C_EVENT_CTRL);
	138	}
	139
	140	static void hisi_l3c_pmu_disable_counter(struct hisi_pmu *l3c_pmu,
	141	struct hw_perf_event *hwc)
	142	{
	143	u32 val;
	144
	145	/* Clear counter index in L3C_EVENT_CTRL register */
	146	val = readl(l3c_pmu->base + L3C_EVENT_CTRL);
	147	val &= ~(1 << hwc->idx);
	148	writel(val, l3c_pmu->base + L3C_EVENT_CTRL);
	149	}
	150
	151	static void hisi_l3c_pmu_enable_counter_int(struct hisi_pmu *l3c_pmu,
	152	struct hw_perf_event *hwc)
	153	{
	154	u32 val;
	155
	156	val = readl(l3c_pmu->base + L3C_INT_MASK);
	157	/* Write 0 to enable interrupt */
	158	val &= ~(1 << hwc->idx);
	159	writel(val, l3c_pmu->base + L3C_INT_MASK);
	160	}
	161
	162	static void hisi_l3c_pmu_disable_counter_int(struct hisi_pmu *l3c_pmu,
	163	struct hw_perf_event *hwc)
	164	{
	165	u32 val;
	166
	167	val = readl(l3c_pmu->base + L3C_INT_MASK);
	168	/* Write 1 to mask interrupt */
	169	val \|= (1 << hwc->idx);
	170	writel(val, l3c_pmu->base + L3C_INT_MASK);
	171	}
	172
	173	static irqreturn_t hisi_l3c_pmu_isr(int irq, void *dev_id)
	174	{
	175	struct hisi_pmu *l3c_pmu = dev_id;
	176	struct perf_event *event;
	177	unsigned long overflown;
	178	int idx;
	179
	180	/* Read L3C_INT_STATUS register */
	181	overflown = readl(l3c_pmu->base + L3C_INT_STATUS);
	182	if (!overflown)
	183	return IRQ_NONE;
	184
	185	/*
	186	* Find the counter index which overflowed if the bit was set
	187	* and handle it.
	188	*/
	189	for_each_set_bit(idx, &overflown, L3C_NR_COUNTERS) {
	190	/* Write 1 to clear the IRQ status flag */
	191	writel((1 << idx), l3c_pmu->base + L3C_INT_CLEAR);
	192
	193	/* Get the corresponding event struct */
	194	event = l3c_pmu->pmu_events.hw_events[idx];
	195	if (!event)
	196	continue;
	197
	198	hisi_uncore_pmu_event_update(event);
	199	hisi_uncore_pmu_set_event_period(event);
	200	}
	201
	202	return IRQ_HANDLED;
	203	}
	204
	205	static int hisi_l3c_pmu_init_irq(struct hisi_pmu *l3c_pmu,
	206	struct platform_device *pdev)
	207	{
	208	int irq, ret;
	209
	210	/* Read and init IRQ */
	211	irq = platform_get_irq(pdev, 0);
	212	if (irq < 0) {
	213	dev_err(&pdev->dev, "L3C PMU get irq fail; irq:%d\n", irq);
	214	return irq;
	215	}
	216
	217	ret = devm_request_irq(&pdev->dev, irq, hisi_l3c_pmu_isr,
	218	IRQF_NOBALANCING \| IRQF_NO_THREAD,
	219	dev_name(&pdev->dev), l3c_pmu);
	220	if (ret < 0) {
	221	dev_err(&pdev->dev,
	222	"Fail to request IRQ:%d ret:%d\n", irq, ret);
	223	return ret;
	224	}
	225
	226	l3c_pmu->irq = irq;
	227
	228	return 0;
	229	}
	230
	231	static const struct acpi_device_id hisi_l3c_pmu_acpi_match[] = {
	232	{ "HISI0213", },
	233	{},
	234	};
	235	MODULE_DEVICE_TABLE(acpi, hisi_l3c_pmu_acpi_match);
	236
	237	static int hisi_l3c_pmu_init_data(struct platform_device *pdev,
	238	struct hisi_pmu *l3c_pmu)
	239	{
	240	unsigned long long id;
	241	struct resource *res;
	242	acpi_status status;
	243
	244	status = acpi_evaluate_integer(ACPI_HANDLE(&pdev->dev),
	245	"_UID", NULL, &id);
	246	if (ACPI_FAILURE(status))
	247	return -EINVAL;
	248
	249	l3c_pmu->index_id = id;
	250
	251	/*
	252	* Use the SCCL_ID and CCL_ID to identify the L3C PMU, while
	253	* SCCL_ID is in MPIDR[aff2] and CCL_ID is in MPIDR[aff1].
	254	*/
	255	if (device_property_read_u32(&pdev->dev, "hisilicon,scl-id",
	256	&l3c_pmu->sccl_id)) {
	257	dev_err(&pdev->dev, "Can not read l3c sccl-id!\n");
	258	return -EINVAL;
	259	}
	260
	261	if (device_property_read_u32(&pdev->dev, "hisilicon,ccl-id",
	262	&l3c_pmu->ccl_id)) {
	263	dev_err(&pdev->dev, "Can not read l3c ccl-id!\n");
	264	return -EINVAL;
	265	}
	266
	267	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
	268	l3c_pmu->base = devm_ioremap_resource(&pdev->dev, res);
	269	if (IS_ERR(l3c_pmu->base)) {
	270	dev_err(&pdev->dev, "ioremap failed for l3c_pmu resource\n");
	271	return PTR_ERR(l3c_pmu->base);
	272	}
	273
	274	return 0;
	275	}
	276
	277	static struct attribute *hisi_l3c_pmu_format_attr[] = {
	278	HISI_PMU_FORMAT_ATTR(event, "config:0-7"),
	279	NULL,
	280	};
	281
	282	static const struct attribute_group hisi_l3c_pmu_format_group = {
	283	.name = "format",
	284	.attrs = hisi_l3c_pmu_format_attr,
	285	};
	286
	287	static struct attribute *hisi_l3c_pmu_events_attr[] = {
	288	HISI_PMU_EVENT_ATTR(rd_cpipe, 0x00),
	289	HISI_PMU_EVENT_ATTR(wr_cpipe, 0x01),
	290	HISI_PMU_EVENT_ATTR(rd_hit_cpipe, 0x02),
	291	HISI_PMU_EVENT_ATTR(wr_hit_cpipe, 0x03),
	292	HISI_PMU_EVENT_ATTR(victim_num, 0x04),
	293	HISI_PMU_EVENT_ATTR(rd_spipe, 0x20),
	294	HISI_PMU_EVENT_ATTR(wr_spipe, 0x21),
	295	HISI_PMU_EVENT_ATTR(rd_hit_spipe, 0x22),
	296	HISI_PMU_EVENT_ATTR(wr_hit_spipe, 0x23),
	297	HISI_PMU_EVENT_ATTR(back_invalid, 0x29),
	298	HISI_PMU_EVENT_ATTR(retry_cpu, 0x40),
	299	HISI_PMU_EVENT_ATTR(retry_ring, 0x41),
	300	HISI_PMU_EVENT_ATTR(prefetch_drop, 0x42),
	301	NULL,
	302	};
	303
	304	static const struct attribute_group hisi_l3c_pmu_events_group = {
	305	.name = "events",
	306	.attrs = hisi_l3c_pmu_events_attr,
	307	};
	308
	309	static DEVICE_ATTR(cpumask, 0444, hisi_cpumask_sysfs_show, NULL);
	310
	311	static struct attribute *hisi_l3c_pmu_cpumask_attrs[] = {
	312	&dev_attr_cpumask.attr,
	313	NULL,
	314	};
	315
	316	static const struct attribute_group hisi_l3c_pmu_cpumask_attr_group = {
	317	.attrs = hisi_l3c_pmu_cpumask_attrs,
	318	};
	319
	320	static const struct attribute_group *hisi_l3c_pmu_attr_groups[] = {
	321	&hisi_l3c_pmu_format_group,
	322	&hisi_l3c_pmu_events_group,
	323	&hisi_l3c_pmu_cpumask_attr_group,
	324	NULL,
	325	};
	326
	327	static const struct hisi_uncore_ops hisi_uncore_l3c_ops = {
	328	.write_evtype = hisi_l3c_pmu_write_evtype,
	329	.get_event_idx = hisi_uncore_pmu_get_event_idx,
	330	.start_counters = hisi_l3c_pmu_start_counters,
	331	.stop_counters = hisi_l3c_pmu_stop_counters,
	332	.enable_counter = hisi_l3c_pmu_enable_counter,
	333	.disable_counter = hisi_l3c_pmu_disable_counter,
	334	.enable_counter_int = hisi_l3c_pmu_enable_counter_int,
	335	.disable_counter_int = hisi_l3c_pmu_disable_counter_int,
	336	.write_counter = hisi_l3c_pmu_write_counter,
	337	.read_counter = hisi_l3c_pmu_read_counter,
	338	};
	339
	340	static int hisi_l3c_pmu_dev_probe(struct platform_device *pdev,
	341	struct hisi_pmu *l3c_pmu)
	342	{
	343	int ret;
	344
	345	ret = hisi_l3c_pmu_init_data(pdev, l3c_pmu);
	346	if (ret)
	347	return ret;
	348
	349	ret = hisi_l3c_pmu_init_irq(l3c_pmu, pdev);
	350	if (ret)
	351	return ret;
	352
	353	l3c_pmu->num_counters = L3C_NR_COUNTERS;
	354	l3c_pmu->counter_bits = 48;
	355	l3c_pmu->ops = &hisi_uncore_l3c_ops;
	356	l3c_pmu->dev = &pdev->dev;
	357	l3c_pmu->on_cpu = -1;
	358	l3c_pmu->check_event = 0x59;
	359
	360	return 0;
	361	}
	362
	363	static int hisi_l3c_pmu_probe(struct platform_device *pdev)
	364	{
	365	struct hisi_pmu *l3c_pmu;
	366	char *name;
	367	int ret;
	368
	369	l3c_pmu = devm_kzalloc(&pdev->dev, sizeof(*l3c_pmu), GFP_KERNEL);
	370	if (!l3c_pmu)
	371	return -ENOMEM;
	372
	373	platform_set_drvdata(pdev, l3c_pmu);
	374
	375	ret = hisi_l3c_pmu_dev_probe(pdev, l3c_pmu);
	376	if (ret)
	377	return ret;
	378
	379	ret = cpuhp_state_add_instance(CPUHP_AP_PERF_ARM_HISI_L3_ONLINE,
	380	&l3c_pmu->node);
	381	if (ret) {
	382	dev_err(&pdev->dev, "Error %d registering hotplug\n", ret);
	383	return ret;
	384	}
	385
	386	name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "hisi_sccl%u_l3c%u",
	387	l3c_pmu->sccl_id, l3c_pmu->index_id);
	388	l3c_pmu->pmu = (struct pmu) {
	389	.name = name,
	390	.task_ctx_nr = perf_invalid_context,
	391	.event_init = hisi_uncore_pmu_event_init,
	392	.pmu_enable = hisi_uncore_pmu_enable,
	393	.pmu_disable = hisi_uncore_pmu_disable,
	394	.add = hisi_uncore_pmu_add,
	395	.del = hisi_uncore_pmu_del,
	396	.start = hisi_uncore_pmu_start,
	397	.stop = hisi_uncore_pmu_stop,
	398	.read = hisi_uncore_pmu_read,
	399	.attr_groups = hisi_l3c_pmu_attr_groups,
	400	};
	401
	402	ret = perf_pmu_register(&l3c_pmu->pmu, name, -1);
	403	if (ret) {
	404	dev_err(l3c_pmu->dev, "L3C PMU register failed!\n");
	405	cpuhp_state_remove_instance(CPUHP_AP_PERF_ARM_HISI_L3_ONLINE,
	406	&l3c_pmu->node);
	407	}
	408
	409	return ret;
	410	}
	411
	412	static int hisi_l3c_pmu_remove(struct platform_device *pdev)
	413	{
	414	struct hisi_pmu *l3c_pmu = platform_get_drvdata(pdev);
	415
	416	perf_pmu_unregister(&l3c_pmu->pmu);
	417	cpuhp_state_remove_instance(CPUHP_AP_PERF_ARM_HISI_L3_ONLINE,
	418	&l3c_pmu->node);
	419
	420	return 0;
	421	}
	422
	423	static struct platform_driver hisi_l3c_pmu_driver = {
	424	.driver = {
	425	.name = "hisi_l3c_pmu",
	426	.acpi_match_table = ACPI_PTR(hisi_l3c_pmu_acpi_match),
	427	},
	428	.probe = hisi_l3c_pmu_probe,
	429	.remove = hisi_l3c_pmu_remove,
	430	};
	431
	432	static int __init hisi_l3c_pmu_module_init(void)
	433	{
	434	int ret;
	435
	436	ret = cpuhp_setup_state_multi(CPUHP_AP_PERF_ARM_HISI_L3_ONLINE,
	437	"AP_PERF_ARM_HISI_L3_ONLINE",
	438	hisi_uncore_pmu_online_cpu,
	439	hisi_uncore_pmu_offline_cpu);
	440	if (ret) {
	441	pr_err("L3C PMU: Error setup hotplug, ret = %d\n", ret);
	442	return ret;
	443	}
	444
	445	ret = platform_driver_register(&hisi_l3c_pmu_driver);
	446	if (ret)
	447	cpuhp_remove_multi_state(CPUHP_AP_PERF_ARM_HISI_L3_ONLINE);
	448
	449	return ret;
	450	}
	451	module_init(hisi_l3c_pmu_module_init);
	452
	453	static void __exit hisi_l3c_pmu_module_exit(void)
	454	{
	455	platform_driver_unregister(&hisi_l3c_pmu_driver);
	456	cpuhp_remove_multi_state(CPUHP_AP_PERF_ARM_HISI_L3_ONLINE);
	457	}
	458	module_exit(hisi_l3c_pmu_module_exit);
	459
	460	MODULE_DESCRIPTION("HiSilicon SoC L3C uncore PMU driver");
	461	MODULE_LICENSE("GPL v2");
	462	MODULE_AUTHOR("Anurup M <anurup.m@huawei.com>");
	463	MODULE_AUTHOR("Shaokun Zhang <zhangshaokun@hisilicon.com>");

drivers/perf/hisilicon/hisi_uncore_pmu.c+447−0

@@ -0,0 +1,447 @@
	1	/*
	2	* HiSilicon SoC Hardware event counters support
	3	*
	4	* Copyright (C) 2017 Hisilicon Limited
	5	* Author: Anurup M <anurup.m@huawei.com>
	6	* Shaokun Zhang <zhangshaokun@hisilicon.com>
	7	*
	8	* This code is based on the uncore PMUs like arm-cci and arm-ccn.
	9	*
	10	* This program is free software; you can redistribute it and/or modify
	11	* it under the terms of the GNU General Public License version 2 as
	12	* published by the Free Software Foundation.
	13	*/
	14	#include <linux/bitmap.h>
	15	#include <linux/bitops.h>
	16	#include <linux/bug.h>
	17	#include <linux/err.h>
	18	#include <linux/errno.h>
	19	#include <linux/interrupt.h>
	20
	21	#include <asm/local64.h>
	22
	23	#include "hisi_uncore_pmu.h"
	24
	25	#define HISI_GET_EVENTID(ev) (ev->hw.config_base & 0xff)
	26	#define HISI_MAX_PERIOD(nr) (BIT_ULL(nr) - 1)
	27
	28	/*
	29	* PMU format attributes
	30	*/
	31	ssize_t hisi_format_sysfs_show(struct device *dev,
	32	struct device_attribute attr, char buf)
	33	{
	34	struct dev_ext_attribute *eattr;
	35
	36	eattr = container_of(attr, struct dev_ext_attribute, attr);
	37
	38	return sprintf(buf, "%s\n", (char *)eattr->var);
	39	}
	40
	41	/*
	42	* PMU event attributes
	43	*/
	44	ssize_t hisi_event_sysfs_show(struct device *dev,
	45	struct device_attribute attr, char page)
	46	{
	47	struct dev_ext_attribute *eattr;
	48
	49	eattr = container_of(attr, struct dev_ext_attribute, attr);
	50
	51	return sprintf(page, "config=0x%lx\n", (unsigned long)eattr->var);
	52	}
	53
	54	/*
	55	* sysfs cpumask attributes. For uncore PMU, we only have a single CPU to show
	56	*/
	57	ssize_t hisi_cpumask_sysfs_show(struct device *dev,
	58	struct device_attribute attr, char buf)
	59	{
	60	struct hisi_pmu *hisi_pmu = to_hisi_pmu(dev_get_drvdata(dev));
	61
	62	return sprintf(buf, "%d\n", hisi_pmu->on_cpu);
	63	}
	64
	65	static bool hisi_validate_event_group(struct perf_event *event)
	66	{
	67	struct perf_event sibling, leader = event->group_leader;
	68	struct hisi_pmu *hisi_pmu = to_hisi_pmu(event->pmu);
	69	/* Include count for the event */
	70	int counters = 1;
	71
	72	if (!is_software_event(leader)) {
	73	/*
	74	* We must NOT create groups containing mixed PMUs, although
	75	* software events are acceptable
	76	*/
	77	if (leader->pmu != event->pmu)
	78	return false;
	79
	80	/* Increment counter for the leader */
	81	if (leader != event)
	82	counters++;
	83	}
	84
	85	list_for_each_entry(sibling, &event->group_leader->sibling_list,
	86	group_entry) {
	87	if (is_software_event(sibling))
	88	continue;
	89	if (sibling->pmu != event->pmu)
	90	return false;
	91	/* Increment counter for each sibling */
	92	counters++;
	93	}
	94
	95	/* The group can not count events more than the counters in the HW */
	96	return counters <= hisi_pmu->num_counters;
	97	}
	98
	99	int hisi_uncore_pmu_counter_valid(struct hisi_pmu *hisi_pmu, int idx)
	100	{
	101	return idx >= 0 && idx < hisi_pmu->num_counters;
	102	}
	103
	104	int hisi_uncore_pmu_get_event_idx(struct perf_event *event)
	105	{
	106	struct hisi_pmu *hisi_pmu = to_hisi_pmu(event->pmu);
	107	unsigned long *used_mask = hisi_pmu->pmu_events.used_mask;
	108	u32 num_counters = hisi_pmu->num_counters;
	109	int idx;
	110
	111	idx = find_first_zero_bit(used_mask, num_counters);
	112	if (idx == num_counters)
	113	return -EAGAIN;
	114
	115	set_bit(idx, used_mask);
	116
	117	return idx;
	118	}
	119
	120	static void hisi_uncore_pmu_clear_event_idx(struct hisi_pmu *hisi_pmu, int idx)
	121	{
	122	if (!hisi_uncore_pmu_counter_valid(hisi_pmu, idx)) {
	123	dev_err(hisi_pmu->dev, "Unsupported event index:%d!\n", idx);
	124	return;
	125	}
	126
	127	clear_bit(idx, hisi_pmu->pmu_events.used_mask);
	128	}
	129
	130	int hisi_uncore_pmu_event_init(struct perf_event *event)
	131	{
	132	struct hw_perf_event *hwc = &event->hw;
	133	struct hisi_pmu *hisi_pmu;
	134
	135	if (event->attr.type != event->pmu->type)
	136	return -ENOENT;
	137
	138	/*
	139	* We do not support sampling as the counters are all
	140	* shared by all CPU cores in a CPU die(SCCL). Also we
	141	* do not support attach to a task(per-process mode)
	142	*/
	143	if (is_sampling_event(event) \|\| event->attach_state & PERF_ATTACH_TASK)
	144	return -EOPNOTSUPP;
	145
	146	/* counters do not have these bits */
	147	if (event->attr.exclude_user \|\|
	148	event->attr.exclude_kernel \|\|
	149	event->attr.exclude_host \|\|
	150	event->attr.exclude_guest \|\|
	151	event->attr.exclude_hv \|\|
	152	event->attr.exclude_idle)
	153	return -EINVAL;
	154
	155	/*
	156	* The uncore counters not specific to any CPU, so cannot
	157	* support per-task
	158	*/
	159	if (event->cpu < 0)
	160	return -EINVAL;
	161
	162	/*
	163	* Validate if the events in group does not exceed the
	164	* available counters in hardware.
	165	*/
	166	if (!hisi_validate_event_group(event))
	167	return -EINVAL;
	168
	169	hisi_pmu = to_hisi_pmu(event->pmu);
	170	if (event->attr.config > hisi_pmu->check_event)
	171	return -EINVAL;
	172
	173	if (hisi_pmu->on_cpu == -1)
	174	return -EINVAL;
	175	/*
	176	* We don't assign an index until we actually place the event onto
	177	* hardware. Use -1 to signify that we haven't decided where to put it
	178	* yet.
	179	*/
	180	hwc->idx = -1;
	181	hwc->config_base = event->attr.config;
	182
	183	/* Enforce to use the same CPU for all events in this PMU */
	184	event->cpu = hisi_pmu->on_cpu;
	185
	186	return 0;
	187	}
	188
	189	/*
	190	* Set the counter to count the event that we're interested in,
	191	* and enable interrupt and counter.
	192	*/
	193	static void hisi_uncore_pmu_enable_event(struct perf_event *event)
	194	{
	195	struct hisi_pmu *hisi_pmu = to_hisi_pmu(event->pmu);
	196	struct hw_perf_event *hwc = &event->hw;
	197
	198	hisi_pmu->ops->write_evtype(hisi_pmu, hwc->idx,
	199	HISI_GET_EVENTID(event));
	200
	201	hisi_pmu->ops->enable_counter_int(hisi_pmu, hwc);
	202	hisi_pmu->ops->enable_counter(hisi_pmu, hwc);
	203	}
	204
	205	/*
	206	* Disable counter and interrupt.
	207	*/
	208	static void hisi_uncore_pmu_disable_event(struct perf_event *event)
	209	{
	210	struct hisi_pmu *hisi_pmu = to_hisi_pmu(event->pmu);
	211	struct hw_perf_event *hwc = &event->hw;
	212
	213	hisi_pmu->ops->disable_counter(hisi_pmu, hwc);
	214	hisi_pmu->ops->disable_counter_int(hisi_pmu, hwc);
	215	}
	216
	217	void hisi_uncore_pmu_set_event_period(struct perf_event *event)
	218	{
	219	struct hisi_pmu *hisi_pmu = to_hisi_pmu(event->pmu);
	220	struct hw_perf_event *hwc = &event->hw;
	221
	222	/*
	223	* The HiSilicon PMU counters support 32 bits or 48 bits, depending on
	224	* the PMU. We reduce it to 2^(counter_bits - 1) to account for the
	225	* extreme interrupt latency. So we could hopefully handle the overflow
	226	* interrupt before another 2^(counter_bits - 1) events occur and the
	227	* counter overtakes its previous value.
	228	*/
	229	u64 val = BIT_ULL(hisi_pmu->counter_bits - 1);
	230
	231	local64_set(&hwc->prev_count, val);
	232	/* Write start value to the hardware event counter */
	233	hisi_pmu->ops->write_counter(hisi_pmu, hwc, val);
	234	}
	235
	236	void hisi_uncore_pmu_event_update(struct perf_event *event)
	237	{
	238	struct hisi_pmu *hisi_pmu = to_hisi_pmu(event->pmu);
	239	struct hw_perf_event *hwc = &event->hw;
	240	u64 delta, prev_raw_count, new_raw_count;
	241
	242	do {
	243	/* Read the count from the counter register */
	244	new_raw_count = hisi_pmu->ops->read_counter(hisi_pmu, hwc);
	245	prev_raw_count = local64_read(&hwc->prev_count);
	246	} while (local64_cmpxchg(&hwc->prev_count, prev_raw_count,
	247	new_raw_count) != prev_raw_count);
	248	/*
	249	* compute the delta
	250	*/
	251	delta = (new_raw_count - prev_raw_count) &
	252	HISI_MAX_PERIOD(hisi_pmu->counter_bits);
	253	local64_add(delta, &event->count);
	254	}
	255
	256	void hisi_uncore_pmu_start(struct perf_event *event, int flags)
	257	{
	258	struct hisi_pmu *hisi_pmu = to_hisi_pmu(event->pmu);
	259	struct hw_perf_event *hwc = &event->hw;
	260
	261	if (WARN_ON_ONCE(!(hwc->state & PERF_HES_STOPPED)))
	262	return;
	263
	264	WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE));
	265	hwc->state = 0;
	266	hisi_uncore_pmu_set_event_period(event);
	267
	268	if (flags & PERF_EF_RELOAD) {
	269	u64 prev_raw_count = local64_read(&hwc->prev_count);
	270
	271	hisi_pmu->ops->write_counter(hisi_pmu, hwc, prev_raw_count);
	272	}
	273
	274	hisi_uncore_pmu_enable_event(event);
	275	perf_event_update_userpage(event);
	276	}
	277
	278	void hisi_uncore_pmu_stop(struct perf_event *event, int flags)
	279	{
	280	struct hw_perf_event *hwc = &event->hw;
	281
	282	hisi_uncore_pmu_disable_event(event);
	283	WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED);
	284	hwc->state \|= PERF_HES_STOPPED;
	285
	286	if (hwc->state & PERF_HES_UPTODATE)
	287	return;
	288
	289	/* Read hardware counter and update the perf counter statistics */
	290	hisi_uncore_pmu_event_update(event);
	291	hwc->state \|= PERF_HES_UPTODATE;
	292	}
	293
	294	int hisi_uncore_pmu_add(struct perf_event *event, int flags)
	295	{
	296	struct hisi_pmu *hisi_pmu = to_hisi_pmu(event->pmu);
	297	struct hw_perf_event *hwc = &event->hw;
	298	int idx;
	299
	300	hwc->state = PERF_HES_STOPPED \| PERF_HES_UPTODATE;
	301
	302	/* Get an available counter index for counting */
	303	idx = hisi_pmu->ops->get_event_idx(event);
	304	if (idx < 0)
	305	return idx;
	306
	307	event->hw.idx = idx;
	308	hisi_pmu->pmu_events.hw_events[idx] = event;
	309
	310	if (flags & PERF_EF_START)
	311	hisi_uncore_pmu_start(event, PERF_EF_RELOAD);
	312
	313	return 0;
	314	}
	315
	316	void hisi_uncore_pmu_del(struct perf_event *event, int flags)
	317	{
	318	struct hisi_pmu *hisi_pmu = to_hisi_pmu(event->pmu);
	319	struct hw_perf_event *hwc = &event->hw;
	320
	321	hisi_uncore_pmu_stop(event, PERF_EF_UPDATE);
	322	hisi_uncore_pmu_clear_event_idx(hisi_pmu, hwc->idx);
	323	perf_event_update_userpage(event);
	324	hisi_pmu->pmu_events.hw_events[hwc->idx] = NULL;
	325	}
	326
	327	void hisi_uncore_pmu_read(struct perf_event *event)
	328	{
	329	/* Read hardware counter and update the perf counter statistics */
	330	hisi_uncore_pmu_event_update(event);
	331	}
	332
	333	void hisi_uncore_pmu_enable(struct pmu *pmu)
	334	{
	335	struct hisi_pmu *hisi_pmu = to_hisi_pmu(pmu);
	336	int enabled = bitmap_weight(hisi_pmu->pmu_events.used_mask,
	337	hisi_pmu->num_counters);
	338
	339	if (!enabled)
	340	return;
	341
	342	hisi_pmu->ops->start_counters(hisi_pmu);
	343	}
	344
	345	void hisi_uncore_pmu_disable(struct pmu *pmu)
	346	{
	347	struct hisi_pmu *hisi_pmu = to_hisi_pmu(pmu);
	348
	349	hisi_pmu->ops->stop_counters(hisi_pmu);
	350	}
	351
	352	/*
	353	* Read Super CPU cluster and CPU cluster ID from MPIDR_EL1.
	354	* If multi-threading is supported, SCCL_ID is in MPIDR[aff3] and CCL_ID
	355	* is in MPIDR[aff2]; if not, SCCL_ID is in MPIDR[aff2] and CCL_ID is
	356	* in MPIDR[aff1]. If this changes in future, this shall be updated.
	357	*/
	358	static void hisi_read_sccl_and_ccl_id(int sccl_id, int ccl_id)
	359	{
	360	u64 mpidr = read_cpuid_mpidr();
	361
	362	if (mpidr & MPIDR_MT_BITMASK) {
	363	if (sccl_id)
	364	*sccl_id = MPIDR_AFFINITY_LEVEL(mpidr, 3);
	365	if (ccl_id)
	366	*ccl_id = MPIDR_AFFINITY_LEVEL(mpidr, 2);
	367	} else {
	368	if (sccl_id)
	369	*sccl_id = MPIDR_AFFINITY_LEVEL(mpidr, 2);
	370	if (ccl_id)
	371	*ccl_id = MPIDR_AFFINITY_LEVEL(mpidr, 1);
	372	}
	373	}
	374
	375	/*
	376	* Check whether the CPU is associated with this uncore PMU
	377	*/
	378	static bool hisi_pmu_cpu_is_associated_pmu(struct hisi_pmu *hisi_pmu)
	379	{
	380	int sccl_id, ccl_id;
	381
	382	if (hisi_pmu->ccl_id == -1) {
	383	/* If CCL_ID is -1, the PMU only shares the same SCCL */
	384	hisi_read_sccl_and_ccl_id(&sccl_id, NULL);
	385
	386	return sccl_id == hisi_pmu->sccl_id;
	387	}
	388
	389	hisi_read_sccl_and_ccl_id(&sccl_id, &ccl_id);
	390
	391	return sccl_id == hisi_pmu->sccl_id && ccl_id == hisi_pmu->ccl_id;
	392	}
	393
	394	int hisi_uncore_pmu_online_cpu(unsigned int cpu, struct hlist_node *node)
	395	{
	396	struct hisi_pmu *hisi_pmu = hlist_entry_safe(node, struct hisi_pmu,
	397	node);
	398
	399	if (!hisi_pmu_cpu_is_associated_pmu(hisi_pmu))
	400	return 0;
	401
	402	cpumask_set_cpu(cpu, &hisi_pmu->associated_cpus);
	403
	404	/* If another CPU is already managing this PMU, simply return. */
	405	if (hisi_pmu->on_cpu != -1)
	406	return 0;
	407
	408	/* Use this CPU in cpumask for event counting */
	409	hisi_pmu->on_cpu = cpu;
	410
	411	/* Overflow interrupt also should use the same CPU */
	412	WARN_ON(irq_set_affinity(hisi_pmu->irq, cpumask_of(cpu)));
	413
	414	return 0;
	415	}
	416
	417	int hisi_uncore_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node)
	418	{
	419	struct hisi_pmu *hisi_pmu = hlist_entry_safe(node, struct hisi_pmu,
	420	node);
	421	cpumask_t pmu_online_cpus;
	422	unsigned int target;
	423
	424	if (!cpumask_test_and_clear_cpu(cpu, &hisi_pmu->associated_cpus))
	425	return 0;
	426
	427	/* Nothing to do if this CPU doesn't own the PMU */
	428	if (hisi_pmu->on_cpu != cpu)
	429	return 0;
	430
	431	/* Give up ownership of the PMU */
	432	hisi_pmu->on_cpu = -1;
	433
	434	/* Choose a new CPU to migrate ownership of the PMU to */
	435	cpumask_and(&pmu_online_cpus, &hisi_pmu->associated_cpus,
	436	cpu_online_mask);
	437	target = cpumask_any_but(&pmu_online_cpus, cpu);
	438	if (target >= nr_cpu_ids)
	439	return 0;
	440
	441	perf_pmu_migrate_context(&hisi_pmu->pmu, cpu, target);
	442	/* Use this CPU for event counting */
	443	hisi_pmu->on_cpu = target;
	444	WARN_ON(irq_set_affinity(hisi_pmu->irq, cpumask_of(target)));
	445
	446	return 0;
	447	}

drivers/perf/hisilicon/hisi_uncore_pmu.h+102−0

@@ -0,0 +1,102 @@
	1	/*
	2	* HiSilicon SoC Hardware event counters support
	3	*
	4	* Copyright (C) 2017 Hisilicon Limited
	5	* Author: Anurup M <anurup.m@huawei.com>
	6	* Shaokun Zhang <zhangshaokun@hisilicon.com>
	7	*
	8	* This code is based on the uncore PMUs like arm-cci and arm-ccn.
	9	*
	10	* This program is free software; you can redistribute it and/or modify
	11	* it under the terms of the GNU General Public License version 2 as
	12	* published by the Free Software Foundation.
	13	*/
	14	#ifndef __HISI_UNCORE_PMU_H__
	15	#define __HISI_UNCORE_PMU_H__
	16
	17	#include <linux/cpumask.h>
	18	#include <linux/device.h>
	19	#include <linux/kernel.h>
	20	#include <linux/perf_event.h>
	21	#include <linux/types.h>
	22
	23	#undef pr_fmt
	24	#define pr_fmt(fmt) "hisi_pmu: " fmt
	25
	26	#define HISI_MAX_COUNTERS 0x10
	27	#define to_hisi_pmu(p) (container_of(p, struct hisi_pmu, pmu))
	28
	29	#define HISI_PMU_ATTR(_name, _func, _config) \
	30	(&((struct dev_ext_attribute[]) { \
	31	{ __ATTR(_name, 0444, _func, NULL), (void *)_config } \
	32	})[0].attr.attr)
	33
	34	#define HISI_PMU_FORMAT_ATTR(_name, _config) \
	35	HISI_PMU_ATTR(_name, hisi_format_sysfs_show, (void *)_config)
	36	#define HISI_PMU_EVENT_ATTR(_name, _config) \
	37	HISI_PMU_ATTR(_name, hisi_event_sysfs_show, (unsigned long)_config)
	38
	39	struct hisi_pmu;
	40
	41	struct hisi_uncore_ops {
	42	void (write_evtype)(struct hisi_pmu , int, u32);
	43	int (get_event_idx)(struct perf_event );
	44	u64 (read_counter)(struct hisi_pmu , struct hw_perf_event *);
	45	void (write_counter)(struct hisi_pmu , struct hw_perf_event *, u64);
	46	void (enable_counter)(struct hisi_pmu , struct hw_perf_event *);
	47	void (disable_counter)(struct hisi_pmu , struct hw_perf_event *);
	48	void (enable_counter_int)(struct hisi_pmu , struct hw_perf_event *);
	49	void (disable_counter_int)(struct hisi_pmu , struct hw_perf_event *);
	50	void (start_counters)(struct hisi_pmu );
	51	void (stop_counters)(struct hisi_pmu );
	52	};
	53
	54	struct hisi_pmu_hwevents {
	55	struct perf_event *hw_events[HISI_MAX_COUNTERS];
	56	DECLARE_BITMAP(used_mask, HISI_MAX_COUNTERS);
	57	};
	58
	59	/* Generic pmu struct for different pmu types */
	60	struct hisi_pmu {
	61	struct pmu pmu;
	62	const struct hisi_uncore_ops *ops;
	63	struct hisi_pmu_hwevents pmu_events;
	64	/* associated_cpus: All CPUs associated with the PMU */
	65	cpumask_t associated_cpus;
	66	/* CPU used for counting */
	67	int on_cpu;
	68	int irq;
	69	struct device *dev;
	70	struct hlist_node node;
	71	int sccl_id;
	72	int ccl_id;
	73	void __iomem *base;
	74	/* the ID of the PMU modules */
	75	u32 index_id;
	76	int num_counters;
	77	int counter_bits;
	78	/* check event code range */
	79	int check_event;
	80	};
	81
	82	int hisi_uncore_pmu_counter_valid(struct hisi_pmu *hisi_pmu, int idx);
	83	int hisi_uncore_pmu_get_event_idx(struct perf_event *event);
	84	void hisi_uncore_pmu_read(struct perf_event *event);
	85	int hisi_uncore_pmu_add(struct perf_event *event, int flags);
	86	void hisi_uncore_pmu_del(struct perf_event *event, int flags);
	87	void hisi_uncore_pmu_start(struct perf_event *event, int flags);
	88	void hisi_uncore_pmu_stop(struct perf_event *event, int flags);
	89	void hisi_uncore_pmu_set_event_period(struct perf_event *event);
	90	void hisi_uncore_pmu_event_update(struct perf_event *event);
	91	int hisi_uncore_pmu_event_init(struct perf_event *event);
	92	void hisi_uncore_pmu_enable(struct pmu *pmu);
	93	void hisi_uncore_pmu_disable(struct pmu *pmu);
	94	ssize_t hisi_event_sysfs_show(struct device *dev,
	95	struct device_attribute attr, char buf);
	96	ssize_t hisi_format_sysfs_show(struct device *dev,
	97	struct device_attribute attr, char buf);
	98	ssize_t hisi_cpumask_sysfs_show(struct device *dev,
	99	struct device_attribute attr, char buf);
	100	int hisi_uncore_pmu_online_cpu(unsigned int cpu, struct hlist_node *node);
	101	int hisi_uncore_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node);
	102	#endif /* __HISI_UNCORE_PMU_H__ */

drivers/perf/qcom_l2_pmu.c+54−0

@@ -92,6 +92,21 @@ · 91 unmodified lines
92		92
93	#define reg_idx(reg, i) (((i) * IA_L2_REG_OFFSET) + reg##_BASE)	93	#define reg_idx(reg, i) (((i) * IA_L2_REG_OFFSET) + reg##_BASE)
94		94
		95	/*
		96	* Events
		97	*/
		98	#define L2_EVENT_CYCLES 0xfe
		99	#define L2_EVENT_DCACHE_OPS 0x400
		100	#define L2_EVENT_ICACHE_OPS 0x401
		101	#define L2_EVENT_TLBI 0x402
		102	#define L2_EVENT_BARRIERS 0x403
		103	#define L2_EVENT_TOTAL_READS 0x405
		104	#define L2_EVENT_TOTAL_WRITES 0x406
		105	#define L2_EVENT_TOTAL_REQUESTS 0x407
		106	#define L2_EVENT_LDREX 0x420
		107	#define L2_EVENT_STREX 0x421
		108	#define L2_EVENT_CLREX 0x422
		109
95	static DEFINE_RAW_SPINLOCK(l2_access_lock);	110	static DEFINE_RAW_SPINLOCK(l2_access_lock);
96		111
97	/**	112	/**
@@ -700,9 +715,12 @@ static struct attribute_group l2_cache_pmu_cpumask_group = { · 602 unmodified lines
700	/* CCG format for perf RAW codes. */	715	/* CCG format for perf RAW codes. */
701	PMU_FORMAT_ATTR(l2_code, "config:4-11");	716	PMU_FORMAT_ATTR(l2_code, "config:4-11");
702	PMU_FORMAT_ATTR(l2_group, "config:0-3");	717	PMU_FORMAT_ATTR(l2_group, "config:0-3");
		718	PMU_FORMAT_ATTR(event, "config:0-11");
		719
703	static struct attribute *l2_cache_pmu_formats[] = {	720	static struct attribute *l2_cache_pmu_formats[] = {
704	&format_attr_l2_code.attr,	721	&format_attr_l2_code.attr,
705	&format_attr_l2_group.attr,	722	&format_attr_l2_group.attr,
		723	&format_attr_event.attr,
706	NULL,	724	NULL,
707	};	725	};
708		726
@@ -711,9 +729,45 @@ static struct attribute_group l2_cache_pmu_format_group = { · 2 unmodified lines
711	.attrs = l2_cache_pmu_formats,	729	.attrs = l2_cache_pmu_formats,
712	};	730	};
713		731
		732	static ssize_t l2cache_pmu_event_show(struct device *dev,
		733	struct device_attribute attr, char page)
		734	{
		735	struct perf_pmu_events_attr *pmu_attr;
		736
		737	pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr);
		738	return sprintf(page, "event=0x%02llx\n", pmu_attr->id);
		739	}
		740
		741	#define L2CACHE_EVENT_ATTR(_name, _id) \
		742	(&((struct perf_pmu_events_attr[]) { \
		743	{ .attr = __ATTR(_name, 0444, l2cache_pmu_event_show, NULL), \
		744	.id = _id, } \
		745	})[0].attr.attr)
		746
		747	static struct attribute *l2_cache_pmu_events[] = {
		748	L2CACHE_EVENT_ATTR(cycles, L2_EVENT_CYCLES),
		749	L2CACHE_EVENT_ATTR(dcache-ops, L2_EVENT_DCACHE_OPS),
		750	L2CACHE_EVENT_ATTR(icache-ops, L2_EVENT_ICACHE_OPS),
		751	L2CACHE_EVENT_ATTR(tlbi, L2_EVENT_TLBI),
		752	L2CACHE_EVENT_ATTR(barriers, L2_EVENT_BARRIERS),
		753	L2CACHE_EVENT_ATTR(total-reads, L2_EVENT_TOTAL_READS),
		754	L2CACHE_EVENT_ATTR(total-writes, L2_EVENT_TOTAL_WRITES),
		755	L2CACHE_EVENT_ATTR(total-requests, L2_EVENT_TOTAL_REQUESTS),
		756	L2CACHE_EVENT_ATTR(ldrex, L2_EVENT_LDREX),
		757	L2CACHE_EVENT_ATTR(strex, L2_EVENT_STREX),
		758	L2CACHE_EVENT_ATTR(clrex, L2_EVENT_CLREX),
		759	NULL
		760	};
		761
		762	static struct attribute_group l2_cache_pmu_events_group = {
		763	.name = "events",
		764	.attrs = l2_cache_pmu_events,
		765	};
		766
714	static const struct attribute_group *l2_cache_pmu_attr_grps[] = {	767	static const struct attribute_group *l2_cache_pmu_attr_grps[] = {
715	&l2_cache_pmu_format_group,	768	&l2_cache_pmu_format_group,
716	&l2_cache_pmu_cpumask_group,	769	&l2_cache_pmu_cpumask_group,
		770	&l2_cache_pmu_events_group,
717	NULL,	771	NULL,
718	};	772	};
719		773

fs/binfmt_elf.c+5−5

@@ -1699,7 +1699,7 @@ static int fill_thread_core_info(struct elf_thread_core_info *t, · 1698 unmodified lines
1699	long signr, size_t *total)	1699	long signr, size_t *total)
1700	{	1700	{
1701	unsigned int i;	1701	unsigned int i;
1702	unsigned int regset_size = view->regsets[0].n * view->regsets[0].size;	1702	unsigned int regset0_size = regset_size(t->task, &view->regsets[0]);
1703		1703
1704	/*	1704	/*
1705	* NT_PRSTATUS is the one special case, because the regset data	1705	* NT_PRSTATUS is the one special case, because the regset data
@@ -1708,11 +1708,11 @@ static int fill_thread_core_info(struct elf_thread_core_info *t, · 2 unmodified lines
1708	* We assume that regset 0 is NT_PRSTATUS.	1708	* We assume that regset 0 is NT_PRSTATUS.
1709	*/	1709	*/
1710	fill_prstatus(&t->prstatus, t->task, signr);	1710	fill_prstatus(&t->prstatus, t->task, signr);
1711	(void) view->regsets[0].get(t->task, &view->regsets[0], 0, regset_size,	1711	(void) view->regsets[0].get(t->task, &view->regsets[0], 0, regset0_size,
1712	&t->prstatus.pr_reg, NULL);	1712	&t->prstatus.pr_reg, NULL);
1713		1713
1714	fill_note(&t->notes[0], "CORE", NT_PRSTATUS,	1714	fill_note(&t->notes[0], "CORE", NT_PRSTATUS,
1715	PRSTATUS_SIZE(t->prstatus, regset_size), &t->prstatus);	1715	PRSTATUS_SIZE(t->prstatus, regset0_size), &t->prstatus);
1716	*total += notesize(&t->notes[0]);	1716	*total += notesize(&t->notes[0]);
1717		1717
1718	do_thread_regset_writeback(t->task, &view->regsets[0]);	1718	do_thread_regset_writeback(t->task, &view->regsets[0]);
@@ -1728,7 +1728,7 @@ static int fill_thread_core_info(struct elf_thread_core_info *t, · 9 unmodified lines
1728	if (regset->core_note_type && regset->get &&	1728	if (regset->core_note_type && regset->get &&
1729	(!regset->active \|\| regset->active(t->task, regset))) {	1729	(!regset->active \|\| regset->active(t->task, regset))) {
1730	int ret;	1730	int ret;
1731	size_t size = regset->n * regset->size;	1731	size_t size = regset_size(t->task, regset);
1732	void *data = kmalloc(size, GFP_KERNEL);	1732	void *data = kmalloc(size, GFP_KERNEL);
1733	if (unlikely(!data))	1733	if (unlikely(!data))
1734	return 0;	1734	return 0;
@@ -1743,7 +1743,7 @@ static int fill_thread_core_info(struct elf_thread_core_info *t, · 8 unmodified lines
1743	size, data);	1743	size, data);
1744	else {	1744	else {
1745	SET_PR_FPVALID(&t->prstatus,	1745	SET_PR_FPVALID(&t->prstatus,
1746	1, regset_size);	1746	1, regset0_size);
1747	fill_note(&t->notes[i], "CORE",	1747	fill_note(&t->notes[i], "CORE",
1748	NT_PRFPREG, size, data);	1748	NT_PRFPREG, size, data);
1749	}	1749	}

include/clocksource/arm_arch_timer.h+9−1

 #define ARCH_TIMER_USR_VT_ACCESS_EN	(1 << 8) /* virtual timer registers */
 #define ARCH_TIMER_USR_PT_ACCESS_EN	(1 << 9) /* physical timer registers */
-#define ARCH_TIMER_EVT_STREAM_FREQ	10000	/* 100us */
+#define ARCH_TIMER_EVT_STREAM_PERIOD_US	100
+#define ARCH_TIMER_EVT_STREAM_FREQ				\
+	(USEC_PER_SEC / ARCH_TIMER_EVT_STREAM_PERIOD_US)
 struct arch_timer_kvm_info {
 	struct timecounter timecounter;
 extern u32 arch_timer_get_rate(void);
 extern u64 (*arch_timer_read_counter)(void);
 extern struct arch_timer_kvm_info *arch_timer_get_kvm_info(void);
+extern bool arch_timer_evtstrm_available(void);
 #else
 	return 0;
 }
+static inline bool arch_timer_evtstrm_available(void)
+{
+	return false;
+}
 #endif
 #endif

include/linux/acpi_iort.h+2−2

@@ -49,8 +49,8 @@ static inline void acpi_configure_pmsi_domain(struct device *dev) { } · 48 unmodified lines
49	/* IOMMU interface */	49	/* IOMMU interface */
50	static inline void iort_dma_setup(struct device dev, u64 dma_addr,	50	static inline void iort_dma_setup(struct device dev, u64 dma_addr,
51	u64 *size) { }	51	u64 *size) { }
52	static inline	52	static inline const struct iommu_ops *iort_iommu_configure(
53	const struct iommu_ops iort_iommu_configure(struct device dev)	53	struct device *dev)
54	{ return NULL; }	54	{ return NULL; }
55	#endif	55	#endif
56		56

include/linux/cpuhotplug.h+3−0

 	CPUHP_AP_PERF_S390_SF_ONLINE,
 	CPUHP_AP_PERF_ARM_CCI_ONLINE,
 	CPUHP_AP_PERF_ARM_CCN_ONLINE,
+	CPUHP_AP_PERF_ARM_HISI_DDRC_ONLINE,
+	CPUHP_AP_PERF_ARM_HISI_HHA_ONLINE,
+	CPUHP_AP_PERF_ARM_HISI_L3_ONLINE,
 	CPUHP_AP_PERF_ARM_L2X0_ONLINE,
 	CPUHP_AP_PERF_ARM_QCOM_L2_ONLINE,
 	CPUHP_AP_PERF_ARM_QCOM_L3_ONLINE,

include/linux/irqdesc.h+8−0

@@ -246,6 +246,14 @@ static inline int irq_is_percpu(unsigned int irq) · 245 unmodified lines
246	return desc->status_use_accessors & IRQ_PER_CPU;	246	return desc->status_use_accessors & IRQ_PER_CPU;
247	}	247	}
248		248
		249	static inline int irq_is_percpu_devid(unsigned int irq)
		250	{
		251	struct irq_desc *desc;
		252
		253	desc = irq_to_desc(irq);
		254	return desc->status_use_accessors & IRQ_PER_CPU_DEVID;
		255	}
		256
249	static inline void	257	static inline void
250	irq_set_lockdep_class(unsigned int irq, struct lock_class_key *class)	258	irq_set_lockdep_class(unsigned int irq, struct lock_class_key *class)
251	{	259	{

include/linux/regset.h+60−7

@@ -106,6 +106,28 @@ typedef int user_regset_writeback_fn(struct task_struct *target, · 105 unmodified lines
106	const struct user_regset *regset,	106	const struct user_regset *regset,
107	int immediate);	107	int immediate);
108		108
		109	/**
		110	* user_regset_get_size_fn - type of @get_size function in &struct user_regset
		111	* @target: thread being examined
		112	* @regset: regset being examined
		113	*
		114	* This call is optional; usually the pointer is %NULL.
		115	*
		116	* When provided, this function must return the current size of regset
		117	* data, as observed by the @get function in &struct user_regset. The
		118	* value returned must be a multiple of @size. The returned size is
		119	* required to be valid only until the next time (if any) @regset is
		120	* modified for @target.
		121	*
		122	* This function is intended for dynamically sized regsets. A regset
		123	* that is statically sized does not need to implement it.
		124	*
		125	* This function should not be called directly: instead, callers should
		126	* call regset_size() to determine the current size of a regset.
		127	*/
		128	typedef unsigned int user_regset_get_size_fn(struct task_struct *target,
		129	const struct user_regset *regset);
		130
109	/**	131	/**
110	* struct user_regset - accessible thread CPU state	132	* struct user_regset - accessible thread CPU state
111	* @n: Number of slots (registers).	133	* @n: Number of slots (registers).
@@ -117,19 +139,33 @@ typedef int user_regset_writeback_fn(struct task_struct *target, · 5 unmodified lines
117	* @set: Function to store values.	139	* @set: Function to store values.
118	* @active: Function to report if regset is active, or %NULL.	140	* @active: Function to report if regset is active, or %NULL.
119	* @writeback: Function to write data back to user memory, or %NULL.	141	* @writeback: Function to write data back to user memory, or %NULL.
		142	* @get_size: Function to return the regset's size, or %NULL.
120	*	143	*
121	* This data structure describes a machine resource we call a register set.	144	* This data structure describes a machine resource we call a register set.
122	* This is part of the state of an individual thread, not necessarily	145	* This is part of the state of an individual thread, not necessarily
123	* actual CPU registers per se. A register set consists of a number of	146	* actual CPU registers per se. A register set consists of a number of
124	* similar slots, given by @n. Each slot is @size bytes, and aligned to	147	* similar slots, given by @n. Each slot is @size bytes, and aligned to
125	* @align bytes (which is at least @size).	148	* @align bytes (which is at least @size). For dynamically-sized
		149	* regsets, @n must contain the maximum possible number of slots for the
		150	* regset, and @get_size must point to a function that returns the
		151	* current regset size.
126	*	152	*
127	* These functions must be called only on the current thread or on a	153	* Callers that need to know only the current size of the regset and do
128	* thread that is in %TASK_STOPPED or %TASK_TRACED state, that we are	154	* not care about its internal structure should call regset_size()
129	* guaranteed will not be woken up and return to user mode, and that we	155	* instead of inspecting @n or calling @get_size.
130	* have called wait_task_inactive() on. (The target thread always might	156	*
131	* wake up for SIGKILL while these functions are working, in which case	157	* For backward compatibility, the @get and @set methods must pad to, or
132	* that thread's user_regset state might be scrambled.)	158	* accept, @n * @size bytes, even if the current regset size is smaller.
		159	* The precise semantics of these operations depend on the regset being
		160	* accessed.
		161	*
		162	* The functions to which &struct user_regset members point must be
		163	* called only on the current thread or on a thread that is in
		164	* %TASK_STOPPED or %TASK_TRACED state, that we are guaranteed will not
		165	* be woken up and return to user mode, and that we have called
		166	* wait_task_inactive() on. (The target thread always might wake up for
		167	* SIGKILL while these functions are working, in which case that
		168	* thread's user_regset state might be scrambled.)
133	*	169	*
134	* The @pos argument must be aligned according to @align; the @count	170	* The @pos argument must be aligned according to @align; the @count
135	* argument must be a multiple of @size. These functions are not	171	* argument must be a multiple of @size. These functions are not
@@ -156,6 +192,7 @@ struct user_regset { · 20 unmodified lines
156	user_regset_set_fn *set;	192	user_regset_set_fn *set;
157	user_regset_active_fn *active;	193	user_regset_active_fn *active;
158	user_regset_writeback_fn *writeback;	194	user_regset_writeback_fn *writeback;
		195	user_regset_get_size_fn *get_size;
159	unsigned int n;	196	unsigned int n;
160	unsigned int size;	197	unsigned int size;
161	unsigned int align;	198	unsigned int align;
@@ -371,5 +408,21 @@ static inline int copy_regset_from_user(struct task_struct *target, · 209 unmodified lines
371	return regset->set(target, regset, offset, size, NULL, data);	408	return regset->set(target, regset, offset, size, NULL, data);
372	}	409	}
373		410
		411	/**
		412	* regset_size - determine the current size of a regset
		413	* @target: thread to be examined
		414	* @regset: regset to be examined
		415	*
		416	* Note that the returned size is valid only until the next time
		417	* (if any) @regset is modified for @target.
		418	*/
		419	static inline unsigned int regset_size(struct task_struct *target,
		420	const struct user_regset *regset)
		421	{
		422	if (!regset->get_size)
		423	return regset->n * regset->size;
		424	else
		425	return regset->get_size(target, regset);
		426	}
374		427
375	#endif /* <linux/regset.h> */	428	#endif /* <linux/regset.h> */

include/uapi/linux/elf.h+1−0

@@ -418,6 +418,7 @@ typedef struct elf64_shdr { · 417 unmodified lines
418	#define NT_ARM_HW_BREAK 0x402 /* ARM hardware breakpoint registers */	418	#define NT_ARM_HW_BREAK 0x402 /* ARM hardware breakpoint registers */
419	#define NT_ARM_HW_WATCH 0x403 /* ARM hardware watchpoint registers */	419	#define NT_ARM_HW_WATCH 0x403 /* ARM hardware watchpoint registers */
420	#define NT_ARM_SYSTEM_CALL 0x404 /* ARM system call number */	420	#define NT_ARM_SYSTEM_CALL 0x404 /* ARM system call number */
		421	#define NT_ARM_SVE 0x405 /* ARM Scalable Vector Extension registers */
421	#define NT_METAG_CBUF 0x500 /* Metag catch buffer registers */	422	#define NT_METAG_CBUF 0x500 /* Metag catch buffer registers */
422	#define NT_METAG_RPIPE 0x501 /* Metag read pipeline state */	423	#define NT_METAG_RPIPE 0x501 /* Metag read pipeline state */
423	#define NT_METAG_TLS 0x502 /* Metag TLS pointer */	424	#define NT_METAG_TLS 0x502 /* Metag TLS pointer */

include/uapi/linux/perf_event.h+1−0

@@ -942,6 +942,7 @@ enum perf_callchain_context { · 941 unmodified lines
942	#define PERF_AUX_FLAG_TRUNCATED 0x01 /* record was truncated to fit */	942	#define PERF_AUX_FLAG_TRUNCATED 0x01 /* record was truncated to fit */
943	#define PERF_AUX_FLAG_OVERWRITE 0x02 /* snapshot from overwrite mode */	943	#define PERF_AUX_FLAG_OVERWRITE 0x02 /* snapshot from overwrite mode */
944	#define PERF_AUX_FLAG_PARTIAL 0x04 /* record contains gaps */	944	#define PERF_AUX_FLAG_PARTIAL 0x04 /* record contains gaps */
		945	#define PERF_AUX_FLAG_COLLISION 0x08 /* sample collided with another */
945		946
946	#define PERF_FLAG_FD_NO_GROUP (1UL << 0)	947	#define PERF_FLAG_FD_NO_GROUP (1UL << 0)
947	#define PERF_FLAG_FD_OUTPUT (1UL << 1)	948	#define PERF_FLAG_FD_OUTPUT (1UL << 1)

include/uapi/linux/prctl.h+9−0

@@ -198,4 +198,13 @@ struct prctl_mm_map { · 197 unmodified lines
198	# define PR_CAP_AMBIENT_LOWER 3	198	# define PR_CAP_AMBIENT_LOWER 3
199	# define PR_CAP_AMBIENT_CLEAR_ALL 4	199	# define PR_CAP_AMBIENT_CLEAR_ALL 4
200		200
		201	/* arm64 Scalable Vector Extension controls */
		202	/* Flag values must be kept in sync with ptrace NT_ARM_SVE interface */
		203	#define PR_SVE_SET_VL 50 /* set task vector length */
		204	# define PR_SVE_SET_VL_ONEXEC (1 << 18) /* defer effect until exec */
		205	#define PR_SVE_GET_VL 51 /* get task vector length */
		206	/* Bits common to PR_SVE_SET_VL and PR_SVE_GET_VL */
		207	# define PR_SVE_VL_LEN_MASK 0xffff
		208	# define PR_SVE_VL_INHERIT (1 << 17) /* inherit across exec */
		209
201	#endif /* _LINUX_PRCTL_H */	210	#endif /* _LINUX_PRCTL_H */

kernel/events/ring_buffer.c+4−0

 	return NULL;
 }
+EXPORT_SYMBOL_GPL(perf_aux_output_begin);
 static bool __always_inline rb_need_aux_wakeup(struct ring_buffer *rb)
 {
 	rb_free_aux(rb);
 	ring_buffer_put(rb);
 }
+EXPORT_SYMBOL_GPL(perf_aux_output_end);
 /*
  * Skip over a given number of bytes in the AUX buffer, due to, for example,
 	return 0;
 }
+EXPORT_SYMBOL_GPL(perf_aux_output_skip);
 void *perf_get_aux(struct perf_output_handle *handle)
 {
 	return handle->rb->aux_priv;
 }
+EXPORT_SYMBOL_GPL(perf_get_aux);
 #define PERF_AUX_GFP	(GFP_KERNEL | __GFP_ZERO | __GFP_NOWARN | __GFP_NORETRY)

kernel/irq/irqdesc.c+1−0

@@ -862,6 +862,7 @@ int irq_get_percpu_devid_partition(unsigned int irq, struct cpumask *affinity) · 861 unmodified lines
862		862
863	return 0;	863	return 0;
864	}	864	}
		865	EXPORT_SYMBOL_GPL(irq_get_percpu_devid_partition);
865		866
866	void kstat_incr_irq_this_cpu(unsigned int irq)	867	void kstat_incr_irq_this_cpu(unsigned int irq)
867	{	868	{

kernel/sys.c+12−0

 #ifndef SET_FP_MODE
 # define SET_FP_MODE(a,b)	(-EINVAL)
 #endif
+#ifndef SVE_SET_VL
+# define SVE_SET_VL(a)		(-EINVAL)
+#endif
+#ifndef SVE_GET_VL
+# define SVE_GET_VL()		(-EINVAL)
+#endif
 /*
  * this is where the system-wide overflow UID and GID are defined, for
 	case PR_GET_FP_MODE:
 		error = GET_FP_MODE(me);
 		break;
+	case PR_SVE_SET_VL:
+		error = SVE_SET_VL(arg2);
+		break;
+	case PR_SVE_GET_VL:
+		error = SVE_GET_VL();
+		break;
 	default:
 		error = -EINVAL;
 		break;

virt/kvm/arm/arm.c+3−0

@@ -652,6 +652,9 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu vcpu, struct kvm_run run) · 651 unmodified lines
652	*/	652	*/
653	preempt_disable();	653	preempt_disable();
654		654
		655	/* Flush FP/SIMD state that can't survive guest entry/exit */
		656	kvm_fpsimd_flush_cpu_state();
		657
655	kvm_pmu_flush_hwstate(vcpu);	658	kvm_pmu_flush_hwstate(vcpu);
656		659
657	kvm_timer_flush_hwstate(vcpu);	660	kvm_timer_flush_hwstate(vcpu);

Merge tag 'arm64-upstream' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code