/* SPDX-License-Identifier: GPL-2.0-only *//* * Based on arch/arm/include/asm/assembler.h, arch/arm/mm/proc-macros.S * * Copyright (C) 1996-2000 Russell King * Copyright (C) 2012 ARM Ltd. */#ifndef __ASSEMBLY__#error "Only include this from assembly code"#endif #ifndef __ASM_ASSEMBLER_H#define __ASM_ASSEMBLER_H #include <asm-generic/export.h> #include <asm/asm-offsets.h>#include <asm/alternative.h>#include <asm/asm-bug.h>#include <asm/cpufeature.h>#include <asm/cputype.h>#include <asm/debug-monitors.h>#include <asm/page.h>#include <asm/pgtable-hwdef.h>#include <asm/ptrace.h>#include <asm/thread_info.h> 	/*	 * Provide a wxN alias for each wN register so what we can paste a xN	 * reference after a 'w' to obtain the 32-bit version.	 */	.irp	n,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30	wx\n	.req	w\n	.endr 	.macro save_and_disable_daif, flags	mrs	\flags, daif	msr	daifset, #0xf	.endm 	.macro disable_daif	msr	daifset, #0xf	.endm 	.macro enable_daif	msr	daifclr, #0xf	.endm 	.macro	restore_daif, flags:req	msr	daif, \flags	.endm 	/* IRQ/FIQ are the lowest priority flags, unconditionally unmask the rest. */	.macro enable_da	msr	daifclr, #(8 | 4)	.endm /* * Save/restore interrupts. */	.macro	save_and_disable_irq, flags	mrs	\flags, daif	msr	daifset, #3	.endm 	.macro	restore_irq, flags	msr	daif, \flags	.endm 	.macro	enable_dbg	msr	daifclr, #8	.endm 	.macro	disable_step_tsk, flgs, tmp	tbz	\flgs, #TIF_SINGLESTEP, 9990f	mrs	\tmp, mdscr_el1	bic	\tmp, \tmp, #DBG_MDSCR_SS	msr	mdscr_el1, \tmp	isb	// Synchronise with enable_dbg9990:	.endm 	/* call with daif masked */	.macro	enable_step_tsk, flgs, tmp	tbz	\flgs, #TIF_SINGLESTEP, 9990f	mrs	\tmp, mdscr_el1	orr	\tmp, \tmp, #DBG_MDSCR_SS	msr	mdscr_el1, \tmp9990:	.endm /* * RAS Error Synchronization barrier */	.macro  esb#ifdef CONFIG_ARM64_RAS_EXTN	hint    #16#else	nop#endif	.endm /* * Value prediction barrier */	.macro	csdb	hint	#20	.endm /* * Speculation barrier */	.macro	sbalternative_if_not ARM64_HAS_SB	dsb	nsh	isbalternative_else	SB_BARRIER_INSN	nopalternative_endif	.endm /* * NOP sequence */	.macro	nops, num	.rept	\num	nop	.endr	.endm /* * Emit an entry into the exception table */	.macro		_asm_extable, from, to	.pushsection	__ex_table, "a"	.align		3	.long		(\from - .), (\to - .)	.popsection	.endm #define USER(l, x...)				\9999:	x;					\	_asm_extable	9999b, l /* * Register aliases. */lr	.req	x30		// link register /* * Vector entry */	 .macro	ventry	label	.align	7	b	\label	.endm /* * Select code when configured for BE. */#ifdef CONFIG_CPU_BIG_ENDIAN#define CPU_BE(code...) code#else#define CPU_BE(code...)#endif /* * Select code when configured for LE. */#ifdef CONFIG_CPU_BIG_ENDIAN#define CPU_LE(code...)#else#define CPU_LE(code...) code#endif /* * Define a macro that constructs a 64-bit value by concatenating two * 32-bit registers. Note that on big endian systems the order of the * registers is swapped. */#ifndef CONFIG_CPU_BIG_ENDIAN	.macro	regs_to_64, rd, lbits, hbits#else	.macro	regs_to_64, rd, hbits, lbits#endif	orr	\rd, \lbits, \hbits, lsl #32	.endm /* * Pseudo-ops for PC-relative adr/ldr/str <reg>, <symbol> where * <symbol> is within the range +/- 4 GB of the PC. */	/*	 * @dst: destination register (64 bit wide)	 * @sym: name of the symbol	 */	.macro	adr_l, dst, sym	adrp	\dst, \sym	add	\dst, \dst, :lo12:\sym	.endm 	/*	 * @dst: destination register (32 or 64 bit wide)	 * @sym: name of the symbol	 * @tmp: optional 64-bit scratch register to be used if <dst> is a	 *       32-bit wide register, in which case it cannot be used to hold	 *       the address	 */	.macro	ldr_l, dst, sym, tmp=	.ifb	\tmp	adrp	\dst, \sym	ldr	\dst, [\dst, :lo12:\sym]	.else	adrp	\tmp, \sym	ldr	\dst, [\tmp, :lo12:\sym]	.endif	.endm 	/*	 * @src: source register (32 or 64 bit wide)	 * @sym: name of the symbol	 * @tmp: mandatory 64-bit scratch register to calculate the address	 *       while <src> needs to be preserved.	 */	.macro	str_l, src, sym, tmp	adrp	\tmp, \sym	str	\src, [\tmp, :lo12:\sym]	.endm 	/*	 * @dst: destination register	 */#if defined(__KVM_NVHE_HYPERVISOR__) || defined(__KVM_VHE_HYPERVISOR__)	.macro	this_cpu_offset, dst	mrs	\dst, tpidr_el2	.endm#else	.macro	this_cpu_offset, dstalternative_if_not ARM64_HAS_VIRT_HOST_EXTN	mrs	\dst, tpidr_el1alternative_else	mrs	\dst, tpidr_el2alternative_endif	.endm#endif 	/*	 * @dst: Result of per_cpu(sym, smp_processor_id()) (can be SP)	 * @sym: The name of the per-cpu variable	 * @tmp: scratch register	 */	.macro adr_this_cpu, dst, sym, tmp	adrp	\tmp, \sym	add	\dst, \tmp, #:lo12:\sym	this_cpu_offset \tmp	add	\dst, \dst, \tmp	.endm 	/*	 * @dst: Result of READ_ONCE(per_cpu(sym, smp_processor_id()))	 * @sym: The name of the per-cpu variable	 * @tmp: scratch register	 */	.macro ldr_this_cpu dst, sym, tmp	adr_l	\dst, \sym	this_cpu_offset \tmp	ldr	\dst, [\dst, \tmp]	.endm /* * vma_vm_mm - get mm pointer from vma pointer (vma->vm_mm) */	.macro	vma_vm_mm, rd, rn	ldr	\rd, [\rn, #VMA_VM_MM]	.endm /* * read_ctr - read CTR_EL0. If the system has mismatched register fields, * provide the system wide safe value from arm64_ftr_reg_ctrel0.sys_val */	.macro	read_ctr, reg#ifndef __KVM_NVHE_HYPERVISOR__alternative_if_not ARM64_MISMATCHED_CACHE_TYPE	mrs	\reg, ctr_el0			// read CTR	nopalternative_else	ldr_l	\reg, arm64_ftr_reg_ctrel0 + ARM64_FTR_SYSVALalternative_endif#elsealternative_if_not ARM64_KVM_PROTECTED_MODE	ASM_BUG()alternative_else_nop_endifalternative_cb kvm_compute_final_ctr_el0	movz	\reg, #0	movk	\reg, #0, lsl #16	movk	\reg, #0, lsl #32	movk	\reg, #0, lsl #48alternative_cb_end#endif	.endm  /* * raw_dcache_line_size - get the minimum D-cache line size on this CPU * from the CTR register. */	.macro	raw_dcache_line_size, reg, tmp	mrs	\tmp, ctr_el0			// read CTR	ubfm	\tmp, \tmp, #16, #19		// cache line size encoding	mov	\reg, #4			// bytes per word	lsl	\reg, \reg, \tmp		// actual cache line size	.endm /* * dcache_line_size - get the safe D-cache line size across all CPUs */	.macro	dcache_line_size, reg, tmp	read_ctr	\tmp	ubfm		\tmp, \tmp, #16, #19	// cache line size encoding	mov		\reg, #4		// bytes per word	lsl		\reg, \reg, \tmp	// actual cache line size	.endm /* * raw_icache_line_size - get the minimum I-cache line size on this CPU * from the CTR register. */	.macro	raw_icache_line_size, reg, tmp	mrs	\tmp, ctr_el0			// read CTR	and	\tmp, \tmp, #0xf		// cache line size encoding	mov	\reg, #4			// bytes per word	lsl	\reg, \reg, \tmp		// actual cache line size	.endm /* * icache_line_size - get the safe I-cache line size across all CPUs */	.macro	icache_line_size, reg, tmp	read_ctr	\tmp	and		\tmp, \tmp, #0xf	// cache line size encoding	mov		\reg, #4		// bytes per word	lsl		\reg, \reg, \tmp	// actual cache line size	.endm /* * tcr_set_t0sz - update TCR.T0SZ so that we can load the ID map */	.macro	tcr_set_t0sz, valreg, t0sz	bfi	\valreg, \t0sz, #TCR_T0SZ_OFFSET, #TCR_TxSZ_WIDTH	.endm /* * tcr_set_t1sz - update TCR.T1SZ */	.macro	tcr_set_t1sz, valreg, t1sz	bfi	\valreg, \t1sz, #TCR_T1SZ_OFFSET, #TCR_TxSZ_WIDTH	.endm /* * tcr_compute_pa_size - set TCR.(I)PS to the highest supported * ID_AA64MMFR0_EL1.PARange value * *	tcr:		register with the TCR_ELx value to be updated *	pos:		IPS or PS bitfield position *	tmp{0,1}:	temporary registers */	.macro	tcr_compute_pa_size, tcr, pos, tmp0, tmp1	mrs	\tmp0, ID_AA64MMFR0_EL1	// Narrow PARange to fit the PS field in TCR_ELx	ubfx	\tmp0, \tmp0, #ID_AA64MMFR0_PARANGE_SHIFT, #3	mov	\tmp1, #ID_AA64MMFR0_PARANGE_MAX	cmp	\tmp0, \tmp1	csel	\tmp0, \tmp1, \tmp0, hi	bfi	\tcr, \tmp0, \pos, #3	.endm /* * Macro to perform a data cache maintenance for the interval * [kaddr, kaddr + size) * * 	op:		operation passed to dc instruction * 	domain:		domain used in dsb instruciton * 	kaddr:		starting virtual address of the region * 	size:		size of the region * 	Corrupts:	kaddr, size, tmp1, tmp2 */	.macro __dcache_op_workaround_clean_cache, op, kaddralternative_if_not ARM64_WORKAROUND_CLEAN_CACHE	dc	\op, \kaddralternative_else	dc	civac, \kaddralternative_endif	.endm 	.macro dcache_by_line_op op, domain, kaddr, size, tmp1, tmp2	dcache_line_size \tmp1, \tmp2	add	\size, \kaddr, \size	sub	\tmp2, \tmp1, #1	bic	\kaddr, \kaddr, \tmp29998:	.ifc	\op, cvau	__dcache_op_workaround_clean_cache \op, \kaddr	.else	.ifc	\op, cvac	__dcache_op_workaround_clean_cache \op, \kaddr	.else	.ifc	\op, cvap	sys	3, c7, c12, 1, \kaddr	// dc cvap	.else	.ifc	\op, cvadp	sys	3, c7, c13, 1, \kaddr	// dc cvadp	.else	dc	\op, \kaddr	.endif	.endif	.endif	.endif	add	\kaddr, \kaddr, \tmp1	cmp	\kaddr, \size	b.lo	9998b	dsb	\domain	.endm /* * Macro to perform an instruction cache maintenance for the interval * [start, end) * * 	start, end:	virtual addresses describing the region *	label:		A label to branch to on user fault. * 	Corrupts:	tmp1, tmp2 */	.macro invalidate_icache_by_line start, end, tmp1, tmp2, label	icache_line_size \tmp1, \tmp2	sub	\tmp2, \tmp1, #1	bic	\tmp2, \start, \tmp29997:USER(\label, ic	ivau, \tmp2)			// invalidate I line PoU	add	\tmp2, \tmp2, \tmp1	cmp	\tmp2, \end	b.lo	9997b	dsb	ish	isb	.endm /* * reset_pmuserenr_el0 - reset PMUSERENR_EL0 if PMUv3 present */	.macro	reset_pmuserenr_el0, tmpreg	mrs	\tmpreg, id_aa64dfr0_el1	sbfx	\tmpreg, \tmpreg, #ID_AA64DFR0_PMUVER_SHIFT, #4	cmp	\tmpreg, #1			// Skip if no PMU present	b.lt	9000f	msr	pmuserenr_el0, xzr		// Disable PMU access from EL09000:	.endm /* * reset_amuserenr_el0 - reset AMUSERENR_EL0 if AMUv1 present */	.macro	reset_amuserenr_el0, tmpreg	mrs	\tmpreg, id_aa64pfr0_el1	// Check ID_AA64PFR0_EL1	ubfx	\tmpreg, \tmpreg, #ID_AA64PFR0_AMU_SHIFT, #4	cbz	\tmpreg, .Lskip_\@		// Skip if no AMU present	msr_s	SYS_AMUSERENR_EL0, xzr		// Disable AMU access from EL0.Lskip_\@:	.endm/* * copy_page - copy src to dest using temp registers t1-t8 */	.macro copy_page dest:req src:req t1:req t2:req t3:req t4:req t5:req t6:req t7:req t8:req9998:	ldp	\t1, \t2, [\src]	ldp	\t3, \t4, [\src, #16]	ldp	\t5, \t6, [\src, #32]	ldp	\t7, \t8, [\src, #48]	add	\src, \src, #64	stnp	\t1, \t2, [\dest]	stnp	\t3, \t4, [\dest, #16]	stnp	\t5, \t6, [\dest, #32]	stnp	\t7, \t8, [\dest, #48]	add	\dest, \dest, #64	tst	\src, #(PAGE_SIZE - 1)	b.ne	9998b	.endm /* * Annotate a function as being unsuitable for kprobes. */#ifdef CONFIG_KPROBES#define NOKPROBE(x)				\	.pushsection "_kprobe_blacklist", "aw";	\	.quad	x;				\	.popsection;#else#define NOKPROBE(x)#endif #if defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)#define EXPORT_SYMBOL_NOKASAN(name)#else

Lines 1–500 of 804. Load more