/* SPDX-License-Identifier: GPL-2.0-only *//* * Based on arch/arm/include/asm/assembler.h, arch/arm/mm/proc-macros.S * * Copyright (C) 1996-2000 Russell King * Copyright (C) 2012 ARM Ltd. */#ifndef __ASSEMBLER__#error "Only include this from assembly code"#endif #ifndef __ASM_ASSEMBLER_H#define __ASM_ASSEMBLER_H #include <linux/export.h> #include <asm/alternative.h>#include <asm/asm-bug.h>#include <asm/asm-extable.h>#include <asm/asm-offsets.h>#include <asm/cpufeature.h>#include <asm/cputype.h>#include <asm/debug-monitors.h>#include <asm/page.h>#include <asm/pgtable-hwdef.h>#include <asm/ptrace.h>#include <asm/thread_info.h> 	/*	 * Provide a wxN alias for each wN register so what we can paste a xN	 * reference after a 'w' to obtain the 32-bit version.	 */	.irp	n,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30	wx\n	.req	w\n	.endr /* * Save/restore interrupts. */	.macro save_and_disable_daif, flags	mrs	\flags, daif	msr	daifset, #0xf	.endm 	.macro	save_and_disable_irq, flags	mrs	\flags, daif	msr	daifset, #3	.endm 	.macro	restore_irq, flags	msr	daif, \flags	.endm 	.macro	disable_step_tsk, flgs, tmp	tbz	\flgs, #TIF_SINGLESTEP, 9990f	mrs	\tmp, mdscr_el1	bic	\tmp, \tmp, #MDSCR_EL1_SS	msr	mdscr_el1, \tmp	isb	// Take effect before a subsequent clear of DAIF.D9990:	.endm 	/* call with daif masked */	.macro	enable_step_tsk, flgs, tmp	tbz	\flgs, #TIF_SINGLESTEP, 9990f	mrs	\tmp, mdscr_el1	orr	\tmp, \tmp, #MDSCR_EL1_SS	msr	mdscr_el1, \tmp9990:	.endm /* * RAS Error Synchronization barrier */	.macro  esb#ifdef CONFIG_ARM64_RAS_EXTN	hint    #16#else	nop#endif	.endm /* * Value prediction barrier */	.macro	csdb	hint	#20	.endm /* * Clear Branch History instruction */	.macro clearbhb	hint	#22	.endm /* * Speculation barrier */	.macro	sbalternative_if_not ARM64_HAS_SB	dsb	nsh	isbalternative_else	SB_BARRIER_INSN	nopalternative_endif	.endm /* * NOP sequence */	.macro	nops, num	.rept	\num	nop	.endr	.endm /* * Register aliases. */lr	.req	x30		// link register /* * Vector entry */	 .macro	ventry	label	.align	7	b	\label	.endm /* * Select code when configured for BE. */#ifdef CONFIG_CPU_BIG_ENDIAN#define CPU_BE(code...) code#else#define CPU_BE(code...)#endif /* * Select code when configured for LE. */#ifdef CONFIG_CPU_BIG_ENDIAN#define CPU_LE(code...)#else#define CPU_LE(code...) code#endif /* * Define a macro that constructs a 64-bit value by concatenating two * 32-bit registers. Note that on big endian systems the order of the * registers is swapped. */#ifndef CONFIG_CPU_BIG_ENDIAN	.macro	regs_to_64, rd, lbits, hbits#else	.macro	regs_to_64, rd, hbits, lbits#endif	orr	\rd, \lbits, \hbits, lsl #32	.endm /* * Pseudo-ops for PC-relative adr/ldr/str <reg>, <symbol> where * <symbol> is within the range +/- 4 GB of the PC. */	/*	 * @dst: destination register (64 bit wide)	 * @sym: name of the symbol	 */	.macro	adr_l, dst, sym	adrp	\dst, \sym	add	\dst, \dst, :lo12:\sym	.endm 	/*	 * @dst: destination register (32 or 64 bit wide)	 * @sym: name of the symbol	 * @tmp: optional 64-bit scratch register to be used if <dst> is a	 *       32-bit wide register, in which case it cannot be used to hold	 *       the address	 */	.macro	ldr_l, dst, sym, tmp=	.ifb	\tmp	adrp	\dst, \sym	ldr	\dst, [\dst, :lo12:\sym]	.else	adrp	\tmp, \sym	ldr	\dst, [\tmp, :lo12:\sym]	.endif	.endm 	/*	 * @src: source register (32 or 64 bit wide)	 * @sym: name of the symbol	 * @tmp: mandatory 64-bit scratch register to calculate the address	 *       while <src> needs to be preserved.	 */	.macro	str_l, src, sym, tmp	adrp	\tmp, \sym	str	\src, [\tmp, :lo12:\sym]	.endm 	/*	 * @dst: destination register	 */#if defined(__KVM_NVHE_HYPERVISOR__) || defined(__KVM_VHE_HYPERVISOR__)	.macro	get_this_cpu_offset, dst	mrs	\dst, tpidr_el2	.endm#else	.macro	get_this_cpu_offset, dstalternative_if_not ARM64_HAS_VIRT_HOST_EXTN	mrs	\dst, tpidr_el1alternative_else	mrs	\dst, tpidr_el2alternative_endif	.endm 	.macro	set_this_cpu_offset, srcalternative_if_not ARM64_HAS_VIRT_HOST_EXTN	msr	tpidr_el1, \srcalternative_else	msr	tpidr_el2, \srcalternative_endif	.endm#endif 	/*	 * @dst: Result of per_cpu(sym, smp_processor_id()) (can be SP)	 * @sym: The name of the per-cpu variable	 * @tmp: scratch register	 */	.macro adr_this_cpu, dst, sym, tmp	adrp	\tmp, \sym	add	\dst, \tmp, #:lo12:\sym	get_this_cpu_offset \tmp	add	\dst, \dst, \tmp	.endm 	/*	 * @dst: Result of READ_ONCE(per_cpu(sym, smp_processor_id()))	 * @sym: The name of the per-cpu variable	 * @tmp: scratch register	 */	.macro ldr_this_cpu dst, sym, tmp	adr_l	\dst, \sym	get_this_cpu_offset \tmp	ldr	\dst, [\dst, \tmp]	.endm /* * read_ctr - read CTR_EL0. If the system has mismatched register fields, * provide the system wide safe value from arm64_ftr_reg_ctrel0.sys_val */	.macro	read_ctr, reg#ifndef __KVM_NVHE_HYPERVISOR__alternative_if_not ARM64_MISMATCHED_CACHE_TYPE	mrs	\reg, ctr_el0			// read CTR	nopalternative_else	ldr_l	\reg, arm64_ftr_reg_ctrel0 + ARM64_FTR_SYSVALalternative_endif#elsealternative_if_not ARM64_KVM_PROTECTED_MODE	ASM_BUG()alternative_else_nop_endifalternative_cb ARM64_ALWAYS_SYSTEM, kvm_compute_final_ctr_el0	movz	\reg, #0	movk	\reg, #0, lsl #16	movk	\reg, #0, lsl #32	movk	\reg, #0, lsl #48alternative_cb_end#endif	.endm  /* * raw_dcache_line_size - get the minimum D-cache line size on this CPU * from the CTR register. */	.macro	raw_dcache_line_size, reg, tmp	mrs	\tmp, ctr_el0			// read CTR	ubfm	\tmp, \tmp, #16, #19		// cache line size encoding	mov	\reg, #4			// bytes per word	lsl	\reg, \reg, \tmp		// actual cache line size	.endm /* * dcache_line_size - get the safe D-cache line size across all CPUs */	.macro	dcache_line_size, reg, tmp	read_ctr	\tmp	ubfm		\tmp, \tmp, #16, #19	// cache line size encoding	mov		\reg, #4		// bytes per word	lsl		\reg, \reg, \tmp	// actual cache line size	.endm /* * raw_icache_line_size - get the minimum I-cache line size on this CPU * from the CTR register. */	.macro	raw_icache_line_size, reg, tmp	mrs	\tmp, ctr_el0			// read CTR	and	\tmp, \tmp, #0xf		// cache line size encoding	mov	\reg, #4			// bytes per word	lsl	\reg, \reg, \tmp		// actual cache line size	.endm /* * icache_line_size - get the safe I-cache line size across all CPUs */	.macro	icache_line_size, reg, tmp	read_ctr	\tmp	and		\tmp, \tmp, #0xf	// cache line size encoding	mov		\reg, #4		// bytes per word	lsl		\reg, \reg, \tmp	// actual cache line size	.endm /* * tcr_set_t0sz - update TCR.T0SZ so that we can load the ID map */	.macro	tcr_set_t0sz, valreg, t0sz	bfi	\valreg, \t0sz, #TCR_EL1_T0SZ_SHIFT, #TCR_EL1_T0SZ_WIDTH	.endm /* * tcr_set_t1sz - update TCR.T1SZ */	.macro	tcr_set_t1sz, valreg, t1sz	bfi	\valreg, \t1sz, #TCR_EL1_T1SZ_SHIFT, #TCR_EL1_T1SZ_WIDTH	.endm /* * tcr_compute_pa_size - set TCR.(I)PS to the highest supported * ID_AA64MMFR0_EL1.PARange value * *	tcr:		register with the TCR_ELx value to be updated *	pos:		IPS or PS bitfield position *	tmp{0,1}:	temporary registers */	.macro	tcr_compute_pa_size, tcr, pos, tmp0, tmp1	mrs	\tmp0, ID_AA64MMFR0_EL1	// Narrow PARange to fit the PS field in TCR_ELx	ubfx	\tmp0, \tmp0, #ID_AA64MMFR0_EL1_PARANGE_SHIFT, #3	mov	\tmp1, #ID_AA64MMFR0_EL1_PARANGE_MAX#ifdef CONFIG_ARM64_LPA2alternative_if_not ARM64_HAS_VA52	mov	\tmp1, #ID_AA64MMFR0_EL1_PARANGE_48alternative_else_nop_endif#endif	cmp	\tmp0, \tmp1	csel	\tmp0, \tmp1, \tmp0, hi	bfi	\tcr, \tmp0, \pos, #3	.endm 	.macro __dcache_op_workaround_clean_cache, op, addralternative_if_not ARM64_WORKAROUND_CLEAN_CACHE	dc	\op, \addralternative_else	dc	civac, \addralternative_endif	.endm /* * Macro to perform a data cache maintenance for the interval * [start, end) with dcache line size explicitly provided. * * 	op:		operation passed to dc instruction * 	start:          starting virtual address of the region * 	end:            end virtual address of the region *	linesz:		dcache line size * 	fixup:		optional label to branch to on user fault * 	Corrupts:       start, end, tmp */	.macro dcache_by_myline_op_nosync op, start, end, linesz, tmp, fixup	sub	\tmp, \linesz, #1	bic	\start, \start, \tmpalternative_if ARM64_WORKAROUND_4311569	mov	\tmp, \startalternative_else_nop_endif.Ldcache_op\@:	.ifc	\op, cvau	__dcache_op_workaround_clean_cache \op, \start	.else	.ifc	\op, cvac	__dcache_op_workaround_clean_cache \op, \start	.else	.ifc	\op, cvap	sys	3, c7, c12, 1, \start	// dc cvap	.else	.ifc	\op, cvadp	sys	3, c7, c13, 1, \start	// dc cvadp	.else	dc	\op, \start	.endif	.endif	.endif	.endif	add	\start, \start, \linesz	cmp	\start, \end	b.lo	.Ldcache_op\@alternative_if ARM64_WORKAROUND_4311569	.ifnc	\op, cvau	mov	\start, \tmp	mov	\tmp, xzr	cbnz	\start, .Ldcache_op\@	.endifalternative_else_nop_endif 	_cond_uaccess_extable .Ldcache_op\@, \fixup	.endm /* * Macro to perform a data cache maintenance for the interval * [start, end) without waiting for completion * * 	op:		operation passed to dc instruction * 	start:          starting virtual address of the region * 	end:            end virtual address of the region * 	fixup:		optional label to branch to on user fault * 	Corrupts:       start, end, tmp1, tmp2 */	.macro dcache_by_line_op_nosync op, start, end, tmp1, tmp2, fixup	dcache_line_size \tmp1, \tmp2	dcache_by_myline_op_nosync \op, \start, \end, \tmp1, \tmp2, \fixup	.endm /* * Macro to perform a data cache maintenance for the interval * [start, end) and wait for completion * * 	op:		operation passed to dc instruction * 	domain:		domain used in dsb instruction * 	start:          starting virtual address of the region * 	end:            end virtual address of the region * 	fixup:		optional label to branch to on user fault * 	Corrupts:       start, end, tmp1, tmp2 */	.macro dcache_by_line_op op, domain, start, end, tmp1, tmp2, fixup	dcache_by_line_op_nosync \op, \start, \end, \tmp1, \tmp2, \fixup	dsb \domain	.endm /* * Macro to perform an instruction cache maintenance for the interval * [start, end) * * 	start, end:	virtual addresses describing the region *	fixup:		optional label to branch to on user fault * 	Corrupts:	tmp1, tmp2 */	.macro invalidate_icache_by_line start, end, tmp1, tmp2, fixup	icache_line_size \tmp1, \tmp2	sub	\tmp2, \tmp1, #1	bic	\tmp2, \start, \tmp2.Licache_op\@:	ic	ivau, \tmp2			// invalidate I line PoU	add	\tmp2, \tmp2, \tmp1	cmp	\tmp2, \end	b.lo	.Licache_op\@	dsb	ish	isb 	_cond_uaccess_extable .Licache_op\@, \fixup	.endm /* * load_ttbr1 - install @pgtbl as a TTBR1 page table * pgtbl preserved * tmp1/tmp2 clobbered, either may overlap with pgtbl */	.macro		load_ttbr1, pgtbl, tmp1, tmp2	phys_to_ttbr	\tmp1, \pgtbl	offset_ttbr1 	\tmp1, \tmp2	msr		ttbr1_el1, \tmp1	isb	.endm /* * To prevent the possibility of old and new partial table walks being visible * in the tlb, switch the ttbr to a zero page when we invalidate the old * records. D4.7.1 'General TLB maintenance requirements' in ARM DDI 0487A.i * Even switching to our copied tables will cause a changed output address at * each stage of the walk. */	.macro break_before_make_ttbr_switch zero_page, page_table, tmp, tmp2	phys_to_ttbr \tmp, \zero_page	msr	ttbr1_el1, \tmp	isb	tlbi	vmalle1	dsb	nsh	load_ttbr1 \page_table, \tmp, \tmp2	.endm /* * reset_pmuserenr_el0 - reset PMUSERENR_EL0 if PMUv3 present */	.macro	reset_pmuserenr_el0, tmpreg	mrs	\tmpreg, id_aa64dfr0_el1

Lines 1–500 of 866. Load more