/* SPDX-License-Identifier: GPL-2.0-only *//* *  Copyright (C) 1994-2002 Russell King *  Copyright (c) 2003, 2020 ARM Limited *  All Rights Reserved */ #include <linux/init.h>#include <linux/linkage.h>#include <asm/assembler.h>#include <asm/page.h> #ifdef __ARMEB__#define LOW_OFFSET	0x4#define HIGH_OFFSET	0x0#else#define LOW_OFFSET	0x0#define HIGH_OFFSET	0x4#endif /* * __fixup_pv_table - patch the stub instructions with the delta between *                    PHYS_OFFSET and PAGE_OFFSET, which is assumed to be *                    2 MiB aligned. * * Called from head.S, which expects the following registers to be preserved: *   r1 = machine no, r2 = atags or dtb, *   r8 = phys_offset, r9 = cpuid, r10 = procinfo */	__HEADENTRY(__fixup_pv_table)	mov	r0, r8, lsr #PAGE_SHIFT	@ convert to PFN	str_l	r0, __pv_phys_pfn_offset, r3 	adr_l	r0, __pv_offset	subs	r3, r8, #PAGE_OFFSET	@ PHYS_OFFSET - PAGE_OFFSET	mvn	ip, #0	strcc	ip, [r0, #HIGH_OFFSET]	@ save to __pv_offset high bits	str	r3, [r0, #LOW_OFFSET]	@ save to __pv_offset low bits 	mov	r0, r3, lsr #21		@ constant for add/sub instructions	teq	r3, r0, lsl #21 	@ must be 2 MiB aligned	bne	0f 	adr_l	r4, __pv_table_begin	adr_l	r5, __pv_table_end	b	__fixup_a_pv_table 0:	mov	r0, r0			@ deadloop on error	b	0bENDPROC(__fixup_pv_table) 	.text__fixup_a_pv_table:	adr_l	r6, __pv_offset	ldr	r0, [r6, #HIGH_OFFSET]	@ pv_offset high word	ldr	r6, [r6, #LOW_OFFSET]	@ pv_offset low word	cmn	r0, #1#ifdef CONFIG_THUMB2_KERNEL	@	@ The Thumb-2 versions of the patchable sequences are	@	@ phys-to-virt:			movw	<reg>, #offset<31:21>	@				lsl	<reg>, #21	@				sub	<VA>, <PA>, <reg>	@	@ virt-to-phys (non-LPAE):	movw	<reg>, #offset<31:21>	@				lsl	<reg>, #21	@				add	<PA>, <VA>, <reg>	@	@ virt-to-phys (LPAE):		movw	<reg>, #offset<31:21>	@				lsl	<reg>, #21	@				adds	<PAlo>, <VA>, <reg>	@				mov	<PAhi>, #offset<39:32>	@				adc	<PAhi>, <PAhi>, #0	@	@ In the non-LPAE case, all patchable instructions are MOVW	@ instructions, where we need to patch in the offset into the	@ second halfword of the opcode (the 16-bit immediate is encoded	@ as imm4:i:imm3:imm8)	@	@       15       11 10  9           4 3    0  15  14  12 11 8 7    0	@      +-----------+---+-------------+------++---+------+----+------+	@ MOVW | 1 1 1 1 0 | i | 1 0 0 1 0 0 | imm4 || 0 | imm3 | Rd | imm8 |	@      +-----------+---+-------------+------++---+------+----+------+	@	@ In the LPAE case, we also need to patch in the high word of the	@ offset into the immediate field of the MOV instruction, or patch it	@ to a MVN instruction if the offset is negative. In this case, we	@ need to inspect the first halfword of the opcode, to check whether	@ it is MOVW or MOV/MVN, and to perform the MOV to MVN patching if	@ needed. The encoding of the immediate is rather complex for values	@ of i:imm3 != 0b0000, but fortunately, we never need more than 8 lower	@ order bits, which can be patched into imm8 directly (and i:imm3	@ cleared)	@	@      15       11 10  9        5         0  15  14  12 11 8 7    0	@     +-----------+---+---------------------++---+------+----+------+	@ MOV | 1 1 1 1 0 | i | 0 0 0 1 0 0 1 1 1 1 || 0 | imm3 | Rd | imm8 |	@ MVN | 1 1 1 1 0 | i | 0 0 0 1 1 0 1 1 1 1 || 0 | imm3 | Rd | imm8 |	@     +-----------+---+---------------------++---+------+----+------+	@	moveq	r0, #0x200000		@ set bit 21, mov to mvn instruction	lsrs	r3, r6, #29		@ isolate top 3 bits of displacement	ubfx	r6, r6, #21, #8		@ put bits 28:21 into the MOVW imm8 field	bfi	r6, r3, #12, #3		@ put bits 31:29 into the MOVW imm3 field	b	.Lnext.Lloop:	add	r7, r4	adds	r4, #4			@ clears Z flag#ifdef CONFIG_ARM_LPAE	ldrh	ip, [r7]ARM_BE8(rev16	ip, ip)	tst	ip, #0x200		@ MOVW has bit 9 set, MVN has it clear	bne	0f			@ skip to MOVW handling (Z flag is clear)	bic	ip, #0x20		@ clear bit 5 (MVN -> MOV)	orr	ip, ip, r0, lsr #16	@ MOV -> MVN if offset < 0ARM_BE8(rev16	ip, ip)	strh	ip, [r7]	@ Z flag is set0:#endif	ldrh	ip, [r7, #2]ARM_BE8(rev16	ip, ip)	and	ip, #0xf00		@ clear everything except Rd field	orreq	ip, r0			@ Z flag set -> MOV/MVN -> patch in high bits	orrne	ip, r6			@ Z flag clear -> MOVW -> patch in low bitsARM_BE8(rev16	ip, ip)	strh	ip, [r7, #2]#else#ifdef CONFIG_CPU_ENDIAN_BE8@ in BE8, we load data in BE, but instructions still in LE#define PV_BIT24	0x00000001#define PV_IMM8_MASK	0xff000000#define PV_IMMR_MSB	0x00080000#else#define PV_BIT24	0x01000000#define PV_IMM8_MASK	0x000000ff#define PV_IMMR_MSB	0x00000800#endif 	@	@ The ARM versions of the patchable sequences are	@	@ phys-to-virt:			sub	<VA>, <PA>, #offset<31:24>, lsl #24	@				sub	<VA>, <PA>, #offset<23:16>, lsl #16	@	@ virt-to-phys (non-LPAE):	add	<PA>, <VA>, #offset<31:24>, lsl #24	@				add	<PA>, <VA>, #offset<23:16>, lsl #16	@	@ virt-to-phys (LPAE):		movw	<reg>, #offset<31:20>	@				adds	<PAlo>, <VA>, <reg>, lsl #20	@				mov	<PAhi>, #offset<39:32>	@				adc	<PAhi>, <PAhi>, #0	@	@ In the non-LPAE case, all patchable instructions are ADD or SUB	@ instructions, where we need to patch in the offset into the	@ immediate field of the opcode, which is emitted with the correct	@ rotation value. (The effective value of the immediate is imm12<7:0>	@ rotated right by [2 * imm12<11:8>] bits)	@	@      31   28 27      23 22  20 19  16 15  12 11    0	@      +------+-----------------+------+------+-------+	@  ADD | cond | 0 0 1 0 1 0 0 0 |  Rn  |  Rd  | imm12 |	@  SUB | cond | 0 0 1 0 0 1 0 0 |  Rn  |  Rd  | imm12 |	@  MOV | cond | 0 0 1 1 1 0 1 0 |  Rn  |  Rd  | imm12 |	@  MVN | cond | 0 0 1 1 1 1 1 0 |  Rn  |  Rd  | imm12 |	@      +------+-----------------+------+------+-------+	@	@ In the LPAE case, we use a MOVW instruction to carry the low offset	@ word, and patch in the high word of the offset into the immediate	@ field of the subsequent MOV instruction, or patch it to a MVN	@ instruction if the offset is negative. We can distinguish MOVW	@ instructions based on bits 23:22 of the opcode, and ADD/SUB can be	@ distinguished from MOV/MVN (all using the encodings above) using	@ bit 24.	@	@      31   28 27      23 22  20 19  16 15  12 11    0	@      +------+-----------------+------+------+-------+	@ MOVW | cond | 0 0 1 1 0 0 0 0 | imm4 |  Rd  | imm12 |	@      +------+-----------------+------+------+-------+	@	moveq	r0, #0x400000		@ set bit 22, mov to mvn instruction	mov	r3, r6, lsr #16		@ put offset bits 31-16 into r3	mov	r6, r6, lsr #24		@ put offset bits 31-24 into r6	and	r3, r3, #0xf0		@ only keep offset bits 23-20 in r3	b	.Lnext.Lloop:	ldr	ip, [r7, r4]#ifdef CONFIG_ARM_LPAE	tst	ip, #PV_BIT24		@ ADD/SUB have bit 24 clear	beq	1fARM_BE8(rev	ip, ip)	tst	ip, #0xc00000		@ MOVW has bits 23:22 clear	bic	ip, ip, #0x400000	@ clear bit 22	bfc	ip, #0, #12		@ clear imm12 field of MOV[W] instruction	orreq	ip, ip, r6, lsl #4	@ MOVW -> mask in offset bits 31-24	orreq	ip, ip, r3, lsr #4	@ MOVW -> mask in offset bits 23-20	orrne	ip, ip, r0		@ MOV  -> mask in offset bits 7-0 (or bit 22)ARM_BE8(rev	ip, ip)	b	2f1:#endif	tst	ip, #PV_IMMR_MSB		@ rotation value >= 16 ?	bic	ip, ip, #PV_IMM8_MASK	orreq	ip, ip, r6 ARM_BE8(, lsl #24)	@ mask in offset bits 31-24	orrne	ip, ip, r3 ARM_BE8(, lsl #24)	@ mask in offset bits 23-202:	str	ip, [r7, r4]	add	r4, r4, #4#endif .Lnext:	cmp	r4, r5	ldrcc	r7, [r4]		@ use branch for delay slot	bcc	.Lloop	ret	lrENDPROC(__fixup_a_pv_table) ENTRY(fixup_pv_table)	stmfd	sp!, {r4 - r7, lr}	mov	r4, r0			@ r0 = table start	add	r5, r0, r1		@ r1 = table size	bl	__fixup_a_pv_table	ldmfd	sp!, {r4 - r7, pc}ENDPROC(fixup_pv_table) 	.data	.align	2	.globl	__pv_phys_pfn_offset	.type	__pv_phys_pfn_offset, %object__pv_phys_pfn_offset:	.word	0	.size	__pv_phys_pfn_offset, . -__pv_phys_pfn_offset 	.globl	__pv_offset	.type	__pv_offset, %object__pv_offset:	.quad	0	.size	__pv_offset, . -__pv_offset