// SPDX-License-Identifier: GPL-2.0-only/* * Copyright (C) 2022 - Google LLC * Author: Ard Biesheuvel <ardb@google.com> */ #include <linux/bug.h>#include <linux/errno.h>#include <linux/init.h>#include <linux/linkage.h>#include <linux/printk.h>#include <linux/types.h> #include <asm/cacheflush.h>#include <asm/scs.h> //// This minimal DWARF CFI parser is partially based on the code in// arch/arc/kernel/unwind.c, and on the document below:// https://refspecs.linuxbase.org/LSB_4.0.0/LSB-Core-generic/LSB-Core-generic/ehframechpt.html// #define DW_CFA_nop                          0x00#define DW_CFA_set_loc                      0x01#define DW_CFA_advance_loc1                 0x02#define DW_CFA_advance_loc2                 0x03#define DW_CFA_advance_loc4                 0x04#define DW_CFA_offset_extended              0x05#define DW_CFA_restore_extended             0x06#define DW_CFA_undefined                    0x07#define DW_CFA_same_value                   0x08#define DW_CFA_register                     0x09#define DW_CFA_remember_state               0x0a#define DW_CFA_restore_state                0x0b#define DW_CFA_def_cfa                      0x0c#define DW_CFA_def_cfa_register             0x0d#define DW_CFA_def_cfa_offset               0x0e#define DW_CFA_def_cfa_expression           0x0f#define DW_CFA_expression                   0x10#define DW_CFA_offset_extended_sf           0x11#define DW_CFA_def_cfa_sf                   0x12#define DW_CFA_def_cfa_offset_sf            0x13#define DW_CFA_val_offset                   0x14#define DW_CFA_val_offset_sf                0x15#define DW_CFA_val_expression               0x16#define DW_CFA_lo_user                      0x1c#define DW_CFA_negate_ra_state              0x2d#define DW_CFA_GNU_args_size                0x2e#define DW_CFA_GNU_negative_offset_extended 0x2f#define DW_CFA_hi_user                      0x3f extern const u8 __eh_frame_start[], __eh_frame_end[]; enum {	PACIASP		= 0xd503233f,	AUTIASP		= 0xd50323bf,	SCS_PUSH	= 0xf800865e,	SCS_POP		= 0xf85f8e5e,}; static void __always_inline scs_patch_loc(u64 loc){	u32 insn = le32_to_cpup((void *)loc); 	switch (insn) {	case PACIASP:		*(u32 *)loc = cpu_to_le32(SCS_PUSH);		break;	case AUTIASP:		*(u32 *)loc = cpu_to_le32(SCS_POP);		break;	default:		/*		 * While the DW_CFA_negate_ra_state directive is guaranteed to		 * appear right after a PACIASP/AUTIASP instruction, it may		 * also appear after a DW_CFA_restore_state directive that		 * restores a state that is only partially accurate, and is		 * followed by DW_CFA_negate_ra_state directive to toggle the		 * PAC bit again. So we permit other instructions here, and ignore		 * them.		 */		return;	}	dcache_clean_pou(loc, loc + sizeof(u32));} /* * Skip one uleb128/sleb128 encoded quantity from the opcode stream. All bytes * except the last one have bit #7 set. */static int __always_inline skip_xleb128(const u8 **opcode, int size){	u8 c; 	do {		c = *(*opcode)++;		size--;	} while (c & BIT(7)); 	return size;} struct eh_frame {	/*	 * The size of this frame if 0 < size < U32_MAX, 0 terminates the list.	 */	u32	size; 	/*	 * The first frame is a Common Information Entry (CIE) frame, followed	 * by one or more Frame Description Entry (FDE) frames. In the former	 * case, this field is 0, otherwise it is the negated offset relative	 * to the associated CIE frame.	 */	u32	cie_id_or_pointer; 	union {		struct { // CIE			u8	version;			u8	augmentation_string[];		}; 		struct { // FDE			s32	initial_loc;			s32	range;			u8	opcodes[];		};	};}; static int noinstr scs_handle_fde_frame(const struct eh_frame *frame,					bool fde_has_augmentation_data,					int code_alignment_factor){	int size = frame->size - offsetof(struct eh_frame, opcodes) + 4;	u64 loc = (u64)offset_to_ptr(&frame->initial_loc);	const u8 *opcode = frame->opcodes; 	if (fde_has_augmentation_data) {		int l; 		// assume single byte uleb128_t		if (WARN_ON(*opcode & BIT(7)))			return -ENOEXEC; 		l = *opcode++;		opcode += l;		size -= l + 1;	} 	/*	 * Starting from 'loc', apply the CFA opcodes that advance the location	 * pointer, and identify the locations of the PAC instructions.	 */	while (size-- > 0) {		switch (*opcode++) {		case DW_CFA_nop:		case DW_CFA_remember_state:		case DW_CFA_restore_state:			break; 		case DW_CFA_advance_loc1:			loc += *opcode++ * code_alignment_factor;			size--;			break; 		case DW_CFA_advance_loc2:			loc += *opcode++ * code_alignment_factor;			loc += (*opcode++ << 8) * code_alignment_factor;			size -= 2;			break; 		case DW_CFA_def_cfa:		case DW_CFA_offset_extended:			size = skip_xleb128(&opcode, size);			fallthrough;		case DW_CFA_def_cfa_offset:		case DW_CFA_def_cfa_offset_sf:		case DW_CFA_def_cfa_register:		case DW_CFA_same_value:		case DW_CFA_restore_extended:		case 0x80 ... 0xbf:			size = skip_xleb128(&opcode, size);			break; 		case DW_CFA_negate_ra_state:			scs_patch_loc(loc - 4);			break; 		case 0x40 ... 0x7f:			// advance loc			loc += (opcode[-1] & 0x3f) * code_alignment_factor;			break; 		case 0xc0 ... 0xff:			break; 		default:			pr_err("unhandled opcode: %02x in FDE frame %lx\n", opcode[-1], (uintptr_t)frame);			return -ENOEXEC;		}	}	return 0;} int noinstr scs_patch(const u8 eh_frame[], int size){	const u8 *p = eh_frame; 	while (size > 4) {		const struct eh_frame *frame = (const void *)p;		bool fde_has_augmentation_data = true;		int code_alignment_factor = 1;		int ret; 		if (frame->size == 0 ||		    frame->size == U32_MAX ||		    frame->size > size)			break; 		if (frame->cie_id_or_pointer == 0) {			const u8 *p = frame->augmentation_string; 			/* a 'z' in the augmentation string must come first */			fde_has_augmentation_data = *p == 'z'; 			/*			 * The code alignment factor is a uleb128 encoded field			 * but given that the only sensible values are 1 or 4,			 * there is no point in decoding the whole thing.			 */			p += strlen(p) + 1;			if (!WARN_ON(*p & BIT(7)))				code_alignment_factor = *p;		} else {			ret = scs_handle_fde_frame(frame,						   fde_has_augmentation_data,						   code_alignment_factor);			if (ret)				return ret;		} 		p += sizeof(frame->size) + frame->size;		size -= sizeof(frame->size) + frame->size;	}	return 0;} asmlinkage void __init scs_patch_vmlinux(void){	if (!should_patch_pac_into_scs())		return; 	WARN_ON(scs_patch(__eh_frame_start, __eh_frame_end - __eh_frame_start));	icache_inval_all_pou();	isb();}