// SPDX-License-Identifier: GPL-2.0-only#define pr_fmt(fmt) "SMP alternatives: " fmt #include <linux/mmu_context.h>#include <linux/perf_event.h>#include <linux/vmalloc.h>#include <linux/memory.h>#include <linux/execmem.h> #include <asm/text-patching.h>#include <asm/insn.h>#include <asm/insn-eval.h>#include <asm/ibt.h>#include <asm/set_memory.h>#include <asm/nmi.h> int __read_mostly alternatives_patched; EXPORT_SYMBOL_GPL(alternatives_patched); #define MAX_PATCH_LEN (255-1) #define DA_ALL		(~0)#define DA_ALT		0x01#define DA_RET		0x02#define DA_RETPOLINE	0x04#define DA_ENDBR	0x08#define DA_SMP		0x10 static unsigned int debug_alternative; static int __init debug_alt(char *str){	if (str && *str == '=')		str++; 	if (!str || kstrtouint(str, 0, &debug_alternative))		debug_alternative = DA_ALL; 	return 1;}__setup("debug-alternative", debug_alt); #define DPRINTK(type, fmt, args...)					\do {									\	if (debug_alternative & DA_##type)				\		printk(KERN_DEBUG pr_fmt(fmt) "\n", ##args);		\} while (0) #define DUMP_BYTES(type, buf, len, fmt, args...)			\do {									\	if (unlikely(debug_alternative & DA_##type)) {			\		int j;							\									\		if (!(len))						\			break;						\									\		printk(KERN_DEBUG pr_fmt(fmt), ##args);			\		for (j = 0; j < (len) - 1; j++)				\			printk(KERN_CONT "%02hhx ", buf[j]);		\		printk(KERN_CONT "%02hhx\n", buf[j]);			\	}								\} while (0) static const unsigned char x86nops[] ={	BYTES_NOP1,	BYTES_NOP2,	BYTES_NOP3,	BYTES_NOP4,	BYTES_NOP5,	BYTES_NOP6,	BYTES_NOP7,	BYTES_NOP8,#ifdef CONFIG_64BIT	BYTES_NOP9,	BYTES_NOP10,	BYTES_NOP11,#endif}; const unsigned char * const x86_nops[ASM_NOP_MAX+1] ={	NULL,	x86nops,	x86nops + 1,	x86nops + 1 + 2,	x86nops + 1 + 2 + 3,	x86nops + 1 + 2 + 3 + 4,	x86nops + 1 + 2 + 3 + 4 + 5,	x86nops + 1 + 2 + 3 + 4 + 5 + 6,	x86nops + 1 + 2 + 3 + 4 + 5 + 6 + 7,#ifdef CONFIG_64BIT	x86nops + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8,	x86nops + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9,	x86nops + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + 10,#endif}; #ifdef CONFIG_FINEIBTstatic bool cfi_paranoid __ro_after_init;#endif #ifdef CONFIG_MITIGATION_ITS #ifdef CONFIG_MODULESstatic struct module *its_mod;#endifstatic void *its_page;static unsigned int its_offset;struct its_array its_pages; static void *__its_alloc(struct its_array *pages){	void *page __free(execmem) = execmem_alloc_rw(EXECMEM_MODULE_TEXT, PAGE_SIZE);	if (!page)		return NULL; 	void *tmp = krealloc(pages->pages, (pages->num+1) * sizeof(void *),			     GFP_KERNEL);	if (!tmp)		return NULL; 	pages->pages = tmp;	pages->pages[pages->num++] = page; 	return no_free_ptr(page);} /* Initialize a thunk with the "jmp *reg; int3" instructions. */static void *its_init_thunk(void *thunk, int reg){	u8 *bytes = thunk;	int offset = 0;	int i = 0; #ifdef CONFIG_FINEIBT	if (cfi_paranoid) {		/*		 * When ITS uses indirect branch thunk the fineibt_paranoid		 * caller sequence doesn't fit in the caller site. So put the		 * remaining part of the sequence (UDB + JNE) into the ITS		 * thunk.		 */		bytes[i++] = 0xd6; /* UDB */		bytes[i++] = 0x75; /* JNE */		bytes[i++] = 0xfd; 		offset = 1;	}#endif 	if (reg >= 8) {		bytes[i++] = 0x41; /* REX.B prefix */		reg -= 8;	}	bytes[i++] = 0xff;	bytes[i++] = 0xe0 + reg; /* JMP *reg */	bytes[i++] = 0xcc; 	return thunk + offset;} static void its_pages_protect(struct its_array *pages){	for (int i = 0; i < pages->num; i++) {		void *page = pages->pages[i];		execmem_restore_rox(page, PAGE_SIZE);	}} static void its_fini_core(void){	if (IS_ENABLED(CONFIG_STRICT_KERNEL_RWX))		its_pages_protect(&its_pages);	kfree(its_pages.pages);} #ifdef CONFIG_MODULESvoid its_init_mod(struct module *mod){	if (!cpu_feature_enabled(X86_FEATURE_INDIRECT_THUNK_ITS))		return; 	mutex_lock(&text_mutex);	its_mod = mod;	its_page = NULL;} void its_fini_mod(struct module *mod){	if (!cpu_feature_enabled(X86_FEATURE_INDIRECT_THUNK_ITS))		return; 	WARN_ON_ONCE(its_mod != mod); 	its_mod = NULL;	its_page = NULL;	mutex_unlock(&text_mutex); 	if (IS_ENABLED(CONFIG_STRICT_MODULE_RWX))		its_pages_protect(&mod->arch.its_pages);} void its_free_mod(struct module *mod){	if (!cpu_feature_enabled(X86_FEATURE_INDIRECT_THUNK_ITS))		return; 	for (int i = 0; i < mod->arch.its_pages.num; i++) {		void *page = mod->arch.its_pages.pages[i];		execmem_free(page);	}	kfree(mod->arch.its_pages.pages);}#endif /* CONFIG_MODULES */ static void *its_alloc(void){	struct its_array *pages = &its_pages;	void *page; #ifdef CONFIG_MODULES	if (its_mod)		pages = &its_mod->arch.its_pages;#endif 	page = __its_alloc(pages);	if (!page)		return NULL; 	if (pages == &its_pages)		set_memory_x((unsigned long)page, 1); 	return page;} static void *its_allocate_thunk(int reg){	int size = 3 + (reg / 8);	void *thunk; #ifdef CONFIG_FINEIBT	/*	 * The ITS thunk contains an indirect jump and an int3 instruction so	 * its size is 3 or 4 bytes depending on the register used. If CFI	 * paranoid is used then 3 extra bytes are added in the ITS thunk to	 * complete the fineibt_paranoid caller sequence.	 */	if (cfi_paranoid)		size += 3;#endif 	if (!its_page || (its_offset + size - 1) >= PAGE_SIZE) {		its_page = its_alloc();		if (!its_page) {			pr_err("ITS page allocation failed\n");			return NULL;		}		memset(its_page, INT3_INSN_OPCODE, PAGE_SIZE);		its_offset = 32;	} 	/*	 * If the indirect branch instruction will be in the lower half	 * of a cacheline, then update the offset to reach the upper half.	 */	if ((its_offset + size - 1) % 64 < 32)		its_offset = ((its_offset - 1) | 0x3F) + 33; 	thunk = its_page + its_offset;	its_offset += size; 	return its_init_thunk(thunk, reg);} u8 *its_static_thunk(int reg){	u8 *thunk = __x86_indirect_its_thunk_array[reg]; #ifdef CONFIG_FINEIBT	/* Paranoid thunk starts 2 bytes before */	if (cfi_paranoid)		return thunk - 2;#endif	return thunk;} #elsestatic inline void its_fini_core(void) {}#endif /* CONFIG_MITIGATION_ITS */ /* * Nomenclature for variable names to simplify and clarify this code and ease * any potential staring at it: * * @instr: source address of the original instructions in the kernel text as * generated by the compiler. * * @buf: temporary buffer on which the patching operates. This buffer is * eventually text-poked into the kernel image. * * @replacement/@repl: pointer to the opcodes which are replacing @instr, located * in the .altinstr_replacement section. */ /* * Fill the buffer with a single effective instruction of size @len. * * In order not to issue an ORC stack depth tracking CFI entry (Call Frame Info) * for every single-byte NOP, try to generate the maximally available NOP of * size <= ASM_NOP_MAX such that only a single CFI entry is generated (vs one for * each single-byte NOPs). If @len to fill out is > ASM_NOP_MAX, pad with INT3 and * *jump* over instead of executing long and daft NOPs. */static void add_nop(u8 *buf, unsigned int len){	u8 *target = buf + len; 	if (!len)		return; 	if (len <= ASM_NOP_MAX) {		memcpy(buf, x86_nops[len], len);		return;	} 	if (len < 128) {		__text_gen_insn(buf, JMP8_INSN_OPCODE, buf, target, JMP8_INSN_SIZE);		buf += JMP8_INSN_SIZE;	} else {		__text_gen_insn(buf, JMP32_INSN_OPCODE, buf, target, JMP32_INSN_SIZE);		buf += JMP32_INSN_SIZE;	} 	for (;buf < target; buf++)		*buf = INT3_INSN_OPCODE;} /* * Find the offset of the first non-NOP instruction starting at @offset * but no further than @len. */static int skip_nops(u8 *buf, int offset, int len){	struct insn insn; 	for (; offset < len; offset += insn.length) {		if (insn_decode_kernel(&insn, &buf[offset]))			break; 		if (!insn_is_nop(&insn))			break;	} 	return offset;} /* * "noinline" to cause control flow change and thus invalidate I$ and * cause refetch after modification. */static void noinline optimize_nops(const u8 * const instr, u8 *buf, size_t len){	for (int next, i = 0; i < len; i = next) {		struct insn insn; 		if (insn_decode_kernel(&insn, &buf[i]))			return; 		next = i + insn.length; 		if (insn_is_nop(&insn)) {			int nop = i; 			/* Has the NOP already been optimized? */			if (i + insn.length == len)				return; 			next = skip_nops(buf, next, len); 			add_nop(buf + nop, next - nop);			DUMP_BYTES(ALT, buf, len, "%px: [%d:%d) optimized NOPs: ", instr, nop, next);		}	}} /* * In this context, "source" is where the instructions are placed in the * section .altinstr_replacement, for example during kernel build by the * toolchain. * "Destination" is where the instructions are being patched in by this * machinery. * * The source offset is: * *   src_imm = target - src_next_ip                  (1) * * and the target offset is: * *   dst_imm = target - dst_next_ip                  (2) * * so rework (1) as an expression for target like: * *   target = src_imm + src_next_ip                  (1a) * * and substitute in (2) to get: * *   dst_imm = (src_imm + src_next_ip) - dst_next_ip (3) * * Now, since the instruction stream is 'identical' at src and dst (it * is being copied after all) it can be stated that: * *   src_next_ip = src + ip_offset *   dst_next_ip = dst + ip_offset                   (4) * * Substitute (4) in (3) and observe ip_offset being cancelled out to * obtain: * *   dst_imm = src_imm + (src + ip_offset) - (dst + ip_offset) *           = src_imm + src - dst + ip_offset - ip_offset *           = src_imm + src - dst                   (5) * * IOW, only the relative displacement of the code block matters. */ #define apply_reloc_n(n_, p_, d_)				\	do {							\		s32 v = *(s##n_ *)(p_);				\		v += (d_);					\		BUG_ON((v >> 31) != (v >> (n_-1)));		\		*(s##n_ *)(p_) = (s##n_)v;			\	} while (0)  static __always_inlinevoid apply_reloc(int n, void *ptr, uintptr_t diff){	switch (n) {	case 1: apply_reloc_n(8, ptr, diff); break;	case 2: apply_reloc_n(16, ptr, diff); break;	case 4: apply_reloc_n(32, ptr, diff); break;	default: BUG();	}} static __always_inlinebool need_reloc(unsigned long offset, u8 *src, size_t src_len){	u8 *target = src + offset;	/*	 * If the target is inside the patched block, it's relative to the	 * block itself and does not need relocation.	 */	return (target < src || target > src + src_len);} static void __apply_relocation(u8 *buf, const u8 * const instr, size_t instrlen, u8 *repl, size_t repl_len){	for (int next, i = 0; i < instrlen; i = next) {		struct insn insn; 		if (WARN_ON_ONCE(insn_decode_kernel(&insn, &buf[i])))			return; 		next = i + insn.length; 		switch (insn.opcode.bytes[0]) {		case 0x0f:			if (insn.opcode.bytes[1] < 0x80 ||			    insn.opcode.bytes[1] > 0x8f)				break; 			fallthrough;	/* Jcc.d32 */		case 0x70 ... 0x7f:	/* Jcc.d8 */		case JMP8_INSN_OPCODE:		case JMP32_INSN_OPCODE:		case CALL_INSN_OPCODE:			if (need_reloc(next + insn.immediate.value, repl, repl_len)) {				apply_reloc(insn.immediate.nbytes,					    buf + i + insn_offset_immediate(&insn),					    repl - instr);			} 			/*			 * Where possible, convert JMP.d32 into JMP.d8.			 */			if (insn.opcode.bytes[0] == JMP32_INSN_OPCODE) {				s32 imm = insn.immediate.value;				imm += repl - instr;				imm += JMP32_INSN_SIZE - JMP8_INSN_SIZE;				if ((imm >> 31) == (imm >> 7)) {					buf[i+0] = JMP8_INSN_OPCODE;					buf[i+1] = (s8)imm; 					memset(&buf[i+2], INT3_INSN_OPCODE, insn.length - 2);				}			}			break;		}

Lines 1–500 of 3054. Load more