kasan: arm64: support specialized outlined tag mismatch checks

By using outlined checks we can achieve a significant code size improvement by moving the tag-based ASAN checks into separate functions. Unlike the existing CONFIG_KASAN_OUTLINE mode these functions have a custom calling convention that preserves most registers and is specialized to the register containing the address and the type of access, and as a result we can eliminate the code size and performance overhead of a standard calling convention such as AAPCS for these functions. This change depends on a separate series of changes to Clang [1] to support outlined checks in the kernel, although the change works fine without them (we just don't get outlined checks). This is because the flag -mllvm -hwasan-inline-all-checks=0 has no effect until the Clang changes land. The flag was introduced in the Clang 9.0 timeframe as part of the support for outlined checks in userspace and because our minimum Clang version is 10.0 we can pass it unconditionally. Outlined checks require a new runtime function with a custom calling convention. Add this function to arch/arm64/lib. I measured the code size of defconfig + tag-based KASAN, as well as boot time (i.e. time to init launch) on a DragonBoard 845c with an Android arm64 GKI kernel. The results are below: code size boot time CONFIG_KASAN_INLINE=y before 92824064 6.18s CONFIG_KASAN_INLINE=y after 38822400 6.65s CONFIG_KASAN_OUTLINE=y 39215616 11.48s We can see straight away that specialized outlined checks beat the existing CONFIG_KASAN_OUTLINE=y on both code size and boot time for tag-based ASAN. As for the comparison between CONFIG_KASAN_INLINE=y before and after we saw similar performance numbers in userspace [2] and decided that since the performance overhead is minimal compared to the overhead of tag-based ASAN itself as well as compared to the code size improvements we would just replace the inlined checks with the specialized outlined checks without the option to select between them, and that is what I have implemented in this patch.

Peter Collingbourne committed 5 years ago

Commit 1cbdf601 parent 2e21d856 files · +107 −2Browse files

Unified · Split

6 files

arch/arm64/include/asm/asm-prototypes.h +6 −0
arch/arm64/include/asm/module.lds.h +15 −2
arch/arm64/lib/Makefile +2 −0
arch/arm64/lib/kasan_sw_tags.S +76 −0
mm/kasan/sw_tags.c +7 −0
scripts/Makefile.kasan +1 −0

arch/arm64/include/asm/asm-prototypes.h+6−0

@@ -23,4 +23,10 @@ long long __ashlti3(long long a, int b); · 22 unmodified lines
23	long long __ashrti3(long long a, int b);	23	long long __ashrti3(long long a, int b);
24	long long __lshrti3(long long a, int b);	24	long long __lshrti3(long long a, int b);
25		25
		26	/*
		27	* This function uses a custom calling convention and cannot be called from C so
		28	* this prototype is not entirely accurate.
		29	*/
		30	void __hwasan_tag_mismatch(unsigned long addr, unsigned long access_info);
		31
26	#endif /* __ASM_PROTOTYPES_H */	32	#endif /* __ASM_PROTOTYPES_H */

arch/arm64/include/asm/module.lds.h+15−2

@@ -1,7 +1,20 @@
1	#ifdef CONFIG_ARM64_MODULE_PLTS
2	SECTIONS {	1	SECTIONS {
		2	#ifdef CONFIG_ARM64_MODULE_PLTS
3	.plt 0 (NOLOAD) : { BYTE(0) }	3	.plt 0 (NOLOAD) : { BYTE(0) }
4	.init.plt 0 (NOLOAD) : { BYTE(0) }	4	.init.plt 0 (NOLOAD) : { BYTE(0) }
5	.text.ftrace_trampoline 0 (NOLOAD) : { BYTE(0) }	5	.text.ftrace_trampoline 0 (NOLOAD) : { BYTE(0) }
6	}
7	#endif	6	#endif
		7
		8	#ifdef CONFIG_KASAN_SW_TAGS
		9	/*
		10	* Outlined checks go into comdat-deduplicated sections named .text.hot.
		11	* Because they are in comdats they are not combined by the linker and
		12	* we otherwise end up with multiple sections with the same .text.hot
		13	* name in the .ko file. The kernel module loader warns if it sees
		14	* multiple sections with the same name so we use this sections
		15	* directive to force them into a single section and silence the
		16	* warning.
		17	*/
		18	.text.hot : { *(.text.hot) }
		19	#endif
		20	}

arch/arm64/lib/Makefile+2−0

@@ -18,3 +18,5 @@ obj-$(CONFIG_CRC32) += crc32.o · 17 unmodified lines
18	obj-$(CONFIG_FUNCTION_ERROR_INJECTION) += error-inject.o	18	obj-$(CONFIG_FUNCTION_ERROR_INJECTION) += error-inject.o
19		19
20	obj-$(CONFIG_ARM64_MTE) += mte.o	20	obj-$(CONFIG_ARM64_MTE) += mte.o
		21
		22	obj-$(CONFIG_KASAN_SW_TAGS) += kasan_sw_tags.o

arch/arm64/lib/kasan_sw_tags.S+76−0

@@ -0,0 +1,76 @@
	1	/* SPDX-License-Identifier: GPL-2.0-only */
	2	/*
	3	* Copyright (C) 2020 Google LLC
	4	*/
	5
	6	#include <linux/linkage.h>
	7	#include <asm/assembler.h>
	8
	9	/*
	10	* Report a tag mismatch detected by tag-based KASAN.
	11	*
	12	* A compiler-generated thunk calls this with a non-AAPCS calling
	13	* convention. Upon entry to this function, registers are as follows:
	14	*
	15	* x0: fault address (see below for restore)
	16	* x1: fault description (see below for restore)
	17	* x2 to x15: callee-saved
	18	* x16 to x17: safe to clobber
	19	* x18 to x30: callee-saved
	20	* sp: pre-decremented by 256 bytes (see below for restore)
	21	*
	22	* The caller has decremented the SP by 256 bytes, and created a
	23	* structure on the stack as follows:
	24	*
	25	* sp + 0..15: x0 and x1 to be restored
	26	* sp + 16..231: free for use
	27	* sp + 232..247: x29 and x30 (same as in GPRs)
	28	* sp + 248..255: free for use
	29	*
	30	* Note that this is not a struct pt_regs.
	31	*
	32	* To call a regular AAPCS function we must save x2 to x15 (which we can
	33	* store in the gaps), and create a frame record (for which we can use
	34	* x29 and x30 spilled by the caller as those match the GPRs).
	35	*
	36	* The caller expects x0 and x1 to be restored from the structure, and
	37	* for the structure to be removed from the stack (i.e. the SP must be
	38	* incremented by 256 prior to return).
	39	*/
	40	SYM_CODE_START(__hwasan_tag_mismatch)
	41	#ifdef BTI_C
	42	BTI_C
	43	#endif
	44	add x29, sp, #232
	45	stp x2, x3, [sp, #8 * 2]
	46	stp x4, x5, [sp, #8 * 4]
	47	stp x6, x7, [sp, #8 * 6]
	48	stp x8, x9, [sp, #8 * 8]
	49	stp x10, x11, [sp, #8 * 10]
	50	stp x12, x13, [sp, #8 * 12]
	51	stp x14, x15, [sp, #8 * 14]
	52	#ifndef CONFIG_SHADOW_CALL_STACK
	53	str x18, [sp, #8 * 18]
	54	#endif
	55
	56	mov x2, x30
	57	bl kasan_tag_mismatch
	58
	59	ldp x0, x1, [sp]
	60	ldp x2, x3, [sp, #8 * 2]
	61	ldp x4, x5, [sp, #8 * 4]
	62	ldp x6, x7, [sp, #8 * 6]
	63	ldp x8, x9, [sp, #8 * 8]
	64	ldp x10, x11, [sp, #8 * 10]
	65	ldp x12, x13, [sp, #8 * 12]
	66	ldp x14, x15, [sp, #8 * 14]
	67	#ifndef CONFIG_SHADOW_CALL_STACK
	68	ldr x18, [sp, #8 * 18]
	69	#endif
	70	ldp x29, x30, [sp, #8 * 29]
	71
	72	/* remove the structure from the stack */
	73	add sp, sp, #256
	74	ret
	75	SYM_CODE_END(__hwasan_tag_mismatch)
	76	EXPORT_SYMBOL(__hwasan_tag_mismatch)

mm/kasan/sw_tags.c+7−0

@@ -207,3 +207,10 @@ struct kasan_track kasan_get_free_track(struct kmem_cache cache, · 206 unmodified lines
207		207
208	return &alloc_meta->free_track[i];	208	return &alloc_meta->free_track[i];
209	}	209	}
		210
		211	void kasan_tag_mismatch(unsigned long addr, unsigned long access_info,
		212	unsigned long ret_ip)
		213	{
		214	kasan_report(addr, 1 << (access_info & 0xf), access_info & 0x10,
		215	ret_ip);
		216	}

scripts/Makefile.kasan+1−0

@@ -50,6 +50,7 @@ endif · 49 unmodified lines
50	CFLAGS_KASAN := -fsanitize=kernel-hwaddress \	50	CFLAGS_KASAN := -fsanitize=kernel-hwaddress \
51	$(call cc-param,hwasan-instrument-stack=$(stack_enable)) \	51	$(call cc-param,hwasan-instrument-stack=$(stack_enable)) \
52	$(call cc-param,hwasan-use-short-granules=0) \	52	$(call cc-param,hwasan-use-short-granules=0) \
		53	$(call cc-param,hwasan-inline-all-checks=0) \
53	$(instrumentation_flags)	54	$(instrumentation_flags)
54		55
55	endif # CONFIG_KASAN_SW_TAGS	56	endif # CONFIG_KASAN_SW_TAGS

kasan: arm64: support specialized outlined tag mismatch checks

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code