// SPDX-License-Identifier: GPL-2.0-only/* * VDSO implementations. * * Copyright (C) 2012 ARM Limited * * Author: Will Deacon <will.deacon@arm.com> */ #include <linux/cache.h>#include <linux/clocksource.h>#include <linux/elf.h>#include <linux/err.h>#include <linux/errno.h>#include <linux/gfp.h>#include <linux/kernel.h>#include <linux/mm.h>#include <linux/sched.h>#include <linux/signal.h>#include <linux/slab.h>#include <linux/vdso_datastore.h>#include <linux/vmalloc.h>#include <vdso/datapage.h>#include <vdso/helpers.h>#include <vdso/vsyscall.h> #include <asm/cacheflush.h>#include <asm/signal32.h>#include <asm/vdso.h> enum vdso_abi {	VDSO_ABI_AA64,	VDSO_ABI_AA32,}; struct vdso_abi_info {	const char *name;	const char *vdso_code_start;	const char *vdso_code_end;	unsigned long vdso_pages;	/* Code Mapping */	struct vm_special_mapping *cm;}; static struct vdso_abi_info vdso_info[] __ro_after_init = {	[VDSO_ABI_AA64] = {		.name = "vdso",		.vdso_code_start = vdso_start,		.vdso_code_end = vdso_end,	},#ifdef CONFIG_COMPAT_VDSO	[VDSO_ABI_AA32] = {		.name = "vdso32",		.vdso_code_start = vdso32_start,		.vdso_code_end = vdso32_end,	},#endif /* CONFIG_COMPAT_VDSO */}; static int vdso_mremap(const struct vm_special_mapping *sm,		struct vm_area_struct *new_vma){	current->mm->context.vdso = (void *)new_vma->vm_start; 	return 0;} static int __init __vdso_init(enum vdso_abi abi){	int i;	struct page **vdso_pagelist;	unsigned long pfn; 	if (memcmp(vdso_info[abi].vdso_code_start, "\177ELF", 4)) {		pr_err("vDSO is not a valid ELF object!\n");		return -EINVAL;	} 	vdso_info[abi].vdso_pages = (			vdso_info[abi].vdso_code_end -			vdso_info[abi].vdso_code_start) >>			PAGE_SHIFT; 	vdso_pagelist = kzalloc_objs(struct page *, vdso_info[abi].vdso_pages);	if (vdso_pagelist == NULL)		return -ENOMEM; 	/* Grab the vDSO code pages. */	pfn = sym_to_pfn(vdso_info[abi].vdso_code_start); 	for (i = 0; i < vdso_info[abi].vdso_pages; i++)		vdso_pagelist[i] = pfn_to_page(pfn + i); 	vdso_info[abi].cm->pages = vdso_pagelist; 	return 0;} static int __setup_additional_pages(enum vdso_abi abi,				    struct mm_struct *mm,				    struct linux_binprm *bprm,				    int uses_interp){	unsigned long vdso_base, vdso_text_len, vdso_mapping_len;	unsigned long gp_flags = 0;	void *ret; 	BUILD_BUG_ON(VDSO_NR_PAGES != __VDSO_PAGES); 	vdso_text_len = vdso_info[abi].vdso_pages << PAGE_SHIFT;	/* Be sure to map the data page */	vdso_mapping_len = vdso_text_len + VDSO_NR_PAGES * PAGE_SIZE; 	vdso_base = get_unmapped_area(NULL, 0, vdso_mapping_len, 0, 0);	if (IS_ERR_VALUE(vdso_base)) {		ret = ERR_PTR(vdso_base);		goto up_fail;	} 	ret = vdso_install_vvar_mapping(mm, vdso_base);	if (IS_ERR(ret))		goto up_fail; 	if (system_supports_bti_kernel())		gp_flags = VM_ARM64_BTI; 	vdso_base += VDSO_NR_PAGES * PAGE_SIZE;	mm->context.vdso = (void *)vdso_base;	ret = _install_special_mapping(mm, vdso_base, vdso_text_len,				       VM_READ|VM_EXEC|gp_flags|				       VM_MAYREAD|VM_MAYWRITE|VM_MAYEXEC|				       VM_SEALED_SYSMAP,				       vdso_info[abi].cm);	if (IS_ERR(ret))		goto up_fail; 	return 0; up_fail:	mm->context.vdso = NULL;	return PTR_ERR(ret);} #ifdef CONFIG_COMPAT/* * Create and map the vectors page for AArch32 tasks. */enum aarch32_map {	AA32_MAP_VECTORS, /* kuser helpers */	AA32_MAP_SIGPAGE,	AA32_MAP_VDSO,}; static struct page *aarch32_vectors_page __ro_after_init;static struct page *aarch32_sig_page __ro_after_init; static int aarch32_sigpage_mremap(const struct vm_special_mapping *sm,				  struct vm_area_struct *new_vma){	current->mm->context.sigpage = (void *)new_vma->vm_start; 	return 0;} static struct vm_special_mapping aarch32_vdso_maps[] = {	[AA32_MAP_VECTORS] = {		.name	= "[vectors]", /* ABI */		.pages	= &aarch32_vectors_page,	},	[AA32_MAP_SIGPAGE] = {		.name	= "[sigpage]", /* ABI */		.pages	= &aarch32_sig_page,		.mremap	= aarch32_sigpage_mremap,	},	[AA32_MAP_VDSO] = {		.name = "[vdso]",		.mremap = vdso_mremap,	},}; static int aarch32_alloc_kuser_vdso_page(void){	extern char __kuser_helper_start[], __kuser_helper_end[];	int kuser_sz = __kuser_helper_end - __kuser_helper_start;	unsigned long vdso_page; 	if (!IS_ENABLED(CONFIG_KUSER_HELPERS))		return 0; 	vdso_page = get_zeroed_page(GFP_KERNEL);	if (!vdso_page)		return -ENOMEM; 	memcpy((void *)(vdso_page + 0x1000 - kuser_sz), __kuser_helper_start,	       kuser_sz);	aarch32_vectors_page = virt_to_page((void *)vdso_page);	return 0;} #define COMPAT_SIGPAGE_POISON_WORD	0xe7fddef1static int aarch32_alloc_sigpage(void){	extern char __aarch32_sigret_code_start[], __aarch32_sigret_code_end[];	int sigret_sz = __aarch32_sigret_code_end - __aarch32_sigret_code_start;	__le32 poison = cpu_to_le32(COMPAT_SIGPAGE_POISON_WORD);	void *sigpage; 	sigpage = (void *)__get_free_page(GFP_KERNEL);	if (!sigpage)		return -ENOMEM; 	memset32(sigpage, (__force u32)poison, PAGE_SIZE / sizeof(poison));	memcpy(sigpage, __aarch32_sigret_code_start, sigret_sz);	aarch32_sig_page = virt_to_page(sigpage);	return 0;} static int __init __aarch32_alloc_vdso_pages(void){ 	if (!IS_ENABLED(CONFIG_COMPAT_VDSO))		return 0; 	vdso_info[VDSO_ABI_AA32].cm = &aarch32_vdso_maps[AA32_MAP_VDSO]; 	return __vdso_init(VDSO_ABI_AA32);} static int __init aarch32_alloc_vdso_pages(void){	int ret; 	ret = __aarch32_alloc_vdso_pages();	if (ret)		return ret; 	ret = aarch32_alloc_sigpage();	if (ret)		return ret; 	return aarch32_alloc_kuser_vdso_page();}arch_initcall(aarch32_alloc_vdso_pages); static int aarch32_kuser_helpers_setup(struct mm_struct *mm){	void *ret; 	if (!IS_ENABLED(CONFIG_KUSER_HELPERS))		return 0; 	/*	 * Avoid VM_MAYWRITE for compatibility with arch/arm/, where it's	 * not safe to CoW the page containing the CPU exception vectors.	 */	ret = _install_special_mapping(mm, AARCH32_VECTORS_BASE, PAGE_SIZE,				       VM_READ | VM_EXEC |				       VM_MAYREAD | VM_MAYEXEC |				       VM_SEALED_SYSMAP,				       &aarch32_vdso_maps[AA32_MAP_VECTORS]); 	return PTR_ERR_OR_ZERO(ret);} static int aarch32_sigreturn_setup(struct mm_struct *mm){	unsigned long addr;	void *ret; 	addr = get_unmapped_area(NULL, 0, PAGE_SIZE, 0, 0);	if (IS_ERR_VALUE(addr)) {		ret = ERR_PTR(addr);		goto out;	} 	/*	 * VM_MAYWRITE is required to allow gdb to Copy-on-Write and	 * set breakpoints.	 */	ret = _install_special_mapping(mm, addr, PAGE_SIZE,				       VM_READ | VM_EXEC | VM_MAYREAD |				       VM_MAYWRITE | VM_MAYEXEC |				       VM_SEALED_SYSMAP,				       &aarch32_vdso_maps[AA32_MAP_SIGPAGE]);	if (IS_ERR(ret))		goto out; 	mm->context.sigpage = (void *)addr; out:	return PTR_ERR_OR_ZERO(ret);} int aarch32_setup_additional_pages(struct linux_binprm *bprm, int uses_interp){	struct mm_struct *mm = current->mm;	int ret; 	if (mmap_write_lock_killable(mm))		return -EINTR; 	ret = aarch32_kuser_helpers_setup(mm);	if (ret)		goto out; 	if (IS_ENABLED(CONFIG_COMPAT_VDSO)) {		ret = __setup_additional_pages(VDSO_ABI_AA32, mm, bprm,					       uses_interp);		if (ret)			goto out;	} 	ret = aarch32_sigreturn_setup(mm);out:	mmap_write_unlock(mm);	return ret;}#endif /* CONFIG_COMPAT */ static struct vm_special_mapping aarch64_vdso_map __ro_after_init = {	.name	= "[vdso]",	.mremap = vdso_mremap,}; static int __init vdso_init(void){	vdso_info[VDSO_ABI_AA64].cm = &aarch64_vdso_map; 	return __vdso_init(VDSO_ABI_AA64);}arch_initcall(vdso_init); int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp){	struct mm_struct *mm = current->mm;	int ret; 	if (mmap_write_lock_killable(mm))		return -EINTR; 	ret = __setup_additional_pages(VDSO_ABI_AA64, mm, bprm, uses_interp);	mmap_write_unlock(mm); 	return ret;}