// SPDX-License-Identifier: GPL-2.0/* Copyright(c) 2016-2018 Intel Corporation. All rights reserved. */#include <linux/memremap.h>#include <linux/pagemap.h>#include <linux/module.h>#include <linux/device.h>#include <linux/cdev.h>#include <linux/slab.h>#include <linux/dax.h>#include <linux/fs.h>#include <linux/mm.h>#include <linux/mman.h>#include "dax-private.h"#include "bus.h" static int __check_vma(struct dev_dax *dev_dax, vma_flags_t flags,		       unsigned long start, unsigned long end, struct file *file,		       const char *func){	struct device *dev = &dev_dax->dev;	unsigned long mask; 	if (!dax_alive(dev_dax->dax_dev))		return -ENXIO; 	/* prevent private mappings from being established */	if (!vma_flags_test_any(&flags, VMA_MAYSHARE_BIT)) {		dev_info_ratelimited(dev,				"%s: %s: fail, attempted private mapping\n",				current->comm, func);		return -EINVAL;	} 	mask = dev_dax->align - 1;	if (start & mask || end & mask) {		dev_info_ratelimited(dev,				"%s: %s: fail, unaligned vma (%#lx - %#lx, %#lx)\n",				current->comm, func, start, end,				mask);		return -EINVAL;	} 	if (!file_is_dax(file)) {		dev_info_ratelimited(dev,				"%s: %s: fail, vma is not DAX capable\n",				current->comm, func);		return -EINVAL;	} 	return 0;} static int check_vma(struct dev_dax *dev_dax, struct vm_area_struct *vma,		     const char *func){	return __check_vma(dev_dax, vma->flags, vma->vm_start, vma->vm_end,			   vma->vm_file, func);} static void dax_set_mapping(struct vm_fault *vmf, unsigned long pfn,			      unsigned long fault_size){	unsigned long i, nr_pages = fault_size / PAGE_SIZE;	struct file *filp = vmf->vma->vm_file;	struct dev_dax *dev_dax = filp->private_data;	pgoff_t pgoff; 	/* mapping is only set on the head */	if (dev_dax->pgmap->vmemmap_shift)		nr_pages = 1; 	pgoff = linear_page_index(vmf->vma,			ALIGN_DOWN(vmf->address, fault_size)); 	for (i = 0; i < nr_pages; i++) {		struct folio *folio = pfn_folio(pfn + i); 		if (folio->mapping)			continue; 		folio->mapping = filp->f_mapping;		folio->index = pgoff + i;	}} static vm_fault_t __dev_dax_pte_fault(struct dev_dax *dev_dax,				struct vm_fault *vmf){	struct device *dev = &dev_dax->dev;	phys_addr_t phys;	unsigned long pfn;	unsigned int fault_size = PAGE_SIZE; 	if (check_vma(dev_dax, vmf->vma, __func__))		return VM_FAULT_SIGBUS; 	if (dev_dax->align > PAGE_SIZE) {		dev_dbg(dev, "alignment (%#x) > fault size (%#x)\n",			dev_dax->align, fault_size);		return VM_FAULT_SIGBUS;	} 	if (fault_size != dev_dax->align)		return VM_FAULT_SIGBUS; 	phys = dax_pgoff_to_phys(dev_dax, vmf->pgoff, PAGE_SIZE);	if (phys == -1) {		dev_dbg(dev, "pgoff_to_phys(%#lx) failed\n", vmf->pgoff);		return VM_FAULT_SIGBUS;	} 	pfn = PHYS_PFN(phys); 	dax_set_mapping(vmf, pfn, fault_size); 	return vmf_insert_page_mkwrite(vmf, pfn_to_page(pfn),					vmf->flags & FAULT_FLAG_WRITE);} static vm_fault_t __dev_dax_pmd_fault(struct dev_dax *dev_dax,				struct vm_fault *vmf){	unsigned long pmd_addr = vmf->address & PMD_MASK;	struct device *dev = &dev_dax->dev;	phys_addr_t phys;	pgoff_t pgoff;	unsigned long pfn;	unsigned int fault_size = PMD_SIZE; 	if (check_vma(dev_dax, vmf->vma, __func__))		return VM_FAULT_SIGBUS; 	if (dev_dax->align > PMD_SIZE) {		dev_dbg(dev, "alignment (%#x) > fault size (%#x)\n",			dev_dax->align, fault_size);		return VM_FAULT_SIGBUS;	} 	if (fault_size < dev_dax->align)		return VM_FAULT_SIGBUS;	else if (fault_size > dev_dax->align)		return VM_FAULT_FALLBACK; 	/* if we are outside of the VMA */	if (pmd_addr < vmf->vma->vm_start ||			(pmd_addr + PMD_SIZE) > vmf->vma->vm_end)		return VM_FAULT_SIGBUS; 	pgoff = linear_page_index(vmf->vma, pmd_addr);	phys = dax_pgoff_to_phys(dev_dax, pgoff, PMD_SIZE);	if (phys == -1) {		dev_dbg(dev, "pgoff_to_phys(%#lx) failed\n", pgoff);		return VM_FAULT_SIGBUS;	} 	pfn = PHYS_PFN(phys); 	dax_set_mapping(vmf, pfn, fault_size); 	return vmf_insert_folio_pmd(vmf, page_folio(pfn_to_page(pfn)),				vmf->flags & FAULT_FLAG_WRITE);} #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUDstatic vm_fault_t __dev_dax_pud_fault(struct dev_dax *dev_dax,				struct vm_fault *vmf){	unsigned long pud_addr = vmf->address & PUD_MASK;	struct device *dev = &dev_dax->dev;	phys_addr_t phys;	pgoff_t pgoff;	unsigned long pfn;	unsigned int fault_size = PUD_SIZE;  	if (check_vma(dev_dax, vmf->vma, __func__))		return VM_FAULT_SIGBUS; 	if (dev_dax->align > PUD_SIZE) {		dev_dbg(dev, "alignment (%#x) > fault size (%#x)\n",			dev_dax->align, fault_size);		return VM_FAULT_SIGBUS;	} 	if (fault_size < dev_dax->align)		return VM_FAULT_SIGBUS;	else if (fault_size > dev_dax->align)		return VM_FAULT_FALLBACK; 	/* if we are outside of the VMA */	if (pud_addr < vmf->vma->vm_start ||			(pud_addr + PUD_SIZE) > vmf->vma->vm_end)		return VM_FAULT_SIGBUS; 	pgoff = linear_page_index(vmf->vma, pud_addr);	phys = dax_pgoff_to_phys(dev_dax, pgoff, PUD_SIZE);	if (phys == -1) {		dev_dbg(dev, "pgoff_to_phys(%#lx) failed\n", pgoff);		return VM_FAULT_SIGBUS;	} 	pfn = PHYS_PFN(phys); 	dax_set_mapping(vmf, pfn, fault_size); 	return vmf_insert_folio_pud(vmf, page_folio(pfn_to_page(pfn)),				vmf->flags & FAULT_FLAG_WRITE);}#elsestatic vm_fault_t __dev_dax_pud_fault(struct dev_dax *dev_dax,				struct vm_fault *vmf){	return VM_FAULT_FALLBACK;}#endif /* !CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */ static vm_fault_t dev_dax_huge_fault(struct vm_fault *vmf, unsigned int order){	struct file *filp = vmf->vma->vm_file;	vm_fault_t rc = VM_FAULT_SIGBUS;	int id;	struct dev_dax *dev_dax = filp->private_data; 	dev_dbg(&dev_dax->dev, "%s: op=%s addr=%#lx order=%d\n", current->comm,		(vmf->flags & FAULT_FLAG_WRITE) ? "write" : "read",		vmf->address & ~((1UL << (order + PAGE_SHIFT)) - 1), order); 	id = dax_read_lock();	if (order == 0)		rc = __dev_dax_pte_fault(dev_dax, vmf);	else if (order == PMD_ORDER)		rc = __dev_dax_pmd_fault(dev_dax, vmf);	else if (order == PUD_ORDER)		rc = __dev_dax_pud_fault(dev_dax, vmf);	else		rc = VM_FAULT_SIGBUS; 	dax_read_unlock(id); 	return rc;} static vm_fault_t dev_dax_fault(struct vm_fault *vmf){	return dev_dax_huge_fault(vmf, 0);} static int dev_dax_may_split(struct vm_area_struct *vma, unsigned long addr){	struct file *filp = vma->vm_file;	struct dev_dax *dev_dax = filp->private_data; 	if (!IS_ALIGNED(addr, dev_dax->align))		return -EINVAL;	return 0;} static unsigned long dev_dax_pagesize(struct vm_area_struct *vma){	struct file *filp = vma->vm_file;	struct dev_dax *dev_dax = filp->private_data; 	return dev_dax->align;} static const struct vm_operations_struct dax_vm_ops = {	.fault = dev_dax_fault,	.huge_fault = dev_dax_huge_fault,	.may_split = dev_dax_may_split,	.pagesize = dev_dax_pagesize,}; static int dax_mmap_prepare(struct vm_area_desc *desc){	struct file *filp = desc->file;	struct dev_dax *dev_dax = filp->private_data;	int rc, id; 	dev_dbg(&dev_dax->dev, "trace\n"); 	/*	 * We lock to check dax_dev liveness and will re-check at	 * fault time.	 */	id = dax_read_lock();	rc = __check_vma(dev_dax, desc->vma_flags, desc->start, desc->end, filp,			 __func__);	dax_read_unlock(id);	if (rc)		return rc; 	desc->vm_ops = &dax_vm_ops;	vma_desc_set_flags(desc, VMA_HUGEPAGE_BIT);	return 0;} /* return an unmapped area aligned to the dax region specified alignment */static unsigned long dax_get_unmapped_area(struct file *filp,		unsigned long addr, unsigned long len, unsigned long pgoff,		unsigned long flags){	unsigned long off, off_end, off_align, len_align, addr_align, align;	struct dev_dax *dev_dax = filp ? filp->private_data : NULL; 	if (!dev_dax || addr)		goto out; 	align = dev_dax->align;	off = pgoff << PAGE_SHIFT;	off_end = off + len;	off_align = round_up(off, align); 	if ((off_end <= off_align) || ((off_end - off_align) < align))		goto out; 	len_align = len + align;	if ((off + len_align) < off)		goto out; 	addr_align = mm_get_unmapped_area(filp, addr, len_align, pgoff, flags);	if (!IS_ERR_VALUE(addr_align)) {		addr_align += (off - addr_align) & (align - 1);		return addr_align;	} out:	return mm_get_unmapped_area(filp, addr, len, pgoff, flags);} static const struct address_space_operations dev_dax_aops = {	.dirty_folio	= noop_dirty_folio,}; static int dax_open(struct inode *inode, struct file *filp){	struct dax_device *dax_dev = inode_dax(inode);	struct inode *__dax_inode = dax_inode(dax_dev);	struct dev_dax *dev_dax = dax_get_private(dax_dev); 	dev_dbg(&dev_dax->dev, "trace\n");	inode->i_mapping = __dax_inode->i_mapping;	inode->i_mapping->host = __dax_inode;	inode->i_mapping->a_ops = &dev_dax_aops;	filp->f_mapping = inode->i_mapping;	filp->f_wb_err = filemap_sample_wb_err(filp->f_mapping);	filp->f_sb_err = file_sample_sb_err(filp);	filp->private_data = dev_dax;	inode->i_flags = S_DAX; 	return 0;} static int dax_release(struct inode *inode, struct file *filp){	struct dev_dax *dev_dax = filp->private_data; 	dev_dbg(&dev_dax->dev, "trace\n");	return 0;} static const struct file_operations dax_fops = {	.llseek = noop_llseek,	.owner = THIS_MODULE,	.open = dax_open,	.release = dax_release,	.get_unmapped_area = dax_get_unmapped_area,	.mmap_prepare = dax_mmap_prepare,	.fop_flags = FOP_MMAP_SYNC,}; static void dev_dax_cdev_del(void *cdev){	cdev_del(cdev);} static void dev_dax_kill(void *dev_dax){	kill_dev_dax(dev_dax);} static int dev_dax_probe(struct dev_dax *dev_dax){	struct dax_device *dax_dev = dev_dax->dax_dev;	struct device *dev = &dev_dax->dev;	struct dev_pagemap *pgmap;	struct inode *inode;	struct cdev *cdev;	void *addr;	int rc, i; 	if (static_dev_dax(dev_dax))  {		if (dev_dax->nr_range > 1) {			dev_warn(dev,				"static pgmap / multi-range device conflict\n");			return -EINVAL;		} 		pgmap = dev_dax->pgmap;	} else {		if (dev_dax->pgmap) {			dev_warn(dev,				 "dynamic-dax with pre-populated page map\n");			return -EINVAL;		} 		pgmap = devm_kzalloc(dev,                       struct_size(pgmap, ranges, dev_dax->nr_range - 1),                       GFP_KERNEL);		if (!pgmap)			return -ENOMEM; 		pgmap->nr_range = dev_dax->nr_range;		dev_dax->pgmap = pgmap; 		for (i = 0; i < dev_dax->nr_range; i++) {			struct range *range = &dev_dax->ranges[i].range;			pgmap->ranges[i] = *range;		}	} 	for (i = 0; i < dev_dax->nr_range; i++) {		struct range *range = &dev_dax->ranges[i].range; 		if (!devm_request_mem_region(dev, range->start,					range_len(range), dev_name(dev))) {			dev_warn(dev, "mapping%d: %#llx-%#llx could not reserve range\n",					i, range->start, range->end);			return -EBUSY;		}	} 	pgmap->type = MEMORY_DEVICE_GENERIC;	if (dev_dax->align > PAGE_SIZE)		pgmap->vmemmap_shift =			order_base_2(dev_dax->align >> PAGE_SHIFT);	addr = devm_memremap_pages(dev, pgmap);	if (IS_ERR(addr))		return PTR_ERR(addr); 	inode = dax_inode(dax_dev);	cdev = inode->i_cdev;	cdev_init(cdev, &dax_fops);	cdev->owner = dev->driver->owner;	cdev_set_parent(cdev, &dev->kobj);	rc = cdev_add(cdev, dev->devt, 1);	if (rc)		return rc; 	rc = devm_add_action_or_reset(dev, dev_dax_cdev_del, cdev);	if (rc)		return rc; 	run_dax(dax_dev);	return devm_add_action_or_reset(dev, dev_dax_kill, dev_dax);} static struct dax_device_driver device_dax_driver = {	.probe = dev_dax_probe,	.type = DAXDRV_DEVICE_TYPE,}; static int __init dax_init(void){	return dax_driver_register(&device_dax_driver);} static void __exit dax_exit(void){	dax_driver_unregister(&device_dax_driver);} MODULE_AUTHOR("Intel Corporation");MODULE_DESCRIPTION("Device DAX: direct access device driver");MODULE_LICENSE("GPL v2");module_init(dax_init);module_exit(dax_exit);MODULE_ALIAS_DAX_DEVICE(0);