// SPDX-License-Identifier: GPL-2.0-only/* * Copyright(c) 2017 Intel Corporation. All rights reserved. */#include <linux/pagemap.h>#include <linux/module.h>#include <linux/mount.h>#include <linux/pseudo_fs.h>#include <linux/magic.h>#include <linux/cdev.h>#include <linux/slab.h>#include <linux/uio.h>#include <linux/dax.h>#include <linux/fs.h>#include <linux/cacheinfo.h>#include "dax-private.h"#include "bus.h" /** * struct dax_device - anchor object for dax services * @inode: core vfs * @cdev: optional character interface for "device dax" * @private: dax driver private data * @flags: state and boolean properties * @ops: operations for this device * @holder_data: holder of a dax_device: could be filesystem or mapped device * @holder_ops: operations for the inner holder */struct dax_device {	struct inode inode;	struct cdev cdev;	void *private;	unsigned long flags;	const struct dax_operations *ops;	void *holder_data;	const struct dax_holder_operations *holder_ops;}; static dev_t dax_devt;DEFINE_STATIC_SRCU(dax_srcu);static struct vfsmount *dax_mnt;static DEFINE_IDA(dax_minor_ida);static struct kmem_cache *dax_cache __read_mostly;static struct super_block *dax_superblock __read_mostly; int dax_read_lock(void){	return srcu_read_lock(&dax_srcu);}EXPORT_SYMBOL_GPL(dax_read_lock); void dax_read_unlock(int id){	srcu_read_unlock(&dax_srcu, id);}EXPORT_SYMBOL_GPL(dax_read_unlock); #if defined(CONFIG_BLOCK) && defined(CONFIG_FS_DAX)#include <linux/blkdev.h> static DEFINE_XARRAY(dax_hosts); int dax_add_host(struct dax_device *dax_dev, struct gendisk *disk){	return xa_insert(&dax_hosts, (unsigned long)disk, dax_dev, GFP_KERNEL);}EXPORT_SYMBOL_GPL(dax_add_host); void dax_remove_host(struct gendisk *disk){	xa_erase(&dax_hosts, (unsigned long)disk);}EXPORT_SYMBOL_GPL(dax_remove_host); /** * fs_dax_get_by_bdev() - temporary lookup mechanism for filesystem-dax * @bdev: block device to find a dax_device for * @start_off: returns the byte offset into the dax_device that @bdev starts * @holder: filesystem or mapped device inside the dax_device * @ops: operations for the inner holder */struct dax_device *fs_dax_get_by_bdev(struct block_device *bdev, u64 *start_off,		void *holder, const struct dax_holder_operations *ops){	struct dax_device *dax_dev;	u64 part_size;	int id; 	if (!blk_queue_dax(bdev->bd_disk->queue))		return NULL; 	*start_off = get_start_sect(bdev) * SECTOR_SIZE;	part_size = bdev_nr_sectors(bdev) * SECTOR_SIZE;	if (*start_off % PAGE_SIZE || part_size % PAGE_SIZE) {		pr_info("%pg: error: unaligned partition for dax\n", bdev);		return NULL;	} 	id = dax_read_lock();	dax_dev = xa_load(&dax_hosts, (unsigned long)bdev->bd_disk);	if (!dax_dev || !dax_alive(dax_dev) || !igrab(&dax_dev->inode))		dax_dev = NULL;	else if (holder) {		if (!cmpxchg(&dax_dev->holder_data, NULL, holder))			dax_dev->holder_ops = ops;		else			dax_dev = NULL;	}	dax_read_unlock(id); 	return dax_dev;}EXPORT_SYMBOL_GPL(fs_dax_get_by_bdev); #endif /* CONFIG_BLOCK && CONFIG_FS_DAX */ #if IS_ENABLED(CONFIG_FS_DAX) /** * fs_put_dax() - release holder ownership of a dax_device * @dax_dev: dax device to release (may be NULL) * @holder: the holder pointer previously passed to fs_dax_get() or *          fs_dax_get_by_bdev(); must match exactly, as it is used *          in a cmpxchg to atomically release ownership * * Must only be called by the current holder. Clears holder_ops before * holder_data to avoid a race where a concurrent fs_dax_get() could have * its newly installed holder_ops overwritten. */void fs_put_dax(struct dax_device *dax_dev, void *holder){	if (dax_dev && holder) {		void *prev; 		/*		 * Clear holder_ops before releasing holder_data. A concurrent		 * dax_holder_notify_failure() that sees NULL ops returns		 * -EOPNOTSUPP cleanly. A concurrent fs_dax_get() that acquires		 * holder_data after the cmpxchg below is guaranteed to observe		 * holder_ops=NULL first (cmpxchg provides release ordering), so		 * its subsequent store of new ops will not be overwritten.		 */		WRITE_ONCE(dax_dev->holder_ops, NULL);		prev = cmpxchg(&dax_dev->holder_data, holder, NULL); 		/*		 * prev == holder: normal release.		 * prev == NULL:   already released by kill_dax() when the		 *                 device was removed under a live holder;		 *                 not a bug.		 * prev != holder (non-NULL): fs_put_dax() called by something		 *                 that is not the current holder; an API		 *                 contract violation. A lock would be needed		 *                 to guard against this, but we WARN_ON()		 *                 instead since violating the contract is		 *                 a bug.		 */		WARN_ON(prev && prev != holder);	}	put_dax(dax_dev);}EXPORT_SYMBOL_GPL(fs_put_dax); /** * fs_dax_get() - get ownership of a devdax via holder/holder_ops * * fs-dax file systems call this function to prepare to use a devdax device for * fsdax. This is like fs_dax_get_by_bdev(), but the caller already has struct * dev_dax (and there is no bdev). The holder makes this exclusive. * * @dax_dev: dev to be prepared for fs-dax usage * @holder: filesystem or mapped device inside the dax_device * @hops: operations for the inner holder * * Returns: 0 on success, <0 on failure */int fs_dax_get(struct dax_device *dax_dev, void *holder,	const struct dax_holder_operations *hops){	struct dev_dax *dev_dax;	struct dax_device_driver *dax_drv;	int id; 	id = dax_read_lock();	if (!dax_dev || !dax_alive(dax_dev) || !igrab(&dax_dev->inode)) {		dax_read_unlock(id);		return -ENODEV;	}	dax_read_unlock(id); 	/* Verify the device is bound to fsdev_dax driver */	dev_dax = dax_get_private(dax_dev);	if (!dev_dax) {		iput(&dax_dev->inode);		return -ENODEV;	} 	device_lock(&dev_dax->dev);	if (!dev_dax->dev.driver) {		device_unlock(&dev_dax->dev);		iput(&dax_dev->inode);		return -ENODEV;	}	dax_drv = to_dax_drv(dev_dax->dev.driver);	if (dax_drv->type != DAXDRV_FSDEV_TYPE) {		device_unlock(&dev_dax->dev);		iput(&dax_dev->inode);		return -EOPNOTSUPP;	}	device_unlock(&dev_dax->dev); 	if (cmpxchg(&dax_dev->holder_data, NULL, holder)) {		iput(&dax_dev->inode);		return -EBUSY;	} 	dax_dev->holder_ops = hops; 	return 0;}EXPORT_SYMBOL_GPL(fs_dax_get);#endif /* CONFIG_FS_DAX */ enum dax_device_flags {	/* !alive + rcu grace period == no new operations / mappings */	DAXDEV_ALIVE,	/* gate whether dax_flush() calls the low level flush routine */	DAXDEV_WRITE_CACHE,	/* flag to check if device supports synchronous flush */	DAXDEV_SYNC,	/* do not leave the caches dirty after writes */	DAXDEV_NOCACHE,	/* handle CPU fetch exceptions during reads */	DAXDEV_NOMC,}; /** * dax_direct_access() - translate a device pgoff to an absolute pfn * @dax_dev: a dax_device instance representing the logical memory range * @pgoff: offset in pages from the start of the device to translate * @nr_pages: number of consecutive pages caller can handle relative to @pfn * @mode: indicator on normal access or recovery write * @kaddr: output parameter that returns a virtual address mapping of pfn * @pfn: output parameter that returns an absolute pfn translation of @pgoff * * Return: negative errno if an error occurs, otherwise the number of * pages accessible at the device relative @pgoff. */long dax_direct_access(struct dax_device *dax_dev, pgoff_t pgoff, long nr_pages,		enum dax_access_mode mode, void **kaddr, unsigned long *pfn){	long avail; 	if (!dax_dev)		return -EOPNOTSUPP; 	if (!dax_alive(dax_dev))		return -ENXIO; 	if (!dax_dev->ops)		return -EOPNOTSUPP; 	if (nr_pages < 0)		return -EINVAL; 	avail = dax_dev->ops->direct_access(dax_dev, pgoff, nr_pages,			mode, kaddr, pfn);	if (!avail)		return -ERANGE;	return min(avail, nr_pages);}EXPORT_SYMBOL_GPL(dax_direct_access); size_t dax_copy_from_iter(struct dax_device *dax_dev, pgoff_t pgoff, void *addr,		size_t bytes, struct iov_iter *i){	if (!dax_alive(dax_dev))		return 0; 	/*	 * The userspace address for the memory copy has already been validated	 * via access_ok() in vfs_write, so use the 'no check' version to bypass	 * the HARDENED_USERCOPY overhead.	 */	if (test_bit(DAXDEV_NOCACHE, &dax_dev->flags))		return _copy_from_iter_flushcache(addr, bytes, i);	return _copy_from_iter(addr, bytes, i);} size_t dax_copy_to_iter(struct dax_device *dax_dev, pgoff_t pgoff, void *addr,		size_t bytes, struct iov_iter *i){	if (!dax_alive(dax_dev))		return 0; 	/*	 * The userspace address for the memory copy has already been validated	 * via access_ok() in vfs_red, so use the 'no check' version to bypass	 * the HARDENED_USERCOPY overhead.	 */	if (test_bit(DAXDEV_NOMC, &dax_dev->flags))		return _copy_mc_to_iter(addr, bytes, i);	return _copy_to_iter(addr, bytes, i);} int dax_zero_page_range(struct dax_device *dax_dev, pgoff_t pgoff,			size_t nr_pages){	int ret; 	if (!dax_alive(dax_dev))		return -ENXIO; 	if (!dax_dev->ops)		return -EOPNOTSUPP; 	/*	 * There are no callers that want to zero more than one page as of now.	 * Once users are there, this check can be removed after the	 * device mapper code has been updated to split ranges across targets.	 */	if (nr_pages != 1)		return -EIO; 	ret = dax_dev->ops->zero_page_range(dax_dev, pgoff, nr_pages);	return dax_mem2blk_err(ret);}EXPORT_SYMBOL_GPL(dax_zero_page_range); size_t dax_recovery_write(struct dax_device *dax_dev, pgoff_t pgoff,		void *addr, size_t bytes, struct iov_iter *iter){	if (!dax_dev->ops || !dax_dev->ops->recovery_write)		return 0;	return dax_dev->ops->recovery_write(dax_dev, pgoff, addr, bytes, iter);}EXPORT_SYMBOL_GPL(dax_recovery_write); int dax_holder_notify_failure(struct dax_device *dax_dev, u64 off,			      u64 len, int mf_flags){	const struct dax_holder_operations *ops;	int rc, id; 	id = dax_read_lock();	if (!dax_alive(dax_dev)) {		rc = -ENXIO;		goto out;	} 	/*	 * Read holder_ops once: a concurrent fs_put_dax() can clear it without	 * synchronizing against readers. Without the single fetch the compiler	 * could reload between the NULL check and the call and dereference a	 * NULL ops.	 */	ops = READ_ONCE(dax_dev->holder_ops);	if (!ops) {		rc = -EOPNOTSUPP;		goto out;	} 	rc = ops->notify_failure(dax_dev, off, len, mf_flags);out:	dax_read_unlock(id);	return rc;}EXPORT_SYMBOL_GPL(dax_holder_notify_failure); #ifdef CONFIG_ARCH_HAS_PMEM_APIvoid arch_wb_cache_pmem(void *addr, size_t size);void dax_flush(struct dax_device *dax_dev, void *addr, size_t size){	if (unlikely(!dax_write_cache_enabled(dax_dev)))		return; 	arch_wb_cache_pmem(addr, size);}#elsevoid dax_flush(struct dax_device *dax_dev, void *addr, size_t size){}#endifEXPORT_SYMBOL_GPL(dax_flush); void dax_write_cache(struct dax_device *dax_dev, bool wc){	if (wc)		set_bit(DAXDEV_WRITE_CACHE, &dax_dev->flags);	else		clear_bit(DAXDEV_WRITE_CACHE, &dax_dev->flags);}EXPORT_SYMBOL_GPL(dax_write_cache); bool dax_write_cache_enabled(struct dax_device *dax_dev){	return test_bit(DAXDEV_WRITE_CACHE, &dax_dev->flags);}EXPORT_SYMBOL_GPL(dax_write_cache_enabled); bool dax_synchronous(struct dax_device *dax_dev){	return test_bit(DAXDEV_SYNC, &dax_dev->flags);}EXPORT_SYMBOL_GPL(dax_synchronous); void set_dax_synchronous(struct dax_device *dax_dev){	set_bit(DAXDEV_SYNC, &dax_dev->flags);}EXPORT_SYMBOL_GPL(set_dax_synchronous); void set_dax_nocache(struct dax_device *dax_dev){	set_bit(DAXDEV_NOCACHE, &dax_dev->flags);}EXPORT_SYMBOL_GPL(set_dax_nocache); void set_dax_nomc(struct dax_device *dax_dev){	set_bit(DAXDEV_NOMC, &dax_dev->flags);}EXPORT_SYMBOL_GPL(set_dax_nomc); /** * dax_set_ops - set the dax_operations for a dax_device * @dax_dev: the dax_device to configure * @ops: the operations to set (may be NULL to clear) * * This allows drivers to set the dax_operations after the dax_device * has been allocated. This is needed when the device is created before * the driver that needs specific ops is bound (e.g., fsdev_dax binding * to a dev_dax created by hmem). * * When setting non-NULL ops, fails if ops are already set (returns -EBUSY). * When clearing ops (NULL), always succeeds. * * Return: 0 on success, -EBUSY if ops already set */int dax_set_ops(struct dax_device *dax_dev, const struct dax_operations *ops){	if (ops) {		/* Setting ops: fail if already set */		if (cmpxchg(&dax_dev->ops, NULL, ops) != NULL)			return -EBUSY;	} else {		/* Clearing ops: always allowed */		dax_dev->ops = NULL;	}	return 0;}EXPORT_SYMBOL_GPL(dax_set_ops); bool dax_alive(struct dax_device *dax_dev){	lockdep_assert_held(&dax_srcu);	return test_bit(DAXDEV_ALIVE, &dax_dev->flags);}EXPORT_SYMBOL_GPL(dax_alive); /* * Note, rcu is not protecting the liveness of dax_dev, rcu is ensuring * that any fault handlers or operations that might have seen * dax_alive(), have completed.  Any operations that start after * synchronize_srcu() has run will abort upon seeing !dax_alive(). * * Note, because alloc_dax() returns an ERR_PTR() on error, callers * typically store its result into a local variable in order to check * the result. Therefore, care must be taken to populate the struct * device dax_dev field make sure the dax_dev is not leaked. */void kill_dax(struct dax_device *dax_dev){	if (!dax_dev)		return; 	if (dax_dev->holder_data != NULL)		dax_holder_notify_failure(dax_dev, 0, U64_MAX,				MF_MEM_PRE_REMOVE); 	clear_bit(DAXDEV_ALIVE, &dax_dev->flags);	synchronize_srcu(&dax_srcu); 	/* clear holder data */	dax_dev->holder_ops = NULL;	dax_dev->holder_data = NULL;}EXPORT_SYMBOL_GPL(kill_dax); void run_dax(struct dax_device *dax_dev){	set_bit(DAXDEV_ALIVE, &dax_dev->flags);}EXPORT_SYMBOL_GPL(run_dax); static struct inode *dax_alloc_inode(struct super_block *sb){	struct dax_device *dax_dev;	struct inode *inode; 

Lines 1–500 of 756. Load more