Merge tag 'libnvdimm-for-7.1' of git://git.kernel.org/pub/scm/linux/kernel/git/nvdimm/nvdimm

Pull dax updates from Ira Weiny: "The series adds DAX support required for the upcoming fuse/famfs file system.[1] The support here is required because famfs is backed by devdax rather than pmem. This all lays the groundwork for using shared memory as a file system" Link: https://lore.kernel.org/all/0100019d43e5f632-f5862a3e-361c-4b54-a9a6-96c242a8f17a-000000@email.amazonses.com/ [1] * tag 'libnvdimm-for-7.1' of git://git.kernel.org/pub/scm/linux/kernel/git/nvdimm/nvdimm: dax/fsdev: fix uninitialized kaddr in fsdev_dax_zero_page_range() dax: export dax_dev_get() dax: Add fs_dax_get() func to prepare dax for fs-dax usage dax: Add dax_set_ops() for setting dax_operations at bind time dax: Add dax_operations for use by fs-dax on fsdev dax dax: Save the kva from memremap dax: add fsdev.c driver for fs-dax on character dax dax: Factor out dax_folio_reset_order() helper dax: move dax_pgoff_to_phys from [drivers/dax/] device.c to bus.c

Linus Torvalds committed 6 months ago

Commit bb0bc492 parents c94faa7 45df91111 files · +566 −50Browse files

Unified · Split

11 files

MAINTAINERS +8 −0
drivers/dax/Kconfig +5 −0
drivers/dax/Makefile +2 −0
drivers/dax/bus.c +20 −2
drivers/dax/bus.h +3 −0
drivers/dax/dax-private.h +4 −0
drivers/dax/device.c +0 −23
drivers/dax/fsdev.c +349 −0
drivers/dax/super.c +104 −3
fs/dax.c +56 −18
include/linux/dax.h +15 −4

MAINTAINERS+8−0

@@ -7303,6 +7303,14 @@ L: linux-cxl@vger.kernel.org · 7302 unmodified lines
7303	7303	S: Supported
7304	7304	F: drivers/dax/
7305	7305
	7306	+DEVICE DIRECT ACCESS (DAX) [fsdev_dax]
	7307	+M: John Groves <jgroves@micron.com>
	7308	+M: John Groves <John@Groves.net>
	7309	+L: nvdimm@lists.linux.dev
	7310	+L: linux-cxl@vger.kernel.org
	7311	+S: Supported
	7312	+F: drivers/dax/fsdev.c
	7313	+
7306	7314	DEVICE FREQUENCY (DEVFREQ)
7307	7315	M: MyungJoo Ham <myungjoo.ham@samsung.com>
7308	7316	M: Kyungmin Park <kyungmin.park@samsung.com>

drivers/dax/Kconfig+5−0

@@ -65,6 +65,11 @@ config DEV_DAX_HMEM_DEVICES · 64 unmodified lines
65	65	depends on DEV_DAX_HMEM && DAX
66	66	def_bool y
67	67
	68	+config DEV_DAX_FSDEV
	69	+ tristate
	70	+ depends on DEV_DAX && FS_DAX
	71	+ default DEV_DAX
	72	+
68	73	config DEV_DAX_KMEM
69	74	tristate "KMEM DAX: map dax-devices as System-RAM"
70	75	default DEV_DAX

drivers/dax/Makefile+2−0

@@ -5,9 +5,11 @@ obj-$(CONFIG_DEV_DAX) += device_dax.o · 4 unmodified lines
5	5	obj-$(CONFIG_DEV_DAX_KMEM) += kmem.o
6	6	obj-$(CONFIG_DEV_DAX_PMEM) += dax_pmem.o
7	7	obj-$(CONFIG_DEV_DAX_CXL) += dax_cxl.o
	8	+obj-$(CONFIG_DEV_DAX_FSDEV) += fsdev_dax.o
8	9
9	10	dax-y := super.o
10	11	dax-y += bus.o
11	12	device_dax-y := device.o
12	13	dax_pmem-y := pmem.o
13	14	dax_cxl-y := cxl.o
	15	+fsdev_dax-y := fsdev.o

drivers/dax/bus.c+20−2

@@ -40,8 +40,6 @@ static int dax_bus_uevent(const struct device dev, struct kobj_uevent_env env) · 39 unmodified lines
40	40	return add_uevent_var(env, "MODALIAS=" DAX_DEVICE_MODALIAS_FMT, 0);
41	41	}
42	42
43		−#define to_dax_drv(__drv) container_of_const(__drv, struct dax_device_driver, drv)
44		−
45	43	static struct dax_id __dax_match_id(const struct dax_device_driver dax_drv,
46	44	const char *dev_name)
47	45	{
@@ -1431,6 +1429,26 @@ static const struct device_type dev_dax_type = { · 1383 unmodified lines
1431	1429	.groups = dax_attribute_groups,
1432	1430	};
1433	1431
	1432	+/* see "strong" declaration in tools/testing/nvdimm/dax-dev.c */
	1433	+__weak phys_addr_t dax_pgoff_to_phys(struct dev_dax *dev_dax, pgoff_t pgoff,
	1434	+ unsigned long size)
	1435	+{
	1436	+ for (int i = 0; i < dev_dax->nr_range; i++) {
	1437	+ struct dev_dax_range *dax_range = &dev_dax->ranges[i];
	1438	+ struct range *range = &dax_range->range;
	1439	+ phys_addr_t phys;
	1440	+
	1441	+ if (!in_range(pgoff, dax_range->pgoff, PHYS_PFN(range_len(range))))
	1442	+ continue;
	1443	+ phys = PFN_PHYS(pgoff - dax_range->pgoff) + range->start;
	1444	+ if (phys + size - 1 <= range->end)
	1445	+ return phys;
	1446	+ break;
	1447	+ }
	1448	+ return -1;
	1449	+}
	1450	+EXPORT_SYMBOL_GPL(dax_pgoff_to_phys);
	1451	+
1434	1452	static struct dev_dax __devm_create_dev_dax(struct dev_dax_data data)
1435	1453	{
1436	1454	struct dax_region *dax_region = data->dax_region;

drivers/dax/bus.h+3−0

@@ -33,6 +33,7 @@ struct dev_dax devm_create_dev_dax(struct dev_dax_data data); · 32 unmodified lines
33	33	enum dax_driver_type {
34	34	DAXDRV_KMEM_TYPE,
35	35	DAXDRV_DEVICE_TYPE,
	36	+ DAXDRV_FSDEV_TYPE,
36	37	};
37	38
38	39	struct dax_device_driver {
@@ -43,6 +44,8 @@ struct dax_device_driver { · 4 unmodified lines
43	44	void (remove)(struct dev_dax dev);
44	45	};
45	46
	47	+#define to_dax_drv(__drv) container_of_const(__drv, struct dax_device_driver, drv)
	48	+
46	49	int __dax_driver_register(struct dax_device_driver *dax_drv,
47	50	struct module module, const char mod_name);
48	51	#define dax_driver_register(driver) \

drivers/dax/dax-private.h+4−0

@@ -69,6 +69,8 @@ struct dev_dax_range { · 68 unmodified lines
69	69	* data while the device is activated in the driver.
70	70	* @region: parent region
71	71	* @dax_dev: core dax functionality
	72	+ * @virt_addr: kva from memremap; used by fsdev_dax
	73	+ * @cached_size: size of daxdev cached by fsdev_dax
72	74	* @align: alignment of this instance
73	75	* @target_node: effective numa node if dev_dax memory range is onlined
74	76	* @dyn_id: is this a dynamic or statically created instance
@@ -83,6 +85,8 @@ struct dev_dax_range { · 8 unmodified lines
83	85	struct dev_dax {
84	86	struct dax_region *region;
85	87	struct dax_device *dax_dev;
	88	+ void *virt_addr;
	89	+ u64 cached_size;
86	90	unsigned int align;
87	91	int target_node;
88	92	bool dyn_id;

drivers/dax/device.c+0−23

@@ -57,29 +57,6 @@ static int check_vma(struct dev_dax dev_dax, struct vm_area_struct vma, · 56 unmodified lines
57	57	vma->vm_file, func);
58	58	}
59	59
60		−/* see "strong" declaration in tools/testing/nvdimm/dax-dev.c */
61		−__weak phys_addr_t dax_pgoff_to_phys(struct dev_dax *dev_dax, pgoff_t pgoff,
62		− unsigned long size)
63		−{
64		− int i;
65		−
66		− for (i = 0; i < dev_dax->nr_range; i++) {
67		− struct dev_dax_range *dax_range = &dev_dax->ranges[i];
68		− struct range *range = &dax_range->range;
69		− unsigned long long pgoff_end;
70		− phys_addr_t phys;
71		−
72		− pgoff_end = dax_range->pgoff + PHYS_PFN(range_len(range)) - 1;
73		− if (pgoff < dax_range->pgoff \|\| pgoff > pgoff_end)
74		− continue;
75		− phys = PFN_PHYS(pgoff - dax_range->pgoff) + range->start;
76		− if (phys + size - 1 <= range->end)
77		− return phys;
78		− break;
79		− }
80		− return -1;
81		−}
82		−
83	60	static void dax_set_mapping(struct vm_fault *vmf, unsigned long pfn,
84	61	unsigned long fault_size)
85	62	{

drivers/dax/fsdev.c+349−0

@@ -0,0 +1,349 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+/* Copyright(c) 2026 Micron Technology, Inc. */
	3	+#include <linux/memremap.h>
	4	+#include <linux/pagemap.h>
	5	+#include <linux/module.h>
	6	+#include <linux/device.h>
	7	+#include <linux/cdev.h>
	8	+#include <linux/slab.h>
	9	+#include <linux/dax.h>
	10	+#include <linux/uio.h>
	11	+#include <linux/fs.h>
	12	+#include <linux/mm.h>
	13	+#include "dax-private.h"
	14	+#include "bus.h"
	15	+
	16	+/*
	17	+ * FS-DAX compatible devdax driver
	18	+ *
	19	+ * Unlike drivers/dax/device.c which pre-initializes compound folios based
	20	+ * on device alignment (via vmemmap_shift), this driver leaves folios
	21	+ * uninitialized similar to pmem. This allows fs-dax filesystems like famfs
	22	+ * to work without needing special handling for pre-initialized folios.
	23	+ *
	24	+ * Key differences from device.c:
	25	+ * - pgmap type is MEMORY_DEVICE_FS_DAX (not MEMORY_DEVICE_GENERIC)
	26	+ * - vmemmap_shift is NOT set (folios remain order-0)
	27	+ * - fs-dax can dynamically create compound folios as needed
	28	+ * - No mmap support - all access is through fs-dax/iomap
	29	+ */
	30	+
	31	+static void fsdev_write_dax(void addr, struct page page,
	32	+ unsigned int off, unsigned int len)
	33	+{
	34	+ while (len) {
	35	+ void *mem = kmap_local_page(page);
	36	+ unsigned int chunk = min_t(unsigned int, len, PAGE_SIZE - off);
	37	+
	38	+ memcpy_flushcache(addr, mem + off, chunk);
	39	+ kunmap_local(mem);
	40	+ len -= chunk;
	41	+ off = 0;
	42	+ page++;
	43	+ addr += chunk;
	44	+ }
	45	+}
	46	+
	47	+static long __fsdev_dax_direct_access(struct dax_device *dax_dev, pgoff_t pgoff,
	48	+ long nr_pages, enum dax_access_mode mode, void **kaddr,
	49	+ unsigned long *pfn)
	50	+{
	51	+ struct dev_dax *dev_dax = dax_get_private(dax_dev);
	52	+ size_t size = nr_pages << PAGE_SHIFT;
	53	+ size_t offset = pgoff << PAGE_SHIFT;
	54	+ void *virt_addr = dev_dax->virt_addr + offset;
	55	+ phys_addr_t phys;
	56	+ unsigned long local_pfn;
	57	+
	58	+ phys = dax_pgoff_to_phys(dev_dax, pgoff, size);
	59	+ if (phys == -1) {
	60	+ dev_dbg(&dev_dax->dev,
	61	+ "pgoff (%#lx) out of range\n", pgoff);
	62	+ return -EFAULT;
	63	+ }
	64	+
	65	+ if (kaddr)
	66	+ *kaddr = virt_addr;
	67	+
	68	+ local_pfn = PHYS_PFN(phys);
	69	+ if (pfn)
	70	+ *pfn = local_pfn;
	71	+
	72	+ /*
	73	+ * Use cached_size which was computed at probe time. The size cannot
	74	+ * change while the driver is bound (resize returns -EBUSY).
	75	+ */
	76	+ return PHYS_PFN(min(size, dev_dax->cached_size - offset));
	77	+}
	78	+
	79	+static int fsdev_dax_zero_page_range(struct dax_device *dax_dev,
	80	+ pgoff_t pgoff, size_t nr_pages)
	81	+{
	82	+ void *kaddr;
	83	+ long rc;
	84	+
	85	+ WARN_ONCE(nr_pages > 1, "%s: nr_pages > 1\n", __func__);
	86	+ rc = __fsdev_dax_direct_access(dax_dev, pgoff, 1, DAX_ACCESS, &kaddr, NULL);
	87	+ if (rc < 0)
	88	+ return rc;
	89	+ fsdev_write_dax(kaddr, ZERO_PAGE(0), 0, PAGE_SIZE);
	90	+ return 0;
	91	+}
	92	+
	93	+static long fsdev_dax_direct_access(struct dax_device *dax_dev,
	94	+ pgoff_t pgoff, long nr_pages, enum dax_access_mode mode,
	95	+ void *kaddr, unsigned long pfn)
	96	+{
	97	+ return __fsdev_dax_direct_access(dax_dev, pgoff, nr_pages, mode,
	98	+ kaddr, pfn);
	99	+}
	100	+
	101	+static size_t fsdev_dax_recovery_write(struct dax_device *dax_dev, pgoff_t pgoff,
	102	+ void addr, size_t bytes, struct iov_iter i)
	103	+{
	104	+ return _copy_from_iter_flushcache(addr, bytes, i);
	105	+}
	106	+
	107	+static const struct dax_operations dev_dax_ops = {
	108	+ .direct_access = fsdev_dax_direct_access,
	109	+ .zero_page_range = fsdev_dax_zero_page_range,
	110	+ .recovery_write = fsdev_dax_recovery_write,
	111	+};
	112	+
	113	+static void fsdev_cdev_del(void *cdev)
	114	+{
	115	+ cdev_del(cdev);
	116	+}
	117	+
	118	+static void fsdev_kill(void *dev_dax)
	119	+{
	120	+ kill_dev_dax(dev_dax);
	121	+}
	122	+
	123	+static void fsdev_clear_ops(void *data)
	124	+{
	125	+ struct dev_dax *dev_dax = data;
	126	+
	127	+ dax_set_ops(dev_dax->dax_dev, NULL);
	128	+}
	129	+
	130	+/*
	131	+ * Page map operations for FS-DAX mode
	132	+ * Similar to fsdax_pagemap_ops in drivers/nvdimm/pmem.c
	133	+ *
	134	+ * Note: folio_free callback is not needed for MEMORY_DEVICE_FS_DAX.
	135	+ * The core mm code in free_zone_device_folio() handles the wake_up_var()
	136	+ * directly for this memory type.
	137	+ */
	138	+static int fsdev_pagemap_memory_failure(struct dev_pagemap *pgmap,
	139	+ unsigned long pfn, unsigned long nr_pages, int mf_flags)
	140	+{
	141	+ struct dev_dax *dev_dax = pgmap->owner;
	142	+ u64 offset = PFN_PHYS(pfn) - dev_dax->ranges[0].range.start;
	143	+ u64 len = nr_pages << PAGE_SHIFT;
	144	+
	145	+ return dax_holder_notify_failure(dev_dax->dax_dev, offset,
	146	+ len, mf_flags);
	147	+}
	148	+
	149	+static const struct dev_pagemap_ops fsdev_pagemap_ops = {
	150	+ .memory_failure = fsdev_pagemap_memory_failure,
	151	+};
	152	+
	153	+/*
	154	+ * Clear any stale folio state from pages in the given range.
	155	+ * This is necessary because device_dax pre-initializes compound folios
	156	+ * based on vmemmap_shift, and that state may persist after driver unbind.
	157	+ * Since fsdev_dax uses MEMORY_DEVICE_FS_DAX without vmemmap_shift, fs-dax
	158	+ * expects to find clean order-0 folios that it can build into compound
	159	+ * folios on demand.
	160	+ *
	161	+ * At probe time, no filesystem should be mounted yet, so all mappings
	162	+ * are stale and must be cleared along with compound state.
	163	+ */
	164	+static void fsdev_clear_folio_state(struct dev_dax *dev_dax)
	165	+{
	166	+ for (int i = 0; i < dev_dax->nr_range; i++) {
	167	+ struct range *range = &dev_dax->ranges[i].range;
	168	+ unsigned long pfn = PHYS_PFN(range->start);
	169	+ unsigned long end_pfn = PHYS_PFN(range->end) + 1;
	170	+
	171	+ while (pfn < end_pfn) {
	172	+ struct folio *folio = pfn_folio(pfn);
	173	+ int order = dax_folio_reset_order(folio);
	174	+
	175	+ pfn += 1UL << order;
	176	+ }
	177	+ }
	178	+}
	179	+
	180	+static void fsdev_clear_folio_state_action(void *data)
	181	+{
	182	+ fsdev_clear_folio_state(data);
	183	+}
	184	+
	185	+static int fsdev_open(struct inode inode, struct file filp)
	186	+{
	187	+ struct dax_device *dax_dev = inode_dax(inode);
	188	+ struct dev_dax *dev_dax = dax_get_private(dax_dev);
	189	+
	190	+ filp->private_data = dev_dax;
	191	+
	192	+ return 0;
	193	+}
	194	+
	195	+static int fsdev_release(struct inode inode, struct file filp)
	196	+{
	197	+ return 0;
	198	+}
	199	+
	200	+static const struct file_operations fsdev_fops = {
	201	+ .llseek = noop_llseek,
	202	+ .owner = THIS_MODULE,
	203	+ .open = fsdev_open,
	204	+ .release = fsdev_release,
	205	+};
	206	+
	207	+static int fsdev_dax_probe(struct dev_dax *dev_dax)
	208	+{
	209	+ struct dax_device *dax_dev = dev_dax->dax_dev;
	210	+ struct device *dev = &dev_dax->dev;
	211	+ struct dev_pagemap *pgmap;
	212	+ struct inode *inode;
	213	+ u64 data_offset = 0;
	214	+ struct cdev *cdev;
	215	+ void *addr;
	216	+ int rc, i;
	217	+
	218	+ if (static_dev_dax(dev_dax)) {
	219	+ if (dev_dax->nr_range > 1) {
	220	+ dev_warn(dev, "static pgmap / multi-range device conflict\n");
	221	+ return -EINVAL;
	222	+ }
	223	+
	224	+ pgmap = dev_dax->pgmap;
	225	+ } else {
	226	+ size_t pgmap_size;
	227	+
	228	+ if (dev_dax->pgmap) {
	229	+ dev_warn(dev, "dynamic-dax with pre-populated page map\n");
	230	+ return -EINVAL;
	231	+ }
	232	+
	233	+ pgmap_size = struct_size(pgmap, ranges, dev_dax->nr_range - 1);
	234	+ pgmap = devm_kzalloc(dev, pgmap_size, GFP_KERNEL);
	235	+ if (!pgmap)
	236	+ return -ENOMEM;
	237	+
	238	+ pgmap->nr_range = dev_dax->nr_range;
	239	+ dev_dax->pgmap = pgmap;
	240	+
	241	+ for (i = 0; i < dev_dax->nr_range; i++) {
	242	+ struct range *range = &dev_dax->ranges[i].range;
	243	+
	244	+ pgmap->ranges[i] = *range;
	245	+ }
	246	+ }
	247	+
	248	+ for (i = 0; i < dev_dax->nr_range; i++) {
	249	+ struct range *range = &dev_dax->ranges[i].range;
	250	+
	251	+ if (!devm_request_mem_region(dev, range->start,
	252	+ range_len(range), dev_name(dev))) {
	253	+ dev_warn(dev, "mapping%d: %#llx-%#llx could not reserve range\n",
	254	+ i, range->start, range->end);
	255	+ return -EBUSY;
	256	+ }
	257	+ }
	258	+
	259	+ /* Cache size now; it cannot change while driver is bound */
	260	+ dev_dax->cached_size = 0;
	261	+ for (i = 0; i < dev_dax->nr_range; i++)
	262	+ dev_dax->cached_size += range_len(&dev_dax->ranges[i].range);
	263	+
	264	+ /*
	265	+ * Use MEMORY_DEVICE_FS_DAX without setting vmemmap_shift, leaving
	266	+ * folios at order-0. Unlike device.c (MEMORY_DEVICE_GENERIC), this
	267	+ * lets fs-dax dynamically build compound folios as needed, similar
	268	+ * to pmem behavior.
	269	+ */
	270	+ pgmap->type = MEMORY_DEVICE_FS_DAX;
	271	+ pgmap->ops = &fsdev_pagemap_ops;
	272	+ pgmap->owner = dev_dax;
	273	+
	274	+ addr = devm_memremap_pages(dev, pgmap);
	275	+ if (IS_ERR(addr))
	276	+ return PTR_ERR(addr);
	277	+
	278	+ /*
	279	+ * Clear any stale compound folio state left over from a previous
	280	+ * driver (e.g., device_dax with vmemmap_shift). Also register this
	281	+ * as a devm action so folio state is cleared on unbind, ensuring
	282	+ * clean pages for subsequent drivers (e.g., kmem for system-ram).
	283	+ */
	284	+ fsdev_clear_folio_state(dev_dax);
	285	+ rc = devm_add_action_or_reset(dev, fsdev_clear_folio_state_action,
	286	+ dev_dax);
	287	+ if (rc)
	288	+ return rc;
	289	+
	290	+ /* Detect whether the data is at a non-zero offset into the memory */
	291	+ if (pgmap->range.start != dev_dax->ranges[0].range.start) {
	292	+ u64 phys = dev_dax->ranges[0].range.start;
	293	+ u64 pgmap_phys = dev_dax->pgmap[0].range.start;
	294	+
	295	+ if (!WARN_ON(pgmap_phys > phys))
	296	+ data_offset = phys - pgmap_phys;
	297	+
	298	+ pr_debug("%s: offset detected phys=%llx pgmap_phys=%llx offset=%llx\n",
	299	+ __func__, phys, pgmap_phys, data_offset);
	300	+ }
	301	+ dev_dax->virt_addr = addr + data_offset;
	302	+
	303	+ inode = dax_inode(dax_dev);
	304	+ cdev = inode->i_cdev;
	305	+ cdev_init(cdev, &fsdev_fops);
	306	+ cdev->owner = dev->driver->owner;
	307	+ cdev_set_parent(cdev, &dev->kobj);
	308	+ rc = cdev_add(cdev, dev->devt, 1);
	309	+ if (rc)
	310	+ return rc;
	311	+
	312	+ rc = devm_add_action_or_reset(dev, fsdev_cdev_del, cdev);
	313	+ if (rc)
	314	+ return rc;
	315	+
	316	+ /* Set the dax operations for fs-dax access path */
	317	+ rc = dax_set_ops(dax_dev, &dev_dax_ops);
	318	+ if (rc)
	319	+ return rc;
	320	+
	321	+ rc = devm_add_action_or_reset(dev, fsdev_clear_ops, dev_dax);
	322	+ if (rc)
	323	+ return rc;
	324	+
	325	+ run_dax(dax_dev);
	326	+ return devm_add_action_or_reset(dev, fsdev_kill, dev_dax);
	327	+}
	328	+
	329	+static struct dax_device_driver fsdev_dax_driver = {
	330	+ .probe = fsdev_dax_probe,
	331	+ .type = DAXDRV_FSDEV_TYPE,
	332	+};
	333	+
	334	+static int __init dax_init(void)
	335	+{
	336	+ return dax_driver_register(&fsdev_dax_driver);
	337	+}
	338	+
	339	+static void __exit dax_exit(void)
	340	+{
	341	+ dax_driver_unregister(&fsdev_dax_driver);
	342	+}
	343	+
	344	+MODULE_AUTHOR("John Groves");
	345	+MODULE_DESCRIPTION("FS-DAX Device: fs-dax compatible devdax driver");
	346	+MODULE_LICENSE("GPL");
	347	+module_init(dax_init);
	348	+module_exit(dax_exit);
	349	+MODULE_ALIAS_DAX_DEVICE(0);

drivers/dax/super.c+104−3

@@ -14,6 +14,7 @@ · 13 unmodified lines
14	14	#include <linux/fs.h>
15	15	#include <linux/cacheinfo.h>
16	16	#include "dax-private.h"
	17	+#include "bus.h"
17	18
18	19	/**
19	20	* struct dax_device - anchor object for dax services
@@ -111,6 +112,10 @@ struct dax_device fs_dax_get_by_bdev(struct block_device bdev, u64 *start_off, · 91 unmodified lines
111	112	}
112	113	EXPORT_SYMBOL_GPL(fs_dax_get_by_bdev);
113	114
	115	+#endif /* CONFIG_BLOCK && CONFIG_FS_DAX */
	116	+
	117	+#if IS_ENABLED(CONFIG_FS_DAX)
	118	+
114	119	void fs_put_dax(struct dax_device dax_dev, void holder)
115	120	{
116	121	if (dax_dev && holder &&
@@ -119,7 +124,66 @@ void fs_put_dax(struct dax_device dax_dev, void holder) · 2 unmodified lines
119	124	put_dax(dax_dev);
120	125	}
121	126	EXPORT_SYMBOL_GPL(fs_put_dax);
122		−#endif /* CONFIG_BLOCK && CONFIG_FS_DAX */
	127	+
	128	+/**
	129	+ * fs_dax_get() - get ownership of a devdax via holder/holder_ops
	130	+ *
	131	+ * fs-dax file systems call this function to prepare to use a devdax device for
	132	+ * fsdax. This is like fs_dax_get_by_bdev(), but the caller already has struct
	133	+ * dev_dax (and there is no bdev). The holder makes this exclusive.
	134	+ *
	135	+ * @dax_dev: dev to be prepared for fs-dax usage
	136	+ * @holder: filesystem or mapped device inside the dax_device
	137	+ * @hops: operations for the inner holder
	138	+ *
	139	+ * Returns: 0 on success, <0 on failure
	140	+ */
	141	+int fs_dax_get(struct dax_device dax_dev, void holder,
	142	+ const struct dax_holder_operations *hops)
	143	+{
	144	+ struct dev_dax *dev_dax;
	145	+ struct dax_device_driver *dax_drv;
	146	+ int id;
	147	+
	148	+ id = dax_read_lock();
	149	+ if (!dax_dev \|\| !dax_alive(dax_dev) \|\| !igrab(&dax_dev->inode)) {
	150	+ dax_read_unlock(id);
	151	+ return -ENODEV;
	152	+ }
	153	+ dax_read_unlock(id);
	154	+
	155	+ /* Verify the device is bound to fsdev_dax driver */
	156	+ dev_dax = dax_get_private(dax_dev);
	157	+ if (!dev_dax) {
	158	+ iput(&dax_dev->inode);
	159	+ return -ENODEV;
	160	+ }
	161	+
	162	+ device_lock(&dev_dax->dev);
	163	+ if (!dev_dax->dev.driver) {
	164	+ device_unlock(&dev_dax->dev);
	165	+ iput(&dax_dev->inode);
	166	+ return -ENODEV;
	167	+ }
	168	+ dax_drv = to_dax_drv(dev_dax->dev.driver);
	169	+ if (dax_drv->type != DAXDRV_FSDEV_TYPE) {
	170	+ device_unlock(&dev_dax->dev);
	171	+ iput(&dax_dev->inode);
	172	+ return -EOPNOTSUPP;
	173	+ }
	174	+ device_unlock(&dev_dax->dev);
	175	+
	176	+ if (cmpxchg(&dax_dev->holder_data, NULL, holder)) {
	177	+ iput(&dax_dev->inode);
	178	+ return -EBUSY;
	179	+ }
	180	+
	181	+ dax_dev->holder_ops = hops;
	182	+
	183	+ return 0;
	184	+}
	185	+EXPORT_SYMBOL_GPL(fs_dax_get);
	186	+#endif /* CONFIG_FS_DAX */
123	187
124	188	enum dax_device_flags {
125	189	/* !alive + rcu grace period == no new operations / mappings */
@@ -157,6 +221,9 @@ long dax_direct_access(struct dax_device *dax_dev, pgoff_t pgoff, long nr_pages, · 31 unmodified lines
157	221	if (!dax_alive(dax_dev))
158	222	return -ENXIO;
159	223
	224	+ if (!dax_dev->ops)
	225	+ return -EOPNOTSUPP;
	226	+
160	227	if (nr_pages < 0)
161	228	return -EINVAL;
162	229
@@ -207,6 +274,10 @@ int dax_zero_page_range(struct dax_device *dax_dev, pgoff_t pgoff, · 44 unmodified lines
207	274
208	275	if (!dax_alive(dax_dev))
209	276	return -ENXIO;
	277	+
	278	+ if (!dax_dev->ops)
	279	+ return -EOPNOTSUPP;
	280	+
210	281	/*
211	282	* There are no callers that want to zero more than one page as of now.
212	283	* Once users are there, this check can be removed after the
@@ -223,7 +294,7 @@ EXPORT_SYMBOL_GPL(dax_zero_page_range); · 10 unmodified lines
223	294	size_t dax_recovery_write(struct dax_device *dax_dev, pgoff_t pgoff,
224	295	void addr, size_t bytes, struct iov_iter iter)
225	296	{
226		− if (!dax_dev->ops->recovery_write)
	297	+ if (!dax_dev->ops \|\| !dax_dev->ops->recovery_write)
227	298	return 0;
228	299	return dax_dev->ops->recovery_write(dax_dev, pgoff, addr, bytes, iter);
229	300	}
@@ -307,6 +378,35 @@ void set_dax_nomc(struct dax_device *dax_dev) · 77 unmodified lines
307	378	}
308	379	EXPORT_SYMBOL_GPL(set_dax_nomc);
309	380
	381	+/**
	382	+ * dax_set_ops - set the dax_operations for a dax_device
	383	+ * @dax_dev: the dax_device to configure
	384	+ * @ops: the operations to set (may be NULL to clear)
	385	+ *
	386	+ * This allows drivers to set the dax_operations after the dax_device
	387	+ * has been allocated. This is needed when the device is created before
	388	+ * the driver that needs specific ops is bound (e.g., fsdev_dax binding
	389	+ * to a dev_dax created by hmem).
	390	+ *
	391	+ * When setting non-NULL ops, fails if ops are already set (returns -EBUSY).
	392	+ * When clearing ops (NULL), always succeeds.
	393	+ *
	394	+ * Return: 0 on success, -EBUSY if ops already set
	395	+ */
	396	+int dax_set_ops(struct dax_device dax_dev, const struct dax_operations ops)
	397	+{
	398	+ if (ops) {
	399	+ /* Setting ops: fail if already set */
	400	+ if (cmpxchg(&dax_dev->ops, NULL, ops) != NULL)
	401	+ return -EBUSY;
	402	+ } else {
	403	+ /* Clearing ops: always allowed */
	404	+ dax_dev->ops = NULL;
	405	+ }
	406	+ return 0;
	407	+}
	408	+EXPORT_SYMBOL_GPL(dax_set_ops);
	409	+
310	410	bool dax_alive(struct dax_device *dax_dev)
311	411	{
312	412	lockdep_assert_held(&dax_srcu);
@@ -421,7 +521,7 @@ static int dax_set(struct inode inode, void data) · 108 unmodified lines
421	521	return 0;
422	522	}
423	523
424		−static struct dax_device *dax_dev_get(dev_t devt)
	524	+struct dax_device *dax_dev_get(dev_t devt)
425	525	{
426	526	struct dax_device *dax_dev;
427	527	struct inode *inode;
@@ -444,6 +544,7 @@ static struct dax_device *dax_dev_get(dev_t devt) · 16 unmodified lines
444	544
445	545	return dax_dev;
446	546	}
	547	+EXPORT_SYMBOL_GPL(dax_dev_get);
447	548
448	549	struct dax_device alloc_dax(void private, const struct dax_operations *ops)
449	550	{

fs/dax.c+56−18

@@ -377,6 +377,59 @@ static void dax_folio_make_shared(struct folio *folio) · 376 unmodified lines
377	377	folio->share = 1;
378	378	}
379	379
	380	+/**
	381	+ * dax_folio_reset_order - Reset a compound DAX folio to order-0 pages
	382	+ * @folio: The folio to reset
	383	+ *
	384	+ * Splits a compound folio back into individual order-0 pages,
	385	+ * clearing compound state and restoring pgmap pointers.
	386	+ *
	387	+ * Returns: the original folio order (0 if already order-0)
	388	+ */
	389	+int dax_folio_reset_order(struct folio *folio)
	390	+{
	391	+ struct dev_pagemap *pgmap = page_pgmap(&folio->page);
	392	+ int order = folio_order(folio);
	393	+
	394	+ /*
	395	+ * DAX maintains the invariant that folio->share != 0 only when
	396	+ * folio->mapping == NULL (enforced by dax_folio_make_shared()).
	397	+ * Equivalently: folio->mapping != NULL implies folio->share == 0.
	398	+ * Callers ensure share has been decremented to zero before
	399	+ * calling here, so unconditionally clearing both fields is
	400	+ * correct.
	401	+ */
	402	+ folio->mapping = NULL;
	403	+ folio->share = 0;
	404	+
	405	+ if (!order) {
	406	+ /*
	407	+ * Restore pgmap explicitly even for order-0 folios. For the
	408	+ * dax_folio_put() caller this is a no-op (same value), but
	409	+ * fsdev_clear_folio_state() may call this on folios that
	410	+ * were previously compound and need pgmap re-established.
	411	+ */
	412	+ folio->pgmap = pgmap;
	413	+ return 0;
	414	+ }
	415	+
	416	+ folio_reset_order(folio);
	417	+
	418	+ for (int i = 0; i < (1UL << order); i++) {
	419	+ struct page *page = folio_page(folio, i);
	420	+ struct folio f = (struct folio )page;
	421	+
	422	+ ClearPageHead(page);
	423	+ clear_compound_head(page);
	424	+ f->mapping = NULL;
	425	+ f->share = 0;
	426	+ f->pgmap = pgmap;
	427	+ }
	428	+
	429	+ return order;
	430	+}
	431	+EXPORT_SYMBOL_GPL(dax_folio_reset_order);
	432	+
380	433	static inline unsigned long dax_folio_put(struct folio *folio)
381	434	{
382	435	unsigned long ref;
@@ -390,28 +443,13 @@ static inline unsigned long dax_folio_put(struct folio *folio) · 7 unmodified lines
390	443	if (ref)
391	444	return ref;
392	445
393		− folio->mapping = NULL;
394		− order = folio_order(folio);
395		− if (!order)
396		− return 0;
397		− folio_reset_order(folio);
	446	+ order = dax_folio_reset_order(folio);
398	447
	448	+ /* Debug check: verify refcounts are zero for all sub-folios */
399	449	for (i = 0; i < (1UL << order); i++) {
400		− struct dev_pagemap *pgmap = page_pgmap(&folio->page);
401	450	struct page *page = folio_page(folio, i);
402		− struct folio new_folio = (struct folio )page;
403		−
404		− ClearPageHead(page);
405		− clear_compound_head(page);
406	451
407		− new_folio->mapping = NULL;
408		− /*
409		− * Reset pgmap which was over-written by
410		− * prep_compound_page().
411		− */
412		− new_folio->pgmap = pgmap;
413		− new_folio->share = 0;
414		− WARN_ON_ONCE(folio_ref_count(new_folio));
	452	+ WARN_ON_ONCE(folio_ref_count((struct folio *)page));
415	453	}
416	454
417	455	return ref;

include/linux/dax.h+15−4

@@ -54,6 +54,7 @@ struct dax_device alloc_dax(void private, const struct dax_operations *ops); · 53 unmodified lines
54	54	void dax_holder(struct dax_device dax_dev);
55	55	void put_dax(struct dax_device *dax_dev);
56	56	void kill_dax(struct dax_device *dax_dev);
	57	+struct dax_device *dax_dev_get(dev_t devt);
57	58	void dax_write_cache(struct dax_device *dax_dev, bool wc);
58	59	bool dax_write_cache_enabled(struct dax_device *dax_dev);
59	60	bool dax_synchronous(struct dax_device *dax_dev);
@@ -130,7 +131,6 @@ int dax_add_host(struct dax_device dax_dev, struct gendisk disk); · 70 unmodified lines
130	131	void dax_remove_host(struct gendisk *disk);
131	132	struct dax_device fs_dax_get_by_bdev(struct block_device bdev, u64 *start_off,
132	133	void holder, const struct dax_holder_operations ops);
133		−void fs_put_dax(struct dax_device dax_dev, void holder);
134	134	#else
135	135	static inline int dax_add_host(struct dax_device dax_dev, struct gendisk disk)
136	136	{
@@ -145,14 +145,15 @@ static inline struct dax_device fs_dax_get_by_bdev(struct block_device bdev, · 8 unmodified lines
145	145	{
146	146	return NULL;
147	147	}
148		−static inline void fs_put_dax(struct dax_device dax_dev, void holder)
149		−{
150		−}
151	148	#endif /* CONFIG_BLOCK && CONFIG_FS_DAX */
152	149
153	150	#if IS_ENABLED(CONFIG_FS_DAX)
	151	+void fs_put_dax(struct dax_device dax_dev, void holder);
	152	+int fs_dax_get(struct dax_device dax_dev, void holder,
	153	+ const struct dax_holder_operations *hops);
154	154	int dax_writeback_mapping_range(struct address_space *mapping,
155	155	struct dax_device dax_dev, struct writeback_control wbc);
	156	+int dax_folio_reset_order(struct folio *folio);
156	157
157	158	struct page dax_layout_busy_page(struct address_space mapping);
158	159	struct page dax_layout_busy_page_range(struct address_space mapping, loff_t start, loff_t end);
@@ -163,6 +164,15 @@ dax_entry_t dax_lock_mapping_entry(struct address_space *mapping, · 4 unmodified lines
163	164	void dax_unlock_mapping_entry(struct address_space *mapping,
164	165	unsigned long index, dax_entry_t cookie);
165	166	#else
	167	+static inline void fs_put_dax(struct dax_device dax_dev, void holder)
	168	+{
	169	+}
	170	+
	171	+static inline int fs_dax_get(struct dax_device dax_dev, void holder,
	172	+ const struct dax_holder_operations *hops)
	173	+{
	174	+ return -EOPNOTSUPP;
	175	+}
166	176	static inline struct page dax_layout_busy_page(struct address_space mapping)
167	177	{
168	178	return NULL;
@@ -242,6 +252,7 @@ static inline void dax_break_layout_final(struct inode *inode) · 73 unmodified lines
242	252
243	253	bool dax_alive(struct dax_device *dax_dev);
244	254	void dax_get_private(struct dax_device dax_dev);
	255	+int dax_set_ops(struct dax_device dax_dev, const struct dax_operations ops);
245	256	long dax_direct_access(struct dax_device *dax_dev, pgoff_t pgoff, long nr_pages,
246	257	enum dax_access_mode mode, void *kaddr, unsigned long pfn);
247	258	size_t dax_copy_from_iter(struct dax_device dax_dev, pgoff_t pgoff, void addr,