History

Merge tag 'libnvdimm-for-7.1' of git://git.kernel.org/pub/scm/linux/kernel/git/nvdimm/nvdimm

Pull dax updates from Ira Weiny: "The series adds DAX support required for the upcoming fuse/famfs file system.[1] The support here is required because famfs is backed by devdax rather than pmem. This all lays the groundwork for using shared memory as a file system" Link: https://lore.kernel.org/all/0100019d43e5f632-f5862a3e-361c-4b54-a9a6-96c242a8f17a-000000@email.amazonses.com/ [1] * tag 'libnvdimm-for-7.1' of git://git.kernel.org/pub/scm/linux/kernel/git/nvdimm/nvdimm: dax/fsdev: fix uninitialized kaddr in fsdev_dax_zero_page_range() dax: export dax_dev_get() dax: Add fs_dax_get() func to prepare dax for fs-dax usage dax: Add dax_set_ops() for setting dax_operations at bind time dax: Add dax_operations for use by fs-dax on fsdev dax dax: Save the kva from memremap dax: add fsdev.c driver for fs-dax on character dax dax: Factor out dax_folio_reset_order() helper dax: move dax_pgoff_to_phys from [drivers/dax/] device.c to bus.c

Linus Torvalds committed 6 months ago

Commit bb0bc492 parents c94faa7 45df91111 files · +566 −50Browse files

Unified · Split

11 files
  1. MAINTAINERS +8 −0
  2. drivers/dax/Kconfig +5 −0
  3. drivers/dax/Makefile +2 −0
  4. drivers/dax/bus.c +20 −2
  5. drivers/dax/bus.h +3 −0
  6. drivers/dax/dax-private.h +4 −0
  7. drivers/dax/device.c +0 −23
  8. drivers/dax/fsdev.c +349 −0
  9. drivers/dax/super.c +104 −3
  10. fs/dax.c +56 −18
  11. include/linux/dax.h +15 −4
MAINTAINERS+8−0
@@ -7303,6 +7303,14 @@ L: linux-cxl@vger.kernel.org · 7302 unmodified lines
7303S: Supported7303S: Supported
7304F: drivers/dax/7304F: drivers/dax/
7305 7305
7306DEVICE DIRECT ACCESS (DAX) [fsdev_dax]
7307M: John Groves <jgroves@micron.com>
7308M: John Groves <John@Groves.net>
7309L: nvdimm@lists.linux.dev
7310L: linux-cxl@vger.kernel.org
7311S: Supported
7312F: drivers/dax/fsdev.c
7313
7306DEVICE FREQUENCY (DEVFREQ)7314DEVICE FREQUENCY (DEVFREQ)
7307M: MyungJoo Ham <myungjoo.ham@samsung.com>7315M: MyungJoo Ham <myungjoo.ham@samsung.com>
7308M: Kyungmin Park <kyungmin.park@samsung.com>7316M: Kyungmin Park <kyungmin.park@samsung.com>
drivers/dax/Kconfig+5−0
@@ -65,6 +65,11 @@ config DEV_DAX_HMEM_DEVICES · 64 unmodified lines
65 depends on DEV_DAX_HMEM && DAX65 depends on DEV_DAX_HMEM && DAX
66 def_bool y66 def_bool y
67 67
68config DEV_DAX_FSDEV
69 tristate
70 depends on DEV_DAX && FS_DAX
71 default DEV_DAX
72
68config DEV_DAX_KMEM73config DEV_DAX_KMEM
69 tristate "KMEM DAX: map dax-devices as System-RAM"74 tristate "KMEM DAX: map dax-devices as System-RAM"
70 default DEV_DAX75 default DEV_DAX
drivers/dax/Makefile+2−0
@@ -5,9 +5,11 @@ obj-$(CONFIG_DEV_DAX) += device_dax.o · 4 unmodified lines
5obj-$(CONFIG_DEV_DAX_KMEM) += kmem.o5obj-$(CONFIG_DEV_DAX_KMEM) += kmem.o
6obj-$(CONFIG_DEV_DAX_PMEM) += dax_pmem.o6obj-$(CONFIG_DEV_DAX_PMEM) += dax_pmem.o
7obj-$(CONFIG_DEV_DAX_CXL) += dax_cxl.o7obj-$(CONFIG_DEV_DAX_CXL) += dax_cxl.o
8obj-$(CONFIG_DEV_DAX_FSDEV) += fsdev_dax.o
8 9
9dax-y := super.o10dax-y := super.o
10dax-y += bus.o11dax-y += bus.o
11device_dax-y := device.o12device_dax-y := device.o
12dax_pmem-y := pmem.o13dax_pmem-y := pmem.o
13dax_cxl-y := cxl.o14dax_cxl-y := cxl.o
15fsdev_dax-y := fsdev.o
drivers/dax/bus.c+20−2
@@ -40,8 +40,6 @@ static int dax_bus_uevent(const struct device *dev, struct kobj_uevent_env *env) · 39 unmodified lines
40 return add_uevent_var(env, "MODALIAS=" DAX_DEVICE_MODALIAS_FMT, 0);40 return add_uevent_var(env, "MODALIAS=" DAX_DEVICE_MODALIAS_FMT, 0);
41}41}
42 42
43#define to_dax_drv(__drv) container_of_const(__drv, struct dax_device_driver, drv)
44
45static struct dax_id *__dax_match_id(const struct dax_device_driver *dax_drv,43static struct dax_id *__dax_match_id(const struct dax_device_driver *dax_drv,
46 const char *dev_name)44 const char *dev_name)
47{45{
@@ -1431,6 +1429,26 @@ static const struct device_type dev_dax_type = { · 1383 unmodified lines
1431 .groups = dax_attribute_groups,1429 .groups = dax_attribute_groups,
1432};1430};
1433 1431
1432/* see "strong" declaration in tools/testing/nvdimm/dax-dev.c */
1433__weak phys_addr_t dax_pgoff_to_phys(struct dev_dax *dev_dax, pgoff_t pgoff,
1434 unsigned long size)
1435{
1436 for (int i = 0; i < dev_dax->nr_range; i++) {
1437 struct dev_dax_range *dax_range = &dev_dax->ranges[i];
1438 struct range *range = &dax_range->range;
1439 phys_addr_t phys;
1440
1441 if (!in_range(pgoff, dax_range->pgoff, PHYS_PFN(range_len(range))))
1442 continue;
1443 phys = PFN_PHYS(pgoff - dax_range->pgoff) + range->start;
1444 if (phys + size - 1 <= range->end)
1445 return phys;
1446 break;
1447 }
1448 return -1;
1449}
1450EXPORT_SYMBOL_GPL(dax_pgoff_to_phys);
1451
1434static struct dev_dax *__devm_create_dev_dax(struct dev_dax_data *data)1452static struct dev_dax *__devm_create_dev_dax(struct dev_dax_data *data)
1435{1453{
1436 struct dax_region *dax_region = data->dax_region;1454 struct dax_region *dax_region = data->dax_region;
drivers/dax/bus.h+3−0
@@ -33,6 +33,7 @@ struct dev_dax *devm_create_dev_dax(struct dev_dax_data *data); · 32 unmodified lines
33enum dax_driver_type {33enum dax_driver_type {
34 DAXDRV_KMEM_TYPE,34 DAXDRV_KMEM_TYPE,
35 DAXDRV_DEVICE_TYPE,35 DAXDRV_DEVICE_TYPE,
36 DAXDRV_FSDEV_TYPE,
36};37};
37 38
38struct dax_device_driver {39struct dax_device_driver {
@@ -43,6 +44,8 @@ struct dax_device_driver { · 4 unmodified lines
43 void (*remove)(struct dev_dax *dev);44 void (*remove)(struct dev_dax *dev);
44};45};
45 46
47#define to_dax_drv(__drv) container_of_const(__drv, struct dax_device_driver, drv)
48
46int __dax_driver_register(struct dax_device_driver *dax_drv,49int __dax_driver_register(struct dax_device_driver *dax_drv,
47 struct module *module, const char *mod_name);50 struct module *module, const char *mod_name);
48#define dax_driver_register(driver) \51#define dax_driver_register(driver) \
drivers/dax/dax-private.h+4−0
@@ -69,6 +69,8 @@ struct dev_dax_range { · 68 unmodified lines
69 * data while the device is activated in the driver.69 * data while the device is activated in the driver.
70 * @region: parent region70 * @region: parent region
71 * @dax_dev: core dax functionality71 * @dax_dev: core dax functionality
72 * @virt_addr: kva from memremap; used by fsdev_dax
73 * @cached_size: size of daxdev cached by fsdev_dax
72 * @align: alignment of this instance74 * @align: alignment of this instance
73 * @target_node: effective numa node if dev_dax memory range is onlined75 * @target_node: effective numa node if dev_dax memory range is onlined
74 * @dyn_id: is this a dynamic or statically created instance76 * @dyn_id: is this a dynamic or statically created instance
@@ -83,6 +85,8 @@ struct dev_dax_range { · 8 unmodified lines
83struct dev_dax {85struct dev_dax {
84 struct dax_region *region;86 struct dax_region *region;
85 struct dax_device *dax_dev;87 struct dax_device *dax_dev;
88 void *virt_addr;
89 u64 cached_size;
86 unsigned int align;90 unsigned int align;
87 int target_node;91 int target_node;
88 bool dyn_id;92 bool dyn_id;
drivers/dax/device.c+0−23
@@ -57,29 +57,6 @@ static int check_vma(struct dev_dax *dev_dax, struct vm_area_struct *vma, · 56 unmodified lines
57 vma->vm_file, func);57 vma->vm_file, func);
58}58}
59 59
60/* see "strong" declaration in tools/testing/nvdimm/dax-dev.c */
61__weak phys_addr_t dax_pgoff_to_phys(struct dev_dax *dev_dax, pgoff_t pgoff,
62 unsigned long size)
63{
64 int i;
65
66 for (i = 0; i < dev_dax->nr_range; i++) {
67 struct dev_dax_range *dax_range = &dev_dax->ranges[i];
68 struct range *range = &dax_range->range;
69 unsigned long long pgoff_end;
70 phys_addr_t phys;
71
72 pgoff_end = dax_range->pgoff + PHYS_PFN(range_len(range)) - 1;
73 if (pgoff < dax_range->pgoff || pgoff > pgoff_end)
74 continue;
75 phys = PFN_PHYS(pgoff - dax_range->pgoff) + range->start;
76 if (phys + size - 1 <= range->end)
77 return phys;
78 break;
79 }
80 return -1;
81}
82
83static void dax_set_mapping(struct vm_fault *vmf, unsigned long pfn,60static void dax_set_mapping(struct vm_fault *vmf, unsigned long pfn,
84 unsigned long fault_size)61 unsigned long fault_size)
85{62{
drivers/dax/fsdev.c+349−0
@@ -0,0 +1,349 @@
1// SPDX-License-Identifier: GPL-2.0
2/* Copyright(c) 2026 Micron Technology, Inc. */
3#include <linux/memremap.h>
4#include <linux/pagemap.h>
5#include <linux/module.h>
6#include <linux/device.h>
7#include <linux/cdev.h>
8#include <linux/slab.h>
9#include <linux/dax.h>
10#include <linux/uio.h>
11#include <linux/fs.h>
12#include <linux/mm.h>
13#include "dax-private.h"
14#include "bus.h"
15
16/*
17 * FS-DAX compatible devdax driver
18 *
19 * Unlike drivers/dax/device.c which pre-initializes compound folios based
20 * on device alignment (via vmemmap_shift), this driver leaves folios
21 * uninitialized similar to pmem. This allows fs-dax filesystems like famfs
22 * to work without needing special handling for pre-initialized folios.
23 *
24 * Key differences from device.c:
25 * - pgmap type is MEMORY_DEVICE_FS_DAX (not MEMORY_DEVICE_GENERIC)
26 * - vmemmap_shift is NOT set (folios remain order-0)
27 * - fs-dax can dynamically create compound folios as needed
28 * - No mmap support - all access is through fs-dax/iomap
29 */
30
31static void fsdev_write_dax(void *addr, struct page *page,
32 unsigned int off, unsigned int len)
33{
34 while (len) {
35 void *mem = kmap_local_page(page);
36 unsigned int chunk = min_t(unsigned int, len, PAGE_SIZE - off);
37
38 memcpy_flushcache(addr, mem + off, chunk);
39 kunmap_local(mem);
40 len -= chunk;
41 off = 0;
42 page++;
43 addr += chunk;
44 }
45}
46
47static long __fsdev_dax_direct_access(struct dax_device *dax_dev, pgoff_t pgoff,
48 long nr_pages, enum dax_access_mode mode, void **kaddr,
49 unsigned long *pfn)
50{
51 struct dev_dax *dev_dax = dax_get_private(dax_dev);
52 size_t size = nr_pages << PAGE_SHIFT;
53 size_t offset = pgoff << PAGE_SHIFT;
54 void *virt_addr = dev_dax->virt_addr + offset;
55 phys_addr_t phys;
56 unsigned long local_pfn;
57
58 phys = dax_pgoff_to_phys(dev_dax, pgoff, size);
59 if (phys == -1) {
60 dev_dbg(&dev_dax->dev,
61 "pgoff (%#lx) out of range\n", pgoff);
62 return -EFAULT;
63 }
64
65 if (kaddr)
66 *kaddr = virt_addr;
67
68 local_pfn = PHYS_PFN(phys);
69 if (pfn)
70 *pfn = local_pfn;
71
72 /*
73 * Use cached_size which was computed at probe time. The size cannot
74 * change while the driver is bound (resize returns -EBUSY).
75 */
76 return PHYS_PFN(min(size, dev_dax->cached_size - offset));
77}
78
79static int fsdev_dax_zero_page_range(struct dax_device *dax_dev,
80 pgoff_t pgoff, size_t nr_pages)
81{
82 void *kaddr;
83 long rc;
84
85 WARN_ONCE(nr_pages > 1, "%s: nr_pages > 1\n", __func__);
86 rc = __fsdev_dax_direct_access(dax_dev, pgoff, 1, DAX_ACCESS, &kaddr, NULL);
87 if (rc < 0)
88 return rc;
89 fsdev_write_dax(kaddr, ZERO_PAGE(0), 0, PAGE_SIZE);
90 return 0;
91}
92
93static long fsdev_dax_direct_access(struct dax_device *dax_dev,
94 pgoff_t pgoff, long nr_pages, enum dax_access_mode mode,
95 void **kaddr, unsigned long *pfn)
96{
97 return __fsdev_dax_direct_access(dax_dev, pgoff, nr_pages, mode,
98 kaddr, pfn);
99}
100
101static size_t fsdev_dax_recovery_write(struct dax_device *dax_dev, pgoff_t pgoff,
102 void *addr, size_t bytes, struct iov_iter *i)
103{
104 return _copy_from_iter_flushcache(addr, bytes, i);
105}
106
107static const struct dax_operations dev_dax_ops = {
108 .direct_access = fsdev_dax_direct_access,
109 .zero_page_range = fsdev_dax_zero_page_range,
110 .recovery_write = fsdev_dax_recovery_write,
111};
112
113static void fsdev_cdev_del(void *cdev)
114{
115 cdev_del(cdev);
116}
117
118static void fsdev_kill(void *dev_dax)
119{
120 kill_dev_dax(dev_dax);
121}
122
123static void fsdev_clear_ops(void *data)
124{
125 struct dev_dax *dev_dax = data;
126
127 dax_set_ops(dev_dax->dax_dev, NULL);
128}
129
130/*
131 * Page map operations for FS-DAX mode
132 * Similar to fsdax_pagemap_ops in drivers/nvdimm/pmem.c
133 *
134 * Note: folio_free callback is not needed for MEMORY_DEVICE_FS_DAX.
135 * The core mm code in free_zone_device_folio() handles the wake_up_var()
136 * directly for this memory type.
137 */
138static int fsdev_pagemap_memory_failure(struct dev_pagemap *pgmap,
139 unsigned long pfn, unsigned long nr_pages, int mf_flags)
140{
141 struct dev_dax *dev_dax = pgmap->owner;
142 u64 offset = PFN_PHYS(pfn) - dev_dax->ranges[0].range.start;
143 u64 len = nr_pages << PAGE_SHIFT;
144
145 return dax_holder_notify_failure(dev_dax->dax_dev, offset,
146 len, mf_flags);
147}
148
149static const struct dev_pagemap_ops fsdev_pagemap_ops = {
150 .memory_failure = fsdev_pagemap_memory_failure,
151};
152
153/*
154 * Clear any stale folio state from pages in the given range.
155 * This is necessary because device_dax pre-initializes compound folios
156 * based on vmemmap_shift, and that state may persist after driver unbind.
157 * Since fsdev_dax uses MEMORY_DEVICE_FS_DAX without vmemmap_shift, fs-dax
158 * expects to find clean order-0 folios that it can build into compound
159 * folios on demand.
160 *
161 * At probe time, no filesystem should be mounted yet, so all mappings
162 * are stale and must be cleared along with compound state.
163 */
164static void fsdev_clear_folio_state(struct dev_dax *dev_dax)
165{
166 for (int i = 0; i < dev_dax->nr_range; i++) {
167 struct range *range = &dev_dax->ranges[i].range;
168 unsigned long pfn = PHYS_PFN(range->start);
169 unsigned long end_pfn = PHYS_PFN(range->end) + 1;
170
171 while (pfn < end_pfn) {
172 struct folio *folio = pfn_folio(pfn);
173 int order = dax_folio_reset_order(folio);
174
175 pfn += 1UL << order;
176 }
177 }
178}
179
180static void fsdev_clear_folio_state_action(void *data)
181{
182 fsdev_clear_folio_state(data);
183}
184
185static int fsdev_open(struct inode *inode, struct file *filp)
186{
187 struct dax_device *dax_dev = inode_dax(inode);
188 struct dev_dax *dev_dax = dax_get_private(dax_dev);
189
190 filp->private_data = dev_dax;
191
192 return 0;
193}
194
195static int fsdev_release(struct inode *inode, struct file *filp)
196{
197 return 0;
198}
199
200static const struct file_operations fsdev_fops = {
201 .llseek = noop_llseek,
202 .owner = THIS_MODULE,
203 .open = fsdev_open,
204 .release = fsdev_release,
205};
206
207static int fsdev_dax_probe(struct dev_dax *dev_dax)
208{
209 struct dax_device *dax_dev = dev_dax->dax_dev;
210 struct device *dev = &dev_dax->dev;
211 struct dev_pagemap *pgmap;
212 struct inode *inode;
213 u64 data_offset = 0;
214 struct cdev *cdev;
215 void *addr;
216 int rc, i;
217
218 if (static_dev_dax(dev_dax)) {
219 if (dev_dax->nr_range > 1) {
220 dev_warn(dev, "static pgmap / multi-range device conflict\n");
221 return -EINVAL;
222 }
223
224 pgmap = dev_dax->pgmap;
225 } else {
226 size_t pgmap_size;
227
228 if (dev_dax->pgmap) {
229 dev_warn(dev, "dynamic-dax with pre-populated page map\n");
230 return -EINVAL;
231 }
232
233 pgmap_size = struct_size(pgmap, ranges, dev_dax->nr_range - 1);
234 pgmap = devm_kzalloc(dev, pgmap_size, GFP_KERNEL);
235 if (!pgmap)
236 return -ENOMEM;
237
238 pgmap->nr_range = dev_dax->nr_range;
239 dev_dax->pgmap = pgmap;
240
241 for (i = 0; i < dev_dax->nr_range; i++) {
242 struct range *range = &dev_dax->ranges[i].range;
243
244 pgmap->ranges[i] = *range;
245 }
246 }
247
248 for (i = 0; i < dev_dax->nr_range; i++) {
249 struct range *range = &dev_dax->ranges[i].range;
250
251 if (!devm_request_mem_region(dev, range->start,
252 range_len(range), dev_name(dev))) {
253 dev_warn(dev, "mapping%d: %#llx-%#llx could not reserve range\n",
254 i, range->start, range->end);
255 return -EBUSY;
256 }
257 }
258
259 /* Cache size now; it cannot change while driver is bound */
260 dev_dax->cached_size = 0;
261 for (i = 0; i < dev_dax->nr_range; i++)
262 dev_dax->cached_size += range_len(&dev_dax->ranges[i].range);
263
264 /*
265 * Use MEMORY_DEVICE_FS_DAX without setting vmemmap_shift, leaving
266 * folios at order-0. Unlike device.c (MEMORY_DEVICE_GENERIC), this
267 * lets fs-dax dynamically build compound folios as needed, similar
268 * to pmem behavior.
269 */
270 pgmap->type = MEMORY_DEVICE_FS_DAX;
271 pgmap->ops = &fsdev_pagemap_ops;
272 pgmap->owner = dev_dax;
273
274 addr = devm_memremap_pages(dev, pgmap);
275 if (IS_ERR(addr))
276 return PTR_ERR(addr);
277
278 /*
279 * Clear any stale compound folio state left over from a previous
280 * driver (e.g., device_dax with vmemmap_shift). Also register this
281 * as a devm action so folio state is cleared on unbind, ensuring
282 * clean pages for subsequent drivers (e.g., kmem for system-ram).
283 */
284 fsdev_clear_folio_state(dev_dax);
285 rc = devm_add_action_or_reset(dev, fsdev_clear_folio_state_action,
286 dev_dax);
287 if (rc)
288 return rc;
289
290 /* Detect whether the data is at a non-zero offset into the memory */
291 if (pgmap->range.start != dev_dax->ranges[0].range.start) {
292 u64 phys = dev_dax->ranges[0].range.start;
293 u64 pgmap_phys = dev_dax->pgmap[0].range.start;
294
295 if (!WARN_ON(pgmap_phys > phys))
296 data_offset = phys - pgmap_phys;
297
298 pr_debug("%s: offset detected phys=%llx pgmap_phys=%llx offset=%llx\n",
299 __func__, phys, pgmap_phys, data_offset);
300 }
301 dev_dax->virt_addr = addr + data_offset;
302
303 inode = dax_inode(dax_dev);
304 cdev = inode->i_cdev;
305 cdev_init(cdev, &fsdev_fops);
306 cdev->owner = dev->driver->owner;
307 cdev_set_parent(cdev, &dev->kobj);
308 rc = cdev_add(cdev, dev->devt, 1);
309 if (rc)
310 return rc;
311
312 rc = devm_add_action_or_reset(dev, fsdev_cdev_del, cdev);
313 if (rc)
314 return rc;
315
316 /* Set the dax operations for fs-dax access path */
317 rc = dax_set_ops(dax_dev, &dev_dax_ops);
318 if (rc)
319 return rc;
320
321 rc = devm_add_action_or_reset(dev, fsdev_clear_ops, dev_dax);
322 if (rc)
323 return rc;
324
325 run_dax(dax_dev);
326 return devm_add_action_or_reset(dev, fsdev_kill, dev_dax);
327}
328
329static struct dax_device_driver fsdev_dax_driver = {
330 .probe = fsdev_dax_probe,
331 .type = DAXDRV_FSDEV_TYPE,
332};
333
334static int __init dax_init(void)
335{
336 return dax_driver_register(&fsdev_dax_driver);
337}
338
339static void __exit dax_exit(void)
340{
341 dax_driver_unregister(&fsdev_dax_driver);
342}
343
344MODULE_AUTHOR("John Groves");
345MODULE_DESCRIPTION("FS-DAX Device: fs-dax compatible devdax driver");
346MODULE_LICENSE("GPL");
347module_init(dax_init);
348module_exit(dax_exit);
349MODULE_ALIAS_DAX_DEVICE(0);
drivers/dax/super.c+104−3
@@ -14,6 +14,7 @@ · 13 unmodified lines
14#include <linux/fs.h>14#include <linux/fs.h>
15#include <linux/cacheinfo.h>15#include <linux/cacheinfo.h>
16#include "dax-private.h"16#include "dax-private.h"
17#include "bus.h"
17 18
18/**19/**
19 * struct dax_device - anchor object for dax services20 * struct dax_device - anchor object for dax services
@@ -111,6 +112,10 @@ struct dax_device *fs_dax_get_by_bdev(struct block_device *bdev, u64 *start_off, · 91 unmodified lines
111}112}
112EXPORT_SYMBOL_GPL(fs_dax_get_by_bdev);113EXPORT_SYMBOL_GPL(fs_dax_get_by_bdev);
113 114
115#endif /* CONFIG_BLOCK && CONFIG_FS_DAX */
116
117#if IS_ENABLED(CONFIG_FS_DAX)
118
114void fs_put_dax(struct dax_device *dax_dev, void *holder)119void fs_put_dax(struct dax_device *dax_dev, void *holder)
115{120{
116 if (dax_dev && holder &&121 if (dax_dev && holder &&
@@ -119,7 +124,66 @@ void fs_put_dax(struct dax_device *dax_dev, void *holder) · 2 unmodified lines
119 put_dax(dax_dev);124 put_dax(dax_dev);
120}125}
121EXPORT_SYMBOL_GPL(fs_put_dax);126EXPORT_SYMBOL_GPL(fs_put_dax);
122#endif /* CONFIG_BLOCK && CONFIG_FS_DAX */127
128/**
129 * fs_dax_get() - get ownership of a devdax via holder/holder_ops
130 *
131 * fs-dax file systems call this function to prepare to use a devdax device for
132 * fsdax. This is like fs_dax_get_by_bdev(), but the caller already has struct
133 * dev_dax (and there is no bdev). The holder makes this exclusive.
134 *
135 * @dax_dev: dev to be prepared for fs-dax usage
136 * @holder: filesystem or mapped device inside the dax_device
137 * @hops: operations for the inner holder
138 *
139 * Returns: 0 on success, <0 on failure
140 */
141int fs_dax_get(struct dax_device *dax_dev, void *holder,
142 const struct dax_holder_operations *hops)
143{
144 struct dev_dax *dev_dax;
145 struct dax_device_driver *dax_drv;
146 int id;
147
148 id = dax_read_lock();
149 if (!dax_dev || !dax_alive(dax_dev) || !igrab(&dax_dev->inode)) {
150 dax_read_unlock(id);
151 return -ENODEV;
152 }
153 dax_read_unlock(id);
154
155 /* Verify the device is bound to fsdev_dax driver */
156 dev_dax = dax_get_private(dax_dev);
157 if (!dev_dax) {
158 iput(&dax_dev->inode);
159 return -ENODEV;
160 }
161
162 device_lock(&dev_dax->dev);
163 if (!dev_dax->dev.driver) {
164 device_unlock(&dev_dax->dev);
165 iput(&dax_dev->inode);
166 return -ENODEV;
167 }
168 dax_drv = to_dax_drv(dev_dax->dev.driver);
169 if (dax_drv->type != DAXDRV_FSDEV_TYPE) {
170 device_unlock(&dev_dax->dev);
171 iput(&dax_dev->inode);
172 return -EOPNOTSUPP;
173 }
174 device_unlock(&dev_dax->dev);
175
176 if (cmpxchg(&dax_dev->holder_data, NULL, holder)) {
177 iput(&dax_dev->inode);
178 return -EBUSY;
179 }
180
181 dax_dev->holder_ops = hops;
182
183 return 0;
184}
185EXPORT_SYMBOL_GPL(fs_dax_get);
186#endif /* CONFIG_FS_DAX */
123 187
124enum dax_device_flags {188enum dax_device_flags {
125 /* !alive + rcu grace period == no new operations / mappings */189 /* !alive + rcu grace period == no new operations / mappings */
@@ -157,6 +221,9 @@ long dax_direct_access(struct dax_device *dax_dev, pgoff_t pgoff, long nr_pages, · 31 unmodified lines
157 if (!dax_alive(dax_dev))221 if (!dax_alive(dax_dev))
158 return -ENXIO;222 return -ENXIO;
159 223
224 if (!dax_dev->ops)
225 return -EOPNOTSUPP;
226
160 if (nr_pages < 0)227 if (nr_pages < 0)
161 return -EINVAL;228 return -EINVAL;
162 229
@@ -207,6 +274,10 @@ int dax_zero_page_range(struct dax_device *dax_dev, pgoff_t pgoff, · 44 unmodified lines
207 274
208 if (!dax_alive(dax_dev))275 if (!dax_alive(dax_dev))
209 return -ENXIO;276 return -ENXIO;
277
278 if (!dax_dev->ops)
279 return -EOPNOTSUPP;
280
210 /*281 /*
211 * There are no callers that want to zero more than one page as of now.282 * There are no callers that want to zero more than one page as of now.
212 * Once users are there, this check can be removed after the283 * Once users are there, this check can be removed after the
@@ -223,7 +294,7 @@ EXPORT_SYMBOL_GPL(dax_zero_page_range); · 10 unmodified lines
223size_t dax_recovery_write(struct dax_device *dax_dev, pgoff_t pgoff,294size_t dax_recovery_write(struct dax_device *dax_dev, pgoff_t pgoff,
224 void *addr, size_t bytes, struct iov_iter *iter)295 void *addr, size_t bytes, struct iov_iter *iter)
225{296{
226 if (!dax_dev->ops->recovery_write)297 if (!dax_dev->ops || !dax_dev->ops->recovery_write)
227 return 0;298 return 0;
228 return dax_dev->ops->recovery_write(dax_dev, pgoff, addr, bytes, iter);299 return dax_dev->ops->recovery_write(dax_dev, pgoff, addr, bytes, iter);
229}300}
@@ -307,6 +378,35 @@ void set_dax_nomc(struct dax_device *dax_dev) · 77 unmodified lines
307}378}
308EXPORT_SYMBOL_GPL(set_dax_nomc);379EXPORT_SYMBOL_GPL(set_dax_nomc);
309 380
381/**
382 * dax_set_ops - set the dax_operations for a dax_device
383 * @dax_dev: the dax_device to configure
384 * @ops: the operations to set (may be NULL to clear)
385 *
386 * This allows drivers to set the dax_operations after the dax_device
387 * has been allocated. This is needed when the device is created before
388 * the driver that needs specific ops is bound (e.g., fsdev_dax binding
389 * to a dev_dax created by hmem).
390 *
391 * When setting non-NULL ops, fails if ops are already set (returns -EBUSY).
392 * When clearing ops (NULL), always succeeds.
393 *
394 * Return: 0 on success, -EBUSY if ops already set
395 */
396int dax_set_ops(struct dax_device *dax_dev, const struct dax_operations *ops)
397{
398 if (ops) {
399 /* Setting ops: fail if already set */
400 if (cmpxchg(&dax_dev->ops, NULL, ops) != NULL)
401 return -EBUSY;
402 } else {
403 /* Clearing ops: always allowed */
404 dax_dev->ops = NULL;
405 }
406 return 0;
407}
408EXPORT_SYMBOL_GPL(dax_set_ops);
409
310bool dax_alive(struct dax_device *dax_dev)410bool dax_alive(struct dax_device *dax_dev)
311{411{
312 lockdep_assert_held(&dax_srcu);412 lockdep_assert_held(&dax_srcu);
@@ -421,7 +521,7 @@ static int dax_set(struct inode *inode, void *data) · 108 unmodified lines
421 return 0;521 return 0;
422}522}
423 523
424static struct dax_device *dax_dev_get(dev_t devt)524struct dax_device *dax_dev_get(dev_t devt)
425{525{
426 struct dax_device *dax_dev;526 struct dax_device *dax_dev;
427 struct inode *inode;527 struct inode *inode;
@@ -444,6 +544,7 @@ static struct dax_device *dax_dev_get(dev_t devt) · 16 unmodified lines
444 544
445 return dax_dev;545 return dax_dev;
446}546}
547EXPORT_SYMBOL_GPL(dax_dev_get);
447 548
448struct dax_device *alloc_dax(void *private, const struct dax_operations *ops)549struct dax_device *alloc_dax(void *private, const struct dax_operations *ops)
449{550{
fs/dax.c+56−18
@@ -377,6 +377,59 @@ static void dax_folio_make_shared(struct folio *folio) · 376 unmodified lines
377 folio->share = 1;377 folio->share = 1;
378}378}
379 379
380/**
381 * dax_folio_reset_order - Reset a compound DAX folio to order-0 pages
382 * @folio: The folio to reset
383 *
384 * Splits a compound folio back into individual order-0 pages,
385 * clearing compound state and restoring pgmap pointers.
386 *
387 * Returns: the original folio order (0 if already order-0)
388 */
389int dax_folio_reset_order(struct folio *folio)
390{
391 struct dev_pagemap *pgmap = page_pgmap(&folio->page);
392 int order = folio_order(folio);
393
394 /*
395 * DAX maintains the invariant that folio->share != 0 only when
396 * folio->mapping == NULL (enforced by dax_folio_make_shared()).
397 * Equivalently: folio->mapping != NULL implies folio->share == 0.
398 * Callers ensure share has been decremented to zero before
399 * calling here, so unconditionally clearing both fields is
400 * correct.
401 */
402 folio->mapping = NULL;
403 folio->share = 0;
404
405 if (!order) {
406 /*
407 * Restore pgmap explicitly even for order-0 folios. For the
408 * dax_folio_put() caller this is a no-op (same value), but
409 * fsdev_clear_folio_state() may call this on folios that
410 * were previously compound and need pgmap re-established.
411 */
412 folio->pgmap = pgmap;
413 return 0;
414 }
415
416 folio_reset_order(folio);
417
418 for (int i = 0; i < (1UL << order); i++) {
419 struct page *page = folio_page(folio, i);
420 struct folio *f = (struct folio *)page;
421
422 ClearPageHead(page);
423 clear_compound_head(page);
424 f->mapping = NULL;
425 f->share = 0;
426 f->pgmap = pgmap;
427 }
428
429 return order;
430}
431EXPORT_SYMBOL_GPL(dax_folio_reset_order);
432
380static inline unsigned long dax_folio_put(struct folio *folio)433static inline unsigned long dax_folio_put(struct folio *folio)
381{434{
382 unsigned long ref;435 unsigned long ref;
@@ -390,28 +443,13 @@ static inline unsigned long dax_folio_put(struct folio *folio) · 7 unmodified lines
390 if (ref)443 if (ref)
391 return ref;444 return ref;
392 445
393 folio->mapping = NULL;446 order = dax_folio_reset_order(folio);
394 order = folio_order(folio);
395 if (!order)
396 return 0;
397 folio_reset_order(folio);
398 447
448 /* Debug check: verify refcounts are zero for all sub-folios */
399 for (i = 0; i < (1UL << order); i++) {449 for (i = 0; i < (1UL << order); i++) {
400 struct dev_pagemap *pgmap = page_pgmap(&folio->page);
401 struct page *page = folio_page(folio, i);450 struct page *page = folio_page(folio, i);
402 struct folio *new_folio = (struct folio *)page;
403
404 ClearPageHead(page);
405 clear_compound_head(page);
406 451
407 new_folio->mapping = NULL;452 WARN_ON_ONCE(folio_ref_count((struct folio *)page));
408 /*
409 * Reset pgmap which was over-written by
410 * prep_compound_page().
411 */
412 new_folio->pgmap = pgmap;
413 new_folio->share = 0;
414 WARN_ON_ONCE(folio_ref_count(new_folio));
415 }453 }
416 454
417 return ref;455 return ref;
include/linux/dax.h+15−4
@@ -54,6 +54,7 @@ struct dax_device *alloc_dax(void *private, const struct dax_operations *ops); · 53 unmodified lines
54void *dax_holder(struct dax_device *dax_dev);54void *dax_holder(struct dax_device *dax_dev);
55void put_dax(struct dax_device *dax_dev);55void put_dax(struct dax_device *dax_dev);
56void kill_dax(struct dax_device *dax_dev);56void kill_dax(struct dax_device *dax_dev);
57struct dax_device *dax_dev_get(dev_t devt);
57void dax_write_cache(struct dax_device *dax_dev, bool wc);58void dax_write_cache(struct dax_device *dax_dev, bool wc);
58bool dax_write_cache_enabled(struct dax_device *dax_dev);59bool dax_write_cache_enabled(struct dax_device *dax_dev);
59bool dax_synchronous(struct dax_device *dax_dev);60bool dax_synchronous(struct dax_device *dax_dev);
@@ -130,7 +131,6 @@ int dax_add_host(struct dax_device *dax_dev, struct gendisk *disk); · 70 unmodified lines
130void dax_remove_host(struct gendisk *disk);131void dax_remove_host(struct gendisk *disk);
131struct dax_device *fs_dax_get_by_bdev(struct block_device *bdev, u64 *start_off,132struct dax_device *fs_dax_get_by_bdev(struct block_device *bdev, u64 *start_off,
132 void *holder, const struct dax_holder_operations *ops);133 void *holder, const struct dax_holder_operations *ops);
133void fs_put_dax(struct dax_device *dax_dev, void *holder);
134#else134#else
135static inline int dax_add_host(struct dax_device *dax_dev, struct gendisk *disk)135static inline int dax_add_host(struct dax_device *dax_dev, struct gendisk *disk)
136{136{
@@ -145,14 +145,15 @@ static inline struct dax_device *fs_dax_get_by_bdev(struct block_device *bdev, · 8 unmodified lines
145{145{
146 return NULL;146 return NULL;
147}147}
148static inline void fs_put_dax(struct dax_device *dax_dev, void *holder)
149{
150}
151#endif /* CONFIG_BLOCK && CONFIG_FS_DAX */148#endif /* CONFIG_BLOCK && CONFIG_FS_DAX */
152 149
153#if IS_ENABLED(CONFIG_FS_DAX)150#if IS_ENABLED(CONFIG_FS_DAX)
151void fs_put_dax(struct dax_device *dax_dev, void *holder);
152int fs_dax_get(struct dax_device *dax_dev, void *holder,
153 const struct dax_holder_operations *hops);
154int dax_writeback_mapping_range(struct address_space *mapping,154int dax_writeback_mapping_range(struct address_space *mapping,
155 struct dax_device *dax_dev, struct writeback_control *wbc);155 struct dax_device *dax_dev, struct writeback_control *wbc);
156int dax_folio_reset_order(struct folio *folio);
156 157
157struct page *dax_layout_busy_page(struct address_space *mapping);158struct page *dax_layout_busy_page(struct address_space *mapping);
158struct page *dax_layout_busy_page_range(struct address_space *mapping, loff_t start, loff_t end);159struct page *dax_layout_busy_page_range(struct address_space *mapping, loff_t start, loff_t end);
@@ -163,6 +164,15 @@ dax_entry_t dax_lock_mapping_entry(struct address_space *mapping, · 4 unmodified lines
163void dax_unlock_mapping_entry(struct address_space *mapping,164void dax_unlock_mapping_entry(struct address_space *mapping,
164 unsigned long index, dax_entry_t cookie);165 unsigned long index, dax_entry_t cookie);
165#else166#else
167static inline void fs_put_dax(struct dax_device *dax_dev, void *holder)
168{
169}
170
171static inline int fs_dax_get(struct dax_device *dax_dev, void *holder,
172 const struct dax_holder_operations *hops)
173{
174 return -EOPNOTSUPP;
175}
166static inline struct page *dax_layout_busy_page(struct address_space *mapping)176static inline struct page *dax_layout_busy_page(struct address_space *mapping)
167{177{
168 return NULL;178 return NULL;
@@ -242,6 +252,7 @@ static inline void dax_break_layout_final(struct inode *inode) · 73 unmodified lines
242 252
243bool dax_alive(struct dax_device *dax_dev);253bool dax_alive(struct dax_device *dax_dev);
244void *dax_get_private(struct dax_device *dax_dev);254void *dax_get_private(struct dax_device *dax_dev);
255int dax_set_ops(struct dax_device *dax_dev, const struct dax_operations *ops);
245long dax_direct_access(struct dax_device *dax_dev, pgoff_t pgoff, long nr_pages,256long dax_direct_access(struct dax_device *dax_dev, pgoff_t pgoff, long nr_pages,
246 enum dax_access_mode mode, void **kaddr, unsigned long *pfn);257 enum dax_access_mode mode, void **kaddr, unsigned long *pfn);
247size_t dax_copy_from_iter(struct dax_device *dax_dev, pgoff_t pgoff, void *addr,258size_t dax_copy_from_iter(struct dax_device *dax_dev, pgoff_t pgoff, void *addr,