History

mm, dax: make pmd_fault() and friends be the same as fault()

Instead of passing in multiple parameters in the pmd_fault() handler, a vmf can be passed in just like a fault() handler. This will simplify code and remove the need for the actual pmd fault handlers to allocate a vmf. Related functions are also modified to do the same. [dave.jiang@intel.com: fix issue with xfs_tests stall when DAX option is off] Link: http://lkml.kernel.org/r/148469861071.195597.3619476895250028518.stgit@djiang5-desk3.ch.intel.com Link: http://lkml.kernel.org/r/1484085142-2297-7-git-send-email-ross.zwisler@linux.intel.com Signed-off-by: Dave Jiang <dave.jiang@intel.com> Reviewed-by: Ross Zwisler <ross.zwisler@linux.intel.com> Reviewed-by: Jan Kara <jack@suse.cz> Cc: Dave Chinner <david@fromorbit.com> Cc: Matthew Wilcox <mawilcox@microsoft.com> Cc: Steven Rostedt <rostedt@goodmis.org> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

Dave Jiang committed 9 years ago

Commit d8a849e1 parent 27a7ffa8 files · +39 −55Browse files

Unified · Split

8 files
  1. drivers/dax/dax.c +7 −9
  2. fs/dax.c +11 −17
  3. fs/ext4/file.c +4 −5
  4. fs/xfs/xfs_file.c +4 −6
  5. include/linux/dax.h +3 −4
  6. include/linux/mm.h +1 −2
  7. include/trace/events/fs_dax.h +7 −8
  8. mm/memory.c +2 −4
drivers/dax/dax.c+7−9
@@ -473,10 +473,9 @@ static int dax_dev_fault(struct vm_area_struct *vma, struct vm_fault *vmf) · 472 unmodified lines
473}473}
474 474
475static int __dax_dev_pmd_fault(struct dax_dev *dax_dev,475static int __dax_dev_pmd_fault(struct dax_dev *dax_dev,
476 struct vm_area_struct *vma, unsigned long addr, pmd_t *pmd,476 struct vm_area_struct *vma, struct vm_fault *vmf)
477 unsigned int flags)
478{477{
479 unsigned long pmd_addr = addr & PMD_MASK;478 unsigned long pmd_addr = vmf->address & PMD_MASK;
480 struct device *dev = &dax_dev->dev;479 struct device *dev = &dax_dev->dev;
481 struct dax_region *dax_region;480 struct dax_region *dax_region;
482 phys_addr_t phys;481 phys_addr_t phys;
@@ -508,23 +507,22 @@ static int __dax_dev_pmd_fault(struct dax_dev *dax_dev, · 25 unmodified lines
508 507
509 pfn = phys_to_pfn_t(phys, dax_region->pfn_flags);508 pfn = phys_to_pfn_t(phys, dax_region->pfn_flags);
510 509
511 return vmf_insert_pfn_pmd(vma, addr, pmd, pfn,510 return vmf_insert_pfn_pmd(vma, vmf->address, vmf->pmd, pfn,
512 flags & FAULT_FLAG_WRITE);511 vmf->flags & FAULT_FLAG_WRITE);
513}512}
514 513
515static int dax_dev_pmd_fault(struct vm_area_struct *vma, unsigned long addr,514static int dax_dev_pmd_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
516 pmd_t *pmd, unsigned int flags)
517{515{
518 int rc;516 int rc;
519 struct file *filp = vma->vm_file;517 struct file *filp = vma->vm_file;
520 struct dax_dev *dax_dev = filp->private_data;518 struct dax_dev *dax_dev = filp->private_data;
521 519
522 dev_dbg(&dax_dev->dev, "%s: %s: %s (%#lx - %#lx)\n", __func__,520 dev_dbg(&dax_dev->dev, "%s: %s: %s (%#lx - %#lx)\n", __func__,
523 current->comm, (flags & FAULT_FLAG_WRITE)521 current->comm, (vmf->flags & FAULT_FLAG_WRITE)
524 ? "write" : "read", vma->vm_start, vma->vm_end);522 ? "write" : "read", vma->vm_start, vma->vm_end);
525 523
526 rcu_read_lock();524 rcu_read_lock();
527 rc = __dax_dev_pmd_fault(dax_dev, vma, addr, pmd, flags);525 rc = __dax_dev_pmd_fault(dax_dev, vma, vmf);
528 rcu_read_unlock();526 rcu_read_unlock();
529 527
530 return rc;528 return rc;
fs/dax.c+11−17
@@ -1340,18 +1340,17 @@ fallback: · 1339 unmodified lines
1340 return VM_FAULT_FALLBACK;1340 return VM_FAULT_FALLBACK;
1341}1341}
1342 1342
1343int dax_iomap_pmd_fault(struct vm_area_struct *vma, unsigned long address,1343int dax_iomap_pmd_fault(struct vm_area_struct *vma, struct vm_fault *vmf,
1344 pmd_t *pmd, unsigned int flags, struct iomap_ops *ops)1344 struct iomap_ops *ops)
1345{1345{
1346 struct address_space *mapping = vma->vm_file->f_mapping;1346 struct address_space *mapping = vma->vm_file->f_mapping;
1347 unsigned long pmd_addr = address & PMD_MASK;1347 unsigned long pmd_addr = vmf->address & PMD_MASK;
1348 bool write = flags & FAULT_FLAG_WRITE;1348 bool write = vmf->flags & FAULT_FLAG_WRITE;
1349 unsigned int iomap_flags = (write ? IOMAP_WRITE : 0) | IOMAP_FAULT;1349 unsigned int iomap_flags = (write ? IOMAP_WRITE : 0) | IOMAP_FAULT;
1350 struct inode *inode = mapping->host;1350 struct inode *inode = mapping->host;
1351 int result = VM_FAULT_FALLBACK;1351 int result = VM_FAULT_FALLBACK;
1352 struct iomap iomap = { 0 };1352 struct iomap iomap = { 0 };
1353 pgoff_t max_pgoff, pgoff;1353 pgoff_t max_pgoff, pgoff;
1354 struct vm_fault vmf;
1355 void *entry;1354 void *entry;
1356 loff_t pos;1355 loff_t pos;
1357 int error;1356 int error;
@@ -1364,7 +1363,7 @@ int dax_iomap_pmd_fault(struct vm_area_struct *vma, unsigned long address, · 6 unmodified lines
1364 pgoff = linear_page_index(vma, pmd_addr);1363 pgoff = linear_page_index(vma, pmd_addr);
1365 max_pgoff = (i_size_read(inode) - 1) >> PAGE_SHIFT;1364 max_pgoff = (i_size_read(inode) - 1) >> PAGE_SHIFT;
1366 1365
1367 trace_dax_pmd_fault(inode, vma, address, flags, pgoff, max_pgoff, 0);1366 trace_dax_pmd_fault(inode, vma, vmf, max_pgoff, 0);
1368 1367
1369 /* Fall back to PTEs if we're going to COW */1368 /* Fall back to PTEs if we're going to COW */
1370 if (write && !(vma->vm_flags & VM_SHARED))1369 if (write && !(vma->vm_flags & VM_SHARED))
@@ -1408,21 +1407,17 @@ int dax_iomap_pmd_fault(struct vm_area_struct *vma, unsigned long address, · 37 unmodified lines
1408 if (IS_ERR(entry))1407 if (IS_ERR(entry))
1409 goto finish_iomap;1408 goto finish_iomap;
1410 1409
1411 vmf.pgoff = pgoff;
1412 vmf.flags = flags;
1413 vmf.gfp_mask = mapping_gfp_mask(mapping) | __GFP_IO;
1414
1415 switch (iomap.type) {1410 switch (iomap.type) {
1416 case IOMAP_MAPPED:1411 case IOMAP_MAPPED:
1417 result = dax_pmd_insert_mapping(vma, pmd, &vmf, address,1412 result = dax_pmd_insert_mapping(vma, vmf->pmd, vmf,
1418 &iomap, pos, write, &entry);1413 vmf->address, &iomap, pos, write, &entry);
1419 break;1414 break;
1420 case IOMAP_UNWRITTEN:1415 case IOMAP_UNWRITTEN:
1421 case IOMAP_HOLE:1416 case IOMAP_HOLE:
1422 if (WARN_ON_ONCE(write))1417 if (WARN_ON_ONCE(write))
1423 goto unlock_entry;1418 goto unlock_entry;
1424 result = dax_pmd_load_hole(vma, pmd, &vmf, address, &iomap,1419 result = dax_pmd_load_hole(vma, vmf->pmd, vmf, vmf->address,
1425 &entry);1420 &iomap, &entry);
1426 break;1421 break;
1427 default:1422 default:
1428 WARN_ON_ONCE(1);1423 WARN_ON_ONCE(1);
@@ -1448,12 +1443,11 @@ int dax_iomap_pmd_fault(struct vm_area_struct *vma, unsigned long address, · 19 unmodified lines
1448 }1443 }
1449 fallback:1444 fallback:
1450 if (result == VM_FAULT_FALLBACK) {1445 if (result == VM_FAULT_FALLBACK) {
1451 split_huge_pmd(vma, pmd, address);1446 split_huge_pmd(vma, vmf->pmd, vmf->address);
1452 count_vm_event(THP_FAULT_FALLBACK);1447 count_vm_event(THP_FAULT_FALLBACK);
1453 }1448 }
1454out:1449out:
1455 trace_dax_pmd_fault_done(inode, vma, address, flags, pgoff, max_pgoff,1450 trace_dax_pmd_fault_done(inode, vma, vmf, max_pgoff, result);
1456 result);
1457 return result;1451 return result;
1458}1452}
1459EXPORT_SYMBOL_GPL(dax_iomap_pmd_fault);1453EXPORT_SYMBOL_GPL(dax_iomap_pmd_fault);
fs/ext4/file.c+4−5
@@ -273,21 +273,20 @@ static int ext4_dax_fault(struct vm_area_struct *vma, struct vm_fault *vmf) · 272 unmodified lines
273 return result;273 return result;
274}274}
275 275
276static int ext4_dax_pmd_fault(struct vm_area_struct *vma, unsigned long addr,276static int
277 pmd_t *pmd, unsigned int flags)277ext4_dax_pmd_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
278{278{
279 int result;279 int result;
280 struct inode *inode = file_inode(vma->vm_file);280 struct inode *inode = file_inode(vma->vm_file);
281 struct super_block *sb = inode->i_sb;281 struct super_block *sb = inode->i_sb;
282 bool write = flags & FAULT_FLAG_WRITE;282 bool write = vmf->flags & FAULT_FLAG_WRITE;
283 283
284 if (write) {284 if (write) {
285 sb_start_pagefault(sb);285 sb_start_pagefault(sb);
286 file_update_time(vma->vm_file);286 file_update_time(vma->vm_file);
287 }287 }
288 down_read(&EXT4_I(inode)->i_mmap_sem);288 down_read(&EXT4_I(inode)->i_mmap_sem);
289 result = dax_iomap_pmd_fault(vma, addr, pmd, flags,289 result = dax_iomap_pmd_fault(vma, vmf, &ext4_iomap_ops);
290 &ext4_iomap_ops);
291 up_read(&EXT4_I(inode)->i_mmap_sem);290 up_read(&EXT4_I(inode)->i_mmap_sem);
292 if (write)291 if (write)
293 sb_end_pagefault(sb);292 sb_end_pagefault(sb);
fs/xfs/xfs_file.c+4−6
@@ -1432,9 +1432,7 @@ xfs_filemap_fault( · 1431 unmodified lines
1432STATIC int1432STATIC int
1433xfs_filemap_pmd_fault(1433xfs_filemap_pmd_fault(
1434 struct vm_area_struct *vma,1434 struct vm_area_struct *vma,
1435 unsigned long addr,1435 struct vm_fault *vmf)
1436 pmd_t *pmd,
1437 unsigned int flags)
1438{1436{
1439 struct inode *inode = file_inode(vma->vm_file);1437 struct inode *inode = file_inode(vma->vm_file);
1440 struct xfs_inode *ip = XFS_I(inode);1438 struct xfs_inode *ip = XFS_I(inode);
@@ -1445,16 +1443,16 @@ xfs_filemap_pmd_fault( · 4 unmodified lines
1445 1443
1446 trace_xfs_filemap_pmd_fault(ip);1444 trace_xfs_filemap_pmd_fault(ip);
1447 1445
1448 if (flags & FAULT_FLAG_WRITE) {1446 if (vmf->flags & FAULT_FLAG_WRITE) {
1449 sb_start_pagefault(inode->i_sb);1447 sb_start_pagefault(inode->i_sb);
1450 file_update_time(vma->vm_file);1448 file_update_time(vma->vm_file);
1451 }1449 }
1452 1450
1453 xfs_ilock(XFS_I(inode), XFS_MMAPLOCK_SHARED);1451 xfs_ilock(XFS_I(inode), XFS_MMAPLOCK_SHARED);
1454 ret = dax_iomap_pmd_fault(vma, addr, pmd, flags, &xfs_iomap_ops);1452 ret = dax_iomap_pmd_fault(vma, vmf, &xfs_iomap_ops);
1455 xfs_iunlock(XFS_I(inode), XFS_MMAPLOCK_SHARED);1453 xfs_iunlock(XFS_I(inode), XFS_MMAPLOCK_SHARED);
1456 1454
1457 if (flags & FAULT_FLAG_WRITE)1455 if (vmf->flags & FAULT_FLAG_WRITE)
1458 sb_end_pagefault(inode->i_sb);1456 sb_end_pagefault(inode->i_sb);
1459 1457
1460 return ret;1458 return ret;
include/linux/dax.h+3−4
@@ -71,16 +71,15 @@ static inline unsigned int dax_radix_order(void *entry) · 70 unmodified lines
71 return PMD_SHIFT - PAGE_SHIFT;71 return PMD_SHIFT - PAGE_SHIFT;
72 return 0;72 return 0;
73}73}
74int dax_iomap_pmd_fault(struct vm_area_struct *vma, unsigned long address,74int dax_iomap_pmd_fault(struct vm_area_struct *vma, struct vm_fault *vmf,
75 pmd_t *pmd, unsigned int flags, struct iomap_ops *ops);75 struct iomap_ops *ops);
76#else76#else
77static inline unsigned int dax_radix_order(void *entry)77static inline unsigned int dax_radix_order(void *entry)
78{78{
79 return 0;79 return 0;
80}80}
81static inline int dax_iomap_pmd_fault(struct vm_area_struct *vma,81static inline int dax_iomap_pmd_fault(struct vm_area_struct *vma,
82 unsigned long address, pmd_t *pmd, unsigned int flags,82 struct vm_fault *vmf, struct iomap_ops *ops)
83 struct iomap_ops *ops)
84{83{
85 return VM_FAULT_FALLBACK;84 return VM_FAULT_FALLBACK;
86}85}
include/linux/mm.h+1−2
@@ -351,8 +351,7 @@ struct vm_operations_struct { · 350 unmodified lines
351 void (*close)(struct vm_area_struct * area);351 void (*close)(struct vm_area_struct * area);
352 int (*mremap)(struct vm_area_struct * area);352 int (*mremap)(struct vm_area_struct * area);
353 int (*fault)(struct vm_area_struct *vma, struct vm_fault *vmf);353 int (*fault)(struct vm_area_struct *vma, struct vm_fault *vmf);
354 int (*pmd_fault)(struct vm_area_struct *, unsigned long address,354 int (*pmd_fault)(struct vm_area_struct *vma, struct vm_fault *vmf);
355 pmd_t *, unsigned int flags);
356 void (*map_pages)(struct vm_fault *vmf,355 void (*map_pages)(struct vm_fault *vmf,
357 pgoff_t start_pgoff, pgoff_t end_pgoff);356 pgoff_t start_pgoff, pgoff_t end_pgoff);
358 357
include/trace/events/fs_dax.h+7−8
@@ -8,9 +8,8 @@ · 7 unmodified lines
8 8
9DECLARE_EVENT_CLASS(dax_pmd_fault_class,9DECLARE_EVENT_CLASS(dax_pmd_fault_class,
10 TP_PROTO(struct inode *inode, struct vm_area_struct *vma,10 TP_PROTO(struct inode *inode, struct vm_area_struct *vma,
11 unsigned long address, unsigned int flags, pgoff_t pgoff,11 struct vm_fault *vmf, pgoff_t max_pgoff, int result),
12 pgoff_t max_pgoff, int result),12 TP_ARGS(inode, vma, vmf, max_pgoff, result),
13 TP_ARGS(inode, vma, address, flags, pgoff, max_pgoff, result),
14 TP_STRUCT__entry(13 TP_STRUCT__entry(
15 __field(unsigned long, ino)14 __field(unsigned long, ino)
16 __field(unsigned long, vm_start)15 __field(unsigned long, vm_start)
@@ -29,9 +28,9 @@ DECLARE_EVENT_CLASS(dax_pmd_fault_class, · 12 unmodified lines
29 __entry->vm_start = vma->vm_start;28 __entry->vm_start = vma->vm_start;
30 __entry->vm_end = vma->vm_end;29 __entry->vm_end = vma->vm_end;
31 __entry->vm_flags = vma->vm_flags;30 __entry->vm_flags = vma->vm_flags;
32 __entry->address = address;31 __entry->address = vmf->address;
33 __entry->flags = flags;32 __entry->flags = vmf->flags;
34 __entry->pgoff = pgoff;33 __entry->pgoff = vmf->pgoff;
35 __entry->max_pgoff = max_pgoff;34 __entry->max_pgoff = max_pgoff;
36 __entry->result = result;35 __entry->result = result;
37 ),36 ),
@@ -54,9 +53,9 @@ DECLARE_EVENT_CLASS(dax_pmd_fault_class, · 16 unmodified lines
54#define DEFINE_PMD_FAULT_EVENT(name) \53#define DEFINE_PMD_FAULT_EVENT(name) \
55DEFINE_EVENT(dax_pmd_fault_class, name, \54DEFINE_EVENT(dax_pmd_fault_class, name, \
56 TP_PROTO(struct inode *inode, struct vm_area_struct *vma, \55 TP_PROTO(struct inode *inode, struct vm_area_struct *vma, \
57 unsigned long address, unsigned int flags, pgoff_t pgoff, \56 struct vm_fault *vmf, \
58 pgoff_t max_pgoff, int result), \57 pgoff_t max_pgoff, int result), \
59 TP_ARGS(inode, vma, address, flags, pgoff, max_pgoff, result))58 TP_ARGS(inode, vma, vmf, max_pgoff, result))
60 59
61DEFINE_PMD_FAULT_EVENT(dax_pmd_fault);60DEFINE_PMD_FAULT_EVENT(dax_pmd_fault);
62DEFINE_PMD_FAULT_EVENT(dax_pmd_fault_done);61DEFINE_PMD_FAULT_EVENT(dax_pmd_fault_done);
mm/memory.c+2−4
@@ -3475,8 +3475,7 @@ static int create_huge_pmd(struct vm_fault *vmf) · 3474 unmodified lines
3475 if (vma_is_anonymous(vma))3475 if (vma_is_anonymous(vma))
3476 return do_huge_pmd_anonymous_page(vmf);3476 return do_huge_pmd_anonymous_page(vmf);
3477 if (vma->vm_ops->pmd_fault)3477 if (vma->vm_ops->pmd_fault)
3478 return vma->vm_ops->pmd_fault(vma, vmf->address, vmf->pmd,3478 return vma->vm_ops->pmd_fault(vma, vmf);
3479 vmf->flags);
3480 return VM_FAULT_FALLBACK;3479 return VM_FAULT_FALLBACK;
3481}3480}
3482 3481
@@ -3485,8 +3484,7 @@ static int wp_huge_pmd(struct vm_fault *vmf, pmd_t orig_pmd) · 2 unmodified lines
3485 if (vma_is_anonymous(vmf->vma))3484 if (vma_is_anonymous(vmf->vma))
3486 return do_huge_pmd_wp_page(vmf, orig_pmd);3485 return do_huge_pmd_wp_page(vmf, orig_pmd);
3487 if (vmf->vma->vm_ops->pmd_fault)3486 if (vmf->vma->vm_ops->pmd_fault)
3488 return vmf->vma->vm_ops->pmd_fault(vmf->vma, vmf->address,3487 return vmf->vma->vm_ops->pmd_fault(vmf->vma, vmf);
3489 vmf->pmd, vmf->flags);
3490 3488
3491 /* COW handled on pte level: split pmd */3489 /* COW handled on pte level: split pmd */
3492 VM_BUG_ON_VMA(vmf->vma->vm_flags & VM_SHARED, vmf->vma);3490 VM_BUG_ON_VMA(vmf->vma->vm_flags & VM_SHARED, vmf->vma);