// SPDX-License-Identifier: GPL-2.0/* * This is a module to test the HMM (Heterogeneous Memory Management) * mirror and zone device private memory migration APIs of the kernel. * Userspace programs can register with the driver to mirror their own address * space and can use the device to read/write any valid virtual address. */#include <linux/init.h>#include <linux/fs.h>#include <linux/mm.h>#include <linux/module.h>#include <linux/kernel.h>#include <linux/cdev.h>#include <linux/device.h>#include <linux/memremap.h>#include <linux/mutex.h>#include <linux/rwsem.h>#include <linux/sched.h>#include <linux/slab.h>#include <linux/highmem.h>#include <linux/delay.h>#include <linux/pagemap.h>#include <linux/hmm.h>#include <linux/vmalloc.h>#include <linux/swap.h>#include <linux/swapops.h>#include <linux/sched/mm.h>#include <linux/platform_device.h>#include <linux/rmap.h>#include <linux/mmu_notifier.h>#include <linux/migrate.h> #include "test_hmm_uapi.h" #define DMIRROR_NDEVICES 4#define DMIRROR_RANGE_FAULT_TIMEOUT 1000#define DEVMEM_CHUNK_SIZE (256 * 1024 * 1024U)#define DEVMEM_CHUNKS_RESERVE 16 /* * For device_private pages, dpage is just a dummy struct page * representing a piece of device memory. dmirror_devmem_alloc_page * allocates a real system memory page as backing storage to fake a * real device. zone_device_data points to that backing page. But * for device_coherent memory, the struct page represents real * physical CPU-accessible memory that we can use directly. */#define BACKING_PAGE(page) (is_device_private_page((page)) ? \ (page)->zone_device_data : (page)) static unsigned long spm_addr_dev0;module_param(spm_addr_dev0, long, 0644);MODULE_PARM_DESC(spm_addr_dev0, "Specify start address for SPM (special purpose memory) used for device 0. By setting this Coherent device type will be used. Make sure spm_addr_dev1 is set too. Minimum SPM size should be DEVMEM_CHUNK_SIZE."); static unsigned long spm_addr_dev1;module_param(spm_addr_dev1, long, 0644);MODULE_PARM_DESC(spm_addr_dev1, "Specify start address for SPM (special purpose memory) used for device 1. By setting this Coherent device type will be used. Make sure spm_addr_dev0 is set too. Minimum SPM size should be DEVMEM_CHUNK_SIZE."); static const struct dev_pagemap_ops dmirror_devmem_ops;static const struct mmu_interval_notifier_ops dmirror_min_ops;static dev_t dmirror_dev; struct dmirror_device; struct dmirror_bounce { void *ptr; unsigned long size; unsigned long addr; unsigned long cpages;}; #define DPT_XA_TAG_ATOMIC 1UL#define DPT_XA_TAG_WRITE 3UL /* * Data structure to track address ranges and register for mmu interval * notifier updates. */struct dmirror_interval { struct mmu_interval_notifier notifier; struct dmirror *dmirror;}; /* * Data attached to the open device file. * Note that it might be shared after a fork(). */struct dmirror { struct dmirror_device *mdevice; struct xarray pt; struct mmu_interval_notifier notifier; struct mutex mutex; __u64 flags;}; /* * ZONE_DEVICE pages for migration and simulating device memory. */struct dmirror_chunk { struct dev_pagemap pagemap; struct dmirror_device *mdevice; bool remove;}; /* * Per device data. */struct dmirror_device { struct cdev cdevice; unsigned int zone_device_type; struct device device; unsigned int devmem_capacity; unsigned int devmem_count; struct dmirror_chunk **devmem_chunks; struct mutex devmem_lock; /* protects the above */ unsigned long calloc; unsigned long cfree; struct page *free_pages; struct folio *free_folios; spinlock_t lock; /* protects the above */}; static struct dmirror_device dmirror_devices[DMIRROR_NDEVICES]; static int dmirror_bounce_init(struct dmirror_bounce *bounce, unsigned long addr, unsigned long size){ bounce->addr = addr; bounce->size = size; bounce->cpages = 0; bounce->ptr = vmalloc(size); if (!bounce->ptr) return -ENOMEM; return 0;} static bool dmirror_is_private_zone(struct dmirror_device *mdevice){ return (mdevice->zone_device_type == HMM_DMIRROR_MEMORY_DEVICE_PRIVATE);} static enum migrate_vma_directiondmirror_select_device(struct dmirror *dmirror){ return (dmirror->mdevice->zone_device_type == HMM_DMIRROR_MEMORY_DEVICE_PRIVATE) ? MIGRATE_VMA_SELECT_DEVICE_PRIVATE : MIGRATE_VMA_SELECT_DEVICE_COHERENT;} static void dmirror_bounce_fini(struct dmirror_bounce *bounce){ vfree(bounce->ptr);} static int dmirror_fops_open(struct inode *inode, struct file *filp){ struct cdev *cdev = inode->i_cdev; struct dmirror *dmirror; int ret; /* Mirror this process address space */ dmirror = kzalloc_obj(*dmirror); if (dmirror == NULL) return -ENOMEM; dmirror->mdevice = container_of(cdev, struct dmirror_device, cdevice); mutex_init(&dmirror->mutex); xa_init(&dmirror->pt); ret = mmu_interval_notifier_insert(&dmirror->notifier, current->mm, 0, ULONG_MAX & PAGE_MASK, &dmirror_min_ops); if (ret) { kfree(dmirror); return ret; } filp->private_data = dmirror; return 0;} static void dmirror_device_evict_chunk(struct dmirror_chunk *chunk){ unsigned long start_pfn = chunk->pagemap.range.start >> PAGE_SHIFT; unsigned long end_pfn = chunk->pagemap.range.end >> PAGE_SHIFT; unsigned long npages = end_pfn - start_pfn + 1; unsigned long i; unsigned long *src_pfns; unsigned long *dst_pfns; unsigned int order = 0; src_pfns = kvcalloc(npages, sizeof(*src_pfns), GFP_KERNEL | __GFP_NOFAIL); dst_pfns = kvcalloc(npages, sizeof(*dst_pfns), GFP_KERNEL | __GFP_NOFAIL); migrate_device_range(src_pfns, start_pfn, npages); for (i = 0; i < npages; i++) { struct page *dpage, *spage; spage = migrate_pfn_to_page(src_pfns[i]); if (!spage || !(src_pfns[i] & MIGRATE_PFN_MIGRATE)) continue; if (WARN_ON(!is_device_private_page(spage) && !is_device_coherent_page(spage))) continue; order = folio_order(page_folio(spage)); spage = BACKING_PAGE(spage); if (src_pfns[i] & MIGRATE_PFN_COMPOUND) { dpage = folio_page(folio_alloc(GFP_HIGHUSER_MOVABLE, order), 0); } else { dpage = alloc_page(GFP_HIGHUSER_MOVABLE | __GFP_NOFAIL); order = 0; } /* TODO Support splitting here */ lock_page(dpage); dst_pfns[i] = migrate_pfn(page_to_pfn(dpage)); if (src_pfns[i] & MIGRATE_PFN_WRITE) dst_pfns[i] |= MIGRATE_PFN_WRITE; if (order) dst_pfns[i] |= MIGRATE_PFN_COMPOUND; folio_copy(page_folio(dpage), page_folio(spage)); } migrate_device_pages(src_pfns, dst_pfns, npages); migrate_device_finalize(src_pfns, dst_pfns, npages); kvfree(src_pfns); kvfree(dst_pfns);} static int dmirror_fops_release(struct inode *inode, struct file *filp){ struct dmirror *dmirror = filp->private_data; struct dmirror_device *mdevice = dmirror->mdevice; int i; mmu_interval_notifier_remove(&dmirror->notifier); if (mdevice->devmem_chunks) { for (i = 0; i < mdevice->devmem_count; i++) { struct dmirror_chunk *devmem = mdevice->devmem_chunks[i]; dmirror_device_evict_chunk(devmem); } } xa_destroy(&dmirror->pt); kfree(dmirror); return 0;} static struct dmirror_chunk *dmirror_page_to_chunk(struct page *page){ return container_of(page_pgmap(page), struct dmirror_chunk, pagemap);} static struct dmirror_device *dmirror_page_to_device(struct page *page) { return dmirror_page_to_chunk(page)->mdevice;} static int dmirror_do_fault(struct dmirror *dmirror, struct hmm_range *range){ unsigned long *pfns = range->hmm_pfns; unsigned long pfn; for (pfn = (range->start >> PAGE_SHIFT); pfn < (range->end >> PAGE_SHIFT); pfn++, pfns++) { struct page *page; void *entry; /* * Since we asked for hmm_range_fault() to populate pages, * it shouldn't return an error entry on success. */ WARN_ON(*pfns & HMM_PFN_ERROR); WARN_ON(!(*pfns & HMM_PFN_VALID)); page = hmm_pfn_to_page(*pfns); WARN_ON(!page); entry = page; if (*pfns & HMM_PFN_WRITE) entry = xa_tag_pointer(entry, DPT_XA_TAG_WRITE); else if (WARN_ON(range->default_flags & HMM_PFN_WRITE)) return -EFAULT; entry = xa_store(&dmirror->pt, pfn, entry, GFP_ATOMIC); if (xa_is_err(entry)) return xa_err(entry); } return 0;} static void dmirror_do_update(struct dmirror *dmirror, unsigned long start, unsigned long end){ unsigned long pfn; void *entry; /* * The XArray doesn't hold references to pages since it relies on * the mmu notifier to clear page pointers when they become stale. * Therefore, it is OK to just clear the entry. */ xa_for_each_range(&dmirror->pt, pfn, entry, start >> PAGE_SHIFT, end >> PAGE_SHIFT) xa_erase(&dmirror->pt, pfn);} static bool dmirror_interval_invalidate(struct mmu_interval_notifier *mni, const struct mmu_notifier_range *range, unsigned long cur_seq){ struct dmirror *dmirror = container_of(mni, struct dmirror, notifier); /* * Ignore invalidation callbacks for device private pages since * the invalidation is handled as part of the migration process. */ if (range->event == MMU_NOTIFY_MIGRATE && range->owner == dmirror->mdevice) return true; if (mmu_notifier_range_blockable(range)) mutex_lock(&dmirror->mutex); else if (!mutex_trylock(&dmirror->mutex)) return false; mmu_interval_set_seq(mni, cur_seq); dmirror_do_update(dmirror, range->start, range->end); mutex_unlock(&dmirror->mutex); return true;} static const struct mmu_interval_notifier_ops dmirror_min_ops = { .invalidate = dmirror_interval_invalidate,}; static int dmirror_range_fault(struct dmirror *dmirror, struct hmm_range *range){ struct mm_struct *mm = dmirror->notifier.mm; unsigned long timeout = jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); int ret; while (true) { if (time_after(jiffies, timeout)) { ret = -EBUSY; goto out; } range->notifier_seq = mmu_interval_read_begin(range->notifier); mmap_read_lock(mm); ret = hmm_range_fault(range); mmap_read_unlock(mm); if (ret) { if (ret == -EBUSY) continue; goto out; } mutex_lock(&dmirror->mutex); if (mmu_interval_read_retry(range->notifier, range->notifier_seq)) { mutex_unlock(&dmirror->mutex); continue; } break; } ret = dmirror_do_fault(dmirror, range); mutex_unlock(&dmirror->mutex);out: return ret;} static int dmirror_fault(struct dmirror *dmirror, unsigned long start, unsigned long end, bool write){ struct mm_struct *mm = dmirror->notifier.mm; unsigned long addr; unsigned long pfns[32]; struct hmm_range range = { .notifier = &dmirror->notifier, .hmm_pfns = pfns, .pfn_flags_mask = 0, .default_flags = HMM_PFN_REQ_FAULT | (write ? HMM_PFN_REQ_WRITE : 0), .dev_private_owner = dmirror->mdevice, }; int ret = 0; /* Since the mm is for the mirrored process, get a reference first. */ if (!mmget_not_zero(mm)) return 0; for (addr = start; addr < end; addr = range.end) { range.start = addr; range.end = min(addr + (ARRAY_SIZE(pfns) << PAGE_SHIFT), end); ret = dmirror_range_fault(dmirror, &range); if (ret) break; } mmput(mm); return ret;} static int dmirror_do_read(struct dmirror *dmirror, unsigned long start, unsigned long end, struct dmirror_bounce *bounce){ unsigned long pfn; void *ptr; ptr = bounce->ptr + ((start - bounce->addr) & PAGE_MASK); for (pfn = start >> PAGE_SHIFT; pfn < (end >> PAGE_SHIFT); pfn++) { void *entry; struct page *page; entry = xa_load(&dmirror->pt, pfn); page = xa_untag_pointer(entry); if (!page) return -ENOENT; memcpy_from_page(ptr, page, 0, PAGE_SIZE); ptr += PAGE_SIZE; bounce->cpages++; } return 0;} static int dmirror_read(struct dmirror *dmirror, struct hmm_dmirror_cmd *cmd){ struct dmirror_bounce bounce; unsigned long start, end; unsigned long size = cmd->npages << PAGE_SHIFT; int ret; start = cmd->addr; end = start + size; if (end < start) return -EINVAL; ret = dmirror_bounce_init(&bounce, start, size); if (ret) return ret; while (1) { mutex_lock(&dmirror->mutex); ret = dmirror_do_read(dmirror, start, end, &bounce); mutex_unlock(&dmirror->mutex); if (ret != -ENOENT) break; start = cmd->addr + (bounce.cpages << PAGE_SHIFT); ret = dmirror_fault(dmirror, start, end, false); if (ret) break; cmd->faults++; } if (ret == 0) { if (copy_to_user(u64_to_user_ptr(cmd->ptr), bounce.ptr, bounce.size)) ret = -EFAULT; } cmd->cpages = bounce.cpages; dmirror_bounce_fini(&bounce); return ret;} static int dmirror_do_write(struct dmirror *dmirror, unsigned long start, unsigned long end, struct dmirror_bounce *bounce){ unsigned long pfn; void *ptr; ptr = bounce->ptr + ((start - bounce->addr) & PAGE_MASK); for (pfn = start >> PAGE_SHIFT; pfn < (end >> PAGE_SHIFT); pfn++) { void *entry;Lines 1–500 of 1873. Load more
// SPDX-License-Identifier: GPL-2.0 /* * This is a module to test the HMM (Heterogeneous Memory Management) * mirror and zone device private memory migration APIs of the kernel. * Userspace programs can register with the driver to mirror their own address * space and can use the device to read/write any valid virtual address. */ #include <linux/init.h> #include <linux/fs.h> #include <linux/mm.h> #include <linux/module.h> #include <linux/kernel.h> #include <linux/cdev.h> #include <linux/device.h> #include <linux/memremap.h> #include <linux/mutex.h> #include <linux/rwsem.h> #include <linux/sched.h> #include <linux/slab.h> #include <linux/highmem.h> #include <linux/delay.h> #include <linux/pagemap.h> #include <linux/hmm.h> #include <linux/vmalloc.h> #include <linux/swap.h> #include <linux/swapops.h> #include <linux/sched/mm.h> #include <linux/platform_device.h> #include <linux/rmap.h> #include <linux/mmu_notifier.h> #include <linux/migrate.h> #include "test_hmm_uapi.h" #define DMIRROR_NDEVICES 4 #define DMIRROR_RANGE_FAULT_TIMEOUT 1000 #define DEVMEM_CHUNK_SIZE (256 * 1024 * 1024U) #define DEVMEM_CHUNKS_RESERVE 16 /* * For device_private pages, dpage is just a dummy struct page * representing a piece of device memory. dmirror_devmem_alloc_page * allocates a real system memory page as backing storage to fake a * real device. zone_device_data points to that backing page. But * for device_coherent memory, the struct page represents real * physical CPU-accessible memory that we can use directly. */ #define BACKING_PAGE(page) (is_device_private_page((page)) ? \ (page)->zone_device_data : (page)) static unsigned long spm_addr_dev0; module_param(spm_addr_dev0, long, 0644); MODULE_PARM_DESC(spm_addr_dev0, "Specify start address for SPM (special purpose memory) used for device 0. By setting this Coherent device type will be used. Make sure spm_addr_dev1 is set too. Minimum SPM size should be DEVMEM_CHUNK_SIZE."); static unsigned long spm_addr_dev1; module_param(spm_addr_dev1, long, 0644); MODULE_PARM_DESC(spm_addr_dev1, "Specify start address for SPM (special purpose memory) used for device 1. By setting this Coherent device type will be used. Make sure spm_addr_dev0 is set too. Minimum SPM size should be DEVMEM_CHUNK_SIZE."); static const struct dev_pagemap_ops dmirror_devmem_ops; static const struct mmu_interval_notifier_ops dmirror_min_ops; static dev_t dmirror_dev; struct dmirror_device; struct dmirror_bounce { void *ptr; unsigned long size; unsigned long addr; unsigned long cpages; }; #define DPT_XA_TAG_ATOMIC 1UL #define DPT_XA_TAG_WRITE 3UL /* * Data structure to track address ranges and register for mmu interval * notifier updates. */ struct dmirror_interval { struct mmu_interval_notifier notifier; struct dmirror *dmirror; }; /* * Data attached to the open device file. * Note that it might be shared after a fork(). */ struct dmirror { struct dmirror_device *mdevice; struct xarray pt; struct mmu_interval_notifier notifier; struct mutex mutex; __u64 flags; }; /* * ZONE_DEVICE pages for migration and simulating device memory. */ struct dmirror_chunk { struct dev_pagemap pagemap; struct dmirror_device *mdevice; bool remove; }; /* * Per device data. */ struct dmirror_device { struct cdev cdevice; unsigned int zone_device_type; struct device device; unsigned int devmem_capacity; unsigned int devmem_count; struct dmirror_chunk **devmem_chunks; struct mutex devmem_lock; /* protects the above */ unsigned long calloc; unsigned long cfree; struct page *free_pages; struct folio *free_folios; spinlock_t lock; /* protects the above */ }; static struct dmirror_device dmirror_devices[DMIRROR_NDEVICES]; static int dmirror_bounce_init(struct dmirror_bounce *bounce, unsigned long addr, unsigned long size) { bounce->addr = addr; bounce->size = size; bounce->cpages = 0; bounce->ptr = vmalloc(size); if (!bounce->ptr) return -ENOMEM; return 0; } static bool dmirror_is_private_zone(struct dmirror_device *mdevice) { return (mdevice->zone_device_type == HMM_DMIRROR_MEMORY_DEVICE_PRIVATE); } static enum migrate_vma_direction dmirror_select_device(struct dmirror *dmirror) { return (dmirror->mdevice->zone_device_type == HMM_DMIRROR_MEMORY_DEVICE_PRIVATE) ? MIGRATE_VMA_SELECT_DEVICE_PRIVATE : MIGRATE_VMA_SELECT_DEVICE_COHERENT; } static void dmirror_bounce_fini(struct dmirror_bounce *bounce) { vfree(bounce->ptr); } static int dmirror_fops_open(struct inode *inode, struct file *filp) { struct cdev *cdev = inode->i_cdev; struct dmirror *dmirror; int ret; /* Mirror this process address space */ dmirror = kzalloc_obj(*dmirror); if (dmirror == NULL) return -ENOMEM; dmirror->mdevice = container_of(cdev, struct dmirror_device, cdevice); mutex_init(&dmirror->mutex); xa_init(&dmirror->pt); ret = mmu_interval_notifier_insert(&dmirror->notifier, current->mm, 0, ULONG_MAX & PAGE_MASK, &dmirror_min_ops); if (ret) { kfree(dmirror); return ret; } filp->private_data = dmirror; return 0; } static void dmirror_device_evict_chunk(struct dmirror_chunk *chunk) { unsigned long start_pfn = chunk->pagemap.range.start >> PAGE_SHIFT; unsigned long end_pfn = chunk->pagemap.range.end >> PAGE_SHIFT; unsigned long npages = end_pfn - start_pfn + 1; unsigned long i; unsigned long *src_pfns; unsigned long *dst_pfns; unsigned int order = 0; src_pfns = kvcalloc(npages, sizeof(*src_pfns), GFP_KERNEL | __GFP_NOFAIL); dst_pfns = kvcalloc(npages, sizeof(*dst_pfns), GFP_KERNEL | __GFP_NOFAIL); migrate_device_range(src_pfns, start_pfn, npages); for (i = 0; i < npages; i++) { struct page *dpage, *spage; spage = migrate_pfn_to_page(src_pfns[i]); if (!spage || !(src_pfns[i] & MIGRATE_PFN_MIGRATE)) continue; if (WARN_ON(!is_device_private_page(spage) && !is_device_coherent_page(spage))) continue; order = folio_order(page_folio(spage)); spage = BACKING_PAGE(spage); if (src_pfns[i] & MIGRATE_PFN_COMPOUND) { dpage = folio_page(folio_alloc(GFP_HIGHUSER_MOVABLE, order), 0); } else { dpage = alloc_page(GFP_HIGHUSER_MOVABLE | __GFP_NOFAIL); order = 0; } /* TODO Support splitting here */ lock_page(dpage); dst_pfns[i] = migrate_pfn(page_to_pfn(dpage)); if (src_pfns[i] & MIGRATE_PFN_WRITE) dst_pfns[i] |= MIGRATE_PFN_WRITE; if (order) dst_pfns[i] |= MIGRATE_PFN_COMPOUND; folio_copy(page_folio(dpage), page_folio(spage)); } migrate_device_pages(src_pfns, dst_pfns, npages); migrate_device_finalize(src_pfns, dst_pfns, npages); kvfree(src_pfns); kvfree(dst_pfns); } static int dmirror_fops_release(struct inode *inode, struct file *filp) { struct dmirror *dmirror = filp->private_data; struct dmirror_device *mdevice = dmirror->mdevice; int i; mmu_interval_notifier_remove(&dmirror->notifier); if (mdevice->devmem_chunks) { for (i = 0; i < mdevice->devmem_count; i++) { struct dmirror_chunk *devmem = mdevice->devmem_chunks[i]; dmirror_device_evict_chunk(devmem); } } xa_destroy(&dmirror->pt); kfree(dmirror); return 0; } static struct dmirror_chunk *dmirror_page_to_chunk(struct page *page) { return container_of(page_pgmap(page), struct dmirror_chunk, pagemap); } static struct dmirror_device *dmirror_page_to_device(struct page *page) { return dmirror_page_to_chunk(page)->mdevice; } static int dmirror_do_fault(struct dmirror *dmirror, struct hmm_range *range) { unsigned long *pfns = range->hmm_pfns; unsigned long pfn; for (pfn = (range->start >> PAGE_SHIFT); pfn < (range->end >> PAGE_SHIFT); pfn++, pfns++) { struct page *page; void *entry; /* * Since we asked for hmm_range_fault() to populate pages, * it shouldn't return an error entry on success. */ WARN_ON(*pfns & HMM_PFN_ERROR); WARN_ON(!(*pfns & HMM_PFN_VALID)); page = hmm_pfn_to_page(*pfns); WARN_ON(!page); entry = page; if (*pfns & HMM_PFN_WRITE) entry = xa_tag_pointer(entry, DPT_XA_TAG_WRITE); else if (WARN_ON(range->default_flags & HMM_PFN_WRITE)) return -EFAULT; entry = xa_store(&dmirror->pt, pfn, entry, GFP_ATOMIC); if (xa_is_err(entry)) return xa_err(entry); } return 0; } static void dmirror_do_update(struct dmirror *dmirror, unsigned long start, unsigned long end) { unsigned long pfn; void *entry; /* * The XArray doesn't hold references to pages since it relies on * the mmu notifier to clear page pointers when they become stale. * Therefore, it is OK to just clear the entry. */ xa_for_each_range(&dmirror->pt, pfn, entry, start >> PAGE_SHIFT, end >> PAGE_SHIFT) xa_erase(&dmirror->pt, pfn); } static bool dmirror_interval_invalidate(struct mmu_interval_notifier *mni, const struct mmu_notifier_range *range, unsigned long cur_seq) { struct dmirror *dmirror = container_of(mni, struct dmirror, notifier); /* * Ignore invalidation callbacks for device private pages since * the invalidation is handled as part of the migration process. */ if (range->event == MMU_NOTIFY_MIGRATE && range->owner == dmirror->mdevice) return true; if (mmu_notifier_range_blockable(range)) mutex_lock(&dmirror->mutex); else if (!mutex_trylock(&dmirror->mutex)) return false; mmu_interval_set_seq(mni, cur_seq); dmirror_do_update(dmirror, range->start, range->end); mutex_unlock(&dmirror->mutex); return true; } static const struct mmu_interval_notifier_ops dmirror_min_ops = { .invalidate = dmirror_interval_invalidate, }; static int dmirror_range_fault(struct dmirror *dmirror, struct hmm_range *range) { struct mm_struct *mm = dmirror->notifier.mm; unsigned long timeout = jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); int ret; while (true) { if (time_after(jiffies, timeout)) { ret = -EBUSY; goto out; } range->notifier_seq = mmu_interval_read_begin(range->notifier); mmap_read_lock(mm); ret = hmm_range_fault(range); mmap_read_unlock(mm); if (ret) { if (ret == -EBUSY) continue; goto out; } mutex_lock(&dmirror->mutex); if (mmu_interval_read_retry(range->notifier, range->notifier_seq)) { mutex_unlock(&dmirror->mutex); continue; } break; } ret = dmirror_do_fault(dmirror, range); mutex_unlock(&dmirror->mutex); out: return ret; } static int dmirror_fault(struct dmirror *dmirror, unsigned long start, unsigned long end, bool write) { struct mm_struct *mm = dmirror->notifier.mm; unsigned long addr; unsigned long pfns[32]; struct hmm_range range = { .notifier = &dmirror->notifier, .hmm_pfns = pfns, .pfn_flags_mask = 0, .default_flags = HMM_PFN_REQ_FAULT | (write ? HMM_PFN_REQ_WRITE : 0), .dev_private_owner = dmirror->mdevice, }; int ret = 0; /* Since the mm is for the mirrored process, get a reference first. */ if (!mmget_not_zero(mm)) return 0; for (addr = start; addr < end; addr = range.end) { range.start = addr; range.end = min(addr + (ARRAY_SIZE(pfns) << PAGE_SHIFT), end); ret = dmirror_range_fault(dmirror, &range); if (ret) break; } mmput(mm); return ret; } static int dmirror_do_read(struct dmirror *dmirror, unsigned long start, unsigned long end, struct dmirror_bounce *bounce) { unsigned long pfn; void *ptr; ptr = bounce->ptr + ((start - bounce->addr) & PAGE_MASK); for (pfn = start >> PAGE_SHIFT; pfn < (end >> PAGE_SHIFT); pfn++) { void *entry; struct page *page; entry = xa_load(&dmirror->pt, pfn); page = xa_untag_pointer(entry); if (!page) return -ENOENT; memcpy_from_page(ptr, page, 0, PAGE_SIZE); ptr += PAGE_SIZE; bounce->cpages++; } return 0; } static int dmirror_read(struct dmirror *dmirror, struct hmm_dmirror_cmd *cmd) { struct dmirror_bounce bounce; unsigned long start, end; unsigned long size = cmd->npages << PAGE_SHIFT; int ret; start = cmd->addr; end = start + size; if (end < start) return -EINVAL; ret = dmirror_bounce_init(&bounce, start, size); if (ret) return ret; while (1) { mutex_lock(&dmirror->mutex); ret = dmirror_do_read(dmirror, start, end, &bounce); mutex_unlock(&dmirror->mutex); if (ret != -ENOENT) break; start = cmd->addr + (bounce.cpages << PAGE_SHIFT); ret = dmirror_fault(dmirror, start, end, false); if (ret) break; cmd->faults++; } if (ret == 0) { if (copy_to_user(u64_to_user_ptr(cmd->ptr), bounce.ptr, bounce.size)) ret = -EFAULT; } cmd->cpages = bounce.cpages; dmirror_bounce_fini(&bounce); return ret; } static int dmirror_do_write(struct dmirror *dmirror, unsigned long start, unsigned long end, struct dmirror_bounce *bounce) { unsigned long pfn; void *ptr; ptr = bounce->ptr + ((start - bounce->addr) & PAGE_MASK); for (pfn = start >> PAGE_SHIFT; pfn < (end >> PAGE_SHIFT); pfn++) { void *entry;