Device drivers can use the migrate_vma family of functions to migrate existing private anonymous mappings to device private pages. These pages are backed by memory on the device with drivers being responsible for copying data to and from device memory. Device private pages are freed via the pgmap->page_free() callback when they are unmapped and their refcount drops to zero. Alternatively they may be freed indirectly via migration back to CPU memory in response to a pgmap->migrate_to_ram() callback called whenever the CPU accesses an address mapped to a device private page. In other words drivers cannot control the lifetime of data allocated on the devices and must wait until these pages are freed from userspace. This causes issues when memory needs to reclaimed on the device, either because the device is going away due to a ->release() callback or because another user needs to use the memory. Drivers could use the existing migrate_vma functions to migrate data off the device. However this would require them to track the mappings of each page which is both complicated and not always possible. Instead drivers need to be able to migrate device pages directly so they can free up device memory. To allow that this patch introduces the migrate_device family of functions which are functionally similar to migrate_vma but which skips the initial lookup based on mapping.
Unified · Split
| @@ -210,6 +210,13 @@ struct migrate_vma { · 209 unmodified lines | |||
| 210 | int migrate_vma_setup(struct migrate_vma *args); | 210 | int migrate_vma_setup(struct migrate_vma *args); |
| 211 | void migrate_vma_pages(struct migrate_vma *migrate); | 211 | void migrate_vma_pages(struct migrate_vma *migrate); |
| 212 | void migrate_vma_finalize(struct migrate_vma *migrate); | 212 | void migrate_vma_finalize(struct migrate_vma *migrate); |
| 213 | int migrate_device_range(unsigned long *src_pfns, unsigned long start, | ||
| 214 | unsigned long npages); | ||
| 215 | void migrate_device_pages(unsigned long *src_pfns, unsigned long *dst_pfns, | ||
| 216 | unsigned long npages); | ||
| 217 | void migrate_device_finalize(unsigned long *src_pfns, | ||
| 218 | unsigned long *dst_pfns, unsigned long npages); | ||
| 219 | |||
| 213 | #endif /* CONFIG_MIGRATION */ | 220 | #endif /* CONFIG_MIGRATION */ |
| 214 | 221 | ||
| 215 | #endif /* _LINUX_MIGRATE_H */ | 222 | #endif /* _LINUX_MIGRATE_H */ |
| @@ -693,7 +693,7 @@ abort: · 692 unmodified lines | |||
| 693 | *src &= ~MIGRATE_PFN_MIGRATE; | 693 | *src &= ~MIGRATE_PFN_MIGRATE; |
| 694 | } | 694 | } |
| 695 | 695 | ||
| 696 | static void migrate_device_pages(unsigned long *src_pfns, | 696 | static void __migrate_device_pages(unsigned long *src_pfns, |
| 697 | unsigned long *dst_pfns, unsigned long npages, | 697 | unsigned long *dst_pfns, unsigned long npages, |
| 698 | struct migrate_vma *migrate) | 698 | struct migrate_vma *migrate) |
| 699 | { | 699 | { |
| @@ -715,6 +715,9 @@ static void migrate_device_pages(unsigned long *src_pfns, · 15 unmodified lines | |||
| 715 | if (!page) { | 715 | if (!page) { |
| 716 | unsigned long addr; | 716 | unsigned long addr; |
| 717 | 717 | ||
| 718 | if (!(src_pfns[i] & MIGRATE_PFN_MIGRATE)) | ||
| 719 | continue; | ||
| 720 | |||
| 718 | /* | 721 | /* |
| 719 | * The only time there is no vma is when called from | 722 | * The only time there is no vma is when called from |
| 720 | * migrate_device_coherent_page(). However this isn't | 723 | * migrate_device_coherent_page(). However this isn't |
| @@ -722,8 +725,6 @@ static void migrate_device_pages(unsigned long *src_pfns, · 1 unmodified line | |||
| 722 | */ | 725 | */ |
| 723 | VM_BUG_ON(!migrate); | 726 | VM_BUG_ON(!migrate); |
| 724 | addr = migrate->start + i*PAGE_SIZE; | 727 | addr = migrate->start + i*PAGE_SIZE; |
| 725 | if (!(src_pfns[i] & MIGRATE_PFN_MIGRATE)) | ||
| 726 | continue; | ||
| 727 | if (!notified) { | 728 | if (!notified) { |
| 728 | notified = true; | 729 | notified = true; |
| 729 | 730 | ||
| @@ -778,6 +779,22 @@ static void migrate_device_pages(unsigned long *src_pfns, · 48 unmodified lines | |||
| 778 | mmu_notifier_invalidate_range_only_end(&range); | 779 | mmu_notifier_invalidate_range_only_end(&range); |
| 779 | } | 780 | } |
| 780 | 781 | ||
| 782 | /** | ||
| 783 | * migrate_device_pages() - migrate meta-data from src page to dst page | ||
| 784 | * @src_pfns: src_pfns returned from migrate_device_range() | ||
| 785 | * @dst_pfns: array of pfns allocated by the driver to migrate memory to | ||
| 786 | * @npages: number of pages in the range | ||
| 787 | * | ||
| 788 | * Equivalent to migrate_vma_pages(). This is called to migrate struct page | ||
| 789 | * meta-data from source struct page to destination. | ||
| 790 | */ | ||
| 791 | void migrate_device_pages(unsigned long *src_pfns, unsigned long *dst_pfns, | ||
| 792 | unsigned long npages) | ||
| 793 | { | ||
| 794 | __migrate_device_pages(src_pfns, dst_pfns, npages, NULL); | ||
| 795 | } | ||
| 796 | EXPORT_SYMBOL(migrate_device_pages); | ||
| 797 | |||
| 781 | /** | 798 | /** |
| 782 | * migrate_vma_pages() - migrate meta-data from src page to dst page | 799 | * migrate_vma_pages() - migrate meta-data from src page to dst page |
| 783 | * @migrate: migrate struct containing all migration information | 800 | * @migrate: migrate struct containing all migration information |
| @@ -788,12 +805,22 @@ static void migrate_device_pages(unsigned long *src_pfns, · 4 unmodified lines | |||
| 788 | */ | 805 | */ |
| 789 | void migrate_vma_pages(struct migrate_vma *migrate) | 806 | void migrate_vma_pages(struct migrate_vma *migrate) |
| 790 | { | 807 | { |
| 791 | migrate_device_pages(migrate->src, migrate->dst, migrate->npages, migrate); | 808 | __migrate_device_pages(migrate->src, migrate->dst, migrate->npages, migrate); |
| 792 | } | 809 | } |
| 793 | EXPORT_SYMBOL(migrate_vma_pages); | 810 | EXPORT_SYMBOL(migrate_vma_pages); |
| 794 | 811 | ||
| 795 | static void migrate_device_finalize(unsigned long *src_pfns, | 812 | /* |
| 796 | unsigned long *dst_pfns, unsigned long npages) | 813 | * migrate_device_finalize() - complete page migration |
| 814 | * @src_pfns: src_pfns returned from migrate_device_range() | ||
| 815 | * @dst_pfns: array of pfns allocated by the driver to migrate memory to | ||
| 816 | * @npages: number of pages in the range | ||
| 817 | * | ||
| 818 | * Completes migration of the page by removing special migration entries. | ||
| 819 | * Drivers must ensure copying of page data is complete and visible to the CPU | ||
| 820 | * before calling this. | ||
| 821 | */ | ||
| 822 | void migrate_device_finalize(unsigned long *src_pfns, | ||
| 823 | unsigned long *dst_pfns, unsigned long npages) | ||
| 797 | { | 824 | { |
| 798 | unsigned long i; | 825 | unsigned long i; |
| 799 | 826 | ||
| @@ -837,6 +864,7 @@ static void migrate_device_finalize(unsigned long *src_pfns, · 37 unmodified lines | |||
| 837 | } | 864 | } |
| 838 | } | 865 | } |
| 839 | } | 866 | } |
| 867 | EXPORT_SYMBOL(migrate_device_finalize); | ||
| 840 | 868 | ||
| 841 | /** | 869 | /** |
| 842 | * migrate_vma_finalize() - restore CPU page table entry | 870 | * migrate_vma_finalize() - restore CPU page table entry |
| @@ -855,6 +883,53 @@ void migrate_vma_finalize(struct migrate_vma *migrate) · 12 unmodified lines | |||
| 855 | } | 883 | } |
| 856 | EXPORT_SYMBOL(migrate_vma_finalize); | 884 | EXPORT_SYMBOL(migrate_vma_finalize); |
| 857 | 885 | ||
| 886 | /** | ||
| 887 | * migrate_device_range() - migrate device private pfns to normal memory. | ||
| 888 | * @src_pfns: array large enough to hold migrating source device private pfns. | ||
| 889 | * @start: starting pfn in the range to migrate. | ||
| 890 | * @npages: number of pages to migrate. | ||
| 891 | * | ||
| 892 | * migrate_vma_setup() is similar in concept to migrate_vma_setup() except that | ||
| 893 | * instead of looking up pages based on virtual address mappings a range of | ||
| 894 | * device pfns that should be migrated to system memory is used instead. | ||
| 895 | * | ||
| 896 | * This is useful when a driver needs to free device memory but doesn't know the | ||
| 897 | * virtual mappings of every page that may be in device memory. For example this | ||
| 898 | * is often the case when a driver is being unloaded or unbound from a device. | ||
| 899 | * | ||
| 900 | * Like migrate_vma_setup() this function will take a reference and lock any | ||
| 901 | * migrating pages that aren't free before unmapping them. Drivers may then | ||
| 902 | * allocate destination pages and start copying data from the device to CPU | ||
| 903 | * memory before calling migrate_device_pages(). | ||
| 904 | */ | ||
| 905 | int migrate_device_range(unsigned long *src_pfns, unsigned long start, | ||
| 906 | unsigned long npages) | ||
| 907 | { | ||
| 908 | unsigned long i, pfn; | ||
| 909 | |||
| 910 | for (pfn = start, i = 0; i < npages; pfn++, i++) { | ||
| 911 | struct page *page = pfn_to_page(pfn); | ||
| 912 | |||
| 913 | if (!get_page_unless_zero(page)) { | ||
| 914 | src_pfns[i] = 0; | ||
| 915 | continue; | ||
| 916 | } | ||
| 917 | |||
| 918 | if (!trylock_page(page)) { | ||
| 919 | src_pfns[i] = 0; | ||
| 920 | put_page(page); | ||
| 921 | continue; | ||
| 922 | } | ||
| 923 | |||
| 924 | src_pfns[i] = migrate_pfn(pfn) | MIGRATE_PFN_MIGRATE; | ||
| 925 | } | ||
| 926 | |||
| 927 | migrate_device_unmap(src_pfns, npages, NULL); | ||
| 928 | |||
| 929 | return 0; | ||
| 930 | } | ||
| 931 | EXPORT_SYMBOL(migrate_device_range); | ||
| 932 | |||
| 858 | /* | 933 | /* |
| 859 | * Migrate a device coherent page back to normal memory. The caller should have | 934 | * Migrate a device coherent page back to normal memory. The caller should have |
| 860 | * a reference on page which will be copied to the new page if migration is | 935 | * a reference on page which will be copied to the new page if migration is |
| @@ -885,7 +960,7 @@ int migrate_device_coherent_page(struct page *page) · 24 unmodified lines | |||
| 885 | dst_pfn = migrate_pfn(page_to_pfn(dpage)); | 960 | dst_pfn = migrate_pfn(page_to_pfn(dpage)); |
| 886 | } | 961 | } |
| 887 | 962 | ||
| 888 | migrate_device_pages(&src_pfn, &dst_pfn, 1, NULL); | 963 | migrate_device_pages(&src_pfn, &dst_pfn, 1); |
| 889 | if (src_pfn & MIGRATE_PFN_MIGRATE) | 964 | if (src_pfn & MIGRATE_PFN_MIGRATE) |
| 890 | copy_highpage(dpage, page); | 965 | copy_highpage(dpage, page); |
| 891 | migrate_device_finalize(&src_pfn, &dst_pfn, 1); | 966 | migrate_device_finalize(&src_pfn, &dst_pfn, 1); |