An upcoming patch changes and complicates the refcounting and especially the "put page" aspects of it. In order to keep everything clean, refactor the devmap page release routines: * Rename put_devmap_managed_page() to page_is_devmap_managed(), and limit the functionality to "read only": return a bool, with no side effects. * Add a new routine, put_devmap_managed_page(), to handle decrementing the refcount for ZONE_DEVICE pages. * Change callers (just release_pages() and put_page()) to check page_is_devmap_managed() before calling the new put_devmap_managed_page() routine. This is a performance point: put_page() is a hot path, so we need to avoid non- inline function calls where possible. * Rename __put_devmap_managed_page() to free_devmap_managed_page(), and limit the functionality to unconditionally freeing a devmap page. This is originally based on a separate patch by Ira Weiny, which applied to an early version of the put_user_page() experiments. Since then, Jérôme Glisse suggested the refactoring described above.
Unified · Split
| @@ -947,9 +947,10 @@ static inline bool is_zone_device_page(const struct page *page) · 946 unmodified lines | |||
| 947 | #endif | 947 | #endif |
| 948 | 948 | ||
| 949 | #ifdef CONFIG_DEV_PAGEMAP_OPS | 949 | #ifdef CONFIG_DEV_PAGEMAP_OPS |
| 950 | void __put_devmap_managed_page(struct page *page); | 950 | void free_devmap_managed_page(struct page *page); |
| 951 | DECLARE_STATIC_KEY_FALSE(devmap_managed_key); | 951 | DECLARE_STATIC_KEY_FALSE(devmap_managed_key); |
| 952 | static inline bool put_devmap_managed_page(struct page *page) | 952 | |
| 953 | static inline bool page_is_devmap_managed(struct page *page) | ||
| 953 | { | 954 | { |
| 954 | if (!static_branch_unlikely(&devmap_managed_key)) | 955 | if (!static_branch_unlikely(&devmap_managed_key)) |
| 955 | return false; | 956 | return false; |
| @@ -958,7 +959,6 @@ static inline bool put_devmap_managed_page(struct page *page) · 2 unmodified lines | |||
| 958 | switch (page->pgmap->type) { | 959 | switch (page->pgmap->type) { |
| 959 | case MEMORY_DEVICE_PRIVATE: | 960 | case MEMORY_DEVICE_PRIVATE: |
| 960 | case MEMORY_DEVICE_FS_DAX: | 961 | case MEMORY_DEVICE_FS_DAX: |
| 961 | __put_devmap_managed_page(page); | ||
| 962 | return true; | 962 | return true; |
| 963 | default: | 963 | default: |
| 964 | break; | 964 | break; |
| @@ -966,11 +966,17 @@ static inline bool put_devmap_managed_page(struct page *page) · 1 unmodified line | |||
| 966 | return false; | 966 | return false; |
| 967 | } | 967 | } |
| 968 | 968 | ||
| 969 | void put_devmap_managed_page(struct page *page); | ||
| 970 | |||
| 969 | #else /* CONFIG_DEV_PAGEMAP_OPS */ | 971 | #else /* CONFIG_DEV_PAGEMAP_OPS */ |
| 970 | static inline bool put_devmap_managed_page(struct page *page) | 972 | static inline bool page_is_devmap_managed(struct page *page) |
| 971 | { | 973 | { |
| 972 | return false; | 974 | return false; |
| 973 | } | 975 | } |
| 976 | |||
| 977 | static inline void put_devmap_managed_page(struct page *page) | ||
| 978 | { | ||
| 979 | } | ||
| 974 | #endif /* CONFIG_DEV_PAGEMAP_OPS */ | 980 | #endif /* CONFIG_DEV_PAGEMAP_OPS */ |
| 975 | 981 | ||
| 976 | static inline bool is_device_private_page(const struct page *page) | 982 | static inline bool is_device_private_page(const struct page *page) |
| @@ -1023,8 +1029,10 @@ static inline void put_page(struct page *page) · 46 unmodified lines | |||
| 1023 | * need to inform the device driver through callback. See | 1029 | * need to inform the device driver through callback. See |
| 1024 | * include/linux/memremap.h and HMM for details. | 1030 | * include/linux/memremap.h and HMM for details. |
| 1025 | */ | 1031 | */ |
| 1026 | if (put_devmap_managed_page(page)) | 1032 | if (page_is_devmap_managed(page)) { |
| 1033 | put_devmap_managed_page(page); | ||
| 1027 | return; | 1034 | return; |
| 1035 | } | ||
| 1028 | 1036 | ||
| 1029 | if (put_page_testzero(page)) | 1037 | if (put_page_testzero(page)) |
| 1030 | __put_page(page); | 1038 | __put_page(page); |
| @@ -411,20 +411,8 @@ struct dev_pagemap *get_dev_pagemap(unsigned long pfn, · 410 unmodified lines | |||
| 411 | EXPORT_SYMBOL_GPL(get_dev_pagemap); | 411 | EXPORT_SYMBOL_GPL(get_dev_pagemap); |
| 412 | 412 | ||
| 413 | #ifdef CONFIG_DEV_PAGEMAP_OPS | 413 | #ifdef CONFIG_DEV_PAGEMAP_OPS |
| 414 | void __put_devmap_managed_page(struct page *page) | 414 | void free_devmap_managed_page(struct page *page) |
| 415 | { | 415 | { |
| 416 | int count = page_ref_dec_return(page); | ||
| 417 | |||
| 418 | /* still busy */ | ||
| 419 | if (count > 1) | ||
| 420 | return; | ||
| 421 | |||
| 422 | /* only triggered by the dev_pagemap shutdown path */ | ||
| 423 | if (count == 0) { | ||
| 424 | __put_page(page); | ||
| 425 | return; | ||
| 426 | } | ||
| 427 | |||
| 428 | /* notify page idle for dax */ | 416 | /* notify page idle for dax */ |
| 429 | if (!is_device_private_page(page)) { | 417 | if (!is_device_private_page(page)) { |
| 430 | wake_up_var(&page->_refcount); | 418 | wake_up_var(&page->_refcount); |
| @@ -461,5 +449,4 @@ void __put_devmap_managed_page(struct page *page) · 30 unmodified lines | |||
| 461 | page->mapping = NULL; | 449 | page->mapping = NULL; |
| 462 | page->pgmap->ops->page_free(page); | 450 | page->pgmap->ops->page_free(page); |
| 463 | } | 451 | } |
| 464 | EXPORT_SYMBOL(__put_devmap_managed_page); | ||
| 465 | #endif /* CONFIG_DEV_PAGEMAP_OPS */ | 452 | #endif /* CONFIG_DEV_PAGEMAP_OPS */ |
| @@ -813,8 +813,10 @@ void release_pages(struct page **pages, int nr) · 812 unmodified lines | |||
| 813 | * processing, and instead, expect a call to | 813 | * processing, and instead, expect a call to |
| 814 | * put_page_testzero(). | 814 | * put_page_testzero(). |
| 815 | */ | 815 | */ |
| 816 | if (put_devmap_managed_page(page)) | 816 | if (page_is_devmap_managed(page)) { |
| 817 | put_devmap_managed_page(page); | ||
| 817 | continue; | 818 | continue; |
| 819 | } | ||
| 818 | } | 820 | } |
| 819 | 821 | ||
| 820 | page = compound_head(page); | 822 | page = compound_head(page); |
| @@ -1102,3 +1104,26 @@ void __init swap_setup(void) · 281 unmodified lines | |||
| 1102 | * _really_ don't want to cluster much more | 1104 | * _really_ don't want to cluster much more |
| 1103 | */ | 1105 | */ |
| 1104 | } | 1106 | } |
| 1107 | |||
| 1108 | #ifdef CONFIG_DEV_PAGEMAP_OPS | ||
| 1109 | void put_devmap_managed_page(struct page *page) | ||
| 1110 | { | ||
| 1111 | int count; | ||
| 1112 | |||
| 1113 | if (WARN_ON_ONCE(!page_is_devmap_managed(page))) | ||
| 1114 | return; | ||
| 1115 | |||
| 1116 | count = page_ref_dec_return(page); | ||
| 1117 | |||
| 1118 | /* | ||
| 1119 | * devmap page refcounts are 1-based, rather than 0-based: if | ||
| 1120 | * refcount is 1, then the page is free and the refcount is | ||
| 1121 | * stable because nobody holds a reference on the page. | ||
| 1122 | */ | ||
| 1123 | if (count == 1) | ||
| 1124 | free_devmap_managed_page(page); | ||
| 1125 | else if (!count) | ||
| 1126 | __put_page(page); | ||
| 1127 | } | ||
| 1128 | EXPORT_SYMBOL(put_devmap_managed_page); | ||
| 1129 | #endif | ||