An upcoming patch changes and complicates the refcounting and especially the "put page" aspects of it. In order to keep everything clean, refactor the devmap page release routines: * Rename put_devmap_managed_page() to page_is_devmap_managed(), and limit the functionality to "read only": return a bool, with no side effects. * Add a new routine, put_devmap_managed_page(), to handle decrementing the refcount for ZONE_DEVICE pages. * Change callers (just release_pages() and put_page()) to check page_is_devmap_managed() before calling the new put_devmap_managed_page() routine. This is a performance point: put_page() is a hot path, so we need to avoid non- inline function calls where possible. * Rename __put_devmap_managed_page() to free_devmap_managed_page(), and limit the functionality to unconditionally freeing a devmap page. This is originally based on a separate patch by Ira Weiny, which applied to an early version of the put_user_page() experiments. Since then, Jérôme Glisse suggested the refactoring described above.
Showing the changes to include/linux (1 of 3 files). Show all
Unified · Split
| @@ -947,9 +947,10 @@ static inline bool is_zone_device_page(const struct page *page) · 946 unmodified lines | |||
| 947 | #endif | 947 | #endif |
| 948 | 948 | ||
| 949 | #ifdef CONFIG_DEV_PAGEMAP_OPS | 949 | #ifdef CONFIG_DEV_PAGEMAP_OPS |
| 950 | void __put_devmap_managed_page(struct page *page); | 950 | void free_devmap_managed_page(struct page *page); |
| 951 | DECLARE_STATIC_KEY_FALSE(devmap_managed_key); | 951 | DECLARE_STATIC_KEY_FALSE(devmap_managed_key); |
| 952 | static inline bool put_devmap_managed_page(struct page *page) | 952 | |
| 953 | static inline bool page_is_devmap_managed(struct page *page) | ||
| 953 | { | 954 | { |
| 954 | if (!static_branch_unlikely(&devmap_managed_key)) | 955 | if (!static_branch_unlikely(&devmap_managed_key)) |
| 955 | return false; | 956 | return false; |
| @@ -958,7 +959,6 @@ static inline bool put_devmap_managed_page(struct page *page) · 2 unmodified lines | |||
| 958 | switch (page->pgmap->type) { | 959 | switch (page->pgmap->type) { |
| 959 | case MEMORY_DEVICE_PRIVATE: | 960 | case MEMORY_DEVICE_PRIVATE: |
| 960 | case MEMORY_DEVICE_FS_DAX: | 961 | case MEMORY_DEVICE_FS_DAX: |
| 961 | __put_devmap_managed_page(page); | ||
| 962 | return true; | 962 | return true; |
| 963 | default: | 963 | default: |
| 964 | break; | 964 | break; |
| @@ -966,11 +966,17 @@ static inline bool put_devmap_managed_page(struct page *page) · 1 unmodified line | |||
| 966 | return false; | 966 | return false; |
| 967 | } | 967 | } |
| 968 | 968 | ||
| 969 | void put_devmap_managed_page(struct page *page); | ||
| 970 | |||
| 969 | #else /* CONFIG_DEV_PAGEMAP_OPS */ | 971 | #else /* CONFIG_DEV_PAGEMAP_OPS */ |
| 970 | static inline bool put_devmap_managed_page(struct page *page) | 972 | static inline bool page_is_devmap_managed(struct page *page) |
| 971 | { | 973 | { |
| 972 | return false; | 974 | return false; |
| 973 | } | 975 | } |
| 976 | |||
| 977 | static inline void put_devmap_managed_page(struct page *page) | ||
| 978 | { | ||
| 979 | } | ||
| 974 | #endif /* CONFIG_DEV_PAGEMAP_OPS */ | 980 | #endif /* CONFIG_DEV_PAGEMAP_OPS */ |
| 975 | 981 | ||
| 976 | static inline bool is_device_private_page(const struct page *page) | 982 | static inline bool is_device_private_page(const struct page *page) |
| @@ -1023,8 +1029,10 @@ static inline void put_page(struct page *page) · 46 unmodified lines | |||
| 1023 | * need to inform the device driver through callback. See | 1029 | * need to inform the device driver through callback. See |
| 1024 | * include/linux/memremap.h and HMM for details. | 1030 | * include/linux/memremap.h and HMM for details. |
| 1025 | */ | 1031 | */ |
| 1026 | if (put_devmap_managed_page(page)) | 1032 | if (page_is_devmap_managed(page)) { |
| 1033 | put_devmap_managed_page(page); | ||
| 1027 | return; | 1034 | return; |
| 1035 | } | ||
| 1028 | 1036 | ||
| 1029 | if (put_page_testzero(page)) | 1037 | if (put_page_testzero(page)) |
| 1030 | __put_page(page); | 1038 | __put_page(page); |