All exit paths from gup_pte_range() require pte_unmap() of the original pte page before returning. Refactor the code to have a single exit point to do the unmap. This mirrors the flow of the generic gup_pte_range() in mm/gup.c.
Unified · Split
| @@ -106,36 +106,35 @@ static noinline int gup_pte_range(pmd_t pmd, unsigned long addr, · 105 unmodified lines | |||
| 106 | unsigned long end, int write, struct page **pages, int *nr) | 106 | unsigned long end, int write, struct page **pages, int *nr) |
| 107 | { | 107 | { |
| 108 | struct dev_pagemap *pgmap = NULL; | 108 | struct dev_pagemap *pgmap = NULL; |
| 109 | int nr_start = *nr; | 109 | int nr_start = *nr, ret = 0; |
| 110 | pte_t *ptep; | 110 | pte_t *ptep, *ptem; |
| 111 | 111 | ||
| 112 | ptep = pte_offset_map(&pmd, addr); | 112 | /* |
| 113 | * Keep the original mapped PTE value (ptem) around since we | ||
| 114 | * might increment ptep off the end of the page when finishing | ||
| 115 | * our loop iteration. | ||
| 116 | */ | ||
| 117 | ptem = ptep = pte_offset_map(&pmd, addr); | ||
| 113 | do { | 118 | do { |
| 114 | pte_t pte = gup_get_pte(ptep); | 119 | pte_t pte = gup_get_pte(ptep); |
| 115 | struct page *page; | 120 | struct page *page; |
| 116 | 121 | ||
| 117 | /* Similar to the PMD case, NUMA hinting must take slow path */ | 122 | /* Similar to the PMD case, NUMA hinting must take slow path */ |
| 118 | if (pte_protnone(pte)) { | 123 | if (pte_protnone(pte)) |
| 119 | pte_unmap(ptep); | 124 | break; |
| 120 | return 0; | ||
| 121 | } | ||
| 122 | 125 | ||
| 123 | if (!pte_allows_gup(pte_val(pte), write)) { | 126 | if (!pte_allows_gup(pte_val(pte), write)) |
| 124 | pte_unmap(ptep); | 127 | break; |
| 125 | return 0; | ||
| 126 | } | ||
| 127 | 128 | ||
| 128 | if (pte_devmap(pte)) { | 129 | if (pte_devmap(pte)) { |
| 129 | pgmap = get_dev_pagemap(pte_pfn(pte), pgmap); | 130 | pgmap = get_dev_pagemap(pte_pfn(pte), pgmap); |
| 130 | if (unlikely(!pgmap)) { | 131 | if (unlikely(!pgmap)) { |
| 131 | undo_dev_pagemap(nr, nr_start, pages); | 132 | undo_dev_pagemap(nr, nr_start, pages); |
| 132 | pte_unmap(ptep); | 133 | break; |
| 133 | return 0; | ||
| 134 | } | 134 | } |
| 135 | } else if (pte_special(pte)) { | 135 | } else if (pte_special(pte)) |
| 136 | pte_unmap(ptep); | 136 | break; |
| 137 | return 0; | 137 | |
| 138 | } | ||
| 139 | VM_BUG_ON(!pfn_valid(pte_pfn(pte))); | 138 | VM_BUG_ON(!pfn_valid(pte_pfn(pte))); |
| 140 | page = pte_page(pte); | 139 | page = pte_page(pte); |
| 141 | get_page(page); | 140 | get_page(page); |
| @@ -145,9 +144,11 @@ static noinline int gup_pte_range(pmd_t pmd, unsigned long addr, · 3 unmodified lines | |||
| 145 | (*nr)++; | 144 | (*nr)++; |
| 146 | 145 | ||
| 147 | } while (ptep++, addr += PAGE_SIZE, addr != end); | 146 | } while (ptep++, addr += PAGE_SIZE, addr != end); |
| 148 | pte_unmap(ptep - 1); | 147 | if (addr == end) |
| 148 | ret = 1; | ||
| 149 | pte_unmap(ptem); | ||
| 149 | 150 | ||
| 150 | return 1; | 151 | return ret; |
| 151 | } | 152 | } |
| 152 | 153 | ||
| 153 | static inline void get_head_page_multiple(struct page *page, int nr) | 154 | static inline void get_head_page_multiple(struct page *page, int nr) |