Move memory policy interpretation out of alloc_buddy_hugetlb_folio_with_mpol() and into alloc_hugetlb_folio() to separate reading and interpretation of memory policy from actual allocation. This will later allow memory policy to be interpreted outside of the process of allocating a hugetlb folio entirely. This opens doors for other callers of the HugeTLB folio allocation function, such as guest_memfd, where memory may not always be mapped and hence may not have an associated vma. Introduce struct mempolicy_interpreted to hold all the components of an interpreted memory policy. Rename alloc_buddy_hugetlb_folio_with_mpol() to alloc_buddy_hugetlb_folio() since the function no longer interprets memory policy. No functional change intended.
Unified · Split
| @@ -16,7 +16,7 @@ · 15 unmodified lines | |||
| 16 | */ | 16 | */ |
| 17 | 17 | ||
| 18 | /* Policies */ | 18 | /* Policies */ |
| 19 | enum { | 19 | enum mempolicy_mode { |
| 20 | MPOL_DEFAULT, | 20 | MPOL_DEFAULT, |
| 21 | MPOL_PREFERRED, | 21 | MPOL_PREFERRED, |
| 22 | MPOL_BIND, | 22 | MPOL_BIND, |
| @@ -1316,6 +1316,12 @@ static unsigned long available_huge_pages(struct hstate *h) · 1315 unmodified lines | |||
| 1316 | return h->free_huge_pages - h->resv_huge_pages; | 1316 | return h->free_huge_pages - h->resv_huge_pages; |
| 1317 | } | 1317 | } |
| 1318 | 1318 | ||
| 1319 | struct mempolicy_interpreted { | ||
| 1320 | int nid; | ||
| 1321 | nodemask_t *nodemask; | ||
| 1322 | enum mempolicy_mode mode; | ||
| 1323 | }; | ||
| 1324 | |||
| 1319 | static struct folio *dequeue_hugetlb_folio_vma(struct hstate *h, | 1325 | static struct folio *dequeue_hugetlb_folio_vma(struct hstate *h, |
| 1320 | struct vm_area_struct *vma, | 1326 | struct vm_area_struct *vma, |
| 1321 | unsigned long address) | 1327 | unsigned long address) |
| @@ -2137,32 +2143,28 @@ static struct folio *alloc_migrate_hugetlb_folio(struct hstate *h, gfp_t gfp_mas · 815 unmodified lines | |||
| 2137 | return folio; | 2143 | return folio; |
| 2138 | } | 2144 | } |
| 2139 | 2145 | ||
| 2140 | /* | ||
| 2141 | * Use the VMA's mpolicy to allocate a huge page from the buddy. | ||
| 2142 | */ | ||
| 2143 | static | 2146 | static |
| 2144 | struct folio *alloc_buddy_hugetlb_folio_with_mpol(struct hstate *h, | 2147 | struct folio *alloc_buddy_hugetlb_folio(struct hstate *h, |
| 2145 | struct vm_area_struct *vma, unsigned long addr) | 2148 | gfp_t gfp_mask, struct mempolicy_interpreted *mpoli) |
| 2146 | { | 2149 | { |
| 2147 | struct folio *folio = NULL; | 2150 | struct folio *folio = NULL; |
| 2148 | struct mempolicy *mpol; | 2151 | nodemask_t *nodemask = mpoli->nodemask; |
| 2149 | gfp_t gfp_mask = htlb_alloc_mask(h); | ||
| 2150 | int nid; | ||
| 2151 | nodemask_t *nodemask; | ||
| 2152 | 2152 | ||
| 2153 | nid = huge_node(vma, addr, gfp_mask, &mpol, &nodemask); | 2153 | if (mpoli->mode == MPOL_PREFERRED_MANY) { |
| 2154 | if (mpol_is_preferred_many(mpol)) { | ||
| 2155 | gfp_t gfp = gfp_mask & ~(__GFP_DIRECT_RECLAIM | __GFP_NOFAIL); | 2154 | gfp_t gfp = gfp_mask & ~(__GFP_DIRECT_RECLAIM | __GFP_NOFAIL); |
| 2156 | 2155 | ||
| 2157 | folio = alloc_surplus_hugetlb_folio(h, gfp, nid, nodemask); | 2156 | folio = alloc_surplus_hugetlb_folio(h, gfp, mpoli->nid, |
| 2157 | nodemask); | ||
| 2158 | 2158 | ||
| 2159 | /* Fallback to all nodes if page==NULL */ | 2159 | /* Fallback to all nodes if page==NULL */ |
| 2160 | nodemask = NULL; | 2160 | nodemask = NULL; |
| 2161 | } | 2161 | } |
| 2162 | 2162 | ||
| 2163 | if (!folio) | 2163 | if (!folio) { |
| 2164 | folio = alloc_surplus_hugetlb_folio(h, gfp_mask, nid, nodemask); | 2164 | folio = alloc_surplus_hugetlb_folio(h, gfp_mask, mpoli->nid, |
| 2165 | mpol_cond_put(mpol); | 2165 | nodemask); |
| 2166 | } | ||
| 2167 | |||
| 2166 | return folio; | 2168 | return folio; |
| 2167 | } | 2169 | } |
| 2168 | 2170 | ||
| @@ -2852,7 +2854,7 @@ struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma, · 683 unmodified lines | |||
| 2852 | int ret, idx; | 2854 | int ret, idx; |
| 2853 | struct hugetlb_cgroup *h_cg = NULL; | 2855 | struct hugetlb_cgroup *h_cg = NULL; |
| 2854 | struct hugetlb_cgroup *h_cg_rsvd = NULL; | 2856 | struct hugetlb_cgroup *h_cg_rsvd = NULL; |
| 2855 | gfp_t gfp = htlb_alloc_mask(h) | __GFP_RETRY_MAYFAIL; | 2857 | gfp_t gfp = htlb_alloc_mask(h); |
| 2856 | 2858 | ||
| 2857 | idx = hstate_index(h); | 2859 | idx = hstate_index(h); |
| 2858 | 2860 | ||
| @@ -2924,8 +2926,24 @@ struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma, · 65 unmodified lines | |||
| 2924 | folio = dequeue_hugetlb_folio_vma(h, vma, addr); | 2926 | folio = dequeue_hugetlb_folio_vma(h, vma, addr); |
| 2925 | 2927 | ||
| 2926 | if (!folio) { | 2928 | if (!folio) { |
| 2929 | struct mempolicy_interpreted mpoli; | ||
| 2930 | struct mempolicy *mpol; | ||
| 2931 | nodemask_t *nodemask; | ||
| 2932 | int nid; | ||
| 2933 | |||
| 2927 | spin_unlock_irq(&hugetlb_lock); | 2934 | spin_unlock_irq(&hugetlb_lock); |
| 2928 | folio = alloc_buddy_hugetlb_folio_with_mpol(h, vma, addr); | 2935 | nid = huge_node(vma, addr, gfp, &mpol, &nodemask); |
| 2936 | mpoli = (struct mempolicy_interpreted){ | ||
| 2937 | .nid = nid, | ||
| 2938 | #ifdef CONFIG_NUMA | ||
| 2939 | .mode = mpol ? mpol->mode : MPOL_DEFAULT, | ||
| 2940 | #else | ||
| 2941 | .mode = MPOL_DEFAULT, | ||
| 2942 | #endif | ||
| 2943 | .nodemask = nodemask, | ||
| 2944 | }; | ||
| 2945 | folio = alloc_buddy_hugetlb_folio(h, gfp, &mpoli); | ||
| 2946 | mpol_cond_put(mpol); | ||
| 2929 | if (!folio) | 2947 | if (!folio) |
| 2930 | goto out_uncharge_cgroup; | 2948 | goto out_uncharge_cgroup; |
| 2931 | spin_lock_irq(&hugetlb_lock); | 2949 | spin_lock_irq(&hugetlb_lock); |
| @@ -2981,7 +2999,7 @@ struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma, · 49 unmodified lines | |||
| 2981 | } | 2999 | } |
| 2982 | } | 3000 | } |
| 2983 | 3001 | ||
| 2984 | ret = mem_cgroup_charge_hugetlb(folio, gfp); | 3002 | ret = mem_cgroup_charge_hugetlb(folio, gfp | __GFP_RETRY_MAYFAIL); |
| 2985 | /* | 3003 | /* |
| 2986 | * Unconditionally increment NR_HUGETLB here. If it turns out that | 3004 | * Unconditionally increment NR_HUGETLB here. If it turns out that |
| 2987 | * mem_cgroup_charge_hugetlb failed, then immediately free the page and | 3005 | * mem_cgroup_charge_hugetlb failed, then immediately free the page and |