Move memory policy interpretation out of alloc_buddy_hugetlb_folio_with_mpol() and into alloc_hugetlb_folio() to separate reading and interpretation of memory policy from actual allocation. This will later allow memory policy to be interpreted outside of the process of allocating a hugetlb folio entirely. This opens doors for other callers of the HugeTLB folio allocation function, such as guest_memfd, where memory may not always be mapped and hence may not have an associated vma. Introduce struct mempolicy_interpreted to hold all the components of an interpreted memory policy. Rename alloc_buddy_hugetlb_folio_with_mpol() to alloc_buddy_hugetlb_folio() since the function no longer interprets memory policy. No functional change intended.
Unified · Split
| @@ -16,7 +16,7 @@ · 15 unmodified lines | ||
| 16 | 16 | */ |
| 17 | 17 | |
| 18 | 18 | /* Policies */ |
| 19 | −enum { | |
| 19 | +enum mempolicy_mode { | |
| 20 | 20 | MPOL_DEFAULT, |
| 21 | 21 | MPOL_PREFERRED, |
| 22 | 22 | MPOL_BIND, |
| @@ -1316,6 +1316,12 @@ static unsigned long available_huge_pages(struct hstate *h) · 1315 unmodified lines | ||
| 1316 | 1316 | return h->free_huge_pages - h->resv_huge_pages; |
| 1317 | 1317 | } |
| 1318 | 1318 | |
| 1319 | +struct mempolicy_interpreted { | |
| 1320 | + int nid; | |
| 1321 | + nodemask_t *nodemask; | |
| 1322 | + enum mempolicy_mode mode; | |
| 1323 | +}; | |
| 1324 | + | |
| 1319 | 1325 | static struct folio *dequeue_hugetlb_folio_vma(struct hstate *h, |
| 1320 | 1326 | struct vm_area_struct *vma, |
| 1321 | 1327 | unsigned long address) |
| @@ -2137,32 +2143,28 @@ static struct folio *alloc_migrate_hugetlb_folio(struct hstate *h, gfp_t gfp_mas · 815 unmodified lines | ||
| 2137 | 2143 | return folio; |
| 2138 | 2144 | } |
| 2139 | 2145 | |
| 2140 | −/* | |
| 2141 | − * Use the VMA's mpolicy to allocate a huge page from the buddy. | |
| 2142 | − */ | |
| 2143 | 2146 | static |
| 2144 | −struct folio *alloc_buddy_hugetlb_folio_with_mpol(struct hstate *h, | |
| 2145 | − struct vm_area_struct *vma, unsigned long addr) | |
| 2147 | +struct folio *alloc_buddy_hugetlb_folio(struct hstate *h, | |
| 2148 | + gfp_t gfp_mask, struct mempolicy_interpreted *mpoli) | |
| 2146 | 2149 | { |
| 2147 | 2150 | struct folio *folio = NULL; |
| 2148 | − struct mempolicy *mpol; | |
| 2149 | − gfp_t gfp_mask = htlb_alloc_mask(h); | |
| 2150 | − int nid; | |
| 2151 | − nodemask_t *nodemask; | |
| 2151 | + nodemask_t *nodemask = mpoli->nodemask; | |
| 2152 | 2152 | |
| 2153 | − nid = huge_node(vma, addr, gfp_mask, &mpol, &nodemask); | |
| 2154 | − if (mpol_is_preferred_many(mpol)) { | |
| 2153 | + if (mpoli->mode == MPOL_PREFERRED_MANY) { | |
| 2155 | 2154 | gfp_t gfp = gfp_mask & ~(__GFP_DIRECT_RECLAIM | __GFP_NOFAIL); |
| 2156 | 2155 | |
| 2157 | − folio = alloc_surplus_hugetlb_folio(h, gfp, nid, nodemask); | |
| 2156 | + folio = alloc_surplus_hugetlb_folio(h, gfp, mpoli->nid, | |
| 2157 | + nodemask); | |
| 2158 | 2158 | |
| 2159 | 2159 | /* Fallback to all nodes if page==NULL */ |
| 2160 | 2160 | nodemask = NULL; |
| 2161 | 2161 | } |
| 2162 | 2162 | |
| 2163 | − if (!folio) | |
| 2164 | − folio = alloc_surplus_hugetlb_folio(h, gfp_mask, nid, nodemask); | |
| 2165 | − mpol_cond_put(mpol); | |
| 2163 | + if (!folio) { | |
| 2164 | + folio = alloc_surplus_hugetlb_folio(h, gfp_mask, mpoli->nid, | |
| 2165 | + nodemask); | |
| 2166 | + } | |
| 2167 | + | |
| 2166 | 2168 | return folio; |
| 2167 | 2169 | } |
| 2168 | 2170 | |
| @@ -2852,7 +2854,7 @@ struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma, · 683 unmodified lines | ||
| 2852 | 2854 | int ret, idx; |
| 2853 | 2855 | struct hugetlb_cgroup *h_cg = NULL; |
| 2854 | 2856 | struct hugetlb_cgroup *h_cg_rsvd = NULL; |
| 2855 | − gfp_t gfp = htlb_alloc_mask(h) | __GFP_RETRY_MAYFAIL; | |
| 2857 | + gfp_t gfp = htlb_alloc_mask(h); | |
| 2856 | 2858 | |
| 2857 | 2859 | idx = hstate_index(h); |
| 2858 | 2860 | |
| @@ -2924,8 +2926,24 @@ struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma, · 65 unmodified lines | ||
| 2924 | 2926 | folio = dequeue_hugetlb_folio_vma(h, vma, addr); |
| 2925 | 2927 | |
| 2926 | 2928 | if (!folio) { |
| 2929 | + struct mempolicy_interpreted mpoli; | |
| 2930 | + struct mempolicy *mpol; | |
| 2931 | + nodemask_t *nodemask; | |
| 2932 | + int nid; | |
| 2933 | + | |
| 2927 | 2934 | spin_unlock_irq(&hugetlb_lock); |
| 2928 | − folio = alloc_buddy_hugetlb_folio_with_mpol(h, vma, addr); | |
| 2935 | + nid = huge_node(vma, addr, gfp, &mpol, &nodemask); | |
| 2936 | + mpoli = (struct mempolicy_interpreted){ | |
| 2937 | + .nid = nid, | |
| 2938 | +#ifdef CONFIG_NUMA | |
| 2939 | + .mode = mpol ? mpol->mode : MPOL_DEFAULT, | |
| 2940 | +#else | |
| 2941 | + .mode = MPOL_DEFAULT, | |
| 2942 | +#endif | |
| 2943 | + .nodemask = nodemask, | |
| 2944 | + }; | |
| 2945 | + folio = alloc_buddy_hugetlb_folio(h, gfp, &mpoli); | |
| 2946 | + mpol_cond_put(mpol); | |
| 2929 | 2947 | if (!folio) |
| 2930 | 2948 | goto out_uncharge_cgroup; |
| 2931 | 2949 | spin_lock_irq(&hugetlb_lock); |
| @@ -2981,7 +2999,7 @@ struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma, · 49 unmodified lines | ||
| 2981 | 2999 | } |
| 2982 | 3000 | } |
| 2983 | 3001 | |
| 2984 | − ret = mem_cgroup_charge_hugetlb(folio, gfp); | |
| 3002 | + ret = mem_cgroup_charge_hugetlb(folio, gfp | __GFP_RETRY_MAYFAIL); | |
| 2985 | 3003 | /* |
| 2986 | 3004 | * Unconditionally increment NR_HUGETLB here. If it turns out that |
| 2987 | 3005 | * mem_cgroup_charge_hugetlb failed, then immediately free the page and |