Currently, memalloc_nocma_{save/restore} API that prevents CMA area in page allocation is implemented by using current_gfp_context(). However, there are two problems of this implementation. First, this doesn't work for allocation fastpath. In the fastpath, original gfp_mask is used since current_gfp_context() is introduced in order to control reclaim and it is on slowpath. So, CMA area can be allocated through the allocation fastpath even if memalloc_nocma_{save/restore} APIs are used. Currently, there is just one user for these APIs and it has a fallback method to prevent actual problem. Second, clearing __GFP_MOVABLE in current_gfp_context() has a side effect to exclude the memory on the ZONE_MOVABLE for allocation target. To fix these problems, this patch changes the implementation to exclude CMA area in page allocation. Main point of this change is using the alloc_flags. alloc_flags is mainly used to control allocation so it fits for excluding CMA area in allocation.
Unified · Split
| @@ -175,12 +175,10 @@ static inline bool in_vfork(struct task_struct *tsk) · 174 unmodified lines | ||
| 175 | 175 | * Applies per-task gfp context to the given allocation flags. |
| 176 | 176 | * PF_MEMALLOC_NOIO implies GFP_NOIO |
| 177 | 177 | * PF_MEMALLOC_NOFS implies GFP_NOFS |
| 178 | − * PF_MEMALLOC_NOCMA implies no allocation from CMA region. | |
| 179 | 178 | */ |
| 180 | 179 | static inline gfp_t current_gfp_context(gfp_t flags) |
| 181 | 180 | { |
| 182 | − if (unlikely(current->flags & | |
| 183 | − (PF_MEMALLOC_NOIO | PF_MEMALLOC_NOFS | PF_MEMALLOC_NOCMA))) { | |
| 181 | + if (unlikely(current->flags & (PF_MEMALLOC_NOIO | PF_MEMALLOC_NOFS))) { | |
| 184 | 182 | /* |
| 185 | 183 | * NOIO implies both NOIO and NOFS and it is a weaker context |
| 186 | 184 | * so always make sure it makes precedence |
| @@ -189,10 +187,6 @@ static inline gfp_t current_gfp_context(gfp_t flags) · 2 unmodified lines | ||
| 189 | 187 | flags &= ~(__GFP_IO | __GFP_FS); |
| 190 | 188 | else if (current->flags & PF_MEMALLOC_NOFS) |
| 191 | 189 | flags &= ~__GFP_FS; |
| 192 | −#ifdef CONFIG_CMA | |
| 193 | − if (current->flags & PF_MEMALLOC_NOCMA) | |
| 194 | − flags &= ~__GFP_MOVABLE; | |
| 195 | −#endif | |
| 196 | 190 | } |
| 197 | 191 | return flags; |
| 198 | 192 | } |
| @@ -2785,7 +2785,7 @@ __rmqueue(struct zone *zone, unsigned int order, int migratetype, · 2784 unmodified lines | ||
| 2785 | 2785 | * allocating from CMA when over half of the zone's free memory |
| 2786 | 2786 | * is in the CMA area. |
| 2787 | 2787 | */ |
| 2788 | − if (migratetype == MIGRATE_MOVABLE && | |
| 2788 | + if (alloc_flags & ALLOC_CMA && | |
| 2789 | 2789 | zone_page_state(zone, NR_FREE_CMA_PAGES) > |
| 2790 | 2790 | zone_page_state(zone, NR_FREE_PAGES) / 2) { |
| 2791 | 2791 | page = __rmqueue_cma_fallback(zone, order); |
| @@ -2796,7 +2796,7 @@ __rmqueue(struct zone *zone, unsigned int order, int migratetype, · 4 unmodified lines | ||
| 2796 | 2796 | retry: |
| 2797 | 2797 | page = __rmqueue_smallest(zone, order, migratetype); |
| 2798 | 2798 | if (unlikely(!page)) { |
| 2799 | − if (migratetype == MIGRATE_MOVABLE) | |
| 2799 | + if (alloc_flags & ALLOC_CMA) | |
| 2800 | 2800 | page = __rmqueue_cma_fallback(zone, order); |
| 2801 | 2801 | |
| 2802 | 2802 | if (!page && __rmqueue_fallback(zone, order, migratetype, |
| @@ -3687,6 +3687,20 @@ alloc_flags_nofragment(struct zone *zone, gfp_t gfp_mask) · 884 unmodified lines | ||
| 3687 | 3687 | return alloc_flags; |
| 3688 | 3688 | } |
| 3689 | 3689 | |
| 3690 | +static inline unsigned int current_alloc_flags(gfp_t gfp_mask, | |
| 3691 | + unsigned int alloc_flags) | |
| 3692 | +{ | |
| 3693 | +#ifdef CONFIG_CMA | |
| 3694 | + unsigned int pflags = current->flags; | |
| 3695 | + | |
| 3696 | + if (!(pflags & PF_MEMALLOC_NOCMA) && | |
| 3697 | + gfp_migratetype(gfp_mask) == MIGRATE_MOVABLE) | |
| 3698 | + alloc_flags |= ALLOC_CMA; | |
| 3699 | + | |
| 3700 | +#endif | |
| 3701 | + return alloc_flags; | |
| 3702 | +} | |
| 3703 | + | |
| 3690 | 3704 | /* |
| 3691 | 3705 | * get_page_from_freelist goes through the zonelist trying to allocate |
| 3692 | 3706 | * a page. |
| @@ -4333,10 +4347,8 @@ gfp_to_alloc_flags(gfp_t gfp_mask) · 640 unmodified lines | ||
| 4333 | 4347 | } else if (unlikely(rt_task(current)) && !in_interrupt()) |
| 4334 | 4348 | alloc_flags |= ALLOC_HARDER; |
| 4335 | 4349 | |
| 4336 | −#ifdef CONFIG_CMA | |
| 4337 | − if (gfp_migratetype(gfp_mask) == MIGRATE_MOVABLE) | |
| 4338 | − alloc_flags |= ALLOC_CMA; | |
| 4339 | −#endif | |
| 4350 | + alloc_flags = current_alloc_flags(gfp_mask, alloc_flags); | |
| 4351 | + | |
| 4340 | 4352 | return alloc_flags; |
| 4341 | 4353 | } |
| 4342 | 4354 | |
| @@ -4637,7 +4649,7 @@ retry: · 294 unmodified lines | ||
| 4637 | 4649 | |
| 4638 | 4650 | reserve_flags = __gfp_pfmemalloc_flags(gfp_mask); |
| 4639 | 4651 | if (reserve_flags) |
| 4640 | − alloc_flags = reserve_flags; | |
| 4652 | + alloc_flags = current_alloc_flags(gfp_mask, reserve_flags); | |
| 4641 | 4653 | |
| 4642 | 4654 | /* |
| 4643 | 4655 | * Reset the nodemask and zonelist iterators if memory policies can be |
| @@ -4714,7 +4726,7 @@ retry: · 70 unmodified lines | ||
| 4714 | 4726 | |
| 4715 | 4727 | /* Avoid allocations with no watermarks from looping endlessly */ |
| 4716 | 4728 | if (tsk_is_oom_victim(current) && |
| 4717 | − (alloc_flags == ALLOC_OOM || | |
| 4729 | + (alloc_flags & ALLOC_OOM || | |
| 4718 | 4730 | (gfp_mask & __GFP_NOMEMALLOC))) |
| 4719 | 4731 | goto nopage; |
| 4720 | 4732 | |
| @@ -4806,8 +4818,7 @@ static inline bool prepare_alloc_pages(gfp_t gfp_mask, unsigned int order, · 85 unmodified lines | ||
| 4806 | 4818 | if (should_fail_alloc_page(gfp_mask, order)) |
| 4807 | 4819 | return false; |
| 4808 | 4820 | |
| 4809 | − if (IS_ENABLED(CONFIG_CMA) && ac->migratetype == MIGRATE_MOVABLE) | |
| 4810 | − *alloc_flags |= ALLOC_CMA; | |
| 4821 | + *alloc_flags = current_alloc_flags(gfp_mask, *alloc_flags); | |
| 4811 | 4822 | |
| 4812 | 4823 | return true; |
| 4813 | 4824 | } |