Currently, memalloc_nocma_{save/restore} API that prevents CMA area in page allocation is implemented by using current_gfp_context(). However, there are two problems of this implementation. First, this doesn't work for allocation fastpath. In the fastpath, original gfp_mask is used since current_gfp_context() is introduced in order to control reclaim and it is on slowpath. So, CMA area can be allocated through the allocation fastpath even if memalloc_nocma_{save/restore} APIs are used. Currently, there is just one user for these APIs and it has a fallback method to prevent actual problem. Second, clearing __GFP_MOVABLE in current_gfp_context() has a side effect to exclude the memory on the ZONE_MOVABLE for allocation target. To fix these problems, this patch changes the implementation to exclude CMA area in page allocation. Main point of this change is using the alloc_flags. alloc_flags is mainly used to control allocation so it fits for excluding CMA area in allocation.
Unified · Split
| @@ -175,12 +175,10 @@ static inline bool in_vfork(struct task_struct *tsk) · 174 unmodified lines | |||
| 175 | * Applies per-task gfp context to the given allocation flags. | 175 | * Applies per-task gfp context to the given allocation flags. |
| 176 | * PF_MEMALLOC_NOIO implies GFP_NOIO | 176 | * PF_MEMALLOC_NOIO implies GFP_NOIO |
| 177 | * PF_MEMALLOC_NOFS implies GFP_NOFS | 177 | * PF_MEMALLOC_NOFS implies GFP_NOFS |
| 178 | * PF_MEMALLOC_NOCMA implies no allocation from CMA region. | ||
| 179 | */ | 178 | */ |
| 180 | static inline gfp_t current_gfp_context(gfp_t flags) | 179 | static inline gfp_t current_gfp_context(gfp_t flags) |
| 181 | { | 180 | { |
| 182 | if (unlikely(current->flags & | 181 | if (unlikely(current->flags & (PF_MEMALLOC_NOIO | PF_MEMALLOC_NOFS))) { |
| 183 | (PF_MEMALLOC_NOIO | PF_MEMALLOC_NOFS | PF_MEMALLOC_NOCMA))) { | ||
| 184 | /* | 182 | /* |
| 185 | * NOIO implies both NOIO and NOFS and it is a weaker context | 183 | * NOIO implies both NOIO and NOFS and it is a weaker context |
| 186 | * so always make sure it makes precedence | 184 | * so always make sure it makes precedence |
| @@ -189,10 +187,6 @@ static inline gfp_t current_gfp_context(gfp_t flags) · 2 unmodified lines | |||
| 189 | flags &= ~(__GFP_IO | __GFP_FS); | 187 | flags &= ~(__GFP_IO | __GFP_FS); |
| 190 | else if (current->flags & PF_MEMALLOC_NOFS) | 188 | else if (current->flags & PF_MEMALLOC_NOFS) |
| 191 | flags &= ~__GFP_FS; | 189 | flags &= ~__GFP_FS; |
| 192 | #ifdef CONFIG_CMA | ||
| 193 | if (current->flags & PF_MEMALLOC_NOCMA) | ||
| 194 | flags &= ~__GFP_MOVABLE; | ||
| 195 | #endif | ||
| 196 | } | 190 | } |
| 197 | return flags; | 191 | return flags; |
| 198 | } | 192 | } |
| @@ -2785,7 +2785,7 @@ __rmqueue(struct zone *zone, unsigned int order, int migratetype, · 2784 unmodified lines | |||
| 2785 | * allocating from CMA when over half of the zone's free memory | 2785 | * allocating from CMA when over half of the zone's free memory |
| 2786 | * is in the CMA area. | 2786 | * is in the CMA area. |
| 2787 | */ | 2787 | */ |
| 2788 | if (migratetype == MIGRATE_MOVABLE && | 2788 | if (alloc_flags & ALLOC_CMA && |
| 2789 | zone_page_state(zone, NR_FREE_CMA_PAGES) > | 2789 | zone_page_state(zone, NR_FREE_CMA_PAGES) > |
| 2790 | zone_page_state(zone, NR_FREE_PAGES) / 2) { | 2790 | zone_page_state(zone, NR_FREE_PAGES) / 2) { |
| 2791 | page = __rmqueue_cma_fallback(zone, order); | 2791 | page = __rmqueue_cma_fallback(zone, order); |
| @@ -2796,7 +2796,7 @@ __rmqueue(struct zone *zone, unsigned int order, int migratetype, · 4 unmodified lines | |||
| 2796 | retry: | 2796 | retry: |
| 2797 | page = __rmqueue_smallest(zone, order, migratetype); | 2797 | page = __rmqueue_smallest(zone, order, migratetype); |
| 2798 | if (unlikely(!page)) { | 2798 | if (unlikely(!page)) { |
| 2799 | if (migratetype == MIGRATE_MOVABLE) | 2799 | if (alloc_flags & ALLOC_CMA) |
| 2800 | page = __rmqueue_cma_fallback(zone, order); | 2800 | page = __rmqueue_cma_fallback(zone, order); |
| 2801 | 2801 | ||
| 2802 | if (!page && __rmqueue_fallback(zone, order, migratetype, | 2802 | if (!page && __rmqueue_fallback(zone, order, migratetype, |
| @@ -3687,6 +3687,20 @@ alloc_flags_nofragment(struct zone *zone, gfp_t gfp_mask) · 884 unmodified lines | |||
| 3687 | return alloc_flags; | 3687 | return alloc_flags; |
| 3688 | } | 3688 | } |
| 3689 | 3689 | ||
| 3690 | static inline unsigned int current_alloc_flags(gfp_t gfp_mask, | ||
| 3691 | unsigned int alloc_flags) | ||
| 3692 | { | ||
| 3693 | #ifdef CONFIG_CMA | ||
| 3694 | unsigned int pflags = current->flags; | ||
| 3695 | |||
| 3696 | if (!(pflags & PF_MEMALLOC_NOCMA) && | ||
| 3697 | gfp_migratetype(gfp_mask) == MIGRATE_MOVABLE) | ||
| 3698 | alloc_flags |= ALLOC_CMA; | ||
| 3699 | |||
| 3700 | #endif | ||
| 3701 | return alloc_flags; | ||
| 3702 | } | ||
| 3703 | |||
| 3690 | /* | 3704 | /* |
| 3691 | * get_page_from_freelist goes through the zonelist trying to allocate | 3705 | * get_page_from_freelist goes through the zonelist trying to allocate |
| 3692 | * a page. | 3706 | * a page. |
| @@ -4333,10 +4347,8 @@ gfp_to_alloc_flags(gfp_t gfp_mask) · 640 unmodified lines | |||
| 4333 | } else if (unlikely(rt_task(current)) && !in_interrupt()) | 4347 | } else if (unlikely(rt_task(current)) && !in_interrupt()) |
| 4334 | alloc_flags |= ALLOC_HARDER; | 4348 | alloc_flags |= ALLOC_HARDER; |
| 4335 | 4349 | ||
| 4336 | #ifdef CONFIG_CMA | 4350 | alloc_flags = current_alloc_flags(gfp_mask, alloc_flags); |
| 4337 | if (gfp_migratetype(gfp_mask) == MIGRATE_MOVABLE) | 4351 | |
| 4338 | alloc_flags |= ALLOC_CMA; | ||
| 4339 | #endif | ||
| 4340 | return alloc_flags; | 4352 | return alloc_flags; |
| 4341 | } | 4353 | } |
| 4342 | 4354 | ||
| @@ -4637,7 +4649,7 @@ retry: · 294 unmodified lines | |||
| 4637 | 4649 | ||
| 4638 | reserve_flags = __gfp_pfmemalloc_flags(gfp_mask); | 4650 | reserve_flags = __gfp_pfmemalloc_flags(gfp_mask); |
| 4639 | if (reserve_flags) | 4651 | if (reserve_flags) |
| 4640 | alloc_flags = reserve_flags; | 4652 | alloc_flags = current_alloc_flags(gfp_mask, reserve_flags); |
| 4641 | 4653 | ||
| 4642 | /* | 4654 | /* |
| 4643 | * Reset the nodemask and zonelist iterators if memory policies can be | 4655 | * Reset the nodemask and zonelist iterators if memory policies can be |
| @@ -4714,7 +4726,7 @@ retry: · 70 unmodified lines | |||
| 4714 | 4726 | ||
| 4715 | /* Avoid allocations with no watermarks from looping endlessly */ | 4727 | /* Avoid allocations with no watermarks from looping endlessly */ |
| 4716 | if (tsk_is_oom_victim(current) && | 4728 | if (tsk_is_oom_victim(current) && |
| 4717 | (alloc_flags == ALLOC_OOM || | 4729 | (alloc_flags & ALLOC_OOM || |
| 4718 | (gfp_mask & __GFP_NOMEMALLOC))) | 4730 | (gfp_mask & __GFP_NOMEMALLOC))) |
| 4719 | goto nopage; | 4731 | goto nopage; |
| 4720 | 4732 | ||
| @@ -4806,8 +4818,7 @@ static inline bool prepare_alloc_pages(gfp_t gfp_mask, unsigned int order, · 85 unmodified lines | |||
| 4806 | if (should_fail_alloc_page(gfp_mask, order)) | 4818 | if (should_fail_alloc_page(gfp_mask, order)) |
| 4807 | return false; | 4819 | return false; |
| 4808 | 4820 | ||
| 4809 | if (IS_ENABLED(CONFIG_CMA) && ac->migratetype == MIGRATE_MOVABLE) | 4821 | *alloc_flags = current_alloc_flags(gfp_mask, *alloc_flags); |
| 4810 | *alloc_flags |= ALLOC_CMA; | ||
| 4811 | 4822 | ||
| 4812 | return true; | 4823 | return true; |
| 4813 | } | 4824 | } |