The compaction capturing code assumes the allocation request order and compaction target order are the same. That won't be true once defrag_mode promotes sub-block allocations to pageblock-order compaction: compaction targets the larger order, while capture should remain at the original allocation order. Move the capture_control to the page allocator and give it its own copies of what the page freeing path matches against - zone, migratetype and the allocation order - rather than reaching into compaction's live compact_control. __alloc_pages_direct_compact() fills in migratetype and order, and installs and hides current->capture_control around the whole compaction call; try_to_compact_pages() aims capc->zone at each zone while it is being compacted. compact_zone_order() no longer deals with capture at all. Pass the capture_control through try_to_compact_pages() / compact_zone_order() in place of the bare struct page **. No functional change.
Unified · Split
| @@ -58,6 +58,7 @@ enum compact_result { · 57 unmodified lines | ||
| 58 | 58 | }; |
| 59 | 59 | |
| 60 | 60 | struct alloc_context; /* in mm/internal.h */ |
| 61 | +struct capture_control; /* in mm/internal.h */ | |
| 61 | 62 | |
| 62 | 63 | /* |
| 63 | 64 | * Number of free order-0 pages that should be available above given watermark |
| @@ -92,7 +93,7 @@ extern int fragmentation_index(struct zone *zone, unsigned int order); · 28 unmodified lines | ||
| 92 | 93 | extern enum compact_result try_to_compact_pages(gfp_t gfp_mask, |
| 93 | 94 | unsigned int order, unsigned int alloc_flags, |
| 94 | 95 | const struct alloc_context *ac, enum compact_priority prio, |
| 95 | − struct page **page); | |
| 96 | + struct capture_control *capc); | |
| 96 | 97 | extern void reset_isolation_suitable(pg_data_t *pgdat); |
| 97 | 98 | extern bool compaction_suitable(struct zone *zone, int order, |
| 98 | 99 | unsigned long watermark, int highest_zoneidx); |
| @@ -2800,9 +2800,8 @@ out: · 2799 unmodified lines | ||
| 2800 | 2800 | static enum compact_result compact_zone_order(struct zone *zone, int order, |
| 2801 | 2801 | gfp_t gfp_mask, enum compact_priority prio, |
| 2802 | 2802 | unsigned int alloc_flags, int highest_zoneidx, |
| 2803 | − struct page **capture) | |
| 2803 | + struct capture_control *capc) | |
| 2804 | 2804 | { |
| 2805 | − enum compact_result ret; | |
| 2806 | 2805 | struct compact_control cc = { |
| 2807 | 2806 | .order = order, |
| 2808 | 2807 | .search_order = order, |
| @@ -2817,38 +2816,8 @@ static enum compact_result compact_zone_order(struct zone *zone, int order, · 8 unmodified lines | ||
| 2817 | 2816 | .ignore_skip_hint = (prio == MIN_COMPACT_PRIORITY), |
| 2818 | 2817 | .ignore_block_suitable = (prio == MIN_COMPACT_PRIORITY) |
| 2819 | 2818 | }; |
| 2820 | − struct capture_control capc = { | |
| 2821 | − .cc = &cc, | |
| 2822 | − .page = NULL, | |
| 2823 | − }; | |
| 2824 | − | |
| 2825 | − /* | |
| 2826 | − * Make sure the structs are really initialized before we expose the | |
| 2827 | − * capture control, in case we are interrupted and the interrupt handler | |
| 2828 | − * frees a page. | |
| 2829 | − */ | |
| 2830 | − barrier(); | |
| 2831 | − WRITE_ONCE(current->capture_control, &capc); | |
| 2832 | 2819 | |
| 2833 | − ret = compact_zone(&cc, &capc); | |
| 2834 | − | |
| 2835 | − /* | |
| 2836 | − * Make sure we hide capture control first before we read the captured | |
| 2837 | − * page pointer, otherwise an interrupt could free and capture a page | |
| 2838 | − * and we would leak it. | |
| 2839 | − */ | |
| 2840 | − WRITE_ONCE(current->capture_control, NULL); | |
| 2841 | − *capture = READ_ONCE(capc.page); | |
| 2842 | − /* | |
| 2843 | − * Technically, it is also possible that compaction is skipped but | |
| 2844 | − * the page is still captured out of luck(IRQ came and freed the page). | |
| 2845 | − * Returning COMPACT_SUCCESS in such cases helps in properly accounting | |
| 2846 | − * the COMPACT[STALL|FAIL] when compaction is skipped. | |
| 2847 | − */ | |
| 2848 | − if (*capture) | |
| 2849 | − ret = COMPACT_SUCCESS; | |
| 2850 | − | |
| 2851 | − return ret; | |
| 2820 | + return compact_zone(&cc, capc); | |
| 2852 | 2821 | } |
| 2853 | 2822 | |
| 2854 | 2823 | /** |
| @@ -2858,13 +2827,13 @@ static enum compact_result compact_zone_order(struct zone *zone, int order, · 3 unmodified lines | ||
| 2858 | 2827 | * @alloc_flags: The allocation flags of the current allocation |
| 2859 | 2828 | * @ac: The context of current allocation |
| 2860 | 2829 | * @prio: Determines how hard direct compaction should try to succeed |
| 2861 | − * @capture: Pointer to free page created by compaction will be stored here | |
| 2830 | + * @capc: Free page capture bypassing the freelist | |
| 2862 | 2831 | * |
| 2863 | 2832 | * This is the main entry point for direct page compaction. |
| 2864 | 2833 | */ |
| 2865 | 2834 | enum compact_result try_to_compact_pages(gfp_t gfp_mask, unsigned int order, |
| 2866 | 2835 | unsigned int alloc_flags, const struct alloc_context *ac, |
| 2867 | − enum compact_priority prio, struct page **capture) | |
| 2836 | + enum compact_priority prio, struct capture_control *capc) | |
| 2868 | 2837 | { |
| 2869 | 2838 | struct zoneref *z; |
| 2870 | 2839 | struct zone *zone; |
| @@ -2891,8 +2860,17 @@ enum compact_result try_to_compact_pages(gfp_t gfp_mask, unsigned int order, · 20 unmodified lines | ||
| 2891 | 2860 | continue; |
| 2892 | 2861 | } |
| 2893 | 2862 | |
| 2863 | + WRITE_ONCE(capc->zone, zone); | |
| 2864 | + | |
| 2894 | 2865 | status = compact_zone_order(zone, order, gfp_mask, prio, |
| 2895 | − alloc_flags, ac->highest_zoneidx, capture); | |
| 2866 | + alloc_flags, ac->highest_zoneidx, capc); | |
| 2867 | + | |
| 2868 | + WRITE_ONCE(capc->zone, NULL); | |
| 2869 | + | |
| 2870 | + /* Stop if a page has been captured */ | |
| 2871 | + if (READ_ONCE(capc->page)) | |
| 2872 | + status = COMPACT_SUCCESS; | |
| 2873 | + | |
| 2896 | 2874 | rc = max(status, rc); |
| 2897 | 2875 | |
| 2898 | 2876 | /* The allocation should succeed, stop compacting */ |
| @@ -868,7 +868,9 @@ struct compact_control { · 867 unmodified lines | ||
| 868 | 868 | * immediately when one is created during the free path. |
| 869 | 869 | */ |
| 870 | 870 | struct capture_control { |
| 871 | − struct compact_control *cc; | |
| 871 | + struct zone *zone; | |
| 872 | + int migratetype; | |
| 873 | + int order; | |
| 872 | 874 | struct page *page; |
| 873 | 875 | }; |
| 874 | 876 | |
| @@ -725,14 +725,14 @@ static inline struct capture_control *task_capc(struct zone *zone) · 724 unmodified lines | ||
| 725 | 725 | return unlikely(capc) && |
| 726 | 726 | !(current->flags & PF_KTHREAD) && |
| 727 | 727 | !capc->page && |
| 728 | − capc->cc->zone == zone ? capc : NULL; | |
| 728 | + capc->zone == zone ? capc : NULL; | |
| 729 | 729 | } |
| 730 | 730 | |
| 731 | 731 | static inline bool |
| 732 | 732 | compaction_capture(struct capture_control *capc, struct page *page, |
| 733 | 733 | int order, int migratetype) |
| 734 | 734 | { |
| 735 | − if (!capc || order != capc->cc->order) | |
| 735 | + if (!capc || order != capc->order) | |
| 736 | 736 | return false; |
| 737 | 737 | |
| 738 | 738 | /* Do not accidentally pollute CMA or isolated regions*/ |
| @@ -748,12 +748,12 @@ compaction_capture(struct capture_control *capc, struct page *page, · 9 unmodified lines | ||
| 748 | 748 | * have trouble finding a high-order free page. |
| 749 | 749 | */ |
| 750 | 750 | if (order < pageblock_order && migratetype == MIGRATE_MOVABLE && |
| 751 | − capc->cc->migratetype != MIGRATE_MOVABLE) | |
| 751 | + capc->migratetype != MIGRATE_MOVABLE) | |
| 752 | 752 | return false; |
| 753 | 753 | |
| 754 | − if (migratetype != capc->cc->migratetype) | |
| 755 | − trace_mm_page_alloc_extfrag(page, capc->cc->order, order, | |
| 756 | − capc->cc->migratetype, migratetype); | |
| 754 | + if (migratetype != capc->migratetype) | |
| 755 | + trace_mm_page_alloc_extfrag(page, capc->order, order, | |
| 756 | + capc->migratetype, migratetype); | |
| 757 | 757 | |
| 758 | 758 | capc->page = page; |
| 759 | 759 | return true; |
| @@ -4143,6 +4143,12 @@ __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order, · 3383 unmodified lines | ||
| 4143 | 4143 | struct page *page = NULL; |
| 4144 | 4144 | unsigned long pflags; |
| 4145 | 4145 | unsigned int noreclaim_flag; |
| 4146 | + struct capture_control capc = { | |
| 4147 | + .zone = NULL, | |
| 4148 | + .migratetype = ac->migratetype, | |
| 4149 | + .order = order, | |
| 4150 | + .page = NULL, | |
| 4151 | + }; | |
| 4146 | 4152 | |
| 4147 | 4153 | if (!order) |
| 4148 | 4154 | return NULL; |
| @@ -4152,8 +4158,33 @@ __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order, · 3 unmodified lines | ||
| 4152 | 4158 | fs_reclaim_acquire(gfp_mask); |
| 4153 | 4159 | noreclaim_flag = memalloc_noreclaim_save(); |
| 4154 | 4160 | |
| 4161 | + /* | |
| 4162 | + * Make sure the structs are really initialized before we expose the | |
| 4163 | + * capture control, in case we are interrupted and the interrupt handler | |
| 4164 | + * frees a page. | |
| 4165 | + */ | |
| 4166 | + barrier(); | |
| 4167 | + WRITE_ONCE(current->capture_control, &capc); | |
| 4168 | + | |
| 4155 | 4169 | *compact_result = try_to_compact_pages(gfp_mask, order, alloc_flags, ac, |
| 4156 | − prio, &page); | |
| 4170 | + prio, &capc); | |
| 4171 | + | |
| 4172 | + /* | |
| 4173 | + * Make sure we hide capture control first before we read the captured | |
| 4174 | + * page pointer, otherwise an interrupt could free and capture a page | |
| 4175 | + * and we would leak it. | |
| 4176 | + */ | |
| 4177 | + WRITE_ONCE(current->capture_control, NULL); | |
| 4178 | + page = READ_ONCE(capc.page); | |
| 4179 | + | |
| 4180 | + /* | |
| 4181 | + * Technically, it is also possible that compaction is skipped but | |
| 4182 | + * the page is still captured out of luck(IRQ came and freed the page). | |
| 4183 | + * Returning COMPACT_SUCCESS in such cases helps in properly accounting | |
| 4184 | + * the COMPACT[STALL|FAIL] when compaction is skipped. | |
| 4185 | + */ | |
| 4186 | + if (page) | |
| 4187 | + *compact_result = COMPACT_SUCCESS; | |
| 4157 | 4188 | |
| 4158 | 4189 | memalloc_noreclaim_restore(noreclaim_flag); |
| 4159 | 4190 | fs_reclaim_release(gfp_mask); |