mm: page_alloc: move capture_control to the page allocator

The compaction capturing code assumes the allocation request order and compaction target order are the same. That won't be true once defrag_mode promotes sub-block allocations to pageblock-order compaction: compaction targets the larger order, while capture should remain at the original allocation order. Move the capture_control to the page allocator and give it its own copies of what the page freeing path matches against - zone, migratetype and the allocation order - rather than reaching into compaction's live compact_control. __alloc_pages_direct_compact() fills in migratetype and order, and installs and hides current->capture_control around the whole compaction call; try_to_compact_pages() aims capc->zone at each zone while it is being compacted. compact_zone_order() no longer deals with capture at all. Pass the capture_control through try_to_compact_pages() / compact_zone_order() in place of the bare struct page **. No functional change.

Vlastimil Babka (SUSE) committed 3 months ago

Commit aee220f1 parent 1b4b6974 files · +57 −45Browse files

Unified · Split

4 files

include/linux/compaction.h +2 −1
mm/compaction.c +14 −36
mm/internal.h +3 −1
mm/page_alloc.c +38 −7

include/linux/compaction.h+2−1

 };
 struct alloc_context; /* in mm/internal.h */
+struct capture_control; /* in mm/internal.h */
 /*
  * Number of free order-0 pages that should be available above given watermark
 extern enum compact_result try_to_compact_pages(gfp_t gfp_mask,
 		unsigned int order, unsigned int alloc_flags,
 		const struct alloc_context *ac, enum compact_priority prio,
-		struct page **page);
+		struct capture_control *capc);
 extern void reset_isolation_suitable(pg_data_t *pgdat);
 extern bool compaction_suitable(struct zone *zone, int order,
 				unsigned long watermark, int highest_zoneidx);

mm/compaction.c+14−36

 static enum compact_result compact_zone_order(struct zone *zone, int order,
 		gfp_t gfp_mask, enum compact_priority prio,
 		unsigned int alloc_flags, int highest_zoneidx,
-		struct page **capture)
+		struct capture_control *capc)
 {
-	enum compact_result ret;
 	struct compact_control cc = {
 		.order = order,
 		.search_order = order,
 		.ignore_skip_hint = (prio == MIN_COMPACT_PRIORITY),
 		.ignore_block_suitable = (prio == MIN_COMPACT_PRIORITY)
 	};
-	struct capture_control capc = {
-		.cc = &cc,
-		.page = NULL,
-	};
-	/*
-	 * Make sure the structs are really initialized before we expose the
-	 * capture control, in case we are interrupted and the interrupt handler
-	 * frees a page.
-	 */
-	barrier();
-	WRITE_ONCE(current->capture_control, &capc);
-	ret = compact_zone(&cc, &capc);
+	return compact_zone(&cc, capc);
-	/*
-	 * Make sure we hide capture control first before we read the captured
-	 * page pointer, otherwise an interrupt could free and capture a page
-	 * and we would leak it.
-	 */
-	WRITE_ONCE(current->capture_control, NULL);
-	*capture = READ_ONCE(capc.page);
-	/*
-	 * Technically, it is also possible that compaction is skipped but
-	 * the page is still captured out of luck(IRQ came and freed the page).
-	 * Returning COMPACT_SUCCESS in such cases helps in properly accounting
-	 * the COMPACT[STALL|FAIL] when compaction is skipped.
-	 */
-	if (*capture)
-		ret = COMPACT_SUCCESS;
-	return ret;
 }
 /**
  * @alloc_flags: The allocation flags of the current allocation
  * @ac: The context of current allocation
  * @prio: Determines how hard direct compaction should try to succeed
- * @capture: Pointer to free page created by compaction will be stored here
+ * @capc: Free page capture bypassing the freelist
  *
  * This is the main entry point for direct page compaction.
  */
 enum compact_result try_to_compact_pages(gfp_t gfp_mask, unsigned int order,
 		unsigned int alloc_flags, const struct alloc_context *ac,
-		enum compact_priority prio, struct page **capture)
+		enum compact_priority prio, struct capture_control *capc)
 {
 	struct zoneref *z;
 	struct zone *zone;
 			continue;
 		}
+		WRITE_ONCE(capc->zone, zone);
 		status = compact_zone_order(zone, order, gfp_mask, prio,
-				alloc_flags, ac->highest_zoneidx, capture);
+				alloc_flags, ac->highest_zoneidx, capc);
+		WRITE_ONCE(capc->zone, NULL);
+		/* Stop if a page has been captured */
+		if (READ_ONCE(capc->page))
+			status = COMPACT_SUCCESS;
 		rc = max(status, rc);
 		/* The allocation should succeed, stop compacting */

mm/internal.h+3−1

  * immediately when one is created during the free path.
  */
 struct capture_control {
-	struct compact_control *cc;
+	struct zone *zone;
+	int migratetype;
+	int order;
 	struct page *page;
 };

mm/page_alloc.c+38−7

 	return unlikely(capc) &&
 		!(current->flags & PF_KTHREAD) &&
 		!capc->page &&
-		capc->cc->zone == zone ? capc : NULL;
+		capc->zone == zone ? capc : NULL;
 }
 static inline bool
 compaction_capture(struct capture_control *capc, struct page *page,
 		   int order, int migratetype)
 {
-	if (!capc || order != capc->cc->order)
+	if (!capc || order != capc->order)
 		return false;
 	/* Do not accidentally pollute CMA or isolated regions*/
 	 * have trouble finding a high-order free page.
 	 */
 	if (order < pageblock_order && migratetype == MIGRATE_MOVABLE &&
-	    capc->cc->migratetype != MIGRATE_MOVABLE)
+	    capc->migratetype != MIGRATE_MOVABLE)
 		return false;
-	if (migratetype != capc->cc->migratetype)
+	if (migratetype != capc->migratetype)
-		trace_mm_page_alloc_extfrag(page, capc->cc->order, order,
+		trace_mm_page_alloc_extfrag(page, capc->order, order,
-					    capc->cc->migratetype, migratetype);
+					    capc->migratetype, migratetype);
 	capc->page = page;
 	return true;
 	struct page *page = NULL;
 	unsigned long pflags;
 	unsigned int noreclaim_flag;
+	struct capture_control capc = {
+		.zone = NULL,
+		.migratetype = ac->migratetype,
+		.order = order,
+		.page = NULL,
+	};
 	if (!order)
 		return NULL;
 	fs_reclaim_acquire(gfp_mask);
 	noreclaim_flag = memalloc_noreclaim_save();
+	/*
+	 * Make sure the structs are really initialized before we expose the
+	 * capture control, in case we are interrupted and the interrupt handler
+	 * frees a page.
+	 */
+	barrier();
+	WRITE_ONCE(current->capture_control, &capc);
 	*compact_result = try_to_compact_pages(gfp_mask, order, alloc_flags, ac,
-								prio, &page);
+							       prio, &capc);
+	/*
+	 * Make sure we hide capture control first before we read the captured
+	 * page pointer, otherwise an interrupt could free and capture a page
+	 * and we would leak it.
+	 */
+	WRITE_ONCE(current->capture_control, NULL);
+	page = READ_ONCE(capc.page);
+	/*
+	 * Technically, it is also possible that compaction is skipped but
+	 * the page is still captured out of luck(IRQ came and freed the page).
+	 * Returning COMPACT_SUCCESS in such cases helps in properly accounting
+	 * the COMPACT[STALL|FAIL] when compaction is skipped.
+	 */
+	if (page)
+		*compact_result = COMPACT_SUCCESS;
 	memalloc_noreclaim_restore(noreclaim_flag);
 	fs_reclaim_release(gfp_mask);

mm: page_alloc: move capture_control to the page allocator

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code