// SPDX-License-Identifier: GPL-2.0-only/* * Common interface for implementing a memory balloon, including support * for migration of pages inflated in a memory balloon. * * Copyright (C) 2012, Red Hat, Inc.  Rafael Aquini <aquini@redhat.com> */#include <linux/mm.h>#include <linux/slab.h>#include <linux/export.h>#include <linux/balloon.h> /* * Lock protecting the balloon_dev_info of all devices. We don't really * expect more than one device. */static DEFINE_SPINLOCK(balloon_pages_lock); /** * balloon_page_insert - insert a page into the balloon's page list and make *			 the page->private assignment accordingly. * @balloon : pointer to balloon device * @page    : page to be assigned as a 'balloon page' * * Caller must ensure the balloon_pages_lock is held. */static void balloon_page_insert(struct balloon_dev_info *balloon,				       struct page *page){	lockdep_assert_held(&balloon_pages_lock);	__SetPageOffline(page);	if (IS_ENABLED(CONFIG_BALLOON_MIGRATION)) {		SetPageMovableOps(page);		set_page_private(page, (unsigned long)balloon);	}	list_add(&page->lru, &balloon->pages);} /** * balloon_page_finalize - prepare a balloon page that was removed from the *			   balloon list for release to the page allocator * @page: page to be released to the page allocator * * Caller must ensure the balloon_pages_lock is held. */static void balloon_page_finalize(struct page *page){	lockdep_assert_held(&balloon_pages_lock);	if (IS_ENABLED(CONFIG_BALLOON_MIGRATION))		set_page_private(page, 0);	/* PageOffline is sticky until the page is freed to the buddy. */} static void balloon_page_enqueue_one(struct balloon_dev_info *b_dev_info,				     struct page *page){	balloon_page_insert(b_dev_info, page);	if (b_dev_info->adjust_managed_page_count)		adjust_managed_page_count(page, -1);	__count_vm_event(BALLOON_INFLATE);	inc_node_page_state(page, NR_BALLOON_PAGES);} /** * balloon_page_list_enqueue() - inserts a list of pages into the balloon page *				 list. * @b_dev_info: balloon device descriptor where we will insert a new page to * @pages: pages to enqueue - allocated using balloon_page_alloc. * * Driver must call this function to properly enqueue balloon pages before * definitively removing them from the guest system. * * Return: number of pages that were enqueued. */size_t balloon_page_list_enqueue(struct balloon_dev_info *b_dev_info,				 struct list_head *pages){	struct page *page, *tmp;	unsigned long flags;	size_t n_pages = 0; 	spin_lock_irqsave(&balloon_pages_lock, flags);	list_for_each_entry_safe(page, tmp, pages, lru) {		list_del(&page->lru);		balloon_page_enqueue_one(b_dev_info, page);		n_pages++;	}	spin_unlock_irqrestore(&balloon_pages_lock, flags);	return n_pages;}EXPORT_SYMBOL_GPL(balloon_page_list_enqueue); /** * balloon_page_list_dequeue() - removes pages from balloon's page list and *				 returns a list of the pages. * @b_dev_info: balloon device descriptor where we will grab a page from. * @pages: pointer to the list of pages that would be returned to the caller. * @n_req_pages: number of requested pages. * * Driver must call this function to properly de-allocate a previous enlisted * balloon pages before definitively releasing it back to the guest system. * This function tries to remove @n_req_pages from the ballooned pages and * return them to the caller in the @pages list. * * Note that this function may fail to dequeue some pages even if the balloon * isn't empty - since the page list can be temporarily empty due to compaction * of isolated pages. * * Return: number of pages that were added to the @pages list. */size_t balloon_page_list_dequeue(struct balloon_dev_info *b_dev_info,				 struct list_head *pages, size_t n_req_pages){	struct page *page, *tmp;	unsigned long flags;	size_t n_pages = 0; 	spin_lock_irqsave(&balloon_pages_lock, flags);	list_for_each_entry_safe(page, tmp, &b_dev_info->pages, lru) {		if (n_pages == n_req_pages)			break;		list_del(&page->lru);		if (b_dev_info->adjust_managed_page_count)			adjust_managed_page_count(page, 1);		balloon_page_finalize(page);		__count_vm_event(BALLOON_DEFLATE);		list_add(&page->lru, pages);		dec_node_page_state(page, NR_BALLOON_PAGES);		n_pages++;	}	spin_unlock_irqrestore(&balloon_pages_lock, flags); 	return n_pages;}EXPORT_SYMBOL_GPL(balloon_page_list_dequeue); /** * balloon_page_alloc - allocates a new page for insertion into the balloon *			page list. * * Driver must call this function to properly allocate a new balloon page. * Driver must call balloon_page_enqueue before definitively removing the page * from the guest system. * * Return: struct page for the allocated page or NULL on allocation failure. */struct page *balloon_page_alloc(void){	gfp_t gfp_flags = __GFP_NOMEMALLOC | __GFP_NORETRY | __GFP_NOWARN; 	if (IS_ENABLED(CONFIG_BALLOON_MIGRATION))		gfp_flags |= GFP_HIGHUSER_MOVABLE;	else		gfp_flags |= GFP_HIGHUSER; 	return alloc_page(gfp_flags);}EXPORT_SYMBOL_GPL(balloon_page_alloc); /** * balloon_page_enqueue - inserts a new page into the balloon page list. * * @b_dev_info: balloon device descriptor where we will insert a new page * @page: new page to enqueue - allocated using balloon_page_alloc. * * Drivers must call this function to properly enqueue a new allocated balloon * page before definitively removing the page from the guest system. * * Drivers must not enqueue pages while page->lru is still in * use, and must not use page->lru until a page was unqueued again. */void balloon_page_enqueue(struct balloon_dev_info *b_dev_info,			  struct page *page){	unsigned long flags; 	spin_lock_irqsave(&balloon_pages_lock, flags);	balloon_page_enqueue_one(b_dev_info, page);	spin_unlock_irqrestore(&balloon_pages_lock, flags);}EXPORT_SYMBOL_GPL(balloon_page_enqueue); /** * balloon_page_dequeue - removes a page from balloon's page list and returns *			  its address to allow the driver to release the page. * @b_dev_info: balloon device descriptor where we will grab a page from. * * Driver must call this function to properly dequeue a previously enqueued page * before definitively releasing it back to the guest system. * * Caller must perform its own accounting to ensure that this * function is called only if some pages are actually enqueued. * * Note that this function may fail to dequeue some pages even if there are * some enqueued pages - since the page list can be temporarily empty due to * the compaction of isolated pages. * * TODO: remove the caller accounting requirements, and allow caller to wait * until all pages can be dequeued. * * Return: struct page for the dequeued page, or NULL if no page was dequeued. */struct page *balloon_page_dequeue(struct balloon_dev_info *b_dev_info){	unsigned long flags;	LIST_HEAD(pages);	int n_pages; 	n_pages = balloon_page_list_dequeue(b_dev_info, &pages, 1); 	if (n_pages != 1) {		/*		 * If we are unable to dequeue a balloon page because the page		 * list is empty and there are no isolated pages, then something		 * went out of track and some balloon pages are lost.		 * BUG() here, otherwise the balloon driver may get stuck in		 * an infinite loop while attempting to release all its pages.		 */		spin_lock_irqsave(&balloon_pages_lock, flags);		if (unlikely(list_empty(&b_dev_info->pages) &&			     !b_dev_info->isolated_pages))			BUG();		spin_unlock_irqrestore(&balloon_pages_lock, flags);		return NULL;	}	return list_first_entry(&pages, struct page, lru);}EXPORT_SYMBOL_GPL(balloon_page_dequeue); #ifdef CONFIG_BALLOON_MIGRATIONstatic struct balloon_dev_info *balloon_page_device(struct page *page){	return (struct balloon_dev_info *)page_private(page);} static bool balloon_page_isolate(struct page *page, isolate_mode_t mode) {	struct balloon_dev_info *b_dev_info;	unsigned long flags; 	spin_lock_irqsave(&balloon_pages_lock, flags);	b_dev_info = balloon_page_device(page);	if (!b_dev_info) {		/*		 * The page already got deflated and removed from the		 * balloon list.		 */		spin_unlock_irqrestore(&balloon_pages_lock, flags);		return false;	}	list_del(&page->lru);	b_dev_info->isolated_pages++;	spin_unlock_irqrestore(&balloon_pages_lock, flags); 	return true;} static void balloon_page_putback(struct page *page){	struct balloon_dev_info *b_dev_info = balloon_page_device(page);	unsigned long flags; 	/*	 * When we isolated the page, the page was still inflated in a balloon	 * device. As isolated balloon pages cannot get deflated, we still have	 * a balloon device here.	 */	if (WARN_ON_ONCE(!b_dev_info))		return; 	spin_lock_irqsave(&balloon_pages_lock, flags);	list_add(&page->lru, &b_dev_info->pages);	b_dev_info->isolated_pages--;	spin_unlock_irqrestore(&balloon_pages_lock, flags);} static int balloon_page_migrate(struct page *newpage, struct page *page,		enum migrate_mode mode){	struct balloon_dev_info *b_dev_info = balloon_page_device(page);	unsigned long flags;	int rc; 	/*	 * When we isolated the page, the page was still inflated in a balloon	 * device. As isolated balloon pages cannot get deflated, we still have	 * a balloon device here.	 */	if (WARN_ON_ONCE(!b_dev_info))		return -EAGAIN; 	rc = b_dev_info->migratepage(b_dev_info, newpage, page, mode);	if (rc < 0 && rc != -ENOENT)		return rc; 	spin_lock_irqsave(&balloon_pages_lock, flags);	if (!rc) {		/* Insert the new page into the balloon list. */		get_page(newpage);		balloon_page_insert(b_dev_info, newpage);		__count_vm_event(BALLOON_MIGRATE); 		if (b_dev_info->adjust_managed_page_count &&		    page_zone(page) != page_zone(newpage)) {			/*			 * When we migrate a page to a different zone we			 * have to fixup the count of both involved zones.			 */			adjust_managed_page_count(page, 1);			adjust_managed_page_count(newpage, -1);		}	} else {		/* Old page was deflated but new page not inflated. */		__count_vm_event(BALLOON_DEFLATE); 		if (b_dev_info->adjust_managed_page_count)			adjust_managed_page_count(page, 1);	} 	b_dev_info->isolated_pages--; 	/* Free the now-deflated page we isolated in balloon_page_isolate(). */	balloon_page_finalize(page);	spin_unlock_irqrestore(&balloon_pages_lock, flags); 	put_page(page); 	return 0;} static const struct movable_operations balloon_mops = {	.migrate_page = balloon_page_migrate,	.isolate_page = balloon_page_isolate,	.putback_page = balloon_page_putback,}; static int __init balloon_init(void){	return set_movable_ops(&balloon_mops, PGTY_offline);}core_initcall(balloon_init); #endif /* CONFIG_BALLOON_MIGRATION */