// SPDX-License-Identifier: GPL-2.0/* * VMware Balloon driver. * * Copyright (C) 2000-2018, VMware, Inc. All Rights Reserved. * * This is VMware physical memory management driver for Linux. The driver * acts like a "balloon" that can be inflated to reclaim physical pages by * reserving them in the guest and invalidating them in the monitor, * freeing up the underlying machine pages so they can be allocated to * other guests.  The balloon can also be deflated to allow the guest to * use more physical memory. Higher level policies can control the sizes * of balloons in VMs in order to manage physical memory resources. */ //#define DEBUG#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt #include <linux/types.h>#include <linux/io.h>#include <linux/kernel.h>#include <linux/mm.h>#include <linux/vmalloc.h>#include <linux/sched.h>#include <linux/module.h>#include <linux/workqueue.h>#include <linux/debugfs.h>#include <linux/seq_file.h>#include <linux/rwsem.h>#include <linux/slab.h>#include <linux/spinlock.h>#include <linux/balloon.h>#include <linux/vmw_vmci_defs.h>#include <linux/vmw_vmci_api.h>#include <asm/hypervisor.h> MODULE_AUTHOR("VMware, Inc.");MODULE_DESCRIPTION("VMware Memory Control (Balloon) Driver");MODULE_ALIAS("dmi:*:svnVMware*:*");MODULE_ALIAS("vmware_vmmemctl");MODULE_LICENSE("GPL"); static bool __read_mostly vmwballoon_shrinker_enable;module_param(vmwballoon_shrinker_enable, bool, 0444);MODULE_PARM_DESC(vmwballoon_shrinker_enable,	"Enable non-cooperative out-of-memory protection. Disabled by default as it may degrade performance."); /* Delay in seconds after shrink before inflation. */#define VMBALLOON_SHRINK_DELAY		(5) /* Maximum number of refused pages we accumulate during inflation cycle */#define VMW_BALLOON_MAX_REFUSED		16 /* Magic number for the balloon mount-point */#define BALLOON_VMW_MAGIC		0x0ba11007 /* * Hypervisor communication port definitions. */#define VMW_BALLOON_HV_PORT		0x5670#define VMW_BALLOON_HV_MAGIC		0x456c6d6f#define VMW_BALLOON_GUEST_ID		1	/* Linux */ enum vmwballoon_capabilities {	/*	 * Bit 0 is reserved and not associated to any capability.	 */	VMW_BALLOON_BASIC_CMDS			= (1 << 1),	VMW_BALLOON_BATCHED_CMDS		= (1 << 2),	VMW_BALLOON_BATCHED_2M_CMDS		= (1 << 3),	VMW_BALLOON_SIGNALLED_WAKEUP_CMD	= (1 << 4),	VMW_BALLOON_64_BIT_TARGET		= (1 << 5)}; #define VMW_BALLOON_CAPABILITIES_COMMON	(VMW_BALLOON_BASIC_CMDS \					| VMW_BALLOON_BATCHED_CMDS \					| VMW_BALLOON_BATCHED_2M_CMDS \					| VMW_BALLOON_SIGNALLED_WAKEUP_CMD) #define VMW_BALLOON_2M_ORDER		(PMD_SHIFT - PAGE_SHIFT) /* * 64-bit targets are only supported in 64-bit */#ifdef CONFIG_64BIT#define VMW_BALLOON_CAPABILITIES	(VMW_BALLOON_CAPABILITIES_COMMON \					| VMW_BALLOON_64_BIT_TARGET)#else#define VMW_BALLOON_CAPABILITIES	VMW_BALLOON_CAPABILITIES_COMMON#endif enum vmballoon_page_size_type {	VMW_BALLOON_4K_PAGE,	VMW_BALLOON_2M_PAGE,	VMW_BALLOON_LAST_SIZE = VMW_BALLOON_2M_PAGE}; #define VMW_BALLOON_NUM_PAGE_SIZES	(VMW_BALLOON_LAST_SIZE + 1) static const char * const vmballoon_page_size_names[] = {	[VMW_BALLOON_4K_PAGE]			= "4k",	[VMW_BALLOON_2M_PAGE]			= "2M"}; enum vmballoon_op {	VMW_BALLOON_INFLATE,	VMW_BALLOON_DEFLATE}; enum vmballoon_op_stat_type {	VMW_BALLOON_OP_STAT,	VMW_BALLOON_OP_FAIL_STAT}; #define VMW_BALLOON_OP_STAT_TYPES	(VMW_BALLOON_OP_FAIL_STAT + 1) /** * enum vmballoon_cmd_type - backdoor commands. * * Availability of the commands is as followed: * * %VMW_BALLOON_CMD_START, %VMW_BALLOON_CMD_GET_TARGET and * %VMW_BALLOON_CMD_GUEST_ID are always available. * * If the host reports %VMW_BALLOON_BASIC_CMDS are supported then * %VMW_BALLOON_CMD_LOCK and %VMW_BALLOON_CMD_UNLOCK commands are available. * * If the host reports %VMW_BALLOON_BATCHED_CMDS are supported then * %VMW_BALLOON_CMD_BATCHED_LOCK and VMW_BALLOON_CMD_BATCHED_UNLOCK commands * are available. * * If the host reports %VMW_BALLOON_BATCHED_2M_CMDS are supported then * %VMW_BALLOON_CMD_BATCHED_2M_LOCK and %VMW_BALLOON_CMD_BATCHED_2M_UNLOCK * are supported. * * If the host reports  VMW_BALLOON_SIGNALLED_WAKEUP_CMD is supported then * VMW_BALLOON_CMD_VMCI_DOORBELL_SET command is supported. * * @VMW_BALLOON_CMD_START: Communicating supported version with the hypervisor. * @VMW_BALLOON_CMD_GET_TARGET: Gets the balloon target size. * @VMW_BALLOON_CMD_LOCK: Informs the hypervisor about a ballooned page. * @VMW_BALLOON_CMD_UNLOCK: Informs the hypervisor about a page that is about *			    to be deflated from the balloon. * @VMW_BALLOON_CMD_GUEST_ID: Informs the hypervisor about the type of OS that *			      runs in the VM. * @VMW_BALLOON_CMD_BATCHED_LOCK: Inform the hypervisor about a batch of *				  ballooned pages (up to 512). * @VMW_BALLOON_CMD_BATCHED_UNLOCK: Inform the hypervisor about a batch of *				  pages that are about to be deflated from the *				  balloon (up to 512). * @VMW_BALLOON_CMD_BATCHED_2M_LOCK: Similar to @VMW_BALLOON_CMD_BATCHED_LOCK *				     for 2MB pages. * @VMW_BALLOON_CMD_BATCHED_2M_UNLOCK: Similar to *				       @VMW_BALLOON_CMD_BATCHED_UNLOCK for 2MB *				       pages. * @VMW_BALLOON_CMD_VMCI_DOORBELL_SET: A command to set doorbell notification *				       that would be invoked when the balloon *				       size changes. * @VMW_BALLOON_CMD_LAST: Value of the last command. */enum vmballoon_cmd_type {	VMW_BALLOON_CMD_START,	VMW_BALLOON_CMD_GET_TARGET,	VMW_BALLOON_CMD_LOCK,	VMW_BALLOON_CMD_UNLOCK,	VMW_BALLOON_CMD_GUEST_ID,	/* No command 5 */	VMW_BALLOON_CMD_BATCHED_LOCK = 6,	VMW_BALLOON_CMD_BATCHED_UNLOCK,	VMW_BALLOON_CMD_BATCHED_2M_LOCK,	VMW_BALLOON_CMD_BATCHED_2M_UNLOCK,	VMW_BALLOON_CMD_VMCI_DOORBELL_SET,	VMW_BALLOON_CMD_LAST = VMW_BALLOON_CMD_VMCI_DOORBELL_SET,}; #define VMW_BALLOON_CMD_NUM	(VMW_BALLOON_CMD_LAST + 1) enum vmballoon_error_codes {	VMW_BALLOON_SUCCESS,	VMW_BALLOON_ERROR_CMD_INVALID,	VMW_BALLOON_ERROR_PPN_INVALID,	VMW_BALLOON_ERROR_PPN_LOCKED,	VMW_BALLOON_ERROR_PPN_UNLOCKED,	VMW_BALLOON_ERROR_PPN_PINNED,	VMW_BALLOON_ERROR_PPN_NOTNEEDED,	VMW_BALLOON_ERROR_RESET,	VMW_BALLOON_ERROR_BUSY}; #define VMW_BALLOON_SUCCESS_WITH_CAPABILITIES	(0x03000000) #define VMW_BALLOON_CMD_WITH_TARGET_MASK			\	((1UL << VMW_BALLOON_CMD_GET_TARGET)		|	\	 (1UL << VMW_BALLOON_CMD_LOCK)			|	\	 (1UL << VMW_BALLOON_CMD_UNLOCK)		|	\	 (1UL << VMW_BALLOON_CMD_BATCHED_LOCK)		|	\	 (1UL << VMW_BALLOON_CMD_BATCHED_UNLOCK)	|	\	 (1UL << VMW_BALLOON_CMD_BATCHED_2M_LOCK)	|	\	 (1UL << VMW_BALLOON_CMD_BATCHED_2M_UNLOCK)) static const char * const vmballoon_cmd_names[] = {	[VMW_BALLOON_CMD_START]			= "start",	[VMW_BALLOON_CMD_GET_TARGET]		= "target",	[VMW_BALLOON_CMD_LOCK]			= "lock",	[VMW_BALLOON_CMD_UNLOCK]		= "unlock",	[VMW_BALLOON_CMD_GUEST_ID]		= "guestType",	[VMW_BALLOON_CMD_BATCHED_LOCK]		= "batchLock",	[VMW_BALLOON_CMD_BATCHED_UNLOCK]	= "batchUnlock",	[VMW_BALLOON_CMD_BATCHED_2M_LOCK]	= "2m-lock",	[VMW_BALLOON_CMD_BATCHED_2M_UNLOCK]	= "2m-unlock",	[VMW_BALLOON_CMD_VMCI_DOORBELL_SET]	= "doorbellSet"}; enum vmballoon_stat_page {	VMW_BALLOON_PAGE_STAT_ALLOC,	VMW_BALLOON_PAGE_STAT_ALLOC_FAIL,	VMW_BALLOON_PAGE_STAT_REFUSED_ALLOC,	VMW_BALLOON_PAGE_STAT_REFUSED_FREE,	VMW_BALLOON_PAGE_STAT_FREE,	VMW_BALLOON_PAGE_STAT_LAST = VMW_BALLOON_PAGE_STAT_FREE}; #define VMW_BALLOON_PAGE_STAT_NUM	(VMW_BALLOON_PAGE_STAT_LAST + 1) enum vmballoon_stat_general {	VMW_BALLOON_STAT_TIMER,	VMW_BALLOON_STAT_DOORBELL,	VMW_BALLOON_STAT_RESET,	VMW_BALLOON_STAT_SHRINK,	VMW_BALLOON_STAT_SHRINK_FREE,	VMW_BALLOON_STAT_LAST = VMW_BALLOON_STAT_SHRINK_FREE}; #define VMW_BALLOON_STAT_NUM		(VMW_BALLOON_STAT_LAST + 1) static DEFINE_STATIC_KEY_TRUE(vmw_balloon_batching);static DEFINE_STATIC_KEY_FALSE(balloon_stat_enabled); struct vmballoon_ctl {	struct list_head pages;	struct list_head refused_pages;	struct list_head prealloc_pages;	unsigned int n_refused_pages;	unsigned int n_pages;	enum vmballoon_page_size_type page_size;	enum vmballoon_op op;}; /** * struct vmballoon_batch_entry - a batch entry for lock or unlock. * * @status: the status of the operation, which is written by the hypervisor. * @reserved: reserved for future use. Must be set to zero. * @pfn: the physical frame number of the page to be locked or unlocked. */struct vmballoon_batch_entry {	u64 status : 5;	u64 reserved : PAGE_SHIFT - 5;	u64 pfn : 52;} __packed; struct vmballoon {	/**	 * @max_page_size: maximum supported page size for ballooning.	 *	 * Protected by @conf_sem	 */	enum vmballoon_page_size_type max_page_size; 	/**	 * @size: balloon actual size in basic page size (frames).	 *	 * While we currently do not support size which is bigger than 32-bit,	 * in preparation for future support, use 64-bits.	 */	atomic64_t size; 	/**	 * @target: balloon target size in basic page size (frames).	 *	 * We do not protect the target under the assumption that setting the	 * value is always done through a single write. If this assumption ever	 * breaks, we would have to use X_ONCE for accesses, and suffer the less	 * optimized code. Although we may read stale target value if multiple	 * accesses happen at once, the performance impact should be minor.	 */	unsigned long target; 	/**	 * @reset_required: reset flag	 *	 * Setting this flag may introduce races, but the code is expected to	 * handle them gracefully. In the worst case, another operation will	 * fail as reset did not take place. Clearing the flag is done while	 * holding @conf_sem for write.	 */	bool reset_required; 	/**	 * @capabilities: hypervisor balloon capabilities.	 *	 * Protected by @conf_sem.	 */	unsigned long capabilities; 	/**	 * @batch_page: pointer to communication batch page.	 *	 * When batching is used, batch_page points to a page, which holds up to	 * %VMW_BALLOON_BATCH_MAX_PAGES entries for locking or unlocking.	 */	struct vmballoon_batch_entry *batch_page; 	/**	 * @batch_max_pages: maximum pages that can be locked/unlocked.	 *	 * Indicates the number of pages that the hypervisor can lock or unlock	 * at once, according to whether batching is enabled. If batching is	 * disabled, only a single page can be locked/unlock on each operation.	 *	 * Protected by @conf_sem.	 */	unsigned int batch_max_pages; 	/**	 * @page: page to be locked/unlocked by the hypervisor	 *	 * @page is only used when batching is disabled and a single page is	 * reclaimed on each iteration.	 *	 * Protected by @comm_lock.	 */	struct page *page; 	/**	 * @shrink_timeout: timeout until the next inflation.	 *	 * After an shrink event, indicates the time in jiffies after which	 * inflation is allowed again. Can be written concurrently with reads,	 * so must use READ_ONCE/WRITE_ONCE when accessing.	 */	unsigned long shrink_timeout; 	/* statistics */	struct vmballoon_stats *stats; 	/**	 * @b_dev_info: balloon device information descriptor.	 */	struct balloon_dev_info b_dev_info; 	struct delayed_work dwork; 	/**	 * @huge_pages - list of the inflated 2MB pages.	 *	 * Protected by @huge_pages_lock.	 */	struct list_head huge_pages; 	/**	 * @huge_pages_lock: lock for the list of inflated 2MB pages.	 */	spinlock_t huge_pages_lock; 	/**	 * @vmci_doorbell.	 *	 * Protected by @conf_sem.	 */	struct vmci_handle vmci_doorbell; 	/**	 * @conf_sem: semaphore to protect the configuration and the statistics.	 */	struct rw_semaphore conf_sem; 	/**	 * @comm_lock: lock to protect the communication with the host.	 *	 * Lock ordering: @conf_sem -> @comm_lock .	 */	spinlock_t comm_lock; 	/**	 * @shrinker: shrinker interface that is used to avoid over-inflation.	 */	struct shrinker *shrinker;}; static struct vmballoon balloon; struct vmballoon_stats {	/* timer / doorbell operations */	atomic64_t general_stat[VMW_BALLOON_STAT_NUM]; 	/* allocation statistics for huge and small pages */	atomic64_t	       page_stat[VMW_BALLOON_PAGE_STAT_NUM][VMW_BALLOON_NUM_PAGE_SIZES]; 	/* Monitor operations: total operations, and failures */	atomic64_t ops[VMW_BALLOON_CMD_NUM][VMW_BALLOON_OP_STAT_TYPES];}; static inline bool is_vmballoon_stats_on(void){	return IS_ENABLED(CONFIG_DEBUG_FS) &&		static_branch_unlikely(&balloon_stat_enabled);} static inline void vmballoon_stats_op_inc(struct vmballoon *b, unsigned int op,					  enum vmballoon_op_stat_type type){	if (is_vmballoon_stats_on())		atomic64_inc(&b->stats->ops[op][type]);} static inline void vmballoon_stats_gen_inc(struct vmballoon *b,					   enum vmballoon_stat_general stat){	if (is_vmballoon_stats_on())		atomic64_inc(&b->stats->general_stat[stat]);} static inline void vmballoon_stats_gen_add(struct vmballoon *b,					   enum vmballoon_stat_general stat,					   unsigned int val){	if (is_vmballoon_stats_on())		atomic64_add(val, &b->stats->general_stat[stat]);} static inline void vmballoon_stats_page_inc(struct vmballoon *b,					    enum vmballoon_stat_page stat,					    enum vmballoon_page_size_type size){	if (is_vmballoon_stats_on())		atomic64_inc(&b->stats->page_stat[stat][size]);} static inline void vmballoon_stats_page_add(struct vmballoon *b,					    enum vmballoon_stat_page stat,					    enum vmballoon_page_size_type size,					    unsigned int val){	if (is_vmballoon_stats_on())		atomic64_add(val, &b->stats->page_stat[stat][size]);} static inline unsigned long__vmballoon_cmd(struct vmballoon *b, unsigned long cmd, unsigned long arg1,		unsigned long arg2, unsigned long *result){	unsigned long status, dummy1, dummy2, dummy3, local_result; 	vmballoon_stats_op_inc(b, cmd, VMW_BALLOON_OP_STAT); 	asm volatile ("inl %%dx" :		"=a"(status),		"=c"(dummy1),		"=d"(dummy2),		"=b"(local_result),		"=S"(dummy3) :		"0"(VMW_BALLOON_HV_MAGIC),		"1"(cmd),		"2"(VMW_BALLOON_HV_PORT),		"3"(arg1),		"4"(arg2) :		"memory"); 	/* update the result if needed */	if (result)		*result = (cmd == VMW_BALLOON_CMD_START) ? dummy1 :							   local_result; 	/* update target when applicable */	if (status == VMW_BALLOON_SUCCESS &&	    ((1ul << cmd) & VMW_BALLOON_CMD_WITH_TARGET_MASK))		WRITE_ONCE(b->target, local_result); 	if (status != VMW_BALLOON_SUCCESS &&	    status != VMW_BALLOON_SUCCESS_WITH_CAPABILITIES) {		vmballoon_stats_op_inc(b, cmd, VMW_BALLOON_OP_FAIL_STAT);		pr_debug("%s: %s [0x%lx,0x%lx) failed, returned %ld\n",			 __func__, vmballoon_cmd_names[cmd], arg1, arg2,			 status);	} 	/* mark reset required accordingly */	if (status == VMW_BALLOON_ERROR_RESET)		b->reset_required = true; 	return status;} static __always_inline unsigned longvmballoon_cmd(struct vmballoon *b, unsigned long cmd, unsigned long arg1,	      unsigned long arg2){	unsigned long dummy;

Lines 1–500 of 1873. Load more