/* *  Copyright (C) 2009  Red Hat, Inc. * *  This work is licensed under the terms of the GNU GPL, version 2. See *  the COPYING file in the top-level directory. */ #include <linux/mm.h>#include <linux/sched.h>#include <linux/highmem.h>#include <linux/hugetlb.h>#include <linux/mmu_notifier.h>#include <linux/rmap.h>#include <linux/swap.h>#include <linux/mm_inline.h>#include <linux/kthread.h>#include <linux/khugepaged.h>#include <linux/freezer.h>#include <linux/mman.h>#include <asm/tlb.h>#include <asm/pgalloc.h>#include "internal.h" /* * By default transparent hugepage support is enabled for all mappings * and khugepaged scans all mappings. Defrag is only invoked by * khugepaged hugepage allocations and by page faults inside * MADV_HUGEPAGE regions to avoid the risk of slowing down short lived * allocations. */unsigned long transparent_hugepage_flags __read_mostly =#ifdef CONFIG_TRANSPARENT_HUGEPAGE_ALWAYS	(1<<TRANSPARENT_HUGEPAGE_FLAG)|#endif#ifdef CONFIG_TRANSPARENT_HUGEPAGE_MADVISE	(1<<TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG)|#endif	(1<<TRANSPARENT_HUGEPAGE_DEFRAG_FLAG)|	(1<<TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG); /* default scan 8*512 pte (or vmas) every 30 second */static unsigned int khugepaged_pages_to_scan __read_mostly = HPAGE_PMD_NR*8;static unsigned int khugepaged_pages_collapsed;static unsigned int khugepaged_full_scans;static unsigned int khugepaged_scan_sleep_millisecs __read_mostly = 10000;/* during fragmentation poll the hugepage allocator once every minute */static unsigned int khugepaged_alloc_sleep_millisecs __read_mostly = 60000;static struct task_struct *khugepaged_thread __read_mostly;static DEFINE_MUTEX(khugepaged_mutex);static DEFINE_SPINLOCK(khugepaged_mm_lock);static DECLARE_WAIT_QUEUE_HEAD(khugepaged_wait);/* * default collapse hugepages if there is at least one pte mapped like * it would have happened if the vma was large enough during page * fault. */static unsigned int khugepaged_max_ptes_none __read_mostly = HPAGE_PMD_NR-1; static int khugepaged(void *none);static int mm_slots_hash_init(void);static int khugepaged_slab_init(void);static void khugepaged_slab_free(void); #define MM_SLOTS_HASH_HEADS 1024static struct hlist_head *mm_slots_hash __read_mostly;static struct kmem_cache *mm_slot_cache __read_mostly; /** * struct mm_slot - hash lookup from mm to mm_slot * @hash: hash collision list * @mm_node: khugepaged scan list headed in khugepaged_scan.mm_head * @mm: the mm that this information is valid for */struct mm_slot {	struct hlist_node hash;	struct list_head mm_node;	struct mm_struct *mm;}; /** * struct khugepaged_scan - cursor for scanning * @mm_head: the head of the mm list to scan * @mm_slot: the current mm_slot we are scanning * @address: the next address inside that to be scanned * * There is only the one khugepaged_scan instance of this cursor structure. */struct khugepaged_scan {	struct list_head mm_head;	struct mm_slot *mm_slot;	unsigned long address;};static struct khugepaged_scan khugepaged_scan = {	.mm_head = LIST_HEAD_INIT(khugepaged_scan.mm_head),};  static int set_recommended_min_free_kbytes(void){	struct zone *zone;	int nr_zones = 0;	unsigned long recommended_min;	extern int min_free_kbytes; 	if (!test_bit(TRANSPARENT_HUGEPAGE_FLAG,		      &transparent_hugepage_flags) &&	    !test_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG,		      &transparent_hugepage_flags))		return 0; 	for_each_populated_zone(zone)		nr_zones++; 	/* Make sure at least 2 hugepages are free for MIGRATE_RESERVE */	recommended_min = pageblock_nr_pages * nr_zones * 2; 	/*	 * Make sure that on average at least two pageblocks are almost free	 * of another type, one for a migratetype to fall back to and a	 * second to avoid subsequent fallbacks of other types There are 3	 * MIGRATE_TYPES we care about.	 */	recommended_min += pageblock_nr_pages * nr_zones *			   MIGRATE_PCPTYPES * MIGRATE_PCPTYPES; 	/* don't ever allow to reserve more than 5% of the lowmem */	recommended_min = min(recommended_min,			      (unsigned long) nr_free_buffer_pages() / 20);	recommended_min <<= (PAGE_SHIFT-10); 	if (recommended_min > min_free_kbytes)		min_free_kbytes = recommended_min;	setup_per_zone_wmarks();	return 0;}late_initcall(set_recommended_min_free_kbytes); static int start_khugepaged(void){	int err = 0;	if (khugepaged_enabled()) {		int wakeup;		if (unlikely(!mm_slot_cache || !mm_slots_hash)) {			err = -ENOMEM;			goto out;		}		mutex_lock(&khugepaged_mutex);		if (!khugepaged_thread)			khugepaged_thread = kthread_run(khugepaged, NULL,							"khugepaged");		if (unlikely(IS_ERR(khugepaged_thread))) {			printk(KERN_ERR			       "khugepaged: kthread_run(khugepaged) failed\n");			err = PTR_ERR(khugepaged_thread);			khugepaged_thread = NULL;		}		wakeup = !list_empty(&khugepaged_scan.mm_head);		mutex_unlock(&khugepaged_mutex);		if (wakeup)			wake_up_interruptible(&khugepaged_wait); 		set_recommended_min_free_kbytes();	} else		/* wakeup to exit */		wake_up_interruptible(&khugepaged_wait);out:	return err;} #ifdef CONFIG_SYSFS static ssize_t double_flag_show(struct kobject *kobj,				struct kobj_attribute *attr, char *buf,				enum transparent_hugepage_flag enabled,				enum transparent_hugepage_flag req_madv){	if (test_bit(enabled, &transparent_hugepage_flags)) {		VM_BUG_ON(test_bit(req_madv, &transparent_hugepage_flags));		return sprintf(buf, "[always] madvise never\n");	} else if (test_bit(req_madv, &transparent_hugepage_flags))		return sprintf(buf, "always [madvise] never\n");	else		return sprintf(buf, "always madvise [never]\n");}static ssize_t double_flag_store(struct kobject *kobj,				 struct kobj_attribute *attr,				 const char *buf, size_t count,				 enum transparent_hugepage_flag enabled,				 enum transparent_hugepage_flag req_madv){	if (!memcmp("always", buf,		    min(sizeof("always")-1, count))) {		set_bit(enabled, &transparent_hugepage_flags);		clear_bit(req_madv, &transparent_hugepage_flags);	} else if (!memcmp("madvise", buf,			   min(sizeof("madvise")-1, count))) {		clear_bit(enabled, &transparent_hugepage_flags);		set_bit(req_madv, &transparent_hugepage_flags);	} else if (!memcmp("never", buf,			   min(sizeof("never")-1, count))) {		clear_bit(enabled, &transparent_hugepage_flags);		clear_bit(req_madv, &transparent_hugepage_flags);	} else		return -EINVAL; 	return count;} static ssize_t enabled_show(struct kobject *kobj,			    struct kobj_attribute *attr, char *buf){	return double_flag_show(kobj, attr, buf,				TRANSPARENT_HUGEPAGE_FLAG,				TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG);}static ssize_t enabled_store(struct kobject *kobj,			     struct kobj_attribute *attr,			     const char *buf, size_t count){	ssize_t ret; 	ret = double_flag_store(kobj, attr, buf, count,				TRANSPARENT_HUGEPAGE_FLAG,				TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG); 	if (ret > 0) {		int err = start_khugepaged();		if (err)			ret = err;	} 	if (ret > 0 &&	    (test_bit(TRANSPARENT_HUGEPAGE_FLAG,		      &transparent_hugepage_flags) ||	     test_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG,		      &transparent_hugepage_flags)))		set_recommended_min_free_kbytes(); 	return ret;}static struct kobj_attribute enabled_attr =	__ATTR(enabled, 0644, enabled_show, enabled_store); static ssize_t single_flag_show(struct kobject *kobj,				struct kobj_attribute *attr, char *buf,				enum transparent_hugepage_flag flag){	return sprintf(buf, "%d\n",		       !!test_bit(flag, &transparent_hugepage_flags));} static ssize_t single_flag_store(struct kobject *kobj,				 struct kobj_attribute *attr,				 const char *buf, size_t count,				 enum transparent_hugepage_flag flag){	unsigned long value;	int ret; 	ret = kstrtoul(buf, 10, &value);	if (ret < 0)		return ret;	if (value > 1)		return -EINVAL; 	if (value)		set_bit(flag, &transparent_hugepage_flags);	else		clear_bit(flag, &transparent_hugepage_flags); 	return count;} /* * Currently defrag only disables __GFP_NOWAIT for allocation. A blind * __GFP_REPEAT is too aggressive, it's never worth swapping tons of * memory just to allocate one more hugepage. */static ssize_t defrag_show(struct kobject *kobj,			   struct kobj_attribute *attr, char *buf){	return double_flag_show(kobj, attr, buf,				TRANSPARENT_HUGEPAGE_DEFRAG_FLAG,				TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG);}static ssize_t defrag_store(struct kobject *kobj,			    struct kobj_attribute *attr,			    const char *buf, size_t count){	return double_flag_store(kobj, attr, buf, count,				 TRANSPARENT_HUGEPAGE_DEFRAG_FLAG,				 TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG);}static struct kobj_attribute defrag_attr =	__ATTR(defrag, 0644, defrag_show, defrag_store); #ifdef CONFIG_DEBUG_VMstatic ssize_t debug_cow_show(struct kobject *kobj,				struct kobj_attribute *attr, char *buf){	return single_flag_show(kobj, attr, buf,				TRANSPARENT_HUGEPAGE_DEBUG_COW_FLAG);}static ssize_t debug_cow_store(struct kobject *kobj,			       struct kobj_attribute *attr,			       const char *buf, size_t count){	return single_flag_store(kobj, attr, buf, count,				 TRANSPARENT_HUGEPAGE_DEBUG_COW_FLAG);}static struct kobj_attribute debug_cow_attr =	__ATTR(debug_cow, 0644, debug_cow_show, debug_cow_store);#endif /* CONFIG_DEBUG_VM */ static struct attribute *hugepage_attr[] = {	&enabled_attr.attr,	&defrag_attr.attr,#ifdef CONFIG_DEBUG_VM	&debug_cow_attr.attr,#endif	NULL,}; static struct attribute_group hugepage_attr_group = {	.attrs = hugepage_attr,}; static ssize_t scan_sleep_millisecs_show(struct kobject *kobj,					 struct kobj_attribute *attr,					 char *buf){	return sprintf(buf, "%u\n", khugepaged_scan_sleep_millisecs);} static ssize_t scan_sleep_millisecs_store(struct kobject *kobj,					  struct kobj_attribute *attr,					  const char *buf, size_t count){	unsigned long msecs;	int err; 	err = strict_strtoul(buf, 10, &msecs);	if (err || msecs > UINT_MAX)		return -EINVAL; 	khugepaged_scan_sleep_millisecs = msecs;	wake_up_interruptible(&khugepaged_wait); 	return count;}static struct kobj_attribute scan_sleep_millisecs_attr =	__ATTR(scan_sleep_millisecs, 0644, scan_sleep_millisecs_show,	       scan_sleep_millisecs_store); static ssize_t alloc_sleep_millisecs_show(struct kobject *kobj,					  struct kobj_attribute *attr,					  char *buf){	return sprintf(buf, "%u\n", khugepaged_alloc_sleep_millisecs);} static ssize_t alloc_sleep_millisecs_store(struct kobject *kobj,					   struct kobj_attribute *attr,					   const char *buf, size_t count){	unsigned long msecs;	int err; 	err = strict_strtoul(buf, 10, &msecs);	if (err || msecs > UINT_MAX)		return -EINVAL; 	khugepaged_alloc_sleep_millisecs = msecs;	wake_up_interruptible(&khugepaged_wait); 	return count;}static struct kobj_attribute alloc_sleep_millisecs_attr =	__ATTR(alloc_sleep_millisecs, 0644, alloc_sleep_millisecs_show,	       alloc_sleep_millisecs_store); static ssize_t pages_to_scan_show(struct kobject *kobj,				  struct kobj_attribute *attr,				  char *buf){	return sprintf(buf, "%u\n", khugepaged_pages_to_scan);}static ssize_t pages_to_scan_store(struct kobject *kobj,				   struct kobj_attribute *attr,				   const char *buf, size_t count){	int err;	unsigned long pages; 	err = strict_strtoul(buf, 10, &pages);	if (err || !pages || pages > UINT_MAX)		return -EINVAL; 	khugepaged_pages_to_scan = pages; 	return count;}static struct kobj_attribute pages_to_scan_attr =	__ATTR(pages_to_scan, 0644, pages_to_scan_show,	       pages_to_scan_store); static ssize_t pages_collapsed_show(struct kobject *kobj,				    struct kobj_attribute *attr,				    char *buf){	return sprintf(buf, "%u\n", khugepaged_pages_collapsed);}static struct kobj_attribute pages_collapsed_attr =	__ATTR_RO(pages_collapsed); static ssize_t full_scans_show(struct kobject *kobj,			       struct kobj_attribute *attr,			       char *buf){	return sprintf(buf, "%u\n", khugepaged_full_scans);}static struct kobj_attribute full_scans_attr =	__ATTR_RO(full_scans); static ssize_t khugepaged_defrag_show(struct kobject *kobj,				      struct kobj_attribute *attr, char *buf){	return single_flag_show(kobj, attr, buf,				TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG);}static ssize_t khugepaged_defrag_store(struct kobject *kobj,				       struct kobj_attribute *attr,				       const char *buf, size_t count){	return single_flag_store(kobj, attr, buf, count,				 TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG);}static struct kobj_attribute khugepaged_defrag_attr =	__ATTR(defrag, 0644, khugepaged_defrag_show,	       khugepaged_defrag_store); /* * max_ptes_none controls if khugepaged should collapse hugepages over * any unmapped ptes in turn potentially increasing the memory * footprint of the vmas. When max_ptes_none is 0 khugepaged will not * reduce the available free memory in the system as it * runs. Increasing max_ptes_none will instead potentially reduce the * free memory in the system during the khugepaged scan. */static ssize_t khugepaged_max_ptes_none_show(struct kobject *kobj,					     struct kobj_attribute *attr,					     char *buf){	return sprintf(buf, "%u\n", khugepaged_max_ptes_none);}static ssize_t khugepaged_max_ptes_none_store(struct kobject *kobj,					      struct kobj_attribute *attr,					      const char *buf, size_t count){	int err;	unsigned long max_ptes_none; 	err = strict_strtoul(buf, 10, &max_ptes_none);	if (err || max_ptes_none > HPAGE_PMD_NR-1)		return -EINVAL; 	khugepaged_max_ptes_none = max_ptes_none; 	return count;}static struct kobj_attribute khugepaged_max_ptes_none_attr =	__ATTR(max_ptes_none, 0644, khugepaged_max_ptes_none_show,	       khugepaged_max_ptes_none_store); static struct attribute *khugepaged_attr[] = {	&khugepaged_defrag_attr.attr,	&khugepaged_max_ptes_none_attr.attr,	&pages_to_scan_attr.attr,	&pages_collapsed_attr.attr,	&full_scans_attr.attr,	&scan_sleep_millisecs_attr.attr,	&alloc_sleep_millisecs_attr.attr,	NULL,}; static struct attribute_group khugepaged_attr_group = {	.attrs = khugepaged_attr,	.name = "khugepaged",}; static int __init hugepage_init_sysfs(struct kobject **hugepage_kobj){	int err; 	*hugepage_kobj = kobject_create_and_add("transparent_hugepage", mm_kobj);	if (unlikely(!*hugepage_kobj)) {		printk(KERN_ERR "hugepage: failed kobject create\n");		return -ENOMEM;	} 

Lines 1–500 of 2465. Load more