lael-labs/linux

drivers/base/devres.c+23−1

@@ -755,10 +755,32 @@ void devm_remove_action(struct device dev, void (action)(void ), void data) · 754 unmodified lines
755		755
756	WARN_ON(devres_destroy(dev, devm_action_release, devm_action_match,	756	WARN_ON(devres_destroy(dev, devm_action_release, devm_action_match,
757	&devres));	757	&devres));
758
759	}	758	}
760	EXPORT_SYMBOL_GPL(devm_remove_action);	759	EXPORT_SYMBOL_GPL(devm_remove_action);
761		760
		761	/**
		762	* devm_release_action() - release previously added custom action
		763	* @dev: Device that owns the action
		764	* @action: Function implementing the action
		765	* @data: Pointer to data passed to @action implementation
		766	*
		767	* Releases and removes instance of @action previously added by
		768	* devm_add_action(). Both action and data should match one of the
		769	* existing entries.
		770	*/
		771	void devm_release_action(struct device dev, void (action)(void ), void data)
		772	{
		773	struct action_devres devres = {
		774	.data = data,
		775	.action = action,
		776	};
		777
		778	WARN_ON(devres_release(dev, devm_action_release, devm_action_match,
		779	&devres));
		780
		781	}
		782	EXPORT_SYMBOL_GPL(devm_release_action);
		783
762	/*	784	/*
763	* Managed kmalloc/kfree	785	* Managed kmalloc/kfree
764	*/	786	*/

drivers/dax/device.c+3−10

 	complete(&dev_dax->cmp);
 }
-static void dev_dax_percpu_exit(void *data)
+static void dev_dax_percpu_exit(struct percpu_ref *ref)
 {
-	struct percpu_ref *ref = data;
 	struct dev_dax *dev_dax = ref_to_dev_dax(ref);
 	dev_dbg(&dev_dax->dev, "%s\n", __func__);
 	if (rc)
 		return rc;
-	rc = devm_add_action_or_reset(dev, dev_dax_percpu_exit, &dev_dax->ref);
-	if (rc)
-		return rc;
 	dev_dax->pgmap.ref = &dev_dax->ref;
 	dev_dax->pgmap.kill = dev_dax_percpu_kill;
+	dev_dax->pgmap.cleanup = dev_dax_percpu_exit;
 	addr = devm_memremap_pages(dev, &dev_dax->pgmap);
-	if (IS_ERR(addr)) {
+	if (IS_ERR(addr))
-		devm_remove_action(dev, dev_dax_percpu_exit, &dev_dax->ref);
-		percpu_ref_exit(&dev_dax->ref);
 		return PTR_ERR(addr);
-	}
 	inode = dax_inode(dax_dev);
 	cdev = inode->i_cdev;

drivers/nvdimm/pmem.c+13−4

 	NULL,
 };
-static void pmem_release_queue(void *q)
+static void __pmem_release_queue(struct percpu_ref *ref)
 {
+	struct request_queue *q;
+	q = container_of(ref, typeof(*q), q_usage_counter);
 	blk_cleanup_queue(q);
 }
+static void pmem_release_queue(void *ref)
+{
+	__pmem_release_queue(ref);
+}
 static void pmem_freeze_queue(struct percpu_ref *ref)
 {
 	struct request_queue *q;
 	if (!q)
 		return -ENOMEM;
-	if (devm_add_action_or_reset(dev, pmem_release_queue, q))
-		return -ENOMEM;
 	pmem->pfn_flags = PFN_DEV;
 	pmem->pgmap.ref = &q->q_usage_counter;
 	pmem->pgmap.kill = pmem_freeze_queue;
+	pmem->pgmap.cleanup = __pmem_release_queue;
 	if (is_nd_pfn(dev)) {
 		if (setup_pagemap_fsdax(dev, &pmem->pgmap))
 			return -ENOMEM;
 		pmem->pfn_flags |= PFN_MAP;
 		memcpy(&bb_res, &pmem->pgmap.res, sizeof(bb_res));
 	} else {
+		if (devm_add_action_or_reset(dev, pmem_release_queue,
+					&q->q_usage_counter))
+			return -ENOMEM;
 		addr = devm_memremap(dev, pmem->phys_addr,
 				pmem->size, ARCH_MEMREMAP_PMEM);
 		memcpy(&bb_res, &nsio->res, sizeof(bb_res));

drivers/pci/p2pdma.c+72−43

@@ -20,12 +20,16 @@ · 19 unmodified lines
20	#include <linux/seq_buf.h>	20	#include <linux/seq_buf.h>
21		21
22	struct pci_p2pdma {	22	struct pci_p2pdma {
23	struct percpu_ref devmap_ref;
24	struct completion devmap_ref_done;
25	struct gen_pool *pool;	23	struct gen_pool *pool;
26	bool p2pmem_published;	24	bool p2pmem_published;
27	};	25	};
28		26
		27	struct p2pdma_pagemap {
		28	struct dev_pagemap pgmap;
		29	struct percpu_ref ref;
		30	struct completion ref_done;
		31	};
		32
29	static ssize_t size_show(struct device dev, struct device_attribute attr,	33	static ssize_t size_show(struct device dev, struct device_attribute attr,
30	char *buf)	34	char *buf)
31	{	35	{
@@ -74,41 +78,45 @@ static const struct attribute_group p2pmem_group = { · 42 unmodified lines
74	.name = "p2pmem",	78	.name = "p2pmem",
75	};	79	};
76		80
		81	static struct p2pdma_pagemap to_p2p_pgmap(struct percpu_ref ref)
		82	{
		83	return container_of(ref, struct p2pdma_pagemap, ref);
		84	}
		85
77	static void pci_p2pdma_percpu_release(struct percpu_ref *ref)	86	static void pci_p2pdma_percpu_release(struct percpu_ref *ref)
78	{	87	{
79	struct pci_p2pdma *p2p =	88	struct p2pdma_pagemap *p2p_pgmap = to_p2p_pgmap(ref);
80	container_of(ref, struct pci_p2pdma, devmap_ref);
81		89
82	complete_all(&p2p->devmap_ref_done);	90	complete(&p2p_pgmap->ref_done);
83	}	91	}
84		92
85	static void pci_p2pdma_percpu_kill(struct percpu_ref *ref)	93	static void pci_p2pdma_percpu_kill(struct percpu_ref *ref)
86	{	94	{
87	/*
88	* pci_p2pdma_add_resource() may be called multiple times
89	* by a driver and may register the percpu_kill devm action multiple
90	* times. We only want the first action to actually kill the
91	* percpu_ref.
92	*/
93	if (percpu_ref_is_dying(ref))
94	return;
95
96	percpu_ref_kill(ref);	95	percpu_ref_kill(ref);
97	}	96	}
98		97
		98	static void pci_p2pdma_percpu_cleanup(struct percpu_ref *ref)
		99	{
		100	struct p2pdma_pagemap *p2p_pgmap = to_p2p_pgmap(ref);
		101
		102	wait_for_completion(&p2p_pgmap->ref_done);
		103	percpu_ref_exit(&p2p_pgmap->ref);
		104	}
		105
99	static void pci_p2pdma_release(void *data)	106	static void pci_p2pdma_release(void *data)
100	{	107	{
101	struct pci_dev *pdev = data;	108	struct pci_dev *pdev = data;
		109	struct pci_p2pdma *p2pdma = pdev->p2pdma;
102		110
103	if (!pdev->p2pdma)	111	if (!p2pdma)
104	return;	112	return;
105		113
106	wait_for_completion(&pdev->p2pdma->devmap_ref_done);	114	/* Flush and disable pci_alloc_p2p_mem() */
107	percpu_ref_exit(&pdev->p2pdma->devmap_ref);	115	pdev->p2pdma = NULL;
		116	synchronize_rcu();
108		117
109	gen_pool_destroy(pdev->p2pdma->pool);	118	gen_pool_destroy(p2pdma->pool);
110	sysfs_remove_group(&pdev->dev.kobj, &p2pmem_group);	119	sysfs_remove_group(&pdev->dev.kobj, &p2pmem_group);
111	pdev->p2pdma = NULL;
112	}	120	}
113		121
114	static int pci_p2pdma_setup(struct pci_dev *pdev)	122	static int pci_p2pdma_setup(struct pci_dev *pdev)
@@ -124,12 +132,6 @@ static int pci_p2pdma_setup(struct pci_dev *pdev) · 9 unmodified lines
124	if (!p2p->pool)	132	if (!p2p->pool)
125	goto out;	133	goto out;
126		134
127	init_completion(&p2p->devmap_ref_done);
128	error = percpu_ref_init(&p2p->devmap_ref,
129	pci_p2pdma_percpu_release, 0, GFP_KERNEL);
130	if (error)
131	goto out_pool_destroy;
132
133	error = devm_add_action_or_reset(&pdev->dev, pci_p2pdma_release, pdev);	135	error = devm_add_action_or_reset(&pdev->dev, pci_p2pdma_release, pdev);
134	if (error)	136	if (error)
135	goto out_pool_destroy;	137	goto out_pool_destroy;
@@ -163,6 +165,7 @@ out: · 27 unmodified lines
163	int pci_p2pdma_add_resource(struct pci_dev *pdev, int bar, size_t size,	165	int pci_p2pdma_add_resource(struct pci_dev *pdev, int bar, size_t size,
164	u64 offset)	166	u64 offset)
165	{	167	{
		168	struct p2pdma_pagemap *p2p_pgmap;
166	struct dev_pagemap *pgmap;	169	struct dev_pagemap *pgmap;
167	void *addr;	170	void *addr;
168	int error;	171	int error;
@@ -185,18 +188,27 @@ int pci_p2pdma_add_resource(struct pci_dev *pdev, int bar, size_t size, · 16 unmodified lines
185	return error;	188	return error;
186	}	189	}
187		190
188	pgmap = devm_kzalloc(&pdev->dev, sizeof(*pgmap), GFP_KERNEL);	191	p2p_pgmap = devm_kzalloc(&pdev->dev, sizeof(*p2p_pgmap), GFP_KERNEL);
189	if (!pgmap)	192	if (!p2p_pgmap)
190	return -ENOMEM;	193	return -ENOMEM;
191		194
		195	init_completion(&p2p_pgmap->ref_done);
		196	error = percpu_ref_init(&p2p_pgmap->ref,
		197	pci_p2pdma_percpu_release, 0, GFP_KERNEL);
		198	if (error)
		199	goto pgmap_free;
		200
		201	pgmap = &p2p_pgmap->pgmap;
		202
192	pgmap->res.start = pci_resource_start(pdev, bar) + offset;	203	pgmap->res.start = pci_resource_start(pdev, bar) + offset;
193	pgmap->res.end = pgmap->res.start + size - 1;	204	pgmap->res.end = pgmap->res.start + size - 1;
194	pgmap->res.flags = pci_resource_flags(pdev, bar);	205	pgmap->res.flags = pci_resource_flags(pdev, bar);
195	pgmap->ref = &pdev->p2pdma->devmap_ref;	206	pgmap->ref = &p2p_pgmap->ref;
196	pgmap->type = MEMORY_DEVICE_PCI_P2PDMA;	207	pgmap->type = MEMORY_DEVICE_PCI_P2PDMA;
197	pgmap->pci_p2pdma_bus_offset = pci_bus_address(pdev, bar) -	208	pgmap->pci_p2pdma_bus_offset = pci_bus_address(pdev, bar) -
198	pci_resource_start(pdev, bar);	209	pci_resource_start(pdev, bar);
199	pgmap->kill = pci_p2pdma_percpu_kill;	210	pgmap->kill = pci_p2pdma_percpu_kill;
		211	pgmap->cleanup = pci_p2pdma_percpu_cleanup;
200		212
201	addr = devm_memremap_pages(&pdev->dev, pgmap);	213	addr = devm_memremap_pages(&pdev->dev, pgmap);
202	if (IS_ERR(addr)) {	214	if (IS_ERR(addr)) {
@@ -204,19 +216,22 @@ int pci_p2pdma_add_resource(struct pci_dev *pdev, int bar, size_t size, · 1 unmodified line
204	goto pgmap_free;	216	goto pgmap_free;
205	}	217	}
206		218
207	error = gen_pool_add_virt(pdev->p2pdma->pool, (unsigned long)addr,	219	error = gen_pool_add_owner(pdev->p2pdma->pool, (unsigned long)addr,
208	pci_bus_address(pdev, bar) + offset,	220	pci_bus_address(pdev, bar) + offset,
209	resource_size(&pgmap->res), dev_to_node(&pdev->dev));	221	resource_size(&pgmap->res), dev_to_node(&pdev->dev),
		222	&p2p_pgmap->ref);
210	if (error)	223	if (error)
211	goto pgmap_free;	224	goto pages_free;
212		225
213	pci_info(pdev, "added peer-to-peer DMA memory %pR\n",	226	pci_info(pdev, "added peer-to-peer DMA memory %pR\n",
214	&pgmap->res);	227	&pgmap->res);
215		228
216	return 0;	229	return 0;
217		230
		231	pages_free:
		232	devm_memunmap_pages(&pdev->dev, pgmap);
218	pgmap_free:	233	pgmap_free:
219	devm_kfree(&pdev->dev, pgmap);	234	devm_kfree(&pdev->dev, p2p_pgmap);
220	return error;	235	return error;
221	}	236	}
222	EXPORT_SYMBOL_GPL(pci_p2pdma_add_resource);	237	EXPORT_SYMBOL_GPL(pci_p2pdma_add_resource);
@@ -585,19 +600,30 @@ EXPORT_SYMBOL_GPL(pci_p2pmem_find_many); · 362 unmodified lines
585	*/	600	*/
586	void pci_alloc_p2pmem(struct pci_dev pdev, size_t size)	601	void pci_alloc_p2pmem(struct pci_dev pdev, size_t size)
587	{	602	{
588	void *ret;	603	void *ret = NULL;
		604	struct percpu_ref *ref;
589		605
		606	/*
		607	* Pairs with synchronize_rcu() in pci_p2pdma_release() to
		608	* ensure pdev->p2pdma is non-NULL for the duration of the
		609	* read-lock.
		610	*/
		611	rcu_read_lock();
590	if (unlikely(!pdev->p2pdma))	612	if (unlikely(!pdev->p2pdma))
591	return NULL;	613	goto out;
592
593	if (unlikely(!percpu_ref_tryget_live(&pdev->p2pdma->devmap_ref)))
594	return NULL;
595
596	ret = (void *)gen_pool_alloc(pdev->p2pdma->pool, size);
597		614
598	if (unlikely(!ret))	615	ret = (void *)gen_pool_alloc_owner(pdev->p2pdma->pool, size,
599	percpu_ref_put(&pdev->p2pdma->devmap_ref);	616	(void **) &ref);
		617	if (!ret)
		618	goto out;
600		619
		620	if (unlikely(!percpu_ref_tryget_live(ref))) {
		621	gen_pool_free(pdev->p2pdma->pool, (unsigned long) ret, size);
		622	ret = NULL;
		623	goto out;
		624	}
		625	out:
		626	rcu_read_unlock();
601	return ret;	627	return ret;
602	}	628	}
603	EXPORT_SYMBOL_GPL(pci_alloc_p2pmem);	629	EXPORT_SYMBOL_GPL(pci_alloc_p2pmem);
@@ -610,8 +636,11 @@ EXPORT_SYMBOL_GPL(pci_alloc_p2pmem); · 6 unmodified lines
610	*/	636	*/
611	void pci_free_p2pmem(struct pci_dev pdev, void addr, size_t size)	637	void pci_free_p2pmem(struct pci_dev pdev, void addr, size_t size)
612	{	638	{
613	gen_pool_free(pdev->p2pdma->pool, (uintptr_t)addr, size);	639	struct percpu_ref *ref;
614	percpu_ref_put(&pdev->p2pdma->devmap_ref);	640
		641	gen_pool_free_owner(pdev->p2pdma->pool, (uintptr_t)addr, size,
		642	(void **) &ref);
		643	percpu_ref_put(ref);
615	}	644	}
616	EXPORT_SYMBOL_GPL(pci_free_p2pmem);	645	EXPORT_SYMBOL_GPL(pci_free_p2pmem);
617		646

fs/ocfs2/dcache.c+12−0

@@ -296,6 +296,18 @@ int ocfs2_dentry_attach_lock(struct dentry *dentry, · 295 unmodified lines
296		296
297	out_attach:	297	out_attach:
298	spin_lock(&dentry_attach_lock);	298	spin_lock(&dentry_attach_lock);
		299	if (unlikely(dentry->d_fsdata && !alias)) {
		300	/* d_fsdata is set by a racing thread which is doing
		301	* the same thing as this thread is doing. Leave the racing
		302	* thread going ahead and we return here.
		303	*/
		304	spin_unlock(&dentry_attach_lock);
		305	iput(dl->dl_inode);
		306	ocfs2_lock_res_free(&dl->dl_lockres);
		307	kfree(dl);
		308	return 0;
		309	}
		310
299	dentry->d_fsdata = dl;	311	dentry->d_fsdata = dl;
300	dl->dl_count++;	312	dl->dl_count++;
301	spin_unlock(&dentry_attach_lock);	313	spin_unlock(&dentry_attach_lock);

include/linux/device.h+1−0

@@ -713,6 +713,7 @@ void __iomem devm_of_iomap(struct device dev, · 712 unmodified lines
713	/* allows to add/remove a custom action to devres stack */	713	/* allows to add/remove a custom action to devres stack */
714	int devm_add_action(struct device dev, void (action)(void ), void data);	714	int devm_add_action(struct device dev, void (action)(void ), void data);
715	void devm_remove_action(struct device dev, void (action)(void ), void data);	715	void devm_remove_action(struct device dev, void (action)(void ), void data);
		716	void devm_release_action(struct device dev, void (action)(void ), void data);
716		717
717	static inline int devm_add_action_or_reset(struct device *dev,	718	static inline int devm_add_action_or_reset(struct device *dev,
718	void (action)(void ), void *data)	719	void (action)(void ), void *data)

include/linux/genalloc.h+49−6

@@ -75,6 +75,7 @@ struct gen_pool_chunk { · 74 unmodified lines
75	struct list_head next_chunk; /* next chunk in pool */	75	struct list_head next_chunk; /* next chunk in pool */
76	atomic_long_t avail;	76	atomic_long_t avail;
77	phys_addr_t phys_addr; /* physical starting address of memory chunk */	77	phys_addr_t phys_addr; /* physical starting address of memory chunk */
		78	void owner; / private data to retrieve at alloc time */
78	unsigned long start_addr; /* start address of memory chunk */	79	unsigned long start_addr; /* start address of memory chunk */
79	unsigned long end_addr; /* end address of memory chunk (inclusive) */	80	unsigned long end_addr; /* end address of memory chunk (inclusive) */
80	unsigned long bits[0]; /* bitmap for allocating memory chunk */	81	unsigned long bits[0]; /* bitmap for allocating memory chunk */
@@ -96,8 +97,15 @@ struct genpool_data_fixed { · 15 unmodified lines
96		97
97	extern struct gen_pool *gen_pool_create(int, int);	98	extern struct gen_pool *gen_pool_create(int, int);
98	extern phys_addr_t gen_pool_virt_to_phys(struct gen_pool *pool, unsigned long);	99	extern phys_addr_t gen_pool_virt_to_phys(struct gen_pool *pool, unsigned long);
99	extern int gen_pool_add_virt(struct gen_pool *, unsigned long, phys_addr_t,	100	extern int gen_pool_add_owner(struct gen_pool *, unsigned long, phys_addr_t,
100	size_t, int);	101	size_t, int, void *);
		102
		103	static inline int gen_pool_add_virt(struct gen_pool *pool, unsigned long addr,
		104	phys_addr_t phys, size_t size, int nid)
		105	{
		106	return gen_pool_add_owner(pool, addr, phys, size, nid, NULL);
		107	}
		108
101	/**	109	/**
102	* gen_pool_add - add a new chunk of special memory to the pool	110	* gen_pool_add - add a new chunk of special memory to the pool
103	* @pool: pool to add new memory chunk to	111	* @pool: pool to add new memory chunk to
@@ -116,12 +124,47 @@ static inline int gen_pool_add(struct gen_pool *pool, unsigned long addr, · 12 unmodified lines
116	return gen_pool_add_virt(pool, addr, -1, size, nid);	124	return gen_pool_add_virt(pool, addr, -1, size, nid);
117	}	125	}
118	extern void gen_pool_destroy(struct gen_pool *);	126	extern void gen_pool_destroy(struct gen_pool *);
119	extern unsigned long gen_pool_alloc(struct gen_pool *, size_t);	127	unsigned long gen_pool_alloc_algo_owner(struct gen_pool *pool, size_t size,
120	extern unsigned long gen_pool_alloc_algo(struct gen_pool *, size_t,	128	genpool_algo_t algo, void data, void *owner);
121	genpool_algo_t algo, void *data);	129
		130	static inline unsigned long gen_pool_alloc_owner(struct gen_pool *pool,
		131	size_t size, void **owner)
		132	{
		133	return gen_pool_alloc_algo_owner(pool, size, pool->algo, pool->data,
		134	owner);
		135	}
		136
		137	static inline unsigned long gen_pool_alloc_algo(struct gen_pool *pool,
		138	size_t size, genpool_algo_t algo, void *data)
		139	{
		140	return gen_pool_alloc_algo_owner(pool, size, algo, data, NULL);
		141	}
		142
		143	/**
		144	* gen_pool_alloc - allocate special memory from the pool
		145	* @pool: pool to allocate from
		146	* @size: number of bytes to allocate from the pool
		147	*
		148	* Allocate the requested number of bytes from the specified pool.
		149	* Uses the pool allocation function (with first-fit algorithm by default).
		150	* Can not be used in NMI handler on architectures without
		151	* NMI-safe cmpxchg implementation.
		152	*/
		153	static inline unsigned long gen_pool_alloc(struct gen_pool *pool, size_t size)
		154	{
		155	return gen_pool_alloc_algo(pool, size, pool->algo, pool->data);
		156	}
		157
122	extern void gen_pool_dma_alloc(struct gen_pool pool, size_t size,	158	extern void gen_pool_dma_alloc(struct gen_pool pool, size_t size,
123	dma_addr_t *dma);	159	dma_addr_t *dma);
124	extern void gen_pool_free(struct gen_pool *, unsigned long, size_t);	160	extern void gen_pool_free_owner(struct gen_pool *pool, unsigned long addr,
		161	size_t size, void **owner);
		162	static inline void gen_pool_free(struct gen_pool *pool, unsigned long addr,
		163	size_t size)
		164	{
		165	gen_pool_free_owner(pool, addr, size, NULL);
		166	}
		167
125	extern void gen_pool_for_each_chunk(struct gen_pool *,	168	extern void gen_pool_for_each_chunk(struct gen_pool *,
126	void ()(struct gen_pool , struct gen_pool_chunk , void ), void *);	169	void ()(struct gen_pool , struct gen_pool_chunk , void ), void *);
127	extern size_t gen_pool_avail(struct gen_pool *);	170	extern size_t gen_pool_avail(struct gen_pool *);

include/linux/memcontrol.h+18−8

 struct mem_cgroup_per_node {
 	struct lruvec		lruvec;
+	/* Legacy local VM stats */
+	struct lruvec_stat __percpu *lruvec_stat_local;
+	/* Subtree VM stats (batched updates) */
 	struct lruvec_stat __percpu *lruvec_stat_cpu;
 	atomic_long_t		lruvec_stat[NR_VM_NODE_STAT_ITEMS];
-	atomic_long_t		lruvec_stat_local[NR_VM_NODE_STAT_ITEMS];
 	unsigned long		lru_zone_size[MAX_NR_ZONES][NR_LRU_LISTS];
 	atomic_t		moving_account;
 	struct task_struct	*move_lock_task;
-	/* memory.stat */
+	/* Legacy local VM stats and events */
+	struct memcg_vmstats_percpu __percpu *vmstats_local;
+	/* Subtree VM stats and events (batched updates) */
 	struct memcg_vmstats_percpu __percpu *vmstats_percpu;
 	MEMCG_PADDING(_pad2_);
 	atomic_long_t		vmstats[MEMCG_NR_STAT];
-	atomic_long_t		vmstats_local[MEMCG_NR_STAT];
 	atomic_long_t		vmevents[NR_VM_EVENT_ITEMS];
-	atomic_long_t		vmevents_local[NR_VM_EVENT_ITEMS];
+	/* memory.events */
 	atomic_long_t		memory_events[MEMCG_NR_MEMORY_EVENTS];
 	unsigned long		socket_pressure;
 static inline unsigned long memcg_page_state_local(struct mem_cgroup *memcg,
 						   int idx)
 {
-	long x = atomic_long_read(&memcg->vmstats_local[idx]);
+	long x = 0;
+	int cpu;
+	for_each_possible_cpu(cpu)
+		x += per_cpu(memcg->vmstats_local->stat[idx], cpu);
 #ifdef CONFIG_SMP
 	if (x < 0)
 		x = 0;
 						    enum node_stat_item idx)
 {
 	struct mem_cgroup_per_node *pn;
-	long x;
+	long x = 0;
+	int cpu;
 	if (mem_cgroup_disabled())
 		return node_page_state(lruvec_pgdat(lruvec), idx);
 	pn = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
-	x = atomic_long_read(&pn->lruvec_stat_local[idx]);
+	for_each_possible_cpu(cpu)
+		x += per_cpu(pn->lruvec_stat_local->count[idx], cpu);
 #ifdef CONFIG_SMP
 	if (x < 0)
 		x = 0;

include/linux/memremap.h+8−0

  * @res: physical address range covered by @ref
  * @ref: reference count that pins the devm_memremap_pages() mapping
  * @kill: callback to transition @ref to the dead state
+ * @cleanup: callback to wait for @ref to be idle and reap it
  * @dev: host device of the mapping for debug
  * @data: private data pointer for page_free()
  * @type: memory type: see MEMORY_* in memory_hotplug.h
 	struct resource res;
 	struct percpu_ref *ref;
 	void (*kill)(struct percpu_ref *ref);
+	void (*cleanup)(struct percpu_ref *ref);
 	struct device *dev;
 	void *data;
 	enum memory_type type;
 #ifdef CONFIG_ZONE_DEVICE
 void *devm_memremap_pages(struct device *dev, struct dev_pagemap *pgmap);
+void devm_memunmap_pages(struct device *dev, struct dev_pagemap *pgmap);
 struct dev_pagemap *get_dev_pagemap(unsigned long pfn,
 		struct dev_pagemap *pgmap);
 	return ERR_PTR(-ENXIO);
 }
+static inline void devm_memunmap_pages(struct device *dev,
+		struct dev_pagemap *pgmap)
+{
+}
 static inline struct dev_pagemap *get_dev_pagemap(unsigned long pfn,
 		struct dev_pagemap *pgmap)
 {

include/linux/sched/mm.h+4−0

  * followed by taking the mmap_sem for writing before modifying the
  * vmas or anything the coredump pretends not to change from under it.
  *
+ * It also has to be called when mmgrab() is used in the context of
+ * the process, but then the mm_count refcount is transferred outside
+ * the context of the process to run down_write() on that pinned mm.
+ *
  * NOTE: find_extend_vma() called from GUP context is the only place
  * that can modify the "mm" (notably the vm_start/end) under mmap_sem
  * for reading and outside the context of the process, so it is also

kernel/memremap.c+18−5

 	pgmap->kill(pgmap->ref);
 	for_each_device_pfn(pfn, pgmap)
 		put_page(pfn_to_page(pfn));
+	pgmap->cleanup(pgmap->ref);
 	/* pages are dead and unused, undo the arch mapping */
 	align_start = res->start & ~(SECTION_SIZE - 1);
  * 2/ The altmap field may optionally be initialized, in which case altmap_valid
  *    must be set to true
  *
- * 3/ pgmap->ref must be 'live' on entry and will be killed at
+ * 3/ pgmap->ref must be 'live' on entry and will be killed and reaped
- *    devm_memremap_pages_release() time, or if this routine fails.
+ *    at devm_memremap_pages_release() time, or if this routine fails.
  *
  * 4/ res is expected to be a host memory range that could feasibly be
  *    treated as a "System RAM" range, i.e. not a device mmio range, but
 	pgprot_t pgprot = PAGE_KERNEL;
 	int error, nid, is_ram;
-	if (!pgmap->ref || !pgmap->kill)
+	if (!pgmap->ref || !pgmap->kill || !pgmap->cleanup) {
+		WARN(1, "Missing reference count teardown definition\n");
 		return ERR_PTR(-EINVAL);
+	}
 	align_start = res->start & ~(SECTION_SIZE - 1);
 	align_size = ALIGN(res->start + resource_size(res), SECTION_SIZE)
 	if (conflict_pgmap) {
 		dev_WARN(dev, "Conflicting mapping in same section\n");
 		put_dev_pagemap(conflict_pgmap);
-		return ERR_PTR(-ENOMEM);
+		error = -ENOMEM;
+		goto err_array;
 	}
 	conflict_pgmap = get_dev_pagemap(PHYS_PFN(align_end), NULL);
 	if (conflict_pgmap) {
 		dev_WARN(dev, "Conflicting mapping in same section\n");
 		put_dev_pagemap(conflict_pgmap);
-		return ERR_PTR(-ENOMEM);
+		error = -ENOMEM;
+		goto err_array;
 	}
 	is_ram = region_intersects(align_start, align_size,
 	pgmap_array_delete(res);
  err_array:
 	pgmap->kill(pgmap->ref);
+	pgmap->cleanup(pgmap->ref);
 	return ERR_PTR(error);
 }
 EXPORT_SYMBOL_GPL(devm_memremap_pages);
+void devm_memunmap_pages(struct device *dev, struct dev_pagemap *pgmap)
+{
+	devm_release_action(dev, devm_memremap_pages_release, pgmap);
+}
+EXPORT_SYMBOL_GPL(devm_memunmap_pages);
 unsigned long vmem_altmap_offset(struct vmem_altmap *altmap)
 {
 	/* number of pfns from base where pfn_to_page() is valid */

lib/genalloc.c+25−26

 EXPORT_SYMBOL(gen_pool_create);
 /**
- * gen_pool_add_virt - add a new chunk of special memory to the pool
+ * gen_pool_add_owner- add a new chunk of special memory to the pool
  * @pool: pool to add new memory chunk to
  * @virt: virtual starting address of memory chunk to add to pool
  * @phys: physical starting address of memory chunk to add to pool
  * @size: size in bytes of the memory chunk to add to pool
  * @nid: node id of the node the chunk structure and bitmap should be
  *       allocated on, or -1
+ * @owner: private data the publisher would like to recall at alloc time
  *
  * Add a new chunk of special memory to the specified pool.
  *
  * Returns 0 on success or a -ve errno on failure.
  */
-int gen_pool_add_virt(struct gen_pool *pool, unsigned long virt, phys_addr_t phys,
+int gen_pool_add_owner(struct gen_pool *pool, unsigned long virt, phys_addr_t phys,
-		 size_t size, int nid)
+		 size_t size, int nid, void *owner)
 {
 	struct gen_pool_chunk *chunk;
 	int nbits = size >> pool->min_alloc_order;
 	chunk->phys_addr = phys;
 	chunk->start_addr = virt;
 	chunk->end_addr = virt + size - 1;
+	chunk->owner = owner;
 	atomic_long_set(&chunk->avail, size);
 	spin_lock(&pool->lock);
 	return 0;
 }
-EXPORT_SYMBOL(gen_pool_add_virt);
+EXPORT_SYMBOL(gen_pool_add_owner);
 /**
  * gen_pool_virt_to_phys - return the physical address of memory
 EXPORT_SYMBOL(gen_pool_destroy);
 /**
- * gen_pool_alloc - allocate special memory from the pool
+ * gen_pool_alloc_algo_owner - allocate special memory from the pool
- * @pool: pool to allocate from
- * @size: number of bytes to allocate from the pool
- *
- * Allocate the requested number of bytes from the specified pool.
- * Uses the pool allocation function (with first-fit algorithm by default).
- * Can not be used in NMI handler on architectures without
- * NMI-safe cmpxchg implementation.
- */
-unsigned long gen_pool_alloc(struct gen_pool *pool, size_t size)
-{
-	return gen_pool_alloc_algo(pool, size, pool->algo, pool->data);
-}
-EXPORT_SYMBOL(gen_pool_alloc);
-/**
- * gen_pool_alloc_algo - allocate special memory from the pool
  * @pool: pool to allocate from
  * @size: number of bytes to allocate from the pool
  * @algo: algorithm passed from caller
  * @data: data passed to algorithm
+ * @owner: optionally retrieve the chunk owner
  *
  * Allocate the requested number of bytes from the specified pool.
  * Uses the pool allocation function (with first-fit algorithm by default).
  * Can not be used in NMI handler on architectures without
  * NMI-safe cmpxchg implementation.
  */
-unsigned long gen_pool_alloc_algo(struct gen_pool *pool, size_t size,
+unsigned long gen_pool_alloc_algo_owner(struct gen_pool *pool, size_t size,
-		genpool_algo_t algo, void *data)
+		genpool_algo_t algo, void *data, void **owner)
 {
 	struct gen_pool_chunk *chunk;
 	unsigned long addr = 0;
 	BUG_ON(in_nmi());
 #endif
+	if (owner)
+		*owner = NULL;
 	if (size == 0)
 		return 0;
 		addr = chunk->start_addr + ((unsigned long)start_bit << order);
 		size = nbits << order;
 		atomic_long_sub(size, &chunk->avail);
+		if (owner)
+			*owner = chunk->owner;
 		break;
 	}
 	rcu_read_unlock();
 	return addr;
 }
-EXPORT_SYMBOL(gen_pool_alloc_algo);
+EXPORT_SYMBOL(gen_pool_alloc_algo_owner);
 /**
  * gen_pool_dma_alloc - allocate special memory from the pool for DMA usage
  * @pool: pool to free to
  * @addr: starting address of memory to free back to pool
  * @size: size in bytes of memory to free
+ * @owner: private data stashed at gen_pool_add() time
  *
  * Free previously allocated special memory back to the specified
  * pool.  Can not be used in NMI handler on architectures without
  * NMI-safe cmpxchg implementation.
  */
-void gen_pool_free(struct gen_pool *pool, unsigned long addr, size_t size)
+void gen_pool_free_owner(struct gen_pool *pool, unsigned long addr, size_t size,
+		void **owner)
 {
 	struct gen_pool_chunk *chunk;
 	int order = pool->min_alloc_order;
 	BUG_ON(in_nmi());
 #endif
+	if (owner)
+		*owner = NULL;
 	nbits = (size + (1UL << order) - 1) >> order;
 	rcu_read_lock();
 	list_for_each_entry_rcu(chunk, &pool->chunks, next_chunk) {
 			BUG_ON(remain);
 			size = nbits << order;
 			atomic_long_add(size, &chunk->avail);
+			if (owner)
+				*owner = chunk->owner;
 			rcu_read_unlock();
 			return;
 		}
 	rcu_read_unlock();
 	BUG();
 }
-EXPORT_SYMBOL(gen_pool_free);
+EXPORT_SYMBOL(gen_pool_free_owner);
 /**
  * gen_pool_for_each_chunk - call func for every chunk of generic memory pool

mm/hmm.c+3−11

 	complete(&devmem->completion);
 }
-static void hmm_devmem_ref_exit(void *data)
+static void hmm_devmem_ref_exit(struct percpu_ref *ref)
 {
-	struct percpu_ref *ref = data;
 	struct hmm_devmem *devmem;
 	devmem = container_of(ref, struct hmm_devmem, ref);
 	if (ret)
 		return ERR_PTR(ret);
-	ret = devm_add_action_or_reset(device, hmm_devmem_ref_exit, &devmem->ref);
-	if (ret)
-		return ERR_PTR(ret);
 	size = ALIGN(size, PA_SECTION_SIZE);
 	addr = min((unsigned long)iomem_resource.end,
 		   (1UL << MAX_PHYSMEM_BITS) - 1);
 	devmem->pagemap.ref = &devmem->ref;
 	devmem->pagemap.data = devmem;
 	devmem->pagemap.kill = hmm_devmem_ref_kill;
+	devmem->pagemap.cleanup = hmm_devmem_ref_exit;
 	result = devm_memremap_pages(devmem->device, &devmem->pagemap);
 	if (IS_ERR(result))
 	if (ret)
 		return ERR_PTR(ret);
-	ret = devm_add_action_or_reset(device, hmm_devmem_ref_exit,
-			&devmem->ref);
-	if (ret)
-		return ERR_PTR(ret);
 	devmem->pfn_first = devmem->resource->start >> PAGE_SHIFT;
 	devmem->pfn_last = devmem->pfn_first +
 			   (resource_size(devmem->resource) >> PAGE_SHIFT);
 	devmem->pagemap.ref = &devmem->ref;
 	devmem->pagemap.data = devmem;
 	devmem->pagemap.kill = hmm_devmem_ref_kill;
+	devmem->pagemap.cleanup = hmm_devmem_ref_exit;
 	result = devm_memremap_pages(devmem->device, &devmem->pagemap);
 	if (IS_ERR(result))

mm/khugepaged.c+3−0

 	 * handled by the anon_vma lock + PG_lock.
 	 */
 	down_write(&mm->mmap_sem);
+	result = SCAN_ANY_PROCESS;
+	if (!mmget_still_valid(mm))
+		goto out;
 	result = hugepage_vma_revalidate(mm, address, &vma);
 	if (result)
 		goto out;

mm/list_lru.c+1−1

@@ -354,7 +354,7 @@ static int __memcg_init_list_lru_node(struct list_lru_memcg *memcg_lrus, · 353 unmodified lines
354	}	354	}
355	return 0;	355	return 0;
356	fail:	356	fail:
357	__memcg_destroy_list_lru_node(memcg_lrus, begin, i - 1);	357	__memcg_destroy_list_lru_node(memcg_lrus, begin, i);
358	return -ENOMEM;	358	return -ENOMEM;
359	}	359	}
360		360

mm/memcontrol.c+28−13

 	if (mem_cgroup_disabled())
 		return;
+	__this_cpu_add(memcg->vmstats_local->stat[idx], val);
 	x = val + __this_cpu_read(memcg->vmstats_percpu->stat[idx]);
 	if (unlikely(abs(x) > MEMCG_CHARGE_BATCH)) {
 		struct mem_cgroup *mi;
-		atomic_long_add(x, &memcg->vmstats_local[idx]);
 		for (mi = memcg; mi; mi = parent_mem_cgroup(mi))
 			atomic_long_add(x, &mi->vmstats[idx]);
 		x = 0;
 	__mod_memcg_state(memcg, idx, val);
 	/* Update lruvec */
+	__this_cpu_add(pn->lruvec_stat_local->count[idx], val);
 	x = val + __this_cpu_read(pn->lruvec_stat_cpu->count[idx]);
 	if (unlikely(abs(x) > MEMCG_CHARGE_BATCH)) {
 		struct mem_cgroup_per_node *pi;
-		atomic_long_add(x, &pn->lruvec_stat_local[idx]);
 		for (pi = pn; pi; pi = parent_nodeinfo(pi, pgdat->node_id))
 			atomic_long_add(x, &pi->lruvec_stat[idx]);
 		x = 0;
 	if (mem_cgroup_disabled())
 		return;
+	__this_cpu_add(memcg->vmstats_local->events[idx], count);
 	x = count + __this_cpu_read(memcg->vmstats_percpu->events[idx]);
 	if (unlikely(x > MEMCG_CHARGE_BATCH)) {
 		struct mem_cgroup *mi;
-		atomic_long_add(x, &memcg->vmevents_local[idx]);
 		for (mi = memcg; mi; mi = parent_mem_cgroup(mi))
 			atomic_long_add(x, &mi->vmevents[idx]);
 		x = 0;
 static unsigned long memcg_events_local(struct mem_cgroup *memcg, int event)
 {
-	return atomic_long_read(&memcg->vmevents_local[event]);
+	long x = 0;
+	int cpu;
+	for_each_possible_cpu(cpu)
+		x += per_cpu(memcg->vmstats_local->events[event], cpu);
+	return x;
 }
 static void mem_cgroup_charge_statistics(struct mem_cgroup *memcg,
 			long x;
 			x = this_cpu_xchg(memcg->vmstats_percpu->stat[i], 0);
-			if (x) {
+			if (x)
-				atomic_long_add(x, &memcg->vmstats_local[i]);
 				for (mi = memcg; mi; mi = parent_mem_cgroup(mi))
 					atomic_long_add(x, &memcg->vmstats[i]);
-			}
 			if (i >= NR_VM_NODE_STAT_ITEMS)
 				continue;
 				pn = mem_cgroup_nodeinfo(memcg, nid);
 				x = this_cpu_xchg(pn->lruvec_stat_cpu->count[i], 0);
-				if (x) {
+				if (x)
-					atomic_long_add(x, &pn->lruvec_stat_local[i]);
 					do {
 						atomic_long_add(x, &pn->lruvec_stat[i]);
 					} while ((pn = parent_nodeinfo(pn, nid)));
-				}
 			}
 		}
 			long x;
 			x = this_cpu_xchg(memcg->vmstats_percpu->events[i], 0);
-			if (x) {
+			if (x)
-				atomic_long_add(x, &memcg->vmevents_local[i]);
 				for (mi = memcg; mi; mi = parent_mem_cgroup(mi))
 					atomic_long_add(x, &memcg->vmevents[i]);
-			}
 		}
 	}
 	if (!pn)
 		return 1;
+	pn->lruvec_stat_local = alloc_percpu(struct lruvec_stat);
+	if (!pn->lruvec_stat_local) {
+		kfree(pn);
+		return 1;
+	}
 	pn->lruvec_stat_cpu = alloc_percpu(struct lruvec_stat);
 	if (!pn->lruvec_stat_cpu) {
+		free_percpu(pn->lruvec_stat_local);
 		kfree(pn);
 		return 1;
 	}
 		return;
 	free_percpu(pn->lruvec_stat_cpu);
+	free_percpu(pn->lruvec_stat_local);
 	kfree(pn);
 }
 	for_each_node(node)
 		free_mem_cgroup_per_node_info(memcg, node);
 	free_percpu(memcg->vmstats_percpu);
+	free_percpu(memcg->vmstats_local);
 	kfree(memcg);
 }
 	if (memcg->id.id < 0)
 		goto fail;
+	memcg->vmstats_local = alloc_percpu(struct memcg_vmstats_percpu);
+	if (!memcg->vmstats_local)
+		goto fail;
 	memcg->vmstats_percpu = alloc_percpu(struct memcg_vmstats_percpu);
 	if (!memcg->vmstats_percpu)
 		goto fail;

mm/mlock.c+4−3

  * is also counted.
  * Return value: previously mlocked page counts
  */
-static int count_mm_mlocked_page_nr(struct mm_struct *mm,
+static unsigned long count_mm_mlocked_page_nr(struct mm_struct *mm,
 		unsigned long start, size_t len)
 {
 	struct vm_area_struct *vma;
-	int count = 0;
+	unsigned long count = 0;
 	if (mm == NULL)
 		mm = current->mm;
 	unsigned long lock_limit;
 	int ret;
-	if (!flags || (flags & ~(MCL_CURRENT | MCL_FUTURE | MCL_ONFAULT)))
+	if (!flags || (flags & ~(MCL_CURRENT | MCL_FUTURE | MCL_ONFAULT)) ||
+	    flags == MCL_ONFAULT)
 		return -EINVAL;
 	if (!can_do_mlock())

mm/mmu_gather.c+19−5

 {
 	/*
 	 * If there are parallel threads are doing PTE changes on same range
-	 * under non-exclusive lock(e.g., mmap_sem read-side) but defer TLB
+	 * under non-exclusive lock (e.g., mmap_sem read-side) but defer TLB
-	 * flush by batching, a thread has stable TLB entry can fail to flush
+	 * flush by batching, one thread may end up seeing inconsistent PTEs
-	 * the TLB by observing pte_none|!pte_dirty, for example so flush TLB
+	 * and result in having stale TLB entries.  So flush TLB forcefully
-	 * forcefully if we detect parallel PTE batching threads.
+	 * if we detect parallel PTE batching threads.
+	 *
+	 * However, some syscalls, e.g. munmap(), may free page tables, this
+	 * needs force flush everything in the given range. Otherwise this
+	 * may result in having stale TLB entries for some architectures,
+	 * e.g. aarch64, that could specify flush what level TLB.
 	 */
 	if (mm_tlb_flush_nested(tlb->mm)) {
+		/*
+		 * The aarch64 yields better performance with fullmm by
+		 * avoiding multiple CPUs spamming TLBI messages at the
+		 * same time.
+		 *
+		 * On x86 non-fullmm doesn't yield significant difference
+		 * against fullmm.
+		 */
+		tlb->fullmm = 1;
 		__tlb_reset_range(tlb);
-		__tlb_adjust_range(tlb, start, end - start);
+		tlb->freed_tables = 1;
 	}
 	tlb_flush_mmu(tlb);

mm/vmscan.c+3−3

@@ -1505,7 +1505,7 @@ unsigned long reclaim_clean_pages_from_list(struct zone *zone, · 1504 unmodified lines
1505		1505
1506	list_for_each_entry_safe(page, next, page_list, lru) {	1506	list_for_each_entry_safe(page, next, page_list, lru) {
1507	if (page_is_file_cache(page) && !PageDirty(page) &&	1507	if (page_is_file_cache(page) && !PageDirty(page) &&
1508	!__PageMovable(page)) {	1508	!__PageMovable(page) && !PageUnevictable(page)) {
1509	ClearPageActive(page);	1509	ClearPageActive(page);
1510	list_move(&page->lru, &clean_pages);	1510	list_move(&page->lru, &clean_pages);
1511	}	1511	}
@@ -1953,8 +1953,8 @@ shrink_inactive_list(unsigned long nr_to_scan, struct lruvec *lruvec, · 441 unmodified lines
1953	if (global_reclaim(sc))	1953	if (global_reclaim(sc))
1954	__count_vm_events(item, nr_reclaimed);	1954	__count_vm_events(item, nr_reclaimed);
1955	__count_memcg_events(lruvec_memcg(lruvec), item, nr_reclaimed);	1955	__count_memcg_events(lruvec_memcg(lruvec), item, nr_reclaimed);
1956	reclaim_stat->recent_rotated[0] = stat.nr_activate[0];	1956	reclaim_stat->recent_rotated[0] += stat.nr_activate[0];
1957	reclaim_stat->recent_rotated[1] = stat.nr_activate[1];	1957	reclaim_stat->recent_rotated[1] += stat.nr_activate[1];
1958		1958
1959	move_pages_to_lru(lruvec, &page_list);	1959	move_pages_to_lru(lruvec, &page_list);
1960		1960

scripts/decode_stacktrace.sh+1−1

@@ -73,7 +73,7 @@ parse_symbol() { · 72 unmodified lines
73	if [[ "${cache[$module,$address]+isset}" == "isset" ]]; then	73	if [[ "${cache[$module,$address]+isset}" == "isset" ]]; then
74	local code=${cache[$module,$address]}	74	local code=${cache[$module,$address]}
75	else	75	else
76	local code=$(addr2line -i -e "$objfile" "$address")	76	local code=$(${CROSS_COMPILE}addr2line -i -e "$objfile" "$address")
77	cache[$module,$address]=$code	77	cache[$module,$address]=$code
78	fi	78	fi
79		79

tools/testing/nvdimm/test/iomap.c+2−0

@@ -100,7 +100,9 @@ static void nfit_test_kill(void *_pgmap) · 99 unmodified lines
100	{	100	{
101	struct dev_pagemap *pgmap = _pgmap;	101	struct dev_pagemap *pgmap = _pgmap;
102		102
		103	WARN_ON(!pgmap \|\| !pgmap->ref \|\| !pgmap->kill \|\| !pgmap->cleanup);
103	pgmap->kill(pgmap->ref);	104	pgmap->kill(pgmap->ref);
		105	pgmap->cleanup(pgmap->ref);
104	}	106	}
105		107
106	void __wrap_devm_memremap_pages(struct device dev, struct dev_pagemap *pgmap)	108	void __wrap_devm_memremap_pages(struct device dev, struct dev_pagemap *pgmap)