Merge branch 'akpm' (patches from Andrew)

Merge yet more updates from Andrew Morton: "A few final bits: - large changes to vmalloc, yielding large performance benefits - tweak the console-flush-on-panic code - a few fixes" * emailed patches from Andrew Morton <akpm@linux-foundation.org>: panic: add an option to replay all the printk message in buffer initramfs: don't free a non-existent initrd fs/writeback.c: use rcu_barrier() to wait for inflight wb switches going into workqueue when umount mm/compaction.c: correct zone boundary handling when isolating pages from a pageblock mm/vmap: add DEBUG_AUGMENT_LOWEST_MATCH_CHECK macro mm/vmap: add DEBUG_AUGMENT_PROPAGATE_CHECK macro mm/vmalloc.c: keep track of free blocks for vmap allocation

Linus Torvalds committed 7 years ago

Commit cb6f8732 parents ff8583d de6da1e10 files · +889 −257Browse files

Unified · Split

10 files

Documentation/admin-guide/kernel-parameters.txt +1 −0
arch/powerpc/kernel/traps.c +1 −1
fs/fs-writeback.c +8 −3
include/linux/console.h +6 −1
include/linux/vmalloc.h +5 −1
init/initramfs.c +1 −1
kernel/panic.c +5 −1
kernel/printk/printk.c +11 −1
mm/compaction.c +2 −2
mm/vmalloc.c +849 −246

Documentation/admin-guide/kernel-parameters.txt+1−0

@@ -3212,6 +3212,7 @@ · 3211 unmodified lines
3212	bit 2: print timer info	3212	bit 2: print timer info
3213	bit 3: print locks info if CONFIG_LOCKDEP is on	3213	bit 3: print locks info if CONFIG_LOCKDEP is on
3214	bit 4: print ftrace buffer	3214	bit 4: print ftrace buffer
		3215	bit 5: print all printk messages in buffer
3215		3216
3216	panic_on_warn panic() instead of WARN(). Useful to cause kdump	3217	panic_on_warn panic() instead of WARN(). Useful to cause kdump
3217	on a WARN().	3218	on a WARN().

arch/powerpc/kernel/traps.c+1−1

@@ -179,7 +179,7 @@ extern void panic_flush_kmsg_end(void) · 178 unmodified lines
179	kmsg_dump(KMSG_DUMP_PANIC);	179	kmsg_dump(KMSG_DUMP_PANIC);
180	bust_spinlocks(0);	180	bust_spinlocks(0);
181	debug_locks_off();	181	debug_locks_off();
182	console_flush_on_panic();	182	console_flush_on_panic(CONSOLE_FLUSH_PENDING);
183	}	183	}
184		184
185	static unsigned long oops_begin(struct pt_regs *regs)	185	static unsigned long oops_begin(struct pt_regs *regs)

fs/fs-writeback.c+8−3

 	isw->inode = inode;
-	atomic_inc(&isw_nr_in_flight);
 	/*
 	 * In addition to synchronizing among switchers, I_WB_SWITCH tells
 	 * the RCU protected stat update paths to grab the i_page
 	 * Let's continue after I_WB_SWITCH is guaranteed to be visible.
 	 */
 	call_rcu(&isw->rcu_head, inode_switch_wbs_rcu_fn);
+	atomic_inc(&isw_nr_in_flight);
 	goto out_unlock;
 out_free:
 void cgroup_writeback_umount(void)
 {
 	if (atomic_read(&isw_nr_in_flight)) {
-		synchronize_rcu();
+		/*
+		 * Use rcu_barrier() to wait for all pending callbacks to
+		 * ensure that all in-flight wb switches are in the workqueue.
+		 */
+		rcu_barrier();
 		flush_workqueue(isw_wq);
 	}
 }

include/linux/console.h+6−1

 extern int console_set_on_cmdline;
 extern struct console *early_console;
+enum con_flush_mode {
+	CONSOLE_FLUSH_PENDING,
+	CONSOLE_REPLAY_ALL,
+};
 extern int add_preferred_console(char *name, int idx, char *options);
 extern void register_console(struct console *);
 extern int unregister_console(struct console *);
 extern void console_unlock(void);
 extern void console_conditional_schedule(void);
 extern void console_unblank(void);
-extern void console_flush_on_panic(void);
+extern void console_flush_on_panic(enum con_flush_mode mode);
 extern struct tty_driver *console_device(int *);
 extern void console_stop(struct console *);
 extern void console_start(struct console *);

include/linux/vmalloc.h+5−1

 struct vmap_area {
 	unsigned long va_start;
 	unsigned long va_end;
+	/*
+	 * Largest available free size in subtree.
+	 */
+	unsigned long subtree_max_size;
 	unsigned long flags;
 	struct rb_node rb_node;         /* address sorted rbtree */
 	struct list_head list;          /* address sorted list */
 	struct llist_node purge_list;    /* "lazy purge" list */
 	struct vm_struct *vm;
-	struct rcu_head rcu_head;
 };
 /*

init/initramfs.c+1−1

@@ -669,7 +669,7 @@ done: · 668 unmodified lines
669	* If the initrd region is overlapped with crashkernel reserved region,	669	* If the initrd region is overlapped with crashkernel reserved region,
670	* free only memory that is not part of crashkernel region.	670	* free only memory that is not part of crashkernel region.
671	*/	671	*/
672	if (!do_retain_initrd && !kexec_free_initrd())	672	if (!do_retain_initrd && initrd_start && !kexec_free_initrd())
673	free_initrd_mem(initrd_start, initrd_end);	673	free_initrd_mem(initrd_start, initrd_end);
674	initrd_start = 0;	674	initrd_start = 0;
675	initrd_end = 0;	675	initrd_end = 0;

kernel/panic.c+5−1

 #define PANIC_PRINT_TIMER_INFO		0x00000004
 #define PANIC_PRINT_LOCK_INFO		0x00000008
 #define PANIC_PRINT_FTRACE_INFO		0x00000010
+#define PANIC_PRINT_ALL_PRINTK_MSG	0x00000020
 unsigned long panic_print;
 ATOMIC_NOTIFIER_HEAD(panic_notifier_list);
 static void panic_print_sys_info(void)
 {
+	if (panic_print & PANIC_PRINT_ALL_PRINTK_MSG)
+		console_flush_on_panic(CONSOLE_REPLAY_ALL);
 	if (panic_print & PANIC_PRINT_TASK_INFO)
 		show_state();
 	 * panic() is not being callled from OOPS.
 	 */
 	debug_locks_off();
-	console_flush_on_panic();
+	console_flush_on_panic(CONSOLE_FLUSH_PENDING);
 	panic_print_sys_info();

kernel/printk/printk.c+11−1

 /**
  * console_flush_on_panic - flush console content on panic
+ * @mode: flush all messages in buffer or just the pending ones
  *
  * Immediately output all pending messages no matter what.
  */
-void console_flush_on_panic(void)
+void console_flush_on_panic(enum con_flush_mode mode)
 {
 	/*
 	 * If someone else is holding the console lock, trylock will fail
 	 */
 	console_trylock();
 	console_may_schedule = 0;
+	if (mode == CONSOLE_REPLAY_ALL) {
+		unsigned long flags;
+		logbuf_lock_irqsave(flags);
+		console_seq = log_first_seq;
+		console_idx = log_first_idx;
+		logbuf_unlock_irqrestore(flags);
+	}
 	console_unlock();
 }

mm/compaction.c+2−2

@@ -1230,7 +1230,7 @@ fast_isolate_around(struct compact_control *cc, unsigned long pfn, unsigned long · 1229 unmodified lines
1230		1230
1231	/* Pageblock boundaries */	1231	/* Pageblock boundaries */
1232	start_pfn = pageblock_start_pfn(pfn);	1232	start_pfn = pageblock_start_pfn(pfn);
1233	end_pfn = min(start_pfn + pageblock_nr_pages, zone_end_pfn(cc->zone));	1233	end_pfn = min(pageblock_end_pfn(pfn), zone_end_pfn(cc->zone)) - 1;
1234		1234
1235	/* Scan before */	1235	/* Scan before */
1236	if (start_pfn != pfn) {	1236	if (start_pfn != pfn) {
@@ -1241,7 +1241,7 @@ fast_isolate_around(struct compact_control *cc, unsigned long pfn, unsigned long · 4 unmodified lines
1241		1241
1242	/* Scan after */	1242	/* Scan after */
1243	start_pfn = pfn + nr_isolated;	1243	start_pfn = pfn + nr_isolated;
1244	if (start_pfn != end_pfn)	1244	if (start_pfn < end_pfn)
1245	isolate_freepages_block(cc, &start_pfn, end_pfn, &cc->freepages, 1, false);	1245	isolate_freepages_block(cc, &start_pfn, end_pfn, &cc->freepages, 1, false);
1246		1246
1247	/* Skip this pageblock in the future as it's full or nearly full */	1247	/* Skip this pageblock in the future as it's full or nearly full */

mm/vmalloc.c+849−246

@@ -32,6 +32,7 @@ · 31 unmodified lines
32	#include <linux/compiler.h>	32	#include <linux/compiler.h>
33	#include <linux/llist.h>	33	#include <linux/llist.h>
34	#include <linux/bitops.h>	34	#include <linux/bitops.h>
		35	#include <linux/rbtree_augmented.h>
35		36
36	#include <linux/uaccess.h>	37	#include <linux/uaccess.h>
37	#include <asm/tlbflush.h>	38	#include <asm/tlbflush.h>
@@ -324,6 +325,9 @@ EXPORT_SYMBOL(vmalloc_to_pfn); · 286 unmodified lines
324		325
325	/* Global kva allocator */	326	/* Global kva allocator */
326		327
		328	#define DEBUG_AUGMENT_PROPAGATE_CHECK 0
		329	#define DEBUG_AUGMENT_LOWEST_MATCH_CHECK 0
		330
327	#define VM_LAZY_FREE 0x02	331	#define VM_LAZY_FREE 0x02
328	#define VM_VM_AREA 0x04	332	#define VM_VM_AREA 0x04
329		333
@@ -332,14 +336,67 @@ static DEFINE_SPINLOCK(vmap_area_lock); · 2 unmodified lines
332	LIST_HEAD(vmap_area_list);	336	LIST_HEAD(vmap_area_list);
333	static LLIST_HEAD(vmap_purge_list);	337	static LLIST_HEAD(vmap_purge_list);
334	static struct rb_root vmap_area_root = RB_ROOT;	338	static struct rb_root vmap_area_root = RB_ROOT;
		339	static bool vmap_initialized __read_mostly;
		340
		341	/*
		342	* This kmem_cache is used for vmap_area objects. Instead of
		343	* allocating from slab we reuse an object from this cache to
		344	* make things faster. Especially in "no edge" splitting of
		345	* free block.
		346	*/
		347	static struct kmem_cache *vmap_area_cachep;
		348
		349	/*
		350	* This linked list is used in pair with free_vmap_area_root.
		351	* It gives O(1) access to prev/next to perform fast coalescing.
		352	*/
		353	static LIST_HEAD(free_vmap_area_list);
		354
		355	/*
		356	* This augment red-black tree represents the free vmap space.
		357	* All vmap_area objects in this tree are sorted by va->va_start
		358	* address. It is used for allocation and merging when a vmap
		359	* object is released.
		360	*
		361	* Each vmap_area node contains a maximum available free block
		362	* of its sub-tree, right or left. Therefore it is possible to
		363	* find a lowest match of free area.
		364	*/
		365	static struct rb_root free_vmap_area_root = RB_ROOT;
		366
		367	static __always_inline unsigned long
		368	va_size(struct vmap_area *va)
		369	{
		370	return (va->va_end - va->va_start);
		371	}
		372
		373	static __always_inline unsigned long
		374	get_subtree_max_size(struct rb_node *node)
		375	{
		376	struct vmap_area *va;
		377
		378	va = rb_entry_safe(node, struct vmap_area, rb_node);
		379	return va ? va->subtree_max_size : 0;
		380	}
		381
		382	/*
		383	* Gets called when remove the node and rotate.
		384	*/
		385	static __always_inline unsigned long
		386	compute_subtree_max_size(struct vmap_area *va)
		387	{
		388	return max3(va_size(va),
		389	get_subtree_max_size(va->rb_node.rb_left),
		390	get_subtree_max_size(va->rb_node.rb_right));
		391	}
335		392
336	/* The vmap cache globals are protected by vmap_area_lock */	393	RB_DECLARE_CALLBACKS(static, free_vmap_area_rb_augment_cb,
337	static struct rb_node *free_vmap_cache;	394	struct vmap_area, rb_node, unsigned long, subtree_max_size,
338	static unsigned long cached_hole_size;	395	compute_subtree_max_size)
339	static unsigned long cached_vstart;
340	static unsigned long cached_align;
341		396
342	static unsigned long vmap_area_pcpu_hole;	397	static void purge_vmap_area_lazy(void);
		398	static BLOCKING_NOTIFIER_HEAD(vmap_notify_list);
		399	static unsigned long lazy_max_pages(void);
343		400
344	static struct vmap_area *__find_vmap_area(unsigned long addr)	401	static struct vmap_area *__find_vmap_area(unsigned long addr)
345	{	402	{
@@ -360,41 +417,610 @@ static struct vmap_area *__find_vmap_area(unsigned long addr) · 14 unmodified lines
360	return NULL;	417	return NULL;
361	}	418	}
362		419
363	static void __insert_vmap_area(struct vmap_area *va)	420	/*
364	{	421	* This function returns back addresses of parent node
365	struct rb_node **p = &vmap_area_root.rb_node;	422	* and its left or right link for further processing.
366	struct rb_node *parent = NULL;	423	*/
367	struct rb_node *tmp;	424	static __always_inline struct rb_node **
		425	find_va_links(struct vmap_area *va,
		426	struct rb_root root, struct rb_node from,
		427	struct rb_node **parent)
		428	{
		429	struct vmap_area *tmp_va;
		430	struct rb_node **link;
		431
		432	if (root) {
		433	link = &root->rb_node;
		434	if (unlikely(!*link)) {
		435	*parent = NULL;
		436	return link;
		437	}
		438	} else {
		439	link = &from;
		440	}
368		441
369	while (*p) {	442	/*
370	struct vmap_area *tmp_va;	443	* Go to the bottom of the tree. When we hit the last point
		444	* we end up with parent rb_node and correct direction, i name
		445	* it link, where the new va->rb_node will be attached to.
		446	*/
		447	do {
		448	tmp_va = rb_entry(*link, struct vmap_area, rb_node);
371		449
372	parent = *p;	450	/*
373	tmp_va = rb_entry(parent, struct vmap_area, rb_node);	451	* During the traversal we also do some sanity check.
374	if (va->va_start < tmp_va->va_end)	452	* Trigger the BUG() if there are sides(left/right)
375	p = &(*p)->rb_left;	453	* or full overlaps.
376	else if (va->va_end > tmp_va->va_start)	454	*/
377	p = &(*p)->rb_right;	455	if (va->va_start < tmp_va->va_end &&
		456	va->va_end <= tmp_va->va_start)
		457	link = &(*link)->rb_left;
		458	else if (va->va_end > tmp_va->va_start &&
		459	va->va_start >= tmp_va->va_end)
		460	link = &(*link)->rb_right;
378	else	461	else
379	BUG();	462	BUG();
		463	} while (*link);
		464
		465	*parent = &tmp_va->rb_node;
		466	return link;
		467	}
		468
		469	static __always_inline struct list_head *
		470	get_va_next_sibling(struct rb_node parent, struct rb_node *link)
		471	{
		472	struct list_head *list;
		473
		474	if (unlikely(!parent))
		475	/*
		476	* The red-black tree where we try to find VA neighbors
		477	* before merging or inserting is empty, i.e. it means
		478	* there is no free vmap space. Normally it does not
		479	* happen but we handle this case anyway.
		480	*/
		481	return NULL;
		482
		483	list = &rb_entry(parent, struct vmap_area, rb_node)->list;
		484	return (&parent->rb_right == link ? list->next : list);
		485	}
		486
		487	static __always_inline void
		488	link_va(struct vmap_area va, struct rb_root root,
		489	struct rb_node parent, struct rb_node link, struct list_head head)
		490	{
		491	/*
		492	* VA is still not in the list, but we can
		493	* identify its future previous list_head node.
		494	*/
		495	if (likely(parent)) {
		496	head = &rb_entry(parent, struct vmap_area, rb_node)->list;
		497	if (&parent->rb_right != link)
		498	head = head->prev;
380	}	499	}
381		500
382	rb_link_node(&va->rb_node, parent, p);	501	/* Insert to the rb-tree */
383	rb_insert_color(&va->rb_node, &vmap_area_root);	502	rb_link_node(&va->rb_node, parent, link);
		503	if (root == &free_vmap_area_root) {
		504	/*
		505	* Some explanation here. Just perform simple insertion
		506	* to the tree. We do not set va->subtree_max_size to
		507	* its current size before calling rb_insert_augmented().
		508	* It is because of we populate the tree from the bottom
		509	* to parent levels when the node _is_ in the tree.
		510	*
		511	* Therefore we set subtree_max_size to zero after insertion,
		512	* to let __augment_tree_propagate_from() puts everything to
		513	* the correct order later on.
		514	*/
		515	rb_insert_augmented(&va->rb_node,
		516	root, &free_vmap_area_rb_augment_cb);
		517	va->subtree_max_size = 0;
		518	} else {
		519	rb_insert_color(&va->rb_node, root);
		520	}
384		521
385	/* address-sort this list */	522	/* Address-sort this list */
386	tmp = rb_prev(&va->rb_node);	523	list_add(&va->list, head);
387	if (tmp) {
388	struct vmap_area *prev;
389	prev = rb_entry(tmp, struct vmap_area, rb_node);
390	list_add_rcu(&va->list, &prev->list);
391	} else
392	list_add_rcu(&va->list, &vmap_area_list);
393	}	524	}
394		525
395	static void purge_vmap_area_lazy(void);	526	static __always_inline void
		527	unlink_va(struct vmap_area va, struct rb_root root)
		528	{
		529	/*
		530	* During merging a VA node can be empty, therefore
		531	* not linked with the tree nor list. Just check it.
		532	*/
		533	if (!RB_EMPTY_NODE(&va->rb_node)) {
		534	if (root == &free_vmap_area_root)
		535	rb_erase_augmented(&va->rb_node,
		536	root, &free_vmap_area_rb_augment_cb);
		537	else
		538	rb_erase(&va->rb_node, root);
396		539
397	static BLOCKING_NOTIFIER_HEAD(vmap_notify_list);	540	list_del(&va->list);
		541	RB_CLEAR_NODE(&va->rb_node);
		542	}
		543	}
		544
		545	#if DEBUG_AUGMENT_PROPAGATE_CHECK
		546	static void
		547	augment_tree_propagate_check(struct rb_node *n)
		548	{
		549	struct vmap_area *va;
		550	struct rb_node *node;
		551	unsigned long size;
		552	bool found = false;
		553
		554	if (n == NULL)
		555	return;
		556
		557	va = rb_entry(n, struct vmap_area, rb_node);
		558	size = va->subtree_max_size;
		559	node = n;
		560
		561	while (node) {
		562	va = rb_entry(node, struct vmap_area, rb_node);
		563
		564	if (get_subtree_max_size(node->rb_left) == size) {
		565	node = node->rb_left;
		566	} else {
		567	if (va_size(va) == size) {
		568	found = true;
		569	break;
		570	}
		571
		572	node = node->rb_right;
		573	}
		574	}
		575
		576	if (!found) {
		577	va = rb_entry(n, struct vmap_area, rb_node);
		578	pr_emerg("tree is corrupted: %lu, %lu\n",
		579	va_size(va), va->subtree_max_size);
		580	}
		581
		582	augment_tree_propagate_check(n->rb_left);
		583	augment_tree_propagate_check(n->rb_right);
		584	}
		585	#endif
		586
		587	/*
		588	* This function populates subtree_max_size from bottom to upper
		589	* levels starting from VA point. The propagation must be done
		590	* when VA size is modified by changing its va_start/va_end. Or
		591	* in case of newly inserting of VA to the tree.
		592	*
		593	* It means that __augment_tree_propagate_from() must be called:
		594	* - After VA has been inserted to the tree(free path);
		595	* - After VA has been shrunk(allocation path);
		596	* - After VA has been increased(merging path).
		597	*
		598	* Please note that, it does not mean that upper parent nodes
		599	* and their subtree_max_size are recalculated all the time up
		600	* to the root node.
		601	*
		602	* 4--8
		603	* /\
		604	* / \
		605	* / \
		606	* 2--2 8--8
		607	*
		608	* For example if we modify the node 4, shrinking it to 2, then
		609	* no any modification is required. If we shrink the node 2 to 1
		610	* its subtree_max_size is updated only, and set to 1. If we shrink
		611	* the node 8 to 6, then its subtree_max_size is set to 6 and parent
		612	* node becomes 4--6.
		613	*/
		614	static __always_inline void
		615	augment_tree_propagate_from(struct vmap_area *va)
		616	{
		617	struct rb_node *node = &va->rb_node;
		618	unsigned long new_va_sub_max_size;
		619
		620	while (node) {
		621	va = rb_entry(node, struct vmap_area, rb_node);
		622	new_va_sub_max_size = compute_subtree_max_size(va);
		623
		624	/*
		625	* If the newly calculated maximum available size of the
		626	* subtree is equal to the current one, then it means that
		627	* the tree is propagated correctly. So we have to stop at
		628	* this point to save cycles.
		629	*/
		630	if (va->subtree_max_size == new_va_sub_max_size)
		631	break;
		632
		633	va->subtree_max_size = new_va_sub_max_size;
		634	node = rb_parent(&va->rb_node);
		635	}
		636
		637	#if DEBUG_AUGMENT_PROPAGATE_CHECK
		638	augment_tree_propagate_check(free_vmap_area_root.rb_node);
		639	#endif
		640	}
		641
		642	static void
		643	insert_vmap_area(struct vmap_area *va,
		644	struct rb_root root, struct list_head head)
		645	{
		646	struct rb_node **link;
		647	struct rb_node *parent;
		648
		649	link = find_va_links(va, root, NULL, &parent);
		650	link_va(va, root, parent, link, head);
		651	}
		652
		653	static void
		654	insert_vmap_area_augment(struct vmap_area *va,
		655	struct rb_node from, struct rb_root root,
		656	struct list_head *head)
		657	{
		658	struct rb_node **link;
		659	struct rb_node *parent;
		660
		661	if (from)
		662	link = find_va_links(va, NULL, from, &parent);
		663	else
		664	link = find_va_links(va, root, NULL, &parent);
		665
		666	link_va(va, root, parent, link, head);
		667	augment_tree_propagate_from(va);
		668	}
		669
		670	/*
		671	* Merge de-allocated chunk of VA memory with previous
		672	* and next free blocks. If coalesce is not done a new
		673	* free area is inserted. If VA has been merged, it is
		674	* freed.
		675	*/
		676	static __always_inline void
		677	merge_or_add_vmap_area(struct vmap_area *va,
		678	struct rb_root root, struct list_head head)
		679	{
		680	struct vmap_area *sibling;
		681	struct list_head *next;
		682	struct rb_node **link;
		683	struct rb_node *parent;
		684	bool merged = false;
		685
		686	/*
		687	* Find a place in the tree where VA potentially will be
		688	* inserted, unless it is merged with its sibling/siblings.
		689	*/
		690	link = find_va_links(va, root, NULL, &parent);
		691
		692	/*
		693	* Get next node of VA to check if merging can be done.
		694	*/
		695	next = get_va_next_sibling(parent, link);
		696	if (unlikely(next == NULL))
		697	goto insert;
		698
		699	/*
		700	* start end
		701	* \| \|
		702	* \|<------VA------>\|<-----Next----->\|
		703	* \| \|
		704	* start end
		705	*/
		706	if (next != head) {
		707	sibling = list_entry(next, struct vmap_area, list);
		708	if (sibling->va_start == va->va_end) {
		709	sibling->va_start = va->va_start;
		710
		711	/* Check and update the tree if needed. */
		712	augment_tree_propagate_from(sibling);
		713
		714	/* Remove this VA, it has been merged. */
		715	unlink_va(va, root);
		716
		717	/* Free vmap_area object. */
		718	kmem_cache_free(vmap_area_cachep, va);
		719
		720	/* Point to the new merged area. */
		721	va = sibling;
		722	merged = true;
		723	}
		724	}
		725
		726	/*
		727	* start end
		728	* \| \|
		729	* \|<-----Prev----->\|<------VA------>\|
		730	* \| \|
		731	* start end
		732	*/
		733	if (next->prev != head) {
		734	sibling = list_entry(next->prev, struct vmap_area, list);
		735	if (sibling->va_end == va->va_start) {
		736	sibling->va_end = va->va_end;
		737
		738	/* Check and update the tree if needed. */
		739	augment_tree_propagate_from(sibling);
		740
		741	/* Remove this VA, it has been merged. */
		742	unlink_va(va, root);
		743
		744	/* Free vmap_area object. */
		745	kmem_cache_free(vmap_area_cachep, va);
		746
		747	return;
		748	}
		749	}
		750
		751	insert:
		752	if (!merged) {
		753	link_va(va, root, parent, link, head);
		754	augment_tree_propagate_from(va);
		755	}
		756	}
		757
		758	static __always_inline bool
		759	is_within_this_va(struct vmap_area *va, unsigned long size,
		760	unsigned long align, unsigned long vstart)
		761	{
		762	unsigned long nva_start_addr;
		763
		764	if (va->va_start > vstart)
		765	nva_start_addr = ALIGN(va->va_start, align);
		766	else
		767	nva_start_addr = ALIGN(vstart, align);
		768
		769	/* Can be overflowed due to big size or alignment. */
		770	if (nva_start_addr + size < nva_start_addr \|\|
		771	nva_start_addr < vstart)
		772	return false;
		773
		774	return (nva_start_addr + size <= va->va_end);
		775	}
		776
		777	/*
		778	* Find the first free block(lowest start address) in the tree,
		779	* that will accomplish the request corresponding to passing
		780	* parameters.
		781	*/
		782	static __always_inline struct vmap_area *
		783	find_vmap_lowest_match(unsigned long size,
		784	unsigned long align, unsigned long vstart)
		785	{
		786	struct vmap_area *va;
		787	struct rb_node *node;
		788	unsigned long length;
		789
		790	/* Start from the root. */
		791	node = free_vmap_area_root.rb_node;
		792
		793	/* Adjust the search size for alignment overhead. */
		794	length = size + align - 1;
		795
		796	while (node) {
		797	va = rb_entry(node, struct vmap_area, rb_node);
		798
		799	if (get_subtree_max_size(node->rb_left) >= length &&
		800	vstart < va->va_start) {
		801	node = node->rb_left;
		802	} else {
		803	if (is_within_this_va(va, size, align, vstart))
		804	return va;
		805
		806	/*
		807	* Does not make sense to go deeper towards the right
		808	* sub-tree if it does not have a free block that is
		809	* equal or bigger to the requested search length.
		810	*/
		811	if (get_subtree_max_size(node->rb_right) >= length) {
		812	node = node->rb_right;
		813	continue;
		814	}
		815
		816	/*
		817	* OK. We roll back and find the fist right sub-tree,
		818	* that will satisfy the search criteria. It can happen
		819	* only once due to "vstart" restriction.
		820	*/
		821	while ((node = rb_parent(node))) {
		822	va = rb_entry(node, struct vmap_area, rb_node);
		823	if (is_within_this_va(va, size, align, vstart))
		824	return va;
		825
		826	if (get_subtree_max_size(node->rb_right) >= length &&
		827	vstart <= va->va_start) {
		828	node = node->rb_right;
		829	break;
		830	}
		831	}
		832	}
		833	}
		834
		835	return NULL;
		836	}
		837
		838	#if DEBUG_AUGMENT_LOWEST_MATCH_CHECK
		839	#include <linux/random.h>
		840
		841	static struct vmap_area *
		842	find_vmap_lowest_linear_match(unsigned long size,
		843	unsigned long align, unsigned long vstart)
		844	{
		845	struct vmap_area *va;
		846
		847	list_for_each_entry(va, &free_vmap_area_list, list) {
		848	if (!is_within_this_va(va, size, align, vstart))
		849	continue;
		850
		851	return va;
		852	}
		853
		854	return NULL;
		855	}
		856
		857	static void
		858	find_vmap_lowest_match_check(unsigned long size)
		859	{
		860	struct vmap_area va_1, va_2;
		861	unsigned long vstart;
		862	unsigned int rnd;
		863
		864	get_random_bytes(&rnd, sizeof(rnd));
		865	vstart = VMALLOC_START + rnd;
		866
		867	va_1 = find_vmap_lowest_match(size, 1, vstart);
		868	va_2 = find_vmap_lowest_linear_match(size, 1, vstart);
		869
		870	if (va_1 != va_2)
		871	pr_emerg("not lowest: t: 0x%p, l: 0x%p, v: 0x%lx\n",
		872	va_1, va_2, vstart);
		873	}
		874	#endif
		875
		876	enum fit_type {
		877	NOTHING_FIT = 0,
		878	FL_FIT_TYPE = 1, /* full fit */
		879	LE_FIT_TYPE = 2, /* left edge fit */
		880	RE_FIT_TYPE = 3, /* right edge fit */
		881	NE_FIT_TYPE = 4 /* no edge fit */
		882	};
		883
		884	static __always_inline enum fit_type
		885	classify_va_fit_type(struct vmap_area *va,
		886	unsigned long nva_start_addr, unsigned long size)
		887	{
		888	enum fit_type type;
		889
		890	/* Check if it is within VA. */
		891	if (nva_start_addr < va->va_start \|\|
		892	nva_start_addr + size > va->va_end)
		893	return NOTHING_FIT;
		894
		895	/* Now classify. */
		896	if (va->va_start == nva_start_addr) {
		897	if (va->va_end == nva_start_addr + size)
		898	type = FL_FIT_TYPE;
		899	else
		900	type = LE_FIT_TYPE;
		901	} else if (va->va_end == nva_start_addr + size) {
		902	type = RE_FIT_TYPE;
		903	} else {
		904	type = NE_FIT_TYPE;
		905	}
		906
		907	return type;
		908	}
		909
		910	static __always_inline int
		911	adjust_va_to_fit_type(struct vmap_area *va,
		912	unsigned long nva_start_addr, unsigned long size,
		913	enum fit_type type)
		914	{
		915	struct vmap_area *lva;
		916
		917	if (type == FL_FIT_TYPE) {
		918	/*
		919	* No need to split VA, it fully fits.
		920	*
		921	* \| \|
		922	* V NVA V
		923	* \|---------------\|
		924	*/
		925	unlink_va(va, &free_vmap_area_root);
		926	kmem_cache_free(vmap_area_cachep, va);
		927	} else if (type == LE_FIT_TYPE) {
		928	/*
		929	* Split left edge of fit VA.
		930	*
		931	* \| \|
		932	* V NVA V R
		933	* \|-------\|-------\|
		934	*/
		935	va->va_start += size;
		936	} else if (type == RE_FIT_TYPE) {
		937	/*
		938	* Split right edge of fit VA.
		939	*
		940	* \| \|
		941	* L V NVA V
		942	* \|-------\|-------\|
		943	*/
		944	va->va_end = nva_start_addr;
		945	} else if (type == NE_FIT_TYPE) {
		946	/*
		947	* Split no edge of fit VA.
		948	*
		949	* \| \|
		950	* L V NVA V R
		951	* \|---\|-------\|---\|
		952	*/
		953	lva = kmem_cache_alloc(vmap_area_cachep, GFP_NOWAIT);
		954	if (unlikely(!lva))
		955	return -1;
		956
		957	/*
		958	* Build the remainder.
		959	*/
		960	lva->va_start = va->va_start;
		961	lva->va_end = nva_start_addr;
		962
		963	/*
		964	* Shrink this VA to remaining size.
		965	*/
		966	va->va_start = nva_start_addr + size;
		967	} else {
		968	return -1;
		969	}
		970
		971	if (type != FL_FIT_TYPE) {
		972	augment_tree_propagate_from(va);
		973
		974	if (type == NE_FIT_TYPE)
		975	insert_vmap_area_augment(lva, &va->rb_node,
		976	&free_vmap_area_root, &free_vmap_area_list);
		977	}
		978
		979	return 0;
		980	}
		981
		982	/*
		983	* Returns a start address of the newly allocated area, if success.
		984	* Otherwise a vend is returned that indicates failure.
		985	*/
		986	static __always_inline unsigned long
		987	__alloc_vmap_area(unsigned long size, unsigned long align,
		988	unsigned long vstart, unsigned long vend, int node)
		989	{
		990	unsigned long nva_start_addr;
		991	struct vmap_area *va;
		992	enum fit_type type;
		993	int ret;
		994
		995	va = find_vmap_lowest_match(size, align, vstart);
		996	if (unlikely(!va))
		997	return vend;
		998
		999	if (va->va_start > vstart)
		1000	nva_start_addr = ALIGN(va->va_start, align);
		1001	else
		1002	nva_start_addr = ALIGN(vstart, align);
		1003
		1004	/* Check the "vend" restriction. */
		1005	if (nva_start_addr + size > vend)
		1006	return vend;
		1007
		1008	/* Classify what we have found. */
		1009	type = classify_va_fit_type(va, nva_start_addr, size);
		1010	if (WARN_ON_ONCE(type == NOTHING_FIT))
		1011	return vend;
		1012
		1013	/* Update the free vmap_area. */
		1014	ret = adjust_va_to_fit_type(va, nva_start_addr, size, type);
		1015	if (ret)
		1016	return vend;
		1017
		1018	#if DEBUG_AUGMENT_LOWEST_MATCH_CHECK
		1019	find_vmap_lowest_match_check(size);
		1020	#endif
		1021
		1022	return nva_start_addr;
		1023	}
398		1024
399	/*	1025	/*
400	* Allocate a region of KVA of the specified size and alignment, within the	1026	* Allocate a region of KVA of the specified size and alignment, within the
@@ -406,18 +1032,19 @@ static struct vmap_area *alloc_vmap_area(unsigned long size, · 5 unmodified lines
406	int node, gfp_t gfp_mask)	1032	int node, gfp_t gfp_mask)
407	{	1033	{
408	struct vmap_area *va;	1034	struct vmap_area *va;
409	struct rb_node *n;
410	unsigned long addr;	1035	unsigned long addr;
411	int purged = 0;	1036	int purged = 0;
412	struct vmap_area *first;
413		1037
414	BUG_ON(!size);	1038	BUG_ON(!size);
415	BUG_ON(offset_in_page(size));	1039	BUG_ON(offset_in_page(size));
416	BUG_ON(!is_power_of_2(align));	1040	BUG_ON(!is_power_of_2(align));
417		1041
		1042	if (unlikely(!vmap_initialized))
		1043	return ERR_PTR(-EBUSY);
		1044
418	might_sleep();	1045	might_sleep();
419		1046
420	va = kmalloc_node(sizeof(struct vmap_area),	1047	va = kmem_cache_alloc_node(vmap_area_cachep,
421	gfp_mask & GFP_RECLAIM_MASK, node);	1048	gfp_mask & GFP_RECLAIM_MASK, node);
422	if (unlikely(!va))	1049	if (unlikely(!va))
423	return ERR_PTR(-ENOMEM);	1050	return ERR_PTR(-ENOMEM);
@@ -430,87 +1057,20 @@ static struct vmap_area *alloc_vmap_area(unsigned long size, · 6 unmodified lines
430		1057
431	retry:	1058	retry:
432	spin_lock(&vmap_area_lock);	1059	spin_lock(&vmap_area_lock);
433	/*
434	* Invalidate cache if we have more permissive parameters.
435	* cached_hole_size notes the largest hole noticed _below_
436	* the vmap_area cached in free_vmap_cache: if size fits
437	* into that hole, we want to scan from vstart to reuse
438	* the hole instead of allocating above free_vmap_cache.
439	* Note that __free_vmap_area may update free_vmap_cache
440	* without updating cached_hole_size or cached_align.
441	*/
442	if (!free_vmap_cache \|\|
443	size < cached_hole_size \|\|
444	vstart < cached_vstart \|\|
445	align < cached_align) {
446	nocache:
447	cached_hole_size = 0;
448	free_vmap_cache = NULL;
449	}
450	/* record if we encounter less permissive parameters */
451	cached_vstart = vstart;
452	cached_align = align;
453
454	/* find starting point for our search */
455	if (free_vmap_cache) {
456	first = rb_entry(free_vmap_cache, struct vmap_area, rb_node);
457	addr = ALIGN(first->va_end, align);
458	if (addr < vstart)
459	goto nocache;
460	if (addr + size < addr)
461	goto overflow;
462
463	} else {
464	addr = ALIGN(vstart, align);
465	if (addr + size < addr)
466	goto overflow;
467
468	n = vmap_area_root.rb_node;
469	first = NULL;
470
471	while (n) {
472	struct vmap_area *tmp;
473	tmp = rb_entry(n, struct vmap_area, rb_node);
474	if (tmp->va_end >= addr) {
475	first = tmp;
476	if (tmp->va_start <= addr)
477	break;
478	n = n->rb_left;
479	} else
480	n = n->rb_right;
481	}
482
483	if (!first)
484	goto found;
485	}
486
487	/* from the starting point, walk areas until a suitable hole is found */
488	while (addr + size > first->va_start && addr + size <= vend) {
489	if (addr + cached_hole_size < first->va_start)
490	cached_hole_size = first->va_start - addr;
491	addr = ALIGN(first->va_end, align);
492	if (addr + size < addr)
493	goto overflow;
494
495	if (list_is_last(&first->list, &vmap_area_list))
496	goto found;
497		1060
498	first = list_next_entry(first, list);
499	}
500
501	found:
502	/*	1061	/*
503	* Check also calculated address against the vstart,	1062	* If an allocation fails, the "vend" address is
504	* because it can be 0 because of big align request.	1063	* returned. Therefore trigger the overflow path.
505	*/	1064	*/
506	if (addr + size > vend \|\| addr < vstart)	1065	addr = __alloc_vmap_area(size, align, vstart, vend, node);
		1066	if (unlikely(addr == vend))
507	goto overflow;	1067	goto overflow;
508		1068
509	va->va_start = addr;	1069	va->va_start = addr;
510	va->va_end = addr + size;	1070	va->va_end = addr + size;
511	va->flags = 0;	1071	va->flags = 0;
512	__insert_vmap_area(va);	1072	insert_vmap_area(va, &vmap_area_root, &vmap_area_list);
513	free_vmap_cache = &va->rb_node;	1073
514	spin_unlock(&vmap_area_lock);	1074	spin_unlock(&vmap_area_lock);
515		1075
516	BUG_ON(!IS_ALIGNED(va->va_start, align));	1076	BUG_ON(!IS_ALIGNED(va->va_start, align));
@@ -539,7 +1099,8 @@ overflow: · 22 unmodified lines
539	if (!(gfp_mask & __GFP_NOWARN) && printk_ratelimit())	1099	if (!(gfp_mask & __GFP_NOWARN) && printk_ratelimit())
540	pr_warn("vmap allocation for size %lu failed: use vmalloc=<size> to increase size\n",	1100	pr_warn("vmap allocation for size %lu failed: use vmalloc=<size> to increase size\n",
541	size);	1101	size);
542	kfree(va);	1102
		1103	kmem_cache_free(vmap_area_cachep, va);
543	return ERR_PTR(-EBUSY);	1104	return ERR_PTR(-EBUSY);
544	}	1105	}
545		1106
@@ -559,35 +1120,16 @@ static void __free_vmap_area(struct vmap_area *va) · 13 unmodified lines
559	{	1120	{
560	BUG_ON(RB_EMPTY_NODE(&va->rb_node));	1121	BUG_ON(RB_EMPTY_NODE(&va->rb_node));
561		1122
562	if (free_vmap_cache) {
563	if (va->va_end < cached_vstart) {
564	free_vmap_cache = NULL;
565	} else {
566	struct vmap_area *cache;
567	cache = rb_entry(free_vmap_cache, struct vmap_area, rb_node);
568	if (va->va_start <= cache->va_start) {
569	free_vmap_cache = rb_prev(&va->rb_node);
570	/*
571	* We don't try to update cached_hole_size or
572	* cached_align, but it won't go very wrong.
573	*/
574	}
575	}
576	}
577	rb_erase(&va->rb_node, &vmap_area_root);
578	RB_CLEAR_NODE(&va->rb_node);
579	list_del_rcu(&va->list);
580
581	/*	1123	/*
582	* Track the highest possible candidate for pcpu area	1124	* Remove from the busy tree/list.
583	* allocation. Areas outside of vmalloc area can be returned
584	* here too, consider only end addresses which fall inside
585	* vmalloc area proper.
586	*/	1125	*/
587	if (va->va_end > VMALLOC_START && va->va_end <= VMALLOC_END)	1126	unlink_va(va, &vmap_area_root);
588	vmap_area_pcpu_hole = max(vmap_area_pcpu_hole, va->va_end);
589		1127
590	kfree_rcu(va, rcu_head);	1128	/*
		1129	* Merge VA with its neighbors, otherwise just add it.
		1130	*/
		1131	merge_or_add_vmap_area(va,
		1132	&free_vmap_area_root, &free_vmap_area_list);
591	}	1133	}
592		1134
593	/*	1135	/*
@@ -794,8 +1336,6 @@ static struct vmap_area *find_vmap_area(unsigned long addr) · 200 unmodified lines
794		1336
795	#define VMAP_BLOCK_SIZE (VMAP_BBMAP_BITS * PAGE_SIZE)	1337	#define VMAP_BLOCK_SIZE (VMAP_BBMAP_BITS * PAGE_SIZE)
796		1338
797	static bool vmap_initialized __read_mostly = false;
798
799	struct vmap_block_queue {	1339	struct vmap_block_queue {
800	spinlock_t lock;	1340	spinlock_t lock;
801	struct list_head free;	1341	struct list_head free;
@@ -1256,12 +1796,58 @@ void __init vm_area_register_early(struct vm_struct *vm, size_t align) · 454 unmodified lines
1256	vm_area_add_early(vm);	1796	vm_area_add_early(vm);
1257	}	1797	}
1258		1798
		1799	static void vmap_init_free_space(void)
		1800	{
		1801	unsigned long vmap_start = 1;
		1802	const unsigned long vmap_end = ULONG_MAX;
		1803	struct vmap_area busy, free;
		1804
		1805	/*
		1806	* B F B B B F
		1807	* -\|-----\|.....\|-----\|-----\|-----\|.....\|-
		1808	* \| The KVA space \|
		1809	* \|<--------------------------------->\|
		1810	*/
		1811	list_for_each_entry(busy, &vmap_area_list, list) {
		1812	if (busy->va_start - vmap_start > 0) {
		1813	free = kmem_cache_zalloc(vmap_area_cachep, GFP_NOWAIT);
		1814	if (!WARN_ON_ONCE(!free)) {
		1815	free->va_start = vmap_start;
		1816	free->va_end = busy->va_start;
		1817
		1818	insert_vmap_area_augment(free, NULL,
		1819	&free_vmap_area_root,
		1820	&free_vmap_area_list);
		1821	}
		1822	}
		1823
		1824	vmap_start = busy->va_end;
		1825	}
		1826
		1827	if (vmap_end - vmap_start > 0) {
		1828	free = kmem_cache_zalloc(vmap_area_cachep, GFP_NOWAIT);
		1829	if (!WARN_ON_ONCE(!free)) {
		1830	free->va_start = vmap_start;
		1831	free->va_end = vmap_end;
		1832
		1833	insert_vmap_area_augment(free, NULL,
		1834	&free_vmap_area_root,
		1835	&free_vmap_area_list);
		1836	}
		1837	}
		1838	}
		1839
1259	void __init vmalloc_init(void)	1840	void __init vmalloc_init(void)
1260	{	1841	{
1261	struct vmap_area *va;	1842	struct vmap_area *va;
1262	struct vm_struct *tmp;	1843	struct vm_struct *tmp;
1263	int i;	1844	int i;
1264		1845
		1846	/*
		1847	* Create the cache for vmap_area objects.
		1848	*/
		1849	vmap_area_cachep = KMEM_CACHE(vmap_area, SLAB_PANIC);
		1850
1265	for_each_possible_cpu(i) {	1851	for_each_possible_cpu(i) {
1266	struct vmap_block_queue *vbq;	1852	struct vmap_block_queue *vbq;
1267	struct vfree_deferred *p;	1853	struct vfree_deferred *p;
@@ -1276,16 +1862,21 @@ void __init vmalloc_init(void) · 8 unmodified lines
1276		1862
1277	/* Import existing vmlist entries. */	1863	/* Import existing vmlist entries. */
1278	for (tmp = vmlist; tmp; tmp = tmp->next) {	1864	for (tmp = vmlist; tmp; tmp = tmp->next) {
1279	va = kzalloc(sizeof(struct vmap_area), GFP_NOWAIT);	1865	va = kmem_cache_zalloc(vmap_area_cachep, GFP_NOWAIT);
		1866	if (WARN_ON_ONCE(!va))
		1867	continue;
		1868
1280	va->flags = VM_VM_AREA;	1869	va->flags = VM_VM_AREA;
1281	va->va_start = (unsigned long)tmp->addr;	1870	va->va_start = (unsigned long)tmp->addr;
1282	va->va_end = va->va_start + tmp->size;	1871	va->va_end = va->va_start + tmp->size;
1283	va->vm = tmp;	1872	va->vm = tmp;
1284	__insert_vmap_area(va);	1873	insert_vmap_area(va, &vmap_area_root, &vmap_area_list);
1285	}	1874	}
1286		1875
1287	vmap_area_pcpu_hole = VMALLOC_END;	1876	/*
1288		1877	* Now we can initialize a free vmap space.
		1878	*/
		1879	vmap_init_free_space();
1289	vmap_initialized = true;	1880	vmap_initialized = true;
1290	}	1881	}
1291		1882
@@ -2477,81 +3068,64 @@ static struct vmap_area node_to_va(struct rb_node n) · 1185 unmodified lines
2477	}	3068	}
2478		3069
2479	/**	3070	/**
2480	* pvm_find_next_prev - find the next and prev vmap_area surrounding @end	3071	* pvm_find_va_enclose_addr - find the vmap_area @addr belongs to
2481	* @end: target address	3072	* @addr: target address
2482	* @pnext: out arg for the next vmap_area
2483	* @pprev: out arg for the previous vmap_area
2484	*
2485	* Returns: %true if either or both of next and prev are found,
2486	* %false if no vmap_area exists
2487	*	3073	*
2488	* Find vmap_areas end addresses of which enclose @end. ie. if not	3074	* Returns: vmap_area if it is found. If there is no such area
2489	* NULL, pnext->va_end > @end and pprev->va_end <= @end.	3075	* the first highest(reverse order) vmap_area is returned
		3076	* i.e. va->va_start < addr && va->va_end < addr or NULL
		3077	* if there are no any areas before @addr.
2490	*/	3078	*/
2491	static bool pvm_find_next_prev(unsigned long end,	3079	static struct vmap_area *
2492	struct vmap_area **pnext,	3080	pvm_find_va_enclose_addr(unsigned long addr)
2493	struct vmap_area **pprev)
2494	{	3081	{
2495	struct rb_node *n = vmap_area_root.rb_node;	3082	struct vmap_area va, tmp;
2496	struct vmap_area *va = NULL;	3083	struct rb_node *n;
		3084
		3085	n = free_vmap_area_root.rb_node;
		3086	va = NULL;
2497		3087
2498	while (n) {	3088	while (n) {
2499	va = rb_entry(n, struct vmap_area, rb_node);	3089	tmp = rb_entry(n, struct vmap_area, rb_node);
2500	if (end < va->va_end)	3090	if (tmp->va_start <= addr) {
2501	n = n->rb_left;	3091	va = tmp;
2502	else if (end > va->va_end)	3092	if (tmp->va_end >= addr)
		3093	break;
		3094
2503	n = n->rb_right;	3095	n = n->rb_right;
2504	else	3096	} else {
2505	break;	3097	n = n->rb_left;
		3098	}
2506	}	3099	}
2507		3100
2508	if (!va)	3101	return va;
2509	return false;
2510
2511	if (va->va_end > end) {
2512	*pnext = va;
2513	pprev = node_to_va(rb_prev(&(pnext)->rb_node));
2514	} else {
2515	*pprev = va;
2516	pnext = node_to_va(rb_next(&(pprev)->rb_node));
2517	}
2518	return true;
2519	}	3102	}
2520		3103
2521	/**	3104	/**
2522	* pvm_determine_end - find the highest aligned address between two vmap_areas	3105	* pvm_determine_end_from_reverse - find the highest aligned address
2523	* @pnext: in/out arg for the next vmap_area	3106	* of free block below VMALLOC_END
2524	* @pprev: in/out arg for the previous vmap_area	3107	* @va:
2525	* @align: alignment	3108	* in - the VA we start the search(reverse order);
		3109	* out - the VA with the highest aligned end address.
2526	*	3110	*
2527	* Returns: determined end address	3111	* Returns: determined end address within vmap_area
2528	*
2529	* Find the highest aligned address between @pnext and @pprev below
2530	* VMALLOC_END. @pnext and @pprev are adjusted so that the aligned
2531	* down address is between the end addresses of the two vmap_areas.
2532	*
2533	* Please note that the address returned by this function may fall
2534	* inside *@pnext vmap_area. The caller is responsible for checking
2535	* that.
2536	*/	3112	*/
2537	static unsigned long pvm_determine_end(struct vmap_area **pnext,	3113	static unsigned long
2538	struct vmap_area **pprev,	3114	pvm_determine_end_from_reverse(struct vmap_area **va, unsigned long align)
2539	unsigned long align)
2540	{	3115	{
2541	const unsigned long vmalloc_end = VMALLOC_END & ~(align - 1);	3116	unsigned long vmalloc_end = VMALLOC_END & ~(align - 1);
2542	unsigned long addr;	3117	unsigned long addr;
2543		3118
2544	if (*pnext)	3119	if (likely(*va)) {
2545	addr = min((*pnext)->va_start & ~(align - 1), vmalloc_end);	3120	list_for_each_entry_from_reverse((*va),
2546	else	3121	&free_vmap_area_list, list) {
2547	addr = vmalloc_end;	3122	addr = min((*va)->va_end & ~(align - 1), vmalloc_end);
2548		3123	if ((*va)->va_start < addr)
2549	while (pprev && (pprev)->va_end > addr) {	3124	return addr;
2550	pnext = pprev;	3125	}
2551	pprev = node_to_va(rb_prev(&(pnext)->rb_node));
2552	}	3126	}
2553		3127
2554	return addr;	3128	return 0;
2555	}	3129	}
2556		3130
2557	/**	3131	/**
@@ -2571,12 +3145,12 @@ static unsigned long pvm_determine_end(struct vmap_area **pnext, · 13 unmodified lines
2571	* to gigabytes. To avoid interacting with regular vmallocs, these	3145	* to gigabytes. To avoid interacting with regular vmallocs, these
2572	* areas are allocated from top.	3146	* areas are allocated from top.
2573	*	3147	*
2574	* Despite its complicated look, this allocator is rather simple. It	3148	* Despite its complicated look, this allocator is rather simple. It
2575	* does everything top-down and scans areas from the end looking for	3149	* does everything top-down and scans free blocks from the end looking
2576	* matching slot. While scanning, if any of the areas overlaps with	3150	* for matching base. While scanning, if any of the areas do not fit the
2577	* existing vmap_area, the base address is pulled down to fit the	3151	* base address is pulled down to fit the area. Scanning is repeated till
2578	* area. Scanning is repeated till all the areas fit and then all	3152	* all the areas fit and then all necessary data structures are inserted
2579	* necessary data structures are inserted and the result is returned.	3153	* and the result is returned.
2580	*/	3154	*/
2581	struct vm_struct *pcpu_get_vm_areas(const unsigned long offsets,	3155	struct vm_struct *pcpu_get_vm_areas(const unsigned long offsets,
2582	const size_t *sizes, int nr_vms,	3156	const size_t *sizes, int nr_vms,
@@ -2584,11 +3158,12 @@ struct vm_struct *pcpu_get_vm_areas(const unsigned long offsets, · 1 unmodified line
2584	{	3158	{
2585	const unsigned long vmalloc_start = ALIGN(VMALLOC_START, align);	3159	const unsigned long vmalloc_start = ALIGN(VMALLOC_START, align);
2586	const unsigned long vmalloc_end = VMALLOC_END & ~(align - 1);	3160	const unsigned long vmalloc_end = VMALLOC_END & ~(align - 1);
2587	struct vmap_area *vas, prev, *next;	3161	struct vmap_area *vas, va;
2588	struct vm_struct **vms;	3162	struct vm_struct **vms;
2589	int area, area2, last_area, term_area;	3163	int area, area2, last_area, term_area;
2590	unsigned long base, start, end, last_end;	3164	unsigned long base, start, size, end, last_end;
2591	bool purged = false;	3165	bool purged = false;
		3166	enum fit_type type;
2592		3167
2593	/* verify parameters and allocate data structures */	3168	/* verify parameters and allocate data structures */
2594	BUG_ON(offset_in_page(align) \|\| !is_power_of_2(align));	3169	BUG_ON(offset_in_page(align) \|\| !is_power_of_2(align));
@@ -2624,7 +3199,7 @@ struct vm_struct *pcpu_get_vm_areas(const unsigned long offsets, · 29 unmodified lines
2624	goto err_free2;	3199	goto err_free2;
2625		3200
2626	for (area = 0; area < nr_vms; area++) {	3201	for (area = 0; area < nr_vms; area++) {
2627	vas[area] = kzalloc(sizeof(struct vmap_area), GFP_KERNEL);	3202	vas[area] = kmem_cache_zalloc(vmap_area_cachep, GFP_KERNEL);
2628	vms[area] = kzalloc(sizeof(struct vm_struct), GFP_KERNEL);	3203	vms[area] = kzalloc(sizeof(struct vm_struct), GFP_KERNEL);
2629	if (!vas[area] \|\| !vms[area])	3204	if (!vas[area] \|\| !vms[area])
2630	goto err_free;	3205	goto err_free;
@@ -2637,49 +3212,29 @@ retry: · 6 unmodified lines
2637	start = offsets[area];	3212	start = offsets[area];
2638	end = start + sizes[area];	3213	end = start + sizes[area];
2639		3214
2640	if (!pvm_find_next_prev(vmap_area_pcpu_hole, &next, &prev)) {	3215	va = pvm_find_va_enclose_addr(vmalloc_end);
2641	base = vmalloc_end - last_end;	3216	base = pvm_determine_end_from_reverse(&va, align) - end;
2642	goto found;
2643	}
2644	base = pvm_determine_end(&next, &prev, align) - end;
2645		3217
2646	while (true) {	3218	while (true) {
2647	BUG_ON(next && next->va_end <= base + end);
2648	BUG_ON(prev && prev->va_end > base + end);
2649
2650	/*	3219	/*
2651	* base might have underflowed, add last_end before	3220	* base might have underflowed, add last_end before
2652	* comparing.	3221	* comparing.
2653	*/	3222	*/
2654	if (base + last_end < vmalloc_start + last_end) {	3223	if (base + last_end < vmalloc_start + last_end)
2655	spin_unlock(&vmap_area_lock);	3224	goto overflow;
2656	if (!purged) {
2657	purge_vmap_area_lazy();
2658	purged = true;
2659	goto retry;
2660	}
2661	goto err_free;
2662	}
2663		3225
2664	/*	3226	/*
2665	* If next overlaps, move base downwards so that it's	3227	* Fitting base has not been found.
2666	* right below next and then recheck.
2667	*/	3228	*/
2668	if (next && next->va_start < base + end) {	3229	if (va == NULL)
2669	base = pvm_determine_end(&next, &prev, align) - end;	3230	goto overflow;
2670	term_area = area;
2671	continue;
2672	}
2673		3231
2674	/*	3232	/*
2675	* If prev overlaps, shift down next and prev and move	3233	* If this VA does not fit, move base downwards and recheck.
2676	* base so that it's right below new next and then
2677	* recheck.
2678	*/	3234	*/
2679	if (prev && prev->va_end > base + start) {	3235	if (base + start < va->va_start \|\| base + end > va->va_end) {
2680	next = prev;	3236	va = node_to_va(rb_prev(&va->rb_node));
2681	prev = node_to_va(rb_prev(&next->rb_node));	3237	base = pvm_determine_end_from_reverse(&va, align) - end;
2682	base = pvm_determine_end(&next, &prev, align) - end;
2683	term_area = area;	3238	term_area = area;
2684	continue;	3239	continue;
2685	}	3240	}
@@ -2691,21 +3246,40 @@ retry: · 5 unmodified lines
2691	area = (area + nr_vms - 1) % nr_vms;	3246	area = (area + nr_vms - 1) % nr_vms;
2692	if (area == term_area)	3247	if (area == term_area)
2693	break;	3248	break;
		3249
2694	start = offsets[area];	3250	start = offsets[area];
2695	end = start + sizes[area];	3251	end = start + sizes[area];
2696	pvm_find_next_prev(base + end, &next, &prev);	3252	va = pvm_find_va_enclose_addr(base + end);
2697	}	3253	}
2698	found:	3254
2699	/* we've found a fitting base, insert all va's */	3255	/* we've found a fitting base, insert all va's */
2700	for (area = 0; area < nr_vms; area++) {	3256	for (area = 0; area < nr_vms; area++) {
2701	struct vmap_area *va = vas[area];	3257	int ret;
2702		3258
2703	va->va_start = base + offsets[area];	3259	start = base + offsets[area];
2704	va->va_end = va->va_start + sizes[area];	3260	size = sizes[area];
2705	__insert_vmap_area(va);	3261
2706	}	3262	va = pvm_find_va_enclose_addr(start);
		3263	if (WARN_ON_ONCE(va == NULL))
		3264	/* It is a BUG(), but trigger recovery instead. */
		3265	goto recovery;
		3266
		3267	type = classify_va_fit_type(va, start, size);
		3268	if (WARN_ON_ONCE(type == NOTHING_FIT))
		3269	/* It is a BUG(), but trigger recovery instead. */
		3270	goto recovery;
		3271
		3272	ret = adjust_va_to_fit_type(va, start, size, type);
		3273	if (unlikely(ret))
		3274	goto recovery;
2707		3275
2708	vmap_area_pcpu_hole = base + offsets[last_area];	3276	/* Allocated area. */
		3277	va = vas[area];
		3278	va->va_start = start;
		3279	va->va_end = start + size;
		3280
		3281	insert_vmap_area(va, &vmap_area_root, &vmap_area_list);
		3282	}
2709		3283
2710	spin_unlock(&vmap_area_lock);	3284	spin_unlock(&vmap_area_lock);
2711		3285
@@ -2717,9 +3291,38 @@ found: · 5 unmodified lines
2717	kfree(vas);	3291	kfree(vas);
2718	return vms;	3292	return vms;
2719		3293
		3294	recovery:
		3295	/* Remove previously inserted areas. */
		3296	while (area--) {
		3297	__free_vmap_area(vas[area]);
		3298	vas[area] = NULL;
		3299	}
		3300
		3301	overflow:
		3302	spin_unlock(&vmap_area_lock);
		3303	if (!purged) {
		3304	purge_vmap_area_lazy();
		3305	purged = true;
		3306
		3307	/* Before "retry", check if we recover. */
		3308	for (area = 0; area < nr_vms; area++) {
		3309	if (vas[area])
		3310	continue;
		3311
		3312	vas[area] = kmem_cache_zalloc(
		3313	vmap_area_cachep, GFP_KERNEL);
		3314	if (!vas[area])
		3315	goto err_free;
		3316	}
		3317
		3318	goto retry;
		3319	}
		3320
2720	err_free:	3321	err_free:
2721	for (area = 0; area < nr_vms; area++) {	3322	for (area = 0; area < nr_vms; area++) {
2722	kfree(vas[area]);	3323	if (vas[area])
		3324	kmem_cache_free(vmap_area_cachep, vas[area]);
		3325
2723	kfree(vms[area]);	3326	kfree(vms[area]);
2724	}	3327	}
2725	err_free2:	3328	err_free2: