Merge branch 'akpm' (patches from Andrew)

Merge yet more updates from Andrew Morton: "A few final bits: - large changes to vmalloc, yielding large performance benefits - tweak the console-flush-on-panic code - a few fixes" * emailed patches from Andrew Morton <akpm@linux-foundation.org>: panic: add an option to replay all the printk message in buffer initramfs: don't free a non-existent initrd fs/writeback.c: use rcu_barrier() to wait for inflight wb switches going into workqueue when umount mm/compaction.c: correct zone boundary handling when isolating pages from a pageblock mm/vmap: add DEBUG_AUGMENT_LOWEST_MATCH_CHECK macro mm/vmap: add DEBUG_AUGMENT_PROPAGATE_CHECK macro mm/vmalloc.c: keep track of free blocks for vmap allocation

Linus Torvalds committed 7 years ago

Commit cb6f8732 parents ff8583d de6da1e10 files · +889 −257Browse files

Unified · Split

10 files

Documentation/admin-guide/kernel-parameters.txt +1 −0
arch/powerpc/kernel/traps.c +1 −1
fs/fs-writeback.c +8 −3
include/linux/console.h +6 −1
include/linux/vmalloc.h +5 −1
init/initramfs.c +1 −1
kernel/panic.c +5 −1
kernel/printk/printk.c +11 −1
mm/compaction.c +2 −2
mm/vmalloc.c +849 −246

Documentation/admin-guide/kernel-parameters.txt+1−0

@@ -3212,6 +3212,7 @@ · 3211 unmodified lines
3212	3212	bit 2: print timer info
3213	3213	bit 3: print locks info if CONFIG_LOCKDEP is on
3214	3214	bit 4: print ftrace buffer
	3215	+ bit 5: print all printk messages in buffer
3215	3216
3216	3217	panic_on_warn panic() instead of WARN(). Useful to cause kdump
3217	3218	on a WARN().

arch/powerpc/kernel/traps.c+1−1

@@ -179,7 +179,7 @@ extern void panic_flush_kmsg_end(void) · 178 unmodified lines
179	179	kmsg_dump(KMSG_DUMP_PANIC);
180	180	bust_spinlocks(0);
181	181	debug_locks_off();
182		− console_flush_on_panic();
	182	+ console_flush_on_panic(CONSOLE_FLUSH_PENDING);
183	183	}
184	184
185	185	static unsigned long oops_begin(struct pt_regs *regs)

fs/fs-writeback.c+8−3

@@ -523,8 +523,6 @@ static void inode_switch_wbs(struct inode *inode, int new_wb_id) · 522 unmodified lines
523	523
524	524	isw->inode = inode;
525	525
526		− atomic_inc(&isw_nr_in_flight);
527		−
528	526	/*
529	527	* In addition to synchronizing among switchers, I_WB_SWITCH tells
530	528	* the RCU protected stat update paths to grab the i_page
@@ -532,6 +530,9 @@ static void inode_switch_wbs(struct inode *inode, int new_wb_id) · 1 unmodified line
532	530	* Let's continue after I_WB_SWITCH is guaranteed to be visible.
533	531	*/
534	532	call_rcu(&isw->rcu_head, inode_switch_wbs_rcu_fn);
	533	+
	534	+ atomic_inc(&isw_nr_in_flight);
	535	+
535	536	goto out_unlock;
536	537
537	538	out_free:
@@ -901,7 +902,11 @@ restart: · 363 unmodified lines
901	902	void cgroup_writeback_umount(void)
902	903	{
903	904	if (atomic_read(&isw_nr_in_flight)) {
904		− synchronize_rcu();
	905	+ /*
	906	+ * Use rcu_barrier() to wait for all pending callbacks to
	907	+ * ensure that all in-flight wb switches are in the workqueue.
	908	+ */
	909	+ rcu_barrier();
905	910	flush_workqueue(isw_wq);
906	911	}
907	912	}

include/linux/console.h+6−1

@@ -166,6 +166,11 @@ struct console { · 165 unmodified lines
166	166	extern int console_set_on_cmdline;
167	167	extern struct console *early_console;
168	168
	169	+enum con_flush_mode {
	170	+ CONSOLE_FLUSH_PENDING,
	171	+ CONSOLE_REPLAY_ALL,
	172	+};
	173	+
169	174	extern int add_preferred_console(char name, int idx, char options);
170	175	extern void register_console(struct console *);
171	176	extern int unregister_console(struct console *);
@@ -175,7 +180,7 @@ extern int console_trylock(void); · 3 unmodified lines
175	180	extern void console_unlock(void);
176	181	extern void console_conditional_schedule(void);
177	182	extern void console_unblank(void);
178		−extern void console_flush_on_panic(void);
	183	+extern void console_flush_on_panic(enum con_flush_mode mode);
179	184	extern struct tty_driver console_device(int );
180	185	extern void console_stop(struct console *);
181	186	extern void console_start(struct console *);

include/linux/vmalloc.h+5−1

@@ -50,12 +50,16 @@ struct vm_struct { · 49 unmodified lines
50	50	struct vmap_area {
51	51	unsigned long va_start;
52	52	unsigned long va_end;
	53	+
	54	+ /*
	55	+ * Largest available free size in subtree.
	56	+ */
	57	+ unsigned long subtree_max_size;
53	58	unsigned long flags;
54	59	struct rb_node rb_node; /* address sorted rbtree */
55	60	struct list_head list; /* address sorted list */
56	61	struct llist_node purge_list; /* "lazy purge" list */
57	62	struct vm_struct *vm;
58		− struct rcu_head rcu_head;
59	63	};
60	64
61	65	/*

init/initramfs.c+1−1

@@ -669,7 +669,7 @@ done: · 668 unmodified lines
669	669	* If the initrd region is overlapped with crashkernel reserved region,
670	670	* free only memory that is not part of crashkernel region.
671	671	*/
672		− if (!do_retain_initrd && !kexec_free_initrd())
	672	+ if (!do_retain_initrd && initrd_start && !kexec_free_initrd())
673	673	free_initrd_mem(initrd_start, initrd_end);
674	674	initrd_start = 0;
675	675	initrd_end = 0;

kernel/panic.c+5−1

@@ -51,6 +51,7 @@ EXPORT_SYMBOL_GPL(panic_timeout); · 50 unmodified lines
51	51	#define PANIC_PRINT_TIMER_INFO 0x00000004
52	52	#define PANIC_PRINT_LOCK_INFO 0x00000008
53	53	#define PANIC_PRINT_FTRACE_INFO 0x00000010
	54	+#define PANIC_PRINT_ALL_PRINTK_MSG 0x00000020
54	55	unsigned long panic_print;
55	56
56	57	ATOMIC_NOTIFIER_HEAD(panic_notifier_list);
@@ -134,6 +135,9 @@ EXPORT_SYMBOL(nmi_panic); · 77 unmodified lines
134	135
135	136	static void panic_print_sys_info(void)
136	137	{
	138	+ if (panic_print & PANIC_PRINT_ALL_PRINTK_MSG)
	139	+ console_flush_on_panic(CONSOLE_REPLAY_ALL);
	140	+
137	141	if (panic_print & PANIC_PRINT_TASK_INFO)
138	142	show_state();
139	143
@@ -277,7 +281,7 @@ void panic(const char *fmt, ...) · 137 unmodified lines
277	281	* panic() is not being callled from OOPS.
278	282	*/
279	283	debug_locks_off();
280		− console_flush_on_panic();
	284	+ console_flush_on_panic(CONSOLE_FLUSH_PENDING);
281	285
282	286	panic_print_sys_info();
283	287

kernel/printk/printk.c+11−1

@@ -2535,10 +2535,11 @@ void console_unblank(void) · 2534 unmodified lines
2535	2535
2536	2536	/**
2537	2537	* console_flush_on_panic - flush console content on panic
	2538	+ * @mode: flush all messages in buffer or just the pending ones
2538	2539	*
2539	2540	* Immediately output all pending messages no matter what.
2540	2541	*/
2541		−void console_flush_on_panic(void)
	2542	+void console_flush_on_panic(enum con_flush_mode mode)
2542	2543	{
2543	2544	/*
2544	2545	* If someone else is holding the console lock, trylock will fail
@@ -2549,6 +2550,15 @@ void console_flush_on_panic(void) · 4 unmodified lines
2549	2550	*/
2550	2551	console_trylock();
2551	2552	console_may_schedule = 0;
	2553	+
	2554	+ if (mode == CONSOLE_REPLAY_ALL) {
	2555	+ unsigned long flags;
	2556	+
	2557	+ logbuf_lock_irqsave(flags);
	2558	+ console_seq = log_first_seq;
	2559	+ console_idx = log_first_idx;
	2560	+ logbuf_unlock_irqrestore(flags);
	2561	+ }
2552	2562	console_unlock();
2553	2563	}
2554	2564

mm/compaction.c+2−2

@@ -1230,7 +1230,7 @@ fast_isolate_around(struct compact_control *cc, unsigned long pfn, unsigned long · 1229 unmodified lines
1230	1230
1231	1231	/* Pageblock boundaries */
1232	1232	start_pfn = pageblock_start_pfn(pfn);
1233		− end_pfn = min(start_pfn + pageblock_nr_pages, zone_end_pfn(cc->zone));
	1233	+ end_pfn = min(pageblock_end_pfn(pfn), zone_end_pfn(cc->zone)) - 1;
1234	1234
1235	1235	/* Scan before */
1236	1236	if (start_pfn != pfn) {
@@ -1241,7 +1241,7 @@ fast_isolate_around(struct compact_control *cc, unsigned long pfn, unsigned long · 4 unmodified lines
1241	1241
1242	1242	/* Scan after */
1243	1243	start_pfn = pfn + nr_isolated;
1244		− if (start_pfn != end_pfn)
	1244	+ if (start_pfn < end_pfn)
1245	1245	isolate_freepages_block(cc, &start_pfn, end_pfn, &cc->freepages, 1, false);
1246	1246
1247	1247	/* Skip this pageblock in the future as it's full or nearly full */

mm/vmalloc.c+849−246

@@ -32,6 +32,7 @@ · 31 unmodified lines
32	32	#include <linux/compiler.h>
33	33	#include <linux/llist.h>
34	34	#include <linux/bitops.h>
	35	+#include <linux/rbtree_augmented.h>
35	36
36	37	#include <linux/uaccess.h>
37	38	#include <asm/tlbflush.h>
@@ -324,6 +325,9 @@ EXPORT_SYMBOL(vmalloc_to_pfn); · 286 unmodified lines
324	325
325	326	/* Global kva allocator */
326	327
	328	+#define DEBUG_AUGMENT_PROPAGATE_CHECK 0
	329	+#define DEBUG_AUGMENT_LOWEST_MATCH_CHECK 0
	330	+
327	331	#define VM_LAZY_FREE 0x02
328	332	#define VM_VM_AREA 0x04
329	333
@@ -332,14 +336,67 @@ static DEFINE_SPINLOCK(vmap_area_lock); · 2 unmodified lines
332	336	LIST_HEAD(vmap_area_list);
333	337	static LLIST_HEAD(vmap_purge_list);
334	338	static struct rb_root vmap_area_root = RB_ROOT;
	339	+static bool vmap_initialized __read_mostly;
	340	+
	341	+/*
	342	+ * This kmem_cache is used for vmap_area objects. Instead of
	343	+ * allocating from slab we reuse an object from this cache to
	344	+ * make things faster. Especially in "no edge" splitting of
	345	+ * free block.
	346	+ */
	347	+static struct kmem_cache *vmap_area_cachep;
	348	+
	349	+/*
	350	+ * This linked list is used in pair with free_vmap_area_root.
	351	+ * It gives O(1) access to prev/next to perform fast coalescing.
	352	+ */
	353	+static LIST_HEAD(free_vmap_area_list);
	354	+
	355	+/*
	356	+ * This augment red-black tree represents the free vmap space.
	357	+ * All vmap_area objects in this tree are sorted by va->va_start
	358	+ * address. It is used for allocation and merging when a vmap
	359	+ * object is released.
	360	+ *
	361	+ * Each vmap_area node contains a maximum available free block
	362	+ * of its sub-tree, right or left. Therefore it is possible to
	363	+ * find a lowest match of free area.
	364	+ */
	365	+static struct rb_root free_vmap_area_root = RB_ROOT;
	366	+
	367	+static __always_inline unsigned long
	368	+va_size(struct vmap_area *va)
	369	+{
	370	+ return (va->va_end - va->va_start);
	371	+}
	372	+
	373	+static __always_inline unsigned long
	374	+get_subtree_max_size(struct rb_node *node)
	375	+{
	376	+ struct vmap_area *va;
	377	+
	378	+ va = rb_entry_safe(node, struct vmap_area, rb_node);
	379	+ return va ? va->subtree_max_size : 0;
	380	+}
	381	+
	382	+/*
	383	+ * Gets called when remove the node and rotate.
	384	+ */
	385	+static __always_inline unsigned long
	386	+compute_subtree_max_size(struct vmap_area *va)
	387	+{
	388	+ return max3(va_size(va),
	389	+ get_subtree_max_size(va->rb_node.rb_left),
	390	+ get_subtree_max_size(va->rb_node.rb_right));
	391	+}
335	392
336		−/* The vmap cache globals are protected by vmap_area_lock */
337		−static struct rb_node *free_vmap_cache;
338		−static unsigned long cached_hole_size;
339		−static unsigned long cached_vstart;
340		−static unsigned long cached_align;
	393	+RB_DECLARE_CALLBACKS(static, free_vmap_area_rb_augment_cb,
	394	+ struct vmap_area, rb_node, unsigned long, subtree_max_size,
	395	+ compute_subtree_max_size)
341	396
342		−static unsigned long vmap_area_pcpu_hole;
	397	+static void purge_vmap_area_lazy(void);
	398	+static BLOCKING_NOTIFIER_HEAD(vmap_notify_list);
	399	+static unsigned long lazy_max_pages(void);
343	400
344	401	static struct vmap_area *__find_vmap_area(unsigned long addr)
345	402	{
@@ -360,41 +417,610 @@ static struct vmap_area *__find_vmap_area(unsigned long addr) · 14 unmodified lines
360	417	return NULL;
361	418	}
362	419
363		−static void __insert_vmap_area(struct vmap_area *va)
364		−{
365		− struct rb_node **p = &vmap_area_root.rb_node;
366		− struct rb_node *parent = NULL;
367		− struct rb_node *tmp;
	420	+/*
	421	+ * This function returns back addresses of parent node
	422	+ * and its left or right link for further processing.
	423	+ */
	424	+static __always_inline struct rb_node **
	425	+find_va_links(struct vmap_area *va,
	426	+ struct rb_root root, struct rb_node from,
	427	+ struct rb_node **parent)
	428	+{
	429	+ struct vmap_area *tmp_va;
	430	+ struct rb_node **link;
	431	+
	432	+ if (root) {
	433	+ link = &root->rb_node;
	434	+ if (unlikely(!*link)) {
	435	+ *parent = NULL;
	436	+ return link;
	437	+ }
	438	+ } else {
	439	+ link = &from;
	440	+ }
368	441
369		− while (*p) {
370		− struct vmap_area *tmp_va;
	442	+ /*
	443	+ * Go to the bottom of the tree. When we hit the last point
	444	+ * we end up with parent rb_node and correct direction, i name
	445	+ * it link, where the new va->rb_node will be attached to.
	446	+ */
	447	+ do {
	448	+ tmp_va = rb_entry(*link, struct vmap_area, rb_node);
371	449
372		− parent = *p;
373		− tmp_va = rb_entry(parent, struct vmap_area, rb_node);
374		− if (va->va_start < tmp_va->va_end)
375		− p = &(*p)->rb_left;
376		− else if (va->va_end > tmp_va->va_start)
377		− p = &(*p)->rb_right;
	450	+ /*
	451	+ * During the traversal we also do some sanity check.
	452	+ * Trigger the BUG() if there are sides(left/right)
	453	+ * or full overlaps.
	454	+ */
	455	+ if (va->va_start < tmp_va->va_end &&
	456	+ va->va_end <= tmp_va->va_start)
	457	+ link = &(*link)->rb_left;
	458	+ else if (va->va_end > tmp_va->va_start &&
	459	+ va->va_start >= tmp_va->va_end)
	460	+ link = &(*link)->rb_right;
378	461	else
379	462	BUG();
	463	+ } while (*link);
	464	+
	465	+ *parent = &tmp_va->rb_node;
	466	+ return link;
	467	+}
	468	+
	469	+static __always_inline struct list_head *
	470	+get_va_next_sibling(struct rb_node parent, struct rb_node *link)
	471	+{
	472	+ struct list_head *list;
	473	+
	474	+ if (unlikely(!parent))
	475	+ /*
	476	+ * The red-black tree where we try to find VA neighbors
	477	+ * before merging or inserting is empty, i.e. it means
	478	+ * there is no free vmap space. Normally it does not
	479	+ * happen but we handle this case anyway.
	480	+ */
	481	+ return NULL;
	482	+
	483	+ list = &rb_entry(parent, struct vmap_area, rb_node)->list;
	484	+ return (&parent->rb_right == link ? list->next : list);
	485	+}
	486	+
	487	+static __always_inline void
	488	+link_va(struct vmap_area va, struct rb_root root,
	489	+ struct rb_node parent, struct rb_node link, struct list_head head)
	490	+{
	491	+ /*
	492	+ * VA is still not in the list, but we can
	493	+ * identify its future previous list_head node.
	494	+ */
	495	+ if (likely(parent)) {
	496	+ head = &rb_entry(parent, struct vmap_area, rb_node)->list;
	497	+ if (&parent->rb_right != link)
	498	+ head = head->prev;
380	499	}
381	500
382		− rb_link_node(&va->rb_node, parent, p);
383		− rb_insert_color(&va->rb_node, &vmap_area_root);
	501	+ /* Insert to the rb-tree */
	502	+ rb_link_node(&va->rb_node, parent, link);
	503	+ if (root == &free_vmap_area_root) {
	504	+ /*
	505	+ * Some explanation here. Just perform simple insertion
	506	+ * to the tree. We do not set va->subtree_max_size to
	507	+ * its current size before calling rb_insert_augmented().
	508	+ * It is because of we populate the tree from the bottom
	509	+ * to parent levels when the node _is_ in the tree.
	510	+ *
	511	+ * Therefore we set subtree_max_size to zero after insertion,
	512	+ * to let __augment_tree_propagate_from() puts everything to
	513	+ * the correct order later on.
	514	+ */
	515	+ rb_insert_augmented(&va->rb_node,
	516	+ root, &free_vmap_area_rb_augment_cb);
	517	+ va->subtree_max_size = 0;
	518	+ } else {
	519	+ rb_insert_color(&va->rb_node, root);
	520	+ }
384	521
385		− /* address-sort this list */
386		− tmp = rb_prev(&va->rb_node);
387		− if (tmp) {
388		− struct vmap_area *prev;
389		− prev = rb_entry(tmp, struct vmap_area, rb_node);
390		− list_add_rcu(&va->list, &prev->list);
391		− } else
392		− list_add_rcu(&va->list, &vmap_area_list);
	522	+ /* Address-sort this list */
	523	+ list_add(&va->list, head);
393	524	}
394	525
395		−static void purge_vmap_area_lazy(void);
	526	+static __always_inline void
	527	+unlink_va(struct vmap_area va, struct rb_root root)
	528	+{
	529	+ /*
	530	+ * During merging a VA node can be empty, therefore
	531	+ * not linked with the tree nor list. Just check it.
	532	+ */
	533	+ if (!RB_EMPTY_NODE(&va->rb_node)) {
	534	+ if (root == &free_vmap_area_root)
	535	+ rb_erase_augmented(&va->rb_node,
	536	+ root, &free_vmap_area_rb_augment_cb);
	537	+ else
	538	+ rb_erase(&va->rb_node, root);
396	539
397		−static BLOCKING_NOTIFIER_HEAD(vmap_notify_list);
	540	+ list_del(&va->list);
	541	+ RB_CLEAR_NODE(&va->rb_node);
	542	+ }
	543	+}
	544	+
	545	+#if DEBUG_AUGMENT_PROPAGATE_CHECK
	546	+static void
	547	+augment_tree_propagate_check(struct rb_node *n)
	548	+{
	549	+ struct vmap_area *va;
	550	+ struct rb_node *node;
	551	+ unsigned long size;
	552	+ bool found = false;
	553	+
	554	+ if (n == NULL)
	555	+ return;
	556	+
	557	+ va = rb_entry(n, struct vmap_area, rb_node);
	558	+ size = va->subtree_max_size;
	559	+ node = n;
	560	+
	561	+ while (node) {
	562	+ va = rb_entry(node, struct vmap_area, rb_node);
	563	+
	564	+ if (get_subtree_max_size(node->rb_left) == size) {
	565	+ node = node->rb_left;
	566	+ } else {
	567	+ if (va_size(va) == size) {
	568	+ found = true;
	569	+ break;
	570	+ }
	571	+
	572	+ node = node->rb_right;
	573	+ }
	574	+ }
	575	+
	576	+ if (!found) {
	577	+ va = rb_entry(n, struct vmap_area, rb_node);
	578	+ pr_emerg("tree is corrupted: %lu, %lu\n",
	579	+ va_size(va), va->subtree_max_size);
	580	+ }
	581	+
	582	+ augment_tree_propagate_check(n->rb_left);
	583	+ augment_tree_propagate_check(n->rb_right);
	584	+}
	585	+#endif
	586	+
	587	+/*
	588	+ * This function populates subtree_max_size from bottom to upper
	589	+ * levels starting from VA point. The propagation must be done
	590	+ * when VA size is modified by changing its va_start/va_end. Or
	591	+ * in case of newly inserting of VA to the tree.
	592	+ *
	593	+ * It means that __augment_tree_propagate_from() must be called:
	594	+ * - After VA has been inserted to the tree(free path);
	595	+ * - After VA has been shrunk(allocation path);
	596	+ * - After VA has been increased(merging path).
	597	+ *
	598	+ * Please note that, it does not mean that upper parent nodes
	599	+ * and their subtree_max_size are recalculated all the time up
	600	+ * to the root node.
	601	+ *
	602	+ * 4--8
	603	+ * /\
	604	+ * / \
	605	+ * / \
	606	+ * 2--2 8--8
	607	+ *
	608	+ * For example if we modify the node 4, shrinking it to 2, then
	609	+ * no any modification is required. If we shrink the node 2 to 1
	610	+ * its subtree_max_size is updated only, and set to 1. If we shrink
	611	+ * the node 8 to 6, then its subtree_max_size is set to 6 and parent
	612	+ * node becomes 4--6.
	613	+ */
	614	+static __always_inline void
	615	+augment_tree_propagate_from(struct vmap_area *va)
	616	+{
	617	+ struct rb_node *node = &va->rb_node;
	618	+ unsigned long new_va_sub_max_size;
	619	+
	620	+ while (node) {
	621	+ va = rb_entry(node, struct vmap_area, rb_node);
	622	+ new_va_sub_max_size = compute_subtree_max_size(va);
	623	+
	624	+ /*
	625	+ * If the newly calculated maximum available size of the
	626	+ * subtree is equal to the current one, then it means that
	627	+ * the tree is propagated correctly. So we have to stop at
	628	+ * this point to save cycles.
	629	+ */
	630	+ if (va->subtree_max_size == new_va_sub_max_size)
	631	+ break;
	632	+
	633	+ va->subtree_max_size = new_va_sub_max_size;
	634	+ node = rb_parent(&va->rb_node);
	635	+ }
	636	+
	637	+#if DEBUG_AUGMENT_PROPAGATE_CHECK
	638	+ augment_tree_propagate_check(free_vmap_area_root.rb_node);
	639	+#endif
	640	+}
	641	+
	642	+static void
	643	+insert_vmap_area(struct vmap_area *va,
	644	+ struct rb_root root, struct list_head head)
	645	+{
	646	+ struct rb_node **link;
	647	+ struct rb_node *parent;
	648	+
	649	+ link = find_va_links(va, root, NULL, &parent);
	650	+ link_va(va, root, parent, link, head);
	651	+}
	652	+
	653	+static void
	654	+insert_vmap_area_augment(struct vmap_area *va,
	655	+ struct rb_node from, struct rb_root root,
	656	+ struct list_head *head)
	657	+{
	658	+ struct rb_node **link;
	659	+ struct rb_node *parent;
	660	+
	661	+ if (from)
	662	+ link = find_va_links(va, NULL, from, &parent);
	663	+ else
	664	+ link = find_va_links(va, root, NULL, &parent);
	665	+
	666	+ link_va(va, root, parent, link, head);
	667	+ augment_tree_propagate_from(va);
	668	+}
	669	+
	670	+/*
	671	+ * Merge de-allocated chunk of VA memory with previous
	672	+ * and next free blocks. If coalesce is not done a new
	673	+ * free area is inserted. If VA has been merged, it is
	674	+ * freed.
	675	+ */
	676	+static __always_inline void
	677	+merge_or_add_vmap_area(struct vmap_area *va,
	678	+ struct rb_root root, struct list_head head)
	679	+{
	680	+ struct vmap_area *sibling;
	681	+ struct list_head *next;
	682	+ struct rb_node **link;
	683	+ struct rb_node *parent;
	684	+ bool merged = false;
	685	+
	686	+ /*
	687	+ * Find a place in the tree where VA potentially will be
	688	+ * inserted, unless it is merged with its sibling/siblings.
	689	+ */
	690	+ link = find_va_links(va, root, NULL, &parent);
	691	+
	692	+ /*
	693	+ * Get next node of VA to check if merging can be done.
	694	+ */
	695	+ next = get_va_next_sibling(parent, link);
	696	+ if (unlikely(next == NULL))
	697	+ goto insert;
	698	+
	699	+ /*
	700	+ * start end
	701	+ * \| \|
	702	+ * \|<------VA------>\|<-----Next----->\|
	703	+ * \| \|
	704	+ * start end
	705	+ */
	706	+ if (next != head) {
	707	+ sibling = list_entry(next, struct vmap_area, list);
	708	+ if (sibling->va_start == va->va_end) {
	709	+ sibling->va_start = va->va_start;
	710	+
	711	+ /* Check and update the tree if needed. */
	712	+ augment_tree_propagate_from(sibling);
	713	+
	714	+ /* Remove this VA, it has been merged. */
	715	+ unlink_va(va, root);
	716	+
	717	+ /* Free vmap_area object. */
	718	+ kmem_cache_free(vmap_area_cachep, va);
	719	+
	720	+ /* Point to the new merged area. */
	721	+ va = sibling;
	722	+ merged = true;
	723	+ }
	724	+ }
	725	+
	726	+ /*
	727	+ * start end
	728	+ * \| \|
	729	+ * \|<-----Prev----->\|<------VA------>\|
	730	+ * \| \|
	731	+ * start end
	732	+ */
	733	+ if (next->prev != head) {
	734	+ sibling = list_entry(next->prev, struct vmap_area, list);
	735	+ if (sibling->va_end == va->va_start) {
	736	+ sibling->va_end = va->va_end;
	737	+
	738	+ /* Check and update the tree if needed. */
	739	+ augment_tree_propagate_from(sibling);
	740	+
	741	+ /* Remove this VA, it has been merged. */
	742	+ unlink_va(va, root);
	743	+
	744	+ /* Free vmap_area object. */
	745	+ kmem_cache_free(vmap_area_cachep, va);
	746	+
	747	+ return;
	748	+ }
	749	+ }
	750	+
	751	+insert:
	752	+ if (!merged) {
	753	+ link_va(va, root, parent, link, head);
	754	+ augment_tree_propagate_from(va);
	755	+ }
	756	+}
	757	+
	758	+static __always_inline bool
	759	+is_within_this_va(struct vmap_area *va, unsigned long size,
	760	+ unsigned long align, unsigned long vstart)
	761	+{
	762	+ unsigned long nva_start_addr;
	763	+
	764	+ if (va->va_start > vstart)
	765	+ nva_start_addr = ALIGN(va->va_start, align);
	766	+ else
	767	+ nva_start_addr = ALIGN(vstart, align);
	768	+
	769	+ /* Can be overflowed due to big size or alignment. */
	770	+ if (nva_start_addr + size < nva_start_addr \|\|
	771	+ nva_start_addr < vstart)
	772	+ return false;
	773	+
	774	+ return (nva_start_addr + size <= va->va_end);
	775	+}
	776	+
	777	+/*
	778	+ * Find the first free block(lowest start address) in the tree,
	779	+ * that will accomplish the request corresponding to passing
	780	+ * parameters.
	781	+ */
	782	+static __always_inline struct vmap_area *
	783	+find_vmap_lowest_match(unsigned long size,
	784	+ unsigned long align, unsigned long vstart)
	785	+{
	786	+ struct vmap_area *va;
	787	+ struct rb_node *node;
	788	+ unsigned long length;
	789	+
	790	+ /* Start from the root. */
	791	+ node = free_vmap_area_root.rb_node;
	792	+
	793	+ /* Adjust the search size for alignment overhead. */
	794	+ length = size + align - 1;
	795	+
	796	+ while (node) {
	797	+ va = rb_entry(node, struct vmap_area, rb_node);
	798	+
	799	+ if (get_subtree_max_size(node->rb_left) >= length &&
	800	+ vstart < va->va_start) {
	801	+ node = node->rb_left;
	802	+ } else {
	803	+ if (is_within_this_va(va, size, align, vstart))
	804	+ return va;
	805	+
	806	+ /*
	807	+ * Does not make sense to go deeper towards the right
	808	+ * sub-tree if it does not have a free block that is
	809	+ * equal or bigger to the requested search length.
	810	+ */
	811	+ if (get_subtree_max_size(node->rb_right) >= length) {
	812	+ node = node->rb_right;
	813	+ continue;
	814	+ }
	815	+
	816	+ /*
	817	+ * OK. We roll back and find the fist right sub-tree,
	818	+ * that will satisfy the search criteria. It can happen
	819	+ * only once due to "vstart" restriction.
	820	+ */
	821	+ while ((node = rb_parent(node))) {
	822	+ va = rb_entry(node, struct vmap_area, rb_node);
	823	+ if (is_within_this_va(va, size, align, vstart))
	824	+ return va;
	825	+
	826	+ if (get_subtree_max_size(node->rb_right) >= length &&
	827	+ vstart <= va->va_start) {
	828	+ node = node->rb_right;
	829	+ break;
	830	+ }
	831	+ }
	832	+ }
	833	+ }
	834	+
	835	+ return NULL;
	836	+}
	837	+
	838	+#if DEBUG_AUGMENT_LOWEST_MATCH_CHECK
	839	+#include <linux/random.h>
	840	+
	841	+static struct vmap_area *
	842	+find_vmap_lowest_linear_match(unsigned long size,
	843	+ unsigned long align, unsigned long vstart)
	844	+{
	845	+ struct vmap_area *va;
	846	+
	847	+ list_for_each_entry(va, &free_vmap_area_list, list) {
	848	+ if (!is_within_this_va(va, size, align, vstart))
	849	+ continue;
	850	+
	851	+ return va;
	852	+ }
	853	+
	854	+ return NULL;
	855	+}
	856	+
	857	+static void
	858	+find_vmap_lowest_match_check(unsigned long size)
	859	+{
	860	+ struct vmap_area va_1, va_2;
	861	+ unsigned long vstart;
	862	+ unsigned int rnd;
	863	+
	864	+ get_random_bytes(&rnd, sizeof(rnd));
	865	+ vstart = VMALLOC_START + rnd;
	866	+
	867	+ va_1 = find_vmap_lowest_match(size, 1, vstart);
	868	+ va_2 = find_vmap_lowest_linear_match(size, 1, vstart);
	869	+
	870	+ if (va_1 != va_2)
	871	+ pr_emerg("not lowest: t: 0x%p, l: 0x%p, v: 0x%lx\n",
	872	+ va_1, va_2, vstart);
	873	+}
	874	+#endif
	875	+
	876	+enum fit_type {
	877	+ NOTHING_FIT = 0,
	878	+ FL_FIT_TYPE = 1, /* full fit */
	879	+ LE_FIT_TYPE = 2, /* left edge fit */
	880	+ RE_FIT_TYPE = 3, /* right edge fit */
	881	+ NE_FIT_TYPE = 4 /* no edge fit */
	882	+};
	883	+
	884	+static __always_inline enum fit_type
	885	+classify_va_fit_type(struct vmap_area *va,
	886	+ unsigned long nva_start_addr, unsigned long size)
	887	+{
	888	+ enum fit_type type;
	889	+
	890	+ /* Check if it is within VA. */
	891	+ if (nva_start_addr < va->va_start \|\|
	892	+ nva_start_addr + size > va->va_end)
	893	+ return NOTHING_FIT;
	894	+
	895	+ /* Now classify. */
	896	+ if (va->va_start == nva_start_addr) {
	897	+ if (va->va_end == nva_start_addr + size)
	898	+ type = FL_FIT_TYPE;
	899	+ else
	900	+ type = LE_FIT_TYPE;
	901	+ } else if (va->va_end == nva_start_addr + size) {
	902	+ type = RE_FIT_TYPE;
	903	+ } else {
	904	+ type = NE_FIT_TYPE;
	905	+ }
	906	+
	907	+ return type;
	908	+}
	909	+
	910	+static __always_inline int
	911	+adjust_va_to_fit_type(struct vmap_area *va,
	912	+ unsigned long nva_start_addr, unsigned long size,
	913	+ enum fit_type type)
	914	+{
	915	+ struct vmap_area *lva;
	916	+
	917	+ if (type == FL_FIT_TYPE) {
	918	+ /*
	919	+ * No need to split VA, it fully fits.
	920	+ *
	921	+ * \| \|
	922	+ * V NVA V
	923	+ * \|---------------\|
	924	+ */
	925	+ unlink_va(va, &free_vmap_area_root);
	926	+ kmem_cache_free(vmap_area_cachep, va);
	927	+ } else if (type == LE_FIT_TYPE) {
	928	+ /*
	929	+ * Split left edge of fit VA.
	930	+ *
	931	+ * \| \|
	932	+ * V NVA V R
	933	+ * \|-------\|-------\|
	934	+ */
	935	+ va->va_start += size;
	936	+ } else if (type == RE_FIT_TYPE) {
	937	+ /*
	938	+ * Split right edge of fit VA.
	939	+ *
	940	+ * \| \|
	941	+ * L V NVA V
	942	+ * \|-------\|-------\|
	943	+ */
	944	+ va->va_end = nva_start_addr;
	945	+ } else if (type == NE_FIT_TYPE) {
	946	+ /*
	947	+ * Split no edge of fit VA.
	948	+ *
	949	+ * \| \|
	950	+ * L V NVA V R
	951	+ * \|---\|-------\|---\|
	952	+ */
	953	+ lva = kmem_cache_alloc(vmap_area_cachep, GFP_NOWAIT);
	954	+ if (unlikely(!lva))
	955	+ return -1;
	956	+
	957	+ /*
	958	+ * Build the remainder.
	959	+ */
	960	+ lva->va_start = va->va_start;
	961	+ lva->va_end = nva_start_addr;
	962	+
	963	+ /*
	964	+ * Shrink this VA to remaining size.
	965	+ */
	966	+ va->va_start = nva_start_addr + size;
	967	+ } else {
	968	+ return -1;
	969	+ }
	970	+
	971	+ if (type != FL_FIT_TYPE) {
	972	+ augment_tree_propagate_from(va);
	973	+
	974	+ if (type == NE_FIT_TYPE)
	975	+ insert_vmap_area_augment(lva, &va->rb_node,
	976	+ &free_vmap_area_root, &free_vmap_area_list);
	977	+ }
	978	+
	979	+ return 0;
	980	+}
	981	+
	982	+/*
	983	+ * Returns a start address of the newly allocated area, if success.
	984	+ * Otherwise a vend is returned that indicates failure.
	985	+ */
	986	+static __always_inline unsigned long
	987	+__alloc_vmap_area(unsigned long size, unsigned long align,
	988	+ unsigned long vstart, unsigned long vend, int node)
	989	+{
	990	+ unsigned long nva_start_addr;
	991	+ struct vmap_area *va;
	992	+ enum fit_type type;
	993	+ int ret;
	994	+
	995	+ va = find_vmap_lowest_match(size, align, vstart);
	996	+ if (unlikely(!va))
	997	+ return vend;
	998	+
	999	+ if (va->va_start > vstart)
	1000	+ nva_start_addr = ALIGN(va->va_start, align);
	1001	+ else
	1002	+ nva_start_addr = ALIGN(vstart, align);
	1003	+
	1004	+ /* Check the "vend" restriction. */
	1005	+ if (nva_start_addr + size > vend)
	1006	+ return vend;
	1007	+
	1008	+ /* Classify what we have found. */
	1009	+ type = classify_va_fit_type(va, nva_start_addr, size);
	1010	+ if (WARN_ON_ONCE(type == NOTHING_FIT))
	1011	+ return vend;
	1012	+
	1013	+ /* Update the free vmap_area. */
	1014	+ ret = adjust_va_to_fit_type(va, nva_start_addr, size, type);
	1015	+ if (ret)
	1016	+ return vend;
	1017	+
	1018	+#if DEBUG_AUGMENT_LOWEST_MATCH_CHECK
	1019	+ find_vmap_lowest_match_check(size);
	1020	+#endif
	1021	+
	1022	+ return nva_start_addr;
	1023	+}
398	1024
399	1025	/*
400	1026	* Allocate a region of KVA of the specified size and alignment, within the
@@ -406,18 +1032,19 @@ static struct vmap_area *alloc_vmap_area(unsigned long size, · 5 unmodified lines
406	1032	int node, gfp_t gfp_mask)
407	1033	{
408	1034	struct vmap_area *va;
409		− struct rb_node *n;
410	1035	unsigned long addr;
411	1036	int purged = 0;
412		− struct vmap_area *first;
413	1037
414	1038	BUG_ON(!size);
415	1039	BUG_ON(offset_in_page(size));
416	1040	BUG_ON(!is_power_of_2(align));
417	1041
	1042	+ if (unlikely(!vmap_initialized))
	1043	+ return ERR_PTR(-EBUSY);
	1044	+
418	1045	might_sleep();
419	1046
420		− va = kmalloc_node(sizeof(struct vmap_area),
	1047	+ va = kmem_cache_alloc_node(vmap_area_cachep,
421	1048	gfp_mask & GFP_RECLAIM_MASK, node);
422	1049	if (unlikely(!va))
423	1050	return ERR_PTR(-ENOMEM);
@@ -430,87 +1057,20 @@ static struct vmap_area *alloc_vmap_area(unsigned long size, · 6 unmodified lines
430	1057
431	1058	retry:
432	1059	spin_lock(&vmap_area_lock);
433		− /*
434		− * Invalidate cache if we have more permissive parameters.
435		− * cached_hole_size notes the largest hole noticed _below_
436		− * the vmap_area cached in free_vmap_cache: if size fits
437		− * into that hole, we want to scan from vstart to reuse
438		− * the hole instead of allocating above free_vmap_cache.
439		− * Note that __free_vmap_area may update free_vmap_cache
440		− * without updating cached_hole_size or cached_align.
441		− */
442		− if (!free_vmap_cache \|\|
443		− size < cached_hole_size \|\|
444		− vstart < cached_vstart \|\|
445		− align < cached_align) {
446		−nocache:
447		− cached_hole_size = 0;
448		− free_vmap_cache = NULL;
449		− }
450		− /* record if we encounter less permissive parameters */
451		− cached_vstart = vstart;
452		− cached_align = align;
453		−
454		− /* find starting point for our search */
455		− if (free_vmap_cache) {
456		− first = rb_entry(free_vmap_cache, struct vmap_area, rb_node);
457		− addr = ALIGN(first->va_end, align);
458		− if (addr < vstart)
459		− goto nocache;
460		− if (addr + size < addr)
461		− goto overflow;
462		−
463		− } else {
464		− addr = ALIGN(vstart, align);
465		− if (addr + size < addr)
466		− goto overflow;
467		−
468		− n = vmap_area_root.rb_node;
469		− first = NULL;
470		−
471		− while (n) {
472		− struct vmap_area *tmp;
473		− tmp = rb_entry(n, struct vmap_area, rb_node);
474		− if (tmp->va_end >= addr) {
475		− first = tmp;
476		− if (tmp->va_start <= addr)
477		− break;
478		− n = n->rb_left;
479		− } else
480		− n = n->rb_right;
481		− }
482		−
483		− if (!first)
484		− goto found;
485		− }
486		−
487		− /* from the starting point, walk areas until a suitable hole is found */
488		− while (addr + size > first->va_start && addr + size <= vend) {
489		− if (addr + cached_hole_size < first->va_start)
490		− cached_hole_size = first->va_start - addr;
491		− addr = ALIGN(first->va_end, align);
492		− if (addr + size < addr)
493		− goto overflow;
494		−
495		− if (list_is_last(&first->list, &vmap_area_list))
496		− goto found;
497	1060
498		− first = list_next_entry(first, list);
499		− }
500		−
501		−found:
502	1061	/*
503		− * Check also calculated address against the vstart,
504		− * because it can be 0 because of big align request.
	1062	+ * If an allocation fails, the "vend" address is
	1063	+ * returned. Therefore trigger the overflow path.
505	1064	*/
506		− if (addr + size > vend \|\| addr < vstart)
	1065	+ addr = __alloc_vmap_area(size, align, vstart, vend, node);
	1066	+ if (unlikely(addr == vend))
507	1067	goto overflow;
508	1068
509	1069	va->va_start = addr;
510	1070	va->va_end = addr + size;
511	1071	va->flags = 0;
512		− __insert_vmap_area(va);
513		− free_vmap_cache = &va->rb_node;
	1072	+ insert_vmap_area(va, &vmap_area_root, &vmap_area_list);
	1073	+
514	1074	spin_unlock(&vmap_area_lock);
515	1075
516	1076	BUG_ON(!IS_ALIGNED(va->va_start, align));
@@ -539,7 +1099,8 @@ overflow: · 22 unmodified lines
539	1099	if (!(gfp_mask & __GFP_NOWARN) && printk_ratelimit())
540	1100	pr_warn("vmap allocation for size %lu failed: use vmalloc=<size> to increase size\n",
541	1101	size);
542		− kfree(va);
	1102	+
	1103	+ kmem_cache_free(vmap_area_cachep, va);
543	1104	return ERR_PTR(-EBUSY);
544	1105	}
545	1106
@@ -559,35 +1120,16 @@ static void __free_vmap_area(struct vmap_area *va) · 13 unmodified lines
559	1120	{
560	1121	BUG_ON(RB_EMPTY_NODE(&va->rb_node));
561	1122
562		− if (free_vmap_cache) {
563		− if (va->va_end < cached_vstart) {
564		− free_vmap_cache = NULL;
565		− } else {
566		− struct vmap_area *cache;
567		− cache = rb_entry(free_vmap_cache, struct vmap_area, rb_node);
568		− if (va->va_start <= cache->va_start) {
569		− free_vmap_cache = rb_prev(&va->rb_node);
570		− /*
571		− * We don't try to update cached_hole_size or
572		− * cached_align, but it won't go very wrong.
573		− */
574		− }
575		− }
576		− }
577		− rb_erase(&va->rb_node, &vmap_area_root);
578		− RB_CLEAR_NODE(&va->rb_node);
579		− list_del_rcu(&va->list);
580		−
581	1123	/*
582		− * Track the highest possible candidate for pcpu area
583		− * allocation. Areas outside of vmalloc area can be returned
584		− * here too, consider only end addresses which fall inside
585		− * vmalloc area proper.
	1124	+ * Remove from the busy tree/list.
586	1125	*/
587		− if (va->va_end > VMALLOC_START && va->va_end <= VMALLOC_END)
588		− vmap_area_pcpu_hole = max(vmap_area_pcpu_hole, va->va_end);
	1126	+ unlink_va(va, &vmap_area_root);
589	1127
590		− kfree_rcu(va, rcu_head);
	1128	+ /*
	1129	+ * Merge VA with its neighbors, otherwise just add it.
	1130	+ */
	1131	+ merge_or_add_vmap_area(va,
	1132	+ &free_vmap_area_root, &free_vmap_area_list);
591	1133	}
592	1134
593	1135	/*
@@ -794,8 +1336,6 @@ static struct vmap_area *find_vmap_area(unsigned long addr) · 200 unmodified lines
794	1336
795	1337	#define VMAP_BLOCK_SIZE (VMAP_BBMAP_BITS * PAGE_SIZE)
796	1338
797		−static bool vmap_initialized __read_mostly = false;
798		−
799	1339	struct vmap_block_queue {
800	1340	spinlock_t lock;
801	1341	struct list_head free;
@@ -1256,12 +1796,58 @@ void __init vm_area_register_early(struct vm_struct *vm, size_t align) · 454 unmodified lines
1256	1796	vm_area_add_early(vm);
1257	1797	}
1258	1798
	1799	+static void vmap_init_free_space(void)
	1800	+{
	1801	+ unsigned long vmap_start = 1;
	1802	+ const unsigned long vmap_end = ULONG_MAX;
	1803	+ struct vmap_area busy, free;
	1804	+
	1805	+ /*
	1806	+ * B F B B B F
	1807	+ * -\|-----\|.....\|-----\|-----\|-----\|.....\|-
	1808	+ * \| The KVA space \|
	1809	+ * \|<--------------------------------->\|
	1810	+ */
	1811	+ list_for_each_entry(busy, &vmap_area_list, list) {
	1812	+ if (busy->va_start - vmap_start > 0) {
	1813	+ free = kmem_cache_zalloc(vmap_area_cachep, GFP_NOWAIT);
	1814	+ if (!WARN_ON_ONCE(!free)) {
	1815	+ free->va_start = vmap_start;
	1816	+ free->va_end = busy->va_start;
	1817	+
	1818	+ insert_vmap_area_augment(free, NULL,
	1819	+ &free_vmap_area_root,
	1820	+ &free_vmap_area_list);
	1821	+ }
	1822	+ }
	1823	+
	1824	+ vmap_start = busy->va_end;
	1825	+ }
	1826	+
	1827	+ if (vmap_end - vmap_start > 0) {
	1828	+ free = kmem_cache_zalloc(vmap_area_cachep, GFP_NOWAIT);
	1829	+ if (!WARN_ON_ONCE(!free)) {
	1830	+ free->va_start = vmap_start;
	1831	+ free->va_end = vmap_end;
	1832	+
	1833	+ insert_vmap_area_augment(free, NULL,
	1834	+ &free_vmap_area_root,
	1835	+ &free_vmap_area_list);
	1836	+ }
	1837	+ }
	1838	+}
	1839	+
1259	1840	void __init vmalloc_init(void)
1260	1841	{
1261	1842	struct vmap_area *va;
1262	1843	struct vm_struct *tmp;
1263	1844	int i;
1264	1845
	1846	+ /*
	1847	+ * Create the cache for vmap_area objects.
	1848	+ */
	1849	+ vmap_area_cachep = KMEM_CACHE(vmap_area, SLAB_PANIC);
	1850	+
1265	1851	for_each_possible_cpu(i) {
1266	1852	struct vmap_block_queue *vbq;
1267	1853	struct vfree_deferred *p;
@@ -1276,16 +1862,21 @@ void __init vmalloc_init(void) · 8 unmodified lines
1276	1862
1277	1863	/* Import existing vmlist entries. */
1278	1864	for (tmp = vmlist; tmp; tmp = tmp->next) {
1279		− va = kzalloc(sizeof(struct vmap_area), GFP_NOWAIT);
	1865	+ va = kmem_cache_zalloc(vmap_area_cachep, GFP_NOWAIT);
	1866	+ if (WARN_ON_ONCE(!va))
	1867	+ continue;
	1868	+
1280	1869	va->flags = VM_VM_AREA;
1281	1870	va->va_start = (unsigned long)tmp->addr;
1282	1871	va->va_end = va->va_start + tmp->size;
1283	1872	va->vm = tmp;
1284		− __insert_vmap_area(va);
	1873	+ insert_vmap_area(va, &vmap_area_root, &vmap_area_list);
1285	1874	}
1286	1875
1287		− vmap_area_pcpu_hole = VMALLOC_END;
1288		−
	1876	+ /*
	1877	+ * Now we can initialize a free vmap space.
	1878	+ */
	1879	+ vmap_init_free_space();
1289	1880	vmap_initialized = true;
1290	1881	}
1291	1882
@@ -2477,81 +3068,64 @@ static struct vmap_area node_to_va(struct rb_node n) · 1185 unmodified lines
2477	3068	}
2478	3069
2479	3070	/**
2480		− * pvm_find_next_prev - find the next and prev vmap_area surrounding @end
2481		− * @end: target address
2482		− * @pnext: out arg for the next vmap_area
2483		− * @pprev: out arg for the previous vmap_area
2484		− *
2485		− * Returns: %true if either or both of next and prev are found,
2486		− * %false if no vmap_area exists
	3071	+ * pvm_find_va_enclose_addr - find the vmap_area @addr belongs to
	3072	+ * @addr: target address
2487	3073	*
2488		− * Find vmap_areas end addresses of which enclose @end. ie. if not
2489		− * NULL, pnext->va_end > @end and pprev->va_end <= @end.
	3074	+ * Returns: vmap_area if it is found. If there is no such area
	3075	+ * the first highest(reverse order) vmap_area is returned
	3076	+ * i.e. va->va_start < addr && va->va_end < addr or NULL
	3077	+ * if there are no any areas before @addr.
2490	3078	*/
2491		−static bool pvm_find_next_prev(unsigned long end,
2492		− struct vmap_area **pnext,
2493		− struct vmap_area **pprev)
	3079	+static struct vmap_area *
	3080	+pvm_find_va_enclose_addr(unsigned long addr)
2494	3081	{
2495		− struct rb_node *n = vmap_area_root.rb_node;
2496		− struct vmap_area *va = NULL;
	3082	+ struct vmap_area va, tmp;
	3083	+ struct rb_node *n;
	3084	+
	3085	+ n = free_vmap_area_root.rb_node;
	3086	+ va = NULL;
2497	3087
2498	3088	while (n) {
2499		− va = rb_entry(n, struct vmap_area, rb_node);
2500		− if (end < va->va_end)
2501		− n = n->rb_left;
2502		− else if (end > va->va_end)
	3089	+ tmp = rb_entry(n, struct vmap_area, rb_node);
	3090	+ if (tmp->va_start <= addr) {
	3091	+ va = tmp;
	3092	+ if (tmp->va_end >= addr)
	3093	+ break;
	3094	+
2503	3095	n = n->rb_right;
2504		− else
2505		− break;
	3096	+ } else {
	3097	+ n = n->rb_left;
	3098	+ }
2506	3099	}
2507	3100
2508		− if (!va)
2509		− return false;
2510		−
2511		− if (va->va_end > end) {
2512		− *pnext = va;
2513		− pprev = node_to_va(rb_prev(&(pnext)->rb_node));
2514		− } else {
2515		− *pprev = va;
2516		− pnext = node_to_va(rb_next(&(pprev)->rb_node));
2517		− }
2518		− return true;
	3101	+ return va;
2519	3102	}
2520	3103
2521	3104	/**
2522		− * pvm_determine_end - find the highest aligned address between two vmap_areas
2523		− * @pnext: in/out arg for the next vmap_area
2524		− * @pprev: in/out arg for the previous vmap_area
2525		− * @align: alignment
	3105	+ * pvm_determine_end_from_reverse - find the highest aligned address
	3106	+ * of free block below VMALLOC_END
	3107	+ * @va:
	3108	+ * in - the VA we start the search(reverse order);
	3109	+ * out - the VA with the highest aligned end address.
2526	3110	*
2527		− * Returns: determined end address
2528		− *
2529		− * Find the highest aligned address between @pnext and @pprev below
2530		− * VMALLOC_END. @pnext and @pprev are adjusted so that the aligned
2531		− * down address is between the end addresses of the two vmap_areas.
2532		− *
2533		− * Please note that the address returned by this function may fall
2534		− * inside *@pnext vmap_area. The caller is responsible for checking
2535		− * that.
	3111	+ * Returns: determined end address within vmap_area
2536	3112	*/
2537		−static unsigned long pvm_determine_end(struct vmap_area **pnext,
2538		− struct vmap_area **pprev,
2539		− unsigned long align)
	3113	+static unsigned long
	3114	+pvm_determine_end_from_reverse(struct vmap_area **va, unsigned long align)
2540	3115	{
2541		− const unsigned long vmalloc_end = VMALLOC_END & ~(align - 1);
	3116	+ unsigned long vmalloc_end = VMALLOC_END & ~(align - 1);
2542	3117	unsigned long addr;
2543	3118
2544		− if (*pnext)
2545		− addr = min((*pnext)->va_start & ~(align - 1), vmalloc_end);
2546		− else
2547		− addr = vmalloc_end;
2548		−
2549		− while (pprev && (pprev)->va_end > addr) {
2550		− pnext = pprev;
2551		− pprev = node_to_va(rb_prev(&(pnext)->rb_node));
	3119	+ if (likely(*va)) {
	3120	+ list_for_each_entry_from_reverse((*va),
	3121	+ &free_vmap_area_list, list) {
	3122	+ addr = min((*va)->va_end & ~(align - 1), vmalloc_end);
	3123	+ if ((*va)->va_start < addr)
	3124	+ return addr;
	3125	+ }
2552	3126	}
2553	3127
2554		− return addr;
	3128	+ return 0;
2555	3129	}
2556	3130
2557	3131	/**
@@ -2571,12 +3145,12 @@ static unsigned long pvm_determine_end(struct vmap_area **pnext, · 13 unmodified lines
2571	3145	* to gigabytes. To avoid interacting with regular vmallocs, these
2572	3146	* areas are allocated from top.
2573	3147	*
2574		− * Despite its complicated look, this allocator is rather simple. It
2575		− * does everything top-down and scans areas from the end looking for
2576		− * matching slot. While scanning, if any of the areas overlaps with
2577		− * existing vmap_area, the base address is pulled down to fit the
2578		− * area. Scanning is repeated till all the areas fit and then all
2579		− * necessary data structures are inserted and the result is returned.
	3148	+ * Despite its complicated look, this allocator is rather simple. It
	3149	+ * does everything top-down and scans free blocks from the end looking
	3150	+ * for matching base. While scanning, if any of the areas do not fit the
	3151	+ * base address is pulled down to fit the area. Scanning is repeated till
	3152	+ * all the areas fit and then all necessary data structures are inserted
	3153	+ * and the result is returned.
2580	3154	*/
2581	3155	struct vm_struct *pcpu_get_vm_areas(const unsigned long offsets,
2582	3156	const size_t *sizes, int nr_vms,
@@ -2584,11 +3158,12 @@ struct vm_struct *pcpu_get_vm_areas(const unsigned long offsets, · 1 unmodified line
2584	3158	{
2585	3159	const unsigned long vmalloc_start = ALIGN(VMALLOC_START, align);
2586	3160	const unsigned long vmalloc_end = VMALLOC_END & ~(align - 1);
2587		− struct vmap_area *vas, prev, *next;
	3161	+ struct vmap_area *vas, va;
2588	3162	struct vm_struct **vms;
2589	3163	int area, area2, last_area, term_area;
2590		− unsigned long base, start, end, last_end;
	3164	+ unsigned long base, start, size, end, last_end;
2591	3165	bool purged = false;
	3166	+ enum fit_type type;
2592	3167
2593	3168	/* verify parameters and allocate data structures */
2594	3169	BUG_ON(offset_in_page(align) \|\| !is_power_of_2(align));
@@ -2624,7 +3199,7 @@ struct vm_struct *pcpu_get_vm_areas(const unsigned long offsets, · 29 unmodified lines
2624	3199	goto err_free2;
2625	3200
2626	3201	for (area = 0; area < nr_vms; area++) {
2627		− vas[area] = kzalloc(sizeof(struct vmap_area), GFP_KERNEL);
	3202	+ vas[area] = kmem_cache_zalloc(vmap_area_cachep, GFP_KERNEL);
2628	3203	vms[area] = kzalloc(sizeof(struct vm_struct), GFP_KERNEL);
2629	3204	if (!vas[area] \|\| !vms[area])
2630	3205	goto err_free;
@@ -2637,49 +3212,29 @@ retry: · 6 unmodified lines
2637	3212	start = offsets[area];
2638	3213	end = start + sizes[area];
2639	3214
2640		− if (!pvm_find_next_prev(vmap_area_pcpu_hole, &next, &prev)) {
2641		− base = vmalloc_end - last_end;
2642		− goto found;
2643		− }
2644		− base = pvm_determine_end(&next, &prev, align) - end;
	3215	+ va = pvm_find_va_enclose_addr(vmalloc_end);
	3216	+ base = pvm_determine_end_from_reverse(&va, align) - end;
2645	3217
2646	3218	while (true) {
2647		− BUG_ON(next && next->va_end <= base + end);
2648		− BUG_ON(prev && prev->va_end > base + end);
2649		−
2650	3219	/*
2651	3220	* base might have underflowed, add last_end before
2652	3221	* comparing.
2653	3222	*/
2654		− if (base + last_end < vmalloc_start + last_end) {
2655		− spin_unlock(&vmap_area_lock);
2656		− if (!purged) {
2657		− purge_vmap_area_lazy();
2658		− purged = true;
2659		− goto retry;
2660		− }
2661		− goto err_free;
2662		− }
	3223	+ if (base + last_end < vmalloc_start + last_end)
	3224	+ goto overflow;
2663	3225
2664	3226	/*
2665		− * If next overlaps, move base downwards so that it's
2666		− * right below next and then recheck.
	3227	+ * Fitting base has not been found.
2667	3228	*/
2668		− if (next && next->va_start < base + end) {
2669		− base = pvm_determine_end(&next, &prev, align) - end;
2670		− term_area = area;
2671		− continue;
2672		− }
	3229	+ if (va == NULL)
	3230	+ goto overflow;
2673	3231
2674	3232	/*
2675		− * If prev overlaps, shift down next and prev and move
2676		− * base so that it's right below new next and then
2677		− * recheck.
	3233	+ * If this VA does not fit, move base downwards and recheck.
2678	3234	*/
2679		− if (prev && prev->va_end > base + start) {
2680		− next = prev;
2681		− prev = node_to_va(rb_prev(&next->rb_node));
2682		− base = pvm_determine_end(&next, &prev, align) - end;
	3235	+ if (base + start < va->va_start \|\| base + end > va->va_end) {
	3236	+ va = node_to_va(rb_prev(&va->rb_node));
	3237	+ base = pvm_determine_end_from_reverse(&va, align) - end;
2683	3238	term_area = area;
2684	3239	continue;
2685	3240	}
@@ -2691,21 +3246,40 @@ retry: · 5 unmodified lines
2691	3246	area = (area + nr_vms - 1) % nr_vms;
2692	3247	if (area == term_area)
2693	3248	break;
	3249	+
2694	3250	start = offsets[area];
2695	3251	end = start + sizes[area];
2696		− pvm_find_next_prev(base + end, &next, &prev);
	3252	+ va = pvm_find_va_enclose_addr(base + end);
2697	3253	}
2698		−found:
	3254	+
2699	3255	/* we've found a fitting base, insert all va's */
2700	3256	for (area = 0; area < nr_vms; area++) {
2701		− struct vmap_area *va = vas[area];
	3257	+ int ret;
2702	3258
2703		− va->va_start = base + offsets[area];
2704		− va->va_end = va->va_start + sizes[area];
2705		− __insert_vmap_area(va);
2706		− }
	3259	+ start = base + offsets[area];
	3260	+ size = sizes[area];
	3261	+
	3262	+ va = pvm_find_va_enclose_addr(start);
	3263	+ if (WARN_ON_ONCE(va == NULL))
	3264	+ /* It is a BUG(), but trigger recovery instead. */
	3265	+ goto recovery;
	3266	+
	3267	+ type = classify_va_fit_type(va, start, size);
	3268	+ if (WARN_ON_ONCE(type == NOTHING_FIT))
	3269	+ /* It is a BUG(), but trigger recovery instead. */
	3270	+ goto recovery;
	3271	+
	3272	+ ret = adjust_va_to_fit_type(va, start, size, type);
	3273	+ if (unlikely(ret))
	3274	+ goto recovery;
2707	3275
2708		− vmap_area_pcpu_hole = base + offsets[last_area];
	3276	+ /* Allocated area. */
	3277	+ va = vas[area];
	3278	+ va->va_start = start;
	3279	+ va->va_end = start + size;
	3280	+
	3281	+ insert_vmap_area(va, &vmap_area_root, &vmap_area_list);
	3282	+ }
2709	3283
2710	3284	spin_unlock(&vmap_area_lock);
2711	3285
@@ -2717,9 +3291,38 @@ found: · 5 unmodified lines
2717	3291	kfree(vas);
2718	3292	return vms;
2719	3293
	3294	+recovery:
	3295	+ /* Remove previously inserted areas. */
	3296	+ while (area--) {
	3297	+ __free_vmap_area(vas[area]);
	3298	+ vas[area] = NULL;
	3299	+ }
	3300	+
	3301	+overflow:
	3302	+ spin_unlock(&vmap_area_lock);
	3303	+ if (!purged) {
	3304	+ purge_vmap_area_lazy();
	3305	+ purged = true;
	3306	+
	3307	+ /* Before "retry", check if we recover. */
	3308	+ for (area = 0; area < nr_vms; area++) {
	3309	+ if (vas[area])
	3310	+ continue;
	3311	+
	3312	+ vas[area] = kmem_cache_zalloc(
	3313	+ vmap_area_cachep, GFP_KERNEL);
	3314	+ if (!vas[area])
	3315	+ goto err_free;
	3316	+ }
	3317	+
	3318	+ goto retry;
	3319	+ }
	3320	+
2720	3321	err_free:
2721	3322	for (area = 0; area < nr_vms; area++) {
2722		− kfree(vas[area]);
	3323	+ if (vas[area])
	3324	+ kmem_cache_free(vmap_area_cachep, vas[area]);
	3325	+
2723	3326	kfree(vms[area]);
2724	3327	}
2725	3328	err_free2: