Merge branch 'tip/perf/core-4' of git://git.kernel.org/pub/scm/linux/kernel/git/rostedt/linux-trace into perf/core

Ingo Molnar committed 14 years ago

Commit 19631cb2 parents 1fa2e84 59a094c12 files · +810 −383Browse files

Unified · Split

12 files

arch/x86/Kconfig +0 −1
arch/x86/include/asm/ftrace.h +3 −0
arch/x86/kernel/ftrace.c +343 −168
arch/x86/kernel/nmi.c +5 −5
arch/x86/kernel/traps.c +7 −1
include/linux/ftrace.h +6 −0
include/linux/kernel.h +7 −6
include/linux/ring_buffer.h +4 −2
kernel/trace/ring_buffer.c +140 −108
kernel/trace/trace.c +288 −91
kernel/trace/trace.h +3 −1
kernel/trace/trace_printk.c +4 −0

@@ -40,7 +40,6 @@ config X86 · 39 unmodified lines
40	40	select HAVE_FUNCTION_GRAPH_TRACER
41	41	select HAVE_FUNCTION_GRAPH_FP_TEST
42	42	select HAVE_FUNCTION_TRACE_MCOUNT_TEST
43		− select HAVE_FTRACE_NMI_ENTER if DYNAMIC_FTRACE
44	43	select HAVE_SYSCALL_TRACEPOINTS
45	44	select HAVE_KVM
46	45	select HAVE_ARCH_KGDB

arch/x86/include/asm/ftrace.h+3−0

@@ -34,6 +34,7 @@ · 33 unmodified lines
34	34
35	35	#ifndef __ASSEMBLY__
36	36	extern void mcount(void);
	37	+extern int modifying_ftrace_code;
37	38
38	39	static inline unsigned long ftrace_call_adjust(unsigned long addr)
39	40	{
@@ -50,6 +51,8 @@ struct dyn_arch_ftrace { · 10 unmodified lines
50	51	/* No extra data needed for x86 */
51	52	};
52	53
	54	+int ftrace_int3_handler(struct pt_regs *regs);
	55	+
53	56	#endif /* CONFIG_DYNAMIC_FTRACE */
54	57	#endif /* __ASSEMBLY__ */
55	58	#endif /* CONFIG_FUNCTION_TRACER */

arch/x86/kernel/ftrace.c+343−168

@@ -24,40 +24,21 @@ · 23 unmodified lines
24	24	#include <trace/syscall.h>
25	25
26	26	#include <asm/cacheflush.h>
	27	+#include <asm/kprobes.h>
27	28	#include <asm/ftrace.h>
28	29	#include <asm/nops.h>
29		−#include <asm/nmi.h>
30		−
31	30
32	31	#ifdef CONFIG_DYNAMIC_FTRACE
33	32
34		−/*
35		− * modifying_code is set to notify NMIs that they need to use
36		− * memory barriers when entering or exiting. But we don't want
37		− * to burden NMIs with unnecessary memory barriers when code
38		− * modification is not being done (which is most of the time).
39		− *
40		− * A mutex is already held when ftrace_arch_code_modify_prepare
41		− * and post_process are called. No locks need to be taken here.
42		− *
43		− * Stop machine will make sure currently running NMIs are done
44		− * and new NMIs will see the updated variable before we need
45		− * to worry about NMIs doing memory barriers.
46		− */
47		−static int modifying_code __read_mostly;
48		−static DEFINE_PER_CPU(int, save_modifying_code);
49		−
50	33	int ftrace_arch_code_modify_prepare(void)
51	34	{
52	35	set_kernel_text_rw();
53	36	set_all_modules_text_rw();
54		− modifying_code = 1;
55	37	return 0;
56	38	}
57	39
58	40	int ftrace_arch_code_modify_post_process(void)
59	41	{
60		− modifying_code = 0;
61	42	set_all_modules_text_ro();
62	43	set_kernel_text_ro();
63	44	return 0;
@@ -90,134 +71,6 @@ static unsigned char *ftrace_call_replace(unsigned long ip, unsigned long addr) · 26 unmodified lines
90	71	return calc.code;
91	72	}
92	73
93		−/*
94		− * Modifying code must take extra care. On an SMP machine, if
95		− * the code being modified is also being executed on another CPU
96		− * that CPU will have undefined results and possibly take a GPF.
97		− * We use kstop_machine to stop other CPUS from exectuing code.
98		− * But this does not stop NMIs from happening. We still need
99		− * to protect against that. We separate out the modification of
100		− * the code to take care of this.
101		− *
102		− * Two buffers are added: An IP buffer and a "code" buffer.
103		− *
104		− * 1) Put the instruction pointer into the IP buffer
105		− * and the new code into the "code" buffer.
106		− * 2) Wait for any running NMIs to finish and set a flag that says
107		− * we are modifying code, it is done in an atomic operation.
108		− * 3) Write the code
109		− * 4) clear the flag.
110		− * 5) Wait for any running NMIs to finish.
111		− *
112		− * If an NMI is executed, the first thing it does is to call
113		− * "ftrace_nmi_enter". This will check if the flag is set to write
114		− * and if it is, it will write what is in the IP and "code" buffers.
115		− *
116		− * The trick is, it does not matter if everyone is writing the same
117		− * content to the code location. Also, if a CPU is executing code
118		− * it is OK to write to that code location if the contents being written
119		− * are the same as what exists.
120		− */
121		−
122		−#define MOD_CODE_WRITE_FLAG (1 << 31) /* set when NMI should do the write */
123		−static atomic_t nmi_running = ATOMIC_INIT(0);
124		−static int mod_code_status; /* holds return value of text write */
125		−static void mod_code_ip; / holds the IP to write to */
126		−static const void mod_code_newcode; / holds the text to write to the IP */
127		−
128		−static unsigned nmi_wait_count;
129		−static atomic_t nmi_update_count = ATOMIC_INIT(0);
130		−
131		−int ftrace_arch_read_dyn_info(char *buf, int size)
132		−{
133		− int r;
134		−
135		− r = snprintf(buf, size, "%u %u",
136		− nmi_wait_count,
137		− atomic_read(&nmi_update_count));
138		− return r;
139		−}
140		−
141		−static void clear_mod_flag(void)
142		−{
143		− int old = atomic_read(&nmi_running);
144		−
145		− for (;;) {
146		− int new = old & ~MOD_CODE_WRITE_FLAG;
147		−
148		− if (old == new)
149		− break;
150		−
151		− old = atomic_cmpxchg(&nmi_running, old, new);
152		− }
153		−}
154		−
155		−static void ftrace_mod_code(void)
156		−{
157		− /*
158		− * Yes, more than one CPU process can be writing to mod_code_status.
159		− * (and the code itself)
160		− * But if one were to fail, then they all should, and if one were
161		− * to succeed, then they all should.
162		− */
163		− mod_code_status = probe_kernel_write(mod_code_ip, mod_code_newcode,
164		− MCOUNT_INSN_SIZE);
165		−
166		− /* if we fail, then kill any new writers */
167		− if (mod_code_status)
168		− clear_mod_flag();
169		−}
170		−
171		−void ftrace_nmi_enter(void)
172		−{
173		− __this_cpu_write(save_modifying_code, modifying_code);
174		−
175		− if (!__this_cpu_read(save_modifying_code))
176		− return;
177		−
178		− if (atomic_inc_return(&nmi_running) & MOD_CODE_WRITE_FLAG) {
179		− smp_rmb();
180		− ftrace_mod_code();
181		− atomic_inc(&nmi_update_count);
182		− }
183		− /* Must have previous changes seen before executions */
184		− smp_mb();
185		−}
186		−
187		−void ftrace_nmi_exit(void)
188		−{
189		− if (!__this_cpu_read(save_modifying_code))
190		− return;
191		−
192		− /* Finish all executions before clearing nmi_running */
193		− smp_mb();
194		− atomic_dec(&nmi_running);
195		−}
196		−
197		−static void wait_for_nmi_and_set_mod_flag(void)
198		−{
199		− if (!atomic_cmpxchg(&nmi_running, 0, MOD_CODE_WRITE_FLAG))
200		− return;
201		−
202		− do {
203		− cpu_relax();
204		− } while (atomic_cmpxchg(&nmi_running, 0, MOD_CODE_WRITE_FLAG));
205		−
206		− nmi_wait_count++;
207		−}
208		−
209		−static void wait_for_nmi(void)
210		−{
211		− if (!atomic_read(&nmi_running))
212		− return;
213		−
214		− do {
215		− cpu_relax();
216		− } while (atomic_read(&nmi_running));
217		−
218		− nmi_wait_count++;
219		−}
220		−
221	74	static inline int
222	75	within(unsigned long addr, unsigned long start, unsigned long end)
223	76	{
@@ -238,26 +91,7 @@ do_ftrace_mod_code(unsigned long ip, const void *new_code) · 14 unmodified lines
238	91	if (within(ip, (unsigned long)_text, (unsigned long)_etext))
239	92	ip = (unsigned long)__va(__pa(ip));
240	93
241		− mod_code_ip = (void *)ip;
242		− mod_code_newcode = new_code;
243		−
244		− /* The buffers need to be visible before we let NMIs write them */
245		− smp_mb();
246		−
247		− wait_for_nmi_and_set_mod_flag();
248		−
249		− /* Make sure all running NMIs have finished before we write the code */
250		− smp_mb();
251		−
252		− ftrace_mod_code();
253		−
254		− /* Make sure the write happens before clearing the bit */
255		− smp_mb();
256		−
257		− clear_mod_flag();
258		− wait_for_nmi();
259		−
260		− return mod_code_status;
	94	+ return probe_kernel_write((void *)ip, new_code, MCOUNT_INSN_SIZE);
261	95	}
262	96
263	97	static const unsigned char *ftrace_nop_replace(void)
@@ -334,6 +168,347 @@ int ftrace_update_ftrace_func(ftrace_func_t func) · 70 unmodified lines
334	168	return ret;
335	169	}
336	170
	171	+int modifying_ftrace_code __read_mostly;
	172	+
	173	+/*
	174	+ * A breakpoint was added to the code address we are about to
	175	+ * modify, and this is the handle that will just skip over it.
	176	+ * We are either changing a nop into a trace call, or a trace
	177	+ * call to a nop. While the change is taking place, we treat
	178	+ * it just like it was a nop.
	179	+ */
	180	+int ftrace_int3_handler(struct pt_regs *regs)
	181	+{
	182	+ if (WARN_ON_ONCE(!regs))
	183	+ return 0;
	184	+
	185	+ if (!ftrace_location(regs->ip - 1))
	186	+ return 0;
	187	+
	188	+ regs->ip += MCOUNT_INSN_SIZE - 1;
	189	+
	190	+ return 1;
	191	+}
	192	+
	193	+static int ftrace_write(unsigned long ip, const char *val, int size)
	194	+{
	195	+ /*
	196	+ * On x86_64, kernel text mappings are mapped read-only with
	197	+ * CONFIG_DEBUG_RODATA. So we use the kernel identity mapping instead
	198	+ * of the kernel text mapping to modify the kernel text.
	199	+ *
	200	+ * For 32bit kernels, these mappings are same and we can use
	201	+ * kernel identity mapping to modify code.
	202	+ */
	203	+ if (within(ip, (unsigned long)_text, (unsigned long)_etext))
	204	+ ip = (unsigned long)__va(__pa(ip));
	205	+
	206	+ return probe_kernel_write((void *)ip, val, size);
	207	+}
	208	+
	209	+static int add_break(unsigned long ip, const char *old)
	210	+{
	211	+ unsigned char replaced[MCOUNT_INSN_SIZE];
	212	+ unsigned char brk = BREAKPOINT_INSTRUCTION;
	213	+
	214	+ if (probe_kernel_read(replaced, (void *)ip, MCOUNT_INSN_SIZE))
	215	+ return -EFAULT;
	216	+
	217	+ /* Make sure it is what we expect it to be */
	218	+ if (memcmp(replaced, old, MCOUNT_INSN_SIZE) != 0)
	219	+ return -EINVAL;
	220	+
	221	+ if (ftrace_write(ip, &brk, 1))
	222	+ return -EPERM;
	223	+
	224	+ return 0;
	225	+}
	226	+
	227	+static int add_brk_on_call(struct dyn_ftrace *rec, unsigned long addr)
	228	+{
	229	+ unsigned const char *old;
	230	+ unsigned long ip = rec->ip;
	231	+
	232	+ old = ftrace_call_replace(ip, addr);
	233	+
	234	+ return add_break(rec->ip, old);
	235	+}
	236	+
	237	+
	238	+static int add_brk_on_nop(struct dyn_ftrace *rec)
	239	+{
	240	+ unsigned const char *old;
	241	+
	242	+ old = ftrace_nop_replace();
	243	+
	244	+ return add_break(rec->ip, old);
	245	+}
	246	+
	247	+static int add_breakpoints(struct dyn_ftrace *rec, int enable)
	248	+{
	249	+ unsigned long ftrace_addr;
	250	+ int ret;
	251	+
	252	+ ret = ftrace_test_record(rec, enable);
	253	+
	254	+ ftrace_addr = (unsigned long)FTRACE_ADDR;
	255	+
	256	+ switch (ret) {
	257	+ case FTRACE_UPDATE_IGNORE:
	258	+ return 0;
	259	+
	260	+ case FTRACE_UPDATE_MAKE_CALL:
	261	+ /* converting nop to call */
	262	+ return add_brk_on_nop(rec);
	263	+
	264	+ case FTRACE_UPDATE_MAKE_NOP:
	265	+ /* converting a call to a nop */
	266	+ return add_brk_on_call(rec, ftrace_addr);
	267	+ }
	268	+ return 0;
	269	+}
	270	+
	271	+/*
	272	+ * On error, we need to remove breakpoints. This needs to
	273	+ * be done caefully. If the address does not currently have a
	274	+ * breakpoint, we know we are done. Otherwise, we look at the
	275	+ * remaining 4 bytes of the instruction. If it matches a nop
	276	+ * we replace the breakpoint with the nop. Otherwise we replace
	277	+ * it with the call instruction.
	278	+ */
	279	+static int remove_breakpoint(struct dyn_ftrace *rec)
	280	+{
	281	+ unsigned char ins[MCOUNT_INSN_SIZE];
	282	+ unsigned char brk = BREAKPOINT_INSTRUCTION;
	283	+ const unsigned char *nop;
	284	+ unsigned long ftrace_addr;
	285	+ unsigned long ip = rec->ip;
	286	+
	287	+ /* If we fail the read, just give up */
	288	+ if (probe_kernel_read(ins, (void *)ip, MCOUNT_INSN_SIZE))
	289	+ return -EFAULT;
	290	+
	291	+ /* If this does not have a breakpoint, we are done */
	292	+ if (ins[0] != brk)
	293	+ return -1;
	294	+
	295	+ nop = ftrace_nop_replace();
	296	+
	297	+ /*
	298	+ * If the last 4 bytes of the instruction do not match
	299	+ * a nop, then we assume that this is a call to ftrace_addr.
	300	+ */
	301	+ if (memcmp(&ins[1], &nop[1], MCOUNT_INSN_SIZE - 1) != 0) {
	302	+ /*
	303	+ * For extra paranoidism, we check if the breakpoint is on
	304	+ * a call that would actually jump to the ftrace_addr.
	305	+ * If not, don't touch the breakpoint, we make just create
	306	+ * a disaster.
	307	+ */
	308	+ ftrace_addr = (unsigned long)FTRACE_ADDR;
	309	+ nop = ftrace_call_replace(ip, ftrace_addr);
	310	+
	311	+ if (memcmp(&ins[1], &nop[1], MCOUNT_INSN_SIZE - 1) != 0)
	312	+ return -EINVAL;
	313	+ }
	314	+
	315	+ return probe_kernel_write((void *)ip, &nop[0], 1);
	316	+}
	317	+
	318	+static int add_update_code(unsigned long ip, unsigned const char *new)
	319	+{
	320	+ /* skip breakpoint */
	321	+ ip++;
	322	+ new++;
	323	+ if (ftrace_write(ip, new, MCOUNT_INSN_SIZE - 1))
	324	+ return -EPERM;
	325	+ return 0;
	326	+}
	327	+
	328	+static int add_update_call(struct dyn_ftrace *rec, unsigned long addr)
	329	+{
	330	+ unsigned long ip = rec->ip;
	331	+ unsigned const char *new;
	332	+
	333	+ new = ftrace_call_replace(ip, addr);
	334	+ return add_update_code(ip, new);
	335	+}
	336	+
	337	+static int add_update_nop(struct dyn_ftrace *rec)
	338	+{
	339	+ unsigned long ip = rec->ip;
	340	+ unsigned const char *new;
	341	+
	342	+ new = ftrace_nop_replace();
	343	+ return add_update_code(ip, new);
	344	+}
	345	+
	346	+static int add_update(struct dyn_ftrace *rec, int enable)
	347	+{
	348	+ unsigned long ftrace_addr;
	349	+ int ret;
	350	+
	351	+ ret = ftrace_test_record(rec, enable);
	352	+
	353	+ ftrace_addr = (unsigned long)FTRACE_ADDR;
	354	+
	355	+ switch (ret) {
	356	+ case FTRACE_UPDATE_IGNORE:
	357	+ return 0;
	358	+
	359	+ case FTRACE_UPDATE_MAKE_CALL:
	360	+ /* converting nop to call */
	361	+ return add_update_call(rec, ftrace_addr);
	362	+
	363	+ case FTRACE_UPDATE_MAKE_NOP:
	364	+ /* converting a call to a nop */
	365	+ return add_update_nop(rec);
	366	+ }
	367	+
	368	+ return 0;
	369	+}
	370	+
	371	+static int finish_update_call(struct dyn_ftrace *rec, unsigned long addr)
	372	+{
	373	+ unsigned long ip = rec->ip;
	374	+ unsigned const char *new;
	375	+
	376	+ new = ftrace_call_replace(ip, addr);
	377	+
	378	+ if (ftrace_write(ip, new, 1))
	379	+ return -EPERM;
	380	+
	381	+ return 0;
	382	+}
	383	+
	384	+static int finish_update_nop(struct dyn_ftrace *rec)
	385	+{
	386	+ unsigned long ip = rec->ip;
	387	+ unsigned const char *new;
	388	+
	389	+ new = ftrace_nop_replace();
	390	+
	391	+ if (ftrace_write(ip, new, 1))
	392	+ return -EPERM;
	393	+ return 0;
	394	+}
	395	+
	396	+static int finish_update(struct dyn_ftrace *rec, int enable)
	397	+{
	398	+ unsigned long ftrace_addr;
	399	+ int ret;
	400	+
	401	+ ret = ftrace_update_record(rec, enable);
	402	+
	403	+ ftrace_addr = (unsigned long)FTRACE_ADDR;
	404	+
	405	+ switch (ret) {
	406	+ case FTRACE_UPDATE_IGNORE:
	407	+ return 0;
	408	+
	409	+ case FTRACE_UPDATE_MAKE_CALL:
	410	+ /* converting nop to call */
	411	+ return finish_update_call(rec, ftrace_addr);
	412	+
	413	+ case FTRACE_UPDATE_MAKE_NOP:
	414	+ /* converting a call to a nop */
	415	+ return finish_update_nop(rec);
	416	+ }
	417	+
	418	+ return 0;
	419	+}
	420	+
	421	+static void do_sync_core(void *data)
	422	+{
	423	+ sync_core();
	424	+}
	425	+
	426	+static void run_sync(void)
	427	+{
	428	+ int enable_irqs = irqs_disabled();
	429	+
	430	+ /* We may be called with interrupts disbled (on bootup). */
	431	+ if (enable_irqs)
	432	+ local_irq_enable();
	433	+ on_each_cpu(do_sync_core, NULL, 1);
	434	+ if (enable_irqs)
	435	+ local_irq_disable();
	436	+}
	437	+
	438	+static void ftrace_replace_code(int enable)
	439	+{
	440	+ struct ftrace_rec_iter *iter;
	441	+ struct dyn_ftrace *rec;
	442	+ const char *report = "adding breakpoints";
	443	+ int count = 0;
	444	+ int ret;
	445	+
	446	+ for_ftrace_rec_iter(iter) {
	447	+ rec = ftrace_rec_iter_record(iter);
	448	+
	449	+ ret = add_breakpoints(rec, enable);
	450	+ if (ret)
	451	+ goto remove_breakpoints;
	452	+ count++;
	453	+ }
	454	+
	455	+ run_sync();
	456	+
	457	+ report = "updating code";
	458	+
	459	+ for_ftrace_rec_iter(iter) {
	460	+ rec = ftrace_rec_iter_record(iter);
	461	+
	462	+ ret = add_update(rec, enable);
	463	+ if (ret)
	464	+ goto remove_breakpoints;
	465	+ }
	466	+
	467	+ run_sync();
	468	+
	469	+ report = "removing breakpoints";
	470	+
	471	+ for_ftrace_rec_iter(iter) {
	472	+ rec = ftrace_rec_iter_record(iter);
	473	+
	474	+ ret = finish_update(rec, enable);
	475	+ if (ret)
	476	+ goto remove_breakpoints;
	477	+ }
	478	+
	479	+ run_sync();
	480	+
	481	+ return;
	482	+
	483	+ remove_breakpoints:
	484	+ ftrace_bug(ret, rec ? rec->ip : 0);
	485	+ printk(KERN_WARNING "Failed on %s (%d):\n", report, count);
	486	+ for_ftrace_rec_iter(iter) {
	487	+ rec = ftrace_rec_iter_record(iter);
	488	+ remove_breakpoint(rec);
	489	+ }
	490	+}
	491	+
	492	+void arch_ftrace_update_code(int command)
	493	+{
	494	+ modifying_ftrace_code++;
	495	+
	496	+ if (command & FTRACE_UPDATE_CALLS)
	497	+ ftrace_replace_code(1);
	498	+ else if (command & FTRACE_DISABLE_CALLS)
	499	+ ftrace_replace_code(0);
	500	+
	501	+ if (command & FTRACE_UPDATE_TRACE_FUNC)
	502	+ ftrace_update_ftrace_func(ftrace_trace_function);
	503	+
	504	+ if (command & FTRACE_START_FUNC_RET)
	505	+ ftrace_enable_ftrace_graph_caller();
	506	+ else if (command & FTRACE_STOP_FUNC_RET)
	507	+ ftrace_disable_ftrace_graph_caller();
	508	+
	509	+ modifying_ftrace_code--;
	510	+}
	511	+
337	512	int __init ftrace_dyn_arch_init(void *data)
338	513	{
339	514	/* The return code is retured via data */

arch/x86/kernel/nmi.c+5−5

@@ -84,7 +84,7 @@ __setup("unknown_nmi_panic", setup_unknown_nmi_panic); · 83 unmodified lines
84	84
85	85	#define nmi_to_desc(type) (&nmi_desc[type])
86	86
87		−static int notrace __kprobes nmi_handle(unsigned int type, struct pt_regs *regs, bool b2b)
	87	+static int __kprobes nmi_handle(unsigned int type, struct pt_regs *regs, bool b2b)
88	88	{
89	89	struct nmi_desc *desc = nmi_to_desc(type);
90	90	struct nmiaction *a;
@@ -209,7 +209,7 @@ void unregister_nmi_handler(unsigned int type, const char *name) · 118 unmodified lines
209	209
210	210	EXPORT_SYMBOL_GPL(unregister_nmi_handler);
211	211
212		−static notrace __kprobes void
	212	+static __kprobes void
213	213	pci_serr_error(unsigned char reason, struct pt_regs *regs)
214	214	{
215	215	pr_emerg("NMI: PCI system error (SERR) for reason %02x on CPU %d.\n",
@@ -236,7 +236,7 @@ pci_serr_error(unsigned char reason, struct pt_regs *regs) · 20 unmodified lines
236	236	outb(reason, NMI_REASON_PORT);
237	237	}
238	238
239		−static notrace __kprobes void
	239	+static __kprobes void
240	240	io_check_error(unsigned char reason, struct pt_regs *regs)
241	241	{
242	242	unsigned long i;
@@ -263,7 +263,7 @@ io_check_error(unsigned char reason, struct pt_regs *regs) · 20 unmodified lines
263	263	outb(reason, NMI_REASON_PORT);
264	264	}
265	265
266		−static notrace __kprobes void
	266	+static __kprobes void
267	267	unknown_nmi_error(unsigned char reason, struct pt_regs *regs)
268	268	{
269	269	int handled;
@@ -305,7 +305,7 @@ unknown_nmi_error(unsigned char reason, struct pt_regs *regs) · 35 unmodified lines
305	305	static DEFINE_PER_CPU(bool, swallow_nmi);
306	306	static DEFINE_PER_CPU(unsigned long, last_nmi_rip);
307	307
308		−static notrace __kprobes void default_do_nmi(struct pt_regs *regs)
	308	+static __kprobes void default_do_nmi(struct pt_regs *regs)
309	309	{
310	310	unsigned char reason = 0;
311	311	int handled;

arch/x86/kernel/traps.c+7−1

@@ -50,6 +50,7 @@ · 49 unmodified lines
50	50	#include <asm/processor.h>
51	51	#include <asm/debugreg.h>
52	52	#include <linux/atomic.h>
	53	+#include <asm/ftrace.h>
53	54	#include <asm/traps.h>
54	55	#include <asm/desc.h>
55	56	#include <asm/i387.h>
@@ -303,8 +304,13 @@ gp_in_kernel: · 247 unmodified lines
303	304	}
304	305
305	306	/* May run on IST stack. */
306		−dotraplinkage void __kprobes do_int3(struct pt_regs *regs, long error_code)
	307	+dotraplinkage void __kprobes notrace do_int3(struct pt_regs *regs, long error_code)
307	308	{
	309	+#ifdef CONFIG_DYNAMIC_FTRACE
	310	+ /* ftrace must be first, everything else may cause a recursive crash */
	311	+ if (unlikely(modifying_ftrace_code) && ftrace_int3_handler(regs))
	312	+ return;
	313	+#endif
308	314	#ifdef CONFIG_KGDB_LOW_LEVEL_TRAP
309	315	if (kgdb_ll_trap(DIE_INT3, "int3", regs, error_code, X86_TRAP_BP,
310	316	SIGTRAP) == NOTIFY_STOP)

include/linux/ftrace.h+6−0

@@ -286,6 +286,12 @@ struct ftrace_rec_iter *ftrace_rec_iter_start(void); · 285 unmodified lines
286	286	struct ftrace_rec_iter ftrace_rec_iter_next(struct ftrace_rec_iter iter);
287	287	struct dyn_ftrace ftrace_rec_iter_record(struct ftrace_rec_iter iter);
288	288
	289	+#define for_ftrace_rec_iter(iter) \
	290	+ for (iter = ftrace_rec_iter_start(); \
	291	+ iter; \
	292	+ iter = ftrace_rec_iter_next(iter))
	293	+
	294	+
289	295	int ftrace_update_record(struct dyn_ftrace *rec, int enable);
290	296	int ftrace_test_record(struct dyn_ftrace *rec, int enable);
291	297	void ftrace_run_stop_machine(int command);

include/linux/kernel.h+7−6

@@ -480,15 +480,16 @@ do { \ · 479 unmodified lines
480	480
481	481	#define trace_printk(fmt, args...) \
482	482	do { \
	483	+ static const char *trace_printk_fmt \
	484	+ __attribute__((section("__trace_printk_fmt"))) = \
	485	+ __builtin_constant_p(fmt) ? fmt : NULL; \
	486	+ \
483	487	__trace_printk_check_format(fmt, ##args); \
484		− if (__builtin_constant_p(fmt)) { \
485		− static const char *trace_printk_fmt \
486		− __attribute__((section("__trace_printk_fmt"))) = \
487		− __builtin_constant_p(fmt) ? fmt : NULL; \
488	488	\
	489	+ if (__builtin_constant_p(fmt)) \
489	490	__trace_bprintk(_THIS_IP_, trace_printk_fmt, ##args); \
490		− } else \
491		− __trace_printk(_THIS_IP_, fmt, ##args); \
	491	+ else \
	492	+ __trace_printk(_THIS_IP_, fmt, ##args); \
492	493	} while (0)
493	494
494	495	extern __printf(2, 3)

include/linux/ring_buffer.h+4−2

@@ -96,9 +96,11 @@ __ring_buffer_alloc(unsigned long size, unsigned flags, struct lock_class_key *k · 95 unmodified lines
96	96	__ring_buffer_alloc((size), (flags), &__key); \
97	97	})
98	98
	99	+#define RING_BUFFER_ALL_CPUS -1
	100	+
99	101	void ring_buffer_free(struct ring_buffer *buffer);
100	102
101		−int ring_buffer_resize(struct ring_buffer *buffer, unsigned long size);
	103	+int ring_buffer_resize(struct ring_buffer *buffer, unsigned long size, int cpu);
102	104
103	105	void ring_buffer_change_overwrite(struct ring_buffer *buffer, int val);
104	106
@@ -129,7 +131,7 @@ ring_buffer_read(struct ring_buffer_iter iter, u64 ts); · 24 unmodified lines
129	131	void ring_buffer_iter_reset(struct ring_buffer_iter *iter);
130	132	int ring_buffer_iter_empty(struct ring_buffer_iter *iter);
131	133
132		−unsigned long ring_buffer_size(struct ring_buffer *buffer);
	134	+unsigned long ring_buffer_size(struct ring_buffer *buffer, int cpu);
133	135
134	136	void ring_buffer_reset_cpu(struct ring_buffer *buffer, int cpu);
135	137	void ring_buffer_reset(struct ring_buffer *buffer);

kernel/trace/ring_buffer.c+140−108

@@ -449,6 +449,7 @@ struct ring_buffer_per_cpu { · 448 unmodified lines
449	449	raw_spinlock_t reader_lock; /* serialize readers */
450	450	arch_spinlock_t lock;
451	451	struct lock_class_key lock_key;
	452	+ unsigned int nr_pages;
452	453	struct list_head *pages;
453	454	struct buffer_page head_page; / read from head */
454	455	struct buffer_page tail_page; / write to tail */
@@ -466,10 +467,12 @@ struct ring_buffer_per_cpu { · 11 unmodified lines
466	467	unsigned long read_bytes;
467	468	u64 write_stamp;
468	469	u64 read_stamp;
	470	+ /* ring buffer pages to update, > 0 to add, < 0 to remove */
	471	+ int nr_pages_to_update;
	472	+ struct list_head new_pages; /* new pages to add */
469	473	};
470	474
471	475	struct ring_buffer {
472		− unsigned pages;
473	476	unsigned flags;
474	477	int cpus;
475	478	atomic_t record_disabled;
@@ -963,14 +966,10 @@ static int rb_check_pages(struct ring_buffer_per_cpu *cpu_buffer) · 487 unmodified lines
963	966	return 0;
964	967	}
965	968
966		−static int rb_allocate_pages(struct ring_buffer_per_cpu *cpu_buffer,
967		− unsigned nr_pages)
	969	+static int __rb_allocate_pages(int nr_pages, struct list_head *pages, int cpu)
968	970	{
	971	+ int i;
969	972	struct buffer_page bpage, tmp;
970		− LIST_HEAD(pages);
971		− unsigned i;
972		−
973		− WARN_ON(!nr_pages);
974	973
975	974	for (i = 0; i < nr_pages; i++) {
976	975	struct page *page;
@@ -981,15 +980,13 @@ static int rb_allocate_pages(struct ring_buffer_per_cpu *cpu_buffer, · 4 unmodified lines
981	980	*/
982	981	bpage = kzalloc_node(ALIGN(sizeof(*bpage), cache_line_size()),
983	982	GFP_KERNEL \| __GFP_NORETRY,
984		− cpu_to_node(cpu_buffer->cpu));
	983	+ cpu_to_node(cpu));
985	984	if (!bpage)
986	985	goto free_pages;
987	986
988		− rb_check_bpage(cpu_buffer, bpage);
	987	+ list_add(&bpage->list, pages);
989	988
990		− list_add(&bpage->list, &pages);
991		−
992		− page = alloc_pages_node(cpu_to_node(cpu_buffer->cpu),
	989	+ page = alloc_pages_node(cpu_to_node(cpu),
993	990	GFP_KERNEL \| __GFP_NORETRY, 0);
994	991	if (!page)
995	992	goto free_pages;
@@ -997,6 +994,27 @@ static int rb_allocate_pages(struct ring_buffer_per_cpu *cpu_buffer, · 1 unmodified line
997	994	rb_init_page(bpage->page);
998	995	}
999	996
	997	+ return 0;
	998	+
	999	+free_pages:
	1000	+ list_for_each_entry_safe(bpage, tmp, pages, list) {
	1001	+ list_del_init(&bpage->list);
	1002	+ free_buffer_page(bpage);
	1003	+ }
	1004	+
	1005	+ return -ENOMEM;
	1006	+}
	1007	+
	1008	+static int rb_allocate_pages(struct ring_buffer_per_cpu *cpu_buffer,
	1009	+ unsigned nr_pages)
	1010	+{
	1011	+ LIST_HEAD(pages);
	1012	+
	1013	+ WARN_ON(!nr_pages);
	1014	+
	1015	+ if (__rb_allocate_pages(nr_pages, &pages, cpu_buffer->cpu))
	1016	+ return -ENOMEM;
	1017	+
1000	1018	/*
1001	1019	* The ring buffer page list is a circular list that does not
1002	1020	* start and end with a list head. All page list items point to
@@ -1005,20 +1023,15 @@ static int rb_allocate_pages(struct ring_buffer_per_cpu *cpu_buffer, · 2 unmodified lines
1005	1023	cpu_buffer->pages = pages.next;
1006	1024	list_del(&pages);
1007	1025
	1026	+ cpu_buffer->nr_pages = nr_pages;
	1027	+
1008	1028	rb_check_pages(cpu_buffer);
1009	1029
1010	1030	return 0;
1011		−
1012		− free_pages:
1013		− list_for_each_entry_safe(bpage, tmp, &pages, list) {
1014		− list_del_init(&bpage->list);
1015		− free_buffer_page(bpage);
1016		− }
1017		− return -ENOMEM;
1018	1031	}
1019	1032
1020	1033	static struct ring_buffer_per_cpu *
1021		−rb_allocate_cpu_buffer(struct ring_buffer *buffer, int cpu)
	1034	+rb_allocate_cpu_buffer(struct ring_buffer *buffer, int nr_pages, int cpu)
1022	1035	{
1023	1036	struct ring_buffer_per_cpu *cpu_buffer;
1024	1037	struct buffer_page *bpage;
@@ -1052,7 +1065,7 @@ rb_allocate_cpu_buffer(struct ring_buffer *buffer, int cpu) · 27 unmodified lines
1052	1065
1053	1066	INIT_LIST_HEAD(&cpu_buffer->reader_page->list);
1054	1067
1055		− ret = rb_allocate_pages(cpu_buffer, buffer->pages);
	1068	+ ret = rb_allocate_pages(cpu_buffer, nr_pages);
1056	1069	if (ret < 0)
1057	1070	goto fail_free_reader;
1058	1071
@@ -1113,7 +1126,7 @@ struct ring_buffer *__ring_buffer_alloc(unsigned long size, unsigned flags, · 54 unmodified lines
1113	1126	{
1114	1127	struct ring_buffer *buffer;
1115	1128	int bsize;
1116		− int cpu;
	1129	+ int cpu, nr_pages;
1117	1130
1118	1131	/* keep it in its own cache line */
1119	1132	buffer = kzalloc(ALIGN(sizeof(*buffer), cache_line_size()),
@@ -1124,14 +1137,14 @@ struct ring_buffer *__ring_buffer_alloc(unsigned long size, unsigned flags, · 4 unmodified lines
1124	1137	if (!alloc_cpumask_var(&buffer->cpumask, GFP_KERNEL))
1125	1138	goto fail_free_buffer;
1126	1139
1127		− buffer->pages = DIV_ROUND_UP(size, BUF_PAGE_SIZE);
	1140	+ nr_pages = DIV_ROUND_UP(size, BUF_PAGE_SIZE);
1128	1141	buffer->flags = flags;
1129	1142	buffer->clock = trace_clock_local;
1130	1143	buffer->reader_lock_key = key;
1131	1144
1132	1145	/* need at least two pages */
1133		− if (buffer->pages < 2)
1134		− buffer->pages = 2;
	1146	+ if (nr_pages < 2)
	1147	+ nr_pages = 2;
1135	1148
1136	1149	/*
1137	1150	* In case of non-hotplug cpu, if the ring-buffer is allocated
@@ -1154,7 +1167,7 @@ struct ring_buffer *__ring_buffer_alloc(unsigned long size, unsigned flags, · 16 unmodified lines
1154	1167
1155	1168	for_each_buffer_cpu(buffer, cpu) {
1156	1169	buffer->buffers[cpu] =
1157		− rb_allocate_cpu_buffer(buffer, cpu);
	1170	+ rb_allocate_cpu_buffer(buffer, nr_pages, cpu);
1158	1171	if (!buffer->buffers[cpu])
1159	1172	goto fail_free_buffers;
1160	1173	}
@@ -1276,6 +1289,18 @@ out: · 115 unmodified lines
1276	1289	raw_spin_unlock_irq(&cpu_buffer->reader_lock);
1277	1290	}
1278	1291
	1292	+static void update_pages_handler(struct ring_buffer_per_cpu *cpu_buffer)
	1293	+{
	1294	+ if (cpu_buffer->nr_pages_to_update > 0)
	1295	+ rb_insert_pages(cpu_buffer, &cpu_buffer->new_pages,
	1296	+ cpu_buffer->nr_pages_to_update);
	1297	+ else
	1298	+ rb_remove_pages(cpu_buffer, -cpu_buffer->nr_pages_to_update);
	1299	+ cpu_buffer->nr_pages += cpu_buffer->nr_pages_to_update;
	1300	+ /* reset this value */
	1301	+ cpu_buffer->nr_pages_to_update = 0;
	1302	+}
	1303	+
1279	1304	/**
1280	1305	* ring_buffer_resize - resize the ring buffer
1281	1306	* @buffer: the buffer to resize.
@@ -1285,14 +1310,12 @@ out: · 3 unmodified lines
1285	1310	*
1286	1311	* Returns -1 on failure.
1287	1312	*/
1288		−int ring_buffer_resize(struct ring_buffer *buffer, unsigned long size)
	1313	+int ring_buffer_resize(struct ring_buffer *buffer, unsigned long size,
	1314	+ int cpu_id)
1289	1315	{
1290	1316	struct ring_buffer_per_cpu *cpu_buffer;
1291		− unsigned nr_pages, rm_pages, new_pages;
1292		− struct buffer_page bpage, tmp;
1293		− unsigned long buffer_size;
1294		− LIST_HEAD(pages);
1295		− int i, cpu;
	1317	+ unsigned nr_pages;
	1318	+ int cpu;
1296	1319
1297	1320	/*
1298	1321	* Always succeed at resizing a non-existent buffer:
@@ -1302,15 +1325,11 @@ int ring_buffer_resize(struct ring_buffer *buffer, unsigned long size) · 3 unmodified lines
1302	1325
1303	1326	size = DIV_ROUND_UP(size, BUF_PAGE_SIZE);
1304	1327	size *= BUF_PAGE_SIZE;
1305		− buffer_size = buffer->pages * BUF_PAGE_SIZE;
1306	1328
1307	1329	/* we need a minimum of two pages */
1308	1330	if (size < BUF_PAGE_SIZE * 2)
1309	1331	size = BUF_PAGE_SIZE * 2;
1310	1332
1311		− if (size == buffer_size)
1312		− return size;
1313		−
1314	1333	atomic_inc(&buffer->record_disabled);
1315	1334
1316	1335	/* Make sure all writers are done with this buffer. */
@@ -1321,68 +1340,56 @@ int ring_buffer_resize(struct ring_buffer *buffer, unsigned long size) · 4 unmodified lines
1321	1340
1322	1341	nr_pages = DIV_ROUND_UP(size, BUF_PAGE_SIZE);
1323	1342
1324		− if (size < buffer_size) {
1325		−
1326		− /* easy case, just free pages */
1327		− if (RB_WARN_ON(buffer, nr_pages >= buffer->pages))
1328		− goto out_fail;
1329		−
1330		− rm_pages = buffer->pages - nr_pages;
1331		−
	1343	+ if (cpu_id == RING_BUFFER_ALL_CPUS) {
	1344	+ /* calculate the pages to update */
1332	1345	for_each_buffer_cpu(buffer, cpu) {
1333	1346	cpu_buffer = buffer->buffers[cpu];
1334		− rb_remove_pages(cpu_buffer, rm_pages);
1335		− }
1336		− goto out;
1337		− }
1338	1347
1339		− /*
1340		− * This is a bit more difficult. We only want to add pages
1341		− * when we can allocate enough for all CPUs. We do this
1342		− * by allocating all the pages and storing them on a local
1343		− * link list. If we succeed in our allocation, then we
1344		− * add these pages to the cpu_buffers. Otherwise we just free
1345		− * them all and return -ENOMEM;
1346		− */
1347		− if (RB_WARN_ON(buffer, nr_pages <= buffer->pages))
1348		− goto out_fail;
	1348	+ cpu_buffer->nr_pages_to_update = nr_pages -
	1349	+ cpu_buffer->nr_pages;
1349	1350
1350		− new_pages = nr_pages - buffer->pages;
	1351	+ /*
	1352	+ * nothing more to do for removing pages or no update
	1353	+ */
	1354	+ if (cpu_buffer->nr_pages_to_update <= 0)
	1355	+ continue;
1351	1356
1352		− for_each_buffer_cpu(buffer, cpu) {
1353		− for (i = 0; i < new_pages; i++) {
1354		− struct page *page;
1355	1357	/*
1356		− * __GFP_NORETRY flag makes sure that the allocation
1357		− * fails gracefully without invoking oom-killer and
1358		− * the system is not destabilized.
	1358	+ * to add pages, make sure all new pages can be
	1359	+ * allocated without receiving ENOMEM
1359	1360	*/
1360		− bpage = kzalloc_node(ALIGN(sizeof(*bpage),
1361		− cache_line_size()),
1362		− GFP_KERNEL \| __GFP_NORETRY,
1363		− cpu_to_node(cpu));
1364		− if (!bpage)
1365		− goto free_pages;
1366		− list_add(&bpage->list, &pages);
1367		− page = alloc_pages_node(cpu_to_node(cpu),
1368		− GFP_KERNEL \| __GFP_NORETRY, 0);
1369		− if (!page)
1370		− goto free_pages;
1371		− bpage->page = page_address(page);
1372		− rb_init_page(bpage->page);
	1361	+ INIT_LIST_HEAD(&cpu_buffer->new_pages);
	1362	+ if (__rb_allocate_pages(cpu_buffer->nr_pages_to_update,
	1363	+ &cpu_buffer->new_pages, cpu))
	1364	+ /* not enough memory for new pages */
	1365	+ goto no_mem;
1373	1366	}
1374		− }
1375	1367
1376		− for_each_buffer_cpu(buffer, cpu) {
1377		− cpu_buffer = buffer->buffers[cpu];
1378		− rb_insert_pages(cpu_buffer, &pages, new_pages);
1379		− }
	1368	+ /* wait for all the updates to complete */
	1369	+ for_each_buffer_cpu(buffer, cpu) {
	1370	+ cpu_buffer = buffer->buffers[cpu];
	1371	+ if (cpu_buffer->nr_pages_to_update) {
	1372	+ update_pages_handler(cpu_buffer);
	1373	+ }
	1374	+ }
	1375	+ } else {
	1376	+ cpu_buffer = buffer->buffers[cpu_id];
	1377	+ if (nr_pages == cpu_buffer->nr_pages)
	1378	+ goto out;
1380	1379
1381		− if (RB_WARN_ON(buffer, !list_empty(&pages)))
1382		− goto out_fail;
	1380	+ cpu_buffer->nr_pages_to_update = nr_pages -
	1381	+ cpu_buffer->nr_pages;
	1382	+
	1383	+ INIT_LIST_HEAD(&cpu_buffer->new_pages);
	1384	+ if (cpu_buffer->nr_pages_to_update > 0 &&
	1385	+ __rb_allocate_pages(cpu_buffer->nr_pages_to_update,
	1386	+ &cpu_buffer->new_pages, cpu_id))
	1387	+ goto no_mem;
	1388	+
	1389	+ update_pages_handler(cpu_buffer);
	1390	+ }
1383	1391
1384	1392	out:
1385		− buffer->pages = nr_pages;
1386	1393	put_online_cpus();
1387	1394	mutex_unlock(&buffer->mutex);
1388	1395
@@ -1390,25 +1397,24 @@ int ring_buffer_resize(struct ring_buffer *buffer, unsigned long size) · 1 unmodified line
1390	1397
1391	1398	return size;
1392	1399
1393		− free_pages:
1394		− list_for_each_entry_safe(bpage, tmp, &pages, list) {
1395		− list_del_init(&bpage->list);
1396		− free_buffer_page(bpage);
	1400	+ no_mem:
	1401	+ for_each_buffer_cpu(buffer, cpu) {
	1402	+ struct buffer_page bpage, tmp;
	1403	+ cpu_buffer = buffer->buffers[cpu];
	1404	+ /* reset this number regardless */
	1405	+ cpu_buffer->nr_pages_to_update = 0;
	1406	+ if (list_empty(&cpu_buffer->new_pages))
	1407	+ continue;
	1408	+ list_for_each_entry_safe(bpage, tmp, &cpu_buffer->new_pages,
	1409	+ list) {
	1410	+ list_del_init(&bpage->list);
	1411	+ free_buffer_page(bpage);
	1412	+ }
1397	1413	}
1398	1414	put_online_cpus();
1399	1415	mutex_unlock(&buffer->mutex);
1400	1416	atomic_dec(&buffer->record_disabled);
1401	1417	return -ENOMEM;
1402		−
1403		− /*
1404		− * Something went totally wrong, and we are too paranoid
1405		− * to even clean up the mess.
1406		− */
1407		− out_fail:
1408		− put_online_cpus();
1409		− mutex_unlock(&buffer->mutex);
1410		− atomic_dec(&buffer->record_disabled);
1411		− return -1;
1412	1418	}
1413	1419	EXPORT_SYMBOL_GPL(ring_buffer_resize);
1414	1420
@@ -1510,7 +1516,7 @@ rb_set_commit_to_write(struct ring_buffer_per_cpu *cpu_buffer) · 95 unmodified lines
1510	1516	* assign the commit to the tail.
1511	1517	*/
1512	1518	again:
1513		− max_count = cpu_buffer->buffer->pages * 100;
	1519	+ max_count = cpu_buffer->nr_pages * 100;
1514	1520
1515	1521	while (cpu_buffer->commit_page != cpu_buffer->tail_page) {
1516	1522	if (RB_WARN_ON(cpu_buffer, !(--max_count)))
@@ -3588,9 +3594,18 @@ EXPORT_SYMBOL_GPL(ring_buffer_read); · 2071 unmodified lines
3588	3594	* ring_buffer_size - return the size of the ring buffer (in bytes)
3589	3595	* @buffer: The ring buffer.
3590	3596	*/
3591		−unsigned long ring_buffer_size(struct ring_buffer *buffer)
	3597	+unsigned long ring_buffer_size(struct ring_buffer *buffer, int cpu)
3592	3598	{
3593		− return BUF_PAGE_SIZE * buffer->pages;
	3599	+ /*
	3600	+ * Earlier, this method returned
	3601	+ * BUF_PAGE_SIZE * buffer->nr_pages
	3602	+ * Since the nr_pages field is now removed, we have converted this to
	3603	+ * return the per cpu buffer value.
	3604	+ */
	3605	+ if (!cpumask_test_cpu(cpu, buffer->cpumask))
	3606	+ return 0;
	3607	+
	3608	+ return BUF_PAGE_SIZE * buffer->buffers[cpu]->nr_pages;
3594	3609	}
3595	3610	EXPORT_SYMBOL_GPL(ring_buffer_size);
3596	3611
@@ -3765,8 +3780,11 @@ int ring_buffer_swap_cpu(struct ring_buffer *buffer_a, · 168 unmodified lines
3765	3780	!cpumask_test_cpu(cpu, buffer_b->cpumask))
3766	3781	goto out;
3767	3782
	3783	+ cpu_buffer_a = buffer_a->buffers[cpu];
	3784	+ cpu_buffer_b = buffer_b->buffers[cpu];
	3785	+
3768	3786	/* At least make sure the two buffers are somewhat the same */
3769		− if (buffer_a->pages != buffer_b->pages)
	3787	+ if (cpu_buffer_a->nr_pages != cpu_buffer_b->nr_pages)
3770	3788	goto out;
3771	3789
3772	3790	ret = -EAGAIN;
@@ -3780,9 +3798,6 @@ int ring_buffer_swap_cpu(struct ring_buffer *buffer_a, · 7 unmodified lines
3780	3798	if (atomic_read(&buffer_b->record_disabled))
3781	3799	goto out;
3782	3800
3783		− cpu_buffer_a = buffer_a->buffers[cpu];
3784		− cpu_buffer_b = buffer_b->buffers[cpu];
3785		−
3786	3801	if (atomic_read(&cpu_buffer_a->record_disabled))
3787	3802	goto out;
3788	3803
@@ -4071,6 +4086,8 @@ static int rb_cpu_notify(struct notifier_block *self, · 282 unmodified lines
4071	4086	struct ring_buffer *buffer =
4072	4087	container_of(self, struct ring_buffer, cpu_notify);
4073	4088	long cpu = (long)hcpu;
	4089	+ int cpu_i, nr_pages_same;
	4090	+ unsigned int nr_pages;
4074	4091
4075	4092	switch (action) {
4076	4093	case CPU_UP_PREPARE:
@@ -4078,8 +4095,23 @@ static int rb_cpu_notify(struct notifier_block *self, · 1 unmodified line
4078	4095	if (cpumask_test_cpu(cpu, buffer->cpumask))
4079	4096	return NOTIFY_OK;
4080	4097
	4098	+ nr_pages = 0;
	4099	+ nr_pages_same = 1;
	4100	+ /* check if all cpu sizes are same */
	4101	+ for_each_buffer_cpu(buffer, cpu_i) {
	4102	+ /* fill in the size from first enabled cpu */
	4103	+ if (nr_pages == 0)
	4104	+ nr_pages = buffer->buffers[cpu_i]->nr_pages;
	4105	+ if (nr_pages != buffer->buffers[cpu_i]->nr_pages) {
	4106	+ nr_pages_same = 0;
	4107	+ break;
	4108	+ }
	4109	+ }
	4110	+ /* allocate minimum pages, user can later expand it */
	4111	+ if (!nr_pages_same)
	4112	+ nr_pages = 2;
4081	4113	buffer->buffers[cpu] =
4082		− rb_allocate_cpu_buffer(buffer, cpu);
	4114	+ rb_allocate_cpu_buffer(buffer, nr_pages, cpu);
4083	4115	if (!buffer->buffers[cpu]) {
4084	4116	WARN(1, "failed to allocate ring buffer on CPU %ld\n",
4085	4117	cpu);

kernel/trace/trace.c+288−91

@@ -629,7 +629,6 @@ ssize_t trace_seq_to_user(struct trace_seq s, char __user ubuf, size_t cnt) · 628 unmodified lines
629	629	static ssize_t trace_seq_to_buffer(struct trace_seq s, void buf, size_t cnt)
630	630	{
631	631	int len;
632		− void *ret;
633	632
634	633	if (s->len <= s->readpos)
635	634	return -EBUSY;
@@ -637,9 +636,7 @@ static ssize_t trace_seq_to_buffer(struct trace_seq s, void buf, size_t cnt) · 1 unmodified line
637	636	len = s->len - s->readpos;
638	637	if (cnt > len)
639	638	cnt = len;
640		− ret = memcpy(buf, s->buffer + s->readpos, cnt);
641		− if (!ret)
642		− return -EFAULT;
	639	+ memcpy(buf, s->buffer + s->readpos, cnt);
643	640
644	641	s->readpos += cnt;
645	642	return cnt;
@@ -841,7 +838,8 @@ __acquires(kernel_lock) · 195 unmodified lines
841	838
842	839	/* If we expanded the buffers, make sure the max is expanded too */
843	840	if (ring_buffer_expanded && type->use_max_tr)
844		− ring_buffer_resize(max_tr.buffer, trace_buf_size);
	841	+ ring_buffer_resize(max_tr.buffer, trace_buf_size,
	842	+ RING_BUFFER_ALL_CPUS);
845	843
846	844	/* the test is responsible for initializing and enabling */
847	845	pr_info("Testing tracer %s: ", type->name);
@@ -857,7 +855,8 @@ __acquires(kernel_lock) · 9 unmodified lines
857	855
858	856	/* Shrink the max buffer again */
859	857	if (ring_buffer_expanded && type->use_max_tr)
860		− ring_buffer_resize(max_tr.buffer, 1);
	858	+ ring_buffer_resize(max_tr.buffer, 1,
	859	+ RING_BUFFER_ALL_CPUS);
861	860
862	861	printk(KERN_CONT "PASSED\n");
863	862	}
@@ -1498,25 +1497,119 @@ static void __trace_userstack(struct trace_array *tr, unsigned long flags) · 634 unmodified lines
1498	1497
1499	1498	#endif /* CONFIG_STACKTRACE */
1500	1499
	1500	+/* created for use with alloc_percpu */
	1501	+struct trace_buffer_struct {
	1502	+ char buffer[TRACE_BUF_SIZE];
	1503	+};
	1504	+
	1505	+static struct trace_buffer_struct *trace_percpu_buffer;
	1506	+static struct trace_buffer_struct *trace_percpu_sirq_buffer;
	1507	+static struct trace_buffer_struct *trace_percpu_irq_buffer;
	1508	+static struct trace_buffer_struct *trace_percpu_nmi_buffer;
	1509	+
	1510	+/*
	1511	+ * The buffer used is dependent on the context. There is a per cpu
	1512	+ * buffer for normal context, softirq contex, hard irq context and
	1513	+ * for NMI context. Thise allows for lockless recording.
	1514	+ *
	1515	+ * Note, if the buffers failed to be allocated, then this returns NULL
	1516	+ */
	1517	+static char *get_trace_buf(void)
	1518	+{
	1519	+ struct trace_buffer_struct *percpu_buffer;
	1520	+ struct trace_buffer_struct *buffer;
	1521	+
	1522	+ /*
	1523	+ * If we have allocated per cpu buffers, then we do not
	1524	+ * need to do any locking.
	1525	+ */
	1526	+ if (in_nmi())
	1527	+ percpu_buffer = trace_percpu_nmi_buffer;
	1528	+ else if (in_irq())
	1529	+ percpu_buffer = trace_percpu_irq_buffer;
	1530	+ else if (in_softirq())
	1531	+ percpu_buffer = trace_percpu_sirq_buffer;
	1532	+ else
	1533	+ percpu_buffer = trace_percpu_buffer;
	1534	+
	1535	+ if (!percpu_buffer)
	1536	+ return NULL;
	1537	+
	1538	+ buffer = per_cpu_ptr(percpu_buffer, smp_processor_id());
	1539	+
	1540	+ return buffer->buffer;
	1541	+}
	1542	+
	1543	+static int alloc_percpu_trace_buffer(void)
	1544	+{
	1545	+ struct trace_buffer_struct *buffers;
	1546	+ struct trace_buffer_struct *sirq_buffers;
	1547	+ struct trace_buffer_struct *irq_buffers;
	1548	+ struct trace_buffer_struct *nmi_buffers;
	1549	+
	1550	+ buffers = alloc_percpu(struct trace_buffer_struct);
	1551	+ if (!buffers)
	1552	+ goto err_warn;
	1553	+
	1554	+ sirq_buffers = alloc_percpu(struct trace_buffer_struct);
	1555	+ if (!sirq_buffers)
	1556	+ goto err_sirq;
	1557	+
	1558	+ irq_buffers = alloc_percpu(struct trace_buffer_struct);
	1559	+ if (!irq_buffers)
	1560	+ goto err_irq;
	1561	+
	1562	+ nmi_buffers = alloc_percpu(struct trace_buffer_struct);
	1563	+ if (!nmi_buffers)
	1564	+ goto err_nmi;
	1565	+
	1566	+ trace_percpu_buffer = buffers;
	1567	+ trace_percpu_sirq_buffer = sirq_buffers;
	1568	+ trace_percpu_irq_buffer = irq_buffers;
	1569	+ trace_percpu_nmi_buffer = nmi_buffers;
	1570	+
	1571	+ return 0;
	1572	+
	1573	+ err_nmi:
	1574	+ free_percpu(irq_buffers);
	1575	+ err_irq:
	1576	+ free_percpu(sirq_buffers);
	1577	+ err_sirq:
	1578	+ free_percpu(buffers);
	1579	+ err_warn:
	1580	+ WARN(1, "Could not allocate percpu trace_printk buffer");
	1581	+ return -ENOMEM;
	1582	+}
	1583	+
	1584	+void trace_printk_init_buffers(void)
	1585	+{
	1586	+ static int buffers_allocated;
	1587	+
	1588	+ if (buffers_allocated)
	1589	+ return;
	1590	+
	1591	+ if (alloc_percpu_trace_buffer())
	1592	+ return;
	1593	+
	1594	+ pr_info("ftrace: Allocated trace_printk buffers\n");
	1595	+
	1596	+ buffers_allocated = 1;
	1597	+}
	1598	+
1501	1599	/**
1502	1600	* trace_vbprintk - write binary msg to tracing buffer
1503	1601	*
1504	1602	*/
1505	1603	int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
1506	1604	{
1507		− static arch_spinlock_t trace_buf_lock =
1508		− (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
1509		− static u32 trace_buf[TRACE_BUF_SIZE];
1510		−
1511	1605	struct ftrace_event_call *call = &event_bprint;
1512	1606	struct ring_buffer_event *event;
1513	1607	struct ring_buffer *buffer;
1514	1608	struct trace_array *tr = &global_trace;
1515		− struct trace_array_cpu *data;
1516	1609	struct bprint_entry *entry;
1517	1610	unsigned long flags;
1518		− int disable;
1519		− int cpu, len = 0, size, pc;
	1611	+ char *tbuffer;
	1612	+ int len = 0, size, pc;
1520	1613
1521	1614	if (unlikely(tracing_selftest_running \|\| tracing_disabled))
1522	1615	return 0;
@@ -1526,43 +1619,36 @@ int trace_vbprintk(unsigned long ip, const char *fmt, va_list args) · 3 unmodified lines
1526	1619
1527	1620	pc = preempt_count();
1528	1621	preempt_disable_notrace();
1529		− cpu = raw_smp_processor_id();
1530		− data = tr->data[cpu];
1531	1622
1532		− disable = atomic_inc_return(&data->disabled);
1533		− if (unlikely(disable != 1))
	1623	+ tbuffer = get_trace_buf();
	1624	+ if (!tbuffer) {
	1625	+ len = 0;
1534	1626	goto out;
	1627	+ }
1535	1628
1536		− /* Lockdep uses trace_printk for lock tracing */
1537		− local_irq_save(flags);
1538		− arch_spin_lock(&trace_buf_lock);
1539		− len = vbin_printf(trace_buf, TRACE_BUF_SIZE, fmt, args);
	1629	+ len = vbin_printf((u32 *)tbuffer, TRACE_BUF_SIZE/sizeof(int), fmt, args);
1540	1630
1541		− if (len > TRACE_BUF_SIZE \|\| len < 0)
1542		− goto out_unlock;
	1631	+ if (len > TRACE_BUF_SIZE/sizeof(int) \|\| len < 0)
	1632	+ goto out;
1543	1633
	1634	+ local_save_flags(flags);
1544	1635	size = sizeof(entry) + sizeof(u32) len;
1545	1636	buffer = tr->buffer;
1546	1637	event = trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
1547	1638	flags, pc);
1548	1639	if (!event)
1549		− goto out_unlock;
	1640	+ goto out;
1550	1641	entry = ring_buffer_event_data(event);
1551	1642	entry->ip = ip;
1552	1643	entry->fmt = fmt;
1553	1644
1554		− memcpy(entry->buf, trace_buf, sizeof(u32) * len);
	1645	+ memcpy(entry->buf, tbuffer, sizeof(u32) * len);
1555	1646	if (!filter_check_discard(call, entry, buffer, event)) {
1556	1647	ring_buffer_unlock_commit(buffer, event);
1557	1648	ftrace_trace_stack(buffer, flags, 6, pc);
1558	1649	}
1559	1650
1560		−out_unlock:
1561		− arch_spin_unlock(&trace_buf_lock);
1562		− local_irq_restore(flags);
1563		−
1564	1651	out:
1565		− atomic_dec_return(&data->disabled);
1566	1652	preempt_enable_notrace();
1567	1653	unpause_graph_tracing();
1568	1654
@@ -1588,58 +1674,53 @@ int trace_array_printk(struct trace_array *tr, · 19 unmodified lines
1588	1674	int trace_array_vprintk(struct trace_array *tr,
1589	1675	unsigned long ip, const char *fmt, va_list args)
1590	1676	{
1591		− static arch_spinlock_t trace_buf_lock = __ARCH_SPIN_LOCK_UNLOCKED;
1592		− static char trace_buf[TRACE_BUF_SIZE];
1593		−
1594	1677	struct ftrace_event_call *call = &event_print;
1595	1678	struct ring_buffer_event *event;
1596	1679	struct ring_buffer *buffer;
1597		− struct trace_array_cpu *data;
1598		− int cpu, len = 0, size, pc;
	1680	+ int len = 0, size, pc;
1599	1681	struct print_entry *entry;
1600		− unsigned long irq_flags;
1601		− int disable;
	1682	+ unsigned long flags;
	1683	+ char *tbuffer;
1602	1684
1603	1685	if (tracing_disabled \|\| tracing_selftest_running)
1604	1686	return 0;
1605	1687
	1688	+ /* Don't pollute graph traces with trace_vprintk internals */
	1689	+ pause_graph_tracing();
	1690	+
1606	1691	pc = preempt_count();
1607	1692	preempt_disable_notrace();
1608		− cpu = raw_smp_processor_id();
1609		− data = tr->data[cpu];
1610	1693
1611		− disable = atomic_inc_return(&data->disabled);
1612		− if (unlikely(disable != 1))
	1694	+
	1695	+ tbuffer = get_trace_buf();
	1696	+ if (!tbuffer) {
	1697	+ len = 0;
1613	1698	goto out;
	1699	+ }
1614	1700
1615		− pause_graph_tracing();
1616		− raw_local_irq_save(irq_flags);
1617		− arch_spin_lock(&trace_buf_lock);
1618		− len = vsnprintf(trace_buf, TRACE_BUF_SIZE, fmt, args);
	1701	+ len = vsnprintf(tbuffer, TRACE_BUF_SIZE, fmt, args);
	1702	+ if (len > TRACE_BUF_SIZE)
	1703	+ goto out;
1619	1704
	1705	+ local_save_flags(flags);
1620	1706	size = sizeof(*entry) + len + 1;
1621	1707	buffer = tr->buffer;
1622	1708	event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
1623		− irq_flags, pc);
	1709	+ flags, pc);
1624	1710	if (!event)
1625		− goto out_unlock;
	1711	+ goto out;
1626	1712	entry = ring_buffer_event_data(event);
1627	1713	entry->ip = ip;
1628	1714
1629		− memcpy(&entry->buf, trace_buf, len);
	1715	+ memcpy(&entry->buf, tbuffer, len);
1630	1716	entry->buf[len] = '\0';
1631	1717	if (!filter_check_discard(call, entry, buffer, event)) {
1632	1718	ring_buffer_unlock_commit(buffer, event);
1633		− ftrace_trace_stack(buffer, irq_flags, 6, pc);
	1719	+ ftrace_trace_stack(buffer, flags, 6, pc);
1634	1720	}
1635		−
1636		− out_unlock:
1637		− arch_spin_unlock(&trace_buf_lock);
1638		− raw_local_irq_restore(irq_flags);
1639		− unpause_graph_tracing();
1640	1721	out:
1641		− atomic_dec_return(&data->disabled);
1642	1722	preempt_enable_notrace();
	1723	+ unpause_graph_tracing();
1643	1724
1644	1725	return len;
1645	1726	}
@@ -2974,7 +3055,14 @@ int tracer_init(struct tracer t, struct trace_array tr) · 1328 unmodified lines
2974	3055	return t->init(tr);
2975	3056	}
2976	3057
2977		−static int __tracing_resize_ring_buffer(unsigned long size)
	3058	+static void set_buffer_entries(struct trace_array *tr, unsigned long val)
	3059	+{
	3060	+ int cpu;
	3061	+ for_each_tracing_cpu(cpu)
	3062	+ tr->data[cpu]->entries = val;
	3063	+}
	3064	+
	3065	+static int __tracing_resize_ring_buffer(unsigned long size, int cpu)
2978	3066	{
2979	3067	int ret;
2980	3068
@@ -2985,19 +3073,32 @@ static int __tracing_resize_ring_buffer(unsigned long size) · 4 unmodified lines
2985	3073	*/
2986	3074	ring_buffer_expanded = 1;
2987	3075
2988		− ret = ring_buffer_resize(global_trace.buffer, size);
	3076	+ ret = ring_buffer_resize(global_trace.buffer, size, cpu);
2989	3077	if (ret < 0)
2990	3078	return ret;
2991	3079
2992	3080	if (!current_trace->use_max_tr)
2993	3081	goto out;
2994	3082
2995		− ret = ring_buffer_resize(max_tr.buffer, size);
	3083	+ ret = ring_buffer_resize(max_tr.buffer, size, cpu);
2996	3084	if (ret < 0) {
2997		− int r;
	3085	+ int r = 0;
	3086	+
	3087	+ if (cpu == RING_BUFFER_ALL_CPUS) {
	3088	+ int i;
	3089	+ for_each_tracing_cpu(i) {
	3090	+ r = ring_buffer_resize(global_trace.buffer,
	3091	+ global_trace.data[i]->entries,
	3092	+ i);
	3093	+ if (r < 0)
	3094	+ break;
	3095	+ }
	3096	+ } else {
	3097	+ r = ring_buffer_resize(global_trace.buffer,
	3098	+ global_trace.data[cpu]->entries,
	3099	+ cpu);
	3100	+ }
2998	3101
2999		− r = ring_buffer_resize(global_trace.buffer,
3000		− global_trace.entries);
3001	3102	if (r < 0) {
3002	3103	/*
3003	3104	* AARGH! We are left with different
@@ -3019,14 +3120,21 @@ static int __tracing_resize_ring_buffer(unsigned long size) · 15 unmodified lines
3019	3120	return ret;
3020	3121	}
3021	3122
3022		− max_tr.entries = size;
	3123	+ if (cpu == RING_BUFFER_ALL_CPUS)
	3124	+ set_buffer_entries(&max_tr, size);
	3125	+ else
	3126	+ max_tr.data[cpu]->entries = size;
	3127	+
3023	3128	out:
3024		− global_trace.entries = size;
	3129	+ if (cpu == RING_BUFFER_ALL_CPUS)
	3130	+ set_buffer_entries(&global_trace, size);
	3131	+ else
	3132	+ global_trace.data[cpu]->entries = size;
3025	3133
3026	3134	return ret;
3027	3135	}
3028	3136
3029		−static ssize_t tracing_resize_ring_buffer(unsigned long size)
	3137	+static ssize_t tracing_resize_ring_buffer(unsigned long size, int cpu_id)
3030	3138	{
3031	3139	int cpu, ret = size;
3032	3140
@@ -3042,12 +3150,19 @@ static ssize_t tracing_resize_ring_buffer(unsigned long size) · 9 unmodified lines
3042	3150	atomic_inc(&max_tr.data[cpu]->disabled);
3043	3151	}
3044	3152
3045		− if (size != global_trace.entries)
3046		− ret = __tracing_resize_ring_buffer(size);
	3153	+ if (cpu_id != RING_BUFFER_ALL_CPUS) {
	3154	+ /* make sure, this cpu is enabled in the mask */
	3155	+ if (!cpumask_test_cpu(cpu_id, tracing_buffer_mask)) {
	3156	+ ret = -EINVAL;
	3157	+ goto out;
	3158	+ }
	3159	+ }
3047	3160
	3161	+ ret = __tracing_resize_ring_buffer(size, cpu_id);
3048	3162	if (ret < 0)
3049	3163	ret = -ENOMEM;
3050	3164
	3165	+out:
3051	3166	for_each_tracing_cpu(cpu) {
3052	3167	if (global_trace.data[cpu])
3053	3168	atomic_dec(&global_trace.data[cpu]->disabled);
@@ -3078,7 +3193,8 @@ int tracing_update_buffers(void) · 24 unmodified lines
3078	3193
3079	3194	mutex_lock(&trace_types_lock);
3080	3195	if (!ring_buffer_expanded)
3081		− ret = __tracing_resize_ring_buffer(trace_buf_size);
	3196	+ ret = __tracing_resize_ring_buffer(trace_buf_size,
	3197	+ RING_BUFFER_ALL_CPUS);
3082	3198	mutex_unlock(&trace_types_lock);
3083	3199
3084	3200	return ret;
@@ -3102,7 +3218,8 @@ static int tracing_set_tracer(const char *buf) · 17 unmodified lines
3102	3218	mutex_lock(&trace_types_lock);
3103	3219
3104	3220	if (!ring_buffer_expanded) {
3105		− ret = __tracing_resize_ring_buffer(trace_buf_size);
	3221	+ ret = __tracing_resize_ring_buffer(trace_buf_size,
	3222	+ RING_BUFFER_ALL_CPUS);
3106	3223	if (ret < 0)
3107	3224	goto out;
3108	3225	ret = 0;
@@ -3128,8 +3245,8 @@ static int tracing_set_tracer(const char *buf) · 19 unmodified lines
3128	3245	* The max_tr ring buffer has some state (e.g. ring->clock) and
3129	3246	* we want preserve it.
3130	3247	*/
3131		− ring_buffer_resize(max_tr.buffer, 1);
3132		− max_tr.entries = 1;
	3248	+ ring_buffer_resize(max_tr.buffer, 1, RING_BUFFER_ALL_CPUS);
	3249	+ set_buffer_entries(&max_tr, 1);
3133	3250	}
3134	3251	destroy_trace_option_files(topts);
3135	3252
@@ -3137,10 +3254,17 @@ static int tracing_set_tracer(const char *buf) · 1 unmodified line
3137	3254
3138	3255	topts = create_trace_option_files(current_trace);
3139	3256	if (current_trace->use_max_tr) {
3140		− ret = ring_buffer_resize(max_tr.buffer, global_trace.entries);
3141		− if (ret < 0)
3142		− goto out;
3143		− max_tr.entries = global_trace.entries;
	3257	+ int cpu;
	3258	+ /* we need to make per cpu buffer sizes equivalent */
	3259	+ for_each_tracing_cpu(cpu) {
	3260	+ ret = ring_buffer_resize(max_tr.buffer,
	3261	+ global_trace.data[cpu]->entries,
	3262	+ cpu);
	3263	+ if (ret < 0)
	3264	+ goto out;
	3265	+ max_tr.data[cpu]->entries =
	3266	+ global_trace.data[cpu]->entries;
	3267	+ }
3144	3268	}
3145	3269
3146	3270	if (t->init) {
@@ -3642,30 +3766,82 @@ out_err: · 495 unmodified lines
3642	3766	goto out;
3643	3767	}
3644	3768
	3769	+struct ftrace_entries_info {
	3770	+ struct trace_array *tr;
	3771	+ int cpu;
	3772	+};
	3773	+
	3774	+static int tracing_entries_open(struct inode inode, struct file filp)
	3775	+{
	3776	+ struct ftrace_entries_info *info;
	3777	+
	3778	+ if (tracing_disabled)
	3779	+ return -ENODEV;
	3780	+
	3781	+ info = kzalloc(sizeof(*info), GFP_KERNEL);
	3782	+ if (!info)
	3783	+ return -ENOMEM;
	3784	+
	3785	+ info->tr = &global_trace;
	3786	+ info->cpu = (unsigned long)inode->i_private;
	3787	+
	3788	+ filp->private_data = info;
	3789	+
	3790	+ return 0;
	3791	+}
	3792	+
3645	3793	static ssize_t
3646	3794	tracing_entries_read(struct file filp, char __user ubuf,
3647	3795	size_t cnt, loff_t *ppos)
3648	3796	{
3649		− struct trace_array *tr = filp->private_data;
3650		− char buf[96];
3651		− int r;
	3797	+ struct ftrace_entries_info *info = filp->private_data;
	3798	+ struct trace_array *tr = info->tr;
	3799	+ char buf[64];
	3800	+ int r = 0;
	3801	+ ssize_t ret;
3652	3802
3653	3803	mutex_lock(&trace_types_lock);
3654		− if (!ring_buffer_expanded)
3655		− r = sprintf(buf, "%lu (expanded: %lu)\n",
3656		− tr->entries >> 10,
3657		− trace_buf_size >> 10);
3658		− else
3659		− r = sprintf(buf, "%lu\n", tr->entries >> 10);
	3804	+
	3805	+ if (info->cpu == RING_BUFFER_ALL_CPUS) {
	3806	+ int cpu, buf_size_same;
	3807	+ unsigned long size;
	3808	+
	3809	+ size = 0;
	3810	+ buf_size_same = 1;
	3811	+ /* check if all cpu sizes are same */
	3812	+ for_each_tracing_cpu(cpu) {
	3813	+ /* fill in the size from first enabled cpu */
	3814	+ if (size == 0)
	3815	+ size = tr->data[cpu]->entries;
	3816	+ if (size != tr->data[cpu]->entries) {
	3817	+ buf_size_same = 0;
	3818	+ break;
	3819	+ }
	3820	+ }
	3821	+
	3822	+ if (buf_size_same) {
	3823	+ if (!ring_buffer_expanded)
	3824	+ r = sprintf(buf, "%lu (expanded: %lu)\n",
	3825	+ size >> 10,
	3826	+ trace_buf_size >> 10);
	3827	+ else
	3828	+ r = sprintf(buf, "%lu\n", size >> 10);
	3829	+ } else
	3830	+ r = sprintf(buf, "X\n");
	3831	+ } else
	3832	+ r = sprintf(buf, "%lu\n", tr->data[info->cpu]->entries >> 10);
	3833	+
3660	3834	mutex_unlock(&trace_types_lock);
3661	3835
3662		− return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
	3836	+ ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
	3837	+ return ret;
3663	3838	}
3664	3839
3665	3840	static ssize_t
3666	3841	tracing_entries_write(struct file filp, const char __user ubuf,
3667	3842	size_t cnt, loff_t *ppos)
3668	3843	{
	3844	+ struct ftrace_entries_info *info = filp->private_data;
3669	3845	unsigned long val;
3670	3846	int ret;
3671	3847
@@ -3680,7 +3856,7 @@ tracing_entries_write(struct file filp, const char __user ubuf, · 8 unmodified lines
3680	3856	/* value is in KB */
3681	3857	val <<= 10;
3682	3858
3683		− ret = tracing_resize_ring_buffer(val);
	3859	+ ret = tracing_resize_ring_buffer(val, info->cpu);
3684	3860	if (ret < 0)
3685	3861	return ret;
3686	3862
@@ -3689,6 +3865,16 @@ tracing_entries_write(struct file filp, const char __user ubuf, · 2 unmodified lines
3689	3865	return cnt;
3690	3866	}
3691	3867
	3868	+static int
	3869	+tracing_entries_release(struct inode inode, struct file filp)
	3870	+{
	3871	+ struct ftrace_entries_info *info = filp->private_data;
	3872	+
	3873	+ kfree(info);
	3874	+
	3875	+ return 0;
	3876	+}
	3877	+
3692	3878	static ssize_t
3693	3879	tracing_total_entries_read(struct file filp, char __user ubuf,
3694	3880	size_t cnt, loff_t *ppos)
@@ -3700,7 +3886,7 @@ tracing_total_entries_read(struct file filp, char __user ubuf, · 5 unmodified lines
3700	3886
3701	3887	mutex_lock(&trace_types_lock);
3702	3888	for_each_tracing_cpu(cpu) {
3703		− size += tr->entries >> 10;
	3889	+ size += tr->data[cpu]->entries >> 10;
3704	3890	if (!ring_buffer_expanded)
3705	3891	expanded_size += trace_buf_size >> 10;
3706	3892	}
@@ -3734,7 +3920,7 @@ tracing_free_buffer_release(struct inode inode, struct file filp) · 27 unmodified lines
3734	3920	if (trace_flags & TRACE_ITER_STOP_ON_FREE)
3735	3921	tracing_off();
3736	3922	/* resize the ring buffer to 0 */
3737		− tracing_resize_ring_buffer(0);
	3923	+ tracing_resize_ring_buffer(0, RING_BUFFER_ALL_CPUS);
3738	3924
3739	3925	return 0;
3740	3926	}
@@ -3933,9 +4119,10 @@ static const struct file_operations tracing_pipe_fops = { · 192 unmodified lines
3933	4119	};
3934	4120
3935	4121	static const struct file_operations tracing_entries_fops = {
3936		− .open = tracing_open_generic,
	4122	+ .open = tracing_entries_open,
3937	4123	.read = tracing_entries_read,
3938	4124	.write = tracing_entries_write,
	4125	+ .release = tracing_entries_release,
3939	4126	.llseek = generic_file_llseek,
3940	4127	};
3941	4128
@@ -4387,6 +4574,9 @@ static void tracing_init_debugfs_percpu(long cpu) · 445 unmodified lines
4387	4574
4388	4575	trace_create_file("stats", 0444, d_cpu,
4389	4576	(void *) cpu, &tracing_stats_fops);
	4577	+
	4578	+ trace_create_file("buffer_size_kb", 0444, d_cpu,
	4579	+ (void *) cpu, &tracing_entries_fops);
4390	4580	}
4391	4581
4392	4582	#ifdef CONFIG_FTRACE_SELFTEST
@@ -4716,7 +4906,7 @@ static __init int tracer_init_debugfs(void) · 323 unmodified lines
4716	4906	(void *) TRACE_PIPE_ALL_CPU, &tracing_pipe_fops);
4717	4907
4718	4908	trace_create_file("buffer_size_kb", 0644, d_tracer,
4719		− &global_trace, &tracing_entries_fops);
	4909	+ (void *) RING_BUFFER_ALL_CPUS, &tracing_entries_fops);
4720	4910
4721	4911	trace_create_file("buffer_total_size_kb", 0444, d_tracer,
4722	4912	&global_trace, &tracing_total_entries_fops);
@@ -4955,6 +5145,10 @@ __init static int tracer_alloc_buffers(void) · 232 unmodified lines
4955	5145	if (!alloc_cpumask_var(&tracing_cpumask, GFP_KERNEL))
4956	5146	goto out_free_buffer_mask;
4957	5147
	5148	+ /* Only allocate trace_printk buffers if a trace_printk exists */
	5149	+ if (__stop___trace_bprintk_fmt != __start___trace_bprintk_fmt)
	5150	+ trace_printk_init_buffers();
	5151	+
4958	5152	/* To save memory, keep the ring buffer size to its minimum */
4959	5153	if (ring_buffer_expanded)
4960	5154	ring_buf_size = trace_buf_size;
@@ -4973,7 +5167,6 @@ __init static int tracer_alloc_buffers(void) · 12 unmodified lines
4973	5167	WARN_ON(1);
4974	5168	goto out_free_cpumask;
4975	5169	}
4976		− global_trace.entries = ring_buffer_size(global_trace.buffer);
4977	5170	if (global_trace.buffer_disabled)
4978	5171	tracing_off();
4979	5172
@@ -4986,7 +5179,6 @@ __init static int tracer_alloc_buffers(void) · 6 unmodified lines
4986	5179	ring_buffer_free(global_trace.buffer);
4987	5180	goto out_free_cpumask;
4988	5181	}
4989		− max_tr.entries = 1;
4990	5182	#endif
4991	5183
4992	5184	/* Allocate the first page for all buffers */
@@ -4995,6 +5187,11 @@ __init static int tracer_alloc_buffers(void) · 2 unmodified lines
4995	5187	max_tr.data[i] = &per_cpu(max_tr_data, i);
4996	5188	}
4997	5189
	5190	+ set_buffer_entries(&global_trace, ring_buf_size);
	5191	+#ifdef CONFIG_TRACER_MAX_TRACE
	5192	+ set_buffer_entries(&max_tr, 1);
	5193	+#endif
	5194	+
4998	5195	trace_init_cmdlines();
4999	5196
5000	5197	register_tracer(&nop_trace);

kernel/trace/trace.h+3−1

@@ -131,6 +131,7 @@ struct trace_array_cpu { · 130 unmodified lines
131	131	atomic_t disabled;
132	132	void buffer_page; / ring buffer spare */
133	133
	134	+ unsigned long entries;
134	135	unsigned long saved_latency;
135	136	unsigned long critical_start;
136	137	unsigned long critical_end;
@@ -152,7 +153,6 @@ struct trace_array_cpu { · 15 unmodified lines
152	153	*/
153	154	struct trace_array {
154	155	struct ring_buffer *buffer;
155		− unsigned long entries;
156	156	int cpu;
157	157	int buffer_disabled;
158	158	cycle_t time_start;
@@ -826,6 +826,8 @@ extern struct list_head ftrace_events; · 667 unmodified lines
826	826	extern const char *__start___trace_bprintk_fmt[];
827	827	extern const char *__stop___trace_bprintk_fmt[];
828	828
	829	+void trace_printk_init_buffers(void);
	830	+
829	831	#undef FTRACE_ENTRY
830	832	#define FTRACE_ENTRY(call, struct_name, id, tstruct, print, filter) \
831	833	extern struct ftrace_event_call \

kernel/trace/trace_printk.c+4−0

@@ -51,6 +51,10 @@ void hold_module_trace_bprintk_format(const char start, const char end) · 50 unmodified lines
51	51	const char **iter;
52	52	char *fmt;
53	53
	54	+ /* allocate the trace_printk per cpu buffers */
	55	+ if (start != end)
	56	+ trace_printk_init_buffers();
	57	+
54	58	mutex_lock(&btrace_mutex);
55	59	for (iter = start; iter < end; iter++) {
56	60	struct trace_bprintk_fmt tb_fmt = lookup_format(iter);