History

Merge branch 'tip/perf/core-4' of git://git.kernel.org/pub/scm/linux/kernel/git/rostedt/linux-trace into perf/core

Ingo Molnar committed 14 years ago

Commit 19631cb2 parents 1fa2e84 59a094c12 files · +810 −383Browse files

Showing the changes to kernel (4 of 12 files). Show all

Unified · Split

4 files
  1. kernel/trace/ring_buffer.c +140 −108
  2. kernel/trace/trace.c +288 −91
  3. kernel/trace/trace.h +3 −1
  4. kernel/trace/trace_printk.c +4 −0
kernel/trace/ring_buffer.c+140−108
@@ -449,6 +449,7 @@ struct ring_buffer_per_cpu { · 448 unmodified lines
449449 raw_spinlock_t reader_lock; /* serialize readers */
450450 arch_spinlock_t lock;
451451 struct lock_class_key lock_key;
452+ unsigned int nr_pages;
452453 struct list_head *pages;
453454 struct buffer_page *head_page; /* read from head */
454455 struct buffer_page *tail_page; /* write to tail */
@@ -466,10 +467,12 @@ struct ring_buffer_per_cpu { · 11 unmodified lines
466467 unsigned long read_bytes;
467468 u64 write_stamp;
468469 u64 read_stamp;
470+ /* ring buffer pages to update, > 0 to add, < 0 to remove */
471+ int nr_pages_to_update;
472+ struct list_head new_pages; /* new pages to add */
469473 };
470474
471475 struct ring_buffer {
472− unsigned pages;
473476 unsigned flags;
474477 int cpus;
475478 atomic_t record_disabled;
@@ -963,14 +966,10 @@ static int rb_check_pages(struct ring_buffer_per_cpu *cpu_buffer) · 487 unmodified lines
963966 return 0;
964967 }
965968
966−static int rb_allocate_pages(struct ring_buffer_per_cpu *cpu_buffer,
967− unsigned nr_pages)
969+static int __rb_allocate_pages(int nr_pages, struct list_head *pages, int cpu)
968970 {
971+ int i;
969972 struct buffer_page *bpage, *tmp;
970− LIST_HEAD(pages);
971− unsigned i;
972−
973− WARN_ON(!nr_pages);
974973
975974 for (i = 0; i < nr_pages; i++) {
976975 struct page *page;
@@ -981,15 +980,13 @@ static int rb_allocate_pages(struct ring_buffer_per_cpu *cpu_buffer, · 4 unmodified lines
981980 */
982981 bpage = kzalloc_node(ALIGN(sizeof(*bpage), cache_line_size()),
983982 GFP_KERNEL | __GFP_NORETRY,
984− cpu_to_node(cpu_buffer->cpu));
983+ cpu_to_node(cpu));
985984 if (!bpage)
986985 goto free_pages;
987986
988− rb_check_bpage(cpu_buffer, bpage);
987+ list_add(&bpage->list, pages);
989988
990− list_add(&bpage->list, &pages);
991−
992− page = alloc_pages_node(cpu_to_node(cpu_buffer->cpu),
989+ page = alloc_pages_node(cpu_to_node(cpu),
993990 GFP_KERNEL | __GFP_NORETRY, 0);
994991 if (!page)
995992 goto free_pages;
@@ -997,6 +994,27 @@ static int rb_allocate_pages(struct ring_buffer_per_cpu *cpu_buffer, · 1 unmodified line
997994 rb_init_page(bpage->page);
998995 }
999996
997+ return 0;
998+
999+free_pages:
1000+ list_for_each_entry_safe(bpage, tmp, pages, list) {
1001+ list_del_init(&bpage->list);
1002+ free_buffer_page(bpage);
1003+ }
1004+
1005+ return -ENOMEM;
1006+}
1007+
1008+static int rb_allocate_pages(struct ring_buffer_per_cpu *cpu_buffer,
1009+ unsigned nr_pages)
1010+{
1011+ LIST_HEAD(pages);
1012+
1013+ WARN_ON(!nr_pages);
1014+
1015+ if (__rb_allocate_pages(nr_pages, &pages, cpu_buffer->cpu))
1016+ return -ENOMEM;
1017+
10001018 /*
10011019 * The ring buffer page list is a circular list that does not
10021020 * start and end with a list head. All page list items point to
@@ -1005,20 +1023,15 @@ static int rb_allocate_pages(struct ring_buffer_per_cpu *cpu_buffer, · 2 unmodified lines
10051023 cpu_buffer->pages = pages.next;
10061024 list_del(&pages);
10071025
1026+ cpu_buffer->nr_pages = nr_pages;
1027+
10081028 rb_check_pages(cpu_buffer);
10091029
10101030 return 0;
1011−
1012− free_pages:
1013− list_for_each_entry_safe(bpage, tmp, &pages, list) {
1014− list_del_init(&bpage->list);
1015− free_buffer_page(bpage);
1016− }
1017− return -ENOMEM;
10181031 }
10191032
10201033 static struct ring_buffer_per_cpu *
1021−rb_allocate_cpu_buffer(struct ring_buffer *buffer, int cpu)
1034+rb_allocate_cpu_buffer(struct ring_buffer *buffer, int nr_pages, int cpu)
10221035 {
10231036 struct ring_buffer_per_cpu *cpu_buffer;
10241037 struct buffer_page *bpage;
@@ -1052,7 +1065,7 @@ rb_allocate_cpu_buffer(struct ring_buffer *buffer, int cpu) · 27 unmodified lines
10521065
10531066 INIT_LIST_HEAD(&cpu_buffer->reader_page->list);
10541067
1055− ret = rb_allocate_pages(cpu_buffer, buffer->pages);
1068+ ret = rb_allocate_pages(cpu_buffer, nr_pages);
10561069 if (ret < 0)
10571070 goto fail_free_reader;
10581071
@@ -1113,7 +1126,7 @@ struct ring_buffer *__ring_buffer_alloc(unsigned long size, unsigned flags, · 54 unmodified lines
11131126 {
11141127 struct ring_buffer *buffer;
11151128 int bsize;
1116− int cpu;
1129+ int cpu, nr_pages;
11171130
11181131 /* keep it in its own cache line */
11191132 buffer = kzalloc(ALIGN(sizeof(*buffer), cache_line_size()),
@@ -1124,14 +1137,14 @@ struct ring_buffer *__ring_buffer_alloc(unsigned long size, unsigned flags, · 4 unmodified lines
11241137 if (!alloc_cpumask_var(&buffer->cpumask, GFP_KERNEL))
11251138 goto fail_free_buffer;
11261139
1127− buffer->pages = DIV_ROUND_UP(size, BUF_PAGE_SIZE);
1140+ nr_pages = DIV_ROUND_UP(size, BUF_PAGE_SIZE);
11281141 buffer->flags = flags;
11291142 buffer->clock = trace_clock_local;
11301143 buffer->reader_lock_key = key;
11311144
11321145 /* need at least two pages */
1133− if (buffer->pages < 2)
1134− buffer->pages = 2;
1146+ if (nr_pages < 2)
1147+ nr_pages = 2;
11351148
11361149 /*
11371150 * In case of non-hotplug cpu, if the ring-buffer is allocated
@@ -1154,7 +1167,7 @@ struct ring_buffer *__ring_buffer_alloc(unsigned long size, unsigned flags, · 16 unmodified lines
11541167
11551168 for_each_buffer_cpu(buffer, cpu) {
11561169 buffer->buffers[cpu] =
1157− rb_allocate_cpu_buffer(buffer, cpu);
1170+ rb_allocate_cpu_buffer(buffer, nr_pages, cpu);
11581171 if (!buffer->buffers[cpu])
11591172 goto fail_free_buffers;
11601173 }
@@ -1276,6 +1289,18 @@ out: · 115 unmodified lines
12761289 raw_spin_unlock_irq(&cpu_buffer->reader_lock);
12771290 }
12781291
1292+static void update_pages_handler(struct ring_buffer_per_cpu *cpu_buffer)
1293+{
1294+ if (cpu_buffer->nr_pages_to_update > 0)
1295+ rb_insert_pages(cpu_buffer, &cpu_buffer->new_pages,
1296+ cpu_buffer->nr_pages_to_update);
1297+ else
1298+ rb_remove_pages(cpu_buffer, -cpu_buffer->nr_pages_to_update);
1299+ cpu_buffer->nr_pages += cpu_buffer->nr_pages_to_update;
1300+ /* reset this value */
1301+ cpu_buffer->nr_pages_to_update = 0;
1302+}
1303+
12791304 /**
12801305 * ring_buffer_resize - resize the ring buffer
12811306 * @buffer: the buffer to resize.
@@ -1285,14 +1310,12 @@ out: · 3 unmodified lines
12851310 *
12861311 * Returns -1 on failure.
12871312 */
1288−int ring_buffer_resize(struct ring_buffer *buffer, unsigned long size)
1313+int ring_buffer_resize(struct ring_buffer *buffer, unsigned long size,
1314+ int cpu_id)
12891315 {
12901316 struct ring_buffer_per_cpu *cpu_buffer;
1291− unsigned nr_pages, rm_pages, new_pages;
1292− struct buffer_page *bpage, *tmp;
1293− unsigned long buffer_size;
1294− LIST_HEAD(pages);
1295− int i, cpu;
1317+ unsigned nr_pages;
1318+ int cpu;
12961319
12971320 /*
12981321 * Always succeed at resizing a non-existent buffer:
@@ -1302,15 +1325,11 @@ int ring_buffer_resize(struct ring_buffer *buffer, unsigned long size) · 3 unmodified lines
13021325
13031326 size = DIV_ROUND_UP(size, BUF_PAGE_SIZE);
13041327 size *= BUF_PAGE_SIZE;
1305− buffer_size = buffer->pages * BUF_PAGE_SIZE;
13061328
13071329 /* we need a minimum of two pages */
13081330 if (size < BUF_PAGE_SIZE * 2)
13091331 size = BUF_PAGE_SIZE * 2;
13101332
1311− if (size == buffer_size)
1312− return size;
1313−
13141333 atomic_inc(&buffer->record_disabled);
13151334
13161335 /* Make sure all writers are done with this buffer. */
@@ -1321,68 +1340,56 @@ int ring_buffer_resize(struct ring_buffer *buffer, unsigned long size) · 4 unmodified lines
13211340
13221341 nr_pages = DIV_ROUND_UP(size, BUF_PAGE_SIZE);
13231342
1324− if (size < buffer_size) {
1325−
1326− /* easy case, just free pages */
1327− if (RB_WARN_ON(buffer, nr_pages >= buffer->pages))
1328− goto out_fail;
1329−
1330− rm_pages = buffer->pages - nr_pages;
1331−
1343+ if (cpu_id == RING_BUFFER_ALL_CPUS) {
1344+ /* calculate the pages to update */
13321345 for_each_buffer_cpu(buffer, cpu) {
13331346 cpu_buffer = buffer->buffers[cpu];
1334− rb_remove_pages(cpu_buffer, rm_pages);
1335− }
1336− goto out;
1337− }
13381347
1339− /*
1340− * This is a bit more difficult. We only want to add pages
1341− * when we can allocate enough for all CPUs. We do this
1342− * by allocating all the pages and storing them on a local
1343− * link list. If we succeed in our allocation, then we
1344− * add these pages to the cpu_buffers. Otherwise we just free
1345− * them all and return -ENOMEM;
1346− */
1347− if (RB_WARN_ON(buffer, nr_pages <= buffer->pages))
1348− goto out_fail;
1348+ cpu_buffer->nr_pages_to_update = nr_pages -
1349+ cpu_buffer->nr_pages;
13491350
1350− new_pages = nr_pages - buffer->pages;
1351+ /*
1352+ * nothing more to do for removing pages or no update
1353+ */
1354+ if (cpu_buffer->nr_pages_to_update <= 0)
1355+ continue;
13511356
1352− for_each_buffer_cpu(buffer, cpu) {
1353− for (i = 0; i < new_pages; i++) {
1354− struct page *page;
13551357 /*
1356− * __GFP_NORETRY flag makes sure that the allocation
1357− * fails gracefully without invoking oom-killer and
1358− * the system is not destabilized.
1358+ * to add pages, make sure all new pages can be
1359+ * allocated without receiving ENOMEM
13591360 */
1360− bpage = kzalloc_node(ALIGN(sizeof(*bpage),
1361− cache_line_size()),
1362− GFP_KERNEL | __GFP_NORETRY,
1363− cpu_to_node(cpu));
1364− if (!bpage)
1365− goto free_pages;
1366− list_add(&bpage->list, &pages);
1367− page = alloc_pages_node(cpu_to_node(cpu),
1368− GFP_KERNEL | __GFP_NORETRY, 0);
1369− if (!page)
1370− goto free_pages;
1371− bpage->page = page_address(page);
1372− rb_init_page(bpage->page);
1361+ INIT_LIST_HEAD(&cpu_buffer->new_pages);
1362+ if (__rb_allocate_pages(cpu_buffer->nr_pages_to_update,
1363+ &cpu_buffer->new_pages, cpu))
1364+ /* not enough memory for new pages */
1365+ goto no_mem;
13731366 }
1374− }
13751367
1376− for_each_buffer_cpu(buffer, cpu) {
1377− cpu_buffer = buffer->buffers[cpu];
1378− rb_insert_pages(cpu_buffer, &pages, new_pages);
1379− }
1368+ /* wait for all the updates to complete */
1369+ for_each_buffer_cpu(buffer, cpu) {
1370+ cpu_buffer = buffer->buffers[cpu];
1371+ if (cpu_buffer->nr_pages_to_update) {
1372+ update_pages_handler(cpu_buffer);
1373+ }
1374+ }
1375+ } else {
1376+ cpu_buffer = buffer->buffers[cpu_id];
1377+ if (nr_pages == cpu_buffer->nr_pages)
1378+ goto out;
13801379
1381− if (RB_WARN_ON(buffer, !list_empty(&pages)))
1382− goto out_fail;
1380+ cpu_buffer->nr_pages_to_update = nr_pages -
1381+ cpu_buffer->nr_pages;
1382+
1383+ INIT_LIST_HEAD(&cpu_buffer->new_pages);
1384+ if (cpu_buffer->nr_pages_to_update > 0 &&
1385+ __rb_allocate_pages(cpu_buffer->nr_pages_to_update,
1386+ &cpu_buffer->new_pages, cpu_id))
1387+ goto no_mem;
1388+
1389+ update_pages_handler(cpu_buffer);
1390+ }
13831391
13841392 out:
1385− buffer->pages = nr_pages;
13861393 put_online_cpus();
13871394 mutex_unlock(&buffer->mutex);
13881395
@@ -1390,25 +1397,24 @@ int ring_buffer_resize(struct ring_buffer *buffer, unsigned long size) · 1 unmodified line
13901397
13911398 return size;
13921399
1393− free_pages:
1394− list_for_each_entry_safe(bpage, tmp, &pages, list) {
1395− list_del_init(&bpage->list);
1396− free_buffer_page(bpage);
1400+ no_mem:
1401+ for_each_buffer_cpu(buffer, cpu) {
1402+ struct buffer_page *bpage, *tmp;
1403+ cpu_buffer = buffer->buffers[cpu];
1404+ /* reset this number regardless */
1405+ cpu_buffer->nr_pages_to_update = 0;
1406+ if (list_empty(&cpu_buffer->new_pages))
1407+ continue;
1408+ list_for_each_entry_safe(bpage, tmp, &cpu_buffer->new_pages,
1409+ list) {
1410+ list_del_init(&bpage->list);
1411+ free_buffer_page(bpage);
1412+ }
13971413 }
13981414 put_online_cpus();
13991415 mutex_unlock(&buffer->mutex);
14001416 atomic_dec(&buffer->record_disabled);
14011417 return -ENOMEM;
1402−
1403− /*
1404− * Something went totally wrong, and we are too paranoid
1405− * to even clean up the mess.
1406− */
1407− out_fail:
1408− put_online_cpus();
1409− mutex_unlock(&buffer->mutex);
1410− atomic_dec(&buffer->record_disabled);
1411− return -1;
14121418 }
14131419 EXPORT_SYMBOL_GPL(ring_buffer_resize);
14141420
@@ -1510,7 +1516,7 @@ rb_set_commit_to_write(struct ring_buffer_per_cpu *cpu_buffer) · 95 unmodified lines
15101516 * assign the commit to the tail.
15111517 */
15121518 again:
1513− max_count = cpu_buffer->buffer->pages * 100;
1519+ max_count = cpu_buffer->nr_pages * 100;
15141520
15151521 while (cpu_buffer->commit_page != cpu_buffer->tail_page) {
15161522 if (RB_WARN_ON(cpu_buffer, !(--max_count)))
@@ -3588,9 +3594,18 @@ EXPORT_SYMBOL_GPL(ring_buffer_read); · 2071 unmodified lines
35883594 * ring_buffer_size - return the size of the ring buffer (in bytes)
35893595 * @buffer: The ring buffer.
35903596 */
3591−unsigned long ring_buffer_size(struct ring_buffer *buffer)
3597+unsigned long ring_buffer_size(struct ring_buffer *buffer, int cpu)
35923598 {
3593− return BUF_PAGE_SIZE * buffer->pages;
3599+ /*
3600+ * Earlier, this method returned
3601+ * BUF_PAGE_SIZE * buffer->nr_pages
3602+ * Since the nr_pages field is now removed, we have converted this to
3603+ * return the per cpu buffer value.
3604+ */
3605+ if (!cpumask_test_cpu(cpu, buffer->cpumask))
3606+ return 0;
3607+
3608+ return BUF_PAGE_SIZE * buffer->buffers[cpu]->nr_pages;
35943609 }
35953610 EXPORT_SYMBOL_GPL(ring_buffer_size);
35963611
@@ -3765,8 +3780,11 @@ int ring_buffer_swap_cpu(struct ring_buffer *buffer_a, · 168 unmodified lines
37653780 !cpumask_test_cpu(cpu, buffer_b->cpumask))
37663781 goto out;
37673782
3783+ cpu_buffer_a = buffer_a->buffers[cpu];
3784+ cpu_buffer_b = buffer_b->buffers[cpu];
3785+
37683786 /* At least make sure the two buffers are somewhat the same */
3769− if (buffer_a->pages != buffer_b->pages)
3787+ if (cpu_buffer_a->nr_pages != cpu_buffer_b->nr_pages)
37703788 goto out;
37713789
37723790 ret = -EAGAIN;
@@ -3780,9 +3798,6 @@ int ring_buffer_swap_cpu(struct ring_buffer *buffer_a, · 7 unmodified lines
37803798 if (atomic_read(&buffer_b->record_disabled))
37813799 goto out;
37823800
3783− cpu_buffer_a = buffer_a->buffers[cpu];
3784− cpu_buffer_b = buffer_b->buffers[cpu];
3785−
37863801 if (atomic_read(&cpu_buffer_a->record_disabled))
37873802 goto out;
37883803
@@ -4071,6 +4086,8 @@ static int rb_cpu_notify(struct notifier_block *self, · 282 unmodified lines
40714086 struct ring_buffer *buffer =
40724087 container_of(self, struct ring_buffer, cpu_notify);
40734088 long cpu = (long)hcpu;
4089+ int cpu_i, nr_pages_same;
4090+ unsigned int nr_pages;
40744091
40754092 switch (action) {
40764093 case CPU_UP_PREPARE:
@@ -4078,8 +4095,23 @@ static int rb_cpu_notify(struct notifier_block *self, · 1 unmodified line
40784095 if (cpumask_test_cpu(cpu, buffer->cpumask))
40794096 return NOTIFY_OK;
40804097
4098+ nr_pages = 0;
4099+ nr_pages_same = 1;
4100+ /* check if all cpu sizes are same */
4101+ for_each_buffer_cpu(buffer, cpu_i) {
4102+ /* fill in the size from first enabled cpu */
4103+ if (nr_pages == 0)
4104+ nr_pages = buffer->buffers[cpu_i]->nr_pages;
4105+ if (nr_pages != buffer->buffers[cpu_i]->nr_pages) {
4106+ nr_pages_same = 0;
4107+ break;
4108+ }
4109+ }
4110+ /* allocate minimum pages, user can later expand it */
4111+ if (!nr_pages_same)
4112+ nr_pages = 2;
40814113 buffer->buffers[cpu] =
4082− rb_allocate_cpu_buffer(buffer, cpu);
4114+ rb_allocate_cpu_buffer(buffer, nr_pages, cpu);
40834115 if (!buffer->buffers[cpu]) {
40844116 WARN(1, "failed to allocate ring buffer on CPU %ld\n",
40854117 cpu);
kernel/trace/trace.c+288−91
@@ -629,7 +629,6 @@ ssize_t trace_seq_to_user(struct trace_seq *s, char __user *ubuf, size_t cnt) · 628 unmodified lines
629629 static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt)
630630 {
631631 int len;
632− void *ret;
633632
634633 if (s->len <= s->readpos)
635634 return -EBUSY;
@@ -637,9 +636,7 @@ static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt) · 1 unmodified line
637636 len = s->len - s->readpos;
638637 if (cnt > len)
639638 cnt = len;
640− ret = memcpy(buf, s->buffer + s->readpos, cnt);
641− if (!ret)
642− return -EFAULT;
639+ memcpy(buf, s->buffer + s->readpos, cnt);
643640
644641 s->readpos += cnt;
645642 return cnt;
@@ -841,7 +838,8 @@ __acquires(kernel_lock) · 195 unmodified lines
841838
842839 /* If we expanded the buffers, make sure the max is expanded too */
843840 if (ring_buffer_expanded && type->use_max_tr)
844− ring_buffer_resize(max_tr.buffer, trace_buf_size);
841+ ring_buffer_resize(max_tr.buffer, trace_buf_size,
842+ RING_BUFFER_ALL_CPUS);
845843
846844 /* the test is responsible for initializing and enabling */
847845 pr_info("Testing tracer %s: ", type->name);
@@ -857,7 +855,8 @@ __acquires(kernel_lock) · 9 unmodified lines
857855
858856 /* Shrink the max buffer again */
859857 if (ring_buffer_expanded && type->use_max_tr)
860− ring_buffer_resize(max_tr.buffer, 1);
858+ ring_buffer_resize(max_tr.buffer, 1,
859+ RING_BUFFER_ALL_CPUS);
861860
862861 printk(KERN_CONT "PASSED\n");
863862 }
@@ -1498,25 +1497,119 @@ static void __trace_userstack(struct trace_array *tr, unsigned long flags) · 634 unmodified lines
14981497
14991498 #endif /* CONFIG_STACKTRACE */
15001499
1500+/* created for use with alloc_percpu */
1501+struct trace_buffer_struct {
1502+ char buffer[TRACE_BUF_SIZE];
1503+};
1504+
1505+static struct trace_buffer_struct *trace_percpu_buffer;
1506+static struct trace_buffer_struct *trace_percpu_sirq_buffer;
1507+static struct trace_buffer_struct *trace_percpu_irq_buffer;
1508+static struct trace_buffer_struct *trace_percpu_nmi_buffer;
1509+
1510+/*
1511+ * The buffer used is dependent on the context. There is a per cpu
1512+ * buffer for normal context, softirq contex, hard irq context and
1513+ * for NMI context. Thise allows for lockless recording.
1514+ *
1515+ * Note, if the buffers failed to be allocated, then this returns NULL
1516+ */
1517+static char *get_trace_buf(void)
1518+{
1519+ struct trace_buffer_struct *percpu_buffer;
1520+ struct trace_buffer_struct *buffer;
1521+
1522+ /*
1523+ * If we have allocated per cpu buffers, then we do not
1524+ * need to do any locking.
1525+ */
1526+ if (in_nmi())
1527+ percpu_buffer = trace_percpu_nmi_buffer;
1528+ else if (in_irq())
1529+ percpu_buffer = trace_percpu_irq_buffer;
1530+ else if (in_softirq())
1531+ percpu_buffer = trace_percpu_sirq_buffer;
1532+ else
1533+ percpu_buffer = trace_percpu_buffer;
1534+
1535+ if (!percpu_buffer)
1536+ return NULL;
1537+
1538+ buffer = per_cpu_ptr(percpu_buffer, smp_processor_id());
1539+
1540+ return buffer->buffer;
1541+}
1542+
1543+static int alloc_percpu_trace_buffer(void)
1544+{
1545+ struct trace_buffer_struct *buffers;
1546+ struct trace_buffer_struct *sirq_buffers;
1547+ struct trace_buffer_struct *irq_buffers;
1548+ struct trace_buffer_struct *nmi_buffers;
1549+
1550+ buffers = alloc_percpu(struct trace_buffer_struct);
1551+ if (!buffers)
1552+ goto err_warn;
1553+
1554+ sirq_buffers = alloc_percpu(struct trace_buffer_struct);
1555+ if (!sirq_buffers)
1556+ goto err_sirq;
1557+
1558+ irq_buffers = alloc_percpu(struct trace_buffer_struct);
1559+ if (!irq_buffers)
1560+ goto err_irq;
1561+
1562+ nmi_buffers = alloc_percpu(struct trace_buffer_struct);
1563+ if (!nmi_buffers)
1564+ goto err_nmi;
1565+
1566+ trace_percpu_buffer = buffers;
1567+ trace_percpu_sirq_buffer = sirq_buffers;
1568+ trace_percpu_irq_buffer = irq_buffers;
1569+ trace_percpu_nmi_buffer = nmi_buffers;
1570+
1571+ return 0;
1572+
1573+ err_nmi:
1574+ free_percpu(irq_buffers);
1575+ err_irq:
1576+ free_percpu(sirq_buffers);
1577+ err_sirq:
1578+ free_percpu(buffers);
1579+ err_warn:
1580+ WARN(1, "Could not allocate percpu trace_printk buffer");
1581+ return -ENOMEM;
1582+}
1583+
1584+void trace_printk_init_buffers(void)
1585+{
1586+ static int buffers_allocated;
1587+
1588+ if (buffers_allocated)
1589+ return;
1590+
1591+ if (alloc_percpu_trace_buffer())
1592+ return;
1593+
1594+ pr_info("ftrace: Allocated trace_printk buffers\n");
1595+
1596+ buffers_allocated = 1;
1597+}
1598+
15011599 /**
15021600 * trace_vbprintk - write binary msg to tracing buffer
15031601 *
15041602 */
15051603 int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
15061604 {
1507− static arch_spinlock_t trace_buf_lock =
1508− (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
1509− static u32 trace_buf[TRACE_BUF_SIZE];
1510−
15111605 struct ftrace_event_call *call = &event_bprint;
15121606 struct ring_buffer_event *event;
15131607 struct ring_buffer *buffer;
15141608 struct trace_array *tr = &global_trace;
1515− struct trace_array_cpu *data;
15161609 struct bprint_entry *entry;
15171610 unsigned long flags;
1518− int disable;
1519− int cpu, len = 0, size, pc;
1611+ char *tbuffer;
1612+ int len = 0, size, pc;
15201613
15211614 if (unlikely(tracing_selftest_running || tracing_disabled))
15221615 return 0;
@@ -1526,43 +1619,36 @@ int trace_vbprintk(unsigned long ip, const char *fmt, va_list args) · 3 unmodified lines
15261619
15271620 pc = preempt_count();
15281621 preempt_disable_notrace();
1529− cpu = raw_smp_processor_id();
1530− data = tr->data[cpu];
15311622
1532− disable = atomic_inc_return(&data->disabled);
1533− if (unlikely(disable != 1))
1623+ tbuffer = get_trace_buf();
1624+ if (!tbuffer) {
1625+ len = 0;
15341626 goto out;
1627+ }
15351628
1536− /* Lockdep uses trace_printk for lock tracing */
1537− local_irq_save(flags);
1538− arch_spin_lock(&trace_buf_lock);
1539− len = vbin_printf(trace_buf, TRACE_BUF_SIZE, fmt, args);
1629+ len = vbin_printf((u32 *)tbuffer, TRACE_BUF_SIZE/sizeof(int), fmt, args);
15401630
1541− if (len > TRACE_BUF_SIZE || len < 0)
1542− goto out_unlock;
1631+ if (len > TRACE_BUF_SIZE/sizeof(int) || len < 0)
1632+ goto out;
15431633
1634+ local_save_flags(flags);
15441635 size = sizeof(*entry) + sizeof(u32) * len;
15451636 buffer = tr->buffer;
15461637 event = trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
15471638 flags, pc);
15481639 if (!event)
1549− goto out_unlock;
1640+ goto out;
15501641 entry = ring_buffer_event_data(event);
15511642 entry->ip = ip;
15521643 entry->fmt = fmt;
15531644
1554− memcpy(entry->buf, trace_buf, sizeof(u32) * len);
1645+ memcpy(entry->buf, tbuffer, sizeof(u32) * len);
15551646 if (!filter_check_discard(call, entry, buffer, event)) {
15561647 ring_buffer_unlock_commit(buffer, event);
15571648 ftrace_trace_stack(buffer, flags, 6, pc);
15581649 }
15591650
1560−out_unlock:
1561− arch_spin_unlock(&trace_buf_lock);
1562− local_irq_restore(flags);
1563−
15641651 out:
1565− atomic_dec_return(&data->disabled);
15661652 preempt_enable_notrace();
15671653 unpause_graph_tracing();
15681654
@@ -1588,58 +1674,53 @@ int trace_array_printk(struct trace_array *tr, · 19 unmodified lines
15881674 int trace_array_vprintk(struct trace_array *tr,
15891675 unsigned long ip, const char *fmt, va_list args)
15901676 {
1591− static arch_spinlock_t trace_buf_lock = __ARCH_SPIN_LOCK_UNLOCKED;
1592− static char trace_buf[TRACE_BUF_SIZE];
1593−
15941677 struct ftrace_event_call *call = &event_print;
15951678 struct ring_buffer_event *event;
15961679 struct ring_buffer *buffer;
1597− struct trace_array_cpu *data;
1598− int cpu, len = 0, size, pc;
1680+ int len = 0, size, pc;
15991681 struct print_entry *entry;
1600− unsigned long irq_flags;
1601− int disable;
1682+ unsigned long flags;
1683+ char *tbuffer;
16021684
16031685 if (tracing_disabled || tracing_selftest_running)
16041686 return 0;
16051687
1688+ /* Don't pollute graph traces with trace_vprintk internals */
1689+ pause_graph_tracing();
1690+
16061691 pc = preempt_count();
16071692 preempt_disable_notrace();
1608− cpu = raw_smp_processor_id();
1609− data = tr->data[cpu];
16101693
1611− disable = atomic_inc_return(&data->disabled);
1612− if (unlikely(disable != 1))
1694+
1695+ tbuffer = get_trace_buf();
1696+ if (!tbuffer) {
1697+ len = 0;
16131698 goto out;
1699+ }
16141700
1615− pause_graph_tracing();
1616− raw_local_irq_save(irq_flags);
1617− arch_spin_lock(&trace_buf_lock);
1618− len = vsnprintf(trace_buf, TRACE_BUF_SIZE, fmt, args);
1701+ len = vsnprintf(tbuffer, TRACE_BUF_SIZE, fmt, args);
1702+ if (len > TRACE_BUF_SIZE)
1703+ goto out;
16191704
1705+ local_save_flags(flags);
16201706 size = sizeof(*entry) + len + 1;
16211707 buffer = tr->buffer;
16221708 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
1623− irq_flags, pc);
1709+ flags, pc);
16241710 if (!event)
1625− goto out_unlock;
1711+ goto out;
16261712 entry = ring_buffer_event_data(event);
16271713 entry->ip = ip;
16281714
1629− memcpy(&entry->buf, trace_buf, len);
1715+ memcpy(&entry->buf, tbuffer, len);
16301716 entry->buf[len] = '\0';
16311717 if (!filter_check_discard(call, entry, buffer, event)) {
16321718 ring_buffer_unlock_commit(buffer, event);
1633− ftrace_trace_stack(buffer, irq_flags, 6, pc);
1719+ ftrace_trace_stack(buffer, flags, 6, pc);
16341720 }
1635−
1636− out_unlock:
1637− arch_spin_unlock(&trace_buf_lock);
1638− raw_local_irq_restore(irq_flags);
1639− unpause_graph_tracing();
16401721 out:
1641− atomic_dec_return(&data->disabled);
16421722 preempt_enable_notrace();
1723+ unpause_graph_tracing();
16431724
16441725 return len;
16451726 }
@@ -2974,7 +3055,14 @@ int tracer_init(struct tracer *t, struct trace_array *tr) · 1328 unmodified lines
29743055 return t->init(tr);
29753056 }
29763057
2977−static int __tracing_resize_ring_buffer(unsigned long size)
3058+static void set_buffer_entries(struct trace_array *tr, unsigned long val)
3059+{
3060+ int cpu;
3061+ for_each_tracing_cpu(cpu)
3062+ tr->data[cpu]->entries = val;
3063+}
3064+
3065+static int __tracing_resize_ring_buffer(unsigned long size, int cpu)
29783066 {
29793067 int ret;
29803068
@@ -2985,19 +3073,32 @@ static int __tracing_resize_ring_buffer(unsigned long size) · 4 unmodified lines
29853073 */
29863074 ring_buffer_expanded = 1;
29873075
2988− ret = ring_buffer_resize(global_trace.buffer, size);
3076+ ret = ring_buffer_resize(global_trace.buffer, size, cpu);
29893077 if (ret < 0)
29903078 return ret;
29913079
29923080 if (!current_trace->use_max_tr)
29933081 goto out;
29943082
2995− ret = ring_buffer_resize(max_tr.buffer, size);
3083+ ret = ring_buffer_resize(max_tr.buffer, size, cpu);
29963084 if (ret < 0) {
2997− int r;
3085+ int r = 0;
3086+
3087+ if (cpu == RING_BUFFER_ALL_CPUS) {
3088+ int i;
3089+ for_each_tracing_cpu(i) {
3090+ r = ring_buffer_resize(global_trace.buffer,
3091+ global_trace.data[i]->entries,
3092+ i);
3093+ if (r < 0)
3094+ break;
3095+ }
3096+ } else {
3097+ r = ring_buffer_resize(global_trace.buffer,
3098+ global_trace.data[cpu]->entries,
3099+ cpu);
3100+ }
29983101
2999− r = ring_buffer_resize(global_trace.buffer,
3000− global_trace.entries);
30013102 if (r < 0) {
30023103 /*
30033104 * AARGH! We are left with different
@@ -3019,14 +3120,21 @@ static int __tracing_resize_ring_buffer(unsigned long size) · 15 unmodified lines
30193120 return ret;
30203121 }
30213122
3022− max_tr.entries = size;
3123+ if (cpu == RING_BUFFER_ALL_CPUS)
3124+ set_buffer_entries(&max_tr, size);
3125+ else
3126+ max_tr.data[cpu]->entries = size;
3127+
30233128 out:
3024− global_trace.entries = size;
3129+ if (cpu == RING_BUFFER_ALL_CPUS)
3130+ set_buffer_entries(&global_trace, size);
3131+ else
3132+ global_trace.data[cpu]->entries = size;
30253133
30263134 return ret;
30273135 }
30283136
3029−static ssize_t tracing_resize_ring_buffer(unsigned long size)
3137+static ssize_t tracing_resize_ring_buffer(unsigned long size, int cpu_id)
30303138 {
30313139 int cpu, ret = size;
30323140
@@ -3042,12 +3150,19 @@ static ssize_t tracing_resize_ring_buffer(unsigned long size) · 9 unmodified lines
30423150 atomic_inc(&max_tr.data[cpu]->disabled);
30433151 }
30443152
3045− if (size != global_trace.entries)
3046− ret = __tracing_resize_ring_buffer(size);
3153+ if (cpu_id != RING_BUFFER_ALL_CPUS) {
3154+ /* make sure, this cpu is enabled in the mask */
3155+ if (!cpumask_test_cpu(cpu_id, tracing_buffer_mask)) {
3156+ ret = -EINVAL;
3157+ goto out;
3158+ }
3159+ }
30473160
3161+ ret = __tracing_resize_ring_buffer(size, cpu_id);
30483162 if (ret < 0)
30493163 ret = -ENOMEM;
30503164
3165+out:
30513166 for_each_tracing_cpu(cpu) {
30523167 if (global_trace.data[cpu])
30533168 atomic_dec(&global_trace.data[cpu]->disabled);
@@ -3078,7 +3193,8 @@ int tracing_update_buffers(void) · 24 unmodified lines
30783193
30793194 mutex_lock(&trace_types_lock);
30803195 if (!ring_buffer_expanded)
3081− ret = __tracing_resize_ring_buffer(trace_buf_size);
3196+ ret = __tracing_resize_ring_buffer(trace_buf_size,
3197+ RING_BUFFER_ALL_CPUS);
30823198 mutex_unlock(&trace_types_lock);
30833199
30843200 return ret;
@@ -3102,7 +3218,8 @@ static int tracing_set_tracer(const char *buf) · 17 unmodified lines
31023218 mutex_lock(&trace_types_lock);
31033219
31043220 if (!ring_buffer_expanded) {
3105− ret = __tracing_resize_ring_buffer(trace_buf_size);
3221+ ret = __tracing_resize_ring_buffer(trace_buf_size,
3222+ RING_BUFFER_ALL_CPUS);
31063223 if (ret < 0)
31073224 goto out;
31083225 ret = 0;
@@ -3128,8 +3245,8 @@ static int tracing_set_tracer(const char *buf) · 19 unmodified lines
31283245 * The max_tr ring buffer has some state (e.g. ring->clock) and
31293246 * we want preserve it.
31303247 */
3131− ring_buffer_resize(max_tr.buffer, 1);
3132− max_tr.entries = 1;
3248+ ring_buffer_resize(max_tr.buffer, 1, RING_BUFFER_ALL_CPUS);
3249+ set_buffer_entries(&max_tr, 1);
31333250 }
31343251 destroy_trace_option_files(topts);
31353252
@@ -3137,10 +3254,17 @@ static int tracing_set_tracer(const char *buf) · 1 unmodified line
31373254
31383255 topts = create_trace_option_files(current_trace);
31393256 if (current_trace->use_max_tr) {
3140− ret = ring_buffer_resize(max_tr.buffer, global_trace.entries);
3141− if (ret < 0)
3142− goto out;
3143− max_tr.entries = global_trace.entries;
3257+ int cpu;
3258+ /* we need to make per cpu buffer sizes equivalent */
3259+ for_each_tracing_cpu(cpu) {
3260+ ret = ring_buffer_resize(max_tr.buffer,
3261+ global_trace.data[cpu]->entries,
3262+ cpu);
3263+ if (ret < 0)
3264+ goto out;
3265+ max_tr.data[cpu]->entries =
3266+ global_trace.data[cpu]->entries;
3267+ }
31443268 }
31453269
31463270 if (t->init) {
@@ -3642,30 +3766,82 @@ out_err: · 495 unmodified lines
36423766 goto out;
36433767 }
36443768
3769+struct ftrace_entries_info {
3770+ struct trace_array *tr;
3771+ int cpu;
3772+};
3773+
3774+static int tracing_entries_open(struct inode *inode, struct file *filp)
3775+{
3776+ struct ftrace_entries_info *info;
3777+
3778+ if (tracing_disabled)
3779+ return -ENODEV;
3780+
3781+ info = kzalloc(sizeof(*info), GFP_KERNEL);
3782+ if (!info)
3783+ return -ENOMEM;
3784+
3785+ info->tr = &global_trace;
3786+ info->cpu = (unsigned long)inode->i_private;
3787+
3788+ filp->private_data = info;
3789+
3790+ return 0;
3791+}
3792+
36453793 static ssize_t
36463794 tracing_entries_read(struct file *filp, char __user *ubuf,
36473795 size_t cnt, loff_t *ppos)
36483796 {
3649− struct trace_array *tr = filp->private_data;
3650− char buf[96];
3651− int r;
3797+ struct ftrace_entries_info *info = filp->private_data;
3798+ struct trace_array *tr = info->tr;
3799+ char buf[64];
3800+ int r = 0;
3801+ ssize_t ret;
36523802
36533803 mutex_lock(&trace_types_lock);
3654− if (!ring_buffer_expanded)
3655− r = sprintf(buf, "%lu (expanded: %lu)\n",
3656− tr->entries >> 10,
3657− trace_buf_size >> 10);
3658− else
3659− r = sprintf(buf, "%lu\n", tr->entries >> 10);
3804+
3805+ if (info->cpu == RING_BUFFER_ALL_CPUS) {
3806+ int cpu, buf_size_same;
3807+ unsigned long size;
3808+
3809+ size = 0;
3810+ buf_size_same = 1;
3811+ /* check if all cpu sizes are same */
3812+ for_each_tracing_cpu(cpu) {
3813+ /* fill in the size from first enabled cpu */
3814+ if (size == 0)
3815+ size = tr->data[cpu]->entries;
3816+ if (size != tr->data[cpu]->entries) {
3817+ buf_size_same = 0;
3818+ break;
3819+ }
3820+ }
3821+
3822+ if (buf_size_same) {
3823+ if (!ring_buffer_expanded)
3824+ r = sprintf(buf, "%lu (expanded: %lu)\n",
3825+ size >> 10,
3826+ trace_buf_size >> 10);
3827+ else
3828+ r = sprintf(buf, "%lu\n", size >> 10);
3829+ } else
3830+ r = sprintf(buf, "X\n");
3831+ } else
3832+ r = sprintf(buf, "%lu\n", tr->data[info->cpu]->entries >> 10);
3833+
36603834 mutex_unlock(&trace_types_lock);
36613835
3662− return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3836+ ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3837+ return ret;
36633838 }
36643839
36653840 static ssize_t
36663841 tracing_entries_write(struct file *filp, const char __user *ubuf,
36673842 size_t cnt, loff_t *ppos)
36683843 {
3844+ struct ftrace_entries_info *info = filp->private_data;
36693845 unsigned long val;
36703846 int ret;
36713847
@@ -3680,7 +3856,7 @@ tracing_entries_write(struct file *filp, const char __user *ubuf, · 8 unmodified lines
36803856 /* value is in KB */
36813857 val <<= 10;
36823858
3683− ret = tracing_resize_ring_buffer(val);
3859+ ret = tracing_resize_ring_buffer(val, info->cpu);
36843860 if (ret < 0)
36853861 return ret;
36863862
@@ -3689,6 +3865,16 @@ tracing_entries_write(struct file *filp, const char __user *ubuf, · 2 unmodified lines
36893865 return cnt;
36903866 }
36913867
3868+static int
3869+tracing_entries_release(struct inode *inode, struct file *filp)
3870+{
3871+ struct ftrace_entries_info *info = filp->private_data;
3872+
3873+ kfree(info);
3874+
3875+ return 0;
3876+}
3877+
36923878 static ssize_t
36933879 tracing_total_entries_read(struct file *filp, char __user *ubuf,
36943880 size_t cnt, loff_t *ppos)
@@ -3700,7 +3886,7 @@ tracing_total_entries_read(struct file *filp, char __user *ubuf, · 5 unmodified lines
37003886
37013887 mutex_lock(&trace_types_lock);
37023888 for_each_tracing_cpu(cpu) {
3703− size += tr->entries >> 10;
3889+ size += tr->data[cpu]->entries >> 10;
37043890 if (!ring_buffer_expanded)
37053891 expanded_size += trace_buf_size >> 10;
37063892 }
@@ -3734,7 +3920,7 @@ tracing_free_buffer_release(struct inode *inode, struct file *filp) · 27 unmodified lines
37343920 if (trace_flags & TRACE_ITER_STOP_ON_FREE)
37353921 tracing_off();
37363922 /* resize the ring buffer to 0 */
3737− tracing_resize_ring_buffer(0);
3923+ tracing_resize_ring_buffer(0, RING_BUFFER_ALL_CPUS);
37383924
37393925 return 0;
37403926 }
@@ -3933,9 +4119,10 @@ static const struct file_operations tracing_pipe_fops = { · 192 unmodified lines
39334119 };
39344120
39354121 static const struct file_operations tracing_entries_fops = {
3936− .open = tracing_open_generic,
4122+ .open = tracing_entries_open,
39374123 .read = tracing_entries_read,
39384124 .write = tracing_entries_write,
4125+ .release = tracing_entries_release,
39394126 .llseek = generic_file_llseek,
39404127 };
39414128
@@ -4387,6 +4574,9 @@ static void tracing_init_debugfs_percpu(long cpu) · 445 unmodified lines
43874574
43884575 trace_create_file("stats", 0444, d_cpu,
43894576 (void *) cpu, &tracing_stats_fops);
4577+
4578+ trace_create_file("buffer_size_kb", 0444, d_cpu,
4579+ (void *) cpu, &tracing_entries_fops);
43904580 }
43914581
43924582 #ifdef CONFIG_FTRACE_SELFTEST
@@ -4716,7 +4906,7 @@ static __init int tracer_init_debugfs(void) · 323 unmodified lines
47164906 (void *) TRACE_PIPE_ALL_CPU, &tracing_pipe_fops);
47174907
47184908 trace_create_file("buffer_size_kb", 0644, d_tracer,
4719− &global_trace, &tracing_entries_fops);
4909+ (void *) RING_BUFFER_ALL_CPUS, &tracing_entries_fops);
47204910
47214911 trace_create_file("buffer_total_size_kb", 0444, d_tracer,
47224912 &global_trace, &tracing_total_entries_fops);
@@ -4955,6 +5145,10 @@ __init static int tracer_alloc_buffers(void) · 232 unmodified lines
49555145 if (!alloc_cpumask_var(&tracing_cpumask, GFP_KERNEL))
49565146 goto out_free_buffer_mask;
49575147
5148+ /* Only allocate trace_printk buffers if a trace_printk exists */
5149+ if (__stop___trace_bprintk_fmt != __start___trace_bprintk_fmt)
5150+ trace_printk_init_buffers();
5151+
49585152 /* To save memory, keep the ring buffer size to its minimum */
49595153 if (ring_buffer_expanded)
49605154 ring_buf_size = trace_buf_size;
@@ -4973,7 +5167,6 @@ __init static int tracer_alloc_buffers(void) · 12 unmodified lines
49735167 WARN_ON(1);
49745168 goto out_free_cpumask;
49755169 }
4976− global_trace.entries = ring_buffer_size(global_trace.buffer);
49775170 if (global_trace.buffer_disabled)
49785171 tracing_off();
49795172
@@ -4986,7 +5179,6 @@ __init static int tracer_alloc_buffers(void) · 6 unmodified lines
49865179 ring_buffer_free(global_trace.buffer);
49875180 goto out_free_cpumask;
49885181 }
4989− max_tr.entries = 1;
49905182 #endif
49915183
49925184 /* Allocate the first page for all buffers */
@@ -4995,6 +5187,11 @@ __init static int tracer_alloc_buffers(void) · 2 unmodified lines
49955187 max_tr.data[i] = &per_cpu(max_tr_data, i);
49965188 }
49975189
5190+ set_buffer_entries(&global_trace, ring_buf_size);
5191+#ifdef CONFIG_TRACER_MAX_TRACE
5192+ set_buffer_entries(&max_tr, 1);
5193+#endif
5194+
49985195 trace_init_cmdlines();
49995196
50005197 register_tracer(&nop_trace);
kernel/trace/trace.h+3−1
@@ -131,6 +131,7 @@ struct trace_array_cpu { · 130 unmodified lines
131131 atomic_t disabled;
132132 void *buffer_page; /* ring buffer spare */
133133
134+ unsigned long entries;
134135 unsigned long saved_latency;
135136 unsigned long critical_start;
136137 unsigned long critical_end;
@@ -152,7 +153,6 @@ struct trace_array_cpu { · 15 unmodified lines
152153 */
153154 struct trace_array {
154155 struct ring_buffer *buffer;
155− unsigned long entries;
156156 int cpu;
157157 int buffer_disabled;
158158 cycle_t time_start;
@@ -826,6 +826,8 @@ extern struct list_head ftrace_events; · 667 unmodified lines
826826 extern const char *__start___trace_bprintk_fmt[];
827827 extern const char *__stop___trace_bprintk_fmt[];
828828
829+void trace_printk_init_buffers(void);
830+
829831 #undef FTRACE_ENTRY
830832 #define FTRACE_ENTRY(call, struct_name, id, tstruct, print, filter) \
831833 extern struct ftrace_event_call \
kernel/trace/trace_printk.c+4−0
@@ -51,6 +51,10 @@ void hold_module_trace_bprintk_format(const char **start, const char **end) · 50 unmodified lines
5151 const char **iter;
5252 char *fmt;
5353
54+ /* allocate the trace_printk per cpu buffers */
55+ if (start != end)
56+ trace_printk_init_buffers();
57+
5458 mutex_lock(&btrace_mutex);
5559 for (iter = start; iter < end; iter++) {
5660 struct trace_bprintk_fmt *tb_fmt = lookup_format(*iter);