| @@ -629,7 +629,6 @@ ssize_t trace_seq_to_user(struct trace_seq *s, char __user *ubuf, size_t cnt) · 628 unmodified lines |
| 629 | static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt) | 629 | static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt) |
| 630 | { | 630 | { |
| 631 | int len; | 631 | int len; |
| 632 | void *ret; | | |
| 633 | | 632 | |
| 634 | if (s->len <= s->readpos) | 633 | if (s->len <= s->readpos) |
| 635 | return -EBUSY; | 634 | return -EBUSY; |
| @@ -637,9 +636,7 @@ static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt) · 1 unmodified line |
| 637 | len = s->len - s->readpos; | 636 | len = s->len - s->readpos; |
| 638 | if (cnt > len) | 637 | if (cnt > len) |
| 639 | cnt = len; | 638 | cnt = len; |
| 640 | ret = memcpy(buf, s->buffer + s->readpos, cnt); | 639 | memcpy(buf, s->buffer + s->readpos, cnt); |
| 641 | if (!ret) | | |
| 642 | return -EFAULT; | | |
| 643 | | 640 | |
| 644 | s->readpos += cnt; | 641 | s->readpos += cnt; |
| 645 | return cnt; | 642 | return cnt; |
| @@ -841,7 +838,8 @@ __acquires(kernel_lock) · 195 unmodified lines |
| 841 | | 838 | |
| 842 | /* If we expanded the buffers, make sure the max is expanded too */ | 839 | /* If we expanded the buffers, make sure the max is expanded too */ |
| 843 | if (ring_buffer_expanded && type->use_max_tr) | 840 | if (ring_buffer_expanded && type->use_max_tr) |
| 844 | ring_buffer_resize(max_tr.buffer, trace_buf_size); | 841 | ring_buffer_resize(max_tr.buffer, trace_buf_size, |
| | 842 | RING_BUFFER_ALL_CPUS); |
| 845 | | 843 | |
| 846 | /* the test is responsible for initializing and enabling */ | 844 | /* the test is responsible for initializing and enabling */ |
| 847 | pr_info("Testing tracer %s: ", type->name); | 845 | pr_info("Testing tracer %s: ", type->name); |
| @@ -857,7 +855,8 @@ __acquires(kernel_lock) · 9 unmodified lines |
| 857 | | 855 | |
| 858 | /* Shrink the max buffer again */ | 856 | /* Shrink the max buffer again */ |
| 859 | if (ring_buffer_expanded && type->use_max_tr) | 857 | if (ring_buffer_expanded && type->use_max_tr) |
| 860 | ring_buffer_resize(max_tr.buffer, 1); | 858 | ring_buffer_resize(max_tr.buffer, 1, |
| | 859 | RING_BUFFER_ALL_CPUS); |
| 861 | | 860 | |
| 862 | printk(KERN_CONT "PASSED\n"); | 861 | printk(KERN_CONT "PASSED\n"); |
| 863 | } | 862 | } |
| @@ -1498,25 +1497,119 @@ static void __trace_userstack(struct trace_array *tr, unsigned long flags) · 634 unmodified lines |
| 1498 | | 1497 | |
| 1499 | #endif /* CONFIG_STACKTRACE */ | 1498 | #endif /* CONFIG_STACKTRACE */ |
| 1500 | | 1499 | |
| | 1500 | /* created for use with alloc_percpu */ |
| | 1501 | struct trace_buffer_struct { |
| | 1502 | char buffer[TRACE_BUF_SIZE]; |
| | 1503 | }; |
| | 1504 | |
| | 1505 | static struct trace_buffer_struct *trace_percpu_buffer; |
| | 1506 | static struct trace_buffer_struct *trace_percpu_sirq_buffer; |
| | 1507 | static struct trace_buffer_struct *trace_percpu_irq_buffer; |
| | 1508 | static struct trace_buffer_struct *trace_percpu_nmi_buffer; |
| | 1509 | |
| | 1510 | /* |
| | 1511 | * The buffer used is dependent on the context. There is a per cpu |
| | 1512 | * buffer for normal context, softirq contex, hard irq context and |
| | 1513 | * for NMI context. Thise allows for lockless recording. |
| | 1514 | * |
| | 1515 | * Note, if the buffers failed to be allocated, then this returns NULL |
| | 1516 | */ |
| | 1517 | static char *get_trace_buf(void) |
| | 1518 | { |
| | 1519 | struct trace_buffer_struct *percpu_buffer; |
| | 1520 | struct trace_buffer_struct *buffer; |
| | 1521 | |
| | 1522 | /* |
| | 1523 | * If we have allocated per cpu buffers, then we do not |
| | 1524 | * need to do any locking. |
| | 1525 | */ |
| | 1526 | if (in_nmi()) |
| | 1527 | percpu_buffer = trace_percpu_nmi_buffer; |
| | 1528 | else if (in_irq()) |
| | 1529 | percpu_buffer = trace_percpu_irq_buffer; |
| | 1530 | else if (in_softirq()) |
| | 1531 | percpu_buffer = trace_percpu_sirq_buffer; |
| | 1532 | else |
| | 1533 | percpu_buffer = trace_percpu_buffer; |
| | 1534 | |
| | 1535 | if (!percpu_buffer) |
| | 1536 | return NULL; |
| | 1537 | |
| | 1538 | buffer = per_cpu_ptr(percpu_buffer, smp_processor_id()); |
| | 1539 | |
| | 1540 | return buffer->buffer; |
| | 1541 | } |
| | 1542 | |
| | 1543 | static int alloc_percpu_trace_buffer(void) |
| | 1544 | { |
| | 1545 | struct trace_buffer_struct *buffers; |
| | 1546 | struct trace_buffer_struct *sirq_buffers; |
| | 1547 | struct trace_buffer_struct *irq_buffers; |
| | 1548 | struct trace_buffer_struct *nmi_buffers; |
| | 1549 | |
| | 1550 | buffers = alloc_percpu(struct trace_buffer_struct); |
| | 1551 | if (!buffers) |
| | 1552 | goto err_warn; |
| | 1553 | |
| | 1554 | sirq_buffers = alloc_percpu(struct trace_buffer_struct); |
| | 1555 | if (!sirq_buffers) |
| | 1556 | goto err_sirq; |
| | 1557 | |
| | 1558 | irq_buffers = alloc_percpu(struct trace_buffer_struct); |
| | 1559 | if (!irq_buffers) |
| | 1560 | goto err_irq; |
| | 1561 | |
| | 1562 | nmi_buffers = alloc_percpu(struct trace_buffer_struct); |
| | 1563 | if (!nmi_buffers) |
| | 1564 | goto err_nmi; |
| | 1565 | |
| | 1566 | trace_percpu_buffer = buffers; |
| | 1567 | trace_percpu_sirq_buffer = sirq_buffers; |
| | 1568 | trace_percpu_irq_buffer = irq_buffers; |
| | 1569 | trace_percpu_nmi_buffer = nmi_buffers; |
| | 1570 | |
| | 1571 | return 0; |
| | 1572 | |
| | 1573 | err_nmi: |
| | 1574 | free_percpu(irq_buffers); |
| | 1575 | err_irq: |
| | 1576 | free_percpu(sirq_buffers); |
| | 1577 | err_sirq: |
| | 1578 | free_percpu(buffers); |
| | 1579 | err_warn: |
| | 1580 | WARN(1, "Could not allocate percpu trace_printk buffer"); |
| | 1581 | return -ENOMEM; |
| | 1582 | } |
| | 1583 | |
| | 1584 | void trace_printk_init_buffers(void) |
| | 1585 | { |
| | 1586 | static int buffers_allocated; |
| | 1587 | |
| | 1588 | if (buffers_allocated) |
| | 1589 | return; |
| | 1590 | |
| | 1591 | if (alloc_percpu_trace_buffer()) |
| | 1592 | return; |
| | 1593 | |
| | 1594 | pr_info("ftrace: Allocated trace_printk buffers\n"); |
| | 1595 | |
| | 1596 | buffers_allocated = 1; |
| | 1597 | } |
| | 1598 | |
| 1501 | /** | 1599 | /** |
| 1502 | * trace_vbprintk - write binary msg to tracing buffer | 1600 | * trace_vbprintk - write binary msg to tracing buffer |
| 1503 | * | 1601 | * |
| 1504 | */ | 1602 | */ |
| 1505 | int trace_vbprintk(unsigned long ip, const char *fmt, va_list args) | 1603 | int trace_vbprintk(unsigned long ip, const char *fmt, va_list args) |
| 1506 | { | 1604 | { |
| 1507 | static arch_spinlock_t trace_buf_lock = | | |
| 1508 | (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED; | | |
| 1509 | static u32 trace_buf[TRACE_BUF_SIZE]; | | |
| 1510 | | | |
| 1511 | struct ftrace_event_call *call = &event_bprint; | 1605 | struct ftrace_event_call *call = &event_bprint; |
| 1512 | struct ring_buffer_event *event; | 1606 | struct ring_buffer_event *event; |
| 1513 | struct ring_buffer *buffer; | 1607 | struct ring_buffer *buffer; |
| 1514 | struct trace_array *tr = &global_trace; | 1608 | struct trace_array *tr = &global_trace; |
| 1515 | struct trace_array_cpu *data; | | |
| 1516 | struct bprint_entry *entry; | 1609 | struct bprint_entry *entry; |
| 1517 | unsigned long flags; | 1610 | unsigned long flags; |
| 1518 | int disable; | 1611 | char *tbuffer; |
| 1519 | int cpu, len = 0, size, pc; | 1612 | int len = 0, size, pc; |
| 1520 | | 1613 | |
| 1521 | if (unlikely(tracing_selftest_running || tracing_disabled)) | 1614 | if (unlikely(tracing_selftest_running || tracing_disabled)) |
| 1522 | return 0; | 1615 | return 0; |
| @@ -1526,43 +1619,36 @@ int trace_vbprintk(unsigned long ip, const char *fmt, va_list args) · 3 unmodified lines |
| 1526 | | 1619 | |
| 1527 | pc = preempt_count(); | 1620 | pc = preempt_count(); |
| 1528 | preempt_disable_notrace(); | 1621 | preempt_disable_notrace(); |
| 1529 | cpu = raw_smp_processor_id(); | | |
| 1530 | data = tr->data[cpu]; | | |
| 1531 | | 1622 | |
| 1532 | disable = atomic_inc_return(&data->disabled); | 1623 | tbuffer = get_trace_buf(); |
| 1533 | if (unlikely(disable != 1)) | 1624 | if (!tbuffer) { |
| | 1625 | len = 0; |
| 1534 | goto out; | 1626 | goto out; |
| | 1627 | } |
| 1535 | | 1628 | |
| 1536 | /* Lockdep uses trace_printk for lock tracing */ | 1629 | len = vbin_printf((u32 *)tbuffer, TRACE_BUF_SIZE/sizeof(int), fmt, args); |
| 1537 | local_irq_save(flags); | | |
| 1538 | arch_spin_lock(&trace_buf_lock); | | |
| 1539 | len = vbin_printf(trace_buf, TRACE_BUF_SIZE, fmt, args); | | |
| 1540 | | 1630 | |
| 1541 | if (len > TRACE_BUF_SIZE || len < 0) | 1631 | if (len > TRACE_BUF_SIZE/sizeof(int) || len < 0) |
| 1542 | goto out_unlock; | 1632 | goto out; |
| 1543 | | 1633 | |
| | 1634 | local_save_flags(flags); |
| 1544 | size = sizeof(*entry) + sizeof(u32) * len; | 1635 | size = sizeof(*entry) + sizeof(u32) * len; |
| 1545 | buffer = tr->buffer; | 1636 | buffer = tr->buffer; |
| 1546 | event = trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size, | 1637 | event = trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size, |
| 1547 | flags, pc); | 1638 | flags, pc); |
| 1548 | if (!event) | 1639 | if (!event) |
| 1549 | goto out_unlock; | 1640 | goto out; |
| 1550 | entry = ring_buffer_event_data(event); | 1641 | entry = ring_buffer_event_data(event); |
| 1551 | entry->ip = ip; | 1642 | entry->ip = ip; |
| 1552 | entry->fmt = fmt; | 1643 | entry->fmt = fmt; |
| 1553 | | 1644 | |
| 1554 | memcpy(entry->buf, trace_buf, sizeof(u32) * len); | 1645 | memcpy(entry->buf, tbuffer, sizeof(u32) * len); |
| 1555 | if (!filter_check_discard(call, entry, buffer, event)) { | 1646 | if (!filter_check_discard(call, entry, buffer, event)) { |
| 1556 | ring_buffer_unlock_commit(buffer, event); | 1647 | ring_buffer_unlock_commit(buffer, event); |
| 1557 | ftrace_trace_stack(buffer, flags, 6, pc); | 1648 | ftrace_trace_stack(buffer, flags, 6, pc); |
| 1558 | } | 1649 | } |
| 1559 | | 1650 | |
| 1560 | out_unlock: | | |
| 1561 | arch_spin_unlock(&trace_buf_lock); | | |
| 1562 | local_irq_restore(flags); | | |
| 1563 | | | |
| 1564 | out: | 1651 | out: |
| 1565 | atomic_dec_return(&data->disabled); | | |
| 1566 | preempt_enable_notrace(); | 1652 | preempt_enable_notrace(); |
| 1567 | unpause_graph_tracing(); | 1653 | unpause_graph_tracing(); |
| 1568 | | 1654 | |
| @@ -1588,58 +1674,53 @@ int trace_array_printk(struct trace_array *tr, · 19 unmodified lines |
| 1588 | int trace_array_vprintk(struct trace_array *tr, | 1674 | int trace_array_vprintk(struct trace_array *tr, |
| 1589 | unsigned long ip, const char *fmt, va_list args) | 1675 | unsigned long ip, const char *fmt, va_list args) |
| 1590 | { | 1676 | { |
| 1591 | static arch_spinlock_t trace_buf_lock = __ARCH_SPIN_LOCK_UNLOCKED; | | |
| 1592 | static char trace_buf[TRACE_BUF_SIZE]; | | |
| 1593 | | | |
| 1594 | struct ftrace_event_call *call = &event_print; | 1677 | struct ftrace_event_call *call = &event_print; |
| 1595 | struct ring_buffer_event *event; | 1678 | struct ring_buffer_event *event; |
| 1596 | struct ring_buffer *buffer; | 1679 | struct ring_buffer *buffer; |
| 1597 | struct trace_array_cpu *data; | 1680 | int len = 0, size, pc; |
| 1598 | int cpu, len = 0, size, pc; | | |
| 1599 | struct print_entry *entry; | 1681 | struct print_entry *entry; |
| 1600 | unsigned long irq_flags; | 1682 | unsigned long flags; |
| 1601 | int disable; | 1683 | char *tbuffer; |
| 1602 | | 1684 | |
| 1603 | if (tracing_disabled || tracing_selftest_running) | 1685 | if (tracing_disabled || tracing_selftest_running) |
| 1604 | return 0; | 1686 | return 0; |
| 1605 | | 1687 | |
| | 1688 | /* Don't pollute graph traces with trace_vprintk internals */ |
| | 1689 | pause_graph_tracing(); |
| | 1690 | |
| 1606 | pc = preempt_count(); | 1691 | pc = preempt_count(); |
| 1607 | preempt_disable_notrace(); | 1692 | preempt_disable_notrace(); |
| 1608 | cpu = raw_smp_processor_id(); | | |
| 1609 | data = tr->data[cpu]; | | |
| 1610 | | 1693 | |
| 1611 | disable = atomic_inc_return(&data->disabled); | 1694 | |
| 1612 | if (unlikely(disable != 1)) | 1695 | tbuffer = get_trace_buf(); |
| | 1696 | if (!tbuffer) { |
| | 1697 | len = 0; |
| 1613 | goto out; | 1698 | goto out; |
| | 1699 | } |
| 1614 | | 1700 | |
| 1615 | pause_graph_tracing(); | 1701 | len = vsnprintf(tbuffer, TRACE_BUF_SIZE, fmt, args); |
| 1616 | raw_local_irq_save(irq_flags); | 1702 | if (len > TRACE_BUF_SIZE) |
| 1617 | arch_spin_lock(&trace_buf_lock); | 1703 | goto out; |
| 1618 | len = vsnprintf(trace_buf, TRACE_BUF_SIZE, fmt, args); | | |
| 1619 | | 1704 | |
| | 1705 | local_save_flags(flags); |
| 1620 | size = sizeof(*entry) + len + 1; | 1706 | size = sizeof(*entry) + len + 1; |
| 1621 | buffer = tr->buffer; | 1707 | buffer = tr->buffer; |
| 1622 | event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size, | 1708 | event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size, |
| 1623 | irq_flags, pc); | 1709 | flags, pc); |
| 1624 | if (!event) | 1710 | if (!event) |
| 1625 | goto out_unlock; | 1711 | goto out; |
| 1626 | entry = ring_buffer_event_data(event); | 1712 | entry = ring_buffer_event_data(event); |
| 1627 | entry->ip = ip; | 1713 | entry->ip = ip; |
| 1628 | | 1714 | |
| 1629 | memcpy(&entry->buf, trace_buf, len); | 1715 | memcpy(&entry->buf, tbuffer, len); |
| 1630 | entry->buf[len] = '\0'; | 1716 | entry->buf[len] = '\0'; |
| 1631 | if (!filter_check_discard(call, entry, buffer, event)) { | 1717 | if (!filter_check_discard(call, entry, buffer, event)) { |
| 1632 | ring_buffer_unlock_commit(buffer, event); | 1718 | ring_buffer_unlock_commit(buffer, event); |
| 1633 | ftrace_trace_stack(buffer, irq_flags, 6, pc); | 1719 | ftrace_trace_stack(buffer, flags, 6, pc); |
| 1634 | } | 1720 | } |
| 1635 | | | |
| 1636 | out_unlock: | | |
| 1637 | arch_spin_unlock(&trace_buf_lock); | | |
| 1638 | raw_local_irq_restore(irq_flags); | | |
| 1639 | unpause_graph_tracing(); | | |
| 1640 | out: | 1721 | out: |
| 1641 | atomic_dec_return(&data->disabled); | | |
| 1642 | preempt_enable_notrace(); | 1722 | preempt_enable_notrace(); |
| | 1723 | unpause_graph_tracing(); |
| 1643 | | 1724 | |
| 1644 | return len; | 1725 | return len; |
| 1645 | } | 1726 | } |
| @@ -2974,7 +3055,14 @@ int tracer_init(struct tracer *t, struct trace_array *tr) · 1328 unmodified lines |
| 2974 | return t->init(tr); | 3055 | return t->init(tr); |
| 2975 | } | 3056 | } |
| 2976 | | 3057 | |
| 2977 | static int __tracing_resize_ring_buffer(unsigned long size) | 3058 | static void set_buffer_entries(struct trace_array *tr, unsigned long val) |
| | 3059 | { |
| | 3060 | int cpu; |
| | 3061 | for_each_tracing_cpu(cpu) |
| | 3062 | tr->data[cpu]->entries = val; |
| | 3063 | } |
| | 3064 | |
| | 3065 | static int __tracing_resize_ring_buffer(unsigned long size, int cpu) |
| 2978 | { | 3066 | { |
| 2979 | int ret; | 3067 | int ret; |
| 2980 | | 3068 | |
| @@ -2985,19 +3073,32 @@ static int __tracing_resize_ring_buffer(unsigned long size) · 4 unmodified lines |
| 2985 | */ | 3073 | */ |
| 2986 | ring_buffer_expanded = 1; | 3074 | ring_buffer_expanded = 1; |
| 2987 | | 3075 | |
| 2988 | ret = ring_buffer_resize(global_trace.buffer, size); | 3076 | ret = ring_buffer_resize(global_trace.buffer, size, cpu); |
| 2989 | if (ret < 0) | 3077 | if (ret < 0) |
| 2990 | return ret; | 3078 | return ret; |
| 2991 | | 3079 | |
| 2992 | if (!current_trace->use_max_tr) | 3080 | if (!current_trace->use_max_tr) |
| 2993 | goto out; | 3081 | goto out; |
| 2994 | | 3082 | |
| 2995 | ret = ring_buffer_resize(max_tr.buffer, size); | 3083 | ret = ring_buffer_resize(max_tr.buffer, size, cpu); |
| 2996 | if (ret < 0) { | 3084 | if (ret < 0) { |
| 2997 | int r; | 3085 | int r = 0; |
| | 3086 | |
| | 3087 | if (cpu == RING_BUFFER_ALL_CPUS) { |
| | 3088 | int i; |
| | 3089 | for_each_tracing_cpu(i) { |
| | 3090 | r = ring_buffer_resize(global_trace.buffer, |
| | 3091 | global_trace.data[i]->entries, |
| | 3092 | i); |
| | 3093 | if (r < 0) |
| | 3094 | break; |
| | 3095 | } |
| | 3096 | } else { |
| | 3097 | r = ring_buffer_resize(global_trace.buffer, |
| | 3098 | global_trace.data[cpu]->entries, |
| | 3099 | cpu); |
| | 3100 | } |
| 2998 | | 3101 | |
| 2999 | r = ring_buffer_resize(global_trace.buffer, | | |
| 3000 | global_trace.entries); | | |
| 3001 | if (r < 0) { | 3102 | if (r < 0) { |
| 3002 | /* | 3103 | /* |
| 3003 | * AARGH! We are left with different | 3104 | * AARGH! We are left with different |
| @@ -3019,14 +3120,21 @@ static int __tracing_resize_ring_buffer(unsigned long size) · 15 unmodified lines |
| 3019 | return ret; | 3120 | return ret; |
| 3020 | } | 3121 | } |
| 3021 | | 3122 | |
| 3022 | max_tr.entries = size; | 3123 | if (cpu == RING_BUFFER_ALL_CPUS) |
| | 3124 | set_buffer_entries(&max_tr, size); |
| | 3125 | else |
| | 3126 | max_tr.data[cpu]->entries = size; |
| | 3127 | |
| 3023 | out: | 3128 | out: |
| 3024 | global_trace.entries = size; | 3129 | if (cpu == RING_BUFFER_ALL_CPUS) |
| | 3130 | set_buffer_entries(&global_trace, size); |
| | 3131 | else |
| | 3132 | global_trace.data[cpu]->entries = size; |
| 3025 | | 3133 | |
| 3026 | return ret; | 3134 | return ret; |
| 3027 | } | 3135 | } |
| 3028 | | 3136 | |
| 3029 | static ssize_t tracing_resize_ring_buffer(unsigned long size) | 3137 | static ssize_t tracing_resize_ring_buffer(unsigned long size, int cpu_id) |
| 3030 | { | 3138 | { |
| 3031 | int cpu, ret = size; | 3139 | int cpu, ret = size; |
| 3032 | | 3140 | |
| @@ -3042,12 +3150,19 @@ static ssize_t tracing_resize_ring_buffer(unsigned long size) · 9 unmodified lines |
| 3042 | atomic_inc(&max_tr.data[cpu]->disabled); | 3150 | atomic_inc(&max_tr.data[cpu]->disabled); |
| 3043 | } | 3151 | } |
| 3044 | | 3152 | |
| 3045 | if (size != global_trace.entries) | 3153 | if (cpu_id != RING_BUFFER_ALL_CPUS) { |
| 3046 | ret = __tracing_resize_ring_buffer(size); | 3154 | /* make sure, this cpu is enabled in the mask */ |
| | 3155 | if (!cpumask_test_cpu(cpu_id, tracing_buffer_mask)) { |
| | 3156 | ret = -EINVAL; |
| | 3157 | goto out; |
| | 3158 | } |
| | 3159 | } |
| 3047 | | 3160 | |
| | 3161 | ret = __tracing_resize_ring_buffer(size, cpu_id); |
| 3048 | if (ret < 0) | 3162 | if (ret < 0) |
| 3049 | ret = -ENOMEM; | 3163 | ret = -ENOMEM; |
| 3050 | | 3164 | |
| | 3165 | out: |
| 3051 | for_each_tracing_cpu(cpu) { | 3166 | for_each_tracing_cpu(cpu) { |
| 3052 | if (global_trace.data[cpu]) | 3167 | if (global_trace.data[cpu]) |
| 3053 | atomic_dec(&global_trace.data[cpu]->disabled); | 3168 | atomic_dec(&global_trace.data[cpu]->disabled); |
| @@ -3078,7 +3193,8 @@ int tracing_update_buffers(void) · 24 unmodified lines |
| 3078 | | 3193 | |
| 3079 | mutex_lock(&trace_types_lock); | 3194 | mutex_lock(&trace_types_lock); |
| 3080 | if (!ring_buffer_expanded) | 3195 | if (!ring_buffer_expanded) |
| 3081 | ret = __tracing_resize_ring_buffer(trace_buf_size); | 3196 | ret = __tracing_resize_ring_buffer(trace_buf_size, |
| | 3197 | RING_BUFFER_ALL_CPUS); |
| 3082 | mutex_unlock(&trace_types_lock); | 3198 | mutex_unlock(&trace_types_lock); |
| 3083 | | 3199 | |
| 3084 | return ret; | 3200 | return ret; |
| @@ -3102,7 +3218,8 @@ static int tracing_set_tracer(const char *buf) · 17 unmodified lines |
| 3102 | mutex_lock(&trace_types_lock); | 3218 | mutex_lock(&trace_types_lock); |
| 3103 | | 3219 | |
| 3104 | if (!ring_buffer_expanded) { | 3220 | if (!ring_buffer_expanded) { |
| 3105 | ret = __tracing_resize_ring_buffer(trace_buf_size); | 3221 | ret = __tracing_resize_ring_buffer(trace_buf_size, |
| | 3222 | RING_BUFFER_ALL_CPUS); |
| 3106 | if (ret < 0) | 3223 | if (ret < 0) |
| 3107 | goto out; | 3224 | goto out; |
| 3108 | ret = 0; | 3225 | ret = 0; |
| @@ -3128,8 +3245,8 @@ static int tracing_set_tracer(const char *buf) · 19 unmodified lines |
| 3128 | * The max_tr ring buffer has some state (e.g. ring->clock) and | 3245 | * The max_tr ring buffer has some state (e.g. ring->clock) and |
| 3129 | * we want preserve it. | 3246 | * we want preserve it. |
| 3130 | */ | 3247 | */ |
| 3131 | ring_buffer_resize(max_tr.buffer, 1); | 3248 | ring_buffer_resize(max_tr.buffer, 1, RING_BUFFER_ALL_CPUS); |
| 3132 | max_tr.entries = 1; | 3249 | set_buffer_entries(&max_tr, 1); |
| 3133 | } | 3250 | } |
| 3134 | destroy_trace_option_files(topts); | 3251 | destroy_trace_option_files(topts); |
| 3135 | | 3252 | |
| @@ -3137,10 +3254,17 @@ static int tracing_set_tracer(const char *buf) · 1 unmodified line |
| 3137 | | 3254 | |
| 3138 | topts = create_trace_option_files(current_trace); | 3255 | topts = create_trace_option_files(current_trace); |
| 3139 | if (current_trace->use_max_tr) { | 3256 | if (current_trace->use_max_tr) { |
| 3140 | ret = ring_buffer_resize(max_tr.buffer, global_trace.entries); | 3257 | int cpu; |
| 3141 | if (ret < 0) | 3258 | /* we need to make per cpu buffer sizes equivalent */ |
| 3142 | goto out; | 3259 | for_each_tracing_cpu(cpu) { |
| 3143 | max_tr.entries = global_trace.entries; | 3260 | ret = ring_buffer_resize(max_tr.buffer, |
| | 3261 | global_trace.data[cpu]->entries, |
| | 3262 | cpu); |
| | 3263 | if (ret < 0) |
| | 3264 | goto out; |
| | 3265 | max_tr.data[cpu]->entries = |
| | 3266 | global_trace.data[cpu]->entries; |
| | 3267 | } |
| 3144 | } | 3268 | } |
| 3145 | | 3269 | |
| 3146 | if (t->init) { | 3270 | if (t->init) { |
| @@ -3642,30 +3766,82 @@ out_err: · 495 unmodified lines |
| 3642 | goto out; | 3766 | goto out; |
| 3643 | } | 3767 | } |
| 3644 | | 3768 | |
| | 3769 | struct ftrace_entries_info { |
| | 3770 | struct trace_array *tr; |
| | 3771 | int cpu; |
| | 3772 | }; |
| | 3773 | |
| | 3774 | static int tracing_entries_open(struct inode *inode, struct file *filp) |
| | 3775 | { |
| | 3776 | struct ftrace_entries_info *info; |
| | 3777 | |
| | 3778 | if (tracing_disabled) |
| | 3779 | return -ENODEV; |
| | 3780 | |
| | 3781 | info = kzalloc(sizeof(*info), GFP_KERNEL); |
| | 3782 | if (!info) |
| | 3783 | return -ENOMEM; |
| | 3784 | |
| | 3785 | info->tr = &global_trace; |
| | 3786 | info->cpu = (unsigned long)inode->i_private; |
| | 3787 | |
| | 3788 | filp->private_data = info; |
| | 3789 | |
| | 3790 | return 0; |
| | 3791 | } |
| | 3792 | |
| 3645 | static ssize_t | 3793 | static ssize_t |
| 3646 | tracing_entries_read(struct file *filp, char __user *ubuf, | 3794 | tracing_entries_read(struct file *filp, char __user *ubuf, |
| 3647 | size_t cnt, loff_t *ppos) | 3795 | size_t cnt, loff_t *ppos) |
| 3648 | { | 3796 | { |
| 3649 | struct trace_array *tr = filp->private_data; | 3797 | struct ftrace_entries_info *info = filp->private_data; |
| 3650 | char buf[96]; | 3798 | struct trace_array *tr = info->tr; |
| 3651 | int r; | 3799 | char buf[64]; |
| | 3800 | int r = 0; |
| | 3801 | ssize_t ret; |
| 3652 | | 3802 | |
| 3653 | mutex_lock(&trace_types_lock); | 3803 | mutex_lock(&trace_types_lock); |
| 3654 | if (!ring_buffer_expanded) | 3804 | |
| 3655 | r = sprintf(buf, "%lu (expanded: %lu)\n", | 3805 | if (info->cpu == RING_BUFFER_ALL_CPUS) { |
| 3656 | tr->entries >> 10, | 3806 | int cpu, buf_size_same; |
| 3657 | trace_buf_size >> 10); | 3807 | unsigned long size; |
| 3658 | else | 3808 | |
| 3659 | r = sprintf(buf, "%lu\n", tr->entries >> 10); | 3809 | size = 0; |
| | 3810 | buf_size_same = 1; |
| | 3811 | /* check if all cpu sizes are same */ |
| | 3812 | for_each_tracing_cpu(cpu) { |
| | 3813 | /* fill in the size from first enabled cpu */ |
| | 3814 | if (size == 0) |
| | 3815 | size = tr->data[cpu]->entries; |
| | 3816 | if (size != tr->data[cpu]->entries) { |
| | 3817 | buf_size_same = 0; |
| | 3818 | break; |
| | 3819 | } |
| | 3820 | } |
| | 3821 | |
| | 3822 | if (buf_size_same) { |
| | 3823 | if (!ring_buffer_expanded) |
| | 3824 | r = sprintf(buf, "%lu (expanded: %lu)\n", |
| | 3825 | size >> 10, |
| | 3826 | trace_buf_size >> 10); |
| | 3827 | else |
| | 3828 | r = sprintf(buf, "%lu\n", size >> 10); |
| | 3829 | } else |
| | 3830 | r = sprintf(buf, "X\n"); |
| | 3831 | } else |
| | 3832 | r = sprintf(buf, "%lu\n", tr->data[info->cpu]->entries >> 10); |
| | 3833 | |
| 3660 | mutex_unlock(&trace_types_lock); | 3834 | mutex_unlock(&trace_types_lock); |
| 3661 | | 3835 | |
| 3662 | return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); | 3836 | ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r); |
| | 3837 | return ret; |
| 3663 | } | 3838 | } |
| 3664 | | 3839 | |
| 3665 | static ssize_t | 3840 | static ssize_t |
| 3666 | tracing_entries_write(struct file *filp, const char __user *ubuf, | 3841 | tracing_entries_write(struct file *filp, const char __user *ubuf, |
| 3667 | size_t cnt, loff_t *ppos) | 3842 | size_t cnt, loff_t *ppos) |
| 3668 | { | 3843 | { |
| | 3844 | struct ftrace_entries_info *info = filp->private_data; |
| 3669 | unsigned long val; | 3845 | unsigned long val; |
| 3670 | int ret; | 3846 | int ret; |
| 3671 | | 3847 | |
| @@ -3680,7 +3856,7 @@ tracing_entries_write(struct file *filp, const char __user *ubuf, · 8 unmodified lines |
| 3680 | /* value is in KB */ | 3856 | /* value is in KB */ |
| 3681 | val <<= 10; | 3857 | val <<= 10; |
| 3682 | | 3858 | |
| 3683 | ret = tracing_resize_ring_buffer(val); | 3859 | ret = tracing_resize_ring_buffer(val, info->cpu); |
| 3684 | if (ret < 0) | 3860 | if (ret < 0) |
| 3685 | return ret; | 3861 | return ret; |
| 3686 | | 3862 | |
| @@ -3689,6 +3865,16 @@ tracing_entries_write(struct file *filp, const char __user *ubuf, · 2 unmodified lines |
| 3689 | return cnt; | 3865 | return cnt; |
| 3690 | } | 3866 | } |
| 3691 | | 3867 | |
| | 3868 | static int |
| | 3869 | tracing_entries_release(struct inode *inode, struct file *filp) |
| | 3870 | { |
| | 3871 | struct ftrace_entries_info *info = filp->private_data; |
| | 3872 | |
| | 3873 | kfree(info); |
| | 3874 | |
| | 3875 | return 0; |
| | 3876 | } |
| | 3877 | |
| 3692 | static ssize_t | 3878 | static ssize_t |
| 3693 | tracing_total_entries_read(struct file *filp, char __user *ubuf, | 3879 | tracing_total_entries_read(struct file *filp, char __user *ubuf, |
| 3694 | size_t cnt, loff_t *ppos) | 3880 | size_t cnt, loff_t *ppos) |
| @@ -3700,7 +3886,7 @@ tracing_total_entries_read(struct file *filp, char __user *ubuf, · 5 unmodified lines |
| 3700 | | 3886 | |
| 3701 | mutex_lock(&trace_types_lock); | 3887 | mutex_lock(&trace_types_lock); |
| 3702 | for_each_tracing_cpu(cpu) { | 3888 | for_each_tracing_cpu(cpu) { |
| 3703 | size += tr->entries >> 10; | 3889 | size += tr->data[cpu]->entries >> 10; |
| 3704 | if (!ring_buffer_expanded) | 3890 | if (!ring_buffer_expanded) |
| 3705 | expanded_size += trace_buf_size >> 10; | 3891 | expanded_size += trace_buf_size >> 10; |
| 3706 | } | 3892 | } |
| @@ -3734,7 +3920,7 @@ tracing_free_buffer_release(struct inode *inode, struct file *filp) · 27 unmodified lines |
| 3734 | if (trace_flags & TRACE_ITER_STOP_ON_FREE) | 3920 | if (trace_flags & TRACE_ITER_STOP_ON_FREE) |
| 3735 | tracing_off(); | 3921 | tracing_off(); |
| 3736 | /* resize the ring buffer to 0 */ | 3922 | /* resize the ring buffer to 0 */ |
| 3737 | tracing_resize_ring_buffer(0); | 3923 | tracing_resize_ring_buffer(0, RING_BUFFER_ALL_CPUS); |
| 3738 | | 3924 | |
| 3739 | return 0; | 3925 | return 0; |
| 3740 | } | 3926 | } |
| @@ -3933,9 +4119,10 @@ static const struct file_operations tracing_pipe_fops = { · 192 unmodified lines |
| 3933 | }; | 4119 | }; |
| 3934 | | 4120 | |
| 3935 | static const struct file_operations tracing_entries_fops = { | 4121 | static const struct file_operations tracing_entries_fops = { |
| 3936 | .open = tracing_open_generic, | 4122 | .open = tracing_entries_open, |
| 3937 | .read = tracing_entries_read, | 4123 | .read = tracing_entries_read, |
| 3938 | .write = tracing_entries_write, | 4124 | .write = tracing_entries_write, |
| | 4125 | .release = tracing_entries_release, |
| 3939 | .llseek = generic_file_llseek, | 4126 | .llseek = generic_file_llseek, |
| 3940 | }; | 4127 | }; |
| 3941 | | 4128 | |
| @@ -4387,6 +4574,9 @@ static void tracing_init_debugfs_percpu(long cpu) · 445 unmodified lines |
| 4387 | | 4574 | |
| 4388 | trace_create_file("stats", 0444, d_cpu, | 4575 | trace_create_file("stats", 0444, d_cpu, |
| 4389 | (void *) cpu, &tracing_stats_fops); | 4576 | (void *) cpu, &tracing_stats_fops); |
| | 4577 | |
| | 4578 | trace_create_file("buffer_size_kb", 0444, d_cpu, |
| | 4579 | (void *) cpu, &tracing_entries_fops); |
| 4390 | } | 4580 | } |
| 4391 | | 4581 | |
| 4392 | #ifdef CONFIG_FTRACE_SELFTEST | 4582 | #ifdef CONFIG_FTRACE_SELFTEST |
| @@ -4716,7 +4906,7 @@ static __init int tracer_init_debugfs(void) · 323 unmodified lines |
| 4716 | (void *) TRACE_PIPE_ALL_CPU, &tracing_pipe_fops); | 4906 | (void *) TRACE_PIPE_ALL_CPU, &tracing_pipe_fops); |
| 4717 | | 4907 | |
| 4718 | trace_create_file("buffer_size_kb", 0644, d_tracer, | 4908 | trace_create_file("buffer_size_kb", 0644, d_tracer, |
| 4719 | &global_trace, &tracing_entries_fops); | 4909 | (void *) RING_BUFFER_ALL_CPUS, &tracing_entries_fops); |
| 4720 | | 4910 | |
| 4721 | trace_create_file("buffer_total_size_kb", 0444, d_tracer, | 4911 | trace_create_file("buffer_total_size_kb", 0444, d_tracer, |
| 4722 | &global_trace, &tracing_total_entries_fops); | 4912 | &global_trace, &tracing_total_entries_fops); |
| @@ -4955,6 +5145,10 @@ __init static int tracer_alloc_buffers(void) · 232 unmodified lines |
| 4955 | if (!alloc_cpumask_var(&tracing_cpumask, GFP_KERNEL)) | 5145 | if (!alloc_cpumask_var(&tracing_cpumask, GFP_KERNEL)) |
| 4956 | goto out_free_buffer_mask; | 5146 | goto out_free_buffer_mask; |
| 4957 | | 5147 | |
| | 5148 | /* Only allocate trace_printk buffers if a trace_printk exists */ |
| | 5149 | if (__stop___trace_bprintk_fmt != __start___trace_bprintk_fmt) |
| | 5150 | trace_printk_init_buffers(); |
| | 5151 | |
| 4958 | /* To save memory, keep the ring buffer size to its minimum */ | 5152 | /* To save memory, keep the ring buffer size to its minimum */ |
| 4959 | if (ring_buffer_expanded) | 5153 | if (ring_buffer_expanded) |
| 4960 | ring_buf_size = trace_buf_size; | 5154 | ring_buf_size = trace_buf_size; |
| @@ -4973,7 +5167,6 @@ __init static int tracer_alloc_buffers(void) · 12 unmodified lines |
| 4973 | WARN_ON(1); | 5167 | WARN_ON(1); |
| 4974 | goto out_free_cpumask; | 5168 | goto out_free_cpumask; |
| 4975 | } | 5169 | } |
| 4976 | global_trace.entries = ring_buffer_size(global_trace.buffer); | | |
| 4977 | if (global_trace.buffer_disabled) | 5170 | if (global_trace.buffer_disabled) |
| 4978 | tracing_off(); | 5171 | tracing_off(); |
| 4979 | | 5172 | |
| @@ -4986,7 +5179,6 @@ __init static int tracer_alloc_buffers(void) · 6 unmodified lines |
| 4986 | ring_buffer_free(global_trace.buffer); | 5179 | ring_buffer_free(global_trace.buffer); |
| 4987 | goto out_free_cpumask; | 5180 | goto out_free_cpumask; |
| 4988 | } | 5181 | } |
| 4989 | max_tr.entries = 1; | | |
| 4990 | #endif | 5182 | #endif |
| 4991 | | 5183 | |
| 4992 | /* Allocate the first page for all buffers */ | 5184 | /* Allocate the first page for all buffers */ |
| @@ -4995,6 +5187,11 @@ __init static int tracer_alloc_buffers(void) · 2 unmodified lines |
| 4995 | max_tr.data[i] = &per_cpu(max_tr_data, i); | 5187 | max_tr.data[i] = &per_cpu(max_tr_data, i); |
| 4996 | } | 5188 | } |
| 4997 | | 5189 | |
| | 5190 | set_buffer_entries(&global_trace, ring_buf_size); |
| | 5191 | #ifdef CONFIG_TRACER_MAX_TRACE |
| | 5192 | set_buffer_entries(&max_tr, 1); |
| | 5193 | #endif |
| | 5194 | |
| 4998 | trace_init_cmdlines(); | 5195 | trace_init_cmdlines(); |
| 4999 | | 5196 | |
| 5000 | register_tracer(&nop_trace); | 5197 | register_tracer(&nop_trace); |