/* * Generic ring buffer * * Copyright (C) 2008 Steven Rostedt <srostedt@redhat.com> */#include <linux/ring_buffer.h>#include <linux/spinlock.h>#include <linux/debugfs.h>#include <linux/uaccess.h>#include <linux/module.h>#include <linux/percpu.h>#include <linux/mutex.h>#include <linux/sched.h> /* used for sched_clock() (for now) */#include <linux/init.h>#include <linux/hash.h>#include <linux/list.h>#include <linux/fs.h> #include "trace.h" /* Global flag to disable all recording to ring buffers */static int ring_buffers_off __read_mostly; /** * tracing_on - enable all tracing buffers * * This function enables all tracing buffers that may have been * disabled with tracing_off. */void tracing_on(void){ ring_buffers_off = 0;} /** * tracing_off - turn off all tracing buffers * * This function stops all tracing buffers from recording data. * It does not disable any overhead the tracers themselves may * be causing. This function simply causes all recording to * the ring buffers to fail. */void tracing_off(void){ ring_buffers_off = 1;} /* Up this if you want to test the TIME_EXTENTS and normalization */#define DEBUG_SHIFT 0 /* FIXME!!! */u64 ring_buffer_time_stamp(int cpu){ u64 time; preempt_disable_notrace(); /* shift to debug/test normalization and TIME_EXTENTS */ time = sched_clock() << DEBUG_SHIFT; preempt_enable_notrace(); return time;} void ring_buffer_normalize_time_stamp(int cpu, u64 *ts){ /* Just stupid testing the normalize function and deltas */ *ts >>= DEBUG_SHIFT;} #define RB_EVNT_HDR_SIZE (sizeof(struct ring_buffer_event))#define RB_ALIGNMENT_SHIFT 2#define RB_ALIGNMENT (1 << RB_ALIGNMENT_SHIFT)#define RB_MAX_SMALL_DATA 28 enum { RB_LEN_TIME_EXTEND = 8, RB_LEN_TIME_STAMP = 16,}; /* inline for ring buffer fast paths */static inline unsignedrb_event_length(struct ring_buffer_event *event){ unsigned length; switch (event->type) { case RINGBUF_TYPE_PADDING: /* undefined */ return -1; case RINGBUF_TYPE_TIME_EXTEND: return RB_LEN_TIME_EXTEND; case RINGBUF_TYPE_TIME_STAMP: return RB_LEN_TIME_STAMP; case RINGBUF_TYPE_DATA: if (event->len) length = event->len << RB_ALIGNMENT_SHIFT; else length = event->array[0]; return length + RB_EVNT_HDR_SIZE; default: BUG(); } /* not hit */ return 0;} /** * ring_buffer_event_length - return the length of the event * @event: the event to get the length of */unsigned ring_buffer_event_length(struct ring_buffer_event *event){ return rb_event_length(event);} /* inline for ring buffer fast paths */static inline void *rb_event_data(struct ring_buffer_event *event){ BUG_ON(event->type != RINGBUF_TYPE_DATA); /* If length is in len field, then array[0] has the data */ if (event->len) return (void *)&event->array[0]; /* Otherwise length is in array[0] and array[1] has the data */ return (void *)&event->array[1];} /** * ring_buffer_event_data - return the data of the event * @event: the event to get the data from */void *ring_buffer_event_data(struct ring_buffer_event *event){ return rb_event_data(event);} #define for_each_buffer_cpu(buffer, cpu) \ for_each_cpu_mask(cpu, buffer->cpumask) #define TS_SHIFT 27#define TS_MASK ((1ULL << TS_SHIFT) - 1)#define TS_DELTA_TEST (~TS_MASK) /* * This hack stolen from mm/slob.c. * We can store per page timing information in the page frame of the page. * Thanks to Peter Zijlstra for suggesting this idea. */struct buffer_page { u64 time_stamp; /* page time stamp */ local_t write; /* index for next write */ local_t commit; /* write commited index */ unsigned read; /* index for next read */ struct list_head list; /* list of free pages */ void *page; /* Actual data page */}; /* * Also stolen from mm/slob.c. Thanks to Mathieu Desnoyers for pointing * this issue out. */static inline void free_buffer_page(struct buffer_page *bpage){ if (bpage->page) free_page((unsigned long)bpage->page); kfree(bpage);} /* * We need to fit the time_stamp delta into 27 bits. */static inline int test_time_stamp(u64 delta){ if (delta & TS_DELTA_TEST) return 1; return 0;} #define BUF_PAGE_SIZE PAGE_SIZE /* * head_page == tail_page && head == tail then buffer is empty. */struct ring_buffer_per_cpu { int cpu; struct ring_buffer *buffer; spinlock_t lock; struct lock_class_key lock_key; struct list_head pages; struct buffer_page *head_page; /* read from head */ struct buffer_page *tail_page; /* write to tail */ struct buffer_page *commit_page; /* commited pages */ struct buffer_page *reader_page; unsigned long overrun; unsigned long entries; u64 write_stamp; u64 read_stamp; atomic_t record_disabled;}; struct ring_buffer { unsigned long size; unsigned pages; unsigned flags; int cpus; cpumask_t cpumask; atomic_t record_disabled; struct mutex mutex; struct ring_buffer_per_cpu **buffers;}; struct ring_buffer_iter { struct ring_buffer_per_cpu *cpu_buffer; unsigned long head; struct buffer_page *head_page; u64 read_stamp;}; #define RB_WARN_ON(buffer, cond) \ do { \ if (unlikely(cond)) { \ atomic_inc(&buffer->record_disabled); \ WARN_ON(1); \ } \ } while (0) #define RB_WARN_ON_RET(buffer, cond) \ do { \ if (unlikely(cond)) { \ atomic_inc(&buffer->record_disabled); \ WARN_ON(1); \ return -1; \ } \ } while (0) #define RB_WARN_ON_ONCE(buffer, cond) \ do { \ static int once; \ if (unlikely(cond) && !once) { \ once++; \ atomic_inc(&buffer->record_disabled); \ WARN_ON(1); \ } \ } while (0) /** * check_pages - integrity check of buffer pages * @cpu_buffer: CPU buffer with pages to test * * As a safty measure we check to make sure the data pages have not * been corrupted. */static int rb_check_pages(struct ring_buffer_per_cpu *cpu_buffer){ struct list_head *head = &cpu_buffer->pages; struct buffer_page *page, *tmp; RB_WARN_ON_RET(cpu_buffer, head->next->prev != head); RB_WARN_ON_RET(cpu_buffer, head->prev->next != head); list_for_each_entry_safe(page, tmp, head, list) { RB_WARN_ON_RET(cpu_buffer, page->list.next->prev != &page->list); RB_WARN_ON_RET(cpu_buffer, page->list.prev->next != &page->list); } return 0;} static int rb_allocate_pages(struct ring_buffer_per_cpu *cpu_buffer, unsigned nr_pages){ struct list_head *head = &cpu_buffer->pages; struct buffer_page *page, *tmp; unsigned long addr; LIST_HEAD(pages); unsigned i; for (i = 0; i < nr_pages; i++) { page = kzalloc_node(ALIGN(sizeof(*page), cache_line_size()), GFP_KERNEL, cpu_to_node(cpu_buffer->cpu)); if (!page) goto free_pages; list_add(&page->list, &pages); addr = __get_free_page(GFP_KERNEL); if (!addr) goto free_pages; page->page = (void *)addr; } list_splice(&pages, head); rb_check_pages(cpu_buffer); return 0; free_pages: list_for_each_entry_safe(page, tmp, &pages, list) { list_del_init(&page->list); free_buffer_page(page); } return -ENOMEM;} static struct ring_buffer_per_cpu *rb_allocate_cpu_buffer(struct ring_buffer *buffer, int cpu){ struct ring_buffer_per_cpu *cpu_buffer; struct buffer_page *page; unsigned long addr; int ret; cpu_buffer = kzalloc_node(ALIGN(sizeof(*cpu_buffer), cache_line_size()), GFP_KERNEL, cpu_to_node(cpu)); if (!cpu_buffer) return NULL; cpu_buffer->cpu = cpu; cpu_buffer->buffer = buffer; spin_lock_init(&cpu_buffer->lock); INIT_LIST_HEAD(&cpu_buffer->pages); page = kzalloc_node(ALIGN(sizeof(*page), cache_line_size()), GFP_KERNEL, cpu_to_node(cpu)); if (!page) goto fail_free_buffer; cpu_buffer->reader_page = page; addr = __get_free_page(GFP_KERNEL); if (!addr) goto fail_free_reader; page->page = (void *)addr; INIT_LIST_HEAD(&cpu_buffer->reader_page->list); ret = rb_allocate_pages(cpu_buffer, buffer->pages); if (ret < 0) goto fail_free_reader; cpu_buffer->head_page = list_entry(cpu_buffer->pages.next, struct buffer_page, list); cpu_buffer->tail_page = cpu_buffer->commit_page = cpu_buffer->head_page; return cpu_buffer; fail_free_reader: free_buffer_page(cpu_buffer->reader_page); fail_free_buffer: kfree(cpu_buffer); return NULL;} static void rb_free_cpu_buffer(struct ring_buffer_per_cpu *cpu_buffer){ struct list_head *head = &cpu_buffer->pages; struct buffer_page *page, *tmp; list_del_init(&cpu_buffer->reader_page->list); free_buffer_page(cpu_buffer->reader_page); list_for_each_entry_safe(page, tmp, head, list) { list_del_init(&page->list); free_buffer_page(page); } kfree(cpu_buffer);} /* * Causes compile errors if the struct buffer_page gets bigger * than the struct page. */extern int ring_buffer_page_too_big(void); /** * ring_buffer_alloc - allocate a new ring_buffer * @size: the size in bytes that is needed. * @flags: attributes to set for the ring buffer. * * Currently the only flag that is available is the RB_FL_OVERWRITE * flag. This flag means that the buffer will overwrite old data * when the buffer wraps. If this flag is not set, the buffer will * drop data when the tail hits the head. */struct ring_buffer *ring_buffer_alloc(unsigned long size, unsigned flags){ struct ring_buffer *buffer; int bsize; int cpu; /* Paranoid! Optimizes out when all is well */ if (sizeof(struct buffer_page) > sizeof(struct page)) ring_buffer_page_too_big(); /* keep it in its own cache line */ buffer = kzalloc(ALIGN(sizeof(*buffer), cache_line_size()), GFP_KERNEL); if (!buffer) return NULL; buffer->pages = DIV_ROUND_UP(size, BUF_PAGE_SIZE); buffer->flags = flags; /* need at least two pages */ if (buffer->pages == 1) buffer->pages++; buffer->cpumask = cpu_possible_map; buffer->cpus = nr_cpu_ids; bsize = sizeof(void *) * nr_cpu_ids; buffer->buffers = kzalloc(ALIGN(bsize, cache_line_size()), GFP_KERNEL); if (!buffer->buffers) goto fail_free_buffer; for_each_buffer_cpu(buffer, cpu) { buffer->buffers[cpu] = rb_allocate_cpu_buffer(buffer, cpu); if (!buffer->buffers[cpu]) goto fail_free_buffers; } mutex_init(&buffer->mutex); return buffer; fail_free_buffers: for_each_buffer_cpu(buffer, cpu) { if (buffer->buffers[cpu]) rb_free_cpu_buffer(buffer->buffers[cpu]); } kfree(buffer->buffers); fail_free_buffer: kfree(buffer); return NULL;} /** * ring_buffer_free - free a ring buffer. * @buffer: the buffer to free. */voidring_buffer_free(struct ring_buffer *buffer){ int cpu; for_each_buffer_cpu(buffer, cpu) rb_free_cpu_buffer(buffer->buffers[cpu]); kfree(buffer);} static void rb_reset_cpu(struct ring_buffer_per_cpu *cpu_buffer); static voidrb_remove_pages(struct ring_buffer_per_cpu *cpu_buffer, unsigned nr_pages){ struct buffer_page *page; struct list_head *p; unsigned i; atomic_inc(&cpu_buffer->record_disabled); synchronize_sched(); for (i = 0; i < nr_pages; i++) { BUG_ON(list_empty(&cpu_buffer->pages)); p = cpu_buffer->pages.next; page = list_entry(p, struct buffer_page, list); list_del_init(&page->list); free_buffer_page(page); } BUG_ON(list_empty(&cpu_buffer->pages)); rb_reset_cpu(cpu_buffer); rb_check_pages(cpu_buffer); atomic_dec(&cpu_buffer->record_disabled); } static voidrb_insert_pages(struct ring_buffer_per_cpu *cpu_buffer, struct list_head *pages, unsigned nr_pages){ struct buffer_page *page; struct list_head *p; unsigned i; atomic_inc(&cpu_buffer->record_disabled);Lines 1–500 of 2186. Load more
/* * Generic ring buffer * * Copyright (C) 2008 Steven Rostedt <srostedt@redhat.com> */ #include <linux/ring_buffer.h> #include <linux/spinlock.h> #include <linux/debugfs.h> #include <linux/uaccess.h> #include <linux/module.h> #include <linux/percpu.h> #include <linux/mutex.h> #include <linux/sched.h> /* used for sched_clock() (for now) */ #include <linux/init.h> #include <linux/hash.h> #include <linux/list.h> #include <linux/fs.h> #include "trace.h" /* Global flag to disable all recording to ring buffers */ static int ring_buffers_off __read_mostly; /** * tracing_on - enable all tracing buffers * * This function enables all tracing buffers that may have been * disabled with tracing_off. */ void tracing_on(void) { ring_buffers_off = 0; } /** * tracing_off - turn off all tracing buffers * * This function stops all tracing buffers from recording data. * It does not disable any overhead the tracers themselves may * be causing. This function simply causes all recording to * the ring buffers to fail. */ void tracing_off(void) { ring_buffers_off = 1; } /* Up this if you want to test the TIME_EXTENTS and normalization */ #define DEBUG_SHIFT 0 /* FIXME!!! */ u64 ring_buffer_time_stamp(int cpu) { u64 time; preempt_disable_notrace(); /* shift to debug/test normalization and TIME_EXTENTS */ time = sched_clock() << DEBUG_SHIFT; preempt_enable_notrace(); return time; } void ring_buffer_normalize_time_stamp(int cpu, u64 *ts) { /* Just stupid testing the normalize function and deltas */ *ts >>= DEBUG_SHIFT; } #define RB_EVNT_HDR_SIZE (sizeof(struct ring_buffer_event)) #define RB_ALIGNMENT_SHIFT 2 #define RB_ALIGNMENT (1 << RB_ALIGNMENT_SHIFT) #define RB_MAX_SMALL_DATA 28 enum { RB_LEN_TIME_EXTEND = 8, RB_LEN_TIME_STAMP = 16, }; /* inline for ring buffer fast paths */ static inline unsigned rb_event_length(struct ring_buffer_event *event) { unsigned length; switch (event->type) { case RINGBUF_TYPE_PADDING: /* undefined */ return -1; case RINGBUF_TYPE_TIME_EXTEND: return RB_LEN_TIME_EXTEND; case RINGBUF_TYPE_TIME_STAMP: return RB_LEN_TIME_STAMP; case RINGBUF_TYPE_DATA: if (event->len) length = event->len << RB_ALIGNMENT_SHIFT; else length = event->array[0]; return length + RB_EVNT_HDR_SIZE; default: BUG(); } /* not hit */ return 0; } /** * ring_buffer_event_length - return the length of the event * @event: the event to get the length of */ unsigned ring_buffer_event_length(struct ring_buffer_event *event) { return rb_event_length(event); } /* inline for ring buffer fast paths */ static inline void * rb_event_data(struct ring_buffer_event *event) { BUG_ON(event->type != RINGBUF_TYPE_DATA); /* If length is in len field, then array[0] has the data */ if (event->len) return (void *)&event->array[0]; /* Otherwise length is in array[0] and array[1] has the data */ return (void *)&event->array[1]; } /** * ring_buffer_event_data - return the data of the event * @event: the event to get the data from */ void *ring_buffer_event_data(struct ring_buffer_event *event) { return rb_event_data(event); } #define for_each_buffer_cpu(buffer, cpu) \ for_each_cpu_mask(cpu, buffer->cpumask) #define TS_SHIFT 27 #define TS_MASK ((1ULL << TS_SHIFT) - 1) #define TS_DELTA_TEST (~TS_MASK) /* * This hack stolen from mm/slob.c. * We can store per page timing information in the page frame of the page. * Thanks to Peter Zijlstra for suggesting this idea. */ struct buffer_page { u64 time_stamp; /* page time stamp */ local_t write; /* index for next write */ local_t commit; /* write commited index */ unsigned read; /* index for next read */ struct list_head list; /* list of free pages */ void *page; /* Actual data page */ }; /* * Also stolen from mm/slob.c. Thanks to Mathieu Desnoyers for pointing * this issue out. */ static inline void free_buffer_page(struct buffer_page *bpage) { if (bpage->page) free_page((unsigned long)bpage->page); kfree(bpage); } /* * We need to fit the time_stamp delta into 27 bits. */ static inline int test_time_stamp(u64 delta) { if (delta & TS_DELTA_TEST) return 1; return 0; } #define BUF_PAGE_SIZE PAGE_SIZE /* * head_page == tail_page && head == tail then buffer is empty. */ struct ring_buffer_per_cpu { int cpu; struct ring_buffer *buffer; spinlock_t lock; struct lock_class_key lock_key; struct list_head pages; struct buffer_page *head_page; /* read from head */ struct buffer_page *tail_page; /* write to tail */ struct buffer_page *commit_page; /* commited pages */ struct buffer_page *reader_page; unsigned long overrun; unsigned long entries; u64 write_stamp; u64 read_stamp; atomic_t record_disabled; }; struct ring_buffer { unsigned long size; unsigned pages; unsigned flags; int cpus; cpumask_t cpumask; atomic_t record_disabled; struct mutex mutex; struct ring_buffer_per_cpu **buffers; }; struct ring_buffer_iter { struct ring_buffer_per_cpu *cpu_buffer; unsigned long head; struct buffer_page *head_page; u64 read_stamp; }; #define RB_WARN_ON(buffer, cond) \ do { \ if (unlikely(cond)) { \ atomic_inc(&buffer->record_disabled); \ WARN_ON(1); \ } \ } while (0) #define RB_WARN_ON_RET(buffer, cond) \ do { \ if (unlikely(cond)) { \ atomic_inc(&buffer->record_disabled); \ WARN_ON(1); \ return -1; \ } \ } while (0) #define RB_WARN_ON_ONCE(buffer, cond) \ do { \ static int once; \ if (unlikely(cond) && !once) { \ once++; \ atomic_inc(&buffer->record_disabled); \ WARN_ON(1); \ } \ } while (0) /** * check_pages - integrity check of buffer pages * @cpu_buffer: CPU buffer with pages to test * * As a safty measure we check to make sure the data pages have not * been corrupted. */ static int rb_check_pages(struct ring_buffer_per_cpu *cpu_buffer) { struct list_head *head = &cpu_buffer->pages; struct buffer_page *page, *tmp; RB_WARN_ON_RET(cpu_buffer, head->next->prev != head); RB_WARN_ON_RET(cpu_buffer, head->prev->next != head); list_for_each_entry_safe(page, tmp, head, list) { RB_WARN_ON_RET(cpu_buffer, page->list.next->prev != &page->list); RB_WARN_ON_RET(cpu_buffer, page->list.prev->next != &page->list); } return 0; } static int rb_allocate_pages(struct ring_buffer_per_cpu *cpu_buffer, unsigned nr_pages) { struct list_head *head = &cpu_buffer->pages; struct buffer_page *page, *tmp; unsigned long addr; LIST_HEAD(pages); unsigned i; for (i = 0; i < nr_pages; i++) { page = kzalloc_node(ALIGN(sizeof(*page), cache_line_size()), GFP_KERNEL, cpu_to_node(cpu_buffer->cpu)); if (!page) goto free_pages; list_add(&page->list, &pages); addr = __get_free_page(GFP_KERNEL); if (!addr) goto free_pages; page->page = (void *)addr; } list_splice(&pages, head); rb_check_pages(cpu_buffer); return 0; free_pages: list_for_each_entry_safe(page, tmp, &pages, list) { list_del_init(&page->list); free_buffer_page(page); } return -ENOMEM; } static struct ring_buffer_per_cpu * rb_allocate_cpu_buffer(struct ring_buffer *buffer, int cpu) { struct ring_buffer_per_cpu *cpu_buffer; struct buffer_page *page; unsigned long addr; int ret; cpu_buffer = kzalloc_node(ALIGN(sizeof(*cpu_buffer), cache_line_size()), GFP_KERNEL, cpu_to_node(cpu)); if (!cpu_buffer) return NULL; cpu_buffer->cpu = cpu; cpu_buffer->buffer = buffer; spin_lock_init(&cpu_buffer->lock); INIT_LIST_HEAD(&cpu_buffer->pages); page = kzalloc_node(ALIGN(sizeof(*page), cache_line_size()), GFP_KERNEL, cpu_to_node(cpu)); if (!page) goto fail_free_buffer; cpu_buffer->reader_page = page; addr = __get_free_page(GFP_KERNEL); if (!addr) goto fail_free_reader; page->page = (void *)addr; INIT_LIST_HEAD(&cpu_buffer->reader_page->list); ret = rb_allocate_pages(cpu_buffer, buffer->pages); if (ret < 0) goto fail_free_reader; cpu_buffer->head_page = list_entry(cpu_buffer->pages.next, struct buffer_page, list); cpu_buffer->tail_page = cpu_buffer->commit_page = cpu_buffer->head_page; return cpu_buffer; fail_free_reader: free_buffer_page(cpu_buffer->reader_page); fail_free_buffer: kfree(cpu_buffer); return NULL; } static void rb_free_cpu_buffer(struct ring_buffer_per_cpu *cpu_buffer) { struct list_head *head = &cpu_buffer->pages; struct buffer_page *page, *tmp; list_del_init(&cpu_buffer->reader_page->list); free_buffer_page(cpu_buffer->reader_page); list_for_each_entry_safe(page, tmp, head, list) { list_del_init(&page->list); free_buffer_page(page); } kfree(cpu_buffer); } /* * Causes compile errors if the struct buffer_page gets bigger * than the struct page. */ extern int ring_buffer_page_too_big(void); /** * ring_buffer_alloc - allocate a new ring_buffer * @size: the size in bytes that is needed. * @flags: attributes to set for the ring buffer. * * Currently the only flag that is available is the RB_FL_OVERWRITE * flag. This flag means that the buffer will overwrite old data * when the buffer wraps. If this flag is not set, the buffer will * drop data when the tail hits the head. */ struct ring_buffer *ring_buffer_alloc(unsigned long size, unsigned flags) { struct ring_buffer *buffer; int bsize; int cpu; /* Paranoid! Optimizes out when all is well */ if (sizeof(struct buffer_page) > sizeof(struct page)) ring_buffer_page_too_big(); /* keep it in its own cache line */ buffer = kzalloc(ALIGN(sizeof(*buffer), cache_line_size()), GFP_KERNEL); if (!buffer) return NULL; buffer->pages = DIV_ROUND_UP(size, BUF_PAGE_SIZE); buffer->flags = flags; /* need at least two pages */ if (buffer->pages == 1) buffer->pages++; buffer->cpumask = cpu_possible_map; buffer->cpus = nr_cpu_ids; bsize = sizeof(void *) * nr_cpu_ids; buffer->buffers = kzalloc(ALIGN(bsize, cache_line_size()), GFP_KERNEL); if (!buffer->buffers) goto fail_free_buffer; for_each_buffer_cpu(buffer, cpu) { buffer->buffers[cpu] = rb_allocate_cpu_buffer(buffer, cpu); if (!buffer->buffers[cpu]) goto fail_free_buffers; } mutex_init(&buffer->mutex); return buffer; fail_free_buffers: for_each_buffer_cpu(buffer, cpu) { if (buffer->buffers[cpu]) rb_free_cpu_buffer(buffer->buffers[cpu]); } kfree(buffer->buffers); fail_free_buffer: kfree(buffer); return NULL; } /** * ring_buffer_free - free a ring buffer. * @buffer: the buffer to free. */ void ring_buffer_free(struct ring_buffer *buffer) { int cpu; for_each_buffer_cpu(buffer, cpu) rb_free_cpu_buffer(buffer->buffers[cpu]); kfree(buffer); } static void rb_reset_cpu(struct ring_buffer_per_cpu *cpu_buffer); static void rb_remove_pages(struct ring_buffer_per_cpu *cpu_buffer, unsigned nr_pages) { struct buffer_page *page; struct list_head *p; unsigned i; atomic_inc(&cpu_buffer->record_disabled); synchronize_sched(); for (i = 0; i < nr_pages; i++) { BUG_ON(list_empty(&cpu_buffer->pages)); p = cpu_buffer->pages.next; page = list_entry(p, struct buffer_page, list); list_del_init(&page->list); free_buffer_page(page); } BUG_ON(list_empty(&cpu_buffer->pages)); rb_reset_cpu(cpu_buffer); rb_check_pages(cpu_buffer); atomic_dec(&cpu_buffer->record_disabled); } static void rb_insert_pages(struct ring_buffer_per_cpu *cpu_buffer, struct list_head *pages, unsigned nr_pages) { struct buffer_page *page; struct list_head *p; unsigned i; atomic_inc(&cpu_buffer->record_disabled);