/* * Generic ring buffer * * Copyright (C) 2008 Steven Rostedt <srostedt@redhat.com> */#include <linux/ring_buffer.h>#include <linux/spinlock.h>#include <linux/debugfs.h>#include <linux/uaccess.h>#include <linux/module.h>#include <linux/percpu.h>#include <linux/mutex.h>#include <linux/sched.h> /* used for sched_clock() (for now) */#include <linux/init.h>#include <linux/hash.h>#include <linux/list.h>#include <linux/fs.h> #include "trace.h" /* * A fast way to enable or disable all ring buffers is to * call tracing_on or tracing_off. Turning off the ring buffers * prevents all ring buffers from being recorded to. * Turning this switch on, makes it OK to write to the * ring buffer, if the ring buffer is enabled itself. * * There's three layers that must be on in order to write * to the ring buffer. * * 1) This global flag must be set. * 2) The ring buffer must be enabled for recording. * 3) The per cpu buffer must be enabled for recording. * * In case of an anomaly, this global flag has a bit set that * will permantly disable all ring buffers. */ /* * Global flag to disable all recording to ring buffers * This has two bits: ON, DISABLED * * ON DISABLED * ---- ---------- * 0 0 : ring buffers are off * 1 0 : ring buffers are on * X 1 : ring buffers are permanently disabled */ enum { RB_BUFFERS_ON_BIT = 0, RB_BUFFERS_DISABLED_BIT = 1,}; enum { RB_BUFFERS_ON = 1 << RB_BUFFERS_ON_BIT, RB_BUFFERS_DISABLED = 1 << RB_BUFFERS_DISABLED_BIT,}; static long ring_buffer_flags __read_mostly = RB_BUFFERS_ON; /** * tracing_on - enable all tracing buffers * * This function enables all tracing buffers that may have been * disabled with tracing_off. */void tracing_on(void){ set_bit(RB_BUFFERS_ON_BIT, &ring_buffer_flags);}EXPORT_SYMBOL_GPL(tracing_on); /** * tracing_off - turn off all tracing buffers * * This function stops all tracing buffers from recording data. * It does not disable any overhead the tracers themselves may * be causing. This function simply causes all recording to * the ring buffers to fail. */void tracing_off(void){ clear_bit(RB_BUFFERS_ON_BIT, &ring_buffer_flags);}EXPORT_SYMBOL_GPL(tracing_off); /** * tracing_off_permanent - permanently disable ring buffers * * This function, once called, will disable all ring buffers * permanenty. */void tracing_off_permanent(void){ set_bit(RB_BUFFERS_DISABLED_BIT, &ring_buffer_flags);} #include "trace.h" /* Up this if you want to test the TIME_EXTENTS and normalization */#define DEBUG_SHIFT 0 /* FIXME!!! */u64 ring_buffer_time_stamp(int cpu){ u64 time; preempt_disable_notrace(); /* shift to debug/test normalization and TIME_EXTENTS */ time = sched_clock() << DEBUG_SHIFT; preempt_enable_no_resched_notrace(); return time;}EXPORT_SYMBOL_GPL(ring_buffer_time_stamp); void ring_buffer_normalize_time_stamp(int cpu, u64 *ts){ /* Just stupid testing the normalize function and deltas */ *ts >>= DEBUG_SHIFT;}EXPORT_SYMBOL_GPL(ring_buffer_normalize_time_stamp); #define RB_EVNT_HDR_SIZE (sizeof(struct ring_buffer_event))#define RB_ALIGNMENT_SHIFT 2#define RB_ALIGNMENT (1 << RB_ALIGNMENT_SHIFT)#define RB_MAX_SMALL_DATA 28 enum { RB_LEN_TIME_EXTEND = 8, RB_LEN_TIME_STAMP = 16,}; /* inline for ring buffer fast paths */static inline unsignedrb_event_length(struct ring_buffer_event *event){ unsigned length; switch (event->type) { case RINGBUF_TYPE_PADDING: /* undefined */ return -1; case RINGBUF_TYPE_TIME_EXTEND: return RB_LEN_TIME_EXTEND; case RINGBUF_TYPE_TIME_STAMP: return RB_LEN_TIME_STAMP; case RINGBUF_TYPE_DATA: if (event->len) length = event->len << RB_ALIGNMENT_SHIFT; else length = event->array[0]; return length + RB_EVNT_HDR_SIZE; default: BUG(); } /* not hit */ return 0;} /** * ring_buffer_event_length - return the length of the event * @event: the event to get the length of */unsigned ring_buffer_event_length(struct ring_buffer_event *event){ unsigned length = rb_event_length(event); if (event->type != RINGBUF_TYPE_DATA) return length; length -= RB_EVNT_HDR_SIZE; if (length > RB_MAX_SMALL_DATA + sizeof(event->array[0])) length -= sizeof(event->array[0]); return length;}EXPORT_SYMBOL_GPL(ring_buffer_event_length); /* inline for ring buffer fast paths */static inline void *rb_event_data(struct ring_buffer_event *event){ BUG_ON(event->type != RINGBUF_TYPE_DATA); /* If length is in len field, then array[0] has the data */ if (event->len) return (void *)&event->array[0]; /* Otherwise length is in array[0] and array[1] has the data */ return (void *)&event->array[1];} /** * ring_buffer_event_data - return the data of the event * @event: the event to get the data from */void *ring_buffer_event_data(struct ring_buffer_event *event){ return rb_event_data(event);}EXPORT_SYMBOL_GPL(ring_buffer_event_data); #define for_each_buffer_cpu(buffer, cpu) \ for_each_cpu(cpu, buffer->cpumask) #define TS_SHIFT 27#define TS_MASK ((1ULL << TS_SHIFT) - 1)#define TS_DELTA_TEST (~TS_MASK) struct buffer_data_page { u64 time_stamp; /* page time stamp */ local_t commit; /* write commited index */ unsigned char data[]; /* data of buffer page */}; struct buffer_page { local_t write; /* index for next write */ unsigned read; /* index for next read */ struct list_head list; /* list of free pages */ struct buffer_data_page *page; /* Actual data page */}; static void rb_init_page(struct buffer_data_page *bpage){ local_set(&bpage->commit, 0);} /* * Also stolen from mm/slob.c. Thanks to Mathieu Desnoyers for pointing * this issue out. */static inline void free_buffer_page(struct buffer_page *bpage){ if (bpage->page) free_page((unsigned long)bpage->page); kfree(bpage);} /* * We need to fit the time_stamp delta into 27 bits. */static inline int test_time_stamp(u64 delta){ if (delta & TS_DELTA_TEST) return 1; return 0;} #define BUF_PAGE_SIZE (PAGE_SIZE - offsetof(struct buffer_data_page, data)) /* * head_page == tail_page && head == tail then buffer is empty. */struct ring_buffer_per_cpu { int cpu; struct ring_buffer *buffer; spinlock_t reader_lock; /* serialize readers */ raw_spinlock_t lock; struct lock_class_key lock_key; struct list_head pages; struct buffer_page *head_page; /* read from head */ struct buffer_page *tail_page; /* write to tail */ struct buffer_page *commit_page; /* commited pages */ struct buffer_page *reader_page; unsigned long overrun; unsigned long entries; u64 write_stamp; u64 read_stamp; atomic_t record_disabled;}; struct ring_buffer { unsigned pages; unsigned flags; int cpus; cpumask_var_t cpumask; atomic_t record_disabled; struct mutex mutex; struct ring_buffer_per_cpu **buffers;}; struct ring_buffer_iter { struct ring_buffer_per_cpu *cpu_buffer; unsigned long head; struct buffer_page *head_page; u64 read_stamp;}; /* buffer may be either ring_buffer or ring_buffer_per_cpu */#define RB_WARN_ON(buffer, cond) \ ({ \ int _____ret = unlikely(cond); \ if (_____ret) { \ atomic_inc(&buffer->record_disabled); \ WARN_ON(1); \ } \ _____ret; \ }) /** * check_pages - integrity check of buffer pages * @cpu_buffer: CPU buffer with pages to test * * As a safty measure we check to make sure the data pages have not * been corrupted. */static int rb_check_pages(struct ring_buffer_per_cpu *cpu_buffer){ struct list_head *head = &cpu_buffer->pages; struct buffer_page *bpage, *tmp; if (RB_WARN_ON(cpu_buffer, head->next->prev != head)) return -1; if (RB_WARN_ON(cpu_buffer, head->prev->next != head)) return -1; list_for_each_entry_safe(bpage, tmp, head, list) { if (RB_WARN_ON(cpu_buffer, bpage->list.next->prev != &bpage->list)) return -1; if (RB_WARN_ON(cpu_buffer, bpage->list.prev->next != &bpage->list)) return -1; } return 0;} static int rb_allocate_pages(struct ring_buffer_per_cpu *cpu_buffer, unsigned nr_pages){ struct list_head *head = &cpu_buffer->pages; struct buffer_page *bpage, *tmp; unsigned long addr; LIST_HEAD(pages); unsigned i; for (i = 0; i < nr_pages; i++) { bpage = kzalloc_node(ALIGN(sizeof(*bpage), cache_line_size()), GFP_KERNEL, cpu_to_node(cpu_buffer->cpu)); if (!bpage) goto free_pages; list_add(&bpage->list, &pages); addr = __get_free_page(GFP_KERNEL); if (!addr) goto free_pages; bpage->page = (void *)addr; rb_init_page(bpage->page); } list_splice(&pages, head); rb_check_pages(cpu_buffer); return 0; free_pages: list_for_each_entry_safe(bpage, tmp, &pages, list) { list_del_init(&bpage->list); free_buffer_page(bpage); } return -ENOMEM;} static struct ring_buffer_per_cpu *rb_allocate_cpu_buffer(struct ring_buffer *buffer, int cpu){ struct ring_buffer_per_cpu *cpu_buffer; struct buffer_page *bpage; unsigned long addr; int ret; cpu_buffer = kzalloc_node(ALIGN(sizeof(*cpu_buffer), cache_line_size()), GFP_KERNEL, cpu_to_node(cpu)); if (!cpu_buffer) return NULL; cpu_buffer->cpu = cpu; cpu_buffer->buffer = buffer; spin_lock_init(&cpu_buffer->reader_lock); cpu_buffer->lock = (raw_spinlock_t)__RAW_SPIN_LOCK_UNLOCKED; INIT_LIST_HEAD(&cpu_buffer->pages); bpage = kzalloc_node(ALIGN(sizeof(*bpage), cache_line_size()), GFP_KERNEL, cpu_to_node(cpu)); if (!bpage) goto fail_free_buffer; cpu_buffer->reader_page = bpage; addr = __get_free_page(GFP_KERNEL); if (!addr) goto fail_free_reader; bpage->page = (void *)addr; rb_init_page(bpage->page); INIT_LIST_HEAD(&cpu_buffer->reader_page->list); ret = rb_allocate_pages(cpu_buffer, buffer->pages); if (ret < 0) goto fail_free_reader; cpu_buffer->head_page = list_entry(cpu_buffer->pages.next, struct buffer_page, list); cpu_buffer->tail_page = cpu_buffer->commit_page = cpu_buffer->head_page; return cpu_buffer; fail_free_reader: free_buffer_page(cpu_buffer->reader_page); fail_free_buffer: kfree(cpu_buffer); return NULL;} static void rb_free_cpu_buffer(struct ring_buffer_per_cpu *cpu_buffer){ struct list_head *head = &cpu_buffer->pages; struct buffer_page *bpage, *tmp; list_del_init(&cpu_buffer->reader_page->list); free_buffer_page(cpu_buffer->reader_page); list_for_each_entry_safe(bpage, tmp, head, list) { list_del_init(&bpage->list); free_buffer_page(bpage); } kfree(cpu_buffer);} /* * Causes compile errors if the struct buffer_page gets bigger * than the struct page. */extern int ring_buffer_page_too_big(void); /** * ring_buffer_alloc - allocate a new ring_buffer * @size: the size in bytes per cpu that is needed. * @flags: attributes to set for the ring buffer. * * Currently the only flag that is available is the RB_FL_OVERWRITE * flag. This flag means that the buffer will overwrite old data * when the buffer wraps. If this flag is not set, the buffer will * drop data when the tail hits the head. */struct ring_buffer *ring_buffer_alloc(unsigned long size, unsigned flags){ struct ring_buffer *buffer; int bsize; int cpu; /* Paranoid! Optimizes out when all is well */ if (sizeof(struct buffer_page) > sizeof(struct page)) ring_buffer_page_too_big(); /* keep it in its own cache line */ buffer = kzalloc(ALIGN(sizeof(*buffer), cache_line_size()), GFP_KERNEL); if (!buffer) return NULL; if (!alloc_cpumask_var(&buffer->cpumask, GFP_KERNEL)) goto fail_free_buffer; buffer->pages = DIV_ROUND_UP(size, BUF_PAGE_SIZE); buffer->flags = flags; /* need at least two pages */ if (buffer->pages == 1) buffer->pages++; cpumask_copy(buffer->cpumask, cpu_possible_mask); buffer->cpus = nr_cpu_ids; bsize = sizeof(void *) * nr_cpu_ids; buffer->buffers = kzalloc(ALIGN(bsize, cache_line_size()), GFP_KERNEL); if (!buffer->buffers) goto fail_free_cpumask; for_each_buffer_cpu(buffer, cpu) { buffer->buffers[cpu] = rb_allocate_cpu_buffer(buffer, cpu); if (!buffer->buffers[cpu]) goto fail_free_buffers; } mutex_init(&buffer->mutex); return buffer; fail_free_buffers: for_each_buffer_cpu(buffer, cpu) { if (buffer->buffers[cpu]) rb_free_cpu_buffer(buffer->buffers[cpu]);Lines 1–500 of 2534. Load more
/* * Generic ring buffer * * Copyright (C) 2008 Steven Rostedt <srostedt@redhat.com> */ #include <linux/ring_buffer.h> #include <linux/spinlock.h> #include <linux/debugfs.h> #include <linux/uaccess.h> #include <linux/module.h> #include <linux/percpu.h> #include <linux/mutex.h> #include <linux/sched.h> /* used for sched_clock() (for now) */ #include <linux/init.h> #include <linux/hash.h> #include <linux/list.h> #include <linux/fs.h> #include "trace.h" /* * A fast way to enable or disable all ring buffers is to * call tracing_on or tracing_off. Turning off the ring buffers * prevents all ring buffers from being recorded to. * Turning this switch on, makes it OK to write to the * ring buffer, if the ring buffer is enabled itself. * * There's three layers that must be on in order to write * to the ring buffer. * * 1) This global flag must be set. * 2) The ring buffer must be enabled for recording. * 3) The per cpu buffer must be enabled for recording. * * In case of an anomaly, this global flag has a bit set that * will permantly disable all ring buffers. */ /* * Global flag to disable all recording to ring buffers * This has two bits: ON, DISABLED * * ON DISABLED * ---- ---------- * 0 0 : ring buffers are off * 1 0 : ring buffers are on * X 1 : ring buffers are permanently disabled */ enum { RB_BUFFERS_ON_BIT = 0, RB_BUFFERS_DISABLED_BIT = 1, }; enum { RB_BUFFERS_ON = 1 << RB_BUFFERS_ON_BIT, RB_BUFFERS_DISABLED = 1 << RB_BUFFERS_DISABLED_BIT, }; static long ring_buffer_flags __read_mostly = RB_BUFFERS_ON; /** * tracing_on - enable all tracing buffers * * This function enables all tracing buffers that may have been * disabled with tracing_off. */ void tracing_on(void) { set_bit(RB_BUFFERS_ON_BIT, &ring_buffer_flags); } EXPORT_SYMBOL_GPL(tracing_on); /** * tracing_off - turn off all tracing buffers * * This function stops all tracing buffers from recording data. * It does not disable any overhead the tracers themselves may * be causing. This function simply causes all recording to * the ring buffers to fail. */ void tracing_off(void) { clear_bit(RB_BUFFERS_ON_BIT, &ring_buffer_flags); } EXPORT_SYMBOL_GPL(tracing_off); /** * tracing_off_permanent - permanently disable ring buffers * * This function, once called, will disable all ring buffers * permanenty. */ void tracing_off_permanent(void) { set_bit(RB_BUFFERS_DISABLED_BIT, &ring_buffer_flags); } #include "trace.h" /* Up this if you want to test the TIME_EXTENTS and normalization */ #define DEBUG_SHIFT 0 /* FIXME!!! */ u64 ring_buffer_time_stamp(int cpu) { u64 time; preempt_disable_notrace(); /* shift to debug/test normalization and TIME_EXTENTS */ time = sched_clock() << DEBUG_SHIFT; preempt_enable_no_resched_notrace(); return time; } EXPORT_SYMBOL_GPL(ring_buffer_time_stamp); void ring_buffer_normalize_time_stamp(int cpu, u64 *ts) { /* Just stupid testing the normalize function and deltas */ *ts >>= DEBUG_SHIFT; } EXPORT_SYMBOL_GPL(ring_buffer_normalize_time_stamp); #define RB_EVNT_HDR_SIZE (sizeof(struct ring_buffer_event)) #define RB_ALIGNMENT_SHIFT 2 #define RB_ALIGNMENT (1 << RB_ALIGNMENT_SHIFT) #define RB_MAX_SMALL_DATA 28 enum { RB_LEN_TIME_EXTEND = 8, RB_LEN_TIME_STAMP = 16, }; /* inline for ring buffer fast paths */ static inline unsigned rb_event_length(struct ring_buffer_event *event) { unsigned length; switch (event->type) { case RINGBUF_TYPE_PADDING: /* undefined */ return -1; case RINGBUF_TYPE_TIME_EXTEND: return RB_LEN_TIME_EXTEND; case RINGBUF_TYPE_TIME_STAMP: return RB_LEN_TIME_STAMP; case RINGBUF_TYPE_DATA: if (event->len) length = event->len << RB_ALIGNMENT_SHIFT; else length = event->array[0]; return length + RB_EVNT_HDR_SIZE; default: BUG(); } /* not hit */ return 0; } /** * ring_buffer_event_length - return the length of the event * @event: the event to get the length of */ unsigned ring_buffer_event_length(struct ring_buffer_event *event) { unsigned length = rb_event_length(event); if (event->type != RINGBUF_TYPE_DATA) return length; length -= RB_EVNT_HDR_SIZE; if (length > RB_MAX_SMALL_DATA + sizeof(event->array[0])) length -= sizeof(event->array[0]); return length; } EXPORT_SYMBOL_GPL(ring_buffer_event_length); /* inline for ring buffer fast paths */ static inline void * rb_event_data(struct ring_buffer_event *event) { BUG_ON(event->type != RINGBUF_TYPE_DATA); /* If length is in len field, then array[0] has the data */ if (event->len) return (void *)&event->array[0]; /* Otherwise length is in array[0] and array[1] has the data */ return (void *)&event->array[1]; } /** * ring_buffer_event_data - return the data of the event * @event: the event to get the data from */ void *ring_buffer_event_data(struct ring_buffer_event *event) { return rb_event_data(event); } EXPORT_SYMBOL_GPL(ring_buffer_event_data); #define for_each_buffer_cpu(buffer, cpu) \ for_each_cpu(cpu, buffer->cpumask) #define TS_SHIFT 27 #define TS_MASK ((1ULL << TS_SHIFT) - 1) #define TS_DELTA_TEST (~TS_MASK) struct buffer_data_page { u64 time_stamp; /* page time stamp */ local_t commit; /* write commited index */ unsigned char data[]; /* data of buffer page */ }; struct buffer_page { local_t write; /* index for next write */ unsigned read; /* index for next read */ struct list_head list; /* list of free pages */ struct buffer_data_page *page; /* Actual data page */ }; static void rb_init_page(struct buffer_data_page *bpage) { local_set(&bpage->commit, 0); } /* * Also stolen from mm/slob.c. Thanks to Mathieu Desnoyers for pointing * this issue out. */ static inline void free_buffer_page(struct buffer_page *bpage) { if (bpage->page) free_page((unsigned long)bpage->page); kfree(bpage); } /* * We need to fit the time_stamp delta into 27 bits. */ static inline int test_time_stamp(u64 delta) { if (delta & TS_DELTA_TEST) return 1; return 0; } #define BUF_PAGE_SIZE (PAGE_SIZE - offsetof(struct buffer_data_page, data)) /* * head_page == tail_page && head == tail then buffer is empty. */ struct ring_buffer_per_cpu { int cpu; struct ring_buffer *buffer; spinlock_t reader_lock; /* serialize readers */ raw_spinlock_t lock; struct lock_class_key lock_key; struct list_head pages; struct buffer_page *head_page; /* read from head */ struct buffer_page *tail_page; /* write to tail */ struct buffer_page *commit_page; /* commited pages */ struct buffer_page *reader_page; unsigned long overrun; unsigned long entries; u64 write_stamp; u64 read_stamp; atomic_t record_disabled; }; struct ring_buffer { unsigned pages; unsigned flags; int cpus; cpumask_var_t cpumask; atomic_t record_disabled; struct mutex mutex; struct ring_buffer_per_cpu **buffers; }; struct ring_buffer_iter { struct ring_buffer_per_cpu *cpu_buffer; unsigned long head; struct buffer_page *head_page; u64 read_stamp; }; /* buffer may be either ring_buffer or ring_buffer_per_cpu */ #define RB_WARN_ON(buffer, cond) \ ({ \ int _____ret = unlikely(cond); \ if (_____ret) { \ atomic_inc(&buffer->record_disabled); \ WARN_ON(1); \ } \ _____ret; \ }) /** * check_pages - integrity check of buffer pages * @cpu_buffer: CPU buffer with pages to test * * As a safty measure we check to make sure the data pages have not * been corrupted. */ static int rb_check_pages(struct ring_buffer_per_cpu *cpu_buffer) { struct list_head *head = &cpu_buffer->pages; struct buffer_page *bpage, *tmp; if (RB_WARN_ON(cpu_buffer, head->next->prev != head)) return -1; if (RB_WARN_ON(cpu_buffer, head->prev->next != head)) return -1; list_for_each_entry_safe(bpage, tmp, head, list) { if (RB_WARN_ON(cpu_buffer, bpage->list.next->prev != &bpage->list)) return -1; if (RB_WARN_ON(cpu_buffer, bpage->list.prev->next != &bpage->list)) return -1; } return 0; } static int rb_allocate_pages(struct ring_buffer_per_cpu *cpu_buffer, unsigned nr_pages) { struct list_head *head = &cpu_buffer->pages; struct buffer_page *bpage, *tmp; unsigned long addr; LIST_HEAD(pages); unsigned i; for (i = 0; i < nr_pages; i++) { bpage = kzalloc_node(ALIGN(sizeof(*bpage), cache_line_size()), GFP_KERNEL, cpu_to_node(cpu_buffer->cpu)); if (!bpage) goto free_pages; list_add(&bpage->list, &pages); addr = __get_free_page(GFP_KERNEL); if (!addr) goto free_pages; bpage->page = (void *)addr; rb_init_page(bpage->page); } list_splice(&pages, head); rb_check_pages(cpu_buffer); return 0; free_pages: list_for_each_entry_safe(bpage, tmp, &pages, list) { list_del_init(&bpage->list); free_buffer_page(bpage); } return -ENOMEM; } static struct ring_buffer_per_cpu * rb_allocate_cpu_buffer(struct ring_buffer *buffer, int cpu) { struct ring_buffer_per_cpu *cpu_buffer; struct buffer_page *bpage; unsigned long addr; int ret; cpu_buffer = kzalloc_node(ALIGN(sizeof(*cpu_buffer), cache_line_size()), GFP_KERNEL, cpu_to_node(cpu)); if (!cpu_buffer) return NULL; cpu_buffer->cpu = cpu; cpu_buffer->buffer = buffer; spin_lock_init(&cpu_buffer->reader_lock); cpu_buffer->lock = (raw_spinlock_t)__RAW_SPIN_LOCK_UNLOCKED; INIT_LIST_HEAD(&cpu_buffer->pages); bpage = kzalloc_node(ALIGN(sizeof(*bpage), cache_line_size()), GFP_KERNEL, cpu_to_node(cpu)); if (!bpage) goto fail_free_buffer; cpu_buffer->reader_page = bpage; addr = __get_free_page(GFP_KERNEL); if (!addr) goto fail_free_reader; bpage->page = (void *)addr; rb_init_page(bpage->page); INIT_LIST_HEAD(&cpu_buffer->reader_page->list); ret = rb_allocate_pages(cpu_buffer, buffer->pages); if (ret < 0) goto fail_free_reader; cpu_buffer->head_page = list_entry(cpu_buffer->pages.next, struct buffer_page, list); cpu_buffer->tail_page = cpu_buffer->commit_page = cpu_buffer->head_page; return cpu_buffer; fail_free_reader: free_buffer_page(cpu_buffer->reader_page); fail_free_buffer: kfree(cpu_buffer); return NULL; } static void rb_free_cpu_buffer(struct ring_buffer_per_cpu *cpu_buffer) { struct list_head *head = &cpu_buffer->pages; struct buffer_page *bpage, *tmp; list_del_init(&cpu_buffer->reader_page->list); free_buffer_page(cpu_buffer->reader_page); list_for_each_entry_safe(bpage, tmp, head, list) { list_del_init(&bpage->list); free_buffer_page(bpage); } kfree(cpu_buffer); } /* * Causes compile errors if the struct buffer_page gets bigger * than the struct page. */ extern int ring_buffer_page_too_big(void); /** * ring_buffer_alloc - allocate a new ring_buffer * @size: the size in bytes per cpu that is needed. * @flags: attributes to set for the ring buffer. * * Currently the only flag that is available is the RB_FL_OVERWRITE * flag. This flag means that the buffer will overwrite old data * when the buffer wraps. If this flag is not set, the buffer will * drop data when the tail hits the head. */ struct ring_buffer *ring_buffer_alloc(unsigned long size, unsigned flags) { struct ring_buffer *buffer; int bsize; int cpu; /* Paranoid! Optimizes out when all is well */ if (sizeof(struct buffer_page) > sizeof(struct page)) ring_buffer_page_too_big(); /* keep it in its own cache line */ buffer = kzalloc(ALIGN(sizeof(*buffer), cache_line_size()), GFP_KERNEL); if (!buffer) return NULL; if (!alloc_cpumask_var(&buffer->cpumask, GFP_KERNEL)) goto fail_free_buffer; buffer->pages = DIV_ROUND_UP(size, BUF_PAGE_SIZE); buffer->flags = flags; /* need at least two pages */ if (buffer->pages == 1) buffer->pages++; cpumask_copy(buffer->cpumask, cpu_possible_mask); buffer->cpus = nr_cpu_ids; bsize = sizeof(void *) * nr_cpu_ids; buffer->buffers = kzalloc(ALIGN(bsize, cache_line_size()), GFP_KERNEL); if (!buffer->buffers) goto fail_free_cpumask; for_each_buffer_cpu(buffer, cpu) { buffer->buffers[cpu] = rb_allocate_cpu_buffer(buffer, cpu); if (!buffer->buffers[cpu]) goto fail_free_buffers; } mutex_init(&buffer->mutex); return buffer; fail_free_buffers: for_each_buffer_cpu(buffer, cpu) { if (buffer->buffers[cpu]) rb_free_cpu_buffer(buffer->buffers[cpu]);