History

ring-buffer: Introducing ring-buffer mapping functions

In preparation for allowing the user-space to map a ring-buffer, add a set of mapping functions: ring_buffer_{map,unmap}() And controls on the ring-buffer: ring_buffer_map_get_reader() /* swap reader and head */ Mapping the ring-buffer also involves: A unique ID for each subbuf of the ring-buffer, currently they are only identified through their in-kernel VA. A meta-page, where are stored ring-buffer statistics and a description for the current reader The linear mapping exposes the meta-page, and each subbuf of the ring-buffer, ordered following their unique ID, assigned during the first mapping. Once mapped, no subbuf can get in or out of the ring-buffer: the buffer size will remain unmodified and the splice enabling functions will in reality simply memcpy the data instead of swapping subbufs. Link: https://lore.kernel.org/linux-trace-kernel/20240510140435.3550353-3-vdonnefort@google.com

Vincent Donnefort committed 2 years ago

Commit 117c3921 parent c09d4163 files · +463 −3Browse files

Unified · Split

3 files
  1. include/linux/ring_buffer.h +6 −0
  2. include/uapi/linux/trace_mmap.h +46 −0
  3. kernel/trace/ring_buffer.c +411 −3
include/linux/ring_buffer.h+6−0
@@ -6,6 +6,8 @@ · 5 unmodified lines
6#include <linux/seq_file.h>6#include <linux/seq_file.h>
7#include <linux/poll.h>7#include <linux/poll.h>
8 8
9#include <uapi/linux/trace_mmap.h>
10
9struct trace_buffer;11struct trace_buffer;
10struct ring_buffer_iter;12struct ring_buffer_iter;
11 13
@@ -223,4 +225,8 @@ int trace_rb_cpu_prepare(unsigned int cpu, struct hlist_node *node); · 211 unmodified lines
223#define trace_rb_cpu_prepare NULL225#define trace_rb_cpu_prepare NULL
224#endif226#endif
225 227
228int ring_buffer_map(struct trace_buffer *buffer, int cpu,
229 struct vm_area_struct *vma);
230int ring_buffer_unmap(struct trace_buffer *buffer, int cpu);
231int ring_buffer_map_get_reader(struct trace_buffer *buffer, int cpu);
226#endif /* _LINUX_RING_BUFFER_H */232#endif /* _LINUX_RING_BUFFER_H */
include/uapi/linux/trace_mmap.h+46−0
@@ -0,0 +1,46 @@
1/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
2#ifndef _TRACE_MMAP_H_
3#define _TRACE_MMAP_H_
4
5#include <linux/types.h>
6
7/**
8 * struct trace_buffer_meta - Ring-buffer Meta-page description
9 * @meta_page_size: Size of this meta-page.
10 * @meta_struct_len: Size of this structure.
11 * @subbuf_size: Size of each sub-buffer.
12 * @nr_subbufs: Number of subbfs in the ring-buffer, including the reader.
13 * @reader.lost_events: Number of events lost at the time of the reader swap.
14 * @reader.id: subbuf ID of the current reader. ID range [0 : @nr_subbufs - 1]
15 * @reader.read: Number of bytes read on the reader subbuf.
16 * @flags: Placeholder for now, 0 until new features are supported.
17 * @entries: Number of entries in the ring-buffer.
18 * @overrun: Number of entries lost in the ring-buffer.
19 * @read: Number of entries that have been read.
20 * @Reserved1: Internal use only.
21 * @Reserved2: Internal use only.
22 */
23struct trace_buffer_meta {
24 __u32 meta_page_size;
25 __u32 meta_struct_len;
26
27 __u32 subbuf_size;
28 __u32 nr_subbufs;
29
30 struct {
31 __u64 lost_events;
32 __u32 id;
33 __u32 read;
34 } reader;
35
36 __u64 flags;
37
38 __u64 entries;
39 __u64 overrun;
40 __u64 read;
41
42 __u64 Reserved1;
43 __u64 Reserved2;
44};
45
46#endif /* _TRACE_MMAP_H_ */
kernel/trace/ring_buffer.c+411−3
@@ -9,6 +9,7 @@ · 8 unmodified lines
9#include <linux/ring_buffer.h>9#include <linux/ring_buffer.h>
10#include <linux/trace_clock.h>10#include <linux/trace_clock.h>
11#include <linux/sched/clock.h>11#include <linux/sched/clock.h>
12#include <linux/cacheflush.h>
12#include <linux/trace_seq.h>13#include <linux/trace_seq.h>
13#include <linux/spinlock.h>14#include <linux/spinlock.h>
14#include <linux/irq_work.h>15#include <linux/irq_work.h>
@@ -26,6 +27,7 @@ · 11 unmodified lines
26#include <linux/list.h>27#include <linux/list.h>
27#include <linux/cpu.h>28#include <linux/cpu.h>
28#include <linux/oom.h>29#include <linux/oom.h>
30#include <linux/mm.h>
29 31
30#include <asm/local64.h>32#include <asm/local64.h>
31#include <asm/local.h>33#include <asm/local.h>
@@ -338,6 +340,7 @@ struct buffer_page { · 306 unmodified lines
338 local_t entries; /* entries on this page */340 local_t entries; /* entries on this page */
339 unsigned long real_end; /* real end of data */341 unsigned long real_end; /* real end of data */
340 unsigned order; /* order of the page */342 unsigned order; /* order of the page */
343 u32 id; /* ID for external mapping */
341 struct buffer_data_page *page; /* Actual data page */344 struct buffer_data_page *page; /* Actual data page */
342};345};
343 346
@@ -484,6 +487,12 @@ struct ring_buffer_per_cpu { · 140 unmodified lines
484 u64 read_stamp;487 u64 read_stamp;
485 /* pages removed since last reset */488 /* pages removed since last reset */
486 unsigned long pages_removed;489 unsigned long pages_removed;
490
491 unsigned int mapped;
492 struct mutex mapping_lock;
493 unsigned long *subbuf_ids; /* ID to subbuf VA */
494 struct trace_buffer_meta *meta_page;
495
487 /* ring buffer pages to update, > 0 to add, < 0 to remove */496 /* ring buffer pages to update, > 0 to add, < 0 to remove */
488 long nr_pages_to_update;497 long nr_pages_to_update;
489 struct list_head new_pages; /* new pages to add */498 struct list_head new_pages; /* new pages to add */
@@ -1599,6 +1608,7 @@ rb_allocate_cpu_buffer(struct trace_buffer *buffer, long nr_pages, int cpu) · 1109 unmodified lines
1599 init_irq_work(&cpu_buffer->irq_work.work, rb_wake_up_waiters);1608 init_irq_work(&cpu_buffer->irq_work.work, rb_wake_up_waiters);
1600 init_waitqueue_head(&cpu_buffer->irq_work.waiters);1609 init_waitqueue_head(&cpu_buffer->irq_work.waiters);
1601 init_waitqueue_head(&cpu_buffer->irq_work.full_waiters);1610 init_waitqueue_head(&cpu_buffer->irq_work.full_waiters);
1611 mutex_init(&cpu_buffer->mapping_lock);
1602 1612
1603 bpage = kzalloc_node(ALIGN(sizeof(*bpage), cache_line_size()),1613 bpage = kzalloc_node(ALIGN(sizeof(*bpage), cache_line_size()),
1604 GFP_KERNEL, cpu_to_node(cpu));1614 GFP_KERNEL, cpu_to_node(cpu));
@@ -1789,8 +1799,6 @@ bool ring_buffer_time_stamp_abs(struct trace_buffer *buffer) · 184 unmodified lines
1789 return buffer->time_stamp_abs;1799 return buffer->time_stamp_abs;
1790}1800}
1791 1801
1792static void rb_reset_cpu(struct ring_buffer_per_cpu *cpu_buffer);
1793
1794static inline unsigned long rb_page_entries(struct buffer_page *bpage)1802static inline unsigned long rb_page_entries(struct buffer_page *bpage)
1795{1803{
1796 return local_read(&bpage->entries) & RB_WRITE_MASK;1804 return local_read(&bpage->entries) & RB_WRITE_MASK;
@@ -5211,6 +5219,22 @@ static void rb_clear_buffer_page(struct buffer_page *page) · 3414 unmodified lines
5211 page->read = 0;5219 page->read = 0;
5212}5220}
5213 5221
5222static void rb_update_meta_page(struct ring_buffer_per_cpu *cpu_buffer)
5223{
5224 struct trace_buffer_meta *meta = cpu_buffer->meta_page;
5225
5226 meta->reader.read = cpu_buffer->reader_page->read;
5227 meta->reader.id = cpu_buffer->reader_page->id;
5228 meta->reader.lost_events = cpu_buffer->lost_events;
5229
5230 meta->entries = local_read(&cpu_buffer->entries);
5231 meta->overrun = local_read(&cpu_buffer->overrun);
5232 meta->read = cpu_buffer->read;
5233
5234 /* Some archs do not have data cache coherency between kernel and user-space */
5235 flush_dcache_folio(virt_to_folio(cpu_buffer->meta_page));
5236}
5237
5214static void5238static void
5215rb_reset_cpu(struct ring_buffer_per_cpu *cpu_buffer)5239rb_reset_cpu(struct ring_buffer_per_cpu *cpu_buffer)
5216{5240{
@@ -5255,6 +5279,9 @@ rb_reset_cpu(struct ring_buffer_per_cpu *cpu_buffer) · 38 unmodified lines
5255 cpu_buffer->lost_events = 0;5279 cpu_buffer->lost_events = 0;
5256 cpu_buffer->last_overrun = 0;5280 cpu_buffer->last_overrun = 0;
5257 5281
5282 if (cpu_buffer->mapped)
5283 rb_update_meta_page(cpu_buffer);
5284
5258 rb_head_page_activate(cpu_buffer);5285 rb_head_page_activate(cpu_buffer);
5259 cpu_buffer->pages_removed = 0;5286 cpu_buffer->pages_removed = 0;
5260}5287}
@@ -5469,6 +5496,12 @@ int ring_buffer_swap_cpu(struct trace_buffer *buffer_a, · 208 unmodified lines
5469 cpu_buffer_a = buffer_a->buffers[cpu];5496 cpu_buffer_a = buffer_a->buffers[cpu];
5470 cpu_buffer_b = buffer_b->buffers[cpu];5497 cpu_buffer_b = buffer_b->buffers[cpu];
5471 5498
5499 /* It's up to the callers to not try to swap mapped buffers */
5500 if (WARN_ON_ONCE(cpu_buffer_a->mapped || cpu_buffer_b->mapped)) {
5501 ret = -EBUSY;
5502 goto out;
5503 }
5504
5472 /* At least make sure the two buffers are somewhat the same */5505 /* At least make sure the two buffers are somewhat the same */
5473 if (cpu_buffer_a->nr_pages != cpu_buffer_b->nr_pages)5506 if (cpu_buffer_a->nr_pages != cpu_buffer_b->nr_pages)
5474 goto out;5507 goto out;
@@ -5733,7 +5766,8 @@ int ring_buffer_read_page(struct trace_buffer *buffer, · 258 unmodified lines
5733 * Otherwise, we can simply swap the page with the one passed in.5766 * Otherwise, we can simply swap the page with the one passed in.
5734 */5767 */
5735 if (read || (len < (commit - read)) ||5768 if (read || (len < (commit - read)) ||
5736 cpu_buffer->reader_page == cpu_buffer->commit_page) {5769 cpu_buffer->reader_page == cpu_buffer->commit_page ||
5770 cpu_buffer->mapped) {
5737 struct buffer_data_page *rpage = cpu_buffer->reader_page->page;5771 struct buffer_data_page *rpage = cpu_buffer->reader_page->page;
5738 unsigned int rpos = read;5772 unsigned int rpos = read;
5739 unsigned int pos = 0;5773 unsigned int pos = 0;
@@ -5956,6 +5990,11 @@ int ring_buffer_subbuf_order_set(struct trace_buffer *buffer, int order) · 216 unmodified lines
5956 5990
5957 cpu_buffer = buffer->buffers[cpu];5991 cpu_buffer = buffer->buffers[cpu];
5958 5992
5993 if (cpu_buffer->mapped) {
5994 err = -EBUSY;
5995 goto error;
5996 }
5997
5959 /* Update the number of pages to match the new size */5998 /* Update the number of pages to match the new size */
5960 nr_pages = old_size * buffer->buffers[cpu]->nr_pages;5999 nr_pages = old_size * buffer->buffers[cpu]->nr_pages;
5961 nr_pages = DIV_ROUND_UP(nr_pages, buffer->subbuf_size);6000 nr_pages = DIV_ROUND_UP(nr_pages, buffer->subbuf_size);
@@ -6057,6 +6096,375 @@ error: · 95 unmodified lines
6057}6096}
6058EXPORT_SYMBOL_GPL(ring_buffer_subbuf_order_set);6097EXPORT_SYMBOL_GPL(ring_buffer_subbuf_order_set);
6059 6098
6099static int rb_alloc_meta_page(struct ring_buffer_per_cpu *cpu_buffer)
6100{
6101 struct page *page;
6102
6103 if (cpu_buffer->meta_page)
6104 return 0;
6105
6106 page = alloc_page(GFP_USER | __GFP_ZERO);
6107 if (!page)
6108 return -ENOMEM;
6109
6110 cpu_buffer->meta_page = page_to_virt(page);
6111
6112 return 0;
6113}
6114
6115static void rb_free_meta_page(struct ring_buffer_per_cpu *cpu_buffer)
6116{
6117 unsigned long addr = (unsigned long)cpu_buffer->meta_page;
6118
6119 free_page(addr);
6120 cpu_buffer->meta_page = NULL;
6121}
6122
6123static void rb_setup_ids_meta_page(struct ring_buffer_per_cpu *cpu_buffer,
6124 unsigned long *subbuf_ids)
6125{
6126 struct trace_buffer_meta *meta = cpu_buffer->meta_page;
6127 unsigned int nr_subbufs = cpu_buffer->nr_pages + 1;
6128 struct buffer_page *first_subbuf, *subbuf;
6129 int id = 0;
6130
6131 subbuf_ids[id] = (unsigned long)cpu_buffer->reader_page->page;
6132 cpu_buffer->reader_page->id = id++;
6133
6134 first_subbuf = subbuf = rb_set_head_page(cpu_buffer);
6135 do {
6136 if (WARN_ON(id >= nr_subbufs))
6137 break;
6138
6139 subbuf_ids[id] = (unsigned long)subbuf->page;
6140 subbuf->id = id;
6141
6142 rb_inc_page(&subbuf);
6143 id++;
6144 } while (subbuf != first_subbuf);
6145
6146 /* install subbuf ID to kern VA translation */
6147 cpu_buffer->subbuf_ids = subbuf_ids;
6148
6149 meta->meta_page_size = PAGE_SIZE;
6150 meta->meta_struct_len = sizeof(*meta);
6151 meta->nr_subbufs = nr_subbufs;
6152 meta->subbuf_size = cpu_buffer->buffer->subbuf_size + BUF_PAGE_HDR_SIZE;
6153
6154 rb_update_meta_page(cpu_buffer);
6155}
6156
6157static struct ring_buffer_per_cpu *
6158rb_get_mapped_buffer(struct trace_buffer *buffer, int cpu)
6159{
6160 struct ring_buffer_per_cpu *cpu_buffer;
6161
6162 if (!cpumask_test_cpu(cpu, buffer->cpumask))
6163 return ERR_PTR(-EINVAL);
6164
6165 cpu_buffer = buffer->buffers[cpu];
6166
6167 mutex_lock(&cpu_buffer->mapping_lock);
6168
6169 if (!cpu_buffer->mapped) {
6170 mutex_unlock(&cpu_buffer->mapping_lock);
6171 return ERR_PTR(-ENODEV);
6172 }
6173
6174 return cpu_buffer;
6175}
6176
6177static void rb_put_mapped_buffer(struct ring_buffer_per_cpu *cpu_buffer)
6178{
6179 mutex_unlock(&cpu_buffer->mapping_lock);
6180}
6181
6182/*
6183 * Fast-path for rb_buffer_(un)map(). Called whenever the meta-page doesn't need
6184 * to be set-up or torn-down.
6185 */
6186static int __rb_inc_dec_mapped(struct ring_buffer_per_cpu *cpu_buffer,
6187 bool inc)
6188{
6189 unsigned long flags;
6190
6191 lockdep_assert_held(&cpu_buffer->mapping_lock);
6192
6193 if (inc && cpu_buffer->mapped == UINT_MAX)
6194 return -EBUSY;
6195
6196 if (WARN_ON(!inc && cpu_buffer->mapped == 0))
6197 return -EINVAL;
6198
6199 mutex_lock(&cpu_buffer->buffer->mutex);
6200 raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
6201
6202 if (inc)
6203 cpu_buffer->mapped++;
6204 else
6205 cpu_buffer->mapped--;
6206
6207 raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
6208 mutex_unlock(&cpu_buffer->buffer->mutex);
6209
6210 return 0;
6211}
6212
6213/*
6214 * +--------------+ pgoff == 0
6215 * | meta page |
6216 * +--------------+ pgoff == 1
6217 * | subbuffer 0 |
6218 * | |
6219 * +--------------+ pgoff == (1 + (1 << subbuf_order))
6220 * | subbuffer 1 |
6221 * | |
6222 * ...
6223 */
6224#ifdef CONFIG_MMU
6225static int __rb_map_vma(struct ring_buffer_per_cpu *cpu_buffer,
6226 struct vm_area_struct *vma)
6227{
6228 unsigned long nr_subbufs, nr_pages, vma_pages, pgoff = vma->vm_pgoff;
6229 unsigned int subbuf_pages, subbuf_order;
6230 struct page **pages;
6231 int p = 0, s = 0;
6232 int err;
6233
6234 /* Refuse MP_PRIVATE or writable mappings */
6235 if (vma->vm_flags & VM_WRITE || vma->vm_flags & VM_EXEC ||
6236 !(vma->vm_flags & VM_MAYSHARE))
6237 return -EPERM;
6238
6239 /*
6240 * Make sure the mapping cannot become writable later. Also tell the VM
6241 * to not touch these pages (VM_DONTCOPY | VM_DONTEXPAND).
6242 */
6243 vm_flags_mod(vma, VM_DONTCOPY | VM_DONTEXPAND | VM_DONTDUMP,
6244 VM_MAYWRITE);
6245
6246 lockdep_assert_held(&cpu_buffer->mapping_lock);
6247
6248 subbuf_order = cpu_buffer->buffer->subbuf_order;
6249 subbuf_pages = 1 << subbuf_order;
6250
6251 nr_subbufs = cpu_buffer->nr_pages + 1; /* + reader-subbuf */
6252 nr_pages = ((nr_subbufs) << subbuf_order) - pgoff + 1; /* + meta-page */
6253
6254 vma_pages = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
6255 if (!vma_pages || vma_pages > nr_pages)
6256 return -EINVAL;
6257
6258 nr_pages = vma_pages;
6259
6260 pages = kcalloc(nr_pages, sizeof(*pages), GFP_KERNEL);
6261 if (!pages)
6262 return -ENOMEM;
6263
6264 if (!pgoff) {
6265 pages[p++] = virt_to_page(cpu_buffer->meta_page);
6266
6267 /*
6268 * TODO: Align sub-buffers on their size, once
6269 * vm_insert_pages() supports the zero-page.
6270 */
6271 } else {
6272 /* Skip the meta-page */
6273 pgoff--;
6274
6275 if (pgoff % subbuf_pages) {
6276 err = -EINVAL;
6277 goto out;
6278 }
6279
6280 s += pgoff / subbuf_pages;
6281 }
6282
6283 while (p < nr_pages) {
6284 struct page *page = virt_to_page(cpu_buffer->subbuf_ids[s]);
6285 int off = 0;
6286
6287 if (WARN_ON_ONCE(s >= nr_subbufs)) {
6288 err = -EINVAL;
6289 goto out;
6290 }
6291
6292 for (; off < (1 << (subbuf_order)); off++, page++) {
6293 if (p >= nr_pages)
6294 break;
6295
6296 pages[p++] = page;
6297 }
6298 s++;
6299 }
6300
6301 err = vm_insert_pages(vma, vma->vm_start, pages, &nr_pages);
6302
6303out:
6304 kfree(pages);
6305
6306 return err;
6307}
6308#else
6309static int __rb_map_vma(struct ring_buffer_per_cpu *cpu_buffer,
6310 struct vm_area_struct *vma)
6311{
6312 return -EOPNOTSUPP;
6313}
6314#endif
6315
6316int ring_buffer_map(struct trace_buffer *buffer, int cpu,
6317 struct vm_area_struct *vma)
6318{
6319 struct ring_buffer_per_cpu *cpu_buffer;
6320 unsigned long flags, *subbuf_ids;
6321 int err = 0;
6322
6323 if (!cpumask_test_cpu(cpu, buffer->cpumask))
6324 return -EINVAL;
6325
6326 cpu_buffer = buffer->buffers[cpu];
6327
6328 mutex_lock(&cpu_buffer->mapping_lock);
6329
6330 if (cpu_buffer->mapped) {
6331 err = __rb_map_vma(cpu_buffer, vma);
6332 if (!err)
6333 err = __rb_inc_dec_mapped(cpu_buffer, true);
6334 mutex_unlock(&cpu_buffer->mapping_lock);
6335 return err;
6336 }
6337
6338 /* prevent another thread from changing buffer/sub-buffer sizes */
6339 mutex_lock(&buffer->mutex);
6340
6341 err = rb_alloc_meta_page(cpu_buffer);
6342 if (err)
6343 goto unlock;
6344
6345 /* subbuf_ids include the reader while nr_pages does not */
6346 subbuf_ids = kcalloc(cpu_buffer->nr_pages + 1, sizeof(*subbuf_ids), GFP_KERNEL);
6347 if (!subbuf_ids) {
6348 rb_free_meta_page(cpu_buffer);
6349 err = -ENOMEM;
6350 goto unlock;
6351 }
6352
6353 atomic_inc(&cpu_buffer->resize_disabled);
6354
6355 /*
6356 * Lock all readers to block any subbuf swap until the subbuf IDs are
6357 * assigned.
6358 */
6359 raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
6360 rb_setup_ids_meta_page(cpu_buffer, subbuf_ids);
6361 raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
6362
6363 err = __rb_map_vma(cpu_buffer, vma);
6364 if (!err) {
6365 raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
6366 cpu_buffer->mapped = 1;
6367 raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
6368 } else {
6369 kfree(cpu_buffer->subbuf_ids);
6370 cpu_buffer->subbuf_ids = NULL;
6371 rb_free_meta_page(cpu_buffer);
6372 }
6373
6374unlock:
6375 mutex_unlock(&buffer->mutex);
6376 mutex_unlock(&cpu_buffer->mapping_lock);
6377
6378 return err;
6379}
6380
6381int ring_buffer_unmap(struct trace_buffer *buffer, int cpu)
6382{
6383 struct ring_buffer_per_cpu *cpu_buffer;
6384 unsigned long flags;
6385 int err = 0;
6386
6387 if (!cpumask_test_cpu(cpu, buffer->cpumask))
6388 return -EINVAL;
6389
6390 cpu_buffer = buffer->buffers[cpu];
6391
6392 mutex_lock(&cpu_buffer->mapping_lock);
6393
6394 if (!cpu_buffer->mapped) {
6395 err = -ENODEV;
6396 goto out;
6397 } else if (cpu_buffer->mapped > 1) {
6398 __rb_inc_dec_mapped(cpu_buffer, false);
6399 goto out;
6400 }
6401
6402 mutex_lock(&buffer->mutex);
6403 raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
6404
6405 cpu_buffer->mapped = 0;
6406
6407 raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
6408
6409 kfree(cpu_buffer->subbuf_ids);
6410 cpu_buffer->subbuf_ids = NULL;
6411 rb_free_meta_page(cpu_buffer);
6412 atomic_dec(&cpu_buffer->resize_disabled);
6413
6414 mutex_unlock(&buffer->mutex);
6415
6416out:
6417 mutex_unlock(&cpu_buffer->mapping_lock);
6418
6419 return err;
6420}
6421
6422int ring_buffer_map_get_reader(struct trace_buffer *buffer, int cpu)
6423{
6424 struct ring_buffer_per_cpu *cpu_buffer;
6425 unsigned long reader_size;
6426 unsigned long flags;
6427
6428 cpu_buffer = rb_get_mapped_buffer(buffer, cpu);
6429 if (IS_ERR(cpu_buffer))
6430 return (int)PTR_ERR(cpu_buffer);
6431
6432 raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
6433
6434consume:
6435 if (rb_per_cpu_empty(cpu_buffer))
6436 goto out;
6437
6438 reader_size = rb_page_size(cpu_buffer->reader_page);
6439
6440 /*
6441 * There are data to be read on the current reader page, we can
6442 * return to the caller. But before that, we assume the latter will read
6443 * everything. Let's update the kernel reader accordingly.
6444 */
6445 if (cpu_buffer->reader_page->read < reader_size) {
6446 while (cpu_buffer->reader_page->read < reader_size)
6447 rb_advance_reader(cpu_buffer);
6448 goto out;
6449 }
6450
6451 if (WARN_ON(!rb_get_reader_page(cpu_buffer)))
6452 goto out;
6453
6454 goto consume;
6455
6456out:
6457 /* Some archs do not have data cache coherency between kernel and user-space */
6458 flush_dcache_folio(virt_to_folio(cpu_buffer->reader_page->page));
6459
6460 rb_update_meta_page(cpu_buffer);
6461
6462 raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
6463 rb_put_mapped_buffer(cpu_buffer);
6464
6465 return 0;
6466}
6467
6060/*6468/*
6061 * We only allocate new buffers, never free them if the CPU goes down.6469 * We only allocate new buffers, never free them if the CPU goes down.
6062 * If we were to free the buffer, then the user would lose any trace that was in6470 * If we were to free the buffer, then the user would lose any trace that was in