History

ring-buffer: Add event descriptor to simplify passing data

Add rb_event_info descriptor to pass event info to functions a bit easier than using a bunch of parameters. This will also allow for changing the code around a bit to find better fast paths.

Steven Rostedt (Red Hat) committed 11 years ago

Commit fcc742e1 parent 5e2d5ef1 file · +52 −39Browse files

Unified · Split

kernel/trace/ring_buffer.c+52−39
@@ -399,6 +399,17 @@ struct rb_irq_work { · 398 unmodified lines
399 bool wakeup_full;399 bool wakeup_full;
400};400};
401 401
402/*
403 * Structure to hold event state and handle nested events.
404 */
405struct rb_event_info {
406 u64 ts;
407 u64 delta;
408 unsigned long length;
409 struct buffer_page *tail_page;
410 int add_timestamp;
411};
412
402/*413/*
403 * Used for which event context the event is in.414 * Used for which event context the event is in.
404 * NMI = 0415 * NMI = 0
@@ -2000,9 +2011,12 @@ rb_add_time_stamp(struct ring_buffer_event *event, u64 delta) · 1595 unmodified lines
2000 */2011 */
2001static void2012static void
2002rb_update_event(struct ring_buffer_per_cpu *cpu_buffer,2013rb_update_event(struct ring_buffer_per_cpu *cpu_buffer,
2003 struct ring_buffer_event *event, unsigned length,2014 struct ring_buffer_event *event,
2004 int add_timestamp, u64 delta)2015 struct rb_event_info *info)
2005{2016{
2017 unsigned length = info->length;
2018 u64 delta = info->delta;
2019
2006 /* Only a commit updates the timestamp */2020 /* Only a commit updates the timestamp */
2007 if (unlikely(!rb_event_is_commit(cpu_buffer, event)))2021 if (unlikely(!rb_event_is_commit(cpu_buffer, event)))
2008 delta = 0;2022 delta = 0;
@@ -2011,7 +2025,7 @@ rb_update_event(struct ring_buffer_per_cpu *cpu_buffer, · 2 unmodified lines
2011 * If we need to add a timestamp, then we2025 * If we need to add a timestamp, then we
2012 * add it to the start of the resevered space.2026 * add it to the start of the resevered space.
2013 */2027 */
2014 if (unlikely(add_timestamp)) {2028 if (unlikely(info->add_timestamp)) {
2015 event = rb_add_time_stamp(event, delta);2029 event = rb_add_time_stamp(event, delta);
2016 length -= RB_LEN_TIME_EXTEND;2030 length -= RB_LEN_TIME_EXTEND;
2017 delta = 0;2031 delta = 0;
@@ -2203,10 +2217,11 @@ static unsigned rb_calculate_event_length(unsigned length) · 185 unmodified lines
2203 2217
2204static inline void2218static inline void
2205rb_reset_tail(struct ring_buffer_per_cpu *cpu_buffer,2219rb_reset_tail(struct ring_buffer_per_cpu *cpu_buffer,
2206 struct buffer_page *tail_page,2220 unsigned long tail, struct rb_event_info *info)
2207 unsigned long tail, unsigned long length)
2208{2221{
2222 struct buffer_page *tail_page = info->tail_page;
2209 struct ring_buffer_event *event;2223 struct ring_buffer_event *event;
2224 unsigned long length = info->length;
2210 2225
2211 /*2226 /*
2212 * Only the event that crossed the page boundary2227 * Only the event that crossed the page boundary
@@ -2276,13 +2291,14 @@ rb_reset_tail(struct ring_buffer_per_cpu *cpu_buffer, · 63 unmodified lines
2276 */2291 */
2277static noinline struct ring_buffer_event *2292static noinline struct ring_buffer_event *
2278rb_move_tail(struct ring_buffer_per_cpu *cpu_buffer,2293rb_move_tail(struct ring_buffer_per_cpu *cpu_buffer,
2279 unsigned long length, unsigned long tail,2294 unsigned long tail, struct rb_event_info *info)
2280 struct buffer_page *tail_page, u64 ts)
2281{2295{
2296 struct buffer_page *tail_page = info->tail_page;
2282 struct buffer_page *commit_page = cpu_buffer->commit_page;2297 struct buffer_page *commit_page = cpu_buffer->commit_page;
2283 struct ring_buffer *buffer = cpu_buffer->buffer;2298 struct ring_buffer *buffer = cpu_buffer->buffer;
2284 struct buffer_page *next_page;2299 struct buffer_page *next_page;
2285 int ret;2300 int ret;
2301 u64 ts;
2286 2302
2287 next_page = tail_page;2303 next_page = tail_page;
2288 2304
@@ -2368,25 +2384,24 @@ rb_move_tail(struct ring_buffer_per_cpu *cpu_buffer, · 79 unmodified lines
2368 2384
2369 out_again:2385 out_again:
2370 2386
2371 rb_reset_tail(cpu_buffer, tail_page, tail, length);2387 rb_reset_tail(cpu_buffer, tail, info);
2372 2388
2373 /* fail and let the caller try again */2389 /* fail and let the caller try again */
2374 return ERR_PTR(-EAGAIN);2390 return ERR_PTR(-EAGAIN);
2375 2391
2376 out_reset:2392 out_reset:
2377 /* reset write */2393 /* reset write */
2378 rb_reset_tail(cpu_buffer, tail_page, tail, length);2394 rb_reset_tail(cpu_buffer, tail, info);
2379 2395
2380 return NULL;2396 return NULL;
2381}2397}
2382 2398
2383static struct ring_buffer_event *2399static struct ring_buffer_event *
2384__rb_reserve_next(struct ring_buffer_per_cpu *cpu_buffer,2400__rb_reserve_next(struct ring_buffer_per_cpu *cpu_buffer,
2385 unsigned long length, u64 ts,2401 struct rb_event_info *info)
2386 u64 delta, int add_timestamp)
2387{2402{
2388 struct buffer_page *tail_page;
2389 struct ring_buffer_event *event;2403 struct ring_buffer_event *event;
2404 struct buffer_page *tail_page;
2390 unsigned long tail, write;2405 unsigned long tail, write;
2391 2406
2392 /*2407 /*
@@ -2394,33 +2409,32 @@ __rb_reserve_next(struct ring_buffer_per_cpu *cpu_buffer, · 1 unmodified line
2394 * hold in the time field of the event, then we append a2409 * hold in the time field of the event, then we append a
2395 * TIME EXTEND event ahead of the data event.2410 * TIME EXTEND event ahead of the data event.
2396 */2411 */
2397 if (unlikely(add_timestamp))2412 if (unlikely(info->add_timestamp))
2398 length += RB_LEN_TIME_EXTEND;2413 info->length += RB_LEN_TIME_EXTEND;
2399 2414
2400 tail_page = cpu_buffer->tail_page;2415 tail_page = info->tail_page = cpu_buffer->tail_page;
2401 write = local_add_return(length, &tail_page->write);2416 write = local_add_return(info->length, &tail_page->write);
2402 2417
2403 /* set write to only the index of the write */2418 /* set write to only the index of the write */
2404 write &= RB_WRITE_MASK;2419 write &= RB_WRITE_MASK;
2405 tail = write - length;2420 tail = write - info->length;
2406 2421
2407 /*2422 /*
2408 * If this is the first commit on the page, then it has the same2423 * If this is the first commit on the page, then it has the same
2409 * timestamp as the page itself.2424 * timestamp as the page itself.
2410 */2425 */
2411 if (!tail)2426 if (!tail)
2412 delta = 0;2427 info->delta = 0;
2413 2428
2414 /* See if we shot pass the end of this buffer page */2429 /* See if we shot pass the end of this buffer page */
2415 if (unlikely(write > BUF_PAGE_SIZE))2430 if (unlikely(write > BUF_PAGE_SIZE))
2416 return rb_move_tail(cpu_buffer, length, tail,2431 return rb_move_tail(cpu_buffer, tail, info);
2417 tail_page, ts);
2418 2432
2419 /* We reserved something on the buffer */2433 /* We reserved something on the buffer */
2420 2434
2421 event = __rb_page_index(tail_page, tail);2435 event = __rb_page_index(tail_page, tail);
2422 kmemcheck_annotate_bitfield(event, bitfield);2436 kmemcheck_annotate_bitfield(event, bitfield);
2423 rb_update_event(cpu_buffer, event, length, add_timestamp, delta);2437 rb_update_event(cpu_buffer, event, info);
2424 2438
2425 local_inc(&tail_page->entries);2439 local_inc(&tail_page->entries);
2426 2440
@@ -2429,10 +2443,10 @@ __rb_reserve_next(struct ring_buffer_per_cpu *cpu_buffer, · 2 unmodified lines
2429 * its timestamp.2443 * its timestamp.
2430 */2444 */
2431 if (!tail)2445 if (!tail)
2432 tail_page->page->time_stamp = ts;2446 tail_page->page->time_stamp = info->ts;
2433 2447
2434 /* account for these added bytes */2448 /* account for these added bytes */
2435 local_add(length, &cpu_buffer->entries_bytes);2449 local_add(info->length, &cpu_buffer->entries_bytes);
2436 2450
2437 return event;2451 return event;
2438}2452}
@@ -2521,9 +2535,8 @@ rb_reserve_next_event(struct ring_buffer *buffer, · 82 unmodified lines
2521 unsigned long length)2535 unsigned long length)
2522{2536{
2523 struct ring_buffer_event *event;2537 struct ring_buffer_event *event;
2524 u64 ts, delta;2538 struct rb_event_info info;
2525 int nr_loops = 0;2539 int nr_loops = 0;
2526 int add_timestamp;
2527 u64 diff;2540 u64 diff;
2528 2541
2529 rb_start_commit(cpu_buffer);2542 rb_start_commit(cpu_buffer);
@@ -2543,10 +2556,10 @@ rb_reserve_next_event(struct ring_buffer *buffer, · 13 unmodified lines
2543 }2556 }
2544#endif2557#endif
2545 2558
2546 length = rb_calculate_event_length(length);2559 info.length = rb_calculate_event_length(length);
2547 again:2560 again:
2548 add_timestamp = 0;2561 info.add_timestamp = 0;
2549 delta = 0;2562 info.delta = 0;
2550 2563
2551 /*2564 /*
2552 * We allow for interrupts to reenter here and do a trace.2565 * We allow for interrupts to reenter here and do a trace.
@@ -2560,35 +2573,35 @@ rb_reserve_next_event(struct ring_buffer *buffer, · 7 unmodified lines
2560 if (RB_WARN_ON(cpu_buffer, ++nr_loops > 1000))2573 if (RB_WARN_ON(cpu_buffer, ++nr_loops > 1000))
2561 goto out_fail;2574 goto out_fail;
2562 2575
2563 ts = rb_time_stamp(cpu_buffer->buffer);2576 info.ts = rb_time_stamp(cpu_buffer->buffer);
2564 diff = ts - cpu_buffer->write_stamp;2577 diff = info.ts - cpu_buffer->write_stamp;
2565 2578
2566 /* make sure this diff is calculated here */2579 /* make sure this diff is calculated here */
2567 barrier();2580 barrier();
2568 2581
2569 /* Did the write stamp get updated already? */2582 /* Did the write stamp get updated already? */
2570 if (likely(ts >= cpu_buffer->write_stamp)) {2583 if (likely(info.ts >= cpu_buffer->write_stamp)) {
2571 delta = diff;2584 info.delta = diff;
2572 if (unlikely(test_time_stamp(delta))) {2585 if (unlikely(test_time_stamp(info.delta))) {
2573 int local_clock_stable = 1;2586 int local_clock_stable = 1;
2574#ifdef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK2587#ifdef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK
2575 local_clock_stable = sched_clock_stable();2588 local_clock_stable = sched_clock_stable();
2576#endif2589#endif
2577 WARN_ONCE(delta > (1ULL << 59),2590 WARN_ONCE(info.delta > (1ULL << 59),
2578 KERN_WARNING "Delta way too big! %llu ts=%llu write stamp = %llu\n%s",2591 KERN_WARNING "Delta way too big! %llu ts=%llu write stamp = %llu\n%s",
2579 (unsigned long long)delta,2592 (unsigned long long)info.delta,
2580 (unsigned long long)ts,2593 (unsigned long long)info.ts,
2581 (unsigned long long)cpu_buffer->write_stamp,2594 (unsigned long long)cpu_buffer->write_stamp,
2582 local_clock_stable ? "" :2595 local_clock_stable ? "" :
2583 "If you just came from a suspend/resume,\n"2596 "If you just came from a suspend/resume,\n"
2584 "please switch to the trace global clock:\n"2597 "please switch to the trace global clock:\n"
2585 " echo global > /sys/kernel/debug/tracing/trace_clock\n");2598 " echo global > /sys/kernel/debug/tracing/trace_clock\n");
2586 add_timestamp = 1;2599 info.add_timestamp = 1;
2587 }2600 }
2588 }2601 }
2589 2602
2590 event = __rb_reserve_next(cpu_buffer, length, ts,2603 event = __rb_reserve_next(cpu_buffer, &info);
2591 delta, add_timestamp);2604
2592 if (unlikely(PTR_ERR(event) == -EAGAIN))2605 if (unlikely(PTR_ERR(event) == -EAGAIN))
2593 goto again;2606 goto again;
2594 2607