* No need to worry about races with clearing out the commit. * it only can increment when a commit takes place. But that * only happens in the outer most nested commit. */ local_set(&next_page->page->commit, 0); /* Either we update tail_page or an interrupt does */ if (try_cmpxchg(&cpu_buffer->tail_page, &tail_page, next_page)) local_inc(&cpu_buffer->pages_touched); }} static void rb_check_bpage(struct ring_buffer_per_cpu *cpu_buffer, struct buffer_page *bpage){ unsigned long val = (unsigned long)bpage; RB_WARN_ON(cpu_buffer, val & RB_FLAG_MASK);} static bool rb_check_links(struct ring_buffer_per_cpu *cpu_buffer, struct list_head *list){ if (RB_WARN_ON(cpu_buffer, rb_list_head(rb_list_head(list->next)->prev) != list)) return false; if (RB_WARN_ON(cpu_buffer, rb_list_head(rb_list_head(list->prev)->next) != list)) return false; return true;} /** * rb_check_pages - integrity check of buffer pages * @cpu_buffer: CPU buffer with pages to test * * As a safety measure we check to make sure the data pages have not * been corrupted. */static void rb_check_pages(struct ring_buffer_per_cpu *cpu_buffer){ struct list_head *head, *tmp; unsigned long buffer_cnt; unsigned long flags; int nr_loops = 0; /* * Walk the linked list underpinning the ring buffer and validate all * its next and prev links. * * The check acquires the reader_lock to avoid concurrent processing * with code that could be modifying the list. However, the lock cannot * be held for the entire duration of the walk, as this would make the * time when interrupts are disabled non-deterministic, dependent on the * ring buffer size. Therefore, the code releases and re-acquires the * lock after checking each page. The ring_buffer_per_cpu.cnt variable * is then used to detect if the list was modified while the lock was * not held, in which case the check needs to be restarted. * * The code attempts to perform the check at most three times before * giving up. This is acceptable because this is only a self-validation * to detect problems early on. In practice, the list modification * operations are fairly spaced, and so this check typically succeeds at * most on the second try. */again: if (++nr_loops > 3) return; raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags); head = rb_list_head(cpu_buffer->pages); if (!rb_check_links(cpu_buffer, head)) goto out_locked; buffer_cnt = cpu_buffer->cnt; tmp = head; raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags); while (true) { raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags); if (buffer_cnt != cpu_buffer->cnt) { /* The list was updated, try again. */ raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags); goto again; } tmp = rb_list_head(tmp->next); if (tmp == head) /* The iteration circled back, all is done. */ goto out_locked; if (!rb_check_links(cpu_buffer, tmp)) goto out_locked; raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags); } out_locked: raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);} /* * Take an address, add the meta data size as well as the array of * array subbuffer indexes, then align it to a subbuffer size. * * This is used to help find the next per cpu subbuffer within a mapped range. */static unsigned longrb_range_align_subbuf(unsigned long addr, int subbuf_size, int nr_subbufs){ addr += sizeof(struct ring_buffer_cpu_meta) + sizeof(int) * nr_subbufs; return ALIGN(addr, subbuf_size);} /* * Return the ring_buffer_meta for a given @cpu. */static void *rb_range_meta(struct trace_buffer *buffer, int nr_pages, int cpu){ int subbuf_size = buffer->subbuf_size + BUF_PAGE_HDR_SIZE; struct ring_buffer_cpu_meta *meta; struct ring_buffer_meta *bmeta; unsigned long ptr; int nr_subbufs; bmeta = buffer->meta; if (!bmeta) return NULL; ptr = (unsigned long)bmeta + bmeta->buffers_offset; meta = (struct ring_buffer_cpu_meta *)ptr; /* When nr_pages passed in is zero, the first meta has already been initialized */ if (!nr_pages) { nr_subbufs = meta->nr_subbufs; } else { /* Include the reader page */ nr_subbufs = nr_pages + 1; } /* * The first chunk may not be subbuffer aligned, where as * the rest of the chunks are. */ if (cpu) { ptr = rb_range_align_subbuf(ptr, subbuf_size, nr_subbufs); ptr += subbuf_size * nr_subbufs; /* We can use multiplication to find chunks greater than 1 */ if (cpu > 1) { unsigned long size; unsigned long p; /* Save the beginning of this CPU chunk */ p = ptr; ptr = rb_range_align_subbuf(ptr, subbuf_size, nr_subbufs); ptr += subbuf_size * nr_subbufs; /* Now all chunks after this are the same size */ size = ptr - p; ptr += size * (cpu - 2); } } return (void *)ptr;} /* Return the start of subbufs given the meta pointer */static void *rb_subbufs_from_meta(struct ring_buffer_cpu_meta *meta){ int subbuf_size = meta->subbuf_size; unsigned long ptr; ptr = (unsigned long)meta; ptr = rb_range_align_subbuf(ptr, subbuf_size, meta->nr_subbufs); return (void *)ptr;} /* * Return a specific sub-buffer for a given @cpu defined by @idx. */static void *rb_range_buffer(struct ring_buffer_per_cpu *cpu_buffer, int idx){ struct ring_buffer_cpu_meta *meta; unsigned long ptr; int subbuf_size; meta = rb_range_meta(cpu_buffer->buffer, 0, cpu_buffer->cpu); if (!meta) return NULL; if (WARN_ON_ONCE(idx >= meta->nr_subbufs)) return NULL; subbuf_size = meta->subbuf_size; /* Map this buffer to the order that's in meta->buffers[] */ idx = meta->buffers[idx]; ptr = (unsigned long)rb_subbufs_from_meta(meta); ptr += subbuf_size * idx; if (ptr + subbuf_size > cpu_buffer->buffer->range_addr_end) return NULL; return (void *)ptr;} /* * See if the existing memory contains a valid meta section. * if so, use that, otherwise initialize it. */static bool rb_meta_init(struct trace_buffer *buffer, int scratch_size){ unsigned long ptr = buffer->range_addr_start; struct ring_buffer_meta *bmeta; unsigned long total_size; int struct_sizes; bmeta = (struct ring_buffer_meta *)ptr; buffer->meta = bmeta; total_size = buffer->range_addr_end - buffer->range_addr_start; struct_sizes = sizeof(struct ring_buffer_cpu_meta); struct_sizes |= sizeof(*bmeta) << 16; /* The first buffer will start word size after the meta page */ ptr += sizeof(*bmeta); ptr = ALIGN(ptr, sizeof(long)); ptr += scratch_size; if (bmeta->magic != RING_BUFFER_META_MAGIC) { pr_info("Ring buffer boot meta mismatch of magic\n"); goto init; } if (bmeta->struct_sizes != struct_sizes) { pr_info("Ring buffer boot meta mismatch of struct size\n"); goto init; } if (bmeta->total_size != total_size) { pr_info("Ring buffer boot meta mismatch of total size\n"); goto init; } if (bmeta->buffers_offset > bmeta->total_size) { pr_info("Ring buffer boot meta mismatch of offset outside of total size\n"); goto init; } if (bmeta->buffers_offset != (void *)ptr - (void *)bmeta) { pr_info("Ring buffer boot meta mismatch of first buffer offset\n"); goto init; } return true; init: bmeta->magic = RING_BUFFER_META_MAGIC; bmeta->struct_sizes = struct_sizes; bmeta->total_size = total_size; bmeta->buffers_offset = (void *)ptr - (void *)bmeta; /* Zero out the scratch pad */ memset((void *)bmeta + sizeof(*bmeta), 0, bmeta->buffers_offset - sizeof(*bmeta)); return false;} /* * See if the existing memory contains valid ring buffer data. * As the previous kernel must be the same as this kernel, all * the calculations (size of buffers and number of buffers) * must be the same. */static bool rb_cpu_meta_valid(struct ring_buffer_cpu_meta *meta, int cpu, struct trace_buffer *buffer, int nr_pages, unsigned long *subbuf_mask){ int subbuf_size = PAGE_SIZE; unsigned long buffers_start; unsigned long buffers_end; int i; if (!subbuf_mask) return false; if (meta->subbuf_size != PAGE_SIZE) { pr_info("Ring buffer boot meta [%d] invalid subbuf_size\n", cpu); return false; } buffers_start = meta->first_buffer; buffers_end = meta->first_buffer + (subbuf_size * meta->nr_subbufs); /* Is the head and commit buffers within the range of buffers? */ if (meta->head_buffer < buffers_start || meta->head_buffer >= buffers_end) { pr_info("Ring buffer boot meta [%d] head buffer out of range\n", cpu); return false; } if (meta->commit_buffer < buffers_start || meta->commit_buffer >= buffers_end) { pr_info("Ring buffer boot meta [%d] commit buffer out of range\n", cpu); return false; } bitmap_clear(subbuf_mask, 0, meta->nr_subbufs); /* * Ensure the meta::buffers array has correct data. The data in each subbufs * are checked later in rb_meta_validate_events(). */ for (i = 0; i < meta->nr_subbufs; i++) { if (meta->buffers[i] < 0 || meta->buffers[i] >= meta->nr_subbufs) { pr_info("Ring buffer boot meta [%d] array out of range\n", cpu); return false; } if (test_bit(meta->buffers[i], subbuf_mask)) { pr_info("Ring buffer boot meta [%d] array has duplicates\n", cpu); return false; } set_bit(meta->buffers[i], subbuf_mask); } return true;} static int rb_meta_subbuf_idx(struct ring_buffer_cpu_meta *meta, void *subbuf); static int rb_read_data_buffer(struct buffer_data_page *dpage, int tail, int cpu, unsigned long long *timestamp, u64 *delta_ptr){ struct ring_buffer_event *event; u64 ts, delta; int events = 0; int len; int e; *delta_ptr = 0; *timestamp = 0; ts = dpage->time_stamp; for (e = 0; e < tail; e += len) { event = (struct ring_buffer_event *)(dpage->data + e); len = rb_event_length(event); if (len <= 0 || len > tail - e) return -1; switch (event->type_len) { case RINGBUF_TYPE_TIME_EXTEND: delta = rb_event_time_stamp(event); ts += delta; break; case RINGBUF_TYPE_TIME_STAMP: delta = rb_event_time_stamp(event); delta = rb_fix_abs_ts(delta, ts); if (delta < ts) { *delta_ptr = delta; *timestamp = ts; return -1; } ts = delta; break; case RINGBUF_TYPE_PADDING: if (event->time_delta == 1) break; fallthrough; case RINGBUF_TYPE_DATA: events++; ts += event->time_delta; break; default: return -1; } } *timestamp = ts; return events;} struct rb_validation_state { unsigned long entries; unsigned long entry_bytes; int discarded; u64 ts;}; static int __rb_validate_buffer(struct buffer_page *bpage, int cpu, struct ring_buffer_cpu_meta *meta, u64 prev_ts, u64 next_ts){ struct buffer_data_page *dpage = bpage->page; unsigned long long ts; unsigned long tail; u64 delta; int ret; /* * When a sub-buffer is recovered from a read, the commit value may * have RB_MISSED_* bits set, as these bits are reset on reuse. * Even after clearing these bits, a commit value greater than the * subbuf_size is considered invalid. */ tail = rb_data_page_commit(dpage); if (tail <= meta->subbuf_size - BUF_PAGE_HDR_SIZE) ret = rb_read_data_buffer(dpage, tail, cpu, &ts, &delta); else ret = -1; /* * The timestamp must be greater than @prev_ts and smaller than @next_ts. * Since this function works in both forward (verify) and reverse (unwind) * loop, we don't know both @prev_ts and @next_ts at the same time. * So use the known boundary as the boundary. */ if (ret < 0 || (prev_ts && prev_ts > ts) || (next_ts && ts > next_ts)) { local_set(&bpage->entries, 0); /* * Note, the RB_MISSED_EVENTS is only set inside the main write * buffer by this verification logic. The normal ring buffer * has this bit set when the page is read and passed to the * consumers. */ local_set(&dpage->commit, RB_MISSED_EVENTS); dpage->time_stamp = prev_ts ? prev_ts : next_ts; ret = -1; } else { local_set(&bpage->entries, ret); } return ret;} /** * rb_validate_buffer - validates a single buffer page and updates the state. * @bpage: buffer page to validate * @cpu_buffer: cpu_buffer this page belongs to * @meta: meta of the cpu_buffer * @state: validation state * @prev_ts: previous buffer's timestamp (optional) * @next_ts: next buffer's timestamp (optional) * * If the page is invalid (wrong event length or timestamp), it increments the * discarded counter and warns it. Otherwise, it updates the validation state. */static void rb_validate_buffer(struct buffer_page *bpage, struct ring_buffer_per_cpu *cpu_buffer, struct ring_buffer_cpu_meta *meta, struct rb_validation_state *state, u64 prev_ts, u64 next_ts){ int ret; ret = __rb_validate_buffer(bpage, cpu_buffer->cpu, meta, prev_ts, next_ts); if (ret < 0) { if (!state->discarded) pr_info("Ring buffer meta [%d] invalid buffer page detected\n", cpu_buffer->cpu); state->discarded++; } else { /* If the buffer has content, update pages_touched */ if (ret) local_inc(&cpu_buffer->pages_touched); state->entries += ret; state->entry_bytes += rb_page_size(bpage); state->ts = bpage->page->time_stamp; }} static void rb_meta_inject_reader_page(struct ring_buffer_per_cpu *cpu_buffer, struct ring_buffer_cpu_meta *meta, struct buffer_page *orig_head, struct buffer_page *head_page){ struct buffer_page *bpage = orig_head; int i; rb_dec_page(&bpage); /* * Insert the reader_page before the original head page. * Since the list encode RB_PAGE flags, general list * operations should be avoided. */ cpu_buffer->reader_page->list.next = &orig_head->list;Lines 1501–2000 of 8311. Previous lines Load more
* No need to worry about races with clearing out the commit. * it only can increment when a commit takes place. But that * only happens in the outer most nested commit. */ local_set(&next_page->page->commit, 0); /* Either we update tail_page or an interrupt does */ if (try_cmpxchg(&cpu_buffer->tail_page, &tail_page, next_page)) local_inc(&cpu_buffer->pages_touched); } } static void rb_check_bpage(struct ring_buffer_per_cpu *cpu_buffer, struct buffer_page *bpage) { unsigned long val = (unsigned long)bpage; RB_WARN_ON(cpu_buffer, val & RB_FLAG_MASK); } static bool rb_check_links(struct ring_buffer_per_cpu *cpu_buffer, struct list_head *list) { if (RB_WARN_ON(cpu_buffer, rb_list_head(rb_list_head(list->next)->prev) != list)) return false; if (RB_WARN_ON(cpu_buffer, rb_list_head(rb_list_head(list->prev)->next) != list)) return false; return true; } /** * rb_check_pages - integrity check of buffer pages * @cpu_buffer: CPU buffer with pages to test * * As a safety measure we check to make sure the data pages have not * been corrupted. */ static void rb_check_pages(struct ring_buffer_per_cpu *cpu_buffer) { struct list_head *head, *tmp; unsigned long buffer_cnt; unsigned long flags; int nr_loops = 0; /* * Walk the linked list underpinning the ring buffer and validate all * its next and prev links. * * The check acquires the reader_lock to avoid concurrent processing * with code that could be modifying the list. However, the lock cannot * be held for the entire duration of the walk, as this would make the * time when interrupts are disabled non-deterministic, dependent on the * ring buffer size. Therefore, the code releases and re-acquires the * lock after checking each page. The ring_buffer_per_cpu.cnt variable * is then used to detect if the list was modified while the lock was * not held, in which case the check needs to be restarted. * * The code attempts to perform the check at most three times before * giving up. This is acceptable because this is only a self-validation * to detect problems early on. In practice, the list modification * operations are fairly spaced, and so this check typically succeeds at * most on the second try. */ again: if (++nr_loops > 3) return; raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags); head = rb_list_head(cpu_buffer->pages); if (!rb_check_links(cpu_buffer, head)) goto out_locked; buffer_cnt = cpu_buffer->cnt; tmp = head; raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags); while (true) { raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags); if (buffer_cnt != cpu_buffer->cnt) { /* The list was updated, try again. */ raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags); goto again; } tmp = rb_list_head(tmp->next); if (tmp == head) /* The iteration circled back, all is done. */ goto out_locked; if (!rb_check_links(cpu_buffer, tmp)) goto out_locked; raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags); } out_locked: raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags); } /* * Take an address, add the meta data size as well as the array of * array subbuffer indexes, then align it to a subbuffer size. * * This is used to help find the next per cpu subbuffer within a mapped range. */ static unsigned long rb_range_align_subbuf(unsigned long addr, int subbuf_size, int nr_subbufs) { addr += sizeof(struct ring_buffer_cpu_meta) + sizeof(int) * nr_subbufs; return ALIGN(addr, subbuf_size); } /* * Return the ring_buffer_meta for a given @cpu. */ static void *rb_range_meta(struct trace_buffer *buffer, int nr_pages, int cpu) { int subbuf_size = buffer->subbuf_size + BUF_PAGE_HDR_SIZE; struct ring_buffer_cpu_meta *meta; struct ring_buffer_meta *bmeta; unsigned long ptr; int nr_subbufs; bmeta = buffer->meta; if (!bmeta) return NULL; ptr = (unsigned long)bmeta + bmeta->buffers_offset; meta = (struct ring_buffer_cpu_meta *)ptr; /* When nr_pages passed in is zero, the first meta has already been initialized */ if (!nr_pages) { nr_subbufs = meta->nr_subbufs; } else { /* Include the reader page */ nr_subbufs = nr_pages + 1; } /* * The first chunk may not be subbuffer aligned, where as * the rest of the chunks are. */ if (cpu) { ptr = rb_range_align_subbuf(ptr, subbuf_size, nr_subbufs); ptr += subbuf_size * nr_subbufs; /* We can use multiplication to find chunks greater than 1 */ if (cpu > 1) { unsigned long size; unsigned long p; /* Save the beginning of this CPU chunk */ p = ptr; ptr = rb_range_align_subbuf(ptr, subbuf_size, nr_subbufs); ptr += subbuf_size * nr_subbufs; /* Now all chunks after this are the same size */ size = ptr - p; ptr += size * (cpu - 2); } } return (void *)ptr; } /* Return the start of subbufs given the meta pointer */ static void *rb_subbufs_from_meta(struct ring_buffer_cpu_meta *meta) { int subbuf_size = meta->subbuf_size; unsigned long ptr; ptr = (unsigned long)meta; ptr = rb_range_align_subbuf(ptr, subbuf_size, meta->nr_subbufs); return (void *)ptr; } /* * Return a specific sub-buffer for a given @cpu defined by @idx. */ static void *rb_range_buffer(struct ring_buffer_per_cpu *cpu_buffer, int idx) { struct ring_buffer_cpu_meta *meta; unsigned long ptr; int subbuf_size; meta = rb_range_meta(cpu_buffer->buffer, 0, cpu_buffer->cpu); if (!meta) return NULL; if (WARN_ON_ONCE(idx >= meta->nr_subbufs)) return NULL; subbuf_size = meta->subbuf_size; /* Map this buffer to the order that's in meta->buffers[] */ idx = meta->buffers[idx]; ptr = (unsigned long)rb_subbufs_from_meta(meta); ptr += subbuf_size * idx; if (ptr + subbuf_size > cpu_buffer->buffer->range_addr_end) return NULL; return (void *)ptr; } /* * See if the existing memory contains a valid meta section. * if so, use that, otherwise initialize it. */ static bool rb_meta_init(struct trace_buffer *buffer, int scratch_size) { unsigned long ptr = buffer->range_addr_start; struct ring_buffer_meta *bmeta; unsigned long total_size; int struct_sizes; bmeta = (struct ring_buffer_meta *)ptr; buffer->meta = bmeta; total_size = buffer->range_addr_end - buffer->range_addr_start; struct_sizes = sizeof(struct ring_buffer_cpu_meta); struct_sizes |= sizeof(*bmeta) << 16; /* The first buffer will start word size after the meta page */ ptr += sizeof(*bmeta); ptr = ALIGN(ptr, sizeof(long)); ptr += scratch_size; if (bmeta->magic != RING_BUFFER_META_MAGIC) { pr_info("Ring buffer boot meta mismatch of magic\n"); goto init; } if (bmeta->struct_sizes != struct_sizes) { pr_info("Ring buffer boot meta mismatch of struct size\n"); goto init; } if (bmeta->total_size != total_size) { pr_info("Ring buffer boot meta mismatch of total size\n"); goto init; } if (bmeta->buffers_offset > bmeta->total_size) { pr_info("Ring buffer boot meta mismatch of offset outside of total size\n"); goto init; } if (bmeta->buffers_offset != (void *)ptr - (void *)bmeta) { pr_info("Ring buffer boot meta mismatch of first buffer offset\n"); goto init; } return true; init: bmeta->magic = RING_BUFFER_META_MAGIC; bmeta->struct_sizes = struct_sizes; bmeta->total_size = total_size; bmeta->buffers_offset = (void *)ptr - (void *)bmeta; /* Zero out the scratch pad */ memset((void *)bmeta + sizeof(*bmeta), 0, bmeta->buffers_offset - sizeof(*bmeta)); return false; } /* * See if the existing memory contains valid ring buffer data. * As the previous kernel must be the same as this kernel, all * the calculations (size of buffers and number of buffers) * must be the same. */ static bool rb_cpu_meta_valid(struct ring_buffer_cpu_meta *meta, int cpu, struct trace_buffer *buffer, int nr_pages, unsigned long *subbuf_mask) { int subbuf_size = PAGE_SIZE; unsigned long buffers_start; unsigned long buffers_end; int i; if (!subbuf_mask) return false; if (meta->subbuf_size != PAGE_SIZE) { pr_info("Ring buffer boot meta [%d] invalid subbuf_size\n", cpu); return false; } buffers_start = meta->first_buffer; buffers_end = meta->first_buffer + (subbuf_size * meta->nr_subbufs); /* Is the head and commit buffers within the range of buffers? */ if (meta->head_buffer < buffers_start || meta->head_buffer >= buffers_end) { pr_info("Ring buffer boot meta [%d] head buffer out of range\n", cpu); return false; } if (meta->commit_buffer < buffers_start || meta->commit_buffer >= buffers_end) { pr_info("Ring buffer boot meta [%d] commit buffer out of range\n", cpu); return false; } bitmap_clear(subbuf_mask, 0, meta->nr_subbufs); /* * Ensure the meta::buffers array has correct data. The data in each subbufs * are checked later in rb_meta_validate_events(). */ for (i = 0; i < meta->nr_subbufs; i++) { if (meta->buffers[i] < 0 || meta->buffers[i] >= meta->nr_subbufs) { pr_info("Ring buffer boot meta [%d] array out of range\n", cpu); return false; } if (test_bit(meta->buffers[i], subbuf_mask)) { pr_info("Ring buffer boot meta [%d] array has duplicates\n", cpu); return false; } set_bit(meta->buffers[i], subbuf_mask); } return true; } static int rb_meta_subbuf_idx(struct ring_buffer_cpu_meta *meta, void *subbuf); static int rb_read_data_buffer(struct buffer_data_page *dpage, int tail, int cpu, unsigned long long *timestamp, u64 *delta_ptr) { struct ring_buffer_event *event; u64 ts, delta; int events = 0; int len; int e; *delta_ptr = 0; *timestamp = 0; ts = dpage->time_stamp; for (e = 0; e < tail; e += len) { event = (struct ring_buffer_event *)(dpage->data + e); len = rb_event_length(event); if (len <= 0 || len > tail - e) return -1; switch (event->type_len) { case RINGBUF_TYPE_TIME_EXTEND: delta = rb_event_time_stamp(event); ts += delta; break; case RINGBUF_TYPE_TIME_STAMP: delta = rb_event_time_stamp(event); delta = rb_fix_abs_ts(delta, ts); if (delta < ts) { *delta_ptr = delta; *timestamp = ts; return -1; } ts = delta; break; case RINGBUF_TYPE_PADDING: if (event->time_delta == 1) break; fallthrough; case RINGBUF_TYPE_DATA: events++; ts += event->time_delta; break; default: return -1; } } *timestamp = ts; return events; } struct rb_validation_state { unsigned long entries; unsigned long entry_bytes; int discarded; u64 ts; }; static int __rb_validate_buffer(struct buffer_page *bpage, int cpu, struct ring_buffer_cpu_meta *meta, u64 prev_ts, u64 next_ts) { struct buffer_data_page *dpage = bpage->page; unsigned long long ts; unsigned long tail; u64 delta; int ret; /* * When a sub-buffer is recovered from a read, the commit value may * have RB_MISSED_* bits set, as these bits are reset on reuse. * Even after clearing these bits, a commit value greater than the * subbuf_size is considered invalid. */ tail = rb_data_page_commit(dpage); if (tail <= meta->subbuf_size - BUF_PAGE_HDR_SIZE) ret = rb_read_data_buffer(dpage, tail, cpu, &ts, &delta); else ret = -1; /* * The timestamp must be greater than @prev_ts and smaller than @next_ts. * Since this function works in both forward (verify) and reverse (unwind) * loop, we don't know both @prev_ts and @next_ts at the same time. * So use the known boundary as the boundary. */ if (ret < 0 || (prev_ts && prev_ts > ts) || (next_ts && ts > next_ts)) { local_set(&bpage->entries, 0); /* * Note, the RB_MISSED_EVENTS is only set inside the main write * buffer by this verification logic. The normal ring buffer * has this bit set when the page is read and passed to the * consumers. */ local_set(&dpage->commit, RB_MISSED_EVENTS); dpage->time_stamp = prev_ts ? prev_ts : next_ts; ret = -1; } else { local_set(&bpage->entries, ret); } return ret; } /** * rb_validate_buffer - validates a single buffer page and updates the state. * @bpage: buffer page to validate * @cpu_buffer: cpu_buffer this page belongs to * @meta: meta of the cpu_buffer * @state: validation state * @prev_ts: previous buffer's timestamp (optional) * @next_ts: next buffer's timestamp (optional) * * If the page is invalid (wrong event length or timestamp), it increments the * discarded counter and warns it. Otherwise, it updates the validation state. */ static void rb_validate_buffer(struct buffer_page *bpage, struct ring_buffer_per_cpu *cpu_buffer, struct ring_buffer_cpu_meta *meta, struct rb_validation_state *state, u64 prev_ts, u64 next_ts) { int ret; ret = __rb_validate_buffer(bpage, cpu_buffer->cpu, meta, prev_ts, next_ts); if (ret < 0) { if (!state->discarded) pr_info("Ring buffer meta [%d] invalid buffer page detected\n", cpu_buffer->cpu); state->discarded++; } else { /* If the buffer has content, update pages_touched */ if (ret) local_inc(&cpu_buffer->pages_touched); state->entries += ret; state->entry_bytes += rb_page_size(bpage); state->ts = bpage->page->time_stamp; } } static void rb_meta_inject_reader_page(struct ring_buffer_per_cpu *cpu_buffer, struct ring_buffer_cpu_meta *meta, struct buffer_page *orig_head, struct buffer_page *head_page) { struct buffer_page *bpage = orig_head; int i; rb_dec_page(&bpage); /* * Insert the reader_page before the original head page. * Since the list encode RB_PAGE flags, general list * operations should be avoided. */ cpu_buffer->reader_page->list.next = &orig_head->list;