cpu_buffer->reader_page->list.prev = orig_head->list.prev; orig_head->list.prev = &cpu_buffer->reader_page->list; bpage->list.next = &cpu_buffer->reader_page->list; /* Make the head_page the reader page */ cpu_buffer->reader_page = head_page; bpage = head_page; rb_inc_page(&head_page); head_page->list.prev = bpage->list.prev; rb_dec_page(&bpage); bpage->list.next = &head_page->list; rb_set_list_to_head(&bpage->list); cpu_buffer->pages = &head_page->list; cpu_buffer->head_page = head_page; meta->head_buffer = (unsigned long)head_page->page; /* Reset all the indexes */ bpage = cpu_buffer->reader_page; meta->buffers[0] = rb_meta_subbuf_idx(meta, bpage->page); bpage->id = 0; for (i = 1, bpage = head_page; i < meta->nr_subbufs; i++, rb_inc_page(&bpage)) { meta->buffers[i] = rb_meta_subbuf_idx(meta, bpage->page); bpage->id = i; }} /* If the meta data has been validated, now validate the events */static void rb_meta_validate_events(struct ring_buffer_per_cpu *cpu_buffer){ struct ring_buffer_cpu_meta *meta = cpu_buffer->ring_meta; struct buffer_page *head_page, *orig_head, *orig_reader; struct rb_validation_state state = { 0 }; bool skip = false; int ret; int i; if (!meta || !meta->head_buffer) return; orig_head = head_page = cpu_buffer->head_page; orig_reader = cpu_buffer->reader_page; /* Do the head page first */ ret = __rb_validate_buffer(head_page, cpu_buffer->cpu, meta, 0, 0); if (ret < 0) { pr_info("Ring buffer meta [%d] invalid head page detected\n", cpu_buffer->cpu); /* Don't bother rewinding */ skip = true; state.ts = 0; } else { state.ts = head_page->page->time_stamp; } /* Do the reader page - reader must be previous to head. */ rb_validate_buffer(orig_reader, cpu_buffer, meta, &state, 0, state.ts); if (skip) goto skip_rewind; /* * Try to rewind the head so that we can read the pages which are already * read in the previous boot. */ if (head_page == cpu_buffer->tail_page) goto skip_rewind; rb_dec_page(&head_page); for (i = 0; i < meta->nr_subbufs + 1; i++, rb_dec_page(&head_page)) { /* Rewind until tail (writer) page. */ if (head_page == cpu_buffer->tail_page) break; /* Rewind until unused page (no timestamp, no commit). */ if (!head_page->page->time_stamp && rb_page_commit(head_page) == 0) break; /* * Skip if the page is invalid, or its timestamp is newer than the * previous valid page. */ rb_validate_buffer(head_page, cpu_buffer, meta, &state, 0, state.ts); } if (i) pr_info("Ring buffer [%d] rewound %d pages\n", cpu_buffer->cpu, i); /* The last rewound page must be skipped. */ if (head_page != orig_head) rb_inc_page(&head_page); /* * If the ring buffer was rewound, then inject the reader page * into the location just before the original head page. */ if (head_page != orig_head) { rb_meta_inject_reader_page(cpu_buffer, meta, orig_head, head_page); /* We'll restart verifying from orig_head */ head_page = orig_head; } skip_rewind: /* If the commit_buffer is the reader page, update the commit page */ if (meta->commit_buffer == (unsigned long)cpu_buffer->reader_page->page) { cpu_buffer->commit_page = cpu_buffer->reader_page; /* Nothing more to do, the only page is the reader page */ goto done; } state.ts = head_page->page->time_stamp; /* Iterate until finding the commit page */ for (i = 0; i < meta->nr_subbufs + 1; i++, rb_inc_page(&head_page)) { /* The original reader page has already been checked/counted. */ if (head_page == orig_reader) continue; rb_validate_buffer(head_page, cpu_buffer, meta, &state, state.ts, 0); if (head_page == cpu_buffer->commit_page) break; } if (head_page != cpu_buffer->commit_page) { pr_info("Ring buffer meta [%d] commit page not found\n", cpu_buffer->cpu); goto invalid; } done: local_set(&cpu_buffer->entries, state.entries); local_set(&cpu_buffer->entries_bytes, state.entry_bytes); pr_info("Ring buffer meta [%d] is from previous boot!", cpu_buffer->cpu); if (state.discarded) pr_cont(" (%d pages discarded)", state.discarded); pr_cont("\n"); #ifdef CONFIG_RING_BUFFER_PERSISTENT_INJECT if (meta->nr_invalid) pr_warn("Ring buffer testing [%d] invalid pages: %s (%d/%d)\n", cpu_buffer->cpu, (state.discarded == meta->nr_invalid) ? "PASSED" : "FAILED", state.discarded, meta->nr_invalid); if (meta->entry_bytes) pr_warn("Ring buffer testing [%d] entry_bytes: %s (%ld/%ld)\n", cpu_buffer->cpu, (state.entry_bytes == meta->entry_bytes) ? "PASSED" : "FAILED", (long)state.entry_bytes, (long)meta->entry_bytes); meta->nr_invalid = 0; meta->entry_bytes = 0;#endif return; invalid: /* The content of the buffers are invalid, reset the meta data */ meta->head_buffer = 0; meta->commit_buffer = 0; /* Reset the reader page */ local_set(&cpu_buffer->reader_page->entries, 0); rb_init_data_page(cpu_buffer->reader_page->page); /* Reset all the subbuffers */ for (i = 0; i < meta->nr_subbufs - 1; i++, rb_inc_page(&head_page)) { local_set(&head_page->entries, 0); rb_init_data_page(head_page->page); }} static void rb_range_meta_init(struct trace_buffer *buffer, int nr_pages, int scratch_size){ struct ring_buffer_cpu_meta *meta; unsigned long *subbuf_mask; unsigned long delta; void *subbuf; bool valid = false; int cpu; int i; /* Create a mask to test the subbuf array */ subbuf_mask = bitmap_alloc(nr_pages + 1, GFP_KERNEL); /* If subbuf_mask fails to allocate, then rb_meta_valid() will return false */ if (rb_meta_init(buffer, scratch_size)) valid = true; for (cpu = 0; cpu < nr_cpu_ids; cpu++) { void *next_meta; meta = rb_range_meta(buffer, nr_pages, cpu); if (valid && rb_cpu_meta_valid(meta, cpu, buffer, nr_pages, subbuf_mask)) { /* Make the mappings match the current address */ subbuf = rb_subbufs_from_meta(meta); delta = (unsigned long)subbuf - meta->first_buffer; meta->first_buffer += delta; meta->head_buffer += delta; meta->commit_buffer += delta; continue; } if (cpu < nr_cpu_ids - 1) next_meta = rb_range_meta(buffer, nr_pages, cpu + 1); else next_meta = (void *)buffer->range_addr_end; memset(meta, 0, next_meta - (void *)meta); meta->nr_subbufs = nr_pages + 1; meta->subbuf_size = PAGE_SIZE; subbuf = rb_subbufs_from_meta(meta); meta->first_buffer = (unsigned long)subbuf; /* * The buffers[] array holds the order of the sub-buffers * that are after the meta data. The sub-buffers may * be swapped out when read and inserted into a different * location of the ring buffer. Although their addresses * remain the same, the buffers[] array contains the * index into the sub-buffers holding their actual order. */ for (i = 0; i < meta->nr_subbufs; i++) { meta->buffers[i] = i; rb_init_data_page(subbuf); subbuf += meta->subbuf_size; } } bitmap_free(subbuf_mask);} static void *rbm_start(struct seq_file *m, loff_t *pos){ struct ring_buffer_per_cpu *cpu_buffer = m->private; struct ring_buffer_cpu_meta *meta = cpu_buffer->ring_meta; unsigned long val; if (!meta) return NULL; if (*pos > meta->nr_subbufs) return NULL; val = *pos; val++; return (void *)val;} static void *rbm_next(struct seq_file *m, void *v, loff_t *pos){ (*pos)++; return rbm_start(m, pos);} static int rbm_show(struct seq_file *m, void *v){ struct ring_buffer_per_cpu *cpu_buffer = m->private; struct ring_buffer_cpu_meta *meta = cpu_buffer->ring_meta; unsigned long val = (unsigned long)v; struct buffer_data_page *dpage; if (val == 1) { seq_printf(m, "head_buffer: %d\n", rb_meta_subbuf_idx(meta, (void *)meta->head_buffer)); seq_printf(m, "commit_buffer: %d\n", rb_meta_subbuf_idx(meta, (void *)meta->commit_buffer)); seq_printf(m, "subbuf_size: %d\n", meta->subbuf_size); seq_printf(m, "nr_subbufs: %d\n", meta->nr_subbufs); return 0; } val -= 2; dpage = rb_range_buffer(cpu_buffer, val); seq_printf(m, "buffer[%ld]: %d (commit: %ld)\n", val, meta->buffers[val], dpage ? rb_data_page_commit(dpage) : -1); return 0;} static void rbm_stop(struct seq_file *m, void *p){} static const struct seq_operations rb_meta_seq_ops = { .start = rbm_start, .next = rbm_next, .show = rbm_show, .stop = rbm_stop,}; int ring_buffer_meta_seq_init(struct file *file, struct trace_buffer *buffer, int cpu){ struct seq_file *m; int ret; ret = seq_open(file, &rb_meta_seq_ops); if (ret) return ret; m = file->private_data; m->private = buffer->buffers[cpu]; return 0;} /* Map the buffer_pages to the previous head and commit pages */static void rb_meta_buffer_update(struct ring_buffer_per_cpu *cpu_buffer, struct buffer_page *bpage){ struct ring_buffer_cpu_meta *meta = cpu_buffer->ring_meta; if (meta->head_buffer == (unsigned long)bpage->page) cpu_buffer->head_page = bpage; if (meta->commit_buffer == (unsigned long)bpage->page) { cpu_buffer->commit_page = bpage; cpu_buffer->tail_page = bpage; }} static struct ring_buffer_desc *ring_buffer_desc(struct trace_buffer_desc *trace_desc, int cpu){ struct ring_buffer_desc *desc, *end; size_t len; int i; if (!trace_desc || !trace_desc->nr_cpus) return NULL; end = (struct ring_buffer_desc *)((void *)trace_desc + trace_desc->struct_len); desc = __first_ring_buffer_desc(trace_desc); len = struct_size(desc, page_va, desc->nr_page_va); desc = (struct ring_buffer_desc *)((void *)desc + (len * cpu)); if (desc < end && desc->cpu == cpu) return desc; /* Missing CPUs, need to linear search */ for_each_ring_buffer_desc(desc, i, trace_desc) { if (desc->cpu == cpu) return desc; } return NULL;} static void *ring_buffer_desc_page(struct ring_buffer_desc *desc, unsigned int page_id){ return page_id >= desc->nr_page_va ? NULL : (void *)desc->page_va[page_id];} static int __rb_allocate_pages(struct ring_buffer_per_cpu *cpu_buffer, long nr_pages, struct list_head *pages){ struct trace_buffer *buffer = cpu_buffer->buffer; struct ring_buffer_cpu_meta *meta = NULL; struct buffer_page *bpage, *tmp; bool user_thread = current->mm != NULL; struct ring_buffer_desc *desc = NULL; long i; /* * Check if the available memory is there first. * Note, si_mem_available() only gives us a rough estimate of available * memory. It may not be accurate. But we don't care, we just want * to prevent doing any allocation when it is obvious that it is * not going to succeed. */ i = si_mem_available(); if (i < nr_pages) return -ENOMEM; /* * If a user thread allocates too much, and si_mem_available() * reports there's enough memory, even though there is not. * Make sure the OOM killer kills this thread. This can happen * even with RETRY_MAYFAIL because another task may be doing * an allocation after this task has taken all memory. * This is the task the OOM killer needs to take out during this * loop, even if it was triggered by an allocation somewhere else. */ if (user_thread) set_current_oom_origin(); if (buffer->range_addr_start) meta = rb_range_meta(buffer, nr_pages, cpu_buffer->cpu); if (buffer->remote) { desc = ring_buffer_desc(buffer->remote->desc, cpu_buffer->cpu); if (!desc || WARN_ON(desc->nr_page_va != (nr_pages + 1))) return -EINVAL; } for (i = 0; i < nr_pages; i++) { bpage = alloc_cpu_page(cpu_buffer->cpu); if (!bpage) goto free_pages; rb_check_bpage(cpu_buffer, bpage); /* * Append the pages as for mapped buffers we want to keep * the order */ list_add_tail(&bpage->list, pages); if (meta) { /* A range was given. Use that for the buffer page */ bpage->page = rb_range_buffer(cpu_buffer, i + 1); if (!bpage->page) goto free_pages; /* If this is valid from a previous boot */ if (meta->head_buffer) rb_meta_buffer_update(cpu_buffer, bpage); bpage->range = 1; bpage->id = i + 1; } else if (desc) { void *p = ring_buffer_desc_page(desc, i + 1); if (WARN_ON(!p)) goto free_pages; bpage->page = p; bpage->range = 1; /* bpage->page can't be freed */ bpage->id = i + 1; cpu_buffer->subbuf_ids[i + 1] = bpage; } else { int order = cpu_buffer->buffer->subbuf_order; bpage->page = alloc_cpu_data(cpu_buffer->cpu, order); if (!bpage->page) goto free_pages; } bpage->order = cpu_buffer->buffer->subbuf_order; if (user_thread && fatal_signal_pending(current)) goto free_pages; } if (user_thread) clear_current_oom_origin(); return 0; free_pages: list_for_each_entry_safe(bpage, tmp, pages, list) { list_del_init(&bpage->list); free_buffer_page(bpage); } if (user_thread) clear_current_oom_origin(); return -ENOMEM;} static int rb_allocate_pages(struct ring_buffer_per_cpu *cpu_buffer, unsigned long nr_pages){ LIST_HEAD(pages); WARN_ON(!nr_pages); if (__rb_allocate_pages(cpu_buffer, nr_pages, &pages)) return -ENOMEM; /* * The ring buffer page list is a circular list that does not * start and end with a list head. All page list items point to * other pages. */ cpu_buffer->pages = pages.next; list_del(&pages); cpu_buffer->nr_pages = nr_pages; rb_check_pages(cpu_buffer); return 0;} static struct ring_buffer_per_cpu *rb_allocate_cpu_buffer(struct trace_buffer *buffer, long nr_pages, int cpu){ struct ring_buffer_per_cpu *cpu_buffer __free(kfree) = alloc_cpu_buffer(cpu); struct ring_buffer_cpu_meta *meta; struct buffer_page *bpage; int ret; if (!cpu_buffer) return NULL; cpu_buffer->cpu = cpu; cpu_buffer->buffer = buffer; raw_spin_lock_init(&cpu_buffer->reader_lock);Lines 2001–2500 of 8311. Previous lines Load more
cpu_buffer->reader_page->list.prev = orig_head->list.prev; orig_head->list.prev = &cpu_buffer->reader_page->list; bpage->list.next = &cpu_buffer->reader_page->list; /* Make the head_page the reader page */ cpu_buffer->reader_page = head_page; bpage = head_page; rb_inc_page(&head_page); head_page->list.prev = bpage->list.prev; rb_dec_page(&bpage); bpage->list.next = &head_page->list; rb_set_list_to_head(&bpage->list); cpu_buffer->pages = &head_page->list; cpu_buffer->head_page = head_page; meta->head_buffer = (unsigned long)head_page->page; /* Reset all the indexes */ bpage = cpu_buffer->reader_page; meta->buffers[0] = rb_meta_subbuf_idx(meta, bpage->page); bpage->id = 0; for (i = 1, bpage = head_page; i < meta->nr_subbufs; i++, rb_inc_page(&bpage)) { meta->buffers[i] = rb_meta_subbuf_idx(meta, bpage->page); bpage->id = i; } } /* If the meta data has been validated, now validate the events */ static void rb_meta_validate_events(struct ring_buffer_per_cpu *cpu_buffer) { struct ring_buffer_cpu_meta *meta = cpu_buffer->ring_meta; struct buffer_page *head_page, *orig_head, *orig_reader; struct rb_validation_state state = { 0 }; bool skip = false; int ret; int i; if (!meta || !meta->head_buffer) return; orig_head = head_page = cpu_buffer->head_page; orig_reader = cpu_buffer->reader_page; /* Do the head page first */ ret = __rb_validate_buffer(head_page, cpu_buffer->cpu, meta, 0, 0); if (ret < 0) { pr_info("Ring buffer meta [%d] invalid head page detected\n", cpu_buffer->cpu); /* Don't bother rewinding */ skip = true; state.ts = 0; } else { state.ts = head_page->page->time_stamp; } /* Do the reader page - reader must be previous to head. */ rb_validate_buffer(orig_reader, cpu_buffer, meta, &state, 0, state.ts); if (skip) goto skip_rewind; /* * Try to rewind the head so that we can read the pages which are already * read in the previous boot. */ if (head_page == cpu_buffer->tail_page) goto skip_rewind; rb_dec_page(&head_page); for (i = 0; i < meta->nr_subbufs + 1; i++, rb_dec_page(&head_page)) { /* Rewind until tail (writer) page. */ if (head_page == cpu_buffer->tail_page) break; /* Rewind until unused page (no timestamp, no commit). */ if (!head_page->page->time_stamp && rb_page_commit(head_page) == 0) break; /* * Skip if the page is invalid, or its timestamp is newer than the * previous valid page. */ rb_validate_buffer(head_page, cpu_buffer, meta, &state, 0, state.ts); } if (i) pr_info("Ring buffer [%d] rewound %d pages\n", cpu_buffer->cpu, i); /* The last rewound page must be skipped. */ if (head_page != orig_head) rb_inc_page(&head_page); /* * If the ring buffer was rewound, then inject the reader page * into the location just before the original head page. */ if (head_page != orig_head) { rb_meta_inject_reader_page(cpu_buffer, meta, orig_head, head_page); /* We'll restart verifying from orig_head */ head_page = orig_head; } skip_rewind: /* If the commit_buffer is the reader page, update the commit page */ if (meta->commit_buffer == (unsigned long)cpu_buffer->reader_page->page) { cpu_buffer->commit_page = cpu_buffer->reader_page; /* Nothing more to do, the only page is the reader page */ goto done; } state.ts = head_page->page->time_stamp; /* Iterate until finding the commit page */ for (i = 0; i < meta->nr_subbufs + 1; i++, rb_inc_page(&head_page)) { /* The original reader page has already been checked/counted. */ if (head_page == orig_reader) continue; rb_validate_buffer(head_page, cpu_buffer, meta, &state, state.ts, 0); if (head_page == cpu_buffer->commit_page) break; } if (head_page != cpu_buffer->commit_page) { pr_info("Ring buffer meta [%d] commit page not found\n", cpu_buffer->cpu); goto invalid; } done: local_set(&cpu_buffer->entries, state.entries); local_set(&cpu_buffer->entries_bytes, state.entry_bytes); pr_info("Ring buffer meta [%d] is from previous boot!", cpu_buffer->cpu); if (state.discarded) pr_cont(" (%d pages discarded)", state.discarded); pr_cont("\n"); #ifdef CONFIG_RING_BUFFER_PERSISTENT_INJECT if (meta->nr_invalid) pr_warn("Ring buffer testing [%d] invalid pages: %s (%d/%d)\n", cpu_buffer->cpu, (state.discarded == meta->nr_invalid) ? "PASSED" : "FAILED", state.discarded, meta->nr_invalid); if (meta->entry_bytes) pr_warn("Ring buffer testing [%d] entry_bytes: %s (%ld/%ld)\n", cpu_buffer->cpu, (state.entry_bytes == meta->entry_bytes) ? "PASSED" : "FAILED", (long)state.entry_bytes, (long)meta->entry_bytes); meta->nr_invalid = 0; meta->entry_bytes = 0; #endif return; invalid: /* The content of the buffers are invalid, reset the meta data */ meta->head_buffer = 0; meta->commit_buffer = 0; /* Reset the reader page */ local_set(&cpu_buffer->reader_page->entries, 0); rb_init_data_page(cpu_buffer->reader_page->page); /* Reset all the subbuffers */ for (i = 0; i < meta->nr_subbufs - 1; i++, rb_inc_page(&head_page)) { local_set(&head_page->entries, 0); rb_init_data_page(head_page->page); } } static void rb_range_meta_init(struct trace_buffer *buffer, int nr_pages, int scratch_size) { struct ring_buffer_cpu_meta *meta; unsigned long *subbuf_mask; unsigned long delta; void *subbuf; bool valid = false; int cpu; int i; /* Create a mask to test the subbuf array */ subbuf_mask = bitmap_alloc(nr_pages + 1, GFP_KERNEL); /* If subbuf_mask fails to allocate, then rb_meta_valid() will return false */ if (rb_meta_init(buffer, scratch_size)) valid = true; for (cpu = 0; cpu < nr_cpu_ids; cpu++) { void *next_meta; meta = rb_range_meta(buffer, nr_pages, cpu); if (valid && rb_cpu_meta_valid(meta, cpu, buffer, nr_pages, subbuf_mask)) { /* Make the mappings match the current address */ subbuf = rb_subbufs_from_meta(meta); delta = (unsigned long)subbuf - meta->first_buffer; meta->first_buffer += delta; meta->head_buffer += delta; meta->commit_buffer += delta; continue; } if (cpu < nr_cpu_ids - 1) next_meta = rb_range_meta(buffer, nr_pages, cpu + 1); else next_meta = (void *)buffer->range_addr_end; memset(meta, 0, next_meta - (void *)meta); meta->nr_subbufs = nr_pages + 1; meta->subbuf_size = PAGE_SIZE; subbuf = rb_subbufs_from_meta(meta); meta->first_buffer = (unsigned long)subbuf; /* * The buffers[] array holds the order of the sub-buffers * that are after the meta data. The sub-buffers may * be swapped out when read and inserted into a different * location of the ring buffer. Although their addresses * remain the same, the buffers[] array contains the * index into the sub-buffers holding their actual order. */ for (i = 0; i < meta->nr_subbufs; i++) { meta->buffers[i] = i; rb_init_data_page(subbuf); subbuf += meta->subbuf_size; } } bitmap_free(subbuf_mask); } static void *rbm_start(struct seq_file *m, loff_t *pos) { struct ring_buffer_per_cpu *cpu_buffer = m->private; struct ring_buffer_cpu_meta *meta = cpu_buffer->ring_meta; unsigned long val; if (!meta) return NULL; if (*pos > meta->nr_subbufs) return NULL; val = *pos; val++; return (void *)val; } static void *rbm_next(struct seq_file *m, void *v, loff_t *pos) { (*pos)++; return rbm_start(m, pos); } static int rbm_show(struct seq_file *m, void *v) { struct ring_buffer_per_cpu *cpu_buffer = m->private; struct ring_buffer_cpu_meta *meta = cpu_buffer->ring_meta; unsigned long val = (unsigned long)v; struct buffer_data_page *dpage; if (val == 1) { seq_printf(m, "head_buffer: %d\n", rb_meta_subbuf_idx(meta, (void *)meta->head_buffer)); seq_printf(m, "commit_buffer: %d\n", rb_meta_subbuf_idx(meta, (void *)meta->commit_buffer)); seq_printf(m, "subbuf_size: %d\n", meta->subbuf_size); seq_printf(m, "nr_subbufs: %d\n", meta->nr_subbufs); return 0; } val -= 2; dpage = rb_range_buffer(cpu_buffer, val); seq_printf(m, "buffer[%ld]: %d (commit: %ld)\n", val, meta->buffers[val], dpage ? rb_data_page_commit(dpage) : -1); return 0; } static void rbm_stop(struct seq_file *m, void *p) { } static const struct seq_operations rb_meta_seq_ops = { .start = rbm_start, .next = rbm_next, .show = rbm_show, .stop = rbm_stop, }; int ring_buffer_meta_seq_init(struct file *file, struct trace_buffer *buffer, int cpu) { struct seq_file *m; int ret; ret = seq_open(file, &rb_meta_seq_ops); if (ret) return ret; m = file->private_data; m->private = buffer->buffers[cpu]; return 0; } /* Map the buffer_pages to the previous head and commit pages */ static void rb_meta_buffer_update(struct ring_buffer_per_cpu *cpu_buffer, struct buffer_page *bpage) { struct ring_buffer_cpu_meta *meta = cpu_buffer->ring_meta; if (meta->head_buffer == (unsigned long)bpage->page) cpu_buffer->head_page = bpage; if (meta->commit_buffer == (unsigned long)bpage->page) { cpu_buffer->commit_page = bpage; cpu_buffer->tail_page = bpage; } } static struct ring_buffer_desc *ring_buffer_desc(struct trace_buffer_desc *trace_desc, int cpu) { struct ring_buffer_desc *desc, *end; size_t len; int i; if (!trace_desc || !trace_desc->nr_cpus) return NULL; end = (struct ring_buffer_desc *)((void *)trace_desc + trace_desc->struct_len); desc = __first_ring_buffer_desc(trace_desc); len = struct_size(desc, page_va, desc->nr_page_va); desc = (struct ring_buffer_desc *)((void *)desc + (len * cpu)); if (desc < end && desc->cpu == cpu) return desc; /* Missing CPUs, need to linear search */ for_each_ring_buffer_desc(desc, i, trace_desc) { if (desc->cpu == cpu) return desc; } return NULL; } static void *ring_buffer_desc_page(struct ring_buffer_desc *desc, unsigned int page_id) { return page_id >= desc->nr_page_va ? NULL : (void *)desc->page_va[page_id]; } static int __rb_allocate_pages(struct ring_buffer_per_cpu *cpu_buffer, long nr_pages, struct list_head *pages) { struct trace_buffer *buffer = cpu_buffer->buffer; struct ring_buffer_cpu_meta *meta = NULL; struct buffer_page *bpage, *tmp; bool user_thread = current->mm != NULL; struct ring_buffer_desc *desc = NULL; long i; /* * Check if the available memory is there first. * Note, si_mem_available() only gives us a rough estimate of available * memory. It may not be accurate. But we don't care, we just want * to prevent doing any allocation when it is obvious that it is * not going to succeed. */ i = si_mem_available(); if (i < nr_pages) return -ENOMEM; /* * If a user thread allocates too much, and si_mem_available() * reports there's enough memory, even though there is not. * Make sure the OOM killer kills this thread. This can happen * even with RETRY_MAYFAIL because another task may be doing * an allocation after this task has taken all memory. * This is the task the OOM killer needs to take out during this * loop, even if it was triggered by an allocation somewhere else. */ if (user_thread) set_current_oom_origin(); if (buffer->range_addr_start) meta = rb_range_meta(buffer, nr_pages, cpu_buffer->cpu); if (buffer->remote) { desc = ring_buffer_desc(buffer->remote->desc, cpu_buffer->cpu); if (!desc || WARN_ON(desc->nr_page_va != (nr_pages + 1))) return -EINVAL; } for (i = 0; i < nr_pages; i++) { bpage = alloc_cpu_page(cpu_buffer->cpu); if (!bpage) goto free_pages; rb_check_bpage(cpu_buffer, bpage); /* * Append the pages as for mapped buffers we want to keep * the order */ list_add_tail(&bpage->list, pages); if (meta) { /* A range was given. Use that for the buffer page */ bpage->page = rb_range_buffer(cpu_buffer, i + 1); if (!bpage->page) goto free_pages; /* If this is valid from a previous boot */ if (meta->head_buffer) rb_meta_buffer_update(cpu_buffer, bpage); bpage->range = 1; bpage->id = i + 1; } else if (desc) { void *p = ring_buffer_desc_page(desc, i + 1); if (WARN_ON(!p)) goto free_pages; bpage->page = p; bpage->range = 1; /* bpage->page can't be freed */ bpage->id = i + 1; cpu_buffer->subbuf_ids[i + 1] = bpage; } else { int order = cpu_buffer->buffer->subbuf_order; bpage->page = alloc_cpu_data(cpu_buffer->cpu, order); if (!bpage->page) goto free_pages; } bpage->order = cpu_buffer->buffer->subbuf_order; if (user_thread && fatal_signal_pending(current)) goto free_pages; } if (user_thread) clear_current_oom_origin(); return 0; free_pages: list_for_each_entry_safe(bpage, tmp, pages, list) { list_del_init(&bpage->list); free_buffer_page(bpage); } if (user_thread) clear_current_oom_origin(); return -ENOMEM; } static int rb_allocate_pages(struct ring_buffer_per_cpu *cpu_buffer, unsigned long nr_pages) { LIST_HEAD(pages); WARN_ON(!nr_pages); if (__rb_allocate_pages(cpu_buffer, nr_pages, &pages)) return -ENOMEM; /* * The ring buffer page list is a circular list that does not * start and end with a list head. All page list items point to * other pages. */ cpu_buffer->pages = pages.next; list_del(&pages); cpu_buffer->nr_pages = nr_pages; rb_check_pages(cpu_buffer); return 0; } static struct ring_buffer_per_cpu * rb_allocate_cpu_buffer(struct trace_buffer *buffer, long nr_pages, int cpu) { struct ring_buffer_per_cpu *cpu_buffer __free(kfree) = alloc_cpu_buffer(cpu); struct ring_buffer_cpu_meta *meta; struct buffer_page *bpage; int ret; if (!cpu_buffer) return NULL; cpu_buffer->cpu = cpu; cpu_buffer->buffer = buffer; raw_spin_lock_init(&cpu_buffer->reader_lock);