History

Merge branch 'perf-urgent-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull perf fixes from Ingo Molnar: "Four fixes. The mmap ones are unfortunately larger than desired - fuzzing uncovered bugs that needed perf context life time management changes to fix properly" * 'perf-urgent-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: perf/x86: Fix broken PEBS-LL support on SNB-EP/IVB-EP perf: Fix mmap() accounting hole perf: Fix perf mmap bugs kprobes: Fix to free gone and unused optprobes

Linus Torvalds committed 13 years ago

Commit 86c76672 parents 805e318 f1a52785 files · +187 −85Browse files

Unified · Split

5 files
  1. arch/x86/kernel/cpu/perf_event_intel.c +1 −1
  2. include/linux/perf_event.h +1 −2
  3. kernel/events/core.c +161 −72
  4. kernel/events/internal.h +4 −0
  5. kernel/kprobes.c +20 −10
arch/x86/kernel/cpu/perf_event_intel.c+1−1
@@ -165,13 +165,13 @@ static struct extra_reg intel_snb_extra_regs[] __read_mostly = { · 164 unmodified lines
165165 INTEL_EVENT_EXTRA_REG(0xb7, MSR_OFFCORE_RSP_0, 0x3f807f8fffull, RSP_0),
166166 INTEL_EVENT_EXTRA_REG(0xbb, MSR_OFFCORE_RSP_1, 0x3f807f8fffull, RSP_1),
167167 INTEL_UEVENT_PEBS_LDLAT_EXTRA_REG(0x01cd),
168− INTEL_UEVENT_PEBS_LDLAT_EXTRA_REG(0x01cd),
169168 EVENT_EXTRA_END
170169 };
171170
172171 static struct extra_reg intel_snbep_extra_regs[] __read_mostly = {
173172 INTEL_EVENT_EXTRA_REG(0xb7, MSR_OFFCORE_RSP_0, 0x3fffff8fffull, RSP_0),
174173 INTEL_EVENT_EXTRA_REG(0xbb, MSR_OFFCORE_RSP_1, 0x3fffff8fffull, RSP_1),
174+ INTEL_UEVENT_PEBS_LDLAT_EXTRA_REG(0x01cd),
175175 EVENT_EXTRA_END
176176 };
177177
include/linux/perf_event.h+1−2
@@ -389,8 +389,7 @@ struct perf_event { · 388 unmodified lines
389389 /* mmap bits */
390390 struct mutex mmap_mutex;
391391 atomic_t mmap_count;
392− int mmap_locked;
393− struct user_struct *mmap_user;
392+
394393 struct ring_buffer *rb;
395394 struct list_head rb_entry;
396395
kernel/events/core.c+161−72
@@ -196,9 +196,6 @@ static void cpu_ctx_sched_in(struct perf_cpu_context *cpuctx, · 195 unmodified lines
196196 static void update_context_time(struct perf_event_context *ctx);
197197 static u64 perf_event_time(struct perf_event *event);
198198
199−static void ring_buffer_attach(struct perf_event *event,
200− struct ring_buffer *rb);
201−
202199 void __weak perf_event_print_debug(void) { }
203200
204201 extern __weak const char *perf_pmu_name(void)
@@ -2918,6 +2915,7 @@ static void free_event_rcu(struct rcu_head *head) · 2713 unmodified lines
29182915 }
29192916
29202917 static void ring_buffer_put(struct ring_buffer *rb);
2918+static void ring_buffer_detach(struct perf_event *event, struct ring_buffer *rb);
29212919
29222920 static void free_event(struct perf_event *event)
29232921 {
@@ -2942,15 +2940,30 @@ static void free_event(struct perf_event *event) · 18 unmodified lines
29422940 if (has_branch_stack(event)) {
29432941 static_key_slow_dec_deferred(&perf_sched_events);
29442942 /* is system-wide event */
2945− if (!(event->attach_state & PERF_ATTACH_TASK))
2943+ if (!(event->attach_state & PERF_ATTACH_TASK)) {
29462944 atomic_dec(&per_cpu(perf_branch_stack_events,
29472945 event->cpu));
2946+ }
29482947 }
29492948 }
29502949
29512950 if (event->rb) {
2952− ring_buffer_put(event->rb);
2953− event->rb = NULL;
2951+ struct ring_buffer *rb;
2952+
2953+ /*
2954+ * Can happen when we close an event with re-directed output.
2955+ *
2956+ * Since we have a 0 refcount, perf_mmap_close() will skip
2957+ * over us; possibly making our ring_buffer_put() the last.
2958+ */
2959+ mutex_lock(&event->mmap_mutex);
2960+ rb = event->rb;
2961+ if (rb) {
2962+ rcu_assign_pointer(event->rb, NULL);
2963+ ring_buffer_detach(event, rb);
2964+ ring_buffer_put(rb); /* could be last */
2965+ }
2966+ mutex_unlock(&event->mmap_mutex);
29542967 }
29552968
29562969 if (is_cgroup_event(event))
@@ -3188,30 +3201,13 @@ static unsigned int perf_poll(struct file *file, poll_table *wait) · 231 unmodified lines
31883201 unsigned int events = POLL_HUP;
31893202
31903203 /*
3191− * Race between perf_event_set_output() and perf_poll(): perf_poll()
3192− * grabs the rb reference but perf_event_set_output() overrides it.
3193− * Here is the timeline for two threads T1, T2:
3194− * t0: T1, rb = rcu_dereference(event->rb)
3195− * t1: T2, old_rb = event->rb
3196− * t2: T2, event->rb = new rb
3197− * t3: T2, ring_buffer_detach(old_rb)
3198− * t4: T1, ring_buffer_attach(rb1)
3199− * t5: T1, poll_wait(event->waitq)
3200− *
3201− * To avoid this problem, we grab mmap_mutex in perf_poll()
3202− * thereby ensuring that the assignment of the new ring buffer
3203− * and the detachment of the old buffer appear atomic to perf_poll()
3204+ * Pin the event->rb by taking event->mmap_mutex; otherwise
3205+ * perf_event_set_output() can swizzle our rb and make us miss wakeups.
32043206 */
32053207 mutex_lock(&event->mmap_mutex);
3206−
3207− rcu_read_lock();
3208− rb = rcu_dereference(event->rb);
3209− if (rb) {
3210− ring_buffer_attach(event, rb);
3208+ rb = event->rb;
3209+ if (rb)
32113210 events = atomic_xchg(&rb->poll, 0);
3212− }
3213− rcu_read_unlock();
3214−
32153211 mutex_unlock(&event->mmap_mutex);
32163212
32173213 poll_wait(file, &event->waitq, wait);
@@ -3521,16 +3517,12 @@ static void ring_buffer_attach(struct perf_event *event, · 303 unmodified lines
35213517 return;
35223518
35233519 spin_lock_irqsave(&rb->event_lock, flags);
3524− if (!list_empty(&event->rb_entry))
3525− goto unlock;
3526−
3527− list_add(&event->rb_entry, &rb->event_list);
3528−unlock:
3520+ if (list_empty(&event->rb_entry))
3521+ list_add(&event->rb_entry, &rb->event_list);
35293522 spin_unlock_irqrestore(&rb->event_lock, flags);
35303523 }
35313524
3532−static void ring_buffer_detach(struct perf_event *event,
3533− struct ring_buffer *rb)
3525+static void ring_buffer_detach(struct perf_event *event, struct ring_buffer *rb)
35343526 {
35353527 unsigned long flags;
35363528
@@ -3549,13 +3541,10 @@ static void ring_buffer_wakeup(struct perf_event *event) · 12 unmodified lines
35493541
35503542 rcu_read_lock();
35513543 rb = rcu_dereference(event->rb);
3552− if (!rb)
3553− goto unlock;
3554−
3555− list_for_each_entry_rcu(event, &rb->event_list, rb_entry)
3556− wake_up_all(&event->waitq);
3557−
3558−unlock:
3544+ if (rb) {
3545+ list_for_each_entry_rcu(event, &rb->event_list, rb_entry)
3546+ wake_up_all(&event->waitq);
3547+ }
35593548 rcu_read_unlock();
35603549 }
35613550
@@ -3584,18 +3573,10 @@ static struct ring_buffer *ring_buffer_get(struct perf_event *event) · 22 unmodified lines
35843573
35853574 static void ring_buffer_put(struct ring_buffer *rb)
35863575 {
3587− struct perf_event *event, *n;
3588− unsigned long flags;
3589−
35903576 if (!atomic_dec_and_test(&rb->refcount))
35913577 return;
35923578
3593− spin_lock_irqsave(&rb->event_lock, flags);
3594− list_for_each_entry_safe(event, n, &rb->event_list, rb_entry) {
3595− list_del_init(&event->rb_entry);
3596− wake_up_all(&event->waitq);
3597− }
3598− spin_unlock_irqrestore(&rb->event_lock, flags);
3579+ WARN_ON_ONCE(!list_empty(&rb->event_list));
35993580
36003581 call_rcu(&rb->rcu_head, rb_free_rcu);
36013582 }
@@ -3605,26 +3586,100 @@ static void perf_mmap_open(struct vm_area_struct *vma) · 3 unmodified lines
36053586 struct perf_event *event = vma->vm_file->private_data;
36063587
36073588 atomic_inc(&event->mmap_count);
3589+ atomic_inc(&event->rb->mmap_count);
36083590 }
36093591
3592+/*
3593+ * A buffer can be mmap()ed multiple times; either directly through the same
3594+ * event, or through other events by use of perf_event_set_output().
3595+ *
3596+ * In order to undo the VM accounting done by perf_mmap() we need to destroy
3597+ * the buffer here, where we still have a VM context. This means we need
3598+ * to detach all events redirecting to us.
3599+ */
36103600 static void perf_mmap_close(struct vm_area_struct *vma)
36113601 {
36123602 struct perf_event *event = vma->vm_file->private_data;
36133603
3614− if (atomic_dec_and_mutex_lock(&event->mmap_count, &event->mmap_mutex)) {
3615− unsigned long size = perf_data_size(event->rb);
3616− struct user_struct *user = event->mmap_user;
3617− struct ring_buffer *rb = event->rb;
3604+ struct ring_buffer *rb = event->rb;
3605+ struct user_struct *mmap_user = rb->mmap_user;
3606+ int mmap_locked = rb->mmap_locked;
3607+ unsigned long size = perf_data_size(rb);
3608+
3609+ atomic_dec(&rb->mmap_count);
3610+
3611+ if (!atomic_dec_and_mutex_lock(&event->mmap_count, &event->mmap_mutex))
3612+ return;
36183613
3619− atomic_long_sub((size >> PAGE_SHIFT) + 1, &user->locked_vm);
3620− vma->vm_mm->pinned_vm -= event->mmap_locked;
3621− rcu_assign_pointer(event->rb, NULL);
3622− ring_buffer_detach(event, rb);
3614+ /* Detach current event from the buffer. */
3615+ rcu_assign_pointer(event->rb, NULL);
3616+ ring_buffer_detach(event, rb);
3617+ mutex_unlock(&event->mmap_mutex);
3618+
3619+ /* If there's still other mmap()s of this buffer, we're done. */
3620+ if (atomic_read(&rb->mmap_count)) {
3621+ ring_buffer_put(rb); /* can't be last */
3622+ return;
3623+ }
3624+
3625+ /*
3626+ * No other mmap()s, detach from all other events that might redirect
3627+ * into the now unreachable buffer. Somewhat complicated by the
3628+ * fact that rb::event_lock otherwise nests inside mmap_mutex.
3629+ */
3630+again:
3631+ rcu_read_lock();
3632+ list_for_each_entry_rcu(event, &rb->event_list, rb_entry) {
3633+ if (!atomic_long_inc_not_zero(&event->refcount)) {
3634+ /*
3635+ * This event is en-route to free_event() which will
3636+ * detach it and remove it from the list.
3637+ */
3638+ continue;
3639+ }
3640+ rcu_read_unlock();
3641+
3642+ mutex_lock(&event->mmap_mutex);
3643+ /*
3644+ * Check we didn't race with perf_event_set_output() which can
3645+ * swizzle the rb from under us while we were waiting to
3646+ * acquire mmap_mutex.
3647+ *
3648+ * If we find a different rb; ignore this event, a next
3649+ * iteration will no longer find it on the list. We have to
3650+ * still restart the iteration to make sure we're not now
3651+ * iterating the wrong list.
3652+ */
3653+ if (event->rb == rb) {
3654+ rcu_assign_pointer(event->rb, NULL);
3655+ ring_buffer_detach(event, rb);
3656+ ring_buffer_put(rb); /* can't be last, we still have one */
3657+ }
36233658 mutex_unlock(&event->mmap_mutex);
3659+ put_event(event);
36243660
3625− ring_buffer_put(rb);
3626− free_uid(user);
3661+ /*
3662+ * Restart the iteration; either we're on the wrong list or
3663+ * destroyed its integrity by doing a deletion.
3664+ */
3665+ goto again;
36273666 }
3667+ rcu_read_unlock();
3668+
3669+ /*
3670+ * It could be there's still a few 0-ref events on the list; they'll
3671+ * get cleaned up by free_event() -- they'll also still have their
3672+ * ref on the rb and will free it whenever they are done with it.
3673+ *
3674+ * Aside from that, this buffer is 'fully' detached and unmapped,
3675+ * undo the VM accounting.
3676+ */
3677+
3678+ atomic_long_sub((size >> PAGE_SHIFT) + 1, &mmap_user->locked_vm);
3679+ vma->vm_mm->pinned_vm -= mmap_locked;
3680+ free_uid(mmap_user);
3681+
3682+ ring_buffer_put(rb); /* could be last */
36283683 }
36293684
36303685 static const struct vm_operations_struct perf_mmap_vmops = {
@@ -3674,12 +3729,24 @@ static int perf_mmap(struct file *file, struct vm_area_struct *vma) · 43 unmodified lines
36743729 return -EINVAL;
36753730
36763731 WARN_ON_ONCE(event->ctx->parent_ctx);
3732+again:
36773733 mutex_lock(&event->mmap_mutex);
36783734 if (event->rb) {
3679− if (event->rb->nr_pages == nr_pages)
3680− atomic_inc(&event->rb->refcount);
3681− else
3735+ if (event->rb->nr_pages != nr_pages) {
36823736 ret = -EINVAL;
3737+ goto unlock;
3738+ }
3739+
3740+ if (!atomic_inc_not_zero(&event->rb->mmap_count)) {
3741+ /*
3742+ * Raced against perf_mmap_close() through
3743+ * perf_event_set_output(). Try again, hope for better
3744+ * luck.
3745+ */
3746+ mutex_unlock(&event->mmap_mutex);
3747+ goto again;
3748+ }
3749+
36833750 goto unlock;
36843751 }
36853752
@@ -3720,12 +3787,16 @@ static int perf_mmap(struct file *file, struct vm_area_struct *vma) · 34 unmodified lines
37203787 ret = -ENOMEM;
37213788 goto unlock;
37223789 }
3723− rcu_assign_pointer(event->rb, rb);
3790+
3791+ atomic_set(&rb->mmap_count, 1);
3792+ rb->mmap_locked = extra;
3793+ rb->mmap_user = get_current_user();
37243794
37253795 atomic_long_add(user_extra, &user->locked_vm);
3726− event->mmap_locked = extra;
3727− event->mmap_user = get_current_user();
3728− vma->vm_mm->pinned_vm += event->mmap_locked;
3796+ vma->vm_mm->pinned_vm += extra;
3797+
3798+ ring_buffer_attach(event, rb);
3799+ rcu_assign_pointer(event->rb, rb);
37293800
37303801 perf_event_update_userpage(event);
37313802
@@ -3734,7 +3805,11 @@ unlock: · 2 unmodified lines
37343805 atomic_inc(&event->mmap_count);
37353806 mutex_unlock(&event->mmap_mutex);
37363807
3737− vma->vm_flags |= VM_DONTEXPAND | VM_DONTDUMP;
3808+ /*
3809+ * Since pinned accounting is per vm we cannot allow fork() to copy our
3810+ * vma.
3811+ */
3812+ vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND | VM_DONTDUMP;
37383813 vma->vm_ops = &perf_mmap_vmops;
37393814
37403815 return ret;
@@ -6412,6 +6487,8 @@ set: · 2671 unmodified lines
64126487 if (atomic_read(&event->mmap_count))
64136488 goto unlock;
64146489
6490+ old_rb = event->rb;
6491+
64156492 if (output_event) {
64166493 /* get the rb we want to redirect to */
64176494 rb = ring_buffer_get(output_event);
@@ -6419,16 +6496,28 @@ set: · 1 unmodified line
64196496 goto unlock;
64206497 }
64216498
6422− old_rb = event->rb;
6423− rcu_assign_pointer(event->rb, rb);
64246499 if (old_rb)
64256500 ring_buffer_detach(event, old_rb);
6501+
6502+ if (rb)
6503+ ring_buffer_attach(event, rb);
6504+
6505+ rcu_assign_pointer(event->rb, rb);
6506+
6507+ if (old_rb) {
6508+ ring_buffer_put(old_rb);
6509+ /*
6510+ * Since we detached before setting the new rb, so that we
6511+ * could attach the new rb, we could have missed a wakeup.
6512+ * Provide it now.
6513+ */
6514+ wake_up_all(&event->waitq);
6515+ }
6516+
64266517 ret = 0;
64276518 unlock:
64286519 mutex_unlock(&event->mmap_mutex);
64296520
6430− if (old_rb)
6431− ring_buffer_put(old_rb);
64326521 out:
64336522 return ret;
64346523 }
kernel/events/internal.h+4−0
@@ -31,6 +31,10 @@ struct ring_buffer { · 30 unmodified lines
3131 spinlock_t event_lock;
3232 struct list_head event_list;
3333
34+ atomic_t mmap_count;
35+ unsigned long mmap_locked;
36+ struct user_struct *mmap_user;
37+
3438 struct perf_event_mmap_page *user_page;
3539 void *data_pages[0];
3640 };
kernel/kprobes.c+20−10
@@ -467,6 +467,7 @@ static struct kprobe *__kprobes get_optimized_kprobe(unsigned long addr) · 466 unmodified lines
467467 /* Optimization staging list, protected by kprobe_mutex */
468468 static LIST_HEAD(optimizing_list);
469469 static LIST_HEAD(unoptimizing_list);
470+static LIST_HEAD(freeing_list);
470471
471472 static void kprobe_optimizer(struct work_struct *work);
472473 static DECLARE_DELAYED_WORK(optimizing_work, kprobe_optimizer);
@@ -504,7 +505,7 @@ static __kprobes void do_optimize_kprobes(void) · 31 unmodified lines
504505 * Unoptimize (replace a jump with a breakpoint and remove the breakpoint
505506 * if need) kprobes listed on unoptimizing_list.
506507 */
507−static __kprobes void do_unoptimize_kprobes(struct list_head *free_list)
508+static __kprobes void do_unoptimize_kprobes(void)
508509 {
509510 struct optimized_kprobe *op, *tmp;
510511
@@ -515,9 +516,9 @@ static __kprobes void do_unoptimize_kprobes(struct list_head *free_list) · 4 unmodified lines
515516 /* Ditto to do_optimize_kprobes */
516517 get_online_cpus();
517518 mutex_lock(&text_mutex);
518− arch_unoptimize_kprobes(&unoptimizing_list, free_list);
519+ arch_unoptimize_kprobes(&unoptimizing_list, &freeing_list);
519520 /* Loop free_list for disarming */
520− list_for_each_entry_safe(op, tmp, free_list, list) {
521+ list_for_each_entry_safe(op, tmp, &freeing_list, list) {
521522 /* Disarm probes if marked disabled */
522523 if (kprobe_disabled(&op->kp))
523524 arch_disarm_kprobe(&op->kp);
@@ -536,11 +537,11 @@ static __kprobes void do_unoptimize_kprobes(struct list_head *free_list) · 12 unmodified lines
536537 }
537538
538539 /* Reclaim all kprobes on the free_list */
539−static __kprobes void do_free_cleaned_kprobes(struct list_head *free_list)
540+static __kprobes void do_free_cleaned_kprobes(void)
540541 {
541542 struct optimized_kprobe *op, *tmp;
542543
543− list_for_each_entry_safe(op, tmp, free_list, list) {
544+ list_for_each_entry_safe(op, tmp, &freeing_list, list) {
544545 BUG_ON(!kprobe_unused(&op->kp));
545546 list_del_init(&op->list);
546547 free_aggr_kprobe(&op->kp);
@@ -556,8 +557,6 @@ static __kprobes void kick_kprobe_optimizer(void) · 9 unmodified lines
556557 /* Kprobe jump optimizer */
557558 static __kprobes void kprobe_optimizer(struct work_struct *work)
558559 {
559− LIST_HEAD(free_list);
560−
561560 mutex_lock(&kprobe_mutex);
562561 /* Lock modules while optimizing kprobes */
563562 mutex_lock(&module_mutex);
@@ -566,7 +565,7 @@ static __kprobes void kprobe_optimizer(struct work_struct *work) · 2 unmodified lines
566565 * Step 1: Unoptimize kprobes and collect cleaned (unused and disarmed)
567566 * kprobes before waiting for quiesence period.
568567 */
569− do_unoptimize_kprobes(&free_list);
568+ do_unoptimize_kprobes();
570569
571570 /*
572571 * Step 2: Wait for quiesence period to ensure all running interrupts
@@ -581,7 +580,7 @@ static __kprobes void kprobe_optimizer(struct work_struct *work) · 8 unmodified lines
581580 do_optimize_kprobes();
582581
583582 /* Step 4: Free cleaned kprobes after quiesence period */
584− do_free_cleaned_kprobes(&free_list);
583+ do_free_cleaned_kprobes();
585584
586585 mutex_unlock(&module_mutex);
587586 mutex_unlock(&kprobe_mutex);
@@ -723,8 +722,19 @@ static void __kprobes kill_optimized_kprobe(struct kprobe *p) · 135 unmodified lines
723722 if (!list_empty(&op->list))
724723 /* Dequeue from the (un)optimization queue */
725724 list_del_init(&op->list);
726−
727725 op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED;
726+
727+ if (kprobe_unused(p)) {
728+ /* Enqueue if it is unused */
729+ list_add(&op->list, &freeing_list);
730+ /*
731+ * Remove unused probes from the hash list. After waiting
732+ * for synchronization, this probe is reclaimed.
733+ * (reclaiming is done by do_free_cleaned_kprobes().)
734+ */
735+ hlist_del_rcu(&op->kp.hlist);
736+ }
737+
728738 /* Don't touch the code, because it is already freed. */
729739 arch_remove_optimized_kprobe(op);
730740 }

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t