History

Merge branch 'perf-urgent-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull perf fixes from Thomas Gleixner: "A rather largish series of 12 patches addressing a maze of race conditions in the perf core code from Peter Zijlstra" * 'perf-urgent-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: perf: Robustify task_function_call() perf: Fix scaling vs. perf_install_in_context() perf: Fix scaling vs. perf_event_enable() perf: Fix scaling vs. perf_event_enable_on_exec() perf: Fix ctx time tracking by introducing EVENT_TIME perf: Cure event->pending_disable race perf: Fix race between event install and jump_labels perf: Fix cloning perf: Only update context time when active perf: Allow perf_release() with !event->ctx perf: Do not double free perf: Close install vs. exit race

Linus Torvalds committed 10 years ago

Commit 1b9540c2 parents 4b696dc 0da4cf32 files · +244 −131Browse files

Unified · Split

2 files
  1. include/linux/perf_event.h +4 −3
  2. kernel/events/core.c +240 −128
include/linux/perf_event.h+4−3
@@ -397,6 +397,7 @@ struct pmu { · 396 unmodified lines
397397 * enum perf_event_active_state - the states of a event
398398 */
399399 enum perf_event_active_state {
400+ PERF_EVENT_STATE_DEAD = -4,
400401 PERF_EVENT_STATE_EXIT = -3,
401402 PERF_EVENT_STATE_ERROR = -2,
402403 PERF_EVENT_STATE_OFF = -1,
@@ -905,7 +906,7 @@ perf_sw_event_sched(u32 event_id, u64 nr, u64 addr) · 502 unmodified lines
905906 }
906907 }
907908
908−extern struct static_key_deferred perf_sched_events;
909+extern struct static_key_false perf_sched_events;
909910
910911 static __always_inline bool
911912 perf_sw_migrate_enabled(void)
@@ -924,7 +925,7 @@ static inline void perf_event_task_migrate(struct task_struct *task) · 12 unmodified lines
924925 static inline void perf_event_task_sched_in(struct task_struct *prev,
925926 struct task_struct *task)
926927 {
927− if (static_key_false(&perf_sched_events.key))
928+ if (static_branch_unlikely(&perf_sched_events))
928929 __perf_event_task_sched_in(prev, task);
929930
930931 if (perf_sw_migrate_enabled() && task->sched_migrated) {
@@ -941,7 +942,7 @@ static inline void perf_event_task_sched_out(struct task_struct *prev, · 10 unmodified lines
941942 {
942943 perf_sw_event_sched(PERF_COUNT_SW_CONTEXT_SWITCHES, 1, 0);
943944
944− if (static_key_false(&perf_sched_events.key))
945+ if (static_branch_unlikely(&perf_sched_events))
945946 __perf_event_task_sched_out(prev, next);
946947 }
947948
kernel/events/core.c+240−128
@@ -64,8 +64,17 @@ static void remote_function(void *data) · 63 unmodified lines
6464 struct task_struct *p = tfc->p;
6565
6666 if (p) {
67− tfc->ret = -EAGAIN;
68− if (task_cpu(p) != smp_processor_id() || !task_curr(p))
67+ /* -EAGAIN */
68+ if (task_cpu(p) != smp_processor_id())
69+ return;
70+
71+ /*
72+ * Now that we're on right CPU with IRQs disabled, we can test
73+ * if we hit the right task without races.
74+ */
75+
76+ tfc->ret = -ESRCH; /* No such (running) process */
77+ if (p != current)
6978 return;
7079 }
7180
@@ -92,13 +101,17 @@ task_function_call(struct task_struct *p, remote_function_f func, void *info) · 20 unmodified lines
92101 .p = p,
93102 .func = func,
94103 .info = info,
95− .ret = -ESRCH, /* No such (running) process */
104+ .ret = -EAGAIN,
96105 };
106+ int ret;
97107
98− if (task_curr(p))
99− smp_call_function_single(task_cpu(p), remote_function, &data, 1);
108+ do {
109+ ret = smp_call_function_single(task_cpu(p), remote_function, &data, 1);
110+ if (!ret)
111+ ret = data.ret;
112+ } while (ret == -EAGAIN);
100113
101− return data.ret;
114+ return ret;
102115 }
103116
104117 /**
@@ -169,19 +182,6 @@ static bool is_kernel_event(struct perf_event *event) · 64 unmodified lines
169182 * rely on ctx->is_active and therefore cannot use event_function_call().
170183 * See perf_install_in_context().
171184 *
172− * This is because we need a ctx->lock serialized variable (ctx->is_active)
173− * to reliably determine if a particular task/context is scheduled in. The
174− * task_curr() use in task_function_call() is racy in that a remote context
175− * switch is not a single atomic operation.
176− *
177− * As is, the situation is 'safe' because we set rq->curr before we do the
178− * actual context switch. This means that task_curr() will fail early, but
179− * we'll continue spinning on ctx->is_active until we've passed
180− * perf_event_task_sched_out().
181− *
182− * Without this ctx->lock serialized variable we could have race where we find
183− * the task (and hence the context) would not be active while in fact they are.
184− *
185185 * If ctx->nr_events, then ctx->is_active and cpuctx->task_ctx are set.
186186 */
187187
@@ -212,7 +212,7 @@ static int event_function(void *info) · 24 unmodified lines
212212 */
213213 if (ctx->task) {
214214 if (ctx->task != current) {
215− ret = -EAGAIN;
215+ ret = -ESRCH;
216216 goto unlock;
217217 }
218218
@@ -276,10 +276,10 @@ static void event_function_call(struct perf_event *event, event_f func, void *da · 57 unmodified lines
276276 return;
277277 }
278278
279−again:
280279 if (task == TASK_TOMBSTONE)
281280 return;
282281
282+again:
283283 if (!task_function_call(task, event_function, &efs))
284284 return;
285285
@@ -289,13 +289,15 @@ again: · 3 unmodified lines
289289 * a concurrent perf_event_context_sched_out().
290290 */
291291 task = ctx->task;
292− if (task != TASK_TOMBSTONE) {
293− if (ctx->is_active) {
294− raw_spin_unlock_irq(&ctx->lock);
295− goto again;
296− }
297− func(event, NULL, ctx, data);
292+ if (task == TASK_TOMBSTONE) {
293+ raw_spin_unlock_irq(&ctx->lock);
294+ return;
298295 }
296+ if (ctx->is_active) {
297+ raw_spin_unlock_irq(&ctx->lock);
298+ goto again;
299+ }
300+ func(event, NULL, ctx, data);
299301 raw_spin_unlock_irq(&ctx->lock);
300302 }
301303
@@ -314,6 +316,7 @@ again: · 12 unmodified lines
314316 enum event_type_t {
315317 EVENT_FLEXIBLE = 0x1,
316318 EVENT_PINNED = 0x2,
319+ EVENT_TIME = 0x4,
317320 EVENT_ALL = EVENT_FLEXIBLE | EVENT_PINNED,
318321 };
319322
@@ -321,7 +324,13 @@ enum event_type_t { · 1 unmodified line
321324 * perf_sched_events : >0 events exist
322325 * perf_cgroup_events: >0 per-cpu cgroup events exist on this cpu
323326 */
324−struct static_key_deferred perf_sched_events __read_mostly;
327+
328+static void perf_sched_delayed(struct work_struct *work);
329+DEFINE_STATIC_KEY_FALSE(perf_sched_events);
330+static DECLARE_DELAYED_WORK(perf_sched_work, perf_sched_delayed);
331+static DEFINE_MUTEX(perf_sched_mutex);
332+static atomic_t perf_sched_count;
333+
325334 static DEFINE_PER_CPU(atomic_t, perf_cgroup_events);
326335 static DEFINE_PER_CPU(int, perf_sched_cb_usages);
327336
@@ -1288,16 +1297,18 @@ static u64 perf_event_time(struct perf_event *event) · 960 unmodified lines
12881297
12891298 /*
12901299 * Update the total_time_enabled and total_time_running fields for a event.
1291− * The caller of this function needs to hold the ctx->lock.
12921300 */
12931301 static void update_event_times(struct perf_event *event)
12941302 {
12951303 struct perf_event_context *ctx = event->ctx;
12961304 u64 run_end;
12971305
1306+ lockdep_assert_held(&ctx->lock);
1307+
12981308 if (event->state < PERF_EVENT_STATE_INACTIVE ||
12991309 event->group_leader->state < PERF_EVENT_STATE_INACTIVE)
13001310 return;
1311+
13011312 /*
13021313 * in cgroup mode, time_enabled represents
13031314 * the time the event was enabled AND active
@@ -1645,7 +1656,7 @@ out: · 341 unmodified lines
16451656
16461657 static bool is_orphaned_event(struct perf_event *event)
16471658 {
1648− return event->state == PERF_EVENT_STATE_EXIT;
1659+ return event->state == PERF_EVENT_STATE_DEAD;
16491660 }
16501661
16511662 static inline int pmu_filter_match(struct perf_event *event)
@@ -1690,14 +1701,14 @@ event_sched_out(struct perf_event *event, · 38 unmodified lines
16901701
16911702 perf_pmu_disable(event->pmu);
16921703
1704+ event->tstamp_stopped = tstamp;
1705+ event->pmu->del(event, 0);
1706+ event->oncpu = -1;
16931707 event->state = PERF_EVENT_STATE_INACTIVE;
16941708 if (event->pending_disable) {
16951709 event->pending_disable = 0;
16961710 event->state = PERF_EVENT_STATE_OFF;
16971711 }
1698− event->tstamp_stopped = tstamp;
1699− event->pmu->del(event, 0);
1700− event->oncpu = -1;
17011712
17021713 if (!is_software_event(event))
17031714 cpuctx->active_oncpu--;
@@ -1732,7 +1743,6 @@ group_sched_out(struct perf_event *group_event, · 28 unmodified lines
17321743 }
17331744
17341745 #define DETACH_GROUP 0x01UL
1735−#define DETACH_STATE 0x02UL
17361746
17371747 /*
17381748 * Cross CPU call to remove a performance event
@@ -1752,8 +1762,6 @@ __perf_remove_from_context(struct perf_event *event, · 13 unmodified lines
17521762 if (flags & DETACH_GROUP)
17531763 perf_group_detach(event);
17541764 list_del_event(event, ctx);
1755− if (flags & DETACH_STATE)
1756− event->state = PERF_EVENT_STATE_EXIT;
17571765
17581766 if (!ctx->nr_events && ctx->is_active) {
17591767 ctx->is_active = 0;
@@ -2063,14 +2071,27 @@ static void add_event_to_ctx(struct perf_event *event, · 303 unmodified lines
20632071 event->tstamp_stopped = tstamp;
20642072 }
20652073
2066−static void task_ctx_sched_out(struct perf_cpu_context *cpuctx,
2067− struct perf_event_context *ctx);
2074+static void ctx_sched_out(struct perf_event_context *ctx,
2075+ struct perf_cpu_context *cpuctx,
2076+ enum event_type_t event_type);
20682077 static void
20692078 ctx_sched_in(struct perf_event_context *ctx,
20702079 struct perf_cpu_context *cpuctx,
20712080 enum event_type_t event_type,
20722081 struct task_struct *task);
20732082
2083+static void task_ctx_sched_out(struct perf_cpu_context *cpuctx,
2084+ struct perf_event_context *ctx)
2085+{
2086+ if (!cpuctx->task_ctx)
2087+ return;
2088+
2089+ if (WARN_ON_ONCE(ctx != cpuctx->task_ctx))
2090+ return;
2091+
2092+ ctx_sched_out(ctx, cpuctx, EVENT_ALL);
2093+}
2094+
20742095 static void perf_event_sched_in(struct perf_cpu_context *cpuctx,
20752096 struct perf_event_context *ctx,
20762097 struct task_struct *task)
@@ -2097,49 +2118,68 @@ static void ctx_resched(struct perf_cpu_context *cpuctx, · 20 unmodified lines
20972118 /*
20982119 * Cross CPU call to install and enable a performance event
20992120 *
2100− * Must be called with ctx->mutex held
2121+ * Very similar to remote_function() + event_function() but cannot assume that
2122+ * things like ctx->is_active and cpuctx->task_ctx are set.
21012123 */
21022124 static int __perf_install_in_context(void *info)
21032125 {
2104− struct perf_event_context *ctx = info;
2126+ struct perf_event *event = info;
2127+ struct perf_event_context *ctx = event->ctx;
21052128 struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
21062129 struct perf_event_context *task_ctx = cpuctx->task_ctx;
2130+ bool activate = true;
2131+ int ret = 0;
21072132
21082133 raw_spin_lock(&cpuctx->ctx.lock);
21092134 if (ctx->task) {
21102135 raw_spin_lock(&ctx->lock);
2111− /*
2112− * If we hit the 'wrong' task, we've since scheduled and
2113− * everything should be sorted, nothing to do!
2114− */
21152136 task_ctx = ctx;
2116− if (ctx->task != current)
2137+
2138+ /* If we're on the wrong CPU, try again */
2139+ if (task_cpu(ctx->task) != smp_processor_id()) {
2140+ ret = -ESRCH;
21172141 goto unlock;
2142+ }
21182143
21192144 /*
2120− * If task_ctx is set, it had better be to us.
2145+ * If we're on the right CPU, see if the task we target is
2146+ * current, if not we don't have to activate the ctx, a future
2147+ * context switch will do that for us.
21212148 */
2122− WARN_ON_ONCE(cpuctx->task_ctx != ctx && cpuctx->task_ctx);
2149+ if (ctx->task != current)
2150+ activate = false;
2151+ else
2152+ WARN_ON_ONCE(cpuctx->task_ctx && cpuctx->task_ctx != ctx);
2153+
21232154 } else if (task_ctx) {
21242155 raw_spin_lock(&task_ctx->lock);
21252156 }
21262157
2127− ctx_resched(cpuctx, task_ctx);
2158+ if (activate) {
2159+ ctx_sched_out(ctx, cpuctx, EVENT_TIME);
2160+ add_event_to_ctx(event, ctx);
2161+ ctx_resched(cpuctx, task_ctx);
2162+ } else {
2163+ add_event_to_ctx(event, ctx);
2164+ }
2165+
21282166 unlock:
21292167 perf_ctx_unlock(cpuctx, task_ctx);
21302168
2131− return 0;
2169+ return ret;
21322170 }
21332171
21342172 /*
2135− * Attach a performance event to a context
2173+ * Attach a performance event to a context.
2174+ *
2175+ * Very similar to event_function_call, see comment there.
21362176 */
21372177 static void
21382178 perf_install_in_context(struct perf_event_context *ctx,
21392179 struct perf_event *event,
21402180 int cpu)
21412181 {
2142− struct task_struct *task = NULL;
2182+ struct task_struct *task = READ_ONCE(ctx->task);
21432183
21442184 lockdep_assert_held(&ctx->mutex);
21452185
@@ -2147,40 +2187,46 @@ perf_install_in_context(struct perf_event_context *ctx, · 1 unmodified line
21472187 if (event->cpu != -1)
21482188 event->cpu = cpu;
21492189
2190+ if (!task) {
2191+ cpu_function_call(cpu, __perf_install_in_context, event);
2192+ return;
2193+ }
2194+
2195+ /*
2196+ * Should not happen, we validate the ctx is still alive before calling.
2197+ */
2198+ if (WARN_ON_ONCE(task == TASK_TOMBSTONE))
2199+ return;
2200+
21502201 /*
21512202 * Installing events is tricky because we cannot rely on ctx->is_active
21522203 * to be set in case this is the nr_events 0 -> 1 transition.
2153− *
2154− * So what we do is we add the event to the list here, which will allow
2155− * a future context switch to DTRT and then send a racy IPI. If the IPI
2156− * fails to hit the right task, this means a context switch must have
2157− * happened and that will have taken care of business.
21582204 */
2159− raw_spin_lock_irq(&ctx->lock);
2160− task = ctx->task;
2205+again:
21612206 /*
2162− * Worse, we cannot even rely on the ctx actually existing anymore. If
2163− * between find_get_context() and perf_install_in_context() the task
2164− * went through perf_event_exit_task() its dead and we should not be
2165− * adding new events.
2207+ * Cannot use task_function_call() because we need to run on the task's
2208+ * CPU regardless of whether its current or not.
21662209 */
2167− if (task == TASK_TOMBSTONE) {
2210+ if (!cpu_function_call(task_cpu(task), __perf_install_in_context, event))
2211+ return;
2212+
2213+ raw_spin_lock_irq(&ctx->lock);
2214+ task = ctx->task;
2215+ if (WARN_ON_ONCE(task == TASK_TOMBSTONE)) {
2216+ /*
2217+ * Cannot happen because we already checked above (which also
2218+ * cannot happen), and we hold ctx->mutex, which serializes us
2219+ * against perf_event_exit_task_context().
2220+ */
21682221 raw_spin_unlock_irq(&ctx->lock);
21692222 return;
21702223 }
2171− update_context_time(ctx);
2224+ raw_spin_unlock_irq(&ctx->lock);
21722225 /*
2173− * Update cgrp time only if current cgrp matches event->cgrp.
2174− * Must be done before calling add_event_to_ctx().
2226+ * Since !ctx->is_active doesn't mean anything, we must IPI
2227+ * unconditionally.
21752228 */
2176− update_cgrp_time_from_event(event);
2177− add_event_to_ctx(event, ctx);
2178− raw_spin_unlock_irq(&ctx->lock);
2179−
2180− if (task)
2181− task_function_call(task, __perf_install_in_context, ctx);
2182− else
2183− cpu_function_call(cpu, __perf_install_in_context, ctx);
2229+ goto again;
21842230 }
21852231
21862232 /*
@@ -2219,17 +2265,18 @@ static void __perf_event_enable(struct perf_event *event, · 32 unmodified lines
22192265 event->state <= PERF_EVENT_STATE_ERROR)
22202266 return;
22212267
2222− update_context_time(ctx);
2268+ if (ctx->is_active)
2269+ ctx_sched_out(ctx, cpuctx, EVENT_TIME);
2270+
22232271 __perf_event_mark_enabled(event);
22242272
22252273 if (!ctx->is_active)
22262274 return;
22272275
22282276 if (!event_filter_match(event)) {
2229− if (is_cgroup_event(event)) {
2230− perf_cgroup_set_timestamp(current, ctx); // XXX ?
2277+ if (is_cgroup_event(event))
22312278 perf_cgroup_defer_enabled(event);
2232− }
2279+ ctx_sched_in(ctx, cpuctx, EVENT_TIME, current);
22332280 return;
22342281 }
22352282
@@ -2237,8 +2284,10 @@ static void __perf_event_enable(struct perf_event *event, · 1 unmodified line
22372284 * If the event is in a group and isn't the group leader,
22382285 * then don't put it on unless the group is on.
22392286 */
2240− if (leader != event && leader->state != PERF_EVENT_STATE_ACTIVE)
2287+ if (leader != event && leader->state != PERF_EVENT_STATE_ACTIVE) {
2288+ ctx_sched_in(ctx, cpuctx, EVENT_TIME, current);
22412289 return;
2290+ }
22422291
22432292 task_ctx = cpuctx->task_ctx;
22442293 if (ctx->task)
@@ -2344,24 +2393,33 @@ static void ctx_sched_out(struct perf_event_context *ctx, · 99 unmodified lines
23442393 }
23452394
23462395 ctx->is_active &= ~event_type;
2396+ if (!(ctx->is_active & EVENT_ALL))
2397+ ctx->is_active = 0;
2398+
23472399 if (ctx->task) {
23482400 WARN_ON_ONCE(cpuctx->task_ctx != ctx);
23492401 if (!ctx->is_active)
23502402 cpuctx->task_ctx = NULL;
23512403 }
23522404
2353− update_context_time(ctx);
2354− update_cgrp_time_from_cpuctx(cpuctx);
2355− if (!ctx->nr_active)
2405+ is_active ^= ctx->is_active; /* changed bits */
2406+
2407+ if (is_active & EVENT_TIME) {
2408+ /* update (and stop) ctx time */
2409+ update_context_time(ctx);
2410+ update_cgrp_time_from_cpuctx(cpuctx);
2411+ }
2412+
2413+ if (!ctx->nr_active || !(is_active & EVENT_ALL))
23562414 return;
23572415
23582416 perf_pmu_disable(ctx->pmu);
2359− if ((is_active & EVENT_PINNED) && (event_type & EVENT_PINNED)) {
2417+ if (is_active & EVENT_PINNED) {
23602418 list_for_each_entry(event, &ctx->pinned_groups, group_entry)
23612419 group_sched_out(event, cpuctx, ctx);
23622420 }
23632421
2364− if ((is_active & EVENT_FLEXIBLE) && (event_type & EVENT_FLEXIBLE)) {
2422+ if (is_active & EVENT_FLEXIBLE) {
23652423 list_for_each_entry(event, &ctx->flexible_groups, group_entry)
23662424 group_sched_out(event, cpuctx, ctx);
23672425 }
@@ -2641,18 +2699,6 @@ void __perf_event_task_sched_out(struct task_struct *task, · 273 unmodified lines
26412699 perf_cgroup_sched_out(task, next);
26422700 }
26432701
2644−static void task_ctx_sched_out(struct perf_cpu_context *cpuctx,
2645− struct perf_event_context *ctx)
2646−{
2647− if (!cpuctx->task_ctx)
2648− return;
2649−
2650− if (WARN_ON_ONCE(ctx != cpuctx->task_ctx))
2651− return;
2652−
2653− ctx_sched_out(ctx, cpuctx, EVENT_ALL);
2654−}
2655−
26562702 /*
26572703 * Called with IRQs disabled
26582704 */
@@ -2735,7 +2781,7 @@ ctx_sched_in(struct perf_event_context *ctx, · 76 unmodified lines
27352781 if (likely(!ctx->nr_events))
27362782 return;
27372783
2738− ctx->is_active |= event_type;
2784+ ctx->is_active |= (event_type | EVENT_TIME);
27392785 if (ctx->task) {
27402786 if (!is_active)
27412787 cpuctx->task_ctx = ctx;
@@ -2743,18 +2789,24 @@ ctx_sched_in(struct perf_event_context *ctx, · 1 unmodified line
27432789 WARN_ON_ONCE(cpuctx->task_ctx != ctx);
27442790 }
27452791
2746− now = perf_clock();
2747− ctx->timestamp = now;
2748− perf_cgroup_set_timestamp(task, ctx);
2792+ is_active ^= ctx->is_active; /* changed bits */
2793+
2794+ if (is_active & EVENT_TIME) {
2795+ /* start ctx time */
2796+ now = perf_clock();
2797+ ctx->timestamp = now;
2798+ perf_cgroup_set_timestamp(task, ctx);
2799+ }
2800+
27492801 /*
27502802 * First go through the list and put on any pinned groups
27512803 * in order to give them the best chance of going on.
27522804 */
2753− if (!(is_active & EVENT_PINNED) && (event_type & EVENT_PINNED))
2805+ if (is_active & EVENT_PINNED)
27542806 ctx_pinned_sched_in(ctx, cpuctx);
27552807
27562808 /* Then walk through the lower prio flexible groups */
2757− if (!(is_active & EVENT_FLEXIBLE) && (event_type & EVENT_FLEXIBLE))
2809+ if (is_active & EVENT_FLEXIBLE)
27582810 ctx_flexible_sched_in(ctx, cpuctx);
27592811 }
27602812
@@ -3120,6 +3172,7 @@ static void perf_event_enable_on_exec(int ctxn) · 359 unmodified lines
31203172
31213173 cpuctx = __get_cpu_context(ctx);
31223174 perf_ctx_lock(cpuctx, ctx);
3175+ ctx_sched_out(ctx, cpuctx, EVENT_TIME);
31233176 list_for_each_entry(event, &ctx->event_list, event_entry)
31243177 enabled |= event_enable_on_exec(event, ctx);
31253178
@@ -3537,12 +3590,22 @@ static void unaccount_event(struct perf_event *event) · 411 unmodified lines
35373590 if (has_branch_stack(event))
35383591 dec = true;
35393592
3540− if (dec)
3541− static_key_slow_dec_deferred(&perf_sched_events);
3593+ if (dec) {
3594+ if (!atomic_add_unless(&perf_sched_count, -1, 1))
3595+ schedule_delayed_work(&perf_sched_work, HZ);
3596+ }
35423597
35433598 unaccount_event_cpu(event, event->cpu);
35443599 }
35453600
3601+static void perf_sched_delayed(struct work_struct *work)
3602+{
3603+ mutex_lock(&perf_sched_mutex);
3604+ if (atomic_dec_and_test(&perf_sched_count))
3605+ static_branch_disable(&perf_sched_events);
3606+ mutex_unlock(&perf_sched_mutex);
3607+}
3608+
35463609 /*
35473610 * The following implement mutual exclusion of events on "exclusive" pmus
35483611 * (PERF_PMU_CAP_EXCLUSIVE). Such pmus can only have one event scheduled
@@ -3752,30 +3815,42 @@ static void put_event(struct perf_event *event) · 203 unmodified lines
37523815 */
37533816 int perf_event_release_kernel(struct perf_event *event)
37543817 {
3755− struct perf_event_context *ctx;
3818+ struct perf_event_context *ctx = event->ctx;
37563819 struct perf_event *child, *tmp;
37573820
3821+ /*
3822+ * If we got here through err_file: fput(event_file); we will not have
3823+ * attached to a context yet.
3824+ */
3825+ if (!ctx) {
3826+ WARN_ON_ONCE(event->attach_state &
3827+ (PERF_ATTACH_CONTEXT|PERF_ATTACH_GROUP));
3828+ goto no_ctx;
3829+ }
3830+
37583831 if (!is_kernel_event(event))
37593832 perf_remove_from_owner(event);
37603833
37613834 ctx = perf_event_ctx_lock(event);
37623835 WARN_ON_ONCE(ctx->parent_ctx);
3763− perf_remove_from_context(event, DETACH_GROUP | DETACH_STATE);
3764− perf_event_ctx_unlock(event, ctx);
3836+ perf_remove_from_context(event, DETACH_GROUP);
37653837
3838+ raw_spin_lock_irq(&ctx->lock);
37663839 /*
3767− * At this point we must have event->state == PERF_EVENT_STATE_EXIT,
3768− * either from the above perf_remove_from_context() or through
3769− * perf_event_exit_event().
3840+ * Mark this even as STATE_DEAD, there is no external reference to it
3841+ * anymore.
37703842 *
3771− * Therefore, anybody acquiring event->child_mutex after the below
3772− * loop _must_ also see this, most importantly inherit_event() which
3773− * will avoid placing more children on the list.
3843+ * Anybody acquiring event->child_mutex after the below loop _must_
3844+ * also see this, most importantly inherit_event() which will avoid
3845+ * placing more children on the list.
37743846 *
37753847 * Thus this guarantees that we will in fact observe and kill _ALL_
37763848 * child events.
37773849 */
3778− WARN_ON_ONCE(event->state != PERF_EVENT_STATE_EXIT);
3850+ event->state = PERF_EVENT_STATE_DEAD;
3851+ raw_spin_unlock_irq(&ctx->lock);
3852+
3853+ perf_event_ctx_unlock(event, ctx);
37793854
37803855 again:
37813856 mutex_lock(&event->child_mutex);
@@ -3830,8 +3905,8 @@ again: · 48 unmodified lines
38303905 }
38313906 mutex_unlock(&event->child_mutex);
38323907
3833− /* Must be the last reference */
3834− put_event(event);
3908+no_ctx:
3909+ put_event(event); /* Must be the 'last' reference */
38353910 return 0;
38363911 }
38373912 EXPORT_SYMBOL_GPL(perf_event_release_kernel);
@@ -3988,7 +4063,7 @@ static bool is_event_hup(struct perf_event *event) · 150 unmodified lines
39884063 {
39894064 bool no_children;
39904065
3991− if (event->state != PERF_EVENT_STATE_EXIT)
4066+ if (event->state > PERF_EVENT_STATE_EXIT)
39924067 return false;
39934068
39944069 mutex_lock(&event->child_mutex);
@@ -7769,8 +7844,28 @@ static void account_event(struct perf_event *event) · 3774 unmodified lines
77697844 if (is_cgroup_event(event))
77707845 inc = true;
77717846
7772− if (inc)
7773− static_key_slow_inc(&perf_sched_events.key);
7847+ if (inc) {
7848+ if (atomic_inc_not_zero(&perf_sched_count))
7849+ goto enabled;
7850+
7851+ mutex_lock(&perf_sched_mutex);
7852+ if (!atomic_read(&perf_sched_count)) {
7853+ static_branch_enable(&perf_sched_events);
7854+ /*
7855+ * Guarantee that all CPUs observe they key change and
7856+ * call the perf scheduling hooks before proceeding to
7857+ * install events that need them.
7858+ */
7859+ synchronize_sched();
7860+ }
7861+ /*
7862+ * Now that we have waited for the sync_sched(), allow further
7863+ * increments to by-pass the mutex.
7864+ */
7865+ atomic_inc(&perf_sched_count);
7866+ mutex_unlock(&perf_sched_mutex);
7867+ }
7868+enabled:
77747869
77757870 account_event_cpu(event, event->cpu);
77767871 }
@@ -8389,10 +8484,19 @@ SYSCALL_DEFINE5(perf_event_open, · 612 unmodified lines
83898484 if (move_group) {
83908485 gctx = group_leader->ctx;
83918486 mutex_lock_double(&gctx->mutex, &ctx->mutex);
8487+ if (gctx->task == TASK_TOMBSTONE) {
8488+ err = -ESRCH;
8489+ goto err_locked;
8490+ }
83928491 } else {
83938492 mutex_lock(&ctx->mutex);
83948493 }
83958494
8495+ if (ctx->task == TASK_TOMBSTONE) {
8496+ err = -ESRCH;
8497+ goto err_locked;
8498+ }
8499+
83968500 if (!perf_event_validate_size(event)) {
83978501 err = -E2BIG;
83988502 goto err_locked;
@@ -8509,7 +8613,12 @@ err_context: · 110 unmodified lines
85098613 perf_unpin_context(ctx);
85108614 put_ctx(ctx);
85118615 err_alloc:
8512− free_event(event);
8616+ /*
8617+ * If event_file is set, the fput() above will have called ->release()
8618+ * and that will take care of freeing the event.
8619+ */
8620+ if (!event_file)
8621+ free_event(event);
85138622 err_cpus:
85148623 put_online_cpus();
85158624 err_task:
@@ -8563,12 +8672,14 @@ perf_event_create_kernel_counter(struct perf_event_attr *attr, int cpu, · 47 unmodified lines
85638672
85648673 WARN_ON_ONCE(ctx->parent_ctx);
85658674 mutex_lock(&ctx->mutex);
8675+ if (ctx->task == TASK_TOMBSTONE) {
8676+ err = -ESRCH;
8677+ goto err_unlock;
8678+ }
8679+
85668680 if (!exclusive_event_installable(event, ctx)) {
8567− mutex_unlock(&ctx->mutex);
8568− perf_unpin_context(ctx);
8569− put_ctx(ctx);
85708681 err = -EBUSY;
8571− goto err_free;
8682+ goto err_unlock;
85728683 }
85738684
85748685 perf_install_in_context(ctx, event, cpu);
@@ -8577,6 +8688,10 @@ perf_event_create_kernel_counter(struct perf_event_attr *attr, int cpu, · 2 unmodified lines
85778688
85788689 return event;
85798690
8691+err_unlock:
8692+ mutex_unlock(&ctx->mutex);
8693+ perf_unpin_context(ctx);
8694+ put_ctx(ctx);
85808695 err_free:
85818696 free_event(event);
85828697 err:
@@ -8695,7 +8810,7 @@ perf_event_exit_event(struct perf_event *child_event, · 112 unmodified lines
86958810 if (parent_event)
86968811 perf_group_detach(child_event);
86978812 list_del_event(child_event, child_ctx);
8698− child_event->state = PERF_EVENT_STATE_EXIT; /* see perf_event_release_kernel() */
8813+ child_event->state = PERF_EVENT_STATE_EXIT; /* is_event_hup() */
86998814 raw_spin_unlock_irq(&child_ctx->lock);
87008815
87018816 /*
@@ -9313,9 +9428,6 @@ void __init perf_event_init(void) · 611 unmodified lines
93139428 ret = init_hw_breakpoint();
93149429 WARN(ret, "hw_breakpoint initialization failed with: %d", ret);
93159430
9316− /* do not patch jump label more than once per second */
9317− jump_label_rate_limit(&perf_sched_events, HZ);
9318−
93199431 /*
93209432 * Build time assertion that we keep the data_head at the intended
93219433 * location. IOW, validation we got the __reserved[] size right.

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t