History

perf/x86: Improve HT workaround GP counter constraint

The (SNB/IVB/HSW) HT bug only affects events that can be programmed onto GP counters, therefore we should only limit the number of GP counters that can be used per cpu -- iow we should not constrain the FP counters. Furthermore, we should only enfore such a limit when there are in fact exclusive events being scheduled on either sibling.

Peter Zijlstra committed 11 years ago

Commit cc1790c1 parent b371b594 files · +53 −30Browse files

Unified · Split

4 files
  1. arch/x86/kernel/cpu/perf_event.c +30 −6
  2. arch/x86/kernel/cpu/perf_event.h +12 −3
  3. arch/x86/kernel/cpu/perf_event_intel.c +10 −20
  4. arch/x86/kernel/cpu/perf_event_intel_uncore.c +1 −1
arch/x86/kernel/cpu/perf_event.c+30−6
@@ -611,6 +611,7 @@ struct sched_state { · 610 unmodified lines
611611 int event; /* event index */
612612 int counter; /* counter index */
613613 int unassigned; /* number of events to be assigned left */
614+ int nr_gp; /* number of GP counters used */
614615 unsigned long used[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
615616 };
616617
@@ -620,9 +621,10 @@ struct sched_state { · 3 unmodified lines
620621 struct perf_sched {
621622 int max_weight;
622623 int max_events;
624+ int max_gp;
625+ int saved_states;
623626 struct event_constraint **constraints;
624627 struct sched_state state;
625− int saved_states;
626628 struct sched_state saved[SCHED_STATES_MAX];
627629 };
628630
@@ -630,13 +632,14 @@ struct perf_sched { · 1 unmodified line
630632 * Initialize interator that runs through all events and counters.
631633 */
632634 static void perf_sched_init(struct perf_sched *sched, struct event_constraint **constraints,
633− int num, int wmin, int wmax)
635+ int num, int wmin, int wmax, int gpmax)
634636 {
635637 int idx;
636638
637639 memset(sched, 0, sizeof(*sched));
638640 sched->max_events = num;
639641 sched->max_weight = wmax;
642+ sched->max_gp = gpmax;
640643 sched->constraints = constraints;
641644
642645 for (idx = 0; idx < num; idx++) {
@@ -696,11 +699,16 @@ static bool __perf_sched_find_counter(struct perf_sched *sched) · 53 unmodified lines
696699 goto done;
697700 }
698701 }
702+
699703 /* Grab the first unused counter starting with idx */
700704 idx = sched->state.counter;
701705 for_each_set_bit_from(idx, c->idxmsk, INTEL_PMC_IDX_FIXED) {
702− if (!__test_and_set_bit(idx, sched->state.used))
706+ if (!__test_and_set_bit(idx, sched->state.used)) {
707+ if (sched->state.nr_gp++ >= sched->max_gp)
708+ return false;
709+
703710 goto done;
711+ }
704712 }
705713
706714 return false;
@@ -757,11 +765,11 @@ static bool perf_sched_next_event(struct perf_sched *sched) · 50 unmodified lines
757765 * Assign a counter for each event.
758766 */
759767 int perf_assign_events(struct event_constraint **constraints, int n,
760− int wmin, int wmax, int *assign)
768+ int wmin, int wmax, int gpmax, int *assign)
761769 {
762770 struct perf_sched sched;
763771
764− perf_sched_init(&sched, constraints, n, wmin, wmax);
772+ perf_sched_init(&sched, constraints, n, wmin, wmax, gpmax);
765773
766774 do {
767775 if (!perf_sched_find_counter(&sched))
@@ -822,8 +830,24 @@ int x86_schedule_events(struct cpu_hw_events *cpuc, int n, int *assign) · 54 unmodified lines
822830
823831 /* slow path */
824832 if (i != n) {
833+ int gpmax = x86_pmu.num_counters;
834+
835+ /*
836+ * Do not allow scheduling of more than half the available
837+ * generic counters.
838+ *
839+ * This helps avoid counter starvation of sibling thread by
840+ * ensuring at most half the counters cannot be in exclusive
841+ * mode. There is no designated counters for the limits. Any
842+ * N/2 counters can be used. This helps with events with
843+ * specific counter constraints.
844+ */
845+ if (is_ht_workaround_enabled() && !cpuc->is_fake &&
846+ READ_ONCE(cpuc->excl_cntrs->exclusive_present))
847+ gpmax /= 2;
848+
825849 unsched = perf_assign_events(cpuc->event_constraint, n, wmin,
826− wmax, assign);
850+ wmax, gpmax, assign);
827851 }
828852
829853 /*
arch/x86/kernel/cpu/perf_event.h+12−3
@@ -74,6 +74,7 @@ struct event_constraint { · 73 unmodified lines
7474 #define PERF_X86_EVENT_EXCL 0x0040 /* HT exclusivity on counter */
7575 #define PERF_X86_EVENT_DYNAMIC 0x0080 /* dynamic alloc'd constraint */
7676 #define PERF_X86_EVENT_RDPMC_ALLOWED 0x0100 /* grant rdpmc permission */
77+#define PERF_X86_EVENT_EXCL_ACCT 0x0200 /* accounted EXCL event */
7778
7879
7980 struct amd_nb {
@@ -134,8 +135,6 @@ enum intel_excl_state_type { · 54 unmodified lines
134135 struct intel_excl_states {
135136 enum intel_excl_state_type init_state[X86_PMC_IDX_MAX];
136137 enum intel_excl_state_type state[X86_PMC_IDX_MAX];
137− int num_alloc_cntrs;/* #counters allocated */
138− int max_alloc_cntrs;/* max #counters allowed */
139138 bool sched_started; /* true if scheduling has started */
140139 };
141140
@@ -144,6 +143,11 @@ struct intel_excl_cntrs { · 2 unmodified lines
144143
145144 struct intel_excl_states states[2];
146145
146+ union {
147+ u16 has_exclusive[2];
148+ u32 exclusive_present;
149+ };
150+
147151 int refcnt; /* per-core: #HT threads */
148152 unsigned core_id; /* per-core: core id */
149153 };
@@ -176,6 +180,7 @@ struct cpu_hw_events { · 26 unmodified lines
176180 struct perf_event *event_list[X86_PMC_IDX_MAX]; /* in enabled order */
177181 struct event_constraint *event_constraint[X86_PMC_IDX_MAX];
178182
183+ int n_excl; /* the number of exclusive events */
179184
180185 unsigned int group_flag;
181186 int is_fake;
@@ -719,7 +724,7 @@ static inline void __x86_pmu_enable_event(struct hw_perf_event *hwc, · 537 unmodified lines
719724 void x86_pmu_enable_all(int added);
720725
721726 int perf_assign_events(struct event_constraint **constraints, int n,
722− int wmin, int wmax, int *assign);
727+ int wmin, int wmax, int gpmax, int *assign);
723728 int x86_schedule_events(struct cpu_hw_events *cpuc, int n, int *assign);
724729
725730 void x86_pmu_stop(struct perf_event *event, int flags);
@@ -930,4 +935,8 @@ static inline struct intel_shared_regs *allocate_shared_regs(int cpu) · 204 unmodified lines
930935 return NULL;
931936 }
932937
938+static inline int is_ht_workaround_enabled(void)
939+{
940+ return 0;
941+}
933942 #endif /* CONFIG_CPU_SUP_INTEL */
arch/x86/kernel/cpu/perf_event_intel.c+10−20
@@ -1923,7 +1923,6 @@ intel_start_scheduling(struct cpu_hw_events *cpuc) · 1922 unmodified lines
19231923 xl = &excl_cntrs->states[tid];
19241924
19251925 xl->sched_started = true;
1926− xl->num_alloc_cntrs = 0;
19271926 /*
19281927 * lock shared state until we are done scheduling
19291928 * in stop_event_scheduling()
@@ -2000,6 +1999,11 @@ intel_get_excl_constraints(struct cpu_hw_events *cpuc, struct perf_event *event, · 70 unmodified lines
20001999 * across HT threads
20012000 */
20022001 is_excl = c->flags & PERF_X86_EVENT_EXCL;
2002+ if (is_excl && !(event->hw.flags & PERF_X86_EVENT_EXCL_ACCT)) {
2003+ event->hw.flags |= PERF_X86_EVENT_EXCL_ACCT;
2004+ if (!cpuc->n_excl++)
2005+ WRITE_ONCE(excl_cntrs->has_exclusive[tid], 1);
2006+ }
20032007
20042008 /*
20052009 * xl = state of current HT
@@ -2008,18 +2012,6 @@ intel_get_excl_constraints(struct cpu_hw_events *cpuc, struct perf_event *event, · 2 unmodified lines
20082012 xl = &excl_cntrs->states[tid];
20092013 xlo = &excl_cntrs->states[o_tid];
20102014
2011− /*
2012− * do not allow scheduling of more than max_alloc_cntrs
2013− * which is set to half the available generic counters.
2014− * this helps avoid counter starvation of sibling thread
2015− * by ensuring at most half the counters cannot be in
2016− * exclusive mode. There is not designated counters for the
2017− * limits. Any N/2 counters can be used. This helps with
2018− * events with specifix counter constraints
2019− */
2020− if (xl->num_alloc_cntrs++ == xl->max_alloc_cntrs)
2021− return &emptyconstraint;
2022−
20232015 cx = c;
20242016
20252017 /*
@@ -2150,6 +2142,11 @@ static void intel_put_excl_constraints(struct cpu_hw_events *cpuc, · 124 unmodified lines
21502142
21512143 xl = &excl_cntrs->states[tid];
21522144 xlo = &excl_cntrs->states[o_tid];
2145+ if (hwc->flags & PERF_X86_EVENT_EXCL_ACCT) {
2146+ hwc->flags &= ~PERF_X86_EVENT_EXCL_ACCT;
2147+ if (!--cpuc->n_excl)
2148+ WRITE_ONCE(excl_cntrs->has_exclusive[tid], 0);
2149+ }
21532150
21542151 /*
21552152 * put_constraint may be called from x86_schedule_events()
@@ -2632,8 +2629,6 @@ static void intel_pmu_cpu_starting(int cpu) · 476 unmodified lines
26322629 cpuc->lbr_sel = &cpuc->shared_regs->regs[EXTRA_REG_LBR];
26332630
26342631 if (x86_pmu.flags & PMU_FL_EXCL_CNTRS) {
2635− int h = x86_pmu.num_counters >> 1;
2636−
26372632 for_each_cpu(i, topology_thread_cpumask(cpu)) {
26382633 struct intel_excl_cntrs *c;
26392634
@@ -2647,11 +2642,6 @@ static void intel_pmu_cpu_starting(int cpu) · 7 unmodified lines
26472642 }
26482643 cpuc->excl_cntrs->core_id = core_id;
26492644 cpuc->excl_cntrs->refcnt++;
2650− /*
2651− * set hard limit to half the number of generic counters
2652− */
2653− cpuc->excl_cntrs->states[0].max_alloc_cntrs = h;
2654− cpuc->excl_cntrs->states[1].max_alloc_cntrs = h;
26552645 }
26562646 }
26572647
arch/x86/kernel/cpu/perf_event_intel_uncore.c+1−1
@@ -395,7 +395,7 @@ static int uncore_assign_events(struct intel_uncore_box *box, int assign[], int · 394 unmodified lines
395395 /* slow path */
396396 if (i != n)
397397 ret = perf_assign_events(box->event_constraint, n,
398− wmin, wmax, assign);
398+ wmin, wmax, n, assign);
399399
400400 if (!assign || ret) {
401401 for (i = 0; i < n; i++)