History

Merge tag 'perf-urgent-2020-04-12' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull perf fixes from Thomas Gleixner: "Three fixes/updates for perf: - Fix the perf event cgroup tracking which tries to track the cgroup even for disabled events. - Add Ice Lake server support for uncore events - Disable pagefaults when retrieving the physical address in the sampling code" * tag 'perf-urgent-2020-04-12' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: perf/core: Disable page faults when getting phys address perf/x86/intel/uncore: Add Ice Lake server uncore support perf/cgroup: Correct indirection in perf_less_group_idx() perf/core: Fix event cgroup tracking

Linus Torvalds committed 6 years ago

Commit 20e2aa82 parents 652fa53 d3296fb4 files · +573 −31Browse files

Unified · Split

4 files
  1. arch/x86/events/intel/uncore.c +8 −0
  2. arch/x86/events/intel/uncore.h +3 −0
  3. arch/x86/events/intel/uncore_snbep.c +511 −0
  4. kernel/events/core.c +51 −31
arch/x86/events/intel/uncore.c+8−0
@@ -1476,6 +1476,12 @@ static const struct intel_uncore_init_fun tgl_l_uncore_init __initconst = { · 1475 unmodified lines
14761476 .mmio_init = tgl_l_uncore_mmio_init,
14771477 };
14781478
1479+static const struct intel_uncore_init_fun icx_uncore_init __initconst = {
1480+ .cpu_init = icx_uncore_cpu_init,
1481+ .pci_init = icx_uncore_pci_init,
1482+ .mmio_init = icx_uncore_mmio_init,
1483+};
1484+
14791485 static const struct intel_uncore_init_fun snr_uncore_init __initconst = {
14801486 .cpu_init = snr_uncore_cpu_init,
14811487 .pci_init = snr_uncore_pci_init,
@@ -1511,6 +1517,8 @@ static const struct x86_cpu_id intel_uncore_match[] __initconst = { · 29 unmodified lines
15111517 X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_L, &icl_uncore_init),
15121518 X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_NNPI, &icl_uncore_init),
15131519 X86_MATCH_INTEL_FAM6_MODEL(ICELAKE, &icl_uncore_init),
1520+ X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_D, &icx_uncore_init),
1521+ X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_X, &icx_uncore_init),
15141522 X86_MATCH_INTEL_FAM6_MODEL(TIGERLAKE_L, &tgl_l_uncore_init),
15151523 X86_MATCH_INTEL_FAM6_MODEL(TIGERLAKE, &tgl_uncore_init),
15161524 X86_MATCH_INTEL_FAM6_MODEL(ATOM_TREMONT_D, &snr_uncore_init),
arch/x86/events/intel/uncore.h+3−0
@@ -550,6 +550,9 @@ void skx_uncore_cpu_init(void); · 549 unmodified lines
550550 int snr_uncore_pci_init(void);
551551 void snr_uncore_cpu_init(void);
552552 void snr_uncore_mmio_init(void);
553+int icx_uncore_pci_init(void);
554+void icx_uncore_cpu_init(void);
555+void icx_uncore_mmio_init(void);
553556
554557 /* uncore_nhmex.c */
555558 void nhmex_uncore_cpu_init(void);
arch/x86/events/intel/uncore_snbep.c+511−0
@@ -382,6 +382,42 @@ · 381 unmodified lines
382382 #define SNR_IMC_MMIO_MEM0_OFFSET 0xd8
383383 #define SNR_IMC_MMIO_MEM0_MASK 0x7FF
384384
385+/* ICX CHA */
386+#define ICX_C34_MSR_PMON_CTR0 0xb68
387+#define ICX_C34_MSR_PMON_CTL0 0xb61
388+#define ICX_C34_MSR_PMON_BOX_CTL 0xb60
389+#define ICX_C34_MSR_PMON_BOX_FILTER0 0xb65
390+
391+/* ICX IIO */
392+#define ICX_IIO_MSR_PMON_CTL0 0xa58
393+#define ICX_IIO_MSR_PMON_CTR0 0xa51
394+#define ICX_IIO_MSR_PMON_BOX_CTL 0xa50
395+
396+/* ICX IRP */
397+#define ICX_IRP0_MSR_PMON_CTL0 0xa4d
398+#define ICX_IRP0_MSR_PMON_CTR0 0xa4b
399+#define ICX_IRP0_MSR_PMON_BOX_CTL 0xa4a
400+
401+/* ICX M2PCIE */
402+#define ICX_M2PCIE_MSR_PMON_CTL0 0xa46
403+#define ICX_M2PCIE_MSR_PMON_CTR0 0xa41
404+#define ICX_M2PCIE_MSR_PMON_BOX_CTL 0xa40
405+
406+/* ICX UPI */
407+#define ICX_UPI_PCI_PMON_CTL0 0x350
408+#define ICX_UPI_PCI_PMON_CTR0 0x320
409+#define ICX_UPI_PCI_PMON_BOX_CTL 0x318
410+#define ICX_UPI_CTL_UMASK_EXT 0xffffff
411+
412+/* ICX M3UPI*/
413+#define ICX_M3UPI_PCI_PMON_CTL0 0xd8
414+#define ICX_M3UPI_PCI_PMON_CTR0 0xa8
415+#define ICX_M3UPI_PCI_PMON_BOX_CTL 0xa0
416+
417+/* ICX IMC */
418+#define ICX_NUMBER_IMC_CHN 2
419+#define ICX_IMC_MEM_STRIDE 0x4
420+
385421 DEFINE_UNCORE_FORMAT_ATTR(event, event, "config:0-7");
386422 DEFINE_UNCORE_FORMAT_ATTR(event2, event, "config:0-6");
387423 DEFINE_UNCORE_FORMAT_ATTR(event_ext, event, "config:0-7,21");
@@ -390,6 +426,7 @@ DEFINE_UNCORE_FORMAT_ATTR(umask, umask, "config:8-15"); · 2 unmodified lines
390426 DEFINE_UNCORE_FORMAT_ATTR(umask_ext, umask, "config:8-15,32-43,45-55");
391427 DEFINE_UNCORE_FORMAT_ATTR(umask_ext2, umask, "config:8-15,32-57");
392428 DEFINE_UNCORE_FORMAT_ATTR(umask_ext3, umask, "config:8-15,32-39");
429+DEFINE_UNCORE_FORMAT_ATTR(umask_ext4, umask, "config:8-15,32-55");
393430 DEFINE_UNCORE_FORMAT_ATTR(qor, qor, "config:16");
394431 DEFINE_UNCORE_FORMAT_ATTR(edge, edge, "config:18");
395432 DEFINE_UNCORE_FORMAT_ATTR(tid_en, tid_en, "config:19");
@@ -4551,3 +4588,477 @@ void snr_uncore_mmio_init(void) · 4155 unmodified lines
45514588 }
45524589
45534590 /* end of SNR uncore support */
4591+
4592+/* ICX uncore support */
4593+
4594+static unsigned icx_cha_msr_offsets[] = {
4595+ 0x2a0, 0x2ae, 0x2bc, 0x2ca, 0x2d8, 0x2e6, 0x2f4, 0x302, 0x310,
4596+ 0x31e, 0x32c, 0x33a, 0x348, 0x356, 0x364, 0x372, 0x380, 0x38e,
4597+ 0x3aa, 0x3b8, 0x3c6, 0x3d4, 0x3e2, 0x3f0, 0x3fe, 0x40c, 0x41a,
4598+ 0x428, 0x436, 0x444, 0x452, 0x460, 0x46e, 0x47c, 0x0, 0xe,
4599+ 0x1c, 0x2a, 0x38, 0x46,
4600+};
4601+
4602+static int icx_cha_hw_config(struct intel_uncore_box *box, struct perf_event *event)
4603+{
4604+ struct hw_perf_event_extra *reg1 = &event->hw.extra_reg;
4605+ bool tie_en = !!(event->hw.config & SNBEP_CBO_PMON_CTL_TID_EN);
4606+
4607+ if (tie_en) {
4608+ reg1->reg = ICX_C34_MSR_PMON_BOX_FILTER0 +
4609+ icx_cha_msr_offsets[box->pmu->pmu_idx];
4610+ reg1->config = event->attr.config1 & SKX_CHA_MSR_PMON_BOX_FILTER_TID;
4611+ reg1->idx = 0;
4612+ }
4613+
4614+ return 0;
4615+}
4616+
4617+static struct intel_uncore_ops icx_uncore_chabox_ops = {
4618+ .init_box = ivbep_uncore_msr_init_box,
4619+ .disable_box = snbep_uncore_msr_disable_box,
4620+ .enable_box = snbep_uncore_msr_enable_box,
4621+ .disable_event = snbep_uncore_msr_disable_event,
4622+ .enable_event = snr_cha_enable_event,
4623+ .read_counter = uncore_msr_read_counter,
4624+ .hw_config = icx_cha_hw_config,
4625+};
4626+
4627+static struct intel_uncore_type icx_uncore_chabox = {
4628+ .name = "cha",
4629+ .num_counters = 4,
4630+ .perf_ctr_bits = 48,
4631+ .event_ctl = ICX_C34_MSR_PMON_CTL0,
4632+ .perf_ctr = ICX_C34_MSR_PMON_CTR0,
4633+ .box_ctl = ICX_C34_MSR_PMON_BOX_CTL,
4634+ .msr_offsets = icx_cha_msr_offsets,
4635+ .event_mask = HSWEP_S_MSR_PMON_RAW_EVENT_MASK,
4636+ .event_mask_ext = SNR_CHA_RAW_EVENT_MASK_EXT,
4637+ .constraints = skx_uncore_chabox_constraints,
4638+ .ops = &icx_uncore_chabox_ops,
4639+ .format_group = &snr_uncore_chabox_format_group,
4640+};
4641+
4642+static unsigned icx_msr_offsets[] = {
4643+ 0x0, 0x20, 0x40, 0x90, 0xb0, 0xd0,
4644+};
4645+
4646+static struct event_constraint icx_uncore_iio_constraints[] = {
4647+ UNCORE_EVENT_CONSTRAINT(0x02, 0x3),
4648+ UNCORE_EVENT_CONSTRAINT(0x03, 0x3),
4649+ UNCORE_EVENT_CONSTRAINT(0x83, 0x3),
4650+ UNCORE_EVENT_CONSTRAINT(0xc0, 0xc),
4651+ UNCORE_EVENT_CONSTRAINT(0xc5, 0xc),
4652+ EVENT_CONSTRAINT_END
4653+};
4654+
4655+static struct intel_uncore_type icx_uncore_iio = {
4656+ .name = "iio",
4657+ .num_counters = 4,
4658+ .num_boxes = 6,
4659+ .perf_ctr_bits = 48,
4660+ .event_ctl = ICX_IIO_MSR_PMON_CTL0,
4661+ .perf_ctr = ICX_IIO_MSR_PMON_CTR0,
4662+ .event_mask = SNBEP_PMON_RAW_EVENT_MASK,
4663+ .event_mask_ext = SNR_IIO_PMON_RAW_EVENT_MASK_EXT,
4664+ .box_ctl = ICX_IIO_MSR_PMON_BOX_CTL,
4665+ .msr_offsets = icx_msr_offsets,
4666+ .constraints = icx_uncore_iio_constraints,
4667+ .ops = &skx_uncore_iio_ops,
4668+ .format_group = &snr_uncore_iio_format_group,
4669+};
4670+
4671+static struct intel_uncore_type icx_uncore_irp = {
4672+ .name = "irp",
4673+ .num_counters = 2,
4674+ .num_boxes = 6,
4675+ .perf_ctr_bits = 48,
4676+ .event_ctl = ICX_IRP0_MSR_PMON_CTL0,
4677+ .perf_ctr = ICX_IRP0_MSR_PMON_CTR0,
4678+ .event_mask = SNBEP_PMON_RAW_EVENT_MASK,
4679+ .box_ctl = ICX_IRP0_MSR_PMON_BOX_CTL,
4680+ .msr_offsets = icx_msr_offsets,
4681+ .ops = &ivbep_uncore_msr_ops,
4682+ .format_group = &ivbep_uncore_format_group,
4683+};
4684+
4685+static struct event_constraint icx_uncore_m2pcie_constraints[] = {
4686+ UNCORE_EVENT_CONSTRAINT(0x14, 0x3),
4687+ UNCORE_EVENT_CONSTRAINT(0x23, 0x3),
4688+ UNCORE_EVENT_CONSTRAINT(0x2d, 0x3),
4689+ EVENT_CONSTRAINT_END
4690+};
4691+
4692+static struct intel_uncore_type icx_uncore_m2pcie = {
4693+ .name = "m2pcie",
4694+ .num_counters = 4,
4695+ .num_boxes = 6,
4696+ .perf_ctr_bits = 48,
4697+ .event_ctl = ICX_M2PCIE_MSR_PMON_CTL0,
4698+ .perf_ctr = ICX_M2PCIE_MSR_PMON_CTR0,
4699+ .box_ctl = ICX_M2PCIE_MSR_PMON_BOX_CTL,
4700+ .msr_offsets = icx_msr_offsets,
4701+ .constraints = icx_uncore_m2pcie_constraints,
4702+ .event_mask = SNBEP_PMON_RAW_EVENT_MASK,
4703+ .ops = &ivbep_uncore_msr_ops,
4704+ .format_group = &ivbep_uncore_format_group,
4705+};
4706+
4707+enum perf_uncore_icx_iio_freerunning_type_id {
4708+ ICX_IIO_MSR_IOCLK,
4709+ ICX_IIO_MSR_BW_IN,
4710+
4711+ ICX_IIO_FREERUNNING_TYPE_MAX,
4712+};
4713+
4714+static unsigned icx_iio_clk_freerunning_box_offsets[] = {
4715+ 0x0, 0x20, 0x40, 0x90, 0xb0, 0xd0,
4716+};
4717+
4718+static unsigned icx_iio_bw_freerunning_box_offsets[] = {
4719+ 0x0, 0x10, 0x20, 0x90, 0xa0, 0xb0,
4720+};
4721+
4722+static struct freerunning_counters icx_iio_freerunning[] = {
4723+ [ICX_IIO_MSR_IOCLK] = { 0xa55, 0x1, 0x20, 1, 48, icx_iio_clk_freerunning_box_offsets },
4724+ [ICX_IIO_MSR_BW_IN] = { 0xaa0, 0x1, 0x10, 8, 48, icx_iio_bw_freerunning_box_offsets },
4725+};
4726+
4727+static struct uncore_event_desc icx_uncore_iio_freerunning_events[] = {
4728+ /* Free-Running IIO CLOCKS Counter */
4729+ INTEL_UNCORE_EVENT_DESC(ioclk, "event=0xff,umask=0x10"),
4730+ /* Free-Running IIO BANDWIDTH IN Counters */
4731+ INTEL_UNCORE_EVENT_DESC(bw_in_port0, "event=0xff,umask=0x20"),
4732+ INTEL_UNCORE_EVENT_DESC(bw_in_port0.scale, "3.814697266e-6"),
4733+ INTEL_UNCORE_EVENT_DESC(bw_in_port0.unit, "MiB"),
4734+ INTEL_UNCORE_EVENT_DESC(bw_in_port1, "event=0xff,umask=0x21"),
4735+ INTEL_UNCORE_EVENT_DESC(bw_in_port1.scale, "3.814697266e-6"),
4736+ INTEL_UNCORE_EVENT_DESC(bw_in_port1.unit, "MiB"),
4737+ INTEL_UNCORE_EVENT_DESC(bw_in_port2, "event=0xff,umask=0x22"),
4738+ INTEL_UNCORE_EVENT_DESC(bw_in_port2.scale, "3.814697266e-6"),
4739+ INTEL_UNCORE_EVENT_DESC(bw_in_port2.unit, "MiB"),
4740+ INTEL_UNCORE_EVENT_DESC(bw_in_port3, "event=0xff,umask=0x23"),
4741+ INTEL_UNCORE_EVENT_DESC(bw_in_port3.scale, "3.814697266e-6"),
4742+ INTEL_UNCORE_EVENT_DESC(bw_in_port3.unit, "MiB"),
4743+ INTEL_UNCORE_EVENT_DESC(bw_in_port4, "event=0xff,umask=0x24"),
4744+ INTEL_UNCORE_EVENT_DESC(bw_in_port4.scale, "3.814697266e-6"),
4745+ INTEL_UNCORE_EVENT_DESC(bw_in_port4.unit, "MiB"),
4746+ INTEL_UNCORE_EVENT_DESC(bw_in_port5, "event=0xff,umask=0x25"),
4747+ INTEL_UNCORE_EVENT_DESC(bw_in_port5.scale, "3.814697266e-6"),
4748+ INTEL_UNCORE_EVENT_DESC(bw_in_port5.unit, "MiB"),
4749+ INTEL_UNCORE_EVENT_DESC(bw_in_port6, "event=0xff,umask=0x26"),
4750+ INTEL_UNCORE_EVENT_DESC(bw_in_port6.scale, "3.814697266e-6"),
4751+ INTEL_UNCORE_EVENT_DESC(bw_in_port6.unit, "MiB"),
4752+ INTEL_UNCORE_EVENT_DESC(bw_in_port7, "event=0xff,umask=0x27"),
4753+ INTEL_UNCORE_EVENT_DESC(bw_in_port7.scale, "3.814697266e-6"),
4754+ INTEL_UNCORE_EVENT_DESC(bw_in_port7.unit, "MiB"),
4755+ { /* end: all zeroes */ },
4756+};
4757+
4758+static struct intel_uncore_type icx_uncore_iio_free_running = {
4759+ .name = "iio_free_running",
4760+ .num_counters = 9,
4761+ .num_boxes = 6,
4762+ .num_freerunning_types = ICX_IIO_FREERUNNING_TYPE_MAX,
4763+ .freerunning = icx_iio_freerunning,
4764+ .ops = &skx_uncore_iio_freerunning_ops,
4765+ .event_descs = icx_uncore_iio_freerunning_events,
4766+ .format_group = &skx_uncore_iio_freerunning_format_group,
4767+};
4768+
4769+static struct intel_uncore_type *icx_msr_uncores[] = {
4770+ &skx_uncore_ubox,
4771+ &icx_uncore_chabox,
4772+ &icx_uncore_iio,
4773+ &icx_uncore_irp,
4774+ &icx_uncore_m2pcie,
4775+ &skx_uncore_pcu,
4776+ &icx_uncore_iio_free_running,
4777+ NULL,
4778+};
4779+
4780+/*
4781+ * To determine the number of CHAs, it should read CAPID6(Low) and CAPID7 (High)
4782+ * registers which located at Device 30, Function 3
4783+ */
4784+#define ICX_CAPID6 0x9c
4785+#define ICX_CAPID7 0xa0
4786+
4787+static u64 icx_count_chabox(void)
4788+{
4789+ struct pci_dev *dev = NULL;
4790+ u64 caps = 0;
4791+
4792+ dev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x345b, dev);
4793+ if (!dev)
4794+ goto out;
4795+
4796+ pci_read_config_dword(dev, ICX_CAPID6, (u32 *)&caps);
4797+ pci_read_config_dword(dev, ICX_CAPID7, (u32 *)&caps + 1);
4798+out:
4799+ pci_dev_put(dev);
4800+ return hweight64(caps);
4801+}
4802+
4803+void icx_uncore_cpu_init(void)
4804+{
4805+ u64 num_boxes = icx_count_chabox();
4806+
4807+ if (WARN_ON(num_boxes > ARRAY_SIZE(icx_cha_msr_offsets)))
4808+ return;
4809+ icx_uncore_chabox.num_boxes = num_boxes;
4810+ uncore_msr_uncores = icx_msr_uncores;
4811+}
4812+
4813+static struct intel_uncore_type icx_uncore_m2m = {
4814+ .name = "m2m",
4815+ .num_counters = 4,
4816+ .num_boxes = 4,
4817+ .perf_ctr_bits = 48,
4818+ .perf_ctr = SNR_M2M_PCI_PMON_CTR0,
4819+ .event_ctl = SNR_M2M_PCI_PMON_CTL0,
4820+ .event_mask = SNBEP_PMON_RAW_EVENT_MASK,
4821+ .box_ctl = SNR_M2M_PCI_PMON_BOX_CTL,
4822+ .ops = &snr_m2m_uncore_pci_ops,
4823+ .format_group = &skx_uncore_format_group,
4824+};
4825+
4826+static struct attribute *icx_upi_uncore_formats_attr[] = {
4827+ &format_attr_event.attr,
4828+ &format_attr_umask_ext4.attr,
4829+ &format_attr_edge.attr,
4830+ &format_attr_inv.attr,
4831+ &format_attr_thresh8.attr,
4832+ NULL,
4833+};
4834+
4835+static const struct attribute_group icx_upi_uncore_format_group = {
4836+ .name = "format",
4837+ .attrs = icx_upi_uncore_formats_attr,
4838+};
4839+
4840+static struct intel_uncore_type icx_uncore_upi = {
4841+ .name = "upi",
4842+ .num_counters = 4,
4843+ .num_boxes = 3,
4844+ .perf_ctr_bits = 48,
4845+ .perf_ctr = ICX_UPI_PCI_PMON_CTR0,
4846+ .event_ctl = ICX_UPI_PCI_PMON_CTL0,
4847+ .event_mask = SNBEP_PMON_RAW_EVENT_MASK,
4848+ .event_mask_ext = ICX_UPI_CTL_UMASK_EXT,
4849+ .box_ctl = ICX_UPI_PCI_PMON_BOX_CTL,
4850+ .ops = &skx_upi_uncore_pci_ops,
4851+ .format_group = &icx_upi_uncore_format_group,
4852+};
4853+
4854+static struct event_constraint icx_uncore_m3upi_constraints[] = {
4855+ UNCORE_EVENT_CONSTRAINT(0x1c, 0x1),
4856+ UNCORE_EVENT_CONSTRAINT(0x1d, 0x1),
4857+ UNCORE_EVENT_CONSTRAINT(0x1e, 0x1),
4858+ UNCORE_EVENT_CONSTRAINT(0x1f, 0x1),
4859+ UNCORE_EVENT_CONSTRAINT(0x40, 0x7),
4860+ UNCORE_EVENT_CONSTRAINT(0x4e, 0x7),
4861+ UNCORE_EVENT_CONSTRAINT(0x4f, 0x7),
4862+ UNCORE_EVENT_CONSTRAINT(0x50, 0x7),
4863+ EVENT_CONSTRAINT_END
4864+};
4865+
4866+static struct intel_uncore_type icx_uncore_m3upi = {
4867+ .name = "m3upi",
4868+ .num_counters = 4,
4869+ .num_boxes = 3,
4870+ .perf_ctr_bits = 48,
4871+ .perf_ctr = ICX_M3UPI_PCI_PMON_CTR0,
4872+ .event_ctl = ICX_M3UPI_PCI_PMON_CTL0,
4873+ .event_mask = SNBEP_PMON_RAW_EVENT_MASK,
4874+ .box_ctl = ICX_M3UPI_PCI_PMON_BOX_CTL,
4875+ .constraints = icx_uncore_m3upi_constraints,
4876+ .ops = &ivbep_uncore_pci_ops,
4877+ .format_group = &skx_uncore_format_group,
4878+};
4879+
4880+enum {
4881+ ICX_PCI_UNCORE_M2M,
4882+ ICX_PCI_UNCORE_UPI,
4883+ ICX_PCI_UNCORE_M3UPI,
4884+};
4885+
4886+static struct intel_uncore_type *icx_pci_uncores[] = {
4887+ [ICX_PCI_UNCORE_M2M] = &icx_uncore_m2m,
4888+ [ICX_PCI_UNCORE_UPI] = &icx_uncore_upi,
4889+ [ICX_PCI_UNCORE_M3UPI] = &icx_uncore_m3upi,
4890+ NULL,
4891+};
4892+
4893+static const struct pci_device_id icx_uncore_pci_ids[] = {
4894+ { /* M2M 0 */
4895+ PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x344a),
4896+ .driver_data = UNCORE_PCI_DEV_FULL_DATA(12, 0, ICX_PCI_UNCORE_M2M, 0),
4897+ },
4898+ { /* M2M 1 */
4899+ PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x344a),
4900+ .driver_data = UNCORE_PCI_DEV_FULL_DATA(13, 0, ICX_PCI_UNCORE_M2M, 1),
4901+ },
4902+ { /* M2M 2 */
4903+ PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x344a),
4904+ .driver_data = UNCORE_PCI_DEV_FULL_DATA(14, 0, ICX_PCI_UNCORE_M2M, 2),
4905+ },
4906+ { /* M2M 3 */
4907+ PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x344a),
4908+ .driver_data = UNCORE_PCI_DEV_FULL_DATA(15, 0, ICX_PCI_UNCORE_M2M, 3),
4909+ },
4910+ { /* UPI Link 0 */
4911+ PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x3441),
4912+ .driver_data = UNCORE_PCI_DEV_FULL_DATA(2, 1, ICX_PCI_UNCORE_UPI, 0),
4913+ },
4914+ { /* UPI Link 1 */
4915+ PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x3441),
4916+ .driver_data = UNCORE_PCI_DEV_FULL_DATA(3, 1, ICX_PCI_UNCORE_UPI, 1),
4917+ },
4918+ { /* UPI Link 2 */
4919+ PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x3441),
4920+ .driver_data = UNCORE_PCI_DEV_FULL_DATA(4, 1, ICX_PCI_UNCORE_UPI, 2),
4921+ },
4922+ { /* M3UPI Link 0 */
4923+ PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x3446),
4924+ .driver_data = UNCORE_PCI_DEV_FULL_DATA(5, 1, ICX_PCI_UNCORE_M3UPI, 0),
4925+ },
4926+ { /* M3UPI Link 1 */
4927+ PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x3446),
4928+ .driver_data = UNCORE_PCI_DEV_FULL_DATA(6, 1, ICX_PCI_UNCORE_M3UPI, 1),
4929+ },
4930+ { /* M3UPI Link 2 */
4931+ PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x3446),
4932+ .driver_data = UNCORE_PCI_DEV_FULL_DATA(7, 1, ICX_PCI_UNCORE_M3UPI, 2),
4933+ },
4934+ { /* end: all zeroes */ }
4935+};
4936+
4937+static struct pci_driver icx_uncore_pci_driver = {
4938+ .name = "icx_uncore",
4939+ .id_table = icx_uncore_pci_ids,
4940+};
4941+
4942+int icx_uncore_pci_init(void)
4943+{
4944+ /* ICX UBOX DID */
4945+ int ret = snbep_pci2phy_map_init(0x3450, SKX_CPUNODEID,
4946+ SKX_GIDNIDMAP, true);
4947+
4948+ if (ret)
4949+ return ret;
4950+
4951+ uncore_pci_uncores = icx_pci_uncores;
4952+ uncore_pci_driver = &icx_uncore_pci_driver;
4953+ return 0;
4954+}
4955+
4956+static void icx_uncore_imc_init_box(struct intel_uncore_box *box)
4957+{
4958+ unsigned int box_ctl = box->pmu->type->box_ctl +
4959+ box->pmu->type->mmio_offset * (box->pmu->pmu_idx % ICX_NUMBER_IMC_CHN);
4960+ int mem_offset = (box->pmu->pmu_idx / ICX_NUMBER_IMC_CHN) * ICX_IMC_MEM_STRIDE +
4961+ SNR_IMC_MMIO_MEM0_OFFSET;
4962+
4963+ __snr_uncore_mmio_init_box(box, box_ctl, mem_offset);
4964+}
4965+
4966+static struct intel_uncore_ops icx_uncore_mmio_ops = {
4967+ .init_box = icx_uncore_imc_init_box,
4968+ .exit_box = uncore_mmio_exit_box,
4969+ .disable_box = snr_uncore_mmio_disable_box,
4970+ .enable_box = snr_uncore_mmio_enable_box,
4971+ .disable_event = snr_uncore_mmio_disable_event,
4972+ .enable_event = snr_uncore_mmio_enable_event,
4973+ .read_counter = uncore_mmio_read_counter,
4974+};
4975+
4976+static struct intel_uncore_type icx_uncore_imc = {
4977+ .name = "imc",
4978+ .num_counters = 4,
4979+ .num_boxes = 8,
4980+ .perf_ctr_bits = 48,
4981+ .fixed_ctr_bits = 48,
4982+ .fixed_ctr = SNR_IMC_MMIO_PMON_FIXED_CTR,
4983+ .fixed_ctl = SNR_IMC_MMIO_PMON_FIXED_CTL,
4984+ .event_descs = hswep_uncore_imc_events,
4985+ .perf_ctr = SNR_IMC_MMIO_PMON_CTR0,
4986+ .event_ctl = SNR_IMC_MMIO_PMON_CTL0,
4987+ .event_mask = SNBEP_PMON_RAW_EVENT_MASK,
4988+ .box_ctl = SNR_IMC_MMIO_PMON_BOX_CTL,
4989+ .mmio_offset = SNR_IMC_MMIO_OFFSET,
4990+ .ops = &icx_uncore_mmio_ops,
4991+ .format_group = &skx_uncore_format_group,
4992+};
4993+
4994+enum perf_uncore_icx_imc_freerunning_type_id {
4995+ ICX_IMC_DCLK,
4996+ ICX_IMC_DDR,
4997+ ICX_IMC_DDRT,
4998+
4999+ ICX_IMC_FREERUNNING_TYPE_MAX,
5000+};
5001+
5002+static struct freerunning_counters icx_imc_freerunning[] = {
5003+ [ICX_IMC_DCLK] = { 0x22b0, 0x0, 0, 1, 48 },
5004+ [ICX_IMC_DDR] = { 0x2290, 0x8, 0, 2, 48 },
5005+ [ICX_IMC_DDRT] = { 0x22a0, 0x8, 0, 2, 48 },
5006+};
5007+
5008+static struct uncore_event_desc icx_uncore_imc_freerunning_events[] = {
5009+ INTEL_UNCORE_EVENT_DESC(dclk, "event=0xff,umask=0x10"),
5010+
5011+ INTEL_UNCORE_EVENT_DESC(read, "event=0xff,umask=0x20"),
5012+ INTEL_UNCORE_EVENT_DESC(read.scale, "3.814697266e-6"),
5013+ INTEL_UNCORE_EVENT_DESC(read.unit, "MiB"),
5014+ INTEL_UNCORE_EVENT_DESC(write, "event=0xff,umask=0x21"),
5015+ INTEL_UNCORE_EVENT_DESC(write.scale, "3.814697266e-6"),
5016+ INTEL_UNCORE_EVENT_DESC(write.unit, "MiB"),
5017+
5018+ INTEL_UNCORE_EVENT_DESC(ddrt_read, "event=0xff,umask=0x30"),
5019+ INTEL_UNCORE_EVENT_DESC(ddrt_read.scale, "3.814697266e-6"),
5020+ INTEL_UNCORE_EVENT_DESC(ddrt_read.unit, "MiB"),
5021+ INTEL_UNCORE_EVENT_DESC(ddrt_write, "event=0xff,umask=0x31"),
5022+ INTEL_UNCORE_EVENT_DESC(ddrt_write.scale, "3.814697266e-6"),
5023+ INTEL_UNCORE_EVENT_DESC(ddrt_write.unit, "MiB"),
5024+ { /* end: all zeroes */ },
5025+};
5026+
5027+static void icx_uncore_imc_freerunning_init_box(struct intel_uncore_box *box)
5028+{
5029+ int mem_offset = box->pmu->pmu_idx * ICX_IMC_MEM_STRIDE +
5030+ SNR_IMC_MMIO_MEM0_OFFSET;
5031+
5032+ __snr_uncore_mmio_init_box(box, uncore_mmio_box_ctl(box), mem_offset);
5033+}
5034+
5035+static struct intel_uncore_ops icx_uncore_imc_freerunning_ops = {
5036+ .init_box = icx_uncore_imc_freerunning_init_box,
5037+ .exit_box = uncore_mmio_exit_box,
5038+ .read_counter = uncore_mmio_read_counter,
5039+ .hw_config = uncore_freerunning_hw_config,
5040+};
5041+
5042+static struct intel_uncore_type icx_uncore_imc_free_running = {
5043+ .name = "imc_free_running",
5044+ .num_counters = 5,
5045+ .num_boxes = 4,
5046+ .num_freerunning_types = ICX_IMC_FREERUNNING_TYPE_MAX,
5047+ .freerunning = icx_imc_freerunning,
5048+ .ops = &icx_uncore_imc_freerunning_ops,
5049+ .event_descs = icx_uncore_imc_freerunning_events,
5050+ .format_group = &skx_uncore_iio_freerunning_format_group,
5051+};
5052+
5053+static struct intel_uncore_type *icx_mmio_uncores[] = {
5054+ &icx_uncore_imc,
5055+ &icx_uncore_imc_free_running,
5056+ NULL,
5057+};
5058+
5059+void icx_uncore_mmio_init(void)
5060+{
5061+ uncore_mmio_uncores = icx_mmio_uncores;
5062+}
5063+
5064+/* end of ICX uncore support */
kernel/events/core.c+51−31
@@ -983,16 +983,10 @@ perf_cgroup_set_shadow_time(struct perf_event *event, u64 now) · 982 unmodified lines
983983 event->shadow_ctx_time = now - t->timestamp;
984984 }
985985
986−/*
987− * Update cpuctx->cgrp so that it is set when first cgroup event is added and
988− * cleared when last cgroup event is removed.
989− */
990986 static inline void
991−list_update_cgroup_event(struct perf_event *event,
992− struct perf_event_context *ctx, bool add)
987+perf_cgroup_event_enable(struct perf_event *event, struct perf_event_context *ctx)
993988 {
994989 struct perf_cpu_context *cpuctx;
995− struct list_head *cpuctx_entry;
996990
997991 if (!is_cgroup_event(event))
998992 return;
@@ -1009,28 +1003,41 @@ list_update_cgroup_event(struct perf_event *event, · 10 unmodified lines
10091003 * because if the first would mismatch, the second would not try again
10101004 * and we would leave cpuctx->cgrp unset.
10111005 */
1012− if (add && !cpuctx->cgrp) {
1006+ if (ctx->is_active && !cpuctx->cgrp) {
10131007 struct perf_cgroup *cgrp = perf_cgroup_from_task(current, ctx);
10141008
10151009 if (cgroup_is_descendant(cgrp->css.cgroup, event->cgrp->css.cgroup))
10161010 cpuctx->cgrp = cgrp;
10171011 }
10181012
1019− if (add && ctx->nr_cgroups++)
1013+ if (ctx->nr_cgroups++)
10201014 return;
1021− else if (!add && --ctx->nr_cgroups)
1015+
1016+ list_add(&cpuctx->cgrp_cpuctx_entry,
1017+ per_cpu_ptr(&cgrp_cpuctx_list, event->cpu));
1018+}
1019+
1020+static inline void
1021+perf_cgroup_event_disable(struct perf_event *event, struct perf_event_context *ctx)
1022+{
1023+ struct perf_cpu_context *cpuctx;
1024+
1025+ if (!is_cgroup_event(event))
1026+ return;
1027+
1028+ /*
1029+ * Because cgroup events are always per-cpu events,
1030+ * @ctx == &cpuctx->ctx.
1031+ */
1032+ cpuctx = container_of(ctx, struct perf_cpu_context, ctx);
1033+
1034+ if (--ctx->nr_cgroups)
10221035 return;
10231036
1024− /* no cgroup running */
1025− if (!add)
1037+ if (ctx->is_active && cpuctx->cgrp)
10261038 cpuctx->cgrp = NULL;
10271039
1028− cpuctx_entry = &cpuctx->cgrp_cpuctx_entry;
1029− if (add)
1030− list_add(cpuctx_entry,
1031− per_cpu_ptr(&cgrp_cpuctx_list, event->cpu));
1032− else
1033− list_del(cpuctx_entry);
1040+ list_del(&cpuctx->cgrp_cpuctx_entry);
10341041 }
10351042
10361043 #else /* !CONFIG_CGROUP_PERF */
@@ -1096,11 +1103,14 @@ static inline u64 perf_cgroup_event_time(struct perf_event *event) · 59 unmodified lines
10961103 }
10971104
10981105 static inline void
1099−list_update_cgroup_event(struct perf_event *event,
1100− struct perf_event_context *ctx, bool add)
1106+perf_cgroup_event_enable(struct perf_event *event, struct perf_event_context *ctx)
11011107 {
11021108 }
11031109
1110+static inline void
1111+perf_cgroup_event_disable(struct perf_event *event, struct perf_event_context *ctx)
1112+{
1113+}
11041114 #endif
11051115
11061116 /*
@@ -1791,13 +1801,14 @@ list_add_event(struct perf_event *event, struct perf_event_context *ctx) · 684 unmodified lines
17911801 add_event_to_groups(event, ctx);
17921802 }
17931803
1794− list_update_cgroup_event(event, ctx, true);
1795−
17961804 list_add_rcu(&event->event_entry, &ctx->event_list);
17971805 ctx->nr_events++;
17981806 if (event->attr.inherit_stat)
17991807 ctx->nr_stat++;
18001808
1809+ if (event->state > PERF_EVENT_STATE_OFF)
1810+ perf_cgroup_event_enable(event, ctx);
1811+
18011812 ctx->generation++;
18021813 }
18031814
@@ -1976,8 +1987,6 @@ list_del_event(struct perf_event *event, struct perf_event_context *ctx) · 172 unmodified lines
19761987
19771988 event->attach_state &= ~PERF_ATTACH_CONTEXT;
19781989
1979− list_update_cgroup_event(event, ctx, false);
1980−
19811990 ctx->nr_events--;
19821991 if (event->attr.inherit_stat)
19831992 ctx->nr_stat--;
@@ -1994,8 +2003,10 @@ list_del_event(struct perf_event *event, struct perf_event_context *ctx) · 10 unmodified lines
19942003 * of error state is by explicit re-enabling
19952004 * of the event
19962005 */
1997− if (event->state > PERF_EVENT_STATE_OFF)
2006+ if (event->state > PERF_EVENT_STATE_OFF) {
2007+ perf_cgroup_event_disable(event, ctx);
19982008 perf_event_set_state(event, PERF_EVENT_STATE_OFF);
2009+ }
19992010
20002011 ctx->generation++;
20012012 }
@@ -2226,6 +2237,7 @@ event_sched_out(struct perf_event *event, · 224 unmodified lines
22262237
22272238 if (READ_ONCE(event->pending_disable) >= 0) {
22282239 WRITE_ONCE(event->pending_disable, -1);
2240+ perf_cgroup_event_disable(event, ctx);
22292241 state = PERF_EVENT_STATE_OFF;
22302242 }
22312243 perf_event_set_state(event, state);
@@ -2363,6 +2375,7 @@ static void __perf_event_disable(struct perf_event *event, · 131 unmodified lines
23632375 event_sched_out(event, cpuctx, ctx);
23642376
23652377 perf_event_set_state(event, PERF_EVENT_STATE_OFF);
2378+ perf_cgroup_event_disable(event, ctx);
23662379 }
23672380
23682381 /*
@@ -2746,7 +2759,7 @@ static int __perf_install_in_context(void *info) · 377 unmodified lines
27462759 }
27472760
27482761 #ifdef CONFIG_CGROUP_PERF
2749− if (is_cgroup_event(event)) {
2762+ if (event->state > PERF_EVENT_STATE_OFF && is_cgroup_event(event)) {
27502763 /*
27512764 * If the current cgroup doesn't match the event's
27522765 * cgroup, we should not try to schedule it.
@@ -2906,6 +2919,7 @@ static void __perf_event_enable(struct perf_event *event, · 153 unmodified lines
29062919 ctx_sched_out(ctx, cpuctx, EVENT_TIME);
29072920
29082921 perf_event_set_state(event, PERF_EVENT_STATE_INACTIVE);
2922+ perf_cgroup_event_enable(event, ctx);
29092923
29102924 if (!ctx->is_active)
29112925 return;
@@ -3508,7 +3522,8 @@ static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx, · 596 unmodified lines
35083522
35093523 static bool perf_less_group_idx(const void *l, const void *r)
35103524 {
3511− const struct perf_event *le = l, *re = r;
3525+ const struct perf_event *le = *(const struct perf_event **)l;
3526+ const struct perf_event *re = *(const struct perf_event **)r;
35123527
35133528 return le->group_index < re->group_index;
35143529 }
@@ -3616,8 +3631,10 @@ static int merge_sched_in(struct perf_event *event, void *data) · 101 unmodified lines
36163631 }
36173632
36183633 if (event->state == PERF_EVENT_STATE_INACTIVE) {
3619− if (event->attr.pinned)
3634+ if (event->attr.pinned) {
3635+ perf_cgroup_event_disable(event, ctx);
36203636 perf_event_set_state(event, PERF_EVENT_STATE_ERROR);
3637+ }
36213638
36223639 *can_add_hw = 0;
36233640 ctx->rotate_necessary = 1;
@@ -6917,9 +6934,12 @@ static u64 perf_virt_to_phys(u64 virt) · 3293 unmodified lines
69176934 * Try IRQ-safe __get_user_pages_fast first.
69186935 * If failed, leave phys_addr as 0.
69196936 */
6920− if ((current->mm != NULL) &&
6921− (__get_user_pages_fast(virt, 1, 0, &p) == 1))
6922− phys_addr = page_to_phys(p) + virt % PAGE_SIZE;
6937+ if (current->mm != NULL) {
6938+ pagefault_disable();
6939+ if (__get_user_pages_fast(virt, 1, 0, &p) == 1)
6940+ phys_addr = page_to_phys(p) + virt % PAGE_SIZE;
6941+ pagefault_enable();
6942+ }
69236943
69246944 if (p)
69256945 put_page(p);

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t