History

Merge tag 'perf-urgent-2020-04-12' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull perf fixes from Thomas Gleixner: "Three fixes/updates for perf: - Fix the perf event cgroup tracking which tries to track the cgroup even for disabled events. - Add Ice Lake server support for uncore events - Disable pagefaults when retrieving the physical address in the sampling code" * tag 'perf-urgent-2020-04-12' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: perf/core: Disable page faults when getting phys address perf/x86/intel/uncore: Add Ice Lake server uncore support perf/cgroup: Correct indirection in perf_less_group_idx() perf/core: Fix event cgroup tracking

Linus Torvalds committed 6 years ago

Commit 20e2aa82 parents 652fa53 d3296fb4 files · +573 −31Browse files

Unified · Split

4 files
  1. arch/x86/events/intel/uncore.c +8 −0
  2. arch/x86/events/intel/uncore.h +3 −0
  3. arch/x86/events/intel/uncore_snbep.c +511 −0
  4. kernel/events/core.c +51 −31
arch/x86/events/intel/uncore.c+8−0
@@ -1476,6 +1476,12 @@ static const struct intel_uncore_init_fun tgl_l_uncore_init __initconst = { · 1475 unmodified lines
1476 .mmio_init = tgl_l_uncore_mmio_init,1476 .mmio_init = tgl_l_uncore_mmio_init,
1477};1477};
1478 1478
1479static const struct intel_uncore_init_fun icx_uncore_init __initconst = {
1480 .cpu_init = icx_uncore_cpu_init,
1481 .pci_init = icx_uncore_pci_init,
1482 .mmio_init = icx_uncore_mmio_init,
1483};
1484
1479static const struct intel_uncore_init_fun snr_uncore_init __initconst = {1485static const struct intel_uncore_init_fun snr_uncore_init __initconst = {
1480 .cpu_init = snr_uncore_cpu_init,1486 .cpu_init = snr_uncore_cpu_init,
1481 .pci_init = snr_uncore_pci_init,1487 .pci_init = snr_uncore_pci_init,
@@ -1511,6 +1517,8 @@ static const struct x86_cpu_id intel_uncore_match[] __initconst = { · 29 unmodified lines
1511 X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_L, &icl_uncore_init),1517 X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_L, &icl_uncore_init),
1512 X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_NNPI, &icl_uncore_init),1518 X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_NNPI, &icl_uncore_init),
1513 X86_MATCH_INTEL_FAM6_MODEL(ICELAKE, &icl_uncore_init),1519 X86_MATCH_INTEL_FAM6_MODEL(ICELAKE, &icl_uncore_init),
1520 X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_D, &icx_uncore_init),
1521 X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_X, &icx_uncore_init),
1514 X86_MATCH_INTEL_FAM6_MODEL(TIGERLAKE_L, &tgl_l_uncore_init),1522 X86_MATCH_INTEL_FAM6_MODEL(TIGERLAKE_L, &tgl_l_uncore_init),
1515 X86_MATCH_INTEL_FAM6_MODEL(TIGERLAKE, &tgl_uncore_init),1523 X86_MATCH_INTEL_FAM6_MODEL(TIGERLAKE, &tgl_uncore_init),
1516 X86_MATCH_INTEL_FAM6_MODEL(ATOM_TREMONT_D, &snr_uncore_init),1524 X86_MATCH_INTEL_FAM6_MODEL(ATOM_TREMONT_D, &snr_uncore_init),
arch/x86/events/intel/uncore.h+3−0
@@ -550,6 +550,9 @@ void skx_uncore_cpu_init(void); · 549 unmodified lines
550int snr_uncore_pci_init(void);550int snr_uncore_pci_init(void);
551void snr_uncore_cpu_init(void);551void snr_uncore_cpu_init(void);
552void snr_uncore_mmio_init(void);552void snr_uncore_mmio_init(void);
553int icx_uncore_pci_init(void);
554void icx_uncore_cpu_init(void);
555void icx_uncore_mmio_init(void);
553 556
554/* uncore_nhmex.c */557/* uncore_nhmex.c */
555void nhmex_uncore_cpu_init(void);558void nhmex_uncore_cpu_init(void);
arch/x86/events/intel/uncore_snbep.c+511−0
@@ -382,6 +382,42 @@ · 381 unmodified lines
382#define SNR_IMC_MMIO_MEM0_OFFSET 0xd8382#define SNR_IMC_MMIO_MEM0_OFFSET 0xd8
383#define SNR_IMC_MMIO_MEM0_MASK 0x7FF383#define SNR_IMC_MMIO_MEM0_MASK 0x7FF
384 384
385/* ICX CHA */
386#define ICX_C34_MSR_PMON_CTR0 0xb68
387#define ICX_C34_MSR_PMON_CTL0 0xb61
388#define ICX_C34_MSR_PMON_BOX_CTL 0xb60
389#define ICX_C34_MSR_PMON_BOX_FILTER0 0xb65
390
391/* ICX IIO */
392#define ICX_IIO_MSR_PMON_CTL0 0xa58
393#define ICX_IIO_MSR_PMON_CTR0 0xa51
394#define ICX_IIO_MSR_PMON_BOX_CTL 0xa50
395
396/* ICX IRP */
397#define ICX_IRP0_MSR_PMON_CTL0 0xa4d
398#define ICX_IRP0_MSR_PMON_CTR0 0xa4b
399#define ICX_IRP0_MSR_PMON_BOX_CTL 0xa4a
400
401/* ICX M2PCIE */
402#define ICX_M2PCIE_MSR_PMON_CTL0 0xa46
403#define ICX_M2PCIE_MSR_PMON_CTR0 0xa41
404#define ICX_M2PCIE_MSR_PMON_BOX_CTL 0xa40
405
406/* ICX UPI */
407#define ICX_UPI_PCI_PMON_CTL0 0x350
408#define ICX_UPI_PCI_PMON_CTR0 0x320
409#define ICX_UPI_PCI_PMON_BOX_CTL 0x318
410#define ICX_UPI_CTL_UMASK_EXT 0xffffff
411
412/* ICX M3UPI*/
413#define ICX_M3UPI_PCI_PMON_CTL0 0xd8
414#define ICX_M3UPI_PCI_PMON_CTR0 0xa8
415#define ICX_M3UPI_PCI_PMON_BOX_CTL 0xa0
416
417/* ICX IMC */
418#define ICX_NUMBER_IMC_CHN 2
419#define ICX_IMC_MEM_STRIDE 0x4
420
385DEFINE_UNCORE_FORMAT_ATTR(event, event, "config:0-7");421DEFINE_UNCORE_FORMAT_ATTR(event, event, "config:0-7");
386DEFINE_UNCORE_FORMAT_ATTR(event2, event, "config:0-6");422DEFINE_UNCORE_FORMAT_ATTR(event2, event, "config:0-6");
387DEFINE_UNCORE_FORMAT_ATTR(event_ext, event, "config:0-7,21");423DEFINE_UNCORE_FORMAT_ATTR(event_ext, event, "config:0-7,21");
@@ -390,6 +426,7 @@ DEFINE_UNCORE_FORMAT_ATTR(umask, umask, "config:8-15"); · 2 unmodified lines
390DEFINE_UNCORE_FORMAT_ATTR(umask_ext, umask, "config:8-15,32-43,45-55");426DEFINE_UNCORE_FORMAT_ATTR(umask_ext, umask, "config:8-15,32-43,45-55");
391DEFINE_UNCORE_FORMAT_ATTR(umask_ext2, umask, "config:8-15,32-57");427DEFINE_UNCORE_FORMAT_ATTR(umask_ext2, umask, "config:8-15,32-57");
392DEFINE_UNCORE_FORMAT_ATTR(umask_ext3, umask, "config:8-15,32-39");428DEFINE_UNCORE_FORMAT_ATTR(umask_ext3, umask, "config:8-15,32-39");
429DEFINE_UNCORE_FORMAT_ATTR(umask_ext4, umask, "config:8-15,32-55");
393DEFINE_UNCORE_FORMAT_ATTR(qor, qor, "config:16");430DEFINE_UNCORE_FORMAT_ATTR(qor, qor, "config:16");
394DEFINE_UNCORE_FORMAT_ATTR(edge, edge, "config:18");431DEFINE_UNCORE_FORMAT_ATTR(edge, edge, "config:18");
395DEFINE_UNCORE_FORMAT_ATTR(tid_en, tid_en, "config:19");432DEFINE_UNCORE_FORMAT_ATTR(tid_en, tid_en, "config:19");
@@ -4551,3 +4588,477 @@ void snr_uncore_mmio_init(void) · 4155 unmodified lines
4551}4588}
4552 4589
4553/* end of SNR uncore support */4590/* end of SNR uncore support */
4591
4592/* ICX uncore support */
4593
4594static unsigned icx_cha_msr_offsets[] = {
4595 0x2a0, 0x2ae, 0x2bc, 0x2ca, 0x2d8, 0x2e6, 0x2f4, 0x302, 0x310,
4596 0x31e, 0x32c, 0x33a, 0x348, 0x356, 0x364, 0x372, 0x380, 0x38e,
4597 0x3aa, 0x3b8, 0x3c6, 0x3d4, 0x3e2, 0x3f0, 0x3fe, 0x40c, 0x41a,
4598 0x428, 0x436, 0x444, 0x452, 0x460, 0x46e, 0x47c, 0x0, 0xe,
4599 0x1c, 0x2a, 0x38, 0x46,
4600};
4601
4602static int icx_cha_hw_config(struct intel_uncore_box *box, struct perf_event *event)
4603{
4604 struct hw_perf_event_extra *reg1 = &event->hw.extra_reg;
4605 bool tie_en = !!(event->hw.config & SNBEP_CBO_PMON_CTL_TID_EN);
4606
4607 if (tie_en) {
4608 reg1->reg = ICX_C34_MSR_PMON_BOX_FILTER0 +
4609 icx_cha_msr_offsets[box->pmu->pmu_idx];
4610 reg1->config = event->attr.config1 & SKX_CHA_MSR_PMON_BOX_FILTER_TID;
4611 reg1->idx = 0;
4612 }
4613
4614 return 0;
4615}
4616
4617static struct intel_uncore_ops icx_uncore_chabox_ops = {
4618 .init_box = ivbep_uncore_msr_init_box,
4619 .disable_box = snbep_uncore_msr_disable_box,
4620 .enable_box = snbep_uncore_msr_enable_box,
4621 .disable_event = snbep_uncore_msr_disable_event,
4622 .enable_event = snr_cha_enable_event,
4623 .read_counter = uncore_msr_read_counter,
4624 .hw_config = icx_cha_hw_config,
4625};
4626
4627static struct intel_uncore_type icx_uncore_chabox = {
4628 .name = "cha",
4629 .num_counters = 4,
4630 .perf_ctr_bits = 48,
4631 .event_ctl = ICX_C34_MSR_PMON_CTL0,
4632 .perf_ctr = ICX_C34_MSR_PMON_CTR0,
4633 .box_ctl = ICX_C34_MSR_PMON_BOX_CTL,
4634 .msr_offsets = icx_cha_msr_offsets,
4635 .event_mask = HSWEP_S_MSR_PMON_RAW_EVENT_MASK,
4636 .event_mask_ext = SNR_CHA_RAW_EVENT_MASK_EXT,
4637 .constraints = skx_uncore_chabox_constraints,
4638 .ops = &icx_uncore_chabox_ops,
4639 .format_group = &snr_uncore_chabox_format_group,
4640};
4641
4642static unsigned icx_msr_offsets[] = {
4643 0x0, 0x20, 0x40, 0x90, 0xb0, 0xd0,
4644};
4645
4646static struct event_constraint icx_uncore_iio_constraints[] = {
4647 UNCORE_EVENT_CONSTRAINT(0x02, 0x3),
4648 UNCORE_EVENT_CONSTRAINT(0x03, 0x3),
4649 UNCORE_EVENT_CONSTRAINT(0x83, 0x3),
4650 UNCORE_EVENT_CONSTRAINT(0xc0, 0xc),
4651 UNCORE_EVENT_CONSTRAINT(0xc5, 0xc),
4652 EVENT_CONSTRAINT_END
4653};
4654
4655static struct intel_uncore_type icx_uncore_iio = {
4656 .name = "iio",
4657 .num_counters = 4,
4658 .num_boxes = 6,
4659 .perf_ctr_bits = 48,
4660 .event_ctl = ICX_IIO_MSR_PMON_CTL0,
4661 .perf_ctr = ICX_IIO_MSR_PMON_CTR0,
4662 .event_mask = SNBEP_PMON_RAW_EVENT_MASK,
4663 .event_mask_ext = SNR_IIO_PMON_RAW_EVENT_MASK_EXT,
4664 .box_ctl = ICX_IIO_MSR_PMON_BOX_CTL,
4665 .msr_offsets = icx_msr_offsets,
4666 .constraints = icx_uncore_iio_constraints,
4667 .ops = &skx_uncore_iio_ops,
4668 .format_group = &snr_uncore_iio_format_group,
4669};
4670
4671static struct intel_uncore_type icx_uncore_irp = {
4672 .name = "irp",
4673 .num_counters = 2,
4674 .num_boxes = 6,
4675 .perf_ctr_bits = 48,
4676 .event_ctl = ICX_IRP0_MSR_PMON_CTL0,
4677 .perf_ctr = ICX_IRP0_MSR_PMON_CTR0,
4678 .event_mask = SNBEP_PMON_RAW_EVENT_MASK,
4679 .box_ctl = ICX_IRP0_MSR_PMON_BOX_CTL,
4680 .msr_offsets = icx_msr_offsets,
4681 .ops = &ivbep_uncore_msr_ops,
4682 .format_group = &ivbep_uncore_format_group,
4683};
4684
4685static struct event_constraint icx_uncore_m2pcie_constraints[] = {
4686 UNCORE_EVENT_CONSTRAINT(0x14, 0x3),
4687 UNCORE_EVENT_CONSTRAINT(0x23, 0x3),
4688 UNCORE_EVENT_CONSTRAINT(0x2d, 0x3),
4689 EVENT_CONSTRAINT_END
4690};
4691
4692static struct intel_uncore_type icx_uncore_m2pcie = {
4693 .name = "m2pcie",
4694 .num_counters = 4,
4695 .num_boxes = 6,
4696 .perf_ctr_bits = 48,
4697 .event_ctl = ICX_M2PCIE_MSR_PMON_CTL0,
4698 .perf_ctr = ICX_M2PCIE_MSR_PMON_CTR0,
4699 .box_ctl = ICX_M2PCIE_MSR_PMON_BOX_CTL,
4700 .msr_offsets = icx_msr_offsets,
4701 .constraints = icx_uncore_m2pcie_constraints,
4702 .event_mask = SNBEP_PMON_RAW_EVENT_MASK,
4703 .ops = &ivbep_uncore_msr_ops,
4704 .format_group = &ivbep_uncore_format_group,
4705};
4706
4707enum perf_uncore_icx_iio_freerunning_type_id {
4708 ICX_IIO_MSR_IOCLK,
4709 ICX_IIO_MSR_BW_IN,
4710
4711 ICX_IIO_FREERUNNING_TYPE_MAX,
4712};
4713
4714static unsigned icx_iio_clk_freerunning_box_offsets[] = {
4715 0x0, 0x20, 0x40, 0x90, 0xb0, 0xd0,
4716};
4717
4718static unsigned icx_iio_bw_freerunning_box_offsets[] = {
4719 0x0, 0x10, 0x20, 0x90, 0xa0, 0xb0,
4720};
4721
4722static struct freerunning_counters icx_iio_freerunning[] = {
4723 [ICX_IIO_MSR_IOCLK] = { 0xa55, 0x1, 0x20, 1, 48, icx_iio_clk_freerunning_box_offsets },
4724 [ICX_IIO_MSR_BW_IN] = { 0xaa0, 0x1, 0x10, 8, 48, icx_iio_bw_freerunning_box_offsets },
4725};
4726
4727static struct uncore_event_desc icx_uncore_iio_freerunning_events[] = {
4728 /* Free-Running IIO CLOCKS Counter */
4729 INTEL_UNCORE_EVENT_DESC(ioclk, "event=0xff,umask=0x10"),
4730 /* Free-Running IIO BANDWIDTH IN Counters */
4731 INTEL_UNCORE_EVENT_DESC(bw_in_port0, "event=0xff,umask=0x20"),
4732 INTEL_UNCORE_EVENT_DESC(bw_in_port0.scale, "3.814697266e-6"),
4733 INTEL_UNCORE_EVENT_DESC(bw_in_port0.unit, "MiB"),
4734 INTEL_UNCORE_EVENT_DESC(bw_in_port1, "event=0xff,umask=0x21"),
4735 INTEL_UNCORE_EVENT_DESC(bw_in_port1.scale, "3.814697266e-6"),
4736 INTEL_UNCORE_EVENT_DESC(bw_in_port1.unit, "MiB"),
4737 INTEL_UNCORE_EVENT_DESC(bw_in_port2, "event=0xff,umask=0x22"),
4738 INTEL_UNCORE_EVENT_DESC(bw_in_port2.scale, "3.814697266e-6"),
4739 INTEL_UNCORE_EVENT_DESC(bw_in_port2.unit, "MiB"),
4740 INTEL_UNCORE_EVENT_DESC(bw_in_port3, "event=0xff,umask=0x23"),
4741 INTEL_UNCORE_EVENT_DESC(bw_in_port3.scale, "3.814697266e-6"),
4742 INTEL_UNCORE_EVENT_DESC(bw_in_port3.unit, "MiB"),
4743 INTEL_UNCORE_EVENT_DESC(bw_in_port4, "event=0xff,umask=0x24"),
4744 INTEL_UNCORE_EVENT_DESC(bw_in_port4.scale, "3.814697266e-6"),
4745 INTEL_UNCORE_EVENT_DESC(bw_in_port4.unit, "MiB"),
4746 INTEL_UNCORE_EVENT_DESC(bw_in_port5, "event=0xff,umask=0x25"),
4747 INTEL_UNCORE_EVENT_DESC(bw_in_port5.scale, "3.814697266e-6"),
4748 INTEL_UNCORE_EVENT_DESC(bw_in_port5.unit, "MiB"),
4749 INTEL_UNCORE_EVENT_DESC(bw_in_port6, "event=0xff,umask=0x26"),
4750 INTEL_UNCORE_EVENT_DESC(bw_in_port6.scale, "3.814697266e-6"),
4751 INTEL_UNCORE_EVENT_DESC(bw_in_port6.unit, "MiB"),
4752 INTEL_UNCORE_EVENT_DESC(bw_in_port7, "event=0xff,umask=0x27"),
4753 INTEL_UNCORE_EVENT_DESC(bw_in_port7.scale, "3.814697266e-6"),
4754 INTEL_UNCORE_EVENT_DESC(bw_in_port7.unit, "MiB"),
4755 { /* end: all zeroes */ },
4756};
4757
4758static struct intel_uncore_type icx_uncore_iio_free_running = {
4759 .name = "iio_free_running",
4760 .num_counters = 9,
4761 .num_boxes = 6,
4762 .num_freerunning_types = ICX_IIO_FREERUNNING_TYPE_MAX,
4763 .freerunning = icx_iio_freerunning,
4764 .ops = &skx_uncore_iio_freerunning_ops,
4765 .event_descs = icx_uncore_iio_freerunning_events,
4766 .format_group = &skx_uncore_iio_freerunning_format_group,
4767};
4768
4769static struct intel_uncore_type *icx_msr_uncores[] = {
4770 &skx_uncore_ubox,
4771 &icx_uncore_chabox,
4772 &icx_uncore_iio,
4773 &icx_uncore_irp,
4774 &icx_uncore_m2pcie,
4775 &skx_uncore_pcu,
4776 &icx_uncore_iio_free_running,
4777 NULL,
4778};
4779
4780/*
4781 * To determine the number of CHAs, it should read CAPID6(Low) and CAPID7 (High)
4782 * registers which located at Device 30, Function 3
4783 */
4784#define ICX_CAPID6 0x9c
4785#define ICX_CAPID7 0xa0
4786
4787static u64 icx_count_chabox(void)
4788{
4789 struct pci_dev *dev = NULL;
4790 u64 caps = 0;
4791
4792 dev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x345b, dev);
4793 if (!dev)
4794 goto out;
4795
4796 pci_read_config_dword(dev, ICX_CAPID6, (u32 *)&caps);
4797 pci_read_config_dword(dev, ICX_CAPID7, (u32 *)&caps + 1);
4798out:
4799 pci_dev_put(dev);
4800 return hweight64(caps);
4801}
4802
4803void icx_uncore_cpu_init(void)
4804{
4805 u64 num_boxes = icx_count_chabox();
4806
4807 if (WARN_ON(num_boxes > ARRAY_SIZE(icx_cha_msr_offsets)))
4808 return;
4809 icx_uncore_chabox.num_boxes = num_boxes;
4810 uncore_msr_uncores = icx_msr_uncores;
4811}
4812
4813static struct intel_uncore_type icx_uncore_m2m = {
4814 .name = "m2m",
4815 .num_counters = 4,
4816 .num_boxes = 4,
4817 .perf_ctr_bits = 48,
4818 .perf_ctr = SNR_M2M_PCI_PMON_CTR0,
4819 .event_ctl = SNR_M2M_PCI_PMON_CTL0,
4820 .event_mask = SNBEP_PMON_RAW_EVENT_MASK,
4821 .box_ctl = SNR_M2M_PCI_PMON_BOX_CTL,
4822 .ops = &snr_m2m_uncore_pci_ops,
4823 .format_group = &skx_uncore_format_group,
4824};
4825
4826static struct attribute *icx_upi_uncore_formats_attr[] = {
4827 &format_attr_event.attr,
4828 &format_attr_umask_ext4.attr,
4829 &format_attr_edge.attr,
4830 &format_attr_inv.attr,
4831 &format_attr_thresh8.attr,
4832 NULL,
4833};
4834
4835static const struct attribute_group icx_upi_uncore_format_group = {
4836 .name = "format",
4837 .attrs = icx_upi_uncore_formats_attr,
4838};
4839
4840static struct intel_uncore_type icx_uncore_upi = {
4841 .name = "upi",
4842 .num_counters = 4,
4843 .num_boxes = 3,
4844 .perf_ctr_bits = 48,
4845 .perf_ctr = ICX_UPI_PCI_PMON_CTR0,
4846 .event_ctl = ICX_UPI_PCI_PMON_CTL0,
4847 .event_mask = SNBEP_PMON_RAW_EVENT_MASK,
4848 .event_mask_ext = ICX_UPI_CTL_UMASK_EXT,
4849 .box_ctl = ICX_UPI_PCI_PMON_BOX_CTL,
4850 .ops = &skx_upi_uncore_pci_ops,
4851 .format_group = &icx_upi_uncore_format_group,
4852};
4853
4854static struct event_constraint icx_uncore_m3upi_constraints[] = {
4855 UNCORE_EVENT_CONSTRAINT(0x1c, 0x1),
4856 UNCORE_EVENT_CONSTRAINT(0x1d, 0x1),
4857 UNCORE_EVENT_CONSTRAINT(0x1e, 0x1),
4858 UNCORE_EVENT_CONSTRAINT(0x1f, 0x1),
4859 UNCORE_EVENT_CONSTRAINT(0x40, 0x7),
4860 UNCORE_EVENT_CONSTRAINT(0x4e, 0x7),
4861 UNCORE_EVENT_CONSTRAINT(0x4f, 0x7),
4862 UNCORE_EVENT_CONSTRAINT(0x50, 0x7),
4863 EVENT_CONSTRAINT_END
4864};
4865
4866static struct intel_uncore_type icx_uncore_m3upi = {
4867 .name = "m3upi",
4868 .num_counters = 4,
4869 .num_boxes = 3,
4870 .perf_ctr_bits = 48,
4871 .perf_ctr = ICX_M3UPI_PCI_PMON_CTR0,
4872 .event_ctl = ICX_M3UPI_PCI_PMON_CTL0,
4873 .event_mask = SNBEP_PMON_RAW_EVENT_MASK,
4874 .box_ctl = ICX_M3UPI_PCI_PMON_BOX_CTL,
4875 .constraints = icx_uncore_m3upi_constraints,
4876 .ops = &ivbep_uncore_pci_ops,
4877 .format_group = &skx_uncore_format_group,
4878};
4879
4880enum {
4881 ICX_PCI_UNCORE_M2M,
4882 ICX_PCI_UNCORE_UPI,
4883 ICX_PCI_UNCORE_M3UPI,
4884};
4885
4886static struct intel_uncore_type *icx_pci_uncores[] = {
4887 [ICX_PCI_UNCORE_M2M] = &icx_uncore_m2m,
4888 [ICX_PCI_UNCORE_UPI] = &icx_uncore_upi,
4889 [ICX_PCI_UNCORE_M3UPI] = &icx_uncore_m3upi,
4890 NULL,
4891};
4892
4893static const struct pci_device_id icx_uncore_pci_ids[] = {
4894 { /* M2M 0 */
4895 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x344a),
4896 .driver_data = UNCORE_PCI_DEV_FULL_DATA(12, 0, ICX_PCI_UNCORE_M2M, 0),
4897 },
4898 { /* M2M 1 */
4899 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x344a),
4900 .driver_data = UNCORE_PCI_DEV_FULL_DATA(13, 0, ICX_PCI_UNCORE_M2M, 1),
4901 },
4902 { /* M2M 2 */
4903 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x344a),
4904 .driver_data = UNCORE_PCI_DEV_FULL_DATA(14, 0, ICX_PCI_UNCORE_M2M, 2),
4905 },
4906 { /* M2M 3 */
4907 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x344a),
4908 .driver_data = UNCORE_PCI_DEV_FULL_DATA(15, 0, ICX_PCI_UNCORE_M2M, 3),
4909 },
4910 { /* UPI Link 0 */
4911 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x3441),
4912 .driver_data = UNCORE_PCI_DEV_FULL_DATA(2, 1, ICX_PCI_UNCORE_UPI, 0),
4913 },
4914 { /* UPI Link 1 */
4915 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x3441),
4916 .driver_data = UNCORE_PCI_DEV_FULL_DATA(3, 1, ICX_PCI_UNCORE_UPI, 1),
4917 },
4918 { /* UPI Link 2 */
4919 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x3441),
4920 .driver_data = UNCORE_PCI_DEV_FULL_DATA(4, 1, ICX_PCI_UNCORE_UPI, 2),
4921 },
4922 { /* M3UPI Link 0 */
4923 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x3446),
4924 .driver_data = UNCORE_PCI_DEV_FULL_DATA(5, 1, ICX_PCI_UNCORE_M3UPI, 0),
4925 },
4926 { /* M3UPI Link 1 */
4927 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x3446),
4928 .driver_data = UNCORE_PCI_DEV_FULL_DATA(6, 1, ICX_PCI_UNCORE_M3UPI, 1),
4929 },
4930 { /* M3UPI Link 2 */
4931 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x3446),
4932 .driver_data = UNCORE_PCI_DEV_FULL_DATA(7, 1, ICX_PCI_UNCORE_M3UPI, 2),
4933 },
4934 { /* end: all zeroes */ }
4935};
4936
4937static struct pci_driver icx_uncore_pci_driver = {
4938 .name = "icx_uncore",
4939 .id_table = icx_uncore_pci_ids,
4940};
4941
4942int icx_uncore_pci_init(void)
4943{
4944 /* ICX UBOX DID */
4945 int ret = snbep_pci2phy_map_init(0x3450, SKX_CPUNODEID,
4946 SKX_GIDNIDMAP, true);
4947
4948 if (ret)
4949 return ret;
4950
4951 uncore_pci_uncores = icx_pci_uncores;
4952 uncore_pci_driver = &icx_uncore_pci_driver;
4953 return 0;
4954}
4955
4956static void icx_uncore_imc_init_box(struct intel_uncore_box *box)
4957{
4958 unsigned int box_ctl = box->pmu->type->box_ctl +
4959 box->pmu->type->mmio_offset * (box->pmu->pmu_idx % ICX_NUMBER_IMC_CHN);
4960 int mem_offset = (box->pmu->pmu_idx / ICX_NUMBER_IMC_CHN) * ICX_IMC_MEM_STRIDE +
4961 SNR_IMC_MMIO_MEM0_OFFSET;
4962
4963 __snr_uncore_mmio_init_box(box, box_ctl, mem_offset);
4964}
4965
4966static struct intel_uncore_ops icx_uncore_mmio_ops = {
4967 .init_box = icx_uncore_imc_init_box,
4968 .exit_box = uncore_mmio_exit_box,
4969 .disable_box = snr_uncore_mmio_disable_box,
4970 .enable_box = snr_uncore_mmio_enable_box,
4971 .disable_event = snr_uncore_mmio_disable_event,
4972 .enable_event = snr_uncore_mmio_enable_event,
4973 .read_counter = uncore_mmio_read_counter,
4974};
4975
4976static struct intel_uncore_type icx_uncore_imc = {
4977 .name = "imc",
4978 .num_counters = 4,
4979 .num_boxes = 8,
4980 .perf_ctr_bits = 48,
4981 .fixed_ctr_bits = 48,
4982 .fixed_ctr = SNR_IMC_MMIO_PMON_FIXED_CTR,
4983 .fixed_ctl = SNR_IMC_MMIO_PMON_FIXED_CTL,
4984 .event_descs = hswep_uncore_imc_events,
4985 .perf_ctr = SNR_IMC_MMIO_PMON_CTR0,
4986 .event_ctl = SNR_IMC_MMIO_PMON_CTL0,
4987 .event_mask = SNBEP_PMON_RAW_EVENT_MASK,
4988 .box_ctl = SNR_IMC_MMIO_PMON_BOX_CTL,
4989 .mmio_offset = SNR_IMC_MMIO_OFFSET,
4990 .ops = &icx_uncore_mmio_ops,
4991 .format_group = &skx_uncore_format_group,
4992};
4993
4994enum perf_uncore_icx_imc_freerunning_type_id {
4995 ICX_IMC_DCLK,
4996 ICX_IMC_DDR,
4997 ICX_IMC_DDRT,
4998
4999 ICX_IMC_FREERUNNING_TYPE_MAX,
5000};
5001
5002static struct freerunning_counters icx_imc_freerunning[] = {
5003 [ICX_IMC_DCLK] = { 0x22b0, 0x0, 0, 1, 48 },
5004 [ICX_IMC_DDR] = { 0x2290, 0x8, 0, 2, 48 },
5005 [ICX_IMC_DDRT] = { 0x22a0, 0x8, 0, 2, 48 },
5006};
5007
5008static struct uncore_event_desc icx_uncore_imc_freerunning_events[] = {
5009 INTEL_UNCORE_EVENT_DESC(dclk, "event=0xff,umask=0x10"),
5010
5011 INTEL_UNCORE_EVENT_DESC(read, "event=0xff,umask=0x20"),
5012 INTEL_UNCORE_EVENT_DESC(read.scale, "3.814697266e-6"),
5013 INTEL_UNCORE_EVENT_DESC(read.unit, "MiB"),
5014 INTEL_UNCORE_EVENT_DESC(write, "event=0xff,umask=0x21"),
5015 INTEL_UNCORE_EVENT_DESC(write.scale, "3.814697266e-6"),
5016 INTEL_UNCORE_EVENT_DESC(write.unit, "MiB"),
5017
5018 INTEL_UNCORE_EVENT_DESC(ddrt_read, "event=0xff,umask=0x30"),
5019 INTEL_UNCORE_EVENT_DESC(ddrt_read.scale, "3.814697266e-6"),
5020 INTEL_UNCORE_EVENT_DESC(ddrt_read.unit, "MiB"),
5021 INTEL_UNCORE_EVENT_DESC(ddrt_write, "event=0xff,umask=0x31"),
5022 INTEL_UNCORE_EVENT_DESC(ddrt_write.scale, "3.814697266e-6"),
5023 INTEL_UNCORE_EVENT_DESC(ddrt_write.unit, "MiB"),
5024 { /* end: all zeroes */ },
5025};
5026
5027static void icx_uncore_imc_freerunning_init_box(struct intel_uncore_box *box)
5028{
5029 int mem_offset = box->pmu->pmu_idx * ICX_IMC_MEM_STRIDE +
5030 SNR_IMC_MMIO_MEM0_OFFSET;
5031
5032 __snr_uncore_mmio_init_box(box, uncore_mmio_box_ctl(box), mem_offset);
5033}
5034
5035static struct intel_uncore_ops icx_uncore_imc_freerunning_ops = {
5036 .init_box = icx_uncore_imc_freerunning_init_box,
5037 .exit_box = uncore_mmio_exit_box,
5038 .read_counter = uncore_mmio_read_counter,
5039 .hw_config = uncore_freerunning_hw_config,
5040};
5041
5042static struct intel_uncore_type icx_uncore_imc_free_running = {
5043 .name = "imc_free_running",
5044 .num_counters = 5,
5045 .num_boxes = 4,
5046 .num_freerunning_types = ICX_IMC_FREERUNNING_TYPE_MAX,
5047 .freerunning = icx_imc_freerunning,
5048 .ops = &icx_uncore_imc_freerunning_ops,
5049 .event_descs = icx_uncore_imc_freerunning_events,
5050 .format_group = &skx_uncore_iio_freerunning_format_group,
5051};
5052
5053static struct intel_uncore_type *icx_mmio_uncores[] = {
5054 &icx_uncore_imc,
5055 &icx_uncore_imc_free_running,
5056 NULL,
5057};
5058
5059void icx_uncore_mmio_init(void)
5060{
5061 uncore_mmio_uncores = icx_mmio_uncores;
5062}
5063
5064/* end of ICX uncore support */
kernel/events/core.c+51−31
@@ -983,16 +983,10 @@ perf_cgroup_set_shadow_time(struct perf_event *event, u64 now) · 982 unmodified lines
983 event->shadow_ctx_time = now - t->timestamp;983 event->shadow_ctx_time = now - t->timestamp;
984}984}
985 985
986/*
987 * Update cpuctx->cgrp so that it is set when first cgroup event is added and
988 * cleared when last cgroup event is removed.
989 */
990static inline void986static inline void
991list_update_cgroup_event(struct perf_event *event,987perf_cgroup_event_enable(struct perf_event *event, struct perf_event_context *ctx)
992 struct perf_event_context *ctx, bool add)
993{988{
994 struct perf_cpu_context *cpuctx;989 struct perf_cpu_context *cpuctx;
995 struct list_head *cpuctx_entry;
996 990
997 if (!is_cgroup_event(event))991 if (!is_cgroup_event(event))
998 return;992 return;
@@ -1009,28 +1003,41 @@ list_update_cgroup_event(struct perf_event *event, · 10 unmodified lines
1009 * because if the first would mismatch, the second would not try again1003 * because if the first would mismatch, the second would not try again
1010 * and we would leave cpuctx->cgrp unset.1004 * and we would leave cpuctx->cgrp unset.
1011 */1005 */
1012 if (add && !cpuctx->cgrp) {1006 if (ctx->is_active && !cpuctx->cgrp) {
1013 struct perf_cgroup *cgrp = perf_cgroup_from_task(current, ctx);1007 struct perf_cgroup *cgrp = perf_cgroup_from_task(current, ctx);
1014 1008
1015 if (cgroup_is_descendant(cgrp->css.cgroup, event->cgrp->css.cgroup))1009 if (cgroup_is_descendant(cgrp->css.cgroup, event->cgrp->css.cgroup))
1016 cpuctx->cgrp = cgrp;1010 cpuctx->cgrp = cgrp;
1017 }1011 }
1018 1012
1019 if (add && ctx->nr_cgroups++)1013 if (ctx->nr_cgroups++)
1020 return;1014 return;
1021 else if (!add && --ctx->nr_cgroups)1015
1016 list_add(&cpuctx->cgrp_cpuctx_entry,
1017 per_cpu_ptr(&cgrp_cpuctx_list, event->cpu));
1018}
1019
1020static inline void
1021perf_cgroup_event_disable(struct perf_event *event, struct perf_event_context *ctx)
1022{
1023 struct perf_cpu_context *cpuctx;
1024
1025 if (!is_cgroup_event(event))
1026 return;
1027
1028 /*
1029 * Because cgroup events are always per-cpu events,
1030 * @ctx == &cpuctx->ctx.
1031 */
1032 cpuctx = container_of(ctx, struct perf_cpu_context, ctx);
1033
1034 if (--ctx->nr_cgroups)
1022 return;1035 return;
1023 1036
1024 /* no cgroup running */1037 if (ctx->is_active && cpuctx->cgrp)
1025 if (!add)
1026 cpuctx->cgrp = NULL;1038 cpuctx->cgrp = NULL;
1027 1039
1028 cpuctx_entry = &cpuctx->cgrp_cpuctx_entry;1040 list_del(&cpuctx->cgrp_cpuctx_entry);
1029 if (add)
1030 list_add(cpuctx_entry,
1031 per_cpu_ptr(&cgrp_cpuctx_list, event->cpu));
1032 else
1033 list_del(cpuctx_entry);
1034}1041}
1035 1042
1036#else /* !CONFIG_CGROUP_PERF */1043#else /* !CONFIG_CGROUP_PERF */
@@ -1096,11 +1103,14 @@ static inline u64 perf_cgroup_event_time(struct perf_event *event) · 59 unmodified lines
1096}1103}
1097 1104
1098static inline void1105static inline void
1099list_update_cgroup_event(struct perf_event *event,1106perf_cgroup_event_enable(struct perf_event *event, struct perf_event_context *ctx)
1100 struct perf_event_context *ctx, bool add)
1101{1107{
1102}1108}
1103 1109
1110static inline void
1111perf_cgroup_event_disable(struct perf_event *event, struct perf_event_context *ctx)
1112{
1113}
1104#endif1114#endif
1105 1115
1106/*1116/*
@@ -1791,13 +1801,14 @@ list_add_event(struct perf_event *event, struct perf_event_context *ctx) · 684 unmodified lines
1791 add_event_to_groups(event, ctx);1801 add_event_to_groups(event, ctx);
1792 }1802 }
1793 1803
1794 list_update_cgroup_event(event, ctx, true);
1795
1796 list_add_rcu(&event->event_entry, &ctx->event_list);1804 list_add_rcu(&event->event_entry, &ctx->event_list);
1797 ctx->nr_events++;1805 ctx->nr_events++;
1798 if (event->attr.inherit_stat)1806 if (event->attr.inherit_stat)
1799 ctx->nr_stat++;1807 ctx->nr_stat++;
1800 1808
1809 if (event->state > PERF_EVENT_STATE_OFF)
1810 perf_cgroup_event_enable(event, ctx);
1811
1801 ctx->generation++;1812 ctx->generation++;
1802}1813}
1803 1814
@@ -1976,8 +1987,6 @@ list_del_event(struct perf_event *event, struct perf_event_context *ctx) · 172 unmodified lines
1976 1987
1977 event->attach_state &= ~PERF_ATTACH_CONTEXT;1988 event->attach_state &= ~PERF_ATTACH_CONTEXT;
1978 1989
1979 list_update_cgroup_event(event, ctx, false);
1980
1981 ctx->nr_events--;1990 ctx->nr_events--;
1982 if (event->attr.inherit_stat)1991 if (event->attr.inherit_stat)
1983 ctx->nr_stat--;1992 ctx->nr_stat--;
@@ -1994,8 +2003,10 @@ list_del_event(struct perf_event *event, struct perf_event_context *ctx) · 10 unmodified lines
1994 * of error state is by explicit re-enabling2003 * of error state is by explicit re-enabling
1995 * of the event2004 * of the event
1996 */2005 */
1997 if (event->state > PERF_EVENT_STATE_OFF)2006 if (event->state > PERF_EVENT_STATE_OFF) {
2007 perf_cgroup_event_disable(event, ctx);
1998 perf_event_set_state(event, PERF_EVENT_STATE_OFF);2008 perf_event_set_state(event, PERF_EVENT_STATE_OFF);
2009 }
1999 2010
2000 ctx->generation++;2011 ctx->generation++;
2001}2012}
@@ -2226,6 +2237,7 @@ event_sched_out(struct perf_event *event, · 224 unmodified lines
2226 2237
2227 if (READ_ONCE(event->pending_disable) >= 0) {2238 if (READ_ONCE(event->pending_disable) >= 0) {
2228 WRITE_ONCE(event->pending_disable, -1);2239 WRITE_ONCE(event->pending_disable, -1);
2240 perf_cgroup_event_disable(event, ctx);
2229 state = PERF_EVENT_STATE_OFF;2241 state = PERF_EVENT_STATE_OFF;
2230 }2242 }
2231 perf_event_set_state(event, state);2243 perf_event_set_state(event, state);
@@ -2363,6 +2375,7 @@ static void __perf_event_disable(struct perf_event *event, · 131 unmodified lines
2363 event_sched_out(event, cpuctx, ctx);2375 event_sched_out(event, cpuctx, ctx);
2364 2376
2365 perf_event_set_state(event, PERF_EVENT_STATE_OFF);2377 perf_event_set_state(event, PERF_EVENT_STATE_OFF);
2378 perf_cgroup_event_disable(event, ctx);
2366}2379}
2367 2380
2368/*2381/*
@@ -2746,7 +2759,7 @@ static int __perf_install_in_context(void *info) · 377 unmodified lines
2746 }2759 }
2747 2760
2748#ifdef CONFIG_CGROUP_PERF2761#ifdef CONFIG_CGROUP_PERF
2749 if (is_cgroup_event(event)) {2762 if (event->state > PERF_EVENT_STATE_OFF && is_cgroup_event(event)) {
2750 /*2763 /*
2751 * If the current cgroup doesn't match the event's2764 * If the current cgroup doesn't match the event's
2752 * cgroup, we should not try to schedule it.2765 * cgroup, we should not try to schedule it.
@@ -2906,6 +2919,7 @@ static void __perf_event_enable(struct perf_event *event, · 153 unmodified lines
2906 ctx_sched_out(ctx, cpuctx, EVENT_TIME);2919 ctx_sched_out(ctx, cpuctx, EVENT_TIME);
2907 2920
2908 perf_event_set_state(event, PERF_EVENT_STATE_INACTIVE);2921 perf_event_set_state(event, PERF_EVENT_STATE_INACTIVE);
2922 perf_cgroup_event_enable(event, ctx);
2909 2923
2910 if (!ctx->is_active)2924 if (!ctx->is_active)
2911 return;2925 return;
@@ -3508,7 +3522,8 @@ static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx, · 596 unmodified lines
3508 3522
3509static bool perf_less_group_idx(const void *l, const void *r)3523static bool perf_less_group_idx(const void *l, const void *r)
3510{3524{
3511 const struct perf_event *le = l, *re = r;3525 const struct perf_event *le = *(const struct perf_event **)l;
3526 const struct perf_event *re = *(const struct perf_event **)r;
3512 3527
3513 return le->group_index < re->group_index;3528 return le->group_index < re->group_index;
3514}3529}
@@ -3616,8 +3631,10 @@ static int merge_sched_in(struct perf_event *event, void *data) · 101 unmodified lines
3616 }3631 }
3617 3632
3618 if (event->state == PERF_EVENT_STATE_INACTIVE) {3633 if (event->state == PERF_EVENT_STATE_INACTIVE) {
3619 if (event->attr.pinned)3634 if (event->attr.pinned) {
3635 perf_cgroup_event_disable(event, ctx);
3620 perf_event_set_state(event, PERF_EVENT_STATE_ERROR);3636 perf_event_set_state(event, PERF_EVENT_STATE_ERROR);
3637 }
3621 3638
3622 *can_add_hw = 0;3639 *can_add_hw = 0;
3623 ctx->rotate_necessary = 1;3640 ctx->rotate_necessary = 1;
@@ -6917,9 +6934,12 @@ static u64 perf_virt_to_phys(u64 virt) · 3293 unmodified lines
6917 * Try IRQ-safe __get_user_pages_fast first.6934 * Try IRQ-safe __get_user_pages_fast first.
6918 * If failed, leave phys_addr as 0.6935 * If failed, leave phys_addr as 0.
6919 */6936 */
6920 if ((current->mm != NULL) &&6937 if (current->mm != NULL) {
6921 (__get_user_pages_fast(virt, 1, 0, &p) == 1))6938 pagefault_disable();
6922 phys_addr = page_to_phys(p) + virt % PAGE_SIZE;6939 if (__get_user_pages_fast(virt, 1, 0, &p) == 1)
6940 phys_addr = page_to_phys(p) + virt % PAGE_SIZE;
6941 pagefault_enable();
6942 }
6923 6943
6924 if (p)6944 if (p)
6925 put_page(p);6945 put_page(p);

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t