History

drivers/perf: hisi: Add driver for HiSilicon PCIe PMU

PCIe PMU Root Complex Integrated End Point(RCiEP) device is supported to sample bandwidth, latency, buffer occupation etc. Each PMU RCiEP device monitors multiple Root Ports, and each RCiEP is registered as a PMU in /sys/bus/event_source/devices, so users can select target PMU, and use filter to do further sets. Filtering options contains: event - select the event. port - select target Root Ports. Information of Root Ports are shown under sysfs. bdf - select requester_id of target EP device. trig_len - set trigger condition for starting event statistics. trig_mode - set trigger mode. 0 means starting to statistic when bigger than trigger condition, and 1 means smaller. thr_len - set threshold for statistics. thr_mode - set threshold mode. 0 means count when bigger than threshold, and 1 means smaller.

Qi Liu committed 4 years ago

Commit 8404b0f1 parent c8602005 files · +962 −0Browse files

Unified · Split

5 files
  1. MAINTAINERS +2 −0
  2. drivers/perf/hisilicon/Kconfig +9 −0
  3. drivers/perf/hisilicon/Makefile +2 −0
  4. drivers/perf/hisilicon/hisi_pcie_pmu.c +948 −0
  5. include/linux/cpuhotplug.h +1 −0
MAINTAINERS+2−0
@@ -8607,8 +8607,10 @@ F: drivers/misc/hisi_hikey_usb.c · 8606 unmodified lines
8607 8607
8608HISILICON PMU DRIVER8608HISILICON PMU DRIVER
8609M: Shaokun Zhang <zhangshaokun@hisilicon.com>8609M: Shaokun Zhang <zhangshaokun@hisilicon.com>
8610M: Qi Liu <liuqi115@huawei.com>
8610S: Supported8611S: Supported
8611W: http://www.hisilicon.com8612W: http://www.hisilicon.com
8613F: Documentation/admin-guide/perf/hisi-pcie-pmu.rst
8612F: Documentation/admin-guide/perf/hisi-pmu.rst8614F: Documentation/admin-guide/perf/hisi-pmu.rst
8613F: drivers/perf/hisilicon8615F: drivers/perf/hisilicon
8614 8616
drivers/perf/hisilicon/Kconfig+9−0
@@ -5,3 +5,12 @@ config HISI_PMU · 4 unmodified lines
5 help5 help
6 Support for HiSilicon SoC L3 Cache performance monitor, Hydra Home6 Support for HiSilicon SoC L3 Cache performance monitor, Hydra Home
7 Agent performance monitor and DDR Controller performance monitor.7 Agent performance monitor and DDR Controller performance monitor.
8
9config HISI_PCIE_PMU
10 tristate "HiSilicon PCIE PERF PMU"
11 depends on PCI && ARM64
12 help
13 Provide support for HiSilicon PCIe performance monitoring unit (PMU)
14 RCiEP devices.
15 Adds the PCIe PMU into perf events system for monitoring latency,
16 bandwidth etc.
drivers/perf/hisilicon/Makefile+2−0
@@ -2,3 +2,5 @@ · 1 unmodified line
2obj-$(CONFIG_HISI_PMU) += hisi_uncore_pmu.o hisi_uncore_l3c_pmu.o \2obj-$(CONFIG_HISI_PMU) += hisi_uncore_pmu.o hisi_uncore_l3c_pmu.o \
3 hisi_uncore_hha_pmu.o hisi_uncore_ddrc_pmu.o hisi_uncore_sllc_pmu.o \3 hisi_uncore_hha_pmu.o hisi_uncore_ddrc_pmu.o hisi_uncore_sllc_pmu.o \
4 hisi_uncore_pa_pmu.o4 hisi_uncore_pa_pmu.o
5
6obj-$(CONFIG_HISI_PCIE_PMU) += hisi_pcie_pmu.o
drivers/perf/hisilicon/hisi_pcie_pmu.c+948−0
@@ -0,0 +1,948 @@
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * This driver adds support for PCIe PMU RCiEP device. Related
4 * perf events are bandwidth, latency etc.
5 *
6 * Copyright (C) 2021 HiSilicon Limited
7 * Author: Qi Liu <liuqi115@huawei.com>
8 */
9#include <linux/bitfield.h>
10#include <linux/bitmap.h>
11#include <linux/bug.h>
12#include <linux/device.h>
13#include <linux/err.h>
14#include <linux/interrupt.h>
15#include <linux/irq.h>
16#include <linux/kernel.h>
17#include <linux/list.h>
18#include <linux/module.h>
19#include <linux/pci.h>
20#include <linux/perf_event.h>
21
22#define DRV_NAME "hisi_pcie_pmu"
23/* Define registers */
24#define HISI_PCIE_GLOBAL_CTRL 0x00
25#define HISI_PCIE_EVENT_CTRL 0x010
26#define HISI_PCIE_CNT 0x090
27#define HISI_PCIE_EXT_CNT 0x110
28#define HISI_PCIE_INT_STAT 0x150
29#define HISI_PCIE_INT_MASK 0x154
30#define HISI_PCIE_REG_BDF 0xfe0
31#define HISI_PCIE_REG_VERSION 0xfe4
32#define HISI_PCIE_REG_INFO 0xfe8
33
34/* Define command in HISI_PCIE_GLOBAL_CTRL */
35#define HISI_PCIE_GLOBAL_EN 0x01
36#define HISI_PCIE_GLOBAL_NONE 0
37
38/* Define command in HISI_PCIE_EVENT_CTRL */
39#define HISI_PCIE_EVENT_EN BIT_ULL(20)
40#define HISI_PCIE_RESET_CNT BIT_ULL(22)
41#define HISI_PCIE_INIT_SET BIT_ULL(34)
42#define HISI_PCIE_THR_EN BIT_ULL(26)
43#define HISI_PCIE_TARGET_EN BIT_ULL(32)
44#define HISI_PCIE_TRIG_EN BIT_ULL(52)
45
46/* Define offsets in HISI_PCIE_EVENT_CTRL */
47#define HISI_PCIE_EVENT_M GENMASK_ULL(15, 0)
48#define HISI_PCIE_THR_MODE_M GENMASK_ULL(27, 27)
49#define HISI_PCIE_THR_M GENMASK_ULL(31, 28)
50#define HISI_PCIE_TARGET_M GENMASK_ULL(52, 36)
51#define HISI_PCIE_TRIG_MODE_M GENMASK_ULL(53, 53)
52#define HISI_PCIE_TRIG_M GENMASK_ULL(59, 56)
53
54#define HISI_PCIE_MAX_COUNTERS 8
55#define HISI_PCIE_REG_STEP 8
56#define HISI_PCIE_THR_MAX_VAL 10
57#define HISI_PCIE_TRIG_MAX_VAL 10
58#define HISI_PCIE_MAX_PERIOD (GENMASK_ULL(63, 0))
59#define HISI_PCIE_INIT_VAL BIT_ULL(63)
60
61struct hisi_pcie_pmu {
62 struct perf_event *hw_events[HISI_PCIE_MAX_COUNTERS];
63 struct hlist_node node;
64 struct pci_dev *pdev;
65 struct pmu pmu;
66 void __iomem *base;
67 int irq;
68 u32 identifier;
69 /* Minimum and maximum BDF of root ports monitored by PMU */
70 u16 bdf_min;
71 u16 bdf_max;
72 int on_cpu;
73};
74
75struct hisi_pcie_reg_pair {
76 u16 lo;
77 u16 hi;
78};
79
80#define to_pcie_pmu(p) (container_of((p), struct hisi_pcie_pmu, pmu))
81#define GET_PCI_DEVFN(bdf) ((bdf) & 0xff)
82
83#define HISI_PCIE_PMU_FILTER_ATTR(_name, _config, _hi, _lo) \
84 static u64 hisi_pcie_get_##_name(struct perf_event *event) \
85 { \
86 return FIELD_GET(GENMASK(_hi, _lo), event->attr._config); \
87 } \
88
89HISI_PCIE_PMU_FILTER_ATTR(event, config, 16, 0);
90HISI_PCIE_PMU_FILTER_ATTR(thr_len, config1, 3, 0);
91HISI_PCIE_PMU_FILTER_ATTR(thr_mode, config1, 4, 4);
92HISI_PCIE_PMU_FILTER_ATTR(trig_len, config1, 8, 5);
93HISI_PCIE_PMU_FILTER_ATTR(trig_mode, config1, 9, 9);
94HISI_PCIE_PMU_FILTER_ATTR(port, config2, 15, 0);
95HISI_PCIE_PMU_FILTER_ATTR(bdf, config2, 31, 16);
96
97static ssize_t hisi_pcie_format_sysfs_show(struct device *dev, struct device_attribute *attr,
98 char *buf)
99{
100 struct dev_ext_attribute *eattr;
101
102 eattr = container_of(attr, struct dev_ext_attribute, attr);
103
104 return sysfs_emit(buf, "%s\n", (char *)eattr->var);
105}
106
107static ssize_t hisi_pcie_event_sysfs_show(struct device *dev, struct device_attribute *attr,
108 char *buf)
109{
110 struct perf_pmu_events_attr *pmu_attr =
111 container_of(attr, struct perf_pmu_events_attr, attr);
112
113 return sysfs_emit(buf, "config=0x%llx\n", pmu_attr->id);
114}
115
116#define HISI_PCIE_PMU_FORMAT_ATTR(_name, _format) \
117 (&((struct dev_ext_attribute[]){ \
118 { .attr = __ATTR(_name, 0444, hisi_pcie_format_sysfs_show, \
119 NULL), \
120 .var = (void *)_format } \
121 })[0].attr.attr)
122
123#define HISI_PCIE_PMU_EVENT_ATTR(_name, _id) \
124 PMU_EVENT_ATTR_ID(_name, hisi_pcie_event_sysfs_show, _id)
125
126static ssize_t cpumask_show(struct device *dev, struct device_attribute *attr, char *buf)
127{
128 struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(dev_get_drvdata(dev));
129
130 return cpumap_print_to_pagebuf(true, buf, cpumask_of(pcie_pmu->on_cpu));
131}
132static DEVICE_ATTR_RO(cpumask);
133
134static ssize_t identifier_show(struct device *dev, struct device_attribute *attr, char *buf)
135{
136 struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(dev_get_drvdata(dev));
137
138 return sysfs_emit(buf, "%#x\n", pcie_pmu->identifier);
139}
140static DEVICE_ATTR_RO(identifier);
141
142static ssize_t bus_show(struct device *dev, struct device_attribute *attr, char *buf)
143{
144 struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(dev_get_drvdata(dev));
145
146 return sysfs_emit(buf, "%#04x\n", PCI_BUS_NUM(pcie_pmu->bdf_min));
147}
148static DEVICE_ATTR_RO(bus);
149
150static struct hisi_pcie_reg_pair
151hisi_pcie_parse_reg_value(struct hisi_pcie_pmu *pcie_pmu, u32 reg_off)
152{
153 u32 val = readl_relaxed(pcie_pmu->base + reg_off);
154 struct hisi_pcie_reg_pair regs = {
155 .lo = val,
156 .hi = val >> 16,
157 };
158
159 return regs;
160}
161
162/*
163 * Hardware counter and ext_counter work together for bandwidth, latency, bus
164 * utilization and buffer occupancy events. For example, RX memory write latency
165 * events(index = 0x0010), counter counts total delay cycles and ext_counter
166 * counts RX memory write PCIe packets number.
167 *
168 * As we don't want PMU driver to process these two data, "delay cycles" can
169 * be treated as an independent event(index = 0x0010), "RX memory write packets
170 * number" as another(index = 0x10010). BIT 16 is used to distinguish and 0-15
171 * bits are "real" event index, which can be used to set HISI_PCIE_EVENT_CTRL.
172 */
173#define EXT_COUNTER_IS_USED(idx) ((idx) & BIT(16))
174
175static u32 hisi_pcie_get_real_event(struct perf_event *event)
176{
177 return hisi_pcie_get_event(event) & GENMASK(15, 0);
178}
179
180static u32 hisi_pcie_pmu_get_offset(u32 offset, u32 idx)
181{
182 return offset + HISI_PCIE_REG_STEP * idx;
183}
184
185static u32 hisi_pcie_pmu_readl(struct hisi_pcie_pmu *pcie_pmu, u32 reg_offset,
186 u32 idx)
187{
188 u32 offset = hisi_pcie_pmu_get_offset(reg_offset, idx);
189
190 return readl_relaxed(pcie_pmu->base + offset);
191}
192
193static void hisi_pcie_pmu_writel(struct hisi_pcie_pmu *pcie_pmu, u32 reg_offset, u32 idx, u32 val)
194{
195 u32 offset = hisi_pcie_pmu_get_offset(reg_offset, idx);
196
197 writel_relaxed(val, pcie_pmu->base + offset);
198}
199
200static u64 hisi_pcie_pmu_readq(struct hisi_pcie_pmu *pcie_pmu, u32 reg_offset, u32 idx)
201{
202 u32 offset = hisi_pcie_pmu_get_offset(reg_offset, idx);
203
204 return readq_relaxed(pcie_pmu->base + offset);
205}
206
207static void hisi_pcie_pmu_writeq(struct hisi_pcie_pmu *pcie_pmu, u32 reg_offset, u32 idx, u64 val)
208{
209 u32 offset = hisi_pcie_pmu_get_offset(reg_offset, idx);
210
211 writeq_relaxed(val, pcie_pmu->base + offset);
212}
213
214static void hisi_pcie_pmu_config_filter(struct perf_event *event)
215{
216 struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(event->pmu);
217 struct hw_perf_event *hwc = &event->hw;
218 u64 reg = HISI_PCIE_INIT_SET;
219 u64 port, trig_len, thr_len;
220
221 /* Config HISI_PCIE_EVENT_CTRL according to event. */
222 reg |= FIELD_PREP(HISI_PCIE_EVENT_M, hisi_pcie_get_real_event(event));
223
224 /* Config HISI_PCIE_EVENT_CTRL according to root port or EP device. */
225 port = hisi_pcie_get_port(event);
226 if (port)
227 reg |= FIELD_PREP(HISI_PCIE_TARGET_M, port);
228 else
229 reg |= HISI_PCIE_TARGET_EN |
230 FIELD_PREP(HISI_PCIE_TARGET_M, hisi_pcie_get_bdf(event));
231
232 /* Config HISI_PCIE_EVENT_CTRL according to trigger condition. */
233 trig_len = hisi_pcie_get_trig_len(event);
234 if (trig_len) {
235 reg |= FIELD_PREP(HISI_PCIE_TRIG_M, trig_len);
236 reg |= FIELD_PREP(HISI_PCIE_TRIG_MODE_M, hisi_pcie_get_trig_mode(event));
237 reg |= HISI_PCIE_TRIG_EN;
238 }
239
240 /* Config HISI_PCIE_EVENT_CTRL according to threshold condition. */
241 thr_len = hisi_pcie_get_thr_len(event);
242 if (thr_len) {
243 reg |= FIELD_PREP(HISI_PCIE_THR_M, thr_len);
244 reg |= FIELD_PREP(HISI_PCIE_THR_MODE_M, hisi_pcie_get_thr_mode(event));
245 reg |= HISI_PCIE_THR_EN;
246 }
247
248 hisi_pcie_pmu_writeq(pcie_pmu, HISI_PCIE_EVENT_CTRL, hwc->idx, reg);
249}
250
251static void hisi_pcie_pmu_clear_filter(struct perf_event *event)
252{
253 struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(event->pmu);
254 struct hw_perf_event *hwc = &event->hw;
255
256 hisi_pcie_pmu_writeq(pcie_pmu, HISI_PCIE_EVENT_CTRL, hwc->idx, HISI_PCIE_INIT_SET);
257}
258
259static bool hisi_pcie_pmu_valid_requester_id(struct hisi_pcie_pmu *pcie_pmu, u32 bdf)
260{
261 struct pci_dev *root_port, *pdev;
262 u16 rp_bdf;
263
264 pdev = pci_get_domain_bus_and_slot(pci_domain_nr(pcie_pmu->pdev->bus), PCI_BUS_NUM(bdf),
265 GET_PCI_DEVFN(bdf));
266 if (!pdev)
267 return false;
268
269 root_port = pcie_find_root_port(pdev);
270 if (!root_port) {
271 pci_dev_put(pdev);
272 return false;
273 }
274
275 pci_dev_put(pdev);
276 rp_bdf = pci_dev_id(root_port);
277 return rp_bdf >= pcie_pmu->bdf_min && rp_bdf <= pcie_pmu->bdf_max;
278}
279
280static bool hisi_pcie_pmu_valid_filter(struct perf_event *event,
281 struct hisi_pcie_pmu *pcie_pmu)
282{
283 u32 requester_id = hisi_pcie_get_bdf(event);
284
285 if (hisi_pcie_get_thr_len(event) > HISI_PCIE_THR_MAX_VAL)
286 return false;
287
288 if (hisi_pcie_get_trig_len(event) > HISI_PCIE_TRIG_MAX_VAL)
289 return false;
290
291 if (requester_id) {
292 if (!hisi_pcie_pmu_valid_requester_id(pcie_pmu, requester_id))
293 return false;
294 }
295
296 return true;
297}
298
299static bool hisi_pcie_pmu_cmp_event(struct perf_event *target,
300 struct perf_event *event)
301{
302 return hisi_pcie_get_real_event(target) == hisi_pcie_get_real_event(event);
303}
304
305static bool hisi_pcie_pmu_validate_event_group(struct perf_event *event)
306{
307 struct perf_event *sibling, *leader = event->group_leader;
308 struct perf_event *event_group[HISI_PCIE_MAX_COUNTERS];
309 int counters = 1;
310 int num;
311
312 event_group[0] = leader;
313 if (!is_software_event(leader)) {
314 if (leader->pmu != event->pmu)
315 return false;
316
317 if (leader != event && !hisi_pcie_pmu_cmp_event(leader, event))
318 event_group[counters++] = event;
319 }
320
321 for_each_sibling_event(sibling, event->group_leader) {
322 if (is_software_event(sibling))
323 continue;
324
325 if (sibling->pmu != event->pmu)
326 return false;
327
328 for (num = 0; num < counters; num++) {
329 if (hisi_pcie_pmu_cmp_event(event_group[num], sibling))
330 break;
331 }
332
333 if (num == counters)
334 event_group[counters++] = sibling;
335 }
336
337 return counters <= HISI_PCIE_MAX_COUNTERS;
338}
339
340static int hisi_pcie_pmu_event_init(struct perf_event *event)
341{
342 struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(event->pmu);
343 struct hw_perf_event *hwc = &event->hw;
344
345 event->cpu = pcie_pmu->on_cpu;
346
347 if (EXT_COUNTER_IS_USED(hisi_pcie_get_event(event)))
348 hwc->event_base = HISI_PCIE_EXT_CNT;
349 else
350 hwc->event_base = HISI_PCIE_CNT;
351
352 if (event->attr.type != event->pmu->type)
353 return -ENOENT;
354
355 /* Sampling is not supported. */
356 if (is_sampling_event(event) || event->attach_state & PERF_ATTACH_TASK)
357 return -EOPNOTSUPP;
358
359 if (!hisi_pcie_pmu_valid_filter(event, pcie_pmu))
360 return -EINVAL;
361
362 if (!hisi_pcie_pmu_validate_event_group(event))
363 return -EINVAL;
364
365 return 0;
366}
367
368static u64 hisi_pcie_pmu_read_counter(struct perf_event *event)
369{
370 struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(event->pmu);
371 u32 idx = event->hw.idx;
372
373 return hisi_pcie_pmu_readq(pcie_pmu, event->hw.event_base, idx);
374}
375
376static int hisi_pcie_pmu_find_related_event(struct hisi_pcie_pmu *pcie_pmu,
377 struct perf_event *event)
378{
379 struct perf_event *sibling;
380 int idx;
381
382 for (idx = 0; idx < HISI_PCIE_MAX_COUNTERS; idx++) {
383 sibling = pcie_pmu->hw_events[idx];
384 if (!sibling)
385 continue;
386
387 if (!hisi_pcie_pmu_cmp_event(sibling, event))
388 continue;
389
390 /* Related events must be used in group */
391 if (sibling->group_leader == event->group_leader)
392 return idx;
393 else
394 return -EINVAL;
395 }
396
397 return idx;
398}
399
400static int hisi_pcie_pmu_get_event_idx(struct hisi_pcie_pmu *pcie_pmu)
401{
402 int idx;
403
404 for (idx = 0; idx < HISI_PCIE_MAX_COUNTERS; idx++) {
405 if (!pcie_pmu->hw_events[idx])
406 return idx;
407 }
408
409 return -EINVAL;
410}
411
412static void hisi_pcie_pmu_event_update(struct perf_event *event)
413{
414 struct hw_perf_event *hwc = &event->hw;
415 u64 new_cnt, prev_cnt, delta;
416
417 do {
418 prev_cnt = local64_read(&hwc->prev_count);
419 new_cnt = hisi_pcie_pmu_read_counter(event);
420 } while (local64_cmpxchg(&hwc->prev_count, prev_cnt,
421 new_cnt) != prev_cnt);
422
423 delta = (new_cnt - prev_cnt) & HISI_PCIE_MAX_PERIOD;
424 local64_add(delta, &event->count);
425}
426
427static void hisi_pcie_pmu_read(struct perf_event *event)
428{
429 hisi_pcie_pmu_event_update(event);
430}
431
432static void hisi_pcie_pmu_set_period(struct perf_event *event)
433{
434 struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(event->pmu);
435 struct hw_perf_event *hwc = &event->hw;
436 int idx = hwc->idx;
437
438 local64_set(&hwc->prev_count, HISI_PCIE_INIT_VAL);
439 hisi_pcie_pmu_writeq(pcie_pmu, HISI_PCIE_CNT, idx, HISI_PCIE_INIT_VAL);
440 hisi_pcie_pmu_writeq(pcie_pmu, HISI_PCIE_EXT_CNT, idx, HISI_PCIE_INIT_VAL);
441}
442
443static void hisi_pcie_pmu_enable_counter(struct hisi_pcie_pmu *pcie_pmu, struct hw_perf_event *hwc)
444{
445 u32 idx = hwc->idx;
446 u64 val;
447
448 val = hisi_pcie_pmu_readq(pcie_pmu, HISI_PCIE_EVENT_CTRL, idx);
449 val |= HISI_PCIE_EVENT_EN;
450 hisi_pcie_pmu_writeq(pcie_pmu, HISI_PCIE_EVENT_CTRL, idx, val);
451}
452
453static void hisi_pcie_pmu_disable_counter(struct hisi_pcie_pmu *pcie_pmu, struct hw_perf_event *hwc)
454{
455 u32 idx = hwc->idx;
456 u64 val;
457
458 val = hisi_pcie_pmu_readq(pcie_pmu, HISI_PCIE_EVENT_CTRL, idx);
459 val &= ~HISI_PCIE_EVENT_EN;
460 hisi_pcie_pmu_writeq(pcie_pmu, HISI_PCIE_EVENT_CTRL, idx, val);
461}
462
463static void hisi_pcie_pmu_enable_int(struct hisi_pcie_pmu *pcie_pmu, struct hw_perf_event *hwc)
464{
465 u32 idx = hwc->idx;
466
467 hisi_pcie_pmu_writel(pcie_pmu, HISI_PCIE_INT_MASK, idx, 0);
468}
469
470static void hisi_pcie_pmu_disable_int(struct hisi_pcie_pmu *pcie_pmu, struct hw_perf_event *hwc)
471{
472 u32 idx = hwc->idx;
473
474 hisi_pcie_pmu_writel(pcie_pmu, HISI_PCIE_INT_MASK, idx, 1);
475}
476
477static void hisi_pcie_pmu_reset_counter(struct hisi_pcie_pmu *pcie_pmu, int idx)
478{
479 hisi_pcie_pmu_writeq(pcie_pmu, HISI_PCIE_EVENT_CTRL, idx, HISI_PCIE_RESET_CNT);
480 hisi_pcie_pmu_writeq(pcie_pmu, HISI_PCIE_EVENT_CTRL, idx, HISI_PCIE_INIT_SET);
481}
482
483static void hisi_pcie_pmu_start(struct perf_event *event, int flags)
484{
485 struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(event->pmu);
486 struct hw_perf_event *hwc = &event->hw;
487 int idx = hwc->idx;
488 u64 prev_cnt;
489
490 if (WARN_ON_ONCE(!(hwc->state & PERF_HES_STOPPED)))
491 return;
492
493 WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE));
494 hwc->state = 0;
495
496 hisi_pcie_pmu_config_filter(event);
497 hisi_pcie_pmu_enable_counter(pcie_pmu, hwc);
498 hisi_pcie_pmu_enable_int(pcie_pmu, hwc);
499 hisi_pcie_pmu_set_period(event);
500
501 if (flags & PERF_EF_RELOAD) {
502 prev_cnt = local64_read(&hwc->prev_count);
503 hisi_pcie_pmu_writeq(pcie_pmu, hwc->event_base, idx, prev_cnt);
504 }
505
506 perf_event_update_userpage(event);
507}
508
509static void hisi_pcie_pmu_stop(struct perf_event *event, int flags)
510{
511 struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(event->pmu);
512 struct hw_perf_event *hwc = &event->hw;
513
514 hisi_pcie_pmu_event_update(event);
515 hisi_pcie_pmu_disable_int(pcie_pmu, hwc);
516 hisi_pcie_pmu_disable_counter(pcie_pmu, hwc);
517 hisi_pcie_pmu_clear_filter(event);
518 WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED);
519 hwc->state |= PERF_HES_STOPPED;
520
521 if (hwc->state & PERF_HES_UPTODATE)
522 return;
523
524 hwc->state |= PERF_HES_UPTODATE;
525}
526
527static int hisi_pcie_pmu_add(struct perf_event *event, int flags)
528{
529 struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(event->pmu);
530 struct hw_perf_event *hwc = &event->hw;
531 int idx;
532
533 hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
534
535 /* Check all working events to find a related event. */
536 idx = hisi_pcie_pmu_find_related_event(pcie_pmu, event);
537 if (idx < 0)
538 return idx;
539
540 /* Current event shares an enabled counter with the related event */
541 if (idx < HISI_PCIE_MAX_COUNTERS) {
542 hwc->idx = idx;
543 goto start_count;
544 }
545
546 idx = hisi_pcie_pmu_get_event_idx(pcie_pmu);
547 if (idx < 0)
548 return idx;
549
550 hwc->idx = idx;
551 pcie_pmu->hw_events[idx] = event;
552 /* Reset Counter to avoid previous statistic interference. */
553 hisi_pcie_pmu_reset_counter(pcie_pmu, idx);
554
555start_count:
556 if (flags & PERF_EF_START)
557 hisi_pcie_pmu_start(event, PERF_EF_RELOAD);
558
559 return 0;
560}
561
562static void hisi_pcie_pmu_del(struct perf_event *event, int flags)
563{
564 struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(event->pmu);
565 struct hw_perf_event *hwc = &event->hw;
566
567 hisi_pcie_pmu_stop(event, PERF_EF_UPDATE);
568 pcie_pmu->hw_events[hwc->idx] = NULL;
569 perf_event_update_userpage(event);
570}
571
572static void hisi_pcie_pmu_enable(struct pmu *pmu)
573{
574 struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(pmu);
575 int num;
576
577 for (num = 0; num < HISI_PCIE_MAX_COUNTERS; num++) {
578 if (pcie_pmu->hw_events[num])
579 break;
580 }
581
582 if (num == HISI_PCIE_MAX_COUNTERS)
583 return;
584
585 writel(HISI_PCIE_GLOBAL_EN, pcie_pmu->base + HISI_PCIE_GLOBAL_CTRL);
586}
587
588static void hisi_pcie_pmu_disable(struct pmu *pmu)
589{
590 struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(pmu);
591
592 writel(HISI_PCIE_GLOBAL_NONE, pcie_pmu->base + HISI_PCIE_GLOBAL_CTRL);
593}
594
595static irqreturn_t hisi_pcie_pmu_irq(int irq, void *data)
596{
597 struct hisi_pcie_pmu *pcie_pmu = data;
598 irqreturn_t ret = IRQ_NONE;
599 struct perf_event *event;
600 u32 overflown;
601 int idx;
602
603 for (idx = 0; idx < HISI_PCIE_MAX_COUNTERS; idx++) {
604 overflown = hisi_pcie_pmu_readl(pcie_pmu, HISI_PCIE_INT_STAT, idx);
605 if (!overflown)
606 continue;
607
608 /* Clear status of interrupt. */
609 hisi_pcie_pmu_writel(pcie_pmu, HISI_PCIE_INT_STAT, idx, 1);
610 event = pcie_pmu->hw_events[idx];
611 if (!event)
612 continue;
613
614 hisi_pcie_pmu_event_update(event);
615 hisi_pcie_pmu_set_period(event);
616 ret = IRQ_HANDLED;
617 }
618
619 return ret;
620}
621
622static int hisi_pcie_pmu_irq_register(struct pci_dev *pdev, struct hisi_pcie_pmu *pcie_pmu)
623{
624 int irq, ret;
625
626 ret = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_MSI);
627 if (ret < 0) {
628 pci_err(pdev, "Failed to enable MSI vectors: %d\n", ret);
629 return ret;
630 }
631
632 irq = pci_irq_vector(pdev, 0);
633 ret = request_irq(irq, hisi_pcie_pmu_irq, IRQF_NOBALANCING | IRQF_NO_THREAD, DRV_NAME,
634 pcie_pmu);
635 if (ret) {
636 pci_err(pdev, "Failed to register IRQ: %d\n", ret);
637 pci_free_irq_vectors(pdev);
638 return ret;
639 }
640
641 pcie_pmu->irq = irq;
642
643 return 0;
644}
645
646static void hisi_pcie_pmu_irq_unregister(struct pci_dev *pdev, struct hisi_pcie_pmu *pcie_pmu)
647{
648 free_irq(pcie_pmu->irq, pcie_pmu);
649 pci_free_irq_vectors(pdev);
650}
651
652static int hisi_pcie_pmu_online_cpu(unsigned int cpu, struct hlist_node *node)
653{
654 struct hisi_pcie_pmu *pcie_pmu = hlist_entry_safe(node, struct hisi_pcie_pmu, node);
655
656 if (pcie_pmu->on_cpu == -1) {
657 pcie_pmu->on_cpu = cpu;
658 WARN_ON(irq_set_affinity(pcie_pmu->irq, cpumask_of(cpu)));
659 }
660
661 return 0;
662}
663
664static int hisi_pcie_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node)
665{
666 struct hisi_pcie_pmu *pcie_pmu = hlist_entry_safe(node, struct hisi_pcie_pmu, node);
667 unsigned int target;
668
669 /* Nothing to do if this CPU doesn't own the PMU */
670 if (pcie_pmu->on_cpu != cpu)
671 return 0;
672
673 pcie_pmu->on_cpu = -1;
674 /* Choose a new CPU from all online cpus. */
675 target = cpumask_first(cpu_online_mask);
676 if (target >= nr_cpu_ids) {
677 pci_err(pcie_pmu->pdev, "There is no CPU to set\n");
678 return 0;
679 }
680
681 perf_pmu_migrate_context(&pcie_pmu->pmu, cpu, target);
682 /* Use this CPU for event counting */
683 pcie_pmu->on_cpu = target;
684 WARN_ON(irq_set_affinity(pcie_pmu->irq, cpumask_of(target)));
685
686 return 0;
687}
688
689static struct attribute *hisi_pcie_pmu_events_attr[] = {
690 HISI_PCIE_PMU_EVENT_ATTR(rx_mwr_latency, 0x0010),
691 HISI_PCIE_PMU_EVENT_ATTR(rx_mwr_cnt, 0x10010),
692 HISI_PCIE_PMU_EVENT_ATTR(rx_mrd_latency, 0x0210),
693 HISI_PCIE_PMU_EVENT_ATTR(rx_mrd_cnt, 0x10210),
694 HISI_PCIE_PMU_EVENT_ATTR(tx_mrd_latency, 0x0011),
695 HISI_PCIE_PMU_EVENT_ATTR(tx_mrd_cnt, 0x10011),
696 HISI_PCIE_PMU_EVENT_ATTR(rx_mrd_flux, 0x1005),
697 HISI_PCIE_PMU_EVENT_ATTR(rx_mrd_time, 0x11005),
698 HISI_PCIE_PMU_EVENT_ATTR(tx_mrd_flux, 0x2004),
699 HISI_PCIE_PMU_EVENT_ATTR(tx_mrd_time, 0x12004),
700 NULL
701};
702
703static struct attribute_group hisi_pcie_pmu_events_group = {
704 .name = "events",
705 .attrs = hisi_pcie_pmu_events_attr,
706};
707
708static struct attribute *hisi_pcie_pmu_format_attr[] = {
709 HISI_PCIE_PMU_FORMAT_ATTR(event, "config:0-16"),
710 HISI_PCIE_PMU_FORMAT_ATTR(thr_len, "config1:0-3"),
711 HISI_PCIE_PMU_FORMAT_ATTR(thr_mode, "config1:4"),
712 HISI_PCIE_PMU_FORMAT_ATTR(trig_len, "config1:5-8"),
713 HISI_PCIE_PMU_FORMAT_ATTR(trig_mode, "config1:9"),
714 HISI_PCIE_PMU_FORMAT_ATTR(port, "config2:0-15"),
715 HISI_PCIE_PMU_FORMAT_ATTR(bdf, "config2:16-31"),
716 NULL
717};
718
719static const struct attribute_group hisi_pcie_pmu_format_group = {
720 .name = "format",
721 .attrs = hisi_pcie_pmu_format_attr,
722};
723
724static struct attribute *hisi_pcie_pmu_bus_attrs[] = {
725 &dev_attr_bus.attr,
726 NULL
727};
728
729static const struct attribute_group hisi_pcie_pmu_bus_attr_group = {
730 .attrs = hisi_pcie_pmu_bus_attrs,
731};
732
733static struct attribute *hisi_pcie_pmu_cpumask_attrs[] = {
734 &dev_attr_cpumask.attr,
735 NULL
736};
737
738static const struct attribute_group hisi_pcie_pmu_cpumask_attr_group = {
739 .attrs = hisi_pcie_pmu_cpumask_attrs,
740};
741
742static struct attribute *hisi_pcie_pmu_identifier_attrs[] = {
743 &dev_attr_identifier.attr,
744 NULL
745};
746
747static const struct attribute_group hisi_pcie_pmu_identifier_attr_group = {
748 .attrs = hisi_pcie_pmu_identifier_attrs,
749};
750
751static const struct attribute_group *hisi_pcie_pmu_attr_groups[] = {
752 &hisi_pcie_pmu_events_group,
753 &hisi_pcie_pmu_format_group,
754 &hisi_pcie_pmu_bus_attr_group,
755 &hisi_pcie_pmu_cpumask_attr_group,
756 &hisi_pcie_pmu_identifier_attr_group,
757 NULL
758};
759
760static int hisi_pcie_alloc_pmu(struct pci_dev *pdev, struct hisi_pcie_pmu *pcie_pmu)
761{
762 struct hisi_pcie_reg_pair regs;
763 u16 sicl_id, core_id;
764 char *name;
765
766 regs = hisi_pcie_parse_reg_value(pcie_pmu, HISI_PCIE_REG_BDF);
767 pcie_pmu->bdf_min = regs.lo;
768 pcie_pmu->bdf_max = regs.hi;
769
770 regs = hisi_pcie_parse_reg_value(pcie_pmu, HISI_PCIE_REG_INFO);
771 sicl_id = regs.hi;
772 core_id = regs.lo;
773
774 name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "hisi_pcie%u_core%u", sicl_id, core_id);
775 if (!name)
776 return -ENOMEM;
777
778 pcie_pmu->pdev = pdev;
779 pcie_pmu->on_cpu = -1;
780 pcie_pmu->identifier = readl(pcie_pmu->base + HISI_PCIE_REG_VERSION);
781 pcie_pmu->pmu = (struct pmu) {
782 .name = name,
783 .module = THIS_MODULE,
784 .event_init = hisi_pcie_pmu_event_init,
785 .pmu_enable = hisi_pcie_pmu_enable,
786 .pmu_disable = hisi_pcie_pmu_disable,
787 .add = hisi_pcie_pmu_add,
788 .del = hisi_pcie_pmu_del,
789 .start = hisi_pcie_pmu_start,
790 .stop = hisi_pcie_pmu_stop,
791 .read = hisi_pcie_pmu_read,
792 .task_ctx_nr = perf_invalid_context,
793 .attr_groups = hisi_pcie_pmu_attr_groups,
794 .capabilities = PERF_PMU_CAP_NO_EXCLUDE,
795 };
796
797 return 0;
798}
799
800static int hisi_pcie_init_pmu(struct pci_dev *pdev, struct hisi_pcie_pmu *pcie_pmu)
801{
802 int ret;
803
804 pcie_pmu->base = pci_ioremap_bar(pdev, 2);
805 if (!pcie_pmu->base) {
806 pci_err(pdev, "Ioremap failed for pcie_pmu resource\n");
807 return -ENOMEM;
808 }
809
810 ret = hisi_pcie_alloc_pmu(pdev, pcie_pmu);
811 if (ret)
812 goto err_iounmap;
813
814 ret = hisi_pcie_pmu_irq_register(pdev, pcie_pmu);
815 if (ret)
816 goto err_iounmap;
817
818 ret = cpuhp_state_add_instance(CPUHP_AP_PERF_ARM_HISI_PCIE_PMU_ONLINE, &pcie_pmu->node);
819 if (ret) {
820 pci_err(pdev, "Failed to register hotplug: %d\n", ret);
821 goto err_irq_unregister;
822 }
823
824 ret = perf_pmu_register(&pcie_pmu->pmu, pcie_pmu->pmu.name, -1);
825 if (ret) {
826 pci_err(pdev, "Failed to register PCIe PMU: %d\n", ret);
827 goto err_hotplug_unregister;
828 }
829
830 return ret;
831
832err_hotplug_unregister:
833 cpuhp_state_remove_instance_nocalls(
834 CPUHP_AP_PERF_ARM_HISI_PCIE_PMU_ONLINE, &pcie_pmu->node);
835
836err_irq_unregister:
837 hisi_pcie_pmu_irq_unregister(pdev, pcie_pmu);
838
839err_iounmap:
840 iounmap(pcie_pmu->base);
841
842 return ret;
843}
844
845static void hisi_pcie_uninit_pmu(struct pci_dev *pdev)
846{
847 struct hisi_pcie_pmu *pcie_pmu = pci_get_drvdata(pdev);
848
849 perf_pmu_unregister(&pcie_pmu->pmu);
850 cpuhp_state_remove_instance_nocalls(
851 CPUHP_AP_PERF_ARM_HISI_PCIE_PMU_ONLINE, &pcie_pmu->node);
852 hisi_pcie_pmu_irq_unregister(pdev, pcie_pmu);
853 iounmap(pcie_pmu->base);
854}
855
856static int hisi_pcie_init_dev(struct pci_dev *pdev)
857{
858 int ret;
859
860 ret = pcim_enable_device(pdev);
861 if (ret) {
862 pci_err(pdev, "Failed to enable PCI device: %d\n", ret);
863 return ret;
864 }
865
866 ret = pcim_iomap_regions(pdev, BIT(2), DRV_NAME);
867 if (ret < 0) {
868 pci_err(pdev, "Failed to request PCI mem regions: %d\n", ret);
869 return ret;
870 }
871
872 pci_set_master(pdev);
873
874 return 0;
875}
876
877static int hisi_pcie_pmu_probe(struct pci_dev *pdev, const struct pci_device_id *id)
878{
879 struct hisi_pcie_pmu *pcie_pmu;
880 int ret;
881
882 pcie_pmu = devm_kzalloc(&pdev->dev, sizeof(*pcie_pmu), GFP_KERNEL);
883 if (!pcie_pmu)
884 return -ENOMEM;
885
886 ret = hisi_pcie_init_dev(pdev);
887 if (ret)
888 return ret;
889
890 ret = hisi_pcie_init_pmu(pdev, pcie_pmu);
891 if (ret)
892 return ret;
893
894 pci_set_drvdata(pdev, pcie_pmu);
895
896 return ret;
897}
898
899static void hisi_pcie_pmu_remove(struct pci_dev *pdev)
900{
901 hisi_pcie_uninit_pmu(pdev);
902 pci_set_drvdata(pdev, NULL);
903}
904
905static const struct pci_device_id hisi_pcie_pmu_ids[] = {
906 { PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, 0xa12d) },
907 { 0, }
908};
909MODULE_DEVICE_TABLE(pci, hisi_pcie_pmu_ids);
910
911static struct pci_driver hisi_pcie_pmu_driver = {
912 .name = DRV_NAME,
913 .id_table = hisi_pcie_pmu_ids,
914 .probe = hisi_pcie_pmu_probe,
915 .remove = hisi_pcie_pmu_remove,
916};
917
918static int __init hisi_pcie_module_init(void)
919{
920 int ret;
921
922 ret = cpuhp_setup_state_multi(CPUHP_AP_PERF_ARM_HISI_PCIE_PMU_ONLINE,
923 "AP_PERF_ARM_HISI_PCIE_PMU_ONLINE",
924 hisi_pcie_pmu_online_cpu,
925 hisi_pcie_pmu_offline_cpu);
926 if (ret) {
927 pr_err("Failed to setup PCIe PMU hotplug: %d\n", ret);
928 return ret;
929 }
930
931 ret = pci_register_driver(&hisi_pcie_pmu_driver);
932 if (ret)
933 cpuhp_remove_multi_state(CPUHP_AP_PERF_ARM_HISI_PCIE_PMU_ONLINE);
934
935 return ret;
936}
937module_init(hisi_pcie_module_init);
938
939static void __exit hisi_pcie_module_exit(void)
940{
941 pci_unregister_driver(&hisi_pcie_pmu_driver);
942 cpuhp_remove_multi_state(CPUHP_AP_PERF_ARM_HISI_PCIE_PMU_ONLINE);
943}
944module_exit(hisi_pcie_module_exit);
945
946MODULE_DESCRIPTION("HiSilicon PCIe PMU driver");
947MODULE_LICENSE("GPL v2");
948MODULE_AUTHOR("Qi Liu <liuqi115@huawei.com>");
include/linux/cpuhotplug.h+1−0
@@ -225,6 +225,7 @@ enum cpuhp_state { · 224 unmodified lines
225 CPUHP_AP_PERF_ARM_HISI_L3_ONLINE,225 CPUHP_AP_PERF_ARM_HISI_L3_ONLINE,
226 CPUHP_AP_PERF_ARM_HISI_PA_ONLINE,226 CPUHP_AP_PERF_ARM_HISI_PA_ONLINE,
227 CPUHP_AP_PERF_ARM_HISI_SLLC_ONLINE,227 CPUHP_AP_PERF_ARM_HISI_SLLC_ONLINE,
228 CPUHP_AP_PERF_ARM_HISI_PCIE_PMU_ONLINE,
228 CPUHP_AP_PERF_ARM_L2X0_ONLINE,229 CPUHP_AP_PERF_ARM_L2X0_ONLINE,
229 CPUHP_AP_PERF_ARM_QCOM_L2_ONLINE,230 CPUHP_AP_PERF_ARM_QCOM_L2_ONLINE,
230 CPUHP_AP_PERF_ARM_QCOM_L3_ONLINE,231 CPUHP_AP_PERF_ARM_QCOM_L3_ONLINE,

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t