| @@ -0,0 +1,948 @@ |
| 1 | +// SPDX-License-Identifier: GPL-2.0-only |
| 2 | +/* |
| 3 | + * This driver adds support for PCIe PMU RCiEP device. Related |
| 4 | + * perf events are bandwidth, latency etc. |
| 5 | + * |
| 6 | + * Copyright (C) 2021 HiSilicon Limited |
| 7 | + * Author: Qi Liu <liuqi115@huawei.com> |
| 8 | + */ |
| 9 | +#include <linux/bitfield.h> |
| 10 | +#include <linux/bitmap.h> |
| 11 | +#include <linux/bug.h> |
| 12 | +#include <linux/device.h> |
| 13 | +#include <linux/err.h> |
| 14 | +#include <linux/interrupt.h> |
| 15 | +#include <linux/irq.h> |
| 16 | +#include <linux/kernel.h> |
| 17 | +#include <linux/list.h> |
| 18 | +#include <linux/module.h> |
| 19 | +#include <linux/pci.h> |
| 20 | +#include <linux/perf_event.h> |
| 21 | + |
| 22 | +#define DRV_NAME "hisi_pcie_pmu" |
| 23 | +/* Define registers */ |
| 24 | +#define HISI_PCIE_GLOBAL_CTRL 0x00 |
| 25 | +#define HISI_PCIE_EVENT_CTRL 0x010 |
| 26 | +#define HISI_PCIE_CNT 0x090 |
| 27 | +#define HISI_PCIE_EXT_CNT 0x110 |
| 28 | +#define HISI_PCIE_INT_STAT 0x150 |
| 29 | +#define HISI_PCIE_INT_MASK 0x154 |
| 30 | +#define HISI_PCIE_REG_BDF 0xfe0 |
| 31 | +#define HISI_PCIE_REG_VERSION 0xfe4 |
| 32 | +#define HISI_PCIE_REG_INFO 0xfe8 |
| 33 | + |
| 34 | +/* Define command in HISI_PCIE_GLOBAL_CTRL */ |
| 35 | +#define HISI_PCIE_GLOBAL_EN 0x01 |
| 36 | +#define HISI_PCIE_GLOBAL_NONE 0 |
| 37 | + |
| 38 | +/* Define command in HISI_PCIE_EVENT_CTRL */ |
| 39 | +#define HISI_PCIE_EVENT_EN BIT_ULL(20) |
| 40 | +#define HISI_PCIE_RESET_CNT BIT_ULL(22) |
| 41 | +#define HISI_PCIE_INIT_SET BIT_ULL(34) |
| 42 | +#define HISI_PCIE_THR_EN BIT_ULL(26) |
| 43 | +#define HISI_PCIE_TARGET_EN BIT_ULL(32) |
| 44 | +#define HISI_PCIE_TRIG_EN BIT_ULL(52) |
| 45 | + |
| 46 | +/* Define offsets in HISI_PCIE_EVENT_CTRL */ |
| 47 | +#define HISI_PCIE_EVENT_M GENMASK_ULL(15, 0) |
| 48 | +#define HISI_PCIE_THR_MODE_M GENMASK_ULL(27, 27) |
| 49 | +#define HISI_PCIE_THR_M GENMASK_ULL(31, 28) |
| 50 | +#define HISI_PCIE_TARGET_M GENMASK_ULL(52, 36) |
| 51 | +#define HISI_PCIE_TRIG_MODE_M GENMASK_ULL(53, 53) |
| 52 | +#define HISI_PCIE_TRIG_M GENMASK_ULL(59, 56) |
| 53 | + |
| 54 | +#define HISI_PCIE_MAX_COUNTERS 8 |
| 55 | +#define HISI_PCIE_REG_STEP 8 |
| 56 | +#define HISI_PCIE_THR_MAX_VAL 10 |
| 57 | +#define HISI_PCIE_TRIG_MAX_VAL 10 |
| 58 | +#define HISI_PCIE_MAX_PERIOD (GENMASK_ULL(63, 0)) |
| 59 | +#define HISI_PCIE_INIT_VAL BIT_ULL(63) |
| 60 | + |
| 61 | +struct hisi_pcie_pmu { |
| 62 | + struct perf_event *hw_events[HISI_PCIE_MAX_COUNTERS]; |
| 63 | + struct hlist_node node; |
| 64 | + struct pci_dev *pdev; |
| 65 | + struct pmu pmu; |
| 66 | + void __iomem *base; |
| 67 | + int irq; |
| 68 | + u32 identifier; |
| 69 | + /* Minimum and maximum BDF of root ports monitored by PMU */ |
| 70 | + u16 bdf_min; |
| 71 | + u16 bdf_max; |
| 72 | + int on_cpu; |
| 73 | +}; |
| 74 | + |
| 75 | +struct hisi_pcie_reg_pair { |
| 76 | + u16 lo; |
| 77 | + u16 hi; |
| 78 | +}; |
| 79 | + |
| 80 | +#define to_pcie_pmu(p) (container_of((p), struct hisi_pcie_pmu, pmu)) |
| 81 | +#define GET_PCI_DEVFN(bdf) ((bdf) & 0xff) |
| 82 | + |
| 83 | +#define HISI_PCIE_PMU_FILTER_ATTR(_name, _config, _hi, _lo) \ |
| 84 | + static u64 hisi_pcie_get_##_name(struct perf_event *event) \ |
| 85 | + { \ |
| 86 | + return FIELD_GET(GENMASK(_hi, _lo), event->attr._config); \ |
| 87 | + } \ |
| 88 | + |
| 89 | +HISI_PCIE_PMU_FILTER_ATTR(event, config, 16, 0); |
| 90 | +HISI_PCIE_PMU_FILTER_ATTR(thr_len, config1, 3, 0); |
| 91 | +HISI_PCIE_PMU_FILTER_ATTR(thr_mode, config1, 4, 4); |
| 92 | +HISI_PCIE_PMU_FILTER_ATTR(trig_len, config1, 8, 5); |
| 93 | +HISI_PCIE_PMU_FILTER_ATTR(trig_mode, config1, 9, 9); |
| 94 | +HISI_PCIE_PMU_FILTER_ATTR(port, config2, 15, 0); |
| 95 | +HISI_PCIE_PMU_FILTER_ATTR(bdf, config2, 31, 16); |
| 96 | + |
| 97 | +static ssize_t hisi_pcie_format_sysfs_show(struct device *dev, struct device_attribute *attr, |
| 98 | + char *buf) |
| 99 | +{ |
| 100 | + struct dev_ext_attribute *eattr; |
| 101 | + |
| 102 | + eattr = container_of(attr, struct dev_ext_attribute, attr); |
| 103 | + |
| 104 | + return sysfs_emit(buf, "%s\n", (char *)eattr->var); |
| 105 | +} |
| 106 | + |
| 107 | +static ssize_t hisi_pcie_event_sysfs_show(struct device *dev, struct device_attribute *attr, |
| 108 | + char *buf) |
| 109 | +{ |
| 110 | + struct perf_pmu_events_attr *pmu_attr = |
| 111 | + container_of(attr, struct perf_pmu_events_attr, attr); |
| 112 | + |
| 113 | + return sysfs_emit(buf, "config=0x%llx\n", pmu_attr->id); |
| 114 | +} |
| 115 | + |
| 116 | +#define HISI_PCIE_PMU_FORMAT_ATTR(_name, _format) \ |
| 117 | + (&((struct dev_ext_attribute[]){ \ |
| 118 | + { .attr = __ATTR(_name, 0444, hisi_pcie_format_sysfs_show, \ |
| 119 | + NULL), \ |
| 120 | + .var = (void *)_format } \ |
| 121 | + })[0].attr.attr) |
| 122 | + |
| 123 | +#define HISI_PCIE_PMU_EVENT_ATTR(_name, _id) \ |
| 124 | + PMU_EVENT_ATTR_ID(_name, hisi_pcie_event_sysfs_show, _id) |
| 125 | + |
| 126 | +static ssize_t cpumask_show(struct device *dev, struct device_attribute *attr, char *buf) |
| 127 | +{ |
| 128 | + struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(dev_get_drvdata(dev)); |
| 129 | + |
| 130 | + return cpumap_print_to_pagebuf(true, buf, cpumask_of(pcie_pmu->on_cpu)); |
| 131 | +} |
| 132 | +static DEVICE_ATTR_RO(cpumask); |
| 133 | + |
| 134 | +static ssize_t identifier_show(struct device *dev, struct device_attribute *attr, char *buf) |
| 135 | +{ |
| 136 | + struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(dev_get_drvdata(dev)); |
| 137 | + |
| 138 | + return sysfs_emit(buf, "%#x\n", pcie_pmu->identifier); |
| 139 | +} |
| 140 | +static DEVICE_ATTR_RO(identifier); |
| 141 | + |
| 142 | +static ssize_t bus_show(struct device *dev, struct device_attribute *attr, char *buf) |
| 143 | +{ |
| 144 | + struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(dev_get_drvdata(dev)); |
| 145 | + |
| 146 | + return sysfs_emit(buf, "%#04x\n", PCI_BUS_NUM(pcie_pmu->bdf_min)); |
| 147 | +} |
| 148 | +static DEVICE_ATTR_RO(bus); |
| 149 | + |
| 150 | +static struct hisi_pcie_reg_pair |
| 151 | +hisi_pcie_parse_reg_value(struct hisi_pcie_pmu *pcie_pmu, u32 reg_off) |
| 152 | +{ |
| 153 | + u32 val = readl_relaxed(pcie_pmu->base + reg_off); |
| 154 | + struct hisi_pcie_reg_pair regs = { |
| 155 | + .lo = val, |
| 156 | + .hi = val >> 16, |
| 157 | + }; |
| 158 | + |
| 159 | + return regs; |
| 160 | +} |
| 161 | + |
| 162 | +/* |
| 163 | + * Hardware counter and ext_counter work together for bandwidth, latency, bus |
| 164 | + * utilization and buffer occupancy events. For example, RX memory write latency |
| 165 | + * events(index = 0x0010), counter counts total delay cycles and ext_counter |
| 166 | + * counts RX memory write PCIe packets number. |
| 167 | + * |
| 168 | + * As we don't want PMU driver to process these two data, "delay cycles" can |
| 169 | + * be treated as an independent event(index = 0x0010), "RX memory write packets |
| 170 | + * number" as another(index = 0x10010). BIT 16 is used to distinguish and 0-15 |
| 171 | + * bits are "real" event index, which can be used to set HISI_PCIE_EVENT_CTRL. |
| 172 | + */ |
| 173 | +#define EXT_COUNTER_IS_USED(idx) ((idx) & BIT(16)) |
| 174 | + |
| 175 | +static u32 hisi_pcie_get_real_event(struct perf_event *event) |
| 176 | +{ |
| 177 | + return hisi_pcie_get_event(event) & GENMASK(15, 0); |
| 178 | +} |
| 179 | + |
| 180 | +static u32 hisi_pcie_pmu_get_offset(u32 offset, u32 idx) |
| 181 | +{ |
| 182 | + return offset + HISI_PCIE_REG_STEP * idx; |
| 183 | +} |
| 184 | + |
| 185 | +static u32 hisi_pcie_pmu_readl(struct hisi_pcie_pmu *pcie_pmu, u32 reg_offset, |
| 186 | + u32 idx) |
| 187 | +{ |
| 188 | + u32 offset = hisi_pcie_pmu_get_offset(reg_offset, idx); |
| 189 | + |
| 190 | + return readl_relaxed(pcie_pmu->base + offset); |
| 191 | +} |
| 192 | + |
| 193 | +static void hisi_pcie_pmu_writel(struct hisi_pcie_pmu *pcie_pmu, u32 reg_offset, u32 idx, u32 val) |
| 194 | +{ |
| 195 | + u32 offset = hisi_pcie_pmu_get_offset(reg_offset, idx); |
| 196 | + |
| 197 | + writel_relaxed(val, pcie_pmu->base + offset); |
| 198 | +} |
| 199 | + |
| 200 | +static u64 hisi_pcie_pmu_readq(struct hisi_pcie_pmu *pcie_pmu, u32 reg_offset, u32 idx) |
| 201 | +{ |
| 202 | + u32 offset = hisi_pcie_pmu_get_offset(reg_offset, idx); |
| 203 | + |
| 204 | + return readq_relaxed(pcie_pmu->base + offset); |
| 205 | +} |
| 206 | + |
| 207 | +static void hisi_pcie_pmu_writeq(struct hisi_pcie_pmu *pcie_pmu, u32 reg_offset, u32 idx, u64 val) |
| 208 | +{ |
| 209 | + u32 offset = hisi_pcie_pmu_get_offset(reg_offset, idx); |
| 210 | + |
| 211 | + writeq_relaxed(val, pcie_pmu->base + offset); |
| 212 | +} |
| 213 | + |
| 214 | +static void hisi_pcie_pmu_config_filter(struct perf_event *event) |
| 215 | +{ |
| 216 | + struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(event->pmu); |
| 217 | + struct hw_perf_event *hwc = &event->hw; |
| 218 | + u64 reg = HISI_PCIE_INIT_SET; |
| 219 | + u64 port, trig_len, thr_len; |
| 220 | + |
| 221 | + /* Config HISI_PCIE_EVENT_CTRL according to event. */ |
| 222 | + reg |= FIELD_PREP(HISI_PCIE_EVENT_M, hisi_pcie_get_real_event(event)); |
| 223 | + |
| 224 | + /* Config HISI_PCIE_EVENT_CTRL according to root port or EP device. */ |
| 225 | + port = hisi_pcie_get_port(event); |
| 226 | + if (port) |
| 227 | + reg |= FIELD_PREP(HISI_PCIE_TARGET_M, port); |
| 228 | + else |
| 229 | + reg |= HISI_PCIE_TARGET_EN | |
| 230 | + FIELD_PREP(HISI_PCIE_TARGET_M, hisi_pcie_get_bdf(event)); |
| 231 | + |
| 232 | + /* Config HISI_PCIE_EVENT_CTRL according to trigger condition. */ |
| 233 | + trig_len = hisi_pcie_get_trig_len(event); |
| 234 | + if (trig_len) { |
| 235 | + reg |= FIELD_PREP(HISI_PCIE_TRIG_M, trig_len); |
| 236 | + reg |= FIELD_PREP(HISI_PCIE_TRIG_MODE_M, hisi_pcie_get_trig_mode(event)); |
| 237 | + reg |= HISI_PCIE_TRIG_EN; |
| 238 | + } |
| 239 | + |
| 240 | + /* Config HISI_PCIE_EVENT_CTRL according to threshold condition. */ |
| 241 | + thr_len = hisi_pcie_get_thr_len(event); |
| 242 | + if (thr_len) { |
| 243 | + reg |= FIELD_PREP(HISI_PCIE_THR_M, thr_len); |
| 244 | + reg |= FIELD_PREP(HISI_PCIE_THR_MODE_M, hisi_pcie_get_thr_mode(event)); |
| 245 | + reg |= HISI_PCIE_THR_EN; |
| 246 | + } |
| 247 | + |
| 248 | + hisi_pcie_pmu_writeq(pcie_pmu, HISI_PCIE_EVENT_CTRL, hwc->idx, reg); |
| 249 | +} |
| 250 | + |
| 251 | +static void hisi_pcie_pmu_clear_filter(struct perf_event *event) |
| 252 | +{ |
| 253 | + struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(event->pmu); |
| 254 | + struct hw_perf_event *hwc = &event->hw; |
| 255 | + |
| 256 | + hisi_pcie_pmu_writeq(pcie_pmu, HISI_PCIE_EVENT_CTRL, hwc->idx, HISI_PCIE_INIT_SET); |
| 257 | +} |
| 258 | + |
| 259 | +static bool hisi_pcie_pmu_valid_requester_id(struct hisi_pcie_pmu *pcie_pmu, u32 bdf) |
| 260 | +{ |
| 261 | + struct pci_dev *root_port, *pdev; |
| 262 | + u16 rp_bdf; |
| 263 | + |
| 264 | + pdev = pci_get_domain_bus_and_slot(pci_domain_nr(pcie_pmu->pdev->bus), PCI_BUS_NUM(bdf), |
| 265 | + GET_PCI_DEVFN(bdf)); |
| 266 | + if (!pdev) |
| 267 | + return false; |
| 268 | + |
| 269 | + root_port = pcie_find_root_port(pdev); |
| 270 | + if (!root_port) { |
| 271 | + pci_dev_put(pdev); |
| 272 | + return false; |
| 273 | + } |
| 274 | + |
| 275 | + pci_dev_put(pdev); |
| 276 | + rp_bdf = pci_dev_id(root_port); |
| 277 | + return rp_bdf >= pcie_pmu->bdf_min && rp_bdf <= pcie_pmu->bdf_max; |
| 278 | +} |
| 279 | + |
| 280 | +static bool hisi_pcie_pmu_valid_filter(struct perf_event *event, |
| 281 | + struct hisi_pcie_pmu *pcie_pmu) |
| 282 | +{ |
| 283 | + u32 requester_id = hisi_pcie_get_bdf(event); |
| 284 | + |
| 285 | + if (hisi_pcie_get_thr_len(event) > HISI_PCIE_THR_MAX_VAL) |
| 286 | + return false; |
| 287 | + |
| 288 | + if (hisi_pcie_get_trig_len(event) > HISI_PCIE_TRIG_MAX_VAL) |
| 289 | + return false; |
| 290 | + |
| 291 | + if (requester_id) { |
| 292 | + if (!hisi_pcie_pmu_valid_requester_id(pcie_pmu, requester_id)) |
| 293 | + return false; |
| 294 | + } |
| 295 | + |
| 296 | + return true; |
| 297 | +} |
| 298 | + |
| 299 | +static bool hisi_pcie_pmu_cmp_event(struct perf_event *target, |
| 300 | + struct perf_event *event) |
| 301 | +{ |
| 302 | + return hisi_pcie_get_real_event(target) == hisi_pcie_get_real_event(event); |
| 303 | +} |
| 304 | + |
| 305 | +static bool hisi_pcie_pmu_validate_event_group(struct perf_event *event) |
| 306 | +{ |
| 307 | + struct perf_event *sibling, *leader = event->group_leader; |
| 308 | + struct perf_event *event_group[HISI_PCIE_MAX_COUNTERS]; |
| 309 | + int counters = 1; |
| 310 | + int num; |
| 311 | + |
| 312 | + event_group[0] = leader; |
| 313 | + if (!is_software_event(leader)) { |
| 314 | + if (leader->pmu != event->pmu) |
| 315 | + return false; |
| 316 | + |
| 317 | + if (leader != event && !hisi_pcie_pmu_cmp_event(leader, event)) |
| 318 | + event_group[counters++] = event; |
| 319 | + } |
| 320 | + |
| 321 | + for_each_sibling_event(sibling, event->group_leader) { |
| 322 | + if (is_software_event(sibling)) |
| 323 | + continue; |
| 324 | + |
| 325 | + if (sibling->pmu != event->pmu) |
| 326 | + return false; |
| 327 | + |
| 328 | + for (num = 0; num < counters; num++) { |
| 329 | + if (hisi_pcie_pmu_cmp_event(event_group[num], sibling)) |
| 330 | + break; |
| 331 | + } |
| 332 | + |
| 333 | + if (num == counters) |
| 334 | + event_group[counters++] = sibling; |
| 335 | + } |
| 336 | + |
| 337 | + return counters <= HISI_PCIE_MAX_COUNTERS; |
| 338 | +} |
| 339 | + |
| 340 | +static int hisi_pcie_pmu_event_init(struct perf_event *event) |
| 341 | +{ |
| 342 | + struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(event->pmu); |
| 343 | + struct hw_perf_event *hwc = &event->hw; |
| 344 | + |
| 345 | + event->cpu = pcie_pmu->on_cpu; |
| 346 | + |
| 347 | + if (EXT_COUNTER_IS_USED(hisi_pcie_get_event(event))) |
| 348 | + hwc->event_base = HISI_PCIE_EXT_CNT; |
| 349 | + else |
| 350 | + hwc->event_base = HISI_PCIE_CNT; |
| 351 | + |
| 352 | + if (event->attr.type != event->pmu->type) |
| 353 | + return -ENOENT; |
| 354 | + |
| 355 | + /* Sampling is not supported. */ |
| 356 | + if (is_sampling_event(event) || event->attach_state & PERF_ATTACH_TASK) |
| 357 | + return -EOPNOTSUPP; |
| 358 | + |
| 359 | + if (!hisi_pcie_pmu_valid_filter(event, pcie_pmu)) |
| 360 | + return -EINVAL; |
| 361 | + |
| 362 | + if (!hisi_pcie_pmu_validate_event_group(event)) |
| 363 | + return -EINVAL; |
| 364 | + |
| 365 | + return 0; |
| 366 | +} |
| 367 | + |
| 368 | +static u64 hisi_pcie_pmu_read_counter(struct perf_event *event) |
| 369 | +{ |
| 370 | + struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(event->pmu); |
| 371 | + u32 idx = event->hw.idx; |
| 372 | + |
| 373 | + return hisi_pcie_pmu_readq(pcie_pmu, event->hw.event_base, idx); |
| 374 | +} |
| 375 | + |
| 376 | +static int hisi_pcie_pmu_find_related_event(struct hisi_pcie_pmu *pcie_pmu, |
| 377 | + struct perf_event *event) |
| 378 | +{ |
| 379 | + struct perf_event *sibling; |
| 380 | + int idx; |
| 381 | + |
| 382 | + for (idx = 0; idx < HISI_PCIE_MAX_COUNTERS; idx++) { |
| 383 | + sibling = pcie_pmu->hw_events[idx]; |
| 384 | + if (!sibling) |
| 385 | + continue; |
| 386 | + |
| 387 | + if (!hisi_pcie_pmu_cmp_event(sibling, event)) |
| 388 | + continue; |
| 389 | + |
| 390 | + /* Related events must be used in group */ |
| 391 | + if (sibling->group_leader == event->group_leader) |
| 392 | + return idx; |
| 393 | + else |
| 394 | + return -EINVAL; |
| 395 | + } |
| 396 | + |
| 397 | + return idx; |
| 398 | +} |
| 399 | + |
| 400 | +static int hisi_pcie_pmu_get_event_idx(struct hisi_pcie_pmu *pcie_pmu) |
| 401 | +{ |
| 402 | + int idx; |
| 403 | + |
| 404 | + for (idx = 0; idx < HISI_PCIE_MAX_COUNTERS; idx++) { |
| 405 | + if (!pcie_pmu->hw_events[idx]) |
| 406 | + return idx; |
| 407 | + } |
| 408 | + |
| 409 | + return -EINVAL; |
| 410 | +} |
| 411 | + |
| 412 | +static void hisi_pcie_pmu_event_update(struct perf_event *event) |
| 413 | +{ |
| 414 | + struct hw_perf_event *hwc = &event->hw; |
| 415 | + u64 new_cnt, prev_cnt, delta; |
| 416 | + |
| 417 | + do { |
| 418 | + prev_cnt = local64_read(&hwc->prev_count); |
| 419 | + new_cnt = hisi_pcie_pmu_read_counter(event); |
| 420 | + } while (local64_cmpxchg(&hwc->prev_count, prev_cnt, |
| 421 | + new_cnt) != prev_cnt); |
| 422 | + |
| 423 | + delta = (new_cnt - prev_cnt) & HISI_PCIE_MAX_PERIOD; |
| 424 | + local64_add(delta, &event->count); |
| 425 | +} |
| 426 | + |
| 427 | +static void hisi_pcie_pmu_read(struct perf_event *event) |
| 428 | +{ |
| 429 | + hisi_pcie_pmu_event_update(event); |
| 430 | +} |
| 431 | + |
| 432 | +static void hisi_pcie_pmu_set_period(struct perf_event *event) |
| 433 | +{ |
| 434 | + struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(event->pmu); |
| 435 | + struct hw_perf_event *hwc = &event->hw; |
| 436 | + int idx = hwc->idx; |
| 437 | + |
| 438 | + local64_set(&hwc->prev_count, HISI_PCIE_INIT_VAL); |
| 439 | + hisi_pcie_pmu_writeq(pcie_pmu, HISI_PCIE_CNT, idx, HISI_PCIE_INIT_VAL); |
| 440 | + hisi_pcie_pmu_writeq(pcie_pmu, HISI_PCIE_EXT_CNT, idx, HISI_PCIE_INIT_VAL); |
| 441 | +} |
| 442 | + |
| 443 | +static void hisi_pcie_pmu_enable_counter(struct hisi_pcie_pmu *pcie_pmu, struct hw_perf_event *hwc) |
| 444 | +{ |
| 445 | + u32 idx = hwc->idx; |
| 446 | + u64 val; |
| 447 | + |
| 448 | + val = hisi_pcie_pmu_readq(pcie_pmu, HISI_PCIE_EVENT_CTRL, idx); |
| 449 | + val |= HISI_PCIE_EVENT_EN; |
| 450 | + hisi_pcie_pmu_writeq(pcie_pmu, HISI_PCIE_EVENT_CTRL, idx, val); |
| 451 | +} |
| 452 | + |
| 453 | +static void hisi_pcie_pmu_disable_counter(struct hisi_pcie_pmu *pcie_pmu, struct hw_perf_event *hwc) |
| 454 | +{ |
| 455 | + u32 idx = hwc->idx; |
| 456 | + u64 val; |
| 457 | + |
| 458 | + val = hisi_pcie_pmu_readq(pcie_pmu, HISI_PCIE_EVENT_CTRL, idx); |
| 459 | + val &= ~HISI_PCIE_EVENT_EN; |
| 460 | + hisi_pcie_pmu_writeq(pcie_pmu, HISI_PCIE_EVENT_CTRL, idx, val); |
| 461 | +} |
| 462 | + |
| 463 | +static void hisi_pcie_pmu_enable_int(struct hisi_pcie_pmu *pcie_pmu, struct hw_perf_event *hwc) |
| 464 | +{ |
| 465 | + u32 idx = hwc->idx; |
| 466 | + |
| 467 | + hisi_pcie_pmu_writel(pcie_pmu, HISI_PCIE_INT_MASK, idx, 0); |
| 468 | +} |
| 469 | + |
| 470 | +static void hisi_pcie_pmu_disable_int(struct hisi_pcie_pmu *pcie_pmu, struct hw_perf_event *hwc) |
| 471 | +{ |
| 472 | + u32 idx = hwc->idx; |
| 473 | + |
| 474 | + hisi_pcie_pmu_writel(pcie_pmu, HISI_PCIE_INT_MASK, idx, 1); |
| 475 | +} |
| 476 | + |
| 477 | +static void hisi_pcie_pmu_reset_counter(struct hisi_pcie_pmu *pcie_pmu, int idx) |
| 478 | +{ |
| 479 | + hisi_pcie_pmu_writeq(pcie_pmu, HISI_PCIE_EVENT_CTRL, idx, HISI_PCIE_RESET_CNT); |
| 480 | + hisi_pcie_pmu_writeq(pcie_pmu, HISI_PCIE_EVENT_CTRL, idx, HISI_PCIE_INIT_SET); |
| 481 | +} |
| 482 | + |
| 483 | +static void hisi_pcie_pmu_start(struct perf_event *event, int flags) |
| 484 | +{ |
| 485 | + struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(event->pmu); |
| 486 | + struct hw_perf_event *hwc = &event->hw; |
| 487 | + int idx = hwc->idx; |
| 488 | + u64 prev_cnt; |
| 489 | + |
| 490 | + if (WARN_ON_ONCE(!(hwc->state & PERF_HES_STOPPED))) |
| 491 | + return; |
| 492 | + |
| 493 | + WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE)); |
| 494 | + hwc->state = 0; |
| 495 | + |
| 496 | + hisi_pcie_pmu_config_filter(event); |
| 497 | + hisi_pcie_pmu_enable_counter(pcie_pmu, hwc); |
| 498 | + hisi_pcie_pmu_enable_int(pcie_pmu, hwc); |
| 499 | + hisi_pcie_pmu_set_period(event); |
| 500 | + |
| 501 | + if (flags & PERF_EF_RELOAD) { |
| 502 | + prev_cnt = local64_read(&hwc->prev_count); |
| 503 | + hisi_pcie_pmu_writeq(pcie_pmu, hwc->event_base, idx, prev_cnt); |
| 504 | + } |
| 505 | + |
| 506 | + perf_event_update_userpage(event); |
| 507 | +} |
| 508 | + |
| 509 | +static void hisi_pcie_pmu_stop(struct perf_event *event, int flags) |
| 510 | +{ |
| 511 | + struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(event->pmu); |
| 512 | + struct hw_perf_event *hwc = &event->hw; |
| 513 | + |
| 514 | + hisi_pcie_pmu_event_update(event); |
| 515 | + hisi_pcie_pmu_disable_int(pcie_pmu, hwc); |
| 516 | + hisi_pcie_pmu_disable_counter(pcie_pmu, hwc); |
| 517 | + hisi_pcie_pmu_clear_filter(event); |
| 518 | + WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED); |
| 519 | + hwc->state |= PERF_HES_STOPPED; |
| 520 | + |
| 521 | + if (hwc->state & PERF_HES_UPTODATE) |
| 522 | + return; |
| 523 | + |
| 524 | + hwc->state |= PERF_HES_UPTODATE; |
| 525 | +} |
| 526 | + |
| 527 | +static int hisi_pcie_pmu_add(struct perf_event *event, int flags) |
| 528 | +{ |
| 529 | + struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(event->pmu); |
| 530 | + struct hw_perf_event *hwc = &event->hw; |
| 531 | + int idx; |
| 532 | + |
| 533 | + hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE; |
| 534 | + |
| 535 | + /* Check all working events to find a related event. */ |
| 536 | + idx = hisi_pcie_pmu_find_related_event(pcie_pmu, event); |
| 537 | + if (idx < 0) |
| 538 | + return idx; |
| 539 | + |
| 540 | + /* Current event shares an enabled counter with the related event */ |
| 541 | + if (idx < HISI_PCIE_MAX_COUNTERS) { |
| 542 | + hwc->idx = idx; |
| 543 | + goto start_count; |
| 544 | + } |
| 545 | + |
| 546 | + idx = hisi_pcie_pmu_get_event_idx(pcie_pmu); |
| 547 | + if (idx < 0) |
| 548 | + return idx; |
| 549 | + |
| 550 | + hwc->idx = idx; |
| 551 | + pcie_pmu->hw_events[idx] = event; |
| 552 | + /* Reset Counter to avoid previous statistic interference. */ |
| 553 | + hisi_pcie_pmu_reset_counter(pcie_pmu, idx); |
| 554 | + |
| 555 | +start_count: |
| 556 | + if (flags & PERF_EF_START) |
| 557 | + hisi_pcie_pmu_start(event, PERF_EF_RELOAD); |
| 558 | + |
| 559 | + return 0; |
| 560 | +} |
| 561 | + |
| 562 | +static void hisi_pcie_pmu_del(struct perf_event *event, int flags) |
| 563 | +{ |
| 564 | + struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(event->pmu); |
| 565 | + struct hw_perf_event *hwc = &event->hw; |
| 566 | + |
| 567 | + hisi_pcie_pmu_stop(event, PERF_EF_UPDATE); |
| 568 | + pcie_pmu->hw_events[hwc->idx] = NULL; |
| 569 | + perf_event_update_userpage(event); |
| 570 | +} |
| 571 | + |
| 572 | +static void hisi_pcie_pmu_enable(struct pmu *pmu) |
| 573 | +{ |
| 574 | + struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(pmu); |
| 575 | + int num; |
| 576 | + |
| 577 | + for (num = 0; num < HISI_PCIE_MAX_COUNTERS; num++) { |
| 578 | + if (pcie_pmu->hw_events[num]) |
| 579 | + break; |
| 580 | + } |
| 581 | + |
| 582 | + if (num == HISI_PCIE_MAX_COUNTERS) |
| 583 | + return; |
| 584 | + |
| 585 | + writel(HISI_PCIE_GLOBAL_EN, pcie_pmu->base + HISI_PCIE_GLOBAL_CTRL); |
| 586 | +} |
| 587 | + |
| 588 | +static void hisi_pcie_pmu_disable(struct pmu *pmu) |
| 589 | +{ |
| 590 | + struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(pmu); |
| 591 | + |
| 592 | + writel(HISI_PCIE_GLOBAL_NONE, pcie_pmu->base + HISI_PCIE_GLOBAL_CTRL); |
| 593 | +} |
| 594 | + |
| 595 | +static irqreturn_t hisi_pcie_pmu_irq(int irq, void *data) |
| 596 | +{ |
| 597 | + struct hisi_pcie_pmu *pcie_pmu = data; |
| 598 | + irqreturn_t ret = IRQ_NONE; |
| 599 | + struct perf_event *event; |
| 600 | + u32 overflown; |
| 601 | + int idx; |
| 602 | + |
| 603 | + for (idx = 0; idx < HISI_PCIE_MAX_COUNTERS; idx++) { |
| 604 | + overflown = hisi_pcie_pmu_readl(pcie_pmu, HISI_PCIE_INT_STAT, idx); |
| 605 | + if (!overflown) |
| 606 | + continue; |
| 607 | + |
| 608 | + /* Clear status of interrupt. */ |
| 609 | + hisi_pcie_pmu_writel(pcie_pmu, HISI_PCIE_INT_STAT, idx, 1); |
| 610 | + event = pcie_pmu->hw_events[idx]; |
| 611 | + if (!event) |
| 612 | + continue; |
| 613 | + |
| 614 | + hisi_pcie_pmu_event_update(event); |
| 615 | + hisi_pcie_pmu_set_period(event); |
| 616 | + ret = IRQ_HANDLED; |
| 617 | + } |
| 618 | + |
| 619 | + return ret; |
| 620 | +} |
| 621 | + |
| 622 | +static int hisi_pcie_pmu_irq_register(struct pci_dev *pdev, struct hisi_pcie_pmu *pcie_pmu) |
| 623 | +{ |
| 624 | + int irq, ret; |
| 625 | + |
| 626 | + ret = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_MSI); |
| 627 | + if (ret < 0) { |
| 628 | + pci_err(pdev, "Failed to enable MSI vectors: %d\n", ret); |
| 629 | + return ret; |
| 630 | + } |
| 631 | + |
| 632 | + irq = pci_irq_vector(pdev, 0); |
| 633 | + ret = request_irq(irq, hisi_pcie_pmu_irq, IRQF_NOBALANCING | IRQF_NO_THREAD, DRV_NAME, |
| 634 | + pcie_pmu); |
| 635 | + if (ret) { |
| 636 | + pci_err(pdev, "Failed to register IRQ: %d\n", ret); |
| 637 | + pci_free_irq_vectors(pdev); |
| 638 | + return ret; |
| 639 | + } |
| 640 | + |
| 641 | + pcie_pmu->irq = irq; |
| 642 | + |
| 643 | + return 0; |
| 644 | +} |
| 645 | + |
| 646 | +static void hisi_pcie_pmu_irq_unregister(struct pci_dev *pdev, struct hisi_pcie_pmu *pcie_pmu) |
| 647 | +{ |
| 648 | + free_irq(pcie_pmu->irq, pcie_pmu); |
| 649 | + pci_free_irq_vectors(pdev); |
| 650 | +} |
| 651 | + |
| 652 | +static int hisi_pcie_pmu_online_cpu(unsigned int cpu, struct hlist_node *node) |
| 653 | +{ |
| 654 | + struct hisi_pcie_pmu *pcie_pmu = hlist_entry_safe(node, struct hisi_pcie_pmu, node); |
| 655 | + |
| 656 | + if (pcie_pmu->on_cpu == -1) { |
| 657 | + pcie_pmu->on_cpu = cpu; |
| 658 | + WARN_ON(irq_set_affinity(pcie_pmu->irq, cpumask_of(cpu))); |
| 659 | + } |
| 660 | + |
| 661 | + return 0; |
| 662 | +} |
| 663 | + |
| 664 | +static int hisi_pcie_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node) |
| 665 | +{ |
| 666 | + struct hisi_pcie_pmu *pcie_pmu = hlist_entry_safe(node, struct hisi_pcie_pmu, node); |
| 667 | + unsigned int target; |
| 668 | + |
| 669 | + /* Nothing to do if this CPU doesn't own the PMU */ |
| 670 | + if (pcie_pmu->on_cpu != cpu) |
| 671 | + return 0; |
| 672 | + |
| 673 | + pcie_pmu->on_cpu = -1; |
| 674 | + /* Choose a new CPU from all online cpus. */ |
| 675 | + target = cpumask_first(cpu_online_mask); |
| 676 | + if (target >= nr_cpu_ids) { |
| 677 | + pci_err(pcie_pmu->pdev, "There is no CPU to set\n"); |
| 678 | + return 0; |
| 679 | + } |
| 680 | + |
| 681 | + perf_pmu_migrate_context(&pcie_pmu->pmu, cpu, target); |
| 682 | + /* Use this CPU for event counting */ |
| 683 | + pcie_pmu->on_cpu = target; |
| 684 | + WARN_ON(irq_set_affinity(pcie_pmu->irq, cpumask_of(target))); |
| 685 | + |
| 686 | + return 0; |
| 687 | +} |
| 688 | + |
| 689 | +static struct attribute *hisi_pcie_pmu_events_attr[] = { |
| 690 | + HISI_PCIE_PMU_EVENT_ATTR(rx_mwr_latency, 0x0010), |
| 691 | + HISI_PCIE_PMU_EVENT_ATTR(rx_mwr_cnt, 0x10010), |
| 692 | + HISI_PCIE_PMU_EVENT_ATTR(rx_mrd_latency, 0x0210), |
| 693 | + HISI_PCIE_PMU_EVENT_ATTR(rx_mrd_cnt, 0x10210), |
| 694 | + HISI_PCIE_PMU_EVENT_ATTR(tx_mrd_latency, 0x0011), |
| 695 | + HISI_PCIE_PMU_EVENT_ATTR(tx_mrd_cnt, 0x10011), |
| 696 | + HISI_PCIE_PMU_EVENT_ATTR(rx_mrd_flux, 0x1005), |
| 697 | + HISI_PCIE_PMU_EVENT_ATTR(rx_mrd_time, 0x11005), |
| 698 | + HISI_PCIE_PMU_EVENT_ATTR(tx_mrd_flux, 0x2004), |
| 699 | + HISI_PCIE_PMU_EVENT_ATTR(tx_mrd_time, 0x12004), |
| 700 | + NULL |
| 701 | +}; |
| 702 | + |
| 703 | +static struct attribute_group hisi_pcie_pmu_events_group = { |
| 704 | + .name = "events", |
| 705 | + .attrs = hisi_pcie_pmu_events_attr, |
| 706 | +}; |
| 707 | + |
| 708 | +static struct attribute *hisi_pcie_pmu_format_attr[] = { |
| 709 | + HISI_PCIE_PMU_FORMAT_ATTR(event, "config:0-16"), |
| 710 | + HISI_PCIE_PMU_FORMAT_ATTR(thr_len, "config1:0-3"), |
| 711 | + HISI_PCIE_PMU_FORMAT_ATTR(thr_mode, "config1:4"), |
| 712 | + HISI_PCIE_PMU_FORMAT_ATTR(trig_len, "config1:5-8"), |
| 713 | + HISI_PCIE_PMU_FORMAT_ATTR(trig_mode, "config1:9"), |
| 714 | + HISI_PCIE_PMU_FORMAT_ATTR(port, "config2:0-15"), |
| 715 | + HISI_PCIE_PMU_FORMAT_ATTR(bdf, "config2:16-31"), |
| 716 | + NULL |
| 717 | +}; |
| 718 | + |
| 719 | +static const struct attribute_group hisi_pcie_pmu_format_group = { |
| 720 | + .name = "format", |
| 721 | + .attrs = hisi_pcie_pmu_format_attr, |
| 722 | +}; |
| 723 | + |
| 724 | +static struct attribute *hisi_pcie_pmu_bus_attrs[] = { |
| 725 | + &dev_attr_bus.attr, |
| 726 | + NULL |
| 727 | +}; |
| 728 | + |
| 729 | +static const struct attribute_group hisi_pcie_pmu_bus_attr_group = { |
| 730 | + .attrs = hisi_pcie_pmu_bus_attrs, |
| 731 | +}; |
| 732 | + |
| 733 | +static struct attribute *hisi_pcie_pmu_cpumask_attrs[] = { |
| 734 | + &dev_attr_cpumask.attr, |
| 735 | + NULL |
| 736 | +}; |
| 737 | + |
| 738 | +static const struct attribute_group hisi_pcie_pmu_cpumask_attr_group = { |
| 739 | + .attrs = hisi_pcie_pmu_cpumask_attrs, |
| 740 | +}; |
| 741 | + |
| 742 | +static struct attribute *hisi_pcie_pmu_identifier_attrs[] = { |
| 743 | + &dev_attr_identifier.attr, |
| 744 | + NULL |
| 745 | +}; |
| 746 | + |
| 747 | +static const struct attribute_group hisi_pcie_pmu_identifier_attr_group = { |
| 748 | + .attrs = hisi_pcie_pmu_identifier_attrs, |
| 749 | +}; |
| 750 | + |
| 751 | +static const struct attribute_group *hisi_pcie_pmu_attr_groups[] = { |
| 752 | + &hisi_pcie_pmu_events_group, |
| 753 | + &hisi_pcie_pmu_format_group, |
| 754 | + &hisi_pcie_pmu_bus_attr_group, |
| 755 | + &hisi_pcie_pmu_cpumask_attr_group, |
| 756 | + &hisi_pcie_pmu_identifier_attr_group, |
| 757 | + NULL |
| 758 | +}; |
| 759 | + |
| 760 | +static int hisi_pcie_alloc_pmu(struct pci_dev *pdev, struct hisi_pcie_pmu *pcie_pmu) |
| 761 | +{ |
| 762 | + struct hisi_pcie_reg_pair regs; |
| 763 | + u16 sicl_id, core_id; |
| 764 | + char *name; |
| 765 | + |
| 766 | + regs = hisi_pcie_parse_reg_value(pcie_pmu, HISI_PCIE_REG_BDF); |
| 767 | + pcie_pmu->bdf_min = regs.lo; |
| 768 | + pcie_pmu->bdf_max = regs.hi; |
| 769 | + |
| 770 | + regs = hisi_pcie_parse_reg_value(pcie_pmu, HISI_PCIE_REG_INFO); |
| 771 | + sicl_id = regs.hi; |
| 772 | + core_id = regs.lo; |
| 773 | + |
| 774 | + name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "hisi_pcie%u_core%u", sicl_id, core_id); |
| 775 | + if (!name) |
| 776 | + return -ENOMEM; |
| 777 | + |
| 778 | + pcie_pmu->pdev = pdev; |
| 779 | + pcie_pmu->on_cpu = -1; |
| 780 | + pcie_pmu->identifier = readl(pcie_pmu->base + HISI_PCIE_REG_VERSION); |
| 781 | + pcie_pmu->pmu = (struct pmu) { |
| 782 | + .name = name, |
| 783 | + .module = THIS_MODULE, |
| 784 | + .event_init = hisi_pcie_pmu_event_init, |
| 785 | + .pmu_enable = hisi_pcie_pmu_enable, |
| 786 | + .pmu_disable = hisi_pcie_pmu_disable, |
| 787 | + .add = hisi_pcie_pmu_add, |
| 788 | + .del = hisi_pcie_pmu_del, |
| 789 | + .start = hisi_pcie_pmu_start, |
| 790 | + .stop = hisi_pcie_pmu_stop, |
| 791 | + .read = hisi_pcie_pmu_read, |
| 792 | + .task_ctx_nr = perf_invalid_context, |
| 793 | + .attr_groups = hisi_pcie_pmu_attr_groups, |
| 794 | + .capabilities = PERF_PMU_CAP_NO_EXCLUDE, |
| 795 | + }; |
| 796 | + |
| 797 | + return 0; |
| 798 | +} |
| 799 | + |
| 800 | +static int hisi_pcie_init_pmu(struct pci_dev *pdev, struct hisi_pcie_pmu *pcie_pmu) |
| 801 | +{ |
| 802 | + int ret; |
| 803 | + |
| 804 | + pcie_pmu->base = pci_ioremap_bar(pdev, 2); |
| 805 | + if (!pcie_pmu->base) { |
| 806 | + pci_err(pdev, "Ioremap failed for pcie_pmu resource\n"); |
| 807 | + return -ENOMEM; |
| 808 | + } |
| 809 | + |
| 810 | + ret = hisi_pcie_alloc_pmu(pdev, pcie_pmu); |
| 811 | + if (ret) |
| 812 | + goto err_iounmap; |
| 813 | + |
| 814 | + ret = hisi_pcie_pmu_irq_register(pdev, pcie_pmu); |
| 815 | + if (ret) |
| 816 | + goto err_iounmap; |
| 817 | + |
| 818 | + ret = cpuhp_state_add_instance(CPUHP_AP_PERF_ARM_HISI_PCIE_PMU_ONLINE, &pcie_pmu->node); |
| 819 | + if (ret) { |
| 820 | + pci_err(pdev, "Failed to register hotplug: %d\n", ret); |
| 821 | + goto err_irq_unregister; |
| 822 | + } |
| 823 | + |
| 824 | + ret = perf_pmu_register(&pcie_pmu->pmu, pcie_pmu->pmu.name, -1); |
| 825 | + if (ret) { |
| 826 | + pci_err(pdev, "Failed to register PCIe PMU: %d\n", ret); |
| 827 | + goto err_hotplug_unregister; |
| 828 | + } |
| 829 | + |
| 830 | + return ret; |
| 831 | + |
| 832 | +err_hotplug_unregister: |
| 833 | + cpuhp_state_remove_instance_nocalls( |
| 834 | + CPUHP_AP_PERF_ARM_HISI_PCIE_PMU_ONLINE, &pcie_pmu->node); |
| 835 | + |
| 836 | +err_irq_unregister: |
| 837 | + hisi_pcie_pmu_irq_unregister(pdev, pcie_pmu); |
| 838 | + |
| 839 | +err_iounmap: |
| 840 | + iounmap(pcie_pmu->base); |
| 841 | + |
| 842 | + return ret; |
| 843 | +} |
| 844 | + |
| 845 | +static void hisi_pcie_uninit_pmu(struct pci_dev *pdev) |
| 846 | +{ |
| 847 | + struct hisi_pcie_pmu *pcie_pmu = pci_get_drvdata(pdev); |
| 848 | + |
| 849 | + perf_pmu_unregister(&pcie_pmu->pmu); |
| 850 | + cpuhp_state_remove_instance_nocalls( |
| 851 | + CPUHP_AP_PERF_ARM_HISI_PCIE_PMU_ONLINE, &pcie_pmu->node); |
| 852 | + hisi_pcie_pmu_irq_unregister(pdev, pcie_pmu); |
| 853 | + iounmap(pcie_pmu->base); |
| 854 | +} |
| 855 | + |
| 856 | +static int hisi_pcie_init_dev(struct pci_dev *pdev) |
| 857 | +{ |
| 858 | + int ret; |
| 859 | + |
| 860 | + ret = pcim_enable_device(pdev); |
| 861 | + if (ret) { |
| 862 | + pci_err(pdev, "Failed to enable PCI device: %d\n", ret); |
| 863 | + return ret; |
| 864 | + } |
| 865 | + |
| 866 | + ret = pcim_iomap_regions(pdev, BIT(2), DRV_NAME); |
| 867 | + if (ret < 0) { |
| 868 | + pci_err(pdev, "Failed to request PCI mem regions: %d\n", ret); |
| 869 | + return ret; |
| 870 | + } |
| 871 | + |
| 872 | + pci_set_master(pdev); |
| 873 | + |
| 874 | + return 0; |
| 875 | +} |
| 876 | + |
| 877 | +static int hisi_pcie_pmu_probe(struct pci_dev *pdev, const struct pci_device_id *id) |
| 878 | +{ |
| 879 | + struct hisi_pcie_pmu *pcie_pmu; |
| 880 | + int ret; |
| 881 | + |
| 882 | + pcie_pmu = devm_kzalloc(&pdev->dev, sizeof(*pcie_pmu), GFP_KERNEL); |
| 883 | + if (!pcie_pmu) |
| 884 | + return -ENOMEM; |
| 885 | + |
| 886 | + ret = hisi_pcie_init_dev(pdev); |
| 887 | + if (ret) |
| 888 | + return ret; |
| 889 | + |
| 890 | + ret = hisi_pcie_init_pmu(pdev, pcie_pmu); |
| 891 | + if (ret) |
| 892 | + return ret; |
| 893 | + |
| 894 | + pci_set_drvdata(pdev, pcie_pmu); |
| 895 | + |
| 896 | + return ret; |
| 897 | +} |
| 898 | + |
| 899 | +static void hisi_pcie_pmu_remove(struct pci_dev *pdev) |
| 900 | +{ |
| 901 | + hisi_pcie_uninit_pmu(pdev); |
| 902 | + pci_set_drvdata(pdev, NULL); |
| 903 | +} |
| 904 | + |
| 905 | +static const struct pci_device_id hisi_pcie_pmu_ids[] = { |
| 906 | + { PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, 0xa12d) }, |
| 907 | + { 0, } |
| 908 | +}; |
| 909 | +MODULE_DEVICE_TABLE(pci, hisi_pcie_pmu_ids); |
| 910 | + |
| 911 | +static struct pci_driver hisi_pcie_pmu_driver = { |
| 912 | + .name = DRV_NAME, |
| 913 | + .id_table = hisi_pcie_pmu_ids, |
| 914 | + .probe = hisi_pcie_pmu_probe, |
| 915 | + .remove = hisi_pcie_pmu_remove, |
| 916 | +}; |
| 917 | + |
| 918 | +static int __init hisi_pcie_module_init(void) |
| 919 | +{ |
| 920 | + int ret; |
| 921 | + |
| 922 | + ret = cpuhp_setup_state_multi(CPUHP_AP_PERF_ARM_HISI_PCIE_PMU_ONLINE, |
| 923 | + "AP_PERF_ARM_HISI_PCIE_PMU_ONLINE", |
| 924 | + hisi_pcie_pmu_online_cpu, |
| 925 | + hisi_pcie_pmu_offline_cpu); |
| 926 | + if (ret) { |
| 927 | + pr_err("Failed to setup PCIe PMU hotplug: %d\n", ret); |
| 928 | + return ret; |
| 929 | + } |
| 930 | + |
| 931 | + ret = pci_register_driver(&hisi_pcie_pmu_driver); |
| 932 | + if (ret) |
| 933 | + cpuhp_remove_multi_state(CPUHP_AP_PERF_ARM_HISI_PCIE_PMU_ONLINE); |
| 934 | + |
| 935 | + return ret; |
| 936 | +} |
| 937 | +module_init(hisi_pcie_module_init); |
| 938 | + |
| 939 | +static void __exit hisi_pcie_module_exit(void) |
| 940 | +{ |
| 941 | + pci_unregister_driver(&hisi_pcie_pmu_driver); |
| 942 | + cpuhp_remove_multi_state(CPUHP_AP_PERF_ARM_HISI_PCIE_PMU_ONLINE); |
| 943 | +} |
| 944 | +module_exit(hisi_pcie_module_exit); |
| 945 | + |
| 946 | +MODULE_DESCRIPTION("HiSilicon PCIe PMU driver"); |
| 947 | +MODULE_LICENSE("GPL v2"); |
| 948 | +MODULE_AUTHOR("Qi Liu <liuqi115@huawei.com>"); |