// SPDX-License-Identifier: GPL-2.0-only/* * Xtensa Performance Monitor Module driver * See Tensilica Debug User's Guide for PMU registers documentation. * * Copyright (C) 2015 Cadence Design Systems Inc. */ #include <linux/interrupt.h>#include <linux/irqdomain.h>#include <linux/module.h>#include <linux/of.h>#include <linux/perf_event.h>#include <linux/platform_device.h> #include <asm/core.h>#include <asm/processor.h>#include <asm/stacktrace.h> #define XTENSA_HWVERSION_RG_2015_0	260000 #if XCHAL_HW_MIN_VERSION >= XTENSA_HWVERSION_RG_2015_0#define XTENSA_PMU_ERI_BASE		0x00101000#else#define XTENSA_PMU_ERI_BASE		0x00001000#endif /* Global control/status for all perf counters */#define XTENSA_PMU_PMG			XTENSA_PMU_ERI_BASE/* Perf counter values */#define XTENSA_PMU_PM(i)		(XTENSA_PMU_ERI_BASE + 0x80 + (i) * 4)/* Perf counter control registers */#define XTENSA_PMU_PMCTRL(i)		(XTENSA_PMU_ERI_BASE + 0x100 + (i) * 4)/* Perf counter status registers */#define XTENSA_PMU_PMSTAT(i)		(XTENSA_PMU_ERI_BASE + 0x180 + (i) * 4) #define XTENSA_PMU_PMG_PMEN		0x1 #define XTENSA_PMU_COUNTER_MASK		0xffffffffULL#define XTENSA_PMU_COUNTER_MAX		0x7fffffff #define XTENSA_PMU_PMCTRL_INTEN		0x00000001#define XTENSA_PMU_PMCTRL_KRNLCNT	0x00000008#define XTENSA_PMU_PMCTRL_TRACELEVEL	0x000000f0#define XTENSA_PMU_PMCTRL_SELECT_SHIFT	8#define XTENSA_PMU_PMCTRL_SELECT	0x00001f00#define XTENSA_PMU_PMCTRL_MASK_SHIFT	16#define XTENSA_PMU_PMCTRL_MASK		0xffff0000 #define XTENSA_PMU_MASK(select, mask) \	(((select) << XTENSA_PMU_PMCTRL_SELECT_SHIFT) | \	 ((mask) << XTENSA_PMU_PMCTRL_MASK_SHIFT) | \	 XTENSA_PMU_PMCTRL_TRACELEVEL | \	 XTENSA_PMU_PMCTRL_INTEN) #define XTENSA_PMU_PMSTAT_OVFL		0x00000001#define XTENSA_PMU_PMSTAT_INTASRT	0x00000010 struct xtensa_pmu_events {	/* Array of events currently on this core */	struct perf_event *event[XCHAL_NUM_PERF_COUNTERS];	/* Bitmap of used hardware counters */	unsigned long used_mask[BITS_TO_LONGS(XCHAL_NUM_PERF_COUNTERS)];};static DEFINE_PER_CPU(struct xtensa_pmu_events, xtensa_pmu_events); static const u32 xtensa_hw_ctl[] = {	[PERF_COUNT_HW_CPU_CYCLES]		= XTENSA_PMU_MASK(0, 0x1),	[PERF_COUNT_HW_INSTRUCTIONS]		= XTENSA_PMU_MASK(2, 0xffff),	[PERF_COUNT_HW_CACHE_REFERENCES]	= XTENSA_PMU_MASK(10, 0x1),	[PERF_COUNT_HW_CACHE_MISSES]		= XTENSA_PMU_MASK(12, 0x1),	/* Taken and non-taken branches + taken loop ends */	[PERF_COUNT_HW_BRANCH_INSTRUCTIONS]	= XTENSA_PMU_MASK(2, 0x490),	/* Instruction-related + other global stall cycles */	[PERF_COUNT_HW_STALLED_CYCLES_FRONTEND]	= XTENSA_PMU_MASK(4, 0x1ff),	/* Data-related global stall cycles */	[PERF_COUNT_HW_STALLED_CYCLES_BACKEND]	= XTENSA_PMU_MASK(3, 0x1ff),}; #define C(_x) PERF_COUNT_HW_CACHE_##_x static const u32 xtensa_cache_ctl[][C(OP_MAX)][C(RESULT_MAX)] = {	[C(L1D)] = {		[C(OP_READ)] = {			[C(RESULT_ACCESS)]	= XTENSA_PMU_MASK(10, 0x1),			[C(RESULT_MISS)]	= XTENSA_PMU_MASK(10, 0x2),		},		[C(OP_WRITE)] = {			[C(RESULT_ACCESS)]	= XTENSA_PMU_MASK(11, 0x1),			[C(RESULT_MISS)]	= XTENSA_PMU_MASK(11, 0x2),		},	},	[C(L1I)] = {		[C(OP_READ)] = {			[C(RESULT_ACCESS)]	= XTENSA_PMU_MASK(8, 0x1),			[C(RESULT_MISS)]	= XTENSA_PMU_MASK(8, 0x2),		},	},	[C(DTLB)] = {		[C(OP_READ)] = {			[C(RESULT_ACCESS)]	= XTENSA_PMU_MASK(9, 0x1),			[C(RESULT_MISS)]	= XTENSA_PMU_MASK(9, 0x8),		},	},	[C(ITLB)] = {		[C(OP_READ)] = {			[C(RESULT_ACCESS)]	= XTENSA_PMU_MASK(7, 0x1),			[C(RESULT_MISS)]	= XTENSA_PMU_MASK(7, 0x8),		},	},}; static int xtensa_pmu_cache_event(u64 config){	unsigned int cache_type, cache_op, cache_result;	int ret; 	cache_type = (config >>  0) & 0xff;	cache_op = (config >>  8) & 0xff;	cache_result = (config >> 16) & 0xff; 	if (cache_type >= ARRAY_SIZE(xtensa_cache_ctl) ||	    cache_op >= C(OP_MAX) ||	    cache_result >= C(RESULT_MAX))		return -EINVAL; 	ret = xtensa_cache_ctl[cache_type][cache_op][cache_result]; 	if (ret == 0)		return -EINVAL; 	return ret;} static inline uint32_t xtensa_pmu_read_counter(int idx){	return get_er(XTENSA_PMU_PM(idx));} static inline void xtensa_pmu_write_counter(int idx, uint32_t v){	set_er(v, XTENSA_PMU_PM(idx));} static void xtensa_perf_event_update(struct perf_event *event,				     struct hw_perf_event *hwc, int idx){	uint64_t prev_raw_count, new_raw_count;	int64_t delta; 	do {		prev_raw_count = local64_read(&hwc->prev_count);		new_raw_count = xtensa_pmu_read_counter(event->hw.idx);	} while (local64_cmpxchg(&hwc->prev_count, prev_raw_count,				 new_raw_count) != prev_raw_count); 	delta = (new_raw_count - prev_raw_count) & XTENSA_PMU_COUNTER_MASK; 	local64_add(delta, &event->count);	local64_sub(delta, &hwc->period_left);} static bool xtensa_perf_event_set_period(struct perf_event *event,					 struct hw_perf_event *hwc, int idx){	bool rc = false;	s64 left; 	if (!is_sampling_event(event)) {		left = XTENSA_PMU_COUNTER_MAX;	} else {		s64 period = hwc->sample_period; 		left = local64_read(&hwc->period_left);		if (left <= -period) {			left = period;			local64_set(&hwc->period_left, left);			hwc->last_period = period;			rc = true;		} else if (left <= 0) {			left += period;			local64_set(&hwc->period_left, left);			hwc->last_period = period;			rc = true;		}		if (left > XTENSA_PMU_COUNTER_MAX)			left = XTENSA_PMU_COUNTER_MAX;	} 	local64_set(&hwc->prev_count, -left);	xtensa_pmu_write_counter(idx, -left);	perf_event_update_userpage(event); 	return rc;} static void xtensa_pmu_enable(struct pmu *pmu){	set_er(get_er(XTENSA_PMU_PMG) | XTENSA_PMU_PMG_PMEN, XTENSA_PMU_PMG);} static void xtensa_pmu_disable(struct pmu *pmu){	set_er(get_er(XTENSA_PMU_PMG) & ~XTENSA_PMU_PMG_PMEN, XTENSA_PMU_PMG);} static int xtensa_pmu_event_init(struct perf_event *event){	int ret; 	switch (event->attr.type) {	case PERF_TYPE_HARDWARE:		if (event->attr.config >= ARRAY_SIZE(xtensa_hw_ctl) ||		    xtensa_hw_ctl[event->attr.config] == 0)			return -EINVAL;		event->hw.config = xtensa_hw_ctl[event->attr.config];		return 0; 	case PERF_TYPE_HW_CACHE:		ret = xtensa_pmu_cache_event(event->attr.config);		if (ret < 0)			return ret;		event->hw.config = ret;		return 0; 	case PERF_TYPE_RAW:		/* Not 'previous counter' select */		if ((event->attr.config & XTENSA_PMU_PMCTRL_SELECT) ==		    (1 << XTENSA_PMU_PMCTRL_SELECT_SHIFT))			return -EINVAL;		event->hw.config = (event->attr.config &				    (XTENSA_PMU_PMCTRL_KRNLCNT |				     XTENSA_PMU_PMCTRL_TRACELEVEL |				     XTENSA_PMU_PMCTRL_SELECT |				     XTENSA_PMU_PMCTRL_MASK)) |			XTENSA_PMU_PMCTRL_INTEN;		return 0; 	default:		return -ENOENT;	}} /* * Starts/Stops a counter present on the PMU. The PMI handler * should stop the counter when perf_event_overflow() returns * !0. ->start() will be used to continue. */static void xtensa_pmu_start(struct perf_event *event, int flags){	struct hw_perf_event *hwc = &event->hw;	int idx = hwc->idx; 	if (WARN_ON_ONCE(idx == -1))		return; 	if (flags & PERF_EF_RELOAD) {		WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE));		xtensa_perf_event_set_period(event, hwc, idx);	} 	hwc->state = 0; 	set_er(hwc->config, XTENSA_PMU_PMCTRL(idx));} static void xtensa_pmu_stop(struct perf_event *event, int flags){	struct hw_perf_event *hwc = &event->hw;	int idx = hwc->idx; 	if (!(hwc->state & PERF_HES_STOPPED)) {		set_er(0, XTENSA_PMU_PMCTRL(idx));		set_er(get_er(XTENSA_PMU_PMSTAT(idx)),		       XTENSA_PMU_PMSTAT(idx));		hwc->state |= PERF_HES_STOPPED;	} 	if ((flags & PERF_EF_UPDATE) &&	    !(event->hw.state & PERF_HES_UPTODATE)) {		xtensa_perf_event_update(event, &event->hw, idx);		event->hw.state |= PERF_HES_UPTODATE;	}} /* * Adds/Removes a counter to/from the PMU, can be done inside * a transaction, see the ->*_txn() methods. */static int xtensa_pmu_add(struct perf_event *event, int flags){	struct xtensa_pmu_events *ev = this_cpu_ptr(&xtensa_pmu_events);	struct hw_perf_event *hwc = &event->hw;	int idx = hwc->idx; 	if (__test_and_set_bit(idx, ev->used_mask)) {		idx = find_first_zero_bit(ev->used_mask,					  XCHAL_NUM_PERF_COUNTERS);		if (idx == XCHAL_NUM_PERF_COUNTERS)			return -EAGAIN; 		__set_bit(idx, ev->used_mask);		hwc->idx = idx;	}	ev->event[idx] = event; 	hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED; 	if (flags & PERF_EF_START)		xtensa_pmu_start(event, PERF_EF_RELOAD); 	perf_event_update_userpage(event);	return 0;} static void xtensa_pmu_del(struct perf_event *event, int flags){	struct xtensa_pmu_events *ev = this_cpu_ptr(&xtensa_pmu_events); 	xtensa_pmu_stop(event, PERF_EF_UPDATE);	__clear_bit(event->hw.idx, ev->used_mask);	perf_event_update_userpage(event);} static void xtensa_pmu_read(struct perf_event *event){	xtensa_perf_event_update(event, &event->hw, event->hw.idx);} static int callchain_trace(struct stackframe *frame, void *data){	struct perf_callchain_entry_ctx *entry = data; 	perf_callchain_store(entry, frame->pc);	return 0;} void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry,			   struct pt_regs *regs){	xtensa_backtrace_kernel(regs, entry->max_stack,				callchain_trace, NULL, entry);} void perf_callchain_user(struct perf_callchain_entry_ctx *entry,			 struct pt_regs *regs){	xtensa_backtrace_user(regs, entry->max_stack,			      callchain_trace, entry);} void perf_event_print_debug(void){	unsigned long flags;	unsigned i; 	local_irq_save(flags);	pr_info("CPU#%d: PMG: 0x%08lx\n", smp_processor_id(),		get_er(XTENSA_PMU_PMG));	for (i = 0; i < XCHAL_NUM_PERF_COUNTERS; ++i)		pr_info("PM%d: 0x%08lx, PMCTRL%d: 0x%08lx, PMSTAT%d: 0x%08lx\n",			i, get_er(XTENSA_PMU_PM(i)),			i, get_er(XTENSA_PMU_PMCTRL(i)),			i, get_er(XTENSA_PMU_PMSTAT(i)));	local_irq_restore(flags);} irqreturn_t xtensa_pmu_irq_handler(int irq, void *dev_id){	irqreturn_t rc = IRQ_NONE;	struct xtensa_pmu_events *ev = this_cpu_ptr(&xtensa_pmu_events);	unsigned i; 	for_each_set_bit(i, ev->used_mask, XCHAL_NUM_PERF_COUNTERS) {		uint32_t v = get_er(XTENSA_PMU_PMSTAT(i));		struct perf_event *event = ev->event[i];		struct hw_perf_event *hwc = &event->hw;		u64 last_period; 		if (!(v & XTENSA_PMU_PMSTAT_OVFL))			continue; 		set_er(v, XTENSA_PMU_PMSTAT(i));		xtensa_perf_event_update(event, hwc, i);		last_period = hwc->last_period;		if (xtensa_perf_event_set_period(event, hwc, i)) {			struct perf_sample_data data;			struct pt_regs *regs = get_irq_regs(); 			perf_sample_data_init(&data, 0, last_period);			perf_event_overflow(event, &data, regs);		} 		rc = IRQ_HANDLED;	}	return rc;} static struct pmu xtensa_pmu = {	.pmu_enable = xtensa_pmu_enable,	.pmu_disable = xtensa_pmu_disable,	.event_init = xtensa_pmu_event_init,	.add = xtensa_pmu_add,	.del = xtensa_pmu_del,	.start = xtensa_pmu_start,	.stop = xtensa_pmu_stop,	.read = xtensa_pmu_read,}; static int xtensa_pmu_setup(unsigned int cpu){	unsigned i; 	set_er(0, XTENSA_PMU_PMG);	for (i = 0; i < XCHAL_NUM_PERF_COUNTERS; ++i) {		set_er(0, XTENSA_PMU_PMCTRL(i));		set_er(get_er(XTENSA_PMU_PMSTAT(i)), XTENSA_PMU_PMSTAT(i));	}	return 0;} static int __init xtensa_pmu_init(void){	int ret;	int irq = irq_create_mapping(NULL, XCHAL_PROFILING_INTERRUPT); 	ret = cpuhp_setup_state(CPUHP_AP_PERF_XTENSA_STARTING,				"perf/xtensa:starting", xtensa_pmu_setup,				NULL);	if (ret) {		pr_err("xtensa_pmu: failed to register CPU-hotplug.\n");		return ret;	}#if XTENSA_FAKE_NMI	enable_irq(irq);#else	ret = request_irq(irq, xtensa_pmu_irq_handler, IRQF_PERCPU,			  "pmu", NULL);	if (ret < 0)		return ret;#endif 	ret = perf_pmu_register(&xtensa_pmu, "cpu", PERF_TYPE_RAW);	if (ret)		free_irq(irq, NULL); 	return ret;}early_initcall(xtensa_pmu_init);