// SPDX-License-Identifier: GPL-2.0-only/* * Resource Director Technology(RDT) * - Monitoring code * * Copyright (C) 2017 Intel Corporation * * Author: *    Vikas Shivappa <vikas.shivappa@intel.com> * * This replaces the cqm.c based on perf but we reuse a lot of * code and datastructures originally from Peter Zijlstra and Matt Fleming. * * More information about RDT be found in the Intel (R) x86 Architecture * Software Developer Manual June 2016, volume 3, section 17.17. */ #include <linux/module.h>#include <linux/sizes.h>#include <linux/slab.h> #include <asm/cpu_device_id.h>#include <asm/resctrl.h> #include "internal.h" struct rmid_entry {	u32				rmid;	int				busy;	struct list_head		list;}; /** * @rmid_free_lru    A least recently used list of free RMIDs *     These RMIDs are guaranteed to have an occupancy less than the *     threshold occupancy */static LIST_HEAD(rmid_free_lru); /** * @rmid_limbo_count     count of currently unused but (potentially) *     dirty RMIDs. *     This counts RMIDs that no one is currently using but that *     may have a occupancy value > resctrl_rmid_realloc_threshold. User can *     change the threshold occupancy value. */static unsigned int rmid_limbo_count; /** * @rmid_entry - The entry in the limbo and free lists. */static struct rmid_entry	*rmid_ptrs; /* * Global boolean for rdt_monitor which is true if any * resource monitoring is enabled. */bool rdt_mon_capable; /* * Global to indicate which monitoring events are enabled. */unsigned int rdt_mon_features; /* * This is the threshold cache occupancy in bytes at which we will consider an * RMID available for re-allocation. */unsigned int resctrl_rmid_realloc_threshold; /* * This is the maximum value for the reallocation threshold, in bytes. */unsigned int resctrl_rmid_realloc_limit; #define CF(cf)	((unsigned long)(1048576 * (cf) + 0.5)) /* * The correction factor table is documented in Documentation/x86/resctrl.rst. * If rmid > rmid threshold, MBM total and local values should be multiplied * by the correction factor. * * The original table is modified for better code: * * 1. The threshold 0 is changed to rmid count - 1 so don't do correction *    for the case. * 2. MBM total and local correction table indexed by core counter which is *    equal to (x86_cache_max_rmid + 1) / 8 - 1 and is from 0 up to 27. * 3. The correction factor is normalized to 2^20 (1048576) so it's faster *    to calculate corrected value by shifting: *    corrected_value = (original_value * correction_factor) >> 20 */static const struct mbm_correction_factor_table {	u32 rmidthreshold;	u64 cf;} mbm_cf_table[] __initconst = {	{7,	CF(1.000000)},	{15,	CF(1.000000)},	{15,	CF(0.969650)},	{31,	CF(1.000000)},	{31,	CF(1.066667)},	{31,	CF(0.969650)},	{47,	CF(1.142857)},	{63,	CF(1.000000)},	{63,	CF(1.185115)},	{63,	CF(1.066553)},	{79,	CF(1.454545)},	{95,	CF(1.000000)},	{95,	CF(1.230769)},	{95,	CF(1.142857)},	{95,	CF(1.066667)},	{127,	CF(1.000000)},	{127,	CF(1.254863)},	{127,	CF(1.185255)},	{151,	CF(1.000000)},	{127,	CF(1.066667)},	{167,	CF(1.000000)},	{159,	CF(1.454334)},	{183,	CF(1.000000)},	{127,	CF(0.969744)},	{191,	CF(1.280246)},	{191,	CF(1.230921)},	{215,	CF(1.000000)},	{191,	CF(1.143118)},}; static u32 mbm_cf_rmidthreshold __read_mostly = UINT_MAX;static u64 mbm_cf __read_mostly; static inline u64 get_corrected_mbm_count(u32 rmid, unsigned long val){	/* Correct MBM value. */	if (rmid > mbm_cf_rmidthreshold)		val = (val * mbm_cf) >> 20; 	return val;} static inline struct rmid_entry *__rmid_entry(u32 rmid){	struct rmid_entry *entry; 	entry = &rmid_ptrs[rmid];	WARN_ON(entry->rmid != rmid); 	return entry;} static int __rmid_read(u32 rmid, enum resctrl_event_id eventid, u64 *val){	u64 msr_val; 	/*	 * As per the SDM, when IA32_QM_EVTSEL.EvtID (bits 7:0) is configured	 * with a valid event code for supported resource type and the bits	 * IA32_QM_EVTSEL.RMID (bits 41:32) are configured with valid RMID,	 * IA32_QM_CTR.data (bits 61:0) reports the monitored data.	 * IA32_QM_CTR.Error (bit 63) and IA32_QM_CTR.Unavailable (bit 62)	 * are error bits.	 */	wrmsr(MSR_IA32_QM_EVTSEL, eventid, rmid);	rdmsrl(MSR_IA32_QM_CTR, msr_val); 	if (msr_val & RMID_VAL_ERROR)		return -EIO;	if (msr_val & RMID_VAL_UNAVAIL)		return -EINVAL; 	*val = msr_val;	return 0;} static struct arch_mbm_state *get_arch_mbm_state(struct rdt_hw_domain *hw_dom,						 u32 rmid,						 enum resctrl_event_id eventid){	switch (eventid) {	case QOS_L3_OCCUP_EVENT_ID:		return NULL;	case QOS_L3_MBM_TOTAL_EVENT_ID:		return &hw_dom->arch_mbm_total[rmid];	case QOS_L3_MBM_LOCAL_EVENT_ID:		return &hw_dom->arch_mbm_local[rmid];	} 	/* Never expect to get here */	WARN_ON_ONCE(1); 	return NULL;} void resctrl_arch_reset_rmid(struct rdt_resource *r, struct rdt_domain *d,			     u32 rmid, enum resctrl_event_id eventid){	struct rdt_hw_domain *hw_dom = resctrl_to_arch_dom(d);	struct arch_mbm_state *am; 	am = get_arch_mbm_state(hw_dom, rmid, eventid);	if (am) {		memset(am, 0, sizeof(*am)); 		/* Record any initial, non-zero count value. */		__rmid_read(rmid, eventid, &am->prev_msr);	}} static u64 mbm_overflow_count(u64 prev_msr, u64 cur_msr, unsigned int width){	u64 shift = 64 - width, chunks; 	chunks = (cur_msr << shift) - (prev_msr << shift);	return chunks >> shift;} int resctrl_arch_rmid_read(struct rdt_resource *r, struct rdt_domain *d,			   u32 rmid, enum resctrl_event_id eventid, u64 *val){	struct rdt_hw_resource *hw_res = resctrl_to_arch_res(r);	struct rdt_hw_domain *hw_dom = resctrl_to_arch_dom(d);	struct arch_mbm_state *am;	u64 msr_val, chunks;	int ret; 	if (!cpumask_test_cpu(smp_processor_id(), &d->cpu_mask))		return -EINVAL; 	ret = __rmid_read(rmid, eventid, &msr_val);	if (ret)		return ret; 	am = get_arch_mbm_state(hw_dom, rmid, eventid);	if (am) {		am->chunks += mbm_overflow_count(am->prev_msr, msr_val,						 hw_res->mbm_width);		chunks = get_corrected_mbm_count(rmid, am->chunks);		am->prev_msr = msr_val;	} else {		chunks = msr_val;	} 	*val = chunks * hw_res->mon_scale; 	return 0;} /* * Check the RMIDs that are marked as busy for this domain. If the * reported LLC occupancy is below the threshold clear the busy bit and * decrement the count. If the busy count gets to zero on an RMID, we * free the RMID */void __check_limbo(struct rdt_domain *d, bool force_free){	struct rdt_resource *r = &rdt_resources_all[RDT_RESOURCE_L3].r_resctrl;	struct rmid_entry *entry;	u32 crmid = 1, nrmid;	bool rmid_dirty;	u64 val = 0; 	/*	 * Skip RMID 0 and start from RMID 1 and check all the RMIDs that	 * are marked as busy for occupancy < threshold. If the occupancy	 * is less than the threshold decrement the busy counter of the	 * RMID and move it to the free list when the counter reaches 0.	 */	for (;;) {		nrmid = find_next_bit(d->rmid_busy_llc, r->num_rmid, crmid);		if (nrmid >= r->num_rmid)			break; 		entry = __rmid_entry(nrmid); 		if (resctrl_arch_rmid_read(r, d, entry->rmid,					   QOS_L3_OCCUP_EVENT_ID, &val)) {			rmid_dirty = true;		} else {			rmid_dirty = (val >= resctrl_rmid_realloc_threshold);		} 		if (force_free || !rmid_dirty) {			clear_bit(entry->rmid, d->rmid_busy_llc);			if (!--entry->busy) {				rmid_limbo_count--;				list_add_tail(&entry->list, &rmid_free_lru);			}		}		crmid = nrmid + 1;	}} bool has_busy_rmid(struct rdt_resource *r, struct rdt_domain *d){	return find_first_bit(d->rmid_busy_llc, r->num_rmid) != r->num_rmid;} /* * As of now the RMIDs allocation is global. * However we keep track of which packages the RMIDs * are used to optimize the limbo list management. */int alloc_rmid(void){	struct rmid_entry *entry; 	lockdep_assert_held(&rdtgroup_mutex); 	if (list_empty(&rmid_free_lru))		return rmid_limbo_count ? -EBUSY : -ENOSPC; 	entry = list_first_entry(&rmid_free_lru,				 struct rmid_entry, list);	list_del(&entry->list); 	return entry->rmid;} static void add_rmid_to_limbo(struct rmid_entry *entry){	struct rdt_resource *r = &rdt_resources_all[RDT_RESOURCE_L3].r_resctrl;	struct rdt_domain *d;	int cpu, err;	u64 val = 0; 	entry->busy = 0;	cpu = get_cpu();	list_for_each_entry(d, &r->domains, list) {		if (cpumask_test_cpu(cpu, &d->cpu_mask)) {			err = resctrl_arch_rmid_read(r, d, entry->rmid,						     QOS_L3_OCCUP_EVENT_ID,						     &val);			if (err || val <= resctrl_rmid_realloc_threshold)				continue;		} 		/*		 * For the first limbo RMID in the domain,		 * setup up the limbo worker.		 */		if (!has_busy_rmid(r, d))			cqm_setup_limbo_handler(d, CQM_LIMBOCHECK_INTERVAL);		set_bit(entry->rmid, d->rmid_busy_llc);		entry->busy++;	}	put_cpu(); 	if (entry->busy)		rmid_limbo_count++;	else		list_add_tail(&entry->list, &rmid_free_lru);} void free_rmid(u32 rmid){	struct rmid_entry *entry; 	if (!rmid)		return; 	lockdep_assert_held(&rdtgroup_mutex); 	entry = __rmid_entry(rmid); 	if (is_llc_occupancy_enabled())		add_rmid_to_limbo(entry);	else		list_add_tail(&entry->list, &rmid_free_lru);} static int __mon_event_count(u32 rmid, struct rmid_read *rr){	struct mbm_state *m;	u64 tval = 0; 	if (rr->first)		resctrl_arch_reset_rmid(rr->r, rr->d, rmid, rr->evtid); 	rr->err = resctrl_arch_rmid_read(rr->r, rr->d, rmid, rr->evtid, &tval);	if (rr->err)		return rr->err; 	switch (rr->evtid) {	case QOS_L3_OCCUP_EVENT_ID:		rr->val += tval;		return 0;	case QOS_L3_MBM_TOTAL_EVENT_ID:		m = &rr->d->mbm_total[rmid];		break;	case QOS_L3_MBM_LOCAL_EVENT_ID:		m = &rr->d->mbm_local[rmid];		break;	default:		/*		 * Code would never reach here because an invalid		 * event id would fail in resctrl_arch_rmid_read().		 */		return -EINVAL;	} 	if (rr->first) {		memset(m, 0, sizeof(struct mbm_state));		return 0;	} 	rr->val += tval; 	return 0;} /* * mbm_bw_count() - Update bw count from values previously read by *		    __mon_event_count(). * @rmid:	The rmid used to identify the cached mbm_state. * @rr:		The struct rmid_read populated by __mon_event_count(). * * Supporting function to calculate the memory bandwidth * and delta bandwidth in MBps. The chunks value previously read by * __mon_event_count() is compared with the chunks value from the previous * invocation. This must be called once per second to maintain values in MBps. */static void mbm_bw_count(u32 rmid, struct rmid_read *rr){	struct mbm_state *m = &rr->d->mbm_local[rmid];	u64 cur_bw, bytes, cur_bytes; 	cur_bytes = rr->val;	bytes = cur_bytes - m->prev_bw_bytes;	m->prev_bw_bytes = cur_bytes; 	cur_bw = bytes / SZ_1M; 	if (m->delta_comp)		m->delta_bw = abs(cur_bw - m->prev_bw);	m->delta_comp = false;	m->prev_bw = cur_bw;} /* * This is called via IPI to read the CQM/MBM counters * on a domain. */void mon_event_count(void *info){	struct rdtgroup *rdtgrp, *entry;	struct rmid_read *rr = info;	struct list_head *head;	int ret; 	rdtgrp = rr->rgrp; 	ret = __mon_event_count(rdtgrp->mon.rmid, rr); 	/*	 * For Ctrl groups read data from child monitor groups and	 * add them together. Count events which are read successfully.	 * Discard the rmid_read's reporting errors.	 */	head = &rdtgrp->mon.crdtgrp_list; 	if (rdtgrp->type == RDTCTRL_GROUP) {		list_for_each_entry(entry, head, mon.crdtgrp_list) {			if (__mon_event_count(entry->mon.rmid, rr) == 0)				ret = 0;		}	} 	/*	 * __mon_event_count() calls for newly created monitor groups may	 * report -EINVAL/Unavailable if the monitor hasn't seen any traffic.	 * Discard error if any of the monitor event reads succeeded.	 */	if (ret == 0)		rr->err = 0;} /* * Feedback loop for MBA software controller (mba_sc) * * mba_sc is a feedback loop where we periodically read MBM counters and * adjust the bandwidth percentage values via the IA32_MBA_THRTL_MSRs so * that: * *   current bandwidth(cur_bw) < user specified bandwidth(user_bw) * * This uses the MBM counters to measure the bandwidth and MBA throttle * MSRs to control the bandwidth for a particular rdtgrp. It builds on the * fact that resctrl rdtgroups have both monitoring and control. * * The frequency of the checks is 1s and we just tag along the MBM overflow * timer. Having 1s interval makes the calculation of bandwidth simpler. * * Although MBA's goal is to restrict the bandwidth to a maximum, there may * be a need to increase the bandwidth to avoid unnecessarily restricting * the L2 <-> L3 traffic. * * Since MBA controls the L2 external bandwidth where as MBM measures the * L3 external bandwidth the following sequence could lead to such a * situation. * * Consider an rdtgroup which had high L3 <-> memory traffic in initial * phases -> mba_sc kicks in and reduced bandwidth percentage values -> but

Lines 1–500 of 822. Load more