// SPDX-License-Identifier: GPL-2.0-only/* * Resource Director Technology(RDT) * - Monitoring code * * Copyright (C) 2017 Intel Corporation * * Author: *    Vikas Shivappa <vikas.shivappa@intel.com> * * This replaces the cqm.c based on perf but we reuse a lot of * code and datastructures originally from Peter Zijlstra and Matt Fleming. * * More information about RDT be found in the Intel (R) x86 Architecture * Software Developer Manual June 2016, volume 3, section 17.17. */ #define pr_fmt(fmt)	"resctrl: " fmt #include <linux/cpu.h>#include <linux/module.h>#include <linux/sizes.h>#include <linux/slab.h> #include <asm/cpu_device_id.h>#include <asm/resctrl.h> #include "internal.h"#include "trace.h" /** * struct rmid_entry - dirty tracking for all RMID. * @closid:	The CLOSID for this entry. * @rmid:	The RMID for this entry. * @busy:	The number of domains with cached data using this RMID. * @list:	Member of the rmid_free_lru list when busy == 0. * * Depending on the architecture the correct monitor is accessed using * both @closid and @rmid, or @rmid only. * * Take the rdtgroup_mutex when accessing. */struct rmid_entry {	u32				closid;	u32				rmid;	int				busy;	struct list_head		list;}; /* * @rmid_free_lru - A least recently used list of free RMIDs *     These RMIDs are guaranteed to have an occupancy less than the *     threshold occupancy */static LIST_HEAD(rmid_free_lru); /* * @closid_num_dirty_rmid    The number of dirty RMID each CLOSID has. *     Only allocated when CONFIG_RESCTRL_RMID_DEPENDS_ON_CLOSID is defined. *     Indexed by CLOSID. Protected by rdtgroup_mutex. */static u32 *closid_num_dirty_rmid; /* * @rmid_limbo_count - count of currently unused but (potentially) *     dirty RMIDs. *     This counts RMIDs that no one is currently using but that *     may have a occupancy value > resctrl_rmid_realloc_threshold. User can *     change the threshold occupancy value. */static unsigned int rmid_limbo_count; /* * @rmid_entry - The entry in the limbo and free lists. */static struct rmid_entry	*rmid_ptrs; /* * Global boolean for rdt_monitor which is true if any * resource monitoring is enabled. */bool rdt_mon_capable; /* * Global to indicate which monitoring events are enabled. */unsigned int rdt_mon_features; /* * This is the threshold cache occupancy in bytes at which we will consider an * RMID available for re-allocation. */unsigned int resctrl_rmid_realloc_threshold; /* * This is the maximum value for the reallocation threshold, in bytes. */unsigned int resctrl_rmid_realloc_limit; #define CF(cf)	((unsigned long)(1048576 * (cf) + 0.5)) static int snc_nodes_per_l3_cache = 1; /* * The correction factor table is documented in Documentation/arch/x86/resctrl.rst. * If rmid > rmid threshold, MBM total and local values should be multiplied * by the correction factor. * * The original table is modified for better code: * * 1. The threshold 0 is changed to rmid count - 1 so don't do correction *    for the case. * 2. MBM total and local correction table indexed by core counter which is *    equal to (x86_cache_max_rmid + 1) / 8 - 1 and is from 0 up to 27. * 3. The correction factor is normalized to 2^20 (1048576) so it's faster *    to calculate corrected value by shifting: *    corrected_value = (original_value * correction_factor) >> 20 */static const struct mbm_correction_factor_table {	u32 rmidthreshold;	u64 cf;} mbm_cf_table[] __initconst = {	{7,	CF(1.000000)},	{15,	CF(1.000000)},	{15,	CF(0.969650)},	{31,	CF(1.000000)},	{31,	CF(1.066667)},	{31,	CF(0.969650)},	{47,	CF(1.142857)},	{63,	CF(1.000000)},	{63,	CF(1.185115)},	{63,	CF(1.066553)},	{79,	CF(1.454545)},	{95,	CF(1.000000)},	{95,	CF(1.230769)},	{95,	CF(1.142857)},	{95,	CF(1.066667)},	{127,	CF(1.000000)},	{127,	CF(1.254863)},	{127,	CF(1.185255)},	{151,	CF(1.000000)},	{127,	CF(1.066667)},	{167,	CF(1.000000)},	{159,	CF(1.454334)},	{183,	CF(1.000000)},	{127,	CF(0.969744)},	{191,	CF(1.280246)},	{191,	CF(1.230921)},	{215,	CF(1.000000)},	{191,	CF(1.143118)},}; static u32 mbm_cf_rmidthreshold __read_mostly = UINT_MAX;static u64 mbm_cf __read_mostly; static inline u64 get_corrected_mbm_count(u32 rmid, unsigned long val){	/* Correct MBM value. */	if (rmid > mbm_cf_rmidthreshold)		val = (val * mbm_cf) >> 20; 	return val;} /* * x86 and arm64 differ in their handling of monitoring. * x86's RMID are independent numbers, there is only one source of traffic * with an RMID value of '1'. * arm64's PMG extends the PARTID/CLOSID space, there are multiple sources of * traffic with a PMG value of '1', one for each CLOSID, meaning the RMID * value is no longer unique. * To account for this, resctrl uses an index. On x86 this is just the RMID, * on arm64 it encodes the CLOSID and RMID. This gives a unique number. * * The domain's rmid_busy_llc and rmid_ptrs[] are sized by index. The arch code * must accept an attempt to read every index. */static inline struct rmid_entry *__rmid_entry(u32 idx){	struct rmid_entry *entry;	u32 closid, rmid; 	entry = &rmid_ptrs[idx];	resctrl_arch_rmid_idx_decode(idx, &closid, &rmid); 	WARN_ON_ONCE(entry->closid != closid);	WARN_ON_ONCE(entry->rmid != rmid); 	return entry;} /* * When Sub-NUMA Cluster (SNC) mode is not enabled (as indicated by * "snc_nodes_per_l3_cache == 1") no translation of the RMID value is * needed. The physical RMID is the same as the logical RMID. * * On a platform with SNC mode enabled, Linux enables RMID sharing mode * via MSR 0xCA0 (see the "RMID Sharing Mode" section in the "Intel * Resource Director Technology Architecture Specification" for a full * description of RMID sharing mode). * * In RMID sharing mode there are fewer "logical RMID" values available * to accumulate data ("physical RMIDs" are divided evenly between SNC * nodes that share an L3 cache). Linux creates an rdt_mon_domain for * each SNC node. * * The value loaded into IA32_PQR_ASSOC is the "logical RMID". * * Data is collected independently on each SNC node and can be retrieved * using the "physical RMID" value computed by this function and loaded * into IA32_QM_EVTSEL. @cpu can be any CPU in the SNC node. * * The scope of the IA32_QM_EVTSEL and IA32_QM_CTR MSRs is at the L3 * cache.  So a "physical RMID" may be read from any CPU that shares * the L3 cache with the desired SNC node, not just from a CPU in * the specific SNC node. */static int logical_rmid_to_physical_rmid(int cpu, int lrmid){	struct rdt_resource *r = &rdt_resources_all[RDT_RESOURCE_L3].r_resctrl; 	if (snc_nodes_per_l3_cache == 1)		return lrmid; 	return lrmid + (cpu_to_node(cpu) % snc_nodes_per_l3_cache) * r->num_rmid;} static int __rmid_read_phys(u32 prmid, enum resctrl_event_id eventid, u64 *val){	u64 msr_val; 	/*	 * As per the SDM, when IA32_QM_EVTSEL.EvtID (bits 7:0) is configured	 * with a valid event code for supported resource type and the bits	 * IA32_QM_EVTSEL.RMID (bits 41:32) are configured with valid RMID,	 * IA32_QM_CTR.data (bits 61:0) reports the monitored data.	 * IA32_QM_CTR.Error (bit 63) and IA32_QM_CTR.Unavailable (bit 62)	 * are error bits.	 */	wrmsr(MSR_IA32_QM_EVTSEL, eventid, prmid);	rdmsrl(MSR_IA32_QM_CTR, msr_val); 	if (msr_val & RMID_VAL_ERROR)		return -EIO;	if (msr_val & RMID_VAL_UNAVAIL)		return -EINVAL; 	*val = msr_val;	return 0;} static struct arch_mbm_state *get_arch_mbm_state(struct rdt_hw_mon_domain *hw_dom,						 u32 rmid,						 enum resctrl_event_id eventid){	switch (eventid) {	case QOS_L3_OCCUP_EVENT_ID:		return NULL;	case QOS_L3_MBM_TOTAL_EVENT_ID:		return &hw_dom->arch_mbm_total[rmid];	case QOS_L3_MBM_LOCAL_EVENT_ID:		return &hw_dom->arch_mbm_local[rmid];	} 	/* Never expect to get here */	WARN_ON_ONCE(1); 	return NULL;} void resctrl_arch_reset_rmid(struct rdt_resource *r, struct rdt_mon_domain *d,			     u32 unused, u32 rmid,			     enum resctrl_event_id eventid){	struct rdt_hw_mon_domain *hw_dom = resctrl_to_arch_mon_dom(d);	int cpu = cpumask_any(&d->hdr.cpu_mask);	struct arch_mbm_state *am;	u32 prmid; 	am = get_arch_mbm_state(hw_dom, rmid, eventid);	if (am) {		memset(am, 0, sizeof(*am)); 		prmid = logical_rmid_to_physical_rmid(cpu, rmid);		/* Record any initial, non-zero count value. */		__rmid_read_phys(prmid, eventid, &am->prev_msr);	}} /* * Assumes that hardware counters are also reset and thus that there is * no need to record initial non-zero counts. */void resctrl_arch_reset_rmid_all(struct rdt_resource *r, struct rdt_mon_domain *d){	struct rdt_hw_mon_domain *hw_dom = resctrl_to_arch_mon_dom(d); 	if (is_mbm_total_enabled())		memset(hw_dom->arch_mbm_total, 0,		       sizeof(*hw_dom->arch_mbm_total) * r->num_rmid); 	if (is_mbm_local_enabled())		memset(hw_dom->arch_mbm_local, 0,		       sizeof(*hw_dom->arch_mbm_local) * r->num_rmid);} static u64 mbm_overflow_count(u64 prev_msr, u64 cur_msr, unsigned int width){	u64 shift = 64 - width, chunks; 	chunks = (cur_msr << shift) - (prev_msr << shift);	return chunks >> shift;} int resctrl_arch_rmid_read(struct rdt_resource *r, struct rdt_mon_domain *d,			   u32 unused, u32 rmid, enum resctrl_event_id eventid,			   u64 *val, void *ignored){	struct rdt_hw_mon_domain *hw_dom = resctrl_to_arch_mon_dom(d);	struct rdt_hw_resource *hw_res = resctrl_to_arch_res(r);	int cpu = cpumask_any(&d->hdr.cpu_mask);	struct arch_mbm_state *am;	u64 msr_val, chunks;	u32 prmid;	int ret; 	resctrl_arch_rmid_read_context_check(); 	prmid = logical_rmid_to_physical_rmid(cpu, rmid);	ret = __rmid_read_phys(prmid, eventid, &msr_val);	if (ret)		return ret; 	am = get_arch_mbm_state(hw_dom, rmid, eventid);	if (am) {		am->chunks += mbm_overflow_count(am->prev_msr, msr_val,						 hw_res->mbm_width);		chunks = get_corrected_mbm_count(rmid, am->chunks);		am->prev_msr = msr_val;	} else {		chunks = msr_val;	} 	*val = chunks * hw_res->mon_scale; 	return 0;} static void limbo_release_entry(struct rmid_entry *entry){	lockdep_assert_held(&rdtgroup_mutex); 	rmid_limbo_count--;	list_add_tail(&entry->list, &rmid_free_lru); 	if (IS_ENABLED(CONFIG_RESCTRL_RMID_DEPENDS_ON_CLOSID))		closid_num_dirty_rmid[entry->closid]--;} /* * Check the RMIDs that are marked as busy for this domain. If the * reported LLC occupancy is below the threshold clear the busy bit and * decrement the count. If the busy count gets to zero on an RMID, we * free the RMID */void __check_limbo(struct rdt_mon_domain *d, bool force_free){	struct rdt_resource *r = &rdt_resources_all[RDT_RESOURCE_L3].r_resctrl;	u32 idx_limit = resctrl_arch_system_num_rmid_idx();	struct rmid_entry *entry;	u32 idx, cur_idx = 1;	void *arch_mon_ctx;	bool rmid_dirty;	u64 val = 0; 	arch_mon_ctx = resctrl_arch_mon_ctx_alloc(r, QOS_L3_OCCUP_EVENT_ID);	if (IS_ERR(arch_mon_ctx)) {		pr_warn_ratelimited("Failed to allocate monitor context: %ld",				    PTR_ERR(arch_mon_ctx));		return;	} 	/*	 * Skip RMID 0 and start from RMID 1 and check all the RMIDs that	 * are marked as busy for occupancy < threshold. If the occupancy	 * is less than the threshold decrement the busy counter of the	 * RMID and move it to the free list when the counter reaches 0.	 */	for (;;) {		idx = find_next_bit(d->rmid_busy_llc, idx_limit, cur_idx);		if (idx >= idx_limit)			break; 		entry = __rmid_entry(idx);		if (resctrl_arch_rmid_read(r, d, entry->closid, entry->rmid,					   QOS_L3_OCCUP_EVENT_ID, &val,					   arch_mon_ctx)) {			rmid_dirty = true;		} else {			rmid_dirty = (val >= resctrl_rmid_realloc_threshold); 			/*			 * x86's CLOSID and RMID are independent numbers, so the entry's			 * CLOSID is an empty CLOSID (X86_RESCTRL_EMPTY_CLOSID). On Arm the			 * RMID (PMG) extends the CLOSID (PARTID) space with bits that aren't			 * used to select the configuration. It is thus necessary to track both			 * CLOSID and RMID because there may be dependencies between them			 * on some architectures.			 */			trace_mon_llc_occupancy_limbo(entry->closid, entry->rmid, d->hdr.id, val);		} 		if (force_free || !rmid_dirty) {			clear_bit(idx, d->rmid_busy_llc);			if (!--entry->busy)				limbo_release_entry(entry);		}		cur_idx = idx + 1;	} 	resctrl_arch_mon_ctx_free(r, QOS_L3_OCCUP_EVENT_ID, arch_mon_ctx);} bool has_busy_rmid(struct rdt_mon_domain *d){	u32 idx_limit = resctrl_arch_system_num_rmid_idx(); 	return find_first_bit(d->rmid_busy_llc, idx_limit) != idx_limit;} static struct rmid_entry *resctrl_find_free_rmid(u32 closid){	struct rmid_entry *itr;	u32 itr_idx, cmp_idx; 	if (list_empty(&rmid_free_lru))		return rmid_limbo_count ? ERR_PTR(-EBUSY) : ERR_PTR(-ENOSPC); 	list_for_each_entry(itr, &rmid_free_lru, list) {		/*		 * Get the index of this free RMID, and the index it would need		 * to be if it were used with this CLOSID.		 * If the CLOSID is irrelevant on this architecture, the two		 * index values are always the same on every entry and thus the		 * very first entry will be returned.		 */		itr_idx = resctrl_arch_rmid_idx_encode(itr->closid, itr->rmid);		cmp_idx = resctrl_arch_rmid_idx_encode(closid, itr->rmid); 		if (itr_idx == cmp_idx)			return itr;	} 	return ERR_PTR(-ENOSPC);} /** * resctrl_find_cleanest_closid() - Find a CLOSID where all the associated *                                  RMID are clean, or the CLOSID that has *                                  the most clean RMID. * * MPAM's equivalent of RMID are per-CLOSID, meaning a freshly allocated CLOSID * may not be able to allocate clean RMID. To avoid this the allocator will * choose the CLOSID with the most clean RMID. * * When the CLOSID and RMID are independent numbers, the first free CLOSID will * be returned. */int resctrl_find_cleanest_closid(void){	u32 cleanest_closid = ~0;	int i = 0; 	lockdep_assert_held(&rdtgroup_mutex); 	if (!IS_ENABLED(CONFIG_RESCTRL_RMID_DEPENDS_ON_CLOSID))		return -EIO; 	for (i = 0; i < closids_supported(); i++) {		int num_dirty; 		if (closid_allocated(i))			continue; 		num_dirty = closid_num_dirty_rmid[i];		if (num_dirty == 0)			return i; 		if (cleanest_closid == ~0)			cleanest_closid = i; 		if (num_dirty < closid_num_dirty_rmid[cleanest_closid])			cleanest_closid = i;	} 	if (cleanest_closid == ~0)		return -ENOSPC; 	return cleanest_closid;}

Lines 1–500 of 1259. Load more