// SPDX-License-Identifier: GPL-2.0-only/* * Resource Director Technology(RDT) * - Monitoring code * * Copyright (C) 2017 Intel Corporation * * Author: * Vikas Shivappa <vikas.shivappa@intel.com> * * This replaces the cqm.c based on perf but we reuse a lot of * code and datastructures originally from Peter Zijlstra and Matt Fleming. * * More information about RDT be found in the Intel (R) x86 Architecture * Software Developer Manual June 2016, volume 3, section 17.17. */ #define pr_fmt(fmt) "resctrl: " fmt #include <linux/cpu.h>#include <linux/resctrl.h>#include <linux/sizes.h>#include <linux/slab.h> #include "internal.h" #define CREATE_TRACE_POINTS #include "monitor_trace.h" /** * struct rmid_entry - dirty tracking for all RMID. * @closid: The CLOSID for this entry. * @rmid: The RMID for this entry. * @busy: The number of domains with cached data using this RMID. * @list: Member of the rmid_free_lru list when busy == 0. * * Depending on the architecture the correct monitor is accessed using * both @closid and @rmid, or @rmid only. * * Take the rdtgroup_mutex when accessing. */struct rmid_entry { u32 closid; u32 rmid; int busy; struct list_head list;}; /* * @rmid_free_lru - A least recently used list of free RMIDs * These RMIDs are guaranteed to have an occupancy less than the * threshold occupancy */static LIST_HEAD(rmid_free_lru); /* * @closid_num_dirty_rmid The number of dirty RMID each CLOSID has. * Only allocated when CONFIG_RESCTRL_RMID_DEPENDS_ON_CLOSID is defined. * Indexed by CLOSID. Protected by rdtgroup_mutex. */static u32 *closid_num_dirty_rmid; /* * @rmid_limbo_count - count of currently unused but (potentially) * dirty RMIDs. * This counts RMIDs that no one is currently using but that * may have a occupancy value > resctrl_rmid_realloc_threshold. User can * change the threshold occupancy value. */static unsigned int rmid_limbo_count; /* * @rmid_entry - The entry in the limbo and free lists. */static struct rmid_entry *rmid_ptrs; /* * This is the threshold cache occupancy in bytes at which we will consider an * RMID available for re-allocation. */unsigned int resctrl_rmid_realloc_threshold; /* * This is the maximum value for the reallocation threshold, in bytes. */unsigned int resctrl_rmid_realloc_limit; /* * x86 and arm64 differ in their handling of monitoring. * x86's RMID are independent numbers, there is only one source of traffic * with an RMID value of '1'. * arm64's PMG extends the PARTID/CLOSID space, there are multiple sources of * traffic with a PMG value of '1', one for each CLOSID, meaning the RMID * value is no longer unique. * To account for this, resctrl uses an index. On x86 this is just the RMID, * on arm64 it encodes the CLOSID and RMID. This gives a unique number. * * The domain's rmid_busy_llc and rmid_ptrs[] are sized by index. The arch code * must accept an attempt to read every index. */static inline struct rmid_entry *__rmid_entry(u32 idx){ struct rmid_entry *entry; u32 closid, rmid; entry = &rmid_ptrs[idx]; resctrl_arch_rmid_idx_decode(idx, &closid, &rmid); WARN_ON_ONCE(entry->closid != closid); WARN_ON_ONCE(entry->rmid != rmid); return entry;} static void limbo_release_entry(struct rmid_entry *entry){ lockdep_assert_held(&rdtgroup_mutex); rmid_limbo_count--; list_add_tail(&entry->list, &rmid_free_lru); if (IS_ENABLED(CONFIG_RESCTRL_RMID_DEPENDS_ON_CLOSID)) closid_num_dirty_rmid[entry->closid]--;} /* * Check the RMIDs that are marked as busy for this domain. If the * reported LLC occupancy is below the threshold clear the busy bit and * decrement the count. If the busy count gets to zero on an RMID, we * free the RMID */void __check_limbo(struct rdt_l3_mon_domain *d, bool force_free){ struct rdt_resource *r = resctrl_arch_get_resource(RDT_RESOURCE_L3); u32 idx_limit = resctrl_arch_system_num_rmid_idx(); struct rmid_entry *entry; u32 idx, cur_idx = 1; void *arch_mon_ctx; void *arch_priv; bool rmid_dirty; u64 val = 0; arch_priv = mon_event_all[QOS_L3_OCCUP_EVENT_ID].arch_priv; arch_mon_ctx = resctrl_arch_mon_ctx_alloc(r, QOS_L3_OCCUP_EVENT_ID); if (IS_ERR(arch_mon_ctx)) { pr_warn_ratelimited("Failed to allocate monitor context: %ld", PTR_ERR(arch_mon_ctx)); return; } /* * Skip RMID 0 and start from RMID 1 and check all the RMIDs that * are marked as busy for occupancy < threshold. If the occupancy * is less than the threshold decrement the busy counter of the * RMID and move it to the free list when the counter reaches 0. */ for (;;) { idx = find_next_bit(d->rmid_busy_llc, idx_limit, cur_idx); if (idx >= idx_limit) break; entry = __rmid_entry(idx); if (resctrl_arch_rmid_read(r, &d->hdr, entry->closid, entry->rmid, QOS_L3_OCCUP_EVENT_ID, arch_priv, &val, arch_mon_ctx)) { rmid_dirty = true; } else { rmid_dirty = (val >= resctrl_rmid_realloc_threshold); /* * x86's CLOSID and RMID are independent numbers, so the entry's * CLOSID is an empty CLOSID (X86_RESCTRL_EMPTY_CLOSID). On Arm the * RMID (PMG) extends the CLOSID (PARTID) space with bits that aren't * used to select the configuration. It is thus necessary to track both * CLOSID and RMID because there may be dependencies between them * on some architectures. */ trace_mon_llc_occupancy_limbo(entry->closid, entry->rmid, d->hdr.id, val); } if (force_free || !rmid_dirty) { clear_bit(idx, d->rmid_busy_llc); if (!--entry->busy) limbo_release_entry(entry); } cur_idx = idx + 1; } resctrl_arch_mon_ctx_free(r, QOS_L3_OCCUP_EVENT_ID, arch_mon_ctx);} bool has_busy_rmid(struct rdt_l3_mon_domain *d){ u32 idx_limit = resctrl_arch_system_num_rmid_idx(); return find_first_bit(d->rmid_busy_llc, idx_limit) != idx_limit;} static struct rmid_entry *resctrl_find_free_rmid(u32 closid){ struct rmid_entry *itr; u32 itr_idx, cmp_idx; if (list_empty(&rmid_free_lru)) return rmid_limbo_count ? ERR_PTR(-EBUSY) : ERR_PTR(-ENOSPC); list_for_each_entry(itr, &rmid_free_lru, list) { /* * Get the index of this free RMID, and the index it would need * to be if it were used with this CLOSID. * If the CLOSID is irrelevant on this architecture, the two * index values are always the same on every entry and thus the * very first entry will be returned. */ itr_idx = resctrl_arch_rmid_idx_encode(itr->closid, itr->rmid); cmp_idx = resctrl_arch_rmid_idx_encode(closid, itr->rmid); if (itr_idx == cmp_idx) return itr; } return ERR_PTR(-ENOSPC);} /** * resctrl_find_cleanest_closid() - Find a CLOSID where all the associated * RMID are clean, or the CLOSID that has * the most clean RMID. * * MPAM's equivalent of RMID are per-CLOSID, meaning a freshly allocated CLOSID * may not be able to allocate clean RMID. To avoid this the allocator will * choose the CLOSID with the most clean RMID. * * When the CLOSID and RMID are independent numbers, the first free CLOSID will * be returned. */int resctrl_find_cleanest_closid(void){ u32 cleanest_closid = ~0; int i = 0; lockdep_assert_held(&rdtgroup_mutex); if (!IS_ENABLED(CONFIG_RESCTRL_RMID_DEPENDS_ON_CLOSID)) return -EIO; for (i = 0; i < closids_supported(); i++) { int num_dirty; if (closid_allocated(i)) continue; num_dirty = closid_num_dirty_rmid[i]; if (num_dirty == 0) return i; if (cleanest_closid == ~0) cleanest_closid = i; if (num_dirty < closid_num_dirty_rmid[cleanest_closid]) cleanest_closid = i; } if (cleanest_closid == ~0) return -ENOSPC; return cleanest_closid;} /* * For MPAM the RMID value is not unique, and has to be considered with * the CLOSID. The (CLOSID, RMID) pair is allocated on all domains, which * allows all domains to be managed by a single free list. * Each domain also has a rmid_busy_llc to reduce the work of the limbo handler. */int alloc_rmid(u32 closid){ struct rmid_entry *entry; lockdep_assert_held(&rdtgroup_mutex); entry = resctrl_find_free_rmid(closid); if (IS_ERR(entry)) return PTR_ERR(entry); list_del(&entry->list); return entry->rmid;} static void add_rmid_to_limbo(struct rmid_entry *entry){ struct rdt_resource *r = resctrl_arch_get_resource(RDT_RESOURCE_L3); struct rdt_l3_mon_domain *d; u32 idx; lockdep_assert_held(&rdtgroup_mutex); /* Walking r->domains, ensure it can't race with cpuhp */ lockdep_assert_cpus_held(); idx = resctrl_arch_rmid_idx_encode(entry->closid, entry->rmid); entry->busy = 0; list_for_each_entry(d, &r->mon_domains, hdr.list) { /* * For the first limbo RMID in the domain, * setup up the limbo worker. */ if (!has_busy_rmid(d)) cqm_setup_limbo_handler(d, CQM_LIMBOCHECK_INTERVAL, RESCTRL_PICK_ANY_CPU); set_bit(idx, d->rmid_busy_llc); entry->busy++; } rmid_limbo_count++; if (IS_ENABLED(CONFIG_RESCTRL_RMID_DEPENDS_ON_CLOSID)) closid_num_dirty_rmid[entry->closid]++;} void free_rmid(u32 closid, u32 rmid){ u32 idx = resctrl_arch_rmid_idx_encode(closid, rmid); struct rmid_entry *entry; lockdep_assert_held(&rdtgroup_mutex); /* * Do not allow the default rmid to be free'd. Comparing by index * allows architectures that ignore the closid parameter to avoid an * unnecessary check. */ if (!resctrl_arch_mon_capable() || idx == resctrl_arch_rmid_idx_encode(RESCTRL_RESERVED_CLOSID, RESCTRL_RESERVED_RMID)) return; entry = __rmid_entry(idx); if (resctrl_is_mon_event_enabled(QOS_L3_OCCUP_EVENT_ID)) add_rmid_to_limbo(entry); else list_add_tail(&entry->list, &rmid_free_lru);} static struct mbm_state *get_mbm_state(struct rdt_l3_mon_domain *d, u32 closid, u32 rmid, enum resctrl_event_id evtid){ u32 idx = resctrl_arch_rmid_idx_encode(closid, rmid); struct mbm_state *state; if (!resctrl_is_mbm_event(evtid)) return NULL; state = d->mbm_states[MBM_STATE_IDX(evtid)]; return state ? &state[idx] : NULL;} /* * mbm_cntr_get() - Return the counter ID for the matching @evtid and @rdtgrp. * * Return: * Valid counter ID on success, or -ENOENT on failure. */static int mbm_cntr_get(struct rdt_resource *r, struct rdt_l3_mon_domain *d, struct rdtgroup *rdtgrp, enum resctrl_event_id evtid){ int cntr_id; if (!r->mon.mbm_cntr_assignable) return -ENOENT; if (!resctrl_is_mbm_event(evtid)) return -ENOENT; for (cntr_id = 0; cntr_id < r->mon.num_mbm_cntrs; cntr_id++) { if (d->cntr_cfg[cntr_id].rdtgrp == rdtgrp && d->cntr_cfg[cntr_id].evtid == evtid) return cntr_id; } return -ENOENT;} /* * mbm_cntr_alloc() - Initialize and return a new counter ID in the domain @d. * Caller must ensure that the specified event is not assigned already. * * Return: * Valid counter ID on success, or -ENOSPC on failure. */static int mbm_cntr_alloc(struct rdt_resource *r, struct rdt_l3_mon_domain *d, struct rdtgroup *rdtgrp, enum resctrl_event_id evtid){ int cntr_id; for (cntr_id = 0; cntr_id < r->mon.num_mbm_cntrs; cntr_id++) { if (!d->cntr_cfg[cntr_id].rdtgrp) { d->cntr_cfg[cntr_id].rdtgrp = rdtgrp; d->cntr_cfg[cntr_id].evtid = evtid; return cntr_id; } } return -ENOSPC;} /* * mbm_cntr_free() - Clear the counter ID configuration details in the domain @d. */static void mbm_cntr_free(struct rdt_l3_mon_domain *d, int cntr_id){ memset(&d->cntr_cfg[cntr_id], 0, sizeof(*d->cntr_cfg));} static int __l3_mon_event_count(struct rdtgroup *rdtgrp, struct rmid_read *rr){ int cpu = smp_processor_id(); u32 closid = rdtgrp->closid; u32 rmid = rdtgrp->mon.rmid; struct rdt_l3_mon_domain *d; int cntr_id = -ENOENT; struct mbm_state *m; u64 tval = 0; if (!domain_header_is_valid(rr->hdr, RESCTRL_MON_DOMAIN, RDT_RESOURCE_L3)) { rr->err = -EIO; return -EINVAL; } d = container_of(rr->hdr, struct rdt_l3_mon_domain, hdr); if (rr->is_mbm_cntr) { cntr_id = mbm_cntr_get(rr->r, d, rdtgrp, rr->evt->evtid); if (cntr_id < 0) { rr->err = -ENOENT; return -EINVAL; } } if (rr->first) { if (rr->is_mbm_cntr) resctrl_arch_reset_cntr(rr->r, d, closid, rmid, cntr_id, rr->evt->evtid); else resctrl_arch_reset_rmid(rr->r, d, closid, rmid, rr->evt->evtid); m = get_mbm_state(d, closid, rmid, rr->evt->evtid); if (m) memset(m, 0, sizeof(struct mbm_state)); return 0; } /* Reading a single domain, must be on a CPU in that domain. */ if (!cpumask_test_cpu(cpu, &d->hdr.cpu_mask)) return -EINVAL; if (rr->is_mbm_cntr) rr->err = resctrl_arch_cntr_read(rr->r, d, closid, rmid, cntr_id, rr->evt->evtid, &tval); else rr->err = resctrl_arch_rmid_read(rr->r, rr->hdr, closid, rmid, rr->evt->evtid, rr->evt->arch_priv, &tval, rr->arch_mon_ctx); if (rr->err) return rr->err; rr->val += tval; return 0;} static int __l3_mon_event_count_sum(struct rdtgroup *rdtgrp, struct rmid_read *rr){ int cpu = smp_processor_id(); u32 closid = rdtgrp->closid; u32 rmid = rdtgrp->mon.rmid; struct rdt_l3_mon_domain *d; u64 tval = 0; int err, ret; /* * Summing across domains is only done for systems that implement * Sub-NUMA Cluster. There is no overlap with systems that support * assignable counters. */ if (rr->is_mbm_cntr) { pr_warn_once("Summing domains using assignable counters is not supported\n"); rr->err = -EINVAL; return -EINVAL; } /* Summing domains that share a cache, must be on a CPU for that cache. */ if (!cpumask_test_cpu(cpu, &rr->ci->shared_cpu_map)) return -EINVAL; /* * Legacy files must report the sum of an event across all * domains that share the same L3 cache instance. * Report success if a read from any domain succeeds, -EINVAL * (translated to "Unavailable" for user space) if reading from * all domains fail for any reason.Lines 1–500 of 1903. Load more
// SPDX-License-Identifier: GPL-2.0-only /* * Resource Director Technology(RDT) * - Monitoring code * * Copyright (C) 2017 Intel Corporation * * Author: * Vikas Shivappa <vikas.shivappa@intel.com> * * This replaces the cqm.c based on perf but we reuse a lot of * code and datastructures originally from Peter Zijlstra and Matt Fleming. * * More information about RDT be found in the Intel (R) x86 Architecture * Software Developer Manual June 2016, volume 3, section 17.17. */ #define pr_fmt(fmt) "resctrl: " fmt #include <linux/cpu.h> #include <linux/resctrl.h> #include <linux/sizes.h> #include <linux/slab.h> #include "internal.h" #define CREATE_TRACE_POINTS #include "monitor_trace.h" /** * struct rmid_entry - dirty tracking for all RMID. * @closid: The CLOSID for this entry. * @rmid: The RMID for this entry. * @busy: The number of domains with cached data using this RMID. * @list: Member of the rmid_free_lru list when busy == 0. * * Depending on the architecture the correct monitor is accessed using * both @closid and @rmid, or @rmid only. * * Take the rdtgroup_mutex when accessing. */ struct rmid_entry { u32 closid; u32 rmid; int busy; struct list_head list; }; /* * @rmid_free_lru - A least recently used list of free RMIDs * These RMIDs are guaranteed to have an occupancy less than the * threshold occupancy */ static LIST_HEAD(rmid_free_lru); /* * @closid_num_dirty_rmid The number of dirty RMID each CLOSID has. * Only allocated when CONFIG_RESCTRL_RMID_DEPENDS_ON_CLOSID is defined. * Indexed by CLOSID. Protected by rdtgroup_mutex. */ static u32 *closid_num_dirty_rmid; /* * @rmid_limbo_count - count of currently unused but (potentially) * dirty RMIDs. * This counts RMIDs that no one is currently using but that * may have a occupancy value > resctrl_rmid_realloc_threshold. User can * change the threshold occupancy value. */ static unsigned int rmid_limbo_count; /* * @rmid_entry - The entry in the limbo and free lists. */ static struct rmid_entry *rmid_ptrs; /* * This is the threshold cache occupancy in bytes at which we will consider an * RMID available for re-allocation. */ unsigned int resctrl_rmid_realloc_threshold; /* * This is the maximum value for the reallocation threshold, in bytes. */ unsigned int resctrl_rmid_realloc_limit; /* * x86 and arm64 differ in their handling of monitoring. * x86's RMID are independent numbers, there is only one source of traffic * with an RMID value of '1'. * arm64's PMG extends the PARTID/CLOSID space, there are multiple sources of * traffic with a PMG value of '1', one for each CLOSID, meaning the RMID * value is no longer unique. * To account for this, resctrl uses an index. On x86 this is just the RMID, * on arm64 it encodes the CLOSID and RMID. This gives a unique number. * * The domain's rmid_busy_llc and rmid_ptrs[] are sized by index. The arch code * must accept an attempt to read every index. */ static inline struct rmid_entry *__rmid_entry(u32 idx) { struct rmid_entry *entry; u32 closid, rmid; entry = &rmid_ptrs[idx]; resctrl_arch_rmid_idx_decode(idx, &closid, &rmid); WARN_ON_ONCE(entry->closid != closid); WARN_ON_ONCE(entry->rmid != rmid); return entry; } static void limbo_release_entry(struct rmid_entry *entry) { lockdep_assert_held(&rdtgroup_mutex); rmid_limbo_count--; list_add_tail(&entry->list, &rmid_free_lru); if (IS_ENABLED(CONFIG_RESCTRL_RMID_DEPENDS_ON_CLOSID)) closid_num_dirty_rmid[entry->closid]--; } /* * Check the RMIDs that are marked as busy for this domain. If the * reported LLC occupancy is below the threshold clear the busy bit and * decrement the count. If the busy count gets to zero on an RMID, we * free the RMID */ void __check_limbo(struct rdt_l3_mon_domain *d, bool force_free) { struct rdt_resource *r = resctrl_arch_get_resource(RDT_RESOURCE_L3); u32 idx_limit = resctrl_arch_system_num_rmid_idx(); struct rmid_entry *entry; u32 idx, cur_idx = 1; void *arch_mon_ctx; void *arch_priv; bool rmid_dirty; u64 val = 0; arch_priv = mon_event_all[QOS_L3_OCCUP_EVENT_ID].arch_priv; arch_mon_ctx = resctrl_arch_mon_ctx_alloc(r, QOS_L3_OCCUP_EVENT_ID); if (IS_ERR(arch_mon_ctx)) { pr_warn_ratelimited("Failed to allocate monitor context: %ld", PTR_ERR(arch_mon_ctx)); return; } /* * Skip RMID 0 and start from RMID 1 and check all the RMIDs that * are marked as busy for occupancy < threshold. If the occupancy * is less than the threshold decrement the busy counter of the * RMID and move it to the free list when the counter reaches 0. */ for (;;) { idx = find_next_bit(d->rmid_busy_llc, idx_limit, cur_idx); if (idx >= idx_limit) break; entry = __rmid_entry(idx); if (resctrl_arch_rmid_read(r, &d->hdr, entry->closid, entry->rmid, QOS_L3_OCCUP_EVENT_ID, arch_priv, &val, arch_mon_ctx)) { rmid_dirty = true; } else { rmid_dirty = (val >= resctrl_rmid_realloc_threshold); /* * x86's CLOSID and RMID are independent numbers, so the entry's * CLOSID is an empty CLOSID (X86_RESCTRL_EMPTY_CLOSID). On Arm the * RMID (PMG) extends the CLOSID (PARTID) space with bits that aren't * used to select the configuration. It is thus necessary to track both * CLOSID and RMID because there may be dependencies between them * on some architectures. */ trace_mon_llc_occupancy_limbo(entry->closid, entry->rmid, d->hdr.id, val); } if (force_free || !rmid_dirty) { clear_bit(idx, d->rmid_busy_llc); if (!--entry->busy) limbo_release_entry(entry); } cur_idx = idx + 1; } resctrl_arch_mon_ctx_free(r, QOS_L3_OCCUP_EVENT_ID, arch_mon_ctx); } bool has_busy_rmid(struct rdt_l3_mon_domain *d) { u32 idx_limit = resctrl_arch_system_num_rmid_idx(); return find_first_bit(d->rmid_busy_llc, idx_limit) != idx_limit; } static struct rmid_entry *resctrl_find_free_rmid(u32 closid) { struct rmid_entry *itr; u32 itr_idx, cmp_idx; if (list_empty(&rmid_free_lru)) return rmid_limbo_count ? ERR_PTR(-EBUSY) : ERR_PTR(-ENOSPC); list_for_each_entry(itr, &rmid_free_lru, list) { /* * Get the index of this free RMID, and the index it would need * to be if it were used with this CLOSID. * If the CLOSID is irrelevant on this architecture, the two * index values are always the same on every entry and thus the * very first entry will be returned. */ itr_idx = resctrl_arch_rmid_idx_encode(itr->closid, itr->rmid); cmp_idx = resctrl_arch_rmid_idx_encode(closid, itr->rmid); if (itr_idx == cmp_idx) return itr; } return ERR_PTR(-ENOSPC); } /** * resctrl_find_cleanest_closid() - Find a CLOSID where all the associated * RMID are clean, or the CLOSID that has * the most clean RMID. * * MPAM's equivalent of RMID are per-CLOSID, meaning a freshly allocated CLOSID * may not be able to allocate clean RMID. To avoid this the allocator will * choose the CLOSID with the most clean RMID. * * When the CLOSID and RMID are independent numbers, the first free CLOSID will * be returned. */ int resctrl_find_cleanest_closid(void) { u32 cleanest_closid = ~0; int i = 0; lockdep_assert_held(&rdtgroup_mutex); if (!IS_ENABLED(CONFIG_RESCTRL_RMID_DEPENDS_ON_CLOSID)) return -EIO; for (i = 0; i < closids_supported(); i++) { int num_dirty; if (closid_allocated(i)) continue; num_dirty = closid_num_dirty_rmid[i]; if (num_dirty == 0) return i; if (cleanest_closid == ~0) cleanest_closid = i; if (num_dirty < closid_num_dirty_rmid[cleanest_closid]) cleanest_closid = i; } if (cleanest_closid == ~0) return -ENOSPC; return cleanest_closid; } /* * For MPAM the RMID value is not unique, and has to be considered with * the CLOSID. The (CLOSID, RMID) pair is allocated on all domains, which * allows all domains to be managed by a single free list. * Each domain also has a rmid_busy_llc to reduce the work of the limbo handler. */ int alloc_rmid(u32 closid) { struct rmid_entry *entry; lockdep_assert_held(&rdtgroup_mutex); entry = resctrl_find_free_rmid(closid); if (IS_ERR(entry)) return PTR_ERR(entry); list_del(&entry->list); return entry->rmid; } static void add_rmid_to_limbo(struct rmid_entry *entry) { struct rdt_resource *r = resctrl_arch_get_resource(RDT_RESOURCE_L3); struct rdt_l3_mon_domain *d; u32 idx; lockdep_assert_held(&rdtgroup_mutex); /* Walking r->domains, ensure it can't race with cpuhp */ lockdep_assert_cpus_held(); idx = resctrl_arch_rmid_idx_encode(entry->closid, entry->rmid); entry->busy = 0; list_for_each_entry(d, &r->mon_domains, hdr.list) { /* * For the first limbo RMID in the domain, * setup up the limbo worker. */ if (!has_busy_rmid(d)) cqm_setup_limbo_handler(d, CQM_LIMBOCHECK_INTERVAL, RESCTRL_PICK_ANY_CPU); set_bit(idx, d->rmid_busy_llc); entry->busy++; } rmid_limbo_count++; if (IS_ENABLED(CONFIG_RESCTRL_RMID_DEPENDS_ON_CLOSID)) closid_num_dirty_rmid[entry->closid]++; } void free_rmid(u32 closid, u32 rmid) { u32 idx = resctrl_arch_rmid_idx_encode(closid, rmid); struct rmid_entry *entry; lockdep_assert_held(&rdtgroup_mutex); /* * Do not allow the default rmid to be free'd. Comparing by index * allows architectures that ignore the closid parameter to avoid an * unnecessary check. */ if (!resctrl_arch_mon_capable() || idx == resctrl_arch_rmid_idx_encode(RESCTRL_RESERVED_CLOSID, RESCTRL_RESERVED_RMID)) return; entry = __rmid_entry(idx); if (resctrl_is_mon_event_enabled(QOS_L3_OCCUP_EVENT_ID)) add_rmid_to_limbo(entry); else list_add_tail(&entry->list, &rmid_free_lru); } static struct mbm_state *get_mbm_state(struct rdt_l3_mon_domain *d, u32 closid, u32 rmid, enum resctrl_event_id evtid) { u32 idx = resctrl_arch_rmid_idx_encode(closid, rmid); struct mbm_state *state; if (!resctrl_is_mbm_event(evtid)) return NULL; state = d->mbm_states[MBM_STATE_IDX(evtid)]; return state ? &state[idx] : NULL; } /* * mbm_cntr_get() - Return the counter ID for the matching @evtid and @rdtgrp. * * Return: * Valid counter ID on success, or -ENOENT on failure. */ static int mbm_cntr_get(struct rdt_resource *r, struct rdt_l3_mon_domain *d, struct rdtgroup *rdtgrp, enum resctrl_event_id evtid) { int cntr_id; if (!r->mon.mbm_cntr_assignable) return -ENOENT; if (!resctrl_is_mbm_event(evtid)) return -ENOENT; for (cntr_id = 0; cntr_id < r->mon.num_mbm_cntrs; cntr_id++) { if (d->cntr_cfg[cntr_id].rdtgrp == rdtgrp && d->cntr_cfg[cntr_id].evtid == evtid) return cntr_id; } return -ENOENT; } /* * mbm_cntr_alloc() - Initialize and return a new counter ID in the domain @d. * Caller must ensure that the specified event is not assigned already. * * Return: * Valid counter ID on success, or -ENOSPC on failure. */ static int mbm_cntr_alloc(struct rdt_resource *r, struct rdt_l3_mon_domain *d, struct rdtgroup *rdtgrp, enum resctrl_event_id evtid) { int cntr_id; for (cntr_id = 0; cntr_id < r->mon.num_mbm_cntrs; cntr_id++) { if (!d->cntr_cfg[cntr_id].rdtgrp) { d->cntr_cfg[cntr_id].rdtgrp = rdtgrp; d->cntr_cfg[cntr_id].evtid = evtid; return cntr_id; } } return -ENOSPC; } /* * mbm_cntr_free() - Clear the counter ID configuration details in the domain @d. */ static void mbm_cntr_free(struct rdt_l3_mon_domain *d, int cntr_id) { memset(&d->cntr_cfg[cntr_id], 0, sizeof(*d->cntr_cfg)); } static int __l3_mon_event_count(struct rdtgroup *rdtgrp, struct rmid_read *rr) { int cpu = smp_processor_id(); u32 closid = rdtgrp->closid; u32 rmid = rdtgrp->mon.rmid; struct rdt_l3_mon_domain *d; int cntr_id = -ENOENT; struct mbm_state *m; u64 tval = 0; if (!domain_header_is_valid(rr->hdr, RESCTRL_MON_DOMAIN, RDT_RESOURCE_L3)) { rr->err = -EIO; return -EINVAL; } d = container_of(rr->hdr, struct rdt_l3_mon_domain, hdr); if (rr->is_mbm_cntr) { cntr_id = mbm_cntr_get(rr->r, d, rdtgrp, rr->evt->evtid); if (cntr_id < 0) { rr->err = -ENOENT; return -EINVAL; } } if (rr->first) { if (rr->is_mbm_cntr) resctrl_arch_reset_cntr(rr->r, d, closid, rmid, cntr_id, rr->evt->evtid); else resctrl_arch_reset_rmid(rr->r, d, closid, rmid, rr->evt->evtid); m = get_mbm_state(d, closid, rmid, rr->evt->evtid); if (m) memset(m, 0, sizeof(struct mbm_state)); return 0; } /* Reading a single domain, must be on a CPU in that domain. */ if (!cpumask_test_cpu(cpu, &d->hdr.cpu_mask)) return -EINVAL; if (rr->is_mbm_cntr) rr->err = resctrl_arch_cntr_read(rr->r, d, closid, rmid, cntr_id, rr->evt->evtid, &tval); else rr->err = resctrl_arch_rmid_read(rr->r, rr->hdr, closid, rmid, rr->evt->evtid, rr->evt->arch_priv, &tval, rr->arch_mon_ctx); if (rr->err) return rr->err; rr->val += tval; return 0; } static int __l3_mon_event_count_sum(struct rdtgroup *rdtgrp, struct rmid_read *rr) { int cpu = smp_processor_id(); u32 closid = rdtgrp->closid; u32 rmid = rdtgrp->mon.rmid; struct rdt_l3_mon_domain *d; u64 tval = 0; int err, ret; /* * Summing across domains is only done for systems that implement * Sub-NUMA Cluster. There is no overlap with systems that support * assignable counters. */ if (rr->is_mbm_cntr) { pr_warn_once("Summing domains using assignable counters is not supported\n"); rr->err = -EINVAL; return -EINVAL; } /* Summing domains that share a cache, must be on a CPU for that cache. */ if (!cpumask_test_cpu(cpu, &rr->ci->shared_cpu_map)) return -EINVAL; /* * Legacy files must report the sum of an event across all * domains that share the same L3 cache instance. * Report success if a read from any domain succeeds, -EINVAL * (translated to "Unavailable" for user space) if reading from * all domains fail for any reason.