// SPDX-License-Identifier: GPL-2.0// Copyright (C) 2025 Arm Ltd. #define pr_fmt(fmt) "%s:%s: " fmt, KBUILD_MODNAME, __func__ #include <linux/arm_mpam.h>#include <linux/cacheinfo.h>#include <linux/cpu.h>#include <linux/cpumask.h>#include <linux/errno.h>#include <linux/limits.h>#include <linux/list.h>#include <linux/math.h>#include <linux/printk.h>#include <linux/rculist.h>#include <linux/resctrl.h>#include <linux/slab.h>#include <linux/types.h>#include <linux/wait.h> #include <asm/mpam.h> #include "mpam_internal.h" static DECLARE_WAIT_QUEUE_HEAD(resctrl_mon_ctx_waiters); /* * The classes we've picked to map to resctrl resources, wrapped * in with their resctrl structure. * Class pointer may be NULL. */static struct mpam_resctrl_res mpam_resctrl_controls[RDT_NUM_RESOURCES]; #define for_each_mpam_resctrl_control(res, rid) \ for (rid = 0, res = &mpam_resctrl_controls[rid]; \ rid < RDT_NUM_RESOURCES; \ rid++, res = &mpam_resctrl_controls[rid]) /* * The classes we've picked to map to resctrl events. * Resctrl believes all the worlds a Xeon, and these are all on the L3. This * array lets us find the actual class backing the event counters. e.g. * the only memory bandwidth counters may be on the memory controller, but to * make use of them, we pretend they are on L3. Restrict the events considered * to those supported by MPAM. * Class pointer may be NULL. */#define MPAM_MAX_EVENT QOS_L3_MBM_TOTAL_EVENT_IDstatic struct mpam_resctrl_mon mpam_resctrl_counters[MPAM_MAX_EVENT + 1]; #define for_each_mpam_resctrl_mon(mon, eventid) \ for (eventid = QOS_FIRST_EVENT, mon = &mpam_resctrl_counters[eventid]; \ eventid <= MPAM_MAX_EVENT; \ eventid++, mon = &mpam_resctrl_counters[eventid]) /* The lock for modifying resctrl's domain lists from cpuhp callbacks. */static DEFINE_MUTEX(domain_list_lock); /* * MPAM emulates CDP by setting different PARTID in the I/D fields of MPAM0_EL1. * This applies globally to all traffic the CPU generates. */static bool cdp_enabled; /* * We use cacheinfo to discover the size of the caches and their id. cacheinfo * populates this from a device_initcall(). mpam_resctrl_setup() must wait. */static bool cacheinfo_ready;static DECLARE_WAIT_QUEUE_HEAD(wait_cacheinfo_ready); /* * If resctrl_init() succeeded, resctrl_exit() can be used to remove support * for the filesystem in the event of an error. */static bool resctrl_enabled; bool resctrl_arch_alloc_capable(void){ struct mpam_resctrl_res *res; enum resctrl_res_level rid; for_each_mpam_resctrl_control(res, rid) { if (res->resctrl_res.alloc_capable) return true; } return false;} bool resctrl_arch_mon_capable(void){ struct mpam_resctrl_res *res = &mpam_resctrl_controls[RDT_RESOURCE_L3]; struct rdt_resource *l3 = &res->resctrl_res; /* All monitors are presented as being on the L3 cache */ return l3->mon_capable;} bool resctrl_arch_is_evt_configurable(enum resctrl_event_id evt){ return false;} void resctrl_arch_mon_event_config_read(void *info){} void resctrl_arch_mon_event_config_write(void *info){} void resctrl_arch_reset_rmid_all(struct rdt_resource *r, struct rdt_l3_mon_domain *d){} void resctrl_arch_reset_rmid(struct rdt_resource *r, struct rdt_l3_mon_domain *d, u32 closid, u32 rmid, enum resctrl_event_id eventid){} void resctrl_arch_reset_cntr(struct rdt_resource *r, struct rdt_l3_mon_domain *d, u32 closid, u32 rmid, int cntr_id, enum resctrl_event_id eventid){} void resctrl_arch_config_cntr(struct rdt_resource *r, struct rdt_l3_mon_domain *d, enum resctrl_event_id evtid, u32 rmid, u32 closid, u32 cntr_id, bool assign){} int resctrl_arch_cntr_read(struct rdt_resource *r, struct rdt_l3_mon_domain *d, u32 unused, u32 rmid, int cntr_id, enum resctrl_event_id eventid, u64 *val){ return -EOPNOTSUPP;} bool resctrl_arch_mbm_cntr_assign_enabled(struct rdt_resource *r){ return false;} int resctrl_arch_mbm_cntr_assign_set(struct rdt_resource *r, bool enable){ return -EINVAL;} int resctrl_arch_io_alloc_enable(struct rdt_resource *r, bool enable){ return -EOPNOTSUPP;} bool resctrl_arch_get_io_alloc_enabled(struct rdt_resource *r){ return false;} void resctrl_arch_pre_mount(void){} bool resctrl_arch_get_cdp_enabled(enum resctrl_res_level rid){ return mpam_resctrl_controls[rid].cdp_enabled;} /** * resctrl_reset_task_closids() - Reset the PARTID/PMG values for all tasks. * * At boot, all existing tasks use partid zero for D and I. * To enable/disable CDP emulation, all these tasks need relabelling. */static void resctrl_reset_task_closids(void){ struct task_struct *p, *t; read_lock(&tasklist_lock); for_each_process_thread(p, t) { resctrl_arch_set_closid_rmid(t, RESCTRL_RESERVED_CLOSID, RESCTRL_RESERVED_RMID); } read_unlock(&tasklist_lock);} int resctrl_arch_set_cdp_enabled(enum resctrl_res_level rid, bool enable){ u32 partid_i = RESCTRL_RESERVED_CLOSID, partid_d = RESCTRL_RESERVED_CLOSID; struct mpam_resctrl_res *res = &mpam_resctrl_controls[RDT_RESOURCE_L3]; struct rdt_resource *l3 = &res->resctrl_res; int cpu; if (!IS_ENABLED(CONFIG_EXPERT) && enable) { /* * If the resctrl fs is mounted more than once, sequentially, * then CDP can lead to the use of out of range PARTIDs. */ pr_warn("CDP not supported\n"); return -EOPNOTSUPP; } if (enable) pr_warn("CDP is an expert feature and may cause MPAM to malfunction.\n"); /* * resctrl_arch_set_cdp_enabled() is only called with enable set to * false on error and unmount. */ cdp_enabled = enable; mpam_resctrl_controls[rid].cdp_enabled = enable; if (enable) l3->mon.num_rmid = resctrl_arch_system_num_rmid_idx() / 2; else l3->mon.num_rmid = resctrl_arch_system_num_rmid_idx(); /* The mbw_max feature can't hide cdp as it's a per-partid maximum. */ if (cdp_enabled && !mpam_resctrl_controls[RDT_RESOURCE_MBA].cdp_enabled) mpam_resctrl_controls[RDT_RESOURCE_MBA].resctrl_res.alloc_capable = false; /* * If resctrl has attempted to enable CDP on MBA, re-enable MBA as two * configurations will be provided so there is no aliasing problem. */ if (mpam_resctrl_controls[RDT_RESOURCE_MBA].cdp_enabled && mpam_resctrl_controls[RDT_RESOURCE_MBA].class) mpam_resctrl_controls[RDT_RESOURCE_MBA].resctrl_res.alloc_capable = true; /* On unmount when CDP is disabled, re-enable MBA */ if (!cdp_enabled && mpam_resctrl_controls[RDT_RESOURCE_MBA].class) mpam_resctrl_controls[RDT_RESOURCE_MBA].resctrl_res.alloc_capable = true; if (enable) { if (mpam_partid_max < 1) return -EINVAL; partid_d = resctrl_get_config_index(RESCTRL_RESERVED_CLOSID, CDP_DATA); partid_i = resctrl_get_config_index(RESCTRL_RESERVED_CLOSID, CDP_CODE); } mpam_set_task_partid_pmg(current, partid_d, partid_i, 0, 0); WRITE_ONCE(arm64_mpam_global_default, mpam_get_regval(current)); resctrl_reset_task_closids(); for_each_possible_cpu(cpu) mpam_set_cpu_defaults(cpu, partid_d, partid_i, 0, 0); on_each_cpu(resctrl_arch_sync_cpu_closid_rmid, NULL, 1); return 0;} static bool mpam_resctrl_hide_cdp(enum resctrl_res_level rid){ return cdp_enabled && !resctrl_arch_get_cdp_enabled(rid);} /* * MSC may raise an error interrupt if it sees an out or range partid/pmg, * and go on to truncate the value. Regardless of what the hardware supports, * only the system wide safe value is safe to use. */u32 resctrl_arch_get_num_closid(struct rdt_resource *ignored){ return mpam_partid_max + 1;} u32 resctrl_arch_system_num_rmid_idx(void){ return (mpam_pmg_max + 1) * (mpam_partid_max + 1);} u32 resctrl_arch_rmid_idx_encode(u32 closid, u32 rmid){ return closid * (mpam_pmg_max + 1) + rmid;} void resctrl_arch_rmid_idx_decode(u32 idx, u32 *closid, u32 *rmid){ *closid = idx / (mpam_pmg_max + 1); *rmid = idx % (mpam_pmg_max + 1);} void resctrl_arch_sched_in(struct task_struct *tsk){ lockdep_assert_preemption_disabled(); mpam_thread_switch(tsk);} void resctrl_arch_set_cpu_default_closid_rmid(int cpu, u32 closid, u32 rmid){ WARN_ON_ONCE(closid > U16_MAX); WARN_ON_ONCE(rmid > U8_MAX); if (!cdp_enabled) { mpam_set_cpu_defaults(cpu, closid, closid, rmid, rmid); } else { /* * When CDP is enabled, resctrl halves the closid range and we * use odd/even partid for one closid. */ u32 partid_d = resctrl_get_config_index(closid, CDP_DATA); u32 partid_i = resctrl_get_config_index(closid, CDP_CODE); mpam_set_cpu_defaults(cpu, partid_d, partid_i, rmid, rmid); }} void resctrl_arch_sync_cpu_closid_rmid(void *info){ struct resctrl_cpu_defaults *r = info; lockdep_assert_preemption_disabled(); if (r) { resctrl_arch_set_cpu_default_closid_rmid(smp_processor_id(), r->closid, r->rmid); } resctrl_arch_sched_in(current);} void resctrl_arch_set_closid_rmid(struct task_struct *tsk, u32 closid, u32 rmid){ WARN_ON_ONCE(closid > U16_MAX); WARN_ON_ONCE(rmid > U8_MAX); if (!cdp_enabled) { mpam_set_task_partid_pmg(tsk, closid, closid, rmid, rmid); } else { u32 partid_d = resctrl_get_config_index(closid, CDP_DATA); u32 partid_i = resctrl_get_config_index(closid, CDP_CODE); mpam_set_task_partid_pmg(tsk, partid_d, partid_i, rmid, rmid); }} bool resctrl_arch_match_closid(struct task_struct *tsk, u32 closid){ u64 regval = mpam_get_regval(tsk); u32 tsk_closid = FIELD_GET(MPAM0_EL1_PARTID_D, regval); if (cdp_enabled) tsk_closid >>= 1; return tsk_closid == closid;} /* The task's pmg is not unique, the partid must be considered too */bool resctrl_arch_match_rmid(struct task_struct *tsk, u32 closid, u32 rmid){ u64 regval = mpam_get_regval(tsk); u32 tsk_closid = FIELD_GET(MPAM0_EL1_PARTID_D, regval); u32 tsk_rmid = FIELD_GET(MPAM0_EL1_PMG_D, regval); if (cdp_enabled) tsk_closid >>= 1; return (tsk_closid == closid) && (tsk_rmid == rmid);} struct rdt_resource *resctrl_arch_get_resource(enum resctrl_res_level l){ if (l >= RDT_NUM_RESOURCES) return NULL; return &mpam_resctrl_controls[l].resctrl_res;} static int resctrl_arch_mon_ctx_alloc_no_wait(enum resctrl_event_id evtid){ struct mpam_resctrl_mon *mon = &mpam_resctrl_counters[evtid]; if (!mpam_is_enabled()) return -EINVAL; if (!mon->class) return -EINVAL; switch (evtid) { case QOS_L3_OCCUP_EVENT_ID: /* With CDP, one monitor gets used for both code/data reads */ return mpam_alloc_csu_mon(mon->class); case QOS_L3_MBM_LOCAL_EVENT_ID: case QOS_L3_MBM_TOTAL_EVENT_ID: return USE_PRE_ALLOCATED; default: return -EOPNOTSUPP; }} void *resctrl_arch_mon_ctx_alloc(struct rdt_resource *r, enum resctrl_event_id evtid){ DEFINE_WAIT(wait); int *ret; ret = kmalloc_obj(*ret); if (!ret) return ERR_PTR(-ENOMEM); do { prepare_to_wait(&resctrl_mon_ctx_waiters, &wait, TASK_INTERRUPTIBLE); *ret = resctrl_arch_mon_ctx_alloc_no_wait(evtid); if (*ret == -ENOSPC) schedule(); } while (*ret == -ENOSPC && !signal_pending(current)); finish_wait(&resctrl_mon_ctx_waiters, &wait); return ret;} static void resctrl_arch_mon_ctx_free_no_wait(enum resctrl_event_id evtid, u32 mon_idx){ struct mpam_resctrl_mon *mon = &mpam_resctrl_counters[evtid]; if (!mpam_is_enabled()) return; if (!mon->class) return; if (evtid == QOS_L3_OCCUP_EVENT_ID) mpam_free_csu_mon(mon->class, mon_idx); wake_up(&resctrl_mon_ctx_waiters);} void resctrl_arch_mon_ctx_free(struct rdt_resource *r, enum resctrl_event_id evtid, void *arch_mon_ctx){ u32 mon_idx = *(u32 *)arch_mon_ctx; kfree(arch_mon_ctx); resctrl_arch_mon_ctx_free_no_wait(evtid, mon_idx);} static int __read_mon(struct mpam_resctrl_mon *mon, struct mpam_component *mon_comp, enum mpam_device_features mon_type, int mon_idx, enum resctrl_conf_type cdp_type, u32 closid, u32 rmid, u64 *val){ struct mon_cfg cfg; if (!mpam_is_enabled()) return -EINVAL; /* Shift closid to account for CDP */ closid = resctrl_get_config_index(closid, cdp_type); if (irqs_disabled()) { /* Check if we can access this domain without an IPI */ return -EIO; } cfg = (struct mon_cfg) { .mon = mon_idx, .match_pmg = true, .partid = closid, .pmg = rmid, }; return mpam_msmon_read(mon_comp, &cfg, mon_type, val);} static int read_mon_cdp_safe(struct mpam_resctrl_mon *mon, struct mpam_component *mon_comp, enum mpam_device_features mon_type, int mon_idx, u32 closid, u32 rmid, u64 *val){ if (cdp_enabled) { u64 code_val = 0, data_val = 0; int err; err = __read_mon(mon, mon_comp, mon_type, mon_idx, CDP_CODE, closid, rmid, &code_val); if (err) return err; err = __read_mon(mon, mon_comp, mon_type, mon_idx, CDP_DATA, closid, rmid, &data_val); if (err) return err; *val += code_val + data_val; return 0; } return __read_mon(mon, mon_comp, mon_type, mon_idx, CDP_NONE, closid, rmid, val);} /* MBWU when not in ABMC mode (not supported), and CSU counters. */int resctrl_arch_rmid_read(struct rdt_resource *r, struct rdt_domain_hdr *hdr, u32 closid, u32 rmid, enum resctrl_event_id eventid,Lines 1–500 of 1712. Load more
// SPDX-License-Identifier: GPL-2.0 // Copyright (C) 2025 Arm Ltd. #define pr_fmt(fmt) "%s:%s: " fmt, KBUILD_MODNAME, __func__ #include <linux/arm_mpam.h> #include <linux/cacheinfo.h> #include <linux/cpu.h> #include <linux/cpumask.h> #include <linux/errno.h> #include <linux/limits.h> #include <linux/list.h> #include <linux/math.h> #include <linux/printk.h> #include <linux/rculist.h> #include <linux/resctrl.h> #include <linux/slab.h> #include <linux/types.h> #include <linux/wait.h> #include <asm/mpam.h> #include "mpam_internal.h" static DECLARE_WAIT_QUEUE_HEAD(resctrl_mon_ctx_waiters); /* * The classes we've picked to map to resctrl resources, wrapped * in with their resctrl structure. * Class pointer may be NULL. */ static struct mpam_resctrl_res mpam_resctrl_controls[RDT_NUM_RESOURCES]; #define for_each_mpam_resctrl_control(res, rid) \ for (rid = 0, res = &mpam_resctrl_controls[rid]; \ rid < RDT_NUM_RESOURCES; \ rid++, res = &mpam_resctrl_controls[rid]) /* * The classes we've picked to map to resctrl events. * Resctrl believes all the worlds a Xeon, and these are all on the L3. This * array lets us find the actual class backing the event counters. e.g. * the only memory bandwidth counters may be on the memory controller, but to * make use of them, we pretend they are on L3. Restrict the events considered * to those supported by MPAM. * Class pointer may be NULL. */ #define MPAM_MAX_EVENT QOS_L3_MBM_TOTAL_EVENT_ID static struct mpam_resctrl_mon mpam_resctrl_counters[MPAM_MAX_EVENT + 1]; #define for_each_mpam_resctrl_mon(mon, eventid) \ for (eventid = QOS_FIRST_EVENT, mon = &mpam_resctrl_counters[eventid]; \ eventid <= MPAM_MAX_EVENT; \ eventid++, mon = &mpam_resctrl_counters[eventid]) /* The lock for modifying resctrl's domain lists from cpuhp callbacks. */ static DEFINE_MUTEX(domain_list_lock); /* * MPAM emulates CDP by setting different PARTID in the I/D fields of MPAM0_EL1. * This applies globally to all traffic the CPU generates. */ static bool cdp_enabled; /* * We use cacheinfo to discover the size of the caches and their id. cacheinfo * populates this from a device_initcall(). mpam_resctrl_setup() must wait. */ static bool cacheinfo_ready; static DECLARE_WAIT_QUEUE_HEAD(wait_cacheinfo_ready); /* * If resctrl_init() succeeded, resctrl_exit() can be used to remove support * for the filesystem in the event of an error. */ static bool resctrl_enabled; bool resctrl_arch_alloc_capable(void) { struct mpam_resctrl_res *res; enum resctrl_res_level rid; for_each_mpam_resctrl_control(res, rid) { if (res->resctrl_res.alloc_capable) return true; } return false; } bool resctrl_arch_mon_capable(void) { struct mpam_resctrl_res *res = &mpam_resctrl_controls[RDT_RESOURCE_L3]; struct rdt_resource *l3 = &res->resctrl_res; /* All monitors are presented as being on the L3 cache */ return l3->mon_capable; } bool resctrl_arch_is_evt_configurable(enum resctrl_event_id evt) { return false; } void resctrl_arch_mon_event_config_read(void *info) { } void resctrl_arch_mon_event_config_write(void *info) { } void resctrl_arch_reset_rmid_all(struct rdt_resource *r, struct rdt_l3_mon_domain *d) { } void resctrl_arch_reset_rmid(struct rdt_resource *r, struct rdt_l3_mon_domain *d, u32 closid, u32 rmid, enum resctrl_event_id eventid) { } void resctrl_arch_reset_cntr(struct rdt_resource *r, struct rdt_l3_mon_domain *d, u32 closid, u32 rmid, int cntr_id, enum resctrl_event_id eventid) { } void resctrl_arch_config_cntr(struct rdt_resource *r, struct rdt_l3_mon_domain *d, enum resctrl_event_id evtid, u32 rmid, u32 closid, u32 cntr_id, bool assign) { } int resctrl_arch_cntr_read(struct rdt_resource *r, struct rdt_l3_mon_domain *d, u32 unused, u32 rmid, int cntr_id, enum resctrl_event_id eventid, u64 *val) { return -EOPNOTSUPP; } bool resctrl_arch_mbm_cntr_assign_enabled(struct rdt_resource *r) { return false; } int resctrl_arch_mbm_cntr_assign_set(struct rdt_resource *r, bool enable) { return -EINVAL; } int resctrl_arch_io_alloc_enable(struct rdt_resource *r, bool enable) { return -EOPNOTSUPP; } bool resctrl_arch_get_io_alloc_enabled(struct rdt_resource *r) { return false; } void resctrl_arch_pre_mount(void) { } bool resctrl_arch_get_cdp_enabled(enum resctrl_res_level rid) { return mpam_resctrl_controls[rid].cdp_enabled; } /** * resctrl_reset_task_closids() - Reset the PARTID/PMG values for all tasks. * * At boot, all existing tasks use partid zero for D and I. * To enable/disable CDP emulation, all these tasks need relabelling. */ static void resctrl_reset_task_closids(void) { struct task_struct *p, *t; read_lock(&tasklist_lock); for_each_process_thread(p, t) { resctrl_arch_set_closid_rmid(t, RESCTRL_RESERVED_CLOSID, RESCTRL_RESERVED_RMID); } read_unlock(&tasklist_lock); } int resctrl_arch_set_cdp_enabled(enum resctrl_res_level rid, bool enable) { u32 partid_i = RESCTRL_RESERVED_CLOSID, partid_d = RESCTRL_RESERVED_CLOSID; struct mpam_resctrl_res *res = &mpam_resctrl_controls[RDT_RESOURCE_L3]; struct rdt_resource *l3 = &res->resctrl_res; int cpu; if (!IS_ENABLED(CONFIG_EXPERT) && enable) { /* * If the resctrl fs is mounted more than once, sequentially, * then CDP can lead to the use of out of range PARTIDs. */ pr_warn("CDP not supported\n"); return -EOPNOTSUPP; } if (enable) pr_warn("CDP is an expert feature and may cause MPAM to malfunction.\n"); /* * resctrl_arch_set_cdp_enabled() is only called with enable set to * false on error and unmount. */ cdp_enabled = enable; mpam_resctrl_controls[rid].cdp_enabled = enable; if (enable) l3->mon.num_rmid = resctrl_arch_system_num_rmid_idx() / 2; else l3->mon.num_rmid = resctrl_arch_system_num_rmid_idx(); /* The mbw_max feature can't hide cdp as it's a per-partid maximum. */ if (cdp_enabled && !mpam_resctrl_controls[RDT_RESOURCE_MBA].cdp_enabled) mpam_resctrl_controls[RDT_RESOURCE_MBA].resctrl_res.alloc_capable = false; /* * If resctrl has attempted to enable CDP on MBA, re-enable MBA as two * configurations will be provided so there is no aliasing problem. */ if (mpam_resctrl_controls[RDT_RESOURCE_MBA].cdp_enabled && mpam_resctrl_controls[RDT_RESOURCE_MBA].class) mpam_resctrl_controls[RDT_RESOURCE_MBA].resctrl_res.alloc_capable = true; /* On unmount when CDP is disabled, re-enable MBA */ if (!cdp_enabled && mpam_resctrl_controls[RDT_RESOURCE_MBA].class) mpam_resctrl_controls[RDT_RESOURCE_MBA].resctrl_res.alloc_capable = true; if (enable) { if (mpam_partid_max < 1) return -EINVAL; partid_d = resctrl_get_config_index(RESCTRL_RESERVED_CLOSID, CDP_DATA); partid_i = resctrl_get_config_index(RESCTRL_RESERVED_CLOSID, CDP_CODE); } mpam_set_task_partid_pmg(current, partid_d, partid_i, 0, 0); WRITE_ONCE(arm64_mpam_global_default, mpam_get_regval(current)); resctrl_reset_task_closids(); for_each_possible_cpu(cpu) mpam_set_cpu_defaults(cpu, partid_d, partid_i, 0, 0); on_each_cpu(resctrl_arch_sync_cpu_closid_rmid, NULL, 1); return 0; } static bool mpam_resctrl_hide_cdp(enum resctrl_res_level rid) { return cdp_enabled && !resctrl_arch_get_cdp_enabled(rid); } /* * MSC may raise an error interrupt if it sees an out or range partid/pmg, * and go on to truncate the value. Regardless of what the hardware supports, * only the system wide safe value is safe to use. */ u32 resctrl_arch_get_num_closid(struct rdt_resource *ignored) { return mpam_partid_max + 1; } u32 resctrl_arch_system_num_rmid_idx(void) { return (mpam_pmg_max + 1) * (mpam_partid_max + 1); } u32 resctrl_arch_rmid_idx_encode(u32 closid, u32 rmid) { return closid * (mpam_pmg_max + 1) + rmid; } void resctrl_arch_rmid_idx_decode(u32 idx, u32 *closid, u32 *rmid) { *closid = idx / (mpam_pmg_max + 1); *rmid = idx % (mpam_pmg_max + 1); } void resctrl_arch_sched_in(struct task_struct *tsk) { lockdep_assert_preemption_disabled(); mpam_thread_switch(tsk); } void resctrl_arch_set_cpu_default_closid_rmid(int cpu, u32 closid, u32 rmid) { WARN_ON_ONCE(closid > U16_MAX); WARN_ON_ONCE(rmid > U8_MAX); if (!cdp_enabled) { mpam_set_cpu_defaults(cpu, closid, closid, rmid, rmid); } else { /* * When CDP is enabled, resctrl halves the closid range and we * use odd/even partid for one closid. */ u32 partid_d = resctrl_get_config_index(closid, CDP_DATA); u32 partid_i = resctrl_get_config_index(closid, CDP_CODE); mpam_set_cpu_defaults(cpu, partid_d, partid_i, rmid, rmid); } } void resctrl_arch_sync_cpu_closid_rmid(void *info) { struct resctrl_cpu_defaults *r = info; lockdep_assert_preemption_disabled(); if (r) { resctrl_arch_set_cpu_default_closid_rmid(smp_processor_id(), r->closid, r->rmid); } resctrl_arch_sched_in(current); } void resctrl_arch_set_closid_rmid(struct task_struct *tsk, u32 closid, u32 rmid) { WARN_ON_ONCE(closid > U16_MAX); WARN_ON_ONCE(rmid > U8_MAX); if (!cdp_enabled) { mpam_set_task_partid_pmg(tsk, closid, closid, rmid, rmid); } else { u32 partid_d = resctrl_get_config_index(closid, CDP_DATA); u32 partid_i = resctrl_get_config_index(closid, CDP_CODE); mpam_set_task_partid_pmg(tsk, partid_d, partid_i, rmid, rmid); } } bool resctrl_arch_match_closid(struct task_struct *tsk, u32 closid) { u64 regval = mpam_get_regval(tsk); u32 tsk_closid = FIELD_GET(MPAM0_EL1_PARTID_D, regval); if (cdp_enabled) tsk_closid >>= 1; return tsk_closid == closid; } /* The task's pmg is not unique, the partid must be considered too */ bool resctrl_arch_match_rmid(struct task_struct *tsk, u32 closid, u32 rmid) { u64 regval = mpam_get_regval(tsk); u32 tsk_closid = FIELD_GET(MPAM0_EL1_PARTID_D, regval); u32 tsk_rmid = FIELD_GET(MPAM0_EL1_PMG_D, regval); if (cdp_enabled) tsk_closid >>= 1; return (tsk_closid == closid) && (tsk_rmid == rmid); } struct rdt_resource *resctrl_arch_get_resource(enum resctrl_res_level l) { if (l >= RDT_NUM_RESOURCES) return NULL; return &mpam_resctrl_controls[l].resctrl_res; } static int resctrl_arch_mon_ctx_alloc_no_wait(enum resctrl_event_id evtid) { struct mpam_resctrl_mon *mon = &mpam_resctrl_counters[evtid]; if (!mpam_is_enabled()) return -EINVAL; if (!mon->class) return -EINVAL; switch (evtid) { case QOS_L3_OCCUP_EVENT_ID: /* With CDP, one monitor gets used for both code/data reads */ return mpam_alloc_csu_mon(mon->class); case QOS_L3_MBM_LOCAL_EVENT_ID: case QOS_L3_MBM_TOTAL_EVENT_ID: return USE_PRE_ALLOCATED; default: return -EOPNOTSUPP; } } void *resctrl_arch_mon_ctx_alloc(struct rdt_resource *r, enum resctrl_event_id evtid) { DEFINE_WAIT(wait); int *ret; ret = kmalloc_obj(*ret); if (!ret) return ERR_PTR(-ENOMEM); do { prepare_to_wait(&resctrl_mon_ctx_waiters, &wait, TASK_INTERRUPTIBLE); *ret = resctrl_arch_mon_ctx_alloc_no_wait(evtid); if (*ret == -ENOSPC) schedule(); } while (*ret == -ENOSPC && !signal_pending(current)); finish_wait(&resctrl_mon_ctx_waiters, &wait); return ret; } static void resctrl_arch_mon_ctx_free_no_wait(enum resctrl_event_id evtid, u32 mon_idx) { struct mpam_resctrl_mon *mon = &mpam_resctrl_counters[evtid]; if (!mpam_is_enabled()) return; if (!mon->class) return; if (evtid == QOS_L3_OCCUP_EVENT_ID) mpam_free_csu_mon(mon->class, mon_idx); wake_up(&resctrl_mon_ctx_waiters); } void resctrl_arch_mon_ctx_free(struct rdt_resource *r, enum resctrl_event_id evtid, void *arch_mon_ctx) { u32 mon_idx = *(u32 *)arch_mon_ctx; kfree(arch_mon_ctx); resctrl_arch_mon_ctx_free_no_wait(evtid, mon_idx); } static int __read_mon(struct mpam_resctrl_mon *mon, struct mpam_component *mon_comp, enum mpam_device_features mon_type, int mon_idx, enum resctrl_conf_type cdp_type, u32 closid, u32 rmid, u64 *val) { struct mon_cfg cfg; if (!mpam_is_enabled()) return -EINVAL; /* Shift closid to account for CDP */ closid = resctrl_get_config_index(closid, cdp_type); if (irqs_disabled()) { /* Check if we can access this domain without an IPI */ return -EIO; } cfg = (struct mon_cfg) { .mon = mon_idx, .match_pmg = true, .partid = closid, .pmg = rmid, }; return mpam_msmon_read(mon_comp, &cfg, mon_type, val); } static int read_mon_cdp_safe(struct mpam_resctrl_mon *mon, struct mpam_component *mon_comp, enum mpam_device_features mon_type, int mon_idx, u32 closid, u32 rmid, u64 *val) { if (cdp_enabled) { u64 code_val = 0, data_val = 0; int err; err = __read_mon(mon, mon_comp, mon_type, mon_idx, CDP_CODE, closid, rmid, &code_val); if (err) return err; err = __read_mon(mon, mon_comp, mon_type, mon_idx, CDP_DATA, closid, rmid, &data_val); if (err) return err; *val += code_val + data_val; return 0; } return __read_mon(mon, mon_comp, mon_type, mon_idx, CDP_NONE, closid, rmid, val); } /* MBWU when not in ABMC mode (not supported), and CSU counters. */ int resctrl_arch_rmid_read(struct rdt_resource *r, struct rdt_domain_hdr *hdr, u32 closid, u32 rmid, enum resctrl_event_id eventid,