// SPDX-License-Identifier: GPL-2.0 OR MIT/* * Copyright 2014-2022 Advanced Micro Devices, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. */ #include <linux/mutex.h>#include <linux/log2.h>#include <linux/sched.h>#include <linux/sched/mm.h>#include <linux/sched/task.h>#include <linux/mmu_context.h>#include <linux/slab.h>#include <linux/notifier.h>#include <linux/compat.h>#include <linux/mman.h>#include <linux/file.h>#include <linux/pm_runtime.h>#include <drm/ttm/ttm_bo.h>#include "amdgpu_amdkfd.h"#include "amdgpu.h"#include "amdgpu_reset.h" struct mm_struct; #include "kfd_priv.h"#include "kfd_device_queue_manager.h"#include "kfd_svm.h"#include "kfd_smi_events.h"#include "kfd_debug.h" /* * List of struct kfd_process (field kfd_process). * Unique/indexed by mm_struct* */DEFINE_HASHTABLE(kfd_processes_table, KFD_PROCESS_TABLE_SIZE);DEFINE_MUTEX(kfd_processes_mutex); DEFINE_SRCU(kfd_processes_srcu); /* For process termination handling */static struct workqueue_struct *kfd_process_wq; /* Ordered, single-threaded workqueue for restoring evicted * processes. Restoring multiple processes concurrently under memory * pressure can lead to processes blocking each other from validating * their BOs and result in a live-lock situation where processes * remain evicted indefinitely. */static struct workqueue_struct *kfd_restore_wq; static struct kfd_process *find_process(const struct task_struct *thread, bool ref);static void kfd_process_ref_release(struct kref *ref); static void evict_process_worker(struct work_struct *work);static void restore_process_worker(struct work_struct *work); static void kfd_process_device_destroy_cwsr_dgpu(struct kfd_process_device *pdd); struct kfd_procfs_tree { struct kobject *kobj;}; static struct kfd_procfs_tree procfs; /* * Structure for SDMA activity tracking */struct kfd_sdma_activity_handler_workarea { struct work_struct sdma_activity_work; struct kfd_process_device *pdd; uint64_t sdma_activity_counter;}; struct temp_sdma_queue_list { uint64_t __user *rptr; uint64_t sdma_val; unsigned int queue_id; struct list_head list;}; static void kfd_sdma_activity_worker(struct work_struct *work){ struct kfd_sdma_activity_handler_workarea *workarea; struct kfd_process_device *pdd; uint64_t val; struct mm_struct *mm; struct queue *q; struct qcm_process_device *qpd; struct device_queue_manager *dqm; int ret = 0; struct temp_sdma_queue_list sdma_q_list; struct temp_sdma_queue_list *sdma_q, *next; workarea = container_of(work, struct kfd_sdma_activity_handler_workarea, sdma_activity_work); pdd = workarea->pdd; if (!pdd) return; dqm = pdd->dev->dqm; qpd = &pdd->qpd; if (!dqm || !qpd) return; /* * Total SDMA activity is current SDMA activity + past SDMA activity * Past SDMA count is stored in pdd. * To get the current activity counters for all active SDMA queues, * we loop over all SDMA queues and get their counts from user-space. * * We cannot call get_user() with dqm_lock held as it can cause * a circular lock dependency situation. To read the SDMA stats, * we need to do the following: * * 1. Create a temporary list of SDMA queue nodes from the qpd->queues_list, * with dqm_lock/dqm_unlock(). * 2. Call get_user() for each node in temporary list without dqm_lock. * Save the SDMA count for each node and also add the count to the total * SDMA count counter. * Its possible, during this step, a few SDMA queue nodes got deleted * from the qpd->queues_list. * 3. Do a second pass over qpd->queues_list to check if any nodes got deleted. * If any node got deleted, its SDMA count would be captured in the sdma * past activity counter. So subtract the SDMA counter stored in step 2 * for this node from the total SDMA count. */ INIT_LIST_HEAD(&sdma_q_list.list); /* * Create the temp list of all SDMA queues */ dqm_lock(dqm); list_for_each_entry(q, &qpd->queues_list, list) { if ((q->properties.type != KFD_QUEUE_TYPE_SDMA) && (q->properties.type != KFD_QUEUE_TYPE_SDMA_XGMI)) continue; if (dqm->dev->kfd2kgd->hqd_sdma_get_counter) { val = 0; ret = dqm->dev->kfd2kgd->hqd_sdma_get_counter( dqm->dev->adev, q->mqd, dqm->dev->kfd->device_info.num_sdma_queues_per_engine, &val); if (ret) pr_debug("Failed to read SDMA queue active counter %i\n", ret); else workarea->sdma_activity_counter += val; continue; } sdma_q = kzalloc_obj(struct temp_sdma_queue_list); if (!sdma_q) { dqm_unlock(dqm); goto cleanup; } INIT_LIST_HEAD(&sdma_q->list); sdma_q->rptr = (uint64_t __user *)q->properties.read_ptr; sdma_q->queue_id = q->properties.queue_id; list_add_tail(&sdma_q->list, &sdma_q_list.list); } /* * If the temp list is empty, then no SDMA queues nodes were found in * qpd->queues_list. Return the past activity count as the total sdma * count */ if (list_empty(&sdma_q_list.list)) { workarea->sdma_activity_counter += pdd->sdma_past_activity_counter; dqm_unlock(dqm); return; } dqm_unlock(dqm); /* * Get the usage count for each SDMA queue in temp_list. */ mm = get_task_mm(pdd->process->lead_thread); if (!mm) goto cleanup; kthread_use_mm(mm); list_for_each_entry(sdma_q, &sdma_q_list.list, list) { val = 0; ret = read_sdma_queue_counter(sdma_q->rptr, &val); if (ret) { pr_debug("Failed to read SDMA queue active counter for queue id: %d", sdma_q->queue_id); } else { sdma_q->sdma_val = val; workarea->sdma_activity_counter += val; } } kthread_unuse_mm(mm); mmput(mm); /* * Do a second iteration over qpd_queues_list to check if any SDMA * nodes got deleted while fetching SDMA counter. */ dqm_lock(dqm); workarea->sdma_activity_counter += pdd->sdma_past_activity_counter; list_for_each_entry(q, &qpd->queues_list, list) { if (list_empty(&sdma_q_list.list)) break; if ((q->properties.type != KFD_QUEUE_TYPE_SDMA) && (q->properties.type != KFD_QUEUE_TYPE_SDMA_XGMI)) continue; list_for_each_entry_safe(sdma_q, next, &sdma_q_list.list, list) { if (((uint64_t __user *)q->properties.read_ptr == sdma_q->rptr) && (sdma_q->queue_id == q->properties.queue_id)) { list_del(&sdma_q->list); kfree(sdma_q); break; } } } dqm_unlock(dqm); /* * If temp list is not empty, it implies some queues got deleted * from qpd->queues_list during SDMA usage read. Subtract the SDMA * count for each node from the total SDMA count. */ list_for_each_entry_safe(sdma_q, next, &sdma_q_list.list, list) { workarea->sdma_activity_counter -= sdma_q->sdma_val; list_del(&sdma_q->list); kfree(sdma_q); } return; cleanup: list_for_each_entry_safe(sdma_q, next, &sdma_q_list.list, list) { list_del(&sdma_q->list); kfree(sdma_q); }} /** * kfd_get_cu_occupancy - Collect number of waves in-flight on this device * by current process. Translates acquired wave count into number of compute units * that are occupied. * * @attr: Handle of attribute that allows reporting of wave count. The attribute * handle encapsulates GPU device it is associated with, thereby allowing collection * of waves in flight, etc * @buffer: Handle of user provided buffer updated with wave count * * Return: Number of bytes written to user buffer or an error value */static int kfd_get_cu_occupancy(struct attribute *attr, char *buffer){ int cu_cnt; int wave_cnt; int max_waves_per_cu; struct kfd_node *dev = NULL; struct kfd_process *proc = NULL; struct kfd_process_device *pdd = NULL; int i; struct kfd_cu_occupancy *cu_occupancy; u32 queue_format; pdd = container_of(attr, struct kfd_process_device, attr_cu_occupancy); dev = pdd->dev; if (dev->kfd2kgd->get_cu_occupancy == NULL) return -EINVAL; cu_cnt = 0; proc = pdd->process; if (pdd->qpd.queue_count == 0) { pr_debug("Gpu-Id: %d has no active queues for process pid %d\n", dev->id, (int)proc->lead_thread->pid); return snprintf(buffer, PAGE_SIZE, "%d\n", cu_cnt); } /* Collect wave count from device if it supports */ wave_cnt = 0; max_waves_per_cu = 0; cu_occupancy = kzalloc_objs(*cu_occupancy, AMDGPU_MAX_QUEUES); if (!cu_occupancy) return -ENOMEM; /* * For GFX 9.4.3, fetch the CU occupancy from the first XCC in the partition. * For AQL queues, because of cooperative dispatch we multiply the wave count * by number of XCCs in the partition to get the total wave counts across all * XCCs in the partition. * For PM4 queues, there is no cooperative dispatch so wave_cnt stay as it is. */ dev->kfd2kgd->get_cu_occupancy(dev->adev, cu_occupancy, &max_waves_per_cu, ffs(dev->xcc_mask) - 1); for (i = 0; i < AMDGPU_MAX_QUEUES; i++) { if (cu_occupancy[i].wave_cnt != 0 && kfd_dqm_is_queue_in_process(dev->dqm, &pdd->qpd, cu_occupancy[i].doorbell_off, &queue_format)) { if (unlikely(queue_format == KFD_QUEUE_FORMAT_PM4)) wave_cnt += cu_occupancy[i].wave_cnt; else wave_cnt += (NUM_XCC(dev->xcc_mask) * cu_occupancy[i].wave_cnt); } } /* Translate wave count to number of compute units */ cu_cnt = (wave_cnt + (max_waves_per_cu - 1)) / max_waves_per_cu; kfree(cu_occupancy); return snprintf(buffer, PAGE_SIZE, "%d\n", cu_cnt);} static ssize_t kfd_procfs_show(struct kobject *kobj, struct attribute *attr, char *buffer){ if (strcmp(attr->name, "pasid") == 0) return snprintf(buffer, PAGE_SIZE, "%d\n", 0); else if (strncmp(attr->name, "vram_", 5) == 0) { struct kfd_process_device *pdd = container_of(attr, struct kfd_process_device, attr_vram); return snprintf(buffer, PAGE_SIZE, "%llu\n", atomic64_read(&pdd->vram_usage)); } else if (strncmp(attr->name, "sdma_", 5) == 0) { struct kfd_process_device *pdd = container_of(attr, struct kfd_process_device, attr_sdma); struct kfd_sdma_activity_handler_workarea sdma_activity_work_handler; INIT_WORK_ONSTACK(&sdma_activity_work_handler.sdma_activity_work, kfd_sdma_activity_worker); sdma_activity_work_handler.pdd = pdd; sdma_activity_work_handler.sdma_activity_counter = 0; schedule_work(&sdma_activity_work_handler.sdma_activity_work); flush_work(&sdma_activity_work_handler.sdma_activity_work); destroy_work_on_stack(&sdma_activity_work_handler.sdma_activity_work); return snprintf(buffer, PAGE_SIZE, "%llu\n", (sdma_activity_work_handler.sdma_activity_counter)/ SDMA_ACTIVITY_DIVISOR); } else { pr_err("Invalid attribute"); return -EINVAL; } return 0;} static void kfd_procfs_kobj_release(struct kobject *kobj){ kfree(kobj);} static const struct sysfs_ops kfd_procfs_ops = { .show = kfd_procfs_show,}; static const struct kobj_type procfs_type = { .release = kfd_procfs_kobj_release, .sysfs_ops = &kfd_procfs_ops,}; void kfd_procfs_init(void){ int ret = 0; procfs.kobj = kfd_alloc_struct(procfs.kobj); if (!procfs.kobj) return; ret = kobject_init_and_add(procfs.kobj, &procfs_type, &kfd_device->kobj, "proc"); if (ret) { pr_warn("Could not create procfs proc folder"); /* If we fail to create the procfs, clean up */ kfd_procfs_shutdown(); }} void kfd_procfs_shutdown(void){ if (procfs.kobj) { kobject_del(procfs.kobj); kobject_put(procfs.kobj); procfs.kobj = NULL; }} static ssize_t kfd_procfs_queue_show(struct kobject *kobj, struct attribute *attr, char *buffer){ struct queue *q = container_of(kobj, struct queue, kobj); if (!strcmp(attr->name, "size")) return snprintf(buffer, PAGE_SIZE, "%llu", q->properties.queue_size); else if (!strcmp(attr->name, "type")) return snprintf(buffer, PAGE_SIZE, "%d", q->properties.type); else if (!strcmp(attr->name, "gpuid")) return snprintf(buffer, PAGE_SIZE, "%u", q->device->id); else pr_err("Invalid attribute"); return 0;} static ssize_t kfd_procfs_stats_show(struct kobject *kobj, struct attribute *attr, char *buffer){ if (strcmp(attr->name, "evicted_ms") == 0) { struct kfd_process_device *pdd = container_of(attr, struct kfd_process_device, attr_evict); uint64_t evict_jiffies; evict_jiffies = atomic64_read(&pdd->evict_duration_counter); return snprintf(buffer, PAGE_SIZE, "%llu\n", jiffies64_to_msecs(evict_jiffies)); /* Sysfs handle that gets CU occupancy is per device */ } else if (strcmp(attr->name, "cu_occupancy") == 0) { return kfd_get_cu_occupancy(attr, buffer); } else { pr_err("Invalid attribute"); } return 0;} static ssize_t kfd_sysfs_counters_show(struct kobject *kobj, struct attribute *attr, char *buf){ struct kfd_process_device *pdd; if (!strcmp(attr->name, "faults")) { pdd = container_of(attr, struct kfd_process_device, attr_faults); return sysfs_emit(buf, "%llu\n", READ_ONCE(pdd->faults)); } if (!strcmp(attr->name, "page_in")) { pdd = container_of(attr, struct kfd_process_device, attr_page_in); return sysfs_emit(buf, "%llu\n", READ_ONCE(pdd->page_in)); } if (!strcmp(attr->name, "page_out")) { pdd = container_of(attr, struct kfd_process_device, attr_page_out); return sysfs_emit(buf, "%llu\n", READ_ONCE(pdd->page_out)); } return 0;} static struct attribute attr_queue_size = { .name = "size", .mode = KFD_SYSFS_FILE_MODE}; static struct attribute attr_queue_type = { .name = "type", .mode = KFD_SYSFS_FILE_MODE}; static struct attribute attr_queue_gpuid = { .name = "gpuid", .mode = KFD_SYSFS_FILE_MODE}; Lines 1–500 of 2486. Load more
// SPDX-License-Identifier: GPL-2.0 OR MIT /* * Copyright 2014-2022 Advanced Micro Devices, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. */ #include <linux/mutex.h> #include <linux/log2.h> #include <linux/sched.h> #include <linux/sched/mm.h> #include <linux/sched/task.h> #include <linux/mmu_context.h> #include <linux/slab.h> #include <linux/notifier.h> #include <linux/compat.h> #include <linux/mman.h> #include <linux/file.h> #include <linux/pm_runtime.h> #include <drm/ttm/ttm_bo.h> #include "amdgpu_amdkfd.h" #include "amdgpu.h" #include "amdgpu_reset.h" struct mm_struct; #include "kfd_priv.h" #include "kfd_device_queue_manager.h" #include "kfd_svm.h" #include "kfd_smi_events.h" #include "kfd_debug.h" /* * List of struct kfd_process (field kfd_process). * Unique/indexed by mm_struct* */ DEFINE_HASHTABLE(kfd_processes_table, KFD_PROCESS_TABLE_SIZE); DEFINE_MUTEX(kfd_processes_mutex); DEFINE_SRCU(kfd_processes_srcu); /* For process termination handling */ static struct workqueue_struct *kfd_process_wq; /* Ordered, single-threaded workqueue for restoring evicted * processes. Restoring multiple processes concurrently under memory * pressure can lead to processes blocking each other from validating * their BOs and result in a live-lock situation where processes * remain evicted indefinitely. */ static struct workqueue_struct *kfd_restore_wq; static struct kfd_process *find_process(const struct task_struct *thread, bool ref); static void kfd_process_ref_release(struct kref *ref); static void evict_process_worker(struct work_struct *work); static void restore_process_worker(struct work_struct *work); static void kfd_process_device_destroy_cwsr_dgpu(struct kfd_process_device *pdd); struct kfd_procfs_tree { struct kobject *kobj; }; static struct kfd_procfs_tree procfs; /* * Structure for SDMA activity tracking */ struct kfd_sdma_activity_handler_workarea { struct work_struct sdma_activity_work; struct kfd_process_device *pdd; uint64_t sdma_activity_counter; }; struct temp_sdma_queue_list { uint64_t __user *rptr; uint64_t sdma_val; unsigned int queue_id; struct list_head list; }; static void kfd_sdma_activity_worker(struct work_struct *work) { struct kfd_sdma_activity_handler_workarea *workarea; struct kfd_process_device *pdd; uint64_t val; struct mm_struct *mm; struct queue *q; struct qcm_process_device *qpd; struct device_queue_manager *dqm; int ret = 0; struct temp_sdma_queue_list sdma_q_list; struct temp_sdma_queue_list *sdma_q, *next; workarea = container_of(work, struct kfd_sdma_activity_handler_workarea, sdma_activity_work); pdd = workarea->pdd; if (!pdd) return; dqm = pdd->dev->dqm; qpd = &pdd->qpd; if (!dqm || !qpd) return; /* * Total SDMA activity is current SDMA activity + past SDMA activity * Past SDMA count is stored in pdd. * To get the current activity counters for all active SDMA queues, * we loop over all SDMA queues and get their counts from user-space. * * We cannot call get_user() with dqm_lock held as it can cause * a circular lock dependency situation. To read the SDMA stats, * we need to do the following: * * 1. Create a temporary list of SDMA queue nodes from the qpd->queues_list, * with dqm_lock/dqm_unlock(). * 2. Call get_user() for each node in temporary list without dqm_lock. * Save the SDMA count for each node and also add the count to the total * SDMA count counter. * Its possible, during this step, a few SDMA queue nodes got deleted * from the qpd->queues_list. * 3. Do a second pass over qpd->queues_list to check if any nodes got deleted. * If any node got deleted, its SDMA count would be captured in the sdma * past activity counter. So subtract the SDMA counter stored in step 2 * for this node from the total SDMA count. */ INIT_LIST_HEAD(&sdma_q_list.list); /* * Create the temp list of all SDMA queues */ dqm_lock(dqm); list_for_each_entry(q, &qpd->queues_list, list) { if ((q->properties.type != KFD_QUEUE_TYPE_SDMA) && (q->properties.type != KFD_QUEUE_TYPE_SDMA_XGMI)) continue; if (dqm->dev->kfd2kgd->hqd_sdma_get_counter) { val = 0; ret = dqm->dev->kfd2kgd->hqd_sdma_get_counter( dqm->dev->adev, q->mqd, dqm->dev->kfd->device_info.num_sdma_queues_per_engine, &val); if (ret) pr_debug("Failed to read SDMA queue active counter %i\n", ret); else workarea->sdma_activity_counter += val; continue; } sdma_q = kzalloc_obj(struct temp_sdma_queue_list); if (!sdma_q) { dqm_unlock(dqm); goto cleanup; } INIT_LIST_HEAD(&sdma_q->list); sdma_q->rptr = (uint64_t __user *)q->properties.read_ptr; sdma_q->queue_id = q->properties.queue_id; list_add_tail(&sdma_q->list, &sdma_q_list.list); } /* * If the temp list is empty, then no SDMA queues nodes were found in * qpd->queues_list. Return the past activity count as the total sdma * count */ if (list_empty(&sdma_q_list.list)) { workarea->sdma_activity_counter += pdd->sdma_past_activity_counter; dqm_unlock(dqm); return; } dqm_unlock(dqm); /* * Get the usage count for each SDMA queue in temp_list. */ mm = get_task_mm(pdd->process->lead_thread); if (!mm) goto cleanup; kthread_use_mm(mm); list_for_each_entry(sdma_q, &sdma_q_list.list, list) { val = 0; ret = read_sdma_queue_counter(sdma_q->rptr, &val); if (ret) { pr_debug("Failed to read SDMA queue active counter for queue id: %d", sdma_q->queue_id); } else { sdma_q->sdma_val = val; workarea->sdma_activity_counter += val; } } kthread_unuse_mm(mm); mmput(mm); /* * Do a second iteration over qpd_queues_list to check if any SDMA * nodes got deleted while fetching SDMA counter. */ dqm_lock(dqm); workarea->sdma_activity_counter += pdd->sdma_past_activity_counter; list_for_each_entry(q, &qpd->queues_list, list) { if (list_empty(&sdma_q_list.list)) break; if ((q->properties.type != KFD_QUEUE_TYPE_SDMA) && (q->properties.type != KFD_QUEUE_TYPE_SDMA_XGMI)) continue; list_for_each_entry_safe(sdma_q, next, &sdma_q_list.list, list) { if (((uint64_t __user *)q->properties.read_ptr == sdma_q->rptr) && (sdma_q->queue_id == q->properties.queue_id)) { list_del(&sdma_q->list); kfree(sdma_q); break; } } } dqm_unlock(dqm); /* * If temp list is not empty, it implies some queues got deleted * from qpd->queues_list during SDMA usage read. Subtract the SDMA * count for each node from the total SDMA count. */ list_for_each_entry_safe(sdma_q, next, &sdma_q_list.list, list) { workarea->sdma_activity_counter -= sdma_q->sdma_val; list_del(&sdma_q->list); kfree(sdma_q); } return; cleanup: list_for_each_entry_safe(sdma_q, next, &sdma_q_list.list, list) { list_del(&sdma_q->list); kfree(sdma_q); } } /** * kfd_get_cu_occupancy - Collect number of waves in-flight on this device * by current process. Translates acquired wave count into number of compute units * that are occupied. * * @attr: Handle of attribute that allows reporting of wave count. The attribute * handle encapsulates GPU device it is associated with, thereby allowing collection * of waves in flight, etc * @buffer: Handle of user provided buffer updated with wave count * * Return: Number of bytes written to user buffer or an error value */ static int kfd_get_cu_occupancy(struct attribute *attr, char *buffer) { int cu_cnt; int wave_cnt; int max_waves_per_cu; struct kfd_node *dev = NULL; struct kfd_process *proc = NULL; struct kfd_process_device *pdd = NULL; int i; struct kfd_cu_occupancy *cu_occupancy; u32 queue_format; pdd = container_of(attr, struct kfd_process_device, attr_cu_occupancy); dev = pdd->dev; if (dev->kfd2kgd->get_cu_occupancy == NULL) return -EINVAL; cu_cnt = 0; proc = pdd->process; if (pdd->qpd.queue_count == 0) { pr_debug("Gpu-Id: %d has no active queues for process pid %d\n", dev->id, (int)proc->lead_thread->pid); return snprintf(buffer, PAGE_SIZE, "%d\n", cu_cnt); } /* Collect wave count from device if it supports */ wave_cnt = 0; max_waves_per_cu = 0; cu_occupancy = kzalloc_objs(*cu_occupancy, AMDGPU_MAX_QUEUES); if (!cu_occupancy) return -ENOMEM; /* * For GFX 9.4.3, fetch the CU occupancy from the first XCC in the partition. * For AQL queues, because of cooperative dispatch we multiply the wave count * by number of XCCs in the partition to get the total wave counts across all * XCCs in the partition. * For PM4 queues, there is no cooperative dispatch so wave_cnt stay as it is. */ dev->kfd2kgd->get_cu_occupancy(dev->adev, cu_occupancy, &max_waves_per_cu, ffs(dev->xcc_mask) - 1); for (i = 0; i < AMDGPU_MAX_QUEUES; i++) { if (cu_occupancy[i].wave_cnt != 0 && kfd_dqm_is_queue_in_process(dev->dqm, &pdd->qpd, cu_occupancy[i].doorbell_off, &queue_format)) { if (unlikely(queue_format == KFD_QUEUE_FORMAT_PM4)) wave_cnt += cu_occupancy[i].wave_cnt; else wave_cnt += (NUM_XCC(dev->xcc_mask) * cu_occupancy[i].wave_cnt); } } /* Translate wave count to number of compute units */ cu_cnt = (wave_cnt + (max_waves_per_cu - 1)) / max_waves_per_cu; kfree(cu_occupancy); return snprintf(buffer, PAGE_SIZE, "%d\n", cu_cnt); } static ssize_t kfd_procfs_show(struct kobject *kobj, struct attribute *attr, char *buffer) { if (strcmp(attr->name, "pasid") == 0) return snprintf(buffer, PAGE_SIZE, "%d\n", 0); else if (strncmp(attr->name, "vram_", 5) == 0) { struct kfd_process_device *pdd = container_of(attr, struct kfd_process_device, attr_vram); return snprintf(buffer, PAGE_SIZE, "%llu\n", atomic64_read(&pdd->vram_usage)); } else if (strncmp(attr->name, "sdma_", 5) == 0) { struct kfd_process_device *pdd = container_of(attr, struct kfd_process_device, attr_sdma); struct kfd_sdma_activity_handler_workarea sdma_activity_work_handler; INIT_WORK_ONSTACK(&sdma_activity_work_handler.sdma_activity_work, kfd_sdma_activity_worker); sdma_activity_work_handler.pdd = pdd; sdma_activity_work_handler.sdma_activity_counter = 0; schedule_work(&sdma_activity_work_handler.sdma_activity_work); flush_work(&sdma_activity_work_handler.sdma_activity_work); destroy_work_on_stack(&sdma_activity_work_handler.sdma_activity_work); return snprintf(buffer, PAGE_SIZE, "%llu\n", (sdma_activity_work_handler.sdma_activity_counter)/ SDMA_ACTIVITY_DIVISOR); } else { pr_err("Invalid attribute"); return -EINVAL; } return 0; } static void kfd_procfs_kobj_release(struct kobject *kobj) { kfree(kobj); } static const struct sysfs_ops kfd_procfs_ops = { .show = kfd_procfs_show, }; static const struct kobj_type procfs_type = { .release = kfd_procfs_kobj_release, .sysfs_ops = &kfd_procfs_ops, }; void kfd_procfs_init(void) { int ret = 0; procfs.kobj = kfd_alloc_struct(procfs.kobj); if (!procfs.kobj) return; ret = kobject_init_and_add(procfs.kobj, &procfs_type, &kfd_device->kobj, "proc"); if (ret) { pr_warn("Could not create procfs proc folder"); /* If we fail to create the procfs, clean up */ kfd_procfs_shutdown(); } } void kfd_procfs_shutdown(void) { if (procfs.kobj) { kobject_del(procfs.kobj); kobject_put(procfs.kobj); procfs.kobj = NULL; } } static ssize_t kfd_procfs_queue_show(struct kobject *kobj, struct attribute *attr, char *buffer) { struct queue *q = container_of(kobj, struct queue, kobj); if (!strcmp(attr->name, "size")) return snprintf(buffer, PAGE_SIZE, "%llu", q->properties.queue_size); else if (!strcmp(attr->name, "type")) return snprintf(buffer, PAGE_SIZE, "%d", q->properties.type); else if (!strcmp(attr->name, "gpuid")) return snprintf(buffer, PAGE_SIZE, "%u", q->device->id); else pr_err("Invalid attribute"); return 0; } static ssize_t kfd_procfs_stats_show(struct kobject *kobj, struct attribute *attr, char *buffer) { if (strcmp(attr->name, "evicted_ms") == 0) { struct kfd_process_device *pdd = container_of(attr, struct kfd_process_device, attr_evict); uint64_t evict_jiffies; evict_jiffies = atomic64_read(&pdd->evict_duration_counter); return snprintf(buffer, PAGE_SIZE, "%llu\n", jiffies64_to_msecs(evict_jiffies)); /* Sysfs handle that gets CU occupancy is per device */ } else if (strcmp(attr->name, "cu_occupancy") == 0) { return kfd_get_cu_occupancy(attr, buffer); } else { pr_err("Invalid attribute"); } return 0; } static ssize_t kfd_sysfs_counters_show(struct kobject *kobj, struct attribute *attr, char *buf) { struct kfd_process_device *pdd; if (!strcmp(attr->name, "faults")) { pdd = container_of(attr, struct kfd_process_device, attr_faults); return sysfs_emit(buf, "%llu\n", READ_ONCE(pdd->faults)); } if (!strcmp(attr->name, "page_in")) { pdd = container_of(attr, struct kfd_process_device, attr_page_in); return sysfs_emit(buf, "%llu\n", READ_ONCE(pdd->page_in)); } if (!strcmp(attr->name, "page_out")) { pdd = container_of(attr, struct kfd_process_device, attr_page_out); return sysfs_emit(buf, "%llu\n", READ_ONCE(pdd->page_out)); } return 0; } static struct attribute attr_queue_size = { .name = "size", .mode = KFD_SYSFS_FILE_MODE }; static struct attribute attr_queue_type = { .name = "type", .mode = KFD_SYSFS_FILE_MODE }; static struct attribute attr_queue_gpuid = { .name = "gpuid", .mode = KFD_SYSFS_FILE_MODE };