/* * Copyright 2023 Advanced Micro Devices, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. */ #include "kfd_debug.h"#include "kfd_device_queue_manager.h"#include "kfd_topology.h"#include <linux/file.h>#include <uapi/linux/kfd_ioctl.h>#include <uapi/linux/kfd_sysfs.h> #define MAX_WATCH_ADDRESSES 4 int kfd_dbg_ev_query_debug_event(struct kfd_process *process, unsigned int *queue_id, unsigned int *gpu_id, uint64_t exception_clear_mask, uint64_t *event_status){ struct process_queue_manager *pqm; struct process_queue_node *pqn; int i; if (!(process && process->debug_trap_enabled)) return -ENODATA; mutex_lock(&process->event_mutex); *event_status = 0; *queue_id = 0; *gpu_id = 0; /* find and report queue events */ pqm = &process->pqm; list_for_each_entry(pqn, &pqm->queues, process_queue_list) { uint64_t tmp = process->exception_enable_mask; if (!pqn->q) continue; tmp &= pqn->q->properties.exception_status; if (!tmp) continue; *event_status = pqn->q->properties.exception_status; *queue_id = pqn->q->properties.queue_id; *gpu_id = pqn->q->device->id; pqn->q->properties.exception_status &= ~exception_clear_mask; goto out; } /* find and report device events */ for (i = 0; i < process->n_pdds; i++) { struct kfd_process_device *pdd = process->pdds[i]; uint64_t tmp = process->exception_enable_mask & pdd->exception_status; if (!tmp) continue; *event_status = pdd->exception_status; *gpu_id = pdd->dev->id; pdd->exception_status &= ~exception_clear_mask; goto out; } /* report process events */ if (process->exception_enable_mask & process->exception_status) { *event_status = process->exception_status; process->exception_status &= ~exception_clear_mask; } out: mutex_unlock(&process->event_mutex); return *event_status ? 0 : -EAGAIN;} void debug_event_write_work_handler(struct work_struct *work){ struct kfd_process *process; static const char write_data = '.'; loff_t pos = 0; process = container_of(work, struct kfd_process, debug_event_workarea); if (process->debug_trap_enabled && process->dbg_ev_file) kernel_write(process->dbg_ev_file, &write_data, 1, &pos);} /* update process/device/queue exception status, write to descriptor * only if exception_status is enabled. */bool kfd_dbg_ev_raise(uint64_t event_mask, struct kfd_process *process, struct kfd_node *dev, unsigned int source_id, bool use_worker, void *exception_data, size_t exception_data_size){ struct process_queue_manager *pqm; struct process_queue_node *pqn; int i; static const char write_data = '.'; loff_t pos = 0; bool is_subscribed = true; if (!(process && process->debug_trap_enabled)) return false; mutex_lock(&process->event_mutex); if (event_mask & KFD_EC_MASK_DEVICE) { for (i = 0; i < process->n_pdds; i++) { struct kfd_process_device *pdd = process->pdds[i]; if (pdd->dev != dev) continue; pdd->exception_status |= event_mask & KFD_EC_MASK_DEVICE; if (event_mask & KFD_EC_MASK(EC_DEVICE_MEMORY_VIOLATION)) { if (!pdd->vm_fault_exc_data) { pdd->vm_fault_exc_data = kmemdup( exception_data, exception_data_size, GFP_KERNEL); if (!pdd->vm_fault_exc_data) pr_debug("Failed to allocate exception data memory"); } else { pr_debug("Debugger exception data not saved\n"); print_hex_dump_bytes("exception data: ", DUMP_PREFIX_OFFSET, exception_data, exception_data_size); } } break; } } else if (event_mask & KFD_EC_MASK_PROCESS) { process->exception_status |= event_mask & KFD_EC_MASK_PROCESS; } else { pqm = &process->pqm; list_for_each_entry(pqn, &pqm->queues, process_queue_list) { int target_id; if (!pqn->q) continue; target_id = event_mask & KFD_EC_MASK(EC_QUEUE_NEW) ? pqn->q->properties.queue_id : pqn->q->doorbell_id; if (pqn->q->device != dev || target_id != source_id) continue; pqn->q->properties.exception_status |= event_mask; break; } } if (process->exception_enable_mask & event_mask) { if (use_worker) schedule_work(&process->debug_event_workarea); else kernel_write(process->dbg_ev_file, &write_data, 1, &pos); } else { is_subscribed = false; } mutex_unlock(&process->event_mutex); return is_subscribed;} /* set pending event queue entry from ring entry */bool kfd_set_dbg_ev_from_interrupt(struct kfd_node *dev, unsigned int pasid, uint32_t doorbell_id, uint64_t trap_mask, void *exception_data, size_t exception_data_size){ struct kfd_process *p; struct kfd_process_device *pdd = NULL; bool signaled_to_debugger_or_runtime = false; p = kfd_lookup_process_by_pasid(pasid, &pdd); if (!pdd) return false; if (!kfd_dbg_ev_raise(trap_mask, p, dev, doorbell_id, true, exception_data, exception_data_size)) { struct process_queue_manager *pqm; struct process_queue_node *pqn; if (!!(trap_mask & KFD_EC_MASK_QUEUE) && p->runtime_info.runtime_state == DEBUG_RUNTIME_STATE_ENABLED) { mutex_lock(&p->mutex); pqm = &p->pqm; list_for_each_entry(pqn, &pqm->queues, process_queue_list) { if (!(pqn->q && pqn->q->device == dev && pqn->q->doorbell_id == doorbell_id)) continue; kfd_send_exception_to_runtime(p, pqn->q->properties.queue_id, trap_mask); signaled_to_debugger_or_runtime = true; break; } mutex_unlock(&p->mutex); } else if (trap_mask & KFD_EC_MASK(EC_DEVICE_MEMORY_VIOLATION)) { kfd_evict_process_device(pdd); kfd_signal_vm_fault_event(pdd, NULL, exception_data); signaled_to_debugger_or_runtime = true; } } else { signaled_to_debugger_or_runtime = true; } kfd_unref_process(p); return signaled_to_debugger_or_runtime;} int kfd_dbg_send_exception_to_runtime(struct kfd_process *p, unsigned int dev_id, unsigned int queue_id, uint64_t error_reason){ if (error_reason & KFD_EC_MASK(EC_DEVICE_MEMORY_VIOLATION)) { struct kfd_process_device *pdd = NULL; struct kfd_hsa_memory_exception_data *data; int i; for (i = 0; i < p->n_pdds; i++) { if (p->pdds[i]->dev->id == dev_id) { pdd = p->pdds[i]; break; } } if (!pdd) return -ENODEV; data = (struct kfd_hsa_memory_exception_data *) pdd->vm_fault_exc_data; kfd_evict_process_device(pdd); kfd_signal_vm_fault_event(pdd, NULL, data); error_reason &= ~KFD_EC_MASK(EC_DEVICE_MEMORY_VIOLATION); } if (error_reason & (KFD_EC_MASK(EC_PROCESS_RUNTIME))) { /* * block should only happen after the debugger receives runtime * enable notice. */ up(&p->runtime_enable_sema); error_reason &= ~KFD_EC_MASK(EC_PROCESS_RUNTIME); } if (error_reason) return kfd_send_exception_to_runtime(p, queue_id, error_reason); return 0;} static int kfd_dbg_set_queue_workaround(struct queue *q, bool enable){ struct mqd_update_info minfo = {0}; int err; if (!q) return 0; if (!kfd_dbg_has_cwsr_workaround(q->device)) return 0; if (enable && q->properties.is_user_cu_masked) return -EBUSY; minfo.update_flag = enable ? UPDATE_FLAG_DBG_WA_ENABLE : UPDATE_FLAG_DBG_WA_DISABLE; q->properties.is_dbg_wa = enable; err = q->device->dqm->ops.update_queue(q->device->dqm, q, &minfo); if (err) q->properties.is_dbg_wa = false; return err;} static int kfd_dbg_set_workaround(struct kfd_process *target, bool enable){ struct process_queue_manager *pqm = &target->pqm; struct process_queue_node *pqn; int r = 0; list_for_each_entry(pqn, &pqm->queues, process_queue_list) { r = kfd_dbg_set_queue_workaround(pqn->q, enable); if (enable && r) goto unwind; } return 0; unwind: list_for_each_entry(pqn, &pqm->queues, process_queue_list) kfd_dbg_set_queue_workaround(pqn->q, false); if (enable) target->runtime_info.runtime_state = r == -EBUSY ? DEBUG_RUNTIME_STATE_ENABLED_BUSY : DEBUG_RUNTIME_STATE_ENABLED_ERROR; return r;} int kfd_dbg_set_mes_debug_mode(struct kfd_process_device *pdd, bool sq_trap_en){ uint32_t spi_dbg_cntl = pdd->spi_dbg_override | pdd->spi_dbg_launch_mode; uint32_t flags = pdd->process->dbg_flags; struct amdgpu_device *adev = pdd->dev->adev; int r; if (!kfd_dbg_is_per_vmid_supported(pdd->dev)) return 0; if (!pdd->proc_ctx_cpu_ptr) { r = amdgpu_amdkfd_alloc_kernel_mem(adev, AMDGPU_MES_PROC_CTX_SIZE, AMDGPU_GEM_DOMAIN_GTT, &pdd->proc_ctx_bo, &pdd->proc_ctx_gpu_addr, &pdd->proc_ctx_cpu_ptr, false); if (r) { dev_err(adev->dev, "failed to allocate process context bo\n"); return r; } memset(pdd->proc_ctx_cpu_ptr, 0, AMDGPU_MES_PROC_CTX_SIZE); } return amdgpu_mes_set_shader_debugger(pdd->dev->adev, pdd->proc_ctx_gpu_addr, spi_dbg_cntl, pdd->watch_points, flags, sq_trap_en, ffs(pdd->dev->xcc_mask) - 1);} #define KFD_DEBUGGER_INVALID_WATCH_POINT_ID -1static int kfd_dbg_get_dev_watch_id(struct kfd_process_device *pdd, int *watch_id){ int i; *watch_id = KFD_DEBUGGER_INVALID_WATCH_POINT_ID; spin_lock(&pdd->dev->watch_points_lock); for (i = 0; i < MAX_WATCH_ADDRESSES; i++) { /* device watchpoint in use so skip */ if ((pdd->dev->alloc_watch_ids >> i) & 0x1) continue; pdd->alloc_watch_ids |= 0x1 << i; pdd->dev->alloc_watch_ids |= 0x1 << i; *watch_id = i; spin_unlock(&pdd->dev->watch_points_lock); return 0; } spin_unlock(&pdd->dev->watch_points_lock); return -ENOMEM;} static void kfd_dbg_clear_dev_watch_id(struct kfd_process_device *pdd, u32 watch_id){ spin_lock(&pdd->dev->watch_points_lock); /* process owns device watch point so safe to clear */ if (pdd->alloc_watch_ids & BIT(watch_id)) { pdd->alloc_watch_ids &= ~BIT(watch_id); pdd->dev->alloc_watch_ids &= ~BIT(watch_id); } spin_unlock(&pdd->dev->watch_points_lock);} static bool kfd_dbg_owns_dev_watch_id(struct kfd_process_device *pdd, u32 watch_id){ bool owns_watch_id = false; spin_lock(&pdd->dev->watch_points_lock); owns_watch_id = pdd->alloc_watch_ids & BIT(watch_id); spin_unlock(&pdd->dev->watch_points_lock); return owns_watch_id;} int kfd_dbg_trap_clear_dev_address_watch(struct kfd_process_device *pdd, uint32_t watch_id){ int r; if (watch_id >= MAX_WATCH_ADDRESSES) return -EINVAL; if (!kfd_dbg_owns_dev_watch_id(pdd, watch_id)) return -EINVAL; if (!pdd->dev->kfd->shared_resources.enable_mes) { r = debug_lock_and_unmap(pdd->dev->dqm); if (r) return r; } amdgpu_gfx_off_ctrl(pdd->dev->adev, false); pdd->watch_points[watch_id] = pdd->dev->kfd2kgd->clear_address_watch( pdd->dev->adev, watch_id); amdgpu_gfx_off_ctrl(pdd->dev->adev, true); if (!pdd->dev->kfd->shared_resources.enable_mes) r = debug_map_and_unlock(pdd->dev->dqm); else r = kfd_dbg_set_mes_debug_mode(pdd, true); kfd_dbg_clear_dev_watch_id(pdd, watch_id); return r;} int kfd_dbg_trap_set_dev_address_watch(struct kfd_process_device *pdd, uint64_t watch_address, uint32_t watch_address_mask, uint32_t *watch_id, uint32_t watch_mode){ int xcc_id, r = kfd_dbg_get_dev_watch_id(pdd, watch_id); uint32_t xcc_mask = pdd->dev->xcc_mask; if (r) return r; if (*watch_id >= MAX_WATCH_ADDRESSES) return -EINVAL; if (!pdd->dev->kfd->shared_resources.enable_mes) { r = debug_lock_and_unmap(pdd->dev->dqm); if (r) { kfd_dbg_clear_dev_watch_id(pdd, *watch_id); return r; } } amdgpu_gfx_off_ctrl(pdd->dev->adev, false); for_each_inst(xcc_id, xcc_mask) pdd->watch_points[*watch_id] = pdd->dev->kfd2kgd->set_address_watch( pdd->dev->adev, watch_address, watch_address_mask, *watch_id, watch_mode, pdd->dev->vm_info.last_vmid_kfd, xcc_id); amdgpu_gfx_off_ctrl(pdd->dev->adev, true); if (!pdd->dev->kfd->shared_resources.enable_mes) r = debug_map_and_unlock(pdd->dev->dqm);Lines 1–500 of 1183. Load more
/* * Copyright 2023 Advanced Micro Devices, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. */ #include "kfd_debug.h" #include "kfd_device_queue_manager.h" #include "kfd_topology.h" #include <linux/file.h> #include <uapi/linux/kfd_ioctl.h> #include <uapi/linux/kfd_sysfs.h> #define MAX_WATCH_ADDRESSES 4 int kfd_dbg_ev_query_debug_event(struct kfd_process *process, unsigned int *queue_id, unsigned int *gpu_id, uint64_t exception_clear_mask, uint64_t *event_status) { struct process_queue_manager *pqm; struct process_queue_node *pqn; int i; if (!(process && process->debug_trap_enabled)) return -ENODATA; mutex_lock(&process->event_mutex); *event_status = 0; *queue_id = 0; *gpu_id = 0; /* find and report queue events */ pqm = &process->pqm; list_for_each_entry(pqn, &pqm->queues, process_queue_list) { uint64_t tmp = process->exception_enable_mask; if (!pqn->q) continue; tmp &= pqn->q->properties.exception_status; if (!tmp) continue; *event_status = pqn->q->properties.exception_status; *queue_id = pqn->q->properties.queue_id; *gpu_id = pqn->q->device->id; pqn->q->properties.exception_status &= ~exception_clear_mask; goto out; } /* find and report device events */ for (i = 0; i < process->n_pdds; i++) { struct kfd_process_device *pdd = process->pdds[i]; uint64_t tmp = process->exception_enable_mask & pdd->exception_status; if (!tmp) continue; *event_status = pdd->exception_status; *gpu_id = pdd->dev->id; pdd->exception_status &= ~exception_clear_mask; goto out; } /* report process events */ if (process->exception_enable_mask & process->exception_status) { *event_status = process->exception_status; process->exception_status &= ~exception_clear_mask; } out: mutex_unlock(&process->event_mutex); return *event_status ? 0 : -EAGAIN; } void debug_event_write_work_handler(struct work_struct *work) { struct kfd_process *process; static const char write_data = '.'; loff_t pos = 0; process = container_of(work, struct kfd_process, debug_event_workarea); if (process->debug_trap_enabled && process->dbg_ev_file) kernel_write(process->dbg_ev_file, &write_data, 1, &pos); } /* update process/device/queue exception status, write to descriptor * only if exception_status is enabled. */ bool kfd_dbg_ev_raise(uint64_t event_mask, struct kfd_process *process, struct kfd_node *dev, unsigned int source_id, bool use_worker, void *exception_data, size_t exception_data_size) { struct process_queue_manager *pqm; struct process_queue_node *pqn; int i; static const char write_data = '.'; loff_t pos = 0; bool is_subscribed = true; if (!(process && process->debug_trap_enabled)) return false; mutex_lock(&process->event_mutex); if (event_mask & KFD_EC_MASK_DEVICE) { for (i = 0; i < process->n_pdds; i++) { struct kfd_process_device *pdd = process->pdds[i]; if (pdd->dev != dev) continue; pdd->exception_status |= event_mask & KFD_EC_MASK_DEVICE; if (event_mask & KFD_EC_MASK(EC_DEVICE_MEMORY_VIOLATION)) { if (!pdd->vm_fault_exc_data) { pdd->vm_fault_exc_data = kmemdup( exception_data, exception_data_size, GFP_KERNEL); if (!pdd->vm_fault_exc_data) pr_debug("Failed to allocate exception data memory"); } else { pr_debug("Debugger exception data not saved\n"); print_hex_dump_bytes("exception data: ", DUMP_PREFIX_OFFSET, exception_data, exception_data_size); } } break; } } else if (event_mask & KFD_EC_MASK_PROCESS) { process->exception_status |= event_mask & KFD_EC_MASK_PROCESS; } else { pqm = &process->pqm; list_for_each_entry(pqn, &pqm->queues, process_queue_list) { int target_id; if (!pqn->q) continue; target_id = event_mask & KFD_EC_MASK(EC_QUEUE_NEW) ? pqn->q->properties.queue_id : pqn->q->doorbell_id; if (pqn->q->device != dev || target_id != source_id) continue; pqn->q->properties.exception_status |= event_mask; break; } } if (process->exception_enable_mask & event_mask) { if (use_worker) schedule_work(&process->debug_event_workarea); else kernel_write(process->dbg_ev_file, &write_data, 1, &pos); } else { is_subscribed = false; } mutex_unlock(&process->event_mutex); return is_subscribed; } /* set pending event queue entry from ring entry */ bool kfd_set_dbg_ev_from_interrupt(struct kfd_node *dev, unsigned int pasid, uint32_t doorbell_id, uint64_t trap_mask, void *exception_data, size_t exception_data_size) { struct kfd_process *p; struct kfd_process_device *pdd = NULL; bool signaled_to_debugger_or_runtime = false; p = kfd_lookup_process_by_pasid(pasid, &pdd); if (!pdd) return false; if (!kfd_dbg_ev_raise(trap_mask, p, dev, doorbell_id, true, exception_data, exception_data_size)) { struct process_queue_manager *pqm; struct process_queue_node *pqn; if (!!(trap_mask & KFD_EC_MASK_QUEUE) && p->runtime_info.runtime_state == DEBUG_RUNTIME_STATE_ENABLED) { mutex_lock(&p->mutex); pqm = &p->pqm; list_for_each_entry(pqn, &pqm->queues, process_queue_list) { if (!(pqn->q && pqn->q->device == dev && pqn->q->doorbell_id == doorbell_id)) continue; kfd_send_exception_to_runtime(p, pqn->q->properties.queue_id, trap_mask); signaled_to_debugger_or_runtime = true; break; } mutex_unlock(&p->mutex); } else if (trap_mask & KFD_EC_MASK(EC_DEVICE_MEMORY_VIOLATION)) { kfd_evict_process_device(pdd); kfd_signal_vm_fault_event(pdd, NULL, exception_data); signaled_to_debugger_or_runtime = true; } } else { signaled_to_debugger_or_runtime = true; } kfd_unref_process(p); return signaled_to_debugger_or_runtime; } int kfd_dbg_send_exception_to_runtime(struct kfd_process *p, unsigned int dev_id, unsigned int queue_id, uint64_t error_reason) { if (error_reason & KFD_EC_MASK(EC_DEVICE_MEMORY_VIOLATION)) { struct kfd_process_device *pdd = NULL; struct kfd_hsa_memory_exception_data *data; int i; for (i = 0; i < p->n_pdds; i++) { if (p->pdds[i]->dev->id == dev_id) { pdd = p->pdds[i]; break; } } if (!pdd) return -ENODEV; data = (struct kfd_hsa_memory_exception_data *) pdd->vm_fault_exc_data; kfd_evict_process_device(pdd); kfd_signal_vm_fault_event(pdd, NULL, data); error_reason &= ~KFD_EC_MASK(EC_DEVICE_MEMORY_VIOLATION); } if (error_reason & (KFD_EC_MASK(EC_PROCESS_RUNTIME))) { /* * block should only happen after the debugger receives runtime * enable notice. */ up(&p->runtime_enable_sema); error_reason &= ~KFD_EC_MASK(EC_PROCESS_RUNTIME); } if (error_reason) return kfd_send_exception_to_runtime(p, queue_id, error_reason); return 0; } static int kfd_dbg_set_queue_workaround(struct queue *q, bool enable) { struct mqd_update_info minfo = {0}; int err; if (!q) return 0; if (!kfd_dbg_has_cwsr_workaround(q->device)) return 0; if (enable && q->properties.is_user_cu_masked) return -EBUSY; minfo.update_flag = enable ? UPDATE_FLAG_DBG_WA_ENABLE : UPDATE_FLAG_DBG_WA_DISABLE; q->properties.is_dbg_wa = enable; err = q->device->dqm->ops.update_queue(q->device->dqm, q, &minfo); if (err) q->properties.is_dbg_wa = false; return err; } static int kfd_dbg_set_workaround(struct kfd_process *target, bool enable) { struct process_queue_manager *pqm = &target->pqm; struct process_queue_node *pqn; int r = 0; list_for_each_entry(pqn, &pqm->queues, process_queue_list) { r = kfd_dbg_set_queue_workaround(pqn->q, enable); if (enable && r) goto unwind; } return 0; unwind: list_for_each_entry(pqn, &pqm->queues, process_queue_list) kfd_dbg_set_queue_workaround(pqn->q, false); if (enable) target->runtime_info.runtime_state = r == -EBUSY ? DEBUG_RUNTIME_STATE_ENABLED_BUSY : DEBUG_RUNTIME_STATE_ENABLED_ERROR; return r; } int kfd_dbg_set_mes_debug_mode(struct kfd_process_device *pdd, bool sq_trap_en) { uint32_t spi_dbg_cntl = pdd->spi_dbg_override | pdd->spi_dbg_launch_mode; uint32_t flags = pdd->process->dbg_flags; struct amdgpu_device *adev = pdd->dev->adev; int r; if (!kfd_dbg_is_per_vmid_supported(pdd->dev)) return 0; if (!pdd->proc_ctx_cpu_ptr) { r = amdgpu_amdkfd_alloc_kernel_mem(adev, AMDGPU_MES_PROC_CTX_SIZE, AMDGPU_GEM_DOMAIN_GTT, &pdd->proc_ctx_bo, &pdd->proc_ctx_gpu_addr, &pdd->proc_ctx_cpu_ptr, false); if (r) { dev_err(adev->dev, "failed to allocate process context bo\n"); return r; } memset(pdd->proc_ctx_cpu_ptr, 0, AMDGPU_MES_PROC_CTX_SIZE); } return amdgpu_mes_set_shader_debugger(pdd->dev->adev, pdd->proc_ctx_gpu_addr, spi_dbg_cntl, pdd->watch_points, flags, sq_trap_en, ffs(pdd->dev->xcc_mask) - 1); } #define KFD_DEBUGGER_INVALID_WATCH_POINT_ID -1 static int kfd_dbg_get_dev_watch_id(struct kfd_process_device *pdd, int *watch_id) { int i; *watch_id = KFD_DEBUGGER_INVALID_WATCH_POINT_ID; spin_lock(&pdd->dev->watch_points_lock); for (i = 0; i < MAX_WATCH_ADDRESSES; i++) { /* device watchpoint in use so skip */ if ((pdd->dev->alloc_watch_ids >> i) & 0x1) continue; pdd->alloc_watch_ids |= 0x1 << i; pdd->dev->alloc_watch_ids |= 0x1 << i; *watch_id = i; spin_unlock(&pdd->dev->watch_points_lock); return 0; } spin_unlock(&pdd->dev->watch_points_lock); return -ENOMEM; } static void kfd_dbg_clear_dev_watch_id(struct kfd_process_device *pdd, u32 watch_id) { spin_lock(&pdd->dev->watch_points_lock); /* process owns device watch point so safe to clear */ if (pdd->alloc_watch_ids & BIT(watch_id)) { pdd->alloc_watch_ids &= ~BIT(watch_id); pdd->dev->alloc_watch_ids &= ~BIT(watch_id); } spin_unlock(&pdd->dev->watch_points_lock); } static bool kfd_dbg_owns_dev_watch_id(struct kfd_process_device *pdd, u32 watch_id) { bool owns_watch_id = false; spin_lock(&pdd->dev->watch_points_lock); owns_watch_id = pdd->alloc_watch_ids & BIT(watch_id); spin_unlock(&pdd->dev->watch_points_lock); return owns_watch_id; } int kfd_dbg_trap_clear_dev_address_watch(struct kfd_process_device *pdd, uint32_t watch_id) { int r; if (watch_id >= MAX_WATCH_ADDRESSES) return -EINVAL; if (!kfd_dbg_owns_dev_watch_id(pdd, watch_id)) return -EINVAL; if (!pdd->dev->kfd->shared_resources.enable_mes) { r = debug_lock_and_unmap(pdd->dev->dqm); if (r) return r; } amdgpu_gfx_off_ctrl(pdd->dev->adev, false); pdd->watch_points[watch_id] = pdd->dev->kfd2kgd->clear_address_watch( pdd->dev->adev, watch_id); amdgpu_gfx_off_ctrl(pdd->dev->adev, true); if (!pdd->dev->kfd->shared_resources.enable_mes) r = debug_map_and_unlock(pdd->dev->dqm); else r = kfd_dbg_set_mes_debug_mode(pdd, true); kfd_dbg_clear_dev_watch_id(pdd, watch_id); return r; } int kfd_dbg_trap_set_dev_address_watch(struct kfd_process_device *pdd, uint64_t watch_address, uint32_t watch_address_mask, uint32_t *watch_id, uint32_t watch_mode) { int xcc_id, r = kfd_dbg_get_dev_watch_id(pdd, watch_id); uint32_t xcc_mask = pdd->dev->xcc_mask; if (r) return r; if (*watch_id >= MAX_WATCH_ADDRESSES) return -EINVAL; if (!pdd->dev->kfd->shared_resources.enable_mes) { r = debug_lock_and_unmap(pdd->dev->dqm); if (r) { kfd_dbg_clear_dev_watch_id(pdd, *watch_id); return r; } } amdgpu_gfx_off_ctrl(pdd->dev->adev, false); for_each_inst(xcc_id, xcc_mask) pdd->watch_points[*watch_id] = pdd->dev->kfd2kgd->set_address_watch( pdd->dev->adev, watch_address, watch_address_mask, *watch_id, watch_mode, pdd->dev->vm_info.last_vmid_kfd, xcc_id); amdgpu_gfx_off_ctrl(pdd->dev->adev, true); if (!pdd->dev->kfd->shared_resources.enable_mes) r = debug_map_and_unlock(pdd->dev->dqm);