// SPDX-License-Identifier: MIT/* * Copyright 2023 Advanced Micro Devices, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. * */ #include <drm/drm_auth.h>#include <drm/drm_exec.h>#include <linux/pm_runtime.h>#include <drm/drm_drv.h> #include "amdgpu.h"#include "amdgpu_reset.h"#include "amdgpu_vm.h"#include "amdgpu_userq.h"#include "amdgpu_hmm.h"#include "amdgpu_userq_fence.h" u32 amdgpu_userq_get_supported_ip_mask(struct amdgpu_device *adev){ int i; u32 userq_ip_mask = 0; for (i = 0; i < AMDGPU_HW_IP_NUM; i++) { if (adev->userq_funcs[i]) userq_ip_mask |= (1 << i); } return userq_ip_mask;} static bool amdgpu_userq_is_reset_type_supported(struct amdgpu_device *adev, enum amdgpu_ring_type ring_type, int reset_type){ if (ring_type < 0 || ring_type >= AMDGPU_RING_TYPE_MAX) return false; switch (ring_type) { case AMDGPU_RING_TYPE_GFX: if (adev->gfx.gfx_supported_reset & reset_type) return true; break; case AMDGPU_RING_TYPE_COMPUTE: if (adev->gfx.compute_supported_reset & reset_type) return true; break; case AMDGPU_RING_TYPE_SDMA: if (adev->sdma.supported_reset & reset_type) return true; break; case AMDGPU_RING_TYPE_VCN_DEC: case AMDGPU_RING_TYPE_VCN_ENC: if (adev->vcn.supported_reset & reset_type) return true; break; case AMDGPU_RING_TYPE_VCN_JPEG: if (adev->jpeg.supported_reset & reset_type) return true; break; default: break; } return false;} static void amdgpu_userq_gpu_reset(struct amdgpu_device *adev){ if (amdgpu_device_should_recover_gpu(adev)) { amdgpu_reset_domain_schedule(adev->reset_domain, &adev->userq_reset_work); /* Wait for the reset job to complete */ flush_work(&adev->userq_reset_work); }} static intamdgpu_userq_detect_and_reset_queues(struct amdgpu_userq_mgr *uq_mgr){ struct amdgpu_device *adev = uq_mgr->adev; const int queue_types[] = { AMDGPU_RING_TYPE_COMPUTE, AMDGPU_RING_TYPE_GFX, AMDGPU_RING_TYPE_SDMA }; const int num_queue_types = ARRAY_SIZE(queue_types); bool gpu_reset = false; int r = 0; int i; /* Warning if current process mutex is not held */ WARN_ON(!mutex_is_locked(&uq_mgr->userq_mutex)); if (unlikely(adev->debug_disable_gpu_ring_reset)) { dev_err(adev->dev, "userq reset disabled by debug mask\n"); return 0; } /* * If GPU recovery feature is disabled system-wide, * skip all reset detection logic */ if (!amdgpu_gpu_recovery) return 0; /* * Iterate through all queue types to detect and reset problematic queues * Process each queue type in the defined order */ for (i = 0; i < num_queue_types; i++) { int ring_type = queue_types[i]; const struct amdgpu_userq_funcs *funcs = adev->userq_funcs[ring_type]; if (!amdgpu_userq_is_reset_type_supported(adev, ring_type, AMDGPU_RESET_TYPE_PER_QUEUE)) continue; if (atomic_read(&uq_mgr->userq_count[ring_type]) > 0 && funcs && funcs->detect_and_reset) { r = funcs->detect_and_reset(adev, ring_type); if (r) { gpu_reset = true; break; } } } if (gpu_reset) amdgpu_userq_gpu_reset(adev); return r;} static void amdgpu_userq_hang_detect_work(struct work_struct *work){ struct amdgpu_usermode_queue *queue = container_of(work, struct amdgpu_usermode_queue, hang_detect_work.work); struct dma_fence *fence; struct amdgpu_userq_mgr *uq_mgr; if (!queue || !queue->userq_mgr) return; uq_mgr = queue->userq_mgr; fence = READ_ONCE(queue->hang_detect_fence); /* Fence already signaled – no action needed */ if (!fence || dma_fence_is_signaled(fence)) return; mutex_lock(&uq_mgr->userq_mutex); amdgpu_userq_detect_and_reset_queues(uq_mgr); mutex_unlock(&uq_mgr->userq_mutex);} /* * Start hang detection for a user queue fence. A delayed work will be scheduled * to check if the fence is still pending after the timeout period.*/void amdgpu_userq_start_hang_detect_work(struct amdgpu_usermode_queue *queue){ struct amdgpu_device *adev; unsigned long timeout_ms; if (!queue || !queue->userq_mgr || !queue->userq_mgr->adev) return; adev = queue->userq_mgr->adev; /* Determine timeout based on queue type */ switch (queue->queue_type) { case AMDGPU_RING_TYPE_GFX: timeout_ms = adev->gfx_timeout; break; case AMDGPU_RING_TYPE_COMPUTE: timeout_ms = adev->compute_timeout; break; case AMDGPU_RING_TYPE_SDMA: timeout_ms = adev->sdma_timeout; break; default: timeout_ms = adev->gfx_timeout; break; } /* Store the fence to monitor and schedule hang detection */ WRITE_ONCE(queue->hang_detect_fence, queue->last_fence); schedule_delayed_work(&queue->hang_detect_work, msecs_to_jiffies(timeout_ms));} static void amdgpu_userq_init_hang_detect_work(struct amdgpu_usermode_queue *queue){ INIT_DELAYED_WORK(&queue->hang_detect_work, amdgpu_userq_hang_detect_work); queue->hang_detect_fence = NULL;} static int amdgpu_userq_buffer_va_list_add(struct amdgpu_usermode_queue *queue, struct amdgpu_bo_va_mapping *va_map, u64 addr){ struct amdgpu_userq_va_cursor *va_cursor; struct userq_va_list; va_cursor = kzalloc_obj(*va_cursor); if (!va_cursor) return -ENOMEM; INIT_LIST_HEAD(&va_cursor->list); va_cursor->gpu_addr = addr; atomic_set(&va_map->bo_va->userq_va_mapped, 1); list_add(&va_cursor->list, &queue->userq_va_list); return 0;} int amdgpu_userq_input_va_validate(struct amdgpu_device *adev, struct amdgpu_usermode_queue *queue, u64 addr, u64 expected_size){ struct amdgpu_bo_va_mapping *va_map; struct amdgpu_vm *vm = queue->vm; u64 user_addr; u64 size; int r = 0; user_addr = (addr & AMDGPU_GMC_HOLE_MASK) >> AMDGPU_GPU_PAGE_SHIFT; size = expected_size >> AMDGPU_GPU_PAGE_SHIFT; r = amdgpu_bo_reserve(vm->root.bo, false); if (r) return r; va_map = amdgpu_vm_bo_lookup_mapping(vm, user_addr); if (!va_map) { r = -EINVAL; goto out_err; } /* Only validate the userq whether resident in the VM mapping range */ if (user_addr >= va_map->start && va_map->last - user_addr + 1 >= size) { amdgpu_userq_buffer_va_list_add(queue, va_map, user_addr); amdgpu_bo_unreserve(vm->root.bo); return 0; } r = -EINVAL;out_err: amdgpu_bo_unreserve(vm->root.bo); return r;} static bool amdgpu_userq_buffer_va_mapped(struct amdgpu_vm *vm, u64 addr){ struct amdgpu_bo_va_mapping *mapping; bool r; if (amdgpu_bo_reserve(vm->root.bo, false)) return false; mapping = amdgpu_vm_bo_lookup_mapping(vm, addr); if (!IS_ERR_OR_NULL(mapping) && atomic_read(&mapping->bo_va->userq_va_mapped)) r = true; else r = false; amdgpu_bo_unreserve(vm->root.bo); return r;} static bool amdgpu_userq_buffer_vas_mapped(struct amdgpu_usermode_queue *queue){ struct amdgpu_userq_va_cursor *va_cursor, *tmp; int r = 0; list_for_each_entry_safe(va_cursor, tmp, &queue->userq_va_list, list) { r += amdgpu_userq_buffer_va_mapped(queue->vm, va_cursor->gpu_addr); dev_dbg(queue->userq_mgr->adev->dev, "validate the userq mapping:%p va:%llx r:%d\n", queue, va_cursor->gpu_addr, r); } if (r != 0) return true; return false;} static void amdgpu_userq_buffer_va_list_del(struct amdgpu_bo_va_mapping *mapping, struct amdgpu_userq_va_cursor *va_cursor){ atomic_set(&mapping->bo_va->userq_va_mapped, 0); list_del(&va_cursor->list); kfree(va_cursor);} static int amdgpu_userq_buffer_vas_list_cleanup(struct amdgpu_device *adev, struct amdgpu_usermode_queue *queue){ struct amdgpu_userq_va_cursor *va_cursor, *tmp; struct amdgpu_bo_va_mapping *mapping; int r; r = amdgpu_bo_reserve(queue->vm->root.bo, false); if (r) return r; list_for_each_entry_safe(va_cursor, tmp, &queue->userq_va_list, list) { mapping = amdgpu_vm_bo_lookup_mapping(queue->vm, va_cursor->gpu_addr); if (!mapping) { r = -EINVAL; goto err; } dev_dbg(adev->dev, "delete the userq:%p va:%llx\n", queue, va_cursor->gpu_addr); amdgpu_userq_buffer_va_list_del(mapping, va_cursor); }err: amdgpu_bo_unreserve(queue->vm->root.bo); return r;} static int amdgpu_userq_preempt_helper(struct amdgpu_usermode_queue *queue){ struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr; struct amdgpu_device *adev = uq_mgr->adev; const struct amdgpu_userq_funcs *userq_funcs = adev->userq_funcs[queue->queue_type]; bool found_hung_queue = false; int r = 0; if (queue->state == AMDGPU_USERQ_STATE_MAPPED) { r = userq_funcs->preempt(queue); if (r) { queue->state = AMDGPU_USERQ_STATE_HUNG; found_hung_queue = true; } else { queue->state = AMDGPU_USERQ_STATE_PREEMPTED; } } if (found_hung_queue) amdgpu_userq_detect_and_reset_queues(uq_mgr); return r;} static int amdgpu_userq_restore_helper(struct amdgpu_usermode_queue *queue){ struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr; struct amdgpu_device *adev = uq_mgr->adev; const struct amdgpu_userq_funcs *userq_funcs = adev->userq_funcs[queue->queue_type]; int r = 0; if (queue->state == AMDGPU_USERQ_STATE_PREEMPTED) { r = userq_funcs->restore(queue); if (r) { queue->state = AMDGPU_USERQ_STATE_HUNG; } else { queue->state = AMDGPU_USERQ_STATE_MAPPED; } } return r;} static int amdgpu_userq_unmap_helper(struct amdgpu_usermode_queue *queue){ struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr; struct amdgpu_device *adev = uq_mgr->adev; const struct amdgpu_userq_funcs *userq_funcs = adev->userq_funcs[queue->queue_type]; bool found_hung_queue = false; int r = 0; if ((queue->state == AMDGPU_USERQ_STATE_MAPPED) || (queue->state == AMDGPU_USERQ_STATE_PREEMPTED)) { r = userq_funcs->unmap(queue); if (r) { queue->state = AMDGPU_USERQ_STATE_HUNG; found_hung_queue = true; } else { queue->state = AMDGPU_USERQ_STATE_UNMAPPED; } } if (found_hung_queue) amdgpu_userq_detect_and_reset_queues(uq_mgr); return r;} static int amdgpu_userq_map_helper(struct amdgpu_usermode_queue *queue){ struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr; struct amdgpu_device *adev = uq_mgr->adev; const struct amdgpu_userq_funcs *userq_funcs = adev->userq_funcs[queue->queue_type]; int r = 0; if (queue->state == AMDGPU_USERQ_STATE_UNMAPPED) { r = userq_funcs->map(queue); if (r) { queue->state = AMDGPU_USERQ_STATE_HUNG; amdgpu_userq_detect_and_reset_queues(uq_mgr); } else { queue->state = AMDGPU_USERQ_STATE_MAPPED; } } return r;} static int amdgpu_userq_wait_for_last_fence(struct amdgpu_usermode_queue *queue){ struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr; struct dma_fence *f = queue->last_fence; int ret = 0; if (f && !dma_fence_is_signaled(f)) { ret = dma_fence_wait_timeout(f, true, MAX_SCHEDULE_TIMEOUT); if (ret <= 0) { drm_file_err(uq_mgr->file, "Timed out waiting for fence=%llu:%llu\n", f->context, f->seqno); queue->state = AMDGPU_USERQ_STATE_HUNG; return -ETIME; } } return ret;} static void amdgpu_userq_cleanup(struct amdgpu_usermode_queue *queue){ struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr; struct amdgpu_device *adev = uq_mgr->adev; const struct amdgpu_userq_funcs *uq_funcs = adev->userq_funcs[queue->queue_type]; /* Wait for mode-1 reset to complete */ down_read(&adev->reset_domain->sem); /* Drop the userq reference. */ amdgpu_userq_buffer_vas_list_cleanup(adev, queue); uq_funcs->mqd_destroy(queue); amdgpu_userq_fence_driver_free(queue); /* Use interrupt-safe locking since IRQ handlers may access these XArrays */ xa_erase_irq(&adev->userq_doorbell_xa, queue->doorbell_index); queue->userq_mgr = NULL; list_del(&queue->userq_va_list); kfree(queue); up_read(&adev->reset_domain->sem);} voidamdgpu_userq_ensure_ev_fence(struct amdgpu_userq_mgr *uq_mgr, struct amdgpu_eviction_fence_mgr *evf_mgr){ struct amdgpu_eviction_fence *ev_fence; retry: /* Flush any pending resume work to create ev_fence */ flush_delayed_work(&uq_mgr->resume_work); mutex_lock(&uq_mgr->userq_mutex); spin_lock(&evf_mgr->ev_fence_lock); ev_fence = evf_mgr->ev_fence; spin_unlock(&evf_mgr->ev_fence_lock); if (!ev_fence || dma_fence_is_signaled(&ev_fence->base)) { mutex_unlock(&uq_mgr->userq_mutex); /* * Looks like there was no pending resume work, * add one now to create a valid eviction fence */ schedule_delayed_work(&uq_mgr->resume_work, 0); goto retry; }} int amdgpu_userq_create_object(struct amdgpu_userq_mgr *uq_mgr, struct amdgpu_userq_obj *userq_obj, int size){ struct amdgpu_device *adev = uq_mgr->adev;Lines 1–500 of 1631. Load more
// SPDX-License-Identifier: MIT /* * Copyright 2023 Advanced Micro Devices, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. * */ #include <drm/drm_auth.h> #include <drm/drm_exec.h> #include <linux/pm_runtime.h> #include <drm/drm_drv.h> #include "amdgpu.h" #include "amdgpu_reset.h" #include "amdgpu_vm.h" #include "amdgpu_userq.h" #include "amdgpu_hmm.h" #include "amdgpu_userq_fence.h" u32 amdgpu_userq_get_supported_ip_mask(struct amdgpu_device *adev) { int i; u32 userq_ip_mask = 0; for (i = 0; i < AMDGPU_HW_IP_NUM; i++) { if (adev->userq_funcs[i]) userq_ip_mask |= (1 << i); } return userq_ip_mask; } static bool amdgpu_userq_is_reset_type_supported(struct amdgpu_device *adev, enum amdgpu_ring_type ring_type, int reset_type) { if (ring_type < 0 || ring_type >= AMDGPU_RING_TYPE_MAX) return false; switch (ring_type) { case AMDGPU_RING_TYPE_GFX: if (adev->gfx.gfx_supported_reset & reset_type) return true; break; case AMDGPU_RING_TYPE_COMPUTE: if (adev->gfx.compute_supported_reset & reset_type) return true; break; case AMDGPU_RING_TYPE_SDMA: if (adev->sdma.supported_reset & reset_type) return true; break; case AMDGPU_RING_TYPE_VCN_DEC: case AMDGPU_RING_TYPE_VCN_ENC: if (adev->vcn.supported_reset & reset_type) return true; break; case AMDGPU_RING_TYPE_VCN_JPEG: if (adev->jpeg.supported_reset & reset_type) return true; break; default: break; } return false; } static void amdgpu_userq_gpu_reset(struct amdgpu_device *adev) { if (amdgpu_device_should_recover_gpu(adev)) { amdgpu_reset_domain_schedule(adev->reset_domain, &adev->userq_reset_work); /* Wait for the reset job to complete */ flush_work(&adev->userq_reset_work); } } static int amdgpu_userq_detect_and_reset_queues(struct amdgpu_userq_mgr *uq_mgr) { struct amdgpu_device *adev = uq_mgr->adev; const int queue_types[] = { AMDGPU_RING_TYPE_COMPUTE, AMDGPU_RING_TYPE_GFX, AMDGPU_RING_TYPE_SDMA }; const int num_queue_types = ARRAY_SIZE(queue_types); bool gpu_reset = false; int r = 0; int i; /* Warning if current process mutex is not held */ WARN_ON(!mutex_is_locked(&uq_mgr->userq_mutex)); if (unlikely(adev->debug_disable_gpu_ring_reset)) { dev_err(adev->dev, "userq reset disabled by debug mask\n"); return 0; } /* * If GPU recovery feature is disabled system-wide, * skip all reset detection logic */ if (!amdgpu_gpu_recovery) return 0; /* * Iterate through all queue types to detect and reset problematic queues * Process each queue type in the defined order */ for (i = 0; i < num_queue_types; i++) { int ring_type = queue_types[i]; const struct amdgpu_userq_funcs *funcs = adev->userq_funcs[ring_type]; if (!amdgpu_userq_is_reset_type_supported(adev, ring_type, AMDGPU_RESET_TYPE_PER_QUEUE)) continue; if (atomic_read(&uq_mgr->userq_count[ring_type]) > 0 && funcs && funcs->detect_and_reset) { r = funcs->detect_and_reset(adev, ring_type); if (r) { gpu_reset = true; break; } } } if (gpu_reset) amdgpu_userq_gpu_reset(adev); return r; } static void amdgpu_userq_hang_detect_work(struct work_struct *work) { struct amdgpu_usermode_queue *queue = container_of(work, struct amdgpu_usermode_queue, hang_detect_work.work); struct dma_fence *fence; struct amdgpu_userq_mgr *uq_mgr; if (!queue || !queue->userq_mgr) return; uq_mgr = queue->userq_mgr; fence = READ_ONCE(queue->hang_detect_fence); /* Fence already signaled – no action needed */ if (!fence || dma_fence_is_signaled(fence)) return; mutex_lock(&uq_mgr->userq_mutex); amdgpu_userq_detect_and_reset_queues(uq_mgr); mutex_unlock(&uq_mgr->userq_mutex); } /* * Start hang detection for a user queue fence. A delayed work will be scheduled * to check if the fence is still pending after the timeout period. */ void amdgpu_userq_start_hang_detect_work(struct amdgpu_usermode_queue *queue) { struct amdgpu_device *adev; unsigned long timeout_ms; if (!queue || !queue->userq_mgr || !queue->userq_mgr->adev) return; adev = queue->userq_mgr->adev; /* Determine timeout based on queue type */ switch (queue->queue_type) { case AMDGPU_RING_TYPE_GFX: timeout_ms = adev->gfx_timeout; break; case AMDGPU_RING_TYPE_COMPUTE: timeout_ms = adev->compute_timeout; break; case AMDGPU_RING_TYPE_SDMA: timeout_ms = adev->sdma_timeout; break; default: timeout_ms = adev->gfx_timeout; break; } /* Store the fence to monitor and schedule hang detection */ WRITE_ONCE(queue->hang_detect_fence, queue->last_fence); schedule_delayed_work(&queue->hang_detect_work, msecs_to_jiffies(timeout_ms)); } static void amdgpu_userq_init_hang_detect_work(struct amdgpu_usermode_queue *queue) { INIT_DELAYED_WORK(&queue->hang_detect_work, amdgpu_userq_hang_detect_work); queue->hang_detect_fence = NULL; } static int amdgpu_userq_buffer_va_list_add(struct amdgpu_usermode_queue *queue, struct amdgpu_bo_va_mapping *va_map, u64 addr) { struct amdgpu_userq_va_cursor *va_cursor; struct userq_va_list; va_cursor = kzalloc_obj(*va_cursor); if (!va_cursor) return -ENOMEM; INIT_LIST_HEAD(&va_cursor->list); va_cursor->gpu_addr = addr; atomic_set(&va_map->bo_va->userq_va_mapped, 1); list_add(&va_cursor->list, &queue->userq_va_list); return 0; } int amdgpu_userq_input_va_validate(struct amdgpu_device *adev, struct amdgpu_usermode_queue *queue, u64 addr, u64 expected_size) { struct amdgpu_bo_va_mapping *va_map; struct amdgpu_vm *vm = queue->vm; u64 user_addr; u64 size; int r = 0; user_addr = (addr & AMDGPU_GMC_HOLE_MASK) >> AMDGPU_GPU_PAGE_SHIFT; size = expected_size >> AMDGPU_GPU_PAGE_SHIFT; r = amdgpu_bo_reserve(vm->root.bo, false); if (r) return r; va_map = amdgpu_vm_bo_lookup_mapping(vm, user_addr); if (!va_map) { r = -EINVAL; goto out_err; } /* Only validate the userq whether resident in the VM mapping range */ if (user_addr >= va_map->start && va_map->last - user_addr + 1 >= size) { amdgpu_userq_buffer_va_list_add(queue, va_map, user_addr); amdgpu_bo_unreserve(vm->root.bo); return 0; } r = -EINVAL; out_err: amdgpu_bo_unreserve(vm->root.bo); return r; } static bool amdgpu_userq_buffer_va_mapped(struct amdgpu_vm *vm, u64 addr) { struct amdgpu_bo_va_mapping *mapping; bool r; if (amdgpu_bo_reserve(vm->root.bo, false)) return false; mapping = amdgpu_vm_bo_lookup_mapping(vm, addr); if (!IS_ERR_OR_NULL(mapping) && atomic_read(&mapping->bo_va->userq_va_mapped)) r = true; else r = false; amdgpu_bo_unreserve(vm->root.bo); return r; } static bool amdgpu_userq_buffer_vas_mapped(struct amdgpu_usermode_queue *queue) { struct amdgpu_userq_va_cursor *va_cursor, *tmp; int r = 0; list_for_each_entry_safe(va_cursor, tmp, &queue->userq_va_list, list) { r += amdgpu_userq_buffer_va_mapped(queue->vm, va_cursor->gpu_addr); dev_dbg(queue->userq_mgr->adev->dev, "validate the userq mapping:%p va:%llx r:%d\n", queue, va_cursor->gpu_addr, r); } if (r != 0) return true; return false; } static void amdgpu_userq_buffer_va_list_del(struct amdgpu_bo_va_mapping *mapping, struct amdgpu_userq_va_cursor *va_cursor) { atomic_set(&mapping->bo_va->userq_va_mapped, 0); list_del(&va_cursor->list); kfree(va_cursor); } static int amdgpu_userq_buffer_vas_list_cleanup(struct amdgpu_device *adev, struct amdgpu_usermode_queue *queue) { struct amdgpu_userq_va_cursor *va_cursor, *tmp; struct amdgpu_bo_va_mapping *mapping; int r; r = amdgpu_bo_reserve(queue->vm->root.bo, false); if (r) return r; list_for_each_entry_safe(va_cursor, tmp, &queue->userq_va_list, list) { mapping = amdgpu_vm_bo_lookup_mapping(queue->vm, va_cursor->gpu_addr); if (!mapping) { r = -EINVAL; goto err; } dev_dbg(adev->dev, "delete the userq:%p va:%llx\n", queue, va_cursor->gpu_addr); amdgpu_userq_buffer_va_list_del(mapping, va_cursor); } err: amdgpu_bo_unreserve(queue->vm->root.bo); return r; } static int amdgpu_userq_preempt_helper(struct amdgpu_usermode_queue *queue) { struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr; struct amdgpu_device *adev = uq_mgr->adev; const struct amdgpu_userq_funcs *userq_funcs = adev->userq_funcs[queue->queue_type]; bool found_hung_queue = false; int r = 0; if (queue->state == AMDGPU_USERQ_STATE_MAPPED) { r = userq_funcs->preempt(queue); if (r) { queue->state = AMDGPU_USERQ_STATE_HUNG; found_hung_queue = true; } else { queue->state = AMDGPU_USERQ_STATE_PREEMPTED; } } if (found_hung_queue) amdgpu_userq_detect_and_reset_queues(uq_mgr); return r; } static int amdgpu_userq_restore_helper(struct amdgpu_usermode_queue *queue) { struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr; struct amdgpu_device *adev = uq_mgr->adev; const struct amdgpu_userq_funcs *userq_funcs = adev->userq_funcs[queue->queue_type]; int r = 0; if (queue->state == AMDGPU_USERQ_STATE_PREEMPTED) { r = userq_funcs->restore(queue); if (r) { queue->state = AMDGPU_USERQ_STATE_HUNG; } else { queue->state = AMDGPU_USERQ_STATE_MAPPED; } } return r; } static int amdgpu_userq_unmap_helper(struct amdgpu_usermode_queue *queue) { struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr; struct amdgpu_device *adev = uq_mgr->adev; const struct amdgpu_userq_funcs *userq_funcs = adev->userq_funcs[queue->queue_type]; bool found_hung_queue = false; int r = 0; if ((queue->state == AMDGPU_USERQ_STATE_MAPPED) || (queue->state == AMDGPU_USERQ_STATE_PREEMPTED)) { r = userq_funcs->unmap(queue); if (r) { queue->state = AMDGPU_USERQ_STATE_HUNG; found_hung_queue = true; } else { queue->state = AMDGPU_USERQ_STATE_UNMAPPED; } } if (found_hung_queue) amdgpu_userq_detect_and_reset_queues(uq_mgr); return r; } static int amdgpu_userq_map_helper(struct amdgpu_usermode_queue *queue) { struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr; struct amdgpu_device *adev = uq_mgr->adev; const struct amdgpu_userq_funcs *userq_funcs = adev->userq_funcs[queue->queue_type]; int r = 0; if (queue->state == AMDGPU_USERQ_STATE_UNMAPPED) { r = userq_funcs->map(queue); if (r) { queue->state = AMDGPU_USERQ_STATE_HUNG; amdgpu_userq_detect_and_reset_queues(uq_mgr); } else { queue->state = AMDGPU_USERQ_STATE_MAPPED; } } return r; } static int amdgpu_userq_wait_for_last_fence(struct amdgpu_usermode_queue *queue) { struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr; struct dma_fence *f = queue->last_fence; int ret = 0; if (f && !dma_fence_is_signaled(f)) { ret = dma_fence_wait_timeout(f, true, MAX_SCHEDULE_TIMEOUT); if (ret <= 0) { drm_file_err(uq_mgr->file, "Timed out waiting for fence=%llu:%llu\n", f->context, f->seqno); queue->state = AMDGPU_USERQ_STATE_HUNG; return -ETIME; } } return ret; } static void amdgpu_userq_cleanup(struct amdgpu_usermode_queue *queue) { struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr; struct amdgpu_device *adev = uq_mgr->adev; const struct amdgpu_userq_funcs *uq_funcs = adev->userq_funcs[queue->queue_type]; /* Wait for mode-1 reset to complete */ down_read(&adev->reset_domain->sem); /* Drop the userq reference. */ amdgpu_userq_buffer_vas_list_cleanup(adev, queue); uq_funcs->mqd_destroy(queue); amdgpu_userq_fence_driver_free(queue); /* Use interrupt-safe locking since IRQ handlers may access these XArrays */ xa_erase_irq(&adev->userq_doorbell_xa, queue->doorbell_index); queue->userq_mgr = NULL; list_del(&queue->userq_va_list); kfree(queue); up_read(&adev->reset_domain->sem); } void amdgpu_userq_ensure_ev_fence(struct amdgpu_userq_mgr *uq_mgr, struct amdgpu_eviction_fence_mgr *evf_mgr) { struct amdgpu_eviction_fence *ev_fence; retry: /* Flush any pending resume work to create ev_fence */ flush_delayed_work(&uq_mgr->resume_work); mutex_lock(&uq_mgr->userq_mutex); spin_lock(&evf_mgr->ev_fence_lock); ev_fence = evf_mgr->ev_fence; spin_unlock(&evf_mgr->ev_fence_lock); if (!ev_fence || dma_fence_is_signaled(&ev_fence->base)) { mutex_unlock(&uq_mgr->userq_mutex); /* * Looks like there was no pending resume work, * add one now to create a valid eviction fence */ schedule_delayed_work(&uq_mgr->resume_work, 0); goto retry; } } int amdgpu_userq_create_object(struct amdgpu_userq_mgr *uq_mgr, struct amdgpu_userq_obj *userq_obj, int size) { struct amdgpu_device *adev = uq_mgr->adev;