1631 lines · 42.1 KB · C
RawDownload
// SPDX-License-Identifier: MIT/* * Copyright 2023 Advanced Micro Devices, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. * */ #include <drm/drm_auth.h>#include <drm/drm_exec.h>#include <linux/pm_runtime.h>#include <drm/drm_drv.h> #include "amdgpu.h"#include "amdgpu_reset.h"#include "amdgpu_vm.h"#include "amdgpu_userq.h"#include "amdgpu_hmm.h"#include "amdgpu_userq_fence.h" u32 amdgpu_userq_get_supported_ip_mask(struct amdgpu_device *adev){	int i;	u32 userq_ip_mask = 0; 	for (i = 0; i < AMDGPU_HW_IP_NUM; i++) {		if (adev->userq_funcs[i])			userq_ip_mask |= (1 << i);	} 	return userq_ip_mask;} static bool amdgpu_userq_is_reset_type_supported(struct amdgpu_device *adev,				enum amdgpu_ring_type ring_type, int reset_type){ 	if (ring_type < 0 || ring_type >= AMDGPU_RING_TYPE_MAX)		return false; 	switch (ring_type) {	case AMDGPU_RING_TYPE_GFX:		if (adev->gfx.gfx_supported_reset & reset_type)			return true;		break;	case AMDGPU_RING_TYPE_COMPUTE:		if (adev->gfx.compute_supported_reset & reset_type)			return true;		break;	case AMDGPU_RING_TYPE_SDMA:		if (adev->sdma.supported_reset & reset_type)			return true;		break;	case AMDGPU_RING_TYPE_VCN_DEC:	case AMDGPU_RING_TYPE_VCN_ENC:		if (adev->vcn.supported_reset & reset_type)			return true;		break;	case AMDGPU_RING_TYPE_VCN_JPEG:		if (adev->jpeg.supported_reset & reset_type)			return true;		break;	default:		break;	}	return false;} static void amdgpu_userq_gpu_reset(struct amdgpu_device *adev){	if (amdgpu_device_should_recover_gpu(adev)) {		amdgpu_reset_domain_schedule(adev->reset_domain,					     &adev->userq_reset_work);		/* Wait for the reset job to complete */		flush_work(&adev->userq_reset_work);	}} static intamdgpu_userq_detect_and_reset_queues(struct amdgpu_userq_mgr *uq_mgr){	struct amdgpu_device *adev = uq_mgr->adev;	const int queue_types[] = {		AMDGPU_RING_TYPE_COMPUTE,		AMDGPU_RING_TYPE_GFX,		AMDGPU_RING_TYPE_SDMA	};	const int num_queue_types = ARRAY_SIZE(queue_types);	bool gpu_reset = false;	int r = 0;	int i; 	/* Warning if current process mutex is not held */	WARN_ON(!mutex_is_locked(&uq_mgr->userq_mutex)); 	if (unlikely(adev->debug_disable_gpu_ring_reset)) {		dev_err(adev->dev, "userq reset disabled by debug mask\n");		return 0;	} 	/*	 * If GPU recovery feature is disabled system-wide,	 * skip all reset detection logic	 */	if (!amdgpu_gpu_recovery)		return 0; 	/*	 * Iterate through all queue types to detect and reset problematic queues	 * Process each queue type in the defined order	 */	for (i = 0; i < num_queue_types; i++) {		int ring_type = queue_types[i];		const struct amdgpu_userq_funcs *funcs = adev->userq_funcs[ring_type]; 		if (!amdgpu_userq_is_reset_type_supported(adev, ring_type, AMDGPU_RESET_TYPE_PER_QUEUE))				continue; 		if (atomic_read(&uq_mgr->userq_count[ring_type]) > 0 &&		    funcs && funcs->detect_and_reset) {			r = funcs->detect_and_reset(adev, ring_type);			if (r) {				gpu_reset = true;				break;			}		}	} 	if (gpu_reset)		amdgpu_userq_gpu_reset(adev); 	return r;} static void amdgpu_userq_hang_detect_work(struct work_struct *work){	struct amdgpu_usermode_queue *queue = container_of(work,							  struct amdgpu_usermode_queue,							  hang_detect_work.work);	struct dma_fence *fence;	struct amdgpu_userq_mgr *uq_mgr; 	if (!queue || !queue->userq_mgr)		return; 	uq_mgr = queue->userq_mgr;	fence = READ_ONCE(queue->hang_detect_fence);	/* Fence already signaled – no action needed */	if (!fence || dma_fence_is_signaled(fence))		return; 	mutex_lock(&uq_mgr->userq_mutex);	amdgpu_userq_detect_and_reset_queues(uq_mgr);	mutex_unlock(&uq_mgr->userq_mutex);} /* * Start hang detection for a user queue fence. A delayed work will be scheduled * to check if the fence is still pending after the timeout period.*/void amdgpu_userq_start_hang_detect_work(struct amdgpu_usermode_queue *queue){	struct amdgpu_device *adev;	unsigned long timeout_ms; 	if (!queue || !queue->userq_mgr || !queue->userq_mgr->adev)		return; 	adev = queue->userq_mgr->adev;	/* Determine timeout based on queue type */	switch (queue->queue_type) {	case AMDGPU_RING_TYPE_GFX:		timeout_ms = adev->gfx_timeout;		break;	case AMDGPU_RING_TYPE_COMPUTE:		timeout_ms = adev->compute_timeout;		break;	case AMDGPU_RING_TYPE_SDMA:		timeout_ms = adev->sdma_timeout;		break;	default:		timeout_ms = adev->gfx_timeout;		break;	} 	/* Store the fence to monitor and schedule hang detection */	WRITE_ONCE(queue->hang_detect_fence, queue->last_fence);	schedule_delayed_work(&queue->hang_detect_work,		     msecs_to_jiffies(timeout_ms));} static void amdgpu_userq_init_hang_detect_work(struct amdgpu_usermode_queue *queue){	INIT_DELAYED_WORK(&queue->hang_detect_work, amdgpu_userq_hang_detect_work);	queue->hang_detect_fence = NULL;} static int amdgpu_userq_buffer_va_list_add(struct amdgpu_usermode_queue *queue,					   struct amdgpu_bo_va_mapping *va_map, u64 addr){	struct amdgpu_userq_va_cursor *va_cursor;	struct userq_va_list; 	va_cursor = kzalloc_obj(*va_cursor);	if (!va_cursor)		return -ENOMEM; 	INIT_LIST_HEAD(&va_cursor->list);	va_cursor->gpu_addr = addr;	atomic_set(&va_map->bo_va->userq_va_mapped, 1);	list_add(&va_cursor->list, &queue->userq_va_list); 	return 0;} int amdgpu_userq_input_va_validate(struct amdgpu_device *adev,				   struct amdgpu_usermode_queue *queue,				   u64 addr, u64 expected_size){	struct amdgpu_bo_va_mapping *va_map;	struct amdgpu_vm *vm = queue->vm;	u64 user_addr;	u64 size;	int r = 0; 	user_addr = (addr & AMDGPU_GMC_HOLE_MASK) >> AMDGPU_GPU_PAGE_SHIFT;	size = expected_size >> AMDGPU_GPU_PAGE_SHIFT; 	r = amdgpu_bo_reserve(vm->root.bo, false);	if (r)		return r; 	va_map = amdgpu_vm_bo_lookup_mapping(vm, user_addr);	if (!va_map) {		r = -EINVAL;		goto out_err;	}	/* Only validate the userq whether resident in the VM mapping range */	if (user_addr >= va_map->start  &&	    va_map->last - user_addr + 1 >= size) {		amdgpu_userq_buffer_va_list_add(queue, va_map, user_addr);		amdgpu_bo_unreserve(vm->root.bo);		return 0;	} 	r = -EINVAL;out_err:	amdgpu_bo_unreserve(vm->root.bo);	return r;} static bool amdgpu_userq_buffer_va_mapped(struct amdgpu_vm *vm, u64 addr){	struct amdgpu_bo_va_mapping *mapping;	bool r; 	if (amdgpu_bo_reserve(vm->root.bo, false))		return false; 	mapping = amdgpu_vm_bo_lookup_mapping(vm, addr);	if (!IS_ERR_OR_NULL(mapping) && atomic_read(&mapping->bo_va->userq_va_mapped))		r = true;	else		r = false;	amdgpu_bo_unreserve(vm->root.bo); 	return r;} static bool amdgpu_userq_buffer_vas_mapped(struct amdgpu_usermode_queue *queue){	struct amdgpu_userq_va_cursor *va_cursor, *tmp;	int r = 0; 	list_for_each_entry_safe(va_cursor, tmp, &queue->userq_va_list, list) {		r += amdgpu_userq_buffer_va_mapped(queue->vm, va_cursor->gpu_addr);		dev_dbg(queue->userq_mgr->adev->dev,			"validate the userq mapping:%p va:%llx r:%d\n",			queue, va_cursor->gpu_addr, r);	} 	if (r != 0)		return true; 	return false;} static void amdgpu_userq_buffer_va_list_del(struct amdgpu_bo_va_mapping *mapping,					    struct amdgpu_userq_va_cursor *va_cursor){	atomic_set(&mapping->bo_va->userq_va_mapped, 0);	list_del(&va_cursor->list);	kfree(va_cursor);} static int amdgpu_userq_buffer_vas_list_cleanup(struct amdgpu_device *adev,						struct amdgpu_usermode_queue *queue){	struct amdgpu_userq_va_cursor *va_cursor, *tmp;	struct amdgpu_bo_va_mapping *mapping;	int r; 	r = amdgpu_bo_reserve(queue->vm->root.bo, false);	if (r)		return r; 	list_for_each_entry_safe(va_cursor, tmp, &queue->userq_va_list, list) {		mapping = amdgpu_vm_bo_lookup_mapping(queue->vm, va_cursor->gpu_addr);		if (!mapping) {			r = -EINVAL;			goto err;		}		dev_dbg(adev->dev, "delete the userq:%p va:%llx\n",			queue, va_cursor->gpu_addr);		amdgpu_userq_buffer_va_list_del(mapping, va_cursor);	}err:	amdgpu_bo_unreserve(queue->vm->root.bo);	return r;} static int amdgpu_userq_preempt_helper(struct amdgpu_usermode_queue *queue){	struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr;	struct amdgpu_device *adev = uq_mgr->adev;	const struct amdgpu_userq_funcs *userq_funcs =		adev->userq_funcs[queue->queue_type];	bool found_hung_queue = false;	int r = 0; 	if (queue->state == AMDGPU_USERQ_STATE_MAPPED) {		r = userq_funcs->preempt(queue);		if (r) {			queue->state = AMDGPU_USERQ_STATE_HUNG;			found_hung_queue = true;		} else {			queue->state = AMDGPU_USERQ_STATE_PREEMPTED;		}	} 	if (found_hung_queue)		amdgpu_userq_detect_and_reset_queues(uq_mgr); 	return r;} static int amdgpu_userq_restore_helper(struct amdgpu_usermode_queue *queue){	struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr;	struct amdgpu_device *adev = uq_mgr->adev;	const struct amdgpu_userq_funcs *userq_funcs =		adev->userq_funcs[queue->queue_type];	int r = 0; 	if (queue->state == AMDGPU_USERQ_STATE_PREEMPTED) {		r = userq_funcs->restore(queue);		if (r) {			queue->state = AMDGPU_USERQ_STATE_HUNG;		} else {			queue->state = AMDGPU_USERQ_STATE_MAPPED;		}	} 	return r;} static int amdgpu_userq_unmap_helper(struct amdgpu_usermode_queue *queue){	struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr;	struct amdgpu_device *adev = uq_mgr->adev;	const struct amdgpu_userq_funcs *userq_funcs =		adev->userq_funcs[queue->queue_type];	bool found_hung_queue = false;	int r = 0; 	if ((queue->state == AMDGPU_USERQ_STATE_MAPPED) ||		(queue->state == AMDGPU_USERQ_STATE_PREEMPTED)) {		r = userq_funcs->unmap(queue);		if (r) {			queue->state = AMDGPU_USERQ_STATE_HUNG;			found_hung_queue = true;		} else {			queue->state = AMDGPU_USERQ_STATE_UNMAPPED;		}	} 	if (found_hung_queue)		amdgpu_userq_detect_and_reset_queues(uq_mgr); 	return r;} static int amdgpu_userq_map_helper(struct amdgpu_usermode_queue *queue){	struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr;	struct amdgpu_device *adev = uq_mgr->adev;	const struct amdgpu_userq_funcs *userq_funcs =		adev->userq_funcs[queue->queue_type];	int r = 0; 	if (queue->state == AMDGPU_USERQ_STATE_UNMAPPED) {		r = userq_funcs->map(queue);		if (r) {			queue->state = AMDGPU_USERQ_STATE_HUNG;			amdgpu_userq_detect_and_reset_queues(uq_mgr);		} else {			queue->state = AMDGPU_USERQ_STATE_MAPPED;		}	} 	return r;} static int amdgpu_userq_wait_for_last_fence(struct amdgpu_usermode_queue *queue){	struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr;	struct dma_fence *f = queue->last_fence;	int ret = 0; 	if (f && !dma_fence_is_signaled(f)) {		ret = dma_fence_wait_timeout(f, true, MAX_SCHEDULE_TIMEOUT);		if (ret <= 0) {			drm_file_err(uq_mgr->file, "Timed out waiting for fence=%llu:%llu\n",				     f->context, f->seqno);			queue->state = AMDGPU_USERQ_STATE_HUNG;			return -ETIME;		}	} 	return ret;} static void amdgpu_userq_cleanup(struct amdgpu_usermode_queue *queue){	struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr;	struct amdgpu_device *adev = uq_mgr->adev;	const struct amdgpu_userq_funcs *uq_funcs = adev->userq_funcs[queue->queue_type]; 	/* Wait for mode-1 reset to complete */	down_read(&adev->reset_domain->sem); 	/* Drop the userq reference. */	amdgpu_userq_buffer_vas_list_cleanup(adev, queue);	uq_funcs->mqd_destroy(queue);	amdgpu_userq_fence_driver_free(queue);	/* Use interrupt-safe locking since IRQ handlers may access these XArrays */	xa_erase_irq(&adev->userq_doorbell_xa, queue->doorbell_index);	queue->userq_mgr = NULL;	list_del(&queue->userq_va_list);	kfree(queue); 	up_read(&adev->reset_domain->sem);} voidamdgpu_userq_ensure_ev_fence(struct amdgpu_userq_mgr *uq_mgr,			     struct amdgpu_eviction_fence_mgr *evf_mgr){	struct amdgpu_eviction_fence *ev_fence; retry:	/* Flush any pending resume work to create ev_fence */	flush_delayed_work(&uq_mgr->resume_work); 	mutex_lock(&uq_mgr->userq_mutex);	spin_lock(&evf_mgr->ev_fence_lock);	ev_fence = evf_mgr->ev_fence;	spin_unlock(&evf_mgr->ev_fence_lock);	if (!ev_fence || dma_fence_is_signaled(&ev_fence->base)) {		mutex_unlock(&uq_mgr->userq_mutex);		/*		 * Looks like there was no pending resume work,		 * add one now to create a valid eviction fence		 */		schedule_delayed_work(&uq_mgr->resume_work, 0);		goto retry;	}} int amdgpu_userq_create_object(struct amdgpu_userq_mgr *uq_mgr,			       struct amdgpu_userq_obj *userq_obj,			       int size){	struct amdgpu_device *adev = uq_mgr->adev;

Lines 1–500 of 1631.

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t