// SPDX-License-Identifier: MIT/* * Copyright 2023 Advanced Micro Devices, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. * */ #include <drm/drm_auth.h>#include <drm/drm_exec.h>#include <linux/pm_runtime.h>#include <linux/overflow.h>#include <drm/drm_drv.h> #include "amdgpu.h"#include "amdgpu_reset.h"#include "amdgpu_vm.h"#include "amdgpu_userq.h"#include "amdgpu_hmm.h"#include "amdgpu_userq_fence.h"#include "amdgpu_trace.h" u32 amdgpu_userq_get_supported_ip_mask(struct amdgpu_device *adev){	int i;	u32 userq_ip_mask = 0; 	for (i = 0; i < AMDGPU_HW_IP_NUM; i++) {		if (adev->userq_funcs[i])			userq_ip_mask |= (1 << i);	} 	return userq_ip_mask;} static bool amdgpu_userq_is_reset_type_supported(struct amdgpu_device *adev,				enum amdgpu_ring_type ring_type, int reset_type){ 	if (ring_type < 0 || ring_type >= AMDGPU_RING_TYPE_MAX)		return false; 	switch (ring_type) {	case AMDGPU_RING_TYPE_GFX:		if (adev->gfx.gfx_supported_reset & reset_type)			return true;		break;	case AMDGPU_RING_TYPE_COMPUTE:		if (adev->gfx.compute_supported_reset & reset_type)			return true;		break;	case AMDGPU_RING_TYPE_SDMA:		if (adev->sdma.supported_reset & reset_type)			return true;		break;	case AMDGPU_RING_TYPE_VCN_DEC:	case AMDGPU_RING_TYPE_VCN_ENC:		if (adev->vcn.supported_reset & reset_type)			return true;		break;	case AMDGPU_RING_TYPE_VCN_JPEG:		if (adev->jpeg.supported_reset & reset_type)			return true;		break;	default:		break;	}	return false;} static void amdgpu_userq_mgr_reset_work(struct work_struct *work){	struct amdgpu_userq_mgr *uq_mgr =		container_of(work, struct amdgpu_userq_mgr,			     reset_work);	struct amdgpu_device *adev = uq_mgr->adev;	struct amdgpu_reset_context reset_context; 	if (unlikely(adev->debug_disable_gpu_ring_reset)) {		dev_err(adev->dev, "userq reset disabled by debug mask\n");		return;	} 	/*	 * If GPU recovery feature is disabled system-wide,	 * skip all reset detection logic	 */	if (!amdgpu_gpu_recovery)		return; 	memset(&reset_context, 0, sizeof(reset_context)); 	reset_context.method = AMD_RESET_METHOD_NONE;	reset_context.reset_req_dev = adev;	reset_context.src = AMDGPU_RESET_SRC_USERQ;	set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);	/*set_bit(AMDGPU_SKIP_COREDUMP, &reset_context.flags);*/ 	amdgpu_device_gpu_recover(adev, NULL, &reset_context);} static void amdgpu_userq_hang_detect_work(struct work_struct *work){	struct amdgpu_usermode_queue *queue =		container_of(work, struct amdgpu_usermode_queue,			     hang_detect_work.work);	struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr;	struct amdgpu_device *adev = uq_mgr->adev;	const struct amdgpu_userq_funcs *userq_funcs =		adev->userq_funcs[queue->queue_type];	struct drm_wedge_task_info *info = NULL;	struct amdgpu_task_info *ti = NULL;	bool gpu_reset = false; 	if (unlikely(adev->debug_disable_gpu_ring_reset)) {		dev_err(adev->dev, "userq reset disabled by debug mask\n");		return;	} 	/*	 * If GPU recovery feature is disabled system-wide,	 * skip all reset detection logic	 */	if (!amdgpu_gpu_recovery)		return; 	if (queue->vm && queue->vm->pasid) {		ti = amdgpu_vm_get_task_info_pasid(adev, queue->vm->pasid);		if (ti) {			amdgpu_vm_print_task_info(adev, ti);			info = &ti->task;		}	} 	if (amdgpu_userq_is_reset_type_supported(adev, queue->queue_type,						 AMDGPU_RESET_TYPE_PER_QUEUE)) {		int r; 		if (queue->queue_type == AMDGPU_HW_IP_COMPUTE)			r = amdgpu_gfx_reset_mes_compute(adev, NULL, NULL,							 queue, NULL, NULL);		else			r = userq_funcs->reset(queue);		if (r) {			gpu_reset = true;		} else {			atomic_inc(&adev->gpu_reset_counter);			amdgpu_userq_fence_driver_force_completion(queue);			drm_dev_wedged_event(adev_to_drm(adev), DRM_WEDGE_RECOVERY_NONE, info);		}	} else {		gpu_reset = true;	}	amdgpu_vm_put_task_info(ti); 	/*	 * Don't schedule the work here! Scheduling or queue work from one reset	 * handler to another is illegal if you don't take extra precautions!	 */	if (gpu_reset)		amdgpu_userq_mgr_reset_work(&queue->userq_mgr->reset_work);} /* * Start hang detection for a user queue fence. A delayed work will be scheduled * to reset the queues when the fence doesn't signal in time. */void amdgpu_userq_start_hang_detect_work(struct amdgpu_usermode_queue *queue){	struct amdgpu_device *adev;	unsigned long timeout_ms; 	adev = queue->userq_mgr->adev;	/* Determine timeout based on queue type */	switch (queue->queue_type) {	case AMDGPU_RING_TYPE_GFX:		timeout_ms = adev->gfx_timeout;		break;	case AMDGPU_RING_TYPE_COMPUTE:		timeout_ms = adev->compute_timeout;		break;	case AMDGPU_RING_TYPE_SDMA:		timeout_ms = adev->sdma_timeout;		break;	default:		timeout_ms = adev->gfx_timeout;		break;	} 	queue_delayed_work(adev->reset_domain->wq, &queue->hang_detect_work,			   msecs_to_jiffies(timeout_ms));} void amdgpu_userq_process_fence_irq(struct amdgpu_device *adev, u32 doorbell){	struct xarray *xa = &adev->userq_doorbell_xa;	struct amdgpu_usermode_queue *queue;	unsigned long flags;	int r; 	xa_lock_irqsave(xa, flags);	queue = xa_load(xa, doorbell);	if (queue) {		r = amdgpu_userq_fence_driver_process(queue->fence_drv);		/*		 * We are in interrupt context here, this *can't* wait for		 * reset work to finish.		 */		if (r >= 0)			cancel_delayed_work(&queue->hang_detect_work); 		/* Restart the timer when there are still fences pending */		if (r == 1)			amdgpu_userq_start_hang_detect_work(queue);	}	xa_unlock_irqrestore(xa, flags);} int amdgpu_userq_input_va_validate(struct amdgpu_device *adev,				   struct amdgpu_usermode_queue *queue,				   u64 addr, u64 expected_size,				   u64 *va_out){	struct amdgpu_bo_va_mapping *va_map;	struct amdgpu_vm *vm = queue->vm;	u64 start_addr;	u64 end_addr;	u64 start_page; 	/* Caller must hold vm->root.bo reservation */	dma_resv_assert_held(queue->vm->root.bo->tbo.base.resv); 	if (!expected_size)		return -EINVAL; 	start_addr = addr & AMDGPU_GMC_HOLE_MASK;	if (check_add_overflow(start_addr, expected_size - 1, &end_addr))		return -EINVAL; 	start_page = start_addr >> AMDGPU_GPU_PAGE_SHIFT; 	va_map = amdgpu_vm_bo_lookup_mapping(vm, start_page);	if (!va_map)		return -EINVAL; 	/* Lookup guarantees start_page is mapped; ensure full span is covered. */	if ((end_addr >> AMDGPU_GPU_PAGE_SHIFT) <= va_map->last) {		va_map->bo_va->userq_va_mapped = true;		*va_out = start_page;		return 0;	} 	return -EINVAL;} static bool amdgpu_userq_buffer_va_mapped(struct amdgpu_vm *vm, u64 addr){	struct amdgpu_bo_va_mapping *mapping;	bool r; 	dma_resv_assert_held(vm->root.bo->tbo.base.resv); 	mapping = amdgpu_vm_bo_lookup_mapping(vm, addr);	if (!IS_ERR_OR_NULL(mapping) && mapping->bo_va->userq_va_mapped)		r = true;	else		r = false; 	return r;} static bool amdgpu_userq_buffer_vas_mapped(struct amdgpu_usermode_queue *queue){	int i;	bool mapped; 	for (i = 0; i < ARRAY_SIZE(queue->userq_vas.va_array); i++) {		if (!queue->userq_vas.va_array[i])			continue; 		mapped = amdgpu_userq_buffer_va_mapped(queue->vm,						   queue->userq_vas.va_array[i]);		dev_dbg(queue->userq_mgr->adev->dev,			"validate the userq mapping:%p va:%llx r:%d\n",			queue, queue->userq_vas.va_array[i], mapped); 		if (!mapped)			return false;	} 	return true;}   static int amdgpu_userq_preempt_helper(struct amdgpu_usermode_queue *queue){	struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr;	struct amdgpu_device *adev = uq_mgr->adev;	const struct amdgpu_userq_funcs *userq_funcs =		adev->userq_funcs[queue->queue_type];	int r; 	if (queue->state == AMDGPU_USERQ_STATE_MAPPED) {		trace_amdgpu_userq_state_start(queue); 		r = userq_funcs->preempt(queue);		if (r) {			trace_amdgpu_userq_state_changed(queue, AMDGPU_USERQ_STATE_HUNG);			queue->state = AMDGPU_USERQ_STATE_HUNG;			return r;		} else {			trace_amdgpu_userq_state_changed(queue, AMDGPU_USERQ_STATE_PREEMPTED);			queue->state = AMDGPU_USERQ_STATE_PREEMPTED;		}	}	return 0;} static int amdgpu_userq_restore_helper(struct amdgpu_usermode_queue *queue){	struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr;	struct amdgpu_device *adev = uq_mgr->adev;	const struct amdgpu_userq_funcs *userq_funcs =		adev->userq_funcs[queue->queue_type];	int r = 0; 	if (queue->state == AMDGPU_USERQ_STATE_PREEMPTED) {		trace_amdgpu_userq_state_start(queue); 		r = userq_funcs->restore(queue);		if (r) {			trace_amdgpu_userq_state_changed(queue, AMDGPU_USERQ_STATE_HUNG);			queue->state = AMDGPU_USERQ_STATE_HUNG;		} else {			trace_amdgpu_userq_state_changed(queue, AMDGPU_USERQ_STATE_MAPPED);			queue->state = AMDGPU_USERQ_STATE_MAPPED;		}	} 	return r;} static int amdgpu_userq_unmap_helper(struct amdgpu_usermode_queue *queue){	struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr;	struct amdgpu_device *adev = uq_mgr->adev;	const struct amdgpu_userq_funcs *userq_funcs =		adev->userq_funcs[queue->queue_type];	int r; 	if ((queue->state == AMDGPU_USERQ_STATE_MAPPED) ||	    (queue->state == AMDGPU_USERQ_STATE_PREEMPTED)) {		trace_amdgpu_userq_state_start(queue); 		r = userq_funcs->unmap(queue);		if (r) {			trace_amdgpu_userq_state_changed(queue, AMDGPU_USERQ_STATE_HUNG);			queue->state = AMDGPU_USERQ_STATE_HUNG;			return r;		} else {			trace_amdgpu_userq_state_changed(queue, AMDGPU_USERQ_STATE_UNMAPPED);			queue->state = AMDGPU_USERQ_STATE_UNMAPPED;		}	} 	return 0;} static int amdgpu_userq_map_helper(struct amdgpu_usermode_queue *queue){	struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr;	struct amdgpu_device *adev = uq_mgr->adev;	const struct amdgpu_userq_funcs *userq_funcs =		adev->userq_funcs[queue->queue_type];	int r; 	if (queue->state == AMDGPU_USERQ_STATE_UNMAPPED) {		trace_amdgpu_userq_state_start(queue); 		r = userq_funcs->map(queue);		if (r) {			trace_amdgpu_userq_state_changed(queue, AMDGPU_USERQ_STATE_HUNG);			queue->state = AMDGPU_USERQ_STATE_HUNG;			return r;		} else {			trace_amdgpu_userq_state_changed(queue, AMDGPU_USERQ_STATE_MAPPED);			queue->state = AMDGPU_USERQ_STATE_MAPPED;		}	} 	return 0;} static void amdgpu_userq_wait_for_last_fence(struct amdgpu_usermode_queue *queue){	struct dma_fence *f = queue->last_fence; 	if (!f)		return; 	dma_fence_wait(f, false);} static void amdgpu_userq_detach_doorbell(struct amdgpu_usermode_queue *queue){	struct amdgpu_device *adev = queue->userq_mgr->adev; 	down_read(&adev->reset_domain->sem);	xa_erase_irq(&adev->userq_doorbell_xa, queue->doorbell_index);	up_read(&adev->reset_domain->sem);} /** * amdgpu_userq_ensure_ev_fence - ensure a valid, unsignaled eviction fence exists * @uq_mgr: the usermode queue manager for this process * @evf_mgr: the eviction fence manager to check and rearm * * Ensures that a valid and not yet signaled eviction fence is attached to the * usermode queue before any queue operations proceed. If it is signalled, then * rearm a new eviction fence. */voidamdgpu_userq_ensure_ev_fence(struct amdgpu_userq_mgr *uq_mgr,			     struct amdgpu_eviction_fence_mgr *evf_mgr){	struct dma_fence *ev_fence; retry:	/* Flush any pending resume work to create ev_fence */	flush_delayed_work(&uq_mgr->resume_work); 	mutex_lock(&uq_mgr->userq_mutex);	ev_fence = amdgpu_evf_mgr_get_fence(evf_mgr);	if (dma_fence_is_signaled(ev_fence)) {		dma_fence_put(ev_fence);		mutex_unlock(&uq_mgr->userq_mutex);		/*		 * Looks like there was no pending resume work,		 * add one now to create a valid eviction fence		 */		schedule_delayed_work(&uq_mgr->resume_work, 0);		goto retry;	}	dma_fence_put(ev_fence);}   static intamdgpu_userq_get_doorbell_index(struct amdgpu_userq_mgr *uq_mgr,				struct amdgpu_db_info *db_info,				struct drm_file *filp,				u64 *index){	u64 doorbell_index;	struct drm_gem_object *gobj;	struct amdgpu_userq_obj *db_obj = db_info->db_obj;	int r, db_size; 	gobj = drm_gem_object_lookup(filp, db_info->doorbell_handle);	if (gobj == NULL) {		drm_file_err(uq_mgr->file, "Can't find GEM object for doorbell\n");		return -EINVAL;	} 	db_obj->obj = amdgpu_bo_ref(gem_to_amdgpu_bo(gobj));	drm_gem_object_put(gobj); 	r = amdgpu_bo_reserve(db_obj->obj, true);	if (r) {		drm_file_err(uq_mgr->file, "[Usermode queues] Failed to pin doorbell object\n");		goto unref_bo;	} 	/* Pin the BO before generating the index, unpin in queue destroy */	r = amdgpu_bo_pin(db_obj->obj, AMDGPU_GEM_DOMAIN_DOORBELL);	if (r) {		drm_file_err(uq_mgr->file, "[Usermode queues] Failed to pin doorbell object\n");		goto unresv_bo;	} 	switch (db_info->queue_type) {	case AMDGPU_HW_IP_GFX:

Lines 1–500 of 1567. Load more