// SPDX-License-Identifier: GPL-2.0 OR MIT/* * Copyright 2014-2022 Advanced Micro Devices, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. * */ #include <linux/ratelimit.h>#include <linux/printk.h>#include <linux/slab.h>#include <linux/list.h>#include <linux/types.h>#include <linux/bitops.h>#include <linux/sched.h>#include "kfd_priv.h"#include "kfd_device_queue_manager.h"#include "kfd_mqd_manager.h"#include "cik_regs.h"#include "kfd_kernel_queue.h"#include "amdgpu_amdkfd.h"#include "amdgpu_reset.h"#include "amdgpu_sdma.h"#include "amdgpu_ring.h"#include "amdgpu_mes.h"#include "kfd_debug.h" /* Size of the per-pipe EOP queue */#define CIK_HPD_EOP_BYTES_LOG2 11#define CIK_HPD_EOP_BYTES (1U << CIK_HPD_EOP_BYTES_LOG2)/* See unmap_queues_cpsch() */#define USE_DEFAULT_GRACE_PERIOD 0xffffffff static int set_pasid_vmid_mapping(struct device_queue_manager *dqm,				  u32 pasid, unsigned int vmid); static int execute_queues_cpsch(struct device_queue_manager *dqm,				enum kfd_unmap_queues_filter filter,				uint32_t filter_param,				uint32_t grace_period);static int unmap_queues_cpsch(struct device_queue_manager *dqm,				enum kfd_unmap_queues_filter filter,				uint32_t filter_param,				uint32_t grace_period,				bool reset); static int map_queues_cpsch(struct device_queue_manager *dqm); static void deallocate_sdma_queue(struct device_queue_manager *dqm,				struct queue *q); static inline void deallocate_hqd(struct device_queue_manager *dqm,				struct queue *q);static int allocate_hqd(struct device_queue_manager *dqm, struct queue *q);static int allocate_sdma_queue(struct device_queue_manager *dqm,				struct queue *q, const uint32_t *restore_sdma_id); static int reset_queues_on_hws_hang(struct device_queue_manager *dqm, bool is_sdma);static struct queue *find_queue_by_doorbell_offset(struct device_queue_manager *dqm,						   u32 doorbell_offset);static void set_queue_as_reset(struct device_queue_manager *dqm, struct queue *q,			       struct qcm_process_device *qpd);static int reset_queues_mes(struct device_queue_manager *dqm, struct queue *q); static inlineenum KFD_MQD_TYPE get_mqd_type_from_queue_type(enum kfd_queue_type type){	if (type == KFD_QUEUE_TYPE_SDMA || type == KFD_QUEUE_TYPE_SDMA_XGMI)		return KFD_MQD_TYPE_SDMA;	return KFD_MQD_TYPE_CP;} static bool is_pipe_enabled(struct device_queue_manager *dqm, int mec, int pipe){	int i;	int pipe_offset = (mec * dqm->dev->kfd->shared_resources.num_pipe_per_mec		+ pipe) * dqm->dev->kfd->shared_resources.num_queue_per_pipe; 	/* queue is available for KFD usage if bit is 1 */	for (i = 0; i <  dqm->dev->kfd->shared_resources.num_queue_per_pipe; ++i)		if (test_bit(pipe_offset + i,			      dqm->dev->kfd->shared_resources.cp_queue_bitmap))			return true;	return false;} unsigned int get_cp_queues_num(struct device_queue_manager *dqm){	return bitmap_weight(dqm->dev->kfd->shared_resources.cp_queue_bitmap,				AMDGPU_MAX_QUEUES);} unsigned int get_queues_per_pipe(struct device_queue_manager *dqm){	return dqm->dev->kfd->shared_resources.num_queue_per_pipe;} unsigned int get_pipes_per_mec(struct device_queue_manager *dqm){	return dqm->dev->kfd->shared_resources.num_pipe_per_mec;} static unsigned int get_num_all_sdma_engines(struct device_queue_manager *dqm){	return kfd_get_num_sdma_engines(dqm->dev) +		kfd_get_num_xgmi_sdma_engines(dqm->dev);} unsigned int get_num_sdma_queues(struct device_queue_manager *dqm){	return kfd_get_num_sdma_engines(dqm->dev) *		dqm->dev->kfd->device_info.num_sdma_queues_per_engine;} unsigned int get_num_xgmi_sdma_queues(struct device_queue_manager *dqm){	return kfd_get_num_xgmi_sdma_engines(dqm->dev) *		dqm->dev->kfd->device_info.num_sdma_queues_per_engine;} static void init_sdma_bitmaps(struct device_queue_manager *dqm){	bitmap_zero(dqm->sdma_bitmap, KFD_MAX_SDMA_QUEUES);	bitmap_set(dqm->sdma_bitmap, 0, get_num_sdma_queues(dqm)); 	bitmap_zero(dqm->xgmi_sdma_bitmap, KFD_MAX_SDMA_QUEUES);	bitmap_set(dqm->xgmi_sdma_bitmap, 0, get_num_xgmi_sdma_queues(dqm)); 	/* Mask out the reserved queues */	bitmap_clear(dqm->sdma_bitmap, 0, kfd_get_num_sdma_engines(dqm->dev) *			dqm->dev->kfd->device_info.num_reserved_sdma_queues_per_engine);	bitmap_clear(dqm->xgmi_sdma_bitmap, 0, kfd_get_num_xgmi_sdma_engines(dqm->dev) *			dqm->dev->kfd->device_info.num_reserved_sdma_queues_per_engine);} void program_sh_mem_settings(struct device_queue_manager *dqm,					struct qcm_process_device *qpd){	uint32_t xcc_mask = dqm->dev->xcc_mask;	int xcc_id; 	for_each_inst(xcc_id, xcc_mask)		dqm->dev->kfd2kgd->program_sh_mem_settings(			dqm->dev->adev, qpd->vmid, qpd->sh_mem_config,			qpd->sh_mem_ape1_base, qpd->sh_mem_ape1_limit,			qpd->sh_mem_bases, xcc_id);} static void kfd_hws_hang(struct device_queue_manager *dqm){	struct device_process_node *cur;	struct qcm_process_device *qpd;	struct queue *q; 	/* Mark all device queues as reset. */	list_for_each_entry(cur, &dqm->queues, list) {		qpd = cur->qpd;		list_for_each_entry(q, &qpd->queues_list, list) {			struct kfd_process_device *pdd = qpd_to_pdd(qpd); 			pdd->has_reset_queue = true;		}	} 	/*	 * Issue a GPU reset if HWS is unresponsive	 */	amdgpu_amdkfd_gpu_reset(dqm->dev->adev);} static int convert_to_amdgpu_ring_type(int queue_type){	int amdgpu_ring_type; 	switch (queue_type) {	case KFD_QUEUE_TYPE_COMPUTE:		amdgpu_ring_type = AMDGPU_RING_TYPE_COMPUTE;		break;	case KFD_QUEUE_TYPE_SDMA:		amdgpu_ring_type = AMDGPU_RING_TYPE_SDMA;		break;	default:		WARN(1, "Invalid queue type %d", queue_type);		amdgpu_ring_type = -EINVAL;		break;	} 	return amdgpu_ring_type;} static int add_queue_mes(struct device_queue_manager *dqm, struct queue *q,			 struct qcm_process_device *qpd){	struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev;	struct kfd_process_device *pdd = qpd_to_pdd(qpd);	struct mes_add_queue_input queue_input;	int r, queue_type;	uint64_t wptr_addr_off; 	if (!dqm->sched_running || dqm->sched_halt)		return 0;	if (!down_read_trylock(&adev->reset_domain->sem))		return -EIO; 	memset(&queue_input, 0x0, sizeof(struct mes_add_queue_input));	queue_input.process_id = pdd->pasid;	queue_input.page_table_base_addr =  qpd->page_table_base;	queue_input.process_va_start = 0;	queue_input.process_va_end = adev->vm_manager.max_pfn - 1;	/* MES unit for quantum is 100ns */	queue_input.process_quantum = KFD_MES_PROCESS_QUANTUM;  /* Equivalent to 10ms. */	queue_input.process_context_addr = pdd->proc_ctx_gpu_addr;	queue_input.process_context_array_index = pdd->proc_ctx_array_index;	queue_input.gang_quantum = KFD_MES_GANG_QUANTUM; /* Equivalent to 1ms */	queue_input.gang_context_addr = q->gang_ctx_gpu_addr;	queue_input.gang_context_array_index = q->gang_ctx_array_index;	queue_input.inprocess_gang_priority = q->properties.priority;	queue_input.gang_global_priority_level =					AMDGPU_MES_PRIORITY_LEVEL_NORMAL;	queue_input.doorbell_offset = q->properties.doorbell_off;	queue_input.mqd_addr = q->gart_mqd_addr;	queue_input.wptr_addr = (uint64_t)q->properties.write_ptr; 	wptr_addr_off = (uint64_t)q->properties.write_ptr & (PAGE_SIZE - 1);	queue_input.wptr_mc_addr = amdgpu_bo_gpu_offset(q->properties.wptr_bo) + wptr_addr_off; 	queue_input.is_kfd_process = 1;	queue_input.is_aql_queue = (q->properties.format == KFD_QUEUE_FORMAT_AQL);	queue_input.queue_size = q->properties.queue_size >> 2; 	queue_input.paging = false;	queue_input.tba_addr = qpd->tba_addr;	queue_input.tma_addr = qpd->tma_addr;	queue_input.trap_en = !kfd_dbg_has_cwsr_workaround(q->device);	queue_input.skip_process_ctx_clear =		qpd->pqm->process->runtime_info.runtime_state == DEBUG_RUNTIME_STATE_ENABLED &&						(qpd->pqm->process->debug_trap_enabled ||						 kfd_dbg_has_ttmps_always_setup(q->device)); 	queue_type = convert_to_amdgpu_ring_type(q->properties.type);	if (queue_type < 0) {		dev_err(adev->dev, "Queue type not supported with MES, queue:%d\n",			q->properties.type);		up_read(&adev->reset_domain->sem);		return -EINVAL;	}	queue_input.queue_type = (uint32_t)queue_type; 	queue_input.exclusively_scheduled = q->properties.is_gws;	queue_input.sh_mem_config_data = qpd->sh_mem_config;	queue_input.vm_cntx_cntl = qpd->vm_cntx_cntl;	queue_input.xcc_id = ffs(dqm->dev->xcc_mask) - 1; 	amdgpu_mes_lock(&adev->mes);	r = adev->mes.funcs->add_hw_queue(&adev->mes, &queue_input);	amdgpu_mes_unlock(&adev->mes);	up_read(&adev->reset_domain->sem);	if (r) {		dev_err(adev->dev, "failed to add hardware queue to MES, doorbell=0x%x\n",			q->properties.doorbell_off);		dev_err(adev->dev, "MES might be in unrecoverable state, issue a GPU reset\n");		kfd_hws_hang(dqm);	} 	return r;} static int remove_queue_mes_on_reset_option(struct device_queue_manager *dqm, struct queue *q,					    struct qcm_process_device *qpd,					    bool is_for_reset,					    bool flush_mes_queue){	struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev;	int r;	struct mes_remove_queue_input queue_input; 	/* queue was already removed during reset */	if (q->properties.is_reset)		return 0; 	if (!dqm->sched_running || dqm->sched_halt)		return 0;	if (!down_read_trylock(&adev->reset_domain->sem))		return -EIO; 	memset(&queue_input, 0x0, sizeof(struct mes_remove_queue_input));	queue_input.doorbell_offset = q->properties.doorbell_off;	queue_input.gang_context_addr = q->gang_ctx_gpu_addr;	queue_input.queue_type = convert_to_amdgpu_ring_type(q->properties.type);	queue_input.remove_queue_after_reset = flush_mes_queue;	queue_input.xcc_id = ffs(dqm->dev->xcc_mask) - 1;	queue_input.gang_context_array_index = q->gang_ctx_array_index; 	amdgpu_mes_lock(&adev->mes);	r = adev->mes.funcs->remove_hw_queue(&adev->mes, &queue_input);	amdgpu_mes_unlock(&adev->mes);	up_read(&adev->reset_domain->sem); 	/* If is_for_reset set, it is a mes internal cleanup */	if (!r || is_for_reset)		return r; 	/* remove_hw_queue failure indicates a queue hang. reset the queue */	r = reset_queues_mes(dqm, q);	if (r && amdgpu_gpu_recovery) {		dev_err(adev->dev, "failed to remove queue from MES, doorbell=0x%x\n",			q->properties.doorbell_off);		dev_err(adev->dev, "MES might be in unrecoverable state, issue a GPU reset\n");		kfd_hws_hang(dqm);	} 	return r;} static void set_perfcount(struct device_queue_manager *dqm, int enable){	struct device_process_node *cur;	struct qcm_process_device *qpd;	struct queue *q;	struct mqd_update_info minfo = { 0 }; 	if (!dqm)		return; 	minfo.update_flag = (enable == 1 ? UPDATE_FLAG_PERFCOUNT_ENABLE :						 UPDATE_FLAG_PERFCOUNT_DISABLE);	dqm_lock(dqm);	list_for_each_entry(cur, &dqm->queues, list) {		qpd = cur->qpd;		list_for_each_entry(q, &qpd->queues_list, list) {			pqm_update_mqd(qpd->pqm, q->properties.queue_id,						&minfo);		}	}	dqm_unlock(dqm);} static int remove_queue_mes(struct device_queue_manager *dqm, struct queue *q,			    struct qcm_process_device *qpd){	return remove_queue_mes_on_reset_option(dqm, q, qpd, false, false);} static int remove_all_kfd_queues_mes(struct device_queue_manager *dqm){	struct device_process_node *cur;	struct device *dev = dqm->dev->adev->dev;	struct qcm_process_device *qpd;	struct queue *q;	int retval = 0; 	list_for_each_entry(cur, &dqm->queues, list) {		qpd = cur->qpd;		list_for_each_entry(q, &qpd->queues_list, list) {			if (q->properties.is_active) {				retval = remove_queue_mes(dqm, q, qpd);				if (retval) {					dev_err(dev, "%s: Failed to remove queue %d for dev %d",						__func__,						q->properties.queue_id,						dqm->dev->id);					return retval;				}			}		}	} 	return retval;} static int add_all_kfd_queues_mes(struct device_queue_manager *dqm){	struct device_process_node *cur;	struct device *dev = dqm->dev->adev->dev;	struct qcm_process_device *qpd;	struct queue *q;	int retval = 0; 	list_for_each_entry(cur, &dqm->queues, list) {		qpd = cur->qpd;		list_for_each_entry(q, &qpd->queues_list, list) {			if (!q->properties.is_active)				continue;			retval = add_queue_mes(dqm, q, qpd);			if (retval) {				dev_err(dev, "%s: Failed to add queue %d for dev %d",					__func__,					q->properties.queue_id,					dqm->dev->id);				return retval;			}		}	} 	return retval;} static int reset_queue_mes(struct device_queue_manager *dqm, struct queue *q,			   int queue_type, int pipe, int queue, unsigned int db){	struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev;	struct kfd_process_device *pdd;	bool use_mmio = adev->gfx.mec.use_mmio_for_reset;	int r; 	pdd = kfd_get_process_device_data(q->device, q->process);	if (!pdd)		return -ENODEV; 	if (use_mmio)		r = amdgpu_mes_reset_queue_mmio(adev, queue_type, 0, 1, pipe, queue,						ffs(dqm->dev->xcc_mask) - 1);	else		r = amdgpu_mes_reset_user_queue(adev, queue_type, db,						ffs(dqm->dev->xcc_mask) - 1);	if (r)		return r;	/* Proceed remove_queue with reset=true */	remove_queue_mes_on_reset_option(dqm, q, &pdd->qpd, true, true);	set_queue_as_reset(dqm, q, &pdd->qpd);	return 0;} int kfd_reset_queue_mes(struct device_queue_manager *dqm, int queue_type,			int pipe, int queue, unsigned int db){	struct queue *q; 	q = find_queue_by_doorbell_offset(dqm, db);	if (!q)		return 0;	return reset_queue_mes(dqm, q, queue_type, pipe, queue, db);} static int reset_queues_mes(struct device_queue_manager *dqm, struct queue *q){	struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev;	struct drm_wedge_task_info *info = NULL;	struct amdgpu_task_info *ti = NULL;	struct kfd_process_device *pdd;	unsigned int num_hung = 0;	int r = 0;	struct mes_remove_queue_input queue_input; 	if (!amdgpu_mes_queue_reset_by_mes_supported(adev)) {		r = -ENOTRECOVERABLE;		goto fail;	} 	/* reset should be used only in dqm locked queue reset */	if (WARN_ON(dqm->detect_hang_count > 0))		return 0; 	if (!amdgpu_gpu_recovery) {		r = -ENOTRECOVERABLE;		goto fail;	} 	memset(&queue_input, 0x0, sizeof(struct mes_remove_queue_input));	queue_input.doorbell_offset = q->properties.doorbell_off;	queue_input.gang_context_addr = q->gang_ctx_gpu_addr;	queue_input.queue_type = convert_to_amdgpu_ring_type(q->properties.type);	queue_input.remove_queue_after_reset = false;	queue_input.xcc_id = ffs(dqm->dev->xcc_mask) - 1;	/* pass the known bad queue info to the reset function */	r = amdgpu_gfx_reset_mes_compute(adev, NULL, NULL, NULL, &num_hung, &queue_input);	if (r)		goto fail;	pdd = kfd_get_process_device_data(q->device, q->process);	if (pdd) {		ti = amdgpu_vm_get_task_info_pasid(adev, pdd->pasid);		if (ti) {			amdgpu_vm_print_task_info(adev, ti);			info = &ti->task;		}	} 	dqm->detect_hang_count = num_hung;	/* When MES doesn't detect any queue hang, no reset happens. Don't signal reset	 * event.	 */	if (dqm->detect_hang_count) {		kfd_signal_reset_event(dqm->dev);		if (pdd && pdd->has_reset_queue) {

Lines 1–500 of 4058. Load more