/* * Copyright 2019 Advanced Micro Devices, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. * */ #include <linux/firmware.h>#include <drm/drm_exec.h> #include "amdgpu_mes.h"#include "amdgpu.h"#include "soc15_common.h"#include "amdgpu_mes_ctx.h" #define AMDGPU_MES_MAX_NUM_OF_QUEUES_PER_PROCESS 1024#define AMDGPU_ONE_DOORBELL_SIZE 8 int amdgpu_mes_doorbell_process_slice(struct amdgpu_device *adev){	return roundup(AMDGPU_ONE_DOORBELL_SIZE *		       AMDGPU_MES_MAX_NUM_OF_QUEUES_PER_PROCESS,		       PAGE_SIZE);} static int amdgpu_mes_kernel_doorbell_get(struct amdgpu_device *adev,					 int ip_type, uint64_t *doorbell_index){	unsigned int offset, found;	struct amdgpu_mes *mes = &adev->mes; 	if (ip_type == AMDGPU_RING_TYPE_SDMA)		offset = adev->doorbell_index.sdma_engine[0];	else		offset = 0; 	found = find_next_zero_bit(mes->doorbell_bitmap, mes->num_mes_dbs, offset);	if (found >= mes->num_mes_dbs) {		DRM_WARN("No doorbell available\n");		return -ENOSPC;	} 	set_bit(found, mes->doorbell_bitmap); 	/* Get the absolute doorbell index on BAR */	*doorbell_index = mes->db_start_dw_offset + found * 2;	return 0;} static void amdgpu_mes_kernel_doorbell_free(struct amdgpu_device *adev,					   uint32_t doorbell_index){	unsigned int old, rel_index;	struct amdgpu_mes *mes = &adev->mes; 	/* Find the relative index of the doorbell in this object */	rel_index = (doorbell_index - mes->db_start_dw_offset) / 2;	old = test_and_clear_bit(rel_index, mes->doorbell_bitmap);	WARN_ON(!old);} static int amdgpu_mes_doorbell_init(struct amdgpu_device *adev){	int i;	struct amdgpu_mes *mes = &adev->mes; 	/* Bitmap for dynamic allocation of kernel doorbells */	mes->doorbell_bitmap = bitmap_zalloc(PAGE_SIZE / sizeof(u32), GFP_KERNEL);	if (!mes->doorbell_bitmap) {		DRM_ERROR("Failed to allocate MES doorbell bitmap\n");		return -ENOMEM;	} 	mes->num_mes_dbs = PAGE_SIZE / AMDGPU_ONE_DOORBELL_SIZE;	for (i = 0; i < AMDGPU_MES_PRIORITY_NUM_LEVELS; i++) {		adev->mes.aggregated_doorbells[i] = mes->db_start_dw_offset + i * 2;		set_bit(i, mes->doorbell_bitmap);	} 	return 0;} static int amdgpu_mes_event_log_init(struct amdgpu_device *adev){	int r; 	if (!amdgpu_mes_log_enable)		return 0; 	r = amdgpu_bo_create_kernel(adev, adev->mes.event_log_size, PAGE_SIZE,				    AMDGPU_GEM_DOMAIN_VRAM,				    &adev->mes.event_log_gpu_obj,				    &adev->mes.event_log_gpu_addr,				    &adev->mes.event_log_cpu_addr);	if (r) {		dev_warn(adev->dev, "failed to create MES event log buffer (%d)", r);		return r;	} 	memset(adev->mes.event_log_cpu_addr, 0, adev->mes.event_log_size); 	return  0; } static void amdgpu_mes_doorbell_free(struct amdgpu_device *adev){	bitmap_free(adev->mes.doorbell_bitmap);} int amdgpu_mes_init(struct amdgpu_device *adev){	int i, r; 	adev->mes.adev = adev; 	idr_init(&adev->mes.pasid_idr);	idr_init(&adev->mes.gang_id_idr);	idr_init(&adev->mes.queue_id_idr);	ida_init(&adev->mes.doorbell_ida);	spin_lock_init(&adev->mes.queue_id_lock);	mutex_init(&adev->mes.mutex_hidden); 	for (i = 0; i < AMDGPU_MAX_MES_PIPES; i++)		spin_lock_init(&adev->mes.ring_lock[i]); 	adev->mes.total_max_queue = AMDGPU_FENCE_MES_QUEUE_ID_MASK;	adev->mes.vmid_mask_mmhub = 0xffffff00;	adev->mes.vmid_mask_gfxhub = 0xffffff00; 	for (i = 0; i < AMDGPU_MES_MAX_COMPUTE_PIPES; i++) {		/* use only 1st MEC pipes */		if (i >= adev->gfx.mec.num_pipe_per_mec)			continue;		adev->mes.compute_hqd_mask[i] = 0xc;	} 	for (i = 0; i < AMDGPU_MES_MAX_GFX_PIPES; i++)		adev->mes.gfx_hqd_mask[i] = i ? 0 : 0xfffffffe; 	for (i = 0; i < AMDGPU_MES_MAX_SDMA_PIPES; i++) {		if (amdgpu_ip_version(adev, SDMA0_HWIP, 0) <		    IP_VERSION(6, 0, 0))			adev->mes.sdma_hqd_mask[i] = i ? 0 : 0x3fc;		/* zero sdma_hqd_mask for non-existent engine */		else if (adev->sdma.num_instances == 1)			adev->mes.sdma_hqd_mask[i] = i ? 0 : 0xfc;		else			adev->mes.sdma_hqd_mask[i] = 0xfc;	} 	for (i = 0; i < AMDGPU_MAX_MES_PIPES; i++) {		r = amdgpu_device_wb_get(adev, &adev->mes.sch_ctx_offs[i]);		if (r) {			dev_err(adev->dev,				"(%d) ring trail_fence_offs wb alloc failed\n",				r);			goto error;		}		adev->mes.sch_ctx_gpu_addr[i] =			adev->wb.gpu_addr + (adev->mes.sch_ctx_offs[i] * 4);		adev->mes.sch_ctx_ptr[i] =			(uint64_t *)&adev->wb.wb[adev->mes.sch_ctx_offs[i]]; 		r = amdgpu_device_wb_get(adev,				 &adev->mes.query_status_fence_offs[i]);		if (r) {			dev_err(adev->dev,			      "(%d) query_status_fence_offs wb alloc failed\n",			      r);			goto error;		}		adev->mes.query_status_fence_gpu_addr[i] = adev->wb.gpu_addr +			(adev->mes.query_status_fence_offs[i] * 4);		adev->mes.query_status_fence_ptr[i] =			(uint64_t *)&adev->wb.wb[adev->mes.query_status_fence_offs[i]];	} 	r = amdgpu_mes_doorbell_init(adev);	if (r)		goto error; 	r = amdgpu_mes_event_log_init(adev);	if (r)		goto error_doorbell; 	return 0; error_doorbell:	amdgpu_mes_doorbell_free(adev);error:	for (i = 0; i < AMDGPU_MAX_MES_PIPES; i++) {		if (adev->mes.sch_ctx_ptr[i])			amdgpu_device_wb_free(adev, adev->mes.sch_ctx_offs[i]);		if (adev->mes.query_status_fence_ptr[i])			amdgpu_device_wb_free(adev,				      adev->mes.query_status_fence_offs[i]);	} 	idr_destroy(&adev->mes.pasid_idr);	idr_destroy(&adev->mes.gang_id_idr);	idr_destroy(&adev->mes.queue_id_idr);	ida_destroy(&adev->mes.doorbell_ida);	mutex_destroy(&adev->mes.mutex_hidden);	return r;} void amdgpu_mes_fini(struct amdgpu_device *adev){	int i; 	amdgpu_bo_free_kernel(&adev->mes.event_log_gpu_obj,			      &adev->mes.event_log_gpu_addr,			      &adev->mes.event_log_cpu_addr); 	for (i = 0; i < AMDGPU_MAX_MES_PIPES; i++) {		if (adev->mes.sch_ctx_ptr[i])			amdgpu_device_wb_free(adev, adev->mes.sch_ctx_offs[i]);		if (adev->mes.query_status_fence_ptr[i])			amdgpu_device_wb_free(adev,				      adev->mes.query_status_fence_offs[i]);	} 	amdgpu_mes_doorbell_free(adev); 	idr_destroy(&adev->mes.pasid_idr);	idr_destroy(&adev->mes.gang_id_idr);	idr_destroy(&adev->mes.queue_id_idr);	ida_destroy(&adev->mes.doorbell_ida);	mutex_destroy(&adev->mes.mutex_hidden);} static void amdgpu_mes_queue_free_mqd(struct amdgpu_mes_queue *q){	amdgpu_bo_free_kernel(&q->mqd_obj,			      &q->mqd_gpu_addr,			      &q->mqd_cpu_ptr);} int amdgpu_mes_create_process(struct amdgpu_device *adev, int pasid,			      struct amdgpu_vm *vm){	struct amdgpu_mes_process *process;	int r; 	/* allocate the mes process buffer */	process = kzalloc(sizeof(struct amdgpu_mes_process), GFP_KERNEL);	if (!process) {		DRM_ERROR("no more memory to create mes process\n");		return -ENOMEM;	} 	/* allocate the process context bo and map it */	r = amdgpu_bo_create_kernel(adev, AMDGPU_MES_PROC_CTX_SIZE, PAGE_SIZE,				    AMDGPU_GEM_DOMAIN_GTT,				    &process->proc_ctx_bo,				    &process->proc_ctx_gpu_addr,				    &process->proc_ctx_cpu_ptr);	if (r) {		DRM_ERROR("failed to allocate process context bo\n");		goto clean_up_memory;	}	memset(process->proc_ctx_cpu_ptr, 0, AMDGPU_MES_PROC_CTX_SIZE); 	/*	 * Avoid taking any other locks under MES lock to avoid circular	 * lock dependencies.	 */	amdgpu_mes_lock(&adev->mes); 	/* add the mes process to idr list */	r = idr_alloc(&adev->mes.pasid_idr, process, pasid, pasid + 1,		      GFP_KERNEL);	if (r < 0) {		DRM_ERROR("failed to lock pasid=%d\n", pasid);		goto clean_up_ctx;	} 	INIT_LIST_HEAD(&process->gang_list);	process->vm = vm;	process->pasid = pasid;	process->process_quantum = adev->mes.default_process_quantum;	process->pd_gpu_addr = amdgpu_bo_gpu_offset(vm->root.bo); 	amdgpu_mes_unlock(&adev->mes);	return 0; clean_up_ctx:	amdgpu_mes_unlock(&adev->mes);	amdgpu_bo_free_kernel(&process->proc_ctx_bo,			      &process->proc_ctx_gpu_addr,			      &process->proc_ctx_cpu_ptr);clean_up_memory:	kfree(process);	return r;} void amdgpu_mes_destroy_process(struct amdgpu_device *adev, int pasid){	struct amdgpu_mes_process *process;	struct amdgpu_mes_gang *gang, *tmp1;	struct amdgpu_mes_queue *queue, *tmp2;	struct mes_remove_queue_input queue_input;	unsigned long flags;	int r; 	/*	 * Avoid taking any other locks under MES lock to avoid circular	 * lock dependencies.	 */	amdgpu_mes_lock(&adev->mes); 	process = idr_find(&adev->mes.pasid_idr, pasid);	if (!process) {		DRM_WARN("pasid %d doesn't exist\n", pasid);		amdgpu_mes_unlock(&adev->mes);		return;	} 	/* Remove all queues from hardware */	list_for_each_entry_safe(gang, tmp1, &process->gang_list, list) {		list_for_each_entry_safe(queue, tmp2, &gang->queue_list, list) {			spin_lock_irqsave(&adev->mes.queue_id_lock, flags);			idr_remove(&adev->mes.queue_id_idr, queue->queue_id);			spin_unlock_irqrestore(&adev->mes.queue_id_lock, flags); 			queue_input.doorbell_offset = queue->doorbell_off;			queue_input.gang_context_addr = gang->gang_ctx_gpu_addr; 			r = adev->mes.funcs->remove_hw_queue(&adev->mes,							     &queue_input);			if (r)				DRM_WARN("failed to remove hardware queue\n");		} 		idr_remove(&adev->mes.gang_id_idr, gang->gang_id);	} 	idr_remove(&adev->mes.pasid_idr, pasid);	amdgpu_mes_unlock(&adev->mes); 	/* free all memory allocated by the process */	list_for_each_entry_safe(gang, tmp1, &process->gang_list, list) {		/* free all queues in the gang */		list_for_each_entry_safe(queue, tmp2, &gang->queue_list, list) {			amdgpu_mes_queue_free_mqd(queue);			list_del(&queue->list);			kfree(queue);		}		amdgpu_bo_free_kernel(&gang->gang_ctx_bo,				      &gang->gang_ctx_gpu_addr,				      &gang->gang_ctx_cpu_ptr);		list_del(&gang->list);		kfree(gang); 	}	amdgpu_bo_free_kernel(&process->proc_ctx_bo,			      &process->proc_ctx_gpu_addr,			      &process->proc_ctx_cpu_ptr);	kfree(process);} int amdgpu_mes_add_gang(struct amdgpu_device *adev, int pasid,			struct amdgpu_mes_gang_properties *gprops,			int *gang_id){	struct amdgpu_mes_process *process;	struct amdgpu_mes_gang *gang;	int r; 	/* allocate the mes gang buffer */	gang = kzalloc(sizeof(struct amdgpu_mes_gang), GFP_KERNEL);	if (!gang) {		return -ENOMEM;	} 	/* allocate the gang context bo and map it to cpu space */	r = amdgpu_bo_create_kernel(adev, AMDGPU_MES_GANG_CTX_SIZE, PAGE_SIZE,				    AMDGPU_GEM_DOMAIN_GTT,				    &gang->gang_ctx_bo,				    &gang->gang_ctx_gpu_addr,				    &gang->gang_ctx_cpu_ptr);	if (r) {		DRM_ERROR("failed to allocate process context bo\n");		goto clean_up_mem;	}	memset(gang->gang_ctx_cpu_ptr, 0, AMDGPU_MES_GANG_CTX_SIZE); 	/*	 * Avoid taking any other locks under MES lock to avoid circular	 * lock dependencies.	 */	amdgpu_mes_lock(&adev->mes); 	process = idr_find(&adev->mes.pasid_idr, pasid);	if (!process) {		DRM_ERROR("pasid %d doesn't exist\n", pasid);		r = -EINVAL;		goto clean_up_ctx;	} 	/* add the mes gang to idr list */	r = idr_alloc(&adev->mes.gang_id_idr, gang, 1, 0,		      GFP_KERNEL);	if (r < 0) {		DRM_ERROR("failed to allocate idr for gang\n");		goto clean_up_ctx;	} 	gang->gang_id = r;	*gang_id = r; 	INIT_LIST_HEAD(&gang->queue_list);	gang->process = process;	gang->priority = gprops->priority;	gang->gang_quantum = gprops->gang_quantum ?		gprops->gang_quantum : adev->mes.default_gang_quantum;	gang->global_priority_level = gprops->global_priority_level;	gang->inprocess_gang_priority = gprops->inprocess_gang_priority;	list_add_tail(&gang->list, &process->gang_list); 	amdgpu_mes_unlock(&adev->mes);	return 0; clean_up_ctx:	amdgpu_mes_unlock(&adev->mes);	amdgpu_bo_free_kernel(&gang->gang_ctx_bo,			      &gang->gang_ctx_gpu_addr,			      &gang->gang_ctx_cpu_ptr);clean_up_mem:	kfree(gang);	return r;} int amdgpu_mes_remove_gang(struct amdgpu_device *adev, int gang_id){	struct amdgpu_mes_gang *gang; 	/*	 * Avoid taking any other locks under MES lock to avoid circular	 * lock dependencies.	 */	amdgpu_mes_lock(&adev->mes); 	gang = idr_find(&adev->mes.gang_id_idr, gang_id);	if (!gang) {		DRM_ERROR("gang id %d doesn't exist\n", gang_id);		amdgpu_mes_unlock(&adev->mes);		return -EINVAL;	} 	if (!list_empty(&gang->queue_list)) {		DRM_ERROR("queue list is not empty\n");		amdgpu_mes_unlock(&adev->mes);		return -EBUSY;	} 	idr_remove(&adev->mes.gang_id_idr, gang->gang_id);	list_del(&gang->list);	amdgpu_mes_unlock(&adev->mes); 	amdgpu_bo_free_kernel(&gang->gang_ctx_bo,			      &gang->gang_ctx_gpu_addr,			      &gang->gang_ctx_cpu_ptr); 	kfree(gang); 	return 0;} int amdgpu_mes_suspend(struct amdgpu_device *adev){	struct mes_suspend_gang_input input;	int r; 	if (!amdgpu_mes_suspend_resume_all_supported(adev))		return 0; 	memset(&input, 0x0, sizeof(struct mes_suspend_gang_input));	input.suspend_all_gangs = 1; 	/*	 * Avoid taking any other locks under MES lock to avoid circular	 * lock dependencies.

Lines 1–500 of 1732. Load more