/* * Copyright 2019 Advanced Micro Devices, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. * */ #include "amdgpu_mes.h"#include "amdgpu.h"#include "soc15_common.h"#include "amdgpu_mes_ctx.h" #define AMDGPU_MES_MAX_NUM_OF_QUEUES_PER_PROCESS 1024#define AMDGPU_ONE_DOORBELL_SIZE 8 int amdgpu_mes_doorbell_process_slice(struct amdgpu_device *adev){	return roundup(AMDGPU_ONE_DOORBELL_SIZE *		       AMDGPU_MES_MAX_NUM_OF_QUEUES_PER_PROCESS,		       PAGE_SIZE);} int amdgpu_mes_alloc_process_doorbells(struct amdgpu_device *adev,				      unsigned int *doorbell_index){	int r = ida_simple_get(&adev->mes.doorbell_ida, 2,			       adev->mes.max_doorbell_slices,			       GFP_KERNEL);	if (r > 0)		*doorbell_index = r; 	return r;} void amdgpu_mes_free_process_doorbells(struct amdgpu_device *adev,				      unsigned int doorbell_index){	if (doorbell_index)		ida_simple_remove(&adev->mes.doorbell_ida, doorbell_index);} unsigned int amdgpu_mes_get_doorbell_dw_offset_in_bar(					struct amdgpu_device *adev,					uint32_t doorbell_index,					unsigned int doorbell_id){	return ((doorbell_index *		amdgpu_mes_doorbell_process_slice(adev)) / sizeof(u32) +		doorbell_id * 2);} static int amdgpu_mes_queue_doorbell_get(struct amdgpu_device *adev,					 struct amdgpu_mes_process *process,					 int ip_type, uint64_t *doorbell_index){	unsigned int offset, found; 	if (ip_type == AMDGPU_RING_TYPE_SDMA) {		offset = adev->doorbell_index.sdma_engine[0];		found = find_next_zero_bit(process->doorbell_bitmap,					   AMDGPU_MES_MAX_NUM_OF_QUEUES_PER_PROCESS,					   offset);	} else {		found = find_first_zero_bit(process->doorbell_bitmap,					    AMDGPU_MES_MAX_NUM_OF_QUEUES_PER_PROCESS);	} 	if (found >= AMDGPU_MES_MAX_NUM_OF_QUEUES_PER_PROCESS) {		DRM_WARN("No doorbell available\n");		return -ENOSPC;	} 	set_bit(found, process->doorbell_bitmap); 	*doorbell_index = amdgpu_mes_get_doorbell_dw_offset_in_bar(adev,				process->doorbell_index, found); 	return 0;} static void amdgpu_mes_queue_doorbell_free(struct amdgpu_device *adev,					   struct amdgpu_mes_process *process,					   uint32_t doorbell_index){	unsigned int old, doorbell_id; 	doorbell_id = doorbell_index -		(process->doorbell_index *		 amdgpu_mes_doorbell_process_slice(adev)) / sizeof(u32);	doorbell_id /= 2; 	old = test_and_clear_bit(doorbell_id, process->doorbell_bitmap);	WARN_ON(!old);} static int amdgpu_mes_doorbell_init(struct amdgpu_device *adev){	size_t doorbell_start_offset;	size_t doorbell_aperture_size;	size_t doorbell_process_limit;	size_t aggregated_doorbell_start;	int i; 	aggregated_doorbell_start = (adev->doorbell_index.max_assignment + 1) * sizeof(u32);	aggregated_doorbell_start =		roundup(aggregated_doorbell_start, PAGE_SIZE); 	doorbell_start_offset = aggregated_doorbell_start + PAGE_SIZE;	doorbell_start_offset =		roundup(doorbell_start_offset,			amdgpu_mes_doorbell_process_slice(adev)); 	doorbell_aperture_size = adev->doorbell.size;	doorbell_aperture_size =			rounddown(doorbell_aperture_size,				  amdgpu_mes_doorbell_process_slice(adev)); 	if (doorbell_aperture_size > doorbell_start_offset)		doorbell_process_limit =			(doorbell_aperture_size - doorbell_start_offset) /			amdgpu_mes_doorbell_process_slice(adev);	else		return -ENOSPC; 	adev->mes.doorbell_id_offset = doorbell_start_offset / sizeof(u32);	adev->mes.max_doorbell_slices = doorbell_process_limit; 	/* allocate Qword range for aggregated doorbell */	for (i = 0; i < AMDGPU_MES_PRIORITY_NUM_LEVELS; i++)		adev->mes.aggregated_doorbells[i] =			aggregated_doorbell_start / sizeof(u32) + i * 2; 	DRM_INFO("max_doorbell_slices=%zu\n", doorbell_process_limit);	return 0;} int amdgpu_mes_init(struct amdgpu_device *adev){	int i, r; 	adev->mes.adev = adev; 	idr_init(&adev->mes.pasid_idr);	idr_init(&adev->mes.gang_id_idr);	idr_init(&adev->mes.queue_id_idr);	ida_init(&adev->mes.doorbell_ida);	spin_lock_init(&adev->mes.queue_id_lock);	spin_lock_init(&adev->mes.ring_lock);	mutex_init(&adev->mes.mutex_hidden); 	adev->mes.total_max_queue = AMDGPU_FENCE_MES_QUEUE_ID_MASK;	adev->mes.vmid_mask_mmhub = 0xffffff00;	adev->mes.vmid_mask_gfxhub = 0xffffff00; 	for (i = 0; i < AMDGPU_MES_MAX_COMPUTE_PIPES; i++) {		/* use only 1st MEC pipes */		if (i >= 4)			continue;		adev->mes.compute_hqd_mask[i] = 0xc;	} 	for (i = 0; i < AMDGPU_MES_MAX_GFX_PIPES; i++)		adev->mes.gfx_hqd_mask[i] = i ? 0 : 0xfffffffe; 	for (i = 0; i < AMDGPU_MES_MAX_SDMA_PIPES; i++) {		if (adev->ip_versions[SDMA0_HWIP][0] < IP_VERSION(6, 0, 0))			adev->mes.sdma_hqd_mask[i] = i ? 0 : 0x3fc;		/* zero sdma_hqd_mask for non-existent engine */		else if (adev->sdma.num_instances == 1)			adev->mes.sdma_hqd_mask[i] = i ? 0 : 0xfc;		else			adev->mes.sdma_hqd_mask[i] = 0xfc;	} 	r = amdgpu_device_wb_get(adev, &adev->mes.sch_ctx_offs);	if (r) {		dev_err(adev->dev,			"(%d) ring trail_fence_offs wb alloc failed\n", r);		goto error_ids;	}	adev->mes.sch_ctx_gpu_addr =		adev->wb.gpu_addr + (adev->mes.sch_ctx_offs * 4);	adev->mes.sch_ctx_ptr =		(uint64_t *)&adev->wb.wb[adev->mes.sch_ctx_offs]; 	r = amdgpu_device_wb_get(adev, &adev->mes.query_status_fence_offs);	if (r) {		amdgpu_device_wb_free(adev, adev->mes.sch_ctx_offs);		dev_err(adev->dev,			"(%d) query_status_fence_offs wb alloc failed\n", r);		goto error_ids;	}	adev->mes.query_status_fence_gpu_addr =		adev->wb.gpu_addr + (adev->mes.query_status_fence_offs * 4);	adev->mes.query_status_fence_ptr =		(uint64_t *)&adev->wb.wb[adev->mes.query_status_fence_offs]; 	r = amdgpu_device_wb_get(adev, &adev->mes.read_val_offs);	if (r) {		amdgpu_device_wb_free(adev, adev->mes.sch_ctx_offs);		amdgpu_device_wb_free(adev, adev->mes.query_status_fence_offs);		dev_err(adev->dev,			"(%d) read_val_offs alloc failed\n", r);		goto error_ids;	}	adev->mes.read_val_gpu_addr =		adev->wb.gpu_addr + (adev->mes.read_val_offs * 4);	adev->mes.read_val_ptr =		(uint32_t *)&adev->wb.wb[adev->mes.read_val_offs]; 	r = amdgpu_mes_doorbell_init(adev);	if (r)		goto error; 	return 0; error:	amdgpu_device_wb_free(adev, adev->mes.sch_ctx_offs);	amdgpu_device_wb_free(adev, adev->mes.query_status_fence_offs);	amdgpu_device_wb_free(adev, adev->mes.read_val_offs);error_ids:	idr_destroy(&adev->mes.pasid_idr);	idr_destroy(&adev->mes.gang_id_idr);	idr_destroy(&adev->mes.queue_id_idr);	ida_destroy(&adev->mes.doorbell_ida);	mutex_destroy(&adev->mes.mutex_hidden);	return r;} void amdgpu_mes_fini(struct amdgpu_device *adev){	amdgpu_device_wb_free(adev, adev->mes.sch_ctx_offs);	amdgpu_device_wb_free(adev, adev->mes.query_status_fence_offs);	amdgpu_device_wb_free(adev, adev->mes.read_val_offs); 	idr_destroy(&adev->mes.pasid_idr);	idr_destroy(&adev->mes.gang_id_idr);	idr_destroy(&adev->mes.queue_id_idr);	ida_destroy(&adev->mes.doorbell_ida);	mutex_destroy(&adev->mes.mutex_hidden);} static void amdgpu_mes_queue_free_mqd(struct amdgpu_mes_queue *q){	amdgpu_bo_free_kernel(&q->mqd_obj,			      &q->mqd_gpu_addr,			      &q->mqd_cpu_ptr);} int amdgpu_mes_create_process(struct amdgpu_device *adev, int pasid,			      struct amdgpu_vm *vm){	struct amdgpu_mes_process *process;	int r; 	/* allocate the mes process buffer */	process = kzalloc(sizeof(struct amdgpu_mes_process), GFP_KERNEL);	if (!process) {		DRM_ERROR("no more memory to create mes process\n");		return -ENOMEM;	} 	process->doorbell_bitmap =		kzalloc(DIV_ROUND_UP(AMDGPU_MES_MAX_NUM_OF_QUEUES_PER_PROCESS,				     BITS_PER_BYTE), GFP_KERNEL);	if (!process->doorbell_bitmap) {		DRM_ERROR("failed to allocate doorbell bitmap\n");		kfree(process);		return -ENOMEM;	} 	/* allocate the process context bo and map it */	r = amdgpu_bo_create_kernel(adev, AMDGPU_MES_PROC_CTX_SIZE, PAGE_SIZE,				    AMDGPU_GEM_DOMAIN_GTT,				    &process->proc_ctx_bo,				    &process->proc_ctx_gpu_addr,				    &process->proc_ctx_cpu_ptr);	if (r) {		DRM_ERROR("failed to allocate process context bo\n");		goto clean_up_memory;	}	memset(process->proc_ctx_cpu_ptr, 0, AMDGPU_MES_PROC_CTX_SIZE); 	/*	 * Avoid taking any other locks under MES lock to avoid circular	 * lock dependencies.	 */	amdgpu_mes_lock(&adev->mes); 	/* add the mes process to idr list */	r = idr_alloc(&adev->mes.pasid_idr, process, pasid, pasid + 1,		      GFP_KERNEL);	if (r < 0) {		DRM_ERROR("failed to lock pasid=%d\n", pasid);		goto clean_up_ctx;	} 	/* allocate the starting doorbell index of the process */	r = amdgpu_mes_alloc_process_doorbells(adev, &process->doorbell_index);	if (r < 0) {		DRM_ERROR("failed to allocate doorbell for process\n");		goto clean_up_pasid;	} 	DRM_DEBUG("process doorbell index = %d\n", process->doorbell_index); 	INIT_LIST_HEAD(&process->gang_list);	process->vm = vm;	process->pasid = pasid;	process->process_quantum = adev->mes.default_process_quantum;	process->pd_gpu_addr = amdgpu_bo_gpu_offset(vm->root.bo); 	amdgpu_mes_unlock(&adev->mes);	return 0; clean_up_pasid:	idr_remove(&adev->mes.pasid_idr, pasid);	amdgpu_mes_unlock(&adev->mes);clean_up_ctx:	amdgpu_bo_free_kernel(&process->proc_ctx_bo,			      &process->proc_ctx_gpu_addr,			      &process->proc_ctx_cpu_ptr);clean_up_memory:	kfree(process->doorbell_bitmap);	kfree(process);	return r;} void amdgpu_mes_destroy_process(struct amdgpu_device *adev, int pasid){	struct amdgpu_mes_process *process;	struct amdgpu_mes_gang *gang, *tmp1;	struct amdgpu_mes_queue *queue, *tmp2;	struct mes_remove_queue_input queue_input;	unsigned long flags;	int r; 	/*	 * Avoid taking any other locks under MES lock to avoid circular	 * lock dependencies.	 */	amdgpu_mes_lock(&adev->mes); 	process = idr_find(&adev->mes.pasid_idr, pasid);	if (!process) {		DRM_WARN("pasid %d doesn't exist\n", pasid);		amdgpu_mes_unlock(&adev->mes);		return;	} 	/* Remove all queues from hardware */	list_for_each_entry_safe(gang, tmp1, &process->gang_list, list) {		list_for_each_entry_safe(queue, tmp2, &gang->queue_list, list) {			spin_lock_irqsave(&adev->mes.queue_id_lock, flags);			idr_remove(&adev->mes.queue_id_idr, queue->queue_id);			spin_unlock_irqrestore(&adev->mes.queue_id_lock, flags); 			queue_input.doorbell_offset = queue->doorbell_off;			queue_input.gang_context_addr = gang->gang_ctx_gpu_addr; 			r = adev->mes.funcs->remove_hw_queue(&adev->mes,							     &queue_input);			if (r)				DRM_WARN("failed to remove hardware queue\n");		} 		idr_remove(&adev->mes.gang_id_idr, gang->gang_id);	} 	amdgpu_mes_free_process_doorbells(adev, process->doorbell_index);	idr_remove(&adev->mes.pasid_idr, pasid);	amdgpu_mes_unlock(&adev->mes); 	/* free all memory allocated by the process */	list_for_each_entry_safe(gang, tmp1, &process->gang_list, list) {		/* free all queues in the gang */		list_for_each_entry_safe(queue, tmp2, &gang->queue_list, list) {			amdgpu_mes_queue_free_mqd(queue);			list_del(&queue->list);			kfree(queue);		}		amdgpu_bo_free_kernel(&gang->gang_ctx_bo,				      &gang->gang_ctx_gpu_addr,				      &gang->gang_ctx_cpu_ptr);		list_del(&gang->list);		kfree(gang); 	}	amdgpu_bo_free_kernel(&process->proc_ctx_bo,			      &process->proc_ctx_gpu_addr,			      &process->proc_ctx_cpu_ptr);	kfree(process->doorbell_bitmap);	kfree(process);} int amdgpu_mes_add_gang(struct amdgpu_device *adev, int pasid,			struct amdgpu_mes_gang_properties *gprops,			int *gang_id){	struct amdgpu_mes_process *process;	struct amdgpu_mes_gang *gang;	int r; 	/* allocate the mes gang buffer */	gang = kzalloc(sizeof(struct amdgpu_mes_gang), GFP_KERNEL);	if (!gang) {		return -ENOMEM;	} 	/* allocate the gang context bo and map it to cpu space */	r = amdgpu_bo_create_kernel(adev, AMDGPU_MES_GANG_CTX_SIZE, PAGE_SIZE,				    AMDGPU_GEM_DOMAIN_GTT,				    &gang->gang_ctx_bo,				    &gang->gang_ctx_gpu_addr,				    &gang->gang_ctx_cpu_ptr);	if (r) {		DRM_ERROR("failed to allocate process context bo\n");		goto clean_up_mem;	}	memset(gang->gang_ctx_cpu_ptr, 0, AMDGPU_MES_GANG_CTX_SIZE); 	/*	 * Avoid taking any other locks under MES lock to avoid circular	 * lock dependencies.	 */	amdgpu_mes_lock(&adev->mes); 	process = idr_find(&adev->mes.pasid_idr, pasid);	if (!process) {		DRM_ERROR("pasid %d doesn't exist\n", pasid);		r = -EINVAL;		goto clean_up_ctx;	} 	/* add the mes gang to idr list */	r = idr_alloc(&adev->mes.gang_id_idr, gang, 1, 0,		      GFP_KERNEL);	if (r < 0) {		DRM_ERROR("failed to allocate idr for gang\n");		goto clean_up_ctx;	} 	gang->gang_id = r;	*gang_id = r; 	INIT_LIST_HEAD(&gang->queue_list);	gang->process = process;	gang->priority = gprops->priority;	gang->gang_quantum = gprops->gang_quantum ?		gprops->gang_quantum : adev->mes.default_gang_quantum;	gang->global_priority_level = gprops->global_priority_level;	gang->inprocess_gang_priority = gprops->inprocess_gang_priority;	list_add_tail(&gang->list, &process->gang_list); 	amdgpu_mes_unlock(&adev->mes);	return 0; clean_up_ctx:	amdgpu_mes_unlock(&adev->mes);	amdgpu_bo_free_kernel(&gang->gang_ctx_bo,			      &gang->gang_ctx_gpu_addr,			      &gang->gang_ctx_cpu_ptr);clean_up_mem:	kfree(gang);	return r;} int amdgpu_mes_remove_gang(struct amdgpu_device *adev, int gang_id){	struct amdgpu_mes_gang *gang; 	/*	 * Avoid taking any other locks under MES lock to avoid circular	 * lock dependencies.	 */	amdgpu_mes_lock(&adev->mes); 	gang = idr_find(&adev->mes.gang_id_idr, gang_id);	if (!gang) {		DRM_ERROR("gang id %d doesn't exist\n", gang_id);		amdgpu_mes_unlock(&adev->mes);		return -EINVAL;	} 

Lines 1–500 of 1425. Load more