// SPDX-License-Identifier: GPL-2.0-only OR MIT/* Copyright (c) 2023 Imagination Technologies Ltd. */ #include <drm/drm_managed.h>#include <drm/gpu_scheduler.h> #include "pvr_cccb.h"#include "pvr_context.h"#include "pvr_device.h"#include "pvr_drv.h"#include "pvr_job.h"#include "pvr_queue.h"#include "pvr_vm.h" #include "pvr_rogue_fwif_client.h" #define MAX_DEADLINE_MS 30000 #define CTX_COMPUTE_CCCB_SIZE_LOG2 15#define CTX_FRAG_CCCB_SIZE_LOG2 15#define CTX_GEOM_CCCB_SIZE_LOG2 15#define CTX_TRANSFER_CCCB_SIZE_LOG2 15 static int get_xfer_ctx_state_size(struct pvr_device *pvr_dev){	u32 num_isp_store_registers; 	if (PVR_HAS_FEATURE(pvr_dev, xe_memory_hierarchy)) {		num_isp_store_registers = 1;	} else {		int err; 		err = PVR_FEATURE_VALUE(pvr_dev, num_isp_ipp_pipes, &num_isp_store_registers);		if (WARN_ON(err))			return err;	} 	return sizeof(struct rogue_fwif_frag_ctx_state) +	       (num_isp_store_registers *		sizeof(((struct rogue_fwif_frag_ctx_state *)0)->frag_reg_isp_store[0]));} static int get_frag_ctx_state_size(struct pvr_device *pvr_dev){	u32 num_isp_store_registers;	int err; 	if (PVR_HAS_FEATURE(pvr_dev, xe_memory_hierarchy)) {		err = PVR_FEATURE_VALUE(pvr_dev, num_raster_pipes, &num_isp_store_registers);		if (WARN_ON(err))			return err; 		if (PVR_HAS_FEATURE(pvr_dev, gpu_multicore_support)) {			u32 xpu_max_slaves; 			err = PVR_FEATURE_VALUE(pvr_dev, xpu_max_slaves, &xpu_max_slaves);			if (WARN_ON(err))				return err; 			num_isp_store_registers *= (1 + xpu_max_slaves);		}	} else {		err = PVR_FEATURE_VALUE(pvr_dev, num_isp_ipp_pipes, &num_isp_store_registers);		if (WARN_ON(err))			return err;	} 	return sizeof(struct rogue_fwif_frag_ctx_state) +	       (num_isp_store_registers *		sizeof(((struct rogue_fwif_frag_ctx_state *)0)->frag_reg_isp_store[0]));} static int get_ctx_state_size(struct pvr_device *pvr_dev, enum drm_pvr_job_type type){	switch (type) {	case DRM_PVR_JOB_TYPE_GEOMETRY:		return sizeof(struct rogue_fwif_geom_ctx_state);	case DRM_PVR_JOB_TYPE_FRAGMENT:		return get_frag_ctx_state_size(pvr_dev);	case DRM_PVR_JOB_TYPE_COMPUTE:		return sizeof(struct rogue_fwif_compute_ctx_state);	case DRM_PVR_JOB_TYPE_TRANSFER_FRAG:		return get_xfer_ctx_state_size(pvr_dev);	} 	WARN(1, "Invalid queue type");	return -EINVAL;} static u32 get_ctx_offset(enum drm_pvr_job_type type){	switch (type) {	case DRM_PVR_JOB_TYPE_GEOMETRY:		return offsetof(struct rogue_fwif_fwrendercontext, geom_context);	case DRM_PVR_JOB_TYPE_FRAGMENT:		return offsetof(struct rogue_fwif_fwrendercontext, frag_context);	case DRM_PVR_JOB_TYPE_COMPUTE:		return offsetof(struct rogue_fwif_fwcomputecontext, cdm_context);	case DRM_PVR_JOB_TYPE_TRANSFER_FRAG:		return offsetof(struct rogue_fwif_fwtransfercontext, tq_context);	} 	return 0;} static const char *pvr_queue_fence_get_driver_name(struct dma_fence *f){	return PVR_DRIVER_NAME;} static void pvr_queue_fence_release(struct dma_fence *f){	struct pvr_queue_fence *fence = container_of(f, struct pvr_queue_fence, base); 	pvr_context_put(fence->queue->ctx);	dma_fence_free(f);} static const char *pvr_queue_job_fence_get_timeline_name(struct dma_fence *f){	struct pvr_queue_fence *fence = container_of(f, struct pvr_queue_fence, base); 	switch (fence->queue->type) {	case DRM_PVR_JOB_TYPE_GEOMETRY:		return "geometry"; 	case DRM_PVR_JOB_TYPE_FRAGMENT:		return "fragment"; 	case DRM_PVR_JOB_TYPE_COMPUTE:		return "compute"; 	case DRM_PVR_JOB_TYPE_TRANSFER_FRAG:		return "transfer";	} 	WARN(1, "Invalid queue type");	return "invalid";} static const char *pvr_queue_cccb_fence_get_timeline_name(struct dma_fence *f){	struct pvr_queue_fence *fence = container_of(f, struct pvr_queue_fence, base); 	switch (fence->queue->type) {	case DRM_PVR_JOB_TYPE_GEOMETRY:		return "geometry-cccb"; 	case DRM_PVR_JOB_TYPE_FRAGMENT:		return "fragment-cccb"; 	case DRM_PVR_JOB_TYPE_COMPUTE:		return "compute-cccb"; 	case DRM_PVR_JOB_TYPE_TRANSFER_FRAG:		return "transfer-cccb";	} 	WARN(1, "Invalid queue type");	return "invalid";} static const struct dma_fence_ops pvr_queue_job_fence_ops = {	.get_driver_name = pvr_queue_fence_get_driver_name,	.get_timeline_name = pvr_queue_job_fence_get_timeline_name,	.release = pvr_queue_fence_release,}; /** * to_pvr_queue_job_fence() - Return a pvr_queue_fence object if the fence is * backed by a UFO. * @f: The dma_fence to turn into a pvr_queue_fence. * * Return: *  * A non-NULL pvr_queue_fence object if the dma_fence is backed by a UFO, or *  * NULL otherwise. */static struct pvr_queue_fence *to_pvr_queue_job_fence(struct dma_fence *f){	struct drm_sched_fence *sched_fence = to_drm_sched_fence(f); 	if (sched_fence)		f = sched_fence->parent; 	if (f && f->ops == &pvr_queue_job_fence_ops)		return container_of(f, struct pvr_queue_fence, base); 	return NULL;} static const struct dma_fence_ops pvr_queue_cccb_fence_ops = {	.get_driver_name = pvr_queue_fence_get_driver_name,	.get_timeline_name = pvr_queue_cccb_fence_get_timeline_name,	.release = pvr_queue_fence_release,}; /** * pvr_queue_fence_put() - Put wrapper for pvr_queue_fence objects. * @f: The dma_fence object to put. * * If the pvr_queue_fence has been initialized, we call dma_fence_put(), * otherwise we free the object with dma_fence_free(). This allows us * to do the right thing before and after pvr_queue_fence_init() had been * called. */static void pvr_queue_fence_put(struct dma_fence *f){	if (!f)		return; 	if (WARN_ON(f->ops &&		    f->ops != &pvr_queue_cccb_fence_ops &&		    f->ops != &pvr_queue_job_fence_ops))		return; 	/* If the fence hasn't been initialized yet, free the object directly. */	if (f->ops)		dma_fence_put(f);	else		dma_fence_free(f);} /** * pvr_queue_fence_alloc() - Allocate a pvr_queue_fence fence object * * Call this function to allocate job CCCB and done fences. This only * allocates the objects. Initialization happens when the underlying * dma_fence object is to be returned to drm_sched (in prepare_job() or * run_job()). * * Return: *  * A valid pointer if the allocation succeeds, or *  * NULL if the allocation fails. */static struct dma_fence *pvr_queue_fence_alloc(void){	struct pvr_queue_fence *fence; 	fence = kzalloc(sizeof(*fence), GFP_KERNEL);	if (!fence)		return NULL; 	return &fence->base;} /** * pvr_queue_fence_init() - Initializes a pvr_queue_fence object. * @f: The fence to initialize * @queue: The queue this fence belongs to. * @fence_ops: The fence operations. * @fence_ctx: The fence context. * * Wrapper around dma_fence_init() that takes care of initializing the * pvr_queue_fence::queue field too. */static voidpvr_queue_fence_init(struct dma_fence *f,		     struct pvr_queue *queue,		     const struct dma_fence_ops *fence_ops,		     struct pvr_queue_fence_ctx *fence_ctx){	struct pvr_queue_fence *fence = container_of(f, struct pvr_queue_fence, base); 	pvr_context_get(queue->ctx);	fence->queue = queue;	dma_fence_init(&fence->base, fence_ops,		       &fence_ctx->lock, fence_ctx->id,		       atomic_inc_return(&fence_ctx->seqno));} /** * pvr_queue_cccb_fence_init() - Initializes a CCCB fence object. * @fence: The fence to initialize. * @queue: The queue this fence belongs to. * * Initializes a fence that can be used to wait for CCCB space. * * Should be called in the ::prepare_job() path, so the fence returned to * drm_sched is valid. */static voidpvr_queue_cccb_fence_init(struct dma_fence *fence, struct pvr_queue *queue){	pvr_queue_fence_init(fence, queue, &pvr_queue_cccb_fence_ops,			     &queue->cccb_fence_ctx.base);} /** * pvr_queue_job_fence_init() - Initializes a job done fence object. * @fence: The fence to initialize. * @queue: The queue this fence belongs to. * * Initializes a fence that will be signaled when the GPU is done executing * a job. * * Should be called *before* the ::run_job() path, so the fence is initialised * before being placed in the pending_list. */static voidpvr_queue_job_fence_init(struct dma_fence *fence, struct pvr_queue *queue){	pvr_queue_fence_init(fence, queue, &pvr_queue_job_fence_ops,			     &queue->job_fence_ctx);} /** * pvr_queue_fence_ctx_init() - Queue fence context initialization. * @fence_ctx: The context to initialize */static voidpvr_queue_fence_ctx_init(struct pvr_queue_fence_ctx *fence_ctx){	spin_lock_init(&fence_ctx->lock);	fence_ctx->id = dma_fence_context_alloc(1);	atomic_set(&fence_ctx->seqno, 0);} static u32 ufo_cmds_size(u32 elem_count){	/* We can pass at most ROGUE_FWIF_CCB_CMD_MAX_UFOS per UFO-related command. */	u32 full_cmd_count = elem_count / ROGUE_FWIF_CCB_CMD_MAX_UFOS;	u32 remaining_elems = elem_count % ROGUE_FWIF_CCB_CMD_MAX_UFOS;	u32 size = full_cmd_count *		   pvr_cccb_get_size_of_cmd_with_hdr(ROGUE_FWIF_CCB_CMD_MAX_UFOS *						     sizeof(struct rogue_fwif_ufo)); 	if (remaining_elems) {		size += pvr_cccb_get_size_of_cmd_with_hdr(remaining_elems *							  sizeof(struct rogue_fwif_ufo));	} 	return size;} static u32 job_cmds_size(struct pvr_job *job, u32 ufo_wait_count){	/* One UFO cmd for the fence signaling, one UFO cmd per native fence native,	 * and a command for the job itself.	 */	return ufo_cmds_size(1) + ufo_cmds_size(ufo_wait_count) +	       pvr_cccb_get_size_of_cmd_with_hdr(job->cmd_len);} /** * job_count_remaining_native_deps() - Count the number of non-signaled native dependencies. * @job: Job to operate on. * * Returns: Number of non-signaled native deps remaining. */static unsigned long job_count_remaining_native_deps(struct pvr_job *job){	unsigned long remaining_count = 0;	struct dma_fence *fence = NULL;	unsigned long index; 	xa_for_each(&job->base.dependencies, index, fence) {		struct pvr_queue_fence *jfence; 		jfence = to_pvr_queue_job_fence(fence);		if (!jfence)			continue; 		if (!dma_fence_is_signaled(&jfence->base))			remaining_count++;	} 	return remaining_count;} /** * pvr_queue_get_job_cccb_fence() - Get the CCCB fence attached to a job. * @queue: The queue this job will be submitted to. * @job: The job to get the CCCB fence on. * * The CCCB fence is a synchronization primitive allowing us to delay job * submission until there's enough space in the CCCB to submit the job. * * Return: *  * NULL if there's enough space in the CCCB to submit this job, or *  * A valid dma_fence object otherwise. */static struct dma_fence *pvr_queue_get_job_cccb_fence(struct pvr_queue *queue, struct pvr_job *job){	struct pvr_queue_fence *cccb_fence;	unsigned int native_deps_remaining; 	/* If the fence is NULL, that means we already checked that we had	 * enough space in the cccb for our job.	 */	if (!job->cccb_fence)		return NULL; 	mutex_lock(&queue->cccb_fence_ctx.job_lock); 	/* Count remaining native dependencies and check if the job fits in the CCCB. */	native_deps_remaining = job_count_remaining_native_deps(job);	if (pvr_cccb_cmdseq_fits(&queue->cccb, job_cmds_size(job, native_deps_remaining))) {		pvr_queue_fence_put(job->cccb_fence);		job->cccb_fence = NULL;		goto out_unlock;	} 	/* There should be no job attached to the CCCB fence context:	 * drm_sched_entity guarantees that jobs are submitted one at a time.	 */	if (WARN_ON(queue->cccb_fence_ctx.job))		pvr_job_put(queue->cccb_fence_ctx.job); 	queue->cccb_fence_ctx.job = pvr_job_get(job); 	/* Initialize the fence before returning it. */	cccb_fence = container_of(job->cccb_fence, struct pvr_queue_fence, base);	if (!WARN_ON(cccb_fence->queue))		pvr_queue_cccb_fence_init(job->cccb_fence, queue); out_unlock:	mutex_unlock(&queue->cccb_fence_ctx.job_lock); 	return dma_fence_get(job->cccb_fence);} /** * pvr_queue_get_job_kccb_fence() - Get the KCCB fence attached to a job. * @queue: The queue this job will be submitted to. * @job: The job to get the KCCB fence on. * * The KCCB fence is a synchronization primitive allowing us to delay job * submission until there's enough space in the KCCB to submit the job. * * Return: *  * NULL if there's enough space in the KCCB to submit this job, or *  * A valid dma_fence object otherwise. */static struct dma_fence *pvr_queue_get_job_kccb_fence(struct pvr_queue *queue, struct pvr_job *job){	struct pvr_device *pvr_dev = queue->ctx->pvr_dev;	struct dma_fence *kccb_fence = NULL; 	/* If the fence is NULL, that means we already checked that we had	 * enough space in the KCCB for our job.	 */	if (!job->kccb_fence)		return NULL; 	if (!WARN_ON(job->kccb_fence->ops)) {		kccb_fence = pvr_kccb_reserve_slot(pvr_dev, job->kccb_fence);		job->kccb_fence = NULL;	} 	return kccb_fence;} static struct dma_fence *pvr_queue_get_paired_frag_job_dep(struct pvr_queue *queue, struct pvr_job *job){	struct pvr_job *frag_job = job->type == DRM_PVR_JOB_TYPE_GEOMETRY ?				   job->paired_job : NULL;	struct dma_fence *f;	unsigned long index; 	if (!frag_job)		return NULL; 	xa_for_each(&frag_job->base.dependencies, index, f) {		/* Skip already signaled fences. */		if (dma_fence_is_signaled(f))			continue; 		/* Skip our own fence. */		if (f == &job->base.s_fence->scheduled)			continue; 		return dma_fence_get(f);	} 	return frag_job->base.sched->ops->prepare_job(&frag_job->base, &queue->entity);} /** * pvr_queue_prepare_job() - Return the next internal dependencies expressed as a dma_fence. * @sched_job: The job to query the next internal dependency on * @s_entity: The entity this job is queue on. * * After iterating over drm_sched_job::dependencies, drm_sched let the driver return * its own internal dependencies. We use this function to return our internal dependencies. */static struct dma_fence *pvr_queue_prepare_job(struct drm_sched_job *sched_job,		      struct drm_sched_entity *s_entity){	struct pvr_job *job = container_of(sched_job, struct pvr_job, base);	struct pvr_queue *queue = container_of(s_entity, struct pvr_queue, entity);	struct dma_fence *internal_dep = NULL;

Lines 1–500 of 1432. Load more