// SPDX-License-Identifier: GPL-2.0-only OR MIT/* Copyright (c) 2023 Imagination Technologies Ltd. */ #include <drm/drm_managed.h>#include <drm/gpu_scheduler.h>#include <linux/overflow.h> #include "pvr_cccb.h"#include "pvr_context.h"#include "pvr_device.h"#include "pvr_drv.h"#include "pvr_job.h"#include "pvr_queue.h"#include "pvr_trace.h"#include "pvr_vm.h" #include "pvr_rogue_fwif_client.h" #define MAX_DEADLINE_MS 30000#define SCHED_TIMEOUT_PERIOD (2 * MAX_DEADLINE_MS) #define CTX_COMPUTE_CCCB_SIZE_LOG2 15#define CTX_FRAG_CCCB_SIZE_LOG2 15#define CTX_GEOM_CCCB_SIZE_LOG2 15#define CTX_TRANSFER_CCCB_SIZE_LOG2 15 static int get_xfer_ctx_state_size(struct pvr_device *pvr_dev){	u32 num_isp_store_registers; 	if (PVR_HAS_FEATURE(pvr_dev, xe_memory_hierarchy)) {		num_isp_store_registers = 1;	} else {		int err; 		err = PVR_FEATURE_VALUE(pvr_dev, num_isp_ipp_pipes, &num_isp_store_registers);		if (WARN_ON(err))			return err;	} 	return struct_size_t(struct rogue_fwif_frag_ctx_state,			     frag_reg_isp_store, num_isp_store_registers);} static int get_frag_ctx_state_size(struct pvr_device *pvr_dev){	u32 num_isp_store_registers;	int err; 	if (PVR_HAS_FEATURE(pvr_dev, xe_memory_hierarchy)) {		err = PVR_FEATURE_VALUE(pvr_dev, num_raster_pipes, &num_isp_store_registers);		if (WARN_ON(err))			return err; 		if (PVR_HAS_FEATURE(pvr_dev, gpu_multicore_support)) {			u32 xpu_max_slaves; 			err = PVR_FEATURE_VALUE(pvr_dev, xpu_max_slaves, &xpu_max_slaves);			if (WARN_ON(err))				return err; 			num_isp_store_registers *= (1 + xpu_max_slaves);		}	} else {		err = PVR_FEATURE_VALUE(pvr_dev, num_isp_ipp_pipes, &num_isp_store_registers);		if (WARN_ON(err))			return err;	} 	return struct_size_t(struct rogue_fwif_frag_ctx_state,			     frag_reg_isp_store, num_isp_store_registers);} static int get_ctx_state_size(struct pvr_device *pvr_dev, enum drm_pvr_job_type type){	switch (type) {	case DRM_PVR_JOB_TYPE_GEOMETRY:		return sizeof(struct rogue_fwif_geom_ctx_state);	case DRM_PVR_JOB_TYPE_FRAGMENT:		return get_frag_ctx_state_size(pvr_dev);	case DRM_PVR_JOB_TYPE_COMPUTE:		return sizeof(struct rogue_fwif_compute_ctx_state);	case DRM_PVR_JOB_TYPE_TRANSFER_FRAG:		return get_xfer_ctx_state_size(pvr_dev);	} 	WARN(1, "Invalid queue type");	return -EINVAL;} static u32 get_ctx_offset(enum drm_pvr_job_type type){	switch (type) {	case DRM_PVR_JOB_TYPE_GEOMETRY:		return offsetof(struct rogue_fwif_fwrendercontext, geom_context);	case DRM_PVR_JOB_TYPE_FRAGMENT:		return offsetof(struct rogue_fwif_fwrendercontext, frag_context);	case DRM_PVR_JOB_TYPE_COMPUTE:		return offsetof(struct rogue_fwif_fwcomputecontext, cdm_context);	case DRM_PVR_JOB_TYPE_TRANSFER_FRAG:		return offsetof(struct rogue_fwif_fwtransfercontext, tq_context);	} 	return 0;} static const char *pvr_queue_fence_get_driver_name(struct dma_fence *f){	return PVR_DRIVER_NAME;} static void pvr_queue_fence_release_work(struct work_struct *w){	struct pvr_queue_fence *fence = container_of(w, struct pvr_queue_fence, release_work); 	pvr_context_put(fence->queue->ctx);	dma_fence_free(&fence->base);} static void pvr_queue_fence_release(struct dma_fence *f){	struct pvr_queue_fence *fence = container_of(f, struct pvr_queue_fence, base);	struct pvr_device *pvr_dev = fence->queue->ctx->pvr_dev; 	queue_work(pvr_dev->sched_wq, &fence->release_work);} static const char *pvr_queue_job_fence_get_timeline_name(struct dma_fence *f){	struct pvr_queue_fence *fence = container_of(f, struct pvr_queue_fence, base); 	switch (fence->queue->type) {	case DRM_PVR_JOB_TYPE_GEOMETRY:		return "geometry"; 	case DRM_PVR_JOB_TYPE_FRAGMENT:		return "fragment"; 	case DRM_PVR_JOB_TYPE_COMPUTE:		return "compute"; 	case DRM_PVR_JOB_TYPE_TRANSFER_FRAG:		return "transfer";	} 	WARN(1, "Invalid queue type");	return "invalid";} static const char *pvr_queue_cccb_fence_get_timeline_name(struct dma_fence *f){	struct pvr_queue_fence *fence = container_of(f, struct pvr_queue_fence, base); 	switch (fence->queue->type) {	case DRM_PVR_JOB_TYPE_GEOMETRY:		return "geometry-cccb"; 	case DRM_PVR_JOB_TYPE_FRAGMENT:		return "fragment-cccb"; 	case DRM_PVR_JOB_TYPE_COMPUTE:		return "compute-cccb"; 	case DRM_PVR_JOB_TYPE_TRANSFER_FRAG:		return "transfer-cccb";	} 	WARN(1, "Invalid queue type");	return "invalid";} static const struct dma_fence_ops pvr_queue_job_fence_ops = {	.get_driver_name = pvr_queue_fence_get_driver_name,	.get_timeline_name = pvr_queue_job_fence_get_timeline_name,	.release = pvr_queue_fence_release,}; /** * pvr_queue_fence_is_ufo_backed() - Check if a dma_fence is backed by a UFO. * @f: The dma_fence to check. * * Return: * * true if the dma_fence is backed by a UFO, or * * false otherwise. */static inline boolpvr_queue_fence_is_ufo_backed(struct dma_fence *f){	/*	 * Currently the only dma_fence backed by a UFO object is the job fence,	 * e.g. pvr_job::done_fence, wrapped by a pvr_queue_fence object.	 */	return f && f->ops == &pvr_queue_job_fence_ops;} /** * to_pvr_queue_job_fence() - Return a pvr_queue_fence object if the fence is * already backed by a UFO. * @f: The dma_fence to turn into a pvr_queue_fence. * * This could be called on: * - a job fence directly, in which case it simply returns the containing pvr_queue_fence; * - a drm_sched_fence's scheduled or finished fence, in which case it will first try to follow *   the parent pointer to find the job fence (note that the parent pointer is initialized *   only after the run_job() callback is called on the drm_sched_fence's owning job); * - any other dma_fence, in which case it will return NULL. * * Return: *  * A non-NULL pvr_queue_fence object if the dma_fence is backed by a UFO, or *  * NULL otherwise. */static struct pvr_queue_fence *to_pvr_queue_job_fence(struct dma_fence *f){	struct drm_sched_fence *sched_fence = to_drm_sched_fence(f); 	if (sched_fence)		f = sched_fence->parent; 	if (pvr_queue_fence_is_ufo_backed(f))		return container_of(f, struct pvr_queue_fence, base); 	return NULL;} static const struct dma_fence_ops pvr_queue_cccb_fence_ops = {	.get_driver_name = pvr_queue_fence_get_driver_name,	.get_timeline_name = pvr_queue_cccb_fence_get_timeline_name,	.release = pvr_queue_fence_release,}; /** * pvr_queue_fence_put() - Put wrapper for pvr_queue_fence objects. * @f: The dma_fence object to put. * * If the pvr_queue_fence has been initialized, we call dma_fence_put(), * otherwise we free the object with dma_fence_free(). This allows us * to do the right thing before and after pvr_queue_fence_init() had been * called. */static void pvr_queue_fence_put(struct dma_fence *f){	if (!f)		return; 	if (WARN_ON(f->ops &&		    f->ops != &pvr_queue_cccb_fence_ops &&		    f->ops != &pvr_queue_job_fence_ops))		return; 	/* If the fence hasn't been initialized yet, free the object directly. */	if (f->ops)		dma_fence_put(f);	else		dma_fence_free(f);} /** * pvr_queue_fence_alloc() - Allocate a pvr_queue_fence fence object * * Call this function to allocate job CCCB and done fences. This only * allocates the objects. Initialization happens when the underlying * dma_fence object is to be returned to drm_sched (in prepare_job() or * run_job()). * * Return: *  * A valid pointer if the allocation succeeds, or *  * NULL if the allocation fails. */static struct dma_fence *pvr_queue_fence_alloc(void){	struct pvr_queue_fence *fence; 	fence = kzalloc_obj(*fence);	if (!fence)		return NULL; 	return &fence->base;} /** * pvr_queue_fence_init() - Initializes a pvr_queue_fence object. * @f: The fence to initialize * @queue: The queue this fence belongs to. * @fence_ops: The fence operations. * @fence_ctx: The fence context. * * Wrapper around dma_fence_init() that takes care of initializing the * pvr_queue_fence::queue field too. */static voidpvr_queue_fence_init(struct dma_fence *f,		     struct pvr_queue *queue,		     const struct dma_fence_ops *fence_ops,		     struct pvr_queue_fence_ctx *fence_ctx){	struct pvr_queue_fence *fence = container_of(f, struct pvr_queue_fence, base); 	pvr_context_get(queue->ctx);	fence->queue = queue;	INIT_WORK(&fence->release_work, pvr_queue_fence_release_work);	dma_fence_init(&fence->base, fence_ops,		       &fence_ctx->lock, fence_ctx->id,		       atomic_inc_return(&fence_ctx->seqno));} /** * pvr_queue_cccb_fence_init() - Initializes a CCCB fence object. * @fence: The fence to initialize. * @queue: The queue this fence belongs to. * * Initializes a fence that can be used to wait for CCCB space. * * Should be called in the ::prepare_job() path, so the fence returned to * drm_sched is valid. */static voidpvr_queue_cccb_fence_init(struct dma_fence *fence, struct pvr_queue *queue){	pvr_queue_fence_init(fence, queue, &pvr_queue_cccb_fence_ops,			     &queue->cccb_fence_ctx.base);} /** * pvr_queue_job_fence_init() - Initializes a job done fence object. * @fence: The fence to initialize. * @queue: The queue this fence belongs to. * * Initializes a fence that will be signaled when the GPU is done executing * a job. * * Should be called *before* the ::run_job() path, so the fence is initialised * before being placed in the pending_list. */static voidpvr_queue_job_fence_init(struct dma_fence *fence, struct pvr_queue *queue){	if (!fence->ops)		pvr_queue_fence_init(fence, queue, &pvr_queue_job_fence_ops,				     &queue->job_fence_ctx);} /** * pvr_queue_fence_ctx_init() - Queue fence context initialization. * @fence_ctx: The context to initialize */static voidpvr_queue_fence_ctx_init(struct pvr_queue_fence_ctx *fence_ctx){	spin_lock_init(&fence_ctx->lock);	fence_ctx->id = dma_fence_context_alloc(1);	atomic_set(&fence_ctx->seqno, 0);} static u32 ufo_cmds_size(u32 elem_count){	/* We can pass at most ROGUE_FWIF_CCB_CMD_MAX_UFOS per UFO-related command. */	u32 full_cmd_count = elem_count / ROGUE_FWIF_CCB_CMD_MAX_UFOS;	u32 remaining_elems = elem_count % ROGUE_FWIF_CCB_CMD_MAX_UFOS;	u32 size = full_cmd_count *		   pvr_cccb_get_size_of_cmd_with_hdr(ROGUE_FWIF_CCB_CMD_MAX_UFOS *						     sizeof(struct rogue_fwif_ufo)); 	if (remaining_elems) {		size += pvr_cccb_get_size_of_cmd_with_hdr(remaining_elems *							  sizeof(struct rogue_fwif_ufo));	} 	return size;} static u32 job_cmds_size(struct pvr_job *job, u32 ufo_wait_count){	/*	 * One UFO command per native fence this job will be waiting on (unless any are	 * signaled by the time the job is submitted), plus a command for the job itself,	 * plus one UFO command for the fence signaling.	 */	return ufo_cmds_size(ufo_wait_count) +	       pvr_cccb_get_size_of_cmd_with_hdr(job->cmd_len) +	       ufo_cmds_size(1);} static boolis_paired_job_fence(struct dma_fence *fence, struct pvr_job *job){	/* This assumes "fence" is one of "job"'s drm_sched_job::dependencies */	return job->type == DRM_PVR_JOB_TYPE_FRAGMENT &&	       job->paired_job &&	       &job->paired_job->base.s_fence->scheduled == fence;} /** * job_count_remaining_native_deps() - Count the number of non-signaled native dependencies. * @job: Job to operate on. * * Returns: Number of non-signaled native deps remaining. */static unsigned long job_count_remaining_native_deps(struct pvr_job *job){	unsigned long remaining_count = 0;	struct dma_fence *fence = NULL;	unsigned long index; 	xa_for_each(&job->base.dependencies, index, fence) {		struct pvr_queue_fence *jfence; 		if (is_paired_job_fence(fence, job)) {			/*			 * A fence between paired jobs won't resolve to a pvr_queue_fence (i.e.			 * be backed by a UFO) until the jobs have been submitted, together.			 * The submitting code will insert a partial render fence command for this.			 */			WARN_ON(dma_fence_is_signaled(fence));			remaining_count++;			continue;		} 		jfence = to_pvr_queue_job_fence(fence);		if (!jfence)			continue; 		if (!dma_fence_is_signaled(&jfence->base))			remaining_count++;	} 	return remaining_count;} /** * pvr_queue_get_job_cccb_fence() - Get the CCCB fence attached to a job. * @queue: The queue this job will be submitted to. * @job: The job to get the CCCB fence on. * * The CCCB fence is a synchronization primitive allowing us to delay job * submission until there's enough space in the CCCB to submit the job. * * Return: *  * NULL if there's enough space in the CCCB to submit this job, or *  * A valid dma_fence object otherwise. */static struct dma_fence *pvr_queue_get_job_cccb_fence(struct pvr_queue *queue, struct pvr_job *job){	struct pvr_queue_fence *cccb_fence;	unsigned int native_deps_remaining; 	/* If the fence is NULL, that means we already checked that we had	 * enough space in the cccb for our job.	 */	if (!job->cccb_fence)		return NULL; 	mutex_lock(&queue->cccb_fence_ctx.job_lock); 	/* Count remaining native dependencies and check if the job fits in the CCCB. */	native_deps_remaining = job_count_remaining_native_deps(job);	if (pvr_cccb_cmdseq_fits(&queue->cccb, job_cmds_size(job, native_deps_remaining))) {		pvr_queue_fence_put(job->cccb_fence);		job->cccb_fence = NULL;		goto out_unlock;	} 	/* There should be no job attached to the CCCB fence context:	 * drm_sched_entity guarantees that jobs are submitted one at a time.	 */	if (WARN_ON(queue->cccb_fence_ctx.job))		pvr_job_put(queue->cccb_fence_ctx.job); 	queue->cccb_fence_ctx.job = pvr_job_get(job); 	/* Initialize the fence before returning it. */	cccb_fence = container_of(job->cccb_fence, struct pvr_queue_fence, base);	if (!WARN_ON(cccb_fence->queue))		pvr_queue_cccb_fence_init(job->cccb_fence, queue); out_unlock:	mutex_unlock(&queue->cccb_fence_ctx.job_lock); 	return dma_fence_get(job->cccb_fence);} /** * pvr_queue_get_job_kccb_fence() - Get the KCCB fence attached to a job. * @queue: The queue this job will be submitted to. * @job: The job to get the KCCB fence on. * * The KCCB fence is a synchronization primitive allowing us to delay job * submission until there's enough space in the KCCB to submit the job. * * Return: *  * NULL if there's enough space in the KCCB to submit this job, or *  * A valid dma_fence object otherwise. */static struct dma_fence *pvr_queue_get_job_kccb_fence(struct pvr_queue *queue, struct pvr_job *job)

Lines 1–500 of 1512. Load more