// SPDX-License-Identifier: GPL-2.0-only OR MIT/* Copyright (c) 2023 Imagination Technologies Ltd. */ #include <uapi/drm/pvr_drm.h> #include <drm/drm_syncobj.h>#include <drm/gpu_scheduler.h>#include <linux/xarray.h>#include <linux/dma-fence-unwrap.h> #include "pvr_device.h"#include "pvr_queue.h"#include "pvr_sync.h" static intpvr_check_sync_op(const struct drm_pvr_sync_op *sync_op){	u8 handle_type; 	if (sync_op->flags & ~DRM_PVR_SYNC_OP_FLAGS_MASK)		return -EINVAL; 	handle_type = sync_op->flags & DRM_PVR_SYNC_OP_FLAG_HANDLE_TYPE_MASK;	if (handle_type != DRM_PVR_SYNC_OP_FLAG_HANDLE_TYPE_SYNCOBJ &&	    handle_type != DRM_PVR_SYNC_OP_FLAG_HANDLE_TYPE_TIMELINE_SYNCOBJ)		return -EINVAL; 	if (handle_type == DRM_PVR_SYNC_OP_FLAG_HANDLE_TYPE_SYNCOBJ &&	    sync_op->value != 0)		return -EINVAL; 	return 0;} static voidpvr_sync_signal_free(struct pvr_sync_signal *sig_sync){	if (!sig_sync)		return; 	drm_syncobj_put(sig_sync->syncobj);	dma_fence_chain_free(sig_sync->chain);	dma_fence_put(sig_sync->fence);	kfree(sig_sync);} voidpvr_sync_signal_array_cleanup(struct xarray *array){	struct pvr_sync_signal *sig_sync;	unsigned long i; 	xa_for_each(array, i, sig_sync)		pvr_sync_signal_free(sig_sync); 	xa_destroy(array);} static struct pvr_sync_signal *pvr_sync_signal_array_add(struct xarray *array, struct drm_file *file, u32 handle, u64 point){	struct pvr_sync_signal *sig_sync;	struct dma_fence *cur_fence;	int err;	u32 id; 	sig_sync = kzalloc_obj(*sig_sync);	if (!sig_sync)		return ERR_PTR(-ENOMEM); 	sig_sync->handle = handle;	sig_sync->point = point; 	if (point > 0) {		sig_sync->chain = dma_fence_chain_alloc();		if (!sig_sync->chain) {			err = -ENOMEM;			goto err_free_sig_sync;		}	} 	sig_sync->syncobj = drm_syncobj_find(file, handle);	if (!sig_sync->syncobj) {		err = -EINVAL;		goto err_free_sig_sync;	} 	/* Retrieve the current fence attached to that point. It's	 * perfectly fine to get a NULL fence here, it just means there's	 * no fence attached to that point yet.	 */	if (!drm_syncobj_find_fence(file, handle, point, 0, &cur_fence))		sig_sync->fence = cur_fence; 	err = xa_alloc(array, &id, sig_sync, xa_limit_32b, GFP_KERNEL);	if (err)		goto err_free_sig_sync; 	return sig_sync; err_free_sig_sync:	pvr_sync_signal_free(sig_sync);	return ERR_PTR(err);} static struct pvr_sync_signal *pvr_sync_signal_array_search(struct xarray *array, u32 handle, u64 point){	struct pvr_sync_signal *sig_sync;	unsigned long i; 	xa_for_each(array, i, sig_sync) {		if (handle == sig_sync->handle && point == sig_sync->point)			return sig_sync;	} 	return NULL;} static struct pvr_sync_signal *pvr_sync_signal_array_get(struct xarray *array, struct drm_file *file, u32 handle, u64 point){	struct pvr_sync_signal *sig_sync; 	sig_sync = pvr_sync_signal_array_search(array, handle, point);	if (sig_sync)		return sig_sync; 	return pvr_sync_signal_array_add(array, file, handle, point);} intpvr_sync_signal_array_collect_ops(struct xarray *array,				  struct drm_file *file,				  u32 sync_op_count,				  const struct drm_pvr_sync_op *sync_ops){	for (u32 i = 0; i < sync_op_count; i++) {		struct pvr_sync_signal *sig_sync;		int ret; 		if (!(sync_ops[i].flags & DRM_PVR_SYNC_OP_FLAG_SIGNAL))			continue; 		ret = pvr_check_sync_op(&sync_ops[i]);		if (ret)			return ret; 		sig_sync = pvr_sync_signal_array_get(array, file,						     sync_ops[i].handle,						     sync_ops[i].value);		if (IS_ERR(sig_sync))			return PTR_ERR(sig_sync);	} 	return 0;} intpvr_sync_signal_array_update_fences(struct xarray *array,				    u32 sync_op_count,				    const struct drm_pvr_sync_op *sync_ops,				    struct dma_fence *finished_fence){	for (u32 i = 0; i < sync_op_count; i++) {		struct dma_fence *old_fence;		struct pvr_sync_signal *sig_sync; 		if (!(sync_ops[i].flags & DRM_PVR_SYNC_OP_FLAG_SIGNAL))			continue; 		sig_sync = pvr_sync_signal_array_search(array, sync_ops[i].handle,							sync_ops[i].value);		if (WARN_ON(!sig_sync))			return -EINVAL; 		old_fence = sig_sync->fence;		sig_sync->fence = dma_fence_get(finished_fence);		dma_fence_put(old_fence); 		if (WARN_ON(!sig_sync->fence))			return -EINVAL;	} 	return 0;} voidpvr_sync_signal_array_push_fences(struct xarray *array){	struct pvr_sync_signal *sig_sync;	unsigned long i; 	xa_for_each(array, i, sig_sync) {		if (sig_sync->chain) {			drm_syncobj_add_point(sig_sync->syncobj, sig_sync->chain,					      sig_sync->fence, sig_sync->point);			sig_sync->chain = NULL;		} else {			drm_syncobj_replace_fence(sig_sync->syncobj, sig_sync->fence);		}	}} static intpvr_sync_add_dep_to_job(struct drm_sched_job *job, struct dma_fence *f){	struct dma_fence_unwrap iter;	u32 native_fence_count = 0;	struct dma_fence *uf;	int err = 0; 	dma_fence_unwrap_for_each(uf, &iter, f) {		if (pvr_queue_fence_is_native(uf))			native_fence_count++;	} 	/* No need to unwrap the fence if it's fully non-native. */	if (!native_fence_count)		return drm_sched_job_add_dependency(job, f); 	dma_fence_unwrap_for_each(uf, &iter, f) {		/* There's no dma_fence_unwrap_stop() helper cleaning up the refs		 * owned by dma_fence_unwrap(), so let's just iterate over all		 * entries without doing anything when something failed.		 */		if (err)			continue; 		if (pvr_queue_fence_is_native(uf)) {			struct drm_sched_fence *s_fence = to_drm_sched_fence(uf); 			/* If this is a native dependency, we wait for the scheduled fence,			 * and we will let pvr_queue_run_job() issue FW waits.			 */			err = drm_sched_job_add_dependency(job,							   dma_fence_get(&s_fence->scheduled));		} else {			err = drm_sched_job_add_dependency(job, dma_fence_get(uf));		}	} 	dma_fence_put(f);	return err;} intpvr_sync_add_deps_to_job(struct pvr_file *pvr_file, struct drm_sched_job *job,			 u32 sync_op_count,			 const struct drm_pvr_sync_op *sync_ops,			 struct xarray *signal_array){	int err = 0; 	if (!sync_op_count)		return 0; 	for (u32 i = 0; i < sync_op_count; i++) {		struct pvr_sync_signal *sig_sync;		struct dma_fence *fence; 		if (sync_ops[i].flags & DRM_PVR_SYNC_OP_FLAG_SIGNAL)			continue; 		err = pvr_check_sync_op(&sync_ops[i]);		if (err)			return err; 		sig_sync = pvr_sync_signal_array_search(signal_array, sync_ops[i].handle,							sync_ops[i].value);		if (sig_sync) {			if (WARN_ON(!sig_sync->fence))				return -EINVAL; 			fence = dma_fence_get(sig_sync->fence);		} else {			err = drm_syncobj_find_fence(from_pvr_file(pvr_file), sync_ops[i].handle,						     sync_ops[i].value, 0, &fence);			if (err)				return err;		} 		err = pvr_sync_add_dep_to_job(job, fence);		if (err)			return err;	} 	return 0;}