// SPDX-License-Identifier: MIT/* * Copyright © 2021 Intel Corporation */ #include "xe_exec_queue.h" #include <linux/nospec.h> #include <drm/drm_device.h>#include <drm/drm_file.h>#include <drm/xe_drm.h> #include "xe_device.h"#include "xe_gt.h"#include "xe_hw_engine_class_sysfs.h"#include "xe_hw_fence.h"#include "xe_lrc.h"#include "xe_macros.h"#include "xe_migrate.h"#include "xe_pm.h"#include "xe_ring_ops_types.h"#include "xe_trace.h"#include "xe_vm.h" enum xe_exec_queue_sched_prop {	XE_EXEC_QUEUE_JOB_TIMEOUT = 0,	XE_EXEC_QUEUE_TIMESLICE = 1,	XE_EXEC_QUEUE_PREEMPT_TIMEOUT = 2,	XE_EXEC_QUEUE_SCHED_PROP_MAX = 3,}; static struct xe_exec_queue *__xe_exec_queue_create(struct xe_device *xe,						    struct xe_vm *vm,						    u32 logical_mask,						    u16 width, struct xe_hw_engine *hwe,						    u32 flags){	struct xe_exec_queue *q;	struct xe_gt *gt = hwe->gt;	int err;	int i; 	/* only kernel queues can be permanent */	XE_WARN_ON((flags & EXEC_QUEUE_FLAG_PERMANENT) && !(flags & EXEC_QUEUE_FLAG_KERNEL)); 	q = kzalloc(sizeof(*q) + sizeof(struct xe_lrc) * width, GFP_KERNEL);	if (!q)		return ERR_PTR(-ENOMEM); 	kref_init(&q->refcount);	q->flags = flags;	q->hwe = hwe;	q->gt = gt;	if (vm)		q->vm = xe_vm_get(vm);	q->class = hwe->class;	q->width = width;	q->logical_mask = logical_mask;	q->fence_irq = &gt->fence_irq[hwe->class];	q->ring_ops = gt->ring_ops[hwe->class];	q->ops = gt->exec_queue_ops;	INIT_LIST_HEAD(&q->compute.link);	INIT_LIST_HEAD(&q->multi_gt_link); 	q->sched_props.timeslice_us = hwe->eclass->sched_props.timeslice_us;	q->sched_props.preempt_timeout_us =				hwe->eclass->sched_props.preempt_timeout_us;	if (q->flags & EXEC_QUEUE_FLAG_KERNEL &&	    q->flags & EXEC_QUEUE_FLAG_HIGH_PRIORITY)		q->sched_props.priority = XE_EXEC_QUEUE_PRIORITY_KERNEL;	else		q->sched_props.priority = XE_EXEC_QUEUE_PRIORITY_NORMAL; 	if (xe_exec_queue_is_parallel(q)) {		q->parallel.composite_fence_ctx = dma_fence_context_alloc(1);		q->parallel.composite_fence_seqno = XE_FENCE_INITIAL_SEQNO;	}	if (q->flags & EXEC_QUEUE_FLAG_VM) {		q->bind.fence_ctx = dma_fence_context_alloc(1);		q->bind.fence_seqno = XE_FENCE_INITIAL_SEQNO;	} 	for (i = 0; i < width; ++i) {		err = xe_lrc_init(q->lrc + i, hwe, q, vm, SZ_16K);		if (err)			goto err_lrc;	} 	err = q->ops->init(q);	if (err)		goto err_lrc; 	/*	 * Normally the user vm holds an rpm ref to keep the device	 * awake, and the context holds a ref for the vm, however for	 * some engines we use the kernels migrate vm underneath which offers no	 * such rpm ref, or we lack a vm. Make sure we keep a ref here, so we	 * can perform GuC CT actions when needed. Caller is expected to have	 * already grabbed the rpm ref outside any sensitive locks.	 */	if (!(q->flags & EXEC_QUEUE_FLAG_PERMANENT) && (q->flags & EXEC_QUEUE_FLAG_VM || !vm))		drm_WARN_ON(&xe->drm, !xe_device_mem_access_get_if_ongoing(xe)); 	return q; err_lrc:	for (i = i - 1; i >= 0; --i)		xe_lrc_finish(q->lrc + i);	kfree(q);	return ERR_PTR(err);} struct xe_exec_queue *xe_exec_queue_create(struct xe_device *xe, struct xe_vm *vm,					   u32 logical_mask, u16 width,					   struct xe_hw_engine *hwe, u32 flags){	struct xe_exec_queue *q;	int err; 	if (vm) {		err = xe_vm_lock(vm, true);		if (err)			return ERR_PTR(err);	}	q = __xe_exec_queue_create(xe, vm, logical_mask, width, hwe, flags);	if (vm)		xe_vm_unlock(vm); 	return q;} struct xe_exec_queue *xe_exec_queue_create_class(struct xe_device *xe, struct xe_gt *gt,						 struct xe_vm *vm,						 enum xe_engine_class class, u32 flags){	struct xe_hw_engine *hwe, *hwe0 = NULL;	enum xe_hw_engine_id id;	u32 logical_mask = 0; 	for_each_hw_engine(hwe, gt, id) {		if (xe_hw_engine_is_reserved(hwe))			continue; 		if (hwe->class == class) {			logical_mask |= BIT(hwe->logical_instance);			if (!hwe0)				hwe0 = hwe;		}	} 	if (!logical_mask)		return ERR_PTR(-ENODEV); 	return xe_exec_queue_create(xe, vm, logical_mask, 1, hwe0, flags);} void xe_exec_queue_destroy(struct kref *ref){	struct xe_exec_queue *q = container_of(ref, struct xe_exec_queue, refcount);	struct xe_exec_queue *eq, *next; 	xe_exec_queue_last_fence_put_unlocked(q);	if (!(q->flags & EXEC_QUEUE_FLAG_BIND_ENGINE_CHILD)) {		list_for_each_entry_safe(eq, next, &q->multi_gt_list,					 multi_gt_link)			xe_exec_queue_put(eq);	} 	q->ops->fini(q);} void xe_exec_queue_fini(struct xe_exec_queue *q){	int i; 	for (i = 0; i < q->width; ++i)		xe_lrc_finish(q->lrc + i);	if (!(q->flags & EXEC_QUEUE_FLAG_PERMANENT) && (q->flags & EXEC_QUEUE_FLAG_VM || !q->vm))		xe_device_mem_access_put(gt_to_xe(q->gt));	if (q->vm)		xe_vm_put(q->vm); 	kfree(q);} void xe_exec_queue_assign_name(struct xe_exec_queue *q, u32 instance){	switch (q->class) {	case XE_ENGINE_CLASS_RENDER:		sprintf(q->name, "rcs%d", instance);		break;	case XE_ENGINE_CLASS_VIDEO_DECODE:		sprintf(q->name, "vcs%d", instance);		break;	case XE_ENGINE_CLASS_VIDEO_ENHANCE:		sprintf(q->name, "vecs%d", instance);		break;	case XE_ENGINE_CLASS_COPY:		sprintf(q->name, "bcs%d", instance);		break;	case XE_ENGINE_CLASS_COMPUTE:		sprintf(q->name, "ccs%d", instance);		break;	case XE_ENGINE_CLASS_OTHER:		sprintf(q->name, "gsccs%d", instance);		break;	default:		XE_WARN_ON(q->class);	}} struct xe_exec_queue *xe_exec_queue_lookup(struct xe_file *xef, u32 id){	struct xe_exec_queue *q; 	mutex_lock(&xef->exec_queue.lock);	q = xa_load(&xef->exec_queue.xa, id);	if (q)		xe_exec_queue_get(q);	mutex_unlock(&xef->exec_queue.lock); 	return q;} enum xe_exec_queue_priorityxe_exec_queue_device_get_max_priority(struct xe_device *xe){	return capable(CAP_SYS_NICE) ? XE_EXEC_QUEUE_PRIORITY_HIGH :				       XE_EXEC_QUEUE_PRIORITY_NORMAL;} static int exec_queue_set_priority(struct xe_device *xe, struct xe_exec_queue *q,				   u64 value, bool create){	if (XE_IOCTL_DBG(xe, value > XE_EXEC_QUEUE_PRIORITY_HIGH))		return -EINVAL; 	if (XE_IOCTL_DBG(xe, value > xe_exec_queue_device_get_max_priority(xe)))		return -EPERM; 	return q->ops->set_priority(q, value);} static bool xe_exec_queue_enforce_schedule_limit(void){#if IS_ENABLED(CONFIG_DRM_XE_ENABLE_SCHEDTIMEOUT_LIMIT)	return true;#else	return !capable(CAP_SYS_NICE);#endif} static voidxe_exec_queue_get_prop_minmax(struct xe_hw_engine_class_intf *eclass,			      enum xe_exec_queue_sched_prop prop,			      u32 *min, u32 *max){	switch (prop) {	case XE_EXEC_QUEUE_JOB_TIMEOUT:		*min = eclass->sched_props.job_timeout_min;		*max = eclass->sched_props.job_timeout_max;		break;	case XE_EXEC_QUEUE_TIMESLICE:		*min = eclass->sched_props.timeslice_min;		*max = eclass->sched_props.timeslice_max;		break;	case XE_EXEC_QUEUE_PREEMPT_TIMEOUT:		*min = eclass->sched_props.preempt_timeout_min;		*max = eclass->sched_props.preempt_timeout_max;		break;	default:		break;	}#if IS_ENABLED(CONFIG_DRM_XE_ENABLE_SCHEDTIMEOUT_LIMIT)	if (capable(CAP_SYS_NICE)) {		switch (prop) {		case XE_EXEC_QUEUE_JOB_TIMEOUT:			*min = XE_HW_ENGINE_JOB_TIMEOUT_MIN;			*max = XE_HW_ENGINE_JOB_TIMEOUT_MAX;			break;		case XE_EXEC_QUEUE_TIMESLICE:			*min = XE_HW_ENGINE_TIMESLICE_MIN;			*max = XE_HW_ENGINE_TIMESLICE_MAX;			break;		case XE_EXEC_QUEUE_PREEMPT_TIMEOUT:			*min = XE_HW_ENGINE_PREEMPT_TIMEOUT_MIN;			*max = XE_HW_ENGINE_PREEMPT_TIMEOUT_MAX;			break;		default:			break;		}	}#endif} static int exec_queue_set_timeslice(struct xe_device *xe, struct xe_exec_queue *q,				    u64 value, bool create){	u32 min = 0, max = 0; 	xe_exec_queue_get_prop_minmax(q->hwe->eclass,				      XE_EXEC_QUEUE_TIMESLICE, &min, &max); 	if (xe_exec_queue_enforce_schedule_limit() &&	    !xe_hw_engine_timeout_in_range(value, min, max))		return -EINVAL; 	return q->ops->set_timeslice(q, value);} typedef int (*xe_exec_queue_set_property_fn)(struct xe_device *xe,					     struct xe_exec_queue *q,					     u64 value, bool create); static const xe_exec_queue_set_property_fn exec_queue_set_property_funcs[] = {	[DRM_XE_EXEC_QUEUE_SET_PROPERTY_PRIORITY] = exec_queue_set_priority,	[DRM_XE_EXEC_QUEUE_SET_PROPERTY_TIMESLICE] = exec_queue_set_timeslice,}; static int exec_queue_user_ext_set_property(struct xe_device *xe,					    struct xe_exec_queue *q,					    u64 extension,					    bool create){	u64 __user *address = u64_to_user_ptr(extension);	struct drm_xe_ext_set_property ext;	int err;	u32 idx; 	err = __copy_from_user(&ext, address, sizeof(ext));	if (XE_IOCTL_DBG(xe, err))		return -EFAULT; 	if (XE_IOCTL_DBG(xe, ext.property >=			 ARRAY_SIZE(exec_queue_set_property_funcs)) ||	    XE_IOCTL_DBG(xe, ext.pad) ||	    XE_IOCTL_DBG(xe, ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_PRIORITY &&			 ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_TIMESLICE))		return -EINVAL; 	idx = array_index_nospec(ext.property, ARRAY_SIZE(exec_queue_set_property_funcs));	if (!exec_queue_set_property_funcs[idx])		return -EINVAL; 	return exec_queue_set_property_funcs[idx](xe, q, ext.value,  create);} typedef int (*xe_exec_queue_user_extension_fn)(struct xe_device *xe,					       struct xe_exec_queue *q,					       u64 extension,					       bool create); static const xe_exec_queue_set_property_fn exec_queue_user_extension_funcs[] = {	[DRM_XE_EXEC_QUEUE_EXTENSION_SET_PROPERTY] = exec_queue_user_ext_set_property,}; #define MAX_USER_EXTENSIONS	16static int exec_queue_user_extensions(struct xe_device *xe, struct xe_exec_queue *q,				      u64 extensions, int ext_number, bool create){	u64 __user *address = u64_to_user_ptr(extensions);	struct drm_xe_user_extension ext;	int err;	u32 idx; 	if (XE_IOCTL_DBG(xe, ext_number >= MAX_USER_EXTENSIONS))		return -E2BIG; 	err = __copy_from_user(&ext, address, sizeof(ext));	if (XE_IOCTL_DBG(xe, err))		return -EFAULT; 	if (XE_IOCTL_DBG(xe, ext.pad) ||	    XE_IOCTL_DBG(xe, ext.name >=			 ARRAY_SIZE(exec_queue_user_extension_funcs)))		return -EINVAL; 	idx = array_index_nospec(ext.name,				 ARRAY_SIZE(exec_queue_user_extension_funcs));	err = exec_queue_user_extension_funcs[idx](xe, q, extensions, create);	if (XE_IOCTL_DBG(xe, err))		return err; 	if (ext.next_extension)		return exec_queue_user_extensions(xe, q, ext.next_extension,					      ++ext_number, create); 	return 0;} static const enum xe_engine_class user_to_xe_engine_class[] = {	[DRM_XE_ENGINE_CLASS_RENDER] = XE_ENGINE_CLASS_RENDER,	[DRM_XE_ENGINE_CLASS_COPY] = XE_ENGINE_CLASS_COPY,	[DRM_XE_ENGINE_CLASS_VIDEO_DECODE] = XE_ENGINE_CLASS_VIDEO_DECODE,	[DRM_XE_ENGINE_CLASS_VIDEO_ENHANCE] = XE_ENGINE_CLASS_VIDEO_ENHANCE,	[DRM_XE_ENGINE_CLASS_COMPUTE] = XE_ENGINE_CLASS_COMPUTE,}; static struct xe_hw_engine *find_hw_engine(struct xe_device *xe,	       struct drm_xe_engine_class_instance eci){	u32 idx; 	if (eci.engine_class > ARRAY_SIZE(user_to_xe_engine_class))		return NULL; 	if (eci.gt_id >= xe->info.gt_count)		return NULL; 	idx = array_index_nospec(eci.engine_class,				 ARRAY_SIZE(user_to_xe_engine_class)); 	return xe_gt_hw_engine(xe_device_get_gt(xe, eci.gt_id),			       user_to_xe_engine_class[idx],			       eci.engine_instance, true);} static u32 bind_exec_queue_logical_mask(struct xe_device *xe, struct xe_gt *gt,					struct drm_xe_engine_class_instance *eci,					u16 width, u16 num_placements){	struct xe_hw_engine *hwe;	enum xe_hw_engine_id id;	u32 logical_mask = 0; 	if (XE_IOCTL_DBG(xe, width != 1))		return 0;	if (XE_IOCTL_DBG(xe, num_placements != 1))		return 0;	if (XE_IOCTL_DBG(xe, eci[0].engine_instance != 0))		return 0; 	eci[0].engine_class = DRM_XE_ENGINE_CLASS_COPY; 	for_each_hw_engine(hwe, gt, id) {		if (xe_hw_engine_is_reserved(hwe))			continue; 		if (hwe->class ==		    user_to_xe_engine_class[DRM_XE_ENGINE_CLASS_COPY])			logical_mask |= BIT(hwe->logical_instance);	} 	return logical_mask;} static u32 calc_validate_logical_mask(struct xe_device *xe, struct xe_gt *gt,				      struct drm_xe_engine_class_instance *eci,				      u16 width, u16 num_placements){	int len = width * num_placements;	int i, j, n;	u16 class;	u16 gt_id;	u32 return_mask = 0, prev_mask; 	if (XE_IOCTL_DBG(xe, !xe_device_uc_enabled(xe) &&			 len > 1))		return 0; 	for (i = 0; i < width; ++i) {		u32 current_mask = 0; 		for (j = 0; j < num_placements; ++j) {			struct xe_hw_engine *hwe; 			n = j * width + i; 			hwe = find_hw_engine(xe, eci[n]);			if (XE_IOCTL_DBG(xe, !hwe))				return 0; 			if (XE_IOCTL_DBG(xe, xe_hw_engine_is_reserved(hwe)))				return 0; 			if (XE_IOCTL_DBG(xe, n && eci[n].gt_id != gt_id) ||			    XE_IOCTL_DBG(xe, n && eci[n].engine_class != class))				return 0; 			class = eci[n].engine_class;			gt_id = eci[n].gt_id; 			if (width == 1 || !i)				return_mask |= BIT(eci[n].engine_instance);			current_mask |= BIT(eci[n].engine_instance);		} 		/* Parallel submissions must be logically contiguous */		if (i && XE_IOCTL_DBG(xe, current_mask != prev_mask << 1))			return 0; 		prev_mask = current_mask;	} 	return return_mask;} int xe_exec_queue_create_ioctl(struct drm_device *dev, void *data,

Lines 1–500 of 860. Load more