// SPDX-License-Identifier: MIT/* * Copyright © 2021 Intel Corporation */ #include "xe_exec_queue.h" #include <linux/nospec.h> #include <drm/drm_device.h>#include <drm/drm_drv.h>#include <drm/drm_file.h>#include <uapi/drm/xe_drm.h> #include "xe_device.h"#include "xe_gt.h"#include "xe_hw_engine_class_sysfs.h"#include "xe_hw_engine_group.h"#include "xe_hw_fence.h"#include "xe_lrc.h"#include "xe_macros.h"#include "xe_migrate.h"#include "xe_pm.h"#include "xe_ring_ops_types.h"#include "xe_trace.h"#include "xe_vm.h" enum xe_exec_queue_sched_prop {	XE_EXEC_QUEUE_JOB_TIMEOUT = 0,	XE_EXEC_QUEUE_TIMESLICE = 1,	XE_EXEC_QUEUE_PREEMPT_TIMEOUT = 2,	XE_EXEC_QUEUE_SCHED_PROP_MAX = 3,}; static int exec_queue_user_extensions(struct xe_device *xe, struct xe_exec_queue *q,				      u64 extensions, int ext_number); static void __xe_exec_queue_free(struct xe_exec_queue *q){	if (q->vm)		xe_vm_put(q->vm); 	if (q->xef)		xe_file_put(q->xef); 	kfree(q);} static struct xe_exec_queue *__xe_exec_queue_alloc(struct xe_device *xe,						   struct xe_vm *vm,						   u32 logical_mask,						   u16 width, struct xe_hw_engine *hwe,						   u32 flags, u64 extensions){	struct xe_exec_queue *q;	struct xe_gt *gt = hwe->gt;	int err; 	/* only kernel queues can be permanent */	XE_WARN_ON((flags & EXEC_QUEUE_FLAG_PERMANENT) && !(flags & EXEC_QUEUE_FLAG_KERNEL)); 	q = kzalloc(struct_size(q, lrc, width), GFP_KERNEL);	if (!q)		return ERR_PTR(-ENOMEM); 	kref_init(&q->refcount);	q->flags = flags;	q->hwe = hwe;	q->gt = gt;	q->class = hwe->class;	q->width = width;	q->logical_mask = logical_mask;	q->fence_irq = &gt->fence_irq[hwe->class];	q->ring_ops = gt->ring_ops[hwe->class];	q->ops = gt->exec_queue_ops;	INIT_LIST_HEAD(&q->lr.link);	INIT_LIST_HEAD(&q->multi_gt_link);	INIT_LIST_HEAD(&q->hw_engine_group_link); 	q->sched_props.timeslice_us = hwe->eclass->sched_props.timeslice_us;	q->sched_props.preempt_timeout_us =				hwe->eclass->sched_props.preempt_timeout_us;	q->sched_props.job_timeout_ms =				hwe->eclass->sched_props.job_timeout_ms;	if (q->flags & EXEC_QUEUE_FLAG_KERNEL &&	    q->flags & EXEC_QUEUE_FLAG_HIGH_PRIORITY)		q->sched_props.priority = XE_EXEC_QUEUE_PRIORITY_KERNEL;	else		q->sched_props.priority = XE_EXEC_QUEUE_PRIORITY_NORMAL; 	if (vm)		q->vm = xe_vm_get(vm); 	if (extensions) {		/*		 * may set q->usm, must come before xe_lrc_create(),		 * may overwrite q->sched_props, must come before q->ops->init()		 */		err = exec_queue_user_extensions(xe, q, extensions, 0);		if (err) {			__xe_exec_queue_free(q);			return ERR_PTR(err);		}	} 	return q;} static int __xe_exec_queue_init(struct xe_exec_queue *q){	struct xe_vm *vm = q->vm;	int i, err; 	if (vm) {		err = xe_vm_lock(vm, true);		if (err)			return err;	} 	for (i = 0; i < q->width; ++i) {		q->lrc[i] = xe_lrc_create(q->hwe, q->vm, SZ_16K);		if (IS_ERR(q->lrc[i])) {			err = PTR_ERR(q->lrc[i]);			goto err_unlock;		}	} 	if (vm)		xe_vm_unlock(vm); 	err = q->ops->init(q);	if (err)		goto err_lrc; 	return 0; err_unlock:	if (vm)		xe_vm_unlock(vm);err_lrc:	for (i = i - 1; i >= 0; --i)		xe_lrc_put(q->lrc[i]);	return err;} struct xe_exec_queue *xe_exec_queue_create(struct xe_device *xe, struct xe_vm *vm,					   u32 logical_mask, u16 width,					   struct xe_hw_engine *hwe, u32 flags,					   u64 extensions){	struct xe_exec_queue *q;	int err; 	q = __xe_exec_queue_alloc(xe, vm, logical_mask, width, hwe, flags,				  extensions);	if (IS_ERR(q))		return q; 	err = __xe_exec_queue_init(q);	if (err)		goto err_post_alloc; 	return q; err_post_alloc:	__xe_exec_queue_free(q);	return ERR_PTR(err);} struct xe_exec_queue *xe_exec_queue_create_class(struct xe_device *xe, struct xe_gt *gt,						 struct xe_vm *vm,						 enum xe_engine_class class,						 u32 flags, u64 extensions){	struct xe_hw_engine *hwe, *hwe0 = NULL;	enum xe_hw_engine_id id;	u32 logical_mask = 0; 	for_each_hw_engine(hwe, gt, id) {		if (xe_hw_engine_is_reserved(hwe))			continue; 		if (hwe->class == class) {			logical_mask |= BIT(hwe->logical_instance);			if (!hwe0)				hwe0 = hwe;		}	} 	if (!logical_mask)		return ERR_PTR(-ENODEV); 	return xe_exec_queue_create(xe, vm, logical_mask, 1, hwe0, flags, extensions);} /** * xe_exec_queue_create_bind() - Create bind exec queue. * @xe: Xe device. * @tile: tile which bind exec queue belongs to. * @flags: exec queue creation flags * @extensions: exec queue creation extensions * * Normalize bind exec queue creation. Bind exec queue is tied to migration VM * for access to physical memory required for page table programming. On a * faulting devices the reserved copy engine instance must be used to avoid * deadlocking (user binds cannot get stuck behind faults as kernel binds which * resolve faults depend on user binds). On non-faulting devices any copy engine * can be used. * * Returns exec queue on success, ERR_PTR on failure */struct xe_exec_queue *xe_exec_queue_create_bind(struct xe_device *xe,						struct xe_tile *tile,						u32 flags, u64 extensions){	struct xe_gt *gt = tile->primary_gt;	struct xe_exec_queue *q;	struct xe_vm *migrate_vm; 	migrate_vm = xe_migrate_get_vm(tile->migrate);	if (xe->info.has_usm) {		struct xe_hw_engine *hwe = xe_gt_hw_engine(gt,							   XE_ENGINE_CLASS_COPY,							   gt->usm.reserved_bcs_instance,							   false); 		if (!hwe) {			xe_vm_put(migrate_vm);			return ERR_PTR(-EINVAL);		} 		q = xe_exec_queue_create(xe, migrate_vm,					 BIT(hwe->logical_instance), 1, hwe,					 flags, extensions);	} else {		q = xe_exec_queue_create_class(xe, gt, migrate_vm,					       XE_ENGINE_CLASS_COPY, flags,					       extensions);	}	xe_vm_put(migrate_vm); 	return q;} void xe_exec_queue_destroy(struct kref *ref){	struct xe_exec_queue *q = container_of(ref, struct xe_exec_queue, refcount);	struct xe_exec_queue *eq, *next; 	xe_exec_queue_last_fence_put_unlocked(q);	if (!(q->flags & EXEC_QUEUE_FLAG_BIND_ENGINE_CHILD)) {		list_for_each_entry_safe(eq, next, &q->multi_gt_list,					 multi_gt_link)			xe_exec_queue_put(eq);	} 	q->ops->fini(q);} void xe_exec_queue_fini(struct xe_exec_queue *q){	int i; 	/*	 * Before releasing our ref to lrc and xef, accumulate our run ticks	 */	xe_exec_queue_update_run_ticks(q); 	for (i = 0; i < q->width; ++i)		xe_lrc_put(q->lrc[i]); 	__xe_exec_queue_free(q);} void xe_exec_queue_assign_name(struct xe_exec_queue *q, u32 instance){	switch (q->class) {	case XE_ENGINE_CLASS_RENDER:		snprintf(q->name, sizeof(q->name), "rcs%d", instance);		break;	case XE_ENGINE_CLASS_VIDEO_DECODE:		snprintf(q->name, sizeof(q->name), "vcs%d", instance);		break;	case XE_ENGINE_CLASS_VIDEO_ENHANCE:		snprintf(q->name, sizeof(q->name), "vecs%d", instance);		break;	case XE_ENGINE_CLASS_COPY:		snprintf(q->name, sizeof(q->name), "bcs%d", instance);		break;	case XE_ENGINE_CLASS_COMPUTE:		snprintf(q->name, sizeof(q->name), "ccs%d", instance);		break;	case XE_ENGINE_CLASS_OTHER:		snprintf(q->name, sizeof(q->name), "gsccs%d", instance);		break;	default:		XE_WARN_ON(q->class);	}} struct xe_exec_queue *xe_exec_queue_lookup(struct xe_file *xef, u32 id){	struct xe_exec_queue *q; 	mutex_lock(&xef->exec_queue.lock);	q = xa_load(&xef->exec_queue.xa, id);	if (q)		xe_exec_queue_get(q);	mutex_unlock(&xef->exec_queue.lock); 	return q;} enum xe_exec_queue_priorityxe_exec_queue_device_get_max_priority(struct xe_device *xe){	return capable(CAP_SYS_NICE) ? XE_EXEC_QUEUE_PRIORITY_HIGH :				       XE_EXEC_QUEUE_PRIORITY_NORMAL;} static int exec_queue_set_priority(struct xe_device *xe, struct xe_exec_queue *q,				   u64 value){	if (XE_IOCTL_DBG(xe, value > XE_EXEC_QUEUE_PRIORITY_HIGH))		return -EINVAL; 	if (XE_IOCTL_DBG(xe, value > xe_exec_queue_device_get_max_priority(xe)))		return -EPERM; 	q->sched_props.priority = value;	return 0;} static bool xe_exec_queue_enforce_schedule_limit(void){#if IS_ENABLED(CONFIG_DRM_XE_ENABLE_SCHEDTIMEOUT_LIMIT)	return true;#else	return !capable(CAP_SYS_NICE);#endif} static voidxe_exec_queue_get_prop_minmax(struct xe_hw_engine_class_intf *eclass,			      enum xe_exec_queue_sched_prop prop,			      u32 *min, u32 *max){	switch (prop) {	case XE_EXEC_QUEUE_JOB_TIMEOUT:		*min = eclass->sched_props.job_timeout_min;		*max = eclass->sched_props.job_timeout_max;		break;	case XE_EXEC_QUEUE_TIMESLICE:		*min = eclass->sched_props.timeslice_min;		*max = eclass->sched_props.timeslice_max;		break;	case XE_EXEC_QUEUE_PREEMPT_TIMEOUT:		*min = eclass->sched_props.preempt_timeout_min;		*max = eclass->sched_props.preempt_timeout_max;		break;	default:		break;	}#if IS_ENABLED(CONFIG_DRM_XE_ENABLE_SCHEDTIMEOUT_LIMIT)	if (capable(CAP_SYS_NICE)) {		switch (prop) {		case XE_EXEC_QUEUE_JOB_TIMEOUT:			*min = XE_HW_ENGINE_JOB_TIMEOUT_MIN;			*max = XE_HW_ENGINE_JOB_TIMEOUT_MAX;			break;		case XE_EXEC_QUEUE_TIMESLICE:			*min = XE_HW_ENGINE_TIMESLICE_MIN;			*max = XE_HW_ENGINE_TIMESLICE_MAX;			break;		case XE_EXEC_QUEUE_PREEMPT_TIMEOUT:			*min = XE_HW_ENGINE_PREEMPT_TIMEOUT_MIN;			*max = XE_HW_ENGINE_PREEMPT_TIMEOUT_MAX;			break;		default:			break;		}	}#endif} static int exec_queue_set_timeslice(struct xe_device *xe, struct xe_exec_queue *q,				    u64 value){	u32 min = 0, max = 0; 	xe_exec_queue_get_prop_minmax(q->hwe->eclass,				      XE_EXEC_QUEUE_TIMESLICE, &min, &max); 	if (xe_exec_queue_enforce_schedule_limit() &&	    !xe_hw_engine_timeout_in_range(value, min, max))		return -EINVAL; 	q->sched_props.timeslice_us = value;	return 0;} typedef int (*xe_exec_queue_set_property_fn)(struct xe_device *xe,					     struct xe_exec_queue *q,					     u64 value); static const xe_exec_queue_set_property_fn exec_queue_set_property_funcs[] = {	[DRM_XE_EXEC_QUEUE_SET_PROPERTY_PRIORITY] = exec_queue_set_priority,	[DRM_XE_EXEC_QUEUE_SET_PROPERTY_TIMESLICE] = exec_queue_set_timeslice,}; static int exec_queue_user_ext_set_property(struct xe_device *xe,					    struct xe_exec_queue *q,					    u64 extension){	u64 __user *address = u64_to_user_ptr(extension);	struct drm_xe_ext_set_property ext;	int err;	u32 idx; 	err = __copy_from_user(&ext, address, sizeof(ext));	if (XE_IOCTL_DBG(xe, err))		return -EFAULT; 	if (XE_IOCTL_DBG(xe, ext.property >=			 ARRAY_SIZE(exec_queue_set_property_funcs)) ||	    XE_IOCTL_DBG(xe, ext.pad) ||	    XE_IOCTL_DBG(xe, ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_PRIORITY &&			 ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_TIMESLICE))		return -EINVAL; 	idx = array_index_nospec(ext.property, ARRAY_SIZE(exec_queue_set_property_funcs));	if (!exec_queue_set_property_funcs[idx])		return -EINVAL; 	return exec_queue_set_property_funcs[idx](xe, q, ext.value);} typedef int (*xe_exec_queue_user_extension_fn)(struct xe_device *xe,					       struct xe_exec_queue *q,					       u64 extension); static const xe_exec_queue_user_extension_fn exec_queue_user_extension_funcs[] = {	[DRM_XE_EXEC_QUEUE_EXTENSION_SET_PROPERTY] = exec_queue_user_ext_set_property,}; #define MAX_USER_EXTENSIONS	16static int exec_queue_user_extensions(struct xe_device *xe, struct xe_exec_queue *q,				      u64 extensions, int ext_number){	u64 __user *address = u64_to_user_ptr(extensions);	struct drm_xe_user_extension ext;	int err;	u32 idx; 	if (XE_IOCTL_DBG(xe, ext_number >= MAX_USER_EXTENSIONS))		return -E2BIG; 	err = __copy_from_user(&ext, address, sizeof(ext));	if (XE_IOCTL_DBG(xe, err))		return -EFAULT; 	if (XE_IOCTL_DBG(xe, ext.pad) ||	    XE_IOCTL_DBG(xe, ext.name >=			 ARRAY_SIZE(exec_queue_user_extension_funcs)))		return -EINVAL; 	idx = array_index_nospec(ext.name,				 ARRAY_SIZE(exec_queue_user_extension_funcs));	err = exec_queue_user_extension_funcs[idx](xe, q, extensions);	if (XE_IOCTL_DBG(xe, err))		return err; 	if (ext.next_extension)		return exec_queue_user_extensions(xe, q, ext.next_extension,						  ++ext_number); 	return 0;} static u32 calc_validate_logical_mask(struct xe_device *xe, struct xe_gt *gt,				      struct drm_xe_engine_class_instance *eci,				      u16 width, u16 num_placements){	int len = width * num_placements;	int i, j, n;	u16 class;	u16 gt_id;	u32 return_mask = 0, prev_mask; 	if (XE_IOCTL_DBG(xe, !xe_device_uc_enabled(xe) &&			 len > 1))		return 0; 	for (i = 0; i < width; ++i) {		u32 current_mask = 0; 		for (j = 0; j < num_placements; ++j) {			struct xe_hw_engine *hwe; 			n = j * width + i;

Lines 1–500 of 981. Load more