// SPDX-License-Identifier: MIT/* * Copyright © 2022 Intel Corporation */ #include "xe_exec.h" #include <drm/drm_device.h>#include <drm/drm_exec.h>#include <drm/drm_file.h>#include <uapi/drm/xe_drm.h>#include <linux/delay.h> #include "xe_device.h"#include "xe_exec_queue.h"#include "xe_hw_engine_group.h"#include "xe_macros.h"#include "xe_pm.h"#include "xe_ring_ops_types.h"#include "xe_sched_job.h"#include "xe_sync.h"#include "xe_svm.h"#include "xe_trace.h"#include "xe_vm.h" /** * DOC: Execbuf (User GPU command submission) * * Execs have historically been rather complicated in DRM drivers (at least in * the i915) because a few things: * * - Passing in a list BO which are read / written to creating implicit syncs * - Binding at exec time * - Flow controlling the ring at exec time * * In Xe we avoid all of this complication by not allowing a BO list to be * passed into an exec, using the dma-buf implicit sync uAPI, have binds as * separate operations, and using the DRM scheduler to flow control the ring. * Let's deep dive on each of these. * * We can get away from a BO list by forcing the user to use in / out fences on * every exec rather than the kernel tracking dependencies of BO (e.g. if the * user knows an exec writes to a BO and reads from the BO in the next exec, it * is the user's responsibility to pass in / out fence between the two execs). * * We do not allow a user to trigger a bind at exec time rather we have a VM * bind IOCTL which uses the same in / out fence interface as exec. In that * sense, a VM bind is basically the same operation as an exec from the user * perspective. e.g. If an exec depends on a VM bind use the in / out fence * interface (struct drm_xe_sync) to synchronize like syncing between two * dependent execs. * * Although a user cannot trigger a bind, we still have to rebind userptrs in * the VM that have been invalidated since the last exec, likewise we also have * to rebind BOs that have been evicted by the kernel. We schedule these rebinds * behind any pending kernel operations on any external BOs in VM or any BOs * private to the VM. This is accomplished by the rebinds waiting on BOs * DMA_RESV_USAGE_KERNEL slot (kernel ops) and kernel ops waiting on all BOs * slots (inflight execs are in the DMA_RESV_USAGE_BOOKKEEP for private BOs and * for external BOs). * * Rebinds / dma-resv usage applies to non-compute mode VMs only as for compute * mode VMs we use preempt fences and a rebind worker (TODO: add link). * * There is no need to flow control the ring in the exec as we write the ring at * submission time and set the DRM scheduler max job limit SIZE_OF_RING / * MAX_JOB_SIZE. The DRM scheduler will then hold all jobs until space in the * ring is available. * * All of this results in a rather simple exec implementation. * * Flow * ~~~~ * * .. code-block:: * * Parse input arguments * Wait for any async VM bind passed as in-fences to start * <----------------------------------------------------------------------| * Lock global VM lock in read mode | * Pin userptrs (also finds userptr invalidated since last exec) | * Lock exec (VM dma-resv lock, external BOs dma-resv locks) | * Validate BOs that have been evicted | * Create job | * Rebind invalidated userptrs + evicted BOs (non-compute-mode) | * Add rebind fence dependency to job | * Add job VM dma-resv bookkeeping slot (non-compute mode) | * Add job to external BOs dma-resv write slots (non-compute mode) | * Check if any userptrs invalidated since pin ------ Drop locks ---------| * Install in / out fences for job * Submit job * Unlock all */ /* * Add validation and rebinding to the drm_exec locking loop, since both can * trigger eviction which may require sleeping dma_resv locks. */static int xe_exec_fn(struct drm_gpuvm_exec *vm_exec){ struct xe_vm *vm = container_of(vm_exec->vm, struct xe_vm, gpuvm); int ret; /* The fence slot added here is intended for the exec sched job. */ xe_vm_set_validation_exec(vm, &vm_exec->exec); ret = xe_vm_validate_rebind(vm, &vm_exec->exec, 1); xe_vm_set_validation_exec(vm, NULL); return ret;} int xe_exec_ioctl(struct drm_device *dev, void *data, struct drm_file *file){ struct xe_device *xe = to_xe_device(dev); struct xe_file *xef = to_xe_file(file); struct drm_xe_exec *args = data; struct drm_xe_sync __user *syncs_user = u64_to_user_ptr(args->syncs); u64 __user *addresses_user = u64_to_user_ptr(args->address); struct xe_exec_queue *q; struct xe_sync_entry *syncs = NULL; u64 addresses[XE_HW_ENGINE_MAX_INSTANCE]; struct drm_gpuvm_exec vm_exec = {.extra.fn = xe_exec_fn}; struct drm_exec *exec = &vm_exec.exec; u32 i, num_syncs, num_in_sync = 0, num_ufence = 0; struct xe_validation_ctx ctx; struct xe_sched_job *job; struct xe_vm *vm; bool write_locked; int err = 0; struct xe_hw_engine_group *group; enum xe_hw_engine_group_execution_mode mode, previous_mode; if (XE_IOCTL_DBG(xe, args->extensions) || XE_IOCTL_DBG(xe, args->pad[0] || args->pad[1] || args->pad[2]) || XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1]) || XE_IOCTL_DBG(xe, args->num_syncs > DRM_XE_MAX_SYNCS)) return -EINVAL; q = xe_exec_queue_lookup(xef, args->exec_queue_id); if (XE_IOCTL_DBG(xe, !q)) return -ENOENT; if (XE_IOCTL_DBG(xe, q->flags & EXEC_QUEUE_FLAG_VM)) { err = -EINVAL; goto err_exec_queue; } if (XE_IOCTL_DBG(xe, args->num_batch_buffer && q->width != args->num_batch_buffer)) { err = -EINVAL; goto err_exec_queue; } if (XE_IOCTL_DBG(xe, q->ops->reset_status(q))) { err = -ECANCELED; goto err_exec_queue; } if (atomic_read(&q->job_cnt) >= XE_MAX_JOB_COUNT_PER_EXEC_QUEUE) { trace_xe_exec_queue_reach_max_job_count(q, XE_MAX_JOB_COUNT_PER_EXEC_QUEUE); err = -EAGAIN; goto err_exec_queue; } if (args->num_syncs) { syncs = kzalloc_objs(*syncs, args->num_syncs); if (!syncs) { err = -ENOMEM; goto err_exec_queue; } } vm = q->vm; for (num_syncs = 0; num_syncs < args->num_syncs; num_syncs++) { err = xe_sync_entry_parse(xe, xef, &syncs[num_syncs], &syncs_user[num_syncs], NULL, 0, SYNC_PARSE_FLAG_EXEC | (xe_vm_in_lr_mode(vm) ? SYNC_PARSE_FLAG_LR_MODE : 0)); if (err) goto err_syncs; if (xe_sync_is_ufence(&syncs[num_syncs])) num_ufence++; if (!num_in_sync && xe_sync_needs_wait(&syncs[num_syncs])) num_in_sync++; } if (XE_IOCTL_DBG(xe, num_ufence > 1)) { err = -EINVAL; goto err_syncs; } if (args->num_batch_buffer && xe_exec_queue_is_parallel(q)) { err = copy_from_user(addresses, addresses_user, sizeof(u64) * q->width); if (err) { err = -EFAULT; goto err_syncs; } } group = q->hwe->hw_engine_group; mode = xe_hw_engine_group_find_exec_mode(q); if (mode == EXEC_MODE_DMA_FENCE) { err = xe_hw_engine_group_get_mode(group, mode, &previous_mode, syncs, num_in_sync ? num_syncs : 0); if (err) goto err_syncs; } retry: if (!xe_vm_in_lr_mode(vm) && xe_vm_userptr_check_repin(vm)) { err = down_write_killable(&vm->lock); write_locked = true; } else { /* We don't allow execs while the VM is in error state */ err = down_read_interruptible(&vm->lock); write_locked = false; } if (err) goto err_hw_exec_mode; if (write_locked) { err = xe_vm_userptr_pin(vm); downgrade_write(&vm->lock); write_locked = false; if (err) goto err_unlock_list; } if (!args->num_batch_buffer) { err = xe_vm_lock(vm, true); if (err) goto err_unlock_list; if (!xe_vm_in_lr_mode(vm)) { struct dma_fence *fence; fence = xe_sync_in_fence_get(syncs, num_syncs, q, vm); if (IS_ERR(fence)) { err = PTR_ERR(fence); xe_vm_unlock(vm); goto err_unlock_list; } for (i = 0; i < num_syncs; i++) xe_sync_entry_signal(&syncs[i], fence); xe_exec_queue_last_fence_set(q, vm, fence); dma_fence_put(fence); } xe_vm_unlock(vm); goto err_unlock_list; } /* * It's OK to block interruptible here with the vm lock held, since * on task freezing during suspend / hibernate, the call will * return -ERESTARTSYS and the IOCTL will be rerun. */ err = xe_pm_block_on_suspend(xe); if (err) goto err_unlock_list; if (!xe_vm_in_lr_mode(vm)) { vm_exec.vm = &vm->gpuvm; vm_exec.flags = DRM_EXEC_INTERRUPTIBLE_WAIT; err = xe_validation_exec_lock(&ctx, &vm_exec, &xe->val); if (err) goto err_unlock_list; } if (xe_vm_is_closed_or_banned(q->vm)) { drm_warn(&xe->drm, "Trying to schedule after vm is closed or banned\n"); err = -ECANCELED; goto err_exec; } if (xe_exec_queue_uses_pxp(q)) { err = xe_vm_validate_protected(q->vm); if (err) goto err_exec; } job = xe_sched_job_create(q, xe_exec_queue_is_parallel(q) ? addresses : &args->address); if (IS_ERR(job)) { err = PTR_ERR(job); goto err_exec; } /* * Wait behind rebinds and any kernel operations (evictions, defrag * moves, ...) on the VM and all external BOs. The VM's private BOs * carry their kernel ops in the VM dma-resv KERNEL slot, while each * external BO carries them in its own dma-resv KERNEL slot; both are * covered by iterating every object locked by the exec, mirroring the * drm_gpuvm_resv_add_fence() below. */ if (!xe_vm_in_lr_mode(vm)) { struct drm_gem_object *obj; drm_exec_for_each_locked_object(exec, obj) { err = xe_sched_job_add_deps(job, obj->resv, DMA_RESV_USAGE_KERNEL); if (err) goto err_put_job; } } for (i = 0; i < num_syncs && !err; i++) err = xe_sync_entry_add_deps(&syncs[i], job); if (err) goto err_put_job; if (!xe_vm_in_lr_mode(vm)) { err = xe_svm_notifier_lock_interruptible(vm); if (err) goto err_put_job; err = __xe_vm_userptr_needs_repin(vm); if (err) goto err_repin; } /* * Point of no return, if we error after this point just set an error on * the job and let the DRM scheduler / backend clean up the job. */ xe_sched_job_arm(job); if (!xe_vm_in_lr_mode(vm)) drm_gpuvm_resv_add_fence(&vm->gpuvm, exec, &job->drm.s_fence->finished, DMA_RESV_USAGE_BOOKKEEP, DMA_RESV_USAGE_BOOKKEEP); for (i = 0; i < num_syncs; i++) { xe_sync_entry_signal(&syncs[i], &job->drm.s_fence->finished); xe_sched_job_init_user_fence(job, &syncs[i]); } if (!xe_vm_in_lr_mode(vm)) xe_exec_queue_last_fence_set(q, vm, &job->drm.s_fence->finished); xe_sched_job_push(job); xe_vm_reactivate_rebind(vm); if (!err && !xe_vm_in_lr_mode(vm)) { spin_lock(&xe->ttm.lru_lock); ttm_lru_bulk_move_tail(&vm->lru_bulk_move); spin_unlock(&xe->ttm.lru_lock); } if (mode == EXEC_MODE_LR) xe_hw_engine_group_resume_faulting_lr_jobs(group); err_repin: if (!xe_vm_in_lr_mode(vm)) xe_svm_notifier_unlock(vm);err_put_job: if (err) xe_sched_job_put(job);err_exec: if (!xe_vm_in_lr_mode(vm)) xe_validation_ctx_fini(&ctx);err_unlock_list: up_read(&vm->lock); if (err == -EAGAIN) goto retry;err_hw_exec_mode: if (mode == EXEC_MODE_DMA_FENCE) xe_hw_engine_group_put(group);err_syncs: while (num_syncs--) xe_sync_entry_cleanup(&syncs[num_syncs]); kfree(syncs);err_exec_queue: xe_exec_queue_put(q); return err;}
// SPDX-License-Identifier: MIT
/*
* Copyright © 2022 Intel Corporation
*/
#include "xe_exec.h"
#include <drm/drm_device.h>
#include <drm/drm_exec.h>
#include <drm/drm_file.h>
#include <uapi/drm/xe_drm.h>
#include <linux/delay.h>
#include "xe_device.h"
#include "xe_exec_queue.h"
#include "xe_hw_engine_group.h"
#include "xe_macros.h"
#include "xe_pm.h"
#include "xe_ring_ops_types.h"
#include "xe_sched_job.h"
#include "xe_sync.h"
#include "xe_svm.h"
#include "xe_trace.h"
#include "xe_vm.h"
/**
* DOC: Execbuf (User GPU command submission)
*
* Execs have historically been rather complicated in DRM drivers (at least in
* the i915) because a few things:
*
* - Passing in a list BO which are read / written to creating implicit syncs
* - Binding at exec time
* - Flow controlling the ring at exec time
*
* In Xe we avoid all of this complication by not allowing a BO list to be
* passed into an exec, using the dma-buf implicit sync uAPI, have binds as
* separate operations, and using the DRM scheduler to flow control the ring.
* Let's deep dive on each of these.
*
* We can get away from a BO list by forcing the user to use in / out fences on
* every exec rather than the kernel tracking dependencies of BO (e.g. if the
* user knows an exec writes to a BO and reads from the BO in the next exec, it
* is the user's responsibility to pass in / out fence between the two execs).
*
* We do not allow a user to trigger a bind at exec time rather we have a VM
* bind IOCTL which uses the same in / out fence interface as exec. In that
* sense, a VM bind is basically the same operation as an exec from the user
* perspective. e.g. If an exec depends on a VM bind use the in / out fence
* interface (struct drm_xe_sync) to synchronize like syncing between two
* dependent execs.
*
* Although a user cannot trigger a bind, we still have to rebind userptrs in
* the VM that have been invalidated since the last exec, likewise we also have
* to rebind BOs that have been evicted by the kernel. We schedule these rebinds
* behind any pending kernel operations on any external BOs in VM or any BOs
* private to the VM. This is accomplished by the rebinds waiting on BOs
* DMA_RESV_USAGE_KERNEL slot (kernel ops) and kernel ops waiting on all BOs
* slots (inflight execs are in the DMA_RESV_USAGE_BOOKKEEP for private BOs and
* for external BOs).
*
* Rebinds / dma-resv usage applies to non-compute mode VMs only as for compute
* mode VMs we use preempt fences and a rebind worker (TODO: add link).
*
* There is no need to flow control the ring in the exec as we write the ring at
* submission time and set the DRM scheduler max job limit SIZE_OF_RING /
* MAX_JOB_SIZE. The DRM scheduler will then hold all jobs until space in the
* ring is available.
*
* All of this results in a rather simple exec implementation.
*
* Flow
* ~~~~
*
* .. code-block::
*
* Parse input arguments
* Wait for any async VM bind passed as in-fences to start
* <----------------------------------------------------------------------|
* Lock global VM lock in read mode |
* Pin userptrs (also finds userptr invalidated since last exec) |
* Lock exec (VM dma-resv lock, external BOs dma-resv locks) |
* Validate BOs that have been evicted |
* Create job |
* Rebind invalidated userptrs + evicted BOs (non-compute-mode) |
* Add rebind fence dependency to job |
* Add job VM dma-resv bookkeeping slot (non-compute mode) |
* Add job to external BOs dma-resv write slots (non-compute mode) |
* Check if any userptrs invalidated since pin ------ Drop locks ---------|
* Install in / out fences for job
* Submit job
* Unlock all
*/
/*
* Add validation and rebinding to the drm_exec locking loop, since both can
* trigger eviction which may require sleeping dma_resv locks.
*/
static int xe_exec_fn(struct drm_gpuvm_exec *vm_exec)
{
struct xe_vm *vm = container_of(vm_exec->vm, struct xe_vm, gpuvm);
int ret;
/* The fence slot added here is intended for the exec sched job. */
xe_vm_set_validation_exec(vm, &vm_exec->exec);
ret = xe_vm_validate_rebind(vm, &vm_exec->exec, 1);
xe_vm_set_validation_exec(vm, NULL);
return ret;
}
int xe_exec_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
{
struct xe_device *xe = to_xe_device(dev);
struct xe_file *xef = to_xe_file(file);
struct drm_xe_exec *args = data;
struct drm_xe_sync __user *syncs_user = u64_to_user_ptr(args->syncs);
u64 __user *addresses_user = u64_to_user_ptr(args->address);
struct xe_exec_queue *q;
struct xe_sync_entry *syncs = NULL;
u64 addresses[XE_HW_ENGINE_MAX_INSTANCE];
struct drm_gpuvm_exec vm_exec = {.extra.fn = xe_exec_fn};
struct drm_exec *exec = &vm_exec.exec;
u32 i, num_syncs, num_in_sync = 0, num_ufence = 0;
struct xe_validation_ctx ctx;
struct xe_sched_job *job;
struct xe_vm *vm;
bool write_locked;
int err = 0;
struct xe_hw_engine_group *group;
enum xe_hw_engine_group_execution_mode mode, previous_mode;
if (XE_IOCTL_DBG(xe, args->extensions) ||
XE_IOCTL_DBG(xe, args->pad[0] || args->pad[1] || args->pad[2]) ||
XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1]) ||
XE_IOCTL_DBG(xe, args->num_syncs > DRM_XE_MAX_SYNCS))
return -EINVAL;
q = xe_exec_queue_lookup(xef, args->exec_queue_id);
if (XE_IOCTL_DBG(xe, !q))
return -ENOENT;
if (XE_IOCTL_DBG(xe, q->flags & EXEC_QUEUE_FLAG_VM)) {
err = -EINVAL;
goto err_exec_queue;
}
if (XE_IOCTL_DBG(xe, args->num_batch_buffer &&
q->width != args->num_batch_buffer)) {
err = -EINVAL;
goto err_exec_queue;
}
if (XE_IOCTL_DBG(xe, q->ops->reset_status(q))) {
err = -ECANCELED;
goto err_exec_queue;
}
if (atomic_read(&q->job_cnt) >= XE_MAX_JOB_COUNT_PER_EXEC_QUEUE) {
trace_xe_exec_queue_reach_max_job_count(q, XE_MAX_JOB_COUNT_PER_EXEC_QUEUE);
err = -EAGAIN;
goto err_exec_queue;
}
if (args->num_syncs) {
syncs = kzalloc_objs(*syncs, args->num_syncs);
if (!syncs) {
err = -ENOMEM;
goto err_exec_queue;
}
}
vm = q->vm;
for (num_syncs = 0; num_syncs < args->num_syncs; num_syncs++) {
err = xe_sync_entry_parse(xe, xef, &syncs[num_syncs],
&syncs_user[num_syncs], NULL, 0,
SYNC_PARSE_FLAG_EXEC |
(xe_vm_in_lr_mode(vm) ?
SYNC_PARSE_FLAG_LR_MODE : 0));
if (err)
goto err_syncs;
if (xe_sync_is_ufence(&syncs[num_syncs]))
num_ufence++;
if (!num_in_sync && xe_sync_needs_wait(&syncs[num_syncs]))
num_in_sync++;
}
if (XE_IOCTL_DBG(xe, num_ufence > 1)) {
err = -EINVAL;
goto err_syncs;
}
if (args->num_batch_buffer && xe_exec_queue_is_parallel(q)) {
err = copy_from_user(addresses, addresses_user,
sizeof(u64) * q->width);
if (err) {
err = -EFAULT;
goto err_syncs;
}
}
group = q->hwe->hw_engine_group;
mode = xe_hw_engine_group_find_exec_mode(q);
if (mode == EXEC_MODE_DMA_FENCE) {
err = xe_hw_engine_group_get_mode(group, mode, &previous_mode,
syncs, num_in_sync ?
num_syncs : 0);
if (err)
goto err_syncs;
}
retry:
if (!xe_vm_in_lr_mode(vm) && xe_vm_userptr_check_repin(vm)) {
err = down_write_killable(&vm->lock);
write_locked = true;
} else {
/* We don't allow execs while the VM is in error state */
err = down_read_interruptible(&vm->lock);
write_locked = false;
}
if (err)
goto err_hw_exec_mode;
if (write_locked) {
err = xe_vm_userptr_pin(vm);
downgrade_write(&vm->lock);
write_locked = false;
if (err)
goto err_unlock_list;
}
if (!args->num_batch_buffer) {
err = xe_vm_lock(vm, true);
if (err)
goto err_unlock_list;
if (!xe_vm_in_lr_mode(vm)) {
struct dma_fence *fence;
fence = xe_sync_in_fence_get(syncs, num_syncs, q, vm);
if (IS_ERR(fence)) {
err = PTR_ERR(fence);
xe_vm_unlock(vm);
goto err_unlock_list;
}
for (i = 0; i < num_syncs; i++)
xe_sync_entry_signal(&syncs[i], fence);
xe_exec_queue_last_fence_set(q, vm, fence);
dma_fence_put(fence);
}
xe_vm_unlock(vm);
goto err_unlock_list;
}
/*
* It's OK to block interruptible here with the vm lock held, since
* on task freezing during suspend / hibernate, the call will
* return -ERESTARTSYS and the IOCTL will be rerun.
*/
err = xe_pm_block_on_suspend(xe);
if (err)
goto err_unlock_list;
if (!xe_vm_in_lr_mode(vm)) {
vm_exec.vm = &vm->gpuvm;
vm_exec.flags = DRM_EXEC_INTERRUPTIBLE_WAIT;
err = xe_validation_exec_lock(&ctx, &vm_exec, &xe->val);
if (err)
goto err_unlock_list;
}
if (xe_vm_is_closed_or_banned(q->vm)) {
drm_warn(&xe->drm, "Trying to schedule after vm is closed or banned\n");
err = -ECANCELED;
goto err_exec;
}
if (xe_exec_queue_uses_pxp(q)) {
err = xe_vm_validate_protected(q->vm);
if (err)
goto err_exec;
}
job = xe_sched_job_create(q, xe_exec_queue_is_parallel(q) ?
addresses : &args->address);
if (IS_ERR(job)) {
err = PTR_ERR(job);
goto err_exec;
}
/*
* Wait behind rebinds and any kernel operations (evictions, defrag
* moves, ...) on the VM and all external BOs. The VM's private BOs
* carry their kernel ops in the VM dma-resv KERNEL slot, while each
* external BO carries them in its own dma-resv KERNEL slot; both are
* covered by iterating every object locked by the exec, mirroring the
* drm_gpuvm_resv_add_fence() below.
*/
if (!xe_vm_in_lr_mode(vm)) {
struct drm_gem_object *obj;
drm_exec_for_each_locked_object(exec, obj) {
err = xe_sched_job_add_deps(job, obj->resv,
DMA_RESV_USAGE_KERNEL);
if (err)
goto err_put_job;
}
}
for (i = 0; i < num_syncs && !err; i++)
err = xe_sync_entry_add_deps(&syncs[i], job);
if (err)
goto err_put_job;
if (!xe_vm_in_lr_mode(vm)) {
err = xe_svm_notifier_lock_interruptible(vm);
if (err)
goto err_put_job;
err = __xe_vm_userptr_needs_repin(vm);
if (err)
goto err_repin;
}
/*
* Point of no return, if we error after this point just set an error on
* the job and let the DRM scheduler / backend clean up the job.
*/
xe_sched_job_arm(job);
if (!xe_vm_in_lr_mode(vm))
drm_gpuvm_resv_add_fence(&vm->gpuvm, exec, &job->drm.s_fence->finished,
DMA_RESV_USAGE_BOOKKEEP,
DMA_RESV_USAGE_BOOKKEEP);
for (i = 0; i < num_syncs; i++) {
xe_sync_entry_signal(&syncs[i], &job->drm.s_fence->finished);
xe_sched_job_init_user_fence(job, &syncs[i]);
}
if (!xe_vm_in_lr_mode(vm))
xe_exec_queue_last_fence_set(q, vm, &job->drm.s_fence->finished);
xe_sched_job_push(job);
xe_vm_reactivate_rebind(vm);
if (!err && !xe_vm_in_lr_mode(vm)) {
spin_lock(&xe->ttm.lru_lock);
ttm_lru_bulk_move_tail(&vm->lru_bulk_move);
spin_unlock(&xe->ttm.lru_lock);
}
if (mode == EXEC_MODE_LR)
xe_hw_engine_group_resume_faulting_lr_jobs(group);
err_repin:
if (!xe_vm_in_lr_mode(vm))
xe_svm_notifier_unlock(vm);
err_put_job:
if (err)
xe_sched_job_put(job);
err_exec:
if (!xe_vm_in_lr_mode(vm))
xe_validation_ctx_fini(&ctx);
err_unlock_list:
up_read(&vm->lock);
if (err == -EAGAIN)
goto retry;
err_hw_exec_mode:
if (mode == EXEC_MODE_DMA_FENCE)
xe_hw_engine_group_put(group);
err_syncs:
while (num_syncs--)
xe_sync_entry_cleanup(&syncs[num_syncs]);
kfree(syncs);
err_exec_queue:
xe_exec_queue_put(q);
return err;
}