// SPDX-License-Identifier: MIT/* * Copyright 2024 Advanced Micro Devices, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. * */#include <linux/sched.h>#include <drm/drm_exec.h>#include "amdgpu.h" static const char *amdgpu_eviction_fence_get_driver_name(struct dma_fence *fence){	return "amdgpu_eviction_fence";} static const char *amdgpu_eviction_fence_get_timeline_name(struct dma_fence *f){	struct amdgpu_eviction_fence *ef; 	ef = container_of(f, struct amdgpu_eviction_fence, base);	return ef->timeline_name;} static bool amdgpu_eviction_fence_enable_signaling(struct dma_fence *f){	struct amdgpu_eviction_fence *ev_fence = to_ev_fence(f); 	schedule_work(&ev_fence->evf_mgr->suspend_work);	return true;} static const struct dma_fence_ops amdgpu_eviction_fence_ops = {	.get_driver_name = amdgpu_eviction_fence_get_driver_name,	.get_timeline_name = amdgpu_eviction_fence_get_timeline_name,	.enable_signaling = amdgpu_eviction_fence_enable_signaling,}; static voidamdgpu_eviction_fence_suspend_worker(struct work_struct *work){	struct amdgpu_eviction_fence_mgr *evf_mgr =		container_of(work, struct amdgpu_eviction_fence_mgr,			     suspend_work);	struct amdgpu_fpriv *fpriv =		container_of(evf_mgr, struct amdgpu_fpriv, evf_mgr);	struct amdgpu_userq_mgr *uq_mgr = &fpriv->userq_mgr;	struct dma_fence *ev_fence;	bool cookie; 	mutex_lock(&uq_mgr->userq_mutex); 	/*	 * This is intentionally after taking the userq_mutex since we do	 * allocate memory while holding this lock, but only after ensuring that	 * the eviction fence is signaled.	 */	cookie = dma_fence_begin_signalling(); 	ev_fence = amdgpu_evf_mgr_get_fence(evf_mgr);	amdgpu_userq_evict(uq_mgr); 	/*	 * Signaling the eviction fence must be done while holding the	 * userq_mutex. Otherwise we won't resume the queues before issuing the	 * next fence.	 */	dma_fence_signal(ev_fence);	dma_fence_end_signalling(cookie);	dma_fence_put(ev_fence); 	if (!evf_mgr->shutdown)		schedule_delayed_work(&uq_mgr->resume_work, 0); 	mutex_unlock(&uq_mgr->userq_mutex);} int amdgpu_evf_mgr_attach_fence(struct amdgpu_eviction_fence_mgr *evf_mgr,				struct amdgpu_bo *bo){	struct dma_fence *ev_fence = amdgpu_evf_mgr_get_fence(evf_mgr);	struct ttm_operation_ctx ctx = { false, false };	struct dma_resv *resv = bo->tbo.base.resv;	int ret; 	if (!dma_fence_is_signaled(ev_fence)) { 		amdgpu_bo_placement_from_domain(bo, bo->allowed_domains);		ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);		if (!ret)			dma_resv_add_fence(resv, ev_fence,					   DMA_RESV_USAGE_BOOKKEEP);	} else {		ret = 0;	} 	dma_fence_put(ev_fence);	return ret;} int amdgpu_evf_mgr_rearm(struct amdgpu_eviction_fence_mgr *evf_mgr,			 struct drm_exec *exec){	struct amdgpu_eviction_fence *ev_fence;	struct drm_gem_object *obj; 	/* Create and initialize a new eviction fence */	ev_fence = kzalloc_obj(*ev_fence);	if (!ev_fence)		return -ENOMEM; 	ev_fence->evf_mgr = evf_mgr;	get_task_comm(ev_fence->timeline_name, current);	spin_lock_init(&ev_fence->lock);	dma_fence_init64(&ev_fence->base, &amdgpu_eviction_fence_ops,			 &ev_fence->lock, evf_mgr->ev_fence_ctx,			 atomic_inc_return(&evf_mgr->ev_fence_seq)); 	/* Remember it for newly added BOs */	dma_fence_put(evf_mgr->ev_fence);	evf_mgr->ev_fence = &ev_fence->base; 	/* And add it to all existing BOs */	drm_exec_for_each_locked_object(exec, obj) {		struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj); 		amdgpu_evf_mgr_attach_fence(evf_mgr, bo);	}	return 0;} void amdgpu_evf_mgr_detach_fence(struct amdgpu_eviction_fence_mgr *evf_mgr,				 struct amdgpu_bo *bo){	struct dma_fence *stub = dma_fence_get_stub(); 	dma_resv_replace_fences(bo->tbo.base.resv, evf_mgr->ev_fence_ctx,				stub, DMA_RESV_USAGE_BOOKKEEP);	dma_fence_put(stub);} void amdgpu_evf_mgr_init(struct amdgpu_eviction_fence_mgr *evf_mgr){	atomic_set(&evf_mgr->ev_fence_seq, 0);	evf_mgr->ev_fence_ctx = dma_fence_context_alloc(1);	evf_mgr->ev_fence = dma_fence_get_stub(); 	INIT_WORK(&evf_mgr->suspend_work, amdgpu_eviction_fence_suspend_worker);} void amdgpu_evf_mgr_shutdown(struct amdgpu_eviction_fence_mgr *evf_mgr){	evf_mgr->shutdown = true;	/* Make sure that the shutdown is visible to the suspend work */	flush_work(&evf_mgr->suspend_work);} void amdgpu_evf_mgr_flush_suspend(struct amdgpu_eviction_fence_mgr *evf_mgr){	dma_fence_wait(rcu_dereference_protected(evf_mgr->ev_fence, true),		       false);	/* Make sure that we are done with the last suspend work */	flush_work(&evf_mgr->suspend_work);} void amdgpu_evf_mgr_fini(struct amdgpu_eviction_fence_mgr *evf_mgr){	dma_fence_put(evf_mgr->ev_fence);}