/* * Copyright 2009 Jerome Glisse. * All Rights Reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the * "Software"), to deal in the Software without restriction, including * without limitation the rights to use, copy, modify, merge, publish, * distribute, sub license, and/or sell copies of the Software, and to * permit persons to whom the Software is furnished to do so, subject to * the following conditions: * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE * USE OR OTHER DEALINGS IN THE SOFTWARE. * * The above copyright notice and this permission notice (including the * next paragraph) shall be included in all copies or substantial portions * of the Software. * *//* * Authors: *    Jerome Glisse <glisse@freedesktop.org> *    Dave Airlie */#include <linux/seq_file.h>#include <linux/atomic.h>#include <linux/wait.h>#include <linux/kref.h>#include <linux/slab.h>#include <linux/firmware.h>#include <drm/drmP.h>#include "amdgpu.h"#include "amdgpu_trace.h" /* * Fences * Fences mark an event in the GPUs pipeline and are used * for GPU/CPU synchronization.  When the fence is written, * it is expected that all buffers associated with that fence * are no longer in use by the associated ring on the GPU and * that the the relevant GPU caches have been flushed. */ /** * amdgpu_fence_write - write a fence value * * @ring: ring the fence is associated with * @seq: sequence number to write * * Writes a fence value to memory (all asics). */static void amdgpu_fence_write(struct amdgpu_ring *ring, u32 seq){	struct amdgpu_fence_driver *drv = &ring->fence_drv; 	if (drv->cpu_addr)		*drv->cpu_addr = cpu_to_le32(seq);} /** * amdgpu_fence_read - read a fence value * * @ring: ring the fence is associated with * * Reads a fence value from memory (all asics). * Returns the value of the fence read from memory. */static u32 amdgpu_fence_read(struct amdgpu_ring *ring){	struct amdgpu_fence_driver *drv = &ring->fence_drv;	u32 seq = 0; 	if (drv->cpu_addr)		seq = le32_to_cpu(*drv->cpu_addr);	else		seq = lower_32_bits(atomic64_read(&drv->last_seq)); 	return seq;} /** * amdgpu_fence_schedule_check - schedule lockup check * * @ring: pointer to struct amdgpu_ring * * Queues a delayed work item to check for lockups. */static void amdgpu_fence_schedule_check(struct amdgpu_ring *ring){	/*	 * Do not reset the timer here with mod_delayed_work,	 * this can livelock in an interaction with TTM delayed destroy.	 */	queue_delayed_work(system_power_efficient_wq,		&ring->fence_drv.lockup_work,		AMDGPU_FENCE_JIFFIES_TIMEOUT);} /** * amdgpu_fence_emit - emit a fence on the requested ring * * @ring: ring the fence is associated with * @owner: creator of the fence * @fence: amdgpu fence object * * Emits a fence command on the requested ring (all asics). * Returns 0 on success, -ENOMEM on failure. */int amdgpu_fence_emit(struct amdgpu_ring *ring, void *owner,		      struct amdgpu_fence **fence){	struct amdgpu_device *adev = ring->adev; 	/* we are protected by the ring emission mutex */	*fence = kmalloc(sizeof(struct amdgpu_fence), GFP_KERNEL);	if ((*fence) == NULL) {		return -ENOMEM;	}	(*fence)->seq = ++ring->fence_drv.sync_seq[ring->idx];	(*fence)->ring = ring;	(*fence)->owner = owner;	fence_init(&(*fence)->base, &amdgpu_fence_ops,		&adev->fence_queue.lock, adev->fence_context + ring->idx,		(*fence)->seq);	amdgpu_ring_emit_fence(ring, ring->fence_drv.gpu_addr,			       (*fence)->seq,			       AMDGPU_FENCE_FLAG_INT);	trace_amdgpu_fence_emit(ring->adev->ddev, ring->idx, (*fence)->seq);	return 0;} /** * amdgpu_fence_recreate - recreate a fence from an user fence * * @ring: ring the fence is associated with * @owner: creator of the fence * @seq: user fence sequence number * @fence: resulting amdgpu fence object * * Recreates a fence command from the user fence sequence number (all asics). * Returns 0 on success, -ENOMEM on failure. */int amdgpu_fence_recreate(struct amdgpu_ring *ring, void *owner,			  uint64_t seq, struct amdgpu_fence **fence){	struct amdgpu_device *adev = ring->adev; 	if (seq > ring->fence_drv.sync_seq[ring->idx])		return -EINVAL; 	*fence = kmalloc(sizeof(struct amdgpu_fence), GFP_KERNEL);	if ((*fence) == NULL)		return -ENOMEM; 	(*fence)->seq = seq;	(*fence)->ring = ring;	(*fence)->owner = owner;	fence_init(&(*fence)->base, &amdgpu_fence_ops,		&adev->fence_queue.lock, adev->fence_context + ring->idx,		(*fence)->seq);	return 0;} /** * amdgpu_fence_check_signaled - callback from fence_queue * * this function is called with fence_queue lock held, which is also used * for the fence locking itself, so unlocked variants are used for * fence_signal, and remove_wait_queue. */static int amdgpu_fence_check_signaled(wait_queue_t *wait, unsigned mode, int flags, void *key){	struct amdgpu_fence *fence;	struct amdgpu_device *adev;	u64 seq;	int ret; 	fence = container_of(wait, struct amdgpu_fence, fence_wake);	adev = fence->ring->adev; 	/*	 * We cannot use amdgpu_fence_process here because we're already	 * in the waitqueue, in a call from wake_up_all.	 */	seq = atomic64_read(&fence->ring->fence_drv.last_seq);	if (seq >= fence->seq) {		ret = fence_signal_locked(&fence->base);		if (!ret)			FENCE_TRACE(&fence->base, "signaled from irq context\n");		else			FENCE_TRACE(&fence->base, "was already signaled\n"); 		amdgpu_irq_put(adev, fence->ring->fence_drv.irq_src,				fence->ring->fence_drv.irq_type);		__remove_wait_queue(&adev->fence_queue, &fence->fence_wake);		fence_put(&fence->base);	} else		FENCE_TRACE(&fence->base, "pending\n");	return 0;} /** * amdgpu_fence_activity - check for fence activity * * @ring: pointer to struct amdgpu_ring * * Checks the current fence value and calculates the last * signalled fence value. Returns true if activity occured * on the ring, and the fence_queue should be waken up. */static bool amdgpu_fence_activity(struct amdgpu_ring *ring){	uint64_t seq, last_seq, last_emitted;	unsigned count_loop = 0;	bool wake = false; 	/* Note there is a scenario here for an infinite loop but it's	 * very unlikely to happen. For it to happen, the current polling	 * process need to be interrupted by another process and another	 * process needs to update the last_seq btw the atomic read and	 * xchg of the current process.	 *	 * More over for this to go in infinite loop there need to be	 * continuously new fence signaled ie amdgpu_fence_read needs	 * to return a different value each time for both the currently	 * polling process and the other process that xchg the last_seq	 * btw atomic read and xchg of the current process. And the	 * value the other process set as last seq must be higher than	 * the seq value we just read. Which means that current process	 * need to be interrupted after amdgpu_fence_read and before	 * atomic xchg.	 *	 * To be even more safe we count the number of time we loop and	 * we bail after 10 loop just accepting the fact that we might	 * have temporarly set the last_seq not to the true real last	 * seq but to an older one.	 */	last_seq = atomic64_read(&ring->fence_drv.last_seq);	do {		last_emitted = ring->fence_drv.sync_seq[ring->idx];		seq = amdgpu_fence_read(ring);		seq |= last_seq & 0xffffffff00000000LL;		if (seq < last_seq) {			seq &= 0xffffffff;			seq |= last_emitted & 0xffffffff00000000LL;		} 		if (seq <= last_seq || seq > last_emitted) {			break;		}		/* If we loop over we don't want to return without		 * checking if a fence is signaled as it means that the		 * seq we just read is different from the previous on.		 */		wake = true;		last_seq = seq;		if ((count_loop++) > 10) {			/* We looped over too many time leave with the			 * fact that we might have set an older fence			 * seq then the current real last seq as signaled			 * by the hw.			 */			break;		}	} while (atomic64_xchg(&ring->fence_drv.last_seq, seq) > seq); 	if (seq < last_emitted)		amdgpu_fence_schedule_check(ring); 	return wake;} /** * amdgpu_fence_check_lockup - check for hardware lockup * * @work: delayed work item * * Checks for fence activity and if there is none probe * the hardware if a lockup occured. */static void amdgpu_fence_check_lockup(struct work_struct *work){	struct amdgpu_fence_driver *fence_drv;	struct amdgpu_ring *ring; 	fence_drv = container_of(work, struct amdgpu_fence_driver,				lockup_work.work);	ring = fence_drv->ring; 	if (!down_read_trylock(&ring->adev->exclusive_lock)) {		/* just reschedule the check if a reset is going on */		amdgpu_fence_schedule_check(ring);		return;	} 	if (fence_drv->delayed_irq && ring->adev->ddev->irq_enabled) {		fence_drv->delayed_irq = false;		amdgpu_irq_update(ring->adev, fence_drv->irq_src,				fence_drv->irq_type);	} 	if (amdgpu_fence_activity(ring))		wake_up_all(&ring->adev->fence_queue);	else if (amdgpu_ring_is_lockup(ring)) {		/* good news we believe it's a lockup */		dev_warn(ring->adev->dev, "GPU lockup (current fence id "			"0x%016llx last fence id 0x%016llx on ring %d)\n",			(uint64_t)atomic64_read(&fence_drv->last_seq),			fence_drv->sync_seq[ring->idx], ring->idx); 		/* remember that we need an reset */		ring->adev->needs_reset = true;		wake_up_all(&ring->adev->fence_queue);	}	up_read(&ring->adev->exclusive_lock);} /** * amdgpu_fence_process - process a fence * * @adev: amdgpu_device pointer * @ring: ring index the fence is associated with * * Checks the current fence value and wakes the fence queue * if the sequence number has increased (all asics). */void amdgpu_fence_process(struct amdgpu_ring *ring){	uint64_t seq, last_seq, last_emitted;	unsigned count_loop = 0;	bool wake = false; 	/* Note there is a scenario here for an infinite loop but it's	 * very unlikely to happen. For it to happen, the current polling	 * process need to be interrupted by another process and another	 * process needs to update the last_seq btw the atomic read and	 * xchg of the current process.	 *	 * More over for this to go in infinite loop there need to be	 * continuously new fence signaled ie amdgpu_fence_read needs	 * to return a different value each time for both the currently	 * polling process and the other process that xchg the last_seq	 * btw atomic read and xchg of the current process. And the	 * value the other process set as last seq must be higher than	 * the seq value we just read. Which means that current process	 * need to be interrupted after amdgpu_fence_read and before	 * atomic xchg.	 *	 * To be even more safe we count the number of time we loop and	 * we bail after 10 loop just accepting the fact that we might	 * have temporarly set the last_seq not to the true real last	 * seq but to an older one.	 */	last_seq = atomic64_read(&ring->fence_drv.last_seq);	do {		last_emitted = ring->fence_drv.sync_seq[ring->idx];		seq = amdgpu_fence_read(ring);		seq |= last_seq & 0xffffffff00000000LL;		if (seq < last_seq) {			seq &= 0xffffffff;			seq |= last_emitted & 0xffffffff00000000LL;		} 		if (seq <= last_seq || seq > last_emitted) {			break;		}		/* If we loop over we don't want to return without		 * checking if a fence is signaled as it means that the		 * seq we just read is different from the previous on.		 */		wake = true;		last_seq = seq;		if ((count_loop++) > 10) {			/* We looped over too many time leave with the			 * fact that we might have set an older fence			 * seq then the current real last seq as signaled			 * by the hw.			 */			break;		}	} while (atomic64_xchg(&ring->fence_drv.last_seq, seq) > seq); 	if (wake)		wake_up_all(&ring->adev->fence_queue);} /** * amdgpu_fence_seq_signaled - check if a fence sequence number has signaled * * @ring: ring the fence is associated with * @seq: sequence number * * Check if the last signaled fence sequnce number is >= the requested * sequence number (all asics). * Returns true if the fence has signaled (current fence value * is >= requested value) or false if it has not (current fence * value is < the requested value.  Helper function for * amdgpu_fence_signaled(). */static bool amdgpu_fence_seq_signaled(struct amdgpu_ring *ring, u64 seq){	if (atomic64_read(&ring->fence_drv.last_seq) >= seq)		return true; 	/* poll new last sequence at least once */	amdgpu_fence_process(ring);	if (atomic64_read(&ring->fence_drv.last_seq) >= seq)		return true; 	return false;} static bool amdgpu_fence_is_signaled(struct fence *f){	struct amdgpu_fence *fence = to_amdgpu_fence(f);	struct amdgpu_ring *ring = fence->ring;	struct amdgpu_device *adev = ring->adev; 	if (atomic64_read(&ring->fence_drv.last_seq) >= fence->seq)		return true; 	if (down_read_trylock(&adev->exclusive_lock)) {		amdgpu_fence_process(ring);		up_read(&adev->exclusive_lock); 		if (atomic64_read(&ring->fence_drv.last_seq) >= fence->seq)			return true;	}	return false;} /** * amdgpu_fence_enable_signaling - enable signalling on fence * @fence: fence * * This function is called with fence_queue lock held, and adds a callback * to fence_queue that checks if this fence is signaled, and if so it * signals the fence and removes itself. */static bool amdgpu_fence_enable_signaling(struct fence *f){	struct amdgpu_fence *fence = to_amdgpu_fence(f);	struct amdgpu_ring *ring = fence->ring;	struct amdgpu_device *adev = ring->adev; 	if (atomic64_read(&ring->fence_drv.last_seq) >= fence->seq)		return false; 	if (down_read_trylock(&adev->exclusive_lock)) {		amdgpu_irq_get(adev, ring->fence_drv.irq_src,			ring->fence_drv.irq_type);		if (amdgpu_fence_activity(ring))			wake_up_all_locked(&adev->fence_queue); 		/* did fence get signaled after we enabled the sw irq? */		if (atomic64_read(&ring->fence_drv.last_seq) >= fence->seq) {			amdgpu_irq_put(adev, ring->fence_drv.irq_src,				ring->fence_drv.irq_type);			up_read(&adev->exclusive_lock);			return false;		} 		up_read(&adev->exclusive_lock);	} else {		/* we're probably in a lockup, lets not fiddle too much */		if (amdgpu_irq_get_delayed(adev, ring->fence_drv.irq_src,			ring->fence_drv.irq_type))			ring->fence_drv.delayed_irq = true;		amdgpu_fence_schedule_check(ring);	} 	fence->fence_wake.flags = 0;	fence->fence_wake.private = NULL;	fence->fence_wake.func = amdgpu_fence_check_signaled;	__add_wait_queue(&adev->fence_queue, &fence->fence_wake);	fence_get(f);	FENCE_TRACE(&fence->base, "armed on ring %i!\n", ring->idx);	return true;} /** * amdgpu_fence_signaled - check if a fence has signaled * * @fence: amdgpu fence object * * Check if the requested fence has signaled (all asics). * Returns true if the fence has signaled or false if it has not. */bool amdgpu_fence_signaled(struct amdgpu_fence *fence){	if (!fence)		return true; 	if (amdgpu_fence_seq_signaled(fence->ring, fence->seq)) {

Lines 1–500 of 1168. Load more