/* * Copyright 2022 Advanced Micro Devices, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. * */#include <linux/slab.h>#include <drm/drm_print.h> #include "amdgpu_ring_mux.h"#include "amdgpu_ring.h"#include "amdgpu.h" #define AMDGPU_MUX_RESUBMIT_JIFFIES_TIMEOUT (HZ / 2)#define AMDGPU_MAX_LAST_UNSIGNALED_THRESHOLD_US 10000 static const struct ring_info {	unsigned int hw_pio;	const char *ring_name;} sw_ring_info[] = {	{ AMDGPU_RING_PRIO_DEFAULT, "gfx_low"},	{ AMDGPU_RING_PRIO_2, "gfx_high"},}; static struct kmem_cache *amdgpu_mux_chunk_slab; static inline struct amdgpu_mux_entry *amdgpu_ring_mux_sw_entry(struct amdgpu_ring_mux *mux,								struct amdgpu_ring *ring){	return ring->entry_index < mux->ring_entry_size ?			&mux->ring_entry[ring->entry_index] : NULL;} /* copy packages on sw ring range[begin, end) */static void amdgpu_ring_mux_copy_pkt_from_sw_ring(struct amdgpu_ring_mux *mux,						  struct amdgpu_ring *ring,						  u64 s_start, u64 s_end){	u64 start, end;	struct amdgpu_ring *real_ring = mux->real_ring; 	start = s_start & ring->buf_mask;	end = s_end & ring->buf_mask; 	if (start == end) {		DRM_ERROR("no more data copied from sw ring\n");		return;	}	if (start > end) {		amdgpu_ring_alloc(real_ring, (ring->ring_size >> 2) + end - start);		amdgpu_ring_write_multiple(real_ring, (void *)&ring->ring[start],					   (ring->ring_size >> 2) - start);		amdgpu_ring_write_multiple(real_ring, (void *)&ring->ring[0], end);	} else {		amdgpu_ring_alloc(real_ring, end - start);		amdgpu_ring_write_multiple(real_ring, (void *)&ring->ring[start], end - start);	}} static void amdgpu_mux_resubmit_chunks(struct amdgpu_ring_mux *mux){	struct amdgpu_mux_entry *e = NULL;	struct amdgpu_mux_chunk *chunk;	uint32_t seq, last_seq;	int i; 	/*find low priority entries:*/	if (!mux->s_resubmit)		return; 	for (i = 0; i < mux->num_ring_entries; i++) {		if (mux->ring_entry[i].ring->hw_prio <= AMDGPU_RING_PRIO_DEFAULT) {			e = &mux->ring_entry[i];			break;		}	} 	if (!e) {		DRM_ERROR("%s no low priority ring found\n", __func__);		return;	} 	last_seq = atomic_read(&e->ring->fence_drv.last_seq);	seq = mux->seqno_to_resubmit;	if (last_seq < seq) {		/*resubmit all the fences between (last_seq, seq]*/		list_for_each_entry(chunk, &e->list, entry) {			if (chunk->sync_seq > last_seq && chunk->sync_seq <= seq) {				amdgpu_fence_update_start_timestamp(e->ring,								    chunk->sync_seq,								    ktime_get());				if (chunk->sync_seq ==					le32_to_cpu(*(e->ring->fence_drv.cpu_addr + 2))) {					if (chunk->cntl_offset <= e->ring->buf_mask)						amdgpu_ring_patch_cntl(e->ring,								       chunk->cntl_offset);					if (chunk->ce_offset <= e->ring->buf_mask)						amdgpu_ring_patch_ce(e->ring, chunk->ce_offset);					if (chunk->de_offset <= e->ring->buf_mask)						amdgpu_ring_patch_de(e->ring, chunk->de_offset);				}				amdgpu_ring_mux_copy_pkt_from_sw_ring(mux, e->ring,								      chunk->start,								      chunk->end);				mux->wptr_resubmit = chunk->end;				amdgpu_ring_commit(mux->real_ring);			}		}	} 	timer_delete(&mux->resubmit_timer);	mux->s_resubmit = false;} static void amdgpu_ring_mux_schedule_resubmit(struct amdgpu_ring_mux *mux){	mod_timer(&mux->resubmit_timer, jiffies + AMDGPU_MUX_RESUBMIT_JIFFIES_TIMEOUT);} static void amdgpu_mux_resubmit_fallback(struct timer_list *t){	struct amdgpu_ring_mux *mux = timer_container_of(mux, t,							 resubmit_timer); 	if (!spin_trylock(&mux->lock)) {		amdgpu_ring_mux_schedule_resubmit(mux);		DRM_ERROR("reschedule resubmit\n");		return;	}	amdgpu_mux_resubmit_chunks(mux);	spin_unlock(&mux->lock);} int amdgpu_ring_mux_init(struct amdgpu_ring_mux *mux, struct amdgpu_ring *ring,			 unsigned int entry_size){	mux->real_ring = ring;	mux->num_ring_entries = 0; 	mux->ring_entry = kzalloc_objs(struct amdgpu_mux_entry, entry_size);	if (!mux->ring_entry)		return -ENOMEM; 	mux->ring_entry_size = entry_size;	mux->s_resubmit = false; 	amdgpu_mux_chunk_slab = KMEM_CACHE(amdgpu_mux_chunk, SLAB_HWCACHE_ALIGN);	if (!amdgpu_mux_chunk_slab) {		DRM_ERROR("create amdgpu_mux_chunk cache failed\n");		return -ENOMEM;	} 	spin_lock_init(&mux->lock);	timer_setup(&mux->resubmit_timer, amdgpu_mux_resubmit_fallback, 0); 	return 0;} void amdgpu_ring_mux_fini(struct amdgpu_ring_mux *mux){	struct amdgpu_mux_entry *e;	struct amdgpu_mux_chunk *chunk, *chunk2;	int i; 	for (i = 0; i < mux->num_ring_entries; i++) {		e = &mux->ring_entry[i];		list_for_each_entry_safe(chunk, chunk2, &e->list, entry) {			list_del(&chunk->entry);			kmem_cache_free(amdgpu_mux_chunk_slab, chunk);		}	}	kmem_cache_destroy(amdgpu_mux_chunk_slab);	kfree(mux->ring_entry);	mux->ring_entry = NULL;	mux->num_ring_entries = 0;	mux->ring_entry_size = 0;} int amdgpu_ring_mux_add_sw_ring(struct amdgpu_ring_mux *mux, struct amdgpu_ring *ring){	struct amdgpu_mux_entry *e; 	if (mux->num_ring_entries >= mux->ring_entry_size) {		DRM_ERROR("add sw ring exceeding max entry size\n");		return -ENOENT;	} 	e = &mux->ring_entry[mux->num_ring_entries];	ring->entry_index = mux->num_ring_entries;	e->ring = ring; 	INIT_LIST_HEAD(&e->list);	mux->num_ring_entries += 1;	return 0;} void amdgpu_ring_mux_set_wptr(struct amdgpu_ring_mux *mux, struct amdgpu_ring *ring, u64 wptr){	struct amdgpu_mux_entry *e; 	spin_lock(&mux->lock); 	if (ring->hw_prio <= AMDGPU_RING_PRIO_DEFAULT)		amdgpu_mux_resubmit_chunks(mux); 	e = amdgpu_ring_mux_sw_entry(mux, ring);	if (!e) {		DRM_ERROR("cannot find entry for sw ring\n");		spin_unlock(&mux->lock);		return;	} 	/* We could skip this set wptr as preemption in process. */	if (ring->hw_prio <= AMDGPU_RING_PRIO_DEFAULT && mux->pending_trailing_fence_signaled) {		spin_unlock(&mux->lock);		return;	} 	e->sw_cptr = e->sw_wptr;	/* Update cptr if the package already copied in resubmit functions */	if (ring->hw_prio <= AMDGPU_RING_PRIO_DEFAULT && e->sw_cptr < mux->wptr_resubmit)		e->sw_cptr = mux->wptr_resubmit;	e->sw_wptr = wptr;	e->start_ptr_in_hw_ring = mux->real_ring->wptr; 	/* Skip copying for the packages already resubmitted.*/	if (ring->hw_prio > AMDGPU_RING_PRIO_DEFAULT || mux->wptr_resubmit < wptr) {		amdgpu_ring_mux_copy_pkt_from_sw_ring(mux, ring, e->sw_cptr, wptr);		e->end_ptr_in_hw_ring = mux->real_ring->wptr;		amdgpu_ring_commit(mux->real_ring);	} else {		e->end_ptr_in_hw_ring = mux->real_ring->wptr;	}	spin_unlock(&mux->lock);} u64 amdgpu_ring_mux_get_wptr(struct amdgpu_ring_mux *mux, struct amdgpu_ring *ring){	struct amdgpu_mux_entry *e; 	e = amdgpu_ring_mux_sw_entry(mux, ring);	if (!e) {		DRM_ERROR("cannot find entry for sw ring\n");		return 0;	} 	return e->sw_wptr;} /** * amdgpu_ring_mux_get_rptr - get the readptr of the software ring * @mux: the multiplexer the software rings attach to * @ring: the software ring of which we calculate the readptr * * The return value of the readptr is not precise while the other rings could * write data onto the real ring buffer.After overwriting on the real ring, we * can not decide if our packages have been excuted or not read yet. However, * this function is only called by the tools such as umr to collect the latest * packages for the hang analysis. We assume the hang happens near our latest * submit. Thus we could use the following logic to give the clue: * If the readptr is between start and end, then we return the copy pointer * plus the distance from start to readptr. If the readptr is before start, we * return the copy pointer. Lastly, if the readptr is past end, we return the * write pointer. */u64 amdgpu_ring_mux_get_rptr(struct amdgpu_ring_mux *mux, struct amdgpu_ring *ring){	struct amdgpu_mux_entry *e;	u64 readp, offset, start, end; 	e = amdgpu_ring_mux_sw_entry(mux, ring);	if (!e) {		DRM_ERROR("no sw entry found!\n");		return 0;	} 	readp = amdgpu_ring_get_rptr(mux->real_ring); 	start = e->start_ptr_in_hw_ring & mux->real_ring->buf_mask;	end = e->end_ptr_in_hw_ring & mux->real_ring->buf_mask;	if (start > end) {		if (readp <= end)			readp += mux->real_ring->ring_size >> 2;		end += mux->real_ring->ring_size >> 2;	} 	if (start <= readp && readp <= end) {		offset = readp - start;		e->sw_rptr = (e->sw_cptr + offset) & ring->buf_mask;	} else if (readp < start) {		e->sw_rptr = e->sw_cptr;	} else {		/* end < readptr */		e->sw_rptr = e->sw_wptr;	} 	return e->sw_rptr;} u64 amdgpu_sw_ring_get_rptr_gfx(struct amdgpu_ring *ring){	struct amdgpu_device *adev = ring->adev;	struct amdgpu_ring_mux *mux = &adev->gfx.muxer; 	WARN_ON(!ring->is_sw_ring);	return amdgpu_ring_mux_get_rptr(mux, ring);} u64 amdgpu_sw_ring_get_wptr_gfx(struct amdgpu_ring *ring){	struct amdgpu_device *adev = ring->adev;	struct amdgpu_ring_mux *mux = &adev->gfx.muxer; 	WARN_ON(!ring->is_sw_ring);	return amdgpu_ring_mux_get_wptr(mux, ring);} void amdgpu_sw_ring_set_wptr_gfx(struct amdgpu_ring *ring){	struct amdgpu_device *adev = ring->adev;	struct amdgpu_ring_mux *mux = &adev->gfx.muxer; 	WARN_ON(!ring->is_sw_ring);	amdgpu_ring_mux_set_wptr(mux, ring, ring->wptr);} /* Override insert_nop to prevent emitting nops to the software rings */void amdgpu_sw_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count){	WARN_ON(!ring->is_sw_ring);} const char *amdgpu_sw_ring_name(int idx){	return idx < ARRAY_SIZE(sw_ring_info) ?		sw_ring_info[idx].ring_name : NULL;} unsigned int amdgpu_sw_ring_priority(int idx){	return idx < ARRAY_SIZE(sw_ring_info) ?		sw_ring_info[idx].hw_pio : AMDGPU_RING_PRIO_DEFAULT;} /*Scan on low prio rings to have unsignaled fence and high ring has no fence.*/static int amdgpu_mcbp_scan(struct amdgpu_ring_mux *mux){	struct amdgpu_ring *ring;	int i, need_preempt; 	need_preempt = 0;	for (i = 0; i < mux->num_ring_entries; i++) {		ring = mux->ring_entry[i].ring;		if (ring->hw_prio > AMDGPU_RING_PRIO_DEFAULT &&		    amdgpu_fence_count_emitted(ring) > 0)			return 0;		if (ring->hw_prio <= AMDGPU_RING_PRIO_DEFAULT &&		    amdgpu_fence_last_unsignaled_time_us(ring) >		    AMDGPU_MAX_LAST_UNSIGNALED_THRESHOLD_US)			need_preempt = 1;	}	return need_preempt && !mux->s_resubmit;} /* Trigger Mid-Command Buffer Preemption (MCBP) and find if we need to resubmit. */static int amdgpu_mcbp_trigger_preempt(struct amdgpu_ring_mux *mux){	int r; 	spin_lock(&mux->lock);	mux->pending_trailing_fence_signaled = true;	r = amdgpu_ring_preempt_ib(mux->real_ring);	spin_unlock(&mux->lock);	return r;} void amdgpu_sw_ring_ib_begin(struct amdgpu_ring *ring){	struct amdgpu_device *adev = ring->adev;	struct amdgpu_ring_mux *mux = &adev->gfx.muxer; 	WARN_ON(!ring->is_sw_ring);	if (adev->gfx.mcbp && ring->hw_prio > AMDGPU_RING_PRIO_DEFAULT) {		if (amdgpu_mcbp_scan(mux) > 0)			amdgpu_mcbp_trigger_preempt(mux);		return;	} 	amdgpu_ring_mux_start_ib(mux, ring);} void amdgpu_sw_ring_ib_end(struct amdgpu_ring *ring){	struct amdgpu_device *adev = ring->adev;	struct amdgpu_ring_mux *mux = &adev->gfx.muxer; 	WARN_ON(!ring->is_sw_ring);	if (adev->gfx.mcbp && ring->hw_prio > AMDGPU_RING_PRIO_DEFAULT)		return;	amdgpu_ring_mux_end_ib(mux, ring);} void amdgpu_sw_ring_ib_mark_offset(struct amdgpu_ring *ring, enum amdgpu_ring_mux_offset_type type){	struct amdgpu_device *adev = ring->adev;	struct amdgpu_ring_mux *mux = &adev->gfx.muxer;	unsigned offset; 	if (ring->hw_prio > AMDGPU_RING_PRIO_DEFAULT)		return; 	offset = ring->wptr & ring->buf_mask; 	amdgpu_ring_mux_ib_mark_offset(mux, ring, offset, type);} void amdgpu_ring_mux_start_ib(struct amdgpu_ring_mux *mux, struct amdgpu_ring *ring){	struct amdgpu_mux_entry *e;	struct amdgpu_mux_chunk *chunk; 	spin_lock(&mux->lock);	amdgpu_mux_resubmit_chunks(mux);	spin_unlock(&mux->lock); 	e = amdgpu_ring_mux_sw_entry(mux, ring);	if (!e) {		DRM_ERROR("cannot find entry!\n");		return;	} 	chunk = kmem_cache_alloc(amdgpu_mux_chunk_slab, GFP_KERNEL);	if (!chunk) {		DRM_ERROR("alloc amdgpu_mux_chunk_slab failed\n");		return;	} 	chunk->start = ring->wptr;	/* the initialized value used to check if they are set by the ib submission*/	chunk->cntl_offset = ring->buf_mask + 1;	chunk->de_offset = ring->buf_mask + 1;	chunk->ce_offset = ring->buf_mask + 1;	list_add_tail(&chunk->entry, &e->list);} static void scan_and_remove_signaled_chunk(struct amdgpu_ring_mux *mux, struct amdgpu_ring *ring){	uint32_t last_seq = 0;	struct amdgpu_mux_entry *e;	struct amdgpu_mux_chunk *chunk, *tmp; 	e = amdgpu_ring_mux_sw_entry(mux, ring);	if (!e) {		DRM_ERROR("cannot find entry!\n");		return;	} 	last_seq = atomic_read(&ring->fence_drv.last_seq); 	list_for_each_entry_safe(chunk, tmp, &e->list, entry) {		if (chunk->sync_seq <= last_seq) {			list_del(&chunk->entry);			kmem_cache_free(amdgpu_mux_chunk_slab, chunk);		}	}} void amdgpu_ring_mux_ib_mark_offset(struct amdgpu_ring_mux *mux,				    struct amdgpu_ring *ring, u64 offset,				    enum amdgpu_ring_mux_offset_type type){	struct amdgpu_mux_entry *e;	struct amdgpu_mux_chunk *chunk; 	e = amdgpu_ring_mux_sw_entry(mux, ring);	if (!e) {		DRM_ERROR("cannot find entry!\n");		return;	} 	chunk = list_last_entry(&e->list, struct amdgpu_mux_chunk, entry);	if (!chunk) {		DRM_ERROR("cannot find chunk!\n");		return;

Lines 1–500 of 576. Load more