/* * Copyright © 2008-2010 Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice (including the next * paragraph) shall be included in all copies or substantial portions of the * Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS * IN THE SOFTWARE. * * Authors: *    Eric Anholt <eric@anholt.net> *    Zou Nan hai <nanhai.zou@intel.com> *    Xiang Hai hao<haihao.xiang@intel.com> * */ #include <drm/drmP.h>#include "i915_drv.h"#include <drm/i915_drm.h>#include "i915_trace.h"#include "intel_drv.h" boolintel_ring_initialized(struct intel_engine_cs *ring){	struct drm_device *dev = ring->dev; 	if (!dev)		return false; 	if (i915.enable_execlists) {		struct intel_context *dctx = ring->default_context;		struct intel_ringbuffer *ringbuf = dctx->engine[ring->id].ringbuf; 		return ringbuf->obj;	} else		return ring->buffer && ring->buffer->obj;} int __intel_ring_space(int head, int tail, int size){	int space = head - tail;	if (space <= 0)		space += size;	return space - I915_RING_FREE_SPACE;} void intel_ring_update_space(struct intel_ringbuffer *ringbuf){	if (ringbuf->last_retired_head != -1) {		ringbuf->head = ringbuf->last_retired_head;		ringbuf->last_retired_head = -1;	} 	ringbuf->space = __intel_ring_space(ringbuf->head & HEAD_ADDR,					    ringbuf->tail, ringbuf->size);} int intel_ring_space(struct intel_ringbuffer *ringbuf){	intel_ring_update_space(ringbuf);	return ringbuf->space;} bool intel_ring_stopped(struct intel_engine_cs *ring){	struct drm_i915_private *dev_priv = ring->dev->dev_private;	return dev_priv->gpu_error.stop_rings & intel_ring_flag(ring);} void __intel_ring_advance(struct intel_engine_cs *ring){	struct intel_ringbuffer *ringbuf = ring->buffer;	ringbuf->tail &= ringbuf->size - 1;	if (intel_ring_stopped(ring))		return;	ring->write_tail(ring, ringbuf->tail);} static intgen2_render_ring_flush(struct intel_engine_cs *ring,		       u32	invalidate_domains,		       u32	flush_domains){	u32 cmd;	int ret; 	cmd = MI_FLUSH;	if (((invalidate_domains|flush_domains) & I915_GEM_DOMAIN_RENDER) == 0)		cmd |= MI_NO_WRITE_FLUSH; 	if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER)		cmd |= MI_READ_FLUSH; 	ret = intel_ring_begin(ring, 2);	if (ret)		return ret; 	intel_ring_emit(ring, cmd);	intel_ring_emit(ring, MI_NOOP);	intel_ring_advance(ring); 	return 0;} static intgen4_render_ring_flush(struct intel_engine_cs *ring,		       u32	invalidate_domains,		       u32	flush_domains){	struct drm_device *dev = ring->dev;	u32 cmd;	int ret; 	/*	 * read/write caches:	 *	 * I915_GEM_DOMAIN_RENDER is always invalidated, but is	 * only flushed if MI_NO_WRITE_FLUSH is unset.  On 965, it is	 * also flushed at 2d versus 3d pipeline switches.	 *	 * read-only caches:	 *	 * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if	 * MI_READ_FLUSH is set, and is always flushed on 965.	 *	 * I915_GEM_DOMAIN_COMMAND may not exist?	 *	 * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is	 * invalidated when MI_EXE_FLUSH is set.	 *	 * I915_GEM_DOMAIN_VERTEX, which exists on 965, is	 * invalidated with every MI_FLUSH.	 *	 * TLBs:	 *	 * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND	 * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and	 * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER	 * are flushed at any MI_FLUSH.	 */ 	cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;	if ((invalidate_domains|flush_domains) & I915_GEM_DOMAIN_RENDER)		cmd &= ~MI_NO_WRITE_FLUSH;	if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)		cmd |= MI_EXE_FLUSH; 	if (invalidate_domains & I915_GEM_DOMAIN_COMMAND &&	    (IS_G4X(dev) || IS_GEN5(dev)))		cmd |= MI_INVALIDATE_ISP; 	ret = intel_ring_begin(ring, 2);	if (ret)		return ret; 	intel_ring_emit(ring, cmd);	intel_ring_emit(ring, MI_NOOP);	intel_ring_advance(ring); 	return 0;} /** * Emits a PIPE_CONTROL with a non-zero post-sync operation, for * implementing two workarounds on gen6.  From section 1.4.7.1 * "PIPE_CONTROL" of the Sandy Bridge PRM volume 2 part 1: * * [DevSNB-C+{W/A}] Before any depth stall flush (including those * produced by non-pipelined state commands), software needs to first * send a PIPE_CONTROL with no bits set except Post-Sync Operation != * 0. * * [Dev-SNB{W/A}]: Before a PIPE_CONTROL with Write Cache Flush Enable * =1, a PIPE_CONTROL with any non-zero post-sync-op is required. * * And the workaround for these two requires this workaround first: * * [Dev-SNB{W/A}]: Pipe-control with CS-stall bit set must be sent * BEFORE the pipe-control with a post-sync op and no write-cache * flushes. * * And this last workaround is tricky because of the requirements on * that bit.  From section 1.4.7.2.3 "Stall" of the Sandy Bridge PRM * volume 2 part 1: * *     "1 of the following must also be set: *      - Render Target Cache Flush Enable ([12] of DW1) *      - Depth Cache Flush Enable ([0] of DW1) *      - Stall at Pixel Scoreboard ([1] of DW1) *      - Depth Stall ([13] of DW1) *      - Post-Sync Operation ([13] of DW1) *      - Notify Enable ([8] of DW1)" * * The cache flushes require the workaround flush that triggered this * one, so we can't use it.  Depth stall would trigger the same. * Post-sync nonzero is what triggered this second workaround, so we * can't use that one either.  Notify enable is IRQs, which aren't * really our business.  That leaves only stall at scoreboard. */static intintel_emit_post_sync_nonzero_flush(struct intel_engine_cs *ring){	u32 scratch_addr = ring->scratch.gtt_offset + 2 * CACHELINE_BYTES;	int ret;  	ret = intel_ring_begin(ring, 6);	if (ret)		return ret; 	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5));	intel_ring_emit(ring, PIPE_CONTROL_CS_STALL |			PIPE_CONTROL_STALL_AT_SCOREBOARD);	intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT); /* address */	intel_ring_emit(ring, 0); /* low dword */	intel_ring_emit(ring, 0); /* high dword */	intel_ring_emit(ring, MI_NOOP);	intel_ring_advance(ring); 	ret = intel_ring_begin(ring, 6);	if (ret)		return ret; 	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5));	intel_ring_emit(ring, PIPE_CONTROL_QW_WRITE);	intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT); /* address */	intel_ring_emit(ring, 0);	intel_ring_emit(ring, 0);	intel_ring_emit(ring, MI_NOOP);	intel_ring_advance(ring); 	return 0;} static intgen6_render_ring_flush(struct intel_engine_cs *ring,                         u32 invalidate_domains, u32 flush_domains){	u32 flags = 0;	u32 scratch_addr = ring->scratch.gtt_offset + 2 * CACHELINE_BYTES;	int ret; 	/* Force SNB workarounds for PIPE_CONTROL flushes */	ret = intel_emit_post_sync_nonzero_flush(ring);	if (ret)		return ret; 	/* Just flush everything.  Experiments have shown that reducing the	 * number of bits based on the write domains has little performance	 * impact.	 */	if (flush_domains) {		flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;		flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;		/*		 * Ensure that any following seqno writes only happen		 * when the render cache is indeed flushed.		 */		flags |= PIPE_CONTROL_CS_STALL;	}	if (invalidate_domains) {		flags |= PIPE_CONTROL_TLB_INVALIDATE;		flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;		flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;		flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;		flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;		flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;		/*		 * TLB invalidate requires a post-sync write.		 */		flags |= PIPE_CONTROL_QW_WRITE | PIPE_CONTROL_CS_STALL;	} 	ret = intel_ring_begin(ring, 4);	if (ret)		return ret; 	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4));	intel_ring_emit(ring, flags);	intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT);	intel_ring_emit(ring, 0);	intel_ring_advance(ring); 	return 0;} static intgen7_render_ring_cs_stall_wa(struct intel_engine_cs *ring){	int ret; 	ret = intel_ring_begin(ring, 4);	if (ret)		return ret; 	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4));	intel_ring_emit(ring, PIPE_CONTROL_CS_STALL |			      PIPE_CONTROL_STALL_AT_SCOREBOARD);	intel_ring_emit(ring, 0);	intel_ring_emit(ring, 0);	intel_ring_advance(ring); 	return 0;} static intgen7_render_ring_flush(struct intel_engine_cs *ring,		       u32 invalidate_domains, u32 flush_domains){	u32 flags = 0;	u32 scratch_addr = ring->scratch.gtt_offset + 2 * CACHELINE_BYTES;	int ret; 	/*	 * Ensure that any following seqno writes only happen when the render	 * cache is indeed flushed.	 *	 * Workaround: 4th PIPE_CONTROL command (except the ones with only	 * read-cache invalidate bits set) must have the CS_STALL bit set. We	 * don't try to be clever and just set it unconditionally.	 */	flags |= PIPE_CONTROL_CS_STALL; 	/* Just flush everything.  Experiments have shown that reducing the	 * number of bits based on the write domains has little performance	 * impact.	 */	if (flush_domains) {		flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;		flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;	}	if (invalidate_domains) {		flags |= PIPE_CONTROL_TLB_INVALIDATE;		flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;		flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;		flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;		flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;		flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;		flags |= PIPE_CONTROL_MEDIA_STATE_CLEAR;		/*		 * TLB invalidate requires a post-sync write.		 */		flags |= PIPE_CONTROL_QW_WRITE;		flags |= PIPE_CONTROL_GLOBAL_GTT_IVB; 		flags |= PIPE_CONTROL_STALL_AT_SCOREBOARD; 		/* Workaround: we must issue a pipe_control with CS-stall bit		 * set before a pipe_control command that has the state cache		 * invalidate bit set. */		gen7_render_ring_cs_stall_wa(ring);	} 	ret = intel_ring_begin(ring, 4);	if (ret)		return ret; 	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4));	intel_ring_emit(ring, flags);	intel_ring_emit(ring, scratch_addr);	intel_ring_emit(ring, 0);	intel_ring_advance(ring); 	return 0;} static intgen8_emit_pipe_control(struct intel_engine_cs *ring,		       u32 flags, u32 scratch_addr){	int ret; 	ret = intel_ring_begin(ring, 6);	if (ret)		return ret; 	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(6));	intel_ring_emit(ring, flags);	intel_ring_emit(ring, scratch_addr);	intel_ring_emit(ring, 0);	intel_ring_emit(ring, 0);	intel_ring_emit(ring, 0);	intel_ring_advance(ring); 	return 0;} static intgen8_render_ring_flush(struct intel_engine_cs *ring,		       u32 invalidate_domains, u32 flush_domains){	u32 flags = 0;	u32 scratch_addr = ring->scratch.gtt_offset + 2 * CACHELINE_BYTES;	int ret; 	flags |= PIPE_CONTROL_CS_STALL; 	if (flush_domains) {		flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;		flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;	}	if (invalidate_domains) {		flags |= PIPE_CONTROL_TLB_INVALIDATE;		flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;		flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;		flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;		flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;		flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;		flags |= PIPE_CONTROL_QW_WRITE;		flags |= PIPE_CONTROL_GLOBAL_GTT_IVB; 		/* WaCsStallBeforeStateCacheInvalidate:bdw,chv */		ret = gen8_emit_pipe_control(ring,					     PIPE_CONTROL_CS_STALL |					     PIPE_CONTROL_STALL_AT_SCOREBOARD,					     0);		if (ret)			return ret;	} 	return gen8_emit_pipe_control(ring, flags, scratch_addr);} static void ring_write_tail(struct intel_engine_cs *ring,			    u32 value){	struct drm_i915_private *dev_priv = ring->dev->dev_private;	I915_WRITE_TAIL(ring, value);} u64 intel_ring_get_active_head(struct intel_engine_cs *ring){	struct drm_i915_private *dev_priv = ring->dev->dev_private;	u64 acthd; 	if (INTEL_INFO(ring->dev)->gen >= 8)		acthd = I915_READ64_2x32(RING_ACTHD(ring->mmio_base),					 RING_ACTHD_UDW(ring->mmio_base));	else if (INTEL_INFO(ring->dev)->gen >= 4)		acthd = I915_READ(RING_ACTHD(ring->mmio_base));	else		acthd = I915_READ(ACTHD); 	return acthd;} static void ring_setup_phys_status_page(struct intel_engine_cs *ring){	struct drm_i915_private *dev_priv = ring->dev->dev_private;	u32 addr; 	addr = dev_priv->status_page_dmah->busaddr;	if (INTEL_INFO(ring->dev)->gen >= 4)		addr |= (dev_priv->status_page_dmah->busaddr >> 28) & 0xf0;	I915_WRITE(HWS_PGA, addr);} static void intel_ring_setup_status_page(struct intel_engine_cs *ring){	struct drm_device *dev = ring->dev;	struct drm_i915_private *dev_priv = ring->dev->dev_private;	u32 mmio = 0; 	/* The ring status page addresses are no longer next to the rest of	 * the ring registers as of gen7.	 */	if (IS_GEN7(dev)) {		switch (ring->id) {		case RCS:			mmio = RENDER_HWS_PGA_GEN7;			break;		case BCS:			mmio = BLT_HWS_PGA_GEN7;			break;		/*		 * VCS2 actually doesn't exist on Gen7. Only shut up		 * gcc switch check warning		 */		case VCS2:		case VCS:			mmio = BSD_HWS_PGA_GEN7;			break;		case VECS:			mmio = VEBOX_HWS_PGA_GEN7;			break;		}

Lines 1–500 of 2872. Load more