/* * Copyright © 2008-2010 Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice (including the next * paragraph) shall be included in all copies or substantial portions of the * Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS * IN THE SOFTWARE. * * Authors: *    Eric Anholt <eric@anholt.net> *    Zou Nan hai <nanhai.zou@intel.com> *    Xiang Hai hao<haihao.xiang@intel.com> * */ #include <drm/drmP.h>#include "i915_drv.h"#include <drm/i915_drm.h>#include "i915_trace.h"#include "intel_drv.h" /* * 965+ support PIPE_CONTROL commands, which provide finer grained control * over cache flushing. */struct pipe_control {	struct drm_i915_gem_object *obj;	volatile u32 *cpu_page;	u32 gtt_offset;}; static inline int ring_space(struct intel_ring_buffer *ring){	int space = (ring->head & HEAD_ADDR) - (ring->tail + 8);	if (space < 0)		space += ring->size;	return space;} static intgen2_render_ring_flush(struct intel_ring_buffer *ring,		       u32	invalidate_domains,		       u32	flush_domains){	u32 cmd;	int ret; 	cmd = MI_FLUSH;	if (((invalidate_domains|flush_domains) & I915_GEM_DOMAIN_RENDER) == 0)		cmd |= MI_NO_WRITE_FLUSH; 	if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER)		cmd |= MI_READ_FLUSH; 	ret = intel_ring_begin(ring, 2);	if (ret)		return ret; 	intel_ring_emit(ring, cmd);	intel_ring_emit(ring, MI_NOOP);	intel_ring_advance(ring); 	return 0;} static intgen4_render_ring_flush(struct intel_ring_buffer *ring,		       u32	invalidate_domains,		       u32	flush_domains){	struct drm_device *dev = ring->dev;	u32 cmd;	int ret; 	/*	 * read/write caches:	 *	 * I915_GEM_DOMAIN_RENDER is always invalidated, but is	 * only flushed if MI_NO_WRITE_FLUSH is unset.  On 965, it is	 * also flushed at 2d versus 3d pipeline switches.	 *	 * read-only caches:	 *	 * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if	 * MI_READ_FLUSH is set, and is always flushed on 965.	 *	 * I915_GEM_DOMAIN_COMMAND may not exist?	 *	 * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is	 * invalidated when MI_EXE_FLUSH is set.	 *	 * I915_GEM_DOMAIN_VERTEX, which exists on 965, is	 * invalidated with every MI_FLUSH.	 *	 * TLBs:	 *	 * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND	 * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and	 * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER	 * are flushed at any MI_FLUSH.	 */ 	cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;	if ((invalidate_domains|flush_domains) & I915_GEM_DOMAIN_RENDER)		cmd &= ~MI_NO_WRITE_FLUSH;	if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)		cmd |= MI_EXE_FLUSH; 	if (invalidate_domains & I915_GEM_DOMAIN_COMMAND &&	    (IS_G4X(dev) || IS_GEN5(dev)))		cmd |= MI_INVALIDATE_ISP; 	ret = intel_ring_begin(ring, 2);	if (ret)		return ret; 	intel_ring_emit(ring, cmd);	intel_ring_emit(ring, MI_NOOP);	intel_ring_advance(ring); 	return 0;} /** * Emits a PIPE_CONTROL with a non-zero post-sync operation, for * implementing two workarounds on gen6.  From section 1.4.7.1 * "PIPE_CONTROL" of the Sandy Bridge PRM volume 2 part 1: * * [DevSNB-C+{W/A}] Before any depth stall flush (including those * produced by non-pipelined state commands), software needs to first * send a PIPE_CONTROL with no bits set except Post-Sync Operation != * 0. * * [Dev-SNB{W/A}]: Before a PIPE_CONTROL with Write Cache Flush Enable * =1, a PIPE_CONTROL with any non-zero post-sync-op is required. * * And the workaround for these two requires this workaround first: * * [Dev-SNB{W/A}]: Pipe-control with CS-stall bit set must be sent * BEFORE the pipe-control with a post-sync op and no write-cache * flushes. * * And this last workaround is tricky because of the requirements on * that bit.  From section 1.4.7.2.3 "Stall" of the Sandy Bridge PRM * volume 2 part 1: * *     "1 of the following must also be set: *      - Render Target Cache Flush Enable ([12] of DW1) *      - Depth Cache Flush Enable ([0] of DW1) *      - Stall at Pixel Scoreboard ([1] of DW1) *      - Depth Stall ([13] of DW1) *      - Post-Sync Operation ([13] of DW1) *      - Notify Enable ([8] of DW1)" * * The cache flushes require the workaround flush that triggered this * one, so we can't use it.  Depth stall would trigger the same. * Post-sync nonzero is what triggered this second workaround, so we * can't use that one either.  Notify enable is IRQs, which aren't * really our business.  That leaves only stall at scoreboard. */static intintel_emit_post_sync_nonzero_flush(struct intel_ring_buffer *ring){	struct pipe_control *pc = ring->private;	u32 scratch_addr = pc->gtt_offset + 128;	int ret;  	ret = intel_ring_begin(ring, 6);	if (ret)		return ret; 	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5));	intel_ring_emit(ring, PIPE_CONTROL_CS_STALL |			PIPE_CONTROL_STALL_AT_SCOREBOARD);	intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT); /* address */	intel_ring_emit(ring, 0); /* low dword */	intel_ring_emit(ring, 0); /* high dword */	intel_ring_emit(ring, MI_NOOP);	intel_ring_advance(ring); 	ret = intel_ring_begin(ring, 6);	if (ret)		return ret; 	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5));	intel_ring_emit(ring, PIPE_CONTROL_QW_WRITE);	intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT); /* address */	intel_ring_emit(ring, 0);	intel_ring_emit(ring, 0);	intel_ring_emit(ring, MI_NOOP);	intel_ring_advance(ring); 	return 0;} static intgen6_render_ring_flush(struct intel_ring_buffer *ring,                         u32 invalidate_domains, u32 flush_domains){	u32 flags = 0;	struct pipe_control *pc = ring->private;	u32 scratch_addr = pc->gtt_offset + 128;	int ret; 	/* Force SNB workarounds for PIPE_CONTROL flushes */	ret = intel_emit_post_sync_nonzero_flush(ring);	if (ret)		return ret; 	/* Just flush everything.  Experiments have shown that reducing the	 * number of bits based on the write domains has little performance	 * impact.	 */	if (flush_domains) {		flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;		flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;		/*		 * Ensure that any following seqno writes only happen		 * when the render cache is indeed flushed.		 */		flags |= PIPE_CONTROL_CS_STALL;	}	if (invalidate_domains) {		flags |= PIPE_CONTROL_TLB_INVALIDATE;		flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;		flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;		flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;		flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;		flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;		/*		 * TLB invalidate requires a post-sync write.		 */		flags |= PIPE_CONTROL_QW_WRITE;	} 	ret = intel_ring_begin(ring, 4);	if (ret)		return ret; 	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4));	intel_ring_emit(ring, flags);	intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT);	intel_ring_emit(ring, 0);	intel_ring_advance(ring); 	return 0;} static intgen7_render_ring_cs_stall_wa(struct intel_ring_buffer *ring){	int ret; 	ret = intel_ring_begin(ring, 4);	if (ret)		return ret; 	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4));	intel_ring_emit(ring, PIPE_CONTROL_CS_STALL |			      PIPE_CONTROL_STALL_AT_SCOREBOARD);	intel_ring_emit(ring, 0);	intel_ring_emit(ring, 0);	intel_ring_advance(ring); 	return 0;} static intgen7_render_ring_flush(struct intel_ring_buffer *ring,		       u32 invalidate_domains, u32 flush_domains){	u32 flags = 0;	struct pipe_control *pc = ring->private;	u32 scratch_addr = pc->gtt_offset + 128;	int ret; 	/*	 * Ensure that any following seqno writes only happen when the render	 * cache is indeed flushed.	 *	 * Workaround: 4th PIPE_CONTROL command (except the ones with only	 * read-cache invalidate bits set) must have the CS_STALL bit set. We	 * don't try to be clever and just set it unconditionally.	 */	flags |= PIPE_CONTROL_CS_STALL; 	/* Just flush everything.  Experiments have shown that reducing the	 * number of bits based on the write domains has little performance	 * impact.	 */	if (flush_domains) {		flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;		flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;	}	if (invalidate_domains) {		flags |= PIPE_CONTROL_TLB_INVALIDATE;		flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;		flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;		flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;		flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;		flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;		/*		 * TLB invalidate requires a post-sync write.		 */		flags |= PIPE_CONTROL_QW_WRITE; 		/* Workaround: we must issue a pipe_control with CS-stall bit		 * set before a pipe_control command that has the state cache		 * invalidate bit set. */		gen7_render_ring_cs_stall_wa(ring);	} 	ret = intel_ring_begin(ring, 4);	if (ret)		return ret; 	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4));	intel_ring_emit(ring, flags);	intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT);	intel_ring_emit(ring, 0);	intel_ring_advance(ring); 	return 0;} static void ring_write_tail(struct intel_ring_buffer *ring,			    u32 value){	drm_i915_private_t *dev_priv = ring->dev->dev_private;	I915_WRITE_TAIL(ring, value);} u32 intel_ring_get_active_head(struct intel_ring_buffer *ring){	drm_i915_private_t *dev_priv = ring->dev->dev_private;	u32 acthd_reg = INTEL_INFO(ring->dev)->gen >= 4 ?			RING_ACTHD(ring->mmio_base) : ACTHD; 	return I915_READ(acthd_reg);} static int init_ring_common(struct intel_ring_buffer *ring){	struct drm_device *dev = ring->dev;	drm_i915_private_t *dev_priv = dev->dev_private;	struct drm_i915_gem_object *obj = ring->obj;	int ret = 0;	u32 head; 	if (HAS_FORCE_WAKE(dev))		gen6_gt_force_wake_get(dev_priv); 	/* Stop the ring if it's running. */	I915_WRITE_CTL(ring, 0);	I915_WRITE_HEAD(ring, 0);	ring->write_tail(ring, 0); 	head = I915_READ_HEAD(ring) & HEAD_ADDR; 	/* G45 ring initialization fails to reset head to zero */	if (head != 0) {		DRM_DEBUG_KMS("%s head not reset to zero "			      "ctl %08x head %08x tail %08x start %08x\n",			      ring->name,			      I915_READ_CTL(ring),			      I915_READ_HEAD(ring),			      I915_READ_TAIL(ring),			      I915_READ_START(ring)); 		I915_WRITE_HEAD(ring, 0); 		if (I915_READ_HEAD(ring) & HEAD_ADDR) {			DRM_ERROR("failed to set %s head to zero "				  "ctl %08x head %08x tail %08x start %08x\n",				  ring->name,				  I915_READ_CTL(ring),				  I915_READ_HEAD(ring),				  I915_READ_TAIL(ring),				  I915_READ_START(ring));		}	} 	/* Initialize the ring. This must happen _after_ we've cleared the ring	 * registers with the above sequence (the readback of the HEAD registers	 * also enforces ordering), otherwise the hw might lose the new ring	 * register values. */	I915_WRITE_START(ring, obj->gtt_offset);	I915_WRITE_CTL(ring,			((ring->size - PAGE_SIZE) & RING_NR_PAGES)			| RING_VALID); 	/* If the head is still not zero, the ring is dead */	if (wait_for((I915_READ_CTL(ring) & RING_VALID) != 0 &&		     I915_READ_START(ring) == obj->gtt_offset &&		     (I915_READ_HEAD(ring) & HEAD_ADDR) == 0, 50)) {		DRM_ERROR("%s initialization failed "				"ctl %08x head %08x tail %08x start %08x\n",				ring->name,				I915_READ_CTL(ring),				I915_READ_HEAD(ring),				I915_READ_TAIL(ring),				I915_READ_START(ring));		ret = -EIO;		goto out;	} 	if (!drm_core_check_feature(ring->dev, DRIVER_MODESET))		i915_kernel_lost_context(ring->dev);	else {		ring->head = I915_READ_HEAD(ring);		ring->tail = I915_READ_TAIL(ring) & TAIL_ADDR;		ring->space = ring_space(ring);		ring->last_retired_head = -1;	} out:	if (HAS_FORCE_WAKE(dev))		gen6_gt_force_wake_put(dev_priv); 	return ret;} static intinit_pipe_control(struct intel_ring_buffer *ring){	struct pipe_control *pc;	struct drm_i915_gem_object *obj;	int ret; 	if (ring->private)		return 0; 	pc = kmalloc(sizeof(*pc), GFP_KERNEL);	if (!pc)		return -ENOMEM; 	obj = i915_gem_alloc_object(ring->dev, 4096);	if (obj == NULL) {		DRM_ERROR("Failed to allocate seqno page\n");		ret = -ENOMEM;		goto err;	} 	i915_gem_object_set_cache_level(obj, I915_CACHE_LLC); 	ret = i915_gem_object_pin(obj, 4096, true, false);	if (ret)		goto err_unref; 	pc->gtt_offset = obj->gtt_offset;	pc->cpu_page =  kmap(sg_page(obj->pages->sgl));	if (pc->cpu_page == NULL)		goto err_unpin; 	pc->obj = obj;	ring->private = pc;	return 0; err_unpin:	i915_gem_object_unpin(obj);err_unref:	drm_gem_object_unreference(&obj->base);err:	kfree(pc);	return ret;} static voidcleanup_pipe_control(struct intel_ring_buffer *ring){	struct pipe_control *pc = ring->private;	struct drm_i915_gem_object *obj; 	if (!ring->private)		return; 	obj = pc->obj; 	kunmap(sg_page(obj->pages->sgl));	i915_gem_object_unpin(obj);	drm_gem_object_unreference(&obj->base); 	kfree(pc);	ring->private = NULL;}

Lines 1–500 of 1689. Load more