/* * Copyright © 2008-2010 Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice (including the next * paragraph) shall be included in all copies or substantial portions of the * Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS * IN THE SOFTWARE. * * Authors: *    Eric Anholt <eric@anholt.net> *    Zou Nan hai <nanhai.zou@intel.com> *    Xiang Hai hao<haihao.xiang@intel.com> * */ #include "drmP.h"#include "drm.h"#include "i915_drv.h"#include "i915_drm.h"#include "i915_trace.h"#include "intel_drv.h" /* * 965+ support PIPE_CONTROL commands, which provide finer grained control * over cache flushing. */struct pipe_control {	struct drm_i915_gem_object *obj;	volatile u32 *cpu_page;	u32 gtt_offset;}; static inline int ring_space(struct intel_ring_buffer *ring){	int space = (ring->head & HEAD_ADDR) - (ring->tail + 8);	if (space < 0)		space += ring->size;	return space;} static u32 i915_gem_get_seqno(struct drm_device *dev){	drm_i915_private_t *dev_priv = dev->dev_private;	u32 seqno; 	seqno = dev_priv->next_seqno; 	/* reserve 0 for non-seqno */	if (++dev_priv->next_seqno == 0)		dev_priv->next_seqno = 1; 	return seqno;} static intrender_ring_flush(struct intel_ring_buffer *ring,		  u32	invalidate_domains,		  u32	flush_domains){	struct drm_device *dev = ring->dev;	u32 cmd;	int ret; 	/*	 * read/write caches:	 *	 * I915_GEM_DOMAIN_RENDER is always invalidated, but is	 * only flushed if MI_NO_WRITE_FLUSH is unset.  On 965, it is	 * also flushed at 2d versus 3d pipeline switches.	 *	 * read-only caches:	 *	 * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if	 * MI_READ_FLUSH is set, and is always flushed on 965.	 *	 * I915_GEM_DOMAIN_COMMAND may not exist?	 *	 * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is	 * invalidated when MI_EXE_FLUSH is set.	 *	 * I915_GEM_DOMAIN_VERTEX, which exists on 965, is	 * invalidated with every MI_FLUSH.	 *	 * TLBs:	 *	 * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND	 * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and	 * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER	 * are flushed at any MI_FLUSH.	 */ 	cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;	if ((invalidate_domains|flush_domains) &	    I915_GEM_DOMAIN_RENDER)		cmd &= ~MI_NO_WRITE_FLUSH;	if (INTEL_INFO(dev)->gen < 4) {		/*		 * On the 965, the sampler cache always gets flushed		 * and this bit is reserved.		 */		if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER)			cmd |= MI_READ_FLUSH;	}	if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)		cmd |= MI_EXE_FLUSH; 	if (invalidate_domains & I915_GEM_DOMAIN_COMMAND &&	    (IS_G4X(dev) || IS_GEN5(dev)))		cmd |= MI_INVALIDATE_ISP; 	ret = intel_ring_begin(ring, 2);	if (ret)		return ret; 	intel_ring_emit(ring, cmd);	intel_ring_emit(ring, MI_NOOP);	intel_ring_advance(ring); 	return 0;} /** * Emits a PIPE_CONTROL with a non-zero post-sync operation, for * implementing two workarounds on gen6.  From section 1.4.7.1 * "PIPE_CONTROL" of the Sandy Bridge PRM volume 2 part 1: * * [DevSNB-C+{W/A}] Before any depth stall flush (including those * produced by non-pipelined state commands), software needs to first * send a PIPE_CONTROL with no bits set except Post-Sync Operation != * 0. * * [Dev-SNB{W/A}]: Before a PIPE_CONTROL with Write Cache Flush Enable * =1, a PIPE_CONTROL with any non-zero post-sync-op is required. * * And the workaround for these two requires this workaround first: * * [Dev-SNB{W/A}]: Pipe-control with CS-stall bit set must be sent * BEFORE the pipe-control with a post-sync op and no write-cache * flushes. * * And this last workaround is tricky because of the requirements on * that bit.  From section 1.4.7.2.3 "Stall" of the Sandy Bridge PRM * volume 2 part 1: * *     "1 of the following must also be set: *      - Render Target Cache Flush Enable ([12] of DW1) *      - Depth Cache Flush Enable ([0] of DW1) *      - Stall at Pixel Scoreboard ([1] of DW1) *      - Depth Stall ([13] of DW1) *      - Post-Sync Operation ([13] of DW1) *      - Notify Enable ([8] of DW1)" * * The cache flushes require the workaround flush that triggered this * one, so we can't use it.  Depth stall would trigger the same. * Post-sync nonzero is what triggered this second workaround, so we * can't use that one either.  Notify enable is IRQs, which aren't * really our business.  That leaves only stall at scoreboard. */static intintel_emit_post_sync_nonzero_flush(struct intel_ring_buffer *ring){	struct pipe_control *pc = ring->private;	u32 scratch_addr = pc->gtt_offset + 128;	int ret;  	ret = intel_ring_begin(ring, 6);	if (ret)		return ret; 	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5));	intel_ring_emit(ring, PIPE_CONTROL_CS_STALL |			PIPE_CONTROL_STALL_AT_SCOREBOARD);	intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT); /* address */	intel_ring_emit(ring, 0); /* low dword */	intel_ring_emit(ring, 0); /* high dword */	intel_ring_emit(ring, MI_NOOP);	intel_ring_advance(ring); 	ret = intel_ring_begin(ring, 6);	if (ret)		return ret; 	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5));	intel_ring_emit(ring, PIPE_CONTROL_QW_WRITE);	intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT); /* address */	intel_ring_emit(ring, 0);	intel_ring_emit(ring, 0);	intel_ring_emit(ring, MI_NOOP);	intel_ring_advance(ring); 	return 0;} static intgen6_render_ring_flush(struct intel_ring_buffer *ring,                         u32 invalidate_domains, u32 flush_domains){	u32 flags = 0;	struct pipe_control *pc = ring->private;	u32 scratch_addr = pc->gtt_offset + 128;	int ret; 	/* Force SNB workarounds for PIPE_CONTROL flushes */	intel_emit_post_sync_nonzero_flush(ring); 	/* Just flush everything.  Experiments have shown that reducing the	 * number of bits based on the write domains has little performance	 * impact.	 */	flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;	flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;	flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;	flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;	flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;	flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;	flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE; 	ret = intel_ring_begin(ring, 6);	if (ret)		return ret; 	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5));	intel_ring_emit(ring, flags);	intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT);	intel_ring_emit(ring, 0); /* lower dword */	intel_ring_emit(ring, 0); /* uppwer dword */	intel_ring_emit(ring, MI_NOOP);	intel_ring_advance(ring); 	return 0;} static void ring_write_tail(struct intel_ring_buffer *ring,			    u32 value){	drm_i915_private_t *dev_priv = ring->dev->dev_private;	I915_WRITE_TAIL(ring, value);} u32 intel_ring_get_active_head(struct intel_ring_buffer *ring){	drm_i915_private_t *dev_priv = ring->dev->dev_private;	u32 acthd_reg = INTEL_INFO(ring->dev)->gen >= 4 ?			RING_ACTHD(ring->mmio_base) : ACTHD; 	return I915_READ(acthd_reg);} static int init_ring_common(struct intel_ring_buffer *ring){	drm_i915_private_t *dev_priv = ring->dev->dev_private;	struct drm_i915_gem_object *obj = ring->obj;	u32 head; 	/* Stop the ring if it's running. */	I915_WRITE_CTL(ring, 0);	I915_WRITE_HEAD(ring, 0);	ring->write_tail(ring, 0); 	/* Initialize the ring. */	I915_WRITE_START(ring, obj->gtt_offset);	head = I915_READ_HEAD(ring) & HEAD_ADDR; 	/* G45 ring initialization fails to reset head to zero */	if (head != 0) {		DRM_DEBUG_KMS("%s head not reset to zero "			      "ctl %08x head %08x tail %08x start %08x\n",			      ring->name,			      I915_READ_CTL(ring),			      I915_READ_HEAD(ring),			      I915_READ_TAIL(ring),			      I915_READ_START(ring)); 		I915_WRITE_HEAD(ring, 0); 		if (I915_READ_HEAD(ring) & HEAD_ADDR) {			DRM_ERROR("failed to set %s head to zero "				  "ctl %08x head %08x tail %08x start %08x\n",				  ring->name,				  I915_READ_CTL(ring),				  I915_READ_HEAD(ring),				  I915_READ_TAIL(ring),				  I915_READ_START(ring));		}	} 	I915_WRITE_CTL(ring,			((ring->size - PAGE_SIZE) & RING_NR_PAGES)			| RING_REPORT_64K | RING_VALID); 	/* If the head is still not zero, the ring is dead */	if ((I915_READ_CTL(ring) & RING_VALID) == 0 ||	    I915_READ_START(ring) != obj->gtt_offset ||	    (I915_READ_HEAD(ring) & HEAD_ADDR) != 0) {		DRM_ERROR("%s initialization failed "				"ctl %08x head %08x tail %08x start %08x\n",				ring->name,				I915_READ_CTL(ring),				I915_READ_HEAD(ring),				I915_READ_TAIL(ring),				I915_READ_START(ring));		return -EIO;	} 	if (!drm_core_check_feature(ring->dev, DRIVER_MODESET))		i915_kernel_lost_context(ring->dev);	else {		ring->head = I915_READ_HEAD(ring);		ring->tail = I915_READ_TAIL(ring) & TAIL_ADDR;		ring->space = ring_space(ring);	} 	return 0;} static intinit_pipe_control(struct intel_ring_buffer *ring){	struct pipe_control *pc;	struct drm_i915_gem_object *obj;	int ret; 	if (ring->private)		return 0; 	pc = kmalloc(sizeof(*pc), GFP_KERNEL);	if (!pc)		return -ENOMEM; 	obj = i915_gem_alloc_object(ring->dev, 4096);	if (obj == NULL) {		DRM_ERROR("Failed to allocate seqno page\n");		ret = -ENOMEM;		goto err;	} 	i915_gem_object_set_cache_level(obj, I915_CACHE_LLC); 	ret = i915_gem_object_pin(obj, 4096, true);	if (ret)		goto err_unref; 	pc->gtt_offset = obj->gtt_offset;	pc->cpu_page =  kmap(obj->pages[0]);	if (pc->cpu_page == NULL)		goto err_unpin; 	pc->obj = obj;	ring->private = pc;	return 0; err_unpin:	i915_gem_object_unpin(obj);err_unref:	drm_gem_object_unreference(&obj->base);err:	kfree(pc);	return ret;} static voidcleanup_pipe_control(struct intel_ring_buffer *ring){	struct pipe_control *pc = ring->private;	struct drm_i915_gem_object *obj; 	if (!ring->private)		return; 	obj = pc->obj;	kunmap(obj->pages[0]);	i915_gem_object_unpin(obj);	drm_gem_object_unreference(&obj->base); 	kfree(pc);	ring->private = NULL;} static int init_render_ring(struct intel_ring_buffer *ring){	struct drm_device *dev = ring->dev;	struct drm_i915_private *dev_priv = dev->dev_private;	int ret = init_ring_common(ring); 	if (INTEL_INFO(dev)->gen > 3) {		int mode = VS_TIMER_DISPATCH << 16 | VS_TIMER_DISPATCH;		if (IS_GEN6(dev) || IS_GEN7(dev))			mode |= MI_FLUSH_ENABLE << 16 | MI_FLUSH_ENABLE;		I915_WRITE(MI_MODE, mode);		if (IS_GEN7(dev))			I915_WRITE(GFX_MODE_GEN7,				   GFX_MODE_DISABLE(GFX_TLB_INVALIDATE_ALWAYS) |				   GFX_MODE_ENABLE(GFX_REPLAY_MODE));	} 	if (INTEL_INFO(dev)->gen >= 5) {		ret = init_pipe_control(ring);		if (ret)			return ret;	} 	return ret;} static void render_ring_cleanup(struct intel_ring_buffer *ring){	if (!ring->private)		return; 	cleanup_pipe_control(ring);} static voidupdate_mboxes(struct intel_ring_buffer *ring,	    u32 seqno,	    u32 mmio_offset){	intel_ring_emit(ring, MI_SEMAPHORE_MBOX |			      MI_SEMAPHORE_GLOBAL_GTT |			      MI_SEMAPHORE_REGISTER |			      MI_SEMAPHORE_UPDATE);	intel_ring_emit(ring, seqno);	intel_ring_emit(ring, mmio_offset);} /** * gen6_add_request - Update the semaphore mailbox registers *  * @ring - ring that is adding a request * @seqno - return seqno stuck into the ring * * Update the mailbox registers in the *other* rings with the current seqno. * This acts like a signal in the canonical semaphore. */static intgen6_add_request(struct intel_ring_buffer *ring,		 u32 *seqno){	u32 mbox1_reg;	u32 mbox2_reg;	int ret; 	ret = intel_ring_begin(ring, 10);	if (ret)		return ret; 	mbox1_reg = ring->signal_mbox[0];	mbox2_reg = ring->signal_mbox[1]; 	*seqno = i915_gem_get_seqno(ring->dev); 	update_mboxes(ring, *seqno, mbox1_reg);	update_mboxes(ring, *seqno, mbox2_reg);	intel_ring_emit(ring, MI_STORE_DWORD_INDEX);	intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);	intel_ring_emit(ring, *seqno);	intel_ring_emit(ring, MI_USER_INTERRUPT);	intel_ring_advance(ring); 	return 0;} /** * intel_ring_sync - sync the waiter to the signaller on seqno * * @waiter - ring that is waiting * @signaller - ring which has, or will signal * @seqno - seqno which the waiter will block on */static intintel_ring_sync(struct intel_ring_buffer *waiter,		struct intel_ring_buffer *signaller,		int ring,		u32 seqno){	int ret;	u32 dw1 = MI_SEMAPHORE_MBOX |		  MI_SEMAPHORE_COMPARE |		  MI_SEMAPHORE_REGISTER; 	ret = intel_ring_begin(waiter, 4);	if (ret)		return ret; 	intel_ring_emit(waiter, dw1 | signaller->semaphore_register[ring]);

Lines 1–500 of 1556. Load more