/* * Copyright © 2008-2010 Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice (including the next * paragraph) shall be included in all copies or substantial portions of the * Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS * IN THE SOFTWARE. * * Authors: * Eric Anholt <eric@anholt.net> * Zou Nan hai <nanhai.zou@intel.com> * Xiang Hai hao<haihao.xiang@intel.com> * */ #include <linux/log2.h>#include <drm/drmP.h>#include "i915_drv.h"#include <drm/i915_drm.h>#include "i915_trace.h"#include "intel_drv.h" /* Rough estimate of the typical request size, performing a flush, * set-context and then emitting the batch. */#define LEGACY_REQUEST_SIZE 200 static unsigned int __intel_ring_space(unsigned int head, unsigned int tail, unsigned int size){ /* * "If the Ring Buffer Head Pointer and the Tail Pointer are on the * same cacheline, the Head Pointer must not be greater than the Tail * Pointer." */ GEM_BUG_ON(!is_power_of_2(size)); return (head - tail - CACHELINE_BYTES) & (size - 1);} unsigned int intel_ring_update_space(struct intel_ring *ring){ unsigned int space; space = __intel_ring_space(ring->head, ring->emit, ring->size); ring->space = space; return space;} static intgen2_render_ring_flush(struct drm_i915_gem_request *req, u32 mode){ u32 cmd, *cs; cmd = MI_FLUSH; if (mode & EMIT_INVALIDATE) cmd |= MI_READ_FLUSH; cs = intel_ring_begin(req, 2); if (IS_ERR(cs)) return PTR_ERR(cs); *cs++ = cmd; *cs++ = MI_NOOP; intel_ring_advance(req, cs); return 0;} static intgen4_render_ring_flush(struct drm_i915_gem_request *req, u32 mode){ u32 cmd, *cs; /* * read/write caches: * * I915_GEM_DOMAIN_RENDER is always invalidated, but is * only flushed if MI_NO_WRITE_FLUSH is unset. On 965, it is * also flushed at 2d versus 3d pipeline switches. * * read-only caches: * * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if * MI_READ_FLUSH is set, and is always flushed on 965. * * I915_GEM_DOMAIN_COMMAND may not exist? * * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is * invalidated when MI_EXE_FLUSH is set. * * I915_GEM_DOMAIN_VERTEX, which exists on 965, is * invalidated with every MI_FLUSH. * * TLBs: * * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER * are flushed at any MI_FLUSH. */ cmd = MI_FLUSH; if (mode & EMIT_INVALIDATE) { cmd |= MI_EXE_FLUSH; if (IS_G4X(req->i915) || IS_GEN5(req->i915)) cmd |= MI_INVALIDATE_ISP; } cs = intel_ring_begin(req, 2); if (IS_ERR(cs)) return PTR_ERR(cs); *cs++ = cmd; *cs++ = MI_NOOP; intel_ring_advance(req, cs); return 0;} /** * Emits a PIPE_CONTROL with a non-zero post-sync operation, for * implementing two workarounds on gen6. From section 1.4.7.1 * "PIPE_CONTROL" of the Sandy Bridge PRM volume 2 part 1: * * [DevSNB-C+{W/A}] Before any depth stall flush (including those * produced by non-pipelined state commands), software needs to first * send a PIPE_CONTROL with no bits set except Post-Sync Operation != * 0. * * [Dev-SNB{W/A}]: Before a PIPE_CONTROL with Write Cache Flush Enable * =1, a PIPE_CONTROL with any non-zero post-sync-op is required. * * And the workaround for these two requires this workaround first: * * [Dev-SNB{W/A}]: Pipe-control with CS-stall bit set must be sent * BEFORE the pipe-control with a post-sync op and no write-cache * flushes. * * And this last workaround is tricky because of the requirements on * that bit. From section 1.4.7.2.3 "Stall" of the Sandy Bridge PRM * volume 2 part 1: * * "1 of the following must also be set: * - Render Target Cache Flush Enable ([12] of DW1) * - Depth Cache Flush Enable ([0] of DW1) * - Stall at Pixel Scoreboard ([1] of DW1) * - Depth Stall ([13] of DW1) * - Post-Sync Operation ([13] of DW1) * - Notify Enable ([8] of DW1)" * * The cache flushes require the workaround flush that triggered this * one, so we can't use it. Depth stall would trigger the same. * Post-sync nonzero is what triggered this second workaround, so we * can't use that one either. Notify enable is IRQs, which aren't * really our business. That leaves only stall at scoreboard. */static intintel_emit_post_sync_nonzero_flush(struct drm_i915_gem_request *req){ u32 scratch_addr = i915_ggtt_offset(req->engine->scratch) + 2 * CACHELINE_BYTES; u32 *cs; cs = intel_ring_begin(req, 6); if (IS_ERR(cs)) return PTR_ERR(cs); *cs++ = GFX_OP_PIPE_CONTROL(5); *cs++ = PIPE_CONTROL_CS_STALL | PIPE_CONTROL_STALL_AT_SCOREBOARD; *cs++ = scratch_addr | PIPE_CONTROL_GLOBAL_GTT; *cs++ = 0; /* low dword */ *cs++ = 0; /* high dword */ *cs++ = MI_NOOP; intel_ring_advance(req, cs); cs = intel_ring_begin(req, 6); if (IS_ERR(cs)) return PTR_ERR(cs); *cs++ = GFX_OP_PIPE_CONTROL(5); *cs++ = PIPE_CONTROL_QW_WRITE; *cs++ = scratch_addr | PIPE_CONTROL_GLOBAL_GTT; *cs++ = 0; *cs++ = 0; *cs++ = MI_NOOP; intel_ring_advance(req, cs); return 0;} static intgen6_render_ring_flush(struct drm_i915_gem_request *req, u32 mode){ u32 scratch_addr = i915_ggtt_offset(req->engine->scratch) + 2 * CACHELINE_BYTES; u32 *cs, flags = 0; int ret; /* Force SNB workarounds for PIPE_CONTROL flushes */ ret = intel_emit_post_sync_nonzero_flush(req); if (ret) return ret; /* Just flush everything. Experiments have shown that reducing the * number of bits based on the write domains has little performance * impact. */ if (mode & EMIT_FLUSH) { flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH; flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH; /* * Ensure that any following seqno writes only happen * when the render cache is indeed flushed. */ flags |= PIPE_CONTROL_CS_STALL; } if (mode & EMIT_INVALIDATE) { flags |= PIPE_CONTROL_TLB_INVALIDATE; flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE; flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE; flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE; flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE; flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE; /* * TLB invalidate requires a post-sync write. */ flags |= PIPE_CONTROL_QW_WRITE | PIPE_CONTROL_CS_STALL; } cs = intel_ring_begin(req, 4); if (IS_ERR(cs)) return PTR_ERR(cs); *cs++ = GFX_OP_PIPE_CONTROL(4); *cs++ = flags; *cs++ = scratch_addr | PIPE_CONTROL_GLOBAL_GTT; *cs++ = 0; intel_ring_advance(req, cs); return 0;} static intgen7_render_ring_cs_stall_wa(struct drm_i915_gem_request *req){ u32 *cs; cs = intel_ring_begin(req, 4); if (IS_ERR(cs)) return PTR_ERR(cs); *cs++ = GFX_OP_PIPE_CONTROL(4); *cs++ = PIPE_CONTROL_CS_STALL | PIPE_CONTROL_STALL_AT_SCOREBOARD; *cs++ = 0; *cs++ = 0; intel_ring_advance(req, cs); return 0;} static intgen7_render_ring_flush(struct drm_i915_gem_request *req, u32 mode){ u32 scratch_addr = i915_ggtt_offset(req->engine->scratch) + 2 * CACHELINE_BYTES; u32 *cs, flags = 0; /* * Ensure that any following seqno writes only happen when the render * cache is indeed flushed. * * Workaround: 4th PIPE_CONTROL command (except the ones with only * read-cache invalidate bits set) must have the CS_STALL bit set. We * don't try to be clever and just set it unconditionally. */ flags |= PIPE_CONTROL_CS_STALL; /* Just flush everything. Experiments have shown that reducing the * number of bits based on the write domains has little performance * impact. */ if (mode & EMIT_FLUSH) { flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH; flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH; flags |= PIPE_CONTROL_DC_FLUSH_ENABLE; flags |= PIPE_CONTROL_FLUSH_ENABLE; } if (mode & EMIT_INVALIDATE) { flags |= PIPE_CONTROL_TLB_INVALIDATE; flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE; flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE; flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE; flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE; flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE; flags |= PIPE_CONTROL_MEDIA_STATE_CLEAR; /* * TLB invalidate requires a post-sync write. */ flags |= PIPE_CONTROL_QW_WRITE; flags |= PIPE_CONTROL_GLOBAL_GTT_IVB; flags |= PIPE_CONTROL_STALL_AT_SCOREBOARD; /* Workaround: we must issue a pipe_control with CS-stall bit * set before a pipe_control command that has the state cache * invalidate bit set. */ gen7_render_ring_cs_stall_wa(req); } cs = intel_ring_begin(req, 4); if (IS_ERR(cs)) return PTR_ERR(cs); *cs++ = GFX_OP_PIPE_CONTROL(4); *cs++ = flags; *cs++ = scratch_addr; *cs++ = 0; intel_ring_advance(req, cs); return 0;} static intgen8_render_ring_flush(struct drm_i915_gem_request *req, u32 mode){ u32 flags; u32 *cs; cs = intel_ring_begin(req, mode & EMIT_INVALIDATE ? 12 : 6); if (IS_ERR(cs)) return PTR_ERR(cs); flags = PIPE_CONTROL_CS_STALL; if (mode & EMIT_FLUSH) { flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH; flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH; flags |= PIPE_CONTROL_DC_FLUSH_ENABLE; flags |= PIPE_CONTROL_FLUSH_ENABLE; } if (mode & EMIT_INVALIDATE) { flags |= PIPE_CONTROL_TLB_INVALIDATE; flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE; flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE; flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE; flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE; flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE; flags |= PIPE_CONTROL_QW_WRITE; flags |= PIPE_CONTROL_GLOBAL_GTT_IVB; /* WaCsStallBeforeStateCacheInvalidate:bdw,chv */ cs = gen8_emit_pipe_control(cs, PIPE_CONTROL_CS_STALL | PIPE_CONTROL_STALL_AT_SCOREBOARD, 0); } cs = gen8_emit_pipe_control(cs, flags, i915_ggtt_offset(req->engine->scratch) + 2 * CACHELINE_BYTES); intel_ring_advance(req, cs); return 0;} static void ring_setup_phys_status_page(struct intel_engine_cs *engine){ struct drm_i915_private *dev_priv = engine->i915; u32 addr; addr = dev_priv->status_page_dmah->busaddr; if (INTEL_GEN(dev_priv) >= 4) addr |= (dev_priv->status_page_dmah->busaddr >> 28) & 0xf0; I915_WRITE(HWS_PGA, addr);} static void intel_ring_setup_status_page(struct intel_engine_cs *engine){ struct drm_i915_private *dev_priv = engine->i915; i915_reg_t mmio; /* The ring status page addresses are no longer next to the rest of * the ring registers as of gen7. */ if (IS_GEN7(dev_priv)) { switch (engine->id) { case RCS: mmio = RENDER_HWS_PGA_GEN7; break; case BCS: mmio = BLT_HWS_PGA_GEN7; break; /* * VCS2 actually doesn't exist on Gen7. Only shut up * gcc switch check warning */ case VCS2: case VCS: mmio = BSD_HWS_PGA_GEN7; break; case VECS: mmio = VEBOX_HWS_PGA_GEN7; break; } } else if (IS_GEN6(dev_priv)) { mmio = RING_HWS_PGA_GEN6(engine->mmio_base); } else { /* XXX: gen8 returns to sanity */ mmio = RING_HWS_PGA(engine->mmio_base); } I915_WRITE(mmio, engine->status_page.ggtt_offset); POSTING_READ(mmio); /* * Flush the TLB for this page * * FIXME: These two bits have disappeared on gen8, so a question * arises: do we still need this and if so how should we go about * invalidating the TLB? */ if (IS_GEN(dev_priv, 6, 7)) { i915_reg_t reg = RING_INSTPM(engine->mmio_base); /* ring should be idle before issuing a sync flush*/ WARN_ON((I915_READ_MODE(engine) & MODE_IDLE) == 0); I915_WRITE(reg, _MASKED_BIT_ENABLE(INSTPM_TLB_INVALIDATE | INSTPM_SYNC_FLUSH)); if (intel_wait_for_register(dev_priv, reg, INSTPM_SYNC_FLUSH, 0, 1000)) DRM_ERROR("%s: wait for SyncFlush to complete for TLB invalidation timed out\n", engine->name); }} static bool stop_ring(struct intel_engine_cs *engine){ struct drm_i915_private *dev_priv = engine->i915; if (INTEL_GEN(dev_priv) > 2) { I915_WRITE_MODE(engine, _MASKED_BIT_ENABLE(STOP_RING)); if (intel_wait_for_register(dev_priv, RING_MI_MODE(engine->mmio_base), MODE_IDLE, MODE_IDLE, 1000)) { DRM_ERROR("%s : timed out trying to stop ring\n", engine->name); /* Sometimes we observe that the idle flag is not * set even though the ring is empty. So double * check before giving up. */ if (I915_READ_HEAD(engine) != I915_READ_TAIL(engine)) return false; } } I915_WRITE_CTL(engine, 0); I915_WRITE_HEAD(engine, 0); I915_WRITE_TAIL(engine, 0); if (INTEL_GEN(dev_priv) > 2) { (void)I915_READ_CTL(engine); I915_WRITE_MODE(engine, _MASKED_BIT_DISABLE(STOP_RING)); } return (I915_READ_HEAD(engine) & HEAD_ADDR) == 0;} static int init_ring_common(struct intel_engine_cs *engine){ struct drm_i915_private *dev_priv = engine->i915; struct intel_ring *ring = engine->buffer; int ret = 0; intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL); if (!stop_ring(engine)) { /* G45 ring initialization often fails to reset head to zero */Lines 1–500 of 2282. Load more
/* * Copyright © 2008-2010 Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice (including the next * paragraph) shall be included in all copies or substantial portions of the * Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS * IN THE SOFTWARE. * * Authors: * Eric Anholt <eric@anholt.net> * Zou Nan hai <nanhai.zou@intel.com> * Xiang Hai hao<haihao.xiang@intel.com> * */ #include <linux/log2.h> #include <drm/drmP.h> #include "i915_drv.h" #include <drm/i915_drm.h> #include "i915_trace.h" #include "intel_drv.h" /* Rough estimate of the typical request size, performing a flush, * set-context and then emitting the batch. */ #define LEGACY_REQUEST_SIZE 200 static unsigned int __intel_ring_space(unsigned int head, unsigned int tail, unsigned int size) { /* * "If the Ring Buffer Head Pointer and the Tail Pointer are on the * same cacheline, the Head Pointer must not be greater than the Tail * Pointer." */ GEM_BUG_ON(!is_power_of_2(size)); return (head - tail - CACHELINE_BYTES) & (size - 1); } unsigned int intel_ring_update_space(struct intel_ring *ring) { unsigned int space; space = __intel_ring_space(ring->head, ring->emit, ring->size); ring->space = space; return space; } static int gen2_render_ring_flush(struct drm_i915_gem_request *req, u32 mode) { u32 cmd, *cs; cmd = MI_FLUSH; if (mode & EMIT_INVALIDATE) cmd |= MI_READ_FLUSH; cs = intel_ring_begin(req, 2); if (IS_ERR(cs)) return PTR_ERR(cs); *cs++ = cmd; *cs++ = MI_NOOP; intel_ring_advance(req, cs); return 0; } static int gen4_render_ring_flush(struct drm_i915_gem_request *req, u32 mode) { u32 cmd, *cs; /* * read/write caches: * * I915_GEM_DOMAIN_RENDER is always invalidated, but is * only flushed if MI_NO_WRITE_FLUSH is unset. On 965, it is * also flushed at 2d versus 3d pipeline switches. * * read-only caches: * * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if * MI_READ_FLUSH is set, and is always flushed on 965. * * I915_GEM_DOMAIN_COMMAND may not exist? * * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is * invalidated when MI_EXE_FLUSH is set. * * I915_GEM_DOMAIN_VERTEX, which exists on 965, is * invalidated with every MI_FLUSH. * * TLBs: * * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER * are flushed at any MI_FLUSH. */ cmd = MI_FLUSH; if (mode & EMIT_INVALIDATE) { cmd |= MI_EXE_FLUSH; if (IS_G4X(req->i915) || IS_GEN5(req->i915)) cmd |= MI_INVALIDATE_ISP; } cs = intel_ring_begin(req, 2); if (IS_ERR(cs)) return PTR_ERR(cs); *cs++ = cmd; *cs++ = MI_NOOP; intel_ring_advance(req, cs); return 0; } /** * Emits a PIPE_CONTROL with a non-zero post-sync operation, for * implementing two workarounds on gen6. From section 1.4.7.1 * "PIPE_CONTROL" of the Sandy Bridge PRM volume 2 part 1: * * [DevSNB-C+{W/A}] Before any depth stall flush (including those * produced by non-pipelined state commands), software needs to first * send a PIPE_CONTROL with no bits set except Post-Sync Operation != * 0. * * [Dev-SNB{W/A}]: Before a PIPE_CONTROL with Write Cache Flush Enable * =1, a PIPE_CONTROL with any non-zero post-sync-op is required. * * And the workaround for these two requires this workaround first: * * [Dev-SNB{W/A}]: Pipe-control with CS-stall bit set must be sent * BEFORE the pipe-control with a post-sync op and no write-cache * flushes. * * And this last workaround is tricky because of the requirements on * that bit. From section 1.4.7.2.3 "Stall" of the Sandy Bridge PRM * volume 2 part 1: * * "1 of the following must also be set: * - Render Target Cache Flush Enable ([12] of DW1) * - Depth Cache Flush Enable ([0] of DW1) * - Stall at Pixel Scoreboard ([1] of DW1) * - Depth Stall ([13] of DW1) * - Post-Sync Operation ([13] of DW1) * - Notify Enable ([8] of DW1)" * * The cache flushes require the workaround flush that triggered this * one, so we can't use it. Depth stall would trigger the same. * Post-sync nonzero is what triggered this second workaround, so we * can't use that one either. Notify enable is IRQs, which aren't * really our business. That leaves only stall at scoreboard. */ static int intel_emit_post_sync_nonzero_flush(struct drm_i915_gem_request *req) { u32 scratch_addr = i915_ggtt_offset(req->engine->scratch) + 2 * CACHELINE_BYTES; u32 *cs; cs = intel_ring_begin(req, 6); if (IS_ERR(cs)) return PTR_ERR(cs); *cs++ = GFX_OP_PIPE_CONTROL(5); *cs++ = PIPE_CONTROL_CS_STALL | PIPE_CONTROL_STALL_AT_SCOREBOARD; *cs++ = scratch_addr | PIPE_CONTROL_GLOBAL_GTT; *cs++ = 0; /* low dword */ *cs++ = 0; /* high dword */ *cs++ = MI_NOOP; intel_ring_advance(req, cs); cs = intel_ring_begin(req, 6); if (IS_ERR(cs)) return PTR_ERR(cs); *cs++ = GFX_OP_PIPE_CONTROL(5); *cs++ = PIPE_CONTROL_QW_WRITE; *cs++ = scratch_addr | PIPE_CONTROL_GLOBAL_GTT; *cs++ = 0; *cs++ = 0; *cs++ = MI_NOOP; intel_ring_advance(req, cs); return 0; } static int gen6_render_ring_flush(struct drm_i915_gem_request *req, u32 mode) { u32 scratch_addr = i915_ggtt_offset(req->engine->scratch) + 2 * CACHELINE_BYTES; u32 *cs, flags = 0; int ret; /* Force SNB workarounds for PIPE_CONTROL flushes */ ret = intel_emit_post_sync_nonzero_flush(req); if (ret) return ret; /* Just flush everything. Experiments have shown that reducing the * number of bits based on the write domains has little performance * impact. */ if (mode & EMIT_FLUSH) { flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH; flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH; /* * Ensure that any following seqno writes only happen * when the render cache is indeed flushed. */ flags |= PIPE_CONTROL_CS_STALL; } if (mode & EMIT_INVALIDATE) { flags |= PIPE_CONTROL_TLB_INVALIDATE; flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE; flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE; flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE; flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE; flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE; /* * TLB invalidate requires a post-sync write. */ flags |= PIPE_CONTROL_QW_WRITE | PIPE_CONTROL_CS_STALL; } cs = intel_ring_begin(req, 4); if (IS_ERR(cs)) return PTR_ERR(cs); *cs++ = GFX_OP_PIPE_CONTROL(4); *cs++ = flags; *cs++ = scratch_addr | PIPE_CONTROL_GLOBAL_GTT; *cs++ = 0; intel_ring_advance(req, cs); return 0; } static int gen7_render_ring_cs_stall_wa(struct drm_i915_gem_request *req) { u32 *cs; cs = intel_ring_begin(req, 4); if (IS_ERR(cs)) return PTR_ERR(cs); *cs++ = GFX_OP_PIPE_CONTROL(4); *cs++ = PIPE_CONTROL_CS_STALL | PIPE_CONTROL_STALL_AT_SCOREBOARD; *cs++ = 0; *cs++ = 0; intel_ring_advance(req, cs); return 0; } static int gen7_render_ring_flush(struct drm_i915_gem_request *req, u32 mode) { u32 scratch_addr = i915_ggtt_offset(req->engine->scratch) + 2 * CACHELINE_BYTES; u32 *cs, flags = 0; /* * Ensure that any following seqno writes only happen when the render * cache is indeed flushed. * * Workaround: 4th PIPE_CONTROL command (except the ones with only * read-cache invalidate bits set) must have the CS_STALL bit set. We * don't try to be clever and just set it unconditionally. */ flags |= PIPE_CONTROL_CS_STALL; /* Just flush everything. Experiments have shown that reducing the * number of bits based on the write domains has little performance * impact. */ if (mode & EMIT_FLUSH) { flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH; flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH; flags |= PIPE_CONTROL_DC_FLUSH_ENABLE; flags |= PIPE_CONTROL_FLUSH_ENABLE; } if (mode & EMIT_INVALIDATE) { flags |= PIPE_CONTROL_TLB_INVALIDATE; flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE; flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE; flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE; flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE; flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE; flags |= PIPE_CONTROL_MEDIA_STATE_CLEAR; /* * TLB invalidate requires a post-sync write. */ flags |= PIPE_CONTROL_QW_WRITE; flags |= PIPE_CONTROL_GLOBAL_GTT_IVB; flags |= PIPE_CONTROL_STALL_AT_SCOREBOARD; /* Workaround: we must issue a pipe_control with CS-stall bit * set before a pipe_control command that has the state cache * invalidate bit set. */ gen7_render_ring_cs_stall_wa(req); } cs = intel_ring_begin(req, 4); if (IS_ERR(cs)) return PTR_ERR(cs); *cs++ = GFX_OP_PIPE_CONTROL(4); *cs++ = flags; *cs++ = scratch_addr; *cs++ = 0; intel_ring_advance(req, cs); return 0; } static int gen8_render_ring_flush(struct drm_i915_gem_request *req, u32 mode) { u32 flags; u32 *cs; cs = intel_ring_begin(req, mode & EMIT_INVALIDATE ? 12 : 6); if (IS_ERR(cs)) return PTR_ERR(cs); flags = PIPE_CONTROL_CS_STALL; if (mode & EMIT_FLUSH) { flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH; flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH; flags |= PIPE_CONTROL_DC_FLUSH_ENABLE; flags |= PIPE_CONTROL_FLUSH_ENABLE; } if (mode & EMIT_INVALIDATE) { flags |= PIPE_CONTROL_TLB_INVALIDATE; flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE; flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE; flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE; flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE; flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE; flags |= PIPE_CONTROL_QW_WRITE; flags |= PIPE_CONTROL_GLOBAL_GTT_IVB; /* WaCsStallBeforeStateCacheInvalidate:bdw,chv */ cs = gen8_emit_pipe_control(cs, PIPE_CONTROL_CS_STALL | PIPE_CONTROL_STALL_AT_SCOREBOARD, 0); } cs = gen8_emit_pipe_control(cs, flags, i915_ggtt_offset(req->engine->scratch) + 2 * CACHELINE_BYTES); intel_ring_advance(req, cs); return 0; } static void ring_setup_phys_status_page(struct intel_engine_cs *engine) { struct drm_i915_private *dev_priv = engine->i915; u32 addr; addr = dev_priv->status_page_dmah->busaddr; if (INTEL_GEN(dev_priv) >= 4) addr |= (dev_priv->status_page_dmah->busaddr >> 28) & 0xf0; I915_WRITE(HWS_PGA, addr); } static void intel_ring_setup_status_page(struct intel_engine_cs *engine) { struct drm_i915_private *dev_priv = engine->i915; i915_reg_t mmio; /* The ring status page addresses are no longer next to the rest of * the ring registers as of gen7. */ if (IS_GEN7(dev_priv)) { switch (engine->id) { case RCS: mmio = RENDER_HWS_PGA_GEN7; break; case BCS: mmio = BLT_HWS_PGA_GEN7; break; /* * VCS2 actually doesn't exist on Gen7. Only shut up * gcc switch check warning */ case VCS2: case VCS: mmio = BSD_HWS_PGA_GEN7; break; case VECS: mmio = VEBOX_HWS_PGA_GEN7; break; } } else if (IS_GEN6(dev_priv)) { mmio = RING_HWS_PGA_GEN6(engine->mmio_base); } else { /* XXX: gen8 returns to sanity */ mmio = RING_HWS_PGA(engine->mmio_base); } I915_WRITE(mmio, engine->status_page.ggtt_offset); POSTING_READ(mmio); /* * Flush the TLB for this page * * FIXME: These two bits have disappeared on gen8, so a question * arises: do we still need this and if so how should we go about * invalidating the TLB? */ if (IS_GEN(dev_priv, 6, 7)) { i915_reg_t reg = RING_INSTPM(engine->mmio_base); /* ring should be idle before issuing a sync flush*/ WARN_ON((I915_READ_MODE(engine) & MODE_IDLE) == 0); I915_WRITE(reg, _MASKED_BIT_ENABLE(INSTPM_TLB_INVALIDATE | INSTPM_SYNC_FLUSH)); if (intel_wait_for_register(dev_priv, reg, INSTPM_SYNC_FLUSH, 0, 1000)) DRM_ERROR("%s: wait for SyncFlush to complete for TLB invalidation timed out\n", engine->name); } } static bool stop_ring(struct intel_engine_cs *engine) { struct drm_i915_private *dev_priv = engine->i915; if (INTEL_GEN(dev_priv) > 2) { I915_WRITE_MODE(engine, _MASKED_BIT_ENABLE(STOP_RING)); if (intel_wait_for_register(dev_priv, RING_MI_MODE(engine->mmio_base), MODE_IDLE, MODE_IDLE, 1000)) { DRM_ERROR("%s : timed out trying to stop ring\n", engine->name); /* Sometimes we observe that the idle flag is not * set even though the ring is empty. So double * check before giving up. */ if (I915_READ_HEAD(engine) != I915_READ_TAIL(engine)) return false; } } I915_WRITE_CTL(engine, 0); I915_WRITE_HEAD(engine, 0); I915_WRITE_TAIL(engine, 0); if (INTEL_GEN(dev_priv) > 2) { (void)I915_READ_CTL(engine); I915_WRITE_MODE(engine, _MASKED_BIT_DISABLE(STOP_RING)); } return (I915_READ_HEAD(engine) & HEAD_ADDR) == 0; } static int init_ring_common(struct intel_engine_cs *engine) { struct drm_i915_private *dev_priv = engine->i915; struct intel_ring *ring = engine->buffer; int ret = 0; intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL); if (!stop_ring(engine)) { /* G45 ring initialization often fails to reset head to zero */