Pull drm fixes from Dave Airlie: "Small set of fixes this week, i915 and a few misc ones. I didn't see an amd pull so maybe next week it'll have a few more on that driver. ttm: - NULL ptr deref fix panel: - add missing MODULE_DEVICE_TABLE imx/ipuv3: - timing fix i915: - Fix bug in getting msg length in AUX CH registers handler - Gen12 AUX invalidation fixes - Fix premature release of request's reusable memory" * tag 'drm-fixes-2023-08-04' of git://anongit.freedesktop.org/drm/drm: drm/panel: samsung-s6d7aa0: Add MODULE_DEVICE_TABLE drm/i915: Fix premature release of request's reusable memory drm/i915/gt: Support aux invalidation on all engines drm/i915/gt: Poll aux invalidation register bit on invalidation drm/i915/gt: Enable the CCS_FLUSH bit in the pipe control and in the CS drm/i915/gt: Rename flags with bit_group_X according to the datasheet drm/i915/gt: Ensure memory quiesced before invalidation drm/i915: Add the gen12_needs_ccs_aux_inv helper drm/i915/gt: Cleanup aux invalidation registers drm/i915/gvt: Fix bug in getting msg length in AUX CH registers handler drm/imx/ipuv3: Fix front porch adjustment upon hactive aligning drm/ttm: check null pointer before accessing when swapping
Unified · Split
| @@ -165,14 +165,60 @@ static u32 preparser_disable(bool state) · 164 unmodified lines | |||
| 165 | return MI_ARB_CHECK | 1 << 8 | state; | 165 | return MI_ARB_CHECK | 1 << 8 | state; |
| 166 | } | 166 | } |
| 167 | 167 | ||
| 168 | u32 *gen12_emit_aux_table_inv(struct intel_gt *gt, u32 *cs, const i915_reg_t inv_reg) | 168 | static i915_reg_t gen12_get_aux_inv_reg(struct intel_engine_cs *engine) |
| 169 | { | 169 | { |
| 170 | u32 gsi_offset = gt->uncore->gsi_offset; | 170 | switch (engine->id) { |
| 171 | case RCS0: | ||
| 172 | return GEN12_CCS_AUX_INV; | ||
| 173 | case BCS0: | ||
| 174 | return GEN12_BCS0_AUX_INV; | ||
| 175 | case VCS0: | ||
| 176 | return GEN12_VD0_AUX_INV; | ||
| 177 | case VCS2: | ||
| 178 | return GEN12_VD2_AUX_INV; | ||
| 179 | case VECS0: | ||
| 180 | return GEN12_VE0_AUX_INV; | ||
| 181 | case CCS0: | ||
| 182 | return GEN12_CCS0_AUX_INV; | ||
| 183 | default: | ||
| 184 | return INVALID_MMIO_REG; | ||
| 185 | } | ||
| 186 | } | ||
| 187 | |||
| 188 | static bool gen12_needs_ccs_aux_inv(struct intel_engine_cs *engine) | ||
| 189 | { | ||
| 190 | i915_reg_t reg = gen12_get_aux_inv_reg(engine); | ||
| 191 | |||
| 192 | if (IS_PONTEVECCHIO(engine->i915)) | ||
| 193 | return false; | ||
| 194 | |||
| 195 | /* | ||
| 196 | * So far platforms supported by i915 having flat ccs do not require | ||
| 197 | * AUX invalidation. Check also whether the engine requires it. | ||
| 198 | */ | ||
| 199 | return i915_mmio_reg_valid(reg) && !HAS_FLAT_CCS(engine->i915); | ||
| 200 | } | ||
| 201 | |||
| 202 | u32 *gen12_emit_aux_table_inv(struct intel_engine_cs *engine, u32 *cs) | ||
| 203 | { | ||
| 204 | i915_reg_t inv_reg = gen12_get_aux_inv_reg(engine); | ||
| 205 | u32 gsi_offset = engine->gt->uncore->gsi_offset; | ||
| 206 | |||
| 207 | if (!gen12_needs_ccs_aux_inv(engine)) | ||
| 208 | return cs; | ||
| 171 | 209 | ||
| 172 | *cs++ = MI_LOAD_REGISTER_IMM(1) | MI_LRI_MMIO_REMAP_EN; | 210 | *cs++ = MI_LOAD_REGISTER_IMM(1) | MI_LRI_MMIO_REMAP_EN; |
| 173 | *cs++ = i915_mmio_reg_offset(inv_reg) + gsi_offset; | 211 | *cs++ = i915_mmio_reg_offset(inv_reg) + gsi_offset; |
| 174 | *cs++ = AUX_INV; | 212 | *cs++ = AUX_INV; |
| 175 | *cs++ = MI_NOOP; | 213 | |
| 214 | *cs++ = MI_SEMAPHORE_WAIT_TOKEN | | ||
| 215 | MI_SEMAPHORE_REGISTER_POLL | | ||
| 216 | MI_SEMAPHORE_POLL | | ||
| 217 | MI_SEMAPHORE_SAD_EQ_SDD; | ||
| 218 | *cs++ = 0; | ||
| 219 | *cs++ = i915_mmio_reg_offset(inv_reg) + gsi_offset; | ||
| 220 | *cs++ = 0; | ||
| 221 | *cs++ = 0; | ||
| 176 | 222 | ||
| 177 | return cs; | 223 | return cs; |
| 178 | } | 224 | } |
| @@ -202,8 +248,13 @@ int gen12_emit_flush_rcs(struct i915_request *rq, u32 mode) · 23 unmodified lines | |||
| 202 | { | 248 | { |
| 203 | struct intel_engine_cs *engine = rq->engine; | 249 | struct intel_engine_cs *engine = rq->engine; |
| 204 | 250 | ||
| 205 | if (mode & EMIT_FLUSH) { | 251 | /* |
| 206 | u32 flags = 0; | 252 | * On Aux CCS platforms the invalidation of the Aux |
| 253 | * table requires quiescing memory traffic beforehand | ||
| 254 | */ | ||
| 255 | if (mode & EMIT_FLUSH || gen12_needs_ccs_aux_inv(engine)) { | ||
| 256 | u32 bit_group_0 = 0; | ||
| 257 | u32 bit_group_1 = 0; | ||
| 207 | int err; | 258 | int err; |
| 208 | u32 *cs; | 259 | u32 *cs; |
| 209 | 260 | ||
| @@ -211,32 +262,40 @@ int gen12_emit_flush_rcs(struct i915_request *rq, u32 mode) · 1 unmodified line | |||
| 211 | if (err) | 262 | if (err) |
| 212 | return err; | 263 | return err; |
| 213 | 264 | ||
| 214 | flags |= PIPE_CONTROL_TILE_CACHE_FLUSH; | 265 | bit_group_0 |= PIPE_CONTROL0_HDC_PIPELINE_FLUSH; |
| 215 | flags |= PIPE_CONTROL_FLUSH_L3; | 266 | |
| 216 | flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH; | 267 | /* |
| 217 | flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH; | 268 | * When required, in MTL and beyond platforms we |
| 269 | * need to set the CCS_FLUSH bit in the pipe control | ||
| 270 | */ | ||
| 271 | if (GRAPHICS_VER_FULL(rq->i915) >= IP_VER(12, 70)) | ||
| 272 | bit_group_0 |= PIPE_CONTROL_CCS_FLUSH; | ||
| 273 | |||
| 274 | bit_group_1 |= PIPE_CONTROL_TILE_CACHE_FLUSH; | ||
| 275 | bit_group_1 |= PIPE_CONTROL_FLUSH_L3; | ||
| 276 | bit_group_1 |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH; | ||
| 277 | bit_group_1 |= PIPE_CONTROL_DEPTH_CACHE_FLUSH; | ||
| 218 | /* Wa_1409600907:tgl,adl-p */ | 278 | /* Wa_1409600907:tgl,adl-p */ |
| 219 | flags |= PIPE_CONTROL_DEPTH_STALL; | 279 | bit_group_1 |= PIPE_CONTROL_DEPTH_STALL; |
| 220 | flags |= PIPE_CONTROL_DC_FLUSH_ENABLE; | 280 | bit_group_1 |= PIPE_CONTROL_DC_FLUSH_ENABLE; |
| 221 | flags |= PIPE_CONTROL_FLUSH_ENABLE; | 281 | bit_group_1 |= PIPE_CONTROL_FLUSH_ENABLE; |
| 222 | 282 | ||
| 223 | flags |= PIPE_CONTROL_STORE_DATA_INDEX; | 283 | bit_group_1 |= PIPE_CONTROL_STORE_DATA_INDEX; |
| 224 | flags |= PIPE_CONTROL_QW_WRITE; | 284 | bit_group_1 |= PIPE_CONTROL_QW_WRITE; |
| 225 | 285 | ||
| 226 | flags |= PIPE_CONTROL_CS_STALL; | 286 | bit_group_1 |= PIPE_CONTROL_CS_STALL; |
| 227 | 287 | ||
| 228 | if (!HAS_3D_PIPELINE(engine->i915)) | 288 | if (!HAS_3D_PIPELINE(engine->i915)) |
| 229 | flags &= ~PIPE_CONTROL_3D_ARCH_FLAGS; | 289 | bit_group_1 &= ~PIPE_CONTROL_3D_ARCH_FLAGS; |
| 230 | else if (engine->class == COMPUTE_CLASS) | 290 | else if (engine->class == COMPUTE_CLASS) |
| 231 | flags &= ~PIPE_CONTROL_3D_ENGINE_FLAGS; | 291 | bit_group_1 &= ~PIPE_CONTROL_3D_ENGINE_FLAGS; |
| 232 | 292 | ||
| 233 | cs = intel_ring_begin(rq, 6); | 293 | cs = intel_ring_begin(rq, 6); |
| 234 | if (IS_ERR(cs)) | 294 | if (IS_ERR(cs)) |
| 235 | return PTR_ERR(cs); | 295 | return PTR_ERR(cs); |
| 236 | 296 | ||
| 237 | cs = gen12_emit_pipe_control(cs, | 297 | cs = gen12_emit_pipe_control(cs, bit_group_0, bit_group_1, |
| 238 | PIPE_CONTROL0_HDC_PIPELINE_FLUSH, | 298 | LRC_PPHWSP_SCRATCH_ADDR); |
| 239 | flags, LRC_PPHWSP_SCRATCH_ADDR); | ||
| 240 | intel_ring_advance(rq, cs); | 299 | intel_ring_advance(rq, cs); |
| 241 | } | 300 | } |
| 242 | 301 | ||
| @@ -267,10 +326,9 @@ int gen12_emit_flush_rcs(struct i915_request *rq, u32 mode) · 24 unmodified lines | |||
| 267 | else if (engine->class == COMPUTE_CLASS) | 326 | else if (engine->class == COMPUTE_CLASS) |
| 268 | flags &= ~PIPE_CONTROL_3D_ENGINE_FLAGS; | 327 | flags &= ~PIPE_CONTROL_3D_ENGINE_FLAGS; |
| 269 | 328 | ||
| 270 | if (!HAS_FLAT_CCS(rq->engine->i915)) | 329 | count = 8; |
| 271 | count = 8 + 4; | 330 | if (gen12_needs_ccs_aux_inv(rq->engine)) |
| 272 | else | 331 | count += 8; |
| 273 | count = 8; | ||
| 274 | 332 | ||
| 275 | cs = intel_ring_begin(rq, count); | 333 | cs = intel_ring_begin(rq, count); |
| 276 | if (IS_ERR(cs)) | 334 | if (IS_ERR(cs)) |
| @@ -285,11 +343,7 @@ int gen12_emit_flush_rcs(struct i915_request *rq, u32 mode) · 8 unmodified lines | |||
| 285 | 343 | ||
| 286 | cs = gen8_emit_pipe_control(cs, flags, LRC_PPHWSP_SCRATCH_ADDR); | 344 | cs = gen8_emit_pipe_control(cs, flags, LRC_PPHWSP_SCRATCH_ADDR); |
| 287 | 345 | ||
| 288 | if (!HAS_FLAT_CCS(rq->engine->i915)) { | 346 | cs = gen12_emit_aux_table_inv(engine, cs); |
| 289 | /* hsdes: 1809175790 */ | ||
| 290 | cs = gen12_emit_aux_table_inv(rq->engine->gt, | ||
| 291 | cs, GEN12_GFX_CCS_AUX_NV); | ||
| 292 | } | ||
| 293 | 347 | ||
| 294 | *cs++ = preparser_disable(false); | 348 | *cs++ = preparser_disable(false); |
| 295 | intel_ring_advance(rq, cs); | 349 | intel_ring_advance(rq, cs); |
| @@ -300,21 +354,14 @@ int gen12_emit_flush_rcs(struct i915_request *rq, u32 mode) · 4 unmodified lines | |||
| 300 | 354 | ||
| 301 | int gen12_emit_flush_xcs(struct i915_request *rq, u32 mode) | 355 | int gen12_emit_flush_xcs(struct i915_request *rq, u32 mode) |
| 302 | { | 356 | { |
| 303 | intel_engine_mask_t aux_inv = 0; | 357 | u32 cmd = 4; |
| 304 | u32 cmd, *cs; | 358 | u32 *cs; |
| 305 | 359 | ||
| 306 | cmd = 4; | ||
| 307 | if (mode & EMIT_INVALIDATE) { | 360 | if (mode & EMIT_INVALIDATE) { |
| 308 | cmd += 2; | 361 | cmd += 2; |
| 309 | 362 | ||
| 310 | if (!HAS_FLAT_CCS(rq->engine->i915) && | 363 | if (gen12_needs_ccs_aux_inv(rq->engine)) |
| 311 | (rq->engine->class == VIDEO_DECODE_CLASS || | 364 | cmd += 8; |
| 312 | rq->engine->class == VIDEO_ENHANCEMENT_CLASS)) { | ||
| 313 | aux_inv = rq->engine->mask & | ||
| 314 | ~GENMASK(_BCS(I915_MAX_BCS - 1), BCS0); | ||
| 315 | if (aux_inv) | ||
| 316 | cmd += 4; | ||
| 317 | } | ||
| 318 | } | 365 | } |
| 319 | 366 | ||
| 320 | cs = intel_ring_begin(rq, cmd); | 367 | cs = intel_ring_begin(rq, cmd); |
| @@ -338,6 +385,10 @@ int gen12_emit_flush_xcs(struct i915_request *rq, u32 mode) · 17 unmodified lines | |||
| 338 | cmd |= MI_INVALIDATE_TLB; | 385 | cmd |= MI_INVALIDATE_TLB; |
| 339 | if (rq->engine->class == VIDEO_DECODE_CLASS) | 386 | if (rq->engine->class == VIDEO_DECODE_CLASS) |
| 340 | cmd |= MI_INVALIDATE_BSD; | 387 | cmd |= MI_INVALIDATE_BSD; |
| 388 | |||
| 389 | if (gen12_needs_ccs_aux_inv(rq->engine) && | ||
| 390 | rq->engine->class == COPY_ENGINE_CLASS) | ||
| 391 | cmd |= MI_FLUSH_DW_CCS; | ||
| 341 | } | 392 | } |
| 342 | 393 | ||
| 343 | *cs++ = cmd; | 394 | *cs++ = cmd; |
| @@ -345,14 +396,7 @@ int gen12_emit_flush_xcs(struct i915_request *rq, u32 mode) · 1 unmodified line | |||
| 345 | *cs++ = 0; /* upper addr */ | 396 | *cs++ = 0; /* upper addr */ |
| 346 | *cs++ = 0; /* value */ | 397 | *cs++ = 0; /* value */ |
| 347 | 398 | ||
| 348 | if (aux_inv) { /* hsdes: 1809175790 */ | 399 | cs = gen12_emit_aux_table_inv(rq->engine, cs); |
| 349 | if (rq->engine->class == VIDEO_DECODE_CLASS) | ||
| 350 | cs = gen12_emit_aux_table_inv(rq->engine->gt, | ||
| 351 | cs, GEN12_VD0_AUX_NV); | ||
| 352 | else | ||
| 353 | cs = gen12_emit_aux_table_inv(rq->engine->gt, | ||
| 354 | cs, GEN12_VE0_AUX_NV); | ||
| 355 | } | ||
| 356 | 400 | ||
| 357 | if (mode & EMIT_INVALIDATE) | 401 | if (mode & EMIT_INVALIDATE) |
| 358 | *cs++ = preparser_disable(false); | 402 | *cs++ = preparser_disable(false); |
| @@ -13,6 +13,7 @@ · 12 unmodified lines | |||
| 13 | #include "intel_gt_regs.h" | 13 | #include "intel_gt_regs.h" |
| 14 | #include "intel_gpu_commands.h" | 14 | #include "intel_gpu_commands.h" |
| 15 | 15 | ||
| 16 | struct intel_engine_cs; | ||
| 16 | struct intel_gt; | 17 | struct intel_gt; |
| 17 | struct i915_request; | 18 | struct i915_request; |
| 18 | 19 | ||
| @@ -46,28 +47,32 @@ u32 *gen8_emit_fini_breadcrumb_rcs(struct i915_request *rq, u32 *cs); · 27 unmodified lines | |||
| 46 | u32 *gen11_emit_fini_breadcrumb_rcs(struct i915_request *rq, u32 *cs); | 47 | u32 *gen11_emit_fini_breadcrumb_rcs(struct i915_request *rq, u32 *cs); |
| 47 | u32 *gen12_emit_fini_breadcrumb_rcs(struct i915_request *rq, u32 *cs); | 48 | u32 *gen12_emit_fini_breadcrumb_rcs(struct i915_request *rq, u32 *cs); |
| 48 | 49 | ||
| 49 | u32 *gen12_emit_aux_table_inv(struct intel_gt *gt, u32 *cs, const i915_reg_t inv_reg); | 50 | u32 *gen12_emit_aux_table_inv(struct intel_engine_cs *engine, u32 *cs); |
| 50 | 51 | ||
| 51 | static inline u32 * | 52 | static inline u32 * |
| 52 | __gen8_emit_pipe_control(u32 *batch, u32 flags0, u32 flags1, u32 offset) | 53 | __gen8_emit_pipe_control(u32 *batch, u32 bit_group_0, |
| 54 | u32 bit_group_1, u32 offset) | ||
| 53 | { | 55 | { |
| 54 | memset(batch, 0, 6 * sizeof(u32)); | 56 | memset(batch, 0, 6 * sizeof(u32)); |
| 55 | 57 | ||
| 56 | batch[0] = GFX_OP_PIPE_CONTROL(6) | flags0; | 58 | batch[0] = GFX_OP_PIPE_CONTROL(6) | bit_group_0; |
| 57 | batch[1] = flags1; | 59 | batch[1] = bit_group_1; |
| 58 | batch[2] = offset; | 60 | batch[2] = offset; |
| 59 | 61 | ||
| 60 | return batch + 6; | 62 | return batch + 6; |
| 61 | } | 63 | } |
| 62 | 64 | ||
| 63 | static inline u32 *gen8_emit_pipe_control(u32 *batch, u32 flags, u32 offset) | 65 | static inline u32 *gen8_emit_pipe_control(u32 *batch, |
| 66 | u32 bit_group_1, u32 offset) | ||
| 64 | { | 67 | { |
| 65 | return __gen8_emit_pipe_control(batch, 0, flags, offset); | 68 | return __gen8_emit_pipe_control(batch, 0, bit_group_1, offset); |
| 66 | } | 69 | } |
| 67 | 70 | ||
| 68 | static inline u32 *gen12_emit_pipe_control(u32 *batch, u32 flags0, u32 flags1, u32 offset) | 71 | static inline u32 *gen12_emit_pipe_control(u32 *batch, u32 bit_group_0, |
| 72 | u32 bit_group_1, u32 offset) | ||
| 69 | { | 73 | { |
| 70 | return __gen8_emit_pipe_control(batch, flags0, flags1, offset); | 74 | return __gen8_emit_pipe_control(batch, bit_group_0, |
| 75 | bit_group_1, offset); | ||
| 71 | } | 76 | } |
| 72 | 77 | ||
| 73 | static inline u32 * | 78 | static inline u32 * |
| @@ -121,6 +121,7 @@ · 120 unmodified lines | |||
| 121 | #define MI_SEMAPHORE_TARGET(engine) ((engine)<<15) | 121 | #define MI_SEMAPHORE_TARGET(engine) ((engine)<<15) |
| 122 | #define MI_SEMAPHORE_WAIT MI_INSTR(0x1c, 2) /* GEN8+ */ | 122 | #define MI_SEMAPHORE_WAIT MI_INSTR(0x1c, 2) /* GEN8+ */ |
| 123 | #define MI_SEMAPHORE_WAIT_TOKEN MI_INSTR(0x1c, 3) /* GEN12+ */ | 123 | #define MI_SEMAPHORE_WAIT_TOKEN MI_INSTR(0x1c, 3) /* GEN12+ */ |
| 124 | #define MI_SEMAPHORE_REGISTER_POLL (1 << 16) | ||
| 124 | #define MI_SEMAPHORE_POLL (1 << 15) | 125 | #define MI_SEMAPHORE_POLL (1 << 15) |
| 125 | #define MI_SEMAPHORE_SAD_GT_SDD (0 << 12) | 126 | #define MI_SEMAPHORE_SAD_GT_SDD (0 << 12) |
| 126 | #define MI_SEMAPHORE_SAD_GTE_SDD (1 << 12) | 127 | #define MI_SEMAPHORE_SAD_GTE_SDD (1 << 12) |
| @@ -299,6 +300,7 @@ · 172 unmodified lines | |||
| 299 | #define PIPE_CONTROL_QW_WRITE (1<<14) | 300 | #define PIPE_CONTROL_QW_WRITE (1<<14) |
| 300 | #define PIPE_CONTROL_POST_SYNC_OP_MASK (3<<14) | 301 | #define PIPE_CONTROL_POST_SYNC_OP_MASK (3<<14) |
| 301 | #define PIPE_CONTROL_DEPTH_STALL (1<<13) | 302 | #define PIPE_CONTROL_DEPTH_STALL (1<<13) |
| 303 | #define PIPE_CONTROL_CCS_FLUSH (1<<13) /* MTL+ */ | ||
| 302 | #define PIPE_CONTROL_WRITE_FLUSH (1<<12) | 304 | #define PIPE_CONTROL_WRITE_FLUSH (1<<12) |
| 303 | #define PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH (1<<12) /* gen6+ */ | 305 | #define PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH (1<<12) /* gen6+ */ |
| 304 | #define PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE (1<<11) /* MBZ on ILK */ | 306 | #define PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE (1<<11) /* MBZ on ILK */ |
| @@ -332,9 +332,11 @@ · 331 unmodified lines | |||
| 332 | #define GEN8_PRIVATE_PAT_HI _MMIO(0x40e0 + 4) | 332 | #define GEN8_PRIVATE_PAT_HI _MMIO(0x40e0 + 4) |
| 333 | #define GEN10_PAT_INDEX(index) _MMIO(0x40e0 + (index) * 4) | 333 | #define GEN10_PAT_INDEX(index) _MMIO(0x40e0 + (index) * 4) |
| 334 | #define BSD_HWS_PGA_GEN7 _MMIO(0x4180) | 334 | #define BSD_HWS_PGA_GEN7 _MMIO(0x4180) |
| 335 | #define GEN12_GFX_CCS_AUX_NV _MMIO(0x4208) | 335 | |
| 336 | #define GEN12_VD0_AUX_NV _MMIO(0x4218) | 336 | #define GEN12_CCS_AUX_INV _MMIO(0x4208) |
| 337 | #define GEN12_VD1_AUX_NV _MMIO(0x4228) | 337 | #define GEN12_VD0_AUX_INV _MMIO(0x4218) |
| 338 | #define GEN12_VE0_AUX_INV _MMIO(0x4238) | ||
| 339 | #define GEN12_BCS0_AUX_INV _MMIO(0x4248) | ||
| 338 | 340 | ||
| 339 | #define GEN8_RTCR _MMIO(0x4260) | 341 | #define GEN8_RTCR _MMIO(0x4260) |
| 340 | #define GEN8_M1TCR _MMIO(0x4264) | 342 | #define GEN8_M1TCR _MMIO(0x4264) |
| @@ -342,14 +344,12 @@ · 1 unmodified line | |||
| 342 | #define GEN8_BTCR _MMIO(0x426c) | 344 | #define GEN8_BTCR _MMIO(0x426c) |
| 343 | #define GEN8_VTCR _MMIO(0x4270) | 345 | #define GEN8_VTCR _MMIO(0x4270) |
| 344 | 346 | ||
| 345 | #define GEN12_VD2_AUX_NV _MMIO(0x4298) | ||
| 346 | #define GEN12_VD3_AUX_NV _MMIO(0x42a8) | ||
| 347 | #define GEN12_VE0_AUX_NV _MMIO(0x4238) | ||
| 348 | |||
| 349 | #define BLT_HWS_PGA_GEN7 _MMIO(0x4280) | 347 | #define BLT_HWS_PGA_GEN7 _MMIO(0x4280) |
| 350 | 348 | ||
| 351 | #define GEN12_VE1_AUX_NV _MMIO(0x42b8) | 349 | #define GEN12_VD2_AUX_INV _MMIO(0x4298) |
| 350 | #define GEN12_CCS0_AUX_INV _MMIO(0x42c8) | ||
| 352 | #define AUX_INV REG_BIT(0) | 351 | #define AUX_INV REG_BIT(0) |
| 352 | |||
| 353 | #define VEBOX_HWS_PGA_GEN7 _MMIO(0x4380) | 353 | #define VEBOX_HWS_PGA_GEN7 _MMIO(0x4380) |
| 354 | 354 | ||
| 355 | #define GEN12_AUX_ERR_DBG _MMIO(0x43f4) | 355 | #define GEN12_AUX_ERR_DBG _MMIO(0x43f4) |
| @@ -1364,10 +1364,7 @@ gen12_emit_indirect_ctx_rcs(const struct intel_context *ce, u32 *cs) · 1363 unmodified lines | |||
| 1364 | IS_DG2_G11(ce->engine->i915)) | 1364 | IS_DG2_G11(ce->engine->i915)) |
| 1365 | cs = gen8_emit_pipe_control(cs, PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE, 0); | 1365 | cs = gen8_emit_pipe_control(cs, PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE, 0); |
| 1366 | 1366 | ||
| 1367 | /* hsdes: 1809175790 */ | 1367 | cs = gen12_emit_aux_table_inv(ce->engine, cs); |
| 1368 | if (!HAS_FLAT_CCS(ce->engine->i915)) | ||
| 1369 | cs = gen12_emit_aux_table_inv(ce->engine->gt, | ||
| 1370 | cs, GEN12_GFX_CCS_AUX_NV); | ||
| 1371 | 1368 | ||
| 1372 | /* Wa_16014892111 */ | 1369 | /* Wa_16014892111 */ |
| 1373 | if (IS_MTL_GRAPHICS_STEP(ce->engine->i915, M, STEP_A0, STEP_B0) || | 1370 | if (IS_MTL_GRAPHICS_STEP(ce->engine->i915, M, STEP_A0, STEP_B0) || |
| @@ -1392,17 +1389,7 @@ gen12_emit_indirect_ctx_xcs(const struct intel_context *ce, u32 *cs) · 18 unmodified lines | |||
| 1392 | PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE, | 1389 | PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE, |
| 1393 | 0); | 1390 | 0); |
| 1394 | 1391 | ||
| 1395 | /* hsdes: 1809175790 */ | 1392 | return gen12_emit_aux_table_inv(ce->engine, cs); |
| 1396 | if (!HAS_FLAT_CCS(ce->engine->i915)) { | ||
| 1397 | if (ce->engine->class == VIDEO_DECODE_CLASS) | ||
| 1398 | cs = gen12_emit_aux_table_inv(ce->engine->gt, | ||
| 1399 | cs, GEN12_VD0_AUX_NV); | ||
| 1400 | else if (ce->engine->class == VIDEO_ENHANCEMENT_CLASS) | ||
| 1401 | cs = gen12_emit_aux_table_inv(ce->engine->gt, | ||
| 1402 | cs, GEN12_VE0_AUX_NV); | ||
| 1403 | } | ||
| 1404 | |||
| 1405 | return cs; | ||
| 1406 | } | 1393 | } |
| 1407 | 1394 | ||
| 1408 | static void | 1395 | static void |
| @@ -491,7 +491,7 @@ void intel_gvt_i2c_handle_aux_ch_write(struct intel_vgpu *vgpu, · 490 unmodified lines | |||
| 491 | return; | 491 | return; |
| 492 | } | 492 | } |
| 493 | 493 | ||
| 494 | msg_length = REG_FIELD_GET(DP_AUX_CH_CTL_MESSAGE_SIZE_MASK, reg); | 494 | msg_length = REG_FIELD_GET(DP_AUX_CH_CTL_MESSAGE_SIZE_MASK, value); |
| 495 | 495 | ||
| 496 | // check the msg in DATA register. | 496 | // check the msg in DATA register. |
| 497 | msg = vgpu_vreg(vgpu, offset + 4); | 497 | msg = vgpu_vreg(vgpu, offset + 4); |
| @@ -449,8 +449,11 @@ int i915_active_add_request(struct i915_active *ref, struct i915_request *rq) · 448 unmodified lines | |||
| 449 | } | 449 | } |
| 450 | } while (unlikely(is_barrier(active))); | 450 | } while (unlikely(is_barrier(active))); |
| 451 | 451 | ||
| 452 | if (!__i915_active_fence_set(active, fence)) | 452 | fence = __i915_active_fence_set(active, fence); |
| 453 | if (!fence) | ||
| 453 | __i915_active_acquire(ref); | 454 | __i915_active_acquire(ref); |
| 455 | else | ||
| 456 | dma_fence_put(fence); | ||
| 454 | 457 | ||
| 455 | out: | 458 | out: |
| 456 | i915_active_release(ref); | 459 | i915_active_release(ref); |
| @@ -469,13 +472,9 @@ __i915_active_set_fence(struct i915_active *ref, · 12 unmodified lines | |||
| 469 | return NULL; | 472 | return NULL; |
| 470 | } | 473 | } |
| 471 | 474 | ||
| 472 | rcu_read_lock(); | ||
| 473 | prev = __i915_active_fence_set(active, fence); | 475 | prev = __i915_active_fence_set(active, fence); |
| 474 | if (prev) | 476 | if (!prev) |
| 475 | prev = dma_fence_get_rcu(prev); | ||
| 476 | else | ||
| 477 | __i915_active_acquire(ref); | 477 | __i915_active_acquire(ref); |
| 478 | rcu_read_unlock(); | ||
| 479 | 478 | ||
| 480 | return prev; | 479 | return prev; |
| 481 | } | 480 | } |
| @@ -1019,10 +1018,11 @@ void i915_request_add_active_barriers(struct i915_request *rq) · 537 unmodified lines | |||
| 1019 | * | 1018 | * |
| 1020 | * Records the new @fence as the last active fence along its timeline in | 1019 | * Records the new @fence as the last active fence along its timeline in |
| 1021 | * this active tracker, moving the tracking callbacks from the previous | 1020 | * this active tracker, moving the tracking callbacks from the previous |
| 1022 | * fence onto this one. Returns the previous fence (if not already completed), | 1021 | * fence onto this one. Gets and returns a reference to the previous fence |
| 1023 | * which the caller must ensure is executed before the new fence. To ensure | 1022 | * (if not already completed), which the caller must put after making sure |
| 1024 | * that the order of fences within the timeline of the i915_active_fence is | 1023 | * that it is executed before the new fence. To ensure that the order of |
| 1025 | * understood, it should be locked by the caller. | 1024 | * fences within the timeline of the i915_active_fence is understood, it |
| 1025 | * should be locked by the caller. | ||
| 1026 | */ | 1026 | */ |
| 1027 | struct dma_fence * | 1027 | struct dma_fence * |
| 1028 | __i915_active_fence_set(struct i915_active_fence *active, | 1028 | __i915_active_fence_set(struct i915_active_fence *active, |
| @@ -1031,7 +1031,23 @@ __i915_active_fence_set(struct i915_active_fence *active, · 2 unmodified lines | |||
| 1031 | struct dma_fence *prev; | 1031 | struct dma_fence *prev; |
| 1032 | unsigned long flags; | 1032 | unsigned long flags; |
| 1033 | 1033 | ||
| 1034 | if (fence == rcu_access_pointer(active->fence)) | 1034 | /* |
| 1035 | * In case of fences embedded in i915_requests, their memory is | ||
| 1036 | * SLAB_FAILSAFE_BY_RCU, then it can be reused right after release | ||
| 1037 | * by new requests. Then, there is a risk of passing back a pointer | ||
| 1038 | * to a new, completely unrelated fence that reuses the same memory | ||
| 1039 | * while tracked under a different active tracker. Combined with i915 | ||
| 1040 | * perf open/close operations that build await dependencies between | ||
| 1041 | * engine kernel context requests and user requests from different | ||
| 1042 | * timelines, this can lead to dependency loops and infinite waits. | ||
| 1043 | * | ||
| 1044 | * As a countermeasure, we try to get a reference to the active->fence | ||
| 1045 | * first, so if we succeed and pass it back to our user then it is not | ||
| 1046 | * released and potentially reused by an unrelated request before the | ||
| 1047 | * user has a chance to set up an await dependency on it. | ||
| 1048 | */ | ||
| 1049 | prev = i915_active_fence_get(active); | ||
| 1050 | if (fence == prev) | ||
| 1035 | return fence; | 1051 | return fence; |
| 1036 | 1052 | ||
| 1037 | GEM_BUG_ON(test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)); | 1053 | GEM_BUG_ON(test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)); |
| @@ -1040,27 +1056,56 @@ __i915_active_fence_set(struct i915_active_fence *active, · 2 unmodified lines | |||
| 1040 | * Consider that we have two threads arriving (A and B), with | 1056 | * Consider that we have two threads arriving (A and B), with |
| 1041 | * C already resident as the active->fence. | 1057 | * C already resident as the active->fence. |
| 1042 | * | 1058 | * |
| 1043 | * A does the xchg first, and so it sees C or NULL depending | 1059 | * Both A and B have got a reference to C or NULL, depending on the |
| 1044 | * on the timing of the interrupt handler. If it is NULL, the | 1060 | * timing of the interrupt handler. Let's assume that if A has got C |
| 1045 | * previous fence must have been signaled and we know that | 1061 | * then it has locked C first (before B). |
| 1046 | * we are first on the timeline. If it is still present, | ||
| 1047 | * we acquire the lock on that fence and serialise with the interrupt | ||
| 1048 | * handler, in the process removing it from any future interrupt | ||
| 1049 | * callback. A will then wait on C before executing (if present). | ||
| 1050 | * | ||
| 1051 | * As B is second, it sees A as the previous fence and so waits for | ||
| 1052 | * it to complete its transition and takes over the occupancy for | ||
| 1053 | * itself -- remembering that it needs to wait on A before executing. | ||
| 1054 | * | 1062 | * |
| 1055 | * Note the strong ordering of the timeline also provides consistent | 1063 | * Note the strong ordering of the timeline also provides consistent |
| 1056 | * nesting rules for the fence->lock; the inner lock is always the | 1064 | * nesting rules for the fence->lock; the inner lock is always the |
| 1057 | * older lock. | 1065 | * older lock. |
| 1058 | */ | 1066 | */ |
| 1059 | spin_lock_irqsave(fence->lock, flags); | 1067 | spin_lock_irqsave(fence->lock, flags); |
| 1060 | prev = xchg(__active_fence_slot(active), fence); | 1068 | if (prev) |
| 1061 | if (prev) { | ||
| 1062 | GEM_BUG_ON(prev == fence); | ||
| 1063 | spin_lock_nested(prev->lock, SINGLE_DEPTH_NESTING); | 1069 | spin_lock_nested(prev->lock, SINGLE_DEPTH_NESTING); |
| 1070 | |||
| 1071 | /* | ||
| 1072 | * A does the cmpxchg first, and so it sees C or NULL, as before, or | ||
| 1073 | * something else, depending on the timing of other threads and/or | ||
| 1074 | * interrupt handler. If not the same as before then A unlocks C if | ||
| 1075 | * applicable and retries, starting from an attempt to get a new | ||
| 1076 | * active->fence. Meanwhile, B follows the same path as A. | ||
| 1077 | * Once A succeeds with cmpxch, B fails again, retires, gets A from | ||
| 1078 | * active->fence, locks it as soon as A completes, and possibly | ||
| 1079 | * succeeds with cmpxchg. | ||
| 1080 | */ | ||
| 1081 | while (cmpxchg(__active_fence_slot(active), prev, fence) != prev) { | ||
| 1082 | if (prev) { | ||
| 1083 | spin_unlock(prev->lock); | ||
| 1084 | dma_fence_put(prev); | ||
| 1085 | } | ||
| 1086 | spin_unlock_irqrestore(fence->lock, flags); | ||
| 1087 | |||
| 1088 | prev = i915_active_fence_get(active); | ||
| 1089 | GEM_BUG_ON(prev == fence); | ||
| 1090 | |||
| 1091 | spin_lock_irqsave(fence->lock, flags); | ||
| 1092 | if (prev) | ||
| 1093 | spin_lock_nested(prev->lock, SINGLE_DEPTH_NESTING); | ||
| 1094 | } | ||
| 1095 | |||
| 1096 | /* | ||
| 1097 | * If prev is NULL then the previous fence must have been signaled | ||
| 1098 | * and we know that we are first on the timeline. If it is still | ||
| 1099 | * present then, having the lock on that fence already acquired, we | ||
| 1100 | * serialise with the interrupt handler, in the process of removing it | ||
| 1101 | * from any future interrupt callback. A will then wait on C before | ||
| 1102 | * executing (if present). | ||
| 1103 | * | ||
| 1104 | * As B is second, it sees A as the previous fence and so waits for | ||
| 1105 | * it to complete its transition and takes over the occupancy for | ||
| 1106 | * itself -- remembering that it needs to wait on A before executing. | ||
| 1107 | */ | ||
| 1108 | if (prev) { | ||
| 1064 | __list_del_entry(&active->cb.node); | 1109 | __list_del_entry(&active->cb.node); |
| 1065 | spin_unlock(prev->lock); /* serialise with prev->cb_list */ | 1110 | spin_unlock(prev->lock); /* serialise with prev->cb_list */ |
| 1066 | } | 1111 | } |
| @@ -1077,11 +1122,7 @@ int i915_active_fence_set(struct i915_active_fence *active, · 10 unmodified lines | |||
| 1077 | int err = 0; | 1122 | int err = 0; |
| 1078 | 1123 | ||
| 1079 | /* Must maintain timeline ordering wrt previous active requests */ | 1124 | /* Must maintain timeline ordering wrt previous active requests */ |
| 1080 | rcu_read_lock(); | ||
| 1081 | fence = __i915_active_fence_set(active, &rq->fence); | 1125 | fence = __i915_active_fence_set(active, &rq->fence); |
| 1082 | if (fence) /* but the previous fence may not belong to that timeline! */ | ||
| 1083 | fence = dma_fence_get_rcu(fence); | ||
| 1084 | rcu_read_unlock(); | ||
| 1085 | if (fence) { | 1126 | if (fence) { |
| 1086 | err = i915_request_await_dma_fence(rq, fence); | 1127 | err = i915_request_await_dma_fence(rq, fence); |
| 1087 | dma_fence_put(fence); | 1128 | dma_fence_put(fence); |
| @@ -1661,6 +1661,11 @@ __i915_request_ensure_parallel_ordering(struct i915_request *rq, · 1660 unmodified lines | |||
| 1661 | 1661 | ||
| 1662 | request_to_parent(rq)->parallel.last_rq = i915_request_get(rq); | 1662 | request_to_parent(rq)->parallel.last_rq = i915_request_get(rq); |
| 1663 | 1663 | ||
| 1664 | /* | ||
| 1665 | * Users have to put a reference potentially got by | ||
| 1666 | * __i915_active_fence_set() to the returned request | ||
| 1667 | * when no longer needed | ||
| 1668 | */ | ||
| 1664 | return to_request(__i915_active_fence_set(&timeline->last_request, | 1669 | return to_request(__i915_active_fence_set(&timeline->last_request, |
| 1665 | &rq->fence)); | 1670 | &rq->fence)); |
| 1666 | } | 1671 | } |
| @@ -1707,6 +1712,10 @@ __i915_request_ensure_ordering(struct i915_request *rq, · 40 unmodified lines | |||
| 1707 | 0); | 1712 | 0); |
| 1708 | } | 1713 | } |
| 1709 | 1714 | ||
| 1715 | /* | ||
| 1716 | * Users have to put the reference to prev potentially got | ||
| 1717 | * by __i915_active_fence_set() when no longer needed | ||
| 1718 | */ | ||
| 1710 | return prev; | 1719 | return prev; |
| 1711 | } | 1720 | } |
| 1712 | 1721 | ||
| @@ -1760,6 +1769,8 @@ __i915_request_add_to_timeline(struct i915_request *rq) · 47 unmodified lines | |||
| 1760 | prev = __i915_request_ensure_ordering(rq, timeline); | 1769 | prev = __i915_request_ensure_ordering(rq, timeline); |
| 1761 | else | 1770 | else |
| 1762 | prev = __i915_request_ensure_parallel_ordering(rq, timeline); | 1771 | prev = __i915_request_ensure_parallel_ordering(rq, timeline); |
| 1772 | if (prev) | ||
| 1773 | i915_request_put(prev); | ||
| 1763 | 1774 | ||
| 1764 | /* | 1775 | /* |
| 1765 | * Make sure that no request gazumped us - if it was allocated after | 1776 | * Make sure that no request gazumped us - if it was allocated after |
| @@ -310,7 +310,7 @@ static void ipu_crtc_mode_set_nofb(struct drm_crtc *crtc) · 309 unmodified lines | |||
| 310 | dev_warn(ipu_crtc->dev, "8-pixel align hactive %d -> %d\n", | 310 | dev_warn(ipu_crtc->dev, "8-pixel align hactive %d -> %d\n", |
| 311 | sig_cfg.mode.hactive, new_hactive); | 311 | sig_cfg.mode.hactive, new_hactive); |
| 312 | 312 | ||
| 313 | sig_cfg.mode.hfront_porch = new_hactive - sig_cfg.mode.hactive; | 313 | sig_cfg.mode.hfront_porch -= new_hactive - sig_cfg.mode.hactive; |
| 314 | sig_cfg.mode.hactive = new_hactive; | 314 | sig_cfg.mode.hactive = new_hactive; |
| 315 | } | 315 | } |
| 316 | 316 | ||
| @@ -569,6 +569,7 @@ static const struct of_device_id s6d7aa0_of_match[] = { · 568 unmodified lines | |||
| 569 | }, | 569 | }, |
| 570 | { /* sentinel */ } | 570 | { /* sentinel */ } |
| 571 | }; | 571 | }; |
| 572 | MODULE_DEVICE_TABLE(of, s6d7aa0_of_match); | ||
| 572 | 573 | ||
| 573 | static struct mipi_dsi_driver s6d7aa0_driver = { | 574 | static struct mipi_dsi_driver s6d7aa0_driver = { |
| 574 | .probe = s6d7aa0_probe, | 575 | .probe = s6d7aa0_probe, |
| @@ -519,7 +519,8 @@ static bool ttm_bo_evict_swapout_allowable(struct ttm_buffer_object *bo, · 518 unmodified lines | |||
| 519 | 519 | ||
| 520 | if (bo->pin_count) { | 520 | if (bo->pin_count) { |
| 521 | *locked = false; | 521 | *locked = false; |
| 522 | *busy = false; | 522 | if (busy) |
| 523 | *busy = false; | ||
| 523 | return false; | 524 | return false; |
| 524 | } | 525 | } |
| 525 | 526 | ||